feat: accept a custom context window per model for Windmill AI

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

backend/windmill-api/openapi.yaml

@@ -28825,6 +28825,16 @@ components:
             type: integer
             minimum: 1
             maximum: 2000000
+        context_window_per_model:
+          type: object
+          description: >-
+            Context window, in tokens, the AI chat budgets against for a model, keyed
+            `provider:model`. Overrides the built-in window, or the assumed one for a
+            model the chat does not know.
+          additionalProperties:
+            type: integer
+            minimum: 1000
+            maximum: 10000000
         free_tier:
           $ref: "#/components/schemas/FreeTierInfo"
         model_pricing:

backend/windmill-api/src/ai.rs

@@ -436,6 +436,10 @@ pub struct AIConfig {
     pub custom_prompts: Option<HashMap<String, String>>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub max_tokens_per_model: Option<HashMap<String, i32>>,
+    /// Context windows the chat budgets against, keyed `provider:model` like
+    /// `max_tokens_per_model`. Only models whose window differs from the built-in one are stored.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub context_window_per_model: Option<HashMap<String, i32>>,
     /// Response-only: this same struct is the request body for saving a workspace's AI
     /// config, and `skip_deserializing` is what stops a client from storing a forged
     /// free-tier marker. Only the server sets it, per-request.

frontend/src/lib/aiStore.ts

@@ -51,6 +51,8 @@ export const copilotInfo = writable<{
 	aiModels: AIProviderModel[]
 	customPrompts?: Record<string, string>
 	maxTokensPerModel?: Record<string, number>
+	/** Context windows per `provider:model`, overriding the built-in table. */
+	contextWindowPerModel?: Record<string, number>
 	/** Negotiated rates per `provider:model`, overriding the built-in price table. */
 	modelPricing?: Record<string, ModelPriceOverride>
 	webSearchEnabledProviders?: Partial<Record<AIProvider, boolean>>
@@ -67,6 +69,7 @@ export const copilotInfo = writable<{
 	aiModels: [],
 	customPrompts: {},
 	maxTokensPerModel: {},
+	contextWindowPerModel: {},
 	modelPricing: {},
 	webSearchEnabledProviders: {}
 })
@@ -144,6 +147,7 @@ export function setCopilotInfo(aiConfig: AIConfig) {
 			aiModels: aiModels,
 			customPrompts: aiConfig.custom_prompts ?? {},
 			maxTokensPerModel: aiConfig.max_tokens_per_model ?? {},
+			contextWindowPerModel: aiConfig.context_window_per_model ?? {},
 			webSearchEnabledProviders,
 			modelPricing: aiConfig.model_pricing ?? {},
 			freeTier: aiConfig.free_tier
@@ -160,6 +164,7 @@ export function setCopilotInfo(aiConfig: AIConfig) {
 			aiModels: [],
 			customPrompts: {},
 			maxTokensPerModel: {},
+			contextWindowPerModel: {},
 			webSearchEnabledProviders: {},
 			modelPricing: {},
 			// An exhausted free grant lands here — no providers, but the reason AI is off

frontend/src/lib/components/copilot/autocomplete/Autocompletor.ts

@@ -94,8 +94,12 @@ export class Autocompletor {
 
 		const deletionsCues = editor.createDecorationsCollection()
 
-		const completionModel = get(copilotInfo).codeCompletionModel
-		this.#contextWindow = getModelContextWindow(completionModel?.model ?? '')
+		const { codeCompletionModel: completionModel, contextWindowPerModel } = get(copilotInfo)
+		this.#contextWindow = getModelContextWindow(
+			completionModel?.provider ?? '',
+			completionModel?.model ?? '',
+			contextWindowPerModel
+		)
 
 		this.#completionDisposable = languages.registerInlineCompletionsProvider(
 			{ pattern: '**' },

frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts

@@ -3855,7 +3855,9 @@ export class AIChatManager implements ChatViewHost {
 			// assumed window rather than no limit: without one the context grows
 			// unbounded until the provider (or a proxy in front of it) times out.
 			// Guessing low only compacts earlier, which is always recoverable.
-			const contextWindow = model ? getModelContextWindow(model.model) : undefined
+			const contextWindow = model
+				? getModelContextWindow(model.provider, model.model, get(copilotInfo).contextWindowPerModel)
+				: undefined
 			if (
 				contextWindow !== undefined &&
 				projectedContextTokens >= contextWindow * COMPACTION_TRIGGER_RATIO

frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte

@@ -1,6 +1,6 @@
 <script lang="ts">
 	import { copilotInfo, copilotSessionModel } from '$lib/aiStore'
-	import { getKnownModelContextWindow, getModelContextWindow } from '../modelConfig'
+	import { getConfiguredContextWindow, getModelContextWindow } from '../modelConfig'
 	import { getChatViewHost } from './chatViewHost'
 	import { AIMode } from './AIChatManager.svelte'
 	import UsageMeter from './UsageMeter.svelte'
@@ -19,10 +19,21 @@
 	// model is listed, otherwise the conservative window the trigger assumes.
 	// The tooltip marks the assumed case so the guess never reads as a spec.
 	let contextWindow = $derived(
-		providerModel ? getModelContextWindow(providerModel.model) : undefined
+		providerModel
+			? getModelContextWindow(
+					providerModel.provider,
+					providerModel.model,
+					$copilotInfo.contextWindowPerModel
+				)
+			: undefined
 	)
 	let windowIsAssumed = $derived(
-		providerModel !== undefined && getKnownModelContextWindow(providerModel.model) === undefined
+		providerModel !== undefined &&
+			getConfiguredContextWindow(
+				providerModel.provider,
+				providerModel.model,
+				$copilotInfo.contextWindowPerModel
+			) === undefined
 	)
 	// The same number the compaction trigger uses: the provider's report when
 	// one describes the current history (one turn stale by nature), otherwise

frontend/src/lib/components/copilot/chat/script/core.ts

@@ -24,7 +24,8 @@ import { createWorkspaceMutationTools } from '../workspaceTools'
 import { setupTypeAcquisition, type DepsToGet } from '$lib/ata'
 import { getModelContextWindow } from '../../modelConfig'
 import type { ReviewChangesOpts } from '../monaco-adapter'
-import { getCurrentModel } from '$lib/aiStore'
+import { copilotInfo, getCurrentModel } from '$lib/aiStore'
+import { get } from 'svelte/store'
 import { getDbSchemas } from '$lib/components/apps/components/display/dbtable/metadata'
 import { getScriptPrompt, getWorkflowAsCodePrompt } from '$system_prompts'
 
@@ -548,7 +549,11 @@ export async function searchExternalIntegrationResources(args: { query: string }
 		)
 
 		const model = getCurrentModel()
-		const modelContextWindow = getModelContextWindow(model.model)
+		const modelContextWindow = getModelContextWindow(
+			model.provider,
+			model.model,
+			get(copilotInfo).contextWindowPerModel
+		)
 		const results: PackageSearchResult[] = await Promise.all(
 			filtered.map(async (r: PackageSearchQuery) => {
 				let documentation = ''

frontend/src/lib/components/copilot/lib.test.ts

@@ -289,12 +289,20 @@ describe('model context windows', () => {
 		// a version between "qwen3" and "-max" must not claim the 256K entry, and
 		// there is deliberately no qwen family entry (variant windows range 8K–1M)
 		expect(getKnownModelContextWindow('qwen3.8-max')).toBeUndefined()
-		expect(getModelContextWindow('qwen3.8-max')).toBe(128000)
+		expect(getModelContextWindow('openai', 'qwen3.8-max', undefined)).toBe(128000)
 	})
 
 	it('returns undefined for unrecognized models, 128K via the defaulting wrapper', () => {
 		expect(getKnownModelContextWindow('some-custom-model')).toBeUndefined()
-		expect(getModelContextWindow('some-custom-model')).toBe(128000)
+		expect(getModelContextWindow('customai', 'some-custom-model', undefined)).toBe(128000)
+	})
+
+	it('prefers the configured window for the exact provider:model', () => {
+		const overrides = { 'customai:some-custom-model': 32000, 'anthropic:claude-opus-5': 200000 }
+		expect(getModelContextWindow('customai', 'some-custom-model', overrides)).toBe(32000)
+		expect(getModelContextWindow('anthropic', 'claude-opus-5', overrides)).toBe(200000)
+		// keyed by provider: the same id through another route keeps the table's window
+		expect(getModelContextWindow('openrouter', 'claude-opus-5', overrides)).toBe(1000000)
 	})
 })
 

frontend/src/lib/components/copilot/modelConfig.ts

@@ -162,9 +162,7 @@ export function buildModelMatchers<T>(
 			// separator is normalized. Only a short segment: a date is digits as well
 			// (`-20251101`) and stays a decoration.
 			strictVariants ? '(?!-\\d{1,3}(?:$|-))' : '',
-			strictVariants
-				? `(?!-(?!(?:v\\d|${DECORATIVE_SUFFIXES.join('|')})$)[a-z])`
-				: ''
+			strictVariants ? `(?!-(?!(?:v\\d|${DECORATIVE_SUFFIXES.join('|')})$)[a-z])` : ''
 		].join('')
 		return [new RegExp(pattern + guards), value]
 	})
@@ -172,7 +170,7 @@ export function buildModelMatchers<T>(
 
 /**
  * The `provider:model` key the workspace AI settings use for their per-model maps
- * (`max_tokens_per_model`, `model_pricing`). A bare model id is not enough: the
+ * (`max_tokens_per_model`, `context_window_per_model`, `model_pricing`). A bare model id is not enough: the
  * same id can be served by more than one provider at different rates.
  *
  * Matched exactly, unlike the fuzzy tables above. Those tables generalize across
@@ -196,9 +194,22 @@ export function getKnownModelContextWindow(model: string): number | undefined {
 	return matchModel(MODEL_CONTEXT_WINDOW_MATCHERS, model)
 }
 
-export function getModelContextWindow(model: string) {
+/** The admin's `context_window_per_model` entry wins over the table. */
+export function getConfiguredContextWindow(
+	provider: AIProvider | string,
+	model: string,
+	overrides: Record<string, number> | undefined
+): number | undefined {
+	return overrides?.[modelKey(provider, model)] ?? getKnownModelContextWindow(model)
+}
+
+export function getModelContextWindow(
+	provider: AIProvider | string,
+	model: string,
+	overrides: Record<string, number> | undefined
+) {
 	// Trim/compaction logic needs a number; assume a conservative window when unknown.
-	return getKnownModelContextWindow(model) ?? 128000
+	return getConfiguredContextWindow(provider, model, overrides) ?? 128000
 }
 
 /**

frontend/src/lib/components/workspaceSettings/AISettings.svelte

@@ -11,7 +11,12 @@
 	import { workspaceStore } from '$lib/stores'
 	import { copilotInfo } from '$lib/aiStore'
 	import { sendUserToast } from '$lib/toast'
-	import { AI_PROVIDERS, fetchAvailableModels, providerSupportsWebSearch } from '../copilot/lib'
+	import {
+		AI_PROVIDERS,
+		fetchAvailableModels,
+		getModelMaxTokens,
+		providerSupportsWebSearch
+	} from '../copilot/lib'
 	import { supportsAutocomplete } from '../copilot/utils'
 	import TestAiKey from '../copilot/TestAIKey.svelte'
 	import Label from '../Label.svelte'
@@ -25,6 +30,7 @@
 	import Badge from '../common/badge/Badge.svelte'
 	import Tooltip from '../Tooltip.svelte'
 	import ModelTokenLimits from './ModelTokenLimits.svelte'
+	import { getModelContextWindow } from '../copilot/modelConfig'
 	import ModelPricing from './ModelPricing.svelte'
 	import AiUsagePanel from './AiUsagePanel.svelte'
 	import { setCopilotInfo } from '$lib/aiStore'
@@ -77,6 +83,7 @@
 	let metadataModel: string | undefined = $state(undefined)
 	let customPrompts: Record<string, string> = $state({})
 	let maxTokensPerModel: Record<string, number> = $state({})
+	let contextWindowPerModel: Record<string, number> = $state({})
 	let modelPricing: Record<string, ModelPriceOverride> = $state({})
 	let usingOpenaiClientCredentialsOauth = $state(false)
 	let workspaceOverrideEditorOpened = $state(false)
@@ -91,6 +98,7 @@
 	let initialMetadataModel: string | undefined = $state(undefined)
 	let initialCustomPrompts: Record<string, string> = $state({})
 	let initialMaxTokensPerModel: Record<string, number> = $state({})
+	let initialContextWindowPerModel: Record<string, number> = $state({})
 	let initialModelPricing: Record<string, ModelPriceOverride> = $state({})
 	let initialPrompts: Record<string, string> = $state({})
 	let initialCopilotDisabled = $state(false)
@@ -122,6 +130,7 @@
 		codeCompletionModel = config?.code_completion_model?.model
 		customPrompts = clone(config?.custom_prompts ?? {})
 		maxTokensPerModel = clone(config?.max_tokens_per_model ?? {})
+		contextWindowPerModel = clone(config?.context_window_per_model ?? {})
 		modelPricing = clone(config?.model_pricing ?? {})
 		copilotDisabled = config?.copilot_disabled === true
 		sessionsStorageDisabled = config?.sessions_storage_disabled === true
@@ -140,6 +149,7 @@
 		initialCodeCompletionModel = codeCompletionModel
 		initialCustomPrompts = clone(customPrompts)
 		initialMaxTokensPerModel = clone(maxTokensPerModel)
+		initialContextWindowPerModel = clone(contextWindowPerModel)
 		initialModelPricing = clone(modelPricing)
 		initialPrompts = clone(customPrompts)
 		initialCopilotDisabled = copilotDisabled
@@ -159,6 +169,7 @@
 		codeCompletionModel = initialCodeCompletionModel
 		customPrompts = clone(initialCustomPrompts)
 		maxTokensPerModel = clone(initialMaxTokensPerModel)
+		contextWindowPerModel = clone(initialContextWindowPerModel)
 		modelPricing = clone(initialModelPricing)
 		copilotDisabled = initialCopilotDisabled
 		sessionsStorageDisabled = initialSessionsStorageDisabled
@@ -197,6 +208,7 @@
 			codeCompletionModel !== initialCodeCompletionModel ||
 			JSON.stringify(customPrompts) !== JSON.stringify(initialCustomPrompts) ||
 			JSON.stringify(maxTokensPerModel) !== JSON.stringify(initialMaxTokensPerModel) ||
+			JSON.stringify(contextWindowPerModel) !== JSON.stringify(initialContextWindowPerModel) ||
 			JSON.stringify(modelPricing) !== JSON.stringify(initialModelPricing) ||
 			copilotDisabled !== initialCopilotDisabled ||
 			sessionsStorageDisabled !== initialSessionsStorageDisabled ||
@@ -319,6 +331,8 @@
 					custom_prompts: Object.keys(custom_prompts).length > 0 ? custom_prompts : undefined,
 					max_tokens_per_model:
 						Object.keys(maxTokensPerModel).length > 0 ? maxTokensPerModel : undefined,
+					context_window_per_model:
+						Object.keys(contextWindowPerModel).length > 0 ? contextWindowPerModel : undefined,
 					model_pricing: Object.keys(modelPricing).length > 0 ? modelPricing : undefined,
 					copilot_disabled,
 					sessions_storage_disabled,
@@ -625,7 +639,24 @@
 			{/if}
 		</div>
 
-		<ModelTokenLimits {aiProviders} bind:maxTokensPerModel />
+		<ModelTokenLimits
+			{aiProviders}
+			bind:limits={contextWindowPerModel}
+			label="Model context windows"
+			description="Configure the context window of each model. AI chats compact their history before reaching it. Set it for models Windmill does not know, which otherwise assume 128K tokens."
+			defaultFor={(provider, model) => getModelContextWindow(provider, model, undefined)}
+			min={1000}
+			max={10_000_000}
+		/>
+
+		<ModelTokenLimits
+			{aiProviders}
+			bind:limits={maxTokensPerModel}
+			label="Model output limits"
+			description="Configure maximum token limits for each model. These limits apply to all AI chat interactions in the workspace."
+			defaultFor={getModelMaxTokens}
+			max={2_000_000}
+		/>
 
 		<SettingCard label="Custom system prompts" description={promptDescription}>
 			<div class="flex items-center gap-2 pt-1">

frontend/src/lib/components/workspaceSettings/ModelTokenLimits.svelte

@@ -1,19 +1,28 @@
 <script lang="ts">
 	import type { AIConfig, AIProvider } from '$lib/gen'
 	import { Badge, Button } from '../common'
-	import { getModelMaxTokens } from '../copilot/lib'
 	import { ChevronDown, ChevronUp } from 'lucide-svelte'
 	import { slide } from 'svelte/transition'
 	import SettingCard from '../instanceSettings/SettingCard.svelte'
 
-	const MAX_TOKENS_LIMIT = 2000000
-
+	// Edits one `provider:model` map of the AI config. Only values that differ from
+	// `defaultFor` are stored.
 	let {
 		aiProviders,
-		maxTokensPerModel = $bindable()
+		limits = $bindable(),
+		label,
+		description,
+		defaultFor,
+		min = 1,
+		max
 	}: {
 		aiProviders: Exclude<AIConfig['providers'], undefined>
-		maxTokensPerModel: Record<string, number>
+		limits: Record<string, number>
+		label: string
+		description: string
+		defaultFor: (provider: AIProvider, model: string) => number
+		min?: number
+		max: number
 	} = $props()
 
 	let errors = $state<Record<string, string>>({})
@@ -37,32 +46,28 @@
 		return `${provider}:${model}`
 	}
 
-	function getDefaultTokensForModel(provider: AIProvider, model: string): number {
-		return getModelMaxTokens(provider, model)
-	}
-
 	function getCurrentTokensForModel(provider: AIProvider, model: string): number {
 		const modelKey = getModelKey(provider, model)
-		return maxTokensPerModel[modelKey] ?? getDefaultTokensForModel(provider, model)
+		return limits[modelKey] ?? defaultFor(provider, model)
 	}
 
 	function updateTokensForModel(provider: AIProvider, model: string, tokens: number) {
 		const modelKey = getModelKey(provider, model)
-		if (tokens < 1 || tokens > MAX_TOKENS_LIMIT) {
-			errors[modelKey] = 'Token limit must be between 1 and ' + MAX_TOKENS_LIMIT
+		if (tokens < min || tokens > max) {
+			errors[modelKey] = `Token limit must be between ${min} and ${max}`
 			return
 		}
 
-		const defaultTokens = getDefaultTokensForModel(provider, model)
+		const defaultTokens = defaultFor(provider, model)
 
 		if (tokens === defaultTokens) {
 			// Remove from object if it's the default value
-			const newSettings = { ...maxTokensPerModel }
+			const newSettings = { ...limits }
 			delete newSettings[modelKey]
-			maxTokensPerModel = newSettings
+			limits = newSettings
 		} else {
-			maxTokensPerModel = {
-				...maxTokensPerModel,
+			limits = {
+				...limits,
 				[modelKey]: tokens
 			}
 		}
@@ -71,14 +76,15 @@
 
 	function resetModelToDefault(provider: AIProvider, model: string) {
 		const modelKey = getModelKey(provider, model)
-		const newSettings = { ...maxTokensPerModel }
+		const newSettings = { ...limits }
 		delete newSettings[modelKey]
-		maxTokensPerModel = newSettings
+		limits = newSettings
+		errors[modelKey] = ''
 	}
 
 	function isModelAtDefault(provider: AIProvider, model: string): boolean {
 		const currentTokens = getCurrentTokensForModel(provider, model)
-		const defaultTokens = getDefaultTokensForModel(provider, model)
+		const defaultTokens = defaultFor(provider, model)
 		return currentTokens === defaultTokens
 	}
 
@@ -99,10 +105,7 @@
 </script>
 
 {#if Object.keys(aiProviders).length > 0}
-	<SettingCard
-		label="Model output limits"
-		description="Configure maximum token limits for each model. These limits apply to all AI chat interactions in the workspace."
-	>
+	<SettingCard {label} {description}>
 		<div class="flex flex-col gap-3">
 			{#each Object.entries(modelsByProvider).filter(([provider, models]) => models.length > 0) as [provider, models]}
 				{@const isExpanded = !collapsedProviders[provider]}
@@ -131,7 +134,7 @@
 							<div class="space-y-3">
 								{#each models as { model }}
 									{@const currentTokens = getCurrentTokensForModel(provider as AIProvider, model)}
-									{@const defaultTokens = getDefaultTokensForModel(provider as AIProvider, model)}
+									{@const defaultTokens = defaultFor(provider as AIProvider, model)}
 									{@const isAtDefault = isModelAtDefault(provider as AIProvider, model)}
 									<div class="flex flex-col gap-1">
 										<div class="flex items-center gap-3">
@@ -141,8 +144,8 @@
 											<div class="flex items-center gap-2">
 												<input
 													type="number"
-													min="1"
-													max={MAX_TOKENS_LIMIT}
+													{min}
+													{max}
 													value={currentTokens}
 													oninput={(e) => {
 														const value = parseInt(e.currentTarget.value)
@@ -150,7 +153,7 @@
 															updateTokensForModel(provider as AIProvider, model, value)
 														}
 													}}
-													class="w-20 px-2 py-1 text-xs text-center border border-gray-200 dark:border-gray-700 rounded bg-surface focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent"
+													class="w-24 px-2 py-1 text-xs text-center border border-gray-200 dark:border-gray-700 rounded bg-surface focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent"
 												/>
 												<span class="text-xs text-secondary whitespace-nowrap">tokens</span>
 											</div>
@@ -166,11 +169,11 @@
 													Reset
 												</Button>
 											</div>
-											{#if errors[getModelKey(provider as AIProvider, model)]}
-												<div class="text-xs text-red-500"
-													>{errors[getModelKey(provider as AIProvider, model)]}</div
-												>
-											{/if}
+										{/if}
+										{#if errors[getModelKey(provider as AIProvider, model)]}
+											<div class="text-xs text-red-500"
+												>{errors[getModelKey(provider as AIProvider, model)]}</div
+											>
 										{/if}
 									</div>
 								{/each}