diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index c385a78363..b6ca1b56c8 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -14514,6 +14514,12 @@ components: type: object additionalProperties: type: string + max_tokens_per_model: + type: object + additionalProperties: + type: integer + minimum: 1 + maximum: 2000000 Alert: type: object diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index 9f13d32cf5..abde227214 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -272,6 +272,8 @@ pub struct AIConfig { pub code_completion_model: Option, #[serde(skip_serializing_if = "Option::is_none")] pub custom_prompts: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_tokens_per_model: Option>, } pub fn global_service() -> Router { diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index b19f47b6cb..1b9d928cf4 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -923,6 +923,7 @@ async fn get_copilot_info( default_model: None, code_completion_model: None, custom_prompts: None, + max_tokens_per_model: None, })) } } diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index 75489ecfaa..b3f624a9d2 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -84,7 +84,9 @@ } }) - const isLastMessageTool = $derived(messages.length > 0 && messages[messages.length - 1].role === 'tool') + const isLastMessageTool = $derived( + messages.length > 0 && messages[messages.length - 1].role === 'tool' + )
diff --git a/frontend/src/lib/components/copilot/chat/ProviderModelSelector.svelte b/frontend/src/lib/components/copilot/chat/ProviderModelSelector.svelte index 6c7068a80c..ffd2b7075a 100644 --- a/frontend/src/lib/components/copilot/chat/ProviderModelSelector.svelte +++ b/frontend/src/lib/components/copilot/chat/ProviderModelSelector.svelte @@ -26,7 +26,7 @@
{providerModel.model} {#if multipleModels} -
+
{/if} diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index addcda69e2..0ccc7ab975 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -1,5 +1,6 @@ import type { AIProvider, AIProviderModel } from '$lib/gen' import { + copilotInfo, getCurrentModel, workspaceStore, type DBSchema, @@ -157,7 +158,7 @@ export async function fetchAvailableModels( return data?.data.map((m) => m.id) ?? [] } -function getModelMaxTokens(provider: AIProvider, model: string) { +export function getModelMaxTokens(provider: AIProvider, model: string) { if (model.startsWith('gpt-5')) { return 128000 } else if ((provider === 'azure_openai' || provider === 'openai') && model.startsWith('o')) { @@ -196,7 +197,10 @@ function getModelSpecificConfig( modelProvider: AIProviderModel, tools?: OpenAI.Chat.Completions.ChatCompletionTool[] ) { - const maxTokens = getModelMaxTokens(modelProvider.provider, modelProvider.model) + const defaultMaxTokens = getModelMaxTokens(modelProvider.provider, modelProvider.model) + const modelKey = `${modelProvider.provider}:${modelProvider.model}` + const customMaxTokensStore = get(copilotInfo)?.maxTokensPerModel + const maxTokens = customMaxTokensStore?.[modelKey] ?? defaultMaxTokens if ( (modelProvider.provider === 'openai' || modelProvider.provider === 'azure_openai') && (modelProvider.model.startsWith('o') || modelProvider.model.startsWith('gpt-5')) diff --git a/frontend/src/lib/components/workspaceSettings/AISettings.svelte b/frontend/src/lib/components/workspaceSettings/AISettings.svelte index 0b34bf0b54..06afffee6f 100644 --- a/frontend/src/lib/components/workspaceSettings/AISettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/AISettings.svelte @@ -18,6 +18,7 @@ import ToggleButtonGroup from '../common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from '../common/toggleButton-v2/ToggleButton.svelte' import autosize from '$lib/autosize' + import ModelTokenLimits from './ModelTokenLimits.svelte' const MAX_CUSTOM_PROMPT_LENGTH = 5000 @@ -26,12 +27,14 @@ codeCompletionModel = $bindable(), defaultModel = $bindable(), customPrompts = $bindable(), + maxTokensPerModel = $bindable(), usingOpenaiClientCredentialsOauth = $bindable() }: { aiProviders: Exclude codeCompletionModel: string | undefined defaultModel: string | undefined customPrompts: Record + maxTokensPerModel: Record usingOpenaiClientCredentialsOauth: boolean } = $props() @@ -101,7 +104,8 @@ providers: aiProviders, code_completion_model, default_model, - custom_prompts: Object.keys(custom_prompts).length > 0 ? custom_prompts : undefined + custom_prompts: Object.keys(custom_prompts).length > 0 ? custom_prompts : undefined, + max_tokens_per_model: Object.keys(maxTokensPerModel).length > 0 ? maxTokensPerModel : undefined } await WorkspaceService.editCopilotConfig({ workspace: $workspaceStore!, @@ -317,6 +321,10 @@
{/if} + {#if Object.keys(aiProviders).length > 0} + + {/if} + {#if Object.keys(aiProviders).length > 0}

Custom system prompts

diff --git a/frontend/src/lib/components/workspaceSettings/ModelTokenLimits.svelte b/frontend/src/lib/components/workspaceSettings/ModelTokenLimits.svelte new file mode 100644 index 0000000000..1ecf775c08 --- /dev/null +++ b/frontend/src/lib/components/workspaceSettings/ModelTokenLimits.svelte @@ -0,0 +1,192 @@ + + +{#if Object.keys(aiProviders).length > 0} +
+
+

Model Output Limits

+

+ Configure maximum token limits for each model. These limits apply to all AI chat + interactions in the workspace. +

+
+ +
+ {#each Object.entries(modelsByProvider).filter(([provider, models]) => models.length > 0) as [provider, models]} + {@const isExpanded = !collapsedProviders[provider]} + {@const hasCustom = hasCustomSettings(provider, models)} +
+ + + {#if isExpanded} +
+
+ {#each models as { model }} + {@const currentTokens = getCurrentTokensForModel(provider as AIProvider, model)} + {@const defaultTokens = getDefaultTokensForModel(provider as AIProvider, model)} + {@const isAtDefault = isModelAtDefault(provider as AIProvider, model)} +
+
+
+ {model} +
+
+ { + const value = parseInt(e.currentTarget.value) + if (!isNaN(value)) { + updateTokensForModel(provider as AIProvider, model, value) + } + }} + class="w-20 px-2 py-1 text-xs text-center border border-gray-200 dark:border-gray-700 rounded bg-surface focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent" + /> + tokens +
+
+ {#if !isAtDefault} +
+ Default: {defaultTokens} tokens + +
+ {#if errors[getModelKey(provider as AIProvider, model)]} +
{errors[getModelKey(provider as AIProvider, model)]}
+ {/if} + {/if} +
+ {/each} +
+
+ {/if} +
+ {/each} +
+
+{/if} diff --git a/frontend/src/lib/stores.ts b/frontend/src/lib/stores.ts index 61f22fc478..5c33b2cca6 100644 --- a/frontend/src/lib/stores.ts +++ b/frontend/src/lib/stores.ts @@ -102,12 +102,14 @@ export const copilotInfo = writable<{ defaultModel?: AIProviderModel aiModels: AIProviderModel[] customPrompts?: Record + maxTokensPerModel?: Record }>({ enabled: false, codeCompletionModel: undefined, defaultModel: undefined, aiModels: [], - customPrompts: {} + customPrompts: {}, + maxTokensPerModel: {} }) export async function loadCopilot(workspace: string) { @@ -143,7 +145,8 @@ export function setCopilotInfo(aiConfig: AIConfig) { codeCompletionModel: aiConfig.code_completion_model, defaultModel: aiConfig.default_model, aiModels: aiModels, - customPrompts: aiConfig.custom_prompts ?? {} + customPrompts: aiConfig.custom_prompts ?? {}, + maxTokensPerModel: aiConfig.max_tokens_per_model ?? {} }) } else { copilotSessionModel.set(undefined) @@ -153,7 +156,8 @@ export function setCopilotInfo(aiConfig: AIConfig) { codeCompletionModel: undefined, defaultModel: undefined, aiModels: [], - customPrompts: {} + customPrompts: {}, + maxTokensPerModel: {} }) } } @@ -197,6 +201,7 @@ export const copilotSessionModel = writable( } : undefined ) + export const usedTriggerKinds = writable([]) type SQLBaseSchema = { diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index f7251ec844..47fd6e33b7 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -81,6 +81,7 @@ let codeCompletionModel: string | undefined = $state(undefined) let defaultModel: string | undefined = $state(undefined) let customPrompts: Record = $state({}) + let maxTokensPerModel: Record = $state({}) let s3ResourceSettings: S3ResourceSettings = $state({ resourceType: 's3', @@ -246,6 +247,7 @@ defaultModel = settings.ai_config?.default_model?.model codeCompletionModel = settings.ai_config?.code_completion_model?.model customPrompts = settings.ai_config?.custom_prompts ?? {} + maxTokensPerModel = settings.ai_config?.max_tokens_per_model ?? {} for (const mode of Object.values(AIMode)) { if (!(mode in customPrompts)) { customPrompts[mode] = '' @@ -813,6 +815,7 @@ bind:codeCompletionModel bind:defaultModel bind:customPrompts + bind:maxTokensPerModel bind:usingOpenaiClientCredentialsOauth /> {:else if tab == 'windmill_lfs'}