From 7751d3e43ee1abbba9a6cca026be78504f0c5dff Mon Sep 17 00:00:00 2001 From: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Date: Mon, 24 Aug 2026 08:50:23 +0200 Subject: [PATCH 01/23] feat(frontend): warn when COEP blocks cross-origin resources in raw app editor preview (#10328) * feat(frontend): warn when COEP blocks cross-origin resources in raw app editor preview Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Gr3udvsSKYyH6mnDvGDqEE * fix(frontend): hedge COEP toast wording and attach warning on detached preview initial load Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Gr3udvsSKYyH6mnDvGDqEE * chore(frontend): condense COEP warning rationale comment Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Gr3udvsSKYyH6mnDvGDqEE --------- Co-authored-by: Claude Fable 5 --- .../raw_apps/RawAppCoepWarning.svelte | 92 +++++++++++++++++++ .../components/raw_apps/RawAppEditor.svelte | 9 ++ 2 files changed, 101 insertions(+) create mode 100644 frontend/src/lib/components/raw_apps/RawAppCoepWarning.svelte diff --git a/frontend/src/lib/components/raw_apps/RawAppCoepWarning.svelte b/frontend/src/lib/components/raw_apps/RawAppCoepWarning.svelte new file mode 100644 index 0000000000..fca969de27 --- /dev/null +++ b/frontend/src/lib/components/raw_apps/RawAppCoepWarning.svelte @@ -0,0 +1,92 @@ + + + +
+

+ The app editor runs in a cross-origin isolated context (COOP/COEP headers). This is + required for SharedArrayBuffer, which powers the TypeScript language workers and + lets the editor build and preview your frontend live in the browser. +

+

+ A side effect is that the browser refuses to load cross-origin resources (images, scripts, + stylesheets, media…) unless the remote server explicitly opts in with CORS or a + Cross-Origin-Resource-Policy header. Resources from servers that don't are + blocked in the editor preview only. When this is the cause, the browser console shows + ERR_BLOCKED_BY_RESPONSE — a plain 404 or DNS error instead means the URL itself is + broken and will fail on the deployed app too. +

+

+ The deployed app is served without these headers, so the same resources load normally there — + open the deployed app link to verify. If you control the remote server, sending + Cross-Origin-Resource-Policy: cross-origin makes the resource load in the editor too. +

+
+ diff --git a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte index 0778fda4e4..4da35eb253 100644 --- a/frontend/src/lib/components/raw_apps/RawAppEditor.svelte +++ b/frontend/src/lib/components/raw_apps/RawAppEditor.svelte @@ -29,6 +29,7 @@ } from './utils' import { runDomQueryOnHtml, type RawAppDomQuery, type RawAppDomRequester } from './rawAppDom' import InlineElementPrompt from './InlineElementPrompt.svelte' + import RawAppCoepWarning from './RawAppCoepWarning.svelte' import DarkModeObserver from '../DarkModeObserver.svelte' import { getAppliedDarkModeVariant, type DarkModeVariant } from '$lib/darkModeVariant' import RawAppSidebar from './RawAppSidebar.svelte' @@ -351,6 +352,7 @@ let iframe: HTMLIFrameElement | undefined = $state(undefined) const PREVIEW_SHELL_URL = '/ui_builder/app-preview.html' let previewIframe: HTMLIFrameElement | undefined = $state(undefined) + let coepWarning: RawAppCoepWarning | undefined = $state(undefined) let previewIframeLoaded = $state(false) let lastBuild: { css: string; js: string } | undefined = undefined // Detached preview tab/window rendering the same app-preview bundle as the @@ -1164,6 +1166,9 @@ ) { externalPreviewReady = true feedExternalPreview() + // The detached window is cross-origin isolated like the inline preview, + // so blocked external resources warrant the same COEP warning. + coepWarning?.attachTo(externalPreviewWindow) return } @@ -1504,6 +1509,9 @@ win.addEventListener('load', () => { externalPreviewReady = true feedExternalPreview() + // Attach here too: against an artifact that predates the handshake, this + // is the only place the freshly opened window is ever seen loaded. + coepWarning?.attachTo(win) }) } @@ -2539,6 +2547,7 @@ src={PREVIEW_SHELL_URL} class="w-full flex-1 block" > + {#if buildError} From b6e059116aa55fa5aa1226f5b3300bb2c8683f1a Mon Sep 17 00:00:00 2001 From: hugocasa Date: Mon, 24 Aug 2026 11:08:13 +0200 Subject: [PATCH 02/23] feat: track token cost in AI sessions and chats (#10688) * feat: track token cost in AI sessions and chats * fix: address review findings on AI cost tracking * fix: price inherited and overridden models at their real rates * fix: stop newer model revisions inheriting an older price * fix: stop a sub-model inheriting its family's price * fix: keep alias suffixes resolving to their model's price * fix: count OpenRouter cache writes and drop unverifiable rates * refactor: move AI spend out of the chat into workspace and user settings * fix: pin the usage workspace per turn and stop inventing cache rates * fix: leave Sonnet 5 unpriced while its promotional rate runs * docs: record the new table in the schema summary and tighten comments * fix: mark estimated AI costs with ~ and drop session grouping Co-Authored-By: Claude Opus 5 (1M context) * fix: name the workspace in the self-scoped AI usage title Co-Authored-By: Claude Opus 5 (1M context) * docs: state that overrides never replace a provider-returned cost Co-Authored-By: Claude Opus 5 (1M context) * fix: let a cleared cache rate inherit again and flag partial totals Co-Authored-By: Claude Opus 5 (1M context) * fix: clear a refused rate's error when the input snaps back Co-Authored-By: Claude Opus 5 (1M context) * fix: stop a revision variant inheriting its base family's rate Co-Authored-By: Claude Opus 5 (1M context) * fix: report AI usage before tools run and price self usage consistently Co-Authored-By: Claude Opus 5 (1M context) * fix: key pricing rows on the model id usage is reported under Co-Authored-By: Claude Opus 5 (1M context) * fix: surface Bedrock and Gemini usage the chat proxy was dropping Co-Authored-By: Claude Opus 5 (1M context) * fix: count Gemini tool-use prompt tokens as input Co-Authored-By: Claude Opus 5 (1M context) * feat: price flat-rate Gemini Flash models Co-Authored-By: Claude Opus 5 (1M context) * docs: state the tool-use token invariant once Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- ...1c19b2310075a644d81c21784c991e52b4001.json | 24 ++ ...d573168161abe350b047402cda4ac4a4d13e4.json | 74 ++++ .../20260813164338_ai_token_usage.down.sql | 1 + .../20260813164338_ai_token_usage.up.sql | 42 +++ backend/summarized_schema.txt | 2 + backend/windmill-ai/src/ai_google.rs | 124 ++++++- backend/windmill-ai/src/ai_types.rs | 54 +++ backend/windmill-ai/src/providers/bedrock.rs | 72 ++++ backend/windmill-api-settings/src/lib.rs | 7 + backend/windmill-api/openapi.yaml | 153 ++++++++ backend/windmill-api/src/ai.rs | 331 +++++++++++++++++- backend/windmill-api/src/workspaces.rs | 2 + frontend/src/lib/aiStore.ts | 12 +- .../src/lib/components/UserSettings.svelte | 13 + .../copilot/chat/AIChatManager.svelte.ts | 55 ++- .../copilot/chat/ContextUsageIndicator.svelte | 11 +- .../lib/components/copilot/chat/anthropic.ts | 3 +- .../components/copilot/chat/chatLoop.test.ts | 2 +- .../lib/components/copilot/chat/chatLoop.ts | 28 +- .../copilot/chat/openai-responses.ts | 3 +- .../copilot/chat/tokenUsage.test.ts | 61 ++++ .../lib/components/copilot/chat/tokenUsage.ts | 87 ++++- frontend/src/lib/components/copilot/lib.ts | 36 +- .../src/lib/components/copilot/modelConfig.ts | 78 ++++- .../components/copilot/modelPricing.test.ts | 187 ++++++++++ .../lib/components/copilot/modelPricing.ts | 288 +++++++++++++++ .../workspaceSettings/AISettings.svelte | 35 +- .../workspaceSettings/AiUsagePanel.svelte | 226 ++++++++++++ .../workspaceSettings/ModelPricing.svelte | 252 +++++++++++++ frontend/src/lib/utils/aiUsageReporter.ts | 154 ++++++++ 30 files changed, 2358 insertions(+), 59 deletions(-) create mode 100644 backend/.sqlx/query-24fcc2b69f30953915f0cbf246e1c19b2310075a644d81c21784c991e52b4001.json create mode 100644 backend/.sqlx/query-6a17a1dfeb75808e5d0726f1f8bd573168161abe350b047402cda4ac4a4d13e4.json create mode 100644 backend/migrations/20260813164338_ai_token_usage.down.sql create mode 100644 backend/migrations/20260813164338_ai_token_usage.up.sql create mode 100644 frontend/src/lib/components/copilot/chat/tokenUsage.test.ts create mode 100644 frontend/src/lib/components/copilot/modelPricing.test.ts create mode 100644 frontend/src/lib/components/copilot/modelPricing.ts create mode 100644 frontend/src/lib/components/workspaceSettings/AiUsagePanel.svelte create mode 100644 frontend/src/lib/components/workspaceSettings/ModelPricing.svelte create mode 100644 frontend/src/lib/utils/aiUsageReporter.ts diff --git a/backend/.sqlx/query-24fcc2b69f30953915f0cbf246e1c19b2310075a644d81c21784c991e52b4001.json b/backend/.sqlx/query-24fcc2b69f30953915f0cbf246e1c19b2310075a644d81c21784c991e52b4001.json new file mode 100644 index 0000000000..a2cd96857a --- /dev/null +++ b/backend/.sqlx/query-24fcc2b69f30953915f0cbf246e1c19b2310075a644d81c21784c991e52b4001.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_token_usage (workspace_id, email, provider, model, session_id, input_tokens, cache_read_tokens, cache_write_tokens, output_tokens, reported_cost_nano_usd, requests)\n SELECT $1, $2, * FROM UNNEST($3::text[], $4::text[], $5::text[], $6::bigint[], $7::bigint[], $8::bigint[], $9::bigint[], $10::bigint[], $11::bigint[])\n ON CONFLICT (workspace_id, day, email, provider, model, session_id)\n DO UPDATE SET\n input_tokens = ai_token_usage.input_tokens + EXCLUDED.input_tokens,\n cache_read_tokens = ai_token_usage.cache_read_tokens + EXCLUDED.cache_read_tokens,\n cache_write_tokens = ai_token_usage.cache_write_tokens + EXCLUDED.cache_write_tokens,\n output_tokens = ai_token_usage.output_tokens + EXCLUDED.output_tokens,\n reported_cost_nano_usd = CASE\n WHEN EXCLUDED.reported_cost_nano_usd IS NULL\n THEN ai_token_usage.reported_cost_nano_usd\n ELSE COALESCE(ai_token_usage.reported_cost_nano_usd, 0)\n + EXCLUDED.reported_cost_nano_usd\n END,\n requests = ai_token_usage.requests + EXCLUDED.requests,\n updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "TextArray", + "TextArray", + "TextArray", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array" + ] + }, + "nullable": [] + }, + "hash": "24fcc2b69f30953915f0cbf246e1c19b2310075a644d81c21784c991e52b4001" +} diff --git a/backend/.sqlx/query-6a17a1dfeb75808e5d0726f1f8bd573168161abe350b047402cda4ac4a4d13e4.json b/backend/.sqlx/query-6a17a1dfeb75808e5d0726f1f8bd573168161abe350b047402cda4ac4a4d13e4.json new file mode 100644 index 0000000000..6ced022760 --- /dev/null +++ b/backend/.sqlx/query-6a17a1dfeb75808e5d0726f1f8bd573168161abe350b047402cda4ac4a4d13e4.json @@ -0,0 +1,74 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (CASE $3::text\n WHEN 'day' THEN day::text\n WHEN 'user' THEN email\n ELSE ''\n END) AS \"key!\",\n provider AS \"provider!\",\n model AS \"model!\",\n SUM(input_tokens)::bigint AS \"input_tokens!\",\n SUM(cache_read_tokens)::bigint AS \"cache_read_tokens!\",\n SUM(cache_write_tokens)::bigint AS \"cache_write_tokens!\",\n SUM(output_tokens)::bigint AS \"output_tokens!\",\n SUM(reported_cost_nano_usd)::bigint AS \"reported_cost_nano_usd\",\n SUM(requests)::bigint AS \"requests!\"\n FROM ai_token_usage\n WHERE workspace_id = $1 AND day > CURRENT_DATE - $2::int\n AND ($5::text IS NULL OR email = $5)\n GROUP BY 1, provider, model\n ORDER BY SUM(input_tokens + cache_read_tokens + cache_write_tokens + output_tokens) DESC\n LIMIT $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "key!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "provider!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "model!", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "input_tokens!", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "cache_read_tokens!", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "cache_write_tokens!", + "type_info": "Int8" + }, + { + "ordinal": 6, + "name": "output_tokens!", + "type_info": "Int8" + }, + { + "ordinal": 7, + "name": "reported_cost_nano_usd", + "type_info": "Int8" + }, + { + "ordinal": 8, + "name": "requests!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Int4", + "Text", + "Int8", + "Text" + ] + }, + "nullable": [ + null, + false, + false, + null, + null, + null, + null, + null, + null + ] + }, + "hash": "6a17a1dfeb75808e5d0726f1f8bd573168161abe350b047402cda4ac4a4d13e4" +} diff --git a/backend/migrations/20260813164338_ai_token_usage.down.sql b/backend/migrations/20260813164338_ai_token_usage.down.sql new file mode 100644 index 0000000000..0fa1b47462 --- /dev/null +++ b/backend/migrations/20260813164338_ai_token_usage.down.sql @@ -0,0 +1 @@ +DROP TABLE ai_token_usage; diff --git a/backend/migrations/20260813164338_ai_token_usage.up.sql b/backend/migrations/20260813164338_ai_token_usage.up.sql new file mode 100644 index 0000000000..b20e471db5 --- /dev/null +++ b/backend/migrations/20260813164338_ai_token_usage.up.sql @@ -0,0 +1,42 @@ +-- Per-workspace AI token spend, accumulated from the chat client. Rows hold token +-- counts rather than money: prices live in the frontend price table plus the +-- workspace's `ai_config.model_pricing` overrides and are applied at read time, so +-- correcting a price also corrects the history. `reported_cost_nano_usd` is the +-- exception — a few providers (OpenRouter) return what they actually charged, and +-- that figure wins over the estimate. +-- +-- Distinct from `feature_usage`, which is anonymous telemetry that leaves the +-- instance and is pruned after 60 days; spend is per-user and kept. +CREATE TABLE ai_token_usage ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + day DATE NOT NULL DEFAULT CURRENT_DATE, + email VARCHAR(255) NOT NULL, + provider VARCHAR(50) NOT NULL, + model VARCHAR(255) NOT NULL, + -- Empty for chats that are not attached to an AI session. + session_id VARCHAR(50) NOT NULL DEFAULT '', + -- Uncached input only; the two cache columns hold the rest of the prompt, so + -- each column maps to exactly one price and they never double-count. + input_tokens BIGINT NOT NULL DEFAULT 0, + cache_read_tokens BIGINT NOT NULL DEFAULT 0, + cache_write_tokens BIGINT NOT NULL DEFAULT 0, + output_tokens BIGINT NOT NULL DEFAULT 0, + reported_cost_nano_usd BIGINT, + requests BIGINT NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, day, email, provider, model, session_id) +); + +-- The usage listing filters on workspace and a date range; the PK only reaches +-- `day` through `email`, so it cannot serve that on its own. +CREATE INDEX idx_ai_token_usage_ws_day ON ai_token_usage (workspace_id, day DESC); + +GRANT ALL ON ai_token_usage TO windmill_admin; +GRANT ALL ON ai_token_usage TO windmill_user; + +-- Both handlers go through the raw pool, so no policy is needed for them to work. +-- Enabling RLS with an admin-only policy is the backstop: a future query that +-- reaches this table through UserDB sees nothing rather than every user's spend. +ALTER TABLE ai_token_usage ENABLE ROW LEVEL SECURITY; + +CREATE POLICY admin_policy ON ai_token_usage FOR ALL TO windmill_admin USING (true); diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 732f1ece52..a9865cebd3 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -38,6 +38,8 @@ account: workspace_id(char), id(int), expires_at(ts), refresh_token(char), clien FK: (workspace_id) -> workspace(id) agent_token_blacklist: token(char), expires_at(ts), blacklisted_at(ts), blacklisted_by(char) ai_agent_memory: workspace_id(char), conversation_id(uuid), step_id(char), messages(jsonb), created_at(ts), updated_at(ts) +ai_token_usage: workspace_id(char), day(date), email(char), provider(char), model(char), session_id(char), input_tokens(bigint), cache_read_tokens(bigint), cache_write_tokens(bigint), output_tokens(bigint), reported_cost_nano_usd(bigint), requests(bigint), updated_at(ts) + FK: (workspace_id) -> workspace(id) alerts: id(int), alert_type(char), message(text), created_at(ts), acknowledged(bool), workspace_id(text), acknowledged_workspace(bool), resource(text) app: id(bigint), workspace_id(char), path(char), summary(char), policy(jsonb), versions(bigint[]), extra_perms(jsonb), draft_only(bool), custom_path(text), labels(text[]) FK: (workspace_id) -> workspace(id) diff --git a/backend/windmill-ai/src/ai_google.rs b/backend/windmill-ai/src/ai_google.rs index 4374cb8628..b90bda9e1c 100644 --- a/backend/windmill-ai/src/ai_google.rs +++ b/backend/windmill-ai/src/ai_google.rs @@ -277,7 +277,7 @@ pub struct GeminiSSECandidate { } /// Token usage from the `usageMetadata` field of a Gemini SSE event. -#[derive(Deserialize, Debug, Clone)] +#[derive(Deserialize, Debug, Clone, Default)] pub struct GeminiUsageMetadata { #[serde(rename = "promptTokenCount", default)] pub prompt_token_count: Option, @@ -285,6 +285,39 @@ pub struct GeminiUsageMetadata { pub candidates_token_count: Option, #[serde(rename = "totalTokenCount", default)] pub total_token_count: Option, + /// Subset of `promptTokenCount` served from context cache, billed at a reduced + /// rate. Reported separately so the client can price it separately. + #[serde(rename = "cachedContentTokenCount", default)] + pub cached_content_token_count: Option, + /// Thinking tokens, billed as output but counted apart from `candidatesTokenCount`. + #[serde(rename = "thoughtsTokenCount", default)] + pub thoughts_token_count: Option, + /// Input tokens spent on tool-use prompts, counted apart from `promptTokenCount` + /// rather than within it. + #[serde(rename = "toolUsePromptTokenCount", default)] + pub tool_use_prompt_token_count: Option, +} + +/// Input tokens as billed. Gemini reports tool-use prompts in their own field, and +/// they are disjoint from `promptTokenCount`: a live tool call returns 17 prompt + +/// 60 tool-use + 17 candidates + 52 thoughts against a `totalTokenCount` of 146, so +/// leaving them out under-reports the input of every tool-using turn. Cached tokens +/// are not added here, being already part of `promptTokenCount`. +fn gemini_prompt_tokens(usage: &GeminiUsageMetadata) -> i32 { + usage + .prompt_token_count + .unwrap_or(0) + .saturating_add(usage.tool_use_prompt_token_count.unwrap_or(0)) +} + +/// Output tokens as billed: Gemini counts thinking apart from `candidatesTokenCount` +/// but charges it at the output rate, so a reply that thought would otherwise be +/// reported as far cheaper than it was. +fn gemini_completion_tokens(usage: &GeminiUsageMetadata) -> i32 { + usage + .candidates_token_count + .unwrap_or(0) + .saturating_add(usage.thoughts_token_count.unwrap_or(0)) } /// Top-level structure of one Gemini SSE event. @@ -588,9 +621,12 @@ pub fn gemini_response_to_openai(parsed: &GeminiParsedEvent, model: &str) -> ser let usage = parsed.usage.as_ref().map(|u| { serde_json::json!({ - "prompt_tokens": u.prompt_token_count.unwrap_or(0), - "completion_tokens": u.candidates_token_count.unwrap_or(0), + "prompt_tokens": gemini_prompt_tokens(u), + "completion_tokens": gemini_completion_tokens(u), "total_tokens": u.total_token_count.unwrap_or(0), + "prompt_tokens_details": { + "cached_tokens": u.cached_content_token_count.unwrap_or(0) + }, }) }); @@ -680,8 +716,8 @@ pub fn gemini_event_to_openai_sse_chunks( // OpenAI's `stream_options.include_usage` terminal chunk (top-level `usage`, // empty `choices`) so the frontend's `'usage' in chunk` path records them. if let Some(usage) = &parsed.usage { - let prompt_tokens = usage.prompt_token_count.unwrap_or(0); - let completion_tokens = usage.candidates_token_count.unwrap_or(0); + let prompt_tokens = gemini_prompt_tokens(usage); + let completion_tokens = gemini_completion_tokens(usage); let total_tokens = usage .total_token_count .unwrap_or(prompt_tokens + completion_tokens); @@ -694,6 +730,9 @@ pub fn gemini_event_to_openai_sse_chunks( "prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, "total_tokens": total_tokens, + "prompt_tokens_details": { + "cached_tokens": usage.cached_content_token_count.unwrap_or(0) + }, } }); chunks.push(format!("data: {}\n\n", chunk)); @@ -943,6 +982,7 @@ mod tests { prompt_token_count: Some(12), candidates_token_count: Some(7), total_token_count: Some(19), + ..Default::default() }), ..Default::default() }; @@ -969,6 +1009,79 @@ mod tests { assert_eq!(usage_chunk["choices"], serde_json::json!([])); } + #[test] + fn gemini_usage_chunk_splits_cached_and_bills_thoughts() { + let parsed = GeminiParsedEvent { + text: Some("the answer".to_string()), + usage: Some(GeminiUsageMetadata { + prompt_token_count: Some(1000), + candidates_token_count: Some(20), + total_token_count: Some(1120), + cached_content_token_count: Some(900), + thoughts_token_count: Some(100), + ..Default::default() + }), + ..Default::default() + }; + + let mut tool_call_index = 0; + let chunks = gemini_event_to_openai_sse_chunks( + &parsed, + "chatcmpl-test", + "gemini-3-flash-preview", + &mut tool_call_index, + ); + let usage_chunk = chunks + .iter() + .map(|c| parse_sse_chunk(c)) + .find(|v| v.get("usage").map(|u| !u.is_null()).unwrap_or(false)) + .expect("a chunk should carry top-level usage"); + + // Gemini's prompt count already includes the cached tokens, so it passes + // through unchanged and the cached share is reported alongside it; thinking + // is billed as output but counted apart from the candidates. + assert_eq!(usage_chunk["usage"]["prompt_tokens"], 1000); + assert_eq!(usage_chunk["usage"]["prompt_tokens_details"]["cached_tokens"], 900); + assert_eq!(usage_chunk["usage"]["completion_tokens"], 120); + } + + #[test] + fn gemini_usage_chunk_counts_tool_use_prompt_tokens() { + let parsed = GeminiParsedEvent { + text: Some("Canberra".to_string()), + usage: Some(GeminiUsageMetadata { + prompt_token_count: Some(17), + candidates_token_count: Some(17), + total_token_count: Some(146), + tool_use_prompt_token_count: Some(60), + thoughts_token_count: Some(52), + ..Default::default() + }), + ..Default::default() + }; + + let mut tool_call_index = 0; + let chunks = gemini_event_to_openai_sse_chunks( + &parsed, + "chatcmpl-test", + "gemini-2.5-flash", + &mut tool_call_index, + ); + let usage_chunk = chunks + .iter() + .map(|c| parse_sse_chunk(c)) + .find(|v| v.get("usage").map(|u| !u.is_null()).unwrap_or(false)) + .expect("a chunk should carry top-level usage"); + + assert_eq!(usage_chunk["usage"]["prompt_tokens"], 77); + assert_eq!(usage_chunk["usage"]["completion_tokens"], 69); + assert_eq!( + usage_chunk["usage"]["prompt_tokens"].as_i64().unwrap() + + usage_chunk["usage"]["completion_tokens"].as_i64().unwrap(), + 146 + ); + } + #[test] fn gemini_streaming_usage_total_falls_back_to_prompt_plus_completion() { let parsed = GeminiParsedEvent { @@ -976,6 +1089,7 @@ mod tests { prompt_token_count: Some(5), candidates_token_count: Some(3), total_token_count: None, + ..Default::default() }), ..Default::default() }; diff --git a/backend/windmill-ai/src/ai_types.rs b/backend/windmill-ai/src/ai_types.rs index b80e237920..69c8ac4538 100644 --- a/backend/windmill-ai/src/ai_types.rs +++ b/backend/windmill-ai/src/ai_types.rs @@ -175,3 +175,57 @@ pub struct OpenAIMessage { #[serde(skip_serializing_if = "Option::is_none")] pub annotations: Option>, } + +// ============================================================================ +// Model pricing +// ============================================================================ + +/// Far above any real per-million-token rate, so a value beyond it is a unit +/// mistake rather than a price. The floor matters more: a negative rate would make +/// spend subtract, and NaN/infinity would poison every total derived from it. +pub const MAX_MODEL_RATE: f64 = 1000.0; + +/// Bound the `model_pricing` map of an AI config that is only available untyped — +/// the instance config is stored through the generic global-settings endpoint, +/// which never deserializes it into `AIConfig`, so the typed check on the +/// workspace path does not cover it. +pub fn validate_model_pricing_json(ai_config: &serde_json::Value) -> Result<(), String> { + // The container itself has to be checked too: a non-object `ai_config` persists + // here and then fails to deserialize as `AIConfig`, which drops the whole + // instance config back to its default for every workspace inheriting it. + if !ai_config.is_null() && !ai_config.is_object() { + return Err("ai_config must be an object".to_string()); + } + let pricing = match ai_config.get("model_pricing") { + None | Some(serde_json::Value::Null) => return Ok(()), + // A present-but-wrong shape must be rejected, not skipped: it would persist + // and then fail to deserialize as `AIConfig`, which silently drops the whole + // instance config back to its default for every workspace inheriting it. + Some(v) => v + .as_object() + .ok_or_else(|| "model_pricing must be an object".to_string())?, + }; + for (key, price) in pricing { + let Some(price) = price.as_object() else { + return Err(format!("Price override for {} is not an object", key)); + }; + for field in ["input", "output", "cache_read", "cache_write"] { + let Some(rate) = price.get(field) else { continue }; + let rate = rate + .as_f64() + .filter(|r| r.is_finite() && *r >= 0.0 && *r <= MAX_MODEL_RATE); + if rate.is_none() { + return Err(format!( + "Price override for {}: {} must be between 0 and {}", + key, field, MAX_MODEL_RATE + )); + } + } + for required in ["input", "output"] { + if !price.contains_key(required) { + return Err(format!("Price override for {} is missing {}", key, required)); + } + } + } + Ok(()) +} diff --git a/backend/windmill-ai/src/providers/bedrock.rs b/backend/windmill-ai/src/providers/bedrock.rs index 453607eea5..cf977f2484 100644 --- a/backend/windmill-ai/src/providers/bedrock.rs +++ b/backend/windmill-ai/src/providers/bedrock.rs @@ -660,6 +660,41 @@ fn bedrock_sse_chunks_for_event( chunks.push(Bytes::from(format!("data: {}\n\n", chunk))); } + // Usage arrives only on the trailing Metadata event, and only this converter + // reaches the chat: without a chunk for it a Bedrock chat reports no tokens at + // all. Bedrock counts cache reads and writes apart from `inputTokens`, while the + // OpenAI shape the client parses treats `prompt_tokens` as the whole input, so + // they are folded in here and split back out through `prompt_tokens_details`. + if let aws_sdk_bedrockruntime::types::ConverseStreamOutput::Metadata(metadata) = event { + if let Some(token_usage) = metadata.usage() { + let cache_read = token_usage.cache_read_input_tokens().unwrap_or(0); + let cache_write = token_usage.cache_write_input_tokens().unwrap_or(0); + let prompt_tokens = token_usage + .input_tokens() + .saturating_add(cache_read) + .saturating_add(cache_write); + + let chunk = serde_json::json!({ + "id": state.id, + "object": "chat.completion.chunk", + "created": state.created, + "model": state.model, + "choices": [], + "usage": { + "prompt_tokens": prompt_tokens, + "completion_tokens": token_usage.output_tokens(), + "total_tokens": token_usage.total_tokens(), + "prompt_tokens_details": { + "cached_tokens": cache_read, + "cache_write_tokens": cache_write + } + } + }); + + chunks.push(Bytes::from(format!("data: {}\n\n", chunk))); + } + } + chunks } @@ -1190,6 +1225,43 @@ mod tests { serde_json::from_str(payload).expect("chunk should contain JSON") } + #[test] + fn metadata_event_emits_usage_chunk_with_cache_split() { + let mut state = BedrockSseStreamState::new("id".to_string(), "model".to_string(), 0); + let event = ConverseStreamOutput::Metadata( + aws_sdk_bedrockruntime::types::ConverseStreamMetadataEvent::builder() + .usage( + aws_sdk_bedrockruntime::types::TokenUsage::builder() + .input_tokens(10) + .output_tokens(7) + .total_tokens(1017) + .cache_read_input_tokens(900) + .cache_write_input_tokens(100) + .build() + .expect("usage"), + ) + .build(), + ); + + let chunks = bedrock_sse_chunks_for_event(&event, &mut state); + let usage = chunks + .iter() + .map(sse_json) + .find(|v| v.get("usage").map(|u| !u.is_null()).unwrap_or(false)) + .expect("the metadata event should carry usage"); + + // Bedrock reports cache reads and writes apart from `inputTokens`; the OpenAI + // shape the client parses treats `prompt_tokens` as the whole input, and + // recovers the uncached share by subtracting the details back out. + assert_eq!(usage["usage"]["prompt_tokens"], 1010); + assert_eq!(usage["usage"]["completion_tokens"], 7); + assert_eq!(usage["usage"]["prompt_tokens_details"]["cached_tokens"], 900); + assert_eq!( + usage["usage"]["prompt_tokens_details"]["cache_write_tokens"], + 100 + ); + } + #[test] fn determine_auth_config_prioritizes_bearer_token() { let config = determine_auth_config( diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index a9bc601591..cead2f4b27 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -891,6 +891,13 @@ async fn run_setting_pre_write_hook( value: &serde_json::Value, ) -> error::Result<()> { match key { + // The instance AI config is written as an untyped blob through this generic + // endpoint, so it never passes the typed check the workspace handler applies. + // Rates that reach a cost total unbounded would make it negative or infinite. + AI_CONFIG_SETTING => { + windmill_ai::ai_types::validate_model_pricing_json(value) + .map_err(error::Error::BadRequest)?; + } AUTOMATE_USERNAME_CREATION_SETTING => { if value.as_bool().unwrap_or(false) { generate_instance_username_for_all_users(db) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 8986bd7f5c..d2f61a5630 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -11986,6 +11986,73 @@ paths: schema: type: string + /w/{workspace}/ai/usage: + post: + summary: record AI token usage for the calling user + operationId: recordAiUsage + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - events + properties: + events: + type: array + items: + $ref: "#/components/schemas/AITokenUsageEvent" + responses: + "204": + description: usage recorded + get: + summary: list aggregated AI token usage + operationId: listAiUsage + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: days + in: query + schema: + type: integer + minimum: 1 + maximum: 365 + - name: group_by + in: query + schema: + type: string + enum: [day, user, model] + - name: scope + in: query + description: workspace-wide usage (admin only) or the calling user's own + schema: + type: string + enum: [workspace, self] + responses: + "200": + description: usage buckets + content: + application/json: + schema: + type: object + required: + - buckets + - truncated + properties: + buckets: + type: array + items: + $ref: "#/components/schemas/AITokenUsageBucket" + truncated: + type: boolean + description: more buckets matched than were returned, so summing them under-reports + /w/{workspace}/ai_skills/list: get: summary: list the workspace AI chat skills (name + description only) @@ -26300,6 +26367,92 @@ components: type: integer minimum: 1 maximum: 2000000 + model_pricing: + type: object + additionalProperties: + $ref: "#/components/schemas/ModelPriceOverride" + + ModelPriceOverride: + type: object + description: negotiated rates in USD per million tokens, keyed `provider:model` + properties: + input: + type: number + minimum: 0 + maximum: 1000 + output: + type: number + minimum: 0 + maximum: 1000 + cache_read: + type: number + minimum: 0 + maximum: 1000 + cache_write: + type: number + minimum: 0 + maximum: 1000 + required: + - input + - output + + AITokenUsageEvent: + type: object + properties: + provider: + $ref: "#/components/schemas/AIProvider" + model: + type: string + session_id: + type: string + input_tokens: + type: integer + cache_read_tokens: + type: integer + cache_write_tokens: + type: integer + output_tokens: + type: integer + reported_cost_nano_usd: + type: integer + description: only set by providers that bill back an exact figure + requests: + type: integer + required: + - provider + - model + + AITokenUsageBucket: + type: object + properties: + key: + type: string + description: the grouped dimension's value; empty when grouping by model + provider: + type: string + model: + type: string + input_tokens: + type: integer + cache_read_tokens: + type: integer + cache_write_tokens: + type: integer + output_tokens: + type: integer + reported_cost_nano_usd: + type: integer + requests: + type: integer + required: + - key + - provider + - model + - input_tokens + - cache_read_tokens + - cache_write_tokens + - output_tokens + - requests InstanceAIProviderSummary: type: object diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index efc2ba7e60..1a5f9dba87 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -3,11 +3,16 @@ use crate::utils::check_scopes; #[cfg(feature = "bedrock")] use axum::routing::get; -#[cfg(feature = "bedrock")] use axum::Json; -use axum::{body::Bytes, extract::Path, response::IntoResponse, routing::post, Extension, Router}; +use axum::{ + body::Bytes, + extract::{DefaultBodyLimit, Path, Query}, + response::IntoResponse, + routing::post, + Extension, Router, +}; use futures::StreamExt; -use http::{HeaderMap, Method}; +use http::{HeaderMap, Method, StatusCode}; use quick_cache::sync::Cache; use reqwest::{Client, RequestBuilder}; use serde::{Deserialize, Serialize}; @@ -18,6 +23,7 @@ use windmill_ai::ai_cache::current_instance_ai_config_revision; use windmill_ai::ai_providers::{ empty_string_as_none, AIPlatform, AIProvider, ProviderConfig, ProviderModel, }; +use windmill_ai::ai_types::MAX_MODEL_RATE; use windmill_ai::credentials::ProviderCredentials; #[cfg(feature = "bedrock")] use windmill_ai::providers::bedrock::{ @@ -37,7 +43,7 @@ use windmill_ai::proxy::{ use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::db::UserDB; use windmill_common::error::{to_anyhow, Error, Result}; -use windmill_common::utils::configure_client; +use windmill_common::utils::{configure_client, require_admin}; use windmill_common::variables::{get_variable_or_self, get_variable_or_self_as}; // AI timeout configuration constants @@ -417,9 +423,54 @@ pub struct AIConfig { pub custom_prompts: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub max_tokens_per_model: Option>, + /// Per-model price overrides, keyed `provider:model` like `max_tokens_per_model`. + /// Only models whose rates differ from the built-in table are stored. + #[serde(skip_serializing_if = "Option::is_none")] + pub model_pricing: Option>, +} + +/// Negotiated rates in USD per million tokens. An unset cache rate is read as the +/// provider's own multiple of the input rate where the model has a published one, +/// and as the input rate itself where it does not — an unstated discount is never +/// filled in from another vendor's. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ModelPriceOverride { + pub input: f64, + pub output: f64, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_read: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_write: Option, +} + +impl ModelPriceOverride { + pub fn validate(&self, key: &str) -> Result<()> { + for (field, rate) in [ + ("input", Some(self.input)), + ("output", Some(self.output)), + ("cache_read", self.cache_read), + ("cache_write", self.cache_write), + ] { + let Some(rate) = rate else { continue }; + if !rate.is_finite() || rate < 0.0 || rate > MAX_MODEL_RATE { + return Err(Error::BadRequest(format!( + "Price override for {}: {} must be between 0 and {}", + key, field, MAX_MODEL_RATE + ))); + } + } + Ok(()) + } } impl AIConfig { + pub fn validate_model_pricing(&self) -> Result<()> { + for (key, price) in self.model_pricing.iter().flatten() { + price.validate(key)?; + } + Ok(()) + } + pub fn has_providers(&self) -> bool { self.providers .as_ref() @@ -432,7 +483,18 @@ pub fn global_service() -> Router { } pub fn workspaced_service() -> Router { - let router = Router::new().route("/proxy/{*ai}", post(proxy).get(proxy)); + let router = Router::new() + .route("/proxy/{*ai}", post(proxy).get(proxy)) + .route( + "/usage", + post(record_ai_usage) + .get(list_ai_usage) + // The handler caps how many events it *stores*, but Json deserializes + // the whole array first — without a body limit an authenticated member + // could make the server allocate and parse an arbitrarily large one. + // Sized well above a full batch of the shape below. + .layer(DefaultBodyLimit::max(AI_USAGE_BODY_LIMIT)), + ); #[cfg(feature = "bedrock")] let router = router.route("/check_bedrock_credentials", get(check_bedrock_credentials)); @@ -440,6 +502,265 @@ pub fn workspaced_service() -> Router { router } +/// One provider request's worth of tokens, as counted by the chat client. +#[derive(Deserialize)] +struct AIUsageEvent { + provider: String, + model: String, + #[serde(default)] + session_id: String, + #[serde(default)] + input_tokens: i64, + #[serde(default)] + cache_read_tokens: i64, + #[serde(default)] + cache_write_tokens: i64, + #[serde(default)] + output_tokens: i64, + /// Only the providers that bill back an exact figure set this. + #[serde(default)] + reported_cost_nano_usd: Option, + #[serde(default)] + requests: Option, +} + +#[derive(Deserialize)] +struct RecordAIUsagePayload { + events: Vec, +} + +const MAX_AI_USAGE_EVENTS: usize = 50; +/// 64 KiB — a 50-event batch is a few kB even with the longest model ids. +const AI_USAGE_BODY_LIMIT: usize = 64 * 1024; +/// Well above any single conversation and far below an i64 overflow, so a client +/// bug caps out at one absurd row instead of poisoning the running total. +const MAX_TOKENS_PER_EVENT: i64 = 100_000_000; +/// $1000 in nano-USD. +const MAX_REPORTED_COST_PER_EVENT: i64 = 1_000_000_000_000; + +/// Model ids carry vendor prefixes and variant suffixes (`anthropic/claude-opus-5:thinking`), +/// so the shape check is looser than an identifier but still excludes whitespace and +/// anything that would not be a model id. +fn is_model_shaped(s: &str, max_len: usize) -> bool { + !s.is_empty() + && s.len() <= max_len + && s.chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | ':' | '.' | '/' | '~')) +} + +/// Accumulate one workspace's AI token spend. Values are clamped and the caller's +/// email comes from the session, never the payload — the client is trusted to +/// report its own usage, not to attribute it to someone else. +async fn record_ai_usage( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(payload): Json, +) -> Result { + // Pre-sum duplicate keys: two rows hitting the same conflict target in a single + // INSERT error out ("cannot affect row a second time"). + let mut agg: HashMap<(String, String, String), AIUsageTotals> = HashMap::new(); + for e in payload.events.into_iter().take(MAX_AI_USAGE_EVENTS) { + if AIProvider::try_from(e.provider.as_str()).is_err() + || !is_model_shaped(&e.model, 255) + || !(e.session_id.is_empty() || is_model_shaped(&e.session_id, 50)) + { + continue; + } + let totals = agg + .entry((e.provider, e.model, e.session_id)) + .or_insert_with(AIUsageTotals::default); + totals.input += e.input_tokens.clamp(0, MAX_TOKENS_PER_EVENT); + totals.cache_read += e.cache_read_tokens.clamp(0, MAX_TOKENS_PER_EVENT); + totals.cache_write += e.cache_write_tokens.clamp(0, MAX_TOKENS_PER_EVENT); + totals.output += e.output_tokens.clamp(0, MAX_TOKENS_PER_EVENT); + totals.requests += e.requests.unwrap_or(1).clamp(0, MAX_AI_USAGE_EVENTS as i64); + if let Some(cost) = e.reported_cost_nano_usd { + totals.reported_cost = Some( + totals.reported_cost.unwrap_or(0) + cost.clamp(0, MAX_REPORTED_COST_PER_EVENT), + ); + } + } + if agg.is_empty() { + return Ok(StatusCode::NO_CONTENT); + } + + let mut providers = Vec::with_capacity(agg.len()); + let mut models = Vec::with_capacity(agg.len()); + let mut session_ids = Vec::with_capacity(agg.len()); + let mut inputs = Vec::with_capacity(agg.len()); + let mut cache_reads = Vec::with_capacity(agg.len()); + let mut cache_writes = Vec::with_capacity(agg.len()); + let mut outputs = Vec::with_capacity(agg.len()); + let mut reported_costs: Vec> = Vec::with_capacity(agg.len()); + let mut requests = Vec::with_capacity(agg.len()); + for ((provider, model, session_id), totals) in agg { + providers.push(provider); + models.push(model); + session_ids.push(session_id); + inputs.push(totals.input); + cache_reads.push(totals.cache_read); + cache_writes.push(totals.cache_write); + outputs.push(totals.output); + reported_costs.push(totals.reported_cost); + requests.push(totals.requests); + } + + sqlx::query!( + "INSERT INTO ai_token_usage (workspace_id, email, provider, model, session_id, \ + input_tokens, cache_read_tokens, cache_write_tokens, output_tokens, \ + reported_cost_nano_usd, requests) + SELECT $1, $2, * FROM UNNEST($3::text[], $4::text[], $5::text[], $6::bigint[], \ + $7::bigint[], $8::bigint[], $9::bigint[], $10::bigint[], $11::bigint[]) + ON CONFLICT (workspace_id, day, email, provider, model, session_id) + DO UPDATE SET + input_tokens = ai_token_usage.input_tokens + EXCLUDED.input_tokens, + cache_read_tokens = ai_token_usage.cache_read_tokens + EXCLUDED.cache_read_tokens, + cache_write_tokens = ai_token_usage.cache_write_tokens + EXCLUDED.cache_write_tokens, + output_tokens = ai_token_usage.output_tokens + EXCLUDED.output_tokens, + reported_cost_nano_usd = CASE + WHEN EXCLUDED.reported_cost_nano_usd IS NULL + THEN ai_token_usage.reported_cost_nano_usd + ELSE COALESCE(ai_token_usage.reported_cost_nano_usd, 0) + + EXCLUDED.reported_cost_nano_usd + END, + requests = ai_token_usage.requests + EXCLUDED.requests, + updated_at = now()", + &w_id, + &authed.email, + &providers, + &models, + &session_ids, + &inputs, + &cache_reads, + &cache_writes, + &outputs, + &reported_costs as &[Option], + &requests + ) + .execute(&db) + .await?; + + Ok(StatusCode::NO_CONTENT) +} + +#[derive(Default)] +struct AIUsageTotals { + input: i64, + cache_read: i64, + cache_write: i64, + output: i64, + reported_cost: Option, + requests: i64, +} + +#[derive(Deserialize)] +struct ListAIUsageQuery { + days: Option, + group_by: Option, + scope: Option, +} + +/// A bucket always carries its provider and model: the caller prices it from a +/// per-model rate table, which a bucket spanning several models could not be +/// resolved against. +#[derive(Serialize)] +struct AITokenUsageBucket { + key: String, + provider: String, + model: String, + input_tokens: i64, + cache_read_tokens: i64, + cache_write_tokens: i64, + output_tokens: i64, + reported_cost_nano_usd: Option, + requests: i64, +} + +/// Grouping by day over a long range, or by model across many models, can produce +/// more buckets than a table is worth rendering, so the listing is capped. +/// `truncated` says so explicitly — a caller that sums the rows into a total must be +/// able to tell that the total is partial rather than silently under-reporting spend. +#[derive(Serialize)] +struct AITokenUsageListing { + buckets: Vec, + truncated: bool, +} + +const AI_USAGE_MAX_BUCKETS: i64 = 1000; + +async fn list_ai_usage( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Query(query): Query, +) -> Result> { + // Reading the whole workspace's spend is an admin view; reading your own is + // not, so a member can see what they are costing without being shown their + // colleagues'. The filter is the session's email, never a parameter. + let own_email = match query.scope.as_deref().unwrap_or("workspace") { + "workspace" => { + require_admin(authed.is_admin, &authed.username)?; + None + } + "self" => Some(authed.email.clone()), + scope => return Err(Error::BadRequest(format!("Unsupported scope: {}", scope))), + }; + + let days = query.days.unwrap_or(30).clamp(1, 365); + let group_by = query.group_by.as_deref().unwrap_or("day"); + // No `session`: a session is identified by a client-generated id whose name + // lives only in the browser that made it, so a bucket keyed on one is a label + // nobody can resolve. `session_id` is still stored, at the grain the client + // batches on, should sessions ever gain a server-side name. + if !matches!(group_by, "day" | "user" | "model") { + return Err(Error::BadRequest(format!( + "Unsupported group_by: {}", + group_by + ))); + } + + // Fetch one past the cap to detect truncation. Ordering is by token volume, not + // by cost: rates are applied by the caller, so this query cannot know what a + // bucket cost. Volume is the closest proxy available here, and the caller is told + // the listing was capped rather than being left to sum a partial set silently. + let mut rows = sqlx::query_as!( + AITokenUsageBucket, + r#"SELECT + (CASE $3::text + WHEN 'day' THEN day::text + WHEN 'user' THEN email + ELSE '' + END) AS "key!", + provider AS "provider!", + model AS "model!", + SUM(input_tokens)::bigint AS "input_tokens!", + SUM(cache_read_tokens)::bigint AS "cache_read_tokens!", + SUM(cache_write_tokens)::bigint AS "cache_write_tokens!", + SUM(output_tokens)::bigint AS "output_tokens!", + SUM(reported_cost_nano_usd)::bigint AS "reported_cost_nano_usd", + SUM(requests)::bigint AS "requests!" + FROM ai_token_usage + WHERE workspace_id = $1 AND day > CURRENT_DATE - $2::int + AND ($5::text IS NULL OR email = $5) + GROUP BY 1, provider, model + ORDER BY SUM(input_tokens + cache_read_tokens + cache_write_tokens + output_tokens) DESC + LIMIT $4"#, + &w_id, + days, + group_by, + AI_USAGE_MAX_BUCKETS + 1, + own_email.as_deref() + ) + .fetch_all(&db) + .await?; + + let truncated = rows.len() as i64 > AI_USAGE_MAX_BUCKETS; + rows.truncate(AI_USAGE_MAX_BUCKETS as usize); + + Ok(Json(AITokenUsageListing { buckets: rows, truncated })) +} + /// Check if AWS Bedrock credentials are available from environment variables. #[cfg(feature = "bedrock")] async fn check_bedrock_credentials( diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 23ce68fb80..01eb32b7f6 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -108,6 +108,8 @@ async fn edit_copilot_config( } } + ai_config.validate_model_pricing()?; + let mut tx = db.begin().await?; sqlx::query!( diff --git a/frontend/src/lib/aiStore.ts b/frontend/src/lib/aiStore.ts index 1e8fb6ac61..0955f68ee5 100644 --- a/frontend/src/lib/aiStore.ts +++ b/frontend/src/lib/aiStore.ts @@ -3,7 +3,12 @@ // import aiStore back, and such a cycle crashes the app once the bundler splits it across // chunks (docs/frontend-import-cycles.md; the build fails on the chunk cycle, not on this). import { writable, get } from 'svelte/store' -import { type AIProviderModel, type AIProvider, type AIConfig } from './gen' +import { + type AIProviderModel, + type AIProvider, + type AIConfig, + type ModelPriceOverride +} from './gen' import { aiUserDisabled, COPILOT_SESSION_MODEL_SETTING_NAME, @@ -41,6 +46,8 @@ export const copilotInfo = writable<{ aiModels: AIProviderModel[] customPrompts?: Record maxTokensPerModel?: Record + /** Negotiated rates per `provider:model`, overriding the built-in price table. */ + modelPricing?: Record webSearchEnabledProviders?: Partial> }>({ enabled: false, @@ -50,6 +57,7 @@ export const copilotInfo = writable<{ aiModels: [], customPrompts: {}, maxTokensPerModel: {}, + modelPricing: {}, webSearchEnabledProviders: {} }) @@ -124,6 +132,7 @@ export function setCopilotInfo(aiConfig: AIConfig) { aiModels: aiModels, customPrompts: aiConfig.custom_prompts ?? {}, maxTokensPerModel: aiConfig.max_tokens_per_model ?? {}, + modelPricing: aiConfig.model_pricing ?? {}, webSearchEnabledProviders }) } else { @@ -137,6 +146,7 @@ export function setCopilotInfo(aiConfig: AIConfig) { aiModels: [], customPrompts: {}, maxTokensPerModel: {}, + modelPricing: {}, webSearchEnabledProviders: {} }) } diff --git a/frontend/src/lib/components/UserSettings.svelte b/frontend/src/lib/components/UserSettings.svelte index d3956f7f12..d5811e737b 100644 --- a/frontend/src/lib/components/UserSettings.svelte +++ b/frontend/src/lib/components/UserSettings.svelte @@ -9,6 +9,8 @@ import { createEventDispatcher } from 'svelte' import UserInfoSettings from './settings/UserInfoSettings.svelte' import AIUserSettings from './settings/AIUserSettings.svelte' + import AiUsagePanel from './workspaceSettings/AiUsagePanel.svelte' + import { copilotInfo, copilotWorkspace } from '$lib/aiStore' import { getDarkModeVariant, setDarkModeVariant, @@ -105,6 +107,17 @@ + + {#if $copilotWorkspace} + + {/if} {/if}
diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index e08b9ba2a3..f68501b293 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -58,7 +58,7 @@ import { import { dfs } from '$lib/components/flows/previousResults' import { SvelteMap, SvelteSet } from 'svelte/reactivity' import { createLongHash } from '$lib/editorLangUtils' -import type { UserDraftItemKind } from '$lib/gen' +import type { AIProvider, UserDraftItemKind } from '$lib/gen' import { maskKey } from '$lib/components/sessions/modifiedItemsMask' import { getStringError } from './utils' import { type PasteAttachment } from './pasteTokens' @@ -101,7 +101,12 @@ import type AIChatInput from './AIChatInput.svelte' import { prepareApiSystemMessage, prepareApiUserMessage } from './api/core' import { runChatLoop, truncateToToolPairedPrefix } from './chatLoop' import { sanitizeToolCallArguments } from './toolCallArguments' -import { normalizeContextUsage } from './tokenUsage' +import { + billedTokens, + normalizeContextUsage, + type ChatTokenUsage +} from './tokenUsage' +import { logAiUsage } from '$lib/utils/aiUsageReporter' import type { ReviewChangesOpts } from './monaco-adapter' import { getCurrentModel, @@ -709,6 +714,39 @@ export class AIChatManager { await this.#persistModifiedItems() } + /** Report one completed provider response's tokens to the workspace usage view. + * Called per response rather than per turn: a tool loop makes several, each + * separately billed, and a turn that fails partway through has still spent + * everything up to that point. + * + * Only token counts leave the browser — rates are applied when the usage is + * read, so a corrected price also corrects everything already recorded. */ + private recordUsage( + usage: ChatTokenUsage, + provider: AIProvider, + model: string, + workspace: string | undefined + ) { + // A provider that reports no usage still yields an all-zero report. Recording + // it would add a $0 row to the usage view, claiming the request cost nothing + // rather than that it went uncounted. + if (usage.total === 0 && usage.prompt === 0 && usage.completion === 0) { + return + } + const tokens = billedTokens(usage) + logAiUsage({ + provider, + model, + sessionId: this.sessionId, + inputTokens: tokens.input, + cacheReadTokens: tokens.cacheRead, + cacheWriteTokens: tokens.cacheWrite, + outputTokens: tokens.output, + costUsd: usage.cost, + workspace + }) + } + // Serialized, snapshot-at-write-time persistence: two rapid dock actions // would otherwise race their saveChat writes, and the earlier (staler) // snapshot could land last — dropping the later mutation until the next @@ -2458,6 +2496,11 @@ export class AIChatManager { // on each iteration. This is critical for changeModeTool (Navigator → Script/Flow) // which reassigns this.tools, this.helpers, this.systemMessage mid-loop. const self = this + // Pinned for the whole turn, like the `workspace` the loop routes through: + // the global chat's operating workspace follows workspaceStore, so a switch + // while a response streams would bill it to the workspace the user landed + // on rather than the one whose credentials and proxy served it. + const usageWorkspace = this.operatingWorkspace const result = await runChatLoop({ messages, addedMessages, @@ -2535,6 +2578,14 @@ export class AIChatManager { } return undefined }, + onUsage: (usage, modelProvider) => { + // Accounting must never take a turn down with it. + try { + this.recordUsage(usage, modelProvider.provider, modelProvider.model, usageWorkspace) + } catch (e) { + console.error('Failed to record AI usage', e) + } + }, onBeforeIteration: async (tools, _helpers, modelProvider) => { this.lastIterationModel = modelProvider for (const tool of tools) { diff --git a/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte b/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte index a307fad255..ffa06ca5df 100644 --- a/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte +++ b/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte @@ -4,6 +4,7 @@ import { getAiChatManager } from './aiChatManagerContext' import { AIMode } from './AIChatManager.svelte' import Tooltip from '$lib/components/meltComponents/Tooltip.svelte' + import { formatTokenCount } from './tokenUsage' const aiChatManager = getAiChatManager() @@ -45,16 +46,6 @@ ? 'bg-amber-500' : 'bg-surface-accent-primary' ) - - function formatTokenCount(tokens: number): string { - if (tokens >= 1_000_000) { - return `${(tokens / 1_000_000).toFixed(1).replace(/\.0$/, '')}M` - } - if (tokens >= 1000) { - return `${Math.round(tokens / 1000)}k` - } - return `${tokens}` - } {#if visible} diff --git a/frontend/src/lib/components/copilot/chat/anthropic.ts b/frontend/src/lib/components/copilot/chat/anthropic.ts index 52074a84c9..8d38f10916 100644 --- a/frontend/src/lib/components/copilot/chat/anthropic.ts +++ b/frontend/src/lib/components/copilot/chat/anthropic.ts @@ -195,7 +195,7 @@ export async function parseAnthropicCompletion( tools: Tool[], helpers: any, abortController?: AbortController, - options?: { workspace?: string } + options?: { workspace?: string; onTokenUsage?: (usage: ChatTokenUsage) => void } ): Promise { let toolCallsToProcess: ChatCompletionMessageFunctionToolCall[] = [] let error = null @@ -417,6 +417,7 @@ export async function parseAnthropicCompletion( const finalMessage = await completion.finalMessage() const tokenUsage = anthropicUsageToChatTokenUsage(finalMessage.usage) + options?.onTokenUsage?.(tokenUsage) // Process tool calls if any if (toolCallsToProcess.length > 0) { diff --git a/frontend/src/lib/components/copilot/chat/chatLoop.test.ts b/frontend/src/lib/components/copilot/chat/chatLoop.test.ts index 2c4eb466e9..a010f1369a 100644 --- a/frontend/src/lib/components/copilot/chat/chatLoop.test.ts +++ b/frontend/src/lib/components/copilot/chat/chatLoop.test.ts @@ -518,7 +518,7 @@ describe('runChatLoop lastIterationUsage', () => { expect(result.lastIterationUsage).toEqual({ prompt: 1200, completion: 80, total: 1280 }) // the aggregate keeps summing across iterations - expect(result.tokenUsage).toEqual({ prompt: 2200, completion: 130, total: 2330 }) + expect(result.tokenUsage).toMatchObject({ prompt: 2200, completion: 130, total: 2330 }) }) it('ignores empty usage reports and returns null when none are real', async () => { diff --git a/frontend/src/lib/components/copilot/chat/chatLoop.ts b/frontend/src/lib/components/copilot/chat/chatLoop.ts index cc963e4b25..91dd3ab29c 100644 --- a/frontend/src/lib/components/copilot/chat/chatLoop.ts +++ b/frontend/src/lib/components/copilot/chat/chatLoop.ts @@ -77,11 +77,16 @@ export interface ChatLoopConfig { helpers: any, modelProvider: ReasoningProviderModel ) => Promise + /** Fired for each completed provider response, before the loop continues. The + * loop can fail or be aborted at any iteration, so spend has to be handed over + * as it happens — a callback only at the end would discard everything the + * earlier iterations were already billed for. */ + onUsage?: (usage: ChatTokenUsage, modelProvider: ReasoningProviderModel) => void } export interface ChatLoopResult { addedMessages: ChatCompletionMessageParam[] - /** Sum of usage across all loop iterations (suitable for cost accounting). */ + /** Sum of usage across all loop iterations. */ tokenUsage: ChatTokenUsage lastIterationUsage: ChatTokenUsage | null hitMaxIterations: boolean @@ -328,6 +333,20 @@ export async function runChatLoop(config: ChatLoopConfig): Promise { + if (usage && iterationModel) { + config.onUsage?.(usage, iterationModel) + } + } const trackUsage = (usage: ChatTokenUsage | null | undefined) => { tokenUsage = addChatTokenUsage(tokenUsage, usage) @@ -351,6 +370,7 @@ export async function runChatLoop(config: ChatLoopConfig): Promise t.def) - const parseOptions = { workspace, provider: modelProvider.provider } + const parseOptions = { + workspace, + provider: modelProvider.provider, + onTokenUsage: reportUsage + } if (isOpenAI) { const reasoningSummaryCacheKey = getReasoningSummaryCacheKey(workspace, modelProvider) diff --git a/frontend/src/lib/components/copilot/chat/openai-responses.ts b/frontend/src/lib/components/copilot/chat/openai-responses.ts index 19efff2cd4..4118eb492d 100644 --- a/frontend/src/lib/components/copilot/chat/openai-responses.ts +++ b/frontend/src/lib/components/copilot/chat/openai-responses.ts @@ -392,7 +392,7 @@ export async function parseOpenAIResponsesCompletion( addedMessages: ChatCompletionMessageParam[], tools: Tool[], helpers: any, - options?: { workspace?: string } + options?: { workspace?: string; onTokenUsage?: (usage: ChatTokenUsage) => void } ): Promise { let toolCallsToProcess: ChatCompletionMessageFunctionToolCall[] = [] let error: OpenAIError | ResponseErrorEvent | null = null @@ -566,6 +566,7 @@ export async function parseOpenAIResponsesCompletion( const finalResponse = await runner.finalResponse() const tokenUsage = openAIResponsesUsageToChatTokenUsage(finalResponse.usage) + options?.onTokenUsage?.(tokenUsage) for (const item of finalResponse.output ?? []) { if (item.type === 'web_search_call' && !surfacedWebSearchCalls.has(item.id)) { diff --git a/frontend/src/lib/components/copilot/chat/tokenUsage.test.ts b/frontend/src/lib/components/copilot/chat/tokenUsage.test.ts new file mode 100644 index 0000000000..f23064f5a3 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/tokenUsage.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from 'vitest' +import { + anthropicUsageToChatTokenUsage, + billedTokens, + openAICompletionsUsageToChatTokenUsage +} from './tokenUsage' + +// The two providers report cache tokens under opposite conventions — Anthropic's +// input_tokens excludes them, OpenAI's includes them. Both are normalized so that +// `prompt` is the whole input, which is what makes `prompt - cached` the uncached +// share. Getting this backwards double-counts (or loses) the cached prefix, which +// is most of a long chat's input. +describe('billedTokens', () => { + it('derives uncached input under the Anthropic convention', () => { + const usage = anthropicUsageToChatTokenUsage({ + input_tokens: 1000, + output_tokens: 200, + cache_creation_input_tokens: 300, + cache_read_input_tokens: 5000 + }) + expect(usage.prompt).toBe(6300) + expect(billedTokens(usage)).toEqual({ + input: 1000, + cacheRead: 5000, + cacheWrite: 300, + output: 200 + }) + }) + + it('derives uncached input under the OpenAI convention', () => { + const usage = openAICompletionsUsageToChatTokenUsage({ + prompt_tokens: 6000, + completion_tokens: 200, + prompt_tokens_details: { cached_tokens: 5000 } + }) + expect(usage.prompt).toBe(6000) + expect(billedTokens(usage)).toEqual({ + input: 1000, + cacheRead: 5000, + cacheWrite: 0, + output: 200 + }) + }) + + // OpenRouter extends the OpenAI shape with cache-creation tokens, counted + // inside prompt_tokens like the reads beside them. Missing the field bills + // them as uncached input. + it('splits out OpenRouter cache-creation tokens', () => { + const usage = openAICompletionsUsageToChatTokenUsage({ + prompt_tokens: 6300, + completion_tokens: 200, + prompt_tokens_details: { cached_tokens: 5000, cache_write_tokens: 300 } + }) + expect(billedTokens(usage)).toEqual({ + input: 1000, + cacheRead: 5000, + cacheWrite: 300, + output: 200 + }) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/tokenUsage.ts b/frontend/src/lib/components/copilot/chat/tokenUsage.ts index 0091e11227..06ef5e71d8 100644 --- a/frontend/src/lib/components/copilot/chat/tokenUsage.ts +++ b/frontend/src/lib/components/copilot/chat/tokenUsage.ts @@ -1,7 +1,19 @@ +import type { PricedTokens } from '../modelPricing' + export interface ChatTokenUsage { prompt: number completion: number total: number + /** + * Subsets of `prompt`, split out because they are billed at different rates + * (a cached read is a fraction of an uncached one). `prompt` stays the whole + * input so the context gauge keeps measuring the whole request; uncached + * input is `prompt - cacheRead - cacheWrite`. + */ + cacheRead: number + cacheWrite: number + /** Cost in USD as billed, for the providers that report one. */ + cost?: number } /** @@ -28,7 +40,7 @@ export function normalizeContextUsage( } export function emptyChatTokenUsage(): ChatTokenUsage { - return { prompt: 0, completion: 0, total: 0 } + return { prompt: 0, completion: 0, total: 0, cacheRead: 0, cacheWrite: 0 } } export function addChatTokenUsage( @@ -39,10 +51,48 @@ export function addChatTokenUsage( return total } + const cost = + total.cost === undefined && usage.cost === undefined + ? undefined + : (total.cost ?? 0) + (usage.cost ?? 0) + return { prompt: total.prompt + usage.prompt, completion: total.completion + usage.completion, - total: total.total + usage.total + total: total.total + usage.total, + // `?? 0`: the cache split is newer than the field it lives on, so a usage + // object read back from storage may predate it. + cacheRead: (total.cacheRead ?? 0) + (usage.cacheRead ?? 0), + cacheWrite: (total.cacheWrite ?? 0) + (usage.cacheWrite ?? 0), + ...(cost === undefined ? {} : { cost }) + } +} + +/** Compact token count for readouts and tables (`1.2M`, `34k`, `567`). */ +export function formatTokenCount(tokens: number): string { + if (tokens >= 1_000_000) { + return `${(tokens / 1_000_000).toFixed(1).replace(/\.0$/, '')}M` + } + if (tokens >= 1000) { + return `${Math.round(tokens / 1000)}k` + } + return `${tokens}` +} + +/** + * Split a usage report into the four separately-billed token classes. `prompt` + * counts the whole input, so the uncached share is whatever the cached classes + * do not account for — which holds for both provider conventions below + * (Anthropic adds its cache counts into `prompt`, OpenAI's already includes them). + */ +export function billedTokens(usage: ChatTokenUsage): PricedTokens { + const cacheRead = usage.cacheRead ?? 0 + const cacheWrite = usage.cacheWrite ?? 0 + return { + input: Math.max(0, usage.prompt - cacheRead - cacheWrite), + cacheRead, + cacheWrite, + output: usage.completion } } @@ -57,16 +107,17 @@ export function anthropicUsageToChatTokenUsage( | null | undefined ): ChatTokenUsage { - const prompt = - (usage?.input_tokens ?? 0) + - (usage?.cache_creation_input_tokens ?? 0) + - (usage?.cache_read_input_tokens ?? 0) + const cacheWrite = usage?.cache_creation_input_tokens ?? 0 + const cacheRead = usage?.cache_read_input_tokens ?? 0 + const prompt = (usage?.input_tokens ?? 0) + cacheWrite + cacheRead const completion = usage?.output_tokens ?? 0 return { prompt, completion, - total: prompt + completion + total: prompt + completion, + cacheRead, + cacheWrite } } @@ -89,7 +140,11 @@ export function openAIResponsesUsageToChatTokenUsage( return { prompt, completion, - total: usage?.total_tokens ?? prompt + completion + total: usage?.total_tokens ?? prompt + completion, + cacheRead: usage?.input_tokens_details?.cached_tokens ?? 0, + // Automatic caching: nothing is billed for populating it, and no usage + // field reports it either. + cacheWrite: 0 } } @@ -101,7 +156,16 @@ export function openAICompletionsUsageToChatTokenUsage( prompt_tokens?: number | null completion_tokens?: number | null total_tokens?: number | null - prompt_tokens_details?: { cached_tokens?: number | null } | null + prompt_tokens_details?: { + cached_tokens?: number | null + /** Cache creation, reported by the providers that bill for it: OpenRouter + * passes Anthropic's through, and the Bedrock proxy folds + * `cacheWriteInputTokens` in here. OpenAI, whose caching is automatic and + * unbilled, reports no such field. */ + cache_write_tokens?: number | null + } | null + /** OpenRouter reports what it actually charged when the request opts in. */ + cost?: number | null } | null | undefined @@ -112,6 +176,9 @@ export function openAICompletionsUsageToChatTokenUsage( return { prompt, completion, - total: usage?.total_tokens ?? prompt + completion + total: usage?.total_tokens ?? prompt + completion, + cacheRead: usage?.prompt_tokens_details?.cached_tokens ?? 0, + cacheWrite: usage?.prompt_tokens_details?.cache_write_tokens ?? 0, + ...(typeof usage?.cost === 'number' ? { cost: usage.cost } : {}) } } diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index 09f34806c2..fc8a00727f 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -1089,6 +1089,23 @@ export async function getFimCompletion( } } +// A streamed OpenAI-compatible response carries no usage at all unless the request +// asks for it, so a provider missing from this set reports zero tokens — no context +// gauge, no cost. `stream_options.include_usage` is part of the OpenAI streaming +// spec and these providers document supporting it; `customai` is deliberately absent +// because it points at an arbitrary endpoint that may reject the field outright. +const STREAM_USAGE_PROVIDERS = new Set([ + 'openai', + 'azure_openai', + 'azure_foundry', + 'googleai', + 'openrouter', + 'groq', + 'deepseek', + 'mistral', + 'togetherai' +]) + export async function getCompletion( messages: ChatCompletionMessageParam[], abortController: AbortController, @@ -1132,17 +1149,17 @@ export async function getCompletion( // Use Completions API for other providers const client = options?.openaiClient ?? workspaceAIClients.getOpenaiClient() const completionConfig = applyReasoningToConfig( - (provider === 'openai' || - provider === 'azure_openai' || - provider === 'azure_foundry' || - provider === 'googleai') && - config.stream + config.stream && STREAM_USAGE_PROVIDERS.has(provider) ? { ...config, stream_options: { ...(config.stream_options ?? {}), include_usage: true - } + }, + // OpenRouter's own extension, on top of stream_options: it returns the + // credits actually charged next to the token counts, which is the one + // route by which the chat sees a real cost rather than an estimate. + ...(provider === 'openrouter' ? { usage: { include: true } } : {}) } : config, provider === 'deepseek' ? 'deepseek' : provider === 'mistral' ? 'mistral' : 'completions', @@ -1178,7 +1195,11 @@ export async function parseOpenAICompletion( tools: Tool[], helpers: any, _abortController?: AbortController, // unused, for signature compatibility with parseAnthropicCompletion - options?: { workspace?: string; provider?: string } + options?: { + workspace?: string + provider?: string + onTokenUsage?: (usage: ChatTokenUsage) => void + } ): Promise<{ shouldContinue: boolean; tokenUsage: ChatTokenUsage }> { const finalToolCalls: Record = {} // The tool call currently receiving argument deltas; when the stream moves on @@ -1328,6 +1349,7 @@ export async function parseOpenAICompletion( callbacks.onMessageEnd() + options?.onTokenUsage?.(tokenUsage) // Stream over: every parsed call is queued until its turn in processToolCall. for (const toolCall of Object.values(finalToolCalls)) { if (toolCall.id) { diff --git a/frontend/src/lib/components/copilot/modelConfig.ts b/frontend/src/lib/components/copilot/modelConfig.ts index a11fd5c034..cc45a23dac 100644 --- a/frontend/src/lib/components/copilot/modelConfig.ts +++ b/frontend/src/lib/components/copilot/modelConfig.ts @@ -123,21 +123,77 @@ function normalizeVersionSeparators(model: string): string { return model.replace(/\./g, '-') } -// An entry that ends on a version digit must not run into a longer version: -// `gpt-4.1` collapses to `gpt-4-1`, which would otherwise claim the 128K -// `gpt-4-1106-preview` as a 1M model. Suffixes that continue with a separator -// (`claude-opus-4-8` in `...-4-8-v1`, `gpt-5` in `gpt-5-mini`) still match. -// Family fallbacks ending on a letter get no such guard — a version welded -// straight onto the name (`llama3.1`) is exactly what they exist to catch. -const MODEL_CONTEXT_WINDOW_MATCHERS: [matcher: RegExp, contextWindow: number][] = - MODEL_CONTEXT_WINDOWS.map(([name, contextWindow]) => { +/** Suffixes that name a route to a model rather than a different model. */ +const DECORATIVE_SUFFIXES = ['latest', 'preview', 'beta', 'stable'] + +/** + * Compile a most-specific-first `[name, value]` table into matchers against the + * bare model id. Shared with the pricing table so both resolve the same set of + * ids — a model whose window is known but whose price is not (or vice versa) + * should be a gap in one table, never a difference in matching. + * + * An entry that ends on a version digit must not run into a longer version: + * `gpt-4.1` collapses to `gpt-4-1`, which would otherwise claim + * `gpt-4-1106-preview`. Suffixes that continue with a separator + * (`claude-opus-4-8` in `...-4-8-v1`, `gpt-5` in `gpt-5-mini`) still match. + * Family fallbacks ending on a letter get no such guard — a version welded + * straight onto the name (`llama3.1`) is exactly what they exist to catch. + */ +export function buildModelMatchers( + entries: [name: string, value: T][], + { strictVariants = false }: { strictVariants?: boolean } = {} +): [RegExp, T][] { + return entries.map(([name, value]) => { const pattern = normalizeVersionSeparators(name).replace(/[.*+?^${}()|[\]\\]/g, '\\$&') - return [new RegExp(/\d$/.test(pattern) ? `${pattern}(?!\\d)` : pattern), contextWindow] + const guards = [ + // An entry ending on a version digit must not run into a longer version. + /\d$/.test(pattern) ? '(?!\\d)' : '', + // A named sub-model (`gpt-5-pro`, `gpt-5-mini`) is a different model with + // its own price, not another route to this one — so under strictVariants an + // entry does not match when a further *name* segment follows. What follows + // is only a decoration when it is a date (`-20251101`), Bedrock's `-v1`, or + // one of the alias words below, at the very end of the id + // (`claude-3-5-haiku-latest` is the same model as `claude-3-5-haiku`, and is + // a shipped default; `gpt-5-preview-pro` would be a different one again). + // Off by default: for a context window an inherited value is a safe + // approximation, for a price it is a wrong number. + // A further revision segment (`gpt-5` vs `gpt-5-4-mini`) is a different model + // too, and the entry-ends-on-a-digit guard above does not catch it once the + // separator is normalized. Only a short segment: a date is digits as well + // (`-20251101`) and stays a decoration. + strictVariants ? '(?!-\\d{1,3}(?:$|-))' : '', + strictVariants + ? `(?!-(?!(?:v\\d|${DECORATIVE_SUFFIXES.join('|')})$)[a-z])` + : '' + ].join('') + return [new RegExp(pattern + guards), value] }) +} + +/** + * The `provider:model` key the workspace AI settings use for their per-model maps + * (`max_tokens_per_model`, `model_pricing`). A bare model id is not enough: the + * same id can be served by more than one provider at different rates. + * + * Matched exactly, unlike the fuzzy tables above. Those tables generalize across + * every route to one model on purpose; a per-model *setting* must not, or an + * admin could not give two variants of a family different values — and the key is + * built from the exact id the provider config lists, which is the same string the + * chat sends. + */ +export function modelKey(provider: AIProvider | string, model: string): string { + return `${provider}:${model}` +} + +export function matchModel(matchers: [RegExp, T][], model: string): T | undefined { + const id = normalizeVersionSeparators(parseModelId(model).base) + return matchers.find(([matcher]) => matcher.test(id))?.[1] +} + +const MODEL_CONTEXT_WINDOW_MATCHERS = buildModelMatchers(MODEL_CONTEXT_WINDOWS) export function getKnownModelContextWindow(model: string): number | undefined { - const id = normalizeVersionSeparators(parseModelId(model).base) - return MODEL_CONTEXT_WINDOW_MATCHERS.find(([matcher]) => matcher.test(id))?.[1] + return matchModel(MODEL_CONTEXT_WINDOW_MATCHERS, model) } export function getModelContextWindow(model: string) { diff --git a/frontend/src/lib/components/copilot/modelPricing.test.ts b/frontend/src/lib/components/copilot/modelPricing.test.ts new file mode 100644 index 0000000000..e295682187 --- /dev/null +++ b/frontend/src/lib/components/copilot/modelPricing.test.ts @@ -0,0 +1,187 @@ +import { describe, expect, it } from 'vitest' +import { billedTokens } from './chat/tokenUsage' +import { estimateCost, priceSpend, resolveModelPrice } from './modelPricing' + +describe('resolveModelPrice', () => { + it('resolves the same model across the routes that decorate its id', () => { + const direct = resolveModelPrice('anthropic', 'claude-opus-5', undefined) + expect(direct?.price.input).toBe(5) + // A gateway prefix, a dot-versioned id, a date suffix and a variant suffix + // must all land on the same entry — a miss here silently under-reports cost. + for (const id of [ + 'anthropic/claude-opus-5', + 'anthropic/claude-opus-4.8', + 'claude-opus-4-8-20260101', + 'anthropic/claude-opus-5:thinking' + ]) { + expect(resolveModelPrice('openrouter', id, undefined)?.price.input).toBe(5) + } + }) + + it('does not let a version-digit entry claim a longer version', () => { + expect(resolveModelPrice('openai', 'gpt-4.1', undefined)?.price.input).toBe(2) + expect(resolveModelPrice('openai', 'gpt-4-1106-preview', undefined)?.price.input).not.toBe(2) + }) + + it('prices flat-rate Gemini Flash while leaving the tiered Pro alone', () => { + expect(resolveModelPrice('googleai', 'gemini-2.5-flash', undefined)?.price.input).toBe(0.3) + expect(resolveModelPrice('googleai', 'gemini-2.5-flash-lite', undefined)?.price.input).toBe(0.1) + expect(resolveModelPrice('googleai', 'gemini-3.5-flash', undefined)?.price.output).toBe(9) + // Pro charges roughly double above a 200k prompt, which a per-model rate cannot + // express, so it must stay unpriced rather than be estimated at the low tier. + expect(resolveModelPrice('googleai', 'gemini-2.5-pro', undefined)).toBeUndefined() + expect(resolveModelPrice('googleai', 'gemini-3.1-pro', undefined)).toBeUndefined() + // Promotional rates carry an end date a timeless table cannot represent. + expect(resolveModelPrice('googleai', 'gemini-3.7-flash', undefined)).toBeUndefined() + }) + + it('reports an unknown model as unpriced rather than guessing', () => { + expect(resolveModelPrice('customai', 'some-in-house-model', undefined)).toBeUndefined() + }) + + it('does not let another model inherit a price through a shared prefix', () => { + // A sub-model (`-pro`) or a newer revision (`gpt-5.6` → `gpt-5-6`) is a + // different model at a different rate; inheriting `gpt-5`'s would be off by + // an order of magnitude, and silently so. + expect(resolveModelPrice('openai', 'gpt-5', undefined)?.price.input).toBe(1.25) + expect(resolveModelPrice('openai', 'gpt-5-mini', undefined)?.price.input).toBe(0.25) + expect(resolveModelPrice('openai', 'gpt-5-pro', undefined)).toBeUndefined() + expect(resolveModelPrice('openai', 'gpt-5.6', undefined)).toBeUndefined() + expect(resolveModelPrice('googleai', 'gemini-3.1', undefined)).toBeUndefined() + // A revision carrying a variant has to be caught by the matcher, not by an + // explicit entry: `gpt-5.4-mini` cannot match the `gpt-5.4` one (the `-mini` + // makes it a sub-model), so nothing but the guard stops it reaching `gpt-5`. + expect(resolveModelPrice('openai', 'gpt-5.4-mini', undefined)).toBeUndefined() + expect(resolveModelPrice('openai', 'gpt-5.5-pro', undefined)).toBeUndefined() + }) + + it('still resolves the route decorations that name the same model', () => { + // Dates, Bedrock's -v1 and floating aliases are ways of spelling one model, + // not sub-models. `claude-3-5-haiku-latest` is a shipped picker default, so + // unpricing it would silently disable cost tracking out of the box. + expect(resolveModelPrice('anthropic', 'claude-opus-4-5-20251101', undefined)?.price.input).toBe(5) + // The revision guard must not swallow a date, which is digits too. + expect(resolveModelPrice('openai', 'gpt-5-2026-01-01', undefined)?.price.input).toBe(1.25) + expect( + resolveModelPrice('bedrock', 'anthropic.claude-sonnet-4-6-20250101-v1:0', undefined)?.price + .input + ).toBe(3) + expect(resolveModelPrice('anthropic', 'claude-3-5-haiku-latest', undefined)?.price.input).toBe( + 0.8 + ) + // …while a genuine sub-model stays unpriced, including one hiding behind a + // decoration. + expect(resolveModelPrice('openai', 'gpt-5-pro', undefined)).toBeUndefined() + expect(resolveModelPrice('openai', 'gpt-5-preview-pro', undefined)).toBeUndefined() + // A family fallback must not price a model the table deliberately left out, + // nor the floating alias pointing at it. + expect(resolveModelPrice('anthropic', 'claude-sonnet-5', undefined)).toBeUndefined() + expect( + resolveModelPrice('openrouter', '~anthropic/claude-sonnet-latest', undefined) + ).toBeUndefined() + }) + + it('prefers a workspace override, keeping the model’s own cache ratios', () => { + const resolved = resolveModelPrice('anthropic', 'claude-opus-5', { + 'anthropic:claude-opus-5': { input: 2, output: 8 } + }) + expect(resolved?.source).toBe('override') + expect(resolved?.price.input).toBe(2) + // Anthropic reads a cached prefix at a tenth and writes at 1.25x. + expect(resolved?.price.cacheRead).toBeCloseTo(0.2) + expect(resolved?.price.cacheWrite).toBeCloseTo(2.5) + }) + + it('applies the overridden model’s own cache discount, not Anthropic’s', () => { + // gpt-4o discounts a cached read by half, not by a tenth — an override that + // only states input/output must not silently inherit the Anthropic ratio. + const resolved = resolveModelPrice('openai', 'gpt-4o', { + 'openai:gpt-4o': { input: 2, output: 8 } + }) + expect(resolved?.price.cacheRead).toBeCloseTo(1) + }) + + it('bills an unpriced model’s cached tokens at its input rate', () => { + // Gemini Pro is deliberately unpriced, so there is no ratio to inherit. Falling + // back to Anthropic's tenth would invent a discount the provider may not give; + // the admin states the cache rates explicitly or pays full input. + const resolved = resolveModelPrice('googleai', 'gemini-2.5-pro', { + 'googleai:gemini-2.5-pro': { input: 2, output: 8 } + }) + expect(resolved?.price.cacheRead).toBe(2) + expect(resolved?.price.cacheWrite).toBe(2) + + const stated = resolveModelPrice('googleai', 'gemini-2.5-pro', { + 'googleai:gemini-2.5-pro': { input: 2, output: 8, cache_read: 0.5, cache_write: 1 } + }) + expect(stated?.price.cacheRead).toBe(0.5) + expect(stated?.price.cacheWrite).toBe(1) + }) + + it('ignores an override whose rates could not be a price', () => { + for (const bad of [{ input: -1, output: 8 }, { input: 1e9, output: 8 }]) { + const resolved = resolveModelPrice('anthropic', 'claude-opus-5', { + 'anthropic:claude-opus-5': bad + }) + expect(resolved?.source).toBe('builtin') + } + }) +}) + +describe('estimateCost', () => { + it('bills each token class at its own rate', () => { + const cost = estimateCost( + { input: 1_000_000, cacheRead: 1_000_000, cacheWrite: 1_000_000, output: 1_000_000 }, + { input: 5, output: 25, cacheRead: 0.5, cacheWrite: 6.25 } + ) + expect(cost).toBeCloseTo(5 + 0.5 + 6.25 + 25) + }) + + it('charges a cached prefix less than an uncached one', () => { + const usage = { + prompt: 100_000, + completion: 0, + total: 100_000, + cacheRead: 90_000, + cacheWrite: 0 + } + const uncached = { ...usage, cacheRead: 0 } + const price = { input: 5, output: 25, cacheRead: 0.5, cacheWrite: 6.25 } + expect(estimateCost(billedTokens(usage), price)).toBeLessThan( + estimateCost(billedTokens(uncached), price) + ) + }) +}) + +describe('priceSpend', () => { + it('prefers a provider-reported cost over the estimate', () => { + const priced = priceSpend( + [ + { + provider: 'openrouter', + model: 'anthropic/claude-opus-5', + tokens: { input: 1_000_000, cacheRead: 0, cacheWrite: 0, output: 0 }, + reportedCostUsd: 0.42 + } + ], + undefined + ) + expect(priced.total).toBe(0.42) + expect(priced.hasReported).toBe(true) + }) + + it('flags an unpriced model instead of counting it as free', () => { + const priced = priceSpend( + [ + { + provider: 'customai', + model: 'some-in-house-model', + tokens: { input: 1_000_000, cacheRead: 0, cacheWrite: 0, output: 0 } + } + ], + undefined + ) + expect(priced.hasUnpriced).toBe(true) + expect(priced.rows[0].cost).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/copilot/modelPricing.ts b/frontend/src/lib/components/copilot/modelPricing.ts new file mode 100644 index 0000000000..1e5b63247b --- /dev/null +++ b/frontend/src/lib/components/copilot/modelPricing.ts @@ -0,0 +1,288 @@ +import type { AIProvider, ModelPriceOverride } from '$lib/gen' +import { buildModelMatchers, matchModel, modelKey } from './modelConfig' + +/** Rates in USD per million tokens, one per billed token class. */ +export type ModelPrice = { + input: number + output: number + cacheRead: number + cacheWrite: number +} + +export type ModelPriceSource = 'override' | 'builtin' + +export type ResolvedModelPrice = { + price: ModelPrice + source: ModelPriceSource +} + +/** What a chat spent on one model, in tokens. */ +export type PricedTokens = { + input: number + cacheRead: number + cacheWrite: number + output: number +} + +// Fallbacks for entries that do not price their cache separately: Anthropic reads a +// cached prefix at a tenth of the input rate and writes one at 1.25x (5-minute TTL, +// the default the chat uses). The read ratio is NOT universal — OpenAI and Google +// discount a cached read far less — so every non-Anthropic entry below states its own +// `cacheRead` rather than inheriting this. Providers whose caching is automatic never +// report a cache write, so their write rate is unused. +const CACHE_READ_RATIO = 0.1 +const CACHE_WRITE_RATIO = 1.25 + +type PriceEntry = { + input: number + output: number + cacheRead?: number + cacheWrite?: number +} + +/** + * Published list prices, most specific entry first — the first name found in the + * bare model id wins, so vendor-namespaced and date-suffixed ids + * (anthropic/claude-opus-5, gpt-5-2026-01-01) still resolve. Matching is shared + * with the context-window table via `buildModelMatchers`. + * + * This is a best-effort snapshot: vendors change rates, ship models faster than + * this table is updated, and negotiated rates differ from list. A model that is + * not listed resolves to undefined and is reported as unpriced rather than + * guessed at, and any entry can be corrected per workspace from the AI settings. + * Providers whose catalogue turns over too quickly to track (DeepSeek, Mistral, + * Groq, TogetherAI, custom deployments) are deliberately absent. + * + * `null` marks a model that is known to exist but whose rates are not. Unpriced is + * a supported state (the UI says so and points at the override); a confidently + * wrong number is not — which is also why these matchers are built with + * `strictVariants`, so an unlisted sub-model (`gpt-5-pro`) reports no rate instead + * of inheriting its family's. + * + * One known gap the per-model shape cannot express: Anthropic's 1M-context beta + * charges more above a threshold. Usage is aggregated per model before pricing, so + * those requests are estimated at the standard tier and understate. An affected + * workspace can set the higher rate as its override. + */ +const MODEL_PRICES: [name: string, price: PriceEntry | null][] = [ + // Anthropic — Opus 4.1 and older bill at the pre-4.5 Opus rate, so the family + // fallback sits below the explicit entries rather than covering them. + ['claude-fable-5', { input: 10, output: 50 }], + ['claude-mythos-5', { input: 10, output: 50 }], + ['claude-opus-5', { input: 5, output: 25 }], + ['claude-opus-4-8', { input: 5, output: 25 }], + ['claude-opus-4-7', { input: 5, output: 25 }], + ['claude-opus-4-6', { input: 5, output: 25 }], + ['claude-opus-4-5', { input: 5, output: 25 }], + ['claude-opus-4-1', { input: 15, output: 75 }], + ['claude-opus-4', { input: 15, output: 75 }], + // Sonnet 5 runs a promotional rate with a published end date, and + // `claude-sonnet-latest` floats to it. Rates carry no date and apply at read + // time, so either figure misstates one side of that boundary — unpriced until + // the rate is a single number again. + ['claude-sonnet-5', null], + ['claude-sonnet-latest', null], + ['claude-sonnet-4-6', { input: 3, output: 15 }], + ['claude-sonnet-4-5', { input: 3, output: 15 }], + ['claude-sonnet-4', { input: 3, output: 15 }], + ['claude-haiku-4-5', { input: 1, output: 5 }], + ['claude-3-5-haiku', { input: 0.8, output: 4 }], + ['claude-opus', { input: 5, output: 25 }], + ['claude-sonnet', { input: 3, output: 15 }], + ['claude-haiku', { input: 1, output: 5 }], + // OpenAI — the cached-input discount varies by family (a tenth on gpt-5, a + // quarter on 4.1 and the o-series, half on 4o), so each entry carries its own + // rate. There is no charge for writing the cache and no usage field reporting + // one, so the write rate never applies. The -mini/-nano entries must precede + // the family entry, which would otherwise claim them. + // Revisions past gpt-5 are priced separately by OpenAI and are not tracked here. + // The matcher's revision guard already keeps them off the family rate; these + // entries stay so a revision the guard admits still resolves to no rate. + ['gpt-5.6', null], + ['gpt-5.5', null], + ['gpt-5.4', null], + ['gpt-5.2', null], + ['gpt-5.1', null], + ['gpt-5-mini', { input: 0.25, output: 2, cacheRead: 0.025 }], + ['gpt-5-nano', { input: 0.05, output: 0.4, cacheRead: 0.005 }], + ['gpt-5', { input: 1.25, output: 10, cacheRead: 0.125 }], + ['gpt-4.1-mini', { input: 0.4, output: 1.6, cacheRead: 0.1 }], + ['gpt-4.1-nano', { input: 0.1, output: 0.4, cacheRead: 0.025 }], + ['gpt-4.1', { input: 2, output: 8, cacheRead: 0.5 }], + ['gpt-4o-mini', { input: 0.15, output: 0.6, cacheRead: 0.075 }], + ['gpt-4o', { input: 2.5, output: 10, cacheRead: 1.25 }], + ['o4-mini', { input: 1.1, output: 4.4, cacheRead: 0.275 }], + ['o3-mini', { input: 1.1, output: 4.4, cacheRead: 0.55 }], + ['o3', { input: 2, output: 8, cacheRead: 0.5 }], + // Google — Flash takes a flat rate and is priced; Pro is not, because both its + // input and output roughly double above a 200k-token prompt and a per-model rate + // cannot express a threshold. Explicit context caching also bills storage per hour, + // which nothing here represents, so a workspace using it sees an underestimate. + // Gemini 3.7 and 3.6 Flash run a promotional rate with an end date, and stay + // unpriced for the same reason Sonnet 5 does. + ['gemini-2.5-flash-lite', { input: 0.1, output: 0.4, cacheRead: 0.01 }], + ['gemini-2.5-flash', { input: 0.3, output: 2.5, cacheRead: 0.03 }], + ['gemini-3.5-flash-lite', { input: 0.3, output: 2.5, cacheRead: 0.03 }], + ['gemini-3.5-flash', { input: 1.5, output: 9, cacheRead: 0.15 }], + ['gemini-3.7', null], + ['gemini-3.6', null], + ['gemini-3.1', null], + ['gemini-3', null], + ['gemini-2.5', null] +] + +const MODEL_PRICE_MATCHERS = buildModelMatchers( + MODEL_PRICES.map(([name, entry]): [string, ModelPrice | null] => [ + name, + entry && { + input: entry.input, + output: entry.output, + cacheRead: entry.cacheRead ?? entry.input * CACHE_READ_RATIO, + cacheWrite: entry.cacheWrite ?? entry.input * CACHE_WRITE_RATIO + } + ]), + { strictVariants: true } +) + +export function getKnownModelPrice(model: string): ModelPrice | undefined { + return matchModel(MODEL_PRICE_MATCHERS, model) ?? undefined +} + +/** + * Rates the API bounds on the way in — but an instance-level config is stored as an + * untyped settings blob that bypasses that handler, so the reader enforces the same + * bounds rather than rendering a negative, infinite or absurd total. + */ +const MAX_MODEL_RATE = 1000 + +function isUsableRate(rate: number | undefined): boolean { + return rate === undefined || (Number.isFinite(rate) && rate >= 0 && rate <= MAX_MODEL_RATE) +} + +/** What a cache rate falls back to when an override leaves it unset: the model's + * own published multiple of the input rate where the table has one, and the input + * rate itself where it does not, so an unstated discount is never borrowed from + * another vendor. Shared with the rates editor, which shows these as placeholders. */ +export function inheritedCacheRates( + model: string, + input: number +): { cacheRead: number; cacheWrite: number } { + const builtin = getKnownModelPrice(model) + return { + cacheRead: input * (builtin ? builtin.cacheRead / builtin.input : 1), + cacheWrite: input * (builtin ? builtin.cacheWrite / builtin.input : 1) + } +} + +/** + * The rate a workspace should be billed at for one model: its override when an + * admin set one, otherwise the published list price, otherwise nothing. An override + * that omits a cache rate takes it from `inheritedCacheRates`. + */ +export function resolveModelPrice( + provider: AIProvider | string, + model: string, + overrides: Record | undefined +): ResolvedModelPrice | undefined { + const builtin = getKnownModelPrice(model) + const candidate = overrides?.[modelKey(provider, model)] + const override = + candidate && + isUsableRate(candidate.input) && + isUsableRate(candidate.output) && + isUsableRate(candidate.cache_read) && + isUsableRate(candidate.cache_write) + ? candidate + : undefined + if (override) { + const inherited = inheritedCacheRates(model, override.input) + return { + source: 'override', + price: { + input: override.input, + output: override.output, + cacheRead: override.cache_read ?? inherited.cacheRead, + cacheWrite: override.cache_write ?? inherited.cacheWrite + } + } + } + return builtin ? { source: 'builtin', price: builtin } : undefined +} + +/** Cost in USD of `tokens` at `price`. */ +export function estimateCost(tokens: PricedTokens, price: ModelPrice): number { + return ( + (tokens.input * price.input + + tokens.cacheRead * price.cacheRead + + tokens.cacheWrite * price.cacheWrite + + tokens.output * price.output) / + 1_000_000 + ) +} + +/** Tokens spent on one model, from a chat's running totals or the usage API. */ +export type ModelSpend = { + provider: string + model: string + tokens: PricedTokens + /** What the provider billed, where it reports a figure. */ + reportedCostUsd?: number +} + +export type Priced = { + /** Undefined when no rate is known for the model — reported as unpriced, never guessed. */ + cost: number | undefined + source: ModelPriceSource | 'reported' | undefined +} + +export type PricedSpend = { + /** The input entries, each with its cost — callers carry their own fields through + * rather than zipping the result back against the input by index. */ + rows: (T & Priced)[] + total: number + /** True when at least one row has no rate, so `total` understates the truth. */ + hasUnpriced: boolean + /** True when at least one row is a figure the provider billed rather than an estimate. */ + hasReported: boolean +} + +/** + * Cost a set of per-model token counts. A provider-reported figure always wins: + * it is what was actually charged, where everything else is list price times + * tokens. `source` says which, so a view never presents an estimate as a bill. + */ +export function priceSpend( + spend: T[], + overrides: Record | undefined +): PricedSpend { + let total = 0 + let hasUnpriced = false + let hasReported = false + const rows = spend.map((entry): T & Priced => { + if (entry.reportedCostUsd !== undefined) { + hasReported = true + total += entry.reportedCostUsd + return { ...entry, cost: entry.reportedCostUsd, source: 'reported' } + } + const resolved = resolveModelPrice(entry.provider, entry.model, overrides) + if (!resolved) { + hasUnpriced = true + return { ...entry, cost: undefined, source: undefined } + } + const cost = estimateCost(entry.tokens, resolved.price) + total += cost + return { ...entry, cost, source: resolved.source } + }) + return { rows, total, hasUnpriced, hasReported } +} + +/** + * Money, at the precision the amount deserves: sub-cent spend is where a chat + * spends most of its life, and rounding it to `$0.00` would read as free. + */ +export function formatUsd(amount: number): string { + if (amount === 0) return '$0' + if (amount < 0.01) return `$${amount.toFixed(4)}` + if (amount < 1) return `$${amount.toFixed(3)}` + return `$${amount.toFixed(2)}` +} diff --git a/frontend/src/lib/components/workspaceSettings/AISettings.svelte b/frontend/src/lib/components/workspaceSettings/AISettings.svelte index 60d1cb77a0..5979e8a535 100644 --- a/frontend/src/lib/components/workspaceSettings/AISettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/AISettings.svelte @@ -5,9 +5,11 @@ type AIConfig, type AIProvider, type GetCopilotSettingsStateResponse, - type InstanceAISummary + type InstanceAISummary, + type ModelPriceOverride } from '$lib/gen' import { workspaceStore } from '$lib/stores' + import { copilotInfo } from '$lib/aiStore' import { sendUserToast } from '$lib/toast' import { AI_PROVIDERS, fetchAvailableModels, providerSupportsWebSearch } from '../copilot/lib' import { supportsAutocomplete } from '../copilot/utils' @@ -25,6 +27,8 @@ import Badge from '../common/badge/Badge.svelte' import Tooltip from '../Tooltip.svelte' import ModelTokenLimits from './ModelTokenLimits.svelte' + import ModelPricing from './ModelPricing.svelte' + import AiUsagePanel from './AiUsagePanel.svelte' import { setCopilotInfo } from '$lib/aiStore' import AIPromptsModal from '../settings/AIPromptsModal.svelte' import { Settings } from 'lucide-svelte' @@ -73,6 +77,7 @@ let metadataModel: string | undefined = $state(undefined) let customPrompts: Record = $state({}) let maxTokensPerModel: Record = $state({}) + let modelPricing: Record = $state({}) let usingOpenaiClientCredentialsOauth = $state(false) let workspaceOverrideEditorOpened = $state(false) @@ -83,6 +88,7 @@ let initialMetadataModel: string | undefined = $state(undefined) let initialCustomPrompts: Record = $state({}) let initialMaxTokensPerModel: Record = $state({}) + let initialModelPricing: Record = $state({}) let initialPrompts: Record = $state({}) let lastLoadedConfigKey = $state(undefined) @@ -110,6 +116,7 @@ codeCompletionModel = config?.code_completion_model?.model customPrompts = clone(config?.custom_prompts ?? {}) maxTokensPerModel = clone(config?.max_tokens_per_model ?? {}) + modelPricing = clone(config?.model_pricing ?? {}) for (const mode of ['edit', 'fix', 'gen']) { if (!(mode in customPrompts)) { customPrompts[mode] = '' @@ -124,6 +131,7 @@ initialCodeCompletionModel = codeCompletionModel initialCustomPrompts = clone(customPrompts) initialMaxTokensPerModel = clone(maxTokensPerModel) + initialModelPricing = clone(modelPricing) initialPrompts = clone(customPrompts) } @@ -139,6 +147,7 @@ codeCompletionModel = initialCodeCompletionModel customPrompts = clone(initialCustomPrompts) maxTokensPerModel = clone(initialMaxTokensPerModel) + modelPricing = clone(initialModelPricing) } $effect(() => { @@ -172,7 +181,8 @@ metadataModel !== initialMetadataModel || codeCompletionModel !== initialCodeCompletionModel || JSON.stringify(customPrompts) !== JSON.stringify(initialCustomPrompts) || - JSON.stringify(maxTokensPerModel) !== JSON.stringify(initialMaxTokensPerModel) + JSON.stringify(maxTokensPerModel) !== JSON.stringify(initialMaxTokensPerModel) || + JSON.stringify(modelPricing) !== JSON.stringify(initialModelPricing) ) $effect(() => { @@ -285,7 +295,8 @@ metadata_model, custom_prompts: Object.keys(custom_prompts).length > 0 ? custom_prompts : undefined, max_tokens_per_model: - Object.keys(maxTokensPerModel).length > 0 ? maxTokensPerModel : undefined + Object.keys(maxTokensPerModel).length > 0 ? maxTokensPerModel : undefined, + model_pricing: Object.keys(modelPricing).length > 0 ? modelPricing : undefined } : {} } @@ -610,6 +621,24 @@ scope={promptScope} /> +{#if promptScope === 'workspace'} + + +{/if} + + +{#if showWorkspaceOverrideEditor} + +{/if} + {#if showWorkspaceOverrideEditor} + import { AiService, ApiError, type AITokenUsageBucket, type ModelPriceOverride } from '$lib/gen' + import { formatUsd, priceSpend, type ModelSpend } from '../copilot/modelPricing' + import { formatTokenCount } from '../copilot/chat/tokenUsage' + import SettingCard from '../instanceSettings/SettingCard.svelte' + import Select from '../select/Select.svelte' + import ToggleButtonGroup from '../common/toggleButton-v2/ToggleButtonGroup.svelte' + import ToggleButton from '../common/toggleButton-v2/ToggleButton.svelte' + import { resource } from 'runed' + import Tooltip from '../meltComponents/Tooltip.svelte' + import DataTable from '../table/DataTable.svelte' + import Head from '../table/Head.svelte' + import Cell from '../table/Cell.svelte' + + // Workspace and rates are both passed in rather than read from a store: the + // settings component that mounts this one also serves the instance scope, and + // the rates that priced a chat are the workspace's *effective* ones, which an + // inheriting workspace does not hold itself. + let { + workspace, + modelPricing, + scope = 'workspace' + }: { + workspace: string + modelPricing: Record + scope?: 'workspace' | 'self' + } = $props() + + type GroupBy = 'day' | 'user' | 'model' + + let days = $state(30) + let groupBy = $state('day') + + const rangeOptions = [ + { label: 'Last 7 days', value: 7 }, + { label: 'Last 30 days', value: 30 }, + { label: 'Last 90 days', value: 90 } + ] + + let usage = resource( + () => ({ workspace, days, groupBy, scope }), + async ({ workspace, days, groupBy, scope }) => + workspace ? await AiService.listAiUsage({ workspace, days, groupBy, scope }) : undefined + ) + + // The API groups by (dimension, provider, model) so every bucket resolves to a + // single rate; the table folds those back into one line per dimension value. + type Bucket = ModelSpend & { key: string; requests: number } + + function toSpend(bucket: AITokenUsageBucket): Bucket { + return { + // Grouping by model has no separate dimension — the model is the key. + key: groupBy === 'model' ? `${bucket.provider}/${bucket.model}` : bucket.key || '—', + requests: bucket.requests, + provider: bucket.provider, + model: bucket.model, + tokens: { + input: bucket.input_tokens, + cacheRead: bucket.cache_read_tokens, + cacheWrite: bucket.cache_write_tokens, + output: bucket.output_tokens + }, + reportedCostUsd: + bucket.reported_cost_nano_usd != undefined + ? bucket.reported_cost_nano_usd / 1_000_000_000 + : undefined + } + } + + let priced = $derived(priceSpend((usage.current?.buckets ?? []).map(toSpend), modelPricing)) + + type Row = { + key: string + cost: number | undefined + /** Every model behind this line was billed back by its provider, so the + * figure is an invoice rather than an estimate. A line mixing sources — or + * one holding a model with no rate, whose spend the figure omits entirely — + * makes the weaker claim. */ + reported: boolean + tokensIn: number + tokensOut: number + requests: number + } + + // Only a 403 on the workspace scope is a permission problem; reading your own + // usage is open to any member. Attributing every failure to permissions sends an + // admin looking for access they already hold, and buries the real cause of the + // far more common transient ones (an expired session, a database hiccup). + function usageError(error: unknown): string { + if (scope === 'workspace' && error instanceof ApiError && error.status === 403) { + return 'Only workspace admins can read workspace usage.' + } + return 'Could not load usage. Try again in a moment.' + } + + // The headline sums both kinds, so it only escapes the ~ when nothing under it + // was estimated. + let totalIsEstimated = $derived( + priced.rows.some((row) => row.cost !== undefined && row.source !== 'reported') + ) + + let rows = $derived.by(() => { + const byKey = new Map() + for (const row of priced.rows) { + const existing = byKey.get(row.key) ?? { + key: row.key, + cost: undefined, + reported: true, + tokensIn: 0, + tokensOut: 0, + requests: 0 + } + existing.tokensIn += row.tokens.input + row.tokens.cacheRead + row.tokens.cacheWrite + existing.tokensOut += row.tokens.output + existing.requests += row.requests + if (row.cost !== undefined) { + existing.cost = (existing.cost ?? 0) + row.cost + } + existing.reported &&= row.source === 'reported' + byKey.set(row.key, existing) + } + return [...byKey.values()].sort((a, b) => (b.cost ?? 0) - (a.cost ?? 0)) + }) + + + +
+
+
+
+ + {#snippet endSnippet({ item, close })} + +
+ {/snippet} + + {#if dataset && hoveringDataset} +
+
+ {/if} +
+ {/if} + + + + +
+ {#snippet actions()} + + {/snippet} + diff --git a/frontend/src/lib/components/aiEvals/EvalRunsList.svelte b/frontend/src/lib/components/aiEvals/EvalRunsList.svelte new file mode 100644 index 0000000000..5d84ef88ce --- /dev/null +++ b/frontend/src/lib/components/aiEvals/EvalRunsList.svelte @@ -0,0 +1,170 @@ + + + + + + + + + + + + + Run + Dataset + Cases + Scores + When + + + + {#each experiments as experiment (experiment.id)} + onOpen(experiment)}> + +
+
+ {experimentName(experiment)} + + {subjectLabel(experiment, deployedHash, currentVersion)} + +
+ {experiment.created_by} +
+ + + {@const summary = datasetSummary(datasets, experiment.dataset)} + + + + {experiment.case_count} + + +
+ {#each experiment.scores ?? [] as score (score.scorer_id)} + {@const value = headline(score)} + + + {#if score.kind === 'agent'} + + {:else} + + {/if} + {score.name} + {#if value != undefined} + {value} + {:else if score.failed > 0} + failed + {:else if experiment.running} + + {:else} + — + {/if} + + + {/each} + {#if (experiment.scores ?? []).length === 0} + {#if experiment.running} + + + scoring + + {:else} + not scored + {/if} + {/if} +
+ + + + + + + + {/each} + {#if experiments.length === 0 && !loaded} + + + + + + {:else if experiments.length === 0} + + +
+ No runs yet + + A run answers every case of a dataset and scores the answers. Each one is kept, so the + next has something to be compared against. + + +
+ + + {/if} + + diff --git a/frontend/src/lib/components/aiEvals/EvalScorers.svelte b/frontend/src/lib/components/aiEvals/EvalScorers.svelte new file mode 100644 index 0000000000..68287b0fbc --- /dev/null +++ b/frontend/src/lib/components/aiEvals/EvalScorers.svelte @@ -0,0 +1,383 @@ + + +
+
+ Scorers + {scorers.length} +
+ openAdd('agent', 'new') }, + { + displayName: 'Existing AI judge', + icon: Bot, + action: () => openAdd('agent', 'existing') + }, + { displayName: 'New code scorer', icon: Code2, action: () => openAdd('script', 'new') }, + { + displayName: 'Existing code scorer', + icon: Code2, + action: () => openAdd('script', 'existing') + } + ]} + placement="bottom-end" + > + {#snippet buttonReplacement()} + + {/snippet} + +
+ +
+ {#if scorers.length === 0} +
+ A scorer reads one run and returns a number. Every run of this dataset is measured by all of + them, which is what makes two runs comparable. +
+ {:else} +
+ {#each scorers as scorer (scorer.id)} +
+ {#if scorer.kind === 'agent'} + + {:else} + + {/if} +
+ + {scorerLabel(scorer)} + + {scorer.path} +
+ {#if scorer.pass_if != undefined} + + ≥ {scorer.pass_if} + + {/if} +
+ {/each} +
+ {/if} +
+
+ + + scorerDrawer?.closeDrawer()} + > + {#if workspace && datasetPath} + {#key scorerFormGeneration} + + scriptEditorDrawer + ?.openDrawer(hash, onChanged) + .catch((e) => sendUserToast(`Failed to open the scorer: ${e}`, true))} + /> + {/key} + {/if} + {#snippet actions()} + {@const state = addScorerForm?.submitState()} + + {/snippet} + + + + + settingsDrawer?.closeDrawer()}> + {#if settingsScorer} +
+ + + +
+ {/if} + {#snippet actions()} + + {/snippet} + + + + + + + + (removingScorer = undefined)} + on:confirmed={async () => { + const target = removingScorer + removingScorer = undefined + if (!target) return + try { + await saveScorers(scorers.filter((s) => s.id !== target.id)) + } catch (e) { + sendUserToast(`Failed to remove the scorer: ${e}`, true) + } + }} +> + + The column goes from every run of this dataset, the ones already recorded included. Adding it + again starts a new column, which fills from the next run on. + + diff --git a/frontend/src/lib/components/aiEvals/EvalsPane.svelte b/frontend/src/lib/components/aiEvals/EvalsPane.svelte new file mode 100644 index 0000000000..d9e5b16fe4 --- /dev/null +++ b/frontend/src/lib/components/aiEvals/EvalsPane.svelte @@ -0,0 +1,978 @@ + + +
+
+ {#if viewingRun} + + {/if} +
+ {#if viewingRun && experiment?.run_job_id} + + Open the job + + + {/if} + {#if !viewingRun && loaded && datasets.length > 0} + + {#if experiments.length > 0} + + + {/if} + {/if} +
+ +
+ + +
+ {#if loaded && loadError} +
+ Could not load evals + + The datasets or runs could not be read. Check your access to this agent and reload. + +
+ {:else if loaded && datasets.length === 0} +
+ No dataset yet + + A dataset is the set of cases this agent is measured on. Runs are of a dataset, so + it is the first thing to make. + + +
+ {:else if !viewingRun || !loaded} + openRun(e.id)} + onEditDataset={async (path) => { + if (await useDataset(path)) datasetDrawer?.openDrawer('edit') + }} + onNew={() => (runDialogOpen = true)} + /> + {:else} + + + + + {#each scorers as scorer (scorer.id)} + + {/each} + + + + Case + Answer + {#each scorers as scorer, index (scorer.id)} + {@const mean = means.find((m) => m.scorer_id === scorer.id)} + {@const headline = columnHeadline(scorer, mean)} + + +
+ + {#if scorer.kind === 'agent'} + + {:else} + + {/if} + {scorerLabel(scorer)} + + + {#if headline} + + {headline.value} + + {#if headline.delta && headline.direction !== 0} + 0 ? 'text-green-500' : headline.direction < 0 ? 'text-red-500' : 'text-tertiary'}`} + > + {headline.delta} + + {/if} + {/if} + +
+ + {/each} + + + + {#each displayRows as row (row.case_id)} + {@const status = statusOf(row.status)} + openCase(row)} + > + + {caseLabel(row)} + + + + + {#if row.output != undefined} + {row.output} + {:else if status === STATUS.not_run} + not run + {:else} + {status.label.toLowerCase()} + {/if} + + + {#each scorers as scorer, index (scorer.id)} + {@const cell = row.scores.find((s) => s.scorer_id === scorer.id)} + + {#if cell?.pending} + + + + {:else if cell?.score != undefined} + + {#snippet text()} +
+ {#if cell.reason} + {cell.reason} + {/if} + {#each checksOf(cell) as check (check.name)} + + + {check.passed ? '✓' : '✗'} + + {check.name} + {#if check.detail} + {check.detail} + {/if} + + {/each} +
+ {/snippet} + + {#if cell.passed != undefined} + + {cell.passed ? '✓' : '✗'} + + {/if} + + {formatScore(cell.score)} + + {#if cell.baseline != undefined && cell.score !== cell.baseline} + {@const delta = cell.score - cell.baseline} + 0 ? 'text-green-500' : 'text-red-500'}`} + > + {formatDelta(delta)} + + {/if} + + + {:else if cell?.not_applicable} + + {#snippet text()} + {cell.reason} + {/snippet} + + n/a + + + {:else if cell?.error} + + {#snippet text()} + {cell.error} + {/snippet} + failed + + {:else} + — + {/if} + + {/each} + + {/each} + + + {/if} +
+ + {#if selectedRow} + {@const openRow = selectedRow} + +
+
+ + {openRow.input?.user_message ?? caseLabel(openRow)} + +
+ {#if openRow.job_id} + + Open the case job + + + {/if} +
+
+ {#if openRow.expected != undefined && openRow.expected !== ''} + + {/if} + {#if scorers.length > 0 && openRow.scores.length > 0} + + {/if} + {#if experiment && (openRow.job_id || openRow.output != undefined)} +
+
+ + Case result + +
+
+ {#if openRow.output != undefined} +
+ +
+ {:else if openRow.status === 'running'} + + + Running + + {:else} + {statusOf(openRow.status).label} + {/if} +
+
+ {/if} +
+
+ + {/if} + +
+
+ + { + if (await useDataset(path)) { + resumeRunDialog = true + datasetDrawer?.openDrawer('edit') + } + }} + onNewDataset={() => { + resumeRunDialog = true + datasetDrawer?.openDrawer('new') + }} +/> + + { + if (!resumeRunDialog) return + resumeRunDialog = false + // On the dataset the drawer was just in: the dialog opens on the pane's own, which + // creating or editing one has already moved to it. + runDialogOpen = true + }} +/> diff --git a/frontend/src/lib/components/aiEvals/evalUtils.test.ts b/frontend/src/lib/components/aiEvals/evalUtils.test.ts new file mode 100644 index 0000000000..d1f65e0d96 --- /dev/null +++ b/frontend/src/lib/components/aiEvals/evalUtils.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, it } from 'vitest' +import type { EvalExperiment } from '$lib/gen' +import { parseThreshold, subjectLabel } from './evalUtils' + +describe('parseThreshold', () => { + it('keeps 0 as a threshold and reads only empty text as no threshold', () => { + expect(parseThreshold(0)).toEqual({ value: 0, error: false }) + expect(parseThreshold('0')).toEqual({ value: 0, error: false }) + expect(parseThreshold('')).toEqual({ error: false }) + expect(parseThreshold(' ')).toEqual({ error: false }) + expect(parseThreshold(null)).toEqual({ error: false }) + expect(parseThreshold(undefined)).toEqual({ error: false }) + }) + + it('refuses anything outside 0 to 1 or not a number', () => { + expect(parseThreshold('0.5')).toEqual({ value: 0.5, error: false }) + expect(parseThreshold('1')).toEqual({ value: 1, error: false }) + expect(parseThreshold('1.5')).toEqual({ error: true }) + expect(parseThreshold('-0.1')).toEqual({ error: true }) + expect(parseThreshold('abc')).toEqual({ error: true }) + }) +}) + +describe('subjectLabel', () => { + function run(subject: Record): EvalExperiment { + return { subject: { path: 'u/me/agent', ...subject } } as unknown as EvalExperiment + } + + it('names a deployed run and a pinned version by their number', () => { + expect(subjectLabel(run({ kind: 'agent', version: 4 }))).toBe('v4') + expect(subjectLabel(run({ kind: 'agent_version', version: 2 }))).toBe('v2') + }) + + it('says a draft run is edits on top of the version it was an edit of', () => { + expect(subjectLabel(run({ kind: 'agent_draft', version: 4, draft_hash: 'h1' }))).toBe( + 'v4 + edits' + ) + expect(subjectLabel(run({ kind: 'agent_draft', draft_hash: 'h1' }))).toBe('edits') + }) + + it('reads a draft whose configuration is now deployed as the current version', () => { + const draft = run({ kind: 'agent_draft', version: 4, draft_hash: 'h1' }) + expect(subjectLabel(draft, 'h1', 5)).toBe('v5') + expect(subjectLabel(draft, 'other', 5)).toBe('v4 + edits') + }) +}) diff --git a/frontend/src/lib/components/aiEvals/evalUtils.ts b/frontend/src/lib/components/aiEvals/evalUtils.ts new file mode 100644 index 0000000000..fcc5419f48 --- /dev/null +++ b/frontend/src/lib/components/aiEvals/evalUtils.ts @@ -0,0 +1,107 @@ +import type { + EvalCase, + EvalCaseInput, + EvalDataset, + EvalExperiment, + NewEvalCase, + Scorer +} from '$lib/gen' + +/** The case being edited in the drawer, before it is either run or saved to a dataset. */ +export type CaseDraft = NewEvalCase & { id?: string } + +/** A level the evals pane is on, and the way out of it. */ +export type EvalsLocation = { label: string; back: () => void } + +export type ScorerKind = Scorer['kind'] + +export function emptyCase(): CaseDraft { + return { input: { user_message: '' } } +} + +export function fromStoredCase(c: EvalCase): CaseDraft { + const { created_at: _created_at, created_by: _created_by, ...rest } = c + return rest +} + +export function caseLabel(c: { input?: EvalCaseInput }): string { + const message = c.input?.user_message?.trim() + if (message) return message.length > 60 ? message.slice(0, 60) + '…' : message + return 'Untitled case' +} + +export function experimentName(experiment: EvalExperiment): string { + return `Run ${experiment.run_number}` +} + +/** + * What ran: a deployed version, or a version with edits sitting on top of it. + * + * The list and the results endpoint restamp a draft run whose configuration was later deployed, so + * the kind is usually enough; `deployedHash` and `currentVersion` resolve the one still unstamped. + */ +export function subjectLabel( + experiment: EvalExperiment, + deployedHash?: string, + currentVersion?: number +): string { + if (experiment.subject.kind === 'agent_version') { + return experiment.subject.version ? `v${experiment.subject.version}` : 'a past version' + } + const deployed = + experiment.subject.kind === 'agent' || + (experiment.subject.draft_hash != undefined && experiment.subject.draft_hash === deployedHash) + if (deployed) { + const version = + experiment.subject.kind === 'agent' ? experiment.subject.version : currentVersion + return version ? `v${version}` : 'deployed' + } + return experiment.subject.version ? `v${experiment.subject.version} + edits` : 'edits' +} + +/** A scorer keeps its id when renamed, so its name is the column header and nothing else. */ +export function scorerLabel(scorer: Scorer): string { + return scorer.name || scorer.path.split('/').pop() || scorer.path +} + +export function kindLabel(kind: ScorerKind): string { + return kind === 'agent' ? 'Judge agent' : 'Script' +} + +export function formatScore(score: number | undefined): string { + return score == undefined ? '—' : score.toFixed(2) +} + +export function formatDelta(delta: number): string { + if (delta === 0) return '0.00' + return `${delta > 0 ? '+' : '−'}${Math.abs(delta).toFixed(2)}` +} + +/** What a dataset is for, where it says so: the path names it either way. */ +export function datasetSummary(datasets: EvalDataset[], path: unknown): string | undefined { + return datasets.find((d) => d.path === path)?.summary || undefined +} + +/** + * A pass threshold, as a field holds it. Empty is `''` or null, never a number: a number input + * coerces the text, so a valid threshold of 0 would otherwise read as empty and be dropped. The + * server refuses anything outside 0 to 1, caught here so the form blocks instead of the save. + */ +export function parseThreshold(text: string | number | null | undefined): { + value?: number + error: boolean +} { + const trimmed = typeof text === 'string' ? text.trim() : text + if (trimmed === '' || trimmed == undefined) return { error: false } + const value = Number(trimmed) + if (Number.isNaN(value) || value < 0 || value > 1) return { error: true } + return { value, error: false } +} + +export function summaryToName(summary: string): string { + return summary + .toLowerCase() + .replace(/[^a-z0-9_]/g, '_') + .replace(/_+/g, '_') + .replace(/^_|_$/g, '') +} diff --git a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.css b/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.css new file mode 100644 index 0000000000..1ac9f8b31c --- /dev/null +++ b/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.css @@ -0,0 +1,26 @@ +/* MultilineCellEditor: a popup positioned over the cell, so it has to paint the cell's own frame + rather than inherit it. */ +.ag-theme-alpine .wm-multiline-cell-editor, +.ag-theme-alpine-dark .wm-multiline-cell-editor { + background-color: var(--ag-background-color); +} +.ag-theme-alpine .wm-multiline-cell-editor textarea, +.ag-theme-alpine-dark .wm-multiline-cell-editor textarea { + display: block; + box-sizing: border-box; + /* Horizontal only: the vertical padding is set by the editor, which knows the height of the row + it is replacing. `line-height` here is what it computes against. */ + padding: 0 calc(var(--ag-cell-horizontal-padding) - 1px); + border: 1px solid var(--ag-input-focus-border-color); + border-radius: 3px; + outline: none; + resize: none; + /* Past this it scrolls rather than growing. */ + max-height: 40vh; + overflow-y: auto; + background-color: var(--ag-background-color); + color: var(--ag-foreground-color); + font: inherit; + line-height: 20px; + white-space: pre-wrap; +} diff --git a/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.ts b/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.ts new file mode 100644 index 0000000000..00975955c9 --- /dev/null +++ b/frontend/src/lib/components/apps/components/display/table/multilineCellEditor.ts @@ -0,0 +1,108 @@ +import type { ColDef, ICellEditorComp, ICellEditorParams } from 'ag-grid-community' +// Beside the editor rather than in the AgGrid theme: that file is the vendored theme, and a rule +// added to it is one the next copy of it drops. +import './multilineCellEditor.css' + +/** Kept in step with the `line-height` the stylesheet gives the textarea. */ +const LINE_HEIGHT = 20 + +/** + * A text cell editor that starts the height of the cell and grows as lines are added, for columns + * holding prose rather than a value. Enter commits, Shift+Enter adds a line, Escape cancels. + * + * Rendered as a popup positioned over the cell: an in-cell editor is clipped to the row height, so + * growing is only visible if the editor is allowed to paint outside it. + */ +export class MultilineCellEditor implements ICellEditorComp { + private eGui!: HTMLDivElement + private textarea!: HTMLTextAreaElement + private params!: ICellEditorParams + private wasEmpty = false + + init(params: ICellEditorParams) { + this.params = params + this.eGui = document.createElement('div') + this.eGui.className = 'wm-multiline-cell-editor' + + this.wasEmpty = params.value == undefined + + this.textarea = document.createElement('textarea') + this.textarea.rows = 1 + // A keystroke that opened the edit replaces the value, as it does in every other cell; F2 + // and double-click keep it to be edited. + this.textarea.value = params.eventKey?.length === 1 ? params.eventKey : (params.value ?? '') + this.textarea.style.width = `${params.column.getActualWidth() - 2}px` + // Padded so one line fills the cell it replaces and a second costs a line rather than a row. + // From the row rather than from `--ag-row-height`, which is the theme's figure and not + // necessarily this grid's. + const rowHeight = params.node.rowHeight ?? 28 + const padding = Math.max(0, (rowHeight - LINE_HEIGHT - 2) / 2) + this.textarea.style.paddingTop = `${padding}px` + this.textarea.style.paddingBottom = `${padding}px` + + this.textarea.addEventListener('input', () => this.resize()) + this.textarea.addEventListener('keydown', (e) => { + if (e.key === 'Escape') { + // Kept from whatever is around the grid: a grid in a drawer or a dialog is under a + // surface that closes on Escape, and leaving an edit is not asking to leave that. + e.preventDefault() + e.stopPropagation() + this.params.api.stopEditing(true) + return + } + if (e.key !== 'Enter' || e.isComposing) return + // Both branches keep the key from the grid, which ends the edit on Enter whether or not + // Shift is held: Shift+Enter falls through to the textarea's own newline, and plain Enter + // ends the edit here instead. + e.stopPropagation() + if (!e.shiftKey) { + e.preventDefault() + this.params.stopEditing() + } + }) + this.eGui.appendChild(this.textarea) + } + + private resize() { + this.textarea.style.height = 'auto' + this.textarea.style.height = `${this.textarea.scrollHeight}px` + } + + getGui() { + return this.eGui + } + + afterGuiAttached() { + this.resize() + this.textarea.focus() + // At the end rather than selected: a selection is a keystroke away from erasing the cell. + const end = this.textarea.value.length + this.textarea.setSelectionRange(end, end) + } + + getValue() { + // Nothing typed into a cell that held nothing is not an edit: returning '' here would write + // an empty string over a null, which the grid would see as a change and commit. + if (this.wasEmpty && this.textarea.value === '') return this.params.value + return this.textarea.value + } + + isPopup() { + return true + } + + getPopupPosition(): 'over' | 'under' { + return 'over' + } +} + +/** + * What a column of prose needs, ready to spread into a colDef. `suppressKeyboardEvent` as well as + * the editor: the grid ends an edit on Enter from a handler a popup editor's DOM does not sit + * under, so the editor cannot keep Shift+Enter for itself on its own. + */ +export const multilineCellColDef: Pick = { + cellEditor: MultilineCellEditor, + suppressKeyboardEvent: (p) => + p.editing && (p.event as KeyboardEvent).key === 'Enter' && (p.event as KeyboardEvent).shiftKey +} diff --git a/frontend/src/lib/components/common/drawer/Disposable.svelte b/frontend/src/lib/components/common/drawer/Disposable.svelte index 6a27357cf2..642d1366f6 100644 --- a/frontend/src/lib/components/common/drawer/Disposable.svelte +++ b/frontend/src/lib/components/common/drawer/Disposable.svelte @@ -94,6 +94,13 @@ return open } + /** Whether this is the overlay on top, i.e. the one a key press is for. Overlays that keep + * Escape for themselves (`preventEscape`) have to ask, or they answer keys aimed at whatever + * is stacked above them. Same condition the handler below arbitrates on. */ + export function isTopmost() { + return stack.val.length === 0 || stack.val[stack.val.length - 1] === id + } + function handleClickAway(e) { const last = stack.val[stack.val.length - 1] if (last === id) { diff --git a/frontend/src/lib/components/common/modal/Modal.svelte b/frontend/src/lib/components/common/modal/Modal.svelte index 20821a33f9..9cfa11b84e 100644 --- a/frontend/src/lib/components/common/modal/Modal.svelte +++ b/frontend/src/lib/components/common/modal/Modal.svelte @@ -1,5 +1,16 @@ + + -
+
{#if agent} -
-
+
+ +
(showDetail = !showDetail)} + onkeydown={(e) => { + // Keys aimed at the buttons inside the row bubble through here; leave them theirs. + if (e.target !== e.currentTarget) return + if (e.key === 'Enter' || e.key === ' ') { + e.preventDefault() + showDetail = !showDetail + } + }} + > - Linked to - +
{agent} e.stopPropagation()}>{agent} - - {#snippet text()} - Read-only: the configuration comes from this saved agent, and only the message and - inputs are set in this flow. Edit changes the agent everywhere it's used. Unlink forks - an editable copy into just this step. - {/snippet} - - -
+ {#if version != undefined} + + v{version} + + {/if} +
+
+ {#if brainParams.length > 0 || inheritedTools.length > 0} + + {#if showDetail} + + {:else} + + {/if} + + {/if} +
- {#if brainParams.length > 0 || inheritedTools.length > 0} -
+ {#if showDetail && (brainParams.length > 0 || inheritedTools.length > 0)} +
{#each brainParams as param (param.label)}
{param.label}
@@ -478,12 +609,7 @@
Tools
{#each inheritedTools as tool (tool.id)} - - {toolLabel(tool)} - + {toolLabel(tool)} {/each}
@@ -502,14 +628,56 @@ {/if} {:else if editingPath}
- - Editing - {editingPath} -
+
+ +
+
+ {editingPath} + {#if version != undefined} + + v{version} + + {/if} + {#if edited} + + unsaved changes + + {/if} +
+
+ saving updates every flow using it + {#snippet text()} + The edits live in this step until you decide: Evals runs them as they are here, Save + changes writes them to the agent, Cancel drops them and re-links the step. + {/snippet} + +
+
+
+
+ + -
-

- Editing the saved agent. Save changes updates it and re-links this step — the update - propagates to every flow that links to it. Cancel keeps your edits here as a standalone step - instead. -

{#if providerSaveError} -

+

{providerSaveError}

{/if} {:else} -
-
- -
- or - -
+ {/if}
@@ -552,7 +711,8 @@

Save this AI agent's configuration and tools as a reusable resource. Other flows can then - link to it, and updates propagate automatically. + link to it, updates propagate automatically, and it gains a dataset of eval cases of its + own.

+ + + + + + + { + confirmCancel = false + const path = editingPath + if (path) relink(path) + }} + onCanceled={() => (confirmCancel = false)} +> + + The step goes back to {editingPath} as it is deployed, and the edits are not kept anywhere. Save + changes writes them to the agent instead. + + diff --git a/frontend/src/lib/components/flows/content/ScriptEditorDrawer.svelte b/frontend/src/lib/components/flows/content/ScriptEditorDrawer.svelte index 538bcf26c7..362b7bca37 100644 --- a/frontend/src/lib/components/flows/content/ScriptEditorDrawer.svelte +++ b/frontend/src/lib/components/flows/content/ScriptEditorDrawer.svelte @@ -175,52 +175,6 @@ let settingsDrawer: Drawer | undefined = $state() - { - unsavedModalOpen = false - }} - on:confirmed={() => { - console.log('confirmed') - closeAnyway = true - unsavedModalOpen = false - scriptEditorDrawer?.closeDrawer() - }} -> -
- Are you sure you want to discard the changes you have made? - -
- + + { + unsavedModalOpen = false + }} + on:confirmed={() => { + closeAnyway = true + unsavedModalOpen = false + scriptEditorDrawer?.closeDrawer() + }} + > +
+ Are you sure you want to discard the changes you have made? + +
+ { +export async function createAiAgent( + id: string, + agentPath?: string +): Promise<[FlowModule, FlowModuleState]> { const storedConfig = loadStoredConfig() const providerValue = storedConfig ?? { kind: 'openai', resource: '', model: '' } + // A step linked to a saved agent reads its brain and tools from the resource, so it carries only + // the flow-local inputs: seeding `provider`/`output_type` would leave transforms it never reads. const aiAgentFlowModules: FlowModule = { id, value: { type: 'aiagent', + ...(agentPath ? { agent: agentPath } : {}), tools: [], input_transforms: { - provider: { type: 'static', value: providerValue }, - output_type: { type: 'static', value: 'text' }, + ...(agentPath + ? {} + : { + provider: { type: 'static', value: providerValue }, + output_type: { type: 'static', value: 'text' } + }), user_message: { type: 'static', value: undefined } } } diff --git a/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte b/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte index c98c10b1a2..ef2dd38c39 100644 --- a/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte +++ b/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte @@ -165,7 +165,8 @@ kind: InsertKind, wsScript?: { path: string; summary: string; hash: string | undefined }, wsFlow?: { path: string; summary: string }, - inlineScript?: InlineScript + inlineScript?: InlineScript, + agentPath?: string ): Promise { let module = emptyModule(flowStateStore.val, flowStore.val, kind == 'flow') let state = emptyFlowModuleState() @@ -190,7 +191,7 @@ } else if (kind == 'branchall') { ;[module, state] = await createBranchAll(module.id) } else if (kind == 'aiagent') { - ;[module, state] = await createAiAgent(module.id) + ;[module, state] = await createAiAgent(module.id, agentPath) } else if (inlineScript) { const { language, kind, subkind, summary } = inlineScript ;[module, state] = await createInlineScriptModule(language, kind, subkind, module.id, summary) @@ -751,7 +752,8 @@ detail.kind as InsertKind, detail.script, detail.flow ? { path: detail.flow.path, summary: detail.flow.summary } : undefined, - detail.inlineScript + detail.inlineScript, + detail.agentPath ) const index = detail.index ?? 0 const extraModules: FlowModule[] = [module] diff --git a/frontend/src/lib/components/flows/map/InsertModuleInner.svelte b/frontend/src/lib/components/flows/map/InsertModuleInner.svelte index 0b3e77a848..3316463bbb 100644 --- a/frontend/src/lib/components/flows/map/InsertModuleInner.svelte +++ b/frontend/src/lib/components/flows/map/InsertModuleInner.svelte @@ -10,6 +10,11 @@ import ToggleHubWorkspaceQuick from '$lib/components/ToggleHubWorkspaceQuick.svelte' import TopLevelNode from '../pickers/TopLevelNode.svelte' import RefreshButton from '$lib/components/common/button/RefreshButton.svelte' + import Button from '$lib/components/common/button/Button.svelte' + import { ResourceService } from '$lib/gen' + import { workspaceStore } from '$lib/stores' + import type { FlowEditorContext } from '../types' + import { BotIcon, Loader2, Plus } from 'lucide-svelte' const dispatch = createEventDispatcher() interface Props { @@ -42,11 +47,44 @@ | 'approval' | 'flow' | 'failure' - | 'aisandbox' = $state(untrack(() => kind)) + | 'aisandbox' + | 'aiagent' = $state(untrack(() => kind)) let preFilter: 'all' | 'workspace' | 'hub' = $state('all') let loading = $state(false) let small = $derived(smallProp ?? (kind === 'preprocessor' || kind === 'failure')) + // Optional: this picker also renders outside the flow editor's context (the triggers wrapper). + const flowEditorContext = getContext('FlowEditorContext') + let ws = $derived(flowEditorContext?.opWorkspace?.() ?? $workspaceStore) + + let savedAgents = $state<{ path: string; description?: string }[]>([]) + let savedAgentsLoading = $state(false) + let savedAgentsWs: string | undefined = undefined + async function loadSavedAgents() { + if (!ws || savedAgentsWs === ws) { + return + } + savedAgentsLoading = true + try { + const rs = await ResourceService.listResource({ + workspace: ws, + resourceType: 'ai_agent', + perPage: 1000 + }) + savedAgents = rs.map((r) => ({ path: r.path, description: r.description })) + savedAgentsWs = ws + } catch { + savedAgents = [] + } finally { + savedAgentsLoading = false + } + } + let filteredAgents = $derived( + funcDesc + ? savedAgents.filter((a) => a.path.toLowerCase().includes(funcDesc.toLowerCase())) + : savedAgents + ) + let height = $state(0) let owners = $state([]) // Only the content-sized host (TriggersWrapper) grows past this. The fixed-height hosts top out @@ -81,6 +119,10 @@ {loading} onClick={() => { refreshCount.val += 1 + if (selectedKind === 'aiagent') { + savedAgentsWs = undefined + loadSavedAgents() + } }} />
@@ -184,9 +226,10 @@ {#if customUi?.aiAgent != false} { - dispatch('close') - dispatch('new', { kind: 'aiagent' }) + selectedKind = 'aiagent' + loadSavedAgents() }} /> {/if} @@ -203,7 +246,52 @@
{/if} - {#if selectedKind === 'aisandbox'} + {#if selectedKind === 'aiagent'} +
+ + {#if savedAgentsLoading} +
+ Loading saved agents +
+ {:else if filteredAgents.length > 0} +
Saved agents
+ {#each filteredAgents as agent (agent.path)} + + {/each} + {:else} +
+ {savedAgents.length > 0 + ? 'No saved agent matches this search' + : 'No saved agent in this workspace yet. Configure a blank one, then Save as reusable agent to reuse it.'} +
+ {/if} +
+ {:else if selectedKind === 'aisandbox'}
Promise diff --git a/frontend/src/lib/components/graph/graphBuilder.svelte.ts b/frontend/src/lib/components/graph/graphBuilder.svelte.ts index 450debbef4..3491cd3126 100644 --- a/frontend/src/lib/components/graph/graphBuilder.svelte.ts +++ b/frontend/src/lib/components/graph/graphBuilder.svelte.ts @@ -58,6 +58,8 @@ export type GraphEventHandlers = { inlineScript?: InlineScript script?: PathScript flow?: { path: string; summary: string } + /** Saved `ai_agent` resource the inserted agent step links to, for `kind: 'aiagent'`. */ + agentPath?: string isPreprocessor?: boolean }) => void deleteBranch: (detail: { id: string; index: number }, label: string) => void diff --git a/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte b/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte index 89e7c88c94..5678621062 100644 --- a/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte +++ b/frontend/src/lib/components/graph/renderers/edges/BaseEdge.svelte @@ -192,7 +192,8 @@ branch: data.branch, index: data.index, kind: e.detail.kind, - inlineScript: e.detail.inlineScript + inlineScript: e.detail.inlineScript, + agentPath: e.detail.agentPath }) }} on:pickScript={(e) => { diff --git a/frontend/src/lib/components/select/SelectDropdown.svelte b/frontend/src/lib/components/select/SelectDropdown.svelte index d9fc2dcf70..8dd952e5bf 100644 --- a/frontend/src/lib/components/select/SelectDropdown.svelte +++ b/frontend/src/lib/components/select/SelectDropdown.svelte @@ -145,17 +145,21 @@ }} > {@render startSnippet?.({ item, close: () => (open = false) })} - - {item.label || '\xa0'} - + +
+ + {item.label || '\xa0'} + + {#if item.subtitle} +
{item.subtitle}
+ {/if} +
{#if item.__is_create} {:else} {@render endSnippet?.({ item, close: () => (open = false) })} {/if} - {#if item.subtitle} -
{item.subtitle}
- {/if} {/each} diff --git a/frontend/src/routes/(root)/(logged)/resources/+page.svelte b/frontend/src/routes/(root)/(logged)/resources/+page.svelte index 5dbe7378fe..6f3fd1a6dc 100644 --- a/frontend/src/routes/(root)/(logged)/resources/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/resources/+page.svelte @@ -68,6 +68,7 @@ Plus, RotateCw, Save, + FlaskConical, SearchX, Shield, Trash, @@ -82,6 +83,7 @@ assetCanBeExplored } from '../../../../lib/components/ExploreAssetButton.svelte' import NoDirectDeployAlert from '$lib/components/NoDirectDeployAlert.svelte' + import AgentEvalModal from '$lib/components/aiEvals/AgentEvalModal.svelte' type ResourceW = ListableResource & { canWrite: boolean; marked?: string } type ResourceTypeW = ResourceType & { canWrite: boolean } @@ -133,6 +135,8 @@ let deleteConfirmedCallback: (() => void) | undefined = $state(undefined) let deleteIsLinked = $state(false) let deletePath = $state('') + let evalsOpen = $state(false) + let evalsAgentPath = $state(undefined) let loading = $state({ resources: true, types: true @@ -1262,6 +1266,18 @@ { + evalsAgentPath = path + evalsOpen = true + } + } + ] + : []), { displayName: 'Permissions', icon: Shield, @@ -1462,6 +1478,8 @@ + + Date: Mon, 24 Aug 2026 22:29:44 +0200 Subject: [PATCH 04/23] fix: patch sqlx so a cancelled BEGIN cannot poison a pooled connection (#10823) * fix: patch sqlx so a cancelled BEGIN cannot poison a pooled connection Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Qzqmh52NU8fB9RBQNNkJGt * test: drop the migration run and fixed sleep from the sqlx patch guard Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Qzqmh52NU8fB9RBQNNkJGt * test: ignore the sqlx patch guard by default and point at it from where sqlx is changed Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01Qzqmh52NU8fB9RBQNNkJGt --------- Co-authored-by: Claude Opus 5 (1M context) --- backend/Cargo.lock | 21 ++---- backend/Cargo.toml | 20 ++++++ .../tests/sqlx_begin_cancel_safe.rs | 72 +++++++++++++++++++ docs/validation.md | 1 + 4 files changed, 100 insertions(+), 14 deletions(-) create mode 100644 backend/windmill-common/tests/sqlx_begin_cancel_safe.rs diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 43ff602ec1..6b76f995f9 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -11849,8 +11849,7 @@ dependencies = [ [[package]] name = "sqlx" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" +source = "git+https://github.com/windmill-labs/sqlx?rev=6bdaee94fa62a01561125646da3f99eb341f2457#6bdaee94fa62a01561125646da3f99eb341f2457" dependencies = [ "sqlx-core", "sqlx-macros", @@ -11862,8 +11861,7 @@ dependencies = [ [[package]] name = "sqlx-core" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" +source = "git+https://github.com/windmill-labs/sqlx?rev=6bdaee94fa62a01561125646da3f99eb341f2457#6bdaee94fa62a01561125646da3f99eb341f2457" dependencies = [ "base64 0.22.1", "bigdecimal", @@ -11901,8 +11899,7 @@ dependencies = [ [[package]] name = "sqlx-macros" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" +source = "git+https://github.com/windmill-labs/sqlx?rev=6bdaee94fa62a01561125646da3f99eb341f2457#6bdaee94fa62a01561125646da3f99eb341f2457" dependencies = [ "proc-macro2", "quote", @@ -11914,8 +11911,7 @@ dependencies = [ [[package]] name = "sqlx-macros-core" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" +source = "git+https://github.com/windmill-labs/sqlx?rev=6bdaee94fa62a01561125646da3f99eb341f2457#6bdaee94fa62a01561125646da3f99eb341f2457" dependencies = [ "dotenvy", "either", @@ -11939,8 +11935,7 @@ dependencies = [ [[package]] name = "sqlx-mysql" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" +source = "git+https://github.com/windmill-labs/sqlx?rev=6bdaee94fa62a01561125646da3f99eb341f2457#6bdaee94fa62a01561125646da3f99eb341f2457" dependencies = [ "atoi", "base64 0.22.1", @@ -11984,8 +11979,7 @@ dependencies = [ [[package]] name = "sqlx-postgres" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" +source = "git+https://github.com/windmill-labs/sqlx?rev=6bdaee94fa62a01561125646da3f99eb341f2457#6bdaee94fa62a01561125646da3f99eb341f2457" dependencies = [ "atoi", "base64 0.22.1", @@ -12025,8 +12019,7 @@ dependencies = [ [[package]] name = "sqlx-sqlite" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" +source = "git+https://github.com/windmill-labs/sqlx?rev=6bdaee94fa62a01561125646da3f99eb341f2457#6bdaee94fa62a01561125646da3f99eb341f2457" dependencies = [ "atoi", "chrono", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 0bfa52d3e6..0e08b0f060 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -214,6 +214,26 @@ all_sqlx_features = ["all_languages", "enterprise", "enterprise_saml", "embeddin "windmill-git-sync/all_sqlx_features"] [patch.crates-io] +# v0.8.6 plus one commit: `Pool::begin` is not cancel-safe on Postgres. sqlx raises the +# transaction depth its rollback-on-drop guard keys on only *after* the BEGIN round trip, so +# a cancelled caller (a disconnecting API client, a `timeout`, an aborted task) leaves the +# session in a transaction nothing will end, and the pool hands that connection out again — +# every later query on it fails with 25P02 until max_lifetime recycles it 30 minutes on. +# Reported upstream in 2022 (launchbadge/sqlx#2054), fixed for SQLite only, and still present +# in 0.9.0. Drop this the moment upstream carries the fix. +# The whole family has to move together: `sqlx-postgres` depends on `sqlx-core` by path +# inside the sqlx workspace, so patching it alone leaves two incompatible `sqlx-core`s and +# `Postgres` stops implementing the `Database` the macros expect. +# Changing any of this — a bump, a rebase of the fork, dropping these lines — still compiles +# clean, so run the guard that actually checks the behaviour is still there: +# cargo test -p windmill-common --test sqlx_begin_cancel_safe -- --ignored +sqlx = { git = "https://github.com/windmill-labs/sqlx", rev = "6bdaee94fa62a01561125646da3f99eb341f2457" } +sqlx-core = { git = "https://github.com/windmill-labs/sqlx", rev = "6bdaee94fa62a01561125646da3f99eb341f2457" } +sqlx-macros = { git = "https://github.com/windmill-labs/sqlx", rev = "6bdaee94fa62a01561125646da3f99eb341f2457" } +sqlx-macros-core = { git = "https://github.com/windmill-labs/sqlx", rev = "6bdaee94fa62a01561125646da3f99eb341f2457" } +sqlx-postgres = { git = "https://github.com/windmill-labs/sqlx", rev = "6bdaee94fa62a01561125646da3f99eb341f2457" } +sqlx-mysql = { git = "https://github.com/windmill-labs/sqlx", rev = "6bdaee94fa62a01561125646da3f99eb341f2457" } +sqlx-sqlite = { git = "https://github.com/windmill-labs/sqlx", rev = "6bdaee94fa62a01561125646da3f99eb341f2457" } object_store = { git = "https://github.com/apache/arrow-rs-object-store", rev = "36752c975d4f29e20b57c91f81a10872dcd48ae7" } # Use tiberius main branch for libgssapi 0.8.1 fix (https://github.com/prisma/tiberius/issues/343) tiberius = { git = "https://github.com/prisma/tiberius", rev = "59db57960a14b422fb3a1309aa4aa47880896ff8" } diff --git a/backend/windmill-common/tests/sqlx_begin_cancel_safe.rs b/backend/windmill-common/tests/sqlx_begin_cancel_safe.rs new file mode 100644 index 0000000000..1f387984b0 --- /dev/null +++ b/backend/windmill-common/tests/sqlx_begin_cancel_safe.rs @@ -0,0 +1,72 @@ +//! Guards the `sqlx` entries in `[patch.crates-io]` — `backend/Cargo.toml` carries the why. +//! Dropping the patch still compiles, so a test is what notices. +//! +//! Ignored by default: it only has something to say when the sqlx dependency moves, and it +//! spends a couple of seconds waiting on a deliberately slow round trip. Run it whenever you +//! touch sqlx — a version bump, a change to the patch entries, a fork rebase: +//! +//! ```text +//! cargo test -p windmill-common --test sqlx_begin_cancel_safe -- --ignored +//! ``` + +use sqlx::{Connection, PgConnection, Pool, Postgres}; +use std::time::{Duration, Instant}; + +#[sqlx::test] +#[ignore = "run with --ignored after any sqlx bump or change to [patch.crates-io]"] +async fn begin_cancelled_mid_round_trip_leaves_no_open_transaction(db: Pool) { + // One connection, so the session inspected below is the one the cancelled begin used. + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .min_connections(0) + .connect_with((*db.connect_options()).clone()) + .await + .expect("failed to build pool"); + let pid: i32 = sqlx::query_scalar("SELECT pg_backend_pid()") + .fetch_one(&pool) + .await + .unwrap(); + + // A plain `BEGIN` answers in well under a millisecond, which is too narrow to cancel + // reliably; appending a sleep widens the round trip and runs through the same + // `PgTransactionManager::begin` the patch fixes. + let cancelled = tokio::time::timeout( + Duration::from_millis(300), + pool.begin_with("BEGIN; SELECT pg_sleep(2);"), + ) + .await; + assert!(cancelled.is_err(), "the begin must not have completed"); + + let mut admin = PgConnection::connect_with(&(*db.connect_options()).clone()) + .await + .expect("failed to open an observing connection"); + + // sqlx only flushes the queued ROLLBACK once the abandoned statement has answered, so + // wait for the session to stop running rather than sleeping a fixed time a loaded runner + // could overshoot. + let deadline = Instant::now() + Duration::from_secs(30); + let state = loop { + let state: String = sqlx::query_scalar("SELECT state FROM pg_stat_activity WHERE pid = $1") + .bind(pid) + .fetch_optional(&mut admin) + .await + .unwrap() + .flatten() + .unwrap_or_default(); + if state != "active" || Instant::now() >= deadline { + break state; + } + tokio::time::sleep(Duration::from_millis(100)).await; + }; + + assert!( + !state.starts_with("idle in transaction"), + "connection returned to the pool still inside a transaction (state {state:?}) — is \ + the sqlx patch in backend/Cargo.toml still applied?" + ); + + sqlx::query_scalar::<_, i32>("SELECT 1") + .fetch_one(&pool) + .await + .expect("pool must still serve queries"); +} diff --git a/docs/validation.md b/docs/validation.md index 050eacd75b..74e9cf5677 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -16,6 +16,7 @@ After making changes, run the appropriate checks and fix all errors before consi | Multiple gated modules | `cargo check --features enterprise,parquet` | Combine only the flags you need | | API route changes | `cargo check` | Then update `openapi.yaml` and run `npm run generate-backend-client` | | Database migrations | `cargo check` | Test migration applies cleanly with `sqlx migrate run` | +| The `sqlx` dependency (version bump, `[patch.crates-io]` entries, fork rebase) | `cargo test -p windmill-common --test sqlx_begin_cancel_safe -- --ignored` | Windmill runs a patched `sqlx`: upstream's `Pool::begin` is not cancel-safe on Postgres, and a cancelled one poisons the pooled connection for 30 minutes. Losing the patch still compiles, so this ignored test is the only thing that notices. `backend/Cargo.toml` has the detail | **Never** use `--features all_sqlx_features` — it compiles everything and is very slow. Check `backend/Cargo.toml` `[features]` to find the right flags. From 541b6c849657d13fed3580407a00a996e891ad9e Mon Sep 17 00:00:00 2001 From: hugocasa Date: Mon, 24 Aug 2026 22:30:45 +0200 Subject: [PATCH 05/23] fix: keep ai chat messages when leaving the page mid-generation (#10809) * fix: persist ai chat turns mid-generation so leaving the page keeps them * fix: stop chat checkpoints once the turn commits, keep streamed text visible * fix: checkpoint streamed answers as they grow and keep half-run tool batches * fix: checkpoint text as received so a backgrounded tab keeps capturing * fix: keep buffered tool screenshots in mid-batch chat checkpoints * fix: decide committed-text at the flush site, condense checkpoint comments * fix: checkpoint only live streamed text, never text the parser owns * fix: don't swap the chat transcript out from under a running turn * fix: close the pre-loading window in the conversation-switch guard --- .../copilot/chat/AIChatDisplay.svelte | 6 +- .../copilot/chat/AIChatManager.svelte.ts | 181 ++++++++- .../copilot/chat/AIChatManager.test.ts | 362 ++++++++++++++++++ .../components/copilot/chat/chatLoop.test.ts | 26 +- .../lib/components/copilot/chat/chatLoop.ts | 32 ++ .../src/lib/components/copilot/chat/shared.ts | 25 +- .../copilot/chat/typewriterReveal.test.ts | 20 + .../copilot/chat/typewriterReveal.ts | 9 + 8 files changed, 635 insertions(+), 26 deletions(-) diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index 2dd8f7bef7..9b55bd9217 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -611,7 +611,11 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. -->
{#each pastChats as chat (chat.id)} @@ -135,6 +215,8 @@
{#if icon} {@const SvelteComponent = icon} - + {/if} {#if !isCollapsed} diff --git a/frontend/src/lib/components/sidebar/OperatorMenu.svelte b/frontend/src/lib/components/sidebar/OperatorMenu.svelte index e6519ffafc..0914068607 100644 --- a/frontend/src/lib/components/sidebar/OperatorMenu.svelte +++ b/frontend/src/lib/components/sidebar/OperatorMenu.svelte @@ -40,7 +40,6 @@ import type { FavoriteKind } from './FavoriteMenu.svelte' let darkMode: boolean = $state(false) let showExtraTriggers = $state(false) - let menubarEl: HTMLElement | undefined = $state() interface Props { isCollapsed?: boolean @@ -171,211 +170,227 @@ ) - -
{ - const btn = menubarEl?.querySelector('[data-melt-menubar-trigger]') - if (btn instanceof HTMLElement) btn.click() - }} -> - - {#snippet children({ createMenu })} - (showExtraTriggers = false)}> - {#snippet triggr({ trigger })} - - {/snippet} - {#snippet children({ item })} -
- {#each favoriteLinks ?? [] as favorite (favorite.href)} - - - {#if favorite.kind == 'script'} - - {:else if favorite.kind == 'flow'} - - {:else if favorite.kind == 'app' || favorite.kind == 'raw_app'} - - {:else if favorite.kind == 'asset'} - - {/if} - - - {favorite.label} - - - {/each} + + {#snippet children({ createMenu })} + (showExtraTriggers = false)} + > + {#snippet triggr({ trigger, pinned })} + + {/snippet} + {#snippet children({ item })} +
+ {#each favoriteLinks ?? [] as favorite (favorite.href)} + + + {#if favorite.kind == 'script'} + + {:else if favorite.kind == 'flow'} + + {:else if favorite.kind == 'app' || favorite.kind == 'raw_app'} + + {:else if favorite.kind == 'asset'} + + {/if} + + + {favorite.label} + + + {/each} +
+ + {#each mainMenuLinks as menuLink (menuLink.href ?? menuLink.label)} + + {/each} + +
+
+ + + Account settings +
- {#each mainMenuLinks as menuLink (menuLink.href ?? menuLink.label)} - - {/each} - -
-
- - - Account settings - -
- -
- { - if (!document.documentElement.classList.contains('dark')) { - document.documentElement.classList.add('dark') - window.localStorage.setItem('dark-mode', 'dark') - } else { - document.documentElement.classList.remove('dark') - window.localStorage.setItem('dark-mode', 'light') - } - }} - lightMode - class={twMerge( - 'w-full flex gap-3.5 px-2 py-2', - sidebarClasses.hoverBg, - sidebarClasses.text - )} - {item} - > - {#if darkMode} - - {:else} - - {/if} - Switch theme - - clearWorkspaceFromStorage()} - lightMode - class={twMerge( - 'flex gap-3.5 px-2 py-2', - sidebarClasses.hoverBg, - sidebarClasses.text - )} - {item} - > - - All workspaces - - - {#if $superadmin} - - - Instance settings - +
+ { + if (!document.documentElement.classList.contains('dark')) { + document.documentElement.classList.add('dark') + window.localStorage.setItem('dark-mode', 'dark') + } else { + document.documentElement.classList.remove('dark') + window.localStorage.setItem('dark-mode', 'light') + } + }} + lightMode + class={twMerge( + 'w-full flex gap-3.5 px-2 py-2', + 'transition-colors', + sidebarClasses.text, + 'data-[highlighted]:bg-surface-hover data-[highlighted]:text-primary' + )} + {item} + > + {#if darkMode} + + {:else} + {/if} + Switch theme + + clearWorkspaceFromStorage()} + lightMode + class={twMerge( + 'flex gap-3.5 px-2 py-2', + 'transition-colors', + sidebarClasses.text, + 'data-[highlighted]:bg-surface-hover data-[highlighted]:text-primary' + )} + {item} + > + + All workspaces + + {#if $superadmin} logout()} + href="#superadmin-settings" class={twMerge( - 'flex flex-row gap-3.5 items-center px-2 py-2 w-full', - 'text-primary text-xs', - 'hover:bg-surface-hover cursor-pointer', + 'flex flex-row gap-3.5 items-center px-2 py-2 ', + 'text-secondary text-xs', + 'cursor-pointer', 'data-[highlighted]:bg-surface-hover data-[highlighted]:text-primary' )} {item} > - - Sign out + + Instance settings -
-
- {#snippet renderSecondMenuLinks(menuLinks: SecondMenuLink[])} - {#each menuLinks as menuLink (menuLink.href ?? menuLink.label)} - - {menuLink.label} - - {/each} - {/snippet} - {#if secondMenuLinks.length || secondMenuTriggerLinks.length || extraTriggerLinks.length} -
- {#if secondMenuLinks.length}
{@render renderSecondMenuLinks(secondMenuLinks)}
{/if} - {#if secondMenuTriggerLinks.length}
{@render renderSecondMenuLinks(secondMenuTriggerLinks)}
{/if} - {#if extraTriggerLinks.length}
- -
{ - e.stopPropagation() - showExtraTriggers = !showExtraTriggers - }} + {/if} + + logout()} + class={twMerge( + 'flex flex-row gap-3.5 items-center px-2 py-2 w-full', + 'text-primary text-xs', + 'cursor-pointer', + 'data-[highlighted]:bg-surface-hover data-[highlighted]:text-primary' + )} + {item} + > + + Sign out + +
+
+ {#snippet renderSecondMenuLinks(menuLinks: SecondMenuLink[])} + {#each menuLinks as menuLink (menuLink.href ?? menuLink.label)} + + {menuLink.label} + + {/each} + {/snippet} + {#if secondMenuLinks.length || secondMenuTriggerLinks.length || extraTriggerLinks.length} +
+ {#if secondMenuLinks.length}
{@render renderSecondMenuLinks(secondMenuLinks)}
{/if} + {#if secondMenuTriggerLinks.length}
{@render renderSecondMenuLinks(secondMenuTriggerLinks)}
{/if} + {#if extraTriggerLinks.length}
+ +
{ + // This row expands the list below it instead of acting on the selection, and + // melt keeps the menu open only for a click it sees as defaultPrevented. + // Svelte delegates onclick to the root, which runs after melt's own listener, + // and a capture listener on the item itself would be ordered only by + // registration, so an ancestor's capture phase is what reliably wins. + e.preventDefault() + showExtraTriggers = !showExtraTriggers + }} + > + More triggers -
- {#if showExtraTriggers} - {#each extraTriggerLinks as menuLink (menuLink.href)} - - {menuLink.label} - - {/each} - {/if} -
{/if} -
- {/if} - {#if $enterpriseLicense} - - {/if} -
+ +
+ {#if showExtraTriggers} + {#each extraTriggerLinks as menuLink (menuLink.href)} + + {menuLink.label} + + {/each} + {/if} +
{/if} +
+ {/if} + {#if $enterpriseLicense} + + {/if}
- {/snippet} -
- {/snippet} - -
+
+ {/snippet} + + {/snippet} + From 46c363ffa4bc72bef6b367ece4bdbeef5e0eadc9 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Wed, 26 Aug 2026 00:49:13 +0200 Subject: [PATCH 21/23] fix: require admin on workspace tarball settings export (#10817) * fix: require admin on workspace tarball settings export Co-Authored-By: Claude Opus 5 * fix: name the refused flag in the settings export error Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- backend/tests/workspace_export.rs | 76 ++++++++++++++++++- backend/windmill-api/src/workspaces_export.rs | 18 +++-- 2 files changed, 86 insertions(+), 8 deletions(-) diff --git a/backend/tests/workspace_export.rs b/backend/tests/workspace_export.rs index 8568b8ff24..5894ca8394 100644 --- a/backend/tests/workspace_export.rs +++ b/backend/tests/workspace_export.rs @@ -1,6 +1,6 @@ use sqlx::postgres::Postgres; use sqlx::Pool; -use windmill_test_utils::{initialize_tracing, ApiServer}; +use windmill_test_utils::{initialize_tracing, set_jwt_secret, ApiServer}; /// Integration test: exercises every explicit-column query in `tarball_workspace`. /// @@ -287,3 +287,77 @@ async fn test_tarball_export_gates_values_on_item_scopes(db: Pool) -> Ok(()) } + +/// `settings.json` carries the admin-managed integration config that `get_settings` +/// is admin-only for (the webhook URL, ai_config, git_sync, handler extra_args), so +/// `include_settings` takes the same admin check as `get_settings` rather than +/// riding on the route's `workspaces:read`. Git sync exports settings through the +/// same route, so the gate must still admit its system identity. +#[sqlx::test(fixtures("base"))] +async fn test_tarball_export_settings_are_admin_only(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + set_jwt_secret().await; + let server = ApiServer::start(db.clone()).await?; + let base_url = format!("http://localhost:{}", server.addr.port()); + + sqlx::query( + r#"UPDATE workspace_settings + SET webhook = 'https://hook.example/?token=WEBHOOK_SECRET', + ai_config = '{"providers":{"openai":{"api_key":"AI_CONFIG_SECRET"}}}'::jsonb + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + + let export = async |token: &str| -> anyhow::Result<(u16, String)> { + let resp = reqwest::Client::new() + .get(format!( + "{base_url}/api/w/test-workspace/workspaces/tarball?include_settings=true&settings_version=v2" + )) + .bearer_auth(token) + .send() + .await?; + let status = resp.status().as_u16(); + // Lossy: a successful export is a tar, not UTF-8. Only the values matter here. + Ok(( + status, + String::from_utf8_lossy(&resp.bytes().await?).into_owned(), + )) + }; + + // SECRET_TOKEN_2 belongs to test-user-2, a non-admin member of test-workspace. + let (status, body) = export("SECRET_TOKEN_2").await?; + assert_eq!(status, 403, "non-admin exported settings: {body}"); + + let (status, body) = export("SECRET_TOKEN").await?; + assert_eq!(status, 200, "admin denied settings: {body}"); + assert!( + body.contains("WEBHOOK_SECRET") && body.contains("AI_CONFIG_SECRET"), + "admin got no settings" + ); + + // Git sync pushes the workspace to the repo by exporting it under + // `superadmin_sync@windmill.dev`, which belongs to no workspace: the job token + // it runs with is the export's only admin claim. + let sync_email = windmill_common::users::SUPERADMIN_SYNC_EMAIL; + let sync_token = windmill_common::auth::create_token_for_owner( + &db, + "test-workspace", + sync_email, + "git-sync", + 300, + sync_email, + &uuid::Uuid::new_v4(), + None, + None, + ) + .await?; + let (status, body) = export(&sync_token).await?; + assert_eq!(status, 200, "git-sync identity denied settings: {body}"); + assert!( + body.contains("WEBHOOK_SECRET"), + "git-sync identity got no settings" + ); + + Ok(()) +} diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 9116021626..05cd354cd4 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -643,6 +643,16 @@ pub(crate) async fn tarball_workspace( windmill_api_auth::forbid_scoped_token_workspace_key(&authed)?; } + // settings.json carries the admin-managed integration config that `get_settings` + // is admin-only for (ai_config, the webhook URL, git_sync, handler extra_args), + // so it takes the same check. Not a per-field redaction: fields silently dropped + // from settings.json come back as null on the next `wmill sync push`. + if include_settings.unwrap_or(false) && !authed.is_admin { + return Err(Error::PermissionDenied( + "include_settings requires workspace admin".to_string(), + )); + } + // The route is gated by workspaces:read, but the tarball also carries the item // values that the per-item routes gate on their own domain (get_resource_value, // get_variable). A whole-workspace export cannot be confined to a path, so it @@ -1626,13 +1636,7 @@ pub(crate) async fn tarball_workspace( slack_name: row.slack_name.clone(), slack_command_script: row.slack_command_script.clone(), slack_oauth_client_id: row.slack_oauth_client_id.clone(), - // Mirror the non-admin redaction in `get_settings`: the OAuth - // client secret is admin-only and must not leak via tarball. - slack_oauth_client_secret: if authed.is_admin { - row.slack_oauth_client_secret.clone() - } else { - None - }, + slack_oauth_client_secret: row.slack_oauth_client_secret.clone(), }; serde_json::to_value(settings) .map(|v| serde_json::to_string_pretty(&v).ok()) From 72763c9ba582c4e55b0f1ebfe17fe1435353bdaa Mon Sep 17 00:00:00 2001 From: Guilhem Date: Wed, 26 Aug 2026 00:49:48 +0200 Subject: [PATCH 22/23] tighten spacing between login email and password fields (#10811) Claude-Session: https://claude.ai/code/session_01BmEVHF8afJmgv6saBRYN6w Co-authored-by: Claude Opus 5 (1M context) --- frontend/src/lib/components/Login.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/lib/components/Login.svelte b/frontend/src/lib/components/Login.svelte index 9c4666bf4c..d1dc3ec7a1 100644 --- a/frontend/src/lib/components/Login.svelte +++ b/frontend/src/lib/components/Login.svelte @@ -732,7 +732,7 @@ contact@windmill.dev

{/if} -
+
From ffdf17ef8dc5575dd92d62d0d0ba887c1e378576 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Wed, 26 Aug 2026 08:23:25 +0200 Subject: [PATCH 23/23] fix: force HTTP router rebuild on trigger-change notification (#10849) * fix: force HTTP router rebuild on trigger-change notification Co-Authored-By: Claude Opus 5 * fix: coalesce http trigger change events into one forced rebuild Co-Authored-By: Claude Opus 5 * fix: retry the coalesced http router rebuild when it fails Co-Authored-By: Claude Opus 5 * fix: mark http routers stale when a forced rebuild fails Co-Authored-By: Claude Opus 5 * fix: keep the router invalidation across an in-flight rebuild Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 --- backend/src/main.rs | 50 +++++++++------ .../windmill-api/src/triggers/http/handler.rs | 4 +- backend/windmill-trigger-http/src/lib.rs | 41 ++++++++++-- .../tests/refresh_routers.rs | 62 +++++++++++++++++++ 4 files changed, 130 insertions(+), 27 deletions(-) create mode 100644 backend/windmill-trigger-http/tests/refresh_routers.rs diff --git a/backend/src/main.rs b/backend/src/main.rs index 10d1434f98..f8b150383a 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -1434,21 +1434,30 @@ Windmill Community Edition {GIT_VERSION} // Poll for new events from notify_event table match windmill_common::notify_events::poll_notify_events(&db, last_event_id).await { Ok(events) => { + let mut http_trigger_change_handled = false; for event in events { if !*windmill_common::QUIET_LOGS { tracing::info!("Processing notify event: channel={}, payload={}", event.channel, event.payload); } - process_notify_event( - &event.channel, - &event.payload, - &db, - &conn, - &tx, - server_mode, - worker_mode, - #[cfg(feature = "parquet")] - disable_s3_store, - ).await; + let is_http_trigger_change = event.channel == "notify_http_trigger_change"; + // Every changed http_trigger row emits its own event and each one forces + // a full router rebuild, but the batch's first successful rebuild already + // read every row the batch committed. A failed rebuild leaves the flag + // clear so the next event in the batch retries it. + if !(is_http_trigger_change && http_trigger_change_handled) { + let handled = process_notify_event( + &event.channel, + &event.payload, + &db, + &conn, + &tx, + server_mode, + worker_mode, + #[cfg(feature = "parquet")] + disable_s3_store, + ).await; + http_trigger_change_handled |= is_http_trigger_change && handled; + } last_event_id = last_event_id.max(event.id); } } @@ -1670,6 +1679,9 @@ Windmill Community Edition {GIT_VERSION} /// Process a single notify event from the polling-based event system. /// This replaces the old PgListener notification handling. +/// +/// Returns `false` when the event still needs handling. Only the HTTP router rebuild reports +/// that, because the poll loop coalesces those events and must not swallow the retry. #[allow(unused_variables)] async fn process_notify_event( channel: &str, @@ -1680,7 +1692,7 @@ async fn process_notify_event( server_mode: bool, worker_mode: bool, #[cfg(feature = "parquet")] disable_s3_store: bool, -) { +) -> bool { match channel { "notify_config_change" => { if payload == "server" && server_mode { @@ -1825,17 +1837,14 @@ async fn process_notify_event( #[cfg(feature = "http_trigger")] "notify_http_trigger_change" => { tracing::info!("HTTP trigger change detected: {}", payload); - match windmill_api::triggers::http::refresh_routers(db).await { - Ok((true, _)) => { + match windmill_api::triggers::http::refresh_routers(db, true).await { + Ok(_) => { tracing::info!("Refreshed HTTP routers (trigger change)"); } - Ok((false, _)) => { - tracing::warn!( - "Should have refreshed HTTP routers (trigger change) but did not" - ); - } Err(err) => { tracing::error!("Error refreshing HTTP routers (trigger change): {err:#}"); + windmill_api::triggers::http::invalidate_routers(); + return false; } }; } @@ -2059,7 +2068,7 @@ async fn process_notify_event( tracing::error!(error = %e, "Could not reload http route workspaced route setting"); } #[cfg(feature = "http_trigger")] - match windmill_api::triggers::http::refresh_routers(db).await { + match windmill_api::triggers::http::refresh_routers(db, false).await { Ok((true, _)) => { tracing::info!( "Refreshed HTTP routers (http workspaced route setting change)" @@ -2179,6 +2188,7 @@ async fn process_notify_event( tracing::warn!("Unknown notification channel: {}", channel); } } + true } fn display_config(envs: &[&str]) { diff --git a/backend/windmill-api/src/triggers/http/handler.rs b/backend/windmill-api/src/triggers/http/handler.rs index 37442c8111..e58aab44a2 100644 --- a/backend/windmill-api/src/triggers/http/handler.rs +++ b/backend/windmill-api/src/triggers/http/handler.rs @@ -123,7 +123,9 @@ async fn get_http_route_trigger( let routers_cache = if routers_cache.routers.is_empty() { tracing::warn!("HTTP routers are not loaded, loading from db"); - let (_, routers_cache) = refresh_routers(db).await?; + // refresh_routers takes the write lock, so holding this read guard across it deadlocks. + drop(routers_cache); + let (_, routers_cache) = refresh_routers(db, false).await?; routers_cache } else { routers_cache diff --git a/backend/windmill-trigger-http/src/lib.rs b/backend/windmill-trigger-http/src/lib.rs index f89247dd16..b78276c28d 100644 --- a/backend/windmill-trigger-http/src/lib.rs +++ b/backend/windmill-trigger-http/src/lib.rs @@ -1,4 +1,5 @@ use std::collections::HashMap; +use std::sync::atomic::{AtomicU64, Ordering}; use quick_cache::sync::Cache; use serde::{Deserialize, Serialize}; @@ -27,9 +28,12 @@ lazy_static::lazy_static! { pub static ref HTTP_ROUTERS_CACHE: RwLock = RwLock::new(RoutersCache { routers: HashMap::new(), version: 0, + invalidations: 0, }); } +static HTTP_ROUTERS_INVALIDATIONS: AtomicU64 = AtomicU64::new(0); + #[derive(Debug, Deserialize, Clone)] pub struct TriggerRoute { pub path: String, @@ -56,6 +60,10 @@ pub struct TriggerRoute { pub struct RoutersCache { pub routers: HashMap>, pub version: i64, + /// `HTTP_ROUTERS_INVALIDATIONS` as of the moment these rows were read. A rebuild that + /// started before an invalidation publishes a count behind the current one, which is what + /// stops it from passing its own stale rows off as covering that invalidation. + invalidations: u64, } #[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Copy, Hash, Eq, PartialEq)] @@ -223,12 +231,24 @@ pub fn validate_authentication_method( } } -pub async fn refresh_routers(db: &DB) -> Result<(bool, RwLockReadGuard<'_, RoutersCache>)> { +/// `force` rebuilds unconditionally. `nextval` on `http_trigger_version_seq` runs inside the +/// writing transaction and sequences are non-transactional, so another session can cache the +/// bumped version against still-uncommitted rows, after which every version-gated refresh is a +/// no-op. Force when reacting to a bump that could have been observed before its own rows were. +pub async fn refresh_routers( + db: &DB, + force: bool, +) -> Result<(bool, RwLockReadGuard<'_, RoutersCache>)> { + let invalidations = HTTP_ROUTERS_INVALIDATIONS.load(Ordering::Relaxed); let version = sqlx::query_scalar!("SELECT last_value FROM http_trigger_version_seq",) .fetch_one(db) .await?; let routers_cache = HTTP_ROUTERS_CACHE.read().await; - if routers_cache.version == 0 || version > routers_cache.version { + if force + || routers_cache.version == 0 + || version > routers_cache.version + || invalidations != routers_cache.invalidations + { drop(routers_cache); let mut routers = HashMap::new(); @@ -274,7 +294,8 @@ pub async fn refresh_routers(db: &DB) -> Result<(bool, RwLockReadGuard<'_, Route .await?; let mut router = matchit::Router::new(); - let http_route_workspaced = HTTP_ROUTE_WORKSPACED_ROUTE.load(std::sync::atomic::Ordering::Relaxed); + let http_route_workspaced = + HTTP_ROUTE_WORKSPACED_ROUTE.load(std::sync::atomic::Ordering::Relaxed); for trigger in triggers { let full_path = @@ -306,7 +327,7 @@ pub async fn refresh_routers(db: &DB) -> Result<(bool, RwLockReadGuard<'_, Route } let mut routers_cache = HTTP_ROUTERS_CACHE.write().await; - *routers_cache = RoutersCache { routers, version }; + *routers_cache = RoutersCache { routers, version, invalidations }; Ok((true, routers_cache.downgrade())) } else { @@ -315,11 +336,19 @@ pub async fn refresh_routers(db: &DB) -> Result<(bool, RwLockReadGuard<'_, Route } } +/// Record that the cache no longer covers everything committed, so the next refresh rebuilds +/// whatever the version says. The routes already loaded keep being served in the meantime. Use +/// after a forced refresh fails: its change is inside the cached version, so nothing else would +/// retry it. +pub fn invalidate_routers() { + HTTP_ROUTERS_INVALIDATIONS.fetch_add(1, Ordering::Relaxed); +} + pub async fn refresh_routers_loop( db: &DB, mut killpill_rx: tokio::sync::broadcast::Receiver<()>, ) -> () { - match refresh_routers(db).await { + match refresh_routers(db, false).await { Ok(_) => { tracing::info!("Loaded HTTP routers"); } @@ -335,7 +364,7 @@ pub async fn refresh_routers_loop( break; } _ = tokio::time::sleep(std::time::Duration::from_secs(60)) => { - match refresh_routers(&db).await { + match refresh_routers(&db, false).await { Ok((true, _)) => { tracing::info!("Refreshed HTTP routers"); } diff --git a/backend/windmill-trigger-http/tests/refresh_routers.rs b/backend/windmill-trigger-http/tests/refresh_routers.rs new file mode 100644 index 0000000000..f47da2ee0e --- /dev/null +++ b/backend/windmill-trigger-http/tests/refresh_routers.rs @@ -0,0 +1,62 @@ +use sqlx::{Pool, Postgres}; +use windmill_trigger_http::{invalidate_routers, refresh_routers, HttpMethod, RoutersCache}; + +async fn insert_trigger(db: &Pool, path: &str, route_path: &str) { + sqlx::query( + "INSERT INTO http_trigger ( + path, route_path, route_path_key, script_path, is_flow, workspace_id, edited_by, + permissioned_as, http_method, authentication_method, request_type, is_static_website, + workspaced_route, wrap_body, raw_string, mode + ) VALUES ($1, $2, $2, 'f/test/handler', false, 'test-workspace', 'test-user', + 'u/test-user', 'get', 'none', 'async', false, false, false, false, 'enabled')", + ) + .bind(path) + .bind(route_path) + .execute(db) + .await + .expect("insert http_trigger"); +} + +fn routes(cache: &RoutersCache, path: &str) -> bool { + cache.routers[&HttpMethod::Get].at(path).is_ok() +} + +// A trigger row can commit without advancing http_trigger_version_seq past what the cache +// already holds, because `nextval` runs ahead of the commit it belongs to. The version gate +// cannot see such a row; only forcing, or an invalidation, recovers the route. +#[sqlx::test(migrations = "../migrations")] +async fn rebuilds_a_change_the_cached_version_does_not_cover(db: Pool) { + insert_trigger(&db, "f/test/first", "first").await; + let (rebuilt, cache) = refresh_routers(&db, false).await.unwrap(); + assert!(rebuilt); + assert!(routes(&cache, "/first")); + drop(cache); + + insert_trigger(&db, "f/test/second", "second").await; + + let (rebuilt, cache) = refresh_routers(&db, false).await.unwrap(); + assert!(!rebuilt, "an unchanged version must not rebuild"); + assert!(!routes(&cache, "/second")); + drop(cache); + + let (rebuilt, cache) = refresh_routers(&db, true).await.unwrap(); + assert!(rebuilt, "force must rebuild whatever the version says"); + assert!(routes(&cache, "/second")); + drop(cache); + + // A forced refresh that failed leaves its change inside the cached version, so the periodic + // version-gated refresh has to rebuild on the invalidation alone. + insert_trigger(&db, "f/test/third", "third").await; + invalidate_routers(); + + let (rebuilt, cache) = refresh_routers(&db, false).await.unwrap(); + assert!( + rebuilt, + "an invalidation must rebuild through the version gate" + ); + assert!(routes(&cache, "/third")); + drop(cache); + + let (rebuilt, _) = refresh_routers(&db, false).await.unwrap(); + assert!(!rebuilt, "a served invalidation must not rebuild forever"); +}