diff --git a/backend/.sqlx/query-24fcc2b69f30953915f0cbf246e1c19b2310075a644d81c21784c991e52b4001.json b/backend/.sqlx/query-24fcc2b69f30953915f0cbf246e1c19b2310075a644d81c21784c991e52b4001.json new file mode 100644 index 0000000000..a2cd96857a --- /dev/null +++ b/backend/.sqlx/query-24fcc2b69f30953915f0cbf246e1c19b2310075a644d81c21784c991e52b4001.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_token_usage (workspace_id, email, provider, model, session_id, input_tokens, cache_read_tokens, cache_write_tokens, output_tokens, reported_cost_nano_usd, requests)\n SELECT $1, $2, * FROM UNNEST($3::text[], $4::text[], $5::text[], $6::bigint[], $7::bigint[], $8::bigint[], $9::bigint[], $10::bigint[], $11::bigint[])\n ON CONFLICT (workspace_id, day, email, provider, model, session_id)\n DO UPDATE SET\n input_tokens = ai_token_usage.input_tokens + EXCLUDED.input_tokens,\n cache_read_tokens = ai_token_usage.cache_read_tokens + EXCLUDED.cache_read_tokens,\n cache_write_tokens = ai_token_usage.cache_write_tokens + EXCLUDED.cache_write_tokens,\n output_tokens = ai_token_usage.output_tokens + EXCLUDED.output_tokens,\n reported_cost_nano_usd = CASE\n WHEN EXCLUDED.reported_cost_nano_usd IS NULL\n THEN ai_token_usage.reported_cost_nano_usd\n ELSE COALESCE(ai_token_usage.reported_cost_nano_usd, 0)\n + EXCLUDED.reported_cost_nano_usd\n END,\n requests = ai_token_usage.requests + EXCLUDED.requests,\n updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "TextArray", + "TextArray", + "TextArray", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array" + ] + }, + "nullable": [] + }, + "hash": "24fcc2b69f30953915f0cbf246e1c19b2310075a644d81c21784c991e52b4001" +} diff --git a/backend/.sqlx/query-6a17a1dfeb75808e5d0726f1f8bd573168161abe350b047402cda4ac4a4d13e4.json b/backend/.sqlx/query-6a17a1dfeb75808e5d0726f1f8bd573168161abe350b047402cda4ac4a4d13e4.json new file mode 100644 index 0000000000..6ced022760 --- /dev/null +++ b/backend/.sqlx/query-6a17a1dfeb75808e5d0726f1f8bd573168161abe350b047402cda4ac4a4d13e4.json @@ -0,0 +1,74 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (CASE $3::text\n WHEN 'day' THEN day::text\n WHEN 'user' THEN email\n ELSE ''\n END) AS \"key!\",\n provider AS \"provider!\",\n model AS \"model!\",\n SUM(input_tokens)::bigint AS \"input_tokens!\",\n SUM(cache_read_tokens)::bigint AS \"cache_read_tokens!\",\n SUM(cache_write_tokens)::bigint AS \"cache_write_tokens!\",\n SUM(output_tokens)::bigint AS \"output_tokens!\",\n SUM(reported_cost_nano_usd)::bigint AS \"reported_cost_nano_usd\",\n SUM(requests)::bigint AS \"requests!\"\n FROM ai_token_usage\n WHERE workspace_id = $1 AND day > CURRENT_DATE - $2::int\n AND ($5::text IS NULL OR email = $5)\n GROUP BY 1, provider, model\n ORDER BY SUM(input_tokens + cache_read_tokens + cache_write_tokens + output_tokens) DESC\n LIMIT $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "key!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "provider!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "model!", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "input_tokens!", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "cache_read_tokens!", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "cache_write_tokens!", + "type_info": "Int8" + }, + { + "ordinal": 6, + "name": "output_tokens!", + "type_info": "Int8" + }, + { + "ordinal": 7, + "name": "reported_cost_nano_usd", + "type_info": "Int8" + }, + { + "ordinal": 8, + "name": "requests!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Int4", + "Text", + "Int8", + "Text" + ] + }, + "nullable": [ + null, + false, + false, + null, + null, + null, + null, + null, + null + ] + }, + "hash": "6a17a1dfeb75808e5d0726f1f8bd573168161abe350b047402cda4ac4a4d13e4" +} diff --git a/backend/migrations/20260813164338_ai_token_usage.down.sql b/backend/migrations/20260813164338_ai_token_usage.down.sql new file mode 100644 index 0000000000..0fa1b47462 --- /dev/null +++ b/backend/migrations/20260813164338_ai_token_usage.down.sql @@ -0,0 +1 @@ +DROP TABLE ai_token_usage; diff --git a/backend/migrations/20260813164338_ai_token_usage.up.sql b/backend/migrations/20260813164338_ai_token_usage.up.sql new file mode 100644 index 0000000000..b20e471db5 --- /dev/null +++ b/backend/migrations/20260813164338_ai_token_usage.up.sql @@ -0,0 +1,42 @@ +-- Per-workspace AI token spend, accumulated from the chat client. Rows hold token +-- counts rather than money: prices live in the frontend price table plus the +-- workspace's `ai_config.model_pricing` overrides and are applied at read time, so +-- correcting a price also corrects the history. `reported_cost_nano_usd` is the +-- exception — a few providers (OpenRouter) return what they actually charged, and +-- that figure wins over the estimate. +-- +-- Distinct from `feature_usage`, which is anonymous telemetry that leaves the +-- instance and is pruned after 60 days; spend is per-user and kept. +CREATE TABLE ai_token_usage ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + day DATE NOT NULL DEFAULT CURRENT_DATE, + email VARCHAR(255) NOT NULL, + provider VARCHAR(50) NOT NULL, + model VARCHAR(255) NOT NULL, + -- Empty for chats that are not attached to an AI session. + session_id VARCHAR(50) NOT NULL DEFAULT '', + -- Uncached input only; the two cache columns hold the rest of the prompt, so + -- each column maps to exactly one price and they never double-count. + input_tokens BIGINT NOT NULL DEFAULT 0, + cache_read_tokens BIGINT NOT NULL DEFAULT 0, + cache_write_tokens BIGINT NOT NULL DEFAULT 0, + output_tokens BIGINT NOT NULL DEFAULT 0, + reported_cost_nano_usd BIGINT, + requests BIGINT NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, day, email, provider, model, session_id) +); + +-- The usage listing filters on workspace and a date range; the PK only reaches +-- `day` through `email`, so it cannot serve that on its own. +CREATE INDEX idx_ai_token_usage_ws_day ON ai_token_usage (workspace_id, day DESC); + +GRANT ALL ON ai_token_usage TO windmill_admin; +GRANT ALL ON ai_token_usage TO windmill_user; + +-- Both handlers go through the raw pool, so no policy is needed for them to work. +-- Enabling RLS with an admin-only policy is the backstop: a future query that +-- reaches this table through UserDB sees nothing rather than every user's spend. +ALTER TABLE ai_token_usage ENABLE ROW LEVEL SECURITY; + +CREATE POLICY admin_policy ON ai_token_usage FOR ALL TO windmill_admin USING (true); diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 732f1ece52..a9865cebd3 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -38,6 +38,8 @@ account: workspace_id(char), id(int), expires_at(ts), refresh_token(char), clien FK: (workspace_id) -> workspace(id) agent_token_blacklist: token(char), expires_at(ts), blacklisted_at(ts), blacklisted_by(char) ai_agent_memory: workspace_id(char), conversation_id(uuid), step_id(char), messages(jsonb), created_at(ts), updated_at(ts) +ai_token_usage: workspace_id(char), day(date), email(char), provider(char), model(char), session_id(char), input_tokens(bigint), cache_read_tokens(bigint), cache_write_tokens(bigint), output_tokens(bigint), reported_cost_nano_usd(bigint), requests(bigint), updated_at(ts) + FK: (workspace_id) -> workspace(id) alerts: id(int), alert_type(char), message(text), created_at(ts), acknowledged(bool), workspace_id(text), acknowledged_workspace(bool), resource(text) app: id(bigint), workspace_id(char), path(char), summary(char), policy(jsonb), versions(bigint[]), extra_perms(jsonb), draft_only(bool), custom_path(text), labels(text[]) FK: (workspace_id) -> workspace(id) diff --git a/backend/windmill-ai/src/ai_google.rs b/backend/windmill-ai/src/ai_google.rs index 4374cb8628..b90bda9e1c 100644 --- a/backend/windmill-ai/src/ai_google.rs +++ b/backend/windmill-ai/src/ai_google.rs @@ -277,7 +277,7 @@ pub struct GeminiSSECandidate { } /// Token usage from the `usageMetadata` field of a Gemini SSE event. -#[derive(Deserialize, Debug, Clone)] +#[derive(Deserialize, Debug, Clone, Default)] pub struct GeminiUsageMetadata { #[serde(rename = "promptTokenCount", default)] pub prompt_token_count: Option, @@ -285,6 +285,39 @@ pub struct GeminiUsageMetadata { pub candidates_token_count: Option, #[serde(rename = "totalTokenCount", default)] pub total_token_count: Option, + /// Subset of `promptTokenCount` served from context cache, billed at a reduced + /// rate. Reported separately so the client can price it separately. + #[serde(rename = "cachedContentTokenCount", default)] + pub cached_content_token_count: Option, + /// Thinking tokens, billed as output but counted apart from `candidatesTokenCount`. + #[serde(rename = "thoughtsTokenCount", default)] + pub thoughts_token_count: Option, + /// Input tokens spent on tool-use prompts, counted apart from `promptTokenCount` + /// rather than within it. + #[serde(rename = "toolUsePromptTokenCount", default)] + pub tool_use_prompt_token_count: Option, +} + +/// Input tokens as billed. Gemini reports tool-use prompts in their own field, and +/// they are disjoint from `promptTokenCount`: a live tool call returns 17 prompt + +/// 60 tool-use + 17 candidates + 52 thoughts against a `totalTokenCount` of 146, so +/// leaving them out under-reports the input of every tool-using turn. Cached tokens +/// are not added here, being already part of `promptTokenCount`. +fn gemini_prompt_tokens(usage: &GeminiUsageMetadata) -> i32 { + usage + .prompt_token_count + .unwrap_or(0) + .saturating_add(usage.tool_use_prompt_token_count.unwrap_or(0)) +} + +/// Output tokens as billed: Gemini counts thinking apart from `candidatesTokenCount` +/// but charges it at the output rate, so a reply that thought would otherwise be +/// reported as far cheaper than it was. +fn gemini_completion_tokens(usage: &GeminiUsageMetadata) -> i32 { + usage + .candidates_token_count + .unwrap_or(0) + .saturating_add(usage.thoughts_token_count.unwrap_or(0)) } /// Top-level structure of one Gemini SSE event. @@ -588,9 +621,12 @@ pub fn gemini_response_to_openai(parsed: &GeminiParsedEvent, model: &str) -> ser let usage = parsed.usage.as_ref().map(|u| { serde_json::json!({ - "prompt_tokens": u.prompt_token_count.unwrap_or(0), - "completion_tokens": u.candidates_token_count.unwrap_or(0), + "prompt_tokens": gemini_prompt_tokens(u), + "completion_tokens": gemini_completion_tokens(u), "total_tokens": u.total_token_count.unwrap_or(0), + "prompt_tokens_details": { + "cached_tokens": u.cached_content_token_count.unwrap_or(0) + }, }) }); @@ -680,8 +716,8 @@ pub fn gemini_event_to_openai_sse_chunks( // OpenAI's `stream_options.include_usage` terminal chunk (top-level `usage`, // empty `choices`) so the frontend's `'usage' in chunk` path records them. if let Some(usage) = &parsed.usage { - let prompt_tokens = usage.prompt_token_count.unwrap_or(0); - let completion_tokens = usage.candidates_token_count.unwrap_or(0); + let prompt_tokens = gemini_prompt_tokens(usage); + let completion_tokens = gemini_completion_tokens(usage); let total_tokens = usage .total_token_count .unwrap_or(prompt_tokens + completion_tokens); @@ -694,6 +730,9 @@ pub fn gemini_event_to_openai_sse_chunks( "prompt_tokens": prompt_tokens, "completion_tokens": completion_tokens, "total_tokens": total_tokens, + "prompt_tokens_details": { + "cached_tokens": usage.cached_content_token_count.unwrap_or(0) + }, } }); chunks.push(format!("data: {}\n\n", chunk)); @@ -943,6 +982,7 @@ mod tests { prompt_token_count: Some(12), candidates_token_count: Some(7), total_token_count: Some(19), + ..Default::default() }), ..Default::default() }; @@ -969,6 +1009,79 @@ mod tests { assert_eq!(usage_chunk["choices"], serde_json::json!([])); } + #[test] + fn gemini_usage_chunk_splits_cached_and_bills_thoughts() { + let parsed = GeminiParsedEvent { + text: Some("the answer".to_string()), + usage: Some(GeminiUsageMetadata { + prompt_token_count: Some(1000), + candidates_token_count: Some(20), + total_token_count: Some(1120), + cached_content_token_count: Some(900), + thoughts_token_count: Some(100), + ..Default::default() + }), + ..Default::default() + }; + + let mut tool_call_index = 0; + let chunks = gemini_event_to_openai_sse_chunks( + &parsed, + "chatcmpl-test", + "gemini-3-flash-preview", + &mut tool_call_index, + ); + let usage_chunk = chunks + .iter() + .map(|c| parse_sse_chunk(c)) + .find(|v| v.get("usage").map(|u| !u.is_null()).unwrap_or(false)) + .expect("a chunk should carry top-level usage"); + + // Gemini's prompt count already includes the cached tokens, so it passes + // through unchanged and the cached share is reported alongside it; thinking + // is billed as output but counted apart from the candidates. + assert_eq!(usage_chunk["usage"]["prompt_tokens"], 1000); + assert_eq!(usage_chunk["usage"]["prompt_tokens_details"]["cached_tokens"], 900); + assert_eq!(usage_chunk["usage"]["completion_tokens"], 120); + } + + #[test] + fn gemini_usage_chunk_counts_tool_use_prompt_tokens() { + let parsed = GeminiParsedEvent { + text: Some("Canberra".to_string()), + usage: Some(GeminiUsageMetadata { + prompt_token_count: Some(17), + candidates_token_count: Some(17), + total_token_count: Some(146), + tool_use_prompt_token_count: Some(60), + thoughts_token_count: Some(52), + ..Default::default() + }), + ..Default::default() + }; + + let mut tool_call_index = 0; + let chunks = gemini_event_to_openai_sse_chunks( + &parsed, + "chatcmpl-test", + "gemini-2.5-flash", + &mut tool_call_index, + ); + let usage_chunk = chunks + .iter() + .map(|c| parse_sse_chunk(c)) + .find(|v| v.get("usage").map(|u| !u.is_null()).unwrap_or(false)) + .expect("a chunk should carry top-level usage"); + + assert_eq!(usage_chunk["usage"]["prompt_tokens"], 77); + assert_eq!(usage_chunk["usage"]["completion_tokens"], 69); + assert_eq!( + usage_chunk["usage"]["prompt_tokens"].as_i64().unwrap() + + usage_chunk["usage"]["completion_tokens"].as_i64().unwrap(), + 146 + ); + } + #[test] fn gemini_streaming_usage_total_falls_back_to_prompt_plus_completion() { let parsed = GeminiParsedEvent { @@ -976,6 +1089,7 @@ mod tests { prompt_token_count: Some(5), candidates_token_count: Some(3), total_token_count: None, + ..Default::default() }), ..Default::default() }; diff --git a/backend/windmill-ai/src/ai_types.rs b/backend/windmill-ai/src/ai_types.rs index b80e237920..69c8ac4538 100644 --- a/backend/windmill-ai/src/ai_types.rs +++ b/backend/windmill-ai/src/ai_types.rs @@ -175,3 +175,57 @@ pub struct OpenAIMessage { #[serde(skip_serializing_if = "Option::is_none")] pub annotations: Option>, } + +// ============================================================================ +// Model pricing +// ============================================================================ + +/// Far above any real per-million-token rate, so a value beyond it is a unit +/// mistake rather than a price. The floor matters more: a negative rate would make +/// spend subtract, and NaN/infinity would poison every total derived from it. +pub const MAX_MODEL_RATE: f64 = 1000.0; + +/// Bound the `model_pricing` map of an AI config that is only available untyped — +/// the instance config is stored through the generic global-settings endpoint, +/// which never deserializes it into `AIConfig`, so the typed check on the +/// workspace path does not cover it. +pub fn validate_model_pricing_json(ai_config: &serde_json::Value) -> Result<(), String> { + // The container itself has to be checked too: a non-object `ai_config` persists + // here and then fails to deserialize as `AIConfig`, which drops the whole + // instance config back to its default for every workspace inheriting it. + if !ai_config.is_null() && !ai_config.is_object() { + return Err("ai_config must be an object".to_string()); + } + let pricing = match ai_config.get("model_pricing") { + None | Some(serde_json::Value::Null) => return Ok(()), + // A present-but-wrong shape must be rejected, not skipped: it would persist + // and then fail to deserialize as `AIConfig`, which silently drops the whole + // instance config back to its default for every workspace inheriting it. + Some(v) => v + .as_object() + .ok_or_else(|| "model_pricing must be an object".to_string())?, + }; + for (key, price) in pricing { + let Some(price) = price.as_object() else { + return Err(format!("Price override for {} is not an object", key)); + }; + for field in ["input", "output", "cache_read", "cache_write"] { + let Some(rate) = price.get(field) else { continue }; + let rate = rate + .as_f64() + .filter(|r| r.is_finite() && *r >= 0.0 && *r <= MAX_MODEL_RATE); + if rate.is_none() { + return Err(format!( + "Price override for {}: {} must be between 0 and {}", + key, field, MAX_MODEL_RATE + )); + } + } + for required in ["input", "output"] { + if !price.contains_key(required) { + return Err(format!("Price override for {} is missing {}", key, required)); + } + } + } + Ok(()) +} diff --git a/backend/windmill-ai/src/providers/bedrock.rs b/backend/windmill-ai/src/providers/bedrock.rs index 453607eea5..cf977f2484 100644 --- a/backend/windmill-ai/src/providers/bedrock.rs +++ b/backend/windmill-ai/src/providers/bedrock.rs @@ -660,6 +660,41 @@ fn bedrock_sse_chunks_for_event( chunks.push(Bytes::from(format!("data: {}\n\n", chunk))); } + // Usage arrives only on the trailing Metadata event, and only this converter + // reaches the chat: without a chunk for it a Bedrock chat reports no tokens at + // all. Bedrock counts cache reads and writes apart from `inputTokens`, while the + // OpenAI shape the client parses treats `prompt_tokens` as the whole input, so + // they are folded in here and split back out through `prompt_tokens_details`. + if let aws_sdk_bedrockruntime::types::ConverseStreamOutput::Metadata(metadata) = event { + if let Some(token_usage) = metadata.usage() { + let cache_read = token_usage.cache_read_input_tokens().unwrap_or(0); + let cache_write = token_usage.cache_write_input_tokens().unwrap_or(0); + let prompt_tokens = token_usage + .input_tokens() + .saturating_add(cache_read) + .saturating_add(cache_write); + + let chunk = serde_json::json!({ + "id": state.id, + "object": "chat.completion.chunk", + "created": state.created, + "model": state.model, + "choices": [], + "usage": { + "prompt_tokens": prompt_tokens, + "completion_tokens": token_usage.output_tokens(), + "total_tokens": token_usage.total_tokens(), + "prompt_tokens_details": { + "cached_tokens": cache_read, + "cache_write_tokens": cache_write + } + } + }); + + chunks.push(Bytes::from(format!("data: {}\n\n", chunk))); + } + } + chunks } @@ -1190,6 +1225,43 @@ mod tests { serde_json::from_str(payload).expect("chunk should contain JSON") } + #[test] + fn metadata_event_emits_usage_chunk_with_cache_split() { + let mut state = BedrockSseStreamState::new("id".to_string(), "model".to_string(), 0); + let event = ConverseStreamOutput::Metadata( + aws_sdk_bedrockruntime::types::ConverseStreamMetadataEvent::builder() + .usage( + aws_sdk_bedrockruntime::types::TokenUsage::builder() + .input_tokens(10) + .output_tokens(7) + .total_tokens(1017) + .cache_read_input_tokens(900) + .cache_write_input_tokens(100) + .build() + .expect("usage"), + ) + .build(), + ); + + let chunks = bedrock_sse_chunks_for_event(&event, &mut state); + let usage = chunks + .iter() + .map(sse_json) + .find(|v| v.get("usage").map(|u| !u.is_null()).unwrap_or(false)) + .expect("the metadata event should carry usage"); + + // Bedrock reports cache reads and writes apart from `inputTokens`; the OpenAI + // shape the client parses treats `prompt_tokens` as the whole input, and + // recovers the uncached share by subtracting the details back out. + assert_eq!(usage["usage"]["prompt_tokens"], 1010); + assert_eq!(usage["usage"]["completion_tokens"], 7); + assert_eq!(usage["usage"]["prompt_tokens_details"]["cached_tokens"], 900); + assert_eq!( + usage["usage"]["prompt_tokens_details"]["cache_write_tokens"], + 100 + ); + } + #[test] fn determine_auth_config_prioritizes_bearer_token() { let config = determine_auth_config( diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index a9bc601591..cead2f4b27 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -891,6 +891,13 @@ async fn run_setting_pre_write_hook( value: &serde_json::Value, ) -> error::Result<()> { match key { + // The instance AI config is written as an untyped blob through this generic + // endpoint, so it never passes the typed check the workspace handler applies. + // Rates that reach a cost total unbounded would make it negative or infinite. + AI_CONFIG_SETTING => { + windmill_ai::ai_types::validate_model_pricing_json(value) + .map_err(error::Error::BadRequest)?; + } AUTOMATE_USERNAME_CREATION_SETTING => { if value.as_bool().unwrap_or(false) { generate_instance_username_for_all_users(db) diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 8986bd7f5c..d2f61a5630 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -11986,6 +11986,73 @@ paths: schema: type: string + /w/{workspace}/ai/usage: + post: + summary: record AI token usage for the calling user + operationId: recordAiUsage + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - events + properties: + events: + type: array + items: + $ref: "#/components/schemas/AITokenUsageEvent" + responses: + "204": + description: usage recorded + get: + summary: list aggregated AI token usage + operationId: listAiUsage + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: days + in: query + schema: + type: integer + minimum: 1 + maximum: 365 + - name: group_by + in: query + schema: + type: string + enum: [day, user, model] + - name: scope + in: query + description: workspace-wide usage (admin only) or the calling user's own + schema: + type: string + enum: [workspace, self] + responses: + "200": + description: usage buckets + content: + application/json: + schema: + type: object + required: + - buckets + - truncated + properties: + buckets: + type: array + items: + $ref: "#/components/schemas/AITokenUsageBucket" + truncated: + type: boolean + description: more buckets matched than were returned, so summing them under-reports + /w/{workspace}/ai_skills/list: get: summary: list the workspace AI chat skills (name + description only) @@ -26300,6 +26367,92 @@ components: type: integer minimum: 1 maximum: 2000000 + model_pricing: + type: object + additionalProperties: + $ref: "#/components/schemas/ModelPriceOverride" + + ModelPriceOverride: + type: object + description: negotiated rates in USD per million tokens, keyed `provider:model` + properties: + input: + type: number + minimum: 0 + maximum: 1000 + output: + type: number + minimum: 0 + maximum: 1000 + cache_read: + type: number + minimum: 0 + maximum: 1000 + cache_write: + type: number + minimum: 0 + maximum: 1000 + required: + - input + - output + + AITokenUsageEvent: + type: object + properties: + provider: + $ref: "#/components/schemas/AIProvider" + model: + type: string + session_id: + type: string + input_tokens: + type: integer + cache_read_tokens: + type: integer + cache_write_tokens: + type: integer + output_tokens: + type: integer + reported_cost_nano_usd: + type: integer + description: only set by providers that bill back an exact figure + requests: + type: integer + required: + - provider + - model + + AITokenUsageBucket: + type: object + properties: + key: + type: string + description: the grouped dimension's value; empty when grouping by model + provider: + type: string + model: + type: string + input_tokens: + type: integer + cache_read_tokens: + type: integer + cache_write_tokens: + type: integer + output_tokens: + type: integer + reported_cost_nano_usd: + type: integer + requests: + type: integer + required: + - key + - provider + - model + - input_tokens + - cache_read_tokens + - cache_write_tokens + - output_tokens + - requests InstanceAIProviderSummary: type: object diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index efc2ba7e60..1a5f9dba87 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -3,11 +3,16 @@ use crate::utils::check_scopes; #[cfg(feature = "bedrock")] use axum::routing::get; -#[cfg(feature = "bedrock")] use axum::Json; -use axum::{body::Bytes, extract::Path, response::IntoResponse, routing::post, Extension, Router}; +use axum::{ + body::Bytes, + extract::{DefaultBodyLimit, Path, Query}, + response::IntoResponse, + routing::post, + Extension, Router, +}; use futures::StreamExt; -use http::{HeaderMap, Method}; +use http::{HeaderMap, Method, StatusCode}; use quick_cache::sync::Cache; use reqwest::{Client, RequestBuilder}; use serde::{Deserialize, Serialize}; @@ -18,6 +23,7 @@ use windmill_ai::ai_cache::current_instance_ai_config_revision; use windmill_ai::ai_providers::{ empty_string_as_none, AIPlatform, AIProvider, ProviderConfig, ProviderModel, }; +use windmill_ai::ai_types::MAX_MODEL_RATE; use windmill_ai::credentials::ProviderCredentials; #[cfg(feature = "bedrock")] use windmill_ai::providers::bedrock::{ @@ -37,7 +43,7 @@ use windmill_ai::proxy::{ use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::db::UserDB; use windmill_common::error::{to_anyhow, Error, Result}; -use windmill_common::utils::configure_client; +use windmill_common::utils::{configure_client, require_admin}; use windmill_common::variables::{get_variable_or_self, get_variable_or_self_as}; // AI timeout configuration constants @@ -417,9 +423,54 @@ pub struct AIConfig { pub custom_prompts: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub max_tokens_per_model: Option>, + /// Per-model price overrides, keyed `provider:model` like `max_tokens_per_model`. + /// Only models whose rates differ from the built-in table are stored. + #[serde(skip_serializing_if = "Option::is_none")] + pub model_pricing: Option>, +} + +/// Negotiated rates in USD per million tokens. An unset cache rate is read as the +/// provider's own multiple of the input rate where the model has a published one, +/// and as the input rate itself where it does not — an unstated discount is never +/// filled in from another vendor's. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ModelPriceOverride { + pub input: f64, + pub output: f64, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_read: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_write: Option, +} + +impl ModelPriceOverride { + pub fn validate(&self, key: &str) -> Result<()> { + for (field, rate) in [ + ("input", Some(self.input)), + ("output", Some(self.output)), + ("cache_read", self.cache_read), + ("cache_write", self.cache_write), + ] { + let Some(rate) = rate else { continue }; + if !rate.is_finite() || rate < 0.0 || rate > MAX_MODEL_RATE { + return Err(Error::BadRequest(format!( + "Price override for {}: {} must be between 0 and {}", + key, field, MAX_MODEL_RATE + ))); + } + } + Ok(()) + } } impl AIConfig { + pub fn validate_model_pricing(&self) -> Result<()> { + for (key, price) in self.model_pricing.iter().flatten() { + price.validate(key)?; + } + Ok(()) + } + pub fn has_providers(&self) -> bool { self.providers .as_ref() @@ -432,7 +483,18 @@ pub fn global_service() -> Router { } pub fn workspaced_service() -> Router { - let router = Router::new().route("/proxy/{*ai}", post(proxy).get(proxy)); + let router = Router::new() + .route("/proxy/{*ai}", post(proxy).get(proxy)) + .route( + "/usage", + post(record_ai_usage) + .get(list_ai_usage) + // The handler caps how many events it *stores*, but Json deserializes + // the whole array first — without a body limit an authenticated member + // could make the server allocate and parse an arbitrarily large one. + // Sized well above a full batch of the shape below. + .layer(DefaultBodyLimit::max(AI_USAGE_BODY_LIMIT)), + ); #[cfg(feature = "bedrock")] let router = router.route("/check_bedrock_credentials", get(check_bedrock_credentials)); @@ -440,6 +502,265 @@ pub fn workspaced_service() -> Router { router } +/// One provider request's worth of tokens, as counted by the chat client. +#[derive(Deserialize)] +struct AIUsageEvent { + provider: String, + model: String, + #[serde(default)] + session_id: String, + #[serde(default)] + input_tokens: i64, + #[serde(default)] + cache_read_tokens: i64, + #[serde(default)] + cache_write_tokens: i64, + #[serde(default)] + output_tokens: i64, + /// Only the providers that bill back an exact figure set this. + #[serde(default)] + reported_cost_nano_usd: Option, + #[serde(default)] + requests: Option, +} + +#[derive(Deserialize)] +struct RecordAIUsagePayload { + events: Vec, +} + +const MAX_AI_USAGE_EVENTS: usize = 50; +/// 64 KiB — a 50-event batch is a few kB even with the longest model ids. +const AI_USAGE_BODY_LIMIT: usize = 64 * 1024; +/// Well above any single conversation and far below an i64 overflow, so a client +/// bug caps out at one absurd row instead of poisoning the running total. +const MAX_TOKENS_PER_EVENT: i64 = 100_000_000; +/// $1000 in nano-USD. +const MAX_REPORTED_COST_PER_EVENT: i64 = 1_000_000_000_000; + +/// Model ids carry vendor prefixes and variant suffixes (`anthropic/claude-opus-5:thinking`), +/// so the shape check is looser than an identifier but still excludes whitespace and +/// anything that would not be a model id. +fn is_model_shaped(s: &str, max_len: usize) -> bool { + !s.is_empty() + && s.len() <= max_len + && s.chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | ':' | '.' | '/' | '~')) +} + +/// Accumulate one workspace's AI token spend. Values are clamped and the caller's +/// email comes from the session, never the payload — the client is trusted to +/// report its own usage, not to attribute it to someone else. +async fn record_ai_usage( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(payload): Json, +) -> Result { + // Pre-sum duplicate keys: two rows hitting the same conflict target in a single + // INSERT error out ("cannot affect row a second time"). + let mut agg: HashMap<(String, String, String), AIUsageTotals> = HashMap::new(); + for e in payload.events.into_iter().take(MAX_AI_USAGE_EVENTS) { + if AIProvider::try_from(e.provider.as_str()).is_err() + || !is_model_shaped(&e.model, 255) + || !(e.session_id.is_empty() || is_model_shaped(&e.session_id, 50)) + { + continue; + } + let totals = agg + .entry((e.provider, e.model, e.session_id)) + .or_insert_with(AIUsageTotals::default); + totals.input += e.input_tokens.clamp(0, MAX_TOKENS_PER_EVENT); + totals.cache_read += e.cache_read_tokens.clamp(0, MAX_TOKENS_PER_EVENT); + totals.cache_write += e.cache_write_tokens.clamp(0, MAX_TOKENS_PER_EVENT); + totals.output += e.output_tokens.clamp(0, MAX_TOKENS_PER_EVENT); + totals.requests += e.requests.unwrap_or(1).clamp(0, MAX_AI_USAGE_EVENTS as i64); + if let Some(cost) = e.reported_cost_nano_usd { + totals.reported_cost = Some( + totals.reported_cost.unwrap_or(0) + cost.clamp(0, MAX_REPORTED_COST_PER_EVENT), + ); + } + } + if agg.is_empty() { + return Ok(StatusCode::NO_CONTENT); + } + + let mut providers = Vec::with_capacity(agg.len()); + let mut models = Vec::with_capacity(agg.len()); + let mut session_ids = Vec::with_capacity(agg.len()); + let mut inputs = Vec::with_capacity(agg.len()); + let mut cache_reads = Vec::with_capacity(agg.len()); + let mut cache_writes = Vec::with_capacity(agg.len()); + let mut outputs = Vec::with_capacity(agg.len()); + let mut reported_costs: Vec> = Vec::with_capacity(agg.len()); + let mut requests = Vec::with_capacity(agg.len()); + for ((provider, model, session_id), totals) in agg { + providers.push(provider); + models.push(model); + session_ids.push(session_id); + inputs.push(totals.input); + cache_reads.push(totals.cache_read); + cache_writes.push(totals.cache_write); + outputs.push(totals.output); + reported_costs.push(totals.reported_cost); + requests.push(totals.requests); + } + + sqlx::query!( + "INSERT INTO ai_token_usage (workspace_id, email, provider, model, session_id, \ + input_tokens, cache_read_tokens, cache_write_tokens, output_tokens, \ + reported_cost_nano_usd, requests) + SELECT $1, $2, * FROM UNNEST($3::text[], $4::text[], $5::text[], $6::bigint[], \ + $7::bigint[], $8::bigint[], $9::bigint[], $10::bigint[], $11::bigint[]) + ON CONFLICT (workspace_id, day, email, provider, model, session_id) + DO UPDATE SET + input_tokens = ai_token_usage.input_tokens + EXCLUDED.input_tokens, + cache_read_tokens = ai_token_usage.cache_read_tokens + EXCLUDED.cache_read_tokens, + cache_write_tokens = ai_token_usage.cache_write_tokens + EXCLUDED.cache_write_tokens, + output_tokens = ai_token_usage.output_tokens + EXCLUDED.output_tokens, + reported_cost_nano_usd = CASE + WHEN EXCLUDED.reported_cost_nano_usd IS NULL + THEN ai_token_usage.reported_cost_nano_usd + ELSE COALESCE(ai_token_usage.reported_cost_nano_usd, 0) + + EXCLUDED.reported_cost_nano_usd + END, + requests = ai_token_usage.requests + EXCLUDED.requests, + updated_at = now()", + &w_id, + &authed.email, + &providers, + &models, + &session_ids, + &inputs, + &cache_reads, + &cache_writes, + &outputs, + &reported_costs as &[Option], + &requests + ) + .execute(&db) + .await?; + + Ok(StatusCode::NO_CONTENT) +} + +#[derive(Default)] +struct AIUsageTotals { + input: i64, + cache_read: i64, + cache_write: i64, + output: i64, + reported_cost: Option, + requests: i64, +} + +#[derive(Deserialize)] +struct ListAIUsageQuery { + days: Option, + group_by: Option, + scope: Option, +} + +/// A bucket always carries its provider and model: the caller prices it from a +/// per-model rate table, which a bucket spanning several models could not be +/// resolved against. +#[derive(Serialize)] +struct AITokenUsageBucket { + key: String, + provider: String, + model: String, + input_tokens: i64, + cache_read_tokens: i64, + cache_write_tokens: i64, + output_tokens: i64, + reported_cost_nano_usd: Option, + requests: i64, +} + +/// Grouping by day over a long range, or by model across many models, can produce +/// more buckets than a table is worth rendering, so the listing is capped. +/// `truncated` says so explicitly — a caller that sums the rows into a total must be +/// able to tell that the total is partial rather than silently under-reporting spend. +#[derive(Serialize)] +struct AITokenUsageListing { + buckets: Vec, + truncated: bool, +} + +const AI_USAGE_MAX_BUCKETS: i64 = 1000; + +async fn list_ai_usage( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Query(query): Query, +) -> Result> { + // Reading the whole workspace's spend is an admin view; reading your own is + // not, so a member can see what they are costing without being shown their + // colleagues'. The filter is the session's email, never a parameter. + let own_email = match query.scope.as_deref().unwrap_or("workspace") { + "workspace" => { + require_admin(authed.is_admin, &authed.username)?; + None + } + "self" => Some(authed.email.clone()), + scope => return Err(Error::BadRequest(format!("Unsupported scope: {}", scope))), + }; + + let days = query.days.unwrap_or(30).clamp(1, 365); + let group_by = query.group_by.as_deref().unwrap_or("day"); + // No `session`: a session is identified by a client-generated id whose name + // lives only in the browser that made it, so a bucket keyed on one is a label + // nobody can resolve. `session_id` is still stored, at the grain the client + // batches on, should sessions ever gain a server-side name. + if !matches!(group_by, "day" | "user" | "model") { + return Err(Error::BadRequest(format!( + "Unsupported group_by: {}", + group_by + ))); + } + + // Fetch one past the cap to detect truncation. Ordering is by token volume, not + // by cost: rates are applied by the caller, so this query cannot know what a + // bucket cost. Volume is the closest proxy available here, and the caller is told + // the listing was capped rather than being left to sum a partial set silently. + let mut rows = sqlx::query_as!( + AITokenUsageBucket, + r#"SELECT + (CASE $3::text + WHEN 'day' THEN day::text + WHEN 'user' THEN email + ELSE '' + END) AS "key!", + provider AS "provider!", + model AS "model!", + SUM(input_tokens)::bigint AS "input_tokens!", + SUM(cache_read_tokens)::bigint AS "cache_read_tokens!", + SUM(cache_write_tokens)::bigint AS "cache_write_tokens!", + SUM(output_tokens)::bigint AS "output_tokens!", + SUM(reported_cost_nano_usd)::bigint AS "reported_cost_nano_usd", + SUM(requests)::bigint AS "requests!" + FROM ai_token_usage + WHERE workspace_id = $1 AND day > CURRENT_DATE - $2::int + AND ($5::text IS NULL OR email = $5) + GROUP BY 1, provider, model + ORDER BY SUM(input_tokens + cache_read_tokens + cache_write_tokens + output_tokens) DESC + LIMIT $4"#, + &w_id, + days, + group_by, + AI_USAGE_MAX_BUCKETS + 1, + own_email.as_deref() + ) + .fetch_all(&db) + .await?; + + let truncated = rows.len() as i64 > AI_USAGE_MAX_BUCKETS; + rows.truncate(AI_USAGE_MAX_BUCKETS as usize); + + Ok(Json(AITokenUsageListing { buckets: rows, truncated })) +} + /// Check if AWS Bedrock credentials are available from environment variables. #[cfg(feature = "bedrock")] async fn check_bedrock_credentials( diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index 23ce68fb80..01eb32b7f6 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -108,6 +108,8 @@ async fn edit_copilot_config( } } + ai_config.validate_model_pricing()?; + let mut tx = db.begin().await?; sqlx::query!( diff --git a/frontend/src/lib/aiStore.ts b/frontend/src/lib/aiStore.ts index 1e8fb6ac61..0955f68ee5 100644 --- a/frontend/src/lib/aiStore.ts +++ b/frontend/src/lib/aiStore.ts @@ -3,7 +3,12 @@ // import aiStore back, and such a cycle crashes the app once the bundler splits it across // chunks (docs/frontend-import-cycles.md; the build fails on the chunk cycle, not on this). import { writable, get } from 'svelte/store' -import { type AIProviderModel, type AIProvider, type AIConfig } from './gen' +import { + type AIProviderModel, + type AIProvider, + type AIConfig, + type ModelPriceOverride +} from './gen' import { aiUserDisabled, COPILOT_SESSION_MODEL_SETTING_NAME, @@ -41,6 +46,8 @@ export const copilotInfo = writable<{ aiModels: AIProviderModel[] customPrompts?: Record maxTokensPerModel?: Record + /** Negotiated rates per `provider:model`, overriding the built-in price table. */ + modelPricing?: Record webSearchEnabledProviders?: Partial> }>({ enabled: false, @@ -50,6 +57,7 @@ export const copilotInfo = writable<{ aiModels: [], customPrompts: {}, maxTokensPerModel: {}, + modelPricing: {}, webSearchEnabledProviders: {} }) @@ -124,6 +132,7 @@ export function setCopilotInfo(aiConfig: AIConfig) { aiModels: aiModels, customPrompts: aiConfig.custom_prompts ?? {}, maxTokensPerModel: aiConfig.max_tokens_per_model ?? {}, + modelPricing: aiConfig.model_pricing ?? {}, webSearchEnabledProviders }) } else { @@ -137,6 +146,7 @@ export function setCopilotInfo(aiConfig: AIConfig) { aiModels: [], customPrompts: {}, maxTokensPerModel: {}, + modelPricing: {}, webSearchEnabledProviders: {} }) } diff --git a/frontend/src/lib/components/UserSettings.svelte b/frontend/src/lib/components/UserSettings.svelte index d3956f7f12..d5811e737b 100644 --- a/frontend/src/lib/components/UserSettings.svelte +++ b/frontend/src/lib/components/UserSettings.svelte @@ -9,6 +9,8 @@ import { createEventDispatcher } from 'svelte' import UserInfoSettings from './settings/UserInfoSettings.svelte' import AIUserSettings from './settings/AIUserSettings.svelte' + import AiUsagePanel from './workspaceSettings/AiUsagePanel.svelte' + import { copilotInfo, copilotWorkspace } from '$lib/aiStore' import { getDarkModeVariant, setDarkModeVariant, @@ -105,6 +107,17 @@ + + {#if $copilotWorkspace} + + {/if} {/if}
diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index e08b9ba2a3..f68501b293 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -58,7 +58,7 @@ import { import { dfs } from '$lib/components/flows/previousResults' import { SvelteMap, SvelteSet } from 'svelte/reactivity' import { createLongHash } from '$lib/editorLangUtils' -import type { UserDraftItemKind } from '$lib/gen' +import type { AIProvider, UserDraftItemKind } from '$lib/gen' import { maskKey } from '$lib/components/sessions/modifiedItemsMask' import { getStringError } from './utils' import { type PasteAttachment } from './pasteTokens' @@ -101,7 +101,12 @@ import type AIChatInput from './AIChatInput.svelte' import { prepareApiSystemMessage, prepareApiUserMessage } from './api/core' import { runChatLoop, truncateToToolPairedPrefix } from './chatLoop' import { sanitizeToolCallArguments } from './toolCallArguments' -import { normalizeContextUsage } from './tokenUsage' +import { + billedTokens, + normalizeContextUsage, + type ChatTokenUsage +} from './tokenUsage' +import { logAiUsage } from '$lib/utils/aiUsageReporter' import type { ReviewChangesOpts } from './monaco-adapter' import { getCurrentModel, @@ -709,6 +714,39 @@ export class AIChatManager { await this.#persistModifiedItems() } + /** Report one completed provider response's tokens to the workspace usage view. + * Called per response rather than per turn: a tool loop makes several, each + * separately billed, and a turn that fails partway through has still spent + * everything up to that point. + * + * Only token counts leave the browser — rates are applied when the usage is + * read, so a corrected price also corrects everything already recorded. */ + private recordUsage( + usage: ChatTokenUsage, + provider: AIProvider, + model: string, + workspace: string | undefined + ) { + // A provider that reports no usage still yields an all-zero report. Recording + // it would add a $0 row to the usage view, claiming the request cost nothing + // rather than that it went uncounted. + if (usage.total === 0 && usage.prompt === 0 && usage.completion === 0) { + return + } + const tokens = billedTokens(usage) + logAiUsage({ + provider, + model, + sessionId: this.sessionId, + inputTokens: tokens.input, + cacheReadTokens: tokens.cacheRead, + cacheWriteTokens: tokens.cacheWrite, + outputTokens: tokens.output, + costUsd: usage.cost, + workspace + }) + } + // Serialized, snapshot-at-write-time persistence: two rapid dock actions // would otherwise race their saveChat writes, and the earlier (staler) // snapshot could land last — dropping the later mutation until the next @@ -2458,6 +2496,11 @@ export class AIChatManager { // on each iteration. This is critical for changeModeTool (Navigator → Script/Flow) // which reassigns this.tools, this.helpers, this.systemMessage mid-loop. const self = this + // Pinned for the whole turn, like the `workspace` the loop routes through: + // the global chat's operating workspace follows workspaceStore, so a switch + // while a response streams would bill it to the workspace the user landed + // on rather than the one whose credentials and proxy served it. + const usageWorkspace = this.operatingWorkspace const result = await runChatLoop({ messages, addedMessages, @@ -2535,6 +2578,14 @@ export class AIChatManager { } return undefined }, + onUsage: (usage, modelProvider) => { + // Accounting must never take a turn down with it. + try { + this.recordUsage(usage, modelProvider.provider, modelProvider.model, usageWorkspace) + } catch (e) { + console.error('Failed to record AI usage', e) + } + }, onBeforeIteration: async (tools, _helpers, modelProvider) => { this.lastIterationModel = modelProvider for (const tool of tools) { diff --git a/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte b/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte index a307fad255..ffa06ca5df 100644 --- a/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte +++ b/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte @@ -4,6 +4,7 @@ import { getAiChatManager } from './aiChatManagerContext' import { AIMode } from './AIChatManager.svelte' import Tooltip from '$lib/components/meltComponents/Tooltip.svelte' + import { formatTokenCount } from './tokenUsage' const aiChatManager = getAiChatManager() @@ -45,16 +46,6 @@ ? 'bg-amber-500' : 'bg-surface-accent-primary' ) - - function formatTokenCount(tokens: number): string { - if (tokens >= 1_000_000) { - return `${(tokens / 1_000_000).toFixed(1).replace(/\.0$/, '')}M` - } - if (tokens >= 1000) { - return `${Math.round(tokens / 1000)}k` - } - return `${tokens}` - } {#if visible} diff --git a/frontend/src/lib/components/copilot/chat/anthropic.ts b/frontend/src/lib/components/copilot/chat/anthropic.ts index 52074a84c9..8d38f10916 100644 --- a/frontend/src/lib/components/copilot/chat/anthropic.ts +++ b/frontend/src/lib/components/copilot/chat/anthropic.ts @@ -195,7 +195,7 @@ export async function parseAnthropicCompletion( tools: Tool[], helpers: any, abortController?: AbortController, - options?: { workspace?: string } + options?: { workspace?: string; onTokenUsage?: (usage: ChatTokenUsage) => void } ): Promise { let toolCallsToProcess: ChatCompletionMessageFunctionToolCall[] = [] let error = null @@ -417,6 +417,7 @@ export async function parseAnthropicCompletion( const finalMessage = await completion.finalMessage() const tokenUsage = anthropicUsageToChatTokenUsage(finalMessage.usage) + options?.onTokenUsage?.(tokenUsage) // Process tool calls if any if (toolCallsToProcess.length > 0) { diff --git a/frontend/src/lib/components/copilot/chat/chatLoop.test.ts b/frontend/src/lib/components/copilot/chat/chatLoop.test.ts index 2c4eb466e9..a010f1369a 100644 --- a/frontend/src/lib/components/copilot/chat/chatLoop.test.ts +++ b/frontend/src/lib/components/copilot/chat/chatLoop.test.ts @@ -518,7 +518,7 @@ describe('runChatLoop lastIterationUsage', () => { expect(result.lastIterationUsage).toEqual({ prompt: 1200, completion: 80, total: 1280 }) // the aggregate keeps summing across iterations - expect(result.tokenUsage).toEqual({ prompt: 2200, completion: 130, total: 2330 }) + expect(result.tokenUsage).toMatchObject({ prompt: 2200, completion: 130, total: 2330 }) }) it('ignores empty usage reports and returns null when none are real', async () => { diff --git a/frontend/src/lib/components/copilot/chat/chatLoop.ts b/frontend/src/lib/components/copilot/chat/chatLoop.ts index cc963e4b25..91dd3ab29c 100644 --- a/frontend/src/lib/components/copilot/chat/chatLoop.ts +++ b/frontend/src/lib/components/copilot/chat/chatLoop.ts @@ -77,11 +77,16 @@ export interface ChatLoopConfig { helpers: any, modelProvider: ReasoningProviderModel ) => Promise + /** Fired for each completed provider response, before the loop continues. The + * loop can fail or be aborted at any iteration, so spend has to be handed over + * as it happens — a callback only at the end would discard everything the + * earlier iterations were already billed for. */ + onUsage?: (usage: ChatTokenUsage, modelProvider: ReasoningProviderModel) => void } export interface ChatLoopResult { addedMessages: ChatCompletionMessageParam[] - /** Sum of usage across all loop iterations (suitable for cost accounting). */ + /** Sum of usage across all loop iterations. */ tokenUsage: ChatTokenUsage lastIterationUsage: ChatTokenUsage | null hitMaxIterations: boolean @@ -328,6 +333,20 @@ export async function runChatLoop(config: ChatLoopConfig): Promise { + if (usage && iterationModel) { + config.onUsage?.(usage, iterationModel) + } + } const trackUsage = (usage: ChatTokenUsage | null | undefined) => { tokenUsage = addChatTokenUsage(tokenUsage, usage) @@ -351,6 +370,7 @@ export async function runChatLoop(config: ChatLoopConfig): Promise t.def) - const parseOptions = { workspace, provider: modelProvider.provider } + const parseOptions = { + workspace, + provider: modelProvider.provider, + onTokenUsage: reportUsage + } if (isOpenAI) { const reasoningSummaryCacheKey = getReasoningSummaryCacheKey(workspace, modelProvider) diff --git a/frontend/src/lib/components/copilot/chat/openai-responses.ts b/frontend/src/lib/components/copilot/chat/openai-responses.ts index 19efff2cd4..4118eb492d 100644 --- a/frontend/src/lib/components/copilot/chat/openai-responses.ts +++ b/frontend/src/lib/components/copilot/chat/openai-responses.ts @@ -392,7 +392,7 @@ export async function parseOpenAIResponsesCompletion( addedMessages: ChatCompletionMessageParam[], tools: Tool[], helpers: any, - options?: { workspace?: string } + options?: { workspace?: string; onTokenUsage?: (usage: ChatTokenUsage) => void } ): Promise { let toolCallsToProcess: ChatCompletionMessageFunctionToolCall[] = [] let error: OpenAIError | ResponseErrorEvent | null = null @@ -566,6 +566,7 @@ export async function parseOpenAIResponsesCompletion( const finalResponse = await runner.finalResponse() const tokenUsage = openAIResponsesUsageToChatTokenUsage(finalResponse.usage) + options?.onTokenUsage?.(tokenUsage) for (const item of finalResponse.output ?? []) { if (item.type === 'web_search_call' && !surfacedWebSearchCalls.has(item.id)) { diff --git a/frontend/src/lib/components/copilot/chat/tokenUsage.test.ts b/frontend/src/lib/components/copilot/chat/tokenUsage.test.ts new file mode 100644 index 0000000000..f23064f5a3 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/tokenUsage.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from 'vitest' +import { + anthropicUsageToChatTokenUsage, + billedTokens, + openAICompletionsUsageToChatTokenUsage +} from './tokenUsage' + +// The two providers report cache tokens under opposite conventions — Anthropic's +// input_tokens excludes them, OpenAI's includes them. Both are normalized so that +// `prompt` is the whole input, which is what makes `prompt - cached` the uncached +// share. Getting this backwards double-counts (or loses) the cached prefix, which +// is most of a long chat's input. +describe('billedTokens', () => { + it('derives uncached input under the Anthropic convention', () => { + const usage = anthropicUsageToChatTokenUsage({ + input_tokens: 1000, + output_tokens: 200, + cache_creation_input_tokens: 300, + cache_read_input_tokens: 5000 + }) + expect(usage.prompt).toBe(6300) + expect(billedTokens(usage)).toEqual({ + input: 1000, + cacheRead: 5000, + cacheWrite: 300, + output: 200 + }) + }) + + it('derives uncached input under the OpenAI convention', () => { + const usage = openAICompletionsUsageToChatTokenUsage({ + prompt_tokens: 6000, + completion_tokens: 200, + prompt_tokens_details: { cached_tokens: 5000 } + }) + expect(usage.prompt).toBe(6000) + expect(billedTokens(usage)).toEqual({ + input: 1000, + cacheRead: 5000, + cacheWrite: 0, + output: 200 + }) + }) + + // OpenRouter extends the OpenAI shape with cache-creation tokens, counted + // inside prompt_tokens like the reads beside them. Missing the field bills + // them as uncached input. + it('splits out OpenRouter cache-creation tokens', () => { + const usage = openAICompletionsUsageToChatTokenUsage({ + prompt_tokens: 6300, + completion_tokens: 200, + prompt_tokens_details: { cached_tokens: 5000, cache_write_tokens: 300 } + }) + expect(billedTokens(usage)).toEqual({ + input: 1000, + cacheRead: 5000, + cacheWrite: 300, + output: 200 + }) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/tokenUsage.ts b/frontend/src/lib/components/copilot/chat/tokenUsage.ts index 0091e11227..06ef5e71d8 100644 --- a/frontend/src/lib/components/copilot/chat/tokenUsage.ts +++ b/frontend/src/lib/components/copilot/chat/tokenUsage.ts @@ -1,7 +1,19 @@ +import type { PricedTokens } from '../modelPricing' + export interface ChatTokenUsage { prompt: number completion: number total: number + /** + * Subsets of `prompt`, split out because they are billed at different rates + * (a cached read is a fraction of an uncached one). `prompt` stays the whole + * input so the context gauge keeps measuring the whole request; uncached + * input is `prompt - cacheRead - cacheWrite`. + */ + cacheRead: number + cacheWrite: number + /** Cost in USD as billed, for the providers that report one. */ + cost?: number } /** @@ -28,7 +40,7 @@ export function normalizeContextUsage( } export function emptyChatTokenUsage(): ChatTokenUsage { - return { prompt: 0, completion: 0, total: 0 } + return { prompt: 0, completion: 0, total: 0, cacheRead: 0, cacheWrite: 0 } } export function addChatTokenUsage( @@ -39,10 +51,48 @@ export function addChatTokenUsage( return total } + const cost = + total.cost === undefined && usage.cost === undefined + ? undefined + : (total.cost ?? 0) + (usage.cost ?? 0) + return { prompt: total.prompt + usage.prompt, completion: total.completion + usage.completion, - total: total.total + usage.total + total: total.total + usage.total, + // `?? 0`: the cache split is newer than the field it lives on, so a usage + // object read back from storage may predate it. + cacheRead: (total.cacheRead ?? 0) + (usage.cacheRead ?? 0), + cacheWrite: (total.cacheWrite ?? 0) + (usage.cacheWrite ?? 0), + ...(cost === undefined ? {} : { cost }) + } +} + +/** Compact token count for readouts and tables (`1.2M`, `34k`, `567`). */ +export function formatTokenCount(tokens: number): string { + if (tokens >= 1_000_000) { + return `${(tokens / 1_000_000).toFixed(1).replace(/\.0$/, '')}M` + } + if (tokens >= 1000) { + return `${Math.round(tokens / 1000)}k` + } + return `${tokens}` +} + +/** + * Split a usage report into the four separately-billed token classes. `prompt` + * counts the whole input, so the uncached share is whatever the cached classes + * do not account for — which holds for both provider conventions below + * (Anthropic adds its cache counts into `prompt`, OpenAI's already includes them). + */ +export function billedTokens(usage: ChatTokenUsage): PricedTokens { + const cacheRead = usage.cacheRead ?? 0 + const cacheWrite = usage.cacheWrite ?? 0 + return { + input: Math.max(0, usage.prompt - cacheRead - cacheWrite), + cacheRead, + cacheWrite, + output: usage.completion } } @@ -57,16 +107,17 @@ export function anthropicUsageToChatTokenUsage( | null | undefined ): ChatTokenUsage { - const prompt = - (usage?.input_tokens ?? 0) + - (usage?.cache_creation_input_tokens ?? 0) + - (usage?.cache_read_input_tokens ?? 0) + const cacheWrite = usage?.cache_creation_input_tokens ?? 0 + const cacheRead = usage?.cache_read_input_tokens ?? 0 + const prompt = (usage?.input_tokens ?? 0) + cacheWrite + cacheRead const completion = usage?.output_tokens ?? 0 return { prompt, completion, - total: prompt + completion + total: prompt + completion, + cacheRead, + cacheWrite } } @@ -89,7 +140,11 @@ export function openAIResponsesUsageToChatTokenUsage( return { prompt, completion, - total: usage?.total_tokens ?? prompt + completion + total: usage?.total_tokens ?? prompt + completion, + cacheRead: usage?.input_tokens_details?.cached_tokens ?? 0, + // Automatic caching: nothing is billed for populating it, and no usage + // field reports it either. + cacheWrite: 0 } } @@ -101,7 +156,16 @@ export function openAICompletionsUsageToChatTokenUsage( prompt_tokens?: number | null completion_tokens?: number | null total_tokens?: number | null - prompt_tokens_details?: { cached_tokens?: number | null } | null + prompt_tokens_details?: { + cached_tokens?: number | null + /** Cache creation, reported by the providers that bill for it: OpenRouter + * passes Anthropic's through, and the Bedrock proxy folds + * `cacheWriteInputTokens` in here. OpenAI, whose caching is automatic and + * unbilled, reports no such field. */ + cache_write_tokens?: number | null + } | null + /** OpenRouter reports what it actually charged when the request opts in. */ + cost?: number | null } | null | undefined @@ -112,6 +176,9 @@ export function openAICompletionsUsageToChatTokenUsage( return { prompt, completion, - total: usage?.total_tokens ?? prompt + completion + total: usage?.total_tokens ?? prompt + completion, + cacheRead: usage?.prompt_tokens_details?.cached_tokens ?? 0, + cacheWrite: usage?.prompt_tokens_details?.cache_write_tokens ?? 0, + ...(typeof usage?.cost === 'number' ? { cost: usage.cost } : {}) } } diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index 09f34806c2..fc8a00727f 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -1089,6 +1089,23 @@ export async function getFimCompletion( } } +// A streamed OpenAI-compatible response carries no usage at all unless the request +// asks for it, so a provider missing from this set reports zero tokens — no context +// gauge, no cost. `stream_options.include_usage` is part of the OpenAI streaming +// spec and these providers document supporting it; `customai` is deliberately absent +// because it points at an arbitrary endpoint that may reject the field outright. +const STREAM_USAGE_PROVIDERS = new Set([ + 'openai', + 'azure_openai', + 'azure_foundry', + 'googleai', + 'openrouter', + 'groq', + 'deepseek', + 'mistral', + 'togetherai' +]) + export async function getCompletion( messages: ChatCompletionMessageParam[], abortController: AbortController, @@ -1132,17 +1149,17 @@ export async function getCompletion( // Use Completions API for other providers const client = options?.openaiClient ?? workspaceAIClients.getOpenaiClient() const completionConfig = applyReasoningToConfig( - (provider === 'openai' || - provider === 'azure_openai' || - provider === 'azure_foundry' || - provider === 'googleai') && - config.stream + config.stream && STREAM_USAGE_PROVIDERS.has(provider) ? { ...config, stream_options: { ...(config.stream_options ?? {}), include_usage: true - } + }, + // OpenRouter's own extension, on top of stream_options: it returns the + // credits actually charged next to the token counts, which is the one + // route by which the chat sees a real cost rather than an estimate. + ...(provider === 'openrouter' ? { usage: { include: true } } : {}) } : config, provider === 'deepseek' ? 'deepseek' : provider === 'mistral' ? 'mistral' : 'completions', @@ -1178,7 +1195,11 @@ export async function parseOpenAICompletion( tools: Tool[], helpers: any, _abortController?: AbortController, // unused, for signature compatibility with parseAnthropicCompletion - options?: { workspace?: string; provider?: string } + options?: { + workspace?: string + provider?: string + onTokenUsage?: (usage: ChatTokenUsage) => void + } ): Promise<{ shouldContinue: boolean; tokenUsage: ChatTokenUsage }> { const finalToolCalls: Record = {} // The tool call currently receiving argument deltas; when the stream moves on @@ -1328,6 +1349,7 @@ export async function parseOpenAICompletion( callbacks.onMessageEnd() + options?.onTokenUsage?.(tokenUsage) // Stream over: every parsed call is queued until its turn in processToolCall. for (const toolCall of Object.values(finalToolCalls)) { if (toolCall.id) { diff --git a/frontend/src/lib/components/copilot/modelConfig.ts b/frontend/src/lib/components/copilot/modelConfig.ts index a11fd5c034..cc45a23dac 100644 --- a/frontend/src/lib/components/copilot/modelConfig.ts +++ b/frontend/src/lib/components/copilot/modelConfig.ts @@ -123,21 +123,77 @@ function normalizeVersionSeparators(model: string): string { return model.replace(/\./g, '-') } -// An entry that ends on a version digit must not run into a longer version: -// `gpt-4.1` collapses to `gpt-4-1`, which would otherwise claim the 128K -// `gpt-4-1106-preview` as a 1M model. Suffixes that continue with a separator -// (`claude-opus-4-8` in `...-4-8-v1`, `gpt-5` in `gpt-5-mini`) still match. -// Family fallbacks ending on a letter get no such guard — a version welded -// straight onto the name (`llama3.1`) is exactly what they exist to catch. -const MODEL_CONTEXT_WINDOW_MATCHERS: [matcher: RegExp, contextWindow: number][] = - MODEL_CONTEXT_WINDOWS.map(([name, contextWindow]) => { +/** Suffixes that name a route to a model rather than a different model. */ +const DECORATIVE_SUFFIXES = ['latest', 'preview', 'beta', 'stable'] + +/** + * Compile a most-specific-first `[name, value]` table into matchers against the + * bare model id. Shared with the pricing table so both resolve the same set of + * ids — a model whose window is known but whose price is not (or vice versa) + * should be a gap in one table, never a difference in matching. + * + * An entry that ends on a version digit must not run into a longer version: + * `gpt-4.1` collapses to `gpt-4-1`, which would otherwise claim + * `gpt-4-1106-preview`. Suffixes that continue with a separator + * (`claude-opus-4-8` in `...-4-8-v1`, `gpt-5` in `gpt-5-mini`) still match. + * Family fallbacks ending on a letter get no such guard — a version welded + * straight onto the name (`llama3.1`) is exactly what they exist to catch. + */ +export function buildModelMatchers( + entries: [name: string, value: T][], + { strictVariants = false }: { strictVariants?: boolean } = {} +): [RegExp, T][] { + return entries.map(([name, value]) => { const pattern = normalizeVersionSeparators(name).replace(/[.*+?^${}()|[\]\\]/g, '\\$&') - return [new RegExp(/\d$/.test(pattern) ? `${pattern}(?!\\d)` : pattern), contextWindow] + const guards = [ + // An entry ending on a version digit must not run into a longer version. + /\d$/.test(pattern) ? '(?!\\d)' : '', + // A named sub-model (`gpt-5-pro`, `gpt-5-mini`) is a different model with + // its own price, not another route to this one — so under strictVariants an + // entry does not match when a further *name* segment follows. What follows + // is only a decoration when it is a date (`-20251101`), Bedrock's `-v1`, or + // one of the alias words below, at the very end of the id + // (`claude-3-5-haiku-latest` is the same model as `claude-3-5-haiku`, and is + // a shipped default; `gpt-5-preview-pro` would be a different one again). + // Off by default: for a context window an inherited value is a safe + // approximation, for a price it is a wrong number. + // A further revision segment (`gpt-5` vs `gpt-5-4-mini`) is a different model + // too, and the entry-ends-on-a-digit guard above does not catch it once the + // separator is normalized. Only a short segment: a date is digits as well + // (`-20251101`) and stays a decoration. + strictVariants ? '(?!-\\d{1,3}(?:$|-))' : '', + strictVariants + ? `(?!-(?!(?:v\\d|${DECORATIVE_SUFFIXES.join('|')})$)[a-z])` + : '' + ].join('') + return [new RegExp(pattern + guards), value] }) +} + +/** + * The `provider:model` key the workspace AI settings use for their per-model maps + * (`max_tokens_per_model`, `model_pricing`). A bare model id is not enough: the + * same id can be served by more than one provider at different rates. + * + * Matched exactly, unlike the fuzzy tables above. Those tables generalize across + * every route to one model on purpose; a per-model *setting* must not, or an + * admin could not give two variants of a family different values — and the key is + * built from the exact id the provider config lists, which is the same string the + * chat sends. + */ +export function modelKey(provider: AIProvider | string, model: string): string { + return `${provider}:${model}` +} + +export function matchModel(matchers: [RegExp, T][], model: string): T | undefined { + const id = normalizeVersionSeparators(parseModelId(model).base) + return matchers.find(([matcher]) => matcher.test(id))?.[1] +} + +const MODEL_CONTEXT_WINDOW_MATCHERS = buildModelMatchers(MODEL_CONTEXT_WINDOWS) export function getKnownModelContextWindow(model: string): number | undefined { - const id = normalizeVersionSeparators(parseModelId(model).base) - return MODEL_CONTEXT_WINDOW_MATCHERS.find(([matcher]) => matcher.test(id))?.[1] + return matchModel(MODEL_CONTEXT_WINDOW_MATCHERS, model) } export function getModelContextWindow(model: string) { diff --git a/frontend/src/lib/components/copilot/modelPricing.test.ts b/frontend/src/lib/components/copilot/modelPricing.test.ts new file mode 100644 index 0000000000..e295682187 --- /dev/null +++ b/frontend/src/lib/components/copilot/modelPricing.test.ts @@ -0,0 +1,187 @@ +import { describe, expect, it } from 'vitest' +import { billedTokens } from './chat/tokenUsage' +import { estimateCost, priceSpend, resolveModelPrice } from './modelPricing' + +describe('resolveModelPrice', () => { + it('resolves the same model across the routes that decorate its id', () => { + const direct = resolveModelPrice('anthropic', 'claude-opus-5', undefined) + expect(direct?.price.input).toBe(5) + // A gateway prefix, a dot-versioned id, a date suffix and a variant suffix + // must all land on the same entry — a miss here silently under-reports cost. + for (const id of [ + 'anthropic/claude-opus-5', + 'anthropic/claude-opus-4.8', + 'claude-opus-4-8-20260101', + 'anthropic/claude-opus-5:thinking' + ]) { + expect(resolveModelPrice('openrouter', id, undefined)?.price.input).toBe(5) + } + }) + + it('does not let a version-digit entry claim a longer version', () => { + expect(resolveModelPrice('openai', 'gpt-4.1', undefined)?.price.input).toBe(2) + expect(resolveModelPrice('openai', 'gpt-4-1106-preview', undefined)?.price.input).not.toBe(2) + }) + + it('prices flat-rate Gemini Flash while leaving the tiered Pro alone', () => { + expect(resolveModelPrice('googleai', 'gemini-2.5-flash', undefined)?.price.input).toBe(0.3) + expect(resolveModelPrice('googleai', 'gemini-2.5-flash-lite', undefined)?.price.input).toBe(0.1) + expect(resolveModelPrice('googleai', 'gemini-3.5-flash', undefined)?.price.output).toBe(9) + // Pro charges roughly double above a 200k prompt, which a per-model rate cannot + // express, so it must stay unpriced rather than be estimated at the low tier. + expect(resolveModelPrice('googleai', 'gemini-2.5-pro', undefined)).toBeUndefined() + expect(resolveModelPrice('googleai', 'gemini-3.1-pro', undefined)).toBeUndefined() + // Promotional rates carry an end date a timeless table cannot represent. + expect(resolveModelPrice('googleai', 'gemini-3.7-flash', undefined)).toBeUndefined() + }) + + it('reports an unknown model as unpriced rather than guessing', () => { + expect(resolveModelPrice('customai', 'some-in-house-model', undefined)).toBeUndefined() + }) + + it('does not let another model inherit a price through a shared prefix', () => { + // A sub-model (`-pro`) or a newer revision (`gpt-5.6` → `gpt-5-6`) is a + // different model at a different rate; inheriting `gpt-5`'s would be off by + // an order of magnitude, and silently so. + expect(resolveModelPrice('openai', 'gpt-5', undefined)?.price.input).toBe(1.25) + expect(resolveModelPrice('openai', 'gpt-5-mini', undefined)?.price.input).toBe(0.25) + expect(resolveModelPrice('openai', 'gpt-5-pro', undefined)).toBeUndefined() + expect(resolveModelPrice('openai', 'gpt-5.6', undefined)).toBeUndefined() + expect(resolveModelPrice('googleai', 'gemini-3.1', undefined)).toBeUndefined() + // A revision carrying a variant has to be caught by the matcher, not by an + // explicit entry: `gpt-5.4-mini` cannot match the `gpt-5.4` one (the `-mini` + // makes it a sub-model), so nothing but the guard stops it reaching `gpt-5`. + expect(resolveModelPrice('openai', 'gpt-5.4-mini', undefined)).toBeUndefined() + expect(resolveModelPrice('openai', 'gpt-5.5-pro', undefined)).toBeUndefined() + }) + + it('still resolves the route decorations that name the same model', () => { + // Dates, Bedrock's -v1 and floating aliases are ways of spelling one model, + // not sub-models. `claude-3-5-haiku-latest` is a shipped picker default, so + // unpricing it would silently disable cost tracking out of the box. + expect(resolveModelPrice('anthropic', 'claude-opus-4-5-20251101', undefined)?.price.input).toBe(5) + // The revision guard must not swallow a date, which is digits too. + expect(resolveModelPrice('openai', 'gpt-5-2026-01-01', undefined)?.price.input).toBe(1.25) + expect( + resolveModelPrice('bedrock', 'anthropic.claude-sonnet-4-6-20250101-v1:0', undefined)?.price + .input + ).toBe(3) + expect(resolveModelPrice('anthropic', 'claude-3-5-haiku-latest', undefined)?.price.input).toBe( + 0.8 + ) + // …while a genuine sub-model stays unpriced, including one hiding behind a + // decoration. + expect(resolveModelPrice('openai', 'gpt-5-pro', undefined)).toBeUndefined() + expect(resolveModelPrice('openai', 'gpt-5-preview-pro', undefined)).toBeUndefined() + // A family fallback must not price a model the table deliberately left out, + // nor the floating alias pointing at it. + expect(resolveModelPrice('anthropic', 'claude-sonnet-5', undefined)).toBeUndefined() + expect( + resolveModelPrice('openrouter', '~anthropic/claude-sonnet-latest', undefined) + ).toBeUndefined() + }) + + it('prefers a workspace override, keeping the model’s own cache ratios', () => { + const resolved = resolveModelPrice('anthropic', 'claude-opus-5', { + 'anthropic:claude-opus-5': { input: 2, output: 8 } + }) + expect(resolved?.source).toBe('override') + expect(resolved?.price.input).toBe(2) + // Anthropic reads a cached prefix at a tenth and writes at 1.25x. + expect(resolved?.price.cacheRead).toBeCloseTo(0.2) + expect(resolved?.price.cacheWrite).toBeCloseTo(2.5) + }) + + it('applies the overridden model’s own cache discount, not Anthropic’s', () => { + // gpt-4o discounts a cached read by half, not by a tenth — an override that + // only states input/output must not silently inherit the Anthropic ratio. + const resolved = resolveModelPrice('openai', 'gpt-4o', { + 'openai:gpt-4o': { input: 2, output: 8 } + }) + expect(resolved?.price.cacheRead).toBeCloseTo(1) + }) + + it('bills an unpriced model’s cached tokens at its input rate', () => { + // Gemini Pro is deliberately unpriced, so there is no ratio to inherit. Falling + // back to Anthropic's tenth would invent a discount the provider may not give; + // the admin states the cache rates explicitly or pays full input. + const resolved = resolveModelPrice('googleai', 'gemini-2.5-pro', { + 'googleai:gemini-2.5-pro': { input: 2, output: 8 } + }) + expect(resolved?.price.cacheRead).toBe(2) + expect(resolved?.price.cacheWrite).toBe(2) + + const stated = resolveModelPrice('googleai', 'gemini-2.5-pro', { + 'googleai:gemini-2.5-pro': { input: 2, output: 8, cache_read: 0.5, cache_write: 1 } + }) + expect(stated?.price.cacheRead).toBe(0.5) + expect(stated?.price.cacheWrite).toBe(1) + }) + + it('ignores an override whose rates could not be a price', () => { + for (const bad of [{ input: -1, output: 8 }, { input: 1e9, output: 8 }]) { + const resolved = resolveModelPrice('anthropic', 'claude-opus-5', { + 'anthropic:claude-opus-5': bad + }) + expect(resolved?.source).toBe('builtin') + } + }) +}) + +describe('estimateCost', () => { + it('bills each token class at its own rate', () => { + const cost = estimateCost( + { input: 1_000_000, cacheRead: 1_000_000, cacheWrite: 1_000_000, output: 1_000_000 }, + { input: 5, output: 25, cacheRead: 0.5, cacheWrite: 6.25 } + ) + expect(cost).toBeCloseTo(5 + 0.5 + 6.25 + 25) + }) + + it('charges a cached prefix less than an uncached one', () => { + const usage = { + prompt: 100_000, + completion: 0, + total: 100_000, + cacheRead: 90_000, + cacheWrite: 0 + } + const uncached = { ...usage, cacheRead: 0 } + const price = { input: 5, output: 25, cacheRead: 0.5, cacheWrite: 6.25 } + expect(estimateCost(billedTokens(usage), price)).toBeLessThan( + estimateCost(billedTokens(uncached), price) + ) + }) +}) + +describe('priceSpend', () => { + it('prefers a provider-reported cost over the estimate', () => { + const priced = priceSpend( + [ + { + provider: 'openrouter', + model: 'anthropic/claude-opus-5', + tokens: { input: 1_000_000, cacheRead: 0, cacheWrite: 0, output: 0 }, + reportedCostUsd: 0.42 + } + ], + undefined + ) + expect(priced.total).toBe(0.42) + expect(priced.hasReported).toBe(true) + }) + + it('flags an unpriced model instead of counting it as free', () => { + const priced = priceSpend( + [ + { + provider: 'customai', + model: 'some-in-house-model', + tokens: { input: 1_000_000, cacheRead: 0, cacheWrite: 0, output: 0 } + } + ], + undefined + ) + expect(priced.hasUnpriced).toBe(true) + expect(priced.rows[0].cost).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/copilot/modelPricing.ts b/frontend/src/lib/components/copilot/modelPricing.ts new file mode 100644 index 0000000000..1e5b63247b --- /dev/null +++ b/frontend/src/lib/components/copilot/modelPricing.ts @@ -0,0 +1,288 @@ +import type { AIProvider, ModelPriceOverride } from '$lib/gen' +import { buildModelMatchers, matchModel, modelKey } from './modelConfig' + +/** Rates in USD per million tokens, one per billed token class. */ +export type ModelPrice = { + input: number + output: number + cacheRead: number + cacheWrite: number +} + +export type ModelPriceSource = 'override' | 'builtin' + +export type ResolvedModelPrice = { + price: ModelPrice + source: ModelPriceSource +} + +/** What a chat spent on one model, in tokens. */ +export type PricedTokens = { + input: number + cacheRead: number + cacheWrite: number + output: number +} + +// Fallbacks for entries that do not price their cache separately: Anthropic reads a +// cached prefix at a tenth of the input rate and writes one at 1.25x (5-minute TTL, +// the default the chat uses). The read ratio is NOT universal — OpenAI and Google +// discount a cached read far less — so every non-Anthropic entry below states its own +// `cacheRead` rather than inheriting this. Providers whose caching is automatic never +// report a cache write, so their write rate is unused. +const CACHE_READ_RATIO = 0.1 +const CACHE_WRITE_RATIO = 1.25 + +type PriceEntry = { + input: number + output: number + cacheRead?: number + cacheWrite?: number +} + +/** + * Published list prices, most specific entry first — the first name found in the + * bare model id wins, so vendor-namespaced and date-suffixed ids + * (anthropic/claude-opus-5, gpt-5-2026-01-01) still resolve. Matching is shared + * with the context-window table via `buildModelMatchers`. + * + * This is a best-effort snapshot: vendors change rates, ship models faster than + * this table is updated, and negotiated rates differ from list. A model that is + * not listed resolves to undefined and is reported as unpriced rather than + * guessed at, and any entry can be corrected per workspace from the AI settings. + * Providers whose catalogue turns over too quickly to track (DeepSeek, Mistral, + * Groq, TogetherAI, custom deployments) are deliberately absent. + * + * `null` marks a model that is known to exist but whose rates are not. Unpriced is + * a supported state (the UI says so and points at the override); a confidently + * wrong number is not — which is also why these matchers are built with + * `strictVariants`, so an unlisted sub-model (`gpt-5-pro`) reports no rate instead + * of inheriting its family's. + * + * One known gap the per-model shape cannot express: Anthropic's 1M-context beta + * charges more above a threshold. Usage is aggregated per model before pricing, so + * those requests are estimated at the standard tier and understate. An affected + * workspace can set the higher rate as its override. + */ +const MODEL_PRICES: [name: string, price: PriceEntry | null][] = [ + // Anthropic — Opus 4.1 and older bill at the pre-4.5 Opus rate, so the family + // fallback sits below the explicit entries rather than covering them. + ['claude-fable-5', { input: 10, output: 50 }], + ['claude-mythos-5', { input: 10, output: 50 }], + ['claude-opus-5', { input: 5, output: 25 }], + ['claude-opus-4-8', { input: 5, output: 25 }], + ['claude-opus-4-7', { input: 5, output: 25 }], + ['claude-opus-4-6', { input: 5, output: 25 }], + ['claude-opus-4-5', { input: 5, output: 25 }], + ['claude-opus-4-1', { input: 15, output: 75 }], + ['claude-opus-4', { input: 15, output: 75 }], + // Sonnet 5 runs a promotional rate with a published end date, and + // `claude-sonnet-latest` floats to it. Rates carry no date and apply at read + // time, so either figure misstates one side of that boundary — unpriced until + // the rate is a single number again. + ['claude-sonnet-5', null], + ['claude-sonnet-latest', null], + ['claude-sonnet-4-6', { input: 3, output: 15 }], + ['claude-sonnet-4-5', { input: 3, output: 15 }], + ['claude-sonnet-4', { input: 3, output: 15 }], + ['claude-haiku-4-5', { input: 1, output: 5 }], + ['claude-3-5-haiku', { input: 0.8, output: 4 }], + ['claude-opus', { input: 5, output: 25 }], + ['claude-sonnet', { input: 3, output: 15 }], + ['claude-haiku', { input: 1, output: 5 }], + // OpenAI — the cached-input discount varies by family (a tenth on gpt-5, a + // quarter on 4.1 and the o-series, half on 4o), so each entry carries its own + // rate. There is no charge for writing the cache and no usage field reporting + // one, so the write rate never applies. The -mini/-nano entries must precede + // the family entry, which would otherwise claim them. + // Revisions past gpt-5 are priced separately by OpenAI and are not tracked here. + // The matcher's revision guard already keeps them off the family rate; these + // entries stay so a revision the guard admits still resolves to no rate. + ['gpt-5.6', null], + ['gpt-5.5', null], + ['gpt-5.4', null], + ['gpt-5.2', null], + ['gpt-5.1', null], + ['gpt-5-mini', { input: 0.25, output: 2, cacheRead: 0.025 }], + ['gpt-5-nano', { input: 0.05, output: 0.4, cacheRead: 0.005 }], + ['gpt-5', { input: 1.25, output: 10, cacheRead: 0.125 }], + ['gpt-4.1-mini', { input: 0.4, output: 1.6, cacheRead: 0.1 }], + ['gpt-4.1-nano', { input: 0.1, output: 0.4, cacheRead: 0.025 }], + ['gpt-4.1', { input: 2, output: 8, cacheRead: 0.5 }], + ['gpt-4o-mini', { input: 0.15, output: 0.6, cacheRead: 0.075 }], + ['gpt-4o', { input: 2.5, output: 10, cacheRead: 1.25 }], + ['o4-mini', { input: 1.1, output: 4.4, cacheRead: 0.275 }], + ['o3-mini', { input: 1.1, output: 4.4, cacheRead: 0.55 }], + ['o3', { input: 2, output: 8, cacheRead: 0.5 }], + // Google — Flash takes a flat rate and is priced; Pro is not, because both its + // input and output roughly double above a 200k-token prompt and a per-model rate + // cannot express a threshold. Explicit context caching also bills storage per hour, + // which nothing here represents, so a workspace using it sees an underestimate. + // Gemini 3.7 and 3.6 Flash run a promotional rate with an end date, and stay + // unpriced for the same reason Sonnet 5 does. + ['gemini-2.5-flash-lite', { input: 0.1, output: 0.4, cacheRead: 0.01 }], + ['gemini-2.5-flash', { input: 0.3, output: 2.5, cacheRead: 0.03 }], + ['gemini-3.5-flash-lite', { input: 0.3, output: 2.5, cacheRead: 0.03 }], + ['gemini-3.5-flash', { input: 1.5, output: 9, cacheRead: 0.15 }], + ['gemini-3.7', null], + ['gemini-3.6', null], + ['gemini-3.1', null], + ['gemini-3', null], + ['gemini-2.5', null] +] + +const MODEL_PRICE_MATCHERS = buildModelMatchers( + MODEL_PRICES.map(([name, entry]): [string, ModelPrice | null] => [ + name, + entry && { + input: entry.input, + output: entry.output, + cacheRead: entry.cacheRead ?? entry.input * CACHE_READ_RATIO, + cacheWrite: entry.cacheWrite ?? entry.input * CACHE_WRITE_RATIO + } + ]), + { strictVariants: true } +) + +export function getKnownModelPrice(model: string): ModelPrice | undefined { + return matchModel(MODEL_PRICE_MATCHERS, model) ?? undefined +} + +/** + * Rates the API bounds on the way in — but an instance-level config is stored as an + * untyped settings blob that bypasses that handler, so the reader enforces the same + * bounds rather than rendering a negative, infinite or absurd total. + */ +const MAX_MODEL_RATE = 1000 + +function isUsableRate(rate: number | undefined): boolean { + return rate === undefined || (Number.isFinite(rate) && rate >= 0 && rate <= MAX_MODEL_RATE) +} + +/** What a cache rate falls back to when an override leaves it unset: the model's + * own published multiple of the input rate where the table has one, and the input + * rate itself where it does not, so an unstated discount is never borrowed from + * another vendor. Shared with the rates editor, which shows these as placeholders. */ +export function inheritedCacheRates( + model: string, + input: number +): { cacheRead: number; cacheWrite: number } { + const builtin = getKnownModelPrice(model) + return { + cacheRead: input * (builtin ? builtin.cacheRead / builtin.input : 1), + cacheWrite: input * (builtin ? builtin.cacheWrite / builtin.input : 1) + } +} + +/** + * The rate a workspace should be billed at for one model: its override when an + * admin set one, otherwise the published list price, otherwise nothing. An override + * that omits a cache rate takes it from `inheritedCacheRates`. + */ +export function resolveModelPrice( + provider: AIProvider | string, + model: string, + overrides: Record | undefined +): ResolvedModelPrice | undefined { + const builtin = getKnownModelPrice(model) + const candidate = overrides?.[modelKey(provider, model)] + const override = + candidate && + isUsableRate(candidate.input) && + isUsableRate(candidate.output) && + isUsableRate(candidate.cache_read) && + isUsableRate(candidate.cache_write) + ? candidate + : undefined + if (override) { + const inherited = inheritedCacheRates(model, override.input) + return { + source: 'override', + price: { + input: override.input, + output: override.output, + cacheRead: override.cache_read ?? inherited.cacheRead, + cacheWrite: override.cache_write ?? inherited.cacheWrite + } + } + } + return builtin ? { source: 'builtin', price: builtin } : undefined +} + +/** Cost in USD of `tokens` at `price`. */ +export function estimateCost(tokens: PricedTokens, price: ModelPrice): number { + return ( + (tokens.input * price.input + + tokens.cacheRead * price.cacheRead + + tokens.cacheWrite * price.cacheWrite + + tokens.output * price.output) / + 1_000_000 + ) +} + +/** Tokens spent on one model, from a chat's running totals or the usage API. */ +export type ModelSpend = { + provider: string + model: string + tokens: PricedTokens + /** What the provider billed, where it reports a figure. */ + reportedCostUsd?: number +} + +export type Priced = { + /** Undefined when no rate is known for the model — reported as unpriced, never guessed. */ + cost: number | undefined + source: ModelPriceSource | 'reported' | undefined +} + +export type PricedSpend = { + /** The input entries, each with its cost — callers carry their own fields through + * rather than zipping the result back against the input by index. */ + rows: (T & Priced)[] + total: number + /** True when at least one row has no rate, so `total` understates the truth. */ + hasUnpriced: boolean + /** True when at least one row is a figure the provider billed rather than an estimate. */ + hasReported: boolean +} + +/** + * Cost a set of per-model token counts. A provider-reported figure always wins: + * it is what was actually charged, where everything else is list price times + * tokens. `source` says which, so a view never presents an estimate as a bill. + */ +export function priceSpend( + spend: T[], + overrides: Record | undefined +): PricedSpend { + let total = 0 + let hasUnpriced = false + let hasReported = false + const rows = spend.map((entry): T & Priced => { + if (entry.reportedCostUsd !== undefined) { + hasReported = true + total += entry.reportedCostUsd + return { ...entry, cost: entry.reportedCostUsd, source: 'reported' } + } + const resolved = resolveModelPrice(entry.provider, entry.model, overrides) + if (!resolved) { + hasUnpriced = true + return { ...entry, cost: undefined, source: undefined } + } + const cost = estimateCost(entry.tokens, resolved.price) + total += cost + return { ...entry, cost, source: resolved.source } + }) + return { rows, total, hasUnpriced, hasReported } +} + +/** + * Money, at the precision the amount deserves: sub-cent spend is where a chat + * spends most of its life, and rounding it to `$0.00` would read as free. + */ +export function formatUsd(amount: number): string { + if (amount === 0) return '$0' + if (amount < 0.01) return `$${amount.toFixed(4)}` + if (amount < 1) return `$${amount.toFixed(3)}` + return `$${amount.toFixed(2)}` +} diff --git a/frontend/src/lib/components/workspaceSettings/AISettings.svelte b/frontend/src/lib/components/workspaceSettings/AISettings.svelte index 60d1cb77a0..5979e8a535 100644 --- a/frontend/src/lib/components/workspaceSettings/AISettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/AISettings.svelte @@ -5,9 +5,11 @@ type AIConfig, type AIProvider, type GetCopilotSettingsStateResponse, - type InstanceAISummary + type InstanceAISummary, + type ModelPriceOverride } from '$lib/gen' import { workspaceStore } from '$lib/stores' + import { copilotInfo } from '$lib/aiStore' import { sendUserToast } from '$lib/toast' import { AI_PROVIDERS, fetchAvailableModels, providerSupportsWebSearch } from '../copilot/lib' import { supportsAutocomplete } from '../copilot/utils' @@ -25,6 +27,8 @@ import Badge from '../common/badge/Badge.svelte' import Tooltip from '../Tooltip.svelte' import ModelTokenLimits from './ModelTokenLimits.svelte' + import ModelPricing from './ModelPricing.svelte' + import AiUsagePanel from './AiUsagePanel.svelte' import { setCopilotInfo } from '$lib/aiStore' import AIPromptsModal from '../settings/AIPromptsModal.svelte' import { Settings } from 'lucide-svelte' @@ -73,6 +77,7 @@ let metadataModel: string | undefined = $state(undefined) let customPrompts: Record = $state({}) let maxTokensPerModel: Record = $state({}) + let modelPricing: Record = $state({}) let usingOpenaiClientCredentialsOauth = $state(false) let workspaceOverrideEditorOpened = $state(false) @@ -83,6 +88,7 @@ let initialMetadataModel: string | undefined = $state(undefined) let initialCustomPrompts: Record = $state({}) let initialMaxTokensPerModel: Record = $state({}) + let initialModelPricing: Record = $state({}) let initialPrompts: Record = $state({}) let lastLoadedConfigKey = $state(undefined) @@ -110,6 +116,7 @@ codeCompletionModel = config?.code_completion_model?.model customPrompts = clone(config?.custom_prompts ?? {}) maxTokensPerModel = clone(config?.max_tokens_per_model ?? {}) + modelPricing = clone(config?.model_pricing ?? {}) for (const mode of ['edit', 'fix', 'gen']) { if (!(mode in customPrompts)) { customPrompts[mode] = '' @@ -124,6 +131,7 @@ initialCodeCompletionModel = codeCompletionModel initialCustomPrompts = clone(customPrompts) initialMaxTokensPerModel = clone(maxTokensPerModel) + initialModelPricing = clone(modelPricing) initialPrompts = clone(customPrompts) } @@ -139,6 +147,7 @@ codeCompletionModel = initialCodeCompletionModel customPrompts = clone(initialCustomPrompts) maxTokensPerModel = clone(initialMaxTokensPerModel) + modelPricing = clone(initialModelPricing) } $effect(() => { @@ -172,7 +181,8 @@ metadataModel !== initialMetadataModel || codeCompletionModel !== initialCodeCompletionModel || JSON.stringify(customPrompts) !== JSON.stringify(initialCustomPrompts) || - JSON.stringify(maxTokensPerModel) !== JSON.stringify(initialMaxTokensPerModel) + JSON.stringify(maxTokensPerModel) !== JSON.stringify(initialMaxTokensPerModel) || + JSON.stringify(modelPricing) !== JSON.stringify(initialModelPricing) ) $effect(() => { @@ -285,7 +295,8 @@ metadata_model, custom_prompts: Object.keys(custom_prompts).length > 0 ? custom_prompts : undefined, max_tokens_per_model: - Object.keys(maxTokensPerModel).length > 0 ? maxTokensPerModel : undefined + Object.keys(maxTokensPerModel).length > 0 ? maxTokensPerModel : undefined, + model_pricing: Object.keys(modelPricing).length > 0 ? modelPricing : undefined } : {} } @@ -610,6 +621,24 @@ scope={promptScope} /> +{#if promptScope === 'workspace'} + + +{/if} + + +{#if showWorkspaceOverrideEditor} + +{/if} + {#if showWorkspaceOverrideEditor} + import { AiService, ApiError, type AITokenUsageBucket, type ModelPriceOverride } from '$lib/gen' + import { formatUsd, priceSpend, type ModelSpend } from '../copilot/modelPricing' + import { formatTokenCount } from '../copilot/chat/tokenUsage' + import SettingCard from '../instanceSettings/SettingCard.svelte' + import Select from '../select/Select.svelte' + import ToggleButtonGroup from '../common/toggleButton-v2/ToggleButtonGroup.svelte' + import ToggleButton from '../common/toggleButton-v2/ToggleButton.svelte' + import { resource } from 'runed' + import Tooltip from '../meltComponents/Tooltip.svelte' + import DataTable from '../table/DataTable.svelte' + import Head from '../table/Head.svelte' + import Cell from '../table/Cell.svelte' + + // Workspace and rates are both passed in rather than read from a store: the + // settings component that mounts this one also serves the instance scope, and + // the rates that priced a chat are the workspace's *effective* ones, which an + // inheriting workspace does not hold itself. + let { + workspace, + modelPricing, + scope = 'workspace' + }: { + workspace: string + modelPricing: Record + scope?: 'workspace' | 'self' + } = $props() + + type GroupBy = 'day' | 'user' | 'model' + + let days = $state(30) + let groupBy = $state('day') + + const rangeOptions = [ + { label: 'Last 7 days', value: 7 }, + { label: 'Last 30 days', value: 30 }, + { label: 'Last 90 days', value: 90 } + ] + + let usage = resource( + () => ({ workspace, days, groupBy, scope }), + async ({ workspace, days, groupBy, scope }) => + workspace ? await AiService.listAiUsage({ workspace, days, groupBy, scope }) : undefined + ) + + // The API groups by (dimension, provider, model) so every bucket resolves to a + // single rate; the table folds those back into one line per dimension value. + type Bucket = ModelSpend & { key: string; requests: number } + + function toSpend(bucket: AITokenUsageBucket): Bucket { + return { + // Grouping by model has no separate dimension — the model is the key. + key: groupBy === 'model' ? `${bucket.provider}/${bucket.model}` : bucket.key || '—', + requests: bucket.requests, + provider: bucket.provider, + model: bucket.model, + tokens: { + input: bucket.input_tokens, + cacheRead: bucket.cache_read_tokens, + cacheWrite: bucket.cache_write_tokens, + output: bucket.output_tokens + }, + reportedCostUsd: + bucket.reported_cost_nano_usd != undefined + ? bucket.reported_cost_nano_usd / 1_000_000_000 + : undefined + } + } + + let priced = $derived(priceSpend((usage.current?.buckets ?? []).map(toSpend), modelPricing)) + + type Row = { + key: string + cost: number | undefined + /** Every model behind this line was billed back by its provider, so the + * figure is an invoice rather than an estimate. A line mixing sources — or + * one holding a model with no rate, whose spend the figure omits entirely — + * makes the weaker claim. */ + reported: boolean + tokensIn: number + tokensOut: number + requests: number + } + + // Only a 403 on the workspace scope is a permission problem; reading your own + // usage is open to any member. Attributing every failure to permissions sends an + // admin looking for access they already hold, and buries the real cause of the + // far more common transient ones (an expired session, a database hiccup). + function usageError(error: unknown): string { + if (scope === 'workspace' && error instanceof ApiError && error.status === 403) { + return 'Only workspace admins can read workspace usage.' + } + return 'Could not load usage. Try again in a moment.' + } + + // The headline sums both kinds, so it only escapes the ~ when nothing under it + // was estimated. + let totalIsEstimated = $derived( + priced.rows.some((row) => row.cost !== undefined && row.source !== 'reported') + ) + + let rows = $derived.by(() => { + const byKey = new Map() + for (const row of priced.rows) { + const existing = byKey.get(row.key) ?? { + key: row.key, + cost: undefined, + reported: true, + tokensIn: 0, + tokensOut: 0, + requests: 0 + } + existing.tokensIn += row.tokens.input + row.tokens.cacheRead + row.tokens.cacheWrite + existing.tokensOut += row.tokens.output + existing.requests += row.requests + if (row.cost !== undefined) { + existing.cost = (existing.cost ?? 0) + row.cost + } + existing.reported &&= row.source === 'reported' + byKey.set(row.key, existing) + } + return [...byKey.values()].sort((a, b) => (b.cost ?? 0) - (a.cost ?? 0)) + }) + + + +
+
+
+
+