diff --git a/backend/.sqlx/query-24fcc2b69f30953915f0cbf246e1c19b2310075a644d81c21784c991e52b4001.json b/backend/.sqlx/query-24fcc2b69f30953915f0cbf246e1c19b2310075a644d81c21784c991e52b4001.json new file mode 100644 index 0000000000..a2cd96857a --- /dev/null +++ b/backend/.sqlx/query-24fcc2b69f30953915f0cbf246e1c19b2310075a644d81c21784c991e52b4001.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO ai_token_usage (workspace_id, email, provider, model, session_id, input_tokens, cache_read_tokens, cache_write_tokens, output_tokens, reported_cost_nano_usd, requests)\n SELECT $1, $2, * FROM UNNEST($3::text[], $4::text[], $5::text[], $6::bigint[], $7::bigint[], $8::bigint[], $9::bigint[], $10::bigint[], $11::bigint[])\n ON CONFLICT (workspace_id, day, email, provider, model, session_id)\n DO UPDATE SET\n input_tokens = ai_token_usage.input_tokens + EXCLUDED.input_tokens,\n cache_read_tokens = ai_token_usage.cache_read_tokens + EXCLUDED.cache_read_tokens,\n cache_write_tokens = ai_token_usage.cache_write_tokens + EXCLUDED.cache_write_tokens,\n output_tokens = ai_token_usage.output_tokens + EXCLUDED.output_tokens,\n reported_cost_nano_usd = CASE\n WHEN EXCLUDED.reported_cost_nano_usd IS NULL\n THEN ai_token_usage.reported_cost_nano_usd\n ELSE COALESCE(ai_token_usage.reported_cost_nano_usd, 0)\n + EXCLUDED.reported_cost_nano_usd\n END,\n requests = ai_token_usage.requests + EXCLUDED.requests,\n updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "TextArray", + "TextArray", + "TextArray", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array", + "Int8Array" + ] + }, + "nullable": [] + }, + "hash": "24fcc2b69f30953915f0cbf246e1c19b2310075a644d81c21784c991e52b4001" +} diff --git a/backend/.sqlx/query-43ec1c4adf6e453848b532bf3fab5de8825d247ea0b337007ee43c9cdffac83d.json b/backend/.sqlx/query-43ec1c4adf6e453848b532bf3fab5de8825d247ea0b337007ee43c9cdffac83d.json new file mode 100644 index 0000000000..efda9b2a2b --- /dev/null +++ b/backend/.sqlx/query-43ec1c4adf6e453848b532bf3fab5de8825d247ea0b337007ee43c9cdffac83d.json @@ -0,0 +1,72 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n (CASE $3::text\n WHEN 'day' THEN day::text\n WHEN 'user' THEN email\n WHEN 'session' THEN session_id\n ELSE ''\n END) AS \"key!\",\n provider AS \"provider!\",\n model AS \"model!\",\n SUM(input_tokens)::bigint AS \"input_tokens!\",\n SUM(cache_read_tokens)::bigint AS \"cache_read_tokens!\",\n SUM(cache_write_tokens)::bigint AS \"cache_write_tokens!\",\n SUM(output_tokens)::bigint AS \"output_tokens!\",\n SUM(reported_cost_nano_usd)::bigint AS \"reported_cost_nano_usd\",\n SUM(requests)::bigint AS \"requests!\"\n FROM ai_token_usage\n WHERE workspace_id = $1 AND day >= CURRENT_DATE - $2::int\n GROUP BY 1, provider, model\n ORDER BY 1 DESC, SUM(input_tokens + output_tokens) DESC\n LIMIT 1000", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "key!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "provider!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "model!", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "input_tokens!", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "cache_read_tokens!", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "cache_write_tokens!", + "type_info": "Int8" + }, + { + "ordinal": 6, + "name": "output_tokens!", + "type_info": "Int8" + }, + { + "ordinal": 7, + "name": "reported_cost_nano_usd", + "type_info": "Int8" + }, + { + "ordinal": 8, + "name": "requests!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Int4", + "Text" + ] + }, + "nullable": [ + null, + false, + false, + null, + null, + null, + null, + null, + null + ] + }, + "hash": "43ec1c4adf6e453848b532bf3fab5de8825d247ea0b337007ee43c9cdffac83d" +} diff --git a/backend/migrations/20260813164338_ai_token_usage.down.sql b/backend/migrations/20260813164338_ai_token_usage.down.sql new file mode 100644 index 0000000000..0fa1b47462 --- /dev/null +++ b/backend/migrations/20260813164338_ai_token_usage.down.sql @@ -0,0 +1 @@ +DROP TABLE ai_token_usage; diff --git a/backend/migrations/20260813164338_ai_token_usage.up.sql b/backend/migrations/20260813164338_ai_token_usage.up.sql new file mode 100644 index 0000000000..b20e471db5 --- /dev/null +++ b/backend/migrations/20260813164338_ai_token_usage.up.sql @@ -0,0 +1,42 @@ +-- Per-workspace AI token spend, accumulated from the chat client. Rows hold token +-- counts rather than money: prices live in the frontend price table plus the +-- workspace's `ai_config.model_pricing` overrides and are applied at read time, so +-- correcting a price also corrects the history. `reported_cost_nano_usd` is the +-- exception — a few providers (OpenRouter) return what they actually charged, and +-- that figure wins over the estimate. +-- +-- Distinct from `feature_usage`, which is anonymous telemetry that leaves the +-- instance and is pruned after 60 days; spend is per-user and kept. +CREATE TABLE ai_token_usage ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE, + day DATE NOT NULL DEFAULT CURRENT_DATE, + email VARCHAR(255) NOT NULL, + provider VARCHAR(50) NOT NULL, + model VARCHAR(255) NOT NULL, + -- Empty for chats that are not attached to an AI session. + session_id VARCHAR(50) NOT NULL DEFAULT '', + -- Uncached input only; the two cache columns hold the rest of the prompt, so + -- each column maps to exactly one price and they never double-count. + input_tokens BIGINT NOT NULL DEFAULT 0, + cache_read_tokens BIGINT NOT NULL DEFAULT 0, + cache_write_tokens BIGINT NOT NULL DEFAULT 0, + output_tokens BIGINT NOT NULL DEFAULT 0, + reported_cost_nano_usd BIGINT, + requests BIGINT NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, day, email, provider, model, session_id) +); + +-- The usage listing filters on workspace and a date range; the PK only reaches +-- `day` through `email`, so it cannot serve that on its own. +CREATE INDEX idx_ai_token_usage_ws_day ON ai_token_usage (workspace_id, day DESC); + +GRANT ALL ON ai_token_usage TO windmill_admin; +GRANT ALL ON ai_token_usage TO windmill_user; + +-- Both handlers go through the raw pool, so no policy is needed for them to work. +-- Enabling RLS with an admin-only policy is the backstop: a future query that +-- reaches this table through UserDB sees nothing rather than every user's spend. +ALTER TABLE ai_token_usage ENABLE ROW LEVEL SECURITY; + +CREATE POLICY admin_policy ON ai_token_usage FOR ALL TO windmill_admin USING (true); diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index bcd5c66e34..be6e1fd0d2 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -11838,6 +11838,58 @@ paths: schema: type: string + /w/{workspace}/ai/usage: + post: + summary: record AI token usage for the calling user + operationId: recordAiUsage + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - events + properties: + events: + type: array + items: + $ref: "#/components/schemas/AITokenUsageEvent" + responses: + "204": + description: usage recorded + get: + summary: list aggregated AI token usage for the workspace (admin only) + operationId: listAiUsage + tags: + - ai + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: days + in: query + schema: + type: integer + minimum: 1 + maximum: 365 + - name: group_by + in: query + schema: + type: string + enum: [day, user, model, session] + responses: + "200": + description: usage buckets + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/AITokenUsageBucket" + /w/{workspace}/ai_skills/list: get: summary: list the workspace AI chat skills (name + description only) @@ -25999,6 +26051,84 @@ components: type: integer minimum: 1 maximum: 2000000 + model_pricing: + type: object + additionalProperties: + $ref: "#/components/schemas/ModelPriceOverride" + + ModelPriceOverride: + type: object + description: negotiated rates in USD per million tokens, keyed `provider:model` + properties: + input: + type: number + output: + type: number + cache_read: + type: number + cache_write: + type: number + required: + - input + - output + + AITokenUsageEvent: + type: object + properties: + provider: + $ref: "#/components/schemas/AIProvider" + model: + type: string + session_id: + type: string + input_tokens: + type: integer + cache_read_tokens: + type: integer + cache_write_tokens: + type: integer + output_tokens: + type: integer + reported_cost_nano_usd: + type: integer + description: only set by providers that bill back an exact figure + requests: + type: integer + required: + - provider + - model + + AITokenUsageBucket: + type: object + properties: + key: + type: string + description: the grouped dimension's value; empty when grouping by model + provider: + type: string + model: + type: string + input_tokens: + type: integer + cache_read_tokens: + type: integer + cache_write_tokens: + type: integer + output_tokens: + type: integer + reported_cost_nano_usd: + type: integer + requests: + type: integer + required: + - key + - provider + - model + - input_tokens + - cache_read_tokens + - cache_write_tokens + - output_tokens + - requests InstanceAIProviderSummary: type: object diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index efc2ba7e60..24b7987acc 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -3,11 +3,16 @@ use crate::utils::check_scopes; #[cfg(feature = "bedrock")] use axum::routing::get; -#[cfg(feature = "bedrock")] use axum::Json; -use axum::{body::Bytes, extract::Path, response::IntoResponse, routing::post, Extension, Router}; +use axum::{ + body::Bytes, + extract::{Path, Query}, + response::IntoResponse, + routing::post, + Extension, Router, +}; use futures::StreamExt; -use http::{HeaderMap, Method}; +use http::{HeaderMap, Method, StatusCode}; use quick_cache::sync::Cache; use reqwest::{Client, RequestBuilder}; use serde::{Deserialize, Serialize}; @@ -37,7 +42,7 @@ use windmill_ai::proxy::{ use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::db::UserDB; use windmill_common::error::{to_anyhow, Error, Result}; -use windmill_common::utils::configure_client; +use windmill_common::utils::{configure_client, require_admin}; use windmill_common::variables::{get_variable_or_self, get_variable_or_self_as}; // AI timeout configuration constants @@ -417,6 +422,23 @@ pub struct AIConfig { pub custom_prompts: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub max_tokens_per_model: Option>, + /// Per-model price overrides, keyed `provider:model` like `max_tokens_per_model`. + /// Only models whose rates differ from the built-in table are stored. + #[serde(skip_serializing_if = "Option::is_none")] + pub model_pricing: Option>, +} + +/// Negotiated rates in USD per million tokens. Cache rates fall back to the +/// provider's usual multiples of the input rate when left unset, so an admin who +/// only knows their input/output pricing does not have to invent the other two. +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct ModelPriceOverride { + pub input: f64, + pub output: f64, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_read: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_write: Option, } impl AIConfig { @@ -432,7 +454,9 @@ pub fn global_service() -> Router { } pub fn workspaced_service() -> Router { - let router = Router::new().route("/proxy/{*ai}", post(proxy).get(proxy)); + let router = Router::new() + .route("/proxy/{*ai}", post(proxy).get(proxy)) + .route("/usage", post(record_ai_usage).get(list_ai_usage)); #[cfg(feature = "bedrock")] let router = router.route("/check_bedrock_credentials", get(check_bedrock_credentials)); @@ -440,6 +464,227 @@ pub fn workspaced_service() -> Router { router } +/// One provider request's worth of tokens, as counted by the chat client. +#[derive(Deserialize)] +struct AIUsageEvent { + provider: String, + model: String, + #[serde(default)] + session_id: String, + #[serde(default)] + input_tokens: i64, + #[serde(default)] + cache_read_tokens: i64, + #[serde(default)] + cache_write_tokens: i64, + #[serde(default)] + output_tokens: i64, + /// Only the providers that bill back an exact figure set this. + #[serde(default)] + reported_cost_nano_usd: Option, + #[serde(default)] + requests: Option, +} + +#[derive(Deserialize)] +struct RecordAIUsagePayload { + events: Vec, +} + +const MAX_AI_USAGE_EVENTS: usize = 50; +/// Well above any single conversation and far below an i64 overflow, so a client +/// bug caps out at one absurd row instead of poisoning the running total. +const MAX_TOKENS_PER_EVENT: i64 = 100_000_000; +/// $1000 in nano-USD. +const MAX_REPORTED_COST_PER_EVENT: i64 = 1_000_000_000_000; + +/// Model ids carry vendor prefixes and variant suffixes (`anthropic/claude-opus-5:thinking`), +/// so the shape check is looser than an identifier but still excludes whitespace and +/// anything that would not be a model id. +fn is_model_shaped(s: &str, max_len: usize) -> bool { + !s.is_empty() + && s.len() <= max_len + && s.chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | ':' | '.' | '/' | '~')) +} + +/// Accumulate one workspace's AI token spend. Values are clamped and the caller's +/// email comes from the session, never the payload — the client is trusted to +/// report its own usage, not to attribute it to someone else. +async fn record_ai_usage( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Json(payload): Json, +) -> Result { + // Pre-sum duplicate keys: two rows hitting the same conflict target in a single + // INSERT error out ("cannot affect row a second time"). + let mut agg: HashMap<(String, String, String), AIUsageTotals> = HashMap::new(); + for e in payload.events.into_iter().take(MAX_AI_USAGE_EVENTS) { + if AIProvider::try_from(e.provider.as_str()).is_err() + || !is_model_shaped(&e.model, 255) + || !(e.session_id.is_empty() || is_model_shaped(&e.session_id, 50)) + { + continue; + } + let totals = agg + .entry((e.provider, e.model, e.session_id)) + .or_insert_with(AIUsageTotals::default); + totals.input += e.input_tokens.clamp(0, MAX_TOKENS_PER_EVENT); + totals.cache_read += e.cache_read_tokens.clamp(0, MAX_TOKENS_PER_EVENT); + totals.cache_write += e.cache_write_tokens.clamp(0, MAX_TOKENS_PER_EVENT); + totals.output += e.output_tokens.clamp(0, MAX_TOKENS_PER_EVENT); + totals.requests += e.requests.unwrap_or(1).clamp(0, MAX_AI_USAGE_EVENTS as i64); + if let Some(cost) = e.reported_cost_nano_usd { + totals.reported_cost = Some( + totals.reported_cost.unwrap_or(0) + cost.clamp(0, MAX_REPORTED_COST_PER_EVENT), + ); + } + } + if agg.is_empty() { + return Ok(StatusCode::NO_CONTENT); + } + + let mut providers = Vec::with_capacity(agg.len()); + let mut models = Vec::with_capacity(agg.len()); + let mut session_ids = Vec::with_capacity(agg.len()); + let mut inputs = Vec::with_capacity(agg.len()); + let mut cache_reads = Vec::with_capacity(agg.len()); + let mut cache_writes = Vec::with_capacity(agg.len()); + let mut outputs = Vec::with_capacity(agg.len()); + let mut reported_costs: Vec> = Vec::with_capacity(agg.len()); + let mut requests = Vec::with_capacity(agg.len()); + for ((provider, model, session_id), totals) in agg { + providers.push(provider); + models.push(model); + session_ids.push(session_id); + inputs.push(totals.input); + cache_reads.push(totals.cache_read); + cache_writes.push(totals.cache_write); + outputs.push(totals.output); + reported_costs.push(totals.reported_cost); + requests.push(totals.requests); + } + + sqlx::query!( + "INSERT INTO ai_token_usage (workspace_id, email, provider, model, session_id, \ + input_tokens, cache_read_tokens, cache_write_tokens, output_tokens, \ + reported_cost_nano_usd, requests) + SELECT $1, $2, * FROM UNNEST($3::text[], $4::text[], $5::text[], $6::bigint[], \ + $7::bigint[], $8::bigint[], $9::bigint[], $10::bigint[], $11::bigint[]) + ON CONFLICT (workspace_id, day, email, provider, model, session_id) + DO UPDATE SET + input_tokens = ai_token_usage.input_tokens + EXCLUDED.input_tokens, + cache_read_tokens = ai_token_usage.cache_read_tokens + EXCLUDED.cache_read_tokens, + cache_write_tokens = ai_token_usage.cache_write_tokens + EXCLUDED.cache_write_tokens, + output_tokens = ai_token_usage.output_tokens + EXCLUDED.output_tokens, + reported_cost_nano_usd = CASE + WHEN EXCLUDED.reported_cost_nano_usd IS NULL + THEN ai_token_usage.reported_cost_nano_usd + ELSE COALESCE(ai_token_usage.reported_cost_nano_usd, 0) + + EXCLUDED.reported_cost_nano_usd + END, + requests = ai_token_usage.requests + EXCLUDED.requests, + updated_at = now()", + &w_id, + &authed.email, + &providers, + &models, + &session_ids, + &inputs, + &cache_reads, + &cache_writes, + &outputs, + &reported_costs as &[Option], + &requests + ) + .execute(&db) + .await?; + + Ok(StatusCode::NO_CONTENT) +} + +#[derive(Default)] +struct AIUsageTotals { + input: i64, + cache_read: i64, + cache_write: i64, + output: i64, + reported_cost: Option, + requests: i64, +} + +#[derive(Deserialize)] +struct ListAIUsageQuery { + days: Option, + group_by: Option, +} + +/// A bucket always carries its provider and model: the caller prices it from a +/// per-model rate table, which a bucket spanning several models could not be +/// resolved against. +#[derive(Serialize)] +struct AITokenUsageBucket { + key: String, + provider: String, + model: String, + input_tokens: i64, + cache_read_tokens: i64, + cache_write_tokens: i64, + output_tokens: i64, + reported_cost_nano_usd: Option, + requests: i64, +} + +async fn list_ai_usage( + authed: ApiAuthed, + Extension(db): Extension, + Path(w_id): Path, + Query(query): Query, +) -> Result>> { + require_admin(authed.is_admin, &authed.username)?; + + let days = query.days.unwrap_or(30).clamp(1, 365); + let group_by = query.group_by.as_deref().unwrap_or("day"); + if !matches!(group_by, "day" | "user" | "model" | "session") { + return Err(Error::BadRequest(format!( + "Unsupported group_by: {}", + group_by + ))); + } + + let rows = sqlx::query_as!( + AITokenUsageBucket, + r#"SELECT + (CASE $3::text + WHEN 'day' THEN day::text + WHEN 'user' THEN email + WHEN 'session' THEN session_id + ELSE '' + END) AS "key!", + provider AS "provider!", + model AS "model!", + SUM(input_tokens)::bigint AS "input_tokens!", + SUM(cache_read_tokens)::bigint AS "cache_read_tokens!", + SUM(cache_write_tokens)::bigint AS "cache_write_tokens!", + SUM(output_tokens)::bigint AS "output_tokens!", + SUM(reported_cost_nano_usd)::bigint AS "reported_cost_nano_usd", + SUM(requests)::bigint AS "requests!" + FROM ai_token_usage + WHERE workspace_id = $1 AND day >= CURRENT_DATE - $2::int + GROUP BY 1, provider, model + ORDER BY 1 DESC, SUM(input_tokens + output_tokens) DESC + LIMIT 1000"#, + &w_id, + days, + group_by + ) + .fetch_all(&db) + .await?; + + Ok(Json(rows)) +} + /// Check if AWS Bedrock credentials are available from environment variables. #[cfg(feature = "bedrock")] async fn check_bedrock_credentials( diff --git a/frontend/src/lib/aiStore.ts b/frontend/src/lib/aiStore.ts index 1e8fb6ac61..0955f68ee5 100644 --- a/frontend/src/lib/aiStore.ts +++ b/frontend/src/lib/aiStore.ts @@ -3,7 +3,12 @@ // import aiStore back, and such a cycle crashes the app once the bundler splits it across // chunks (docs/frontend-import-cycles.md; the build fails on the chunk cycle, not on this). import { writable, get } from 'svelte/store' -import { type AIProviderModel, type AIProvider, type AIConfig } from './gen' +import { + type AIProviderModel, + type AIProvider, + type AIConfig, + type ModelPriceOverride +} from './gen' import { aiUserDisabled, COPILOT_SESSION_MODEL_SETTING_NAME, @@ -41,6 +46,8 @@ export const copilotInfo = writable<{ aiModels: AIProviderModel[] customPrompts?: Record maxTokensPerModel?: Record + /** Negotiated rates per `provider:model`, overriding the built-in price table. */ + modelPricing?: Record webSearchEnabledProviders?: Partial> }>({ enabled: false, @@ -50,6 +57,7 @@ export const copilotInfo = writable<{ aiModels: [], customPrompts: {}, maxTokensPerModel: {}, + modelPricing: {}, webSearchEnabledProviders: {} }) @@ -124,6 +132,7 @@ export function setCopilotInfo(aiConfig: AIConfig) { aiModels: aiModels, customPrompts: aiConfig.custom_prompts ?? {}, maxTokensPerModel: aiConfig.max_tokens_per_model ?? {}, + modelPricing: aiConfig.model_pricing ?? {}, webSearchEnabledProviders }) } else { @@ -137,6 +146,7 @@ export function setCopilotInfo(aiConfig: AIConfig) { aiModels: [], customPrompts: {}, maxTokensPerModel: {}, + modelPricing: {}, webSearchEnabledProviders: {} }) } diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index c3875f702f..9df24d394c 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -28,6 +28,7 @@ import type { ContextElement } from './context' import ChatQuickActions from './ChatQuickActions.svelte' import ContextUsageIndicator from './ContextUsageIndicator.svelte' + import CostIndicator from './CostIndicator.svelte' import AIChatModelSettings from './AIChatModelSettings.svelte' import ChatMode from './ChatMode.svelte' import DatatableCreationPolicy from './DatatableCreationPolicy.svelte' @@ -981,6 +982,7 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> {/if} + {#if aiChatManager.mode === AIMode.APP && appContext && (appContext.inspectorElement || appContext.codeSelection)} diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index 1faa9c003b..e27508aad3 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -99,7 +99,13 @@ import type AIChatInput from './AIChatInput.svelte' import { prepareApiSystemMessage, prepareApiUserMessage } from './api/core' import { runChatLoop, truncateToToolPairedPrefix } from './chatLoop' import { sanitizeToolCallArguments } from './toolCallArguments' -import { normalizeContextUsage } from './tokenUsage' +import { + addModelTokenUsage, + billedTokens, + normalizeContextUsage, + type ModelTokenUsageTotals +} from './tokenUsage' +import { logAiUsage } from '$lib/utils/aiUsageReporter' import type { ReviewChangesOpts } from './monaco-adapter' import { getCurrentModel, @@ -514,6 +520,11 @@ export class AIChatManager { * (provider never reported, turn failed, history rewound). Never holds a * guess: readers go through `contextTokens`, which estimates lazily. */ contextUsage = $state(undefined) + /** What this conversation has spent, in tokens, per `provider:model`. Unlike + * `contextUsage` this describes money already spent rather than the current + * history, so compaction and rewinds leave it alone — it is cleared only when + * the conversation is (New chat) and restored when one is loaded. */ + usageByModel = $state({}) // Circuit breaker for summary-based compaction: after repeated failures the // summary round-trip is skipped in favor of drop-oldest. Reset on any // successful summarization. Not persisted — a fresh load gets a fresh chance. @@ -645,6 +656,42 @@ export class AIChatManager { await this.#persistModifiedItems() } + /** Plain snapshot for persistence; undefined while nothing has been spent, so + * a chat that never ran a turn stores no usage field at all. */ + private usageSnapshot(): ModelTokenUsageTotals | undefined { + const snapshot = $state.snapshot(this.usageByModel) as ModelTokenUsageTotals + return Object.keys(snapshot).length > 0 ? snapshot : undefined + } + + /** Fold a completed turn's usage into the conversation's running spend and + * report it for the workspace usage view. Only token counts leave the browser: + * rates are applied when the usage is read, so a corrected price also corrects + * everything already recorded. */ + private recordUsage(byModel: ModelTokenUsageTotals | undefined) { + // Accounting must never take a turn down with it: a path that reports no + // per-model breakdown simply records nothing. + for (const entry of Object.values(byModel ?? {})) { + this.usageByModel = addModelTokenUsage( + this.usageByModel, + entry.provider, + entry.model, + entry.usage + ) + const tokens = billedTokens(entry.usage) + logAiUsage({ + provider: entry.provider, + model: entry.model, + sessionId: this.sessionId, + inputTokens: tokens.input, + cacheReadTokens: tokens.cacheRead, + cacheWriteTokens: tokens.cacheWrite, + outputTokens: tokens.output, + costUsd: entry.usage.cost, + workspace: this.operatingWorkspace + }) + } + } + // Serialized, snapshot-at-write-time persistence: two rapid dock actions // would otherwise race their saveChat writes, and the earlier (staler) // snapshot could land last — dropping the later mutation until the next @@ -657,7 +704,8 @@ export class AIChatManager { this.displayMessages, this.messages, this.contextUsage, - this.modifiedItems ? [...this.modifiedItems] : undefined + this.modifiedItems ? [...this.modifiedItems] : undefined, + this.usageSnapshot() ) // Swallow (and log) a failed write so it can't wedge the queue as a // rejected link — the next persist snapshots the full current set, so @@ -961,6 +1009,7 @@ export class AIChatManager { this.messages, this.contextUsage, undefined, + this.usageSnapshot(), $state.snapshot(this.backgroundJobs) ) .catch((e) => console.error('Failed to persist background jobs', e)) @@ -1397,7 +1446,8 @@ export class AIChatManager { this.displayMessages, this.messages, this.contextUsage, - this.modifiedItems ? [...this.modifiedItems] : undefined + this.modifiedItems ? [...this.modifiedItems] : undefined, + this.usageSnapshot() ) sendUserToast('Conversation compacted.') break @@ -2343,6 +2393,9 @@ export class AIChatManager { } } }) + if (result.tokenUsage.total > 0) { + this.recordUsage(result.tokenUsageByModel) + } if (this.isSessionChat && this.sessionId && result.tokenUsage.total > 0) { logFeatureUsage('ai_session', 'tokens', { entityId: this.sessionId, @@ -2961,7 +3014,8 @@ export class AIChatManager { this.displayMessages, this.messages, this.contextUsage, - this.modifiedItems ? [...this.modifiedItems] : undefined + this.modifiedItems ? [...this.modifiedItems] : undefined, + this.usageSnapshot() ) this.replyReveal.reset() @@ -3010,7 +3064,8 @@ export class AIChatManager { this.displayMessages, this.messages, this.contextUsage, - this.modifiedItems ? [...this.modifiedItems] : undefined + this.modifiedItems ? [...this.modifiedItems] : undefined, + this.usageSnapshot() ) } } @@ -3162,7 +3217,8 @@ export class AIChatManager { this.displayMessages, this.messages, this.contextUsage, - this.modifiedItems ? [...this.modifiedItems] : undefined + this.modifiedItems ? [...this.modifiedItems] : undefined, + this.usageSnapshot() ) // Still counts as the saved first turn — skipping the hook here would // permanently miss it (the next turn isn't "first" anymore). @@ -3211,7 +3267,8 @@ export class AIChatManager { this.displayMessages, this.messages, this.contextUsage, - this.modifiedItems ? [...this.modifiedItems] : undefined + this.modifiedItems ? [...this.modifiedItems] : undefined, + this.usageSnapshot() ) } if (!wasAborted) { @@ -3235,7 +3292,8 @@ export class AIChatManager { this.displayMessages, this.messages, this.contextUsage, - this.modifiedItems ? [...this.modifiedItems] : undefined + this.modifiedItems ? [...this.modifiedItems] : undefined, + this.usageSnapshot() ) // Only this branch is a clean send: the queued-message flush below // auto-sends the next message after it (set after saveChat so a @@ -3296,7 +3354,8 @@ export class AIChatManager { this.displayMessages, this.messages, this.contextUsage, - this.modifiedItems ? [...this.modifiedItems] : undefined + this.modifiedItems ? [...this.modifiedItems] : undefined, + this.usageSnapshot() ) } catch (saveErr) { console.error('Failed to persist partial chat after error', saveErr) @@ -3510,11 +3569,13 @@ export class AIChatManager { this.displayMessages, this.messages, this.contextUsage, - this.modifiedItems ? [...this.modifiedItems] : undefined + this.modifiedItems ? [...this.modifiedItems] : undefined, + this.usageSnapshot() ) this.displayMessages = [] this.messages = [] this.contextUsage = undefined + this.usageByModel = {} // The mask belongs to the conversation just saved — the fresh chat starts // its own (empty) tracking; carrying entries over would claim the previous // conversation's edits for the new one. Untracked chats stay untracked. @@ -3545,6 +3606,7 @@ export class AIChatManager { this.displayMessages = chat.displayMessages this.messages = chat.actualMessages this.contextUsage = normalizeContextUsage(chat.contextUsage) + this.usageByModel = chat.usageByModel ? { ...chat.usageByModel } : {} // Seed the modified-items mask from the stored chat. A session's Edits // surface is scoped strictly to what this session edited, so it must never // fall back to showing every draft in the (possibly forked) workspace: a diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts index 3b7dd521c0..d19e75e924 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts @@ -1832,7 +1832,13 @@ describe('AIChatManager context compaction', () => { // compaction-time save) so a rolled-back turn keeps a consistent value // 4th arg: the modified-items mask rides on every save (undefined here — // this bare manager never initialised tracking). - expect(saveChat).toHaveBeenCalledWith(expect.anything(), expect.anything(), 650_000, undefined) + expect(saveChat).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + 650_000, + undefined, + undefined + ) // At commit, the no-report turn clears the stored value; the readable // number falls back to estimating the now-tiny compacted history expect(manager.contextUsage).toBeUndefined() @@ -2898,14 +2904,14 @@ describe('AIChatManager background job completion', () => { expect(saveChat).not.toHaveBeenCalled() manager.updateJob('job-1', { status: 'success' }) await vi.waitFor(() => expect(saveChat).toHaveBeenCalledTimes(1)) - expect(saveChat.mock.calls[0][4]).toEqual([ + expect(saveChat.mock.calls[0][5]).toEqual([ expect.objectContaining({ jobId: 'job-1', status: 'success' }) ]) // Reviewing persists the flag; re-reviewing is a no-op (no extra write). manager.markJobsReviewed(['job-1']) await vi.waitFor(() => expect(saveChat).toHaveBeenCalledTimes(2)) - expect(saveChat.mock.calls[1][4]).toEqual([ + expect(saveChat.mock.calls[1][5]).toEqual([ expect.objectContaining({ jobId: 'job-1', reviewed: true }) ]) manager.markJobsReviewed(['job-1']) diff --git a/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte b/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte index a307fad255..ffa06ca5df 100644 --- a/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte +++ b/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte @@ -4,6 +4,7 @@ import { getAiChatManager } from './aiChatManagerContext' import { AIMode } from './AIChatManager.svelte' import Tooltip from '$lib/components/meltComponents/Tooltip.svelte' + import { formatTokenCount } from './tokenUsage' const aiChatManager = getAiChatManager() @@ -45,16 +46,6 @@ ? 'bg-amber-500' : 'bg-surface-accent-primary' ) - - function formatTokenCount(tokens: number): string { - if (tokens >= 1_000_000) { - return `${(tokens / 1_000_000).toFixed(1).replace(/\.0$/, '')}M` - } - if (tokens >= 1000) { - return `${Math.round(tokens / 1000)}k` - } - return `${tokens}` - } {#if visible} diff --git a/frontend/src/lib/components/copilot/chat/CostIndicator.svelte b/frontend/src/lib/components/copilot/chat/CostIndicator.svelte new file mode 100644 index 0000000000..9b14d3f4fc --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/CostIndicator.svelte @@ -0,0 +1,87 @@ + + +{#if visible} + +
+ {label} +
+ {#snippet text()} +
+

+ {aiChatManager.isSessionChat ? 'Session cost' : 'Chat cost'} +

+
+ {#each priced.rows as row (`${row.provider}:${row.model}`)} +
+

{row.model}

+

+ {formatTokenCount(row.tokens.input)} in + {#if row.tokens.cacheRead > 0 || row.tokens.cacheWrite > 0} + · {formatTokenCount(row.tokens.cacheRead + row.tokens.cacheWrite)} cached + {/if} + · {formatTokenCount(row.tokens.output)} out · {row.cost === undefined + ? 'no rate' + : formatUsd(row.cost)}{row.source === 'reported' ? ' billed' : ''} +

+
+ {/each} +
+ {#if priced.hasReported} +

"billed" is the amount the provider charged.

+ {/if} + {#if estimatedRows.length > 0} +

{estimateSource}

+ {/if} + {#if unpricedModels.length > 0} +

+ No price for {unpricedModels.map((r) => r.model).join(', ')}. Set one in workspace + settings, under AI. +

+ {/if} +
+ {/snippet} + +{/if} diff --git a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts index 24d7284b95..0c09b4c795 100644 --- a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts @@ -5,7 +5,7 @@ import { createLongHash } from '$lib/editorLangUtils' import { userScopedDb, type UserScopedDbMigrateDeps } from '$lib/userScopedDb' import { scopedKey } from '$lib/userScopedStorage' import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs' -import type { PersistedContextUsage } from './tokenUsage' +import type { ModelTokenUsageTotals, PersistedContextUsage } from './tokenUsage' import { IMAGE_OMITTED_PLACEHOLDER, type AttachedImage } from './imageUtils' import { randomUUID } from '$lib/utils/uuid' @@ -44,6 +44,10 @@ interface ChatSchema extends IDBSchema { // in-flight job's tray row and completion survive a reload. Absent on // chats predating this feature. Persisted out-of-band like modifiedItems. backgroundJobs?: ChatJob[] + // Tokens this chat spent, per `provider:model`, so its cost survives a + // reload. Absent on chats predating this feature and on chats that never + // ran a turn — both simply show no cost. + usageByModel?: ModelTokenUsageTotals } } // Image bytes, out-of-band from the chat record on purpose: the record is @@ -178,6 +182,7 @@ export default class HistoryManager { contextUsage?: PersistedContextUsage modifiedItems?: string[] backgroundJobs?: ChatJob[] + usageByModel?: ModelTokenUsageTotals } > = $state({}) @@ -475,6 +480,7 @@ export default class HistoryManager { messages: ChatCompletionMessageParam[], contextUsage?: number, modifiedItems?: string[], + usageByModel?: ModelTokenUsageTotals, backgroundJobs?: ChatJob[] ) { if (displayMessages.length > 0) { @@ -539,6 +545,13 @@ export default class HistoryManager { ? { backgroundJobs: $state.snapshot(this.savedChats[this.currentChatId].backgroundJobs) } + : {}), + // Same "don't erase on omit" guard again: a background-jobs save mid-turn + // must not drop the spend recorded by the turns before it. + ...(usageByModel !== undefined + ? { usageByModel } + : this.savedChats[this.currentChatId]?.usageByModel !== undefined + ? { usageByModel: $state.snapshot(this.savedChats[this.currentChatId].usageByModel) } : {}) } // The mirror mirrors what the DB holds (refs — the snapshot is @@ -581,9 +594,17 @@ export default class HistoryManager { messages: ChatCompletionMessageParam[], contextUsage?: number, modifiedItems?: string[], + usageByModel?: ModelTokenUsageTotals, backgroundJobs?: ChatJob[] ) { - await this.saveChat(displayMessages, messages, contextUsage, modifiedItems, backgroundJobs) + await this.saveChat( + displayMessages, + messages, + contextUsage, + modifiedItems, + usageByModel, + backgroundJobs + ) this.currentChatId = createLongHash() this.pruneImageIds(this.currentChatId) } diff --git a/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts b/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts index ffcc065157..a60adf4f3c 100644 --- a/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts @@ -712,7 +712,14 @@ describe('HistoryManager mirror convergence under concurrent metadata saves', () // mirror, or s3's backgroundJobs fallback below reads the stale record // and permanently erases the job. const p1 = hm.saveChat(display, [] as ChatCompletionMessageParam[], undefined, ['script:a']) - const p2 = hm.saveChat(display, [] as ChatCompletionMessageParam[], undefined, undefined, [job]) + const p2 = hm.saveChat( + display, + [] as ChatCompletionMessageParam[], + undefined, + undefined, + undefined, + [job] + ) await p1 const p3 = hm.saveChat(display, [] as ChatCompletionMessageParam[], undefined, [ 'script:a', diff --git a/frontend/src/lib/components/copilot/chat/chatLoop.test.ts b/frontend/src/lib/components/copilot/chat/chatLoop.test.ts index 2c4eb466e9..a010f1369a 100644 --- a/frontend/src/lib/components/copilot/chat/chatLoop.test.ts +++ b/frontend/src/lib/components/copilot/chat/chatLoop.test.ts @@ -518,7 +518,7 @@ describe('runChatLoop lastIterationUsage', () => { expect(result.lastIterationUsage).toEqual({ prompt: 1200, completion: 80, total: 1280 }) // the aggregate keeps summing across iterations - expect(result.tokenUsage).toEqual({ prompt: 2200, completion: 130, total: 2330 }) + expect(result.tokenUsage).toMatchObject({ prompt: 2200, completion: 130, total: 2330 }) }) it('ignores empty usage reports and returns null when none are real', async () => { diff --git a/frontend/src/lib/components/copilot/chat/chatLoop.ts b/frontend/src/lib/components/copilot/chat/chatLoop.ts index cc963e4b25..ceb9d421e7 100644 --- a/frontend/src/lib/components/copilot/chat/chatLoop.ts +++ b/frontend/src/lib/components/copilot/chat/chatLoop.ts @@ -21,7 +21,13 @@ import { } from './openai-responses' import type { Tool, ToolCallbacks } from './shared' import { sanitizeToolCallArguments } from './toolCallArguments' -import { addChatTokenUsage, emptyChatTokenUsage, type ChatTokenUsage } from './tokenUsage' +import { + addChatTokenUsage, + addModelTokenUsage, + emptyChatTokenUsage, + type ChatTokenUsage, + type ModelTokenUsageTotals +} from './tokenUsage' export interface ChatClients { openai: OpenAI @@ -83,6 +89,8 @@ export interface ChatLoopResult { addedMessages: ChatCompletionMessageParam[] /** Sum of usage across all loop iterations (suitable for cost accounting). */ tokenUsage: ChatTokenUsage + /** The same usage split per model, so a turn that switched model prices correctly. */ + tokenUsageByModel: ModelTokenUsageTotals lastIterationUsage: ChatTokenUsage | null hitMaxIterations: boolean } @@ -325,12 +333,25 @@ export async function runChatLoop(config: ChatLoopConfig): Promise { tokenUsage = addChatTokenUsage(tokenUsage, usage) + if (iterationModel) { + tokenUsageByModel = addModelTokenUsage( + tokenUsageByModel, + iterationModel.provider, + iterationModel.model, + usage + ) + } // Some providers/paths report no usage (prompt 0); keep the last real one. if (usage && usage.prompt > 0) { lastIterationUsage = usage @@ -351,6 +372,7 @@ export async function runChatLoop(config: ChatLoopConfig): Promise { + it('derives uncached input under the Anthropic convention', () => { + const usage = anthropicUsageToChatTokenUsage({ + input_tokens: 1000, + output_tokens: 200, + cache_creation_input_tokens: 300, + cache_read_input_tokens: 5000 + }) + expect(usage.prompt).toBe(6300) + expect(billedTokens(usage)).toEqual({ + input: 1000, + cacheRead: 5000, + cacheWrite: 300, + output: 200 + }) + }) + + it('derives uncached input under the OpenAI convention', () => { + const usage = openAICompletionsUsageToChatTokenUsage({ + prompt_tokens: 6000, + completion_tokens: 200, + prompt_tokens_details: { cached_tokens: 5000 } + }) + expect(usage.prompt).toBe(6000) + expect(billedTokens(usage)).toEqual({ + input: 1000, + cacheRead: 5000, + cacheWrite: 0, + output: 200 + }) + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/tokenUsage.ts b/frontend/src/lib/components/copilot/chat/tokenUsage.ts index 0091e11227..cb38d6adbd 100644 --- a/frontend/src/lib/components/copilot/chat/tokenUsage.ts +++ b/frontend/src/lib/components/copilot/chat/tokenUsage.ts @@ -1,7 +1,21 @@ +import type { AIProvider } from '$lib/gen' +import { modelKey } from '../modelConfig' +import type { PricedTokens } from '../modelPricing' + export interface ChatTokenUsage { prompt: number completion: number total: number + /** + * Subsets of `prompt`, split out because they are billed at different rates + * (a cached read is a fraction of an uncached one). `prompt` stays the whole + * input so the context gauge keeps measuring the whole request; uncached + * input is `prompt - cacheRead - cacheWrite`. + */ + cacheRead: number + cacheWrite: number + /** Cost in USD as billed, for the providers that report one. */ + cost?: number } /** @@ -28,7 +42,7 @@ export function normalizeContextUsage( } export function emptyChatTokenUsage(): ChatTokenUsage { - return { prompt: 0, completion: 0, total: 0 } + return { prompt: 0, completion: 0, total: 0, cacheRead: 0, cacheWrite: 0 } } export function addChatTokenUsage( @@ -39,10 +53,83 @@ export function addChatTokenUsage( return total } + const cost = + total.cost === undefined && usage.cost === undefined + ? undefined + : (total.cost ?? 0) + (usage.cost ?? 0) + return { prompt: total.prompt + usage.prompt, completion: total.completion + usage.completion, - total: total.total + usage.total + total: total.total + usage.total, + // `?? 0`: the cache split is newer than the field it lives on, so a usage + // object read back from storage may predate it. + cacheRead: (total.cacheRead ?? 0) + (usage.cacheRead ?? 0), + cacheWrite: (total.cacheWrite ?? 0) + (usage.cacheWrite ?? 0), + ...(cost === undefined ? {} : { cost }) + } +} + +/** Compact token count for chips and tooltips (`1.2M`, `34k`, `567`). */ +export function formatTokenCount(tokens: number): string { + if (tokens >= 1_000_000) { + return `${(tokens / 1_000_000).toFixed(1).replace(/\.0$/, '')}M` + } + if (tokens >= 1000) { + return `${Math.round(tokens / 1000)}k` + } + return `${tokens}` +} + +/** + * A chat's spend on one model. Usage is bucketed per model rather than summed + * because each model bills at its own rate, and a chat can switch model between + * turns (or mid-turn, via the model selector). + */ +export type ModelTokenUsage = { + provider: AIProvider + model: string + usage: ChatTokenUsage +} + +export type ModelTokenUsageTotals = Record + +/** Fold one report into the per-model totals, keyed `provider:model`. */ +export function addModelTokenUsage( + totals: ModelTokenUsageTotals, + provider: AIProvider, + model: string, + usage: ChatTokenUsage | null | undefined +): ModelTokenUsageTotals { + if (!usage) { + return totals + } + const key = modelKey(provider, model) + const existing = totals[key] + return { + ...totals, + [key]: { + provider, + model, + usage: addChatTokenUsage(existing?.usage ?? emptyChatTokenUsage(), usage) + } + } +} + +/** + * Split a usage report into the four separately-billed token classes. `prompt` + * counts the whole input, so the uncached share is whatever the cached classes + * do not account for — which holds for both provider conventions below + * (Anthropic adds its cache counts into `prompt`, OpenAI's already includes them). + */ +export function billedTokens(usage: ChatTokenUsage): PricedTokens { + const cacheRead = usage.cacheRead ?? 0 + const cacheWrite = usage.cacheWrite ?? 0 + return { + input: Math.max(0, usage.prompt - cacheRead - cacheWrite), + cacheRead, + cacheWrite, + output: usage.completion } } @@ -57,16 +144,17 @@ export function anthropicUsageToChatTokenUsage( | null | undefined ): ChatTokenUsage { - const prompt = - (usage?.input_tokens ?? 0) + - (usage?.cache_creation_input_tokens ?? 0) + - (usage?.cache_read_input_tokens ?? 0) + const cacheWrite = usage?.cache_creation_input_tokens ?? 0 + const cacheRead = usage?.cache_read_input_tokens ?? 0 + const prompt = (usage?.input_tokens ?? 0) + cacheWrite + cacheRead const completion = usage?.output_tokens ?? 0 return { prompt, completion, - total: prompt + completion + total: prompt + completion, + cacheRead, + cacheWrite } } @@ -89,7 +177,11 @@ export function openAIResponsesUsageToChatTokenUsage( return { prompt, completion, - total: usage?.total_tokens ?? prompt + completion + total: usage?.total_tokens ?? prompt + completion, + cacheRead: usage?.input_tokens_details?.cached_tokens ?? 0, + // Automatic caching: nothing is billed for populating it, and no usage + // field reports it either. + cacheWrite: 0 } } @@ -102,6 +194,8 @@ export function openAICompletionsUsageToChatTokenUsage( completion_tokens?: number | null total_tokens?: number | null prompt_tokens_details?: { cached_tokens?: number | null } | null + /** OpenRouter reports what it actually charged when the request opts in. */ + cost?: number | null } | null | undefined @@ -112,6 +206,9 @@ export function openAICompletionsUsageToChatTokenUsage( return { prompt, completion, - total: usage?.total_tokens ?? prompt + completion + total: usage?.total_tokens ?? prompt + completion, + cacheRead: usage?.prompt_tokens_details?.cached_tokens ?? 0, + cacheWrite: 0, + ...(typeof usage?.cost === 'number' ? { cost: usage.cost } : {}) } } diff --git a/frontend/src/lib/components/copilot/lib.ts b/frontend/src/lib/components/copilot/lib.ts index fc2cb0abd2..b12ba7cbed 100644 --- a/frontend/src/lib/components/copilot/lib.ts +++ b/frontend/src/lib/components/copilot/lib.ts @@ -1056,6 +1056,23 @@ export async function getFimCompletion( } } +// A streamed OpenAI-compatible response carries no usage at all unless the request +// asks for it, so a provider missing from this set reports zero tokens — no context +// gauge, no cost. `stream_options.include_usage` is part of the OpenAI streaming +// spec and these providers document supporting it; `customai` is deliberately absent +// because it points at an arbitrary endpoint that may reject the field outright. +const STREAM_USAGE_PROVIDERS = new Set([ + 'openai', + 'azure_openai', + 'azure_foundry', + 'googleai', + 'openrouter', + 'groq', + 'deepseek', + 'mistral', + 'togetherai' +]) + export async function getCompletion( messages: ChatCompletionMessageParam[], abortController: AbortController, @@ -1099,17 +1116,17 @@ export async function getCompletion( // Use Completions API for other providers const client = options?.openaiClient ?? workspaceAIClients.getOpenaiClient() const completionConfig = applyReasoningToConfig( - (provider === 'openai' || - provider === 'azure_openai' || - provider === 'azure_foundry' || - provider === 'googleai') && - config.stream + config.stream && STREAM_USAGE_PROVIDERS.has(provider) ? { ...config, stream_options: { ...(config.stream_options ?? {}), include_usage: true - } + }, + // OpenRouter's own extension, on top of stream_options: it returns the + // credits actually charged next to the token counts, which is the one + // route by which the chat sees a real cost rather than an estimate. + ...(provider === 'openrouter' ? { usage: { include: true } } : {}) } : config, provider === 'deepseek' ? 'deepseek' : provider === 'mistral' ? 'mistral' : 'completions', diff --git a/frontend/src/lib/components/copilot/modelConfig.ts b/frontend/src/lib/components/copilot/modelConfig.ts index ee034f0500..f38708da20 100644 --- a/frontend/src/lib/components/copilot/modelConfig.ts +++ b/frontend/src/lib/components/copilot/modelConfig.ts @@ -116,21 +116,44 @@ function normalizeVersionSeparators(model: string): string { return model.replace(/\./g, '-') } -// An entry that ends on a version digit must not run into a longer version: -// `gpt-4.1` collapses to `gpt-4-1`, which would otherwise claim the 128K -// `gpt-4-1106-preview` as a 1M model. Suffixes that continue with a separator -// (`claude-opus-4-8` in `...-4-8-v1`, `gpt-5` in `gpt-5-mini`) still match. -// Family fallbacks ending on a letter get no such guard — a version welded -// straight onto the name (`llama3.1`) is exactly what they exist to catch. -const MODEL_CONTEXT_WINDOW_MATCHERS: [matcher: RegExp, contextWindow: number][] = - MODEL_CONTEXT_WINDOWS.map(([name, contextWindow]) => { +/** + * Compile a most-specific-first `[name, value]` table into matchers against the + * bare model id. Shared with the pricing table so both resolve the same set of + * ids — a model whose window is known but whose price is not (or vice versa) + * should be a gap in one table, never a difference in matching. + * + * An entry that ends on a version digit must not run into a longer version: + * `gpt-4.1` collapses to `gpt-4-1`, which would otherwise claim + * `gpt-4-1106-preview`. Suffixes that continue with a separator + * (`claude-opus-4-8` in `...-4-8-v1`, `gpt-5` in `gpt-5-mini`) still match. + * Family fallbacks ending on a letter get no such guard — a version welded + * straight onto the name (`llama3.1`) is exactly what they exist to catch. + */ +export function buildModelMatchers(entries: [name: string, value: T][]): [RegExp, T][] { + return entries.map(([name, value]) => { const pattern = normalizeVersionSeparators(name).replace(/[.*+?^${}()|[\]\\]/g, '\\$&') - return [new RegExp(/\d$/.test(pattern) ? `${pattern}(?!\\d)` : pattern), contextWindow] + return [new RegExp(/\d$/.test(pattern) ? `${pattern}(?!\\d)` : pattern), value] }) +} + +/** + * The `provider:model` key the workspace AI settings use for their per-model maps + * (`max_tokens_per_model`, `model_pricing`). A bare model id is not enough: the + * same id can be served by more than one provider at different rates. + */ +export function modelKey(provider: AIProvider | string, model: string): string { + return `${provider}:${model}` +} + +export function matchModel(matchers: [RegExp, T][], model: string): T | undefined { + const id = normalizeVersionSeparators(parseModelId(model).base) + return matchers.find(([matcher]) => matcher.test(id))?.[1] +} + +const MODEL_CONTEXT_WINDOW_MATCHERS = buildModelMatchers(MODEL_CONTEXT_WINDOWS) export function getKnownModelContextWindow(model: string): number | undefined { - const id = normalizeVersionSeparators(parseModelId(model).base) - return MODEL_CONTEXT_WINDOW_MATCHERS.find(([matcher]) => matcher.test(id))?.[1] + return matchModel(MODEL_CONTEXT_WINDOW_MATCHERS, model) } export function getModelContextWindow(model: string) { diff --git a/frontend/src/lib/components/copilot/modelPricing.test.ts b/frontend/src/lib/components/copilot/modelPricing.test.ts new file mode 100644 index 0000000000..8bd3bf11b8 --- /dev/null +++ b/frontend/src/lib/components/copilot/modelPricing.test.ts @@ -0,0 +1,97 @@ +import { describe, expect, it } from 'vitest' +import { billedTokens } from './chat/tokenUsage' +import { estimateCost, priceSpend, resolveModelPrice } from './modelPricing' + +describe('resolveModelPrice', () => { + it('resolves the same model across the routes that decorate its id', () => { + const direct = resolveModelPrice('anthropic', 'claude-opus-5', undefined) + expect(direct?.price.input).toBe(5) + // A gateway prefix, a dot-versioned id, a date suffix and a variant suffix + // must all land on the same entry — a miss here silently under-reports cost. + for (const id of [ + 'anthropic/claude-opus-5', + 'anthropic/claude-opus-4.8', + 'claude-opus-4-8-20260101', + 'anthropic/claude-opus-5:thinking' + ]) { + expect(resolveModelPrice('openrouter', id, undefined)?.price.input).toBe(5) + } + }) + + it('does not let a version-digit entry claim a longer version', () => { + expect(resolveModelPrice('openai', 'gpt-4.1', undefined)?.price.input).toBe(2) + expect(resolveModelPrice('openai', 'gpt-4-1106-preview', undefined)?.price.input).not.toBe(2) + }) + + it('reports an unknown model as unpriced rather than guessing', () => { + expect(resolveModelPrice('customai', 'some-in-house-model', undefined)).toBeUndefined() + }) + + it('prefers a workspace override, defaulting its cache rates off its own input rate', () => { + const resolved = resolveModelPrice('anthropic', 'claude-opus-5', { + 'anthropic:claude-opus-5': { input: 2, output: 8 } + }) + expect(resolved?.source).toBe('override') + expect(resolved?.price.input).toBe(2) + expect(resolved?.price.cacheRead).toBeCloseTo(0.2) + expect(resolved?.price.cacheWrite).toBeCloseTo(2.5) + }) +}) + +describe('estimateCost', () => { + it('bills each token class at its own rate', () => { + const cost = estimateCost( + { input: 1_000_000, cacheRead: 1_000_000, cacheWrite: 1_000_000, output: 1_000_000 }, + { input: 5, output: 25, cacheRead: 0.5, cacheWrite: 6.25 } + ) + expect(cost).toBeCloseTo(5 + 0.5 + 6.25 + 25) + }) + + it('charges a cached prefix less than an uncached one', () => { + const usage = { + prompt: 100_000, + completion: 0, + total: 100_000, + cacheRead: 90_000, + cacheWrite: 0 + } + const uncached = { ...usage, cacheRead: 0 } + const price = { input: 5, output: 25, cacheRead: 0.5, cacheWrite: 6.25 } + expect(estimateCost(billedTokens(usage), price)).toBeLessThan( + estimateCost(billedTokens(uncached), price) + ) + }) +}) + +describe('priceSpend', () => { + it('prefers a provider-reported cost over the estimate', () => { + const priced = priceSpend( + [ + { + provider: 'openrouter', + model: 'anthropic/claude-opus-5', + tokens: { input: 1_000_000, cacheRead: 0, cacheWrite: 0, output: 0 }, + reportedCostUsd: 0.42 + } + ], + undefined + ) + expect(priced.total).toBe(0.42) + expect(priced.hasReported).toBe(true) + }) + + it('flags an unpriced model instead of counting it as free', () => { + const priced = priceSpend( + [ + { + provider: 'customai', + model: 'some-in-house-model', + tokens: { input: 1_000_000, cacheRead: 0, cacheWrite: 0, output: 0 } + } + ], + undefined + ) + expect(priced.hasUnpriced).toBe(true) + expect(priced.rows[0].cost).toBeUndefined() + }) +}) diff --git a/frontend/src/lib/components/copilot/modelPricing.ts b/frontend/src/lib/components/copilot/modelPricing.ts new file mode 100644 index 0000000000..8424047887 --- /dev/null +++ b/frontend/src/lib/components/copilot/modelPricing.ts @@ -0,0 +1,217 @@ +import type { AIProvider, ModelPriceOverride } from '$lib/gen' +import { buildModelMatchers, matchModel, modelKey } from './modelConfig' + +/** Rates in USD per million tokens, one per billed token class. */ +export type ModelPrice = { + input: number + output: number + cacheRead: number + cacheWrite: number +} + +export type ModelPriceSource = 'override' | 'builtin' + +export type ResolvedModelPrice = { + price: ModelPrice + source: ModelPriceSource +} + +/** What a chat spent on one model, in tokens. */ +export type PricedTokens = { + input: number + cacheRead: number + cacheWrite: number + output: number +} + +// Rates that fall out of the input rate unless a provider prices them separately. +// Anthropic reads a cached prefix at a tenth of the input rate and writes one at +// 1.25x (5-minute TTL, the default the chat uses). Providers whose caching is +// automatic never report a cache write, so their write rate is unused. +const CACHE_READ_RATIO = 0.1 +const CACHE_WRITE_RATIO = 1.25 + +type PriceEntry = { + input: number + output: number + cacheRead?: number + cacheWrite?: number +} + +/** + * Published list prices, most specific entry first — the first name found in the + * bare model id wins, so vendor-namespaced and date-suffixed ids + * (anthropic/claude-opus-5, gpt-5-2026-01-01) still resolve. Matching is shared + * with the context-window table via `buildModelMatchers`. + * + * This is a best-effort snapshot: vendors change rates, ship models faster than + * this table is updated, and negotiated rates differ from list. A model that is + * not listed resolves to undefined and is reported as unpriced rather than + * guessed at, and any entry can be corrected per workspace from the AI settings. + * Providers whose catalogue turns over too quickly to track (DeepSeek, Mistral, + * Groq, TogetherAI, custom deployments) are deliberately absent. + */ +const MODEL_PRICES: [name: string, price: PriceEntry][] = [ + // Anthropic — Opus 4.1 and older bill at the pre-4.5 Opus rate, so the family + // fallback sits below the explicit entries rather than covering them. + ['claude-fable-5', { input: 10, output: 50 }], + ['claude-mythos-5', { input: 10, output: 50 }], + ['claude-opus-5', { input: 5, output: 25 }], + ['claude-opus-4-8', { input: 5, output: 25 }], + ['claude-opus-4-7', { input: 5, output: 25 }], + ['claude-opus-4-6', { input: 5, output: 25 }], + ['claude-opus-4-5', { input: 5, output: 25 }], + ['claude-opus-4-1', { input: 15, output: 75 }], + ['claude-opus-4', { input: 15, output: 75 }], + ['claude-sonnet-5', { input: 3, output: 15 }], + ['claude-sonnet-4-6', { input: 3, output: 15 }], + ['claude-sonnet-4-5', { input: 3, output: 15 }], + ['claude-sonnet-4', { input: 3, output: 15 }], + ['claude-haiku-4-5', { input: 1, output: 5 }], + ['claude-3-5-haiku', { input: 0.8, output: 4 }], + ['claude-opus', { input: 5, output: 25 }], + ['claude-sonnet', { input: 3, output: 15 }], + ['claude-haiku', { input: 1, output: 5 }], + // OpenAI — cached input is a tenth of input, and there is no separate charge + // for writing the cache, so the write rate never applies (the OpenAI usage + // parsers report no cache-write tokens). The -mini/-nano entries must precede + // the family entry, which would otherwise claim them. + ['gpt-5-mini', { input: 0.25, output: 2 }], + ['gpt-5-nano', { input: 0.05, output: 0.4 }], + ['gpt-5', { input: 1.25, output: 10 }], + ['gpt-4.1-mini', { input: 0.4, output: 1.6 }], + ['gpt-4.1-nano', { input: 0.1, output: 0.4 }], + ['gpt-4.1', { input: 2, output: 8 }], + ['gpt-4o-mini', { input: 0.15, output: 0.6 }], + ['gpt-4o', { input: 2.5, output: 10 }], + ['o4-mini', { input: 1.1, output: 4.4 }], + ['o3-mini', { input: 1.1, output: 4.4 }], + ['o3', { input: 2, output: 8 }], + // Google + ['gemini-2.5-flash-lite', { input: 0.1, output: 0.4 }], + ['gemini-2.5-flash', { input: 0.3, output: 2.5 }], + ['gemini-2.5-pro', { input: 1.25, output: 10 }] +] + +const MODEL_PRICE_MATCHERS = buildModelMatchers( + MODEL_PRICES.map(([name, entry]): [string, ModelPrice] => [ + name, + { + input: entry.input, + output: entry.output, + cacheRead: entry.cacheRead ?? entry.input * CACHE_READ_RATIO, + cacheWrite: entry.cacheWrite ?? entry.input * CACHE_WRITE_RATIO + } + ]) +) + +export function getKnownModelPrice(model: string): ModelPrice | undefined { + return matchModel(MODEL_PRICE_MATCHERS, model) +} + +/** + * The rate a workspace should be billed at for one model: its override when an + * admin set one, otherwise the published list price, otherwise nothing. An + * override that omits the cache rates keeps the usual multiples of its own input + * rate, so an admin who only knows their input/output pricing does not have to + * invent the other two. + */ +export function resolveModelPrice( + provider: AIProvider | string, + model: string, + overrides: Record | undefined +): ResolvedModelPrice | undefined { + const override = overrides?.[modelKey(provider, model)] + if (override) { + return { + source: 'override', + price: { + input: override.input, + output: override.output, + cacheRead: override.cache_read ?? override.input * CACHE_READ_RATIO, + cacheWrite: override.cache_write ?? override.input * CACHE_WRITE_RATIO + } + } + } + const builtin = getKnownModelPrice(model) + return builtin ? { source: 'builtin', price: builtin } : undefined +} + +/** Cost in USD of `tokens` at `price`. */ +export function estimateCost(tokens: PricedTokens, price: ModelPrice): number { + return ( + (tokens.input * price.input + + tokens.cacheRead * price.cacheRead + + tokens.cacheWrite * price.cacheWrite + + tokens.output * price.output) / + 1_000_000 + ) +} + +/** Tokens spent on one model, from a chat's running totals or the usage API. */ +export type ModelSpend = { + provider: string + model: string + tokens: PricedTokens + /** What the provider billed, where it reports a figure. */ + reportedCostUsd?: number +} + +export type Priced = { + /** Undefined when no rate is known for the model — reported as unpriced, never guessed. */ + cost: number | undefined + source: ModelPriceSource | 'reported' | undefined +} + +export type PricedSpend = { + /** The input entries, each with its cost — callers carry their own fields through + * rather than zipping the result back against the input by index. */ + rows: (T & Priced)[] + total: number + /** True when at least one row has no rate, so `total` understates the truth. */ + hasUnpriced: boolean + /** True when at least one row is a figure the provider billed rather than an estimate. */ + hasReported: boolean +} + +/** + * Cost a set of per-model token counts. A provider-reported figure always wins: + * it is what was actually charged, where everything else is list price times + * tokens. Shared by the chat's cost chip and the workspace usage view so both + * apply the same rates and the same estimated/reported labelling. + */ +export function priceSpend( + spend: T[], + overrides: Record | undefined +): PricedSpend { + let total = 0 + let hasUnpriced = false + let hasReported = false + const rows = spend.map((entry): T & Priced => { + if (entry.reportedCostUsd !== undefined) { + hasReported = true + total += entry.reportedCostUsd + return { ...entry, cost: entry.reportedCostUsd, source: 'reported' } + } + const resolved = resolveModelPrice(entry.provider, entry.model, overrides) + if (!resolved) { + hasUnpriced = true + return { ...entry, cost: undefined, source: undefined } + } + const cost = estimateCost(entry.tokens, resolved.price) + total += cost + return { ...entry, cost, source: resolved.source } + }) + return { rows, total, hasUnpriced, hasReported } +} + +/** + * Money, at the precision the amount deserves: sub-cent spend is where a chat + * spends most of its life, and rounding it to `$0.00` would read as free. + */ +export function formatUsd(amount: number): string { + if (amount === 0) return '$0' + if (amount < 0.01) return `$${amount.toFixed(4)}` + if (amount < 1) return `$${amount.toFixed(3)}` + return `$${amount.toFixed(2)}` +} diff --git a/frontend/src/lib/components/workspaceSettings/AISettings.svelte b/frontend/src/lib/components/workspaceSettings/AISettings.svelte index 60d1cb77a0..e5ed2c9f09 100644 --- a/frontend/src/lib/components/workspaceSettings/AISettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/AISettings.svelte @@ -5,7 +5,8 @@ type AIConfig, type AIProvider, type GetCopilotSettingsStateResponse, - type InstanceAISummary + type InstanceAISummary, + type ModelPriceOverride } from '$lib/gen' import { workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' @@ -25,6 +26,8 @@ import Badge from '../common/badge/Badge.svelte' import Tooltip from '../Tooltip.svelte' import ModelTokenLimits from './ModelTokenLimits.svelte' + import ModelPricing from './ModelPricing.svelte' + import AiUsagePanel from './AiUsagePanel.svelte' import { setCopilotInfo } from '$lib/aiStore' import AIPromptsModal from '../settings/AIPromptsModal.svelte' import { Settings } from 'lucide-svelte' @@ -73,6 +76,7 @@ let metadataModel: string | undefined = $state(undefined) let customPrompts: Record = $state({}) let maxTokensPerModel: Record = $state({}) + let modelPricing: Record = $state({}) let usingOpenaiClientCredentialsOauth = $state(false) let workspaceOverrideEditorOpened = $state(false) @@ -83,6 +87,7 @@ let initialMetadataModel: string | undefined = $state(undefined) let initialCustomPrompts: Record = $state({}) let initialMaxTokensPerModel: Record = $state({}) + let initialModelPricing: Record = $state({}) let initialPrompts: Record = $state({}) let lastLoadedConfigKey = $state(undefined) @@ -110,6 +115,7 @@ codeCompletionModel = config?.code_completion_model?.model customPrompts = clone(config?.custom_prompts ?? {}) maxTokensPerModel = clone(config?.max_tokens_per_model ?? {}) + modelPricing = clone(config?.model_pricing ?? {}) for (const mode of ['edit', 'fix', 'gen']) { if (!(mode in customPrompts)) { customPrompts[mode] = '' @@ -124,6 +130,7 @@ initialCodeCompletionModel = codeCompletionModel initialCustomPrompts = clone(customPrompts) initialMaxTokensPerModel = clone(maxTokensPerModel) + initialModelPricing = clone(modelPricing) initialPrompts = clone(customPrompts) } @@ -139,6 +146,7 @@ codeCompletionModel = initialCodeCompletionModel customPrompts = clone(initialCustomPrompts) maxTokensPerModel = clone(initialMaxTokensPerModel) + modelPricing = clone(initialModelPricing) } $effect(() => { @@ -172,7 +180,8 @@ metadataModel !== initialMetadataModel || codeCompletionModel !== initialCodeCompletionModel || JSON.stringify(customPrompts) !== JSON.stringify(initialCustomPrompts) || - JSON.stringify(maxTokensPerModel) !== JSON.stringify(initialMaxTokensPerModel) + JSON.stringify(maxTokensPerModel) !== JSON.stringify(initialMaxTokensPerModel) || + JSON.stringify(modelPricing) !== JSON.stringify(initialModelPricing) ) $effect(() => { @@ -285,7 +294,8 @@ metadata_model, custom_prompts: Object.keys(custom_prompts).length > 0 ? custom_prompts : undefined, max_tokens_per_model: - Object.keys(maxTokensPerModel).length > 0 ? maxTokensPerModel : undefined + Object.keys(maxTokensPerModel).length > 0 ? maxTokensPerModel : undefined, + model_pricing: Object.keys(modelPricing).length > 0 ? modelPricing : undefined } : {} } @@ -576,6 +586,10 @@ + + + +
+
+ {/if} + {#if errors[key]} +
{errors[key]}
+ {/if} + + {/each} + + + {/if} + + {/each} + + +{/if} diff --git a/frontend/src/lib/utils/aiUsageReporter.ts b/frontend/src/lib/utils/aiUsageReporter.ts new file mode 100644 index 0000000000..a57103da87 Binary files /dev/null and b/frontend/src/lib/utils/aiUsageReporter.ts differ