feat: track token cost in AI sessions and chats (#10688)

* feat: track token cost in AI sessions and chats

* fix: address review findings on AI cost tracking

* fix: price inherited and overridden models at their real rates

* fix: stop newer model revisions inheriting an older price

* fix: stop a sub-model inheriting its family's price

* fix: keep alias suffixes resolving to their model's price

* fix: count OpenRouter cache writes and drop unverifiable rates

* refactor: move AI spend out of the chat into workspace and user settings

* fix: pin the usage workspace per turn and stop inventing cache rates

* fix: leave Sonnet 5 unpriced while its promotional rate runs

* docs: record the new table in the schema summary and tighten comments

* fix: mark estimated AI costs with ~ and drop session grouping

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix: name the workspace in the self-scoped AI usage title

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* docs: state that overrides never replace a provider-returned cost

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix: let a cleared cache rate inherit again and flag partial totals

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix: clear a refused rate's error when the input snaps back

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix: stop a revision variant inheriting its base family's rate

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix: report AI usage before tools run and price self usage consistently

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix: key pricing rows on the model id usage is reported under

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix: surface Bedrock and Gemini usage the chat proxy was dropping

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* fix: count Gemini tool-use prompt tokens as input

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* feat: price flat-rate Gemini Flash models

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

* docs: state the tool-use token invariant once

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
hugocasa
2026-08-24 11:08:13 +02:00
committed by GitHub
parent 7751d3e43e
commit b6e059116a
30 changed files with 2358 additions and 59 deletions
@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO ai_token_usage (workspace_id, email, provider, model, session_id, input_tokens, cache_read_tokens, cache_write_tokens, output_tokens, reported_cost_nano_usd, requests)\n SELECT $1, $2, * FROM UNNEST($3::text[], $4::text[], $5::text[], $6::bigint[], $7::bigint[], $8::bigint[], $9::bigint[], $10::bigint[], $11::bigint[])\n ON CONFLICT (workspace_id, day, email, provider, model, session_id)\n DO UPDATE SET\n input_tokens = ai_token_usage.input_tokens + EXCLUDED.input_tokens,\n cache_read_tokens = ai_token_usage.cache_read_tokens + EXCLUDED.cache_read_tokens,\n cache_write_tokens = ai_token_usage.cache_write_tokens + EXCLUDED.cache_write_tokens,\n output_tokens = ai_token_usage.output_tokens + EXCLUDED.output_tokens,\n reported_cost_nano_usd = CASE\n WHEN EXCLUDED.reported_cost_nano_usd IS NULL\n THEN ai_token_usage.reported_cost_nano_usd\n ELSE COALESCE(ai_token_usage.reported_cost_nano_usd, 0)\n + EXCLUDED.reported_cost_nano_usd\n END,\n requests = ai_token_usage.requests + EXCLUDED.requests,\n updated_at = now()",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"TextArray",
"TextArray",
"TextArray",
"Int8Array",
"Int8Array",
"Int8Array",
"Int8Array",
"Int8Array",
"Int8Array"
]
},
"nullable": []
},
"hash": "24fcc2b69f30953915f0cbf246e1c19b2310075a644d81c21784c991e52b4001"
}
@@ -0,0 +1,74 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n (CASE $3::text\n WHEN 'day' THEN day::text\n WHEN 'user' THEN email\n ELSE ''\n END) AS \"key!\",\n provider AS \"provider!\",\n model AS \"model!\",\n SUM(input_tokens)::bigint AS \"input_tokens!\",\n SUM(cache_read_tokens)::bigint AS \"cache_read_tokens!\",\n SUM(cache_write_tokens)::bigint AS \"cache_write_tokens!\",\n SUM(output_tokens)::bigint AS \"output_tokens!\",\n SUM(reported_cost_nano_usd)::bigint AS \"reported_cost_nano_usd\",\n SUM(requests)::bigint AS \"requests!\"\n FROM ai_token_usage\n WHERE workspace_id = $1 AND day > CURRENT_DATE - $2::int\n AND ($5::text IS NULL OR email = $5)\n GROUP BY 1, provider, model\n ORDER BY SUM(input_tokens + cache_read_tokens + cache_write_tokens + output_tokens) DESC\n LIMIT $4",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "key!",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "provider!",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "model!",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "input_tokens!",
"type_info": "Int8"
},
{
"ordinal": 4,
"name": "cache_read_tokens!",
"type_info": "Int8"
},
{
"ordinal": 5,
"name": "cache_write_tokens!",
"type_info": "Int8"
},
{
"ordinal": 6,
"name": "output_tokens!",
"type_info": "Int8"
},
{
"ordinal": 7,
"name": "reported_cost_nano_usd",
"type_info": "Int8"
},
{
"ordinal": 8,
"name": "requests!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"Int4",
"Text",
"Int8",
"Text"
]
},
"nullable": [
null,
false,
false,
null,
null,
null,
null,
null,
null
]
},
"hash": "6a17a1dfeb75808e5d0726f1f8bd573168161abe350b047402cda4ac4a4d13e4"
}
@@ -0,0 +1 @@
DROP TABLE ai_token_usage;
@@ -0,0 +1,42 @@
-- Per-workspace AI token spend, accumulated from the chat client. Rows hold token
-- counts rather than money: prices live in the frontend price table plus the
-- workspace's `ai_config.model_pricing` overrides and are applied at read time, so
-- correcting a price also corrects the history. `reported_cost_nano_usd` is the
-- exception — a few providers (OpenRouter) return what they actually charged, and
-- that figure wins over the estimate.
--
-- Distinct from `feature_usage`, which is anonymous telemetry that leaves the
-- instance and is pruned after 60 days; spend is per-user and kept.
CREATE TABLE ai_token_usage (
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE,
day DATE NOT NULL DEFAULT CURRENT_DATE,
email VARCHAR(255) NOT NULL,
provider VARCHAR(50) NOT NULL,
model VARCHAR(255) NOT NULL,
-- Empty for chats that are not attached to an AI session.
session_id VARCHAR(50) NOT NULL DEFAULT '',
-- Uncached input only; the two cache columns hold the rest of the prompt, so
-- each column maps to exactly one price and they never double-count.
input_tokens BIGINT NOT NULL DEFAULT 0,
cache_read_tokens BIGINT NOT NULL DEFAULT 0,
cache_write_tokens BIGINT NOT NULL DEFAULT 0,
output_tokens BIGINT NOT NULL DEFAULT 0,
reported_cost_nano_usd BIGINT,
requests BIGINT NOT NULL DEFAULT 0,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (workspace_id, day, email, provider, model, session_id)
);
-- The usage listing filters on workspace and a date range; the PK only reaches
-- `day` through `email`, so it cannot serve that on its own.
CREATE INDEX idx_ai_token_usage_ws_day ON ai_token_usage (workspace_id, day DESC);
GRANT ALL ON ai_token_usage TO windmill_admin;
GRANT ALL ON ai_token_usage TO windmill_user;
-- Both handlers go through the raw pool, so no policy is needed for them to work.
-- Enabling RLS with an admin-only policy is the backstop: a future query that
-- reaches this table through UserDB sees nothing rather than every user's spend.
ALTER TABLE ai_token_usage ENABLE ROW LEVEL SECURITY;
CREATE POLICY admin_policy ON ai_token_usage FOR ALL TO windmill_admin USING (true);
+2
View File
@@ -38,6 +38,8 @@ account: workspace_id(char), id(int), expires_at(ts), refresh_token(char), clien
FK: (workspace_id) -> workspace(id)
agent_token_blacklist: token(char), expires_at(ts), blacklisted_at(ts), blacklisted_by(char)
ai_agent_memory: workspace_id(char), conversation_id(uuid), step_id(char), messages(jsonb), created_at(ts), updated_at(ts)
ai_token_usage: workspace_id(char), day(date), email(char), provider(char), model(char), session_id(char), input_tokens(bigint), cache_read_tokens(bigint), cache_write_tokens(bigint), output_tokens(bigint), reported_cost_nano_usd(bigint), requests(bigint), updated_at(ts)
FK: (workspace_id) -> workspace(id)
alerts: id(int), alert_type(char), message(text), created_at(ts), acknowledged(bool), workspace_id(text), acknowledged_workspace(bool), resource(text)
app: id(bigint), workspace_id(char), path(char), summary(char), policy(jsonb), versions(bigint[]), extra_perms(jsonb), draft_only(bool), custom_path(text), labels(text[])
FK: (workspace_id) -> workspace(id)
+119 -5
View File
@@ -277,7 +277,7 @@ pub struct GeminiSSECandidate {
}
/// Token usage from the `usageMetadata` field of a Gemini SSE event.
#[derive(Deserialize, Debug, Clone)]
#[derive(Deserialize, Debug, Clone, Default)]
pub struct GeminiUsageMetadata {
#[serde(rename = "promptTokenCount", default)]
pub prompt_token_count: Option<i32>,
@@ -285,6 +285,39 @@ pub struct GeminiUsageMetadata {
pub candidates_token_count: Option<i32>,
#[serde(rename = "totalTokenCount", default)]
pub total_token_count: Option<i32>,
/// Subset of `promptTokenCount` served from context cache, billed at a reduced
/// rate. Reported separately so the client can price it separately.
#[serde(rename = "cachedContentTokenCount", default)]
pub cached_content_token_count: Option<i32>,
/// Thinking tokens, billed as output but counted apart from `candidatesTokenCount`.
#[serde(rename = "thoughtsTokenCount", default)]
pub thoughts_token_count: Option<i32>,
/// Input tokens spent on tool-use prompts, counted apart from `promptTokenCount`
/// rather than within it.
#[serde(rename = "toolUsePromptTokenCount", default)]
pub tool_use_prompt_token_count: Option<i32>,
}
/// Input tokens as billed. Gemini reports tool-use prompts in their own field, and
/// they are disjoint from `promptTokenCount`: a live tool call returns 17 prompt +
/// 60 tool-use + 17 candidates + 52 thoughts against a `totalTokenCount` of 146, so
/// leaving them out under-reports the input of every tool-using turn. Cached tokens
/// are not added here, being already part of `promptTokenCount`.
fn gemini_prompt_tokens(usage: &GeminiUsageMetadata) -> i32 {
usage
.prompt_token_count
.unwrap_or(0)
.saturating_add(usage.tool_use_prompt_token_count.unwrap_or(0))
}
/// Output tokens as billed: Gemini counts thinking apart from `candidatesTokenCount`
/// but charges it at the output rate, so a reply that thought would otherwise be
/// reported as far cheaper than it was.
fn gemini_completion_tokens(usage: &GeminiUsageMetadata) -> i32 {
usage
.candidates_token_count
.unwrap_or(0)
.saturating_add(usage.thoughts_token_count.unwrap_or(0))
}
/// Top-level structure of one Gemini SSE event.
@@ -588,9 +621,12 @@ pub fn gemini_response_to_openai(parsed: &GeminiParsedEvent, model: &str) -> ser
let usage = parsed.usage.as_ref().map(|u| {
serde_json::json!({
"prompt_tokens": u.prompt_token_count.unwrap_or(0),
"completion_tokens": u.candidates_token_count.unwrap_or(0),
"prompt_tokens": gemini_prompt_tokens(u),
"completion_tokens": gemini_completion_tokens(u),
"total_tokens": u.total_token_count.unwrap_or(0),
"prompt_tokens_details": {
"cached_tokens": u.cached_content_token_count.unwrap_or(0)
},
})
});
@@ -680,8 +716,8 @@ pub fn gemini_event_to_openai_sse_chunks(
// OpenAI's `stream_options.include_usage` terminal chunk (top-level `usage`,
// empty `choices`) so the frontend's `'usage' in chunk` path records them.
if let Some(usage) = &parsed.usage {
let prompt_tokens = usage.prompt_token_count.unwrap_or(0);
let completion_tokens = usage.candidates_token_count.unwrap_or(0);
let prompt_tokens = gemini_prompt_tokens(usage);
let completion_tokens = gemini_completion_tokens(usage);
let total_tokens = usage
.total_token_count
.unwrap_or(prompt_tokens + completion_tokens);
@@ -694,6 +730,9 @@ pub fn gemini_event_to_openai_sse_chunks(
"prompt_tokens": prompt_tokens,
"completion_tokens": completion_tokens,
"total_tokens": total_tokens,
"prompt_tokens_details": {
"cached_tokens": usage.cached_content_token_count.unwrap_or(0)
},
}
});
chunks.push(format!("data: {}\n\n", chunk));
@@ -943,6 +982,7 @@ mod tests {
prompt_token_count: Some(12),
candidates_token_count: Some(7),
total_token_count: Some(19),
..Default::default()
}),
..Default::default()
};
@@ -969,6 +1009,79 @@ mod tests {
assert_eq!(usage_chunk["choices"], serde_json::json!([]));
}
#[test]
fn gemini_usage_chunk_splits_cached_and_bills_thoughts() {
let parsed = GeminiParsedEvent {
text: Some("the answer".to_string()),
usage: Some(GeminiUsageMetadata {
prompt_token_count: Some(1000),
candidates_token_count: Some(20),
total_token_count: Some(1120),
cached_content_token_count: Some(900),
thoughts_token_count: Some(100),
..Default::default()
}),
..Default::default()
};
let mut tool_call_index = 0;
let chunks = gemini_event_to_openai_sse_chunks(
&parsed,
"chatcmpl-test",
"gemini-3-flash-preview",
&mut tool_call_index,
);
let usage_chunk = chunks
.iter()
.map(|c| parse_sse_chunk(c))
.find(|v| v.get("usage").map(|u| !u.is_null()).unwrap_or(false))
.expect("a chunk should carry top-level usage");
// Gemini's prompt count already includes the cached tokens, so it passes
// through unchanged and the cached share is reported alongside it; thinking
// is billed as output but counted apart from the candidates.
assert_eq!(usage_chunk["usage"]["prompt_tokens"], 1000);
assert_eq!(usage_chunk["usage"]["prompt_tokens_details"]["cached_tokens"], 900);
assert_eq!(usage_chunk["usage"]["completion_tokens"], 120);
}
#[test]
fn gemini_usage_chunk_counts_tool_use_prompt_tokens() {
let parsed = GeminiParsedEvent {
text: Some("Canberra".to_string()),
usage: Some(GeminiUsageMetadata {
prompt_token_count: Some(17),
candidates_token_count: Some(17),
total_token_count: Some(146),
tool_use_prompt_token_count: Some(60),
thoughts_token_count: Some(52),
..Default::default()
}),
..Default::default()
};
let mut tool_call_index = 0;
let chunks = gemini_event_to_openai_sse_chunks(
&parsed,
"chatcmpl-test",
"gemini-2.5-flash",
&mut tool_call_index,
);
let usage_chunk = chunks
.iter()
.map(|c| parse_sse_chunk(c))
.find(|v| v.get("usage").map(|u| !u.is_null()).unwrap_or(false))
.expect("a chunk should carry top-level usage");
assert_eq!(usage_chunk["usage"]["prompt_tokens"], 77);
assert_eq!(usage_chunk["usage"]["completion_tokens"], 69);
assert_eq!(
usage_chunk["usage"]["prompt_tokens"].as_i64().unwrap()
+ usage_chunk["usage"]["completion_tokens"].as_i64().unwrap(),
146
);
}
#[test]
fn gemini_streaming_usage_total_falls_back_to_prompt_plus_completion() {
let parsed = GeminiParsedEvent {
@@ -976,6 +1089,7 @@ mod tests {
prompt_token_count: Some(5),
candidates_token_count: Some(3),
total_token_count: None,
..Default::default()
}),
..Default::default()
};
+54
View File
@@ -175,3 +175,57 @@ pub struct OpenAIMessage {
#[serde(skip_serializing_if = "Option::is_none")]
pub annotations: Option<Vec<UrlCitation>>,
}
// ============================================================================
// Model pricing
// ============================================================================
/// Far above any real per-million-token rate, so a value beyond it is a unit
/// mistake rather than a price. The floor matters more: a negative rate would make
/// spend subtract, and NaN/infinity would poison every total derived from it.
pub const MAX_MODEL_RATE: f64 = 1000.0;
/// Bound the `model_pricing` map of an AI config that is only available untyped —
/// the instance config is stored through the generic global-settings endpoint,
/// which never deserializes it into `AIConfig`, so the typed check on the
/// workspace path does not cover it.
pub fn validate_model_pricing_json(ai_config: &serde_json::Value) -> Result<(), String> {
// The container itself has to be checked too: a non-object `ai_config` persists
// here and then fails to deserialize as `AIConfig`, which drops the whole
// instance config back to its default for every workspace inheriting it.
if !ai_config.is_null() && !ai_config.is_object() {
return Err("ai_config must be an object".to_string());
}
let pricing = match ai_config.get("model_pricing") {
None | Some(serde_json::Value::Null) => return Ok(()),
// A present-but-wrong shape must be rejected, not skipped: it would persist
// and then fail to deserialize as `AIConfig`, which silently drops the whole
// instance config back to its default for every workspace inheriting it.
Some(v) => v
.as_object()
.ok_or_else(|| "model_pricing must be an object".to_string())?,
};
for (key, price) in pricing {
let Some(price) = price.as_object() else {
return Err(format!("Price override for {} is not an object", key));
};
for field in ["input", "output", "cache_read", "cache_write"] {
let Some(rate) = price.get(field) else { continue };
let rate = rate
.as_f64()
.filter(|r| r.is_finite() && *r >= 0.0 && *r <= MAX_MODEL_RATE);
if rate.is_none() {
return Err(format!(
"Price override for {}: {} must be between 0 and {}",
key, field, MAX_MODEL_RATE
));
}
}
for required in ["input", "output"] {
if !price.contains_key(required) {
return Err(format!("Price override for {} is missing {}", key, required));
}
}
}
Ok(())
}
@@ -660,6 +660,41 @@ fn bedrock_sse_chunks_for_event(
chunks.push(Bytes::from(format!("data: {}\n\n", chunk)));
}
// Usage arrives only on the trailing Metadata event, and only this converter
// reaches the chat: without a chunk for it a Bedrock chat reports no tokens at
// all. Bedrock counts cache reads and writes apart from `inputTokens`, while the
// OpenAI shape the client parses treats `prompt_tokens` as the whole input, so
// they are folded in here and split back out through `prompt_tokens_details`.
if let aws_sdk_bedrockruntime::types::ConverseStreamOutput::Metadata(metadata) = event {
if let Some(token_usage) = metadata.usage() {
let cache_read = token_usage.cache_read_input_tokens().unwrap_or(0);
let cache_write = token_usage.cache_write_input_tokens().unwrap_or(0);
let prompt_tokens = token_usage
.input_tokens()
.saturating_add(cache_read)
.saturating_add(cache_write);
let chunk = serde_json::json!({
"id": state.id,
"object": "chat.completion.chunk",
"created": state.created,
"model": state.model,
"choices": [],
"usage": {
"prompt_tokens": prompt_tokens,
"completion_tokens": token_usage.output_tokens(),
"total_tokens": token_usage.total_tokens(),
"prompt_tokens_details": {
"cached_tokens": cache_read,
"cache_write_tokens": cache_write
}
}
});
chunks.push(Bytes::from(format!("data: {}\n\n", chunk)));
}
}
chunks
}
@@ -1190,6 +1225,43 @@ mod tests {
serde_json::from_str(payload).expect("chunk should contain JSON")
}
#[test]
fn metadata_event_emits_usage_chunk_with_cache_split() {
let mut state = BedrockSseStreamState::new("id".to_string(), "model".to_string(), 0);
let event = ConverseStreamOutput::Metadata(
aws_sdk_bedrockruntime::types::ConverseStreamMetadataEvent::builder()
.usage(
aws_sdk_bedrockruntime::types::TokenUsage::builder()
.input_tokens(10)
.output_tokens(7)
.total_tokens(1017)
.cache_read_input_tokens(900)
.cache_write_input_tokens(100)
.build()
.expect("usage"),
)
.build(),
);
let chunks = bedrock_sse_chunks_for_event(&event, &mut state);
let usage = chunks
.iter()
.map(sse_json)
.find(|v| v.get("usage").map(|u| !u.is_null()).unwrap_or(false))
.expect("the metadata event should carry usage");
// Bedrock reports cache reads and writes apart from `inputTokens`; the OpenAI
// shape the client parses treats `prompt_tokens` as the whole input, and
// recovers the uncached share by subtracting the details back out.
assert_eq!(usage["usage"]["prompt_tokens"], 1010);
assert_eq!(usage["usage"]["completion_tokens"], 7);
assert_eq!(usage["usage"]["prompt_tokens_details"]["cached_tokens"], 900);
assert_eq!(
usage["usage"]["prompt_tokens_details"]["cache_write_tokens"],
100
);
}
#[test]
fn determine_auth_config_prioritizes_bearer_token() {
let config = determine_auth_config(
+7
View File
@@ -891,6 +891,13 @@ async fn run_setting_pre_write_hook(
value: &serde_json::Value,
) -> error::Result<()> {
match key {
// The instance AI config is written as an untyped blob through this generic
// endpoint, so it never passes the typed check the workspace handler applies.
// Rates that reach a cost total unbounded would make it negative or infinite.
AI_CONFIG_SETTING => {
windmill_ai::ai_types::validate_model_pricing_json(value)
.map_err(error::Error::BadRequest)?;
}
AUTOMATE_USERNAME_CREATION_SETTING => {
if value.as_bool().unwrap_or(false) {
generate_instance_username_for_all_users(db)
+153
View File
@@ -11986,6 +11986,73 @@ paths:
schema:
type: string
/w/{workspace}/ai/usage:
post:
summary: record AI token usage for the calling user
operationId: recordAiUsage
tags:
- ai
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
required: true
content:
application/json:
schema:
type: object
required:
- events
properties:
events:
type: array
items:
$ref: "#/components/schemas/AITokenUsageEvent"
responses:
"204":
description: usage recorded
get:
summary: list aggregated AI token usage
operationId: listAiUsage
tags:
- ai
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: days
in: query
schema:
type: integer
minimum: 1
maximum: 365
- name: group_by
in: query
schema:
type: string
enum: [day, user, model]
- name: scope
in: query
description: workspace-wide usage (admin only) or the calling user's own
schema:
type: string
enum: [workspace, self]
responses:
"200":
description: usage buckets
content:
application/json:
schema:
type: object
required:
- buckets
- truncated
properties:
buckets:
type: array
items:
$ref: "#/components/schemas/AITokenUsageBucket"
truncated:
type: boolean
description: more buckets matched than were returned, so summing them under-reports
/w/{workspace}/ai_skills/list:
get:
summary: list the workspace AI chat skills (name + description only)
@@ -26300,6 +26367,92 @@ components:
type: integer
minimum: 1
maximum: 2000000
model_pricing:
type: object
additionalProperties:
$ref: "#/components/schemas/ModelPriceOverride"
ModelPriceOverride:
type: object
description: negotiated rates in USD per million tokens, keyed `provider:model`
properties:
input:
type: number
minimum: 0
maximum: 1000
output:
type: number
minimum: 0
maximum: 1000
cache_read:
type: number
minimum: 0
maximum: 1000
cache_write:
type: number
minimum: 0
maximum: 1000
required:
- input
- output
AITokenUsageEvent:
type: object
properties:
provider:
$ref: "#/components/schemas/AIProvider"
model:
type: string
session_id:
type: string
input_tokens:
type: integer
cache_read_tokens:
type: integer
cache_write_tokens:
type: integer
output_tokens:
type: integer
reported_cost_nano_usd:
type: integer
description: only set by providers that bill back an exact figure
requests:
type: integer
required:
- provider
- model
AITokenUsageBucket:
type: object
properties:
key:
type: string
description: the grouped dimension's value; empty when grouping by model
provider:
type: string
model:
type: string
input_tokens:
type: integer
cache_read_tokens:
type: integer
cache_write_tokens:
type: integer
output_tokens:
type: integer
reported_cost_nano_usd:
type: integer
requests:
type: integer
required:
- key
- provider
- model
- input_tokens
- cache_read_tokens
- cache_write_tokens
- output_tokens
- requests
InstanceAIProviderSummary:
type: object
+326 -5
View File
@@ -3,11 +3,16 @@ use crate::utils::check_scopes;
#[cfg(feature = "bedrock")]
use axum::routing::get;
#[cfg(feature = "bedrock")]
use axum::Json;
use axum::{body::Bytes, extract::Path, response::IntoResponse, routing::post, Extension, Router};
use axum::{
body::Bytes,
extract::{DefaultBodyLimit, Path, Query},
response::IntoResponse,
routing::post,
Extension, Router,
};
use futures::StreamExt;
use http::{HeaderMap, Method};
use http::{HeaderMap, Method, StatusCode};
use quick_cache::sync::Cache;
use reqwest::{Client, RequestBuilder};
use serde::{Deserialize, Serialize};
@@ -18,6 +23,7 @@ use windmill_ai::ai_cache::current_instance_ai_config_revision;
use windmill_ai::ai_providers::{
empty_string_as_none, AIPlatform, AIProvider, ProviderConfig, ProviderModel,
};
use windmill_ai::ai_types::MAX_MODEL_RATE;
use windmill_ai::credentials::ProviderCredentials;
#[cfg(feature = "bedrock")]
use windmill_ai::providers::bedrock::{
@@ -37,7 +43,7 @@ use windmill_ai::proxy::{
use windmill_audit::{audit_oss::audit_log, ActionKind};
use windmill_common::db::UserDB;
use windmill_common::error::{to_anyhow, Error, Result};
use windmill_common::utils::configure_client;
use windmill_common::utils::{configure_client, require_admin};
use windmill_common::variables::{get_variable_or_self, get_variable_or_self_as};
// AI timeout configuration constants
@@ -417,9 +423,54 @@ pub struct AIConfig {
pub custom_prompts: Option<HashMap<String, String>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub max_tokens_per_model: Option<HashMap<String, i32>>,
/// Per-model price overrides, keyed `provider:model` like `max_tokens_per_model`.
/// Only models whose rates differ from the built-in table are stored.
#[serde(skip_serializing_if = "Option::is_none")]
pub model_pricing: Option<HashMap<String, ModelPriceOverride>>,
}
/// Negotiated rates in USD per million tokens. An unset cache rate is read as the
/// provider's own multiple of the input rate where the model has a published one,
/// and as the input rate itself where it does not — an unstated discount is never
/// filled in from another vendor's.
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct ModelPriceOverride {
pub input: f64,
pub output: f64,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_write: Option<f64>,
}
impl ModelPriceOverride {
pub fn validate(&self, key: &str) -> Result<()> {
for (field, rate) in [
("input", Some(self.input)),
("output", Some(self.output)),
("cache_read", self.cache_read),
("cache_write", self.cache_write),
] {
let Some(rate) = rate else { continue };
if !rate.is_finite() || rate < 0.0 || rate > MAX_MODEL_RATE {
return Err(Error::BadRequest(format!(
"Price override for {}: {} must be between 0 and {}",
key, field, MAX_MODEL_RATE
)));
}
}
Ok(())
}
}
impl AIConfig {
pub fn validate_model_pricing(&self) -> Result<()> {
for (key, price) in self.model_pricing.iter().flatten() {
price.validate(key)?;
}
Ok(())
}
pub fn has_providers(&self) -> bool {
self.providers
.as_ref()
@@ -432,7 +483,18 @@ pub fn global_service() -> Router {
}
pub fn workspaced_service() -> Router {
let router = Router::new().route("/proxy/{*ai}", post(proxy).get(proxy));
let router = Router::new()
.route("/proxy/{*ai}", post(proxy).get(proxy))
.route(
"/usage",
post(record_ai_usage)
.get(list_ai_usage)
// The handler caps how many events it *stores*, but Json deserializes
// the whole array first — without a body limit an authenticated member
// could make the server allocate and parse an arbitrarily large one.
// Sized well above a full batch of the shape below.
.layer(DefaultBodyLimit::max(AI_USAGE_BODY_LIMIT)),
);
#[cfg(feature = "bedrock")]
let router = router.route("/check_bedrock_credentials", get(check_bedrock_credentials));
@@ -440,6 +502,265 @@ pub fn workspaced_service() -> Router {
router
}
/// One provider request's worth of tokens, as counted by the chat client.
#[derive(Deserialize)]
struct AIUsageEvent {
provider: String,
model: String,
#[serde(default)]
session_id: String,
#[serde(default)]
input_tokens: i64,
#[serde(default)]
cache_read_tokens: i64,
#[serde(default)]
cache_write_tokens: i64,
#[serde(default)]
output_tokens: i64,
/// Only the providers that bill back an exact figure set this.
#[serde(default)]
reported_cost_nano_usd: Option<i64>,
#[serde(default)]
requests: Option<i64>,
}
#[derive(Deserialize)]
struct RecordAIUsagePayload {
events: Vec<AIUsageEvent>,
}
const MAX_AI_USAGE_EVENTS: usize = 50;
/// 64 KiB — a 50-event batch is a few kB even with the longest model ids.
const AI_USAGE_BODY_LIMIT: usize = 64 * 1024;
/// Well above any single conversation and far below an i64 overflow, so a client
/// bug caps out at one absurd row instead of poisoning the running total.
const MAX_TOKENS_PER_EVENT: i64 = 100_000_000;
/// $1000 in nano-USD.
const MAX_REPORTED_COST_PER_EVENT: i64 = 1_000_000_000_000;
/// Model ids carry vendor prefixes and variant suffixes (`anthropic/claude-opus-5:thinking`),
/// so the shape check is looser than an identifier but still excludes whitespace and
/// anything that would not be a model id.
fn is_model_shaped(s: &str, max_len: usize) -> bool {
!s.is_empty()
&& s.len() <= max_len
&& s.chars()
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | ':' | '.' | '/' | '~'))
}
/// Accumulate one workspace's AI token spend. Values are clamped and the caller's
/// email comes from the session, never the payload — the client is trusted to
/// report its own usage, not to attribute it to someone else.
async fn record_ai_usage(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Json(payload): Json<RecordAIUsagePayload>,
) -> Result<StatusCode> {
// Pre-sum duplicate keys: two rows hitting the same conflict target in a single
// INSERT error out ("cannot affect row a second time").
let mut agg: HashMap<(String, String, String), AIUsageTotals> = HashMap::new();
for e in payload.events.into_iter().take(MAX_AI_USAGE_EVENTS) {
if AIProvider::try_from(e.provider.as_str()).is_err()
|| !is_model_shaped(&e.model, 255)
|| !(e.session_id.is_empty() || is_model_shaped(&e.session_id, 50))
{
continue;
}
let totals = agg
.entry((e.provider, e.model, e.session_id))
.or_insert_with(AIUsageTotals::default);
totals.input += e.input_tokens.clamp(0, MAX_TOKENS_PER_EVENT);
totals.cache_read += e.cache_read_tokens.clamp(0, MAX_TOKENS_PER_EVENT);
totals.cache_write += e.cache_write_tokens.clamp(0, MAX_TOKENS_PER_EVENT);
totals.output += e.output_tokens.clamp(0, MAX_TOKENS_PER_EVENT);
totals.requests += e.requests.unwrap_or(1).clamp(0, MAX_AI_USAGE_EVENTS as i64);
if let Some(cost) = e.reported_cost_nano_usd {
totals.reported_cost = Some(
totals.reported_cost.unwrap_or(0) + cost.clamp(0, MAX_REPORTED_COST_PER_EVENT),
);
}
}
if agg.is_empty() {
return Ok(StatusCode::NO_CONTENT);
}
let mut providers = Vec::with_capacity(agg.len());
let mut models = Vec::with_capacity(agg.len());
let mut session_ids = Vec::with_capacity(agg.len());
let mut inputs = Vec::with_capacity(agg.len());
let mut cache_reads = Vec::with_capacity(agg.len());
let mut cache_writes = Vec::with_capacity(agg.len());
let mut outputs = Vec::with_capacity(agg.len());
let mut reported_costs: Vec<Option<i64>> = Vec::with_capacity(agg.len());
let mut requests = Vec::with_capacity(agg.len());
for ((provider, model, session_id), totals) in agg {
providers.push(provider);
models.push(model);
session_ids.push(session_id);
inputs.push(totals.input);
cache_reads.push(totals.cache_read);
cache_writes.push(totals.cache_write);
outputs.push(totals.output);
reported_costs.push(totals.reported_cost);
requests.push(totals.requests);
}
sqlx::query!(
"INSERT INTO ai_token_usage (workspace_id, email, provider, model, session_id, \
input_tokens, cache_read_tokens, cache_write_tokens, output_tokens, \
reported_cost_nano_usd, requests)
SELECT $1, $2, * FROM UNNEST($3::text[], $4::text[], $5::text[], $6::bigint[], \
$7::bigint[], $8::bigint[], $9::bigint[], $10::bigint[], $11::bigint[])
ON CONFLICT (workspace_id, day, email, provider, model, session_id)
DO UPDATE SET
input_tokens = ai_token_usage.input_tokens + EXCLUDED.input_tokens,
cache_read_tokens = ai_token_usage.cache_read_tokens + EXCLUDED.cache_read_tokens,
cache_write_tokens = ai_token_usage.cache_write_tokens + EXCLUDED.cache_write_tokens,
output_tokens = ai_token_usage.output_tokens + EXCLUDED.output_tokens,
reported_cost_nano_usd = CASE
WHEN EXCLUDED.reported_cost_nano_usd IS NULL
THEN ai_token_usage.reported_cost_nano_usd
ELSE COALESCE(ai_token_usage.reported_cost_nano_usd, 0)
+ EXCLUDED.reported_cost_nano_usd
END,
requests = ai_token_usage.requests + EXCLUDED.requests,
updated_at = now()",
&w_id,
&authed.email,
&providers,
&models,
&session_ids,
&inputs,
&cache_reads,
&cache_writes,
&outputs,
&reported_costs as &[Option<i64>],
&requests
)
.execute(&db)
.await?;
Ok(StatusCode::NO_CONTENT)
}
#[derive(Default)]
struct AIUsageTotals {
input: i64,
cache_read: i64,
cache_write: i64,
output: i64,
reported_cost: Option<i64>,
requests: i64,
}
#[derive(Deserialize)]
struct ListAIUsageQuery {
days: Option<i32>,
group_by: Option<String>,
scope: Option<String>,
}
/// A bucket always carries its provider and model: the caller prices it from a
/// per-model rate table, which a bucket spanning several models could not be
/// resolved against.
#[derive(Serialize)]
struct AITokenUsageBucket {
key: String,
provider: String,
model: String,
input_tokens: i64,
cache_read_tokens: i64,
cache_write_tokens: i64,
output_tokens: i64,
reported_cost_nano_usd: Option<i64>,
requests: i64,
}
/// Grouping by day over a long range, or by model across many models, can produce
/// more buckets than a table is worth rendering, so the listing is capped.
/// `truncated` says so explicitly — a caller that sums the rows into a total must be
/// able to tell that the total is partial rather than silently under-reporting spend.
#[derive(Serialize)]
struct AITokenUsageListing {
buckets: Vec<AITokenUsageBucket>,
truncated: bool,
}
const AI_USAGE_MAX_BUCKETS: i64 = 1000;
async fn list_ai_usage(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Query(query): Query<ListAIUsageQuery>,
) -> Result<Json<AITokenUsageListing>> {
// Reading the whole workspace's spend is an admin view; reading your own is
// not, so a member can see what they are costing without being shown their
// colleagues'. The filter is the session's email, never a parameter.
let own_email = match query.scope.as_deref().unwrap_or("workspace") {
"workspace" => {
require_admin(authed.is_admin, &authed.username)?;
None
}
"self" => Some(authed.email.clone()),
scope => return Err(Error::BadRequest(format!("Unsupported scope: {}", scope))),
};
let days = query.days.unwrap_or(30).clamp(1, 365);
let group_by = query.group_by.as_deref().unwrap_or("day");
// No `session`: a session is identified by a client-generated id whose name
// lives only in the browser that made it, so a bucket keyed on one is a label
// nobody can resolve. `session_id` is still stored, at the grain the client
// batches on, should sessions ever gain a server-side name.
if !matches!(group_by, "day" | "user" | "model") {
return Err(Error::BadRequest(format!(
"Unsupported group_by: {}",
group_by
)));
}
// Fetch one past the cap to detect truncation. Ordering is by token volume, not
// by cost: rates are applied by the caller, so this query cannot know what a
// bucket cost. Volume is the closest proxy available here, and the caller is told
// the listing was capped rather than being left to sum a partial set silently.
let mut rows = sqlx::query_as!(
AITokenUsageBucket,
r#"SELECT
(CASE $3::text
WHEN 'day' THEN day::text
WHEN 'user' THEN email
ELSE ''
END) AS "key!",
provider AS "provider!",
model AS "model!",
SUM(input_tokens)::bigint AS "input_tokens!",
SUM(cache_read_tokens)::bigint AS "cache_read_tokens!",
SUM(cache_write_tokens)::bigint AS "cache_write_tokens!",
SUM(output_tokens)::bigint AS "output_tokens!",
SUM(reported_cost_nano_usd)::bigint AS "reported_cost_nano_usd",
SUM(requests)::bigint AS "requests!"
FROM ai_token_usage
WHERE workspace_id = $1 AND day > CURRENT_DATE - $2::int
AND ($5::text IS NULL OR email = $5)
GROUP BY 1, provider, model
ORDER BY SUM(input_tokens + cache_read_tokens + cache_write_tokens + output_tokens) DESC
LIMIT $4"#,
&w_id,
days,
group_by,
AI_USAGE_MAX_BUCKETS + 1,
own_email.as_deref()
)
.fetch_all(&db)
.await?;
let truncated = rows.len() as i64 > AI_USAGE_MAX_BUCKETS;
rows.truncate(AI_USAGE_MAX_BUCKETS as usize);
Ok(Json(AITokenUsageListing { buckets: rows, truncated }))
}
/// Check if AWS Bedrock credentials are available from environment variables.
#[cfg(feature = "bedrock")]
async fn check_bedrock_credentials(
+2
View File
@@ -108,6 +108,8 @@ async fn edit_copilot_config(
}
}
ai_config.validate_model_pricing()?;
let mut tx = db.begin().await?;
sqlx::query!(
+11 -1
View File
@@ -3,7 +3,12 @@
// import aiStore back, and such a cycle crashes the app once the bundler splits it across
// chunks (docs/frontend-import-cycles.md; the build fails on the chunk cycle, not on this).
import { writable, get } from 'svelte/store'
import { type AIProviderModel, type AIProvider, type AIConfig } from './gen'
import {
type AIProviderModel,
type AIProvider,
type AIConfig,
type ModelPriceOverride
} from './gen'
import {
aiUserDisabled,
COPILOT_SESSION_MODEL_SETTING_NAME,
@@ -41,6 +46,8 @@ export const copilotInfo = writable<{
aiModels: AIProviderModel[]
customPrompts?: Record<string, string>
maxTokensPerModel?: Record<string, number>
/** Negotiated rates per `provider:model`, overriding the built-in price table. */
modelPricing?: Record<string, ModelPriceOverride>
webSearchEnabledProviders?: Partial<Record<AIProvider, boolean>>
}>({
enabled: false,
@@ -50,6 +57,7 @@ export const copilotInfo = writable<{
aiModels: [],
customPrompts: {},
maxTokensPerModel: {},
modelPricing: {},
webSearchEnabledProviders: {}
})
@@ -124,6 +132,7 @@ export function setCopilotInfo(aiConfig: AIConfig) {
aiModels: aiModels,
customPrompts: aiConfig.custom_prompts ?? {},
maxTokensPerModel: aiConfig.max_tokens_per_model ?? {},
modelPricing: aiConfig.model_pricing ?? {},
webSearchEnabledProviders
})
} else {
@@ -137,6 +146,7 @@ export function setCopilotInfo(aiConfig: AIConfig) {
aiModels: [],
customPrompts: {},
maxTokensPerModel: {},
modelPricing: {},
webSearchEnabledProviders: {}
})
}
@@ -9,6 +9,8 @@
import { createEventDispatcher } from 'svelte'
import UserInfoSettings from './settings/UserInfoSettings.svelte'
import AIUserSettings from './settings/AIUserSettings.svelte'
import AiUsagePanel from './workspaceSettings/AiUsagePanel.svelte'
import { copilotInfo, copilotWorkspace } from '$lib/aiStore'
import {
getDarkModeVariant,
setDarkModeVariant,
@@ -105,6 +107,17 @@
<AIUserSettings />
</div>
</div>
<!-- Keyed on the workspace `copilotInfo` reflects, not on `$workspaceStore`:
a session acting on another workspace loads that workspace's AI config
while navigation stays put, and pricing usage from one workspace with
another's rates would silently misstate it. -->
{#if $copilotWorkspace}
<AiUsagePanel
workspace={$copilotWorkspace}
modelPricing={$copilotInfo.modelPricing ?? {}}
scope="self"
/>
{/if}
{/if}
<div class="grow min-h-0">
@@ -58,7 +58,7 @@ import {
import { dfs } from '$lib/components/flows/previousResults'
import { SvelteMap, SvelteSet } from 'svelte/reactivity'
import { createLongHash } from '$lib/editorLangUtils'
import type { UserDraftItemKind } from '$lib/gen'
import type { AIProvider, UserDraftItemKind } from '$lib/gen'
import { maskKey } from '$lib/components/sessions/modifiedItemsMask'
import { getStringError } from './utils'
import { type PasteAttachment } from './pasteTokens'
@@ -101,7 +101,12 @@ import type AIChatInput from './AIChatInput.svelte'
import { prepareApiSystemMessage, prepareApiUserMessage } from './api/core'
import { runChatLoop, truncateToToolPairedPrefix } from './chatLoop'
import { sanitizeToolCallArguments } from './toolCallArguments'
import { normalizeContextUsage } from './tokenUsage'
import {
billedTokens,
normalizeContextUsage,
type ChatTokenUsage
} from './tokenUsage'
import { logAiUsage } from '$lib/utils/aiUsageReporter'
import type { ReviewChangesOpts } from './monaco-adapter'
import {
getCurrentModel,
@@ -709,6 +714,39 @@ export class AIChatManager {
await this.#persistModifiedItems()
}
/** Report one completed provider response's tokens to the workspace usage view.
* Called per response rather than per turn: a tool loop makes several, each
* separately billed, and a turn that fails partway through has still spent
* everything up to that point.
*
* Only token counts leave the browser — rates are applied when the usage is
* read, so a corrected price also corrects everything already recorded. */
private recordUsage(
usage: ChatTokenUsage,
provider: AIProvider,
model: string,
workspace: string | undefined
) {
// A provider that reports no usage still yields an all-zero report. Recording
// it would add a $0 row to the usage view, claiming the request cost nothing
// rather than that it went uncounted.
if (usage.total === 0 && usage.prompt === 0 && usage.completion === 0) {
return
}
const tokens = billedTokens(usage)
logAiUsage({
provider,
model,
sessionId: this.sessionId,
inputTokens: tokens.input,
cacheReadTokens: tokens.cacheRead,
cacheWriteTokens: tokens.cacheWrite,
outputTokens: tokens.output,
costUsd: usage.cost,
workspace
})
}
// Serialized, snapshot-at-write-time persistence: two rapid dock actions
// would otherwise race their saveChat writes, and the earlier (staler)
// snapshot could land last — dropping the later mutation until the next
@@ -2458,6 +2496,11 @@ export class AIChatManager {
// on each iteration. This is critical for changeModeTool (Navigator → Script/Flow)
// which reassigns this.tools, this.helpers, this.systemMessage mid-loop.
const self = this
// Pinned for the whole turn, like the `workspace` the loop routes through:
// the global chat's operating workspace follows workspaceStore, so a switch
// while a response streams would bill it to the workspace the user landed
// on rather than the one whose credentials and proxy served it.
const usageWorkspace = this.operatingWorkspace
const result = await runChatLoop({
messages,
addedMessages,
@@ -2535,6 +2578,14 @@ export class AIChatManager {
}
return undefined
},
onUsage: (usage, modelProvider) => {
// Accounting must never take a turn down with it.
try {
this.recordUsage(usage, modelProvider.provider, modelProvider.model, usageWorkspace)
} catch (e) {
console.error('Failed to record AI usage', e)
}
},
onBeforeIteration: async (tools, _helpers, modelProvider) => {
this.lastIterationModel = modelProvider
for (const tool of tools) {
@@ -4,6 +4,7 @@
import { getAiChatManager } from './aiChatManagerContext'
import { AIMode } from './AIChatManager.svelte'
import Tooltip from '$lib/components/meltComponents/Tooltip.svelte'
import { formatTokenCount } from './tokenUsage'
const aiChatManager = getAiChatManager()
@@ -45,16 +46,6 @@
? 'bg-amber-500'
: 'bg-surface-accent-primary'
)
function formatTokenCount(tokens: number): string {
if (tokens >= 1_000_000) {
return `${(tokens / 1_000_000).toFixed(1).replace(/\.0$/, '')}M`
}
if (tokens >= 1000) {
return `${Math.round(tokens / 1000)}k`
}
return `${tokens}`
}
</script>
{#if visible}
@@ -195,7 +195,7 @@ export async function parseAnthropicCompletion(
tools: Tool<any>[],
helpers: any,
abortController?: AbortController,
options?: { workspace?: string }
options?: { workspace?: string; onTokenUsage?: (usage: ChatTokenUsage) => void }
): Promise<ParsedCompletionResult> {
let toolCallsToProcess: ChatCompletionMessageFunctionToolCall[] = []
let error = null
@@ -417,6 +417,7 @@ export async function parseAnthropicCompletion(
const finalMessage = await completion.finalMessage()
const tokenUsage = anthropicUsageToChatTokenUsage(finalMessage.usage)
options?.onTokenUsage?.(tokenUsage)
// Process tool calls if any
if (toolCallsToProcess.length > 0) {
@@ -518,7 +518,7 @@ describe('runChatLoop lastIterationUsage', () => {
expect(result.lastIterationUsage).toEqual({ prompt: 1200, completion: 80, total: 1280 })
// the aggregate keeps summing across iterations
expect(result.tokenUsage).toEqual({ prompt: 2200, completion: 130, total: 2330 })
expect(result.tokenUsage).toMatchObject({ prompt: 2200, completion: 130, total: 2330 })
})
it('ignores empty usage reports and returns null when none are real', async () => {
@@ -77,11 +77,16 @@ export interface ChatLoopConfig {
helpers: any,
modelProvider: ReasoningProviderModel
) => Promise<void>
/** Fired for each completed provider response, before the loop continues. The
* loop can fail or be aborted at any iteration, so spend has to be handed over
* as it happens — a callback only at the end would discard everything the
* earlier iterations were already billed for. */
onUsage?: (usage: ChatTokenUsage, modelProvider: ReasoningProviderModel) => void
}
export interface ChatLoopResult {
addedMessages: ChatCompletionMessageParam[]
/** Sum of usage across all loop iterations (suitable for cost accounting). */
/** Sum of usage across all loop iterations. */
tokenUsage: ChatTokenUsage
lastIterationUsage: ChatTokenUsage | null
hitMaxIterations: boolean
@@ -328,6 +333,20 @@ export async function runChatLoop(config: ChatLoopConfig): Promise<ChatLoopResul
let lastIterationUsage: ChatTokenUsage | null = null
let iterations = 0
let hitMaxIterations = false
// The model of the iteration currently in flight; re-read per iteration like
// `config.modelProvider` itself, so usage is attributed to the model that
// actually served it rather than to whatever is selected when the loop ends.
let iterationModel: ReasoningProviderModel | undefined
// Reported as the provider's usage arrives, not when the parser returns: a parser
// waits on tool execution, which can wait on a person, and a tab closed in that
// gap would drop a response that was already billed. Accounting for the turn's
// own totals stays on the return path, where every parser reports uniformly.
const reportUsage = (usage: ChatTokenUsage | null | undefined) => {
if (usage && iterationModel) {
config.onUsage?.(usage, iterationModel)
}
}
const trackUsage = (usage: ChatTokenUsage | null | undefined) => {
tokenUsage = addChatTokenUsage(tokenUsage, usage)
@@ -351,6 +370,7 @@ export async function runChatLoop(config: ChatLoopConfig): Promise<ChatLoopResul
const helpers = config.helpers
const systemMessage = config.systemMessage
const modelProvider = config.modelProvider
iterationModel = modelProvider
const webSearchCacheKey = getWebSearchCacheKey(workspace, modelProvider)
const webSearch =
(config.webSearch ?? true) &&
@@ -386,7 +406,11 @@ export async function runChatLoop(config: ChatLoopConfig): Promise<ChatLoopResul
...(pendingUserMessage ? [pendingUserMessage] : [])
]
const toolDefs = tools.map((t) => t.def)
const parseOptions = { workspace, provider: modelProvider.provider }
const parseOptions = {
workspace,
provider: modelProvider.provider,
onTokenUsage: reportUsage
}
if (isOpenAI) {
const reasoningSummaryCacheKey = getReasoningSummaryCacheKey(workspace, modelProvider)
@@ -392,7 +392,7 @@ export async function parseOpenAIResponsesCompletion(
addedMessages: ChatCompletionMessageParam[],
tools: Tool<any>[],
helpers: any,
options?: { workspace?: string }
options?: { workspace?: string; onTokenUsage?: (usage: ChatTokenUsage) => void }
): Promise<ParsedCompletionResult> {
let toolCallsToProcess: ChatCompletionMessageFunctionToolCall[] = []
let error: OpenAIError | ResponseErrorEvent | null = null
@@ -566,6 +566,7 @@ export async function parseOpenAIResponsesCompletion(
const finalResponse = await runner.finalResponse()
const tokenUsage = openAIResponsesUsageToChatTokenUsage(finalResponse.usage)
options?.onTokenUsage?.(tokenUsage)
for (const item of finalResponse.output ?? []) {
if (item.type === 'web_search_call' && !surfacedWebSearchCalls.has(item.id)) {
@@ -0,0 +1,61 @@
import { describe, expect, it } from 'vitest'
import {
anthropicUsageToChatTokenUsage,
billedTokens,
openAICompletionsUsageToChatTokenUsage
} from './tokenUsage'
// The two providers report cache tokens under opposite conventions — Anthropic's
// input_tokens excludes them, OpenAI's includes them. Both are normalized so that
// `prompt` is the whole input, which is what makes `prompt - cached` the uncached
// share. Getting this backwards double-counts (or loses) the cached prefix, which
// is most of a long chat's input.
describe('billedTokens', () => {
it('derives uncached input under the Anthropic convention', () => {
const usage = anthropicUsageToChatTokenUsage({
input_tokens: 1000,
output_tokens: 200,
cache_creation_input_tokens: 300,
cache_read_input_tokens: 5000
})
expect(usage.prompt).toBe(6300)
expect(billedTokens(usage)).toEqual({
input: 1000,
cacheRead: 5000,
cacheWrite: 300,
output: 200
})
})
it('derives uncached input under the OpenAI convention', () => {
const usage = openAICompletionsUsageToChatTokenUsage({
prompt_tokens: 6000,
completion_tokens: 200,
prompt_tokens_details: { cached_tokens: 5000 }
})
expect(usage.prompt).toBe(6000)
expect(billedTokens(usage)).toEqual({
input: 1000,
cacheRead: 5000,
cacheWrite: 0,
output: 200
})
})
// OpenRouter extends the OpenAI shape with cache-creation tokens, counted
// inside prompt_tokens like the reads beside them. Missing the field bills
// them as uncached input.
it('splits out OpenRouter cache-creation tokens', () => {
const usage = openAICompletionsUsageToChatTokenUsage({
prompt_tokens: 6300,
completion_tokens: 200,
prompt_tokens_details: { cached_tokens: 5000, cache_write_tokens: 300 }
})
expect(billedTokens(usage)).toEqual({
input: 1000,
cacheRead: 5000,
cacheWrite: 300,
output: 200
})
})
})
@@ -1,7 +1,19 @@
import type { PricedTokens } from '../modelPricing'
export interface ChatTokenUsage {
prompt: number
completion: number
total: number
/**
* Subsets of `prompt`, split out because they are billed at different rates
* (a cached read is a fraction of an uncached one). `prompt` stays the whole
* input so the context gauge keeps measuring the whole request; uncached
* input is `prompt - cacheRead - cacheWrite`.
*/
cacheRead: number
cacheWrite: number
/** Cost in USD as billed, for the providers that report one. */
cost?: number
}
/**
@@ -28,7 +40,7 @@ export function normalizeContextUsage(
}
export function emptyChatTokenUsage(): ChatTokenUsage {
return { prompt: 0, completion: 0, total: 0 }
return { prompt: 0, completion: 0, total: 0, cacheRead: 0, cacheWrite: 0 }
}
export function addChatTokenUsage(
@@ -39,10 +51,48 @@ export function addChatTokenUsage(
return total
}
const cost =
total.cost === undefined && usage.cost === undefined
? undefined
: (total.cost ?? 0) + (usage.cost ?? 0)
return {
prompt: total.prompt + usage.prompt,
completion: total.completion + usage.completion,
total: total.total + usage.total
total: total.total + usage.total,
// `?? 0`: the cache split is newer than the field it lives on, so a usage
// object read back from storage may predate it.
cacheRead: (total.cacheRead ?? 0) + (usage.cacheRead ?? 0),
cacheWrite: (total.cacheWrite ?? 0) + (usage.cacheWrite ?? 0),
...(cost === undefined ? {} : { cost })
}
}
/** Compact token count for readouts and tables (`1.2M`, `34k`, `567`). */
export function formatTokenCount(tokens: number): string {
if (tokens >= 1_000_000) {
return `${(tokens / 1_000_000).toFixed(1).replace(/\.0$/, '')}M`
}
if (tokens >= 1000) {
return `${Math.round(tokens / 1000)}k`
}
return `${tokens}`
}
/**
* Split a usage report into the four separately-billed token classes. `prompt`
* counts the whole input, so the uncached share is whatever the cached classes
* do not account for — which holds for both provider conventions below
* (Anthropic adds its cache counts into `prompt`, OpenAI's already includes them).
*/
export function billedTokens(usage: ChatTokenUsage): PricedTokens {
const cacheRead = usage.cacheRead ?? 0
const cacheWrite = usage.cacheWrite ?? 0
return {
input: Math.max(0, usage.prompt - cacheRead - cacheWrite),
cacheRead,
cacheWrite,
output: usage.completion
}
}
@@ -57,16 +107,17 @@ export function anthropicUsageToChatTokenUsage(
| null
| undefined
): ChatTokenUsage {
const prompt =
(usage?.input_tokens ?? 0) +
(usage?.cache_creation_input_tokens ?? 0) +
(usage?.cache_read_input_tokens ?? 0)
const cacheWrite = usage?.cache_creation_input_tokens ?? 0
const cacheRead = usage?.cache_read_input_tokens ?? 0
const prompt = (usage?.input_tokens ?? 0) + cacheWrite + cacheRead
const completion = usage?.output_tokens ?? 0
return {
prompt,
completion,
total: prompt + completion
total: prompt + completion,
cacheRead,
cacheWrite
}
}
@@ -89,7 +140,11 @@ export function openAIResponsesUsageToChatTokenUsage(
return {
prompt,
completion,
total: usage?.total_tokens ?? prompt + completion
total: usage?.total_tokens ?? prompt + completion,
cacheRead: usage?.input_tokens_details?.cached_tokens ?? 0,
// Automatic caching: nothing is billed for populating it, and no usage
// field reports it either.
cacheWrite: 0
}
}
@@ -101,7 +156,16 @@ export function openAICompletionsUsageToChatTokenUsage(
prompt_tokens?: number | null
completion_tokens?: number | null
total_tokens?: number | null
prompt_tokens_details?: { cached_tokens?: number | null } | null
prompt_tokens_details?: {
cached_tokens?: number | null
/** Cache creation, reported by the providers that bill for it: OpenRouter
* passes Anthropic's through, and the Bedrock proxy folds
* `cacheWriteInputTokens` in here. OpenAI, whose caching is automatic and
* unbilled, reports no such field. */
cache_write_tokens?: number | null
} | null
/** OpenRouter reports what it actually charged when the request opts in. */
cost?: number | null
}
| null
| undefined
@@ -112,6 +176,9 @@ export function openAICompletionsUsageToChatTokenUsage(
return {
prompt,
completion,
total: usage?.total_tokens ?? prompt + completion
total: usage?.total_tokens ?? prompt + completion,
cacheRead: usage?.prompt_tokens_details?.cached_tokens ?? 0,
cacheWrite: usage?.prompt_tokens_details?.cache_write_tokens ?? 0,
...(typeof usage?.cost === 'number' ? { cost: usage.cost } : {})
}
}
+29 -7
View File
@@ -1089,6 +1089,23 @@ export async function getFimCompletion(
}
}
// A streamed OpenAI-compatible response carries no usage at all unless the request
// asks for it, so a provider missing from this set reports zero tokens — no context
// gauge, no cost. `stream_options.include_usage` is part of the OpenAI streaming
// spec and these providers document supporting it; `customai` is deliberately absent
// because it points at an arbitrary endpoint that may reject the field outright.
const STREAM_USAGE_PROVIDERS = new Set<AIProvider>([
'openai',
'azure_openai',
'azure_foundry',
'googleai',
'openrouter',
'groq',
'deepseek',
'mistral',
'togetherai'
])
export async function getCompletion(
messages: ChatCompletionMessageParam[],
abortController: AbortController,
@@ -1132,17 +1149,17 @@ export async function getCompletion(
// Use Completions API for other providers
const client = options?.openaiClient ?? workspaceAIClients.getOpenaiClient()
const completionConfig = applyReasoningToConfig(
(provider === 'openai' ||
provider === 'azure_openai' ||
provider === 'azure_foundry' ||
provider === 'googleai') &&
config.stream
config.stream && STREAM_USAGE_PROVIDERS.has(provider)
? {
...config,
stream_options: {
...(config.stream_options ?? {}),
include_usage: true
}
},
// OpenRouter's own extension, on top of stream_options: it returns the
// credits actually charged next to the token counts, which is the one
// route by which the chat sees a real cost rather than an estimate.
...(provider === 'openrouter' ? { usage: { include: true } } : {})
}
: config,
provider === 'deepseek' ? 'deepseek' : provider === 'mistral' ? 'mistral' : 'completions',
@@ -1178,7 +1195,11 @@ export async function parseOpenAICompletion(
tools: Tool<any>[],
helpers: any,
_abortController?: AbortController, // unused, for signature compatibility with parseAnthropicCompletion
options?: { workspace?: string; provider?: string }
options?: {
workspace?: string
provider?: string
onTokenUsage?: (usage: ChatTokenUsage) => void
}
): Promise<{ shouldContinue: boolean; tokenUsage: ChatTokenUsage }> {
const finalToolCalls: Record<number, ChatCompletionChunk.Choice.Delta.ToolCall> = {}
// The tool call currently receiving argument deltas; when the stream moves on
@@ -1328,6 +1349,7 @@ export async function parseOpenAICompletion(
callbacks.onMessageEnd()
options?.onTokenUsage?.(tokenUsage)
// Stream over: every parsed call is queued until its turn in processToolCall.
for (const toolCall of Object.values(finalToolCalls)) {
if (toolCall.id) {
@@ -123,21 +123,77 @@ function normalizeVersionSeparators(model: string): string {
return model.replace(/\./g, '-')
}
// An entry that ends on a version digit must not run into a longer version:
// `gpt-4.1` collapses to `gpt-4-1`, which would otherwise claim the 128K
// `gpt-4-1106-preview` as a 1M model. Suffixes that continue with a separator
// (`claude-opus-4-8` in `...-4-8-v1`, `gpt-5` in `gpt-5-mini`) still match.
// Family fallbacks ending on a letter get no such guard — a version welded
// straight onto the name (`llama3.1`) is exactly what they exist to catch.
const MODEL_CONTEXT_WINDOW_MATCHERS: [matcher: RegExp, contextWindow: number][] =
MODEL_CONTEXT_WINDOWS.map(([name, contextWindow]) => {
/** Suffixes that name a route to a model rather than a different model. */
const DECORATIVE_SUFFIXES = ['latest', 'preview', 'beta', 'stable']
/**
* Compile a most-specific-first `[name, value]` table into matchers against the
* bare model id. Shared with the pricing table so both resolve the same set of
* ids — a model whose window is known but whose price is not (or vice versa)
* should be a gap in one table, never a difference in matching.
*
* An entry that ends on a version digit must not run into a longer version:
* `gpt-4.1` collapses to `gpt-4-1`, which would otherwise claim
* `gpt-4-1106-preview`. Suffixes that continue with a separator
* (`claude-opus-4-8` in `...-4-8-v1`, `gpt-5` in `gpt-5-mini`) still match.
* Family fallbacks ending on a letter get no such guard — a version welded
* straight onto the name (`llama3.1`) is exactly what they exist to catch.
*/
export function buildModelMatchers<T>(
entries: [name: string, value: T][],
{ strictVariants = false }: { strictVariants?: boolean } = {}
): [RegExp, T][] {
return entries.map(([name, value]) => {
const pattern = normalizeVersionSeparators(name).replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
return [new RegExp(/\d$/.test(pattern) ? `${pattern}(?!\\d)` : pattern), contextWindow]
const guards = [
// An entry ending on a version digit must not run into a longer version.
/\d$/.test(pattern) ? '(?!\\d)' : '',
// A named sub-model (`gpt-5-pro`, `gpt-5-mini`) is a different model with
// its own price, not another route to this one — so under strictVariants an
// entry does not match when a further *name* segment follows. What follows
// is only a decoration when it is a date (`-20251101`), Bedrock's `-v1`, or
// one of the alias words below, at the very end of the id
// (`claude-3-5-haiku-latest` is the same model as `claude-3-5-haiku`, and is
// a shipped default; `gpt-5-preview-pro` would be a different one again).
// Off by default: for a context window an inherited value is a safe
// approximation, for a price it is a wrong number.
// A further revision segment (`gpt-5` vs `gpt-5-4-mini`) is a different model
// too, and the entry-ends-on-a-digit guard above does not catch it once the
// separator is normalized. Only a short segment: a date is digits as well
// (`-20251101`) and stays a decoration.
strictVariants ? '(?!-\\d{1,3}(?:$|-))' : '',
strictVariants
? `(?!-(?!(?:v\\d|${DECORATIVE_SUFFIXES.join('|')})$)[a-z])`
: ''
].join('')
return [new RegExp(pattern + guards), value]
})
}
/**
* The `provider:model` key the workspace AI settings use for their per-model maps
* (`max_tokens_per_model`, `model_pricing`). A bare model id is not enough: the
* same id can be served by more than one provider at different rates.
*
* Matched exactly, unlike the fuzzy tables above. Those tables generalize across
* every route to one model on purpose; a per-model *setting* must not, or an
* admin could not give two variants of a family different values — and the key is
* built from the exact id the provider config lists, which is the same string the
* chat sends.
*/
export function modelKey(provider: AIProvider | string, model: string): string {
return `${provider}:${model}`
}
export function matchModel<T>(matchers: [RegExp, T][], model: string): T | undefined {
const id = normalizeVersionSeparators(parseModelId(model).base)
return matchers.find(([matcher]) => matcher.test(id))?.[1]
}
const MODEL_CONTEXT_WINDOW_MATCHERS = buildModelMatchers(MODEL_CONTEXT_WINDOWS)
export function getKnownModelContextWindow(model: string): number | undefined {
const id = normalizeVersionSeparators(parseModelId(model).base)
return MODEL_CONTEXT_WINDOW_MATCHERS.find(([matcher]) => matcher.test(id))?.[1]
return matchModel(MODEL_CONTEXT_WINDOW_MATCHERS, model)
}
export function getModelContextWindow(model: string) {
@@ -0,0 +1,187 @@
import { describe, expect, it } from 'vitest'
import { billedTokens } from './chat/tokenUsage'
import { estimateCost, priceSpend, resolveModelPrice } from './modelPricing'
describe('resolveModelPrice', () => {
it('resolves the same model across the routes that decorate its id', () => {
const direct = resolveModelPrice('anthropic', 'claude-opus-5', undefined)
expect(direct?.price.input).toBe(5)
// A gateway prefix, a dot-versioned id, a date suffix and a variant suffix
// must all land on the same entry — a miss here silently under-reports cost.
for (const id of [
'anthropic/claude-opus-5',
'anthropic/claude-opus-4.8',
'claude-opus-4-8-20260101',
'anthropic/claude-opus-5:thinking'
]) {
expect(resolveModelPrice('openrouter', id, undefined)?.price.input).toBe(5)
}
})
it('does not let a version-digit entry claim a longer version', () => {
expect(resolveModelPrice('openai', 'gpt-4.1', undefined)?.price.input).toBe(2)
expect(resolveModelPrice('openai', 'gpt-4-1106-preview', undefined)?.price.input).not.toBe(2)
})
it('prices flat-rate Gemini Flash while leaving the tiered Pro alone', () => {
expect(resolveModelPrice('googleai', 'gemini-2.5-flash', undefined)?.price.input).toBe(0.3)
expect(resolveModelPrice('googleai', 'gemini-2.5-flash-lite', undefined)?.price.input).toBe(0.1)
expect(resolveModelPrice('googleai', 'gemini-3.5-flash', undefined)?.price.output).toBe(9)
// Pro charges roughly double above a 200k prompt, which a per-model rate cannot
// express, so it must stay unpriced rather than be estimated at the low tier.
expect(resolveModelPrice('googleai', 'gemini-2.5-pro', undefined)).toBeUndefined()
expect(resolveModelPrice('googleai', 'gemini-3.1-pro', undefined)).toBeUndefined()
// Promotional rates carry an end date a timeless table cannot represent.
expect(resolveModelPrice('googleai', 'gemini-3.7-flash', undefined)).toBeUndefined()
})
it('reports an unknown model as unpriced rather than guessing', () => {
expect(resolveModelPrice('customai', 'some-in-house-model', undefined)).toBeUndefined()
})
it('does not let another model inherit a price through a shared prefix', () => {
// A sub-model (`-pro`) or a newer revision (`gpt-5.6` → `gpt-5-6`) is a
// different model at a different rate; inheriting `gpt-5`'s would be off by
// an order of magnitude, and silently so.
expect(resolveModelPrice('openai', 'gpt-5', undefined)?.price.input).toBe(1.25)
expect(resolveModelPrice('openai', 'gpt-5-mini', undefined)?.price.input).toBe(0.25)
expect(resolveModelPrice('openai', 'gpt-5-pro', undefined)).toBeUndefined()
expect(resolveModelPrice('openai', 'gpt-5.6', undefined)).toBeUndefined()
expect(resolveModelPrice('googleai', 'gemini-3.1', undefined)).toBeUndefined()
// A revision carrying a variant has to be caught by the matcher, not by an
// explicit entry: `gpt-5.4-mini` cannot match the `gpt-5.4` one (the `-mini`
// makes it a sub-model), so nothing but the guard stops it reaching `gpt-5`.
expect(resolveModelPrice('openai', 'gpt-5.4-mini', undefined)).toBeUndefined()
expect(resolveModelPrice('openai', 'gpt-5.5-pro', undefined)).toBeUndefined()
})
it('still resolves the route decorations that name the same model', () => {
// Dates, Bedrock's -v1 and floating aliases are ways of spelling one model,
// not sub-models. `claude-3-5-haiku-latest` is a shipped picker default, so
// unpricing it would silently disable cost tracking out of the box.
expect(resolveModelPrice('anthropic', 'claude-opus-4-5-20251101', undefined)?.price.input).toBe(5)
// The revision guard must not swallow a date, which is digits too.
expect(resolveModelPrice('openai', 'gpt-5-2026-01-01', undefined)?.price.input).toBe(1.25)
expect(
resolveModelPrice('bedrock', 'anthropic.claude-sonnet-4-6-20250101-v1:0', undefined)?.price
.input
).toBe(3)
expect(resolveModelPrice('anthropic', 'claude-3-5-haiku-latest', undefined)?.price.input).toBe(
0.8
)
// …while a genuine sub-model stays unpriced, including one hiding behind a
// decoration.
expect(resolveModelPrice('openai', 'gpt-5-pro', undefined)).toBeUndefined()
expect(resolveModelPrice('openai', 'gpt-5-preview-pro', undefined)).toBeUndefined()
// A family fallback must not price a model the table deliberately left out,
// nor the floating alias pointing at it.
expect(resolveModelPrice('anthropic', 'claude-sonnet-5', undefined)).toBeUndefined()
expect(
resolveModelPrice('openrouter', '~anthropic/claude-sonnet-latest', undefined)
).toBeUndefined()
})
it('prefers a workspace override, keeping the models own cache ratios', () => {
const resolved = resolveModelPrice('anthropic', 'claude-opus-5', {
'anthropic:claude-opus-5': { input: 2, output: 8 }
})
expect(resolved?.source).toBe('override')
expect(resolved?.price.input).toBe(2)
// Anthropic reads a cached prefix at a tenth and writes at 1.25x.
expect(resolved?.price.cacheRead).toBeCloseTo(0.2)
expect(resolved?.price.cacheWrite).toBeCloseTo(2.5)
})
it('applies the overridden models own cache discount, not Anthropics', () => {
// gpt-4o discounts a cached read by half, not by a tenth — an override that
// only states input/output must not silently inherit the Anthropic ratio.
const resolved = resolveModelPrice('openai', 'gpt-4o', {
'openai:gpt-4o': { input: 2, output: 8 }
})
expect(resolved?.price.cacheRead).toBeCloseTo(1)
})
it('bills an unpriced models cached tokens at its input rate', () => {
// Gemini Pro is deliberately unpriced, so there is no ratio to inherit. Falling
// back to Anthropic's tenth would invent a discount the provider may not give;
// the admin states the cache rates explicitly or pays full input.
const resolved = resolveModelPrice('googleai', 'gemini-2.5-pro', {
'googleai:gemini-2.5-pro': { input: 2, output: 8 }
})
expect(resolved?.price.cacheRead).toBe(2)
expect(resolved?.price.cacheWrite).toBe(2)
const stated = resolveModelPrice('googleai', 'gemini-2.5-pro', {
'googleai:gemini-2.5-pro': { input: 2, output: 8, cache_read: 0.5, cache_write: 1 }
})
expect(stated?.price.cacheRead).toBe(0.5)
expect(stated?.price.cacheWrite).toBe(1)
})
it('ignores an override whose rates could not be a price', () => {
for (const bad of [{ input: -1, output: 8 }, { input: 1e9, output: 8 }]) {
const resolved = resolveModelPrice('anthropic', 'claude-opus-5', {
'anthropic:claude-opus-5': bad
})
expect(resolved?.source).toBe('builtin')
}
})
})
describe('estimateCost', () => {
it('bills each token class at its own rate', () => {
const cost = estimateCost(
{ input: 1_000_000, cacheRead: 1_000_000, cacheWrite: 1_000_000, output: 1_000_000 },
{ input: 5, output: 25, cacheRead: 0.5, cacheWrite: 6.25 }
)
expect(cost).toBeCloseTo(5 + 0.5 + 6.25 + 25)
})
it('charges a cached prefix less than an uncached one', () => {
const usage = {
prompt: 100_000,
completion: 0,
total: 100_000,
cacheRead: 90_000,
cacheWrite: 0
}
const uncached = { ...usage, cacheRead: 0 }
const price = { input: 5, output: 25, cacheRead: 0.5, cacheWrite: 6.25 }
expect(estimateCost(billedTokens(usage), price)).toBeLessThan(
estimateCost(billedTokens(uncached), price)
)
})
})
describe('priceSpend', () => {
it('prefers a provider-reported cost over the estimate', () => {
const priced = priceSpend(
[
{
provider: 'openrouter',
model: 'anthropic/claude-opus-5',
tokens: { input: 1_000_000, cacheRead: 0, cacheWrite: 0, output: 0 },
reportedCostUsd: 0.42
}
],
undefined
)
expect(priced.total).toBe(0.42)
expect(priced.hasReported).toBe(true)
})
it('flags an unpriced model instead of counting it as free', () => {
const priced = priceSpend(
[
{
provider: 'customai',
model: 'some-in-house-model',
tokens: { input: 1_000_000, cacheRead: 0, cacheWrite: 0, output: 0 }
}
],
undefined
)
expect(priced.hasUnpriced).toBe(true)
expect(priced.rows[0].cost).toBeUndefined()
})
})
@@ -0,0 +1,288 @@
import type { AIProvider, ModelPriceOverride } from '$lib/gen'
import { buildModelMatchers, matchModel, modelKey } from './modelConfig'
/** Rates in USD per million tokens, one per billed token class. */
export type ModelPrice = {
input: number
output: number
cacheRead: number
cacheWrite: number
}
export type ModelPriceSource = 'override' | 'builtin'
export type ResolvedModelPrice = {
price: ModelPrice
source: ModelPriceSource
}
/** What a chat spent on one model, in tokens. */
export type PricedTokens = {
input: number
cacheRead: number
cacheWrite: number
output: number
}
// Fallbacks for entries that do not price their cache separately: Anthropic reads a
// cached prefix at a tenth of the input rate and writes one at 1.25x (5-minute TTL,
// the default the chat uses). The read ratio is NOT universal — OpenAI and Google
// discount a cached read far less — so every non-Anthropic entry below states its own
// `cacheRead` rather than inheriting this. Providers whose caching is automatic never
// report a cache write, so their write rate is unused.
const CACHE_READ_RATIO = 0.1
const CACHE_WRITE_RATIO = 1.25
type PriceEntry = {
input: number
output: number
cacheRead?: number
cacheWrite?: number
}
/**
* Published list prices, most specific entry first — the first name found in the
* bare model id wins, so vendor-namespaced and date-suffixed ids
* (anthropic/claude-opus-5, gpt-5-2026-01-01) still resolve. Matching is shared
* with the context-window table via `buildModelMatchers`.
*
* This is a best-effort snapshot: vendors change rates, ship models faster than
* this table is updated, and negotiated rates differ from list. A model that is
* not listed resolves to undefined and is reported as unpriced rather than
* guessed at, and any entry can be corrected per workspace from the AI settings.
* Providers whose catalogue turns over too quickly to track (DeepSeek, Mistral,
* Groq, TogetherAI, custom deployments) are deliberately absent.
*
* `null` marks a model that is known to exist but whose rates are not. Unpriced is
* a supported state (the UI says so and points at the override); a confidently
* wrong number is not — which is also why these matchers are built with
* `strictVariants`, so an unlisted sub-model (`gpt-5-pro`) reports no rate instead
* of inheriting its family's.
*
* One known gap the per-model shape cannot express: Anthropic's 1M-context beta
* charges more above a threshold. Usage is aggregated per model before pricing, so
* those requests are estimated at the standard tier and understate. An affected
* workspace can set the higher rate as its override.
*/
const MODEL_PRICES: [name: string, price: PriceEntry | null][] = [
// Anthropic — Opus 4.1 and older bill at the pre-4.5 Opus rate, so the family
// fallback sits below the explicit entries rather than covering them.
['claude-fable-5', { input: 10, output: 50 }],
['claude-mythos-5', { input: 10, output: 50 }],
['claude-opus-5', { input: 5, output: 25 }],
['claude-opus-4-8', { input: 5, output: 25 }],
['claude-opus-4-7', { input: 5, output: 25 }],
['claude-opus-4-6', { input: 5, output: 25 }],
['claude-opus-4-5', { input: 5, output: 25 }],
['claude-opus-4-1', { input: 15, output: 75 }],
['claude-opus-4', { input: 15, output: 75 }],
// Sonnet 5 runs a promotional rate with a published end date, and
// `claude-sonnet-latest` floats to it. Rates carry no date and apply at read
// time, so either figure misstates one side of that boundary — unpriced until
// the rate is a single number again.
['claude-sonnet-5', null],
['claude-sonnet-latest', null],
['claude-sonnet-4-6', { input: 3, output: 15 }],
['claude-sonnet-4-5', { input: 3, output: 15 }],
['claude-sonnet-4', { input: 3, output: 15 }],
['claude-haiku-4-5', { input: 1, output: 5 }],
['claude-3-5-haiku', { input: 0.8, output: 4 }],
['claude-opus', { input: 5, output: 25 }],
['claude-sonnet', { input: 3, output: 15 }],
['claude-haiku', { input: 1, output: 5 }],
// OpenAI — the cached-input discount varies by family (a tenth on gpt-5, a
// quarter on 4.1 and the o-series, half on 4o), so each entry carries its own
// rate. There is no charge for writing the cache and no usage field reporting
// one, so the write rate never applies. The -mini/-nano entries must precede
// the family entry, which would otherwise claim them.
// Revisions past gpt-5 are priced separately by OpenAI and are not tracked here.
// The matcher's revision guard already keeps them off the family rate; these
// entries stay so a revision the guard admits still resolves to no rate.
['gpt-5.6', null],
['gpt-5.5', null],
['gpt-5.4', null],
['gpt-5.2', null],
['gpt-5.1', null],
['gpt-5-mini', { input: 0.25, output: 2, cacheRead: 0.025 }],
['gpt-5-nano', { input: 0.05, output: 0.4, cacheRead: 0.005 }],
['gpt-5', { input: 1.25, output: 10, cacheRead: 0.125 }],
['gpt-4.1-mini', { input: 0.4, output: 1.6, cacheRead: 0.1 }],
['gpt-4.1-nano', { input: 0.1, output: 0.4, cacheRead: 0.025 }],
['gpt-4.1', { input: 2, output: 8, cacheRead: 0.5 }],
['gpt-4o-mini', { input: 0.15, output: 0.6, cacheRead: 0.075 }],
['gpt-4o', { input: 2.5, output: 10, cacheRead: 1.25 }],
['o4-mini', { input: 1.1, output: 4.4, cacheRead: 0.275 }],
['o3-mini', { input: 1.1, output: 4.4, cacheRead: 0.55 }],
['o3', { input: 2, output: 8, cacheRead: 0.5 }],
// Google — Flash takes a flat rate and is priced; Pro is not, because both its
// input and output roughly double above a 200k-token prompt and a per-model rate
// cannot express a threshold. Explicit context caching also bills storage per hour,
// which nothing here represents, so a workspace using it sees an underestimate.
// Gemini 3.7 and 3.6 Flash run a promotional rate with an end date, and stay
// unpriced for the same reason Sonnet 5 does.
['gemini-2.5-flash-lite', { input: 0.1, output: 0.4, cacheRead: 0.01 }],
['gemini-2.5-flash', { input: 0.3, output: 2.5, cacheRead: 0.03 }],
['gemini-3.5-flash-lite', { input: 0.3, output: 2.5, cacheRead: 0.03 }],
['gemini-3.5-flash', { input: 1.5, output: 9, cacheRead: 0.15 }],
['gemini-3.7', null],
['gemini-3.6', null],
['gemini-3.1', null],
['gemini-3', null],
['gemini-2.5', null]
]
const MODEL_PRICE_MATCHERS = buildModelMatchers(
MODEL_PRICES.map(([name, entry]): [string, ModelPrice | null] => [
name,
entry && {
input: entry.input,
output: entry.output,
cacheRead: entry.cacheRead ?? entry.input * CACHE_READ_RATIO,
cacheWrite: entry.cacheWrite ?? entry.input * CACHE_WRITE_RATIO
}
]),
{ strictVariants: true }
)
export function getKnownModelPrice(model: string): ModelPrice | undefined {
return matchModel(MODEL_PRICE_MATCHERS, model) ?? undefined
}
/**
* Rates the API bounds on the way in — but an instance-level config is stored as an
* untyped settings blob that bypasses that handler, so the reader enforces the same
* bounds rather than rendering a negative, infinite or absurd total.
*/
const MAX_MODEL_RATE = 1000
function isUsableRate(rate: number | undefined): boolean {
return rate === undefined || (Number.isFinite(rate) && rate >= 0 && rate <= MAX_MODEL_RATE)
}
/** What a cache rate falls back to when an override leaves it unset: the model's
* own published multiple of the input rate where the table has one, and the input
* rate itself where it does not, so an unstated discount is never borrowed from
* another vendor. Shared with the rates editor, which shows these as placeholders. */
export function inheritedCacheRates(
model: string,
input: number
): { cacheRead: number; cacheWrite: number } {
const builtin = getKnownModelPrice(model)
return {
cacheRead: input * (builtin ? builtin.cacheRead / builtin.input : 1),
cacheWrite: input * (builtin ? builtin.cacheWrite / builtin.input : 1)
}
}
/**
* The rate a workspace should be billed at for one model: its override when an
* admin set one, otherwise the published list price, otherwise nothing. An override
* that omits a cache rate takes it from `inheritedCacheRates`.
*/
export function resolveModelPrice(
provider: AIProvider | string,
model: string,
overrides: Record<string, ModelPriceOverride> | undefined
): ResolvedModelPrice | undefined {
const builtin = getKnownModelPrice(model)
const candidate = overrides?.[modelKey(provider, model)]
const override =
candidate &&
isUsableRate(candidate.input) &&
isUsableRate(candidate.output) &&
isUsableRate(candidate.cache_read) &&
isUsableRate(candidate.cache_write)
? candidate
: undefined
if (override) {
const inherited = inheritedCacheRates(model, override.input)
return {
source: 'override',
price: {
input: override.input,
output: override.output,
cacheRead: override.cache_read ?? inherited.cacheRead,
cacheWrite: override.cache_write ?? inherited.cacheWrite
}
}
}
return builtin ? { source: 'builtin', price: builtin } : undefined
}
/** Cost in USD of `tokens` at `price`. */
export function estimateCost(tokens: PricedTokens, price: ModelPrice): number {
return (
(tokens.input * price.input +
tokens.cacheRead * price.cacheRead +
tokens.cacheWrite * price.cacheWrite +
tokens.output * price.output) /
1_000_000
)
}
/** Tokens spent on one model, from a chat's running totals or the usage API. */
export type ModelSpend = {
provider: string
model: string
tokens: PricedTokens
/** What the provider billed, where it reports a figure. */
reportedCostUsd?: number
}
export type Priced = {
/** Undefined when no rate is known for the model — reported as unpriced, never guessed. */
cost: number | undefined
source: ModelPriceSource | 'reported' | undefined
}
export type PricedSpend<T extends ModelSpend> = {
/** The input entries, each with its cost — callers carry their own fields through
* rather than zipping the result back against the input by index. */
rows: (T & Priced)[]
total: number
/** True when at least one row has no rate, so `total` understates the truth. */
hasUnpriced: boolean
/** True when at least one row is a figure the provider billed rather than an estimate. */
hasReported: boolean
}
/**
* Cost a set of per-model token counts. A provider-reported figure always wins:
* it is what was actually charged, where everything else is list price times
* tokens. `source` says which, so a view never presents an estimate as a bill.
*/
export function priceSpend<T extends ModelSpend>(
spend: T[],
overrides: Record<string, ModelPriceOverride> | undefined
): PricedSpend<T> {
let total = 0
let hasUnpriced = false
let hasReported = false
const rows = spend.map((entry): T & Priced => {
if (entry.reportedCostUsd !== undefined) {
hasReported = true
total += entry.reportedCostUsd
return { ...entry, cost: entry.reportedCostUsd, source: 'reported' }
}
const resolved = resolveModelPrice(entry.provider, entry.model, overrides)
if (!resolved) {
hasUnpriced = true
return { ...entry, cost: undefined, source: undefined }
}
const cost = estimateCost(entry.tokens, resolved.price)
total += cost
return { ...entry, cost, source: resolved.source }
})
return { rows, total, hasUnpriced, hasReported }
}
/**
* Money, at the precision the amount deserves: sub-cent spend is where a chat
* spends most of its life, and rounding it to `$0.00` would read as free.
*/
export function formatUsd(amount: number): string {
if (amount === 0) return '$0'
if (amount < 0.01) return `$${amount.toFixed(4)}`
if (amount < 1) return `$${amount.toFixed(3)}`
return `$${amount.toFixed(2)}`
}
@@ -5,9 +5,11 @@
type AIConfig,
type AIProvider,
type GetCopilotSettingsStateResponse,
type InstanceAISummary
type InstanceAISummary,
type ModelPriceOverride
} from '$lib/gen'
import { workspaceStore } from '$lib/stores'
import { copilotInfo } from '$lib/aiStore'
import { sendUserToast } from '$lib/toast'
import { AI_PROVIDERS, fetchAvailableModels, providerSupportsWebSearch } from '../copilot/lib'
import { supportsAutocomplete } from '../copilot/utils'
@@ -25,6 +27,8 @@
import Badge from '../common/badge/Badge.svelte'
import Tooltip from '../Tooltip.svelte'
import ModelTokenLimits from './ModelTokenLimits.svelte'
import ModelPricing from './ModelPricing.svelte'
import AiUsagePanel from './AiUsagePanel.svelte'
import { setCopilotInfo } from '$lib/aiStore'
import AIPromptsModal from '../settings/AIPromptsModal.svelte'
import { Settings } from 'lucide-svelte'
@@ -73,6 +77,7 @@
let metadataModel: string | undefined = $state(undefined)
let customPrompts: Record<string, string> = $state({})
let maxTokensPerModel: Record<string, number> = $state({})
let modelPricing: Record<string, ModelPriceOverride> = $state({})
let usingOpenaiClientCredentialsOauth = $state(false)
let workspaceOverrideEditorOpened = $state(false)
@@ -83,6 +88,7 @@
let initialMetadataModel: string | undefined = $state(undefined)
let initialCustomPrompts: Record<string, string> = $state({})
let initialMaxTokensPerModel: Record<string, number> = $state({})
let initialModelPricing: Record<string, ModelPriceOverride> = $state({})
let initialPrompts: Record<string, string> = $state({})
let lastLoadedConfigKey = $state<string | undefined>(undefined)
@@ -110,6 +116,7 @@
codeCompletionModel = config?.code_completion_model?.model
customPrompts = clone(config?.custom_prompts ?? {})
maxTokensPerModel = clone(config?.max_tokens_per_model ?? {})
modelPricing = clone(config?.model_pricing ?? {})
for (const mode of ['edit', 'fix', 'gen']) {
if (!(mode in customPrompts)) {
customPrompts[mode] = ''
@@ -124,6 +131,7 @@
initialCodeCompletionModel = codeCompletionModel
initialCustomPrompts = clone(customPrompts)
initialMaxTokensPerModel = clone(maxTokensPerModel)
initialModelPricing = clone(modelPricing)
initialPrompts = clone(customPrompts)
}
@@ -139,6 +147,7 @@
codeCompletionModel = initialCodeCompletionModel
customPrompts = clone(initialCustomPrompts)
maxTokensPerModel = clone(initialMaxTokensPerModel)
modelPricing = clone(initialModelPricing)
}
$effect(() => {
@@ -172,7 +181,8 @@
metadataModel !== initialMetadataModel ||
codeCompletionModel !== initialCodeCompletionModel ||
JSON.stringify(customPrompts) !== JSON.stringify(initialCustomPrompts) ||
JSON.stringify(maxTokensPerModel) !== JSON.stringify(initialMaxTokensPerModel)
JSON.stringify(maxTokensPerModel) !== JSON.stringify(initialMaxTokensPerModel) ||
JSON.stringify(modelPricing) !== JSON.stringify(initialModelPricing)
)
$effect(() => {
@@ -285,7 +295,8 @@
metadata_model,
custom_prompts: Object.keys(custom_prompts).length > 0 ? custom_prompts : undefined,
max_tokens_per_model:
Object.keys(maxTokensPerModel).length > 0 ? maxTokensPerModel : undefined
Object.keys(maxTokensPerModel).length > 0 ? maxTokensPerModel : undefined,
model_pricing: Object.keys(modelPricing).length > 0 ? modelPricing : undefined
}
: {}
}
@@ -610,6 +621,24 @@
scope={promptScope}
/>
{#if promptScope === 'workspace'}
<!-- Recorded usage must be priced with the rates the chats actually ran under.
A workspace on instance defaults has no rates of its own, so the effective
ones come from copilotInfo rather than from this form's (empty) workspace
config. -->
<AiUsagePanel
workspace={effectiveWorkspace}
modelPricing={usesInstanceAiConfig ? ($copilotInfo.modelPricing ?? {}) : modelPricing}
/>
{/if}
<!-- Below the usage it explains: the rates are read as a correction to what the
table above already shows. Kept on its own `showWorkspaceOverrideEditor` gate so
the instance scope, which has no usage panel, still edits rates. -->
{#if showWorkspaceOverrideEditor}
<ModelPricing {aiProviders} bind:modelPricing />
{/if}
{#if showWorkspaceOverrideEditor}
<SettingsFooter
hasUnsavedChanges={dirty}
@@ -0,0 +1,226 @@
<script lang="ts">
import { AiService, ApiError, type AITokenUsageBucket, type ModelPriceOverride } from '$lib/gen'
import { formatUsd, priceSpend, type ModelSpend } from '../copilot/modelPricing'
import { formatTokenCount } from '../copilot/chat/tokenUsage'
import SettingCard from '../instanceSettings/SettingCard.svelte'
import Select from '../select/Select.svelte'
import ToggleButtonGroup from '../common/toggleButton-v2/ToggleButtonGroup.svelte'
import ToggleButton from '../common/toggleButton-v2/ToggleButton.svelte'
import { resource } from 'runed'
import Tooltip from '../meltComponents/Tooltip.svelte'
import DataTable from '../table/DataTable.svelte'
import Head from '../table/Head.svelte'
import Cell from '../table/Cell.svelte'
// Workspace and rates are both passed in rather than read from a store: the
// settings component that mounts this one also serves the instance scope, and
// the rates that priced a chat are the workspace's *effective* ones, which an
// inheriting workspace does not hold itself.
let {
workspace,
modelPricing,
scope = 'workspace'
}: {
workspace: string
modelPricing: Record<string, ModelPriceOverride>
scope?: 'workspace' | 'self'
} = $props()
type GroupBy = 'day' | 'user' | 'model'
let days = $state(30)
let groupBy = $state<GroupBy>('day')
const rangeOptions = [
{ label: 'Last 7 days', value: 7 },
{ label: 'Last 30 days', value: 30 },
{ label: 'Last 90 days', value: 90 }
]
let usage = resource(
() => ({ workspace, days, groupBy, scope }),
async ({ workspace, days, groupBy, scope }) =>
workspace ? await AiService.listAiUsage({ workspace, days, groupBy, scope }) : undefined
)
// The API groups by (dimension, provider, model) so every bucket resolves to a
// single rate; the table folds those back into one line per dimension value.
type Bucket = ModelSpend & { key: string; requests: number }
function toSpend(bucket: AITokenUsageBucket): Bucket {
return {
// Grouping by model has no separate dimension — the model is the key.
key: groupBy === 'model' ? `${bucket.provider}/${bucket.model}` : bucket.key || '—',
requests: bucket.requests,
provider: bucket.provider,
model: bucket.model,
tokens: {
input: bucket.input_tokens,
cacheRead: bucket.cache_read_tokens,
cacheWrite: bucket.cache_write_tokens,
output: bucket.output_tokens
},
reportedCostUsd:
bucket.reported_cost_nano_usd != undefined
? bucket.reported_cost_nano_usd / 1_000_000_000
: undefined
}
}
let priced = $derived(priceSpend((usage.current?.buckets ?? []).map(toSpend), modelPricing))
type Row = {
key: string
cost: number | undefined
/** Every model behind this line was billed back by its provider, so the
* figure is an invoice rather than an estimate. A line mixing sources — or
* one holding a model with no rate, whose spend the figure omits entirely —
* makes the weaker claim. */
reported: boolean
tokensIn: number
tokensOut: number
requests: number
}
// Only a 403 on the workspace scope is a permission problem; reading your own
// usage is open to any member. Attributing every failure to permissions sends an
// admin looking for access they already hold, and buries the real cause of the
// far more common transient ones (an expired session, a database hiccup).
function usageError(error: unknown): string {
if (scope === 'workspace' && error instanceof ApiError && error.status === 403) {
return 'Only workspace admins can read workspace usage.'
}
return 'Could not load usage. Try again in a moment.'
}
// The headline sums both kinds, so it only escapes the ~ when nothing under it
// was estimated.
let totalIsEstimated = $derived(
priced.rows.some((row) => row.cost !== undefined && row.source !== 'reported')
)
let rows = $derived.by(() => {
const byKey = new Map<string, Row>()
for (const row of priced.rows) {
const existing = byKey.get(row.key) ?? {
key: row.key,
cost: undefined,
reported: true,
tokensIn: 0,
tokensOut: 0,
requests: 0
}
existing.tokensIn += row.tokens.input + row.tokens.cacheRead + row.tokens.cacheWrite
existing.tokensOut += row.tokens.output
existing.requests += row.requests
if (row.cost !== undefined) {
existing.cost = (existing.cost ?? 0) + row.cost
}
existing.reported &&= row.source === 'reported'
byKey.set(row.key, existing)
}
return [...byKey.values()].sort((a, b) => (b.cost ?? 0) - (a.cost ?? 0))
})
</script>
<SettingCard
label={scope === 'self' ? `Your AI usage in ${workspace}` : 'AI usage'}
description={scope === 'self'
? "Token spend from your own AI chats in this workspace. Costs are estimated from the workspace's model rates unless the provider reported one."
: "Token spend across this workspace's AI chats, grouped by day, user or model. Costs are estimated from the workspace's effective model rates unless the provider reported one."}
>
<div class="flex flex-col gap-3">
<div class="flex flex-row items-center justify-between gap-3 flex-wrap">
<div class="flex flex-row items-center gap-2 flex-wrap">
<div class="w-40">
<Select items={rangeOptions} bind:value={days} />
</div>
<ToggleButtonGroup noWFull bind:selected={groupBy}>
{#snippet children({ item })}
<ToggleButton value="day" label="By day" {item} />
{#if scope !== 'self'}
<ToggleButton value="user" label="By user" {item} />
{/if}
<ToggleButton value="model" label="By model" {item} />
{/snippet}
</ToggleButtonGroup>
</div>
{#if !usage.loading && !usage.error && rows.length > 0}
<div
class="flex flex-row items-baseline gap-2"
title={priced.hasUnpriced
? 'Models with no rate set are not counted, so this figure is lower than the real spend.'
: undefined}
>
<span class="text-lg font-semibold tabular-nums">
{priced.total === 0 && priced.hasUnpriced
? '—'
: `${totalIsEstimated ? '~' : ''}${formatUsd(priced.total)}`}
</span>
<span class="text-xs text-tertiary">
{usage.current?.truncated ? 'across the rows below' : 'total'}{priced.hasUnpriced
? ' (partial)'
: ''}
</span>
</div>
{/if}
</div>
{#if usage.loading}
<p class="text-xs text-tertiary">Loading…</p>
{:else if usage.error}
<p class="text-xs text-tertiary">{usageError(usage.error)}</p>
{:else if rows.length === 0}
<p class="text-xs text-tertiary">No AI usage recorded in this period.</p>
{:else}
{#if usage.current?.truncated}
<p class="text-xs text-tertiary">
More rows matched than are shown; the highest-volume ones are listed. Narrow the range
or group differently to see the rest.
</p>
{/if}
<DataTable size="sm" noBorder={false} rounded={true}>
<Head>
<tr>
<Cell head first>{groupBy}</Cell>
<Cell head numeric>In</Cell>
<Cell head numeric>Out</Cell>
<Cell head numeric>Requests</Cell>
<Cell head numeric last>
<span class="inline-flex flex-row items-center gap-1">
Cost
<Tooltip small placement="left">
{#snippet text()}
A cost marked ~ is estimated from this workspace's model rates. A cost
without one was returned by the provider's API for those requests, and is used
as is. "no rate" means the model has no price set, so its spend stays out of
the total.
{/snippet}
</Tooltip>
</span>
</Cell>
</tr>
</Head>
<tbody>
{#each rows as row (row.key)}
<tr class="border-b last:border-b-0">
<Cell first class="font-mono truncate max-w-xs text-primary">{row.key}</Cell>
<Cell numeric class="tabular-nums text-secondary"
>{formatTokenCount(row.tokensIn)}</Cell
>
<Cell numeric class="tabular-nums text-secondary"
>{formatTokenCount(row.tokensOut)}</Cell
>
<Cell numeric class="tabular-nums text-secondary">{row.requests}</Cell>
<Cell numeric last class="tabular-nums text-primary">
{row.cost === undefined
? 'no rate'
: `${row.reported ? '' : '~'}${formatUsd(row.cost)}`}
</Cell>
</tr>
{/each}
</tbody>
</DataTable>
{/if}
</div>
</SettingCard>
@@ -0,0 +1,252 @@
<script lang="ts">
import type { AIConfig, AIProvider, ModelPriceOverride } from '$lib/gen'
import { Badge, Button } from '../common'
import TextInput from '../text_input/TextInput.svelte'
import { getKnownModelPrice, inheritedCacheRates } from '../copilot/modelPricing'
import { modelKey } from '../copilot/modelConfig'
import { stripLegacyThinkingSuffix } from '../copilot/reasoningRegistry'
import { ChevronDown, ChevronUp } from 'lucide-svelte'
import { slide } from 'svelte/transition'
import SettingCard from '../instanceSettings/SettingCard.svelte'
// A rate above this is far more likely a unit mistake (per-token instead of
// per-million) than a real price, and a wrong rate silently inflates every
// figure derived from it.
const MAX_RATE = 1000
let {
aiProviders,
modelPricing = $bindable()
}: {
aiProviders: Exclude<AIConfig['providers'], undefined>
modelPricing: Record<string, ModelPriceOverride>
} = $props()
let errors = $state<Record<string, string>>({})
let collapsedProviders = $state<Record<string, boolean>>({})
// Rates are keyed by the id the chat reports usage under, and `setCopilotInfo`
// strips the deprecated `/thinking` suffix before the chat ever sees a model. A
// row built from the raw config would save an override under a key nothing
// reports, so it would sit in settings looking applied and never price anything.
// Stripping can collapse two configured slots onto one model, hence the dedupe.
const modelsByProvider = $derived(
Object.entries(aiProviders).reduce(
(acc, [provider, config]) => {
const seen = new Set<string>()
acc[provider] = config.models.flatMap((configured) => {
const model = stripLegacyThinkingSuffix(configured)
if (seen.has(model)) {
return []
}
seen.add(model)
return [{ provider: provider as AIProvider, model }]
})
return acc
},
{} as Record<string, Array<{ provider: AIProvider; model: string }>>
)
)
type Field = 'input' | 'output' | 'cache_read' | 'cache_write'
type CacheField = Extract<Field, 'cache_read' | 'cache_write'>
type Rates = { input: number; output: number; cache_read?: number; cache_write?: number }
const RATE_FIELDS: Field[] = ['input', 'output', 'cache_read', 'cache_write']
// Show what a blank cache rate falls back to, so the inherited figure is visible
// rather than implied. Read from the resolver's own helper: a placeholder that
// computed the rule separately would drift from the price actually charged.
function inheritedCacheRate(model: string, field: Field, rates: Rates | undefined): string {
if (field !== 'cache_read' && field !== 'cache_write') return '—'
const input = rates?.input
if (input === undefined) return '—'
const inherited = inheritedCacheRates(model, input)
return `${+(field === 'cache_read' ? inherited.cacheRead : inherited.cacheWrite).toFixed(4)}`
}
function currentRates(provider: AIProvider, model: string): Rates | undefined {
const override = modelPricing[modelKey(provider, model)]
if (override) {
return {
input: override.input,
output: override.output,
cache_read: override.cache_read,
cache_write: override.cache_write
}
}
const builtin = getKnownModelPrice(model)
return builtin ? { input: builtin.input, output: builtin.output } : undefined
}
function isOverridden(provider: AIProvider, model: string): boolean {
return modelPricing[modelKey(provider, model)] !== undefined
}
function updateRate(provider: AIProvider, model: string, field: Field, value: number) {
const key = modelKey(provider, model)
if (!(value >= 0) || value > MAX_RATE) {
errors[key] = `Rate must be between 0 and ${MAX_RATE}`
return
}
// An edit to either field pins both: a half-specified override would leave
// the other rate silently tracking a built-in price the admin did not choose.
const current = currentRates(provider, model) ?? { input: 0, output: 0 }
modelPricing = {
...modelPricing,
[key]: { ...modelPricing[key], input: current.input, output: current.output, [field]: value }
}
errors[key] = ''
}
// Emptying a cache field means "inherit again", so the key has to go: leaving the
// old number in the override would keep charging it while the field shows the
// inherited placeholder. Input and output are required, so an empty one is not a
// state the override can hold; the input snaps back to the stored value on blur.
function clearCacheRate(provider: AIProvider, model: string, field: CacheField) {
const key = modelKey(provider, model)
const override = modelPricing[key]
if (!override || override[field] === undefined) {
return
}
const next = { ...override }
delete next[field]
modelPricing = { ...modelPricing, [key]: next }
errors[key] = ''
}
function resetModel(provider: AIProvider, model: string) {
const key = modelKey(provider, model)
const next = { ...modelPricing }
delete next[key]
modelPricing = next
errors[key] = ''
}
function toggleProvider(provider: string) {
collapsedProviders[provider] = !collapsedProviders[provider]
}
function hasOverrides(provider: string, models: Array<{ model: string }>): boolean {
return models.some((m) => isOverridden(provider as AIProvider, m.model))
}
$effect(() => {
collapsedProviders = {
...Object.fromEntries(Object.keys(aiProviders).map((provider) => [provider, true]))
}
})
</script>
{#if Object.keys(aiProviders).length > 0}
<SettingCard
label="Model pricing"
description="Rates in USD per million tokens, used to cost AI chat usage. Built-in list prices are a best-effort snapshot you can adjust."
tooltip="Rates apply only where the provider returned no cost of its own; a cost it returned is used as is. A model with no built-in price starts empty and reports usage without a cost until you set one. An empty cache rate uses the figure shown in the field, and a rate of 0 prices those tokens as free."
>
<div class="flex flex-col gap-3">
{#each Object.entries(modelsByProvider).filter(([_, models]) => models.length > 0) as [provider, models]}
{@const isExpanded = !collapsedProviders[provider]}
<div class="border rounded-md bg-surface-tertiary">
<Button
variant="subtle"
unifiedSize="sm"
onclick={() => toggleProvider(provider)}
wrapperClasses="w-full"
btnClasses="w-full px-4 min-h-8 justify-between rounded-t-md rounded-b-none"
endIcon={{ icon: isExpanded ? ChevronUp : ChevronDown }}
>
<div class="flex items-center gap-2">
<h4 class="font-medium text-xs capitalize">{provider}</h4>
{#if hasOverrides(provider, models)}
<Badge color="blue">Modified</Badge>
{/if}
</div>
</Button>
{#if isExpanded}
<div transition:slide|local={{ duration: 200 }} class="p-4 border-t">
<div class="space-y-3">
{#each models as { model }}
{@const key = modelKey(provider as AIProvider, model)}
{@const rates = currentRates(provider as AIProvider, model)}
{@const overridden = isOverridden(provider as AIProvider, model)}
<div class="flex flex-col gap-1">
<div class="flex items-center gap-3 flex-wrap">
<!-- Floor the name's width so it keeps a readable share and the rate
fields wrap below it, rather than the name collapsing to an
ellipsis on a narrow panel. -->
<div class="flex-1 min-w-[10rem]">
<span class="text-xs text-primary truncate block">{model}</span>
</div>
<div class="flex items-center gap-3 flex-wrap">
{#each RATE_FIELDS as field}
<div class="flex items-center gap-1">
<span class="text-xs text-secondary whitespace-nowrap">
{field.replace('_', ' ')}
</span>
<div class="w-24">
<TextInput
value={rates?.[field] ?? ''}
size="sm"
error={!!errors[key]}
inputProps={{
type: 'number',
min: 0,
max: MAX_RATE,
step: 0.01,
placeholder: inheritedCacheRate(model, field, rates),
oninput: (e: Event & { currentTarget: HTMLInputElement }) => {
if (e.currentTarget.value === '') {
if (field === 'cache_read' || field === 'cache_write') {
clearCacheRate(provider as AIProvider, model, field)
}
return
}
const value = parseFloat(e.currentTarget.value)
if (!isNaN(value)) {
updateRate(provider as AIProvider, model, field, value)
}
},
onblur: (e: Event & { currentTarget: HTMLInputElement }) => {
// Resync a field the state refused, so what is shown is what is stored.
const stored = currentRates(provider as AIProvider, model)?.[
field
]
e.currentTarget.value =
stored === undefined ? '' : String(stored)
errors[key] = ''
}
}}
/>
</div>
</div>
{/each}
<span class="text-xs text-secondary whitespace-nowrap">$ / 1M</span>
</div>
</div>
{#if overridden}
<div class="text-xs text-tertiary flex flex-row items-center gap-2">
<span>Overriding the built-in price</span>
<button
type="button"
class="text-xs text-blue-500 hover:underline"
onclick={() => resetModel(provider as AIProvider, model)}
>
Reset
</button>
</div>
{/if}
{#if errors[key]}
<div class="text-xs text-red-500">{errors[key]}</div>
{/if}
</div>
{/each}
</div>
</div>
{/if}
</div>
{/each}
</div>
</SettingCard>
{/if}
+154
View File
@@ -0,0 +1,154 @@
import { get } from 'svelte/store'
import { OpenAPI } from '$lib/gen'
import { workspaceStore } from '$lib/stores'
// Per-workspace AI token spend, batched into the backend `ai_token_usage`
// accumulator that powers the workspace and per-user usage views.
//
// Deliberately separate from `featureUsage.ts`: that buffer carries anonymous
// product telemetry that leaves the instance, and its events must not identify a
// user. These events are attributed to the caller (server-side, from the session)
// and never leave the instance, so the two must not share a transport.
//
// Only token counts are sent. Money is derived when the usage is read, from the
// price table plus the workspace's overrides, so correcting a rate also corrects
// history. The one exception is a cost the provider itself billed back.
export interface AiUsageEvent {
provider: string
model: string
/** Empty for chats not attached to an AI session. */
sessionId?: string
inputTokens: number
cacheReadTokens: number
cacheWriteTokens: number
outputTokens: number
/** Set only where the provider reports what it actually charged, in USD. */
costUsd?: number
/** Workspace whose API route carries the batch; defaults to the active workspace. */
workspace?: string
}
interface AiUsageEventPayload {
provider: string
model: string
session_id: string
input_tokens: number
cache_read_tokens: number
cache_write_tokens: number
output_tokens: number
reported_cost_nano_usd?: number
requests: number
}
const FLUSH_INTERVAL_MS = 15_000
// Backend caps a batch at 50 events; chunk larger flushes.
const MAX_EVENTS_PER_REQUEST = 50
const NANO_USD_PER_USD = 1_000_000_000
// One accumulator per (workspace, provider, model, session): a chat that sends
// several turns before a flush produces one upsert instead of one per turn.
const pending = new Map<string, { workspace: string; event: AiUsageEventPayload }>()
let timer: ReturnType<typeof setTimeout> | undefined
/**
* Record AI token spend. Fire-and-forget: events are summed locally and flushed
* in batches.
*/
export function logAiUsage(event: AiUsageEvent): void {
const workspace = event.workspace ?? get(workspaceStore) ?? undefined
if (!workspace) return
const sessionId = event.sessionId ?? ''
const mapKey = JSON.stringify([workspace, event.provider, event.model, sessionId])
const existing = pending.get(mapKey)?.event
const target: AiUsageEventPayload = existing ?? {
provider: event.provider,
model: event.model,
session_id: sessionId,
input_tokens: 0,
cache_read_tokens: 0,
cache_write_tokens: 0,
output_tokens: 0,
requests: 0
}
target.input_tokens += Math.max(0, Math.round(event.inputTokens))
target.cache_read_tokens += Math.max(0, Math.round(event.cacheReadTokens))
target.cache_write_tokens += Math.max(0, Math.round(event.cacheWriteTokens))
target.output_tokens += Math.max(0, Math.round(event.outputTokens))
target.requests += 1
if (event.costUsd !== undefined) {
target.reported_cost_nano_usd =
(target.reported_cost_nano_usd ?? 0) +
Math.max(0, Math.round(event.costUsd * NANO_USD_PER_USD))
}
pending.set(mapKey, { workspace, event: target })
if (timer === undefined) {
timer = setTimeout(() => {
timer = undefined
void flushAiUsage()
}, FLUSH_INTERVAL_MS)
}
}
export async function flushAiUsage(): Promise<void> {
if (timer !== undefined) {
clearTimeout(timer)
timer = undefined
}
if (pending.size === 0) return
const byWorkspace = new Map<string, AiUsageEventPayload[]>()
for (const { workspace, event } of pending.values()) {
let events = byWorkspace.get(workspace)
if (!events) {
events = []
byWorkspace.set(workspace, events)
}
events.push(event)
}
pending.clear()
// Start every chunk request synchronously before awaiting: the pagehide flush
// only protects requests that were already issued (keepalive can't help a fetch
// that never started).
const inflight: Promise<void>[] = []
for (const [workspace, events] of byWorkspace) {
for (let i = 0; i < events.length; i += MAX_EVENTS_PER_REQUEST) {
inflight.push(send(workspace, events.slice(i, i + MAX_EVENTS_PER_REQUEST)))
}
}
await Promise.all(inflight)
}
async function send(workspace: string, events: AiUsageEventPayload[]): Promise<void> {
try {
// Raw fetch instead of the generated client: `keepalive` lets the request
// finish after tab close/navigation, which is when the final flush runs.
// Auth rides on the token cookie (WITH_CREDENTIALS app setup).
await fetch(`${OpenAPI.BASE}/w/${encodeURIComponent(workspace)}/ai/usage`, {
method: 'POST',
credentials: 'include',
keepalive: true,
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ events })
})
} catch {
// Accounting is best-effort: a dropped batch under-reports spend, which is
// better than surfacing a network error in the middle of a chat.
}
}
if (typeof document !== 'undefined') {
// Flush what's buffered before the tab goes away. pagehide covers
// close/navigation paths where visibilitychange is not delivered.
document.addEventListener('visibilitychange', () => {
if (document.visibilityState === 'hidden') {
void flushAiUsage()
}
})
window.addEventListener('pagehide', () => {
void flushAiUsage()
})
}