mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-08-25 00:01:55 +00:00
feat: track token cost in AI sessions and chats (#10688)
* feat: track token cost in AI sessions and chats * fix: address review findings on AI cost tracking * fix: price inherited and overridden models at their real rates * fix: stop newer model revisions inheriting an older price * fix: stop a sub-model inheriting its family's price * fix: keep alias suffixes resolving to their model's price * fix: count OpenRouter cache writes and drop unverifiable rates * refactor: move AI spend out of the chat into workspace and user settings * fix: pin the usage workspace per turn and stop inventing cache rates * fix: leave Sonnet 5 unpriced while its promotional rate runs * docs: record the new table in the schema summary and tighten comments * fix: mark estimated AI costs with ~ and drop session grouping Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * fix: name the workspace in the self-scoped AI usage title Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * docs: state that overrides never replace a provider-returned cost Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * fix: let a cleared cache rate inherit again and flag partial totals Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * fix: clear a refused rate's error when the input snaps back Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * fix: stop a revision variant inheriting its base family's rate Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * fix: report AI usage before tools run and price self usage consistently Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * fix: key pricing rows on the model id usage is reported under Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * fix: surface Bedrock and Gemini usage the chat proxy was dropping Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * fix: count Gemini tool-use prompt tokens as input Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * feat: price flat-rate Gemini Flash models Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * docs: state the tool-use token invariant once Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
+24
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO ai_token_usage (workspace_id, email, provider, model, session_id, input_tokens, cache_read_tokens, cache_write_tokens, output_tokens, reported_cost_nano_usd, requests)\n SELECT $1, $2, * FROM UNNEST($3::text[], $4::text[], $5::text[], $6::bigint[], $7::bigint[], $8::bigint[], $9::bigint[], $10::bigint[], $11::bigint[])\n ON CONFLICT (workspace_id, day, email, provider, model, session_id)\n DO UPDATE SET\n input_tokens = ai_token_usage.input_tokens + EXCLUDED.input_tokens,\n cache_read_tokens = ai_token_usage.cache_read_tokens + EXCLUDED.cache_read_tokens,\n cache_write_tokens = ai_token_usage.cache_write_tokens + EXCLUDED.cache_write_tokens,\n output_tokens = ai_token_usage.output_tokens + EXCLUDED.output_tokens,\n reported_cost_nano_usd = CASE\n WHEN EXCLUDED.reported_cost_nano_usd IS NULL\n THEN ai_token_usage.reported_cost_nano_usd\n ELSE COALESCE(ai_token_usage.reported_cost_nano_usd, 0)\n + EXCLUDED.reported_cost_nano_usd\n END,\n requests = ai_token_usage.requests + EXCLUDED.requests,\n updated_at = now()",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "24fcc2b69f30953915f0cbf246e1c19b2310075a644d81c21784c991e52b4001"
|
||||
}
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT\n (CASE $3::text\n WHEN 'day' THEN day::text\n WHEN 'user' THEN email\n ELSE ''\n END) AS \"key!\",\n provider AS \"provider!\",\n model AS \"model!\",\n SUM(input_tokens)::bigint AS \"input_tokens!\",\n SUM(cache_read_tokens)::bigint AS \"cache_read_tokens!\",\n SUM(cache_write_tokens)::bigint AS \"cache_write_tokens!\",\n SUM(output_tokens)::bigint AS \"output_tokens!\",\n SUM(reported_cost_nano_usd)::bigint AS \"reported_cost_nano_usd\",\n SUM(requests)::bigint AS \"requests!\"\n FROM ai_token_usage\n WHERE workspace_id = $1 AND day > CURRENT_DATE - $2::int\n AND ($5::text IS NULL OR email = $5)\n GROUP BY 1, provider, model\n ORDER BY SUM(input_tokens + cache_read_tokens + cache_write_tokens + output_tokens) DESC\n LIMIT $4",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "key!",
|
||||
"type_info": "Text"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "provider!",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "model!",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "input_tokens!",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "cache_read_tokens!",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "cache_write_tokens!",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"name": "output_tokens!",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 7,
|
||||
"name": "reported_cost_nano_usd",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 8,
|
||||
"name": "requests!",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Int4",
|
||||
"Text",
|
||||
"Int8",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null,
|
||||
false,
|
||||
false,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "6a17a1dfeb75808e5d0726f1f8bd573168161abe350b047402cda4ac4a4d13e4"
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
DROP TABLE ai_token_usage;
|
||||
@@ -0,0 +1,42 @@
|
||||
-- Per-workspace AI token spend, accumulated from the chat client. Rows hold token
|
||||
-- counts rather than money: prices live in the frontend price table plus the
|
||||
-- workspace's `ai_config.model_pricing` overrides and are applied at read time, so
|
||||
-- correcting a price also corrects the history. `reported_cost_nano_usd` is the
|
||||
-- exception — a few providers (OpenRouter) return what they actually charged, and
|
||||
-- that figure wins over the estimate.
|
||||
--
|
||||
-- Distinct from `feature_usage`, which is anonymous telemetry that leaves the
|
||||
-- instance and is pruned after 60 days; spend is per-user and kept.
|
||||
CREATE TABLE ai_token_usage (
|
||||
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE,
|
||||
day DATE NOT NULL DEFAULT CURRENT_DATE,
|
||||
email VARCHAR(255) NOT NULL,
|
||||
provider VARCHAR(50) NOT NULL,
|
||||
model VARCHAR(255) NOT NULL,
|
||||
-- Empty for chats that are not attached to an AI session.
|
||||
session_id VARCHAR(50) NOT NULL DEFAULT '',
|
||||
-- Uncached input only; the two cache columns hold the rest of the prompt, so
|
||||
-- each column maps to exactly one price and they never double-count.
|
||||
input_tokens BIGINT NOT NULL DEFAULT 0,
|
||||
cache_read_tokens BIGINT NOT NULL DEFAULT 0,
|
||||
cache_write_tokens BIGINT NOT NULL DEFAULT 0,
|
||||
output_tokens BIGINT NOT NULL DEFAULT 0,
|
||||
reported_cost_nano_usd BIGINT,
|
||||
requests BIGINT NOT NULL DEFAULT 0,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (workspace_id, day, email, provider, model, session_id)
|
||||
);
|
||||
|
||||
-- The usage listing filters on workspace and a date range; the PK only reaches
|
||||
-- `day` through `email`, so it cannot serve that on its own.
|
||||
CREATE INDEX idx_ai_token_usage_ws_day ON ai_token_usage (workspace_id, day DESC);
|
||||
|
||||
GRANT ALL ON ai_token_usage TO windmill_admin;
|
||||
GRANT ALL ON ai_token_usage TO windmill_user;
|
||||
|
||||
-- Both handlers go through the raw pool, so no policy is needed for them to work.
|
||||
-- Enabling RLS with an admin-only policy is the backstop: a future query that
|
||||
-- reaches this table through UserDB sees nothing rather than every user's spend.
|
||||
ALTER TABLE ai_token_usage ENABLE ROW LEVEL SECURITY;
|
||||
|
||||
CREATE POLICY admin_policy ON ai_token_usage FOR ALL TO windmill_admin USING (true);
|
||||
@@ -38,6 +38,8 @@ account: workspace_id(char), id(int), expires_at(ts), refresh_token(char), clien
|
||||
FK: (workspace_id) -> workspace(id)
|
||||
agent_token_blacklist: token(char), expires_at(ts), blacklisted_at(ts), blacklisted_by(char)
|
||||
ai_agent_memory: workspace_id(char), conversation_id(uuid), step_id(char), messages(jsonb), created_at(ts), updated_at(ts)
|
||||
ai_token_usage: workspace_id(char), day(date), email(char), provider(char), model(char), session_id(char), input_tokens(bigint), cache_read_tokens(bigint), cache_write_tokens(bigint), output_tokens(bigint), reported_cost_nano_usd(bigint), requests(bigint), updated_at(ts)
|
||||
FK: (workspace_id) -> workspace(id)
|
||||
alerts: id(int), alert_type(char), message(text), created_at(ts), acknowledged(bool), workspace_id(text), acknowledged_workspace(bool), resource(text)
|
||||
app: id(bigint), workspace_id(char), path(char), summary(char), policy(jsonb), versions(bigint[]), extra_perms(jsonb), draft_only(bool), custom_path(text), labels(text[])
|
||||
FK: (workspace_id) -> workspace(id)
|
||||
|
||||
@@ -277,7 +277,7 @@ pub struct GeminiSSECandidate {
|
||||
}
|
||||
|
||||
/// Token usage from the `usageMetadata` field of a Gemini SSE event.
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
#[derive(Deserialize, Debug, Clone, Default)]
|
||||
pub struct GeminiUsageMetadata {
|
||||
#[serde(rename = "promptTokenCount", default)]
|
||||
pub prompt_token_count: Option<i32>,
|
||||
@@ -285,6 +285,39 @@ pub struct GeminiUsageMetadata {
|
||||
pub candidates_token_count: Option<i32>,
|
||||
#[serde(rename = "totalTokenCount", default)]
|
||||
pub total_token_count: Option<i32>,
|
||||
/// Subset of `promptTokenCount` served from context cache, billed at a reduced
|
||||
/// rate. Reported separately so the client can price it separately.
|
||||
#[serde(rename = "cachedContentTokenCount", default)]
|
||||
pub cached_content_token_count: Option<i32>,
|
||||
/// Thinking tokens, billed as output but counted apart from `candidatesTokenCount`.
|
||||
#[serde(rename = "thoughtsTokenCount", default)]
|
||||
pub thoughts_token_count: Option<i32>,
|
||||
/// Input tokens spent on tool-use prompts, counted apart from `promptTokenCount`
|
||||
/// rather than within it.
|
||||
#[serde(rename = "toolUsePromptTokenCount", default)]
|
||||
pub tool_use_prompt_token_count: Option<i32>,
|
||||
}
|
||||
|
||||
/// Input tokens as billed. Gemini reports tool-use prompts in their own field, and
|
||||
/// they are disjoint from `promptTokenCount`: a live tool call returns 17 prompt +
|
||||
/// 60 tool-use + 17 candidates + 52 thoughts against a `totalTokenCount` of 146, so
|
||||
/// leaving them out under-reports the input of every tool-using turn. Cached tokens
|
||||
/// are not added here, being already part of `promptTokenCount`.
|
||||
fn gemini_prompt_tokens(usage: &GeminiUsageMetadata) -> i32 {
|
||||
usage
|
||||
.prompt_token_count
|
||||
.unwrap_or(0)
|
||||
.saturating_add(usage.tool_use_prompt_token_count.unwrap_or(0))
|
||||
}
|
||||
|
||||
/// Output tokens as billed: Gemini counts thinking apart from `candidatesTokenCount`
|
||||
/// but charges it at the output rate, so a reply that thought would otherwise be
|
||||
/// reported as far cheaper than it was.
|
||||
fn gemini_completion_tokens(usage: &GeminiUsageMetadata) -> i32 {
|
||||
usage
|
||||
.candidates_token_count
|
||||
.unwrap_or(0)
|
||||
.saturating_add(usage.thoughts_token_count.unwrap_or(0))
|
||||
}
|
||||
|
||||
/// Top-level structure of one Gemini SSE event.
|
||||
@@ -588,9 +621,12 @@ pub fn gemini_response_to_openai(parsed: &GeminiParsedEvent, model: &str) -> ser
|
||||
|
||||
let usage = parsed.usage.as_ref().map(|u| {
|
||||
serde_json::json!({
|
||||
"prompt_tokens": u.prompt_token_count.unwrap_or(0),
|
||||
"completion_tokens": u.candidates_token_count.unwrap_or(0),
|
||||
"prompt_tokens": gemini_prompt_tokens(u),
|
||||
"completion_tokens": gemini_completion_tokens(u),
|
||||
"total_tokens": u.total_token_count.unwrap_or(0),
|
||||
"prompt_tokens_details": {
|
||||
"cached_tokens": u.cached_content_token_count.unwrap_or(0)
|
||||
},
|
||||
})
|
||||
});
|
||||
|
||||
@@ -680,8 +716,8 @@ pub fn gemini_event_to_openai_sse_chunks(
|
||||
// OpenAI's `stream_options.include_usage` terminal chunk (top-level `usage`,
|
||||
// empty `choices`) so the frontend's `'usage' in chunk` path records them.
|
||||
if let Some(usage) = &parsed.usage {
|
||||
let prompt_tokens = usage.prompt_token_count.unwrap_or(0);
|
||||
let completion_tokens = usage.candidates_token_count.unwrap_or(0);
|
||||
let prompt_tokens = gemini_prompt_tokens(usage);
|
||||
let completion_tokens = gemini_completion_tokens(usage);
|
||||
let total_tokens = usage
|
||||
.total_token_count
|
||||
.unwrap_or(prompt_tokens + completion_tokens);
|
||||
@@ -694,6 +730,9 @@ pub fn gemini_event_to_openai_sse_chunks(
|
||||
"prompt_tokens": prompt_tokens,
|
||||
"completion_tokens": completion_tokens,
|
||||
"total_tokens": total_tokens,
|
||||
"prompt_tokens_details": {
|
||||
"cached_tokens": usage.cached_content_token_count.unwrap_or(0)
|
||||
},
|
||||
}
|
||||
});
|
||||
chunks.push(format!("data: {}\n\n", chunk));
|
||||
@@ -943,6 +982,7 @@ mod tests {
|
||||
prompt_token_count: Some(12),
|
||||
candidates_token_count: Some(7),
|
||||
total_token_count: Some(19),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
@@ -969,6 +1009,79 @@ mod tests {
|
||||
assert_eq!(usage_chunk["choices"], serde_json::json!([]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gemini_usage_chunk_splits_cached_and_bills_thoughts() {
|
||||
let parsed = GeminiParsedEvent {
|
||||
text: Some("the answer".to_string()),
|
||||
usage: Some(GeminiUsageMetadata {
|
||||
prompt_token_count: Some(1000),
|
||||
candidates_token_count: Some(20),
|
||||
total_token_count: Some(1120),
|
||||
cached_content_token_count: Some(900),
|
||||
thoughts_token_count: Some(100),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let mut tool_call_index = 0;
|
||||
let chunks = gemini_event_to_openai_sse_chunks(
|
||||
&parsed,
|
||||
"chatcmpl-test",
|
||||
"gemini-3-flash-preview",
|
||||
&mut tool_call_index,
|
||||
);
|
||||
let usage_chunk = chunks
|
||||
.iter()
|
||||
.map(|c| parse_sse_chunk(c))
|
||||
.find(|v| v.get("usage").map(|u| !u.is_null()).unwrap_or(false))
|
||||
.expect("a chunk should carry top-level usage");
|
||||
|
||||
// Gemini's prompt count already includes the cached tokens, so it passes
|
||||
// through unchanged and the cached share is reported alongside it; thinking
|
||||
// is billed as output but counted apart from the candidates.
|
||||
assert_eq!(usage_chunk["usage"]["prompt_tokens"], 1000);
|
||||
assert_eq!(usage_chunk["usage"]["prompt_tokens_details"]["cached_tokens"], 900);
|
||||
assert_eq!(usage_chunk["usage"]["completion_tokens"], 120);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gemini_usage_chunk_counts_tool_use_prompt_tokens() {
|
||||
let parsed = GeminiParsedEvent {
|
||||
text: Some("Canberra".to_string()),
|
||||
usage: Some(GeminiUsageMetadata {
|
||||
prompt_token_count: Some(17),
|
||||
candidates_token_count: Some(17),
|
||||
total_token_count: Some(146),
|
||||
tool_use_prompt_token_count: Some(60),
|
||||
thoughts_token_count: Some(52),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let mut tool_call_index = 0;
|
||||
let chunks = gemini_event_to_openai_sse_chunks(
|
||||
&parsed,
|
||||
"chatcmpl-test",
|
||||
"gemini-2.5-flash",
|
||||
&mut tool_call_index,
|
||||
);
|
||||
let usage_chunk = chunks
|
||||
.iter()
|
||||
.map(|c| parse_sse_chunk(c))
|
||||
.find(|v| v.get("usage").map(|u| !u.is_null()).unwrap_or(false))
|
||||
.expect("a chunk should carry top-level usage");
|
||||
|
||||
assert_eq!(usage_chunk["usage"]["prompt_tokens"], 77);
|
||||
assert_eq!(usage_chunk["usage"]["completion_tokens"], 69);
|
||||
assert_eq!(
|
||||
usage_chunk["usage"]["prompt_tokens"].as_i64().unwrap()
|
||||
+ usage_chunk["usage"]["completion_tokens"].as_i64().unwrap(),
|
||||
146
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gemini_streaming_usage_total_falls_back_to_prompt_plus_completion() {
|
||||
let parsed = GeminiParsedEvent {
|
||||
@@ -976,6 +1089,7 @@ mod tests {
|
||||
prompt_token_count: Some(5),
|
||||
candidates_token_count: Some(3),
|
||||
total_token_count: None,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -175,3 +175,57 @@ pub struct OpenAIMessage {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub annotations: Option<Vec<UrlCitation>>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Model pricing
|
||||
// ============================================================================
|
||||
|
||||
/// Far above any real per-million-token rate, so a value beyond it is a unit
|
||||
/// mistake rather than a price. The floor matters more: a negative rate would make
|
||||
/// spend subtract, and NaN/infinity would poison every total derived from it.
|
||||
pub const MAX_MODEL_RATE: f64 = 1000.0;
|
||||
|
||||
/// Bound the `model_pricing` map of an AI config that is only available untyped —
|
||||
/// the instance config is stored through the generic global-settings endpoint,
|
||||
/// which never deserializes it into `AIConfig`, so the typed check on the
|
||||
/// workspace path does not cover it.
|
||||
pub fn validate_model_pricing_json(ai_config: &serde_json::Value) -> Result<(), String> {
|
||||
// The container itself has to be checked too: a non-object `ai_config` persists
|
||||
// here and then fails to deserialize as `AIConfig`, which drops the whole
|
||||
// instance config back to its default for every workspace inheriting it.
|
||||
if !ai_config.is_null() && !ai_config.is_object() {
|
||||
return Err("ai_config must be an object".to_string());
|
||||
}
|
||||
let pricing = match ai_config.get("model_pricing") {
|
||||
None | Some(serde_json::Value::Null) => return Ok(()),
|
||||
// A present-but-wrong shape must be rejected, not skipped: it would persist
|
||||
// and then fail to deserialize as `AIConfig`, which silently drops the whole
|
||||
// instance config back to its default for every workspace inheriting it.
|
||||
Some(v) => v
|
||||
.as_object()
|
||||
.ok_or_else(|| "model_pricing must be an object".to_string())?,
|
||||
};
|
||||
for (key, price) in pricing {
|
||||
let Some(price) = price.as_object() else {
|
||||
return Err(format!("Price override for {} is not an object", key));
|
||||
};
|
||||
for field in ["input", "output", "cache_read", "cache_write"] {
|
||||
let Some(rate) = price.get(field) else { continue };
|
||||
let rate = rate
|
||||
.as_f64()
|
||||
.filter(|r| r.is_finite() && *r >= 0.0 && *r <= MAX_MODEL_RATE);
|
||||
if rate.is_none() {
|
||||
return Err(format!(
|
||||
"Price override for {}: {} must be between 0 and {}",
|
||||
key, field, MAX_MODEL_RATE
|
||||
));
|
||||
}
|
||||
}
|
||||
for required in ["input", "output"] {
|
||||
if !price.contains_key(required) {
|
||||
return Err(format!("Price override for {} is missing {}", key, required));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -660,6 +660,41 @@ fn bedrock_sse_chunks_for_event(
|
||||
chunks.push(Bytes::from(format!("data: {}\n\n", chunk)));
|
||||
}
|
||||
|
||||
// Usage arrives only on the trailing Metadata event, and only this converter
|
||||
// reaches the chat: without a chunk for it a Bedrock chat reports no tokens at
|
||||
// all. Bedrock counts cache reads and writes apart from `inputTokens`, while the
|
||||
// OpenAI shape the client parses treats `prompt_tokens` as the whole input, so
|
||||
// they are folded in here and split back out through `prompt_tokens_details`.
|
||||
if let aws_sdk_bedrockruntime::types::ConverseStreamOutput::Metadata(metadata) = event {
|
||||
if let Some(token_usage) = metadata.usage() {
|
||||
let cache_read = token_usage.cache_read_input_tokens().unwrap_or(0);
|
||||
let cache_write = token_usage.cache_write_input_tokens().unwrap_or(0);
|
||||
let prompt_tokens = token_usage
|
||||
.input_tokens()
|
||||
.saturating_add(cache_read)
|
||||
.saturating_add(cache_write);
|
||||
|
||||
let chunk = serde_json::json!({
|
||||
"id": state.id,
|
||||
"object": "chat.completion.chunk",
|
||||
"created": state.created,
|
||||
"model": state.model,
|
||||
"choices": [],
|
||||
"usage": {
|
||||
"prompt_tokens": prompt_tokens,
|
||||
"completion_tokens": token_usage.output_tokens(),
|
||||
"total_tokens": token_usage.total_tokens(),
|
||||
"prompt_tokens_details": {
|
||||
"cached_tokens": cache_read,
|
||||
"cache_write_tokens": cache_write
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
chunks.push(Bytes::from(format!("data: {}\n\n", chunk)));
|
||||
}
|
||||
}
|
||||
|
||||
chunks
|
||||
}
|
||||
|
||||
@@ -1190,6 +1225,43 @@ mod tests {
|
||||
serde_json::from_str(payload).expect("chunk should contain JSON")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_event_emits_usage_chunk_with_cache_split() {
|
||||
let mut state = BedrockSseStreamState::new("id".to_string(), "model".to_string(), 0);
|
||||
let event = ConverseStreamOutput::Metadata(
|
||||
aws_sdk_bedrockruntime::types::ConverseStreamMetadataEvent::builder()
|
||||
.usage(
|
||||
aws_sdk_bedrockruntime::types::TokenUsage::builder()
|
||||
.input_tokens(10)
|
||||
.output_tokens(7)
|
||||
.total_tokens(1017)
|
||||
.cache_read_input_tokens(900)
|
||||
.cache_write_input_tokens(100)
|
||||
.build()
|
||||
.expect("usage"),
|
||||
)
|
||||
.build(),
|
||||
);
|
||||
|
||||
let chunks = bedrock_sse_chunks_for_event(&event, &mut state);
|
||||
let usage = chunks
|
||||
.iter()
|
||||
.map(sse_json)
|
||||
.find(|v| v.get("usage").map(|u| !u.is_null()).unwrap_or(false))
|
||||
.expect("the metadata event should carry usage");
|
||||
|
||||
// Bedrock reports cache reads and writes apart from `inputTokens`; the OpenAI
|
||||
// shape the client parses treats `prompt_tokens` as the whole input, and
|
||||
// recovers the uncached share by subtracting the details back out.
|
||||
assert_eq!(usage["usage"]["prompt_tokens"], 1010);
|
||||
assert_eq!(usage["usage"]["completion_tokens"], 7);
|
||||
assert_eq!(usage["usage"]["prompt_tokens_details"]["cached_tokens"], 900);
|
||||
assert_eq!(
|
||||
usage["usage"]["prompt_tokens_details"]["cache_write_tokens"],
|
||||
100
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn determine_auth_config_prioritizes_bearer_token() {
|
||||
let config = determine_auth_config(
|
||||
|
||||
@@ -891,6 +891,13 @@ async fn run_setting_pre_write_hook(
|
||||
value: &serde_json::Value,
|
||||
) -> error::Result<()> {
|
||||
match key {
|
||||
// The instance AI config is written as an untyped blob through this generic
|
||||
// endpoint, so it never passes the typed check the workspace handler applies.
|
||||
// Rates that reach a cost total unbounded would make it negative or infinite.
|
||||
AI_CONFIG_SETTING => {
|
||||
windmill_ai::ai_types::validate_model_pricing_json(value)
|
||||
.map_err(error::Error::BadRequest)?;
|
||||
}
|
||||
AUTOMATE_USERNAME_CREATION_SETTING => {
|
||||
if value.as_bool().unwrap_or(false) {
|
||||
generate_instance_username_for_all_users(db)
|
||||
|
||||
@@ -11986,6 +11986,73 @@ paths:
|
||||
schema:
|
||||
type: string
|
||||
|
||||
/w/{workspace}/ai/usage:
|
||||
post:
|
||||
summary: record AI token usage for the calling user
|
||||
operationId: recordAiUsage
|
||||
tags:
|
||||
- ai
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
required:
|
||||
- events
|
||||
properties:
|
||||
events:
|
||||
type: array
|
||||
items:
|
||||
$ref: "#/components/schemas/AITokenUsageEvent"
|
||||
responses:
|
||||
"204":
|
||||
description: usage recorded
|
||||
get:
|
||||
summary: list aggregated AI token usage
|
||||
operationId: listAiUsage
|
||||
tags:
|
||||
- ai
|
||||
parameters:
|
||||
- $ref: "#/components/parameters/WorkspaceId"
|
||||
- name: days
|
||||
in: query
|
||||
schema:
|
||||
type: integer
|
||||
minimum: 1
|
||||
maximum: 365
|
||||
- name: group_by
|
||||
in: query
|
||||
schema:
|
||||
type: string
|
||||
enum: [day, user, model]
|
||||
- name: scope
|
||||
in: query
|
||||
description: workspace-wide usage (admin only) or the calling user's own
|
||||
schema:
|
||||
type: string
|
||||
enum: [workspace, self]
|
||||
responses:
|
||||
"200":
|
||||
description: usage buckets
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
required:
|
||||
- buckets
|
||||
- truncated
|
||||
properties:
|
||||
buckets:
|
||||
type: array
|
||||
items:
|
||||
$ref: "#/components/schemas/AITokenUsageBucket"
|
||||
truncated:
|
||||
type: boolean
|
||||
description: more buckets matched than were returned, so summing them under-reports
|
||||
|
||||
/w/{workspace}/ai_skills/list:
|
||||
get:
|
||||
summary: list the workspace AI chat skills (name + description only)
|
||||
@@ -26300,6 +26367,92 @@ components:
|
||||
type: integer
|
||||
minimum: 1
|
||||
maximum: 2000000
|
||||
model_pricing:
|
||||
type: object
|
||||
additionalProperties:
|
||||
$ref: "#/components/schemas/ModelPriceOverride"
|
||||
|
||||
ModelPriceOverride:
|
||||
type: object
|
||||
description: negotiated rates in USD per million tokens, keyed `provider:model`
|
||||
properties:
|
||||
input:
|
||||
type: number
|
||||
minimum: 0
|
||||
maximum: 1000
|
||||
output:
|
||||
type: number
|
||||
minimum: 0
|
||||
maximum: 1000
|
||||
cache_read:
|
||||
type: number
|
||||
minimum: 0
|
||||
maximum: 1000
|
||||
cache_write:
|
||||
type: number
|
||||
minimum: 0
|
||||
maximum: 1000
|
||||
required:
|
||||
- input
|
||||
- output
|
||||
|
||||
AITokenUsageEvent:
|
||||
type: object
|
||||
properties:
|
||||
provider:
|
||||
$ref: "#/components/schemas/AIProvider"
|
||||
model:
|
||||
type: string
|
||||
session_id:
|
||||
type: string
|
||||
input_tokens:
|
||||
type: integer
|
||||
cache_read_tokens:
|
||||
type: integer
|
||||
cache_write_tokens:
|
||||
type: integer
|
||||
output_tokens:
|
||||
type: integer
|
||||
reported_cost_nano_usd:
|
||||
type: integer
|
||||
description: only set by providers that bill back an exact figure
|
||||
requests:
|
||||
type: integer
|
||||
required:
|
||||
- provider
|
||||
- model
|
||||
|
||||
AITokenUsageBucket:
|
||||
type: object
|
||||
properties:
|
||||
key:
|
||||
type: string
|
||||
description: the grouped dimension's value; empty when grouping by model
|
||||
provider:
|
||||
type: string
|
||||
model:
|
||||
type: string
|
||||
input_tokens:
|
||||
type: integer
|
||||
cache_read_tokens:
|
||||
type: integer
|
||||
cache_write_tokens:
|
||||
type: integer
|
||||
output_tokens:
|
||||
type: integer
|
||||
reported_cost_nano_usd:
|
||||
type: integer
|
||||
requests:
|
||||
type: integer
|
||||
required:
|
||||
- key
|
||||
- provider
|
||||
- model
|
||||
- input_tokens
|
||||
- cache_read_tokens
|
||||
- cache_write_tokens
|
||||
- output_tokens
|
||||
- requests
|
||||
|
||||
InstanceAIProviderSummary:
|
||||
type: object
|
||||
|
||||
@@ -3,11 +3,16 @@ use crate::utils::check_scopes;
|
||||
|
||||
#[cfg(feature = "bedrock")]
|
||||
use axum::routing::get;
|
||||
#[cfg(feature = "bedrock")]
|
||||
use axum::Json;
|
||||
use axum::{body::Bytes, extract::Path, response::IntoResponse, routing::post, Extension, Router};
|
||||
use axum::{
|
||||
body::Bytes,
|
||||
extract::{DefaultBodyLimit, Path, Query},
|
||||
response::IntoResponse,
|
||||
routing::post,
|
||||
Extension, Router,
|
||||
};
|
||||
use futures::StreamExt;
|
||||
use http::{HeaderMap, Method};
|
||||
use http::{HeaderMap, Method, StatusCode};
|
||||
use quick_cache::sync::Cache;
|
||||
use reqwest::{Client, RequestBuilder};
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -18,6 +23,7 @@ use windmill_ai::ai_cache::current_instance_ai_config_revision;
|
||||
use windmill_ai::ai_providers::{
|
||||
empty_string_as_none, AIPlatform, AIProvider, ProviderConfig, ProviderModel,
|
||||
};
|
||||
use windmill_ai::ai_types::MAX_MODEL_RATE;
|
||||
use windmill_ai::credentials::ProviderCredentials;
|
||||
#[cfg(feature = "bedrock")]
|
||||
use windmill_ai::providers::bedrock::{
|
||||
@@ -37,7 +43,7 @@ use windmill_ai::proxy::{
|
||||
use windmill_audit::{audit_oss::audit_log, ActionKind};
|
||||
use windmill_common::db::UserDB;
|
||||
use windmill_common::error::{to_anyhow, Error, Result};
|
||||
use windmill_common::utils::configure_client;
|
||||
use windmill_common::utils::{configure_client, require_admin};
|
||||
use windmill_common::variables::{get_variable_or_self, get_variable_or_self_as};
|
||||
|
||||
// AI timeout configuration constants
|
||||
@@ -417,9 +423,54 @@ pub struct AIConfig {
|
||||
pub custom_prompts: Option<HashMap<String, String>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub max_tokens_per_model: Option<HashMap<String, i32>>,
|
||||
/// Per-model price overrides, keyed `provider:model` like `max_tokens_per_model`.
|
||||
/// Only models whose rates differ from the built-in table are stored.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub model_pricing: Option<HashMap<String, ModelPriceOverride>>,
|
||||
}
|
||||
|
||||
/// Negotiated rates in USD per million tokens. An unset cache rate is read as the
|
||||
/// provider's own multiple of the input rate where the model has a published one,
|
||||
/// and as the input rate itself where it does not — an unstated discount is never
|
||||
/// filled in from another vendor's.
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct ModelPriceOverride {
|
||||
pub input: f64,
|
||||
pub output: f64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub cache_read: Option<f64>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub cache_write: Option<f64>,
|
||||
}
|
||||
|
||||
impl ModelPriceOverride {
|
||||
pub fn validate(&self, key: &str) -> Result<()> {
|
||||
for (field, rate) in [
|
||||
("input", Some(self.input)),
|
||||
("output", Some(self.output)),
|
||||
("cache_read", self.cache_read),
|
||||
("cache_write", self.cache_write),
|
||||
] {
|
||||
let Some(rate) = rate else { continue };
|
||||
if !rate.is_finite() || rate < 0.0 || rate > MAX_MODEL_RATE {
|
||||
return Err(Error::BadRequest(format!(
|
||||
"Price override for {}: {} must be between 0 and {}",
|
||||
key, field, MAX_MODEL_RATE
|
||||
)));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl AIConfig {
|
||||
pub fn validate_model_pricing(&self) -> Result<()> {
|
||||
for (key, price) in self.model_pricing.iter().flatten() {
|
||||
price.validate(key)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn has_providers(&self) -> bool {
|
||||
self.providers
|
||||
.as_ref()
|
||||
@@ -432,7 +483,18 @@ pub fn global_service() -> Router {
|
||||
}
|
||||
|
||||
pub fn workspaced_service() -> Router {
|
||||
let router = Router::new().route("/proxy/{*ai}", post(proxy).get(proxy));
|
||||
let router = Router::new()
|
||||
.route("/proxy/{*ai}", post(proxy).get(proxy))
|
||||
.route(
|
||||
"/usage",
|
||||
post(record_ai_usage)
|
||||
.get(list_ai_usage)
|
||||
// The handler caps how many events it *stores*, but Json deserializes
|
||||
// the whole array first — without a body limit an authenticated member
|
||||
// could make the server allocate and parse an arbitrarily large one.
|
||||
// Sized well above a full batch of the shape below.
|
||||
.layer(DefaultBodyLimit::max(AI_USAGE_BODY_LIMIT)),
|
||||
);
|
||||
|
||||
#[cfg(feature = "bedrock")]
|
||||
let router = router.route("/check_bedrock_credentials", get(check_bedrock_credentials));
|
||||
@@ -440,6 +502,265 @@ pub fn workspaced_service() -> Router {
|
||||
router
|
||||
}
|
||||
|
||||
/// One provider request's worth of tokens, as counted by the chat client.
|
||||
#[derive(Deserialize)]
|
||||
struct AIUsageEvent {
|
||||
provider: String,
|
||||
model: String,
|
||||
#[serde(default)]
|
||||
session_id: String,
|
||||
#[serde(default)]
|
||||
input_tokens: i64,
|
||||
#[serde(default)]
|
||||
cache_read_tokens: i64,
|
||||
#[serde(default)]
|
||||
cache_write_tokens: i64,
|
||||
#[serde(default)]
|
||||
output_tokens: i64,
|
||||
/// Only the providers that bill back an exact figure set this.
|
||||
#[serde(default)]
|
||||
reported_cost_nano_usd: Option<i64>,
|
||||
#[serde(default)]
|
||||
requests: Option<i64>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct RecordAIUsagePayload {
|
||||
events: Vec<AIUsageEvent>,
|
||||
}
|
||||
|
||||
const MAX_AI_USAGE_EVENTS: usize = 50;
|
||||
/// 64 KiB — a 50-event batch is a few kB even with the longest model ids.
|
||||
const AI_USAGE_BODY_LIMIT: usize = 64 * 1024;
|
||||
/// Well above any single conversation and far below an i64 overflow, so a client
|
||||
/// bug caps out at one absurd row instead of poisoning the running total.
|
||||
const MAX_TOKENS_PER_EVENT: i64 = 100_000_000;
|
||||
/// $1000 in nano-USD.
|
||||
const MAX_REPORTED_COST_PER_EVENT: i64 = 1_000_000_000_000;
|
||||
|
||||
/// Model ids carry vendor prefixes and variant suffixes (`anthropic/claude-opus-5:thinking`),
|
||||
/// so the shape check is looser than an identifier but still excludes whitespace and
|
||||
/// anything that would not be a model id.
|
||||
fn is_model_shaped(s: &str, max_len: usize) -> bool {
|
||||
!s.is_empty()
|
||||
&& s.len() <= max_len
|
||||
&& s.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | ':' | '.' | '/' | '~'))
|
||||
}
|
||||
|
||||
/// Accumulate one workspace's AI token spend. Values are clamped and the caller's
|
||||
/// email comes from the session, never the payload — the client is trusted to
|
||||
/// report its own usage, not to attribute it to someone else.
|
||||
async fn record_ai_usage(
|
||||
authed: ApiAuthed,
|
||||
Extension(db): Extension<DB>,
|
||||
Path(w_id): Path<String>,
|
||||
Json(payload): Json<RecordAIUsagePayload>,
|
||||
) -> Result<StatusCode> {
|
||||
// Pre-sum duplicate keys: two rows hitting the same conflict target in a single
|
||||
// INSERT error out ("cannot affect row a second time").
|
||||
let mut agg: HashMap<(String, String, String), AIUsageTotals> = HashMap::new();
|
||||
for e in payload.events.into_iter().take(MAX_AI_USAGE_EVENTS) {
|
||||
if AIProvider::try_from(e.provider.as_str()).is_err()
|
||||
|| !is_model_shaped(&e.model, 255)
|
||||
|| !(e.session_id.is_empty() || is_model_shaped(&e.session_id, 50))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let totals = agg
|
||||
.entry((e.provider, e.model, e.session_id))
|
||||
.or_insert_with(AIUsageTotals::default);
|
||||
totals.input += e.input_tokens.clamp(0, MAX_TOKENS_PER_EVENT);
|
||||
totals.cache_read += e.cache_read_tokens.clamp(0, MAX_TOKENS_PER_EVENT);
|
||||
totals.cache_write += e.cache_write_tokens.clamp(0, MAX_TOKENS_PER_EVENT);
|
||||
totals.output += e.output_tokens.clamp(0, MAX_TOKENS_PER_EVENT);
|
||||
totals.requests += e.requests.unwrap_or(1).clamp(0, MAX_AI_USAGE_EVENTS as i64);
|
||||
if let Some(cost) = e.reported_cost_nano_usd {
|
||||
totals.reported_cost = Some(
|
||||
totals.reported_cost.unwrap_or(0) + cost.clamp(0, MAX_REPORTED_COST_PER_EVENT),
|
||||
);
|
||||
}
|
||||
}
|
||||
if agg.is_empty() {
|
||||
return Ok(StatusCode::NO_CONTENT);
|
||||
}
|
||||
|
||||
let mut providers = Vec::with_capacity(agg.len());
|
||||
let mut models = Vec::with_capacity(agg.len());
|
||||
let mut session_ids = Vec::with_capacity(agg.len());
|
||||
let mut inputs = Vec::with_capacity(agg.len());
|
||||
let mut cache_reads = Vec::with_capacity(agg.len());
|
||||
let mut cache_writes = Vec::with_capacity(agg.len());
|
||||
let mut outputs = Vec::with_capacity(agg.len());
|
||||
let mut reported_costs: Vec<Option<i64>> = Vec::with_capacity(agg.len());
|
||||
let mut requests = Vec::with_capacity(agg.len());
|
||||
for ((provider, model, session_id), totals) in agg {
|
||||
providers.push(provider);
|
||||
models.push(model);
|
||||
session_ids.push(session_id);
|
||||
inputs.push(totals.input);
|
||||
cache_reads.push(totals.cache_read);
|
||||
cache_writes.push(totals.cache_write);
|
||||
outputs.push(totals.output);
|
||||
reported_costs.push(totals.reported_cost);
|
||||
requests.push(totals.requests);
|
||||
}
|
||||
|
||||
sqlx::query!(
|
||||
"INSERT INTO ai_token_usage (workspace_id, email, provider, model, session_id, \
|
||||
input_tokens, cache_read_tokens, cache_write_tokens, output_tokens, \
|
||||
reported_cost_nano_usd, requests)
|
||||
SELECT $1, $2, * FROM UNNEST($3::text[], $4::text[], $5::text[], $6::bigint[], \
|
||||
$7::bigint[], $8::bigint[], $9::bigint[], $10::bigint[], $11::bigint[])
|
||||
ON CONFLICT (workspace_id, day, email, provider, model, session_id)
|
||||
DO UPDATE SET
|
||||
input_tokens = ai_token_usage.input_tokens + EXCLUDED.input_tokens,
|
||||
cache_read_tokens = ai_token_usage.cache_read_tokens + EXCLUDED.cache_read_tokens,
|
||||
cache_write_tokens = ai_token_usage.cache_write_tokens + EXCLUDED.cache_write_tokens,
|
||||
output_tokens = ai_token_usage.output_tokens + EXCLUDED.output_tokens,
|
||||
reported_cost_nano_usd = CASE
|
||||
WHEN EXCLUDED.reported_cost_nano_usd IS NULL
|
||||
THEN ai_token_usage.reported_cost_nano_usd
|
||||
ELSE COALESCE(ai_token_usage.reported_cost_nano_usd, 0)
|
||||
+ EXCLUDED.reported_cost_nano_usd
|
||||
END,
|
||||
requests = ai_token_usage.requests + EXCLUDED.requests,
|
||||
updated_at = now()",
|
||||
&w_id,
|
||||
&authed.email,
|
||||
&providers,
|
||||
&models,
|
||||
&session_ids,
|
||||
&inputs,
|
||||
&cache_reads,
|
||||
&cache_writes,
|
||||
&outputs,
|
||||
&reported_costs as &[Option<i64>],
|
||||
&requests
|
||||
)
|
||||
.execute(&db)
|
||||
.await?;
|
||||
|
||||
Ok(StatusCode::NO_CONTENT)
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct AIUsageTotals {
|
||||
input: i64,
|
||||
cache_read: i64,
|
||||
cache_write: i64,
|
||||
output: i64,
|
||||
reported_cost: Option<i64>,
|
||||
requests: i64,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ListAIUsageQuery {
|
||||
days: Option<i32>,
|
||||
group_by: Option<String>,
|
||||
scope: Option<String>,
|
||||
}
|
||||
|
||||
/// A bucket always carries its provider and model: the caller prices it from a
|
||||
/// per-model rate table, which a bucket spanning several models could not be
|
||||
/// resolved against.
|
||||
#[derive(Serialize)]
|
||||
struct AITokenUsageBucket {
|
||||
key: String,
|
||||
provider: String,
|
||||
model: String,
|
||||
input_tokens: i64,
|
||||
cache_read_tokens: i64,
|
||||
cache_write_tokens: i64,
|
||||
output_tokens: i64,
|
||||
reported_cost_nano_usd: Option<i64>,
|
||||
requests: i64,
|
||||
}
|
||||
|
||||
/// Grouping by day over a long range, or by model across many models, can produce
|
||||
/// more buckets than a table is worth rendering, so the listing is capped.
|
||||
/// `truncated` says so explicitly — a caller that sums the rows into a total must be
|
||||
/// able to tell that the total is partial rather than silently under-reporting spend.
|
||||
#[derive(Serialize)]
|
||||
struct AITokenUsageListing {
|
||||
buckets: Vec<AITokenUsageBucket>,
|
||||
truncated: bool,
|
||||
}
|
||||
|
||||
const AI_USAGE_MAX_BUCKETS: i64 = 1000;
|
||||
|
||||
async fn list_ai_usage(
|
||||
authed: ApiAuthed,
|
||||
Extension(db): Extension<DB>,
|
||||
Path(w_id): Path<String>,
|
||||
Query(query): Query<ListAIUsageQuery>,
|
||||
) -> Result<Json<AITokenUsageListing>> {
|
||||
// Reading the whole workspace's spend is an admin view; reading your own is
|
||||
// not, so a member can see what they are costing without being shown their
|
||||
// colleagues'. The filter is the session's email, never a parameter.
|
||||
let own_email = match query.scope.as_deref().unwrap_or("workspace") {
|
||||
"workspace" => {
|
||||
require_admin(authed.is_admin, &authed.username)?;
|
||||
None
|
||||
}
|
||||
"self" => Some(authed.email.clone()),
|
||||
scope => return Err(Error::BadRequest(format!("Unsupported scope: {}", scope))),
|
||||
};
|
||||
|
||||
let days = query.days.unwrap_or(30).clamp(1, 365);
|
||||
let group_by = query.group_by.as_deref().unwrap_or("day");
|
||||
// No `session`: a session is identified by a client-generated id whose name
|
||||
// lives only in the browser that made it, so a bucket keyed on one is a label
|
||||
// nobody can resolve. `session_id` is still stored, at the grain the client
|
||||
// batches on, should sessions ever gain a server-side name.
|
||||
if !matches!(group_by, "day" | "user" | "model") {
|
||||
return Err(Error::BadRequest(format!(
|
||||
"Unsupported group_by: {}",
|
||||
group_by
|
||||
)));
|
||||
}
|
||||
|
||||
// Fetch one past the cap to detect truncation. Ordering is by token volume, not
|
||||
// by cost: rates are applied by the caller, so this query cannot know what a
|
||||
// bucket cost. Volume is the closest proxy available here, and the caller is told
|
||||
// the listing was capped rather than being left to sum a partial set silently.
|
||||
let mut rows = sqlx::query_as!(
|
||||
AITokenUsageBucket,
|
||||
r#"SELECT
|
||||
(CASE $3::text
|
||||
WHEN 'day' THEN day::text
|
||||
WHEN 'user' THEN email
|
||||
ELSE ''
|
||||
END) AS "key!",
|
||||
provider AS "provider!",
|
||||
model AS "model!",
|
||||
SUM(input_tokens)::bigint AS "input_tokens!",
|
||||
SUM(cache_read_tokens)::bigint AS "cache_read_tokens!",
|
||||
SUM(cache_write_tokens)::bigint AS "cache_write_tokens!",
|
||||
SUM(output_tokens)::bigint AS "output_tokens!",
|
||||
SUM(reported_cost_nano_usd)::bigint AS "reported_cost_nano_usd",
|
||||
SUM(requests)::bigint AS "requests!"
|
||||
FROM ai_token_usage
|
||||
WHERE workspace_id = $1 AND day > CURRENT_DATE - $2::int
|
||||
AND ($5::text IS NULL OR email = $5)
|
||||
GROUP BY 1, provider, model
|
||||
ORDER BY SUM(input_tokens + cache_read_tokens + cache_write_tokens + output_tokens) DESC
|
||||
LIMIT $4"#,
|
||||
&w_id,
|
||||
days,
|
||||
group_by,
|
||||
AI_USAGE_MAX_BUCKETS + 1,
|
||||
own_email.as_deref()
|
||||
)
|
||||
.fetch_all(&db)
|
||||
.await?;
|
||||
|
||||
let truncated = rows.len() as i64 > AI_USAGE_MAX_BUCKETS;
|
||||
rows.truncate(AI_USAGE_MAX_BUCKETS as usize);
|
||||
|
||||
Ok(Json(AITokenUsageListing { buckets: rows, truncated }))
|
||||
}
|
||||
|
||||
/// Check if AWS Bedrock credentials are available from environment variables.
|
||||
#[cfg(feature = "bedrock")]
|
||||
async fn check_bedrock_credentials(
|
||||
|
||||
@@ -108,6 +108,8 @@ async fn edit_copilot_config(
|
||||
}
|
||||
}
|
||||
|
||||
ai_config.validate_model_pricing()?;
|
||||
|
||||
let mut tx = db.begin().await?;
|
||||
|
||||
sqlx::query!(
|
||||
|
||||
@@ -3,7 +3,12 @@
|
||||
// import aiStore back, and such a cycle crashes the app once the bundler splits it across
|
||||
// chunks (docs/frontend-import-cycles.md; the build fails on the chunk cycle, not on this).
|
||||
import { writable, get } from 'svelte/store'
|
||||
import { type AIProviderModel, type AIProvider, type AIConfig } from './gen'
|
||||
import {
|
||||
type AIProviderModel,
|
||||
type AIProvider,
|
||||
type AIConfig,
|
||||
type ModelPriceOverride
|
||||
} from './gen'
|
||||
import {
|
||||
aiUserDisabled,
|
||||
COPILOT_SESSION_MODEL_SETTING_NAME,
|
||||
@@ -41,6 +46,8 @@ export const copilotInfo = writable<{
|
||||
aiModels: AIProviderModel[]
|
||||
customPrompts?: Record<string, string>
|
||||
maxTokensPerModel?: Record<string, number>
|
||||
/** Negotiated rates per `provider:model`, overriding the built-in price table. */
|
||||
modelPricing?: Record<string, ModelPriceOverride>
|
||||
webSearchEnabledProviders?: Partial<Record<AIProvider, boolean>>
|
||||
}>({
|
||||
enabled: false,
|
||||
@@ -50,6 +57,7 @@ export const copilotInfo = writable<{
|
||||
aiModels: [],
|
||||
customPrompts: {},
|
||||
maxTokensPerModel: {},
|
||||
modelPricing: {},
|
||||
webSearchEnabledProviders: {}
|
||||
})
|
||||
|
||||
@@ -124,6 +132,7 @@ export function setCopilotInfo(aiConfig: AIConfig) {
|
||||
aiModels: aiModels,
|
||||
customPrompts: aiConfig.custom_prompts ?? {},
|
||||
maxTokensPerModel: aiConfig.max_tokens_per_model ?? {},
|
||||
modelPricing: aiConfig.model_pricing ?? {},
|
||||
webSearchEnabledProviders
|
||||
})
|
||||
} else {
|
||||
@@ -137,6 +146,7 @@ export function setCopilotInfo(aiConfig: AIConfig) {
|
||||
aiModels: [],
|
||||
customPrompts: {},
|
||||
maxTokensPerModel: {},
|
||||
modelPricing: {},
|
||||
webSearchEnabledProviders: {}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -9,6 +9,8 @@
|
||||
import { createEventDispatcher } from 'svelte'
|
||||
import UserInfoSettings from './settings/UserInfoSettings.svelte'
|
||||
import AIUserSettings from './settings/AIUserSettings.svelte'
|
||||
import AiUsagePanel from './workspaceSettings/AiUsagePanel.svelte'
|
||||
import { copilotInfo, copilotWorkspace } from '$lib/aiStore'
|
||||
import {
|
||||
getDarkModeVariant,
|
||||
setDarkModeVariant,
|
||||
@@ -105,6 +107,17 @@
|
||||
<AIUserSettings />
|
||||
</div>
|
||||
</div>
|
||||
<!-- Keyed on the workspace `copilotInfo` reflects, not on `$workspaceStore`:
|
||||
a session acting on another workspace loads that workspace's AI config
|
||||
while navigation stays put, and pricing usage from one workspace with
|
||||
another's rates would silently misstate it. -->
|
||||
{#if $copilotWorkspace}
|
||||
<AiUsagePanel
|
||||
workspace={$copilotWorkspace}
|
||||
modelPricing={$copilotInfo.modelPricing ?? {}}
|
||||
scope="self"
|
||||
/>
|
||||
{/if}
|
||||
{/if}
|
||||
|
||||
<div class="grow min-h-0">
|
||||
|
||||
@@ -58,7 +58,7 @@ import {
|
||||
import { dfs } from '$lib/components/flows/previousResults'
|
||||
import { SvelteMap, SvelteSet } from 'svelte/reactivity'
|
||||
import { createLongHash } from '$lib/editorLangUtils'
|
||||
import type { UserDraftItemKind } from '$lib/gen'
|
||||
import type { AIProvider, UserDraftItemKind } from '$lib/gen'
|
||||
import { maskKey } from '$lib/components/sessions/modifiedItemsMask'
|
||||
import { getStringError } from './utils'
|
||||
import { type PasteAttachment } from './pasteTokens'
|
||||
@@ -101,7 +101,12 @@ import type AIChatInput from './AIChatInput.svelte'
|
||||
import { prepareApiSystemMessage, prepareApiUserMessage } from './api/core'
|
||||
import { runChatLoop, truncateToToolPairedPrefix } from './chatLoop'
|
||||
import { sanitizeToolCallArguments } from './toolCallArguments'
|
||||
import { normalizeContextUsage } from './tokenUsage'
|
||||
import {
|
||||
billedTokens,
|
||||
normalizeContextUsage,
|
||||
type ChatTokenUsage
|
||||
} from './tokenUsage'
|
||||
import { logAiUsage } from '$lib/utils/aiUsageReporter'
|
||||
import type { ReviewChangesOpts } from './monaco-adapter'
|
||||
import {
|
||||
getCurrentModel,
|
||||
@@ -709,6 +714,39 @@ export class AIChatManager {
|
||||
await this.#persistModifiedItems()
|
||||
}
|
||||
|
||||
/** Report one completed provider response's tokens to the workspace usage view.
|
||||
* Called per response rather than per turn: a tool loop makes several, each
|
||||
* separately billed, and a turn that fails partway through has still spent
|
||||
* everything up to that point.
|
||||
*
|
||||
* Only token counts leave the browser — rates are applied when the usage is
|
||||
* read, so a corrected price also corrects everything already recorded. */
|
||||
private recordUsage(
|
||||
usage: ChatTokenUsage,
|
||||
provider: AIProvider,
|
||||
model: string,
|
||||
workspace: string | undefined
|
||||
) {
|
||||
// A provider that reports no usage still yields an all-zero report. Recording
|
||||
// it would add a $0 row to the usage view, claiming the request cost nothing
|
||||
// rather than that it went uncounted.
|
||||
if (usage.total === 0 && usage.prompt === 0 && usage.completion === 0) {
|
||||
return
|
||||
}
|
||||
const tokens = billedTokens(usage)
|
||||
logAiUsage({
|
||||
provider,
|
||||
model,
|
||||
sessionId: this.sessionId,
|
||||
inputTokens: tokens.input,
|
||||
cacheReadTokens: tokens.cacheRead,
|
||||
cacheWriteTokens: tokens.cacheWrite,
|
||||
outputTokens: tokens.output,
|
||||
costUsd: usage.cost,
|
||||
workspace
|
||||
})
|
||||
}
|
||||
|
||||
// Serialized, snapshot-at-write-time persistence: two rapid dock actions
|
||||
// would otherwise race their saveChat writes, and the earlier (staler)
|
||||
// snapshot could land last — dropping the later mutation until the next
|
||||
@@ -2458,6 +2496,11 @@ export class AIChatManager {
|
||||
// on each iteration. This is critical for changeModeTool (Navigator → Script/Flow)
|
||||
// which reassigns this.tools, this.helpers, this.systemMessage mid-loop.
|
||||
const self = this
|
||||
// Pinned for the whole turn, like the `workspace` the loop routes through:
|
||||
// the global chat's operating workspace follows workspaceStore, so a switch
|
||||
// while a response streams would bill it to the workspace the user landed
|
||||
// on rather than the one whose credentials and proxy served it.
|
||||
const usageWorkspace = this.operatingWorkspace
|
||||
const result = await runChatLoop({
|
||||
messages,
|
||||
addedMessages,
|
||||
@@ -2535,6 +2578,14 @@ export class AIChatManager {
|
||||
}
|
||||
return undefined
|
||||
},
|
||||
onUsage: (usage, modelProvider) => {
|
||||
// Accounting must never take a turn down with it.
|
||||
try {
|
||||
this.recordUsage(usage, modelProvider.provider, modelProvider.model, usageWorkspace)
|
||||
} catch (e) {
|
||||
console.error('Failed to record AI usage', e)
|
||||
}
|
||||
},
|
||||
onBeforeIteration: async (tools, _helpers, modelProvider) => {
|
||||
this.lastIterationModel = modelProvider
|
||||
for (const tool of tools) {
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
import { getAiChatManager } from './aiChatManagerContext'
|
||||
import { AIMode } from './AIChatManager.svelte'
|
||||
import Tooltip from '$lib/components/meltComponents/Tooltip.svelte'
|
||||
import { formatTokenCount } from './tokenUsage'
|
||||
|
||||
const aiChatManager = getAiChatManager()
|
||||
|
||||
@@ -45,16 +46,6 @@
|
||||
? 'bg-amber-500'
|
||||
: 'bg-surface-accent-primary'
|
||||
)
|
||||
|
||||
function formatTokenCount(tokens: number): string {
|
||||
if (tokens >= 1_000_000) {
|
||||
return `${(tokens / 1_000_000).toFixed(1).replace(/\.0$/, '')}M`
|
||||
}
|
||||
if (tokens >= 1000) {
|
||||
return `${Math.round(tokens / 1000)}k`
|
||||
}
|
||||
return `${tokens}`
|
||||
}
|
||||
</script>
|
||||
|
||||
{#if visible}
|
||||
|
||||
@@ -195,7 +195,7 @@ export async function parseAnthropicCompletion(
|
||||
tools: Tool<any>[],
|
||||
helpers: any,
|
||||
abortController?: AbortController,
|
||||
options?: { workspace?: string }
|
||||
options?: { workspace?: string; onTokenUsage?: (usage: ChatTokenUsage) => void }
|
||||
): Promise<ParsedCompletionResult> {
|
||||
let toolCallsToProcess: ChatCompletionMessageFunctionToolCall[] = []
|
||||
let error = null
|
||||
@@ -417,6 +417,7 @@ export async function parseAnthropicCompletion(
|
||||
|
||||
const finalMessage = await completion.finalMessage()
|
||||
const tokenUsage = anthropicUsageToChatTokenUsage(finalMessage.usage)
|
||||
options?.onTokenUsage?.(tokenUsage)
|
||||
|
||||
// Process tool calls if any
|
||||
if (toolCallsToProcess.length > 0) {
|
||||
|
||||
@@ -518,7 +518,7 @@ describe('runChatLoop lastIterationUsage', () => {
|
||||
|
||||
expect(result.lastIterationUsage).toEqual({ prompt: 1200, completion: 80, total: 1280 })
|
||||
// the aggregate keeps summing across iterations
|
||||
expect(result.tokenUsage).toEqual({ prompt: 2200, completion: 130, total: 2330 })
|
||||
expect(result.tokenUsage).toMatchObject({ prompt: 2200, completion: 130, total: 2330 })
|
||||
})
|
||||
|
||||
it('ignores empty usage reports and returns null when none are real', async () => {
|
||||
|
||||
@@ -77,11 +77,16 @@ export interface ChatLoopConfig {
|
||||
helpers: any,
|
||||
modelProvider: ReasoningProviderModel
|
||||
) => Promise<void>
|
||||
/** Fired for each completed provider response, before the loop continues. The
|
||||
* loop can fail or be aborted at any iteration, so spend has to be handed over
|
||||
* as it happens — a callback only at the end would discard everything the
|
||||
* earlier iterations were already billed for. */
|
||||
onUsage?: (usage: ChatTokenUsage, modelProvider: ReasoningProviderModel) => void
|
||||
}
|
||||
|
||||
export interface ChatLoopResult {
|
||||
addedMessages: ChatCompletionMessageParam[]
|
||||
/** Sum of usage across all loop iterations (suitable for cost accounting). */
|
||||
/** Sum of usage across all loop iterations. */
|
||||
tokenUsage: ChatTokenUsage
|
||||
lastIterationUsage: ChatTokenUsage | null
|
||||
hitMaxIterations: boolean
|
||||
@@ -328,6 +333,20 @@ export async function runChatLoop(config: ChatLoopConfig): Promise<ChatLoopResul
|
||||
let lastIterationUsage: ChatTokenUsage | null = null
|
||||
let iterations = 0
|
||||
let hitMaxIterations = false
|
||||
// The model of the iteration currently in flight; re-read per iteration like
|
||||
// `config.modelProvider` itself, so usage is attributed to the model that
|
||||
// actually served it rather than to whatever is selected when the loop ends.
|
||||
let iterationModel: ReasoningProviderModel | undefined
|
||||
|
||||
// Reported as the provider's usage arrives, not when the parser returns: a parser
|
||||
// waits on tool execution, which can wait on a person, and a tab closed in that
|
||||
// gap would drop a response that was already billed. Accounting for the turn's
|
||||
// own totals stays on the return path, where every parser reports uniformly.
|
||||
const reportUsage = (usage: ChatTokenUsage | null | undefined) => {
|
||||
if (usage && iterationModel) {
|
||||
config.onUsage?.(usage, iterationModel)
|
||||
}
|
||||
}
|
||||
|
||||
const trackUsage = (usage: ChatTokenUsage | null | undefined) => {
|
||||
tokenUsage = addChatTokenUsage(tokenUsage, usage)
|
||||
@@ -351,6 +370,7 @@ export async function runChatLoop(config: ChatLoopConfig): Promise<ChatLoopResul
|
||||
const helpers = config.helpers
|
||||
const systemMessage = config.systemMessage
|
||||
const modelProvider = config.modelProvider
|
||||
iterationModel = modelProvider
|
||||
const webSearchCacheKey = getWebSearchCacheKey(workspace, modelProvider)
|
||||
const webSearch =
|
||||
(config.webSearch ?? true) &&
|
||||
@@ -386,7 +406,11 @@ export async function runChatLoop(config: ChatLoopConfig): Promise<ChatLoopResul
|
||||
...(pendingUserMessage ? [pendingUserMessage] : [])
|
||||
]
|
||||
const toolDefs = tools.map((t) => t.def)
|
||||
const parseOptions = { workspace, provider: modelProvider.provider }
|
||||
const parseOptions = {
|
||||
workspace,
|
||||
provider: modelProvider.provider,
|
||||
onTokenUsage: reportUsage
|
||||
}
|
||||
|
||||
if (isOpenAI) {
|
||||
const reasoningSummaryCacheKey = getReasoningSummaryCacheKey(workspace, modelProvider)
|
||||
|
||||
@@ -392,7 +392,7 @@ export async function parseOpenAIResponsesCompletion(
|
||||
addedMessages: ChatCompletionMessageParam[],
|
||||
tools: Tool<any>[],
|
||||
helpers: any,
|
||||
options?: { workspace?: string }
|
||||
options?: { workspace?: string; onTokenUsage?: (usage: ChatTokenUsage) => void }
|
||||
): Promise<ParsedCompletionResult> {
|
||||
let toolCallsToProcess: ChatCompletionMessageFunctionToolCall[] = []
|
||||
let error: OpenAIError | ResponseErrorEvent | null = null
|
||||
@@ -566,6 +566,7 @@ export async function parseOpenAIResponsesCompletion(
|
||||
|
||||
const finalResponse = await runner.finalResponse()
|
||||
const tokenUsage = openAIResponsesUsageToChatTokenUsage(finalResponse.usage)
|
||||
options?.onTokenUsage?.(tokenUsage)
|
||||
|
||||
for (const item of finalResponse.output ?? []) {
|
||||
if (item.type === 'web_search_call' && !surfacedWebSearchCalls.has(item.id)) {
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
anthropicUsageToChatTokenUsage,
|
||||
billedTokens,
|
||||
openAICompletionsUsageToChatTokenUsage
|
||||
} from './tokenUsage'
|
||||
|
||||
// The two providers report cache tokens under opposite conventions — Anthropic's
|
||||
// input_tokens excludes them, OpenAI's includes them. Both are normalized so that
|
||||
// `prompt` is the whole input, which is what makes `prompt - cached` the uncached
|
||||
// share. Getting this backwards double-counts (or loses) the cached prefix, which
|
||||
// is most of a long chat's input.
|
||||
describe('billedTokens', () => {
|
||||
it('derives uncached input under the Anthropic convention', () => {
|
||||
const usage = anthropicUsageToChatTokenUsage({
|
||||
input_tokens: 1000,
|
||||
output_tokens: 200,
|
||||
cache_creation_input_tokens: 300,
|
||||
cache_read_input_tokens: 5000
|
||||
})
|
||||
expect(usage.prompt).toBe(6300)
|
||||
expect(billedTokens(usage)).toEqual({
|
||||
input: 1000,
|
||||
cacheRead: 5000,
|
||||
cacheWrite: 300,
|
||||
output: 200
|
||||
})
|
||||
})
|
||||
|
||||
it('derives uncached input under the OpenAI convention', () => {
|
||||
const usage = openAICompletionsUsageToChatTokenUsage({
|
||||
prompt_tokens: 6000,
|
||||
completion_tokens: 200,
|
||||
prompt_tokens_details: { cached_tokens: 5000 }
|
||||
})
|
||||
expect(usage.prompt).toBe(6000)
|
||||
expect(billedTokens(usage)).toEqual({
|
||||
input: 1000,
|
||||
cacheRead: 5000,
|
||||
cacheWrite: 0,
|
||||
output: 200
|
||||
})
|
||||
})
|
||||
|
||||
// OpenRouter extends the OpenAI shape with cache-creation tokens, counted
|
||||
// inside prompt_tokens like the reads beside them. Missing the field bills
|
||||
// them as uncached input.
|
||||
it('splits out OpenRouter cache-creation tokens', () => {
|
||||
const usage = openAICompletionsUsageToChatTokenUsage({
|
||||
prompt_tokens: 6300,
|
||||
completion_tokens: 200,
|
||||
prompt_tokens_details: { cached_tokens: 5000, cache_write_tokens: 300 }
|
||||
})
|
||||
expect(billedTokens(usage)).toEqual({
|
||||
input: 1000,
|
||||
cacheRead: 5000,
|
||||
cacheWrite: 300,
|
||||
output: 200
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -1,7 +1,19 @@
|
||||
import type { PricedTokens } from '../modelPricing'
|
||||
|
||||
export interface ChatTokenUsage {
|
||||
prompt: number
|
||||
completion: number
|
||||
total: number
|
||||
/**
|
||||
* Subsets of `prompt`, split out because they are billed at different rates
|
||||
* (a cached read is a fraction of an uncached one). `prompt` stays the whole
|
||||
* input so the context gauge keeps measuring the whole request; uncached
|
||||
* input is `prompt - cacheRead - cacheWrite`.
|
||||
*/
|
||||
cacheRead: number
|
||||
cacheWrite: number
|
||||
/** Cost in USD as billed, for the providers that report one. */
|
||||
cost?: number
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -28,7 +40,7 @@ export function normalizeContextUsage(
|
||||
}
|
||||
|
||||
export function emptyChatTokenUsage(): ChatTokenUsage {
|
||||
return { prompt: 0, completion: 0, total: 0 }
|
||||
return { prompt: 0, completion: 0, total: 0, cacheRead: 0, cacheWrite: 0 }
|
||||
}
|
||||
|
||||
export function addChatTokenUsage(
|
||||
@@ -39,10 +51,48 @@ export function addChatTokenUsage(
|
||||
return total
|
||||
}
|
||||
|
||||
const cost =
|
||||
total.cost === undefined && usage.cost === undefined
|
||||
? undefined
|
||||
: (total.cost ?? 0) + (usage.cost ?? 0)
|
||||
|
||||
return {
|
||||
prompt: total.prompt + usage.prompt,
|
||||
completion: total.completion + usage.completion,
|
||||
total: total.total + usage.total
|
||||
total: total.total + usage.total,
|
||||
// `?? 0`: the cache split is newer than the field it lives on, so a usage
|
||||
// object read back from storage may predate it.
|
||||
cacheRead: (total.cacheRead ?? 0) + (usage.cacheRead ?? 0),
|
||||
cacheWrite: (total.cacheWrite ?? 0) + (usage.cacheWrite ?? 0),
|
||||
...(cost === undefined ? {} : { cost })
|
||||
}
|
||||
}
|
||||
|
||||
/** Compact token count for readouts and tables (`1.2M`, `34k`, `567`). */
|
||||
export function formatTokenCount(tokens: number): string {
|
||||
if (tokens >= 1_000_000) {
|
||||
return `${(tokens / 1_000_000).toFixed(1).replace(/\.0$/, '')}M`
|
||||
}
|
||||
if (tokens >= 1000) {
|
||||
return `${Math.round(tokens / 1000)}k`
|
||||
}
|
||||
return `${tokens}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Split a usage report into the four separately-billed token classes. `prompt`
|
||||
* counts the whole input, so the uncached share is whatever the cached classes
|
||||
* do not account for — which holds for both provider conventions below
|
||||
* (Anthropic adds its cache counts into `prompt`, OpenAI's already includes them).
|
||||
*/
|
||||
export function billedTokens(usage: ChatTokenUsage): PricedTokens {
|
||||
const cacheRead = usage.cacheRead ?? 0
|
||||
const cacheWrite = usage.cacheWrite ?? 0
|
||||
return {
|
||||
input: Math.max(0, usage.prompt - cacheRead - cacheWrite),
|
||||
cacheRead,
|
||||
cacheWrite,
|
||||
output: usage.completion
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,16 +107,17 @@ export function anthropicUsageToChatTokenUsage(
|
||||
| null
|
||||
| undefined
|
||||
): ChatTokenUsage {
|
||||
const prompt =
|
||||
(usage?.input_tokens ?? 0) +
|
||||
(usage?.cache_creation_input_tokens ?? 0) +
|
||||
(usage?.cache_read_input_tokens ?? 0)
|
||||
const cacheWrite = usage?.cache_creation_input_tokens ?? 0
|
||||
const cacheRead = usage?.cache_read_input_tokens ?? 0
|
||||
const prompt = (usage?.input_tokens ?? 0) + cacheWrite + cacheRead
|
||||
const completion = usage?.output_tokens ?? 0
|
||||
|
||||
return {
|
||||
prompt,
|
||||
completion,
|
||||
total: prompt + completion
|
||||
total: prompt + completion,
|
||||
cacheRead,
|
||||
cacheWrite
|
||||
}
|
||||
}
|
||||
|
||||
@@ -89,7 +140,11 @@ export function openAIResponsesUsageToChatTokenUsage(
|
||||
return {
|
||||
prompt,
|
||||
completion,
|
||||
total: usage?.total_tokens ?? prompt + completion
|
||||
total: usage?.total_tokens ?? prompt + completion,
|
||||
cacheRead: usage?.input_tokens_details?.cached_tokens ?? 0,
|
||||
// Automatic caching: nothing is billed for populating it, and no usage
|
||||
// field reports it either.
|
||||
cacheWrite: 0
|
||||
}
|
||||
}
|
||||
|
||||
@@ -101,7 +156,16 @@ export function openAICompletionsUsageToChatTokenUsage(
|
||||
prompt_tokens?: number | null
|
||||
completion_tokens?: number | null
|
||||
total_tokens?: number | null
|
||||
prompt_tokens_details?: { cached_tokens?: number | null } | null
|
||||
prompt_tokens_details?: {
|
||||
cached_tokens?: number | null
|
||||
/** Cache creation, reported by the providers that bill for it: OpenRouter
|
||||
* passes Anthropic's through, and the Bedrock proxy folds
|
||||
* `cacheWriteInputTokens` in here. OpenAI, whose caching is automatic and
|
||||
* unbilled, reports no such field. */
|
||||
cache_write_tokens?: number | null
|
||||
} | null
|
||||
/** OpenRouter reports what it actually charged when the request opts in. */
|
||||
cost?: number | null
|
||||
}
|
||||
| null
|
||||
| undefined
|
||||
@@ -112,6 +176,9 @@ export function openAICompletionsUsageToChatTokenUsage(
|
||||
return {
|
||||
prompt,
|
||||
completion,
|
||||
total: usage?.total_tokens ?? prompt + completion
|
||||
total: usage?.total_tokens ?? prompt + completion,
|
||||
cacheRead: usage?.prompt_tokens_details?.cached_tokens ?? 0,
|
||||
cacheWrite: usage?.prompt_tokens_details?.cache_write_tokens ?? 0,
|
||||
...(typeof usage?.cost === 'number' ? { cost: usage.cost } : {})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1089,6 +1089,23 @@ export async function getFimCompletion(
|
||||
}
|
||||
}
|
||||
|
||||
// A streamed OpenAI-compatible response carries no usage at all unless the request
|
||||
// asks for it, so a provider missing from this set reports zero tokens — no context
|
||||
// gauge, no cost. `stream_options.include_usage` is part of the OpenAI streaming
|
||||
// spec and these providers document supporting it; `customai` is deliberately absent
|
||||
// because it points at an arbitrary endpoint that may reject the field outright.
|
||||
const STREAM_USAGE_PROVIDERS = new Set<AIProvider>([
|
||||
'openai',
|
||||
'azure_openai',
|
||||
'azure_foundry',
|
||||
'googleai',
|
||||
'openrouter',
|
||||
'groq',
|
||||
'deepseek',
|
||||
'mistral',
|
||||
'togetherai'
|
||||
])
|
||||
|
||||
export async function getCompletion(
|
||||
messages: ChatCompletionMessageParam[],
|
||||
abortController: AbortController,
|
||||
@@ -1132,17 +1149,17 @@ export async function getCompletion(
|
||||
// Use Completions API for other providers
|
||||
const client = options?.openaiClient ?? workspaceAIClients.getOpenaiClient()
|
||||
const completionConfig = applyReasoningToConfig(
|
||||
(provider === 'openai' ||
|
||||
provider === 'azure_openai' ||
|
||||
provider === 'azure_foundry' ||
|
||||
provider === 'googleai') &&
|
||||
config.stream
|
||||
config.stream && STREAM_USAGE_PROVIDERS.has(provider)
|
||||
? {
|
||||
...config,
|
||||
stream_options: {
|
||||
...(config.stream_options ?? {}),
|
||||
include_usage: true
|
||||
}
|
||||
},
|
||||
// OpenRouter's own extension, on top of stream_options: it returns the
|
||||
// credits actually charged next to the token counts, which is the one
|
||||
// route by which the chat sees a real cost rather than an estimate.
|
||||
...(provider === 'openrouter' ? { usage: { include: true } } : {})
|
||||
}
|
||||
: config,
|
||||
provider === 'deepseek' ? 'deepseek' : provider === 'mistral' ? 'mistral' : 'completions',
|
||||
@@ -1178,7 +1195,11 @@ export async function parseOpenAICompletion(
|
||||
tools: Tool<any>[],
|
||||
helpers: any,
|
||||
_abortController?: AbortController, // unused, for signature compatibility with parseAnthropicCompletion
|
||||
options?: { workspace?: string; provider?: string }
|
||||
options?: {
|
||||
workspace?: string
|
||||
provider?: string
|
||||
onTokenUsage?: (usage: ChatTokenUsage) => void
|
||||
}
|
||||
): Promise<{ shouldContinue: boolean; tokenUsage: ChatTokenUsage }> {
|
||||
const finalToolCalls: Record<number, ChatCompletionChunk.Choice.Delta.ToolCall> = {}
|
||||
// The tool call currently receiving argument deltas; when the stream moves on
|
||||
@@ -1328,6 +1349,7 @@ export async function parseOpenAICompletion(
|
||||
|
||||
callbacks.onMessageEnd()
|
||||
|
||||
options?.onTokenUsage?.(tokenUsage)
|
||||
// Stream over: every parsed call is queued until its turn in processToolCall.
|
||||
for (const toolCall of Object.values(finalToolCalls)) {
|
||||
if (toolCall.id) {
|
||||
|
||||
@@ -123,21 +123,77 @@ function normalizeVersionSeparators(model: string): string {
|
||||
return model.replace(/\./g, '-')
|
||||
}
|
||||
|
||||
// An entry that ends on a version digit must not run into a longer version:
|
||||
// `gpt-4.1` collapses to `gpt-4-1`, which would otherwise claim the 128K
|
||||
// `gpt-4-1106-preview` as a 1M model. Suffixes that continue with a separator
|
||||
// (`claude-opus-4-8` in `...-4-8-v1`, `gpt-5` in `gpt-5-mini`) still match.
|
||||
// Family fallbacks ending on a letter get no such guard — a version welded
|
||||
// straight onto the name (`llama3.1`) is exactly what they exist to catch.
|
||||
const MODEL_CONTEXT_WINDOW_MATCHERS: [matcher: RegExp, contextWindow: number][] =
|
||||
MODEL_CONTEXT_WINDOWS.map(([name, contextWindow]) => {
|
||||
/** Suffixes that name a route to a model rather than a different model. */
|
||||
const DECORATIVE_SUFFIXES = ['latest', 'preview', 'beta', 'stable']
|
||||
|
||||
/**
|
||||
* Compile a most-specific-first `[name, value]` table into matchers against the
|
||||
* bare model id. Shared with the pricing table so both resolve the same set of
|
||||
* ids — a model whose window is known but whose price is not (or vice versa)
|
||||
* should be a gap in one table, never a difference in matching.
|
||||
*
|
||||
* An entry that ends on a version digit must not run into a longer version:
|
||||
* `gpt-4.1` collapses to `gpt-4-1`, which would otherwise claim
|
||||
* `gpt-4-1106-preview`. Suffixes that continue with a separator
|
||||
* (`claude-opus-4-8` in `...-4-8-v1`, `gpt-5` in `gpt-5-mini`) still match.
|
||||
* Family fallbacks ending on a letter get no such guard — a version welded
|
||||
* straight onto the name (`llama3.1`) is exactly what they exist to catch.
|
||||
*/
|
||||
export function buildModelMatchers<T>(
|
||||
entries: [name: string, value: T][],
|
||||
{ strictVariants = false }: { strictVariants?: boolean } = {}
|
||||
): [RegExp, T][] {
|
||||
return entries.map(([name, value]) => {
|
||||
const pattern = normalizeVersionSeparators(name).replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
|
||||
return [new RegExp(/\d$/.test(pattern) ? `${pattern}(?!\\d)` : pattern), contextWindow]
|
||||
const guards = [
|
||||
// An entry ending on a version digit must not run into a longer version.
|
||||
/\d$/.test(pattern) ? '(?!\\d)' : '',
|
||||
// A named sub-model (`gpt-5-pro`, `gpt-5-mini`) is a different model with
|
||||
// its own price, not another route to this one — so under strictVariants an
|
||||
// entry does not match when a further *name* segment follows. What follows
|
||||
// is only a decoration when it is a date (`-20251101`), Bedrock's `-v1`, or
|
||||
// one of the alias words below, at the very end of the id
|
||||
// (`claude-3-5-haiku-latest` is the same model as `claude-3-5-haiku`, and is
|
||||
// a shipped default; `gpt-5-preview-pro` would be a different one again).
|
||||
// Off by default: for a context window an inherited value is a safe
|
||||
// approximation, for a price it is a wrong number.
|
||||
// A further revision segment (`gpt-5` vs `gpt-5-4-mini`) is a different model
|
||||
// too, and the entry-ends-on-a-digit guard above does not catch it once the
|
||||
// separator is normalized. Only a short segment: a date is digits as well
|
||||
// (`-20251101`) and stays a decoration.
|
||||
strictVariants ? '(?!-\\d{1,3}(?:$|-))' : '',
|
||||
strictVariants
|
||||
? `(?!-(?!(?:v\\d|${DECORATIVE_SUFFIXES.join('|')})$)[a-z])`
|
||||
: ''
|
||||
].join('')
|
||||
return [new RegExp(pattern + guards), value]
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* The `provider:model` key the workspace AI settings use for their per-model maps
|
||||
* (`max_tokens_per_model`, `model_pricing`). A bare model id is not enough: the
|
||||
* same id can be served by more than one provider at different rates.
|
||||
*
|
||||
* Matched exactly, unlike the fuzzy tables above. Those tables generalize across
|
||||
* every route to one model on purpose; a per-model *setting* must not, or an
|
||||
* admin could not give two variants of a family different values — and the key is
|
||||
* built from the exact id the provider config lists, which is the same string the
|
||||
* chat sends.
|
||||
*/
|
||||
export function modelKey(provider: AIProvider | string, model: string): string {
|
||||
return `${provider}:${model}`
|
||||
}
|
||||
|
||||
export function matchModel<T>(matchers: [RegExp, T][], model: string): T | undefined {
|
||||
const id = normalizeVersionSeparators(parseModelId(model).base)
|
||||
return matchers.find(([matcher]) => matcher.test(id))?.[1]
|
||||
}
|
||||
|
||||
const MODEL_CONTEXT_WINDOW_MATCHERS = buildModelMatchers(MODEL_CONTEXT_WINDOWS)
|
||||
|
||||
export function getKnownModelContextWindow(model: string): number | undefined {
|
||||
const id = normalizeVersionSeparators(parseModelId(model).base)
|
||||
return MODEL_CONTEXT_WINDOW_MATCHERS.find(([matcher]) => matcher.test(id))?.[1]
|
||||
return matchModel(MODEL_CONTEXT_WINDOW_MATCHERS, model)
|
||||
}
|
||||
|
||||
export function getModelContextWindow(model: string) {
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { billedTokens } from './chat/tokenUsage'
|
||||
import { estimateCost, priceSpend, resolveModelPrice } from './modelPricing'
|
||||
|
||||
describe('resolveModelPrice', () => {
|
||||
it('resolves the same model across the routes that decorate its id', () => {
|
||||
const direct = resolveModelPrice('anthropic', 'claude-opus-5', undefined)
|
||||
expect(direct?.price.input).toBe(5)
|
||||
// A gateway prefix, a dot-versioned id, a date suffix and a variant suffix
|
||||
// must all land on the same entry — a miss here silently under-reports cost.
|
||||
for (const id of [
|
||||
'anthropic/claude-opus-5',
|
||||
'anthropic/claude-opus-4.8',
|
||||
'claude-opus-4-8-20260101',
|
||||
'anthropic/claude-opus-5:thinking'
|
||||
]) {
|
||||
expect(resolveModelPrice('openrouter', id, undefined)?.price.input).toBe(5)
|
||||
}
|
||||
})
|
||||
|
||||
it('does not let a version-digit entry claim a longer version', () => {
|
||||
expect(resolveModelPrice('openai', 'gpt-4.1', undefined)?.price.input).toBe(2)
|
||||
expect(resolveModelPrice('openai', 'gpt-4-1106-preview', undefined)?.price.input).not.toBe(2)
|
||||
})
|
||||
|
||||
it('prices flat-rate Gemini Flash while leaving the tiered Pro alone', () => {
|
||||
expect(resolveModelPrice('googleai', 'gemini-2.5-flash', undefined)?.price.input).toBe(0.3)
|
||||
expect(resolveModelPrice('googleai', 'gemini-2.5-flash-lite', undefined)?.price.input).toBe(0.1)
|
||||
expect(resolveModelPrice('googleai', 'gemini-3.5-flash', undefined)?.price.output).toBe(9)
|
||||
// Pro charges roughly double above a 200k prompt, which a per-model rate cannot
|
||||
// express, so it must stay unpriced rather than be estimated at the low tier.
|
||||
expect(resolveModelPrice('googleai', 'gemini-2.5-pro', undefined)).toBeUndefined()
|
||||
expect(resolveModelPrice('googleai', 'gemini-3.1-pro', undefined)).toBeUndefined()
|
||||
// Promotional rates carry an end date a timeless table cannot represent.
|
||||
expect(resolveModelPrice('googleai', 'gemini-3.7-flash', undefined)).toBeUndefined()
|
||||
})
|
||||
|
||||
it('reports an unknown model as unpriced rather than guessing', () => {
|
||||
expect(resolveModelPrice('customai', 'some-in-house-model', undefined)).toBeUndefined()
|
||||
})
|
||||
|
||||
it('does not let another model inherit a price through a shared prefix', () => {
|
||||
// A sub-model (`-pro`) or a newer revision (`gpt-5.6` → `gpt-5-6`) is a
|
||||
// different model at a different rate; inheriting `gpt-5`'s would be off by
|
||||
// an order of magnitude, and silently so.
|
||||
expect(resolveModelPrice('openai', 'gpt-5', undefined)?.price.input).toBe(1.25)
|
||||
expect(resolveModelPrice('openai', 'gpt-5-mini', undefined)?.price.input).toBe(0.25)
|
||||
expect(resolveModelPrice('openai', 'gpt-5-pro', undefined)).toBeUndefined()
|
||||
expect(resolveModelPrice('openai', 'gpt-5.6', undefined)).toBeUndefined()
|
||||
expect(resolveModelPrice('googleai', 'gemini-3.1', undefined)).toBeUndefined()
|
||||
// A revision carrying a variant has to be caught by the matcher, not by an
|
||||
// explicit entry: `gpt-5.4-mini` cannot match the `gpt-5.4` one (the `-mini`
|
||||
// makes it a sub-model), so nothing but the guard stops it reaching `gpt-5`.
|
||||
expect(resolveModelPrice('openai', 'gpt-5.4-mini', undefined)).toBeUndefined()
|
||||
expect(resolveModelPrice('openai', 'gpt-5.5-pro', undefined)).toBeUndefined()
|
||||
})
|
||||
|
||||
it('still resolves the route decorations that name the same model', () => {
|
||||
// Dates, Bedrock's -v1 and floating aliases are ways of spelling one model,
|
||||
// not sub-models. `claude-3-5-haiku-latest` is a shipped picker default, so
|
||||
// unpricing it would silently disable cost tracking out of the box.
|
||||
expect(resolveModelPrice('anthropic', 'claude-opus-4-5-20251101', undefined)?.price.input).toBe(5)
|
||||
// The revision guard must not swallow a date, which is digits too.
|
||||
expect(resolveModelPrice('openai', 'gpt-5-2026-01-01', undefined)?.price.input).toBe(1.25)
|
||||
expect(
|
||||
resolveModelPrice('bedrock', 'anthropic.claude-sonnet-4-6-20250101-v1:0', undefined)?.price
|
||||
.input
|
||||
).toBe(3)
|
||||
expect(resolveModelPrice('anthropic', 'claude-3-5-haiku-latest', undefined)?.price.input).toBe(
|
||||
0.8
|
||||
)
|
||||
// …while a genuine sub-model stays unpriced, including one hiding behind a
|
||||
// decoration.
|
||||
expect(resolveModelPrice('openai', 'gpt-5-pro', undefined)).toBeUndefined()
|
||||
expect(resolveModelPrice('openai', 'gpt-5-preview-pro', undefined)).toBeUndefined()
|
||||
// A family fallback must not price a model the table deliberately left out,
|
||||
// nor the floating alias pointing at it.
|
||||
expect(resolveModelPrice('anthropic', 'claude-sonnet-5', undefined)).toBeUndefined()
|
||||
expect(
|
||||
resolveModelPrice('openrouter', '~anthropic/claude-sonnet-latest', undefined)
|
||||
).toBeUndefined()
|
||||
})
|
||||
|
||||
it('prefers a workspace override, keeping the model’s own cache ratios', () => {
|
||||
const resolved = resolveModelPrice('anthropic', 'claude-opus-5', {
|
||||
'anthropic:claude-opus-5': { input: 2, output: 8 }
|
||||
})
|
||||
expect(resolved?.source).toBe('override')
|
||||
expect(resolved?.price.input).toBe(2)
|
||||
// Anthropic reads a cached prefix at a tenth and writes at 1.25x.
|
||||
expect(resolved?.price.cacheRead).toBeCloseTo(0.2)
|
||||
expect(resolved?.price.cacheWrite).toBeCloseTo(2.5)
|
||||
})
|
||||
|
||||
it('applies the overridden model’s own cache discount, not Anthropic’s', () => {
|
||||
// gpt-4o discounts a cached read by half, not by a tenth — an override that
|
||||
// only states input/output must not silently inherit the Anthropic ratio.
|
||||
const resolved = resolveModelPrice('openai', 'gpt-4o', {
|
||||
'openai:gpt-4o': { input: 2, output: 8 }
|
||||
})
|
||||
expect(resolved?.price.cacheRead).toBeCloseTo(1)
|
||||
})
|
||||
|
||||
it('bills an unpriced model’s cached tokens at its input rate', () => {
|
||||
// Gemini Pro is deliberately unpriced, so there is no ratio to inherit. Falling
|
||||
// back to Anthropic's tenth would invent a discount the provider may not give;
|
||||
// the admin states the cache rates explicitly or pays full input.
|
||||
const resolved = resolveModelPrice('googleai', 'gemini-2.5-pro', {
|
||||
'googleai:gemini-2.5-pro': { input: 2, output: 8 }
|
||||
})
|
||||
expect(resolved?.price.cacheRead).toBe(2)
|
||||
expect(resolved?.price.cacheWrite).toBe(2)
|
||||
|
||||
const stated = resolveModelPrice('googleai', 'gemini-2.5-pro', {
|
||||
'googleai:gemini-2.5-pro': { input: 2, output: 8, cache_read: 0.5, cache_write: 1 }
|
||||
})
|
||||
expect(stated?.price.cacheRead).toBe(0.5)
|
||||
expect(stated?.price.cacheWrite).toBe(1)
|
||||
})
|
||||
|
||||
it('ignores an override whose rates could not be a price', () => {
|
||||
for (const bad of [{ input: -1, output: 8 }, { input: 1e9, output: 8 }]) {
|
||||
const resolved = resolveModelPrice('anthropic', 'claude-opus-5', {
|
||||
'anthropic:claude-opus-5': bad
|
||||
})
|
||||
expect(resolved?.source).toBe('builtin')
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('estimateCost', () => {
|
||||
it('bills each token class at its own rate', () => {
|
||||
const cost = estimateCost(
|
||||
{ input: 1_000_000, cacheRead: 1_000_000, cacheWrite: 1_000_000, output: 1_000_000 },
|
||||
{ input: 5, output: 25, cacheRead: 0.5, cacheWrite: 6.25 }
|
||||
)
|
||||
expect(cost).toBeCloseTo(5 + 0.5 + 6.25 + 25)
|
||||
})
|
||||
|
||||
it('charges a cached prefix less than an uncached one', () => {
|
||||
const usage = {
|
||||
prompt: 100_000,
|
||||
completion: 0,
|
||||
total: 100_000,
|
||||
cacheRead: 90_000,
|
||||
cacheWrite: 0
|
||||
}
|
||||
const uncached = { ...usage, cacheRead: 0 }
|
||||
const price = { input: 5, output: 25, cacheRead: 0.5, cacheWrite: 6.25 }
|
||||
expect(estimateCost(billedTokens(usage), price)).toBeLessThan(
|
||||
estimateCost(billedTokens(uncached), price)
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
describe('priceSpend', () => {
|
||||
it('prefers a provider-reported cost over the estimate', () => {
|
||||
const priced = priceSpend(
|
||||
[
|
||||
{
|
||||
provider: 'openrouter',
|
||||
model: 'anthropic/claude-opus-5',
|
||||
tokens: { input: 1_000_000, cacheRead: 0, cacheWrite: 0, output: 0 },
|
||||
reportedCostUsd: 0.42
|
||||
}
|
||||
],
|
||||
undefined
|
||||
)
|
||||
expect(priced.total).toBe(0.42)
|
||||
expect(priced.hasReported).toBe(true)
|
||||
})
|
||||
|
||||
it('flags an unpriced model instead of counting it as free', () => {
|
||||
const priced = priceSpend(
|
||||
[
|
||||
{
|
||||
provider: 'customai',
|
||||
model: 'some-in-house-model',
|
||||
tokens: { input: 1_000_000, cacheRead: 0, cacheWrite: 0, output: 0 }
|
||||
}
|
||||
],
|
||||
undefined
|
||||
)
|
||||
expect(priced.hasUnpriced).toBe(true)
|
||||
expect(priced.rows[0].cost).toBeUndefined()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,288 @@
|
||||
import type { AIProvider, ModelPriceOverride } from '$lib/gen'
|
||||
import { buildModelMatchers, matchModel, modelKey } from './modelConfig'
|
||||
|
||||
/** Rates in USD per million tokens, one per billed token class. */
|
||||
export type ModelPrice = {
|
||||
input: number
|
||||
output: number
|
||||
cacheRead: number
|
||||
cacheWrite: number
|
||||
}
|
||||
|
||||
export type ModelPriceSource = 'override' | 'builtin'
|
||||
|
||||
export type ResolvedModelPrice = {
|
||||
price: ModelPrice
|
||||
source: ModelPriceSource
|
||||
}
|
||||
|
||||
/** What a chat spent on one model, in tokens. */
|
||||
export type PricedTokens = {
|
||||
input: number
|
||||
cacheRead: number
|
||||
cacheWrite: number
|
||||
output: number
|
||||
}
|
||||
|
||||
// Fallbacks for entries that do not price their cache separately: Anthropic reads a
|
||||
// cached prefix at a tenth of the input rate and writes one at 1.25x (5-minute TTL,
|
||||
// the default the chat uses). The read ratio is NOT universal — OpenAI and Google
|
||||
// discount a cached read far less — so every non-Anthropic entry below states its own
|
||||
// `cacheRead` rather than inheriting this. Providers whose caching is automatic never
|
||||
// report a cache write, so their write rate is unused.
|
||||
const CACHE_READ_RATIO = 0.1
|
||||
const CACHE_WRITE_RATIO = 1.25
|
||||
|
||||
type PriceEntry = {
|
||||
input: number
|
||||
output: number
|
||||
cacheRead?: number
|
||||
cacheWrite?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Published list prices, most specific entry first — the first name found in the
|
||||
* bare model id wins, so vendor-namespaced and date-suffixed ids
|
||||
* (anthropic/claude-opus-5, gpt-5-2026-01-01) still resolve. Matching is shared
|
||||
* with the context-window table via `buildModelMatchers`.
|
||||
*
|
||||
* This is a best-effort snapshot: vendors change rates, ship models faster than
|
||||
* this table is updated, and negotiated rates differ from list. A model that is
|
||||
* not listed resolves to undefined and is reported as unpriced rather than
|
||||
* guessed at, and any entry can be corrected per workspace from the AI settings.
|
||||
* Providers whose catalogue turns over too quickly to track (DeepSeek, Mistral,
|
||||
* Groq, TogetherAI, custom deployments) are deliberately absent.
|
||||
*
|
||||
* `null` marks a model that is known to exist but whose rates are not. Unpriced is
|
||||
* a supported state (the UI says so and points at the override); a confidently
|
||||
* wrong number is not — which is also why these matchers are built with
|
||||
* `strictVariants`, so an unlisted sub-model (`gpt-5-pro`) reports no rate instead
|
||||
* of inheriting its family's.
|
||||
*
|
||||
* One known gap the per-model shape cannot express: Anthropic's 1M-context beta
|
||||
* charges more above a threshold. Usage is aggregated per model before pricing, so
|
||||
* those requests are estimated at the standard tier and understate. An affected
|
||||
* workspace can set the higher rate as its override.
|
||||
*/
|
||||
const MODEL_PRICES: [name: string, price: PriceEntry | null][] = [
|
||||
// Anthropic — Opus 4.1 and older bill at the pre-4.5 Opus rate, so the family
|
||||
// fallback sits below the explicit entries rather than covering them.
|
||||
['claude-fable-5', { input: 10, output: 50 }],
|
||||
['claude-mythos-5', { input: 10, output: 50 }],
|
||||
['claude-opus-5', { input: 5, output: 25 }],
|
||||
['claude-opus-4-8', { input: 5, output: 25 }],
|
||||
['claude-opus-4-7', { input: 5, output: 25 }],
|
||||
['claude-opus-4-6', { input: 5, output: 25 }],
|
||||
['claude-opus-4-5', { input: 5, output: 25 }],
|
||||
['claude-opus-4-1', { input: 15, output: 75 }],
|
||||
['claude-opus-4', { input: 15, output: 75 }],
|
||||
// Sonnet 5 runs a promotional rate with a published end date, and
|
||||
// `claude-sonnet-latest` floats to it. Rates carry no date and apply at read
|
||||
// time, so either figure misstates one side of that boundary — unpriced until
|
||||
// the rate is a single number again.
|
||||
['claude-sonnet-5', null],
|
||||
['claude-sonnet-latest', null],
|
||||
['claude-sonnet-4-6', { input: 3, output: 15 }],
|
||||
['claude-sonnet-4-5', { input: 3, output: 15 }],
|
||||
['claude-sonnet-4', { input: 3, output: 15 }],
|
||||
['claude-haiku-4-5', { input: 1, output: 5 }],
|
||||
['claude-3-5-haiku', { input: 0.8, output: 4 }],
|
||||
['claude-opus', { input: 5, output: 25 }],
|
||||
['claude-sonnet', { input: 3, output: 15 }],
|
||||
['claude-haiku', { input: 1, output: 5 }],
|
||||
// OpenAI — the cached-input discount varies by family (a tenth on gpt-5, a
|
||||
// quarter on 4.1 and the o-series, half on 4o), so each entry carries its own
|
||||
// rate. There is no charge for writing the cache and no usage field reporting
|
||||
// one, so the write rate never applies. The -mini/-nano entries must precede
|
||||
// the family entry, which would otherwise claim them.
|
||||
// Revisions past gpt-5 are priced separately by OpenAI and are not tracked here.
|
||||
// The matcher's revision guard already keeps them off the family rate; these
|
||||
// entries stay so a revision the guard admits still resolves to no rate.
|
||||
['gpt-5.6', null],
|
||||
['gpt-5.5', null],
|
||||
['gpt-5.4', null],
|
||||
['gpt-5.2', null],
|
||||
['gpt-5.1', null],
|
||||
['gpt-5-mini', { input: 0.25, output: 2, cacheRead: 0.025 }],
|
||||
['gpt-5-nano', { input: 0.05, output: 0.4, cacheRead: 0.005 }],
|
||||
['gpt-5', { input: 1.25, output: 10, cacheRead: 0.125 }],
|
||||
['gpt-4.1-mini', { input: 0.4, output: 1.6, cacheRead: 0.1 }],
|
||||
['gpt-4.1-nano', { input: 0.1, output: 0.4, cacheRead: 0.025 }],
|
||||
['gpt-4.1', { input: 2, output: 8, cacheRead: 0.5 }],
|
||||
['gpt-4o-mini', { input: 0.15, output: 0.6, cacheRead: 0.075 }],
|
||||
['gpt-4o', { input: 2.5, output: 10, cacheRead: 1.25 }],
|
||||
['o4-mini', { input: 1.1, output: 4.4, cacheRead: 0.275 }],
|
||||
['o3-mini', { input: 1.1, output: 4.4, cacheRead: 0.55 }],
|
||||
['o3', { input: 2, output: 8, cacheRead: 0.5 }],
|
||||
// Google — Flash takes a flat rate and is priced; Pro is not, because both its
|
||||
// input and output roughly double above a 200k-token prompt and a per-model rate
|
||||
// cannot express a threshold. Explicit context caching also bills storage per hour,
|
||||
// which nothing here represents, so a workspace using it sees an underestimate.
|
||||
// Gemini 3.7 and 3.6 Flash run a promotional rate with an end date, and stay
|
||||
// unpriced for the same reason Sonnet 5 does.
|
||||
['gemini-2.5-flash-lite', { input: 0.1, output: 0.4, cacheRead: 0.01 }],
|
||||
['gemini-2.5-flash', { input: 0.3, output: 2.5, cacheRead: 0.03 }],
|
||||
['gemini-3.5-flash-lite', { input: 0.3, output: 2.5, cacheRead: 0.03 }],
|
||||
['gemini-3.5-flash', { input: 1.5, output: 9, cacheRead: 0.15 }],
|
||||
['gemini-3.7', null],
|
||||
['gemini-3.6', null],
|
||||
['gemini-3.1', null],
|
||||
['gemini-3', null],
|
||||
['gemini-2.5', null]
|
||||
]
|
||||
|
||||
const MODEL_PRICE_MATCHERS = buildModelMatchers(
|
||||
MODEL_PRICES.map(([name, entry]): [string, ModelPrice | null] => [
|
||||
name,
|
||||
entry && {
|
||||
input: entry.input,
|
||||
output: entry.output,
|
||||
cacheRead: entry.cacheRead ?? entry.input * CACHE_READ_RATIO,
|
||||
cacheWrite: entry.cacheWrite ?? entry.input * CACHE_WRITE_RATIO
|
||||
}
|
||||
]),
|
||||
{ strictVariants: true }
|
||||
)
|
||||
|
||||
export function getKnownModelPrice(model: string): ModelPrice | undefined {
|
||||
return matchModel(MODEL_PRICE_MATCHERS, model) ?? undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Rates the API bounds on the way in — but an instance-level config is stored as an
|
||||
* untyped settings blob that bypasses that handler, so the reader enforces the same
|
||||
* bounds rather than rendering a negative, infinite or absurd total.
|
||||
*/
|
||||
const MAX_MODEL_RATE = 1000
|
||||
|
||||
function isUsableRate(rate: number | undefined): boolean {
|
||||
return rate === undefined || (Number.isFinite(rate) && rate >= 0 && rate <= MAX_MODEL_RATE)
|
||||
}
|
||||
|
||||
/** What a cache rate falls back to when an override leaves it unset: the model's
|
||||
* own published multiple of the input rate where the table has one, and the input
|
||||
* rate itself where it does not, so an unstated discount is never borrowed from
|
||||
* another vendor. Shared with the rates editor, which shows these as placeholders. */
|
||||
export function inheritedCacheRates(
|
||||
model: string,
|
||||
input: number
|
||||
): { cacheRead: number; cacheWrite: number } {
|
||||
const builtin = getKnownModelPrice(model)
|
||||
return {
|
||||
cacheRead: input * (builtin ? builtin.cacheRead / builtin.input : 1),
|
||||
cacheWrite: input * (builtin ? builtin.cacheWrite / builtin.input : 1)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The rate a workspace should be billed at for one model: its override when an
|
||||
* admin set one, otherwise the published list price, otherwise nothing. An override
|
||||
* that omits a cache rate takes it from `inheritedCacheRates`.
|
||||
*/
|
||||
export function resolveModelPrice(
|
||||
provider: AIProvider | string,
|
||||
model: string,
|
||||
overrides: Record<string, ModelPriceOverride> | undefined
|
||||
): ResolvedModelPrice | undefined {
|
||||
const builtin = getKnownModelPrice(model)
|
||||
const candidate = overrides?.[modelKey(provider, model)]
|
||||
const override =
|
||||
candidate &&
|
||||
isUsableRate(candidate.input) &&
|
||||
isUsableRate(candidate.output) &&
|
||||
isUsableRate(candidate.cache_read) &&
|
||||
isUsableRate(candidate.cache_write)
|
||||
? candidate
|
||||
: undefined
|
||||
if (override) {
|
||||
const inherited = inheritedCacheRates(model, override.input)
|
||||
return {
|
||||
source: 'override',
|
||||
price: {
|
||||
input: override.input,
|
||||
output: override.output,
|
||||
cacheRead: override.cache_read ?? inherited.cacheRead,
|
||||
cacheWrite: override.cache_write ?? inherited.cacheWrite
|
||||
}
|
||||
}
|
||||
}
|
||||
return builtin ? { source: 'builtin', price: builtin } : undefined
|
||||
}
|
||||
|
||||
/** Cost in USD of `tokens` at `price`. */
|
||||
export function estimateCost(tokens: PricedTokens, price: ModelPrice): number {
|
||||
return (
|
||||
(tokens.input * price.input +
|
||||
tokens.cacheRead * price.cacheRead +
|
||||
tokens.cacheWrite * price.cacheWrite +
|
||||
tokens.output * price.output) /
|
||||
1_000_000
|
||||
)
|
||||
}
|
||||
|
||||
/** Tokens spent on one model, from a chat's running totals or the usage API. */
|
||||
export type ModelSpend = {
|
||||
provider: string
|
||||
model: string
|
||||
tokens: PricedTokens
|
||||
/** What the provider billed, where it reports a figure. */
|
||||
reportedCostUsd?: number
|
||||
}
|
||||
|
||||
export type Priced = {
|
||||
/** Undefined when no rate is known for the model — reported as unpriced, never guessed. */
|
||||
cost: number | undefined
|
||||
source: ModelPriceSource | 'reported' | undefined
|
||||
}
|
||||
|
||||
export type PricedSpend<T extends ModelSpend> = {
|
||||
/** The input entries, each with its cost — callers carry their own fields through
|
||||
* rather than zipping the result back against the input by index. */
|
||||
rows: (T & Priced)[]
|
||||
total: number
|
||||
/** True when at least one row has no rate, so `total` understates the truth. */
|
||||
hasUnpriced: boolean
|
||||
/** True when at least one row is a figure the provider billed rather than an estimate. */
|
||||
hasReported: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Cost a set of per-model token counts. A provider-reported figure always wins:
|
||||
* it is what was actually charged, where everything else is list price times
|
||||
* tokens. `source` says which, so a view never presents an estimate as a bill.
|
||||
*/
|
||||
export function priceSpend<T extends ModelSpend>(
|
||||
spend: T[],
|
||||
overrides: Record<string, ModelPriceOverride> | undefined
|
||||
): PricedSpend<T> {
|
||||
let total = 0
|
||||
let hasUnpriced = false
|
||||
let hasReported = false
|
||||
const rows = spend.map((entry): T & Priced => {
|
||||
if (entry.reportedCostUsd !== undefined) {
|
||||
hasReported = true
|
||||
total += entry.reportedCostUsd
|
||||
return { ...entry, cost: entry.reportedCostUsd, source: 'reported' }
|
||||
}
|
||||
const resolved = resolveModelPrice(entry.provider, entry.model, overrides)
|
||||
if (!resolved) {
|
||||
hasUnpriced = true
|
||||
return { ...entry, cost: undefined, source: undefined }
|
||||
}
|
||||
const cost = estimateCost(entry.tokens, resolved.price)
|
||||
total += cost
|
||||
return { ...entry, cost, source: resolved.source }
|
||||
})
|
||||
return { rows, total, hasUnpriced, hasReported }
|
||||
}
|
||||
|
||||
/**
|
||||
* Money, at the precision the amount deserves: sub-cent spend is where a chat
|
||||
* spends most of its life, and rounding it to `$0.00` would read as free.
|
||||
*/
|
||||
export function formatUsd(amount: number): string {
|
||||
if (amount === 0) return '$0'
|
||||
if (amount < 0.01) return `$${amount.toFixed(4)}`
|
||||
if (amount < 1) return `$${amount.toFixed(3)}`
|
||||
return `$${amount.toFixed(2)}`
|
||||
}
|
||||
@@ -5,9 +5,11 @@
|
||||
type AIConfig,
|
||||
type AIProvider,
|
||||
type GetCopilotSettingsStateResponse,
|
||||
type InstanceAISummary
|
||||
type InstanceAISummary,
|
||||
type ModelPriceOverride
|
||||
} from '$lib/gen'
|
||||
import { workspaceStore } from '$lib/stores'
|
||||
import { copilotInfo } from '$lib/aiStore'
|
||||
import { sendUserToast } from '$lib/toast'
|
||||
import { AI_PROVIDERS, fetchAvailableModels, providerSupportsWebSearch } from '../copilot/lib'
|
||||
import { supportsAutocomplete } from '../copilot/utils'
|
||||
@@ -25,6 +27,8 @@
|
||||
import Badge from '../common/badge/Badge.svelte'
|
||||
import Tooltip from '../Tooltip.svelte'
|
||||
import ModelTokenLimits from './ModelTokenLimits.svelte'
|
||||
import ModelPricing from './ModelPricing.svelte'
|
||||
import AiUsagePanel from './AiUsagePanel.svelte'
|
||||
import { setCopilotInfo } from '$lib/aiStore'
|
||||
import AIPromptsModal from '../settings/AIPromptsModal.svelte'
|
||||
import { Settings } from 'lucide-svelte'
|
||||
@@ -73,6 +77,7 @@
|
||||
let metadataModel: string | undefined = $state(undefined)
|
||||
let customPrompts: Record<string, string> = $state({})
|
||||
let maxTokensPerModel: Record<string, number> = $state({})
|
||||
let modelPricing: Record<string, ModelPriceOverride> = $state({})
|
||||
let usingOpenaiClientCredentialsOauth = $state(false)
|
||||
let workspaceOverrideEditorOpened = $state(false)
|
||||
|
||||
@@ -83,6 +88,7 @@
|
||||
let initialMetadataModel: string | undefined = $state(undefined)
|
||||
let initialCustomPrompts: Record<string, string> = $state({})
|
||||
let initialMaxTokensPerModel: Record<string, number> = $state({})
|
||||
let initialModelPricing: Record<string, ModelPriceOverride> = $state({})
|
||||
let initialPrompts: Record<string, string> = $state({})
|
||||
let lastLoadedConfigKey = $state<string | undefined>(undefined)
|
||||
|
||||
@@ -110,6 +116,7 @@
|
||||
codeCompletionModel = config?.code_completion_model?.model
|
||||
customPrompts = clone(config?.custom_prompts ?? {})
|
||||
maxTokensPerModel = clone(config?.max_tokens_per_model ?? {})
|
||||
modelPricing = clone(config?.model_pricing ?? {})
|
||||
for (const mode of ['edit', 'fix', 'gen']) {
|
||||
if (!(mode in customPrompts)) {
|
||||
customPrompts[mode] = ''
|
||||
@@ -124,6 +131,7 @@
|
||||
initialCodeCompletionModel = codeCompletionModel
|
||||
initialCustomPrompts = clone(customPrompts)
|
||||
initialMaxTokensPerModel = clone(maxTokensPerModel)
|
||||
initialModelPricing = clone(modelPricing)
|
||||
initialPrompts = clone(customPrompts)
|
||||
}
|
||||
|
||||
@@ -139,6 +147,7 @@
|
||||
codeCompletionModel = initialCodeCompletionModel
|
||||
customPrompts = clone(initialCustomPrompts)
|
||||
maxTokensPerModel = clone(initialMaxTokensPerModel)
|
||||
modelPricing = clone(initialModelPricing)
|
||||
}
|
||||
|
||||
$effect(() => {
|
||||
@@ -172,7 +181,8 @@
|
||||
metadataModel !== initialMetadataModel ||
|
||||
codeCompletionModel !== initialCodeCompletionModel ||
|
||||
JSON.stringify(customPrompts) !== JSON.stringify(initialCustomPrompts) ||
|
||||
JSON.stringify(maxTokensPerModel) !== JSON.stringify(initialMaxTokensPerModel)
|
||||
JSON.stringify(maxTokensPerModel) !== JSON.stringify(initialMaxTokensPerModel) ||
|
||||
JSON.stringify(modelPricing) !== JSON.stringify(initialModelPricing)
|
||||
)
|
||||
|
||||
$effect(() => {
|
||||
@@ -285,7 +295,8 @@
|
||||
metadata_model,
|
||||
custom_prompts: Object.keys(custom_prompts).length > 0 ? custom_prompts : undefined,
|
||||
max_tokens_per_model:
|
||||
Object.keys(maxTokensPerModel).length > 0 ? maxTokensPerModel : undefined
|
||||
Object.keys(maxTokensPerModel).length > 0 ? maxTokensPerModel : undefined,
|
||||
model_pricing: Object.keys(modelPricing).length > 0 ? modelPricing : undefined
|
||||
}
|
||||
: {}
|
||||
}
|
||||
@@ -610,6 +621,24 @@
|
||||
scope={promptScope}
|
||||
/>
|
||||
|
||||
{#if promptScope === 'workspace'}
|
||||
<!-- Recorded usage must be priced with the rates the chats actually ran under.
|
||||
A workspace on instance defaults has no rates of its own, so the effective
|
||||
ones come from copilotInfo rather than from this form's (empty) workspace
|
||||
config. -->
|
||||
<AiUsagePanel
|
||||
workspace={effectiveWorkspace}
|
||||
modelPricing={usesInstanceAiConfig ? ($copilotInfo.modelPricing ?? {}) : modelPricing}
|
||||
/>
|
||||
{/if}
|
||||
|
||||
<!-- Below the usage it explains: the rates are read as a correction to what the
|
||||
table above already shows. Kept on its own `showWorkspaceOverrideEditor` gate so
|
||||
the instance scope, which has no usage panel, still edits rates. -->
|
||||
{#if showWorkspaceOverrideEditor}
|
||||
<ModelPricing {aiProviders} bind:modelPricing />
|
||||
{/if}
|
||||
|
||||
{#if showWorkspaceOverrideEditor}
|
||||
<SettingsFooter
|
||||
hasUnsavedChanges={dirty}
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
<script lang="ts">
|
||||
import { AiService, ApiError, type AITokenUsageBucket, type ModelPriceOverride } from '$lib/gen'
|
||||
import { formatUsd, priceSpend, type ModelSpend } from '../copilot/modelPricing'
|
||||
import { formatTokenCount } from '../copilot/chat/tokenUsage'
|
||||
import SettingCard from '../instanceSettings/SettingCard.svelte'
|
||||
import Select from '../select/Select.svelte'
|
||||
import ToggleButtonGroup from '../common/toggleButton-v2/ToggleButtonGroup.svelte'
|
||||
import ToggleButton from '../common/toggleButton-v2/ToggleButton.svelte'
|
||||
import { resource } from 'runed'
|
||||
import Tooltip from '../meltComponents/Tooltip.svelte'
|
||||
import DataTable from '../table/DataTable.svelte'
|
||||
import Head from '../table/Head.svelte'
|
||||
import Cell from '../table/Cell.svelte'
|
||||
|
||||
// Workspace and rates are both passed in rather than read from a store: the
|
||||
// settings component that mounts this one also serves the instance scope, and
|
||||
// the rates that priced a chat are the workspace's *effective* ones, which an
|
||||
// inheriting workspace does not hold itself.
|
||||
let {
|
||||
workspace,
|
||||
modelPricing,
|
||||
scope = 'workspace'
|
||||
}: {
|
||||
workspace: string
|
||||
modelPricing: Record<string, ModelPriceOverride>
|
||||
scope?: 'workspace' | 'self'
|
||||
} = $props()
|
||||
|
||||
type GroupBy = 'day' | 'user' | 'model'
|
||||
|
||||
let days = $state(30)
|
||||
let groupBy = $state<GroupBy>('day')
|
||||
|
||||
const rangeOptions = [
|
||||
{ label: 'Last 7 days', value: 7 },
|
||||
{ label: 'Last 30 days', value: 30 },
|
||||
{ label: 'Last 90 days', value: 90 }
|
||||
]
|
||||
|
||||
let usage = resource(
|
||||
() => ({ workspace, days, groupBy, scope }),
|
||||
async ({ workspace, days, groupBy, scope }) =>
|
||||
workspace ? await AiService.listAiUsage({ workspace, days, groupBy, scope }) : undefined
|
||||
)
|
||||
|
||||
// The API groups by (dimension, provider, model) so every bucket resolves to a
|
||||
// single rate; the table folds those back into one line per dimension value.
|
||||
type Bucket = ModelSpend & { key: string; requests: number }
|
||||
|
||||
function toSpend(bucket: AITokenUsageBucket): Bucket {
|
||||
return {
|
||||
// Grouping by model has no separate dimension — the model is the key.
|
||||
key: groupBy === 'model' ? `${bucket.provider}/${bucket.model}` : bucket.key || '—',
|
||||
requests: bucket.requests,
|
||||
provider: bucket.provider,
|
||||
model: bucket.model,
|
||||
tokens: {
|
||||
input: bucket.input_tokens,
|
||||
cacheRead: bucket.cache_read_tokens,
|
||||
cacheWrite: bucket.cache_write_tokens,
|
||||
output: bucket.output_tokens
|
||||
},
|
||||
reportedCostUsd:
|
||||
bucket.reported_cost_nano_usd != undefined
|
||||
? bucket.reported_cost_nano_usd / 1_000_000_000
|
||||
: undefined
|
||||
}
|
||||
}
|
||||
|
||||
let priced = $derived(priceSpend((usage.current?.buckets ?? []).map(toSpend), modelPricing))
|
||||
|
||||
type Row = {
|
||||
key: string
|
||||
cost: number | undefined
|
||||
/** Every model behind this line was billed back by its provider, so the
|
||||
* figure is an invoice rather than an estimate. A line mixing sources — or
|
||||
* one holding a model with no rate, whose spend the figure omits entirely —
|
||||
* makes the weaker claim. */
|
||||
reported: boolean
|
||||
tokensIn: number
|
||||
tokensOut: number
|
||||
requests: number
|
||||
}
|
||||
|
||||
// Only a 403 on the workspace scope is a permission problem; reading your own
|
||||
// usage is open to any member. Attributing every failure to permissions sends an
|
||||
// admin looking for access they already hold, and buries the real cause of the
|
||||
// far more common transient ones (an expired session, a database hiccup).
|
||||
function usageError(error: unknown): string {
|
||||
if (scope === 'workspace' && error instanceof ApiError && error.status === 403) {
|
||||
return 'Only workspace admins can read workspace usage.'
|
||||
}
|
||||
return 'Could not load usage. Try again in a moment.'
|
||||
}
|
||||
|
||||
// The headline sums both kinds, so it only escapes the ~ when nothing under it
|
||||
// was estimated.
|
||||
let totalIsEstimated = $derived(
|
||||
priced.rows.some((row) => row.cost !== undefined && row.source !== 'reported')
|
||||
)
|
||||
|
||||
let rows = $derived.by(() => {
|
||||
const byKey = new Map<string, Row>()
|
||||
for (const row of priced.rows) {
|
||||
const existing = byKey.get(row.key) ?? {
|
||||
key: row.key,
|
||||
cost: undefined,
|
||||
reported: true,
|
||||
tokensIn: 0,
|
||||
tokensOut: 0,
|
||||
requests: 0
|
||||
}
|
||||
existing.tokensIn += row.tokens.input + row.tokens.cacheRead + row.tokens.cacheWrite
|
||||
existing.tokensOut += row.tokens.output
|
||||
existing.requests += row.requests
|
||||
if (row.cost !== undefined) {
|
||||
existing.cost = (existing.cost ?? 0) + row.cost
|
||||
}
|
||||
existing.reported &&= row.source === 'reported'
|
||||
byKey.set(row.key, existing)
|
||||
}
|
||||
return [...byKey.values()].sort((a, b) => (b.cost ?? 0) - (a.cost ?? 0))
|
||||
})
|
||||
</script>
|
||||
|
||||
<SettingCard
|
||||
label={scope === 'self' ? `Your AI usage in ${workspace}` : 'AI usage'}
|
||||
description={scope === 'self'
|
||||
? "Token spend from your own AI chats in this workspace. Costs are estimated from the workspace's model rates unless the provider reported one."
|
||||
: "Token spend across this workspace's AI chats, grouped by day, user or model. Costs are estimated from the workspace's effective model rates unless the provider reported one."}
|
||||
>
|
||||
<div class="flex flex-col gap-3">
|
||||
<div class="flex flex-row items-center justify-between gap-3 flex-wrap">
|
||||
<div class="flex flex-row items-center gap-2 flex-wrap">
|
||||
<div class="w-40">
|
||||
<Select items={rangeOptions} bind:value={days} />
|
||||
</div>
|
||||
<ToggleButtonGroup noWFull bind:selected={groupBy}>
|
||||
{#snippet children({ item })}
|
||||
<ToggleButton value="day" label="By day" {item} />
|
||||
{#if scope !== 'self'}
|
||||
<ToggleButton value="user" label="By user" {item} />
|
||||
{/if}
|
||||
<ToggleButton value="model" label="By model" {item} />
|
||||
{/snippet}
|
||||
</ToggleButtonGroup>
|
||||
</div>
|
||||
{#if !usage.loading && !usage.error && rows.length > 0}
|
||||
<div
|
||||
class="flex flex-row items-baseline gap-2"
|
||||
title={priced.hasUnpriced
|
||||
? 'Models with no rate set are not counted, so this figure is lower than the real spend.'
|
||||
: undefined}
|
||||
>
|
||||
<span class="text-lg font-semibold tabular-nums">
|
||||
{priced.total === 0 && priced.hasUnpriced
|
||||
? '—'
|
||||
: `${totalIsEstimated ? '~' : ''}${formatUsd(priced.total)}`}
|
||||
</span>
|
||||
<span class="text-xs text-tertiary">
|
||||
{usage.current?.truncated ? 'across the rows below' : 'total'}{priced.hasUnpriced
|
||||
? ' (partial)'
|
||||
: ''}
|
||||
</span>
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
{#if usage.loading}
|
||||
<p class="text-xs text-tertiary">Loading…</p>
|
||||
{:else if usage.error}
|
||||
<p class="text-xs text-tertiary">{usageError(usage.error)}</p>
|
||||
{:else if rows.length === 0}
|
||||
<p class="text-xs text-tertiary">No AI usage recorded in this period.</p>
|
||||
{:else}
|
||||
{#if usage.current?.truncated}
|
||||
<p class="text-xs text-tertiary">
|
||||
More rows matched than are shown; the highest-volume ones are listed. Narrow the range
|
||||
or group differently to see the rest.
|
||||
</p>
|
||||
{/if}
|
||||
<DataTable size="sm" noBorder={false} rounded={true}>
|
||||
<Head>
|
||||
<tr>
|
||||
<Cell head first>{groupBy}</Cell>
|
||||
<Cell head numeric>In</Cell>
|
||||
<Cell head numeric>Out</Cell>
|
||||
<Cell head numeric>Requests</Cell>
|
||||
<Cell head numeric last>
|
||||
<span class="inline-flex flex-row items-center gap-1">
|
||||
Cost
|
||||
<Tooltip small placement="left">
|
||||
{#snippet text()}
|
||||
A cost marked ~ is estimated from this workspace's model rates. A cost
|
||||
without one was returned by the provider's API for those requests, and is used
|
||||
as is. "no rate" means the model has no price set, so its spend stays out of
|
||||
the total.
|
||||
{/snippet}
|
||||
</Tooltip>
|
||||
</span>
|
||||
</Cell>
|
||||
</tr>
|
||||
</Head>
|
||||
<tbody>
|
||||
{#each rows as row (row.key)}
|
||||
<tr class="border-b last:border-b-0">
|
||||
<Cell first class="font-mono truncate max-w-xs text-primary">{row.key}</Cell>
|
||||
<Cell numeric class="tabular-nums text-secondary"
|
||||
>{formatTokenCount(row.tokensIn)}</Cell
|
||||
>
|
||||
<Cell numeric class="tabular-nums text-secondary"
|
||||
>{formatTokenCount(row.tokensOut)}</Cell
|
||||
>
|
||||
<Cell numeric class="tabular-nums text-secondary">{row.requests}</Cell>
|
||||
<Cell numeric last class="tabular-nums text-primary">
|
||||
{row.cost === undefined
|
||||
? 'no rate'
|
||||
: `${row.reported ? '' : '~'}${formatUsd(row.cost)}`}
|
||||
</Cell>
|
||||
</tr>
|
||||
{/each}
|
||||
</tbody>
|
||||
</DataTable>
|
||||
{/if}
|
||||
</div>
|
||||
</SettingCard>
|
||||
@@ -0,0 +1,252 @@
|
||||
<script lang="ts">
|
||||
import type { AIConfig, AIProvider, ModelPriceOverride } from '$lib/gen'
|
||||
import { Badge, Button } from '../common'
|
||||
import TextInput from '../text_input/TextInput.svelte'
|
||||
import { getKnownModelPrice, inheritedCacheRates } from '../copilot/modelPricing'
|
||||
import { modelKey } from '../copilot/modelConfig'
|
||||
import { stripLegacyThinkingSuffix } from '../copilot/reasoningRegistry'
|
||||
import { ChevronDown, ChevronUp } from 'lucide-svelte'
|
||||
import { slide } from 'svelte/transition'
|
||||
import SettingCard from '../instanceSettings/SettingCard.svelte'
|
||||
|
||||
// A rate above this is far more likely a unit mistake (per-token instead of
|
||||
// per-million) than a real price, and a wrong rate silently inflates every
|
||||
// figure derived from it.
|
||||
const MAX_RATE = 1000
|
||||
|
||||
let {
|
||||
aiProviders,
|
||||
modelPricing = $bindable()
|
||||
}: {
|
||||
aiProviders: Exclude<AIConfig['providers'], undefined>
|
||||
modelPricing: Record<string, ModelPriceOverride>
|
||||
} = $props()
|
||||
|
||||
let errors = $state<Record<string, string>>({})
|
||||
let collapsedProviders = $state<Record<string, boolean>>({})
|
||||
|
||||
// Rates are keyed by the id the chat reports usage under, and `setCopilotInfo`
|
||||
// strips the deprecated `/thinking` suffix before the chat ever sees a model. A
|
||||
// row built from the raw config would save an override under a key nothing
|
||||
// reports, so it would sit in settings looking applied and never price anything.
|
||||
// Stripping can collapse two configured slots onto one model, hence the dedupe.
|
||||
const modelsByProvider = $derived(
|
||||
Object.entries(aiProviders).reduce(
|
||||
(acc, [provider, config]) => {
|
||||
const seen = new Set<string>()
|
||||
acc[provider] = config.models.flatMap((configured) => {
|
||||
const model = stripLegacyThinkingSuffix(configured)
|
||||
if (seen.has(model)) {
|
||||
return []
|
||||
}
|
||||
seen.add(model)
|
||||
return [{ provider: provider as AIProvider, model }]
|
||||
})
|
||||
return acc
|
||||
},
|
||||
{} as Record<string, Array<{ provider: AIProvider; model: string }>>
|
||||
)
|
||||
)
|
||||
|
||||
type Field = 'input' | 'output' | 'cache_read' | 'cache_write'
|
||||
type CacheField = Extract<Field, 'cache_read' | 'cache_write'>
|
||||
type Rates = { input: number; output: number; cache_read?: number; cache_write?: number }
|
||||
|
||||
const RATE_FIELDS: Field[] = ['input', 'output', 'cache_read', 'cache_write']
|
||||
|
||||
// Show what a blank cache rate falls back to, so the inherited figure is visible
|
||||
// rather than implied. Read from the resolver's own helper: a placeholder that
|
||||
// computed the rule separately would drift from the price actually charged.
|
||||
function inheritedCacheRate(model: string, field: Field, rates: Rates | undefined): string {
|
||||
if (field !== 'cache_read' && field !== 'cache_write') return '—'
|
||||
const input = rates?.input
|
||||
if (input === undefined) return '—'
|
||||
const inherited = inheritedCacheRates(model, input)
|
||||
return `${+(field === 'cache_read' ? inherited.cacheRead : inherited.cacheWrite).toFixed(4)}`
|
||||
}
|
||||
|
||||
function currentRates(provider: AIProvider, model: string): Rates | undefined {
|
||||
const override = modelPricing[modelKey(provider, model)]
|
||||
if (override) {
|
||||
return {
|
||||
input: override.input,
|
||||
output: override.output,
|
||||
cache_read: override.cache_read,
|
||||
cache_write: override.cache_write
|
||||
}
|
||||
}
|
||||
const builtin = getKnownModelPrice(model)
|
||||
return builtin ? { input: builtin.input, output: builtin.output } : undefined
|
||||
}
|
||||
|
||||
function isOverridden(provider: AIProvider, model: string): boolean {
|
||||
return modelPricing[modelKey(provider, model)] !== undefined
|
||||
}
|
||||
|
||||
function updateRate(provider: AIProvider, model: string, field: Field, value: number) {
|
||||
const key = modelKey(provider, model)
|
||||
if (!(value >= 0) || value > MAX_RATE) {
|
||||
errors[key] = `Rate must be between 0 and ${MAX_RATE}`
|
||||
return
|
||||
}
|
||||
// An edit to either field pins both: a half-specified override would leave
|
||||
// the other rate silently tracking a built-in price the admin did not choose.
|
||||
const current = currentRates(provider, model) ?? { input: 0, output: 0 }
|
||||
modelPricing = {
|
||||
...modelPricing,
|
||||
[key]: { ...modelPricing[key], input: current.input, output: current.output, [field]: value }
|
||||
}
|
||||
errors[key] = ''
|
||||
}
|
||||
|
||||
// Emptying a cache field means "inherit again", so the key has to go: leaving the
|
||||
// old number in the override would keep charging it while the field shows the
|
||||
// inherited placeholder. Input and output are required, so an empty one is not a
|
||||
// state the override can hold; the input snaps back to the stored value on blur.
|
||||
function clearCacheRate(provider: AIProvider, model: string, field: CacheField) {
|
||||
const key = modelKey(provider, model)
|
||||
const override = modelPricing[key]
|
||||
if (!override || override[field] === undefined) {
|
||||
return
|
||||
}
|
||||
const next = { ...override }
|
||||
delete next[field]
|
||||
modelPricing = { ...modelPricing, [key]: next }
|
||||
errors[key] = ''
|
||||
}
|
||||
|
||||
function resetModel(provider: AIProvider, model: string) {
|
||||
const key = modelKey(provider, model)
|
||||
const next = { ...modelPricing }
|
||||
delete next[key]
|
||||
modelPricing = next
|
||||
errors[key] = ''
|
||||
}
|
||||
|
||||
function toggleProvider(provider: string) {
|
||||
collapsedProviders[provider] = !collapsedProviders[provider]
|
||||
}
|
||||
|
||||
function hasOverrides(provider: string, models: Array<{ model: string }>): boolean {
|
||||
return models.some((m) => isOverridden(provider as AIProvider, m.model))
|
||||
}
|
||||
|
||||
$effect(() => {
|
||||
collapsedProviders = {
|
||||
...Object.fromEntries(Object.keys(aiProviders).map((provider) => [provider, true]))
|
||||
}
|
||||
})
|
||||
</script>
|
||||
|
||||
{#if Object.keys(aiProviders).length > 0}
|
||||
<SettingCard
|
||||
label="Model pricing"
|
||||
description="Rates in USD per million tokens, used to cost AI chat usage. Built-in list prices are a best-effort snapshot you can adjust."
|
||||
tooltip="Rates apply only where the provider returned no cost of its own; a cost it returned is used as is. A model with no built-in price starts empty and reports usage without a cost until you set one. An empty cache rate uses the figure shown in the field, and a rate of 0 prices those tokens as free."
|
||||
>
|
||||
<div class="flex flex-col gap-3">
|
||||
{#each Object.entries(modelsByProvider).filter(([_, models]) => models.length > 0) as [provider, models]}
|
||||
{@const isExpanded = !collapsedProviders[provider]}
|
||||
<div class="border rounded-md bg-surface-tertiary">
|
||||
<Button
|
||||
variant="subtle"
|
||||
unifiedSize="sm"
|
||||
onclick={() => toggleProvider(provider)}
|
||||
wrapperClasses="w-full"
|
||||
btnClasses="w-full px-4 min-h-8 justify-between rounded-t-md rounded-b-none"
|
||||
endIcon={{ icon: isExpanded ? ChevronUp : ChevronDown }}
|
||||
>
|
||||
<div class="flex items-center gap-2">
|
||||
<h4 class="font-medium text-xs capitalize">{provider}</h4>
|
||||
{#if hasOverrides(provider, models)}
|
||||
<Badge color="blue">Modified</Badge>
|
||||
{/if}
|
||||
</div>
|
||||
</Button>
|
||||
|
||||
{#if isExpanded}
|
||||
<div transition:slide|local={{ duration: 200 }} class="p-4 border-t">
|
||||
<div class="space-y-3">
|
||||
{#each models as { model }}
|
||||
{@const key = modelKey(provider as AIProvider, model)}
|
||||
{@const rates = currentRates(provider as AIProvider, model)}
|
||||
{@const overridden = isOverridden(provider as AIProvider, model)}
|
||||
<div class="flex flex-col gap-1">
|
||||
<div class="flex items-center gap-3 flex-wrap">
|
||||
<!-- Floor the name's width so it keeps a readable share and the rate
|
||||
fields wrap below it, rather than the name collapsing to an
|
||||
ellipsis on a narrow panel. -->
|
||||
<div class="flex-1 min-w-[10rem]">
|
||||
<span class="text-xs text-primary truncate block">{model}</span>
|
||||
</div>
|
||||
<div class="flex items-center gap-3 flex-wrap">
|
||||
{#each RATE_FIELDS as field}
|
||||
<div class="flex items-center gap-1">
|
||||
<span class="text-xs text-secondary whitespace-nowrap">
|
||||
{field.replace('_', ' ')}
|
||||
</span>
|
||||
<div class="w-24">
|
||||
<TextInput
|
||||
value={rates?.[field] ?? ''}
|
||||
size="sm"
|
||||
error={!!errors[key]}
|
||||
inputProps={{
|
||||
type: 'number',
|
||||
min: 0,
|
||||
max: MAX_RATE,
|
||||
step: 0.01,
|
||||
placeholder: inheritedCacheRate(model, field, rates),
|
||||
oninput: (e: Event & { currentTarget: HTMLInputElement }) => {
|
||||
if (e.currentTarget.value === '') {
|
||||
if (field === 'cache_read' || field === 'cache_write') {
|
||||
clearCacheRate(provider as AIProvider, model, field)
|
||||
}
|
||||
return
|
||||
}
|
||||
const value = parseFloat(e.currentTarget.value)
|
||||
if (!isNaN(value)) {
|
||||
updateRate(provider as AIProvider, model, field, value)
|
||||
}
|
||||
},
|
||||
onblur: (e: Event & { currentTarget: HTMLInputElement }) => {
|
||||
// Resync a field the state refused, so what is shown is what is stored.
|
||||
const stored = currentRates(provider as AIProvider, model)?.[
|
||||
field
|
||||
]
|
||||
e.currentTarget.value =
|
||||
stored === undefined ? '' : String(stored)
|
||||
errors[key] = ''
|
||||
}
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
{/each}
|
||||
<span class="text-xs text-secondary whitespace-nowrap">$ / 1M</span>
|
||||
</div>
|
||||
</div>
|
||||
{#if overridden}
|
||||
<div class="text-xs text-tertiary flex flex-row items-center gap-2">
|
||||
<span>Overriding the built-in price</span>
|
||||
<button
|
||||
type="button"
|
||||
class="text-xs text-blue-500 hover:underline"
|
||||
onclick={() => resetModel(provider as AIProvider, model)}
|
||||
>
|
||||
Reset
|
||||
</button>
|
||||
</div>
|
||||
{/if}
|
||||
{#if errors[key]}
|
||||
<div class="text-xs text-red-500">{errors[key]}</div>
|
||||
{/if}
|
||||
</div>
|
||||
{/each}
|
||||
</div>
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
{/each}
|
||||
</div>
|
||||
</SettingCard>
|
||||
{/if}
|
||||
@@ -0,0 +1,154 @@
|
||||
import { get } from 'svelte/store'
|
||||
import { OpenAPI } from '$lib/gen'
|
||||
import { workspaceStore } from '$lib/stores'
|
||||
|
||||
// Per-workspace AI token spend, batched into the backend `ai_token_usage`
|
||||
// accumulator that powers the workspace and per-user usage views.
|
||||
//
|
||||
// Deliberately separate from `featureUsage.ts`: that buffer carries anonymous
|
||||
// product telemetry that leaves the instance, and its events must not identify a
|
||||
// user. These events are attributed to the caller (server-side, from the session)
|
||||
// and never leave the instance, so the two must not share a transport.
|
||||
//
|
||||
// Only token counts are sent. Money is derived when the usage is read, from the
|
||||
// price table plus the workspace's overrides, so correcting a rate also corrects
|
||||
// history. The one exception is a cost the provider itself billed back.
|
||||
|
||||
export interface AiUsageEvent {
|
||||
provider: string
|
||||
model: string
|
||||
/** Empty for chats not attached to an AI session. */
|
||||
sessionId?: string
|
||||
inputTokens: number
|
||||
cacheReadTokens: number
|
||||
cacheWriteTokens: number
|
||||
outputTokens: number
|
||||
/** Set only where the provider reports what it actually charged, in USD. */
|
||||
costUsd?: number
|
||||
/** Workspace whose API route carries the batch; defaults to the active workspace. */
|
||||
workspace?: string
|
||||
}
|
||||
|
||||
interface AiUsageEventPayload {
|
||||
provider: string
|
||||
model: string
|
||||
session_id: string
|
||||
input_tokens: number
|
||||
cache_read_tokens: number
|
||||
cache_write_tokens: number
|
||||
output_tokens: number
|
||||
reported_cost_nano_usd?: number
|
||||
requests: number
|
||||
}
|
||||
|
||||
const FLUSH_INTERVAL_MS = 15_000
|
||||
// Backend caps a batch at 50 events; chunk larger flushes.
|
||||
const MAX_EVENTS_PER_REQUEST = 50
|
||||
|
||||
const NANO_USD_PER_USD = 1_000_000_000
|
||||
|
||||
// One accumulator per (workspace, provider, model, session): a chat that sends
|
||||
// several turns before a flush produces one upsert instead of one per turn.
|
||||
const pending = new Map<string, { workspace: string; event: AiUsageEventPayload }>()
|
||||
let timer: ReturnType<typeof setTimeout> | undefined
|
||||
|
||||
/**
|
||||
* Record AI token spend. Fire-and-forget: events are summed locally and flushed
|
||||
* in batches.
|
||||
*/
|
||||
export function logAiUsage(event: AiUsageEvent): void {
|
||||
const workspace = event.workspace ?? get(workspaceStore) ?? undefined
|
||||
if (!workspace) return
|
||||
const sessionId = event.sessionId ?? ''
|
||||
const mapKey = JSON.stringify([workspace, event.provider, event.model, sessionId])
|
||||
const existing = pending.get(mapKey)?.event
|
||||
const target: AiUsageEventPayload = existing ?? {
|
||||
provider: event.provider,
|
||||
model: event.model,
|
||||
session_id: sessionId,
|
||||
input_tokens: 0,
|
||||
cache_read_tokens: 0,
|
||||
cache_write_tokens: 0,
|
||||
output_tokens: 0,
|
||||
requests: 0
|
||||
}
|
||||
target.input_tokens += Math.max(0, Math.round(event.inputTokens))
|
||||
target.cache_read_tokens += Math.max(0, Math.round(event.cacheReadTokens))
|
||||
target.cache_write_tokens += Math.max(0, Math.round(event.cacheWriteTokens))
|
||||
target.output_tokens += Math.max(0, Math.round(event.outputTokens))
|
||||
target.requests += 1
|
||||
if (event.costUsd !== undefined) {
|
||||
target.reported_cost_nano_usd =
|
||||
(target.reported_cost_nano_usd ?? 0) +
|
||||
Math.max(0, Math.round(event.costUsd * NANO_USD_PER_USD))
|
||||
}
|
||||
pending.set(mapKey, { workspace, event: target })
|
||||
|
||||
if (timer === undefined) {
|
||||
timer = setTimeout(() => {
|
||||
timer = undefined
|
||||
void flushAiUsage()
|
||||
}, FLUSH_INTERVAL_MS)
|
||||
}
|
||||
}
|
||||
|
||||
export async function flushAiUsage(): Promise<void> {
|
||||
if (timer !== undefined) {
|
||||
clearTimeout(timer)
|
||||
timer = undefined
|
||||
}
|
||||
if (pending.size === 0) return
|
||||
|
||||
const byWorkspace = new Map<string, AiUsageEventPayload[]>()
|
||||
for (const { workspace, event } of pending.values()) {
|
||||
let events = byWorkspace.get(workspace)
|
||||
if (!events) {
|
||||
events = []
|
||||
byWorkspace.set(workspace, events)
|
||||
}
|
||||
events.push(event)
|
||||
}
|
||||
pending.clear()
|
||||
|
||||
// Start every chunk request synchronously before awaiting: the pagehide flush
|
||||
// only protects requests that were already issued (keepalive can't help a fetch
|
||||
// that never started).
|
||||
const inflight: Promise<void>[] = []
|
||||
for (const [workspace, events] of byWorkspace) {
|
||||
for (let i = 0; i < events.length; i += MAX_EVENTS_PER_REQUEST) {
|
||||
inflight.push(send(workspace, events.slice(i, i + MAX_EVENTS_PER_REQUEST)))
|
||||
}
|
||||
}
|
||||
await Promise.all(inflight)
|
||||
}
|
||||
|
||||
async function send(workspace: string, events: AiUsageEventPayload[]): Promise<void> {
|
||||
try {
|
||||
// Raw fetch instead of the generated client: `keepalive` lets the request
|
||||
// finish after tab close/navigation, which is when the final flush runs.
|
||||
// Auth rides on the token cookie (WITH_CREDENTIALS app setup).
|
||||
await fetch(`${OpenAPI.BASE}/w/${encodeURIComponent(workspace)}/ai/usage`, {
|
||||
method: 'POST',
|
||||
credentials: 'include',
|
||||
keepalive: true,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ events })
|
||||
})
|
||||
} catch {
|
||||
// Accounting is best-effort: a dropped batch under-reports spend, which is
|
||||
// better than surfacing a network error in the middle of a chat.
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof document !== 'undefined') {
|
||||
// Flush what's buffered before the tab goes away. pagehide covers
|
||||
// close/navigation paths where visibilitychange is not delivered.
|
||||
document.addEventListener('visibilitychange', () => {
|
||||
if (document.visibilityState === 'hidden') {
|
||||
void flushAiUsage()
|
||||
}
|
||||
})
|
||||
window.addEventListener('pagehide', () => {
|
||||
void flushAiUsage()
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user