feat: track token cost in AI sessions and chats

This commit is contained in:
hugocasa
2026-08-13 19:15:39 +02:00
parent 71b9989daa
commit c3fab84cb5
26 changed files with 1624 additions and 64 deletions
@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO ai_token_usage (workspace_id, email, provider, model, session_id, input_tokens, cache_read_tokens, cache_write_tokens, output_tokens, reported_cost_nano_usd, requests)\n SELECT $1, $2, * FROM UNNEST($3::text[], $4::text[], $5::text[], $6::bigint[], $7::bigint[], $8::bigint[], $9::bigint[], $10::bigint[], $11::bigint[])\n ON CONFLICT (workspace_id, day, email, provider, model, session_id)\n DO UPDATE SET\n input_tokens = ai_token_usage.input_tokens + EXCLUDED.input_tokens,\n cache_read_tokens = ai_token_usage.cache_read_tokens + EXCLUDED.cache_read_tokens,\n cache_write_tokens = ai_token_usage.cache_write_tokens + EXCLUDED.cache_write_tokens,\n output_tokens = ai_token_usage.output_tokens + EXCLUDED.output_tokens,\n reported_cost_nano_usd = CASE\n WHEN EXCLUDED.reported_cost_nano_usd IS NULL\n THEN ai_token_usage.reported_cost_nano_usd\n ELSE COALESCE(ai_token_usage.reported_cost_nano_usd, 0)\n + EXCLUDED.reported_cost_nano_usd\n END,\n requests = ai_token_usage.requests + EXCLUDED.requests,\n updated_at = now()",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"TextArray",
"TextArray",
"TextArray",
"Int8Array",
"Int8Array",
"Int8Array",
"Int8Array",
"Int8Array",
"Int8Array"
]
},
"nullable": []
},
"hash": "24fcc2b69f30953915f0cbf246e1c19b2310075a644d81c21784c991e52b4001"
}
@@ -0,0 +1,72 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n (CASE $3::text\n WHEN 'day' THEN day::text\n WHEN 'user' THEN email\n WHEN 'session' THEN session_id\n ELSE ''\n END) AS \"key!\",\n provider AS \"provider!\",\n model AS \"model!\",\n SUM(input_tokens)::bigint AS \"input_tokens!\",\n SUM(cache_read_tokens)::bigint AS \"cache_read_tokens!\",\n SUM(cache_write_tokens)::bigint AS \"cache_write_tokens!\",\n SUM(output_tokens)::bigint AS \"output_tokens!\",\n SUM(reported_cost_nano_usd)::bigint AS \"reported_cost_nano_usd\",\n SUM(requests)::bigint AS \"requests!\"\n FROM ai_token_usage\n WHERE workspace_id = $1 AND day >= CURRENT_DATE - $2::int\n GROUP BY 1, provider, model\n ORDER BY 1 DESC, SUM(input_tokens + output_tokens) DESC\n LIMIT 1000",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "key!",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "provider!",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "model!",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "input_tokens!",
"type_info": "Int8"
},
{
"ordinal": 4,
"name": "cache_read_tokens!",
"type_info": "Int8"
},
{
"ordinal": 5,
"name": "cache_write_tokens!",
"type_info": "Int8"
},
{
"ordinal": 6,
"name": "output_tokens!",
"type_info": "Int8"
},
{
"ordinal": 7,
"name": "reported_cost_nano_usd",
"type_info": "Int8"
},
{
"ordinal": 8,
"name": "requests!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"Int4",
"Text"
]
},
"nullable": [
null,
false,
false,
null,
null,
null,
null,
null,
null
]
},
"hash": "43ec1c4adf6e453848b532bf3fab5de8825d247ea0b337007ee43c9cdffac83d"
}
@@ -0,0 +1 @@
DROP TABLE ai_token_usage;
@@ -0,0 +1,42 @@
-- Per-workspace AI token spend, accumulated from the chat client. Rows hold token
-- counts rather than money: prices live in the frontend price table plus the
-- workspace's `ai_config.model_pricing` overrides and are applied at read time, so
-- correcting a price also corrects the history. `reported_cost_nano_usd` is the
-- exception — a few providers (OpenRouter) return what they actually charged, and
-- that figure wins over the estimate.
--
-- Distinct from `feature_usage`, which is anonymous telemetry that leaves the
-- instance and is pruned after 60 days; spend is per-user and kept.
CREATE TABLE ai_token_usage (
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE,
day DATE NOT NULL DEFAULT CURRENT_DATE,
email VARCHAR(255) NOT NULL,
provider VARCHAR(50) NOT NULL,
model VARCHAR(255) NOT NULL,
-- Empty for chats that are not attached to an AI session.
session_id VARCHAR(50) NOT NULL DEFAULT '',
-- Uncached input only; the two cache columns hold the rest of the prompt, so
-- each column maps to exactly one price and they never double-count.
input_tokens BIGINT NOT NULL DEFAULT 0,
cache_read_tokens BIGINT NOT NULL DEFAULT 0,
cache_write_tokens BIGINT NOT NULL DEFAULT 0,
output_tokens BIGINT NOT NULL DEFAULT 0,
reported_cost_nano_usd BIGINT,
requests BIGINT NOT NULL DEFAULT 0,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (workspace_id, day, email, provider, model, session_id)
);
-- The usage listing filters on workspace and a date range; the PK only reaches
-- `day` through `email`, so it cannot serve that on its own.
CREATE INDEX idx_ai_token_usage_ws_day ON ai_token_usage (workspace_id, day DESC);
GRANT ALL ON ai_token_usage TO windmill_admin;
GRANT ALL ON ai_token_usage TO windmill_user;
-- Both handlers go through the raw pool, so no policy is needed for them to work.
-- Enabling RLS with an admin-only policy is the backstop: a future query that
-- reaches this table through UserDB sees nothing rather than every user's spend.
ALTER TABLE ai_token_usage ENABLE ROW LEVEL SECURITY;
CREATE POLICY admin_policy ON ai_token_usage FOR ALL TO windmill_admin USING (true);
+130
View File
@@ -11838,6 +11838,58 @@ paths:
schema:
type: string
/w/{workspace}/ai/usage:
post:
summary: record AI token usage for the calling user
operationId: recordAiUsage
tags:
- ai
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
required: true
content:
application/json:
schema:
type: object
required:
- events
properties:
events:
type: array
items:
$ref: "#/components/schemas/AITokenUsageEvent"
responses:
"204":
description: usage recorded
get:
summary: list aggregated AI token usage for the workspace (admin only)
operationId: listAiUsage
tags:
- ai
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- name: days
in: query
schema:
type: integer
minimum: 1
maximum: 365
- name: group_by
in: query
schema:
type: string
enum: [day, user, model, session]
responses:
"200":
description: usage buckets
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/AITokenUsageBucket"
/w/{workspace}/ai_skills/list:
get:
summary: list the workspace AI chat skills (name + description only)
@@ -25999,6 +26051,84 @@ components:
type: integer
minimum: 1
maximum: 2000000
model_pricing:
type: object
additionalProperties:
$ref: "#/components/schemas/ModelPriceOverride"
ModelPriceOverride:
type: object
description: negotiated rates in USD per million tokens, keyed `provider:model`
properties:
input:
type: number
output:
type: number
cache_read:
type: number
cache_write:
type: number
required:
- input
- output
AITokenUsageEvent:
type: object
properties:
provider:
$ref: "#/components/schemas/AIProvider"
model:
type: string
session_id:
type: string
input_tokens:
type: integer
cache_read_tokens:
type: integer
cache_write_tokens:
type: integer
output_tokens:
type: integer
reported_cost_nano_usd:
type: integer
description: only set by providers that bill back an exact figure
requests:
type: integer
required:
- provider
- model
AITokenUsageBucket:
type: object
properties:
key:
type: string
description: the grouped dimension's value; empty when grouping by model
provider:
type: string
model:
type: string
input_tokens:
type: integer
cache_read_tokens:
type: integer
cache_write_tokens:
type: integer
output_tokens:
type: integer
reported_cost_nano_usd:
type: integer
requests:
type: integer
required:
- key
- provider
- model
- input_tokens
- cache_read_tokens
- cache_write_tokens
- output_tokens
- requests
InstanceAIProviderSummary:
type: object
+250 -5
View File
@@ -3,11 +3,16 @@ use crate::utils::check_scopes;
#[cfg(feature = "bedrock")]
use axum::routing::get;
#[cfg(feature = "bedrock")]
use axum::Json;
use axum::{body::Bytes, extract::Path, response::IntoResponse, routing::post, Extension, Router};
use axum::{
body::Bytes,
extract::{Path, Query},
response::IntoResponse,
routing::post,
Extension, Router,
};
use futures::StreamExt;
use http::{HeaderMap, Method};
use http::{HeaderMap, Method, StatusCode};
use quick_cache::sync::Cache;
use reqwest::{Client, RequestBuilder};
use serde::{Deserialize, Serialize};
@@ -37,7 +42,7 @@ use windmill_ai::proxy::{
use windmill_audit::{audit_oss::audit_log, ActionKind};
use windmill_common::db::UserDB;
use windmill_common::error::{to_anyhow, Error, Result};
use windmill_common::utils::configure_client;
use windmill_common::utils::{configure_client, require_admin};
use windmill_common::variables::{get_variable_or_self, get_variable_or_self_as};
// AI timeout configuration constants
@@ -417,6 +422,23 @@ pub struct AIConfig {
pub custom_prompts: Option<HashMap<String, String>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub max_tokens_per_model: Option<HashMap<String, i32>>,
/// Per-model price overrides, keyed `provider:model` like `max_tokens_per_model`.
/// Only models whose rates differ from the built-in table are stored.
#[serde(skip_serializing_if = "Option::is_none")]
pub model_pricing: Option<HashMap<String, ModelPriceOverride>>,
}
/// Negotiated rates in USD per million tokens. Cache rates fall back to the
/// provider's usual multiples of the input rate when left unset, so an admin who
/// only knows their input/output pricing does not have to invent the other two.
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct ModelPriceOverride {
pub input: f64,
pub output: f64,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_read: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_write: Option<f64>,
}
impl AIConfig {
@@ -432,7 +454,9 @@ pub fn global_service() -> Router {
}
pub fn workspaced_service() -> Router {
let router = Router::new().route("/proxy/{*ai}", post(proxy).get(proxy));
let router = Router::new()
.route("/proxy/{*ai}", post(proxy).get(proxy))
.route("/usage", post(record_ai_usage).get(list_ai_usage));
#[cfg(feature = "bedrock")]
let router = router.route("/check_bedrock_credentials", get(check_bedrock_credentials));
@@ -440,6 +464,227 @@ pub fn workspaced_service() -> Router {
router
}
/// One provider request's worth of tokens, as counted by the chat client.
#[derive(Deserialize)]
struct AIUsageEvent {
provider: String,
model: String,
#[serde(default)]
session_id: String,
#[serde(default)]
input_tokens: i64,
#[serde(default)]
cache_read_tokens: i64,
#[serde(default)]
cache_write_tokens: i64,
#[serde(default)]
output_tokens: i64,
/// Only the providers that bill back an exact figure set this.
#[serde(default)]
reported_cost_nano_usd: Option<i64>,
#[serde(default)]
requests: Option<i64>,
}
#[derive(Deserialize)]
struct RecordAIUsagePayload {
events: Vec<AIUsageEvent>,
}
const MAX_AI_USAGE_EVENTS: usize = 50;
/// Well above any single conversation and far below an i64 overflow, so a client
/// bug caps out at one absurd row instead of poisoning the running total.
const MAX_TOKENS_PER_EVENT: i64 = 100_000_000;
/// $1000 in nano-USD.
const MAX_REPORTED_COST_PER_EVENT: i64 = 1_000_000_000_000;
/// Model ids carry vendor prefixes and variant suffixes (`anthropic/claude-opus-5:thinking`),
/// so the shape check is looser than an identifier but still excludes whitespace and
/// anything that would not be a model id.
fn is_model_shaped(s: &str, max_len: usize) -> bool {
!s.is_empty()
&& s.len() <= max_len
&& s.chars()
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | ':' | '.' | '/' | '~'))
}
/// Accumulate one workspace's AI token spend. Values are clamped and the caller's
/// email comes from the session, never the payload — the client is trusted to
/// report its own usage, not to attribute it to someone else.
async fn record_ai_usage(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Json(payload): Json<RecordAIUsagePayload>,
) -> Result<StatusCode> {
// Pre-sum duplicate keys: two rows hitting the same conflict target in a single
// INSERT error out ("cannot affect row a second time").
let mut agg: HashMap<(String, String, String), AIUsageTotals> = HashMap::new();
for e in payload.events.into_iter().take(MAX_AI_USAGE_EVENTS) {
if AIProvider::try_from(e.provider.as_str()).is_err()
|| !is_model_shaped(&e.model, 255)
|| !(e.session_id.is_empty() || is_model_shaped(&e.session_id, 50))
{
continue;
}
let totals = agg
.entry((e.provider, e.model, e.session_id))
.or_insert_with(AIUsageTotals::default);
totals.input += e.input_tokens.clamp(0, MAX_TOKENS_PER_EVENT);
totals.cache_read += e.cache_read_tokens.clamp(0, MAX_TOKENS_PER_EVENT);
totals.cache_write += e.cache_write_tokens.clamp(0, MAX_TOKENS_PER_EVENT);
totals.output += e.output_tokens.clamp(0, MAX_TOKENS_PER_EVENT);
totals.requests += e.requests.unwrap_or(1).clamp(0, MAX_AI_USAGE_EVENTS as i64);
if let Some(cost) = e.reported_cost_nano_usd {
totals.reported_cost = Some(
totals.reported_cost.unwrap_or(0) + cost.clamp(0, MAX_REPORTED_COST_PER_EVENT),
);
}
}
if agg.is_empty() {
return Ok(StatusCode::NO_CONTENT);
}
let mut providers = Vec::with_capacity(agg.len());
let mut models = Vec::with_capacity(agg.len());
let mut session_ids = Vec::with_capacity(agg.len());
let mut inputs = Vec::with_capacity(agg.len());
let mut cache_reads = Vec::with_capacity(agg.len());
let mut cache_writes = Vec::with_capacity(agg.len());
let mut outputs = Vec::with_capacity(agg.len());
let mut reported_costs: Vec<Option<i64>> = Vec::with_capacity(agg.len());
let mut requests = Vec::with_capacity(agg.len());
for ((provider, model, session_id), totals) in agg {
providers.push(provider);
models.push(model);
session_ids.push(session_id);
inputs.push(totals.input);
cache_reads.push(totals.cache_read);
cache_writes.push(totals.cache_write);
outputs.push(totals.output);
reported_costs.push(totals.reported_cost);
requests.push(totals.requests);
}
sqlx::query!(
"INSERT INTO ai_token_usage (workspace_id, email, provider, model, session_id, \
input_tokens, cache_read_tokens, cache_write_tokens, output_tokens, \
reported_cost_nano_usd, requests)
SELECT $1, $2, * FROM UNNEST($3::text[], $4::text[], $5::text[], $6::bigint[], \
$7::bigint[], $8::bigint[], $9::bigint[], $10::bigint[], $11::bigint[])
ON CONFLICT (workspace_id, day, email, provider, model, session_id)
DO UPDATE SET
input_tokens = ai_token_usage.input_tokens + EXCLUDED.input_tokens,
cache_read_tokens = ai_token_usage.cache_read_tokens + EXCLUDED.cache_read_tokens,
cache_write_tokens = ai_token_usage.cache_write_tokens + EXCLUDED.cache_write_tokens,
output_tokens = ai_token_usage.output_tokens + EXCLUDED.output_tokens,
reported_cost_nano_usd = CASE
WHEN EXCLUDED.reported_cost_nano_usd IS NULL
THEN ai_token_usage.reported_cost_nano_usd
ELSE COALESCE(ai_token_usage.reported_cost_nano_usd, 0)
+ EXCLUDED.reported_cost_nano_usd
END,
requests = ai_token_usage.requests + EXCLUDED.requests,
updated_at = now()",
&w_id,
&authed.email,
&providers,
&models,
&session_ids,
&inputs,
&cache_reads,
&cache_writes,
&outputs,
&reported_costs as &[Option<i64>],
&requests
)
.execute(&db)
.await?;
Ok(StatusCode::NO_CONTENT)
}
#[derive(Default)]
struct AIUsageTotals {
input: i64,
cache_read: i64,
cache_write: i64,
output: i64,
reported_cost: Option<i64>,
requests: i64,
}
#[derive(Deserialize)]
struct ListAIUsageQuery {
days: Option<i32>,
group_by: Option<String>,
}
/// A bucket always carries its provider and model: the caller prices it from a
/// per-model rate table, which a bucket spanning several models could not be
/// resolved against.
#[derive(Serialize)]
struct AITokenUsageBucket {
key: String,
provider: String,
model: String,
input_tokens: i64,
cache_read_tokens: i64,
cache_write_tokens: i64,
output_tokens: i64,
reported_cost_nano_usd: Option<i64>,
requests: i64,
}
async fn list_ai_usage(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Query(query): Query<ListAIUsageQuery>,
) -> Result<Json<Vec<AITokenUsageBucket>>> {
require_admin(authed.is_admin, &authed.username)?;
let days = query.days.unwrap_or(30).clamp(1, 365);
let group_by = query.group_by.as_deref().unwrap_or("day");
if !matches!(group_by, "day" | "user" | "model" | "session") {
return Err(Error::BadRequest(format!(
"Unsupported group_by: {}",
group_by
)));
}
let rows = sqlx::query_as!(
AITokenUsageBucket,
r#"SELECT
(CASE $3::text
WHEN 'day' THEN day::text
WHEN 'user' THEN email
WHEN 'session' THEN session_id
ELSE ''
END) AS "key!",
provider AS "provider!",
model AS "model!",
SUM(input_tokens)::bigint AS "input_tokens!",
SUM(cache_read_tokens)::bigint AS "cache_read_tokens!",
SUM(cache_write_tokens)::bigint AS "cache_write_tokens!",
SUM(output_tokens)::bigint AS "output_tokens!",
SUM(reported_cost_nano_usd)::bigint AS "reported_cost_nano_usd",
SUM(requests)::bigint AS "requests!"
FROM ai_token_usage
WHERE workspace_id = $1 AND day >= CURRENT_DATE - $2::int
GROUP BY 1, provider, model
ORDER BY 1 DESC, SUM(input_tokens + output_tokens) DESC
LIMIT 1000"#,
&w_id,
days,
group_by
)
.fetch_all(&db)
.await?;
Ok(Json(rows))
}
/// Check if AWS Bedrock credentials are available from environment variables.
#[cfg(feature = "bedrock")]
async fn check_bedrock_credentials(
+11 -1
View File
@@ -3,7 +3,12 @@
// import aiStore back, and such a cycle crashes the app once the bundler splits it across
// chunks (docs/frontend-import-cycles.md; the build fails on the chunk cycle, not on this).
import { writable, get } from 'svelte/store'
import { type AIProviderModel, type AIProvider, type AIConfig } from './gen'
import {
type AIProviderModel,
type AIProvider,
type AIConfig,
type ModelPriceOverride
} from './gen'
import {
aiUserDisabled,
COPILOT_SESSION_MODEL_SETTING_NAME,
@@ -41,6 +46,8 @@ export const copilotInfo = writable<{
aiModels: AIProviderModel[]
customPrompts?: Record<string, string>
maxTokensPerModel?: Record<string, number>
/** Negotiated rates per `provider:model`, overriding the built-in price table. */
modelPricing?: Record<string, ModelPriceOverride>
webSearchEnabledProviders?: Partial<Record<AIProvider, boolean>>
}>({
enabled: false,
@@ -50,6 +57,7 @@ export const copilotInfo = writable<{
aiModels: [],
customPrompts: {},
maxTokensPerModel: {},
modelPricing: {},
webSearchEnabledProviders: {}
})
@@ -124,6 +132,7 @@ export function setCopilotInfo(aiConfig: AIConfig) {
aiModels: aiModels,
customPrompts: aiConfig.custom_prompts ?? {},
maxTokensPerModel: aiConfig.max_tokens_per_model ?? {},
modelPricing: aiConfig.model_pricing ?? {},
webSearchEnabledProviders
})
} else {
@@ -137,6 +146,7 @@ export function setCopilotInfo(aiConfig: AIConfig) {
aiModels: [],
customPrompts: {},
maxTokensPerModel: {},
modelPricing: {},
webSearchEnabledProviders: {}
})
}
@@ -28,6 +28,7 @@
import type { ContextElement } from './context'
import ChatQuickActions from './ChatQuickActions.svelte'
import ContextUsageIndicator from './ContextUsageIndicator.svelte'
import CostIndicator from './CostIndicator.svelte'
import AIChatModelSettings from './AIChatModelSettings.svelte'
import ChatMode from './ChatMode.svelte'
import DatatableCreationPolicy from './DatatableCreationPolicy.svelte'
@@ -981,6 +982,7 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. -->
<DatatableCreationPolicy />
{/if}
<ContextUsageIndicator />
<CostIndicator />
<AIChatModelSettings />
{#if aiChatManager.mode === AIMode.APP && appContext && (appContext.inspectorElement || appContext.codeSelection)}
@@ -99,7 +99,13 @@ import type AIChatInput from './AIChatInput.svelte'
import { prepareApiSystemMessage, prepareApiUserMessage } from './api/core'
import { runChatLoop, truncateToToolPairedPrefix } from './chatLoop'
import { sanitizeToolCallArguments } from './toolCallArguments'
import { normalizeContextUsage } from './tokenUsage'
import {
addModelTokenUsage,
billedTokens,
normalizeContextUsage,
type ModelTokenUsageTotals
} from './tokenUsage'
import { logAiUsage } from '$lib/utils/aiUsageReporter'
import type { ReviewChangesOpts } from './monaco-adapter'
import {
getCurrentModel,
@@ -514,6 +520,11 @@ export class AIChatManager {
* (provider never reported, turn failed, history rewound). Never holds a
* guess: readers go through `contextTokens`, which estimates lazily. */
contextUsage = $state<number | undefined>(undefined)
/** What this conversation has spent, in tokens, per `provider:model`. Unlike
* `contextUsage` this describes money already spent rather than the current
* history, so compaction and rewinds leave it alone — it is cleared only when
* the conversation is (New chat) and restored when one is loaded. */
usageByModel = $state<ModelTokenUsageTotals>({})
// Circuit breaker for summary-based compaction: after repeated failures the
// summary round-trip is skipped in favor of drop-oldest. Reset on any
// successful summarization. Not persisted — a fresh load gets a fresh chance.
@@ -645,6 +656,42 @@ export class AIChatManager {
await this.#persistModifiedItems()
}
/** Plain snapshot for persistence; undefined while nothing has been spent, so
* a chat that never ran a turn stores no usage field at all. */
private usageSnapshot(): ModelTokenUsageTotals | undefined {
const snapshot = $state.snapshot(this.usageByModel) as ModelTokenUsageTotals
return Object.keys(snapshot).length > 0 ? snapshot : undefined
}
/** Fold a completed turn's usage into the conversation's running spend and
* report it for the workspace usage view. Only token counts leave the browser:
* rates are applied when the usage is read, so a corrected price also corrects
* everything already recorded. */
private recordUsage(byModel: ModelTokenUsageTotals | undefined) {
// Accounting must never take a turn down with it: a path that reports no
// per-model breakdown simply records nothing.
for (const entry of Object.values(byModel ?? {})) {
this.usageByModel = addModelTokenUsage(
this.usageByModel,
entry.provider,
entry.model,
entry.usage
)
const tokens = billedTokens(entry.usage)
logAiUsage({
provider: entry.provider,
model: entry.model,
sessionId: this.sessionId,
inputTokens: tokens.input,
cacheReadTokens: tokens.cacheRead,
cacheWriteTokens: tokens.cacheWrite,
outputTokens: tokens.output,
costUsd: entry.usage.cost,
workspace: this.operatingWorkspace
})
}
}
// Serialized, snapshot-at-write-time persistence: two rapid dock actions
// would otherwise race their saveChat writes, and the earlier (staler)
// snapshot could land last — dropping the later mutation until the next
@@ -657,7 +704,8 @@ export class AIChatManager {
this.displayMessages,
this.messages,
this.contextUsage,
this.modifiedItems ? [...this.modifiedItems] : undefined
this.modifiedItems ? [...this.modifiedItems] : undefined,
this.usageSnapshot()
)
// Swallow (and log) a failed write so it can't wedge the queue as a
// rejected link — the next persist snapshots the full current set, so
@@ -961,6 +1009,7 @@ export class AIChatManager {
this.messages,
this.contextUsage,
undefined,
this.usageSnapshot(),
$state.snapshot(this.backgroundJobs)
)
.catch((e) => console.error('Failed to persist background jobs', e))
@@ -1397,7 +1446,8 @@ export class AIChatManager {
this.displayMessages,
this.messages,
this.contextUsage,
this.modifiedItems ? [...this.modifiedItems] : undefined
this.modifiedItems ? [...this.modifiedItems] : undefined,
this.usageSnapshot()
)
sendUserToast('Conversation compacted.')
break
@@ -2343,6 +2393,9 @@ export class AIChatManager {
}
}
})
if (result.tokenUsage.total > 0) {
this.recordUsage(result.tokenUsageByModel)
}
if (this.isSessionChat && this.sessionId && result.tokenUsage.total > 0) {
logFeatureUsage('ai_session', 'tokens', {
entityId: this.sessionId,
@@ -2961,7 +3014,8 @@ export class AIChatManager {
this.displayMessages,
this.messages,
this.contextUsage,
this.modifiedItems ? [...this.modifiedItems] : undefined
this.modifiedItems ? [...this.modifiedItems] : undefined,
this.usageSnapshot()
)
this.replyReveal.reset()
@@ -3010,7 +3064,8 @@ export class AIChatManager {
this.displayMessages,
this.messages,
this.contextUsage,
this.modifiedItems ? [...this.modifiedItems] : undefined
this.modifiedItems ? [...this.modifiedItems] : undefined,
this.usageSnapshot()
)
}
}
@@ -3162,7 +3217,8 @@ export class AIChatManager {
this.displayMessages,
this.messages,
this.contextUsage,
this.modifiedItems ? [...this.modifiedItems] : undefined
this.modifiedItems ? [...this.modifiedItems] : undefined,
this.usageSnapshot()
)
// Still counts as the saved first turn — skipping the hook here would
// permanently miss it (the next turn isn't "first" anymore).
@@ -3211,7 +3267,8 @@ export class AIChatManager {
this.displayMessages,
this.messages,
this.contextUsage,
this.modifiedItems ? [...this.modifiedItems] : undefined
this.modifiedItems ? [...this.modifiedItems] : undefined,
this.usageSnapshot()
)
}
if (!wasAborted) {
@@ -3235,7 +3292,8 @@ export class AIChatManager {
this.displayMessages,
this.messages,
this.contextUsage,
this.modifiedItems ? [...this.modifiedItems] : undefined
this.modifiedItems ? [...this.modifiedItems] : undefined,
this.usageSnapshot()
)
// Only this branch is a clean send: the queued-message flush below
// auto-sends the next message after it (set after saveChat so a
@@ -3296,7 +3354,8 @@ export class AIChatManager {
this.displayMessages,
this.messages,
this.contextUsage,
this.modifiedItems ? [...this.modifiedItems] : undefined
this.modifiedItems ? [...this.modifiedItems] : undefined,
this.usageSnapshot()
)
} catch (saveErr) {
console.error('Failed to persist partial chat after error', saveErr)
@@ -3510,11 +3569,13 @@ export class AIChatManager {
this.displayMessages,
this.messages,
this.contextUsage,
this.modifiedItems ? [...this.modifiedItems] : undefined
this.modifiedItems ? [...this.modifiedItems] : undefined,
this.usageSnapshot()
)
this.displayMessages = []
this.messages = []
this.contextUsage = undefined
this.usageByModel = {}
// The mask belongs to the conversation just saved — the fresh chat starts
// its own (empty) tracking; carrying entries over would claim the previous
// conversation's edits for the new one. Untracked chats stay untracked.
@@ -3545,6 +3606,7 @@ export class AIChatManager {
this.displayMessages = chat.displayMessages
this.messages = chat.actualMessages
this.contextUsage = normalizeContextUsage(chat.contextUsage)
this.usageByModel = chat.usageByModel ? { ...chat.usageByModel } : {}
// Seed the modified-items mask from the stored chat. A session's Edits
// surface is scoped strictly to what this session edited, so it must never
// fall back to showing every draft in the (possibly forked) workspace: a
@@ -1832,7 +1832,13 @@ describe('AIChatManager context compaction', () => {
// compaction-time save) so a rolled-back turn keeps a consistent value
// 4th arg: the modified-items mask rides on every save (undefined here —
// this bare manager never initialised tracking).
expect(saveChat).toHaveBeenCalledWith(expect.anything(), expect.anything(), 650_000, undefined)
expect(saveChat).toHaveBeenCalledWith(
expect.anything(),
expect.anything(),
650_000,
undefined,
undefined
)
// At commit, the no-report turn clears the stored value; the readable
// number falls back to estimating the now-tiny compacted history
expect(manager.contextUsage).toBeUndefined()
@@ -2898,14 +2904,14 @@ describe('AIChatManager background job completion', () => {
expect(saveChat).not.toHaveBeenCalled()
manager.updateJob('job-1', { status: 'success' })
await vi.waitFor(() => expect(saveChat).toHaveBeenCalledTimes(1))
expect(saveChat.mock.calls[0][4]).toEqual([
expect(saveChat.mock.calls[0][5]).toEqual([
expect.objectContaining({ jobId: 'job-1', status: 'success' })
])
// Reviewing persists the flag; re-reviewing is a no-op (no extra write).
manager.markJobsReviewed(['job-1'])
await vi.waitFor(() => expect(saveChat).toHaveBeenCalledTimes(2))
expect(saveChat.mock.calls[1][4]).toEqual([
expect(saveChat.mock.calls[1][5]).toEqual([
expect.objectContaining({ jobId: 'job-1', reviewed: true })
])
manager.markJobsReviewed(['job-1'])
@@ -4,6 +4,7 @@
import { getAiChatManager } from './aiChatManagerContext'
import { AIMode } from './AIChatManager.svelte'
import Tooltip from '$lib/components/meltComponents/Tooltip.svelte'
import { formatTokenCount } from './tokenUsage'
const aiChatManager = getAiChatManager()
@@ -45,16 +46,6 @@
? 'bg-amber-500'
: 'bg-surface-accent-primary'
)
function formatTokenCount(tokens: number): string {
if (tokens >= 1_000_000) {
return `${(tokens / 1_000_000).toFixed(1).replace(/\.0$/, '')}M`
}
if (tokens >= 1000) {
return `${Math.round(tokens / 1000)}k`
}
return `${tokens}`
}
</script>
{#if visible}
@@ -0,0 +1,87 @@
<script lang="ts">
import { copilotInfo } from '$lib/aiStore'
import Tooltip from '$lib/components/meltComponents/Tooltip.svelte'
import { formatUsd, priceSpend, type ModelSpend } from '../modelPricing'
import { getAiChatManager } from './aiChatManagerContext'
import { billedTokens, formatTokenCount } from './tokenUsage'
const aiChatManager = getAiChatManager()
let spend = $derived(
Object.values(aiChatManager.usageByModel).map(
(entry): ModelSpend => ({
provider: entry.provider,
model: entry.model,
tokens: billedTokens(entry.usage),
reportedCostUsd: entry.usage.cost
})
)
)
let priced = $derived(priceSpend(spend, $copilotInfo.modelPricing))
let visible = $derived(priced.rows.length > 0)
// A priced total of zero next to unpriced models would read as "this was free",
// so a chat whose models have no rate shows no figure at all — the tooltip says
// why and where to fix it.
let label = $derived(
priced.total === 0 && priced.hasUnpriced
? '—'
: `${formatUsd(priced.total)}${priced.hasUnpriced ? '+' : ''}`
)
let unpricedModels = $derived(priced.rows.filter((r) => r.cost === undefined))
// Where each number came from, so an estimate is never mistaken for a bill. A
// chat can mix the two (one model priced from a table, another billed back by
// its provider), so the reported ones are marked per row and the caveat below
// covers only the estimated remainder.
let estimatedRows = $derived(
priced.rows.filter((r) => r.source === 'builtin' || r.source === 'override')
)
let estimateSource = $derived(
estimatedRows.some((r) => r.source === 'override')
? 'Estimated from the rates set for this workspace.'
: 'Estimated from list prices.'
)
</script>
{#if visible}
<Tooltip small placement="top">
<div class="flex items-center h-5 px-1 text-2xs tabular-nums text-tertiary">
{label}
</div>
{#snippet text()}
<div class="text-xs whitespace-nowrap">
<p class="font-semibold">
{aiChatManager.isSessionChat ? 'Session cost' : 'Chat cost'}
</p>
<div class="mt-1 flex flex-col gap-1">
{#each priced.rows as row (`${row.provider}:${row.model}`)}
<div>
<p class="font-mono">{row.model}</p>
<p class="tabular-nums text-tertiary">
{formatTokenCount(row.tokens.input)} in
{#if row.tokens.cacheRead > 0 || row.tokens.cacheWrite > 0}
· {formatTokenCount(row.tokens.cacheRead + row.tokens.cacheWrite)} cached
{/if}
· {formatTokenCount(row.tokens.output)} out · {row.cost === undefined
? 'no rate'
: formatUsd(row.cost)}{row.source === 'reported' ? ' billed' : ''}
</p>
</div>
{/each}
</div>
{#if priced.hasReported}
<p class="mt-1 text-tertiary">"billed" is the amount the provider charged.</p>
{/if}
{#if estimatedRows.length > 0}
<p class="mt-1 text-tertiary">{estimateSource}</p>
{/if}
{#if unpricedModels.length > 0}
<p class="mt-1 text-tertiary">
No price for {unpricedModels.map((r) => r.model).join(', ')}. Set one in workspace
settings, under AI.
</p>
{/if}
</div>
{/snippet}
</Tooltip>
{/if}
@@ -5,7 +5,7 @@ import { createLongHash } from '$lib/editorLangUtils'
import { userScopedDb, type UserScopedDbMigrateDeps } from '$lib/userScopedDb'
import { scopedKey } from '$lib/userScopedStorage'
import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs'
import type { PersistedContextUsage } from './tokenUsage'
import type { ModelTokenUsageTotals, PersistedContextUsage } from './tokenUsage'
import { IMAGE_OMITTED_PLACEHOLDER, type AttachedImage } from './imageUtils'
import { randomUUID } from '$lib/utils/uuid'
@@ -44,6 +44,10 @@ interface ChatSchema extends IDBSchema {
// in-flight job's tray row and completion survive a reload. Absent on
// chats predating this feature. Persisted out-of-band like modifiedItems.
backgroundJobs?: ChatJob[]
// Tokens this chat spent, per `provider:model`, so its cost survives a
// reload. Absent on chats predating this feature and on chats that never
// ran a turn — both simply show no cost.
usageByModel?: ModelTokenUsageTotals
}
}
// Image bytes, out-of-band from the chat record on purpose: the record is
@@ -178,6 +182,7 @@ export default class HistoryManager {
contextUsage?: PersistedContextUsage
modifiedItems?: string[]
backgroundJobs?: ChatJob[]
usageByModel?: ModelTokenUsageTotals
}
> = $state({})
@@ -475,6 +480,7 @@ export default class HistoryManager {
messages: ChatCompletionMessageParam[],
contextUsage?: number,
modifiedItems?: string[],
usageByModel?: ModelTokenUsageTotals,
backgroundJobs?: ChatJob[]
) {
if (displayMessages.length > 0) {
@@ -539,6 +545,13 @@ export default class HistoryManager {
? {
backgroundJobs: $state.snapshot(this.savedChats[this.currentChatId].backgroundJobs)
}
: {}),
// Same "don't erase on omit" guard again: a background-jobs save mid-turn
// must not drop the spend recorded by the turns before it.
...(usageByModel !== undefined
? { usageByModel }
: this.savedChats[this.currentChatId]?.usageByModel !== undefined
? { usageByModel: $state.snapshot(this.savedChats[this.currentChatId].usageByModel) }
: {})
}
// The mirror mirrors what the DB holds (refs — the snapshot is
@@ -581,9 +594,17 @@ export default class HistoryManager {
messages: ChatCompletionMessageParam[],
contextUsage?: number,
modifiedItems?: string[],
usageByModel?: ModelTokenUsageTotals,
backgroundJobs?: ChatJob[]
) {
await this.saveChat(displayMessages, messages, contextUsage, modifiedItems, backgroundJobs)
await this.saveChat(
displayMessages,
messages,
contextUsage,
modifiedItems,
usageByModel,
backgroundJobs
)
this.currentChatId = createLongHash()
this.pruneImageIds(this.currentChatId)
}
@@ -712,7 +712,14 @@ describe('HistoryManager mirror convergence under concurrent metadata saves', ()
// mirror, or s3's backgroundJobs fallback below reads the stale record
// and permanently erases the job.
const p1 = hm.saveChat(display, [] as ChatCompletionMessageParam[], undefined, ['script:a'])
const p2 = hm.saveChat(display, [] as ChatCompletionMessageParam[], undefined, undefined, [job])
const p2 = hm.saveChat(
display,
[] as ChatCompletionMessageParam[],
undefined,
undefined,
undefined,
[job]
)
await p1
const p3 = hm.saveChat(display, [] as ChatCompletionMessageParam[], undefined, [
'script:a',
@@ -518,7 +518,7 @@ describe('runChatLoop lastIterationUsage', () => {
expect(result.lastIterationUsage).toEqual({ prompt: 1200, completion: 80, total: 1280 })
// the aggregate keeps summing across iterations
expect(result.tokenUsage).toEqual({ prompt: 2200, completion: 130, total: 2330 })
expect(result.tokenUsage).toMatchObject({ prompt: 2200, completion: 130, total: 2330 })
})
it('ignores empty usage reports and returns null when none are real', async () => {
@@ -21,7 +21,13 @@ import {
} from './openai-responses'
import type { Tool, ToolCallbacks } from './shared'
import { sanitizeToolCallArguments } from './toolCallArguments'
import { addChatTokenUsage, emptyChatTokenUsage, type ChatTokenUsage } from './tokenUsage'
import {
addChatTokenUsage,
addModelTokenUsage,
emptyChatTokenUsage,
type ChatTokenUsage,
type ModelTokenUsageTotals
} from './tokenUsage'
export interface ChatClients {
openai: OpenAI
@@ -83,6 +89,8 @@ export interface ChatLoopResult {
addedMessages: ChatCompletionMessageParam[]
/** Sum of usage across all loop iterations (suitable for cost accounting). */
tokenUsage: ChatTokenUsage
/** The same usage split per model, so a turn that switched model prices correctly. */
tokenUsageByModel: ModelTokenUsageTotals
lastIterationUsage: ChatTokenUsage | null
hitMaxIterations: boolean
}
@@ -325,12 +333,25 @@ export async function runChatLoop(config: ChatLoopConfig): Promise<ChatLoopResul
const addedMessages: ChatCompletionMessageParam[] = config.addedMessages ?? []
let tokenUsage = emptyChatTokenUsage()
let tokenUsageByModel: ModelTokenUsageTotals = {}
let lastIterationUsage: ChatTokenUsage | null = null
let iterations = 0
let hitMaxIterations = false
// The model of the iteration currently in flight; re-read per iteration like
// `config.modelProvider` itself, so usage is attributed to the model that
// actually served it rather than to whatever is selected when the loop ends.
let iterationModel: ReasoningProviderModel | undefined
const trackUsage = (usage: ChatTokenUsage | null | undefined) => {
tokenUsage = addChatTokenUsage(tokenUsage, usage)
if (iterationModel) {
tokenUsageByModel = addModelTokenUsage(
tokenUsageByModel,
iterationModel.provider,
iterationModel.model,
usage
)
}
// Some providers/paths report no usage (prompt 0); keep the last real one.
if (usage && usage.prompt > 0) {
lastIterationUsage = usage
@@ -351,6 +372,7 @@ export async function runChatLoop(config: ChatLoopConfig): Promise<ChatLoopResul
const helpers = config.helpers
const systemMessage = config.systemMessage
const modelProvider = config.modelProvider
iterationModel = modelProvider
const webSearchCacheKey = getWebSearchCacheKey(workspace, modelProvider)
const webSearch =
(config.webSearch ?? true) &&
@@ -572,5 +594,5 @@ export async function runChatLoop(config: ChatLoopConfig): Promise<ChatLoopResul
}
}
return { addedMessages, tokenUsage, lastIterationUsage, hitMaxIterations }
return { addedMessages, tokenUsage, tokenUsageByModel, lastIterationUsage, hitMaxIterations }
}
@@ -0,0 +1,44 @@
import { describe, expect, it } from 'vitest'
import {
anthropicUsageToChatTokenUsage,
billedTokens,
openAICompletionsUsageToChatTokenUsage
} from './tokenUsage'
// The two providers report cache tokens under opposite conventions — Anthropic's
// input_tokens excludes them, OpenAI's includes them. Both are normalized so that
// `prompt` is the whole input, which is what makes `prompt - cached` the uncached
// share. Getting this backwards double-counts (or loses) the cached prefix, which
// is most of a long chat's input.
describe('billedTokens', () => {
it('derives uncached input under the Anthropic convention', () => {
const usage = anthropicUsageToChatTokenUsage({
input_tokens: 1000,
output_tokens: 200,
cache_creation_input_tokens: 300,
cache_read_input_tokens: 5000
})
expect(usage.prompt).toBe(6300)
expect(billedTokens(usage)).toEqual({
input: 1000,
cacheRead: 5000,
cacheWrite: 300,
output: 200
})
})
it('derives uncached input under the OpenAI convention', () => {
const usage = openAICompletionsUsageToChatTokenUsage({
prompt_tokens: 6000,
completion_tokens: 200,
prompt_tokens_details: { cached_tokens: 5000 }
})
expect(usage.prompt).toBe(6000)
expect(billedTokens(usage)).toEqual({
input: 1000,
cacheRead: 5000,
cacheWrite: 0,
output: 200
})
})
})
@@ -1,7 +1,21 @@
import type { AIProvider } from '$lib/gen'
import { modelKey } from '../modelConfig'
import type { PricedTokens } from '../modelPricing'
export interface ChatTokenUsage {
prompt: number
completion: number
total: number
/**
* Subsets of `prompt`, split out because they are billed at different rates
* (a cached read is a fraction of an uncached one). `prompt` stays the whole
* input so the context gauge keeps measuring the whole request; uncached
* input is `prompt - cacheRead - cacheWrite`.
*/
cacheRead: number
cacheWrite: number
/** Cost in USD as billed, for the providers that report one. */
cost?: number
}
/**
@@ -28,7 +42,7 @@ export function normalizeContextUsage(
}
export function emptyChatTokenUsage(): ChatTokenUsage {
return { prompt: 0, completion: 0, total: 0 }
return { prompt: 0, completion: 0, total: 0, cacheRead: 0, cacheWrite: 0 }
}
export function addChatTokenUsage(
@@ -39,10 +53,83 @@ export function addChatTokenUsage(
return total
}
const cost =
total.cost === undefined && usage.cost === undefined
? undefined
: (total.cost ?? 0) + (usage.cost ?? 0)
return {
prompt: total.prompt + usage.prompt,
completion: total.completion + usage.completion,
total: total.total + usage.total
total: total.total + usage.total,
// `?? 0`: the cache split is newer than the field it lives on, so a usage
// object read back from storage may predate it.
cacheRead: (total.cacheRead ?? 0) + (usage.cacheRead ?? 0),
cacheWrite: (total.cacheWrite ?? 0) + (usage.cacheWrite ?? 0),
...(cost === undefined ? {} : { cost })
}
}
/** Compact token count for chips and tooltips (`1.2M`, `34k`, `567`). */
export function formatTokenCount(tokens: number): string {
if (tokens >= 1_000_000) {
return `${(tokens / 1_000_000).toFixed(1).replace(/\.0$/, '')}M`
}
if (tokens >= 1000) {
return `${Math.round(tokens / 1000)}k`
}
return `${tokens}`
}
/**
* A chat's spend on one model. Usage is bucketed per model rather than summed
* because each model bills at its own rate, and a chat can switch model between
* turns (or mid-turn, via the model selector).
*/
export type ModelTokenUsage = {
provider: AIProvider
model: string
usage: ChatTokenUsage
}
export type ModelTokenUsageTotals = Record<string, ModelTokenUsage>
/** Fold one report into the per-model totals, keyed `provider:model`. */
export function addModelTokenUsage(
totals: ModelTokenUsageTotals,
provider: AIProvider,
model: string,
usage: ChatTokenUsage | null | undefined
): ModelTokenUsageTotals {
if (!usage) {
return totals
}
const key = modelKey(provider, model)
const existing = totals[key]
return {
...totals,
[key]: {
provider,
model,
usage: addChatTokenUsage(existing?.usage ?? emptyChatTokenUsage(), usage)
}
}
}
/**
* Split a usage report into the four separately-billed token classes. `prompt`
* counts the whole input, so the uncached share is whatever the cached classes
* do not account for which holds for both provider conventions below
* (Anthropic adds its cache counts into `prompt`, OpenAI's already includes them).
*/
export function billedTokens(usage: ChatTokenUsage): PricedTokens {
const cacheRead = usage.cacheRead ?? 0
const cacheWrite = usage.cacheWrite ?? 0
return {
input: Math.max(0, usage.prompt - cacheRead - cacheWrite),
cacheRead,
cacheWrite,
output: usage.completion
}
}
@@ -57,16 +144,17 @@ export function anthropicUsageToChatTokenUsage(
| null
| undefined
): ChatTokenUsage {
const prompt =
(usage?.input_tokens ?? 0) +
(usage?.cache_creation_input_tokens ?? 0) +
(usage?.cache_read_input_tokens ?? 0)
const cacheWrite = usage?.cache_creation_input_tokens ?? 0
const cacheRead = usage?.cache_read_input_tokens ?? 0
const prompt = (usage?.input_tokens ?? 0) + cacheWrite + cacheRead
const completion = usage?.output_tokens ?? 0
return {
prompt,
completion,
total: prompt + completion
total: prompt + completion,
cacheRead,
cacheWrite
}
}
@@ -89,7 +177,11 @@ export function openAIResponsesUsageToChatTokenUsage(
return {
prompt,
completion,
total: usage?.total_tokens ?? prompt + completion
total: usage?.total_tokens ?? prompt + completion,
cacheRead: usage?.input_tokens_details?.cached_tokens ?? 0,
// Automatic caching: nothing is billed for populating it, and no usage
// field reports it either.
cacheWrite: 0
}
}
@@ -102,6 +194,8 @@ export function openAICompletionsUsageToChatTokenUsage(
completion_tokens?: number | null
total_tokens?: number | null
prompt_tokens_details?: { cached_tokens?: number | null } | null
/** OpenRouter reports what it actually charged when the request opts in. */
cost?: number | null
}
| null
| undefined
@@ -112,6 +206,9 @@ export function openAICompletionsUsageToChatTokenUsage(
return {
prompt,
completion,
total: usage?.total_tokens ?? prompt + completion
total: usage?.total_tokens ?? prompt + completion,
cacheRead: usage?.prompt_tokens_details?.cached_tokens ?? 0,
cacheWrite: 0,
...(typeof usage?.cost === 'number' ? { cost: usage.cost } : {})
}
}
+23 -6
View File
@@ -1056,6 +1056,23 @@ export async function getFimCompletion(
}
}
// A streamed OpenAI-compatible response carries no usage at all unless the request
// asks for it, so a provider missing from this set reports zero tokens — no context
// gauge, no cost. `stream_options.include_usage` is part of the OpenAI streaming
// spec and these providers document supporting it; `customai` is deliberately absent
// because it points at an arbitrary endpoint that may reject the field outright.
const STREAM_USAGE_PROVIDERS = new Set<AIProvider>([
'openai',
'azure_openai',
'azure_foundry',
'googleai',
'openrouter',
'groq',
'deepseek',
'mistral',
'togetherai'
])
export async function getCompletion(
messages: ChatCompletionMessageParam[],
abortController: AbortController,
@@ -1099,17 +1116,17 @@ export async function getCompletion(
// Use Completions API for other providers
const client = options?.openaiClient ?? workspaceAIClients.getOpenaiClient()
const completionConfig = applyReasoningToConfig(
(provider === 'openai' ||
provider === 'azure_openai' ||
provider === 'azure_foundry' ||
provider === 'googleai') &&
config.stream
config.stream && STREAM_USAGE_PROVIDERS.has(provider)
? {
...config,
stream_options: {
...(config.stream_options ?? {}),
include_usage: true
}
},
// OpenRouter's own extension, on top of stream_options: it returns the
// credits actually charged next to the token counts, which is the one
// route by which the chat sees a real cost rather than an estimate.
...(provider === 'openrouter' ? { usage: { include: true } } : {})
}
: config,
provider === 'deepseek' ? 'deepseek' : provider === 'mistral' ? 'mistral' : 'completions',
@@ -116,21 +116,44 @@ function normalizeVersionSeparators(model: string): string {
return model.replace(/\./g, '-')
}
// An entry that ends on a version digit must not run into a longer version:
// `gpt-4.1` collapses to `gpt-4-1`, which would otherwise claim the 128K
// `gpt-4-1106-preview` as a 1M model. Suffixes that continue with a separator
// (`claude-opus-4-8` in `...-4-8-v1`, `gpt-5` in `gpt-5-mini`) still match.
// Family fallbacks ending on a letter get no such guard — a version welded
// straight onto the name (`llama3.1`) is exactly what they exist to catch.
const MODEL_CONTEXT_WINDOW_MATCHERS: [matcher: RegExp, contextWindow: number][] =
MODEL_CONTEXT_WINDOWS.map(([name, contextWindow]) => {
/**
* Compile a most-specific-first `[name, value]` table into matchers against the
* bare model id. Shared with the pricing table so both resolve the same set of
* ids a model whose window is known but whose price is not (or vice versa)
* should be a gap in one table, never a difference in matching.
*
* An entry that ends on a version digit must not run into a longer version:
* `gpt-4.1` collapses to `gpt-4-1`, which would otherwise claim
* `gpt-4-1106-preview`. Suffixes that continue with a separator
* (`claude-opus-4-8` in `...-4-8-v1`, `gpt-5` in `gpt-5-mini`) still match.
* Family fallbacks ending on a letter get no such guard a version welded
* straight onto the name (`llama3.1`) is exactly what they exist to catch.
*/
export function buildModelMatchers<T>(entries: [name: string, value: T][]): [RegExp, T][] {
return entries.map(([name, value]) => {
const pattern = normalizeVersionSeparators(name).replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
return [new RegExp(/\d$/.test(pattern) ? `${pattern}(?!\\d)` : pattern), contextWindow]
return [new RegExp(/\d$/.test(pattern) ? `${pattern}(?!\\d)` : pattern), value]
})
}
/**
* The `provider:model` key the workspace AI settings use for their per-model maps
* (`max_tokens_per_model`, `model_pricing`). A bare model id is not enough: the
* same id can be served by more than one provider at different rates.
*/
export function modelKey(provider: AIProvider | string, model: string): string {
return `${provider}:${model}`
}
export function matchModel<T>(matchers: [RegExp, T][], model: string): T | undefined {
const id = normalizeVersionSeparators(parseModelId(model).base)
return matchers.find(([matcher]) => matcher.test(id))?.[1]
}
const MODEL_CONTEXT_WINDOW_MATCHERS = buildModelMatchers(MODEL_CONTEXT_WINDOWS)
export function getKnownModelContextWindow(model: string): number | undefined {
const id = normalizeVersionSeparators(parseModelId(model).base)
return MODEL_CONTEXT_WINDOW_MATCHERS.find(([matcher]) => matcher.test(id))?.[1]
return matchModel(MODEL_CONTEXT_WINDOW_MATCHERS, model)
}
export function getModelContextWindow(model: string) {
@@ -0,0 +1,97 @@
import { describe, expect, it } from 'vitest'
import { billedTokens } from './chat/tokenUsage'
import { estimateCost, priceSpend, resolveModelPrice } from './modelPricing'
describe('resolveModelPrice', () => {
it('resolves the same model across the routes that decorate its id', () => {
const direct = resolveModelPrice('anthropic', 'claude-opus-5', undefined)
expect(direct?.price.input).toBe(5)
// A gateway prefix, a dot-versioned id, a date suffix and a variant suffix
// must all land on the same entry — a miss here silently under-reports cost.
for (const id of [
'anthropic/claude-opus-5',
'anthropic/claude-opus-4.8',
'claude-opus-4-8-20260101',
'anthropic/claude-opus-5:thinking'
]) {
expect(resolveModelPrice('openrouter', id, undefined)?.price.input).toBe(5)
}
})
it('does not let a version-digit entry claim a longer version', () => {
expect(resolveModelPrice('openai', 'gpt-4.1', undefined)?.price.input).toBe(2)
expect(resolveModelPrice('openai', 'gpt-4-1106-preview', undefined)?.price.input).not.toBe(2)
})
it('reports an unknown model as unpriced rather than guessing', () => {
expect(resolveModelPrice('customai', 'some-in-house-model', undefined)).toBeUndefined()
})
it('prefers a workspace override, defaulting its cache rates off its own input rate', () => {
const resolved = resolveModelPrice('anthropic', 'claude-opus-5', {
'anthropic:claude-opus-5': { input: 2, output: 8 }
})
expect(resolved?.source).toBe('override')
expect(resolved?.price.input).toBe(2)
expect(resolved?.price.cacheRead).toBeCloseTo(0.2)
expect(resolved?.price.cacheWrite).toBeCloseTo(2.5)
})
})
describe('estimateCost', () => {
it('bills each token class at its own rate', () => {
const cost = estimateCost(
{ input: 1_000_000, cacheRead: 1_000_000, cacheWrite: 1_000_000, output: 1_000_000 },
{ input: 5, output: 25, cacheRead: 0.5, cacheWrite: 6.25 }
)
expect(cost).toBeCloseTo(5 + 0.5 + 6.25 + 25)
})
it('charges a cached prefix less than an uncached one', () => {
const usage = {
prompt: 100_000,
completion: 0,
total: 100_000,
cacheRead: 90_000,
cacheWrite: 0
}
const uncached = { ...usage, cacheRead: 0 }
const price = { input: 5, output: 25, cacheRead: 0.5, cacheWrite: 6.25 }
expect(estimateCost(billedTokens(usage), price)).toBeLessThan(
estimateCost(billedTokens(uncached), price)
)
})
})
describe('priceSpend', () => {
it('prefers a provider-reported cost over the estimate', () => {
const priced = priceSpend(
[
{
provider: 'openrouter',
model: 'anthropic/claude-opus-5',
tokens: { input: 1_000_000, cacheRead: 0, cacheWrite: 0, output: 0 },
reportedCostUsd: 0.42
}
],
undefined
)
expect(priced.total).toBe(0.42)
expect(priced.hasReported).toBe(true)
})
it('flags an unpriced model instead of counting it as free', () => {
const priced = priceSpend(
[
{
provider: 'customai',
model: 'some-in-house-model',
tokens: { input: 1_000_000, cacheRead: 0, cacheWrite: 0, output: 0 }
}
],
undefined
)
expect(priced.hasUnpriced).toBe(true)
expect(priced.rows[0].cost).toBeUndefined()
})
})
@@ -0,0 +1,217 @@
import type { AIProvider, ModelPriceOverride } from '$lib/gen'
import { buildModelMatchers, matchModel, modelKey } from './modelConfig'
/** Rates in USD per million tokens, one per billed token class. */
export type ModelPrice = {
input: number
output: number
cacheRead: number
cacheWrite: number
}
export type ModelPriceSource = 'override' | 'builtin'
export type ResolvedModelPrice = {
price: ModelPrice
source: ModelPriceSource
}
/** What a chat spent on one model, in tokens. */
export type PricedTokens = {
input: number
cacheRead: number
cacheWrite: number
output: number
}
// Rates that fall out of the input rate unless a provider prices them separately.
// Anthropic reads a cached prefix at a tenth of the input rate and writes one at
// 1.25x (5-minute TTL, the default the chat uses). Providers whose caching is
// automatic never report a cache write, so their write rate is unused.
const CACHE_READ_RATIO = 0.1
const CACHE_WRITE_RATIO = 1.25
type PriceEntry = {
input: number
output: number
cacheRead?: number
cacheWrite?: number
}
/**
* Published list prices, most specific entry first the first name found in the
* bare model id wins, so vendor-namespaced and date-suffixed ids
* (anthropic/claude-opus-5, gpt-5-2026-01-01) still resolve. Matching is shared
* with the context-window table via `buildModelMatchers`.
*
* This is a best-effort snapshot: vendors change rates, ship models faster than
* this table is updated, and negotiated rates differ from list. A model that is
* not listed resolves to undefined and is reported as unpriced rather than
* guessed at, and any entry can be corrected per workspace from the AI settings.
* Providers whose catalogue turns over too quickly to track (DeepSeek, Mistral,
* Groq, TogetherAI, custom deployments) are deliberately absent.
*/
const MODEL_PRICES: [name: string, price: PriceEntry][] = [
// Anthropic — Opus 4.1 and older bill at the pre-4.5 Opus rate, so the family
// fallback sits below the explicit entries rather than covering them.
['claude-fable-5', { input: 10, output: 50 }],
['claude-mythos-5', { input: 10, output: 50 }],
['claude-opus-5', { input: 5, output: 25 }],
['claude-opus-4-8', { input: 5, output: 25 }],
['claude-opus-4-7', { input: 5, output: 25 }],
['claude-opus-4-6', { input: 5, output: 25 }],
['claude-opus-4-5', { input: 5, output: 25 }],
['claude-opus-4-1', { input: 15, output: 75 }],
['claude-opus-4', { input: 15, output: 75 }],
['claude-sonnet-5', { input: 3, output: 15 }],
['claude-sonnet-4-6', { input: 3, output: 15 }],
['claude-sonnet-4-5', { input: 3, output: 15 }],
['claude-sonnet-4', { input: 3, output: 15 }],
['claude-haiku-4-5', { input: 1, output: 5 }],
['claude-3-5-haiku', { input: 0.8, output: 4 }],
['claude-opus', { input: 5, output: 25 }],
['claude-sonnet', { input: 3, output: 15 }],
['claude-haiku', { input: 1, output: 5 }],
// OpenAI — cached input is a tenth of input, and there is no separate charge
// for writing the cache, so the write rate never applies (the OpenAI usage
// parsers report no cache-write tokens). The -mini/-nano entries must precede
// the family entry, which would otherwise claim them.
['gpt-5-mini', { input: 0.25, output: 2 }],
['gpt-5-nano', { input: 0.05, output: 0.4 }],
['gpt-5', { input: 1.25, output: 10 }],
['gpt-4.1-mini', { input: 0.4, output: 1.6 }],
['gpt-4.1-nano', { input: 0.1, output: 0.4 }],
['gpt-4.1', { input: 2, output: 8 }],
['gpt-4o-mini', { input: 0.15, output: 0.6 }],
['gpt-4o', { input: 2.5, output: 10 }],
['o4-mini', { input: 1.1, output: 4.4 }],
['o3-mini', { input: 1.1, output: 4.4 }],
['o3', { input: 2, output: 8 }],
// Google
['gemini-2.5-flash-lite', { input: 0.1, output: 0.4 }],
['gemini-2.5-flash', { input: 0.3, output: 2.5 }],
['gemini-2.5-pro', { input: 1.25, output: 10 }]
]
const MODEL_PRICE_MATCHERS = buildModelMatchers(
MODEL_PRICES.map(([name, entry]): [string, ModelPrice] => [
name,
{
input: entry.input,
output: entry.output,
cacheRead: entry.cacheRead ?? entry.input * CACHE_READ_RATIO,
cacheWrite: entry.cacheWrite ?? entry.input * CACHE_WRITE_RATIO
}
])
)
export function getKnownModelPrice(model: string): ModelPrice | undefined {
return matchModel(MODEL_PRICE_MATCHERS, model)
}
/**
* The rate a workspace should be billed at for one model: its override when an
* admin set one, otherwise the published list price, otherwise nothing. An
* override that omits the cache rates keeps the usual multiples of its own input
* rate, so an admin who only knows their input/output pricing does not have to
* invent the other two.
*/
export function resolveModelPrice(
provider: AIProvider | string,
model: string,
overrides: Record<string, ModelPriceOverride> | undefined
): ResolvedModelPrice | undefined {
const override = overrides?.[modelKey(provider, model)]
if (override) {
return {
source: 'override',
price: {
input: override.input,
output: override.output,
cacheRead: override.cache_read ?? override.input * CACHE_READ_RATIO,
cacheWrite: override.cache_write ?? override.input * CACHE_WRITE_RATIO
}
}
}
const builtin = getKnownModelPrice(model)
return builtin ? { source: 'builtin', price: builtin } : undefined
}
/** Cost in USD of `tokens` at `price`. */
export function estimateCost(tokens: PricedTokens, price: ModelPrice): number {
return (
(tokens.input * price.input +
tokens.cacheRead * price.cacheRead +
tokens.cacheWrite * price.cacheWrite +
tokens.output * price.output) /
1_000_000
)
}
/** Tokens spent on one model, from a chat's running totals or the usage API. */
export type ModelSpend = {
provider: string
model: string
tokens: PricedTokens
/** What the provider billed, where it reports a figure. */
reportedCostUsd?: number
}
export type Priced = {
/** Undefined when no rate is known for the model — reported as unpriced, never guessed. */
cost: number | undefined
source: ModelPriceSource | 'reported' | undefined
}
export type PricedSpend<T extends ModelSpend> = {
/** The input entries, each with its cost callers carry their own fields through
* rather than zipping the result back against the input by index. */
rows: (T & Priced)[]
total: number
/** True when at least one row has no rate, so `total` understates the truth. */
hasUnpriced: boolean
/** True when at least one row is a figure the provider billed rather than an estimate. */
hasReported: boolean
}
/**
* Cost a set of per-model token counts. A provider-reported figure always wins:
* it is what was actually charged, where everything else is list price times
* tokens. Shared by the chat's cost chip and the workspace usage view so both
* apply the same rates and the same estimated/reported labelling.
*/
export function priceSpend<T extends ModelSpend>(
spend: T[],
overrides: Record<string, ModelPriceOverride> | undefined
): PricedSpend<T> {
let total = 0
let hasUnpriced = false
let hasReported = false
const rows = spend.map((entry): T & Priced => {
if (entry.reportedCostUsd !== undefined) {
hasReported = true
total += entry.reportedCostUsd
return { ...entry, cost: entry.reportedCostUsd, source: 'reported' }
}
const resolved = resolveModelPrice(entry.provider, entry.model, overrides)
if (!resolved) {
hasUnpriced = true
return { ...entry, cost: undefined, source: undefined }
}
const cost = estimateCost(entry.tokens, resolved.price)
total += cost
return { ...entry, cost, source: resolved.source }
})
return { rows, total, hasUnpriced, hasReported }
}
/**
* Money, at the precision the amount deserves: sub-cent spend is where a chat
* spends most of its life, and rounding it to `$0.00` would read as free.
*/
export function formatUsd(amount: number): string {
if (amount === 0) return '$0'
if (amount < 0.01) return `$${amount.toFixed(4)}`
if (amount < 1) return `$${amount.toFixed(3)}`
return `$${amount.toFixed(2)}`
}
@@ -5,7 +5,8 @@
type AIConfig,
type AIProvider,
type GetCopilotSettingsStateResponse,
type InstanceAISummary
type InstanceAISummary,
type ModelPriceOverride
} from '$lib/gen'
import { workspaceStore } from '$lib/stores'
import { sendUserToast } from '$lib/toast'
@@ -25,6 +26,8 @@
import Badge from '../common/badge/Badge.svelte'
import Tooltip from '../Tooltip.svelte'
import ModelTokenLimits from './ModelTokenLimits.svelte'
import ModelPricing from './ModelPricing.svelte'
import AiUsagePanel from './AiUsagePanel.svelte'
import { setCopilotInfo } from '$lib/aiStore'
import AIPromptsModal from '../settings/AIPromptsModal.svelte'
import { Settings } from 'lucide-svelte'
@@ -73,6 +76,7 @@
let metadataModel: string | undefined = $state(undefined)
let customPrompts: Record<string, string> = $state({})
let maxTokensPerModel: Record<string, number> = $state({})
let modelPricing: Record<string, ModelPriceOverride> = $state({})
let usingOpenaiClientCredentialsOauth = $state(false)
let workspaceOverrideEditorOpened = $state(false)
@@ -83,6 +87,7 @@
let initialMetadataModel: string | undefined = $state(undefined)
let initialCustomPrompts: Record<string, string> = $state({})
let initialMaxTokensPerModel: Record<string, number> = $state({})
let initialModelPricing: Record<string, ModelPriceOverride> = $state({})
let initialPrompts: Record<string, string> = $state({})
let lastLoadedConfigKey = $state<string | undefined>(undefined)
@@ -110,6 +115,7 @@
codeCompletionModel = config?.code_completion_model?.model
customPrompts = clone(config?.custom_prompts ?? {})
maxTokensPerModel = clone(config?.max_tokens_per_model ?? {})
modelPricing = clone(config?.model_pricing ?? {})
for (const mode of ['edit', 'fix', 'gen']) {
if (!(mode in customPrompts)) {
customPrompts[mode] = ''
@@ -124,6 +130,7 @@
initialCodeCompletionModel = codeCompletionModel
initialCustomPrompts = clone(customPrompts)
initialMaxTokensPerModel = clone(maxTokensPerModel)
initialModelPricing = clone(modelPricing)
initialPrompts = clone(customPrompts)
}
@@ -139,6 +146,7 @@
codeCompletionModel = initialCodeCompletionModel
customPrompts = clone(initialCustomPrompts)
maxTokensPerModel = clone(initialMaxTokensPerModel)
modelPricing = clone(initialModelPricing)
}
$effect(() => {
@@ -172,7 +180,8 @@
metadataModel !== initialMetadataModel ||
codeCompletionModel !== initialCodeCompletionModel ||
JSON.stringify(customPrompts) !== JSON.stringify(initialCustomPrompts) ||
JSON.stringify(maxTokensPerModel) !== JSON.stringify(initialMaxTokensPerModel)
JSON.stringify(maxTokensPerModel) !== JSON.stringify(initialMaxTokensPerModel) ||
JSON.stringify(modelPricing) !== JSON.stringify(initialModelPricing)
)
$effect(() => {
@@ -285,7 +294,8 @@
metadata_model,
custom_prompts: Object.keys(custom_prompts).length > 0 ? custom_prompts : undefined,
max_tokens_per_model:
Object.keys(maxTokensPerModel).length > 0 ? maxTokensPerModel : undefined
Object.keys(maxTokensPerModel).length > 0 ? maxTokensPerModel : undefined,
model_pricing: Object.keys(modelPricing).length > 0 ? modelPricing : undefined
}
: {}
}
@@ -576,6 +586,10 @@
<ModelTokenLimits {aiProviders} bind:maxTokensPerModel />
<ModelPricing {aiProviders} bind:modelPricing />
<AiUsagePanel {modelPricing} />
<SettingCard label="Custom system prompts" description={promptDescription}>
<div class="flex items-center gap-2 pt-1">
<Button
@@ -0,0 +1,144 @@
<script lang="ts">
import { AiService, type AITokenUsageBucket, type ModelPriceOverride } from '$lib/gen'
import { workspaceStore } from '$lib/stores'
import { formatUsd, priceSpend, type ModelSpend } from '../copilot/modelPricing'
import { formatTokenCount } from '../copilot/chat/tokenUsage'
import SettingCard from '../instanceSettings/SettingCard.svelte'
import Select from '../select/Select.svelte'
import ToggleButtonGroup from '../common/toggleButton-v2/ToggleButtonGroup.svelte'
import ToggleButton from '../common/toggleButton-v2/ToggleButton.svelte'
import { resource } from 'runed'
let { modelPricing }: { modelPricing: Record<string, ModelPriceOverride> } = $props()
type GroupBy = 'day' | 'user' | 'model' | 'session'
let days = $state(30)
let groupBy = $state<GroupBy>('day')
const rangeOptions = [
{ label: 'Last 7 days', value: 7 },
{ label: 'Last 30 days', value: 30 },
{ label: 'Last 90 days', value: 90 }
]
let usage = resource(
() => ({ workspace: $workspaceStore, days, groupBy }),
async ({ workspace, days, groupBy }) =>
workspace ? await AiService.listAiUsage({ workspace, days, groupBy }) : []
)
// The API groups by (dimension, provider, model) so every bucket resolves to a
// single rate; the table folds those back into one line per dimension value.
type Bucket = ModelSpend & { key: string; requests: number }
function toSpend(bucket: AITokenUsageBucket): Bucket {
return {
// Grouping by model has no separate dimension — the model is the key.
key: groupBy === 'model' ? `${bucket.provider}/${bucket.model}` : bucket.key || '—',
requests: bucket.requests,
provider: bucket.provider,
model: bucket.model,
tokens: {
input: bucket.input_tokens,
cacheRead: bucket.cache_read_tokens,
cacheWrite: bucket.cache_write_tokens,
output: bucket.output_tokens
},
reportedCostUsd:
bucket.reported_cost_nano_usd != undefined
? bucket.reported_cost_nano_usd / 1_000_000_000
: undefined
}
}
let priced = $derived(priceSpend((usage.current ?? []).map(toSpend), modelPricing))
let rows = $derived.by(() => {
const byKey = new Map<
string,
{ key: string; cost: number | undefined; tokensIn: number; tokensOut: number; requests: number }
>()
for (const row of priced.rows) {
const existing = byKey.get(row.key) ?? {
key: row.key,
cost: undefined,
tokensIn: 0,
tokensOut: 0,
requests: 0
}
existing.tokensIn += row.tokens.input + row.tokens.cacheRead + row.tokens.cacheWrite
existing.tokensOut += row.tokens.output
existing.requests += row.requests
if (row.cost !== undefined) {
existing.cost = (existing.cost ?? 0) + row.cost
}
byKey.set(row.key, existing)
}
return [...byKey.values()].sort((a, b) => (b.cost ?? 0) - (a.cost ?? 0))
})
</script>
<SettingCard
label="AI usage"
description="Token spend across this workspace's AI chats. Costs are estimated from the rates above unless the provider reported one."
>
<div class="flex flex-col gap-3">
<div class="flex flex-row items-center gap-2 flex-wrap">
<div class="w-40">
<Select items={rangeOptions} bind:value={days} />
</div>
<ToggleButtonGroup bind:selected={groupBy}>
{#snippet children({ item })}
<ToggleButton value="day" label="By day" {item} />
<ToggleButton value="user" label="By user" {item} />
<ToggleButton value="model" label="By model" {item} />
<ToggleButton value="session" label="By session" {item} />
{/snippet}
</ToggleButtonGroup>
</div>
{#if usage.loading}
<p class="text-xs text-tertiary">Loading…</p>
{:else if usage.error}
<p class="text-xs text-tertiary">
Could not load usage. Only workspace admins can read it.
</p>
{:else if rows.length === 0}
<p class="text-xs text-tertiary">No AI usage recorded in this period.</p>
{:else}
<div class="flex flex-row items-baseline gap-2">
<span class="text-lg font-semibold tabular-nums">{formatUsd(priced.total)}</span>
<span class="text-xs text-tertiary">
total{priced.hasUnpriced ? ', excluding models with no price' : ''}
</span>
</div>
<div class="overflow-x-auto border rounded-md">
<table class="w-full text-xs">
<thead class="bg-surface-secondary">
<tr class="text-left text-secondary">
<th class="px-3 py-2 font-medium">{groupBy}</th>
<th class="px-3 py-2 font-medium text-right">In</th>
<th class="px-3 py-2 font-medium text-right">Out</th>
<th class="px-3 py-2 font-medium text-right">Requests</th>
<th class="px-3 py-2 font-medium text-right">Cost</th>
</tr>
</thead>
<tbody>
{#each rows as row (row.key)}
<tr class="border-t">
<td class="px-3 py-2 font-mono truncate max-w-xs">{row.key}</td>
<td class="px-3 py-2 text-right tabular-nums">{formatTokenCount(row.tokensIn)}</td>
<td class="px-3 py-2 text-right tabular-nums">{formatTokenCount(row.tokensOut)}</td>
<td class="px-3 py-2 text-right tabular-nums">{row.requests}</td>
<td class="px-3 py-2 text-right tabular-nums">
{row.cost === undefined ? 'no rate' : formatUsd(row.cost)}
</td>
</tr>
{/each}
</tbody>
</table>
</div>
{/if}
</div>
</SettingCard>
@@ -0,0 +1,185 @@
<script lang="ts">
import type { AIConfig, AIProvider, ModelPriceOverride } from '$lib/gen'
import { Badge, Button } from '../common'
import { getKnownModelPrice } from '../copilot/modelPricing'
import { modelKey } from '../copilot/modelConfig'
import { ChevronDown, ChevronUp } from 'lucide-svelte'
import { slide } from 'svelte/transition'
import SettingCard from '../instanceSettings/SettingCard.svelte'
// A rate above this is far more likely a unit mistake (per-token instead of
// per-million) than a real price, and a wrong rate silently inflates every
// figure derived from it.
const MAX_RATE = 1000
let {
aiProviders,
modelPricing = $bindable()
}: {
aiProviders: Exclude<AIConfig['providers'], undefined>
modelPricing: Record<string, ModelPriceOverride>
} = $props()
let errors = $state<Record<string, string>>({})
let collapsedProviders = $state<Record<string, boolean>>({})
const modelsByProvider = $derived(
Object.entries(aiProviders).reduce(
(acc, [provider, config]) => {
acc[provider] = config.models.map((model) => ({
provider: provider as AIProvider,
model
}))
return acc
},
{} as Record<string, Array<{ provider: AIProvider; model: string }>>
)
)
type Field = 'input' | 'output'
function currentRates(
provider: AIProvider,
model: string
): { input: number; output: number } | undefined {
const override = modelPricing[modelKey(provider, model)]
if (override) return { input: override.input, output: override.output }
const builtin = getKnownModelPrice(model)
return builtin ? { input: builtin.input, output: builtin.output } : undefined
}
function isOverridden(provider: AIProvider, model: string): boolean {
return modelPricing[modelKey(provider, model)] !== undefined
}
function updateRate(provider: AIProvider, model: string, field: Field, value: number) {
const key = modelKey(provider, model)
if (!(value >= 0) || value > MAX_RATE) {
errors[key] = `Rate must be between 0 and ${MAX_RATE}`
return
}
// An edit to either field pins both: a half-specified override would leave
// the other rate silently tracking a built-in price the admin did not choose.
const current = currentRates(provider, model) ?? { input: 0, output: 0 }
modelPricing = {
...modelPricing,
[key]: { ...modelPricing[key], input: current.input, output: current.output, [field]: value }
}
errors[key] = ''
}
function resetModel(provider: AIProvider, model: string) {
const key = modelKey(provider, model)
const next = { ...modelPricing }
delete next[key]
modelPricing = next
errors[key] = ''
}
function toggleProvider(provider: string) {
collapsedProviders[provider] = !collapsedProviders[provider]
}
function hasOverrides(provider: string, models: Array<{ model: string }>): boolean {
return models.some((m) => isOverridden(provider as AIProvider, m.model))
}
$effect(() => {
collapsedProviders = {
...Object.fromEntries(Object.keys(aiProviders).map((provider) => [provider, true]))
}
})
</script>
{#if Object.keys(aiProviders).length > 0}
<SettingCard
label="Model pricing"
description="Rates in USD per million tokens, used to cost AI chat usage. Built-in list prices are a best-effort snapshot; set a rate here to use your negotiated one, or to price a model that has none."
>
<div class="flex flex-col gap-3">
{#each Object.entries(modelsByProvider).filter(([_, models]) => models.length > 0) as [provider, models]}
{@const isExpanded = !collapsedProviders[provider]}
<div class="border rounded-md bg-surface-tertiary">
<button
type="button"
onclick={() => toggleProvider(provider)}
class="w-full px-4 py-1 min-h-8 flex items-center justify-between hover:bg-surface-hover transition-colors rounded-t-md"
>
<div class="flex items-center gap-2">
<h4 class="font-medium text-xs capitalize">{provider}</h4>
{#if hasOverrides(provider, models)}
<Badge color="blue">Modified</Badge>
{/if}
</div>
{#if isExpanded}
<ChevronUp size={16} class="text-gray-500" />
{:else}
<ChevronDown size={16} class="text-gray-500" />
{/if}
</button>
{#if isExpanded}
<div transition:slide|local={{ duration: 200 }} class="p-4 border-t">
<div class="space-y-3">
{#each models as { model }}
{@const key = modelKey(provider as AIProvider, model)}
{@const rates = currentRates(provider as AIProvider, model)}
{@const overridden = isOverridden(provider as AIProvider, model)}
<div class="flex flex-col gap-1">
<div class="flex items-center gap-3">
<div class="flex-1 min-w-0">
<span class="text-xs text-primary truncate block">{model}</span>
</div>
{#each ['input', 'output'] as const as field}
<div class="flex items-center gap-1">
<span class="text-xs text-secondary">{field}</span>
<input
type="number"
min="0"
max={MAX_RATE}
step="0.01"
value={rates?.[field] ?? ''}
placeholder="—"
oninput={(e) => {
const value = parseFloat(e.currentTarget.value)
if (!isNaN(value)) {
updateRate(provider as AIProvider, model, field, value)
}
}}
class="w-20 px-2 py-1 text-xs text-center border border-gray-200 dark:border-gray-700 rounded bg-surface focus:outline-none focus:ring-2 focus:ring-blue-500 focus:border-transparent"
/>
</div>
{/each}
<span class="text-xs text-secondary whitespace-nowrap">$ / 1M</span>
</div>
{#if !rates}
<div class="text-xs text-tertiary">
No built-in price — usage on this model is reported without a cost until you
set one.
</div>
{/if}
{#if overridden}
<div class="text-xs text-primary flex flex-row items-center gap-1">
<span>Overriding the built-in price</span>
<Button
variant="default"
unifiedSize="xs"
onclick={() => resetModel(provider as AIProvider, model)}
>
Reset
</Button>
</div>
{/if}
{#if errors[key]}
<div class="text-xs text-red-500">{errors[key]}</div>
{/if}
</div>
{/each}
</div>
</div>
{/if}
</div>
{/each}
</div>
</SettingCard>
{/if}
Binary file not shown.