From 11fda89b520cc8d1ce30cdba36bd10355cf025cc Mon Sep 17 00:00:00 2001 From: hugocasa Date: Mon, 20 Jul 2026 20:56:57 +0200 Subject: [PATCH] feat(telemetry): generic feature-usage telemetry with AI session metrics (#10200) * feat(telemetry): add generic feature_usage table and batched logging endpoint Co-Authored-By: Claude Fable 5 * feat(telemetry): log AI session usage events and document them in telemetry settings Co-Authored-By: Claude Fable 5 * fix(telemetry): use escape sequence instead of literal NUL bytes in buffer key Co-Authored-By: Claude Fable 5 * fix(telemetry): validate dimensions, decouple retention, keepalive flush Co-Authored-By: Claude Fable 5 * fix(telemetry): allowlist feature-usage dimensions and index retention scans Co-Authored-By: Claude Fable 5 * fix(telemetry): pin tool-name allowlist and deploy session attribution Co-Authored-By: Claude Fable 5 * refactor(telemetry): route AI chat usage through feature_usage and drop ai_chat_usage Co-Authored-By: Claude Fable 5 * refactor(telemetry): slim dimension validation to registered kinds plus key shape Co-Authored-By: Claude Fable 5 * fix(telemetry): backfill ai_chat_usage into feature_usage before dropping it Co-Authored-By: Claude Fable 5 * fix(telemetry): disclose provider and model identifiers in telemetry settings text Co-Authored-By: Claude Fable 5 * fix(telemetry): issue all flush chunks before awaiting so pagehide keeps them Co-Authored-By: Claude Fable 5 * chore: update ee-repo-ref to 6306c072a50937ea9af44a5bcf42345543207486 This commit updates the EE repository reference after PR #672 was merged in windmill-ee-private. Previous ee-repo-ref: 964f242a0eb44db7f7d26636cc8d76aeabea2b73 New ee-repo-ref: 6306c072a50937ea9af44a5bcf42345543207486 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Fable 5 Co-authored-by: windmill-internal-app[bot] Co-authored-by: Ruben Fiszel --- ...b5014e1c889ba9181ee3331f128c56c9d22de.json | 12 ++ ...42bcbf66f0cc5900c7777141df47776e32fa3.json | 18 +++ ...83030e7a7ce32971a5621aceb738a3673f943.json | 44 ------ ...86122e4ceb34cc8993937dff24cdd7ba3fe5f.json | 12 -- ...d523ec2219920a64783cd5d2972f1826114cc.json | 17 --- ...3f75fb05a27ee759972b76e812b70843eb5e4.json | 62 ++++++++ backend/ee-repo-ref.txt | 2 +- .../20260720081307_add_feature_usage.down.sql | 1 + .../20260720081307_add_feature_usage.up.sql | 17 +++ ...20260720094300_drop_ai_chat_usage.down.sql | 11 ++ .../20260720094300_drop_ai_chat_usage.up.sql | 22 +++ backend/src/monitor.rs | 10 ++ .../windmill-api-workspaces/src/workspaces.rs | 99 +++++++++++-- backend/windmill-api/openapi.yaml | 37 +++-- .../lib/components/InstanceSettings.svelte | 6 +- .../copilot/chat/AIChatManager.svelte.ts | 42 ++++-- .../copilot/chat/AIChatManager.test.ts | 5 +- .../src/lib/components/copilot/chat/shared.ts | 6 + .../sessions/sessionDeployModel.svelte.ts | 10 ++ .../sessions/sessionPreviewTabs.svelte.ts | 4 + .../sessions/sessionRuntime.svelte.ts | 19 ++- .../sessions/sessionState.svelte.ts | 15 +- frontend/src/lib/utils/featureUsage.test.ts | 80 ++++++++++ frontend/src/lib/utils/featureUsage.ts | 137 ++++++++++++++++++ 24 files changed, 566 insertions(+), 122 deletions(-) create mode 100644 backend/.sqlx/query-0390d9e4fae597aabdeef940aa5b5014e1c889ba9181ee3331f128c56c9d22de.json create mode 100644 backend/.sqlx/query-2cdb9076747b61c01b6d389157e42bcbf66f0cc5900c7777141df47776e32fa3.json delete mode 100644 backend/.sqlx/query-3ec92c1682f3ce701028f66f7ce83030e7a7ce32971a5621aceb738a3673f943.json delete mode 100644 backend/.sqlx/query-98746829ab854dff922a04822bc86122e4ceb34cc8993937dff24cdd7ba3fe5f.json delete mode 100644 backend/.sqlx/query-b64f0f337aed44840cd68f4c81bd523ec2219920a64783cd5d2972f1826114cc.json create mode 100644 backend/.sqlx/query-c3a973b0eea69be747140426cd03f75fb05a27ee759972b76e812b70843eb5e4.json create mode 100644 backend/migrations/20260720081307_add_feature_usage.down.sql create mode 100644 backend/migrations/20260720081307_add_feature_usage.up.sql create mode 100644 backend/migrations/20260720094300_drop_ai_chat_usage.down.sql create mode 100644 backend/migrations/20260720094300_drop_ai_chat_usage.up.sql create mode 100644 frontend/src/lib/utils/featureUsage.test.ts create mode 100644 frontend/src/lib/utils/featureUsage.ts diff --git a/backend/.sqlx/query-0390d9e4fae597aabdeef940aa5b5014e1c889ba9181ee3331f128c56c9d22de.json b/backend/.sqlx/query-0390d9e4fae597aabdeef940aa5b5014e1c889ba9181ee3331f128c56c9d22de.json new file mode 100644 index 0000000000..bacf0f76c2 --- /dev/null +++ b/backend/.sqlx/query-0390d9e4fae597aabdeef940aa5b5014e1c889ba9181ee3331f128c56c9d22de.json @@ -0,0 +1,12 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM feature_usage WHERE day < CURRENT_DATE - 60", + "describe": { + "columns": [], + "parameters": { + "Left": [] + }, + "nullable": [] + }, + "hash": "0390d9e4fae597aabdeef940aa5b5014e1c889ba9181ee3331f128c56c9d22de" +} diff --git a/backend/.sqlx/query-2cdb9076747b61c01b6d389157e42bcbf66f0cc5900c7777141df47776e32fa3.json b/backend/.sqlx/query-2cdb9076747b61c01b6d389157e42bcbf66f0cc5900c7777141df47776e32fa3.json new file mode 100644 index 0000000000..2a67e64615 --- /dev/null +++ b/backend/.sqlx/query-2cdb9076747b61c01b6d389157e42bcbf66f0cc5900c7777141df47776e32fa3.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO feature_usage (feature, kind, key, entity_id, value)\n SELECT * FROM UNNEST($1::text[], $2::text[], $3::text[], $4::text[], $5::bigint[])\n ON CONFLICT (feature, kind, key, entity_id, day)\n DO UPDATE SET value = feature_usage.value + EXCLUDED.value, updated_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "TextArray", + "TextArray", + "TextArray", + "TextArray", + "Int8Array" + ] + }, + "nullable": [] + }, + "hash": "2cdb9076747b61c01b6d389157e42bcbf66f0cc5900c7777141df47776e32fa3" +} diff --git a/backend/.sqlx/query-3ec92c1682f3ce701028f66f7ce83030e7a7ce32971a5621aceb738a3673f943.json b/backend/.sqlx/query-3ec92c1682f3ce701028f66f7ce83030e7a7ce32971a5621aceb738a3673f943.json deleted file mode 100644 index 3d91feace1..0000000000 --- a/backend/.sqlx/query-3ec92c1682f3ce701028f66f7ce83030e7a7ce32971a5621aceb738a3673f943.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT provider, model, mode,\n COUNT(*)::BIGINT as \"session_count!\",\n COALESCE(SUM(message_count), 0)::BIGINT as \"message_count!\"\n FROM ai_chat_usage\n WHERE created_at > NOW() - INTERVAL '30 days'\n GROUP BY provider, model, mode\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "provider", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "model", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "mode", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "session_count!", - "type_info": "Int8" - }, - { - "ordinal": 4, - "name": "message_count!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - null, - null - ] - }, - "hash": "3ec92c1682f3ce701028f66f7ce83030e7a7ce32971a5621aceb738a3673f943" -} diff --git a/backend/.sqlx/query-98746829ab854dff922a04822bc86122e4ceb34cc8993937dff24cdd7ba3fe5f.json b/backend/.sqlx/query-98746829ab854dff922a04822bc86122e4ceb34cc8993937dff24cdd7ba3fe5f.json deleted file mode 100644 index 58f8729225..0000000000 --- a/backend/.sqlx/query-98746829ab854dff922a04822bc86122e4ceb34cc8993937dff24cdd7ba3fe5f.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM ai_chat_usage WHERE created_at < NOW() - INTERVAL '60 days'", - "describe": { - "columns": [], - "parameters": { - "Left": [] - }, - "nullable": [] - }, - "hash": "98746829ab854dff922a04822bc86122e4ceb34cc8993937dff24cdd7ba3fe5f" -} diff --git a/backend/.sqlx/query-b64f0f337aed44840cd68f4c81bd523ec2219920a64783cd5d2972f1826114cc.json b/backend/.sqlx/query-b64f0f337aed44840cd68f4c81bd523ec2219920a64783cd5d2972f1826114cc.json deleted file mode 100644 index ed4288ea5c..0000000000 --- a/backend/.sqlx/query-b64f0f337aed44840cd68f4c81bd523ec2219920a64783cd5d2972f1826114cc.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO ai_chat_usage (session_id, provider, model, mode) VALUES ($1, $2, $3, $4)\n ON CONFLICT (session_id) DO UPDATE SET message_count = ai_chat_usage.message_count + 1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Varchar", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "b64f0f337aed44840cd68f4c81bd523ec2219920a64783cd5d2972f1826114cc" -} diff --git a/backend/.sqlx/query-c3a973b0eea69be747140426cd03f75fb05a27ee759972b76e812b70843eb5e4.json b/backend/.sqlx/query-c3a973b0eea69be747140426cd03f75fb05a27ee759972b76e812b70843eb5e4.json new file mode 100644 index 0000000000..fa96843cc9 --- /dev/null +++ b/backend/.sqlx/query-c3a973b0eea69be747140426cd03f75fb05a27ee759972b76e812b70843eb5e4.json @@ -0,0 +1,62 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH per_entity AS (\n SELECT feature, kind, key, entity_id,\n SUM(value)::BIGINT AS value,\n MAX(day) AS last_day\n FROM feature_usage\n WHERE day > CURRENT_DATE - 30\n GROUP BY feature, kind, key, entity_id\n )\n SELECT feature, kind, key,\n COUNT(*)::BIGINT AS \"entity_count!\",\n COALESCE(SUM(value), 0)::BIGINT AS \"total_value!\",\n COALESCE(PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY value), 0)::DOUBLE PRECISION AS \"median_value!\",\n COALESCE(PERCENTILE_CONT(0.9) WITHIN GROUP (ORDER BY value), 0)::DOUBLE PRECISION AS \"p90_value!\",\n (COUNT(*) FILTER (WHERE last_day < CURRENT_DATE - 3))::BIGINT AS \"inactive_3d_entity_count!\"\n FROM per_entity\n GROUP BY feature, kind, key\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "feature", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "kind", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "key", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "entity_count!", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "total_value!", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "median_value!", + "type_info": "Float8" + }, + { + "ordinal": 6, + "name": "p90_value!", + "type_info": "Float8" + }, + { + "ordinal": 7, + "name": "inactive_3d_entity_count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false, + null, + null, + null, + null, + null + ] + }, + "hash": "c3a973b0eea69be747140426cd03f75fb05a27ee759972b76e812b70843eb5e4" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index dcb517e0eb..1a8edeed9c 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -a3adea1ffb406e709cc480871df58fab6c51aca1 +6306c072a50937ea9af44a5bcf42345543207486 diff --git a/backend/migrations/20260720081307_add_feature_usage.down.sql b/backend/migrations/20260720081307_add_feature_usage.down.sql new file mode 100644 index 0000000000..1b12797d72 --- /dev/null +++ b/backend/migrations/20260720081307_add_feature_usage.down.sql @@ -0,0 +1 @@ +DROP TABLE feature_usage; diff --git a/backend/migrations/20260720081307_add_feature_usage.up.sql b/backend/migrations/20260720081307_add_feature_usage.up.sql new file mode 100644 index 0000000000..6d17777b97 --- /dev/null +++ b/backend/migrations/20260720081307_add_feature_usage.up.sql @@ -0,0 +1,17 @@ +-- Generic product-telemetry accumulator: day-bucketed counters (entity_id = '') +-- and per-entity accumulators (e.g. messages per AI session). Aggregated into +-- the anonymous usage stats payload and pruned after 60 days. +CREATE TABLE feature_usage ( + feature VARCHAR(50) NOT NULL, + kind VARCHAR(50) NOT NULL, + key VARCHAR(100) NOT NULL DEFAULT '', + entity_id VARCHAR(50) NOT NULL DEFAULT '', + day DATE NOT NULL DEFAULT CURRENT_DATE, + value BIGINT NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (feature, kind, key, entity_id, day) +); + +-- The periodic retention delete filters on day alone; without this it would +-- full-scan the table (the PK only reaches day through four other columns). +CREATE INDEX idx_feature_usage_day ON feature_usage (day); diff --git a/backend/migrations/20260720094300_drop_ai_chat_usage.down.sql b/backend/migrations/20260720094300_drop_ai_chat_usage.down.sql new file mode 100644 index 0000000000..f4b3030011 --- /dev/null +++ b/backend/migrations/20260720094300_drop_ai_chat_usage.down.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS ai_chat_usage ( + id BIGSERIAL PRIMARY KEY, + session_id VARCHAR(36) NOT NULL UNIQUE, + provider VARCHAR(50) NOT NULL, + model VARCHAR(255) NOT NULL, + mode VARCHAR(50) NOT NULL, + message_count INT NOT NULL DEFAULT 1, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_ai_chat_usage_created_at ON ai_chat_usage (created_at); diff --git a/backend/migrations/20260720094300_drop_ai_chat_usage.up.sql b/backend/migrations/20260720094300_drop_ai_chat_usage.up.sql new file mode 100644 index 0000000000..7c72379b4f --- /dev/null +++ b/backend/migrations/20260720094300_drop_ai_chat_usage.up.sql @@ -0,0 +1,22 @@ +-- AI chat usage telemetry now flows through the generic feature_usage table +-- (ai_chat/message and ai_chat/model events). Backfill the accumulated rows so +-- no reporting window is lost, then drop the old table. Day-bucketing uses the +-- chat's first-message date; values are filtered to the identifier shape the +-- logging endpoint enforces. +INSERT INTO feature_usage (feature, kind, key, entity_id, day, value, updated_at) +SELECT 'ai_chat', 'message', mode, session_id, created_at::date, message_count, created_at +FROM ai_chat_usage +WHERE mode ~ '^[A-Za-z0-9_:./-]{1,100}$' + AND session_id ~ '^[A-Za-z0-9_:./-]{1,50}$' +ON CONFLICT (feature, kind, key, entity_id, day) +DO UPDATE SET value = feature_usage.value + EXCLUDED.value; + +INSERT INTO feature_usage (feature, kind, key, entity_id, day, value, updated_at) +SELECT 'ai_chat', 'model', provider || ':' || model, session_id, created_at::date, message_count, created_at +FROM ai_chat_usage +WHERE (provider || ':' || model) ~ '^[A-Za-z0-9_:./-]{1,100}$' + AND session_id ~ '^[A-Za-z0-9_:./-]{1,50}$' +ON CONFLICT (feature, kind, key, entity_id, day) +DO UPDATE SET value = feature_usage.value + EXCLUDED.value; + +DROP TABLE ai_chat_usage; diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 58af60c525..86f26aeb33 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -1362,6 +1362,16 @@ pub async fn delete_expired_items(db: &DB) -> () { tracing::error!("Error reaping stale join_pending_inputs slots: {:?}", e); } + // 60-day retention for anonymous feature-usage counters. Runs here (not only + // in the telemetry sender) so rows are pruned even when telemetry is disabled + // or the build has no stats scheduler. + if let Err(e) = sqlx::query!("DELETE FROM feature_usage WHERE day < CURRENT_DATE - 60") + .execute(db) + .await + { + tracing::error!("Error deleting old feature_usage rows: {e}"); + } + match sqlx::query_scalar!( "DELETE FROM agent_token_blacklist WHERE expires_at <= now() RETURNING token", ) diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 74b31c1410..7b5ff16bb1 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -199,7 +199,7 @@ pub fn workspaced_service() -> Router { "/protection_rules/{rule_name}", post(update_protection_rule).delete(delete_protection_rule), ) - .route("/log_chat", post(log_ai_chat)) + .route("/log_feature_usage", post(log_feature_usage)) .route("/cloud_quotas", get(get_cloud_quotas)) .route("/prune_versions", post(prune_versions)) .route("/list_ws_specific", get(list_ws_specific)) @@ -9559,25 +9559,96 @@ const TRIGGER_OR_SCHEDULE_TABLES: &[&str] = &[ "email_trigger", ]; +const MAX_FEATURE_USAGE_EVENTS: usize = 50; + #[derive(Deserialize)] -struct LogAiChatPayload { - session_id: String, - provider: String, - model: String, - mode: String, +struct FeatureUsageEvent { + feature: String, + kind: String, + #[serde(default)] + key: String, + #[serde(default)] + entity_id: String, + value: Option, } -async fn log_ai_chat( +#[derive(Deserialize)] +struct LogFeatureUsagePayload { + events: Vec, +} + +// Only registered (feature, kind) actions are accepted, so telemetry stays +// limited to predefined feature actions. Keys are shape-checked (identifier-like, +// no spaces) rather than pinned to value sets: they come from our own frontend +// (modes, tab/draft kinds, tool names, provider:model) and pinning every value +// server-side was not worth the maintenance. +const FEATURE_USAGE_KINDS: &[(&str, &str)] = &[ + ("ai_session", "created"), + ("ai_session", "message"), + ("ai_session", "autonomy"), + ("ai_session", "tab"), + ("ai_session", "tokens"), + ("ai_session", "deployed"), + ("ai_session", "archived"), + ("ai_session", "deleted"), + ("ai_chat", "message"), + ("ai_chat", "model"), + ("ai_chat", "tool"), +]; + +fn is_identifier_shaped(s: &str, max_len: usize) -> bool { + !s.is_empty() + && s.len() <= max_len + && s.chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | ':' | '.' | '/')) +} + +fn valid_feature_usage_event(e: &FeatureUsageEvent) -> bool { + FEATURE_USAGE_KINDS.contains(&(e.feature.as_str(), e.kind.as_str())) + && (e.key.is_empty() || is_identifier_shaped(&e.key, 100)) + && (e.entity_id.is_empty() || is_identifier_shaped(&e.entity_id, 50)) +} + +async fn log_feature_usage( Extension(db): Extension, - Json(payload): Json, + Json(payload): Json, ) -> Result { + // Pre-sum duplicate keys: two rows hitting the same conflict target in a + // single INSERT error out ("cannot affect row a second time"). + let mut agg: HashMap<(String, String, String, String), i64> = HashMap::new(); + for e in payload.events.into_iter().take(MAX_FEATURE_USAGE_EVENTS) { + if !valid_feature_usage_event(&e) { + continue; + } + let value = e.value.unwrap_or(1).clamp(1, 1_000_000); + *agg.entry((e.feature, e.kind, e.key, e.entity_id)) + .or_insert(0) += value; + } + if agg.is_empty() { + return Ok(StatusCode::NO_CONTENT); + } + let mut features = Vec::with_capacity(agg.len()); + let mut kinds = Vec::with_capacity(agg.len()); + let mut keys = Vec::with_capacity(agg.len()); + let mut entity_ids = Vec::with_capacity(agg.len()); + let mut values = Vec::with_capacity(agg.len()); + for ((feature, kind, key, entity_id), value) in agg { + features.push(feature); + kinds.push(kind); + keys.push(key); + entity_ids.push(entity_id); + values.push(value); + } sqlx::query!( - "INSERT INTO ai_chat_usage (session_id, provider, model, mode) VALUES ($1, $2, $3, $4) - ON CONFLICT (session_id) DO UPDATE SET message_count = ai_chat_usage.message_count + 1", - &payload.session_id, - &payload.provider, - &payload.model, - &payload.mode + "INSERT INTO feature_usage (feature, kind, key, entity_id, value) + SELECT * FROM UNNEST($1::text[], $2::text[], $3::text[], $4::text[], $5::bigint[]) + ON CONFLICT (feature, kind, key, entity_id, day) + DO UPDATE SET value = feature_usage.value + EXCLUDED.value, updated_at = now()", + &features, + &kinds, + &keys, + &entity_ids, + &values ) .execute(&db) .await?; diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index e5bc24d360..883ab5ec74 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -6505,10 +6505,10 @@ paths: "400": description: invalid input or request closed - /w/{workspace}/workspaces/log_chat: + /w/{workspace}/workspaces/log_feature_usage: post: - summary: log AI chat message - operationId: logAiChat + summary: log anonymous feature usage telemetry events + operationId: logFeatureUsage tags: - workspace parameters: @@ -6520,19 +6520,26 @@ paths: schema: type: object required: - - session_id - - provider - - model - - mode + - events properties: - session_id: - type: string - provider: - type: string - model: - type: string - mode: - type: string + events: + type: array + items: + type: object + required: + - feature + - kind + properties: + feature: + type: string + kind: + type: string + key: + type: string + entity_id: + type: string + value: + type: integer responses: "204": description: logged diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index b3d0ee1446..13a7a41ccd 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -1061,7 +1061,8 @@
  • job usage (language, total duration, count)
  • git sync repo count (sync vs promotion mode)
  • AI chat usage (provider, model, mode, session count, message count — last 30 days)
    • feature usage telemetry: aggregated AI chat and AI session usage counts, including AI + provider and model identifiers (last 30 days)
  • resource counts (workspaces, scripts per language, flows, workflows as code, low-code @@ -1107,7 +1108,8 @@
  • user usage (author count, operator count)
  • development instance status
  • AI chat usage (provider, model, mode, session count, message count — last 30 days)
    • feature usage telemetry: aggregated AI chat and AI session usage counts, including AI + provider and model identifiers (last 30 days)
  • resource counts (workspaces, scripts per language, flows, workflows as code, low-code diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index b9649200ab..6e07271a60 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -1,5 +1,5 @@ import type { ScriptLang } from '$lib/gen/types.gen' -import { WorkspaceService, JobService, type CompletedJob } from '$lib/gen' +import { JobService, type CompletedJob } from '$lib/gen' import type { FlowOptions, ScriptOptions } from './ContextManager.svelte' import { flowTools, @@ -45,6 +45,7 @@ import { prepareScriptUserMessage } from './script/core' import { prepareNavigatorUserMessage } from './navigator/core' import { sendUserToast } from '$lib/toast' import { workspaceAIClients, getNonStreamingCompletion } from '../lib' +import { logFeatureUsage } from '$lib/utils/featureUsage' import { modelSupportsVision } from '../modelConfig' import { getKnownModelContextWindow } from '../modelConfig' import { @@ -2078,6 +2079,13 @@ export class AIChatManager { } } }) + if (this.isSessionChat && this.sessionId && result.tokenUsage.total > 0) { + logFeatureUsage('ai_session', 'tokens', { + entityId: this.sessionId, + value: result.tokenUsage.total, + workspace: this.operatingWorkspace + }) + } return result } catch (err) { console.log('chatRequest error', err) @@ -2435,15 +2443,29 @@ export class AIChatManager { const model = tryGetCurrentModel() if (model) { - WorkspaceService.logAiChat({ - workspace: this.operatingWorkspace ?? '', - requestBody: { - session_id: this.historyManager.getCurrentChatId(), - provider: model.provider, - model: model.model, - mode: this.mode - } - }).catch(() => {}) + const chatId = this.historyManager.getCurrentChatId() + logFeatureUsage('ai_chat', 'message', { + key: this.mode, + entityId: chatId, + workspace: this.operatingWorkspace + }) + logFeatureUsage('ai_chat', 'model', { + key: `${model.provider}:${model.model}`, + entityId: chatId, + workspace: this.operatingWorkspace + }) + } + if (this.isSessionChat && this.sessionId) { + logFeatureUsage('ai_session', 'message', { + key: this.mode, + entityId: this.sessionId, + workspace: this.operatingWorkspace + }) + logFeatureUsage('ai_session', 'autonomy', { + key: this.autonomyMode, + entityId: this.sessionId, + workspace: this.operatingWorkspace + }) } if (this.mode === AIMode.FLOW && !this.flowAiChatHelpers) { diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts index 55bea0d7e6..ba6508e87c 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts @@ -23,7 +23,6 @@ const mocks = vi.hoisted(() => ({ getCurrentModel: vi.fn(), tryGetCurrentModel: vi.fn(), isWebSearchEnabledForProvider: vi.fn(), - logAiChat: vi.fn(), sendUserToast: vi.fn(), getOpenaiClient: vi.fn(), getAnthropicClient: vi.fn(), @@ -38,9 +37,10 @@ vi.mock('monaco-editor', () => ({ Selection: class Selection {} })) +vi.mock('$lib/utils/featureUsage', () => ({ logFeatureUsage: vi.fn() })) + vi.mock('$lib/gen', () => ({ WorkspaceService: { - logAiChat: mocks.logAiChat, listAiSkills: mocks.listAiSkills }, ScriptService: {}, @@ -129,7 +129,6 @@ beforeEach(() => { mocks.getCurrentModel.mockReturnValue(undefined) mocks.tryGetCurrentModel.mockReturnValue(undefined) mocks.isWebSearchEnabledForProvider.mockReturnValue(true) - mocks.logAiChat.mockResolvedValue(undefined) mocks.getOpenaiClient.mockReturnValue({}) mocks.getAnthropicClient.mockReturnValue({}) mocks.listAiSkills.mockResolvedValue([]) diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index e37ae1da7f..a1f79f181e 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -39,6 +39,7 @@ import { } from '$lib/gen' import uFuzzy from '@leeoniya/ufuzzy' import { emptyString } from '$lib/utils' +import { logFeatureUsage } from '$lib/utils/featureUsage' import { forLater } from '$lib/forLater' import { scriptLangToEditorLang } from '$lib/scripts' import { getCurrentModel } from '$lib/aiStore' @@ -763,6 +764,11 @@ export async function processToolCall({ } let result = '' + // Key by the resolved tool's declared name, not the model-provided string, + // so hallucinated tool names never enter telemetry. + if (tool) { + logFeatureUsage('ai_chat', 'tool', { key: tool.def.function.name, workspace: workspaceId }) + } try { result = await callTool({ tools, diff --git a/frontend/src/lib/components/sessions/sessionDeployModel.svelte.ts b/frontend/src/lib/components/sessions/sessionDeployModel.svelte.ts index 28f0075880..8332782cdf 100644 --- a/frontend/src/lib/components/sessions/sessionDeployModel.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionDeployModel.svelte.ts @@ -23,6 +23,8 @@ import { type DeployPlanEntry } from './sessionDeployModel' import { maskKey } from './modifiedItemsMask' +import { sessionState } from './sessionState.svelte' +import { logFeatureUsage } from '$lib/utils/featureUsage' export type DeploymentStatus = { status: 'loading' | 'failed'; error?: string } @@ -261,6 +263,9 @@ export function useSessionDeployModel(getArgs: () => SessionDeployModelArgs) { async function deployOne(item: DeployItem, discard = false): Promise { const plan = discard ? discardPlanFor(item) : deployPlanFor(item) if (!plan) return false + // Snapshot before the await: the user may switch sessions while the + // deploy runs, and the event belongs to the initiating session. + const initiatingSessionId = sessionState.currentSessionId // Don't attempt a deploy we know the user can't make (no write permission // on the path, or blocked by the operator / deployer rule) — the UI // disables it too; this is the guard behind that. @@ -280,6 +285,11 @@ export function useSessionDeployModel(getArgs: () => SessionDeployModelArgs) { .add(item.key) .add(maskKey(item.draftKind, item.displayPath)) getArgs().onItemDeployed?.(item) + logFeatureUsage('ai_session', 'deployed', { + key: item.draftKind, + entityId: initiatingSessionId, + workspace: getArgs().workspaceId + }) } } return res.success diff --git a/frontend/src/lib/components/sessions/sessionPreviewTabs.svelte.ts b/frontend/src/lib/components/sessions/sessionPreviewTabs.svelte.ts index c88387d1b4..27b0dc2454 100644 --- a/frontend/src/lib/components/sessions/sessionPreviewTabs.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionPreviewTabs.svelte.ts @@ -35,6 +35,9 @@ export type PreviewTabsAdapter = { // Fired synchronously on every tab-set change, so the runtime can drop editor // cells no open tab references anymore (a closed / navigated-away item). onTabsChanged?: () => void + // Fired when open() creates a brand-new tab (not focus/retarget of an + // existing one), with the tab's initial URL. + onTabOpened?: (url: string) => void } // True when a tab's URL is the live editor for a specific editable item. Every @@ -268,6 +271,7 @@ export class SessionPreviewTabs { this.#tabs.push(tab) this.#activeId = tab.id this.#flush() + this.#adapter.onTabOpened?.(url) return { status: 'opened' } } diff --git a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts index fa0dc52946..7b864e28e4 100644 --- a/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionRuntime.svelte.ts @@ -49,7 +49,13 @@ import { previewTargetForSessionTarget, selectPreviewTabsToClose } from './sessionPreviewTabs.svelte' -import { matchPreviewPage, parsePreviewItemRoute, previewLocationLabel } from './previewRouter' +import { + matchPreviewPage, + parsePreviewItemRoute, + previewLocationLabel, + resolvePreviewTab +} from './previewRouter' +import { logFeatureUsage } from '$lib/utils/featureUsage' import { UserDraft } from '$lib/userDraft.svelte' import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' import { armRestartOnFirstInteraction } from '$lib/userDraftToast' @@ -432,7 +438,16 @@ function createRuntime(session: Session): SessionRuntime { // Only persist a real width; undefined means "never resized" (defaults to 50). if (snap.previewSize != null) setSessionPreviewSize(session.id, snap.previewSize) }, - onTabsChanged: pruneEditorCells + onTabsChanged: pruneEditorCells, + onTabOpened: (url) => { + const slot = resolvePreviewTab(url) + logFeatureUsage('ai_session', 'tab', { + key: + slot.kind === 'editor' ? slot.editorKind : slot.kind === 'artifact' ? 'artifact' : 'page', + entityId: session.id, + workspace: getEffectiveWorkspaceId(session) + }) + } }) // Let the jobs tray open a run in this session's preview panel (as an iframe diff --git a/frontend/src/lib/components/sessions/sessionState.svelte.ts b/frontend/src/lib/components/sessions/sessionState.svelte.ts index 1c38838787..26b90abd11 100644 --- a/frontend/src/lib/components/sessions/sessionState.svelte.ts +++ b/frontend/src/lib/components/sessions/sessionState.svelte.ts @@ -18,6 +18,7 @@ import { protectionRulesState } from '$lib/workspaceProtectionRules.svelte' import { getLocalSetting, storeLocalSetting } from '$lib/utils' +import { logFeatureUsage } from '$lib/utils/featureUsage' import { workspaceRootId } from './sessionScope.svelte' import { type DBSchema, type IDBPDatabase } from 'idb' import { userScopedDb } from '$lib/userScopedDb' @@ -795,6 +796,7 @@ export async function commitSessionWorkspace( // The draft prompt has been consumed as the first message. delete s.draftPrompt await putSession(s) + logFeatureUsage('ai_session', 'created', { key: 'fork', entityId: s.id, workspace: newId }) // The global workspaceStore is intentionally left untouched: the session // chat targets its own workspace via AIChatManager.operatingWorkspace, so // committing must not yank the user's active (navigation-mode) workspace. @@ -809,6 +811,12 @@ export async function commitSessionWorkspace( // The draft prompt has been consumed as the first message. delete s.draftPrompt await putSession(s) + // A picked workspace can itself be an existing fork — classify by root. + logFeatureUsage('ai_session', 'created', { + key: ws === s.workspace_root_id ? 'root' : 'fork', + entityId: s.id, + workspace: ws + }) // The global workspaceStore is intentionally left untouched (see the fork // branch above): the session chat reads its committed workspace through the // manager's workspace resolver, not the active workspaceStore. @@ -958,8 +966,10 @@ export function setSessionArchived(id: string, archived: boolean) { if (!s) return const next = archived ? true : undefined if (s.archived === next && (archived || !s.archivedByWorkspace)) return - if (archived) s.archived = true - else { + if (archived) { + s.archived = true + logFeatureUsage('ai_session', 'archived', { entityId: s.id, workspace: s.workspace_id }) + } else { delete s.archived delete s.archivedByWorkspace } @@ -982,6 +992,7 @@ export function deleteSession(id: string) { // GC any linked files and artifacts persisted for this session. void deleteItemsForSession(id) void deleteArtifactsForSession(id) + logFeatureUsage('ai_session', 'deleted', { entityId: id, workspace: s.workspace_id }) } export function setSessionChatId(sessionId: string, chatId: string) { diff --git a/frontend/src/lib/utils/featureUsage.test.ts b/frontend/src/lib/utils/featureUsage.test.ts new file mode 100644 index 0000000000..3e13ff17d7 --- /dev/null +++ b/frontend/src/lib/utils/featureUsage.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('$lib/gen', () => ({ OpenAPI: { BASE: '/api' } })) +vi.mock('$lib/stores', () => ({ workspaceStore: { subscribe: () => () => {} } })) + +import { createFeatureUsageBuffer, type FeatureUsageEventPayload } from './featureUsage' + +describe('createFeatureUsageBuffer', () => { + it('sums repeated events per (feature, kind, key, entity) and flushes one batch', async () => { + const send = vi.fn().mockResolvedValue(undefined) + const buffer = createFeatureUsageBuffer(send, () => 'ws1') + + buffer.log('ai_session', 'message', { key: 'global', entityId: 's1' }) + buffer.log('ai_session', 'message', { key: 'global', entityId: 's1' }) + buffer.log('ai_session', 'tokens', { entityId: 's1', value: 120 }) + buffer.log('ai_session', 'message', { key: 'global', entityId: 's2' }) + await buffer.flush() + + expect(send).toHaveBeenCalledTimes(1) + const [workspace, events] = send.mock.calls[0] + expect(workspace).toBe('ws1') + expect(events).toEqual( + expect.arrayContaining([ + { feature: 'ai_session', kind: 'message', key: 'global', entity_id: 's1', value: 2 }, + { feature: 'ai_session', kind: 'tokens', key: '', entity_id: 's1', value: 120 }, + { feature: 'ai_session', kind: 'message', key: 'global', entity_id: 's2', value: 1 } + ]) + ) + expect(events).toHaveLength(3) + + // Flushed events must not be re-sent. + await buffer.flush() + expect(send).toHaveBeenCalledTimes(1) + }) + + it('splits batches per workspace and drops events without any workspace', async () => { + const send = vi.fn().mockResolvedValue(undefined) + const buffer = createFeatureUsageBuffer(send, () => undefined) + + buffer.log('ai_session', 'created', { key: 'fork' }) // no workspace -> dropped + buffer.log('ai_session', 'created', { key: 'fork', workspace: 'ws1' }) + buffer.log('ai_session', 'created', { key: 'root', workspace: 'ws2' }) + await buffer.flush() + + expect(send).toHaveBeenCalledTimes(2) + const workspaces = send.mock.calls.map((c) => c[0]).sort() + expect(workspaces).toEqual(['ws1', 'ws2']) + }) + + it('starts every chunk request before any send resolves (pagehide flush)', async () => { + const send = vi.fn().mockReturnValue(new Promise(() => {})) + const buffer = createFeatureUsageBuffer(send, () => 'ws1') + + for (let i = 0; i < 60; i++) { + buffer.log('ai_session', 'tool', { key: `tool_${i}` }) + } + buffer.log('ai_session', 'message', { workspace: 'ws2' }) + void buffer.flush() + await Promise.resolve() + + // keepalive only protects requests that were issued; a sequential flush + // would have started just the first chunk here. + expect(send).toHaveBeenCalledTimes(3) + }) + + it('chunks flushes above the per-request cap and survives send failures', async () => { + const send = vi.fn().mockRejectedValueOnce(new Error('network')).mockResolvedValue(undefined) + const buffer = createFeatureUsageBuffer(send, () => 'ws1') + + for (let i = 0; i < 60; i++) { + buffer.log('ai_session', 'tool', { key: `tool_${i}` }) + } + await expect(buffer.flush()).resolves.toBeUndefined() + + expect(send).toHaveBeenCalledTimes(2) + const sent = send.mock.calls.flatMap((c) => c[1] as FeatureUsageEventPayload[]) + expect(send.mock.calls[0][1]).toHaveLength(50) + expect(sent).toHaveLength(60) + }) +}) diff --git a/frontend/src/lib/utils/featureUsage.ts b/frontend/src/lib/utils/featureUsage.ts new file mode 100644 index 0000000000..6e02e4ce0e --- /dev/null +++ b/frontend/src/lib/utils/featureUsage.ts @@ -0,0 +1,137 @@ +import { get } from 'svelte/store' +import { OpenAPI } from '$lib/gen' +import { workspaceStore } from '$lib/stores' + +// Anonymous product-usage counters (e.g. AI session activity), batched into the +// backend `feature_usage` accumulator. Only aggregated counts ever leave the +// instance, and only when telemetry is enabled and not in minimal mode — never +// log paths, prompts, code, or user identifiers here (entity ids must be +// opaque random ids). + +export interface FeatureUsageOpts { + key?: string + entityId?: string + value?: number + /** Workspace whose API route carries the batch; defaults to the active workspace. */ + workspace?: string +} + +type SendFn = (workspace: string, events: FeatureUsageEventPayload[]) => Promise + +export interface FeatureUsageEventPayload { + feature: string + kind: string + key?: string + entity_id?: string + value?: number +} + +const FLUSH_INTERVAL_MS = 30_000 +// Backend caps a batch at 50 events; chunk larger flushes. +const MAX_EVENTS_PER_REQUEST = 50 + +export function createFeatureUsageBuffer( + send: SendFn, + getDefaultWorkspace: () => string | undefined, + flushIntervalMs = FLUSH_INTERVAL_MS +) { + // One accumulator per (workspace, feature, kind, key, entityId): repeated + // events sum locally so a chatty UI still produces one upsert per flush. + const pending = new Map() + let timer: ReturnType | undefined + + function log(feature: string, kind: string, opts: FeatureUsageOpts = {}): void { + const workspace = opts.workspace ?? getDefaultWorkspace() + if (!workspace) return + const key = opts.key ?? '' + const entityId = opts.entityId ?? '' + const value = Math.max(1, Math.round(opts.value ?? 1)) + const mapKey = `${workspace}\u0000${feature}\u0000${kind}\u0000${key}\u0000${entityId}` + const existing = pending.get(mapKey) + if (existing) { + existing.event.value = (existing.event.value ?? 1) + value + } else { + pending.set(mapKey, { + workspace, + event: { feature, kind, key, entity_id: entityId, value } + }) + } + if (timer === undefined) { + timer = setTimeout(() => { + timer = undefined + void flush() + }, flushIntervalMs) + } + } + + async function flush(): Promise { + if (timer !== undefined) { + clearTimeout(timer) + timer = undefined + } + if (pending.size === 0) return + const byWorkspace = new Map() + for (const { workspace, event } of pending.values()) { + let events = byWorkspace.get(workspace) + if (!events) { + events = [] + byWorkspace.set(workspace, events) + } + events.push(event) + } + pending.clear() + // Start every chunk request synchronously before awaiting: the pagehide + // flush only protects requests that were already issued (keepalive can't + // help a fetch that never started). + const inflight: Promise[] = [] + for (const [workspace, events] of byWorkspace) { + for (let i = 0; i < events.length; i += MAX_EVENTS_PER_REQUEST) { + inflight.push( + send(workspace, events.slice(i, i + MAX_EVENTS_PER_REQUEST)).catch(() => { + // Telemetry is best-effort: drop the batch rather than retry. + }) + ) + } + } + await Promise.all(inflight) + } + + return { log, flush } +} + +const buffer = createFeatureUsageBuffer( + async (workspace, events) => { + // Raw fetch instead of the generated client: `keepalive` lets the request + // finish after tab close/navigation, which is when the final flush runs. + // Auth rides on the token cookie (WITH_CREDENTIALS app setup). + await fetch(`${OpenAPI.BASE}/w/${encodeURIComponent(workspace)}/workspaces/log_feature_usage`, { + method: 'POST', + credentials: 'include', + keepalive: true, + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ events }) + }) + }, + () => get(workspaceStore) ?? undefined +) + +if (typeof document !== 'undefined') { + // Flush what's buffered before the tab goes away. pagehide covers + // close/navigation paths where visibilitychange is not delivered. + document.addEventListener('visibilitychange', () => { + if (document.visibilityState === 'hidden') { + void buffer.flush() + } + }) + window.addEventListener('pagehide', () => { + void buffer.flush() + }) +} + +/** + * Record an anonymous feature-usage event. Fire-and-forget: events are summed + * locally per (feature, kind, key, entityId) and flushed in batches. + */ +export function logFeatureUsage(feature: string, kind: string, opts: FeatureUsageOpts = {}): void { + buffer.log(feature, kind, opts) +}