mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-08-18 16:02:10 +00:00
feat(telemetry): generic feature-usage telemetry with AI session metrics (#10200)
* feat(telemetry): add generic feature_usage table and batched logging endpoint Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * feat(telemetry): log AI session usage events and document them in telemetry settings Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(telemetry): use escape sequence instead of literal NUL bytes in buffer key Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(telemetry): validate dimensions, decouple retention, keepalive flush Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(telemetry): allowlist feature-usage dimensions and index retention scans Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(telemetry): pin tool-name allowlist and deploy session attribution Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * refactor(telemetry): route AI chat usage through feature_usage and drop ai_chat_usage Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * refactor(telemetry): slim dimension validation to registered kinds plus key shape Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(telemetry): backfill ai_chat_usage into feature_usage before dropping it Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(telemetry): disclose provider and model identifiers in telemetry settings text Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(telemetry): issue all flush chunks before awaiting so pagehide keeps them Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * chore: update ee-repo-ref to 6306c072a50937ea9af44a5bcf42345543207486 This commit updates the EE repository reference after PR #672 was merged in windmill-ee-private. Previous ee-repo-ref: 964f242a0eb44db7f7d26636cc8d76aeabea2b73 New ee-repo-ref: 6306c072a50937ea9af44a5bcf42345543207486 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com> Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
This commit is contained in:
hugocasa
GitHub
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "DELETE FROM feature_usage WHERE day < CURRENT_DATE - 60",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": []
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "0390d9e4fae597aabdeef940aa5b5014e1c889ba9181ee3331f128c56c9d22de"
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO feature_usage (feature, kind, key, entity_id, value)\n SELECT * FROM UNNEST($1::text[], $2::text[], $3::text[], $4::text[], $5::bigint[])\n ON CONFLICT (feature, kind, key, entity_id, day)\n DO UPDATE SET value = feature_usage.value + EXCLUDED.value, updated_at = now()",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"Int8Array"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "2cdb9076747b61c01b6d389157e42bcbf66f0cc5900c7777141df47776e32fa3"
|
||||
}
|
||||
-44
@@ -1,44 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n SELECT provider, model, mode,\n COUNT(*)::BIGINT as \"session_count!\",\n COALESCE(SUM(message_count), 0)::BIGINT as \"message_count!\"\n FROM ai_chat_usage\n WHERE created_at > NOW() - INTERVAL '30 days'\n GROUP BY provider, model, mode\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "provider",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "model",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "mode",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "session_count!",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "message_count!",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": []
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "3ec92c1682f3ce701028f66f7ce83030e7a7ce32971a5621aceb738a3673f943"
|
||||
}
|
||||
-12
@@ -1,12 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "DELETE FROM ai_chat_usage WHERE created_at < NOW() - INTERVAL '60 days'",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": []
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "98746829ab854dff922a04822bc86122e4ceb34cc8993937dff24cdd7ba3fe5f"
|
||||
}
|
||||
-17
@@ -1,17 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO ai_chat_usage (session_id, provider, model, mode) VALUES ($1, $2, $3, $4)\n ON CONFLICT (session_id) DO UPDATE SET message_count = ai_chat_usage.message_count + 1",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Varchar"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "b64f0f337aed44840cd68f4c81bd523ec2219920a64783cd5d2972f1826114cc"
|
||||
}
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n WITH per_entity AS (\n SELECT feature, kind, key, entity_id,\n SUM(value)::BIGINT AS value,\n MAX(day) AS last_day\n FROM feature_usage\n WHERE day > CURRENT_DATE - 30\n GROUP BY feature, kind, key, entity_id\n )\n SELECT feature, kind, key,\n COUNT(*)::BIGINT AS \"entity_count!\",\n COALESCE(SUM(value), 0)::BIGINT AS \"total_value!\",\n COALESCE(PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY value), 0)::DOUBLE PRECISION AS \"median_value!\",\n COALESCE(PERCENTILE_CONT(0.9) WITHIN GROUP (ORDER BY value), 0)::DOUBLE PRECISION AS \"p90_value!\",\n (COUNT(*) FILTER (WHERE last_day < CURRENT_DATE - 3))::BIGINT AS \"inactive_3d_entity_count!\"\n FROM per_entity\n GROUP BY feature, kind, key\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "feature",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "kind",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "key",
|
||||
"type_info": "Varchar"
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "entity_count!",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "total_value!",
|
||||
"type_info": "Int8"
|
||||
},
|
||||
{
|
||||
"ordinal": 5,
|
||||
"name": "median_value!",
|
||||
"type_info": "Float8"
|
||||
},
|
||||
{
|
||||
"ordinal": 6,
|
||||
"name": "p90_value!",
|
||||
"type_info": "Float8"
|
||||
},
|
||||
{
|
||||
"ordinal": 7,
|
||||
"name": "inactive_3d_entity_count!",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": []
|
||||
},
|
||||
"nullable": [
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null,
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "c3a973b0eea69be747140426cd03f75fb05a27ee759972b76e812b70843eb5e4"
|
||||
}
|
||||
@@ -1 +1 @@
|
||||
a3adea1ffb406e709cc480871df58fab6c51aca1
|
||||
6306c072a50937ea9af44a5bcf42345543207486
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
DROP TABLE feature_usage;
|
||||
@@ -0,0 +1,17 @@
|
||||
-- Generic product-telemetry accumulator: day-bucketed counters (entity_id = '')
|
||||
-- and per-entity accumulators (e.g. messages per AI session). Aggregated into
|
||||
-- the anonymous usage stats payload and pruned after 60 days.
|
||||
CREATE TABLE feature_usage (
|
||||
feature VARCHAR(50) NOT NULL,
|
||||
kind VARCHAR(50) NOT NULL,
|
||||
key VARCHAR(100) NOT NULL DEFAULT '',
|
||||
entity_id VARCHAR(50) NOT NULL DEFAULT '',
|
||||
day DATE NOT NULL DEFAULT CURRENT_DATE,
|
||||
value BIGINT NOT NULL DEFAULT 0,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (feature, kind, key, entity_id, day)
|
||||
);
|
||||
|
||||
-- The periodic retention delete filters on day alone; without this it would
|
||||
-- full-scan the table (the PK only reaches day through four other columns).
|
||||
CREATE INDEX idx_feature_usage_day ON feature_usage (day);
|
||||
@@ -0,0 +1,11 @@
|
||||
CREATE TABLE IF NOT EXISTS ai_chat_usage (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
session_id VARCHAR(36) NOT NULL UNIQUE,
|
||||
provider VARCHAR(50) NOT NULL,
|
||||
model VARCHAR(255) NOT NULL,
|
||||
mode VARCHAR(50) NOT NULL,
|
||||
message_count INT NOT NULL DEFAULT 1,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_ai_chat_usage_created_at ON ai_chat_usage (created_at);
|
||||
@@ -0,0 +1,22 @@
|
||||
-- AI chat usage telemetry now flows through the generic feature_usage table
|
||||
-- (ai_chat/message and ai_chat/model events). Backfill the accumulated rows so
|
||||
-- no reporting window is lost, then drop the old table. Day-bucketing uses the
|
||||
-- chat's first-message date; values are filtered to the identifier shape the
|
||||
-- logging endpoint enforces.
|
||||
INSERT INTO feature_usage (feature, kind, key, entity_id, day, value, updated_at)
|
||||
SELECT 'ai_chat', 'message', mode, session_id, created_at::date, message_count, created_at
|
||||
FROM ai_chat_usage
|
||||
WHERE mode ~ '^[A-Za-z0-9_:./-]{1,100}$'
|
||||
AND session_id ~ '^[A-Za-z0-9_:./-]{1,50}$'
|
||||
ON CONFLICT (feature, kind, key, entity_id, day)
|
||||
DO UPDATE SET value = feature_usage.value + EXCLUDED.value;
|
||||
|
||||
INSERT INTO feature_usage (feature, kind, key, entity_id, day, value, updated_at)
|
||||
SELECT 'ai_chat', 'model', provider || ':' || model, session_id, created_at::date, message_count, created_at
|
||||
FROM ai_chat_usage
|
||||
WHERE (provider || ':' || model) ~ '^[A-Za-z0-9_:./-]{1,100}$'
|
||||
AND session_id ~ '^[A-Za-z0-9_:./-]{1,50}$'
|
||||
ON CONFLICT (feature, kind, key, entity_id, day)
|
||||
DO UPDATE SET value = feature_usage.value + EXCLUDED.value;
|
||||
|
||||
DROP TABLE ai_chat_usage;
|
||||
@@ -1362,6 +1362,16 @@ pub async fn delete_expired_items(db: &DB) -> () {
|
||||
tracing::error!("Error reaping stale join_pending_inputs slots: {:?}", e);
|
||||
}
|
||||
|
||||
// 60-day retention for anonymous feature-usage counters. Runs here (not only
|
||||
// in the telemetry sender) so rows are pruned even when telemetry is disabled
|
||||
// or the build has no stats scheduler.
|
||||
if let Err(e) = sqlx::query!("DELETE FROM feature_usage WHERE day < CURRENT_DATE - 60")
|
||||
.execute(db)
|
||||
.await
|
||||
{
|
||||
tracing::error!("Error deleting old feature_usage rows: {e}");
|
||||
}
|
||||
|
||||
match sqlx::query_scalar!(
|
||||
"DELETE FROM agent_token_blacklist WHERE expires_at <= now() RETURNING token",
|
||||
)
|
||||
|
||||
@@ -199,7 +199,7 @@ pub fn workspaced_service() -> Router {
|
||||
"/protection_rules/{rule_name}",
|
||||
post(update_protection_rule).delete(delete_protection_rule),
|
||||
)
|
||||
.route("/log_chat", post(log_ai_chat))
|
||||
.route("/log_feature_usage", post(log_feature_usage))
|
||||
.route("/cloud_quotas", get(get_cloud_quotas))
|
||||
.route("/prune_versions", post(prune_versions))
|
||||
.route("/list_ws_specific", get(list_ws_specific))
|
||||
@@ -9559,25 +9559,96 @@ const TRIGGER_OR_SCHEDULE_TABLES: &[&str] = &[
|
||||
"email_trigger",
|
||||
];
|
||||
|
||||
const MAX_FEATURE_USAGE_EVENTS: usize = 50;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct LogAiChatPayload {
|
||||
session_id: String,
|
||||
provider: String,
|
||||
model: String,
|
||||
mode: String,
|
||||
struct FeatureUsageEvent {
|
||||
feature: String,
|
||||
kind: String,
|
||||
#[serde(default)]
|
||||
key: String,
|
||||
#[serde(default)]
|
||||
entity_id: String,
|
||||
value: Option<i64>,
|
||||
}
|
||||
|
||||
async fn log_ai_chat(
|
||||
#[derive(Deserialize)]
|
||||
struct LogFeatureUsagePayload {
|
||||
events: Vec<FeatureUsageEvent>,
|
||||
}
|
||||
|
||||
// Only registered (feature, kind) actions are accepted, so telemetry stays
|
||||
// limited to predefined feature actions. Keys are shape-checked (identifier-like,
|
||||
// no spaces) rather than pinned to value sets: they come from our own frontend
|
||||
// (modes, tab/draft kinds, tool names, provider:model) and pinning every value
|
||||
// server-side was not worth the maintenance.
|
||||
const FEATURE_USAGE_KINDS: &[(&str, &str)] = &[
|
||||
("ai_session", "created"),
|
||||
("ai_session", "message"),
|
||||
("ai_session", "autonomy"),
|
||||
("ai_session", "tab"),
|
||||
("ai_session", "tokens"),
|
||||
("ai_session", "deployed"),
|
||||
("ai_session", "archived"),
|
||||
("ai_session", "deleted"),
|
||||
("ai_chat", "message"),
|
||||
("ai_chat", "model"),
|
||||
("ai_chat", "tool"),
|
||||
];
|
||||
|
||||
fn is_identifier_shaped(s: &str, max_len: usize) -> bool {
|
||||
!s.is_empty()
|
||||
&& s.len() <= max_len
|
||||
&& s.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | ':' | '.' | '/'))
|
||||
}
|
||||
|
||||
fn valid_feature_usage_event(e: &FeatureUsageEvent) -> bool {
|
||||
FEATURE_USAGE_KINDS.contains(&(e.feature.as_str(), e.kind.as_str()))
|
||||
&& (e.key.is_empty() || is_identifier_shaped(&e.key, 100))
|
||||
&& (e.entity_id.is_empty() || is_identifier_shaped(&e.entity_id, 50))
|
||||
}
|
||||
|
||||
async fn log_feature_usage(
|
||||
Extension(db): Extension<DB>,
|
||||
Json(payload): Json<LogAiChatPayload>,
|
||||
Json(payload): Json<LogFeatureUsagePayload>,
|
||||
) -> Result<StatusCode> {
|
||||
// Pre-sum duplicate keys: two rows hitting the same conflict target in a
|
||||
// single INSERT error out ("cannot affect row a second time").
|
||||
let mut agg: HashMap<(String, String, String, String), i64> = HashMap::new();
|
||||
for e in payload.events.into_iter().take(MAX_FEATURE_USAGE_EVENTS) {
|
||||
if !valid_feature_usage_event(&e) {
|
||||
continue;
|
||||
}
|
||||
let value = e.value.unwrap_or(1).clamp(1, 1_000_000);
|
||||
*agg.entry((e.feature, e.kind, e.key, e.entity_id))
|
||||
.or_insert(0) += value;
|
||||
}
|
||||
if agg.is_empty() {
|
||||
return Ok(StatusCode::NO_CONTENT);
|
||||
}
|
||||
let mut features = Vec::with_capacity(agg.len());
|
||||
let mut kinds = Vec::with_capacity(agg.len());
|
||||
let mut keys = Vec::with_capacity(agg.len());
|
||||
let mut entity_ids = Vec::with_capacity(agg.len());
|
||||
let mut values = Vec::with_capacity(agg.len());
|
||||
for ((feature, kind, key, entity_id), value) in agg {
|
||||
features.push(feature);
|
||||
kinds.push(kind);
|
||||
keys.push(key);
|
||||
entity_ids.push(entity_id);
|
||||
values.push(value);
|
||||
}
|
||||
sqlx::query!(
|
||||
"INSERT INTO ai_chat_usage (session_id, provider, model, mode) VALUES ($1, $2, $3, $4)
|
||||
ON CONFLICT (session_id) DO UPDATE SET message_count = ai_chat_usage.message_count + 1",
|
||||
&payload.session_id,
|
||||
&payload.provider,
|
||||
&payload.model,
|
||||
&payload.mode
|
||||
"INSERT INTO feature_usage (feature, kind, key, entity_id, value)
|
||||
SELECT * FROM UNNEST($1::text[], $2::text[], $3::text[], $4::text[], $5::bigint[])
|
||||
ON CONFLICT (feature, kind, key, entity_id, day)
|
||||
DO UPDATE SET value = feature_usage.value + EXCLUDED.value, updated_at = now()",
|
||||
&features,
|
||||
&kinds,
|
||||
&keys,
|
||||
&entity_ids,
|
||||
&values
|
||||
)
|
||||
.execute(&db)
|
||||
.await?;
|
||||
|
||||
@@ -6505,10 +6505,10 @@ paths:
|
||||
"400":
|
||||
description: invalid input or request closed
|
||||
|
||||
/w/{workspace}/workspaces/log_chat:
|
||||
/w/{workspace}/workspaces/log_feature_usage:
|
||||
post:
|
||||
summary: log AI chat message
|
||||
operationId: logAiChat
|
||||
summary: log anonymous feature usage telemetry events
|
||||
operationId: logFeatureUsage
|
||||
tags:
|
||||
- workspace
|
||||
parameters:
|
||||
@@ -6520,19 +6520,26 @@ paths:
|
||||
schema:
|
||||
type: object
|
||||
required:
|
||||
- session_id
|
||||
- provider
|
||||
- model
|
||||
- mode
|
||||
- events
|
||||
properties:
|
||||
session_id:
|
||||
type: string
|
||||
provider:
|
||||
type: string
|
||||
model:
|
||||
type: string
|
||||
mode:
|
||||
type: string
|
||||
events:
|
||||
type: array
|
||||
items:
|
||||
type: object
|
||||
required:
|
||||
- feature
|
||||
- kind
|
||||
properties:
|
||||
feature:
|
||||
type: string
|
||||
kind:
|
||||
type: string
|
||||
key:
|
||||
type: string
|
||||
entity_id:
|
||||
type: string
|
||||
value:
|
||||
type: integer
|
||||
responses:
|
||||
"204":
|
||||
description: logged
|
||||
|
||||
@@ -1061,7 +1061,8 @@
|
||||
<li>job usage (language, total duration, count)</li>
|
||||
<li>git sync repo count (sync vs promotion mode)</li>
|
||||
<li
|
||||
>AI chat usage (provider, model, mode, session count, message count — last 30 days)</li
|
||||
>feature usage telemetry: aggregated AI chat and AI session usage counts, including AI
|
||||
provider and model identifiers (last 30 days)</li
|
||||
>
|
||||
<li
|
||||
>resource counts (workspaces, scripts per language, flows, workflows as code, low-code
|
||||
@@ -1107,7 +1108,8 @@
|
||||
<li>user usage (author count, operator count)</li>
|
||||
<li>development instance status</li>
|
||||
<li
|
||||
>AI chat usage (provider, model, mode, session count, message count — last 30 days)</li
|
||||
>feature usage telemetry: aggregated AI chat and AI session usage counts, including AI
|
||||
provider and model identifiers (last 30 days)</li
|
||||
>
|
||||
<li
|
||||
>resource counts (workspaces, scripts per language, flows, workflows as code, low-code
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { ScriptLang } from '$lib/gen/types.gen'
|
||||
import { WorkspaceService, JobService, type CompletedJob } from '$lib/gen'
|
||||
import { JobService, type CompletedJob } from '$lib/gen'
|
||||
import type { FlowOptions, ScriptOptions } from './ContextManager.svelte'
|
||||
import {
|
||||
flowTools,
|
||||
@@ -45,6 +45,7 @@ import { prepareScriptUserMessage } from './script/core'
|
||||
import { prepareNavigatorUserMessage } from './navigator/core'
|
||||
import { sendUserToast } from '$lib/toast'
|
||||
import { workspaceAIClients, getNonStreamingCompletion } from '../lib'
|
||||
import { logFeatureUsage } from '$lib/utils/featureUsage'
|
||||
import { modelSupportsVision } from '../modelConfig'
|
||||
import { getKnownModelContextWindow } from '../modelConfig'
|
||||
import {
|
||||
@@ -2078,6 +2079,13 @@ export class AIChatManager {
|
||||
}
|
||||
}
|
||||
})
|
||||
if (this.isSessionChat && this.sessionId && result.tokenUsage.total > 0) {
|
||||
logFeatureUsage('ai_session', 'tokens', {
|
||||
entityId: this.sessionId,
|
||||
value: result.tokenUsage.total,
|
||||
workspace: this.operatingWorkspace
|
||||
})
|
||||
}
|
||||
return result
|
||||
} catch (err) {
|
||||
console.log('chatRequest error', err)
|
||||
@@ -2435,15 +2443,29 @@ export class AIChatManager {
|
||||
|
||||
const model = tryGetCurrentModel()
|
||||
if (model) {
|
||||
WorkspaceService.logAiChat({
|
||||
workspace: this.operatingWorkspace ?? '',
|
||||
requestBody: {
|
||||
session_id: this.historyManager.getCurrentChatId(),
|
||||
provider: model.provider,
|
||||
model: model.model,
|
||||
mode: this.mode
|
||||
}
|
||||
}).catch(() => {})
|
||||
const chatId = this.historyManager.getCurrentChatId()
|
||||
logFeatureUsage('ai_chat', 'message', {
|
||||
key: this.mode,
|
||||
entityId: chatId,
|
||||
workspace: this.operatingWorkspace
|
||||
})
|
||||
logFeatureUsage('ai_chat', 'model', {
|
||||
key: `${model.provider}:${model.model}`,
|
||||
entityId: chatId,
|
||||
workspace: this.operatingWorkspace
|
||||
})
|
||||
}
|
||||
if (this.isSessionChat && this.sessionId) {
|
||||
logFeatureUsage('ai_session', 'message', {
|
||||
key: this.mode,
|
||||
entityId: this.sessionId,
|
||||
workspace: this.operatingWorkspace
|
||||
})
|
||||
logFeatureUsage('ai_session', 'autonomy', {
|
||||
key: this.autonomyMode,
|
||||
entityId: this.sessionId,
|
||||
workspace: this.operatingWorkspace
|
||||
})
|
||||
}
|
||||
|
||||
if (this.mode === AIMode.FLOW && !this.flowAiChatHelpers) {
|
||||
|
||||
@@ -23,7 +23,6 @@ const mocks = vi.hoisted(() => ({
|
||||
getCurrentModel: vi.fn(),
|
||||
tryGetCurrentModel: vi.fn(),
|
||||
isWebSearchEnabledForProvider: vi.fn(),
|
||||
logAiChat: vi.fn(),
|
||||
sendUserToast: vi.fn(),
|
||||
getOpenaiClient: vi.fn(),
|
||||
getAnthropicClient: vi.fn(),
|
||||
@@ -38,9 +37,10 @@ vi.mock('monaco-editor', () => ({
|
||||
Selection: class Selection {}
|
||||
}))
|
||||
|
||||
vi.mock('$lib/utils/featureUsage', () => ({ logFeatureUsage: vi.fn() }))
|
||||
|
||||
vi.mock('$lib/gen', () => ({
|
||||
WorkspaceService: {
|
||||
logAiChat: mocks.logAiChat,
|
||||
listAiSkills: mocks.listAiSkills
|
||||
},
|
||||
ScriptService: {},
|
||||
@@ -129,7 +129,6 @@ beforeEach(() => {
|
||||
mocks.getCurrentModel.mockReturnValue(undefined)
|
||||
mocks.tryGetCurrentModel.mockReturnValue(undefined)
|
||||
mocks.isWebSearchEnabledForProvider.mockReturnValue(true)
|
||||
mocks.logAiChat.mockResolvedValue(undefined)
|
||||
mocks.getOpenaiClient.mockReturnValue({})
|
||||
mocks.getAnthropicClient.mockReturnValue({})
|
||||
mocks.listAiSkills.mockResolvedValue([])
|
||||
|
||||
@@ -39,6 +39,7 @@ import {
|
||||
} from '$lib/gen'
|
||||
import uFuzzy from '@leeoniya/ufuzzy'
|
||||
import { emptyString } from '$lib/utils'
|
||||
import { logFeatureUsage } from '$lib/utils/featureUsage'
|
||||
import { forLater } from '$lib/forLater'
|
||||
import { scriptLangToEditorLang } from '$lib/scripts'
|
||||
import { getCurrentModel } from '$lib/aiStore'
|
||||
@@ -763,6 +764,11 @@ export async function processToolCall<T>({
|
||||
}
|
||||
|
||||
let result = ''
|
||||
// Key by the resolved tool's declared name, not the model-provided string,
|
||||
// so hallucinated tool names never enter telemetry.
|
||||
if (tool) {
|
||||
logFeatureUsage('ai_chat', 'tool', { key: tool.def.function.name, workspace: workspaceId })
|
||||
}
|
||||
try {
|
||||
result = await callTool({
|
||||
tools,
|
||||
|
||||
@@ -23,6 +23,8 @@ import {
|
||||
type DeployPlanEntry
|
||||
} from './sessionDeployModel'
|
||||
import { maskKey } from './modifiedItemsMask'
|
||||
import { sessionState } from './sessionState.svelte'
|
||||
import { logFeatureUsage } from '$lib/utils/featureUsage'
|
||||
|
||||
export type DeploymentStatus = { status: 'loading' | 'failed'; error?: string }
|
||||
|
||||
@@ -261,6 +263,9 @@ export function useSessionDeployModel(getArgs: () => SessionDeployModelArgs) {
|
||||
async function deployOne(item: DeployItem, discard = false): Promise<boolean> {
|
||||
const plan = discard ? discardPlanFor(item) : deployPlanFor(item)
|
||||
if (!plan) return false
|
||||
// Snapshot before the await: the user may switch sessions while the
|
||||
// deploy runs, and the event belongs to the initiating session.
|
||||
const initiatingSessionId = sessionState.currentSessionId
|
||||
// Don't attempt a deploy we know the user can't make (no write permission
|
||||
// on the path, or blocked by the operator / deployer rule) — the UI
|
||||
// disables it too; this is the guard behind that.
|
||||
@@ -280,6 +285,11 @@ export function useSessionDeployModel(getArgs: () => SessionDeployModelArgs) {
|
||||
.add(item.key)
|
||||
.add(maskKey(item.draftKind, item.displayPath))
|
||||
getArgs().onItemDeployed?.(item)
|
||||
logFeatureUsage('ai_session', 'deployed', {
|
||||
key: item.draftKind,
|
||||
entityId: initiatingSessionId,
|
||||
workspace: getArgs().workspaceId
|
||||
})
|
||||
}
|
||||
}
|
||||
return res.success
|
||||
|
||||
@@ -35,6 +35,9 @@ export type PreviewTabsAdapter = {
|
||||
// Fired synchronously on every tab-set change, so the runtime can drop editor
|
||||
// cells no open tab references anymore (a closed / navigated-away item).
|
||||
onTabsChanged?: () => void
|
||||
// Fired when open() creates a brand-new tab (not focus/retarget of an
|
||||
// existing one), with the tab's initial URL.
|
||||
onTabOpened?: (url: string) => void
|
||||
}
|
||||
|
||||
// True when a tab's URL is the live editor for a specific editable item. Every
|
||||
@@ -268,6 +271,7 @@ export class SessionPreviewTabs {
|
||||
this.#tabs.push(tab)
|
||||
this.#activeId = tab.id
|
||||
this.#flush()
|
||||
this.#adapter.onTabOpened?.(url)
|
||||
return { status: 'opened' }
|
||||
}
|
||||
|
||||
|
||||
@@ -49,7 +49,13 @@ import {
|
||||
previewTargetForSessionTarget,
|
||||
selectPreviewTabsToClose
|
||||
} from './sessionPreviewTabs.svelte'
|
||||
import { matchPreviewPage, parsePreviewItemRoute, previewLocationLabel } from './previewRouter'
|
||||
import {
|
||||
matchPreviewPage,
|
||||
parsePreviewItemRoute,
|
||||
previewLocationLabel,
|
||||
resolvePreviewTab
|
||||
} from './previewRouter'
|
||||
import { logFeatureUsage } from '$lib/utils/featureUsage'
|
||||
import { UserDraft } from '$lib/userDraft.svelte'
|
||||
import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte'
|
||||
import { armRestartOnFirstInteraction } from '$lib/userDraftToast'
|
||||
@@ -432,7 +438,16 @@ function createRuntime(session: Session): SessionRuntime {
|
||||
// Only persist a real width; undefined means "never resized" (defaults to 50).
|
||||
if (snap.previewSize != null) setSessionPreviewSize(session.id, snap.previewSize)
|
||||
},
|
||||
onTabsChanged: pruneEditorCells
|
||||
onTabsChanged: pruneEditorCells,
|
||||
onTabOpened: (url) => {
|
||||
const slot = resolvePreviewTab(url)
|
||||
logFeatureUsage('ai_session', 'tab', {
|
||||
key:
|
||||
slot.kind === 'editor' ? slot.editorKind : slot.kind === 'artifact' ? 'artifact' : 'page',
|
||||
entityId: session.id,
|
||||
workspace: getEffectiveWorkspaceId(session)
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
// Let the jobs tray open a run in this session's preview panel (as an iframe
|
||||
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
protectionRulesState
|
||||
} from '$lib/workspaceProtectionRules.svelte'
|
||||
import { getLocalSetting, storeLocalSetting } from '$lib/utils'
|
||||
import { logFeatureUsage } from '$lib/utils/featureUsage'
|
||||
import { workspaceRootId } from './sessionScope.svelte'
|
||||
import { type DBSchema, type IDBPDatabase } from 'idb'
|
||||
import { userScopedDb } from '$lib/userScopedDb'
|
||||
@@ -795,6 +796,7 @@ export async function commitSessionWorkspace(
|
||||
// The draft prompt has been consumed as the first message.
|
||||
delete s.draftPrompt
|
||||
await putSession(s)
|
||||
logFeatureUsage('ai_session', 'created', { key: 'fork', entityId: s.id, workspace: newId })
|
||||
// The global workspaceStore is intentionally left untouched: the session
|
||||
// chat targets its own workspace via AIChatManager.operatingWorkspace, so
|
||||
// committing must not yank the user's active (navigation-mode) workspace.
|
||||
@@ -809,6 +811,12 @@ export async function commitSessionWorkspace(
|
||||
// The draft prompt has been consumed as the first message.
|
||||
delete s.draftPrompt
|
||||
await putSession(s)
|
||||
// A picked workspace can itself be an existing fork — classify by root.
|
||||
logFeatureUsage('ai_session', 'created', {
|
||||
key: ws === s.workspace_root_id ? 'root' : 'fork',
|
||||
entityId: s.id,
|
||||
workspace: ws
|
||||
})
|
||||
// The global workspaceStore is intentionally left untouched (see the fork
|
||||
// branch above): the session chat reads its committed workspace through the
|
||||
// manager's workspace resolver, not the active workspaceStore.
|
||||
@@ -958,8 +966,10 @@ export function setSessionArchived(id: string, archived: boolean) {
|
||||
if (!s) return
|
||||
const next = archived ? true : undefined
|
||||
if (s.archived === next && (archived || !s.archivedByWorkspace)) return
|
||||
if (archived) s.archived = true
|
||||
else {
|
||||
if (archived) {
|
||||
s.archived = true
|
||||
logFeatureUsage('ai_session', 'archived', { entityId: s.id, workspace: s.workspace_id })
|
||||
} else {
|
||||
delete s.archived
|
||||
delete s.archivedByWorkspace
|
||||
}
|
||||
@@ -982,6 +992,7 @@ export function deleteSession(id: string) {
|
||||
// GC any linked files and artifacts persisted for this session.
|
||||
void deleteItemsForSession(id)
|
||||
void deleteArtifactsForSession(id)
|
||||
logFeatureUsage('ai_session', 'deleted', { entityId: id, workspace: s.workspace_id })
|
||||
}
|
||||
|
||||
export function setSessionChatId(sessionId: string, chatId: string) {
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
vi.mock('$lib/gen', () => ({ OpenAPI: { BASE: '/api' } }))
|
||||
vi.mock('$lib/stores', () => ({ workspaceStore: { subscribe: () => () => {} } }))
|
||||
|
||||
import { createFeatureUsageBuffer, type FeatureUsageEventPayload } from './featureUsage'
|
||||
|
||||
describe('createFeatureUsageBuffer', () => {
|
||||
it('sums repeated events per (feature, kind, key, entity) and flushes one batch', async () => {
|
||||
const send = vi.fn().mockResolvedValue(undefined)
|
||||
const buffer = createFeatureUsageBuffer(send, () => 'ws1')
|
||||
|
||||
buffer.log('ai_session', 'message', { key: 'global', entityId: 's1' })
|
||||
buffer.log('ai_session', 'message', { key: 'global', entityId: 's1' })
|
||||
buffer.log('ai_session', 'tokens', { entityId: 's1', value: 120 })
|
||||
buffer.log('ai_session', 'message', { key: 'global', entityId: 's2' })
|
||||
await buffer.flush()
|
||||
|
||||
expect(send).toHaveBeenCalledTimes(1)
|
||||
const [workspace, events] = send.mock.calls[0]
|
||||
expect(workspace).toBe('ws1')
|
||||
expect(events).toEqual(
|
||||
expect.arrayContaining([
|
||||
{ feature: 'ai_session', kind: 'message', key: 'global', entity_id: 's1', value: 2 },
|
||||
{ feature: 'ai_session', kind: 'tokens', key: '', entity_id: 's1', value: 120 },
|
||||
{ feature: 'ai_session', kind: 'message', key: 'global', entity_id: 's2', value: 1 }
|
||||
])
|
||||
)
|
||||
expect(events).toHaveLength(3)
|
||||
|
||||
// Flushed events must not be re-sent.
|
||||
await buffer.flush()
|
||||
expect(send).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('splits batches per workspace and drops events without any workspace', async () => {
|
||||
const send = vi.fn().mockResolvedValue(undefined)
|
||||
const buffer = createFeatureUsageBuffer(send, () => undefined)
|
||||
|
||||
buffer.log('ai_session', 'created', { key: 'fork' }) // no workspace -> dropped
|
||||
buffer.log('ai_session', 'created', { key: 'fork', workspace: 'ws1' })
|
||||
buffer.log('ai_session', 'created', { key: 'root', workspace: 'ws2' })
|
||||
await buffer.flush()
|
||||
|
||||
expect(send).toHaveBeenCalledTimes(2)
|
||||
const workspaces = send.mock.calls.map((c) => c[0]).sort()
|
||||
expect(workspaces).toEqual(['ws1', 'ws2'])
|
||||
})
|
||||
|
||||
it('starts every chunk request before any send resolves (pagehide flush)', async () => {
|
||||
const send = vi.fn().mockReturnValue(new Promise<void>(() => {}))
|
||||
const buffer = createFeatureUsageBuffer(send, () => 'ws1')
|
||||
|
||||
for (let i = 0; i < 60; i++) {
|
||||
buffer.log('ai_session', 'tool', { key: `tool_${i}` })
|
||||
}
|
||||
buffer.log('ai_session', 'message', { workspace: 'ws2' })
|
||||
void buffer.flush()
|
||||
await Promise.resolve()
|
||||
|
||||
// keepalive only protects requests that were issued; a sequential flush
|
||||
// would have started just the first chunk here.
|
||||
expect(send).toHaveBeenCalledTimes(3)
|
||||
})
|
||||
|
||||
it('chunks flushes above the per-request cap and survives send failures', async () => {
|
||||
const send = vi.fn().mockRejectedValueOnce(new Error('network')).mockResolvedValue(undefined)
|
||||
const buffer = createFeatureUsageBuffer(send, () => 'ws1')
|
||||
|
||||
for (let i = 0; i < 60; i++) {
|
||||
buffer.log('ai_session', 'tool', { key: `tool_${i}` })
|
||||
}
|
||||
await expect(buffer.flush()).resolves.toBeUndefined()
|
||||
|
||||
expect(send).toHaveBeenCalledTimes(2)
|
||||
const sent = send.mock.calls.flatMap((c) => c[1] as FeatureUsageEventPayload[])
|
||||
expect(send.mock.calls[0][1]).toHaveLength(50)
|
||||
expect(sent).toHaveLength(60)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,137 @@
|
||||
import { get } from 'svelte/store'
|
||||
import { OpenAPI } from '$lib/gen'
|
||||
import { workspaceStore } from '$lib/stores'
|
||||
|
||||
// Anonymous product-usage counters (e.g. AI session activity), batched into the
|
||||
// backend `feature_usage` accumulator. Only aggregated counts ever leave the
|
||||
// instance, and only when telemetry is enabled and not in minimal mode — never
|
||||
// log paths, prompts, code, or user identifiers here (entity ids must be
|
||||
// opaque random ids).
|
||||
|
||||
export interface FeatureUsageOpts {
|
||||
key?: string
|
||||
entityId?: string
|
||||
value?: number
|
||||
/** Workspace whose API route carries the batch; defaults to the active workspace. */
|
||||
workspace?: string
|
||||
}
|
||||
|
||||
type SendFn = (workspace: string, events: FeatureUsageEventPayload[]) => Promise<void>
|
||||
|
||||
export interface FeatureUsageEventPayload {
|
||||
feature: string
|
||||
kind: string
|
||||
key?: string
|
||||
entity_id?: string
|
||||
value?: number
|
||||
}
|
||||
|
||||
const FLUSH_INTERVAL_MS = 30_000
|
||||
// Backend caps a batch at 50 events; chunk larger flushes.
|
||||
const MAX_EVENTS_PER_REQUEST = 50
|
||||
|
||||
export function createFeatureUsageBuffer(
|
||||
send: SendFn,
|
||||
getDefaultWorkspace: () => string | undefined,
|
||||
flushIntervalMs = FLUSH_INTERVAL_MS
|
||||
) {
|
||||
// One accumulator per (workspace, feature, kind, key, entityId): repeated
|
||||
// events sum locally so a chatty UI still produces one upsert per flush.
|
||||
const pending = new Map<string, { workspace: string; event: FeatureUsageEventPayload }>()
|
||||
let timer: ReturnType<typeof setTimeout> | undefined
|
||||
|
||||
function log(feature: string, kind: string, opts: FeatureUsageOpts = {}): void {
|
||||
const workspace = opts.workspace ?? getDefaultWorkspace()
|
||||
if (!workspace) return
|
||||
const key = opts.key ?? ''
|
||||
const entityId = opts.entityId ?? ''
|
||||
const value = Math.max(1, Math.round(opts.value ?? 1))
|
||||
const mapKey = `${workspace}\u0000${feature}\u0000${kind}\u0000${key}\u0000${entityId}`
|
||||
const existing = pending.get(mapKey)
|
||||
if (existing) {
|
||||
existing.event.value = (existing.event.value ?? 1) + value
|
||||
} else {
|
||||
pending.set(mapKey, {
|
||||
workspace,
|
||||
event: { feature, kind, key, entity_id: entityId, value }
|
||||
})
|
||||
}
|
||||
if (timer === undefined) {
|
||||
timer = setTimeout(() => {
|
||||
timer = undefined
|
||||
void flush()
|
||||
}, flushIntervalMs)
|
||||
}
|
||||
}
|
||||
|
||||
async function flush(): Promise<void> {
|
||||
if (timer !== undefined) {
|
||||
clearTimeout(timer)
|
||||
timer = undefined
|
||||
}
|
||||
if (pending.size === 0) return
|
||||
const byWorkspace = new Map<string, FeatureUsageEventPayload[]>()
|
||||
for (const { workspace, event } of pending.values()) {
|
||||
let events = byWorkspace.get(workspace)
|
||||
if (!events) {
|
||||
events = []
|
||||
byWorkspace.set(workspace, events)
|
||||
}
|
||||
events.push(event)
|
||||
}
|
||||
pending.clear()
|
||||
// Start every chunk request synchronously before awaiting: the pagehide
|
||||
// flush only protects requests that were already issued (keepalive can't
|
||||
// help a fetch that never started).
|
||||
const inflight: Promise<void>[] = []
|
||||
for (const [workspace, events] of byWorkspace) {
|
||||
for (let i = 0; i < events.length; i += MAX_EVENTS_PER_REQUEST) {
|
||||
inflight.push(
|
||||
send(workspace, events.slice(i, i + MAX_EVENTS_PER_REQUEST)).catch(() => {
|
||||
// Telemetry is best-effort: drop the batch rather than retry.
|
||||
})
|
||||
)
|
||||
}
|
||||
}
|
||||
await Promise.all(inflight)
|
||||
}
|
||||
|
||||
return { log, flush }
|
||||
}
|
||||
|
||||
const buffer = createFeatureUsageBuffer(
|
||||
async (workspace, events) => {
|
||||
// Raw fetch instead of the generated client: `keepalive` lets the request
|
||||
// finish after tab close/navigation, which is when the final flush runs.
|
||||
// Auth rides on the token cookie (WITH_CREDENTIALS app setup).
|
||||
await fetch(`${OpenAPI.BASE}/w/${encodeURIComponent(workspace)}/workspaces/log_feature_usage`, {
|
||||
method: 'POST',
|
||||
credentials: 'include',
|
||||
keepalive: true,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ events })
|
||||
})
|
||||
},
|
||||
() => get(workspaceStore) ?? undefined
|
||||
)
|
||||
|
||||
if (typeof document !== 'undefined') {
|
||||
// Flush what's buffered before the tab goes away. pagehide covers
|
||||
// close/navigation paths where visibilitychange is not delivered.
|
||||
document.addEventListener('visibilitychange', () => {
|
||||
if (document.visibilityState === 'hidden') {
|
||||
void buffer.flush()
|
||||
}
|
||||
})
|
||||
window.addEventListener('pagehide', () => {
|
||||
void buffer.flush()
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Record an anonymous feature-usage event. Fire-and-forget: events are summed
|
||||
* locally per (feature, kind, key, entityId) and flushed in batches.
|
||||
*/
|
||||
export function logFeatureUsage(feature: string, kind: string, opts: FeatureUsageOpts = {}): void {
|
||||
buffer.log(feature, kind, opts)
|
||||
}
|
||||
Reference in New Issue
Block a user