feat(telemetry): generic feature-usage telemetry with AI session metrics (#10200)

* feat(telemetry): add generic feature_usage table and batched logging endpoint Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * feat(telemetry): log AI session usage events and document them in telemetry settings Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(telemetry): use escape sequence instead of literal NUL bytes in buffer key Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(telemetry): validate dimensions, decouple retention, keepalive flush Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(telemetry): allowlist feature-usage dimensions and index retention scans Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(telemetry): pin tool-name allowlist and deploy session attribution Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * refactor(telemetry): route AI chat usage through feature_usage and drop ai_chat_usage Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * refactor(telemetry): slim dimension validation to registered kinds plus key shape Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(telemetry): backfill ai_chat_usage into feature_usage before dropping it Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(telemetry): disclose provider and model identifiers in telemetry settings text Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(telemetry): issue all flush chunks before awaiting so pagehide keeps them Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * chore: update ee-repo-ref to 6306c072a50937ea9af44a5bcf42345543207486 This commit updates the EE repository reference after PR #672 was merged in windmill-ee-private. Previous ee-repo-ref: 964f242a0eb44db7f7d26636cc8d76aeabea2b73 New ee-repo-ref: 6306c072a50937ea9af44a5bcf42345543207486 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com> Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
2026-09-09 00:04:10 +00:00 · 2026-07-20 20:56:57 +02:00
parent f635bd5ae7
commit 11fda89b52
24 changed files with 566 additions and 122 deletions
@@ -1061,7 +1061,8 @@
 						<li>job usage (language, total duration, count)</li>
 						<li>git sync repo count (sync vs promotion mode)</li>
 						<li
-							>AI chat usage (provider, model, mode, session count, message count — last 30 days)</li
+							>feature usage telemetry: aggregated AI chat and AI session usage counts, including AI
+							provider and model identifiers (last 30 days)</li
 						>
 						<li
 							>resource counts (workspaces, scripts per language, flows, workflows as code, low-code
@@ -1107,7 +1108,8 @@
 						<li>user usage (author count, operator count)</li>
 						<li>development instance status</li>
 						<li
-							>AI chat usage (provider, model, mode, session count, message count — last 30 days)</li
+							>feature usage telemetry: aggregated AI chat and AI session usage counts, including AI
+							provider and model identifiers (last 30 days)</li
 						>
 						<li
 							>resource counts (workspaces, scripts per language, flows, workflows as code, low-code
@@ -1,5 +1,5 @@
 import type { ScriptLang } from '$lib/gen/types.gen'
-import { WorkspaceService, JobService, type CompletedJob } from '$lib/gen'
+import { JobService, type CompletedJob } from '$lib/gen'
 import type { FlowOptions, ScriptOptions } from './ContextManager.svelte'
 import {
 	flowTools,
@@ -45,6 +45,7 @@ import { prepareScriptUserMessage } from './script/core'
 import { prepareNavigatorUserMessage } from './navigator/core'
 import { sendUserToast } from '$lib/toast'
 import { workspaceAIClients, getNonStreamingCompletion } from '../lib'
+import { logFeatureUsage } from '$lib/utils/featureUsage'
 import { modelSupportsVision } from '../modelConfig'
 import { getKnownModelContextWindow } from '../modelConfig'
 import {
@@ -2078,6 +2079,13 @@ export class AIChatManager {
 					}
 				}
 			})
+			if (this.isSessionChat && this.sessionId && result.tokenUsage.total > 0) {
+				logFeatureUsage('ai_session', 'tokens', {
+					entityId: this.sessionId,
+					value: result.tokenUsage.total,
+					workspace: this.operatingWorkspace
+				})
+			}
 			return result
 		} catch (err) {
 			console.log('chatRequest error', err)
@@ -2435,15 +2443,29 @@ export class AIChatManager {

 			const model = tryGetCurrentModel()
 			if (model) {
-				WorkspaceService.logAiChat({
-					workspace: this.operatingWorkspace ?? '',
-					requestBody: {
-						session_id: this.historyManager.getCurrentChatId(),
-						provider: model.provider,
-						model: model.model,
-						mode: this.mode
-					}
-				}).catch(() => {})
+				const chatId = this.historyManager.getCurrentChatId()
+				logFeatureUsage('ai_chat', 'message', {
+					key: this.mode,
+					entityId: chatId,
+					workspace: this.operatingWorkspace
+				})
+				logFeatureUsage('ai_chat', 'model', {
+					key: `${model.provider}:${model.model}`,
+					entityId: chatId,
+					workspace: this.operatingWorkspace
+				})
+			}
+			if (this.isSessionChat && this.sessionId) {
+				logFeatureUsage('ai_session', 'message', {
+					key: this.mode,
+					entityId: this.sessionId,
+					workspace: this.operatingWorkspace
+				})
+				logFeatureUsage('ai_session', 'autonomy', {
+					key: this.autonomyMode,
+					entityId: this.sessionId,
+					workspace: this.operatingWorkspace
+				})
 			}

 			if (this.mode === AIMode.FLOW && !this.flowAiChatHelpers) {
@@ -23,7 +23,6 @@ const mocks = vi.hoisted(() => ({
 	getCurrentModel: vi.fn(),
 	tryGetCurrentModel: vi.fn(),
 	isWebSearchEnabledForProvider: vi.fn(),
-	logAiChat: vi.fn(),
 	sendUserToast: vi.fn(),
 	getOpenaiClient: vi.fn(),
 	getAnthropicClient: vi.fn(),
@@ -38,9 +37,10 @@ vi.mock('monaco-editor', () => ({
 	Selection: class Selection {}
 }))

+vi.mock('$lib/utils/featureUsage', () => ({ logFeatureUsage: vi.fn() }))
+
 vi.mock('$lib/gen', () => ({
 	WorkspaceService: {
-		logAiChat: mocks.logAiChat,
 		listAiSkills: mocks.listAiSkills
 	},
 	ScriptService: {},
@@ -129,7 +129,6 @@ beforeEach(() => {
 	mocks.getCurrentModel.mockReturnValue(undefined)
 	mocks.tryGetCurrentModel.mockReturnValue(undefined)
 	mocks.isWebSearchEnabledForProvider.mockReturnValue(true)
-	mocks.logAiChat.mockResolvedValue(undefined)
 	mocks.getOpenaiClient.mockReturnValue({})
 	mocks.getAnthropicClient.mockReturnValue({})
 	mocks.listAiSkills.mockResolvedValue([])
@@ -39,6 +39,7 @@ import {
 } from '$lib/gen'
 import uFuzzy from '@leeoniya/ufuzzy'
 import { emptyString } from '$lib/utils'
+import { logFeatureUsage } from '$lib/utils/featureUsage'
 import { forLater } from '$lib/forLater'
 import { scriptLangToEditorLang } from '$lib/scripts'
 import { getCurrentModel } from '$lib/aiStore'
@@ -763,6 +764,11 @@ export async function processToolCall<T>({
 		}

 		let result = ''
+		// Key by the resolved tool's declared name, not the model-provided string,
+		// so hallucinated tool names never enter telemetry.
+		if (tool) {
+			logFeatureUsage('ai_chat', 'tool', { key: tool.def.function.name, workspace: workspaceId })
+		}
 		try {
 			result = await callTool({
 				tools,
@@ -23,6 +23,8 @@ import {
 	type DeployPlanEntry
 } from './sessionDeployModel'
 import { maskKey } from './modifiedItemsMask'
+import { sessionState } from './sessionState.svelte'
+import { logFeatureUsage } from '$lib/utils/featureUsage'

 export type DeploymentStatus = { status: 'loading' | 'failed'; error?: string }

@@ -261,6 +263,9 @@ export function useSessionDeployModel(getArgs: () => SessionDeployModelArgs) {
 	async function deployOne(item: DeployItem, discard = false): Promise<boolean> {
 		const plan = discard ? discardPlanFor(item) : deployPlanFor(item)
 		if (!plan) return false
+		// Snapshot before the await: the user may switch sessions while the
+		// deploy runs, and the event belongs to the initiating session.
+		const initiatingSessionId = sessionState.currentSessionId
 		// Don't attempt a deploy we know the user can't make (no write permission
 		// on the path, or blocked by the operator / deployer rule) — the UI
 		// disables it too; this is the guard behind that.
@@ -280,6 +285,11 @@ export function useSessionDeployModel(getArgs: () => SessionDeployModelArgs) {
 						.add(item.key)
 						.add(maskKey(item.draftKind, item.displayPath))
 					getArgs().onItemDeployed?.(item)
+					logFeatureUsage('ai_session', 'deployed', {
+						key: item.draftKind,
+						entityId: initiatingSessionId,
+						workspace: getArgs().workspaceId
+					})
 				}
 			}
 			return res.success
@@ -35,6 +35,9 @@ export type PreviewTabsAdapter = {
 	// Fired synchronously on every tab-set change, so the runtime can drop editor
 	// cells no open tab references anymore (a closed / navigated-away item).
 	onTabsChanged?: () => void
+	// Fired when open() creates a brand-new tab (not focus/retarget of an
+	// existing one), with the tab's initial URL.
+	onTabOpened?: (url: string) => void
 }

 // True when a tab's URL is the live editor for a specific editable item. Every
@@ -268,6 +271,7 @@ export class SessionPreviewTabs {
 		this.#tabs.push(tab)
 		this.#activeId = tab.id
 		this.#flush()
+		this.#adapter.onTabOpened?.(url)
 		return { status: 'opened' }
 	}

@@ -49,7 +49,13 @@ import {
 	previewTargetForSessionTarget,
 	selectPreviewTabsToClose
 } from './sessionPreviewTabs.svelte'
-import { matchPreviewPage, parsePreviewItemRoute, previewLocationLabel } from './previewRouter'
+import {
+	matchPreviewPage,
+	parsePreviewItemRoute,
+	previewLocationLabel,
+	resolvePreviewTab
+} from './previewRouter'
+import { logFeatureUsage } from '$lib/utils/featureUsage'
 import { UserDraft } from '$lib/userDraft.svelte'
 import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte'
 import { armRestartOnFirstInteraction } from '$lib/userDraftToast'
@@ -432,7 +438,16 @@ function createRuntime(session: Session): SessionRuntime {
 			// Only persist a real width; undefined means "never resized" (defaults to 50).
 			if (snap.previewSize != null) setSessionPreviewSize(session.id, snap.previewSize)
 		},
-		onTabsChanged: pruneEditorCells
+		onTabsChanged: pruneEditorCells,
+		onTabOpened: (url) => {
+			const slot = resolvePreviewTab(url)
+			logFeatureUsage('ai_session', 'tab', {
+				key:
+					slot.kind === 'editor' ? slot.editorKind : slot.kind === 'artifact' ? 'artifact' : 'page',
+				entityId: session.id,
+				workspace: getEffectiveWorkspaceId(session)
+			})
+		}
 	})

 	// Let the jobs tray open a run in this session's preview panel (as an iframe
@@ -18,6 +18,7 @@ import {
 	protectionRulesState
 } from '$lib/workspaceProtectionRules.svelte'
 import { getLocalSetting, storeLocalSetting } from '$lib/utils'
+import { logFeatureUsage } from '$lib/utils/featureUsage'
 import { workspaceRootId } from './sessionScope.svelte'
 import { type DBSchema, type IDBPDatabase } from 'idb'
 import { userScopedDb } from '$lib/userScopedDb'
@@ -795,6 +796,7 @@ export async function commitSessionWorkspace(
 		// The draft prompt has been consumed as the first message.
 		delete s.draftPrompt
 		await putSession(s)
+		logFeatureUsage('ai_session', 'created', { key: 'fork', entityId: s.id, workspace: newId })
 		// The global workspaceStore is intentionally left untouched: the session
 		// chat targets its own workspace via AIChatManager.operatingWorkspace, so
 		// committing must not yank the user's active (navigation-mode) workspace.
@@ -809,6 +811,12 @@ export async function commitSessionWorkspace(
 	// The draft prompt has been consumed as the first message.
 	delete s.draftPrompt
 	await putSession(s)
+	// A picked workspace can itself be an existing fork — classify by root.
+	logFeatureUsage('ai_session', 'created', {
+		key: ws === s.workspace_root_id ? 'root' : 'fork',
+		entityId: s.id,
+		workspace: ws
+	})
 	// The global workspaceStore is intentionally left untouched (see the fork
 	// branch above): the session chat reads its committed workspace through the
 	// manager's workspace resolver, not the active workspaceStore.
@@ -958,8 +966,10 @@ export function setSessionArchived(id: string, archived: boolean) {
 	if (!s) return
 	const next = archived ? true : undefined
 	if (s.archived === next && (archived || !s.archivedByWorkspace)) return
-	if (archived) s.archived = true
-	else {
+	if (archived) {
+		s.archived = true
+		logFeatureUsage('ai_session', 'archived', { entityId: s.id, workspace: s.workspace_id })
+	} else {
 		delete s.archived
 		delete s.archivedByWorkspace
 	}
@@ -982,6 +992,7 @@ export function deleteSession(id: string) {
 	// GC any linked files and artifacts persisted for this session.
 	void deleteItemsForSession(id)
 	void deleteArtifactsForSession(id)
+	logFeatureUsage('ai_session', 'deleted', { entityId: id, workspace: s.workspace_id })
 }

 export function setSessionChatId(sessionId: string, chatId: string) {
@@ -0,0 +1,80 @@
+import { describe, expect, it, vi } from 'vitest'
+
+vi.mock('$lib/gen', () => ({ OpenAPI: { BASE: '/api' } }))
+vi.mock('$lib/stores', () => ({ workspaceStore: { subscribe: () => () => {} } }))
+
+import { createFeatureUsageBuffer, type FeatureUsageEventPayload } from './featureUsage'
+
+describe('createFeatureUsageBuffer', () => {
+	it('sums repeated events per (feature, kind, key, entity) and flushes one batch', async () => {
+		const send = vi.fn().mockResolvedValue(undefined)
+		const buffer = createFeatureUsageBuffer(send, () => 'ws1')
+
+		buffer.log('ai_session', 'message', { key: 'global', entityId: 's1' })
+		buffer.log('ai_session', 'message', { key: 'global', entityId: 's1' })
+		buffer.log('ai_session', 'tokens', { entityId: 's1', value: 120 })
+		buffer.log('ai_session', 'message', { key: 'global', entityId: 's2' })
+		await buffer.flush()
+
+		expect(send).toHaveBeenCalledTimes(1)
+		const [workspace, events] = send.mock.calls[0]
+		expect(workspace).toBe('ws1')
+		expect(events).toEqual(
+			expect.arrayContaining([
+				{ feature: 'ai_session', kind: 'message', key: 'global', entity_id: 's1', value: 2 },
+				{ feature: 'ai_session', kind: 'tokens', key: '', entity_id: 's1', value: 120 },
+				{ feature: 'ai_session', kind: 'message', key: 'global', entity_id: 's2', value: 1 }
+			])
+		)
+		expect(events).toHaveLength(3)
+
+		// Flushed events must not be re-sent.
+		await buffer.flush()
+		expect(send).toHaveBeenCalledTimes(1)
+	})
+
+	it('splits batches per workspace and drops events without any workspace', async () => {
+		const send = vi.fn().mockResolvedValue(undefined)
+		const buffer = createFeatureUsageBuffer(send, () => undefined)
+
+		buffer.log('ai_session', 'created', { key: 'fork' }) // no workspace -> dropped
+		buffer.log('ai_session', 'created', { key: 'fork', workspace: 'ws1' })
+		buffer.log('ai_session', 'created', { key: 'root', workspace: 'ws2' })
+		await buffer.flush()
+
+		expect(send).toHaveBeenCalledTimes(2)
+		const workspaces = send.mock.calls.map((c) => c[0]).sort()
+		expect(workspaces).toEqual(['ws1', 'ws2'])
+	})
+
+	it('starts every chunk request before any send resolves (pagehide flush)', async () => {
+		const send = vi.fn().mockReturnValue(new Promise<void>(() => {}))
+		const buffer = createFeatureUsageBuffer(send, () => 'ws1')
+
+		for (let i = 0; i < 60; i++) {
+			buffer.log('ai_session', 'tool', { key: `tool_${i}` })
+		}
+		buffer.log('ai_session', 'message', { workspace: 'ws2' })
+		void buffer.flush()
+		await Promise.resolve()
+
+		// keepalive only protects requests that were issued; a sequential flush
+		// would have started just the first chunk here.
+		expect(send).toHaveBeenCalledTimes(3)
+	})
+
+	it('chunks flushes above the per-request cap and survives send failures', async () => {
+		const send = vi.fn().mockRejectedValueOnce(new Error('network')).mockResolvedValue(undefined)
+		const buffer = createFeatureUsageBuffer(send, () => 'ws1')
+
+		for (let i = 0; i < 60; i++) {
+			buffer.log('ai_session', 'tool', { key: `tool_${i}` })
+		}
+		await expect(buffer.flush()).resolves.toBeUndefined()
+
+		expect(send).toHaveBeenCalledTimes(2)
+		const sent = send.mock.calls.flatMap((c) => c[1] as FeatureUsageEventPayload[])
+		expect(send.mock.calls[0][1]).toHaveLength(50)
+		expect(sent).toHaveLength(60)
+	})
+})
@@ -0,0 +1,137 @@
+import { get } from 'svelte/store'
+import { OpenAPI } from '$lib/gen'
+import { workspaceStore } from '$lib/stores'
+
+// Anonymous product-usage counters (e.g. AI session activity), batched into the
+// backend `feature_usage` accumulator. Only aggregated counts ever leave the
+// instance, and only when telemetry is enabled and not in minimal mode — never
+// log paths, prompts, code, or user identifiers here (entity ids must be
+// opaque random ids).
+
+export interface FeatureUsageOpts {
+	key?: string
+	entityId?: string
+	value?: number
+	/** Workspace whose API route carries the batch; defaults to the active workspace. */
+	workspace?: string
+}
+
+type SendFn = (workspace: string, events: FeatureUsageEventPayload[]) => Promise<void>
+
+export interface FeatureUsageEventPayload {
+	feature: string
+	kind: string
+	key?: string
+	entity_id?: string
+	value?: number
+}
+
+const FLUSH_INTERVAL_MS = 30_000
+// Backend caps a batch at 50 events; chunk larger flushes.
+const MAX_EVENTS_PER_REQUEST = 50
+
+export function createFeatureUsageBuffer(
+	send: SendFn,
+	getDefaultWorkspace: () => string | undefined,
+	flushIntervalMs = FLUSH_INTERVAL_MS
+) {
+	// One accumulator per (workspace, feature, kind, key, entityId): repeated
+	// events sum locally so a chatty UI still produces one upsert per flush.
+	const pending = new Map<string, { workspace: string; event: FeatureUsageEventPayload }>()
+	let timer: ReturnType<typeof setTimeout> | undefined
+
+	function log(feature: string, kind: string, opts: FeatureUsageOpts = {}): void {
+		const workspace = opts.workspace ?? getDefaultWorkspace()
+		if (!workspace) return
+		const key = opts.key ?? ''
+		const entityId = opts.entityId ?? ''
+		const value = Math.max(1, Math.round(opts.value ?? 1))
+		const mapKey = `${workspace}\u0000${feature}\u0000${kind}\u0000${key}\u0000${entityId}`
+		const existing = pending.get(mapKey)
+		if (existing) {
+			existing.event.value = (existing.event.value ?? 1) + value
+		} else {
+			pending.set(mapKey, {
+				workspace,
+				event: { feature, kind, key, entity_id: entityId, value }
+			})
+		}
+		if (timer === undefined) {
+			timer = setTimeout(() => {
+				timer = undefined
+				void flush()
+			}, flushIntervalMs)
+		}
+	}
+
+	async function flush(): Promise<void> {
+		if (timer !== undefined) {
+			clearTimeout(timer)
+			timer = undefined
+		}
+		if (pending.size === 0) return
+		const byWorkspace = new Map<string, FeatureUsageEventPayload[]>()
+		for (const { workspace, event } of pending.values()) {
+			let events = byWorkspace.get(workspace)
+			if (!events) {
+				events = []
+				byWorkspace.set(workspace, events)
+			}
+			events.push(event)
+		}
+		pending.clear()
+		// Start every chunk request synchronously before awaiting: the pagehide
+		// flush only protects requests that were already issued (keepalive can't
+		// help a fetch that never started).
+		const inflight: Promise<void>[] = []
+		for (const [workspace, events] of byWorkspace) {
+			for (let i = 0; i < events.length; i += MAX_EVENTS_PER_REQUEST) {
+				inflight.push(
+					send(workspace, events.slice(i, i + MAX_EVENTS_PER_REQUEST)).catch(() => {
+						// Telemetry is best-effort: drop the batch rather than retry.
+					})
+				)
+			}
+		}
+		await Promise.all(inflight)
+	}
+
+	return { log, flush }
+}
+
+const buffer = createFeatureUsageBuffer(
+	async (workspace, events) => {
+		// Raw fetch instead of the generated client: `keepalive` lets the request
+		// finish after tab close/navigation, which is when the final flush runs.
+		// Auth rides on the token cookie (WITH_CREDENTIALS app setup).
+		await fetch(`${OpenAPI.BASE}/w/${encodeURIComponent(workspace)}/workspaces/log_feature_usage`, {
+			method: 'POST',
+			credentials: 'include',
+			keepalive: true,
+			headers: { 'Content-Type': 'application/json' },
+			body: JSON.stringify({ events })
+		})
+	},
+	() => get(workspaceStore) ?? undefined
+)
+
+if (typeof document !== 'undefined') {
+	// Flush what's buffered before the tab goes away. pagehide covers
+	// close/navigation paths where visibilitychange is not delivered.
+	document.addEventListener('visibilitychange', () => {
+		if (document.visibilityState === 'hidden') {
+			void buffer.flush()
+		}
+	})
+	window.addEventListener('pagehide', () => {
+		void buffer.flush()
+	})
+}
+
+/**
+ * Record an anonymous feature-usage event. Fire-and-forget: events are summed
+ * locally per (feature, kind, key, entityId) and flushed in batches.
+ */
+export function logFeatureUsage(feature: string, kind: string, opts: FeatureUsageOpts = {}): void {
+	buffer.log(feature, kind, opts)
+}