feat(frontend): record the outcome of every AI chat tool call (#10746)

* feat(frontend): record the outcome of every AI chat tool call The `ai_chat`/`tool` counter fired before execution, so nothing recorded whether a tool call succeeded, and the three paths that refuse a call before it runs recorded nothing at all. Log once per call on whichever path ends it, keyed `<tool_name>:<status>` over ok, error, declined, rejected and blocked_plan_mode. Per-tool totals now need `split_part(key, ':', 1)` downstream; rows keyed by the bare tool name coexist for up to 60 days. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> * docs(frontend): state what the tool-call telemetry statuses do not cover `ok` means the tool function resolved, which includes tools that report failure by returning an error string, and a call abandoned mid-execution logs nothing. Also pin that a hallucinated tool name reaches telemetry nowhere. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-12 00:06:14 +00:00 · 2026-08-18 12:26:09 +02:00
parent 6749015fbf
commit 7b17e358b3
2 changed files with 93 additions and 7 deletions
@@ -7,6 +7,11 @@ vi.mock('monaco-editor', () => ({
 	editor: {}
 }))

+vi.mock('$lib/utils/featureUsage', () => ({
+	logFeatureUsage: vi.fn(),
+	logHubScriptPick: vi.fn()
+}))
+
 const userHolder = vi.hoisted(() => ({
 	current: { is_super_admin: true } as { is_super_admin: boolean }
 }))
@@ -902,6 +907,63 @@ describe('processToolCall', () => {
 			})
 		)
 	})
+
+	// The counter is silently dropped by the backend when the key is malformed, so nothing
+	// here fails loudly if a path stops logging or logs the wrong status.
+	it('logs one feature-usage outcome per tool call, keyed <tool>:<status>', async () => {
+		const { createToolDef, processToolCall } = await import('./shared')
+		const { logFeatureUsage } = await import('$lib/utils/featureUsage')
+
+		const outcomeKeys = async (
+			tool: Partial<import('./shared').Tool<any>> = {},
+			toolCallbacks: Partial<import('./shared').ToolCallbacks> = {}
+		) => {
+			vi.mocked(logFeatureUsage).mockClear()
+			await runToolCall(
+				{
+					def: createToolDef(z.object({}), 'run_script', 'Run script'),
+					fn: vi.fn().mockResolvedValue('done'),
+					...tool
+				},
+				toolCallbacks
+			)
+			return vi
+				.mocked(logFeatureUsage)
+				.mock.calls.map(([feature, kind, opts]) => [feature, kind, opts?.key])
+		}
+
+		expect(await outcomeKeys()).toEqual([['ai_chat', 'tool', 'run_script:ok']])
+		expect(await outcomeKeys({ fn: vi.fn().mockRejectedValue(new Error('boom')) })).toEqual([
+			['ai_chat', 'tool', 'run_script:error']
+		])
+		expect(await outcomeKeys({ validateBeforeConfirmation: () => 'not deployed' })).toEqual([
+			['ai_chat', 'tool', 'run_script:rejected']
+		])
+		expect(
+			await outcomeKeys(
+				{ requiresConfirmation: true },
+				{ requestConfirmation: vi.fn().mockResolvedValue(false) }
+			)
+		).toEqual([['ai_chat', 'tool', 'run_script:declined']])
+		expect(await outcomeKeys({}, { isPlanModeActive: () => true })).toEqual([
+			['ai_chat', 'tool', 'run_script:blocked_plan_mode']
+		])
+
+		// A name the model invented resolves to no tool, and must never reach telemetry.
+		vi.mocked(logFeatureUsage).mockClear()
+		await processToolCall({
+			tools: [{ def: createToolDef(z.object({}), 'run_script', 'Run script'), fn: vi.fn() }],
+			toolCall: {
+				id: 'call_ghost',
+				type: 'function',
+				function: { name: 'hallucinated_tool', arguments: '{}' }
+			},
+			helpers: {},
+			workspace: 'test-workspace',
+			toolCallbacks: { setToolStatus: vi.fn(), removeToolStatus: vi.fn() }
+		})
+		expect(logFeatureUsage).not.toHaveBeenCalled()
+	})
 })

 async function runToolCall(
@@ -797,6 +797,15 @@ function stringifyErrorBody(body: unknown): string {
 	}
 }

+/**
+ * Closed vocabulary for the `ai_chat`/`tool` counter's `<name>:<status>` key.
+ * `ok` means the tool function resolved — tools that report failure by returning an
+ * error string instead of throwing land there too. A call abandoned mid-execution
+ * (tab closed while a tool polls) logs nothing, so the statuses sum to the calls that
+ * finished, not to the calls made.
+ */
+type ToolCallStatus = 'ok' | 'error' | 'declined' | 'rejected' | 'blocked_plan_mode'
+
 export async function processToolCall<T>({
 	tools,
 	toolCall,
@@ -810,10 +819,24 @@ export async function processToolCall<T>({
 	toolCallbacks: ToolCallbacks
 	workspace?: string
 }): Promise<ChatCompletionMessageParam> {
+	const tool = tools.find((t) => t.def.function.name === toolCall.function.name)
+	const workspaceId = workspace ?? get(workspaceStore) ?? ''
+
+	// Exactly once per call, on whichever path ends it. Keyed by the resolved tool's
+	// declared name, not the model-provided string, so hallucinated tool names never
+	// enter telemetry — an unresolved name is counted nowhere.
+	let outcomeLogged = false
+	const logToolOutcome = (status: ToolCallStatus) => {
+		if (!tool || outcomeLogged) return
+		outcomeLogged = true
+		logFeatureUsage('ai_chat', 'tool', {
+			key: `${tool.def.function.name}:${status}`,
+			workspace: workspaceId
+		})
+	}
+
 	try {
 		const args = JSON.parse(toolCall.function.arguments || '{}')
-		const tool = tools.find((t) => t.def.function.name === toolCall.function.name)
-		const workspaceId = workspace ?? get(workspaceStore) ?? ''

 		// Fails closed: untagged is blocked, only the safety tag exempt. Runs before anything
 		// belonging to the tool, so a validator cannot probe while planning — and again after
@@ -830,6 +853,7 @@ export async function processToolCall<T>({
 					: { label: PLAN_MODE_MESSAGES.blockedLabel, result: PLAN_MODE_MESSAGES.blockedResult }
 			if (!refusal) return undefined
 			toolCallbacks.onToolBlockedByPlanMode?.()
+			logToolOutcome('blocked_plan_mode')
 			toolCallbacks.setToolStatus(toolCall.id, {
 				content: refusal.label,
 				parameters: args,
@@ -858,6 +882,7 @@ export async function processToolCall<T>({
 			await tool?.validateBeforeConfirmation?.({ args, workspace: workspaceId, helpers })
 		)
 		if (rejection) {
+			logToolOutcome('rejected')
 			toolCallbacks.setToolStatus(toolCall.id, {
 				content: rejection.label,
 				parameters: args,
@@ -912,6 +937,7 @@ export async function processToolCall<T>({
 			const confirmed = await toolCallbacks.requestConfirmation(toolCall.id, toolCall.function.name)

 			if (!confirmed) {
+				logToolOutcome('declined')
 				toolCallbacks.setToolStatus(toolCall.id, {
 					content: 'Cancelled by user',
 					isLoading: false,
@@ -940,11 +966,6 @@ export async function processToolCall<T>({
 		}

 		let result = ''
-		// Key by the resolved tool's declared name, not the model-provided string,
-		// so hallucinated tool names never enter telemetry.
-		if (tool) {
-			logFeatureUsage('ai_chat', 'tool', { key: tool.def.function.name, workspace: workspaceId })
-		}
 		try {
 			result = await callTool({
 				tools,
@@ -955,12 +976,14 @@ export async function processToolCall<T>({
 				toolCallbacks,
 				toolId: toolCall.id
 			})
+			logToolOutcome('ok')
 			toolCallbacks.setToolStatus(toolCall.id, {
 				isLoading: false,
 				isStreamingArguments: false
 			})
 		} catch (err) {
 			console.error(err)
+			logToolOutcome('error')
 			const errorMessage = formatToolError(err)
 			toolCallbacks.setToolStatus(toolCall.id, {
 				isLoading: false,
@@ -977,6 +1000,7 @@ export async function processToolCall<T>({
 		return toAdd
 	} catch (err) {
 		console.error(err)
+		logToolOutcome('error')
 		const errorMessage = formatToolError(err)
 		toolCallbacks.setToolStatus(toolCall.id, {
 			isLoading: false,