From 7b17e358b35bf4ef8c213ea13252a456e87acb32 Mon Sep 17 00:00:00 2001 From: Guilhem Date: Tue, 18 Aug 2026 12:26:09 +0200 Subject: [PATCH] feat(frontend): record the outcome of every AI chat tool call (#10746) * feat(frontend): record the outcome of every AI chat tool call The `ai_chat`/`tool` counter fired before execution, so nothing recorded whether a tool call succeeded, and the three paths that refuse a call before it runs recorded nothing at all. Log once per call on whichever path ends it, keyed `:` over ok, error, declined, rejected and blocked_plan_mode. Per-tool totals now need `split_part(key, ':', 1)` downstream; rows keyed by the bare tool name coexist for up to 60 days. Co-Authored-By: Claude Opus 5 (1M context) * docs(frontend): state what the tool-call telemetry statuses do not cover `ok` means the tool function resolved, which includes tools that report failure by returning an error string, and a call abandoned mid-execution logs nothing. Also pin that a hallucinated tool name reaches telemetry nowhere. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- .../components/copilot/chat/shared.test.ts | 62 +++++++++++++++++++ .../src/lib/components/copilot/chat/shared.ts | 38 +++++++++--- 2 files changed, 93 insertions(+), 7 deletions(-) diff --git a/frontend/src/lib/components/copilot/chat/shared.test.ts b/frontend/src/lib/components/copilot/chat/shared.test.ts index 635ad2a7c4..086d4cc594 100644 --- a/frontend/src/lib/components/copilot/chat/shared.test.ts +++ b/frontend/src/lib/components/copilot/chat/shared.test.ts @@ -7,6 +7,11 @@ vi.mock('monaco-editor', () => ({ editor: {} })) +vi.mock('$lib/utils/featureUsage', () => ({ + logFeatureUsage: vi.fn(), + logHubScriptPick: vi.fn() +})) + const userHolder = vi.hoisted(() => ({ current: { is_super_admin: true } as { is_super_admin: boolean } })) @@ -902,6 +907,63 @@ describe('processToolCall', () => { }) ) }) + + // The counter is silently dropped by the backend when the key is malformed, so nothing + // here fails loudly if a path stops logging or logs the wrong status. + it('logs one feature-usage outcome per tool call, keyed :', async () => { + const { createToolDef, processToolCall } = await import('./shared') + const { logFeatureUsage } = await import('$lib/utils/featureUsage') + + const outcomeKeys = async ( + tool: Partial> = {}, + toolCallbacks: Partial = {} + ) => { + vi.mocked(logFeatureUsage).mockClear() + await runToolCall( + { + def: createToolDef(z.object({}), 'run_script', 'Run script'), + fn: vi.fn().mockResolvedValue('done'), + ...tool + }, + toolCallbacks + ) + return vi + .mocked(logFeatureUsage) + .mock.calls.map(([feature, kind, opts]) => [feature, kind, opts?.key]) + } + + expect(await outcomeKeys()).toEqual([['ai_chat', 'tool', 'run_script:ok']]) + expect(await outcomeKeys({ fn: vi.fn().mockRejectedValue(new Error('boom')) })).toEqual([ + ['ai_chat', 'tool', 'run_script:error'] + ]) + expect(await outcomeKeys({ validateBeforeConfirmation: () => 'not deployed' })).toEqual([ + ['ai_chat', 'tool', 'run_script:rejected'] + ]) + expect( + await outcomeKeys( + { requiresConfirmation: true }, + { requestConfirmation: vi.fn().mockResolvedValue(false) } + ) + ).toEqual([['ai_chat', 'tool', 'run_script:declined']]) + expect(await outcomeKeys({}, { isPlanModeActive: () => true })).toEqual([ + ['ai_chat', 'tool', 'run_script:blocked_plan_mode'] + ]) + + // A name the model invented resolves to no tool, and must never reach telemetry. + vi.mocked(logFeatureUsage).mockClear() + await processToolCall({ + tools: [{ def: createToolDef(z.object({}), 'run_script', 'Run script'), fn: vi.fn() }], + toolCall: { + id: 'call_ghost', + type: 'function', + function: { name: 'hallucinated_tool', arguments: '{}' } + }, + helpers: {}, + workspace: 'test-workspace', + toolCallbacks: { setToolStatus: vi.fn(), removeToolStatus: vi.fn() } + }) + expect(logFeatureUsage).not.toHaveBeenCalled() + }) }) async function runToolCall( diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index 83c586a34b..bdf4c52036 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -797,6 +797,15 @@ function stringifyErrorBody(body: unknown): string { } } +/** + * Closed vocabulary for the `ai_chat`/`tool` counter's `:` key. + * `ok` means the tool function resolved — tools that report failure by returning an + * error string instead of throwing land there too. A call abandoned mid-execution + * (tab closed while a tool polls) logs nothing, so the statuses sum to the calls that + * finished, not to the calls made. + */ +type ToolCallStatus = 'ok' | 'error' | 'declined' | 'rejected' | 'blocked_plan_mode' + export async function processToolCall({ tools, toolCall, @@ -810,10 +819,24 @@ export async function processToolCall({ toolCallbacks: ToolCallbacks workspace?: string }): Promise { + const tool = tools.find((t) => t.def.function.name === toolCall.function.name) + const workspaceId = workspace ?? get(workspaceStore) ?? '' + + // Exactly once per call, on whichever path ends it. Keyed by the resolved tool's + // declared name, not the model-provided string, so hallucinated tool names never + // enter telemetry — an unresolved name is counted nowhere. + let outcomeLogged = false + const logToolOutcome = (status: ToolCallStatus) => { + if (!tool || outcomeLogged) return + outcomeLogged = true + logFeatureUsage('ai_chat', 'tool', { + key: `${tool.def.function.name}:${status}`, + workspace: workspaceId + }) + } + try { const args = JSON.parse(toolCall.function.arguments || '{}') - const tool = tools.find((t) => t.def.function.name === toolCall.function.name) - const workspaceId = workspace ?? get(workspaceStore) ?? '' // Fails closed: untagged is blocked, only the safety tag exempt. Runs before anything // belonging to the tool, so a validator cannot probe while planning — and again after @@ -830,6 +853,7 @@ export async function processToolCall({ : { label: PLAN_MODE_MESSAGES.blockedLabel, result: PLAN_MODE_MESSAGES.blockedResult } if (!refusal) return undefined toolCallbacks.onToolBlockedByPlanMode?.() + logToolOutcome('blocked_plan_mode') toolCallbacks.setToolStatus(toolCall.id, { content: refusal.label, parameters: args, @@ -858,6 +882,7 @@ export async function processToolCall({ await tool?.validateBeforeConfirmation?.({ args, workspace: workspaceId, helpers }) ) if (rejection) { + logToolOutcome('rejected') toolCallbacks.setToolStatus(toolCall.id, { content: rejection.label, parameters: args, @@ -912,6 +937,7 @@ export async function processToolCall({ const confirmed = await toolCallbacks.requestConfirmation(toolCall.id, toolCall.function.name) if (!confirmed) { + logToolOutcome('declined') toolCallbacks.setToolStatus(toolCall.id, { content: 'Cancelled by user', isLoading: false, @@ -940,11 +966,6 @@ export async function processToolCall({ } let result = '' - // Key by the resolved tool's declared name, not the model-provided string, - // so hallucinated tool names never enter telemetry. - if (tool) { - logFeatureUsage('ai_chat', 'tool', { key: tool.def.function.name, workspace: workspaceId }) - } try { result = await callTool({ tools, @@ -955,12 +976,14 @@ export async function processToolCall({ toolCallbacks, toolId: toolCall.id }) + logToolOutcome('ok') toolCallbacks.setToolStatus(toolCall.id, { isLoading: false, isStreamingArguments: false }) } catch (err) { console.error(err) + logToolOutcome('error') const errorMessage = formatToolError(err) toolCallbacks.setToolStatus(toolCall.id, { isLoading: false, @@ -977,6 +1000,7 @@ export async function processToolCall({ return toAdd } catch (err) { console.error(err) + logToolOutcome('error') const errorMessage = formatToolError(err) toolCallbacks.setToolStatus(toolCall.id, { isLoading: false,