From add5d4305d8047d33554f2207b8f40ec494dbacc Mon Sep 17 00:00:00 2001 From: brennanb2025 Date: Fri, 29 May 2026 02:34:56 -0700 Subject: [PATCH] Track agent prompt telemetry from hooks Co-authored-by: Orca --- docs/reference/telemetry-availability.md | 2 +- src/main/agent-hooks/server.test.ts | 577 +++++++++++++++++- src/main/agent-hooks/server.ts | 121 +++- src/main/opencode/hook-service.test.ts | 4 +- src/main/opencode/hook-service.ts | 2 +- ...ay-session-agent-hooks.integration.test.ts | 107 ++++ src/main/ssh/ssh-relay-session.ts | 15 + src/main/telemetry/validator.test.ts | 4 +- src/relay/agent-hook-server.ts | 1 + src/renderer/src/hooks/useComposerState.ts | 26 +- .../lib/agent-prompt-sent-telemetry.test.ts | 79 --- .../src/lib/agent-prompt-sent-telemetry.ts | 25 - .../src/lib/launch-agent-in-new-tab.test.ts | 16 +- .../src/lib/launch-agent-in-new-tab.ts | 20 +- src/renderer/src/lib/new-workspace.test.ts | 30 +- src/renderer/src/lib/new-workspace.ts | 12 +- .../runtime-terminal-inspection.test.ts | 4 +- .../runtime/runtime-terminal-inspection.ts | 3 + src/shared/agent-hook-listener.test.ts | 66 ++ src/shared/agent-hook-listener.ts | 204 ++++++- src/shared/agent-hook-relay.ts | 3 + src/shared/telemetry-events.test.ts | 10 +- 22 files changed, 1098 insertions(+), 233 deletions(-) delete mode 100644 src/renderer/src/lib/agent-prompt-sent-telemetry.test.ts delete mode 100644 src/renderer/src/lib/agent-prompt-sent-telemetry.ts diff --git a/docs/reference/telemetry-availability.md b/docs/reference/telemetry-availability.md index 86be5373d4a..4626c972eab 100644 --- a/docs/reference/telemetry-availability.md +++ b/docs/reference/telemetry-availability.md @@ -39,7 +39,7 @@ D1+/D3+/D7+ retention means the user fired `app_opened` at least once after 24/7 - `source = 'unknown'` is not a real product surface. It means the caller omitted a source or the value failed schema validation. - `workspace_created` means create-worktree IPC succeeded. It is not a general "usable workspace exists" or "workspace revealed" marker. - `agent_started` means PTY spawn succeeded with agent telemetry attached. It is not first-repo activation and does not prove the user sent a prompt. -- `agent_prompt_sent` means Orca submitted a non-draft prompt/message through an agent launch or post-ready agent input path. It excludes agent auto-start, bare shells, and draft prefill. +- `agent_prompt_sent` means a live agent hook observed an explicit non-empty user prompt. It excludes hydrated/replayed status, agent auto-start, bare shells, draft prefill, and hookless sessions; missing rows mean no hook-confirmed interaction was observed, not proof the user never typed. - Workspace-outcome joins are native Electron coverage unless the query explicitly proves remote/web instrumentation. Remote runtime and web paths can bypass native repo/worktree telemetry, so do not interpret missing workspace outcome rows as product drop-off for SSH, remote, or web users. ## Rollouts diff --git a/src/main/agent-hooks/server.test.ts b/src/main/agent-hooks/server.test.ts index 3f4c2d6449f..926f24d8903 100644 --- a/src/main/agent-hooks/server.test.ts +++ b/src/main/agent-hooks/server.test.ts @@ -20,7 +20,8 @@ import { } from '../../shared/agent-status-types' import { makePaneKey } from '../../shared/stable-pane-id' -const { trackMock } = vi.hoisted(() => ({ +const { getCohortAtEmitMock, trackMock } = vi.hoisted(() => ({ + getCohortAtEmitMock: vi.fn(), trackMock: vi.fn() })) @@ -28,6 +29,10 @@ vi.mock('../telemetry/client', () => ({ track: trackMock })) +vi.mock('../telemetry/cohort-classifier', () => ({ + getCohortAtEmit: getCohortAtEmitMock +})) + const LEAF_1 = '11111111-1111-4111-8111-111111111111' const LEAF_2 = '22222222-2222-4222-8222-222222222222' const LEAF_3 = '33333333-3333-4333-8333-333333333333' @@ -62,6 +67,8 @@ function buildBody(payload: Record, overrides: Partial = beforeEach(() => { _internals.resetCachesForTests() trackMock.mockReset() + getCohortAtEmitMock.mockReset() + getCohortAtEmitMock.mockReturnValue({ nth_repo_added: 2 }) }) afterEach(() => { @@ -2171,6 +2178,537 @@ describe('AgentHookServer listener replay', () => { }) }) +describe('AgentHookServer prompt-sent telemetry', () => { + it('tracks a live local hook explicit prompt with conservative attribution', async () => { + const server = new AgentHookServer() + await server.start({ env: 'production' }) + try { + const env = server.buildPtyEnv() + const response = await fetch(`http://127.0.0.1:${env.ORCA_AGENT_HOOK_PORT}/hook/claude`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Orca-Agent-Hook-Token': env.ORCA_AGENT_HOOK_TOKEN + }, + body: JSON.stringify( + buildBody({ + hook_event_name: 'UserPromptSubmit', + prompt: ' fix the spinner ' + }) + ) + }) + + expect(response.status).toBe(204) + expect(trackMock).toHaveBeenCalledWith('agent_prompt_sent', { + agent_kind: 'claude-code', + launch_source: 'unknown', + request_kind: 'followup', + nth_repo_added: 2 + }) + } finally { + server.stop() + } + }) + + it('tracks a live SSH hook explicit prompt through ingestRemote', () => { + const server = new AgentHookServer() + + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: 'remote prompt', agentType: 'codex' } + }, + 'conn-1' + ) + + expect(trackMock).toHaveBeenCalledWith('agent_prompt_sent', { + agent_kind: 'codex', + launch_source: 'unknown', + request_kind: 'followup', + nth_repo_added: 2 + }) + }) + + it('dedupes adjacent same-turn reports without considering hook state', () => { + vi.useFakeTimers() + vi.setSystemTime(1_000) + try { + const server = new AgentHookServer() + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: 'same turn', agentType: 'gemini' } + }, + 'conn-1' + ) + vi.setSystemTime(1_500) + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'done', prompt: 'same turn', agentType: 'gemini' } + }, + 'conn-1' + ) + + expect(trackMock).toHaveBeenCalledTimes(1) + expect(trackMock).toHaveBeenCalledWith('agent_prompt_sent', { + agent_kind: 'gemini', + launch_source: 'unknown', + request_kind: 'followup', + nth_repo_added: 2 + }) + } finally { + vi.useRealTimers() + } + }) + + it('tracks the same prompt again after a completed turn starts over', () => { + vi.useFakeTimers() + vi.setSystemTime(1_000) + try { + const server = new AgentHookServer() + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: 'continue', agentType: 'codex' } + }, + 'conn-1' + ) + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + payload: { state: 'done', prompt: 'continue', agentType: 'codex' } + }, + 'conn-1' + ) + vi.setSystemTime(1_500) + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: 'continue', agentType: 'codex' } + }, + 'conn-1' + ) + + expect(trackMock).toHaveBeenCalledTimes(2) + } finally { + vi.useRealTimers() + } + }) + + it('dedupes duplicate Command Code stop hooks but tracks same-prompt reruns', () => { + const server = new AgentHookServer() + + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + promptInteractionKey: 'command-code-transcript-user-1', + payload: { state: 'done', prompt: 'rerun', agentType: 'command-code' } + }, + 'conn-1' + ) + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + promptInteractionKey: 'command-code-transcript-user-1', + payload: { state: 'done', prompt: 'rerun', agentType: 'command-code' } + }, + 'conn-1' + ) + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + promptInteractionKey: 'command-code-transcript-user-2', + payload: { state: 'done', prompt: 'rerun', agentType: 'command-code' } + }, + 'conn-1' + ) + + expect(trackMock).toHaveBeenCalledTimes(2) + }) + + it('dedupes Command Code direct prompt hooks followed by transcript-backed stop hooks', () => { + const server = new AgentHookServer() + + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: 'same command', agentType: 'command-code' } + }, + 'conn-1' + ) + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + promptInteractionKey: 'command-code-transcript-a-1', + payload: { state: 'done', prompt: 'same command', agentType: 'command-code' } + }, + 'conn-1' + ) + + expect(trackMock).toHaveBeenCalledTimes(1) + }) + + it('does not let a reused interaction key suppress different prompt text', () => { + const server = new AgentHookServer() + + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + promptInteractionKey: 'command-code-transcript-reused', + payload: { state: 'done', prompt: 'first command', agentType: 'command-code' } + }, + 'conn-1' + ) + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + promptInteractionKey: 'command-code-transcript-reused', + payload: { state: 'done', prompt: 'second command', agentType: 'command-code' } + }, + 'conn-1' + ) + + expect(trackMock).toHaveBeenCalledTimes(2) + }) + + it('does not treat Command Code cached prompts as explicit prompt evidence', () => { + const server = new AgentHookServer() + + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'done', prompt: 'cached prompt', agentType: 'command-code' } + }, + 'conn-1' + ) + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: false, + payload: { state: 'done', prompt: 'cached prompt', agentType: 'command-code' } + }, + 'conn-1' + ) + + expect(trackMock).toHaveBeenCalledTimes(1) + }) + + it('preserves prompt dedupe when a live status row is dismissed', () => { + const server = new AgentHookServer() + + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: 'long turn', agentType: 'codex' } + }, + 'conn-1' + ) + server.dropStatusEntry(PANE) + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: 'long turn', agentType: 'codex' } + }, + 'conn-1' + ) + + expect(trackMock).toHaveBeenCalledTimes(1) + }) + + it('lets a dismissed completed row start the same prompt again', () => { + const server = new AgentHookServer() + + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'done', prompt: 'rerun after done', agentType: 'codex' } + }, + 'conn-1' + ) + server.dropStatusEntry(PANE) + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: 'rerun after done', agentType: 'codex' } + }, + 'conn-1' + ) + + expect(trackMock).toHaveBeenCalledTimes(2) + }) + + it('dedupes the same prompt until a completed turn boundary is observed', () => { + vi.useFakeTimers() + vi.setSystemTime(1_000) + try { + const server = new AgentHookServer() + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: 'repeat later', agentType: 'codex' } + }, + 'conn-1' + ) + vi.setSystemTime(32_000) + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: 'repeat later', agentType: 'codex' } + }, + 'conn-1' + ) + + expect(trackMock).toHaveBeenCalledTimes(1) + } finally { + vi.useRealTimers() + } + }) + + it('does not track replays, empty prompts, or inherited prompt snapshots', () => { + const server = new AgentHookServer() + + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + isReplay: true, + payload: { state: 'working', prompt: 'replayed prompt', agentType: 'codex' } + }, + 'conn-1' + ) + server.ingestRemote( + { + paneKey: GOOD_PANE, + tabId: 'tab-good', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: ' ', agentType: 'codex' } + }, + 'conn-1' + ) + server.ingestRemote( + { + paneKey: FRESH_PANE, + tabId: 'tab-fresh', + worktreeId: 'wt-1', + payload: { state: 'working', prompt: 'inherited prompt', agentType: 'codex' } + }, + 'conn-1' + ) + + expect(trackMock).not.toHaveBeenCalledWith('agent_prompt_sent', expect.anything()) + }) + + it('does not track hook status messages that preserve a cached prompt', () => { + const server = new AgentHookServer() + + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: 'real prompt', agentType: 'droid' } + }, + 'conn-1' + ) + trackMock.mockClear() + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: false, + payload: { state: 'waiting', prompt: 'real prompt', agentType: 'droid' } + }, + 'conn-1' + ) + + expect(trackMock).not.toHaveBeenCalledWith('agent_prompt_sent', expect.anything()) + }) + + it('tracks OpenCode user MessagePart hooks once per message id', async () => { + const server = new AgentHookServer() + await server.start({ env: 'production' }) + try { + const env = server.buildPtyEnv() + const response = await fetch(`http://127.0.0.1:${env.ORCA_AGENT_HOOK_PORT}/hook/opencode`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Orca-Agent-Hook-Token': env.ORCA_AGENT_HOOK_TOKEN + }, + body: JSON.stringify( + buildBody({ + hook_event_name: 'MessagePart', + role: 'user', + text: 'fix', + messageID: 'msg-1' + }) + ) + }) + const updatedResponse = await fetch( + `http://127.0.0.1:${env.ORCA_AGENT_HOOK_PORT}/hook/opencode`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'X-Orca-Agent-Hook-Token': env.ORCA_AGENT_HOOK_TOKEN + }, + body: JSON.stringify( + buildBody({ + hook_event_name: 'MessagePart', + role: 'user', + text: 'fix tests', + messageID: 'msg-1' + }) + ) + } + ) + + expect(response.status).toBe(204) + expect(updatedResponse.status).toBe(204) + expect(server.getStatusSnapshot()[0]).toMatchObject({ + state: 'working', + prompt: 'fix tests', + agentType: 'opencode' + }) + expect(trackMock).toHaveBeenCalledTimes(1) + expect(trackMock).toHaveBeenCalledWith('agent_prompt_sent', { + agent_kind: 'opencode', + launch_source: 'unknown', + request_kind: 'followup', + nth_repo_added: 2 + }) + } finally { + server.stop() + } + }) + + it('maps custom hook agent types to other', () => { + const server = new AgentHookServer() + + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: 'custom prompt', agentType: 'my-local-agent' } + }, + 'conn-1' + ) + + expect(trackMock).toHaveBeenCalledWith('agent_prompt_sent', { + agent_kind: 'other', + launch_source: 'unknown', + request_kind: 'followup', + nth_repo_added: 2 + }) + }) + + it('does not block status cache mutation or listener fanout when telemetry throws', () => { + const server = new AgentHookServer() + const listener = vi.fn() + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}) + trackMock.mockImplementationOnce(() => { + throw new Error('telemetry unavailable') + }) + server.setListener(listener) + + server.ingestRemote( + { + paneKey: PANE, + tabId: 'tab-1', + worktreeId: 'wt-1', + hasExplicitPrompt: true, + payload: { state: 'working', prompt: 'keep status moving', agentType: 'codex' } + }, + 'conn-1' + ) + + expect(server.getStatusSnapshot()).toEqual([ + expect.objectContaining({ + paneKey: PANE, + state: 'working', + prompt: 'keep status moving', + agentType: 'codex' + }) + ]) + expect(listener).toHaveBeenCalledWith( + expect.objectContaining({ + paneKey: PANE, + payload: expect.objectContaining({ prompt: 'keep status moving' }) + }) + ) + errorSpy.mockRestore() + }) +}) + describe('Claude hook normalization', () => { it('PostToolUse for Edit surfaces toolName + file_path preview', () => { const result = _internals.normalizeHookPayload( @@ -2808,11 +3346,18 @@ describe('OpenCode hook normalization', () => { it('MessagePart with role=user surfaces text as the prompt and stays working', () => { const result = _internals.normalizeHookPayload( 'opencode', - buildBody({ hook_event_name: 'MessagePart', role: 'user', text: 'hi there' }), + buildBody({ + hook_event_name: 'MessagePart', + role: 'user', + text: 'hi there', + messageID: 'msg-1' + }), 'production' ) expect(result?.payload.state).toBe('working') expect(result?.payload.prompt).toBe('hi there') + expect(result?.hasExplicitPrompt).toBe(true) + expect(result?.promptInteractionKey).toBe('opencode-message-msg-1') }) it('MessagePart with role=assistant populates lastAssistantMessage', () => { @@ -3108,6 +3653,7 @@ describe('Droid hook normalization', () => { expect(done?.payload.state).toBe('done') expect(done?.payload.prompt).toBe('write tests') + expect(done?.hasExplicitPrompt).toBe(false) }) it('Notification ignores confirmation status text rather than treating it as permission', () => { @@ -3598,6 +4144,7 @@ describe('Copilot hook normalization', () => { expect(result?.payload.state).toBe('blocked') expect(result?.payload.prompt).toBe('deploy the app') expect(result?.payload.lastAssistantMessage).toBe('Which environment?') + expect(result?.hasExplicitPrompt).toBe(false) }) it('Notification(elicitation_dialog) accepts camelCase type and surfaces the question', () => { @@ -4215,6 +4762,32 @@ describe('Last-status persistence', () => { } }) + it('does not write prompt interaction keys to last-status.json', async () => { + const server = new AgentHookServer() + await server.start({ + env: 'production', + userDataPath + }) + try { + await postHookEvent( + server, + buildBody({ + hook_event_name: 'MessagePart', + role: 'user', + text: 'persist status only', + messageID: 'opencode-local-message-id' + }), + '/hook/opencode' + ) + server.flushStatusPersistSync() + const file = JSON.parse(readFileSync(lastStatusPath(), 'utf8')) + expect(file.entries[PANE].payload.prompt).toBe('persist status only') + expect(file.entries[PANE].promptInteractionKey).toBeUndefined() + } finally { + server.stop() + } + }) + it('hydrates last-status.json into the cache before listener registration', async () => { // Pre-populate the file directly to simulate a prior session. mkdirSync(join(userDataPath, 'agent-hooks'), { recursive: true }) diff --git a/src/main/agent-hooks/server.ts b/src/main/agent-hooks/server.ts index 0ff58601ddb..636b1e4affb 100644 --- a/src/main/agent-hooks/server.ts +++ b/src/main/agent-hooks/server.ts @@ -11,11 +11,13 @@ // - the on-disk last-status cache (`last-status.json`) that survives // Orca restart so retained dashboard rows reappear on relaunch import { createServer, type IncomingMessage, type ServerResponse } from 'http' -import { randomUUID } from 'crypto' +import { createHash, randomBytes, randomUUID } from 'crypto' import { chmodSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from 'fs' import { join } from 'path' import { track } from '../telemetry/client' +import { getCohortAtEmit } from '../telemetry/cohort-classifier' +import { AGENT_KIND_VALUES, type AgentKind } from '../../shared/telemetry-events' import { ORCA_HOOK_PROTOCOL_VERSION } from '../../shared/agent-hook-types' import { clearAllListenerCaches, @@ -99,6 +101,7 @@ const LAST_STATUS_FILE_VERSION = 2 // hook-server batching; quit-time uses flushStatusPersistSync() for the // guaranteed final flush. const STATUS_PERSIST_DEBOUNCE_MS = 250 +const AGENT_PROMPT_SENT_AGENT_KINDS = new Set(AGENT_KIND_VALUES) // Why: bound the on-disk file's growth across many sessions. PTY-teardown // eviction handles closed panes, but daemon-restored PTYs that never re-attach @@ -113,6 +116,25 @@ type LastStatusFile = { entries: Record } +type AgentPromptSentDedupeEntry = { + agentKind: AgentKind + promptHash: string + promptInteractionKey?: string +} + +function agentTypeToPromptSentAgentKind(agentType: AgentType | undefined): AgentKind { + const normalized = agentType?.trim().toLowerCase() + if (!normalized || normalized === 'unknown') { + return 'other' + } + if (normalized === 'claude') { + return 'claude-code' + } + return AGENT_PROMPT_SENT_AGENT_KINDS.has(normalized as AgentKind) + ? (normalized as AgentKind) + : 'other' +} + function equivalentInterruptAgentType( actual: AgentType | undefined, baseline: AgentType | undefined @@ -383,6 +405,8 @@ export class AgentHookServer { // server instances in the same process (tests) don't share state. private statusPersistTimer: ReturnType | null = null private assistantMessageRetryTimers = new Map>() + private promptSentDedupeByPaneKey = new Map() + private promptSentHashSalt = randomBytes(16).toString('hex') // Why: identity check — skip writes when the JSON-stringified contents // exactly match the last successful disk write. Cheap protection against // re-firing trailing timers when nothing changed. @@ -529,6 +553,76 @@ export class AgentHookServer { } } + private hashPromptForTelemetryDedupe(prompt: string): string { + return createHash('sha256') + .update(this.promptSentHashSalt) + .update('\0') + .update(prompt) + .digest('hex') + } + + private maybeTrackAgentPromptSent( + payload: AgentHookEventPayload, + previousStatus: EnrichedAgentHookEventPayload | undefined + ): void { + if (payload.isReplay === true || payload.hasExplicitPrompt !== true) { + return + } + const prompt = payload.payload.prompt?.trim() ?? '' + if (prompt.length === 0) { + return + } + const agentKind = agentTypeToPromptSentAgentKind(payload.payload.agentType) + const promptHash = this.hashPromptForTelemetryDedupe(prompt) + const promptInteractionKey = + typeof payload.promptInteractionKey === 'string' && + payload.promptInteractionKey.trim().length > 0 + ? payload.promptInteractionKey.trim() + : undefined + const previousDedupe = this.promptSentDedupeByPaneKey.get(payload.paneKey) + const isCompletedTurnBoundary = + previousStatus?.payload.state === 'done' && payload.payload.state === 'working' + if ( + previousDedupe?.agentKind === agentKind && + previousDedupe.promptInteractionKey !== undefined && + previousDedupe.promptInteractionKey === promptInteractionKey && + (agentKind === 'opencode' || previousDedupe.promptHash === promptHash) + ) { + return + } + if ( + previousDedupe?.agentKind === agentKind && + previousDedupe.promptHash === promptHash && + !( + previousStatus?.payload.state === 'done' && + payload.payload.state === 'done' && + previousDedupe.promptInteractionKey !== undefined && + promptInteractionKey !== undefined && + previousDedupe.promptInteractionKey !== promptInteractionKey + ) && + !isCompletedTurnBoundary + ) { + return + } + this.promptSentDedupeByPaneKey.set(payload.paneKey, { + agentKind, + promptHash, + promptInteractionKey + }) + try { + // Why: hooks prove the user submitted a turn, but do not know which UI + // launched the terminal; keep attribution low-cardinality and conservative. + track('agent_prompt_sent', { + agent_kind: agentKind, + launch_source: 'unknown', + request_kind: 'followup', + ...getCohortAtEmit() + }) + } catch (err) { + console.error('[agent-hooks] prompt-sent telemetry failed', err) + } + } + private applyNormalizedStatus(payload: AgentHookEventPayload): EnrichedAgentHookEventPayload { const previous = this.state.lastStatusByPaneKey.get(payload.paneKey) as | EnrichedAgentHookEventPayload @@ -567,6 +661,7 @@ export class AgentHookServer { ) { this.clearAssistantMessageRetry(effectivePayload.paneKey) } + this.maybeTrackAgentPromptSent(effectivePayload, previous) const enriched = this.attachStatusTiming(effectivePayload) this.runtimeObservedStatusPaneKeys.add(enriched.paneKey) this.state.lastStatusByPaneKey.set(enriched.paneKey, enriched) @@ -702,6 +797,7 @@ export class AgentHookServer { if (entry.ptyId === ptyId) { this.legacyPaneKeyAliases.delete(legacyPaneKey) clearPaneCacheState(this.state, legacyPaneKey) + this.promptSentDedupeByPaneKey.delete(legacyPaneKey) const shouldClearStablePaneKey = options?.shouldClearStablePaneKey?.(entry.stablePaneKey) ?? true if (shouldClearStablePaneKey && this.state.lastStatusByPaneKey.has(entry.stablePaneKey)) { @@ -713,6 +809,7 @@ export class AgentHookServer { // cleanup is the only path that can evict that retained status. clearPaneCacheState(this.state, entry.stablePaneKey) this.runtimeObservedStatusPaneKeys.delete(entry.stablePaneKey) + this.promptSentDedupeByPaneKey.delete(entry.stablePaneKey) } aliasChanged = true } @@ -763,6 +860,7 @@ export class AgentHookServer { env?: string version?: string hasExplicitPrompt?: boolean + promptInteractionKey?: string hookEventName?: string toolUseId?: string toolAgentId?: string @@ -824,6 +922,11 @@ export class AgentHookServer { typeof envelope.hookEventName === 'string' && envelope.hookEventName.trim().length > 0 ? envelope.hookEventName.trim() : undefined + const promptInteractionKey = + typeof envelope.promptInteractionKey === 'string' && + envelope.promptInteractionKey.trim().length > 0 + ? envelope.promptInteractionKey.trim() + : undefined const toolUseId = typeof envelope.toolUseId === 'string' && envelope.toolUseId.trim().length > 0 ? envelope.toolUseId.trim() @@ -860,6 +963,7 @@ export class AgentHookServer { worktreeId, connectionId: trimmedConnectionId, hasExplicitPrompt: envelope.hasExplicitPrompt === true ? true : undefined, + promptInteractionKey, hookEventName, toolUseId, toolAgentId, @@ -1001,6 +1105,7 @@ export class AgentHookServer { this.lastStatusFilePath = null this.lastWrittenJson = null this.runtimeObservedStatusPaneKeys.clear() + this.promptSentDedupeByPaneKey.clear() this.legacyPaneKeyAliases.clear() clearAllListenerCaches(this.state) this.notifyStatusChangeListeners() @@ -1018,9 +1123,13 @@ export class AgentHookServer { if (!this.state.lastStatusByPaneKey.has(resolvedPaneKey)) { return } + const existing = this.state.lastStatusByPaneKey.get(resolvedPaneKey) this.state.lastStatusByPaneKey.delete(resolvedPaneKey) this.clearAssistantMessageRetry(resolvedPaneKey) this.runtimeObservedStatusPaneKeys.delete(resolvedPaneKey) + if (existing?.payload.state === 'done') { + this.promptSentDedupeByPaneKey.delete(resolvedPaneKey) + } this.scheduleStatusPersist() this.notifyStatusChangeListeners() } @@ -1034,11 +1143,13 @@ export class AgentHookServer { const hadStatus = this.state.lastStatusByPaneKey.has(resolvedPaneKey) this.clearAssistantMessageRetry(resolvedPaneKey) clearPaneCacheState(this.state, resolvedPaneKey) + this.promptSentDedupeByPaneKey.delete(resolvedPaneKey) let clearedAlias = false for (const [legacyPaneKey, stablePaneKey] of this.legacyPaneKeyAliases) { if (stablePaneKey.stablePaneKey === resolvedPaneKey) { this.legacyPaneKeyAliases.delete(legacyPaneKey) clearPaneCacheState(this.state, legacyPaneKey) + this.promptSentDedupeByPaneKey.delete(legacyPaneKey) clearedAlias = true } } @@ -1190,7 +1301,8 @@ export class AgentHookServer { if (!isValidPaneKey(paneKey)) { continue } - entries[paneKey] = payload as EnrichedAgentHookEventPayload + const { promptInteractionKey: _promptInteractionKey, ...persistedPayload } = payload + entries[paneKey] = persistedPayload as EnrichedAgentHookEventPayload } const file: LastStatusFile = { version: LAST_STATUS_FILE_VERSION, entries } return JSON.stringify(file) @@ -1270,6 +1382,10 @@ export class AgentHookServer { _getStateForTests(): HookListenerState { return this.state } + + _resetPromptSentDedupeForTests(): void { + this.promptSentDedupeByPaneKey.clear() + } } export const agentHookServer = new AgentHookServer() @@ -1287,5 +1403,6 @@ export const _internals = { parseFormEncodedBody, resetCachesForTests: (): void => { clearAllListenerCaches(agentHookServer._getStateForTests()) + agentHookServer._resetPromptSentDedupeForTests() } } diff --git a/src/main/opencode/hook-service.test.ts b/src/main/opencode/hook-service.test.ts index 53896f56e40..ebbcf17524c 100644 --- a/src/main/opencode/hook-service.test.ts +++ b/src/main/opencode/hook-service.test.ts @@ -210,7 +210,9 @@ describe('OpenCodeHookService buildPtyEnv / clearPty round-trip', () => { const pluginPath = join(env.OPENCODE_CONFIG_DIR!, 'plugins', 'orca-opencode-status.js') expect(existsSync(pluginPath)).toBe(true) // Sanity-check the file has plugin source, not a stray write. - expect(readFileSync(pluginPath, 'utf8')).toContain('OrcaOpenCodeStatusPlugin') + const pluginSource = readFileSync(pluginPath, 'utf8') + expect(pluginSource).toContain('OrcaOpenCodeStatusPlugin') + expect(pluginSource).toContain('messageID: part.messageID') }) it('clearPty removes the same directory buildPtyEnv created', () => { diff --git a/src/main/opencode/hook-service.ts b/src/main/opencode/hook-service.ts index 074ed6720a0..c96b3063933 100644 --- a/src/main/opencode/hook-service.ts +++ b/src/main/opencode/hook-service.ts @@ -305,7 +305,7 @@ function getOpenCodePluginSource(): string { ' if (!part || part.type !== "text" || !part.text) return;', ' const role = messageRoleById.get(part.messageID);', ' if (!role) return;', - ' await post("MessagePart", { role, text: part.text });', + ' await post("MessagePart", { role, text: part.text, messageID: part.messageID });', ' return;', ' }', '', diff --git a/src/main/ssh/ssh-relay-session-agent-hooks.integration.test.ts b/src/main/ssh/ssh-relay-session-agent-hooks.integration.test.ts index f4ea0b22187..057e095de12 100644 --- a/src/main/ssh/ssh-relay-session-agent-hooks.integration.test.ts +++ b/src/main/ssh/ssh-relay-session-agent-hooks.integration.test.ts @@ -19,6 +19,19 @@ import { agentHookServer, _internals as agentHookInternals } from '../agent-hook import { getSshPtyProvider } from '../ipc/pty' import { toAppSshPtyId } from '../providers/ssh-pty-id' +const { getCohortAtEmitMock, trackMock } = vi.hoisted(() => ({ + getCohortAtEmitMock: vi.fn(), + trackMock: vi.fn() +})) + +vi.mock('../telemetry/client', () => ({ + track: trackMock +})) + +vi.mock('../telemetry/cohort-classifier', () => ({ + getCohortAtEmit: getCohortAtEmitMock +})) + vi.mock('./ssh-relay-deploy', () => ({ deployAndLaunchRelay: vi.fn() })) @@ -184,6 +197,9 @@ describe('SshRelaySession agent hooks over a fake relay transport', () => { beforeEach(() => { vi.clearAllMocks() + trackMock.mockReset() + getCohortAtEmitMock.mockReset() + getCohortAtEmitMock.mockReturnValue({ nth_repo_added: 4 }) previousRemoteHooksFlag = process.env[ORCA_FEATURE_REMOTE_AGENT_HOOKS_ENV] process.env[ORCA_FEATURE_REMOTE_AGENT_HOOKS_ENV] = '1' warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}) @@ -388,6 +404,97 @@ describe('SshRelaySession agent hooks over a fake relay transport', () => { ) }) + it('forwards remote hook transition metadata into main ingest', async () => { + relay = createFakeRelay() + vi.mocked(deployAndLaunchRelay).mockResolvedValue({ + transport: relay.transport, + platform: 'linux-x64' + }) + const ingestSpy = vi.spyOn(agentHookServer, 'ingestRemote') + + session = createSession('conn-hook-metadata') + await session.establish({} as SshConnection) + + relay.notifyAgentHook( + makeEnvelope({ + source: 'claude', + hookEventName: 'PreToolUse', + promptInteractionKey: 'command-code-transcript-user-3', + toolUseId: 'toolu-1', + toolAgentId: 'agent-subagent-a', + toolAgentType: 'Review', + payload: { + state: 'working', + prompt: 'remote prompt', + agentType: 'claude', + toolName: 'Bash', + toolInput: 'pnpm test' + } + }) + ) + + await vi.waitFor(() => + expect(ingestSpy).toHaveBeenCalledWith( + expect.objectContaining({ + hookEventName: 'PreToolUse', + promptInteractionKey: 'command-code-transcript-user-3', + toolUseId: 'toolu-1', + toolAgentId: 'agent-subagent-a', + toolAgentType: 'Review' + }), + 'conn-hook-metadata' + ) + ) + ingestSpy.mockRestore() + }) + + it('tracks prompt sent from live SSH agent hooks but not replayed hooks', async () => { + relay = createFakeRelay() + vi.mocked(deployAndLaunchRelay).mockResolvedValue({ + transport: relay.transport, + platform: 'linux-x64' + }) + + session = createSession('conn-live-telemetry') + await session.establish({} as SshConnection) + + relay.notifyAgentHook( + makeEnvelope({ + hasExplicitPrompt: true, + payload: { + state: 'working', + prompt: 'ssh live user prompt', + agentType: 'codex' + } + }) + ) + + await vi.waitFor(() => + expect(trackMock).toHaveBeenCalledWith('agent_prompt_sent', { + agent_kind: 'codex', + launch_source: 'unknown', + request_kind: 'followup', + nth_repo_added: 4 + }) + ) + + trackMock.mockClear() + relay.notifyAgentHook( + makeEnvelope({ + hasExplicitPrompt: true, + isReplay: true, + payload: { + state: 'working', + prompt: 'ssh replayed user prompt', + agentType: 'codex' + } + }) + ) + + await new Promise((resolve) => setImmediate(resolve)) + expect(trackMock).not.toHaveBeenCalledWith('agent_prompt_sent', expect.anything()) + }) + it('preserves replay metadata from remote hook notifications', async () => { relay = createFakeRelay() vi.mocked(deployAndLaunchRelay).mockResolvedValue({ diff --git a/src/main/ssh/ssh-relay-session.ts b/src/main/ssh/ssh-relay-session.ts index 73b72b8647b..76a9ac4aca7 100644 --- a/src/main/ssh/ssh-relay-session.ts +++ b/src/main/ssh/ssh-relay-session.ts @@ -642,6 +642,11 @@ export class SshRelaySession { env?: unknown version?: unknown hasExplicitPrompt?: unknown + promptInteractionKey?: unknown + hookEventName?: unknown + toolUseId?: unknown + toolAgentId?: unknown + toolAgentType?: unknown isReplay?: unknown payload?: unknown } @@ -661,6 +666,16 @@ export class SshRelaySession { env: typeof envelope.env === 'string' ? envelope.env : undefined, version: typeof envelope.version === 'string' ? envelope.version : undefined, hasExplicitPrompt: envelope.hasExplicitPrompt === true ? true : undefined, + promptInteractionKey: + typeof envelope.promptInteractionKey === 'string' + ? envelope.promptInteractionKey + : undefined, + hookEventName: + typeof envelope.hookEventName === 'string' ? envelope.hookEventName : undefined, + toolUseId: typeof envelope.toolUseId === 'string' ? envelope.toolUseId : undefined, + toolAgentId: typeof envelope.toolAgentId === 'string' ? envelope.toolAgentId : undefined, + toolAgentType: + typeof envelope.toolAgentType === 'string' ? envelope.toolAgentType : undefined, isReplay: envelope.isReplay === true ? true : undefined, payload: envelope.payload }, diff --git a/src/main/telemetry/validator.test.ts b/src/main/telemetry/validator.test.ts index 996e3e3d5ee..d8b1d6d5bb5 100644 --- a/src/main/telemetry/validator.test.ts +++ b/src/main/telemetry/validator.test.ts @@ -34,8 +34,8 @@ describe('validate', () => { it('accepts a well-formed agent_prompt_sent payload', () => { const result = validate('agent_prompt_sent', { agent_kind: 'claude-code', - launch_source: 'new_workspace_composer', - request_kind: 'new', + launch_source: 'unknown', + request_kind: 'followup', nth_repo_added: 1 }) expect(result.ok).toBe(true) diff --git a/src/relay/agent-hook-server.ts b/src/relay/agent-hook-server.ts index 2789fffde74..2dd9c23b0a4 100644 --- a/src/relay/agent-hook-server.ts +++ b/src/relay/agent-hook-server.ts @@ -292,6 +292,7 @@ export class RelayAgentHookServer { worktreeId: event.worktreeId, connectionId: null, hasExplicitPrompt: event.hasExplicitPrompt, + promptInteractionKey: event.promptInteractionKey, hookEventName: event.hookEventName, toolUseId: event.toolUseId, toolAgentId: event.toolAgentId, diff --git a/src/renderer/src/hooks/useComposerState.ts b/src/renderer/src/hooks/useComposerState.ts index 2a6a9fdfefb..08aa47de717 100644 --- a/src/renderer/src/hooks/useComposerState.ts +++ b/src/renderer/src/hooks/useComposerState.ts @@ -19,8 +19,6 @@ import { filterEnabledTuiAgents, isTuiAgentEnabled } from '../../../shared/tui-a import { tuiAgentToAgentKind } from '@/lib/telemetry' import { isGitRepoKind } from '../../../shared/repo-kind' import { callRuntimeRpc, getActiveRuntimeTarget } from '@/runtime/runtime-rpc-client' -import type { EventProps } from '../../../shared/telemetry-events' -import { trackFoldedStartupPromptSent } from '@/lib/agent-prompt-sent-telemetry' import type { GitHubWorkItem, GitPushTarget, @@ -1872,8 +1870,7 @@ export function useComposerState(options: UseComposerStateOptions): UseComposerS launch_source: telemetrySource === 'onboarding' ? 'onboarding' : 'new_workspace_composer', request_kind: 'new' } - const composerPromptTelemetry: EventProps<'agent_prompt_sent'> = composerTelemetry - const activation = activateAndRevealWorktree(worktree.id, { + activateAndRevealWorktree(worktree.id, { sidebarRevealBehavior: 'auto', setup: result.setup, issueCommand, @@ -1896,16 +1893,9 @@ export function useComposerState(options: UseComposerStateOptions): UseComposerS : {}) }) if (startupPlan) { - trackFoldedStartupPromptSent({ - prompt: submitStartupPrompt, - startupPlan, - promptTelemetry: composerPromptTelemetry, - startupQueued: activation !== false && activation.primaryTabId !== null - }) void ensureAgentStartupInTerminal({ worktreeId: worktree.id, - startup: startupPlan, - promptTelemetry: composerPromptTelemetry + startup: startupPlan }) } setSidebarOpen(true) @@ -2148,8 +2138,7 @@ export function useComposerState(options: UseComposerStateOptions): UseComposerS telemetrySource === 'onboarding' ? 'onboarding' : 'new_workspace_composer', request_kind: 'new' } - const quickPromptTelemetry: EventProps<'agent_prompt_sent'> | null = quickTelemetry - const activation = activateAndRevealWorktree(worktree.id, { + activateAndRevealWorktree(worktree.id, { sidebarRevealBehavior: 'auto', setup: result.setup, ...(startupPlan @@ -2171,16 +2160,9 @@ export function useComposerState(options: UseComposerStateOptions): UseComposerS : {}) }) if (startupPlan) { - trackFoldedStartupPromptSent({ - prompt: quickPrompt, - startupPlan, - promptTelemetry: quickPromptTelemetry, - startupQueued: activation !== false && activation.primaryTabId !== null - }) void ensureAgentStartupInTerminal({ worktreeId: worktree.id, - startup: startupPlan, - ...(quickPromptTelemetry ? { promptTelemetry: quickPromptTelemetry } : {}) + startup: startupPlan }) } setSidebarOpen(true) diff --git a/src/renderer/src/lib/agent-prompt-sent-telemetry.test.ts b/src/renderer/src/lib/agent-prompt-sent-telemetry.test.ts deleted file mode 100644 index 966bf3c5ebf..00000000000 --- a/src/renderer/src/lib/agent-prompt-sent-telemetry.test.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const mockTrack = vi.hoisted(() => vi.fn()) - -vi.mock('@/lib/telemetry', () => ({ - track: mockTrack -})) - -import { trackFoldedStartupPromptSent } from './agent-prompt-sent-telemetry' - -describe('trackFoldedStartupPromptSent', () => { - beforeEach(() => { - vi.clearAllMocks() - }) - - it('tracks a folded startup prompt once with bounded telemetry', () => { - trackFoldedStartupPromptSent({ - prompt: 'fix the spinner', - startupPlan: { - followupPrompt: null, - draftPrompt: null - }, - startupQueued: true, - promptTelemetry: { - agent_kind: 'codex', - launch_source: 'new_workspace_composer', - request_kind: 'new' - } - }) - - expect(mockTrack).toHaveBeenCalledOnce() - expect(mockTrack).toHaveBeenCalledWith('agent_prompt_sent', { - agent_kind: 'codex', - launch_source: 'new_workspace_composer', - request_kind: 'new' - }) - }) - - it('does not track empty prompts, follow-up prompts, drafts, or bare shells', () => { - const promptTelemetry = { - agent_kind: 'codex', - launch_source: 'new_workspace_composer', - request_kind: 'new' - } as const - - trackFoldedStartupPromptSent({ - prompt: ' ', - startupPlan: { followupPrompt: null, draftPrompt: null }, - startupQueued: true, - promptTelemetry - }) - trackFoldedStartupPromptSent({ - prompt: 'fix the spinner', - startupPlan: { followupPrompt: 'fix the spinner', draftPrompt: null }, - startupQueued: true, - promptTelemetry - }) - trackFoldedStartupPromptSent({ - prompt: 'https://example.test/issue/1', - startupPlan: { followupPrompt: null, draftPrompt: 'https://example.test/issue/1' }, - startupQueued: true, - promptTelemetry - }) - trackFoldedStartupPromptSent({ - prompt: 'fix the spinner', - startupPlan: { followupPrompt: null, draftPrompt: null }, - startupQueued: true, - promptTelemetry: null - }) - trackFoldedStartupPromptSent({ - prompt: 'fix the spinner', - startupPlan: { followupPrompt: null, draftPrompt: null }, - startupQueued: false, - promptTelemetry - }) - - expect(mockTrack).not.toHaveBeenCalled() - }) -}) diff --git a/src/renderer/src/lib/agent-prompt-sent-telemetry.ts b/src/renderer/src/lib/agent-prompt-sent-telemetry.ts deleted file mode 100644 index c49c26e435f..00000000000 --- a/src/renderer/src/lib/agent-prompt-sent-telemetry.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { track } from '@/lib/telemetry' -import type { AgentStartupPlan } from '@/lib/tui-agent-startup' -import type { EventProps } from '../../../shared/telemetry-events' - -export function trackFoldedStartupPromptSent(args: { - prompt: string - startupPlan: Pick - promptTelemetry: EventProps<'agent_prompt_sent'> | null - startupQueued: boolean -}): void { - const { prompt, startupPlan, promptTelemetry, startupQueued } = args - if (!startupQueued) { - return - } - if (!promptTelemetry) { - return - } - if (prompt.trim().length === 0) { - return - } - if (startupPlan.followupPrompt !== null || startupPlan.draftPrompt) { - return - } - track('agent_prompt_sent', promptTelemetry) -} diff --git a/src/renderer/src/lib/launch-agent-in-new-tab.test.ts b/src/renderer/src/lib/launch-agent-in-new-tab.test.ts index 20696d9c99a..f89c9375f97 100644 --- a/src/renderer/src/lib/launch-agent-in-new-tab.test.ts +++ b/src/renderer/src/lib/launch-agent-in-new-tab.test.ts @@ -89,7 +89,7 @@ describe('launchAgentInNewTab', () => { ) }) - it('tracks one prompt-sent event for argv prompt launches', async () => { + it('does not track prompt-sent for argv prompt launches', async () => { const { launchAgentInNewTab } = await import('./launch-agent-in-new-tab') launchAgentInNewTab({ @@ -99,12 +99,7 @@ describe('launchAgentInNewTab', () => { launchSource: 'onboarding' }) - expect(mockTrack).toHaveBeenCalledTimes(1) - expect(mockTrack).toHaveBeenCalledWith('agent_prompt_sent', { - agent_kind: 'codex', - launch_source: 'onboarding', - request_kind: 'new' - }) + expect(mockTrack).not.toHaveBeenCalledWith('agent_prompt_sent', expect.anything()) }) it('does not track prompt-sent for draft launches', async () => { @@ -152,12 +147,7 @@ describe('launchAgentInNewTab', () => { prompt: 'large generated prompt', agentType: 'command-code' }) - expect(mockTrack).toHaveBeenCalledTimes(1) - expect(mockTrack).toHaveBeenCalledWith('agent_prompt_sent', { - agent_kind: 'command-code', - launch_source: 'tab_bar_quick_launch', - request_kind: 'new' - }) + expect(mockTrack).not.toHaveBeenCalledWith('agent_prompt_sent', expect.anything()) }) it('does not track prompt-sent when submit-after-ready delivery fails', async () => { diff --git a/src/renderer/src/lib/launch-agent-in-new-tab.ts b/src/renderer/src/lib/launch-agent-in-new-tab.ts index fbec392e273..0b4f084c2e6 100644 --- a/src/renderer/src/lib/launch-agent-in-new-tab.ts +++ b/src/renderer/src/lib/launch-agent-in-new-tab.ts @@ -12,7 +12,7 @@ import { pasteDraftWhenAgentReady } from '@/lib/agent-paste-draft' import { TUI_AGENT_CONFIG } from '../../../shared/tui-agent-config' import { makePaneKey } from '../../../shared/stable-pane-id' import type { TuiAgent } from '../../../shared/types' -import type { EventProps, LaunchSource } from '../../../shared/telemetry-events' +import type { LaunchSource } from '../../../shared/telemetry-events' export type LaunchAgentInNewTabArgs = { agent: TuiAgent @@ -93,12 +93,6 @@ export function launchAgentInNewTab(args: LaunchAgentInNewTabArgs): LaunchAgentI const trimmedPrompt = prompt?.trim() ?? '' const hasPrompt = trimmedPrompt.length > 0 const isFollowupPath = TUI_AGENT_CONFIG[agent].promptInjectionMode === 'stdin-after-start' - const promptTelemetry: EventProps<'agent_prompt_sent'> = { - agent_kind: tuiAgentToAgentKind(agent), - launch_source: launchSource ?? 'tab_bar_quick_launch', - request_kind: 'new' - } - // Why: argv/flag agents fold the prompt into the launch command and // auto-submit — keeping behavior consistent with the composer/tab-bar `+` // mental model, where the prompt is "the first turn the user sent". @@ -194,15 +188,6 @@ export function launchAgentInNewTab(args: LaunchAgentInNewTabArgs): LaunchAgentI request_kind: 'new' } }) - if ( - hasPrompt && - promptDelivery === 'auto-submit' && - startupPlan.followupPrompt === null && - pasteDraftAfterLaunch === null - ) { - track('agent_prompt_sent', promptTelemetry) - } - // Why: schedule the bracketed-paste-after-ready follow-up immediately after // the startup command is queued. Fire-and-forget so callers keep their // synchronous `{ tabId, startupPlan }` signature. The helper short-circuits @@ -247,9 +232,6 @@ export function launchAgentInNewTab(args: LaunchAgentInNewTabArgs): LaunchAgentI } }).then((delivered) => { if (delivered) { - if (submitPastedPrompt) { - track('agent_prompt_sent', promptTelemetry) - } if (agent === 'command-code' && submitPastedPrompt) { // Why: Command Code has no prompt-submit hook; when Orca submits a // generated prompt after readiness, seed working at delivery time. diff --git a/src/renderer/src/lib/new-workspace.test.ts b/src/renderer/src/lib/new-workspace.test.ts index 89c7eabbf14..59812d340e9 100644 --- a/src/renderer/src/lib/new-workspace.test.ts +++ b/src/renderer/src/lib/new-workspace.test.ts @@ -168,7 +168,7 @@ describe('isGitLabIssueUrl', () => { }) }) -describe('ensureAgentStartupInTerminal prompt telemetry', () => { +describe('ensureAgentStartupInTerminal prompt delivery', () => { beforeEach(() => { vi.clearAllMocks() store.settings = {} @@ -183,7 +183,7 @@ describe('ensureAgentStartupInTerminal prompt telemetry', () => { mockPasteDraftWhenAgentReady.mockResolvedValue(true) }) - it('tracks after a follow-up prompt is accepted by the terminal runtime', async () => { + it('sends a follow-up prompt through the terminal runtime without renderer telemetry', async () => { await ensureAgentStartupInTerminal({ worktreeId: 'wt-1', startup: { @@ -191,20 +191,11 @@ describe('ensureAgentStartupInTerminal prompt telemetry', () => { launchCommand: 'aider', expectedProcess: 'aider', followupPrompt: 'fix the spinner' - }, - promptTelemetry: { - agent_kind: 'aider', - launch_source: 'new_workspace_composer', - request_kind: 'new' } }) expect(mockSendRuntimePtyInputVerified).toHaveBeenCalledWith({}, 'pty-1', 'fix the spinner\r') - expect(mockTrack).toHaveBeenCalledWith('agent_prompt_sent', { - agent_kind: 'aider', - launch_source: 'new_workspace_composer', - request_kind: 'new' - }) + expect(mockTrack).not.toHaveBeenCalledWith('agent_prompt_sent', expect.anything()) }) it('does not track when follow-up prompt delivery is rejected by the terminal runtime', async () => { @@ -217,11 +208,6 @@ describe('ensureAgentStartupInTerminal prompt telemetry', () => { launchCommand: 'aider', expectedProcess: 'aider', followupPrompt: 'fix the spinner' - }, - promptTelemetry: { - agent_kind: 'aider', - launch_source: 'new_workspace_composer', - request_kind: 'new' } }) @@ -239,11 +225,6 @@ describe('ensureAgentStartupInTerminal prompt telemetry', () => { launchCommand: 'aider', expectedProcess: 'aider', followupPrompt: 'fix the spinner' - }, - promptTelemetry: { - agent_kind: 'aider', - launch_source: 'new_workspace_composer', - request_kind: 'new' } }) ).resolves.toBeUndefined() @@ -260,11 +241,6 @@ describe('ensureAgentStartupInTerminal prompt telemetry', () => { expectedProcess: 'claude', followupPrompt: null, draftPrompt: 'review this before sending' - }, - promptTelemetry: { - agent_kind: 'claude-code', - launch_source: 'new_workspace_composer', - request_kind: 'new' } }) diff --git a/src/renderer/src/lib/new-workspace.ts b/src/renderer/src/lib/new-workspace.ts index d8c6fe604ae..6cc91987c96 100644 --- a/src/renderer/src/lib/new-workspace.ts +++ b/src/renderer/src/lib/new-workspace.ts @@ -4,10 +4,8 @@ import { inspectRuntimeTerminalProcess, sendRuntimePtyInputVerified } from '@/runtime/runtime-terminal-inspection' -import { track } from '@/lib/telemetry' import type { AgentStartupPlan } from '@/lib/tui-agent-startup' import { isShellProcess } from '@/lib/tui-agent-startup' -import type { EventProps } from '../../../shared/telemetry-events' import type { OrcaHooks, TaskViewPresetId } from '../../../shared/types' import { resolveHookCommandSourcePolicy } from '../../../shared/hook-command-source-policy' import { isExpectedAgentProcess } from '../../../shared/agent-process-recognition' @@ -210,9 +208,8 @@ export function getWorkspaceSeedName(args: { export async function ensureAgentStartupInTerminal(args: { worktreeId: string startup: AgentStartupPlan - promptTelemetry?: EventProps<'agent_prompt_sent'> }): Promise { - const { worktreeId, startup, promptTelemetry } = args + const { worktreeId, startup } = args const draftPrompt = startup.draftPrompt ?? null if (startup.followupPrompt === null && draftPrompt === null) { return @@ -247,12 +244,7 @@ export async function ensureAgentStartupInTerminal(args: { // session and submitted. Wait until the agent owns the PTY before writing. if (startup.followupPrompt) { await waitForAgentForeground(ptyId, startup.expectedProcess) - const sent = await sendFollowupPrompt(ptyId, startup.followupPrompt) - if (sent) { - if (promptTelemetry) { - track('agent_prompt_sent', promptTelemetry) - } - } + await sendFollowupPrompt(ptyId, startup.followupPrompt) } // Why: draftPrompt uses bracketed-paste so the URL lands atomically in the diff --git a/src/renderer/src/runtime/runtime-terminal-inspection.test.ts b/src/renderer/src/runtime/runtime-terminal-inspection.test.ts index 3cfeb26a1bc..e3dd83c4601 100644 --- a/src/renderer/src/runtime/runtime-terminal-inspection.test.ts +++ b/src/renderer/src/runtime/runtime-terminal-inspection.test.ts @@ -143,12 +143,12 @@ describe('runtime terminal owner routing', () => { expect(localWrite).not.toHaveBeenCalled() }) - it('falls back to fire-and-forget writes when local acceptance cannot be verified', async () => { + it('reports success after fallback fire-and-forget writes when local acceptance cannot be verified', async () => { localWriteAccepted.mockResolvedValue(false) await expect( sendRuntimePtyInputVerified({ activeRuntimeEnvironmentId: null }, 'local-pty', 'x') - ).resolves.toBe(false) + ).resolves.toBe(true) expect(localWriteAccepted).toHaveBeenCalledWith('local-pty', 'x') expect(localWrite).toHaveBeenCalledWith('local-pty', 'x') diff --git a/src/renderer/src/runtime/runtime-terminal-inspection.ts b/src/renderer/src/runtime/runtime-terminal-inspection.ts index 44348f19973..b292e5dad1b 100644 --- a/src/renderer/src/runtime/runtime-terminal-inspection.ts +++ b/src/renderer/src/runtime/runtime-terminal-inspection.ts @@ -110,6 +110,9 @@ export async function sendRuntimePtyInputVerified( const accepted = await window.api.pty.writeAccepted(ptyId, data) if (!accepted) { window.api.pty.write(ptyId, data) + // Why: SSH/local fallback writes are fire-and-forget. Callers use this + // boolean to continue UX flow, while hook telemetry confirms real turns. + return true } return accepted } diff --git a/src/shared/agent-hook-listener.test.ts b/src/shared/agent-hook-listener.test.ts index 669da848010..03ce703c3d3 100644 --- a/src/shared/agent-hook-listener.test.ts +++ b/src/shared/agent-hook-listener.test.ts @@ -196,6 +196,52 @@ describe('shared agent-hook-listener', () => { toolName: 'shell_command', toolInput: 'pwd' }) + expect(tool?.hasExplicitPrompt).toBe(true) + expect(tool?.promptInteractionKey).toMatch(/^command-code-transcript-[a-f0-9]{12}-/) + + const directPrompt = normalizeHookPayload( + createHookListenerState(), + 'command-code', + { + paneKey: PANE_KEY, + payload: { + hook_event_name: 'PreToolUse', + prompt: 'Direct command prompt' + } + }, + 'production' + ) + expect(directPrompt?.hasExplicitPrompt).toBe(true) + + const directPromptWithTranscript = normalizeHookPayload( + createHookListenerState(), + 'command-code', + { + paneKey: PANE_KEY, + payload: { + hook_event_name: 'PreToolUse', + prompt: 'Run pwd and report it', + transcript_path: transcriptPath + } + }, + 'production' + ) + expect(directPromptWithTranscript?.hasExplicitPrompt).toBe(true) + expect(directPromptWithTranscript?.promptInteractionKey).toBe(tool?.promptInteractionKey) + + const statusMessage = normalizeHookPayload( + createHookListenerState(), + 'command-code', + { + paneKey: PANE_KEY, + payload: { + hook_event_name: 'PreToolUse', + message: 'Preparing tool call' + } + }, + 'production' + ) + expect(statusMessage?.hasExplicitPrompt).toBe(false) const done = normalizeHookPayload( state, @@ -219,6 +265,25 @@ describe('shared agent-hook-listener', () => { agentType: 'command-code', lastAssistantMessage: 'The output is /tmp/project.' }) + expect(done?.promptInteractionKey).toBe(tool?.promptInteractionKey) + + const cachedOnly = normalizeHookPayload( + state, + 'command-code', + { + paneKey: PANE_KEY, + tabId: 'tab-1', + worktreeId: 'wt', + env: 'production', + version: '1', + payload: { + hook_event_name: 'Stop' + } + }, + 'production' + ) + expect(cachedOnly?.payload.prompt).toBe('Run pwd and report it') + expect(cachedOnly?.hasExplicitPrompt).toBe(false) } finally { rmSync(tmpDir, { recursive: true, force: true }) } @@ -376,6 +441,7 @@ describe('shared agent-hook-listener', () => { prompt: 'Fix the failing test', agentType: 'antigravity' }) + expect(started?.hasExplicitPrompt).toBe(true) } finally { rmSync(tmpDir, { recursive: true, force: true }) } diff --git a/src/shared/agent-hook-listener.ts b/src/shared/agent-hook-listener.ts index e29f9d61e8c..619f50c1f56 100644 --- a/src/shared/agent-hook-listener.ts +++ b/src/shared/agent-hook-listener.ts @@ -11,7 +11,7 @@ // which pull `electron` — so it is safe to import from `src/relay/`. See // docs/design/agent-status-over-ssh.md §3 ("relay normalizes; Orca routes"). import type { IncomingMessage } from 'http' -import { randomUUID } from 'crypto' +import { createHash, randomUUID } from 'crypto' import { homedir } from 'os' import { chmodSync, @@ -141,6 +141,9 @@ export type AgentHookEventPayload = { /** True when this hook event carried prompt text directly, instead of using * the listener's cached prompt from an earlier event in the same pane. */ hasExplicitPrompt?: boolean + /** Stable per-turn key when a source exposes enough local hook context to + * distinguish duplicate hook delivery from a same-text prompt rerun. */ + promptInteractionKey?: string /** Raw agent hook event name, used by main-process transition guards. */ hookEventName?: string /** Claude tool-use identifier when the hook source exposes one. */ @@ -235,7 +238,21 @@ export function readRequestBody(req: IncomingMessage): Promise { // ─── Per-pane field caches + extractors ───────────────────────────── -function extractPromptText(hookPayload: Record): string { +type ExtractedPromptText = { + text: string + source: + | 'prompt' + | 'user_prompt' + | 'userPrompt' + | 'initial_prompt' + | 'initialPrompt' + | 'user_message' + | 'message' + | 'role_user_text' + | null +} + +function extractPromptText(hookPayload: Record): ExtractedPromptText { const candidateKeys = [ 'prompt', 'user_prompt', @@ -250,7 +267,7 @@ function extractPromptText(hookPayload: Record): string { if (typeof value === 'string' && value.trim().length > 0) { // Why: trim so prompts match what readStringField produces elsewhere — // surrounding whitespace would otherwise leak into UI and caches. - return value.trim() + return { text: value.trim(), source: key as Exclude } } } // Why: OpenCode's plugin sends MessagePart events with { role, text }. When @@ -259,10 +276,10 @@ function extractPromptText(hookPayload: Record): string { if (hookPayload.role === 'user' && typeof hookPayload.text === 'string') { const trimmed = hookPayload.text.trim() if (trimmed.length > 0) { - return trimmed + return { text: trimmed, source: 'role_user_text' } } } - return '' + return { text: '', source: null } } function stripGrokUserQueryWrapper(promptText: string): string { @@ -664,6 +681,72 @@ function extractCommandCodeUserPromptFromLine(line: string): string | undefined return record.role === 'user' ? extractAssistantContentText(record.content) : undefined } +function hashInteractionKeyPart(value: string): string { + return createHash('sha256').update(value).digest('hex').slice(0, 12) +} + +function readLastCommandCodeUserPromptEntryFromTranscript( + transcriptPath: unknown +): { text: string; interactionKey: string } | undefined { + if (typeof transcriptPath !== 'string' || transcriptPath.length === 0) { + return undefined + } + try { + const stats = statSync(transcriptPath) + const size = stats.size + if (size <= 0) { + return undefined + } + const bytesToRead = Math.min(size, TRANSCRIPT_MAX_SCAN_BYTES) + const position = size - bytesToRead + const fd = openSync(transcriptPath, 'r') + try { + const buffer = Buffer.alloc(bytesToRead) + let filled = 0 + while (filled < bytesToRead) { + const n = readSync(fd, buffer, filled, bytesToRead - filled, position + filled) + if (n === 0) { + break + } + filled += n + } + let text = buffer.subarray(0, filled).toString('utf8') + let textBasePosition = position + if (position > 0) { + const firstNewline = text.indexOf('\n') + textBasePosition += firstNewline + 1 + text = firstNewline === -1 ? '' : text.slice(firstNewline + 1) + } + let lastPrompt: string | undefined + let lastPromptOffset = 0 + let lineStart = 0 + for (const line of text.split('\n')) { + const prompt = extractCommandCodeUserPromptFromLine(line.trim()) + if (prompt !== undefined) { + lastPrompt = prompt + lastPromptOffset = textBasePosition + Buffer.byteLength(text.slice(0, lineStart), 'utf8') + } + lineStart += line.length + 1 + } + return lastPrompt + ? { + text: lastPrompt, + interactionKey: [ + 'command-code-transcript', + hashInteractionKeyPart(transcriptPath), + String(lastPromptOffset), + hashInteractionKeyPart(lastPrompt) + ].join('-') + } + : undefined + } finally { + closeSync(fd) + } + } catch { + return undefined + } +} + function extractCommandCodeAssistantTextFromLine(line: string): string | undefined { let entry: unknown try { @@ -698,13 +781,6 @@ function extractCommandCodeAssistantTextFromLine(line: string): string | undefin return extractAssistantContentText(content) } -function readLastCommandCodeUserPromptFromTranscript(transcriptPath: unknown): string | undefined { - if (typeof transcriptPath !== 'string' || transcriptPath.length === 0) { - return undefined - } - return readLastTextFromTranscriptOnce(transcriptPath, extractCommandCodeUserPromptFromLine) -} - function readLastCommandCodeAssistantFromTranscript(transcriptPath: unknown): string | undefined { if (typeof transcriptPath !== 'string' || transcriptPath.length === 0) { return undefined @@ -1665,6 +1741,52 @@ function isNewTurnEvent(source: AgentHookSource, eventName: unknown): boolean { } } +function hasExplicitUserPrompt( + source: AgentHookSource, + eventName: unknown, + extractedPrompt: ExtractedPromptText, + resolvedPromptText: string, + hasTranscriptPromptEvidence = false +): boolean { + if ( + source === 'command-code' && + (eventName === 'PreToolUse' || eventName === 'Stop') && + (extractedPrompt.source !== 'message' || hasTranscriptPromptEvidence) && + resolvedPromptText.trim().length > 0 + ) { + // Why: Command Code exposes the submitted prompt through its transcript + // rather than direct hook fields. Treat the transcript-backed prompt as + // explicit so hook telemetry covers real Command Code turns. + return true + } + if ( + source === 'antigravity' && + isNewTurnEvent(source, eventName) && + resolvedPromptText.trim().length > 0 + ) { + return true + } + if (extractedPrompt.source === 'role_user_text') { + return source === 'opencode' && eventName === 'MessagePart' + } + if (extractedPrompt.text.length === 0) { + return false + } + // Why: bare `message` fields often contain permission or status copy. They + // may update visible status prompts, but they are not proof of user submit. + if (extractedPrompt.source === 'message') { + return false + } + if ( + extractedPrompt.source === 'user_prompt' || + extractedPrompt.source === 'userPrompt' || + extractedPrompt.source === 'user_message' + ) { + return isNewTurnEvent(source, eventName) + } + return isNewTurnEvent(source, eventName) +} + function extractToolFields( source: AgentHookSource, eventName: unknown, @@ -2222,12 +2344,6 @@ function normalizeCommandCodeEvent( return null } - const effectivePrompt = - promptText || - readLastCommandCodeUserPromptFromTranscript( - hookPayload.transcript_path ?? hookPayload.transcriptPath - ) || - '' const snapshot = resolveToolState( state, paneKey, @@ -2238,7 +2354,7 @@ function normalizeCommandCodeEvent( return parseAgentStatusPayload( JSON.stringify({ state: stateName, - prompt: resolvePrompt(state, paneKey, effectivePrompt, { + prompt: resolvePrompt(state, paneKey, promptText, { resetOnNewTurn: isNewTurnEvent('command-code', eventName) }), agentType: 'command-code', @@ -2424,11 +2540,15 @@ export function normalizeHookPayload( const worktreeId = readStringField(record, 'worktreeId') const hookPayloadRecord = hookPayload as Record + let promptInteractionKey: string | undefined const eventName = readFirstString(record, ['hook_event_name', 'hookEventName', 'hook_type', 'hookType']) ?? hookPayloadRecord.hook_event_name ?? hookPayloadRecord.hookEventName - const promptText = extractPromptText(hookPayload as Record) + const extractedPrompt = extractPromptText(hookPayload as Record) + const promptText = extractedPrompt.text + let resolvedPromptText = promptText + let hasTranscriptPromptEvidence = false // Why: exhaustive switch so adding a source to AgentHookSource fails // typecheck here instead of silently routing through OpenCode's normalizer. let payload: ParsedAgentStatusPayload | null @@ -2443,9 +2563,25 @@ export function normalizeHookPayload( payload = normalizeGeminiEvent(state, eventName, promptText, paneKey, hookPayloadRecord) break case 'antigravity': + if (isNewTurnEvent('antigravity', eventName)) { + resolvedPromptText = + promptText || + readLastUserPromptFromTranscript( + readFirstString(hookPayloadRecord, ['transcriptPath', 'transcript_path']) + ) || + '' + } payload = normalizeAntigravityEvent(state, eventName, promptText, paneKey, hookPayloadRecord) break case 'opencode': + if (extractedPrompt.source === 'role_user_text') { + const messageId = readFirstString(hookPayloadRecord, [ + 'messageID', + 'messageId', + 'message_id' + ]) + promptInteractionKey = messageId ? `opencode-message-${messageId}` : undefined + } payload = normalizeOpenCodeEvent(state, eventName, promptText, paneKey, hookPayloadRecord) break case 'cursor': @@ -2475,7 +2611,24 @@ export function normalizeHookPayload( payload = normalizeDroidEvent(state, eventName, promptText, paneKey, hookPayloadRecord) break case 'command-code': - payload = normalizeCommandCodeEvent(state, eventName, promptText, paneKey, hookPayloadRecord) + { + const transcriptPrompt = readLastCommandCodeUserPromptEntryFromTranscript( + hookPayloadRecord.transcript_path ?? hookPayloadRecord.transcriptPath + ) + hasTranscriptPromptEvidence = transcriptPrompt !== undefined + promptInteractionKey = transcriptPrompt?.interactionKey + resolvedPromptText = transcriptPrompt?.text ?? '' + if (promptText && extractedPrompt.source !== 'message') { + resolvedPromptText = promptText + } + } + payload = normalizeCommandCodeEvent( + state, + eventName, + resolvedPromptText, + paneKey, + hookPayloadRecord + ) break case 'grok': payload = normalizeGrokEvent(state, eventName, promptText, paneKey, hookPayloadRecord) @@ -2503,7 +2656,14 @@ export function normalizeHookPayload( tabId, worktreeId, connectionId: null, - hasExplicitPrompt: promptText.length > 0, + hasExplicitPrompt: hasExplicitUserPrompt( + source, + eventName, + extractedPrompt, + resolvedPromptText, + hasTranscriptPromptEvidence + ), + promptInteractionKey, hookEventName: typeof eventName === 'string' ? eventName : undefined, toolUseId: readFirstString(hookPayloadRecord, ['tool_use_id', 'toolUseId']), toolAgentId: readFirstString(hookPayloadRecord, ['agent_id', 'agentId']), diff --git a/src/shared/agent-hook-relay.ts b/src/shared/agent-hook-relay.ts index fa2d696ddf7..12cc2144020 100644 --- a/src/shared/agent-hook-relay.ts +++ b/src/shared/agent-hook-relay.ts @@ -60,6 +60,9 @@ export type AgentHookRelayEnvelope = { /** Preserved from the relay-side normalized hook event so Orca can * distinguish a true same-prompt retry from a cached-prompt tool ping. */ hasExplicitPrompt?: boolean + /** Optional stable per-turn key from the relay-side listener. Used only for + * in-memory dedupe; never included in product telemetry payloads. */ + promptInteractionKey?: string /** Hook discriminator preserved for main-process transition rules. */ hookEventName?: string /** Claude tool execution id, when the source hook provides one. */ diff --git a/src/shared/telemetry-events.test.ts b/src/shared/telemetry-events.test.ts index 8d0aa7621da..37972dd915f 100644 --- a/src/shared/telemetry-events.test.ts +++ b/src/shared/telemetry-events.test.ts @@ -112,11 +112,11 @@ describe('agent_started schema', () => { }) describe('agent_prompt_sent schema', () => { - it('accepts a bounded prompt-send payload with cohort context', () => { + it('accepts a hook-confirmed prompt-send payload with cohort context', () => { const parsed = eventSchemas.agent_prompt_sent.safeParse({ agent_kind: 'codex', - launch_source: 'onboarding', - request_kind: 'new', + launch_source: 'unknown', + request_kind: 'followup', nth_repo_added: 1 }) expect(parsed.success).toBe(true) @@ -125,8 +125,8 @@ describe('agent_prompt_sent schema', () => { it('rejects prompt text via .strict()', () => { const parsed = eventSchemas.agent_prompt_sent.safeParse({ agent_kind: 'claude-code', - launch_source: 'new_workspace_composer', - request_kind: 'new', + launch_source: 'unknown', + request_kind: 'followup', prompt: 'please inspect /Users/alice/private-repo' }) expect(parsed.success).toBe(false)