From a64876ccccd7eaba0a3f167d7a492f69d3945005 Mon Sep 17 00:00:00 2001 From: brennanb2025 Date: Fri, 29 May 2026 00:27:35 -0700 Subject: [PATCH] Add agent prompt sent telemetry Co-authored-by: Orca --- docs/reference/agent-prompt-sent-telemetry.md | 74 +++++++++ docs/reference/telemetry-availability.md | 4 +- src/main/telemetry/validator.test.ts | 10 ++ src/renderer/src/hooks/useComposerState.ts | 26 ++- .../src/lib/agent-paste-draft.test.ts | 69 ++++++-- src/renderer/src/lib/agent-paste-draft.ts | 16 +- .../lib/agent-prompt-sent-telemetry.test.ts | 79 +++++++++ .../src/lib/agent-prompt-sent-telemetry.ts | 25 +++ .../src/lib/launch-agent-in-new-tab.test.ts | 55 +++++- .../src/lib/launch-agent-in-new-tab.ts | 18 +- src/renderer/src/lib/new-workspace.test.ts | 156 +++++++++++++++++- src/renderer/src/lib/new-workspace.ts | 22 ++- .../runtime-terminal-inspection.test.ts | 72 +++++++- .../runtime/runtime-terminal-inspection.ts | 17 +- src/shared/telemetry-events.test.ts | 22 +++ src/shared/telemetry-events.ts | 10 ++ 16 files changed, 639 insertions(+), 36 deletions(-) create mode 100644 docs/reference/agent-prompt-sent-telemetry.md create mode 100644 src/renderer/src/lib/agent-prompt-sent-telemetry.test.ts create mode 100644 src/renderer/src/lib/agent-prompt-sent-telemetry.ts diff --git a/docs/reference/agent-prompt-sent-telemetry.md b/docs/reference/agent-prompt-sent-telemetry.md new file mode 100644 index 00000000000..5ad8c091f52 --- /dev/null +++ b/docs/reference/agent-prompt-sent-telemetry.md @@ -0,0 +1,74 @@ +# Agent Prompt Sent Telemetry + +This change adds `agent_prompt_sent` so onboarding and retention analysis can tell the difference between an agent being started and Orca initiating or completing a covered prompt submission to that agent. + +## Why this exists + +`agent_started` only proves Orca launched an agent-backed PTY with telemetry attached. It does not prove the user reached the product value moment of asking the agent to do work. + +For onboarding completion, that distinction matters. A user who creates a workspace and starts an agent may still drop before sending a prompt. `agent_prompt_sent` gives us a cleaner signal for "workspace plus real agent intent." It is not proof that the model processed the prompt; folded argv/flag prompts are recorded after local startup is queued, while post-ready terminal input is recorded only after the write path accepts it. + +## Event contract + +Event: `agent_prompt_sent` + +Allowed properties: + +| Property | Meaning | +| --- | --- | +| `agent_kind` | Bounded agent family, using the same mapping as `agent_started`. | +| `launch_source` | Bounded source for where the agent prompt flow came from, such as onboarding or quick launch. | +| `request_kind` | Whether this is a new request or a supported bounded request type. | +| `nth_repo_added` | Optional repo-onboarding cohort marker when available. | + +The event does not include prompt text, file paths, repository names, branch names, raw command text, or raw error details. + +## When it fires + +The event records only non-empty, non-draft prompt/message paths that Orca submits or queues for local agent startup: + +- A prompt folded into an agent launch command after local startup is queued. +- A prompt sent after the agent is ready through the workspace startup path and verified terminal input accepts it. +- A post-ready pasted prompt that Orca submits on behalf of the user after verified terminal input accepts it. + +The implementation covers renderer paths that launch an agent and submit the user's first prompt through Orca's launch/startup helpers: + +- `src/renderer/src/hooks/useComposerState.ts` +- `src/renderer/src/lib/launch-agent-in-new-tab.ts` +- `src/renderer/src/lib/new-workspace.ts` + +That includes onboarding/new workspace launches, tab-bar quick launch, and other `launchAgentInNewTab` submit-after-ready surfaces such as notes send, conflict resolution, source-control recovery, and task pages. + +For local PTYs, "accepted" means main accepted the write through `pty:writeAccepted`. For remote web runtime PTYs, it means the runtime `terminal.send` RPC returned `accepted: true`. SSH relay PTYs cannot acknowledge writes yet, so these paths still attempt the prompt write but intentionally do not emit `agent_prompt_sent` when acceptance cannot be known. + +## When it does not fire + +The event intentionally does not fire for: + +- Bare agent starts with no user prompt. +- Draft-prefill flows where Orca places text in the composer but does not submit it. +- Empty or whitespace-only prompts. +- Incidental terminal input outside the agent prompt submission paths. + +## How to use it + +For onboarding retention, prefer splitting users like this: + +| Segment | Interpretation | +| --- | --- | +| `workspace_created`, no `agent_started` | Workspace created, but no agent launched. | +| `agent_started`, no `agent_prompt_sent` | Agent process launched, but no clear user message. | +| `agent_prompt_sent` | User submitted a real agent request. Stronger activation signal. | + +This should make the "workspace/setup action only" and "agent started only" buckets less ambiguous. It also gives us a better way to identify users who technically completed onboarding but never reached a meaningful agent interaction. + +## Validation + +The worker validated the change with: + +- `pnpm test src/shared/telemetry-events.test.ts src/main/telemetry/validator.test.ts src/renderer/src/lib/agent-prompt-sent-telemetry.test.ts src/renderer/src/lib/agent-paste-draft.test.ts src/renderer/src/lib/launch-agent-in-new-tab.test.ts src/renderer/src/lib/new-workspace.test.ts src/renderer/src/runtime/runtime-terminal-inspection.test.ts` +- `pnpm run typecheck:web` +- `pnpm run typecheck:node` +- `git diff --check` + +PostHog first-seen validation remains release-dependent. After this ships, update `docs/reference/telemetry-availability.md` with the first observed production timestamp for `agent_prompt_sent`. diff --git a/docs/reference/telemetry-availability.md b/docs/reference/telemetry-availability.md index 9c3091b9ad9..86be5373d4a 100644 --- a/docs/reference/telemetry-availability.md +++ b/docs/reference/telemetry-availability.md @@ -39,13 +39,14 @@ D1+/D3+/D7+ retention means the user fired `app_opened` at least once after 24/7 - `source = 'unknown'` is not a real product surface. It means the caller omitted a source or the value failed schema validation. - `workspace_created` means create-worktree IPC succeeded. It is not a general "usable workspace exists" or "workspace revealed" marker. - `agent_started` means PTY spawn succeeded with agent telemetry attached. It is not first-repo activation and does not prove the user sent a prompt. +- `agent_prompt_sent` means Orca submitted a non-draft prompt/message through an agent launch or post-ready agent input path. It excludes agent auto-start, bare shells, and draft prefill. - Workspace-outcome joins are native Electron coverage unless the query explicitly proves remote/web instrumentation. Remote runtime and web paths can bypass native repo/worktree telemetry, so do not interpret missing workspace outcome rows as product drop-off for SSH, remote, or web users. ## Rollouts ### 2026-05-08 - Repo Cohort Property -Scope: `nth_repo_added` on repo/activation/retention events. Current schemas declare it on `app_opened`, `repo_added`, `add_repo_setup_step_action`, `add_repo_existing_workspaces_detected`, `workspace_created`, `workspace_create_failed`, `setup_script_prompt_shown`, `setup_script_prompt_action`, `agent_started`, and `agent_error`. The original rollout covered `app_opened`, `repo_added`, `add_repo_setup_step_action`, `workspace_created`, `workspace_create_failed`, `agent_started`, and `agent_error`; later events have their own first-seen timestamps below. +Scope: `nth_repo_added` on repo/activation/retention events. Current schemas declare it on `app_opened`, `repo_added`, `add_repo_setup_step_action`, `add_repo_existing_workspaces_detected`, `workspace_created`, `workspace_create_failed`, `setup_script_prompt_shown`, `setup_script_prompt_action`, `agent_started`, `agent_prompt_sent`, and `agent_error`. The original rollout covered `app_opened`, `repo_added`, `add_repo_setup_step_action`, `workspace_created`, `workspace_create_failed`, `agent_started`, and `agent_error`; later events have their own first-seen timestamps below. | Field | Value | | --- | --- | @@ -65,6 +66,7 @@ PostHog evidence checked at `2026-05-23T23:34:32Z`: | `app_opened` | `2026-05-08T18:40:00.354Z` (`1.3.42-rc.1`) | | `workspace_created` | `2026-05-08T19:15:06.913Z` | | `agent_started` | `2026-05-08T19:15:07.130Z` | +| `agent_prompt_sent` | `TBD` | | `repo_added` | `2026-05-08T20:01:06.364Z` (`1.3.42`) | | `add_repo_setup_step_action` | `2026-05-08T20:01:20.897Z` | | `workspace_create_failed` | `2026-05-08T23:04:32.315Z` | diff --git a/src/main/telemetry/validator.test.ts b/src/main/telemetry/validator.test.ts index 934b2a09055..996e3e3d5ee 100644 --- a/src/main/telemetry/validator.test.ts +++ b/src/main/telemetry/validator.test.ts @@ -31,6 +31,16 @@ describe('validate', () => { expect(result.ok).toBe(true) }) + it('accepts a well-formed agent_prompt_sent payload', () => { + const result = validate('agent_prompt_sent', { + agent_kind: 'claude-code', + launch_source: 'new_workspace_composer', + request_kind: 'new', + nth_repo_added: 1 + }) + expect(result.ok).toBe(true) + }) + it('drops unknown event names', () => { const result = validate('not_a_real_event' as never, {}) expect(result.ok).toBe(false) diff --git a/src/renderer/src/hooks/useComposerState.ts b/src/renderer/src/hooks/useComposerState.ts index 08aa47de717..2a6a9fdfefb 100644 --- a/src/renderer/src/hooks/useComposerState.ts +++ b/src/renderer/src/hooks/useComposerState.ts @@ -19,6 +19,8 @@ import { filterEnabledTuiAgents, isTuiAgentEnabled } from '../../../shared/tui-a import { tuiAgentToAgentKind } from '@/lib/telemetry' import { isGitRepoKind } from '../../../shared/repo-kind' import { callRuntimeRpc, getActiveRuntimeTarget } from '@/runtime/runtime-rpc-client' +import type { EventProps } from '../../../shared/telemetry-events' +import { trackFoldedStartupPromptSent } from '@/lib/agent-prompt-sent-telemetry' import type { GitHubWorkItem, GitPushTarget, @@ -1870,7 +1872,8 @@ export function useComposerState(options: UseComposerStateOptions): UseComposerS launch_source: telemetrySource === 'onboarding' ? 'onboarding' : 'new_workspace_composer', request_kind: 'new' } - activateAndRevealWorktree(worktree.id, { + const composerPromptTelemetry: EventProps<'agent_prompt_sent'> = composerTelemetry + const activation = activateAndRevealWorktree(worktree.id, { sidebarRevealBehavior: 'auto', setup: result.setup, issueCommand, @@ -1893,9 +1896,16 @@ export function useComposerState(options: UseComposerStateOptions): UseComposerS : {}) }) if (startupPlan) { + trackFoldedStartupPromptSent({ + prompt: submitStartupPrompt, + startupPlan, + promptTelemetry: composerPromptTelemetry, + startupQueued: activation !== false && activation.primaryTabId !== null + }) void ensureAgentStartupInTerminal({ worktreeId: worktree.id, - startup: startupPlan + startup: startupPlan, + promptTelemetry: composerPromptTelemetry }) } setSidebarOpen(true) @@ -2138,7 +2148,8 @@ export function useComposerState(options: UseComposerStateOptions): UseComposerS telemetrySource === 'onboarding' ? 'onboarding' : 'new_workspace_composer', request_kind: 'new' } - activateAndRevealWorktree(worktree.id, { + const quickPromptTelemetry: EventProps<'agent_prompt_sent'> | null = quickTelemetry + const activation = activateAndRevealWorktree(worktree.id, { sidebarRevealBehavior: 'auto', setup: result.setup, ...(startupPlan @@ -2160,9 +2171,16 @@ export function useComposerState(options: UseComposerStateOptions): UseComposerS : {}) }) if (startupPlan) { + trackFoldedStartupPromptSent({ + prompt: quickPrompt, + startupPlan, + promptTelemetry: quickPromptTelemetry, + startupQueued: activation !== false && activation.primaryTabId !== null + }) void ensureAgentStartupInTerminal({ worktreeId: worktree.id, - startup: startupPlan + startup: startupPlan, + ...(quickPromptTelemetry ? { promptTelemetry: quickPromptTelemetry } : {}) }) } setSidebarOpen(true) diff --git a/src/renderer/src/lib/agent-paste-draft.test.ts b/src/renderer/src/lib/agent-paste-draft.test.ts index 36c0cd510fb..efd4960e501 100644 --- a/src/renderer/src/lib/agent-paste-draft.test.ts +++ b/src/renderer/src/lib/agent-paste-draft.test.ts @@ -10,7 +10,7 @@ const testState = vi.hoisted(() => ({ unsubscribe: vi.fn(), subscribeToPtyData: vi.fn(), isRemoteRuntimePtyId: vi.fn(), - sendRuntimePtyInput: vi.fn(), + sendRuntimePtyInputVerified: vi.fn(), subscribeToRuntimeTerminalData: vi.fn() })) @@ -26,7 +26,7 @@ vi.mock('@/components/terminal-pane/pty-dispatcher', () => ({ vi.mock('@/runtime/runtime-terminal-inspection', () => ({ isRemoteRuntimePtyId: testState.isRemoteRuntimePtyId, - sendRuntimePtyInput: testState.sendRuntimePtyInput + sendRuntimePtyInputVerified: testState.sendRuntimePtyInputVerified })) vi.mock('@/runtime/runtime-terminal-stream', () => ({ @@ -58,7 +58,8 @@ describe('pasteDraftWhenAgentReady', () => { ) testState.isRemoteRuntimePtyId.mockReset() testState.isRemoteRuntimePtyId.mockReturnValue(false) - testState.sendRuntimePtyInput.mockReset() + testState.sendRuntimePtyInputVerified.mockReset() + testState.sendRuntimePtyInputVerified.mockResolvedValue(true) testState.subscribeToRuntimeTerminalData.mockReset() }) @@ -77,16 +78,20 @@ describe('pasteDraftWhenAgentReady', () => { testState.ptyObserver?.(CODEX_COMPOSER_PROMPT_RENDER) await flushMicrotasks() - expect(testState.sendRuntimePtyInput).not.toHaveBeenCalled() + expect(testState.sendRuntimePtyInputVerified).not.toHaveBeenCalled() testState.ptyObserver?.(DECSET_BRACKETED_PASTE) await flushMicrotasks() - expect(testState.sendRuntimePtyInput).not.toHaveBeenCalled() + expect(testState.sendRuntimePtyInputVerified).not.toHaveBeenCalled() testState.ptyObserver?.(CODEX_COMPOSER_PROMPT_RENDER) await expect(promise).resolves.toBe(true) - expect(testState.sendRuntimePtyInput).toHaveBeenCalledWith({}, 'pty-1', PASTED_ISSUE_URL) + expect(testState.sendRuntimePtyInputVerified).toHaveBeenCalledWith( + {}, + 'pty-1', + PASTED_ISSUE_URL + ) expect(vi.getTimerCount()).toBe(0) }) @@ -103,7 +108,11 @@ describe('pasteDraftWhenAgentReady', () => { ) await expect(promise).resolves.toBe(true) - expect(testState.sendRuntimePtyInput).toHaveBeenCalledWith({}, 'pty-1', PASTED_ISSUE_URL) + expect(testState.sendRuntimePtyInputVerified).toHaveBeenCalledWith( + {}, + 'pty-1', + PASTED_ISSUE_URL + ) }) it('keeps the render-quiet wait for agents without the Codex ready signal', async () => { @@ -116,15 +125,19 @@ describe('pasteDraftWhenAgentReady', () => { testState.ptyObserver?.(DECSET_BRACKETED_PASTE) await flushMicrotasks() - expect(testState.sendRuntimePtyInput).not.toHaveBeenCalled() + expect(testState.sendRuntimePtyInputVerified).not.toHaveBeenCalled() await vi.advanceTimersByTimeAsync(1499) - expect(testState.sendRuntimePtyInput).not.toHaveBeenCalled() + expect(testState.sendRuntimePtyInputVerified).not.toHaveBeenCalled() await vi.advanceTimersByTimeAsync(1) await expect(promise).resolves.toBe(true) - expect(testState.sendRuntimePtyInput).toHaveBeenCalledWith({}, 'pty-1', PASTED_ISSUE_URL) + expect(testState.sendRuntimePtyInputVerified).toHaveBeenCalledWith( + {}, + 'pty-1', + PASTED_ISSUE_URL + ) }) it('does not paste for agents that already use native draft prefill', async () => { @@ -137,7 +150,7 @@ describe('pasteDraftWhenAgentReady', () => { ).resolves.toBe(false) expect(testState.subscribeToPtyData).not.toHaveBeenCalled() - expect(testState.sendRuntimePtyInput).not.toHaveBeenCalled() + expect(testState.sendRuntimePtyInputVerified).not.toHaveBeenCalled() }) it('can force paste and submit for native-prefill agents', async () => { @@ -154,7 +167,39 @@ describe('pasteDraftWhenAgentReady', () => { await vi.advanceTimersByTimeAsync(1500) await expect(promise).resolves.toBe(true) - expect(testState.sendRuntimePtyInput).toHaveBeenCalledWith({}, 'pty-1', `${PASTED_ISSUE_URL}\r`) + expect(testState.sendRuntimePtyInputVerified).toHaveBeenCalledWith( + {}, + 'pty-1', + `${PASTED_ISSUE_URL}\r` + ) + }) + + it('reports false when verified input delivery fails', async () => { + testState.sendRuntimePtyInputVerified.mockResolvedValue(false) + const promise = pasteDraftWhenAgentReady({ + tabId: 'tab-1', + content: ISSUE_URL, + agent: 'codex' + }) + await flushMicrotasks() + + testState.ptyObserver?.(`${DECSET_BRACKETED_PASTE}${CODEX_COMPOSER_PROMPT_RENDER}`) + + await expect(promise).resolves.toBe(false) + }) + + it('reports false when verified input delivery rejects', async () => { + testState.sendRuntimePtyInputVerified.mockRejectedValue(new Error('runtime timeout')) + const promise = pasteDraftWhenAgentReady({ + tabId: 'tab-1', + content: ISSUE_URL, + agent: 'codex' + }) + await flushMicrotasks() + + testState.ptyObserver?.(`${DECSET_BRACKETED_PASTE}${CODEX_COMPOSER_PROMPT_RENDER}`) + + await expect(promise).resolves.toBe(false) }) }) diff --git a/src/renderer/src/lib/agent-paste-draft.ts b/src/renderer/src/lib/agent-paste-draft.ts index b133c3591d8..00dcf7207fc 100644 --- a/src/renderer/src/lib/agent-paste-draft.ts +++ b/src/renderer/src/lib/agent-paste-draft.ts @@ -4,7 +4,6 @@ import { useAppStore } from '@/store' import { subscribeToPtyData } from '@/components/terminal-pane/pty-dispatcher' import { isRemoteRuntimePtyId, - sendRuntimePtyInput, sendRuntimePtyInputVerified } from '@/runtime/runtime-terminal-inspection' import { subscribeToRuntimeTerminalData } from '@/runtime/runtime-terminal-stream' @@ -95,12 +94,15 @@ export async function pasteDraftWhenAgentReady(args: { return false } - sendRuntimePtyInput( - useAppStore.getState().settings, - ptyId, - `${BRACKETED_PASTE_BEGIN}${content}${BRACKETED_PASTE_END}${submit ? '\r' : ''}` - ) - return true + try { + return await sendRuntimePtyInputVerified( + useAppStore.getState().settings, + ptyId, + `${BRACKETED_PASTE_BEGIN}${content}${BRACKETED_PASTE_END}${submit ? '\r' : ''}` + ) + } catch { + return false + } } export async function submitPromptToAgentTab(args: { diff --git a/src/renderer/src/lib/agent-prompt-sent-telemetry.test.ts b/src/renderer/src/lib/agent-prompt-sent-telemetry.test.ts new file mode 100644 index 00000000000..966bf3c5ebf --- /dev/null +++ b/src/renderer/src/lib/agent-prompt-sent-telemetry.test.ts @@ -0,0 +1,79 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mockTrack = vi.hoisted(() => vi.fn()) + +vi.mock('@/lib/telemetry', () => ({ + track: mockTrack +})) + +import { trackFoldedStartupPromptSent } from './agent-prompt-sent-telemetry' + +describe('trackFoldedStartupPromptSent', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + + it('tracks a folded startup prompt once with bounded telemetry', () => { + trackFoldedStartupPromptSent({ + prompt: 'fix the spinner', + startupPlan: { + followupPrompt: null, + draftPrompt: null + }, + startupQueued: true, + promptTelemetry: { + agent_kind: 'codex', + launch_source: 'new_workspace_composer', + request_kind: 'new' + } + }) + + expect(mockTrack).toHaveBeenCalledOnce() + expect(mockTrack).toHaveBeenCalledWith('agent_prompt_sent', { + agent_kind: 'codex', + launch_source: 'new_workspace_composer', + request_kind: 'new' + }) + }) + + it('does not track empty prompts, follow-up prompts, drafts, or bare shells', () => { + const promptTelemetry = { + agent_kind: 'codex', + launch_source: 'new_workspace_composer', + request_kind: 'new' + } as const + + trackFoldedStartupPromptSent({ + prompt: ' ', + startupPlan: { followupPrompt: null, draftPrompt: null }, + startupQueued: true, + promptTelemetry + }) + trackFoldedStartupPromptSent({ + prompt: 'fix the spinner', + startupPlan: { followupPrompt: 'fix the spinner', draftPrompt: null }, + startupQueued: true, + promptTelemetry + }) + trackFoldedStartupPromptSent({ + prompt: 'https://example.test/issue/1', + startupPlan: { followupPrompt: null, draftPrompt: 'https://example.test/issue/1' }, + startupQueued: true, + promptTelemetry + }) + trackFoldedStartupPromptSent({ + prompt: 'fix the spinner', + startupPlan: { followupPrompt: null, draftPrompt: null }, + startupQueued: true, + promptTelemetry: null + }) + trackFoldedStartupPromptSent({ + prompt: 'fix the spinner', + startupPlan: { followupPrompt: null, draftPrompt: null }, + startupQueued: false, + promptTelemetry + }) + + expect(mockTrack).not.toHaveBeenCalled() + }) +}) diff --git a/src/renderer/src/lib/agent-prompt-sent-telemetry.ts b/src/renderer/src/lib/agent-prompt-sent-telemetry.ts new file mode 100644 index 00000000000..c49c26e435f --- /dev/null +++ b/src/renderer/src/lib/agent-prompt-sent-telemetry.ts @@ -0,0 +1,25 @@ +import { track } from '@/lib/telemetry' +import type { AgentStartupPlan } from '@/lib/tui-agent-startup' +import type { EventProps } from '../../../shared/telemetry-events' + +export function trackFoldedStartupPromptSent(args: { + prompt: string + startupPlan: Pick + promptTelemetry: EventProps<'agent_prompt_sent'> | null + startupQueued: boolean +}): void { + const { prompt, startupPlan, promptTelemetry, startupQueued } = args + if (!startupQueued) { + return + } + if (!promptTelemetry) { + return + } + if (prompt.trim().length === 0) { + return + } + if (startupPlan.followupPrompt !== null || startupPlan.draftPrompt) { + return + } + track('agent_prompt_sent', promptTelemetry) +} diff --git a/src/renderer/src/lib/launch-agent-in-new-tab.test.ts b/src/renderer/src/lib/launch-agent-in-new-tab.test.ts index 8ae121335af..20696d9c99a 100644 --- a/src/renderer/src/lib/launch-agent-in-new-tab.test.ts +++ b/src/renderer/src/lib/launch-agent-in-new-tab.test.ts @@ -6,6 +6,7 @@ const mockSetActiveTabType = vi.fn() const mockSetTabBarOrder = vi.fn() const mockSetAgentStatus = vi.fn() const mockPasteDraftWhenAgentReady = vi.fn() +const mockTrack = vi.fn() const LEAF_ID = '11111111-1111-4111-8111-111111111111' @@ -50,7 +51,7 @@ vi.mock('@/lib/agent-paste-draft', () => ({ })) vi.mock('@/lib/telemetry', () => ({ - track: vi.fn(), + track: mockTrack, tuiAgentToAgentKind: (agent: string) => agent })) @@ -88,6 +89,37 @@ describe('launchAgentInNewTab', () => { ) }) + it('tracks one prompt-sent event for argv prompt launches', async () => { + const { launchAgentInNewTab } = await import('./launch-agent-in-new-tab') + + launchAgentInNewTab({ + agent: 'codex', + worktreeId: 'wt-1', + prompt: 'fix the spinner', + launchSource: 'onboarding' + }) + + expect(mockTrack).toHaveBeenCalledTimes(1) + expect(mockTrack).toHaveBeenCalledWith('agent_prompt_sent', { + agent_kind: 'codex', + launch_source: 'onboarding', + request_kind: 'new' + }) + }) + + it('does not track prompt-sent for draft launches', async () => { + const { launchAgentInNewTab } = await import('./launch-agent-in-new-tab') + + launchAgentInNewTab({ + agent: 'claude', + worktreeId: 'wt-1', + prompt: 'review this before sending', + promptDelivery: 'draft' + }) + + expect(mockTrack).not.toHaveBeenCalledWith('agent_prompt_sent', expect.anything()) + }) + it('seeds working after Command Code submit-after-ready prompt delivery', async () => { const { launchAgentInNewTab } = await import('./launch-agent-in-new-tab') @@ -120,5 +152,26 @@ describe('launchAgentInNewTab', () => { prompt: 'large generated prompt', agentType: 'command-code' }) + expect(mockTrack).toHaveBeenCalledTimes(1) + expect(mockTrack).toHaveBeenCalledWith('agent_prompt_sent', { + agent_kind: 'command-code', + launch_source: 'tab_bar_quick_launch', + request_kind: 'new' + }) + }) + + it('does not track prompt-sent when submit-after-ready delivery fails', async () => { + mockPasteDraftWhenAgentReady.mockResolvedValue(false) + const { launchAgentInNewTab } = await import('./launch-agent-in-new-tab') + + launchAgentInNewTab({ + agent: 'command-code', + worktreeId: 'wt-1', + prompt: 'large generated prompt', + promptDelivery: 'submit-after-ready' + }) + await Promise.resolve() + + expect(mockTrack).not.toHaveBeenCalledWith('agent_prompt_sent', expect.anything()) }) }) diff --git a/src/renderer/src/lib/launch-agent-in-new-tab.ts b/src/renderer/src/lib/launch-agent-in-new-tab.ts index dd305a18972..fbec392e273 100644 --- a/src/renderer/src/lib/launch-agent-in-new-tab.ts +++ b/src/renderer/src/lib/launch-agent-in-new-tab.ts @@ -12,7 +12,7 @@ import { pasteDraftWhenAgentReady } from '@/lib/agent-paste-draft' import { TUI_AGENT_CONFIG } from '../../../shared/tui-agent-config' import { makePaneKey } from '../../../shared/stable-pane-id' import type { TuiAgent } from '../../../shared/types' -import type { LaunchSource } from '../../../shared/telemetry-events' +import type { EventProps, LaunchSource } from '../../../shared/telemetry-events' export type LaunchAgentInNewTabArgs = { agent: TuiAgent @@ -93,6 +93,11 @@ export function launchAgentInNewTab(args: LaunchAgentInNewTabArgs): LaunchAgentI const trimmedPrompt = prompt?.trim() ?? '' const hasPrompt = trimmedPrompt.length > 0 const isFollowupPath = TUI_AGENT_CONFIG[agent].promptInjectionMode === 'stdin-after-start' + const promptTelemetry: EventProps<'agent_prompt_sent'> = { + agent_kind: tuiAgentToAgentKind(agent), + launch_source: launchSource ?? 'tab_bar_quick_launch', + request_kind: 'new' + } // Why: argv/flag agents fold the prompt into the launch command and // auto-submit — keeping behavior consistent with the composer/tab-bar `+` @@ -189,6 +194,14 @@ export function launchAgentInNewTab(args: LaunchAgentInNewTabArgs): LaunchAgentI request_kind: 'new' } }) + if ( + hasPrompt && + promptDelivery === 'auto-submit' && + startupPlan.followupPrompt === null && + pasteDraftAfterLaunch === null + ) { + track('agent_prompt_sent', promptTelemetry) + } // Why: schedule the bracketed-paste-after-ready follow-up immediately after // the startup command is queued. Fire-and-forget so callers keep their @@ -234,6 +247,9 @@ export function launchAgentInNewTab(args: LaunchAgentInNewTabArgs): LaunchAgentI } }).then((delivered) => { if (delivered) { + if (submitPastedPrompt) { + track('agent_prompt_sent', promptTelemetry) + } if (agent === 'command-code' && submitPastedPrompt) { // Why: Command Code has no prompt-submit hook; when Orca submits a // generated prompt after readiness, seed working at delivery time. diff --git a/src/renderer/src/lib/new-workspace.test.ts b/src/renderer/src/lib/new-workspace.test.ts index 17d7ae21b76..89c7eabbf14 100644 --- a/src/renderer/src/lib/new-workspace.test.ts +++ b/src/renderer/src/lib/new-workspace.test.ts @@ -1,5 +1,48 @@ -import { describe, expect, it } from 'vitest' -import { getWorkspaceSeedName, isGitLabIssueUrl } from './new-workspace' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { + mockInspectRuntimeTerminalProcess, + mockSendRuntimePtyInputVerified, + mockPasteDraftWhenAgentReady, + mockTrack, + store +} = vi.hoisted(() => ({ + mockInspectRuntimeTerminalProcess: vi.fn(), + mockSendRuntimePtyInputVerified: vi.fn(), + mockPasteDraftWhenAgentReady: vi.fn(), + mockTrack: vi.fn(), + store: { + settings: {}, + activeTabIdByWorktree: { 'wt-1': 'tab-1' } as Record, + tabsByWorktree: { 'wt-1': [{ id: 'tab-1' }] } as Record, + ptyIdsByTabId: { 'tab-1': ['pty-1'] } as Record + } +})) + +vi.mock('@/store', () => ({ + useAppStore: { + getState: () => store + } +})) + +vi.mock('@/runtime/runtime-terminal-inspection', () => ({ + inspectRuntimeTerminalProcess: mockInspectRuntimeTerminalProcess, + sendRuntimePtyInputVerified: mockSendRuntimePtyInputVerified +})) + +vi.mock('@/lib/agent-paste-draft', () => ({ + pasteDraftWhenAgentReady: mockPasteDraftWhenAgentReady +})) + +vi.mock('@/lib/telemetry', () => ({ + track: mockTrack +})) + +import { + ensureAgentStartupInTerminal, + getWorkspaceSeedName, + isGitLabIssueUrl +} from './new-workspace' describe('getWorkspaceSeedName', () => { it('prefers an explicit name', () => { @@ -124,3 +167,112 @@ describe('isGitLabIssueUrl', () => { expect(isGitLabIssueUrl('https://github.com/group/project/issues/123')).toBe(false) }) }) + +describe('ensureAgentStartupInTerminal prompt telemetry', () => { + beforeEach(() => { + vi.clearAllMocks() + store.settings = {} + store.activeTabIdByWorktree = { 'wt-1': 'tab-1' } + store.tabsByWorktree = { 'wt-1': [{ id: 'tab-1' }] } + store.ptyIdsByTabId = { 'tab-1': ['pty-1'] } + mockInspectRuntimeTerminalProcess.mockResolvedValue({ + foregroundProcess: 'aider', + hasChildProcesses: true + }) + mockSendRuntimePtyInputVerified.mockResolvedValue(true) + mockPasteDraftWhenAgentReady.mockResolvedValue(true) + }) + + it('tracks after a follow-up prompt is accepted by the terminal runtime', async () => { + await ensureAgentStartupInTerminal({ + worktreeId: 'wt-1', + startup: { + agent: 'aider', + launchCommand: 'aider', + expectedProcess: 'aider', + followupPrompt: 'fix the spinner' + }, + promptTelemetry: { + agent_kind: 'aider', + launch_source: 'new_workspace_composer', + request_kind: 'new' + } + }) + + expect(mockSendRuntimePtyInputVerified).toHaveBeenCalledWith({}, 'pty-1', 'fix the spinner\r') + expect(mockTrack).toHaveBeenCalledWith('agent_prompt_sent', { + agent_kind: 'aider', + launch_source: 'new_workspace_composer', + request_kind: 'new' + }) + }) + + it('does not track when follow-up prompt delivery is rejected by the terminal runtime', async () => { + mockSendRuntimePtyInputVerified.mockResolvedValue(false) + + await ensureAgentStartupInTerminal({ + worktreeId: 'wt-1', + startup: { + agent: 'aider', + launchCommand: 'aider', + expectedProcess: 'aider', + followupPrompt: 'fix the spinner' + }, + promptTelemetry: { + agent_kind: 'aider', + launch_source: 'new_workspace_composer', + request_kind: 'new' + } + }) + + expect(mockTrack).not.toHaveBeenCalledWith('agent_prompt_sent', expect.anything()) + }) + + it('does not track when follow-up prompt delivery rejects', async () => { + mockSendRuntimePtyInputVerified.mockRejectedValue(new Error('runtime timeout')) + + await expect( + ensureAgentStartupInTerminal({ + worktreeId: 'wt-1', + startup: { + agent: 'aider', + launchCommand: 'aider', + expectedProcess: 'aider', + followupPrompt: 'fix the spinner' + }, + promptTelemetry: { + agent_kind: 'aider', + launch_source: 'new_workspace_composer', + request_kind: 'new' + } + }) + ).resolves.toBeUndefined() + + expect(mockTrack).not.toHaveBeenCalledWith('agent_prompt_sent', expect.anything()) + }) + + it('does not track draft prompt delivery as a sent prompt', async () => { + await ensureAgentStartupInTerminal({ + worktreeId: 'wt-1', + startup: { + agent: 'claude', + launchCommand: 'claude', + expectedProcess: 'claude', + followupPrompt: null, + draftPrompt: 'review this before sending' + }, + promptTelemetry: { + agent_kind: 'claude-code', + launch_source: 'new_workspace_composer', + request_kind: 'new' + } + }) + + expect(mockPasteDraftWhenAgentReady).toHaveBeenCalledWith({ + tabId: 'tab-1', + content: 'review this before sending', + agent: 'claude' + }) + expect(mockTrack).not.toHaveBeenCalledWith('agent_prompt_sent', expect.anything()) + }) +}) diff --git a/src/renderer/src/lib/new-workspace.ts b/src/renderer/src/lib/new-workspace.ts index 4c32f0b395b..d8c6fe604ae 100644 --- a/src/renderer/src/lib/new-workspace.ts +++ b/src/renderer/src/lib/new-workspace.ts @@ -2,10 +2,12 @@ import { useAppStore } from '@/store' import { pasteDraftWhenAgentReady } from '@/lib/agent-paste-draft' import { inspectRuntimeTerminalProcess, - sendRuntimePtyInput + sendRuntimePtyInputVerified } from '@/runtime/runtime-terminal-inspection' +import { track } from '@/lib/telemetry' import type { AgentStartupPlan } from '@/lib/tui-agent-startup' import { isShellProcess } from '@/lib/tui-agent-startup' +import type { EventProps } from '../../../shared/telemetry-events' import type { OrcaHooks, TaskViewPresetId } from '../../../shared/types' import { resolveHookCommandSourcePolicy } from '../../../shared/hook-command-source-policy' import { isExpectedAgentProcess } from '../../../shared/agent-process-recognition' @@ -208,8 +210,9 @@ export function getWorkspaceSeedName(args: { export async function ensureAgentStartupInTerminal(args: { worktreeId: string startup: AgentStartupPlan + promptTelemetry?: EventProps<'agent_prompt_sent'> }): Promise { - const { worktreeId, startup } = args + const { worktreeId, startup, promptTelemetry } = args const draftPrompt = startup.draftPrompt ?? null if (startup.followupPrompt === null && draftPrompt === null) { return @@ -244,7 +247,12 @@ export async function ensureAgentStartupInTerminal(args: { // session and submitted. Wait until the agent owns the PTY before writing. if (startup.followupPrompt) { await waitForAgentForeground(ptyId, startup.expectedProcess) - sendRuntimePtyInput(useAppStore.getState().settings, ptyId, `${startup.followupPrompt}\r`) + const sent = await sendFollowupPrompt(ptyId, startup.followupPrompt) + if (sent) { + if (promptTelemetry) { + track('agent_prompt_sent', promptTelemetry) + } + } } // Why: draftPrompt uses bracketed-paste so the URL lands atomically in the @@ -259,6 +267,14 @@ export async function ensureAgentStartupInTerminal(args: { } } +async function sendFollowupPrompt(ptyId: string, prompt: string): Promise { + try { + return await sendRuntimePtyInputVerified(useAppStore.getState().settings, ptyId, `${prompt}\r`) + } catch { + return false + } +} + // Why: legacy followupPrompt path used `agentOwnsForeground` exclusively (with // a hasChildProcesses fallback after several polls). Preserve that behavior so // stdin-after-start agents still receive their prompt under the same diff --git a/src/renderer/src/runtime/runtime-terminal-inspection.test.ts b/src/renderer/src/runtime/runtime-terminal-inspection.test.ts index eab9d599c26..3cfeb26a1bc 100644 --- a/src/renderer/src/runtime/runtime-terminal-inspection.test.ts +++ b/src/renderer/src/runtime/runtime-terminal-inspection.test.ts @@ -1,5 +1,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' -import { inspectRuntimeTerminalProcess, sendRuntimePtyInput } from './runtime-terminal-inspection' +import { + inspectRuntimeTerminalProcess, + sendRuntimePtyInput, + sendRuntimePtyInputVerified +} from './runtime-terminal-inspection' import { createCompatibleRuntimeStatusResponseIfNeeded, type RuntimeEnvironmentCallRequest @@ -10,6 +14,7 @@ describe('runtime terminal owner routing', () => { const runtimeCall = vi.fn() const runtimeTransportCall = vi.fn() const localWrite = vi.fn() + const localWriteAccepted = vi.fn() const localForeground = vi.fn() const localHasChildren = vi.fn() @@ -29,6 +34,7 @@ describe('runtime terminal owner routing', () => { runtimeEnvironments: { call: runtimeTransportCall }, pty: { write: localWrite, + writeAccepted: localWriteAccepted, getForegroundProcess: localForeground, hasChildProcesses: localHasChildren } @@ -83,4 +89,68 @@ describe('runtime terminal owner routing', () => { ) ).resolves.toEqual({ foregroundProcess: null, hasChildProcesses: false }) }) + + it('reports stale remote terminal handles as rejected during verified send', async () => { + runtimeCall.mockResolvedValue({ + ok: false, + error: { code: 'terminal_handle_stale', message: 'terminal_handle_stale' } + }) + + await expect( + sendRuntimePtyInputVerified( + { activeRuntimeEnvironmentId: 'env-2' }, + 'remote:env-1@@terminal-stale', + 'x' + ) + ).resolves.toBe(false) + }) + + it('reports declined remote terminal sends as rejected during verified send', async () => { + runtimeCall.mockResolvedValue({ + ok: true, + result: { send: { handle: 'terminal-1', accepted: false, bytesWritten: 0 } }, + _meta: { runtimeId: 'runtime-1' } + }) + + await expect( + sendRuntimePtyInputVerified( + { activeRuntimeEnvironmentId: 'env-2' }, + 'remote:env-1@@terminal-1', + 'x' + ) + ).resolves.toBe(false) + + expect(runtimeCall).toHaveBeenCalledWith({ + selector: 'env-1', + method: 'terminal.send', + params: { + terminal: 'terminal-1', + text: 'x', + client: { id: 'orca-desktop', type: 'desktop' } + }, + timeoutMs: 15_000 + }) + }) + + it('uses accepted local writes for verified input', async () => { + localWriteAccepted.mockResolvedValue(true) + + await expect( + sendRuntimePtyInputVerified({ activeRuntimeEnvironmentId: null }, 'local-pty', 'x') + ).resolves.toBe(true) + + expect(localWriteAccepted).toHaveBeenCalledWith('local-pty', 'x') + expect(localWrite).not.toHaveBeenCalled() + }) + + it('falls back to fire-and-forget writes when local acceptance cannot be verified', async () => { + localWriteAccepted.mockResolvedValue(false) + + await expect( + sendRuntimePtyInputVerified({ activeRuntimeEnvironmentId: null }, 'local-pty', 'x') + ).resolves.toBe(false) + + expect(localWriteAccepted).toHaveBeenCalledWith('local-pty', 'x') + expect(localWrite).toHaveBeenCalledWith('local-pty', 'x') + }) }) diff --git a/src/renderer/src/runtime/runtime-terminal-inspection.ts b/src/renderer/src/runtime/runtime-terminal-inspection.ts index 18d87ec4a45..44348f19973 100644 --- a/src/renderer/src/runtime/runtime-terminal-inspection.ts +++ b/src/renderer/src/runtime/runtime-terminal-inspection.ts @@ -1,4 +1,5 @@ import type { GlobalSettings } from '../../../shared/types' +import type { RuntimeTerminalSend } from '../../../shared/runtime-types' import { RuntimeRpcCallError, callRuntimeRpc, getActiveRuntimeTarget } from './runtime-rpc-client' import { getRemoteRuntimePtyEnvironmentId, @@ -106,13 +107,21 @@ export async function sendRuntimePtyInputVerified( : getActiveRuntimeTarget(settings) const terminal = getRemoteRuntimeTerminalHandle(ptyId) if (target.kind !== 'environment' || !terminal) { - window.api.pty.write(ptyId, data) - return true + const accepted = await window.api.pty.writeAccepted(ptyId, data) + if (!accepted) { + window.api.pty.write(ptyId, data) + } + return accepted } try { - await callRuntimeRpc(target, 'terminal.send', { terminal, text: data }, { timeoutMs: 15_000 }) - return true + const result = await callRuntimeRpc<{ send: RuntimeTerminalSend }>( + target, + 'terminal.send', + { terminal, text: data, client: { id: 'orca-desktop', type: 'desktop' } }, + { timeoutMs: 15_000 } + ) + return result.send.accepted === true } catch (error) { if (isTerminalGoneError(error)) { return false diff --git a/src/shared/telemetry-events.test.ts b/src/shared/telemetry-events.test.ts index 1f8f10dd2c8..8d0aa7621da 100644 --- a/src/shared/telemetry-events.test.ts +++ b/src/shared/telemetry-events.test.ts @@ -111,6 +111,28 @@ describe('agent_started schema', () => { }) }) +describe('agent_prompt_sent schema', () => { + it('accepts a bounded prompt-send payload with cohort context', () => { + const parsed = eventSchemas.agent_prompt_sent.safeParse({ + agent_kind: 'codex', + launch_source: 'onboarding', + request_kind: 'new', + nth_repo_added: 1 + }) + expect(parsed.success).toBe(true) + }) + + it('rejects prompt text via .strict()', () => { + const parsed = eventSchemas.agent_prompt_sent.safeParse({ + agent_kind: 'claude-code', + launch_source: 'new_workspace_composer', + request_kind: 'new', + prompt: 'please inspect /Users/alice/private-repo' + }) + expect(parsed.success).toBe(false) + }) +}) + describe('agent_hook_unattributed schema', () => { it('accepts the two bounded attribution failure reasons', () => { for (const reason of ['empty_pane_key', 'unknown_tab_id'] as const) { diff --git a/src/shared/telemetry-events.ts b/src/shared/telemetry-events.ts index 3aa012ea434..d84f513117d 100644 --- a/src/shared/telemetry-events.ts +++ b/src/shared/telemetry-events.ts @@ -281,6 +281,14 @@ const agentStartedSchema = z nth_repo_added: nthRepoAddedSchema }) .strict() +const agentPromptSentSchema = z + .object({ + agent_kind: agentKindSchema, + launch_source: launchSourceSchema, + request_kind: requestKindSchema, + nth_repo_added: nthRepoAddedSchema + }) + .strict() // Enum-only by design for both fields. `error_message` and `error_stack` are // deliberately absent — `.strict()` rejects either key if a call site ever @@ -1017,6 +1025,7 @@ export const eventSchemas = { setup_script_prompt_action: setupScriptPromptActionSchema, agent_started: agentStartedSchema, + agent_prompt_sent: agentPromptSentSchema, agent_error: agentErrorSchema, agent_hook_install_failed: agentHookInstallFailedSchema, agent_hook_unattributed: agentHookUnattributedSchema, @@ -1107,6 +1116,7 @@ type _CohortExtendedRoster = | 'setup_script_prompt_shown' | 'setup_script_prompt_action' | 'agent_started' + | 'agent_prompt_sent' | 'agent_error' // Why: `z.object({}).strict()` infers a string index signature, which would // make every key appear present. Ignore index-signature-only keys here so