From 8cc1cc5a14868759d3cb45efa834279e2322b028 Mon Sep 17 00:00:00 2001 From: Neil Date: Thu, 10 Sep 2026 23:10:26 -0700 Subject: [PATCH] fix(renderer): report every credential refusal, and prove the cross-emulator projection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `submitPromptToAgentPty` and `sendBracketedPasteToRunningAgent` collapsed a credential refusal to `false` and never called `onUndelivered`, so the automation dispatch path abandoned a session reuse with no record at all. Both now route through `deliverBracketedPaste`; the dispatch site tracks the refusal without a toast, because it recovers by launching a fresh background session. Quick launch — the highest-volume entry point — showed the readiness-timeout wording and `paste_readiness_timeout` telemetry for a credential refusal. The notice now takes the failure and routes it, which is what the `onUndelivered` rename was for; renamed to match what it reports. The equivalence proof rendered only through `@xterm/headless`, while the pane that feeds the guard is `@xterm/xterm` on a neighbouring beta. Added a parity suite driving the corpus through the real renderer emulator: both project the same bytes, so the skew is immaterial and a future divergence now fails a test. Split the corpus by side to stay under max-lines. --- ...-credential-prompt-wait-vocabulary.test.ts | 6 +- ...pty-visible-screen-emulator-parity.test.ts | 72 ++++++++ .../src/hooks/automation-dispatch-handler.ts | 9 +- ...gent-paste-credential-prompt-guard.test.ts | 6 +- .../agent-paste-credential-prompt-notice.ts | 5 + ...gent-paste-draft-credential-prompt.test.ts | 17 +- src/renderer/src/lib/agent-paste-draft.ts | 18 +- .../src/lib/launch-agent-in-new-tab.ts | 8 +- ...nch-agent-paste-undelivered-notice.test.ts | 88 ++++++++++ ... launch-agent-paste-undelivered-notice.ts} | 18 +- ...minal-agent-auth-summary-screens-corpus.ts | 133 ++++++++++++++ ...rminal-credential-prompt-detection.test.ts | 6 +- ...erminal-legitimate-agent-screens-corpus.ts | 165 ++++++++++++++++++ ...erminal-live-credential-surfaces-corpus.ts | 91 ++++++++++ ...terminal-visible-screen-projection.test.ts | 6 +- 15 files changed, 613 insertions(+), 35 deletions(-) create mode 100644 src/renderer/src/components/terminal-pane/pty-visible-screen-emulator-parity.test.ts create mode 100644 src/renderer/src/lib/launch-agent-paste-undelivered-notice.test.ts rename src/renderer/src/lib/{launch-agent-paste-timeout-notice.ts => launch-agent-paste-undelivered-notice.ts} (69%) create mode 100644 src/shared/terminal-agent-auth-summary-screens-corpus.ts create mode 100644 src/shared/terminal-legitimate-agent-screens-corpus.ts create mode 100644 src/shared/terminal-live-credential-surfaces-corpus.ts diff --git a/src/main/runtime/terminal-credential-prompt-wait-vocabulary.test.ts b/src/main/runtime/terminal-credential-prompt-wait-vocabulary.test.ts index 45d76b5441b..6de55ecccc9 100644 --- a/src/main/runtime/terminal-credential-prompt-wait-vocabulary.test.ts +++ b/src/main/runtime/terminal-credential-prompt-wait-vocabulary.test.ts @@ -2,10 +2,8 @@ // blocks must also reach `agent-credential-prompt` through the wait-blocked vocabulary, which // is what makes `writeTerminalAgentPrompt` refuse the PTY write. import { describe, expect, it } from 'vitest' -import { - LEGITIMATE_AGENT_SCREENS, - LIVE_CREDENTIAL_SURFACES -} from '../../shared/terminal-credential-prompt-corpus' +import { LEGITIMATE_AGENT_SCREENS } from '../../shared/terminal-legitimate-agent-screens-corpus' +import { LIVE_CREDENTIAL_SURFACES } from '../../shared/terminal-live-credential-surfaces-corpus' import { detectTerminalWaitBlockedReason, isKnownReadyPromptPreview diff --git a/src/renderer/src/components/terminal-pane/pty-visible-screen-emulator-parity.test.ts b/src/renderer/src/components/terminal-pane/pty-visible-screen-emulator-parity.test.ts new file mode 100644 index 00000000000..e2570f76fcc --- /dev/null +++ b/src/renderer/src/components/terminal-pane/pty-visible-screen-emulator-parity.test.ts @@ -0,0 +1,72 @@ +// @vitest-environment happy-dom + +// The equivalence proof in src/shared renders through @xterm/headless, but the pane that actually +// feeds the renderer guard is @xterm/xterm — a different package on a neighbouring beta. Byte +// identity proven against one emulator says nothing about the other, so this drives the corpus +// through the real renderer emulator and pins the two projections to each other. +import { Terminal } from '@xterm/xterm' +import { Unicode11Addon } from '@xterm/addon-unicode11' +import { describe, expect, it } from 'vitest' +import { + createRendererParityTerminal, + writeToTerminal +} from '../../../../shared/terminal-restore-parity-fixture' +import { buildTerminalVisibleScreenText } from '../../../../shared/terminal-visible-screen-projection' +import { activateOrcaTerminalUnicodeProvider } from '../../../../shared/terminal-unicode-provider' +import { DESKTOP_TERMINAL_SCROLLBACK_ROWS_DEFAULT } from '../../../../shared/terminal-scrollback-policy' +import { findCredentialPromptIndex } from '../../../../shared/terminal-credential-prompt-detection' +import { LEGITIMATE_AGENT_SCREENS } from '../../../../shared/terminal-legitimate-agent-screens-corpus' +import { LIVE_CREDENTIAL_SURFACES } from '../../../../shared/terminal-live-credential-surfaces-corpus' + +const VIEWPORT = { cols: 120, rows: 24 } + +function renderThroughRendererEmulator(lines: string[]): Promise { + const terminal = new Terminal({ + cols: VIEWPORT.cols, + rows: VIEWPORT.rows, + scrollback: DESKTOP_TERMINAL_SCROLLBACK_ROWS_DEFAULT, + allowProposedApi: true, + vtExtensions: { kittyKeyboard: true } + }) + terminal.loadAddon(new Unicode11Addon()) + activateOrcaTerminalUnicodeProvider(terminal) + return new Promise((resolve) => { + terminal.write(`\x1b[H\x1b[2J${lines.join('\r\n')}`, () => { + resolve(buildTerminalVisibleScreenText(terminal)) + terminal.dispose() + }) + }) +} + +async function renderThroughHeadlessEmulator(lines: string[]): Promise { + const { terminal } = createRendererParityTerminal(VIEWPORT) + await writeToTerminal(terminal, `\x1b[H\x1b[2J${lines.join('\r\n')}`) + return buildTerminalVisibleScreenText(terminal) +} + +describe('@xterm/xterm and @xterm/headless project the same visible screen', () => { + it.each([...LIVE_CREDENTIAL_SURFACES, ...LEGITIMATE_AGENT_SCREENS])( + 'agrees byte-for-byte on %s', + async (_name, lines) => { + expect(await renderThroughRendererEmulator(lines)).toBe( + await renderThroughHeadlessEmulator(lines) + ) + } + ) + + it.each(LIVE_CREDENTIAL_SURFACES)( + 'keeps %s detectable through the renderer emulator', + async (_name, lines) => { + const screen = await renderThroughRendererEmulator(lines) + expect(findCredentialPromptIndex(screen.toLowerCase())).not.toBeNull() + } + ) + + it.each(LEGITIMATE_AGENT_SCREENS)( + 'keeps %s passable through the renderer emulator', + async (_name, lines) => { + const screen = await renderThroughRendererEmulator(lines) + expect(findCredentialPromptIndex(screen.toLowerCase())).toBeNull() + } + ) +}) diff --git a/src/renderer/src/hooks/automation-dispatch-handler.ts b/src/renderer/src/hooks/automation-dispatch-handler.ts index 6df1a64fb58..92be2bc0866 100644 --- a/src/renderer/src/hooks/automation-dispatch-handler.ts +++ b/src/renderer/src/hooks/automation-dispatch-handler.ts @@ -1,6 +1,7 @@ import { listAutomationRunsForTarget } from '@/components/automations/automation-host-client' import { translate } from '@/i18n/i18n' import { submitPromptToAgentPty } from '@/lib/agent-paste-draft' +import { trackAgentPasteCredentialPromptRefusal } from '@/lib/agent-paste-credential-prompt-notice' import { launchAgentBackgroundSession } from '@/lib/launch-agent-background-session' import { observeExistingAutomationSession } from '@/lib/automation-session-observer' import { findReusableAutomationSession } from '@/lib/automation-session-reuse' @@ -111,7 +112,13 @@ export async function handleAutomationDispatchRequest({ const submitted = await submitPromptToAgentPty({ tabId: reusableSession.tabId, ptyId: reusableSession.ptyId, - content: automation.prompt + content: automation.prompt, + // Why no toast: a refused reuse falls through to a fresh background session below, + // so the prompt still lands — but the refusal must not vanish from telemetry. + onUndelivered: (failure) => + failure === 'credential-prompt' + ? trackAgentPasteCredentialPromptRefusal(automation.agentId) + : undefined }) if (!submitted) { completion.cleanupRunObservers() diff --git a/src/renderer/src/lib/agent-paste-credential-prompt-guard.test.ts b/src/renderer/src/lib/agent-paste-credential-prompt-guard.test.ts index 09bd9738adf..50a028bba46 100644 --- a/src/renderer/src/lib/agent-paste-credential-prompt-guard.test.ts +++ b/src/renderer/src/lib/agent-paste-credential-prompt-guard.test.ts @@ -6,10 +6,8 @@ import { createRendererParityTerminal, writeToTerminal } from '../../../shared/terminal-restore-parity-fixture' -import { - LEGITIMATE_AGENT_SCREENS, - LIVE_CREDENTIAL_SURFACES -} from '../../../shared/terminal-credential-prompt-corpus' +import { LEGITIMATE_AGENT_SCREENS } from '../../../shared/terminal-legitimate-agent-screens-corpus' +import { LIVE_CREDENTIAL_SURFACES } from '../../../shared/terminal-live-credential-surfaces-corpus' import { registerPtyVisibleScreen } from '@/components/terminal-pane/pty-visible-screen-registry' import { isAgentPasteBlockedByCredentialPrompt } from './agent-paste-credential-prompt-guard' diff --git a/src/renderer/src/lib/agent-paste-credential-prompt-notice.ts b/src/renderer/src/lib/agent-paste-credential-prompt-notice.ts index 997298415d8..2839832704a 100644 --- a/src/renderer/src/lib/agent-paste-credential-prompt-notice.ts +++ b/src/renderer/src/lib/agent-paste-credential-prompt-notice.ts @@ -18,6 +18,11 @@ export function showAgentPasteCredentialPromptToast(agent: TuiAgent, submitted: { value0: submitted ? 'prompt' : 'notes' } ) ) + trackAgentPasteCredentialPromptRefusal(agent) +} + +/** For callers that recover on their own and so must not toast, but still owe the refusal a record. */ +export function trackAgentPasteCredentialPromptRefusal(agent: TuiAgent): void { // Why 'unknown': errorClassSchema has no credential-refusal slot, and the dashboard's unknown // slice is this repo's established trigger to add one. track('agent_error', { error_class: 'unknown', agent_kind: tuiAgentToAgentKind(agent) }) diff --git a/src/renderer/src/lib/agent-paste-draft-credential-prompt.test.ts b/src/renderer/src/lib/agent-paste-draft-credential-prompt.test.ts index f07bd2e4dd2..efd83dd8f1c 100644 --- a/src/renderer/src/lib/agent-paste-draft-credential-prompt.test.ts +++ b/src/renderer/src/lib/agent-paste-draft-credential-prompt.test.ts @@ -10,6 +10,7 @@ import { registerPtyVisibleScreen } from '@/components/terminal-pane/pty-visible import { pasteDraftToAgentPtyWhenReady, pasteDraftWhenAgentReady, + sendBracketedPasteToRunningAgent, submitPromptToAgentPty } from './agent-paste-draft' @@ -227,10 +228,24 @@ describe('the agent paste lane refuses a live credential prompt', () => { it('guards the automation reuse entry point, which has no readiness wait at all', async () => { await showOnPane('pty-1', SIGN_IN_DIALOG) + const onUndelivered = vi.fn() await expect( - submitPromptToAgentPty({ tabId: 'tab-1', ptyId: 'pty-1', content: PROMPT }) + submitPromptToAgentPty({ tabId: 'tab-1', ptyId: 'pty-1', content: PROMPT, onUndelivered }) ).resolves.toBe(false) expect(testState.sendRuntimePtyInputVerified).not.toHaveBeenCalled() + // A refusal the caller never hears about is the silent drop the guard exists to prevent. + expect(onUndelivered).toHaveBeenCalledExactlyOnceWith('credential-prompt') + }) + + it('reports a refused paste into an already-running agent', async () => { + await showOnPane('pty-1', SIGN_IN_DIALOG) + const onUndelivered = vi.fn() + + await expect( + sendBracketedPasteToRunningAgent({ ptyId: 'pty-1', content: PROMPT, onUndelivered }) + ).resolves.toBe(false) + expect(testState.sendRuntimePtyInputVerified).not.toHaveBeenCalled() + expect(onUndelivered).toHaveBeenCalledExactlyOnceWith('credential-prompt') }) }) diff --git a/src/renderer/src/lib/agent-paste-draft.ts b/src/renderer/src/lib/agent-paste-draft.ts index 39faab78fde..3b69f0d0b4a 100644 --- a/src/renderer/src/lib/agent-paste-draft.ts +++ b/src/renderer/src/lib/agent-paste-draft.ts @@ -189,27 +189,27 @@ export async function submitPromptToAgentPty(args: { tabId: string ptyId: string content: string + onUndelivered?: (failure: AgentDraftDeliveryFailure) => void }): Promise { - return ( - (await sendBracketedPasteToAgent({ + return await deliverBracketedPaste( + { settings: getSettingsForAgentTabRuntimeOwner(args.tabId), ptyId: args.ptyId, content: args.content, submit: true - })) === 'delivered' + }, + args.onUndelivered ) } export async function sendBracketedPasteToRunningAgent(args: { ptyId: string content: string + onUndelivered?: (failure: AgentDraftDeliveryFailure) => void }): Promise { - return ( - (await sendBracketedPasteToAgent({ - ptyId: args.ptyId, - content: args.content, - submit: true - })) === 'delivered' + return await deliverBracketedPaste( + { ptyId: args.ptyId, content: args.content, submit: true }, + args.onUndelivered ) } diff --git a/src/renderer/src/lib/launch-agent-in-new-tab.ts b/src/renderer/src/lib/launch-agent-in-new-tab.ts index 98f2b750b89..5605ea741d6 100644 --- a/src/renderer/src/lib/launch-agent-in-new-tab.ts +++ b/src/renderer/src/lib/launch-agent-in-new-tab.ts @@ -5,7 +5,7 @@ import { CLIENT_PLATFORM } from '@/lib/new-workspace' import { getAgentLaunchPlatformForRepo } from '@/lib/agent-launch-platform' import { persistAgentLaunchTabOrder } from '@/lib/launch-agent-tab-order' import { tuiAgentToAgentKind } from '@/lib/telemetry' -import { createPasteReadinessTimeoutNotice } from '@/lib/launch-agent-paste-timeout-notice' +import { createPasteUndeliveredNotice } from '@/lib/launch-agent-paste-undelivered-notice' import { deliverLaunchPromptToAgentTab, seedNativeChatLaunchDraftForAgentTab @@ -264,7 +264,7 @@ function launchAgentInNewTabInternal( seedNativeChatLaunchDraftForAgentTab({ tabId: tab.id, agent, text: trimmedPrompt }) } if (pasteDraftAfterLaunch !== null) { - const timeoutNotice = createPasteReadinessTimeoutNotice({ + const undeliveredNotice = createPasteUndeliveredNotice({ worktreeId, tabId: tab.id, agent, @@ -276,7 +276,7 @@ function launchAgentInNewTabInternal( agent, submit: submitPastedPrompt, forcePaste: promptDelivery === 'submit-after-ready', - onUndelivered: timeoutNotice.onTimeout + onUndelivered: undeliveredNotice.onUndelivered }).then((delivered) => { if (delivered) { if (agent === 'command-code' && submitPastedPrompt) { @@ -286,7 +286,7 @@ function launchAgentInNewTabInternal( } onPromptDelivered?.() } - return { delivered, failureNotified: !delivered && timeoutNotice.wasNotified() } + return { delivered, failureNotified: !delivered && undeliveredNotice.wasNotified() } }) if (promptDelivery === 'submit-after-ready') { promptDeliveryResult = deliveryPromise diff --git a/src/renderer/src/lib/launch-agent-paste-undelivered-notice.test.ts b/src/renderer/src/lib/launch-agent-paste-undelivered-notice.test.ts new file mode 100644 index 00000000000..fa4b7b6fc94 --- /dev/null +++ b/src/renderer/src/lib/launch-agent-paste-undelivered-notice.test.ts @@ -0,0 +1,88 @@ +// Quick launch is the highest-volume paste entry point, and it used to report every undelivered +// prompt as a readiness timeout — including a credential refusal, which is a different event with +// different advice and different telemetry. +import { describe, expect, it, vi, beforeEach } from 'vitest' +import { createPasteUndeliveredNotice } from './launch-agent-paste-undelivered-notice' + +const testState = vi.hoisted(() => ({ + appState: { + activeWorktreeId: 'wt-1', + tabsByWorktree: { 'wt-1': [{ id: 'tab-1', ptyId: 'pty-1' }] } as Record< + string, + { id: string; ptyId: string | null }[] + > + }, + toastMessage: vi.fn(), + track: vi.fn(), + showCredentialToast: vi.fn() +})) + +vi.mock('@/store', () => ({ useAppStore: { getState: () => testState.appState } })) +vi.mock('sonner', () => ({ toast: { message: testState.toastMessage } })) +vi.mock('@/lib/telemetry', () => ({ + track: testState.track, + tuiAgentToAgentKind: (agent: string) => agent +})) +vi.mock('@/i18n/i18n', () => ({ translate: (_key: string, fallback: string) => fallback })) +vi.mock('@/lib/agent-paste-credential-prompt-notice', () => ({ + showAgentPasteCredentialPromptToast: testState.showCredentialToast +})) + +function notice(): ReturnType { + return createPasteUndeliveredNotice({ + worktreeId: 'wt-1', + tabId: 'tab-1', + agent: 'codex', + submitted: true + }) +} + +describe('createPasteUndeliveredNotice', () => { + beforeEach(() => { + testState.appState.activeWorktreeId = 'wt-1' + testState.appState.tabsByWorktree = { 'wt-1': [{ id: 'tab-1', ptyId: 'pty-1' }] } + testState.toastMessage.mockReset() + testState.track.mockReset() + testState.showCredentialToast.mockReset() + }) + + it('routes a credential refusal to the credential notice, not the timeout one', () => { + const undelivered = notice() + undelivered.onUndelivered('credential-prompt') + + expect(testState.showCredentialToast).toHaveBeenCalledExactlyOnceWith('codex', true) + expect(testState.track).not.toHaveBeenCalled() + expect(undelivered.wasNotified()).toBe(true) + }) + + it('still reports a readiness timeout as a readiness timeout', () => { + const undelivered = notice() + undelivered.onUndelivered('readiness-timeout') + + expect(testState.showCredentialToast).not.toHaveBeenCalled() + expect(testState.track).toHaveBeenCalledExactlyOnceWith('agent_error', { + error_class: 'paste_readiness_timeout', + agent_kind: 'codex' + }) + expect(undelivered.wasNotified()).toBe(true) + }) + + it('stays silent when the PTY never spawned, so the caller owns the sole notice', () => { + testState.appState.tabsByWorktree = { 'wt-1': [{ id: 'tab-1', ptyId: null }] } + const undelivered = notice() + undelivered.onUndelivered('credential-prompt') + + expect(testState.showCredentialToast).not.toHaveBeenCalled() + expect(undelivered.wasNotified()).toBe(false) + }) + + it('suppresses the toast but marks notified once the user has moved on', () => { + testState.appState.activeWorktreeId = 'wt-2' + const undelivered = notice() + undelivered.onUndelivered('credential-prompt') + + expect(testState.showCredentialToast).not.toHaveBeenCalled() + expect(testState.toastMessage).not.toHaveBeenCalled() + expect(undelivered.wasNotified()).toBe(true) + }) +}) diff --git a/src/renderer/src/lib/launch-agent-paste-timeout-notice.ts b/src/renderer/src/lib/launch-agent-paste-undelivered-notice.ts similarity index 69% rename from src/renderer/src/lib/launch-agent-paste-timeout-notice.ts rename to src/renderer/src/lib/launch-agent-paste-undelivered-notice.ts index f72f5b8b3b7..b7a5737fec6 100644 --- a/src/renderer/src/lib/launch-agent-paste-timeout-notice.ts +++ b/src/renderer/src/lib/launch-agent-paste-undelivered-notice.ts @@ -2,25 +2,31 @@ import { toast } from 'sonner' import { useAppStore } from '@/store' import { track, tuiAgentToAgentKind } from '@/lib/telemetry' import { translate } from '@/i18n/i18n' +import { showAgentPasteCredentialPromptToast } from '@/lib/agent-paste-credential-prompt-notice' +import type { AgentDraftDeliveryFailure } from '@/lib/agent-paste-credential-prompt-guard' import type { TuiAgent } from '../../../shared/tui-agent' /** * Notice for a post-launch paste that never landed — a stalled readiness wait * would otherwise drop the user's text silently. * + * Why it takes the failure: the two reasons want different telemetry. Reporting a + * credential refusal as `paste_readiness_timeout` is the same lie the reason argument + * exists to prevent. + * * `wasNotified()` reports whether the user already heard about it, so a * deferred caller can suppress a duplicate toast. */ -export function createPasteReadinessTimeoutNotice(args: { +export function createPasteUndeliveredNotice(args: { worktreeId: string tabId: string agent: TuiAgent submitted: boolean -}): { onTimeout: () => void; wasNotified: () => boolean } { +}): { onUndelivered: (failure: AgentDraftDeliveryFailure) => void; wasNotified: () => boolean } { let notified = false return { wasNotified: () => notified, - onTimeout: () => { + onUndelivered: (failure) => { const state = useAppStore.getState() const currentTab = (state.tabsByWorktree[args.worktreeId] ?? []).find( (tab) => tab.id === args.tabId @@ -34,6 +40,11 @@ export function createPasteReadinessTimeoutNotice(args: { notified = true return } + notified = true + if (failure === 'credential-prompt') { + showAgentPasteCredentialPromptToast(args.agent, args.submitted) + return + } toast.message( translate( 'auto.lib.launch.agent.in.new.tab.a5a1f7033f', @@ -41,7 +52,6 @@ export function createPasteReadinessTimeoutNotice(args: { { value0: args.submitted ? 'prompt' : 'notes' } ) ) - notified = true track('agent_error', { error_class: 'paste_readiness_timeout', agent_kind: tuiAgentToAgentKind(args.agent) diff --git a/src/shared/terminal-agent-auth-summary-screens-corpus.ts b/src/shared/terminal-agent-auth-summary-screens-corpus.ts new file mode 100644 index 00000000000..798e2add72f --- /dev/null +++ b/src/shared/terminal-agent-auth-summary-screens-corpus.ts @@ -0,0 +1,133 @@ +// Agent output ABOUT auth work, which reads exactly like a credential dialog to any rule that +// only looks for auth wording. Every one of these blocked before the pair rule gained a prompt-shape +// requirement, and every one leaves the agent's own composer caret on the last row. +import type { TerminalCredentialPromptCase } from './terminal-live-credential-surfaces-corpus' + +export const AGENT_AUTH_SUMMARY_SCREENS: readonly TerminalCredentialPromptCase[] = [ + // An agent SUMMARISING auth work it just finished, with its own composer on the last row. + // These pair an auth verb with an auth-flow phrase, which is the shape that used to match + // with no prompt terminator and no position requirement at all. + [ + 'codex summarising two-factor work', + [ + '• I implemented two-factor authentication for the login flow.', + ' The authenticator app now generates a 6-digit code.', + '› Ask Codex to do anything' + ] + ], + [ + 'claude summarising two-factor work', + [ + '· Added two-factor authentication. Tests for the authenticator app pass.', + '✳ Claude Code', + '> ' + ] + ], + [ + 'codex summarising a sign-in button', + [ + '• Added a Sign in with Google button; it logs the authorization result.', + '› Ask Codex to do anything' + ] + ], + [ + 'codex reporting an auth error it hit', + [ + ' └ ERROR: authentication required. Please sign in with the CLI.', + '› Ask Codex to do anything' + ] + ], + [ + 'agent asking whether MFA is wanted', + ['· Should the app require MFA, or is authentication via password enough?', '> '] + ], + [ + 'codex summarising a device-code flow it built', + [ + '• The OAuth flow now shows a device code and waits for authentication.', + '› Ask Codex to do anything' + ] + ], + [ + 'rg hit on auth documentation', + [' └ docs/auth.md:12: Users authenticate with the authenticator app.', '> '] + ], + [ + 'rg hits on a login component', + [ + ' └ src/Login.tsx:31: ', + ' └ src/Login.tsx:44: // authorization code exchange', + '› Ask Codex to do anything' + ] + ], + [ + 'codex reporting MFA tests passing', + ['• All MFA tests pass; authentication is wired end to end.', '› Ask Codex to do anything'] + ], + [ + 'codex quoting a build failure', + [ + "• The build failed: 'authorization required'. You need to log in with `vercel login`.", + '› Ask Codex to do anything' + ] + ], + [ + 'rg hit on a readme auth section', + [ + ' └ docs/auth.md:3: ## Authentication', + ' Users sign in with GitHub or an authenticator app.', + '> ' + ] + ], + [ + 'claude summarising an oauth change', + [ + '· Done — the OAuth login now requires authentication via the device code flow.', + '✳ Claude Code', + '> ' + ] + ], + [ + 'gemini summarising SSO work', + ['✦ Added SSO. Users authenticate with Okta; the sign in with SAML path is tested.', '◇ '] + ], + [ + 'opencode wrapping an auth summary', + [ + 'Added requireAuth middleware. Unauthenticated requests get 401; sign in with the', + 'token endpoint returns a JWT.', + '❯ ' + ] + ], + [ + 'stack trace over a codex composer', + [ + 'Error: authentication required', + ' at signInWithToken (auth.ts:22)', + '› Ask Codex to do anything' + ] + ], + [ + 'shell deploy failure', + [ + 'Running deploy...', + 'ERROR: authentication required', + 'Please sign in with the CLI and retry.', + 'exit code 1' + ] + ], + // Printed config whose bare `password:` label is not the screen's bottom row. + [ + 'printed kubernetes secret manifest', + ['kind: Secret', 'stringData:', ' password:', '› Ask Codex to do anything'] + ], + [ + 'printed signup form template', + [ + '• The signup form now has these fields:', + ' email:', + ' password:', + '› Ask Codex to do anything' + ] + ] +] diff --git a/src/shared/terminal-credential-prompt-detection.test.ts b/src/shared/terminal-credential-prompt-detection.test.ts index eb808d9118a..54e2de4e538 100644 --- a/src/shared/terminal-credential-prompt-detection.test.ts +++ b/src/shared/terminal-credential-prompt-detection.test.ts @@ -7,10 +7,8 @@ import { findCredentialPromptIndex, TERMINAL_CREDENTIAL_PROMPT_SENTINEL_RE } from './terminal-credential-prompt-detection' -import { - LEGITIMATE_AGENT_SCREENS, - LIVE_CREDENTIAL_SURFACES -} from './terminal-credential-prompt-corpus' +import { LEGITIMATE_AGENT_SCREENS } from './terminal-legitimate-agent-screens-corpus' +import { LIVE_CREDENTIAL_SURFACES } from './terminal-live-credential-surfaces-corpus' import { TERMINAL_TITLE_CLASSIFICATION_CORPUS } from './terminal-title-classification-corpus' function screen(lines: string[]): string { diff --git a/src/shared/terminal-legitimate-agent-screens-corpus.ts b/src/shared/terminal-legitimate-agent-screens-corpus.ts new file mode 100644 index 00000000000..ecaeabd7e86 --- /dev/null +++ b/src/shared/terminal-legitimate-agent-screens-corpus.ts @@ -0,0 +1,165 @@ +// The other side of the credential-prompt corpus: screens the guard MUST let through. A refusal +// here is worse than a missed prompt — the reason is unconditional, so it also pins an idle agent +// to `permission` in the agent-status store. +import type { TerminalCredentialPromptCase } from './terminal-live-credential-surfaces-corpus' +import { AGENT_AUTH_SUMMARY_SCREENS } from './terminal-agent-auth-summary-screens-corpus' + +export const LEGITIMATE_AGENT_SCREENS: readonly TerminalCredentialPromptCase[] = [ + // An agent narrating credential work and returning to its composer. + [ + 'codex narrating password hashing', + [ + '• I added bcrypt password hashing to src/auth/user.ts.', + '', + '› Ask Codex to do anything', + '', + ' gpt-6 medium · ~/repo' + ] + ], + [ + 'codex narrating api-key wiring', + ['• Wired the API key into .env.example and documented it.', '', '› Ask Codex to do anything'] + ], + [ + 'claude narrating login work', + ['· Updated the login form to use the new session cookie.', '', '✳ Claude Code', '', '> '] + ], + [ + 'codex narrating an oauth refresh', + [ + '• Done. The OAuth device-code flow now refreshes the access token.', + '', + '› Ask Codex to do anything' + ] + ], + [ + 'claude narrating a secret rotation', + ['· I rotated the client secret and pushed the change.', '', '> '] + ], + // An agent legitimately ASKING the user something credential-adjacent. + [ + 'agent asks where to store a password', + [ + '· Should I store the password in the .env file or in the keychain?', + '', + '✳ Claude Code', + '', + '> ' + ] + ], + [ + 'agent asks about reading an api key', + ['· Do you want me to read your api key from process.env.OPENAI_API_KEY?', '', '> '] + ], + [ + 'agent asks which auth provider', + ['· Which auth provider should the sign in page use?', '', '> '] + ], + [ + 'agent asks about a token in CI', + ['· I need to know: does your CI already have a personal access token?', '', '> '] + ], + [ + 'agent asks where a template goes', + ['· Where should I put the verification code template?', '', '> '] + ], + ['agent asks about OTP expiry', ['· Should the OTP expire after 5 minutes or 10?', '', '> ']], + [ + 'agent asks about 2FA delivery', + ['· Do you want 2FA codes emailed or via authenticator app?', '', '> '] + ], + [ + 'agent narrates an upcoming passphrase edit', + ['· Ready. Next I will enter the passphrase handling into the key loader.', '', '> '] + ], + [ + 'agent narrates an api-key edit mid-sentence', + ['· I will enter the API key into the vault once you confirm the vault name', '', '> '] + ], + [ + 'agent asks which secret key to rotate', + ['· Please tell me which secret key you want rotated first', '', '> '] + ], + // The user's own task prompt echoed above the composer. + ['echoed login task prompt', ['> Implement the login form', '', '· Working…']], + [ + 'echoed password-reset task prompt', + ['> add password reset via one-time code', '', '· Working…'] + ], + [ + 'echoed credentials task prompt', + ['> Refactor the credentials module', '', '✳ Claude Code', '', '> '] + ], + // Search output quoting credential-shaped source, the shape that broke earlier detectors. + [ + 'rg hit on a password call', + [ + ' └ src/auth.ts:42: const password = await promptPassword()', + '', + '› Ask Codex to do anything' + ] + ], + [ + 'rg hit on an api-key label literal', + [" └ 118: label: 'Enter your API key'", '', '› Ask Codex to do anything'] + ], + [ + 'rg hit on a password test name', + [" └ tests/auth.test.ts:9: it('prompts for password', () => {", '', '> '] + ], + [ + 'search narration quoting a prompt', + [' └ Search "enter your password" in src/', '', '› Ask Codex to do anything'] + ], + // A diff that ADDS a credential prompt string. + [ + 'diff adding an api-key log', + ['+ console.log("Enter your API key:")', '', '› Ask Codex to do anything'] + ], + ['diff adding a password prompt field', ['+ prompt: "Password:"', '', '> ']], + // Other terminal traffic. + [ + 'cursor approval menu', + [ + 'Run this command?', + ' cat ~/.ssh/id_rsa', + ' Run (once) (enter)', + ' Skip & tell the agent (esc)' + ] + ], + [ + 'vitest auth suite output', + [ + ' ✓ auth > rejects an expired access token (4 ms)', + ' ✓ auth > hashes the password with argon2 (9 ms)', + '', + 'Test Files 1 passed' + ] + ], + [ + 'jest login suite output', + ['PASS src/login.test.ts', '', ' ● login form › submits credentials', '', '> '] + ], + [ + 'git push rejection', + [ + 'remote: Support for password authentication was removed.', + 'fatal: Authentication failed', + '$ ' + ] + ], + ['clean git push', ['Everything up-to-date', '$ ']], + [ + 'rendered readme auth section', + ['## Authentication', '', 'Set `ORCA_API_KEY` in your environment before running.', '', '$ '] + ], + [ + 'agent narrating a failed gh auth', + [ + '• The gh CLI says authentication failed; I skipped the PR step.', + '', + '› Ask Codex to do anything' + ] + ], + ...AGENT_AUTH_SUMMARY_SCREENS +] diff --git a/src/shared/terminal-live-credential-surfaces-corpus.ts b/src/shared/terminal-live-credential-surfaces-corpus.ts new file mode 100644 index 00000000000..202a854863a --- /dev/null +++ b/src/shared/terminal-live-credential-surfaces-corpus.ts @@ -0,0 +1,91 @@ +// Screens a live credential or sign-in surface owns, i.e. every screen the guard MUST refuse. +// Shared by the shared-detector suite, the main-process wait-vocabulary suite and the renderer +// paste-lane suite: one corpus keeps the three lanes provably in agreement. + +export type TerminalCredentialPromptCase = readonly [name: string, lines: string[]] + +export const LIVE_CREDENTIAL_SURFACES: readonly TerminalCredentialPromptCase[] = [ + [ + 'antigravity device-code sign-in drawn over ready chrome (#19749)', + [ + 'Antigravity CLI', + 'gemini 3 pro (high)', + '~/orca/workspaces/orca/crash-closer', + '>', + '', + ' Sign in to Antigravity', + ' Open https://antigravity.google/device and enter the code: KXTD-9PQR', + ' Waiting for authentication…' + ] + ], + [ + 'antigravity auth-method menu over ready chrome', + [ + 'Antigravity CLI', + 'gemini 3 pro (high)', + '>', + '', + '? How would you like to authenticate?', + '❯ Sign in with Google', + ' Use an API key' + ] + ], + [ + 'api-key prompt under a complete Codex ready header', + [ + 'OpenAI Codex', + 'model: gpt-6', + 'directory: ~/repo', + '', + '› Ask Codex to do anything', + '', + 'Enter your API key:' + ] + ], + ['bare api-key ask', ['Enter your API key: ']], + ['vendor-qualified api-key ask with a caret', ['? Enter your Anthropic API key ›']], + ['bare password label', ['Password:']], + ['lowercase password label', ['password: ']], + ['sudo password', ['[sudo] password for neil:']], + ['ssh key passphrase', ["Enter passphrase for key '/Users/neil/.ssh/id_ed25519':"]], + ['git username', ["Username for 'https://github.com': "]], + ['git password', ["Password for 'https://neil@github.com': "]], + ['sms verification code', ['Enter the verification code we sent to your phone:']], + ['one-time code', ['Enter your one-time code:']], + [ + 'two-factor dialog', + ['Two-factor authentication', 'Enter the 6-digit code from your authenticator app:'] + ], + ['personal access token paste', ['Paste your personal access token here:']], + ['sign-in wall', ['Authentication required', 'Sign in with GitHub to continue']], + [ + 'oauth device-code flow', + [ + 'Please open the following url in your browser:', + ' https://github.com/login/device', + '', + 'and enter the code: ABCD-1234' + ] + ], + ['client secret', ['Enter client secret:']], + ['password confirmation', ['Re-enter password:']], + ['access token ask', ['Provide your access token:']], + ['otp ask', ['Type your OTP:']], + ['bare credentials label', ['credentials:']], + ['device code ask', ['Enter device code:']], + [ + 'gh auth login device code', + [ + '! First copy your one-time code: 1A2B-3C4D', + 'Press Enter to open github.com in your browser...' + ] + ], + [ + 'claude /login paste-code screen', + [ + "Browser didn't open? Use the url below to sign in:", + 'https://claude.ai/oauth/authorize?code=true', + 'Paste code here if prompted >' + ] + ] +] diff --git a/src/shared/terminal-visible-screen-projection.test.ts b/src/shared/terminal-visible-screen-projection.test.ts index 4cf78901aed..2bbd9d2ed21 100644 --- a/src/shared/terminal-visible-screen-projection.test.ts +++ b/src/shared/terminal-visible-screen-projection.test.ts @@ -10,10 +10,8 @@ import { visibleNonBlankTerminalLines } from './terminal-visible-screen-projection' import { findCredentialPromptIndex } from './terminal-credential-prompt-detection' -import { - LEGITIMATE_AGENT_SCREENS, - LIVE_CREDENTIAL_SURFACES -} from './terminal-credential-prompt-corpus' +import { LEGITIMATE_AGENT_SCREENS } from './terminal-legitimate-agent-screens-corpus' +import { LIVE_CREDENTIAL_SURFACES } from './terminal-live-credential-surfaces-corpus' const VIEWPORT = { cols: 120, rows: 24 }