From c7850eb59c5ad90f85faee8036589efa00dfbcb2 Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:49:34 -0700 Subject: [PATCH] fix(terminals): prevent retired workers from cold-resuming (#13708) --- config/reliability-gates.jsonc | 30 +- .../terminal-pane/pty-connection.test.ts | 10 +- .../terminal/terminal-tab-actions.test.ts | 51 ++ .../terminal/terminal-tab-actions.ts | 13 + ...completed-worker-retirement-resume.spec.ts | 521 +++++++++++++++ ...eted-worker-retirement-resume.unit.test.ts | 610 ++++++++++++++++++ .../completed-worker-retirement-fixture.ts | 194 ++++++ 7 files changed, 1424 insertions(+), 5 deletions(-) create mode 100644 tests/e2e/completed-worker-retirement-resume.spec.ts create mode 100644 tests/e2e/completed-worker-retirement-resume.unit.test.ts create mode 100644 tests/e2e/helpers/completed-worker-retirement-fixture.ts diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 16e479b2666..f5e5a39caeb 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -1969,7 +1969,7 @@ "invariant": "Close permanently removes the owned provider session, agent descendants, and resume authority even when no TerminalPane is mounted; a terminating id remains reserved through natural exit, duplicate callers await the same completion, and immediate teardown upgrades any graceful request without signalling a recycled PID or a descendant tree after root ownership is lost; process-table work is locale-stable, bounded, fresh for each post-start request, same-turn coalesced, and begins within the requesting caller's deadline, including bulk worktree cleanup; detach and park preserve ownership; aliases prevent a detached agent's immutable physical pane key from being retired with its former tab.", "oracle": "Capture the exact PTY before parking, prove it remains listed while the view is absent, close through the product state boundary, and poll the provider inventory until that exact ID disappears; an agent-marked PTY's detached-pgid child is alive before close and absent afterward; unit tests keep a naturally exited id reserved without re-killing its PID or signalling its captured tree, upgrade pending and post-snapshot graceful kills to immediate, force ps into the C locale, coalesce each bounded bulk-shutdown batch, share duplicate teardown completion, coalesce 20 same-turn process-table requests, start one shared successor without waiting for the prior scan, terminate cyclic-looking traversal, retain the source scan's timestamp, bound both read phases, avoid ambiguous SIGKILL, and assert canonical owner dedupe, exact pane tombstones, chained detach transfer, and restart alias restoration.", "commands": [ - "pnpm dlx node@24 ./node_modules/vitest/vitest.mjs run --config config/vitest.config.ts src/main/agent-hooks/server-pane-authority.test.ts src/main/ipc/agent-hooks.test.ts src/main/ipc/agent-pane-authority-ownership.test.ts src/main/ipc/pty-management.test.ts src/main/persistence.test.ts src/renderer/src/store/slices/agent-pane-authority.test.ts src/renderer/src/store/slices/terminal-pane-detach-agent-identity.test.ts src/renderer/src/store/slices/terminal-tab-retirement.test.ts src/renderer/src/store/slices/terminal-tab-retirement-store.test.ts src/renderer/src/components/shared/kill-all-terminal-surfaces.test.ts", + "pnpm dlx node@24 ./node_modules/vitest/vitest.mjs run --config config/vitest.config.ts src/main/agent-hooks/server-pane-authority.test.ts src/main/ipc/agent-hooks.test.ts src/main/ipc/agent-pane-authority-ownership.test.ts src/main/ipc/pty-management.test.ts src/main/persistence.test.ts src/renderer/src/store/slices/agent-pane-authority.test.ts src/renderer/src/store/slices/terminal-pane-detach-agent-identity.test.ts src/renderer/src/store/slices/terminal-tab-retirement.test.ts src/renderer/src/store/slices/terminal-tab-retirement-store.test.ts tests/e2e/completed-worker-retirement-resume.unit.test.ts src/renderer/src/components/shared/kill-all-terminal-surfaces.test.ts", "pnpm exec vitest run --config config/vitest.config.ts src/main/pty-descendant-termination.test.ts src/main/daemon/session.test.ts src/main/daemon/terminal-host.test.ts src/main/providers/local-pty-provider.test.ts src/main/runtime/worktree-teardown.test.ts", "pnpm run test:e2e -- tests/e2e/terminal-parked-close-retirement.spec.ts --workers=1", "pnpm run test:e2e -- tests/e2e/agent-descendant-process-kill.spec.ts --workers=1" @@ -1989,11 +1989,18 @@ "src/renderer/src/store/slices/terminal-pane-detach-agent-identity.test.ts", "src/renderer/src/store/slices/terminal-tab-retirement.test.ts", "src/renderer/src/store/slices/terminal-tab-retirement-store.test.ts", + "tests/e2e/completed-worker-retirement-resume.unit.test.ts", "src/renderer/src/components/shared/kill-all-terminal-surfaces.test.ts", "tests/e2e/terminal-parked-close-retirement.spec.ts", "tests/e2e/agent-descendant-process-kill.spec.ts" ], "assertionRefs": [ + { + "file": "tests/e2e/completed-worker-retirement-resume.unit.test.ts", + "assertions": [ + "explicit close revokes the exact live recovery record both before and after PTY exit removes the tab, while PTY exit alone preserves interrupted-session recovery" + ] + }, { "file": "tests/e2e/terminal-parked-close-retirement.spec.ts", "assertions": [ @@ -2711,11 +2718,13 @@ "oracle": "Renderer-state tests assert provider-session ownership across preserved and queued panes. Main/provider contracts assert atomic attach-only adoption, stable host/worktree/tab/leaf identity, no fresh spawn on adoption, and safe paired-runtime cancellation. The Electron oracle creates inactive runtime-owned Codex and Setup PTYs plus an unrelated canary, seeds an exact resumable provider session, removes only the target renderer projections, and activates the workspace. It requires byte-stable handle, PTY, incarnation, tab, leaf, process PID, renderer graph, persisted binding, runtime id, graph epoch, and daemon PID across first mount and reload; PID-specific DOM keyboard I/O must remain live with one launch, zero resume argv, zero signals, zero interruption text, and no canary mutation.", "commands": [ "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/lib/resume-sleeping-agent-session.test.ts", + "pnpm exec vitest run --config config/vitest.config.ts tests/e2e/completed-worker-retirement-resume.unit.test.ts --reporter=verbose", "pnpm exec vitest run --config config/vitest.config.ts src/renderer/src/lib/resume-sleeping-agent-session.test.ts src/main/providers/local-pty-provider.test.ts src/main/daemon/terminal-host.test.ts src/main/daemon/daemon-pty-adapter.test.ts src/main/ipc/pty.test.ts src/main/runtime/orca-runtime.test.ts src/renderer/src/lib/pane-manager/pane-fit.test.ts src/renderer/src/components/terminal-pane/pty-connection.test.ts src/renderer/src/components/terminal-pane/pty-transport.test.ts", "pnpm exec electron-vite build --mode e2e && SKIP_BUILD=1 pnpm exec playwright test tests/e2e/live-background-terminal-mount-authority.spec.ts --config tests/playwright.config.ts --project electron-headless --workers=1" ], "testFiles": [ "src/renderer/src/lib/resume-sleeping-agent-session.test.ts", + "tests/e2e/completed-worker-retirement-resume.unit.test.ts", "src/main/providers/local-pty-provider.test.ts", "src/main/daemon/terminal-host.test.ts", "src/main/daemon/daemon-pty-adapter.test.ts", @@ -2727,6 +2736,13 @@ "tests/e2e/live-background-terminal-mount-authority.spec.ts" ], "assertionRefs": [ + { + "file": "tests/e2e/completed-worker-retirement-resume.unit.test.ts", + "assertions": [ + "completed and explicitly retired workers retain no provider-resume authority on first workspace activation", + "the original tab, leaf, PTY, provider session, release state, replacement count, queued resume argv, emitted restored banner, and unrelated canary remain exact across restart" + ] + }, { "file": "src/renderer/src/lib/resume-sleeping-agent-session.test.ts", "assertions": [ @@ -7867,16 +7883,24 @@ "invariant": "A settled Dispatch may close only its one coordinator-created terminal lease. Explicit reuse, real user input, retain, identity or host change, ambiguity, and another resource for the same exact host/pane/process must fence closure. Output preservation and the requested-to-releasing transition are atomic, archives remain readable without the provider file, retries resume idempotently, and orchestration reset removes archive and authority state.", "oracle": "Record release intent for a settled owner, attempt exact reuse before close, and require worker-start to fail with terminal_release_in_progress while the terminal stays open; then release the original owner exactly once. Race retain and real user input against a controlled archive promise and require no committed archive or close. Change host or process identity and inject duplicate resource evidence to require retention. Freeze a structured transcript, delete its source file, and require archived worker-read to return the same bounded redacted messages. Restart a pending mutation, reset orchestration state, and create 50 resources while asserting replay convergence, zero orphan rows, two-query worker listing, and no unrelated close.", "commands": [ - "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/rpc/methods/orchestration-worker-release.test.ts src/main/runtime/rpc/methods/orchestration-worker-release-recovery.test.ts src/main/runtime/rpc/orchestration-mutation-ledger.test.ts src/main/runtime/orchestration/worker-transcript-read.test.ts src/renderer/src/lib/worker-terminal-takeover-report.test.ts --reporter=dot" + "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/rpc/methods/orchestration-worker-release.test.ts src/main/runtime/rpc/methods/orchestration-worker-release-recovery.test.ts src/main/runtime/rpc/orchestration-mutation-ledger.test.ts src/main/runtime/orchestration/worker-transcript-read.test.ts src/renderer/src/lib/worker-terminal-takeover-report.test.ts --reporter=dot", + "pnpm exec vitest run --config config/vitest.config.ts tests/e2e/completed-worker-retirement-resume.unit.test.ts --reporter=verbose" ], "testFiles": [ "src/main/runtime/rpc/methods/orchestration-worker-release.test.ts", "src/main/runtime/rpc/methods/orchestration-worker-release-recovery.test.ts", "src/main/runtime/rpc/orchestration-mutation-ledger.test.ts", "src/main/runtime/orchestration/worker-transcript-read.test.ts", - "src/renderer/src/lib/worker-terminal-takeover-report.test.ts" + "src/renderer/src/lib/worker-terminal-takeover-report.test.ts", + "tests/e2e/completed-worker-retirement-resume.unit.test.ts" ], "assertionRefs": [ + { + "file": "tests/e2e/completed-worker-retirement-resume.unit.test.ts", + "assertions": [ + "an already-exited succeeded Dispatch reaches released while its late exact close revokes the missing tab's stale resume record" + ] + }, { "file": "src/main/runtime/rpc/methods/orchestration-worker-release.test.ts", "assertions": [ diff --git a/src/renderer/src/components/terminal-pane/pty-connection.test.ts b/src/renderer/src/components/terminal-pane/pty-connection.test.ts index 9391b6d614a..2ff73043cc3 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection.test.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection.test.ts @@ -11599,11 +11599,12 @@ describe('connectPanePty', () => { expect(mockStoreState.sleepingAgentSessionsByPaneKey[paneKey]).toBeDefined() }) - it('does not write the restored banner through xterm bytes for sidebar-resumed startup commands', async () => { + it('forwards one sidebar resume spawn without writing the restored banner through xterm', async () => { const { connectPanePty } = await import('./pty-connection') const transport = createMockTransport('pty-1') transportFactoryQueue.push(transport) const pane = createPane(1) + const providerSession = { key: 'session_id', id: 'codex-session-1' } as const connectPanePty( pane as never, @@ -11611,6 +11612,7 @@ describe('connectPanePty', () => { createDeps({ startup: { command: "codex 'resume' 'codex-session-1'", + resumeProviderSession: providerSession, showSessionRestoredBanner: true } }) as never @@ -11626,7 +11628,11 @@ describe('connectPanePty', () => { expect.stringContaining('--- session restored ---'), expect.any(Function) ) - expect(createdTransportOptions[0]?.command).toBe("codex 'resume' 'codex-session-1'") + expect(transport.connect).toHaveBeenCalledTimes(1) + expect(createdTransportOptions[0]).toMatchObject({ + command: "codex 'resume' 'codex-session-1'", + resumeProviderSession: providerSession + }) }) it('does not consume the sleeping record when daemon reattach returns a live snapshot', async () => { diff --git a/src/renderer/src/components/terminal/terminal-tab-actions.test.ts b/src/renderer/src/components/terminal/terminal-tab-actions.test.ts index a8747ec81ff..4b7557ba057 100644 --- a/src/renderer/src/components/terminal/terminal-tab-actions.test.ts +++ b/src/renderer/src/components/terminal/terminal-tab-actions.test.ts @@ -183,6 +183,57 @@ describe('closeTerminalTab', () => { isWebTerminalSurfaceTabIdMock.mockReturnValue(false) }) + it('retires exact resume authority when explicit cleanup arrives after the tab disappeared', () => { + const closeTab = vi.fn() + const onClosed = vi.fn() + getStateMock.mockReturnValue({ + tabsByWorktree: {}, + unifiedTabsByWorktree: {}, + closeTab + }) + + closeTerminalTab('retired-worker-tab', { + hostCloseReason: 'cleanup', + localPtyTeardownOwnedExternally: true, + onClosed + }) + + expect(closeTab).toHaveBeenCalledWith('retired-worker-tab', { + reason: 'cleanup', + localPtyTeardownOwnedExternally: true + }) + expect(onClosed).toHaveBeenCalledOnce() + }) + + it('retires exact resume authority when a user close arrives after the tab disappeared', () => { + const closeTab = vi.fn() + getStateMock.mockReturnValue({ + tabsByWorktree: {}, + unifiedTabsByWorktree: {}, + closeTab + }) + + closeTerminalTab('retired-worker-tab') + + expect(closeTab).toHaveBeenCalledWith('retired-worker-tab', { reason: 'user' }) + }) + + it.each([ + ['local PTY exit', { reason: 'pty-exit' as const }], + ['paired host PTY exit', { hostCloseReason: 'pty-exit' as const }] + ])('preserves missing-tab recovery authority for %s', (_label, options) => { + const closeTab = vi.fn() + getStateMock.mockReturnValue({ + tabsByWorktree: {}, + unifiedTabsByWorktree: {}, + closeTab + }) + + closeTerminalTab('unexpectedly-lost-tab', options) + + expect(closeTab).not.toHaveBeenCalled() + }) + it('delegates host-backed terminal closes to the paired runtime', () => { const closeTab = vi.fn() isWebRuntimeSessionActiveMock.mockReturnValue(true) diff --git a/src/renderer/src/components/terminal/terminal-tab-actions.ts b/src/renderer/src/components/terminal/terminal-tab-actions.ts index 55934e7ae88..8216696d217 100644 --- a/src/renderer/src/components/terminal/terminal-tab-actions.ts +++ b/src/renderer/src/components/terminal/terminal-tab-actions.ts @@ -65,6 +65,19 @@ export function closeTerminalTab( ) const target = resolveTerminalCloseTarget(state, tabId, precomputedCloseState) if (!target) { + const closeReason = options?.reason ?? options?.hostCloseReason ?? 'user' + if (closeReason !== 'pty-exit') { + // Why: late explicit cleanup must still revoke tab-scoped resume authority after PTY exit removed the row. + state.closeTab(tabId, { + reason: closeReason, + ...(options?.localPtyTeardownOwnedExternally + ? { localPtyTeardownOwnedExternally: true } + : {}), + ...(options?.precomputedRetirementPlan + ? { precomputedRetirementPlan: options.precomputedRetirementPlan } + : {}) + }) + } options?.onClosed?.() return } diff --git a/tests/e2e/completed-worker-retirement-resume.spec.ts b/tests/e2e/completed-worker-retirement-resume.spec.ts new file mode 100644 index 00000000000..5540c4d6896 --- /dev/null +++ b/tests/e2e/completed-worker-retirement-resume.spec.ts @@ -0,0 +1,521 @@ +import { test as base, expect } from './helpers/orca-app' +import { ensureTerminalVisible, waitForActiveWorktree, waitForSessionReady } from './helpers/store' +import { + waitForActivePaneHookDescriptor, + waitForActivePanePtyId, + waitForActiveTerminalManager +} from './helpers/terminal' +import { + cleanupCompletedWorkerFixture, + clearCompletedWorkerLedger, + completedWorkerLaunchEnv, + listRuntimeTerminals, + readCompletedWorkerDispatchCapability, + readCompletedWorkerLedger, + readPersistedWorkerRecoveryRecord, + runBuiltOrcaCli, + seedCurrentCodexTranscript, + terminalIdentity +} from './helpers/completed-worker-retirement-fixture' +import { RuntimeClient } from '../../src/cli/runtime-client' +import type { RuntimeTerminalRead, RuntimeTerminalSummary } from '../../src/shared/runtime-types' +import { splitWorktreeIdForFilesystem } from '../../src/shared/worktree-id' + +const PROVIDER_SESSION_ID = '019feb51-2269-71c2-89c6-faa8dc65c8dc' + +const test = base.extend({ + launchEnv: [completedWorkerLaunchEnv, { option: true }] +}) + +test.describe.configure({ mode: 'serial' }) + +test.afterAll(() => { + cleanupCompletedWorkerFixture() +}) + +for (const closeMode of ['terminal-close-cli', 'worker-release'] as const) { + test(`completed background worker ${closeMode} retires resume authority before first activation`, async ({ + orcaPage, + electronApp + }) => { + test.setTimeout(180_000) + clearCompletedWorkerLedger() + await waitForSessionReady(orcaPage) + const coordinatorWorktreeId = await waitForActiveWorktree(orcaPage) + await ensureTerminalVisible(orcaPage) + await waitForActiveTerminalManager(orcaPage) + await waitForActivePanePtyId(orcaPage) + await orcaPage.evaluate(async () => { + await window.__store?.getState().updateSettings({ + disabledTuiAgents: [], + terminalHiddenViewParking: false + }) + }) + + const userDataDir = await electronApp.evaluate(({ app }) => app.getPath('userData')) + const isolatedHome = await electronApp.evaluate(({ app }) => app.getPath('home')) + const client = new RuntimeClient(userDataDir, 30_000, null, null) + const coordinatorPane = await waitForActivePaneHookDescriptor(orcaPage) + const coordinatorResolved = await client.call<{ terminal: { handle: string } }>( + 'terminal.resolvePane', + { paneKey: coordinatorPane.paneKey } + ) + const coordinatorHandle = coordinatorResolved.result.terminal.handle + const coordinator = (await listRuntimeTerminals(client)).find( + (terminal) => terminal.handle === coordinatorHandle + ) + if (!coordinator) { + throw new Error('Coordinator terminal was not runtime-visible') + } + const coordinatorBefore = terminalIdentity(coordinator) + + let targetWorktreeId: string | null = null + await expect + .poll( + async () => { + const listed = await client.call<{ worktrees: { id: string }[] }>('worktree.list', {}) + const rendererWorktreeIds = await orcaPage.evaluate(() => + Object.values(window.__store?.getState().worktreesByRepo ?? {}) + .flat() + .map((worktree) => worktree.id) + ) + targetWorktreeId = + listed.result.worktrees.find( + (worktree) => + worktree.id !== coordinatorWorktreeId && rendererWorktreeIds.includes(worktree.id) + )?.id ?? null + return targetWorktreeId + }, + { timeout: 60_000, message: 'runtime never registered the secondary worktree' } + ) + .not.toBeNull() + if (!targetWorktreeId) { + throw new Error('The seeded repository did not expose its secondary worktree') + } + const targetWorktreePath = splitWorktreeIdForFilesystem(targetWorktreeId)?.worktreePath + if (!targetWorktreePath) { + throw new Error('The secondary worktree did not expose a filesystem path') + } + + expect( + await orcaPage.evaluate( + (worktreeId) => window.__store?.getState().everActivatedWorktreeIds.has(worktreeId), + targetWorktreeId + ) + ).toBe(false) + + const run = await client.call<{ run: { id: string } }>('orchestration.runCreate', { + objective: 'Retire one completed background worker', + from: coordinatorHandle + }) + const task = await client.call<{ task: { id: string } }>('orchestration.taskCreate', { + spec: 'Report completion, then exit normally', + run: run.result.run.id, + callerTerminalHandle: coordinatorHandle + }) + const started = await client.call<{ + dispatchId: string + state: string + effects: { kind: string; role?: string; id?: string }[] + }>('orchestration.workerStart', { + task: task.result.task.id, + from: coordinatorHandle, + worktree: `id:${String(targetWorktreeId)}`, + agent: 'codex', + timeoutMs: 30_000 + }) + expect(started.result.state).toBe('ready') + const workerHandle = started.result.effects.find( + (effect) => effect.kind === 'terminal' && effect.role === 'agent' + )?.id + if (!workerHandle) { + throw new Error('worker-start did not return its agent terminal') + } + + let worker: RuntimeTerminalSummary | undefined + await expect + .poll( + async () => { + worker = (await listRuntimeTerminals(client)).find( + (terminal) => terminal.handle === workerHandle + ) + return worker?.ptyId ?? null + }, + { timeout: 30_000, message: 'background worker never published its PTY identity' } + ) + .not.toBeNull() + if (!worker?.ptyId || !worker.incarnationId) { + throw new Error('Background worker did not publish exact PTY identity') + } + const workerBefore = terminalIdentity(worker) + const workerPaneKey = `${worker.tabId}:${worker.leafId}` + expect(worker.worktreeId).toBe(targetWorktreeId) + await orcaPage.evaluate( + ({ tabId, worktreeId }) => { + window.dispatchEvent( + new CustomEvent('orca-background-mount-terminal-worktree', { + detail: { worktreeId, tabIds: [tabId] } + }) + ) + }, + { tabId: worker.tabId, worktreeId: targetWorktreeId } + ) + await expect + .poll(() => + orcaPage.evaluate((tabId) => Boolean(window.__paneManagers?.get(tabId)), workerBefore.tabId) + ) + .toBe(true) + expect( + await orcaPage.evaluate( + (worktreeId) => window.__store?.getState().everActivatedWorktreeIds.has(worktreeId), + targetWorktreeId + ) + ).toBe(false) + await expect + .poll(() => readCompletedWorkerLedger().filter((event) => event.event === 'spawn')) + .toHaveLength(1) + let dispatchCapability: string | null = null + await expect + .poll(() => { + dispatchCapability = readCompletedWorkerDispatchCapability() + return dispatchCapability + }) + .not.toBeNull() + if (!dispatchCapability) { + throw new Error('Background worker did not receive its dispatch capability') + } + + const transcriptPath = seedCurrentCodexTranscript( + isolatedHome, + PROVIDER_SESSION_ID, + targetWorktreePath + ) + + await orcaPage.evaluate( + ({ paneKey, providerSessionId, tabId, terminalHandle, transcriptPath, worktreeId }) => { + const state = window.__store?.getState() + if (!state) { + throw new Error('Renderer store unavailable') + } + const providerSession = { + key: 'session_id' as const, + id: providerSessionId, + transcriptPath + } + const metadata = { tabId, worktreeId, terminalHandle } + const recovery = { + providerSession, + launchConfig: { + agentCommand: 'codex', + agentArgs: '--dangerously-bypass-approvals-and-sandbox', + agentEnv: {} + } + } + state.setAgentStatus( + paneKey, + { state: 'working', prompt: 'Report completion, then exit normally', agentType: 'codex' }, + 'Completed background worker', + undefined, + metadata, + recovery + ) + state.setAgentStatus( + paneKey, + { state: 'done', prompt: 'Report completion, then exit normally', agentType: 'codex' }, + 'Completed background worker', + undefined, + metadata, + recovery + ) + }, + { + paneKey: workerPaneKey, + providerSessionId: PROVIDER_SESSION_ID, + tabId: worker.tabId, + terminalHandle: workerHandle, + transcriptPath, + worktreeId: targetWorktreeId + } + ) + + const expectedRecovery = { + origin: 'live', + state: 'working', + providerSessionId: PROVIDER_SESSION_ID + } + await expect + .poll(() => + orcaPage.evaluate((paneKey) => { + const record = window.__store?.getState().sleepingAgentSessionsByPaneKey[paneKey] + return record + ? { + origin: record.origin, + state: record.state, + providerSessionId: record.providerSession.id + } + : null + }, workerPaneKey) + ) + .toEqual(expectedRecovery) + await expect + .poll( + () => { + const record = readPersistedWorkerRecoveryRecord(userDataDir, workerPaneKey) + return record + ? { + origin: record.origin, + state: record.state, + providerSessionId: record.providerSession?.id + } + : null + }, + { timeout: 30_000 } + ) + .toEqual(expectedRecovery) + + const completed = await client.call<{ message: { type: string } }>( + 'orchestration.send', + { + from: workerHandle, + subject: 'Completed', + body: 'The fixture completed. It found no work. Nothing remains.', + type: 'worker_done', + payload: JSON.stringify({ + taskId: task.result.task.id, + dispatchId: started.result.dispatchId, + outcome: 'succeeded' + }) + }, + { orchestrationCapability: dispatchCapability } + ) + expect(completed.result.message.type).toBe('worker_done') + await expect + .poll( + async () => { + const dispatch = await client.call<{ dispatch: { status: string } | null }>( + 'orchestration.dispatchShow', + { task: task.result.task.id } + ) + const tasks = await client.call<{ tasks: { id: string; status: string }[] }>( + 'orchestration.taskList', + { run: run.result.run.id } + ) + return { + dispatch: dispatch.result.dispatch?.status ?? null, + task: + tasks.result.tasks.find((candidate) => candidate.id === task.result.task.id) + ?.status ?? null + } + }, + { timeout: 30_000, message: 'worker completion never settled its task and dispatch' } + ) + .toEqual({ dispatch: 'completed', task: 'completed' }) + + await client.call('terminal.send', { + terminal: workerHandle, + text: 'ORCA_E2E_EXIT_AFTER_DONE', + enter: true + }) + await expect + .poll(() => readCompletedWorkerLedger().filter((event) => event.event === 'normal-exit')) + .toHaveLength(1) + expect( + await orcaPage.evaluate( + ({ paneKey, tabId, worktreeId }) => { + const state = window.__store?.getState() + return { + tabPresent: Boolean(state?.tabsByWorktree[worktreeId]?.some((tab) => tab.id === tabId)), + recoveryPresent: Boolean(state?.sleepingAgentSessionsByPaneKey[paneKey]) + } + }, + { paneKey: workerPaneKey, tabId: workerBefore.tabId, worktreeId: targetWorktreeId } + ) + ).toEqual({ tabPresent: true, recoveryPresent: true }) + + await orcaPage.evaluate( + ({ paneKey, tabId, worktreeId }) => { + const store = window.__store + if (!store) { + throw new Error('Renderer store unavailable') + } + type Transition = { tabPresent: boolean; recoveryPresent: boolean } + const e2eWindow = window as typeof window & { + __orcaRetiredWorkerTransitions?: Transition[] + __orcaRetiredWorkerUnsubscribe?: () => void + } + const transitions: Transition[] = [ + { + tabPresent: Boolean( + store.getState().tabsByWorktree[worktreeId]?.some((tab) => tab.id === tabId) + ), + recoveryPresent: Boolean(store.getState().sleepingAgentSessionsByPaneKey[paneKey]) + } + ] + e2eWindow.__orcaRetiredWorkerTransitions = transitions + e2eWindow.__orcaRetiredWorkerUnsubscribe = store.subscribe((state) => { + const next = { + tabPresent: Boolean(state.tabsByWorktree[worktreeId]?.some((tab) => tab.id === tabId)), + recoveryPresent: Boolean(state.sleepingAgentSessionsByPaneKey[paneKey]) + } + const previous = transitions.at(-1) + if ( + !previous || + previous.tabPresent !== next.tabPresent || + previous.recoveryPresent !== next.recoveryPresent + ) { + transitions.push(next) + } + }) + }, + { paneKey: workerPaneKey, tabId: workerBefore.tabId, worktreeId: targetWorktreeId } + ) + + if (closeMode === 'terminal-close-cli') { + const closed = runBuiltOrcaCli(['terminal', 'close', '--terminal', workerHandle, '--json'], { + userDataDir, + cwd: process.cwd() + }) + expect(closed).toMatchObject({ + ok: true, + result: { + close: { + handle: workerHandle, + tabId: workerBefore.tabId, + ptyKilled: true + } + } + }) + } else { + const release = await client.call<{ + dispatchId: string + state: string + processAction: string + }>('orchestration.workerRelease', { dispatch: started.result.dispatchId }) + expect(release.result).toMatchObject({ + dispatchId: started.result.dispatchId, + state: 'released', + processAction: 'closed_agent_terminal' + }) + } + await expect + .poll(() => + orcaPage.evaluate(() => { + type Transition = { tabPresent: boolean; recoveryPresent: boolean } + return (window as typeof window & { __orcaRetiredWorkerTransitions?: Transition[] }) + .__orcaRetiredWorkerTransitions + }) + ) + .toEqual( + expect.arrayContaining([ + { tabPresent: true, recoveryPresent: true }, + { tabPresent: false, recoveryPresent: false } + ]) + ) + await orcaPage.evaluate(() => { + const e2eWindow = window as typeof window & { __orcaRetiredWorkerUnsubscribe?: () => void } + e2eWindow.__orcaRetiredWorkerUnsubscribe?.() + delete e2eWindow.__orcaRetiredWorkerUnsubscribe + }) + await expect + .poll(() => + orcaPage.evaluate( + (paneKey) => window.__store?.getState().sleepingAgentSessionsByPaneKey[paneKey] ?? null, + workerPaneKey + ) + ) + .toBeNull() + + await orcaPage.evaluate(() => window.dispatchEvent(new Event('beforeunload'))) + await expect + .poll(() => + orcaPage.evaluate(async (paneKey) => { + const session = await window.api.session.get() + return session.sleepingAgentSessionsByPaneKey?.[paneKey] ?? null + }, workerPaneKey) + ) + .toBeNull() + await orcaPage.evaluate(() => window.api.session.flush()) + expect(readPersistedWorkerRecoveryRecord(userDataDir, workerPaneKey)).toBeNull() + + await orcaPage.reload() + await waitForSessionReady(orcaPage) + + const beforeActivation = await orcaPage.evaluate((worktreeId) => { + const state = window.__store?.getState() + return { + everActivated: state?.everActivatedWorktreeIds.has(worktreeId) ?? false, + tabCount: state?.tabsByWorktree[worktreeId]?.length ?? 0, + pendingStartupCount: Object.keys(state?.pendingStartupByTabId ?? {}).length + } + }, targetWorktreeId) + expect(beforeActivation).toEqual({ everActivated: false, tabCount: 0, pendingStartupCount: 0 }) + + const targetCard = orcaPage + .locator(`[data-worktree-id="${String(targetWorktreeId)}"]`) + .first() + .locator('[data-worktree-card-surface]') + await targetCard.evaluate((element: HTMLElement) => element.click()) + await expect + .poll(() => orcaPage.evaluate(() => window.__store?.getState().activeWorktreeId)) + .toBe(targetWorktreeId) + await waitForActiveTerminalManager(orcaPage) + await waitForActivePanePtyId(orcaPage) + const activatedPane = await waitForActivePaneHookDescriptor(orcaPage) + expect(activatedPane.worktreeId).toBe(targetWorktreeId) + const activatedResolved = await client.call<{ terminal: { handle: string } }>( + 'terminal.resolvePane', + { paneKey: activatedPane.paneKey } + ) + await expect + .poll( + async () => { + const read = await client.call<{ terminal: RuntimeTerminalRead }>('terminal.read', { + terminal: activatedResolved.result.terminal.handle, + limit: 50 + }) + return read.result.terminal.tail.join('\n') + }, + { timeout: 30_000, message: 'activated fallback terminal never produced output' } + ) + .not.toBe('') + + const spawnEvents = readCompletedWorkerLedger().filter((event) => event.event === 'spawn') + expect(spawnEvents).toHaveLength(1) + expect( + spawnEvents.filter( + (event) => event.args?.includes('resume') && event.args?.includes(PROVIDER_SESSION_ID) + ) + ).toEqual([]) + await expect(orcaPage.locator('.session-restored-banner')).toHaveCount(0) + + const afterActivation = await orcaPage.evaluate( + ({ originalTabId, worktreeId }) => { + const state = window.__store?.getState() + const tabs = state?.tabsByWorktree[worktreeId] ?? [] + return { + everActivated: state?.everActivatedWorktreeIds.has(worktreeId) ?? false, + originalTabPresent: tabs.some((tab) => tab.id === originalTabId), + replacementTabCount: tabs.filter((tab) => tab.id !== originalTabId).length, + pendingResumeSessionIds: Object.values(state?.pendingStartupByTabId ?? {}).flatMap( + (startup) => (startup.resumeProviderSession ? [startup.resumeProviderSession.id] : []) + ), + resumeClaimCount: Object.keys(state?.automaticAgentResumeClaimsByTabId ?? {}).length + } + }, + { originalTabId: workerBefore.tabId, worktreeId: targetWorktreeId } + ) + expect(afterActivation).toEqual({ + everActivated: true, + originalTabPresent: false, + replacementTabCount: 1, + pendingResumeSessionIds: [], + resumeClaimCount: 0 + }) + + const coordinatorAfter = (await listRuntimeTerminals(client)).find( + (terminal) => terminal.handle === coordinatorHandle + ) + expect(coordinatorAfter ? terminalIdentity(coordinatorAfter) : null).toEqual(coordinatorBefore) + expect(coordinatorAfter?.worktreeId).toBe(coordinatorWorktreeId) + }) +} diff --git a/tests/e2e/completed-worker-retirement-resume.unit.test.ts b/tests/e2e/completed-worker-retirement-resume.unit.test.ts new file mode 100644 index 00000000000..03292dfb0e6 --- /dev/null +++ b/tests/e2e/completed-worker-retirement-resume.unit.test.ts @@ -0,0 +1,610 @@ +import path from 'node:path' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { SleepingAgentSessionRecord } from '../../src/shared/agent-session-resume' +import { makePaneKey } from '../../src/shared/stable-pane-id' +import { tokenizeStartupCommand } from '../../src/shared/tui-agent-startup-shell' +import { parseWorkspaceSession } from '../../src/shared/workspace-session-schema' +import type { TerminalTab, Worktree } from '../../src/shared/types' +import { OrchestrationDb } from '../../src/main/runtime/orchestration/db' +import { OrcaRuntimeService } from '../../src/main/runtime/orca-runtime' +import type { RpcContext } from '../../src/main/runtime/rpc/core' +import { ORCHESTRATION_METHODS } from '../../src/main/runtime/rpc/methods/orchestration' +import { closeTerminalTab } from '@/components/terminal/terminal-tab-actions' +import { seedStartupSessionRestoredBanner } from '@/components/terminal-pane/session-restored-banner-pane-state' +import { + resolveLegacyWorkerTerminalRecoveryAction, + rollbackLegacyWorkerTerminalSurfaceInStore +} from '@/hooks/legacy-worker-terminal-recovery-event' +import { useAppStore, type AppState } from '@/store' +import { buildWorkspaceSessionPayload } from '@/lib/workspace-session' +import { activateAndRevealWorktree } from '@/lib/worktree-activation' +import { resumeSleepingAgentSessionsForWorktree } from '@/lib/resume-sleeping-agent-session' + +const PROVIDER_SESSION_ID = '019feb51-2269-71c2-89c6-faa8dc65c8dc' +const ORIGINAL_TAB_ID = '1c897bc8-973b-47b4-9449-ac5fc6b726c3' +const ORIGINAL_LEAF_ID = '0526f763-6729-49af-adf8-85ddbcf2b4e7' +const ORIGINAL_PANE_KEY = makePaneKey(ORIGINAL_TAB_ID, ORIGINAL_LEAF_ID) +const ORIGINAL_PTY_ID = 'pty-background-worker' +const REPO_ID = '32a0226d-9f33-42e8-8b7b-24867dea06d4' +const WORKTREE_PATH = path.join(path.sep, 'workspace', 'factory-pr-4626-git-crypt') +const WORKTREE_ID = `${REPO_ID}::${WORKTREE_PATH}` +const CANARY_WORKTREE_PATH = path.join(path.sep, 'workspace', 'canary') +const CANARY_WORKTREE_ID = `${REPO_ID}::${CANARY_WORKTREE_PATH}` +const CANARY_TAB_ID = 'canary-tab' +const CANARY_LEAF_ID = '22222222-2222-4222-8222-222222222222' +const CANARY_PTY_ID = 'pty-unrelated-canary' +const HELPER_TAB_ID = 'worker-child-terminal' +const HELPER_LEAF_ID = '33333333-3333-4333-8333-333333333333' +const HELPER_PANE_KEY = makePaneKey(HELPER_TAB_ID, HELPER_LEAF_ID) +const TERMINAL_HANDLE = 'terminal-background-worker' +const initialAppStoreState = useAppStore.getState() + +function makeWorktree(id: string, workspacePath: string): Worktree { + return { + id, + repoId: REPO_ID, + path: workspacePath, + head: 'abc123', + branch: 'refs/heads/review', + isBare: false, + isMainWorktree: false, + displayName: path.basename(workspacePath), + comment: '', + linkedIssue: null, + linkedPR: null, + linkedLinearIssue: null, + linkedGitLabMR: null, + linkedGitLabIssue: null, + isArchived: false, + isUnread: false, + isPinned: false, + sortOrder: 0, + lastActivityAt: 0, + hostId: 'local' + } +} + +function makeTab(id: string, worktreeId: string, ptyId: string, title: string): TerminalTab { + return { + id, + ptyId, + worktreeId, + title, + customTitle: null, + color: null, + sortOrder: 0, + createdAt: 1 + } +} + +function makeLayout(leafId: string, ptyId: string) { + return { + root: { type: 'leaf' as const, leafId }, + activeLeafId: leafId, + expandedLeafId: null, + ptyIdsByLeafId: { [leafId]: ptyId } + } +} + +function seedWorkspace(options: { helper?: boolean } = {}): void { + useAppStore.setState(initialAppStoreState, true) + const target = makeWorktree(WORKTREE_ID, WORKTREE_PATH) + const canary = makeWorktree(CANARY_WORKTREE_ID, CANARY_WORKTREE_PATH) + const original = makeTab( + ORIGINAL_TAB_ID, + WORKTREE_ID, + ORIGINAL_PTY_ID, + 'PR 4626 unified correction r3' + ) + const unrelated = makeTab(CANARY_TAB_ID, CANARY_WORKTREE_ID, CANARY_PTY_ID, 'Unrelated') + const helper = makeTab(HELPER_TAB_ID, WORKTREE_ID, 'pty-worker-child', 'Worker child') + useAppStore.setState({ + repos: [ + { + id: REPO_ID, + path: path.join(path.sep, 'workspace'), + displayName: 'repo', + badgeColor: '#000000', + addedAt: 0, + executionHostId: 'local' + } + ], + worktreesByRepo: { [REPO_ID]: [target, canary] }, + activeRepoId: REPO_ID, + activeWorktreeId: CANARY_WORKTREE_ID, + activeTabId: CANARY_TAB_ID, + activeTabType: 'terminal', + activeView: 'terminal', + tabsByWorktree: { + [WORKTREE_ID]: options.helper ? [original, helper] : [original], + [CANARY_WORKTREE_ID]: [unrelated] + }, + ptyIdsByTabId: { + [ORIGINAL_TAB_ID]: [ORIGINAL_PTY_ID], + [CANARY_TAB_ID]: [CANARY_PTY_ID], + ...(options.helper ? { [HELPER_TAB_ID]: ['pty-worker-child'] } : {}) + }, + terminalLayoutsByTabId: { + [ORIGINAL_TAB_ID]: makeLayout(ORIGINAL_LEAF_ID, ORIGINAL_PTY_ID), + [CANARY_TAB_ID]: makeLayout(CANARY_LEAF_ID, CANARY_PTY_ID), + ...(options.helper ? { [HELPER_TAB_ID]: makeLayout(HELPER_LEAF_ID, 'pty-worker-child') } : {}) + }, + activeTabIdByWorktree: { + [WORKTREE_ID]: ORIGINAL_TAB_ID, + [CANARY_WORKTREE_ID]: CANARY_TAB_ID + }, + activeTabTypeByWorktree: { + [WORKTREE_ID]: 'terminal', + [CANARY_WORKTREE_ID]: 'terminal' + }, + tabBarOrderByWorktree: { + [WORKTREE_ID]: options.helper ? [ORIGINAL_TAB_ID, HELPER_TAB_ID] : [ORIGINAL_TAB_ID], + [CANARY_WORKTREE_ID]: [CANARY_TAB_ID] + }, + unifiedTabsByWorktree: {}, + groupsByWorktree: {}, + layoutByWorktree: {}, + activeGroupIdByWorktree: {}, + openFiles: [], + browserTabsByWorktree: {}, + browserPagesByWorkspace: {}, + activeFileIdByWorktree: {}, + activeBrowserTabIdByWorktree: {}, + pendingStartupByTabId: {}, + automaticAgentResumeClaimsByTabId: {}, + agentStatusByPaneKey: {}, + sleepingAgentSessionsByPaneKey: {}, + everActivatedWorktreeIds: new Set([CANARY_WORKTREE_ID]), + settings: { + ...initialAppStoreState.settings, + agentCmdOverrides: {}, + agentDefaultArgs: { codex: '--dangerously-bypass-approvals-and-sandbox' }, + setupScriptLaunchMode: 'new-tab' + }, + markWorktreeVisited: vi.fn(), + recordWorktreeVisit: vi.fn(), + refreshGitHubForWorktreeIfStale: vi.fn(), + revealWorktreeInSidebar: vi.fn() + } as Partial) +} + +function recordWorkingWorker() { + const providerSession = { key: 'session_id' as const, id: PROVIDER_SESSION_ID } + useAppStore + .getState() + .setAgentStatus( + ORIGINAL_PANE_KEY, + { state: 'working', prompt: 'review PR 4626', agentType: 'codex' }, + 'PR 4626 unified correction r3', + { updatedAt: 1_786_361_478_130, stateStartedAt: 1_786_361_478_130 }, + { tabId: ORIGINAL_TAB_ID, worktreeId: WORKTREE_ID, terminalHandle: TERMINAL_HANDLE }, + { providerSession } + ) + return providerSession +} + +function completeRecordedWorker( + providerSession: ReturnType +): SleepingAgentSessionRecord { + useAppStore + .getState() + .setAgentStatus( + ORIGINAL_PANE_KEY, + { state: 'done', prompt: 'review PR 4626', agentType: 'codex' }, + 'PR 4626 unified correction r3', + { updatedAt: 1_786_361_625_666, stateStartedAt: 1_786_361_625_666 }, + { tabId: ORIGINAL_TAB_ID, worktreeId: WORKTREE_ID, terminalHandle: TERMINAL_HANDLE }, + { providerSession } + ) + expect(useAppStore.getState().agentStatusByPaneKey[ORIGINAL_PANE_KEY]).toMatchObject({ + state: 'done', + providerSession + }) + const record = useAppStore.getState().sleepingAgentSessionsByPaneKey[ORIGINAL_PANE_KEY] + expect(record).toMatchObject({ + paneKey: ORIGINAL_PANE_KEY, + tabId: ORIGINAL_TAB_ID, + worktreeId: WORKTREE_ID, + agent: 'codex', + providerSession, + state: 'working', + origin: 'live' + }) + return record! +} + +function recordCompletedWorker(): SleepingAgentSessionRecord { + return completeRecordedWorker(recordWorkingWorker()) +} + +function expectCanaryUnchanged(): void { + const state = useAppStore.getState() + expect(state.tabsByWorktree[CANARY_WORKTREE_ID]).toEqual([ + expect.objectContaining({ id: CANARY_TAB_ID, ptyId: CANARY_PTY_ID }) + ]) + expect(state.terminalLayoutsByTabId[CANARY_TAB_ID]).toEqual( + makeLayout(CANARY_LEAF_ID, CANARY_PTY_ID) + ) +} + +function orchestrationMethod(name: string) { + const method = ORCHESTRATION_METHODS.find((candidate) => candidate.name === name) + if (!method) { + throw new Error(`Missing orchestration method: ${name}`) + } + return method +} + +async function releaseCompletedWorker(terminalState: 'running' | 'exited'): Promise { + const db = new OrchestrationDb(':memory:') + const runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(db) + const coordinatorPaneKey = 'coordinator-tab:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' + const run = db.createRun({ + objective: 'Completed worker retirement reproduction', + coordinatorHandle: 'terminal-coordinator', + coordinatorPaneKey + }) + const ctx: RpcContext = { runtime } + const call = async (name: string, params: Record) => { + const method = orchestrationMethod(name) + const parsed = method.params ? method.params.parse(params) : undefined + return method.handler(parsed, ctx) + } + + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === 'terminal-coordinator' ? coordinatorPaneKey : ORIGINAL_PANE_KEY + ) + vi.spyOn(runtime, 'getTerminalProcessIncarnation').mockImplementation((handle) => + handle === TERMINAL_HANDLE ? 'runtime:test:worker:1' : null + ) + vi.spyOn(runtime, 'getOrchestrationDispatchAuthority').mockImplementation((handle) => + handle === TERMINAL_HANDLE + ? ({ + terminalHandle: TERMINAL_HANDLE, + paneKey: ORIGINAL_PANE_KEY, + processIncarnation: 'runtime:test:worker:1', + hostScope: { kind: 'local', hostId: 'local' } + } as never) + : null + ) + vi.spyOn(runtime, 'validateOrchestrationAgentLauncher').mockImplementation(() => {}) + vi.spyOn(runtime, 'showManagedWorktree').mockResolvedValue({ id: WORKTREE_ID } as never) + vi.spyOn(runtime, 'createTerminal').mockResolvedValue({ + handle: TERMINAL_HANDLE, + worktreeId: WORKTREE_ID, + title: 'PR 4626 unified correction r3' + }) + vi.spyOn(runtime, 'waitForTerminal').mockResolvedValue({ + handle: TERMINAL_HANDLE, + condition: 'tui-idle', + satisfied: true, + status: 'running', + exitCode: null + }) + vi.spyOn(runtime, 'getTerminalOrchestrationCliCommand').mockReturnValue('orca') + vi.spyOn(runtime, 'sendTerminalAgentPrompt').mockResolvedValue({ + handle: TERMINAL_HANDLE, + accepted: true, + bytesWritten: 1 + }) + vi.spyOn(runtime, 'isTerminalRunningAgent').mockResolvedValue(true) + vi.spyOn(runtime, 'getExactWorkerProviderSession').mockReturnValue(null) + vi.spyOn(runtime, 'showTerminal').mockResolvedValue({ + handle: TERMINAL_HANDLE, + worktreeId: WORKTREE_ID, + ...(terminalState === 'exited' ? { connected: false } : { status: 'running' }) + } as never) + vi.spyOn(runtime, 'readTerminal').mockResolvedValue({ + handle: TERMINAL_HANDLE, + status: terminalState, + tail: terminalState === 'exited' ? [] : ['completed worker output'], + truncated: false, + nextCursor: terminalState === 'exited' ? null : '1' + }) + const closeTerminal = vi.spyOn(runtime, 'closeTerminal').mockImplementation(async () => { + closeTerminalTab(ORIGINAL_TAB_ID, { + force: true, + skipRunningProcessConfirm: true, + localPtyTeardownOwnedExternally: true + }) + return { handle: TERMINAL_HANDLE, tabId: ORIGINAL_TAB_ID, ptyKilled: true } + }) + vi.spyOn(runtime, 'notifyMessageArrived').mockImplementation(() => {}) + + try { + const task = db.createTask({ spec: 'release completed worker', runId: run.id }) + const started = (await call('orchestration.workerStart', { + task: task.id, + from: 'terminal-coordinator', + agent: 'codex' + })) as { dispatchId: string; state: string } + expect(started.state).toBe('ready') + expect(db.getWorkerDispatch(started.dispatchId)?.state).toBe('ready') + expect( + db.settleWorkerReport({ + taskId: task.id, + dispatchId: started.dispatchId, + outcome: 'succeeded', + result: 'worker completed' + }).action + ).toBe('settled') + expect(db.getWorkerDispatch(started.dispatchId)?.state).toBe('succeeded') + + await expect( + call('orchestration.workerRelease', { dispatch: started.dispatchId }) + ).resolves.toMatchObject({ + dispatchId: started.dispatchId, + state: 'released', + processAction: terminalState === 'exited' ? 'closed_exited_terminal' : 'closed_agent_terminal' + }) + expect(db.getWorkerDispatch(started.dispatchId)?.state).toBe('succeeded') + expect(db.getWorkerTerminalResourceByOwner(started.dispatchId)).toMatchObject({ + ownership_state: 'released', + release_state: 'released', + pane_key: ORIGINAL_PANE_KEY, + terminal_handle: TERMINAL_HANDLE + }) + expect(closeTerminal).toHaveBeenCalledOnce() + } finally { + db.close() + } +} + +function persistAndParseCurrentSession() { + const payload = buildWorkspaceSessionPayload(useAppStore.getState()) + const parsed = parseWorkspaceSession(JSON.parse(JSON.stringify(payload))) + expect(parsed.ok).toBe(true) + if (!parsed.ok) { + throw new Error(parsed.error) + } + return parsed.value +} + +async function hydrateSession( + session: ReturnType +): Promise { + seedWorkspace() + useAppStore.getState().hydrateWorkspaceSession(session) + useAppStore.getState().hydrateTabsSession(session) + await useAppStore.getState().reconnectPersistedTerminals() +} + +beforeEach(() => { + vi.spyOn(console, 'debug').mockImplementation(() => {}) + vi.stubGlobal('window', { + api: { + pty: { kill: vi.fn().mockResolvedValue(undefined) }, + runtime: { call: vi.fn().mockResolvedValue({ ok: true, result: {} }) }, + runtimeEnvironments: { call: vi.fn().mockResolvedValue({ ok: true, result: {} }) } + } + }) + seedWorkspace() +}) + +afterEach(() => { + vi.restoreAllMocks() + vi.unstubAllGlobals() + useAppStore.setState(initialAppStoreState, true) +}) + +describe('completed background-worker retirement resume matrix', () => { + it('does not cold-resume an explicitly completed and retired worker on first activation', async () => { + // Case 1: task completion alone keeps the still-owned provider session recoverable in place. + const ownedRecord = recordCompletedWorker() + expect(resumeSleepingAgentSessionsForWorktree(WORKTREE_ID)).toBe(0) + expect(useAppStore.getState().tabsByWorktree[WORKTREE_ID]).toHaveLength(1) + expect(useAppStore.getState().sleepingAgentSessionsByPaneKey[ORIGINAL_PANE_KEY]).toBe( + ownedRecord + ) + + // Case 2: the renderer boundary used by an explicit Orca close retires the exact authority. + seedWorkspace() + recordCompletedWorker() + closeTerminalTab(ORIGINAL_TAB_ID, { + force: true, + skipRunningProcessConfirm: true, + localPtyTeardownOwnedExternally: true + }) + expect(useAppStore.getState().sleepingAgentSessionsByPaneKey[ORIGINAL_PANE_KEY]).toBeUndefined() + expectCanaryUnchanged() + + // Case 3: both running and PTY-exit-first release retire exact resume authority. + seedWorkspace() + recordCompletedWorker() + await releaseCompletedWorker('running') + expect(useAppStore.getState().sleepingAgentSessionsByPaneKey[ORIGINAL_PANE_KEY]).toBeUndefined() + expectCanaryUnchanged() + + seedWorkspace() + recordCompletedWorker() + useAppStore.getState().removeAgentStatus(ORIGINAL_PANE_KEY) + useAppStore.getState().closeTab(ORIGINAL_TAB_ID, { reason: 'pty-exit' }) + expect(useAppStore.getState().tabsByWorktree[WORKTREE_ID]).toEqual([]) + expect(useAppStore.getState().terminalLayoutsByTabId[ORIGINAL_TAB_ID]).toBeUndefined() + expect(useAppStore.getState().ptyIdsByTabId[ORIGINAL_TAB_ID]).toBeUndefined() + expect(useAppStore.getState().sleepingAgentSessionsByPaneKey[ORIGINAL_PANE_KEY]).toMatchObject({ + origin: 'live', + state: 'working', + providerSession: { key: 'session_id', id: PROVIDER_SESSION_ID } + }) + await releaseCompletedWorker('exited') + expect(useAppStore.getState().sleepingAgentSessionsByPaneKey[ORIGINAL_PANE_KEY]).toBeUndefined() + + const retiredRestart = persistAndParseCurrentSession() + expect(retiredRestart.tabsByWorktree[WORKTREE_ID]).toEqual([]) + expect(retiredRestart.sleepingAgentSessionsByPaneKey?.[ORIGINAL_PANE_KEY]).toBeUndefined() + + // Case 4: legacy rollback preserves a fenced record; exited resolution clears it. + seedWorkspace() + const legacyRecord = recordCompletedWorker() + useAppStore.setState({ + sleepingAgentSessionsByPaneKey: { + [ORIGINAL_PANE_KEY]: { + ...legacyRecord, + automaticResumeBlockedBy: 'legacy-orchestration-worker' + } + } + }) + const legacyAction = resolveLegacyWorkerTerminalRecoveryAction({ + paneKey: ORIGINAL_PANE_KEY, + resolution: 'rolled_back', + ptyId: ORIGINAL_PTY_ID + }) + expect(legacyAction.kind).toBe('rollback-surface') + if (legacyAction.kind === 'rollback-surface') { + expect( + rollbackLegacyWorkerTerminalSurfaceInStore(useAppStore.getState(), legacyAction.detail) + ).toBe('removed') + } + expect(resumeSleepingAgentSessionsForWorktree(WORKTREE_ID)).toBe(0) + expect( + useAppStore.getState().sleepingAgentSessionsByPaneKey[ORIGINAL_PANE_KEY] + ?.automaticResumeBlockedBy + ).toBe('legacy-orchestration-worker') + const exitedAction = resolveLegacyWorkerTerminalRecoveryAction({ + paneKey: ORIGINAL_PANE_KEY, + resolution: 'exited' + }) + expect(exitedAction).toEqual({ kind: 'clear-sleeping', paneKey: ORIGINAL_PANE_KEY }) + useAppStore.getState().clearSleepingAgentSession(ORIGINAL_PANE_KEY) + + // Case 5: coordinator manual close is the same safe exact-tab retirement boundary. + seedWorkspace() + recordCompletedWorker() + closeTerminalTab(ORIGINAL_TAB_ID, { + force: true, + skipRunningProcessConfirm: true, + localPtyTeardownOwnedExternally: true + }) + expect(useAppStore.getState().sleepingAgentSessionsByPaneKey[ORIGINAL_PANE_KEY]).toBeUndefined() + expectCanaryUnchanged() + + // Case 6: normal Codex exit leaves the shell pane; helper close retires only the helper. + seedWorkspace({ helper: true }) + recordCompletedWorker() + useAppStore.setState({ + sleepingAgentSessionsByPaneKey: { + ...useAppStore.getState().sleepingAgentSessionsByPaneKey, + [HELPER_PANE_KEY]: { + paneKey: HELPER_PANE_KEY, + tabId: HELPER_TAB_ID, + worktreeId: WORKTREE_ID, + agent: 'codex', + providerSession: { key: 'session_id', id: 'unrelated-helper-session' }, + prompt: 'helper', + state: 'working', + capturedAt: 1, + updatedAt: 1, + origin: 'live' + } + } + }) + useAppStore.getState().removeAgentStatus(ORIGINAL_PANE_KEY) + expect(useAppStore.getState().tabsByWorktree[WORKTREE_ID]?.[0]).toMatchObject({ + id: ORIGINAL_TAB_ID, + ptyId: ORIGINAL_PTY_ID + }) + expect(useAppStore.getState().terminalLayoutsByTabId[ORIGINAL_TAB_ID]).toEqual( + makeLayout(ORIGINAL_LEAF_ID, ORIGINAL_PTY_ID) + ) + closeTerminalTab(HELPER_TAB_ID, { + force: true, + skipRunningProcessConfirm: true, + localPtyTeardownOwnedExternally: true + }) + expect(useAppStore.getState().sleepingAgentSessionsByPaneKey[HELPER_PANE_KEY]).toBeUndefined() + expect(useAppStore.getState().sleepingAgentSessionsByPaneKey[ORIGINAL_PANE_KEY]).toBeDefined() + expect(resumeSleepingAgentSessionsForWorktree(WORKTREE_ID)).toBe(0) + expectCanaryUnchanged() + + // Case 7: restart preserves owned panes, while post-retirement restart keeps authority absent. + seedWorkspace() + const workingProviderSession = recordWorkingWorker() + const beforeCompletion = persistAndParseCurrentSession() + expect(beforeCompletion.tabsByWorktree[WORKTREE_ID]?.[0]).toMatchObject({ + id: ORIGINAL_TAB_ID, + ptyId: ORIGINAL_PTY_ID + }) + expect(beforeCompletion.sleepingAgentSessionsByPaneKey?.[ORIGINAL_PANE_KEY]).toMatchObject({ + state: 'working', + origin: 'live', + providerSession: workingProviderSession + }) + await hydrateSession(beforeCompletion) + activateAndRevealWorktree(WORKTREE_ID, { notifyHostRuntime: false }) + expect(useAppStore.getState().tabsByWorktree[WORKTREE_ID]?.[0]?.id).toBe(ORIGINAL_TAB_ID) + expect(Object.keys(useAppStore.getState().pendingStartupByTabId)).toEqual([]) + expect(useAppStore.getState().tabsByWorktree[WORKTREE_ID]?.[0]?.ptyId).toBe(ORIGINAL_PTY_ID) + expect(useAppStore.getState().ptyIdsByTabId[ORIGINAL_TAB_ID]).toEqual([ORIGINAL_PTY_ID]) + completeRecordedWorker(recordWorkingWorker()) + const afterCompletion = persistAndParseCurrentSession() + expect(afterCompletion.tabsByWorktree[WORKTREE_ID]?.[0]).toMatchObject({ + id: ORIGINAL_TAB_ID, + ptyId: ORIGINAL_PTY_ID + }) + expect(afterCompletion.terminalLayoutsByTabId[ORIGINAL_TAB_ID]?.ptyIdsByLeafId).toEqual({ + [ORIGINAL_LEAF_ID]: ORIGINAL_PTY_ID + }) + expect(afterCompletion.sleepingAgentSessionsByPaneKey?.[ORIGINAL_PANE_KEY]).toBeDefined() + await hydrateSession(afterCompletion) + activateAndRevealWorktree(WORKTREE_ID, { notifyHostRuntime: false }) + expect(useAppStore.getState().tabsByWorktree[WORKTREE_ID]?.[0]?.id).toBe(ORIGINAL_TAB_ID) + expect(Object.keys(useAppStore.getState().pendingStartupByTabId)).toEqual([]) + expect(useAppStore.getState().tabsByWorktree[CANARY_WORKTREE_ID]?.[0]?.id).toBe(CANARY_TAB_ID) + + activateAndRevealWorktree(CANARY_WORKTREE_ID, { notifyHostRuntime: false }) + useAppStore.getState().removeAgentStatus(ORIGINAL_PANE_KEY) + useAppStore.getState().closeTab(ORIGINAL_TAB_ID, { reason: 'pty-exit' }) + await releaseCompletedWorker('exited') + const restartAfterRetirement = persistAndParseCurrentSession() + await hydrateSession(restartAfterRetirement) + + // Case 8: first activation of the never-visited target cannot resurrect retired authority. + const beforeActivation = useAppStore.getState() + expect(beforeActivation.everActivatedWorktreeIds.has(WORKTREE_ID)).toBe(false) + expect(beforeActivation.agentStatusByPaneKey[ORIGINAL_PANE_KEY]).toBeUndefined() + expect(beforeActivation.sleepingAgentSessionsByPaneKey[ORIGINAL_PANE_KEY]).toBeUndefined() + expect(Object.keys(beforeActivation.pendingStartupByTabId)).toEqual([]) + const tabCountBeforeActivation = beforeActivation.tabsByWorktree[WORKTREE_ID]?.length ?? 0 + activateAndRevealWorktree(WORKTREE_ID, { notifyHostRuntime: false }) + const activated = useAppStore.getState() + const replacementTabs = (activated.tabsByWorktree[WORKTREE_ID] ?? []).filter( + (tab) => tab.id !== ORIGINAL_TAB_ID + ) + // The provider-ownership gate separately proves this request becomes one transport spawn. + const coldSpawnRequests = replacementTabs.flatMap((tab) => { + const startup = activated.pendingStartupByTabId[tab.id] + if (!startup?.resumeProviderSession) { + return [] + } + const tokens = tokenizeStartupCommand(startup.command, 'posix') + expect(tokens.ok).toBe(true) + const showSessionRestoredBanner = vi.fn() + seedStartupSessionRestoredBanner(startup, 1, showSessionRestoredBanner) + return [ + { + providerSession: startup.resumeProviderSession, + command: startup.command, + argv: tokens.ok ? tokens.tokens : [], + restoredBannerCount: showSessionRestoredBanner.mock.calls.length + } + ] + }) + + expect(tabCountBeforeActivation).toBe(0) + expect(replacementTabs).toHaveLength(1) + expect(activated.tabsByWorktree[WORKTREE_ID]?.some((tab) => tab.id === ORIGINAL_TAB_ID)).toBe( + false + ) + expect(activated.terminalLayoutsByTabId[ORIGINAL_TAB_ID]).toBeUndefined() + expect(activated.ptyIdsByTabId[ORIGINAL_TAB_ID]).toBeUndefined() + expect(coldSpawnRequests).toEqual([]) + expectCanaryUnchanged() + expect(Object.keys(activated.pendingStartupByTabId)).toHaveLength(0) + expect(Object.keys(activated.automaticAgentResumeClaimsByTabId)).toHaveLength(0) + + // Required invariant: explicit completion plus retirement must revoke provider-resume authority. + expect(coldSpawnRequests).toEqual([]) + }) +}) diff --git a/tests/e2e/helpers/completed-worker-retirement-fixture.ts b/tests/e2e/helpers/completed-worker-retirement-fixture.ts new file mode 100644 index 00000000000..d1211234258 --- /dev/null +++ b/tests/e2e/helpers/completed-worker-retirement-fixture.ts @@ -0,0 +1,194 @@ +import { execFileSync } from 'node:child_process' +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync +} from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import type { RuntimeClient } from '../../../src/cli/runtime-client' +import { DEFAULT_LOCAL_ORCA_PROFILE_ID } from '../../../src/shared/orca-profiles' +import type { + RuntimeTerminalListResult, + RuntimeTerminalSummary +} from '../../../src/shared/runtime-types' + +const fakeCliDir = mkdtempSync(path.join(os.tmpdir(), 'orca-e2e-retired-worker-')) +const lifecycleLedgerPath = path.join(fakeCliDir, 'codex-lifecycle.jsonl') +const fakeCodexSource = ` +const { appendFileSync } = require('node:fs') +const ledger = process.env.ORCA_E2E_CODEX_LIFECYCLE_LEDGER +const append = (event) => appendFileSync(ledger, JSON.stringify({ pid: process.pid, ...event }) + '\\n') +const args = process.argv.slice(2) +if (args.includes('app-server')) { + process.stderr.write("error: unrecognized subcommand 'app-server'\\n") + process.exit(2) +} +append({ event: 'spawn', args }) +process.stdout.write('\\u001b]0;Codex Ready\\u0007OpenAI Codex\\nmodel: e2e\\ndirectory: e2e\\n') +process.stdin.on('data', (chunk) => { + const input = chunk.toString() + append({ event: 'input', input }) + if (input.includes('ORCA_E2E_EXIT_AFTER_DONE')) { + append({ event: 'normal-exit' }) + process.exit(0) + } + if (input.includes('\\r')) process.stdout.write('ACK\\n') +}) +process.stdin.resume() +setInterval(() => {}, 60_000) +` + +if (process.platform === 'win32') { + writeFileSync(path.join(fakeCliDir, 'fake-codex.js'), fakeCodexSource) + writeFileSync( + path.join(fakeCliDir, 'codex.cmd'), + '@echo off\r\nnode "%~dp0\\fake-codex.js" %*\r\n' + ) +} else { + const executable = path.join(fakeCliDir, 'codex') + writeFileSync(executable, `#!/usr/bin/env node\n${fakeCodexSource}`) + chmodSync(executable, 0o755) +} + +export const completedWorkerLaunchEnv = { + PATH: `${fakeCliDir}${path.delimiter}${process.env.PATH ?? ''}`, + ORCA_E2E_CODEX_LIFECYCLE_LEDGER: lifecycleLedgerPath +} + +export type LifecycleEvent = { + pid: number + event: 'spawn' | 'input' | 'normal-exit' + args?: string[] + input?: string +} + +export type TerminalIdentity = Pick< + RuntimeTerminalSummary, + 'handle' | 'incarnationId' | 'leafId' | 'ptyId' | 'tabId' | 'worktreeId' +> + +export function clearCompletedWorkerLedger(): void { + rmSync(lifecycleLedgerPath, { force: true }) +} + +export function cleanupCompletedWorkerFixture(): void { + rmSync(fakeCliDir, { recursive: true, force: true }) +} + +export function readCompletedWorkerLedger(): LifecycleEvent[] { + if (!existsSync(lifecycleLedgerPath)) { + return [] + } + const contents = readFileSync(lifecycleLedgerPath, 'utf8') + const lastCompleteLine = contents.lastIndexOf('\n') + if (lastCompleteLine < 0) { + return [] + } + return contents + .slice(0, lastCompleteLine) + .split(/\r?\n/) + .filter(Boolean) + .map((line) => JSON.parse(line) as LifecycleEvent) +} + +export function readCompletedWorkerDispatchCapability(): string | null { + const input = readCompletedWorkerLedger() + .filter((event) => event.event === 'input') + .map((event) => event.input ?? '') + .join('') + return input.match(/--dispatch-capability\s+(\S+)/)?.[1] ?? null +} + +export function runBuiltOrcaCli( + args: string[], + options: { userDataDir: string; cwd: string } +): unknown { + const { + ORCA_ENVIRONMENT: _environment, + ORCA_PAIRING_CODE: _pairingCode, + ORCA_USER_DATA_PATH: _userDataPath, + ...cleanEnv + } = process.env + void _environment + void _pairingCode + void _userDataPath + const output = execFileSync( + process.execPath, + [path.join(process.cwd(), 'out', 'cli', 'index.js'), ...args], + { + cwd: options.cwd, + env: { ...cleanEnv, ORCA_USER_DATA_PATH: options.userDataDir }, + encoding: 'utf8', + timeout: 30_000 + } + ) + return JSON.parse(output) as unknown +} + +export function seedCurrentCodexTranscript( + isolatedHome: string, + providerSessionId: string, + cwd: string +): string { + const now = new Date() + const transcriptDir = path.join( + isolatedHome, + '.codex', + 'sessions', + String(now.getUTCFullYear()), + String(now.getUTCMonth() + 1).padStart(2, '0'), + String(now.getUTCDate()).padStart(2, '0') + ) + mkdirSync(transcriptDir, { recursive: true }) + const transcriptPath = path.join(transcriptDir, `rollout-${providerSessionId}.jsonl`) + writeFileSync( + transcriptPath, + `${JSON.stringify({ + timestamp: now.toISOString(), + type: 'session_meta', + payload: { id: providerSessionId, cwd } + })}\n` + ) + return transcriptPath +} + +export function terminalIdentity(terminal: RuntimeTerminalSummary): TerminalIdentity { + const { handle, incarnationId, leafId, ptyId, tabId, worktreeId } = terminal + return { handle, incarnationId, leafId, ptyId, tabId, worktreeId } +} + +export async function listRuntimeTerminals( + client: RuntimeClient +): Promise { + return (await client.call('terminal.list')).result.terminals +} + +export function readPersistedWorkerRecoveryRecord(userDataDir: string, paneKey: string) { + const dataPath = path.join( + userDataDir, + 'profiles', + DEFAULT_LOCAL_ORCA_PROFILE_ID, + 'orca-data.json' + ) + if (!existsSync(dataPath)) { + return null + } + const data = JSON.parse(readFileSync(dataPath, 'utf8')) as { + workspaceSession?: { + sleepingAgentSessionsByPaneKey?: Record< + string, + { + origin?: unknown + state?: unknown + providerSession?: { id?: unknown } + } + > + } + } + return data.workspaceSession?.sleepingAgentSessionsByPaneKey?.[paneKey] ?? null +}