From afa471da29dab05edfe704b3cf46eba45220696f Mon Sep 17 00:00:00 2001 From: Neil Date: Wed, 11 Nov 2026 04:15:34 -0800 Subject: [PATCH] fix(cloud): RA-11 resume race: Restart resumes in the pane A cloud Restart stopped the pane's host terminal, ensured a --resume in its leaf, then reconnected. But the pane's own transport reacted to the stop ending its stream by resubscribing (session.tabs.activate), which woke the stopped leaf with its ORIGINAL launch before the ensure arrived; the ensure then adopted that live terminal and the resume was lost (a new conversation). Seen 3/3 on golden 4020131de02. - Release the pane's transport before the stop (split releasePaneForRestart out of handleRestartPane), so nothing reattaches on its own. - resumeRemotePaneAgent returns the ensured terminal; on 'adopted' it stops that terminal and ensures once more, else fails (Couldn't restart). - Reconnect only after the host lists that exact terminal live in the pane's tab and leaf (bounded 15 s, else Couldn't restart). - After reconnecting, the pane must bind that handle; any other handle shows Couldn't restart. A failed restart reattaches the pane so it is never left empty. --- .../cloud-strip-restart-run.test.ts | 125 +++++++++++++- .../use-ready-pane-account-restarts.ts | 73 +++++++- .../use-terminal-pane-process-exit-actions.ts | 3 +- .../process-exit/use-pane-process-restart.ts | 44 +++-- .../orca-cloud/cloud-agent-restart-attempt.ts | 9 +- .../cloud-pane-restart-stop.test.ts | 114 ++++++++++++- .../lib/orca-cloud/cloud-pane-restart-stop.ts | 158 ++++++++++++++++-- .../orca-cloud/cloud-restart-bind-check.ts | 63 +++++++ 8 files changed, 544 insertions(+), 45 deletions(-) create mode 100644 src/renderer/src/lib/orca-cloud/cloud-restart-bind-check.ts diff --git a/src/renderer/src/components/terminal-pane/account-restart/cloud-strip-restart-run.test.ts b/src/renderer/src/components/terminal-pane/account-restart/cloud-strip-restart-run.test.ts index a3c38777f0d..ba9dce5dd1e 100644 --- a/src/renderer/src/components/terminal-pane/account-restart/cloud-strip-restart-run.test.ts +++ b/src/renderer/src/components/terminal-pane/account-restart/cloud-strip-restart-run.test.ts @@ -43,6 +43,8 @@ const TAB = 'tab-1' const LEAF = '11111111-1111-4111-8111-111111111111' const PTY = 'env-1@@pty-1' const PANE_KEY = makePaneKey(TAB, LEAF) +/** The terminal a resume ensure placed in the pane's leaf. */ +const RESUMED = { handle: 'term_resumed', disposition: 'created' as const } function seed(state: AgentStatusEntry['state']): void { useAppStore.setState({ @@ -291,6 +293,7 @@ describe('RA-11 Restart on a paired (remote) pane', () => { const providerSession = { key: 'session_id' as const, id: 'sess-tab-1' } const resumeRemotePane = vi.fn(async () => { order.push('resume') + return RESUMED }) const freshIfResumeFoundNothing = vi.fn(async () => false) const handleRestartPane = vi.fn(() => order.push('reconnect')) @@ -309,6 +312,7 @@ describe('RA-11 Restart on a paired (remote) pane', () => { order.push('stop') }, resumeRemotePane, + waitForResumedTerminal: () => Promise.resolve(), freshIfResumeFoundNothing, buildFreshCommand: (agent, leafId) => `fresh ${agent} ${leafId}` }) @@ -368,7 +372,8 @@ describe('RA-11 Restart on a paired (remote) pane', () => { }), handleRestartPane, stopRemotePane: () => Promise.resolve(), - resumeRemotePane: () => Promise.resolve(), + resumeRemotePane: () => Promise.resolve(RESUMED), + waitForResumedTerminal: () => Promise.resolve(), freshIfResumeFoundNothing: (input) => realBackstop({ ...input, sleep: () => Promise.resolve() }), buildFreshCommand: () => 'claude --dangerously-skip-permissions' @@ -397,7 +402,8 @@ describe('RA-11 Restart on a paired (remote) pane', () => { }), handleRestartPane, stopRemotePane: () => Promise.resolve(), - resumeRemotePane: () => Promise.resolve(), + resumeRemotePane: () => Promise.resolve(RESUMED), + waitForResumedTerminal: () => Promise.resolve(), freshIfResumeFoundNothing: async () => true, buildFreshCommand: () => 'claude' }) @@ -425,7 +431,8 @@ describe('RA-11 Restart on a paired (remote) pane', () => { }), handleRestartPane, stopRemotePane: () => Promise.resolve(), - resumeRemotePane: () => Promise.resolve(), + resumeRemotePane: () => Promise.resolve(RESUMED), + waitForResumedTerminal: () => Promise.resolve(), freshIfResumeFoundNothing: () => Promise.reject(new Error('ra11e_fresh_send_failed')), buildFreshCommand: () => 'claude' }) @@ -438,6 +445,118 @@ describe('RA-11 Restart on a paired (remote) pane', () => { warn.mockRestore() }) + // RA-11 resume race: the pane's own transport reattached when the stop ended its stream, woke the + // leaf with its original launch, and the resume was adopted away. + function runResume(over: { + waitForResumedTerminal?: () => Promise + onReconnect?: () => void + order?: string[] + }) { + const order = over.order ?? [] + const handleRestartPane = vi.fn( + (_paneId: number, _startup: unknown, options: { focus: boolean }) => { + order.push(options.focus ? 'reconnect(focus)' : 'reconnect') + over.onReconnect?.() + } + ) + runReadyPaneAccountRestarts({ + panes: [{ id: 7, leafId: LEAF }], + ptyIdOf: () => REMOTE, + readyRestartPtyIds: selectReadyAccountRestartPtyIds(useAppStore.getState(), TAB), + tabId: TAB, + consumePendingPaneAccountRestart: useAppStore.getState().consumePendingPaneAccountRestart, + buildAccountRestartStartup: () => ({ + command: 'claude', + resumeProviderSession: { key: 'session_id' as const, id: 'sess-tab-1' } + }), + handleRestartPane, + releasePaneForRestart: () => { + order.push('release') + }, + stopRemotePane: async () => { + order.push('stop') + }, + resumeRemotePane: async () => { + order.push('ensure') + return { handle: 'term_resumed', disposition: 'created' as const } + }, + waitForResumedTerminal: async () => { + order.push('ready?') + await (over.waitForResumedTerminal?.() ?? Promise.resolve()) + }, + freshIfResumeFoundNothing: async () => false, + buildFreshCommand: () => 'claude' + }) + return { handleRestartPane, order } + } + + function bindPane(handle: string): void { + const layouts = useAppStore.getState().terminalLayoutsByTabId + useAppStore.setState({ + terminalLayoutsByTabId: { + ...layouts, + [TAB]: { ...layouts[TAB]!, ptyIdsByLeafId: { [LEAF]: `remote:env-1@@${handle}` } } + } + }) + } + + it('lets go of the pane before the stop, then stop → resume → ready → one reconnect', async () => { + clickRemoteRestart() + const { handleRestartPane, order } = runResume({}) + await vi.waitFor(() => expect(handleRestartPane).toHaveBeenCalledTimes(1)) + expect(order).toEqual(['release', 'stop', 'ensure', 'ready?', 'reconnect']) + }) + + it('never reconnects before the resumed terminal is live in the leaf', async () => { + clickRemoteRestart() + let ready: () => void = () => undefined + const gate = new Promise((resolve) => { + ready = resolve + }) + const { handleRestartPane, order } = runResume({ waitForResumedTerminal: () => gate }) + await vi.waitFor(() => expect(order).toContain('ready?')) + await Promise.resolve() + expect(handleRestartPane).not.toHaveBeenCalled() + ready() + await vi.waitFor(() => expect(handleRestartPane).toHaveBeenCalledTimes(1)) + }) + + it('a resumed terminal that never shows fails the restart and reattaches the pane (never empty)', async () => { + clickRemoteRestart() + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const { handleRestartPane } = runResume({ + waitForResumedTerminal: () => Promise.reject(new Error('restart_resume_not_ready')) + }) + await vi.waitFor(() => expect(cloudAgentRestartAttempt(PANE_KEY, 0)?.phase).toBe('failed')) + expect(cloudAgentRestartAttempt(PANE_KEY, 0)?.failure).toBe('hostUnresponsive') + expect(handleRestartPane).toHaveBeenCalledTimes(1) + expect(handleRestartPane.mock.calls[0]?.[2]).toEqual({ focus: false }) + warn.mockRestore() + }) + + it('the pane binding the resumed terminal is a clean restart', async () => { + clickRemoteRestart() + const { handleRestartPane } = runResume({ onReconnect: () => bindPane('term_resumed') }) + await vi.waitFor(() => expect(handleRestartPane).toHaveBeenCalledTimes(1)) + await Promise.resolve() + expect(cloudAgentRestartAttempt(PANE_KEY, 0)?.phase).not.toBe('failed') + }) + + it('the pane binding any other terminal fails visibly (a new conversation is not a resume)', async () => { + clickRemoteRestart() + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + const { handleRestartPane } = runResume({ + onReconnect: () => { + settleCloudAgentRestart(PANE_KEY) + bindPane('term_original_launch') + } + }) + await vi.waitFor(() => expect(handleRestartPane).toHaveBeenCalledTimes(1)) + await vi.waitFor(() => expect(cloudAgentRestartAttempt(PANE_KEY, 0)?.phase).toBe('failed')) + expect(cloudAgentRestartAttempt(PANE_KEY, 0)?.ptyId).toBe('remote:env-1@@term_original_launch') + warn.mockRestore() + }) + it('a failed resume on the host shows the failure and never reconnects', async () => { clickRemoteRestart() const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) diff --git a/src/renderer/src/components/terminal-pane/account-restart/use-ready-pane-account-restarts.ts b/src/renderer/src/components/terminal-pane/account-restart/use-ready-pane-account-restarts.ts index c98b91f098f..2874eacff0d 100644 --- a/src/renderer/src/components/terminal-pane/account-restart/use-ready-pane-account-restarts.ts +++ b/src/renderer/src/components/terminal-pane/account-restart/use-ready-pane-account-restarts.ts @@ -8,7 +8,7 @@ import { import { beginCloudAgentRestart, failCloudAgentRestart, - failCloudAgentRestartFresh, + failCloudAgentRestartAfterLaunch, noteCloudAgentRestartFresh, isForcedCloudAgentRestart, noteCloudAgentRestartLaunched @@ -18,8 +18,13 @@ import type { usePaneProcessRestart } from '../process-exit/use-pane-process-res import { freshIfResumeFoundNothing, resumeRemotePaneAgent, + waitForResumedTerminalReady, stopRemotePaneTerminalForRestart } from '@/lib/orca-cloud/cloud-pane-restart-stop' +import { + expectPaneBindsHandle, + remotePtyIdWithHandle +} from '@/lib/orca-cloud/cloud-restart-bind-check' import type { AgentProviderSessionMetadata } from '../../../../../shared/agent-session/lifecycle/agent-session-resume' import { makePaneKey } from '../../../../../shared/workspace-layout/stable-pane-id' import { @@ -39,6 +44,7 @@ export function useReadyPaneAccountRestarts({ tabId, buildAccountRestartStartup, handleRestartPane, + releasePaneForRestart, panePtyLayoutBindings }: Pick< TerminalPaneCloseController, @@ -48,6 +54,7 @@ export function useReadyPaneAccountRestarts({ typeof usePaneAccountRestartStartup >['buildAccountRestartStartup'] handleRestartPane: ReturnType['handleRestartPane'] + releasePaneForRestart: ReturnType['releasePaneForRestart'] panePtyLayoutBindings: TerminalPaneCloseController['savedLayout']['ptyIdsByLeafId'] }) { // Why primitive ids: unrelated status updates stay shallow-equal and skip this pane. @@ -66,12 +73,14 @@ export function useReadyPaneAccountRestarts({ tabId, consumePendingPaneAccountRestart, buildAccountRestartStartup, - handleRestartPane + handleRestartPane, + releasePaneForRestart }) }, [ buildAccountRestartStartup, consumePendingPaneAccountRestart, handleRestartPane, + releasePaneForRestart, panePtyLayoutBindings, readyRestartPtyIds, managerRef, @@ -116,10 +125,18 @@ export function runReadyPaneAccountRestarts< consumePendingPaneAccountRestart: (ptyId: string) => AccountRestartAgent | null buildAccountRestartStartup: (agent: AccountRestartAgent, leafId: string) => Startup | null handleRestartPane: (paneId: number, startup: Startup, options: { focus: boolean }) => void + /** + * Destroys the pane's transport without reconnecting. A paired pane's transport otherwise reacts to + * its own host terminal stopping by reattaching, which wakes the leaf with its original launch + * before the resume lands (RA-11 resume race). + */ + releasePaneForRestart?: (paneId: number) => void /** Stops the one host terminal behind a paired pane before its restart reconnects. */ stopRemotePane?: (ptyId: string, worktreeId: string) => Promise /** Relaunches the paired pane's agent on its own conversation, in the same pane. */ resumeRemotePane?: typeof resumeRemotePaneAgent + /** RA-11 resume race: wait until the resumed terminal is live in the pane's own leaf. */ + waitForResumedTerminal?: typeof waitForResumedTerminalReady /** RA-11e: start fresh in the same pane when a resume found no conversation. */ freshIfResumeFoundNothing?: typeof freshIfResumeFoundNothing /** The fresh-launch command for this tab's pane (no session to resume). */ @@ -128,6 +145,7 @@ export function runReadyPaneAccountRestarts< const args = { stopRemotePane: stopRemotePaneTerminalForRestart, resumeRemotePane: resumeRemotePaneAgent, + waitForResumedTerminal: waitForResumedTerminalReady, freshIfResumeFoundNothing, buildFreshCommand: (agent: AccountRestartAgent, leafId: string) => buildFreshAccountRestartCommand(agent, { @@ -187,7 +205,9 @@ export function runReadyPaneAccountRestarts< noteCloudAgentRestartLaunched(ptyId) } // Why: one Claude switch restarts many panes at once; none was the one the user clicked. - args.handleRestartPane(pane.id, launchedStartup, { focus: agent === 'codex' }) + args.handleRestartPane(pane.id, launchedStartup, { + focus: agent === 'codex' + }) } if (!worktreeId) { launch() @@ -196,6 +216,9 @@ export function runReadyPaneAccountRestarts< // Stop that one host terminal first (its exit is ours, not the pane's), then reconnect. const store = useAppStore.getState() store.suppressPtyExit(ptyId) + // Order is stop → ensure → reconnect: let go of the pane's transport before the stop, or its own + // recovery reattaches the moment the stop ends its stream and wakes the original launch. + args.releasePaneForRestart?.(pane.id) void args .stopRemotePane(ptyId, worktreeId) .then(() => @@ -210,9 +233,44 @@ export function runReadyPaneAccountRestarts< }) : undefined ) + .then(async (resumed) => { + // RA-11 resume race: reconnect only once the resumed terminal is live in this very leaf; + // an earlier reconnect would wake the leaf with its original launch and drop the resume. + if (resumed) { + await args.waitForResumedTerminal({ + ptyId, + worktreeId, + tabId: args.tabId, + leafId: pane.leafId, + handle: resumed.handle + }) + } + return resumed + }) .then( - () => { + (resumed) => { launch() + if (resumed && tracked) { + // The pane must end on the resumed terminal; anything else is a new conversation. + expectPaneBindsHandle({ + tabId: args.tabId, + leafId: pane.leafId, + replacedPtyId: ptyId, + handle: resumed.handle, + onMismatch: (bound) => { + console.warn( + '[orca-cloud] RA-11 restart bound', + bound, + 'not the resumed', + resumed.handle + ) + failCloudAgentRestartAfterLaunch(makePaneKey(args.tabId, pane.leafId), { + agent, + ptyId: remotePtyIdWithHandle(ptyId, bound) + }) + } + }) + } if (providerSession) { // RA-11e: a session with no conversation can't resume; start fresh in the same pane, once. void Promise.resolve() @@ -239,7 +297,7 @@ export function runReadyPaneAccountRestarts< console.warn('[orca-cloud] fresh fallback failed:', err) if (tracked) { // The pane is a bare shell now: say so with Try again, never leave it quiet. - failCloudAgentRestartFresh(makePaneKey(args.tabId, pane.leafId), { + failCloudAgentRestartAfterLaunch(makePaneKey(args.tabId, pane.leafId), { agent, ptyId: useAppStore.getState().terminalLayoutsByTabId[args.tabId]?.ptyIdsByLeafId?.[ @@ -255,6 +313,11 @@ export function runReadyPaneAccountRestarts< useAppStore.getState().consumeSuppressedPtyExit(ptyId) // The host did not stop or relaunch it: the strip names that reason. failCloudAgentRestart(ptyId, 'hostUnresponsive') + if (args.releasePaneForRestart) { + // Its transport was let go before the stop: reattach so the pane is never left empty + // (the still-live terminal, or the leaf's own launch if the stop went through). + args.handleRestartPane(pane.id, launchedStartup, { focus: false }) + } if (!forced && agent === 'codex') { // The switch was answered on the old account: ask again rather than leave it silently stale. useAppStore.getState().reopenCodexRestartPrompt(ptyId) diff --git a/src/renderer/src/components/terminal-pane/controller/use-terminal-pane-process-exit-actions.ts b/src/renderer/src/components/terminal-pane/controller/use-terminal-pane-process-exit-actions.ts index d262a94e2ff..2747c35b02f 100644 --- a/src/renderer/src/components/terminal-pane/controller/use-terminal-pane-process-exit-actions.ts +++ b/src/renderer/src/components/terminal-pane/controller/use-terminal-pane-process-exit-actions.ts @@ -49,7 +49,7 @@ export function useTerminalPaneProcessExitActions(controller: TerminalPaneCloseC const { buildAccountRestartStartup } = usePaneAccountRestartStartup({ tabId, worktreeId }) - const { handleRestartPane } = usePaneProcessRestart({ + const { handleRestartPane, releasePaneForRestart } = usePaneProcessRestart({ clearCodexRestartNotice, clearExitedPanePtyLayoutBinding, clearRuntimePaneTitle, @@ -105,6 +105,7 @@ export function useTerminalPaneProcessExitActions(controller: TerminalPaneCloseC tabId, buildAccountRestartStartup, handleRestartPane, + releasePaneForRestart, panePtyLayoutBindings }) diff --git a/src/renderer/src/components/terminal-pane/process-exit/use-pane-process-restart.ts b/src/renderer/src/components/terminal-pane/process-exit/use-pane-process-restart.ts index 00b2dd52874..d7ce8e99706 100644 --- a/src/renderer/src/components/terminal-pane/process-exit/use-pane-process-restart.ts +++ b/src/renderer/src/components/terminal-pane/process-exit/use-pane-process-restart.ts @@ -77,13 +77,13 @@ export function usePaneProcessRestart({ | 'updateTabTitle' | 'worktreeId' >) { - const handleRestartPane = useCallback( - (paneId: number, startup: PtyConnectionDeps['startup'], options: { focus: boolean }) => { - const manager = managerRef.current - const pane = manager?.getPanes().find((candidate) => candidate.id === paneId) - if (!manager || !pane) { - return - } + /** + * Let go of the pane's current PTY without connecting a new one: its transport is destroyed, so its + * own recovery can no longer reattach (or wake the host leaf) on its own. Returns the PTY id local + * main should replace, if any. + */ + const releasePaneForRestart = useCallback( + (paneId: number): string | null => { const transport = paneTransportsRef.current.get(paneId) const panePtyBinding = panePtyBindingsRef.current.get(paneId) const existingPtyId = transport?.getPtyId() @@ -94,9 +94,32 @@ export function usePaneProcessRestart({ } panePtyBinding?.dispose() panePtyBindingsRef.current.delete(paneId) - syncPanePtyLayoutBinding(paneId, null) + if (transport || panePtyBinding) { + syncPanePtyLayoutBinding(paneId, null) + } const replacesPtyId = releasePaneTransportForRestart(transport) paneTransportsRef.current.delete(paneId) + return replacesPtyId + }, + [ + clearCodexRestartNotice, + clearTabPtyId, + panePtyBindingsRef, + paneTransportsRef, + suppressPtyExit, + syncPanePtyLayoutBinding, + tabId + ] + ) + + const handleRestartPane = useCallback( + (paneId: number, startup: PtyConnectionDeps['startup'], options: { focus: boolean }) => { + const manager = managerRef.current + const pane = manager?.getPanes().find((candidate) => candidate.id === paneId) + if (!manager || !pane) { + return + } + const replacesPtyId = releasePaneForRestart(paneId) setCacheTimerStartedAt(makePaneKey(tabId, pane.leafId), null) setTerminalError(null) setTerminalErrorsByPaneId((current) => clearPaneTerminalError(current, paneId)) @@ -140,7 +163,7 @@ export function usePaneProcessRestart({ } }, [ - clearCodexRestartNotice, + releasePaneForRestart, clearExitedPanePtyLayoutBinding, clearRuntimePaneTitle, clearTabPtyId, @@ -166,7 +189,6 @@ export function usePaneProcessRestart({ setRuntimePaneTitle, setTerminalError, showRestoredSessionBanner, - suppressPtyExit, syncPanePtyLayoutBinding, tabId, updateTabPtyId, @@ -175,5 +197,5 @@ export function usePaneProcessRestart({ setTerminalErrorsByPaneId ] ) - return { handleRestartPane } + return { handleRestartPane, releasePaneForRestart } } diff --git a/src/renderer/src/lib/orca-cloud/cloud-agent-restart-attempt.ts b/src/renderer/src/lib/orca-cloud/cloud-agent-restart-attempt.ts index 56168718aff..95077ed6808 100644 --- a/src/renderer/src/lib/orca-cloud/cloud-agent-restart-attempt.ts +++ b/src/renderer/src/lib/orca-cloud/cloud-agent-restart-attempt.ts @@ -180,11 +180,12 @@ export function cloudAgentRestartStartedFresh(paneKey: string, _revision: number } /** - * RA-11e: the resume found no conversation and the fresh start could not be typed, so the pane is a - * bare shell. The attempt already settled on the new PTY; show "Couldn't restart" again so the - * person can Try again, never a quiet shell. + * The restart relaunched but did not end where it must: the fresh start after an empty resume could + * not be typed (RA-11e, a bare shell), or the pane bound a terminal other than the resumed one (RA-11 + * resume race, a new conversation). The attempt may already have settled on the new PTY; show + * "Couldn't restart" again so the person can Try again, never a quiet wrong result. */ -export function failCloudAgentRestartFresh( +export function failCloudAgentRestartAfterLaunch( paneKey: string, args: { agent: HostAgentSignInAgent; ptyId: string } ): void { diff --git a/src/renderer/src/lib/orca-cloud/cloud-pane-restart-stop.test.ts b/src/renderer/src/lib/orca-cloud/cloud-pane-restart-stop.test.ts index 53dad184a32..d6b9d20ae01 100644 --- a/src/renderer/src/lib/orca-cloud/cloud-pane-restart-stop.test.ts +++ b/src/renderer/src/lib/orca-cloud/cloud-pane-restart-stop.test.ts @@ -8,6 +8,10 @@ let readTail: string[] = [] let sendAnswers: ('ok' | 'timeout' | 'refused')[] = [] /** Tail the host shows after the nth send (the fresh agent came up), if set. */ let tailAfterSend: string[] | null = null +/** What each terminal.ensureAgentSession answers, in order. */ +let ensureAnswers: { handle: string; disposition: 'created' | 'adopted' }[] = [] +/** What terminal.list answers, per call (the last one repeats). */ +let listAnswers: { handle: string; tabId: string; leafId: string; connected?: boolean }[][] = [] vi.mock('@/runtime/rpc/calls/runtime-rpc-call', () => ({ callRuntimeRpc: async (target: { environmentId: string }, method: string, params: unknown) => { @@ -15,6 +19,14 @@ vi.mock('@/runtime/rpc/calls/runtime-rpc-call', () => ({ if (method === 'terminal.read') { return { terminal: { tail: readTail } } } + if (method === 'terminal.ensureAgentSession') { + const answer = ensureAnswers.shift() ?? { handle: 'term_resumed', disposition: 'created' } + return { terminal: { handle: answer.handle }, disposition: answer.disposition } + } + if (method === 'terminal.list') { + const answer = listAnswers.length > 1 ? listAnswers.shift() : listAnswers[0] + return { terminals: answer ?? [] } + } if (method === 'terminal.send') { if (tailAfterSend) { readTail = tailAfterSend @@ -34,8 +46,12 @@ vi.mock( () => ({ agentProviderSessionForHostEnsure: async (_env: string, session: unknown) => session }) ) -const { freshIfResumeFoundNothing, resumeRemotePaneAgent, stopRemotePaneTerminalForRestart } = - await import('./cloud-pane-restart-stop') +const { + freshIfResumeFoundNothing, + resumeRemotePaneAgent, + stopRemotePaneTerminalForRestart, + waitForResumedTerminalReady +} = await import('./cloud-pane-restart-stop') describe('stopRemotePaneTerminalForRestart (RA-11 paired pane)', () => { beforeEach(() => { @@ -72,9 +88,60 @@ describe('stopRemotePaneTerminalForRestart (RA-11 paired pane)', () => { }) }) +const RESUME_ARGS = { + ptyId: 'remote:env-1@@term_a', + worktreeId: 'wt-1', + tabId: 'web-terminal-host-tab%3A%3Aleaf', + leafId: 'leaf-1', + agent: 'claude' as const, + providerSession: { key: 'session_id' as const, id: 'sess-1' } +} + describe('resumeRemotePaneAgent (RA-11 paired pane)', () => { beforeEach(() => { calls.length = 0 + ensureAnswers = [] + showPtyId = 'host-pty-7' + stopResult = { stoppedPtyIds: ['host-pty-7'], postStopVerified: true } + }) + + it('returns the terminal the host ensured, so the pane can bind exactly it', async () => { + ensureAnswers = [{ handle: 'term_resumed', disposition: 'created' }] + await expect(resumeRemotePaneAgent(RESUME_ARGS)).resolves.toEqual({ + handle: 'term_resumed', + disposition: 'created' + }) + }) + + it('an adopted live terminal (the resume never ran) is stopped and the resume ensured once more', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + ensureAnswers = [ + { handle: 'term_original', disposition: 'adopted' }, + { handle: 'term_resumed', disposition: 'created' } + ] + await expect(resumeRemotePaneAgent(RESUME_ARGS)).resolves.toEqual({ + handle: 'term_resumed', + disposition: 'created' + }) + expect(calls.map((c) => c.method)).toEqual([ + 'terminal.ensureAgentSession', + 'terminal.show', + 'terminal.stopExact', + 'terminal.ensureAgentSession' + ]) + expect(calls[1]).toMatchObject({ params: { terminal: 'term_original' } }) + warn.mockRestore() + }) + + it('adopted twice fails the restart (never a silent new conversation)', async () => { + const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined) + ensureAnswers = [ + { handle: 'term_original', disposition: 'adopted' }, + { handle: 'term_original2', disposition: 'adopted' } + ] + await expect(resumeRemotePaneAgent(RESUME_ARGS)).rejects.toThrow('restart_resume_adopted') + expect(calls.filter((c) => c.method === 'terminal.ensureAgentSession')).toHaveLength(2) + warn.mockRestore() }) it('ensures the agent on its own session, placed in the same host tab and leaf', async () => { @@ -100,6 +167,43 @@ describe('resumeRemotePaneAgent (RA-11 paired pane)', () => { }) }) +describe('waitForResumedTerminalReady (RA-11 resume race)', () => { + const sleep = () => Promise.resolve() + let clock = 0 + const now = () => (clock += 300) + const args = { + ptyId: 'remote:env-1@@term_a', + worktreeId: 'wt-1', + tabId: 'web-terminal-host-tab%3A%3Aleaf', + leafId: 'leaf-1', + handle: 'term_resumed', + sleep, + now + } + beforeEach(() => { + calls.length = 0 + clock = 0 + listAnswers = [] + }) + + it('waits until the ensured terminal itself is live in the pane’s own tab and leaf', async () => { + listAnswers = [ + [], + // Some other terminal ready in the leaf is not enough. + [{ handle: 'term_original', tabId: 'host-tab::leaf', leafId: 'leaf-1' }], + [{ handle: 'term_resumed', tabId: 'host-tab::leaf', leafId: 'leaf-1', connected: true }] + ] + await expect(waitForResumedTerminalReady(args)).resolves.toBeUndefined() + expect(calls.filter((c) => c.method === 'terminal.list')).toHaveLength(3) + expect(calls[0]).toMatchObject({ params: { handles: ['term_resumed'] } }) + }) + + it('throws when the ensured terminal never shows in that leaf', async () => { + listAnswers = [[{ handle: 'term_resumed', tabId: 'host-tab::leaf', leafId: 'other-leaf' }]] + await expect(waitForResumedTerminalReady(args)).rejects.toThrow('restart_resume_not_ready') + }) +}) + describe('freshIfResumeFoundNothing (RA-11e)', () => { const sleep = () => Promise.resolve() let clock = 0 @@ -163,7 +267,11 @@ describe('freshIfResumeFoundNothing (RA-11e)', () => { it('never types twice: a timed-out send whose Claude came up anyway is not sent again', async () => { readTail = ['No conversation found with session ID: s1', '$ '] sendAnswers = ['timeout'] - tailAfterSend = ['No conversation found with session ID: s1', '$ claude', 'Claude Code v2.1.319'] + tailAfterSend = [ + 'No conversation found with session ID: s1', + '$ claude', + 'Claude Code v2.1.319' + ] const relaunched = await freshIfResumeFoundNothing({ currentPtyId: () => 'remote:env-1@@term_new', replacedPtyId: 'remote:env-1@@term_old', diff --git a/src/renderer/src/lib/orca-cloud/cloud-pane-restart-stop.ts b/src/renderer/src/lib/orca-cloud/cloud-pane-restart-stop.ts index 3a21d9932d8..7ab7aaf8edd 100644 --- a/src/renderer/src/lib/orca-cloud/cloud-pane-restart-stop.ts +++ b/src/renderer/src/lib/orca-cloud/cloud-pane-restart-stop.ts @@ -3,7 +3,10 @@ // relaunches. Put exactly that one host terminal to sleep first (the reversible stop the pane-sleep // path uses: the leaf stays, as a sleeping surface); the reconnect then wakes it with --resume. // A plain stop is a close: the host drops the tab and there is nothing left to relaunch into. -import { parseRemoteRuntimePtyId } from '../../../../shared/remote-runtime/remote-runtime-pty-id' +import { + parseRemoteRuntimePtyId, + toRemoteRuntimePtyId +} from '../../../../shared/remote-runtime/remote-runtime-pty-id' import type { RuntimeTerminalShow } from '../../../../shared/runtime-terminal-contracts' import { callRuntimeRpc } from '@/runtime/rpc/calls/runtime-rpc-call' import { toRuntimeWorktreeSelector } from '@/runtime/rpc/targets/runtime-worktree-selector' @@ -27,7 +30,10 @@ export async function stopRemotePaneTerminalForRestart( if (!parsed?.environmentId) { throw new Error('restart_remote_pane_unowned') } - const target = { kind: 'environment' as const, environmentId: parsed.environmentId } + const target = { + kind: 'environment' as const, + environmentId: parsed.environmentId + } // The pane names its terminal by handle; the host stops by its own PTY id. const { terminal } = await callRuntimeRpc<{ terminal?: RuntimeTerminalShow }>( target, @@ -57,10 +63,20 @@ export async function stopRemotePaneTerminalForRestart( } } +/** The terminal the host ensured for the resume, and whether it spawned it or adopted a live one. */ +export type RemotePaneResume = { + handle: string + disposition: 'created' | 'adopted' +} + /** * Relaunch the pane's agent on its own conversation, in the same pane. Without this the host wakes * the sleeping pane with its original launch (a new conversation). Only for a resumable session; * a pane with none takes the host's fresh launch, which is what "Starts a new conversation" says. + * + * Returns the ensured terminal's handle so the pane binds to exactly it. `adopted` means a live + * owner already held the leaf (something woke it with its original launch first) and the resume + * never ran: stop that terminal and ensure once more; still adopted throws (Couldn't restart). */ export async function resumeRemotePaneAgent(args: { ptyId: string @@ -69,25 +85,126 @@ export async function resumeRemotePaneAgent(args: { leafId: string agent: AccountRestartAgent providerSession: AgentProviderSessionMetadata +}): Promise { + const environmentId = parseRemoteRuntimePtyId(args.ptyId)?.environmentId + if (!environmentId) { + throw new Error('restart_remote_pane_unowned') + } + const ensure = async (): Promise => { + const result = await callRuntimeRpc<{ + terminal?: { handle?: string } + disposition?: 'created' | 'adopted' + }>( + { kind: 'environment', environmentId }, + 'terminal.ensureAgentSession', + { + kind: 'explicit', + worktree: toRuntimeWorktreeSelector(args.worktreeId), + agent: args.agent, + providerSession: await agentProviderSessionForHostEnsure( + environmentId, + args.providerSession + ), + // The same pane: the host tab and leaf the stopped terminal lived in. + placement: { + tabId: toHostSessionTabId(args.tabId), + leafId: args.leafId + }, + presentation: 'background' + }, + { timeoutMs: 15_000 } + ) + const handle = result?.terminal?.handle + if (!handle) { + throw new Error('restart_resume_no_terminal') + } + return { + handle, + disposition: result?.disposition === 'adopted' ? 'adopted' : 'created' + } + } + const first = await ensure() + if (first.disposition !== 'adopted') { + return first + } + console.warn( + '[orca-cloud] RA-11 resume adopted a live terminal; stopping it and resuming again', + first.handle + ) + await stopRemotePaneTerminalForRestart( + toRemoteRuntimePtyId(first.handle, environmentId), + args.worktreeId + ) + const second = await ensure() + if (second.disposition === 'adopted') { + throw new Error('restart_resume_adopted') + } + return second +} + +// The ensured terminal shows up in the host's inventory within a second or two of the ensure. +const RESUMED_READY_WAIT_MS = 15_000 +const RESUMED_READY_POLL_MS = 300 + +/** + * RA-11 resume race: wait until the host lists exactly the ensured terminal `handle`, live, in the + * pane's own host tab and leaf. Only then may the pane reconnect: its activation reuses a ready leaf, + * and a leaf that is not ready yet is woken with its ORIGINAL launch (the resume is then lost). + * A turn-less `--resume` exits to a shell but its PTY stays live, so it still lists. Throws when the + * terminal never shows, so the restart fails visibly instead of reconnecting into the race. + */ +export async function waitForResumedTerminalReady(args: { + ptyId: string + worktreeId: string + tabId: string + leafId: string + handle: string + sleep?: (ms: number) => Promise + now?: () => number }): Promise { const environmentId = parseRemoteRuntimePtyId(args.ptyId)?.environmentId if (!environmentId) { throw new Error('restart_remote_pane_unowned') } - await callRuntimeRpc( - { kind: 'environment', environmentId }, - 'terminal.ensureAgentSession', - { - kind: 'explicit', - worktree: toRuntimeWorktreeSelector(args.worktreeId), - agent: args.agent, - providerSession: await agentProviderSessionForHostEnsure(environmentId, args.providerSession), - // The same pane: the host tab and leaf the stopped terminal lived in. - placement: { tabId: toHostSessionTabId(args.tabId), leafId: args.leafId }, - presentation: 'background' - }, - { timeoutMs: 15_000 } - ) + const sleep = + args.sleep ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))) + const now = args.now ?? Date.now + const hostTabId = toHostSessionTabId(args.tabId) + const deadline = now() + RESUMED_READY_WAIT_MS + while (now() < deadline) { + const listed = await callRuntimeRpc<{ + terminals?: { + handle?: string + tabId?: string + leafId?: string + connected?: boolean + verdict?: string + }[] + }>( + { kind: 'environment', environmentId }, + 'terminal.list', + { + worktree: toRuntimeWorktreeSelector(args.worktreeId), + handles: [args.handle], + includePreview: false, + includeVisualLayouts: false + }, + { timeoutMs: 5_000 } + ).catch(() => null) + const ready = (listed?.terminals ?? []).some( + (terminal) => + terminal.handle === args.handle && + terminal.tabId === hostTabId && + terminal.leafId === args.leafId && + terminal.connected !== false && + terminal.verdict !== 'dead' + ) + if (ready) { + return + } + await sleep(RESUMED_READY_POLL_MS) + } + throw new Error('restart_resume_not_ready') } /** What Claude prints when `--resume ` names a session that never had a conversation. */ @@ -164,7 +281,9 @@ async function sendFreshOnce( try { // The Enter rides in the text, not `enter: true`: the host holds an Enter send for the launched // Claude's composer (8 s), which never comes once it exited to the shell, then refuses it. - const sent = await callRuntimeRpc<{ send?: { accepted?: boolean; refusedReason?: string } }>( + const sent = await callRuntimeRpc<{ + send?: { accepted?: boolean; refusedReason?: string } + }>( target, 'terminal.send', { terminal: handle, text: `${freshCommand}\r` }, @@ -215,7 +334,10 @@ export async function freshIfResumeFoundNothing(args: { lastSeen = current.handle console.info('[orca-cloud] RA-11e watching relaunched terminal', current.handle) } - const target = { kind: 'environment' as const, environmentId: current.environmentId } + const target = { + kind: 'environment' as const, + environmentId: current.environmentId + } const read = await callRuntimeRpc<{ terminal?: { tail?: string[] } }>( target, 'terminal.read', diff --git a/src/renderer/src/lib/orca-cloud/cloud-restart-bind-check.ts b/src/renderer/src/lib/orca-cloud/cloud-restart-bind-check.ts new file mode 100644 index 00000000000..7059279d22c --- /dev/null +++ b/src/renderer/src/lib/orca-cloud/cloud-restart-bind-check.ts @@ -0,0 +1,63 @@ +// RA-11 resume race: after a cloud Restart reconnects, the pane must sit on the terminal the resume +// ensured. Anything else is the leaf's original launch (a new conversation), which the strip reports. +import { useAppStore } from '@/store' +import { + parseRemoteRuntimePtyId, + toRemoteRuntimePtyId +} from '../../../../shared/remote-runtime/remote-runtime-pty-id' + +// How long the pane may take to bind its relaunched terminal before the check gives up (the +// attempt's own timeout reports a pane that never binds). +const BIND_CHECK_MS = 30_000 + +function paneHandleOf(tabId: string, leafId: string): { ptyId: string; handle: string } | null { + const ptyId = + useAppStore.getState().terminalLayoutsByTabId[tabId]?.ptyIdsByLeafId?.[leafId] ?? null + const parsed = ptyId ? parseRemoteRuntimePtyId(ptyId) : null + return ptyId && parsed ? { ptyId, handle: parsed.handle } : null +} + +/** The pane id for `handle` on the same environment as `ptyId` (falls back to `ptyId`). */ +export function remotePtyIdWithHandle(ptyId: string, handle: string): string { + const environmentId = parseRemoteRuntimePtyId(ptyId)?.environmentId + return environmentId ? toRemoteRuntimePtyId(handle, environmentId) : ptyId +} + +/** Once the pane binds a new PTY, check it is `handle`; report anything else. */ +export function expectPaneBindsHandle(args: { + tabId: string + leafId: string + replacedPtyId: string + handle: string + onMismatch: (boundHandle: string) => void +}): void { + let done = false + const check = (): boolean => { + const bound = paneHandleOf(args.tabId, args.leafId) + if (!bound || bound.ptyId === args.replacedPtyId) { + return false + } + if (bound.handle !== args.handle) { + args.onMismatch(bound.handle) + } + return true + } + if (check()) { + return + } + const finish = (): void => { + done = true + unsubscribe() + clearTimeout(timer) + } + const unsubscribe = useAppStore.subscribe(() => { + if (!done && check()) { + finish() + } + }) + const timer = setTimeout(() => { + if (!done) { + finish() + } + }, BIND_CHECK_MS) +}