fix(cloud): RA-11 resume race: Restart resumes in the pane

A cloud Restart stopped the pane's host terminal, ensured a --resume in its
leaf, then reconnected. But the pane's own transport reacted to the stop
ending its stream by resubscribing (session.tabs.activate), which woke the
stopped leaf with its ORIGINAL launch before the ensure arrived; the ensure
then adopted that live terminal and the resume was lost (a new
conversation). Seen 3/3 on golden 4020131de0.

- Release the pane's transport before the stop (split releasePaneForRestart
  out of handleRestartPane), so nothing reattaches on its own.
- resumeRemotePaneAgent returns the ensured terminal; on 'adopted' it stops
  that terminal and ensures once more, else fails (Couldn't restart).
- Reconnect only after the host lists that exact terminal live in the
  pane's tab and leaf (bounded 15 s, else Couldn't restart).
- After reconnecting, the pane must bind that handle; any other handle
  shows Couldn't restart. A failed restart reattaches the pane so it is
  never left empty.
This commit is contained in:
Neil
2026-11-11 04:15:34 -08:00
parent 7b353c4e0f
commit afa471da29
8 changed files with 544 additions and 45 deletions
@@ -43,6 +43,8 @@ const TAB = 'tab-1'
const LEAF = '11111111-1111-4111-8111-111111111111'
const PTY = 'env-1@@pty-1'
const PANE_KEY = makePaneKey(TAB, LEAF)
/** The terminal a resume ensure placed in the pane's leaf. */
const RESUMED = { handle: 'term_resumed', disposition: 'created' as const }
function seed(state: AgentStatusEntry['state']): void {
useAppStore.setState({
@@ -291,6 +293,7 @@ describe('RA-11 Restart on a paired (remote) pane', () => {
const providerSession = { key: 'session_id' as const, id: 'sess-tab-1' }
const resumeRemotePane = vi.fn(async () => {
order.push('resume')
return RESUMED
})
const freshIfResumeFoundNothing = vi.fn(async () => false)
const handleRestartPane = vi.fn(() => order.push('reconnect'))
@@ -309,6 +312,7 @@ describe('RA-11 Restart on a paired (remote) pane', () => {
order.push('stop')
},
resumeRemotePane,
waitForResumedTerminal: () => Promise.resolve(),
freshIfResumeFoundNothing,
buildFreshCommand: (agent, leafId) => `fresh ${agent} ${leafId}`
})
@@ -368,7 +372,8 @@ describe('RA-11 Restart on a paired (remote) pane', () => {
}),
handleRestartPane,
stopRemotePane: () => Promise.resolve(),
resumeRemotePane: () => Promise.resolve(),
resumeRemotePane: () => Promise.resolve(RESUMED),
waitForResumedTerminal: () => Promise.resolve(),
freshIfResumeFoundNothing: (input) =>
realBackstop({ ...input, sleep: () => Promise.resolve() }),
buildFreshCommand: () => 'claude --dangerously-skip-permissions'
@@ -397,7 +402,8 @@ describe('RA-11 Restart on a paired (remote) pane', () => {
}),
handleRestartPane,
stopRemotePane: () => Promise.resolve(),
resumeRemotePane: () => Promise.resolve(),
resumeRemotePane: () => Promise.resolve(RESUMED),
waitForResumedTerminal: () => Promise.resolve(),
freshIfResumeFoundNothing: async () => true,
buildFreshCommand: () => 'claude'
})
@@ -425,7 +431,8 @@ describe('RA-11 Restart on a paired (remote) pane', () => {
}),
handleRestartPane,
stopRemotePane: () => Promise.resolve(),
resumeRemotePane: () => Promise.resolve(),
resumeRemotePane: () => Promise.resolve(RESUMED),
waitForResumedTerminal: () => Promise.resolve(),
freshIfResumeFoundNothing: () => Promise.reject(new Error('ra11e_fresh_send_failed')),
buildFreshCommand: () => 'claude'
})
@@ -438,6 +445,118 @@ describe('RA-11 Restart on a paired (remote) pane', () => {
warn.mockRestore()
})
// RA-11 resume race: the pane's own transport reattached when the stop ended its stream, woke the
// leaf with its original launch, and the resume was adopted away.
function runResume(over: {
waitForResumedTerminal?: () => Promise<void>
onReconnect?: () => void
order?: string[]
}) {
const order = over.order ?? []
const handleRestartPane = vi.fn(
(_paneId: number, _startup: unknown, options: { focus: boolean }) => {
order.push(options.focus ? 'reconnect(focus)' : 'reconnect')
over.onReconnect?.()
}
)
runReadyPaneAccountRestarts({
panes: [{ id: 7, leafId: LEAF }],
ptyIdOf: () => REMOTE,
readyRestartPtyIds: selectReadyAccountRestartPtyIds(useAppStore.getState(), TAB),
tabId: TAB,
consumePendingPaneAccountRestart: useAppStore.getState().consumePendingPaneAccountRestart,
buildAccountRestartStartup: () => ({
command: 'claude',
resumeProviderSession: { key: 'session_id' as const, id: 'sess-tab-1' }
}),
handleRestartPane,
releasePaneForRestart: () => {
order.push('release')
},
stopRemotePane: async () => {
order.push('stop')
},
resumeRemotePane: async () => {
order.push('ensure')
return { handle: 'term_resumed', disposition: 'created' as const }
},
waitForResumedTerminal: async () => {
order.push('ready?')
await (over.waitForResumedTerminal?.() ?? Promise.resolve())
},
freshIfResumeFoundNothing: async () => false,
buildFreshCommand: () => 'claude'
})
return { handleRestartPane, order }
}
function bindPane(handle: string): void {
const layouts = useAppStore.getState().terminalLayoutsByTabId
useAppStore.setState({
terminalLayoutsByTabId: {
...layouts,
[TAB]: { ...layouts[TAB]!, ptyIdsByLeafId: { [LEAF]: `remote:env-1@@${handle}` } }
}
})
}
it('lets go of the pane before the stop, then stop → resume → ready → one reconnect', async () => {
clickRemoteRestart()
const { handleRestartPane, order } = runResume({})
await vi.waitFor(() => expect(handleRestartPane).toHaveBeenCalledTimes(1))
expect(order).toEqual(['release', 'stop', 'ensure', 'ready?', 'reconnect'])
})
it('never reconnects before the resumed terminal is live in the leaf', async () => {
clickRemoteRestart()
let ready: () => void = () => undefined
const gate = new Promise<void>((resolve) => {
ready = resolve
})
const { handleRestartPane, order } = runResume({ waitForResumedTerminal: () => gate })
await vi.waitFor(() => expect(order).toContain('ready?'))
await Promise.resolve()
expect(handleRestartPane).not.toHaveBeenCalled()
ready()
await vi.waitFor(() => expect(handleRestartPane).toHaveBeenCalledTimes(1))
})
it('a resumed terminal that never shows fails the restart and reattaches the pane (never empty)', async () => {
clickRemoteRestart()
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const { handleRestartPane } = runResume({
waitForResumedTerminal: () => Promise.reject(new Error('restart_resume_not_ready'))
})
await vi.waitFor(() => expect(cloudAgentRestartAttempt(PANE_KEY, 0)?.phase).toBe('failed'))
expect(cloudAgentRestartAttempt(PANE_KEY, 0)?.failure).toBe('hostUnresponsive')
expect(handleRestartPane).toHaveBeenCalledTimes(1)
expect(handleRestartPane.mock.calls[0]?.[2]).toEqual({ focus: false })
warn.mockRestore()
})
it('the pane binding the resumed terminal is a clean restart', async () => {
clickRemoteRestart()
const { handleRestartPane } = runResume({ onReconnect: () => bindPane('term_resumed') })
await vi.waitFor(() => expect(handleRestartPane).toHaveBeenCalledTimes(1))
await Promise.resolve()
expect(cloudAgentRestartAttempt(PANE_KEY, 0)?.phase).not.toBe('failed')
})
it('the pane binding any other terminal fails visibly (a new conversation is not a resume)', async () => {
clickRemoteRestart()
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const { handleRestartPane } = runResume({
onReconnect: () => {
settleCloudAgentRestart(PANE_KEY)
bindPane('term_original_launch')
}
})
await vi.waitFor(() => expect(handleRestartPane).toHaveBeenCalledTimes(1))
await vi.waitFor(() => expect(cloudAgentRestartAttempt(PANE_KEY, 0)?.phase).toBe('failed'))
expect(cloudAgentRestartAttempt(PANE_KEY, 0)?.ptyId).toBe('remote:env-1@@term_original_launch')
warn.mockRestore()
})
it('a failed resume on the host shows the failure and never reconnects', async () => {
clickRemoteRestart()
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
@@ -8,7 +8,7 @@ import {
import {
beginCloudAgentRestart,
failCloudAgentRestart,
failCloudAgentRestartFresh,
failCloudAgentRestartAfterLaunch,
noteCloudAgentRestartFresh,
isForcedCloudAgentRestart,
noteCloudAgentRestartLaunched
@@ -18,8 +18,13 @@ import type { usePaneProcessRestart } from '../process-exit/use-pane-process-res
import {
freshIfResumeFoundNothing,
resumeRemotePaneAgent,
waitForResumedTerminalReady,
stopRemotePaneTerminalForRestart
} from '@/lib/orca-cloud/cloud-pane-restart-stop'
import {
expectPaneBindsHandle,
remotePtyIdWithHandle
} from '@/lib/orca-cloud/cloud-restart-bind-check'
import type { AgentProviderSessionMetadata } from '../../../../../shared/agent-session/lifecycle/agent-session-resume'
import { makePaneKey } from '../../../../../shared/workspace-layout/stable-pane-id'
import {
@@ -39,6 +44,7 @@ export function useReadyPaneAccountRestarts({
tabId,
buildAccountRestartStartup,
handleRestartPane,
releasePaneForRestart,
panePtyLayoutBindings
}: Pick<
TerminalPaneCloseController,
@@ -48,6 +54,7 @@ export function useReadyPaneAccountRestarts({
typeof usePaneAccountRestartStartup
>['buildAccountRestartStartup']
handleRestartPane: ReturnType<typeof usePaneProcessRestart>['handleRestartPane']
releasePaneForRestart: ReturnType<typeof usePaneProcessRestart>['releasePaneForRestart']
panePtyLayoutBindings: TerminalPaneCloseController['savedLayout']['ptyIdsByLeafId']
}) {
// Why primitive ids: unrelated status updates stay shallow-equal and skip this pane.
@@ -66,12 +73,14 @@ export function useReadyPaneAccountRestarts({
tabId,
consumePendingPaneAccountRestart,
buildAccountRestartStartup,
handleRestartPane
handleRestartPane,
releasePaneForRestart
})
}, [
buildAccountRestartStartup,
consumePendingPaneAccountRestart,
handleRestartPane,
releasePaneForRestart,
panePtyLayoutBindings,
readyRestartPtyIds,
managerRef,
@@ -116,10 +125,18 @@ export function runReadyPaneAccountRestarts<
consumePendingPaneAccountRestart: (ptyId: string) => AccountRestartAgent | null
buildAccountRestartStartup: (agent: AccountRestartAgent, leafId: string) => Startup | null
handleRestartPane: (paneId: number, startup: Startup, options: { focus: boolean }) => void
/**
* Destroys the pane's transport without reconnecting. A paired pane's transport otherwise reacts to
* its own host terminal stopping by reattaching, which wakes the leaf with its original launch
* before the resume lands (RA-11 resume race).
*/
releasePaneForRestart?: (paneId: number) => void
/** Stops the one host terminal behind a paired pane before its restart reconnects. */
stopRemotePane?: (ptyId: string, worktreeId: string) => Promise<void>
/** Relaunches the paired pane's agent on its own conversation, in the same pane. */
resumeRemotePane?: typeof resumeRemotePaneAgent
/** RA-11 resume race: wait until the resumed terminal is live in the pane's own leaf. */
waitForResumedTerminal?: typeof waitForResumedTerminalReady
/** RA-11e: start fresh in the same pane when a resume found no conversation. */
freshIfResumeFoundNothing?: typeof freshIfResumeFoundNothing
/** The fresh-launch command for this tab's pane (no session to resume). */
@@ -128,6 +145,7 @@ export function runReadyPaneAccountRestarts<
const args = {
stopRemotePane: stopRemotePaneTerminalForRestart,
resumeRemotePane: resumeRemotePaneAgent,
waitForResumedTerminal: waitForResumedTerminalReady,
freshIfResumeFoundNothing,
buildFreshCommand: (agent: AccountRestartAgent, leafId: string) =>
buildFreshAccountRestartCommand(agent, {
@@ -187,7 +205,9 @@ export function runReadyPaneAccountRestarts<
noteCloudAgentRestartLaunched(ptyId)
}
// Why: one Claude switch restarts many panes at once; none was the one the user clicked.
args.handleRestartPane(pane.id, launchedStartup, { focus: agent === 'codex' })
args.handleRestartPane(pane.id, launchedStartup, {
focus: agent === 'codex'
})
}
if (!worktreeId) {
launch()
@@ -196,6 +216,9 @@ export function runReadyPaneAccountRestarts<
// Stop that one host terminal first (its exit is ours, not the pane's), then reconnect.
const store = useAppStore.getState()
store.suppressPtyExit(ptyId)
// Order is stop → ensure → reconnect: let go of the pane's transport before the stop, or its own
// recovery reattaches the moment the stop ends its stream and wakes the original launch.
args.releasePaneForRestart?.(pane.id)
void args
.stopRemotePane(ptyId, worktreeId)
.then(() =>
@@ -210,9 +233,44 @@ export function runReadyPaneAccountRestarts<
})
: undefined
)
.then(async (resumed) => {
// RA-11 resume race: reconnect only once the resumed terminal is live in this very leaf;
// an earlier reconnect would wake the leaf with its original launch and drop the resume.
if (resumed) {
await args.waitForResumedTerminal({
ptyId,
worktreeId,
tabId: args.tabId,
leafId: pane.leafId,
handle: resumed.handle
})
}
return resumed
})
.then(
() => {
(resumed) => {
launch()
if (resumed && tracked) {
// The pane must end on the resumed terminal; anything else is a new conversation.
expectPaneBindsHandle({
tabId: args.tabId,
leafId: pane.leafId,
replacedPtyId: ptyId,
handle: resumed.handle,
onMismatch: (bound) => {
console.warn(
'[orca-cloud] RA-11 restart bound',
bound,
'not the resumed',
resumed.handle
)
failCloudAgentRestartAfterLaunch(makePaneKey(args.tabId, pane.leafId), {
agent,
ptyId: remotePtyIdWithHandle(ptyId, bound)
})
}
})
}
if (providerSession) {
// RA-11e: a session with no conversation can't resume; start fresh in the same pane, once.
void Promise.resolve()
@@ -239,7 +297,7 @@ export function runReadyPaneAccountRestarts<
console.warn('[orca-cloud] fresh fallback failed:', err)
if (tracked) {
// The pane is a bare shell now: say so with Try again, never leave it quiet.
failCloudAgentRestartFresh(makePaneKey(args.tabId, pane.leafId), {
failCloudAgentRestartAfterLaunch(makePaneKey(args.tabId, pane.leafId), {
agent,
ptyId:
useAppStore.getState().terminalLayoutsByTabId[args.tabId]?.ptyIdsByLeafId?.[
@@ -255,6 +313,11 @@ export function runReadyPaneAccountRestarts<
useAppStore.getState().consumeSuppressedPtyExit(ptyId)
// The host did not stop or relaunch it: the strip names that reason.
failCloudAgentRestart(ptyId, 'hostUnresponsive')
if (args.releasePaneForRestart) {
// Its transport was let go before the stop: reattach so the pane is never left empty
// (the still-live terminal, or the leaf's own launch if the stop went through).
args.handleRestartPane(pane.id, launchedStartup, { focus: false })
}
if (!forced && agent === 'codex') {
// The switch was answered on the old account: ask again rather than leave it silently stale.
useAppStore.getState().reopenCodexRestartPrompt(ptyId)
@@ -49,7 +49,7 @@ export function useTerminalPaneProcessExitActions(controller: TerminalPaneCloseC
const { buildAccountRestartStartup } = usePaneAccountRestartStartup({ tabId, worktreeId })
const { handleRestartPane } = usePaneProcessRestart({
const { handleRestartPane, releasePaneForRestart } = usePaneProcessRestart({
clearCodexRestartNotice,
clearExitedPanePtyLayoutBinding,
clearRuntimePaneTitle,
@@ -105,6 +105,7 @@ export function useTerminalPaneProcessExitActions(controller: TerminalPaneCloseC
tabId,
buildAccountRestartStartup,
handleRestartPane,
releasePaneForRestart,
panePtyLayoutBindings
})
@@ -77,13 +77,13 @@ export function usePaneProcessRestart({
| 'updateTabTitle'
| 'worktreeId'
>) {
const handleRestartPane = useCallback(
(paneId: number, startup: PtyConnectionDeps['startup'], options: { focus: boolean }) => {
const manager = managerRef.current
const pane = manager?.getPanes().find((candidate) => candidate.id === paneId)
if (!manager || !pane) {
return
}
/**
* Let go of the pane's current PTY without connecting a new one: its transport is destroyed, so its
* own recovery can no longer reattach (or wake the host leaf) on its own. Returns the PTY id local
* main should replace, if any.
*/
const releasePaneForRestart = useCallback(
(paneId: number): string | null => {
const transport = paneTransportsRef.current.get(paneId)
const panePtyBinding = panePtyBindingsRef.current.get(paneId)
const existingPtyId = transport?.getPtyId()
@@ -94,9 +94,32 @@ export function usePaneProcessRestart({
}
panePtyBinding?.dispose()
panePtyBindingsRef.current.delete(paneId)
syncPanePtyLayoutBinding(paneId, null)
if (transport || panePtyBinding) {
syncPanePtyLayoutBinding(paneId, null)
}
const replacesPtyId = releasePaneTransportForRestart(transport)
paneTransportsRef.current.delete(paneId)
return replacesPtyId
},
[
clearCodexRestartNotice,
clearTabPtyId,
panePtyBindingsRef,
paneTransportsRef,
suppressPtyExit,
syncPanePtyLayoutBinding,
tabId
]
)
const handleRestartPane = useCallback(
(paneId: number, startup: PtyConnectionDeps['startup'], options: { focus: boolean }) => {
const manager = managerRef.current
const pane = manager?.getPanes().find((candidate) => candidate.id === paneId)
if (!manager || !pane) {
return
}
const replacesPtyId = releasePaneForRestart(paneId)
setCacheTimerStartedAt(makePaneKey(tabId, pane.leafId), null)
setTerminalError(null)
setTerminalErrorsByPaneId((current) => clearPaneTerminalError(current, paneId))
@@ -140,7 +163,7 @@ export function usePaneProcessRestart({
}
},
[
clearCodexRestartNotice,
releasePaneForRestart,
clearExitedPanePtyLayoutBinding,
clearRuntimePaneTitle,
clearTabPtyId,
@@ -166,7 +189,6 @@ export function usePaneProcessRestart({
setRuntimePaneTitle,
setTerminalError,
showRestoredSessionBanner,
suppressPtyExit,
syncPanePtyLayoutBinding,
tabId,
updateTabPtyId,
@@ -175,5 +197,5 @@ export function usePaneProcessRestart({
setTerminalErrorsByPaneId
]
)
return { handleRestartPane }
return { handleRestartPane, releasePaneForRestart }
}
@@ -180,11 +180,12 @@ export function cloudAgentRestartStartedFresh(paneKey: string, _revision: number
}
/**
* RA-11e: the resume found no conversation and the fresh start could not be typed, so the pane is a
* bare shell. The attempt already settled on the new PTY; show "Couldn't restart" again so the
* person can Try again, never a quiet shell.
* The restart relaunched but did not end where it must: the fresh start after an empty resume could
* not be typed (RA-11e, a bare shell), or the pane bound a terminal other than the resumed one (RA-11
* resume race, a new conversation). The attempt may already have settled on the new PTY; show
* "Couldn't restart" again so the person can Try again, never a quiet wrong result.
*/
export function failCloudAgentRestartFresh(
export function failCloudAgentRestartAfterLaunch(
paneKey: string,
args: { agent: HostAgentSignInAgent; ptyId: string }
): void {
@@ -8,6 +8,10 @@ let readTail: string[] = []
let sendAnswers: ('ok' | 'timeout' | 'refused')[] = []
/** Tail the host shows after the nth send (the fresh agent came up), if set. */
let tailAfterSend: string[] | null = null
/** What each terminal.ensureAgentSession answers, in order. */
let ensureAnswers: { handle: string; disposition: 'created' | 'adopted' }[] = []
/** What terminal.list answers, per call (the last one repeats). */
let listAnswers: { handle: string; tabId: string; leafId: string; connected?: boolean }[][] = []
vi.mock('@/runtime/rpc/calls/runtime-rpc-call', () => ({
callRuntimeRpc: async (target: { environmentId: string }, method: string, params: unknown) => {
@@ -15,6 +19,14 @@ vi.mock('@/runtime/rpc/calls/runtime-rpc-call', () => ({
if (method === 'terminal.read') {
return { terminal: { tail: readTail } }
}
if (method === 'terminal.ensureAgentSession') {
const answer = ensureAnswers.shift() ?? { handle: 'term_resumed', disposition: 'created' }
return { terminal: { handle: answer.handle }, disposition: answer.disposition }
}
if (method === 'terminal.list') {
const answer = listAnswers.length > 1 ? listAnswers.shift() : listAnswers[0]
return { terminals: answer ?? [] }
}
if (method === 'terminal.send') {
if (tailAfterSend) {
readTail = tailAfterSend
@@ -34,8 +46,12 @@ vi.mock(
() => ({ agentProviderSessionForHostEnsure: async (_env: string, session: unknown) => session })
)
const { freshIfResumeFoundNothing, resumeRemotePaneAgent, stopRemotePaneTerminalForRestart } =
await import('./cloud-pane-restart-stop')
const {
freshIfResumeFoundNothing,
resumeRemotePaneAgent,
stopRemotePaneTerminalForRestart,
waitForResumedTerminalReady
} = await import('./cloud-pane-restart-stop')
describe('stopRemotePaneTerminalForRestart (RA-11 paired pane)', () => {
beforeEach(() => {
@@ -72,9 +88,60 @@ describe('stopRemotePaneTerminalForRestart (RA-11 paired pane)', () => {
})
})
const RESUME_ARGS = {
ptyId: 'remote:env-1@@term_a',
worktreeId: 'wt-1',
tabId: 'web-terminal-host-tab%3A%3Aleaf',
leafId: 'leaf-1',
agent: 'claude' as const,
providerSession: { key: 'session_id' as const, id: 'sess-1' }
}
describe('resumeRemotePaneAgent (RA-11 paired pane)', () => {
beforeEach(() => {
calls.length = 0
ensureAnswers = []
showPtyId = 'host-pty-7'
stopResult = { stoppedPtyIds: ['host-pty-7'], postStopVerified: true }
})
it('returns the terminal the host ensured, so the pane can bind exactly it', async () => {
ensureAnswers = [{ handle: 'term_resumed', disposition: 'created' }]
await expect(resumeRemotePaneAgent(RESUME_ARGS)).resolves.toEqual({
handle: 'term_resumed',
disposition: 'created'
})
})
it('an adopted live terminal (the resume never ran) is stopped and the resume ensured once more', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
ensureAnswers = [
{ handle: 'term_original', disposition: 'adopted' },
{ handle: 'term_resumed', disposition: 'created' }
]
await expect(resumeRemotePaneAgent(RESUME_ARGS)).resolves.toEqual({
handle: 'term_resumed',
disposition: 'created'
})
expect(calls.map((c) => c.method)).toEqual([
'terminal.ensureAgentSession',
'terminal.show',
'terminal.stopExact',
'terminal.ensureAgentSession'
])
expect(calls[1]).toMatchObject({ params: { terminal: 'term_original' } })
warn.mockRestore()
})
it('adopted twice fails the restart (never a silent new conversation)', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
ensureAnswers = [
{ handle: 'term_original', disposition: 'adopted' },
{ handle: 'term_original2', disposition: 'adopted' }
]
await expect(resumeRemotePaneAgent(RESUME_ARGS)).rejects.toThrow('restart_resume_adopted')
expect(calls.filter((c) => c.method === 'terminal.ensureAgentSession')).toHaveLength(2)
warn.mockRestore()
})
it('ensures the agent on its own session, placed in the same host tab and leaf', async () => {
@@ -100,6 +167,43 @@ describe('resumeRemotePaneAgent (RA-11 paired pane)', () => {
})
})
describe('waitForResumedTerminalReady (RA-11 resume race)', () => {
const sleep = () => Promise.resolve()
let clock = 0
const now = () => (clock += 300)
const args = {
ptyId: 'remote:env-1@@term_a',
worktreeId: 'wt-1',
tabId: 'web-terminal-host-tab%3A%3Aleaf',
leafId: 'leaf-1',
handle: 'term_resumed',
sleep,
now
}
beforeEach(() => {
calls.length = 0
clock = 0
listAnswers = []
})
it('waits until the ensured terminal itself is live in the pane’s own tab and leaf', async () => {
listAnswers = [
[],
// Some other terminal ready in the leaf is not enough.
[{ handle: 'term_original', tabId: 'host-tab::leaf', leafId: 'leaf-1' }],
[{ handle: 'term_resumed', tabId: 'host-tab::leaf', leafId: 'leaf-1', connected: true }]
]
await expect(waitForResumedTerminalReady(args)).resolves.toBeUndefined()
expect(calls.filter((c) => c.method === 'terminal.list')).toHaveLength(3)
expect(calls[0]).toMatchObject({ params: { handles: ['term_resumed'] } })
})
it('throws when the ensured terminal never shows in that leaf', async () => {
listAnswers = [[{ handle: 'term_resumed', tabId: 'host-tab::leaf', leafId: 'other-leaf' }]]
await expect(waitForResumedTerminalReady(args)).rejects.toThrow('restart_resume_not_ready')
})
})
describe('freshIfResumeFoundNothing (RA-11e)', () => {
const sleep = () => Promise.resolve()
let clock = 0
@@ -163,7 +267,11 @@ describe('freshIfResumeFoundNothing (RA-11e)', () => {
it('never types twice: a timed-out send whose Claude came up anyway is not sent again', async () => {
readTail = ['No conversation found with session ID: s1', '$ ']
sendAnswers = ['timeout']
tailAfterSend = ['No conversation found with session ID: s1', '$ claude', 'Claude Code v2.1.319']
tailAfterSend = [
'No conversation found with session ID: s1',
'$ claude',
'Claude Code v2.1.319'
]
const relaunched = await freshIfResumeFoundNothing({
currentPtyId: () => 'remote:env-1@@term_new',
replacedPtyId: 'remote:env-1@@term_old',
@@ -3,7 +3,10 @@
// relaunches. Put exactly that one host terminal to sleep first (the reversible stop the pane-sleep
// path uses: the leaf stays, as a sleeping surface); the reconnect then wakes it with --resume.
// A plain stop is a close: the host drops the tab and there is nothing left to relaunch into.
import { parseRemoteRuntimePtyId } from '../../../../shared/remote-runtime/remote-runtime-pty-id'
import {
parseRemoteRuntimePtyId,
toRemoteRuntimePtyId
} from '../../../../shared/remote-runtime/remote-runtime-pty-id'
import type { RuntimeTerminalShow } from '../../../../shared/runtime-terminal-contracts'
import { callRuntimeRpc } from '@/runtime/rpc/calls/runtime-rpc-call'
import { toRuntimeWorktreeSelector } from '@/runtime/rpc/targets/runtime-worktree-selector'
@@ -27,7 +30,10 @@ export async function stopRemotePaneTerminalForRestart(
if (!parsed?.environmentId) {
throw new Error('restart_remote_pane_unowned')
}
const target = { kind: 'environment' as const, environmentId: parsed.environmentId }
const target = {
kind: 'environment' as const,
environmentId: parsed.environmentId
}
// The pane names its terminal by handle; the host stops by its own PTY id.
const { terminal } = await callRuntimeRpc<{ terminal?: RuntimeTerminalShow }>(
target,
@@ -57,10 +63,20 @@ export async function stopRemotePaneTerminalForRestart(
}
}
/** The terminal the host ensured for the resume, and whether it spawned it or adopted a live one. */
export type RemotePaneResume = {
handle: string
disposition: 'created' | 'adopted'
}
/**
* Relaunch the pane's agent on its own conversation, in the same pane. Without this the host wakes
* the sleeping pane with its original launch (a new conversation). Only for a resumable session;
* a pane with none takes the host's fresh launch, which is what "Starts a new conversation" says.
*
* Returns the ensured terminal's handle so the pane binds to exactly it. `adopted` means a live
* owner already held the leaf (something woke it with its original launch first) and the resume
* never ran: stop that terminal and ensure once more; still adopted throws (Couldn't restart).
*/
export async function resumeRemotePaneAgent(args: {
ptyId: string
@@ -69,25 +85,126 @@ export async function resumeRemotePaneAgent(args: {
leafId: string
agent: AccountRestartAgent
providerSession: AgentProviderSessionMetadata
}): Promise<RemotePaneResume> {
const environmentId = parseRemoteRuntimePtyId(args.ptyId)?.environmentId
if (!environmentId) {
throw new Error('restart_remote_pane_unowned')
}
const ensure = async (): Promise<RemotePaneResume> => {
const result = await callRuntimeRpc<{
terminal?: { handle?: string }
disposition?: 'created' | 'adopted'
}>(
{ kind: 'environment', environmentId },
'terminal.ensureAgentSession',
{
kind: 'explicit',
worktree: toRuntimeWorktreeSelector(args.worktreeId),
agent: args.agent,
providerSession: await agentProviderSessionForHostEnsure(
environmentId,
args.providerSession
),
// The same pane: the host tab and leaf the stopped terminal lived in.
placement: {
tabId: toHostSessionTabId(args.tabId),
leafId: args.leafId
},
presentation: 'background'
},
{ timeoutMs: 15_000 }
)
const handle = result?.terminal?.handle
if (!handle) {
throw new Error('restart_resume_no_terminal')
}
return {
handle,
disposition: result?.disposition === 'adopted' ? 'adopted' : 'created'
}
}
const first = await ensure()
if (first.disposition !== 'adopted') {
return first
}
console.warn(
'[orca-cloud] RA-11 resume adopted a live terminal; stopping it and resuming again',
first.handle
)
await stopRemotePaneTerminalForRestart(
toRemoteRuntimePtyId(first.handle, environmentId),
args.worktreeId
)
const second = await ensure()
if (second.disposition === 'adopted') {
throw new Error('restart_resume_adopted')
}
return second
}
// The ensured terminal shows up in the host's inventory within a second or two of the ensure.
const RESUMED_READY_WAIT_MS = 15_000
const RESUMED_READY_POLL_MS = 300
/**
* RA-11 resume race: wait until the host lists exactly the ensured terminal `handle`, live, in the
* pane's own host tab and leaf. Only then may the pane reconnect: its activation reuses a ready leaf,
* and a leaf that is not ready yet is woken with its ORIGINAL launch (the resume is then lost).
* A turn-less `--resume` exits to a shell but its PTY stays live, so it still lists. Throws when the
* terminal never shows, so the restart fails visibly instead of reconnecting into the race.
*/
export async function waitForResumedTerminalReady(args: {
ptyId: string
worktreeId: string
tabId: string
leafId: string
handle: string
sleep?: (ms: number) => Promise<void>
now?: () => number
}): Promise<void> {
const environmentId = parseRemoteRuntimePtyId(args.ptyId)?.environmentId
if (!environmentId) {
throw new Error('restart_remote_pane_unowned')
}
await callRuntimeRpc(
{ kind: 'environment', environmentId },
'terminal.ensureAgentSession',
{
kind: 'explicit',
worktree: toRuntimeWorktreeSelector(args.worktreeId),
agent: args.agent,
providerSession: await agentProviderSessionForHostEnsure(environmentId, args.providerSession),
// The same pane: the host tab and leaf the stopped terminal lived in.
placement: { tabId: toHostSessionTabId(args.tabId), leafId: args.leafId },
presentation: 'background'
},
{ timeoutMs: 15_000 }
)
const sleep =
args.sleep ?? ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)))
const now = args.now ?? Date.now
const hostTabId = toHostSessionTabId(args.tabId)
const deadline = now() + RESUMED_READY_WAIT_MS
while (now() < deadline) {
const listed = await callRuntimeRpc<{
terminals?: {
handle?: string
tabId?: string
leafId?: string
connected?: boolean
verdict?: string
}[]
}>(
{ kind: 'environment', environmentId },
'terminal.list',
{
worktree: toRuntimeWorktreeSelector(args.worktreeId),
handles: [args.handle],
includePreview: false,
includeVisualLayouts: false
},
{ timeoutMs: 5_000 }
).catch(() => null)
const ready = (listed?.terminals ?? []).some(
(terminal) =>
terminal.handle === args.handle &&
terminal.tabId === hostTabId &&
terminal.leafId === args.leafId &&
terminal.connected !== false &&
terminal.verdict !== 'dead'
)
if (ready) {
return
}
await sleep(RESUMED_READY_POLL_MS)
}
throw new Error('restart_resume_not_ready')
}
/** What Claude prints when `--resume <id>` names a session that never had a conversation. */
@@ -164,7 +281,9 @@ async function sendFreshOnce(
try {
// The Enter rides in the text, not `enter: true`: the host holds an Enter send for the launched
// Claude's composer (8 s), which never comes once it exited to the shell, then refuses it.
const sent = await callRuntimeRpc<{ send?: { accepted?: boolean; refusedReason?: string } }>(
const sent = await callRuntimeRpc<{
send?: { accepted?: boolean; refusedReason?: string }
}>(
target,
'terminal.send',
{ terminal: handle, text: `${freshCommand}\r` },
@@ -215,7 +334,10 @@ export async function freshIfResumeFoundNothing(args: {
lastSeen = current.handle
console.info('[orca-cloud] RA-11e watching relaunched terminal', current.handle)
}
const target = { kind: 'environment' as const, environmentId: current.environmentId }
const target = {
kind: 'environment' as const,
environmentId: current.environmentId
}
const read = await callRuntimeRpc<{ terminal?: { tail?: string[] } }>(
target,
'terminal.read',
@@ -0,0 +1,63 @@
// RA-11 resume race: after a cloud Restart reconnects, the pane must sit on the terminal the resume
// ensured. Anything else is the leaf's original launch (a new conversation), which the strip reports.
import { useAppStore } from '@/store'
import {
parseRemoteRuntimePtyId,
toRemoteRuntimePtyId
} from '../../../../shared/remote-runtime/remote-runtime-pty-id'
// How long the pane may take to bind its relaunched terminal before the check gives up (the
// attempt's own timeout reports a pane that never binds).
const BIND_CHECK_MS = 30_000
function paneHandleOf(tabId: string, leafId: string): { ptyId: string; handle: string } | null {
const ptyId =
useAppStore.getState().terminalLayoutsByTabId[tabId]?.ptyIdsByLeafId?.[leafId] ?? null
const parsed = ptyId ? parseRemoteRuntimePtyId(ptyId) : null
return ptyId && parsed ? { ptyId, handle: parsed.handle } : null
}
/** The pane id for `handle` on the same environment as `ptyId` (falls back to `ptyId`). */
export function remotePtyIdWithHandle(ptyId: string, handle: string): string {
const environmentId = parseRemoteRuntimePtyId(ptyId)?.environmentId
return environmentId ? toRemoteRuntimePtyId(handle, environmentId) : ptyId
}
/** Once the pane binds a new PTY, check it is `handle`; report anything else. */
export function expectPaneBindsHandle(args: {
tabId: string
leafId: string
replacedPtyId: string
handle: string
onMismatch: (boundHandle: string) => void
}): void {
let done = false
const check = (): boolean => {
const bound = paneHandleOf(args.tabId, args.leafId)
if (!bound || bound.ptyId === args.replacedPtyId) {
return false
}
if (bound.handle !== args.handle) {
args.onMismatch(bound.handle)
}
return true
}
if (check()) {
return
}
const finish = (): void => {
done = true
unsubscribe()
clearTimeout(timer)
}
const unsubscribe = useAppStore.subscribe(() => {
if (!done && check()) {
finish()
}
})
const timer = setTimeout(() => {
if (!done) {
finish()
}
}, BIND_CHECK_MS)
}