Merge pull request #429 from nwparker/cloud/ra11-bind-ensure

fix(cloud): RA-11 resume race: Restart resumes in the pane (P1)
This commit is contained in:
Neil
2026-11-11 04:35:14 -08:00
committed by GitHub
8 changed files with 544 additions and 45 deletions
@@ -43,6 +43,8 @@ const TAB = 'tab-1'
const LEAF = '11111111-1111-4111-8111-111111111111'
const PTY = 'env-1@@pty-1'
const PANE_KEY = makePaneKey(TAB, LEAF)
/** The terminal a resume ensure placed in the pane's leaf. */
const RESUMED = { handle: 'term_resumed', disposition: 'created' as const }
function seed(state: AgentStatusEntry['state']): void {
useAppStore.setState({
@@ -291,6 +293,7 @@ describe('RA-11 Restart on a paired (remote) pane', () => {
const providerSession = { key: 'session_id' as const, id: 'sess-tab-1' }
const resumeRemotePane = vi.fn(async () => {
order.push('resume')
return RESUMED
})
const freshIfResumeFoundNothing = vi.fn(async () => false)
const handleRestartPane = vi.fn(() => order.push('reconnect'))
@@ -309,6 +312,7 @@ describe('RA-11 Restart on a paired (remote) pane', () => {
order.push('stop')
},
resumeRemotePane,
waitForResumedTerminal: () => Promise.resolve(),
freshIfResumeFoundNothing,
buildFreshCommand: (agent, leafId) => `fresh ${agent} ${leafId}`
})
@@ -368,7 +372,8 @@ describe('RA-11 Restart on a paired (remote) pane', () => {
}),
handleRestartPane,
stopRemotePane: () => Promise.resolve(),
resumeRemotePane: () => Promise.resolve(),
resumeRemotePane: () => Promise.resolve(RESUMED),
waitForResumedTerminal: () => Promise.resolve(),
freshIfResumeFoundNothing: (input) =>
realBackstop({ ...input, sleep: () => Promise.resolve() }),
buildFreshCommand: () => 'claude --dangerously-skip-permissions'
@@ -397,7 +402,8 @@ describe('RA-11 Restart on a paired (remote) pane', () => {
}),
handleRestartPane,
stopRemotePane: () => Promise.resolve(),
resumeRemotePane: () => Promise.resolve(),
resumeRemotePane: () => Promise.resolve(RESUMED),
waitForResumedTerminal: () => Promise.resolve(),
freshIfResumeFoundNothing: async () => true,
buildFreshCommand: () => 'claude'
})
@@ -425,7 +431,8 @@ describe('RA-11 Restart on a paired (remote) pane', () => {
}),
handleRestartPane,
stopRemotePane: () => Promise.resolve(),
resumeRemotePane: () => Promise.resolve(),
resumeRemotePane: () => Promise.resolve(RESUMED),
waitForResumedTerminal: () => Promise.resolve(),
freshIfResumeFoundNothing: () => Promise.reject(new Error('ra11e_fresh_send_failed')),
buildFreshCommand: () => 'claude'
})
@@ -438,6 +445,118 @@ describe('RA-11 Restart on a paired (remote) pane', () => {
warn.mockRestore()
})
// RA-11 resume race: the pane's own transport reattached when the stop ended its stream, woke the
// leaf with its original launch, and the resume was adopted away.
function runResume(over: {
waitForResumedTerminal?: () => Promise<void>
onReconnect?: () => void
order?: string[]
}) {
const order = over.order ?? []
const handleRestartPane = vi.fn(
(_paneId: number, _startup: unknown, options: { focus: boolean }) => {
order.push(options.focus ? 'reconnect(focus)' : 'reconnect')
over.onReconnect?.()
}
)
runReadyPaneAccountRestarts({
panes: [{ id: 7, leafId: LEAF }],
ptyIdOf: () => REMOTE,
readyRestartPtyIds: selectReadyAccountRestartPtyIds(useAppStore.getState(), TAB),
tabId: TAB,
consumePendingPaneAccountRestart: useAppStore.getState().consumePendingPaneAccountRestart,
buildAccountRestartStartup: () => ({
command: 'claude',
resumeProviderSession: { key: 'session_id' as const, id: 'sess-tab-1' }
}),
handleRestartPane,
releasePaneForRestart: () => {
order.push('release')
},
stopRemotePane: async () => {
order.push('stop')
},
resumeRemotePane: async () => {
order.push('ensure')
return { handle: 'term_resumed', disposition: 'created' as const }
},
waitForResumedTerminal: async () => {
order.push('ready?')
await (over.waitForResumedTerminal?.() ?? Promise.resolve())
},
freshIfResumeFoundNothing: async () => false,
buildFreshCommand: () => 'claude'
})
return { handleRestartPane, order }
}
function bindPane(handle: string): void {
const layouts = useAppStore.getState().terminalLayoutsByTabId
useAppStore.setState({
terminalLayoutsByTabId: {
...layouts,
[TAB]: { ...layouts[TAB]!, ptyIdsByLeafId: { [LEAF]: `remote:env-1@@${handle}` } }
}
})
}
it('lets go of the pane before the stop, then stop → resume → ready → one reconnect', async () => {
clickRemoteRestart()
const { handleRestartPane, order } = runResume({})
await vi.waitFor(() => expect(handleRestartPane).toHaveBeenCalledTimes(1))
expect(order).toEqual(['release', 'stop', 'ensure', 'ready?', 'reconnect'])
})
it('never reconnects before the resumed terminal is live in the leaf', async () => {
clickRemoteRestart()
let ready: () => void = () => undefined
const gate = new Promise<void>((resolve) => {
ready = resolve
})
const { handleRestartPane, order } = runResume({ waitForResumedTerminal: () => gate })
await vi.waitFor(() => expect(order).toContain('ready?'))
await Promise.resolve()
expect(handleRestartPane).not.toHaveBeenCalled()
ready()
await vi.waitFor(() => expect(handleRestartPane).toHaveBeenCalledTimes(1))
})
it('a resumed terminal that never shows fails the restart and reattaches the pane (never empty)', async () => {
clickRemoteRestart()
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const { handleRestartPane } = runResume({
waitForResumedTerminal: () => Promise.reject(new Error('restart_resume_not_ready'))
})
await vi.waitFor(() => expect(cloudAgentRestartAttempt(PANE_KEY, 0)?.phase).toBe('failed'))
expect(cloudAgentRestartAttempt(PANE_KEY, 0)?.failure).toBe('hostUnresponsive')
expect(handleRestartPane).toHaveBeenCalledTimes(1)
expect(handleRestartPane.mock.calls[0]?.[2]).toEqual({ focus: false })
warn.mockRestore()
})
it('the pane binding the resumed terminal is a clean restart', async () => {
clickRemoteRestart()
const { handleRestartPane } = runResume({ onReconnect: () => bindPane('term_resumed') })
await vi.waitFor(() => expect(handleRestartPane).toHaveBeenCalledTimes(1))
await Promise.resolve()
expect(cloudAgentRestartAttempt(PANE_KEY, 0)?.phase).not.toBe('failed')
})
it('the pane binding any other terminal fails visibly (a new conversation is not a resume)', async () => {
clickRemoteRestart()
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
const { handleRestartPane } = runResume({
onReconnect: () => {
settleCloudAgentRestart(PANE_KEY)
bindPane('term_original_launch')
}
})
await vi.waitFor(() => expect(handleRestartPane).toHaveBeenCalledTimes(1))
await vi.waitFor(() => expect(cloudAgentRestartAttempt(PANE_KEY, 0)?.phase).toBe('failed'))
expect(cloudAgentRestartAttempt(PANE_KEY, 0)?.ptyId).toBe('remote:env-1@@term_original_launch')
warn.mockRestore()
})
it('a failed resume on the host shows the failure and never reconnects', async () => {
clickRemoteRestart()
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
@@ -8,7 +8,7 @@ import {
import {
beginCloudAgentRestart,
failCloudAgentRestart,
failCloudAgentRestartFresh,
failCloudAgentRestartAfterLaunch,
noteCloudAgentRestartFresh,
isForcedCloudAgentRestart,
noteCloudAgentRestartLaunched
@@ -18,8 +18,13 @@ import type { usePaneProcessRestart } from '../process-exit/use-pane-process-res
import {
freshIfResumeFoundNothing,
resumeRemotePaneAgent,
waitForResumedTerminalReady,
stopRemotePaneTerminalForRestart
} from '@/lib/orca-cloud/cloud-pane-restart-stop'
import {
expectPaneBindsHandle,
remotePtyIdWithHandle
} from '@/lib/orca-cloud/cloud-restart-bind-check'
import type { AgentProviderSessionMetadata } from '../../../../../shared/agent-session/lifecycle/agent-session-resume'
import { makePaneKey } from '../../../../../shared/workspace-layout/stable-pane-id'
import {
@@ -39,6 +44,7 @@ export function useReadyPaneAccountRestarts({
tabId,
buildAccountRestartStartup,
handleRestartPane,
releasePaneForRestart,
panePtyLayoutBindings
}: Pick<
TerminalPaneCloseController,
@@ -48,6 +54,7 @@ export function useReadyPaneAccountRestarts({
typeof usePaneAccountRestartStartup
>['buildAccountRestartStartup']
handleRestartPane: ReturnType<typeof usePaneProcessRestart>['handleRestartPane']
releasePaneForRestart: ReturnType<typeof usePaneProcessRestart>['releasePaneForRestart']
panePtyLayoutBindings: TerminalPaneCloseController['savedLayout']['ptyIdsByLeafId']
}) {
// Why primitive ids: unrelated status updates stay shallow-equal and skip this pane.
@@ -66,12 +73,14 @@ export function useReadyPaneAccountRestarts({
tabId,
consumePendingPaneAccountRestart,
buildAccountRestartStartup,
handleRestartPane
handleRestartPane,
releasePaneForRestart
})
}, [
buildAccountRestartStartup,
consumePendingPaneAccountRestart,
handleRestartPane,
releasePaneForRestart,
panePtyLayoutBindings,
readyRestartPtyIds,
managerRef,
@@ -116,10 +125,18 @@ export function runReadyPaneAccountRestarts<
consumePendingPaneAccountRestart: (ptyId: string) => AccountRestartAgent | null
buildAccountRestartStartup: (agent: AccountRestartAgent, leafId: string) => Startup | null
handleRestartPane: (paneId: number, startup: Startup, options: { focus: boolean }) => void
/**
* Destroys the pane's transport without reconnecting. A paired pane's transport otherwise reacts to
* its own host terminal stopping by reattaching, which wakes the leaf with its original launch
* before the resume lands (RA-11 resume race).
*/
releasePaneForRestart?: (paneId: number) => void
/** Stops the one host terminal behind a paired pane before its restart reconnects. */
stopRemotePane?: (ptyId: string, worktreeId: string) => Promise<void>
/** Relaunches the paired pane's agent on its own conversation, in the same pane. */
resumeRemotePane?: typeof resumeRemotePaneAgent
/** RA-11 resume race: wait until the resumed terminal is live in the pane's own leaf. */
waitForResumedTerminal?: typeof waitForResumedTerminalReady
/** RA-11e: start fresh in the same pane when a resume found no conversation. */
freshIfResumeFoundNothing?: typeof freshIfResumeFoundNothing
/** The fresh-launch command for this tab's pane (no session to resume). */
@@ -128,6 +145,7 @@ export function runReadyPaneAccountRestarts<
const args = {
stopRemotePane: stopRemotePaneTerminalForRestart,
resumeRemotePane: resumeRemotePaneAgent,
waitForResumedTerminal: waitForResumedTerminalReady,
freshIfResumeFoundNothing,
buildFreshCommand: (agent: AccountRestartAgent, leafId: string) =>
buildFreshAccountRestartCommand(agent, {
@@ -187,7 +205,9 @@ export function runReadyPaneAccountRestarts<
noteCloudAgentRestartLaunched(ptyId)
}
// Why: one Claude switch restarts many panes at once; none was the one the user clicked.
args.handleRestartPane(pane.id, launchedStartup, { focus: agent === 'codex' })
args.handleRestartPane(pane.id, launchedStartup, {
focus: agent === 'codex'
})
}
if (!worktreeId) {
launch()
@@ -196,6 +216,9 @@ export function runReadyPaneAccountRestarts<
// Stop that one host terminal first (its exit is ours, not the pane's), then reconnect.
const store = useAppStore.getState()
store.suppressPtyExit(ptyId)
// Order is stop → ensure → reconnect: let go of the pane's transport before the stop, or its own
// recovery reattaches the moment the stop ends its stream and wakes the original launch.
args.releasePaneForRestart?.(pane.id)
void args
.stopRemotePane(ptyId, worktreeId)
.then(() =>
@@ -210,9 +233,44 @@ export function runReadyPaneAccountRestarts<
})
: undefined
)
.then(async (resumed) => {
// RA-11 resume race: reconnect only once the resumed terminal is live in this very leaf;
// an earlier reconnect would wake the leaf with its original launch and drop the resume.
if (resumed) {
await args.waitForResumedTerminal({
ptyId,
worktreeId,
tabId: args.tabId,
leafId: pane.leafId,
handle: resumed.handle
})
}
return resumed
})
.then(
() => {
(resumed) => {
launch()
if (resumed && tracked) {
// The pane must end on the resumed terminal; anything else is a new conversation.
expectPaneBindsHandle({
tabId: args.tabId,
leafId: pane.leafId,
replacedPtyId: ptyId,
handle: resumed.handle,
onMismatch: (bound) => {
console.warn(
'[orca-cloud] RA-11 restart bound',
bound,
'not the resumed',
resumed.handle
)
failCloudAgentRestartAfterLaunch(makePaneKey(args.tabId, pane.leafId), {
agent,
ptyId: remotePtyIdWithHandle(ptyId, bound)
})
}
})
}
if (providerSession) {
// RA-11e: a session with no conversation can't resume; start fresh in the same pane, once.
void Promise.resolve()
@@ -239,7 +297,7 @@ export function runReadyPaneAccountRestarts<
console.warn('[orca-cloud] fresh fallback failed:', err)
if (tracked) {
// The pane is a bare shell now: say so with Try again, never leave it quiet.
failCloudAgentRestartFresh(makePaneKey(args.tabId, pane.leafId), {
failCloudAgentRestartAfterLaunch(makePaneKey(args.tabId, pane.leafId), {
agent,
ptyId:
useAppStore.getState().terminalLayoutsByTabId[args.tabId]?.ptyIdsByLeafId?.[
@@ -255,6 +313,11 @@ export function runReadyPaneAccountRestarts<
useAppStore.getState().consumeSuppressedPtyExit(ptyId)
// The host did not stop or relaunch it: the strip names that reason.
failCloudAgentRestart(ptyId, 'hostUnresponsive')
if (args.releasePaneForRestart) {
// Its transport was let go before the stop: reattach so the pane is never left empty
// (the still-live terminal, or the leaf's own launch if the stop went through).
args.handleRestartPane(pane.id, launchedStartup, { focus: false })
}
if (!forced && agent === 'codex') {
// The switch was answered on the old account: ask again rather than leave it silently stale.
useAppStore.getState().reopenCodexRestartPrompt(ptyId)
@@ -49,7 +49,7 @@ export function useTerminalPaneProcessExitActions(controller: TerminalPaneCloseC
const { buildAccountRestartStartup } = usePaneAccountRestartStartup({ tabId, worktreeId })
const { handleRestartPane } = usePaneProcessRestart({
const { handleRestartPane, releasePaneForRestart } = usePaneProcessRestart({
clearCodexRestartNotice,
clearExitedPanePtyLayoutBinding,
clearRuntimePaneTitle,
@@ -105,6 +105,7 @@ export function useTerminalPaneProcessExitActions(controller: TerminalPaneCloseC
tabId,
buildAccountRestartStartup,
handleRestartPane,
releasePaneForRestart,
panePtyLayoutBindings
})
@@ -77,13 +77,13 @@ export function usePaneProcessRestart({
| 'updateTabTitle'
| 'worktreeId'
>) {
const handleRestartPane = useCallback(
(paneId: number, startup: PtyConnectionDeps['startup'], options: { focus: boolean }) => {
const manager = managerRef.current
const pane = manager?.getPanes().find((candidate) => candidate.id === paneId)
if (!manager || !pane) {
return
}
/**
* Let go of the pane's current PTY without connecting a new one: its transport is destroyed, so its
* own recovery can no longer reattach (or wake the host leaf) on its own. Returns the PTY id local
* main should replace, if any.
*/
const releasePaneForRestart = useCallback(
(paneId: number): string | null => {
const transport = paneTransportsRef.current.get(paneId)
const panePtyBinding = panePtyBindingsRef.current.get(paneId)
const existingPtyId = transport?.getPtyId()
@@ -94,9 +94,32 @@ export function usePaneProcessRestart({
}
panePtyBinding?.dispose()
panePtyBindingsRef.current.delete(paneId)
syncPanePtyLayoutBinding(paneId, null)
if (transport || panePtyBinding) {
syncPanePtyLayoutBinding(paneId, null)
}
const replacesPtyId = releasePaneTransportForRestart(transport)
paneTransportsRef.current.delete(paneId)
return replacesPtyId
},
[
clearCodexRestartNotice,
clearTabPtyId,
panePtyBindingsRef,
paneTransportsRef,
suppressPtyExit,
syncPanePtyLayoutBinding,
tabId
]
)
const handleRestartPane = useCallback(
(paneId: number, startup: PtyConnectionDeps['startup'], options: { focus: boolean }) => {
const manager = managerRef.current
const pane = manager?.getPanes().find((candidate) => candidate.id === paneId)
if (!manager || !pane) {
return
}
const replacesPtyId = releasePaneForRestart(paneId)
setCacheTimerStartedAt(makePaneKey(tabId, pane.leafId), null)
setTerminalError(null)
setTerminalErrorsByPaneId((current) => clearPaneTerminalError(current, paneId))
@@ -140,7 +163,7 @@ export function usePaneProcessRestart({
}
},
[
clearCodexRestartNotice,
releasePaneForRestart,
clearExitedPanePtyLayoutBinding,
clearRuntimePaneTitle,
clearTabPtyId,
@@ -166,7 +189,6 @@ export function usePaneProcessRestart({
setRuntimePaneTitle,
setTerminalError,
showRestoredSessionBanner,
suppressPtyExit,
syncPanePtyLayoutBinding,
tabId,
updateTabPtyId,
@@ -175,5 +197,5 @@ export function usePaneProcessRestart({
setTerminalErrorsByPaneId
]
)
return { handleRestartPane }
return { handleRestartPane, releasePaneForRestart }
}
@@ -180,11 +180,12 @@ export function cloudAgentRestartStartedFresh(paneKey: string, _revision: number
}
/**
* RA-11e: the resume found no conversation and the fresh start could not be typed, so the pane is a
* bare shell. The attempt already settled on the new PTY; show "Couldn't restart" again so the
* person can Try again, never a quiet shell.
* The restart relaunched but did not end where it must: the fresh start after an empty resume could
* not be typed (RA-11e, a bare shell), or the pane bound a terminal other than the resumed one (RA-11
* resume race, a new conversation). The attempt may already have settled on the new PTY; show
* "Couldn't restart" again so the person can Try again, never a quiet wrong result.
*/
export function failCloudAgentRestartFresh(
export function failCloudAgentRestartAfterLaunch(
paneKey: string,
args: { agent: HostAgentSignInAgent; ptyId: string }
): void {
@@ -8,6 +8,10 @@ let readTail: string[] = []
let sendAnswers: ('ok' | 'timeout' | 'refused')[] = []
/** Tail the host shows after the nth send (the fresh agent came up), if set. */
let tailAfterSend: string[] | null = null
/** What each terminal.ensureAgentSession answers, in order. */
let ensureAnswers: { handle: string; disposition: 'created' | 'adopted' }[] = []
/** What terminal.list answers, per call (the last one repeats). */
let listAnswers: { handle: string; tabId: string; leafId: string; connected?: boolean }[][] = []
vi.mock('@/runtime/rpc/calls/runtime-rpc-call', () => ({
callRuntimeRpc: async (target: { environmentId: string }, method: string, params: unknown) => {
@@ -15,6 +19,14 @@ vi.mock('@/runtime/rpc/calls/runtime-rpc-call', () => ({
if (method === 'terminal.read') {
return { terminal: { tail: readTail } }
}
if (method === 'terminal.ensureAgentSession') {
const answer = ensureAnswers.shift() ?? { handle: 'term_resumed', disposition: 'created' }
return { terminal: { handle: answer.handle }, disposition: answer.disposition }
}
if (method === 'terminal.list') {
const answer = listAnswers.length > 1 ? listAnswers.shift() : listAnswers[0]
return { terminals: answer ?? [] }
}
if (method === 'terminal.send') {
if (tailAfterSend) {
readTail = tailAfterSend
@@ -34,8 +46,12 @@ vi.mock(
() => ({ agentProviderSessionForHostEnsure: async (_env: string, session: unknown) => session })
)
const { freshIfResumeFoundNothing, resumeRemotePaneAgent, stopRemotePaneTerminalForRestart } =
await import('./cloud-pane-restart-stop')
const {
freshIfResumeFoundNothing,
resumeRemotePaneAgent,
stopRemotePaneTerminalForRestart,
waitForResumedTerminalReady
} = await import('./cloud-pane-restart-stop')
describe('stopRemotePaneTerminalForRestart (RA-11 paired pane)', () => {
beforeEach(() => {
@@ -72,9 +88,60 @@ describe('stopRemotePaneTerminalForRestart (RA-11 paired pane)', () => {
})
})
const RESUME_ARGS = {
ptyId: 'remote:env-1@@term_a',
worktreeId: 'wt-1',
tabId: 'web-terminal-host-tab%3A%3Aleaf',
leafId: 'leaf-1',
agent: 'claude' as const,
providerSession: { key: 'session_id' as const, id: 'sess-1' }
}
describe('resumeRemotePaneAgent (RA-11 paired pane)', () => {
beforeEach(() => {
calls.length = 0
ensureAnswers = []
showPtyId = 'host-pty-7'
stopResult = { stoppedPtyIds: ['host-pty-7'], postStopVerified: true }
})
it('returns the terminal the host ensured, so the pane can bind exactly it', async () => {
ensureAnswers = [{ handle: 'term_resumed', disposition: 'created' }]
await expect(resumeRemotePaneAgent(RESUME_ARGS)).resolves.toEqual({
handle: 'term_resumed',
disposition: 'created'
})
})
it('an adopted live terminal (the resume never ran) is stopped and the resume ensured once more', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
ensureAnswers = [
{ handle: 'term_original', disposition: 'adopted' },
{ handle: 'term_resumed', disposition: 'created' }
]
await expect(resumeRemotePaneAgent(RESUME_ARGS)).resolves.toEqual({
handle: 'term_resumed',
disposition: 'created'
})
expect(calls.map((c) => c.method)).toEqual([
'terminal.ensureAgentSession',
'terminal.show',
'terminal.stopExact',
'terminal.ensureAgentSession'
])
expect(calls[1]).toMatchObject({ params: { terminal: 'term_original' } })
warn.mockRestore()
})
it('adopted twice fails the restart (never a silent new conversation)', async () => {
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
ensureAnswers = [
{ handle: 'term_original', disposition: 'adopted' },
{ handle: 'term_original2', disposition: 'adopted' }
]
await expect(resumeRemotePaneAgent(RESUME_ARGS)).rejects.toThrow('restart_resume_adopted')
expect(calls.filter((c) => c.method === 'terminal.ensureAgentSession')).toHaveLength(2)
warn.mockRestore()
})
it('ensures the agent on its own session, placed in the same host tab and leaf', async () => {
@@ -100,6 +167,43 @@ describe('resumeRemotePaneAgent (RA-11 paired pane)', () => {
})
})
describe('waitForResumedTerminalReady (RA-11 resume race)', () => {
const sleep = () => Promise.resolve()
let clock = 0
const now = () => (clock += 300)
const args = {
ptyId: 'remote:env-1@@term_a',
worktreeId: 'wt-1',
tabId: 'web-terminal-host-tab%3A%3Aleaf',
leafId: 'leaf-1',
handle: 'term_resumed',
sleep,
now
}
beforeEach(() => {
calls.length = 0
clock = 0
listAnswers = []
})
it('waits until the ensured terminal itself is live in the pane’s own tab and leaf', async () => {
listAnswers = [
[],
// Some other terminal ready in the leaf is not enough.
[{ handle: 'term_original', tabId: 'host-tab::leaf', leafId: 'leaf-1' }],
[{ handle: 'term_resumed', tabId: 'host-tab::leaf', leafId: 'leaf-1', connected: true }]
]
await expect(waitForResumedTerminalReady(args)).resolves.toBeUndefined()
expect(calls.filter((c) => c.method === 'terminal.list')).toHaveLength(3)
expect(calls[0]).toMatchObject({ params: { handles: ['term_resumed'] } })
})
it('throws when the ensured terminal never shows in that leaf', async () => {
listAnswers = [[{ handle: 'term_resumed', tabId: 'host-tab::leaf', leafId: 'other-leaf' }]]
await expect(waitForResumedTerminalReady(args)).rejects.toThrow('restart_resume_not_ready')
})
})
describe('freshIfResumeFoundNothing (RA-11e)', () => {
const sleep = () => Promise.resolve()
let clock = 0
@@ -163,7 +267,11 @@ describe('freshIfResumeFoundNothing (RA-11e)', () => {
it('never types twice: a timed-out send whose Claude came up anyway is not sent again', async () => {
readTail = ['No conversation found with session ID: s1', '$ ']
sendAnswers = ['timeout']
tailAfterSend = ['No conversation found with session ID: s1', '$ claude', 'Claude Code v2.1.319']
tailAfterSend = [
'No conversation found with session ID: s1',
'$ claude',
'Claude Code v2.1.319'
]
const relaunched = await freshIfResumeFoundNothing({
currentPtyId: () => 'remote:env-1@@term_new',
replacedPtyId: 'remote:env-1@@term_old',
@@ -3,7 +3,10 @@
// relaunches. Put exactly that one host terminal to sleep first (the reversible stop the pane-sleep
// path uses: the leaf stays, as a sleeping surface); the reconnect then wakes it with --resume.
// A plain stop is a close: the host drops the tab and there is nothing left to relaunch into.
import { parseRemoteRuntimePtyId } from '../../../../shared/remote-runtime/remote-runtime-pty-id'
import {
parseRemoteRuntimePtyId,
toRemoteRuntimePtyId
} from '../../../../shared/remote-runtime/remote-runtime-pty-id'
import type { RuntimeTerminalShow } from '../../../../shared/runtime-terminal-contracts'
import { callRuntimeRpc } from '@/runtime/rpc/calls/runtime-rpc-call'
import { toRuntimeWorktreeSelector } from '@/runtime/rpc/targets/runtime-worktree-selector'
@@ -27,7 +30,10 @@ export async function stopRemotePaneTerminalForRestart(
if (!parsed?.environmentId) {
throw new Error('restart_remote_pane_unowned')
}
const target = { kind: 'environment' as const, environmentId: parsed.environmentId }
const target = {
kind: 'environment' as const,
environmentId: parsed.environmentId
}
// The pane names its terminal by handle; the host stops by its own PTY id.
const { terminal } = await callRuntimeRpc<{ terminal?: RuntimeTerminalShow }>(
target,
@@ -57,10 +63,20 @@ export async function stopRemotePaneTerminalForRestart(
}
}
/** The terminal the host ensured for the resume, and whether it spawned it or adopted a live one. */
export type RemotePaneResume = {
handle: string
disposition: 'created' | 'adopted'
}
/**
* Relaunch the pane's agent on its own conversation, in the same pane. Without this the host wakes
* the sleeping pane with its original launch (a new conversation). Only for a resumable session;
* a pane with none takes the host's fresh launch, which is what "Starts a new conversation" says.
*
* Returns the ensured terminal's handle so the pane binds to exactly it. `adopted` means a live
* owner already held the leaf (something woke it with its original launch first) and the resume
* never ran: stop that terminal and ensure once more; still adopted throws (Couldn't restart).
*/
export async function resumeRemotePaneAgent(args: {
ptyId: string
@@ -69,25 +85,126 @@ export async function resumeRemotePaneAgent(args: {
leafId: string
agent: AccountRestartAgent
providerSession: AgentProviderSessionMetadata
}): Promise<RemotePaneResume> {
const environmentId = parseRemoteRuntimePtyId(args.ptyId)?.environmentId
if (!environmentId) {
throw new Error('restart_remote_pane_unowned')
}
const ensure = async (): Promise<RemotePaneResume> => {
const result = await callRuntimeRpc<{
terminal?: { handle?: string }
disposition?: 'created' | 'adopted'
}>(
{ kind: 'environment', environmentId },
'terminal.ensureAgentSession',
{
kind: 'explicit',
worktree: toRuntimeWorktreeSelector(args.worktreeId),
agent: args.agent,
providerSession: await agentProviderSessionForHostEnsure(
environmentId,
args.providerSession
),
// The same pane: the host tab and leaf the stopped terminal lived in.
placement: {
tabId: toHostSessionTabId(args.tabId),
leafId: args.leafId
},
presentation: 'background'
},
{ timeoutMs: 15_000 }
)
const handle = result?.terminal?.handle
if (!handle) {
throw new Error('restart_resume_no_terminal')
}
return {
handle,
disposition: result?.disposition === 'adopted' ? 'adopted' : 'created'
}
}
const first = await ensure()
if (first.disposition !== 'adopted') {
return first
}
console.warn(
'[orca-cloud] RA-11 resume adopted a live terminal; stopping it and resuming again',
first.handle
)
await stopRemotePaneTerminalForRestart(
toRemoteRuntimePtyId(first.handle, environmentId),
args.worktreeId
)
const second = await ensure()
if (second.disposition === 'adopted') {
throw new Error('restart_resume_adopted')
}
return second
}
// The ensured terminal shows up in the host's inventory within a second or two of the ensure.
const RESUMED_READY_WAIT_MS = 15_000
const RESUMED_READY_POLL_MS = 300
/**
* RA-11 resume race: wait until the host lists exactly the ensured terminal `handle`, live, in the
* pane's own host tab and leaf. Only then may the pane reconnect: its activation reuses a ready leaf,
* and a leaf that is not ready yet is woken with its ORIGINAL launch (the resume is then lost).
* A turn-less `--resume` exits to a shell but its PTY stays live, so it still lists. Throws when the
* terminal never shows, so the restart fails visibly instead of reconnecting into the race.
*/
export async function waitForResumedTerminalReady(args: {
ptyId: string
worktreeId: string
tabId: string
leafId: string
handle: string
sleep?: (ms: number) => Promise<void>
now?: () => number
}): Promise<void> {
const environmentId = parseRemoteRuntimePtyId(args.ptyId)?.environmentId
if (!environmentId) {
throw new Error('restart_remote_pane_unowned')
}
await callRuntimeRpc(
{ kind: 'environment', environmentId },
'terminal.ensureAgentSession',
{
kind: 'explicit',
worktree: toRuntimeWorktreeSelector(args.worktreeId),
agent: args.agent,
providerSession: await agentProviderSessionForHostEnsure(environmentId, args.providerSession),
// The same pane: the host tab and leaf the stopped terminal lived in.
placement: { tabId: toHostSessionTabId(args.tabId), leafId: args.leafId },
presentation: 'background'
},
{ timeoutMs: 15_000 }
)
const sleep =
args.sleep ?? ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)))
const now = args.now ?? Date.now
const hostTabId = toHostSessionTabId(args.tabId)
const deadline = now() + RESUMED_READY_WAIT_MS
while (now() < deadline) {
const listed = await callRuntimeRpc<{
terminals?: {
handle?: string
tabId?: string
leafId?: string
connected?: boolean
verdict?: string
}[]
}>(
{ kind: 'environment', environmentId },
'terminal.list',
{
worktree: toRuntimeWorktreeSelector(args.worktreeId),
handles: [args.handle],
includePreview: false,
includeVisualLayouts: false
},
{ timeoutMs: 5_000 }
).catch(() => null)
const ready = (listed?.terminals ?? []).some(
(terminal) =>
terminal.handle === args.handle &&
terminal.tabId === hostTabId &&
terminal.leafId === args.leafId &&
terminal.connected !== false &&
terminal.verdict !== 'dead'
)
if (ready) {
return
}
await sleep(RESUMED_READY_POLL_MS)
}
throw new Error('restart_resume_not_ready')
}
/** What Claude prints when `--resume <id>` names a session that never had a conversation. */
@@ -164,7 +281,9 @@ async function sendFreshOnce(
try {
// The Enter rides in the text, not `enter: true`: the host holds an Enter send for the launched
// Claude's composer (8 s), which never comes once it exited to the shell, then refuses it.
const sent = await callRuntimeRpc<{ send?: { accepted?: boolean; refusedReason?: string } }>(
const sent = await callRuntimeRpc<{
send?: { accepted?: boolean; refusedReason?: string }
}>(
target,
'terminal.send',
{ terminal: handle, text: `${freshCommand}\r` },
@@ -215,7 +334,10 @@ export async function freshIfResumeFoundNothing(args: {
lastSeen = current.handle
console.info('[orca-cloud] RA-11e watching relaunched terminal', current.handle)
}
const target = { kind: 'environment' as const, environmentId: current.environmentId }
const target = {
kind: 'environment' as const,
environmentId: current.environmentId
}
const read = await callRuntimeRpc<{ terminal?: { tail?: string[] } }>(
target,
'terminal.read',
@@ -0,0 +1,63 @@
// RA-11 resume race: after a cloud Restart reconnects, the pane must sit on the terminal the resume
// ensured. Anything else is the leaf's original launch (a new conversation), which the strip reports.
import { useAppStore } from '@/store'
import {
parseRemoteRuntimePtyId,
toRemoteRuntimePtyId
} from '../../../../shared/remote-runtime/remote-runtime-pty-id'
// How long the pane may take to bind its relaunched terminal before the check gives up (the
// attempt's own timeout reports a pane that never binds).
const BIND_CHECK_MS = 30_000
function paneHandleOf(tabId: string, leafId: string): { ptyId: string; handle: string } | null {
const ptyId =
useAppStore.getState().terminalLayoutsByTabId[tabId]?.ptyIdsByLeafId?.[leafId] ?? null
const parsed = ptyId ? parseRemoteRuntimePtyId(ptyId) : null
return ptyId && parsed ? { ptyId, handle: parsed.handle } : null
}
/** The pane id for `handle` on the same environment as `ptyId` (falls back to `ptyId`). */
export function remotePtyIdWithHandle(ptyId: string, handle: string): string {
const environmentId = parseRemoteRuntimePtyId(ptyId)?.environmentId
return environmentId ? toRemoteRuntimePtyId(handle, environmentId) : ptyId
}
/** Once the pane binds a new PTY, check it is `handle`; report anything else. */
export function expectPaneBindsHandle(args: {
tabId: string
leafId: string
replacedPtyId: string
handle: string
onMismatch: (boundHandle: string) => void
}): void {
let done = false
const check = (): boolean => {
const bound = paneHandleOf(args.tabId, args.leafId)
if (!bound || bound.ptyId === args.replacedPtyId) {
return false
}
if (bound.handle !== args.handle) {
args.onMismatch(bound.handle)
}
return true
}
if (check()) {
return
}
const finish = (): void => {
done = true
unsubscribe()
clearTimeout(timer)
}
const unsubscribe = useAppStore.subscribe(() => {
if (!done && check()) {
finish()
}
})
const timer = setTimeout(() => {
if (!done) {
finish()
}
}, BIND_CHECK_MS)
}