mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 02:47:34 +00:00
Merge pull request #429 from nwparker/cloud/ra11-bind-ensure
fix(cloud): RA-11 resume race: Restart resumes in the pane (P1)
This commit is contained in:
+122
-3
@@ -43,6 +43,8 @@ const TAB = 'tab-1'
|
||||
const LEAF = '11111111-1111-4111-8111-111111111111'
|
||||
const PTY = 'env-1@@pty-1'
|
||||
const PANE_KEY = makePaneKey(TAB, LEAF)
|
||||
/** The terminal a resume ensure placed in the pane's leaf. */
|
||||
const RESUMED = { handle: 'term_resumed', disposition: 'created' as const }
|
||||
|
||||
function seed(state: AgentStatusEntry['state']): void {
|
||||
useAppStore.setState({
|
||||
@@ -291,6 +293,7 @@ describe('RA-11 Restart on a paired (remote) pane', () => {
|
||||
const providerSession = { key: 'session_id' as const, id: 'sess-tab-1' }
|
||||
const resumeRemotePane = vi.fn(async () => {
|
||||
order.push('resume')
|
||||
return RESUMED
|
||||
})
|
||||
const freshIfResumeFoundNothing = vi.fn(async () => false)
|
||||
const handleRestartPane = vi.fn(() => order.push('reconnect'))
|
||||
@@ -309,6 +312,7 @@ describe('RA-11 Restart on a paired (remote) pane', () => {
|
||||
order.push('stop')
|
||||
},
|
||||
resumeRemotePane,
|
||||
waitForResumedTerminal: () => Promise.resolve(),
|
||||
freshIfResumeFoundNothing,
|
||||
buildFreshCommand: (agent, leafId) => `fresh ${agent} ${leafId}`
|
||||
})
|
||||
@@ -368,7 +372,8 @@ describe('RA-11 Restart on a paired (remote) pane', () => {
|
||||
}),
|
||||
handleRestartPane,
|
||||
stopRemotePane: () => Promise.resolve(),
|
||||
resumeRemotePane: () => Promise.resolve(),
|
||||
resumeRemotePane: () => Promise.resolve(RESUMED),
|
||||
waitForResumedTerminal: () => Promise.resolve(),
|
||||
freshIfResumeFoundNothing: (input) =>
|
||||
realBackstop({ ...input, sleep: () => Promise.resolve() }),
|
||||
buildFreshCommand: () => 'claude --dangerously-skip-permissions'
|
||||
@@ -397,7 +402,8 @@ describe('RA-11 Restart on a paired (remote) pane', () => {
|
||||
}),
|
||||
handleRestartPane,
|
||||
stopRemotePane: () => Promise.resolve(),
|
||||
resumeRemotePane: () => Promise.resolve(),
|
||||
resumeRemotePane: () => Promise.resolve(RESUMED),
|
||||
waitForResumedTerminal: () => Promise.resolve(),
|
||||
freshIfResumeFoundNothing: async () => true,
|
||||
buildFreshCommand: () => 'claude'
|
||||
})
|
||||
@@ -425,7 +431,8 @@ describe('RA-11 Restart on a paired (remote) pane', () => {
|
||||
}),
|
||||
handleRestartPane,
|
||||
stopRemotePane: () => Promise.resolve(),
|
||||
resumeRemotePane: () => Promise.resolve(),
|
||||
resumeRemotePane: () => Promise.resolve(RESUMED),
|
||||
waitForResumedTerminal: () => Promise.resolve(),
|
||||
freshIfResumeFoundNothing: () => Promise.reject(new Error('ra11e_fresh_send_failed')),
|
||||
buildFreshCommand: () => 'claude'
|
||||
})
|
||||
@@ -438,6 +445,118 @@ describe('RA-11 Restart on a paired (remote) pane', () => {
|
||||
warn.mockRestore()
|
||||
})
|
||||
|
||||
// RA-11 resume race: the pane's own transport reattached when the stop ended its stream, woke the
|
||||
// leaf with its original launch, and the resume was adopted away.
|
||||
function runResume(over: {
|
||||
waitForResumedTerminal?: () => Promise<void>
|
||||
onReconnect?: () => void
|
||||
order?: string[]
|
||||
}) {
|
||||
const order = over.order ?? []
|
||||
const handleRestartPane = vi.fn(
|
||||
(_paneId: number, _startup: unknown, options: { focus: boolean }) => {
|
||||
order.push(options.focus ? 'reconnect(focus)' : 'reconnect')
|
||||
over.onReconnect?.()
|
||||
}
|
||||
)
|
||||
runReadyPaneAccountRestarts({
|
||||
panes: [{ id: 7, leafId: LEAF }],
|
||||
ptyIdOf: () => REMOTE,
|
||||
readyRestartPtyIds: selectReadyAccountRestartPtyIds(useAppStore.getState(), TAB),
|
||||
tabId: TAB,
|
||||
consumePendingPaneAccountRestart: useAppStore.getState().consumePendingPaneAccountRestart,
|
||||
buildAccountRestartStartup: () => ({
|
||||
command: 'claude',
|
||||
resumeProviderSession: { key: 'session_id' as const, id: 'sess-tab-1' }
|
||||
}),
|
||||
handleRestartPane,
|
||||
releasePaneForRestart: () => {
|
||||
order.push('release')
|
||||
},
|
||||
stopRemotePane: async () => {
|
||||
order.push('stop')
|
||||
},
|
||||
resumeRemotePane: async () => {
|
||||
order.push('ensure')
|
||||
return { handle: 'term_resumed', disposition: 'created' as const }
|
||||
},
|
||||
waitForResumedTerminal: async () => {
|
||||
order.push('ready?')
|
||||
await (over.waitForResumedTerminal?.() ?? Promise.resolve())
|
||||
},
|
||||
freshIfResumeFoundNothing: async () => false,
|
||||
buildFreshCommand: () => 'claude'
|
||||
})
|
||||
return { handleRestartPane, order }
|
||||
}
|
||||
|
||||
function bindPane(handle: string): void {
|
||||
const layouts = useAppStore.getState().terminalLayoutsByTabId
|
||||
useAppStore.setState({
|
||||
terminalLayoutsByTabId: {
|
||||
...layouts,
|
||||
[TAB]: { ...layouts[TAB]!, ptyIdsByLeafId: { [LEAF]: `remote:env-1@@${handle}` } }
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
it('lets go of the pane before the stop, then stop → resume → ready → one reconnect', async () => {
|
||||
clickRemoteRestart()
|
||||
const { handleRestartPane, order } = runResume({})
|
||||
await vi.waitFor(() => expect(handleRestartPane).toHaveBeenCalledTimes(1))
|
||||
expect(order).toEqual(['release', 'stop', 'ensure', 'ready?', 'reconnect'])
|
||||
})
|
||||
|
||||
it('never reconnects before the resumed terminal is live in the leaf', async () => {
|
||||
clickRemoteRestart()
|
||||
let ready: () => void = () => undefined
|
||||
const gate = new Promise<void>((resolve) => {
|
||||
ready = resolve
|
||||
})
|
||||
const { handleRestartPane, order } = runResume({ waitForResumedTerminal: () => gate })
|
||||
await vi.waitFor(() => expect(order).toContain('ready?'))
|
||||
await Promise.resolve()
|
||||
expect(handleRestartPane).not.toHaveBeenCalled()
|
||||
ready()
|
||||
await vi.waitFor(() => expect(handleRestartPane).toHaveBeenCalledTimes(1))
|
||||
})
|
||||
|
||||
it('a resumed terminal that never shows fails the restart and reattaches the pane (never empty)', async () => {
|
||||
clickRemoteRestart()
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
const { handleRestartPane } = runResume({
|
||||
waitForResumedTerminal: () => Promise.reject(new Error('restart_resume_not_ready'))
|
||||
})
|
||||
await vi.waitFor(() => expect(cloudAgentRestartAttempt(PANE_KEY, 0)?.phase).toBe('failed'))
|
||||
expect(cloudAgentRestartAttempt(PANE_KEY, 0)?.failure).toBe('hostUnresponsive')
|
||||
expect(handleRestartPane).toHaveBeenCalledTimes(1)
|
||||
expect(handleRestartPane.mock.calls[0]?.[2]).toEqual({ focus: false })
|
||||
warn.mockRestore()
|
||||
})
|
||||
|
||||
it('the pane binding the resumed terminal is a clean restart', async () => {
|
||||
clickRemoteRestart()
|
||||
const { handleRestartPane } = runResume({ onReconnect: () => bindPane('term_resumed') })
|
||||
await vi.waitFor(() => expect(handleRestartPane).toHaveBeenCalledTimes(1))
|
||||
await Promise.resolve()
|
||||
expect(cloudAgentRestartAttempt(PANE_KEY, 0)?.phase).not.toBe('failed')
|
||||
})
|
||||
|
||||
it('the pane binding any other terminal fails visibly (a new conversation is not a resume)', async () => {
|
||||
clickRemoteRestart()
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
const { handleRestartPane } = runResume({
|
||||
onReconnect: () => {
|
||||
settleCloudAgentRestart(PANE_KEY)
|
||||
bindPane('term_original_launch')
|
||||
}
|
||||
})
|
||||
await vi.waitFor(() => expect(handleRestartPane).toHaveBeenCalledTimes(1))
|
||||
await vi.waitFor(() => expect(cloudAgentRestartAttempt(PANE_KEY, 0)?.phase).toBe('failed'))
|
||||
expect(cloudAgentRestartAttempt(PANE_KEY, 0)?.ptyId).toBe('remote:env-1@@term_original_launch')
|
||||
warn.mockRestore()
|
||||
})
|
||||
|
||||
it('a failed resume on the host shows the failure and never reconnects', async () => {
|
||||
clickRemoteRestart()
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
|
||||
+68
-5
@@ -8,7 +8,7 @@ import {
|
||||
import {
|
||||
beginCloudAgentRestart,
|
||||
failCloudAgentRestart,
|
||||
failCloudAgentRestartFresh,
|
||||
failCloudAgentRestartAfterLaunch,
|
||||
noteCloudAgentRestartFresh,
|
||||
isForcedCloudAgentRestart,
|
||||
noteCloudAgentRestartLaunched
|
||||
@@ -18,8 +18,13 @@ import type { usePaneProcessRestart } from '../process-exit/use-pane-process-res
|
||||
import {
|
||||
freshIfResumeFoundNothing,
|
||||
resumeRemotePaneAgent,
|
||||
waitForResumedTerminalReady,
|
||||
stopRemotePaneTerminalForRestart
|
||||
} from '@/lib/orca-cloud/cloud-pane-restart-stop'
|
||||
import {
|
||||
expectPaneBindsHandle,
|
||||
remotePtyIdWithHandle
|
||||
} from '@/lib/orca-cloud/cloud-restart-bind-check'
|
||||
import type { AgentProviderSessionMetadata } from '../../../../../shared/agent-session/lifecycle/agent-session-resume'
|
||||
import { makePaneKey } from '../../../../../shared/workspace-layout/stable-pane-id'
|
||||
import {
|
||||
@@ -39,6 +44,7 @@ export function useReadyPaneAccountRestarts({
|
||||
tabId,
|
||||
buildAccountRestartStartup,
|
||||
handleRestartPane,
|
||||
releasePaneForRestart,
|
||||
panePtyLayoutBindings
|
||||
}: Pick<
|
||||
TerminalPaneCloseController,
|
||||
@@ -48,6 +54,7 @@ export function useReadyPaneAccountRestarts({
|
||||
typeof usePaneAccountRestartStartup
|
||||
>['buildAccountRestartStartup']
|
||||
handleRestartPane: ReturnType<typeof usePaneProcessRestart>['handleRestartPane']
|
||||
releasePaneForRestart: ReturnType<typeof usePaneProcessRestart>['releasePaneForRestart']
|
||||
panePtyLayoutBindings: TerminalPaneCloseController['savedLayout']['ptyIdsByLeafId']
|
||||
}) {
|
||||
// Why primitive ids: unrelated status updates stay shallow-equal and skip this pane.
|
||||
@@ -66,12 +73,14 @@ export function useReadyPaneAccountRestarts({
|
||||
tabId,
|
||||
consumePendingPaneAccountRestart,
|
||||
buildAccountRestartStartup,
|
||||
handleRestartPane
|
||||
handleRestartPane,
|
||||
releasePaneForRestart
|
||||
})
|
||||
}, [
|
||||
buildAccountRestartStartup,
|
||||
consumePendingPaneAccountRestart,
|
||||
handleRestartPane,
|
||||
releasePaneForRestart,
|
||||
panePtyLayoutBindings,
|
||||
readyRestartPtyIds,
|
||||
managerRef,
|
||||
@@ -116,10 +125,18 @@ export function runReadyPaneAccountRestarts<
|
||||
consumePendingPaneAccountRestart: (ptyId: string) => AccountRestartAgent | null
|
||||
buildAccountRestartStartup: (agent: AccountRestartAgent, leafId: string) => Startup | null
|
||||
handleRestartPane: (paneId: number, startup: Startup, options: { focus: boolean }) => void
|
||||
/**
|
||||
* Destroys the pane's transport without reconnecting. A paired pane's transport otherwise reacts to
|
||||
* its own host terminal stopping by reattaching, which wakes the leaf with its original launch
|
||||
* before the resume lands (RA-11 resume race).
|
||||
*/
|
||||
releasePaneForRestart?: (paneId: number) => void
|
||||
/** Stops the one host terminal behind a paired pane before its restart reconnects. */
|
||||
stopRemotePane?: (ptyId: string, worktreeId: string) => Promise<void>
|
||||
/** Relaunches the paired pane's agent on its own conversation, in the same pane. */
|
||||
resumeRemotePane?: typeof resumeRemotePaneAgent
|
||||
/** RA-11 resume race: wait until the resumed terminal is live in the pane's own leaf. */
|
||||
waitForResumedTerminal?: typeof waitForResumedTerminalReady
|
||||
/** RA-11e: start fresh in the same pane when a resume found no conversation. */
|
||||
freshIfResumeFoundNothing?: typeof freshIfResumeFoundNothing
|
||||
/** The fresh-launch command for this tab's pane (no session to resume). */
|
||||
@@ -128,6 +145,7 @@ export function runReadyPaneAccountRestarts<
|
||||
const args = {
|
||||
stopRemotePane: stopRemotePaneTerminalForRestart,
|
||||
resumeRemotePane: resumeRemotePaneAgent,
|
||||
waitForResumedTerminal: waitForResumedTerminalReady,
|
||||
freshIfResumeFoundNothing,
|
||||
buildFreshCommand: (agent: AccountRestartAgent, leafId: string) =>
|
||||
buildFreshAccountRestartCommand(agent, {
|
||||
@@ -187,7 +205,9 @@ export function runReadyPaneAccountRestarts<
|
||||
noteCloudAgentRestartLaunched(ptyId)
|
||||
}
|
||||
// Why: one Claude switch restarts many panes at once; none was the one the user clicked.
|
||||
args.handleRestartPane(pane.id, launchedStartup, { focus: agent === 'codex' })
|
||||
args.handleRestartPane(pane.id, launchedStartup, {
|
||||
focus: agent === 'codex'
|
||||
})
|
||||
}
|
||||
if (!worktreeId) {
|
||||
launch()
|
||||
@@ -196,6 +216,9 @@ export function runReadyPaneAccountRestarts<
|
||||
// Stop that one host terminal first (its exit is ours, not the pane's), then reconnect.
|
||||
const store = useAppStore.getState()
|
||||
store.suppressPtyExit(ptyId)
|
||||
// Order is stop → ensure → reconnect: let go of the pane's transport before the stop, or its own
|
||||
// recovery reattaches the moment the stop ends its stream and wakes the original launch.
|
||||
args.releasePaneForRestart?.(pane.id)
|
||||
void args
|
||||
.stopRemotePane(ptyId, worktreeId)
|
||||
.then(() =>
|
||||
@@ -210,9 +233,44 @@ export function runReadyPaneAccountRestarts<
|
||||
})
|
||||
: undefined
|
||||
)
|
||||
.then(async (resumed) => {
|
||||
// RA-11 resume race: reconnect only once the resumed terminal is live in this very leaf;
|
||||
// an earlier reconnect would wake the leaf with its original launch and drop the resume.
|
||||
if (resumed) {
|
||||
await args.waitForResumedTerminal({
|
||||
ptyId,
|
||||
worktreeId,
|
||||
tabId: args.tabId,
|
||||
leafId: pane.leafId,
|
||||
handle: resumed.handle
|
||||
})
|
||||
}
|
||||
return resumed
|
||||
})
|
||||
.then(
|
||||
() => {
|
||||
(resumed) => {
|
||||
launch()
|
||||
if (resumed && tracked) {
|
||||
// The pane must end on the resumed terminal; anything else is a new conversation.
|
||||
expectPaneBindsHandle({
|
||||
tabId: args.tabId,
|
||||
leafId: pane.leafId,
|
||||
replacedPtyId: ptyId,
|
||||
handle: resumed.handle,
|
||||
onMismatch: (bound) => {
|
||||
console.warn(
|
||||
'[orca-cloud] RA-11 restart bound',
|
||||
bound,
|
||||
'not the resumed',
|
||||
resumed.handle
|
||||
)
|
||||
failCloudAgentRestartAfterLaunch(makePaneKey(args.tabId, pane.leafId), {
|
||||
agent,
|
||||
ptyId: remotePtyIdWithHandle(ptyId, bound)
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
if (providerSession) {
|
||||
// RA-11e: a session with no conversation can't resume; start fresh in the same pane, once.
|
||||
void Promise.resolve()
|
||||
@@ -239,7 +297,7 @@ export function runReadyPaneAccountRestarts<
|
||||
console.warn('[orca-cloud] fresh fallback failed:', err)
|
||||
if (tracked) {
|
||||
// The pane is a bare shell now: say so with Try again, never leave it quiet.
|
||||
failCloudAgentRestartFresh(makePaneKey(args.tabId, pane.leafId), {
|
||||
failCloudAgentRestartAfterLaunch(makePaneKey(args.tabId, pane.leafId), {
|
||||
agent,
|
||||
ptyId:
|
||||
useAppStore.getState().terminalLayoutsByTabId[args.tabId]?.ptyIdsByLeafId?.[
|
||||
@@ -255,6 +313,11 @@ export function runReadyPaneAccountRestarts<
|
||||
useAppStore.getState().consumeSuppressedPtyExit(ptyId)
|
||||
// The host did not stop or relaunch it: the strip names that reason.
|
||||
failCloudAgentRestart(ptyId, 'hostUnresponsive')
|
||||
if (args.releasePaneForRestart) {
|
||||
// Its transport was let go before the stop: reattach so the pane is never left empty
|
||||
// (the still-live terminal, or the leaf's own launch if the stop went through).
|
||||
args.handleRestartPane(pane.id, launchedStartup, { focus: false })
|
||||
}
|
||||
if (!forced && agent === 'codex') {
|
||||
// The switch was answered on the old account: ask again rather than leave it silently stale.
|
||||
useAppStore.getState().reopenCodexRestartPrompt(ptyId)
|
||||
|
||||
+2
-1
@@ -49,7 +49,7 @@ export function useTerminalPaneProcessExitActions(controller: TerminalPaneCloseC
|
||||
|
||||
const { buildAccountRestartStartup } = usePaneAccountRestartStartup({ tabId, worktreeId })
|
||||
|
||||
const { handleRestartPane } = usePaneProcessRestart({
|
||||
const { handleRestartPane, releasePaneForRestart } = usePaneProcessRestart({
|
||||
clearCodexRestartNotice,
|
||||
clearExitedPanePtyLayoutBinding,
|
||||
clearRuntimePaneTitle,
|
||||
@@ -105,6 +105,7 @@ export function useTerminalPaneProcessExitActions(controller: TerminalPaneCloseC
|
||||
tabId,
|
||||
buildAccountRestartStartup,
|
||||
handleRestartPane,
|
||||
releasePaneForRestart,
|
||||
panePtyLayoutBindings
|
||||
})
|
||||
|
||||
|
||||
+33
-11
@@ -77,13 +77,13 @@ export function usePaneProcessRestart({
|
||||
| 'updateTabTitle'
|
||||
| 'worktreeId'
|
||||
>) {
|
||||
const handleRestartPane = useCallback(
|
||||
(paneId: number, startup: PtyConnectionDeps['startup'], options: { focus: boolean }) => {
|
||||
const manager = managerRef.current
|
||||
const pane = manager?.getPanes().find((candidate) => candidate.id === paneId)
|
||||
if (!manager || !pane) {
|
||||
return
|
||||
}
|
||||
/**
|
||||
* Let go of the pane's current PTY without connecting a new one: its transport is destroyed, so its
|
||||
* own recovery can no longer reattach (or wake the host leaf) on its own. Returns the PTY id local
|
||||
* main should replace, if any.
|
||||
*/
|
||||
const releasePaneForRestart = useCallback(
|
||||
(paneId: number): string | null => {
|
||||
const transport = paneTransportsRef.current.get(paneId)
|
||||
const panePtyBinding = panePtyBindingsRef.current.get(paneId)
|
||||
const existingPtyId = transport?.getPtyId()
|
||||
@@ -94,9 +94,32 @@ export function usePaneProcessRestart({
|
||||
}
|
||||
panePtyBinding?.dispose()
|
||||
panePtyBindingsRef.current.delete(paneId)
|
||||
syncPanePtyLayoutBinding(paneId, null)
|
||||
if (transport || panePtyBinding) {
|
||||
syncPanePtyLayoutBinding(paneId, null)
|
||||
}
|
||||
const replacesPtyId = releasePaneTransportForRestart(transport)
|
||||
paneTransportsRef.current.delete(paneId)
|
||||
return replacesPtyId
|
||||
},
|
||||
[
|
||||
clearCodexRestartNotice,
|
||||
clearTabPtyId,
|
||||
panePtyBindingsRef,
|
||||
paneTransportsRef,
|
||||
suppressPtyExit,
|
||||
syncPanePtyLayoutBinding,
|
||||
tabId
|
||||
]
|
||||
)
|
||||
|
||||
const handleRestartPane = useCallback(
|
||||
(paneId: number, startup: PtyConnectionDeps['startup'], options: { focus: boolean }) => {
|
||||
const manager = managerRef.current
|
||||
const pane = manager?.getPanes().find((candidate) => candidate.id === paneId)
|
||||
if (!manager || !pane) {
|
||||
return
|
||||
}
|
||||
const replacesPtyId = releasePaneForRestart(paneId)
|
||||
setCacheTimerStartedAt(makePaneKey(tabId, pane.leafId), null)
|
||||
setTerminalError(null)
|
||||
setTerminalErrorsByPaneId((current) => clearPaneTerminalError(current, paneId))
|
||||
@@ -140,7 +163,7 @@ export function usePaneProcessRestart({
|
||||
}
|
||||
},
|
||||
[
|
||||
clearCodexRestartNotice,
|
||||
releasePaneForRestart,
|
||||
clearExitedPanePtyLayoutBinding,
|
||||
clearRuntimePaneTitle,
|
||||
clearTabPtyId,
|
||||
@@ -166,7 +189,6 @@ export function usePaneProcessRestart({
|
||||
setRuntimePaneTitle,
|
||||
setTerminalError,
|
||||
showRestoredSessionBanner,
|
||||
suppressPtyExit,
|
||||
syncPanePtyLayoutBinding,
|
||||
tabId,
|
||||
updateTabPtyId,
|
||||
@@ -175,5 +197,5 @@ export function usePaneProcessRestart({
|
||||
setTerminalErrorsByPaneId
|
||||
]
|
||||
)
|
||||
return { handleRestartPane }
|
||||
return { handleRestartPane, releasePaneForRestart }
|
||||
}
|
||||
|
||||
@@ -180,11 +180,12 @@ export function cloudAgentRestartStartedFresh(paneKey: string, _revision: number
|
||||
}
|
||||
|
||||
/**
|
||||
* RA-11e: the resume found no conversation and the fresh start could not be typed, so the pane is a
|
||||
* bare shell. The attempt already settled on the new PTY; show "Couldn't restart" again so the
|
||||
* person can Try again, never a quiet shell.
|
||||
* The restart relaunched but did not end where it must: the fresh start after an empty resume could
|
||||
* not be typed (RA-11e, a bare shell), or the pane bound a terminal other than the resumed one (RA-11
|
||||
* resume race, a new conversation). The attempt may already have settled on the new PTY; show
|
||||
* "Couldn't restart" again so the person can Try again, never a quiet wrong result.
|
||||
*/
|
||||
export function failCloudAgentRestartFresh(
|
||||
export function failCloudAgentRestartAfterLaunch(
|
||||
paneKey: string,
|
||||
args: { agent: HostAgentSignInAgent; ptyId: string }
|
||||
): void {
|
||||
|
||||
@@ -8,6 +8,10 @@ let readTail: string[] = []
|
||||
let sendAnswers: ('ok' | 'timeout' | 'refused')[] = []
|
||||
/** Tail the host shows after the nth send (the fresh agent came up), if set. */
|
||||
let tailAfterSend: string[] | null = null
|
||||
/** What each terminal.ensureAgentSession answers, in order. */
|
||||
let ensureAnswers: { handle: string; disposition: 'created' | 'adopted' }[] = []
|
||||
/** What terminal.list answers, per call (the last one repeats). */
|
||||
let listAnswers: { handle: string; tabId: string; leafId: string; connected?: boolean }[][] = []
|
||||
|
||||
vi.mock('@/runtime/rpc/calls/runtime-rpc-call', () => ({
|
||||
callRuntimeRpc: async (target: { environmentId: string }, method: string, params: unknown) => {
|
||||
@@ -15,6 +19,14 @@ vi.mock('@/runtime/rpc/calls/runtime-rpc-call', () => ({
|
||||
if (method === 'terminal.read') {
|
||||
return { terminal: { tail: readTail } }
|
||||
}
|
||||
if (method === 'terminal.ensureAgentSession') {
|
||||
const answer = ensureAnswers.shift() ?? { handle: 'term_resumed', disposition: 'created' }
|
||||
return { terminal: { handle: answer.handle }, disposition: answer.disposition }
|
||||
}
|
||||
if (method === 'terminal.list') {
|
||||
const answer = listAnswers.length > 1 ? listAnswers.shift() : listAnswers[0]
|
||||
return { terminals: answer ?? [] }
|
||||
}
|
||||
if (method === 'terminal.send') {
|
||||
if (tailAfterSend) {
|
||||
readTail = tailAfterSend
|
||||
@@ -34,8 +46,12 @@ vi.mock(
|
||||
() => ({ agentProviderSessionForHostEnsure: async (_env: string, session: unknown) => session })
|
||||
)
|
||||
|
||||
const { freshIfResumeFoundNothing, resumeRemotePaneAgent, stopRemotePaneTerminalForRestart } =
|
||||
await import('./cloud-pane-restart-stop')
|
||||
const {
|
||||
freshIfResumeFoundNothing,
|
||||
resumeRemotePaneAgent,
|
||||
stopRemotePaneTerminalForRestart,
|
||||
waitForResumedTerminalReady
|
||||
} = await import('./cloud-pane-restart-stop')
|
||||
|
||||
describe('stopRemotePaneTerminalForRestart (RA-11 paired pane)', () => {
|
||||
beforeEach(() => {
|
||||
@@ -72,9 +88,60 @@ describe('stopRemotePaneTerminalForRestart (RA-11 paired pane)', () => {
|
||||
})
|
||||
})
|
||||
|
||||
const RESUME_ARGS = {
|
||||
ptyId: 'remote:env-1@@term_a',
|
||||
worktreeId: 'wt-1',
|
||||
tabId: 'web-terminal-host-tab%3A%3Aleaf',
|
||||
leafId: 'leaf-1',
|
||||
agent: 'claude' as const,
|
||||
providerSession: { key: 'session_id' as const, id: 'sess-1' }
|
||||
}
|
||||
|
||||
describe('resumeRemotePaneAgent (RA-11 paired pane)', () => {
|
||||
beforeEach(() => {
|
||||
calls.length = 0
|
||||
ensureAnswers = []
|
||||
showPtyId = 'host-pty-7'
|
||||
stopResult = { stoppedPtyIds: ['host-pty-7'], postStopVerified: true }
|
||||
})
|
||||
|
||||
it('returns the terminal the host ensured, so the pane can bind exactly it', async () => {
|
||||
ensureAnswers = [{ handle: 'term_resumed', disposition: 'created' }]
|
||||
await expect(resumeRemotePaneAgent(RESUME_ARGS)).resolves.toEqual({
|
||||
handle: 'term_resumed',
|
||||
disposition: 'created'
|
||||
})
|
||||
})
|
||||
|
||||
it('an adopted live terminal (the resume never ran) is stopped and the resume ensured once more', async () => {
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
ensureAnswers = [
|
||||
{ handle: 'term_original', disposition: 'adopted' },
|
||||
{ handle: 'term_resumed', disposition: 'created' }
|
||||
]
|
||||
await expect(resumeRemotePaneAgent(RESUME_ARGS)).resolves.toEqual({
|
||||
handle: 'term_resumed',
|
||||
disposition: 'created'
|
||||
})
|
||||
expect(calls.map((c) => c.method)).toEqual([
|
||||
'terminal.ensureAgentSession',
|
||||
'terminal.show',
|
||||
'terminal.stopExact',
|
||||
'terminal.ensureAgentSession'
|
||||
])
|
||||
expect(calls[1]).toMatchObject({ params: { terminal: 'term_original' } })
|
||||
warn.mockRestore()
|
||||
})
|
||||
|
||||
it('adopted twice fails the restart (never a silent new conversation)', async () => {
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
|
||||
ensureAnswers = [
|
||||
{ handle: 'term_original', disposition: 'adopted' },
|
||||
{ handle: 'term_original2', disposition: 'adopted' }
|
||||
]
|
||||
await expect(resumeRemotePaneAgent(RESUME_ARGS)).rejects.toThrow('restart_resume_adopted')
|
||||
expect(calls.filter((c) => c.method === 'terminal.ensureAgentSession')).toHaveLength(2)
|
||||
warn.mockRestore()
|
||||
})
|
||||
|
||||
it('ensures the agent on its own session, placed in the same host tab and leaf', async () => {
|
||||
@@ -100,6 +167,43 @@ describe('resumeRemotePaneAgent (RA-11 paired pane)', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('waitForResumedTerminalReady (RA-11 resume race)', () => {
|
||||
const sleep = () => Promise.resolve()
|
||||
let clock = 0
|
||||
const now = () => (clock += 300)
|
||||
const args = {
|
||||
ptyId: 'remote:env-1@@term_a',
|
||||
worktreeId: 'wt-1',
|
||||
tabId: 'web-terminal-host-tab%3A%3Aleaf',
|
||||
leafId: 'leaf-1',
|
||||
handle: 'term_resumed',
|
||||
sleep,
|
||||
now
|
||||
}
|
||||
beforeEach(() => {
|
||||
calls.length = 0
|
||||
clock = 0
|
||||
listAnswers = []
|
||||
})
|
||||
|
||||
it('waits until the ensured terminal itself is live in the pane’s own tab and leaf', async () => {
|
||||
listAnswers = [
|
||||
[],
|
||||
// Some other terminal ready in the leaf is not enough.
|
||||
[{ handle: 'term_original', tabId: 'host-tab::leaf', leafId: 'leaf-1' }],
|
||||
[{ handle: 'term_resumed', tabId: 'host-tab::leaf', leafId: 'leaf-1', connected: true }]
|
||||
]
|
||||
await expect(waitForResumedTerminalReady(args)).resolves.toBeUndefined()
|
||||
expect(calls.filter((c) => c.method === 'terminal.list')).toHaveLength(3)
|
||||
expect(calls[0]).toMatchObject({ params: { handles: ['term_resumed'] } })
|
||||
})
|
||||
|
||||
it('throws when the ensured terminal never shows in that leaf', async () => {
|
||||
listAnswers = [[{ handle: 'term_resumed', tabId: 'host-tab::leaf', leafId: 'other-leaf' }]]
|
||||
await expect(waitForResumedTerminalReady(args)).rejects.toThrow('restart_resume_not_ready')
|
||||
})
|
||||
})
|
||||
|
||||
describe('freshIfResumeFoundNothing (RA-11e)', () => {
|
||||
const sleep = () => Promise.resolve()
|
||||
let clock = 0
|
||||
@@ -163,7 +267,11 @@ describe('freshIfResumeFoundNothing (RA-11e)', () => {
|
||||
it('never types twice: a timed-out send whose Claude came up anyway is not sent again', async () => {
|
||||
readTail = ['No conversation found with session ID: s1', '$ ']
|
||||
sendAnswers = ['timeout']
|
||||
tailAfterSend = ['No conversation found with session ID: s1', '$ claude', 'Claude Code v2.1.319']
|
||||
tailAfterSend = [
|
||||
'No conversation found with session ID: s1',
|
||||
'$ claude',
|
||||
'Claude Code v2.1.319'
|
||||
]
|
||||
const relaunched = await freshIfResumeFoundNothing({
|
||||
currentPtyId: () => 'remote:env-1@@term_new',
|
||||
replacedPtyId: 'remote:env-1@@term_old',
|
||||
|
||||
@@ -3,7 +3,10 @@
|
||||
// relaunches. Put exactly that one host terminal to sleep first (the reversible stop the pane-sleep
|
||||
// path uses: the leaf stays, as a sleeping surface); the reconnect then wakes it with --resume.
|
||||
// A plain stop is a close: the host drops the tab and there is nothing left to relaunch into.
|
||||
import { parseRemoteRuntimePtyId } from '../../../../shared/remote-runtime/remote-runtime-pty-id'
|
||||
import {
|
||||
parseRemoteRuntimePtyId,
|
||||
toRemoteRuntimePtyId
|
||||
} from '../../../../shared/remote-runtime/remote-runtime-pty-id'
|
||||
import type { RuntimeTerminalShow } from '../../../../shared/runtime-terminal-contracts'
|
||||
import { callRuntimeRpc } from '@/runtime/rpc/calls/runtime-rpc-call'
|
||||
import { toRuntimeWorktreeSelector } from '@/runtime/rpc/targets/runtime-worktree-selector'
|
||||
@@ -27,7 +30,10 @@ export async function stopRemotePaneTerminalForRestart(
|
||||
if (!parsed?.environmentId) {
|
||||
throw new Error('restart_remote_pane_unowned')
|
||||
}
|
||||
const target = { kind: 'environment' as const, environmentId: parsed.environmentId }
|
||||
const target = {
|
||||
kind: 'environment' as const,
|
||||
environmentId: parsed.environmentId
|
||||
}
|
||||
// The pane names its terminal by handle; the host stops by its own PTY id.
|
||||
const { terminal } = await callRuntimeRpc<{ terminal?: RuntimeTerminalShow }>(
|
||||
target,
|
||||
@@ -57,10 +63,20 @@ export async function stopRemotePaneTerminalForRestart(
|
||||
}
|
||||
}
|
||||
|
||||
/** The terminal the host ensured for the resume, and whether it spawned it or adopted a live one. */
|
||||
export type RemotePaneResume = {
|
||||
handle: string
|
||||
disposition: 'created' | 'adopted'
|
||||
}
|
||||
|
||||
/**
|
||||
* Relaunch the pane's agent on its own conversation, in the same pane. Without this the host wakes
|
||||
* the sleeping pane with its original launch (a new conversation). Only for a resumable session;
|
||||
* a pane with none takes the host's fresh launch, which is what "Starts a new conversation" says.
|
||||
*
|
||||
* Returns the ensured terminal's handle so the pane binds to exactly it. `adopted` means a live
|
||||
* owner already held the leaf (something woke it with its original launch first) and the resume
|
||||
* never ran: stop that terminal and ensure once more; still adopted throws (Couldn't restart).
|
||||
*/
|
||||
export async function resumeRemotePaneAgent(args: {
|
||||
ptyId: string
|
||||
@@ -69,25 +85,126 @@ export async function resumeRemotePaneAgent(args: {
|
||||
leafId: string
|
||||
agent: AccountRestartAgent
|
||||
providerSession: AgentProviderSessionMetadata
|
||||
}): Promise<RemotePaneResume> {
|
||||
const environmentId = parseRemoteRuntimePtyId(args.ptyId)?.environmentId
|
||||
if (!environmentId) {
|
||||
throw new Error('restart_remote_pane_unowned')
|
||||
}
|
||||
const ensure = async (): Promise<RemotePaneResume> => {
|
||||
const result = await callRuntimeRpc<{
|
||||
terminal?: { handle?: string }
|
||||
disposition?: 'created' | 'adopted'
|
||||
}>(
|
||||
{ kind: 'environment', environmentId },
|
||||
'terminal.ensureAgentSession',
|
||||
{
|
||||
kind: 'explicit',
|
||||
worktree: toRuntimeWorktreeSelector(args.worktreeId),
|
||||
agent: args.agent,
|
||||
providerSession: await agentProviderSessionForHostEnsure(
|
||||
environmentId,
|
||||
args.providerSession
|
||||
),
|
||||
// The same pane: the host tab and leaf the stopped terminal lived in.
|
||||
placement: {
|
||||
tabId: toHostSessionTabId(args.tabId),
|
||||
leafId: args.leafId
|
||||
},
|
||||
presentation: 'background'
|
||||
},
|
||||
{ timeoutMs: 15_000 }
|
||||
)
|
||||
const handle = result?.terminal?.handle
|
||||
if (!handle) {
|
||||
throw new Error('restart_resume_no_terminal')
|
||||
}
|
||||
return {
|
||||
handle,
|
||||
disposition: result?.disposition === 'adopted' ? 'adopted' : 'created'
|
||||
}
|
||||
}
|
||||
const first = await ensure()
|
||||
if (first.disposition !== 'adopted') {
|
||||
return first
|
||||
}
|
||||
console.warn(
|
||||
'[orca-cloud] RA-11 resume adopted a live terminal; stopping it and resuming again',
|
||||
first.handle
|
||||
)
|
||||
await stopRemotePaneTerminalForRestart(
|
||||
toRemoteRuntimePtyId(first.handle, environmentId),
|
||||
args.worktreeId
|
||||
)
|
||||
const second = await ensure()
|
||||
if (second.disposition === 'adopted') {
|
||||
throw new Error('restart_resume_adopted')
|
||||
}
|
||||
return second
|
||||
}
|
||||
|
||||
// The ensured terminal shows up in the host's inventory within a second or two of the ensure.
|
||||
const RESUMED_READY_WAIT_MS = 15_000
|
||||
const RESUMED_READY_POLL_MS = 300
|
||||
|
||||
/**
|
||||
* RA-11 resume race: wait until the host lists exactly the ensured terminal `handle`, live, in the
|
||||
* pane's own host tab and leaf. Only then may the pane reconnect: its activation reuses a ready leaf,
|
||||
* and a leaf that is not ready yet is woken with its ORIGINAL launch (the resume is then lost).
|
||||
* A turn-less `--resume` exits to a shell but its PTY stays live, so it still lists. Throws when the
|
||||
* terminal never shows, so the restart fails visibly instead of reconnecting into the race.
|
||||
*/
|
||||
export async function waitForResumedTerminalReady(args: {
|
||||
ptyId: string
|
||||
worktreeId: string
|
||||
tabId: string
|
||||
leafId: string
|
||||
handle: string
|
||||
sleep?: (ms: number) => Promise<void>
|
||||
now?: () => number
|
||||
}): Promise<void> {
|
||||
const environmentId = parseRemoteRuntimePtyId(args.ptyId)?.environmentId
|
||||
if (!environmentId) {
|
||||
throw new Error('restart_remote_pane_unowned')
|
||||
}
|
||||
await callRuntimeRpc(
|
||||
{ kind: 'environment', environmentId },
|
||||
'terminal.ensureAgentSession',
|
||||
{
|
||||
kind: 'explicit',
|
||||
worktree: toRuntimeWorktreeSelector(args.worktreeId),
|
||||
agent: args.agent,
|
||||
providerSession: await agentProviderSessionForHostEnsure(environmentId, args.providerSession),
|
||||
// The same pane: the host tab and leaf the stopped terminal lived in.
|
||||
placement: { tabId: toHostSessionTabId(args.tabId), leafId: args.leafId },
|
||||
presentation: 'background'
|
||||
},
|
||||
{ timeoutMs: 15_000 }
|
||||
)
|
||||
const sleep =
|
||||
args.sleep ?? ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)))
|
||||
const now = args.now ?? Date.now
|
||||
const hostTabId = toHostSessionTabId(args.tabId)
|
||||
const deadline = now() + RESUMED_READY_WAIT_MS
|
||||
while (now() < deadline) {
|
||||
const listed = await callRuntimeRpc<{
|
||||
terminals?: {
|
||||
handle?: string
|
||||
tabId?: string
|
||||
leafId?: string
|
||||
connected?: boolean
|
||||
verdict?: string
|
||||
}[]
|
||||
}>(
|
||||
{ kind: 'environment', environmentId },
|
||||
'terminal.list',
|
||||
{
|
||||
worktree: toRuntimeWorktreeSelector(args.worktreeId),
|
||||
handles: [args.handle],
|
||||
includePreview: false,
|
||||
includeVisualLayouts: false
|
||||
},
|
||||
{ timeoutMs: 5_000 }
|
||||
).catch(() => null)
|
||||
const ready = (listed?.terminals ?? []).some(
|
||||
(terminal) =>
|
||||
terminal.handle === args.handle &&
|
||||
terminal.tabId === hostTabId &&
|
||||
terminal.leafId === args.leafId &&
|
||||
terminal.connected !== false &&
|
||||
terminal.verdict !== 'dead'
|
||||
)
|
||||
if (ready) {
|
||||
return
|
||||
}
|
||||
await sleep(RESUMED_READY_POLL_MS)
|
||||
}
|
||||
throw new Error('restart_resume_not_ready')
|
||||
}
|
||||
|
||||
/** What Claude prints when `--resume <id>` names a session that never had a conversation. */
|
||||
@@ -164,7 +281,9 @@ async function sendFreshOnce(
|
||||
try {
|
||||
// The Enter rides in the text, not `enter: true`: the host holds an Enter send for the launched
|
||||
// Claude's composer (8 s), which never comes once it exited to the shell, then refuses it.
|
||||
const sent = await callRuntimeRpc<{ send?: { accepted?: boolean; refusedReason?: string } }>(
|
||||
const sent = await callRuntimeRpc<{
|
||||
send?: { accepted?: boolean; refusedReason?: string }
|
||||
}>(
|
||||
target,
|
||||
'terminal.send',
|
||||
{ terminal: handle, text: `${freshCommand}\r` },
|
||||
@@ -215,7 +334,10 @@ export async function freshIfResumeFoundNothing(args: {
|
||||
lastSeen = current.handle
|
||||
console.info('[orca-cloud] RA-11e watching relaunched terminal', current.handle)
|
||||
}
|
||||
const target = { kind: 'environment' as const, environmentId: current.environmentId }
|
||||
const target = {
|
||||
kind: 'environment' as const,
|
||||
environmentId: current.environmentId
|
||||
}
|
||||
const read = await callRuntimeRpc<{ terminal?: { tail?: string[] } }>(
|
||||
target,
|
||||
'terminal.read',
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
// RA-11 resume race: after a cloud Restart reconnects, the pane must sit on the terminal the resume
|
||||
// ensured. Anything else is the leaf's original launch (a new conversation), which the strip reports.
|
||||
import { useAppStore } from '@/store'
|
||||
import {
|
||||
parseRemoteRuntimePtyId,
|
||||
toRemoteRuntimePtyId
|
||||
} from '../../../../shared/remote-runtime/remote-runtime-pty-id'
|
||||
|
||||
// How long the pane may take to bind its relaunched terminal before the check gives up (the
|
||||
// attempt's own timeout reports a pane that never binds).
|
||||
const BIND_CHECK_MS = 30_000
|
||||
|
||||
function paneHandleOf(tabId: string, leafId: string): { ptyId: string; handle: string } | null {
|
||||
const ptyId =
|
||||
useAppStore.getState().terminalLayoutsByTabId[tabId]?.ptyIdsByLeafId?.[leafId] ?? null
|
||||
const parsed = ptyId ? parseRemoteRuntimePtyId(ptyId) : null
|
||||
return ptyId && parsed ? { ptyId, handle: parsed.handle } : null
|
||||
}
|
||||
|
||||
/** The pane id for `handle` on the same environment as `ptyId` (falls back to `ptyId`). */
|
||||
export function remotePtyIdWithHandle(ptyId: string, handle: string): string {
|
||||
const environmentId = parseRemoteRuntimePtyId(ptyId)?.environmentId
|
||||
return environmentId ? toRemoteRuntimePtyId(handle, environmentId) : ptyId
|
||||
}
|
||||
|
||||
/** Once the pane binds a new PTY, check it is `handle`; report anything else. */
|
||||
export function expectPaneBindsHandle(args: {
|
||||
tabId: string
|
||||
leafId: string
|
||||
replacedPtyId: string
|
||||
handle: string
|
||||
onMismatch: (boundHandle: string) => void
|
||||
}): void {
|
||||
let done = false
|
||||
const check = (): boolean => {
|
||||
const bound = paneHandleOf(args.tabId, args.leafId)
|
||||
if (!bound || bound.ptyId === args.replacedPtyId) {
|
||||
return false
|
||||
}
|
||||
if (bound.handle !== args.handle) {
|
||||
args.onMismatch(bound.handle)
|
||||
}
|
||||
return true
|
||||
}
|
||||
if (check()) {
|
||||
return
|
||||
}
|
||||
const finish = (): void => {
|
||||
done = true
|
||||
unsubscribe()
|
||||
clearTimeout(timer)
|
||||
}
|
||||
const unsubscribe = useAppStore.subscribe(() => {
|
||||
if (!done && check()) {
|
||||
finish()
|
||||
}
|
||||
})
|
||||
const timer = setTimeout(() => {
|
||||
if (!done) {
|
||||
finish()
|
||||
}
|
||||
}, BIND_CHECK_MS)
|
||||
}
|
||||
Reference in New Issue
Block a user