fix(remote-runtime): requests wait for an in-flight VM wake; resumed remote agents continue a turn their host cut off; restart offers tolerate host clock skew

This commit is contained in:
Neil
2026-11-09 06:06:57 -08:00
parent e2d32ce606
commit d07e40ccda
14 changed files with 250 additions and 8 deletions
@@ -24,6 +24,10 @@ const ContinueAnswer = z.looseObject({
skipped: z.array(z.unknown()).optional()
})
/** `recordedAt` is the host's clock and `drainStartedAt` this machine's: tolerate skew between
* them so an offer the drain just recorded is not mistaken for an older one. */
export const CLOCK_SKEW_TOLERANCE_MS = 2 * 60_000
function hasCode(error: unknown, code: string): boolean {
return typeof error === 'object' && error !== null && 'code' in error && error.code === code
}
@@ -47,7 +51,9 @@ export async function autoResumeAgentsAfterRollover(
}
const ids = list.data.sessions.flatMap((row) => {
const offer = Offer.safeParse(row)
return offer.success && offer.data.recordedAt >= drainStartedAt ? [offer.data.sessionId] : []
return offer.success && offer.data.recordedAt >= drainStartedAt - CLOCK_SKEW_TOLERANCE_MS
? [offer.data.sessionId]
: []
})
const outcome = { offered: ids.length, continued: 0, skipped: 0, unsupported: false }
for (let offset = 0; offset < ids.length; offset += MAX_RESTART_RESUME_SESSIONS) {
@@ -112,14 +112,15 @@ describe('agent auto-resume after rollover', () => {
if (method === 'agentSession.restartResumable') {
return {
sessions: [
{ sessionId: 'old', recordedAt: 900 },
{ sessionId: 'a', recordedAt: 1000 },
{ sessionId: 'b', recordedAt: 1500 }
{ sessionId: 'old', recordedAt: T0 - 10 * MIN },
// The host's clock runs 30 s behind this machine's: still this drain's offer.
{ sessionId: 'a', recordedAt: T0 - 30_000 },
{ sessionId: 'b', recordedAt: T0 + 500 }
]
}
}
return { continued: [{ sessionId: 'a' }], skipped: ['b'] }
}, 1000)
}, T0)
expect(calls).toEqual([
{ method: 'agentSession.restartResumable', params: undefined },
{ method: 'agentSession.restartContinue', params: { sessionIds: ['a', 'b'] } }
@@ -131,6 +131,44 @@ describe('a request to a recipe VM environment', () => {
}
)
it('holds a request that does not open the workspace while an in-flight wake finishes', async () => {
recordVm('running')
ownerState = 'waking'
const awaitWake = vi.fn(async () => 'up' as const)
registerRecipeVmWaker({
state,
noteOpened: vi.fn(),
holdOpen,
awaitWake,
openEnvironment: async () => 'up'
})
noteRecipeVmWakePairing(VM_ENVIRONMENT.id, 7, true)
await expect(prepareRecipeVm(userDataPath, VM_ENVIRONMENT, 'none')).resolves.toEqual({
wokenPairingRevision: null
})
expect(awaitWake).toHaveBeenCalledWith('workspace-of-vm-1', false, 60_000, undefined)
// It never starts a wake of its own.
expect(holdOpen).not.toHaveBeenCalled()
})
it.each([
['waking', 'runtime_environment_waking'],
['wake-failed', 'runtime_environment_wake_failed'],
['slept', 'runtime_environment_asleep'],
['asleep', 'runtime_environment_asleep']
] as const)('refuses with %s when the in-flight wake ends that way', async (outcome, code) => {
recordVm('running')
ownerState = 'waking'
registerRecipeVmWaker({
state,
noteOpened: vi.fn(),
holdOpen,
awaitWake: async () => outcome,
openEnvironment: async () => 'up'
})
await expect(codeOf('none')).resolves.toBe(code)
})
// A phone gives up after 30 seconds, so its open is held for less.
// A Disconnect made before the open holds: no client's open wakes the VM behind that server.
@@ -111,12 +111,31 @@ export async function prepareRecipeVm(
return { wokenPairingRevision: woken !== wakeBefore ? (woken?.pairingRevision ?? null) : null }
}
const state = waker.state(vm.workspaceId, vm.recordSuspended)
if (state === 'waking' && waker.awaitWake) {
// The wake is already underway (often just reconnecting): a terminal or panel opened in that
// window waits for it instead of failing and never retrying. Nothing here starts a wake.
const wakeBefore = latestRecipeVmWake(environment.id)
const outcome = await waker.awaitWake(
vm.workspaceId,
vm.recordSuspended,
Math.min(holdMs, RECIPE_VM_IN_FLIGHT_WAKE_HOLD_MS),
signal
)
if (outcome !== 'up') {
throw refusal(outcome)
}
const woken = latestRecipeVmWake(environment.id)
return { wokenPairingRevision: woken !== wakeBefore ? (woken?.pairingRevision ?? null) : null }
}
if (state !== 'up') {
throw refusal(state)
}
return UNCHANGED
}
/** How long a request that does not open the workspace waits for a wake already in flight. */
export const RECIPE_VM_IN_FLIGHT_WAKE_HOLD_MS = 60_000
/** The pairing revision a recipe VM wake this request waited for left its server on, if any. */
export type RecipeVmPreparation = { wokenPairingRevision: number | null }
const UNCHANGED: RecipeVmPreparation = { wokenPairingRevision: null }
@@ -0,0 +1,30 @@
import { describe, expect, it } from 'vitest'
import { awaitInFlightWake, WorkspaceSleepRequests, type VmOperation } from './recipe-vm-operations'
describe('awaitInFlightWake', () => {
it('answers at once for a VM that is up or asleep, without starting anything', async () => {
const sleeps = new WorkspaceSleepRequests()
await expect(awaitInFlightWake([], false, sleeps, 'w', 1000)).resolves.toBe('up')
await expect(awaitInFlightWake([], true, sleeps, 'w', 1000)).resolves.toBe('asleep')
})
it('waits for the wake already in flight', async () => {
let finish = (): void => {}
const settled = new Promise<void>((resolve) => {
finish = resolve
})
const pending: VmOperation[] = [{ kind: 'wake', settled }]
const answer = awaitInFlightWake(pending, false, new WorkspaceSleepRequests(), 'w', 5000)
finish()
await expect(answer).resolves.toBe('up')
})
it('ends slept when a Sleep is asked for while it waits, and waking when the hold runs out', async () => {
const sleeps = new WorkspaceSleepRequests()
const never: VmOperation[] = [{ kind: 'wake', settled: new Promise(() => {}) }]
const slept = awaitInFlightWake(never, false, sleeps, 'w', 5000)
sleeps.noteRequested('w')
await expect(slept).resolves.toBe('slept')
await expect(awaitInFlightWake(never, false, sleeps, 'w', 5)).resolves.toBe('waking')
})
})
@@ -124,3 +124,26 @@ export async function holdOpenUntilUp(
stopAbortWatch()
}
}
/** Holds a request that does not open the workspace while the wake already in flight settles; a VM
* that is not waking answers at once with where it stands. Never starts a wake. */
export async function awaitInFlightWake(
pending: readonly VmOperation[],
recordSuspended: boolean,
sleepRequests: WorkspaceSleepRequests,
worktreeId: string,
timeoutMs: number,
signal?: AbortSignal
): Promise<RecipeVmOpening | 'asleep'> {
signal?.throwIfAborted()
const state = vmStateOf(pending, recordSuspended)
if (state !== 'waking') {
return state === 'up' ? 'up' : 'asleep'
}
const sleep = sleepRequests.next(worktreeId)
try {
return await holdOpenUntilUp(pendingVmWake(pending), sleep.requested, timeoutMs, signal)
} finally {
sleep.stop()
}
}
@@ -15,6 +15,14 @@ export type RecipeVmWaker = {
timeoutMs: number,
signal?: AbortSignal
) => Promise<RecipeVmOpening>
/** Holds a request that does not open the workspace while a wake already in flight finishes;
* never starts one. */
awaitWake?: (
workspaceId: string,
recordSuspended: boolean,
timeoutMs: number,
signal?: AbortSignal
) => Promise<RecipeVmOpening | 'asleep'>
/** Holds an open of the workspace behind a saved environment (id or name) for the CLI. */
openEnvironment: (selector: string) => Promise<RecipeVmOpening>
}
@@ -161,6 +161,16 @@ export abstract class RuntimeWorkspaceSleepOperations extends OrcaRuntimeWithRes
return this.workspaceSleep.holdOpen(worktreeId, recordSuspended, timeoutMs, signal)
}
/** Holds a request that does not open this workspace while its in-flight wake finishes. */
awaitWorkspaceVmWake(
worktreeId: string,
recordSuspended: boolean,
timeoutMs: number,
signal?: AbortSignal
): Promise<RecipeVmOpening | 'asleep'> {
return this.workspaceSleep.awaitWake(worktreeId, recordSuspended, timeoutMs, signal)
}
/** The window opened a slept pane; main resumes it and the window attaches. */
resumeSleptPane(pane: RestoredPane, automatic?: boolean): Promise<boolean> {
return this.workspaceSleep.resumePane(pane, automatic)
@@ -1,4 +1,5 @@
import {
awaitInFlightWake,
holdOpenUntilUp,
pendingVmWake,
trackPendingVmOperation,
@@ -209,6 +210,20 @@ export class WorkspaceSleepCoordinator {
}
}
/** Holds a request that does not open the workspace while a wake already in flight finishes;
* never starts a wake or counts as a reopening, so background traffic leaves a VM asleep. */
awaitWake(worktreeId: string, recordSuspended: boolean, timeoutMs: number, signal?: AbortSignal) {
const pending = this.pendingVmOperations.get(worktreeId) ?? []
return awaitInFlightWake(
pending,
recordSuspended,
this.sleepRequests,
worktreeId,
timeoutMs,
signal
)
}
private trackVmOperation<T>(worktreeId: string, operation: VmOperation, run: Promise<T>) {
return trackPendingVmOperation(this.pendingVmOperations, worktreeId, operation, run)
}
@@ -166,6 +166,8 @@ export function initializeMainProcessRuntime(): OrcaRuntimeService {
noteOpened: (workspaceId) => runtime.noteWorkspaceReopened(workspaceId),
holdOpen: (workspaceId, suspended, timeoutMs, signal) =>
runtime.holdWorkspaceVmOpen(workspaceId, suspended, timeoutMs, signal),
awaitWake: (workspaceId, suspended, timeoutMs, signal) =>
runtime.awaitWorkspaceVmWake(workspaceId, suspended, timeoutMs, signal),
openEnvironment: (selector) => {
const root = getRuntimeEnvironmentRegistryRoot()
return openRecipeVmEnvironment(root, resolveEnvironment(root, selector))
@@ -0,0 +1,13 @@
import { describe, expect, it, vi } from 'vitest'
vi.mock('@/store/index', () => ({ useAppStore: { getState: () => ({}) } }))
describe('shouldContinueInterruptedTurn', () => {
it('continues only a remote agent whose last status was working', async () => {
const { shouldContinueInterruptedTurn } = await import('./cold-restore-resume-startup')
expect(shouldContinueInterruptedTurn({ state: 'working' }, 'env-1')).toBe(true)
expect(shouldContinueInterruptedTurn({ state: 'waiting' }, 'env-1')).toBe(false)
expect(shouldContinueInterruptedTurn({ state: 'working' }, null)).toBe(false)
expect(shouldContinueInterruptedTurn(null, 'env-1')).toBe(false)
})
})
@@ -6,7 +6,10 @@ import { normalizeExecutionHostId } from '../../../../../../shared/execution-hos
import { parseWorkspaceKey } from '../../../../../../shared/workspace-scope'
import { useAppStore } from '@/store/index'
import { createNonSecureContextUuid as createBrowserUuid } from '../../../../../../shared/non-secure-context-uuid'
import { buildAgentResumeStartupPlan } from '../../../../../../shared/agent-launch/startup/tui-agent-resume-startup'
import {
INTERRUPTED_TURN_CONTINUE_PROMPT,
buildAgentResumeStartupPlan
} from '../../../../../../shared/agent-launch/startup/tui-agent-resume-startup'
import {
resolveRuntimeHostReportedPlatform,
resolveRuntimeHostTerminalWindowsShell,
@@ -31,6 +34,15 @@ import type { ConnectPanePtySession } from '../connect-pane-pty-session'
export const NOT_REOPENED: HeldPaneReopenOutcome = { reopened: false, agentResumed: false }
/** A cold-restored agent on a remote host whose last status was working lost its turn when the
* host restarted; local panes are left alone (their PTYs outlive app restarts). */
export function shouldContinueInterruptedTurn(
entry: { state: string } | null | undefined,
runtimeEnvironmentId: string | null
): boolean {
return Boolean(runtimeEnvironmentId) && entry?.state === 'working'
}
type AppState = ReturnType<typeof useAppStore.getState>
/** The agent session a cold restore of this pane would resume; read-only, so pre-spawn checks can share it. */
@@ -117,7 +129,15 @@ export function bindBuildColdRestoreAgentResumeStartup(session: ConnectPanePtySe
platform: resumeTarget.platform,
agentHostPlatform: resumeTarget.agentHostPlatform,
shell: resumeTarget.shell,
launchCwd: session.worktree?.path
launchCwd: session.worktree?.path,
// A remote host that restarted under a working agent (reboot, crash, VM rollover) killed it
// mid-turn; its last status still says working. Carry the turn on instead of waiting.
continuePrompt: shouldContinueInterruptedTurn(
entry,
session.transport.getRuntimeEnvironmentId?.() ?? null
)
? INTERRUPTED_TURN_CONTINUE_PROMPT
: null
})
if (!startupPlan) {
return null
@@ -0,0 +1,43 @@
import { describe, expect, it } from 'vitest'
import { buildAgentResumeStartupPlan } from './tui-agent-resume-startup'
const providerSession = { key: 'session_id', id: 'claude-session-1' } as const
describe('resume with a continue prompt', () => {
const base = { cmdOverrides: {}, platform: 'linux' as const, shell: 'posix' as const }
it('submits the prompt on resume for Claude and Codex', () => {
const claude = buildAgentResumeStartupPlan({
...base,
agent: 'claude',
providerSession,
continuePrompt: 'continue'
})
expect(claude?.launchCommand).toMatch(/'?--resume'? '?claude-session-1'? '?continue'?$/)
const codex = buildAgentResumeStartupPlan({
...base,
agent: 'codex',
providerSession: { key: 'session_id', id: 'codex-1' },
continuePrompt: 'continue'
})
expect(codex?.launchCommand).toMatch(/'?resume'? '?codex-1'? '?continue'?$/)
})
it('leaves other agents and empty prompts unchanged', () => {
const plain = buildAgentResumeStartupPlan({ ...base, agent: 'claude', providerSession })
const blank = buildAgentResumeStartupPlan({
...base,
agent: 'claude',
providerSession,
continuePrompt: ' '
})
expect(blank?.launchCommand).toBe(plain?.launchCommand)
const cursor = buildAgentResumeStartupPlan({
...base,
agent: 'cursor',
providerSession: { key: 'conversation_id', id: 'c-1' },
continuePrompt: 'continue'
})
expect(cursor?.launchCommand).not.toMatch(/continue'?$/)
})
})
@@ -37,6 +37,9 @@ export function buildAgentResumeStartupPlan(args: {
fork?: boolean
/** OS that runs the agent when `platform` only picks quoting; null when unreported. */
agentHostPlatform?: NodeJS.Platform | null
/** First message the resumed session submits, e.g. to carry on a turn its host cut off. Only
* agents whose resume takes a prompt argument (Claude, Codex) send it; others ignore it. */
continuePrompt?: string | null
}): AgentStartupPlan | null {
const resumeArgv = args.fork
? getAgentForkArgv(args.agent, args.providerSession)
@@ -45,10 +48,15 @@ export function buildAgentResumeStartupPlan(args: {
return null
}
// Codex otherwise asks for its old folder, even if deleted (#17745). Keep before resume so argv removal preserves the folder choice.
const argv =
const withCwd =
args.agent === 'codex' && args.resumeInLaunchCwd
? [resumeArgv[0], '-c', 'tui.resume_cwd=current', ...resumeArgv.slice(1)]
: resumeArgv
const continuePrompt = args.continuePrompt?.trim()
const argv =
continuePrompt && !args.fork && RESUME_PROMPT_AGENTS.has(args.agent)
? [...withCwd, continuePrompt]
: withCwd
const shell = resolveStartupShell(args.platform, args.shell)
// Why: Codex restores a resumed thread's saved permissions unless the launch passes one, so
// replaying Orca's launch-time flag would undo a change made inside the session (#10886).
@@ -112,6 +120,12 @@ export function buildAgentResumeStartupPlan(args: {
}
}
/** Agents whose resume command accepts a positional prompt it submits on start. */
const RESUME_PROMPT_AGENTS: ReadonlySet<ResumableTuiAgent> = new Set(['claude', 'codex'])
/** What a resumed agent is told when its host went away mid-turn (restart, reboot, crash). */
export const INTERRUPTED_TURN_CONTINUE_PROMPT = 'continue'
/** WSL panes start at a UNC path while the agent reports its Linux path. */
function isSameFolder(agentCwd: string, launchCwd: string): boolean {
const launchPath = parseWslUncPath(launchCwd)?.linuxPath ?? launchCwd