mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 10:58:10 +00:00
fix(remote-runtime): requests wait for an in-flight VM wake; resumed remote agents continue a turn their host cut off; restart offers tolerate host clock skew
This commit is contained in:
@@ -24,6 +24,10 @@ const ContinueAnswer = z.looseObject({
|
||||
skipped: z.array(z.unknown()).optional()
|
||||
})
|
||||
|
||||
/** `recordedAt` is the host's clock and `drainStartedAt` this machine's: tolerate skew between
|
||||
* them so an offer the drain just recorded is not mistaken for an older one. */
|
||||
export const CLOCK_SKEW_TOLERANCE_MS = 2 * 60_000
|
||||
|
||||
function hasCode(error: unknown, code: string): boolean {
|
||||
return typeof error === 'object' && error !== null && 'code' in error && error.code === code
|
||||
}
|
||||
@@ -47,7 +51,9 @@ export async function autoResumeAgentsAfterRollover(
|
||||
}
|
||||
const ids = list.data.sessions.flatMap((row) => {
|
||||
const offer = Offer.safeParse(row)
|
||||
return offer.success && offer.data.recordedAt >= drainStartedAt ? [offer.data.sessionId] : []
|
||||
return offer.success && offer.data.recordedAt >= drainStartedAt - CLOCK_SKEW_TOLERANCE_MS
|
||||
? [offer.data.sessionId]
|
||||
: []
|
||||
})
|
||||
const outcome = { offered: ids.length, continued: 0, skipped: 0, unsupported: false }
|
||||
for (let offset = 0; offset < ids.length; offset += MAX_RESTART_RESUME_SESSIONS) {
|
||||
|
||||
@@ -112,14 +112,15 @@ describe('agent auto-resume after rollover', () => {
|
||||
if (method === 'agentSession.restartResumable') {
|
||||
return {
|
||||
sessions: [
|
||||
{ sessionId: 'old', recordedAt: 900 },
|
||||
{ sessionId: 'a', recordedAt: 1000 },
|
||||
{ sessionId: 'b', recordedAt: 1500 }
|
||||
{ sessionId: 'old', recordedAt: T0 - 10 * MIN },
|
||||
// The host's clock runs 30 s behind this machine's: still this drain's offer.
|
||||
{ sessionId: 'a', recordedAt: T0 - 30_000 },
|
||||
{ sessionId: 'b', recordedAt: T0 + 500 }
|
||||
]
|
||||
}
|
||||
}
|
||||
return { continued: [{ sessionId: 'a' }], skipped: ['b'] }
|
||||
}, 1000)
|
||||
}, T0)
|
||||
expect(calls).toEqual([
|
||||
{ method: 'agentSession.restartResumable', params: undefined },
|
||||
{ method: 'agentSession.restartContinue', params: { sessionIds: ['a', 'b'] } }
|
||||
|
||||
@@ -131,6 +131,44 @@ describe('a request to a recipe VM environment', () => {
|
||||
}
|
||||
)
|
||||
|
||||
it('holds a request that does not open the workspace while an in-flight wake finishes', async () => {
|
||||
recordVm('running')
|
||||
ownerState = 'waking'
|
||||
const awaitWake = vi.fn(async () => 'up' as const)
|
||||
registerRecipeVmWaker({
|
||||
state,
|
||||
noteOpened: vi.fn(),
|
||||
holdOpen,
|
||||
awaitWake,
|
||||
openEnvironment: async () => 'up'
|
||||
})
|
||||
noteRecipeVmWakePairing(VM_ENVIRONMENT.id, 7, true)
|
||||
await expect(prepareRecipeVm(userDataPath, VM_ENVIRONMENT, 'none')).resolves.toEqual({
|
||||
wokenPairingRevision: null
|
||||
})
|
||||
expect(awaitWake).toHaveBeenCalledWith('workspace-of-vm-1', false, 60_000, undefined)
|
||||
// It never starts a wake of its own.
|
||||
expect(holdOpen).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it.each([
|
||||
['waking', 'runtime_environment_waking'],
|
||||
['wake-failed', 'runtime_environment_wake_failed'],
|
||||
['slept', 'runtime_environment_asleep'],
|
||||
['asleep', 'runtime_environment_asleep']
|
||||
] as const)('refuses with %s when the in-flight wake ends that way', async (outcome, code) => {
|
||||
recordVm('running')
|
||||
ownerState = 'waking'
|
||||
registerRecipeVmWaker({
|
||||
state,
|
||||
noteOpened: vi.fn(),
|
||||
holdOpen,
|
||||
awaitWake: async () => outcome,
|
||||
openEnvironment: async () => 'up'
|
||||
})
|
||||
await expect(codeOf('none')).resolves.toBe(code)
|
||||
})
|
||||
|
||||
// A phone gives up after 30 seconds, so its open is held for less.
|
||||
|
||||
// A Disconnect made before the open holds: no client's open wakes the VM behind that server.
|
||||
|
||||
@@ -111,12 +111,31 @@ export async function prepareRecipeVm(
|
||||
return { wokenPairingRevision: woken !== wakeBefore ? (woken?.pairingRevision ?? null) : null }
|
||||
}
|
||||
const state = waker.state(vm.workspaceId, vm.recordSuspended)
|
||||
if (state === 'waking' && waker.awaitWake) {
|
||||
// The wake is already underway (often just reconnecting): a terminal or panel opened in that
|
||||
// window waits for it instead of failing and never retrying. Nothing here starts a wake.
|
||||
const wakeBefore = latestRecipeVmWake(environment.id)
|
||||
const outcome = await waker.awaitWake(
|
||||
vm.workspaceId,
|
||||
vm.recordSuspended,
|
||||
Math.min(holdMs, RECIPE_VM_IN_FLIGHT_WAKE_HOLD_MS),
|
||||
signal
|
||||
)
|
||||
if (outcome !== 'up') {
|
||||
throw refusal(outcome)
|
||||
}
|
||||
const woken = latestRecipeVmWake(environment.id)
|
||||
return { wokenPairingRevision: woken !== wakeBefore ? (woken?.pairingRevision ?? null) : null }
|
||||
}
|
||||
if (state !== 'up') {
|
||||
throw refusal(state)
|
||||
}
|
||||
return UNCHANGED
|
||||
}
|
||||
|
||||
/** How long a request that does not open the workspace waits for a wake already in flight. */
|
||||
export const RECIPE_VM_IN_FLIGHT_WAKE_HOLD_MS = 60_000
|
||||
|
||||
/** The pairing revision a recipe VM wake this request waited for left its server on, if any. */
|
||||
export type RecipeVmPreparation = { wokenPairingRevision: number | null }
|
||||
const UNCHANGED: RecipeVmPreparation = { wokenPairingRevision: null }
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { awaitInFlightWake, WorkspaceSleepRequests, type VmOperation } from './recipe-vm-operations'
|
||||
|
||||
describe('awaitInFlightWake', () => {
|
||||
it('answers at once for a VM that is up or asleep, without starting anything', async () => {
|
||||
const sleeps = new WorkspaceSleepRequests()
|
||||
await expect(awaitInFlightWake([], false, sleeps, 'w', 1000)).resolves.toBe('up')
|
||||
await expect(awaitInFlightWake([], true, sleeps, 'w', 1000)).resolves.toBe('asleep')
|
||||
})
|
||||
|
||||
it('waits for the wake already in flight', async () => {
|
||||
let finish = (): void => {}
|
||||
const settled = new Promise<void>((resolve) => {
|
||||
finish = resolve
|
||||
})
|
||||
const pending: VmOperation[] = [{ kind: 'wake', settled }]
|
||||
const answer = awaitInFlightWake(pending, false, new WorkspaceSleepRequests(), 'w', 5000)
|
||||
finish()
|
||||
await expect(answer).resolves.toBe('up')
|
||||
})
|
||||
|
||||
it('ends slept when a Sleep is asked for while it waits, and waking when the hold runs out', async () => {
|
||||
const sleeps = new WorkspaceSleepRequests()
|
||||
const never: VmOperation[] = [{ kind: 'wake', settled: new Promise(() => {}) }]
|
||||
const slept = awaitInFlightWake(never, false, sleeps, 'w', 5000)
|
||||
sleeps.noteRequested('w')
|
||||
await expect(slept).resolves.toBe('slept')
|
||||
await expect(awaitInFlightWake(never, false, sleeps, 'w', 5)).resolves.toBe('waking')
|
||||
})
|
||||
})
|
||||
@@ -124,3 +124,26 @@ export async function holdOpenUntilUp(
|
||||
stopAbortWatch()
|
||||
}
|
||||
}
|
||||
|
||||
/** Holds a request that does not open the workspace while the wake already in flight settles; a VM
|
||||
* that is not waking answers at once with where it stands. Never starts a wake. */
|
||||
export async function awaitInFlightWake(
|
||||
pending: readonly VmOperation[],
|
||||
recordSuspended: boolean,
|
||||
sleepRequests: WorkspaceSleepRequests,
|
||||
worktreeId: string,
|
||||
timeoutMs: number,
|
||||
signal?: AbortSignal
|
||||
): Promise<RecipeVmOpening | 'asleep'> {
|
||||
signal?.throwIfAborted()
|
||||
const state = vmStateOf(pending, recordSuspended)
|
||||
if (state !== 'waking') {
|
||||
return state === 'up' ? 'up' : 'asleep'
|
||||
}
|
||||
const sleep = sleepRequests.next(worktreeId)
|
||||
try {
|
||||
return await holdOpenUntilUp(pendingVmWake(pending), sleep.requested, timeoutMs, signal)
|
||||
} finally {
|
||||
sleep.stop()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,14 @@ export type RecipeVmWaker = {
|
||||
timeoutMs: number,
|
||||
signal?: AbortSignal
|
||||
) => Promise<RecipeVmOpening>
|
||||
/** Holds a request that does not open the workspace while a wake already in flight finishes;
|
||||
* never starts one. */
|
||||
awaitWake?: (
|
||||
workspaceId: string,
|
||||
recordSuspended: boolean,
|
||||
timeoutMs: number,
|
||||
signal?: AbortSignal
|
||||
) => Promise<RecipeVmOpening | 'asleep'>
|
||||
/** Holds an open of the workspace behind a saved environment (id or name) for the CLI. */
|
||||
openEnvironment: (selector: string) => Promise<RecipeVmOpening>
|
||||
}
|
||||
|
||||
@@ -161,6 +161,16 @@ export abstract class RuntimeWorkspaceSleepOperations extends OrcaRuntimeWithRes
|
||||
return this.workspaceSleep.holdOpen(worktreeId, recordSuspended, timeoutMs, signal)
|
||||
}
|
||||
|
||||
/** Holds a request that does not open this workspace while its in-flight wake finishes. */
|
||||
awaitWorkspaceVmWake(
|
||||
worktreeId: string,
|
||||
recordSuspended: boolean,
|
||||
timeoutMs: number,
|
||||
signal?: AbortSignal
|
||||
): Promise<RecipeVmOpening | 'asleep'> {
|
||||
return this.workspaceSleep.awaitWake(worktreeId, recordSuspended, timeoutMs, signal)
|
||||
}
|
||||
|
||||
/** The window opened a slept pane; main resumes it and the window attaches. */
|
||||
resumeSleptPane(pane: RestoredPane, automatic?: boolean): Promise<boolean> {
|
||||
return this.workspaceSleep.resumePane(pane, automatic)
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import {
|
||||
awaitInFlightWake,
|
||||
holdOpenUntilUp,
|
||||
pendingVmWake,
|
||||
trackPendingVmOperation,
|
||||
@@ -209,6 +210,20 @@ export class WorkspaceSleepCoordinator {
|
||||
}
|
||||
}
|
||||
|
||||
/** Holds a request that does not open the workspace while a wake already in flight finishes;
|
||||
* never starts a wake or counts as a reopening, so background traffic leaves a VM asleep. */
|
||||
awaitWake(worktreeId: string, recordSuspended: boolean, timeoutMs: number, signal?: AbortSignal) {
|
||||
const pending = this.pendingVmOperations.get(worktreeId) ?? []
|
||||
return awaitInFlightWake(
|
||||
pending,
|
||||
recordSuspended,
|
||||
this.sleepRequests,
|
||||
worktreeId,
|
||||
timeoutMs,
|
||||
signal
|
||||
)
|
||||
}
|
||||
|
||||
private trackVmOperation<T>(worktreeId: string, operation: VmOperation, run: Promise<T>) {
|
||||
return trackPendingVmOperation(this.pendingVmOperations, worktreeId, operation, run)
|
||||
}
|
||||
|
||||
@@ -166,6 +166,8 @@ export function initializeMainProcessRuntime(): OrcaRuntimeService {
|
||||
noteOpened: (workspaceId) => runtime.noteWorkspaceReopened(workspaceId),
|
||||
holdOpen: (workspaceId, suspended, timeoutMs, signal) =>
|
||||
runtime.holdWorkspaceVmOpen(workspaceId, suspended, timeoutMs, signal),
|
||||
awaitWake: (workspaceId, suspended, timeoutMs, signal) =>
|
||||
runtime.awaitWorkspaceVmWake(workspaceId, suspended, timeoutMs, signal),
|
||||
openEnvironment: (selector) => {
|
||||
const root = getRuntimeEnvironmentRegistryRoot()
|
||||
return openRecipeVmEnvironment(root, resolveEnvironment(root, selector))
|
||||
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
vi.mock('@/store/index', () => ({ useAppStore: { getState: () => ({}) } }))
|
||||
|
||||
describe('shouldContinueInterruptedTurn', () => {
|
||||
it('continues only a remote agent whose last status was working', async () => {
|
||||
const { shouldContinueInterruptedTurn } = await import('./cold-restore-resume-startup')
|
||||
expect(shouldContinueInterruptedTurn({ state: 'working' }, 'env-1')).toBe(true)
|
||||
expect(shouldContinueInterruptedTurn({ state: 'waiting' }, 'env-1')).toBe(false)
|
||||
expect(shouldContinueInterruptedTurn({ state: 'working' }, null)).toBe(false)
|
||||
expect(shouldContinueInterruptedTurn(null, 'env-1')).toBe(false)
|
||||
})
|
||||
})
|
||||
+22
-2
@@ -6,7 +6,10 @@ import { normalizeExecutionHostId } from '../../../../../../shared/execution-hos
|
||||
import { parseWorkspaceKey } from '../../../../../../shared/workspace-scope'
|
||||
import { useAppStore } from '@/store/index'
|
||||
import { createNonSecureContextUuid as createBrowserUuid } from '../../../../../../shared/non-secure-context-uuid'
|
||||
import { buildAgentResumeStartupPlan } from '../../../../../../shared/agent-launch/startup/tui-agent-resume-startup'
|
||||
import {
|
||||
INTERRUPTED_TURN_CONTINUE_PROMPT,
|
||||
buildAgentResumeStartupPlan
|
||||
} from '../../../../../../shared/agent-launch/startup/tui-agent-resume-startup'
|
||||
import {
|
||||
resolveRuntimeHostReportedPlatform,
|
||||
resolveRuntimeHostTerminalWindowsShell,
|
||||
@@ -31,6 +34,15 @@ import type { ConnectPanePtySession } from '../connect-pane-pty-session'
|
||||
|
||||
export const NOT_REOPENED: HeldPaneReopenOutcome = { reopened: false, agentResumed: false }
|
||||
|
||||
/** A cold-restored agent on a remote host whose last status was working lost its turn when the
|
||||
* host restarted; local panes are left alone (their PTYs outlive app restarts). */
|
||||
export function shouldContinueInterruptedTurn(
|
||||
entry: { state: string } | null | undefined,
|
||||
runtimeEnvironmentId: string | null
|
||||
): boolean {
|
||||
return Boolean(runtimeEnvironmentId) && entry?.state === 'working'
|
||||
}
|
||||
|
||||
type AppState = ReturnType<typeof useAppStore.getState>
|
||||
|
||||
/** The agent session a cold restore of this pane would resume; read-only, so pre-spawn checks can share it. */
|
||||
@@ -117,7 +129,15 @@ export function bindBuildColdRestoreAgentResumeStartup(session: ConnectPanePtySe
|
||||
platform: resumeTarget.platform,
|
||||
agentHostPlatform: resumeTarget.agentHostPlatform,
|
||||
shell: resumeTarget.shell,
|
||||
launchCwd: session.worktree?.path
|
||||
launchCwd: session.worktree?.path,
|
||||
// A remote host that restarted under a working agent (reboot, crash, VM rollover) killed it
|
||||
// mid-turn; its last status still says working. Carry the turn on instead of waiting.
|
||||
continuePrompt: shouldContinueInterruptedTurn(
|
||||
entry,
|
||||
session.transport.getRuntimeEnvironmentId?.() ?? null
|
||||
)
|
||||
? INTERRUPTED_TURN_CONTINUE_PROMPT
|
||||
: null
|
||||
})
|
||||
if (!startupPlan) {
|
||||
return null
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { buildAgentResumeStartupPlan } from './tui-agent-resume-startup'
|
||||
|
||||
const providerSession = { key: 'session_id', id: 'claude-session-1' } as const
|
||||
|
||||
describe('resume with a continue prompt', () => {
|
||||
const base = { cmdOverrides: {}, platform: 'linux' as const, shell: 'posix' as const }
|
||||
|
||||
it('submits the prompt on resume for Claude and Codex', () => {
|
||||
const claude = buildAgentResumeStartupPlan({
|
||||
...base,
|
||||
agent: 'claude',
|
||||
providerSession,
|
||||
continuePrompt: 'continue'
|
||||
})
|
||||
expect(claude?.launchCommand).toMatch(/'?--resume'? '?claude-session-1'? '?continue'?$/)
|
||||
const codex = buildAgentResumeStartupPlan({
|
||||
...base,
|
||||
agent: 'codex',
|
||||
providerSession: { key: 'session_id', id: 'codex-1' },
|
||||
continuePrompt: 'continue'
|
||||
})
|
||||
expect(codex?.launchCommand).toMatch(/'?resume'? '?codex-1'? '?continue'?$/)
|
||||
})
|
||||
|
||||
it('leaves other agents and empty prompts unchanged', () => {
|
||||
const plain = buildAgentResumeStartupPlan({ ...base, agent: 'claude', providerSession })
|
||||
const blank = buildAgentResumeStartupPlan({
|
||||
...base,
|
||||
agent: 'claude',
|
||||
providerSession,
|
||||
continuePrompt: ' '
|
||||
})
|
||||
expect(blank?.launchCommand).toBe(plain?.launchCommand)
|
||||
const cursor = buildAgentResumeStartupPlan({
|
||||
...base,
|
||||
agent: 'cursor',
|
||||
providerSession: { key: 'conversation_id', id: 'c-1' },
|
||||
continuePrompt: 'continue'
|
||||
})
|
||||
expect(cursor?.launchCommand).not.toMatch(/continue'?$/)
|
||||
})
|
||||
})
|
||||
@@ -37,6 +37,9 @@ export function buildAgentResumeStartupPlan(args: {
|
||||
fork?: boolean
|
||||
/** OS that runs the agent when `platform` only picks quoting; null when unreported. */
|
||||
agentHostPlatform?: NodeJS.Platform | null
|
||||
/** First message the resumed session submits, e.g. to carry on a turn its host cut off. Only
|
||||
* agents whose resume takes a prompt argument (Claude, Codex) send it; others ignore it. */
|
||||
continuePrompt?: string | null
|
||||
}): AgentStartupPlan | null {
|
||||
const resumeArgv = args.fork
|
||||
? getAgentForkArgv(args.agent, args.providerSession)
|
||||
@@ -45,10 +48,15 @@ export function buildAgentResumeStartupPlan(args: {
|
||||
return null
|
||||
}
|
||||
// Codex otherwise asks for its old folder, even if deleted (#17745). Keep before resume so argv removal preserves the folder choice.
|
||||
const argv =
|
||||
const withCwd =
|
||||
args.agent === 'codex' && args.resumeInLaunchCwd
|
||||
? [resumeArgv[0], '-c', 'tui.resume_cwd=current', ...resumeArgv.slice(1)]
|
||||
: resumeArgv
|
||||
const continuePrompt = args.continuePrompt?.trim()
|
||||
const argv =
|
||||
continuePrompt && !args.fork && RESUME_PROMPT_AGENTS.has(args.agent)
|
||||
? [...withCwd, continuePrompt]
|
||||
: withCwd
|
||||
const shell = resolveStartupShell(args.platform, args.shell)
|
||||
// Why: Codex restores a resumed thread's saved permissions unless the launch passes one, so
|
||||
// replaying Orca's launch-time flag would undo a change made inside the session (#10886).
|
||||
@@ -112,6 +120,12 @@ export function buildAgentResumeStartupPlan(args: {
|
||||
}
|
||||
}
|
||||
|
||||
/** Agents whose resume command accepts a positional prompt it submits on start. */
|
||||
const RESUME_PROMPT_AGENTS: ReadonlySet<ResumableTuiAgent> = new Set(['claude', 'codex'])
|
||||
|
||||
/** What a resumed agent is told when its host went away mid-turn (restart, reboot, crash). */
|
||||
export const INTERRUPTED_TURN_CONTINUE_PROMPT = 'continue'
|
||||
|
||||
/** WSL panes start at a UNC path while the agent reports its Linux path. */
|
||||
function isSameFolder(agentCwd: string, launchCwd: string): boolean {
|
||||
const launchPath = parseWslUncPath(launchCwd)?.linuxPath ?? launchCwd
|
||||
|
||||
Reference in New Issue
Block a user