diff --git a/src/main/hosted-sandbox/hosted-agent-auto-resume.ts b/src/main/hosted-sandbox/hosted-agent-auto-resume.ts index 2c1a1fd8e47..6637066ec92 100644 --- a/src/main/hosted-sandbox/hosted-agent-auto-resume.ts +++ b/src/main/hosted-sandbox/hosted-agent-auto-resume.ts @@ -24,6 +24,10 @@ const ContinueAnswer = z.looseObject({ skipped: z.array(z.unknown()).optional() }) +/** `recordedAt` is the host's clock and `drainStartedAt` this machine's: tolerate skew between + * them so an offer the drain just recorded is not mistaken for an older one. */ +export const CLOCK_SKEW_TOLERANCE_MS = 2 * 60_000 + function hasCode(error: unknown, code: string): boolean { return typeof error === 'object' && error !== null && 'code' in error && error.code === code } @@ -47,7 +51,9 @@ export async function autoResumeAgentsAfterRollover( } const ids = list.data.sessions.flatMap((row) => { const offer = Offer.safeParse(row) - return offer.success && offer.data.recordedAt >= drainStartedAt ? [offer.data.sessionId] : [] + return offer.success && offer.data.recordedAt >= drainStartedAt - CLOCK_SKEW_TOLERANCE_MS + ? [offer.data.sessionId] + : [] }) const outcome = { offered: ids.length, continued: 0, skipped: 0, unsupported: false } for (let offset = 0; offset < ids.length; offset += MAX_RESTART_RESUME_SESSIONS) { diff --git a/src/main/hosted-sandbox/hosted-session-lease.test.ts b/src/main/hosted-sandbox/hosted-session-lease.test.ts index ef018530cd6..3400ef8afee 100644 --- a/src/main/hosted-sandbox/hosted-session-lease.test.ts +++ b/src/main/hosted-sandbox/hosted-session-lease.test.ts @@ -112,14 +112,15 @@ describe('agent auto-resume after rollover', () => { if (method === 'agentSession.restartResumable') { return { sessions: [ - { sessionId: 'old', recordedAt: 900 }, - { sessionId: 'a', recordedAt: 1000 }, - { sessionId: 'b', recordedAt: 1500 } + { sessionId: 'old', recordedAt: T0 - 10 * MIN }, + // The host's clock runs 30 s behind this machine's: still this drain's offer. + { sessionId: 'a', recordedAt: T0 - 30_000 }, + { sessionId: 'b', recordedAt: T0 + 500 } ] } } return { continued: [{ sessionId: 'a' }], skipped: ['b'] } - }, 1000) + }, T0) expect(calls).toEqual([ { method: 'agentSession.restartResumable', params: undefined }, { method: 'agentSession.restartContinue', params: { sessionIds: ['a', 'b'] } } diff --git a/src/main/ipc/runtime-environments/runtime-environment-recipe-vm-wake.test.ts b/src/main/ipc/runtime-environments/runtime-environment-recipe-vm-wake.test.ts index 6e3c5b7f1bb..009fd9fbe21 100644 --- a/src/main/ipc/runtime-environments/runtime-environment-recipe-vm-wake.test.ts +++ b/src/main/ipc/runtime-environments/runtime-environment-recipe-vm-wake.test.ts @@ -131,6 +131,44 @@ describe('a request to a recipe VM environment', () => { } ) + it('holds a request that does not open the workspace while an in-flight wake finishes', async () => { + recordVm('running') + ownerState = 'waking' + const awaitWake = vi.fn(async () => 'up' as const) + registerRecipeVmWaker({ + state, + noteOpened: vi.fn(), + holdOpen, + awaitWake, + openEnvironment: async () => 'up' + }) + noteRecipeVmWakePairing(VM_ENVIRONMENT.id, 7, true) + await expect(prepareRecipeVm(userDataPath, VM_ENVIRONMENT, 'none')).resolves.toEqual({ + wokenPairingRevision: null + }) + expect(awaitWake).toHaveBeenCalledWith('workspace-of-vm-1', false, 60_000, undefined) + // It never starts a wake of its own. + expect(holdOpen).not.toHaveBeenCalled() + }) + + it.each([ + ['waking', 'runtime_environment_waking'], + ['wake-failed', 'runtime_environment_wake_failed'], + ['slept', 'runtime_environment_asleep'], + ['asleep', 'runtime_environment_asleep'] + ] as const)('refuses with %s when the in-flight wake ends that way', async (outcome, code) => { + recordVm('running') + ownerState = 'waking' + registerRecipeVmWaker({ + state, + noteOpened: vi.fn(), + holdOpen, + awaitWake: async () => outcome, + openEnvironment: async () => 'up' + }) + await expect(codeOf('none')).resolves.toBe(code) + }) + // A phone gives up after 30 seconds, so its open is held for less. // A Disconnect made before the open holds: no client's open wakes the VM behind that server. diff --git a/src/main/ipc/runtime-environments/runtime-environment-recipe-vm-wake.ts b/src/main/ipc/runtime-environments/runtime-environment-recipe-vm-wake.ts index f906a8612f9..e1dd906d203 100644 --- a/src/main/ipc/runtime-environments/runtime-environment-recipe-vm-wake.ts +++ b/src/main/ipc/runtime-environments/runtime-environment-recipe-vm-wake.ts @@ -111,12 +111,31 @@ export async function prepareRecipeVm( return { wokenPairingRevision: woken !== wakeBefore ? (woken?.pairingRevision ?? null) : null } } const state = waker.state(vm.workspaceId, vm.recordSuspended) + if (state === 'waking' && waker.awaitWake) { + // The wake is already underway (often just reconnecting): a terminal or panel opened in that + // window waits for it instead of failing and never retrying. Nothing here starts a wake. + const wakeBefore = latestRecipeVmWake(environment.id) + const outcome = await waker.awaitWake( + vm.workspaceId, + vm.recordSuspended, + Math.min(holdMs, RECIPE_VM_IN_FLIGHT_WAKE_HOLD_MS), + signal + ) + if (outcome !== 'up') { + throw refusal(outcome) + } + const woken = latestRecipeVmWake(environment.id) + return { wokenPairingRevision: woken !== wakeBefore ? (woken?.pairingRevision ?? null) : null } + } if (state !== 'up') { throw refusal(state) } return UNCHANGED } +/** How long a request that does not open the workspace waits for a wake already in flight. */ +export const RECIPE_VM_IN_FLIGHT_WAKE_HOLD_MS = 60_000 + /** The pairing revision a recipe VM wake this request waited for left its server on, if any. */ export type RecipeVmPreparation = { wokenPairingRevision: number | null } const UNCHANGED: RecipeVmPreparation = { wokenPairingRevision: null } diff --git a/src/main/runtime/workspaces/sleep/recipe-vm-in-flight-wake.test.ts b/src/main/runtime/workspaces/sleep/recipe-vm-in-flight-wake.test.ts new file mode 100644 index 00000000000..bcc6b8075ee --- /dev/null +++ b/src/main/runtime/workspaces/sleep/recipe-vm-in-flight-wake.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from 'vitest' +import { awaitInFlightWake, WorkspaceSleepRequests, type VmOperation } from './recipe-vm-operations' + +describe('awaitInFlightWake', () => { + it('answers at once for a VM that is up or asleep, without starting anything', async () => { + const sleeps = new WorkspaceSleepRequests() + await expect(awaitInFlightWake([], false, sleeps, 'w', 1000)).resolves.toBe('up') + await expect(awaitInFlightWake([], true, sleeps, 'w', 1000)).resolves.toBe('asleep') + }) + + it('waits for the wake already in flight', async () => { + let finish = (): void => {} + const settled = new Promise((resolve) => { + finish = resolve + }) + const pending: VmOperation[] = [{ kind: 'wake', settled }] + const answer = awaitInFlightWake(pending, false, new WorkspaceSleepRequests(), 'w', 5000) + finish() + await expect(answer).resolves.toBe('up') + }) + + it('ends slept when a Sleep is asked for while it waits, and waking when the hold runs out', async () => { + const sleeps = new WorkspaceSleepRequests() + const never: VmOperation[] = [{ kind: 'wake', settled: new Promise(() => {}) }] + const slept = awaitInFlightWake(never, false, sleeps, 'w', 5000) + sleeps.noteRequested('w') + await expect(slept).resolves.toBe('slept') + await expect(awaitInFlightWake(never, false, sleeps, 'w', 5)).resolves.toBe('waking') + }) +}) diff --git a/src/main/runtime/workspaces/sleep/recipe-vm-operations.ts b/src/main/runtime/workspaces/sleep/recipe-vm-operations.ts index 00305978ffd..1778f20ef98 100644 --- a/src/main/runtime/workspaces/sleep/recipe-vm-operations.ts +++ b/src/main/runtime/workspaces/sleep/recipe-vm-operations.ts @@ -124,3 +124,26 @@ export async function holdOpenUntilUp( stopAbortWatch() } } + +/** Holds a request that does not open the workspace while the wake already in flight settles; a VM + * that is not waking answers at once with where it stands. Never starts a wake. */ +export async function awaitInFlightWake( + pending: readonly VmOperation[], + recordSuspended: boolean, + sleepRequests: WorkspaceSleepRequests, + worktreeId: string, + timeoutMs: number, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const state = vmStateOf(pending, recordSuspended) + if (state !== 'waking') { + return state === 'up' ? 'up' : 'asleep' + } + const sleep = sleepRequests.next(worktreeId) + try { + return await holdOpenUntilUp(pendingVmWake(pending), sleep.requested, timeoutMs, signal) + } finally { + sleep.stop() + } +} diff --git a/src/main/runtime/workspaces/sleep/recipe-vm-waker-registry.ts b/src/main/runtime/workspaces/sleep/recipe-vm-waker-registry.ts index b2697ae0ecd..cead8b03c5e 100644 --- a/src/main/runtime/workspaces/sleep/recipe-vm-waker-registry.ts +++ b/src/main/runtime/workspaces/sleep/recipe-vm-waker-registry.ts @@ -15,6 +15,14 @@ export type RecipeVmWaker = { timeoutMs: number, signal?: AbortSignal ) => Promise + /** Holds a request that does not open the workspace while a wake already in flight finishes; + * never starts one. */ + awaitWake?: ( + workspaceId: string, + recordSuspended: boolean, + timeoutMs: number, + signal?: AbortSignal + ) => Promise /** Holds an open of the workspace behind a saved environment (id or name) for the CLI. */ openEnvironment: (selector: string) => Promise } diff --git a/src/main/runtime/workspaces/sleep/runtime-workspace-sleep-operations.ts b/src/main/runtime/workspaces/sleep/runtime-workspace-sleep-operations.ts index cb1a161ffd6..ee3b05ffc87 100644 --- a/src/main/runtime/workspaces/sleep/runtime-workspace-sleep-operations.ts +++ b/src/main/runtime/workspaces/sleep/runtime-workspace-sleep-operations.ts @@ -161,6 +161,16 @@ export abstract class RuntimeWorkspaceSleepOperations extends OrcaRuntimeWithRes return this.workspaceSleep.holdOpen(worktreeId, recordSuspended, timeoutMs, signal) } + /** Holds a request that does not open this workspace while its in-flight wake finishes. */ + awaitWorkspaceVmWake( + worktreeId: string, + recordSuspended: boolean, + timeoutMs: number, + signal?: AbortSignal + ): Promise { + return this.workspaceSleep.awaitWake(worktreeId, recordSuspended, timeoutMs, signal) + } + /** The window opened a slept pane; main resumes it and the window attaches. */ resumeSleptPane(pane: RestoredPane, automatic?: boolean): Promise { return this.workspaceSleep.resumePane(pane, automatic) diff --git a/src/main/runtime/workspaces/sleep/workspace-sleep-runtime.ts b/src/main/runtime/workspaces/sleep/workspace-sleep-runtime.ts index 8bd911b0526..cb30955f5d3 100644 --- a/src/main/runtime/workspaces/sleep/workspace-sleep-runtime.ts +++ b/src/main/runtime/workspaces/sleep/workspace-sleep-runtime.ts @@ -1,4 +1,5 @@ import { + awaitInFlightWake, holdOpenUntilUp, pendingVmWake, trackPendingVmOperation, @@ -209,6 +210,20 @@ export class WorkspaceSleepCoordinator { } } + /** Holds a request that does not open the workspace while a wake already in flight finishes; + * never starts a wake or counts as a reopening, so background traffic leaves a VM asleep. */ + awaitWake(worktreeId: string, recordSuspended: boolean, timeoutMs: number, signal?: AbortSignal) { + const pending = this.pendingVmOperations.get(worktreeId) ?? [] + return awaitInFlightWake( + pending, + recordSuspended, + this.sleepRequests, + worktreeId, + timeoutMs, + signal + ) + } + private trackVmOperation(worktreeId: string, operation: VmOperation, run: Promise) { return trackPendingVmOperation(this.pendingVmOperations, worktreeId, operation, run) } diff --git a/src/main/startup/main-process-runtime-service.ts b/src/main/startup/main-process-runtime-service.ts index 7b0405af786..2fc9e8f28f4 100644 --- a/src/main/startup/main-process-runtime-service.ts +++ b/src/main/startup/main-process-runtime-service.ts @@ -166,6 +166,8 @@ export function initializeMainProcessRuntime(): OrcaRuntimeService { noteOpened: (workspaceId) => runtime.noteWorkspaceReopened(workspaceId), holdOpen: (workspaceId, suspended, timeoutMs, signal) => runtime.holdWorkspaceVmOpen(workspaceId, suspended, timeoutMs, signal), + awaitWake: (workspaceId, suspended, timeoutMs, signal) => + runtime.awaitWorkspaceVmWake(workspaceId, suspended, timeoutMs, signal), openEnvironment: (selector) => { const root = getRuntimeEnvironmentRegistryRoot() return openRecipeVmEnvironment(root, resolveEnvironment(root, selector)) diff --git a/src/renderer/src/components/terminal-pane/pty-connection/startup/cold-restore-continue.test.ts b/src/renderer/src/components/terminal-pane/pty-connection/startup/cold-restore-continue.test.ts new file mode 100644 index 00000000000..a0487b3e0a2 --- /dev/null +++ b/src/renderer/src/components/terminal-pane/pty-connection/startup/cold-restore-continue.test.ts @@ -0,0 +1,13 @@ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('@/store/index', () => ({ useAppStore: { getState: () => ({}) } })) + +describe('shouldContinueInterruptedTurn', () => { + it('continues only a remote agent whose last status was working', async () => { + const { shouldContinueInterruptedTurn } = await import('./cold-restore-resume-startup') + expect(shouldContinueInterruptedTurn({ state: 'working' }, 'env-1')).toBe(true) + expect(shouldContinueInterruptedTurn({ state: 'waiting' }, 'env-1')).toBe(false) + expect(shouldContinueInterruptedTurn({ state: 'working' }, null)).toBe(false) + expect(shouldContinueInterruptedTurn(null, 'env-1')).toBe(false) + }) +}) diff --git a/src/renderer/src/components/terminal-pane/pty-connection/startup/cold-restore-resume-startup.ts b/src/renderer/src/components/terminal-pane/pty-connection/startup/cold-restore-resume-startup.ts index 983d4b9c96f..7abba5460db 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection/startup/cold-restore-resume-startup.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection/startup/cold-restore-resume-startup.ts @@ -6,7 +6,10 @@ import { normalizeExecutionHostId } from '../../../../../../shared/execution-hos import { parseWorkspaceKey } from '../../../../../../shared/workspace-scope' import { useAppStore } from '@/store/index' import { createNonSecureContextUuid as createBrowserUuid } from '../../../../../../shared/non-secure-context-uuid' -import { buildAgentResumeStartupPlan } from '../../../../../../shared/agent-launch/startup/tui-agent-resume-startup' +import { + INTERRUPTED_TURN_CONTINUE_PROMPT, + buildAgentResumeStartupPlan +} from '../../../../../../shared/agent-launch/startup/tui-agent-resume-startup' import { resolveRuntimeHostReportedPlatform, resolveRuntimeHostTerminalWindowsShell, @@ -31,6 +34,15 @@ import type { ConnectPanePtySession } from '../connect-pane-pty-session' export const NOT_REOPENED: HeldPaneReopenOutcome = { reopened: false, agentResumed: false } +/** A cold-restored agent on a remote host whose last status was working lost its turn when the + * host restarted; local panes are left alone (their PTYs outlive app restarts). */ +export function shouldContinueInterruptedTurn( + entry: { state: string } | null | undefined, + runtimeEnvironmentId: string | null +): boolean { + return Boolean(runtimeEnvironmentId) && entry?.state === 'working' +} + type AppState = ReturnType /** The agent session a cold restore of this pane would resume; read-only, so pre-spawn checks can share it. */ @@ -117,7 +129,15 @@ export function bindBuildColdRestoreAgentResumeStartup(session: ConnectPanePtySe platform: resumeTarget.platform, agentHostPlatform: resumeTarget.agentHostPlatform, shell: resumeTarget.shell, - launchCwd: session.worktree?.path + launchCwd: session.worktree?.path, + // A remote host that restarted under a working agent (reboot, crash, VM rollover) killed it + // mid-turn; its last status still says working. Carry the turn on instead of waiting. + continuePrompt: shouldContinueInterruptedTurn( + entry, + session.transport.getRuntimeEnvironmentId?.() ?? null + ) + ? INTERRUPTED_TURN_CONTINUE_PROMPT + : null }) if (!startupPlan) { return null diff --git a/src/shared/agent-launch/startup/tui-agent-resume-continue.test.ts b/src/shared/agent-launch/startup/tui-agent-resume-continue.test.ts new file mode 100644 index 00000000000..c8bb1747775 --- /dev/null +++ b/src/shared/agent-launch/startup/tui-agent-resume-continue.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from 'vitest' +import { buildAgentResumeStartupPlan } from './tui-agent-resume-startup' + +const providerSession = { key: 'session_id', id: 'claude-session-1' } as const + +describe('resume with a continue prompt', () => { + const base = { cmdOverrides: {}, platform: 'linux' as const, shell: 'posix' as const } + + it('submits the prompt on resume for Claude and Codex', () => { + const claude = buildAgentResumeStartupPlan({ + ...base, + agent: 'claude', + providerSession, + continuePrompt: 'continue' + }) + expect(claude?.launchCommand).toMatch(/'?--resume'? '?claude-session-1'? '?continue'?$/) + const codex = buildAgentResumeStartupPlan({ + ...base, + agent: 'codex', + providerSession: { key: 'session_id', id: 'codex-1' }, + continuePrompt: 'continue' + }) + expect(codex?.launchCommand).toMatch(/'?resume'? '?codex-1'? '?continue'?$/) + }) + + it('leaves other agents and empty prompts unchanged', () => { + const plain = buildAgentResumeStartupPlan({ ...base, agent: 'claude', providerSession }) + const blank = buildAgentResumeStartupPlan({ + ...base, + agent: 'claude', + providerSession, + continuePrompt: ' ' + }) + expect(blank?.launchCommand).toBe(plain?.launchCommand) + const cursor = buildAgentResumeStartupPlan({ + ...base, + agent: 'cursor', + providerSession: { key: 'conversation_id', id: 'c-1' }, + continuePrompt: 'continue' + }) + expect(cursor?.launchCommand).not.toMatch(/continue'?$/) + }) +}) diff --git a/src/shared/agent-launch/startup/tui-agent-resume-startup.ts b/src/shared/agent-launch/startup/tui-agent-resume-startup.ts index 3f981d79664..a1f61d88365 100644 --- a/src/shared/agent-launch/startup/tui-agent-resume-startup.ts +++ b/src/shared/agent-launch/startup/tui-agent-resume-startup.ts @@ -37,6 +37,9 @@ export function buildAgentResumeStartupPlan(args: { fork?: boolean /** OS that runs the agent when `platform` only picks quoting; null when unreported. */ agentHostPlatform?: NodeJS.Platform | null + /** First message the resumed session submits, e.g. to carry on a turn its host cut off. Only + * agents whose resume takes a prompt argument (Claude, Codex) send it; others ignore it. */ + continuePrompt?: string | null }): AgentStartupPlan | null { const resumeArgv = args.fork ? getAgentForkArgv(args.agent, args.providerSession) @@ -45,10 +48,15 @@ export function buildAgentResumeStartupPlan(args: { return null } // Codex otherwise asks for its old folder, even if deleted (#17745). Keep before resume so argv removal preserves the folder choice. - const argv = + const withCwd = args.agent === 'codex' && args.resumeInLaunchCwd ? [resumeArgv[0], '-c', 'tui.resume_cwd=current', ...resumeArgv.slice(1)] : resumeArgv + const continuePrompt = args.continuePrompt?.trim() + const argv = + continuePrompt && !args.fork && RESUME_PROMPT_AGENTS.has(args.agent) + ? [...withCwd, continuePrompt] + : withCwd const shell = resolveStartupShell(args.platform, args.shell) // Why: Codex restores a resumed thread's saved permissions unless the launch passes one, so // replaying Orca's launch-time flag would undo a change made inside the session (#10886). @@ -112,6 +120,12 @@ export function buildAgentResumeStartupPlan(args: { } } +/** Agents whose resume command accepts a positional prompt it submits on start. */ +const RESUME_PROMPT_AGENTS: ReadonlySet = new Set(['claude', 'codex']) + +/** What a resumed agent is told when its host went away mid-turn (restart, reboot, crash). */ +export const INTERRUPTED_TURN_CONTINUE_PROMPT = 'continue' + /** WSL panes start at a UNC path while the agent reports its Linux path. */ function isSameFolder(agentCwd: string, launchCwd: string): boolean { const launchPath = parseWslUncPath(launchCwd)?.linuxPath ?? launchCwd