diff --git a/src/main/hosted-sandbox/hosted-agent-auto-resume.ts b/src/main/hosted-sandbox/hosted-agent-auto-resume.ts index 2c1a1fd8e47..fced9e50cd1 100644 --- a/src/main/hosted-sandbox/hosted-agent-auto-resume.ts +++ b/src/main/hosted-sandbox/hosted-agent-auto-resume.ts @@ -24,13 +24,51 @@ const ContinueAnswer = z.looseObject({ skipped: z.array(z.unknown()).optional() }) +/** `recordedAt` is the host's clock and `drainStartedAt` this machine's: tolerate skew between + * them so an offer the drain just recorded is not mistaken for an older one. */ +export const CLOCK_SKEW_TOLERANCE_MS = 2 * 60_000 +/** A host too slow to list its offers before a restart is treated as unable to (falls back to time). */ +const OFFER_SNAPSHOT_TIMEOUT_MS = 5_000 + function hasCode(error: unknown, code: string): boolean { return typeof error === 'object' && error !== null && 'code' in error && error.code === code } +/** Offer ids the host already holds before a drain; offers in it were left for the user. Null when + * the host cannot answer (an unresponsive host is often why it is being restarted). */ +export async function listRestartOfferIds(call: RuntimeCall): Promise | null> { + try { + let timer: ReturnType | undefined + const listed = await Promise.race([ + call('agentSession.restartResumable'), + new Promise((resolve) => { + timer = setTimeout(() => resolve(null), OFFER_SNAPSHOT_TIMEOUT_MS) + }) + ]).finally(() => clearTimeout(timer)) + const list = OfferList.safeParse(listed) + if (!list.success) { + return null + } + return new Set( + list.data.sessions.flatMap((row) => { + const offer = Offer.safeParse(row) + return offer.success ? [offer.data.sessionId] : [] + }) + ) + } catch { + return null + } +} + +/** + * Continues the offers this drain recorded. With a pre-drain snapshot of the host's offers, "recorded + * by this drain" is set membership (no clock comparison); without one it falls back to the host + * timestamp against this machine's drain start, allowing for clock skew. + */ export async function autoResumeAgentsAfterRollover( call: RuntimeCall, - drainStartedAt: number + drainStartedAt: number, + offersBeforeDrain: ReadonlySet | null = null ): Promise { let listed: unknown try { @@ -47,7 +85,13 @@ export async function autoResumeAgentsAfterRollover( } const ids = list.data.sessions.flatMap((row) => { const offer = Offer.safeParse(row) - return offer.success && offer.data.recordedAt >= drainStartedAt ? [offer.data.sessionId] : [] + if (!offer.success) { + return [] + } + const recordedByThisDrain = offersBeforeDrain + ? !offersBeforeDrain.has(offer.data.sessionId) + : offer.data.recordedAt >= drainStartedAt - CLOCK_SKEW_TOLERANCE_MS + return recordedByThisDrain ? [offer.data.sessionId] : [] }) const outcome = { offered: ids.length, continued: 0, skipped: 0, unsupported: false } for (let offset = 0; offset < ids.length; offset += MAX_RESTART_RESUME_SESSIONS) { diff --git a/src/main/hosted-sandbox/hosted-session-lease-scheduler.ts b/src/main/hosted-sandbox/hosted-session-lease-scheduler.ts index c1919a26ae3..f300a2b3c35 100644 --- a/src/main/hosted-sandbox/hosted-session-lease-scheduler.ts +++ b/src/main/hosted-sandbox/hosted-session-lease-scheduler.ts @@ -24,7 +24,12 @@ export type HostedSessionLeasePorts = { warn: (minutesLeft: HostedLeaseWarning) => void /** Re-pairs with the rebooted orcad; resolves once the runtime answers again. */ reconnect: (ready: HostedEnvironmentReady) => Promise - resumeAgents: (drainStartedAt: number) => Promise + resumeAgents: ( + drainStartedAt: number, + offersBeforeDrain: ReadonlySet | null + ) => Promise + /** The host's restart offers before the drain, so only the drain's own are continued. */ + snapshotOffers?: () => Promise | null> onSuspended: () => void /** Replaces the driver suspend, e.g. to run the window's ordinary Sleep for the workspace. */ suspend?: () => Promise @@ -158,6 +163,7 @@ export class HostedSessionLeaseScheduler { private async rollover(reason: 'opportunistic' | 'forced'): Promise { const { driver, handle } = this.ports + const offersBeforeDrain = (await this.ports.snapshotOffers?.().catch(() => null)) ?? null const drainStartedAt = this.now() this.log(`[orca-cloud] ${reason} rollover`) this.ports.onRolloverStateChange?.('restarting') @@ -172,7 +178,7 @@ export class HostedSessionLeaseScheduler { } this.ports.onRolloverStateChange?.('running') try { - const outcome = await this.ports.resumeAgents(drainStartedAt) + const outcome = await this.ports.resumeAgents(drainStartedAt, offersBeforeDrain) this.log( `[orca-cloud] continued ${outcome.continued}/${outcome.offered} agents after rollover` ) diff --git a/src/main/hosted-sandbox/hosted-session-lease.test.ts b/src/main/hosted-sandbox/hosted-session-lease.test.ts index 087dc8d1dcc..6c7a4c69b8d 100644 --- a/src/main/hosted-sandbox/hosted-session-lease.test.ts +++ b/src/main/hosted-sandbox/hosted-session-lease.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from 'vitest' import type { HostedEnvironmentReady } from './hosted-environment-provider' -import { autoResumeAgentsAfterRollover } from './hosted-agent-auto-resume' +import { autoResumeAgentsAfterRollover, listRestartOfferIds } from './hosted-agent-auto-resume' import { createHostedSandboxControlPlaneClient } from './hosted-sandbox-control-plane-client' import { createHostedSandboxDriver, HOSTED_SANDBOX_CAPABILITIES } from './hosted-sandbox-driver' import { decideHostedLeaseAction, type HostedActivitySample } from './hosted-session-lease-policy' @@ -102,7 +102,7 @@ describe('hosted session lease scheduler', () => { expect(await scheduler.tick()).toEqual({ kind: 'rollover', reason: 'forced' }) expect(reconnect).toHaveBeenCalledTimes(1) expect(reconnect.mock.calls[0]?.[0].session.startedAt).toBe(now) - expect(resumeAgents).toHaveBeenCalledWith(created.session.deadlineAt) + expect(resumeAgents).toHaveBeenCalledWith(created.session.deadlineAt, null) // The new session is far from its cap, so the next tick does nothing drastic. expect((await scheduler.tick()).kind).not.toBe('rollover') }) @@ -150,14 +150,15 @@ describe('agent auto-resume after rollover', () => { if (method === 'agentSession.restartResumable') { return { sessions: [ - { sessionId: 'old', recordedAt: 900 }, - { sessionId: 'a', recordedAt: 1000 }, - { sessionId: 'b', recordedAt: 1500 } + { sessionId: 'old', recordedAt: T0 - 10 * MIN }, + // The host's clock runs 30 s behind this machine's: still this drain's offer. + { sessionId: 'a', recordedAt: T0 - 30_000 }, + { sessionId: 'b', recordedAt: T0 + 500 } ] } } return { continued: [{ sessionId: 'a' }], skipped: ['b'] } - }, 1000) + }, T0) expect(calls).toEqual([ { method: 'agentSession.restartResumable', params: undefined }, { method: 'agentSession.restartContinue', params: { sessionIds: ['a', 'b'] } } @@ -178,3 +179,33 @@ describe('agent auto-resume after rollover', () => { expect(call).toHaveBeenCalledTimes(1) }) }) + +describe('agent auto-resume with a pre-drain offer snapshot', () => { + it('continues only offers the host did not already hold, whatever the clocks say', async () => { + const offers = [ + { sessionId: 'left-for-user', recordedAt: T0 + 5_000 }, + { sessionId: 'this-drain', recordedAt: T0 - 9 * MIN } + ] + const calls: unknown[] = [] + const call = async (method: string, params?: Record) => { + calls.push({ method, params }) + return method === 'agentSession.restartResumable' ? { sessions: offers } : { continued: [{}] } + } + const before = await listRestartOfferIds(async () => ({ sessions: [offers[0]] })) + expect(before).toEqual(new Set(['left-for-user'])) + const outcome = await autoResumeAgentsAfterRollover(call, T0, before) + expect(calls).toContainEqual({ + method: 'agentSession.restartContinue', + params: { sessionIds: ['this-drain'] } + }) + expect(outcome.offered).toBe(1) + }) + + it('answers null when the host cannot list its offers', async () => { + await expect( + listRestartOfferIds(async () => { + throw new Error('down') + }) + ).resolves.toBeNull() + }) +}) diff --git a/src/main/hosted-sandbox/hosted-session-registry.ts b/src/main/hosted-sandbox/hosted-session-registry.ts index 59e7d4b8332..9150eb10afe 100644 --- a/src/main/hosted-sandbox/hosted-session-registry.ts +++ b/src/main/hosted-sandbox/hosted-session-registry.ts @@ -21,7 +21,7 @@ import { noteRecipeVmWakePairing } from '../ephemeral-vm/recipe-vm-wake-pairings import { getEphemeralVmRecipeResultPairingCode } from '../../shared/ephemeral-vm-recipes' import { environmentHandleFromRecipeResult } from '../environment-providers/environment-provider-recipe-adapter' import type { HostedEnvironmentProviderDriver } from './hosted-environment-provider' -import { autoResumeAgentsAfterRollover } from './hosted-agent-auto-resume' +import { autoResumeAgentsAfterRollover, listRestartOfferIds } from './hosted-agent-auto-resume' import { HOSTED_SANDBOX_DRIVER_ID } from './hosted-sandbox-driver' import { createHostedRuntimeEnvironmentCall } from './hosted-runtime-environment-call' import type { HostedActivitySample } from './hosted-session-lease-policy' @@ -137,7 +137,9 @@ export function createHostedSessionRegistry(deps: { true ) }, - resumeAgents: (drainStartedAt) => autoResumeAgentsAfterRollover(call, drainStartedAt), + snapshotOffers: () => listRestartOfferIds(call), + resumeAgents: (drainStartedAt, offersBeforeDrain) => + autoResumeAgentsAfterRollover(call, drainStartedAt, offersBeforeDrain), onRolloverStateChange: (state) => { const event: HostedSessionStateEvent = { workspaceId: runtime.workspaceId ?? null, state } broadcast('orcaCloud:sessionState', event) diff --git a/src/main/hosted-sandbox/hosted-workspace-runtime.ts b/src/main/hosted-sandbox/hosted-workspace-runtime.ts index 931348febc4..b786880fc3e 100644 --- a/src/main/hosted-sandbox/hosted-workspace-runtime.ts +++ b/src/main/hosted-sandbox/hosted-workspace-runtime.ts @@ -5,6 +5,7 @@ import type { HostedSandboxControlPlaneClient } from './hosted-sandbox-control-p import type { EphemeralVmRuntimeRecord } from '../../shared/ephemeral-vm-runtimes' import type { CloudOomEvent, + CloudWakeProgressEvent, CloudWorkspaceErrorCode, CloudWorkspaceResult } from '../../shared/hosted-sandbox/cloud-workspace-api' @@ -18,6 +19,7 @@ import { } from '../environment-providers/environment-provider-driver' import { environmentHandleFromRecipeResult } from '../environment-providers/environment-provider-recipe-adapter' import { isHostedEntitlementDenied } from './hosted-environment-provider' +import { callRuntimeEnvironment } from '../ipc/runtime-environments/runtime-environment-transport-routing' import { HOSTED_SANDBOX_DRIVER_ID } from './hosted-sandbox-driver' import type { HostedWorkspaceService } from './hosted-workspace-service' @@ -165,6 +167,83 @@ export function getHostedWorkspaceService(): HostedWorkspaceService | null { return current } +/** Polls a cheap terminal listing until the woken server answers it (or the deadline passes). */ +export async function waitForTerminalsReachable( + userDataPath: string, + environmentId: string, + deadline: number, + stillWanted: () => boolean = () => true +): Promise { + while (Date.now() < deadline && stillWanted()) { + const remaining = deadline - Date.now() + // Bounded by the deadline: the probe neither waits past it on a wake in flight nor on a slow server. + const signal = AbortSignal.timeout(Math.max(1, remaining)) + const response = await callRuntimeEnvironment( + userDataPath, + environmentId, + 'terminal.list', + { limit: 1 }, + Math.min(5_000, Math.max(1, remaining)), + undefined, + undefined, + { signal, preparation: { recipeVmHoldMs: Math.max(0, remaining) } } + ).catch(() => null) + if (response?.ok) { + return stillWanted() + } + await new Promise((resolve) => setTimeout(resolve, 250)) + } + return false +} + +/** + * After a wake's step answers: `ready` once the runtime record says running AND the woken server + * answers a terminal listing (panes rebind to the restarted server's terminals a moment after + * running; keys typed before that are dropped by design). `failed` when it never answers. Nothing + * when this wake is no longer the current one (a newer wake, a failure or a Sleep took over). + */ +export async function announceWakeUsable( + userDataPath: string, + workspaceId: string, + startedAt: number, + hooks: { isCurrent: () => boolean; emit: (event: Omit) => void }, + timeoutMs = 60_000 +): Promise { + const deadline = Date.now() + timeoutMs + const running = (): EphemeralVmRuntimeRecord | null => { + const runtime = findCloudRuntime(userDataPath, workspaceId)?.runtime + return runtime?.status === 'running' ? runtime : null + } + while (Date.now() < deadline && hooks.isCurrent()) { + const runtime = running() + if (runtime) { + const reachable = runtime.runtimeEnvironmentId + ? await waitForTerminalsReachable( + userDataPath, + runtime.runtimeEnvironmentId, + deadline, + hooks.isCurrent + ) + : true + if (hooks.isCurrent() && running()) { + hooks.emit( + reachable + ? { workspaceId, phase: 'ready', source: 'open', startedAt } + : { + workspaceId, + phase: 'failed', + source: 'open', + startedAt, + message: 'The workspace did not answer after waking.' + } + ) + } + return + } + await new Promise((resolve) => setTimeout(resolve, 250)) + } +} + /** Whether the control plane says this workspace is archived; unknown counts as not archived. */ export async function isArchivedOnControlPlane( userDataPath: string, @@ -185,7 +264,9 @@ export async function isArchivedOnControlPlane( let archiveCheck: ((workspaceId: string) => Promise) | null = null /** Installed with Orca Cloud; the wake path asks it before resuming a cloud workspace. */ -export function setCloudArchiveCheck(check: ((workspaceId: string) => Promise) | null): void { +export function setCloudArchiveCheck( + check: ((workspaceId: string) => Promise) | null +): void { archiveCheck = check } /** False when Orca Cloud is off in this build. */ diff --git a/src/main/hosted-sandbox/hosted-workspace-service.ts b/src/main/hosted-sandbox/hosted-workspace-service.ts index 0acdc98fcb6..5a86ccb93e8 100644 --- a/src/main/hosted-sandbox/hosted-workspace-service.ts +++ b/src/main/hosted-sandbox/hosted-workspace-service.ts @@ -18,7 +18,7 @@ import type { import { applyResumedRuntimePairing } from '../ephemeral-vm/runtime-resume-pairing' import { noteRecipeVmWakePairing } from '../ephemeral-vm/recipe-vm-wake-pairings' import { invalidateRuntimeEnvironmentTransport } from '../ipc/runtime-environments/runtime-environments' -import { autoResumeAgentsAfterRollover } from './hosted-agent-auto-resume' +import { autoResumeAgentsAfterRollover, listRestartOfferIds } from './hosted-agent-auto-resume' import type { HostedSandboxControlPlaneClient } from './hosted-sandbox-control-plane-client' import { toReady } from './hosted-sandbox-driver' import { createHostedRuntimeEnvironmentCall } from './hosted-runtime-environment-call' @@ -28,6 +28,7 @@ import { failure, findCloudRuntime, readCachedCheckouts, + announceWakeUsable, type CloudRuntime } from './hosted-workspace-runtime' @@ -95,17 +96,11 @@ export function installHostedWorkspaceService(deps: { } /** The step answers before the window re-pairs; `ready` fires once the runtime record says running. */ - async function announceReadyWhenRunning(workspaceId: string, startedAt: number): Promise { - const deadline = Date.now() + 60_000 - while (Date.now() < deadline && wakes.has(workspaceId)) { - const runtime = findCloudRuntime(deps.userDataPath, workspaceId)?.runtime - if (runtime?.status === 'running') { - service.emitWakeProgress({ workspaceId, phase: 'ready', source: 'open', startedAt }) - return - } - await new Promise((resolve) => setTimeout(resolve, 250)) - } - } + const announceReadyWhenRunning = (workspaceId: string, startedAt: number): Promise => + announceWakeUsable(deps.userDataPath, workspaceId, startedAt, { + isCurrent: () => wakes.get(workspaceId)?.startedAt === startedAt, + emit: (event) => service.emitWakeProgress(event) + }) const service: HostedWorkspaceService = { emitWakeProgress(event) { @@ -213,6 +208,11 @@ export function installHostedWorkspaceService(deps: { source: 'restart', startedAt }) + const call = found.runtime.runtimeEnvironmentId + ? createHostedRuntimeEnvironmentCall(deps.userDataPath, found.runtime.runtimeEnvironmentId) + : null + // Offers the host already holds were left for the user; only this restart's are continued. + const offersBeforeDrain = call ? await listRestartOfferIds(call) : null // The scheduler holds the old session's deadline and would warn/roll over the new one; a fresh one starts on sync. deps.stopSession(found.sandboxId) try { @@ -231,14 +231,10 @@ export function installHostedWorkspaceService(deps: { source: 'restart', startedAt }) - if (found.runtime.runtimeEnvironmentId) { + if (call) { // Same continuation as a rollover: chats the restart interrupted carry on by themselves. - const call = createHostedRuntimeEnvironmentCall( - deps.userDataPath, - found.runtime.runtimeEnvironmentId - ) - void autoResumeAgentsAfterRollover(call, startedAt).catch((error: unknown) => - log(`[orca-cloud] auto-resume after restart failed: ${String(error)}`) + void autoResumeAgentsAfterRollover(call, startedAt, offersBeforeDrain).catch( + (error: unknown) => log(`[orca-cloud] auto-resume after restart failed: ${String(error)}`) ) } return { ok: true, value: null } diff --git a/src/main/hosted-sandbox/hosted-workspace-wake-ready.test.ts b/src/main/hosted-sandbox/hosted-workspace-wake-ready.test.ts new file mode 100644 index 00000000000..73a810d260f --- /dev/null +++ b/src/main/hosted-sandbox/hosted-workspace-wake-ready.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it, vi } from 'vitest' + +const runtimes: { status: string; runtimeEnvironmentId?: string }[] = [] +let listOk = true +vi.mock('../ipc/runtime-environments/runtime-environment-transport-routing', () => ({ + callRuntimeEnvironment: vi.fn(async () => + listOk ? { ok: true, result: {} } : { ok: false, error: { code: 'x', message: 'x' } } + ) +})) +vi.mock('../../shared/ephemeral-vm-runtime-store', () => ({ + listEphemeralVmRuntimes: () => + runtimes.map((r) => ({ + ...r, + workspaceId: 'ws', + recipe: { provider: 'orca-cloud' }, + recipeResult: { + schemaVersion: 1, + pairingCode: 'x', + projectRoot: '/p', + orcaCloud: { sandboxId: 'sbx_1' } + } + })) +})) +vi.mock('../environment-providers/environment-provider-recipe-adapter', () => ({ + environmentHandleFromRecipeResult: () => ({ driverState: { sandboxId: 'sbx_1' } }) +})) + +describe('announceWakeUsable', () => { + it('emits ready once running and reachable', async () => { + const { announceWakeUsable } = await import('./hosted-workspace-runtime') + runtimes.splice(0, runtimes.length, { status: 'running', runtimeEnvironmentId: 'env' }) + listOk = true + const emit = vi.fn() + await announceWakeUsable('/u', 'ws', 1, { isCurrent: () => true, emit }, 2_000) + expect(emit).toHaveBeenCalledWith(expect.objectContaining({ phase: 'ready', startedAt: 1 })) + }) + + it('emits failed when the server never answers, and nothing for a superseded wake', async () => { + const { announceWakeUsable } = await import('./hosted-workspace-runtime') + runtimes.splice(0, runtimes.length, { status: 'running', runtimeEnvironmentId: 'env' }) + listOk = false + const emit = vi.fn() + await announceWakeUsable('/u', 'ws', 2, { isCurrent: () => true, emit }, 600) + expect(emit).toHaveBeenCalledWith(expect.objectContaining({ phase: 'failed' })) + const stale = vi.fn() + await announceWakeUsable('/u', 'ws', 3, { isCurrent: () => false, emit: stale }, 600) + expect(stale).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/ipc/runtime-environments/runtime-environment-recipe-vm-wake.test.ts b/src/main/ipc/runtime-environments/runtime-environment-recipe-vm-wake.test.ts index 6e3c5b7f1bb..13f7642f191 100644 --- a/src/main/ipc/runtime-environments/runtime-environment-recipe-vm-wake.test.ts +++ b/src/main/ipc/runtime-environments/runtime-environment-recipe-vm-wake.test.ts @@ -13,7 +13,7 @@ import type { RecipeVmOpening, RecipeVmState } from '../../runtime/workspaces/sleep/recipe-vm-operations' -import { prepareRecipeVm } from './runtime-environment-recipe-vm-wake' +import { holdForInFlightRecipeVmWake, prepareRecipeVm } from './runtime-environment-recipe-vm-wake' import { RemoteRuntimeClientError } from '../../../shared/remote-runtime/remote-runtime-client-error' import { forgetRecipeVmWakePairingsForTests, @@ -131,6 +131,65 @@ describe('a request to a recipe VM environment', () => { } ) + it('holds a request that does not open the workspace for an in-flight wake, outside its call slot', async () => { + recordVm('running') + ownerState = 'waking' + const awaitWake = vi.fn(async () => 'up' as const) + registerRecipeVmWaker({ + state, + noteOpened: vi.fn(), + holdOpen, + awaitWake, + openEnvironment: async () => 'up' + }) + await holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'terminal.list', 5_000) + expect(awaitWake).toHaveBeenCalledWith('workspace-of-vm-1', false, 5_000, undefined) + // Bounded by the caller's hold (0 = never wait) and never for an open (that wakes in its slot). + await holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'terminal.list', 5_000, { + preparation: { recipeVmHoldMs: 0 } + }) + await holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'worktree.activate', 5_000) + expect(awaitWake).toHaveBeenCalledTimes(1) + expect(holdOpen).not.toHaveBeenCalled() + // Inside the slot the request is still answered at once from the owner's state. + await expect(codeOf('none')).resolves.toBe('runtime_environment_waking') + }) + + it('caps the hold at the in-flight maximum and does not wait on a VM that is not waking', async () => { + recordVm('running') + const awaitWake = vi.fn(async () => 'up' as const) + registerRecipeVmWaker({ + state, + noteOpened: vi.fn(), + holdOpen, + awaitWake, + openEnvironment: async () => 'up' + }) + await holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'terminal.list', 600_000) + expect(awaitWake).not.toHaveBeenCalled() + ownerState = 'waking' + await holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'terminal.list', 600_000) + expect(awaitWake).toHaveBeenCalledWith('workspace-of-vm-1', false, 60_000, undefined) + }) + + it('reports the pairing a wake it waited through left, for the caller fence', async () => { + recordVm('running') + ownerState = 'waking' + registerRecipeVmWaker({ + state, + noteOpened: vi.fn(), + holdOpen, + awaitWake: async () => { + noteRecipeVmWakePairing(VM_ENVIRONMENT.id, 9, true) + return 'up' as const + }, + openEnvironment: async () => 'up' + }) + await expect( + holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'terminal.list', 5_000) + ).resolves.toEqual({ before: VM_ENVIRONMENT, wokenPairingRevision: 9 }) + }) + // A phone gives up after 30 seconds, so its open is held for less. // A Disconnect made before the open holds: no client's open wakes the VM behind that server. diff --git a/src/main/ipc/runtime-environments/runtime-environment-recipe-vm-wake.ts b/src/main/ipc/runtime-environments/runtime-environment-recipe-vm-wake.ts index f906a8612f9..dda73c5db0c 100644 --- a/src/main/ipc/runtime-environments/runtime-environment-recipe-vm-wake.ts +++ b/src/main/ipc/runtime-environments/runtime-environment-recipe-vm-wake.ts @@ -14,6 +14,7 @@ import { RUNTIME_ENVIRONMENT_WAKE_FAILED_MESSAGE, RUNTIME_ENVIRONMENT_WAKING_ERROR_CODE, RUNTIME_ENVIRONMENT_WAKING_MESSAGE, + recipeVmWakeIntentFor, type RecipeVmWakeIntent } from '../../../shared/remote-runtime/recipe-vm-wake' import { getRecipeVmWaker } from '../../runtime/workspaces/sleep/recipe-vm-waker-registry' @@ -117,6 +118,46 @@ export async function prepareRecipeVm( return UNCHANGED } +/** How long a request that does not open the workspace waits for a wake already in flight. */ +export const RECIPE_VM_IN_FLIGHT_WAKE_HOLD_MS = 60_000 + +/** + * Before a request that does not open the workspace takes its call slot: while the VM's wake is + * already underway (often just reconnecting), wait for it, up to `holdMs`, instead of being refused + * and never retried. Never starts a wake and never refuses: the request's own preparation decides + * what happens next (it is refused there if the wake failed or is still running). + */ +export async function awaitRecipeVmInFlightWake( + userDataPath: string, + environment: E, + holdMs: number, + signal?: AbortSignal +): Promise | null> { + const waker = getRecipeVmWaker() + if (holdMs <= 0 || !waker?.awaitWake) { + return null + } + const vm = recipeVmOf(userDataPath, environment) + if (!vm || waker.state(vm.workspaceId, vm.recordSuspended) !== 'waking') { + return null + } + const before = latestRecipeVmWake(environment.id) + await waker.awaitWake(vm.workspaceId, vm.recordSuspended, holdMs, signal).catch(() => undefined) + const woken = latestRecipeVmWake(environment.id) + // The wake this request waited through re-paired the server: the caller's fence names the pairing + // before it, exactly as for a request that woke the VM itself. + return woken !== before + ? { before: environment, wokenPairingRevision: woken?.pairingRevision ?? null } + : null +} + +/** A wake a request waited through before taking its call slot: the record from before it and + * the pairing it left. */ +export type HeldRecipeVmWake = { + before: E + wokenPairingRevision: number | null +} + /** The pairing revision a recipe VM wake this request waited for left its server on, if any. */ export type RecipeVmPreparation = { wokenPairingRevision: number | null } const UNCHANGED: RecipeVmPreparation = { wokenPairingRevision: null } @@ -155,3 +196,27 @@ function refusal(outcome: Exclude | 'asleep'): RemoteRunt ) } } + +/** A request that does not open the workspace waits (bounded by its own timeout and hold) for a wake + * already in flight. Called before the request takes a per-environment call slot, so held + * background requests never block an open. */ +export function holdForInFlightRecipeVmWake( + userDataPath: string, + environment: E, + method: string, + timeoutMs: number | undefined, + options?: { preparation?: { recipeVmHoldMs?: number }; signal?: AbortSignal } +): Promise | null> { + if (recipeVmWakeIntentFor(method) !== 'none') { + return Promise.resolve(null) + } + const holdMs = Math.min( + RECIPE_VM_IN_FLIGHT_WAKE_HOLD_MS, + options?.preparation?.recipeVmHoldMs ?? Infinity, + timeoutMs ?? DEFAULT_CALL_TIMEOUT_MS + ) + return awaitRecipeVmInFlightWake(userDataPath, environment, holdMs, options?.signal) +} + +/** The transport's default call timeout, which also bounds the hold for calls that set none. */ +const DEFAULT_CALL_TIMEOUT_MS = 15_000 diff --git a/src/main/ipc/runtime-environments/runtime-environment-request-preparation.test.ts b/src/main/ipc/runtime-environments/runtime-environment-request-preparation.test.ts index 18e6360355c..38bc56ae4a8 100644 --- a/src/main/ipc/runtime-environments/runtime-environment-request-preparation.test.ts +++ b/src/main/ipc/runtime-environments/runtime-environment-request-preparation.test.ts @@ -56,3 +56,46 @@ describe('a request whose own recipe VM wake re-paired its server', () => { expect((await fence(1, 'before')).failure).toMatchObject({ ok: false }) }) }) + +describe('a fenced request that waited through a wake already in flight', () => { + beforeEach(() => { + prepare.mockClear() + // By the time the request takes its slot the record already holds the new pairing; its own + // preparation woke nothing. + prepared.current = { environment: AFTER_WAKE, before: AFTER_WAKE, wokenPairingRevision: null } + }) + + const heldFence = (pairingRevision: number, wokenPairingRevision: number | null) => + prepareFencedRuntimeRequest( + '/user-data', + 'env-vm', + 'terminal.list', + undefined, + { pairingRevision, runtimeId: 'before' }, + undefined, + { before: BEFORE as never, wokenPairingRevision } + ) + + it('passes the pre-wake fence and then expects the pairing that wake made', async () => { + await expect(heldFence(1, 2)).resolves.toMatchObject({ + failure: null, + expected: { pairingRevision: 2, runtimeId: undefined } + }) + }) + + it('still refuses a caller fenced to an older pairing', async () => { + expect((await heldFence(0, 2)).failure).toMatchObject({ ok: false }) + expect(prepare).not.toHaveBeenCalled() + }) + + it('without the held wake, the same call is refused as changed', async () => { + const result = await prepareFencedRuntimeRequest( + '/user-data', + 'env-vm', + 'terminal.list', + undefined, + { pairingRevision: 1, runtimeId: 'before' } + ) + expect(result.failure).toMatchObject({ ok: false }) + }) +}) diff --git a/src/main/ipc/runtime-environments/runtime-environment-request-preparation.ts b/src/main/ipc/runtime-environments/runtime-environment-request-preparation.ts index 5c5cc54739a..e204f046db3 100644 --- a/src/main/ipc/runtime-environments/runtime-environment-request-preparation.ts +++ b/src/main/ipc/runtime-environments/runtime-environment-request-preparation.ts @@ -22,14 +22,17 @@ export async function prepareFencedRuntimeRequest( method: string, preparation: ManagedRuntimeCallPreparation | undefined, expected: Expected, - signal?: AbortSignal + signal?: AbortSignal, + /** A recipe VM wake this request waited through before its call slot: its fence is checked + * against the record from before that wake, as for a wake the request made itself. */ + heldWake?: { before: KnownRuntimeEnvironment; wokenPairingRevision: number | null } ): Promise< | { failure: Rpc.RuntimeRpcResponse } | { failure: null; environment: KnownRuntimeEnvironment; expected: Expected } > { // A stale caller is refused before its request can wake (and re-pair) a VM. const stale = runtimeEnvironmentRevisionFailure( - resolveEnvironment(userDataPath, environmentId), + heldWake?.before ?? resolveEnvironment(userDataPath, environmentId), expected.pairingRevision, method, expected.runtimeId @@ -44,7 +47,10 @@ export async function prepareFencedRuntimeRequest( method, signal ) - const { environment, wokenPairingRevision: woken } = prepared + const { environment } = prepared + const woken = prepared.wokenPairingRevision ?? heldWake?.wokenPairingRevision ?? null + const before = + prepared.wokenPairingRevision !== null ? prepared.before : (heldWake?.before ?? prepared.before) const failure = woken === null ? runtimeEnvironmentRevisionFailure( @@ -54,7 +60,7 @@ export async function prepareFencedRuntimeRequest( expected.runtimeId ) : (runtimeEnvironmentRevisionFailure( - prepared.before, + before, expected.pairingRevision, method, expected.runtimeId diff --git a/src/main/ipc/runtime-environments/runtime-environment-transport-routing.ts b/src/main/ipc/runtime-environments/runtime-environment-transport-routing.ts index 0a4fcce7984..e0fec03129a 100644 --- a/src/main/ipc/runtime-environments/runtime-environment-transport-routing.ts +++ b/src/main/ipc/runtime-environments/runtime-environment-transport-routing.ts @@ -15,6 +15,7 @@ import { resolveManagedRuntimeEnvironment, type ManagedRuntimeCallPreparation } from './runtime-environment-managed-tunnel' +import { markEnvironmentUsedFromResponse } from './runtime-environment-usage-marking' import { recordRuntimeEnvironmentUsage } from '../../../shared/remote-runtime/host-store/runtime-environment-usage-record' import type * as Rpc from '../../../shared/runtime-rpc-envelope' import '../../../shared/remote-runtime/requests/remote-runtime-request-socket' @@ -23,6 +24,7 @@ import type { RemoteRuntimeTransportSubscription } from '../../../shared/remote- import { withRuntimeUnreachableHint } from './runtime-environment-unreachable-hint' import { decodeRuntimeEnvironmentSubscriptionEvents } from './runtime-environment-subscription-events' import { enqueueRuntimeCall } from './runtime-environment-call-queue' +import { holdForInFlightRecipeVmWake } from './runtime-environment-recipe-vm-wake' import { sendRemoteRuntimeConnectionRequestAbortable, sendRemoteRuntimeRequestAbortable @@ -80,6 +82,13 @@ export async function callRuntimeEnvironment( ) } const environment = resolveEnvironment(userDataPath, selector) + const held = await holdForInFlightRecipeVmWake( + userDataPath, + environment, + method, + timeoutMs, + options + ) // Connect failures reject; retain the dispatched endpoint for hints because queued re-pairing can change it. let endpoint = getPreferredPairedRuntimeConnection(environment).endpoint try { @@ -96,7 +105,8 @@ export async function callRuntimeEnvironment( pairingRevision: expectedEnvironmentPairingRevision, runtimeId: options?.expectedEnvironmentRuntimeId }, - options?.signal + options?.signal, + held ?? undefined ) if (prepared.failure) { return prepared.failure @@ -290,18 +300,6 @@ export async function subscribeRuntimeEnvironment( } } -function markEnvironmentUsedFromResponse( - userDataPath: string, - environmentId: string, - response: Rpc.RuntimeRpcResponse -): void { - if (response.ok === true) { - recordRuntimeEnvironmentUsage(userDataPath, environmentId, { - runtimeId: response._meta.runtimeId - }) - } -} - function shouldUseCachedRequestConnection(method: string): boolean { return method === 'terminal.send' || method === 'terminal.updateViewport' } diff --git a/src/main/ipc/runtime-environments/runtime-environment-usage-marking.ts b/src/main/ipc/runtime-environments/runtime-environment-usage-marking.ts new file mode 100644 index 00000000000..83472942edb --- /dev/null +++ b/src/main/ipc/runtime-environments/runtime-environment-usage-marking.ts @@ -0,0 +1,15 @@ +import { recordRuntimeEnvironmentUsage } from '../../../shared/remote-runtime/host-store/runtime-environment-usage-record' +import type * as Rpc from '../../../shared/runtime-rpc-envelope' + +/** A successful answer records which server process served the environment. */ +export function markEnvironmentUsedFromResponse( + userDataPath: string, + environmentId: string, + response: Rpc.RuntimeRpcResponse +): void { + if (response.ok === true) { + recordRuntimeEnvironmentUsage(userDataPath, environmentId, { + runtimeId: response._meta.runtimeId + }) + } +} diff --git a/src/main/runtime/workspaces/sleep/recipe-vm-in-flight-wake.test.ts b/src/main/runtime/workspaces/sleep/recipe-vm-in-flight-wake.test.ts new file mode 100644 index 00000000000..bcc6b8075ee --- /dev/null +++ b/src/main/runtime/workspaces/sleep/recipe-vm-in-flight-wake.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from 'vitest' +import { awaitInFlightWake, WorkspaceSleepRequests, type VmOperation } from './recipe-vm-operations' + +describe('awaitInFlightWake', () => { + it('answers at once for a VM that is up or asleep, without starting anything', async () => { + const sleeps = new WorkspaceSleepRequests() + await expect(awaitInFlightWake([], false, sleeps, 'w', 1000)).resolves.toBe('up') + await expect(awaitInFlightWake([], true, sleeps, 'w', 1000)).resolves.toBe('asleep') + }) + + it('waits for the wake already in flight', async () => { + let finish = (): void => {} + const settled = new Promise((resolve) => { + finish = resolve + }) + const pending: VmOperation[] = [{ kind: 'wake', settled }] + const answer = awaitInFlightWake(pending, false, new WorkspaceSleepRequests(), 'w', 5000) + finish() + await expect(answer).resolves.toBe('up') + }) + + it('ends slept when a Sleep is asked for while it waits, and waking when the hold runs out', async () => { + const sleeps = new WorkspaceSleepRequests() + const never: VmOperation[] = [{ kind: 'wake', settled: new Promise(() => {}) }] + const slept = awaitInFlightWake(never, false, sleeps, 'w', 5000) + sleeps.noteRequested('w') + await expect(slept).resolves.toBe('slept') + await expect(awaitInFlightWake(never, false, sleeps, 'w', 5)).resolves.toBe('waking') + }) +}) diff --git a/src/main/runtime/workspaces/sleep/recipe-vm-operations.ts b/src/main/runtime/workspaces/sleep/recipe-vm-operations.ts index 00305978ffd..1778f20ef98 100644 --- a/src/main/runtime/workspaces/sleep/recipe-vm-operations.ts +++ b/src/main/runtime/workspaces/sleep/recipe-vm-operations.ts @@ -124,3 +124,26 @@ export async function holdOpenUntilUp( stopAbortWatch() } } + +/** Holds a request that does not open the workspace while the wake already in flight settles; a VM + * that is not waking answers at once with where it stands. Never starts a wake. */ +export async function awaitInFlightWake( + pending: readonly VmOperation[], + recordSuspended: boolean, + sleepRequests: WorkspaceSleepRequests, + worktreeId: string, + timeoutMs: number, + signal?: AbortSignal +): Promise { + signal?.throwIfAborted() + const state = vmStateOf(pending, recordSuspended) + if (state !== 'waking') { + return state === 'up' ? 'up' : 'asleep' + } + const sleep = sleepRequests.next(worktreeId) + try { + return await holdOpenUntilUp(pendingVmWake(pending), sleep.requested, timeoutMs, signal) + } finally { + sleep.stop() + } +} diff --git a/src/main/runtime/workspaces/sleep/recipe-vm-waker-registry.ts b/src/main/runtime/workspaces/sleep/recipe-vm-waker-registry.ts index b2697ae0ecd..cead8b03c5e 100644 --- a/src/main/runtime/workspaces/sleep/recipe-vm-waker-registry.ts +++ b/src/main/runtime/workspaces/sleep/recipe-vm-waker-registry.ts @@ -15,6 +15,14 @@ export type RecipeVmWaker = { timeoutMs: number, signal?: AbortSignal ) => Promise + /** Holds a request that does not open the workspace while a wake already in flight finishes; + * never starts one. */ + awaitWake?: ( + workspaceId: string, + recordSuspended: boolean, + timeoutMs: number, + signal?: AbortSignal + ) => Promise /** Holds an open of the workspace behind a saved environment (id or name) for the CLI. */ openEnvironment: (selector: string) => Promise } diff --git a/src/main/runtime/workspaces/sleep/runtime-workspace-sleep-operations.ts b/src/main/runtime/workspaces/sleep/runtime-workspace-sleep-operations.ts index 69c4825efa5..d75bfe20c49 100644 --- a/src/main/runtime/workspaces/sleep/runtime-workspace-sleep-operations.ts +++ b/src/main/runtime/workspaces/sleep/runtime-workspace-sleep-operations.ts @@ -6,7 +6,7 @@ import type { WorkspaceSleepRequest } from '../../../../shared/workspace-sleep' import { WorkspaceSleepCoordinator } from './workspace-sleep-runtime' -import type { RecipeVmOpening, RecipeVmState } from './recipe-vm-operations' +import { awaitInFlightWake, type RecipeVmOpening, type RecipeVmState } from './recipe-vm-operations' import { dropStrandedSleepRecord } from '../../terminal/stop/worktree-sleep-resume-records' import type { RestoredPane } from '../../terminal/recovery/restored-pane-start' import type { SleepingAgentSessionRecord } from '../../../../shared/agent-session/lifecycle/agent-session-resume' @@ -166,6 +166,18 @@ export abstract class RuntimeWorkspaceSleepOperations extends OrcaRuntimeWithRes return this.workspaceSleep.holdOpen(worktreeId, recordSuspended, timeoutMs, signal) } + /** Holds a request that does not open this workspace while its in-flight wake finishes. */ + awaitWorkspaceVmWake( + worktreeId: string, + recordSuspended: boolean, + timeoutMs: number, + signal?: AbortSignal + ): Promise { + const pending = this.workspaceSleep.pendingVmOperations.get(worktreeId) ?? [] + const sleeps = this.workspaceSleep.sleepRequests + return awaitInFlightWake(pending, recordSuspended, sleeps, worktreeId, timeoutMs, signal) + } + /** The window opened a slept pane; main resumes it and the window attaches. */ resumeSleptPane(pane: RestoredPane, automatic?: boolean): Promise { return this.workspaceSleep.resumePane(pane, automatic) diff --git a/src/main/runtime/workspaces/sleep/workspace-sleep-runtime.ts b/src/main/runtime/workspaces/sleep/workspace-sleep-runtime.ts index 49b038d774b..dec406887b3 100644 --- a/src/main/runtime/workspaces/sleep/workspace-sleep-runtime.ts +++ b/src/main/runtime/workspaces/sleep/workspace-sleep-runtime.ts @@ -95,8 +95,9 @@ export class WorkspaceSleepCoordinator { private readonly reopenings = new Map() private readonly vmOperations = new Map>() /** The VM operations asked for per workspace, in order, until each settles. */ - private readonly pendingVmOperations = new Map() - private readonly sleepRequests = new WorkspaceSleepRequests() + /** Read by the in-flight wake hold (awaitInFlightWake); written only here. */ + readonly pendingVmOperations = new Map() + readonly sleepRequests = new WorkspaceSleepRequests() constructor(private readonly runtime: WorkspaceSleepRuntime) {} diff --git a/src/main/startup/main-process-runtime-service.ts b/src/main/startup/main-process-runtime-service.ts index 7b0405af786..38a5ae32e8a 100644 --- a/src/main/startup/main-process-runtime-service.ts +++ b/src/main/startup/main-process-runtime-service.ts @@ -128,13 +128,17 @@ export function initializeMainProcessRuntime(): OrcaRuntimeService { onRuntimeEnvironmentStatusPublished(trackRuntimeEnvironmentVerifiedTransitions(listener)) }, workspaceSleepPlatform: { + // Sleep's own calls and observations never wait on a wake in flight. callEnvironment: (environmentId, method, params, timeoutMs) => callRuntimeEnvironment( getRuntimeEnvironmentRegistryRoot(), environmentId, method, params, - timeoutMs + timeoutMs, + undefined, + undefined, + { preparation: { recipeVmHoldMs: 0 } } ), ownsVm: (workspaceId) => findWorkspaceEphemeralVmRuntime(app.getPath('userData'), workspaceId) !== undefined, @@ -166,6 +170,8 @@ export function initializeMainProcessRuntime(): OrcaRuntimeService { noteOpened: (workspaceId) => runtime.noteWorkspaceReopened(workspaceId), holdOpen: (workspaceId, suspended, timeoutMs, signal) => runtime.holdWorkspaceVmOpen(workspaceId, suspended, timeoutMs, signal), + awaitWake: (workspaceId, suspended, timeoutMs, signal) => + runtime.awaitWorkspaceVmWake(workspaceId, suspended, timeoutMs, signal), openEnvironment: (selector) => { const root = getRuntimeEnvironmentRegistryRoot() return openRecipeVmEnvironment(root, resolveEnvironment(root, selector)) diff --git a/src/renderer/src/components/orca-cloud/CloudWakingOverlay.tsx b/src/renderer/src/components/orca-cloud/CloudWakingOverlay.tsx index 924577d777f..2eee77ebe5f 100644 --- a/src/renderer/src/components/orca-cloud/CloudWakingOverlay.tsx +++ b/src/renderer/src/components/orca-cloud/CloudWakingOverlay.tsx @@ -127,7 +127,9 @@ export function CloudWakingOverlay({ : wake.source === 'restart' ? t('orcaCloudWorkspace.announce.restarting', 'Restarting {{name}}.', { name }) : wake.source === 'unarchive' - ? t('orcaCloudWorkspace.announce.unarchiving', 'Restoring {{name}} from archive.', { name }) + ? t('orcaCloudWorkspace.announce.unarchiving', 'Restoring {{name}} from archive.', { + name + }) : t('orcaCloudWorkspace.announce.waking', 'Waking {{name}}.', { name }) } > diff --git a/src/renderer/src/components/orca-cloud/OrcaCloudWorkspace.fixture.tsx b/src/renderer/src/components/orca-cloud/OrcaCloudWorkspace.fixture.tsx index 7cc31796e56..cbeef9ddcf8 100644 --- a/src/renderer/src/components/orca-cloud/OrcaCloudWorkspace.fixture.tsx +++ b/src/renderer/src/components/orca-cloud/OrcaCloudWorkspace.fixture.tsx @@ -44,7 +44,12 @@ const running: HostedWorkspaceStatus = { archive: null, checkedAt: NOW } -const asleep: HostedWorkspaceStatus = { ...running, state: 'sleeping', metrics: null, uptimeMs: null } +const asleep: HostedWorkspaceStatus = { + ...running, + state: 'sleeping', + metrics: null, + uptimeMs: null +} const entry = ( status: HostedWorkspaceStatus | null, over: Partial = {} @@ -100,7 +105,11 @@ export const panelUnresponsive = defineFixture({ export const panelRestarting = defineFixture({ width: 300, render: () => ( - + ) }) @@ -298,7 +307,7 @@ function IdleToast(): React.JSX.Element { return ( <>
- + ) } diff --git a/src/renderer/src/components/orca-cloud/cloud-workspace-actions.ts b/src/renderer/src/components/orca-cloud/cloud-workspace-actions.ts index 148823b6697..7a637867e5f 100644 --- a/src/renderer/src/components/orca-cloud/cloud-workspace-actions.ts +++ b/src/renderer/src/components/orca-cloud/cloud-workspace-actions.ts @@ -109,7 +109,9 @@ export function runArchiveCloudWorkspace( }, options.undoMs ?? ARCHIVE_UNDO_MS) toast( options.name - ? translate('orcaCloudWorkspace.archivingNamed', 'Archiving {{name}}…', { name: options.name }) + ? translate('orcaCloudWorkspace.archivingNamed', 'Archiving {{name}}…', { + name: options.name + }) : translate('orcaCloudWorkspace.archiving', 'Archiving workspace…'), { id: toastId, diff --git a/src/renderer/src/components/orca-cloud/offline/CloudWorkspaceOfflineView.tsx b/src/renderer/src/components/orca-cloud/offline/CloudWorkspaceOfflineView.tsx index bfcbe286e66..4c8c52fe50e 100644 --- a/src/renderer/src/components/orca-cloud/offline/CloudWorkspaceOfflineView.tsx +++ b/src/renderer/src/components/orca-cloud/offline/CloudWorkspaceOfflineView.tsx @@ -96,7 +96,9 @@ export function CloudWorkspaceOfflineView(props: CloudOfflineViewProps): React.J {t('components.orcaCloud.offline.tabChanges', 'Changes')} - {changeCount > 0 ? {changeCount} : null} + {changeCount > 0 ? ( + {changeCount} + ) : null} diff --git a/src/renderer/src/components/sidebar/workspace-actions/context-menu/WorktreeCloudTransferMenuItem.tsx b/src/renderer/src/components/sidebar/workspace-actions/context-menu/WorktreeCloudTransferMenuItem.tsx index f5a387c913e..086eb415977 100644 --- a/src/renderer/src/components/sidebar/workspace-actions/context-menu/WorktreeCloudTransferMenuItem.tsx +++ b/src/renderer/src/components/sidebar/workspace-actions/context-menu/WorktreeCloudTransferMenuItem.tsx @@ -32,10 +32,7 @@ export function WorktreeCloudTransferMenuItem({ return ( void runOrcaCloudTransferToLocal(worktreeId)}> - {t( - 'auto.components.sidebar.WorktreeContextMenu.moveToThisComputer', - 'Move to this computer' - )} + {t('auto.components.sidebar.WorktreeContextMenu.moveToThisComputer', 'Move to this computer')} ) } diff --git a/src/renderer/src/components/terminal-pane/pty-connection/deferred-cold-restore-and-snapshot.ts b/src/renderer/src/components/terminal-pane/pty-connection/deferred-cold-restore-and-snapshot.ts index 0c22d59bde5..d45980ceb60 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection/deferred-cold-restore-and-snapshot.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection/deferred-cold-restore-and-snapshot.ts @@ -1,3 +1,4 @@ +import { markInterruptedTurnContinued } from './startup/cold-restore-resume-startup' import { paneResumeRecord } from './startup/pane-resume-record' import { useAppStore } from '@/store/index' import type { PtyReplayDataMeta } from '../pty-transport/pty-transport-types' @@ -34,6 +35,7 @@ export function bindDeferredColdRestoreAndSnapshot(session: ConnectPanePtySessio if (!startup || startup.resumeSource?.cancelled) { return false } + markInterruptedTurnContinued(startup.interruptedTurnKey) const state = useAppStore.getState() if (!session.pendingStartupCommand && session.shouldDeliverStartupViaTerminalPaste) { session.pendingStartupCommand = { diff --git a/src/renderer/src/components/terminal-pane/pty-connection/reattach/deferred-session-reattach-connect.ts b/src/renderer/src/components/terminal-pane/pty-connection/reattach/deferred-session-reattach-connect.ts index 46f8bb740ad..052a8b77862 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection/reattach/deferred-session-reattach-connect.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection/reattach/deferred-session-reattach-connect.ts @@ -1,5 +1,6 @@ import { NOT_REOPENED, + markInterruptedTurnContinued, reopenPaneAfterMissingReattach } from '../startup/cold-restore-resume-startup' import { writeStartupCwdFallbackNotice } from '../startup/startup-cwd-fallback-notice' @@ -134,6 +135,10 @@ function connectDeferredSessionReattach( ) session.beginReattachLiveDataDeferral(outputCallbacks.generation) session.transportConnectInFlightSince = Date.now() + // The command goes to the server now: a `continue` it carries is spent. + if (coldRestoreStartup?.command) { + markInterruptedTurnContinued(coldRestoreStartup.interruptedTurnKey) + } const reattachPromise = session.transport.connect({ url: '', cols: session.cols, diff --git a/src/renderer/src/components/terminal-pane/pty-connection/startup/cold-restore-continue.test.ts b/src/renderer/src/components/terminal-pane/pty-connection/startup/cold-restore-continue.test.ts new file mode 100644 index 00000000000..16356718f15 --- /dev/null +++ b/src/renderer/src/components/terminal-pane/pty-connection/startup/cold-restore-continue.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('@/store/index', () => ({ useAppStore: { getState: () => ({}) } })) + +describe('shouldContinueInterruptedTurn', () => { + const now = 1_800_000_000_000 + const working = (stateStartedAt: number) => ({ + state: 'working', + updatedAt: now - 1000, + stateStartedAt + }) + + it('continues a remote agent with a fresh working status, once per interrupted turn', async () => { + const { shouldContinueInterruptedTurn, interruptedTurnKey, markInterruptedTurnContinued } = + await import('./cold-restore-resume-startup') + const ctx = { paneKey: 'p1', hasSleepingRecord: false, now } + expect(shouldContinueInterruptedTurn(working(1), 'env-1', ctx)).toBe(true) + // Building (and discarding) the startup again does not spend the turn. + expect(shouldContinueInterruptedTurn(working(1), 'env-1', ctx)).toBe(true) + // Once the resume launched, a retried cold restore of the same turn does not send it again. + markInterruptedTurnContinued(interruptedTurnKey(working(1), 'p1')) + expect(shouldContinueInterruptedTurn(working(1), 'env-1', ctx)).toBe(false) + // A later interrupted turn does. + expect(shouldContinueInterruptedTurn(working(2), 'env-1', ctx)).toBe(true) + }) + + it('never continues after a deliberate sleep, for stale or non-working status, or locally', async () => { + const { shouldContinueInterruptedTurn } = await import('./cold-restore-resume-startup') + expect( + shouldContinueInterruptedTurn(working(10), 'env-1', { + paneKey: 'p2', + hasSleepingRecord: true, + now + }) + ).toBe(false) + expect( + shouldContinueInterruptedTurn( + { state: 'working', updatedAt: now - 31 * 60_000, stateStartedAt: 11 }, + 'env-1', + { + paneKey: 'p3', + hasSleepingRecord: false, + now + } + ) + ).toBe(false) + expect( + shouldContinueInterruptedTurn( + { state: 'waiting', updatedAt: now, stateStartedAt: 12 }, + 'env-1', + { paneKey: 'p4', hasSleepingRecord: false, now } + ) + ).toBe(false) + expect( + shouldContinueInterruptedTurn(working(13), null, { + paneKey: 'p5', + hasSleepingRecord: false, + now + }) + ).toBe(false) + }) +}) diff --git a/src/renderer/src/components/terminal-pane/pty-connection/startup/cold-restore-resume-startup.ts b/src/renderer/src/components/terminal-pane/pty-connection/startup/cold-restore-resume-startup.ts index 983d4b9c96f..216ecb72b10 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection/startup/cold-restore-resume-startup.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection/startup/cold-restore-resume-startup.ts @@ -6,7 +6,10 @@ import { normalizeExecutionHostId } from '../../../../../../shared/execution-hos import { parseWorkspaceKey } from '../../../../../../shared/workspace-scope' import { useAppStore } from '@/store/index' import { createNonSecureContextUuid as createBrowserUuid } from '../../../../../../shared/non-secure-context-uuid' -import { buildAgentResumeStartupPlan } from '../../../../../../shared/agent-launch/startup/tui-agent-resume-startup' +import { + INTERRUPTED_TURN_CONTINUE_PROMPT, + buildAgentResumeStartupPlan +} from '../../../../../../shared/agent-launch/startup/tui-agent-resume-startup' import { resolveRuntimeHostReportedPlatform, resolveRuntimeHostTerminalWindowsShell, @@ -24,6 +27,7 @@ import { normalizeAgentProviderSession } from '../../../../../../shared/agent-session/lifecycle/agent-session-resume' +import { AGENT_STATUS_STALE_AFTER_MS } from '../../../../../../shared/agent-status/contracts/agent-status-freshness' import type { HeldPaneReopenOutcome } from '../../session-restore/newer-daemon/newer-daemon-hold-state' import type { ColdRestoreAgentResumeStartup } from './fresh-spawn-types' @@ -31,6 +35,45 @@ import type { ConnectPanePtySession } from '../connect-pane-pty-session' export const NOT_REOPENED: HeldPaneReopenOutcome = { reopened: false, agentResumed: false } +/** Interrupted turns already continued, so a retried cold restore never sends `continue` twice. */ +const continuedInterruptedTurns = new Set() + +/** Identity of the turn an agent was in when its host went away. */ +export function interruptedTurnKey( + entry: { updatedAt: number; stateStartedAt?: number }, + paneKey: string +): string { + return `${paneKey}\u0000${entry.stateStartedAt ?? entry.updatedAt}` +} + +/** Called when a resume that carries `continue` actually launches (startups are often built and + * discarded, e.g. on every reattach check). */ +export function markInterruptedTurnContinued(key: string | undefined): void { + if (key) { + continuedInterruptedTurns.add(key) + } +} + +/** + * A cold-restored agent on a remote host whose last status was working lost its turn when the host + * restarted (reboot, crash, rollover): carry it on. Only with fresh evidence (a recent working + * status), never after a deliberate sleep (that leaves a sleeping record), once per interrupted turn. + * Local panes are left alone: their PTYs outlive app restarts. + */ +export function shouldContinueInterruptedTurn( + entry: { state: string; updatedAt: number; stateStartedAt?: number } | null | undefined, + runtimeEnvironmentId: string | null, + context: { paneKey: string; hasSleepingRecord: boolean; now?: number } +): boolean { + if (!runtimeEnvironmentId || context.hasSleepingRecord || entry?.state !== 'working') { + return false + } + if ((context.now ?? Date.now()) - entry.updatedAt > AGENT_STATUS_STALE_AFTER_MS) { + return false + } + return !continuedInterruptedTurns.has(interruptedTurnKey(entry, context.paneKey)) +} + type AppState = ReturnType /** The agent session a cold restore of this pane would resume; read-only, so pre-spawn checks can share it. */ @@ -109,6 +152,12 @@ export function bindBuildColdRestoreAgentResumeStartup(session: ConnectPanePtySe session.executionHostId ) }) + const continueTurn = + entry !== null && + shouldContinueInterruptedTurn(entry, session.transport.getRuntimeEnvironmentId?.() ?? null, { + paneKey: session.cacheKey, + hasSleepingRecord: Boolean(sleepingRecord) + }) const startupPlan = buildAgentResumeStartupPlan({ agent, providerSession, @@ -117,7 +166,10 @@ export function bindBuildColdRestoreAgentResumeStartup(session: ConnectPanePtySe platform: resumeTarget.platform, agentHostPlatform: resumeTarget.agentHostPlatform, shell: resumeTarget.shell, - launchCwd: session.worktree?.path + launchCwd: session.worktree?.path, + // A remote host that restarted under a working agent (reboot, crash, VM rollover) killed it + // mid-turn; its last status still says working. Carry the turn on instead of waiting. + continuePrompt: continueTurn ? INTERRUPTED_TURN_CONTINUE_PROMPT : null }) if (!startupPlan) { return null @@ -151,6 +203,9 @@ export function bindBuildColdRestoreAgentResumeStartup(session: ConnectPanePtySe : {}) }, launchToken: coldRestoreLaunchToken, + ...(continueTurn && entry + ? { interruptedTurnKey: interruptedTurnKey(entry, session.cacheKey) } + : {}), useLiveEntry: Boolean(useLiveEntry), hasSleepingRecord: Boolean(sleepingRecord), sleepingRecordEntry diff --git a/src/renderer/src/components/terminal-pane/pty-connection/startup/fresh-spawn-types.ts b/src/renderer/src/components/terminal-pane/pty-connection/startup/fresh-spawn-types.ts index d3071dcc0a5..31926f92f3d 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection/startup/fresh-spawn-types.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection/startup/fresh-spawn-types.ts @@ -20,6 +20,8 @@ export type FreshSpawnOptions = { } export type ColdRestoreAgentResumeStartup = PendingStartupCommand & { + /** Set when the command carries `continue` for an interrupted turn; marked used on launch. */ + interruptedTurnKey?: string agent: ResumableTuiAgent resumeProviderSession: AgentProviderSessionMetadata connectionId?: string | null diff --git a/src/renderer/src/lib/orca-cloud/orca-cloud-workspace-state.test.ts b/src/renderer/src/lib/orca-cloud/orca-cloud-workspace-state.test.ts index 4b26dcd5762..b19fa8f6c6b 100644 --- a/src/renderer/src/lib/orca-cloud/orca-cloud-workspace-state.test.ts +++ b/src/renderer/src/lib/orca-cloud/orca-cloud-workspace-state.test.ts @@ -14,7 +14,10 @@ import { } from './orca-cloud-workspace-state' const NOW = 1_800_000_000_000 -const status = (state: HostedWorkspaceStatus['state'], archive: HostedWorkspaceStatus['archive'] = null): HostedWorkspaceStatus => ({ +const status = ( + state: HostedWorkspaceStatus['state'], + archive: HostedWorkspaceStatus['archive'] = null +): HostedWorkspaceStatus => ({ state, metrics: null, uptimeMs: null, @@ -32,18 +35,39 @@ const wake = (phase: CloudWakeProgressEvent['phase'], at = NOW): CloudWakeProgre describe('deriveCloudDotState', () => { it('shows an in-flight wake over everything, including a sleeping desktop record', () => { - expect(deriveCloudDotState({ status: status('sleeping'), wake: wake('waking'), sleeping: true, now: NOW })).toBe('waking') - expect(deriveCloudDotState({ status: status('ready'), wake: wake('connecting'), sleeping: false, now: NOW })).toBe('waking') + expect( + deriveCloudDotState({ + status: status('sleeping'), + wake: wake('waking'), + sleeping: true, + now: NOW + }) + ).toBe('waking') + expect( + deriveCloudDotState({ + status: status('ready'), + wake: wake('connecting'), + sleeping: false, + now: NOW + }) + ).toBe('waking') }) it('treats a wake with no terminal event for 4 minutes as stale', () => { expect( - deriveCloudDotState({ status: status('ready'), wake: wake('waking', NOW - 5 * 60_000), sleeping: false, now: NOW }) + deriveCloudDotState({ + status: status('ready'), + wake: wake('waking', NOW - 5 * 60_000), + sleeping: false, + now: NOW + }) ).toBe('running') }) it('archived beats the desktop sleep state; failed wakes show until stale', () => { - expect(deriveCloudDotState({ status: status('archived'), wake: undefined, sleeping: true, now: NOW })).toBe('archived') + expect( + deriveCloudDotState({ status: status('archived'), wake: undefined, sleeping: true, now: NOW }) + ).toBe('archived') expect( deriveCloudDotState({ status: status('sleeping', { archivedAt: NOW, expiresAt: null }), @@ -52,16 +76,45 @@ describe('deriveCloudDotState', () => { now: NOW }) ).toBe('archived') - expect(deriveCloudDotState({ status: status('sleeping'), wake: wake('failed'), sleeping: true, now: NOW })).toBe('failed') + expect( + deriveCloudDotState({ + status: status('sleeping'), + wake: wake('failed'), + sleeping: true, + now: NOW + }) + ).toBe('failed') }) it('maps control-plane states; the local sleep record only fills in before the first read', () => { - expect(deriveCloudDotState({ status: status('ready'), wake: undefined, sleeping: false, now: NOW })).toBe('running') - expect(deriveCloudDotState({ status: status('unresponsive'), wake: undefined, sleeping: false, now: NOW })).toBe('unresponsive') - expect(deriveCloudDotState({ status: status('ready'), wake: undefined, sleeping: true, now: NOW })).toBe('running') - expect(deriveCloudDotState({ status: null, wake: undefined, sleeping: true, now: NOW })).toBe('sleeping') - expect(deriveCloudDotState({ status: status('provisioning'), wake: undefined, sleeping: false, now: NOW })).toBe('waking') - expect(deriveCloudDotState({ status: null, wake: undefined, sleeping: false, now: NOW })).toBe('unknown') + expect( + deriveCloudDotState({ status: status('ready'), wake: undefined, sleeping: false, now: NOW }) + ).toBe('running') + expect( + deriveCloudDotState({ + status: status('unresponsive'), + wake: undefined, + sleeping: false, + now: NOW + }) + ).toBe('unresponsive') + expect( + deriveCloudDotState({ status: status('ready'), wake: undefined, sleeping: true, now: NOW }) + ).toBe('running') + expect(deriveCloudDotState({ status: null, wake: undefined, sleeping: true, now: NOW })).toBe( + 'sleeping' + ) + expect( + deriveCloudDotState({ + status: status('provisioning'), + wake: undefined, + sleeping: false, + now: NOW + }) + ).toBe('waking') + expect(deriveCloudDotState({ status: null, wake: undefined, sleeping: false, now: NOW })).toBe( + 'unknown' + ) }) }) diff --git a/src/renderer/src/lib/orca-cloud/orca-cloud-workspace-store-wake.test.ts b/src/renderer/src/lib/orca-cloud/orca-cloud-workspace-store-wake.test.ts index 8dd132f0d6b..908f0ba2aae 100644 --- a/src/renderer/src/lib/orca-cloud/orca-cloud-workspace-store-wake.test.ts +++ b/src/renderer/src/lib/orca-cloud/orca-cloud-workspace-store-wake.test.ts @@ -19,7 +19,10 @@ const status = (state: HostedWorkspaceStatus['state']): HostedWorkspaceStatus => checkedAt: 1 }) -function stub(getStatus: OrcaCloudWorkspaceApi['getStatus'], ids: string[] = []): OrcaCloudWorkspaceApi { +function stub( + getStatus: OrcaCloudWorkspaceApi['getStatus'], + ids: string[] = [] +): OrcaCloudWorkspaceApi { const bridge = { getStatus: vi.fn(getStatus), listCloudWorkspaceIds: vi.fn(async () => ids) @@ -48,7 +51,13 @@ describe('wake state and archive guard', () => { it('a failed wake stays until dismissed', async () => { stub(async () => ({ ok: true, value: status('sleeping') })) - noteCloudWakeProgress({ workspaceId: 'a', phase: 'failed', source: 'open', startedAt: 1, at: 2 }) + noteCloudWakeProgress({ + workspaceId: 'a', + phase: 'failed', + source: 'open', + startedAt: 1, + at: 2 + }) expect(getCloudWorkspaceSnapshot().wake.a?.phase).toBe('failed') dismissCloudWake('a') expect(getCloudWorkspaceSnapshot().wake.a).toBeUndefined() diff --git a/src/renderer/src/lib/orca-cloud/orca-cloud-workspace-store.test.ts b/src/renderer/src/lib/orca-cloud/orca-cloud-workspace-store.test.ts index 06253c4a0e2..c679e2cb363 100644 --- a/src/renderer/src/lib/orca-cloud/orca-cloud-workspace-store.test.ts +++ b/src/renderer/src/lib/orca-cloud/orca-cloud-workspace-store.test.ts @@ -28,7 +28,7 @@ const ready: HostedWorkspaceStatus = { archive: null, checkedAt: 1 } -const ok = (value: T): CloudWorkspaceResult => ({ ok: true, value }) +const ok = (value: T): CloudWorkspaceResult => ({ ok: true, value }) function stubApi(overrides: Partial = {}): OrcaCloudWorkspaceApi { const bridge: OrcaCloudWorkspaceApi = { @@ -42,7 +42,9 @@ function stubApi(overrides: Partial = {}): OrcaCloudWorks onOom: vi.fn(() => () => {}), ...overrides } - ;(globalThis as unknown as { window: { api: { orcaCloudWorkspace: OrcaCloudWorkspaceApi } } }).window = { + ;( + globalThis as unknown as { window: { api: { orcaCloudWorkspace: OrcaCloudWorkspaceApi } } } + ).window = { api: { orcaCloudWorkspace: bridge } } return bridge @@ -66,7 +68,11 @@ describe('orca cloud workspace store', () => { const bridge = stubApi() await Promise.all([refreshCloudWorkspaceStatus('a'), refreshCloudWorkspaceStatus('a')]) expect(bridge.getStatus).toHaveBeenCalledTimes(1) - bridge.getStatus = vi.fn(async () => ({ ok: false as const, code: 'unavailable' as const, message: 'down' })) + bridge.getStatus = vi.fn(async () => ({ + ok: false as const, + code: 'unavailable' as const, + message: 'down' + })) await refreshCloudWorkspaceStatus('a') const entry = getCloudWorkspaceSnapshot().status.a! expect(entry.status?.state).toBe('ready') @@ -75,7 +81,13 @@ describe('orca cloud workspace store', () => { }) it('forgets a workspace the main process says is not cloud', async () => { - stubApi({ getStatus: vi.fn(async () => ({ ok: false as const, code: 'not_cloud' as const, message: 'no' })) }) + stubApi({ + getStatus: vi.fn(async () => ({ + ok: false as const, + code: 'not_cloud' as const, + message: 'no' + })) + }) seedCloudWorkspaceSnapshot({ ids: new Set(['a']) }) await refreshCloudWorkspaceStatus('a') expect(getCloudWorkspaceSnapshot().ids.has('a')).toBe(false) @@ -117,7 +129,13 @@ describe('orca cloud workspace store', () => { it('tracks wake progress (adding the id) and dismisses OOM notices up to the seen count', async () => { stubApi() - noteCloudWakeProgress({ workspaceId: 'c', phase: 'waking', source: 'message', startedAt: 1, at: 2 }) + noteCloudWakeProgress({ + workspaceId: 'c', + phase: 'waking', + source: 'message', + startedAt: 1, + at: 2 + }) expect(getCloudWorkspaceSnapshot().ids.has('c')).toBe(true) noteCloudOom({ workspaceId: 'c', oomKills: 1, victim: 'node', at: 3 }) dismissCloudOom('c') diff --git a/src/shared/agent-launch/startup/tui-agent-resume-continue.test.ts b/src/shared/agent-launch/startup/tui-agent-resume-continue.test.ts new file mode 100644 index 00000000000..c8bb1747775 --- /dev/null +++ b/src/shared/agent-launch/startup/tui-agent-resume-continue.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, it } from 'vitest' +import { buildAgentResumeStartupPlan } from './tui-agent-resume-startup' + +const providerSession = { key: 'session_id', id: 'claude-session-1' } as const + +describe('resume with a continue prompt', () => { + const base = { cmdOverrides: {}, platform: 'linux' as const, shell: 'posix' as const } + + it('submits the prompt on resume for Claude and Codex', () => { + const claude = buildAgentResumeStartupPlan({ + ...base, + agent: 'claude', + providerSession, + continuePrompt: 'continue' + }) + expect(claude?.launchCommand).toMatch(/'?--resume'? '?claude-session-1'? '?continue'?$/) + const codex = buildAgentResumeStartupPlan({ + ...base, + agent: 'codex', + providerSession: { key: 'session_id', id: 'codex-1' }, + continuePrompt: 'continue' + }) + expect(codex?.launchCommand).toMatch(/'?resume'? '?codex-1'? '?continue'?$/) + }) + + it('leaves other agents and empty prompts unchanged', () => { + const plain = buildAgentResumeStartupPlan({ ...base, agent: 'claude', providerSession }) + const blank = buildAgentResumeStartupPlan({ + ...base, + agent: 'claude', + providerSession, + continuePrompt: ' ' + }) + expect(blank?.launchCommand).toBe(plain?.launchCommand) + const cursor = buildAgentResumeStartupPlan({ + ...base, + agent: 'cursor', + providerSession: { key: 'conversation_id', id: 'c-1' }, + continuePrompt: 'continue' + }) + expect(cursor?.launchCommand).not.toMatch(/continue'?$/) + }) +}) diff --git a/src/shared/agent-launch/startup/tui-agent-resume-startup.ts b/src/shared/agent-launch/startup/tui-agent-resume-startup.ts index 3f981d79664..a1f61d88365 100644 --- a/src/shared/agent-launch/startup/tui-agent-resume-startup.ts +++ b/src/shared/agent-launch/startup/tui-agent-resume-startup.ts @@ -37,6 +37,9 @@ export function buildAgentResumeStartupPlan(args: { fork?: boolean /** OS that runs the agent when `platform` only picks quoting; null when unreported. */ agentHostPlatform?: NodeJS.Platform | null + /** First message the resumed session submits, e.g. to carry on a turn its host cut off. Only + * agents whose resume takes a prompt argument (Claude, Codex) send it; others ignore it. */ + continuePrompt?: string | null }): AgentStartupPlan | null { const resumeArgv = args.fork ? getAgentForkArgv(args.agent, args.providerSession) @@ -45,10 +48,15 @@ export function buildAgentResumeStartupPlan(args: { return null } // Codex otherwise asks for its old folder, even if deleted (#17745). Keep before resume so argv removal preserves the folder choice. - const argv = + const withCwd = args.agent === 'codex' && args.resumeInLaunchCwd ? [resumeArgv[0], '-c', 'tui.resume_cwd=current', ...resumeArgv.slice(1)] : resumeArgv + const continuePrompt = args.continuePrompt?.trim() + const argv = + continuePrompt && !args.fork && RESUME_PROMPT_AGENTS.has(args.agent) + ? [...withCwd, continuePrompt] + : withCwd const shell = resolveStartupShell(args.platform, args.shell) // Why: Codex restores a resumed thread's saved permissions unless the launch passes one, so // replaying Orca's launch-time flag would undo a change made inside the session (#10886). @@ -112,6 +120,12 @@ export function buildAgentResumeStartupPlan(args: { } } +/** Agents whose resume command accepts a positional prompt it submits on start. */ +const RESUME_PROMPT_AGENTS: ReadonlySet = new Set(['claude', 'codex']) + +/** What a resumed agent is told when its host went away mid-turn (restart, reboot, crash). */ +export const INTERRUPTED_TURN_CONTINUE_PROMPT = 'continue' + /** WSL panes start at a UNC path while the agent reports its Linux path. */ function isSameFolder(agentCwd: string, launchCwd: string): boolean { const launchPath = parseWslUncPath(launchCwd)?.linuxPath ?? launchCwd