Merge pull request #77 from nwparker/cloud/w2-wakefix

fix(cloud): wake readiness, wait-for-wake instead of failing, skew-safe agent auto-continue
This commit is contained in:
Neil
2026-11-09 07:12:00 -08:00
committed by GitHub
34 changed files with 836 additions and 86 deletions
@@ -24,13 +24,51 @@ const ContinueAnswer = z.looseObject({
skipped: z.array(z.unknown()).optional()
})
/** `recordedAt` is the host's clock and `drainStartedAt` this machine's: tolerate skew between
* them so an offer the drain just recorded is not mistaken for an older one. */
export const CLOCK_SKEW_TOLERANCE_MS = 2 * 60_000
/** A host too slow to list its offers before a restart is treated as unable to (falls back to time). */
const OFFER_SNAPSHOT_TIMEOUT_MS = 5_000
function hasCode(error: unknown, code: string): boolean {
return typeof error === 'object' && error !== null && 'code' in error && error.code === code
}
/** Offer ids the host already holds before a drain; offers in it were left for the user. Null when
* the host cannot answer (an unresponsive host is often why it is being restarted). */
export async function listRestartOfferIds(call: RuntimeCall): Promise<ReadonlySet<string> | null> {
try {
let timer: ReturnType<typeof setTimeout> | undefined
const listed = await Promise.race([
call('agentSession.restartResumable'),
new Promise<null>((resolve) => {
timer = setTimeout(() => resolve(null), OFFER_SNAPSHOT_TIMEOUT_MS)
})
]).finally(() => clearTimeout(timer))
const list = OfferList.safeParse(listed)
if (!list.success) {
return null
}
return new Set(
list.data.sessions.flatMap((row) => {
const offer = Offer.safeParse(row)
return offer.success ? [offer.data.sessionId] : []
})
)
} catch {
return null
}
}
/**
* Continues the offers this drain recorded. With a pre-drain snapshot of the host's offers, "recorded
* by this drain" is set membership (no clock comparison); without one it falls back to the host
* timestamp against this machine's drain start, allowing for clock skew.
*/
export async function autoResumeAgentsAfterRollover(
call: RuntimeCall,
drainStartedAt: number
drainStartedAt: number,
offersBeforeDrain: ReadonlySet<string> | null = null
): Promise<HostedAgentAutoResumeOutcome> {
let listed: unknown
try {
@@ -47,7 +85,13 @@ export async function autoResumeAgentsAfterRollover(
}
const ids = list.data.sessions.flatMap((row) => {
const offer = Offer.safeParse(row)
return offer.success && offer.data.recordedAt >= drainStartedAt ? [offer.data.sessionId] : []
if (!offer.success) {
return []
}
const recordedByThisDrain = offersBeforeDrain
? !offersBeforeDrain.has(offer.data.sessionId)
: offer.data.recordedAt >= drainStartedAt - CLOCK_SKEW_TOLERANCE_MS
return recordedByThisDrain ? [offer.data.sessionId] : []
})
const outcome = { offered: ids.length, continued: 0, skipped: 0, unsupported: false }
for (let offset = 0; offset < ids.length; offset += MAX_RESTART_RESUME_SESSIONS) {
@@ -24,7 +24,12 @@ export type HostedSessionLeasePorts = {
warn: (minutesLeft: HostedLeaseWarning) => void
/** Re-pairs with the rebooted orcad; resolves once the runtime answers again. */
reconnect: (ready: HostedEnvironmentReady) => Promise<void>
resumeAgents: (drainStartedAt: number) => Promise<HostedAgentAutoResumeOutcome>
resumeAgents: (
drainStartedAt: number,
offersBeforeDrain: ReadonlySet<string> | null
) => Promise<HostedAgentAutoResumeOutcome>
/** The host's restart offers before the drain, so only the drain's own are continued. */
snapshotOffers?: () => Promise<ReadonlySet<string> | null>
onSuspended: () => void
/** Replaces the driver suspend, e.g. to run the window's ordinary Sleep for the workspace. */
suspend?: () => Promise<void>
@@ -158,6 +163,7 @@ export class HostedSessionLeaseScheduler {
private async rollover(reason: 'opportunistic' | 'forced'): Promise<void> {
const { driver, handle } = this.ports
const offersBeforeDrain = (await this.ports.snapshotOffers?.().catch(() => null)) ?? null
const drainStartedAt = this.now()
this.log(`[orca-cloud] ${reason} rollover`)
this.ports.onRolloverStateChange?.('restarting')
@@ -172,7 +178,7 @@ export class HostedSessionLeaseScheduler {
}
this.ports.onRolloverStateChange?.('running')
try {
const outcome = await this.ports.resumeAgents(drainStartedAt)
const outcome = await this.ports.resumeAgents(drainStartedAt, offersBeforeDrain)
this.log(
`[orca-cloud] continued ${outcome.continued}/${outcome.offered} agents after rollover`
)
@@ -1,6 +1,6 @@
import { describe, expect, it, vi } from 'vitest'
import type { HostedEnvironmentReady } from './hosted-environment-provider'
import { autoResumeAgentsAfterRollover } from './hosted-agent-auto-resume'
import { autoResumeAgentsAfterRollover, listRestartOfferIds } from './hosted-agent-auto-resume'
import { createHostedSandboxControlPlaneClient } from './hosted-sandbox-control-plane-client'
import { createHostedSandboxDriver, HOSTED_SANDBOX_CAPABILITIES } from './hosted-sandbox-driver'
import { decideHostedLeaseAction, type HostedActivitySample } from './hosted-session-lease-policy'
@@ -102,7 +102,7 @@ describe('hosted session lease scheduler', () => {
expect(await scheduler.tick()).toEqual({ kind: 'rollover', reason: 'forced' })
expect(reconnect).toHaveBeenCalledTimes(1)
expect(reconnect.mock.calls[0]?.[0].session.startedAt).toBe(now)
expect(resumeAgents).toHaveBeenCalledWith(created.session.deadlineAt)
expect(resumeAgents).toHaveBeenCalledWith(created.session.deadlineAt, null)
// The new session is far from its cap, so the next tick does nothing drastic.
expect((await scheduler.tick()).kind).not.toBe('rollover')
})
@@ -150,14 +150,15 @@ describe('agent auto-resume after rollover', () => {
if (method === 'agentSession.restartResumable') {
return {
sessions: [
{ sessionId: 'old', recordedAt: 900 },
{ sessionId: 'a', recordedAt: 1000 },
{ sessionId: 'b', recordedAt: 1500 }
{ sessionId: 'old', recordedAt: T0 - 10 * MIN },
// The host's clock runs 30 s behind this machine's: still this drain's offer.
{ sessionId: 'a', recordedAt: T0 - 30_000 },
{ sessionId: 'b', recordedAt: T0 + 500 }
]
}
}
return { continued: [{ sessionId: 'a' }], skipped: ['b'] }
}, 1000)
}, T0)
expect(calls).toEqual([
{ method: 'agentSession.restartResumable', params: undefined },
{ method: 'agentSession.restartContinue', params: { sessionIds: ['a', 'b'] } }
@@ -178,3 +179,33 @@ describe('agent auto-resume after rollover', () => {
expect(call).toHaveBeenCalledTimes(1)
})
})
describe('agent auto-resume with a pre-drain offer snapshot', () => {
it('continues only offers the host did not already hold, whatever the clocks say', async () => {
const offers = [
{ sessionId: 'left-for-user', recordedAt: T0 + 5_000 },
{ sessionId: 'this-drain', recordedAt: T0 - 9 * MIN }
]
const calls: unknown[] = []
const call = async (method: string, params?: Record<string, unknown>) => {
calls.push({ method, params })
return method === 'agentSession.restartResumable' ? { sessions: offers } : { continued: [{}] }
}
const before = await listRestartOfferIds(async () => ({ sessions: [offers[0]] }))
expect(before).toEqual(new Set(['left-for-user']))
const outcome = await autoResumeAgentsAfterRollover(call, T0, before)
expect(calls).toContainEqual({
method: 'agentSession.restartContinue',
params: { sessionIds: ['this-drain'] }
})
expect(outcome.offered).toBe(1)
})
it('answers null when the host cannot list its offers', async () => {
await expect(
listRestartOfferIds(async () => {
throw new Error('down')
})
).resolves.toBeNull()
})
})
@@ -21,7 +21,7 @@ import { noteRecipeVmWakePairing } from '../ephemeral-vm/recipe-vm-wake-pairings
import { getEphemeralVmRecipeResultPairingCode } from '../../shared/ephemeral-vm-recipes'
import { environmentHandleFromRecipeResult } from '../environment-providers/environment-provider-recipe-adapter'
import type { HostedEnvironmentProviderDriver } from './hosted-environment-provider'
import { autoResumeAgentsAfterRollover } from './hosted-agent-auto-resume'
import { autoResumeAgentsAfterRollover, listRestartOfferIds } from './hosted-agent-auto-resume'
import { HOSTED_SANDBOX_DRIVER_ID } from './hosted-sandbox-driver'
import { createHostedRuntimeEnvironmentCall } from './hosted-runtime-environment-call'
import type { HostedActivitySample } from './hosted-session-lease-policy'
@@ -137,7 +137,9 @@ export function createHostedSessionRegistry(deps: {
true
)
},
resumeAgents: (drainStartedAt) => autoResumeAgentsAfterRollover(call, drainStartedAt),
snapshotOffers: () => listRestartOfferIds(call),
resumeAgents: (drainStartedAt, offersBeforeDrain) =>
autoResumeAgentsAfterRollover(call, drainStartedAt, offersBeforeDrain),
onRolloverStateChange: (state) => {
const event: HostedSessionStateEvent = { workspaceId: runtime.workspaceId ?? null, state }
broadcast('orcaCloud:sessionState', event)
@@ -5,6 +5,7 @@ import type { HostedSandboxControlPlaneClient } from './hosted-sandbox-control-p
import type { EphemeralVmRuntimeRecord } from '../../shared/ephemeral-vm-runtimes'
import type {
CloudOomEvent,
CloudWakeProgressEvent,
CloudWorkspaceErrorCode,
CloudWorkspaceResult
} from '../../shared/hosted-sandbox/cloud-workspace-api'
@@ -18,6 +19,7 @@ import {
} from '../environment-providers/environment-provider-driver'
import { environmentHandleFromRecipeResult } from '../environment-providers/environment-provider-recipe-adapter'
import { isHostedEntitlementDenied } from './hosted-environment-provider'
import { callRuntimeEnvironment } from '../ipc/runtime-environments/runtime-environment-transport-routing'
import { HOSTED_SANDBOX_DRIVER_ID } from './hosted-sandbox-driver'
import type { HostedWorkspaceService } from './hosted-workspace-service'
@@ -165,6 +167,83 @@ export function getHostedWorkspaceService(): HostedWorkspaceService | null {
return current
}
/** Polls a cheap terminal listing until the woken server answers it (or the deadline passes). */
export async function waitForTerminalsReachable(
userDataPath: string,
environmentId: string,
deadline: number,
stillWanted: () => boolean = () => true
): Promise<boolean> {
while (Date.now() < deadline && stillWanted()) {
const remaining = deadline - Date.now()
// Bounded by the deadline: the probe neither waits past it on a wake in flight nor on a slow server.
const signal = AbortSignal.timeout(Math.max(1, remaining))
const response = await callRuntimeEnvironment(
userDataPath,
environmentId,
'terminal.list',
{ limit: 1 },
Math.min(5_000, Math.max(1, remaining)),
undefined,
undefined,
{ signal, preparation: { recipeVmHoldMs: Math.max(0, remaining) } }
).catch(() => null)
if (response?.ok) {
return stillWanted()
}
await new Promise((resolve) => setTimeout(resolve, 250))
}
return false
}
/**
* After a wake's step answers: `ready` once the runtime record says running AND the woken server
* answers a terminal listing (panes rebind to the restarted server's terminals a moment after
* running; keys typed before that are dropped by design). `failed` when it never answers. Nothing
* when this wake is no longer the current one (a newer wake, a failure or a Sleep took over).
*/
export async function announceWakeUsable(
userDataPath: string,
workspaceId: string,
startedAt: number,
hooks: { isCurrent: () => boolean; emit: (event: Omit<CloudWakeProgressEvent, 'at'>) => void },
timeoutMs = 60_000
): Promise<void> {
const deadline = Date.now() + timeoutMs
const running = (): EphemeralVmRuntimeRecord | null => {
const runtime = findCloudRuntime(userDataPath, workspaceId)?.runtime
return runtime?.status === 'running' ? runtime : null
}
while (Date.now() < deadline && hooks.isCurrent()) {
const runtime = running()
if (runtime) {
const reachable = runtime.runtimeEnvironmentId
? await waitForTerminalsReachable(
userDataPath,
runtime.runtimeEnvironmentId,
deadline,
hooks.isCurrent
)
: true
if (hooks.isCurrent() && running()) {
hooks.emit(
reachable
? { workspaceId, phase: 'ready', source: 'open', startedAt }
: {
workspaceId,
phase: 'failed',
source: 'open',
startedAt,
message: 'The workspace did not answer after waking.'
}
)
}
return
}
await new Promise((resolve) => setTimeout(resolve, 250))
}
}
/** Whether the control plane says this workspace is archived; unknown counts as not archived. */
export async function isArchivedOnControlPlane(
userDataPath: string,
@@ -185,7 +264,9 @@ export async function isArchivedOnControlPlane(
let archiveCheck: ((workspaceId: string) => Promise<boolean>) | null = null
/** Installed with Orca Cloud; the wake path asks it before resuming a cloud workspace. */
export function setCloudArchiveCheck(check: ((workspaceId: string) => Promise<boolean>) | null): void {
export function setCloudArchiveCheck(
check: ((workspaceId: string) => Promise<boolean>) | null
): void {
archiveCheck = check
}
/** False when Orca Cloud is off in this build. */
@@ -18,7 +18,7 @@ import type {
import { applyResumedRuntimePairing } from '../ephemeral-vm/runtime-resume-pairing'
import { noteRecipeVmWakePairing } from '../ephemeral-vm/recipe-vm-wake-pairings'
import { invalidateRuntimeEnvironmentTransport } from '../ipc/runtime-environments/runtime-environments'
import { autoResumeAgentsAfterRollover } from './hosted-agent-auto-resume'
import { autoResumeAgentsAfterRollover, listRestartOfferIds } from './hosted-agent-auto-resume'
import type { HostedSandboxControlPlaneClient } from './hosted-sandbox-control-plane-client'
import { toReady } from './hosted-sandbox-driver'
import { createHostedRuntimeEnvironmentCall } from './hosted-runtime-environment-call'
@@ -28,6 +28,7 @@ import {
failure,
findCloudRuntime,
readCachedCheckouts,
announceWakeUsable,
type CloudRuntime
} from './hosted-workspace-runtime'
@@ -95,17 +96,11 @@ export function installHostedWorkspaceService(deps: {
}
/** The step answers before the window re-pairs; `ready` fires once the runtime record says running. */
async function announceReadyWhenRunning(workspaceId: string, startedAt: number): Promise<void> {
const deadline = Date.now() + 60_000
while (Date.now() < deadline && wakes.has(workspaceId)) {
const runtime = findCloudRuntime(deps.userDataPath, workspaceId)?.runtime
if (runtime?.status === 'running') {
service.emitWakeProgress({ workspaceId, phase: 'ready', source: 'open', startedAt })
return
}
await new Promise((resolve) => setTimeout(resolve, 250))
}
}
const announceReadyWhenRunning = (workspaceId: string, startedAt: number): Promise<void> =>
announceWakeUsable(deps.userDataPath, workspaceId, startedAt, {
isCurrent: () => wakes.get(workspaceId)?.startedAt === startedAt,
emit: (event) => service.emitWakeProgress(event)
})
const service: HostedWorkspaceService = {
emitWakeProgress(event) {
@@ -213,6 +208,11 @@ export function installHostedWorkspaceService(deps: {
source: 'restart',
startedAt
})
const call = found.runtime.runtimeEnvironmentId
? createHostedRuntimeEnvironmentCall(deps.userDataPath, found.runtime.runtimeEnvironmentId)
: null
// Offers the host already holds were left for the user; only this restart's are continued.
const offersBeforeDrain = call ? await listRestartOfferIds(call) : null
// The scheduler holds the old session's deadline and would warn/roll over the new one; a fresh one starts on sync.
deps.stopSession(found.sandboxId)
try {
@@ -231,14 +231,10 @@ export function installHostedWorkspaceService(deps: {
source: 'restart',
startedAt
})
if (found.runtime.runtimeEnvironmentId) {
if (call) {
// Same continuation as a rollover: chats the restart interrupted carry on by themselves.
const call = createHostedRuntimeEnvironmentCall(
deps.userDataPath,
found.runtime.runtimeEnvironmentId
)
void autoResumeAgentsAfterRollover(call, startedAt).catch((error: unknown) =>
log(`[orca-cloud] auto-resume after restart failed: ${String(error)}`)
void autoResumeAgentsAfterRollover(call, startedAt, offersBeforeDrain).catch(
(error: unknown) => log(`[orca-cloud] auto-resume after restart failed: ${String(error)}`)
)
}
return { ok: true, value: null }
@@ -0,0 +1,49 @@
import { describe, expect, it, vi } from 'vitest'
const runtimes: { status: string; runtimeEnvironmentId?: string }[] = []
let listOk = true
vi.mock('../ipc/runtime-environments/runtime-environment-transport-routing', () => ({
callRuntimeEnvironment: vi.fn(async () =>
listOk ? { ok: true, result: {} } : { ok: false, error: { code: 'x', message: 'x' } }
)
}))
vi.mock('../../shared/ephemeral-vm-runtime-store', () => ({
listEphemeralVmRuntimes: () =>
runtimes.map((r) => ({
...r,
workspaceId: 'ws',
recipe: { provider: 'orca-cloud' },
recipeResult: {
schemaVersion: 1,
pairingCode: 'x',
projectRoot: '/p',
orcaCloud: { sandboxId: 'sbx_1' }
}
}))
}))
vi.mock('../environment-providers/environment-provider-recipe-adapter', () => ({
environmentHandleFromRecipeResult: () => ({ driverState: { sandboxId: 'sbx_1' } })
}))
describe('announceWakeUsable', () => {
it('emits ready once running and reachable', async () => {
const { announceWakeUsable } = await import('./hosted-workspace-runtime')
runtimes.splice(0, runtimes.length, { status: 'running', runtimeEnvironmentId: 'env' })
listOk = true
const emit = vi.fn()
await announceWakeUsable('/u', 'ws', 1, { isCurrent: () => true, emit }, 2_000)
expect(emit).toHaveBeenCalledWith(expect.objectContaining({ phase: 'ready', startedAt: 1 }))
})
it('emits failed when the server never answers, and nothing for a superseded wake', async () => {
const { announceWakeUsable } = await import('./hosted-workspace-runtime')
runtimes.splice(0, runtimes.length, { status: 'running', runtimeEnvironmentId: 'env' })
listOk = false
const emit = vi.fn()
await announceWakeUsable('/u', 'ws', 2, { isCurrent: () => true, emit }, 600)
expect(emit).toHaveBeenCalledWith(expect.objectContaining({ phase: 'failed' }))
const stale = vi.fn()
await announceWakeUsable('/u', 'ws', 3, { isCurrent: () => false, emit: stale }, 600)
expect(stale).not.toHaveBeenCalled()
})
})
@@ -13,7 +13,7 @@ import type {
RecipeVmOpening,
RecipeVmState
} from '../../runtime/workspaces/sleep/recipe-vm-operations'
import { prepareRecipeVm } from './runtime-environment-recipe-vm-wake'
import { holdForInFlightRecipeVmWake, prepareRecipeVm } from './runtime-environment-recipe-vm-wake'
import { RemoteRuntimeClientError } from '../../../shared/remote-runtime/remote-runtime-client-error'
import {
forgetRecipeVmWakePairingsForTests,
@@ -131,6 +131,65 @@ describe('a request to a recipe VM environment', () => {
}
)
it('holds a request that does not open the workspace for an in-flight wake, outside its call slot', async () => {
recordVm('running')
ownerState = 'waking'
const awaitWake = vi.fn(async () => 'up' as const)
registerRecipeVmWaker({
state,
noteOpened: vi.fn(),
holdOpen,
awaitWake,
openEnvironment: async () => 'up'
})
await holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'terminal.list', 5_000)
expect(awaitWake).toHaveBeenCalledWith('workspace-of-vm-1', false, 5_000, undefined)
// Bounded by the caller's hold (0 = never wait) and never for an open (that wakes in its slot).
await holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'terminal.list', 5_000, {
preparation: { recipeVmHoldMs: 0 }
})
await holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'worktree.activate', 5_000)
expect(awaitWake).toHaveBeenCalledTimes(1)
expect(holdOpen).not.toHaveBeenCalled()
// Inside the slot the request is still answered at once from the owner's state.
await expect(codeOf('none')).resolves.toBe('runtime_environment_waking')
})
it('caps the hold at the in-flight maximum and does not wait on a VM that is not waking', async () => {
recordVm('running')
const awaitWake = vi.fn(async () => 'up' as const)
registerRecipeVmWaker({
state,
noteOpened: vi.fn(),
holdOpen,
awaitWake,
openEnvironment: async () => 'up'
})
await holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'terminal.list', 600_000)
expect(awaitWake).not.toHaveBeenCalled()
ownerState = 'waking'
await holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'terminal.list', 600_000)
expect(awaitWake).toHaveBeenCalledWith('workspace-of-vm-1', false, 60_000, undefined)
})
it('reports the pairing a wake it waited through left, for the caller fence', async () => {
recordVm('running')
ownerState = 'waking'
registerRecipeVmWaker({
state,
noteOpened: vi.fn(),
holdOpen,
awaitWake: async () => {
noteRecipeVmWakePairing(VM_ENVIRONMENT.id, 9, true)
return 'up' as const
},
openEnvironment: async () => 'up'
})
await expect(
holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'terminal.list', 5_000)
).resolves.toEqual({ before: VM_ENVIRONMENT, wokenPairingRevision: 9 })
})
// A phone gives up after 30 seconds, so its open is held for less.
// A Disconnect made before the open holds: no client's open wakes the VM behind that server.
@@ -14,6 +14,7 @@ import {
RUNTIME_ENVIRONMENT_WAKE_FAILED_MESSAGE,
RUNTIME_ENVIRONMENT_WAKING_ERROR_CODE,
RUNTIME_ENVIRONMENT_WAKING_MESSAGE,
recipeVmWakeIntentFor,
type RecipeVmWakeIntent
} from '../../../shared/remote-runtime/recipe-vm-wake'
import { getRecipeVmWaker } from '../../runtime/workspaces/sleep/recipe-vm-waker-registry'
@@ -117,6 +118,46 @@ export async function prepareRecipeVm(
return UNCHANGED
}
/** How long a request that does not open the workspace waits for a wake already in flight. */
export const RECIPE_VM_IN_FLIGHT_WAKE_HOLD_MS = 60_000
/**
* Before a request that does not open the workspace takes its call slot: while the VM's wake is
* already underway (often just reconnecting), wait for it, up to `holdMs`, instead of being refused
* and never retried. Never starts a wake and never refuses: the request's own preparation decides
* what happens next (it is refused there if the wake failed or is still running).
*/
export async function awaitRecipeVmInFlightWake<E extends RecipeVmEnvironment>(
userDataPath: string,
environment: E,
holdMs: number,
signal?: AbortSignal
): Promise<HeldRecipeVmWake<E> | null> {
const waker = getRecipeVmWaker()
if (holdMs <= 0 || !waker?.awaitWake) {
return null
}
const vm = recipeVmOf(userDataPath, environment)
if (!vm || waker.state(vm.workspaceId, vm.recordSuspended) !== 'waking') {
return null
}
const before = latestRecipeVmWake(environment.id)
await waker.awaitWake(vm.workspaceId, vm.recordSuspended, holdMs, signal).catch(() => undefined)
const woken = latestRecipeVmWake(environment.id)
// The wake this request waited through re-paired the server: the caller's fence names the pairing
// before it, exactly as for a request that woke the VM itself.
return woken !== before
? { before: environment, wokenPairingRevision: woken?.pairingRevision ?? null }
: null
}
/** A wake a request waited through before taking its call slot: the record from before it and
* the pairing it left. */
export type HeldRecipeVmWake<E = RecipeVmEnvironment> = {
before: E
wokenPairingRevision: number | null
}
/** The pairing revision a recipe VM wake this request waited for left its server on, if any. */
export type RecipeVmPreparation = { wokenPairingRevision: number | null }
const UNCHANGED: RecipeVmPreparation = { wokenPairingRevision: null }
@@ -155,3 +196,27 @@ function refusal(outcome: Exclude<RecipeVmOpening, 'up'> | 'asleep'): RemoteRunt
)
}
}
/** A request that does not open the workspace waits (bounded by its own timeout and hold) for a wake
* already in flight. Called before the request takes a per-environment call slot, so held
* background requests never block an open. */
export function holdForInFlightRecipeVmWake<E extends RecipeVmEnvironment>(
userDataPath: string,
environment: E,
method: string,
timeoutMs: number | undefined,
options?: { preparation?: { recipeVmHoldMs?: number }; signal?: AbortSignal }
): Promise<HeldRecipeVmWake<E> | null> {
if (recipeVmWakeIntentFor(method) !== 'none') {
return Promise.resolve(null)
}
const holdMs = Math.min(
RECIPE_VM_IN_FLIGHT_WAKE_HOLD_MS,
options?.preparation?.recipeVmHoldMs ?? Infinity,
timeoutMs ?? DEFAULT_CALL_TIMEOUT_MS
)
return awaitRecipeVmInFlightWake(userDataPath, environment, holdMs, options?.signal)
}
/** The transport's default call timeout, which also bounds the hold for calls that set none. */
const DEFAULT_CALL_TIMEOUT_MS = 15_000
@@ -56,3 +56,46 @@ describe('a request whose own recipe VM wake re-paired its server', () => {
expect((await fence(1, 'before')).failure).toMatchObject({ ok: false })
})
})
describe('a fenced request that waited through a wake already in flight', () => {
beforeEach(() => {
prepare.mockClear()
// By the time the request takes its slot the record already holds the new pairing; its own
// preparation woke nothing.
prepared.current = { environment: AFTER_WAKE, before: AFTER_WAKE, wokenPairingRevision: null }
})
const heldFence = (pairingRevision: number, wokenPairingRevision: number | null) =>
prepareFencedRuntimeRequest(
'/user-data',
'env-vm',
'terminal.list',
undefined,
{ pairingRevision, runtimeId: 'before' },
undefined,
{ before: BEFORE as never, wokenPairingRevision }
)
it('passes the pre-wake fence and then expects the pairing that wake made', async () => {
await expect(heldFence(1, 2)).resolves.toMatchObject({
failure: null,
expected: { pairingRevision: 2, runtimeId: undefined }
})
})
it('still refuses a caller fenced to an older pairing', async () => {
expect((await heldFence(0, 2)).failure).toMatchObject({ ok: false })
expect(prepare).not.toHaveBeenCalled()
})
it('without the held wake, the same call is refused as changed', async () => {
const result = await prepareFencedRuntimeRequest(
'/user-data',
'env-vm',
'terminal.list',
undefined,
{ pairingRevision: 1, runtimeId: 'before' }
)
expect(result.failure).toMatchObject({ ok: false })
})
})
@@ -22,14 +22,17 @@ export async function prepareFencedRuntimeRequest(
method: string,
preparation: ManagedRuntimeCallPreparation | undefined,
expected: Expected,
signal?: AbortSignal
signal?: AbortSignal,
/** A recipe VM wake this request waited through before its call slot: its fence is checked
* against the record from before that wake, as for a wake the request made itself. */
heldWake?: { before: KnownRuntimeEnvironment; wokenPairingRevision: number | null }
): Promise<
| { failure: Rpc.RuntimeRpcResponse<never> }
| { failure: null; environment: KnownRuntimeEnvironment; expected: Expected }
> {
// A stale caller is refused before its request can wake (and re-pair) a VM.
const stale = runtimeEnvironmentRevisionFailure(
resolveEnvironment(userDataPath, environmentId),
heldWake?.before ?? resolveEnvironment(userDataPath, environmentId),
expected.pairingRevision,
method,
expected.runtimeId
@@ -44,7 +47,10 @@ export async function prepareFencedRuntimeRequest(
method,
signal
)
const { environment, wokenPairingRevision: woken } = prepared
const { environment } = prepared
const woken = prepared.wokenPairingRevision ?? heldWake?.wokenPairingRevision ?? null
const before =
prepared.wokenPairingRevision !== null ? prepared.before : (heldWake?.before ?? prepared.before)
const failure =
woken === null
? runtimeEnvironmentRevisionFailure(
@@ -54,7 +60,7 @@ export async function prepareFencedRuntimeRequest(
expected.runtimeId
)
: (runtimeEnvironmentRevisionFailure(
prepared.before,
before,
expected.pairingRevision,
method,
expected.runtimeId
@@ -15,6 +15,7 @@ import {
resolveManagedRuntimeEnvironment,
type ManagedRuntimeCallPreparation
} from './runtime-environment-managed-tunnel'
import { markEnvironmentUsedFromResponse } from './runtime-environment-usage-marking'
import { recordRuntimeEnvironmentUsage } from '../../../shared/remote-runtime/host-store/runtime-environment-usage-record'
import type * as Rpc from '../../../shared/runtime-rpc-envelope'
import '../../../shared/remote-runtime/requests/remote-runtime-request-socket'
@@ -23,6 +24,7 @@ import type { RemoteRuntimeTransportSubscription } from '../../../shared/remote-
import { withRuntimeUnreachableHint } from './runtime-environment-unreachable-hint'
import { decodeRuntimeEnvironmentSubscriptionEvents } from './runtime-environment-subscription-events'
import { enqueueRuntimeCall } from './runtime-environment-call-queue'
import { holdForInFlightRecipeVmWake } from './runtime-environment-recipe-vm-wake'
import {
sendRemoteRuntimeConnectionRequestAbortable,
sendRemoteRuntimeRequestAbortable
@@ -80,6 +82,13 @@ export async function callRuntimeEnvironment(
)
}
const environment = resolveEnvironment(userDataPath, selector)
const held = await holdForInFlightRecipeVmWake(
userDataPath,
environment,
method,
timeoutMs,
options
)
// Connect failures reject; retain the dispatched endpoint for hints because queued re-pairing can change it.
let endpoint = getPreferredPairedRuntimeConnection(environment).endpoint
try {
@@ -96,7 +105,8 @@ export async function callRuntimeEnvironment(
pairingRevision: expectedEnvironmentPairingRevision,
runtimeId: options?.expectedEnvironmentRuntimeId
},
options?.signal
options?.signal,
held ?? undefined
)
if (prepared.failure) {
return prepared.failure
@@ -290,18 +300,6 @@ export async function subscribeRuntimeEnvironment(
}
}
function markEnvironmentUsedFromResponse(
userDataPath: string,
environmentId: string,
response: Rpc.RuntimeRpcResponse<unknown>
): void {
if (response.ok === true) {
recordRuntimeEnvironmentUsage(userDataPath, environmentId, {
runtimeId: response._meta.runtimeId
})
}
}
function shouldUseCachedRequestConnection(method: string): boolean {
return method === 'terminal.send' || method === 'terminal.updateViewport'
}
@@ -0,0 +1,15 @@
import { recordRuntimeEnvironmentUsage } from '../../../shared/remote-runtime/host-store/runtime-environment-usage-record'
import type * as Rpc from '../../../shared/runtime-rpc-envelope'
/** A successful answer records which server process served the environment. */
export function markEnvironmentUsedFromResponse(
userDataPath: string,
environmentId: string,
response: Rpc.RuntimeRpcResponse<unknown>
): void {
if (response.ok === true) {
recordRuntimeEnvironmentUsage(userDataPath, environmentId, {
runtimeId: response._meta.runtimeId
})
}
}
@@ -0,0 +1,30 @@
import { describe, expect, it } from 'vitest'
import { awaitInFlightWake, WorkspaceSleepRequests, type VmOperation } from './recipe-vm-operations'
describe('awaitInFlightWake', () => {
it('answers at once for a VM that is up or asleep, without starting anything', async () => {
const sleeps = new WorkspaceSleepRequests()
await expect(awaitInFlightWake([], false, sleeps, 'w', 1000)).resolves.toBe('up')
await expect(awaitInFlightWake([], true, sleeps, 'w', 1000)).resolves.toBe('asleep')
})
it('waits for the wake already in flight', async () => {
let finish = (): void => {}
const settled = new Promise<void>((resolve) => {
finish = resolve
})
const pending: VmOperation[] = [{ kind: 'wake', settled }]
const answer = awaitInFlightWake(pending, false, new WorkspaceSleepRequests(), 'w', 5000)
finish()
await expect(answer).resolves.toBe('up')
})
it('ends slept when a Sleep is asked for while it waits, and waking when the hold runs out', async () => {
const sleeps = new WorkspaceSleepRequests()
const never: VmOperation[] = [{ kind: 'wake', settled: new Promise(() => {}) }]
const slept = awaitInFlightWake(never, false, sleeps, 'w', 5000)
sleeps.noteRequested('w')
await expect(slept).resolves.toBe('slept')
await expect(awaitInFlightWake(never, false, sleeps, 'w', 5)).resolves.toBe('waking')
})
})
@@ -124,3 +124,26 @@ export async function holdOpenUntilUp(
stopAbortWatch()
}
}
/** Holds a request that does not open the workspace while the wake already in flight settles; a VM
* that is not waking answers at once with where it stands. Never starts a wake. */
export async function awaitInFlightWake(
pending: readonly VmOperation[],
recordSuspended: boolean,
sleepRequests: WorkspaceSleepRequests,
worktreeId: string,
timeoutMs: number,
signal?: AbortSignal
): Promise<RecipeVmOpening | 'asleep'> {
signal?.throwIfAborted()
const state = vmStateOf(pending, recordSuspended)
if (state !== 'waking') {
return state === 'up' ? 'up' : 'asleep'
}
const sleep = sleepRequests.next(worktreeId)
try {
return await holdOpenUntilUp(pendingVmWake(pending), sleep.requested, timeoutMs, signal)
} finally {
sleep.stop()
}
}
@@ -15,6 +15,14 @@ export type RecipeVmWaker = {
timeoutMs: number,
signal?: AbortSignal
) => Promise<RecipeVmOpening>
/** Holds a request that does not open the workspace while a wake already in flight finishes;
* never starts one. */
awaitWake?: (
workspaceId: string,
recordSuspended: boolean,
timeoutMs: number,
signal?: AbortSignal
) => Promise<RecipeVmOpening | 'asleep'>
/** Holds an open of the workspace behind a saved environment (id or name) for the CLI. */
openEnvironment: (selector: string) => Promise<RecipeVmOpening>
}
@@ -6,7 +6,7 @@ import type {
WorkspaceSleepRequest
} from '../../../../shared/workspace-sleep'
import { WorkspaceSleepCoordinator } from './workspace-sleep-runtime'
import type { RecipeVmOpening, RecipeVmState } from './recipe-vm-operations'
import { awaitInFlightWake, type RecipeVmOpening, type RecipeVmState } from './recipe-vm-operations'
import { dropStrandedSleepRecord } from '../../terminal/stop/worktree-sleep-resume-records'
import type { RestoredPane } from '../../terminal/recovery/restored-pane-start'
import type { SleepingAgentSessionRecord } from '../../../../shared/agent-session/lifecycle/agent-session-resume'
@@ -166,6 +166,18 @@ export abstract class RuntimeWorkspaceSleepOperations extends OrcaRuntimeWithRes
return this.workspaceSleep.holdOpen(worktreeId, recordSuspended, timeoutMs, signal)
}
/** Holds a request that does not open this workspace while its in-flight wake finishes. */
awaitWorkspaceVmWake(
worktreeId: string,
recordSuspended: boolean,
timeoutMs: number,
signal?: AbortSignal
): Promise<RecipeVmOpening | 'asleep'> {
const pending = this.workspaceSleep.pendingVmOperations.get(worktreeId) ?? []
const sleeps = this.workspaceSleep.sleepRequests
return awaitInFlightWake(pending, recordSuspended, sleeps, worktreeId, timeoutMs, signal)
}
/** The window opened a slept pane; main resumes it and the window attaches. */
resumeSleptPane(pane: RestoredPane, automatic?: boolean): Promise<boolean> {
return this.workspaceSleep.resumePane(pane, automatic)
@@ -95,8 +95,9 @@ export class WorkspaceSleepCoordinator {
private readonly reopenings = new Map<string, number>()
private readonly vmOperations = new Map<string, Promise<void>>()
/** The VM operations asked for per workspace, in order, until each settles. */
private readonly pendingVmOperations = new Map<string, VmOperation[]>()
private readonly sleepRequests = new WorkspaceSleepRequests()
/** Read by the in-flight wake hold (awaitInFlightWake); written only here. */
readonly pendingVmOperations = new Map<string, VmOperation[]>()
readonly sleepRequests = new WorkspaceSleepRequests()
constructor(private readonly runtime: WorkspaceSleepRuntime) {}
@@ -128,13 +128,17 @@ export function initializeMainProcessRuntime(): OrcaRuntimeService {
onRuntimeEnvironmentStatusPublished(trackRuntimeEnvironmentVerifiedTransitions(listener))
},
workspaceSleepPlatform: {
// Sleep's own calls and observations never wait on a wake in flight.
callEnvironment: (environmentId, method, params, timeoutMs) =>
callRuntimeEnvironment(
getRuntimeEnvironmentRegistryRoot(),
environmentId,
method,
params,
timeoutMs
timeoutMs,
undefined,
undefined,
{ preparation: { recipeVmHoldMs: 0 } }
),
ownsVm: (workspaceId) =>
findWorkspaceEphemeralVmRuntime(app.getPath('userData'), workspaceId) !== undefined,
@@ -166,6 +170,8 @@ export function initializeMainProcessRuntime(): OrcaRuntimeService {
noteOpened: (workspaceId) => runtime.noteWorkspaceReopened(workspaceId),
holdOpen: (workspaceId, suspended, timeoutMs, signal) =>
runtime.holdWorkspaceVmOpen(workspaceId, suspended, timeoutMs, signal),
awaitWake: (workspaceId, suspended, timeoutMs, signal) =>
runtime.awaitWorkspaceVmWake(workspaceId, suspended, timeoutMs, signal),
openEnvironment: (selector) => {
const root = getRuntimeEnvironmentRegistryRoot()
return openRecipeVmEnvironment(root, resolveEnvironment(root, selector))
@@ -127,7 +127,9 @@ export function CloudWakingOverlay({
: wake.source === 'restart'
? t('orcaCloudWorkspace.announce.restarting', 'Restarting {{name}}.', { name })
: wake.source === 'unarchive'
? t('orcaCloudWorkspace.announce.unarchiving', 'Restoring {{name}} from archive.', { name })
? t('orcaCloudWorkspace.announce.unarchiving', 'Restoring {{name}} from archive.', {
name
})
: t('orcaCloudWorkspace.announce.waking', 'Waking {{name}}.', { name })
}
>
@@ -44,7 +44,12 @@ const running: HostedWorkspaceStatus = {
archive: null,
checkedAt: NOW
}
const asleep: HostedWorkspaceStatus = { ...running, state: 'sleeping', metrics: null, uptimeMs: null }
const asleep: HostedWorkspaceStatus = {
...running,
state: 'sleeping',
metrics: null,
uptimeMs: null
}
const entry = (
status: HostedWorkspaceStatus | null,
over: Partial<CloudStatusEntry> = {}
@@ -100,7 +105,11 @@ export const panelUnresponsive = defineFixture({
export const panelRestarting = defineFixture({
width: 300,
render: () => (
<Panel state="unresponsive" value={entry({ ...running, state: 'unresponsive' })} pending="restart" />
<Panel
state="unresponsive"
value={entry({ ...running, state: 'unresponsive' })}
pending="restart"
/>
)
})
@@ -298,7 +307,7 @@ function IdleToast(): React.JSX.Element {
return (
<>
<div className="h-[200px] bg-background" />
<Toaster position="top-left" toastOptions={{ className: "font-sans text-sm" }} />
<Toaster position="top-left" toastOptions={{ className: 'font-sans text-sm' }} />
</>
)
}
@@ -109,7 +109,9 @@ export function runArchiveCloudWorkspace(
}, options.undoMs ?? ARCHIVE_UNDO_MS)
toast(
options.name
? translate('orcaCloudWorkspace.archivingNamed', 'Archiving {{name}}…', { name: options.name })
? translate('orcaCloudWorkspace.archivingNamed', 'Archiving {{name}}…', {
name: options.name
})
: translate('orcaCloudWorkspace.archiving', 'Archiving workspace…'),
{
id: toastId,
@@ -96,7 +96,9 @@ export function CloudWorkspaceOfflineView(props: CloudOfflineViewProps): React.J
</TabsTrigger>
<TabsTrigger value="changes">
{t('components.orcaCloud.offline.tabChanges', 'Changes')}
{changeCount > 0 ? <span className="text-muted-foreground">{changeCount}</span> : null}
{changeCount > 0 ? (
<span className="text-muted-foreground">{changeCount}</span>
) : null}
</TabsTrigger>
</TabsList>
<TabsContent value="chat" className="flex min-h-0 flex-1 flex-col pt-3">
@@ -32,10 +32,7 @@ export function WorktreeCloudTransferMenuItem({
return (
<DropdownMenuItem onSelect={() => void runOrcaCloudTransferToLocal(worktreeId)}>
<Laptop className="size-3.5" />
{t(
'auto.components.sidebar.WorktreeContextMenu.moveToThisComputer',
'Move to this computer'
)}
{t('auto.components.sidebar.WorktreeContextMenu.moveToThisComputer', 'Move to this computer')}
</DropdownMenuItem>
)
}
@@ -1,3 +1,4 @@
import { markInterruptedTurnContinued } from './startup/cold-restore-resume-startup'
import { paneResumeRecord } from './startup/pane-resume-record'
import { useAppStore } from '@/store/index'
import type { PtyReplayDataMeta } from '../pty-transport/pty-transport-types'
@@ -34,6 +35,7 @@ export function bindDeferredColdRestoreAndSnapshot(session: ConnectPanePtySessio
if (!startup || startup.resumeSource?.cancelled) {
return false
}
markInterruptedTurnContinued(startup.interruptedTurnKey)
const state = useAppStore.getState()
if (!session.pendingStartupCommand && session.shouldDeliverStartupViaTerminalPaste) {
session.pendingStartupCommand = {
@@ -1,5 +1,6 @@
import {
NOT_REOPENED,
markInterruptedTurnContinued,
reopenPaneAfterMissingReattach
} from '../startup/cold-restore-resume-startup'
import { writeStartupCwdFallbackNotice } from '../startup/startup-cwd-fallback-notice'
@@ -134,6 +135,10 @@ function connectDeferredSessionReattach(
)
session.beginReattachLiveDataDeferral(outputCallbacks.generation)
session.transportConnectInFlightSince = Date.now()
// The command goes to the server now: a `continue` it carries is spent.
if (coldRestoreStartup?.command) {
markInterruptedTurnContinued(coldRestoreStartup.interruptedTurnKey)
}
const reattachPromise = session.transport.connect({
url: '',
cols: session.cols,
@@ -0,0 +1,62 @@
import { describe, expect, it, vi } from 'vitest'
vi.mock('@/store/index', () => ({ useAppStore: { getState: () => ({}) } }))
describe('shouldContinueInterruptedTurn', () => {
const now = 1_800_000_000_000
const working = (stateStartedAt: number) => ({
state: 'working',
updatedAt: now - 1000,
stateStartedAt
})
it('continues a remote agent with a fresh working status, once per interrupted turn', async () => {
const { shouldContinueInterruptedTurn, interruptedTurnKey, markInterruptedTurnContinued } =
await import('./cold-restore-resume-startup')
const ctx = { paneKey: 'p1', hasSleepingRecord: false, now }
expect(shouldContinueInterruptedTurn(working(1), 'env-1', ctx)).toBe(true)
// Building (and discarding) the startup again does not spend the turn.
expect(shouldContinueInterruptedTurn(working(1), 'env-1', ctx)).toBe(true)
// Once the resume launched, a retried cold restore of the same turn does not send it again.
markInterruptedTurnContinued(interruptedTurnKey(working(1), 'p1'))
expect(shouldContinueInterruptedTurn(working(1), 'env-1', ctx)).toBe(false)
// A later interrupted turn does.
expect(shouldContinueInterruptedTurn(working(2), 'env-1', ctx)).toBe(true)
})
it('never continues after a deliberate sleep, for stale or non-working status, or locally', async () => {
const { shouldContinueInterruptedTurn } = await import('./cold-restore-resume-startup')
expect(
shouldContinueInterruptedTurn(working(10), 'env-1', {
paneKey: 'p2',
hasSleepingRecord: true,
now
})
).toBe(false)
expect(
shouldContinueInterruptedTurn(
{ state: 'working', updatedAt: now - 31 * 60_000, stateStartedAt: 11 },
'env-1',
{
paneKey: 'p3',
hasSleepingRecord: false,
now
}
)
).toBe(false)
expect(
shouldContinueInterruptedTurn(
{ state: 'waiting', updatedAt: now, stateStartedAt: 12 },
'env-1',
{ paneKey: 'p4', hasSleepingRecord: false, now }
)
).toBe(false)
expect(
shouldContinueInterruptedTurn(working(13), null, {
paneKey: 'p5',
hasSleepingRecord: false,
now
})
).toBe(false)
})
})
@@ -6,7 +6,10 @@ import { normalizeExecutionHostId } from '../../../../../../shared/execution-hos
import { parseWorkspaceKey } from '../../../../../../shared/workspace-scope'
import { useAppStore } from '@/store/index'
import { createNonSecureContextUuid as createBrowserUuid } from '../../../../../../shared/non-secure-context-uuid'
import { buildAgentResumeStartupPlan } from '../../../../../../shared/agent-launch/startup/tui-agent-resume-startup'
import {
INTERRUPTED_TURN_CONTINUE_PROMPT,
buildAgentResumeStartupPlan
} from '../../../../../../shared/agent-launch/startup/tui-agent-resume-startup'
import {
resolveRuntimeHostReportedPlatform,
resolveRuntimeHostTerminalWindowsShell,
@@ -24,6 +27,7 @@ import {
normalizeAgentProviderSession
} from '../../../../../../shared/agent-session/lifecycle/agent-session-resume'
import { AGENT_STATUS_STALE_AFTER_MS } from '../../../../../../shared/agent-status/contracts/agent-status-freshness'
import type { HeldPaneReopenOutcome } from '../../session-restore/newer-daemon/newer-daemon-hold-state'
import type { ColdRestoreAgentResumeStartup } from './fresh-spawn-types'
@@ -31,6 +35,45 @@ import type { ConnectPanePtySession } from '../connect-pane-pty-session'
export const NOT_REOPENED: HeldPaneReopenOutcome = { reopened: false, agentResumed: false }
/** Interrupted turns already continued, so a retried cold restore never sends `continue` twice. */
const continuedInterruptedTurns = new Set<string>()
/** Identity of the turn an agent was in when its host went away. */
export function interruptedTurnKey(
entry: { updatedAt: number; stateStartedAt?: number },
paneKey: string
): string {
return `${paneKey}\u0000${entry.stateStartedAt ?? entry.updatedAt}`
}
/** Called when a resume that carries `continue` actually launches (startups are often built and
* discarded, e.g. on every reattach check). */
export function markInterruptedTurnContinued(key: string | undefined): void {
if (key) {
continuedInterruptedTurns.add(key)
}
}
/**
* A cold-restored agent on a remote host whose last status was working lost its turn when the host
* restarted (reboot, crash, rollover): carry it on. Only with fresh evidence (a recent working
* status), never after a deliberate sleep (that leaves a sleeping record), once per interrupted turn.
* Local panes are left alone: their PTYs outlive app restarts.
*/
export function shouldContinueInterruptedTurn(
entry: { state: string; updatedAt: number; stateStartedAt?: number } | null | undefined,
runtimeEnvironmentId: string | null,
context: { paneKey: string; hasSleepingRecord: boolean; now?: number }
): boolean {
if (!runtimeEnvironmentId || context.hasSleepingRecord || entry?.state !== 'working') {
return false
}
if ((context.now ?? Date.now()) - entry.updatedAt > AGENT_STATUS_STALE_AFTER_MS) {
return false
}
return !continuedInterruptedTurns.has(interruptedTurnKey(entry, context.paneKey))
}
type AppState = ReturnType<typeof useAppStore.getState>
/** The agent session a cold restore of this pane would resume; read-only, so pre-spawn checks can share it. */
@@ -109,6 +152,12 @@ export function bindBuildColdRestoreAgentResumeStartup(session: ConnectPanePtySe
session.executionHostId
)
})
const continueTurn =
entry !== null &&
shouldContinueInterruptedTurn(entry, session.transport.getRuntimeEnvironmentId?.() ?? null, {
paneKey: session.cacheKey,
hasSleepingRecord: Boolean(sleepingRecord)
})
const startupPlan = buildAgentResumeStartupPlan({
agent,
providerSession,
@@ -117,7 +166,10 @@ export function bindBuildColdRestoreAgentResumeStartup(session: ConnectPanePtySe
platform: resumeTarget.platform,
agentHostPlatform: resumeTarget.agentHostPlatform,
shell: resumeTarget.shell,
launchCwd: session.worktree?.path
launchCwd: session.worktree?.path,
// A remote host that restarted under a working agent (reboot, crash, VM rollover) killed it
// mid-turn; its last status still says working. Carry the turn on instead of waiting.
continuePrompt: continueTurn ? INTERRUPTED_TURN_CONTINUE_PROMPT : null
})
if (!startupPlan) {
return null
@@ -151,6 +203,9 @@ export function bindBuildColdRestoreAgentResumeStartup(session: ConnectPanePtySe
: {})
},
launchToken: coldRestoreLaunchToken,
...(continueTurn && entry
? { interruptedTurnKey: interruptedTurnKey(entry, session.cacheKey) }
: {}),
useLiveEntry: Boolean(useLiveEntry),
hasSleepingRecord: Boolean(sleepingRecord),
sleepingRecordEntry
@@ -20,6 +20,8 @@ export type FreshSpawnOptions = {
}
export type ColdRestoreAgentResumeStartup = PendingStartupCommand & {
/** Set when the command carries `continue` for an interrupted turn; marked used on launch. */
interruptedTurnKey?: string
agent: ResumableTuiAgent
resumeProviderSession: AgentProviderSessionMetadata
connectionId?: string | null
@@ -14,7 +14,10 @@ import {
} from './orca-cloud-workspace-state'
const NOW = 1_800_000_000_000
const status = (state: HostedWorkspaceStatus['state'], archive: HostedWorkspaceStatus['archive'] = null): HostedWorkspaceStatus => ({
const status = (
state: HostedWorkspaceStatus['state'],
archive: HostedWorkspaceStatus['archive'] = null
): HostedWorkspaceStatus => ({
state,
metrics: null,
uptimeMs: null,
@@ -32,18 +35,39 @@ const wake = (phase: CloudWakeProgressEvent['phase'], at = NOW): CloudWakeProgre
describe('deriveCloudDotState', () => {
it('shows an in-flight wake over everything, including a sleeping desktop record', () => {
expect(deriveCloudDotState({ status: status('sleeping'), wake: wake('waking'), sleeping: true, now: NOW })).toBe('waking')
expect(deriveCloudDotState({ status: status('ready'), wake: wake('connecting'), sleeping: false, now: NOW })).toBe('waking')
expect(
deriveCloudDotState({
status: status('sleeping'),
wake: wake('waking'),
sleeping: true,
now: NOW
})
).toBe('waking')
expect(
deriveCloudDotState({
status: status('ready'),
wake: wake('connecting'),
sleeping: false,
now: NOW
})
).toBe('waking')
})
it('treats a wake with no terminal event for 4 minutes as stale', () => {
expect(
deriveCloudDotState({ status: status('ready'), wake: wake('waking', NOW - 5 * 60_000), sleeping: false, now: NOW })
deriveCloudDotState({
status: status('ready'),
wake: wake('waking', NOW - 5 * 60_000),
sleeping: false,
now: NOW
})
).toBe('running')
})
it('archived beats the desktop sleep state; failed wakes show until stale', () => {
expect(deriveCloudDotState({ status: status('archived'), wake: undefined, sleeping: true, now: NOW })).toBe('archived')
expect(
deriveCloudDotState({ status: status('archived'), wake: undefined, sleeping: true, now: NOW })
).toBe('archived')
expect(
deriveCloudDotState({
status: status('sleeping', { archivedAt: NOW, expiresAt: null }),
@@ -52,16 +76,45 @@ describe('deriveCloudDotState', () => {
now: NOW
})
).toBe('archived')
expect(deriveCloudDotState({ status: status('sleeping'), wake: wake('failed'), sleeping: true, now: NOW })).toBe('failed')
expect(
deriveCloudDotState({
status: status('sleeping'),
wake: wake('failed'),
sleeping: true,
now: NOW
})
).toBe('failed')
})
it('maps control-plane states; the local sleep record only fills in before the first read', () => {
expect(deriveCloudDotState({ status: status('ready'), wake: undefined, sleeping: false, now: NOW })).toBe('running')
expect(deriveCloudDotState({ status: status('unresponsive'), wake: undefined, sleeping: false, now: NOW })).toBe('unresponsive')
expect(deriveCloudDotState({ status: status('ready'), wake: undefined, sleeping: true, now: NOW })).toBe('running')
expect(deriveCloudDotState({ status: null, wake: undefined, sleeping: true, now: NOW })).toBe('sleeping')
expect(deriveCloudDotState({ status: status('provisioning'), wake: undefined, sleeping: false, now: NOW })).toBe('waking')
expect(deriveCloudDotState({ status: null, wake: undefined, sleeping: false, now: NOW })).toBe('unknown')
expect(
deriveCloudDotState({ status: status('ready'), wake: undefined, sleeping: false, now: NOW })
).toBe('running')
expect(
deriveCloudDotState({
status: status('unresponsive'),
wake: undefined,
sleeping: false,
now: NOW
})
).toBe('unresponsive')
expect(
deriveCloudDotState({ status: status('ready'), wake: undefined, sleeping: true, now: NOW })
).toBe('running')
expect(deriveCloudDotState({ status: null, wake: undefined, sleeping: true, now: NOW })).toBe(
'sleeping'
)
expect(
deriveCloudDotState({
status: status('provisioning'),
wake: undefined,
sleeping: false,
now: NOW
})
).toBe('waking')
expect(deriveCloudDotState({ status: null, wake: undefined, sleeping: false, now: NOW })).toBe(
'unknown'
)
})
})
@@ -19,7 +19,10 @@ const status = (state: HostedWorkspaceStatus['state']): HostedWorkspaceStatus =>
checkedAt: 1
})
function stub(getStatus: OrcaCloudWorkspaceApi['getStatus'], ids: string[] = []): OrcaCloudWorkspaceApi {
function stub(
getStatus: OrcaCloudWorkspaceApi['getStatus'],
ids: string[] = []
): OrcaCloudWorkspaceApi {
const bridge = {
getStatus: vi.fn(getStatus),
listCloudWorkspaceIds: vi.fn(async () => ids)
@@ -48,7 +51,13 @@ describe('wake state and archive guard', () => {
it('a failed wake stays until dismissed', async () => {
stub(async () => ({ ok: true, value: status('sleeping') }))
noteCloudWakeProgress({ workspaceId: 'a', phase: 'failed', source: 'open', startedAt: 1, at: 2 })
noteCloudWakeProgress({
workspaceId: 'a',
phase: 'failed',
source: 'open',
startedAt: 1,
at: 2
})
expect(getCloudWorkspaceSnapshot().wake.a?.phase).toBe('failed')
dismissCloudWake('a')
expect(getCloudWorkspaceSnapshot().wake.a).toBeUndefined()
@@ -28,7 +28,7 @@ const ready: HostedWorkspaceStatus = {
archive: null,
checkedAt: 1
}
const ok = <T,>(value: T): CloudWorkspaceResult<T> => ({ ok: true, value })
const ok = <T>(value: T): CloudWorkspaceResult<T> => ({ ok: true, value })
function stubApi(overrides: Partial<OrcaCloudWorkspaceApi> = {}): OrcaCloudWorkspaceApi {
const bridge: OrcaCloudWorkspaceApi = {
@@ -42,7 +42,9 @@ function stubApi(overrides: Partial<OrcaCloudWorkspaceApi> = {}): OrcaCloudWorks
onOom: vi.fn(() => () => {}),
...overrides
}
;(globalThis as unknown as { window: { api: { orcaCloudWorkspace: OrcaCloudWorkspaceApi } } }).window = {
;(
globalThis as unknown as { window: { api: { orcaCloudWorkspace: OrcaCloudWorkspaceApi } } }
).window = {
api: { orcaCloudWorkspace: bridge }
}
return bridge
@@ -66,7 +68,11 @@ describe('orca cloud workspace store', () => {
const bridge = stubApi()
await Promise.all([refreshCloudWorkspaceStatus('a'), refreshCloudWorkspaceStatus('a')])
expect(bridge.getStatus).toHaveBeenCalledTimes(1)
bridge.getStatus = vi.fn(async () => ({ ok: false as const, code: 'unavailable' as const, message: 'down' }))
bridge.getStatus = vi.fn(async () => ({
ok: false as const,
code: 'unavailable' as const,
message: 'down'
}))
await refreshCloudWorkspaceStatus('a')
const entry = getCloudWorkspaceSnapshot().status.a!
expect(entry.status?.state).toBe('ready')
@@ -75,7 +81,13 @@ describe('orca cloud workspace store', () => {
})
it('forgets a workspace the main process says is not cloud', async () => {
stubApi({ getStatus: vi.fn(async () => ({ ok: false as const, code: 'not_cloud' as const, message: 'no' })) })
stubApi({
getStatus: vi.fn(async () => ({
ok: false as const,
code: 'not_cloud' as const,
message: 'no'
}))
})
seedCloudWorkspaceSnapshot({ ids: new Set(['a']) })
await refreshCloudWorkspaceStatus('a')
expect(getCloudWorkspaceSnapshot().ids.has('a')).toBe(false)
@@ -117,7 +129,13 @@ describe('orca cloud workspace store', () => {
it('tracks wake progress (adding the id) and dismisses OOM notices up to the seen count', async () => {
stubApi()
noteCloudWakeProgress({ workspaceId: 'c', phase: 'waking', source: 'message', startedAt: 1, at: 2 })
noteCloudWakeProgress({
workspaceId: 'c',
phase: 'waking',
source: 'message',
startedAt: 1,
at: 2
})
expect(getCloudWorkspaceSnapshot().ids.has('c')).toBe(true)
noteCloudOom({ workspaceId: 'c', oomKills: 1, victim: 'node', at: 3 })
dismissCloudOom('c')
@@ -0,0 +1,43 @@
import { describe, expect, it } from 'vitest'
import { buildAgentResumeStartupPlan } from './tui-agent-resume-startup'
const providerSession = { key: 'session_id', id: 'claude-session-1' } as const
describe('resume with a continue prompt', () => {
const base = { cmdOverrides: {}, platform: 'linux' as const, shell: 'posix' as const }
it('submits the prompt on resume for Claude and Codex', () => {
const claude = buildAgentResumeStartupPlan({
...base,
agent: 'claude',
providerSession,
continuePrompt: 'continue'
})
expect(claude?.launchCommand).toMatch(/'?--resume'? '?claude-session-1'? '?continue'?$/)
const codex = buildAgentResumeStartupPlan({
...base,
agent: 'codex',
providerSession: { key: 'session_id', id: 'codex-1' },
continuePrompt: 'continue'
})
expect(codex?.launchCommand).toMatch(/'?resume'? '?codex-1'? '?continue'?$/)
})
it('leaves other agents and empty prompts unchanged', () => {
const plain = buildAgentResumeStartupPlan({ ...base, agent: 'claude', providerSession })
const blank = buildAgentResumeStartupPlan({
...base,
agent: 'claude',
providerSession,
continuePrompt: ' '
})
expect(blank?.launchCommand).toBe(plain?.launchCommand)
const cursor = buildAgentResumeStartupPlan({
...base,
agent: 'cursor',
providerSession: { key: 'conversation_id', id: 'c-1' },
continuePrompt: 'continue'
})
expect(cursor?.launchCommand).not.toMatch(/continue'?$/)
})
})
@@ -37,6 +37,9 @@ export function buildAgentResumeStartupPlan(args: {
fork?: boolean
/** OS that runs the agent when `platform` only picks quoting; null when unreported. */
agentHostPlatform?: NodeJS.Platform | null
/** First message the resumed session submits, e.g. to carry on a turn its host cut off. Only
* agents whose resume takes a prompt argument (Claude, Codex) send it; others ignore it. */
continuePrompt?: string | null
}): AgentStartupPlan | null {
const resumeArgv = args.fork
? getAgentForkArgv(args.agent, args.providerSession)
@@ -45,10 +48,15 @@ export function buildAgentResumeStartupPlan(args: {
return null
}
// Codex otherwise asks for its old folder, even if deleted (#17745). Keep before resume so argv removal preserves the folder choice.
const argv =
const withCwd =
args.agent === 'codex' && args.resumeInLaunchCwd
? [resumeArgv[0], '-c', 'tui.resume_cwd=current', ...resumeArgv.slice(1)]
: resumeArgv
const continuePrompt = args.continuePrompt?.trim()
const argv =
continuePrompt && !args.fork && RESUME_PROMPT_AGENTS.has(args.agent)
? [...withCwd, continuePrompt]
: withCwd
const shell = resolveStartupShell(args.platform, args.shell)
// Why: Codex restores a resumed thread's saved permissions unless the launch passes one, so
// replaying Orca's launch-time flag would undo a change made inside the session (#10886).
@@ -112,6 +120,12 @@ export function buildAgentResumeStartupPlan(args: {
}
}
/** Agents whose resume command accepts a positional prompt it submits on start. */
const RESUME_PROMPT_AGENTS: ReadonlySet<ResumableTuiAgent> = new Set(['claude', 'codex'])
/** What a resumed agent is told when its host went away mid-turn (restart, reboot, crash). */
export const INTERRUPTED_TURN_CONTINUE_PROMPT = 'continue'
/** WSL panes start at a UNC path while the agent reports its Linux path. */
function isSameFolder(agentCwd: string, launchCwd: string): boolean {
const launchPath = parseWslUncPath(launchCwd)?.linuxPath ?? launchCwd