mirror of
https://github.com/stablyai/orca.git
synced 2026-11-15 18:37:24 +00:00
Merge pull request #77 from nwparker/cloud/w2-wakefix
fix(cloud): wake readiness, wait-for-wake instead of failing, skew-safe agent auto-continue
This commit is contained in:
@@ -24,13 +24,51 @@ const ContinueAnswer = z.looseObject({
|
||||
skipped: z.array(z.unknown()).optional()
|
||||
})
|
||||
|
||||
/** `recordedAt` is the host's clock and `drainStartedAt` this machine's: tolerate skew between
|
||||
* them so an offer the drain just recorded is not mistaken for an older one. */
|
||||
export const CLOCK_SKEW_TOLERANCE_MS = 2 * 60_000
|
||||
/** A host too slow to list its offers before a restart is treated as unable to (falls back to time). */
|
||||
const OFFER_SNAPSHOT_TIMEOUT_MS = 5_000
|
||||
|
||||
function hasCode(error: unknown, code: string): boolean {
|
||||
return typeof error === 'object' && error !== null && 'code' in error && error.code === code
|
||||
}
|
||||
|
||||
/** Offer ids the host already holds before a drain; offers in it were left for the user. Null when
|
||||
* the host cannot answer (an unresponsive host is often why it is being restarted). */
|
||||
export async function listRestartOfferIds(call: RuntimeCall): Promise<ReadonlySet<string> | null> {
|
||||
try {
|
||||
let timer: ReturnType<typeof setTimeout> | undefined
|
||||
const listed = await Promise.race([
|
||||
call('agentSession.restartResumable'),
|
||||
new Promise<null>((resolve) => {
|
||||
timer = setTimeout(() => resolve(null), OFFER_SNAPSHOT_TIMEOUT_MS)
|
||||
})
|
||||
]).finally(() => clearTimeout(timer))
|
||||
const list = OfferList.safeParse(listed)
|
||||
if (!list.success) {
|
||||
return null
|
||||
}
|
||||
return new Set(
|
||||
list.data.sessions.flatMap((row) => {
|
||||
const offer = Offer.safeParse(row)
|
||||
return offer.success ? [offer.data.sessionId] : []
|
||||
})
|
||||
)
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Continues the offers this drain recorded. With a pre-drain snapshot of the host's offers, "recorded
|
||||
* by this drain" is set membership (no clock comparison); without one it falls back to the host
|
||||
* timestamp against this machine's drain start, allowing for clock skew.
|
||||
*/
|
||||
export async function autoResumeAgentsAfterRollover(
|
||||
call: RuntimeCall,
|
||||
drainStartedAt: number
|
||||
drainStartedAt: number,
|
||||
offersBeforeDrain: ReadonlySet<string> | null = null
|
||||
): Promise<HostedAgentAutoResumeOutcome> {
|
||||
let listed: unknown
|
||||
try {
|
||||
@@ -47,7 +85,13 @@ export async function autoResumeAgentsAfterRollover(
|
||||
}
|
||||
const ids = list.data.sessions.flatMap((row) => {
|
||||
const offer = Offer.safeParse(row)
|
||||
return offer.success && offer.data.recordedAt >= drainStartedAt ? [offer.data.sessionId] : []
|
||||
if (!offer.success) {
|
||||
return []
|
||||
}
|
||||
const recordedByThisDrain = offersBeforeDrain
|
||||
? !offersBeforeDrain.has(offer.data.sessionId)
|
||||
: offer.data.recordedAt >= drainStartedAt - CLOCK_SKEW_TOLERANCE_MS
|
||||
return recordedByThisDrain ? [offer.data.sessionId] : []
|
||||
})
|
||||
const outcome = { offered: ids.length, continued: 0, skipped: 0, unsupported: false }
|
||||
for (let offset = 0; offset < ids.length; offset += MAX_RESTART_RESUME_SESSIONS) {
|
||||
|
||||
@@ -24,7 +24,12 @@ export type HostedSessionLeasePorts = {
|
||||
warn: (minutesLeft: HostedLeaseWarning) => void
|
||||
/** Re-pairs with the rebooted orcad; resolves once the runtime answers again. */
|
||||
reconnect: (ready: HostedEnvironmentReady) => Promise<void>
|
||||
resumeAgents: (drainStartedAt: number) => Promise<HostedAgentAutoResumeOutcome>
|
||||
resumeAgents: (
|
||||
drainStartedAt: number,
|
||||
offersBeforeDrain: ReadonlySet<string> | null
|
||||
) => Promise<HostedAgentAutoResumeOutcome>
|
||||
/** The host's restart offers before the drain, so only the drain's own are continued. */
|
||||
snapshotOffers?: () => Promise<ReadonlySet<string> | null>
|
||||
onSuspended: () => void
|
||||
/** Replaces the driver suspend, e.g. to run the window's ordinary Sleep for the workspace. */
|
||||
suspend?: () => Promise<void>
|
||||
@@ -158,6 +163,7 @@ export class HostedSessionLeaseScheduler {
|
||||
|
||||
private async rollover(reason: 'opportunistic' | 'forced'): Promise<void> {
|
||||
const { driver, handle } = this.ports
|
||||
const offersBeforeDrain = (await this.ports.snapshotOffers?.().catch(() => null)) ?? null
|
||||
const drainStartedAt = this.now()
|
||||
this.log(`[orca-cloud] ${reason} rollover`)
|
||||
this.ports.onRolloverStateChange?.('restarting')
|
||||
@@ -172,7 +178,7 @@ export class HostedSessionLeaseScheduler {
|
||||
}
|
||||
this.ports.onRolloverStateChange?.('running')
|
||||
try {
|
||||
const outcome = await this.ports.resumeAgents(drainStartedAt)
|
||||
const outcome = await this.ports.resumeAgents(drainStartedAt, offersBeforeDrain)
|
||||
this.log(
|
||||
`[orca-cloud] continued ${outcome.continued}/${outcome.offered} agents after rollover`
|
||||
)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import type { HostedEnvironmentReady } from './hosted-environment-provider'
|
||||
import { autoResumeAgentsAfterRollover } from './hosted-agent-auto-resume'
|
||||
import { autoResumeAgentsAfterRollover, listRestartOfferIds } from './hosted-agent-auto-resume'
|
||||
import { createHostedSandboxControlPlaneClient } from './hosted-sandbox-control-plane-client'
|
||||
import { createHostedSandboxDriver, HOSTED_SANDBOX_CAPABILITIES } from './hosted-sandbox-driver'
|
||||
import { decideHostedLeaseAction, type HostedActivitySample } from './hosted-session-lease-policy'
|
||||
@@ -102,7 +102,7 @@ describe('hosted session lease scheduler', () => {
|
||||
expect(await scheduler.tick()).toEqual({ kind: 'rollover', reason: 'forced' })
|
||||
expect(reconnect).toHaveBeenCalledTimes(1)
|
||||
expect(reconnect.mock.calls[0]?.[0].session.startedAt).toBe(now)
|
||||
expect(resumeAgents).toHaveBeenCalledWith(created.session.deadlineAt)
|
||||
expect(resumeAgents).toHaveBeenCalledWith(created.session.deadlineAt, null)
|
||||
// The new session is far from its cap, so the next tick does nothing drastic.
|
||||
expect((await scheduler.tick()).kind).not.toBe('rollover')
|
||||
})
|
||||
@@ -150,14 +150,15 @@ describe('agent auto-resume after rollover', () => {
|
||||
if (method === 'agentSession.restartResumable') {
|
||||
return {
|
||||
sessions: [
|
||||
{ sessionId: 'old', recordedAt: 900 },
|
||||
{ sessionId: 'a', recordedAt: 1000 },
|
||||
{ sessionId: 'b', recordedAt: 1500 }
|
||||
{ sessionId: 'old', recordedAt: T0 - 10 * MIN },
|
||||
// The host's clock runs 30 s behind this machine's: still this drain's offer.
|
||||
{ sessionId: 'a', recordedAt: T0 - 30_000 },
|
||||
{ sessionId: 'b', recordedAt: T0 + 500 }
|
||||
]
|
||||
}
|
||||
}
|
||||
return { continued: [{ sessionId: 'a' }], skipped: ['b'] }
|
||||
}, 1000)
|
||||
}, T0)
|
||||
expect(calls).toEqual([
|
||||
{ method: 'agentSession.restartResumable', params: undefined },
|
||||
{ method: 'agentSession.restartContinue', params: { sessionIds: ['a', 'b'] } }
|
||||
@@ -178,3 +179,33 @@ describe('agent auto-resume after rollover', () => {
|
||||
expect(call).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('agent auto-resume with a pre-drain offer snapshot', () => {
|
||||
it('continues only offers the host did not already hold, whatever the clocks say', async () => {
|
||||
const offers = [
|
||||
{ sessionId: 'left-for-user', recordedAt: T0 + 5_000 },
|
||||
{ sessionId: 'this-drain', recordedAt: T0 - 9 * MIN }
|
||||
]
|
||||
const calls: unknown[] = []
|
||||
const call = async (method: string, params?: Record<string, unknown>) => {
|
||||
calls.push({ method, params })
|
||||
return method === 'agentSession.restartResumable' ? { sessions: offers } : { continued: [{}] }
|
||||
}
|
||||
const before = await listRestartOfferIds(async () => ({ sessions: [offers[0]] }))
|
||||
expect(before).toEqual(new Set(['left-for-user']))
|
||||
const outcome = await autoResumeAgentsAfterRollover(call, T0, before)
|
||||
expect(calls).toContainEqual({
|
||||
method: 'agentSession.restartContinue',
|
||||
params: { sessionIds: ['this-drain'] }
|
||||
})
|
||||
expect(outcome.offered).toBe(1)
|
||||
})
|
||||
|
||||
it('answers null when the host cannot list its offers', async () => {
|
||||
await expect(
|
||||
listRestartOfferIds(async () => {
|
||||
throw new Error('down')
|
||||
})
|
||||
).resolves.toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -21,7 +21,7 @@ import { noteRecipeVmWakePairing } from '../ephemeral-vm/recipe-vm-wake-pairings
|
||||
import { getEphemeralVmRecipeResultPairingCode } from '../../shared/ephemeral-vm-recipes'
|
||||
import { environmentHandleFromRecipeResult } from '../environment-providers/environment-provider-recipe-adapter'
|
||||
import type { HostedEnvironmentProviderDriver } from './hosted-environment-provider'
|
||||
import { autoResumeAgentsAfterRollover } from './hosted-agent-auto-resume'
|
||||
import { autoResumeAgentsAfterRollover, listRestartOfferIds } from './hosted-agent-auto-resume'
|
||||
import { HOSTED_SANDBOX_DRIVER_ID } from './hosted-sandbox-driver'
|
||||
import { createHostedRuntimeEnvironmentCall } from './hosted-runtime-environment-call'
|
||||
import type { HostedActivitySample } from './hosted-session-lease-policy'
|
||||
@@ -137,7 +137,9 @@ export function createHostedSessionRegistry(deps: {
|
||||
true
|
||||
)
|
||||
},
|
||||
resumeAgents: (drainStartedAt) => autoResumeAgentsAfterRollover(call, drainStartedAt),
|
||||
snapshotOffers: () => listRestartOfferIds(call),
|
||||
resumeAgents: (drainStartedAt, offersBeforeDrain) =>
|
||||
autoResumeAgentsAfterRollover(call, drainStartedAt, offersBeforeDrain),
|
||||
onRolloverStateChange: (state) => {
|
||||
const event: HostedSessionStateEvent = { workspaceId: runtime.workspaceId ?? null, state }
|
||||
broadcast('orcaCloud:sessionState', event)
|
||||
|
||||
@@ -5,6 +5,7 @@ import type { HostedSandboxControlPlaneClient } from './hosted-sandbox-control-p
|
||||
import type { EphemeralVmRuntimeRecord } from '../../shared/ephemeral-vm-runtimes'
|
||||
import type {
|
||||
CloudOomEvent,
|
||||
CloudWakeProgressEvent,
|
||||
CloudWorkspaceErrorCode,
|
||||
CloudWorkspaceResult
|
||||
} from '../../shared/hosted-sandbox/cloud-workspace-api'
|
||||
@@ -18,6 +19,7 @@ import {
|
||||
} from '../environment-providers/environment-provider-driver'
|
||||
import { environmentHandleFromRecipeResult } from '../environment-providers/environment-provider-recipe-adapter'
|
||||
import { isHostedEntitlementDenied } from './hosted-environment-provider'
|
||||
import { callRuntimeEnvironment } from '../ipc/runtime-environments/runtime-environment-transport-routing'
|
||||
import { HOSTED_SANDBOX_DRIVER_ID } from './hosted-sandbox-driver'
|
||||
import type { HostedWorkspaceService } from './hosted-workspace-service'
|
||||
|
||||
@@ -165,6 +167,83 @@ export function getHostedWorkspaceService(): HostedWorkspaceService | null {
|
||||
return current
|
||||
}
|
||||
|
||||
/** Polls a cheap terminal listing until the woken server answers it (or the deadline passes). */
|
||||
export async function waitForTerminalsReachable(
|
||||
userDataPath: string,
|
||||
environmentId: string,
|
||||
deadline: number,
|
||||
stillWanted: () => boolean = () => true
|
||||
): Promise<boolean> {
|
||||
while (Date.now() < deadline && stillWanted()) {
|
||||
const remaining = deadline - Date.now()
|
||||
// Bounded by the deadline: the probe neither waits past it on a wake in flight nor on a slow server.
|
||||
const signal = AbortSignal.timeout(Math.max(1, remaining))
|
||||
const response = await callRuntimeEnvironment(
|
||||
userDataPath,
|
||||
environmentId,
|
||||
'terminal.list',
|
||||
{ limit: 1 },
|
||||
Math.min(5_000, Math.max(1, remaining)),
|
||||
undefined,
|
||||
undefined,
|
||||
{ signal, preparation: { recipeVmHoldMs: Math.max(0, remaining) } }
|
||||
).catch(() => null)
|
||||
if (response?.ok) {
|
||||
return stillWanted()
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 250))
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* After a wake's step answers: `ready` once the runtime record says running AND the woken server
|
||||
* answers a terminal listing (panes rebind to the restarted server's terminals a moment after
|
||||
* running; keys typed before that are dropped by design). `failed` when it never answers. Nothing
|
||||
* when this wake is no longer the current one (a newer wake, a failure or a Sleep took over).
|
||||
*/
|
||||
export async function announceWakeUsable(
|
||||
userDataPath: string,
|
||||
workspaceId: string,
|
||||
startedAt: number,
|
||||
hooks: { isCurrent: () => boolean; emit: (event: Omit<CloudWakeProgressEvent, 'at'>) => void },
|
||||
timeoutMs = 60_000
|
||||
): Promise<void> {
|
||||
const deadline = Date.now() + timeoutMs
|
||||
const running = (): EphemeralVmRuntimeRecord | null => {
|
||||
const runtime = findCloudRuntime(userDataPath, workspaceId)?.runtime
|
||||
return runtime?.status === 'running' ? runtime : null
|
||||
}
|
||||
while (Date.now() < deadline && hooks.isCurrent()) {
|
||||
const runtime = running()
|
||||
if (runtime) {
|
||||
const reachable = runtime.runtimeEnvironmentId
|
||||
? await waitForTerminalsReachable(
|
||||
userDataPath,
|
||||
runtime.runtimeEnvironmentId,
|
||||
deadline,
|
||||
hooks.isCurrent
|
||||
)
|
||||
: true
|
||||
if (hooks.isCurrent() && running()) {
|
||||
hooks.emit(
|
||||
reachable
|
||||
? { workspaceId, phase: 'ready', source: 'open', startedAt }
|
||||
: {
|
||||
workspaceId,
|
||||
phase: 'failed',
|
||||
source: 'open',
|
||||
startedAt,
|
||||
message: 'The workspace did not answer after waking.'
|
||||
}
|
||||
)
|
||||
}
|
||||
return
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 250))
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether the control plane says this workspace is archived; unknown counts as not archived. */
|
||||
export async function isArchivedOnControlPlane(
|
||||
userDataPath: string,
|
||||
@@ -185,7 +264,9 @@ export async function isArchivedOnControlPlane(
|
||||
|
||||
let archiveCheck: ((workspaceId: string) => Promise<boolean>) | null = null
|
||||
/** Installed with Orca Cloud; the wake path asks it before resuming a cloud workspace. */
|
||||
export function setCloudArchiveCheck(check: ((workspaceId: string) => Promise<boolean>) | null): void {
|
||||
export function setCloudArchiveCheck(
|
||||
check: ((workspaceId: string) => Promise<boolean>) | null
|
||||
): void {
|
||||
archiveCheck = check
|
||||
}
|
||||
/** False when Orca Cloud is off in this build. */
|
||||
|
||||
@@ -18,7 +18,7 @@ import type {
|
||||
import { applyResumedRuntimePairing } from '../ephemeral-vm/runtime-resume-pairing'
|
||||
import { noteRecipeVmWakePairing } from '../ephemeral-vm/recipe-vm-wake-pairings'
|
||||
import { invalidateRuntimeEnvironmentTransport } from '../ipc/runtime-environments/runtime-environments'
|
||||
import { autoResumeAgentsAfterRollover } from './hosted-agent-auto-resume'
|
||||
import { autoResumeAgentsAfterRollover, listRestartOfferIds } from './hosted-agent-auto-resume'
|
||||
import type { HostedSandboxControlPlaneClient } from './hosted-sandbox-control-plane-client'
|
||||
import { toReady } from './hosted-sandbox-driver'
|
||||
import { createHostedRuntimeEnvironmentCall } from './hosted-runtime-environment-call'
|
||||
@@ -28,6 +28,7 @@ import {
|
||||
failure,
|
||||
findCloudRuntime,
|
||||
readCachedCheckouts,
|
||||
announceWakeUsable,
|
||||
type CloudRuntime
|
||||
} from './hosted-workspace-runtime'
|
||||
|
||||
@@ -95,17 +96,11 @@ export function installHostedWorkspaceService(deps: {
|
||||
}
|
||||
|
||||
/** The step answers before the window re-pairs; `ready` fires once the runtime record says running. */
|
||||
async function announceReadyWhenRunning(workspaceId: string, startedAt: number): Promise<void> {
|
||||
const deadline = Date.now() + 60_000
|
||||
while (Date.now() < deadline && wakes.has(workspaceId)) {
|
||||
const runtime = findCloudRuntime(deps.userDataPath, workspaceId)?.runtime
|
||||
if (runtime?.status === 'running') {
|
||||
service.emitWakeProgress({ workspaceId, phase: 'ready', source: 'open', startedAt })
|
||||
return
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 250))
|
||||
}
|
||||
}
|
||||
const announceReadyWhenRunning = (workspaceId: string, startedAt: number): Promise<void> =>
|
||||
announceWakeUsable(deps.userDataPath, workspaceId, startedAt, {
|
||||
isCurrent: () => wakes.get(workspaceId)?.startedAt === startedAt,
|
||||
emit: (event) => service.emitWakeProgress(event)
|
||||
})
|
||||
|
||||
const service: HostedWorkspaceService = {
|
||||
emitWakeProgress(event) {
|
||||
@@ -213,6 +208,11 @@ export function installHostedWorkspaceService(deps: {
|
||||
source: 'restart',
|
||||
startedAt
|
||||
})
|
||||
const call = found.runtime.runtimeEnvironmentId
|
||||
? createHostedRuntimeEnvironmentCall(deps.userDataPath, found.runtime.runtimeEnvironmentId)
|
||||
: null
|
||||
// Offers the host already holds were left for the user; only this restart's are continued.
|
||||
const offersBeforeDrain = call ? await listRestartOfferIds(call) : null
|
||||
// The scheduler holds the old session's deadline and would warn/roll over the new one; a fresh one starts on sync.
|
||||
deps.stopSession(found.sandboxId)
|
||||
try {
|
||||
@@ -231,14 +231,10 @@ export function installHostedWorkspaceService(deps: {
|
||||
source: 'restart',
|
||||
startedAt
|
||||
})
|
||||
if (found.runtime.runtimeEnvironmentId) {
|
||||
if (call) {
|
||||
// Same continuation as a rollover: chats the restart interrupted carry on by themselves.
|
||||
const call = createHostedRuntimeEnvironmentCall(
|
||||
deps.userDataPath,
|
||||
found.runtime.runtimeEnvironmentId
|
||||
)
|
||||
void autoResumeAgentsAfterRollover(call, startedAt).catch((error: unknown) =>
|
||||
log(`[orca-cloud] auto-resume after restart failed: ${String(error)}`)
|
||||
void autoResumeAgentsAfterRollover(call, startedAt, offersBeforeDrain).catch(
|
||||
(error: unknown) => log(`[orca-cloud] auto-resume after restart failed: ${String(error)}`)
|
||||
)
|
||||
}
|
||||
return { ok: true, value: null }
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const runtimes: { status: string; runtimeEnvironmentId?: string }[] = []
|
||||
let listOk = true
|
||||
vi.mock('../ipc/runtime-environments/runtime-environment-transport-routing', () => ({
|
||||
callRuntimeEnvironment: vi.fn(async () =>
|
||||
listOk ? { ok: true, result: {} } : { ok: false, error: { code: 'x', message: 'x' } }
|
||||
)
|
||||
}))
|
||||
vi.mock('../../shared/ephemeral-vm-runtime-store', () => ({
|
||||
listEphemeralVmRuntimes: () =>
|
||||
runtimes.map((r) => ({
|
||||
...r,
|
||||
workspaceId: 'ws',
|
||||
recipe: { provider: 'orca-cloud' },
|
||||
recipeResult: {
|
||||
schemaVersion: 1,
|
||||
pairingCode: 'x',
|
||||
projectRoot: '/p',
|
||||
orcaCloud: { sandboxId: 'sbx_1' }
|
||||
}
|
||||
}))
|
||||
}))
|
||||
vi.mock('../environment-providers/environment-provider-recipe-adapter', () => ({
|
||||
environmentHandleFromRecipeResult: () => ({ driverState: { sandboxId: 'sbx_1' } })
|
||||
}))
|
||||
|
||||
describe('announceWakeUsable', () => {
|
||||
it('emits ready once running and reachable', async () => {
|
||||
const { announceWakeUsable } = await import('./hosted-workspace-runtime')
|
||||
runtimes.splice(0, runtimes.length, { status: 'running', runtimeEnvironmentId: 'env' })
|
||||
listOk = true
|
||||
const emit = vi.fn()
|
||||
await announceWakeUsable('/u', 'ws', 1, { isCurrent: () => true, emit }, 2_000)
|
||||
expect(emit).toHaveBeenCalledWith(expect.objectContaining({ phase: 'ready', startedAt: 1 }))
|
||||
})
|
||||
|
||||
it('emits failed when the server never answers, and nothing for a superseded wake', async () => {
|
||||
const { announceWakeUsable } = await import('./hosted-workspace-runtime')
|
||||
runtimes.splice(0, runtimes.length, { status: 'running', runtimeEnvironmentId: 'env' })
|
||||
listOk = false
|
||||
const emit = vi.fn()
|
||||
await announceWakeUsable('/u', 'ws', 2, { isCurrent: () => true, emit }, 600)
|
||||
expect(emit).toHaveBeenCalledWith(expect.objectContaining({ phase: 'failed' }))
|
||||
const stale = vi.fn()
|
||||
await announceWakeUsable('/u', 'ws', 3, { isCurrent: () => false, emit: stale }, 600)
|
||||
expect(stale).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -13,7 +13,7 @@ import type {
|
||||
RecipeVmOpening,
|
||||
RecipeVmState
|
||||
} from '../../runtime/workspaces/sleep/recipe-vm-operations'
|
||||
import { prepareRecipeVm } from './runtime-environment-recipe-vm-wake'
|
||||
import { holdForInFlightRecipeVmWake, prepareRecipeVm } from './runtime-environment-recipe-vm-wake'
|
||||
import { RemoteRuntimeClientError } from '../../../shared/remote-runtime/remote-runtime-client-error'
|
||||
import {
|
||||
forgetRecipeVmWakePairingsForTests,
|
||||
@@ -131,6 +131,65 @@ describe('a request to a recipe VM environment', () => {
|
||||
}
|
||||
)
|
||||
|
||||
it('holds a request that does not open the workspace for an in-flight wake, outside its call slot', async () => {
|
||||
recordVm('running')
|
||||
ownerState = 'waking'
|
||||
const awaitWake = vi.fn(async () => 'up' as const)
|
||||
registerRecipeVmWaker({
|
||||
state,
|
||||
noteOpened: vi.fn(),
|
||||
holdOpen,
|
||||
awaitWake,
|
||||
openEnvironment: async () => 'up'
|
||||
})
|
||||
await holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'terminal.list', 5_000)
|
||||
expect(awaitWake).toHaveBeenCalledWith('workspace-of-vm-1', false, 5_000, undefined)
|
||||
// Bounded by the caller's hold (0 = never wait) and never for an open (that wakes in its slot).
|
||||
await holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'terminal.list', 5_000, {
|
||||
preparation: { recipeVmHoldMs: 0 }
|
||||
})
|
||||
await holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'worktree.activate', 5_000)
|
||||
expect(awaitWake).toHaveBeenCalledTimes(1)
|
||||
expect(holdOpen).not.toHaveBeenCalled()
|
||||
// Inside the slot the request is still answered at once from the owner's state.
|
||||
await expect(codeOf('none')).resolves.toBe('runtime_environment_waking')
|
||||
})
|
||||
|
||||
it('caps the hold at the in-flight maximum and does not wait on a VM that is not waking', async () => {
|
||||
recordVm('running')
|
||||
const awaitWake = vi.fn(async () => 'up' as const)
|
||||
registerRecipeVmWaker({
|
||||
state,
|
||||
noteOpened: vi.fn(),
|
||||
holdOpen,
|
||||
awaitWake,
|
||||
openEnvironment: async () => 'up'
|
||||
})
|
||||
await holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'terminal.list', 600_000)
|
||||
expect(awaitWake).not.toHaveBeenCalled()
|
||||
ownerState = 'waking'
|
||||
await holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'terminal.list', 600_000)
|
||||
expect(awaitWake).toHaveBeenCalledWith('workspace-of-vm-1', false, 60_000, undefined)
|
||||
})
|
||||
|
||||
it('reports the pairing a wake it waited through left, for the caller fence', async () => {
|
||||
recordVm('running')
|
||||
ownerState = 'waking'
|
||||
registerRecipeVmWaker({
|
||||
state,
|
||||
noteOpened: vi.fn(),
|
||||
holdOpen,
|
||||
awaitWake: async () => {
|
||||
noteRecipeVmWakePairing(VM_ENVIRONMENT.id, 9, true)
|
||||
return 'up' as const
|
||||
},
|
||||
openEnvironment: async () => 'up'
|
||||
})
|
||||
await expect(
|
||||
holdForInFlightRecipeVmWake(userDataPath, VM_ENVIRONMENT, 'terminal.list', 5_000)
|
||||
).resolves.toEqual({ before: VM_ENVIRONMENT, wokenPairingRevision: 9 })
|
||||
})
|
||||
|
||||
// A phone gives up after 30 seconds, so its open is held for less.
|
||||
|
||||
// A Disconnect made before the open holds: no client's open wakes the VM behind that server.
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
RUNTIME_ENVIRONMENT_WAKE_FAILED_MESSAGE,
|
||||
RUNTIME_ENVIRONMENT_WAKING_ERROR_CODE,
|
||||
RUNTIME_ENVIRONMENT_WAKING_MESSAGE,
|
||||
recipeVmWakeIntentFor,
|
||||
type RecipeVmWakeIntent
|
||||
} from '../../../shared/remote-runtime/recipe-vm-wake'
|
||||
import { getRecipeVmWaker } from '../../runtime/workspaces/sleep/recipe-vm-waker-registry'
|
||||
@@ -117,6 +118,46 @@ export async function prepareRecipeVm(
|
||||
return UNCHANGED
|
||||
}
|
||||
|
||||
/** How long a request that does not open the workspace waits for a wake already in flight. */
|
||||
export const RECIPE_VM_IN_FLIGHT_WAKE_HOLD_MS = 60_000
|
||||
|
||||
/**
|
||||
* Before a request that does not open the workspace takes its call slot: while the VM's wake is
|
||||
* already underway (often just reconnecting), wait for it, up to `holdMs`, instead of being refused
|
||||
* and never retried. Never starts a wake and never refuses: the request's own preparation decides
|
||||
* what happens next (it is refused there if the wake failed or is still running).
|
||||
*/
|
||||
export async function awaitRecipeVmInFlightWake<E extends RecipeVmEnvironment>(
|
||||
userDataPath: string,
|
||||
environment: E,
|
||||
holdMs: number,
|
||||
signal?: AbortSignal
|
||||
): Promise<HeldRecipeVmWake<E> | null> {
|
||||
const waker = getRecipeVmWaker()
|
||||
if (holdMs <= 0 || !waker?.awaitWake) {
|
||||
return null
|
||||
}
|
||||
const vm = recipeVmOf(userDataPath, environment)
|
||||
if (!vm || waker.state(vm.workspaceId, vm.recordSuspended) !== 'waking') {
|
||||
return null
|
||||
}
|
||||
const before = latestRecipeVmWake(environment.id)
|
||||
await waker.awaitWake(vm.workspaceId, vm.recordSuspended, holdMs, signal).catch(() => undefined)
|
||||
const woken = latestRecipeVmWake(environment.id)
|
||||
// The wake this request waited through re-paired the server: the caller's fence names the pairing
|
||||
// before it, exactly as for a request that woke the VM itself.
|
||||
return woken !== before
|
||||
? { before: environment, wokenPairingRevision: woken?.pairingRevision ?? null }
|
||||
: null
|
||||
}
|
||||
|
||||
/** A wake a request waited through before taking its call slot: the record from before it and
|
||||
* the pairing it left. */
|
||||
export type HeldRecipeVmWake<E = RecipeVmEnvironment> = {
|
||||
before: E
|
||||
wokenPairingRevision: number | null
|
||||
}
|
||||
|
||||
/** The pairing revision a recipe VM wake this request waited for left its server on, if any. */
|
||||
export type RecipeVmPreparation = { wokenPairingRevision: number | null }
|
||||
const UNCHANGED: RecipeVmPreparation = { wokenPairingRevision: null }
|
||||
@@ -155,3 +196,27 @@ function refusal(outcome: Exclude<RecipeVmOpening, 'up'> | 'asleep'): RemoteRunt
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/** A request that does not open the workspace waits (bounded by its own timeout and hold) for a wake
|
||||
* already in flight. Called before the request takes a per-environment call slot, so held
|
||||
* background requests never block an open. */
|
||||
export function holdForInFlightRecipeVmWake<E extends RecipeVmEnvironment>(
|
||||
userDataPath: string,
|
||||
environment: E,
|
||||
method: string,
|
||||
timeoutMs: number | undefined,
|
||||
options?: { preparation?: { recipeVmHoldMs?: number }; signal?: AbortSignal }
|
||||
): Promise<HeldRecipeVmWake<E> | null> {
|
||||
if (recipeVmWakeIntentFor(method) !== 'none') {
|
||||
return Promise.resolve(null)
|
||||
}
|
||||
const holdMs = Math.min(
|
||||
RECIPE_VM_IN_FLIGHT_WAKE_HOLD_MS,
|
||||
options?.preparation?.recipeVmHoldMs ?? Infinity,
|
||||
timeoutMs ?? DEFAULT_CALL_TIMEOUT_MS
|
||||
)
|
||||
return awaitRecipeVmInFlightWake(userDataPath, environment, holdMs, options?.signal)
|
||||
}
|
||||
|
||||
/** The transport's default call timeout, which also bounds the hold for calls that set none. */
|
||||
const DEFAULT_CALL_TIMEOUT_MS = 15_000
|
||||
|
||||
@@ -56,3 +56,46 @@ describe('a request whose own recipe VM wake re-paired its server', () => {
|
||||
expect((await fence(1, 'before')).failure).toMatchObject({ ok: false })
|
||||
})
|
||||
})
|
||||
|
||||
describe('a fenced request that waited through a wake already in flight', () => {
|
||||
beforeEach(() => {
|
||||
prepare.mockClear()
|
||||
// By the time the request takes its slot the record already holds the new pairing; its own
|
||||
// preparation woke nothing.
|
||||
prepared.current = { environment: AFTER_WAKE, before: AFTER_WAKE, wokenPairingRevision: null }
|
||||
})
|
||||
|
||||
const heldFence = (pairingRevision: number, wokenPairingRevision: number | null) =>
|
||||
prepareFencedRuntimeRequest(
|
||||
'/user-data',
|
||||
'env-vm',
|
||||
'terminal.list',
|
||||
undefined,
|
||||
{ pairingRevision, runtimeId: 'before' },
|
||||
undefined,
|
||||
{ before: BEFORE as never, wokenPairingRevision }
|
||||
)
|
||||
|
||||
it('passes the pre-wake fence and then expects the pairing that wake made', async () => {
|
||||
await expect(heldFence(1, 2)).resolves.toMatchObject({
|
||||
failure: null,
|
||||
expected: { pairingRevision: 2, runtimeId: undefined }
|
||||
})
|
||||
})
|
||||
|
||||
it('still refuses a caller fenced to an older pairing', async () => {
|
||||
expect((await heldFence(0, 2)).failure).toMatchObject({ ok: false })
|
||||
expect(prepare).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('without the held wake, the same call is refused as changed', async () => {
|
||||
const result = await prepareFencedRuntimeRequest(
|
||||
'/user-data',
|
||||
'env-vm',
|
||||
'terminal.list',
|
||||
undefined,
|
||||
{ pairingRevision: 1, runtimeId: 'before' }
|
||||
)
|
||||
expect(result.failure).toMatchObject({ ok: false })
|
||||
})
|
||||
})
|
||||
|
||||
@@ -22,14 +22,17 @@ export async function prepareFencedRuntimeRequest(
|
||||
method: string,
|
||||
preparation: ManagedRuntimeCallPreparation | undefined,
|
||||
expected: Expected,
|
||||
signal?: AbortSignal
|
||||
signal?: AbortSignal,
|
||||
/** A recipe VM wake this request waited through before its call slot: its fence is checked
|
||||
* against the record from before that wake, as for a wake the request made itself. */
|
||||
heldWake?: { before: KnownRuntimeEnvironment; wokenPairingRevision: number | null }
|
||||
): Promise<
|
||||
| { failure: Rpc.RuntimeRpcResponse<never> }
|
||||
| { failure: null; environment: KnownRuntimeEnvironment; expected: Expected }
|
||||
> {
|
||||
// A stale caller is refused before its request can wake (and re-pair) a VM.
|
||||
const stale = runtimeEnvironmentRevisionFailure(
|
||||
resolveEnvironment(userDataPath, environmentId),
|
||||
heldWake?.before ?? resolveEnvironment(userDataPath, environmentId),
|
||||
expected.pairingRevision,
|
||||
method,
|
||||
expected.runtimeId
|
||||
@@ -44,7 +47,10 @@ export async function prepareFencedRuntimeRequest(
|
||||
method,
|
||||
signal
|
||||
)
|
||||
const { environment, wokenPairingRevision: woken } = prepared
|
||||
const { environment } = prepared
|
||||
const woken = prepared.wokenPairingRevision ?? heldWake?.wokenPairingRevision ?? null
|
||||
const before =
|
||||
prepared.wokenPairingRevision !== null ? prepared.before : (heldWake?.before ?? prepared.before)
|
||||
const failure =
|
||||
woken === null
|
||||
? runtimeEnvironmentRevisionFailure(
|
||||
@@ -54,7 +60,7 @@ export async function prepareFencedRuntimeRequest(
|
||||
expected.runtimeId
|
||||
)
|
||||
: (runtimeEnvironmentRevisionFailure(
|
||||
prepared.before,
|
||||
before,
|
||||
expected.pairingRevision,
|
||||
method,
|
||||
expected.runtimeId
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
resolveManagedRuntimeEnvironment,
|
||||
type ManagedRuntimeCallPreparation
|
||||
} from './runtime-environment-managed-tunnel'
|
||||
import { markEnvironmentUsedFromResponse } from './runtime-environment-usage-marking'
|
||||
import { recordRuntimeEnvironmentUsage } from '../../../shared/remote-runtime/host-store/runtime-environment-usage-record'
|
||||
import type * as Rpc from '../../../shared/runtime-rpc-envelope'
|
||||
import '../../../shared/remote-runtime/requests/remote-runtime-request-socket'
|
||||
@@ -23,6 +24,7 @@ import type { RemoteRuntimeTransportSubscription } from '../../../shared/remote-
|
||||
import { withRuntimeUnreachableHint } from './runtime-environment-unreachable-hint'
|
||||
import { decodeRuntimeEnvironmentSubscriptionEvents } from './runtime-environment-subscription-events'
|
||||
import { enqueueRuntimeCall } from './runtime-environment-call-queue'
|
||||
import { holdForInFlightRecipeVmWake } from './runtime-environment-recipe-vm-wake'
|
||||
import {
|
||||
sendRemoteRuntimeConnectionRequestAbortable,
|
||||
sendRemoteRuntimeRequestAbortable
|
||||
@@ -80,6 +82,13 @@ export async function callRuntimeEnvironment(
|
||||
)
|
||||
}
|
||||
const environment = resolveEnvironment(userDataPath, selector)
|
||||
const held = await holdForInFlightRecipeVmWake(
|
||||
userDataPath,
|
||||
environment,
|
||||
method,
|
||||
timeoutMs,
|
||||
options
|
||||
)
|
||||
// Connect failures reject; retain the dispatched endpoint for hints because queued re-pairing can change it.
|
||||
let endpoint = getPreferredPairedRuntimeConnection(environment).endpoint
|
||||
try {
|
||||
@@ -96,7 +105,8 @@ export async function callRuntimeEnvironment(
|
||||
pairingRevision: expectedEnvironmentPairingRevision,
|
||||
runtimeId: options?.expectedEnvironmentRuntimeId
|
||||
},
|
||||
options?.signal
|
||||
options?.signal,
|
||||
held ?? undefined
|
||||
)
|
||||
if (prepared.failure) {
|
||||
return prepared.failure
|
||||
@@ -290,18 +300,6 @@ export async function subscribeRuntimeEnvironment(
|
||||
}
|
||||
}
|
||||
|
||||
function markEnvironmentUsedFromResponse(
|
||||
userDataPath: string,
|
||||
environmentId: string,
|
||||
response: Rpc.RuntimeRpcResponse<unknown>
|
||||
): void {
|
||||
if (response.ok === true) {
|
||||
recordRuntimeEnvironmentUsage(userDataPath, environmentId, {
|
||||
runtimeId: response._meta.runtimeId
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function shouldUseCachedRequestConnection(method: string): boolean {
|
||||
return method === 'terminal.send' || method === 'terminal.updateViewport'
|
||||
}
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
import { recordRuntimeEnvironmentUsage } from '../../../shared/remote-runtime/host-store/runtime-environment-usage-record'
|
||||
import type * as Rpc from '../../../shared/runtime-rpc-envelope'
|
||||
|
||||
/** A successful answer records which server process served the environment. */
|
||||
export function markEnvironmentUsedFromResponse(
|
||||
userDataPath: string,
|
||||
environmentId: string,
|
||||
response: Rpc.RuntimeRpcResponse<unknown>
|
||||
): void {
|
||||
if (response.ok === true) {
|
||||
recordRuntimeEnvironmentUsage(userDataPath, environmentId, {
|
||||
runtimeId: response._meta.runtimeId
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { awaitInFlightWake, WorkspaceSleepRequests, type VmOperation } from './recipe-vm-operations'
|
||||
|
||||
describe('awaitInFlightWake', () => {
|
||||
it('answers at once for a VM that is up or asleep, without starting anything', async () => {
|
||||
const sleeps = new WorkspaceSleepRequests()
|
||||
await expect(awaitInFlightWake([], false, sleeps, 'w', 1000)).resolves.toBe('up')
|
||||
await expect(awaitInFlightWake([], true, sleeps, 'w', 1000)).resolves.toBe('asleep')
|
||||
})
|
||||
|
||||
it('waits for the wake already in flight', async () => {
|
||||
let finish = (): void => {}
|
||||
const settled = new Promise<void>((resolve) => {
|
||||
finish = resolve
|
||||
})
|
||||
const pending: VmOperation[] = [{ kind: 'wake', settled }]
|
||||
const answer = awaitInFlightWake(pending, false, new WorkspaceSleepRequests(), 'w', 5000)
|
||||
finish()
|
||||
await expect(answer).resolves.toBe('up')
|
||||
})
|
||||
|
||||
it('ends slept when a Sleep is asked for while it waits, and waking when the hold runs out', async () => {
|
||||
const sleeps = new WorkspaceSleepRequests()
|
||||
const never: VmOperation[] = [{ kind: 'wake', settled: new Promise(() => {}) }]
|
||||
const slept = awaitInFlightWake(never, false, sleeps, 'w', 5000)
|
||||
sleeps.noteRequested('w')
|
||||
await expect(slept).resolves.toBe('slept')
|
||||
await expect(awaitInFlightWake(never, false, sleeps, 'w', 5)).resolves.toBe('waking')
|
||||
})
|
||||
})
|
||||
@@ -124,3 +124,26 @@ export async function holdOpenUntilUp(
|
||||
stopAbortWatch()
|
||||
}
|
||||
}
|
||||
|
||||
/** Holds a request that does not open the workspace while the wake already in flight settles; a VM
|
||||
* that is not waking answers at once with where it stands. Never starts a wake. */
|
||||
export async function awaitInFlightWake(
|
||||
pending: readonly VmOperation[],
|
||||
recordSuspended: boolean,
|
||||
sleepRequests: WorkspaceSleepRequests,
|
||||
worktreeId: string,
|
||||
timeoutMs: number,
|
||||
signal?: AbortSignal
|
||||
): Promise<RecipeVmOpening | 'asleep'> {
|
||||
signal?.throwIfAborted()
|
||||
const state = vmStateOf(pending, recordSuspended)
|
||||
if (state !== 'waking') {
|
||||
return state === 'up' ? 'up' : 'asleep'
|
||||
}
|
||||
const sleep = sleepRequests.next(worktreeId)
|
||||
try {
|
||||
return await holdOpenUntilUp(pendingVmWake(pending), sleep.requested, timeoutMs, signal)
|
||||
} finally {
|
||||
sleep.stop()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,14 @@ export type RecipeVmWaker = {
|
||||
timeoutMs: number,
|
||||
signal?: AbortSignal
|
||||
) => Promise<RecipeVmOpening>
|
||||
/** Holds a request that does not open the workspace while a wake already in flight finishes;
|
||||
* never starts one. */
|
||||
awaitWake?: (
|
||||
workspaceId: string,
|
||||
recordSuspended: boolean,
|
||||
timeoutMs: number,
|
||||
signal?: AbortSignal
|
||||
) => Promise<RecipeVmOpening | 'asleep'>
|
||||
/** Holds an open of the workspace behind a saved environment (id or name) for the CLI. */
|
||||
openEnvironment: (selector: string) => Promise<RecipeVmOpening>
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ import type {
|
||||
WorkspaceSleepRequest
|
||||
} from '../../../../shared/workspace-sleep'
|
||||
import { WorkspaceSleepCoordinator } from './workspace-sleep-runtime'
|
||||
import type { RecipeVmOpening, RecipeVmState } from './recipe-vm-operations'
|
||||
import { awaitInFlightWake, type RecipeVmOpening, type RecipeVmState } from './recipe-vm-operations'
|
||||
import { dropStrandedSleepRecord } from '../../terminal/stop/worktree-sleep-resume-records'
|
||||
import type { RestoredPane } from '../../terminal/recovery/restored-pane-start'
|
||||
import type { SleepingAgentSessionRecord } from '../../../../shared/agent-session/lifecycle/agent-session-resume'
|
||||
@@ -166,6 +166,18 @@ export abstract class RuntimeWorkspaceSleepOperations extends OrcaRuntimeWithRes
|
||||
return this.workspaceSleep.holdOpen(worktreeId, recordSuspended, timeoutMs, signal)
|
||||
}
|
||||
|
||||
/** Holds a request that does not open this workspace while its in-flight wake finishes. */
|
||||
awaitWorkspaceVmWake(
|
||||
worktreeId: string,
|
||||
recordSuspended: boolean,
|
||||
timeoutMs: number,
|
||||
signal?: AbortSignal
|
||||
): Promise<RecipeVmOpening | 'asleep'> {
|
||||
const pending = this.workspaceSleep.pendingVmOperations.get(worktreeId) ?? []
|
||||
const sleeps = this.workspaceSleep.sleepRequests
|
||||
return awaitInFlightWake(pending, recordSuspended, sleeps, worktreeId, timeoutMs, signal)
|
||||
}
|
||||
|
||||
/** The window opened a slept pane; main resumes it and the window attaches. */
|
||||
resumeSleptPane(pane: RestoredPane, automatic?: boolean): Promise<boolean> {
|
||||
return this.workspaceSleep.resumePane(pane, automatic)
|
||||
|
||||
@@ -95,8 +95,9 @@ export class WorkspaceSleepCoordinator {
|
||||
private readonly reopenings = new Map<string, number>()
|
||||
private readonly vmOperations = new Map<string, Promise<void>>()
|
||||
/** The VM operations asked for per workspace, in order, until each settles. */
|
||||
private readonly pendingVmOperations = new Map<string, VmOperation[]>()
|
||||
private readonly sleepRequests = new WorkspaceSleepRequests()
|
||||
/** Read by the in-flight wake hold (awaitInFlightWake); written only here. */
|
||||
readonly pendingVmOperations = new Map<string, VmOperation[]>()
|
||||
readonly sleepRequests = new WorkspaceSleepRequests()
|
||||
|
||||
constructor(private readonly runtime: WorkspaceSleepRuntime) {}
|
||||
|
||||
|
||||
@@ -128,13 +128,17 @@ export function initializeMainProcessRuntime(): OrcaRuntimeService {
|
||||
onRuntimeEnvironmentStatusPublished(trackRuntimeEnvironmentVerifiedTransitions(listener))
|
||||
},
|
||||
workspaceSleepPlatform: {
|
||||
// Sleep's own calls and observations never wait on a wake in flight.
|
||||
callEnvironment: (environmentId, method, params, timeoutMs) =>
|
||||
callRuntimeEnvironment(
|
||||
getRuntimeEnvironmentRegistryRoot(),
|
||||
environmentId,
|
||||
method,
|
||||
params,
|
||||
timeoutMs
|
||||
timeoutMs,
|
||||
undefined,
|
||||
undefined,
|
||||
{ preparation: { recipeVmHoldMs: 0 } }
|
||||
),
|
||||
ownsVm: (workspaceId) =>
|
||||
findWorkspaceEphemeralVmRuntime(app.getPath('userData'), workspaceId) !== undefined,
|
||||
@@ -166,6 +170,8 @@ export function initializeMainProcessRuntime(): OrcaRuntimeService {
|
||||
noteOpened: (workspaceId) => runtime.noteWorkspaceReopened(workspaceId),
|
||||
holdOpen: (workspaceId, suspended, timeoutMs, signal) =>
|
||||
runtime.holdWorkspaceVmOpen(workspaceId, suspended, timeoutMs, signal),
|
||||
awaitWake: (workspaceId, suspended, timeoutMs, signal) =>
|
||||
runtime.awaitWorkspaceVmWake(workspaceId, suspended, timeoutMs, signal),
|
||||
openEnvironment: (selector) => {
|
||||
const root = getRuntimeEnvironmentRegistryRoot()
|
||||
return openRecipeVmEnvironment(root, resolveEnvironment(root, selector))
|
||||
|
||||
@@ -127,7 +127,9 @@ export function CloudWakingOverlay({
|
||||
: wake.source === 'restart'
|
||||
? t('orcaCloudWorkspace.announce.restarting', 'Restarting {{name}}.', { name })
|
||||
: wake.source === 'unarchive'
|
||||
? t('orcaCloudWorkspace.announce.unarchiving', 'Restoring {{name}} from archive.', { name })
|
||||
? t('orcaCloudWorkspace.announce.unarchiving', 'Restoring {{name}} from archive.', {
|
||||
name
|
||||
})
|
||||
: t('orcaCloudWorkspace.announce.waking', 'Waking {{name}}.', { name })
|
||||
}
|
||||
>
|
||||
|
||||
@@ -44,7 +44,12 @@ const running: HostedWorkspaceStatus = {
|
||||
archive: null,
|
||||
checkedAt: NOW
|
||||
}
|
||||
const asleep: HostedWorkspaceStatus = { ...running, state: 'sleeping', metrics: null, uptimeMs: null }
|
||||
const asleep: HostedWorkspaceStatus = {
|
||||
...running,
|
||||
state: 'sleeping',
|
||||
metrics: null,
|
||||
uptimeMs: null
|
||||
}
|
||||
const entry = (
|
||||
status: HostedWorkspaceStatus | null,
|
||||
over: Partial<CloudStatusEntry> = {}
|
||||
@@ -100,7 +105,11 @@ export const panelUnresponsive = defineFixture({
|
||||
export const panelRestarting = defineFixture({
|
||||
width: 300,
|
||||
render: () => (
|
||||
<Panel state="unresponsive" value={entry({ ...running, state: 'unresponsive' })} pending="restart" />
|
||||
<Panel
|
||||
state="unresponsive"
|
||||
value={entry({ ...running, state: 'unresponsive' })}
|
||||
pending="restart"
|
||||
/>
|
||||
)
|
||||
})
|
||||
|
||||
@@ -298,7 +307,7 @@ function IdleToast(): React.JSX.Element {
|
||||
return (
|
||||
<>
|
||||
<div className="h-[200px] bg-background" />
|
||||
<Toaster position="top-left" toastOptions={{ className: "font-sans text-sm" }} />
|
||||
<Toaster position="top-left" toastOptions={{ className: 'font-sans text-sm' }} />
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -109,7 +109,9 @@ export function runArchiveCloudWorkspace(
|
||||
}, options.undoMs ?? ARCHIVE_UNDO_MS)
|
||||
toast(
|
||||
options.name
|
||||
? translate('orcaCloudWorkspace.archivingNamed', 'Archiving {{name}}…', { name: options.name })
|
||||
? translate('orcaCloudWorkspace.archivingNamed', 'Archiving {{name}}…', {
|
||||
name: options.name
|
||||
})
|
||||
: translate('orcaCloudWorkspace.archiving', 'Archiving workspace…'),
|
||||
{
|
||||
id: toastId,
|
||||
|
||||
@@ -96,7 +96,9 @@ export function CloudWorkspaceOfflineView(props: CloudOfflineViewProps): React.J
|
||||
</TabsTrigger>
|
||||
<TabsTrigger value="changes">
|
||||
{t('components.orcaCloud.offline.tabChanges', 'Changes')}
|
||||
{changeCount > 0 ? <span className="text-muted-foreground">{changeCount}</span> : null}
|
||||
{changeCount > 0 ? (
|
||||
<span className="text-muted-foreground">{changeCount}</span>
|
||||
) : null}
|
||||
</TabsTrigger>
|
||||
</TabsList>
|
||||
<TabsContent value="chat" className="flex min-h-0 flex-1 flex-col pt-3">
|
||||
|
||||
+1
-4
@@ -32,10 +32,7 @@ export function WorktreeCloudTransferMenuItem({
|
||||
return (
|
||||
<DropdownMenuItem onSelect={() => void runOrcaCloudTransferToLocal(worktreeId)}>
|
||||
<Laptop className="size-3.5" />
|
||||
{t(
|
||||
'auto.components.sidebar.WorktreeContextMenu.moveToThisComputer',
|
||||
'Move to this computer'
|
||||
)}
|
||||
{t('auto.components.sidebar.WorktreeContextMenu.moveToThisComputer', 'Move to this computer')}
|
||||
</DropdownMenuItem>
|
||||
)
|
||||
}
|
||||
|
||||
+2
@@ -1,3 +1,4 @@
|
||||
import { markInterruptedTurnContinued } from './startup/cold-restore-resume-startup'
|
||||
import { paneResumeRecord } from './startup/pane-resume-record'
|
||||
import { useAppStore } from '@/store/index'
|
||||
import type { PtyReplayDataMeta } from '../pty-transport/pty-transport-types'
|
||||
@@ -34,6 +35,7 @@ export function bindDeferredColdRestoreAndSnapshot(session: ConnectPanePtySessio
|
||||
if (!startup || startup.resumeSource?.cancelled) {
|
||||
return false
|
||||
}
|
||||
markInterruptedTurnContinued(startup.interruptedTurnKey)
|
||||
const state = useAppStore.getState()
|
||||
if (!session.pendingStartupCommand && session.shouldDeliverStartupViaTerminalPaste) {
|
||||
session.pendingStartupCommand = {
|
||||
|
||||
+5
@@ -1,5 +1,6 @@
|
||||
import {
|
||||
NOT_REOPENED,
|
||||
markInterruptedTurnContinued,
|
||||
reopenPaneAfterMissingReattach
|
||||
} from '../startup/cold-restore-resume-startup'
|
||||
import { writeStartupCwdFallbackNotice } from '../startup/startup-cwd-fallback-notice'
|
||||
@@ -134,6 +135,10 @@ function connectDeferredSessionReattach(
|
||||
)
|
||||
session.beginReattachLiveDataDeferral(outputCallbacks.generation)
|
||||
session.transportConnectInFlightSince = Date.now()
|
||||
// The command goes to the server now: a `continue` it carries is spent.
|
||||
if (coldRestoreStartup?.command) {
|
||||
markInterruptedTurnContinued(coldRestoreStartup.interruptedTurnKey)
|
||||
}
|
||||
const reattachPromise = session.transport.connect({
|
||||
url: '',
|
||||
cols: session.cols,
|
||||
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
vi.mock('@/store/index', () => ({ useAppStore: { getState: () => ({}) } }))
|
||||
|
||||
describe('shouldContinueInterruptedTurn', () => {
|
||||
const now = 1_800_000_000_000
|
||||
const working = (stateStartedAt: number) => ({
|
||||
state: 'working',
|
||||
updatedAt: now - 1000,
|
||||
stateStartedAt
|
||||
})
|
||||
|
||||
it('continues a remote agent with a fresh working status, once per interrupted turn', async () => {
|
||||
const { shouldContinueInterruptedTurn, interruptedTurnKey, markInterruptedTurnContinued } =
|
||||
await import('./cold-restore-resume-startup')
|
||||
const ctx = { paneKey: 'p1', hasSleepingRecord: false, now }
|
||||
expect(shouldContinueInterruptedTurn(working(1), 'env-1', ctx)).toBe(true)
|
||||
// Building (and discarding) the startup again does not spend the turn.
|
||||
expect(shouldContinueInterruptedTurn(working(1), 'env-1', ctx)).toBe(true)
|
||||
// Once the resume launched, a retried cold restore of the same turn does not send it again.
|
||||
markInterruptedTurnContinued(interruptedTurnKey(working(1), 'p1'))
|
||||
expect(shouldContinueInterruptedTurn(working(1), 'env-1', ctx)).toBe(false)
|
||||
// A later interrupted turn does.
|
||||
expect(shouldContinueInterruptedTurn(working(2), 'env-1', ctx)).toBe(true)
|
||||
})
|
||||
|
||||
it('never continues after a deliberate sleep, for stale or non-working status, or locally', async () => {
|
||||
const { shouldContinueInterruptedTurn } = await import('./cold-restore-resume-startup')
|
||||
expect(
|
||||
shouldContinueInterruptedTurn(working(10), 'env-1', {
|
||||
paneKey: 'p2',
|
||||
hasSleepingRecord: true,
|
||||
now
|
||||
})
|
||||
).toBe(false)
|
||||
expect(
|
||||
shouldContinueInterruptedTurn(
|
||||
{ state: 'working', updatedAt: now - 31 * 60_000, stateStartedAt: 11 },
|
||||
'env-1',
|
||||
{
|
||||
paneKey: 'p3',
|
||||
hasSleepingRecord: false,
|
||||
now
|
||||
}
|
||||
)
|
||||
).toBe(false)
|
||||
expect(
|
||||
shouldContinueInterruptedTurn(
|
||||
{ state: 'waiting', updatedAt: now, stateStartedAt: 12 },
|
||||
'env-1',
|
||||
{ paneKey: 'p4', hasSleepingRecord: false, now }
|
||||
)
|
||||
).toBe(false)
|
||||
expect(
|
||||
shouldContinueInterruptedTurn(working(13), null, {
|
||||
paneKey: 'p5',
|
||||
hasSleepingRecord: false,
|
||||
now
|
||||
})
|
||||
).toBe(false)
|
||||
})
|
||||
})
|
||||
+57
-2
@@ -6,7 +6,10 @@ import { normalizeExecutionHostId } from '../../../../../../shared/execution-hos
|
||||
import { parseWorkspaceKey } from '../../../../../../shared/workspace-scope'
|
||||
import { useAppStore } from '@/store/index'
|
||||
import { createNonSecureContextUuid as createBrowserUuid } from '../../../../../../shared/non-secure-context-uuid'
|
||||
import { buildAgentResumeStartupPlan } from '../../../../../../shared/agent-launch/startup/tui-agent-resume-startup'
|
||||
import {
|
||||
INTERRUPTED_TURN_CONTINUE_PROMPT,
|
||||
buildAgentResumeStartupPlan
|
||||
} from '../../../../../../shared/agent-launch/startup/tui-agent-resume-startup'
|
||||
import {
|
||||
resolveRuntimeHostReportedPlatform,
|
||||
resolveRuntimeHostTerminalWindowsShell,
|
||||
@@ -24,6 +27,7 @@ import {
|
||||
normalizeAgentProviderSession
|
||||
} from '../../../../../../shared/agent-session/lifecycle/agent-session-resume'
|
||||
|
||||
import { AGENT_STATUS_STALE_AFTER_MS } from '../../../../../../shared/agent-status/contracts/agent-status-freshness'
|
||||
import type { HeldPaneReopenOutcome } from '../../session-restore/newer-daemon/newer-daemon-hold-state'
|
||||
import type { ColdRestoreAgentResumeStartup } from './fresh-spawn-types'
|
||||
|
||||
@@ -31,6 +35,45 @@ import type { ConnectPanePtySession } from '../connect-pane-pty-session'
|
||||
|
||||
export const NOT_REOPENED: HeldPaneReopenOutcome = { reopened: false, agentResumed: false }
|
||||
|
||||
/** Interrupted turns already continued, so a retried cold restore never sends `continue` twice. */
|
||||
const continuedInterruptedTurns = new Set<string>()
|
||||
|
||||
/** Identity of the turn an agent was in when its host went away. */
|
||||
export function interruptedTurnKey(
|
||||
entry: { updatedAt: number; stateStartedAt?: number },
|
||||
paneKey: string
|
||||
): string {
|
||||
return `${paneKey}\u0000${entry.stateStartedAt ?? entry.updatedAt}`
|
||||
}
|
||||
|
||||
/** Called when a resume that carries `continue` actually launches (startups are often built and
|
||||
* discarded, e.g. on every reattach check). */
|
||||
export function markInterruptedTurnContinued(key: string | undefined): void {
|
||||
if (key) {
|
||||
continuedInterruptedTurns.add(key)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A cold-restored agent on a remote host whose last status was working lost its turn when the host
|
||||
* restarted (reboot, crash, rollover): carry it on. Only with fresh evidence (a recent working
|
||||
* status), never after a deliberate sleep (that leaves a sleeping record), once per interrupted turn.
|
||||
* Local panes are left alone: their PTYs outlive app restarts.
|
||||
*/
|
||||
export function shouldContinueInterruptedTurn(
|
||||
entry: { state: string; updatedAt: number; stateStartedAt?: number } | null | undefined,
|
||||
runtimeEnvironmentId: string | null,
|
||||
context: { paneKey: string; hasSleepingRecord: boolean; now?: number }
|
||||
): boolean {
|
||||
if (!runtimeEnvironmentId || context.hasSleepingRecord || entry?.state !== 'working') {
|
||||
return false
|
||||
}
|
||||
if ((context.now ?? Date.now()) - entry.updatedAt > AGENT_STATUS_STALE_AFTER_MS) {
|
||||
return false
|
||||
}
|
||||
return !continuedInterruptedTurns.has(interruptedTurnKey(entry, context.paneKey))
|
||||
}
|
||||
|
||||
type AppState = ReturnType<typeof useAppStore.getState>
|
||||
|
||||
/** The agent session a cold restore of this pane would resume; read-only, so pre-spawn checks can share it. */
|
||||
@@ -109,6 +152,12 @@ export function bindBuildColdRestoreAgentResumeStartup(session: ConnectPanePtySe
|
||||
session.executionHostId
|
||||
)
|
||||
})
|
||||
const continueTurn =
|
||||
entry !== null &&
|
||||
shouldContinueInterruptedTurn(entry, session.transport.getRuntimeEnvironmentId?.() ?? null, {
|
||||
paneKey: session.cacheKey,
|
||||
hasSleepingRecord: Boolean(sleepingRecord)
|
||||
})
|
||||
const startupPlan = buildAgentResumeStartupPlan({
|
||||
agent,
|
||||
providerSession,
|
||||
@@ -117,7 +166,10 @@ export function bindBuildColdRestoreAgentResumeStartup(session: ConnectPanePtySe
|
||||
platform: resumeTarget.platform,
|
||||
agentHostPlatform: resumeTarget.agentHostPlatform,
|
||||
shell: resumeTarget.shell,
|
||||
launchCwd: session.worktree?.path
|
||||
launchCwd: session.worktree?.path,
|
||||
// A remote host that restarted under a working agent (reboot, crash, VM rollover) killed it
|
||||
// mid-turn; its last status still says working. Carry the turn on instead of waiting.
|
||||
continuePrompt: continueTurn ? INTERRUPTED_TURN_CONTINUE_PROMPT : null
|
||||
})
|
||||
if (!startupPlan) {
|
||||
return null
|
||||
@@ -151,6 +203,9 @@ export function bindBuildColdRestoreAgentResumeStartup(session: ConnectPanePtySe
|
||||
: {})
|
||||
},
|
||||
launchToken: coldRestoreLaunchToken,
|
||||
...(continueTurn && entry
|
||||
? { interruptedTurnKey: interruptedTurnKey(entry, session.cacheKey) }
|
||||
: {}),
|
||||
useLiveEntry: Boolean(useLiveEntry),
|
||||
hasSleepingRecord: Boolean(sleepingRecord),
|
||||
sleepingRecordEntry
|
||||
|
||||
@@ -20,6 +20,8 @@ export type FreshSpawnOptions = {
|
||||
}
|
||||
|
||||
export type ColdRestoreAgentResumeStartup = PendingStartupCommand & {
|
||||
/** Set when the command carries `continue` for an interrupted turn; marked used on launch. */
|
||||
interruptedTurnKey?: string
|
||||
agent: ResumableTuiAgent
|
||||
resumeProviderSession: AgentProviderSessionMetadata
|
||||
connectionId?: string | null
|
||||
|
||||
@@ -14,7 +14,10 @@ import {
|
||||
} from './orca-cloud-workspace-state'
|
||||
|
||||
const NOW = 1_800_000_000_000
|
||||
const status = (state: HostedWorkspaceStatus['state'], archive: HostedWorkspaceStatus['archive'] = null): HostedWorkspaceStatus => ({
|
||||
const status = (
|
||||
state: HostedWorkspaceStatus['state'],
|
||||
archive: HostedWorkspaceStatus['archive'] = null
|
||||
): HostedWorkspaceStatus => ({
|
||||
state,
|
||||
metrics: null,
|
||||
uptimeMs: null,
|
||||
@@ -32,18 +35,39 @@ const wake = (phase: CloudWakeProgressEvent['phase'], at = NOW): CloudWakeProgre
|
||||
|
||||
describe('deriveCloudDotState', () => {
|
||||
it('shows an in-flight wake over everything, including a sleeping desktop record', () => {
|
||||
expect(deriveCloudDotState({ status: status('sleeping'), wake: wake('waking'), sleeping: true, now: NOW })).toBe('waking')
|
||||
expect(deriveCloudDotState({ status: status('ready'), wake: wake('connecting'), sleeping: false, now: NOW })).toBe('waking')
|
||||
expect(
|
||||
deriveCloudDotState({
|
||||
status: status('sleeping'),
|
||||
wake: wake('waking'),
|
||||
sleeping: true,
|
||||
now: NOW
|
||||
})
|
||||
).toBe('waking')
|
||||
expect(
|
||||
deriveCloudDotState({
|
||||
status: status('ready'),
|
||||
wake: wake('connecting'),
|
||||
sleeping: false,
|
||||
now: NOW
|
||||
})
|
||||
).toBe('waking')
|
||||
})
|
||||
|
||||
it('treats a wake with no terminal event for 4 minutes as stale', () => {
|
||||
expect(
|
||||
deriveCloudDotState({ status: status('ready'), wake: wake('waking', NOW - 5 * 60_000), sleeping: false, now: NOW })
|
||||
deriveCloudDotState({
|
||||
status: status('ready'),
|
||||
wake: wake('waking', NOW - 5 * 60_000),
|
||||
sleeping: false,
|
||||
now: NOW
|
||||
})
|
||||
).toBe('running')
|
||||
})
|
||||
|
||||
it('archived beats the desktop sleep state; failed wakes show until stale', () => {
|
||||
expect(deriveCloudDotState({ status: status('archived'), wake: undefined, sleeping: true, now: NOW })).toBe('archived')
|
||||
expect(
|
||||
deriveCloudDotState({ status: status('archived'), wake: undefined, sleeping: true, now: NOW })
|
||||
).toBe('archived')
|
||||
expect(
|
||||
deriveCloudDotState({
|
||||
status: status('sleeping', { archivedAt: NOW, expiresAt: null }),
|
||||
@@ -52,16 +76,45 @@ describe('deriveCloudDotState', () => {
|
||||
now: NOW
|
||||
})
|
||||
).toBe('archived')
|
||||
expect(deriveCloudDotState({ status: status('sleeping'), wake: wake('failed'), sleeping: true, now: NOW })).toBe('failed')
|
||||
expect(
|
||||
deriveCloudDotState({
|
||||
status: status('sleeping'),
|
||||
wake: wake('failed'),
|
||||
sleeping: true,
|
||||
now: NOW
|
||||
})
|
||||
).toBe('failed')
|
||||
})
|
||||
|
||||
it('maps control-plane states; the local sleep record only fills in before the first read', () => {
|
||||
expect(deriveCloudDotState({ status: status('ready'), wake: undefined, sleeping: false, now: NOW })).toBe('running')
|
||||
expect(deriveCloudDotState({ status: status('unresponsive'), wake: undefined, sleeping: false, now: NOW })).toBe('unresponsive')
|
||||
expect(deriveCloudDotState({ status: status('ready'), wake: undefined, sleeping: true, now: NOW })).toBe('running')
|
||||
expect(deriveCloudDotState({ status: null, wake: undefined, sleeping: true, now: NOW })).toBe('sleeping')
|
||||
expect(deriveCloudDotState({ status: status('provisioning'), wake: undefined, sleeping: false, now: NOW })).toBe('waking')
|
||||
expect(deriveCloudDotState({ status: null, wake: undefined, sleeping: false, now: NOW })).toBe('unknown')
|
||||
expect(
|
||||
deriveCloudDotState({ status: status('ready'), wake: undefined, sleeping: false, now: NOW })
|
||||
).toBe('running')
|
||||
expect(
|
||||
deriveCloudDotState({
|
||||
status: status('unresponsive'),
|
||||
wake: undefined,
|
||||
sleeping: false,
|
||||
now: NOW
|
||||
})
|
||||
).toBe('unresponsive')
|
||||
expect(
|
||||
deriveCloudDotState({ status: status('ready'), wake: undefined, sleeping: true, now: NOW })
|
||||
).toBe('running')
|
||||
expect(deriveCloudDotState({ status: null, wake: undefined, sleeping: true, now: NOW })).toBe(
|
||||
'sleeping'
|
||||
)
|
||||
expect(
|
||||
deriveCloudDotState({
|
||||
status: status('provisioning'),
|
||||
wake: undefined,
|
||||
sleeping: false,
|
||||
now: NOW
|
||||
})
|
||||
).toBe('waking')
|
||||
expect(deriveCloudDotState({ status: null, wake: undefined, sleeping: false, now: NOW })).toBe(
|
||||
'unknown'
|
||||
)
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -19,7 +19,10 @@ const status = (state: HostedWorkspaceStatus['state']): HostedWorkspaceStatus =>
|
||||
checkedAt: 1
|
||||
})
|
||||
|
||||
function stub(getStatus: OrcaCloudWorkspaceApi['getStatus'], ids: string[] = []): OrcaCloudWorkspaceApi {
|
||||
function stub(
|
||||
getStatus: OrcaCloudWorkspaceApi['getStatus'],
|
||||
ids: string[] = []
|
||||
): OrcaCloudWorkspaceApi {
|
||||
const bridge = {
|
||||
getStatus: vi.fn(getStatus),
|
||||
listCloudWorkspaceIds: vi.fn(async () => ids)
|
||||
@@ -48,7 +51,13 @@ describe('wake state and archive guard', () => {
|
||||
|
||||
it('a failed wake stays until dismissed', async () => {
|
||||
stub(async () => ({ ok: true, value: status('sleeping') }))
|
||||
noteCloudWakeProgress({ workspaceId: 'a', phase: 'failed', source: 'open', startedAt: 1, at: 2 })
|
||||
noteCloudWakeProgress({
|
||||
workspaceId: 'a',
|
||||
phase: 'failed',
|
||||
source: 'open',
|
||||
startedAt: 1,
|
||||
at: 2
|
||||
})
|
||||
expect(getCloudWorkspaceSnapshot().wake.a?.phase).toBe('failed')
|
||||
dismissCloudWake('a')
|
||||
expect(getCloudWorkspaceSnapshot().wake.a).toBeUndefined()
|
||||
|
||||
@@ -28,7 +28,7 @@ const ready: HostedWorkspaceStatus = {
|
||||
archive: null,
|
||||
checkedAt: 1
|
||||
}
|
||||
const ok = <T,>(value: T): CloudWorkspaceResult<T> => ({ ok: true, value })
|
||||
const ok = <T>(value: T): CloudWorkspaceResult<T> => ({ ok: true, value })
|
||||
|
||||
function stubApi(overrides: Partial<OrcaCloudWorkspaceApi> = {}): OrcaCloudWorkspaceApi {
|
||||
const bridge: OrcaCloudWorkspaceApi = {
|
||||
@@ -42,7 +42,9 @@ function stubApi(overrides: Partial<OrcaCloudWorkspaceApi> = {}): OrcaCloudWorks
|
||||
onOom: vi.fn(() => () => {}),
|
||||
...overrides
|
||||
}
|
||||
;(globalThis as unknown as { window: { api: { orcaCloudWorkspace: OrcaCloudWorkspaceApi } } }).window = {
|
||||
;(
|
||||
globalThis as unknown as { window: { api: { orcaCloudWorkspace: OrcaCloudWorkspaceApi } } }
|
||||
).window = {
|
||||
api: { orcaCloudWorkspace: bridge }
|
||||
}
|
||||
return bridge
|
||||
@@ -66,7 +68,11 @@ describe('orca cloud workspace store', () => {
|
||||
const bridge = stubApi()
|
||||
await Promise.all([refreshCloudWorkspaceStatus('a'), refreshCloudWorkspaceStatus('a')])
|
||||
expect(bridge.getStatus).toHaveBeenCalledTimes(1)
|
||||
bridge.getStatus = vi.fn(async () => ({ ok: false as const, code: 'unavailable' as const, message: 'down' }))
|
||||
bridge.getStatus = vi.fn(async () => ({
|
||||
ok: false as const,
|
||||
code: 'unavailable' as const,
|
||||
message: 'down'
|
||||
}))
|
||||
await refreshCloudWorkspaceStatus('a')
|
||||
const entry = getCloudWorkspaceSnapshot().status.a!
|
||||
expect(entry.status?.state).toBe('ready')
|
||||
@@ -75,7 +81,13 @@ describe('orca cloud workspace store', () => {
|
||||
})
|
||||
|
||||
it('forgets a workspace the main process says is not cloud', async () => {
|
||||
stubApi({ getStatus: vi.fn(async () => ({ ok: false as const, code: 'not_cloud' as const, message: 'no' })) })
|
||||
stubApi({
|
||||
getStatus: vi.fn(async () => ({
|
||||
ok: false as const,
|
||||
code: 'not_cloud' as const,
|
||||
message: 'no'
|
||||
}))
|
||||
})
|
||||
seedCloudWorkspaceSnapshot({ ids: new Set(['a']) })
|
||||
await refreshCloudWorkspaceStatus('a')
|
||||
expect(getCloudWorkspaceSnapshot().ids.has('a')).toBe(false)
|
||||
@@ -117,7 +129,13 @@ describe('orca cloud workspace store', () => {
|
||||
|
||||
it('tracks wake progress (adding the id) and dismisses OOM notices up to the seen count', async () => {
|
||||
stubApi()
|
||||
noteCloudWakeProgress({ workspaceId: 'c', phase: 'waking', source: 'message', startedAt: 1, at: 2 })
|
||||
noteCloudWakeProgress({
|
||||
workspaceId: 'c',
|
||||
phase: 'waking',
|
||||
source: 'message',
|
||||
startedAt: 1,
|
||||
at: 2
|
||||
})
|
||||
expect(getCloudWorkspaceSnapshot().ids.has('c')).toBe(true)
|
||||
noteCloudOom({ workspaceId: 'c', oomKills: 1, victim: 'node', at: 3 })
|
||||
dismissCloudOom('c')
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { buildAgentResumeStartupPlan } from './tui-agent-resume-startup'
|
||||
|
||||
const providerSession = { key: 'session_id', id: 'claude-session-1' } as const
|
||||
|
||||
describe('resume with a continue prompt', () => {
|
||||
const base = { cmdOverrides: {}, platform: 'linux' as const, shell: 'posix' as const }
|
||||
|
||||
it('submits the prompt on resume for Claude and Codex', () => {
|
||||
const claude = buildAgentResumeStartupPlan({
|
||||
...base,
|
||||
agent: 'claude',
|
||||
providerSession,
|
||||
continuePrompt: 'continue'
|
||||
})
|
||||
expect(claude?.launchCommand).toMatch(/'?--resume'? '?claude-session-1'? '?continue'?$/)
|
||||
const codex = buildAgentResumeStartupPlan({
|
||||
...base,
|
||||
agent: 'codex',
|
||||
providerSession: { key: 'session_id', id: 'codex-1' },
|
||||
continuePrompt: 'continue'
|
||||
})
|
||||
expect(codex?.launchCommand).toMatch(/'?resume'? '?codex-1'? '?continue'?$/)
|
||||
})
|
||||
|
||||
it('leaves other agents and empty prompts unchanged', () => {
|
||||
const plain = buildAgentResumeStartupPlan({ ...base, agent: 'claude', providerSession })
|
||||
const blank = buildAgentResumeStartupPlan({
|
||||
...base,
|
||||
agent: 'claude',
|
||||
providerSession,
|
||||
continuePrompt: ' '
|
||||
})
|
||||
expect(blank?.launchCommand).toBe(plain?.launchCommand)
|
||||
const cursor = buildAgentResumeStartupPlan({
|
||||
...base,
|
||||
agent: 'cursor',
|
||||
providerSession: { key: 'conversation_id', id: 'c-1' },
|
||||
continuePrompt: 'continue'
|
||||
})
|
||||
expect(cursor?.launchCommand).not.toMatch(/continue'?$/)
|
||||
})
|
||||
})
|
||||
@@ -37,6 +37,9 @@ export function buildAgentResumeStartupPlan(args: {
|
||||
fork?: boolean
|
||||
/** OS that runs the agent when `platform` only picks quoting; null when unreported. */
|
||||
agentHostPlatform?: NodeJS.Platform | null
|
||||
/** First message the resumed session submits, e.g. to carry on a turn its host cut off. Only
|
||||
* agents whose resume takes a prompt argument (Claude, Codex) send it; others ignore it. */
|
||||
continuePrompt?: string | null
|
||||
}): AgentStartupPlan | null {
|
||||
const resumeArgv = args.fork
|
||||
? getAgentForkArgv(args.agent, args.providerSession)
|
||||
@@ -45,10 +48,15 @@ export function buildAgentResumeStartupPlan(args: {
|
||||
return null
|
||||
}
|
||||
// Codex otherwise asks for its old folder, even if deleted (#17745). Keep before resume so argv removal preserves the folder choice.
|
||||
const argv =
|
||||
const withCwd =
|
||||
args.agent === 'codex' && args.resumeInLaunchCwd
|
||||
? [resumeArgv[0], '-c', 'tui.resume_cwd=current', ...resumeArgv.slice(1)]
|
||||
: resumeArgv
|
||||
const continuePrompt = args.continuePrompt?.trim()
|
||||
const argv =
|
||||
continuePrompt && !args.fork && RESUME_PROMPT_AGENTS.has(args.agent)
|
||||
? [...withCwd, continuePrompt]
|
||||
: withCwd
|
||||
const shell = resolveStartupShell(args.platform, args.shell)
|
||||
// Why: Codex restores a resumed thread's saved permissions unless the launch passes one, so
|
||||
// replaying Orca's launch-time flag would undo a change made inside the session (#10886).
|
||||
@@ -112,6 +120,12 @@ export function buildAgentResumeStartupPlan(args: {
|
||||
}
|
||||
}
|
||||
|
||||
/** Agents whose resume command accepts a positional prompt it submits on start. */
|
||||
const RESUME_PROMPT_AGENTS: ReadonlySet<ResumableTuiAgent> = new Set(['claude', 'codex'])
|
||||
|
||||
/** What a resumed agent is told when its host went away mid-turn (restart, reboot, crash). */
|
||||
export const INTERRUPTED_TURN_CONTINUE_PROMPT = 'continue'
|
||||
|
||||
/** WSL panes start at a UNC path while the agent reports its Linux path. */
|
||||
function isSameFolder(agentCwd: string, launchCwd: string): boolean {
|
||||
const launchPath = parseWslUncPath(launchCwd)?.linuxPath ?? launchCwd
|
||||
|
||||
Reference in New Issue
Block a user