Merge pull request #35 from nwparker/cloud/w2-lifecycle

feat(cloud): workspace lifecycle API (wake, restart, status, archive) + desktop service
This commit is contained in:
Neil
2026-11-09 05:06:27 -08:00
committed by GitHub
28 changed files with 2080 additions and 48 deletions
@@ -0,0 +1,18 @@
-- Workspace lifecycle (wake, status, archive).
-- orcad's public wss endpoint for the current session (not a secret: pairing still needs the code),
-- and when orcad last became ready, so wake callers can report `readyAt` without a provider call.
ALTER TABLE sandboxes ADD COLUMN orcad_endpoint TEXT;
ALTER TABLE sandboxes ADD COLUMN ready_at INTEGER;
-- Archive = stopped with the snapshot kept for a finite TTL; the cron destroys it after archive_expires_at.
-- (`archived_at` itself comes from 0020_api_routines.sql, already applied on dev.)
ALTER TABLE sandboxes ADD COLUMN archive_expires_at INTEGER;
-- What an archived workspace shows without waking: branch, last commits, changed files, file tree, PRs.
-- Captured in the VM at archive (or sent by the desktop from its cache when the VM was already asleep).
-- Bounded (see archive.ts); deleted with the sandbox.
CREATE TABLE sandbox_archives (
sandbox_id TEXT PRIMARY KEY REFERENCES sandboxes (id),
captured_at INTEGER NOT NULL,
source TEXT NOT NULL,
metadata TEXT NOT NULL
);
+18 -1
View File
@@ -129,6 +129,9 @@ function messageBucket(caller: ApiCaller): string {
return caller.via === 'workspace' ? `msg:ws:${caller.sandboxId}` : `msg:${caller.accountId}:${caller.userId}`
}
import { wakeSandbox } from '../workspace-lifecycle'
import { parseWakeWait } from '../wire'
export function isApiPath(pathname: string): boolean {
return /^\/v1\/(me|api-keys|settings|audit|projects|workspaces|routines|oauth-approvals|oauth-grants)(\/|$)/.test(pathname)
}
@@ -409,7 +412,21 @@ async function route(request: Request, url: URL, caller: ApiCaller, ctx: ApiCont
return mutate('sleep', () => sleepWorkspace(life(), row))
}
if (sub === 'wake' && method === 'POST') {
return mutate('wake', () => wakeWorkspace(life(), row))
if (row.status === 'archived' || row.archived_at !== null) {
// The public API's wake brings an archived workspace back.
return mutate('wake', () => wakeWorkspace(life(), row))
}
// Every wake source shares W2's single-flight wake: idempotent, `?wait=30s` long-polls.
requireScope(caller, 'workspaces:write')
const started = performance.now()
const answer = await wakeSandbox(life(), row, {
// API callers expect the VM up when wake answers: wait the full 30 s unless `?wait=` says otherwise.
waitMs: url.searchParams.has('wait') ? parseWakeWait(url, null) : 30_000,
background: ctx.waitUntil
})
await auditCaller(ctx.db, caller, { action: 'workspace.wake', targetType: 'workspace', targetId: row.id, detail: { ms: Math.round(performance.now() - started), state: answer.state } }, ctx.now())
const fresh = (await getWorkspace(ctx.db, caller.accountId, id)) ?? row
return json({ workspace: workspaceView(fresh), ...answer })
}
if (sub === 'archive' && method === 'POST') {
return mutate('archive', () => archiveWorkspace(life(), row))
@@ -3,6 +3,7 @@
import type { MachineSize } from '../entitlements'
import { createSandbox, destroySandbox, resumeSandbox, suspendSandbox, type LifecycleDeps } from '../lifecycle'
import { getSandbox, type SandboxRow } from '../store'
import { archiveSandbox, unarchiveSandbox } from '../workspace-lifecycle'
import { ApiError } from './caller'
import { randomId } from './keys'
@@ -63,6 +64,8 @@ export function apiStatus(row: Pick<SandboxRow, 'status'>): string {
running: 'running',
suspending: 'sleeping',
suspended: 'sleeping',
// The API reports archive separately (`archived: true`); the VM itself is asleep.
archived: 'sleeping',
resuming: 'waking',
destroying: 'destroying',
destroyed: 'destroyed',
@@ -171,19 +174,21 @@ export async function sleepWorkspace(life: LifecycleDeps, row: WorkspaceRow): Pr
}
export async function wakeWorkspace(life: LifecycleDeps, row: WorkspaceRow): Promise<WorkspaceRow> {
const ready = await resumeSandbox(life, row)
// Waking an archived workspace brings it back (restores the snapshot TTL; boot clears archived_at).
const ready = row.status === 'archived' ? await unarchiveSandbox(life, row) : await resumeSandbox(life, row)
if (row.archived_at !== null) {
// Waking an archived workspace brings it back.
await life.db.prepare('UPDATE sandboxes SET archived_at = NULL WHERE id = ?').bind(row.id).run()
}
return (await getSandbox(life.db, ready.row.id)) as WorkspaceRow
}
/** Archive = sleep + out of default lists. The disk is kept until retention or destroy. */
/**
* Archive = stop and keep the snapshot for a TTL, out of default lists (W2's archiveSandbox:
* repo metadata captured in the VM, provider snapshot expiry shortened, cron deletes after the TTL).
*/
export async function archiveWorkspace(life: LifecycleDeps, row: WorkspaceRow): Promise<WorkspaceRow> {
const slept = row.status === 'running' ? await sleepWorkspace(life, row) : row
await life.db.prepare('UPDATE sandboxes SET archived_at = COALESCE(archived_at, ?) WHERE id = ?').bind(life.now(), row.id).run()
return (await getSandbox(life.db, slept.id)) as WorkspaceRow
const archived = await archiveSandbox(life, row, {})
return (await getSandbox(life.db, archived.id)) as WorkspaceRow
}
export async function destroyWorkspace(life: LifecycleDeps, row: WorkspaceRow): Promise<WorkspaceRow> {
+4
View File
@@ -104,4 +104,8 @@ export type Env = {
EGRESS_EXTRA_HOSTS?: string
/** GitHub App webhook secret (secret): verifies X-Hub-Signature-256 on /v1/github/webhook. */
GITHUB_WEBHOOK_SECRET?: string
/** Archived workspaces are destroyed this many days after archive (default 30). */
ARCHIVE_TTL_DAYS?: string
/** Status-panel cost estimate, cents per running hour by machine size: {"standard":75,"large":150}. */
COST_CENTS_PER_HOUR?: string
}
+80 -7
View File
@@ -5,6 +5,13 @@
// GET /v1/sandboxes/:id -> {sandbox}
// POST /v1/sandboxes/:id/suspend | /resume | /rollover -> {sandbox[, connection]}
// POST /v1/sandboxes/:id/lease {extendMs} -> {sandbox}
// POST /v1/sandboxes/:id/wake[?wait=30s] -> {state, endpoint, readyAt, sandboxId} (idempotent, single-flight;
// the public API's POST /v1/workspaces/:id/wake answers the same fields plus `workspace`)
// POST /v1/sandboxes/:id/restart -> {sandbox, connection}
// GET /v1/sandboxes/:id/status -> {sandbox, status} (live metrics, cost)
// POST /v1/sandboxes/:id/archive {pullRequests?, checkouts?} -> {sandbox}
// GET /v1/sandboxes/:id/archive -> {archive}
// POST /v1/sandboxes/:id/unarchive -> {sandbox, connection}
// DELETE /v1/sandboxes/:id -> {sandbox}
// GET /v1/secrets?scope= | PUT, DELETE /v1/secrets/:scope/:NAME (see src/secrets.ts)
// POST /v1/github/connect -> {installUrl, authorizeUrl} (bind a GitHub App installation to the account)
@@ -12,6 +19,7 @@
// Browser (GitHub redirects): GET /v1/github/setup, GET /v1/github/callback
// Operator (OPERATOR_TOKEN): /v1/internal/* (see operator.ts)
// Vercel network-policy forwardURL target (Vercel sandbox OIDC): /v1/egress/proxy
import { z } from 'zod'
import { maintainPool, poolConfigFromEnv } from './pool'
import { authenticateDesktop } from './auth'
import { authenticateApi } from './api/caller'
@@ -68,9 +76,27 @@ import { ProviderError, type SandboxProviderApi } from './sandbox-provider'
import { VmCommandError } from './sandbox-vm-commands'
import { getSandbox, getSandboxByProviderName, listAccountSandboxes, type SandboxRow } from './store'
import { createVercelSandboxApi } from './vercel-sandbox-api'
import { CreateRequestSchema, LeaseRequestSchema, readyView, sandboxView } from './wire'
import { CreateRequestSchema, LeaseRequestSchema, parseWakeWait, readyView, sandboxView } from './wire'
import { ArchiveRequestSchema } from './sandbox-vm-inspect'
import {
archiveSandbox,
readArchive,
restartSandbox,
unarchiveSandbox,
wakeSandbox,
workspaceStatus,
type WorkspaceConfig
} from './workspace-lifecycle'
const MAX_JSON_BYTES = 16 * 1024
/** Archive bodies carry the desktop's cached repo state (file trees) when the VM is already asleep. */
const MAX_ARCHIVE_JSON_BYTES = 256 * 1024
const CentsPerHourSchema = z
.object({ standard: z.number().int().nonnegative(), large: z.number().int().nonnegative() })
.partial()
.strict()
/** Desktops that understand `archived` say so; older strict clients see it as `suspended`. */
const FEATURES_HEADER = 'x-orca-sandbox-features'
/** A pasted .env file: up to 100 values. */
const MAX_BATCH_JSON_BYTES = 160 * 1024
@@ -91,6 +117,25 @@ export type Deps = {
waitUntil?: (work: Promise<unknown>) => void
/** Route groups mounted on the public API (routines, see src/routines). */
apiExtensions?: ApiExtension[]
/** Long-poll sleep (tests advance a fake clock). */
sleep?: (ms: number) => Promise<void>
}
function workspaceConfig(env: Env, deps: Deps): WorkspaceConfig {
const days = Number(env.ARCHIVE_TTL_DAYS)
let centsPerHour: WorkspaceConfig['centsPerHour']
try {
const parsed = CentsPerHourSchema.safeParse(env.COST_CENTS_PER_HOUR ? JSON.parse(env.COST_CENTS_PER_HOUR) : undefined)
centsPerHour = parsed.success ? parsed.data : undefined
} catch {
centsPerHour = undefined
}
return {
archiveTtlMs: Number.isFinite(days) && days >= 1 ? Math.floor(days) * 86_400_000 : undefined,
centsPerHour,
background: deps.waitUntil,
sleep: deps.sleep
}
}
/** The public origin: PUBLIC_API_URL when set, else the request's own origin. */
@@ -565,7 +610,10 @@ export async function handleRequest(request: Request, env: Env, deps: Deps = {})
}
return response
}
const match = /^\/v1\/sandboxes(?:\/([A-Za-z0-9_]+)(?:\/(suspend|resume|lease|rollover|env\/refresh))?)?$/.exec(url.pathname)
const match =
/^\/v1\/sandboxes(?:\/([A-Za-z0-9_]+)(?:\/(suspend|resume|lease|rollover|wake|restart|status|archive|unarchive|env\/refresh))?)?$/.exec(
url.pathname
)
if (!match) {
return fail('not_found', 404)
}
@@ -578,11 +626,13 @@ export async function handleRequest(request: Request, env: Env, deps: Deps = {})
return fail('not_configured', 503)
}
const [, id, action] = match
const archivedAware = (request.headers.get(FEATURES_HEADER) ?? '').split(',').map((f) => f.trim()).includes('archive')
const view = (row: SandboxRow) => sandboxView(row, archivedAware)
if (!id) {
if (request.method === 'GET') {
// Routine runs and other machine-made workspaces stay out of the desktop's list.
const rows = (await listAccountSandboxes(env.SANDBOX_DB, caller.accountId)).filter((row) => (row as { hidden?: number }).hidden !== 1)
return json({ sandboxes: rows.map(sandboxView) })
return json({ sandboxes: rows.map(view) })
}
if (request.method !== 'POST') {
return fail('not_found', 404)
@@ -603,11 +653,11 @@ export async function handleRequest(request: Request, env: Env, deps: Deps = {})
const route = `${request.method} ${action ?? ''}`
switch (route) {
case 'GET ':
return json({ sandbox: sandboxView(row) })
return json({ sandbox: view(row) })
case 'DELETE ':
return json({ sandbox: sandboxView(await destroySandbox(life, row)) })
return json({ sandbox: view(await destroySandbox(life, row)) })
case 'POST suspend':
return json({ sandbox: sandboxView(await suspendSandbox(life, row)) })
return json({ sandbox: view(await suspendSandbox(life, row)) })
case 'POST resume': {
const ready = await resumeSandbox(life, row)
return readyResponse(ready)
@@ -616,6 +666,29 @@ export async function handleRequest(request: Request, env: Env, deps: Deps = {})
const ready = await rolloverSandbox(life, row)
return readyResponse(ready)
}
case 'POST wake':
return json(await wakeSandbox(life, row, { waitMs: parseWakeWait(url, await readJson(request)), ...workspaceConfig(env, deps) }))
case 'POST restart': {
// User-triggered and unbounded otherwise: shares the per-account start rate limit.
await consumeCreateToken(life.db, life.config.guardrails!, caller.accountId, life.now())
const ready = await restartSandbox(life, row)
return json(readyView(ready.row, ready.connection))
}
case 'GET status':
return json({ sandbox: view(row), status: await workspaceStatus(life, row, workspaceConfig(env, deps)) })
case 'POST archive': {
const body = ArchiveRequestSchema.safeParse(await readJson(request, MAX_ARCHIVE_JSON_BYTES))
if (!body.success) {
return fail('invalid_request', 400)
}
return json({ sandbox: view(await archiveSandbox(life, row, body.data, workspaceConfig(env, deps))) })
}
case 'GET archive':
return json({ archive: await readArchive(env.SANDBOX_DB, row.id) })
case 'POST unarchive': {
const ready = await unarchiveSandbox(life, row)
return json(readyView(ready.row, ready.connection))
}
case 'POST env/refresh':
return json({ applied: await refreshSandboxEnv(life, row) })
case 'POST lease': {
@@ -623,7 +696,7 @@ export async function handleRequest(request: Request, env: Env, deps: Deps = {})
if (!body.success) {
return fail('invalid_request', 400)
}
return json({ sandbox: sandboxView(await renewLease(life, row, body.data.extendMs)) })
return json({ sandbox: view(await renewLease(life, row, body.data.extendMs)) })
}
default:
return fail('not_found', 404)
+55 -11
View File
@@ -201,7 +201,7 @@ export function sessionFields(deps: LifecycleDeps, state: ProviderSandboxState):
}
}
async function claim(deps: LifecycleDeps, row: SandboxRow, patch: Partial<SandboxRow>): Promise<SandboxRow> {
export async function claim(deps: LifecycleDeps, row: SandboxRow, patch: Partial<SandboxRow>): Promise<SandboxRow> {
const next = await transition(deps.db, row, patch, deps.now())
if (!next) {
throw new LifecycleError('conflict', 'concurrent_transition')
@@ -213,7 +213,7 @@ export function guardrails(deps: LifecycleDeps): GuardrailConfig {
return deps.config.guardrails ?? DEFAULT_GUARDRAILS
}
async function requireStart(deps: LifecycleDeps, accountId: string, userId: string, machineSize: MachineSize) {
export async function requireStart(deps: LifecycleDeps, accountId: string, userId: string, machineSize: MachineSize) {
const [, runningSandboxes, runningSandboxesForUser] = await Promise.all([
assertStartsOpen(deps.db, guardrails(deps)),
countActiveSandboxes(deps.db, accountId),
@@ -238,6 +238,11 @@ async function requireStart(deps: LifecycleDeps, accountId: string, userId: stri
})
}
function orcadEndpoint(state: ProviderSandboxState): string | null {
const route = state.routes.find((candidate) => candidate.port === ORCAD_PORT)
return route ? route.url.replace(/^https:/, 'wss:') : null
}
async function boot(
deps: LifecycleDeps,
row: SandboxRow,
@@ -281,6 +286,10 @@ async function boot(
const running = await claim(deps, row, {
status: 'running',
last_error: null,
orcad_endpoint: orcadEndpoint(state),
ready_at: deps.now(),
archived_at: null,
archive_expires_at: null,
retention_warned_at: null,
github_token_expires_at: git?.expiresAt ?? null,
egress_secret_hosts: routed.egress_secret_hosts,
@@ -483,9 +492,13 @@ export async function failAndRelease(deps: LifecycleDeps, row: SandboxRow): Prom
}
}
async function stopForSuspend(deps: LifecycleDeps, row: SandboxRow): Promise<SandboxRow> {
export async function stopForSuspend(
deps: LifecycleDeps,
row: SandboxRow,
options: { drainSeconds?: number } = {}
): Promise<SandboxRow> {
if (row.session_id) {
await drainOrcad(deps.provider, row.session_id)
await drainOrcad(deps.provider, row.session_id, options.drainSeconds)
const stopped = await deps.provider.stopSession(row.session_id)
if (stopped.snapshotId) {
await recordSnapshots(deps.db, row.id, [stopped.snapshotId], deps.now())
@@ -504,7 +517,8 @@ export async function markProviderStopped(deps: LifecycleDeps, row: SandboxRow):
}
export async function suspendSandbox(deps: LifecycleDeps, row: SandboxRow): Promise<SandboxRow> {
if (row.status === 'suspended') {
// Archived is already stopped (the desktop's Sleep runs after or before an archive).
if (row.status === 'suspended' || row.status === 'archived') {
return row
}
if (row.status !== 'running') {
@@ -521,10 +535,7 @@ export async function resumeSandbox(deps: LifecycleDeps, row: SandboxRow): Promi
const state = await deps.provider.getSandbox(row.provider_name, { resume: false })
if (state?.session.status === 'running') {
// Idempotent: a running, serving sandbox answers at once (desktop wake-on-open after a restart).
const connection =
(await runningOrcad(deps.provider, state.session.id)) ??
(await startOrcad(deps.provider, state, { sandboxId: row.id }))
return { row, connection }
return adoptRunning(deps, row, state)
}
// The provider ended the session (lease lapsed): fall through to a real resume.
row = await markProviderStopped(deps, row)
@@ -537,7 +548,37 @@ export async function resumeSandbox(deps: LifecycleDeps, row: SandboxRow): Promi
return wake(deps, resuming)
}
async function wake(deps: LifecycleDeps, row: SandboxRow): Promise<Ready> {
/**
* A running session whose orcad may not be up (a retried call, or a row the cron repaired after a
* wake died mid-way): start orcad (idempotent per boot) and record its endpoint if missing.
*/
export async function adoptRunning(deps: LifecycleDeps, row: SandboxRow, state: ProviderSandboxState): Promise<Ready> {
const connection =
(await runningOrcad(deps.provider, state.session.id)) ?? (await startOrcad(deps.provider, state, { sandboxId: row.id }))
if (row.orcad_endpoint && row.ready_at !== null) {
return { row, connection }
}
const updated = await transition(deps.db, row, { orcad_endpoint: orcadEndpoint(state), ready_at: deps.now() }, deps.now())
return { row: updated ?? row, connection }
}
/**
* The VM is running but the boot that should have followed never finished (its Worker was cut
* off, or the cron adopted the session): run the full boot — secrets, repo token, egress policy,
* orcad — against the live session, exactly as a wake would.
*/
export async function bootLiveSession(deps: LifecycleDeps, row: SandboxRow, state: ProviderSandboxState): Promise<Ready> {
const timer = phaseTimer(deps.now)
const prep = await timer.time('prepare', () => prepareBoot(deps, row))
return boot(deps, row, state, prep, {
policyApplied: false,
timer,
// Re-booting orcad in an already-metered session keeps its usage cursor (no re-billing).
billFrom: row.status === 'running' && row.usage_settled_at !== null ? row.usage_settled_at : undefined
})
}
export async function wake(deps: LifecycleDeps, row: SandboxRow): Promise<Ready> {
const timer = phaseTimer(deps.now)
// The provider boot is the long pole; secrets and the repo token resolve alongside it.
const [state, prep] = await Promise.all([
@@ -555,7 +596,8 @@ async function wake(deps: LifecycleDeps, row: SandboxRow): Promise<Ready> {
} catch (error) {
// Leave it resumable: the disk is intact, only this boot failed.
const current = await getSandbox(deps.db, row.id)
if (current?.status === 'resuming') {
// Only undo our own claim: a Restart that took the row over owns it now.
if (current?.status === 'resuming' && current.generation === row.generation) {
await transition(deps.db, current, { status: 'suspended', ...NO_SESSION, last_error: 'resume_failed' }, deps.now())
await deps.hooks?.afterStop?.(current).catch(() => undefined)
}
@@ -615,6 +657,8 @@ export async function destroySandbox(deps: LifecycleDeps, row: SandboxRow): Prom
throw new LifecycleError('provider_unavailable', 'snapshot_cleanup_incomplete')
}
await markSnapshotsDeleted(deps.db, row.id, deps.now())
// Archive metadata (repo state, file tree) goes with the workspace.
await deps.db.prepare('DELETE FROM sandbox_archives WHERE sandbox_id = ?').bind(row.id).run()
const destroyed = await claim(deps, destroying, {
status: 'destroyed',
...NO_SESSION,
@@ -16,6 +16,7 @@ import { killSwitchOn } from './guardrails'
import { drainAll, housekeeping, openHold, type HousekeepingSummary } from './retention'
import { listByStatus, transition, type SandboxRow } from './store'
import { flushUsage, queueUsage } from './usage'
import { expiredArchives } from './workspace-lifecycle'
const STUCK_MS = 15 * 60_000
/** The desktop rolls over at the deadline; the cron steps in only if it did not (provider cap is +10 min). */
@@ -29,6 +30,7 @@ export type ReconcileSummary = Record<
| 'leaseLapsed'
| 'released'
| 'repaired'
| 'archivesExpired'
| 'billingErrors',
number
> & { housekeeping?: HousekeepingSummary }
@@ -42,6 +44,7 @@ export async function reconcile(deps: LifecycleDeps): Promise<ReconcileSummary>
leaseLapsed: 0,
released: 0,
repaired: 0,
archivesExpired: 0,
billingErrors: 0
}
const polled = await guard(async () => {
@@ -81,6 +84,12 @@ export async function reconcile(deps: LifecycleDeps): Promise<ReconcileSummary>
}
})
}
for (const row of await expiredArchives(deps.db, deps.now())) {
await guard(async () => {
await destroySandbox(deps, row)
summary.archivesExpired++
})
}
await flushUsage(deps.db, deps.entitlements, deps.now())
await guard(async () => {
const housekept = await housekeeping(deps)
@@ -89,6 +89,8 @@ export type SandboxProviderApi = {
extendTimeout(sessionId: string, durationMs: number): Promise<ProviderSession>
updateNetworkPolicy(sessionId: string, policy: NetworkPolicy): Promise<void>
runCommand(sessionId: string, params: RunCommandParams): Promise<CommandResult>
/** Provider-side snapshot expiry after last use (archive TTL backstop). */
updateSnapshotExpiration(name: string, expirationMs: number): Promise<void>
/** A small text file from the VM (files API: not kept as command logs). Null when absent. */
readFile(sessionId: string, path: string): Promise<string | null>
/** Resolves when already gone. */
@@ -149,11 +149,13 @@ export async function refreshSessionEnv(
}
/** Best effort: orcad turns working chats into durable restart offers before the VM stops. */
export async function drainOrcad(provider: SandboxProviderApi, sessionId: string): Promise<boolean> {
export async function drainOrcad(provider: SandboxProviderApi, sessionId: string, waitSeconds?: number): Promise<boolean> {
try {
const result = await provider.runCommand(sessionId, {
command: DRAIN_COMMAND,
timeoutMs: DRAIN_TIMEOUT_MS
// The script's own default (280s) leaves room inside DRAIN_TIMEOUT_MS; Restart passes a short wait.
...(waitSeconds ? { args: [String(Math.max(1, Math.floor(waitSeconds)))] } : {}),
timeoutMs: waitSeconds ? (waitSeconds + 20) * 1000 : DRAIN_TIMEOUT_MS
})
return result.exitCode === 0
} catch (error) {
@@ -0,0 +1,170 @@
// Read-only probes the control plane runs in a VM: live metrics (status panel, OOM banner) and the
// git metadata an archived workspace shows. Fixed scripts compiled into the Worker (never user
// input), so they work on every golden image without a rebuild. Output is one JSON line, parsed strictly.
// Bash `\${...}` is escaped in these templates; `${CONST}` interpolates the TypeScript constants.
import { z } from 'zod'
import { ProviderError, type SandboxProviderApi } from './sandbox-provider'
const PROJECT = '/vercel/sandbox/project'
/**
* CPU from two /proc/stat samples 250 ms apart, memory from /proc/meminfo, disk for the project
* volume, uptime, and OOM: the kernel's oom_kill counter (/proc/vmstat, readable unprivileged) plus
* the last killed process name from dmesg when sudo allows it.
*/
export const METRICS_SCRIPT = String.raw`
set -u
cpu() { awk '/^cpu /{idle=$5+$6; t=0; for(i=2;i<=NF;i++) t+=$i; print t, idle}' /proc/stat; }
read t1 i1 < <(cpu); sleep 0.25; read t2 i2 < <(cpu)
dt=$((t2-t1)); di=$((i2-i1)); [ "$dt" -gt 0 ] || dt=1
cpu_pct=$(( (100*(dt-di)) / dt ))
mem_total=$(awk '/^MemTotal:/{print $2*1024}' /proc/meminfo)
mem_avail=$(awk '/^MemAvailable:/{print $2*1024}' /proc/meminfo)
read disk_total disk_used < <(df -B1 --output=size,used ${PROJECT} / 2>/dev/null | sed -n 2p)
uptime_s=$(awk '{printf "%d", $1}' /proc/uptime)
oom=$(awk '/^oom_kill /{print $2}' /proc/vmstat 2>/dev/null); oom=\${oom:-0}
victim=$(sudo -n dmesg 2>/dev/null | grep -oE 'Killed process [0-9]+ \([^)]{1,64}\)' | tail -n 1 | sed -E 's/.*\((.*)\)/\1/' | tr -cd 'A-Za-z0-9._+-' | head -c 64)
orcad=false; pid=$(cat "$HOME/.orca-sandbox/orcad.pid" 2>/dev/null || echo 0); [ "$pid" -gt 1 ] 2>/dev/null && kill -0 "$pid" 2>/dev/null && orcad=true
load=$(awk '{print $1}' /proc/loadavg)
printf '{"cpuPercent":%d,"cpus":%d,"load1":%s,"memTotalBytes":%d,"memUsedBytes":%d,"diskTotalBytes":%d,"diskUsedBytes":%d,"uptimeSeconds":%d,"oomKills":%d,"lastOomVictim":%s,"orcadAlive":%s}\n' \
"$cpu_pct" "$(nproc)" "$load" "$mem_total" "$((mem_total-mem_avail))" "\${disk_total:-0}" "\${disk_used:-0}" "$uptime_s" "$oom" \
"$( [ -n "$victim" ] && printf '"%s"' "$victim" || printf null )" "$orcad"
`.replaceAll('\\${', '${')
export const VmMetricsSchema = z
.object({
cpuPercent: z.number().int().min(0).max(100),
cpus: z.number().int().min(1),
load1: z.number().min(0),
memTotalBytes: z.number().int().min(0),
memUsedBytes: z.number().int().min(0),
diskTotalBytes: z.number().int().min(0),
diskUsedBytes: z.number().int().min(0),
uptimeSeconds: z.number().int().min(0),
oomKills: z.number().int().min(0),
lastOomVictim: z.string().max(64).nullable(),
orcadAlive: z.boolean()
})
.strict()
export type VmMetrics = z.infer<typeof VmMetricsSchema>
const MAX_TREE = 2000
const MAX_CHANGED = 300
const MAX_COMMITS = 20
/** Under the provider's 256 KiB command-output cap, with room for the stream framing. */
export const MAX_OUTPUT_BYTES = 200 * 1024
/**
* Archive metadata for every git checkout in the project (the repo plus its worktrees): branch,
* head, upstream ahead/behind, last commits, uncommitted changes and a capped file tree. Built
* with git's own porcelain output, JSON-escaped by node (always present in the image).
*/
export const ARCHIVE_SCRIPT = String.raw`
set -u
cd ${PROJECT} 2>/dev/null || { echo '{"checkouts":[]}'; exit 0; }
node="$(find /opt/orca/runtime/runtimes -type f -name node 2>/dev/null | head -n 1)"; node=\${node:-node}
git worktree list --porcelain 2>/dev/null | awk '/^worktree /{print substr($0,10)}' > /tmp/orca-archive-wt || true
[ -s /tmp/orca-archive-wt ] || echo ${PROJECT} > /tmp/orca-archive-wt
"$node" -e '
const { execFileSync } = require("child_process")
const fs = require("fs")
const run = (cwd, args) => { try { return execFileSync("git", args, { cwd, encoding: "utf8", maxBuffer: 8 << 20, timeout: 8000 }).trim() } catch { return "" } }
const lines = (s) => s ? s.split("\n") : []
const checkouts = fs.readFileSync("/tmp/orca-archive-wt", "utf8").split("\n").filter(Boolean).slice(0, 20).map((dir) => {
const counts = run(dir, ["rev-list", "--left-right", "--count", "@{upstream}...HEAD"]).split(/\s+/)
const tree = lines(run(dir, ["ls-files", "--cached", "--others", "--exclude-standard"]))
return {
path: dir,
branch: run(dir, ["rev-parse", "--abbrev-ref", "HEAD"]) || null,
head: run(dir, ["rev-parse", "HEAD"]) || null,
upstream: run(dir, ["rev-parse", "--abbrev-ref", "@{upstream}"]) || null,
behind: counts.length === 2 ? Number(counts[0]) : null,
ahead: counts.length === 2 ? Number(counts[1]) : null,
commits: lines(run(dir, ["log", "-n", "${MAX_COMMITS}", "--format=%H%x1f%an%x1f%at%x1f%s"])).map((l) => {
const [sha, author, at, subject] = l.split("\x1f")
return { sha, author: (author || "").slice(0, 120), at: Number(at) * 1000, subject: (subject || "").slice(0, 300) }
}),
changes: lines(run(dir, ["status", "--porcelain=v1", "-uall"])).slice(0, ${MAX_CHANGED}).map((l) => ({ status: l.slice(0, 2).trim() || "?", path: l.slice(3).slice(0, 1024) })),
diffStat: run(dir, ["diff", "HEAD", "--shortstat"]).slice(0, 200) || null,
fileCount: tree.length,
files: tree.slice(0, ${MAX_TREE}).map((f) => f.slice(0, 1024))
}
})
// The provider caps command output (256 KiB): keep the answer well under it. The main checkout keeps
// its file tree first; trees are trimmed, then dropped, until the JSON fits.
const kept = checkouts.filter((c) => c.head || c.branch)
const fits = () => Buffer.byteLength(JSON.stringify({ checkouts: kept })) <= ${MAX_OUTPUT_BYTES}
for (let i = kept.length - 1; i >= 0 && !fits(); i--) { kept[i].files = i === 0 ? kept[i].files.slice(0, 500) : [] }
for (let i = kept.length - 1; i >= 0 && !fits(); i--) { kept[i].files = []; kept[i].changes = kept[i].changes.slice(0, 50) }
while (kept.length > 1 && !fits()) kept.pop()
process.stdout.write(JSON.stringify({ checkouts: kept }) + "\n")
'
`.replaceAll('\\${', '${')
const Commit = z.object({ sha: z.string().max(64), author: z.string().max(120), at: z.number(), subject: z.string().max(300) })
const Checkout = z.object({
path: z.string().max(4096),
branch: z.string().max(255).nullable(),
head: z.string().max(64).nullable(),
upstream: z.string().max(255).nullable(),
ahead: z.number().int().nullable(),
behind: z.number().int().nullable(),
commits: z.array(Commit).max(MAX_COMMITS),
changes: z.array(z.object({ status: z.string().max(2), path: z.string().max(1024) })).max(MAX_CHANGED),
diffStat: z.string().max(200).nullable(),
fileCount: z.number().int().min(0),
files: z.array(z.string().max(1024)).max(MAX_TREE)
})
const PullRequest = z.object({
number: z.number().int().positive(),
title: z.string().max(300),
state: z.enum(['open', 'closed', 'merged', 'draft']),
url: z.url({ protocol: /^https$/ }).max(2048),
branch: z.string().max(255).optional()
})
export const ArchiveMetadataSchema = z.object({
checkouts: z.array(Checkout).max(20),
pullRequests: z.array(PullRequest).max(50).optional()
})
export type ArchiveMetadata = z.infer<typeof ArchiveMetadataSchema>
/** What the desktop may send at archive: PRs it knows, and git state from its cache if the VM is asleep. */
export const ArchiveRequestSchema = z
.object({
pullRequests: z.array(PullRequest).max(50).optional(),
checkouts: z.array(Checkout).max(20).optional()
})
.strict()
async function runJson<T>(
provider: SandboxProviderApi,
sessionId: string,
script: string,
schema: z.ZodType<T>,
timeoutMs: number
): Promise<T | null> {
try {
const result = await provider.runCommand(sessionId, { command: 'bash', args: ['-c', script], timeoutMs })
if (result.exitCode !== 0) {
return null
}
const parsed = schema.safeParse(JSON.parse(result.stdout.trim().split('\n').at(-1) ?? ''))
return parsed.success ? parsed.data : null
} catch (error) {
if (error instanceof ProviderError || error instanceof SyntaxError) {
return null
}
throw error
}
}
/** Null when the VM did not answer in time (the status panel shows "Unresponsive"). */
export function readVmMetrics(provider: SandboxProviderApi, sessionId: string): Promise<VmMetrics | null> {
return runJson(provider, sessionId, METRICS_SCRIPT, VmMetricsSchema, 8_000)
}
export function captureArchiveMetadata(provider: SandboxProviderApi, sessionId: string): Promise<ArchiveMetadata | null> {
return runJson(provider, sessionId, ARCHIVE_SCRIPT, ArchiveMetadataSchema, 30_000)
}
+22 -2
View File
@@ -11,6 +11,8 @@ export type SandboxStatus =
| 'destroying'
| 'destroyed'
| 'failed'
/** Stopped with the snapshot kept for a TTL; unarchive resumes it, the cron destroys it after the TTL. */
| 'archived'
export type SandboxRow = {
id: string
@@ -39,6 +41,12 @@ export type SandboxRow = {
/** Comma list of custom model-provider hosts added to this session's egress allowlist. */
egress_allow_hosts: string | null
retention_warned_at: number | null
/** orcad's public wss endpoint for the current session. */
orcad_endpoint: string | null
/** When orcad last became ready (wake callers report it as `readyAt`). */
ready_at: number | null
archived_at: number | null
archive_expires_at: number | null
}
export type SessionFields = Pick<
@@ -46,7 +54,8 @@ export type SessionFields = Pick<
'session_id' | 'session_started_at' | 'session_deadline_at' | 'lease_expires_at' | 'usage_settled_at'
>
export const NO_SESSION: SessionFields = {
export const NO_SESSION: SessionFields & Pick<SandboxRow, 'orcad_endpoint'> = {
orcad_endpoint: null,
session_id: null,
session_started_at: null,
session_deadline_at: null,
@@ -139,7 +148,8 @@ type Patch = Partial<Omit<SandboxRow, 'id' | 'account_id' | 'generation' | 'crea
const PATCHABLE = new Set<string>([
'status', 'session_id', 'session_started_at', 'session_deadline_at', 'lease_expires_at',
'usage_settled_at', 'last_error', 'destroyed_at', 'github_token_expires_at',
'egress_secret_hosts', 'egress_allow_hosts', 'repo_url', 'retention_warned_at'
'egress_secret_hosts', 'egress_allow_hosts', 'repo_url', 'retention_warned_at', 'orcad_endpoint', 'ready_at',
'archived_at', 'archive_expires_at'
])
/** Applies `patch` only if nobody moved the row since `expected` was read; null on a lost race. */
@@ -188,3 +198,13 @@ export async function recordedSnapshotIds(db: D1Database, sandboxId: string): Pr
.all<{ snapshot_id: string }>()
return results.map((row) => row.snapshot_id)
}
/** Usage so far for one sandbox: settled segments plus the open one (ms). */
export async function sandboxUsageMs(db: D1Database, row: SandboxRow, now: number): Promise<number> {
const settled = await db
.prepare('SELECT COALESCE(SUM(ended_at - started_at), 0) AS ms FROM usage_outbox WHERE sandbox_id = ?')
.bind(row.id)
.first<{ ms: number }>()
const open = row.status === 'running' && row.usage_settled_at !== null ? Math.max(0, now - row.usage_settled_at) : 0
return (settled?.ms ?? 0) + open
}
@@ -197,6 +197,14 @@ export function createVercelSandboxApi(
return readCommandStream(response)
},
async updateSnapshotExpiration(name, expirationMs) {
// Archive shortens the provider-side backstop; unarchive restores it. Same fields as create.
await request('PATCH', `/v2/sandboxes/${encodeURIComponent(name)}`, {
query: { projectId: config.projectId },
body: { snapshotExpiration: expirationMs, keepLastSnapshots: { count: 2, expiration: expirationMs } }
})
},
async readFile(sessionId, path) {
try {
const response = await request('POST', `/v2/sandboxes/sessions/${encodeURIComponent(sessionId)}/fs/read`, {
+18 -2
View File
@@ -38,7 +38,8 @@ export const LeaseRequestSchema = z
})
.strict()
export function sandboxView(row: SandboxRow) {
/** `archivedAware`: the caller's schema knows `archived`; older strict desktops see `suspended`. */
export function sandboxView(row: SandboxRow, archivedAware = false) {
const session =
row.session_id !== null &&
row.session_started_at !== null &&
@@ -54,7 +55,7 @@ export function sandboxView(row: SandboxRow) {
return {
id: row.id,
workspaceId: row.workspace_id,
status: row.status,
status: row.status === 'archived' && !archivedAware ? ('suspended' as const) : row.status,
machineSize: row.machine_size,
region: row.region,
session,
@@ -72,3 +73,18 @@ export function readyView(row: SandboxRow, connection: OrcadReady) {
}
}
}
/** `?wait=30s` / `?wait=30000ms` / `?wait=30`, or body `{wait: true}` (= the 30 s max). 0 when absent. */
export function parseWakeWait(url: URL, body: unknown): number {
const raw = url.searchParams.get('wait')
if (raw !== null) {
const match = /^(\d{1,6})(ms|s)?$/.exec(raw.trim())
if (!match) {
return 0
}
const value = Number(match[1])
return Math.min(30_000, match[2] === 'ms' ? value : value * 1000)
}
const wait = (body as { wait?: unknown } | null)?.wait
return wait === true ? 30_000 : typeof wait === 'number' && Number.isFinite(wait) ? Math.min(30_000, Math.max(0, wait)) : 0
}
@@ -0,0 +1,371 @@
// Workspace lifecycle on top of lifecycle.ts: wake (single-flight, long-poll; every wake source —
// open, message, SSH, preview — goes through it), Restart for unresponsive VMs, archive/unarchive
// with a snapshot TTL, and the status panel's live view (state, metrics, uptime, cost so far).
import type { MachineSize } from './entitlements'
import { GuardrailError, killSwitchOn } from './guardrails'
import {
DEFAULT_SNAPSHOT_EXPIRATION_MS,
LifecycleError,
bootLiveSession,
claim,
guardrails,
markProviderStopped,
requireStart,
resumeSandbox,
stopForSuspend,
wake,
type LifecycleDeps,
type Ready
} from './lifecycle'
import {
ArchiveMetadataSchema,
captureArchiveMetadata,
readVmMetrics,
type ArchiveMetadata,
type VmMetrics
} from './sandbox-vm-inspect'
import { NO_SESSION, getSandbox, sandboxUsageMs, transition, type SandboxRow } from './store'
import { flushUsage } from './usage'
const DAY_MS = 86_400_000
export const DEFAULT_ARCHIVE_TTL_MS = 30 * DAY_MS
/** Provider backstop past Orca's own archive TTL, so Orca's cron always deletes first. */
const ARCHIVE_PROVIDER_GRACE_MS = 7 * DAY_MS
export const MAX_WAKE_WAIT_MS = 30_000
const POLL_MS = 400
/** Restart gives a wedged orcad this long to write durable offers before the stop. */
const RESTART_DRAIN_SECONDS = 15
/** A resuming/suspending row older than this is treated as stuck by Restart (the cron's bar is 15 min). */
const RESTART_STUCK_MS = 5 * 60_000
/**
* A wake whose Worker died (waitUntil cut it off) is picked up by the next wake call after this.
* Longer than any live inline wake (the desktop's boot timeout is 180 s), so a healthy boot is never taken over.
*/
const WAKE_STALE_MS = 4 * 60_000
/** A wake that just failed answers `failed` for this long instead of booting again on every poll. */
const WAKE_FAILURE_HOLD_MS = 60_000
/** Archive gives orcad this long to write durable offers (the desktop already ran its Sleep). */
const ARCHIVE_DRAIN_SECONDS = 60
export const DEFAULT_CENTS_PER_HOUR: Record<MachineSize, number> = { standard: 75, large: 150 }
export type WorkspaceConfig = {
archiveTtlMs?: number
centsPerHour?: Partial<Record<MachineSize, number>>
/** Keeps a wake running after the response (Worker ctx.waitUntil); tests omit it. */
background?: (work: Promise<unknown>) => void
sleep?: (ms: number) => Promise<void>
}
export type WakeState = 'ready' | 'waking' | 'archived' | 'failed' | 'not_found'
export type WakeAnswer = { state: WakeState; endpoint: string | null; readyAt: number | null; sandboxId: string | null }
/** wake/status share one vocabulary with the desktop's host health states. */
export type WorkspaceState =
| 'provisioning'
| 'sleeping'
| 'waking'
| 'ready'
| 'unresponsive'
| 'stopping'
| 'archived'
| 'failed'
function wakeAnswer(row: SandboxRow | null): WakeAnswer {
if (!row || row.status === 'destroyed' || row.status === 'destroying') {
return { state: 'not_found', endpoint: null, readyAt: null, sandboxId: row?.id ?? null }
}
const state: WakeState =
row.status === 'running'
? 'ready'
: row.status === 'archived'
? 'archived'
: row.status === 'failed'
? 'failed'
: 'waking'
return {
state,
endpoint: state === 'ready' ? row.orcad_endpoint : null,
readyAt: state === 'ready' ? row.ready_at : null,
sandboxId: row.id
}
}
function recentlyFailed(deps: LifecycleDeps, row: SandboxRow): boolean {
return row.status === 'suspended' && row.last_error === 'resume_failed' && deps.now() - row.updated_at < WAKE_FAILURE_HOLD_MS
}
const defaultSleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms))
/** The row says running, but a lapsed lease means the provider may have stopped it on its own. */
async function confirmRunning(deps: LifecycleDeps, row: SandboxRow): Promise<SandboxRow> {
if (row.lease_expires_at === null || deps.now() < row.lease_expires_at) {
return row
}
const state = await deps.provider.getSandbox(row.provider_name, { resume: false })
return state?.session.status === 'running' ? row : markProviderStopped(deps, row)
}
/**
* Idempotent wake. A sleeping workspace starts waking (one wake however many callers race: the
* `resuming` claim is a compare-and-set); `waitMs` long-polls until ready or the wait runs out.
* Archived workspaces are not woken implicitly — unarchive is an explicit action.
*/
export async function wakeSandbox(
deps: LifecycleDeps,
row: SandboxRow,
options: { waitMs: number } & WorkspaceConfig
): Promise<WakeAnswer> {
const waitMs = Math.max(0, Math.min(options.waitMs, MAX_WAKE_WAIT_MS))
const sleep = options.sleep ?? defaultSleep
let current = row.status === 'running' ? await confirmRunning(deps, row) : row
let ours: Promise<unknown> | null = null
if (current.status === 'running' && !current.orcad_endpoint) {
// Repaired by the cron after a wake died: the VM runs, orcad may not.
const state = await deps.provider.getSandbox(current.provider_name, { resume: false })
current =
state?.session.status === 'running' ? (await bootLiveSession(deps, current, state)).row : await markProviderStopped(deps, current)
}
if (current.status === 'resuming' && deps.now() - current.updated_at >= WAKE_STALE_MS) {
current = await adoptStaleWake(deps, current)
}
if (recentlyFailed(deps, current)) {
return { state: 'failed', endpoint: null, readyAt: null, sandboxId: current.id }
}
if (current.status === 'suspended') {
await requireStart(deps, current.account_id, current.user_id, current.machine_size)
const resuming = await transition(deps.db, current, { status: 'resuming' }, deps.now())
if (resuming) {
ours = wake(deps, resuming).catch((error: unknown) => {
console.log(JSON.stringify({ event: 'wake_failed', sandboxId: row.id, error: String(error).slice(0, 120) }))
})
options.background?.(ours)
}
// Lost the race: someone else's wake is running; fall through and wait on the row.
current = (await getSandbox(deps.db, row.id)) ?? current
}
const deadline = deps.now() + waitMs
if (ours && waitMs > 0) {
await Promise.race([ours, sleep(waitMs)])
current = (await getSandbox(deps.db, row.id)) ?? current
}
while (current.status === 'resuming' && deps.now() < deadline) {
await sleep(Math.min(POLL_MS, Math.max(1, deadline - deps.now())))
current = (await getSandbox(deps.db, row.id)) ?? current
}
if (recentlyFailed(deps, current)) {
return { state: 'failed', endpoint: null, readyAt: null, sandboxId: current.id }
}
return wakeAnswer(current)
}
/**
* The Worker running a wake was cut off (waitUntil limit) and left the row `resuming`: adopt what
* the provider says — a running session gets orcad started and becomes `running`; otherwise the
* row goes back to `suspended` so this wake can start a fresh one.
*/
async function adoptStaleWake(deps: LifecycleDeps, row: SandboxRow): Promise<SandboxRow> {
const state = await deps.provider.getSandbox(row.provider_name, { resume: false })
if (state?.session.status !== 'running') {
return markProviderStopped(deps, row)
}
try {
return (await bootLiveSession(deps, row, state)).row
} catch (error) {
if (error instanceof LifecycleError && error.code === 'conflict') {
// Someone else moved the row meanwhile; report what they left.
return (await getSandbox(deps.db, row.id)) ?? row
}
// Leave it resumable (and reported as failed for a minute) rather than stuck in `resuming`.
const current = await getSandbox(deps.db, row.id)
if (current?.status === 'resuming' && current.generation === row.generation) {
await transition(deps.db, current, { status: 'suspended', ...NO_SESSION, last_error: 'resume_failed' }, deps.now())
}
throw error
}
}
/**
* "Restart" for an unresponsive workspace: one rehydrate path (short drain, stop with snapshot,
* boot the same disk). Works from running, sleeping, or a wake/sleep that got stuck mid-way.
*/
export async function restartSandbox(deps: LifecycleDeps, row: SandboxRow): Promise<Ready> {
if (await killSwitchOn(deps.db, guardrails(deps))) {
throw new GuardrailError('maintenance')
}
let current = row
if ((current.status === 'resuming' || current.status === 'suspending') && deps.now() - current.updated_at >= RESTART_STUCK_MS) {
const state = await deps.provider.getSandbox(current.provider_name, { resume: false })
if (state?.session.status === 'running') {
await deps.provider.stopSession(state.session.id).catch(() => undefined)
}
current = await markProviderStopped(deps, current)
}
if (current.status === 'running') {
// A lapsed lease means the provider already stopped it; resume instead of stopping a dead session.
current = await confirmRunning(deps, current)
}
if (current.status === 'suspended') {
return resumeSandbox(deps, current)
}
if (current.status !== 'running') {
throw new LifecycleError('conflict', `sandbox_${current.status}`)
}
const suspending = await claim(deps, current, { status: 'suspending' })
const stopped = await stopForSuspend(deps, suspending, { drainSeconds: RESTART_DRAIN_SECONDS })
await flushUsage(deps.db, deps.entitlements, deps.now(), row.id)
return wake(deps, await claim(deps, stopped, { status: 'resuming' }))
}
export async function readArchive(db: D1Database, sandboxId: string): Promise<{ capturedAt: number; source: string; metadata: ArchiveMetadata } | null> {
const stored = await db
.prepare('SELECT captured_at, source, metadata FROM sandbox_archives WHERE sandbox_id = ?')
.bind(sandboxId)
.first<{ captured_at: number; source: string; metadata: string }>()
if (!stored) {
return null
}
const parsed = ArchiveMetadataSchema.safeParse(JSON.parse(stored.metadata))
return parsed.success ? { capturedAt: stored.captured_at, source: stored.source, metadata: parsed.data } : null
}
async function writeArchive(db: D1Database, sandboxId: string, source: string, metadata: ArchiveMetadata, now: number) {
await db
.prepare(
`INSERT INTO sandbox_archives (sandbox_id, captured_at, source, metadata) VALUES (?, ?, ?, ?)
ON CONFLICT (sandbox_id) DO UPDATE SET captured_at = excluded.captured_at, source = excluded.source, metadata = excluded.metadata`
)
.bind(sandboxId, now, source, JSON.stringify(metadata))
.run()
}
export async function deleteArchive(db: D1Database, sandboxId: string): Promise<void> {
await db.prepare('DELETE FROM sandbox_archives WHERE sandbox_id = ?').bind(sandboxId).run()
}
/**
* Archive = stop and keep the snapshot for a TTL. Git metadata is captured in the VM when it is
* running; a sleeping VM is not woken just to archive it — the desktop's cached git state is used.
*/
export async function archiveSandbox(
deps: LifecycleDeps,
input: SandboxRow,
request: Partial<ArchiveMetadata>,
config: WorkspaceConfig = {}
): Promise<SandboxRow> {
let row = input
if (row.status === 'archived') {
return row
}
if (row.status !== 'running' && row.status !== 'suspended') {
throw new LifecycleError('conflict', `sandbox_${row.status}`)
}
const captured = row.status === 'running' && row.session_id ? await captureArchiveMetadata(deps.provider, row.session_id) : null
// The capture takes seconds; a lease renew may have moved the row meanwhile.
row = (await getSandbox(deps.db, row.id)) ?? row
if (row.status !== 'running' && row.status !== 'suspended') {
throw new LifecycleError('conflict', `sandbox_${row.status}`)
}
const previous = await readArchive(deps.db, row.id)
const checkouts = captured?.checkouts ?? request.checkouts ?? previous?.metadata.checkouts ?? []
const source = captured ? 'vm' : request.checkouts ? 'client' : previous ? previous.source : 'none'
await writeArchive(
deps.db,
row.id,
source,
{ checkouts, pullRequests: request.pullRequests ?? previous?.metadata.pullRequests ?? [] },
deps.now()
)
let stopped = row
if (row.status === 'running') {
// Short drain: the desktop's Sleep normally ran first; a long drain would outlive the caller.
stopped = await stopForSuspend(deps, await claim(deps, row, { status: 'suspending' }), { drainSeconds: ARCHIVE_DRAIN_SECONDS })
await flushUsage(deps.db, deps.entitlements, deps.now(), row.id)
}
const ttl = config.archiveTtlMs ?? DEFAULT_ARCHIVE_TTL_MS
await deps.provider
.updateSnapshotExpiration(stopped.provider_name, ttl + ARCHIVE_PROVIDER_GRACE_MS)
.catch((error: unknown) => console.log(JSON.stringify({ event: 'archive_ttl_update_failed', sandboxId: row.id, error: String(error).slice(0, 120) })))
return claim(deps, stopped, {
status: 'archived',
...NO_SESSION,
archived_at: deps.now(),
archive_expires_at: deps.now() + ttl
})
}
/** Unarchive restores the long-lived snapshot backstop and resumes the same disk. */
export async function unarchiveSandbox(deps: LifecycleDeps, row: SandboxRow): Promise<Ready> {
if (row.status !== 'archived') {
if (row.status === 'running' || row.status === 'suspended') {
return resumeSandbox(deps, row)
}
throw new LifecycleError('conflict', `sandbox_${row.status}`)
}
await requireStart(deps, row.account_id, row.user_id, row.machine_size)
await deps.provider.updateSnapshotExpiration(
row.provider_name,
deps.config.snapshotExpirationMs ?? DEFAULT_SNAPSHOT_EXPIRATION_MS
)
const suspended = await claim(deps, row, { status: 'suspended', archived_at: null, archive_expires_at: null })
return resumeSandbox(deps, suspended)
}
/** Cron: archives past their TTL are destroyed (snapshots included). */
export async function expiredArchives(db: D1Database, now: number): Promise<SandboxRow[]> {
const { results } = await db
.prepare(`SELECT * FROM sandboxes WHERE status = 'archived' AND archive_expires_at <= ? ORDER BY archive_expires_at LIMIT 50`)
.bind(now)
.all<SandboxRow>()
return results
}
export type WorkspaceStatus = {
state: WorkspaceState
metrics: VmMetrics | null
/** Since orcad last became ready (this session). */
uptimeMs: number | null
usage: { totalMs: number; centsPerHour: number; costCents: number }
archive: { archivedAt: number; expiresAt: number | null } | null
checkedAt: number
}
function stateOf(row: SandboxRow, metrics: VmMetrics | null, probed: boolean): WorkspaceState {
switch (row.status) {
case 'creating':
return 'provisioning'
case 'resuming':
return 'waking'
case 'suspending':
case 'destroying':
return 'stopping'
case 'suspended':
return 'sleeping'
case 'archived':
return 'archived'
case 'running':
return probed && (!metrics || !metrics.orcadAlive) ? 'unresponsive' : 'ready'
default:
return 'failed'
}
}
/** The status panel's one call. Metrics are read live from the VM only when it is running. */
export async function workspaceStatus(deps: LifecycleDeps, input: SandboxRow, config: WorkspaceConfig = {}): Promise<WorkspaceStatus> {
// A lapsed lease: the provider may have stopped it, which is "sleeping", not "unresponsive".
const row = input.status === 'running' ? await confirmRunning(deps, input) : input
const probed = row.status === 'running' && row.session_id !== null
const metrics = probed ? await readVmMetrics(deps.provider, row.session_id!) : null
const now = deps.now()
const totalMs = await sandboxUsageMs(deps.db, row, now)
const centsPerHour = config.centsPerHour?.[row.machine_size] ?? DEFAULT_CENTS_PER_HOUR[row.machine_size]
return {
state: stateOf(row, metrics, probed),
metrics,
uptimeMs: row.status === 'running' && row.ready_at !== null ? Math.max(0, now - row.ready_at) : null,
usage: { totalMs, centsPerHour, costCents: Math.round((totalMs / 3_600_000) * centsPerHour) },
archive: row.archived_at !== null ? { archivedAt: row.archived_at, expiresAt: row.archive_expires_at } : null,
checkedAt: now
}
}
export { getSandbox }
@@ -36,6 +36,10 @@ export type FakeVercel = SandboxProviderApi & {
failStart: boolean
failEnvRefresh: boolean
failCreate: boolean
/** Answers `bash -c <script>` probes (metrics, archive capture); undefined falls through to 127. */
onScript?: (script: string, sessionId: string) => { exitCode: number; stdout: string; stderr: string } | undefined
/** Provider snapshot expiry per sandbox name, as last set by create or update. */
snapshotExpirations: Map<string, number>
/** Simulates the provider ending a session at its timeout. */
expire(name: string): void
}
@@ -79,6 +83,7 @@ export function createFakeVercel(now: () => number): FakeVercel {
failStart: false,
failEnvRefresh: false,
failCreate: false,
snapshotExpirations: new Map(),
expire(name) {
const box = boxes.get(name)
if (box?.status === 'running') {
@@ -175,8 +180,20 @@ export function createFakeVercel(now: () => number): FakeVercel {
if (params.command === '/bin/sh' && params.args?.[1]?.includes('credentials.json')) {
return { exitCode: 0, stdout: '', stderr: '' }
}
if (params.command === 'bash' && params.args?.[0] === '-c' && fake.onScript) {
const answered = fake.onScript(params.args[1] ?? '', sessionId)
if (answered) {
return answered
}
}
return { exitCode: 127, stdout: '', stderr: 'not found' }
},
async updateSnapshotExpiration(name, expirationMs) {
if (!boxes.has(name)) {
throw new ProviderError(404, 'not_found')
}
fake.snapshotExpirations.set(name, expirationMs)
},
async readFile(sessionId, path) {
return bySession(sessionId).files.get(path) ?? null
},
@@ -0,0 +1,327 @@
import { describe, expect, it } from 'vitest'
import {
HostedArchiveResponseSchema,
HostedSandboxReadyResponseSchema,
HostedWakeResponseSchema,
HostedWorkspaceStatusResponseSchema
} from '../../../../src/shared/hosted-sandbox/hosted-sandbox-wire'
import { DRAIN_COMMAND, START_COMMAND } from '../src/sandbox-vm-commands'
import { ARCHIVE_SCRIPT, METRICS_SCRIPT } from '../src/sandbox-vm-inspect'
import { createHarness } from './harness'
const DAY = 86_400_000
const create = { workspaceId: 'ws-1', machineSize: 'standard' }
const metricsLine = (over: Record<string, unknown> = {}) =>
JSON.stringify({
cpuPercent: 12,
cpus: 4,
load1: 0.4,
memTotalBytes: 8e9,
memUsedBytes: 2e9,
diskTotalBytes: 64e9,
diskUsedBytes: 3e9,
uptimeSeconds: 120,
oomKills: 0,
lastOomVictim: null,
orcadAlive: true,
...over
})
const checkout = {
path: '/vercel/sandbox/project',
branch: 'feature/x',
head: 'a'.repeat(40),
upstream: 'origin/feature/x',
ahead: 2,
behind: 0,
commits: [{ sha: 'a'.repeat(40), author: 'Ada', at: 1_800_000_000_000, subject: 'Add x' }],
changes: [{ status: 'M', path: 'src/x.ts' }],
diffStat: '1 file changed, 2 insertions(+)',
fileCount: 2,
files: ['README.md', 'src/x.ts']
}
async function running() {
const h = await createHarness()
const created = HostedSandboxReadyResponseSchema.parse((await h.call('POST', '/v1/sandboxes', { body: create })).body)
return { h, id: created.sandbox.id }
}
describe('wake', () => {
it('answers ready with the endpoint for a running workspace, by sandbox or workspace id', async () => {
const { h, id } = await running()
const bySandbox = HostedWakeResponseSchema.parse((await h.call('POST', `/v1/sandboxes/${id}/wake`)).body)
expect(bySandbox).toMatchObject({ state: 'ready', sandboxId: id })
expect(bySandbox.endpoint).toMatch(/^wss:\/\/sb-\d+\.vercel\.run$/)
expect(bySandbox.readyAt).toBe(h.now())
// The public API route shares the wake and answers the same fields (plus `workspace`).
const viaApi = (await h.call('POST', `/v1/workspaces/${id}/wake?wait=30s`)).body
expect({ state: viaApi.state, endpoint: viaApi.endpoint, readyAt: viaApi.readyAt, sandboxId: viaApi.sandboxId }).toEqual(bySandbox)
})
it('wakes a sleeping workspace once however many callers race, and long-polls to ready', async () => {
const { h, id } = await running()
await h.call('POST', `/v1/sandboxes/${id}/suspend`)
const starts = () => h.vercel.commands.filter((c) => c.command === START_COMMAND).length
const before = starts()
const answers = await Promise.all([
h.call('POST', `/v1/sandboxes/${id}/wake?wait=30s`),
h.call('POST', `/v1/sandboxes/${id}/wake?wait=30s`),
h.call('POST', `/v1/sandboxes/${id}/wake`, { body: { wait: true } })
])
expect(answers.map((a) => a.body.state)).toEqual(['ready', 'ready', 'ready'])
expect(starts() - before).toBe(1)
})
it('returns waking immediately without a wait and finishes in the background', async () => {
const background: Promise<unknown>[] = []
const h = await createHarness({}, { waitUntil: (work) => void background.push(work) })
const { body } = await h.call('POST', '/v1/sandboxes', { body: create })
const id = body.sandbox.id
await h.call("POST", `/v1/sandboxes/${id}/suspend`)
const answer = HostedWakeResponseSchema.parse((await h.call('POST', `/v1/sandboxes/${id}/wake`)).body)
// The fake provider is synchronous enough that the wake may already be done; either is valid.
expect(['waking', 'ready']).toContain(answer.state)
expect(background).toHaveLength(1)
await Promise.all(background)
expect((await h.call('POST', `/v1/sandboxes/${id}/wake`)).body.state).toBe('ready')
})
it('does not wake an archived workspace implicitly, and 404s an unknown workspace', async () => {
const { h, id } = await running()
await h.call('POST', `/v1/sandboxes/${id}/archive`, { body: {} })
expect((await h.call('POST', `/v1/sandboxes/${id}/wake?wait=5s`)).body).toMatchObject({ state: 'archived', endpoint: null })
expect((await h.call('POST', '/v1/sandboxes/sbx_nope/wake')).status).toBe(404)
})
it('refuses a wake billing denies', async () => {
const { h, id } = await running()
await h.call('POST', `/v1/sandboxes/${id}/suspend`)
h.billing.decision = { allow: false, reason: 'included_hours_exhausted', suspendRunning: false }
expect(await h.call('POST', `/v1/sandboxes/${id}/wake?wait=1s`)).toEqual({
status: 403,
body: { error: 'entitlement_denied', reason: 'included_hours_exhausted' }
})
})
})
describe('restart', () => {
it('rehydrates a running workspace with a short drain and keeps the disk', async () => {
const { h, id } = await running()
const before = h.vercel.snapshots.size
const ready = HostedSandboxReadyResponseSchema.parse((await h.call('POST', `/v1/sandboxes/${id}/restart`)).body)
expect(ready.sandbox.status).toBe('running')
expect(h.vercel.snapshots.size).toBe(before + 1)
const drains = h.vercel.commands.filter((c) => c.command === DRAIN_COMMAND)
expect(drains).toHaveLength(1)
})
it('recovers a wake that got stuck mid-way', async () => {
const { h, id } = await running()
h.raw.prepare(`UPDATE sandboxes SET status = 'resuming', updated_at = ? WHERE id = ?`).run(h.now(), id)
h.advance(2 * 60_000)
// A slow-but-alive wake is not taken over.
expect((await h.call('POST', `/v1/sandboxes/${id}/restart`)).status).toBe(409)
h.advance(4 * 60_000)
const res = await h.call('POST', `/v1/sandboxes/${id}/restart`)
expect(res.status).toBe(200)
expect(res.body.sandbox.status).toBe('running')
})
})
describe('status panel', () => {
it('reports live metrics, uptime and cost so far', async () => {
const { h, id } = await running()
h.vercel.onScript = (script) => (script === METRICS_SCRIPT ? { exitCode: 0, stdout: `${metricsLine()}\n`, stderr: '' } : undefined)
h.advance(90 * 60_000)
const { status } = HostedWorkspaceStatusResponseSchema.parse((await h.call('GET', `/v1/sandboxes/${id}/status`)).body)
expect(status.state).toBe('ready')
expect(status.metrics?.memUsedBytes).toBe(2e9)
expect(status.uptimeMs).toBe(90 * 60_000)
expect(status.usage).toEqual({ totalMs: 90 * 60_000, centsPerHour: 75, costCents: 113 })
})
it('says unresponsive when the VM does not answer or orcad is dead, and surfaces OOM kills', async () => {
const { h, id } = await running()
h.vercel.onScript = () => ({ exitCode: 0, stdout: 'garbage', stderr: '' })
expect((await h.call('GET', `/v1/sandboxes/${id}/status`)).body.status.state).toBe('unresponsive')
h.vercel.onScript = () => ({ exitCode: 0, stdout: metricsLine({ orcadAlive: false, oomKills: 1, lastOomVictim: 'node' }), stderr: '' })
const { status } = HostedWorkspaceStatusResponseSchema.parse((await h.call('GET', `/v1/sandboxes/${id}/status`)).body)
expect(status.state).toBe('unresponsive')
expect(status.metrics).toMatchObject({ oomKills: 1, lastOomVictim: 'node' })
})
it('counts settled sessions in the cost and does not probe a sleeping VM', async () => {
const { h, id } = await running()
h.advance(60 * 60_000)
await h.call('POST', `/v1/sandboxes/${id}/suspend`)
let probes = 0
h.vercel.onScript = () => (probes++, undefined)
const { status } = HostedWorkspaceStatusResponseSchema.parse((await h.call('GET', `/v1/sandboxes/${id}/status`)).body)
expect(status).toMatchObject({ state: 'sleeping', metrics: null, uptimeMs: null, usage: { totalMs: 60 * 60_000, costCents: 75 } })
expect(probes).toBe(0)
})
})
describe('archive', () => {
it('captures metadata in the VM, stops it, keeps the snapshot with a TTL, and unarchives the same disk', async () => {
const { h, id } = await running()
h.vercel.onScript = (script) =>
script === ARCHIVE_SCRIPT ? { exitCode: 0, stdout: `${JSON.stringify({ checkouts: [checkout] })}\n`, stderr: '' } : undefined
const pr = { number: 7, title: 'Add x', state: 'open', url: 'https://github.com/o/r/pull/7', branch: 'feature/x' }
const archived = await h.call('POST', `/v1/sandboxes/${id}/archive`, { body: { pullRequests: [pr] } })
// Callers that don't advertise `archive` (older strict desktops) see it as suspended.
expect(archived.body.sandbox.status).toBe('suspended')
const aware = await h.send(
new Request(`https://sandbox.test/v1/sandboxes/${id}`, {
headers: { authorization: `Bearer ${await h.token('alice')}`, 'x-orca-sandbox-features': 'archive' }
})
)
expect(((await aware.json()) as any).sandbox.status).toBe('archived')
const name = `orca-test-${id}`
expect(h.vercel.boxes.get(name)!.status).toBe('stopped')
expect(h.vercel.snapshotExpirations.get(name)).toBe(37 * DAY)
const { archive } = HostedArchiveResponseSchema.parse((await h.call('GET', `/v1/sandboxes/${id}/archive`)).body)
expect(archive).toMatchObject({ source: 'vm', metadata: { checkouts: [checkout], pullRequests: [pr] } })
const status = HostedWorkspaceStatusResponseSchema.parse((await h.call('GET', `/v1/sandboxes/${id}/status`)).body).status
expect(status.archive).toEqual({ archivedAt: h.now(), expiresAt: h.now() + 30 * DAY })
const restored = HostedSandboxReadyResponseSchema.parse((await h.call('POST', `/v1/sandboxes/${id}/unarchive`)).body)
expect(restored.sandbox.status).toBe('running')
expect(h.vercel.snapshotExpirations.get(name)).toBe(104 * DAY)
expect((await h.call('GET', `/v1/sandboxes/${id}/status`)).body.status.archive).toBeNull()
})
it('archives a sleeping workspace from the desktop cache without waking it', async () => {
const { h, id } = await running()
await h.call('POST', `/v1/sandboxes/${id}/suspend`)
const starts = h.vercel.commands.filter((c) => c.command === START_COMMAND).length
await h.call('POST', `/v1/sandboxes/${id}/archive`, { body: { checkouts: [checkout] } })
expect(h.vercel.commands.filter((c) => c.command === START_COMMAND)).toHaveLength(starts)
const { archive } = HostedArchiveResponseSchema.parse((await h.call('GET', `/v1/sandboxes/${id}/archive`)).body)
expect(archive?.source).toBe('client')
})
it('rejects oversized archive bodies and destroys expired archives with their metadata', async () => {
const { h, id } = await running()
const tooMany = { checkouts: [{ ...checkout, files: Array.from({ length: 2001 }, (_, i) => `f${i}`) }] }
expect((await h.call('POST', `/v1/sandboxes/${id}/archive`, { body: tooMany })).status).toBe(400)
await h.call('POST', `/v1/sandboxes/${id}/archive`, { body: { checkouts: [checkout] } })
h.advance(29 * DAY)
expect((await h.reconcile()).archivesExpired).toBe(0)
h.advance(2 * DAY)
expect((await h.reconcile()).archivesExpired).toBe(1)
expect(h.raw.prepare('SELECT status FROM sandboxes WHERE id = ?').get(id)).toEqual({ status: 'destroyed' })
expect(h.raw.prepare('SELECT COUNT(*) AS n FROM sandbox_archives').get()).toEqual({ n: 0 })
expect([...h.vercel.snapshots.values()].filter((s) => s.status !== 'deleted')).toHaveLength(0)
})
})
describe('desktop client against the real handler', () => {
it('drives wake, status, restart, archive and unarchive through the desktop client', async () => {
const { createHostedSandboxControlPlaneClient } = await import(
'../../../../src/main/hosted-sandbox/hosted-sandbox-control-plane-client'
)
const h = await createHarness()
const client = createHostedSandboxControlPlaneClient({
baseUrl: 'https://sandbox.test',
getAccessToken: () => h.token('alice'),
fetch: (input, init) => h.send(new Request(input as URL, init))
})
const created = await client.create({ workspaceId: 'ws-c', machineSize: 'standard' })
const id = created.sandbox.id
h.vercel.onScript = (script) =>
script === METRICS_SCRIPT
? { exitCode: 0, stdout: metricsLine(), stderr: '' }
: script === ARCHIVE_SCRIPT
? { exitCode: 0, stdout: JSON.stringify({ checkouts: [checkout] }), stderr: '' }
: undefined
expect((await client.wake(id, 30_000)).state).toBe('ready')
expect((await client.workspaceStatus(id)).status.state).toBe('ready')
expect((await client.restart(id)).sandbox.status).toBe('running')
expect((await client.archive(id, {})).status).toBe('archived')
expect((await client.archiveInfo(id))?.metadata.checkouts[0]?.branch).toBe('feature/x')
expect((await client.wake(id, 0)).state).toBe('archived')
expect((await client.unarchive(id)).sandbox.status).toBe('running')
})
})
describe('wake recovery', () => {
it('adopts a wake whose Worker was cut off: running VM gets orcad and an endpoint', async () => {
const { h, id } = await running()
await h.call('POST', `/v1/sandboxes/${id}/suspend`)
// Simulate a wake that booted the VM, then lost its Worker before orcad/claim.
await h.vercel.getSandbox(`orca-test-${id}`, { resume: true })
h.raw.prepare(`UPDATE sandboxes SET status = 'resuming', updated_at = ? WHERE id = ?`).run(h.now(), id)
expect((await h.call('POST', `/v1/sandboxes/${id}/wake`)).body.state).toBe('waking')
// A live inline boot can take minutes: not adopted yet.
h.advance(3 * 60_000)
expect((await h.call('POST', `/v1/sandboxes/${id}/wake`)).body.state).toBe('waking')
h.advance(61_000)
const answer = HostedWakeResponseSchema.parse((await h.call('POST', `/v1/sandboxes/${id}/wake`)).body)
expect(answer.state).toBe('ready')
expect(answer.endpoint).toMatch(/^wss:/)
})
it('a stale wake whose VM never came up goes back to sleep and wakes again', async () => {
const { h, id } = await running()
await h.call('POST', `/v1/sandboxes/${id}/suspend`)
h.raw.prepare(`UPDATE sandboxes SET status = 'resuming', updated_at = ? WHERE id = ?`).run(h.now(), id)
h.advance(4 * 60_000 + 1000)
expect((await h.call('POST', `/v1/sandboxes/${id}/wake?wait=30s`)).body.state).toBe('ready')
})
it('answers failed after a failed wake instead of booting on every poll', async () => {
const { h, id } = await running()
await h.call('POST', `/v1/sandboxes/${id}/suspend`)
h.vercel.failStart = true
expect((await h.call('POST', `/v1/sandboxes/${id}/wake?wait=30s`)).body.state).toBe('failed')
const starts = h.vercel.commands.filter((c) => c.command === START_COMMAND).length
expect((await h.call('POST', `/v1/sandboxes/${id}/wake?wait=30s`)).body.state).toBe('failed')
expect(h.vercel.commands.filter((c) => c.command === START_COMMAND)).toHaveLength(starts)
h.vercel.failStart = false
h.advance(61_000)
expect((await h.call('POST', `/v1/sandboxes/${id}/wake?wait=30s`)).body.state).toBe('ready')
})
it('reports a provider-stopped VM as sleeping, not unresponsive', async () => {
const { h, id } = await running()
h.vercel.expire(`orca-test-${id}`)
h.advance(31 * 60_000)
expect((await h.call('GET', `/v1/sandboxes/${id}/status`)).body.status.state).toBe('sleeping')
})
})
describe('adopted wakes boot like real wakes', () => {
it('starts orcad with the session secrets and the egress policy, and keeps metering intact', async () => {
const h = await createHarness({ SECRETS_KEK: Buffer.alloc(32, 7).toString('base64') })
expect((await h.call('PUT', '/v1/secrets/personal/OPENAI_API_KEY', { body: { value: 'sk-adopted-test' } })).status).toBe(200)
const created = HostedSandboxReadyResponseSchema.parse((await h.call('POST', '/v1/sandboxes', { body: create })).body)
const id = created.sandbox.id
await h.call('POST', `/v1/sandboxes/${id}/suspend`)
await h.vercel.getSandbox(`orca-test-${id}`, { resume: true })
h.raw.prepare(`UPDATE sandboxes SET status = 'resuming', updated_at = ? WHERE id = ?`).run(h.now(), id)
const before = h.vercel.commands.length
h.advance(4 * 60_000 + 1000)
expect((await h.call('POST', `/v1/sandboxes/${id}/wake`)).body.state).toBe('ready')
const start = h.vercel.commands.slice(before).find((c) => c.command === START_COMMAND)!
expect(start.env.ORCA_SECRET_ENV).toContain('OPENAI_API_KEY=')
expect(start.env.ORCA_PUBLIC_WSS).toMatch(/^wss:/)
const row = h.raw.prepare('SELECT status, session_id, orcad_endpoint FROM sandboxes WHERE id = ?').get(id) as any
expect(row.status).toBe('running')
expect(row.orcad_endpoint).toMatch(/^wss:/)
})
it('a cron-repaired running row without orcad gets a full boot and keeps its usage cursor', async () => {
const { h, id } = await running()
h.advance(10 * 60_000)
const cursor = (h.raw.prepare('SELECT usage_settled_at FROM sandboxes WHERE id = ?').get(id) as any).usage_settled_at
h.raw.prepare('UPDATE sandboxes SET orcad_endpoint = NULL WHERE id = ?').run(id)
const before = h.vercel.commands.filter((c) => c.command === START_COMMAND).length
expect((await h.call('POST', `/v1/sandboxes/${id}/wake`)).body.state).toBe('ready')
expect(h.vercel.commands.filter((c) => c.command === START_COMMAND)).toHaveLength(before + 1)
expect((h.raw.prepare('SELECT usage_settled_at FROM sandboxes WHERE id = ?').get(id) as any).usage_settled_at).toBe(cursor)
})
})
@@ -3,6 +3,13 @@
import type { z } from 'zod'
import { HostedEntitlementDeniedError } from './hosted-environment-provider'
import {
HostedArchiveResponseSchema,
HostedWakeResponseSchema,
HostedWorkspaceStatusResponseSchema,
type HostedArchiveRequest,
type HostedArchiveResponse,
type HostedWakeResponse,
type HostedWorkspaceStatus,
HostedSandboxErrorResponseSchema,
HostedSandboxListResponseSchema,
HostedSandboxReadyResponseSchema,
@@ -28,6 +35,22 @@ export type HostedSandboxControlPlaneClient = {
renewLease(sandboxId: string, extendMs: number, signal?: AbortSignal): Promise<HostedSandboxView>
rollover(sandboxId: string, signal?: AbortSignal): Promise<HostedSandboxReadyResponse>
destroy(sandboxId: string, signal?: AbortSignal): Promise<HostedSandboxView>
/** Idempotent wake; `waitMs` (max 30s) long-polls until orcad is ready. */
wake(sandboxId: string, waitMs: number, signal?: AbortSignal): Promise<HostedWakeResponse>
/** Status panel: state, live VM metrics, uptime, cost so far. */
workspaceStatus(
sandboxId: string,
signal?: AbortSignal
): Promise<{ sandbox: HostedSandboxView; status: HostedWorkspaceStatus }>
/** Forced rehydrate for an unresponsive workspace (same disk, new session). */
restart(sandboxId: string, signal?: AbortSignal): Promise<HostedSandboxReadyResponse>
archive(
sandboxId: string,
body: HostedArchiveRequest,
signal?: AbortSignal
): Promise<HostedSandboxView>
archiveInfo(sandboxId: string, signal?: AbortSignal): Promise<HostedArchiveResponse['archive']>
unarchive(sandboxId: string, signal?: AbortSignal): Promise<HostedSandboxReadyResponse>
}
export type HostedSandboxControlPlaneClientOptions = {
@@ -65,6 +88,8 @@ export function createHostedSandboxControlPlaneClient(
headers: {
authorization: `Bearer ${token}`,
accept: 'application/json',
// This client's schemas know the `archived` status.
'x-orca-sandbox-features': 'archive',
...(init.body === undefined ? {} : { 'content-type': 'application/json' })
},
body: init.body === undefined ? undefined : JSON.stringify(init.body),
@@ -144,7 +169,44 @@ export function createHostedSandboxControlPlaneClient(
kind: 'control',
signal
})
).sandbox
).sandbox,
wake: (id, waitMs, signal) =>
call(
'POST',
`${sandboxPath(id, '/wake')}?wait=${Math.max(0, Math.min(30_000, Math.floor(waitMs)))}ms`,
HostedWakeResponseSchema,
{ kind: 'control', signal }
),
workspaceStatus: (id, signal) =>
call('GET', sandboxPath(id, '/status'), HostedWorkspaceStatusResponseSchema, {
kind: 'control',
signal
}),
restart: (id, signal) =>
call('POST', sandboxPath(id, '/restart'), HostedSandboxReadyResponseSchema, {
kind: 'boot',
signal
}),
archive: async (id, body, signal) =>
(
await call('POST', sandboxPath(id, '/archive'), HostedSandboxStatusResponseSchema, {
body,
kind: 'boot',
signal
})
).sandbox,
archiveInfo: async (id, signal) =>
(
await call('GET', sandboxPath(id, '/archive'), HostedArchiveResponseSchema, {
kind: 'control',
signal
})
).archive,
unarchive: (id, signal) =>
call('POST', sandboxPath(id, '/unarchive'), HostedSandboxReadyResponseSchema, {
kind: 'boot',
signal
})
}
}
@@ -41,7 +41,9 @@ const STATUS: Record<HostedSandboxView['status'], EnvironmentStatus> = {
resuming: 'resuming',
destroying: 'destroying',
destroyed: 'destroyed',
failed: 'failed'
failed: 'failed',
// Archived = asleep with a snapshot TTL; the desktop shows it as a sleeping environment.
archived: 'suspended'
}
export function createHostedSandboxDriver(
@@ -141,7 +143,7 @@ function toSnapshot(view: HostedSandboxView): HostedStatusSnapshot {
return { status: STATUS[view.status], session: toSession(view) }
}
function toReady(ready: HostedSandboxReadyResponse): HostedEnvironmentReady {
export function toReady(ready: HostedSandboxReadyResponse): HostedEnvironmentReady {
const session = toSession(ready.sandbox)
if (ready.sandbox.status !== 'running' || !session) {
throw new EnvironmentProviderError(
@@ -10,9 +10,14 @@ import type { PluginService } from '../plugins/plugin-service'
import { getHostedSandboxConfig } from './hosted-sandbox-config'
import {
ORCA_CLOUD_RECIPE,
createHostedSandboxRecipeClient,
createHostedSandboxRecipeDriver,
registerHostedSandboxRecipe
} from './hosted-sandbox-recipe'
import {
installHostedWorkspaceService,
setHostedWorkspaceService
} from './hosted-workspace-service'
import { createHostedSessionRegistry, type HostedSessionRegistry } from './hosted-session-registry'
import { setHostedSessionSync } from './hosted-session-sync'
@@ -34,7 +39,8 @@ export function installHostedSandbox(): void {
if (!config) {
return
}
const base = createHostedSandboxRecipeDriver(config)
const client = createHostedSandboxRecipeClient(config)
const base = createHostedSandboxRecipeDriver(config, client)
// Dev rehearsals shorten idle sleep; packaged builds always use the provider's default.
const idleOverride = app.isPackaged
? Number.NaN
@@ -47,17 +53,37 @@ export function installHostedSandbox(): void {
driver,
windows: () => BrowserWindow.getAllWindows()
})
const workspaces = installHostedWorkspaceService({
userDataPath,
client,
windows: () => BrowserWindow.getAllWindows(),
syncSessions: () => void registry.sync(),
stopSession: (environmentId) => registry.stopEnvironment(environmentId)
})
setHostedWorkspaceService(workspaces)
// Records land after the step answers and wakes can come from any path, so reconcile on a slow beat too.
let beat: ReturnType<typeof setTimeout> | null = null
let beats = 0
const schedule = (ms: number): void => {
beat = setTimeout(() => {
void registry.sync().finally(() => schedule(30_000))
// OOM in the guest is checked every other beat (~1 min) for running cloud workspaces.
const oom = beats++ % 2 === 1 ? workspaces.pollOom() : Promise.resolve()
void Promise.all([registry.sync(), oom]).finally(() => schedule(30_000))
}, ms)
beat.unref?.()
}
registerHostedSandboxRecipe(driver, {
onReady: () => setTimeout(() => void registry.sync(), 3_000).unref?.(),
onStopped: (environmentId) => registry.stopEnvironment(environmentId)
onStopped: (environmentId) => registry.stopEnvironment(environmentId),
onWake: ({ workspaceId, phase, startedAt, message }) =>
workspaces.emitWakeProgress({
workspaceId,
phase,
startedAt,
message,
// Later phases inherit the source recorded at `waking`.
source: phase === 'waking' ? workspaces.takeWakeSource(workspaceId) : 'open'
})
})
installed = { registry }
setHostedSessionSync(() => void registry.sync())
@@ -67,6 +93,8 @@ export function installHostedSandbox(): void {
clearTimeout(beat)
}
registry.dispose()
workspaces.dispose()
setHostedWorkspaceService(null)
})
}
@@ -39,15 +39,18 @@ export const ORCA_CLOUD_RECIPE: OrcaVmRecipe = {
const LifecyclePayload = z.looseObject({ recipeResult: EphemeralVmRecipeResultSchema.optional() })
export function createHostedSandboxRecipeClient(config: HostedSandboxConfig) {
return createHostedSandboxControlPlaneClient({
baseUrl: config.controlPlaneUrl,
getAccessToken: config.getAccessToken
})
}
export function createHostedSandboxRecipeDriver(
config: HostedSandboxConfig
config: HostedSandboxConfig,
client = createHostedSandboxRecipeClient(config)
): HostedEnvironmentProviderDriver {
return createHostedSandboxDriver(
createHostedSandboxControlPlaneClient({
baseUrl: config.controlPlaneUrl,
getAccessToken: config.getAccessToken
})
)
return createHostedSandboxDriver(client)
}
/** https clone URL the control plane can mint a GitHub App token for; null for non-GitHub remotes. */
@@ -101,6 +104,13 @@ export function createHostedSandboxStepProvider(
hooks: {
onReady?: (ready: HostedEnvironmentReady) => void
onStopped?: (environmentId: string) => void
/** Wake progress for the window's "Waking…" overlay (resume only). */
onWake?: (event: {
workspaceId: string
phase: 'waking' | 'connecting' | 'ready' | 'failed'
startedAt: number
message?: string
}) => void
} = {}
): RecipeStepProvider {
return async ({ mode, repoPath, context, stdin, signal, onStderr }) => {
@@ -133,9 +143,29 @@ export function createHostedSandboxStepProvider(
return { stdout: '', stderr: 'Not an Orca Cloud environment.\n', exitCode: 1, signal: null }
}
if (mode === 'resume') {
const ready = await driver.resume(handle, ctx)
hooks.onReady?.(ready)
return ok(JSON.stringify(ready.result))
const workspaceId = context.workspaceId
const startedAt = Date.now()
if (workspaceId) {
hooks.onWake?.({ workspaceId, phase: 'waking', startedAt })
}
try {
const ready = await driver.resume(handle, ctx)
if (workspaceId) {
hooks.onWake?.({ workspaceId, phase: 'connecting', startedAt })
}
hooks.onReady?.(ready)
return ok(JSON.stringify(ready.result))
} catch (error) {
if (workspaceId) {
hooks.onWake?.({
workspaceId,
phase: 'failed',
startedAt,
message: error instanceof Error ? error.message : String(error)
})
}
throw error
}
}
await (mode === 'suspend' ? driver.suspend(handle, ctx) : driver.destroy(handle, ctx))
hooks.onStopped?.(handle.environmentId)
@@ -0,0 +1,84 @@
import { describe, expect, it } from 'vitest'
import type { CloudOomEvent } from '../../shared/hosted-sandbox/cloud-workspace-api'
import type { HostedWorkspaceStatus } from '../../shared/hosted-sandbox/hosted-sandbox-wire'
import { createOomTracker } from './hosted-workspace-runtime'
function status(checkedAt: number, uptimeSeconds: number, oomKills: number): HostedWorkspaceStatus {
return {
state: 'ready',
metrics: {
cpuPercent: 1,
cpus: 4,
load1: 0,
memTotalBytes: 8,
memUsedBytes: 1,
diskTotalBytes: 8,
diskUsedBytes: 1,
uptimeSeconds,
oomKills,
lastOomVictim: oomKills ? 'node' : null,
orcadAlive: true
},
uptimeMs: 0,
usage: { totalMs: 0, centsPerHour: 75, costCents: 0 },
archive: null,
checkedAt
}
}
describe('OOM tracker', () => {
it('announces each new kill once per boot, despite clock jitter', () => {
const events: CloudOomEvent[] = []
const tracker = createOomTracker((event) => events.push(event))
const t0 = 1_000_000_000
tracker.observe('ws', 'sbx', status(t0, 100, 1))
// Same boot, readings straddle a minute boundary: no repeat.
tracker.observe('ws', 'sbx', status(t0 + 60_400, 160, 1))
tracker.observe('ws', 'sbx', status(t0 + 120_000, 220, 2))
expect(events.map((e) => e.oomKills)).toEqual([1, 2])
expect(events[0]?.victim).toBe('node')
})
it('treats a new boot as a fresh counter and forgets stopped sandboxes', () => {
const events: CloudOomEvent[] = []
const tracker = createOomTracker((event) => events.push(event))
tracker.observe('ws', 'sbx', status(1_000_000_000, 100, 1))
// Restarted: uptime is small again, counter restarts at 1.
tracker.observe('ws', 'sbx', status(1_000_600_000, 20, 1))
expect(events).toHaveLength(2)
tracker.prune(new Set())
tracker.observe('ws', 'sbx', status(1_000_660_000, 80, 1))
expect(events).toHaveLength(3)
})
it('ignores a status without metrics', () => {
const events: CloudOomEvent[] = []
createOomTracker((event) => events.push(event)).observe('ws', 'sbx', {
...status(1, 1, 0),
metrics: null
})
expect(events).toEqual([])
})
})
describe('archive checkouts trim', () => {
it('keeps cached repo state under the archive body limit', async () => {
const { trimCheckouts } = await import('./hosted-workspace-runtime')
const big = Array.from({ length: 20 }, (_, i) => ({
path: `/w/${i}`,
branch: 'b',
head: 'h',
upstream: null,
ahead: null,
behind: null,
commits: [],
changes: [],
diffStat: null,
fileCount: 2000,
files: Array.from({ length: 2000 }, (_, f) => `src/some/long/path/file-${f}.ts`)
}))
const trimmed = trimCheckouts(big)
expect(JSON.stringify(trimmed).length).toBeLessThanOrEqual(200 * 1024)
expect(trimmed[0]!.files.length).toBeGreaterThan(0)
})
})
@@ -0,0 +1,165 @@
// Orca Cloud workspace lookups shared by the workspace service and its IPC: workspace id → runtime
// record → sandbox id, error mapping for the window, and OOM-notice dedupe.
import { listEphemeralVmRuntimes } from '../../shared/ephemeral-vm-runtime-store'
import type { EphemeralVmRuntimeRecord } from '../../shared/ephemeral-vm-runtimes'
import type {
CloudOomEvent,
CloudWorkspaceErrorCode,
CloudWorkspaceResult
} from '../../shared/hosted-sandbox/cloud-workspace-api'
import type {
HostedArchiveCheckout,
HostedWorkspaceStatus
} from '../../shared/hosted-sandbox/hosted-sandbox-wire'
import {
isEnvironmentProviderError,
type EnvironmentProviderErrorCode
} from '../environment-providers/environment-provider-driver'
import { environmentHandleFromRecipeResult } from '../environment-providers/environment-provider-recipe-adapter'
import { isHostedEntitlementDenied } from './hosted-environment-provider'
import { HOSTED_SANDBOX_DRIVER_ID } from './hosted-sandbox-driver'
import type { HostedWorkspaceService } from './hosted-workspace-service'
export type CloudRuntime = { runtime: EphemeralVmRuntimeRecord; sandboxId: string }
let cachedCheckouts: ((workspaceId: string) => Promise<HostedArchiveCheckout[] | null>) | null =
null
export function setArchiveCheckoutsProvider(
provider: ((workspaceId: string) => Promise<HostedArchiveCheckout[] | null>) | null
): void {
cachedCheckouts = provider
}
export function findCloudRuntime(userDataPath: string, workspaceId: string): CloudRuntime | null {
for (const runtime of listEphemeralVmRuntimes(userDataPath)) {
if (runtime.recipe?.provider !== 'orca-cloud' || runtime.workspaceId !== workspaceId) {
continue
}
const handle = runtime.recipeResult
? environmentHandleFromRecipeResult(runtime.recipeResult, HOSTED_SANDBOX_DRIVER_ID)
: null
const sandboxId = handle?.driverState.sandboxId
if (typeof sandboxId === 'string' && sandboxId) {
return { runtime, sandboxId }
}
}
return null
}
export function failure<T>(error: unknown): CloudWorkspaceResult<T> {
if (isHostedEntitlementDenied(error)) {
return {
ok: false,
code: 'entitlement_denied',
message: error.message,
reason: error.reason ?? null
}
}
const codes: (CloudWorkspaceErrorCode & EnvironmentProviderErrorCode)[] = [
'unauthorized',
'conflict',
'not_found'
]
for (const code of codes) {
if (isEnvironmentProviderError(error, code)) {
return { ok: false, code, message: error.message, reason: error.reason ?? null }
}
}
if (
isEnvironmentProviderError(error, 'unavailable') ||
isEnvironmentProviderError(error, 'timeout')
) {
return { ok: false, code: 'unavailable', message: error.message }
}
return {
ok: false,
code: 'failed',
message: error instanceof Error ? error.message : String(error)
}
}
export const NOT_CLOUD = {
ok: false,
code: 'not_cloud',
message: 'Not an Orca Cloud workspace.'
} as const
/** Cached repo state for an archive body, trimmed under the control plane's 256 KiB archive limit. */
export async function readCachedCheckouts(
workspaceId: string
): Promise<HostedArchiveCheckout[] | null> {
const checkouts = cachedCheckouts ? await cachedCheckouts(workspaceId).catch(() => null) : null
return checkouts ? trimCheckouts(checkouts) : null
}
const MAX_ARCHIVE_BODY_BYTES = 200 * 1024
export function trimCheckouts(input: HostedArchiveCheckout[]): HostedArchiveCheckout[] {
const kept = input.slice(0, 20).map((c) => ({ ...c, files: c.files.slice(0, 2000) }))
const fits = (): boolean => JSON.stringify(kept).length <= MAX_ARCHIVE_BODY_BYTES
for (let i = kept.length - 1; i >= 0 && !fits(); i--) {
kept[i] = { ...kept[i]!, files: i === 0 ? kept[i]!.files.slice(0, 500) : [] }
}
for (let i = kept.length - 1; i >= 0 && !fits(); i--) {
kept[i] = { ...kept[i]!, files: [], changes: kept[i]!.changes.slice(0, 50) }
}
while (kept.length > 1 && !fits()) {
kept.pop()
}
return kept
}
/**
* Announces each new kernel OOM kill once. The counter resets per VM boot; two readings belong to
* one boot when their implied boot times agree within two minutes (uptime and checkedAt come from
* different clocks).
*/
export function createOomTracker(announce: (event: CloudOomEvent) => void) {
const seen = new Map<string, { bootAt: number; kills: number }>()
return {
observe(
workspaceId: string,
sandboxId: string,
status: HostedWorkspaceStatus,
now = Date.now()
): void {
const metrics = status.metrics
if (!metrics) {
return
}
const bootAt = status.checkedAt - metrics.uptimeSeconds * 1000
const previous = seen.get(sandboxId)
const sameBoot = previous !== undefined && Math.abs(previous.bootAt - bootAt) < 120_000
const announced = sameBoot ? previous.kills : 0
seen.set(sandboxId, {
bootAt: sameBoot ? previous.bootAt : bootAt,
kills: Math.max(announced, metrics.oomKills)
})
if (metrics.oomKills > announced) {
announce({
workspaceId,
oomKills: metrics.oomKills,
victim: metrics.lastOomVictim,
at: now
})
}
},
/** Forget sandboxes that are no longer running. */
prune(live: ReadonlySet<string | undefined>): void {
for (const sandboxId of seen.keys()) {
if (!live.has(sandboxId)) {
seen.delete(sandboxId)
}
}
}
}
}
let current: HostedWorkspaceService | null = null
export function setHostedWorkspaceService(next: HostedWorkspaceService | null): void {
current = next
}
/** Null when Orca Cloud is not configured in this build. */
export function getHostedWorkspaceService(): HostedWorkspaceService | null {
return current
}
@@ -0,0 +1,321 @@
// Orca Cloud workspace surfaces for the window: status panel, Restart, archive/unarchive, wake
// progress and OOM notices. Maps the renderer's workspace id to its Orca Cloud runtime record.
import { ipcMain, type BrowserWindow } from 'electron'
import {
listEphemeralVmRuntimes,
updateEphemeralVmRuntimeStatus
} from '../../shared/ephemeral-vm-runtime-store'
import {
ORCA_CLOUD_WORKSPACE_CHANNELS as CH,
type CloudWakeProgressEvent,
type CloudWakeSource
} from '../../shared/hosted-sandbox/cloud-workspace-api'
import type {
HostedArchivePullRequest,
HostedSandboxReadyResponse
} from '../../shared/hosted-sandbox/hosted-sandbox-wire'
import { applyResumedRuntimePairing } from '../ephemeral-vm/runtime-resume-pairing'
import { noteRecipeVmWakePairing } from '../ephemeral-vm/recipe-vm-wake-pairings'
import { invalidateRuntimeEnvironmentTransport } from '../ipc/runtime-environments/runtime-environments'
import { autoResumeAgentsAfterRollover } from './hosted-agent-auto-resume'
import type { HostedSandboxControlPlaneClient } from './hosted-sandbox-control-plane-client'
import { toReady } from './hosted-sandbox-driver'
import { createHostedRuntimeEnvironmentCall } from './hosted-runtime-environment-call'
import {
NOT_CLOUD,
createOomTracker,
failure,
findCloudRuntime,
readCachedCheckouts,
type CloudRuntime
} from './hosted-workspace-runtime'
export {
findCloudRuntime,
getHostedWorkspaceService,
setArchiveCheckoutsProvider,
setHostedWorkspaceService
} from './hosted-workspace-runtime'
export type HostedWorkspaceService = {
emitWakeProgress(event: Omit<CloudWakeProgressEvent, 'at'>): void
/** Who asked for the next wake of this workspace (message/ssh/preview); read once by the step. */
noteWakeSource(workspaceId: string, source: CloudWakeSource): void
takeWakeSource(workspaceId: string): CloudWakeSource
/** One OOM check for every running cloud workspace (the install beat calls it). */
pollOom(): Promise<void>
dispose(): void
}
/** Git state from the client-side offline cache, used when archiving a VM that is already asleep. */
export function installHostedWorkspaceService(deps: {
userDataPath: string
client: HostedSandboxControlPlaneClient
windows: () => readonly BrowserWindow[]
/** Restart lease schedulers after a lifecycle change. */
syncSessions: () => void
/** Stop the lease scheduler of one environment (its session is about to be replaced). */
stopSession: (environmentId: string) => void
log?: (line: string) => void
}): HostedWorkspaceService {
const log = deps.log ?? ((line: string) => console.error(line))
const wakeSources = new Map<string, CloudWakeSource>()
// Wakes in flight per workspace, so every phase carries the source the first one named.
const wakes = new Map<string, { source: CloudWakeSource; startedAt: number }>()
const broadcast = (channel: string, payload: unknown): void => {
for (const window of deps.windows()) {
if (!window.isDestroyed()) {
window.webContents.send(channel, payload)
}
}
}
const oom = createOomTracker((event) => broadcast(CH.oom, event))
async function reconnect(found: CloudRuntime, ready: HostedSandboxReadyResponse): Promise<void> {
const runtimeEnvironmentId = found.runtime.runtimeEnvironmentId
updateEphemeralVmRuntimeStatus(deps.userDataPath, found.runtime.id, {
status: 'running',
recipeResult: toReady(ready).result
})
if (!runtimeEnvironmentId) {
return
}
const { environment } = await applyResumedRuntimePairing({
userDataPath: deps.userDataPath,
environmentId: runtimeEnvironmentId,
pairingCode: ready.connection.pairingCode,
invalidateTransport: invalidateRuntimeEnvironmentTransport
})
noteRecipeVmWakePairing(
environment.id,
environment.pairingRevision ?? environment.createdAt,
true
)
}
/** The step answers before the window re-pairs; `ready` fires once the runtime record says running. */
async function announceReadyWhenRunning(workspaceId: string, startedAt: number): Promise<void> {
const deadline = Date.now() + 60_000
while (Date.now() < deadline && wakes.has(workspaceId)) {
const runtime = findCloudRuntime(deps.userDataPath, workspaceId)?.runtime
if (runtime?.status === 'running') {
service.emitWakeProgress({ workspaceId, phase: 'ready', source: 'open', startedAt })
return
}
await new Promise((resolve) => setTimeout(resolve, 250))
}
}
const service: HostedWorkspaceService = {
emitWakeProgress(event) {
if (event.phase === 'waking') {
wakes.set(event.workspaceId, { source: event.source, startedAt: event.startedAt })
}
const inFlight = wakes.get(event.workspaceId)
const source = inFlight?.source ?? event.source
if (event.phase === 'ready' || event.phase === 'failed') {
wakes.delete(event.workspaceId)
}
broadcast(CH.wakeProgress, {
...event,
source,
at: Date.now()
} satisfies CloudWakeProgressEvent)
if (event.phase === 'connecting' && event.source !== 'restart') {
void announceReadyWhenRunning(event.workspaceId, event.startedAt)
}
},
noteWakeSource(workspaceId, source) {
wakeSources.set(workspaceId, source)
},
takeWakeSource(workspaceId) {
const source = wakeSources.get(workspaceId) ?? 'open'
wakeSources.delete(workspaceId)
return source
},
async pollOom() {
const live = new Set(
listEphemeralVmRuntimes(deps.userDataPath)
.filter(
(runtime) =>
runtime.recipe?.provider === 'orca-cloud' &&
runtime.status === 'running' &&
runtime.workspaceId
)
.map((runtime) => findCloudRuntime(deps.userDataPath, runtime.workspaceId!)?.sandboxId)
)
oom.prune(live)
for (const runtime of listEphemeralVmRuntimes(deps.userDataPath)) {
if (
runtime.recipe?.provider !== 'orca-cloud' ||
runtime.status !== 'running' ||
!runtime.workspaceId
) {
continue
}
const found = findCloudRuntime(deps.userDataPath, runtime.workspaceId)
if (!found) {
continue
}
try {
const { status } = await deps.client.workspaceStatus(found.sandboxId)
oom.observe(runtime.workspaceId, found.sandboxId, status)
} catch {
// Status is advisory; the next beat retries.
}
}
},
dispose() {
for (const channel of [
CH.getStatus,
CH.restart,
CH.archive,
CH.unarchive,
CH.getArchive,
CH.listCloudWorkspaceIds
]) {
ipcMain.removeHandler(channel)
}
}
}
service.dispose()
ipcMain.handle(CH.listCloudWorkspaceIds, () =>
listEphemeralVmRuntimes(deps.userDataPath)
.filter((runtime) => runtime.recipe?.provider === 'orca-cloud' && runtime.workspaceId)
.map((runtime) => runtime.workspaceId!)
)
ipcMain.handle(CH.getStatus, async (_event, args: { workspaceId: string }) => {
const found = findCloudRuntime(deps.userDataPath, args.workspaceId)
if (!found) {
return NOT_CLOUD
}
try {
const { status } = await deps.client.workspaceStatus(found.sandboxId)
oom.observe(args.workspaceId, found.sandboxId, status)
return { ok: true, value: status }
} catch (error) {
return failure(error)
}
})
ipcMain.handle(CH.restart, async (_event, args: { workspaceId: string }) => {
const found = findCloudRuntime(deps.userDataPath, args.workspaceId)
if (!found) {
return NOT_CLOUD
}
const startedAt = Date.now()
service.emitWakeProgress({
workspaceId: args.workspaceId,
phase: 'waking',
source: 'restart',
startedAt
})
// The scheduler holds the old session's deadline and would warn/roll over the new one; a fresh one starts on sync.
deps.stopSession(found.sandboxId)
try {
const ready = await deps.client.restart(found.sandboxId)
service.emitWakeProgress({
workspaceId: args.workspaceId,
phase: 'connecting',
source: 'restart',
startedAt
})
await reconnect(found, ready)
deps.syncSessions()
service.emitWakeProgress({
workspaceId: args.workspaceId,
phase: 'ready',
source: 'restart',
startedAt
})
if (found.runtime.runtimeEnvironmentId) {
// Same continuation as a rollover: chats the restart interrupted carry on by themselves.
const call = createHostedRuntimeEnvironmentCall(
deps.userDataPath,
found.runtime.runtimeEnvironmentId
)
void autoResumeAgentsAfterRollover(call, startedAt).catch((error: unknown) =>
log(`[orca-cloud] auto-resume after restart failed: ${String(error)}`)
)
}
return { ok: true, value: null }
} catch (error) {
service.emitWakeProgress({
workspaceId: args.workspaceId,
phase: 'failed',
source: 'restart',
startedAt,
message: error instanceof Error ? error.message : String(error)
})
deps.syncSessions()
return failure(error)
}
})
ipcMain.handle(
CH.archive,
async (_event, args: { workspaceId: string; pullRequests?: HostedArchivePullRequest[] }) => {
const found = findCloudRuntime(deps.userDataPath, args.workspaceId)
if (!found) {
return NOT_CLOUD
}
try {
const checkouts = (await readCachedCheckouts(args.workspaceId)) ?? undefined
await deps.client.archive(found.sandboxId, {
...(args.pullRequests?.length ? { pullRequests: args.pullRequests.slice(0, 50) } : {}),
...(checkouts?.length ? { checkouts: checkouts.slice(0, 20) } : {})
})
updateEphemeralVmRuntimeStatus(deps.userDataPath, found.runtime.id, { status: 'suspended' })
deps.syncSessions()
return { ok: true, value: null }
} catch (error) {
return failure(error)
}
}
)
ipcMain.handle(CH.unarchive, async (_event, args: { workspaceId: string }) => {
const found = findCloudRuntime(deps.userDataPath, args.workspaceId)
if (!found) {
return NOT_CLOUD
}
const startedAt = Date.now()
service.emitWakeProgress({
workspaceId: args.workspaceId,
phase: 'waking',
source: 'unarchive',
startedAt
})
try {
// The control plane resumes the disk; the window then runs its ordinary wake, which re-pairs.
await deps.client.unarchive(found.sandboxId)
deps.syncSessions()
return { ok: true, value: null }
} catch (error) {
service.emitWakeProgress({
workspaceId: args.workspaceId,
phase: 'failed',
source: 'unarchive',
startedAt,
message: error instanceof Error ? error.message : String(error)
})
return failure(error)
}
})
ipcMain.handle(CH.getArchive, async (_event, args: { workspaceId: string }) => {
const found = findCloudRuntime(deps.userDataPath, args.workspaceId)
if (!found) {
return NOT_CLOUD
}
try {
return { ok: true, value: await deps.client.archiveInfo(found.sandboxId) }
} catch (error) {
return failure(error)
}
})
return service
}
+3
View File
@@ -28,6 +28,7 @@ import type { DashboardApi, TerminalPreviewApi } from './api/dashboard-api'
import type { DocPreviewApi } from './api/doc-preview-api'
import type { EmulatorApi } from './api/emulator-api'
import type { EphemeralVmApi } from './api/ephemeral-vm-api'
import type { OrcaCloudWorkspaceApi } from '../shared/hosted-sandbox/cloud-workspace-api'
import type { ExportApi, FilesystemApi } from './api/filesystem-api'
import type { GitInspectionApi } from './api/git-inspection-api'
import type { GitOperationApi } from './api/git-operation-api'
@@ -134,6 +135,8 @@ export type PreloadApi = {
emulator: EmulatorApi
hooks: HooksApi
ephemeralVm: EphemeralVmApi
/** Orca Cloud workspace lifecycle (status panel, Restart, archive); absent in builds without Orca Cloud. */
orcaCloudWorkspace: OrcaCloudWorkspaceApi
cache: WorkspaceSessionApi['cache']
session: WorkspaceSessionApi['session']
updater: UpdaterApi
@@ -0,0 +1,28 @@
import { ipcRenderer } from 'electron'
import {
ORCA_CLOUD_WORKSPACE_CHANNELS as CH,
type CloudOomEvent,
type CloudWakeProgressEvent,
type OrcaCloudWorkspaceApi
} from '../../shared/hosted-sandbox/cloud-workspace-api'
export const orcaCloudWorkspaceApi = {
getStatus: (args) => ipcRenderer.invoke(CH.getStatus, args),
restart: (args) => ipcRenderer.invoke(CH.restart, args),
archive: (args) => ipcRenderer.invoke(CH.archive, args),
unarchive: (args) => ipcRenderer.invoke(CH.unarchive, args),
getArchive: (args) => ipcRenderer.invoke(CH.getArchive, args),
listCloudWorkspaceIds: () => ipcRenderer.invoke(CH.listCloudWorkspaceIds),
onWakeProgress: (callback) => {
const listener = (_event: Electron.IpcRendererEvent, event: CloudWakeProgressEvent): void =>
callback(event)
ipcRenderer.on(CH.wakeProgress, listener)
return () => ipcRenderer.removeListener(CH.wakeProgress, listener)
},
onOom: (callback) => {
const listener = (_event: Electron.IpcRendererEvent, event: CloudOomEvent): void =>
callback(event)
ipcRenderer.on(CH.oom, listener)
return () => ipcRenderer.removeListener(CH.oom, listener)
}
} satisfies OrcaCloudWorkspaceApi
+2
View File
@@ -60,6 +60,7 @@ import { browserApi } from './api/browser/browser-bridge'
import { emulatorApi } from './api/emulator-bridge'
import { hooksApi } from './api/hooks-bridge'
import { ephemeralVmApi } from './api/ephemeral-vm-bridge'
import { orcaCloudWorkspaceApi } from './api/orca-cloud-workspace-bridge'
import { cacheApi } from './api/cache-bridge'
import { sessionApi } from './api/session-bridge'
import { updaterApi } from './api/updater-bridge'
@@ -167,6 +168,7 @@ const api = {
emulator: emulatorApi,
hooks: hooksApi,
ephemeralVm: ephemeralVmApi,
orcaCloudWorkspace: orcaCloudWorkspaceApi,
cache: cacheApi,
session: sessionApi,
updater: updaterApi,
@@ -0,0 +1,75 @@
// Window ↔ main contract for Orca Cloud workspace lifecycle surfaces: status panel, Restart,
// archive/unarchive, wake progress, OOM notices. Keyed by the renderer's workspace (worktree) id;
// main maps it to the Orca Cloud runtime record and its sandbox.
import type {
HostedArchivePullRequest,
HostedArchiveResponse,
HostedWorkspaceStatus
} from './hosted-sandbox-wire'
export type CloudWorkspaceErrorCode =
| 'not_cloud'
| 'unauthorized'
| 'entitlement_denied'
| 'conflict'
| 'unavailable'
| 'not_found'
| 'failed'
export type CloudWorkspaceResult<T> =
| { ok: true; value: T }
| { ok: false; code: CloudWorkspaceErrorCode; message: string; reason?: string | null }
/** Why a wake started; every source goes through the control plane's single-flight wake. */
export type CloudWakeSource = 'open' | 'message' | 'ssh' | 'preview' | 'restart' | 'unarchive'
export type CloudWakeProgressEvent = {
workspaceId: string
phase: 'waking' | 'connecting' | 'ready' | 'failed'
source: CloudWakeSource
/** Epoch ms the wake began (the overlay's elapsed timer). */
startedAt: number
at: number
message?: string
}
export type CloudOomEvent = {
workspaceId: string
/** Kernel OOM-kill counter after this notice (monotonic per VM boot). */
oomKills: number
victim: string | null
at: number
}
export type CloudArchiveInfo = NonNullable<HostedArchiveResponse['archive']>
export type OrcaCloudWorkspaceApi = {
/** Live status: state, CPU/mem/disk, uptime, cost so far, archive. Cheap; the panel polls it while open. */
getStatus: (args: { workspaceId: string }) => Promise<CloudWorkspaceResult<HostedWorkspaceStatus>>
/** "Restart" (Unresponsive / OOM): rehydrate the same disk and re-pair; agents auto-resume. */
restart: (args: { workspaceId: string }) => Promise<CloudWorkspaceResult<null>>
/** Sleep, then archive with a snapshot TTL; metadata captured in the VM or from the client cache. */
archive: (args: {
workspaceId: string
pullRequests?: HostedArchivePullRequest[]
}) => Promise<CloudWorkspaceResult<null>>
unarchive: (args: { workspaceId: string }) => Promise<CloudWorkspaceResult<null>>
getArchive: (args: {
workspaceId: string
}) => Promise<CloudWorkspaceResult<CloudArchiveInfo | null>>
/** Workspace ids of every Orca Cloud workspace this desktop knows (the sidebar dot / hover card). */
listCloudWorkspaceIds: () => Promise<string[]>
onWakeProgress: (callback: (event: CloudWakeProgressEvent) => void) => () => void
onOom: (callback: (event: CloudOomEvent) => void) => () => void
}
export const ORCA_CLOUD_WORKSPACE_CHANNELS = {
getStatus: 'orcaCloudWorkspace:getStatus',
restart: 'orcaCloudWorkspace:restart',
archive: 'orcaCloudWorkspace:archive',
unarchive: 'orcaCloudWorkspace:unarchive',
getArchive: 'orcaCloudWorkspace:getArchive',
listCloudWorkspaceIds: 'orcaCloudWorkspace:listIds',
wakeProgress: 'orcaCloudWorkspace:wakeProgress',
oom: 'orcaCloudWorkspace:oom'
} as const
@@ -14,7 +14,8 @@ export const HOSTED_SANDBOX_STATUSES = [
'resuming',
'destroying',
'destroyed',
'failed'
'failed',
'archived'
] as const
export type HostedSandboxStatus = (typeof HOSTED_SANDBOX_STATUSES)[number]
@@ -119,3 +120,131 @@ export const HostedSandboxErrorResponseSchema = z
reason: z.string().max(64).optional()
})
.strict()
/** Every wake source (open, message, SSH, preview) answers in this shape. */
export const HOSTED_WAKE_STATES = ['ready', 'waking', 'archived', 'failed', 'not_found'] as const
export const HostedWakeResponseSchema = z
.object({
state: z.enum(HOSTED_WAKE_STATES),
endpoint: z.string().startsWith('wss://').max(2048).nullable(),
readyAt: Millis.nullable(),
sandboxId: Id.nullable()
})
.strict()
export type HostedWakeResponse = z.infer<typeof HostedWakeResponseSchema>
export const HOSTED_WORKSPACE_STATES = [
'provisioning',
'sleeping',
'waking',
'ready',
'unresponsive',
'stopping',
'archived',
'failed'
] as const
export type HostedWorkspaceState = (typeof HOSTED_WORKSPACE_STATES)[number]
const Bytes = z.number().int().nonnegative()
export const HostedVmMetricsSchema = z
.object({
cpuPercent: z.number().int().min(0).max(100),
cpus: z.number().int().min(1),
load1: z.number().min(0),
memTotalBytes: Bytes,
memUsedBytes: Bytes,
diskTotalBytes: Bytes,
diskUsedBytes: Bytes,
uptimeSeconds: z.number().int().nonnegative(),
oomKills: z.number().int().nonnegative(),
lastOomVictim: z.string().max(64).nullable(),
orcadAlive: z.boolean()
})
.strict()
export type HostedVmMetrics = z.infer<typeof HostedVmMetricsSchema>
export const HostedWorkspaceStatusSchema = z
.object({
state: z.enum(HOSTED_WORKSPACE_STATES),
metrics: HostedVmMetricsSchema.nullable(),
uptimeMs: Millis.nullable(),
usage: z
.object({
totalMs: Millis,
centsPerHour: z.number().int().nonnegative(),
costCents: z.number().int().nonnegative()
})
.strict(),
archive: z.object({ archivedAt: Millis, expiresAt: Millis.nullable() }).strict().nullable(),
checkedAt: Millis
})
.strict()
export type HostedWorkspaceStatus = z.infer<typeof HostedWorkspaceStatusSchema>
export const HostedWorkspaceStatusResponseSchema = z
.object({ sandbox: HostedSandboxViewSchema, status: HostedWorkspaceStatusSchema })
.strict()
const ArchiveCommit = z
.object({
sha: z.string().max(64),
author: z.string().max(120),
at: z.number(),
subject: z.string().max(300)
})
.strict()
export const HostedArchiveCheckoutSchema = z
.object({
path: z.string().max(4096),
branch: z.string().max(255).nullable(),
head: z.string().max(64).nullable(),
upstream: z.string().max(255).nullable(),
ahead: z.number().int().nullable(),
behind: z.number().int().nullable(),
commits: z.array(ArchiveCommit).max(20),
changes: z
.array(z.object({ status: z.string().max(2), path: z.string().max(1024) }).strict())
.max(300),
diffStat: z.string().max(200).nullable(),
fileCount: z.number().int().nonnegative(),
files: z.array(z.string().max(1024)).max(2000)
})
.strict()
export type HostedArchiveCheckout = z.infer<typeof HostedArchiveCheckoutSchema>
export const HostedArchivePullRequestSchema = z
.object({
number: z.number().int().positive(),
title: z.string().max(300),
state: z.enum(['open', 'closed', 'merged', 'draft']),
url: z.url({ protocol: /^https$/ }).max(2048),
branch: z.string().max(255).optional()
})
.strict()
export type HostedArchivePullRequest = z.infer<typeof HostedArchivePullRequestSchema>
export const HostedArchiveRequestSchema = z
.object({
pullRequests: z.array(HostedArchivePullRequestSchema).max(50).optional(),
checkouts: z.array(HostedArchiveCheckoutSchema).max(20).optional()
})
.strict()
export type HostedArchiveRequest = z.infer<typeof HostedArchiveRequestSchema>
export const HostedArchiveResponseSchema = z
.object({
archive: z
.object({
capturedAt: Millis,
source: z.enum(['vm', 'client', 'none']),
metadata: z
.object({
checkouts: z.array(HostedArchiveCheckoutSchema).max(20),
pullRequests: z.array(HostedArchivePullRequestSchema).max(50).optional()
})
.strict()
})
.strict()
.nullable()
})
.strict()
export type HostedArchiveResponse = z.infer<typeof HostedArchiveResponseSchema>