From c250505f75b16df15fb65ed998b0d7c77ba45a18 Mon Sep 17 00:00:00 2001 From: Neil Date: Mon, 9 Nov 2026 05:02:20 -0800 Subject: [PATCH 1/8] feat(cloud): wake (single-flight, long-poll), restart, status metrics, archive/unarchive on the sandbox control plane # Conflicts: # cloud/apps/sandbox-control/src/index.ts # Conflicts: # cloud/apps/sandbox-control/src/env.ts # cloud/apps/sandbox-control/src/index.ts # cloud/apps/sandbox-control/test/fake-vercel.ts --- .../migrations/0006_lifecycle_w2.sql | 18 ++ cloud/apps/sandbox-control/src/api/router.ts | 19 +- .../sandbox-control/src/api/workspaces.ts | 17 +- cloud/apps/sandbox-control/src/env.ts | 4 + cloud/apps/sandbox-control/src/index.ts | 63 +++- cloud/apps/sandbox-control/src/lifecycle.ts | 25 +- cloud/apps/sandbox-control/src/reconcile.ts | 9 + .../sandbox-control/src/sandbox-provider.ts | 2 + .../src/sandbox-vm-commands.ts | 6 +- .../sandbox-control/src/sandbox-vm-inspect.ts | 161 ++++++++++ cloud/apps/sandbox-control/src/store.ts | 24 +- .../sandbox-control/src/vercel-sandbox-api.ts | 8 + cloud/apps/sandbox-control/src/wire.ts | 15 + .../src/workspace-lifecycle.ts | 300 ++++++++++++++++++ .../apps/sandbox-control/test/fake-vercel.ts | 17 + .../test/workspace-lifecycle.test.ts | 210 ++++++++++++ .../hosted-sandbox/hosted-sandbox-wire.ts | 131 +++++++- 17 files changed, 1010 insertions(+), 19 deletions(-) create mode 100644 cloud/apps/sandbox-control/migrations/0006_lifecycle_w2.sql create mode 100644 cloud/apps/sandbox-control/src/sandbox-vm-inspect.ts create mode 100644 cloud/apps/sandbox-control/src/workspace-lifecycle.ts create mode 100644 cloud/apps/sandbox-control/test/workspace-lifecycle.test.ts diff --git a/cloud/apps/sandbox-control/migrations/0006_lifecycle_w2.sql b/cloud/apps/sandbox-control/migrations/0006_lifecycle_w2.sql new file mode 100644 index 00000000000..1d49ff5f5e9 --- /dev/null +++ b/cloud/apps/sandbox-control/migrations/0006_lifecycle_w2.sql @@ -0,0 +1,18 @@ +-- Workspace lifecycle (wake, status, archive). +-- orcad's public wss endpoint for the current session (not a secret: pairing still needs the code), +-- and when orcad last became ready, so wake callers can report `readyAt` without a provider call. +ALTER TABLE sandboxes ADD COLUMN orcad_endpoint TEXT; +ALTER TABLE sandboxes ADD COLUMN ready_at INTEGER; +-- Archive = stopped with the snapshot kept for a finite TTL; the cron destroys it after archive_expires_at. +-- (`archived_at` itself comes from 0020_api_routines.sql, already applied on dev.) +ALTER TABLE sandboxes ADD COLUMN archive_expires_at INTEGER; + +-- What an archived workspace shows without waking: branch, last commits, changed files, file tree, PRs. +-- Captured in the VM at archive (or sent by the desktop from its cache when the VM was already asleep). +-- Bounded (see archive.ts); deleted with the sandbox. +CREATE TABLE sandbox_archives ( + sandbox_id TEXT PRIMARY KEY REFERENCES sandboxes (id), + captured_at INTEGER NOT NULL, + source TEXT NOT NULL, + metadata TEXT NOT NULL +); diff --git a/cloud/apps/sandbox-control/src/api/router.ts b/cloud/apps/sandbox-control/src/api/router.ts index e9ea439fd56..045bed9a658 100644 --- a/cloud/apps/sandbox-control/src/api/router.ts +++ b/cloud/apps/sandbox-control/src/api/router.ts @@ -129,6 +129,9 @@ function messageBucket(caller: ApiCaller): string { return caller.via === 'workspace' ? `msg:ws:${caller.sandboxId}` : `msg:${caller.accountId}:${caller.userId}` } +import { wakeSandbox } from '../workspace-lifecycle' +import { parseWakeWait } from '../wire' + export function isApiPath(pathname: string): boolean { return /^\/v1\/(me|api-keys|settings|audit|projects|workspaces|routines)(\/|$)/.test(pathname) } @@ -398,7 +401,21 @@ async function route(request: Request, url: URL, caller: ApiCaller, ctx: ApiCont return mutate('sleep', () => sleepWorkspace(life(), row)) } if (sub === 'wake' && method === 'POST') { - return mutate('wake', () => wakeWorkspace(life(), row)) + if (row.status === 'archived' || row.archived_at !== null) { + // The public API's wake brings an archived workspace back. + return mutate('wake', () => wakeWorkspace(life(), row)) + } + // Every wake source shares W2's single-flight wake: idempotent, `?wait=30s` long-polls. + requireScope(caller, 'workspaces:write') + const started = performance.now() + const answer = await wakeSandbox(life(), row, { + // API callers expect the VM up when wake answers: wait the full 30 s unless `?wait=` says otherwise. + waitMs: url.searchParams.has('wait') ? parseWakeWait(url, null) : 30_000, + background: ctx.waitUntil + }) + await auditCaller(ctx.db, caller, { action: 'workspace.wake', targetType: 'workspace', targetId: row.id, detail: { ms: Math.round(performance.now() - started), state: answer.state } }, ctx.now()) + const fresh = (await getWorkspace(ctx.db, caller.accountId, id)) ?? row + return json({ workspace: workspaceView(fresh), ...answer }) } if (sub === 'archive' && method === 'POST') { return mutate('archive', () => archiveWorkspace(life(), row)) diff --git a/cloud/apps/sandbox-control/src/api/workspaces.ts b/cloud/apps/sandbox-control/src/api/workspaces.ts index 666bca14cb8..4147318ae5e 100644 --- a/cloud/apps/sandbox-control/src/api/workspaces.ts +++ b/cloud/apps/sandbox-control/src/api/workspaces.ts @@ -3,6 +3,7 @@ import type { MachineSize } from '../entitlements' import { createSandbox, destroySandbox, resumeSandbox, suspendSandbox, type LifecycleDeps } from '../lifecycle' import { getSandbox, type SandboxRow } from '../store' +import { archiveSandbox, unarchiveSandbox } from '../workspace-lifecycle' import { ApiError } from './caller' import { randomId } from './keys' @@ -63,6 +64,8 @@ export function apiStatus(row: Pick): string { running: 'running', suspending: 'sleeping', suspended: 'sleeping', + // The API reports archive separately (`archived: true`); the VM itself is asleep. + archived: 'sleeping', resuming: 'waking', destroying: 'destroying', destroyed: 'destroyed', @@ -171,19 +174,21 @@ export async function sleepWorkspace(life: LifecycleDeps, row: WorkspaceRow): Pr } export async function wakeWorkspace(life: LifecycleDeps, row: WorkspaceRow): Promise { - const ready = await resumeSandbox(life, row) + // Waking an archived workspace brings it back (restores the snapshot TTL; boot clears archived_at). + const ready = row.status === 'archived' ? await unarchiveSandbox(life, row) : await resumeSandbox(life, row) if (row.archived_at !== null) { - // Waking an archived workspace brings it back. await life.db.prepare('UPDATE sandboxes SET archived_at = NULL WHERE id = ?').bind(row.id).run() } return (await getSandbox(life.db, ready.row.id)) as WorkspaceRow } -/** Archive = sleep + out of default lists. The disk is kept until retention or destroy. */ +/** + * Archive = stop and keep the snapshot for a TTL, out of default lists (W2's archiveSandbox: + * repo metadata captured in the VM, provider snapshot expiry shortened, cron deletes after the TTL). + */ export async function archiveWorkspace(life: LifecycleDeps, row: WorkspaceRow): Promise { - const slept = row.status === 'running' ? await sleepWorkspace(life, row) : row - await life.db.prepare('UPDATE sandboxes SET archived_at = COALESCE(archived_at, ?) WHERE id = ?').bind(life.now(), row.id).run() - return (await getSandbox(life.db, slept.id)) as WorkspaceRow + const archived = await archiveSandbox(life, row, {}) + return (await getSandbox(life.db, archived.id)) as WorkspaceRow } export async function destroyWorkspace(life: LifecycleDeps, row: WorkspaceRow): Promise { diff --git a/cloud/apps/sandbox-control/src/env.ts b/cloud/apps/sandbox-control/src/env.ts index cdab8e81cf1..d45c61aefb6 100644 --- a/cloud/apps/sandbox-control/src/env.ts +++ b/cloud/apps/sandbox-control/src/env.ts @@ -101,4 +101,8 @@ export type Env = { EGRESS_EXTRA_HOSTS?: string /** GitHub App webhook secret (secret): verifies X-Hub-Signature-256 on /v1/github/webhook. */ GITHUB_WEBHOOK_SECRET?: string + /** Archived workspaces are destroyed this many days after archive (default 30). */ + ARCHIVE_TTL_DAYS?: string + /** Status-panel cost estimate, cents per running hour by machine size: {"standard":75,"large":150}. */ + COST_CENTS_PER_HOUR?: string } diff --git a/cloud/apps/sandbox-control/src/index.ts b/cloud/apps/sandbox-control/src/index.ts index 6e31d327ba6..0f4479d778f 100644 --- a/cloud/apps/sandbox-control/src/index.ts +++ b/cloud/apps/sandbox-control/src/index.ts @@ -5,6 +5,13 @@ // GET /v1/sandboxes/:id -> {sandbox} // POST /v1/sandboxes/:id/suspend | /resume | /rollover -> {sandbox[, connection]} // POST /v1/sandboxes/:id/lease {extendMs} -> {sandbox} +// POST /v1/sandboxes/:id/wake[?wait=30s] -> {state, endpoint, readyAt, sandboxId} (idempotent, single-flight; +// the public API's POST /v1/workspaces/:id/wake answers the same fields plus `workspace`) +// POST /v1/sandboxes/:id/restart -> {sandbox, connection} +// GET /v1/sandboxes/:id/status -> {sandbox, status} (live metrics, cost) +// POST /v1/sandboxes/:id/archive {pullRequests?, checkouts?} -> {sandbox} +// GET /v1/sandboxes/:id/archive -> {archive} +// POST /v1/sandboxes/:id/unarchive -> {sandbox, connection} // DELETE /v1/sandboxes/:id -> {sandbox} // GET /v1/secrets?scope= | PUT, DELETE /v1/secrets/:scope/:NAME (see src/secrets.ts) // POST /v1/github/connect -> {installUrl, authorizeUrl} (bind a GitHub App installation to the account) @@ -62,7 +69,17 @@ import { ProviderError, type SandboxProviderApi } from './sandbox-provider' import { VmCommandError } from './sandbox-vm-commands' import { getSandbox, getSandboxByProviderName, listAccountSandboxes, type SandboxRow } from './store' import { createVercelSandboxApi } from './vercel-sandbox-api' -import { CreateRequestSchema, LeaseRequestSchema, readyView, sandboxView } from './wire' +import { CreateRequestSchema, LeaseRequestSchema, parseWakeWait, readyView, sandboxView } from './wire' +import { ArchiveRequestSchema } from './sandbox-vm-inspect' +import { + archiveSandbox, + readArchive, + restartSandbox, + unarchiveSandbox, + wakeSandbox, + workspaceStatus, + type WorkspaceConfig +} from './workspace-lifecycle' const MAX_JSON_BYTES = 16 * 1024 @@ -83,6 +100,24 @@ export type Deps = { waitUntil?: (work: Promise) => void /** Route groups mounted on the public API (routines, see src/routines). */ apiExtensions?: ApiExtension[] + /** Long-poll sleep (tests advance a fake clock). */ + sleep?: (ms: number) => Promise +} + +function workspaceConfig(env: Env, deps: Deps): WorkspaceConfig { + const days = Number(env.ARCHIVE_TTL_DAYS) + let centsPerHour: WorkspaceConfig['centsPerHour'] + try { + centsPerHour = env.COST_CENTS_PER_HOUR ? (JSON.parse(env.COST_CENTS_PER_HOUR) as WorkspaceConfig['centsPerHour']) : undefined + } catch { + centsPerHour = undefined + } + return { + archiveTtlMs: Number.isFinite(days) && days >= 1 ? Math.floor(days) * 86_400_000 : undefined, + centsPerHour, + background: deps.waitUntil, + sleep: deps.sleep + } } /** The public origin: PUBLIC_API_URL when set, else the request's own origin. */ @@ -512,7 +547,10 @@ export async function handleRequest(request: Request, env: Env, deps: Deps = {}) } return await handleSecretsRoute(request, url, env.SANDBOX_DB, secrets.keyring, caller, (deps.now ?? Date.now)(), readJson) } - const match = /^\/v1\/sandboxes(?:\/([A-Za-z0-9_]+)(?:\/(suspend|resume|lease|rollover))?)?$/.exec(url.pathname) + const match = + /^\/v1\/sandboxes(?:\/([A-Za-z0-9_]+)(?:\/(suspend|resume|lease|rollover|wake|restart|status|archive|unarchive))?)?$/.exec( + url.pathname + ) if (!match) { return fail('not_found', 404) } @@ -563,6 +601,27 @@ export async function handleRequest(request: Request, env: Env, deps: Deps = {}) const ready = await rolloverSandbox(life, row) return readyResponse(ready) } + case 'POST wake': + return json(await wakeSandbox(life, row, { waitMs: parseWakeWait(url, await readJson(request)), ...workspaceConfig(env, deps) })) + case 'POST restart': { + const ready = await restartSandbox(life, row) + return json(readyView(ready.row, ready.connection)) + } + case 'GET status': + return json({ sandbox: sandboxView(row), status: await workspaceStatus(life, row, workspaceConfig(env, deps)) }) + case 'POST archive': { + const body = ArchiveRequestSchema.safeParse(await readJson(request)) + if (!body.success) { + return fail('invalid_request', 400) + } + return json({ sandbox: sandboxView(await archiveSandbox(life, row, body.data, workspaceConfig(env, deps))) }) + } + case 'GET archive': + return json({ archive: await readArchive(env.SANDBOX_DB, row.id) }) + case 'POST unarchive': { + const ready = await unarchiveSandbox(life, row) + return json(readyView(ready.row, ready.connection)) + } case 'POST lease': { const body = LeaseRequestSchema.safeParse(await readJson(request)) if (!body.success) { diff --git a/cloud/apps/sandbox-control/src/lifecycle.ts b/cloud/apps/sandbox-control/src/lifecycle.ts index 20d157c15c3..6171fe720e9 100644 --- a/cloud/apps/sandbox-control/src/lifecycle.ts +++ b/cloud/apps/sandbox-control/src/lifecycle.ts @@ -202,7 +202,7 @@ export function sessionFields(deps: LifecycleDeps, state: ProviderSandboxState): } } -async function claim(deps: LifecycleDeps, row: SandboxRow, patch: Partial): Promise { +export async function claim(deps: LifecycleDeps, row: SandboxRow, patch: Partial): Promise { const next = await transition(deps.db, row, patch, deps.now()) if (!next) { throw new LifecycleError('conflict', 'concurrent_transition') @@ -214,7 +214,7 @@ export function guardrails(deps: LifecycleDeps): GuardrailConfig { return deps.config.guardrails ?? DEFAULT_GUARDRAILS } -async function requireStart(deps: LifecycleDeps, accountId: string, userId: string, machineSize: MachineSize) { +export async function requireStart(deps: LifecycleDeps, accountId: string, userId: string, machineSize: MachineSize) { const [, runningSandboxes, runningSandboxesForUser] = await Promise.all([ assertStartsOpen(deps.db, guardrails(deps)), countActiveSandboxes(deps.db, accountId), @@ -239,6 +239,11 @@ async function requireStart(deps: LifecycleDeps, accountId: string, userId: stri }) } +function orcadEndpoint(state: ProviderSandboxState): string | null { + const route = state.routes.find((candidate) => candidate.port === ORCAD_PORT) + return route ? route.url.replace(/^https:/, 'wss:') : null +} + async function boot( deps: LifecycleDeps, row: SandboxRow, @@ -279,6 +284,10 @@ async function boot( const running = await claim(deps, row, { status: 'running', last_error: null, + orcad_endpoint: orcadEndpoint(state), + ready_at: deps.now(), + archived_at: null, + archive_expires_at: null, retention_warned_at: null, github_token_expires_at: git?.expiresAt ?? null, egress_secret_hosts: routed.egress_secret_hosts, @@ -497,9 +506,13 @@ export async function failAndRelease(deps: LifecycleDeps, row: SandboxRow): Prom } } -async function stopForSuspend(deps: LifecycleDeps, row: SandboxRow): Promise { +export async function stopForSuspend( + deps: LifecycleDeps, + row: SandboxRow, + options: { drainSeconds?: number } = {} +): Promise { if (row.session_id) { - await drainOrcad(deps.provider, row.session_id) + await drainOrcad(deps.provider, row.session_id, options.drainSeconds) const stopped = await deps.provider.stopSession(row.session_id) if (stopped.snapshotId) { await recordSnapshots(deps.db, row.id, [stopped.snapshotId], deps.now()) @@ -551,7 +564,7 @@ export async function resumeSandbox(deps: LifecycleDeps, row: SandboxRow): Promi return wake(deps, resuming) } -async function wake(deps: LifecycleDeps, row: SandboxRow): Promise { +export async function wake(deps: LifecycleDeps, row: SandboxRow): Promise { const timer = phaseTimer(deps.now) // The provider boot is the long pole; secrets and the repo token resolve alongside it. const [state, prep] = await Promise.all([ @@ -629,6 +642,8 @@ export async function destroySandbox(deps: LifecycleDeps, row: SandboxRow): Prom throw new LifecycleError('provider_unavailable', 'snapshot_cleanup_incomplete') } await markSnapshotsDeleted(deps.db, row.id, deps.now()) + // Archive metadata (repo state, file tree) goes with the workspace. + await deps.db.prepare('DELETE FROM sandbox_archives WHERE sandbox_id = ?').bind(row.id).run() const destroyed = await claim(deps, destroying, { status: 'destroyed', ...NO_SESSION, diff --git a/cloud/apps/sandbox-control/src/reconcile.ts b/cloud/apps/sandbox-control/src/reconcile.ts index 8dc08b323e7..1792e5f395b 100644 --- a/cloud/apps/sandbox-control/src/reconcile.ts +++ b/cloud/apps/sandbox-control/src/reconcile.ts @@ -16,6 +16,7 @@ import { killSwitchOn } from './guardrails' import { drainAll, housekeeping, openHold, type HousekeepingSummary } from './retention' import { listByStatus, transition, type SandboxRow } from './store' import { flushUsage, queueUsage } from './usage' +import { expiredArchives } from './workspace-lifecycle' const STUCK_MS = 15 * 60_000 /** The desktop rolls over at the deadline; the cron steps in only if it did not (provider cap is +10 min). */ @@ -29,6 +30,7 @@ export type ReconcileSummary = Record< | 'leaseLapsed' | 'released' | 'repaired' + | 'archivesExpired' | 'billingErrors', number > & { housekeeping?: HousekeepingSummary } @@ -42,6 +44,7 @@ export async function reconcile(deps: LifecycleDeps): Promise leaseLapsed: 0, released: 0, repaired: 0, + archivesExpired: 0, billingErrors: 0 } const polled = await guard(async () => { @@ -81,6 +84,12 @@ export async function reconcile(deps: LifecycleDeps): Promise } }) } + for (const row of await expiredArchives(deps.db, deps.now())) { + await guard(async () => { + await destroySandbox(deps, row) + summary.archivesExpired++ + }) + } await flushUsage(deps.db, deps.entitlements, deps.now()) await guard(async () => { const housekept = await housekeeping(deps) diff --git a/cloud/apps/sandbox-control/src/sandbox-provider.ts b/cloud/apps/sandbox-control/src/sandbox-provider.ts index 49e720b0543..7d954b40ddd 100644 --- a/cloud/apps/sandbox-control/src/sandbox-provider.ts +++ b/cloud/apps/sandbox-control/src/sandbox-provider.ts @@ -89,6 +89,8 @@ export type SandboxProviderApi = { extendTimeout(sessionId: string, durationMs: number): Promise updateNetworkPolicy(sessionId: string, policy: NetworkPolicy): Promise runCommand(sessionId: string, params: RunCommandParams): Promise + /** Provider-side snapshot expiry after last use (archive TTL backstop). */ + updateSnapshotExpiration(name: string, expirationMs: number): Promise /** Resolves when already gone. */ deleteSandbox(name: string): Promise listSnapshots(name: string): Promise diff --git a/cloud/apps/sandbox-control/src/sandbox-vm-commands.ts b/cloud/apps/sandbox-control/src/sandbox-vm-commands.ts index 4f502ad9fe7..15440d3159f 100644 --- a/cloud/apps/sandbox-control/src/sandbox-vm-commands.ts +++ b/cloud/apps/sandbox-control/src/sandbox-vm-commands.ts @@ -124,11 +124,13 @@ export async function runningOrcad(provider: SandboxProviderApi, sessionId: stri } /** Best effort: orcad turns working chats into durable restart offers before the VM stops. */ -export async function drainOrcad(provider: SandboxProviderApi, sessionId: string): Promise { +export async function drainOrcad(provider: SandboxProviderApi, sessionId: string, waitSeconds?: number): Promise { try { const result = await provider.runCommand(sessionId, { command: DRAIN_COMMAND, - timeoutMs: DRAIN_TIMEOUT_MS + // The script's own default (280s) leaves room inside DRAIN_TIMEOUT_MS; Restart passes a short wait. + ...(waitSeconds ? { args: [String(Math.max(1, Math.floor(waitSeconds)))] } : {}), + timeoutMs: waitSeconds ? (waitSeconds + 20) * 1000 : DRAIN_TIMEOUT_MS }) return result.exitCode === 0 } catch (error) { diff --git a/cloud/apps/sandbox-control/src/sandbox-vm-inspect.ts b/cloud/apps/sandbox-control/src/sandbox-vm-inspect.ts new file mode 100644 index 00000000000..44509222f79 --- /dev/null +++ b/cloud/apps/sandbox-control/src/sandbox-vm-inspect.ts @@ -0,0 +1,161 @@ +// Read-only probes the control plane runs in a VM: live metrics (status panel, OOM banner) and the +// git metadata an archived workspace shows. Fixed scripts compiled into the Worker (never user +// input), so they work on every golden image without a rebuild. Output is one JSON line, parsed strictly. +// Bash `\${...}` is escaped in these templates; `${CONST}` interpolates the TypeScript constants. +import { z } from 'zod' +import { ProviderError, type SandboxProviderApi } from './sandbox-provider' + +const PROJECT = '/vercel/sandbox/project' + +/** + * CPU from two /proc/stat samples 250 ms apart, memory from /proc/meminfo, disk for the project + * volume, uptime, and OOM: the kernel's oom_kill counter (/proc/vmstat, readable unprivileged) plus + * the last killed process name from dmesg when sudo allows it. + */ +export const METRICS_SCRIPT = String.raw` +set -u +cpu() { awk '/^cpu /{idle=$5+$6; t=0; for(i=2;i<=NF;i++) t+=$i; print t, idle}' /proc/stat; } +read t1 i1 < <(cpu); sleep 0.25; read t2 i2 < <(cpu) +dt=$((t2-t1)); di=$((i2-i1)); [ "$dt" -gt 0 ] || dt=1 +cpu_pct=$(( (100*(dt-di)) / dt )) +mem_total=$(awk '/^MemTotal:/{print $2*1024}' /proc/meminfo) +mem_avail=$(awk '/^MemAvailable:/{print $2*1024}' /proc/meminfo) +read disk_total disk_used < <(df -B1 --output=size,used ${PROJECT} / 2>/dev/null | sed -n 2p) +uptime_s=$(awk '{printf "%d", $1}' /proc/uptime) +oom=$(awk '/^oom_kill /{print $2}' /proc/vmstat 2>/dev/null); oom=\${oom:-0} +victim=$(sudo -n dmesg 2>/dev/null | grep -oE 'Killed process [0-9]+ \([^)]{1,64}\)' | tail -n 1 | sed -E 's/.*\((.*)\)/\1/' | tr -cd 'A-Za-z0-9._+-' | head -c 64) +orcad=false; pid=$(cat "$HOME/.orca-sandbox/orcad.pid" 2>/dev/null || echo 0); kill -0 "$pid" 2>/dev/null && orcad=true +load=$(awk '{print $1}' /proc/loadavg) +printf '{"cpuPercent":%d,"cpus":%d,"load1":%s,"memTotalBytes":%d,"memUsedBytes":%d,"diskTotalBytes":%d,"diskUsedBytes":%d,"uptimeSeconds":%d,"oomKills":%d,"lastOomVictim":%s,"orcadAlive":%s}\n' \ + "$cpu_pct" "$(nproc)" "$load" "$mem_total" "$((mem_total-mem_avail))" "\${disk_total:-0}" "\${disk_used:-0}" "$uptime_s" "$oom" \ + "$( [ -n "$victim" ] && printf '"%s"' "$victim" || printf null )" "$orcad" +`.replaceAll('\\${', '${') + +export const VmMetricsSchema = z + .object({ + cpuPercent: z.number().int().min(0).max(100), + cpus: z.number().int().min(1), + load1: z.number().min(0), + memTotalBytes: z.number().int().min(0), + memUsedBytes: z.number().int().min(0), + diskTotalBytes: z.number().int().min(0), + diskUsedBytes: z.number().int().min(0), + uptimeSeconds: z.number().int().min(0), + oomKills: z.number().int().min(0), + lastOomVictim: z.string().max(64).nullable(), + orcadAlive: z.boolean() + }) + .strict() + +export type VmMetrics = z.infer + +const MAX_TREE = 2000 +const MAX_CHANGED = 300 +const MAX_COMMITS = 20 + +/** + * Archive metadata for every git checkout in the project (the repo plus its worktrees): branch, + * head, upstream ahead/behind, last commits, uncommitted changes and a capped file tree. Built + * with git's own porcelain output, JSON-escaped by node (always present in the image). + */ +export const ARCHIVE_SCRIPT = String.raw` +set -u +cd ${PROJECT} 2>/dev/null || { echo '{"checkouts":[]}'; exit 0; } +node="$(find /opt/orca/runtime/runtimes -type f -name node 2>/dev/null | head -n 1)"; node=\${node:-node} +git worktree list --porcelain 2>/dev/null | awk '/^worktree /{print substr($0,10)}' > /tmp/orca-archive-wt || true +[ -s /tmp/orca-archive-wt ] || echo ${PROJECT} > /tmp/orca-archive-wt +"$node" -e ' +const { execFileSync } = require("child_process") +const fs = require("fs") +const run = (cwd, args) => { try { return execFileSync("git", args, { cwd, encoding: "utf8", maxBuffer: 8 << 20, timeout: 8000 }).trim() } catch { return "" } } +const lines = (s) => s ? s.split("\n") : [] +const checkouts = fs.readFileSync("/tmp/orca-archive-wt", "utf8").split("\n").filter(Boolean).slice(0, 20).map((dir) => { + const counts = run(dir, ["rev-list", "--left-right", "--count", "@{upstream}...HEAD"]).split(/\s+/) + const tree = lines(run(dir, ["ls-files", "--cached", "--others", "--exclude-standard"])) + return { + path: dir, + branch: run(dir, ["rev-parse", "--abbrev-ref", "HEAD"]) || null, + head: run(dir, ["rev-parse", "HEAD"]) || null, + upstream: run(dir, ["rev-parse", "--abbrev-ref", "@{upstream}"]) || null, + behind: counts.length === 2 ? Number(counts[0]) : null, + ahead: counts.length === 2 ? Number(counts[1]) : null, + commits: lines(run(dir, ["log", "-n", "${MAX_COMMITS}", "--format=%H%x1f%an%x1f%at%x1f%s"])).map((l) => { + const [sha, author, at, subject] = l.split("\x1f") + return { sha, author: (author || "").slice(0, 120), at: Number(at) * 1000, subject: (subject || "").slice(0, 300) } + }), + changes: lines(run(dir, ["status", "--porcelain=v1", "-uall"])).slice(0, ${MAX_CHANGED}).map((l) => ({ status: l.slice(0, 2).trim() || "?", path: l.slice(3).slice(0, 1024) })), + diffStat: run(dir, ["diff", "HEAD", "--shortstat"]).slice(0, 200) || null, + fileCount: tree.length, + files: tree.slice(0, ${MAX_TREE}).map((f) => f.slice(0, 1024)) + } +}) +process.stdout.write(JSON.stringify({ checkouts }) + "\n") +' +`.replaceAll('\\${', '${') + +const Commit = z.object({ sha: z.string().max(64), author: z.string().max(120), at: z.number(), subject: z.string().max(300) }) +const Checkout = z.object({ + path: z.string().max(4096), + branch: z.string().max(255).nullable(), + head: z.string().max(64).nullable(), + upstream: z.string().max(255).nullable(), + ahead: z.number().int().nullable(), + behind: z.number().int().nullable(), + commits: z.array(Commit).max(MAX_COMMITS), + changes: z.array(z.object({ status: z.string().max(2), path: z.string().max(1024) })).max(MAX_CHANGED), + diffStat: z.string().max(200).nullable(), + fileCount: z.number().int().min(0), + files: z.array(z.string().max(1024)).max(MAX_TREE) +}) +const PullRequest = z.object({ + number: z.number().int().positive(), + title: z.string().max(300), + state: z.enum(['open', 'closed', 'merged', 'draft']), + url: z.url({ protocol: /^https$/ }).max(2048), + branch: z.string().max(255).optional() +}) + +export const ArchiveMetadataSchema = z.object({ + checkouts: z.array(Checkout).max(20), + pullRequests: z.array(PullRequest).max(50).optional() +}) +export type ArchiveMetadata = z.infer + +/** What the desktop may send at archive: PRs it knows, and git state from its cache if the VM is asleep. */ +export const ArchiveRequestSchema = z + .object({ + pullRequests: z.array(PullRequest).max(50).optional(), + checkouts: z.array(Checkout).max(20).optional() + }) + .strict() + +async function runJson( + provider: SandboxProviderApi, + sessionId: string, + script: string, + schema: z.ZodType, + timeoutMs: number +): Promise { + try { + const result = await provider.runCommand(sessionId, { command: 'bash', args: ['-c', script], timeoutMs }) + if (result.exitCode !== 0) { + return null + } + const parsed = schema.safeParse(JSON.parse(result.stdout.trim().split('\n').at(-1) ?? '')) + return parsed.success ? parsed.data : null + } catch (error) { + if (error instanceof ProviderError || error instanceof SyntaxError) { + return null + } + throw error + } +} + +/** Null when the VM did not answer in time (the status panel shows "Unresponsive"). */ +export function readVmMetrics(provider: SandboxProviderApi, sessionId: string): Promise { + return runJson(provider, sessionId, METRICS_SCRIPT, VmMetricsSchema, 8_000) +} + +export function captureArchiveMetadata(provider: SandboxProviderApi, sessionId: string): Promise { + return runJson(provider, sessionId, ARCHIVE_SCRIPT, ArchiveMetadataSchema, 30_000) +} diff --git a/cloud/apps/sandbox-control/src/store.ts b/cloud/apps/sandbox-control/src/store.ts index e2c2e1766ac..f3e2de4ed77 100644 --- a/cloud/apps/sandbox-control/src/store.ts +++ b/cloud/apps/sandbox-control/src/store.ts @@ -11,6 +11,8 @@ export type SandboxStatus = | 'destroying' | 'destroyed' | 'failed' + /** Stopped with the snapshot kept for a TTL; unarchive resumes it, the cron destroys it after the TTL. */ + | 'archived' export type SandboxRow = { id: string @@ -37,6 +39,12 @@ export type SandboxRow = { /** Comma list of model API hosts whose key the egress proxy swaps in this session. */ egress_secret_hosts: string | null retention_warned_at: number | null + /** orcad's public wss endpoint for the current session. */ + orcad_endpoint: string | null + /** When orcad last became ready (wake callers report it as `readyAt`). */ + ready_at: number | null + archived_at: number | null + archive_expires_at: number | null } export type SessionFields = Pick< @@ -44,7 +52,8 @@ export type SessionFields = Pick< 'session_id' | 'session_started_at' | 'session_deadline_at' | 'lease_expires_at' | 'usage_settled_at' > -export const NO_SESSION: SessionFields = { +export const NO_SESSION: SessionFields & Pick = { + orcad_endpoint: null, session_id: null, session_started_at: null, session_deadline_at: null, @@ -137,7 +146,8 @@ type Patch = Partial([ 'status', 'session_id', 'session_started_at', 'session_deadline_at', 'lease_expires_at', 'usage_settled_at', 'last_error', 'destroyed_at', 'github_token_expires_at', - 'egress_secret_hosts', 'repo_url', 'retention_warned_at' + 'egress_secret_hosts', 'repo_url', 'retention_warned_at', 'orcad_endpoint', 'ready_at', + 'archived_at', 'archive_expires_at' ]) /** Applies `patch` only if nobody moved the row since `expected` was read; null on a lost race. */ @@ -186,3 +196,13 @@ export async function recordedSnapshotIds(db: D1Database, sandboxId: string): Pr .all<{ snapshot_id: string }>() return results.map((row) => row.snapshot_id) } + +/** Usage so far for one sandbox: settled segments plus the open one (ms). */ +export async function sandboxUsageMs(db: D1Database, row: SandboxRow, now: number): Promise { + const settled = await db + .prepare('SELECT COALESCE(SUM(ended_at - started_at), 0) AS ms FROM usage_outbox WHERE sandbox_id = ?') + .bind(row.id) + .first<{ ms: number }>() + const open = row.status === 'running' && row.usage_settled_at !== null ? Math.max(0, now - row.usage_settled_at) : 0 + return (settled?.ms ?? 0) + open +} diff --git a/cloud/apps/sandbox-control/src/vercel-sandbox-api.ts b/cloud/apps/sandbox-control/src/vercel-sandbox-api.ts index 19a8b5364cf..c8e54ad1248 100644 --- a/cloud/apps/sandbox-control/src/vercel-sandbox-api.ts +++ b/cloud/apps/sandbox-control/src/vercel-sandbox-api.ts @@ -197,6 +197,14 @@ export function createVercelSandboxApi( return readCommandStream(response) }, + async updateSnapshotExpiration(name, expirationMs) { + // Archive shortens the provider-side backstop; unarchive restores it. Same fields as create. + await request('PATCH', `/v2/sandboxes/${encodeURIComponent(name)}`, { + query: { projectId: config.projectId }, + body: { snapshotExpiration: expirationMs, keepLastSnapshots: { count: 2, expiration: expirationMs } } + }) + }, + deleteSandbox: (name) => missingOk(() => request('DELETE', `/v2/sandboxes/${encodeURIComponent(name)}`, { diff --git a/cloud/apps/sandbox-control/src/wire.ts b/cloud/apps/sandbox-control/src/wire.ts index 1d7a323c477..d270017a8a6 100644 --- a/cloud/apps/sandbox-control/src/wire.ts +++ b/cloud/apps/sandbox-control/src/wire.ts @@ -72,3 +72,18 @@ export function readyView(row: SandboxRow, connection: OrcadReady) { } } } + +/** `?wait=30s` / `?wait=30000ms` / `?wait=30`, or body `{wait: true}` (= the 30 s max). 0 when absent. */ +export function parseWakeWait(url: URL, body: unknown): number { + const raw = url.searchParams.get('wait') + if (raw !== null) { + const match = /^(\d{1,6})(ms|s)?$/.exec(raw.trim()) + if (!match) { + return 0 + } + const value = Number(match[1]) + return Math.min(30_000, match[2] === 'ms' ? value : value * 1000) + } + const wait = (body as { wait?: unknown } | null)?.wait + return wait === true ? 30_000 : typeof wait === 'number' && Number.isFinite(wait) ? Math.min(30_000, Math.max(0, wait)) : 0 +} diff --git a/cloud/apps/sandbox-control/src/workspace-lifecycle.ts b/cloud/apps/sandbox-control/src/workspace-lifecycle.ts new file mode 100644 index 00000000000..f534be2b65d --- /dev/null +++ b/cloud/apps/sandbox-control/src/workspace-lifecycle.ts @@ -0,0 +1,300 @@ +// Workspace lifecycle on top of lifecycle.ts: wake (single-flight, long-poll; every wake source — +// open, message, SSH, preview — goes through it), Restart for unresponsive VMs, archive/unarchive +// with a snapshot TTL, and the status panel's live view (state, metrics, uptime, cost so far). +import type { MachineSize } from './entitlements' +import { GuardrailError, killSwitchOn } from './guardrails' +import { + DEFAULT_SNAPSHOT_EXPIRATION_MS, + LifecycleError, + claim, + guardrails, + markProviderStopped, + requireStart, + resumeSandbox, + stopForSuspend, + suspendSandbox, + wake, + type LifecycleDeps, + type Ready +} from './lifecycle' +import { + ArchiveMetadataSchema, + captureArchiveMetadata, + readVmMetrics, + type ArchiveMetadata, + type VmMetrics +} from './sandbox-vm-inspect' +import { NO_SESSION, getSandbox, sandboxUsageMs, transition, type SandboxRow } from './store' +import { flushUsage } from './usage' + +const DAY_MS = 86_400_000 +export const DEFAULT_ARCHIVE_TTL_MS = 30 * DAY_MS +/** Provider backstop past Orca's own archive TTL, so Orca's cron always deletes first. */ +const ARCHIVE_PROVIDER_GRACE_MS = 7 * DAY_MS +export const MAX_WAKE_WAIT_MS = 30_000 +const POLL_MS = 400 +/** Restart gives a wedged orcad this long to write durable offers before the stop. */ +const RESTART_DRAIN_SECONDS = 15 +/** A resuming/suspending row older than this is treated as stuck by Restart (the cron's bar is 15 min). */ +const RESTART_STUCK_MS = 90_000 +export const DEFAULT_CENTS_PER_HOUR: Record = { standard: 75, large: 150 } + +export type WorkspaceConfig = { + archiveTtlMs?: number + centsPerHour?: Partial> + /** Keeps a wake running after the response (Worker ctx.waitUntil); tests omit it. */ + background?: (work: Promise) => void + sleep?: (ms: number) => Promise +} + +export type WakeState = 'ready' | 'waking' | 'archived' | 'failed' | 'not_found' +export type WakeAnswer = { state: WakeState; endpoint: string | null; readyAt: number | null; sandboxId: string | null } + +/** wake/status share one vocabulary with the desktop's host health states. */ +export type WorkspaceState = + | 'provisioning' + | 'sleeping' + | 'waking' + | 'ready' + | 'unresponsive' + | 'stopping' + | 'archived' + | 'failed' + +function wakeAnswer(row: SandboxRow | null): WakeAnswer { + if (!row || row.status === 'destroyed' || row.status === 'destroying') { + return { state: 'not_found', endpoint: null, readyAt: null, sandboxId: row?.id ?? null } + } + const state: WakeState = + row.status === 'running' + ? 'ready' + : row.status === 'archived' + ? 'archived' + : row.status === 'failed' + ? 'failed' + : 'waking' + return { + state, + endpoint: state === 'ready' ? row.orcad_endpoint : null, + readyAt: state === 'ready' ? row.ready_at : null, + sandboxId: row.id + } +} + +const defaultSleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + +/** The row says running, but a lapsed lease means the provider may have stopped it on its own. */ +async function confirmRunning(deps: LifecycleDeps, row: SandboxRow): Promise { + if (row.lease_expires_at === null || deps.now() < row.lease_expires_at) { + return row + } + const state = await deps.provider.getSandbox(row.provider_name, { resume: false }) + return state?.session.status === 'running' ? row : markProviderStopped(deps, row) +} + +/** + * Idempotent wake. A sleeping workspace starts waking (one wake however many callers race: the + * `resuming` claim is a compare-and-set); `waitMs` long-polls until ready or the wait runs out. + * Archived workspaces are not woken implicitly — unarchive is an explicit action. + */ +export async function wakeSandbox( + deps: LifecycleDeps, + row: SandboxRow, + options: { waitMs: number } & WorkspaceConfig +): Promise { + const waitMs = Math.max(0, Math.min(options.waitMs, MAX_WAKE_WAIT_MS)) + const sleep = options.sleep ?? defaultSleep + let current = row.status === 'running' ? await confirmRunning(deps, row) : row + let ours: Promise | null = null + if (current.status === 'suspended') { + await requireStart(deps, current.account_id, current.user_id, current.machine_size) + const resuming = await transition(deps.db, current, { status: 'resuming' }, deps.now()) + if (resuming) { + ours = wake(deps, resuming).catch((error: unknown) => { + console.log(JSON.stringify({ event: 'wake_failed', sandboxId: row.id, error: String(error).slice(0, 120) })) + }) + options.background?.(ours) + } + // Lost the race: someone else's wake is running; fall through and wait on the row. + current = (await getSandbox(deps.db, row.id)) ?? current + } + const deadline = deps.now() + waitMs + if (ours && waitMs > 0) { + await Promise.race([ours, sleep(waitMs)]) + current = (await getSandbox(deps.db, row.id)) ?? current + } + while (current.status === 'resuming' && deps.now() < deadline) { + await sleep(Math.min(POLL_MS, Math.max(1, deadline - deps.now()))) + current = (await getSandbox(deps.db, row.id)) ?? current + } + return wakeAnswer(current) +} + +/** + * "Restart" for an unresponsive workspace: one rehydrate path (short drain, stop with snapshot, + * boot the same disk). Works from running, sleeping, or a wake/sleep that got stuck mid-way. + */ +export async function restartSandbox(deps: LifecycleDeps, row: SandboxRow): Promise { + if (await killSwitchOn(deps.db, guardrails(deps))) { + throw new GuardrailError('maintenance') + } + let current = row + if ((current.status === 'resuming' || current.status === 'suspending') && deps.now() - current.updated_at >= RESTART_STUCK_MS) { + const state = await deps.provider.getSandbox(current.provider_name, { resume: false }) + if (state?.session.status === 'running') { + await deps.provider.stopSession(state.session.id).catch(() => undefined) + } + current = await markProviderStopped(deps, current) + } + if (current.status === 'suspended') { + return resumeSandbox(deps, current) + } + if (current.status !== 'running') { + throw new LifecycleError('conflict', `sandbox_${current.status}`) + } + const suspending = await claim(deps, current, { status: 'suspending' }) + const stopped = await stopForSuspend(deps, suspending, { drainSeconds: RESTART_DRAIN_SECONDS }) + await flushUsage(deps.db, deps.entitlements, deps.now(), row.id) + return wake(deps, await claim(deps, stopped, { status: 'resuming' })) +} + +export async function readArchive(db: D1Database, sandboxId: string): Promise<{ capturedAt: number; source: string; metadata: ArchiveMetadata } | null> { + const stored = await db + .prepare('SELECT captured_at, source, metadata FROM sandbox_archives WHERE sandbox_id = ?') + .bind(sandboxId) + .first<{ captured_at: number; source: string; metadata: string }>() + if (!stored) { + return null + } + const parsed = ArchiveMetadataSchema.safeParse(JSON.parse(stored.metadata)) + return parsed.success ? { capturedAt: stored.captured_at, source: stored.source, metadata: parsed.data } : null +} + +async function writeArchive(db: D1Database, sandboxId: string, source: string, metadata: ArchiveMetadata, now: number) { + await db + .prepare( + `INSERT INTO sandbox_archives (sandbox_id, captured_at, source, metadata) VALUES (?, ?, ?, ?) + ON CONFLICT (sandbox_id) DO UPDATE SET captured_at = excluded.captured_at, source = excluded.source, metadata = excluded.metadata` + ) + .bind(sandboxId, now, source, JSON.stringify(metadata)) + .run() +} + +export async function deleteArchive(db: D1Database, sandboxId: string): Promise { + await db.prepare('DELETE FROM sandbox_archives WHERE sandbox_id = ?').bind(sandboxId).run() +} + +/** + * Archive = stop and keep the snapshot for a TTL. Git metadata is captured in the VM when it is + * running; a sleeping VM is not woken just to archive it — the desktop's cached git state is used. + */ +export async function archiveSandbox( + deps: LifecycleDeps, + row: SandboxRow, + request: Partial, + config: WorkspaceConfig = {} +): Promise { + if (row.status === 'archived') { + return row + } + if (row.status !== 'running' && row.status !== 'suspended') { + throw new LifecycleError('conflict', `sandbox_${row.status}`) + } + const captured = row.status === 'running' && row.session_id ? await captureArchiveMetadata(deps.provider, row.session_id) : null + const previous = await readArchive(deps.db, row.id) + const checkouts = captured?.checkouts ?? request.checkouts ?? previous?.metadata.checkouts ?? [] + const source = captured ? 'vm' : request.checkouts ? 'client' : previous ? previous.source : 'none' + await writeArchive( + deps.db, + row.id, + source, + { checkouts, pullRequests: request.pullRequests ?? previous?.metadata.pullRequests ?? [] }, + deps.now() + ) + const stopped = row.status === 'running' ? await suspendSandbox(deps, row) : row + const ttl = config.archiveTtlMs ?? DEFAULT_ARCHIVE_TTL_MS + await deps.provider + .updateSnapshotExpiration(stopped.provider_name, ttl + ARCHIVE_PROVIDER_GRACE_MS) + .catch((error: unknown) => console.log(JSON.stringify({ event: 'archive_ttl_update_failed', sandboxId: row.id, error: String(error).slice(0, 120) }))) + return claim(deps, stopped, { + status: 'archived', + ...NO_SESSION, + archived_at: deps.now(), + archive_expires_at: deps.now() + ttl + }) +} + +/** Unarchive restores the long-lived snapshot backstop and resumes the same disk. */ +export async function unarchiveSandbox(deps: LifecycleDeps, row: SandboxRow): Promise { + if (row.status !== 'archived') { + if (row.status === 'running' || row.status === 'suspended') { + return resumeSandbox(deps, row) + } + throw new LifecycleError('conflict', `sandbox_${row.status}`) + } + await requireStart(deps, row.account_id, row.user_id, row.machine_size) + await deps.provider.updateSnapshotExpiration( + row.provider_name, + deps.config.snapshotExpirationMs ?? DEFAULT_SNAPSHOT_EXPIRATION_MS + ) + const suspended = await claim(deps, row, { status: 'suspended', archived_at: null, archive_expires_at: null }) + return resumeSandbox(deps, suspended) +} + +/** Cron: archives past their TTL are destroyed (snapshots included). */ +export async function expiredArchives(db: D1Database, now: number): Promise { + const { results } = await db + .prepare(`SELECT * FROM sandboxes WHERE status = 'archived' AND archive_expires_at <= ? ORDER BY archive_expires_at LIMIT 50`) + .bind(now) + .all() + return results +} + +export type WorkspaceStatus = { + state: WorkspaceState + metrics: VmMetrics | null + /** Since orcad last became ready (this session). */ + uptimeMs: number | null + usage: { totalMs: number; centsPerHour: number; costCents: number } + archive: { archivedAt: number; expiresAt: number | null } | null + checkedAt: number +} + +function stateOf(row: SandboxRow, metrics: VmMetrics | null, probed: boolean): WorkspaceState { + switch (row.status) { + case 'creating': + return 'provisioning' + case 'resuming': + return 'waking' + case 'suspending': + case 'destroying': + return 'stopping' + case 'suspended': + return 'sleeping' + case 'archived': + return 'archived' + case 'running': + return probed && (!metrics || !metrics.orcadAlive) ? 'unresponsive' : 'ready' + default: + return 'failed' + } +} + +/** The status panel's one call. Metrics are read live from the VM only when it is running. */ +export async function workspaceStatus(deps: LifecycleDeps, row: SandboxRow, config: WorkspaceConfig = {}): Promise { + const probed = row.status === 'running' && row.session_id !== null + const metrics = probed ? await readVmMetrics(deps.provider, row.session_id!) : null + const now = deps.now() + const totalMs = await sandboxUsageMs(deps.db, row, now) + const centsPerHour = config.centsPerHour?.[row.machine_size] ?? DEFAULT_CENTS_PER_HOUR[row.machine_size] + return { + state: stateOf(row, metrics, probed), + metrics, + uptimeMs: row.status === 'running' && row.ready_at !== null ? Math.max(0, now - row.ready_at) : null, + usage: { totalMs, centsPerHour, costCents: Math.round((totalMs / 3_600_000) * centsPerHour) }, + archive: row.archived_at !== null ? { archivedAt: row.archived_at, expiresAt: row.archive_expires_at } : null, + checkedAt: now + } +} + +export { getSandbox } diff --git a/cloud/apps/sandbox-control/test/fake-vercel.ts b/cloud/apps/sandbox-control/test/fake-vercel.ts index 1a3f7ce73a5..c68457e345a 100644 --- a/cloud/apps/sandbox-control/test/fake-vercel.ts +++ b/cloud/apps/sandbox-control/test/fake-vercel.ts @@ -31,6 +31,10 @@ export type FakeVercel = SandboxProviderApi & { commandHandler?: (params: RunCommandParams, sessionId: string) => { exitCode: number; stdout: string; stderr: string } | undefined failStart: boolean failCreate: boolean + /** Answers `bash -c