mirror of
https://github.com/stablyai/orca.git
synced 2026-11-15 18:37:24 +00:00
Merge pull request #35 from nwparker/cloud/w2-lifecycle
feat(cloud): workspace lifecycle API (wake, restart, status, archive) + desktop service
This commit is contained in:
@@ -0,0 +1,18 @@
|
||||
-- Workspace lifecycle (wake, status, archive).
|
||||
-- orcad's public wss endpoint for the current session (not a secret: pairing still needs the code),
|
||||
-- and when orcad last became ready, so wake callers can report `readyAt` without a provider call.
|
||||
ALTER TABLE sandboxes ADD COLUMN orcad_endpoint TEXT;
|
||||
ALTER TABLE sandboxes ADD COLUMN ready_at INTEGER;
|
||||
-- Archive = stopped with the snapshot kept for a finite TTL; the cron destroys it after archive_expires_at.
|
||||
-- (`archived_at` itself comes from 0020_api_routines.sql, already applied on dev.)
|
||||
ALTER TABLE sandboxes ADD COLUMN archive_expires_at INTEGER;
|
||||
|
||||
-- What an archived workspace shows without waking: branch, last commits, changed files, file tree, PRs.
|
||||
-- Captured in the VM at archive (or sent by the desktop from its cache when the VM was already asleep).
|
||||
-- Bounded (see archive.ts); deleted with the sandbox.
|
||||
CREATE TABLE sandbox_archives (
|
||||
sandbox_id TEXT PRIMARY KEY REFERENCES sandboxes (id),
|
||||
captured_at INTEGER NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
metadata TEXT NOT NULL
|
||||
);
|
||||
@@ -129,6 +129,9 @@ function messageBucket(caller: ApiCaller): string {
|
||||
return caller.via === 'workspace' ? `msg:ws:${caller.sandboxId}` : `msg:${caller.accountId}:${caller.userId}`
|
||||
}
|
||||
|
||||
import { wakeSandbox } from '../workspace-lifecycle'
|
||||
import { parseWakeWait } from '../wire'
|
||||
|
||||
export function isApiPath(pathname: string): boolean {
|
||||
return /^\/v1\/(me|api-keys|settings|audit|projects|workspaces|routines|oauth-approvals|oauth-grants)(\/|$)/.test(pathname)
|
||||
}
|
||||
@@ -409,7 +412,21 @@ async function route(request: Request, url: URL, caller: ApiCaller, ctx: ApiCont
|
||||
return mutate('sleep', () => sleepWorkspace(life(), row))
|
||||
}
|
||||
if (sub === 'wake' && method === 'POST') {
|
||||
return mutate('wake', () => wakeWorkspace(life(), row))
|
||||
if (row.status === 'archived' || row.archived_at !== null) {
|
||||
// The public API's wake brings an archived workspace back.
|
||||
return mutate('wake', () => wakeWorkspace(life(), row))
|
||||
}
|
||||
// Every wake source shares W2's single-flight wake: idempotent, `?wait=30s` long-polls.
|
||||
requireScope(caller, 'workspaces:write')
|
||||
const started = performance.now()
|
||||
const answer = await wakeSandbox(life(), row, {
|
||||
// API callers expect the VM up when wake answers: wait the full 30 s unless `?wait=` says otherwise.
|
||||
waitMs: url.searchParams.has('wait') ? parseWakeWait(url, null) : 30_000,
|
||||
background: ctx.waitUntil
|
||||
})
|
||||
await auditCaller(ctx.db, caller, { action: 'workspace.wake', targetType: 'workspace', targetId: row.id, detail: { ms: Math.round(performance.now() - started), state: answer.state } }, ctx.now())
|
||||
const fresh = (await getWorkspace(ctx.db, caller.accountId, id)) ?? row
|
||||
return json({ workspace: workspaceView(fresh), ...answer })
|
||||
}
|
||||
if (sub === 'archive' && method === 'POST') {
|
||||
return mutate('archive', () => archiveWorkspace(life(), row))
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import type { MachineSize } from '../entitlements'
|
||||
import { createSandbox, destroySandbox, resumeSandbox, suspendSandbox, type LifecycleDeps } from '../lifecycle'
|
||||
import { getSandbox, type SandboxRow } from '../store'
|
||||
import { archiveSandbox, unarchiveSandbox } from '../workspace-lifecycle'
|
||||
import { ApiError } from './caller'
|
||||
import { randomId } from './keys'
|
||||
|
||||
@@ -63,6 +64,8 @@ export function apiStatus(row: Pick<SandboxRow, 'status'>): string {
|
||||
running: 'running',
|
||||
suspending: 'sleeping',
|
||||
suspended: 'sleeping',
|
||||
// The API reports archive separately (`archived: true`); the VM itself is asleep.
|
||||
archived: 'sleeping',
|
||||
resuming: 'waking',
|
||||
destroying: 'destroying',
|
||||
destroyed: 'destroyed',
|
||||
@@ -171,19 +174,21 @@ export async function sleepWorkspace(life: LifecycleDeps, row: WorkspaceRow): Pr
|
||||
}
|
||||
|
||||
export async function wakeWorkspace(life: LifecycleDeps, row: WorkspaceRow): Promise<WorkspaceRow> {
|
||||
const ready = await resumeSandbox(life, row)
|
||||
// Waking an archived workspace brings it back (restores the snapshot TTL; boot clears archived_at).
|
||||
const ready = row.status === 'archived' ? await unarchiveSandbox(life, row) : await resumeSandbox(life, row)
|
||||
if (row.archived_at !== null) {
|
||||
// Waking an archived workspace brings it back.
|
||||
await life.db.prepare('UPDATE sandboxes SET archived_at = NULL WHERE id = ?').bind(row.id).run()
|
||||
}
|
||||
return (await getSandbox(life.db, ready.row.id)) as WorkspaceRow
|
||||
}
|
||||
|
||||
/** Archive = sleep + out of default lists. The disk is kept until retention or destroy. */
|
||||
/**
|
||||
* Archive = stop and keep the snapshot for a TTL, out of default lists (W2's archiveSandbox:
|
||||
* repo metadata captured in the VM, provider snapshot expiry shortened, cron deletes after the TTL).
|
||||
*/
|
||||
export async function archiveWorkspace(life: LifecycleDeps, row: WorkspaceRow): Promise<WorkspaceRow> {
|
||||
const slept = row.status === 'running' ? await sleepWorkspace(life, row) : row
|
||||
await life.db.prepare('UPDATE sandboxes SET archived_at = COALESCE(archived_at, ?) WHERE id = ?').bind(life.now(), row.id).run()
|
||||
return (await getSandbox(life.db, slept.id)) as WorkspaceRow
|
||||
const archived = await archiveSandbox(life, row, {})
|
||||
return (await getSandbox(life.db, archived.id)) as WorkspaceRow
|
||||
}
|
||||
|
||||
export async function destroyWorkspace(life: LifecycleDeps, row: WorkspaceRow): Promise<WorkspaceRow> {
|
||||
|
||||
@@ -104,4 +104,8 @@ export type Env = {
|
||||
EGRESS_EXTRA_HOSTS?: string
|
||||
/** GitHub App webhook secret (secret): verifies X-Hub-Signature-256 on /v1/github/webhook. */
|
||||
GITHUB_WEBHOOK_SECRET?: string
|
||||
/** Archived workspaces are destroyed this many days after archive (default 30). */
|
||||
ARCHIVE_TTL_DAYS?: string
|
||||
/** Status-panel cost estimate, cents per running hour by machine size: {"standard":75,"large":150}. */
|
||||
COST_CENTS_PER_HOUR?: string
|
||||
}
|
||||
|
||||
@@ -5,6 +5,13 @@
|
||||
// GET /v1/sandboxes/:id -> {sandbox}
|
||||
// POST /v1/sandboxes/:id/suspend | /resume | /rollover -> {sandbox[, connection]}
|
||||
// POST /v1/sandboxes/:id/lease {extendMs} -> {sandbox}
|
||||
// POST /v1/sandboxes/:id/wake[?wait=30s] -> {state, endpoint, readyAt, sandboxId} (idempotent, single-flight;
|
||||
// the public API's POST /v1/workspaces/:id/wake answers the same fields plus `workspace`)
|
||||
// POST /v1/sandboxes/:id/restart -> {sandbox, connection}
|
||||
// GET /v1/sandboxes/:id/status -> {sandbox, status} (live metrics, cost)
|
||||
// POST /v1/sandboxes/:id/archive {pullRequests?, checkouts?} -> {sandbox}
|
||||
// GET /v1/sandboxes/:id/archive -> {archive}
|
||||
// POST /v1/sandboxes/:id/unarchive -> {sandbox, connection}
|
||||
// DELETE /v1/sandboxes/:id -> {sandbox}
|
||||
// GET /v1/secrets?scope= | PUT, DELETE /v1/secrets/:scope/:NAME (see src/secrets.ts)
|
||||
// POST /v1/github/connect -> {installUrl, authorizeUrl} (bind a GitHub App installation to the account)
|
||||
@@ -12,6 +19,7 @@
|
||||
// Browser (GitHub redirects): GET /v1/github/setup, GET /v1/github/callback
|
||||
// Operator (OPERATOR_TOKEN): /v1/internal/* (see operator.ts)
|
||||
// Vercel network-policy forwardURL target (Vercel sandbox OIDC): /v1/egress/proxy
|
||||
import { z } from 'zod'
|
||||
import { maintainPool, poolConfigFromEnv } from './pool'
|
||||
import { authenticateDesktop } from './auth'
|
||||
import { authenticateApi } from './api/caller'
|
||||
@@ -68,9 +76,27 @@ import { ProviderError, type SandboxProviderApi } from './sandbox-provider'
|
||||
import { VmCommandError } from './sandbox-vm-commands'
|
||||
import { getSandbox, getSandboxByProviderName, listAccountSandboxes, type SandboxRow } from './store'
|
||||
import { createVercelSandboxApi } from './vercel-sandbox-api'
|
||||
import { CreateRequestSchema, LeaseRequestSchema, readyView, sandboxView } from './wire'
|
||||
import { CreateRequestSchema, LeaseRequestSchema, parseWakeWait, readyView, sandboxView } from './wire'
|
||||
import { ArchiveRequestSchema } from './sandbox-vm-inspect'
|
||||
import {
|
||||
archiveSandbox,
|
||||
readArchive,
|
||||
restartSandbox,
|
||||
unarchiveSandbox,
|
||||
wakeSandbox,
|
||||
workspaceStatus,
|
||||
type WorkspaceConfig
|
||||
} from './workspace-lifecycle'
|
||||
|
||||
const MAX_JSON_BYTES = 16 * 1024
|
||||
/** Archive bodies carry the desktop's cached repo state (file trees) when the VM is already asleep. */
|
||||
const MAX_ARCHIVE_JSON_BYTES = 256 * 1024
|
||||
const CentsPerHourSchema = z
|
||||
.object({ standard: z.number().int().nonnegative(), large: z.number().int().nonnegative() })
|
||||
.partial()
|
||||
.strict()
|
||||
/** Desktops that understand `archived` say so; older strict clients see it as `suspended`. */
|
||||
const FEATURES_HEADER = 'x-orca-sandbox-features'
|
||||
/** A pasted .env file: up to 100 values. */
|
||||
const MAX_BATCH_JSON_BYTES = 160 * 1024
|
||||
|
||||
@@ -91,6 +117,25 @@ export type Deps = {
|
||||
waitUntil?: (work: Promise<unknown>) => void
|
||||
/** Route groups mounted on the public API (routines, see src/routines). */
|
||||
apiExtensions?: ApiExtension[]
|
||||
/** Long-poll sleep (tests advance a fake clock). */
|
||||
sleep?: (ms: number) => Promise<void>
|
||||
}
|
||||
|
||||
function workspaceConfig(env: Env, deps: Deps): WorkspaceConfig {
|
||||
const days = Number(env.ARCHIVE_TTL_DAYS)
|
||||
let centsPerHour: WorkspaceConfig['centsPerHour']
|
||||
try {
|
||||
const parsed = CentsPerHourSchema.safeParse(env.COST_CENTS_PER_HOUR ? JSON.parse(env.COST_CENTS_PER_HOUR) : undefined)
|
||||
centsPerHour = parsed.success ? parsed.data : undefined
|
||||
} catch {
|
||||
centsPerHour = undefined
|
||||
}
|
||||
return {
|
||||
archiveTtlMs: Number.isFinite(days) && days >= 1 ? Math.floor(days) * 86_400_000 : undefined,
|
||||
centsPerHour,
|
||||
background: deps.waitUntil,
|
||||
sleep: deps.sleep
|
||||
}
|
||||
}
|
||||
|
||||
/** The public origin: PUBLIC_API_URL when set, else the request's own origin. */
|
||||
@@ -565,7 +610,10 @@ export async function handleRequest(request: Request, env: Env, deps: Deps = {})
|
||||
}
|
||||
return response
|
||||
}
|
||||
const match = /^\/v1\/sandboxes(?:\/([A-Za-z0-9_]+)(?:\/(suspend|resume|lease|rollover|env\/refresh))?)?$/.exec(url.pathname)
|
||||
const match =
|
||||
/^\/v1\/sandboxes(?:\/([A-Za-z0-9_]+)(?:\/(suspend|resume|lease|rollover|wake|restart|status|archive|unarchive|env\/refresh))?)?$/.exec(
|
||||
url.pathname
|
||||
)
|
||||
if (!match) {
|
||||
return fail('not_found', 404)
|
||||
}
|
||||
@@ -578,11 +626,13 @@ export async function handleRequest(request: Request, env: Env, deps: Deps = {})
|
||||
return fail('not_configured', 503)
|
||||
}
|
||||
const [, id, action] = match
|
||||
const archivedAware = (request.headers.get(FEATURES_HEADER) ?? '').split(',').map((f) => f.trim()).includes('archive')
|
||||
const view = (row: SandboxRow) => sandboxView(row, archivedAware)
|
||||
if (!id) {
|
||||
if (request.method === 'GET') {
|
||||
// Routine runs and other machine-made workspaces stay out of the desktop's list.
|
||||
const rows = (await listAccountSandboxes(env.SANDBOX_DB, caller.accountId)).filter((row) => (row as { hidden?: number }).hidden !== 1)
|
||||
return json({ sandboxes: rows.map(sandboxView) })
|
||||
return json({ sandboxes: rows.map(view) })
|
||||
}
|
||||
if (request.method !== 'POST') {
|
||||
return fail('not_found', 404)
|
||||
@@ -603,11 +653,11 @@ export async function handleRequest(request: Request, env: Env, deps: Deps = {})
|
||||
const route = `${request.method} ${action ?? ''}`
|
||||
switch (route) {
|
||||
case 'GET ':
|
||||
return json({ sandbox: sandboxView(row) })
|
||||
return json({ sandbox: view(row) })
|
||||
case 'DELETE ':
|
||||
return json({ sandbox: sandboxView(await destroySandbox(life, row)) })
|
||||
return json({ sandbox: view(await destroySandbox(life, row)) })
|
||||
case 'POST suspend':
|
||||
return json({ sandbox: sandboxView(await suspendSandbox(life, row)) })
|
||||
return json({ sandbox: view(await suspendSandbox(life, row)) })
|
||||
case 'POST resume': {
|
||||
const ready = await resumeSandbox(life, row)
|
||||
return readyResponse(ready)
|
||||
@@ -616,6 +666,29 @@ export async function handleRequest(request: Request, env: Env, deps: Deps = {})
|
||||
const ready = await rolloverSandbox(life, row)
|
||||
return readyResponse(ready)
|
||||
}
|
||||
case 'POST wake':
|
||||
return json(await wakeSandbox(life, row, { waitMs: parseWakeWait(url, await readJson(request)), ...workspaceConfig(env, deps) }))
|
||||
case 'POST restart': {
|
||||
// User-triggered and unbounded otherwise: shares the per-account start rate limit.
|
||||
await consumeCreateToken(life.db, life.config.guardrails!, caller.accountId, life.now())
|
||||
const ready = await restartSandbox(life, row)
|
||||
return json(readyView(ready.row, ready.connection))
|
||||
}
|
||||
case 'GET status':
|
||||
return json({ sandbox: view(row), status: await workspaceStatus(life, row, workspaceConfig(env, deps)) })
|
||||
case 'POST archive': {
|
||||
const body = ArchiveRequestSchema.safeParse(await readJson(request, MAX_ARCHIVE_JSON_BYTES))
|
||||
if (!body.success) {
|
||||
return fail('invalid_request', 400)
|
||||
}
|
||||
return json({ sandbox: view(await archiveSandbox(life, row, body.data, workspaceConfig(env, deps))) })
|
||||
}
|
||||
case 'GET archive':
|
||||
return json({ archive: await readArchive(env.SANDBOX_DB, row.id) })
|
||||
case 'POST unarchive': {
|
||||
const ready = await unarchiveSandbox(life, row)
|
||||
return json(readyView(ready.row, ready.connection))
|
||||
}
|
||||
case 'POST env/refresh':
|
||||
return json({ applied: await refreshSandboxEnv(life, row) })
|
||||
case 'POST lease': {
|
||||
@@ -623,7 +696,7 @@ export async function handleRequest(request: Request, env: Env, deps: Deps = {})
|
||||
if (!body.success) {
|
||||
return fail('invalid_request', 400)
|
||||
}
|
||||
return json({ sandbox: sandboxView(await renewLease(life, row, body.data.extendMs)) })
|
||||
return json({ sandbox: view(await renewLease(life, row, body.data.extendMs)) })
|
||||
}
|
||||
default:
|
||||
return fail('not_found', 404)
|
||||
|
||||
@@ -201,7 +201,7 @@ export function sessionFields(deps: LifecycleDeps, state: ProviderSandboxState):
|
||||
}
|
||||
}
|
||||
|
||||
async function claim(deps: LifecycleDeps, row: SandboxRow, patch: Partial<SandboxRow>): Promise<SandboxRow> {
|
||||
export async function claim(deps: LifecycleDeps, row: SandboxRow, patch: Partial<SandboxRow>): Promise<SandboxRow> {
|
||||
const next = await transition(deps.db, row, patch, deps.now())
|
||||
if (!next) {
|
||||
throw new LifecycleError('conflict', 'concurrent_transition')
|
||||
@@ -213,7 +213,7 @@ export function guardrails(deps: LifecycleDeps): GuardrailConfig {
|
||||
return deps.config.guardrails ?? DEFAULT_GUARDRAILS
|
||||
}
|
||||
|
||||
async function requireStart(deps: LifecycleDeps, accountId: string, userId: string, machineSize: MachineSize) {
|
||||
export async function requireStart(deps: LifecycleDeps, accountId: string, userId: string, machineSize: MachineSize) {
|
||||
const [, runningSandboxes, runningSandboxesForUser] = await Promise.all([
|
||||
assertStartsOpen(deps.db, guardrails(deps)),
|
||||
countActiveSandboxes(deps.db, accountId),
|
||||
@@ -238,6 +238,11 @@ async function requireStart(deps: LifecycleDeps, accountId: string, userId: stri
|
||||
})
|
||||
}
|
||||
|
||||
function orcadEndpoint(state: ProviderSandboxState): string | null {
|
||||
const route = state.routes.find((candidate) => candidate.port === ORCAD_PORT)
|
||||
return route ? route.url.replace(/^https:/, 'wss:') : null
|
||||
}
|
||||
|
||||
async function boot(
|
||||
deps: LifecycleDeps,
|
||||
row: SandboxRow,
|
||||
@@ -281,6 +286,10 @@ async function boot(
|
||||
const running = await claim(deps, row, {
|
||||
status: 'running',
|
||||
last_error: null,
|
||||
orcad_endpoint: orcadEndpoint(state),
|
||||
ready_at: deps.now(),
|
||||
archived_at: null,
|
||||
archive_expires_at: null,
|
||||
retention_warned_at: null,
|
||||
github_token_expires_at: git?.expiresAt ?? null,
|
||||
egress_secret_hosts: routed.egress_secret_hosts,
|
||||
@@ -483,9 +492,13 @@ export async function failAndRelease(deps: LifecycleDeps, row: SandboxRow): Prom
|
||||
}
|
||||
}
|
||||
|
||||
async function stopForSuspend(deps: LifecycleDeps, row: SandboxRow): Promise<SandboxRow> {
|
||||
export async function stopForSuspend(
|
||||
deps: LifecycleDeps,
|
||||
row: SandboxRow,
|
||||
options: { drainSeconds?: number } = {}
|
||||
): Promise<SandboxRow> {
|
||||
if (row.session_id) {
|
||||
await drainOrcad(deps.provider, row.session_id)
|
||||
await drainOrcad(deps.provider, row.session_id, options.drainSeconds)
|
||||
const stopped = await deps.provider.stopSession(row.session_id)
|
||||
if (stopped.snapshotId) {
|
||||
await recordSnapshots(deps.db, row.id, [stopped.snapshotId], deps.now())
|
||||
@@ -504,7 +517,8 @@ export async function markProviderStopped(deps: LifecycleDeps, row: SandboxRow):
|
||||
}
|
||||
|
||||
export async function suspendSandbox(deps: LifecycleDeps, row: SandboxRow): Promise<SandboxRow> {
|
||||
if (row.status === 'suspended') {
|
||||
// Archived is already stopped (the desktop's Sleep runs after or before an archive).
|
||||
if (row.status === 'suspended' || row.status === 'archived') {
|
||||
return row
|
||||
}
|
||||
if (row.status !== 'running') {
|
||||
@@ -521,10 +535,7 @@ export async function resumeSandbox(deps: LifecycleDeps, row: SandboxRow): Promi
|
||||
const state = await deps.provider.getSandbox(row.provider_name, { resume: false })
|
||||
if (state?.session.status === 'running') {
|
||||
// Idempotent: a running, serving sandbox answers at once (desktop wake-on-open after a restart).
|
||||
const connection =
|
||||
(await runningOrcad(deps.provider, state.session.id)) ??
|
||||
(await startOrcad(deps.provider, state, { sandboxId: row.id }))
|
||||
return { row, connection }
|
||||
return adoptRunning(deps, row, state)
|
||||
}
|
||||
// The provider ended the session (lease lapsed): fall through to a real resume.
|
||||
row = await markProviderStopped(deps, row)
|
||||
@@ -537,7 +548,37 @@ export async function resumeSandbox(deps: LifecycleDeps, row: SandboxRow): Promi
|
||||
return wake(deps, resuming)
|
||||
}
|
||||
|
||||
async function wake(deps: LifecycleDeps, row: SandboxRow): Promise<Ready> {
|
||||
/**
|
||||
* A running session whose orcad may not be up (a retried call, or a row the cron repaired after a
|
||||
* wake died mid-way): start orcad (idempotent per boot) and record its endpoint if missing.
|
||||
*/
|
||||
export async function adoptRunning(deps: LifecycleDeps, row: SandboxRow, state: ProviderSandboxState): Promise<Ready> {
|
||||
const connection =
|
||||
(await runningOrcad(deps.provider, state.session.id)) ?? (await startOrcad(deps.provider, state, { sandboxId: row.id }))
|
||||
if (row.orcad_endpoint && row.ready_at !== null) {
|
||||
return { row, connection }
|
||||
}
|
||||
const updated = await transition(deps.db, row, { orcad_endpoint: orcadEndpoint(state), ready_at: deps.now() }, deps.now())
|
||||
return { row: updated ?? row, connection }
|
||||
}
|
||||
|
||||
/**
|
||||
* The VM is running but the boot that should have followed never finished (its Worker was cut
|
||||
* off, or the cron adopted the session): run the full boot — secrets, repo token, egress policy,
|
||||
* orcad — against the live session, exactly as a wake would.
|
||||
*/
|
||||
export async function bootLiveSession(deps: LifecycleDeps, row: SandboxRow, state: ProviderSandboxState): Promise<Ready> {
|
||||
const timer = phaseTimer(deps.now)
|
||||
const prep = await timer.time('prepare', () => prepareBoot(deps, row))
|
||||
return boot(deps, row, state, prep, {
|
||||
policyApplied: false,
|
||||
timer,
|
||||
// Re-booting orcad in an already-metered session keeps its usage cursor (no re-billing).
|
||||
billFrom: row.status === 'running' && row.usage_settled_at !== null ? row.usage_settled_at : undefined
|
||||
})
|
||||
}
|
||||
|
||||
export async function wake(deps: LifecycleDeps, row: SandboxRow): Promise<Ready> {
|
||||
const timer = phaseTimer(deps.now)
|
||||
// The provider boot is the long pole; secrets and the repo token resolve alongside it.
|
||||
const [state, prep] = await Promise.all([
|
||||
@@ -555,7 +596,8 @@ async function wake(deps: LifecycleDeps, row: SandboxRow): Promise<Ready> {
|
||||
} catch (error) {
|
||||
// Leave it resumable: the disk is intact, only this boot failed.
|
||||
const current = await getSandbox(deps.db, row.id)
|
||||
if (current?.status === 'resuming') {
|
||||
// Only undo our own claim: a Restart that took the row over owns it now.
|
||||
if (current?.status === 'resuming' && current.generation === row.generation) {
|
||||
await transition(deps.db, current, { status: 'suspended', ...NO_SESSION, last_error: 'resume_failed' }, deps.now())
|
||||
await deps.hooks?.afterStop?.(current).catch(() => undefined)
|
||||
}
|
||||
@@ -615,6 +657,8 @@ export async function destroySandbox(deps: LifecycleDeps, row: SandboxRow): Prom
|
||||
throw new LifecycleError('provider_unavailable', 'snapshot_cleanup_incomplete')
|
||||
}
|
||||
await markSnapshotsDeleted(deps.db, row.id, deps.now())
|
||||
// Archive metadata (repo state, file tree) goes with the workspace.
|
||||
await deps.db.prepare('DELETE FROM sandbox_archives WHERE sandbox_id = ?').bind(row.id).run()
|
||||
const destroyed = await claim(deps, destroying, {
|
||||
status: 'destroyed',
|
||||
...NO_SESSION,
|
||||
|
||||
@@ -16,6 +16,7 @@ import { killSwitchOn } from './guardrails'
|
||||
import { drainAll, housekeeping, openHold, type HousekeepingSummary } from './retention'
|
||||
import { listByStatus, transition, type SandboxRow } from './store'
|
||||
import { flushUsage, queueUsage } from './usage'
|
||||
import { expiredArchives } from './workspace-lifecycle'
|
||||
|
||||
const STUCK_MS = 15 * 60_000
|
||||
/** The desktop rolls over at the deadline; the cron steps in only if it did not (provider cap is +10 min). */
|
||||
@@ -29,6 +30,7 @@ export type ReconcileSummary = Record<
|
||||
| 'leaseLapsed'
|
||||
| 'released'
|
||||
| 'repaired'
|
||||
| 'archivesExpired'
|
||||
| 'billingErrors',
|
||||
number
|
||||
> & { housekeeping?: HousekeepingSummary }
|
||||
@@ -42,6 +44,7 @@ export async function reconcile(deps: LifecycleDeps): Promise<ReconcileSummary>
|
||||
leaseLapsed: 0,
|
||||
released: 0,
|
||||
repaired: 0,
|
||||
archivesExpired: 0,
|
||||
billingErrors: 0
|
||||
}
|
||||
const polled = await guard(async () => {
|
||||
@@ -81,6 +84,12 @@ export async function reconcile(deps: LifecycleDeps): Promise<ReconcileSummary>
|
||||
}
|
||||
})
|
||||
}
|
||||
for (const row of await expiredArchives(deps.db, deps.now())) {
|
||||
await guard(async () => {
|
||||
await destroySandbox(deps, row)
|
||||
summary.archivesExpired++
|
||||
})
|
||||
}
|
||||
await flushUsage(deps.db, deps.entitlements, deps.now())
|
||||
await guard(async () => {
|
||||
const housekept = await housekeeping(deps)
|
||||
|
||||
@@ -89,6 +89,8 @@ export type SandboxProviderApi = {
|
||||
extendTimeout(sessionId: string, durationMs: number): Promise<ProviderSession>
|
||||
updateNetworkPolicy(sessionId: string, policy: NetworkPolicy): Promise<void>
|
||||
runCommand(sessionId: string, params: RunCommandParams): Promise<CommandResult>
|
||||
/** Provider-side snapshot expiry after last use (archive TTL backstop). */
|
||||
updateSnapshotExpiration(name: string, expirationMs: number): Promise<void>
|
||||
/** A small text file from the VM (files API: not kept as command logs). Null when absent. */
|
||||
readFile(sessionId: string, path: string): Promise<string | null>
|
||||
/** Resolves when already gone. */
|
||||
|
||||
@@ -149,11 +149,13 @@ export async function refreshSessionEnv(
|
||||
}
|
||||
|
||||
/** Best effort: orcad turns working chats into durable restart offers before the VM stops. */
|
||||
export async function drainOrcad(provider: SandboxProviderApi, sessionId: string): Promise<boolean> {
|
||||
export async function drainOrcad(provider: SandboxProviderApi, sessionId: string, waitSeconds?: number): Promise<boolean> {
|
||||
try {
|
||||
const result = await provider.runCommand(sessionId, {
|
||||
command: DRAIN_COMMAND,
|
||||
timeoutMs: DRAIN_TIMEOUT_MS
|
||||
// The script's own default (280s) leaves room inside DRAIN_TIMEOUT_MS; Restart passes a short wait.
|
||||
...(waitSeconds ? { args: [String(Math.max(1, Math.floor(waitSeconds)))] } : {}),
|
||||
timeoutMs: waitSeconds ? (waitSeconds + 20) * 1000 : DRAIN_TIMEOUT_MS
|
||||
})
|
||||
return result.exitCode === 0
|
||||
} catch (error) {
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
// Read-only probes the control plane runs in a VM: live metrics (status panel, OOM banner) and the
|
||||
// git metadata an archived workspace shows. Fixed scripts compiled into the Worker (never user
|
||||
// input), so they work on every golden image without a rebuild. Output is one JSON line, parsed strictly.
|
||||
// Bash `\${...}` is escaped in these templates; `${CONST}` interpolates the TypeScript constants.
|
||||
import { z } from 'zod'
|
||||
import { ProviderError, type SandboxProviderApi } from './sandbox-provider'
|
||||
|
||||
const PROJECT = '/vercel/sandbox/project'
|
||||
|
||||
/**
|
||||
* CPU from two /proc/stat samples 250 ms apart, memory from /proc/meminfo, disk for the project
|
||||
* volume, uptime, and OOM: the kernel's oom_kill counter (/proc/vmstat, readable unprivileged) plus
|
||||
* the last killed process name from dmesg when sudo allows it.
|
||||
*/
|
||||
export const METRICS_SCRIPT = String.raw`
|
||||
set -u
|
||||
cpu() { awk '/^cpu /{idle=$5+$6; t=0; for(i=2;i<=NF;i++) t+=$i; print t, idle}' /proc/stat; }
|
||||
read t1 i1 < <(cpu); sleep 0.25; read t2 i2 < <(cpu)
|
||||
dt=$((t2-t1)); di=$((i2-i1)); [ "$dt" -gt 0 ] || dt=1
|
||||
cpu_pct=$(( (100*(dt-di)) / dt ))
|
||||
mem_total=$(awk '/^MemTotal:/{print $2*1024}' /proc/meminfo)
|
||||
mem_avail=$(awk '/^MemAvailable:/{print $2*1024}' /proc/meminfo)
|
||||
read disk_total disk_used < <(df -B1 --output=size,used ${PROJECT} / 2>/dev/null | sed -n 2p)
|
||||
uptime_s=$(awk '{printf "%d", $1}' /proc/uptime)
|
||||
oom=$(awk '/^oom_kill /{print $2}' /proc/vmstat 2>/dev/null); oom=\${oom:-0}
|
||||
victim=$(sudo -n dmesg 2>/dev/null | grep -oE 'Killed process [0-9]+ \([^)]{1,64}\)' | tail -n 1 | sed -E 's/.*\((.*)\)/\1/' | tr -cd 'A-Za-z0-9._+-' | head -c 64)
|
||||
orcad=false; pid=$(cat "$HOME/.orca-sandbox/orcad.pid" 2>/dev/null || echo 0); [ "$pid" -gt 1 ] 2>/dev/null && kill -0 "$pid" 2>/dev/null && orcad=true
|
||||
load=$(awk '{print $1}' /proc/loadavg)
|
||||
printf '{"cpuPercent":%d,"cpus":%d,"load1":%s,"memTotalBytes":%d,"memUsedBytes":%d,"diskTotalBytes":%d,"diskUsedBytes":%d,"uptimeSeconds":%d,"oomKills":%d,"lastOomVictim":%s,"orcadAlive":%s}\n' \
|
||||
"$cpu_pct" "$(nproc)" "$load" "$mem_total" "$((mem_total-mem_avail))" "\${disk_total:-0}" "\${disk_used:-0}" "$uptime_s" "$oom" \
|
||||
"$( [ -n "$victim" ] && printf '"%s"' "$victim" || printf null )" "$orcad"
|
||||
`.replaceAll('\\${', '${')
|
||||
|
||||
export const VmMetricsSchema = z
|
||||
.object({
|
||||
cpuPercent: z.number().int().min(0).max(100),
|
||||
cpus: z.number().int().min(1),
|
||||
load1: z.number().min(0),
|
||||
memTotalBytes: z.number().int().min(0),
|
||||
memUsedBytes: z.number().int().min(0),
|
||||
diskTotalBytes: z.number().int().min(0),
|
||||
diskUsedBytes: z.number().int().min(0),
|
||||
uptimeSeconds: z.number().int().min(0),
|
||||
oomKills: z.number().int().min(0),
|
||||
lastOomVictim: z.string().max(64).nullable(),
|
||||
orcadAlive: z.boolean()
|
||||
})
|
||||
.strict()
|
||||
|
||||
export type VmMetrics = z.infer<typeof VmMetricsSchema>
|
||||
|
||||
const MAX_TREE = 2000
|
||||
const MAX_CHANGED = 300
|
||||
const MAX_COMMITS = 20
|
||||
/** Under the provider's 256 KiB command-output cap, with room for the stream framing. */
|
||||
export const MAX_OUTPUT_BYTES = 200 * 1024
|
||||
|
||||
/**
|
||||
* Archive metadata for every git checkout in the project (the repo plus its worktrees): branch,
|
||||
* head, upstream ahead/behind, last commits, uncommitted changes and a capped file tree. Built
|
||||
* with git's own porcelain output, JSON-escaped by node (always present in the image).
|
||||
*/
|
||||
export const ARCHIVE_SCRIPT = String.raw`
|
||||
set -u
|
||||
cd ${PROJECT} 2>/dev/null || { echo '{"checkouts":[]}'; exit 0; }
|
||||
node="$(find /opt/orca/runtime/runtimes -type f -name node 2>/dev/null | head -n 1)"; node=\${node:-node}
|
||||
git worktree list --porcelain 2>/dev/null | awk '/^worktree /{print substr($0,10)}' > /tmp/orca-archive-wt || true
|
||||
[ -s /tmp/orca-archive-wt ] || echo ${PROJECT} > /tmp/orca-archive-wt
|
||||
"$node" -e '
|
||||
const { execFileSync } = require("child_process")
|
||||
const fs = require("fs")
|
||||
const run = (cwd, args) => { try { return execFileSync("git", args, { cwd, encoding: "utf8", maxBuffer: 8 << 20, timeout: 8000 }).trim() } catch { return "" } }
|
||||
const lines = (s) => s ? s.split("\n") : []
|
||||
const checkouts = fs.readFileSync("/tmp/orca-archive-wt", "utf8").split("\n").filter(Boolean).slice(0, 20).map((dir) => {
|
||||
const counts = run(dir, ["rev-list", "--left-right", "--count", "@{upstream}...HEAD"]).split(/\s+/)
|
||||
const tree = lines(run(dir, ["ls-files", "--cached", "--others", "--exclude-standard"]))
|
||||
return {
|
||||
path: dir,
|
||||
branch: run(dir, ["rev-parse", "--abbrev-ref", "HEAD"]) || null,
|
||||
head: run(dir, ["rev-parse", "HEAD"]) || null,
|
||||
upstream: run(dir, ["rev-parse", "--abbrev-ref", "@{upstream}"]) || null,
|
||||
behind: counts.length === 2 ? Number(counts[0]) : null,
|
||||
ahead: counts.length === 2 ? Number(counts[1]) : null,
|
||||
commits: lines(run(dir, ["log", "-n", "${MAX_COMMITS}", "--format=%H%x1f%an%x1f%at%x1f%s"])).map((l) => {
|
||||
const [sha, author, at, subject] = l.split("\x1f")
|
||||
return { sha, author: (author || "").slice(0, 120), at: Number(at) * 1000, subject: (subject || "").slice(0, 300) }
|
||||
}),
|
||||
changes: lines(run(dir, ["status", "--porcelain=v1", "-uall"])).slice(0, ${MAX_CHANGED}).map((l) => ({ status: l.slice(0, 2).trim() || "?", path: l.slice(3).slice(0, 1024) })),
|
||||
diffStat: run(dir, ["diff", "HEAD", "--shortstat"]).slice(0, 200) || null,
|
||||
fileCount: tree.length,
|
||||
files: tree.slice(0, ${MAX_TREE}).map((f) => f.slice(0, 1024))
|
||||
}
|
||||
})
|
||||
// The provider caps command output (256 KiB): keep the answer well under it. The main checkout keeps
|
||||
// its file tree first; trees are trimmed, then dropped, until the JSON fits.
|
||||
const kept = checkouts.filter((c) => c.head || c.branch)
|
||||
const fits = () => Buffer.byteLength(JSON.stringify({ checkouts: kept })) <= ${MAX_OUTPUT_BYTES}
|
||||
for (let i = kept.length - 1; i >= 0 && !fits(); i--) { kept[i].files = i === 0 ? kept[i].files.slice(0, 500) : [] }
|
||||
for (let i = kept.length - 1; i >= 0 && !fits(); i--) { kept[i].files = []; kept[i].changes = kept[i].changes.slice(0, 50) }
|
||||
while (kept.length > 1 && !fits()) kept.pop()
|
||||
process.stdout.write(JSON.stringify({ checkouts: kept }) + "\n")
|
||||
'
|
||||
`.replaceAll('\\${', '${')
|
||||
|
||||
const Commit = z.object({ sha: z.string().max(64), author: z.string().max(120), at: z.number(), subject: z.string().max(300) })
|
||||
const Checkout = z.object({
|
||||
path: z.string().max(4096),
|
||||
branch: z.string().max(255).nullable(),
|
||||
head: z.string().max(64).nullable(),
|
||||
upstream: z.string().max(255).nullable(),
|
||||
ahead: z.number().int().nullable(),
|
||||
behind: z.number().int().nullable(),
|
||||
commits: z.array(Commit).max(MAX_COMMITS),
|
||||
changes: z.array(z.object({ status: z.string().max(2), path: z.string().max(1024) })).max(MAX_CHANGED),
|
||||
diffStat: z.string().max(200).nullable(),
|
||||
fileCount: z.number().int().min(0),
|
||||
files: z.array(z.string().max(1024)).max(MAX_TREE)
|
||||
})
|
||||
const PullRequest = z.object({
|
||||
number: z.number().int().positive(),
|
||||
title: z.string().max(300),
|
||||
state: z.enum(['open', 'closed', 'merged', 'draft']),
|
||||
url: z.url({ protocol: /^https$/ }).max(2048),
|
||||
branch: z.string().max(255).optional()
|
||||
})
|
||||
|
||||
export const ArchiveMetadataSchema = z.object({
|
||||
checkouts: z.array(Checkout).max(20),
|
||||
pullRequests: z.array(PullRequest).max(50).optional()
|
||||
})
|
||||
export type ArchiveMetadata = z.infer<typeof ArchiveMetadataSchema>
|
||||
|
||||
/** What the desktop may send at archive: PRs it knows, and git state from its cache if the VM is asleep. */
|
||||
export const ArchiveRequestSchema = z
|
||||
.object({
|
||||
pullRequests: z.array(PullRequest).max(50).optional(),
|
||||
checkouts: z.array(Checkout).max(20).optional()
|
||||
})
|
||||
.strict()
|
||||
|
||||
async function runJson<T>(
|
||||
provider: SandboxProviderApi,
|
||||
sessionId: string,
|
||||
script: string,
|
||||
schema: z.ZodType<T>,
|
||||
timeoutMs: number
|
||||
): Promise<T | null> {
|
||||
try {
|
||||
const result = await provider.runCommand(sessionId, { command: 'bash', args: ['-c', script], timeoutMs })
|
||||
if (result.exitCode !== 0) {
|
||||
return null
|
||||
}
|
||||
const parsed = schema.safeParse(JSON.parse(result.stdout.trim().split('\n').at(-1) ?? ''))
|
||||
return parsed.success ? parsed.data : null
|
||||
} catch (error) {
|
||||
if (error instanceof ProviderError || error instanceof SyntaxError) {
|
||||
return null
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/** Null when the VM did not answer in time (the status panel shows "Unresponsive"). */
|
||||
export function readVmMetrics(provider: SandboxProviderApi, sessionId: string): Promise<VmMetrics | null> {
|
||||
return runJson(provider, sessionId, METRICS_SCRIPT, VmMetricsSchema, 8_000)
|
||||
}
|
||||
|
||||
export function captureArchiveMetadata(provider: SandboxProviderApi, sessionId: string): Promise<ArchiveMetadata | null> {
|
||||
return runJson(provider, sessionId, ARCHIVE_SCRIPT, ArchiveMetadataSchema, 30_000)
|
||||
}
|
||||
@@ -11,6 +11,8 @@ export type SandboxStatus =
|
||||
| 'destroying'
|
||||
| 'destroyed'
|
||||
| 'failed'
|
||||
/** Stopped with the snapshot kept for a TTL; unarchive resumes it, the cron destroys it after the TTL. */
|
||||
| 'archived'
|
||||
|
||||
export type SandboxRow = {
|
||||
id: string
|
||||
@@ -39,6 +41,12 @@ export type SandboxRow = {
|
||||
/** Comma list of custom model-provider hosts added to this session's egress allowlist. */
|
||||
egress_allow_hosts: string | null
|
||||
retention_warned_at: number | null
|
||||
/** orcad's public wss endpoint for the current session. */
|
||||
orcad_endpoint: string | null
|
||||
/** When orcad last became ready (wake callers report it as `readyAt`). */
|
||||
ready_at: number | null
|
||||
archived_at: number | null
|
||||
archive_expires_at: number | null
|
||||
}
|
||||
|
||||
export type SessionFields = Pick<
|
||||
@@ -46,7 +54,8 @@ export type SessionFields = Pick<
|
||||
'session_id' | 'session_started_at' | 'session_deadline_at' | 'lease_expires_at' | 'usage_settled_at'
|
||||
>
|
||||
|
||||
export const NO_SESSION: SessionFields = {
|
||||
export const NO_SESSION: SessionFields & Pick<SandboxRow, 'orcad_endpoint'> = {
|
||||
orcad_endpoint: null,
|
||||
session_id: null,
|
||||
session_started_at: null,
|
||||
session_deadline_at: null,
|
||||
@@ -139,7 +148,8 @@ type Patch = Partial<Omit<SandboxRow, 'id' | 'account_id' | 'generation' | 'crea
|
||||
const PATCHABLE = new Set<string>([
|
||||
'status', 'session_id', 'session_started_at', 'session_deadline_at', 'lease_expires_at',
|
||||
'usage_settled_at', 'last_error', 'destroyed_at', 'github_token_expires_at',
|
||||
'egress_secret_hosts', 'egress_allow_hosts', 'repo_url', 'retention_warned_at'
|
||||
'egress_secret_hosts', 'egress_allow_hosts', 'repo_url', 'retention_warned_at', 'orcad_endpoint', 'ready_at',
|
||||
'archived_at', 'archive_expires_at'
|
||||
])
|
||||
|
||||
/** Applies `patch` only if nobody moved the row since `expected` was read; null on a lost race. */
|
||||
@@ -188,3 +198,13 @@ export async function recordedSnapshotIds(db: D1Database, sandboxId: string): Pr
|
||||
.all<{ snapshot_id: string }>()
|
||||
return results.map((row) => row.snapshot_id)
|
||||
}
|
||||
|
||||
/** Usage so far for one sandbox: settled segments plus the open one (ms). */
|
||||
export async function sandboxUsageMs(db: D1Database, row: SandboxRow, now: number): Promise<number> {
|
||||
const settled = await db
|
||||
.prepare('SELECT COALESCE(SUM(ended_at - started_at), 0) AS ms FROM usage_outbox WHERE sandbox_id = ?')
|
||||
.bind(row.id)
|
||||
.first<{ ms: number }>()
|
||||
const open = row.status === 'running' && row.usage_settled_at !== null ? Math.max(0, now - row.usage_settled_at) : 0
|
||||
return (settled?.ms ?? 0) + open
|
||||
}
|
||||
|
||||
@@ -197,6 +197,14 @@ export function createVercelSandboxApi(
|
||||
return readCommandStream(response)
|
||||
},
|
||||
|
||||
async updateSnapshotExpiration(name, expirationMs) {
|
||||
// Archive shortens the provider-side backstop; unarchive restores it. Same fields as create.
|
||||
await request('PATCH', `/v2/sandboxes/${encodeURIComponent(name)}`, {
|
||||
query: { projectId: config.projectId },
|
||||
body: { snapshotExpiration: expirationMs, keepLastSnapshots: { count: 2, expiration: expirationMs } }
|
||||
})
|
||||
},
|
||||
|
||||
async readFile(sessionId, path) {
|
||||
try {
|
||||
const response = await request('POST', `/v2/sandboxes/sessions/${encodeURIComponent(sessionId)}/fs/read`, {
|
||||
|
||||
@@ -38,7 +38,8 @@ export const LeaseRequestSchema = z
|
||||
})
|
||||
.strict()
|
||||
|
||||
export function sandboxView(row: SandboxRow) {
|
||||
/** `archivedAware`: the caller's schema knows `archived`; older strict desktops see `suspended`. */
|
||||
export function sandboxView(row: SandboxRow, archivedAware = false) {
|
||||
const session =
|
||||
row.session_id !== null &&
|
||||
row.session_started_at !== null &&
|
||||
@@ -54,7 +55,7 @@ export function sandboxView(row: SandboxRow) {
|
||||
return {
|
||||
id: row.id,
|
||||
workspaceId: row.workspace_id,
|
||||
status: row.status,
|
||||
status: row.status === 'archived' && !archivedAware ? ('suspended' as const) : row.status,
|
||||
machineSize: row.machine_size,
|
||||
region: row.region,
|
||||
session,
|
||||
@@ -72,3 +73,18 @@ export function readyView(row: SandboxRow, connection: OrcadReady) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** `?wait=30s` / `?wait=30000ms` / `?wait=30`, or body `{wait: true}` (= the 30 s max). 0 when absent. */
|
||||
export function parseWakeWait(url: URL, body: unknown): number {
|
||||
const raw = url.searchParams.get('wait')
|
||||
if (raw !== null) {
|
||||
const match = /^(\d{1,6})(ms|s)?$/.exec(raw.trim())
|
||||
if (!match) {
|
||||
return 0
|
||||
}
|
||||
const value = Number(match[1])
|
||||
return Math.min(30_000, match[2] === 'ms' ? value : value * 1000)
|
||||
}
|
||||
const wait = (body as { wait?: unknown } | null)?.wait
|
||||
return wait === true ? 30_000 : typeof wait === 'number' && Number.isFinite(wait) ? Math.min(30_000, Math.max(0, wait)) : 0
|
||||
}
|
||||
|
||||
@@ -0,0 +1,371 @@
|
||||
// Workspace lifecycle on top of lifecycle.ts: wake (single-flight, long-poll; every wake source —
|
||||
// open, message, SSH, preview — goes through it), Restart for unresponsive VMs, archive/unarchive
|
||||
// with a snapshot TTL, and the status panel's live view (state, metrics, uptime, cost so far).
|
||||
import type { MachineSize } from './entitlements'
|
||||
import { GuardrailError, killSwitchOn } from './guardrails'
|
||||
import {
|
||||
DEFAULT_SNAPSHOT_EXPIRATION_MS,
|
||||
LifecycleError,
|
||||
bootLiveSession,
|
||||
claim,
|
||||
guardrails,
|
||||
markProviderStopped,
|
||||
requireStart,
|
||||
resumeSandbox,
|
||||
stopForSuspend,
|
||||
wake,
|
||||
type LifecycleDeps,
|
||||
type Ready
|
||||
} from './lifecycle'
|
||||
import {
|
||||
ArchiveMetadataSchema,
|
||||
captureArchiveMetadata,
|
||||
readVmMetrics,
|
||||
type ArchiveMetadata,
|
||||
type VmMetrics
|
||||
} from './sandbox-vm-inspect'
|
||||
import { NO_SESSION, getSandbox, sandboxUsageMs, transition, type SandboxRow } from './store'
|
||||
import { flushUsage } from './usage'
|
||||
|
||||
const DAY_MS = 86_400_000
|
||||
export const DEFAULT_ARCHIVE_TTL_MS = 30 * DAY_MS
|
||||
/** Provider backstop past Orca's own archive TTL, so Orca's cron always deletes first. */
|
||||
const ARCHIVE_PROVIDER_GRACE_MS = 7 * DAY_MS
|
||||
export const MAX_WAKE_WAIT_MS = 30_000
|
||||
const POLL_MS = 400
|
||||
/** Restart gives a wedged orcad this long to write durable offers before the stop. */
|
||||
const RESTART_DRAIN_SECONDS = 15
|
||||
/** A resuming/suspending row older than this is treated as stuck by Restart (the cron's bar is 15 min). */
|
||||
const RESTART_STUCK_MS = 5 * 60_000
|
||||
/**
|
||||
* A wake whose Worker died (waitUntil cut it off) is picked up by the next wake call after this.
|
||||
* Longer than any live inline wake (the desktop's boot timeout is 180 s), so a healthy boot is never taken over.
|
||||
*/
|
||||
const WAKE_STALE_MS = 4 * 60_000
|
||||
/** A wake that just failed answers `failed` for this long instead of booting again on every poll. */
|
||||
const WAKE_FAILURE_HOLD_MS = 60_000
|
||||
/** Archive gives orcad this long to write durable offers (the desktop already ran its Sleep). */
|
||||
const ARCHIVE_DRAIN_SECONDS = 60
|
||||
export const DEFAULT_CENTS_PER_HOUR: Record<MachineSize, number> = { standard: 75, large: 150 }
|
||||
|
||||
export type WorkspaceConfig = {
|
||||
archiveTtlMs?: number
|
||||
centsPerHour?: Partial<Record<MachineSize, number>>
|
||||
/** Keeps a wake running after the response (Worker ctx.waitUntil); tests omit it. */
|
||||
background?: (work: Promise<unknown>) => void
|
||||
sleep?: (ms: number) => Promise<void>
|
||||
}
|
||||
|
||||
export type WakeState = 'ready' | 'waking' | 'archived' | 'failed' | 'not_found'
|
||||
export type WakeAnswer = { state: WakeState; endpoint: string | null; readyAt: number | null; sandboxId: string | null }
|
||||
|
||||
/** wake/status share one vocabulary with the desktop's host health states. */
|
||||
export type WorkspaceState =
|
||||
| 'provisioning'
|
||||
| 'sleeping'
|
||||
| 'waking'
|
||||
| 'ready'
|
||||
| 'unresponsive'
|
||||
| 'stopping'
|
||||
| 'archived'
|
||||
| 'failed'
|
||||
|
||||
function wakeAnswer(row: SandboxRow | null): WakeAnswer {
|
||||
if (!row || row.status === 'destroyed' || row.status === 'destroying') {
|
||||
return { state: 'not_found', endpoint: null, readyAt: null, sandboxId: row?.id ?? null }
|
||||
}
|
||||
const state: WakeState =
|
||||
row.status === 'running'
|
||||
? 'ready'
|
||||
: row.status === 'archived'
|
||||
? 'archived'
|
||||
: row.status === 'failed'
|
||||
? 'failed'
|
||||
: 'waking'
|
||||
return {
|
||||
state,
|
||||
endpoint: state === 'ready' ? row.orcad_endpoint : null,
|
||||
readyAt: state === 'ready' ? row.ready_at : null,
|
||||
sandboxId: row.id
|
||||
}
|
||||
}
|
||||
|
||||
function recentlyFailed(deps: LifecycleDeps, row: SandboxRow): boolean {
|
||||
return row.status === 'suspended' && row.last_error === 'resume_failed' && deps.now() - row.updated_at < WAKE_FAILURE_HOLD_MS
|
||||
}
|
||||
|
||||
const defaultSleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms))
|
||||
|
||||
/** The row says running, but a lapsed lease means the provider may have stopped it on its own. */
|
||||
async function confirmRunning(deps: LifecycleDeps, row: SandboxRow): Promise<SandboxRow> {
|
||||
if (row.lease_expires_at === null || deps.now() < row.lease_expires_at) {
|
||||
return row
|
||||
}
|
||||
const state = await deps.provider.getSandbox(row.provider_name, { resume: false })
|
||||
return state?.session.status === 'running' ? row : markProviderStopped(deps, row)
|
||||
}
|
||||
|
||||
/**
|
||||
* Idempotent wake. A sleeping workspace starts waking (one wake however many callers race: the
|
||||
* `resuming` claim is a compare-and-set); `waitMs` long-polls until ready or the wait runs out.
|
||||
* Archived workspaces are not woken implicitly — unarchive is an explicit action.
|
||||
*/
|
||||
export async function wakeSandbox(
|
||||
deps: LifecycleDeps,
|
||||
row: SandboxRow,
|
||||
options: { waitMs: number } & WorkspaceConfig
|
||||
): Promise<WakeAnswer> {
|
||||
const waitMs = Math.max(0, Math.min(options.waitMs, MAX_WAKE_WAIT_MS))
|
||||
const sleep = options.sleep ?? defaultSleep
|
||||
let current = row.status === 'running' ? await confirmRunning(deps, row) : row
|
||||
let ours: Promise<unknown> | null = null
|
||||
if (current.status === 'running' && !current.orcad_endpoint) {
|
||||
// Repaired by the cron after a wake died: the VM runs, orcad may not.
|
||||
const state = await deps.provider.getSandbox(current.provider_name, { resume: false })
|
||||
current =
|
||||
state?.session.status === 'running' ? (await bootLiveSession(deps, current, state)).row : await markProviderStopped(deps, current)
|
||||
}
|
||||
if (current.status === 'resuming' && deps.now() - current.updated_at >= WAKE_STALE_MS) {
|
||||
current = await adoptStaleWake(deps, current)
|
||||
}
|
||||
if (recentlyFailed(deps, current)) {
|
||||
return { state: 'failed', endpoint: null, readyAt: null, sandboxId: current.id }
|
||||
}
|
||||
if (current.status === 'suspended') {
|
||||
await requireStart(deps, current.account_id, current.user_id, current.machine_size)
|
||||
const resuming = await transition(deps.db, current, { status: 'resuming' }, deps.now())
|
||||
if (resuming) {
|
||||
ours = wake(deps, resuming).catch((error: unknown) => {
|
||||
console.log(JSON.stringify({ event: 'wake_failed', sandboxId: row.id, error: String(error).slice(0, 120) }))
|
||||
})
|
||||
options.background?.(ours)
|
||||
}
|
||||
// Lost the race: someone else's wake is running; fall through and wait on the row.
|
||||
current = (await getSandbox(deps.db, row.id)) ?? current
|
||||
}
|
||||
const deadline = deps.now() + waitMs
|
||||
if (ours && waitMs > 0) {
|
||||
await Promise.race([ours, sleep(waitMs)])
|
||||
current = (await getSandbox(deps.db, row.id)) ?? current
|
||||
}
|
||||
while (current.status === 'resuming' && deps.now() < deadline) {
|
||||
await sleep(Math.min(POLL_MS, Math.max(1, deadline - deps.now())))
|
||||
current = (await getSandbox(deps.db, row.id)) ?? current
|
||||
}
|
||||
if (recentlyFailed(deps, current)) {
|
||||
return { state: 'failed', endpoint: null, readyAt: null, sandboxId: current.id }
|
||||
}
|
||||
return wakeAnswer(current)
|
||||
}
|
||||
|
||||
/**
|
||||
* The Worker running a wake was cut off (waitUntil limit) and left the row `resuming`: adopt what
|
||||
* the provider says — a running session gets orcad started and becomes `running`; otherwise the
|
||||
* row goes back to `suspended` so this wake can start a fresh one.
|
||||
*/
|
||||
async function adoptStaleWake(deps: LifecycleDeps, row: SandboxRow): Promise<SandboxRow> {
|
||||
const state = await deps.provider.getSandbox(row.provider_name, { resume: false })
|
||||
if (state?.session.status !== 'running') {
|
||||
return markProviderStopped(deps, row)
|
||||
}
|
||||
try {
|
||||
return (await bootLiveSession(deps, row, state)).row
|
||||
} catch (error) {
|
||||
if (error instanceof LifecycleError && error.code === 'conflict') {
|
||||
// Someone else moved the row meanwhile; report what they left.
|
||||
return (await getSandbox(deps.db, row.id)) ?? row
|
||||
}
|
||||
// Leave it resumable (and reported as failed for a minute) rather than stuck in `resuming`.
|
||||
const current = await getSandbox(deps.db, row.id)
|
||||
if (current?.status === 'resuming' && current.generation === row.generation) {
|
||||
await transition(deps.db, current, { status: 'suspended', ...NO_SESSION, last_error: 'resume_failed' }, deps.now())
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* "Restart" for an unresponsive workspace: one rehydrate path (short drain, stop with snapshot,
|
||||
* boot the same disk). Works from running, sleeping, or a wake/sleep that got stuck mid-way.
|
||||
*/
|
||||
export async function restartSandbox(deps: LifecycleDeps, row: SandboxRow): Promise<Ready> {
|
||||
if (await killSwitchOn(deps.db, guardrails(deps))) {
|
||||
throw new GuardrailError('maintenance')
|
||||
}
|
||||
let current = row
|
||||
if ((current.status === 'resuming' || current.status === 'suspending') && deps.now() - current.updated_at >= RESTART_STUCK_MS) {
|
||||
const state = await deps.provider.getSandbox(current.provider_name, { resume: false })
|
||||
if (state?.session.status === 'running') {
|
||||
await deps.provider.stopSession(state.session.id).catch(() => undefined)
|
||||
}
|
||||
current = await markProviderStopped(deps, current)
|
||||
}
|
||||
if (current.status === 'running') {
|
||||
// A lapsed lease means the provider already stopped it; resume instead of stopping a dead session.
|
||||
current = await confirmRunning(deps, current)
|
||||
}
|
||||
if (current.status === 'suspended') {
|
||||
return resumeSandbox(deps, current)
|
||||
}
|
||||
if (current.status !== 'running') {
|
||||
throw new LifecycleError('conflict', `sandbox_${current.status}`)
|
||||
}
|
||||
const suspending = await claim(deps, current, { status: 'suspending' })
|
||||
const stopped = await stopForSuspend(deps, suspending, { drainSeconds: RESTART_DRAIN_SECONDS })
|
||||
await flushUsage(deps.db, deps.entitlements, deps.now(), row.id)
|
||||
return wake(deps, await claim(deps, stopped, { status: 'resuming' }))
|
||||
}
|
||||
|
||||
export async function readArchive(db: D1Database, sandboxId: string): Promise<{ capturedAt: number; source: string; metadata: ArchiveMetadata } | null> {
|
||||
const stored = await db
|
||||
.prepare('SELECT captured_at, source, metadata FROM sandbox_archives WHERE sandbox_id = ?')
|
||||
.bind(sandboxId)
|
||||
.first<{ captured_at: number; source: string; metadata: string }>()
|
||||
if (!stored) {
|
||||
return null
|
||||
}
|
||||
const parsed = ArchiveMetadataSchema.safeParse(JSON.parse(stored.metadata))
|
||||
return parsed.success ? { capturedAt: stored.captured_at, source: stored.source, metadata: parsed.data } : null
|
||||
}
|
||||
|
||||
async function writeArchive(db: D1Database, sandboxId: string, source: string, metadata: ArchiveMetadata, now: number) {
|
||||
await db
|
||||
.prepare(
|
||||
`INSERT INTO sandbox_archives (sandbox_id, captured_at, source, metadata) VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT (sandbox_id) DO UPDATE SET captured_at = excluded.captured_at, source = excluded.source, metadata = excluded.metadata`
|
||||
)
|
||||
.bind(sandboxId, now, source, JSON.stringify(metadata))
|
||||
.run()
|
||||
}
|
||||
|
||||
export async function deleteArchive(db: D1Database, sandboxId: string): Promise<void> {
|
||||
await db.prepare('DELETE FROM sandbox_archives WHERE sandbox_id = ?').bind(sandboxId).run()
|
||||
}
|
||||
|
||||
/**
|
||||
* Archive = stop and keep the snapshot for a TTL. Git metadata is captured in the VM when it is
|
||||
* running; a sleeping VM is not woken just to archive it — the desktop's cached git state is used.
|
||||
*/
|
||||
export async function archiveSandbox(
|
||||
deps: LifecycleDeps,
|
||||
input: SandboxRow,
|
||||
request: Partial<ArchiveMetadata>,
|
||||
config: WorkspaceConfig = {}
|
||||
): Promise<SandboxRow> {
|
||||
let row = input
|
||||
if (row.status === 'archived') {
|
||||
return row
|
||||
}
|
||||
if (row.status !== 'running' && row.status !== 'suspended') {
|
||||
throw new LifecycleError('conflict', `sandbox_${row.status}`)
|
||||
}
|
||||
const captured = row.status === 'running' && row.session_id ? await captureArchiveMetadata(deps.provider, row.session_id) : null
|
||||
// The capture takes seconds; a lease renew may have moved the row meanwhile.
|
||||
row = (await getSandbox(deps.db, row.id)) ?? row
|
||||
if (row.status !== 'running' && row.status !== 'suspended') {
|
||||
throw new LifecycleError('conflict', `sandbox_${row.status}`)
|
||||
}
|
||||
const previous = await readArchive(deps.db, row.id)
|
||||
const checkouts = captured?.checkouts ?? request.checkouts ?? previous?.metadata.checkouts ?? []
|
||||
const source = captured ? 'vm' : request.checkouts ? 'client' : previous ? previous.source : 'none'
|
||||
await writeArchive(
|
||||
deps.db,
|
||||
row.id,
|
||||
source,
|
||||
{ checkouts, pullRequests: request.pullRequests ?? previous?.metadata.pullRequests ?? [] },
|
||||
deps.now()
|
||||
)
|
||||
let stopped = row
|
||||
if (row.status === 'running') {
|
||||
// Short drain: the desktop's Sleep normally ran first; a long drain would outlive the caller.
|
||||
stopped = await stopForSuspend(deps, await claim(deps, row, { status: 'suspending' }), { drainSeconds: ARCHIVE_DRAIN_SECONDS })
|
||||
await flushUsage(deps.db, deps.entitlements, deps.now(), row.id)
|
||||
}
|
||||
const ttl = config.archiveTtlMs ?? DEFAULT_ARCHIVE_TTL_MS
|
||||
await deps.provider
|
||||
.updateSnapshotExpiration(stopped.provider_name, ttl + ARCHIVE_PROVIDER_GRACE_MS)
|
||||
.catch((error: unknown) => console.log(JSON.stringify({ event: 'archive_ttl_update_failed', sandboxId: row.id, error: String(error).slice(0, 120) })))
|
||||
return claim(deps, stopped, {
|
||||
status: 'archived',
|
||||
...NO_SESSION,
|
||||
archived_at: deps.now(),
|
||||
archive_expires_at: deps.now() + ttl
|
||||
})
|
||||
}
|
||||
|
||||
/** Unarchive restores the long-lived snapshot backstop and resumes the same disk. */
|
||||
export async function unarchiveSandbox(deps: LifecycleDeps, row: SandboxRow): Promise<Ready> {
|
||||
if (row.status !== 'archived') {
|
||||
if (row.status === 'running' || row.status === 'suspended') {
|
||||
return resumeSandbox(deps, row)
|
||||
}
|
||||
throw new LifecycleError('conflict', `sandbox_${row.status}`)
|
||||
}
|
||||
await requireStart(deps, row.account_id, row.user_id, row.machine_size)
|
||||
await deps.provider.updateSnapshotExpiration(
|
||||
row.provider_name,
|
||||
deps.config.snapshotExpirationMs ?? DEFAULT_SNAPSHOT_EXPIRATION_MS
|
||||
)
|
||||
const suspended = await claim(deps, row, { status: 'suspended', archived_at: null, archive_expires_at: null })
|
||||
return resumeSandbox(deps, suspended)
|
||||
}
|
||||
|
||||
/** Cron: archives past their TTL are destroyed (snapshots included). */
|
||||
export async function expiredArchives(db: D1Database, now: number): Promise<SandboxRow[]> {
|
||||
const { results } = await db
|
||||
.prepare(`SELECT * FROM sandboxes WHERE status = 'archived' AND archive_expires_at <= ? ORDER BY archive_expires_at LIMIT 50`)
|
||||
.bind(now)
|
||||
.all<SandboxRow>()
|
||||
return results
|
||||
}
|
||||
|
||||
export type WorkspaceStatus = {
|
||||
state: WorkspaceState
|
||||
metrics: VmMetrics | null
|
||||
/** Since orcad last became ready (this session). */
|
||||
uptimeMs: number | null
|
||||
usage: { totalMs: number; centsPerHour: number; costCents: number }
|
||||
archive: { archivedAt: number; expiresAt: number | null } | null
|
||||
checkedAt: number
|
||||
}
|
||||
|
||||
function stateOf(row: SandboxRow, metrics: VmMetrics | null, probed: boolean): WorkspaceState {
|
||||
switch (row.status) {
|
||||
case 'creating':
|
||||
return 'provisioning'
|
||||
case 'resuming':
|
||||
return 'waking'
|
||||
case 'suspending':
|
||||
case 'destroying':
|
||||
return 'stopping'
|
||||
case 'suspended':
|
||||
return 'sleeping'
|
||||
case 'archived':
|
||||
return 'archived'
|
||||
case 'running':
|
||||
return probed && (!metrics || !metrics.orcadAlive) ? 'unresponsive' : 'ready'
|
||||
default:
|
||||
return 'failed'
|
||||
}
|
||||
}
|
||||
|
||||
/** The status panel's one call. Metrics are read live from the VM only when it is running. */
|
||||
export async function workspaceStatus(deps: LifecycleDeps, input: SandboxRow, config: WorkspaceConfig = {}): Promise<WorkspaceStatus> {
|
||||
// A lapsed lease: the provider may have stopped it, which is "sleeping", not "unresponsive".
|
||||
const row = input.status === 'running' ? await confirmRunning(deps, input) : input
|
||||
const probed = row.status === 'running' && row.session_id !== null
|
||||
const metrics = probed ? await readVmMetrics(deps.provider, row.session_id!) : null
|
||||
const now = deps.now()
|
||||
const totalMs = await sandboxUsageMs(deps.db, row, now)
|
||||
const centsPerHour = config.centsPerHour?.[row.machine_size] ?? DEFAULT_CENTS_PER_HOUR[row.machine_size]
|
||||
return {
|
||||
state: stateOf(row, metrics, probed),
|
||||
metrics,
|
||||
uptimeMs: row.status === 'running' && row.ready_at !== null ? Math.max(0, now - row.ready_at) : null,
|
||||
usage: { totalMs, centsPerHour, costCents: Math.round((totalMs / 3_600_000) * centsPerHour) },
|
||||
archive: row.archived_at !== null ? { archivedAt: row.archived_at, expiresAt: row.archive_expires_at } : null,
|
||||
checkedAt: now
|
||||
}
|
||||
}
|
||||
|
||||
export { getSandbox }
|
||||
@@ -36,6 +36,10 @@ export type FakeVercel = SandboxProviderApi & {
|
||||
failStart: boolean
|
||||
failEnvRefresh: boolean
|
||||
failCreate: boolean
|
||||
/** Answers `bash -c <script>` probes (metrics, archive capture); undefined falls through to 127. */
|
||||
onScript?: (script: string, sessionId: string) => { exitCode: number; stdout: string; stderr: string } | undefined
|
||||
/** Provider snapshot expiry per sandbox name, as last set by create or update. */
|
||||
snapshotExpirations: Map<string, number>
|
||||
/** Simulates the provider ending a session at its timeout. */
|
||||
expire(name: string): void
|
||||
}
|
||||
@@ -79,6 +83,7 @@ export function createFakeVercel(now: () => number): FakeVercel {
|
||||
failStart: false,
|
||||
failEnvRefresh: false,
|
||||
failCreate: false,
|
||||
snapshotExpirations: new Map(),
|
||||
expire(name) {
|
||||
const box = boxes.get(name)
|
||||
if (box?.status === 'running') {
|
||||
@@ -175,8 +180,20 @@ export function createFakeVercel(now: () => number): FakeVercel {
|
||||
if (params.command === '/bin/sh' && params.args?.[1]?.includes('credentials.json')) {
|
||||
return { exitCode: 0, stdout: '', stderr: '' }
|
||||
}
|
||||
if (params.command === 'bash' && params.args?.[0] === '-c' && fake.onScript) {
|
||||
const answered = fake.onScript(params.args[1] ?? '', sessionId)
|
||||
if (answered) {
|
||||
return answered
|
||||
}
|
||||
}
|
||||
return { exitCode: 127, stdout: '', stderr: 'not found' }
|
||||
},
|
||||
async updateSnapshotExpiration(name, expirationMs) {
|
||||
if (!boxes.has(name)) {
|
||||
throw new ProviderError(404, 'not_found')
|
||||
}
|
||||
fake.snapshotExpirations.set(name, expirationMs)
|
||||
},
|
||||
async readFile(sessionId, path) {
|
||||
return bySession(sessionId).files.get(path) ?? null
|
||||
},
|
||||
|
||||
@@ -0,0 +1,327 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
HostedArchiveResponseSchema,
|
||||
HostedSandboxReadyResponseSchema,
|
||||
HostedWakeResponseSchema,
|
||||
HostedWorkspaceStatusResponseSchema
|
||||
} from '../../../../src/shared/hosted-sandbox/hosted-sandbox-wire'
|
||||
import { DRAIN_COMMAND, START_COMMAND } from '../src/sandbox-vm-commands'
|
||||
import { ARCHIVE_SCRIPT, METRICS_SCRIPT } from '../src/sandbox-vm-inspect'
|
||||
import { createHarness } from './harness'
|
||||
|
||||
const DAY = 86_400_000
|
||||
const create = { workspaceId: 'ws-1', machineSize: 'standard' }
|
||||
|
||||
const metricsLine = (over: Record<string, unknown> = {}) =>
|
||||
JSON.stringify({
|
||||
cpuPercent: 12,
|
||||
cpus: 4,
|
||||
load1: 0.4,
|
||||
memTotalBytes: 8e9,
|
||||
memUsedBytes: 2e9,
|
||||
diskTotalBytes: 64e9,
|
||||
diskUsedBytes: 3e9,
|
||||
uptimeSeconds: 120,
|
||||
oomKills: 0,
|
||||
lastOomVictim: null,
|
||||
orcadAlive: true,
|
||||
...over
|
||||
})
|
||||
|
||||
const checkout = {
|
||||
path: '/vercel/sandbox/project',
|
||||
branch: 'feature/x',
|
||||
head: 'a'.repeat(40),
|
||||
upstream: 'origin/feature/x',
|
||||
ahead: 2,
|
||||
behind: 0,
|
||||
commits: [{ sha: 'a'.repeat(40), author: 'Ada', at: 1_800_000_000_000, subject: 'Add x' }],
|
||||
changes: [{ status: 'M', path: 'src/x.ts' }],
|
||||
diffStat: '1 file changed, 2 insertions(+)',
|
||||
fileCount: 2,
|
||||
files: ['README.md', 'src/x.ts']
|
||||
}
|
||||
|
||||
async function running() {
|
||||
const h = await createHarness()
|
||||
const created = HostedSandboxReadyResponseSchema.parse((await h.call('POST', '/v1/sandboxes', { body: create })).body)
|
||||
return { h, id: created.sandbox.id }
|
||||
}
|
||||
|
||||
describe('wake', () => {
|
||||
it('answers ready with the endpoint for a running workspace, by sandbox or workspace id', async () => {
|
||||
const { h, id } = await running()
|
||||
const bySandbox = HostedWakeResponseSchema.parse((await h.call('POST', `/v1/sandboxes/${id}/wake`)).body)
|
||||
expect(bySandbox).toMatchObject({ state: 'ready', sandboxId: id })
|
||||
expect(bySandbox.endpoint).toMatch(/^wss:\/\/sb-\d+\.vercel\.run$/)
|
||||
expect(bySandbox.readyAt).toBe(h.now())
|
||||
// The public API route shares the wake and answers the same fields (plus `workspace`).
|
||||
const viaApi = (await h.call('POST', `/v1/workspaces/${id}/wake?wait=30s`)).body
|
||||
expect({ state: viaApi.state, endpoint: viaApi.endpoint, readyAt: viaApi.readyAt, sandboxId: viaApi.sandboxId }).toEqual(bySandbox)
|
||||
})
|
||||
|
||||
it('wakes a sleeping workspace once however many callers race, and long-polls to ready', async () => {
|
||||
const { h, id } = await running()
|
||||
await h.call('POST', `/v1/sandboxes/${id}/suspend`)
|
||||
const starts = () => h.vercel.commands.filter((c) => c.command === START_COMMAND).length
|
||||
const before = starts()
|
||||
const answers = await Promise.all([
|
||||
h.call('POST', `/v1/sandboxes/${id}/wake?wait=30s`),
|
||||
h.call('POST', `/v1/sandboxes/${id}/wake?wait=30s`),
|
||||
h.call('POST', `/v1/sandboxes/${id}/wake`, { body: { wait: true } })
|
||||
])
|
||||
expect(answers.map((a) => a.body.state)).toEqual(['ready', 'ready', 'ready'])
|
||||
expect(starts() - before).toBe(1)
|
||||
})
|
||||
|
||||
it('returns waking immediately without a wait and finishes in the background', async () => {
|
||||
const background: Promise<unknown>[] = []
|
||||
const h = await createHarness({}, { waitUntil: (work) => void background.push(work) })
|
||||
const { body } = await h.call('POST', '/v1/sandboxes', { body: create })
|
||||
const id = body.sandbox.id
|
||||
await h.call("POST", `/v1/sandboxes/${id}/suspend`)
|
||||
const answer = HostedWakeResponseSchema.parse((await h.call('POST', `/v1/sandboxes/${id}/wake`)).body)
|
||||
// The fake provider is synchronous enough that the wake may already be done; either is valid.
|
||||
expect(['waking', 'ready']).toContain(answer.state)
|
||||
expect(background).toHaveLength(1)
|
||||
await Promise.all(background)
|
||||
expect((await h.call('POST', `/v1/sandboxes/${id}/wake`)).body.state).toBe('ready')
|
||||
})
|
||||
|
||||
it('does not wake an archived workspace implicitly, and 404s an unknown workspace', async () => {
|
||||
const { h, id } = await running()
|
||||
await h.call('POST', `/v1/sandboxes/${id}/archive`, { body: {} })
|
||||
expect((await h.call('POST', `/v1/sandboxes/${id}/wake?wait=5s`)).body).toMatchObject({ state: 'archived', endpoint: null })
|
||||
expect((await h.call('POST', '/v1/sandboxes/sbx_nope/wake')).status).toBe(404)
|
||||
})
|
||||
|
||||
it('refuses a wake billing denies', async () => {
|
||||
const { h, id } = await running()
|
||||
await h.call('POST', `/v1/sandboxes/${id}/suspend`)
|
||||
h.billing.decision = { allow: false, reason: 'included_hours_exhausted', suspendRunning: false }
|
||||
expect(await h.call('POST', `/v1/sandboxes/${id}/wake?wait=1s`)).toEqual({
|
||||
status: 403,
|
||||
body: { error: 'entitlement_denied', reason: 'included_hours_exhausted' }
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe('restart', () => {
|
||||
it('rehydrates a running workspace with a short drain and keeps the disk', async () => {
|
||||
const { h, id } = await running()
|
||||
const before = h.vercel.snapshots.size
|
||||
const ready = HostedSandboxReadyResponseSchema.parse((await h.call('POST', `/v1/sandboxes/${id}/restart`)).body)
|
||||
expect(ready.sandbox.status).toBe('running')
|
||||
expect(h.vercel.snapshots.size).toBe(before + 1)
|
||||
const drains = h.vercel.commands.filter((c) => c.command === DRAIN_COMMAND)
|
||||
expect(drains).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('recovers a wake that got stuck mid-way', async () => {
|
||||
const { h, id } = await running()
|
||||
h.raw.prepare(`UPDATE sandboxes SET status = 'resuming', updated_at = ? WHERE id = ?`).run(h.now(), id)
|
||||
h.advance(2 * 60_000)
|
||||
// A slow-but-alive wake is not taken over.
|
||||
expect((await h.call('POST', `/v1/sandboxes/${id}/restart`)).status).toBe(409)
|
||||
h.advance(4 * 60_000)
|
||||
const res = await h.call('POST', `/v1/sandboxes/${id}/restart`)
|
||||
expect(res.status).toBe(200)
|
||||
expect(res.body.sandbox.status).toBe('running')
|
||||
})
|
||||
})
|
||||
|
||||
describe('status panel', () => {
|
||||
it('reports live metrics, uptime and cost so far', async () => {
|
||||
const { h, id } = await running()
|
||||
h.vercel.onScript = (script) => (script === METRICS_SCRIPT ? { exitCode: 0, stdout: `${metricsLine()}\n`, stderr: '' } : undefined)
|
||||
h.advance(90 * 60_000)
|
||||
const { status } = HostedWorkspaceStatusResponseSchema.parse((await h.call('GET', `/v1/sandboxes/${id}/status`)).body)
|
||||
expect(status.state).toBe('ready')
|
||||
expect(status.metrics?.memUsedBytes).toBe(2e9)
|
||||
expect(status.uptimeMs).toBe(90 * 60_000)
|
||||
expect(status.usage).toEqual({ totalMs: 90 * 60_000, centsPerHour: 75, costCents: 113 })
|
||||
})
|
||||
|
||||
it('says unresponsive when the VM does not answer or orcad is dead, and surfaces OOM kills', async () => {
|
||||
const { h, id } = await running()
|
||||
h.vercel.onScript = () => ({ exitCode: 0, stdout: 'garbage', stderr: '' })
|
||||
expect((await h.call('GET', `/v1/sandboxes/${id}/status`)).body.status.state).toBe('unresponsive')
|
||||
h.vercel.onScript = () => ({ exitCode: 0, stdout: metricsLine({ orcadAlive: false, oomKills: 1, lastOomVictim: 'node' }), stderr: '' })
|
||||
const { status } = HostedWorkspaceStatusResponseSchema.parse((await h.call('GET', `/v1/sandboxes/${id}/status`)).body)
|
||||
expect(status.state).toBe('unresponsive')
|
||||
expect(status.metrics).toMatchObject({ oomKills: 1, lastOomVictim: 'node' })
|
||||
})
|
||||
|
||||
it('counts settled sessions in the cost and does not probe a sleeping VM', async () => {
|
||||
const { h, id } = await running()
|
||||
h.advance(60 * 60_000)
|
||||
await h.call('POST', `/v1/sandboxes/${id}/suspend`)
|
||||
let probes = 0
|
||||
h.vercel.onScript = () => (probes++, undefined)
|
||||
const { status } = HostedWorkspaceStatusResponseSchema.parse((await h.call('GET', `/v1/sandboxes/${id}/status`)).body)
|
||||
expect(status).toMatchObject({ state: 'sleeping', metrics: null, uptimeMs: null, usage: { totalMs: 60 * 60_000, costCents: 75 } })
|
||||
expect(probes).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('archive', () => {
|
||||
it('captures metadata in the VM, stops it, keeps the snapshot with a TTL, and unarchives the same disk', async () => {
|
||||
const { h, id } = await running()
|
||||
h.vercel.onScript = (script) =>
|
||||
script === ARCHIVE_SCRIPT ? { exitCode: 0, stdout: `${JSON.stringify({ checkouts: [checkout] })}\n`, stderr: '' } : undefined
|
||||
const pr = { number: 7, title: 'Add x', state: 'open', url: 'https://github.com/o/r/pull/7', branch: 'feature/x' }
|
||||
const archived = await h.call('POST', `/v1/sandboxes/${id}/archive`, { body: { pullRequests: [pr] } })
|
||||
// Callers that don't advertise `archive` (older strict desktops) see it as suspended.
|
||||
expect(archived.body.sandbox.status).toBe('suspended')
|
||||
const aware = await h.send(
|
||||
new Request(`https://sandbox.test/v1/sandboxes/${id}`, {
|
||||
headers: { authorization: `Bearer ${await h.token('alice')}`, 'x-orca-sandbox-features': 'archive' }
|
||||
})
|
||||
)
|
||||
expect(((await aware.json()) as any).sandbox.status).toBe('archived')
|
||||
const name = `orca-test-${id}`
|
||||
expect(h.vercel.boxes.get(name)!.status).toBe('stopped')
|
||||
expect(h.vercel.snapshotExpirations.get(name)).toBe(37 * DAY)
|
||||
|
||||
const { archive } = HostedArchiveResponseSchema.parse((await h.call('GET', `/v1/sandboxes/${id}/archive`)).body)
|
||||
expect(archive).toMatchObject({ source: 'vm', metadata: { checkouts: [checkout], pullRequests: [pr] } })
|
||||
const status = HostedWorkspaceStatusResponseSchema.parse((await h.call('GET', `/v1/sandboxes/${id}/status`)).body).status
|
||||
expect(status.archive).toEqual({ archivedAt: h.now(), expiresAt: h.now() + 30 * DAY })
|
||||
|
||||
const restored = HostedSandboxReadyResponseSchema.parse((await h.call('POST', `/v1/sandboxes/${id}/unarchive`)).body)
|
||||
expect(restored.sandbox.status).toBe('running')
|
||||
expect(h.vercel.snapshotExpirations.get(name)).toBe(104 * DAY)
|
||||
expect((await h.call('GET', `/v1/sandboxes/${id}/status`)).body.status.archive).toBeNull()
|
||||
})
|
||||
|
||||
it('archives a sleeping workspace from the desktop cache without waking it', async () => {
|
||||
const { h, id } = await running()
|
||||
await h.call('POST', `/v1/sandboxes/${id}/suspend`)
|
||||
const starts = h.vercel.commands.filter((c) => c.command === START_COMMAND).length
|
||||
await h.call('POST', `/v1/sandboxes/${id}/archive`, { body: { checkouts: [checkout] } })
|
||||
expect(h.vercel.commands.filter((c) => c.command === START_COMMAND)).toHaveLength(starts)
|
||||
const { archive } = HostedArchiveResponseSchema.parse((await h.call('GET', `/v1/sandboxes/${id}/archive`)).body)
|
||||
expect(archive?.source).toBe('client')
|
||||
})
|
||||
|
||||
it('rejects oversized archive bodies and destroys expired archives with their metadata', async () => {
|
||||
const { h, id } = await running()
|
||||
const tooMany = { checkouts: [{ ...checkout, files: Array.from({ length: 2001 }, (_, i) => `f${i}`) }] }
|
||||
expect((await h.call('POST', `/v1/sandboxes/${id}/archive`, { body: tooMany })).status).toBe(400)
|
||||
await h.call('POST', `/v1/sandboxes/${id}/archive`, { body: { checkouts: [checkout] } })
|
||||
h.advance(29 * DAY)
|
||||
expect((await h.reconcile()).archivesExpired).toBe(0)
|
||||
h.advance(2 * DAY)
|
||||
expect((await h.reconcile()).archivesExpired).toBe(1)
|
||||
expect(h.raw.prepare('SELECT status FROM sandboxes WHERE id = ?').get(id)).toEqual({ status: 'destroyed' })
|
||||
expect(h.raw.prepare('SELECT COUNT(*) AS n FROM sandbox_archives').get()).toEqual({ n: 0 })
|
||||
expect([...h.vercel.snapshots.values()].filter((s) => s.status !== 'deleted')).toHaveLength(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('desktop client against the real handler', () => {
|
||||
it('drives wake, status, restart, archive and unarchive through the desktop client', async () => {
|
||||
const { createHostedSandboxControlPlaneClient } = await import(
|
||||
'../../../../src/main/hosted-sandbox/hosted-sandbox-control-plane-client'
|
||||
)
|
||||
const h = await createHarness()
|
||||
const client = createHostedSandboxControlPlaneClient({
|
||||
baseUrl: 'https://sandbox.test',
|
||||
getAccessToken: () => h.token('alice'),
|
||||
fetch: (input, init) => h.send(new Request(input as URL, init))
|
||||
})
|
||||
const created = await client.create({ workspaceId: 'ws-c', machineSize: 'standard' })
|
||||
const id = created.sandbox.id
|
||||
h.vercel.onScript = (script) =>
|
||||
script === METRICS_SCRIPT
|
||||
? { exitCode: 0, stdout: metricsLine(), stderr: '' }
|
||||
: script === ARCHIVE_SCRIPT
|
||||
? { exitCode: 0, stdout: JSON.stringify({ checkouts: [checkout] }), stderr: '' }
|
||||
: undefined
|
||||
expect((await client.wake(id, 30_000)).state).toBe('ready')
|
||||
expect((await client.workspaceStatus(id)).status.state).toBe('ready')
|
||||
expect((await client.restart(id)).sandbox.status).toBe('running')
|
||||
expect((await client.archive(id, {})).status).toBe('archived')
|
||||
expect((await client.archiveInfo(id))?.metadata.checkouts[0]?.branch).toBe('feature/x')
|
||||
expect((await client.wake(id, 0)).state).toBe('archived')
|
||||
expect((await client.unarchive(id)).sandbox.status).toBe('running')
|
||||
})
|
||||
})
|
||||
|
||||
describe('wake recovery', () => {
|
||||
it('adopts a wake whose Worker was cut off: running VM gets orcad and an endpoint', async () => {
|
||||
const { h, id } = await running()
|
||||
await h.call('POST', `/v1/sandboxes/${id}/suspend`)
|
||||
// Simulate a wake that booted the VM, then lost its Worker before orcad/claim.
|
||||
await h.vercel.getSandbox(`orca-test-${id}`, { resume: true })
|
||||
h.raw.prepare(`UPDATE sandboxes SET status = 'resuming', updated_at = ? WHERE id = ?`).run(h.now(), id)
|
||||
expect((await h.call('POST', `/v1/sandboxes/${id}/wake`)).body.state).toBe('waking')
|
||||
// A live inline boot can take minutes: not adopted yet.
|
||||
h.advance(3 * 60_000)
|
||||
expect((await h.call('POST', `/v1/sandboxes/${id}/wake`)).body.state).toBe('waking')
|
||||
h.advance(61_000)
|
||||
const answer = HostedWakeResponseSchema.parse((await h.call('POST', `/v1/sandboxes/${id}/wake`)).body)
|
||||
expect(answer.state).toBe('ready')
|
||||
expect(answer.endpoint).toMatch(/^wss:/)
|
||||
})
|
||||
|
||||
it('a stale wake whose VM never came up goes back to sleep and wakes again', async () => {
|
||||
const { h, id } = await running()
|
||||
await h.call('POST', `/v1/sandboxes/${id}/suspend`)
|
||||
h.raw.prepare(`UPDATE sandboxes SET status = 'resuming', updated_at = ? WHERE id = ?`).run(h.now(), id)
|
||||
h.advance(4 * 60_000 + 1000)
|
||||
expect((await h.call('POST', `/v1/sandboxes/${id}/wake?wait=30s`)).body.state).toBe('ready')
|
||||
})
|
||||
|
||||
it('answers failed after a failed wake instead of booting on every poll', async () => {
|
||||
const { h, id } = await running()
|
||||
await h.call('POST', `/v1/sandboxes/${id}/suspend`)
|
||||
h.vercel.failStart = true
|
||||
expect((await h.call('POST', `/v1/sandboxes/${id}/wake?wait=30s`)).body.state).toBe('failed')
|
||||
const starts = h.vercel.commands.filter((c) => c.command === START_COMMAND).length
|
||||
expect((await h.call('POST', `/v1/sandboxes/${id}/wake?wait=30s`)).body.state).toBe('failed')
|
||||
expect(h.vercel.commands.filter((c) => c.command === START_COMMAND)).toHaveLength(starts)
|
||||
h.vercel.failStart = false
|
||||
h.advance(61_000)
|
||||
expect((await h.call('POST', `/v1/sandboxes/${id}/wake?wait=30s`)).body.state).toBe('ready')
|
||||
})
|
||||
|
||||
it('reports a provider-stopped VM as sleeping, not unresponsive', async () => {
|
||||
const { h, id } = await running()
|
||||
h.vercel.expire(`orca-test-${id}`)
|
||||
h.advance(31 * 60_000)
|
||||
expect((await h.call('GET', `/v1/sandboxes/${id}/status`)).body.status.state).toBe('sleeping')
|
||||
})
|
||||
})
|
||||
|
||||
describe('adopted wakes boot like real wakes', () => {
|
||||
it('starts orcad with the session secrets and the egress policy, and keeps metering intact', async () => {
|
||||
const h = await createHarness({ SECRETS_KEK: Buffer.alloc(32, 7).toString('base64') })
|
||||
expect((await h.call('PUT', '/v1/secrets/personal/OPENAI_API_KEY', { body: { value: 'sk-adopted-test' } })).status).toBe(200)
|
||||
const created = HostedSandboxReadyResponseSchema.parse((await h.call('POST', '/v1/sandboxes', { body: create })).body)
|
||||
const id = created.sandbox.id
|
||||
await h.call('POST', `/v1/sandboxes/${id}/suspend`)
|
||||
await h.vercel.getSandbox(`orca-test-${id}`, { resume: true })
|
||||
h.raw.prepare(`UPDATE sandboxes SET status = 'resuming', updated_at = ? WHERE id = ?`).run(h.now(), id)
|
||||
const before = h.vercel.commands.length
|
||||
h.advance(4 * 60_000 + 1000)
|
||||
expect((await h.call('POST', `/v1/sandboxes/${id}/wake`)).body.state).toBe('ready')
|
||||
const start = h.vercel.commands.slice(before).find((c) => c.command === START_COMMAND)!
|
||||
expect(start.env.ORCA_SECRET_ENV).toContain('OPENAI_API_KEY=')
|
||||
expect(start.env.ORCA_PUBLIC_WSS).toMatch(/^wss:/)
|
||||
const row = h.raw.prepare('SELECT status, session_id, orcad_endpoint FROM sandboxes WHERE id = ?').get(id) as any
|
||||
expect(row.status).toBe('running')
|
||||
expect(row.orcad_endpoint).toMatch(/^wss:/)
|
||||
})
|
||||
|
||||
it('a cron-repaired running row without orcad gets a full boot and keeps its usage cursor', async () => {
|
||||
const { h, id } = await running()
|
||||
h.advance(10 * 60_000)
|
||||
const cursor = (h.raw.prepare('SELECT usage_settled_at FROM sandboxes WHERE id = ?').get(id) as any).usage_settled_at
|
||||
h.raw.prepare('UPDATE sandboxes SET orcad_endpoint = NULL WHERE id = ?').run(id)
|
||||
const before = h.vercel.commands.filter((c) => c.command === START_COMMAND).length
|
||||
expect((await h.call('POST', `/v1/sandboxes/${id}/wake`)).body.state).toBe('ready')
|
||||
expect(h.vercel.commands.filter((c) => c.command === START_COMMAND)).toHaveLength(before + 1)
|
||||
expect((h.raw.prepare('SELECT usage_settled_at FROM sandboxes WHERE id = ?').get(id) as any).usage_settled_at).toBe(cursor)
|
||||
})
|
||||
})
|
||||
@@ -3,6 +3,13 @@
|
||||
import type { z } from 'zod'
|
||||
import { HostedEntitlementDeniedError } from './hosted-environment-provider'
|
||||
import {
|
||||
HostedArchiveResponseSchema,
|
||||
HostedWakeResponseSchema,
|
||||
HostedWorkspaceStatusResponseSchema,
|
||||
type HostedArchiveRequest,
|
||||
type HostedArchiveResponse,
|
||||
type HostedWakeResponse,
|
||||
type HostedWorkspaceStatus,
|
||||
HostedSandboxErrorResponseSchema,
|
||||
HostedSandboxListResponseSchema,
|
||||
HostedSandboxReadyResponseSchema,
|
||||
@@ -28,6 +35,22 @@ export type HostedSandboxControlPlaneClient = {
|
||||
renewLease(sandboxId: string, extendMs: number, signal?: AbortSignal): Promise<HostedSandboxView>
|
||||
rollover(sandboxId: string, signal?: AbortSignal): Promise<HostedSandboxReadyResponse>
|
||||
destroy(sandboxId: string, signal?: AbortSignal): Promise<HostedSandboxView>
|
||||
/** Idempotent wake; `waitMs` (max 30s) long-polls until orcad is ready. */
|
||||
wake(sandboxId: string, waitMs: number, signal?: AbortSignal): Promise<HostedWakeResponse>
|
||||
/** Status panel: state, live VM metrics, uptime, cost so far. */
|
||||
workspaceStatus(
|
||||
sandboxId: string,
|
||||
signal?: AbortSignal
|
||||
): Promise<{ sandbox: HostedSandboxView; status: HostedWorkspaceStatus }>
|
||||
/** Forced rehydrate for an unresponsive workspace (same disk, new session). */
|
||||
restart(sandboxId: string, signal?: AbortSignal): Promise<HostedSandboxReadyResponse>
|
||||
archive(
|
||||
sandboxId: string,
|
||||
body: HostedArchiveRequest,
|
||||
signal?: AbortSignal
|
||||
): Promise<HostedSandboxView>
|
||||
archiveInfo(sandboxId: string, signal?: AbortSignal): Promise<HostedArchiveResponse['archive']>
|
||||
unarchive(sandboxId: string, signal?: AbortSignal): Promise<HostedSandboxReadyResponse>
|
||||
}
|
||||
|
||||
export type HostedSandboxControlPlaneClientOptions = {
|
||||
@@ -65,6 +88,8 @@ export function createHostedSandboxControlPlaneClient(
|
||||
headers: {
|
||||
authorization: `Bearer ${token}`,
|
||||
accept: 'application/json',
|
||||
// This client's schemas know the `archived` status.
|
||||
'x-orca-sandbox-features': 'archive',
|
||||
...(init.body === undefined ? {} : { 'content-type': 'application/json' })
|
||||
},
|
||||
body: init.body === undefined ? undefined : JSON.stringify(init.body),
|
||||
@@ -144,7 +169,44 @@ export function createHostedSandboxControlPlaneClient(
|
||||
kind: 'control',
|
||||
signal
|
||||
})
|
||||
).sandbox
|
||||
).sandbox,
|
||||
wake: (id, waitMs, signal) =>
|
||||
call(
|
||||
'POST',
|
||||
`${sandboxPath(id, '/wake')}?wait=${Math.max(0, Math.min(30_000, Math.floor(waitMs)))}ms`,
|
||||
HostedWakeResponseSchema,
|
||||
{ kind: 'control', signal }
|
||||
),
|
||||
workspaceStatus: (id, signal) =>
|
||||
call('GET', sandboxPath(id, '/status'), HostedWorkspaceStatusResponseSchema, {
|
||||
kind: 'control',
|
||||
signal
|
||||
}),
|
||||
restart: (id, signal) =>
|
||||
call('POST', sandboxPath(id, '/restart'), HostedSandboxReadyResponseSchema, {
|
||||
kind: 'boot',
|
||||
signal
|
||||
}),
|
||||
archive: async (id, body, signal) =>
|
||||
(
|
||||
await call('POST', sandboxPath(id, '/archive'), HostedSandboxStatusResponseSchema, {
|
||||
body,
|
||||
kind: 'boot',
|
||||
signal
|
||||
})
|
||||
).sandbox,
|
||||
archiveInfo: async (id, signal) =>
|
||||
(
|
||||
await call('GET', sandboxPath(id, '/archive'), HostedArchiveResponseSchema, {
|
||||
kind: 'control',
|
||||
signal
|
||||
})
|
||||
).archive,
|
||||
unarchive: (id, signal) =>
|
||||
call('POST', sandboxPath(id, '/unarchive'), HostedSandboxReadyResponseSchema, {
|
||||
kind: 'boot',
|
||||
signal
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -41,7 +41,9 @@ const STATUS: Record<HostedSandboxView['status'], EnvironmentStatus> = {
|
||||
resuming: 'resuming',
|
||||
destroying: 'destroying',
|
||||
destroyed: 'destroyed',
|
||||
failed: 'failed'
|
||||
failed: 'failed',
|
||||
// Archived = asleep with a snapshot TTL; the desktop shows it as a sleeping environment.
|
||||
archived: 'suspended'
|
||||
}
|
||||
|
||||
export function createHostedSandboxDriver(
|
||||
@@ -141,7 +143,7 @@ function toSnapshot(view: HostedSandboxView): HostedStatusSnapshot {
|
||||
return { status: STATUS[view.status], session: toSession(view) }
|
||||
}
|
||||
|
||||
function toReady(ready: HostedSandboxReadyResponse): HostedEnvironmentReady {
|
||||
export function toReady(ready: HostedSandboxReadyResponse): HostedEnvironmentReady {
|
||||
const session = toSession(ready.sandbox)
|
||||
if (ready.sandbox.status !== 'running' || !session) {
|
||||
throw new EnvironmentProviderError(
|
||||
|
||||
@@ -10,9 +10,14 @@ import type { PluginService } from '../plugins/plugin-service'
|
||||
import { getHostedSandboxConfig } from './hosted-sandbox-config'
|
||||
import {
|
||||
ORCA_CLOUD_RECIPE,
|
||||
createHostedSandboxRecipeClient,
|
||||
createHostedSandboxRecipeDriver,
|
||||
registerHostedSandboxRecipe
|
||||
} from './hosted-sandbox-recipe'
|
||||
import {
|
||||
installHostedWorkspaceService,
|
||||
setHostedWorkspaceService
|
||||
} from './hosted-workspace-service'
|
||||
import { createHostedSessionRegistry, type HostedSessionRegistry } from './hosted-session-registry'
|
||||
import { setHostedSessionSync } from './hosted-session-sync'
|
||||
|
||||
@@ -34,7 +39,8 @@ export function installHostedSandbox(): void {
|
||||
if (!config) {
|
||||
return
|
||||
}
|
||||
const base = createHostedSandboxRecipeDriver(config)
|
||||
const client = createHostedSandboxRecipeClient(config)
|
||||
const base = createHostedSandboxRecipeDriver(config, client)
|
||||
// Dev rehearsals shorten idle sleep; packaged builds always use the provider's default.
|
||||
const idleOverride = app.isPackaged
|
||||
? Number.NaN
|
||||
@@ -47,17 +53,37 @@ export function installHostedSandbox(): void {
|
||||
driver,
|
||||
windows: () => BrowserWindow.getAllWindows()
|
||||
})
|
||||
const workspaces = installHostedWorkspaceService({
|
||||
userDataPath,
|
||||
client,
|
||||
windows: () => BrowserWindow.getAllWindows(),
|
||||
syncSessions: () => void registry.sync(),
|
||||
stopSession: (environmentId) => registry.stopEnvironment(environmentId)
|
||||
})
|
||||
setHostedWorkspaceService(workspaces)
|
||||
// Records land after the step answers and wakes can come from any path, so reconcile on a slow beat too.
|
||||
let beat: ReturnType<typeof setTimeout> | null = null
|
||||
let beats = 0
|
||||
const schedule = (ms: number): void => {
|
||||
beat = setTimeout(() => {
|
||||
void registry.sync().finally(() => schedule(30_000))
|
||||
// OOM in the guest is checked every other beat (~1 min) for running cloud workspaces.
|
||||
const oom = beats++ % 2 === 1 ? workspaces.pollOom() : Promise.resolve()
|
||||
void Promise.all([registry.sync(), oom]).finally(() => schedule(30_000))
|
||||
}, ms)
|
||||
beat.unref?.()
|
||||
}
|
||||
registerHostedSandboxRecipe(driver, {
|
||||
onReady: () => setTimeout(() => void registry.sync(), 3_000).unref?.(),
|
||||
onStopped: (environmentId) => registry.stopEnvironment(environmentId)
|
||||
onStopped: (environmentId) => registry.stopEnvironment(environmentId),
|
||||
onWake: ({ workspaceId, phase, startedAt, message }) =>
|
||||
workspaces.emitWakeProgress({
|
||||
workspaceId,
|
||||
phase,
|
||||
startedAt,
|
||||
message,
|
||||
// Later phases inherit the source recorded at `waking`.
|
||||
source: phase === 'waking' ? workspaces.takeWakeSource(workspaceId) : 'open'
|
||||
})
|
||||
})
|
||||
installed = { registry }
|
||||
setHostedSessionSync(() => void registry.sync())
|
||||
@@ -67,6 +93,8 @@ export function installHostedSandbox(): void {
|
||||
clearTimeout(beat)
|
||||
}
|
||||
registry.dispose()
|
||||
workspaces.dispose()
|
||||
setHostedWorkspaceService(null)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -39,15 +39,18 @@ export const ORCA_CLOUD_RECIPE: OrcaVmRecipe = {
|
||||
|
||||
const LifecyclePayload = z.looseObject({ recipeResult: EphemeralVmRecipeResultSchema.optional() })
|
||||
|
||||
export function createHostedSandboxRecipeClient(config: HostedSandboxConfig) {
|
||||
return createHostedSandboxControlPlaneClient({
|
||||
baseUrl: config.controlPlaneUrl,
|
||||
getAccessToken: config.getAccessToken
|
||||
})
|
||||
}
|
||||
|
||||
export function createHostedSandboxRecipeDriver(
|
||||
config: HostedSandboxConfig
|
||||
config: HostedSandboxConfig,
|
||||
client = createHostedSandboxRecipeClient(config)
|
||||
): HostedEnvironmentProviderDriver {
|
||||
return createHostedSandboxDriver(
|
||||
createHostedSandboxControlPlaneClient({
|
||||
baseUrl: config.controlPlaneUrl,
|
||||
getAccessToken: config.getAccessToken
|
||||
})
|
||||
)
|
||||
return createHostedSandboxDriver(client)
|
||||
}
|
||||
|
||||
/** https clone URL the control plane can mint a GitHub App token for; null for non-GitHub remotes. */
|
||||
@@ -101,6 +104,13 @@ export function createHostedSandboxStepProvider(
|
||||
hooks: {
|
||||
onReady?: (ready: HostedEnvironmentReady) => void
|
||||
onStopped?: (environmentId: string) => void
|
||||
/** Wake progress for the window's "Waking…" overlay (resume only). */
|
||||
onWake?: (event: {
|
||||
workspaceId: string
|
||||
phase: 'waking' | 'connecting' | 'ready' | 'failed'
|
||||
startedAt: number
|
||||
message?: string
|
||||
}) => void
|
||||
} = {}
|
||||
): RecipeStepProvider {
|
||||
return async ({ mode, repoPath, context, stdin, signal, onStderr }) => {
|
||||
@@ -133,9 +143,29 @@ export function createHostedSandboxStepProvider(
|
||||
return { stdout: '', stderr: 'Not an Orca Cloud environment.\n', exitCode: 1, signal: null }
|
||||
}
|
||||
if (mode === 'resume') {
|
||||
const ready = await driver.resume(handle, ctx)
|
||||
hooks.onReady?.(ready)
|
||||
return ok(JSON.stringify(ready.result))
|
||||
const workspaceId = context.workspaceId
|
||||
const startedAt = Date.now()
|
||||
if (workspaceId) {
|
||||
hooks.onWake?.({ workspaceId, phase: 'waking', startedAt })
|
||||
}
|
||||
try {
|
||||
const ready = await driver.resume(handle, ctx)
|
||||
if (workspaceId) {
|
||||
hooks.onWake?.({ workspaceId, phase: 'connecting', startedAt })
|
||||
}
|
||||
hooks.onReady?.(ready)
|
||||
return ok(JSON.stringify(ready.result))
|
||||
} catch (error) {
|
||||
if (workspaceId) {
|
||||
hooks.onWake?.({
|
||||
workspaceId,
|
||||
phase: 'failed',
|
||||
startedAt,
|
||||
message: error instanceof Error ? error.message : String(error)
|
||||
})
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
await (mode === 'suspend' ? driver.suspend(handle, ctx) : driver.destroy(handle, ctx))
|
||||
hooks.onStopped?.(handle.environmentId)
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import type { CloudOomEvent } from '../../shared/hosted-sandbox/cloud-workspace-api'
|
||||
import type { HostedWorkspaceStatus } from '../../shared/hosted-sandbox/hosted-sandbox-wire'
|
||||
import { createOomTracker } from './hosted-workspace-runtime'
|
||||
|
||||
function status(checkedAt: number, uptimeSeconds: number, oomKills: number): HostedWorkspaceStatus {
|
||||
return {
|
||||
state: 'ready',
|
||||
metrics: {
|
||||
cpuPercent: 1,
|
||||
cpus: 4,
|
||||
load1: 0,
|
||||
memTotalBytes: 8,
|
||||
memUsedBytes: 1,
|
||||
diskTotalBytes: 8,
|
||||
diskUsedBytes: 1,
|
||||
uptimeSeconds,
|
||||
oomKills,
|
||||
lastOomVictim: oomKills ? 'node' : null,
|
||||
orcadAlive: true
|
||||
},
|
||||
uptimeMs: 0,
|
||||
usage: { totalMs: 0, centsPerHour: 75, costCents: 0 },
|
||||
archive: null,
|
||||
checkedAt
|
||||
}
|
||||
}
|
||||
|
||||
describe('OOM tracker', () => {
|
||||
it('announces each new kill once per boot, despite clock jitter', () => {
|
||||
const events: CloudOomEvent[] = []
|
||||
const tracker = createOomTracker((event) => events.push(event))
|
||||
const t0 = 1_000_000_000
|
||||
tracker.observe('ws', 'sbx', status(t0, 100, 1))
|
||||
// Same boot, readings straddle a minute boundary: no repeat.
|
||||
tracker.observe('ws', 'sbx', status(t0 + 60_400, 160, 1))
|
||||
tracker.observe('ws', 'sbx', status(t0 + 120_000, 220, 2))
|
||||
expect(events.map((e) => e.oomKills)).toEqual([1, 2])
|
||||
expect(events[0]?.victim).toBe('node')
|
||||
})
|
||||
|
||||
it('treats a new boot as a fresh counter and forgets stopped sandboxes', () => {
|
||||
const events: CloudOomEvent[] = []
|
||||
const tracker = createOomTracker((event) => events.push(event))
|
||||
tracker.observe('ws', 'sbx', status(1_000_000_000, 100, 1))
|
||||
// Restarted: uptime is small again, counter restarts at 1.
|
||||
tracker.observe('ws', 'sbx', status(1_000_600_000, 20, 1))
|
||||
expect(events).toHaveLength(2)
|
||||
tracker.prune(new Set())
|
||||
tracker.observe('ws', 'sbx', status(1_000_660_000, 80, 1))
|
||||
expect(events).toHaveLength(3)
|
||||
})
|
||||
|
||||
it('ignores a status without metrics', () => {
|
||||
const events: CloudOomEvent[] = []
|
||||
createOomTracker((event) => events.push(event)).observe('ws', 'sbx', {
|
||||
...status(1, 1, 0),
|
||||
metrics: null
|
||||
})
|
||||
expect(events).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('archive checkouts trim', () => {
|
||||
it('keeps cached repo state under the archive body limit', async () => {
|
||||
const { trimCheckouts } = await import('./hosted-workspace-runtime')
|
||||
const big = Array.from({ length: 20 }, (_, i) => ({
|
||||
path: `/w/${i}`,
|
||||
branch: 'b',
|
||||
head: 'h',
|
||||
upstream: null,
|
||||
ahead: null,
|
||||
behind: null,
|
||||
commits: [],
|
||||
changes: [],
|
||||
diffStat: null,
|
||||
fileCount: 2000,
|
||||
files: Array.from({ length: 2000 }, (_, f) => `src/some/long/path/file-${f}.ts`)
|
||||
}))
|
||||
const trimmed = trimCheckouts(big)
|
||||
expect(JSON.stringify(trimmed).length).toBeLessThanOrEqual(200 * 1024)
|
||||
expect(trimmed[0]!.files.length).toBeGreaterThan(0)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,165 @@
|
||||
// Orca Cloud workspace lookups shared by the workspace service and its IPC: workspace id → runtime
|
||||
// record → sandbox id, error mapping for the window, and OOM-notice dedupe.
|
||||
import { listEphemeralVmRuntimes } from '../../shared/ephemeral-vm-runtime-store'
|
||||
import type { EphemeralVmRuntimeRecord } from '../../shared/ephemeral-vm-runtimes'
|
||||
import type {
|
||||
CloudOomEvent,
|
||||
CloudWorkspaceErrorCode,
|
||||
CloudWorkspaceResult
|
||||
} from '../../shared/hosted-sandbox/cloud-workspace-api'
|
||||
import type {
|
||||
HostedArchiveCheckout,
|
||||
HostedWorkspaceStatus
|
||||
} from '../../shared/hosted-sandbox/hosted-sandbox-wire'
|
||||
import {
|
||||
isEnvironmentProviderError,
|
||||
type EnvironmentProviderErrorCode
|
||||
} from '../environment-providers/environment-provider-driver'
|
||||
import { environmentHandleFromRecipeResult } from '../environment-providers/environment-provider-recipe-adapter'
|
||||
import { isHostedEntitlementDenied } from './hosted-environment-provider'
|
||||
import { HOSTED_SANDBOX_DRIVER_ID } from './hosted-sandbox-driver'
|
||||
import type { HostedWorkspaceService } from './hosted-workspace-service'
|
||||
|
||||
export type CloudRuntime = { runtime: EphemeralVmRuntimeRecord; sandboxId: string }
|
||||
|
||||
let cachedCheckouts: ((workspaceId: string) => Promise<HostedArchiveCheckout[] | null>) | null =
|
||||
null
|
||||
export function setArchiveCheckoutsProvider(
|
||||
provider: ((workspaceId: string) => Promise<HostedArchiveCheckout[] | null>) | null
|
||||
): void {
|
||||
cachedCheckouts = provider
|
||||
}
|
||||
|
||||
export function findCloudRuntime(userDataPath: string, workspaceId: string): CloudRuntime | null {
|
||||
for (const runtime of listEphemeralVmRuntimes(userDataPath)) {
|
||||
if (runtime.recipe?.provider !== 'orca-cloud' || runtime.workspaceId !== workspaceId) {
|
||||
continue
|
||||
}
|
||||
const handle = runtime.recipeResult
|
||||
? environmentHandleFromRecipeResult(runtime.recipeResult, HOSTED_SANDBOX_DRIVER_ID)
|
||||
: null
|
||||
const sandboxId = handle?.driverState.sandboxId
|
||||
if (typeof sandboxId === 'string' && sandboxId) {
|
||||
return { runtime, sandboxId }
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
export function failure<T>(error: unknown): CloudWorkspaceResult<T> {
|
||||
if (isHostedEntitlementDenied(error)) {
|
||||
return {
|
||||
ok: false,
|
||||
code: 'entitlement_denied',
|
||||
message: error.message,
|
||||
reason: error.reason ?? null
|
||||
}
|
||||
}
|
||||
const codes: (CloudWorkspaceErrorCode & EnvironmentProviderErrorCode)[] = [
|
||||
'unauthorized',
|
||||
'conflict',
|
||||
'not_found'
|
||||
]
|
||||
for (const code of codes) {
|
||||
if (isEnvironmentProviderError(error, code)) {
|
||||
return { ok: false, code, message: error.message, reason: error.reason ?? null }
|
||||
}
|
||||
}
|
||||
if (
|
||||
isEnvironmentProviderError(error, 'unavailable') ||
|
||||
isEnvironmentProviderError(error, 'timeout')
|
||||
) {
|
||||
return { ok: false, code: 'unavailable', message: error.message }
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
code: 'failed',
|
||||
message: error instanceof Error ? error.message : String(error)
|
||||
}
|
||||
}
|
||||
|
||||
export const NOT_CLOUD = {
|
||||
ok: false,
|
||||
code: 'not_cloud',
|
||||
message: 'Not an Orca Cloud workspace.'
|
||||
} as const
|
||||
|
||||
/** Cached repo state for an archive body, trimmed under the control plane's 256 KiB archive limit. */
|
||||
export async function readCachedCheckouts(
|
||||
workspaceId: string
|
||||
): Promise<HostedArchiveCheckout[] | null> {
|
||||
const checkouts = cachedCheckouts ? await cachedCheckouts(workspaceId).catch(() => null) : null
|
||||
return checkouts ? trimCheckouts(checkouts) : null
|
||||
}
|
||||
|
||||
const MAX_ARCHIVE_BODY_BYTES = 200 * 1024
|
||||
|
||||
export function trimCheckouts(input: HostedArchiveCheckout[]): HostedArchiveCheckout[] {
|
||||
const kept = input.slice(0, 20).map((c) => ({ ...c, files: c.files.slice(0, 2000) }))
|
||||
const fits = (): boolean => JSON.stringify(kept).length <= MAX_ARCHIVE_BODY_BYTES
|
||||
for (let i = kept.length - 1; i >= 0 && !fits(); i--) {
|
||||
kept[i] = { ...kept[i]!, files: i === 0 ? kept[i]!.files.slice(0, 500) : [] }
|
||||
}
|
||||
for (let i = kept.length - 1; i >= 0 && !fits(); i--) {
|
||||
kept[i] = { ...kept[i]!, files: [], changes: kept[i]!.changes.slice(0, 50) }
|
||||
}
|
||||
while (kept.length > 1 && !fits()) {
|
||||
kept.pop()
|
||||
}
|
||||
return kept
|
||||
}
|
||||
|
||||
/**
|
||||
* Announces each new kernel OOM kill once. The counter resets per VM boot; two readings belong to
|
||||
* one boot when their implied boot times agree within two minutes (uptime and checkedAt come from
|
||||
* different clocks).
|
||||
*/
|
||||
export function createOomTracker(announce: (event: CloudOomEvent) => void) {
|
||||
const seen = new Map<string, { bootAt: number; kills: number }>()
|
||||
return {
|
||||
observe(
|
||||
workspaceId: string,
|
||||
sandboxId: string,
|
||||
status: HostedWorkspaceStatus,
|
||||
now = Date.now()
|
||||
): void {
|
||||
const metrics = status.metrics
|
||||
if (!metrics) {
|
||||
return
|
||||
}
|
||||
const bootAt = status.checkedAt - metrics.uptimeSeconds * 1000
|
||||
const previous = seen.get(sandboxId)
|
||||
const sameBoot = previous !== undefined && Math.abs(previous.bootAt - bootAt) < 120_000
|
||||
const announced = sameBoot ? previous.kills : 0
|
||||
seen.set(sandboxId, {
|
||||
bootAt: sameBoot ? previous.bootAt : bootAt,
|
||||
kills: Math.max(announced, metrics.oomKills)
|
||||
})
|
||||
if (metrics.oomKills > announced) {
|
||||
announce({
|
||||
workspaceId,
|
||||
oomKills: metrics.oomKills,
|
||||
victim: metrics.lastOomVictim,
|
||||
at: now
|
||||
})
|
||||
}
|
||||
},
|
||||
/** Forget sandboxes that are no longer running. */
|
||||
prune(live: ReadonlySet<string | undefined>): void {
|
||||
for (const sandboxId of seen.keys()) {
|
||||
if (!live.has(sandboxId)) {
|
||||
seen.delete(sandboxId)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let current: HostedWorkspaceService | null = null
|
||||
export function setHostedWorkspaceService(next: HostedWorkspaceService | null): void {
|
||||
current = next
|
||||
}
|
||||
/** Null when Orca Cloud is not configured in this build. */
|
||||
export function getHostedWorkspaceService(): HostedWorkspaceService | null {
|
||||
return current
|
||||
}
|
||||
@@ -0,0 +1,321 @@
|
||||
// Orca Cloud workspace surfaces for the window: status panel, Restart, archive/unarchive, wake
|
||||
// progress and OOM notices. Maps the renderer's workspace id to its Orca Cloud runtime record.
|
||||
import { ipcMain, type BrowserWindow } from 'electron'
|
||||
import {
|
||||
listEphemeralVmRuntimes,
|
||||
updateEphemeralVmRuntimeStatus
|
||||
} from '../../shared/ephemeral-vm-runtime-store'
|
||||
import {
|
||||
ORCA_CLOUD_WORKSPACE_CHANNELS as CH,
|
||||
type CloudWakeProgressEvent,
|
||||
type CloudWakeSource
|
||||
} from '../../shared/hosted-sandbox/cloud-workspace-api'
|
||||
import type {
|
||||
HostedArchivePullRequest,
|
||||
HostedSandboxReadyResponse
|
||||
} from '../../shared/hosted-sandbox/hosted-sandbox-wire'
|
||||
import { applyResumedRuntimePairing } from '../ephemeral-vm/runtime-resume-pairing'
|
||||
import { noteRecipeVmWakePairing } from '../ephemeral-vm/recipe-vm-wake-pairings'
|
||||
import { invalidateRuntimeEnvironmentTransport } from '../ipc/runtime-environments/runtime-environments'
|
||||
import { autoResumeAgentsAfterRollover } from './hosted-agent-auto-resume'
|
||||
import type { HostedSandboxControlPlaneClient } from './hosted-sandbox-control-plane-client'
|
||||
import { toReady } from './hosted-sandbox-driver'
|
||||
import { createHostedRuntimeEnvironmentCall } from './hosted-runtime-environment-call'
|
||||
import {
|
||||
NOT_CLOUD,
|
||||
createOomTracker,
|
||||
failure,
|
||||
findCloudRuntime,
|
||||
readCachedCheckouts,
|
||||
type CloudRuntime
|
||||
} from './hosted-workspace-runtime'
|
||||
|
||||
export {
|
||||
findCloudRuntime,
|
||||
getHostedWorkspaceService,
|
||||
setArchiveCheckoutsProvider,
|
||||
setHostedWorkspaceService
|
||||
} from './hosted-workspace-runtime'
|
||||
|
||||
export type HostedWorkspaceService = {
|
||||
emitWakeProgress(event: Omit<CloudWakeProgressEvent, 'at'>): void
|
||||
/** Who asked for the next wake of this workspace (message/ssh/preview); read once by the step. */
|
||||
noteWakeSource(workspaceId: string, source: CloudWakeSource): void
|
||||
takeWakeSource(workspaceId: string): CloudWakeSource
|
||||
/** One OOM check for every running cloud workspace (the install beat calls it). */
|
||||
pollOom(): Promise<void>
|
||||
dispose(): void
|
||||
}
|
||||
|
||||
/** Git state from the client-side offline cache, used when archiving a VM that is already asleep. */
|
||||
export function installHostedWorkspaceService(deps: {
|
||||
userDataPath: string
|
||||
client: HostedSandboxControlPlaneClient
|
||||
windows: () => readonly BrowserWindow[]
|
||||
/** Restart lease schedulers after a lifecycle change. */
|
||||
syncSessions: () => void
|
||||
/** Stop the lease scheduler of one environment (its session is about to be replaced). */
|
||||
stopSession: (environmentId: string) => void
|
||||
log?: (line: string) => void
|
||||
}): HostedWorkspaceService {
|
||||
const log = deps.log ?? ((line: string) => console.error(line))
|
||||
const wakeSources = new Map<string, CloudWakeSource>()
|
||||
// Wakes in flight per workspace, so every phase carries the source the first one named.
|
||||
const wakes = new Map<string, { source: CloudWakeSource; startedAt: number }>()
|
||||
const broadcast = (channel: string, payload: unknown): void => {
|
||||
for (const window of deps.windows()) {
|
||||
if (!window.isDestroyed()) {
|
||||
window.webContents.send(channel, payload)
|
||||
}
|
||||
}
|
||||
}
|
||||
const oom = createOomTracker((event) => broadcast(CH.oom, event))
|
||||
|
||||
async function reconnect(found: CloudRuntime, ready: HostedSandboxReadyResponse): Promise<void> {
|
||||
const runtimeEnvironmentId = found.runtime.runtimeEnvironmentId
|
||||
updateEphemeralVmRuntimeStatus(deps.userDataPath, found.runtime.id, {
|
||||
status: 'running',
|
||||
recipeResult: toReady(ready).result
|
||||
})
|
||||
if (!runtimeEnvironmentId) {
|
||||
return
|
||||
}
|
||||
const { environment } = await applyResumedRuntimePairing({
|
||||
userDataPath: deps.userDataPath,
|
||||
environmentId: runtimeEnvironmentId,
|
||||
pairingCode: ready.connection.pairingCode,
|
||||
invalidateTransport: invalidateRuntimeEnvironmentTransport
|
||||
})
|
||||
noteRecipeVmWakePairing(
|
||||
environment.id,
|
||||
environment.pairingRevision ?? environment.createdAt,
|
||||
true
|
||||
)
|
||||
}
|
||||
|
||||
/** The step answers before the window re-pairs; `ready` fires once the runtime record says running. */
|
||||
async function announceReadyWhenRunning(workspaceId: string, startedAt: number): Promise<void> {
|
||||
const deadline = Date.now() + 60_000
|
||||
while (Date.now() < deadline && wakes.has(workspaceId)) {
|
||||
const runtime = findCloudRuntime(deps.userDataPath, workspaceId)?.runtime
|
||||
if (runtime?.status === 'running') {
|
||||
service.emitWakeProgress({ workspaceId, phase: 'ready', source: 'open', startedAt })
|
||||
return
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 250))
|
||||
}
|
||||
}
|
||||
|
||||
const service: HostedWorkspaceService = {
|
||||
emitWakeProgress(event) {
|
||||
if (event.phase === 'waking') {
|
||||
wakes.set(event.workspaceId, { source: event.source, startedAt: event.startedAt })
|
||||
}
|
||||
const inFlight = wakes.get(event.workspaceId)
|
||||
const source = inFlight?.source ?? event.source
|
||||
if (event.phase === 'ready' || event.phase === 'failed') {
|
||||
wakes.delete(event.workspaceId)
|
||||
}
|
||||
broadcast(CH.wakeProgress, {
|
||||
...event,
|
||||
source,
|
||||
at: Date.now()
|
||||
} satisfies CloudWakeProgressEvent)
|
||||
if (event.phase === 'connecting' && event.source !== 'restart') {
|
||||
void announceReadyWhenRunning(event.workspaceId, event.startedAt)
|
||||
}
|
||||
},
|
||||
noteWakeSource(workspaceId, source) {
|
||||
wakeSources.set(workspaceId, source)
|
||||
},
|
||||
takeWakeSource(workspaceId) {
|
||||
const source = wakeSources.get(workspaceId) ?? 'open'
|
||||
wakeSources.delete(workspaceId)
|
||||
return source
|
||||
},
|
||||
async pollOom() {
|
||||
const live = new Set(
|
||||
listEphemeralVmRuntimes(deps.userDataPath)
|
||||
.filter(
|
||||
(runtime) =>
|
||||
runtime.recipe?.provider === 'orca-cloud' &&
|
||||
runtime.status === 'running' &&
|
||||
runtime.workspaceId
|
||||
)
|
||||
.map((runtime) => findCloudRuntime(deps.userDataPath, runtime.workspaceId!)?.sandboxId)
|
||||
)
|
||||
oom.prune(live)
|
||||
for (const runtime of listEphemeralVmRuntimes(deps.userDataPath)) {
|
||||
if (
|
||||
runtime.recipe?.provider !== 'orca-cloud' ||
|
||||
runtime.status !== 'running' ||
|
||||
!runtime.workspaceId
|
||||
) {
|
||||
continue
|
||||
}
|
||||
const found = findCloudRuntime(deps.userDataPath, runtime.workspaceId)
|
||||
if (!found) {
|
||||
continue
|
||||
}
|
||||
try {
|
||||
const { status } = await deps.client.workspaceStatus(found.sandboxId)
|
||||
oom.observe(runtime.workspaceId, found.sandboxId, status)
|
||||
} catch {
|
||||
// Status is advisory; the next beat retries.
|
||||
}
|
||||
}
|
||||
},
|
||||
dispose() {
|
||||
for (const channel of [
|
||||
CH.getStatus,
|
||||
CH.restart,
|
||||
CH.archive,
|
||||
CH.unarchive,
|
||||
CH.getArchive,
|
||||
CH.listCloudWorkspaceIds
|
||||
]) {
|
||||
ipcMain.removeHandler(channel)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
service.dispose()
|
||||
ipcMain.handle(CH.listCloudWorkspaceIds, () =>
|
||||
listEphemeralVmRuntimes(deps.userDataPath)
|
||||
.filter((runtime) => runtime.recipe?.provider === 'orca-cloud' && runtime.workspaceId)
|
||||
.map((runtime) => runtime.workspaceId!)
|
||||
)
|
||||
|
||||
ipcMain.handle(CH.getStatus, async (_event, args: { workspaceId: string }) => {
|
||||
const found = findCloudRuntime(deps.userDataPath, args.workspaceId)
|
||||
if (!found) {
|
||||
return NOT_CLOUD
|
||||
}
|
||||
try {
|
||||
const { status } = await deps.client.workspaceStatus(found.sandboxId)
|
||||
oom.observe(args.workspaceId, found.sandboxId, status)
|
||||
return { ok: true, value: status }
|
||||
} catch (error) {
|
||||
return failure(error)
|
||||
}
|
||||
})
|
||||
|
||||
ipcMain.handle(CH.restart, async (_event, args: { workspaceId: string }) => {
|
||||
const found = findCloudRuntime(deps.userDataPath, args.workspaceId)
|
||||
if (!found) {
|
||||
return NOT_CLOUD
|
||||
}
|
||||
const startedAt = Date.now()
|
||||
service.emitWakeProgress({
|
||||
workspaceId: args.workspaceId,
|
||||
phase: 'waking',
|
||||
source: 'restart',
|
||||
startedAt
|
||||
})
|
||||
// The scheduler holds the old session's deadline and would warn/roll over the new one; a fresh one starts on sync.
|
||||
deps.stopSession(found.sandboxId)
|
||||
try {
|
||||
const ready = await deps.client.restart(found.sandboxId)
|
||||
service.emitWakeProgress({
|
||||
workspaceId: args.workspaceId,
|
||||
phase: 'connecting',
|
||||
source: 'restart',
|
||||
startedAt
|
||||
})
|
||||
await reconnect(found, ready)
|
||||
deps.syncSessions()
|
||||
service.emitWakeProgress({
|
||||
workspaceId: args.workspaceId,
|
||||
phase: 'ready',
|
||||
source: 'restart',
|
||||
startedAt
|
||||
})
|
||||
if (found.runtime.runtimeEnvironmentId) {
|
||||
// Same continuation as a rollover: chats the restart interrupted carry on by themselves.
|
||||
const call = createHostedRuntimeEnvironmentCall(
|
||||
deps.userDataPath,
|
||||
found.runtime.runtimeEnvironmentId
|
||||
)
|
||||
void autoResumeAgentsAfterRollover(call, startedAt).catch((error: unknown) =>
|
||||
log(`[orca-cloud] auto-resume after restart failed: ${String(error)}`)
|
||||
)
|
||||
}
|
||||
return { ok: true, value: null }
|
||||
} catch (error) {
|
||||
service.emitWakeProgress({
|
||||
workspaceId: args.workspaceId,
|
||||
phase: 'failed',
|
||||
source: 'restart',
|
||||
startedAt,
|
||||
message: error instanceof Error ? error.message : String(error)
|
||||
})
|
||||
deps.syncSessions()
|
||||
return failure(error)
|
||||
}
|
||||
})
|
||||
|
||||
ipcMain.handle(
|
||||
CH.archive,
|
||||
async (_event, args: { workspaceId: string; pullRequests?: HostedArchivePullRequest[] }) => {
|
||||
const found = findCloudRuntime(deps.userDataPath, args.workspaceId)
|
||||
if (!found) {
|
||||
return NOT_CLOUD
|
||||
}
|
||||
try {
|
||||
const checkouts = (await readCachedCheckouts(args.workspaceId)) ?? undefined
|
||||
await deps.client.archive(found.sandboxId, {
|
||||
...(args.pullRequests?.length ? { pullRequests: args.pullRequests.slice(0, 50) } : {}),
|
||||
...(checkouts?.length ? { checkouts: checkouts.slice(0, 20) } : {})
|
||||
})
|
||||
updateEphemeralVmRuntimeStatus(deps.userDataPath, found.runtime.id, { status: 'suspended' })
|
||||
deps.syncSessions()
|
||||
return { ok: true, value: null }
|
||||
} catch (error) {
|
||||
return failure(error)
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
ipcMain.handle(CH.unarchive, async (_event, args: { workspaceId: string }) => {
|
||||
const found = findCloudRuntime(deps.userDataPath, args.workspaceId)
|
||||
if (!found) {
|
||||
return NOT_CLOUD
|
||||
}
|
||||
const startedAt = Date.now()
|
||||
service.emitWakeProgress({
|
||||
workspaceId: args.workspaceId,
|
||||
phase: 'waking',
|
||||
source: 'unarchive',
|
||||
startedAt
|
||||
})
|
||||
try {
|
||||
// The control plane resumes the disk; the window then runs its ordinary wake, which re-pairs.
|
||||
await deps.client.unarchive(found.sandboxId)
|
||||
deps.syncSessions()
|
||||
return { ok: true, value: null }
|
||||
} catch (error) {
|
||||
service.emitWakeProgress({
|
||||
workspaceId: args.workspaceId,
|
||||
phase: 'failed',
|
||||
source: 'unarchive',
|
||||
startedAt,
|
||||
message: error instanceof Error ? error.message : String(error)
|
||||
})
|
||||
return failure(error)
|
||||
}
|
||||
})
|
||||
|
||||
ipcMain.handle(CH.getArchive, async (_event, args: { workspaceId: string }) => {
|
||||
const found = findCloudRuntime(deps.userDataPath, args.workspaceId)
|
||||
if (!found) {
|
||||
return NOT_CLOUD
|
||||
}
|
||||
try {
|
||||
return { ok: true, value: await deps.client.archiveInfo(found.sandboxId) }
|
||||
} catch (error) {
|
||||
return failure(error)
|
||||
}
|
||||
})
|
||||
|
||||
return service
|
||||
}
|
||||
@@ -28,6 +28,7 @@ import type { DashboardApi, TerminalPreviewApi } from './api/dashboard-api'
|
||||
import type { DocPreviewApi } from './api/doc-preview-api'
|
||||
import type { EmulatorApi } from './api/emulator-api'
|
||||
import type { EphemeralVmApi } from './api/ephemeral-vm-api'
|
||||
import type { OrcaCloudWorkspaceApi } from '../shared/hosted-sandbox/cloud-workspace-api'
|
||||
import type { ExportApi, FilesystemApi } from './api/filesystem-api'
|
||||
import type { GitInspectionApi } from './api/git-inspection-api'
|
||||
import type { GitOperationApi } from './api/git-operation-api'
|
||||
@@ -134,6 +135,8 @@ export type PreloadApi = {
|
||||
emulator: EmulatorApi
|
||||
hooks: HooksApi
|
||||
ephemeralVm: EphemeralVmApi
|
||||
/** Orca Cloud workspace lifecycle (status panel, Restart, archive); absent in builds without Orca Cloud. */
|
||||
orcaCloudWorkspace: OrcaCloudWorkspaceApi
|
||||
cache: WorkspaceSessionApi['cache']
|
||||
session: WorkspaceSessionApi['session']
|
||||
updater: UpdaterApi
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
import { ipcRenderer } from 'electron'
|
||||
import {
|
||||
ORCA_CLOUD_WORKSPACE_CHANNELS as CH,
|
||||
type CloudOomEvent,
|
||||
type CloudWakeProgressEvent,
|
||||
type OrcaCloudWorkspaceApi
|
||||
} from '../../shared/hosted-sandbox/cloud-workspace-api'
|
||||
|
||||
export const orcaCloudWorkspaceApi = {
|
||||
getStatus: (args) => ipcRenderer.invoke(CH.getStatus, args),
|
||||
restart: (args) => ipcRenderer.invoke(CH.restart, args),
|
||||
archive: (args) => ipcRenderer.invoke(CH.archive, args),
|
||||
unarchive: (args) => ipcRenderer.invoke(CH.unarchive, args),
|
||||
getArchive: (args) => ipcRenderer.invoke(CH.getArchive, args),
|
||||
listCloudWorkspaceIds: () => ipcRenderer.invoke(CH.listCloudWorkspaceIds),
|
||||
onWakeProgress: (callback) => {
|
||||
const listener = (_event: Electron.IpcRendererEvent, event: CloudWakeProgressEvent): void =>
|
||||
callback(event)
|
||||
ipcRenderer.on(CH.wakeProgress, listener)
|
||||
return () => ipcRenderer.removeListener(CH.wakeProgress, listener)
|
||||
},
|
||||
onOom: (callback) => {
|
||||
const listener = (_event: Electron.IpcRendererEvent, event: CloudOomEvent): void =>
|
||||
callback(event)
|
||||
ipcRenderer.on(CH.oom, listener)
|
||||
return () => ipcRenderer.removeListener(CH.oom, listener)
|
||||
}
|
||||
} satisfies OrcaCloudWorkspaceApi
|
||||
@@ -60,6 +60,7 @@ import { browserApi } from './api/browser/browser-bridge'
|
||||
import { emulatorApi } from './api/emulator-bridge'
|
||||
import { hooksApi } from './api/hooks-bridge'
|
||||
import { ephemeralVmApi } from './api/ephemeral-vm-bridge'
|
||||
import { orcaCloudWorkspaceApi } from './api/orca-cloud-workspace-bridge'
|
||||
import { cacheApi } from './api/cache-bridge'
|
||||
import { sessionApi } from './api/session-bridge'
|
||||
import { updaterApi } from './api/updater-bridge'
|
||||
@@ -167,6 +168,7 @@ const api = {
|
||||
emulator: emulatorApi,
|
||||
hooks: hooksApi,
|
||||
ephemeralVm: ephemeralVmApi,
|
||||
orcaCloudWorkspace: orcaCloudWorkspaceApi,
|
||||
cache: cacheApi,
|
||||
session: sessionApi,
|
||||
updater: updaterApi,
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
// Window ↔ main contract for Orca Cloud workspace lifecycle surfaces: status panel, Restart,
|
||||
// archive/unarchive, wake progress, OOM notices. Keyed by the renderer's workspace (worktree) id;
|
||||
// main maps it to the Orca Cloud runtime record and its sandbox.
|
||||
import type {
|
||||
HostedArchivePullRequest,
|
||||
HostedArchiveResponse,
|
||||
HostedWorkspaceStatus
|
||||
} from './hosted-sandbox-wire'
|
||||
|
||||
export type CloudWorkspaceErrorCode =
|
||||
| 'not_cloud'
|
||||
| 'unauthorized'
|
||||
| 'entitlement_denied'
|
||||
| 'conflict'
|
||||
| 'unavailable'
|
||||
| 'not_found'
|
||||
| 'failed'
|
||||
|
||||
export type CloudWorkspaceResult<T> =
|
||||
| { ok: true; value: T }
|
||||
| { ok: false; code: CloudWorkspaceErrorCode; message: string; reason?: string | null }
|
||||
|
||||
/** Why a wake started; every source goes through the control plane's single-flight wake. */
|
||||
export type CloudWakeSource = 'open' | 'message' | 'ssh' | 'preview' | 'restart' | 'unarchive'
|
||||
|
||||
export type CloudWakeProgressEvent = {
|
||||
workspaceId: string
|
||||
phase: 'waking' | 'connecting' | 'ready' | 'failed'
|
||||
source: CloudWakeSource
|
||||
/** Epoch ms the wake began (the overlay's elapsed timer). */
|
||||
startedAt: number
|
||||
at: number
|
||||
message?: string
|
||||
}
|
||||
|
||||
export type CloudOomEvent = {
|
||||
workspaceId: string
|
||||
/** Kernel OOM-kill counter after this notice (monotonic per VM boot). */
|
||||
oomKills: number
|
||||
victim: string | null
|
||||
at: number
|
||||
}
|
||||
|
||||
export type CloudArchiveInfo = NonNullable<HostedArchiveResponse['archive']>
|
||||
|
||||
export type OrcaCloudWorkspaceApi = {
|
||||
/** Live status: state, CPU/mem/disk, uptime, cost so far, archive. Cheap; the panel polls it while open. */
|
||||
getStatus: (args: { workspaceId: string }) => Promise<CloudWorkspaceResult<HostedWorkspaceStatus>>
|
||||
/** "Restart" (Unresponsive / OOM): rehydrate the same disk and re-pair; agents auto-resume. */
|
||||
restart: (args: { workspaceId: string }) => Promise<CloudWorkspaceResult<null>>
|
||||
/** Sleep, then archive with a snapshot TTL; metadata captured in the VM or from the client cache. */
|
||||
archive: (args: {
|
||||
workspaceId: string
|
||||
pullRequests?: HostedArchivePullRequest[]
|
||||
}) => Promise<CloudWorkspaceResult<null>>
|
||||
unarchive: (args: { workspaceId: string }) => Promise<CloudWorkspaceResult<null>>
|
||||
getArchive: (args: {
|
||||
workspaceId: string
|
||||
}) => Promise<CloudWorkspaceResult<CloudArchiveInfo | null>>
|
||||
/** Workspace ids of every Orca Cloud workspace this desktop knows (the sidebar dot / hover card). */
|
||||
listCloudWorkspaceIds: () => Promise<string[]>
|
||||
onWakeProgress: (callback: (event: CloudWakeProgressEvent) => void) => () => void
|
||||
onOom: (callback: (event: CloudOomEvent) => void) => () => void
|
||||
}
|
||||
|
||||
export const ORCA_CLOUD_WORKSPACE_CHANNELS = {
|
||||
getStatus: 'orcaCloudWorkspace:getStatus',
|
||||
restart: 'orcaCloudWorkspace:restart',
|
||||
archive: 'orcaCloudWorkspace:archive',
|
||||
unarchive: 'orcaCloudWorkspace:unarchive',
|
||||
getArchive: 'orcaCloudWorkspace:getArchive',
|
||||
listCloudWorkspaceIds: 'orcaCloudWorkspace:listIds',
|
||||
wakeProgress: 'orcaCloudWorkspace:wakeProgress',
|
||||
oom: 'orcaCloudWorkspace:oom'
|
||||
} as const
|
||||
@@ -14,7 +14,8 @@ export const HOSTED_SANDBOX_STATUSES = [
|
||||
'resuming',
|
||||
'destroying',
|
||||
'destroyed',
|
||||
'failed'
|
||||
'failed',
|
||||
'archived'
|
||||
] as const
|
||||
export type HostedSandboxStatus = (typeof HOSTED_SANDBOX_STATUSES)[number]
|
||||
|
||||
@@ -119,3 +120,131 @@ export const HostedSandboxErrorResponseSchema = z
|
||||
reason: z.string().max(64).optional()
|
||||
})
|
||||
.strict()
|
||||
|
||||
/** Every wake source (open, message, SSH, preview) answers in this shape. */
|
||||
export const HOSTED_WAKE_STATES = ['ready', 'waking', 'archived', 'failed', 'not_found'] as const
|
||||
export const HostedWakeResponseSchema = z
|
||||
.object({
|
||||
state: z.enum(HOSTED_WAKE_STATES),
|
||||
endpoint: z.string().startsWith('wss://').max(2048).nullable(),
|
||||
readyAt: Millis.nullable(),
|
||||
sandboxId: Id.nullable()
|
||||
})
|
||||
.strict()
|
||||
export type HostedWakeResponse = z.infer<typeof HostedWakeResponseSchema>
|
||||
|
||||
export const HOSTED_WORKSPACE_STATES = [
|
||||
'provisioning',
|
||||
'sleeping',
|
||||
'waking',
|
||||
'ready',
|
||||
'unresponsive',
|
||||
'stopping',
|
||||
'archived',
|
||||
'failed'
|
||||
] as const
|
||||
export type HostedWorkspaceState = (typeof HOSTED_WORKSPACE_STATES)[number]
|
||||
|
||||
const Bytes = z.number().int().nonnegative()
|
||||
export const HostedVmMetricsSchema = z
|
||||
.object({
|
||||
cpuPercent: z.number().int().min(0).max(100),
|
||||
cpus: z.number().int().min(1),
|
||||
load1: z.number().min(0),
|
||||
memTotalBytes: Bytes,
|
||||
memUsedBytes: Bytes,
|
||||
diskTotalBytes: Bytes,
|
||||
diskUsedBytes: Bytes,
|
||||
uptimeSeconds: z.number().int().nonnegative(),
|
||||
oomKills: z.number().int().nonnegative(),
|
||||
lastOomVictim: z.string().max(64).nullable(),
|
||||
orcadAlive: z.boolean()
|
||||
})
|
||||
.strict()
|
||||
export type HostedVmMetrics = z.infer<typeof HostedVmMetricsSchema>
|
||||
|
||||
export const HostedWorkspaceStatusSchema = z
|
||||
.object({
|
||||
state: z.enum(HOSTED_WORKSPACE_STATES),
|
||||
metrics: HostedVmMetricsSchema.nullable(),
|
||||
uptimeMs: Millis.nullable(),
|
||||
usage: z
|
||||
.object({
|
||||
totalMs: Millis,
|
||||
centsPerHour: z.number().int().nonnegative(),
|
||||
costCents: z.number().int().nonnegative()
|
||||
})
|
||||
.strict(),
|
||||
archive: z.object({ archivedAt: Millis, expiresAt: Millis.nullable() }).strict().nullable(),
|
||||
checkedAt: Millis
|
||||
})
|
||||
.strict()
|
||||
export type HostedWorkspaceStatus = z.infer<typeof HostedWorkspaceStatusSchema>
|
||||
|
||||
export const HostedWorkspaceStatusResponseSchema = z
|
||||
.object({ sandbox: HostedSandboxViewSchema, status: HostedWorkspaceStatusSchema })
|
||||
.strict()
|
||||
|
||||
const ArchiveCommit = z
|
||||
.object({
|
||||
sha: z.string().max(64),
|
||||
author: z.string().max(120),
|
||||
at: z.number(),
|
||||
subject: z.string().max(300)
|
||||
})
|
||||
.strict()
|
||||
export const HostedArchiveCheckoutSchema = z
|
||||
.object({
|
||||
path: z.string().max(4096),
|
||||
branch: z.string().max(255).nullable(),
|
||||
head: z.string().max(64).nullable(),
|
||||
upstream: z.string().max(255).nullable(),
|
||||
ahead: z.number().int().nullable(),
|
||||
behind: z.number().int().nullable(),
|
||||
commits: z.array(ArchiveCommit).max(20),
|
||||
changes: z
|
||||
.array(z.object({ status: z.string().max(2), path: z.string().max(1024) }).strict())
|
||||
.max(300),
|
||||
diffStat: z.string().max(200).nullable(),
|
||||
fileCount: z.number().int().nonnegative(),
|
||||
files: z.array(z.string().max(1024)).max(2000)
|
||||
})
|
||||
.strict()
|
||||
export type HostedArchiveCheckout = z.infer<typeof HostedArchiveCheckoutSchema>
|
||||
export const HostedArchivePullRequestSchema = z
|
||||
.object({
|
||||
number: z.number().int().positive(),
|
||||
title: z.string().max(300),
|
||||
state: z.enum(['open', 'closed', 'merged', 'draft']),
|
||||
url: z.url({ protocol: /^https$/ }).max(2048),
|
||||
branch: z.string().max(255).optional()
|
||||
})
|
||||
.strict()
|
||||
export type HostedArchivePullRequest = z.infer<typeof HostedArchivePullRequestSchema>
|
||||
|
||||
export const HostedArchiveRequestSchema = z
|
||||
.object({
|
||||
pullRequests: z.array(HostedArchivePullRequestSchema).max(50).optional(),
|
||||
checkouts: z.array(HostedArchiveCheckoutSchema).max(20).optional()
|
||||
})
|
||||
.strict()
|
||||
export type HostedArchiveRequest = z.infer<typeof HostedArchiveRequestSchema>
|
||||
|
||||
export const HostedArchiveResponseSchema = z
|
||||
.object({
|
||||
archive: z
|
||||
.object({
|
||||
capturedAt: Millis,
|
||||
source: z.enum(['vm', 'client', 'none']),
|
||||
metadata: z
|
||||
.object({
|
||||
checkouts: z.array(HostedArchiveCheckoutSchema).max(20),
|
||||
pullRequests: z.array(HostedArchivePullRequestSchema).max(50).optional()
|
||||
})
|
||||
.strict()
|
||||
})
|
||||
.strict()
|
||||
.nullable()
|
||||
})
|
||||
.strict()
|
||||
export type HostedArchiveResponse = z.infer<typeof HostedArchiveResponseSchema>
|
||||
|
||||
Reference in New Issue
Block a user