From c2ebfd00aa4090dd2d58d79e381705ee2d52264a Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Sun, 8 Nov 2026 20:22:19 -0800 Subject: [PATCH] feat(cloud): sandbox control-plane Worker (white-label Vercel) (#11) * feat(cloud): sandbox control-plane Worker (Vercel behind an interface, entitlements, usage, egress proxy) * ci: collect sandbox-control tests --- cloud/apps/sandbox-control/.gitignore | 3 + .../sandbox-control/migrations/0001_init.sql | 52 +++ cloud/apps/sandbox-control/package.json | 24 ++ cloud/apps/sandbox-control/src/auth.ts | 73 +++++ .../apps/sandbox-control/src/egress-proxy.ts | 127 ++++++++ .../apps/sandbox-control/src/entitlements.ts | 89 ++++++ cloud/apps/sandbox-control/src/env.ts | 33 ++ cloud/apps/sandbox-control/src/index.ts | 194 +++++++++++ cloud/apps/sandbox-control/src/lifecycle.ts | 301 ++++++++++++++++++ .../sandbox-control/src/network-policy.ts | 110 +++++++ cloud/apps/sandbox-control/src/reconcile.ts | 98 ++++++ .../sandbox-control/src/sandbox-provider.ts | 90 ++++++ .../src/sandbox-vm-commands.ts | 89 ++++++ cloud/apps/sandbox-control/src/store.ts | 173 ++++++++++ cloud/apps/sandbox-control/src/usage.ts | 75 +++++ .../sandbox-control/src/vercel-sandbox-api.ts | 255 +++++++++++++++ cloud/apps/sandbox-control/src/wire.ts | 70 ++++ cloud/apps/sandbox-control/test/d1-fake.ts | 64 ++++ .../sandbox-control/test/egress-proxy.test.ts | 81 +++++ .../apps/sandbox-control/test/fake-vercel.ts | 161 ++++++++++ cloud/apps/sandbox-control/test/harness.ts | 98 ++++++ .../test/lifecycle-flow.test.ts | 190 +++++++++++ .../test/network-policy.test.ts | 63 ++++ .../test/vercel-sandbox-api.test.ts | 95 ++++++ cloud/apps/sandbox-control/tsconfig.json | 14 + cloud/apps/sandbox-control/vitest.config.ts | 7 + cloud/apps/sandbox-control/wrangler.jsonc | 28 ++ cloud/pnpm-lock.yaml | 28 ++ .../check-test-file-runners.mjs | 9 + 29 files changed, 2694 insertions(+) create mode 100644 cloud/apps/sandbox-control/.gitignore create mode 100644 cloud/apps/sandbox-control/migrations/0001_init.sql create mode 100644 cloud/apps/sandbox-control/package.json create mode 100644 cloud/apps/sandbox-control/src/auth.ts create mode 100644 cloud/apps/sandbox-control/src/egress-proxy.ts create mode 100644 cloud/apps/sandbox-control/src/entitlements.ts create mode 100644 cloud/apps/sandbox-control/src/env.ts create mode 100644 cloud/apps/sandbox-control/src/index.ts create mode 100644 cloud/apps/sandbox-control/src/lifecycle.ts create mode 100644 cloud/apps/sandbox-control/src/network-policy.ts create mode 100644 cloud/apps/sandbox-control/src/reconcile.ts create mode 100644 cloud/apps/sandbox-control/src/sandbox-provider.ts create mode 100644 cloud/apps/sandbox-control/src/sandbox-vm-commands.ts create mode 100644 cloud/apps/sandbox-control/src/store.ts create mode 100644 cloud/apps/sandbox-control/src/usage.ts create mode 100644 cloud/apps/sandbox-control/src/vercel-sandbox-api.ts create mode 100644 cloud/apps/sandbox-control/src/wire.ts create mode 100644 cloud/apps/sandbox-control/test/d1-fake.ts create mode 100644 cloud/apps/sandbox-control/test/egress-proxy.test.ts create mode 100644 cloud/apps/sandbox-control/test/fake-vercel.ts create mode 100644 cloud/apps/sandbox-control/test/harness.ts create mode 100644 cloud/apps/sandbox-control/test/lifecycle-flow.test.ts create mode 100644 cloud/apps/sandbox-control/test/network-policy.test.ts create mode 100644 cloud/apps/sandbox-control/test/vercel-sandbox-api.test.ts create mode 100644 cloud/apps/sandbox-control/tsconfig.json create mode 100644 cloud/apps/sandbox-control/vitest.config.ts create mode 100644 cloud/apps/sandbox-control/wrangler.jsonc diff --git a/cloud/apps/sandbox-control/.gitignore b/cloud/apps/sandbox-control/.gitignore new file mode 100644 index 00000000000..18433e3fe1e --- /dev/null +++ b/cloud/apps/sandbox-control/.gitignore @@ -0,0 +1,3 @@ +.dev.vars +.wrangler/ +node_modules/ diff --git a/cloud/apps/sandbox-control/migrations/0001_init.sql b/cloud/apps/sandbox-control/migrations/0001_init.sql new file mode 100644 index 00000000000..077d6a181c2 --- /dev/null +++ b/cloud/apps/sandbox-control/migrations/0001_init.sql @@ -0,0 +1,52 @@ +-- One row per Orca Cloud workspace sandbox. Written before any provider call (write-ahead), so a +-- crash mid-create leaves a row the cron reconciles instead of an untracked VM. +CREATE TABLE sandboxes ( + id TEXT PRIMARY KEY, + account_id TEXT NOT NULL, + user_id TEXT NOT NULL, + workspace_id TEXT NOT NULL, + provider TEXT NOT NULL, + provider_name TEXT NOT NULL UNIQUE, + status TEXT NOT NULL, + machine_size TEXT NOT NULL, + region TEXT NOT NULL, + -- Bumped on every transition; lifecycle writes compare-and-set on it. + generation INTEGER NOT NULL DEFAULT 0, + session_id TEXT, + session_started_at INTEGER, + session_deadline_at INTEGER, + lease_expires_at INTEGER, + -- Start of the not-yet-reported usage segment of the current session. + usage_settled_at INTEGER, + last_error TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + destroyed_at INTEGER +); + +CREATE UNIQUE INDEX sandboxes_live_workspace + ON sandboxes (account_id, workspace_id) WHERE status <> 'destroyed'; +CREATE INDEX sandboxes_status ON sandboxes (status, updated_at); + +-- Every snapshot id seen for a sandbox, so destroy can prove none outlive it. +CREATE TABLE sandbox_snapshots ( + snapshot_id TEXT PRIMARY KEY, + sandbox_id TEXT NOT NULL REFERENCES sandboxes (id), + recorded_at INTEGER NOT NULL, + deleted_at INTEGER +); +CREATE INDEX sandbox_snapshots_sandbox ON sandbox_snapshots (sandbox_id); + +-- Usage segments owed to billing. Written before the row's settle cursor advances, so a billing +-- outage delays reports instead of dropping them; usage_id makes resends idempotent upstream. +CREATE TABLE usage_outbox ( + usage_id TEXT PRIMARY KEY, + sandbox_id TEXT NOT NULL REFERENCES sandboxes (id), + account_id TEXT NOT NULL, + machine_size TEXT NOT NULL, + started_at INTEGER NOT NULL, + ended_at INTEGER NOT NULL, + attempts INTEGER NOT NULL DEFAULT 0, + sent_at INTEGER +); +CREATE INDEX usage_outbox_pending ON usage_outbox (sent_at, started_at); diff --git a/cloud/apps/sandbox-control/package.json b/cloud/apps/sandbox-control/package.json new file mode 100644 index 00000000000..1da9a6dd95b --- /dev/null +++ b/cloud/apps/sandbox-control/package.json @@ -0,0 +1,24 @@ +{ + "name": "@orca-cloud/sandbox-control", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "wrangler dev --port 8791 --ip 127.0.0.1", + "db:migrate:local": "wrangler d1 migrations apply SANDBOX_DB --local", + "typecheck": "tsc --noEmit", + "test": "vitest run" + }, + "dependencies": { + "jose": "^6.2.13", + "zod": "^4.6.7" + }, + "devDependencies": { + "@cloudflare/workers-types": "^5.20261103.1", + "@types/node": "^24.19.2", + "tsx": "^4.23.15", + "typescript": "^5.9.3", + "vitest": "^5.0.5", + "wrangler": "^4.157.0" + } +} diff --git a/cloud/apps/sandbox-control/src/auth.ts b/cloud/apps/sandbox-control/src/auth.ts new file mode 100644 index 00000000000..46aaa83409f --- /dev/null +++ b/cloud/apps/sandbox-control/src/auth.ts @@ -0,0 +1,73 @@ +// Desktop callers present a short-lived ES256 JWT minted by Orca Cloud auth (aud=orca-sandbox, +// purpose=sandbox), verified against the auth JWKS — the billing Worker's pattern. +import { + createLocalJWKSet, + createRemoteJWKSet, + jwtVerify, + type JSONWebKeySet, + type JWTVerifyGetKey +} from 'jose' +import { z } from 'zod' +import type { Env } from './env' + +const ID = z + .string() + .min(1) + .max(128) + .regex(/^[A-Za-z0-9_.:-]+$/) + +const ClaimsSchema = z.object({ + sub: ID, + org: ID.optional(), + purpose: z.literal('sandbox') +}) + +export type SandboxCaller = { userId: string; accountId: string } + +const keySets = new Map() + +function keysFor(env: Env): JWTVerifyGetKey { + const source = env.SANDBOX_JWKS_URL ?? `inline:${env.SANDBOX_JWKS ?? ''}` + let keys = keySets.get(source) + if (!keys) { + keys = env.SANDBOX_JWKS_URL + ? createRemoteJWKSet(new URL(env.SANDBOX_JWKS_URL)) + : createLocalJWKSet(JSON.parse(env.SANDBOX_JWKS ?? '{"keys":[]}') as JSONWebKeySet) + keySets.set(source, keys) + } + return keys +} + +export function readBearer(value: string | null): string | null { + return /^Bearer ([^\s]+)$/.exec(value ?? '')?.[1] ?? null +} + +/** Same account rule as billing: an org pays for its members, otherwise the user is the account. */ +export function accountIdFor(claims: { sub: string; org?: string }): string { + return claims.org ? `org:${claims.org}` : `user:${claims.sub}` +} + +export async function authenticateDesktop( + env: Env, + request: Request, + now: number = Date.now() +): Promise { + const token = readBearer(request.headers.get('authorization')) + if (!token) { + return null + } + try { + const { payload } = await jwtVerify(token, keysFor(env), { + audience: env.SANDBOX_JWT_AUDIENCE, + algorithms: ['ES256'], + maxTokenAge: '15m', + requiredClaims: ['exp', 'iat'], + currentDate: new Date(now), + ...(env.SANDBOX_AUTH_ISSUER ? { issuer: env.SANDBOX_AUTH_ISSUER } : {}) + }) + const claims = ClaimsSchema.parse(payload) + return { userId: claims.sub, accountId: accountIdFor(claims) } + } catch { + return null + } +} diff --git a/cloud/apps/sandbox-control/src/egress-proxy.ts b/cloud/apps/sandbox-control/src/egress-proxy.ts new file mode 100644 index 00000000000..d5169f0bfdc --- /dev/null +++ b/cloud/apps/sandbox-control/src/egress-proxy.ts @@ -0,0 +1,127 @@ +// Target of Vercel network-policy `forwardURL` rules. Vercel proves which sandbox sent the request +// with an OIDC token; we swap the VM's placeholder for the real credential and return a normally +// framed response (works around undici hanging on transform-rule responses). Real tokens never +// enter the VM or Vercel's policy store. +import { createRemoteJWKSet, decodeJwt, jwtVerify, type JWTVerifyGetKey } from 'jose' +import type { SandboxRow } from './store' + +const VERCEL_OIDC_HOSTNAME = 'oidc.vercel.com' +/** Hosts whose clients are Node (undici); git/curl hosts use transform rules instead. */ +export const PROXIED_HOSTS: ReadonlySet = new Set(['api.github.com']) +const DROP_HEADERS = [ + 'vercel-forwarded-host', + 'vercel-forwarded-scheme', + 'vercel-forwarded-port', + 'vercel-forwarded-path', + 'vercel-sandbox-oidc-token', + 'host', + 'connection', + 'keep-alive', + 'proxy-authorization', + 'transfer-encoding', + 'upgrade' +] + +export type CredentialBroker = { + /** Authorization header value for `host` on behalf of this sandbox's workspace, or null. */ + authorizationFor(row: SandboxRow, host: string): Promise +} + +/** Until the GitHub App installation-token minter exists, proxied hosts are refused. */ +export const NO_CREDENTIALS: CredentialBroker = { authorizationFor: async () => null } + +export type EgressProxyDeps = { + teamId: string + projectId: string + findSandbox: (providerName: string) => Promise + placeholderFor: (row: SandboxRow) => string + credentials: CredentialBroker + fetch?: typeof fetch + /** Tests inject a local key set; production fetches `/.well-known/jwks`. */ + keysForIssuer?: (issuer: string) => JWTVerifyGetKey +} + +const remoteKeys = new Map() +function defaultKeys(issuer: string): JWTVerifyGetKey { + let keys = remoteKeys.get(issuer) + if (!keys) { + keys = createRemoteJWKSet(new URL(`${issuer.replace(/\/$/, '')}/.well-known/jwks`)) + remoteKeys.set(issuer, keys) + } + return keys +} + +function deny(reason: string): Response { + return new Response(reason, { status: 403, headers: { 'content-type': 'text/plain' } }) +} + +/** The audience Vercel signs: the forwardURL, i.e. this request's URL minus the forwarded path. */ +function forwardAudience(requestUrl: URL, forwardedPath: string): string { + const strip = (path: string) => path.replace(/\/+$/, '') || '/' + const suffix = strip(new URL(forwardedPath, requestUrl.origin).pathname) + let pathname = strip(requestUrl.pathname) + if (suffix !== '/' && pathname.endsWith(suffix)) { + pathname = strip(pathname.slice(0, -suffix.length)) + } + return pathname === '/' ? requestUrl.origin : requestUrl.origin + pathname +} + +export async function handleEgressProxy(request: Request, deps: EgressProxyDeps): Promise { + const host = request.headers.get('vercel-forwarded-host')?.toLowerCase() + const scheme = request.headers.get('vercel-forwarded-scheme') + const port = request.headers.get('vercel-forwarded-port') + const path = request.headers.get('vercel-forwarded-path') + const token = request.headers.get('vercel-sandbox-oidc-token') + if (!host || !scheme || !port || !path || !token || !path.startsWith('/')) { + return deny('missing_proxy_headers') + } + if (!PROXIED_HOSTS.has(host) || scheme !== 'https' || port !== '443') { + return deny('host_not_proxied') + } + let claims: Record + try { + const issuer = String(decodeJwt(token).iss ?? '') + const issuerUrl = new URL(issuer) + if (issuerUrl.protocol !== 'https:' || issuerUrl.hostname !== VERCEL_OIDC_HOSTNAME) { + return deny('bad_issuer') + } + const verified = await jwtVerify(token, (deps.keysForIssuer ?? defaultKeys)(issuer), { + issuer, + audience: forwardAudience(new URL(request.url), path), + algorithms: ['RS256'], + clockTolerance: 60 + }) + claims = verified.payload + } catch { + return deny('bad_oidc_token') + } + if (claims.team_id !== deps.teamId || claims.project_id !== deps.projectId) { + return deny('foreign_sandbox') + } + const name = typeof claims.sandbox_name === 'string' ? claims.sandbox_name : null + const row = name ? await deps.findSandbox(name) : null + if (!row || row.status !== 'running') { + return deny('sandbox_not_running') + } + if (request.headers.get('authorization') !== deps.placeholderFor(row)) { + return deny('placeholder_missing') + } + const authorization = await deps.credentials.authorizationFor(row, host) + if (!authorization) { + return deny('credential_unavailable') + } + const headers = new Headers(request.headers) + for (const name of DROP_HEADERS) { + headers.delete(name) + } + headers.set('authorization', authorization) + const hasBody = request.method !== 'GET' && request.method !== 'HEAD' + const upstream = await (deps.fetch ?? fetch)(`https://${host}${path}`, { + method: request.method, + headers, + body: hasBody ? await request.arrayBuffer() : undefined, + redirect: 'manual' + }) + // Re-wrapping lets the runtime frame the body (content-length or chunked) for the VM's client. + return new Response(upstream.body, { status: upstream.status, headers: upstream.headers }) +} diff --git a/cloud/apps/sandbox-control/src/entitlements.ts b/cloud/apps/sandbox-control/src/entitlements.ts new file mode 100644 index 00000000000..ae24d148c36 --- /dev/null +++ b/cloud/apps/sandbox-control/src/entitlements.ts @@ -0,0 +1,89 @@ +// Billing Worker client (cloud/apps/billing on nwparker/billing-worker): start checks and usage. +// Fails closed: without a configured billing URL no sandbox starts, except the local-dev stub. +import { z } from 'zod' +import type { Env } from './env' + +export type MachineSize = 'standard' | 'large' | 'xlarge' + +export type StartDecision = { allow: boolean; reason: string | null } + +export type UsageSegment = { + usageId: string + accountId: string + sandboxId: string + machineSize: MachineSize + startedAt: number + endedAt: number +} + +export type UsageAnswer = { canStartSandbox: boolean; reason: string | null } + +export type EntitlementClient = { + checkStart(request: { + accountId: string + machineSize: MachineSize + runningSandboxes: number + }): Promise + reportUsage(segment: UsageSegment): Promise +} + +export class BillingUnavailableError extends Error { + constructor(message: string) { + super(message) + this.name = 'BillingUnavailableError' + } +} + +const StartAnswer = z.looseObject({ allow: z.boolean(), reason: z.string().nullable() }) +const UsageResponse = z.looseObject({ canStartSandbox: z.boolean(), reason: z.string().nullable() }) + +/** Reasons that mean running sandboxes must stop, not just that new starts wait. */ +export const STOP_RUNNING_REASONS: ReadonlySet = new Set([ + 'included_hours_exhausted', + 'overage_cap_reached', + 'subscription_inactive', + 'no_subscription' +]) + +export function createEntitlementClient(env: Env, doFetch: typeof fetch = fetch): EntitlementClient { + if (env.ENTITLEMENTS_MODE === 'stub') { + if (env.SANDBOX_ENV === 'prod') { + throw new BillingUnavailableError('stub entitlements refused in prod') + } + return { + checkStart: async () => ({ allow: true, reason: null }), + reportUsage: async () => ({ canStartSandbox: true, reason: null }) + } + } + const base = env.BILLING_URL + const token = env.CONTROL_PLANE_TOKEN + async function post(path: string, body: unknown, schema: z.ZodType): Promise { + if (!base || !token) { + throw new BillingUnavailableError('billing not configured') + } + let response: Response + try { + response = await doFetch(new URL(path, base), { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify(body), + redirect: 'manual', + signal: AbortSignal.timeout(10_000) + }) + } catch { + throw new BillingUnavailableError('billing unreachable') + } + if (!response.ok) { + throw new BillingUnavailableError(`billing answered ${response.status}`) + } + const parsed = schema.safeParse(await response.json().catch(() => null)) + if (!parsed.success) { + throw new BillingUnavailableError('billing answered an unexpected body') + } + return parsed.data + } + return { + checkStart: (request) => post('/v1/internal/entitlements/check', request, StartAnswer), + reportUsage: (segment) => post('/v1/internal/usage', segment, UsageResponse) + } +} diff --git a/cloud/apps/sandbox-control/src/env.ts b/cloud/apps/sandbox-control/src/env.ts new file mode 100644 index 00000000000..2ce0646fd46 --- /dev/null +++ b/cloud/apps/sandbox-control/src/env.ts @@ -0,0 +1,33 @@ +// Sandbox control-plane bindings. Secrets are set with `wrangler secret put` (stdin), never in +// wrangler.jsonc or git. Deployed environments live in the private deploy config. +export type Env = { + SANDBOX_DB: D1Database + /** `prod` | `dev` | `local`; tagged on every provider resource so environments never mix. */ + SANDBOX_ENV: string + + // Vercel: Orca's own Pro team. A team-scoped token limited to Sandbox; users never see it. + VERCEL_TOKEN?: string + VERCEL_TEAM_ID?: string + VERCEL_PROJECT_ID?: string + VERCEL_REGION?: string + /** Overrides the Vercel API origin (tests only). */ + VERCEL_API_BASE?: string + /** Golden image (orcad + agent CLIs) every new sandbox boots from; see image/. */ + GOLDEN_SNAPSHOT_ID?: string + + // Desktop tokens: short-lived ES256 JWTs from Orca Cloud auth, aud=SANDBOX_JWT_AUDIENCE. + // SANDBOX_JWKS (inline set) is for local dev and tests only. + SANDBOX_JWKS_URL?: string + SANDBOX_JWKS?: string + SANDBOX_AUTH_ISSUER?: string + SANDBOX_JWT_AUDIENCE: string + + // Billing Worker (entitlement checks + usage). CONTROL_PLANE_TOKEN is shared with it. + BILLING_URL?: string + CONTROL_PLANE_TOKEN?: string + /** `stub` allows every start and records nothing: local dev only, refused when SANDBOX_ENV=prod. */ + ENTITLEMENTS_MODE?: string + + /** Public URL of this Worker's /v1/egress/proxy; Vercel forwardURL rules point here. */ + EGRESS_PROXY_URL?: string +} diff --git a/cloud/apps/sandbox-control/src/index.ts b/cloud/apps/sandbox-control/src/index.ts new file mode 100644 index 00000000000..0420b2fcd69 --- /dev/null +++ b/cloud/apps/sandbox-control/src/index.ts @@ -0,0 +1,194 @@ +// Orca sandbox control plane: owns Orca's Vercel credentials and runs one persistent sandbox per +// Orca Cloud workspace for the signed-in account. Desktop routes (Orca JWT, aud=orca-sandbox): +// POST /v1/sandboxes {workspaceId, machineSize, region?, repo?} -> {sandbox, connection} +// GET /v1/sandboxes -> {sandboxes} +// GET /v1/sandboxes/:id -> {sandbox} +// POST /v1/sandboxes/:id/suspend | /resume | /rollover -> {sandbox[, connection]} +// POST /v1/sandboxes/:id/lease {extendMs} -> {sandbox} +// DELETE /v1/sandboxes/:id -> {sandbox} +// Vercel network-policy forwardURL target (Vercel sandbox OIDC): /v1/egress/proxy +import { authenticateDesktop } from './auth' +import { NO_CREDENTIALS, handleEgressProxy, type CredentialBroker } from './egress-proxy' +import { BillingUnavailableError, createEntitlementClient, type EntitlementClient } from './entitlements' +import type { Env } from './env' +import { + LifecycleError, + createSandbox, + destroySandbox, + githubPlaceholderHeader, + renewLease, + resumeSandbox, + rolloverSandbox, + suspendSandbox, + type LifecycleDeps +} from './lifecycle' +import { reconcile } from './reconcile' +import { ProviderError, type SandboxProviderApi } from './sandbox-provider' +import { VmCommandError } from './sandbox-vm-commands' +import { getSandbox, getSandboxByProviderName, listAccountSandboxes } from './store' +import { createVercelSandboxApi } from './vercel-sandbox-api' +import { CreateRequestSchema, LeaseRequestSchema, readyView, sandboxView } from './wire' + +const MAX_JSON_BYTES = 16 * 1024 + +export type Deps = { + provider?: SandboxProviderApi + entitlements?: EntitlementClient + credentials?: CredentialBroker + now?: () => number + egressFetch?: typeof fetch + egressKeysForIssuer?: Parameters[1]['keysForIssuer'] +} + +function json(body: unknown, status = 200): Response { + return Response.json(body, { status, headers: { 'cache-control': 'no-store' } }) +} + +function fail(error: string, status: number, reason?: string): Response { + return json(reason ? { error, reason } : { error }, status) +} + +async function readJson(request: Request): Promise { + const text = await request.text() + if (text.length > MAX_JSON_BYTES) { + throw new SyntaxError('too_large') + } + return text ? JSON.parse(text) : {} +} + +/** Lifecycle routes refuse to run half-configured rather than guess at Orca's provider account. */ +function lifecycleDeps(env: Env, deps: Deps): LifecycleDeps | null { + if (!env.GOLDEN_SNAPSHOT_ID || (!deps.provider && (!env.VERCEL_TOKEN || !env.VERCEL_TEAM_ID || !env.VERCEL_PROJECT_ID))) { + return null + } + return { + db: env.SANDBOX_DB, + provider: + deps.provider ?? + createVercelSandboxApi({ + token: env.VERCEL_TOKEN!, + teamId: env.VERCEL_TEAM_ID!, + projectId: env.VERCEL_PROJECT_ID!, + baseUrl: env.VERCEL_API_BASE + }), + entitlements: deps.entitlements ?? createEntitlementClient(env), + now: deps.now ?? Date.now, + config: { + environment: env.SANDBOX_ENV, + goldenSnapshotId: env.GOLDEN_SNAPSHOT_ID, + defaultRegion: env.VERCEL_REGION ?? 'iad1', + egressProxyUrl: env.EGRESS_PROXY_URL + } + } +} + +export async function handleRequest(request: Request, env: Env, deps: Deps = {}): Promise { + const url = new URL(request.url) + try { + if (url.pathname === '/v1/egress/proxy' || url.pathname.startsWith('/v1/egress/proxy/')) { + if (!env.VERCEL_TEAM_ID || !env.VERCEL_PROJECT_ID) { + return fail('not_configured', 503) + } + return await handleEgressProxy(request, { + teamId: env.VERCEL_TEAM_ID, + projectId: env.VERCEL_PROJECT_ID, + findSandbox: (name) => getSandboxByProviderName(env.SANDBOX_DB, name), + placeholderFor: githubPlaceholderHeader, + credentials: deps.credentials ?? NO_CREDENTIALS, + fetch: deps.egressFetch, + keysForIssuer: deps.egressKeysForIssuer + }) + } + const match = /^\/v1\/sandboxes(?:\/([A-Za-z0-9_]+)(?:\/(suspend|resume|lease|rollover))?)?$/.exec(url.pathname) + if (!match) { + return fail('not_found', 404) + } + const caller = await authenticateDesktop(env, request, (deps.now ?? Date.now)()) + if (!caller) { + return fail('unauthorized', 401) + } + const life = lifecycleDeps(env, deps) + if (!life) { + return fail('not_configured', 503) + } + const [, id, action] = match + if (!id) { + if (request.method === 'GET') { + const rows = await listAccountSandboxes(env.SANDBOX_DB, caller.accountId) + return json({ sandboxes: rows.map(sandboxView) }) + } + if (request.method !== 'POST') { + return fail('not_found', 404) + } + const body = CreateRequestSchema.safeParse(await readJson(request)) + if (!body.success) { + return fail('invalid_request', 400) + } + const ready = await createSandbox(life, caller, body.data) + return json(readyView(ready.row, ready.connection)) + } + const row = await getSandbox(env.SANDBOX_DB, id) + // Another account's sandbox is indistinguishable from a missing one. + if (!row || row.account_id !== caller.accountId) { + return fail('not_found', 404) + } + const route = `${request.method} ${action ?? ''}` + switch (route) { + case 'GET ': + return json({ sandbox: sandboxView(row) }) + case 'DELETE ': + return json({ sandbox: sandboxView(await destroySandbox(life, row)) }) + case 'POST suspend': + return json({ sandbox: sandboxView(await suspendSandbox(life, row)) }) + case 'POST resume': { + const ready = await resumeSandbox(life, row) + return json(readyView(ready.row, ready.connection)) + } + case 'POST rollover': { + const ready = await rolloverSandbox(life, row) + return json(readyView(ready.row, ready.connection)) + } + case 'POST lease': { + const body = LeaseRequestSchema.safeParse(await readJson(request)) + if (!body.success) { + return fail('invalid_request', 400) + } + return json({ sandbox: sandboxView(await renewLease(life, row, body.data.extendMs)) }) + } + default: + return fail('not_found', 404) + } + } catch (error) { + return errorResponse(error) + } +} + +function errorResponse(error: unknown): Response { + if (error instanceof SyntaxError) { + return fail('invalid_json', 400) + } + if (error instanceof LifecycleError) { + const status = { not_found: 404, conflict: 409, entitlement_denied: 403, provider_unavailable: 502 }[error.code] + return fail(error.code, status, error.reason) + } + if (error instanceof BillingUnavailableError) { + return fail('billing_unavailable', 503) + } + // Provider and VM failures are summarized; their bodies never reach the desktop. + if (error instanceof ProviderError || error instanceof VmCommandError) { + return fail('provider_unavailable', 502, error instanceof VmCommandError ? error.code : undefined) + } + return fail('internal_error', 500) +} + +export default { + fetch(request: Request, env: Env): Promise { + return handleRequest(request, env) + }, + async scheduled(_controller: ScheduledController, env: Env, ctx: ExecutionContext): Promise { + const life = lifecycleDeps(env, {}) + if (life) { + ctx.waitUntil(reconcile(life).then(() => undefined)) + } + } +} satisfies ExportedHandler diff --git a/cloud/apps/sandbox-control/src/lifecycle.ts b/cloud/apps/sandbox-control/src/lifecycle.ts new file mode 100644 index 00000000000..1ff07ff58a9 --- /dev/null +++ b/cloud/apps/sandbox-control/src/lifecycle.ts @@ -0,0 +1,301 @@ +// Sandbox lifecycle: one persistent provider sandbox per Orca workspace, with an Orca-owned lease, +// a forced stop before the provider's 24h cap, usage metering and write-ahead rows. +import type { EntitlementClient, MachineSize } from './entitlements' +import { compileNetworkPolicy, type CredentialRoute } from './network-policy' +import type { ProviderSandboxState, SandboxProviderApi } from './sandbox-provider' +import { ORCAD_PORT, drainOrcad, startOrcad, type OrcadReady } from './sandbox-vm-commands' +import { + NO_SESSION, + countActiveSandboxes, + getSandbox, + insertSandbox, + liveSandboxForWorkspace, + markSnapshotsDeleted, + recordSnapshots, + recordedSnapshotIds, + transition, + type SandboxRow, + type SessionFields +} from './store' +import { flushUsage, queueUsage } from './usage' + +export const LEASE_MS = 30 * 60_000 +const PROVIDER_SESSION_CAP_MS = 24 * 60 * 60_000 +/** Orca stops 10 min early so it, not the provider, ends the session (drain first). */ +export const SESSION_DEADLINE_MS = PROVIDER_SESSION_CAP_MS - 10 * 60_000 +const VCPUS: Record = { standard: 4, large: 8, xlarge: 16 } + +export type LifecycleDeps = { + db: D1Database + provider: SandboxProviderApi + entitlements: EntitlementClient + now: () => number + config: { + environment: string + goldenSnapshotId: string + defaultRegion: string + egressProxyUrl?: string + } +} + +export class LifecycleError extends Error { + constructor( + readonly code: 'not_found' | 'conflict' | 'entitlement_denied' | 'provider_unavailable', + readonly reason?: string + ) { + super(reason ? `${code}:${reason}` : code) + this.name = 'LifecycleError' + } +} + +export type Ready = { row: SandboxRow; connection: OrcadReady } + +function randomId(): string { + const bytes = crypto.getRandomValues(new Uint8Array(12)) + return `sbx_${Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join('')}` +} + +/** What the VM sends as its GitHub API credential; the egress proxy swaps it for the real token. */ +export function githubPlaceholderHeader(row: SandboxRow): string { + return `Bearer orca-ph-${row.id}` +} + +function egressPolicy(deps: LifecycleDeps, row: SandboxRow) { + const credentials: CredentialRoute[] = deps.config.egressProxyUrl + ? [ + { + host: 'api.github.com', + header: 'authorization', + placeholder: githubPlaceholderHeader(row), + via: 'forward', + forwardURL: deps.config.egressProxyUrl + } + ] + : [] + return compileNetworkPolicy({ mode: 'allowlist', credentials }) +} + +export function sessionFields(state: ProviderSandboxState, now: number): SessionFields { + const startedAt = state.session.startedAt ?? now + return { + session_id: state.session.id, + session_started_at: startedAt, + session_deadline_at: startedAt + SESSION_DEADLINE_MS, + lease_expires_at: startedAt + state.session.timeoutMs, + usage_settled_at: startedAt + } +} + +async function claim(deps: LifecycleDeps, row: SandboxRow, patch: Partial): Promise { + const next = await transition(deps.db, row, patch, deps.now()) + if (!next) { + throw new LifecycleError('conflict', 'concurrent_transition') + } + return next +} + +async function requireStart(deps: LifecycleDeps, accountId: string, machineSize: MachineSize) { + const runningSandboxes = await countActiveSandboxes(deps.db, accountId) + const decision = await deps.entitlements.checkStart({ accountId, machineSize, runningSandboxes }) + if (!decision.allow) { + throw new LifecycleError('entitlement_denied', decision.reason ?? 'denied') + } +} + +async function boot(deps: LifecycleDeps, row: SandboxRow, state: ProviderSandboxState, repo?: { url: string; ref?: string }): Promise { + if (state.sandbox.tags.orcaSandbox !== row.id) { + throw new LifecycleError('conflict', 'ownership_mismatch') + } + const connection = await startOrcad(deps.provider, state, { + sandboxId: row.id, + repo, + githubPlaceholder: deps.config.egressProxyUrl ? githubPlaceholderHeader(row) : undefined + }) + const running = await claim(deps, row, { status: 'running', last_error: null, ...sessionFields(state, deps.now()) }) + return { row: running, connection } +} + +export async function createSandbox( + deps: LifecycleDeps, + caller: { accountId: string; userId: string }, + request: { workspaceId: string; machineSize: MachineSize; region?: string; repo?: { url: string; ref?: string } } +): Promise { + const existing = await liveSandboxForWorkspace(deps.db, caller.accountId, request.workspaceId) + if (existing) { + if (existing.status !== 'running' || !existing.session_id) { + throw new LifecycleError('conflict', `sandbox_${existing.status}`) + } + // Retried create after a lost response: hand back the same environment. + const state = await deps.provider.getSandbox(existing.provider_name, { resume: false }) + if (!state || state.session.status !== 'running') { + throw new LifecycleError('conflict', 'sandbox_not_running') + } + return { row: existing, connection: await startOrcad(deps.provider, state, { sandboxId: existing.id }) } + } + await requireStart(deps, caller.accountId, request.machineSize) + const id = randomId() + const row = await insertSandbox( + deps.db, + { + id, + account_id: caller.accountId, + user_id: caller.userId, + workspace_id: request.workspaceId, + provider: 'vercel', + provider_name: `orca-${deps.config.environment}-${id}`, + machine_size: request.machineSize, + region: request.region ?? deps.config.defaultRegion + }, + deps.now() + ) + let state: ProviderSandboxState + try { + state = await deps.provider.createSandbox({ + name: row.provider_name, + snapshotId: deps.config.goldenSnapshotId, + vcpus: VCPUS[row.machine_size], + region: row.region, + timeoutMs: LEASE_MS, + ports: [ORCAD_PORT], + networkPolicy: egressPolicy(deps, row), + tags: { orcaSandbox: row.id, orcaEnv: deps.config.environment } + }) + } catch (error) { + await failAndRelease(deps, row) + throw error + } + try { + return await boot(deps, row, state, request.repo) + } catch (error) { + await failAndRelease(deps, (await getSandbox(deps.db, row.id)) ?? row) + throw error + } +} + +/** A create that never became usable is destroyed now; the cron retries if this also fails. */ +export async function failAndRelease(deps: LifecycleDeps, row: SandboxRow): Promise { + const failed = await transition(deps.db, row, { status: 'failed', last_error: 'create_failed' }, deps.now()) + if (failed) { + await destroySandbox(deps, failed).catch(() => undefined) + } +} + +async function stopForSuspend(deps: LifecycleDeps, row: SandboxRow): Promise { + if (row.session_id) { + await drainOrcad(deps.provider, row.session_id) + const stopped = await deps.provider.stopSession(row.session_id) + if (stopped.snapshotId) { + await recordSnapshots(deps.db, row.id, [stopped.snapshotId], deps.now()) + } + } + await queueUsage(deps.db, row, deps.now()) + return claim(deps, row, { status: 'suspended', ...NO_SESSION }) +} + +/** The provider ended the session on its own (lease lapsed): bill up to when it stopped. */ +export async function markProviderStopped(deps: LifecycleDeps, row: SandboxRow): Promise { + await queueUsage(deps.db, row, Math.min(deps.now(), row.lease_expires_at ?? deps.now())) + return claim(deps, row, { status: 'suspended', ...NO_SESSION }) +} + +export async function suspendSandbox(deps: LifecycleDeps, row: SandboxRow): Promise { + if (row.status === 'suspended') { + return row + } + if (row.status !== 'running') { + throw new LifecycleError('conflict', `sandbox_${row.status}`) + } + const suspending = await claim(deps, row, { status: 'suspending' }) + const suspended = await stopForSuspend(deps, suspending) + await flushUsage(deps.db, deps.entitlements, deps.now(), row.id) + return suspended +} + +export async function resumeSandbox(deps: LifecycleDeps, row: SandboxRow): Promise { + if (row.status === 'running') { + const state = await deps.provider.getSandbox(row.provider_name, { resume: false }) + if (state?.session.status === 'running') { + return { row, connection: await startOrcad(deps.provider, state, { sandboxId: row.id }) } + } + // The provider ended the session (lease lapsed): fall through to a real resume. + row = await markProviderStopped(deps, row) + } + if (row.status !== 'suspended') { + throw new LifecycleError('conflict', `sandbox_${row.status}`) + } + await requireStart(deps, row.account_id, row.machine_size) + const resuming = await claim(deps, row, { status: 'resuming' }) + return wake(deps, resuming) +} + +async function wake(deps: LifecycleDeps, row: SandboxRow): Promise { + const state = await deps.provider.getSandbox(row.provider_name, { resume: true }) + if (!state) { + await claim(deps, row, { status: 'failed', last_error: 'provider_sandbox_missing' }) + throw new LifecycleError('conflict', 'environment_missing') + } + try { + return await boot(deps, row, state) + } catch (error) { + // Leave it resumable: the disk is intact, only this boot failed. + const current = await getSandbox(deps.db, row.id) + if (current?.status === 'resuming') { + await transition(deps.db, current, { status: 'suspended', ...NO_SESSION, last_error: 'resume_failed' }, deps.now()) + } + throw error + } +} + +/** 24h cap or opportunistic refresh: drain, stop (snapshot), boot the same disk again. */ +export async function rolloverSandbox(deps: LifecycleDeps, row: SandboxRow): Promise { + if (row.status !== 'running') { + throw new LifecycleError('conflict', `sandbox_${row.status}`) + } + const suspending = await claim(deps, row, { status: 'suspending' }) + const stopped = await stopForSuspend(deps, suspending) + await flushUsage(deps.db, deps.entitlements, deps.now(), row.id) + return wake(deps, await claim(deps, stopped, { status: 'resuming' })) +} + +export async function renewLease(deps: LifecycleDeps, row: SandboxRow, extendMs: number): Promise { + if (row.status !== 'running' || !row.session_id || row.lease_expires_at === null || row.session_deadline_at === null) { + throw new LifecycleError('conflict', `sandbox_${row.status}`) + } + const target = Math.min(row.session_deadline_at, deps.now() + extendMs) + if (target <= row.lease_expires_at) { + return row + } + const session = await deps.provider.extendTimeout(row.session_id, target - row.lease_expires_at) + const startedAt = row.session_started_at ?? deps.now() + return claim(deps, row, { lease_expires_at: startedAt + session.timeoutMs }) +} + +export async function destroySandbox(deps: LifecycleDeps, row: SandboxRow): Promise { + if (row.status === 'destroyed') { + return row + } + let destroying = row.status === 'destroying' ? row : await claim(deps, row, { status: 'destroying' }) + const settled = await queueUsage(deps.db, destroying, deps.now()) + if (settled !== destroying.usage_settled_at) { + destroying = await claim(deps, destroying, { usage_settled_at: settled }) + } + await deps.provider.deleteSandbox(destroying.provider_name) + // Persistent sandboxes leave automatic snapshots behind delete; prove none survive. + const listed = await deps.provider.listSnapshots(destroying.provider_name) + await recordSnapshots(deps.db, row.id, listed.map((s) => s.id), deps.now()) + for (const id of await recordedSnapshotIds(deps.db, row.id)) { + if (id !== deps.config.goldenSnapshotId) { + await deps.provider.deleteSnapshot(id) + } + } + const remaining = (await deps.provider.listSnapshots(destroying.provider_name)).filter( + (s) => s.status !== 'deleted' && s.id !== deps.config.goldenSnapshotId + ) + if (remaining.length > 0) { + throw new LifecycleError('provider_unavailable', 'snapshot_cleanup_incomplete') + } + await markSnapshotsDeleted(deps.db, row.id, deps.now()) + const destroyed = await claim(deps, destroying, { status: 'destroyed', ...NO_SESSION, destroyed_at: deps.now() }) + await flushUsage(deps.db, deps.entitlements, deps.now(), row.id) + return destroyed +} diff --git a/cloud/apps/sandbox-control/src/network-policy.ts b/cloud/apps/sandbox-control/src/network-policy.ts new file mode 100644 index 00000000000..1d19a9ae02d --- /dev/null +++ b/cloud/apps/sandbox-control/src/network-policy.ts @@ -0,0 +1,110 @@ +// Fail-closed egress for Orca Cloud sandboxes (Vercel v2 record-form network policy). +// - No "*" entry unless a workspace opts into open egress: unlisted domains do not resolve. +// - Credential hosts end with a 403 rule, so only requests carrying our placeholder get through. +// - Hosts Node clients use are forwarded to our egress proxy instead of transformed: with a +// transform rule Vercel's proxy answers HTTP/1.1 without content-length or transfer-encoding and +// undici fetch hangs on the body (bake-off). curl/git and Go clients are unaffected. + +type Matcher = { exact?: string; startsWith?: string } +export type NetworkPolicyRule = + | { + match?: { headers?: { key: Matcher; value: Matcher }[] } + transform: { headers: Record }[] + } + | { match?: { headers?: { key: Matcher; value: Matcher }[] }; forwardURL: string } + | { response: { statusCode: number; body: string; contentType: string } } + +export type NetworkPolicy = { allow: Record } + +export const EGRESS_DENIED_BODY = 'orca-egress-denied' + +/** Registries, git hosting and agent model/sign-in hosts a coding VM needs. */ +export const DEFAULT_EGRESS_ALLOWLIST: readonly string[] = [ + 'registry.npmjs.org', + 'registry.yarnpkg.com', + 'pypi.org', + 'files.pythonhosted.org', + 'crates.io', + 'index.crates.io', + 'static.crates.io', + 'proxy.golang.org', + 'sum.golang.org', + 'rubygems.org', + 'nodejs.org', + 'archive.ubuntu.com', + 'security.ubuntu.com', + 'codeload.github.com', + 'objects.githubusercontent.com', + 'raw.githubusercontent.com', + 'api.anthropic.com', + 'claude.ai', + 'console.anthropic.com', + 'api.openai.com', + 'auth.openai.com', + 'chatgpt.com', + 'generativelanguage.googleapis.com', + 'oauth2.googleapis.com', + 'accounts.google.com', + '*.onorca.dev' +] + +export type CredentialRoute = { + host: string + /** Header the VM sends with the placeholder value, matched exactly. */ + header: string + placeholder: string +} & ( + | { via: 'transform'; value: string } + | { via: 'forward'; forwardURL: string } +) + +export type EgressPolicyInput = { + mode: 'allowlist' | 'open' + allowlist?: readonly string[] + credentials?: readonly CredentialRoute[] +} + +const HOST = /^(\*\.)?([a-z0-9-]+\.)+[a-z]{2,63}$/ + +export function compileNetworkPolicy(input: EgressPolicyInput): NetworkPolicy { + const allow: Record = {} + for (const host of input.allowlist ?? DEFAULT_EGRESS_ALLOWLIST) { + allow[checkedHost(host)] = [] + } + for (const route of input.credentials ?? []) { + const host = checkedHost(route.host) + if (host.startsWith('*.')) { + throw new Error('credential routes need an exact host') + } + const match = { + headers: [{ key: { exact: route.header.toLowerCase() }, value: { exact: route.placeholder } }] + } + const claim: NetworkPolicyRule = + route.via === 'transform' + ? { match, transform: [{ headers: { [route.header.toLowerCase()]: route.value } }] } + : { match, forwardURL: route.forwardURL } + const rules = allow[host]?.filter((rule) => !('response' in rule)) ?? [] + allow[host] = [...rules, claim, denyRule()] + } + if (input.mode === 'open') { + allow['*'] = [] + } + return { allow } +} + +function denyRule(): NetworkPolicyRule { + return { response: { statusCode: 403, body: EGRESS_DENIED_BODY, contentType: 'text/plain' } } +} + +function checkedHost(host: string): string { + const normalized = host.trim().toLowerCase() + if (!HOST.test(normalized)) { + throw new Error(`invalid egress host: ${host}`) + } + return normalized +} + +/** Basic credential git sends when `http..extraheader` carries the placeholder token. */ +export function gitBasicAuthorization(token: string): string { + return `Basic ${btoa(`x-access-token:${token}`)}` +} diff --git a/cloud/apps/sandbox-control/src/reconcile.ts b/cloud/apps/sandbox-control/src/reconcile.ts new file mode 100644 index 00000000000..a4303f59978 --- /dev/null +++ b/cloud/apps/sandbox-control/src/reconcile.ts @@ -0,0 +1,98 @@ +// Cron pass: the control plane, not the desktop, guarantees metering, the session deadline and cleanup, +// so a closed laptop or a crash mid-call can never leave a sandbox billed-but-untracked. +import { STOP_RUNNING_REASONS } from './entitlements' +import { + destroySandbox, + failAndRelease, + markProviderStopped, + sessionFields, + suspendSandbox, + type LifecycleDeps +} from './lifecycle' +import { listByStatus, transition, type SandboxRow } from './store' +import { flushUsage, queueUsage } from './usage' + +const STUCK_MS = 15 * 60_000 + +export type ReconcileSummary = Record< + 'settled' | 'suspendedForDeadline' | 'suspendedForBilling' | 'leaseLapsed' | 'released' | 'repaired', + number +> + +export async function reconcile(deps: LifecycleDeps): Promise { + const summary: ReconcileSummary = { + settled: 0, + suspendedForDeadline: 0, + suspendedForBilling: 0, + leaseLapsed: 0, + released: 0, + repaired: 0 + } + for (const row of await listByStatus(deps.db, ['running'])) { + await guard(() => reconcileRunning(deps, row, summary)) + } + const now = deps.now() + for (const row of await listByStatus(deps.db, ['creating', 'failed', 'destroying', 'suspending', 'resuming'])) { + if (now - row.updated_at < STUCK_MS) { + continue + } + await guard(async () => { + if (row.status === 'creating') { + await failAndRelease(deps, row) + summary.released++ + } else if (row.status === 'failed' || row.status === 'destroying') { + await destroySandbox(deps, row) + summary.released++ + } else { + await repairTransition(deps, row) + summary.repaired++ + } + }) + } + await flushUsage(deps.db, deps.entitlements, deps.now()) + return summary +} + +async function reconcileRunning(deps: LifecycleDeps, row: SandboxRow, summary: ReconcileSummary): Promise { + const now = deps.now() + if (row.session_deadline_at !== null && now >= row.session_deadline_at) { + // The desktop should have rolled over; stop before the provider's hard cap kills it undrained. + await suspendSandbox(deps, row) + summary.suspendedForDeadline++ + return + } + if (row.lease_expires_at !== null && now >= row.lease_expires_at) { + const state = await deps.provider.getSandbox(row.provider_name, { resume: false }) + if (!state || state.session.status !== 'running') { + await markProviderStopped(deps, row) + summary.leaseLapsed++ + return + } + } + const settled = await queueUsage(deps.db, row, now) + const current = settled === row.usage_settled_at ? row : await transition(deps.db, row, { usage_settled_at: settled }, now) + summary.settled++ + const answer = await flushUsage(deps.db, deps.entitlements, now, row.id) + if (current && answer && !answer.canStartSandbox && STOP_RUNNING_REASONS.has(answer.reason ?? '')) { + await suspendSandbox(deps, current) + summary.suspendedForBilling++ + } +} + +/** A suspend/resume call died mid-way: adopt whatever the provider says is true. */ +async function repairTransition(deps: LifecycleDeps, row: SandboxRow): Promise { + const state = await deps.provider.getSandbox(row.provider_name, { resume: false }) + if (state?.session.status === 'running') { + await transition(deps.db, row, { status: 'running', ...sessionFields(state, deps.now()) }, deps.now()) + return + } + await markProviderStopped(deps, row) +} + +async function guard(step: () => Promise): Promise { + try { + await step() + } catch { + // One bad row must not stall the pass; it is retried next tick. + } +} diff --git a/cloud/apps/sandbox-control/src/sandbox-provider.ts b/cloud/apps/sandbox-control/src/sandbox-provider.ts new file mode 100644 index 00000000000..270fefca426 --- /dev/null +++ b/cloud/apps/sandbox-control/src/sandbox-provider.ts @@ -0,0 +1,90 @@ +// The provider calls the control plane needs, shaped after Vercel Sandbox's persistent-sandbox API. +// Lifecycle code depends only on this, so a second provider (Freestyle, E2B) is one more adapter. +import type { NetworkPolicy } from './network-policy' + +export type ProviderSessionStatus = + | 'pending' + | 'running' + | 'stopping' + | 'stopped' + | 'failed' + | 'aborted' + | 'snapshotting' + +export type ProviderSandbox = { + name: string + status: ProviderSessionStatus + currentSessionId: string + currentSnapshotId: string | null + tags: Record +} + +export type ProviderSession = { + id: string + status: ProviderSessionStatus + startedAt: number | null + /** Session timeout in ms, counted from start. */ + timeoutMs: number +} + +export type ProviderRoute = { port: number; url: string } + +export type ProviderSandboxState = { + sandbox: ProviderSandbox + session: ProviderSession + routes: ProviderRoute[] +} + +export type ProviderSnapshot = { + id: string + status: 'created' | 'deleted' | 'failed' + sourceSessionId: string +} + +export type CreateSandboxParams = { + name: string + snapshotId: string + vcpus: number + region: string + timeoutMs: number + ports: number[] + networkPolicy: NetworkPolicy + tags: Record +} + +export type RunCommandParams = { + command: string + args?: string[] + cwd?: string + env?: Record + sudo?: boolean + timeoutMs: number +} + +export type CommandResult = { exitCode: number; stdout: string; stderr: string } + +export type SandboxProviderApi = { + createSandbox(params: CreateSandboxParams): Promise + /** `resume: true` boots a stopped persistent sandbox from its latest snapshot. Null when absent. */ + getSandbox(name: string, options: { resume: boolean }): Promise + stopSession(sessionId: string): Promise<{ snapshotId: string | null }> + extendTimeout(sessionId: string, durationMs: number): Promise + updateNetworkPolicy(sessionId: string, policy: NetworkPolicy): Promise + runCommand(sessionId: string, params: RunCommandParams): Promise + /** Resolves when already gone. */ + deleteSandbox(name: string): Promise + listSnapshots(name: string): Promise + /** Resolves when already gone. */ + deleteSnapshot(snapshotId: string): Promise +} + +/** Provider failed or answered unexpectedly; the body is never surfaced to callers. */ +export class ProviderError extends Error { + constructor( + readonly status: number, + readonly code: string | null + ) { + super(`provider request failed (${status}${code ? ` ${code}` : ''})`) + this.name = 'ProviderError' + } +} diff --git a/cloud/apps/sandbox-control/src/sandbox-vm-commands.ts b/cloud/apps/sandbox-control/src/sandbox-vm-commands.ts new file mode 100644 index 00000000000..fc216504f79 --- /dev/null +++ b/cloud/apps/sandbox-control/src/sandbox-vm-commands.ts @@ -0,0 +1,89 @@ +// Commands the golden image installs in /opt/orca/bin (see image/provision-golden.sh). The control +// plane only ever runs these fixed entry points in a VM, never user-supplied command lines. +import { z } from 'zod' +import { ProviderError, type ProviderSandboxState, type SandboxProviderApi } from './sandbox-provider' + +export const ORCAD_PORT = 6768 +export const START_COMMAND = '/opt/orca/bin/orca-sandbox-start' +export const DRAIN_COMMAND = '/opt/orca/bin/orca-sandbox-drain' +const START_TIMEOUT_MS = 120_000 +/** Matches the parity plan: in-flight turns get up to 5 min to become durable restart offers. */ +export const DRAIN_TIMEOUT_MS = 5 * 60_000 + +const ReadySchema = z + .object({ + pairingCode: z.string().min(1).max(8192), + projectRoot: z.string().startsWith('/').max(4096) + }) + .strict() + +export type OrcadReady = z.infer + +export class VmCommandError extends Error { + constructor(readonly code: 'orcad_start_failed' | 'orcad_ready_invalid' | 'route_missing') { + super(code) + this.name = 'VmCommandError' + } +} + +export type StartInputs = { + sandboxId: string + repo?: { url: string; ref?: string } + /** Value the VM sends for credential hosts; the egress layer swaps it for the real credential. */ + githubPlaceholder?: string +} + +/** Starts orcad (idempotent per boot) and returns its pairing offer; pairing codes are never logged. */ +export async function startOrcad( + provider: SandboxProviderApi, + state: ProviderSandboxState, + inputs: StartInputs +): Promise { + const route = state.routes.find((candidate) => candidate.port === ORCAD_PORT) + if (!route) { + throw new VmCommandError('route_missing') + } + const env: Record = { + ORCA_SANDBOX_ID: inputs.sandboxId, + ORCA_PUBLIC_WSS: route.url.replace(/^https:/, 'wss:') + } + if (inputs.repo) { + env.ORCA_REPO_URL = inputs.repo.url + if (inputs.repo.ref) { + env.ORCA_REPO_REF = inputs.repo.ref + } + } + if (inputs.githubPlaceholder) { + env.ORCA_GITHUB_PLACEHOLDER = inputs.githubPlaceholder + } + const result = await provider.runCommand(state.session.id, { + command: START_COMMAND, + env, + timeoutMs: START_TIMEOUT_MS + }) + if (result.exitCode !== 0) { + throw new VmCommandError('orcad_start_failed') + } + const lastLine = result.stdout.trim().split('\n').at(-1) ?? '' + try { + return ReadySchema.parse(JSON.parse(lastLine)) + } catch { + throw new VmCommandError('orcad_ready_invalid') + } +} + +/** Best effort: orcad turns working chats into durable restart offers before the VM stops. */ +export async function drainOrcad(provider: SandboxProviderApi, sessionId: string): Promise { + try { + const result = await provider.runCommand(sessionId, { + command: DRAIN_COMMAND, + timeoutMs: DRAIN_TIMEOUT_MS + }) + return result.exitCode === 0 + } catch (error) { + if (error instanceof ProviderError) { + return false + } + throw error + } +} diff --git a/cloud/apps/sandbox-control/src/store.ts b/cloud/apps/sandbox-control/src/store.ts new file mode 100644 index 00000000000..b8ff00b8766 --- /dev/null +++ b/cloud/apps/sandbox-control/src/store.ts @@ -0,0 +1,173 @@ +// D1 access for sandbox rows. Every transition is a compare-and-set on `generation`, so two +// concurrent lifecycle calls for one sandbox cannot both act on the provider. +import type { MachineSize } from './entitlements' + +export type SandboxStatus = + | 'creating' + | 'running' + | 'suspending' + | 'suspended' + | 'resuming' + | 'destroying' + | 'destroyed' + | 'failed' + +export type SandboxRow = { + id: string + account_id: string + user_id: string + workspace_id: string + provider: string + provider_name: string + status: SandboxStatus + machine_size: MachineSize + region: string + generation: number + session_id: string | null + session_started_at: number | null + session_deadline_at: number | null + lease_expires_at: number | null + usage_settled_at: number | null + last_error: string | null + created_at: number + updated_at: number + destroyed_at: number | null +} + +export type SessionFields = Pick< + SandboxRow, + 'session_id' | 'session_started_at' | 'session_deadline_at' | 'lease_expires_at' | 'usage_settled_at' +> + +export const NO_SESSION: SessionFields = { + session_id: null, + session_started_at: null, + session_deadline_at: null, + lease_expires_at: null, + usage_settled_at: null +} + +/** Counted against the plan's concurrency limit. */ +export const ACTIVE_STATUSES: readonly SandboxStatus[] = ['creating', 'running', 'resuming', 'suspending'] + +export async function insertSandbox( + db: D1Database, + row: Pick< + SandboxRow, + 'id' | 'account_id' | 'user_id' | 'workspace_id' | 'provider' | 'provider_name' | 'machine_size' | 'region' + >, + now: number +): Promise { + await db + .prepare( + `INSERT INTO sandboxes (id, account_id, user_id, workspace_id, provider, provider_name, status, + machine_size, region, generation, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, 'creating', ?, ?, 0, ?, ?)` + ) + .bind(row.id, row.account_id, row.user_id, row.workspace_id, row.provider, row.provider_name, + row.machine_size, row.region, now, now) + .run() + const inserted = await getSandbox(db, row.id) + if (!inserted) { + throw new Error('sandbox insert lost') + } + return inserted +} + +export function getSandbox(db: D1Database, id: string): Promise { + return db.prepare('SELECT * FROM sandboxes WHERE id = ?').bind(id).first() +} + +export function getSandboxByProviderName(db: D1Database, name: string): Promise { + return db.prepare('SELECT * FROM sandboxes WHERE provider_name = ?').bind(name).first() +} + +export function liveSandboxForWorkspace( + db: D1Database, + accountId: string, + workspaceId: string +): Promise { + return db + .prepare(`SELECT * FROM sandboxes WHERE account_id = ? AND workspace_id = ? AND status <> 'destroyed'`) + .bind(accountId, workspaceId) + .first() +} + +export async function listAccountSandboxes(db: D1Database, accountId: string): Promise { + const { results } = await db + .prepare(`SELECT * FROM sandboxes WHERE account_id = ? AND status <> 'destroyed' ORDER BY created_at`) + .bind(accountId) + .all() + return results +} + +export async function countActiveSandboxes(db: D1Database, accountId: string): Promise { + const placeholders = ACTIVE_STATUSES.map(() => '?').join(', ') + const row = await db + .prepare(`SELECT COUNT(*) AS n FROM sandboxes WHERE account_id = ? AND status IN (${placeholders})`) + .bind(accountId, ...ACTIVE_STATUSES) + .first<{ n: number }>() + return row?.n ?? 0 +} + +export async function listByStatus(db: D1Database, statuses: readonly SandboxStatus[]): Promise { + const placeholders = statuses.map(() => '?').join(', ') + const { results } = await db + .prepare(`SELECT * FROM sandboxes WHERE status IN (${placeholders}) ORDER BY updated_at LIMIT 200`) + .bind(...statuses) + .all() + return results +} + +type Patch = Partial> +const PATCHABLE = new Set([ + 'status', 'session_id', 'session_started_at', 'session_deadline_at', 'lease_expires_at', + 'usage_settled_at', 'last_error', 'destroyed_at' +]) + +/** Applies `patch` only if nobody moved the row since `expected` was read; null on a lost race. */ +export async function transition( + db: D1Database, + expected: SandboxRow, + patch: Patch, + now: number +): Promise { + const entries = Object.entries(patch).filter(([key]) => PATCHABLE.has(key)) + const assignments = entries.map(([key]) => `${key} = ?`).join(', ') + const result = await db + .prepare( + `UPDATE sandboxes SET ${assignments}${assignments ? ', ' : ''}generation = generation + 1, updated_at = ? + WHERE id = ? AND generation = ?` + ) + .bind(...entries.map(([, value]) => value ?? null), now, expected.id, expected.generation) + .run() + return result.meta.changes === 1 ? getSandbox(db, expected.id) : null +} + +export async function recordSnapshots(db: D1Database, sandboxId: string, ids: readonly string[], now: number) { + if (ids.length === 0) { + return + } + await db.batch( + ids.map((id) => + db + .prepare('INSERT OR IGNORE INTO sandbox_snapshots (snapshot_id, sandbox_id, recorded_at) VALUES (?, ?, ?)') + .bind(id, sandboxId, now) + ) + ) +} + +export async function markSnapshotsDeleted(db: D1Database, sandboxId: string, now: number) { + await db + .prepare('UPDATE sandbox_snapshots SET deleted_at = ? WHERE sandbox_id = ? AND deleted_at IS NULL') + .bind(now, sandboxId) + .run() +} + +export async function recordedSnapshotIds(db: D1Database, sandboxId: string): Promise { + const { results } = await db + .prepare('SELECT snapshot_id FROM sandbox_snapshots WHERE sandbox_id = ?') + .bind(sandboxId) + .all<{ snapshot_id: string }>() + return results.map((row) => row.snapshot_id) +} diff --git a/cloud/apps/sandbox-control/src/usage.ts b/cloud/apps/sandbox-control/src/usage.ts new file mode 100644 index 00000000000..381c21c8d42 --- /dev/null +++ b/cloud/apps/sandbox-control/src/usage.ts @@ -0,0 +1,75 @@ +// Metering: running time becomes outbox segments, then billing usage reports (overage → Stripe there). +import type { EntitlementClient, MachineSize, UsageAnswer } from './entitlements' +import type { SandboxRow } from './store' + +const MAX_SEGMENT_MS = 24 * 60 * 60_000 +const MAX_ATTEMPTS = 50 + +type OutboxRow = { + usage_id: string + sandbox_id: string + account_id: string + machine_size: MachineSize + started_at: number + ended_at: number + attempts: number +} + +/** Queues the row's unsettled time up to `until`; returns the new settle cursor (unchanged if none). */ +export async function queueUsage(db: D1Database, row: SandboxRow, until: number): Promise { + const from = row.usage_settled_at + if (from === null || row.session_id === null || until <= from) { + return from + } + const statements = [] + for (let start = from; start < until; start += MAX_SEGMENT_MS) { + const end = Math.min(until, start + MAX_SEGMENT_MS) + statements.push( + db + .prepare( + `INSERT OR IGNORE INTO usage_outbox (usage_id, sandbox_id, account_id, machine_size, started_at, ended_at) + VALUES (?, ?, ?, ?, ?, ?)` + ) + .bind(`${row.id}:${row.session_id}:${start}`, row.id, row.account_id, row.machine_size, start, end) + ) + } + await db.batch(statements) + return until +} + +/** Sends pending segments (one sandbox, or the oldest for the cron). Returns billing's last answer. */ +export async function flushUsage( + db: D1Database, + entitlements: EntitlementClient, + now: number, + sandboxId?: string +): Promise { + const { results } = await db + .prepare( + `SELECT * FROM usage_outbox WHERE sent_at IS NULL AND attempts < ? ${sandboxId ? 'AND sandbox_id = ?' : ''} + ORDER BY started_at LIMIT 100` + ) + .bind(...(sandboxId ? [MAX_ATTEMPTS, sandboxId] : [MAX_ATTEMPTS])) + .all() + let answer: UsageAnswer | null = null + for (const segment of results) { + try { + answer = await entitlements.reportUsage({ + usageId: segment.usage_id, + accountId: segment.account_id, + sandboxId: segment.sandbox_id, + machineSize: segment.machine_size, + startedAt: segment.started_at, + endedAt: segment.ended_at + }) + await db.prepare('UPDATE usage_outbox SET sent_at = ? WHERE usage_id = ?').bind(now, segment.usage_id).run() + } catch { + await db + .prepare('UPDATE usage_outbox SET attempts = attempts + 1 WHERE usage_id = ?') + .bind(segment.usage_id) + .run() + break + } + } + return answer +} diff --git a/cloud/apps/sandbox-control/src/vercel-sandbox-api.ts b/cloud/apps/sandbox-control/src/vercel-sandbox-api.ts new file mode 100644 index 00000000000..f99b48673b0 --- /dev/null +++ b/cloud/apps/sandbox-control/src/vercel-sandbox-api.ts @@ -0,0 +1,255 @@ +// Vercel Sandbox REST client for Workers. `@vercel/sandbox` needs undici/fs/xdg paths, so this +// calls the same /v2 + /v3 endpoints its api-client uses (SDK 3.9.1), with Orca's team token. +import { z } from 'zod' +import type { NetworkPolicy } from './network-policy' +import { + ProviderError, + type CommandResult, + type ProviderSandboxState, + type ProviderSession, + type SandboxProviderApi +} from './sandbox-provider' + +export type VercelSandboxApiConfig = { + token: string + teamId: string + projectId: string + baseUrl?: string + fetch?: typeof fetch +} + +const MAX_COMMAND_OUTPUT = 256 * 1024 + +const Status = z.enum(['pending', 'running', 'stopping', 'stopped', 'failed', 'aborted', 'snapshotting']) +const Session = z.looseObject({ + id: z.string(), + status: Status, + startedAt: z.number().optional(), + timeout: z.number() +}) +const Sandbox = z.looseObject({ + name: z.string(), + status: Status, + currentSessionId: z.string(), + currentSnapshotId: z.string().optional(), + tags: z.record(z.string(), z.string()).optional() +}) +const Route = z.looseObject({ url: z.string(), port: z.number() }) +const SandboxAndSession = z.looseObject({ sandbox: Sandbox, session: Session, routes: z.array(Route) }) +const Snapshot = z.looseObject({ + id: z.string(), + status: z.enum(['created', 'deleted', 'failed']), + sourceSessionId: z.string() +}) +const ErrorBody = z.looseObject({ error: z.looseObject({ code: z.string().optional() }).optional() }) +const CommandLine = z.looseObject({ command: z.looseObject({ exitCode: z.number().nullable() }) }) +const OutputLine = z.looseObject({ stream: z.enum(['stdout', 'stderr']), data: z.string() }) +const FailureLine = z.looseObject({ stream: z.literal('error'), data: z.looseObject({ code: z.string() }) }) + +export function createVercelSandboxApi(config: VercelSandboxApiConfig): SandboxProviderApi { + const base = config.baseUrl ?? 'https://vercel.com/api' + const doFetch = config.fetch ?? fetch + + async function request( + method: string, + path: string, + options: { query?: Record; body?: unknown } = {} + ): Promise { + const url = new URL(`${base}${path}`) + url.searchParams.set('teamId', config.teamId) + for (const [key, value] of Object.entries(options.query ?? {})) { + if (value !== undefined) { + url.searchParams.set(key, value) + } + } + const response = await doFetch(url, { + method, + headers: { + authorization: `Bearer ${config.token}`, + 'content-type': 'application/json' + }, + body: options.body === undefined ? undefined : JSON.stringify(options.body), + redirect: 'manual' + }) + if (!response.ok) { + const parsed = ErrorBody.safeParse(await response.json().catch(() => null)) + throw new ProviderError(response.status, parsed.success ? (parsed.data.error?.code ?? null) : null) + } + return response + } + + async function json(schema: z.ZodType, response: Response): Promise { + const parsed = schema.safeParse(await response.json().catch(() => null)) + if (!parsed.success) { + throw new ProviderError(response.status, 'invalid_response') + } + return parsed.data + } + + const toSession = (session: z.infer): ProviderSession => ({ + id: session.id, + status: session.status, + startedAt: session.startedAt ?? null, + timeoutMs: session.timeout + }) + const toState = (body: z.infer): ProviderSandboxState => ({ + sandbox: { + name: body.sandbox.name, + status: body.sandbox.status, + currentSessionId: body.sandbox.currentSessionId, + currentSnapshotId: body.sandbox.currentSnapshotId ?? null, + tags: body.sandbox.tags ?? {} + }, + session: toSession(body.session), + routes: body.routes.map((route) => ({ port: route.port, url: route.url })) + }) + const missingOk = async (operation: () => Promise): Promise => { + try { + await operation() + } catch (error) { + if (!(error instanceof ProviderError && error.status === 404)) { + throw error + } + } + } + + return { + async createSandbox(params) { + // No `runtime` selects /v3, which boots from `source` (the golden snapshot). + const response = await request('POST', '/v3/sandboxes', { + body: { + projectId: config.projectId, + name: params.name, + source: { type: 'snapshot', snapshotId: params.snapshotId }, + resources: { vcpus: params.vcpus }, + region: params.region, + timeout: params.timeoutMs, + ports: params.ports, + persistent: true, + keepLastSnapshots: { count: 2 }, + snapshotExpiration: 0, + networkPolicy: params.networkPolicy, + tags: params.tags + } + }) + return toState(await json(SandboxAndSession, response)) + }, + + async getSandbox(name, options) { + try { + const response = await request('GET', `/v2/sandboxes/${encodeURIComponent(name)}`, { + query: { projectId: config.projectId, resume: String(options.resume) } + }) + return toState(await json(SandboxAndSession, response)) + } catch (error) { + if (error instanceof ProviderError && error.status === 404) { + return null + } + throw error + } + }, + + async stopSession(sessionId) { + const response = await request('POST', `/v2/sandboxes/sessions/${encodeURIComponent(sessionId)}/stop`) + const body = await json(z.looseObject({ snapshot: Snapshot.optional() }), response) + return { snapshotId: body.snapshot?.id ?? null } + }, + + async extendTimeout(sessionId, durationMs) { + const response = await request( + 'POST', + `/v2/sandboxes/sessions/${encodeURIComponent(sessionId)}/extend-timeout`, + { body: { duration: durationMs } } + ) + return toSession((await json(z.looseObject({ session: Session }), response)).session) + }, + + async updateNetworkPolicy(sessionId, policy: NetworkPolicy) { + await request('POST', `/v2/sandboxes/sessions/${encodeURIComponent(sessionId)}/network-policy`, { + body: policy + }) + }, + + async runCommand(sessionId, params) { + const response = await request('POST', `/v2/sandboxes/sessions/${encodeURIComponent(sessionId)}/cmd`, { + body: { + command: params.command, + args: params.args ?? [], + cwd: params.cwd, + env: params.env ?? {}, + sudo: params.sudo ?? false, + wait: true, + logs: true, + timeout: params.timeoutMs + } + }) + return readCommandStream(response) + }, + + deleteSandbox: (name) => + missingOk(() => + request('DELETE', `/v2/sandboxes/${encodeURIComponent(name)}`, { + query: { projectId: config.projectId, deleteOrphanSnapshots: 'true' } + }) + ), + + async listSnapshots(name) { + const snapshots: z.infer[] = [] + let cursor: string | undefined + do { + const response = await request('GET', '/v2/sandboxes/snapshots', { + query: { project: config.projectId, name, limit: '50', cursor } + }) + const page = await json( + z.looseObject({ + snapshots: z.array(Snapshot), + pagination: z.looseObject({ next: z.string().nullable() }) + }), + response + ) + snapshots.push(...page.snapshots) + cursor = page.pagination.next ?? undefined + } while (cursor && snapshots.length < 1000) + return snapshots.map((s) => ({ id: s.id, status: s.status, sourceSessionId: s.sourceSessionId })) + }, + + deleteSnapshot: (snapshotId) => + missingOk(() => request('DELETE', `/v2/sandboxes/snapshots/${encodeURIComponent(snapshotId)}`)) + } +} + +/** `wait: true` answers ndjson: the command, its log lines, then the finished command. */ +async function readCommandStream(response: Response): Promise { + const text = await response.text() + const output = { stdout: '', stderr: '' } + let exitCode: number | null = null + for (const line of text.split('\n')) { + if (!line.trim()) { + continue + } + let value: unknown + try { + value = JSON.parse(line) + } catch { + throw new ProviderError(response.status, 'invalid_response') + } + const command = CommandLine.safeParse(value) + const log = OutputLine.safeParse(value) + const failure = FailureLine.safeParse(value) + if (command.success) { + exitCode = command.data.command.exitCode ?? exitCode + } else if (log.success) { + if (output[log.data.stream].length < MAX_COMMAND_OUTPUT) { + output[log.data.stream] += log.data.data + } + } else if (failure.success) { + throw new ProviderError(response.status, failure.data.data.code) + } else { + throw new ProviderError(response.status, 'invalid_response') + } + } + if (exitCode === null) { + throw new ProviderError(response.status, 'command_unfinished') + } + return { exitCode, ...output } +} diff --git a/cloud/apps/sandbox-control/src/wire.ts b/cloud/apps/sandbox-control/src/wire.ts new file mode 100644 index 00000000000..2028482af22 --- /dev/null +++ b/cloud/apps/sandbox-control/src/wire.ts @@ -0,0 +1,70 @@ +// Request schemas and response views. Mirrors src/shared/hosted-sandbox/hosted-sandbox-wire.ts on +// the desktop; test/wire-contract.test.ts parses every response with the desktop schemas. +import { z } from 'zod' +import type { OrcadReady } from './sandbox-vm-commands' +import type { SandboxRow } from './store' + +const Id = z + .string() + .min(1) + .max(128) + .regex(/^[A-Za-z0-9_.:-]+$/) + +export const CreateRequestSchema = z + .object({ + workspaceId: Id, + machineSize: z.enum(['standard', 'large', 'xlarge']), + region: z + .string() + .regex(/^[a-z]{3}[0-9]$/) + .optional(), + repo: z + .object({ + url: z.url({ protocol: /^https$/ }).max(2048), + ref: z.string().min(1).max(255).optional() + }) + .strict() + .optional() + }) + .strict() + +export const LeaseRequestSchema = z + .object({ + extendMs: z + .number() + .int() + .min(60_000) + .max(60 * 60_000) + }) + .strict() + +export function sandboxView(row: SandboxRow) { + const session = + row.session_id !== null && + row.session_started_at !== null && + row.session_deadline_at !== null && + row.lease_expires_at !== null + ? { + id: row.session_id, + startedAt: row.session_started_at, + deadlineAt: row.session_deadline_at, + leaseExpiresAt: row.lease_expires_at + } + : null + return { + id: row.id, + workspaceId: row.workspace_id, + status: row.status, + machineSize: row.machine_size, + region: row.region, + session, + createdAt: row.created_at + } +} + +export function readyView(row: SandboxRow, connection: OrcadReady) { + return { + sandbox: sandboxView(row), + connection: { type: 'orca-server' as const, ...connection } + } +} diff --git a/cloud/apps/sandbox-control/test/d1-fake.ts b/cloud/apps/sandbox-control/test/d1-fake.ts new file mode 100644 index 00000000000..906896cc54f --- /dev/null +++ b/cloud/apps/sandbox-control/test/d1-fake.ts @@ -0,0 +1,64 @@ +// A D1Database over node:sqlite for tests: same SQL dialect (SQLite), same prepare/bind/first/ +// all/run/batch surface, batch() in one transaction like D1. +import { readFileSync, readdirSync } from 'node:fs' +import { DatabaseSync } from 'node:sqlite' +import { fileURLToPath } from 'node:url' + +type Param = string | number | null + +class FakeStatement { + constructor( + private readonly db: DatabaseSync, + readonly sql: string, + readonly params: Param[] = [] + ) {} + + bind(...params: unknown[]): FakeStatement { + for (const param of params) { + if (param === undefined || typeof param === 'boolean' || typeof param === 'object' && param !== null) { + throw new Error(`D1 cannot bind ${typeof param}`) + } + } + return new FakeStatement(this.db, this.sql, params as Param[]) + } + + async first(): Promise { + return (this.db.prepare(this.sql).get(...this.params) as T | undefined) ?? null + } + + async all(): Promise<{ results: T[]; success: true; meta: { changes: number } }> { + return { results: this.db.prepare(this.sql).all(...this.params) as T[], success: true, meta: { changes: 0 } } + } + + runSync(): { success: true; meta: { changes: number } } { + const result = this.db.prepare(this.sql).run(...this.params) + return { success: true, meta: { changes: Number(result.changes) } } + } + + async run() { + return this.runSync() + } +} + +export function createFakeD1(): { db: D1Database; raw: DatabaseSync } { + const raw = new DatabaseSync(':memory:') + const dir = fileURLToPath(new URL('../migrations/', import.meta.url).href) + for (const file of readdirSync(dir).filter((name) => name.endsWith('.sql')).sort()) { + raw.exec(readFileSync(`${dir}${file}`, 'utf8')) + } + const db = { + prepare: (sql: string) => new FakeStatement(raw, sql), + async batch(statements: FakeStatement[]) { + raw.exec('BEGIN') + try { + const results = statements.map((statement) => statement.runSync()) + raw.exec('COMMIT') + return results + } catch (error) { + raw.exec('ROLLBACK') + throw error + } + } + } + return { db: db as unknown as D1Database, raw } +} diff --git a/cloud/apps/sandbox-control/test/egress-proxy.test.ts b/cloud/apps/sandbox-control/test/egress-proxy.test.ts new file mode 100644 index 00000000000..2b5d06c18b3 --- /dev/null +++ b/cloud/apps/sandbox-control/test/egress-proxy.test.ts @@ -0,0 +1,81 @@ +import { SignJWT, createLocalJWKSet, exportJWK, generateKeyPair } from 'jose' +import { describe, expect, it } from 'vitest' +import { createHarness } from './harness' + +const PROXY_URL = 'https://sandbox.test/v1/egress/proxy' +const ISSUER = 'https://oidc.vercel.com/team_orca' + +async function setup() { + const { privateKey, publicKey } = await generateKeyPair('RS256') + const jwks = { keys: [{ ...(await exportJWK(publicKey)), kid: 'v1', alg: 'RS256' }] } + const upstream: { url: string; authorization: string | null }[] = [] + const h = await createHarness( + { EGRESS_PROXY_URL: PROXY_URL }, + { + egressKeysForIssuer: () => createLocalJWKSet(jwks), + credentials: { authorizationFor: async (row, host) => (host === 'api.github.com' ? `Bearer ghs_for_${row.workspace_id}` : null) }, + egressFetch: async (input, init) => { + upstream.push({ url: String(input), authorization: new Headers(init?.headers).get('authorization') }) + return new Response('{"login":"orca"}', { status: 200, headers: { 'content-type': 'application/json' } }) + } + } + ) + const created = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard' } }) + const sandboxId: string = created.body.sandbox.id + async function oidc(claims: Record, audience = PROXY_URL) { + return new SignJWT({ team_id: 'team_orca', project_id: 'prj_orca', sandbox_id: 'sbx-v', sandbox_name: `orca-test-${sandboxId}`, ...claims }) + .setProtectedHeader({ alg: 'RS256', kid: 'v1' }) + .setIssuer(ISSUER) + .setAudience(audience) + .setIssuedAt() + .setExpirationTime('5m') + .sign(privateKey) + } + async function proxied(init: { token: string; authorization?: string; host?: string }) { + const response = await h.send( + new Request(`${PROXY_URL}/user`, { + headers: { + 'vercel-forwarded-host': init.host ?? 'api.github.com', + 'vercel-forwarded-scheme': 'https', + 'vercel-forwarded-port': '443', + 'vercel-forwarded-path': '/user', + 'vercel-sandbox-oidc-token': init.token, + ...(init.authorization ? { authorization: init.authorization } : {}) + } + }) + ) + return { status: response.status, body: await response.text() } + } + return { h, sandboxId, oidc, proxied, upstream } +} + +describe('egress proxy (Vercel forwardURL target)', () => { + it('swaps the placeholder for the workspace credential on a verified sandbox request', async () => { + const { sandboxId, oidc, proxied, upstream } = await setup() + const res = await proxied({ token: await oidc({}), authorization: `Bearer orca-ph-${sandboxId}` }) + expect(res).toEqual({ status: 200, body: '{"login":"orca"}' }) + expect(upstream).toEqual([{ url: 'https://api.github.com/user', authorization: 'Bearer ghs_for_ws-1' }]) + }) + + it('refuses requests without the placeholder, from other teams, or for unproxied hosts', async () => { + const { sandboxId, oidc, proxied, upstream } = await setup() + const placeholder = `Bearer orca-ph-${sandboxId}` + expect(await proxied({ token: await oidc({}), authorization: 'Bearer attacker' })).toEqual({ status: 403, body: 'placeholder_missing' }) + expect(await proxied({ token: await oidc({ team_id: 'team_other' }), authorization: placeholder })).toEqual({ status: 403, body: 'foreign_sandbox' }) + expect(await proxied({ token: await oidc({}, 'https://elsewhere.test'), authorization: placeholder })).toEqual({ status: 403, body: 'bad_oidc_token' }) + expect(await proxied({ token: await oidc({}), authorization: placeholder, host: 'example.com' })).toEqual({ status: 403, body: 'host_not_proxied' }) + expect(upstream).toEqual([]) + }) + + it('forwards the placeholder route for api.github.com in the sandbox network policy', async () => { + const { h, sandboxId } = await setup() + const [box] = [...h.vercel.boxes.values()] + expect(box!.policy.allow['api.github.com']).toEqual([ + { + match: { headers: [{ key: { exact: 'authorization' }, value: { exact: `Bearer orca-ph-${sandboxId}` } }] }, + forwardURL: PROXY_URL + }, + { response: { statusCode: 403, body: 'orca-egress-denied', contentType: 'text/plain' } } + ]) + }) +}) diff --git a/cloud/apps/sandbox-control/test/fake-vercel.ts b/cloud/apps/sandbox-control/test/fake-vercel.ts new file mode 100644 index 00000000000..bca4b79500a --- /dev/null +++ b/cloud/apps/sandbox-control/test/fake-vercel.ts @@ -0,0 +1,161 @@ +// In-memory Vercel Sandbox with the behaviours the bake-off measured: stop = reboot from an automatic +// snapshot, resume keeps the name and route, and delete leaves automatic snapshots behind. +import type { NetworkPolicy } from '../src/network-policy' +import { + ProviderError, + type CreateSandboxParams, + type ProviderSandboxState, + type ProviderSnapshot, + type RunCommandParams, + type SandboxProviderApi +} from '../src/sandbox-provider' +import { DRAIN_COMMAND, START_COMMAND } from '../src/sandbox-vm-commands' + +type FakeBox = { + params: CreateSandboxParams + status: 'running' | 'stopped' + sessionId: string + startedAt: number + timeoutMs: number + route: string + policy: NetworkPolicy +} + +export type FakeVercel = SandboxProviderApi & { + boxes: Map + snapshots: Map + commands: { sessionId: string; command: string; env: Record }[] + failStart: boolean + failCreate: boolean + /** Simulates the provider ending a session at its timeout. */ + expire(name: string): void +} + +export function createFakeVercel(now: () => number): FakeVercel { + const boxes = new Map() + const snapshots: FakeVercel['snapshots'] = new Map() + const commands: FakeVercel['commands'] = [] + let counter = 0 + const next = (prefix: string) => `${prefix}_${++counter}` + const state = (box: FakeBox): ProviderSandboxState => ({ + sandbox: { + name: box.params.name, + status: box.status, + currentSessionId: box.sessionId, + currentSnapshotId: null, + tags: box.params.tags + }, + session: { id: box.sessionId, status: box.status, startedAt: box.startedAt, timeoutMs: box.timeoutMs }, + routes: [{ port: 6768, url: box.route }] + }) + const bySession = (sessionId: string): FakeBox => { + const box = [...boxes.values()].find((candidate) => candidate.sessionId === sessionId) + if (!box) { + throw new ProviderError(404, 'not_found') + } + return box + } + const stop = (box: FakeBox): string => { + box.status = 'stopped' + const id = next('snap') + snapshots.set(id, { id, status: 'created', sourceSessionId: box.sessionId, sandboxName: box.params.name }) + return id + } + + const fake: FakeVercel = { + boxes, + snapshots, + commands, + failStart: false, + failCreate: false, + expire(name) { + const box = boxes.get(name) + if (box?.status === 'running') { + stop(box) + } + }, + async createSandbox(params) { + if (fake.failCreate) { + throw new ProviderError(500, 'internal_error') + } + if (boxes.has(params.name)) { + throw new ProviderError(409, 'conflict') + } + const box: FakeBox = { + params, + status: 'running', + sessionId: next('sess'), + startedAt: now(), + timeoutMs: params.timeoutMs, + route: `https://sb-${counter}.vercel.run`, + policy: params.networkPolicy + } + boxes.set(params.name, box) + return state(box) + }, + async getSandbox(name, options) { + const box = boxes.get(name) + if (!box) { + return null + } + if (box.status === 'stopped' && options.resume) { + Object.assign(box, { + status: 'running', + sessionId: next('sess'), + startedAt: now(), + timeoutMs: box.params.timeoutMs + }) + } + return state(box) + }, + async stopSession(sessionId) { + const box = bySession(sessionId) + return { snapshotId: box.status === 'running' ? stop(box) : null } + }, + async extendTimeout(sessionId, durationMs) { + const box = bySession(sessionId) + box.timeoutMs += durationMs + return state(box).session + }, + async updateNetworkPolicy(sessionId, policy) { + bySession(sessionId).policy = policy + }, + async runCommand(sessionId, params: RunCommandParams) { + const box = bySession(sessionId) + if (box.status !== 'running') { + throw new ProviderError(410, 'sandbox_stopped') + } + commands.push({ sessionId, command: params.command, env: params.env ?? {} }) + if (params.command === START_COMMAND) { + if (fake.failStart) { + return { exitCode: 1, stdout: '', stderr: 'orcad crashed' } + } + const ready = { pairingCode: `orca://pair?code=${box.params.name}`, projectRoot: '/vercel/sandbox/project' } + return { exitCode: 0, stdout: `booting\n${JSON.stringify(ready)}\n`, stderr: '' } + } + if (params.command === DRAIN_COMMAND) { + return { exitCode: 0, stdout: '', stderr: '' } + } + return { exitCode: 127, stdout: '', stderr: 'not found' } + }, + async deleteSandbox(name) { + const box = boxes.get(name) + if (box?.status === 'running') { + stop(box) + } + boxes.delete(name) + }, + async listSnapshots(name) { + return [...snapshots.values()] + .filter((snapshot) => snapshot.sandboxName === name) + .map(({ id, status, sourceSessionId }) => ({ id, status, sourceSessionId })) + }, + async deleteSnapshot(snapshotId) { + const snapshot = snapshots.get(snapshotId) + if (snapshot) { + snapshot.status = 'deleted' + } + } + } + return fake +} diff --git a/cloud/apps/sandbox-control/test/harness.ts b/cloud/apps/sandbox-control/test/harness.ts new file mode 100644 index 00000000000..d27e7e5ee8c --- /dev/null +++ b/cloud/apps/sandbox-control/test/harness.ts @@ -0,0 +1,98 @@ +import { SignJWT, exportJWK, generateKeyPair } from 'jose' +import type { EntitlementClient, StartDecision, UsageAnswer, UsageSegment } from '../src/entitlements' +import type { Env } from '../src/env' +import { handleRequest, type Deps } from '../src/index' +import { reconcile } from '../src/reconcile' +import { createFakeD1 } from './d1-fake' +import { createFakeVercel } from './fake-vercel' + +export const GOLDEN = 'snap_golden' + +export async function createHarness(overrides: Partial = {}, extraDeps: Partial = {}) { + const { privateKey, publicKey } = await generateKeyPair('ES256') + const jwk = { ...(await exportJWK(publicKey)), kid: 'k1', alg: 'ES256', use: 'sig' } + const { db, raw } = createFakeD1() + let now = 1_800_000_000_000 + const clock = () => now + const vercel = createFakeVercel(clock) + const billing = { + decision: { allow: true, reason: null } as StartDecision, + usageAnswer: { canStartSandbox: true, reason: null } as UsageAnswer, + down: false, + checks: [] as { accountId: string; runningSandboxes: number }[], + usage: [] as UsageSegment[] + } + const entitlements: EntitlementClient = { + async checkStart(request) { + billing.checks.push({ accountId: request.accountId, runningSandboxes: request.runningSandboxes }) + return billing.decision + }, + async reportUsage(segment) { + if (billing.down) { + throw new Error('billing down') + } + billing.usage.push(segment) + return billing.usageAnswer + } + } + const env: Env = { + SANDBOX_DB: db, + SANDBOX_ENV: 'test', + SANDBOX_JWKS: JSON.stringify({ keys: [jwk] }), + SANDBOX_JWT_AUDIENCE: 'orca-sandbox', + SANDBOX_AUTH_ISSUER: 'https://login.onorca.dev', + GOLDEN_SNAPSHOT_ID: GOLDEN, + VERCEL_TEAM_ID: 'team_orca', + VERCEL_PROJECT_ID: 'prj_orca', + ...overrides + } + const deps: Deps = { provider: vercel, entitlements, now: clock, ...extraDeps } + + async function token(sub: string, org?: string, purpose = 'sandbox') { + const iat = Math.floor(now / 1000) + return new SignJWT({ sub, ...(org ? { org } : {}), purpose }) + .setProtectedHeader({ alg: 'ES256', kid: 'k1' }) + .setIssuer('https://login.onorca.dev') + .setAudience('orca-sandbox') + .setIssuedAt(iat) + .setExpirationTime(iat + 300) + .sign(privateKey) + } + + async function send(request: Request): Promise { + return handleRequest(request, env, deps) + } + + async function call(method: string, path: string, init: { body?: unknown; as?: string; bearer?: string } = {}) { + const bearer = init.bearer ?? (await token(init.as ?? 'alice')) + const request = new Request(`https://sandbox.test${path}`, { + method, + headers: { authorization: `Bearer ${bearer}`, 'content-type': 'application/json' }, + body: init.body === undefined ? undefined : JSON.stringify(init.body) + }) + const response = await handleRequest(request, env, deps) + return { status: response.status, body: (await response.json()) as any } + } + + return { + env, + raw, + vercel, + billing, + call, + send, + token, + advance(ms: number) { + now += ms + }, + now: clock, + reconcile: () => + reconcile({ + db, + provider: vercel, + entitlements, + now: clock, + config: { environment: 'test', goldenSnapshotId: GOLDEN, defaultRegion: 'iad1' } + }) + } +} diff --git a/cloud/apps/sandbox-control/test/lifecycle-flow.test.ts b/cloud/apps/sandbox-control/test/lifecycle-flow.test.ts new file mode 100644 index 00000000000..bcbf176d895 --- /dev/null +++ b/cloud/apps/sandbox-control/test/lifecycle-flow.test.ts @@ -0,0 +1,190 @@ +import { describe, expect, it } from 'vitest' +import { + HostedSandboxListResponseSchema, + HostedSandboxReadyResponseSchema, + HostedSandboxStatusResponseSchema +} from '../../../../src/shared/hosted-sandbox/hosted-sandbox-wire' +import { DRAIN_COMMAND, START_COMMAND } from '../src/sandbox-vm-commands' +import { GOLDEN, createHarness } from './harness' + +const MIN = 60_000 +const create = { workspaceId: 'ws-1', machineSize: 'standard' } + +describe('sandbox control plane lifecycle', () => { + it('creates a white-labelled sandbox from the golden image and returns orcad pairing', async () => { + const h = await createHarness() + const res = await h.call('POST', '/v1/sandboxes', { body: create }) + expect(res.status).toBe(200) + const ready = HostedSandboxReadyResponseSchema.parse(res.body) + expect(ready.sandbox.status).toBe('running') + expect(ready.connection.projectRoot).toBe('/vercel/sandbox/project') + expect(ready.sandbox.session!.deadlineAt - ready.sandbox.session!.startedAt).toBe(23 * 60 * MIN + 50 * MIN) + expect(ready.sandbox.session!.leaseExpiresAt - ready.sandbox.session!.startedAt).toBe(30 * MIN) + + const [box] = [...h.vercel.boxes.values()] + expect(box!.params).toMatchObject({ + name: `orca-test-${ready.sandbox.id}`, + snapshotId: GOLDEN, + vcpus: 4, + ports: [6768], + tags: { orcaSandbox: ready.sandbox.id, orcaEnv: 'test' } + }) + expect(Object.keys(box!.policy.allow)).not.toContain('*') + const start = h.vercel.commands.find((c) => c.command === START_COMMAND) + expect(start!.env.ORCA_PUBLIC_WSS).toBe(box!.route.replace('https:', 'wss:')) + expect(h.billing.checks).toEqual([{ accountId: 'user:alice', runningSandboxes: 0 }]) + }) + + it('refuses a start billing denies without touching the provider', async () => { + const h = await createHarness() + h.billing.decision = { allow: false, reason: 'concurrency_limit' } + const res = await h.call('POST', '/v1/sandboxes', { body: create }) + expect(res).toEqual({ status: 403, body: { error: 'entitlement_denied', reason: 'concurrency_limit' } }) + expect(h.vercel.boxes.size).toBe(0) + expect(h.raw.prepare('SELECT COUNT(*) AS n FROM sandboxes').get()).toEqual({ n: 0 }) + }) + + it('hides one account’s sandbox from another', async () => { + const h = await createHarness() + const { body } = await h.call('POST', '/v1/sandboxes', { body: create }) + expect((await h.call('GET', `/v1/sandboxes/${body.sandbox.id}`, { as: 'bob' })).status).toBe(404) + expect((await h.call('DELETE', `/v1/sandboxes/${body.sandbox.id}`, { as: 'bob' })).status).toBe(404) + expect(HostedSandboxListResponseSchema.parse((await h.call('GET', '/v1/sandboxes', { as: 'bob' })).body).sandboxes).toEqual([]) + expect(h.vercel.boxes.size).toBe(1) + }) + + it('returns the same environment when a create is retried for the workspace', async () => { + const h = await createHarness() + const first = await h.call('POST', '/v1/sandboxes', { body: create }) + const second = await h.call('POST', '/v1/sandboxes', { body: create }) + expect(second.body.sandbox.id).toBe(first.body.sandbox.id) + expect(h.vercel.boxes.size).toBe(1) + }) + + it('drains before stopping, meters the session, and resumes the same disk', async () => { + const h = await createHarness() + const { body } = await h.call('POST', '/v1/sandboxes', { body: create }) + const id = body.sandbox.id + h.advance(20 * MIN) + const suspended = await h.call('POST', `/v1/sandboxes/${id}/suspend`) + expect(HostedSandboxStatusResponseSchema.parse(suspended.body).sandbox).toMatchObject({ status: 'suspended', session: null }) + expect(h.vercel.commands.map((c) => c.command)).toEqual([START_COMMAND, DRAIN_COMMAND]) + expect(h.billing.usage).toEqual([ + expect.objectContaining({ sandboxId: id, accountId: 'user:alice', machineSize: 'standard', endedAt: h.now(), startedAt: h.now() - 20 * MIN }) + ]) + + const resumed = HostedSandboxReadyResponseSchema.parse((await h.call('POST', `/v1/sandboxes/${id}/resume`)).body) + expect(resumed.sandbox.status).toBe('running') + expect(resumed.sandbox.session!.id).not.toBe(body.sandbox.session.id) + expect(h.vercel.boxes.size).toBe(1) + expect(h.billing.checks).toHaveLength(2) + }) + + it('renews the Orca lease but never past the forced-rollover deadline', async () => { + const h = await createHarness() + const { body } = await h.call('POST', '/v1/sandboxes', { body: create }) + const startedAt = body.sandbox.session.startedAt + const renewed = await h.call('POST', `/v1/sandboxes/${body.sandbox.id}/lease`, { body: { extendMs: 60 * MIN } }) + expect(renewed.body.sandbox.session.leaseExpiresAt).toBe(h.now() + 60 * MIN) + h.advance(23 * 60 * MIN + 30 * MIN) + const clamped = await h.call('POST', `/v1/sandboxes/${body.sandbox.id}/lease`, { body: { extendMs: 60 * MIN } }) + expect(clamped.body.sandbox.session.leaseExpiresAt).toBe(startedAt + 23 * 60 * MIN + 50 * MIN) + }) + + it('rolls a session over onto a fresh one from the same disk', async () => { + const h = await createHarness() + const { body } = await h.call('POST', '/v1/sandboxes', { body: create }) + h.advance(12 * 60 * MIN) + const rolled = HostedSandboxReadyResponseSchema.parse((await h.call('POST', `/v1/sandboxes/${body.sandbox.id}/rollover`)).body) + expect(rolled.sandbox.id).toBe(body.sandbox.id) + expect(rolled.sandbox.session!.startedAt).toBe(h.now()) + expect(h.vercel.commands.map((c) => c.command)).toEqual([START_COMMAND, DRAIN_COMMAND, START_COMMAND]) + expect(h.billing.usage.map((u) => u.endedAt - u.startedAt)).toEqual([12 * 60 * MIN]) + }) + + it('destroys the sandbox and every snapshot it left behind, idempotently', async () => { + const h = await createHarness() + const { body } = await h.call('POST', '/v1/sandboxes', { body: create }) + await h.call('POST', `/v1/sandboxes/${body.sandbox.id}/suspend`) + await h.call('POST', `/v1/sandboxes/${body.sandbox.id}/resume`) + expect([...h.vercel.snapshots.values()].filter((s) => s.status === 'created')).toHaveLength(1) + + const destroyed = await h.call('DELETE', `/v1/sandboxes/${body.sandbox.id}`) + expect(destroyed.body.sandbox.status).toBe('destroyed') + expect(h.vercel.boxes.size).toBe(0) + expect([...h.vercel.snapshots.values()].every((s) => s.status === 'deleted')).toBe(true) + expect((await h.call('DELETE', `/v1/sandboxes/${body.sandbox.id}`)).body.sandbox.status).toBe('destroyed') + expect((await h.call('GET', '/v1/sandboxes')).body.sandboxes).toEqual([]) + }) + + it('cleans up a create whose orcad never came up, so the workspace can retry', async () => { + const h = await createHarness() + h.vercel.failStart = true + const res = await h.call('POST', '/v1/sandboxes', { body: create }) + expect(res).toEqual({ status: 502, body: { error: 'provider_unavailable', reason: 'orcad_start_failed' } }) + expect(h.vercel.boxes.size).toBe(0) + h.vercel.failStart = false + expect((await h.call('POST', '/v1/sandboxes', { body: create })).status).toBe(200) + }) + + it('refuses lifecycle calls until Orca’s provider account is configured', async () => { + const h = await createHarness({ GOLDEN_SNAPSHOT_ID: undefined }) + expect(await h.call('POST', '/v1/sandboxes', { body: create })).toEqual({ status: 503, body: { error: 'not_configured' } }) + }) + + it('rejects tokens minted for another purpose', async () => { + const h = await createHarness() + const res = await h.call('GET', '/v1/sandboxes', { bearer: await h.token('alice', undefined, 'billing') }) + expect(res.status).toBe(401) + }) +}) + +describe('sandbox control plane cron', () => { + it('stops a sandbox at the forced-rollover deadline when no desktop rolled it over', async () => { + const h = await createHarness() + const { body } = await h.call('POST', '/v1/sandboxes', { body: create }) + await h.call('POST', `/v1/sandboxes/${body.sandbox.id}/lease`, { body: { extendMs: 60 * MIN } }) + h.advance(23 * 60 * MIN + 50 * MIN) + expect((await h.reconcile()).suspendedForDeadline).toBe(1) + expect((await h.call('GET', `/v1/sandboxes/${body.sandbox.id}`)).body.sandbox.status).toBe('suspended') + }) + + it('bills a lapsed lease only up to when the provider stopped it', async () => { + const h = await createHarness() + const { body } = await h.call('POST', '/v1/sandboxes', { body: create }) + h.advance(45 * MIN) + h.vercel.expire(`orca-test-${body.sandbox.id}`) + expect((await h.reconcile()).leaseLapsed).toBe(1) + expect(h.billing.usage.map((u) => u.endedAt - u.startedAt)).toEqual([30 * MIN]) + }) + + it('stops running sandboxes once billing says the included hours are gone', async () => { + const h = await createHarness() + const { body } = await h.call('POST', '/v1/sandboxes', { body: create }) + h.billing.usageAnswer = { canStartSandbox: false, reason: 'included_hours_exhausted' } + h.advance(10 * MIN) + expect((await h.reconcile()).suspendedForBilling).toBe(1) + expect((await h.call('GET', `/v1/sandboxes/${body.sandbox.id}`)).body.sandbox.status).toBe('suspended') + }) + + it('keeps usage owed while billing is down and reports it once billing is back', async () => { + const h = await createHarness() + const { body } = await h.call('POST', '/v1/sandboxes', { body: create }) + h.billing.down = true + h.advance(5 * MIN) + expect((await h.call('POST', `/v1/sandboxes/${body.sandbox.id}/suspend`)).status).toBe(200) + expect(h.billing.usage).toEqual([]) + h.billing.down = false + await h.reconcile() + expect(h.billing.usage.map((u) => u.endedAt - u.startedAt)).toEqual([5 * MIN]) + }) + + it('releases a sandbox whose create died mid-way', async () => { + const h = await createHarness() + const { body } = await h.call('POST', '/v1/sandboxes', { body: create }) + h.raw.prepare(`UPDATE sandboxes SET status = 'creating' WHERE id = ?`).run(body.sandbox.id) + h.advance(16 * MIN) + expect((await h.reconcile()).released).toBe(1) + expect(h.vercel.boxes.size).toBe(0) + }) +}) diff --git a/cloud/apps/sandbox-control/test/network-policy.test.ts b/cloud/apps/sandbox-control/test/network-policy.test.ts new file mode 100644 index 00000000000..5545a374a87 --- /dev/null +++ b/cloud/apps/sandbox-control/test/network-policy.test.ts @@ -0,0 +1,63 @@ +import { describe, expect, it } from 'vitest' +import { compileNetworkPolicy, gitBasicAuthorization } from '../src/network-policy' + +const deny = { response: { statusCode: 403, body: 'orca-egress-denied', contentType: 'text/plain' } } + +describe('compileNetworkPolicy', () => { + it('fails closed: no wildcard, and credential hosts end with a deny rule', () => { + const policy = compileNetworkPolicy({ + mode: 'allowlist', + allowlist: ['registry.npmjs.org', 'github.com'], + credentials: [ + { + host: 'github.com', + header: 'Authorization', + placeholder: gitBasicAuthorization('orca-ph-1'), + via: 'transform', + value: gitBasicAuthorization('ghs_real') + }, + { + host: 'api.github.com', + header: 'authorization', + placeholder: 'Bearer orca-ph-1', + via: 'forward', + forwardURL: 'https://sandbox.onorca.dev/v1/egress/proxy' + } + ] + }) + expect(policy).toEqual({ + allow: { + 'registry.npmjs.org': [], + 'github.com': [ + { + match: { headers: [{ key: { exact: 'authorization' }, value: { exact: 'Basic eC1hY2Nlc3MtdG9rZW46b3JjYS1waC0x' } }] }, + transform: [{ headers: { authorization: 'Basic eC1hY2Nlc3MtdG9rZW46Z2hzX3JlYWw=' } }] + }, + deny + ], + 'api.github.com': [ + { + match: { headers: [{ key: { exact: 'authorization' }, value: { exact: 'Bearer orca-ph-1' } }] }, + forwardURL: 'https://sandbox.onorca.dev/v1/egress/proxy' + }, + deny + ] + } + }) + }) + + it('adds the wildcard only when a workspace opts into open egress', () => { + expect(compileNetworkPolicy({ mode: 'open', allowlist: [] })).toEqual({ allow: { '*': [] } }) + }) + + it('rejects malformed hosts and wildcard credential hosts', () => { + expect(() => compileNetworkPolicy({ mode: 'allowlist', allowlist: ['https://x.com'] })).toThrow() + expect(() => + compileNetworkPolicy({ + mode: 'allowlist', + allowlist: [], + credentials: [{ host: '*.github.com', header: 'authorization', placeholder: 'p', via: 'transform', value: 'v' }] + }) + ).toThrow(/exact host/) + }) +}) diff --git a/cloud/apps/sandbox-control/test/vercel-sandbox-api.test.ts b/cloud/apps/sandbox-control/test/vercel-sandbox-api.test.ts new file mode 100644 index 00000000000..d6d7a094b53 --- /dev/null +++ b/cloud/apps/sandbox-control/test/vercel-sandbox-api.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from 'vitest' +import { ProviderError } from '../src/sandbox-provider' +import { createVercelSandboxApi } from '../src/vercel-sandbox-api' + +type Seen = { method: string; url: URL; headers: Headers; body: any } + +function api(answer: (seen: Seen) => Response) { + const seen: Seen[] = [] + const client = createVercelSandboxApi({ + token: 'vercel_test_token', + teamId: 'team_orca', + projectId: 'prj_orca', + fetch: async (input, init) => { + const entry = { + method: init?.method ?? 'GET', + url: new URL(String(input)), + headers: new Headers(init?.headers), + body: typeof init?.body === 'string' ? JSON.parse(init.body) : undefined + } + seen.push(entry) + return answer(entry) + } + }) + return { client, seen } +} + +const sandboxBody = { + sandbox: { name: 'orca-prod-sbx_1', status: 'running', currentSessionId: 'sess_1', persistent: true, tags: { orcaSandbox: 'sbx_1' } }, + session: { id: 'sess_1', status: 'running', startedAt: 1000, timeout: 1_800_000, memory: 8192, vcpus: 4 }, + routes: [{ port: 6768, url: 'https://sb-1.vercel.run', subdomain: 'sb-1' }] +} + +describe('Vercel Sandbox REST client', () => { + it('creates a persistent sandbox from the golden snapshot on the v3 endpoint', async () => { + const { client, seen } = api(() => Response.json(sandboxBody)) + const state = await client.createSandbox({ + name: 'orca-prod-sbx_1', + snapshotId: 'snap_golden', + vcpus: 4, + region: 'iad1', + timeoutMs: 1_800_000, + ports: [6768], + networkPolicy: { allow: { 'registry.npmjs.org': [] } }, + tags: { orcaSandbox: 'sbx_1' } + }) + expect(seen[0]!.method).toBe('POST') + expect(`${seen[0]!.url.origin}${seen[0]!.url.pathname}${seen[0]!.url.search}`).toBe( + 'https://vercel.com/api/v3/sandboxes?teamId=team_orca' + ) + expect(seen[0]!.headers.get('authorization')).toBe('Bearer vercel_test_token') + expect(seen[0]!.body).toMatchObject({ + projectId: 'prj_orca', + source: { type: 'snapshot', snapshotId: 'snap_golden' }, + resources: { vcpus: 4 }, + persistent: true, + keepLastSnapshots: { count: 2 }, + snapshotExpiration: 0 + }) + expect(state.routes).toEqual([{ port: 6768, url: 'https://sb-1.vercel.run' }]) + expect(state.session.timeoutMs).toBe(1_800_000) + }) + + it('reads the command ndjson stream to the finished exit code', async () => { + const lines = [ + { command: { id: 'cmd_1', exitCode: null } }, + { stream: 'stdout', data: 'boot\n' }, + { stream: 'stderr', data: 'warn\n' }, + { stream: 'stdout', data: '{"ok":1}\n' }, + { command: { id: 'cmd_1', exitCode: 0 } } + ] + const { client, seen } = api(() => new Response(lines.map((l) => JSON.stringify(l)).join('\n'), { headers: { 'content-type': 'application/x-ndjson' } })) + const result = await client.runCommand('sess_1', { command: '/opt/orca/bin/orca-sandbox-start', timeoutMs: 1000 }) + expect(result).toEqual({ exitCode: 0, stdout: 'boot\n{"ok":1}\n', stderr: 'warn\n' }) + expect(seen[0]!.url.pathname).toBe('/api/v2/sandboxes/sessions/sess_1/cmd') + expect(seen[0]!.body).toMatchObject({ wait: true, logs: true }) + }) + + it('treats a missing sandbox as absent and deletes with orphan snapshots', async () => { + const { client, seen } = api((s) => + s.method === 'GET' ? Response.json({ error: { code: 'not_found' } }, { status: 404 }) : Response.json({ error: { code: 'not_found' } }, { status: 404 }) + ) + expect(await client.getSandbox('orca-prod-sbx_9', { resume: true })).toBeNull() + await client.deleteSandbox('orca-prod-sbx_9') + expect(seen[0]!.url.searchParams.get('resume')).toBe('true') + expect(seen[1]!.method).toBe('DELETE') + expect(seen[1]!.url.searchParams.get('deleteOrphanSnapshots')).toBe('true') + }) + + it('surfaces provider failures as codes without their bodies', async () => { + const { client } = api(() => Response.json({ error: { code: 'rate_limited', message: 'secret detail' } }, { status: 429 })) + const error = await client.stopSession('sess_1').catch((caught: unknown) => caught) + expect(error).toBeInstanceOf(ProviderError) + expect((error as ProviderError).message).toBe('provider request failed (429 rate_limited)') + }) +}) diff --git a/cloud/apps/sandbox-control/tsconfig.json b/cloud/apps/sandbox-control/tsconfig.json new file mode 100644 index 00000000000..043cf347090 --- /dev/null +++ b/cloud/apps/sandbox-control/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "es2022", + "module": "es2022", + "moduleResolution": "bundler", + "lib": ["es2022"], + "types": ["@cloudflare/workers-types", "@types/node"], + "strict": true, + "noUncheckedIndexedAccess": true, + "noEmit": true, + "skipLibCheck": true + }, + "include": ["src", "test", "image"] +} diff --git a/cloud/apps/sandbox-control/vitest.config.ts b/cloud/apps/sandbox-control/vitest.config.ts new file mode 100644 index 00000000000..64447de6145 --- /dev/null +++ b/cloud/apps/sandbox-control/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + // A string tsconfig skips the lookup, which otherwise climbs to the desktop app's tsconfig. + oxc: { tsconfig: false }, + test: { include: ['test/**/*.test.ts', 'image/**/*.test.ts'] } +}) diff --git a/cloud/apps/sandbox-control/wrangler.jsonc b/cloud/apps/sandbox-control/wrangler.jsonc new file mode 100644 index 00000000000..677c810b07c --- /dev/null +++ b/cloud/apps/sandbox-control/wrangler.jsonc @@ -0,0 +1,28 @@ +{ + "$schema": "node_modules/wrangler/config-schema.json", + // Local `wrangler dev` only. No routes and workers.dev is off, so a deploy from this file + // publishes nothing reachable. Deployed environments live in the private deploy config. + // Secrets (stdin only, never here): VERCEL_TOKEN (Orca's team, Sandbox scope) and + // CONTROL_PLANE_TOKEN (shared with the billing Worker). Local dev: an untracked .dev.vars. + "name": "orca-sandbox-control-local", + "main": "src/index.ts", + "compatibility_date": "2026-11-01", + "workers_dev": false, + "d1_databases": [ + { + "binding": "SANDBOX_DB", + "database_name": "orca-sandbox-control-local", + "database_id": "00000000-0000-0000-0000-000000000000", + "migrations_dir": "migrations" + } + ], + // Usage settlement, lease/deadline enforcement and stuck-row reconciliation. + "triggers": { "crons": ["*/5 * * * *"] }, + "vars": { + "SANDBOX_ENV": "local", + "SANDBOX_JWT_AUDIENCE": "orca-sandbox", + "VERCEL_REGION": "iad1", + // Local dev never calls billing; deployed configs set BILLING_URL and drop this. + "ENTITLEMENTS_MODE": "stub" + } +} diff --git a/cloud/pnpm-lock.yaml b/cloud/pnpm-lock.yaml index 4faad933bb0..5e22a6427ca 100644 --- a/cloud/pnpm-lock.yaml +++ b/cloud/pnpm-lock.yaml @@ -348,6 +348,34 @@ importers: specifier: ^5.0.5 version: 5.0.5(@types/node@24.19.2)(vite@8.3.4(@types/node@24.19.2)(esbuild@0.28.2)(tsx@4.23.15)) + apps/sandbox-control: + dependencies: + jose: + specifier: ^6.2.13 + version: 6.2.13 + zod: + specifier: ^4.6.7 + version: 4.6.7 + devDependencies: + '@cloudflare/workers-types': + specifier: ^5.20261103.1 + version: 5.20261104.1 + '@types/node': + specifier: ^24.19.2 + version: 24.19.2 + tsx: + specifier: ^4.23.15 + version: 4.23.15 + typescript: + specifier: ^5.9.3 + version: 5.9.3 + vitest: + specifier: ^5.0.5 + version: 5.0.5(@types/node@24.19.2)(vite@8.3.4(@types/node@24.19.2)(esbuild@0.28.2)(tsx@4.23.15)) + wrangler: + specifier: ^4.157.0 + version: 4.157.0(@cloudflare/workers-types@5.20261104.1)(@types/node@24.19.2) + packages/cloud-database: dependencies: '@orca-cloud/postgres-schema': diff --git a/config/scripts/ci/unit-selection/check-test-file-runners.mjs b/config/scripts/ci/unit-selection/check-test-file-runners.mjs index 24993494bd6..c8f96cd1549 100644 --- a/config/scripts/ci/unit-selection/check-test-file-runners.mjs +++ b/config/scripts/ci/unit-selection/check-test-file-runners.mjs @@ -62,6 +62,15 @@ const NON_UNIT_TEST_RUNNERS = [ sources: ['cloud/apps/relay-edge/vitest.config.ts'], mirrors: ["include: ['test/**/*.test.ts']"] }, + { + name: 'cloud sandbox-control', + include: [ + 'cloud/apps/sandbox-control/test/**/*.test.ts', + 'cloud/apps/sandbox-control/image/**/*.test.ts' + ], + sources: ['cloud/apps/sandbox-control/vitest.config.ts'], + mirrors: ["include: ['test/**/*.test.ts', 'image/**/*.test.ts']"] + }, ...[ 'cloud/apps/relay-fence-broker', 'cloud/packages/cloud-database',