diff --git a/cloud/apps/sandbox-control/migrations/0004_github_bindings_guardrails.sql b/cloud/apps/sandbox-control/migrations/0004_github_bindings_guardrails.sql new file mode 100644 index 00000000000..658fddf980e --- /dev/null +++ b/cloud/apps/sandbox-control/migrations/0004_github_bindings_guardrails.sql @@ -0,0 +1,64 @@ +-- GitHub App installations bound to Orca accounts. Repo tokens are minted only for a caller whose +-- account has a live binding to the installation that covers the repo (deny by default). +CREATE TABLE github_installations ( + installation_id INTEGER NOT NULL, + account_id TEXT NOT NULL, + github_account_id INTEGER NOT NULL, + github_account_login TEXT NOT NULL, + github_account_type TEXT NOT NULL, + -- Orca user who bound it, and the GitHub user proven (OAuth) to own/admin the installation account. + bound_by_user_id TEXT NOT NULL, + bound_by_github_user_id INTEGER, + -- `oauth` (owner/org admin verified with a user-to-server token) | `operator` (support tool). + method TEXT NOT NULL, + created_at INTEGER NOT NULL, + revoked_at INTEGER, + PRIMARY KEY (installation_id, account_id) +); +CREATE INDEX github_installations_account ON github_installations (account_id); + +-- Single-use install/connect state nonces (signed state carries the account; this stops replay). +CREATE TABLE github_connect_states ( + nonce TEXT PRIMARY KEY, + account_id TEXT NOT NULL, + expires_at INTEGER NOT NULL, + consumed_at INTEGER +); + +-- Operator flags (kill switch) and cron bookkeeping; D1 so flipping needs no redeploy. +CREATE TABLE control_state ( + name TEXT PRIMARY KEY, + value TEXT NOT NULL, + updated_at INTEGER NOT NULL +); + +-- Fixed-window create counters per account. +CREATE TABLE rate_windows ( + bucket TEXT PRIMARY KEY, + window_start INTEGER NOT NULL, + count INTEGER NOT NULL +); + +-- Provider resources with no live D1 record: suspended on first sight, deleted after a grace period. +CREATE TABLE provider_orphans ( + kind TEXT NOT NULL, + provider_id TEXT NOT NULL, + first_seen_at INTEGER NOT NULL, + last_seen_at INTEGER NOT NULL, + suspended_at INTEGER, + deleted_at INTEGER, + PRIMARY KEY (kind, provider_id) +); + +-- Accounts whose subscription is canceled/unpaid: data is deleted DATA_RETENTION_DAYS after `since`. +CREATE TABLE account_holds ( + account_id TEXT PRIMARY KEY, + reason TEXT NOT NULL, + since INTEGER NOT NULL, + checked_at INTEGER NOT NULL, + warned_at INTEGER, + purged_at INTEGER +); + +-- Idle-retention warning marker for long-suspended sandboxes. +ALTER TABLE sandboxes ADD COLUMN retention_warned_at INTEGER; diff --git a/cloud/apps/sandbox-control/src/entitlements.ts b/cloud/apps/sandbox-control/src/entitlements.ts index fe4624e0300..e3692fa4259 100644 --- a/cloud/apps/sandbox-control/src/entitlements.ts +++ b/cloud/apps/sandbox-control/src/entitlements.ts @@ -7,7 +7,13 @@ import type { Env } from './env' export type MachineSize = 'standard' | 'large' /** `suspendRunning`: payment grace ran out, so the account's running sandboxes must stop too. */ -export type StartDecision = { allow: boolean; reason: string | null; suspendRunning: boolean } +export type StartDecision = { + allow: boolean + reason: string | null + suspendRunning: boolean + /** Paid seats, when billing reports them; scales the control plane's per-account backstop. */ + seats?: number +} export type UsageSegment = { usageId: string @@ -41,7 +47,8 @@ export class BillingUnavailableError extends Error { const StartAnswer = z.looseObject({ allow: z.boolean(), reason: z.string().nullable(), - suspendRunning: z.boolean().default(false) + suspendRunning: z.boolean().default(false), + seats: z.number().int().positive().max(100_000).optional() }) const UsageResponse = z.looseObject({ canStartSandbox: z.boolean(), @@ -52,6 +59,13 @@ const SuspensionsResponse = z.looseObject({ accounts: z.array(z.looseObject({ accountId: z.string() })).max(10_000) }) +/** Reasons that start the data-retention clock (canceled / unpaid). */ +export const RETENTION_HOLD_REASONS: ReadonlySet = new Set([ + 'subscription_inactive', + 'no_subscription', + 'payment_grace_expired' +]) + /** Reasons that mean running sandboxes must stop, not just that new starts wait. */ export const STOP_RUNNING_REASONS: ReadonlySet = new Set([ 'included_hours_exhausted', diff --git a/cloud/apps/sandbox-control/src/env.ts b/cloud/apps/sandbox-control/src/env.ts index cfa34aec477..3e0c3692546 100644 --- a/cloud/apps/sandbox-control/src/env.ts +++ b/cloud/apps/sandbox-control/src/env.ts @@ -39,7 +39,7 @@ export type Env = { /** Dev-only: Authorization the egress proxy injects for api.github.com until the GitHub App minter exists. */ EGRESS_DEV_STATIC_AUTHORIZATION?: string - // GitHub App (contents/pull_requests write, metadata read). The PKCS#8 key is a secret. + // GitHub App (contents/pull_requests write, metadata read, org members read). The PKCS#8 key is a secret. GITHUB_APP_ID?: string GITHUB_APP_PRIVATE_KEY?: string @@ -55,4 +55,31 @@ export type Env = { * key in (forwardURL path). Anything else: plain env injection. Needs EGRESS_PROXY_URL. */ SECRETS_EGRESS_PLACEHOLDERS?: string + + /** App slug for the install URL (github.com/apps/). */ + GITHUB_APP_SLUG?: string + /** OAuth credentials of the same app: prove who installed it before binding (secret: CLIENT_SECRET). */ + GITHUB_APP_CLIENT_ID?: string + GITHUB_APP_CLIENT_SECRET?: string + /** HMAC key for the signed connect state (secret, >= 32 chars). */ + GITHUB_STATE_SECRET?: string + + /** Operator routes (/v1/internal/*): kill switch, drain, bindings. Secret, >= 32 chars; unset = no routes. */ + OPERATOR_TOKEN?: string + /** `true` refuses every create/resume/rollover (static kill switch; the D1 flag is the runtime one). */ + SANDBOX_CREATES_DISABLED?: string + /** Global cap on active sandboxes across all accounts (default 50). */ + SANDBOX_MAX_RUNNING?: string + /** Per-account active cap per paid seat (default 3); billing enforces the plan limit as well. */ + ACCOUNT_MAX_ACTIVE_PER_SEAT?: string + /** POST /v1/sandboxes per account per minute (default 10). */ + CREATE_RATE_PER_MINUTE?: string + /** Days data is kept after cancel/unpaid before deletion (default 30). */ + DATA_RETENTION_DAYS?: string + /** Days a suspended sandbox may sit untouched before deletion (default 90). */ + IDLE_RETENTION_DAYS?: string + /** Hours an orphan provider resource is kept (suspended) before deletion (default 24). */ + ORPHAN_GRACE_HOURS?: string + /** Provider snapshot expiry after last use, days (default 104, backstop behind idle retention). */ + SNAPSHOT_EXPIRATION_DAYS?: string } diff --git a/cloud/apps/sandbox-control/src/github-app.ts b/cloud/apps/sandbox-control/src/github-app.ts index ebea80124dc..26bb8258c21 100644 --- a/cloud/apps/sandbox-control/src/github-app.ts +++ b/cloud/apps/sandbox-control/src/github-app.ts @@ -1,21 +1,50 @@ // GitHub App installation tokens for sandbox git: 1h, scoped to the workspace's one repository with // contents + pull_requests write. Minted here, injected at the egress layer, never stored in the VM. +// A token is minted only when the caller's Orca account is bound to the installation that covers the +// repo (github-connect.ts records bindings); an app installed on a repo is not by itself a grant. import { SignJWT, importPKCS8 } from 'jose' +import { z } from 'zod' export type GitHubRepoToken = { token: string; expiresAt: number } -export type GitHubAppMinter = { - /** Null when the repo is not on github.com or the app is not installed on it. */ - tokenForRepo(repoUrl: string): Promise +export type GitHubInstallation = { + id: number + account: { id: number; login: string; type: 'User' | 'Organization' } } +export type GitHubAppMinter = { + /** + * Null when the repo is not on github.com, the app is not installed on it, or the installation is + * not bound to `accountId`. Deny by default. + */ + tokenForRepo(repoUrl: string, accountId: string): Promise + /** App-level installation lookup (operator binding). Null when absent. */ + getInstallation(installationId: number): Promise + /** The installation covering a github.com repo URL, or null. */ + getRepoInstallation(repoUrl: string): Promise +} + +/** Whether `accountId` holds a live binding to the installation. */ +export type InstallationBindingCheck = (installationId: number, accountId: string) => Promise + const REFRESH_MARGIN_MS = 5 * 60_000 +const REPO_PERMISSIONS = { contents: 'write', pull_requests: 'write', metadata: 'read' } as const + +const Installation = z.looseObject({ + id: z.number().int().positive(), + account: z.looseObject({ id: z.number().int(), login: z.string(), type: z.enum(['User', 'Organization']) }) +}) +const MintedToken = z.looseObject({ + token: z.string().min(1), + expires_at: z.string(), + repositories: z.array(z.looseObject({ id: z.number(), name: z.string() })).optional() +}) export function parseGitHubRepo(repoUrl: string): { owner: string; repo: string } | null { try { const url = new URL(repoUrl) const match = /^\/([A-Za-z0-9-]+)\/([A-Za-z0-9._-]+?)(?:\.git)?\/?$/.exec(url.pathname) - return url.protocol === 'https:' && url.hostname === 'github.com' && match + return url.protocol === 'https:' && url.hostname === 'github.com' && !url.username && !url.password && match ? { owner: match[1]!, repo: match[2]! } : null } catch { @@ -26,12 +55,15 @@ export function parseGitHubRepo(repoUrl: string): { owner: string; repo: string export function createGitHubAppMinter(config: { appId: string privateKeyPkcs8: string + isBound: InstallationBindingCheck fetch?: typeof fetch now?: () => number + log?: (event: Record) => void }): GitHubAppMinter { const doFetch = config.fetch ?? fetch const now = config.now ?? Date.now - const cache = new Map() + const log = config.log ?? ((event) => console.log(JSON.stringify(event))) + const cache = new Map() let signingKey: CryptoKey | null = null async function appJwt(): Promise { @@ -59,8 +91,30 @@ export function createGitHubAppMinter(config: { }) } + async function installationFor(path: string): Promise { + const response = await github('GET', path) + if (response.status === 404) { + return null + } + if (!response.ok) { + throw new Error(`github installation lookup failed (${response.status})`) + } + const parsed = Installation.safeParse(await response.json().catch(() => null)) + if (!parsed.success) { + throw new Error('github installation lookup answered an unexpected body') + } + return parsed.data + } + return { - async tokenForRepo(repoUrl) { + getInstallation: (installationId) => installationFor(`/app/installations/${installationId}`), + + async getRepoInstallation(repoUrl) { + const parsed = parseGitHubRepo(repoUrl) + return parsed ? installationFor(`/repos/${parsed.owner}/${parsed.repo}/installation`) : null + }, + + async tokenForRepo(repoUrl, accountId) { const parsed = parseGitHubRepo(repoUrl) if (!parsed) { return null @@ -68,26 +122,36 @@ export function createGitHubAppMinter(config: { const key = `${parsed.owner}/${parsed.repo}`.toLowerCase() const cached = cache.get(key) if (cached && cached.expiresAt - now() > REFRESH_MARGIN_MS) { - return cached + // The binding is re-checked on every use: a revoked binding stops a cached token too. + return (await config.isBound(cached.installationId, accountId)) ? { token: cached.token, expiresAt: cached.expiresAt } : null } - const installation = await github('GET', `/repos/${parsed.owner}/${parsed.repo}/installation`) - if (installation.status === 404) { + const installation = await installationFor(`/repos/${parsed.owner}/${parsed.repo}/installation`) + if (!installation) { return null } - if (!installation.ok) { - throw new Error(`github installation lookup failed (${installation.status})`) + if (!(await config.isBound(installation.id, accountId))) { + log({ event: 'github_token_denied', reason: 'installation_not_bound', installationId: installation.id }) + return null } - const { id } = (await installation.json()) as { id: number } - const minted = await github('POST', `/app/installations/${id}/access_tokens`, { + const minted = await github('POST', `/app/installations/${installation.id}/access_tokens`, { repositories: [parsed.repo], - permissions: { contents: 'write', pull_requests: 'write', metadata: 'read' } + permissions: REPO_PERMISSIONS }) if (!minted.ok) { throw new Error(`github token mint failed (${minted.status})`) } - const body = (await minted.json()) as { token: string; expires_at: string } - const token = { token: body.token, expiresAt: Date.parse(body.expires_at) } - cache.set(key, token) + const body = MintedToken.safeParse(await minted.json().catch(() => null)) + // GitHub narrows to the named repo; refuse anything wider than that one repository. + if ( + !body.success || + !body.data.repositories || + body.data.repositories.length !== 1 || + body.data.repositories[0]!.name.toLowerCase() !== parsed.repo.toLowerCase() + ) { + throw new Error('github token mint answered an unexpected scope') + } + const token = { token: body.data.token, expiresAt: Date.parse(body.data.expires_at) } + cache.set(key, { ...token, installationId: installation.id }) return token } } diff --git a/cloud/apps/sandbox-control/src/github-connect.ts b/cloud/apps/sandbox-control/src/github-connect.ts new file mode 100644 index 00000000000..8610acbb13a --- /dev/null +++ b/cloud/apps/sandbox-control/src/github-connect.ts @@ -0,0 +1,305 @@ +// Binds GitHub App installations to Orca accounts. The desktop asks for a connect URL (signed, +// single-use state carrying its Orca account); GitHub sends the browser back to the setup/callback +// URL; we exchange the OAuth code for a user-to-server token and bind an installation only when that +// GitHub user owns the installation's account (personal) or is an active admin of the org. +// Installation ids in a callback URL are never trusted on their own. +import { z } from 'zod' +import type { GitHubInstallation } from './github-app' + +const STATE_TTL_MS = 10 * 60_000 + +export type GitHubConnectConfig = { + appSlug: string + clientId: string + clientSecret: string + stateSecret: string + fetch?: typeof fetch +} + +export type InstallationBinding = { + installation_id: number + account_id: string + github_account_id: number + github_account_login: string + github_account_type: string + bound_by_user_id: string + bound_by_github_user_id: number | null + method: 'oauth' | 'operator' + created_at: number + revoked_at: number | null +} + +// ---- D1 ---- + +export async function isInstallationBound(db: D1Database, installationId: number, accountId: string): Promise { + const row = await db + .prepare('SELECT 1 AS ok FROM github_installations WHERE installation_id = ? AND account_id = ? AND revoked_at IS NULL') + .bind(installationId, accountId) + .first<{ ok: number }>() + return row !== null +} + +export async function bindInstallation( + db: D1Database, + binding: Omit, + now: number +): Promise { + await db + .prepare( + `INSERT INTO github_installations (installation_id, account_id, github_account_id, github_account_login, + github_account_type, bound_by_user_id, bound_by_github_user_id, method, created_at, revoked_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL) + ON CONFLICT (installation_id, account_id) DO UPDATE SET github_account_id = excluded.github_account_id, + github_account_login = excluded.github_account_login, github_account_type = excluded.github_account_type, + bound_by_user_id = excluded.bound_by_user_id, bound_by_github_user_id = excluded.bound_by_github_user_id, + method = excluded.method, created_at = excluded.created_at, revoked_at = NULL` + ) + .bind(binding.installation_id, binding.account_id, binding.github_account_id, binding.github_account_login, + binding.github_account_type, binding.bound_by_user_id, binding.bound_by_github_user_id, binding.method, now) + .run() +} + +export async function revokeInstallation(db: D1Database, installationId: number, accountId: string | null, now: number) { + const result = await db + .prepare( + `UPDATE github_installations SET revoked_at = ? WHERE installation_id = ? AND revoked_at IS NULL + ${accountId ? 'AND account_id = ?' : ''}` + ) + .bind(...(accountId ? [now, installationId, accountId] : [now, installationId])) + .run() + return result.meta.changes +} + +export async function listAccountInstallations(db: D1Database, accountId: string) { + const { results } = await db + .prepare( + `SELECT installation_id, github_account_login, github_account_type, method, created_at + FROM github_installations WHERE account_id = ? AND revoked_at IS NULL ORDER BY created_at` + ) + .bind(accountId) + .all<{ installation_id: number; github_account_login: string; github_account_type: string; method: string; created_at: number }>() + return results +} + +// ---- signed state ---- + +const StatePayload = z.object({ + n: z.string().regex(/^[a-f0-9]{32}$/), + a: z.string().min(1).max(160), + u: z.string().min(1).max(160), + e: z.number().int(), + i: z.number().int().positive().optional() +}) +export type ConnectState = z.infer + +const b64url = (bytes: Uint8Array) => btoa(String.fromCharCode(...bytes)).replaceAll('+', '-').replaceAll('/', '_').replace(/=+$/, '') +const fromB64url = (text: string) => Uint8Array.from(atob(text.replaceAll('-', '+').replaceAll('_', '/')), (c) => c.charCodeAt(0)) + +async function hmac(secret: string, data: string): Promise { + const key = await crypto.subtle.importKey('raw', new TextEncoder().encode(secret), { name: 'HMAC', hash: 'SHA-256' }, false, ['sign']) + return new Uint8Array(await crypto.subtle.sign('HMAC', key, new TextEncoder().encode(data))) +} + +export async function signState(secret: string, payload: ConnectState): Promise { + const body = b64url(new TextEncoder().encode(JSON.stringify(payload))) + return `${body}.${b64url(await hmac(secret, body))}` +} + +/** Null on a bad signature, malformed payload or expiry. Does not consume the nonce. */ +export async function verifyState(secret: string, state: string, now: number): Promise { + const [body, mac, extra] = state.split('.') + if (!body || !mac || extra !== undefined || state.length > 2048) { + return null + } + try { + const expected = await hmac(secret, body) + const given = fromB64url(mac) + if (given.length !== expected.length || !timingSafeEqual(given, expected)) { + return null + } + const parsed = StatePayload.safeParse(JSON.parse(new TextDecoder().decode(fromB64url(body)))) + return parsed.success && parsed.data.e > now ? parsed.data : null + } catch { + return null + } +} + +export function timingSafeEqual(a: Uint8Array, b: Uint8Array): boolean { + let diff = a.length ^ b.length + for (let i = 0; i < Math.max(a.length, b.length); i++) { + diff |= (a[i] ?? 0) ^ (b[i] ?? 0) + } + return diff === 0 +} + +export async function createConnectState( + db: D1Database, + secret: string, + caller: { accountId: string; userId: string }, + now: number +): Promise { + const nonce = Array.from(crypto.getRandomValues(new Uint8Array(16)), (b) => b.toString(16).padStart(2, '0')).join('') + await db + .prepare('INSERT INTO github_connect_states (nonce, account_id, expires_at) VALUES (?, ?, ?)') + .bind(nonce, caller.accountId, now + STATE_TTL_MS) + .run() + return signState(secret, { n: nonce, a: caller.accountId, u: caller.userId, e: now + STATE_TTL_MS }) +} + +/** Marks the nonce used; false when it was already used, unknown, or issued to another account. */ +async function consumeState(db: D1Database, state: ConnectState, now: number): Promise { + const result = await db + .prepare( + `UPDATE github_connect_states SET consumed_at = ? WHERE nonce = ? AND account_id = ? AND consumed_at IS NULL AND expires_at > ?` + ) + .bind(now, state.n, state.a, now) + .run() + return result.meta.changes === 1 +} + +// ---- GitHub user-to-server verification ---- + +const UserBody = z.looseObject({ id: z.number().int(), login: z.string() }) +const UserInstallations = z.looseObject({ + installations: z.array( + z.looseObject({ + id: z.number().int(), + account: z.looseObject({ id: z.number().int(), login: z.string(), type: z.enum(['User', 'Organization']) }) + }) + ) +}) +const Membership = z.looseObject({ state: z.string(), role: z.string() }) + +export type VerifiedInstallation = GitHubInstallation & { githubUserId: number } + +/** + * Exchanges the OAuth code and returns the installations this GitHub user may bind: ones on their + * own account, or on an org where they are an active admin. `only` narrows to one installation id. + */ +export async function verifiedInstallations( + config: GitHubConnectConfig, + code: string, + only?: number +): Promise<{ githubUser: { id: number; login: string }; allowed: VerifiedInstallation[]; denied: number[] }> { + const doFetch = config.fetch ?? fetch + const exchanged = await doFetch('https://github.com/login/oauth/access_token', { + method: 'POST', + headers: { accept: 'application/json', 'content-type': 'application/json', 'user-agent': 'orca-sandbox-control' }, + body: JSON.stringify({ client_id: config.clientId, client_secret: config.clientSecret, code }), + redirect: 'manual' + }) + const token = z.looseObject({ access_token: z.string().min(1) }).safeParse(await exchanged.json().catch(() => null)) + if (!exchanged.ok || !token.success) { + throw new GitHubConnectError('oauth_exchange_failed') + } + const userApi = async (path: string) => + doFetch(`https://api.github.com${path}`, { + headers: { + authorization: `Bearer ${token.data.access_token}`, + accept: 'application/vnd.github+json', + 'x-github-api-version': '2022-11-28', + 'user-agent': 'orca-sandbox-control' + }, + redirect: 'manual' + }) + const userResponse = await userApi('/user') + const user = UserBody.safeParse(await userResponse.json().catch(() => null)) + if (!userResponse.ok || !user.success) { + throw new GitHubConnectError('github_user_unavailable') + } + const listed = await userApi('/user/installations?per_page=100') + const installations = UserInstallations.safeParse(await listed.json().catch(() => null)) + if (!listed.ok || !installations.success) { + throw new GitHubConnectError('github_installations_unavailable') + } + const candidates = installations.data.installations.filter((i) => only === undefined || i.id === only) + const allowed: VerifiedInstallation[] = [] + const denied: number[] = [] + for (const installation of candidates) { + let ok = false + if (installation.account.type === 'User') { + ok = installation.account.id === user.data.id + } else { + // Org: only an active org admin may bind it (needs the app's Members:read org permission). + const membership = await userApi(`/user/memberships/orgs/${encodeURIComponent(installation.account.login)}`) + const body = Membership.safeParse(await membership.json().catch(() => null)) + ok = membership.ok && body.success && body.data.state === 'active' && body.data.role === 'admin' + } + if (ok) { + allowed.push({ id: installation.id, account: installation.account, githubUserId: user.data.id }) + } else { + denied.push(installation.id) + } + } + if (only !== undefined && candidates.length === 0) { + denied.push(only) + } + return { githubUser: user.data, allowed, denied } +} + +export class GitHubConnectError extends Error { + constructor(readonly code: string) { + super(code) + this.name = 'GitHubConnectError' + } +} + +export function authorizeUrl(config: GitHubConnectConfig, origin: string, state: string): string { + const url = new URL('https://github.com/login/oauth/authorize') + url.searchParams.set('client_id', config.clientId) + url.searchParams.set('redirect_uri', `${origin}/v1/github/callback`) + url.searchParams.set('state', state) + url.searchParams.set('allow_signup', 'false') + return url.toString() +} + +export function installUrl(config: GitHubConnectConfig, state: string): string { + return `https://github.com/apps/${encodeURIComponent(config.appSlug)}/installations/new?state=${encodeURIComponent(state)}` +} + +/** + * Setup URL (after install) and OAuth callback. Without a code we send the browser through OAuth so + * GitHub proves who the installer is; the installation id is carried inside a re-signed state. + */ +export async function handleGitHubReturn( + db: D1Database, + config: GitHubConnectConfig, + url: URL, + now: number +): Promise<{ redirect: string } | { bound: string[]; denied: number } | { error: string }> { + const rawState = url.searchParams.get('state') ?? '' + const state = rawState ? await verifyState(config.stateSecret, rawState, now) : null + if (!state) { + return { error: 'invalid_state' } + } + const queryInstallation = Number(url.searchParams.get('installation_id') ?? NaN) + const installationId = state.i ?? (Number.isSafeInteger(queryInstallation) && queryInstallation > 0 ? queryInstallation : undefined) + const code = url.searchParams.get('code') ?? '' + if (!/^[A-Za-z0-9_-]{8,128}$/.test(code)) { + // Carry the (still untrusted) installation id through OAuth; verification happens on return. + const carried = await signState(config.stateSecret, { ...state, ...(installationId ? { i: installationId } : {}) }) + return { redirect: authorizeUrl(config, url.origin, carried) } + } + if (!(await consumeState(db, state, now))) { + return { error: 'state_used' } + } + const result = await verifiedInstallations(config, code, installationId) + for (const installation of result.allowed) { + await bindInstallation( + db, + { + installation_id: installation.id, + account_id: state.a, + github_account_id: installation.account.id, + github_account_login: installation.account.login, + github_account_type: installation.account.type, + bound_by_user_id: state.u, + bound_by_github_user_id: installation.githubUserId, + method: 'oauth' + }, + now + ) + } + console.log(JSON.stringify({ event: 'github_bind', bound: result.allowed.length, denied: result.denied.length })) + return { bound: result.allowed.map((i) => i.account.login), denied: result.denied.length } +} diff --git a/cloud/apps/sandbox-control/src/guardrails.ts b/cloud/apps/sandbox-control/src/guardrails.ts new file mode 100644 index 00000000000..dbb16908908 --- /dev/null +++ b/cloud/apps/sandbox-control/src/guardrails.ts @@ -0,0 +1,129 @@ +// Cost guardrails in front of every start (create, resume, rollover): the operator kill switch, a +// global running cap, a per-account concurrency backstop (billing enforces the plan limit too) and a +// per-account create rate limit. All are control-plane local, so a billing outage cannot lift them. +import { ACTIVE_STATUSES } from './store' + +export type GuardrailConfig = { + /** Static kill switch from the deploy config (`SANDBOX_CREATES_DISABLED=true`). */ + startsDisabled: boolean + maxRunning: number + maxActivePerSeat: number + createsPerMinute: number +} + +export const DEFAULT_GUARDRAILS: GuardrailConfig = { + startsDisabled: false, + maxRunning: 50, + maxActivePerSeat: 3, + createsPerMinute: 10 +} + +export class GuardrailError extends Error { + constructor( + readonly code: 'maintenance' | 'capacity' | 'rate_limited' | 'concurrency_limit', + readonly retryAfterSeconds?: number + ) { + super(code) + this.name = 'GuardrailError' + } +} + +const KILL_SWITCH = 'kill_switch' + +export async function getControlState(db: D1Database, name: string): Promise { + const row = await db.prepare('SELECT value FROM control_state WHERE name = ?').bind(name).first<{ value: string }>() + return row?.value ?? null +} + +export async function setControlState(db: D1Database, name: string, value: string, now: number): Promise { + await db + .prepare( + `INSERT INTO control_state (name, value, updated_at) VALUES (?, ?, ?) + ON CONFLICT (name) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at` + ) + .bind(name, value, now) + .run() +} + +export async function killSwitchOn(db: D1Database, config: GuardrailConfig): Promise { + return config.startsDisabled || (await getControlState(db, KILL_SWITCH)) === 'on' +} + +export function setKillSwitch(db: D1Database, on: boolean, now: number): Promise { + return setControlState(db, KILL_SWITCH, on ? 'on' : 'off', now) +} + +async function countActive(db: D1Database, accountId?: string): Promise { + const placeholders = ACTIVE_STATUSES.map(() => '?').join(', ') + const row = await db + .prepare( + `SELECT COUNT(*) AS n FROM sandboxes WHERE status IN (${placeholders}) ${accountId ? 'AND account_id = ?' : ''}` + ) + .bind(...ACTIVE_STATUSES, ...(accountId ? [accountId] : [])) + .first<{ n: number }>() + return row?.n ?? 0 +} + +export function countAllActive(db: D1Database): Promise { + return countActive(db) +} + +/** Kill switch + global running cap; checked before billing is asked anything. */ +export async function assertStartsOpen(db: D1Database, config: GuardrailConfig): Promise { + if (await killSwitchOn(db, config)) { + throw new GuardrailError('maintenance') + } + if ((await countAllActive(db)) >= config.maxRunning) { + console.log(JSON.stringify({ event: 'global_cap_reached', maxRunning: config.maxRunning })) + throw new GuardrailError('capacity', 60) + } +} + +/** Per-account backstop (3/seat); `seats` comes from billing when it reports one, else 1. */ +export async function assertAccountCapacity( + db: D1Database, + config: GuardrailConfig, + accountId: string, + seats = 1 +): Promise { + if ((await countActive(db, accountId)) >= config.maxActivePerSeat * Math.max(1, Math.floor(seats))) { + throw new GuardrailError('concurrency_limit') + } +} + +/** Fixed one-minute window per account; counts every create request, retries included. */ +export async function consumeCreateToken(db: D1Database, config: GuardrailConfig, accountId: string, now: number) { + const windowStart = now - (now % 60_000) + const bucket = `create:${accountId}` + await db + .prepare( + `INSERT INTO rate_windows (bucket, window_start, count) VALUES (?, ?, 1) + ON CONFLICT (bucket) DO UPDATE SET + count = CASE WHEN rate_windows.window_start = excluded.window_start THEN rate_windows.count + 1 ELSE 1 END, + window_start = excluded.window_start` + ) + .bind(bucket, windowStart) + .run() + const row = await db.prepare('SELECT count FROM rate_windows WHERE bucket = ?').bind(bucket).first<{ count: number }>() + if ((row?.count ?? 0) > config.createsPerMinute) { + throw new GuardrailError('rate_limited', Math.ceil((windowStart + 60_000 - now) / 1000)) + } +} + +export function guardrailsFromEnv(env: { + SANDBOX_CREATES_DISABLED?: string + SANDBOX_MAX_RUNNING?: string + ACCOUNT_MAX_ACTIVE_PER_SEAT?: string + CREATE_RATE_PER_MINUTE?: string +}): GuardrailConfig { + const int = (value: string | undefined, fallback: number) => { + const parsed = Number(value) + return value !== undefined && Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : fallback + } + return { + startsDisabled: env.SANDBOX_CREATES_DISABLED === 'true', + maxRunning: int(env.SANDBOX_MAX_RUNNING, DEFAULT_GUARDRAILS.maxRunning), + maxActivePerSeat: int(env.ACCOUNT_MAX_ACTIVE_PER_SEAT, DEFAULT_GUARDRAILS.maxActivePerSeat), + createsPerMinute: int(env.CREATE_RATE_PER_MINUTE, DEFAULT_GUARDRAILS.createsPerMinute) + } +} diff --git a/cloud/apps/sandbox-control/src/index.ts b/cloud/apps/sandbox-control/src/index.ts index 6f013a32480..87815c21114 100644 --- a/cloud/apps/sandbox-control/src/index.ts +++ b/cloud/apps/sandbox-control/src/index.ts @@ -7,6 +7,10 @@ // POST /v1/sandboxes/:id/lease {extendMs} -> {sandbox} // DELETE /v1/sandboxes/:id -> {sandbox} // GET /v1/secrets?scope= | PUT, DELETE /v1/secrets/:scope/:NAME (see src/secrets.ts) +// POST /v1/github/connect -> {installUrl, authorizeUrl} (bind a GitHub App installation to the account) +// GET /v1/github/installations -> {installations} +// Browser (GitHub redirects): GET /v1/github/setup, GET /v1/github/callback +// Operator (OPERATOR_TOKEN): /v1/internal/* (see operator.ts) // Vercel network-policy forwardURL target (Vercel sandbox OIDC): /v1/egress/proxy import { authenticateDesktop } from './auth' import { NO_CREDENTIALS, handleEgressProxy, type CredentialBroker, type ProxiedRoute } from './egress-proxy' @@ -29,6 +33,19 @@ import { type LifecycleDeps } from './lifecycle' import { createGitHubAppMinter, type GitHubAppMinter } from './github-app' +import { + GitHubConnectError, + authorizeUrl, + createConnectState, + handleGitHubReturn, + installUrl, + isInstallationBound, + listAccountInstallations, + type GitHubConnectConfig +} from './github-connect' +import { GuardrailError, consumeCreateToken, guardrailsFromEnv } from './guardrails' +import { handleOperator, isOperator } from './operator' +import { DEFAULT_RETENTION, type RetentionConfig } from './retention' import { reconcile } from './reconcile' import { egressSecretHosts, secretPlaceholder } from './secret-egress' import { SecretError, handleSecretsRoute, keyringFor, resolveSandboxSecrets } from './secrets' @@ -49,6 +66,9 @@ export type Deps = { now?: () => number egressFetch?: typeof fetch egressKeysForIssuer?: Parameters[1]['keysForIssuer'] + /** GitHub OAuth/user API fetch (tests). */ + githubFetch?: typeof fetch + retention?: RetentionConfig } // Decrypted model keys for the egress proxy, per sandbox session, for a minute: one D1 read and @@ -95,17 +115,52 @@ function egressRoutes(env: Env, deps: Deps): (row: SandboxRow, host: string) => } } -const minters = new Map() +const minters = new Map() function githubMinter(env: Env): GitHubAppMinter | undefined { if (!env.GITHUB_APP_ID || !env.GITHUB_APP_PRIVATE_KEY) { return undefined } - let minter = minters.get(env.GITHUB_APP_ID) - if (!minter) { - minter = createGitHubAppMinter({ appId: env.GITHUB_APP_ID, privateKeyPkcs8: env.GITHUB_APP_PRIVATE_KEY }) - minters.set(env.GITHUB_APP_ID, minter) + let entry = minters.get(env.GITHUB_APP_ID) + if (!entry) { + const db = { current: env.SANDBOX_DB } + entry = { + db, + minter: createGitHubAppMinter({ + appId: env.GITHUB_APP_ID, + privateKeyPkcs8: env.GITHUB_APP_PRIVATE_KEY, + isBound: (installationId, accountId) => isInstallationBound(db.current, installationId, accountId) + }) + } + minters.set(env.GITHUB_APP_ID, entry) + } + entry.db.current = env.SANDBOX_DB + return entry.minter +} + +function githubConnect(env: Env, deps: Deps): GitHubConnectConfig | null { + if (!env.GITHUB_APP_SLUG || !env.GITHUB_APP_CLIENT_ID || !env.GITHUB_APP_CLIENT_SECRET || (env.GITHUB_STATE_SECRET?.length ?? 0) < 32) { + return null + } + return { + appSlug: env.GITHUB_APP_SLUG, + clientId: env.GITHUB_APP_CLIENT_ID, + clientSecret: env.GITHUB_APP_CLIENT_SECRET, + stateSecret: env.GITHUB_STATE_SECRET!, + fetch: deps.githubFetch + } +} + +function retentionFromEnv(env: Env): RetentionConfig { + const num = (value: string | undefined, fallback: number, unitMs: number) => { + const parsed = Number(value) + return value !== undefined && Number.isFinite(parsed) && parsed > 0 ? parsed * unitMs : fallback + } + return { + ...DEFAULT_RETENTION, + dataRetentionMs: num(env.DATA_RETENTION_DAYS, DEFAULT_RETENTION.dataRetentionMs, 86_400_000), + idleRetentionMs: num(env.IDLE_RETENTION_DAYS, DEFAULT_RETENTION.idleRetentionMs, 86_400_000), + orphanGraceMs: num(env.ORPHAN_GRACE_HOURS, DEFAULT_RETENTION.orphanGraceMs, 3_600_000) } - return minter } function egressCredentials(env: Env): CredentialBroker { @@ -118,7 +173,7 @@ function egressCredentials(env: Env): CredentialBroker { if (host !== 'api.github.com' || !row.repo_url) { return null } - const minted = await minter.tokenForRepo(row.repo_url) + const minted = await minter.tokenForRepo(row.repo_url, row.account_id) return minted ? `Bearer ${minted.token}` : null } } @@ -146,8 +201,11 @@ function githubManifestStart(url: URL): Response { // Installable on personal accounts too; installing grants nothing until a repo is selected. public: true, hook_attributes: { url: `${origin}/v1/github/webhook-unused`, active: false }, - default_permissions: { contents: 'write', metadata: 'read', pull_requests: 'write' }, + // members:read lets the bind step confirm an org installer is an org admin (user-to-server token). + default_permissions: { contents: 'write', metadata: 'read', pull_requests: 'write', members: 'read' }, default_events: [], + // The setup URL sends the browser through OAuth itself, so existing installs can be bound too. + callback_urls: [`${origin}/v1/github/callback`], request_oauth_on_install: false } const state = crypto.randomUUID() @@ -221,7 +279,13 @@ async function lifecycleDeps(env: Env, deps: Deps): Promise= 1 + ? Math.floor(Number(env.SNAPSHOT_EXPIRATION_DAYS)) * 86_400_000 + : undefined, environment: env.SANDBOX_ENV, goldenSnapshotId: env.GOLDEN_SNAPSHOT_ID, defaultRegion: env.VERCEL_REGION ?? 'iad1', @@ -268,8 +332,45 @@ export async function handleRequest(request: Request, env: Env, deps: Deps = {}) if (url.pathname === '/v1/github/manifest-start' && env.SANDBOX_ENV !== 'prod') { return githubManifestStart(url) } - if (url.pathname === '/v1/github/setup') { - return html('Orca Cloud (dev) is installed. You can close this tab.') + if (url.pathname === '/v1/github/setup' || url.pathname === '/v1/github/callback') { + return await githubReturn(env, deps, url) + } + if (url.pathname.startsWith('/v1/internal/')) { + if (!(await isOperator(request, env.OPERATOR_TOKEN))) { + return fail('not_found', 404) + } + const life = await lifecycleDeps(env, deps) + if (!life) { + return fail('not_configured', 503) + } + return ( + (await handleOperator(request, url.pathname, { + life, + github: deps.github ?? githubMinter(env), + readJson: () => readJson(request) + })) ?? fail('not_found', 404) + ) + } + if (url.pathname === '/v1/github/connect' || url.pathname === '/v1/github/installations') { + const caller = await authenticateDesktop(env, request, (deps.now ?? Date.now)()) + if (!caller) { + return fail('unauthorized', 401) + } + if (url.pathname === '/v1/github/installations' && request.method === 'GET') { + const rows = await listAccountInstallations(env.SANDBOX_DB, caller.accountId) + return json({ + installations: rows.map((r) => ({ installationId: r.installation_id, account: r.github_account_login, accountType: r.github_account_type, boundAt: r.created_at })) + }) + } + const connect = githubConnect(env, deps) + if (url.pathname !== '/v1/github/connect' || request.method !== 'POST') { + return fail('not_found', 404) + } + if (!connect) { + return fail('not_configured', 503) + } + const state = await createConnectState(env.SANDBOX_DB, connect.stateSecret, caller, (deps.now ?? Date.now)()) + return json({ installUrl: installUrl(connect, state), authorizeUrl: authorizeUrl(connect, url.origin, state) }) } if (url.pathname === '/v1/secrets' || url.pathname.startsWith('/v1/secrets/')) { const caller = await authenticateDesktop(env, request, (deps.now ?? Date.now)()) @@ -303,6 +404,7 @@ export async function handleRequest(request: Request, env: Env, deps: Deps = {}) if (request.method !== 'POST') { return fail('not_found', 404) } + await consumeCreateToken(life.db, life.config.guardrails!, caller.accountId, life.now()) const body = CreateRequestSchema.safeParse(await readJson(request)) if (!body.success) { return fail('invalid_request', 400) @@ -346,6 +448,38 @@ export async function handleRequest(request: Request, env: Env, deps: Deps = {}) } } +async function githubReturn(env: Env, deps: Deps, url: URL): Promise { + const connect = githubConnect(env, deps) + if (!connect) { + // Not configured for binding: an install alone grants nothing. + return html('Orca Cloud is installed on GitHub. Connect it from Orca to use it with your workspaces.') + } + try { + const result = await handleGitHubReturn(env.SANDBOX_DB, connect, url, (deps.now ?? Date.now)()) + if ('redirect' in result) { + return new Response(null, { status: 302, headers: { location: result.redirect, 'cache-control': 'no-store' } }) + } + if ('error' in result) { + return html( + result.error === 'state_used' + ? 'This link was already used. Start again from Orca.' + : 'GitHub is installed, but this page was not opened from Orca. Start from Orca to connect it to your account.' + ) + } + const esc = (value: string) => value.replace(/[^A-Za-z0-9_.-]/g, '') + return html( + result.bound.length > 0 + ? `Connected GitHub ${result.bound.map((login) => `${esc(login)}`).join(', ')} to Orca. You can close this tab.` + : 'Nothing was connected: only the owner of a personal account or an organization admin can connect an installation.' + ) + } catch (error) { + if (error instanceof GitHubConnectError) { + return html('GitHub did not confirm the sign-in. Start again from Orca.') + } + throw error + } +} + function errorResponse(error: unknown): Response { if (error instanceof SyntaxError) { return fail('invalid_json', 400) @@ -353,6 +487,17 @@ function errorResponse(error: unknown): Response { if (error instanceof SecretError) { return fail(error.code, error.status, error.reason) } + if (error instanceof GuardrailError) { + if (error.code === 'concurrency_limit') { + return fail('entitlement_denied', 403, 'concurrency_limit') + } + const status = { maintenance: 503, capacity: 503, rate_limited: 429 }[error.code] + const response = fail(error.code, status) + if (error.retryAfterSeconds) { + response.headers.set('retry-after', String(error.retryAfterSeconds)) + } + return response + } if (error instanceof LifecycleError) { const status = { not_found: 404, conflict: 409, entitlement_denied: 403, provider_unavailable: 502 }[error.code] return fail(error.code, status, error.reason) diff --git a/cloud/apps/sandbox-control/src/lifecycle.ts b/cloud/apps/sandbox-control/src/lifecycle.ts index 1754da3a6af..8d70a55fae0 100644 --- a/cloud/apps/sandbox-control/src/lifecycle.ts +++ b/cloud/apps/sandbox-control/src/lifecycle.ts @@ -2,7 +2,16 @@ // a forced stop before the provider's 24h cap, usage metering and write-ahead rows. import type { EntitlementClient, MachineSize } from './entitlements' import type { GitHubAppMinter } from './github-app' +import { + DEFAULT_GUARDRAILS, + GuardrailError, + assertAccountCapacity, + assertStartsOpen, + killSwitchOn, + type GuardrailConfig +} from './guardrails' import { compileNetworkPolicy, gitBasicAuthorization, type CredentialRoute } from './network-policy' +import type { RetentionConfig } from './retention' import type { ProviderSandboxState, SandboxProviderApi } from './sandbox-provider' import { applySecretPlaceholders, egressSecretHosts, secretPlaceholder } from './secret-egress' import { resolveSandboxSecrets } from './secrets' @@ -29,6 +38,9 @@ const PROVIDER_SESSION_CAP_MS = 24 * 60 * 60_000 /** Orca stops 10 min early so it, not the provider, ends the session (drain first). */ export const SESSION_DEADLINE_MS = PROVIDER_SESSION_CAP_MS - 10 * 60_000 const VCPUS: Record = { standard: 4, large: 8 } +const DAY_MS = 24 * 60 * 60_000 +/** Provider-side backstop: longer than idle retention (90d) + its warning, so Orca deletes first. */ +export const DEFAULT_SNAPSHOT_EXPIRATION_MS = 104 * DAY_MS export type LifecycleDeps = { db: D1Database @@ -39,6 +51,8 @@ export type LifecycleDeps = { /** Secret store; absent (no SECRETS_KEK) means sessions start without injected secrets. */ secrets?: { keyring: Keyring; egressPlaceholders: boolean } now: () => number + /** Orphan/retention policy for the cron; defaults in retention.ts. */ + retention?: RetentionConfig config: { environment: string goldenSnapshotId: string @@ -46,6 +60,8 @@ export type LifecycleDeps = { egressProxyUrl?: string /** Staging rehearsals shorten the cap; production uses SESSION_DEADLINE_MS. */ sessionDeadlineMs?: number + guardrails?: GuardrailConfig + snapshotExpirationMs?: number } } @@ -136,7 +152,12 @@ async function claim(deps: LifecycleDeps, row: SandboxRow, patch: Partial { @@ -168,6 +190,7 @@ async function boot(deps: LifecycleDeps, row: SandboxRow, state: ProviderSandbox const running = await claim(deps, row, { status: 'running', last_error: null, + retention_warned_at: null, github_token_expires_at: git?.expiresAt ?? null, egress_secret_hosts: routed.egress_secret_hosts, ...sessionFields(deps, state) @@ -201,7 +224,7 @@ export async function refreshGitCredentials( if (!deps.github || !row.repo_url) { return null } - const minted = await deps.github.tokenForRepo(row.repo_url) + const minted = await deps.github.tokenForRepo(row.repo_url, row.account_id) if (!minted) { return null } @@ -253,6 +276,7 @@ export async function createSandbox( timeoutMs: LEASE_MS, ports: [ORCAD_PORT], networkPolicy: egressPolicy(deps, row), + snapshotExpirationMs: deps.config.snapshotExpirationMs ?? DEFAULT_SNAPSHOT_EXPIRATION_MS, tags: { orcaSandbox: row.id, orcaEnv: deps.config.environment } }) } catch (error) { @@ -366,6 +390,9 @@ export async function rolloverSandbox(deps: LifecycleDeps, row: SandboxRow): Pro if (row.status !== 'running') { throw new LifecycleError('conflict', `sandbox_${row.status}`) } + if (await killSwitchOn(deps.db, guardrails(deps))) { + throw new GuardrailError('maintenance') + } const suspending = await claim(deps, row, { status: 'suspending' }) const stopped = await stopForSuspend(deps, suspending) await flushUsage(deps.db, deps.entitlements, deps.now(), row.id) @@ -410,7 +437,13 @@ export async function destroySandbox(deps: LifecycleDeps, row: SandboxRow): Prom throw new LifecycleError('provider_unavailable', 'snapshot_cleanup_incomplete') } await markSnapshotsDeleted(deps.db, row.id, deps.now()) - const destroyed = await claim(deps, destroying, { status: 'destroyed', ...NO_SESSION, destroyed_at: deps.now() }) + const destroyed = await claim(deps, destroying, { + status: 'destroyed', + ...NO_SESSION, + destroyed_at: deps.now(), + // Tombstone: the row stays for usage/audit, the workspace's repository does not. + repo_url: null + }) await flushUsage(deps.db, deps.entitlements, deps.now(), row.id) return destroyed } diff --git a/cloud/apps/sandbox-control/src/operator.ts b/cloud/apps/sandbox-control/src/operator.ts new file mode 100644 index 00000000000..b23767bc5e9 --- /dev/null +++ b/cloud/apps/sandbox-control/src/operator.ts @@ -0,0 +1,117 @@ +// Operator-only routes (Bearer OPERATOR_TOKEN, a Worker secret; absent = routes do not exist): +// GET /v1/internal/status +// POST /v1/internal/kill-switch {enabled, drain?} refuse creates/resumes; drain suspends (snapshot kept) +// POST /v1/internal/drain-all +// POST /v1/internal/housekeeping orphan sweep + retention now +// POST /v1/internal/github/bindings {installationId, accountId} support binding (audited as `operator`) +// DELETE /v1/internal/github/bindings {installationId, accountId?} +import { z } from 'zod' +import { readBearer } from './auth' +import type { GitHubAppMinter } from './github-app' +import { bindInstallation, revokeInstallation, timingSafeEqual } from './github-connect' +import { countAllActive, getControlState, killSwitchOn, setKillSwitch } from './guardrails' +import { guardrails, type LifecycleDeps } from './lifecycle' +import { drainAll, housekeeping } from './retention' +import { listByStatus } from './store' + +const KillSwitchBody = z.object({ enabled: z.boolean(), drain: z.boolean().optional() }).strict() +const AccountId = z.string().min(1).max(160).regex(/^(user|org|dev):[A-Za-z0-9_.:-]+$/) +const BindingBody = z + .object({ + installationId: z.number().int().positive().optional(), + /** Alternative to installationId: the installation covering this github.com repo. */ + repoUrl: z.url({ protocol: /^https$/ }).max(2048).optional(), + accountId: AccountId + }) + .strict() + .refine((body) => (body.installationId === undefined) !== (body.repoUrl === undefined)) +const UnbindBody = z.object({ installationId: z.number().int().positive(), accountId: AccountId.optional() }).strict() + +async function digest(value: string): Promise { + return new Uint8Array(await crypto.subtle.digest('SHA-256', new TextEncoder().encode(value))) +} + +export async function isOperator(request: Request, operatorToken: string | undefined): Promise { + const presented = readBearer(request.headers.get('authorization')) + if (!operatorToken || operatorToken.length < 32 || !presented) { + return false + } + return timingSafeEqual(await digest(presented), await digest(operatorToken)) +} + +export async function handleOperator( + request: Request, + path: string, + deps: { life: LifecycleDeps; github?: GitHubAppMinter; readJson: () => Promise } +): Promise { + const { life } = deps + const now = life.now() + const route = `${request.method} ${path}` + switch (route) { + case 'GET /v1/internal/status': + return Response.json({ + killSwitch: await killSwitchOn(life.db, guardrails(life)), + staticDisabled: guardrails(life).startsDisabled, + active: await countAllActive(life.db), + running: (await listByStatus(life.db, ['running'])).length, + guardrails: guardrails(life), + lastHousekeepingAt: Number((await getControlState(life.db, 'housekeeping_at')) ?? 0) || null + }) + case 'POST /v1/internal/kill-switch': { + const body = KillSwitchBody.safeParse(await deps.readJson()) + if (!body.success) { + return Response.json({ error: 'invalid_request' }, { status: 400 }) + } + await setKillSwitch(life.db, body.data.enabled, now) + console.log(JSON.stringify({ event: 'kill_switch', enabled: body.data.enabled })) + const drained = body.data.enabled && body.data.drain !== false ? await drainAll(life) : null + return Response.json({ killSwitch: body.data.enabled, drained }) + } + case 'POST /v1/internal/drain-all': + return Response.json({ drained: await drainAll(life) }) + case 'POST /v1/internal/housekeeping': + return Response.json({ housekeeping: await housekeeping(life, { force: true }) }) + case 'POST /v1/internal/github/bindings': { + const body = BindingBody.safeParse(await deps.readJson()) + if (!body.success) { + return Response.json({ error: 'invalid_request' }, { status: 400 }) + } + if (!deps.github) { + return Response.json({ error: 'not_configured' }, { status: 503 }) + } + const installation = body.data.installationId + ? await deps.github.getInstallation(body.data.installationId) + : await deps.github.getRepoInstallation(body.data.repoUrl!) + if (!installation) { + return Response.json({ error: 'not_found' }, { status: 404 }) + } + await bindInstallation( + life.db, + { + installation_id: installation.id, + account_id: body.data.accountId, + github_account_id: installation.account.id, + github_account_login: installation.account.login, + github_account_type: installation.account.type, + bound_by_user_id: 'operator', + bound_by_github_user_id: null, + method: 'operator' + }, + now + ) + console.log(JSON.stringify({ event: 'github_bind_operator', installationId: installation.id, accountId: body.data.accountId })) + return Response.json({ bound: { installationId: installation.id, accountId: body.data.accountId, githubAccount: installation.account.login } }) + } + case 'DELETE /v1/internal/github/bindings': { + const body = UnbindBody.safeParse(await deps.readJson()) + if (!body.success) { + return Response.json({ error: 'invalid_request' }, { status: 400 }) + } + const revoked = await revokeInstallation(life.db, body.data.installationId, body.data.accountId ?? null, now) + console.log(JSON.stringify({ event: 'github_unbind_operator', installationId: body.data.installationId, revoked })) + return Response.json({ revoked }) + } + default: + return null + } +} diff --git a/cloud/apps/sandbox-control/src/reconcile.ts b/cloud/apps/sandbox-control/src/reconcile.ts index 2f0613a354d..8dc08b323e7 100644 --- a/cloud/apps/sandbox-control/src/reconcile.ts +++ b/cloud/apps/sandbox-control/src/reconcile.ts @@ -11,6 +11,9 @@ import { suspendSandbox, type LifecycleDeps } from './lifecycle' +import { guardrails } from './lifecycle' +import { killSwitchOn } from './guardrails' +import { drainAll, housekeeping, openHold, type HousekeepingSummary } from './retention' import { listByStatus, transition, type SandboxRow } from './store' import { flushUsage, queueUsage } from './usage' @@ -19,6 +22,7 @@ const STUCK_MS = 15 * 60_000 const FORCED_STOP_GRACE_MS = 5 * 60_000 export type ReconcileSummary = Record< + | 'drained' | 'settled' | 'suspendedForDeadline' | 'suspendedForBilling' @@ -27,10 +31,11 @@ export type ReconcileSummary = Record< | 'repaired' | 'billingErrors', number -> +> & { housekeeping?: HousekeepingSummary } export async function reconcile(deps: LifecycleDeps): Promise { const summary: ReconcileSummary = { + drained: 0, settled: 0, suspendedForDeadline: 0, suspendedForBilling: 0, @@ -42,11 +47,19 @@ export async function reconcile(deps: LifecycleDeps): Promise const polled = await guard(async () => { for (const accountId of await deps.entitlements.listSuspensions()) { summary.suspendedForBilling += await suspendAccountRunning(deps, accountId) + // Payment grace expired: the data-retention clock starts (no-op if already running). + await openHold(deps.db, accountId, 'payment_grace_expired', deps.now()) } }) if (!polled) { summary.billingErrors++ } + if (await killSwitchOn(deps.db, guardrails(deps))) { + // Kill switch on: keep draining until nothing runs (a drain call may have timed out mid-way). + await guard(async () => { + summary.drained += (await drainAll(deps)).suspended + }) + } for (const row of await listByStatus(deps.db, ['running'])) { await guard(() => reconcileRunning(deps, row, summary)) } @@ -69,6 +82,12 @@ export async function reconcile(deps: LifecycleDeps): Promise }) } await flushUsage(deps.db, deps.entitlements, deps.now()) + await guard(async () => { + const housekept = await housekeeping(deps) + if (housekept.swept) { + summary.housekeeping = housekept + } + }) return summary } diff --git a/cloud/apps/sandbox-control/src/retention.ts b/cloud/apps/sandbox-control/src/retention.ts new file mode 100644 index 00000000000..1b3f01ebd02 --- /dev/null +++ b/cloud/apps/sandbox-control/src/retention.ts @@ -0,0 +1,399 @@ +// Hourly cron housekeeping: provider-vs-D1 orphan sweep, data deletion after cancel/unpaid, and idle +// retention for long-suspended sandboxes. Every destructive step is two-phase (seen, then acted on +// after a grace period) and logged; partial provider listings never delete anything. +import { RETENTION_HOLD_REASONS } from './entitlements' +import { destroySandbox, suspendSandbox, type LifecycleDeps } from './lifecycle' +import { getControlState, setControlState } from './guardrails' +import { ACTIVE_STATUSES, getSandboxByProviderName, listByStatus, type SandboxRow } from './store' + +const HOUR_MS = 60 * 60_000 +const DAY_MS = 24 * HOUR_MS + +export type RetentionConfig = { + /** Data kept this long after an account goes canceled/unpaid, then destroyed. */ + dataRetentionMs: number + /** Suspended sandboxes untouched this long are destroyed (active subscriptions too). */ + idleRetentionMs: number + /** Warning hook fires this long before either deletion. */ + warningLeadMs: number + /** Orphans are suspended on first sight and deleted once seen for this long. */ + orphanGraceMs: number + /** Housekeeping cadence inside the 5-minute cron. */ + sweepIntervalMs: number + /** Re-ask billing about an account's standing at most this often. */ + holdProbeIntervalMs: number +} + +export const DEFAULT_RETENTION: RetentionConfig = { + dataRetentionMs: 30 * DAY_MS, + idleRetentionMs: 90 * DAY_MS, + warningLeadMs: 7 * DAY_MS, + orphanGraceMs: 24 * HOUR_MS, + sweepIntervalMs: HOUR_MS, + holdProbeIntervalMs: DAY_MS +} + +export type RetentionWarning = { + kind: 'data_deletion' | 'idle_deletion' + accountId: string + sandboxIds: string[] + deleteAt: number + reason: string +} + +/** Email hook. Stubbed: logs a structured event the alerting Worker / mailer can pick up. */ +export type RetentionNotifier = { warn(warning: RetentionWarning): Promise } +export const LOG_NOTIFIER: RetentionNotifier = { + async warn(warning) { + console.log(JSON.stringify({ event: 'retention_warning', ...warning })) + } +} + +export type HousekeepingSummary = { + swept: boolean + orphanSandboxesSuspended: number + orphanSandboxesDeleted: number + orphanSnapshotsSeen: number + orphanSnapshotsDeleted: number + rowsMissingProvider: number + holdsOpened: number + holdsCleared: number + retentionWarnings: number + retentionDestroyed: number + idleWarnings: number + idleDestroyed: number +} + +function log(event: string, fields: Record) { + console.log(JSON.stringify({ event, ...fields })) +} + +function retention(deps: LifecycleDeps): RetentionConfig { + return deps.retention ?? DEFAULT_RETENTION +} + +// ---- account holds (canceled / unpaid) ---- + +type HoldRow = { account_id: string; reason: string; since: number; checked_at: number; warned_at: number | null; purged_at: number | null } + +/** Records that billing says this account is canceled/unpaid; the retention clock starts at first sight. */ +export async function openHold(db: D1Database, accountId: string, reason: string, now: number): Promise { + const existing = await db.prepare('SELECT * FROM account_holds WHERE account_id = ?').bind(accountId).first() + if (existing && existing.reason !== 'active') { + await db.prepare('UPDATE account_holds SET checked_at = ? WHERE account_id = ?').bind(now, accountId).run() + return false + } + await db + .prepare( + `INSERT INTO account_holds (account_id, reason, since, checked_at) VALUES (?, ?, ?, ?) + ON CONFLICT (account_id) DO UPDATE SET reason = excluded.reason, since = excluded.since, + checked_at = excluded.checked_at, warned_at = NULL, purged_at = NULL` + ) + .bind(accountId, reason, now, now) + .run() + log('retention_hold_opened', { accountId, reason }) + return true +} + +/** Billing allows starts again (resubscribed / paid): the hold and its clock are dropped. */ +async function clearHold(db: D1Database, accountId: string, now: number): Promise { + const existing = await db.prepare('SELECT * FROM account_holds WHERE account_id = ?').bind(accountId).first() + await db + .prepare( + `INSERT INTO account_holds (account_id, reason, since, checked_at) VALUES (?, 'active', ?, ?) + ON CONFLICT (account_id) DO UPDATE SET reason = 'active', since = excluded.since, checked_at = excluded.checked_at, + warned_at = NULL, purged_at = NULL` + ) + .bind(accountId, now, now) + .run() + return existing !== null && existing.reason !== 'active' +} + +/** Accounts with stored sandboxes whose standing has not been checked recently. */ +async function accountsToProbe(db: D1Database, before: number): Promise<{ account_id: string; machine_size: SandboxRow['machine_size'] }[]> { + const { results } = await db + .prepare( + `SELECT s.account_id, MIN(s.machine_size) AS machine_size FROM sandboxes s + LEFT JOIN account_holds h ON h.account_id = s.account_id + WHERE s.status <> 'destroyed' AND (h.checked_at IS NULL OR h.checked_at < ?) + GROUP BY s.account_id LIMIT 50` + ) + .bind(before) + .all<{ account_id: string; machine_size: SandboxRow['machine_size'] }>() + return results +} + +async function probeHolds(deps: LifecycleDeps, summary: HousekeepingSummary): Promise { + const now = deps.now() + for (const account of await accountsToProbe(deps.db, now - retention(deps).holdProbeIntervalMs)) { + try { + // runningSandboxes 0: we ask about the account's standing, not concurrency. + const decision = await deps.entitlements.checkStart({ accountId: account.account_id, machineSize: account.machine_size, runningSandboxes: 0 }) + if (!decision.allow && RETENTION_HOLD_REASONS.has(decision.reason ?? '')) { + summary.holdsOpened += (await openHold(deps.db, account.account_id, decision.reason!, now)) ? 1 : 0 + } else if (decision.allow) { + summary.holdsCleared += (await clearHold(deps.db, account.account_id, now)) ? 1 : 0 + } else { + // Other denials (hours exhausted, cap): not a cancellation; just remember we looked. + await deps.db.prepare('UPDATE account_holds SET checked_at = ? WHERE account_id = ?').bind(now, account.account_id).run() + } + } catch { + // Billing unavailable: never start a deletion clock on an outage. + } + } +} + +async function enforceHolds(deps: LifecycleDeps, notifier: RetentionNotifier, summary: HousekeepingSummary) { + const config = retention(deps) + const now = deps.now() + const { results } = await deps.db + .prepare(`SELECT * FROM account_holds WHERE reason <> 'active' AND purged_at IS NULL AND since <= ? LIMIT 100`) + .bind(now - (config.dataRetentionMs - config.warningLeadMs)) + .all() + for (const hold of results) { + const deleteAt = hold.since + config.dataRetentionMs + const rows = await liveRows(deps.db, hold.account_id) + if (hold.warned_at === null) { + await notifier.warn({ kind: 'data_deletion', accountId: hold.account_id, sandboxIds: rows.map((r) => r.id), deleteAt, reason: hold.reason }) + await deps.db.prepare('UPDATE account_holds SET warned_at = ? WHERE account_id = ?').bind(now, hold.account_id).run() + summary.retentionWarnings++ + continue + } + // Deletion waits for both the deadline and a full warning lead after the warning went out. + if (now < deleteAt || now < hold.warned_at + config.warningLeadMs) { + continue + } + let remaining = 0 + for (const row of rows) { + try { + await destroySandbox(deps, row) + summary.retentionDestroyed++ + } catch { + remaining++ + } + } + if (remaining === 0) { + await deps.db.prepare('UPDATE account_holds SET purged_at = ? WHERE account_id = ?').bind(now, hold.account_id).run() + } + log('retention_purge', { accountId: hold.account_id, destroyed: rows.length - remaining, remaining, reason: hold.reason }) + } +} + +async function liveRows(db: D1Database, accountId: string): Promise { + const { results } = await db + .prepare(`SELECT * FROM sandboxes WHERE account_id = ? AND status <> 'destroyed'`) + .bind(accountId) + .all() + return results +} + +// ---- idle retention ---- + +async function enforceIdle(deps: LifecycleDeps, notifier: RetentionNotifier, summary: HousekeepingSummary) { + const config = retention(deps) + const now = deps.now() + const { results } = await deps.db + .prepare(`SELECT * FROM sandboxes WHERE status = 'suspended' AND updated_at <= ? ORDER BY updated_at LIMIT 100`) + .bind(now - (config.idleRetentionMs - config.warningLeadMs)) + .all() + for (const row of results) { + if (row.retention_warned_at === null) { + await notifier.warn({ + kind: 'idle_deletion', + accountId: row.account_id, + sandboxIds: [row.id], + deleteAt: Math.max(row.updated_at + config.idleRetentionMs, now + config.warningLeadMs), + reason: 'idle' + }) + // Not a transition: the idle clock (updated_at) must not move. + await deps.db.prepare('UPDATE sandboxes SET retention_warned_at = ? WHERE id = ? AND generation = ?').bind(now, row.id, row.generation).run() + summary.idleWarnings++ + } else if (now >= row.updated_at + config.idleRetentionMs && now >= row.retention_warned_at + config.warningLeadMs) { + try { + await destroySandbox(deps, row) + summary.idleDestroyed++ + log('idle_retention_destroy', { sandboxId: row.id, accountId: row.account_id, idleDays: Math.floor((now - row.updated_at) / DAY_MS) }) + } catch { + // Retried next sweep. + } + } + } +} + +// ---- orphan sweep ---- + +async function upsertOrphan(db: D1Database, kind: 'sandbox' | 'snapshot', id: string, now: number) { + await db + .prepare( + `INSERT INTO provider_orphans (kind, provider_id, first_seen_at, last_seen_at) VALUES (?, ?, ?, ?) + ON CONFLICT (kind, provider_id) DO UPDATE SET last_seen_at = excluded.last_seen_at` + ) + .bind(kind, id, now, now) + .run() + return (await db + .prepare('SELECT * FROM provider_orphans WHERE kind = ? AND provider_id = ?') + .bind(kind, id) + .first<{ first_seen_at: number; suspended_at: number | null; deleted_at: number | null }>())! +} + +async function clearOrphan(db: D1Database, kind: string, id: string) { + await db.prepare('DELETE FROM provider_orphans WHERE kind = ? AND provider_id = ? AND deleted_at IS NULL').bind(kind, id).run() +} + +async function sweepOrphans(deps: LifecycleDeps, summary: HousekeepingSummary): Promise { + const now = deps.now() + const grace = retention(deps).orphanGraceMs + const env = deps.config.environment + const listing = await deps.provider.listSandboxes() + const providerNames = new Set(listing.sandboxes.map((s) => s.name)) + for (const box of listing.sandboxes) { + // Only this environment's sandboxes; golden-image build boxes and other envs are not ours to judge. + if (box.tags.orcaEnv !== env || !box.tags.orcaSandbox) { + continue + } + const row = await getSandboxByProviderName(deps.db, box.name) + if (row && row.status !== 'destroyed') { + await clearOrphan(deps.db, 'sandbox', box.name) + continue + } + const orphan = await upsertOrphan(deps.db, 'sandbox', box.name, now) + if (orphan.deleted_at !== null) { + continue + } + if (now - orphan.first_seen_at >= grace) { + await deps.provider.deleteSandbox(box.name) + for (const snapshot of await deps.provider.listSnapshots(box.name)) { + if (snapshot.status === 'created' && snapshot.id !== deps.config.goldenSnapshotId) { + await deps.provider.deleteSnapshot(snapshot.id) + } + } + await deps.db.prepare(`UPDATE provider_orphans SET deleted_at = ? WHERE kind = 'sandbox' AND provider_id = ?`).bind(now, box.name).run() + summary.orphanSandboxesDeleted++ + log('orphan_sandbox_deleted', { name: box.name, d1: row ? row.status : 'missing', firstSeenAt: orphan.first_seen_at }) + } else if (box.status === 'running' && orphan.suspended_at === null) { + // Stop the meter now; the automatic snapshot keeps the disk until the grace period ends. + await deps.provider.stopSession(box.currentSessionId) + await deps.db.prepare(`UPDATE provider_orphans SET suspended_at = ? WHERE kind = 'sandbox' AND provider_id = ?`).bind(now, box.name).run() + summary.orphanSandboxesSuspended++ + log('orphan_sandbox_suspended', { name: box.name, d1: row ? row.status : 'missing' }) + } + } + if (!listing.complete) { + log('orphan_sweep_partial', { listed: listing.sandboxes.length }) + return + } + // D1 says it exists, the provider does not: reported, never auto-repaired (wake fails cleanly). + for (const row of await listByStatus(deps.db, ['running', 'suspended'])) { + if (row.provider_name.startsWith(`orca-${env}-`) && !providerNames.has(row.provider_name) && now - row.updated_at > grace) { + summary.rowsMissingProvider++ + } + } + await sweepSnapshots(deps, listing.sandboxes.map((s) => s.name), summary) +} + +/** Automatic snapshots whose sandbox no longer exists anywhere in the project. Golden (manual) ones are never touched. */ +async function sweepSnapshots(deps: LifecycleDeps, sandboxNames: string[], summary: HousekeepingSummary) { + const now = deps.now() + const grace = retention(deps).orphanGraceMs + if (sandboxNames.length > 500) { + log('orphan_snapshot_sweep_skipped', { sandboxes: sandboxNames.length }) + return + } + const listed = await deps.provider.listProjectSnapshots() + if (!listed.complete) { + log('orphan_snapshot_sweep_partial', { listed: listed.snapshots.length }) + return + } + const owned = new Set([deps.config.goldenSnapshotId]) + for (const name of sandboxNames) { + for (const snapshot of await deps.provider.listSnapshots(name)) { + owned.add(snapshot.id) + } + } + const { results } = await deps.db + .prepare( + `SELECT ss.snapshot_id FROM sandbox_snapshots ss JOIN sandboxes s ON s.id = ss.sandbox_id + WHERE ss.deleted_at IS NULL AND s.status <> 'destroyed'` + ) + .all<{ snapshot_id: string }>() + for (const r of results) { + owned.add(r.snapshot_id) + } + for (const snapshot of listed.snapshots) { + if (snapshot.status !== 'created' || snapshot.creationMethod !== 'automatic' || owned.has(snapshot.id)) { + if (owned.has(snapshot.id)) { + await clearOrphan(deps.db, 'snapshot', snapshot.id) + } + continue + } + summary.orphanSnapshotsSeen++ + const orphan = await upsertOrphan(deps.db, 'snapshot', snapshot.id, now) + if (orphan.deleted_at === null && now - orphan.first_seen_at >= grace) { + await deps.provider.deleteSnapshot(snapshot.id) + await deps.db.prepare(`UPDATE provider_orphans SET deleted_at = ? WHERE kind = 'snapshot' AND provider_id = ?`).bind(now, snapshot.id).run() + summary.orphanSnapshotsDeleted++ + log('orphan_snapshot_deleted', { id: snapshot.id, createdAt: snapshot.createdAt, firstSeenAt: orphan.first_seen_at }) + } + } +} + +// ---- entry points ---- + +/** Kill switch drain: snapshot-preserving suspend of every running sandbox (one batch per call). */ +export async function drainAll(deps: LifecycleDeps): Promise<{ suspended: number; failed: number; remaining: number }> { + let suspended = 0 + let failed = 0 + for (const row of await listByStatus(deps.db, ['running'])) { + try { + await suspendSandbox(deps, row) + suspended++ + } catch { + failed++ + } + } + const remaining = (await listByStatus(deps.db, ACTIVE_STATUSES)).length + log('drain_all', { suspended, failed, remaining }) + return { suspended, failed, remaining } +} + +/** Runs at most once per sweep interval; each part is isolated so one failure does not stop the rest. */ +export async function housekeeping( + deps: LifecycleDeps, + options: { force?: boolean; notifier?: RetentionNotifier } = {} +): Promise { + const summary: HousekeepingSummary = { + swept: false, + orphanSandboxesSuspended: 0, + orphanSandboxesDeleted: 0, + orphanSnapshotsSeen: 0, + orphanSnapshotsDeleted: 0, + rowsMissingProvider: 0, + holdsOpened: 0, + holdsCleared: 0, + retentionWarnings: 0, + retentionDestroyed: 0, + idleWarnings: 0, + idleDestroyed: 0 + } + const now = deps.now() + const last = Number((await getControlState(deps.db, 'housekeeping_at')) ?? 0) + if (!options.force && now - last < retention(deps).sweepIntervalMs) { + return summary + } + await setControlState(deps.db, 'housekeeping_at', String(now), now) + summary.swept = true + const notifier = options.notifier ?? LOG_NOTIFIER + for (const step of [ + () => sweepOrphans(deps, summary), + () => probeHolds(deps, summary), + () => enforceHolds(deps, notifier, summary), + () => enforceIdle(deps, notifier, summary) + ]) { + try { + await step() + } catch (error) { + log('housekeeping_step_failed', { message: error instanceof Error ? error.message : 'unknown' }) + } + } + return summary +} diff --git a/cloud/apps/sandbox-control/src/sandbox-provider.ts b/cloud/apps/sandbox-control/src/sandbox-provider.ts index 270fefca426..49e720b0543 100644 --- a/cloud/apps/sandbox-control/src/sandbox-provider.ts +++ b/cloud/apps/sandbox-control/src/sandbox-provider.ts @@ -41,6 +41,22 @@ export type ProviderSnapshot = { sourceSessionId: string } +/** Project-wide listing entries for the orphan sweep. */ +export type ProviderSandboxSummary = { + name: string + status: ProviderSessionStatus + currentSessionId: string + tags: Record +} + +export type ProviderProjectSnapshot = { + id: string + status: 'created' | 'deleted' | 'failed' + /** `automatic` (session stop) | `manual` (golden images); absent on old snapshots. */ + creationMethod: string | null + createdAt: number +} + export type CreateSandboxParams = { name: string snapshotId: string @@ -50,6 +66,8 @@ export type CreateSandboxParams = { ports: number[] networkPolicy: NetworkPolicy tags: Record + /** Provider-side snapshot expiry after last use (backstop behind Orca's own retention); 0 = never. */ + snapshotExpirationMs: number } export type RunCommandParams = { @@ -76,6 +94,10 @@ export type SandboxProviderApi = { listSnapshots(name: string): Promise /** Resolves when already gone. */ deleteSnapshot(snapshotId: string): Promise + /** Every sandbox in Orca's project (all environments). `complete` is false when the listing was cut short. */ + listSandboxes(): Promise<{ sandboxes: ProviderSandboxSummary[]; complete: boolean }> + /** Every snapshot in Orca's project. */ + listProjectSnapshots(): Promise<{ snapshots: ProviderProjectSnapshot[]; complete: boolean }> } /** Provider failed or answered unexpectedly; the body is never surfaced to callers. */ diff --git a/cloud/apps/sandbox-control/src/store.ts b/cloud/apps/sandbox-control/src/store.ts index 7e13b8eed64..a052a109754 100644 --- a/cloud/apps/sandbox-control/src/store.ts +++ b/cloud/apps/sandbox-control/src/store.ts @@ -36,6 +36,7 @@ export type SandboxRow = { github_token_expires_at: number | null /** Comma list of model API hosts whose key the egress proxy swaps in this session. */ egress_secret_hosts: string | null + retention_warned_at: number | null } export type SessionFields = Pick< @@ -127,7 +128,7 @@ type Patch = Partial([ 'status', 'session_id', 'session_started_at', 'session_deadline_at', 'lease_expires_at', 'usage_settled_at', 'last_error', 'destroyed_at', 'github_token_expires_at', - 'egress_secret_hosts' + 'egress_secret_hosts', 'repo_url', 'retention_warned_at' ]) /** Applies `patch` only if nobody moved the row since `expected` was read; null on a lost race. */ diff --git a/cloud/apps/sandbox-control/src/vercel-sandbox-api.ts b/cloud/apps/sandbox-control/src/vercel-sandbox-api.ts index c7c7b181928..ac0835a1f17 100644 --- a/cloud/apps/sandbox-control/src/vercel-sandbox-api.ts +++ b/cloud/apps/sandbox-control/src/vercel-sandbox-api.ts @@ -42,6 +42,14 @@ const Snapshot = z.looseObject({ status: z.enum(['created', 'deleted', 'failed']), sourceSessionId: z.string() }) +const ProjectSnapshot = Snapshot.extend({ creationMethod: z.string().optional(), createdAt: z.number() }) +const SandboxSummary = z.looseObject({ + name: z.string(), + status: Status, + currentSessionId: z.string(), + tags: z.record(z.string(), z.string()).optional() +}) +const MAX_LISTED = 5000 const ErrorBody = z.looseObject({ error: z.looseObject({ code: z.string().optional() }).optional() }) const CommandLine = z.looseObject({ command: z.looseObject({ exitCode: z.number().nullable() }) }) const OutputLine = z.looseObject({ stream: z.enum(['stdout', 'stderr']), data: z.string() }) @@ -129,8 +137,8 @@ export function createVercelSandboxApi( timeout: params.timeoutMs, ports: params.ports, persistent: true, - keepLastSnapshots: { count: 2 }, - snapshotExpiration: 0, + keepLastSnapshots: { count: 2, expiration: params.snapshotExpirationMs }, + snapshotExpiration: params.snapshotExpirationMs, networkPolicy: params.networkPolicy, tags: params.tags } @@ -247,6 +255,46 @@ export function createVercelSandboxApi( return snapshots.map((s) => ({ id: s.id, status: s.status, sourceSessionId: s.sourceSessionId })) }, + async listSandboxes() { + const sandboxes: z.infer[] = [] + let cursor: string | undefined + do { + const response = await request('GET', '/v2/sandboxes', { + query: { project: config.projectId, limit: '50', cursor } + }) + const page = await json( + z.looseObject({ sandboxes: z.array(SandboxSummary), pagination: z.looseObject({ next: z.string().nullable() }) }), + response + ) + sandboxes.push(...page.sandboxes) + cursor = page.pagination.next ?? undefined + } while (cursor && sandboxes.length < MAX_LISTED) + return { + complete: cursor === undefined, + sandboxes: sandboxes.map((s) => ({ name: s.name, status: s.status, currentSessionId: s.currentSessionId, tags: s.tags ?? {} })) + } + }, + + async listProjectSnapshots() { + const snapshots: z.infer[] = [] + let cursor: string | undefined + do { + const response = await request('GET', '/v2/sandboxes/snapshots', { + query: { project: config.projectId, limit: '50', cursor } + }) + const page = await json( + z.looseObject({ snapshots: z.array(ProjectSnapshot), pagination: z.looseObject({ next: z.string().nullable() }) }), + response + ) + snapshots.push(...page.snapshots) + cursor = page.pagination.next ?? undefined + } while (cursor && snapshots.length < MAX_LISTED) + return { + complete: cursor === undefined, + snapshots: snapshots.map((s) => ({ id: s.id, status: s.status, creationMethod: s.creationMethod ?? null, createdAt: s.createdAt })) + } + }, + deleteSnapshot: (snapshotId) => missingOk(() => request('DELETE', `/v2/sandboxes/snapshots/${encodeURIComponent(snapshotId)}`)) } diff --git a/cloud/apps/sandbox-control/test/fake-vercel.ts b/cloud/apps/sandbox-control/test/fake-vercel.ts index 583b2ffda37..43b974b5082 100644 --- a/cloud/apps/sandbox-control/test/fake-vercel.ts +++ b/cloud/apps/sandbox-control/test/fake-vercel.ts @@ -23,7 +23,7 @@ type FakeBox = { export type FakeVercel = SandboxProviderApi & { boxes: Map - snapshots: Map + snapshots: Map commands: { sessionId: string; command: string; env: Record }[] failStart: boolean failCreate: boolean @@ -58,7 +58,7 @@ export function createFakeVercel(now: () => number): FakeVercel { const stop = (box: FakeBox): string => { box.status = 'stopped' const id = next('snap') - snapshots.set(id, { id, status: 'created', sourceSessionId: box.sessionId, sandboxName: box.params.name }) + snapshots.set(id, { id, status: 'created', sourceSessionId: box.sessionId, sandboxName: box.params.name, creationMethod: 'automatic', createdAt: now() }) return id } @@ -150,6 +150,23 @@ export function createFakeVercel(now: () => number): FakeVercel { .filter((snapshot) => snapshot.sandboxName === name) .map(({ id, status, sourceSessionId }) => ({ id, status, sourceSessionId })) }, + async listSandboxes() { + return { + complete: true, + sandboxes: [...boxes.values()].map((box) => ({ + name: box.params.name, + status: box.status, + currentSessionId: box.sessionId, + tags: box.params.tags + })) + } + }, + async listProjectSnapshots() { + return { + complete: true, + snapshots: [...snapshots.values()].map(({ id, status, creationMethod, createdAt }) => ({ id, status, creationMethod, createdAt })) + } + }, async deleteSnapshot(snapshotId) { const snapshot = snapshots.get(snapshotId) if (snapshot) { diff --git a/cloud/apps/sandbox-control/test/github-app.test.ts b/cloud/apps/sandbox-control/test/github-app.test.ts index ff41cd639f8..e56e8e24beb 100644 --- a/cloud/apps/sandbox-control/test/github-app.test.ts +++ b/cloud/apps/sandbox-control/test/github-app.test.ts @@ -1,53 +1,313 @@ import { exportPKCS8, generateKeyPair, decodeJwt } from 'jose' import { describe, expect, it } from 'vitest' import { createGitHubAppMinter } from '../src/github-app' +import { bindInstallation, isInstallationBound, signState, verifyState } from '../src/github-connect' import { gitBasicAuthorization } from '../src/network-policy' import { START_COMMAND } from '../src/sandbox-vm-commands' import { createHarness } from './harness' const MIN = 60_000 +const REPO = 'https://github.com/alice-gh/private-repo.git' +const STATE_SECRET = 's'.repeat(40) +const OPERATOR = 'o'.repeat(40) + +type Call = { method: string; path: string; body: unknown; auth: string | null } + +/** Fake api.github.com + github.com OAuth: alice-gh (id 1) owns installation 77 on her personal account. */ +function fakeGitHub(options: { now: () => number; users?: Record }> }) { + const calls: Call[] = [] + const users = options.users ?? { + code_alice: { id: 1, login: 'alice-gh', installations: [{ id: 77, account: { id: 1, login: 'alice-gh', type: 'User' } }] }, + code_mallory: { id: 666, login: 'mallory-gh', installations: [] } + } + const tokens = new Map() + const fakeFetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = new URL(String(input)) + const auth = new Headers(init?.headers).get('authorization') + const body = init?.body ? JSON.parse(String(init.body)) : undefined + calls.push({ method: init?.method ?? 'GET', path: url.pathname, body, auth }) + if (url.hostname === 'github.com' && url.pathname === '/login/oauth/access_token') { + const user = users[body.code] + if (!user) { + return Response.json({ error: 'bad_verification_code' }) + } + const token = `gho_${body.code}` + tokens.set(token, body.code) + return Response.json({ access_token: token }) + } + const userToken = auth?.startsWith('Bearer gho_') ? users[tokens.get(auth.slice(7)) ?? ''] : undefined + if (url.pathname === '/user') { + return userToken ? Response.json({ id: userToken.id, login: userToken.login }) : new Response(null, { status: 401 }) + } + if (url.pathname === '/user/installations') { + return Response.json({ total_count: userToken!.installations.length, installations: userToken!.installations }) + } + if (url.pathname.startsWith('/user/memberships/orgs/')) { + const membership = userToken?.memberships?.[url.pathname.split('/').pop()!] + return membership ? Response.json(membership) : new Response(null, { status: 404 }) + } + // App-JWT routes: the repo is installed under installation 77 regardless of who asks. + if (url.pathname === '/repos/alice-gh/private-repo/installation' || url.pathname === '/app/installations/77') { + return Response.json({ id: 77, account: { id: 1, login: 'alice-gh', type: 'User' } }) + } + if (url.pathname === '/app/installations/77/access_tokens') { + return Response.json({ + token: `ghs_${calls.length}`, + expires_at: new Date(options.now() + 60 * MIN).toISOString(), + repositories: body.repositories.map((name: string) => ({ id: 9, name })) + }) + } + return new Response(null, { status: 404 }) + }) as typeof globalThis.fetch + return { calls, fetch: fakeFetch } +} + +async function appKey() { + const { privateKey } = await generateKeyPair('RS256', { extractable: true }) + return exportPKCS8(privateKey) +} + +async function harnessWithApp() { + let clock = () => 0 + const github = fakeGitHub({ now: () => clock() }) + const pem = await appKey() + let dbRef: D1Database | null = null + const minter = createGitHubAppMinter({ + appId: '123', + privateKeyPkcs8: pem, + fetch: github.fetch, + now: () => clock(), + isBound: (id, account) => isInstallationBound(dbRef!, id, account), + log: () => undefined + }) + const h = await createHarness( + { + GITHUB_APP_SLUG: 'orca-cloud-test', + GITHUB_APP_CLIENT_ID: 'Iv1.test', + GITHUB_APP_CLIENT_SECRET: 'client-secret', + GITHUB_STATE_SECRET: STATE_SECRET, + OPERATOR_TOKEN: OPERATOR, + EGRESS_PROXY_URL: 'https://sandbox.test/v1/egress/proxy' + }, + { github: minter, githubFetch: github.fetch } + ) + clock = h.now + dbRef = h.env.SANDBOX_DB + return { h, github, minter } +} + +async function browser(h: Awaited>, url: string) { + const response = await h.send(new Request(url)) + return { status: response.status, location: response.headers.get('location'), text: await response.text() } +} + +/** Desktop asks for a connect URL, the browser installs + signs in on GitHub as `code`'s user. */ +async function connect(h: Awaited>, as: string, code: string, installationId = 77) { + const { body } = await h.call('POST', '/v1/github/connect', { as }) + const state = new URL(body.installUrl).searchParams.get('state')! + const setup = await browser(h, `https://sandbox.test/v1/github/setup?installation_id=${installationId}&setup_action=install&state=${encodeURIComponent(state)}`) + expect(setup.status).toBe(302) + const authorize = new URL(setup.location!) + expect(authorize.origin + authorize.pathname).toBe('https://github.com/login/oauth/authorize') + expect(authorize.searchParams.get('redirect_uri')).toBe('https://sandbox.test/v1/github/callback') + const carried = authorize.searchParams.get('state')! + const callback = await browser(h, `https://sandbox.test/v1/github/callback?code=${code}&state=${encodeURIComponent(carried)}`) + return { state, carried, callback, body } +} describe('GitHub App installation tokens', () => { - it('mints a token scoped to the one workspace repo and reuses it until near expiry', async () => { - const { privateKey } = await generateKeyPair('RS256', { extractable: true }) - const calls: { method: string; path: string; body: unknown; iss: unknown }[] = [] + it('mints a single-repo token only for a bound account, and re-checks the binding on cached tokens', async () => { let now = Date.parse('2026-11-09T00:00:00Z') + const github = fakeGitHub({ now: () => now }) + const bound = new Set(['77:user:alice']) const minter = createGitHubAppMinter({ appId: '123', - privateKeyPkcs8: await exportPKCS8(privateKey), + privateKeyPkcs8: await appKey(), now: () => now, - fetch: async (input, init) => { - const path = new URL(String(input)).pathname - const jwt = new Headers(init?.headers).get('authorization')!.replace('Bearer ', '') - calls.push({ method: init?.method ?? 'GET', path, body: init?.body ? JSON.parse(String(init.body)) : undefined, iss: decodeJwt(jwt).iss }) - return path.endsWith('/installation') - ? Response.json({ id: 77 }) - : Response.json({ token: 'ghs_minted', expires_at: new Date(now + 60 * MIN).toISOString() }) + fetch: github.fetch, + isBound: async (id, account) => bound.has(`${id}:${account}`), + log: () => undefined + }) + const token = await minter.tokenForRepo(REPO, 'user:alice') + expect(token).toMatchObject({ expiresAt: now + 60 * MIN }) + expect(github.calls.map((c) => [c.method, c.path, c.body])).toEqual([ + ['GET', '/repos/alice-gh/private-repo/installation', undefined], + [ + 'POST', + '/app/installations/77/access_tokens', + { repositories: ['private-repo'], permissions: { contents: 'write', pull_requests: 'write', metadata: 'read' } } + ] + ]) + expect(decodeJwt(github.calls[0]!.auth!.slice(7)).iss).toBe('123') + + // Cross-tenant: another account naming the same repo gets nothing, and no token is minted. + expect(await minter.tokenForRepo(REPO, 'user:bob')).toBeNull() + expect(github.calls.filter((c) => c.path.endsWith('/access_tokens'))).toHaveLength(1) + + now += 30 * MIN + expect(await minter.tokenForRepo('https://github.com/alice-gh/private-repo', 'user:alice')).toEqual(token) + bound.delete('77:user:alice') + expect(await minter.tokenForRepo(REPO, 'user:alice')).toBeNull() + + expect(await minter.tokenForRepo('https://gitlab.com/a/b', 'user:alice')).toBeNull() + expect(await minter.tokenForRepo('https://x:y@github.com/alice-gh/private-repo', 'user:alice')).toBeNull() + }) + + it('refuses a minted token that covers more than the one repository', async () => { + const minter = createGitHubAppMinter({ + appId: '123', + privateKeyPkcs8: await appKey(), + isBound: async () => true, + fetch: (async (input: RequestInfo | URL) => + String(input).endsWith('/installation') + ? Response.json({ id: 77, account: { id: 1, login: 'alice-gh', type: 'User' } }) + : Response.json({ token: 'ghs_wide', expires_at: new Date().toISOString(), repositories: [{ id: 1, name: 'private-repo' }, { id: 2, name: 'other' }] })) as typeof fetch + }) + await expect(minter.tokenForRepo(REPO, 'user:alice')).rejects.toThrow('unexpected scope') + }) + + it('cross-tenant attack: bob names alice’s repo and gets no credential and no token is minted', async () => { + const { h, github } = await harnessWithApp() + await bindInstallation( + h.env.SANDBOX_DB, + { + installation_id: 77, + account_id: 'user:alice', + github_account_id: 1, + github_account_login: 'alice-gh', + github_account_type: 'User', + bound_by_user_id: 'alice', + bound_by_github_user_id: 1, + method: 'oauth' + }, + h.now() + ) + const res = await h.call('POST', '/v1/sandboxes', { as: 'bob', body: { workspaceId: 'ws-steal', machineSize: 'standard', repo: { url: REPO } } }) + expect(res.status).toBe(200) + const [box] = [...h.vercel.boxes.values()] + expect(box!.policy.allow['github.com'] ?? []).toEqual([]) + expect(h.vercel.commands.find((c) => c.command === START_COMMAND)!.env.ORCA_GIT_EXTRAHEADER).toBeUndefined() + expect(github.calls.filter((c) => c.path.endsWith('/access_tokens'))).toHaveLength(0) + + // Alice, bound, gets her token. + const mine = await h.call('POST', '/v1/sandboxes', { as: 'alice', body: { workspaceId: 'ws-mine', machineSize: 'standard', repo: { url: REPO } } }) + const aliceBox = h.vercel.boxes.get(`orca-test-${mine.body.sandbox.id}`)! + expect(aliceBox.policy.allow['github.com']![0]).toMatchObject({ + transform: [{ headers: { authorization: expect.stringMatching(/^Basic /) } }] + }) + expect(github.calls.filter((c) => c.path.endsWith('/access_tokens'))).toHaveLength(1) + }) + + it('binds an installation only after GitHub proves the installer owns it, single use', async () => { + const { h, minter } = await harnessWithApp() + const { state, carried, callback, body } = await connect(h, 'alice', 'code_alice') + expect(body.installUrl).toMatch(/^https:\/\/github\.com\/apps\/orca-cloud-test\/installations\/new\?state=/) + expect(callback.text).toContain('Connected GitHub alice-gh') + expect(await minter.tokenForRepo(REPO, 'user:alice')).not.toBeNull() + expect(await minter.tokenForRepo(REPO, 'user:bob')).toBeNull() + expect((await h.call('GET', '/v1/github/installations', { as: 'alice' })).body.installations).toMatchObject([ + { installationId: 77, account: 'alice-gh', accountType: 'User' } + ]) + expect((await h.call('GET', '/v1/github/installations', { as: 'bob' })).body.installations).toEqual([]) + + // Replaying the callback (or the original state) binds nothing more. + const replay = await browser(h, `https://sandbox.test/v1/github/callback?code=code_alice&state=${encodeURIComponent(carried)}`) + expect(replay.text).toContain('already used') + expect(state).not.toBe(carried) + }) + + it('attack: mallory starts her own connect and points the setup URL at alice’s installation id', async () => { + const { h, minter } = await harnessWithApp() + const { callback } = await connect(h, 'mallory', 'code_mallory', 77) + expect(callback.text).toContain('Nothing was connected') + expect(await minter.tokenForRepo(REPO, 'user:mallory')).toBeNull() + expect(h.raw.prepare('SELECT COUNT(*) AS n FROM github_installations').get()).toEqual({ n: 0 }) + }) + + it('attack: a forged or tampered state (another account, no signature) is refused', async () => { + const { h } = await harnessWithApp() + const { body } = await h.call('POST', '/v1/github/connect', { as: 'mallory' }) + const state = new URL(body.installUrl).searchParams.get('state')! + const payload = await verifyState(STATE_SECRET, state, h.now()) + const [bodyPart, mac] = state.split('.') + const forged = `${btoa(JSON.stringify({ ...payload, a: 'user:alice' })).replace(/=+$/, '')}.${mac}` + expect(bodyPart).not.toBe(forged.split('.')[0]) + for (const bad of [forged, await signState('x'.repeat(40), payload!), 'garbage']) { + const res = await browser(h, `https://sandbox.test/v1/github/callback?code=code_alice&state=${encodeURIComponent(bad)}`) + expect(res.text).toContain('not opened from Orca') + } + // A validly signed state whose nonce belongs to another account is refused too. + const crossed = await signState(STATE_SECRET, { ...payload!, a: 'user:alice' }) + const res = await browser(h, `https://sandbox.test/v1/github/callback?code=code_alice&state=${encodeURIComponent(crossed)}`) + expect(res.text).toContain('already used') + expect(h.raw.prepare('SELECT COUNT(*) AS n FROM github_installations').get()).toEqual({ n: 0 }) + h.advance(11 * MIN) + const expired = await browser(h, `https://sandbox.test/v1/github/callback?code=code_mallory&state=${encodeURIComponent(state)}`) + expect(expired.text).toContain('not opened from Orca') + }) + + it('org installations bind only for an active org admin', async () => { + let now = 0 + const org = { id: 77, account: { id: 50, login: 'acme', type: 'Organization' } } + const github = fakeGitHub({ + now: () => now, + users: { + code_member: { id: 2, login: 'member', installations: [org], memberships: { acme: { state: 'active', role: 'member' } } }, + code_admin: { id: 3, login: 'admin', installations: [org], memberships: { acme: { state: 'active', role: 'admin' } } }, + code_pending: { id: 4, login: 'pending', installations: [org], memberships: { acme: { state: 'pending', role: 'admin' } } } } }) - expect(await minter.tokenForRepo('https://github.com/nwparker/orca-sandbox-push-test.git')).toEqual({ token: 'ghs_minted', expiresAt: now + 60 * MIN }) - expect(calls).toEqual([ - { method: 'GET', path: '/repos/nwparker/orca-sandbox-push-test/installation', body: undefined, iss: '123' }, - { + const h = await createHarness( + { GITHUB_APP_SLUG: 'orca-cloud-test', GITHUB_APP_CLIENT_ID: 'Iv1.test', GITHUB_APP_CLIENT_SECRET: 'x', GITHUB_STATE_SECRET: STATE_SECRET }, + { githubFetch: github.fetch } + ) + now = h.now() + for (const [code, expected] of [['code_member', 0], ['code_pending', 0], ['code_admin', 1]] as const) { + const { callback } = await connect(h, 'carol', code) + expect(callback.status).toBe(200) + expect(h.raw.prepare(`SELECT COUNT(*) AS n FROM github_installations WHERE account_id = 'user:carol'`).get()).toEqual({ n: expected }) + } + }) + + it('operator binding needs the operator token and is recorded as such', async () => { + const { h, minter } = await harnessWithApp() + const bind = (token: string) => + h.send( + new Request('https://sandbox.test/v1/internal/github/bindings', { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ installationId: 77, accountId: 'dev:push' }) + }) + ) + expect((await bind(await h.token('alice'))).status).toBe(404) + expect((await bind('o'.repeat(39) + 'x')).status).toBe(404) + const ok = await bind(OPERATOR) + expect(await ok.json()).toEqual({ bound: { installationId: 77, accountId: 'dev:push', githubAccount: 'alice-gh' } }) + expect(h.raw.prepare('SELECT method, bound_by_user_id FROM github_installations').get()).toEqual({ method: 'operator', bound_by_user_id: 'operator' }) + expect(await minter.tokenForRepo(REPO, 'dev:push')).not.toBeNull() + const byRepo = await h.send( + new Request('https://sandbox.test/v1/internal/github/bindings', { method: 'POST', - path: '/app/installations/77/access_tokens', - body: { repositories: ['orca-sandbox-push-test'], permissions: { contents: 'write', pull_requests: 'write', metadata: 'read' } }, - iss: '123' - } - ]) - now += 30 * MIN - await minter.tokenForRepo('https://github.com/nwparker/orca-sandbox-push-test') - expect(calls).toHaveLength(2) - now += 26 * MIN - await minter.tokenForRepo('https://github.com/nwparker/orca-sandbox-push-test') - expect(calls).toHaveLength(4) - expect(await minter.tokenForRepo('https://gitlab.com/a/b')).toBeNull() + headers: { authorization: `Bearer ${OPERATOR}` }, + body: JSON.stringify({ repoUrl: REPO, accountId: 'dev:other' }) + }) + ) + expect(await byRepo.json()).toEqual({ bound: { installationId: 77, accountId: 'dev:other', githubAccount: 'alice-gh' } }) + const unbind = await h.send( + new Request('https://sandbox.test/v1/internal/github/bindings', { + method: 'DELETE', + headers: { authorization: `Bearer ${OPERATOR}` }, + body: JSON.stringify({ installationId: 77, accountId: 'dev:push' }) + }) + ) + expect(await unbind.json()).toEqual({ revoked: 1 }) + expect(await minter.tokenForRepo(REPO, 'dev:push')).toBeNull() }) it('injects the repo token for git and refreshes it before it expires', async () => { let expiresAt = 0 const h = await createHarness({}, { - github: { tokenForRepo: async () => ({ token: `ghs_${expiresAt}`, expiresAt }) } + github: { tokenForRepo: async () => ({ token: `ghs_${expiresAt}`, expiresAt }), getInstallation: async () => null, getRepoInstallation: async () => null } }) expiresAt = h.now() + 60 * MIN const { body } = await h.call('POST', '/v1/sandboxes', { diff --git a/cloud/apps/sandbox-control/test/guardrails.test.ts b/cloud/apps/sandbox-control/test/guardrails.test.ts new file mode 100644 index 00000000000..9fa0ec95268 --- /dev/null +++ b/cloud/apps/sandbox-control/test/guardrails.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from 'vitest' +import { createHarness } from './harness' + +const OPERATOR = 'o'.repeat(40) +const create = (workspaceId: string) => ({ body: { workspaceId, machineSize: 'standard' } }) + +async function operator(h: Awaited>, method: string, path: string, body?: unknown) { + const response = await h.send( + new Request(`https://sandbox.test${path}`, { + method, + headers: { authorization: `Bearer ${OPERATOR}`, 'content-type': 'application/json' }, + body: body === undefined ? undefined : JSON.stringify(body) + }) + ) + return { status: response.status, body: (await response.json()) as any } +} + +describe('kill switch and caps', () => { + it('kill switch refuses creates and resumes, drains running sandboxes keeping their disk, and lifts', async () => { + const h = await createHarness({ OPERATOR_TOKEN: OPERATOR }) + const a = await h.call('POST', '/v1/sandboxes', create('ws-a')) + const b = await h.call('POST', '/v1/sandboxes', { ...create('ws-b'), as: 'bob' }) + + expect((await h.call('GET', '/v1/internal/status')).status).toBe(404) + const on = await operator(h, 'POST', '/v1/internal/kill-switch', { enabled: true }) + expect(on.body).toMatchObject({ killSwitch: true, drained: { suspended: 2, failed: 0, remaining: 0 } }) + for (const box of h.vercel.boxes.values()) { + expect(box.status).toBe('stopped') + } + expect([...h.vercel.snapshots.values()].filter((s) => s.status === 'created')).toHaveLength(2) + + expect(await h.call('POST', '/v1/sandboxes', create('ws-c'))).toEqual({ status: 503, body: { error: 'maintenance' } }) + expect((await h.call('POST', `/v1/sandboxes/${a.body.sandbox.id}/resume`)).body).toEqual({ error: 'maintenance' }) + expect(h.billing.checks).toHaveLength(2) + expect((await operator(h, 'GET', '/v1/internal/status')).body).toMatchObject({ killSwitch: true, active: 0 }) + + await operator(h, 'POST', '/v1/internal/kill-switch', { enabled: false }) + expect((await h.call('POST', `/v1/sandboxes/${b.body.sandbox.id}/resume`, { as: 'bob' })).status).toBe(200) + }) + + it('the cron keeps draining while the kill switch is on; the static env switch also refuses', async () => { + const h = await createHarness({ OPERATOR_TOKEN: OPERATOR }) + await operator(h, 'POST', '/v1/internal/kill-switch', { enabled: true, drain: false }) + expect((await h.call('POST', '/v1/sandboxes', create('ws-a'))).status).toBe(503) + await operator(h, 'POST', '/v1/internal/kill-switch', { enabled: false }) + await h.call('POST', '/v1/sandboxes', create('ws-a')) + await operator(h, 'POST', '/v1/internal/kill-switch', { enabled: true, drain: false }) + expect((await h.reconcile()).drained).toBe(1) + + const fixed = await createHarness({ SANDBOX_CREATES_DISABLED: 'true' }) + expect((await fixed.call('POST', '/v1/sandboxes', create('ws-a'))).body).toEqual({ error: 'maintenance' }) + }) + + it('global running cap refuses the next start across accounts', async () => { + const h = await createHarness({ SANDBOX_MAX_RUNNING: '1' }) + expect((await h.call('POST', '/v1/sandboxes', create('ws-a'))).status).toBe(200) + const res = await h.call('POST', '/v1/sandboxes', { ...create('ws-b'), as: 'bob' }) + expect(res).toEqual({ status: 503, body: { error: 'capacity' } }) + expect(h.vercel.boxes.size).toBe(1) + }) + + it('per-account backstop is 3 per seat even when billing would allow more', async () => { + const h = await createHarness() + for (const ws of ['ws-1', 'ws-2', 'ws-3']) { + expect((await h.call('POST', '/v1/sandboxes', create(ws))).status).toBe(200) + } + expect(await h.call('POST', '/v1/sandboxes', create('ws-4'))).toEqual({ + status: 403, + body: { error: 'entitlement_denied', reason: 'concurrency_limit' } + }) + h.billing.decision = { allow: true, reason: null, suspendRunning: false, seats: 2 } + expect((await h.call('POST', '/v1/sandboxes', create('ws-4'))).status).toBe(200) + expect(h.vercel.boxes.size).toBe(4) + }) + + it('rate limits creates per account per minute', async () => { + const h = await createHarness({ CREATE_RATE_PER_MINUTE: '2' }) + expect((await h.call('POST', '/v1/sandboxes', create('ws-1'))).status).toBe(200) + expect((await h.call('POST', '/v1/sandboxes', create('ws-1'))).status).toBe(200) + expect(await h.call('POST', '/v1/sandboxes', create('ws-2'))).toEqual({ status: 429, body: { error: 'rate_limited' } }) + expect((await h.call('POST', '/v1/sandboxes', { ...create('ws-2'), as: 'bob' })).status).toBe(200) + h.advance(60_000) + expect((await h.call('POST', '/v1/sandboxes', create('ws-2'))).status).toBe(200) + }) +}) diff --git a/cloud/apps/sandbox-control/test/retention.test.ts b/cloud/apps/sandbox-control/test/retention.test.ts new file mode 100644 index 00000000000..1120711eafe --- /dev/null +++ b/cloud/apps/sandbox-control/test/retention.test.ts @@ -0,0 +1,180 @@ +import { describe, expect, it } from 'vitest' +import { DEFAULT_RETENTION, housekeeping, type RetentionWarning } from '../src/retention' +import type { LifecycleDeps } from '../src/lifecycle' +import { GOLDEN, createHarness } from './harness' + +const HOUR = 60 * 60_000 +const DAY = 24 * HOUR +const create = (workspaceId: string) => ({ body: { workspaceId, machineSize: 'standard' } }) + +async function setup() { + const h = await createHarness() + const warnings: RetentionWarning[] = [] + const deps: LifecycleDeps = { + db: h.env.SANDBOX_DB, + provider: h.vercel, + entitlements: { + checkStart: async (request) => { + h.billing.checks.push({ accountId: request.accountId, runningSandboxes: request.runningSandboxes }) + return h.billing.decision + }, + reportUsage: async () => h.billing.usageAnswer, + listSuspensions: async () => h.billing.suspensions + }, + now: h.now, + retention: DEFAULT_RETENTION, + config: { environment: 'test', goldenSnapshotId: GOLDEN, defaultRegion: 'iad1' } + } + const sweep = () => housekeeping(deps, { force: true, notifier: { warn: async (w) => void warnings.push(w) } }) + return { h, deps, warnings, sweep } +} + +describe('orphan sweep', () => { + it('suspends a provider sandbox with no live D1 row, deletes it after the grace period, leaves tracked ones alone', async () => { + const { h, sweep } = await setup() + const tracked = await h.call('POST', '/v1/sandboxes', create('ws-1')) + await h.vercel.createSandbox({ + name: 'orca-test-sbx_ghost', + snapshotId: GOLDEN, + vcpus: 4, + region: 'iad1', + timeoutMs: 30 * 60_000, + ports: [], + networkPolicy: { allow: {} }, + tags: { orcaSandbox: 'sbx_ghost', orcaEnv: 'test' }, + snapshotExpirationMs: 0 + }) + // Another environment's sandbox is never judged by this one. + await h.vercel.createSandbox({ + name: 'orca-prod-sbx_other', + snapshotId: GOLDEN, + vcpus: 4, + region: 'iad1', + timeoutMs: 30 * 60_000, + ports: [], + networkPolicy: { allow: {} }, + tags: { orcaSandbox: 'sbx_other', orcaEnv: 'prod' }, + snapshotExpirationMs: 0 + }) + const first = await sweep() + expect(first).toMatchObject({ orphanSandboxesSuspended: 1, orphanSandboxesDeleted: 0 }) + expect(h.vercel.boxes.get('orca-test-sbx_ghost')!.status).toBe('stopped') + expect(h.vercel.boxes.get('orca-prod-sbx_other')!.status).toBe('running') + expect(h.vercel.boxes.get(`orca-test-${tracked.body.sandbox.id}`)!.status).toBe('running') + + h.advance(23 * HOUR) + expect((await sweep()).orphanSandboxesDeleted).toBe(0) + h.advance(2 * HOUR) + expect((await sweep()).orphanSandboxesDeleted).toBe(1) + expect(h.vercel.boxes.has('orca-test-sbx_ghost')).toBe(false) + const ghostSnaps = [...h.vercel.snapshots.values()].filter((s) => s.sandboxName === 'orca-test-sbx_ghost') + expect(ghostSnaps.every((s) => s.status === 'deleted')).toBe(true) + expect(h.vercel.boxes.has('orca-prod-sbx_other')).toBe(true) + expect(h.vercel.boxes.has(`orca-test-${tracked.body.sandbox.id}`)).toBe(true) + }) + + it('deletes automatic snapshots whose sandbox is gone after the grace period, never manual or owned ones', async () => { + const { h, sweep } = await setup() + const tracked = await h.call('POST', '/v1/sandboxes', create('ws-1')) + await h.call('POST', `/v1/sandboxes/${tracked.body.sandbox.id}/suspend`) + const add = (id: string, creationMethod: string, sandboxName: string) => + h.vercel.snapshots.set(id, { id, status: 'created', sourceSessionId: 's', sandboxName, creationMethod, createdAt: h.now() }) + add('snap_leftover', 'automatic', 'orca-test-sbx_deleted_long_ago') + add('snap_old_golden', 'manual', 'golden-build') + const owned = [...h.vercel.snapshots.values()].find((s) => s.sandboxName === `orca-test-${tracked.body.sandbox.id}`)! + + expect((await sweep()).orphanSnapshotsSeen).toBe(1) + h.advance(25 * HOUR) + expect((await sweep()).orphanSnapshotsDeleted).toBe(1) + expect(h.vercel.snapshots.get('snap_leftover')!.status).toBe('deleted') + expect(h.vercel.snapshots.get('snap_old_golden')!.status).toBe('created') + expect(h.vercel.snapshots.get(owned.id)!.status).toBe('created') + expect((await h.call('POST', `/v1/sandboxes/${tracked.body.sandbox.id}/resume`)).status).toBe(200) + }) + + it('a partial provider listing deletes nothing', async () => { + const { h, sweep } = await setup() + h.vercel.snapshots.set('snap_leftover', { id: 'snap_leftover', status: 'created', sourceSessionId: 's', sandboxName: 'gone', creationMethod: 'automatic', createdAt: 0 }) + const list = h.vercel.listSandboxes + h.vercel.listSandboxes = async () => ({ ...(await list()), complete: false }) + await sweep() + h.advance(48 * HOUR) + expect((await sweep()).orphanSnapshotsDeleted).toBe(0) + expect(h.vercel.snapshots.get('snap_leftover')!.status).toBe('created') + }) +}) + +describe('data retention', () => { + it('destroys a canceled account’s sandboxes 30 days after cancel, warning 7 days ahead', async () => { + const { h, sweep, warnings } = await setup() + const box = await h.call('POST', '/v1/sandboxes', create('ws-1')) + h.billing.suspensions = ['user:alice'] + h.billing.decision = { allow: false, reason: 'payment_grace_expired', suspendRunning: true } + await h.reconcile() + expect(h.raw.prepare('SELECT reason FROM account_holds WHERE account_id = ?').get('user:alice')).toEqual({ reason: 'payment_grace_expired' }) + expect((await h.call('GET', `/v1/sandboxes/${box.body.sandbox.id}`)).body.sandbox.status).toBe('suspended') + + h.advance(22 * DAY) + expect((await sweep()).retentionWarnings).toBe(0) + h.advance(1 * DAY) + expect((await sweep()).retentionWarnings).toBe(1) + expect(warnings[0]).toMatchObject({ kind: 'data_deletion', accountId: 'user:alice', sandboxIds: [box.body.sandbox.id] }) + h.advance(6 * DAY) + expect((await sweep()).retentionDestroyed).toBe(0) + h.advance(1 * DAY + HOUR) + expect((await sweep()).retentionDestroyed).toBe(1) + expect(h.vercel.boxes.size).toBe(0) + expect([...h.vercel.snapshots.values()].every((s) => s.status === 'deleted')).toBe(true) + expect(h.raw.prepare('SELECT repo_url, status FROM sandboxes').get()).toEqual({ repo_url: null, status: 'destroyed' }) + }) + + it('a canceled subscription found by the standing probe starts the clock; resubscribing clears it', async () => { + const { h, sweep } = await setup() + await h.call('POST', '/v1/sandboxes', create('ws-1')) + h.billing.decision = { allow: false, reason: 'subscription_inactive', suspendRunning: false } + expect((await sweep()).holdsOpened).toBe(1) + h.advance(2 * DAY) + h.billing.decision = { allow: true, reason: null, suspendRunning: false } + expect((await sweep()).holdsCleared).toBe(1) + h.advance(40 * DAY) + expect((await sweep()).retentionDestroyed).toBe(0) + expect(h.vercel.boxes.size).toBe(1) + }) + + it('an exhausted-hours denial or a billing outage never starts a deletion clock', async () => { + const { h, sweep, deps } = await setup() + await h.call('POST', '/v1/sandboxes', create('ws-1')) + h.billing.decision = { allow: false, reason: 'included_hours_exhausted', suspendRunning: false } + expect((await sweep()).holdsOpened).toBe(0) + deps.entitlements.checkStart = async () => { + throw new Error('billing down') + } + h.advance(2 * DAY) + expect((await sweep()).holdsOpened).toBe(0) + expect(h.raw.prepare(`SELECT COUNT(*) AS n FROM account_holds WHERE reason <> 'active'`).get()).toEqual({ n: 0 }) + }) + + it('idle retention: a suspended sandbox untouched 90 days is warned at 83 and destroyed at 90; a resume resets it', async () => { + const { h, sweep, warnings } = await setup() + const idle = await h.call('POST', '/v1/sandboxes', create('ws-idle')) + const used = await h.call('POST', '/v1/sandboxes', create('ws-used')) + await h.call('POST', `/v1/sandboxes/${idle.body.sandbox.id}/suspend`) + await h.call('POST', `/v1/sandboxes/${used.body.sandbox.id}/suspend`) + h.advance(84 * DAY) + expect((await sweep()).idleWarnings).toBe(2) + expect(warnings.map((w) => w.kind)).toEqual(['idle_deletion', 'idle_deletion']) + await h.call('POST', `/v1/sandboxes/${used.body.sandbox.id}/resume`) + await h.call('POST', `/v1/sandboxes/${used.body.sandbox.id}/suspend`) + h.advance(7 * DAY) + expect((await sweep()).idleDestroyed).toBe(1) + expect((await h.call('GET', `/v1/sandboxes/${idle.body.sandbox.id}`)).status).toBe(200) + expect((await h.call('GET', `/v1/sandboxes/${idle.body.sandbox.id}`)).body.sandbox.status).toBe('destroyed') + expect((await h.call('GET', `/v1/sandboxes/${used.body.sandbox.id}`)).body.sandbox.status).toBe('suspended') + }) + + it('new sandboxes ask the provider for a snapshot expiry beyond idle retention', async () => { + const { h } = await setup() + await h.call('POST', '/v1/sandboxes', create('ws-1')) + expect([...h.vercel.boxes.values()][0]!.params.snapshotExpirationMs).toBe(104 * DAY) + }) +}) diff --git a/cloud/apps/sandbox-control/test/vercel-sandbox-api.test.ts b/cloud/apps/sandbox-control/test/vercel-sandbox-api.test.ts index d6d7a094b53..b960ad57b8b 100644 --- a/cloud/apps/sandbox-control/test/vercel-sandbox-api.test.ts +++ b/cloud/apps/sandbox-control/test/vercel-sandbox-api.test.ts @@ -41,7 +41,8 @@ describe('Vercel Sandbox REST client', () => { timeoutMs: 1_800_000, ports: [6768], networkPolicy: { allow: { 'registry.npmjs.org': [] } }, - tags: { orcaSandbox: 'sbx_1' } + tags: { orcaSandbox: 'sbx_1' }, + snapshotExpirationMs: 104 * 86_400_000 }) expect(seen[0]!.method).toBe('POST') expect(`${seen[0]!.url.origin}${seen[0]!.url.pathname}${seen[0]!.url.search}`).toBe( @@ -53,8 +54,8 @@ describe('Vercel Sandbox REST client', () => { source: { type: 'snapshot', snapshotId: 'snap_golden' }, resources: { vcpus: 4 }, persistent: true, - keepLastSnapshots: { count: 2 }, - snapshotExpiration: 0 + keepLastSnapshots: { count: 2, expiration: 104 * 86_400_000 }, + snapshotExpiration: 104 * 86_400_000 }) expect(state.routes).toEqual([{ port: 6768, url: 'https://sb-1.vercel.run' }]) expect(state.session.timeoutMs).toBe(1_800_000) diff --git a/cloud/apps/sandbox-control/wrangler.dev.jsonc b/cloud/apps/sandbox-control/wrangler.dev.jsonc index 0a05668e44d..f246c2d13ee 100644 --- a/cloud/apps/sandbox-control/wrangler.dev.jsonc +++ b/cloud/apps/sandbox-control/wrangler.dev.jsonc @@ -4,7 +4,9 @@ // wrangler deploy -c wrangler.dev.jsonc // Secrets (wrangler secret put -c wrangler.dev.jsonc, never here): VERCEL_TOKEN, SANDBOX_DEV_JWKS, // EGRESS_DEV_STATIC_AUTHORIZATION, CONTROL_PLANE_TOKEN (same value as on - // orca-billing-dev), GITHUB_APP_ID, GITHUB_APP_PRIVATE_KEY (PKCS#8), SECRETS_KEK (base64 32 bytes). + // orca-billing-dev), GITHUB_APP_ID, GITHUB_APP_PRIVATE_KEY (PKCS#8), SECRETS_KEK (base64 32 bytes), + // OPERATOR_TOKEN (/v1/internal/*), GITHUB_STATE_SECRET; for the GitHub connect flow also + // GITHUB_APP_CLIENT_ID + GITHUB_APP_CLIENT_SECRET (app settings → generate a client secret). "name": "orca-sandbox-control-dev", "main": "src/index.ts", "compatibility_date": "2026-11-01", @@ -29,7 +31,9 @@ "VERCEL_PROJECT_ID": "prj_SkVtv6zJurF9dUHpfTRy4v9avwPq", "VERCEL_REGION": "iad1", "BILLING_URL": "https://orca-billing-dev.lovecast.workers.dev", - // GOLDEN_SNAPSHOT_ID: unset until an image is built (image/build-golden-image.ts); lifecycle routes answer 503 meanwhile. + // Golden image from image/build-golden-image.ts; lifecycle routes answer 503 without one. + "GOLDEN_SNAPSHOT_ID": "snap_0fYmcpIUlc4OUC9viFN0Y4dVCDuD", + "GITHUB_APP_SLUG": "orca-cloud-dev", "EGRESS_PROXY_URL": "https://orca-sandbox-control-dev.lovecast.workers.dev/v1/egress/proxy" } } diff --git a/cloud/docs/vercel-driver-design.md b/cloud/docs/vercel-driver-design.md index f2434cddcdd..6c237f40a87 100644 --- a/cloud/docs/vercel-driver-design.md +++ b/cloud/docs/vercel-driver-design.md @@ -230,6 +230,29 @@ Product decision: "API-key injection" in the Conductor parity matrix (form 2 = t in the same VM. Enable on dev after a live streaming test with the rebuilt image. - Needs: golden image rebuild (new `orca-sandbox-start`), and Orca Cloud auth minting `org_role` on sandbox tokens. +## GitHub installation binding, guardrails, retention (2026-11-09) + +- Repo tokens need a binding: `github_installations(installation_id, account_id)`. `tokenForRepo(url, accountId)` + looks up the repo's installation, refuses unless the caller's account holds a live binding (re-checked on cached + tokens too), mints for that one repo (contents/pull_requests write, metadata read) and rejects a wider response. + Binding: desktop `POST /v1/github/connect` → signed (HMAC), single-use, 10-min state carrying the Orca account → + GitHub install / OAuth → `/v1/github/setup` + `/v1/github/callback` exchange the code for a user-to-server token and + bind only installations on the GitHub user's own account or on an org where they are an active admin + (`/user/memberships/orgs`, needs the app's Members:read). Installation ids in URLs are never trusted. + Operator binding for support/dev: `POST /v1/internal/github/bindings {installationId|repoUrl, accountId}`. +- Guardrails before billing: kill switch (D1 flag via `POST /v1/internal/kill-switch`, or `SANDBOX_CREATES_DISABLED`) + refuses create/resume/rollover with 503 `maintenance` and drains running sandboxes (suspend, snapshot kept; cron + keeps draining while on); global `SANDBOX_MAX_RUNNING` (50, 503 `capacity`); per-account 3 × seats backstop + (403 `concurrency_limit`); `CREATE_RATE_PER_MINUTE` (10, 429). +- Hourly housekeeping: provider sandboxes tagged with this env and no live row are stopped on first sight and + deleted after `ORPHAN_GRACE_HOURS` (24); automatic snapshots whose sandbox no longer exists anywhere in the project + are deleted after the same grace (manual/golden never; partial listings delete nothing). Canceled/unpaid accounts + (suspensions list + daily standing probe; outages never start a clock) get a warning hook at T−7 days and are + destroyed `DATA_RETENTION_DAYS` (30) after; suspended sandboxes idle `IDLE_RETENTION_DAYS` (90) likewise. New + sandboxes set `snapshotExpiration` to 104 days (backstop). Destroyed rows drop `repo_url`. +- Live on dev: unbound account → private clone refused (no token minted); after operator binding → clone + push + + gh/Node via proxy OK; kill switch drill drained 1, refused create/resume, resume OK after lifting. + ## What needs live access (not done) Vercel Pro team id + project id + a Sandbox-scoped token (as Worker secrets), wrangler auth for a dev D1, a