From 8fb42f939ae46b6e534ad53eed4dfdcc3eed8225 Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:42:26 -0800 Subject: [PATCH] feat(cloud): GitHub App repo tokens + billing dev wiring for the sandbox control plane (#20) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(cloud): GitHub App installation tokens for sandbox git (manifest flow, minter, policy refresh) * feat(cloud): billing dev wiring — shared control-plane token via service binding, grace-expired suspensions, standard/large only --- .../image/vm/orca-sandbox-start | 4 + .../migrations/0002_github_repo.sql | 3 + .../apps/sandbox-control/src/entitlements.ts | 50 +++++++--- cloud/apps/sandbox-control/src/env.ts | 6 ++ cloud/apps/sandbox-control/src/github-app.ts | 94 ++++++++++++++++++ cloud/apps/sandbox-control/src/index.ts | 96 ++++++++++++++++++- cloud/apps/sandbox-control/src/lifecycle.ts | 78 +++++++++++++-- cloud/apps/sandbox-control/src/reconcile.ts | 34 ++++++- .../src/sandbox-vm-commands.ts | 5 + cloud/apps/sandbox-control/src/store.ts | 12 ++- cloud/apps/sandbox-control/src/wire.ts | 2 +- .../test/entitlements-client.test.ts | 39 ++++++++ .../sandbox-control/test/github-app.test.ts | 73 ++++++++++++++ cloud/apps/sandbox-control/test/harness.ts | 9 +- .../test/lifecycle-flow.test.ts | 40 +++++++- cloud/apps/sandbox-control/wrangler.dev.jsonc | 9 +- .../hosted-sandbox/hosted-sandbox-wire.ts | 3 +- 17 files changed, 516 insertions(+), 41 deletions(-) create mode 100644 cloud/apps/sandbox-control/migrations/0002_github_repo.sql create mode 100644 cloud/apps/sandbox-control/src/github-app.ts create mode 100644 cloud/apps/sandbox-control/test/entitlements-client.test.ts create mode 100644 cloud/apps/sandbox-control/test/github-app.test.ts diff --git a/cloud/apps/sandbox-control/image/vm/orca-sandbox-start b/cloud/apps/sandbox-control/image/vm/orca-sandbox-start index af3fbf961b8..2c448768c89 100755 --- a/cloud/apps/sandbox-control/image/vm/orca-sandbox-start +++ b/cloud/apps/sandbox-control/image/vm/orca-sandbox-start @@ -22,6 +22,10 @@ fi rm -f "$STATE/ready.json" "$STATE/orcad.pid" printf '%s' "$boot_id" >"$STATE/boot-id" +if [ -n "${ORCA_GIT_EXTRAHEADER:-}" ]; then + # Every github.com request carries the placeholder; the egress policy swaps in a 1h repo token. + git config --global http.https://github.com/.extraheader "$ORCA_GIT_EXTRAHEADER" +fi if [ ! -d "$PROJECT" ]; then if [ -n "${ORCA_REPO_URL:-}" ]; then git clone --quiet ${ORCA_REPO_REF:+--branch "$ORCA_REPO_REF"} -- "$ORCA_REPO_URL" "$PROJECT" diff --git a/cloud/apps/sandbox-control/migrations/0002_github_repo.sql b/cloud/apps/sandbox-control/migrations/0002_github_repo.sql new file mode 100644 index 00000000000..7b4242eeb91 --- /dev/null +++ b/cloud/apps/sandbox-control/migrations/0002_github_repo.sql @@ -0,0 +1,3 @@ +-- The workspace repository (for GitHub App installation tokens) and when the injected token expires. +ALTER TABLE sandboxes ADD COLUMN repo_url TEXT; +ALTER TABLE sandboxes ADD COLUMN github_token_expires_at INTEGER; diff --git a/cloud/apps/sandbox-control/src/entitlements.ts b/cloud/apps/sandbox-control/src/entitlements.ts index ae24d148c36..5eb71ae2664 100644 --- a/cloud/apps/sandbox-control/src/entitlements.ts +++ b/cloud/apps/sandbox-control/src/entitlements.ts @@ -3,9 +3,11 @@ import { z } from 'zod' import type { Env } from './env' -export type MachineSize = 'standard' | 'large' | 'xlarge' +/** What billing sells; anything else is refused there. */ +export type MachineSize = 'standard' | 'large' -export type StartDecision = { allow: boolean; reason: string | null } +/** `suspendRunning`: payment grace ran out, so the account's running sandboxes must stop too. */ +export type StartDecision = { allow: boolean; reason: string | null; suspendRunning: boolean } export type UsageSegment = { usageId: string @@ -16,7 +18,7 @@ export type UsageSegment = { endedAt: number } -export type UsageAnswer = { canStartSandbox: boolean; reason: string | null } +export type UsageAnswer = { canStartSandbox: boolean; reason: string | null; suspendRunning: boolean } export type EntitlementClient = { checkStart(request: { @@ -25,6 +27,8 @@ export type EntitlementClient = { runningSandboxes: number }): Promise reportUsage(segment: UsageSegment): Promise + /** Accounts whose payment grace expired; polled by the cron. */ + listSuspensions(): Promise } export class BillingUnavailableError extends Error { @@ -34,39 +38,55 @@ export class BillingUnavailableError extends Error { } } -const StartAnswer = z.looseObject({ allow: z.boolean(), reason: z.string().nullable() }) -const UsageResponse = z.looseObject({ canStartSandbox: z.boolean(), reason: z.string().nullable() }) +const StartAnswer = z.looseObject({ + allow: z.boolean(), + reason: z.string().nullable(), + suspendRunning: z.boolean().default(false) +}) +const UsageResponse = z.looseObject({ + canStartSandbox: z.boolean(), + reason: z.string().nullable(), + suspendRunning: z.boolean().default(false) +}) +const SuspensionsResponse = z.looseObject({ + accounts: z.array(z.looseObject({ accountId: z.string() })).max(10_000) +}) /** Reasons that mean running sandboxes must stop, not just that new starts wait. */ export const STOP_RUNNING_REASONS: ReadonlySet = new Set([ 'included_hours_exhausted', 'overage_cap_reached', 'subscription_inactive', - 'no_subscription' + 'no_subscription', + 'payment_grace_expired' ]) -export function createEntitlementClient(env: Env, doFetch: typeof fetch = fetch): EntitlementClient { +export function createEntitlementClient( + env: Env, + doFetch: typeof fetch = env.BILLING ? (input, init) => env.BILLING!.fetch(input, init) : fetch +): EntitlementClient { if (env.ENTITLEMENTS_MODE === 'stub') { if (env.SANDBOX_ENV === 'prod') { throw new BillingUnavailableError('stub entitlements refused in prod') } return { - checkStart: async () => ({ allow: true, reason: null }), - reportUsage: async () => ({ canStartSandbox: true, reason: null }) + checkStart: async () => ({ allow: true, reason: null, suspendRunning: false }), + reportUsage: async () => ({ canStartSandbox: true, reason: null, suspendRunning: false }), + listSuspensions: async () => [] } } const base = env.BILLING_URL const token = env.CONTROL_PLANE_TOKEN - async function post(path: string, body: unknown, schema: z.ZodType): Promise { + async function send(method: 'GET' | 'POST', path: string, body: unknown, schema: z.ZodType): Promise { if (!base || !token) { throw new BillingUnavailableError('billing not configured') } let response: Response try { response = await doFetch(new URL(path, base), { - method: 'POST', + method, headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, - body: JSON.stringify(body), + body: body === undefined ? undefined : JSON.stringify(body), redirect: 'manual', signal: AbortSignal.timeout(10_000) }) @@ -83,7 +103,9 @@ export function createEntitlementClient(env: Env, doFetch: typeof fetch = fetch) return parsed.data } return { - checkStart: (request) => post('/v1/internal/entitlements/check', request, StartAnswer), - reportUsage: (segment) => post('/v1/internal/usage', segment, UsageResponse) + checkStart: (request) => send('POST', '/v1/internal/entitlements/check', request, StartAnswer), + reportUsage: (segment) => send('POST', '/v1/internal/usage', segment, UsageResponse), + listSuspensions: async () => + (await send('GET', '/v1/internal/suspensions', undefined, SuspensionsResponse)).accounts.map((a) => a.accountId) } } diff --git a/cloud/apps/sandbox-control/src/env.ts b/cloud/apps/sandbox-control/src/env.ts index fb404195be8..9138af9e9e5 100644 --- a/cloud/apps/sandbox-control/src/env.ts +++ b/cloud/apps/sandbox-control/src/env.ts @@ -26,6 +26,8 @@ export type Env = { // Billing Worker (entitlement checks + usage). CONTROL_PLANE_TOKEN is shared with it. BILLING_URL?: string + /** Service binding to the billing Worker; Workers on one account cannot reach each other over workers.dev. */ + BILLING?: Fetcher CONTROL_PLANE_TOKEN?: string /** `stub` allows every start and records nothing: local dev only, refused when SANDBOX_ENV=prod. */ ENTITLEMENTS_MODE?: string @@ -36,4 +38,8 @@ export type Env = { SESSION_DEADLINE_MS?: string /** Dev-only: Authorization the egress proxy injects for api.github.com until the GitHub App minter exists. */ EGRESS_DEV_STATIC_AUTHORIZATION?: string + + // GitHub App (contents/pull_requests write, metadata read). The PKCS#8 key is a secret. + GITHUB_APP_ID?: string + GITHUB_APP_PRIVATE_KEY?: string } diff --git a/cloud/apps/sandbox-control/src/github-app.ts b/cloud/apps/sandbox-control/src/github-app.ts new file mode 100644 index 00000000000..ebea80124dc --- /dev/null +++ b/cloud/apps/sandbox-control/src/github-app.ts @@ -0,0 +1,94 @@ +// GitHub App installation tokens for sandbox git: 1h, scoped to the workspace's one repository with +// contents + pull_requests write. Minted here, injected at the egress layer, never stored in the VM. +import { SignJWT, importPKCS8 } from 'jose' + +export type GitHubRepoToken = { token: string; expiresAt: number } + +export type GitHubAppMinter = { + /** Null when the repo is not on github.com or the app is not installed on it. */ + tokenForRepo(repoUrl: string): Promise +} + +const REFRESH_MARGIN_MS = 5 * 60_000 + +export function parseGitHubRepo(repoUrl: string): { owner: string; repo: string } | null { + try { + const url = new URL(repoUrl) + const match = /^\/([A-Za-z0-9-]+)\/([A-Za-z0-9._-]+?)(?:\.git)?\/?$/.exec(url.pathname) + return url.protocol === 'https:' && url.hostname === 'github.com' && match + ? { owner: match[1]!, repo: match[2]! } + : null + } catch { + return null + } +} + +export function createGitHubAppMinter(config: { + appId: string + privateKeyPkcs8: string + fetch?: typeof fetch + now?: () => number +}): GitHubAppMinter { + const doFetch = config.fetch ?? fetch + const now = config.now ?? Date.now + const cache = new Map() + let signingKey: CryptoKey | null = null + + async function appJwt(): Promise { + signingKey ??= await importPKCS8(config.privateKeyPkcs8, 'RS256') + const iat = Math.floor(now() / 1000) - 60 + return new SignJWT({}) + .setProtectedHeader({ alg: 'RS256' }) + .setIssuer(config.appId) + .setIssuedAt(iat) + .setExpirationTime(iat + 9 * 60) + .sign(signingKey) + } + + async function github(method: string, path: string, body?: unknown): Promise { + return doFetch(`https://api.github.com${path}`, { + method, + headers: { + authorization: `Bearer ${await appJwt()}`, + accept: 'application/vnd.github+json', + 'x-github-api-version': '2022-11-28', + 'user-agent': 'orca-sandbox-control' + }, + body: body === undefined ? undefined : JSON.stringify(body), + redirect: 'manual' + }) + } + + return { + async tokenForRepo(repoUrl) { + const parsed = parseGitHubRepo(repoUrl) + if (!parsed) { + return null + } + const key = `${parsed.owner}/${parsed.repo}`.toLowerCase() + const cached = cache.get(key) + if (cached && cached.expiresAt - now() > REFRESH_MARGIN_MS) { + return cached + } + const installation = await github('GET', `/repos/${parsed.owner}/${parsed.repo}/installation`) + if (installation.status === 404) { + return null + } + if (!installation.ok) { + throw new Error(`github installation lookup failed (${installation.status})`) + } + const { id } = (await installation.json()) as { id: number } + const minted = await github('POST', `/app/installations/${id}/access_tokens`, { + repositories: [parsed.repo], + permissions: { contents: 'write', pull_requests: 'write', metadata: 'read' } + }) + if (!minted.ok) { + throw new Error(`github token mint failed (${minted.status})`) + } + const body = (await minted.json()) as { token: string; expires_at: string } + const token = { token: body.token, expiresAt: Date.parse(body.expires_at) } + cache.set(key, token) + return token + } + } +} diff --git a/cloud/apps/sandbox-control/src/index.ts b/cloud/apps/sandbox-control/src/index.ts index f1b9936d464..f34e9f270a9 100644 --- a/cloud/apps/sandbox-control/src/index.ts +++ b/cloud/apps/sandbox-control/src/index.ts @@ -22,6 +22,7 @@ import { suspendSandbox, type LifecycleDeps } from './lifecycle' +import { createGitHubAppMinter, type GitHubAppMinter } from './github-app' import { reconcile } from './reconcile' import { ProviderError, type SandboxProviderApi } from './sandbox-provider' import { VmCommandError } from './sandbox-vm-commands' @@ -35,11 +36,91 @@ export type Deps = { provider?: SandboxProviderApi entitlements?: EntitlementClient credentials?: CredentialBroker + github?: GitHubAppMinter now?: () => number egressFetch?: typeof fetch egressKeysForIssuer?: Parameters[1]['keysForIssuer'] } +const minters = new Map() +function githubMinter(env: Env): GitHubAppMinter | undefined { + if (!env.GITHUB_APP_ID || !env.GITHUB_APP_PRIVATE_KEY) { + return undefined + } + let minter = minters.get(env.GITHUB_APP_ID) + if (!minter) { + minter = createGitHubAppMinter({ appId: env.GITHUB_APP_ID, privateKeyPkcs8: env.GITHUB_APP_PRIVATE_KEY }) + minters.set(env.GITHUB_APP_ID, minter) + } + return minter +} + +function egressCredentials(env: Env): CredentialBroker { + const minter = githubMinter(env) + if (!minter) { + return devCredentials(env) + } + return { + authorizationFor: async (row, host) => { + if (host !== 'api.github.com' || !row.repo_url) { + return null + } + const minted = await minter.tokenForRepo(row.repo_url) + return minted ? `Bearer ${minted.token}` : null + } + } +} + +function html(body: string): Response { + return new Response(`${body}`, { + headers: { 'content-type': 'text/html; charset=utf-8', 'cache-control': 'no-store' } + }) +} + +/** + * One-click GitHub App creation (manifest flow). Permissions are the minimum for sandbox git: + * contents write (clone, fetch, push), pull_requests write (open/update PRs from the workspace), + * metadata read (required by GitHub for any repo access). No webhook: tokens are minted on demand. + */ +function githubManifestStart(url: URL): Response { + const origin = url.origin + const manifest = { + name: 'Orca Cloud (dev)', + url: origin, + redirect_url: `${origin}/v1/github/manifest-callback`, + setup_url: `${origin}/v1/github/setup`, + setup_on_update: false, + // Installable on personal accounts too; installing grants nothing until a repo is selected. + public: true, + hook_attributes: { url: `${origin}/v1/github/webhook-unused`, active: false }, + default_permissions: { contents: 'write', metadata: 'read', pull_requests: 'write' }, + default_events: [], + request_oauth_on_install: false + } + const state = crypto.randomUUID() + const escaped = JSON.stringify(manifest).replaceAll('&', '&').replaceAll('"', '"').replaceAll('<', '<') + return html( + `
` + + `
` + + '' + ) +} + +/** GitHub App manifest flow redirect: the operator's exchange script reads the one-time code from logs. */ +function githubManifestCallback(env: Env, url: URL): Response { + if (env.SANDBOX_ENV === 'prod') { + return fail('not_found', 404) + } + const code = url.searchParams.get('code') ?? '' + if (!/^[A-Za-z0-9_-]{8,128}$/.test(code)) { + return html('Missing code.') + } + console.log(JSON.stringify({ event: 'github_manifest_code', code, state: url.searchParams.get('state') })) + return html( + 'GitHub App created. The Orca agent is finishing setup.

Next: install it on nwparker → Only select repositories → orca-sandbox-push-test.' + ) +} + function devCredentials(env: Env): CredentialBroker { const value = env.EGRESS_DEV_STATIC_AUTHORIZATION if (!value || env.SANDBOX_ENV === 'prod') { @@ -80,6 +161,7 @@ function lifecycleDeps(env: Env, deps: Deps): LifecycleDeps | null { baseUrl: env.VERCEL_API_BASE }), entitlements: deps.entitlements ?? createEntitlementClient(env), + github: deps.github ?? githubMinter(env), now: deps.now ?? Date.now, config: { environment: env.SANDBOX_ENV, @@ -104,11 +186,20 @@ export async function handleRequest(request: Request, env: Env, deps: Deps = {}) projectId: env.VERCEL_PROJECT_ID, findSandbox: (name) => getSandboxByProviderName(env.SANDBOX_DB, name), placeholdersFor: githubPlaceholderHeaders, - credentials: deps.credentials ?? devCredentials(env), + credentials: deps.credentials ?? egressCredentials(env), fetch: deps.egressFetch, keysForIssuer: deps.egressKeysForIssuer }) } + if (url.pathname === '/v1/github/manifest-callback') { + return githubManifestCallback(env, url) + } + if (url.pathname === '/v1/github/manifest-start' && env.SANDBOX_ENV !== 'prod') { + return githubManifestStart(url) + } + if (url.pathname === '/v1/github/setup') { + return html('Orca Cloud (dev) is installed. You can close this tab.') + } const match = /^\/v1\/sandboxes(?:\/([A-Za-z0-9_]+)(?:\/(suspend|resume|lease|rollover))?)?$/.exec(url.pathname) if (!match) { return fail('not_found', 404) @@ -198,7 +289,8 @@ export default { async scheduled(_controller: ScheduledController, env: Env, ctx: ExecutionContext): Promise { const life = lifecycleDeps(env, {}) if (life) { - ctx.waitUntil(reconcile(life).then(() => undefined)) + // Counts only; no ids, tokens or pairing material. + ctx.waitUntil(reconcile(life).then((summary) => console.log(JSON.stringify({ event: 'reconcile', ...summary })))) } } } satisfies ExportedHandler diff --git a/cloud/apps/sandbox-control/src/lifecycle.ts b/cloud/apps/sandbox-control/src/lifecycle.ts index 9abed5636b9..82ae764e05b 100644 --- a/cloud/apps/sandbox-control/src/lifecycle.ts +++ b/cloud/apps/sandbox-control/src/lifecycle.ts @@ -1,7 +1,8 @@ // Sandbox lifecycle: one persistent provider sandbox per Orca workspace, with an Orca-owned lease, // a forced stop before the provider's 24h cap, usage metering and write-ahead rows. import type { EntitlementClient, MachineSize } from './entitlements' -import { compileNetworkPolicy, type CredentialRoute } from './network-policy' +import type { GitHubAppMinter } from './github-app' +import { compileNetworkPolicy, gitBasicAuthorization, type CredentialRoute } from './network-policy' import type { ProviderSandboxState, SandboxProviderApi } from './sandbox-provider' import { ORCAD_PORT, drainOrcad, startOrcad, type OrcadReady } from './sandbox-vm-commands' import { @@ -9,6 +10,7 @@ import { countActiveSandboxes, getSandbox, insertSandbox, + listAccountSandboxes, liveSandboxForWorkspace, markSnapshotsDeleted, recordSnapshots, @@ -23,12 +25,14 @@ export const LEASE_MS = 30 * 60_000 const PROVIDER_SESSION_CAP_MS = 24 * 60 * 60_000 /** Orca stops 10 min early so it, not the provider, ends the session (drain first). */ export const SESSION_DEADLINE_MS = PROVIDER_SESSION_CAP_MS - 10 * 60_000 -const VCPUS: Record = { standard: 4, large: 8, xlarge: 16 } +const VCPUS: Record = { standard: 4, large: 8 } export type LifecycleDeps = { db: D1Database provider: SandboxProviderApi entitlements: EntitlementClient + /** GitHub App installation tokens for the workspace repo; absent until the app is configured. */ + github?: GitHubAppMinter now: () => number config: { environment: string @@ -67,7 +71,19 @@ export function githubPlaceholderHeaders(row: SandboxRow): string[] { return [`Bearer ${token}`, `token ${token}`] } -function egressPolicy(deps: LifecycleDeps, row: SandboxRow) { +export function egressPolicy(deps: LifecycleDeps, row: SandboxRow, gitToken?: string) { + const git: CredentialRoute[] = gitToken + ? [ + { + // git (libcurl) is unaffected by the transform-rule framing hang, so it is injected inline. + host: 'github.com', + header: 'authorization', + placeholders: [gitBasicAuthorization(githubPlaceholderToken(row))], + via: 'transform', + value: gitBasicAuthorization(gitToken) + } + ] + : [] const credentials: CredentialRoute[] = deps.config.egressProxyUrl ? [ { @@ -79,7 +95,7 @@ function egressPolicy(deps: LifecycleDeps, row: SandboxRow) { } ] : [] - return compileNetworkPolicy({ mode: 'allowlist', credentials }) + return compileNetworkPolicy({ mode: 'allowlist', credentials: [...git, ...credentials] }) } export function sessionFields(deps: LifecycleDeps, state: ProviderSandboxState): SessionFields { @@ -105,6 +121,9 @@ async function claim(deps: LifecycleDeps, row: SandboxRow, patch: Partial { + if (!deps.github || !row.repo_url) { + return null + } + const minted = await deps.github.tokenForRepo(row.repo_url) + if (!minted) { + return null + } + await deps.provider.updateNetworkPolicy(sessionId, egressPolicy(deps, row, minted.token)) + return { expiresAt: minted.expiresAt } +} + export async function createSandbox( deps: LifecycleDeps, caller: { accountId: string; userId: string }, @@ -152,7 +195,8 @@ export async function createSandbox( provider: 'vercel', provider_name: `orca-${deps.config.environment}-${id}`, machine_size: request.machineSize, - region: request.region ?? deps.config.defaultRegion + region: request.region ?? deps.config.defaultRegion, + repo_url: request.repo?.url ?? null }, deps.now() ) @@ -180,6 +224,26 @@ export async function createSandbox( } } +/** + * Billing says the account's payment grace ran out: stop (snapshot-preserving, never destroy) every + * running sandbox. Resume stays refused by the start check until billing allows starts again. + */ +export async function suspendAccountRunning(deps: LifecycleDeps, accountId: string): Promise { + let suspended = 0 + for (const row of await listAccountSandboxes(deps.db, accountId)) { + if (row.status !== 'running') { + continue + } + try { + await suspendSandbox(deps, row) + suspended++ + } catch { + // A concurrent transition or provider hiccup; the next cron pass retries. + } + } + return suspended +} + /** A create that never became usable is destroyed now; the cron retries if this also fails. */ export async function failAndRelease(deps: LifecycleDeps, row: SandboxRow): Promise { const failed = await transition(deps.db, row, { status: 'failed', last_error: 'create_failed' }, deps.now()) diff --git a/cloud/apps/sandbox-control/src/reconcile.ts b/cloud/apps/sandbox-control/src/reconcile.ts index 81fde8737b7..2f0613a354d 100644 --- a/cloud/apps/sandbox-control/src/reconcile.ts +++ b/cloud/apps/sandbox-control/src/reconcile.ts @@ -5,6 +5,8 @@ import { destroySandbox, failAndRelease, markProviderStopped, + refreshGitCredentials, + suspendAccountRunning, sessionFields, suspendSandbox, type LifecycleDeps @@ -17,7 +19,13 @@ const STUCK_MS = 15 * 60_000 const FORCED_STOP_GRACE_MS = 5 * 60_000 export type ReconcileSummary = Record< - 'settled' | 'suspendedForDeadline' | 'suspendedForBilling' | 'leaseLapsed' | 'released' | 'repaired', + | 'settled' + | 'suspendedForDeadline' + | 'suspendedForBilling' + | 'leaseLapsed' + | 'released' + | 'repaired' + | 'billingErrors', number > @@ -28,7 +36,16 @@ export async function reconcile(deps: LifecycleDeps): Promise suspendedForBilling: 0, leaseLapsed: 0, released: 0, - repaired: 0 + repaired: 0, + billingErrors: 0 + } + const polled = await guard(async () => { + for (const accountId of await deps.entitlements.listSuspensions()) { + summary.suspendedForBilling += await suspendAccountRunning(deps, accountId) + } + }) + if (!polled) { + summary.billingErrors++ } for (const row of await listByStatus(deps.db, ['running'])) { await guard(() => reconcileRunning(deps, row, summary)) @@ -71,11 +88,18 @@ async function reconcileRunning(deps: LifecycleDeps, row: SandboxRow, summary: R return } } + if (row.github_token_expires_at !== null && row.session_id && row.github_token_expires_at - now < 15 * 60_000) { + const git = await refreshGitCredentials(deps, row, row.session_id) + const refreshed = await transition(deps.db, row, { github_token_expires_at: git?.expiresAt ?? null }, now) + row = refreshed ?? row + } const settled = await queueUsage(deps.db, row, now) const current = settled === row.usage_settled_at ? row : await transition(deps.db, row, { usage_settled_at: settled }, now) summary.settled++ const answer = await flushUsage(deps.db, deps.entitlements, now, row.id) - if (current && answer && !answer.canStartSandbox && STOP_RUNNING_REASONS.has(answer.reason ?? '')) { + if (answer?.suspendRunning) { + summary.suspendedForBilling += await suspendAccountRunning(deps, row.account_id) + } else if (current && answer && !answer.canStartSandbox && STOP_RUNNING_REASONS.has(answer.reason ?? '')) { await suspendSandbox(deps, current) summary.suspendedForBilling++ } @@ -91,10 +115,12 @@ async function repairTransition(deps: LifecycleDeps, row: SandboxRow): Promise Promise): Promise { +async function guard(step: () => Promise): Promise { try { await step() + return true } catch { // One bad row must not stall the pass; it is retried next tick. + return false } } diff --git a/cloud/apps/sandbox-control/src/sandbox-vm-commands.ts b/cloud/apps/sandbox-control/src/sandbox-vm-commands.ts index 8487b38b690..240af46c6a5 100644 --- a/cloud/apps/sandbox-control/src/sandbox-vm-commands.ts +++ b/cloud/apps/sandbox-control/src/sandbox-vm-commands.ts @@ -33,6 +33,8 @@ export type StartInputs = { repo?: { url: string; ref?: string } /** Token the VM's GitHub clients send; the egress layer swaps it for a real short-lived one. */ githubPlaceholder?: string + /** git `http.https://github.com/.extraheader` carrying the placeholder, so every git request is claimable. */ + gitExtraHeader?: string } /** Starts orcad (idempotent per boot) and returns its pairing offer; pairing codes are never logged. */ @@ -55,6 +57,9 @@ export async function startOrcad( env.ORCA_REPO_REF = inputs.repo.ref } } + if (inputs.gitExtraHeader) { + env.ORCA_GIT_EXTRAHEADER = inputs.gitExtraHeader + } if (inputs.githubPlaceholder) { env.ORCA_GITHUB_PLACEHOLDER = inputs.githubPlaceholder } diff --git a/cloud/apps/sandbox-control/src/store.ts b/cloud/apps/sandbox-control/src/store.ts index b8ff00b8766..e982da936f1 100644 --- a/cloud/apps/sandbox-control/src/store.ts +++ b/cloud/apps/sandbox-control/src/store.ts @@ -32,6 +32,8 @@ export type SandboxRow = { created_at: number updated_at: number destroyed_at: number | null + repo_url: string | null + github_token_expires_at: number | null } export type SessionFields = Pick< @@ -54,18 +56,18 @@ export async function insertSandbox( db: D1Database, row: Pick< SandboxRow, - 'id' | 'account_id' | 'user_id' | 'workspace_id' | 'provider' | 'provider_name' | 'machine_size' | 'region' + 'id' | 'account_id' | 'user_id' | 'workspace_id' | 'provider' | 'provider_name' | 'machine_size' | 'region' | 'repo_url' >, now: number ): Promise { await db .prepare( `INSERT INTO sandboxes (id, account_id, user_id, workspace_id, provider, provider_name, status, - machine_size, region, generation, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, 'creating', ?, ?, 0, ?, ?)` + machine_size, region, generation, created_at, updated_at, repo_url) + VALUES (?, ?, ?, ?, ?, ?, 'creating', ?, ?, 0, ?, ?, ?)` ) .bind(row.id, row.account_id, row.user_id, row.workspace_id, row.provider, row.provider_name, - row.machine_size, row.region, now, now) + row.machine_size, row.region, now, now, row.repo_url) .run() const inserted = await getSandbox(db, row.id) if (!inserted) { @@ -122,7 +124,7 @@ export async function listByStatus(db: D1Database, statuses: readonly SandboxSta type Patch = Partial> const PATCHABLE = new Set([ 'status', 'session_id', 'session_started_at', 'session_deadline_at', 'lease_expires_at', - 'usage_settled_at', 'last_error', 'destroyed_at' + 'usage_settled_at', 'last_error', 'destroyed_at', 'github_token_expires_at' ]) /** Applies `patch` only if nobody moved the row since `expected` was read; null on a lost race. */ diff --git a/cloud/apps/sandbox-control/src/wire.ts b/cloud/apps/sandbox-control/src/wire.ts index aaee92b168f..1d7a323c477 100644 --- a/cloud/apps/sandbox-control/src/wire.ts +++ b/cloud/apps/sandbox-control/src/wire.ts @@ -13,7 +13,7 @@ const Id = z export const CreateRequestSchema = z .object({ workspaceId: Id, - machineSize: z.enum(['standard', 'large', 'xlarge']), + machineSize: z.enum(['standard', 'large']), region: z .string() .regex(/^[a-z]{3}[0-9]$/) diff --git a/cloud/apps/sandbox-control/test/entitlements-client.test.ts b/cloud/apps/sandbox-control/test/entitlements-client.test.ts new file mode 100644 index 00000000000..69603b012f8 --- /dev/null +++ b/cloud/apps/sandbox-control/test/entitlements-client.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest' +import { BillingUnavailableError, createEntitlementClient } from '../src/entitlements' +import type { Env } from '../src/env' + +const env = { BILLING_URL: 'https://billing.test', CONTROL_PLANE_TOKEN: 'cp_'.padEnd(40, 'x'), SANDBOX_ENV: 'dev' } as Env + +describe('billing client', () => { + it('reads billing dev answers, including grace-expired suspensions', async () => { + const seen: string[] = [] + const client = createEntitlementClient(env, async (input, init) => { + const url = new URL(String(input)) + seen.push(`${init?.method} ${url.pathname} ${new Headers(init?.headers).get('authorization')}`) + if (url.pathname === '/v1/internal/suspensions') { + return Response.json({ accounts: [{ accountId: 'org:o1', pastDueSince: 1, graceEndsAt: 2 }] }) + } + return Response.json({ allow: false, reason: 'payment_grace_expired', suspendRunning: true, entitlement: {} }) + }) + expect(await client.listSuspensions()).toEqual(['org:o1']) + expect(await client.checkStart({ accountId: 'org:o1', machineSize: 'standard', runningSandboxes: 0 })).toEqual({ + allow: false, + reason: 'payment_grace_expired', + suspendRunning: true, + entitlement: {} + }) + expect(seen[0]).toBe(`GET /v1/internal/suspensions Bearer ${env.CONTROL_PLANE_TOKEN}`) + }) + + it('fails closed when billing is unreachable and refuses the stub in prod', async () => { + const down = createEntitlementClient(env, async () => { + throw new Error('offline') + }) + await expect(down.checkStart({ accountId: 'a', machineSize: 'large', runningSandboxes: 0 })).rejects.toBeInstanceOf( + BillingUnavailableError + ) + expect(() => createEntitlementClient({ ...env, ENTITLEMENTS_MODE: 'stub', SANDBOX_ENV: 'prod' })).toThrow( + BillingUnavailableError + ) + }) +}) diff --git a/cloud/apps/sandbox-control/test/github-app.test.ts b/cloud/apps/sandbox-control/test/github-app.test.ts new file mode 100644 index 00000000000..ff41cd639f8 --- /dev/null +++ b/cloud/apps/sandbox-control/test/github-app.test.ts @@ -0,0 +1,73 @@ +import { exportPKCS8, generateKeyPair, decodeJwt } from 'jose' +import { describe, expect, it } from 'vitest' +import { createGitHubAppMinter } from '../src/github-app' +import { gitBasicAuthorization } from '../src/network-policy' +import { START_COMMAND } from '../src/sandbox-vm-commands' +import { createHarness } from './harness' + +const MIN = 60_000 + +describe('GitHub App installation tokens', () => { + it('mints a token scoped to the one workspace repo and reuses it until near expiry', async () => { + const { privateKey } = await generateKeyPair('RS256', { extractable: true }) + const calls: { method: string; path: string; body: unknown; iss: unknown }[] = [] + let now = Date.parse('2026-11-09T00:00:00Z') + const minter = createGitHubAppMinter({ + appId: '123', + privateKeyPkcs8: await exportPKCS8(privateKey), + now: () => now, + fetch: async (input, init) => { + const path = new URL(String(input)).pathname + const jwt = new Headers(init?.headers).get('authorization')!.replace('Bearer ', '') + calls.push({ method: init?.method ?? 'GET', path, body: init?.body ? JSON.parse(String(init.body)) : undefined, iss: decodeJwt(jwt).iss }) + return path.endsWith('/installation') + ? Response.json({ id: 77 }) + : Response.json({ token: 'ghs_minted', expires_at: new Date(now + 60 * MIN).toISOString() }) + } + }) + expect(await minter.tokenForRepo('https://github.com/nwparker/orca-sandbox-push-test.git')).toEqual({ token: 'ghs_minted', expiresAt: now + 60 * MIN }) + expect(calls).toEqual([ + { method: 'GET', path: '/repos/nwparker/orca-sandbox-push-test/installation', body: undefined, iss: '123' }, + { + method: 'POST', + path: '/app/installations/77/access_tokens', + body: { repositories: ['orca-sandbox-push-test'], permissions: { contents: 'write', pull_requests: 'write', metadata: 'read' } }, + iss: '123' + } + ]) + now += 30 * MIN + await minter.tokenForRepo('https://github.com/nwparker/orca-sandbox-push-test') + expect(calls).toHaveLength(2) + now += 26 * MIN + await minter.tokenForRepo('https://github.com/nwparker/orca-sandbox-push-test') + expect(calls).toHaveLength(4) + expect(await minter.tokenForRepo('https://gitlab.com/a/b')).toBeNull() + }) + + it('injects the repo token for git and refreshes it before it expires', async () => { + let expiresAt = 0 + const h = await createHarness({}, { + github: { tokenForRepo: async () => ({ token: `ghs_${expiresAt}`, expiresAt }) } + }) + expiresAt = h.now() + 60 * MIN + const { body } = await h.call('POST', '/v1/sandboxes', { + body: { workspaceId: 'ws-git', machineSize: 'standard', repo: { url: 'https://github.com/nwparker/orca-sandbox-push-test.git' } } + }) + const placeholder = gitBasicAuthorization(`orca-ph-${body.sandbox.id}`) + const [box] = [...h.vercel.boxes.values()] + expect(box!.policy.allow['github.com']![0]).toEqual({ + match: { headers: [{ key: { exact: 'authorization' }, value: { exact: placeholder } }] }, + transform: [{ headers: { authorization: gitBasicAuthorization(`ghs_${expiresAt}`) } }] + }) + const start = h.vercel.commands.find((c) => c.command === START_COMMAND)! + expect(start.env.ORCA_GIT_EXTRAHEADER).toBe(`Authorization: ${placeholder}`) + expect(start.env.ORCA_REPO_URL).toBe('https://github.com/nwparker/orca-sandbox-push-test.git') + + const first = expiresAt + h.advance(50 * MIN) + expiresAt = h.now() + 60 * MIN + await h.reconcile() + expect(box!.policy.allow['github.com']![0]).toMatchObject({ transform: [{ headers: { authorization: gitBasicAuthorization(`ghs_${expiresAt}`) } }] }) + expect(expiresAt).not.toBe(first) + }) +}) diff --git a/cloud/apps/sandbox-control/test/harness.ts b/cloud/apps/sandbox-control/test/harness.ts index d27e7e5ee8c..8911feecb27 100644 --- a/cloud/apps/sandbox-control/test/harness.ts +++ b/cloud/apps/sandbox-control/test/harness.ts @@ -16,8 +16,9 @@ export async function createHarness(overrides: Partial = {}, extraDeps: Par const clock = () => now const vercel = createFakeVercel(clock) const billing = { - decision: { allow: true, reason: null } as StartDecision, - usageAnswer: { canStartSandbox: true, reason: null } as UsageAnswer, + decision: { allow: true, reason: null, suspendRunning: false } as StartDecision, + usageAnswer: { canStartSandbox: true, reason: null, suspendRunning: false } as UsageAnswer, + suspensions: [] as string[], down: false, checks: [] as { accountId: string; runningSandboxes: number }[], usage: [] as UsageSegment[] @@ -33,6 +34,9 @@ export async function createHarness(overrides: Partial = {}, extraDeps: Par } billing.usage.push(segment) return billing.usageAnswer + }, + async listSuspensions() { + return billing.suspensions } } const env: Env = { @@ -91,6 +95,7 @@ export async function createHarness(overrides: Partial = {}, extraDeps: Par db, provider: vercel, entitlements, + github: extraDeps.github, now: clock, config: { environment: 'test', goldenSnapshotId: GOLDEN, defaultRegion: 'iad1' } }) diff --git a/cloud/apps/sandbox-control/test/lifecycle-flow.test.ts b/cloud/apps/sandbox-control/test/lifecycle-flow.test.ts index bf7dc94d6aa..50af6ad9350 100644 --- a/cloud/apps/sandbox-control/test/lifecycle-flow.test.ts +++ b/cloud/apps/sandbox-control/test/lifecycle-flow.test.ts @@ -37,7 +37,7 @@ describe('sandbox control plane lifecycle', () => { it('refuses a start billing denies without touching the provider', async () => { const h = await createHarness() - h.billing.decision = { allow: false, reason: 'concurrency_limit' } + h.billing.decision = { allow: false, reason: 'concurrency_limit', suspendRunning: false } const res = await h.call('POST', '/v1/sandboxes', { body: create }) expect(res).toEqual({ status: 403, body: { error: 'entitlement_denied', reason: 'concurrency_limit' } }) expect(h.vercel.boxes.size).toBe(0) @@ -163,7 +163,7 @@ describe('sandbox control plane cron', () => { it('stops running sandboxes once billing says the included hours are gone', async () => { const h = await createHarness() const { body } = await h.call('POST', '/v1/sandboxes', { body: create }) - h.billing.usageAnswer = { canStartSandbox: false, reason: 'included_hours_exhausted' } + h.billing.usageAnswer = { canStartSandbox: false, reason: 'included_hours_exhausted', suspendRunning: false } h.advance(10 * MIN) expect((await h.reconcile()).suspendedForBilling).toBe(1) expect((await h.call('GET', `/v1/sandboxes/${body.sandbox.id}`)).body.sandbox.status).toBe('suspended') @@ -189,4 +189,40 @@ describe('sandbox control plane cron', () => { expect((await h.reconcile()).released).toBe(1) expect(h.vercel.boxes.size).toBe(0) }) + + it('suspends an account whose payment grace expired, keeps its disk, and resumes only after payment', async () => { + const h = await createHarness() + const a = (await h.call('POST', '/v1/sandboxes', { body: create })).body.sandbox.id + const b = (await h.call('POST', '/v1/sandboxes', { body: { ...create, workspaceId: 'ws-2' } })).body.sandbox.id + const other = (await h.call('POST', '/v1/sandboxes', { body: create, as: 'bob' })).body.sandbox.id + h.billing.suspensions = ['user:alice'] + expect((await h.reconcile()).suspendedForBilling).toBe(2) + expect((await h.call('GET', `/v1/sandboxes/${a}`)).body.sandbox.status).toBe('suspended') + expect((await h.call('GET', `/v1/sandboxes/${b}`)).body.sandbox.status).toBe('suspended') + expect((await h.call('GET', `/v1/sandboxes/${other}`, { as: 'bob' })).body.sandbox.status).toBe('running') + expect(h.vercel.boxes.size).toBe(3) + + h.billing.decision = { allow: false, reason: 'payment_grace_expired', suspendRunning: true } + expect(await h.call('POST', `/v1/sandboxes/${a}/resume`)).toEqual({ + status: 403, + body: { error: 'entitlement_denied', reason: 'payment_grace_expired' } + }) + h.billing.suspensions = [] + h.billing.decision = { allow: true, reason: null, suspendRunning: false } + expect((await h.call('POST', `/v1/sandboxes/${a}/resume`)).body.sandbox.status).toBe('running') + }) + + it('suspends the whole account when a usage report says grace expired', async () => { + const h = await createHarness() + const a = (await h.call('POST', '/v1/sandboxes', { body: create })).body.sandbox.id + h.billing.usageAnswer = { canStartSandbox: false, reason: 'payment_grace_expired', suspendRunning: true } + h.advance(10 * MIN) + expect((await h.reconcile()).suspendedForBilling).toBe(1) + expect((await h.call('GET', `/v1/sandboxes/${a}`)).body.sandbox.status).toBe('suspended') + }) + + it('refuses machine sizes billing does not sell', async () => { + const h = await createHarness() + expect((await h.call('POST', '/v1/sandboxes', { body: { ...create, machineSize: 'xlarge' } })).status).toBe(400) + }) }) diff --git a/cloud/apps/sandbox-control/wrangler.dev.jsonc b/cloud/apps/sandbox-control/wrangler.dev.jsonc index 7d4b1523d21..b6334b814dd 100644 --- a/cloud/apps/sandbox-control/wrangler.dev.jsonc +++ b/cloud/apps/sandbox-control/wrangler.dev.jsonc @@ -3,7 +3,8 @@ // DEV deployment only (workers.dev, no custom domain). Deploy: // wrangler deploy -c wrangler.dev.jsonc // Secrets (wrangler secret put -c wrangler.dev.jsonc, never here): VERCEL_TOKEN, SANDBOX_DEV_JWKS, - // EGRESS_DEV_STATIC_AUTHORIZATION, and CONTROL_PLANE_TOKEN once BILLING_URL points at billing dev. + // EGRESS_DEV_STATIC_AUTHORIZATION, CONTROL_PLANE_TOKEN (same value as on + // orca-billing-dev), GITHUB_APP_ID, GITHUB_APP_PRIVATE_KEY (PKCS#8). "name": "orca-sandbox-control-dev", "main": "src/index.ts", "compatibility_date": "2026-11-01", @@ -16,6 +17,8 @@ "migrations_dir": "migrations" } ], + // Same-account Workers cannot call each other over workers.dev (error 1042); BILLING_URL names the origin. + "services": [{ "binding": "BILLING", "service": "orca-billing-dev" }], "triggers": { "crons": ["*/5 * * * *"] }, "vars": { "SANDBOX_ENV": "dev", @@ -25,8 +28,8 @@ "VERCEL_TEAM_ID": "team_c5IO9DfEYAxKEUgMxrbEtO0a", "VERCEL_PROJECT_ID": "prj_SkVtv6zJurF9dUHpfTRy4v9avwPq", "VERCEL_REGION": "iad1", - "ENTITLEMENTS_MODE": "stub", - // GOLDEN_SNAPSHOT_ID: unset until an image is built (image/build-golden-image.ts); lifecycle routes answer 503 meanwhile. + "BILLING_URL": "https://orca-billing-dev.lovecast.workers.dev", + "GOLDEN_SNAPSHOT_ID": "snap_aaJmS6etH8vyrTSxOQxPU7lx02QZ", "EGRESS_PROXY_URL": "https://orca-sandbox-control-dev.lovecast.workers.dev/v1/egress/proxy" } } diff --git a/src/shared/hosted-sandbox/hosted-sandbox-wire.ts b/src/shared/hosted-sandbox/hosted-sandbox-wire.ts index 203d9a7443d..3a3d64026e8 100644 --- a/src/shared/hosted-sandbox/hosted-sandbox-wire.ts +++ b/src/shared/hosted-sandbox/hosted-sandbox-wire.ts @@ -2,7 +2,8 @@ // The Worker's tests parse its responses with these schemas, so either side drifting fails a test. import { z } from 'zod' -export const HOSTED_SANDBOX_MACHINE_SIZES = ['standard', 'large', 'xlarge'] as const +// Billing sells exactly these; larger boxes need a Vercel Enterprise plan. +export const HOSTED_SANDBOX_MACHINE_SIZES = ['standard', 'large'] as const export type HostedSandboxMachineSize = (typeof HOSTED_SANDBOX_MACHINE_SIZES)[number] export const HOSTED_SANDBOX_STATUSES = [