feat(cloud): GitHub App repo tokens + billing dev wiring for the sandbox control plane (#20)

* feat(cloud): GitHub App installation tokens for sandbox git (manifest flow, minter, policy refresh)

* feat(cloud): billing dev wiring — shared control-plane token via service binding, grace-expired suspensions, standard/large only
This commit is contained in:
Neil
2026-11-09 04:30:14 -08:00
committed by Neil
parent 0306a56a53
commit 8fb42f939a
17 changed files with 516 additions and 41 deletions
@@ -22,6 +22,10 @@ fi
rm -f "$STATE/ready.json" "$STATE/orcad.pid"
printf '%s' "$boot_id" >"$STATE/boot-id"
if [ -n "${ORCA_GIT_EXTRAHEADER:-}" ]; then
# Every github.com request carries the placeholder; the egress policy swaps in a 1h repo token.
git config --global http.https://github.com/.extraheader "$ORCA_GIT_EXTRAHEADER"
fi
if [ ! -d "$PROJECT" ]; then
if [ -n "${ORCA_REPO_URL:-}" ]; then
git clone --quiet ${ORCA_REPO_REF:+--branch "$ORCA_REPO_REF"} -- "$ORCA_REPO_URL" "$PROJECT"
@@ -0,0 +1,3 @@
-- The workspace repository (for GitHub App installation tokens) and when the injected token expires.
ALTER TABLE sandboxes ADD COLUMN repo_url TEXT;
ALTER TABLE sandboxes ADD COLUMN github_token_expires_at INTEGER;
+36 -14
View File
@@ -3,9 +3,11 @@
import { z } from 'zod'
import type { Env } from './env'
export type MachineSize = 'standard' | 'large' | 'xlarge'
/** What billing sells; anything else is refused there. */
export type MachineSize = 'standard' | 'large'
export type StartDecision = { allow: boolean; reason: string | null }
/** `suspendRunning`: payment grace ran out, so the account's running sandboxes must stop too. */
export type StartDecision = { allow: boolean; reason: string | null; suspendRunning: boolean }
export type UsageSegment = {
usageId: string
@@ -16,7 +18,7 @@ export type UsageSegment = {
endedAt: number
}
export type UsageAnswer = { canStartSandbox: boolean; reason: string | null }
export type UsageAnswer = { canStartSandbox: boolean; reason: string | null; suspendRunning: boolean }
export type EntitlementClient = {
checkStart(request: {
@@ -25,6 +27,8 @@ export type EntitlementClient = {
runningSandboxes: number
}): Promise<StartDecision>
reportUsage(segment: UsageSegment): Promise<UsageAnswer>
/** Accounts whose payment grace expired; polled by the cron. */
listSuspensions(): Promise<string[]>
}
export class BillingUnavailableError extends Error {
@@ -34,39 +38,55 @@ export class BillingUnavailableError extends Error {
}
}
const StartAnswer = z.looseObject({ allow: z.boolean(), reason: z.string().nullable() })
const UsageResponse = z.looseObject({ canStartSandbox: z.boolean(), reason: z.string().nullable() })
const StartAnswer = z.looseObject({
allow: z.boolean(),
reason: z.string().nullable(),
suspendRunning: z.boolean().default(false)
})
const UsageResponse = z.looseObject({
canStartSandbox: z.boolean(),
reason: z.string().nullable(),
suspendRunning: z.boolean().default(false)
})
const SuspensionsResponse = z.looseObject({
accounts: z.array(z.looseObject({ accountId: z.string() })).max(10_000)
})
/** Reasons that mean running sandboxes must stop, not just that new starts wait. */
export const STOP_RUNNING_REASONS: ReadonlySet<string> = new Set([
'included_hours_exhausted',
'overage_cap_reached',
'subscription_inactive',
'no_subscription'
'no_subscription',
'payment_grace_expired'
])
export function createEntitlementClient(env: Env, doFetch: typeof fetch = fetch): EntitlementClient {
export function createEntitlementClient(
env: Env,
doFetch: typeof fetch = env.BILLING ? (input, init) => env.BILLING!.fetch(input, init) : fetch
): EntitlementClient {
if (env.ENTITLEMENTS_MODE === 'stub') {
if (env.SANDBOX_ENV === 'prod') {
throw new BillingUnavailableError('stub entitlements refused in prod')
}
return {
checkStart: async () => ({ allow: true, reason: null }),
reportUsage: async () => ({ canStartSandbox: true, reason: null })
checkStart: async () => ({ allow: true, reason: null, suspendRunning: false }),
reportUsage: async () => ({ canStartSandbox: true, reason: null, suspendRunning: false }),
listSuspensions: async () => []
}
}
const base = env.BILLING_URL
const token = env.CONTROL_PLANE_TOKEN
async function post<T>(path: string, body: unknown, schema: z.ZodType<T>): Promise<T> {
async function send<T>(method: 'GET' | 'POST', path: string, body: unknown, schema: z.ZodType<T>): Promise<T> {
if (!base || !token) {
throw new BillingUnavailableError('billing not configured')
}
let response: Response
try {
response = await doFetch(new URL(path, base), {
method: 'POST',
method,
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify(body),
body: body === undefined ? undefined : JSON.stringify(body),
redirect: 'manual',
signal: AbortSignal.timeout(10_000)
})
@@ -83,7 +103,9 @@ export function createEntitlementClient(env: Env, doFetch: typeof fetch = fetch)
return parsed.data
}
return {
checkStart: (request) => post('/v1/internal/entitlements/check', request, StartAnswer),
reportUsage: (segment) => post('/v1/internal/usage', segment, UsageResponse)
checkStart: (request) => send('POST', '/v1/internal/entitlements/check', request, StartAnswer),
reportUsage: (segment) => send('POST', '/v1/internal/usage', segment, UsageResponse),
listSuspensions: async () =>
(await send('GET', '/v1/internal/suspensions', undefined, SuspensionsResponse)).accounts.map((a) => a.accountId)
}
}
+6
View File
@@ -26,6 +26,8 @@ export type Env = {
// Billing Worker (entitlement checks + usage). CONTROL_PLANE_TOKEN is shared with it.
BILLING_URL?: string
/** Service binding to the billing Worker; Workers on one account cannot reach each other over workers.dev. */
BILLING?: Fetcher
CONTROL_PLANE_TOKEN?: string
/** `stub` allows every start and records nothing: local dev only, refused when SANDBOX_ENV=prod. */
ENTITLEMENTS_MODE?: string
@@ -36,4 +38,8 @@ export type Env = {
SESSION_DEADLINE_MS?: string
/** Dev-only: Authorization the egress proxy injects for api.github.com until the GitHub App minter exists. */
EGRESS_DEV_STATIC_AUTHORIZATION?: string
// GitHub App (contents/pull_requests write, metadata read). The PKCS#8 key is a secret.
GITHUB_APP_ID?: string
GITHUB_APP_PRIVATE_KEY?: string
}
@@ -0,0 +1,94 @@
// GitHub App installation tokens for sandbox git: 1h, scoped to the workspace's one repository with
// contents + pull_requests write. Minted here, injected at the egress layer, never stored in the VM.
import { SignJWT, importPKCS8 } from 'jose'
export type GitHubRepoToken = { token: string; expiresAt: number }
export type GitHubAppMinter = {
/** Null when the repo is not on github.com or the app is not installed on it. */
tokenForRepo(repoUrl: string): Promise<GitHubRepoToken | null>
}
const REFRESH_MARGIN_MS = 5 * 60_000
export function parseGitHubRepo(repoUrl: string): { owner: string; repo: string } | null {
try {
const url = new URL(repoUrl)
const match = /^\/([A-Za-z0-9-]+)\/([A-Za-z0-9._-]+?)(?:\.git)?\/?$/.exec(url.pathname)
return url.protocol === 'https:' && url.hostname === 'github.com' && match
? { owner: match[1]!, repo: match[2]! }
: null
} catch {
return null
}
}
export function createGitHubAppMinter(config: {
appId: string
privateKeyPkcs8: string
fetch?: typeof fetch
now?: () => number
}): GitHubAppMinter {
const doFetch = config.fetch ?? fetch
const now = config.now ?? Date.now
const cache = new Map<string, GitHubRepoToken>()
let signingKey: CryptoKey | null = null
async function appJwt(): Promise<string> {
signingKey ??= await importPKCS8(config.privateKeyPkcs8, 'RS256')
const iat = Math.floor(now() / 1000) - 60
return new SignJWT({})
.setProtectedHeader({ alg: 'RS256' })
.setIssuer(config.appId)
.setIssuedAt(iat)
.setExpirationTime(iat + 9 * 60)
.sign(signingKey)
}
async function github(method: string, path: string, body?: unknown): Promise<Response> {
return doFetch(`https://api.github.com${path}`, {
method,
headers: {
authorization: `Bearer ${await appJwt()}`,
accept: 'application/vnd.github+json',
'x-github-api-version': '2022-11-28',
'user-agent': 'orca-sandbox-control'
},
body: body === undefined ? undefined : JSON.stringify(body),
redirect: 'manual'
})
}
return {
async tokenForRepo(repoUrl) {
const parsed = parseGitHubRepo(repoUrl)
if (!parsed) {
return null
}
const key = `${parsed.owner}/${parsed.repo}`.toLowerCase()
const cached = cache.get(key)
if (cached && cached.expiresAt - now() > REFRESH_MARGIN_MS) {
return cached
}
const installation = await github('GET', `/repos/${parsed.owner}/${parsed.repo}/installation`)
if (installation.status === 404) {
return null
}
if (!installation.ok) {
throw new Error(`github installation lookup failed (${installation.status})`)
}
const { id } = (await installation.json()) as { id: number }
const minted = await github('POST', `/app/installations/${id}/access_tokens`, {
repositories: [parsed.repo],
permissions: { contents: 'write', pull_requests: 'write', metadata: 'read' }
})
if (!minted.ok) {
throw new Error(`github token mint failed (${minted.status})`)
}
const body = (await minted.json()) as { token: string; expires_at: string }
const token = { token: body.token, expiresAt: Date.parse(body.expires_at) }
cache.set(key, token)
return token
}
}
}
+94 -2
View File
@@ -22,6 +22,7 @@ import {
suspendSandbox,
type LifecycleDeps
} from './lifecycle'
import { createGitHubAppMinter, type GitHubAppMinter } from './github-app'
import { reconcile } from './reconcile'
import { ProviderError, type SandboxProviderApi } from './sandbox-provider'
import { VmCommandError } from './sandbox-vm-commands'
@@ -35,11 +36,91 @@ export type Deps = {
provider?: SandboxProviderApi
entitlements?: EntitlementClient
credentials?: CredentialBroker
github?: GitHubAppMinter
now?: () => number
egressFetch?: typeof fetch
egressKeysForIssuer?: Parameters<typeof handleEgressProxy>[1]['keysForIssuer']
}
const minters = new Map<string, GitHubAppMinter>()
function githubMinter(env: Env): GitHubAppMinter | undefined {
if (!env.GITHUB_APP_ID || !env.GITHUB_APP_PRIVATE_KEY) {
return undefined
}
let minter = minters.get(env.GITHUB_APP_ID)
if (!minter) {
minter = createGitHubAppMinter({ appId: env.GITHUB_APP_ID, privateKeyPkcs8: env.GITHUB_APP_PRIVATE_KEY })
minters.set(env.GITHUB_APP_ID, minter)
}
return minter
}
function egressCredentials(env: Env): CredentialBroker {
const minter = githubMinter(env)
if (!minter) {
return devCredentials(env)
}
return {
authorizationFor: async (row, host) => {
if (host !== 'api.github.com' || !row.repo_url) {
return null
}
const minted = await minter.tokenForRepo(row.repo_url)
return minted ? `Bearer ${minted.token}` : null
}
}
}
function html(body: string): Response {
return new Response(`<!doctype html><meta charset=utf-8><body style="font:16px system-ui;margin:3em">${body}`, {
headers: { 'content-type': 'text/html; charset=utf-8', 'cache-control': 'no-store' }
})
}
/**
* One-click GitHub App creation (manifest flow). Permissions are the minimum for sandbox git:
* contents write (clone, fetch, push), pull_requests write (open/update PRs from the workspace),
* metadata read (required by GitHub for any repo access). No webhook: tokens are minted on demand.
*/
function githubManifestStart(url: URL): Response {
const origin = url.origin
const manifest = {
name: 'Orca Cloud (dev)',
url: origin,
redirect_url: `${origin}/v1/github/manifest-callback`,
setup_url: `${origin}/v1/github/setup`,
setup_on_update: false,
// Installable on personal accounts too; installing grants nothing until a repo is selected.
public: true,
hook_attributes: { url: `${origin}/v1/github/webhook-unused`, active: false },
default_permissions: { contents: 'write', metadata: 'read', pull_requests: 'write' },
default_events: [],
request_oauth_on_install: false
}
const state = crypto.randomUUID()
const escaped = JSON.stringify(manifest).replaceAll('&', '&amp;').replaceAll('"', '&quot;').replaceAll('<', '&lt;')
return html(
`<form id=f method=post action="https://github.com/organizations/stablyai/settings/apps/new?state=${state}">` +
`<input type=hidden name=manifest value="${escaped}"><button>Create “Orca Cloud (dev)” on stablyai</button></form>` +
'<script>document.getElementById("f").submit()</script>'
)
}
/** GitHub App manifest flow redirect: the operator's exchange script reads the one-time code from logs. */
function githubManifestCallback(env: Env, url: URL): Response {
if (env.SANDBOX_ENV === 'prod') {
return fail('not_found', 404)
}
const code = url.searchParams.get('code') ?? ''
if (!/^[A-Za-z0-9_-]{8,128}$/.test(code)) {
return html('Missing code.')
}
console.log(JSON.stringify({ event: 'github_manifest_code', code, state: url.searchParams.get('state') }))
return html(
'GitHub App created. The Orca agent is finishing setup.<p>Next: <a href="https://github.com/apps/orca-cloud-dev/installations/new">install it</a> on <b>nwparker</b> → Only select repositories → <b>orca-sandbox-push-test</b>.'
)
}
function devCredentials(env: Env): CredentialBroker {
const value = env.EGRESS_DEV_STATIC_AUTHORIZATION
if (!value || env.SANDBOX_ENV === 'prod') {
@@ -80,6 +161,7 @@ function lifecycleDeps(env: Env, deps: Deps): LifecycleDeps | null {
baseUrl: env.VERCEL_API_BASE
}),
entitlements: deps.entitlements ?? createEntitlementClient(env),
github: deps.github ?? githubMinter(env),
now: deps.now ?? Date.now,
config: {
environment: env.SANDBOX_ENV,
@@ -104,11 +186,20 @@ export async function handleRequest(request: Request, env: Env, deps: Deps = {})
projectId: env.VERCEL_PROJECT_ID,
findSandbox: (name) => getSandboxByProviderName(env.SANDBOX_DB, name),
placeholdersFor: githubPlaceholderHeaders,
credentials: deps.credentials ?? devCredentials(env),
credentials: deps.credentials ?? egressCredentials(env),
fetch: deps.egressFetch,
keysForIssuer: deps.egressKeysForIssuer
})
}
if (url.pathname === '/v1/github/manifest-callback') {
return githubManifestCallback(env, url)
}
if (url.pathname === '/v1/github/manifest-start' && env.SANDBOX_ENV !== 'prod') {
return githubManifestStart(url)
}
if (url.pathname === '/v1/github/setup') {
return html('Orca Cloud (dev) is installed. You can close this tab.')
}
const match = /^\/v1\/sandboxes(?:\/([A-Za-z0-9_]+)(?:\/(suspend|resume|lease|rollover))?)?$/.exec(url.pathname)
if (!match) {
return fail('not_found', 404)
@@ -198,7 +289,8 @@ export default {
async scheduled(_controller: ScheduledController, env: Env, ctx: ExecutionContext): Promise<void> {
const life = lifecycleDeps(env, {})
if (life) {
ctx.waitUntil(reconcile(life).then(() => undefined))
// Counts only; no ids, tokens or pairing material.
ctx.waitUntil(reconcile(life).then((summary) => console.log(JSON.stringify({ event: 'reconcile', ...summary }))))
}
}
} satisfies ExportedHandler<Env>
+71 -7
View File
@@ -1,7 +1,8 @@
// Sandbox lifecycle: one persistent provider sandbox per Orca workspace, with an Orca-owned lease,
// a forced stop before the provider's 24h cap, usage metering and write-ahead rows.
import type { EntitlementClient, MachineSize } from './entitlements'
import { compileNetworkPolicy, type CredentialRoute } from './network-policy'
import type { GitHubAppMinter } from './github-app'
import { compileNetworkPolicy, gitBasicAuthorization, type CredentialRoute } from './network-policy'
import type { ProviderSandboxState, SandboxProviderApi } from './sandbox-provider'
import { ORCAD_PORT, drainOrcad, startOrcad, type OrcadReady } from './sandbox-vm-commands'
import {
@@ -9,6 +10,7 @@ import {
countActiveSandboxes,
getSandbox,
insertSandbox,
listAccountSandboxes,
liveSandboxForWorkspace,
markSnapshotsDeleted,
recordSnapshots,
@@ -23,12 +25,14 @@ export const LEASE_MS = 30 * 60_000
const PROVIDER_SESSION_CAP_MS = 24 * 60 * 60_000
/** Orca stops 10 min early so it, not the provider, ends the session (drain first). */
export const SESSION_DEADLINE_MS = PROVIDER_SESSION_CAP_MS - 10 * 60_000
const VCPUS: Record<MachineSize, number> = { standard: 4, large: 8, xlarge: 16 }
const VCPUS: Record<MachineSize, number> = { standard: 4, large: 8 }
export type LifecycleDeps = {
db: D1Database
provider: SandboxProviderApi
entitlements: EntitlementClient
/** GitHub App installation tokens for the workspace repo; absent until the app is configured. */
github?: GitHubAppMinter
now: () => number
config: {
environment: string
@@ -67,7 +71,19 @@ export function githubPlaceholderHeaders(row: SandboxRow): string[] {
return [`Bearer ${token}`, `token ${token}`]
}
function egressPolicy(deps: LifecycleDeps, row: SandboxRow) {
export function egressPolicy(deps: LifecycleDeps, row: SandboxRow, gitToken?: string) {
const git: CredentialRoute[] = gitToken
? [
{
// git (libcurl) is unaffected by the transform-rule framing hang, so it is injected inline.
host: 'github.com',
header: 'authorization',
placeholders: [gitBasicAuthorization(githubPlaceholderToken(row))],
via: 'transform',
value: gitBasicAuthorization(gitToken)
}
]
: []
const credentials: CredentialRoute[] = deps.config.egressProxyUrl
? [
{
@@ -79,7 +95,7 @@ function egressPolicy(deps: LifecycleDeps, row: SandboxRow) {
}
]
: []
return compileNetworkPolicy({ mode: 'allowlist', credentials })
return compileNetworkPolicy({ mode: 'allowlist', credentials: [...git, ...credentials] })
}
export function sessionFields(deps: LifecycleDeps, state: ProviderSandboxState): SessionFields {
@@ -105,6 +121,9 @@ async function claim(deps: LifecycleDeps, row: SandboxRow, patch: Partial<Sandbo
async function requireStart(deps: LifecycleDeps, accountId: string, machineSize: MachineSize) {
const runningSandboxes = await countActiveSandboxes(deps.db, accountId)
const decision = await deps.entitlements.checkStart({ accountId, machineSize, runningSandboxes })
if (decision.suspendRunning) {
await suspendAccountRunning(deps, accountId)
}
if (!decision.allow) {
throw new LifecycleError('entitlement_denied', decision.reason ?? 'denied')
}
@@ -114,15 +133,39 @@ async function boot(deps: LifecycleDeps, row: SandboxRow, state: ProviderSandbox
if (state.sandbox.tags.orcaSandbox !== row.id) {
throw new LifecycleError('conflict', 'ownership_mismatch')
}
const git = await refreshGitCredentials(deps, row, state.session.id)
const connection = await startOrcad(deps.provider, state, {
sandboxId: row.id,
repo,
githubPlaceholder: deps.config.egressProxyUrl ? githubPlaceholderToken(row) : undefined
githubPlaceholder: deps.config.egressProxyUrl ? githubPlaceholderToken(row) : undefined,
gitExtraHeader: git ? `Authorization: ${gitBasicAuthorization(githubPlaceholderToken(row))}` : undefined
})
const running = await claim(deps, row, {
status: 'running',
last_error: null,
github_token_expires_at: git?.expiresAt ?? null,
...sessionFields(deps, state)
})
const running = await claim(deps, row, { status: 'running', last_error: null, ...sessionFields(deps, state) })
return { row: running, connection }
}
/** Mints a fresh repo token and swaps it into the live policy; null when the repo has no app install. */
export async function refreshGitCredentials(
deps: LifecycleDeps,
row: SandboxRow,
sessionId: string
): Promise<{ expiresAt: number } | null> {
if (!deps.github || !row.repo_url) {
return null
}
const minted = await deps.github.tokenForRepo(row.repo_url)
if (!minted) {
return null
}
await deps.provider.updateNetworkPolicy(sessionId, egressPolicy(deps, row, minted.token))
return { expiresAt: minted.expiresAt }
}
export async function createSandbox(
deps: LifecycleDeps,
caller: { accountId: string; userId: string },
@@ -152,7 +195,8 @@ export async function createSandbox(
provider: 'vercel',
provider_name: `orca-${deps.config.environment}-${id}`,
machine_size: request.machineSize,
region: request.region ?? deps.config.defaultRegion
region: request.region ?? deps.config.defaultRegion,
repo_url: request.repo?.url ?? null
},
deps.now()
)
@@ -180,6 +224,26 @@ export async function createSandbox(
}
}
/**
* Billing says the account's payment grace ran out: stop (snapshot-preserving, never destroy) every
* running sandbox. Resume stays refused by the start check until billing allows starts again.
*/
export async function suspendAccountRunning(deps: LifecycleDeps, accountId: string): Promise<number> {
let suspended = 0
for (const row of await listAccountSandboxes(deps.db, accountId)) {
if (row.status !== 'running') {
continue
}
try {
await suspendSandbox(deps, row)
suspended++
} catch {
// A concurrent transition or provider hiccup; the next cron pass retries.
}
}
return suspended
}
/** A create that never became usable is destroyed now; the cron retries if this also fails. */
export async function failAndRelease(deps: LifecycleDeps, row: SandboxRow): Promise<void> {
const failed = await transition(deps.db, row, { status: 'failed', last_error: 'create_failed' }, deps.now())
+30 -4
View File
@@ -5,6 +5,8 @@ import {
destroySandbox,
failAndRelease,
markProviderStopped,
refreshGitCredentials,
suspendAccountRunning,
sessionFields,
suspendSandbox,
type LifecycleDeps
@@ -17,7 +19,13 @@ const STUCK_MS = 15 * 60_000
const FORCED_STOP_GRACE_MS = 5 * 60_000
export type ReconcileSummary = Record<
'settled' | 'suspendedForDeadline' | 'suspendedForBilling' | 'leaseLapsed' | 'released' | 'repaired',
| 'settled'
| 'suspendedForDeadline'
| 'suspendedForBilling'
| 'leaseLapsed'
| 'released'
| 'repaired'
| 'billingErrors',
number
>
@@ -28,7 +36,16 @@ export async function reconcile(deps: LifecycleDeps): Promise<ReconcileSummary>
suspendedForBilling: 0,
leaseLapsed: 0,
released: 0,
repaired: 0
repaired: 0,
billingErrors: 0
}
const polled = await guard(async () => {
for (const accountId of await deps.entitlements.listSuspensions()) {
summary.suspendedForBilling += await suspendAccountRunning(deps, accountId)
}
})
if (!polled) {
summary.billingErrors++
}
for (const row of await listByStatus(deps.db, ['running'])) {
await guard(() => reconcileRunning(deps, row, summary))
@@ -71,11 +88,18 @@ async function reconcileRunning(deps: LifecycleDeps, row: SandboxRow, summary: R
return
}
}
if (row.github_token_expires_at !== null && row.session_id && row.github_token_expires_at - now < 15 * 60_000) {
const git = await refreshGitCredentials(deps, row, row.session_id)
const refreshed = await transition(deps.db, row, { github_token_expires_at: git?.expiresAt ?? null }, now)
row = refreshed ?? row
}
const settled = await queueUsage(deps.db, row, now)
const current = settled === row.usage_settled_at ? row : await transition(deps.db, row, { usage_settled_at: settled }, now)
summary.settled++
const answer = await flushUsage(deps.db, deps.entitlements, now, row.id)
if (current && answer && !answer.canStartSandbox && STOP_RUNNING_REASONS.has(answer.reason ?? '')) {
if (answer?.suspendRunning) {
summary.suspendedForBilling += await suspendAccountRunning(deps, row.account_id)
} else if (current && answer && !answer.canStartSandbox && STOP_RUNNING_REASONS.has(answer.reason ?? '')) {
await suspendSandbox(deps, current)
summary.suspendedForBilling++
}
@@ -91,10 +115,12 @@ async function repairTransition(deps: LifecycleDeps, row: SandboxRow): Promise<v
await markProviderStopped(deps, row)
}
async function guard(step: () => Promise<void>): Promise<void> {
async function guard(step: () => Promise<void>): Promise<boolean> {
try {
await step()
return true
} catch {
// One bad row must not stall the pass; it is retried next tick.
return false
}
}
@@ -33,6 +33,8 @@ export type StartInputs = {
repo?: { url: string; ref?: string }
/** Token the VM's GitHub clients send; the egress layer swaps it for a real short-lived one. */
githubPlaceholder?: string
/** git `http.https://github.com/.extraheader` carrying the placeholder, so every git request is claimable. */
gitExtraHeader?: string
}
/** Starts orcad (idempotent per boot) and returns its pairing offer; pairing codes are never logged. */
@@ -55,6 +57,9 @@ export async function startOrcad(
env.ORCA_REPO_REF = inputs.repo.ref
}
}
if (inputs.gitExtraHeader) {
env.ORCA_GIT_EXTRAHEADER = inputs.gitExtraHeader
}
if (inputs.githubPlaceholder) {
env.ORCA_GITHUB_PLACEHOLDER = inputs.githubPlaceholder
}
+7 -5
View File
@@ -32,6 +32,8 @@ export type SandboxRow = {
created_at: number
updated_at: number
destroyed_at: number | null
repo_url: string | null
github_token_expires_at: number | null
}
export type SessionFields = Pick<
@@ -54,18 +56,18 @@ export async function insertSandbox(
db: D1Database,
row: Pick<
SandboxRow,
'id' | 'account_id' | 'user_id' | 'workspace_id' | 'provider' | 'provider_name' | 'machine_size' | 'region'
'id' | 'account_id' | 'user_id' | 'workspace_id' | 'provider' | 'provider_name' | 'machine_size' | 'region' | 'repo_url'
>,
now: number
): Promise<SandboxRow> {
await db
.prepare(
`INSERT INTO sandboxes (id, account_id, user_id, workspace_id, provider, provider_name, status,
machine_size, region, generation, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, 'creating', ?, ?, 0, ?, ?)`
machine_size, region, generation, created_at, updated_at, repo_url)
VALUES (?, ?, ?, ?, ?, ?, 'creating', ?, ?, 0, ?, ?, ?)`
)
.bind(row.id, row.account_id, row.user_id, row.workspace_id, row.provider, row.provider_name,
row.machine_size, row.region, now, now)
row.machine_size, row.region, now, now, row.repo_url)
.run()
const inserted = await getSandbox(db, row.id)
if (!inserted) {
@@ -122,7 +124,7 @@ export async function listByStatus(db: D1Database, statuses: readonly SandboxSta
type Patch = Partial<Omit<SandboxRow, 'id' | 'account_id' | 'generation' | 'created_at'>>
const PATCHABLE = new Set<string>([
'status', 'session_id', 'session_started_at', 'session_deadline_at', 'lease_expires_at',
'usage_settled_at', 'last_error', 'destroyed_at'
'usage_settled_at', 'last_error', 'destroyed_at', 'github_token_expires_at'
])
/** Applies `patch` only if nobody moved the row since `expected` was read; null on a lost race. */
+1 -1
View File
@@ -13,7 +13,7 @@ const Id = z
export const CreateRequestSchema = z
.object({
workspaceId: Id,
machineSize: z.enum(['standard', 'large', 'xlarge']),
machineSize: z.enum(['standard', 'large']),
region: z
.string()
.regex(/^[a-z]{3}[0-9]$/)
@@ -0,0 +1,39 @@
import { describe, expect, it } from 'vitest'
import { BillingUnavailableError, createEntitlementClient } from '../src/entitlements'
import type { Env } from '../src/env'
const env = { BILLING_URL: 'https://billing.test', CONTROL_PLANE_TOKEN: 'cp_'.padEnd(40, 'x'), SANDBOX_ENV: 'dev' } as Env
describe('billing client', () => {
it('reads billing dev answers, including grace-expired suspensions', async () => {
const seen: string[] = []
const client = createEntitlementClient(env, async (input, init) => {
const url = new URL(String(input))
seen.push(`${init?.method} ${url.pathname} ${new Headers(init?.headers).get('authorization')}`)
if (url.pathname === '/v1/internal/suspensions') {
return Response.json({ accounts: [{ accountId: 'org:o1', pastDueSince: 1, graceEndsAt: 2 }] })
}
return Response.json({ allow: false, reason: 'payment_grace_expired', suspendRunning: true, entitlement: {} })
})
expect(await client.listSuspensions()).toEqual(['org:o1'])
expect(await client.checkStart({ accountId: 'org:o1', machineSize: 'standard', runningSandboxes: 0 })).toEqual({
allow: false,
reason: 'payment_grace_expired',
suspendRunning: true,
entitlement: {}
})
expect(seen[0]).toBe(`GET /v1/internal/suspensions Bearer ${env.CONTROL_PLANE_TOKEN}`)
})
it('fails closed when billing is unreachable and refuses the stub in prod', async () => {
const down = createEntitlementClient(env, async () => {
throw new Error('offline')
})
await expect(down.checkStart({ accountId: 'a', machineSize: 'large', runningSandboxes: 0 })).rejects.toBeInstanceOf(
BillingUnavailableError
)
expect(() => createEntitlementClient({ ...env, ENTITLEMENTS_MODE: 'stub', SANDBOX_ENV: 'prod' })).toThrow(
BillingUnavailableError
)
})
})
@@ -0,0 +1,73 @@
import { exportPKCS8, generateKeyPair, decodeJwt } from 'jose'
import { describe, expect, it } from 'vitest'
import { createGitHubAppMinter } from '../src/github-app'
import { gitBasicAuthorization } from '../src/network-policy'
import { START_COMMAND } from '../src/sandbox-vm-commands'
import { createHarness } from './harness'
const MIN = 60_000
describe('GitHub App installation tokens', () => {
it('mints a token scoped to the one workspace repo and reuses it until near expiry', async () => {
const { privateKey } = await generateKeyPair('RS256', { extractable: true })
const calls: { method: string; path: string; body: unknown; iss: unknown }[] = []
let now = Date.parse('2026-11-09T00:00:00Z')
const minter = createGitHubAppMinter({
appId: '123',
privateKeyPkcs8: await exportPKCS8(privateKey),
now: () => now,
fetch: async (input, init) => {
const path = new URL(String(input)).pathname
const jwt = new Headers(init?.headers).get('authorization')!.replace('Bearer ', '')
calls.push({ method: init?.method ?? 'GET', path, body: init?.body ? JSON.parse(String(init.body)) : undefined, iss: decodeJwt(jwt).iss })
return path.endsWith('/installation')
? Response.json({ id: 77 })
: Response.json({ token: 'ghs_minted', expires_at: new Date(now + 60 * MIN).toISOString() })
}
})
expect(await minter.tokenForRepo('https://github.com/nwparker/orca-sandbox-push-test.git')).toEqual({ token: 'ghs_minted', expiresAt: now + 60 * MIN })
expect(calls).toEqual([
{ method: 'GET', path: '/repos/nwparker/orca-sandbox-push-test/installation', body: undefined, iss: '123' },
{
method: 'POST',
path: '/app/installations/77/access_tokens',
body: { repositories: ['orca-sandbox-push-test'], permissions: { contents: 'write', pull_requests: 'write', metadata: 'read' } },
iss: '123'
}
])
now += 30 * MIN
await minter.tokenForRepo('https://github.com/nwparker/orca-sandbox-push-test')
expect(calls).toHaveLength(2)
now += 26 * MIN
await minter.tokenForRepo('https://github.com/nwparker/orca-sandbox-push-test')
expect(calls).toHaveLength(4)
expect(await minter.tokenForRepo('https://gitlab.com/a/b')).toBeNull()
})
it('injects the repo token for git and refreshes it before it expires', async () => {
let expiresAt = 0
const h = await createHarness({}, {
github: { tokenForRepo: async () => ({ token: `ghs_${expiresAt}`, expiresAt }) }
})
expiresAt = h.now() + 60 * MIN
const { body } = await h.call('POST', '/v1/sandboxes', {
body: { workspaceId: 'ws-git', machineSize: 'standard', repo: { url: 'https://github.com/nwparker/orca-sandbox-push-test.git' } }
})
const placeholder = gitBasicAuthorization(`orca-ph-${body.sandbox.id}`)
const [box] = [...h.vercel.boxes.values()]
expect(box!.policy.allow['github.com']![0]).toEqual({
match: { headers: [{ key: { exact: 'authorization' }, value: { exact: placeholder } }] },
transform: [{ headers: { authorization: gitBasicAuthorization(`ghs_${expiresAt}`) } }]
})
const start = h.vercel.commands.find((c) => c.command === START_COMMAND)!
expect(start.env.ORCA_GIT_EXTRAHEADER).toBe(`Authorization: ${placeholder}`)
expect(start.env.ORCA_REPO_URL).toBe('https://github.com/nwparker/orca-sandbox-push-test.git')
const first = expiresAt
h.advance(50 * MIN)
expiresAt = h.now() + 60 * MIN
await h.reconcile()
expect(box!.policy.allow['github.com']![0]).toMatchObject({ transform: [{ headers: { authorization: gitBasicAuthorization(`ghs_${expiresAt}`) } }] })
expect(expiresAt).not.toBe(first)
})
})
+7 -2
View File
@@ -16,8 +16,9 @@ export async function createHarness(overrides: Partial<Env> = {}, extraDeps: Par
const clock = () => now
const vercel = createFakeVercel(clock)
const billing = {
decision: { allow: true, reason: null } as StartDecision,
usageAnswer: { canStartSandbox: true, reason: null } as UsageAnswer,
decision: { allow: true, reason: null, suspendRunning: false } as StartDecision,
usageAnswer: { canStartSandbox: true, reason: null, suspendRunning: false } as UsageAnswer,
suspensions: [] as string[],
down: false,
checks: [] as { accountId: string; runningSandboxes: number }[],
usage: [] as UsageSegment[]
@@ -33,6 +34,9 @@ export async function createHarness(overrides: Partial<Env> = {}, extraDeps: Par
}
billing.usage.push(segment)
return billing.usageAnswer
},
async listSuspensions() {
return billing.suspensions
}
}
const env: Env = {
@@ -91,6 +95,7 @@ export async function createHarness(overrides: Partial<Env> = {}, extraDeps: Par
db,
provider: vercel,
entitlements,
github: extraDeps.github,
now: clock,
config: { environment: 'test', goldenSnapshotId: GOLDEN, defaultRegion: 'iad1' }
})
@@ -37,7 +37,7 @@ describe('sandbox control plane lifecycle', () => {
it('refuses a start billing denies without touching the provider', async () => {
const h = await createHarness()
h.billing.decision = { allow: false, reason: 'concurrency_limit' }
h.billing.decision = { allow: false, reason: 'concurrency_limit', suspendRunning: false }
const res = await h.call('POST', '/v1/sandboxes', { body: create })
expect(res).toEqual({ status: 403, body: { error: 'entitlement_denied', reason: 'concurrency_limit' } })
expect(h.vercel.boxes.size).toBe(0)
@@ -163,7 +163,7 @@ describe('sandbox control plane cron', () => {
it('stops running sandboxes once billing says the included hours are gone', async () => {
const h = await createHarness()
const { body } = await h.call('POST', '/v1/sandboxes', { body: create })
h.billing.usageAnswer = { canStartSandbox: false, reason: 'included_hours_exhausted' }
h.billing.usageAnswer = { canStartSandbox: false, reason: 'included_hours_exhausted', suspendRunning: false }
h.advance(10 * MIN)
expect((await h.reconcile()).suspendedForBilling).toBe(1)
expect((await h.call('GET', `/v1/sandboxes/${body.sandbox.id}`)).body.sandbox.status).toBe('suspended')
@@ -189,4 +189,40 @@ describe('sandbox control plane cron', () => {
expect((await h.reconcile()).released).toBe(1)
expect(h.vercel.boxes.size).toBe(0)
})
it('suspends an account whose payment grace expired, keeps its disk, and resumes only after payment', async () => {
const h = await createHarness()
const a = (await h.call('POST', '/v1/sandboxes', { body: create })).body.sandbox.id
const b = (await h.call('POST', '/v1/sandboxes', { body: { ...create, workspaceId: 'ws-2' } })).body.sandbox.id
const other = (await h.call('POST', '/v1/sandboxes', { body: create, as: 'bob' })).body.sandbox.id
h.billing.suspensions = ['user:alice']
expect((await h.reconcile()).suspendedForBilling).toBe(2)
expect((await h.call('GET', `/v1/sandboxes/${a}`)).body.sandbox.status).toBe('suspended')
expect((await h.call('GET', `/v1/sandboxes/${b}`)).body.sandbox.status).toBe('suspended')
expect((await h.call('GET', `/v1/sandboxes/${other}`, { as: 'bob' })).body.sandbox.status).toBe('running')
expect(h.vercel.boxes.size).toBe(3)
h.billing.decision = { allow: false, reason: 'payment_grace_expired', suspendRunning: true }
expect(await h.call('POST', `/v1/sandboxes/${a}/resume`)).toEqual({
status: 403,
body: { error: 'entitlement_denied', reason: 'payment_grace_expired' }
})
h.billing.suspensions = []
h.billing.decision = { allow: true, reason: null, suspendRunning: false }
expect((await h.call('POST', `/v1/sandboxes/${a}/resume`)).body.sandbox.status).toBe('running')
})
it('suspends the whole account when a usage report says grace expired', async () => {
const h = await createHarness()
const a = (await h.call('POST', '/v1/sandboxes', { body: create })).body.sandbox.id
h.billing.usageAnswer = { canStartSandbox: false, reason: 'payment_grace_expired', suspendRunning: true }
h.advance(10 * MIN)
expect((await h.reconcile()).suspendedForBilling).toBe(1)
expect((await h.call('GET', `/v1/sandboxes/${a}`)).body.sandbox.status).toBe('suspended')
})
it('refuses machine sizes billing does not sell', async () => {
const h = await createHarness()
expect((await h.call('POST', '/v1/sandboxes', { body: { ...create, machineSize: 'xlarge' } })).status).toBe(400)
})
})
@@ -3,7 +3,8 @@
// DEV deployment only (workers.dev, no custom domain). Deploy:
// wrangler deploy -c wrangler.dev.jsonc
// Secrets (wrangler secret put -c wrangler.dev.jsonc, never here): VERCEL_TOKEN, SANDBOX_DEV_JWKS,
// EGRESS_DEV_STATIC_AUTHORIZATION, and CONTROL_PLANE_TOKEN once BILLING_URL points at billing dev.
// EGRESS_DEV_STATIC_AUTHORIZATION, CONTROL_PLANE_TOKEN (same value as on
// orca-billing-dev), GITHUB_APP_ID, GITHUB_APP_PRIVATE_KEY (PKCS#8).
"name": "orca-sandbox-control-dev",
"main": "src/index.ts",
"compatibility_date": "2026-11-01",
@@ -16,6 +17,8 @@
"migrations_dir": "migrations"
}
],
// Same-account Workers cannot call each other over workers.dev (error 1042); BILLING_URL names the origin.
"services": [{ "binding": "BILLING", "service": "orca-billing-dev" }],
"triggers": { "crons": ["*/5 * * * *"] },
"vars": {
"SANDBOX_ENV": "dev",
@@ -25,8 +28,8 @@
"VERCEL_TEAM_ID": "team_c5IO9DfEYAxKEUgMxrbEtO0a",
"VERCEL_PROJECT_ID": "prj_SkVtv6zJurF9dUHpfTRy4v9avwPq",
"VERCEL_REGION": "iad1",
"ENTITLEMENTS_MODE": "stub",
// GOLDEN_SNAPSHOT_ID: unset until an image is built (image/build-golden-image.ts); lifecycle routes answer 503 meanwhile.
"BILLING_URL": "https://orca-billing-dev.lovecast.workers.dev",
"GOLDEN_SNAPSHOT_ID": "snap_aaJmS6etH8vyrTSxOQxPU7lx02QZ",
"EGRESS_PROXY_URL": "https://orca-sandbox-control-dev.lovecast.workers.dev/v1/egress/proxy"
}
}
@@ -2,7 +2,8 @@
// The Worker's tests parse its responses with these schemas, so either side drifting fails a test.
import { z } from 'zod'
export const HOSTED_SANDBOX_MACHINE_SIZES = ['standard', 'large', 'xlarge'] as const
// Billing sells exactly these; larger boxes need a Vercel Enterprise plan.
export const HOSTED_SANDBOX_MACHINE_SIZES = ['standard', 'large'] as const
export type HostedSandboxMachineSize = (typeof HOSTED_SANDBOX_MACHINE_SIZES)[number]
export const HOSTED_SANDBOX_STATUSES = [