mirror of
https://github.com/stablyai/orca.git
synced 2026-11-19 04:23:57 +00:00
fix(cloud): bind GitHub App installations to Orca accounts; kill switch, caps, rate limit; orphan sweep and retention (#24)
Security (B1): repo tokens were minted for any repo.url where the app was installed. Now an installation must be bound to the caller's account (signed single-use connect state + OAuth proof that the installer owns the account or is an org admin), tokens are scoped to the one repo and the binding is re-checked on cached tokens. Operator binding route for support/dev. Guardrails (B3/G4): D1 kill switch + static flag refuse create/resume/rollover and drain running sandboxes; global running cap; 3/seat per-account backstop; per-account create rate limit. Retention (B4/B6): hourly orphan sweep (stop, then delete after grace; partial listings delete nothing), 30-day deletion after cancel/unpaid with a T-7 warning hook, 90-day idle retention, 104-day provider snapshot expiry backstop, repo_url tombstoned on destroy.
This commit is contained in:
@@ -0,0 +1,64 @@
|
||||
-- GitHub App installations bound to Orca accounts. Repo tokens are minted only for a caller whose
|
||||
-- account has a live binding to the installation that covers the repo (deny by default).
|
||||
CREATE TABLE github_installations (
|
||||
installation_id INTEGER NOT NULL,
|
||||
account_id TEXT NOT NULL,
|
||||
github_account_id INTEGER NOT NULL,
|
||||
github_account_login TEXT NOT NULL,
|
||||
github_account_type TEXT NOT NULL,
|
||||
-- Orca user who bound it, and the GitHub user proven (OAuth) to own/admin the installation account.
|
||||
bound_by_user_id TEXT NOT NULL,
|
||||
bound_by_github_user_id INTEGER,
|
||||
-- `oauth` (owner/org admin verified with a user-to-server token) | `operator` (support tool).
|
||||
method TEXT NOT NULL,
|
||||
created_at INTEGER NOT NULL,
|
||||
revoked_at INTEGER,
|
||||
PRIMARY KEY (installation_id, account_id)
|
||||
);
|
||||
CREATE INDEX github_installations_account ON github_installations (account_id);
|
||||
|
||||
-- Single-use install/connect state nonces (signed state carries the account; this stops replay).
|
||||
CREATE TABLE github_connect_states (
|
||||
nonce TEXT PRIMARY KEY,
|
||||
account_id TEXT NOT NULL,
|
||||
expires_at INTEGER NOT NULL,
|
||||
consumed_at INTEGER
|
||||
);
|
||||
|
||||
-- Operator flags (kill switch) and cron bookkeeping; D1 so flipping needs no redeploy.
|
||||
CREATE TABLE control_state (
|
||||
name TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- Fixed-window create counters per account.
|
||||
CREATE TABLE rate_windows (
|
||||
bucket TEXT PRIMARY KEY,
|
||||
window_start INTEGER NOT NULL,
|
||||
count INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- Provider resources with no live D1 record: suspended on first sight, deleted after a grace period.
|
||||
CREATE TABLE provider_orphans (
|
||||
kind TEXT NOT NULL,
|
||||
provider_id TEXT NOT NULL,
|
||||
first_seen_at INTEGER NOT NULL,
|
||||
last_seen_at INTEGER NOT NULL,
|
||||
suspended_at INTEGER,
|
||||
deleted_at INTEGER,
|
||||
PRIMARY KEY (kind, provider_id)
|
||||
);
|
||||
|
||||
-- Accounts whose subscription is canceled/unpaid: data is deleted DATA_RETENTION_DAYS after `since`.
|
||||
CREATE TABLE account_holds (
|
||||
account_id TEXT PRIMARY KEY,
|
||||
reason TEXT NOT NULL,
|
||||
since INTEGER NOT NULL,
|
||||
checked_at INTEGER NOT NULL,
|
||||
warned_at INTEGER,
|
||||
purged_at INTEGER
|
||||
);
|
||||
|
||||
-- Idle-retention warning marker for long-suspended sandboxes.
|
||||
ALTER TABLE sandboxes ADD COLUMN retention_warned_at INTEGER;
|
||||
@@ -7,7 +7,13 @@ import type { Env } from './env'
|
||||
export type MachineSize = 'standard' | 'large'
|
||||
|
||||
/** `suspendRunning`: payment grace ran out, so the account's running sandboxes must stop too. */
|
||||
export type StartDecision = { allow: boolean; reason: string | null; suspendRunning: boolean }
|
||||
export type StartDecision = {
|
||||
allow: boolean
|
||||
reason: string | null
|
||||
suspendRunning: boolean
|
||||
/** Paid seats, when billing reports them; scales the control plane's per-account backstop. */
|
||||
seats?: number
|
||||
}
|
||||
|
||||
export type UsageSegment = {
|
||||
usageId: string
|
||||
@@ -41,7 +47,8 @@ export class BillingUnavailableError extends Error {
|
||||
const StartAnswer = z.looseObject({
|
||||
allow: z.boolean(),
|
||||
reason: z.string().nullable(),
|
||||
suspendRunning: z.boolean().default(false)
|
||||
suspendRunning: z.boolean().default(false),
|
||||
seats: z.number().int().positive().max(100_000).optional()
|
||||
})
|
||||
const UsageResponse = z.looseObject({
|
||||
canStartSandbox: z.boolean(),
|
||||
@@ -52,6 +59,13 @@ const SuspensionsResponse = z.looseObject({
|
||||
accounts: z.array(z.looseObject({ accountId: z.string() })).max(10_000)
|
||||
})
|
||||
|
||||
/** Reasons that start the data-retention clock (canceled / unpaid). */
|
||||
export const RETENTION_HOLD_REASONS: ReadonlySet<string> = new Set([
|
||||
'subscription_inactive',
|
||||
'no_subscription',
|
||||
'payment_grace_expired'
|
||||
])
|
||||
|
||||
/** Reasons that mean running sandboxes must stop, not just that new starts wait. */
|
||||
export const STOP_RUNNING_REASONS: ReadonlySet<string> = new Set([
|
||||
'included_hours_exhausted',
|
||||
|
||||
@@ -39,7 +39,7 @@ export type Env = {
|
||||
/** Dev-only: Authorization the egress proxy injects for api.github.com until the GitHub App minter exists. */
|
||||
EGRESS_DEV_STATIC_AUTHORIZATION?: string
|
||||
|
||||
// GitHub App (contents/pull_requests write, metadata read). The PKCS#8 key is a secret.
|
||||
// GitHub App (contents/pull_requests write, metadata read, org members read). The PKCS#8 key is a secret.
|
||||
GITHUB_APP_ID?: string
|
||||
GITHUB_APP_PRIVATE_KEY?: string
|
||||
|
||||
@@ -55,4 +55,31 @@ export type Env = {
|
||||
* key in (forwardURL path). Anything else: plain env injection. Needs EGRESS_PROXY_URL.
|
||||
*/
|
||||
SECRETS_EGRESS_PLACEHOLDERS?: string
|
||||
|
||||
/** App slug for the install URL (github.com/apps/<slug>). */
|
||||
GITHUB_APP_SLUG?: string
|
||||
/** OAuth credentials of the same app: prove who installed it before binding (secret: CLIENT_SECRET). */
|
||||
GITHUB_APP_CLIENT_ID?: string
|
||||
GITHUB_APP_CLIENT_SECRET?: string
|
||||
/** HMAC key for the signed connect state (secret, >= 32 chars). */
|
||||
GITHUB_STATE_SECRET?: string
|
||||
|
||||
/** Operator routes (/v1/internal/*): kill switch, drain, bindings. Secret, >= 32 chars; unset = no routes. */
|
||||
OPERATOR_TOKEN?: string
|
||||
/** `true` refuses every create/resume/rollover (static kill switch; the D1 flag is the runtime one). */
|
||||
SANDBOX_CREATES_DISABLED?: string
|
||||
/** Global cap on active sandboxes across all accounts (default 50). */
|
||||
SANDBOX_MAX_RUNNING?: string
|
||||
/** Per-account active cap per paid seat (default 3); billing enforces the plan limit as well. */
|
||||
ACCOUNT_MAX_ACTIVE_PER_SEAT?: string
|
||||
/** POST /v1/sandboxes per account per minute (default 10). */
|
||||
CREATE_RATE_PER_MINUTE?: string
|
||||
/** Days data is kept after cancel/unpaid before deletion (default 30). */
|
||||
DATA_RETENTION_DAYS?: string
|
||||
/** Days a suspended sandbox may sit untouched before deletion (default 90). */
|
||||
IDLE_RETENTION_DAYS?: string
|
||||
/** Hours an orphan provider resource is kept (suspended) before deletion (default 24). */
|
||||
ORPHAN_GRACE_HOURS?: string
|
||||
/** Provider snapshot expiry after last use, days (default 104, backstop behind idle retention). */
|
||||
SNAPSHOT_EXPIRATION_DAYS?: string
|
||||
}
|
||||
|
||||
@@ -1,21 +1,50 @@
|
||||
// GitHub App installation tokens for sandbox git: 1h, scoped to the workspace's one repository with
|
||||
// contents + pull_requests write. Minted here, injected at the egress layer, never stored in the VM.
|
||||
// A token is minted only when the caller's Orca account is bound to the installation that covers the
|
||||
// repo (github-connect.ts records bindings); an app installed on a repo is not by itself a grant.
|
||||
import { SignJWT, importPKCS8 } from 'jose'
|
||||
import { z } from 'zod'
|
||||
|
||||
export type GitHubRepoToken = { token: string; expiresAt: number }
|
||||
|
||||
export type GitHubAppMinter = {
|
||||
/** Null when the repo is not on github.com or the app is not installed on it. */
|
||||
tokenForRepo(repoUrl: string): Promise<GitHubRepoToken | null>
|
||||
export type GitHubInstallation = {
|
||||
id: number
|
||||
account: { id: number; login: string; type: 'User' | 'Organization' }
|
||||
}
|
||||
|
||||
export type GitHubAppMinter = {
|
||||
/**
|
||||
* Null when the repo is not on github.com, the app is not installed on it, or the installation is
|
||||
* not bound to `accountId`. Deny by default.
|
||||
*/
|
||||
tokenForRepo(repoUrl: string, accountId: string): Promise<GitHubRepoToken | null>
|
||||
/** App-level installation lookup (operator binding). Null when absent. */
|
||||
getInstallation(installationId: number): Promise<GitHubInstallation | null>
|
||||
/** The installation covering a github.com repo URL, or null. */
|
||||
getRepoInstallation(repoUrl: string): Promise<GitHubInstallation | null>
|
||||
}
|
||||
|
||||
/** Whether `accountId` holds a live binding to the installation. */
|
||||
export type InstallationBindingCheck = (installationId: number, accountId: string) => Promise<boolean>
|
||||
|
||||
const REFRESH_MARGIN_MS = 5 * 60_000
|
||||
const REPO_PERMISSIONS = { contents: 'write', pull_requests: 'write', metadata: 'read' } as const
|
||||
|
||||
const Installation = z.looseObject({
|
||||
id: z.number().int().positive(),
|
||||
account: z.looseObject({ id: z.number().int(), login: z.string(), type: z.enum(['User', 'Organization']) })
|
||||
})
|
||||
const MintedToken = z.looseObject({
|
||||
token: z.string().min(1),
|
||||
expires_at: z.string(),
|
||||
repositories: z.array(z.looseObject({ id: z.number(), name: z.string() })).optional()
|
||||
})
|
||||
|
||||
export function parseGitHubRepo(repoUrl: string): { owner: string; repo: string } | null {
|
||||
try {
|
||||
const url = new URL(repoUrl)
|
||||
const match = /^\/([A-Za-z0-9-]+)\/([A-Za-z0-9._-]+?)(?:\.git)?\/?$/.exec(url.pathname)
|
||||
return url.protocol === 'https:' && url.hostname === 'github.com' && match
|
||||
return url.protocol === 'https:' && url.hostname === 'github.com' && !url.username && !url.password && match
|
||||
? { owner: match[1]!, repo: match[2]! }
|
||||
: null
|
||||
} catch {
|
||||
@@ -26,12 +55,15 @@ export function parseGitHubRepo(repoUrl: string): { owner: string; repo: string
|
||||
export function createGitHubAppMinter(config: {
|
||||
appId: string
|
||||
privateKeyPkcs8: string
|
||||
isBound: InstallationBindingCheck
|
||||
fetch?: typeof fetch
|
||||
now?: () => number
|
||||
log?: (event: Record<string, unknown>) => void
|
||||
}): GitHubAppMinter {
|
||||
const doFetch = config.fetch ?? fetch
|
||||
const now = config.now ?? Date.now
|
||||
const cache = new Map<string, GitHubRepoToken>()
|
||||
const log = config.log ?? ((event) => console.log(JSON.stringify(event)))
|
||||
const cache = new Map<string, GitHubRepoToken & { installationId: number }>()
|
||||
let signingKey: CryptoKey | null = null
|
||||
|
||||
async function appJwt(): Promise<string> {
|
||||
@@ -59,8 +91,30 @@ export function createGitHubAppMinter(config: {
|
||||
})
|
||||
}
|
||||
|
||||
async function installationFor(path: string): Promise<GitHubInstallation | null> {
|
||||
const response = await github('GET', path)
|
||||
if (response.status === 404) {
|
||||
return null
|
||||
}
|
||||
if (!response.ok) {
|
||||
throw new Error(`github installation lookup failed (${response.status})`)
|
||||
}
|
||||
const parsed = Installation.safeParse(await response.json().catch(() => null))
|
||||
if (!parsed.success) {
|
||||
throw new Error('github installation lookup answered an unexpected body')
|
||||
}
|
||||
return parsed.data
|
||||
}
|
||||
|
||||
return {
|
||||
async tokenForRepo(repoUrl) {
|
||||
getInstallation: (installationId) => installationFor(`/app/installations/${installationId}`),
|
||||
|
||||
async getRepoInstallation(repoUrl) {
|
||||
const parsed = parseGitHubRepo(repoUrl)
|
||||
return parsed ? installationFor(`/repos/${parsed.owner}/${parsed.repo}/installation`) : null
|
||||
},
|
||||
|
||||
async tokenForRepo(repoUrl, accountId) {
|
||||
const parsed = parseGitHubRepo(repoUrl)
|
||||
if (!parsed) {
|
||||
return null
|
||||
@@ -68,26 +122,36 @@ export function createGitHubAppMinter(config: {
|
||||
const key = `${parsed.owner}/${parsed.repo}`.toLowerCase()
|
||||
const cached = cache.get(key)
|
||||
if (cached && cached.expiresAt - now() > REFRESH_MARGIN_MS) {
|
||||
return cached
|
||||
// The binding is re-checked on every use: a revoked binding stops a cached token too.
|
||||
return (await config.isBound(cached.installationId, accountId)) ? { token: cached.token, expiresAt: cached.expiresAt } : null
|
||||
}
|
||||
const installation = await github('GET', `/repos/${parsed.owner}/${parsed.repo}/installation`)
|
||||
if (installation.status === 404) {
|
||||
const installation = await installationFor(`/repos/${parsed.owner}/${parsed.repo}/installation`)
|
||||
if (!installation) {
|
||||
return null
|
||||
}
|
||||
if (!installation.ok) {
|
||||
throw new Error(`github installation lookup failed (${installation.status})`)
|
||||
if (!(await config.isBound(installation.id, accountId))) {
|
||||
log({ event: 'github_token_denied', reason: 'installation_not_bound', installationId: installation.id })
|
||||
return null
|
||||
}
|
||||
const { id } = (await installation.json()) as { id: number }
|
||||
const minted = await github('POST', `/app/installations/${id}/access_tokens`, {
|
||||
const minted = await github('POST', `/app/installations/${installation.id}/access_tokens`, {
|
||||
repositories: [parsed.repo],
|
||||
permissions: { contents: 'write', pull_requests: 'write', metadata: 'read' }
|
||||
permissions: REPO_PERMISSIONS
|
||||
})
|
||||
if (!minted.ok) {
|
||||
throw new Error(`github token mint failed (${minted.status})`)
|
||||
}
|
||||
const body = (await minted.json()) as { token: string; expires_at: string }
|
||||
const token = { token: body.token, expiresAt: Date.parse(body.expires_at) }
|
||||
cache.set(key, token)
|
||||
const body = MintedToken.safeParse(await minted.json().catch(() => null))
|
||||
// GitHub narrows to the named repo; refuse anything wider than that one repository.
|
||||
if (
|
||||
!body.success ||
|
||||
!body.data.repositories ||
|
||||
body.data.repositories.length !== 1 ||
|
||||
body.data.repositories[0]!.name.toLowerCase() !== parsed.repo.toLowerCase()
|
||||
) {
|
||||
throw new Error('github token mint answered an unexpected scope')
|
||||
}
|
||||
const token = { token: body.data.token, expiresAt: Date.parse(body.data.expires_at) }
|
||||
cache.set(key, { ...token, installationId: installation.id })
|
||||
return token
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,305 @@
|
||||
// Binds GitHub App installations to Orca accounts. The desktop asks for a connect URL (signed,
|
||||
// single-use state carrying its Orca account); GitHub sends the browser back to the setup/callback
|
||||
// URL; we exchange the OAuth code for a user-to-server token and bind an installation only when that
|
||||
// GitHub user owns the installation's account (personal) or is an active admin of the org.
|
||||
// Installation ids in a callback URL are never trusted on their own.
|
||||
import { z } from 'zod'
|
||||
import type { GitHubInstallation } from './github-app'
|
||||
|
||||
const STATE_TTL_MS = 10 * 60_000
|
||||
|
||||
export type GitHubConnectConfig = {
|
||||
appSlug: string
|
||||
clientId: string
|
||||
clientSecret: string
|
||||
stateSecret: string
|
||||
fetch?: typeof fetch
|
||||
}
|
||||
|
||||
export type InstallationBinding = {
|
||||
installation_id: number
|
||||
account_id: string
|
||||
github_account_id: number
|
||||
github_account_login: string
|
||||
github_account_type: string
|
||||
bound_by_user_id: string
|
||||
bound_by_github_user_id: number | null
|
||||
method: 'oauth' | 'operator'
|
||||
created_at: number
|
||||
revoked_at: number | null
|
||||
}
|
||||
|
||||
// ---- D1 ----
|
||||
|
||||
export async function isInstallationBound(db: D1Database, installationId: number, accountId: string): Promise<boolean> {
|
||||
const row = await db
|
||||
.prepare('SELECT 1 AS ok FROM github_installations WHERE installation_id = ? AND account_id = ? AND revoked_at IS NULL')
|
||||
.bind(installationId, accountId)
|
||||
.first<{ ok: number }>()
|
||||
return row !== null
|
||||
}
|
||||
|
||||
export async function bindInstallation(
|
||||
db: D1Database,
|
||||
binding: Omit<InstallationBinding, 'created_at' | 'revoked_at'>,
|
||||
now: number
|
||||
): Promise<void> {
|
||||
await db
|
||||
.prepare(
|
||||
`INSERT INTO github_installations (installation_id, account_id, github_account_id, github_account_login,
|
||||
github_account_type, bound_by_user_id, bound_by_github_user_id, method, created_at, revoked_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, NULL)
|
||||
ON CONFLICT (installation_id, account_id) DO UPDATE SET github_account_id = excluded.github_account_id,
|
||||
github_account_login = excluded.github_account_login, github_account_type = excluded.github_account_type,
|
||||
bound_by_user_id = excluded.bound_by_user_id, bound_by_github_user_id = excluded.bound_by_github_user_id,
|
||||
method = excluded.method, created_at = excluded.created_at, revoked_at = NULL`
|
||||
)
|
||||
.bind(binding.installation_id, binding.account_id, binding.github_account_id, binding.github_account_login,
|
||||
binding.github_account_type, binding.bound_by_user_id, binding.bound_by_github_user_id, binding.method, now)
|
||||
.run()
|
||||
}
|
||||
|
||||
export async function revokeInstallation(db: D1Database, installationId: number, accountId: string | null, now: number) {
|
||||
const result = await db
|
||||
.prepare(
|
||||
`UPDATE github_installations SET revoked_at = ? WHERE installation_id = ? AND revoked_at IS NULL
|
||||
${accountId ? 'AND account_id = ?' : ''}`
|
||||
)
|
||||
.bind(...(accountId ? [now, installationId, accountId] : [now, installationId]))
|
||||
.run()
|
||||
return result.meta.changes
|
||||
}
|
||||
|
||||
export async function listAccountInstallations(db: D1Database, accountId: string) {
|
||||
const { results } = await db
|
||||
.prepare(
|
||||
`SELECT installation_id, github_account_login, github_account_type, method, created_at
|
||||
FROM github_installations WHERE account_id = ? AND revoked_at IS NULL ORDER BY created_at`
|
||||
)
|
||||
.bind(accountId)
|
||||
.all<{ installation_id: number; github_account_login: string; github_account_type: string; method: string; created_at: number }>()
|
||||
return results
|
||||
}
|
||||
|
||||
// ---- signed state ----
|
||||
|
||||
const StatePayload = z.object({
|
||||
n: z.string().regex(/^[a-f0-9]{32}$/),
|
||||
a: z.string().min(1).max(160),
|
||||
u: z.string().min(1).max(160),
|
||||
e: z.number().int(),
|
||||
i: z.number().int().positive().optional()
|
||||
})
|
||||
export type ConnectState = z.infer<typeof StatePayload>
|
||||
|
||||
const b64url = (bytes: Uint8Array) => btoa(String.fromCharCode(...bytes)).replaceAll('+', '-').replaceAll('/', '_').replace(/=+$/, '')
|
||||
const fromB64url = (text: string) => Uint8Array.from(atob(text.replaceAll('-', '+').replaceAll('_', '/')), (c) => c.charCodeAt(0))
|
||||
|
||||
async function hmac(secret: string, data: string): Promise<Uint8Array> {
|
||||
const key = await crypto.subtle.importKey('raw', new TextEncoder().encode(secret), { name: 'HMAC', hash: 'SHA-256' }, false, ['sign'])
|
||||
return new Uint8Array(await crypto.subtle.sign('HMAC', key, new TextEncoder().encode(data)))
|
||||
}
|
||||
|
||||
export async function signState(secret: string, payload: ConnectState): Promise<string> {
|
||||
const body = b64url(new TextEncoder().encode(JSON.stringify(payload)))
|
||||
return `${body}.${b64url(await hmac(secret, body))}`
|
||||
}
|
||||
|
||||
/** Null on a bad signature, malformed payload or expiry. Does not consume the nonce. */
|
||||
export async function verifyState(secret: string, state: string, now: number): Promise<ConnectState | null> {
|
||||
const [body, mac, extra] = state.split('.')
|
||||
if (!body || !mac || extra !== undefined || state.length > 2048) {
|
||||
return null
|
||||
}
|
||||
try {
|
||||
const expected = await hmac(secret, body)
|
||||
const given = fromB64url(mac)
|
||||
if (given.length !== expected.length || !timingSafeEqual(given, expected)) {
|
||||
return null
|
||||
}
|
||||
const parsed = StatePayload.safeParse(JSON.parse(new TextDecoder().decode(fromB64url(body))))
|
||||
return parsed.success && parsed.data.e > now ? parsed.data : null
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
export function timingSafeEqual(a: Uint8Array, b: Uint8Array): boolean {
|
||||
let diff = a.length ^ b.length
|
||||
for (let i = 0; i < Math.max(a.length, b.length); i++) {
|
||||
diff |= (a[i] ?? 0) ^ (b[i] ?? 0)
|
||||
}
|
||||
return diff === 0
|
||||
}
|
||||
|
||||
export async function createConnectState(
|
||||
db: D1Database,
|
||||
secret: string,
|
||||
caller: { accountId: string; userId: string },
|
||||
now: number
|
||||
): Promise<string> {
|
||||
const nonce = Array.from(crypto.getRandomValues(new Uint8Array(16)), (b) => b.toString(16).padStart(2, '0')).join('')
|
||||
await db
|
||||
.prepare('INSERT INTO github_connect_states (nonce, account_id, expires_at) VALUES (?, ?, ?)')
|
||||
.bind(nonce, caller.accountId, now + STATE_TTL_MS)
|
||||
.run()
|
||||
return signState(secret, { n: nonce, a: caller.accountId, u: caller.userId, e: now + STATE_TTL_MS })
|
||||
}
|
||||
|
||||
/** Marks the nonce used; false when it was already used, unknown, or issued to another account. */
|
||||
async function consumeState(db: D1Database, state: ConnectState, now: number): Promise<boolean> {
|
||||
const result = await db
|
||||
.prepare(
|
||||
`UPDATE github_connect_states SET consumed_at = ? WHERE nonce = ? AND account_id = ? AND consumed_at IS NULL AND expires_at > ?`
|
||||
)
|
||||
.bind(now, state.n, state.a, now)
|
||||
.run()
|
||||
return result.meta.changes === 1
|
||||
}
|
||||
|
||||
// ---- GitHub user-to-server verification ----
|
||||
|
||||
const UserBody = z.looseObject({ id: z.number().int(), login: z.string() })
|
||||
const UserInstallations = z.looseObject({
|
||||
installations: z.array(
|
||||
z.looseObject({
|
||||
id: z.number().int(),
|
||||
account: z.looseObject({ id: z.number().int(), login: z.string(), type: z.enum(['User', 'Organization']) })
|
||||
})
|
||||
)
|
||||
})
|
||||
const Membership = z.looseObject({ state: z.string(), role: z.string() })
|
||||
|
||||
export type VerifiedInstallation = GitHubInstallation & { githubUserId: number }
|
||||
|
||||
/**
|
||||
* Exchanges the OAuth code and returns the installations this GitHub user may bind: ones on their
|
||||
* own account, or on an org where they are an active admin. `only` narrows to one installation id.
|
||||
*/
|
||||
export async function verifiedInstallations(
|
||||
config: GitHubConnectConfig,
|
||||
code: string,
|
||||
only?: number
|
||||
): Promise<{ githubUser: { id: number; login: string }; allowed: VerifiedInstallation[]; denied: number[] }> {
|
||||
const doFetch = config.fetch ?? fetch
|
||||
const exchanged = await doFetch('https://github.com/login/oauth/access_token', {
|
||||
method: 'POST',
|
||||
headers: { accept: 'application/json', 'content-type': 'application/json', 'user-agent': 'orca-sandbox-control' },
|
||||
body: JSON.stringify({ client_id: config.clientId, client_secret: config.clientSecret, code }),
|
||||
redirect: 'manual'
|
||||
})
|
||||
const token = z.looseObject({ access_token: z.string().min(1) }).safeParse(await exchanged.json().catch(() => null))
|
||||
if (!exchanged.ok || !token.success) {
|
||||
throw new GitHubConnectError('oauth_exchange_failed')
|
||||
}
|
||||
const userApi = async (path: string) =>
|
||||
doFetch(`https://api.github.com${path}`, {
|
||||
headers: {
|
||||
authorization: `Bearer ${token.data.access_token}`,
|
||||
accept: 'application/vnd.github+json',
|
||||
'x-github-api-version': '2022-11-28',
|
||||
'user-agent': 'orca-sandbox-control'
|
||||
},
|
||||
redirect: 'manual'
|
||||
})
|
||||
const userResponse = await userApi('/user')
|
||||
const user = UserBody.safeParse(await userResponse.json().catch(() => null))
|
||||
if (!userResponse.ok || !user.success) {
|
||||
throw new GitHubConnectError('github_user_unavailable')
|
||||
}
|
||||
const listed = await userApi('/user/installations?per_page=100')
|
||||
const installations = UserInstallations.safeParse(await listed.json().catch(() => null))
|
||||
if (!listed.ok || !installations.success) {
|
||||
throw new GitHubConnectError('github_installations_unavailable')
|
||||
}
|
||||
const candidates = installations.data.installations.filter((i) => only === undefined || i.id === only)
|
||||
const allowed: VerifiedInstallation[] = []
|
||||
const denied: number[] = []
|
||||
for (const installation of candidates) {
|
||||
let ok = false
|
||||
if (installation.account.type === 'User') {
|
||||
ok = installation.account.id === user.data.id
|
||||
} else {
|
||||
// Org: only an active org admin may bind it (needs the app's Members:read org permission).
|
||||
const membership = await userApi(`/user/memberships/orgs/${encodeURIComponent(installation.account.login)}`)
|
||||
const body = Membership.safeParse(await membership.json().catch(() => null))
|
||||
ok = membership.ok && body.success && body.data.state === 'active' && body.data.role === 'admin'
|
||||
}
|
||||
if (ok) {
|
||||
allowed.push({ id: installation.id, account: installation.account, githubUserId: user.data.id })
|
||||
} else {
|
||||
denied.push(installation.id)
|
||||
}
|
||||
}
|
||||
if (only !== undefined && candidates.length === 0) {
|
||||
denied.push(only)
|
||||
}
|
||||
return { githubUser: user.data, allowed, denied }
|
||||
}
|
||||
|
||||
export class GitHubConnectError extends Error {
|
||||
constructor(readonly code: string) {
|
||||
super(code)
|
||||
this.name = 'GitHubConnectError'
|
||||
}
|
||||
}
|
||||
|
||||
export function authorizeUrl(config: GitHubConnectConfig, origin: string, state: string): string {
|
||||
const url = new URL('https://github.com/login/oauth/authorize')
|
||||
url.searchParams.set('client_id', config.clientId)
|
||||
url.searchParams.set('redirect_uri', `${origin}/v1/github/callback`)
|
||||
url.searchParams.set('state', state)
|
||||
url.searchParams.set('allow_signup', 'false')
|
||||
return url.toString()
|
||||
}
|
||||
|
||||
export function installUrl(config: GitHubConnectConfig, state: string): string {
|
||||
return `https://github.com/apps/${encodeURIComponent(config.appSlug)}/installations/new?state=${encodeURIComponent(state)}`
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup URL (after install) and OAuth callback. Without a code we send the browser through OAuth so
|
||||
* GitHub proves who the installer is; the installation id is carried inside a re-signed state.
|
||||
*/
|
||||
export async function handleGitHubReturn(
|
||||
db: D1Database,
|
||||
config: GitHubConnectConfig,
|
||||
url: URL,
|
||||
now: number
|
||||
): Promise<{ redirect: string } | { bound: string[]; denied: number } | { error: string }> {
|
||||
const rawState = url.searchParams.get('state') ?? ''
|
||||
const state = rawState ? await verifyState(config.stateSecret, rawState, now) : null
|
||||
if (!state) {
|
||||
return { error: 'invalid_state' }
|
||||
}
|
||||
const queryInstallation = Number(url.searchParams.get('installation_id') ?? NaN)
|
||||
const installationId = state.i ?? (Number.isSafeInteger(queryInstallation) && queryInstallation > 0 ? queryInstallation : undefined)
|
||||
const code = url.searchParams.get('code') ?? ''
|
||||
if (!/^[A-Za-z0-9_-]{8,128}$/.test(code)) {
|
||||
// Carry the (still untrusted) installation id through OAuth; verification happens on return.
|
||||
const carried = await signState(config.stateSecret, { ...state, ...(installationId ? { i: installationId } : {}) })
|
||||
return { redirect: authorizeUrl(config, url.origin, carried) }
|
||||
}
|
||||
if (!(await consumeState(db, state, now))) {
|
||||
return { error: 'state_used' }
|
||||
}
|
||||
const result = await verifiedInstallations(config, code, installationId)
|
||||
for (const installation of result.allowed) {
|
||||
await bindInstallation(
|
||||
db,
|
||||
{
|
||||
installation_id: installation.id,
|
||||
account_id: state.a,
|
||||
github_account_id: installation.account.id,
|
||||
github_account_login: installation.account.login,
|
||||
github_account_type: installation.account.type,
|
||||
bound_by_user_id: state.u,
|
||||
bound_by_github_user_id: installation.githubUserId,
|
||||
method: 'oauth'
|
||||
},
|
||||
now
|
||||
)
|
||||
}
|
||||
console.log(JSON.stringify({ event: 'github_bind', bound: result.allowed.length, denied: result.denied.length }))
|
||||
return { bound: result.allowed.map((i) => i.account.login), denied: result.denied.length }
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
// Cost guardrails in front of every start (create, resume, rollover): the operator kill switch, a
|
||||
// global running cap, a per-account concurrency backstop (billing enforces the plan limit too) and a
|
||||
// per-account create rate limit. All are control-plane local, so a billing outage cannot lift them.
|
||||
import { ACTIVE_STATUSES } from './store'
|
||||
|
||||
export type GuardrailConfig = {
|
||||
/** Static kill switch from the deploy config (`SANDBOX_CREATES_DISABLED=true`). */
|
||||
startsDisabled: boolean
|
||||
maxRunning: number
|
||||
maxActivePerSeat: number
|
||||
createsPerMinute: number
|
||||
}
|
||||
|
||||
export const DEFAULT_GUARDRAILS: GuardrailConfig = {
|
||||
startsDisabled: false,
|
||||
maxRunning: 50,
|
||||
maxActivePerSeat: 3,
|
||||
createsPerMinute: 10
|
||||
}
|
||||
|
||||
export class GuardrailError extends Error {
|
||||
constructor(
|
||||
readonly code: 'maintenance' | 'capacity' | 'rate_limited' | 'concurrency_limit',
|
||||
readonly retryAfterSeconds?: number
|
||||
) {
|
||||
super(code)
|
||||
this.name = 'GuardrailError'
|
||||
}
|
||||
}
|
||||
|
||||
const KILL_SWITCH = 'kill_switch'
|
||||
|
||||
export async function getControlState(db: D1Database, name: string): Promise<string | null> {
|
||||
const row = await db.prepare('SELECT value FROM control_state WHERE name = ?').bind(name).first<{ value: string }>()
|
||||
return row?.value ?? null
|
||||
}
|
||||
|
||||
export async function setControlState(db: D1Database, name: string, value: string, now: number): Promise<void> {
|
||||
await db
|
||||
.prepare(
|
||||
`INSERT INTO control_state (name, value, updated_at) VALUES (?, ?, ?)
|
||||
ON CONFLICT (name) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at`
|
||||
)
|
||||
.bind(name, value, now)
|
||||
.run()
|
||||
}
|
||||
|
||||
export async function killSwitchOn(db: D1Database, config: GuardrailConfig): Promise<boolean> {
|
||||
return config.startsDisabled || (await getControlState(db, KILL_SWITCH)) === 'on'
|
||||
}
|
||||
|
||||
export function setKillSwitch(db: D1Database, on: boolean, now: number): Promise<void> {
|
||||
return setControlState(db, KILL_SWITCH, on ? 'on' : 'off', now)
|
||||
}
|
||||
|
||||
async function countActive(db: D1Database, accountId?: string): Promise<number> {
|
||||
const placeholders = ACTIVE_STATUSES.map(() => '?').join(', ')
|
||||
const row = await db
|
||||
.prepare(
|
||||
`SELECT COUNT(*) AS n FROM sandboxes WHERE status IN (${placeholders}) ${accountId ? 'AND account_id = ?' : ''}`
|
||||
)
|
||||
.bind(...ACTIVE_STATUSES, ...(accountId ? [accountId] : []))
|
||||
.first<{ n: number }>()
|
||||
return row?.n ?? 0
|
||||
}
|
||||
|
||||
export function countAllActive(db: D1Database): Promise<number> {
|
||||
return countActive(db)
|
||||
}
|
||||
|
||||
/** Kill switch + global running cap; checked before billing is asked anything. */
|
||||
export async function assertStartsOpen(db: D1Database, config: GuardrailConfig): Promise<void> {
|
||||
if (await killSwitchOn(db, config)) {
|
||||
throw new GuardrailError('maintenance')
|
||||
}
|
||||
if ((await countAllActive(db)) >= config.maxRunning) {
|
||||
console.log(JSON.stringify({ event: 'global_cap_reached', maxRunning: config.maxRunning }))
|
||||
throw new GuardrailError('capacity', 60)
|
||||
}
|
||||
}
|
||||
|
||||
/** Per-account backstop (3/seat); `seats` comes from billing when it reports one, else 1. */
|
||||
export async function assertAccountCapacity(
|
||||
db: D1Database,
|
||||
config: GuardrailConfig,
|
||||
accountId: string,
|
||||
seats = 1
|
||||
): Promise<void> {
|
||||
if ((await countActive(db, accountId)) >= config.maxActivePerSeat * Math.max(1, Math.floor(seats))) {
|
||||
throw new GuardrailError('concurrency_limit')
|
||||
}
|
||||
}
|
||||
|
||||
/** Fixed one-minute window per account; counts every create request, retries included. */
|
||||
export async function consumeCreateToken(db: D1Database, config: GuardrailConfig, accountId: string, now: number) {
|
||||
const windowStart = now - (now % 60_000)
|
||||
const bucket = `create:${accountId}`
|
||||
await db
|
||||
.prepare(
|
||||
`INSERT INTO rate_windows (bucket, window_start, count) VALUES (?, ?, 1)
|
||||
ON CONFLICT (bucket) DO UPDATE SET
|
||||
count = CASE WHEN rate_windows.window_start = excluded.window_start THEN rate_windows.count + 1 ELSE 1 END,
|
||||
window_start = excluded.window_start`
|
||||
)
|
||||
.bind(bucket, windowStart)
|
||||
.run()
|
||||
const row = await db.prepare('SELECT count FROM rate_windows WHERE bucket = ?').bind(bucket).first<{ count: number }>()
|
||||
if ((row?.count ?? 0) > config.createsPerMinute) {
|
||||
throw new GuardrailError('rate_limited', Math.ceil((windowStart + 60_000 - now) / 1000))
|
||||
}
|
||||
}
|
||||
|
||||
export function guardrailsFromEnv(env: {
|
||||
SANDBOX_CREATES_DISABLED?: string
|
||||
SANDBOX_MAX_RUNNING?: string
|
||||
ACCOUNT_MAX_ACTIVE_PER_SEAT?: string
|
||||
CREATE_RATE_PER_MINUTE?: string
|
||||
}): GuardrailConfig {
|
||||
const int = (value: string | undefined, fallback: number) => {
|
||||
const parsed = Number(value)
|
||||
return value !== undefined && Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : fallback
|
||||
}
|
||||
return {
|
||||
startsDisabled: env.SANDBOX_CREATES_DISABLED === 'true',
|
||||
maxRunning: int(env.SANDBOX_MAX_RUNNING, DEFAULT_GUARDRAILS.maxRunning),
|
||||
maxActivePerSeat: int(env.ACCOUNT_MAX_ACTIVE_PER_SEAT, DEFAULT_GUARDRAILS.maxActivePerSeat),
|
||||
createsPerMinute: int(env.CREATE_RATE_PER_MINUTE, DEFAULT_GUARDRAILS.createsPerMinute)
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,10 @@
|
||||
// POST /v1/sandboxes/:id/lease {extendMs} -> {sandbox}
|
||||
// DELETE /v1/sandboxes/:id -> {sandbox}
|
||||
// GET /v1/secrets?scope= | PUT, DELETE /v1/secrets/:scope/:NAME (see src/secrets.ts)
|
||||
// POST /v1/github/connect -> {installUrl, authorizeUrl} (bind a GitHub App installation to the account)
|
||||
// GET /v1/github/installations -> {installations}
|
||||
// Browser (GitHub redirects): GET /v1/github/setup, GET /v1/github/callback
|
||||
// Operator (OPERATOR_TOKEN): /v1/internal/* (see operator.ts)
|
||||
// Vercel network-policy forwardURL target (Vercel sandbox OIDC): /v1/egress/proxy
|
||||
import { authenticateDesktop } from './auth'
|
||||
import { NO_CREDENTIALS, handleEgressProxy, type CredentialBroker, type ProxiedRoute } from './egress-proxy'
|
||||
@@ -29,6 +33,19 @@ import {
|
||||
type LifecycleDeps
|
||||
} from './lifecycle'
|
||||
import { createGitHubAppMinter, type GitHubAppMinter } from './github-app'
|
||||
import {
|
||||
GitHubConnectError,
|
||||
authorizeUrl,
|
||||
createConnectState,
|
||||
handleGitHubReturn,
|
||||
installUrl,
|
||||
isInstallationBound,
|
||||
listAccountInstallations,
|
||||
type GitHubConnectConfig
|
||||
} from './github-connect'
|
||||
import { GuardrailError, consumeCreateToken, guardrailsFromEnv } from './guardrails'
|
||||
import { handleOperator, isOperator } from './operator'
|
||||
import { DEFAULT_RETENTION, type RetentionConfig } from './retention'
|
||||
import { reconcile } from './reconcile'
|
||||
import { egressSecretHosts, secretPlaceholder } from './secret-egress'
|
||||
import { SecretError, handleSecretsRoute, keyringFor, resolveSandboxSecrets } from './secrets'
|
||||
@@ -49,6 +66,9 @@ export type Deps = {
|
||||
now?: () => number
|
||||
egressFetch?: typeof fetch
|
||||
egressKeysForIssuer?: Parameters<typeof handleEgressProxy>[1]['keysForIssuer']
|
||||
/** GitHub OAuth/user API fetch (tests). */
|
||||
githubFetch?: typeof fetch
|
||||
retention?: RetentionConfig
|
||||
}
|
||||
|
||||
// Decrypted model keys for the egress proxy, per sandbox session, for a minute: one D1 read and
|
||||
@@ -95,17 +115,52 @@ function egressRoutes(env: Env, deps: Deps): (row: SandboxRow, host: string) =>
|
||||
}
|
||||
}
|
||||
|
||||
const minters = new Map<string, GitHubAppMinter>()
|
||||
const minters = new Map<string, { minter: GitHubAppMinter; db: { current: D1Database } }>()
|
||||
function githubMinter(env: Env): GitHubAppMinter | undefined {
|
||||
if (!env.GITHUB_APP_ID || !env.GITHUB_APP_PRIVATE_KEY) {
|
||||
return undefined
|
||||
}
|
||||
let minter = minters.get(env.GITHUB_APP_ID)
|
||||
if (!minter) {
|
||||
minter = createGitHubAppMinter({ appId: env.GITHUB_APP_ID, privateKeyPkcs8: env.GITHUB_APP_PRIVATE_KEY })
|
||||
minters.set(env.GITHUB_APP_ID, minter)
|
||||
let entry = minters.get(env.GITHUB_APP_ID)
|
||||
if (!entry) {
|
||||
const db = { current: env.SANDBOX_DB }
|
||||
entry = {
|
||||
db,
|
||||
minter: createGitHubAppMinter({
|
||||
appId: env.GITHUB_APP_ID,
|
||||
privateKeyPkcs8: env.GITHUB_APP_PRIVATE_KEY,
|
||||
isBound: (installationId, accountId) => isInstallationBound(db.current, installationId, accountId)
|
||||
})
|
||||
}
|
||||
minters.set(env.GITHUB_APP_ID, entry)
|
||||
}
|
||||
entry.db.current = env.SANDBOX_DB
|
||||
return entry.minter
|
||||
}
|
||||
|
||||
function githubConnect(env: Env, deps: Deps): GitHubConnectConfig | null {
|
||||
if (!env.GITHUB_APP_SLUG || !env.GITHUB_APP_CLIENT_ID || !env.GITHUB_APP_CLIENT_SECRET || (env.GITHUB_STATE_SECRET?.length ?? 0) < 32) {
|
||||
return null
|
||||
}
|
||||
return {
|
||||
appSlug: env.GITHUB_APP_SLUG,
|
||||
clientId: env.GITHUB_APP_CLIENT_ID,
|
||||
clientSecret: env.GITHUB_APP_CLIENT_SECRET,
|
||||
stateSecret: env.GITHUB_STATE_SECRET!,
|
||||
fetch: deps.githubFetch
|
||||
}
|
||||
}
|
||||
|
||||
function retentionFromEnv(env: Env): RetentionConfig {
|
||||
const num = (value: string | undefined, fallback: number, unitMs: number) => {
|
||||
const parsed = Number(value)
|
||||
return value !== undefined && Number.isFinite(parsed) && parsed > 0 ? parsed * unitMs : fallback
|
||||
}
|
||||
return {
|
||||
...DEFAULT_RETENTION,
|
||||
dataRetentionMs: num(env.DATA_RETENTION_DAYS, DEFAULT_RETENTION.dataRetentionMs, 86_400_000),
|
||||
idleRetentionMs: num(env.IDLE_RETENTION_DAYS, DEFAULT_RETENTION.idleRetentionMs, 86_400_000),
|
||||
orphanGraceMs: num(env.ORPHAN_GRACE_HOURS, DEFAULT_RETENTION.orphanGraceMs, 3_600_000)
|
||||
}
|
||||
return minter
|
||||
}
|
||||
|
||||
function egressCredentials(env: Env): CredentialBroker {
|
||||
@@ -118,7 +173,7 @@ function egressCredentials(env: Env): CredentialBroker {
|
||||
if (host !== 'api.github.com' || !row.repo_url) {
|
||||
return null
|
||||
}
|
||||
const minted = await minter.tokenForRepo(row.repo_url)
|
||||
const minted = await minter.tokenForRepo(row.repo_url, row.account_id)
|
||||
return minted ? `Bearer ${minted.token}` : null
|
||||
}
|
||||
}
|
||||
@@ -146,8 +201,11 @@ function githubManifestStart(url: URL): Response {
|
||||
// Installable on personal accounts too; installing grants nothing until a repo is selected.
|
||||
public: true,
|
||||
hook_attributes: { url: `${origin}/v1/github/webhook-unused`, active: false },
|
||||
default_permissions: { contents: 'write', metadata: 'read', pull_requests: 'write' },
|
||||
// members:read lets the bind step confirm an org installer is an org admin (user-to-server token).
|
||||
default_permissions: { contents: 'write', metadata: 'read', pull_requests: 'write', members: 'read' },
|
||||
default_events: [],
|
||||
// The setup URL sends the browser through OAuth itself, so existing installs can be bound too.
|
||||
callback_urls: [`${origin}/v1/github/callback`],
|
||||
request_oauth_on_install: false
|
||||
}
|
||||
const state = crypto.randomUUID()
|
||||
@@ -221,7 +279,13 @@ async function lifecycleDeps(env: Env, deps: Deps): Promise<LifecycleDeps | null
|
||||
github: deps.github ?? githubMinter(env),
|
||||
secrets: await secretsDeps(env),
|
||||
now: deps.now ?? Date.now,
|
||||
retention: deps.retention ?? retentionFromEnv(env),
|
||||
config: {
|
||||
guardrails: guardrailsFromEnv(env),
|
||||
snapshotExpirationMs:
|
||||
env.SNAPSHOT_EXPIRATION_DAYS && Number(env.SNAPSHOT_EXPIRATION_DAYS) >= 1
|
||||
? Math.floor(Number(env.SNAPSHOT_EXPIRATION_DAYS)) * 86_400_000
|
||||
: undefined,
|
||||
environment: env.SANDBOX_ENV,
|
||||
goldenSnapshotId: env.GOLDEN_SNAPSHOT_ID,
|
||||
defaultRegion: env.VERCEL_REGION ?? 'iad1',
|
||||
@@ -268,8 +332,45 @@ export async function handleRequest(request: Request, env: Env, deps: Deps = {})
|
||||
if (url.pathname === '/v1/github/manifest-start' && env.SANDBOX_ENV !== 'prod') {
|
||||
return githubManifestStart(url)
|
||||
}
|
||||
if (url.pathname === '/v1/github/setup') {
|
||||
return html('Orca Cloud (dev) is installed. You can close this tab.')
|
||||
if (url.pathname === '/v1/github/setup' || url.pathname === '/v1/github/callback') {
|
||||
return await githubReturn(env, deps, url)
|
||||
}
|
||||
if (url.pathname.startsWith('/v1/internal/')) {
|
||||
if (!(await isOperator(request, env.OPERATOR_TOKEN))) {
|
||||
return fail('not_found', 404)
|
||||
}
|
||||
const life = await lifecycleDeps(env, deps)
|
||||
if (!life) {
|
||||
return fail('not_configured', 503)
|
||||
}
|
||||
return (
|
||||
(await handleOperator(request, url.pathname, {
|
||||
life,
|
||||
github: deps.github ?? githubMinter(env),
|
||||
readJson: () => readJson(request)
|
||||
})) ?? fail('not_found', 404)
|
||||
)
|
||||
}
|
||||
if (url.pathname === '/v1/github/connect' || url.pathname === '/v1/github/installations') {
|
||||
const caller = await authenticateDesktop(env, request, (deps.now ?? Date.now)())
|
||||
if (!caller) {
|
||||
return fail('unauthorized', 401)
|
||||
}
|
||||
if (url.pathname === '/v1/github/installations' && request.method === 'GET') {
|
||||
const rows = await listAccountInstallations(env.SANDBOX_DB, caller.accountId)
|
||||
return json({
|
||||
installations: rows.map((r) => ({ installationId: r.installation_id, account: r.github_account_login, accountType: r.github_account_type, boundAt: r.created_at }))
|
||||
})
|
||||
}
|
||||
const connect = githubConnect(env, deps)
|
||||
if (url.pathname !== '/v1/github/connect' || request.method !== 'POST') {
|
||||
return fail('not_found', 404)
|
||||
}
|
||||
if (!connect) {
|
||||
return fail('not_configured', 503)
|
||||
}
|
||||
const state = await createConnectState(env.SANDBOX_DB, connect.stateSecret, caller, (deps.now ?? Date.now)())
|
||||
return json({ installUrl: installUrl(connect, state), authorizeUrl: authorizeUrl(connect, url.origin, state) })
|
||||
}
|
||||
if (url.pathname === '/v1/secrets' || url.pathname.startsWith('/v1/secrets/')) {
|
||||
const caller = await authenticateDesktop(env, request, (deps.now ?? Date.now)())
|
||||
@@ -303,6 +404,7 @@ export async function handleRequest(request: Request, env: Env, deps: Deps = {})
|
||||
if (request.method !== 'POST') {
|
||||
return fail('not_found', 404)
|
||||
}
|
||||
await consumeCreateToken(life.db, life.config.guardrails!, caller.accountId, life.now())
|
||||
const body = CreateRequestSchema.safeParse(await readJson(request))
|
||||
if (!body.success) {
|
||||
return fail('invalid_request', 400)
|
||||
@@ -346,6 +448,38 @@ export async function handleRequest(request: Request, env: Env, deps: Deps = {})
|
||||
}
|
||||
}
|
||||
|
||||
async function githubReturn(env: Env, deps: Deps, url: URL): Promise<Response> {
|
||||
const connect = githubConnect(env, deps)
|
||||
if (!connect) {
|
||||
// Not configured for binding: an install alone grants nothing.
|
||||
return html('Orca Cloud is installed on GitHub. Connect it from Orca to use it with your workspaces.')
|
||||
}
|
||||
try {
|
||||
const result = await handleGitHubReturn(env.SANDBOX_DB, connect, url, (deps.now ?? Date.now)())
|
||||
if ('redirect' in result) {
|
||||
return new Response(null, { status: 302, headers: { location: result.redirect, 'cache-control': 'no-store' } })
|
||||
}
|
||||
if ('error' in result) {
|
||||
return html(
|
||||
result.error === 'state_used'
|
||||
? 'This link was already used. Start again from Orca.'
|
||||
: 'GitHub is installed, but this page was not opened from Orca. Start from Orca to connect it to your account.'
|
||||
)
|
||||
}
|
||||
const esc = (value: string) => value.replace(/[^A-Za-z0-9_.-]/g, '')
|
||||
return html(
|
||||
result.bound.length > 0
|
||||
? `Connected GitHub ${result.bound.map((login) => `<b>${esc(login)}</b>`).join(', ')} to Orca. You can close this tab.`
|
||||
: 'Nothing was connected: only the owner of a personal account or an organization admin can connect an installation.'
|
||||
)
|
||||
} catch (error) {
|
||||
if (error instanceof GitHubConnectError) {
|
||||
return html('GitHub did not confirm the sign-in. Start again from Orca.')
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
function errorResponse(error: unknown): Response {
|
||||
if (error instanceof SyntaxError) {
|
||||
return fail('invalid_json', 400)
|
||||
@@ -353,6 +487,17 @@ function errorResponse(error: unknown): Response {
|
||||
if (error instanceof SecretError) {
|
||||
return fail(error.code, error.status, error.reason)
|
||||
}
|
||||
if (error instanceof GuardrailError) {
|
||||
if (error.code === 'concurrency_limit') {
|
||||
return fail('entitlement_denied', 403, 'concurrency_limit')
|
||||
}
|
||||
const status = { maintenance: 503, capacity: 503, rate_limited: 429 }[error.code]
|
||||
const response = fail(error.code, status)
|
||||
if (error.retryAfterSeconds) {
|
||||
response.headers.set('retry-after', String(error.retryAfterSeconds))
|
||||
}
|
||||
return response
|
||||
}
|
||||
if (error instanceof LifecycleError) {
|
||||
const status = { not_found: 404, conflict: 409, entitlement_denied: 403, provider_unavailable: 502 }[error.code]
|
||||
return fail(error.code, status, error.reason)
|
||||
|
||||
@@ -2,7 +2,16 @@
|
||||
// a forced stop before the provider's 24h cap, usage metering and write-ahead rows.
|
||||
import type { EntitlementClient, MachineSize } from './entitlements'
|
||||
import type { GitHubAppMinter } from './github-app'
|
||||
import {
|
||||
DEFAULT_GUARDRAILS,
|
||||
GuardrailError,
|
||||
assertAccountCapacity,
|
||||
assertStartsOpen,
|
||||
killSwitchOn,
|
||||
type GuardrailConfig
|
||||
} from './guardrails'
|
||||
import { compileNetworkPolicy, gitBasicAuthorization, type CredentialRoute } from './network-policy'
|
||||
import type { RetentionConfig } from './retention'
|
||||
import type { ProviderSandboxState, SandboxProviderApi } from './sandbox-provider'
|
||||
import { applySecretPlaceholders, egressSecretHosts, secretPlaceholder } from './secret-egress'
|
||||
import { resolveSandboxSecrets } from './secrets'
|
||||
@@ -29,6 +38,9 @@ const PROVIDER_SESSION_CAP_MS = 24 * 60 * 60_000
|
||||
/** Orca stops 10 min early so it, not the provider, ends the session (drain first). */
|
||||
export const SESSION_DEADLINE_MS = PROVIDER_SESSION_CAP_MS - 10 * 60_000
|
||||
const VCPUS: Record<MachineSize, number> = { standard: 4, large: 8 }
|
||||
const DAY_MS = 24 * 60 * 60_000
|
||||
/** Provider-side backstop: longer than idle retention (90d) + its warning, so Orca deletes first. */
|
||||
export const DEFAULT_SNAPSHOT_EXPIRATION_MS = 104 * DAY_MS
|
||||
|
||||
export type LifecycleDeps = {
|
||||
db: D1Database
|
||||
@@ -39,6 +51,8 @@ export type LifecycleDeps = {
|
||||
/** Secret store; absent (no SECRETS_KEK) means sessions start without injected secrets. */
|
||||
secrets?: { keyring: Keyring; egressPlaceholders: boolean }
|
||||
now: () => number
|
||||
/** Orphan/retention policy for the cron; defaults in retention.ts. */
|
||||
retention?: RetentionConfig
|
||||
config: {
|
||||
environment: string
|
||||
goldenSnapshotId: string
|
||||
@@ -46,6 +60,8 @@ export type LifecycleDeps = {
|
||||
egressProxyUrl?: string
|
||||
/** Staging rehearsals shorten the cap; production uses SESSION_DEADLINE_MS. */
|
||||
sessionDeadlineMs?: number
|
||||
guardrails?: GuardrailConfig
|
||||
snapshotExpirationMs?: number
|
||||
}
|
||||
}
|
||||
|
||||
@@ -136,7 +152,12 @@ async function claim(deps: LifecycleDeps, row: SandboxRow, patch: Partial<Sandbo
|
||||
return next
|
||||
}
|
||||
|
||||
export function guardrails(deps: LifecycleDeps): GuardrailConfig {
|
||||
return deps.config.guardrails ?? DEFAULT_GUARDRAILS
|
||||
}
|
||||
|
||||
async function requireStart(deps: LifecycleDeps, accountId: string, machineSize: MachineSize) {
|
||||
await assertStartsOpen(deps.db, guardrails(deps))
|
||||
const runningSandboxes = await countActiveSandboxes(deps.db, accountId)
|
||||
const decision = await deps.entitlements.checkStart({ accountId, machineSize, runningSandboxes })
|
||||
if (decision.suspendRunning) {
|
||||
@@ -145,6 +166,7 @@ async function requireStart(deps: LifecycleDeps, accountId: string, machineSize:
|
||||
if (!decision.allow) {
|
||||
throw new LifecycleError('entitlement_denied', decision.reason ?? 'denied')
|
||||
}
|
||||
await assertAccountCapacity(deps.db, guardrails(deps), accountId, decision.seats)
|
||||
}
|
||||
|
||||
async function boot(deps: LifecycleDeps, row: SandboxRow, state: ProviderSandboxState, repo?: { url: string; ref?: string }): Promise<Ready> {
|
||||
@@ -168,6 +190,7 @@ async function boot(deps: LifecycleDeps, row: SandboxRow, state: ProviderSandbox
|
||||
const running = await claim(deps, row, {
|
||||
status: 'running',
|
||||
last_error: null,
|
||||
retention_warned_at: null,
|
||||
github_token_expires_at: git?.expiresAt ?? null,
|
||||
egress_secret_hosts: routed.egress_secret_hosts,
|
||||
...sessionFields(deps, state)
|
||||
@@ -201,7 +224,7 @@ export async function refreshGitCredentials(
|
||||
if (!deps.github || !row.repo_url) {
|
||||
return null
|
||||
}
|
||||
const minted = await deps.github.tokenForRepo(row.repo_url)
|
||||
const minted = await deps.github.tokenForRepo(row.repo_url, row.account_id)
|
||||
if (!minted) {
|
||||
return null
|
||||
}
|
||||
@@ -253,6 +276,7 @@ export async function createSandbox(
|
||||
timeoutMs: LEASE_MS,
|
||||
ports: [ORCAD_PORT],
|
||||
networkPolicy: egressPolicy(deps, row),
|
||||
snapshotExpirationMs: deps.config.snapshotExpirationMs ?? DEFAULT_SNAPSHOT_EXPIRATION_MS,
|
||||
tags: { orcaSandbox: row.id, orcaEnv: deps.config.environment }
|
||||
})
|
||||
} catch (error) {
|
||||
@@ -366,6 +390,9 @@ export async function rolloverSandbox(deps: LifecycleDeps, row: SandboxRow): Pro
|
||||
if (row.status !== 'running') {
|
||||
throw new LifecycleError('conflict', `sandbox_${row.status}`)
|
||||
}
|
||||
if (await killSwitchOn(deps.db, guardrails(deps))) {
|
||||
throw new GuardrailError('maintenance')
|
||||
}
|
||||
const suspending = await claim(deps, row, { status: 'suspending' })
|
||||
const stopped = await stopForSuspend(deps, suspending)
|
||||
await flushUsage(deps.db, deps.entitlements, deps.now(), row.id)
|
||||
@@ -410,7 +437,13 @@ export async function destroySandbox(deps: LifecycleDeps, row: SandboxRow): Prom
|
||||
throw new LifecycleError('provider_unavailable', 'snapshot_cleanup_incomplete')
|
||||
}
|
||||
await markSnapshotsDeleted(deps.db, row.id, deps.now())
|
||||
const destroyed = await claim(deps, destroying, { status: 'destroyed', ...NO_SESSION, destroyed_at: deps.now() })
|
||||
const destroyed = await claim(deps, destroying, {
|
||||
status: 'destroyed',
|
||||
...NO_SESSION,
|
||||
destroyed_at: deps.now(),
|
||||
// Tombstone: the row stays for usage/audit, the workspace's repository does not.
|
||||
repo_url: null
|
||||
})
|
||||
await flushUsage(deps.db, deps.entitlements, deps.now(), row.id)
|
||||
return destroyed
|
||||
}
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
// Operator-only routes (Bearer OPERATOR_TOKEN, a Worker secret; absent = routes do not exist):
|
||||
// GET /v1/internal/status
|
||||
// POST /v1/internal/kill-switch {enabled, drain?} refuse creates/resumes; drain suspends (snapshot kept)
|
||||
// POST /v1/internal/drain-all
|
||||
// POST /v1/internal/housekeeping orphan sweep + retention now
|
||||
// POST /v1/internal/github/bindings {installationId, accountId} support binding (audited as `operator`)
|
||||
// DELETE /v1/internal/github/bindings {installationId, accountId?}
|
||||
import { z } from 'zod'
|
||||
import { readBearer } from './auth'
|
||||
import type { GitHubAppMinter } from './github-app'
|
||||
import { bindInstallation, revokeInstallation, timingSafeEqual } from './github-connect'
|
||||
import { countAllActive, getControlState, killSwitchOn, setKillSwitch } from './guardrails'
|
||||
import { guardrails, type LifecycleDeps } from './lifecycle'
|
||||
import { drainAll, housekeeping } from './retention'
|
||||
import { listByStatus } from './store'
|
||||
|
||||
const KillSwitchBody = z.object({ enabled: z.boolean(), drain: z.boolean().optional() }).strict()
|
||||
const AccountId = z.string().min(1).max(160).regex(/^(user|org|dev):[A-Za-z0-9_.:-]+$/)
|
||||
const BindingBody = z
|
||||
.object({
|
||||
installationId: z.number().int().positive().optional(),
|
||||
/** Alternative to installationId: the installation covering this github.com repo. */
|
||||
repoUrl: z.url({ protocol: /^https$/ }).max(2048).optional(),
|
||||
accountId: AccountId
|
||||
})
|
||||
.strict()
|
||||
.refine((body) => (body.installationId === undefined) !== (body.repoUrl === undefined))
|
||||
const UnbindBody = z.object({ installationId: z.number().int().positive(), accountId: AccountId.optional() }).strict()
|
||||
|
||||
async function digest(value: string): Promise<Uint8Array> {
|
||||
return new Uint8Array(await crypto.subtle.digest('SHA-256', new TextEncoder().encode(value)))
|
||||
}
|
||||
|
||||
export async function isOperator(request: Request, operatorToken: string | undefined): Promise<boolean> {
|
||||
const presented = readBearer(request.headers.get('authorization'))
|
||||
if (!operatorToken || operatorToken.length < 32 || !presented) {
|
||||
return false
|
||||
}
|
||||
return timingSafeEqual(await digest(presented), await digest(operatorToken))
|
||||
}
|
||||
|
||||
export async function handleOperator(
|
||||
request: Request,
|
||||
path: string,
|
||||
deps: { life: LifecycleDeps; github?: GitHubAppMinter; readJson: () => Promise<unknown> }
|
||||
): Promise<Response | null> {
|
||||
const { life } = deps
|
||||
const now = life.now()
|
||||
const route = `${request.method} ${path}`
|
||||
switch (route) {
|
||||
case 'GET /v1/internal/status':
|
||||
return Response.json({
|
||||
killSwitch: await killSwitchOn(life.db, guardrails(life)),
|
||||
staticDisabled: guardrails(life).startsDisabled,
|
||||
active: await countAllActive(life.db),
|
||||
running: (await listByStatus(life.db, ['running'])).length,
|
||||
guardrails: guardrails(life),
|
||||
lastHousekeepingAt: Number((await getControlState(life.db, 'housekeeping_at')) ?? 0) || null
|
||||
})
|
||||
case 'POST /v1/internal/kill-switch': {
|
||||
const body = KillSwitchBody.safeParse(await deps.readJson())
|
||||
if (!body.success) {
|
||||
return Response.json({ error: 'invalid_request' }, { status: 400 })
|
||||
}
|
||||
await setKillSwitch(life.db, body.data.enabled, now)
|
||||
console.log(JSON.stringify({ event: 'kill_switch', enabled: body.data.enabled }))
|
||||
const drained = body.data.enabled && body.data.drain !== false ? await drainAll(life) : null
|
||||
return Response.json({ killSwitch: body.data.enabled, drained })
|
||||
}
|
||||
case 'POST /v1/internal/drain-all':
|
||||
return Response.json({ drained: await drainAll(life) })
|
||||
case 'POST /v1/internal/housekeeping':
|
||||
return Response.json({ housekeeping: await housekeeping(life, { force: true }) })
|
||||
case 'POST /v1/internal/github/bindings': {
|
||||
const body = BindingBody.safeParse(await deps.readJson())
|
||||
if (!body.success) {
|
||||
return Response.json({ error: 'invalid_request' }, { status: 400 })
|
||||
}
|
||||
if (!deps.github) {
|
||||
return Response.json({ error: 'not_configured' }, { status: 503 })
|
||||
}
|
||||
const installation = body.data.installationId
|
||||
? await deps.github.getInstallation(body.data.installationId)
|
||||
: await deps.github.getRepoInstallation(body.data.repoUrl!)
|
||||
if (!installation) {
|
||||
return Response.json({ error: 'not_found' }, { status: 404 })
|
||||
}
|
||||
await bindInstallation(
|
||||
life.db,
|
||||
{
|
||||
installation_id: installation.id,
|
||||
account_id: body.data.accountId,
|
||||
github_account_id: installation.account.id,
|
||||
github_account_login: installation.account.login,
|
||||
github_account_type: installation.account.type,
|
||||
bound_by_user_id: 'operator',
|
||||
bound_by_github_user_id: null,
|
||||
method: 'operator'
|
||||
},
|
||||
now
|
||||
)
|
||||
console.log(JSON.stringify({ event: 'github_bind_operator', installationId: installation.id, accountId: body.data.accountId }))
|
||||
return Response.json({ bound: { installationId: installation.id, accountId: body.data.accountId, githubAccount: installation.account.login } })
|
||||
}
|
||||
case 'DELETE /v1/internal/github/bindings': {
|
||||
const body = UnbindBody.safeParse(await deps.readJson())
|
||||
if (!body.success) {
|
||||
return Response.json({ error: 'invalid_request' }, { status: 400 })
|
||||
}
|
||||
const revoked = await revokeInstallation(life.db, body.data.installationId, body.data.accountId ?? null, now)
|
||||
console.log(JSON.stringify({ event: 'github_unbind_operator', installationId: body.data.installationId, revoked }))
|
||||
return Response.json({ revoked })
|
||||
}
|
||||
default:
|
||||
return null
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,9 @@ import {
|
||||
suspendSandbox,
|
||||
type LifecycleDeps
|
||||
} from './lifecycle'
|
||||
import { guardrails } from './lifecycle'
|
||||
import { killSwitchOn } from './guardrails'
|
||||
import { drainAll, housekeeping, openHold, type HousekeepingSummary } from './retention'
|
||||
import { listByStatus, transition, type SandboxRow } from './store'
|
||||
import { flushUsage, queueUsage } from './usage'
|
||||
|
||||
@@ -19,6 +22,7 @@ const STUCK_MS = 15 * 60_000
|
||||
const FORCED_STOP_GRACE_MS = 5 * 60_000
|
||||
|
||||
export type ReconcileSummary = Record<
|
||||
| 'drained'
|
||||
| 'settled'
|
||||
| 'suspendedForDeadline'
|
||||
| 'suspendedForBilling'
|
||||
@@ -27,10 +31,11 @@ export type ReconcileSummary = Record<
|
||||
| 'repaired'
|
||||
| 'billingErrors',
|
||||
number
|
||||
>
|
||||
> & { housekeeping?: HousekeepingSummary }
|
||||
|
||||
export async function reconcile(deps: LifecycleDeps): Promise<ReconcileSummary> {
|
||||
const summary: ReconcileSummary = {
|
||||
drained: 0,
|
||||
settled: 0,
|
||||
suspendedForDeadline: 0,
|
||||
suspendedForBilling: 0,
|
||||
@@ -42,11 +47,19 @@ export async function reconcile(deps: LifecycleDeps): Promise<ReconcileSummary>
|
||||
const polled = await guard(async () => {
|
||||
for (const accountId of await deps.entitlements.listSuspensions()) {
|
||||
summary.suspendedForBilling += await suspendAccountRunning(deps, accountId)
|
||||
// Payment grace expired: the data-retention clock starts (no-op if already running).
|
||||
await openHold(deps.db, accountId, 'payment_grace_expired', deps.now())
|
||||
}
|
||||
})
|
||||
if (!polled) {
|
||||
summary.billingErrors++
|
||||
}
|
||||
if (await killSwitchOn(deps.db, guardrails(deps))) {
|
||||
// Kill switch on: keep draining until nothing runs (a drain call may have timed out mid-way).
|
||||
await guard(async () => {
|
||||
summary.drained += (await drainAll(deps)).suspended
|
||||
})
|
||||
}
|
||||
for (const row of await listByStatus(deps.db, ['running'])) {
|
||||
await guard(() => reconcileRunning(deps, row, summary))
|
||||
}
|
||||
@@ -69,6 +82,12 @@ export async function reconcile(deps: LifecycleDeps): Promise<ReconcileSummary>
|
||||
})
|
||||
}
|
||||
await flushUsage(deps.db, deps.entitlements, deps.now())
|
||||
await guard(async () => {
|
||||
const housekept = await housekeeping(deps)
|
||||
if (housekept.swept) {
|
||||
summary.housekeeping = housekept
|
||||
}
|
||||
})
|
||||
return summary
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,399 @@
|
||||
// Hourly cron housekeeping: provider-vs-D1 orphan sweep, data deletion after cancel/unpaid, and idle
|
||||
// retention for long-suspended sandboxes. Every destructive step is two-phase (seen, then acted on
|
||||
// after a grace period) and logged; partial provider listings never delete anything.
|
||||
import { RETENTION_HOLD_REASONS } from './entitlements'
|
||||
import { destroySandbox, suspendSandbox, type LifecycleDeps } from './lifecycle'
|
||||
import { getControlState, setControlState } from './guardrails'
|
||||
import { ACTIVE_STATUSES, getSandboxByProviderName, listByStatus, type SandboxRow } from './store'
|
||||
|
||||
const HOUR_MS = 60 * 60_000
|
||||
const DAY_MS = 24 * HOUR_MS
|
||||
|
||||
export type RetentionConfig = {
|
||||
/** Data kept this long after an account goes canceled/unpaid, then destroyed. */
|
||||
dataRetentionMs: number
|
||||
/** Suspended sandboxes untouched this long are destroyed (active subscriptions too). */
|
||||
idleRetentionMs: number
|
||||
/** Warning hook fires this long before either deletion. */
|
||||
warningLeadMs: number
|
||||
/** Orphans are suspended on first sight and deleted once seen for this long. */
|
||||
orphanGraceMs: number
|
||||
/** Housekeeping cadence inside the 5-minute cron. */
|
||||
sweepIntervalMs: number
|
||||
/** Re-ask billing about an account's standing at most this often. */
|
||||
holdProbeIntervalMs: number
|
||||
}
|
||||
|
||||
export const DEFAULT_RETENTION: RetentionConfig = {
|
||||
dataRetentionMs: 30 * DAY_MS,
|
||||
idleRetentionMs: 90 * DAY_MS,
|
||||
warningLeadMs: 7 * DAY_MS,
|
||||
orphanGraceMs: 24 * HOUR_MS,
|
||||
sweepIntervalMs: HOUR_MS,
|
||||
holdProbeIntervalMs: DAY_MS
|
||||
}
|
||||
|
||||
export type RetentionWarning = {
|
||||
kind: 'data_deletion' | 'idle_deletion'
|
||||
accountId: string
|
||||
sandboxIds: string[]
|
||||
deleteAt: number
|
||||
reason: string
|
||||
}
|
||||
|
||||
/** Email hook. Stubbed: logs a structured event the alerting Worker / mailer can pick up. */
|
||||
export type RetentionNotifier = { warn(warning: RetentionWarning): Promise<void> }
|
||||
export const LOG_NOTIFIER: RetentionNotifier = {
|
||||
async warn(warning) {
|
||||
console.log(JSON.stringify({ event: 'retention_warning', ...warning }))
|
||||
}
|
||||
}
|
||||
|
||||
export type HousekeepingSummary = {
|
||||
swept: boolean
|
||||
orphanSandboxesSuspended: number
|
||||
orphanSandboxesDeleted: number
|
||||
orphanSnapshotsSeen: number
|
||||
orphanSnapshotsDeleted: number
|
||||
rowsMissingProvider: number
|
||||
holdsOpened: number
|
||||
holdsCleared: number
|
||||
retentionWarnings: number
|
||||
retentionDestroyed: number
|
||||
idleWarnings: number
|
||||
idleDestroyed: number
|
||||
}
|
||||
|
||||
function log(event: string, fields: Record<string, unknown>) {
|
||||
console.log(JSON.stringify({ event, ...fields }))
|
||||
}
|
||||
|
||||
function retention(deps: LifecycleDeps): RetentionConfig {
|
||||
return deps.retention ?? DEFAULT_RETENTION
|
||||
}
|
||||
|
||||
// ---- account holds (canceled / unpaid) ----
|
||||
|
||||
type HoldRow = { account_id: string; reason: string; since: number; checked_at: number; warned_at: number | null; purged_at: number | null }
|
||||
|
||||
/** Records that billing says this account is canceled/unpaid; the retention clock starts at first sight. */
|
||||
export async function openHold(db: D1Database, accountId: string, reason: string, now: number): Promise<boolean> {
|
||||
const existing = await db.prepare('SELECT * FROM account_holds WHERE account_id = ?').bind(accountId).first<HoldRow>()
|
||||
if (existing && existing.reason !== 'active') {
|
||||
await db.prepare('UPDATE account_holds SET checked_at = ? WHERE account_id = ?').bind(now, accountId).run()
|
||||
return false
|
||||
}
|
||||
await db
|
||||
.prepare(
|
||||
`INSERT INTO account_holds (account_id, reason, since, checked_at) VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT (account_id) DO UPDATE SET reason = excluded.reason, since = excluded.since,
|
||||
checked_at = excluded.checked_at, warned_at = NULL, purged_at = NULL`
|
||||
)
|
||||
.bind(accountId, reason, now, now)
|
||||
.run()
|
||||
log('retention_hold_opened', { accountId, reason })
|
||||
return true
|
||||
}
|
||||
|
||||
/** Billing allows starts again (resubscribed / paid): the hold and its clock are dropped. */
|
||||
async function clearHold(db: D1Database, accountId: string, now: number): Promise<boolean> {
|
||||
const existing = await db.prepare('SELECT * FROM account_holds WHERE account_id = ?').bind(accountId).first<HoldRow>()
|
||||
await db
|
||||
.prepare(
|
||||
`INSERT INTO account_holds (account_id, reason, since, checked_at) VALUES (?, 'active', ?, ?)
|
||||
ON CONFLICT (account_id) DO UPDATE SET reason = 'active', since = excluded.since, checked_at = excluded.checked_at,
|
||||
warned_at = NULL, purged_at = NULL`
|
||||
)
|
||||
.bind(accountId, now, now)
|
||||
.run()
|
||||
return existing !== null && existing.reason !== 'active'
|
||||
}
|
||||
|
||||
/** Accounts with stored sandboxes whose standing has not been checked recently. */
|
||||
async function accountsToProbe(db: D1Database, before: number): Promise<{ account_id: string; machine_size: SandboxRow['machine_size'] }[]> {
|
||||
const { results } = await db
|
||||
.prepare(
|
||||
`SELECT s.account_id, MIN(s.machine_size) AS machine_size FROM sandboxes s
|
||||
LEFT JOIN account_holds h ON h.account_id = s.account_id
|
||||
WHERE s.status <> 'destroyed' AND (h.checked_at IS NULL OR h.checked_at < ?)
|
||||
GROUP BY s.account_id LIMIT 50`
|
||||
)
|
||||
.bind(before)
|
||||
.all<{ account_id: string; machine_size: SandboxRow['machine_size'] }>()
|
||||
return results
|
||||
}
|
||||
|
||||
async function probeHolds(deps: LifecycleDeps, summary: HousekeepingSummary): Promise<void> {
|
||||
const now = deps.now()
|
||||
for (const account of await accountsToProbe(deps.db, now - retention(deps).holdProbeIntervalMs)) {
|
||||
try {
|
||||
// runningSandboxes 0: we ask about the account's standing, not concurrency.
|
||||
const decision = await deps.entitlements.checkStart({ accountId: account.account_id, machineSize: account.machine_size, runningSandboxes: 0 })
|
||||
if (!decision.allow && RETENTION_HOLD_REASONS.has(decision.reason ?? '')) {
|
||||
summary.holdsOpened += (await openHold(deps.db, account.account_id, decision.reason!, now)) ? 1 : 0
|
||||
} else if (decision.allow) {
|
||||
summary.holdsCleared += (await clearHold(deps.db, account.account_id, now)) ? 1 : 0
|
||||
} else {
|
||||
// Other denials (hours exhausted, cap): not a cancellation; just remember we looked.
|
||||
await deps.db.prepare('UPDATE account_holds SET checked_at = ? WHERE account_id = ?').bind(now, account.account_id).run()
|
||||
}
|
||||
} catch {
|
||||
// Billing unavailable: never start a deletion clock on an outage.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function enforceHolds(deps: LifecycleDeps, notifier: RetentionNotifier, summary: HousekeepingSummary) {
|
||||
const config = retention(deps)
|
||||
const now = deps.now()
|
||||
const { results } = await deps.db
|
||||
.prepare(`SELECT * FROM account_holds WHERE reason <> 'active' AND purged_at IS NULL AND since <= ? LIMIT 100`)
|
||||
.bind(now - (config.dataRetentionMs - config.warningLeadMs))
|
||||
.all<HoldRow>()
|
||||
for (const hold of results) {
|
||||
const deleteAt = hold.since + config.dataRetentionMs
|
||||
const rows = await liveRows(deps.db, hold.account_id)
|
||||
if (hold.warned_at === null) {
|
||||
await notifier.warn({ kind: 'data_deletion', accountId: hold.account_id, sandboxIds: rows.map((r) => r.id), deleteAt, reason: hold.reason })
|
||||
await deps.db.prepare('UPDATE account_holds SET warned_at = ? WHERE account_id = ?').bind(now, hold.account_id).run()
|
||||
summary.retentionWarnings++
|
||||
continue
|
||||
}
|
||||
// Deletion waits for both the deadline and a full warning lead after the warning went out.
|
||||
if (now < deleteAt || now < hold.warned_at + config.warningLeadMs) {
|
||||
continue
|
||||
}
|
||||
let remaining = 0
|
||||
for (const row of rows) {
|
||||
try {
|
||||
await destroySandbox(deps, row)
|
||||
summary.retentionDestroyed++
|
||||
} catch {
|
||||
remaining++
|
||||
}
|
||||
}
|
||||
if (remaining === 0) {
|
||||
await deps.db.prepare('UPDATE account_holds SET purged_at = ? WHERE account_id = ?').bind(now, hold.account_id).run()
|
||||
}
|
||||
log('retention_purge', { accountId: hold.account_id, destroyed: rows.length - remaining, remaining, reason: hold.reason })
|
||||
}
|
||||
}
|
||||
|
||||
async function liveRows(db: D1Database, accountId: string): Promise<SandboxRow[]> {
|
||||
const { results } = await db
|
||||
.prepare(`SELECT * FROM sandboxes WHERE account_id = ? AND status <> 'destroyed'`)
|
||||
.bind(accountId)
|
||||
.all<SandboxRow>()
|
||||
return results
|
||||
}
|
||||
|
||||
// ---- idle retention ----
|
||||
|
||||
async function enforceIdle(deps: LifecycleDeps, notifier: RetentionNotifier, summary: HousekeepingSummary) {
|
||||
const config = retention(deps)
|
||||
const now = deps.now()
|
||||
const { results } = await deps.db
|
||||
.prepare(`SELECT * FROM sandboxes WHERE status = 'suspended' AND updated_at <= ? ORDER BY updated_at LIMIT 100`)
|
||||
.bind(now - (config.idleRetentionMs - config.warningLeadMs))
|
||||
.all<SandboxRow>()
|
||||
for (const row of results) {
|
||||
if (row.retention_warned_at === null) {
|
||||
await notifier.warn({
|
||||
kind: 'idle_deletion',
|
||||
accountId: row.account_id,
|
||||
sandboxIds: [row.id],
|
||||
deleteAt: Math.max(row.updated_at + config.idleRetentionMs, now + config.warningLeadMs),
|
||||
reason: 'idle'
|
||||
})
|
||||
// Not a transition: the idle clock (updated_at) must not move.
|
||||
await deps.db.prepare('UPDATE sandboxes SET retention_warned_at = ? WHERE id = ? AND generation = ?').bind(now, row.id, row.generation).run()
|
||||
summary.idleWarnings++
|
||||
} else if (now >= row.updated_at + config.idleRetentionMs && now >= row.retention_warned_at + config.warningLeadMs) {
|
||||
try {
|
||||
await destroySandbox(deps, row)
|
||||
summary.idleDestroyed++
|
||||
log('idle_retention_destroy', { sandboxId: row.id, accountId: row.account_id, idleDays: Math.floor((now - row.updated_at) / DAY_MS) })
|
||||
} catch {
|
||||
// Retried next sweep.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- orphan sweep ----
|
||||
|
||||
async function upsertOrphan(db: D1Database, kind: 'sandbox' | 'snapshot', id: string, now: number) {
|
||||
await db
|
||||
.prepare(
|
||||
`INSERT INTO provider_orphans (kind, provider_id, first_seen_at, last_seen_at) VALUES (?, ?, ?, ?)
|
||||
ON CONFLICT (kind, provider_id) DO UPDATE SET last_seen_at = excluded.last_seen_at`
|
||||
)
|
||||
.bind(kind, id, now, now)
|
||||
.run()
|
||||
return (await db
|
||||
.prepare('SELECT * FROM provider_orphans WHERE kind = ? AND provider_id = ?')
|
||||
.bind(kind, id)
|
||||
.first<{ first_seen_at: number; suspended_at: number | null; deleted_at: number | null }>())!
|
||||
}
|
||||
|
||||
async function clearOrphan(db: D1Database, kind: string, id: string) {
|
||||
await db.prepare('DELETE FROM provider_orphans WHERE kind = ? AND provider_id = ? AND deleted_at IS NULL').bind(kind, id).run()
|
||||
}
|
||||
|
||||
async function sweepOrphans(deps: LifecycleDeps, summary: HousekeepingSummary): Promise<void> {
|
||||
const now = deps.now()
|
||||
const grace = retention(deps).orphanGraceMs
|
||||
const env = deps.config.environment
|
||||
const listing = await deps.provider.listSandboxes()
|
||||
const providerNames = new Set(listing.sandboxes.map((s) => s.name))
|
||||
for (const box of listing.sandboxes) {
|
||||
// Only this environment's sandboxes; golden-image build boxes and other envs are not ours to judge.
|
||||
if (box.tags.orcaEnv !== env || !box.tags.orcaSandbox) {
|
||||
continue
|
||||
}
|
||||
const row = await getSandboxByProviderName(deps.db, box.name)
|
||||
if (row && row.status !== 'destroyed') {
|
||||
await clearOrphan(deps.db, 'sandbox', box.name)
|
||||
continue
|
||||
}
|
||||
const orphan = await upsertOrphan(deps.db, 'sandbox', box.name, now)
|
||||
if (orphan.deleted_at !== null) {
|
||||
continue
|
||||
}
|
||||
if (now - orphan.first_seen_at >= grace) {
|
||||
await deps.provider.deleteSandbox(box.name)
|
||||
for (const snapshot of await deps.provider.listSnapshots(box.name)) {
|
||||
if (snapshot.status === 'created' && snapshot.id !== deps.config.goldenSnapshotId) {
|
||||
await deps.provider.deleteSnapshot(snapshot.id)
|
||||
}
|
||||
}
|
||||
await deps.db.prepare(`UPDATE provider_orphans SET deleted_at = ? WHERE kind = 'sandbox' AND provider_id = ?`).bind(now, box.name).run()
|
||||
summary.orphanSandboxesDeleted++
|
||||
log('orphan_sandbox_deleted', { name: box.name, d1: row ? row.status : 'missing', firstSeenAt: orphan.first_seen_at })
|
||||
} else if (box.status === 'running' && orphan.suspended_at === null) {
|
||||
// Stop the meter now; the automatic snapshot keeps the disk until the grace period ends.
|
||||
await deps.provider.stopSession(box.currentSessionId)
|
||||
await deps.db.prepare(`UPDATE provider_orphans SET suspended_at = ? WHERE kind = 'sandbox' AND provider_id = ?`).bind(now, box.name).run()
|
||||
summary.orphanSandboxesSuspended++
|
||||
log('orphan_sandbox_suspended', { name: box.name, d1: row ? row.status : 'missing' })
|
||||
}
|
||||
}
|
||||
if (!listing.complete) {
|
||||
log('orphan_sweep_partial', { listed: listing.sandboxes.length })
|
||||
return
|
||||
}
|
||||
// D1 says it exists, the provider does not: reported, never auto-repaired (wake fails cleanly).
|
||||
for (const row of await listByStatus(deps.db, ['running', 'suspended'])) {
|
||||
if (row.provider_name.startsWith(`orca-${env}-`) && !providerNames.has(row.provider_name) && now - row.updated_at > grace) {
|
||||
summary.rowsMissingProvider++
|
||||
}
|
||||
}
|
||||
await sweepSnapshots(deps, listing.sandboxes.map((s) => s.name), summary)
|
||||
}
|
||||
|
||||
/** Automatic snapshots whose sandbox no longer exists anywhere in the project. Golden (manual) ones are never touched. */
|
||||
async function sweepSnapshots(deps: LifecycleDeps, sandboxNames: string[], summary: HousekeepingSummary) {
|
||||
const now = deps.now()
|
||||
const grace = retention(deps).orphanGraceMs
|
||||
if (sandboxNames.length > 500) {
|
||||
log('orphan_snapshot_sweep_skipped', { sandboxes: sandboxNames.length })
|
||||
return
|
||||
}
|
||||
const listed = await deps.provider.listProjectSnapshots()
|
||||
if (!listed.complete) {
|
||||
log('orphan_snapshot_sweep_partial', { listed: listed.snapshots.length })
|
||||
return
|
||||
}
|
||||
const owned = new Set<string>([deps.config.goldenSnapshotId])
|
||||
for (const name of sandboxNames) {
|
||||
for (const snapshot of await deps.provider.listSnapshots(name)) {
|
||||
owned.add(snapshot.id)
|
||||
}
|
||||
}
|
||||
const { results } = await deps.db
|
||||
.prepare(
|
||||
`SELECT ss.snapshot_id FROM sandbox_snapshots ss JOIN sandboxes s ON s.id = ss.sandbox_id
|
||||
WHERE ss.deleted_at IS NULL AND s.status <> 'destroyed'`
|
||||
)
|
||||
.all<{ snapshot_id: string }>()
|
||||
for (const r of results) {
|
||||
owned.add(r.snapshot_id)
|
||||
}
|
||||
for (const snapshot of listed.snapshots) {
|
||||
if (snapshot.status !== 'created' || snapshot.creationMethod !== 'automatic' || owned.has(snapshot.id)) {
|
||||
if (owned.has(snapshot.id)) {
|
||||
await clearOrphan(deps.db, 'snapshot', snapshot.id)
|
||||
}
|
||||
continue
|
||||
}
|
||||
summary.orphanSnapshotsSeen++
|
||||
const orphan = await upsertOrphan(deps.db, 'snapshot', snapshot.id, now)
|
||||
if (orphan.deleted_at === null && now - orphan.first_seen_at >= grace) {
|
||||
await deps.provider.deleteSnapshot(snapshot.id)
|
||||
await deps.db.prepare(`UPDATE provider_orphans SET deleted_at = ? WHERE kind = 'snapshot' AND provider_id = ?`).bind(now, snapshot.id).run()
|
||||
summary.orphanSnapshotsDeleted++
|
||||
log('orphan_snapshot_deleted', { id: snapshot.id, createdAt: snapshot.createdAt, firstSeenAt: orphan.first_seen_at })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- entry points ----
|
||||
|
||||
/** Kill switch drain: snapshot-preserving suspend of every running sandbox (one batch per call). */
|
||||
export async function drainAll(deps: LifecycleDeps): Promise<{ suspended: number; failed: number; remaining: number }> {
|
||||
let suspended = 0
|
||||
let failed = 0
|
||||
for (const row of await listByStatus(deps.db, ['running'])) {
|
||||
try {
|
||||
await suspendSandbox(deps, row)
|
||||
suspended++
|
||||
} catch {
|
||||
failed++
|
||||
}
|
||||
}
|
||||
const remaining = (await listByStatus(deps.db, ACTIVE_STATUSES)).length
|
||||
log('drain_all', { suspended, failed, remaining })
|
||||
return { suspended, failed, remaining }
|
||||
}
|
||||
|
||||
/** Runs at most once per sweep interval; each part is isolated so one failure does not stop the rest. */
|
||||
export async function housekeeping(
|
||||
deps: LifecycleDeps,
|
||||
options: { force?: boolean; notifier?: RetentionNotifier } = {}
|
||||
): Promise<HousekeepingSummary> {
|
||||
const summary: HousekeepingSummary = {
|
||||
swept: false,
|
||||
orphanSandboxesSuspended: 0,
|
||||
orphanSandboxesDeleted: 0,
|
||||
orphanSnapshotsSeen: 0,
|
||||
orphanSnapshotsDeleted: 0,
|
||||
rowsMissingProvider: 0,
|
||||
holdsOpened: 0,
|
||||
holdsCleared: 0,
|
||||
retentionWarnings: 0,
|
||||
retentionDestroyed: 0,
|
||||
idleWarnings: 0,
|
||||
idleDestroyed: 0
|
||||
}
|
||||
const now = deps.now()
|
||||
const last = Number((await getControlState(deps.db, 'housekeeping_at')) ?? 0)
|
||||
if (!options.force && now - last < retention(deps).sweepIntervalMs) {
|
||||
return summary
|
||||
}
|
||||
await setControlState(deps.db, 'housekeeping_at', String(now), now)
|
||||
summary.swept = true
|
||||
const notifier = options.notifier ?? LOG_NOTIFIER
|
||||
for (const step of [
|
||||
() => sweepOrphans(deps, summary),
|
||||
() => probeHolds(deps, summary),
|
||||
() => enforceHolds(deps, notifier, summary),
|
||||
() => enforceIdle(deps, notifier, summary)
|
||||
]) {
|
||||
try {
|
||||
await step()
|
||||
} catch (error) {
|
||||
log('housekeeping_step_failed', { message: error instanceof Error ? error.message : 'unknown' })
|
||||
}
|
||||
}
|
||||
return summary
|
||||
}
|
||||
@@ -41,6 +41,22 @@ export type ProviderSnapshot = {
|
||||
sourceSessionId: string
|
||||
}
|
||||
|
||||
/** Project-wide listing entries for the orphan sweep. */
|
||||
export type ProviderSandboxSummary = {
|
||||
name: string
|
||||
status: ProviderSessionStatus
|
||||
currentSessionId: string
|
||||
tags: Record<string, string>
|
||||
}
|
||||
|
||||
export type ProviderProjectSnapshot = {
|
||||
id: string
|
||||
status: 'created' | 'deleted' | 'failed'
|
||||
/** `automatic` (session stop) | `manual` (golden images); absent on old snapshots. */
|
||||
creationMethod: string | null
|
||||
createdAt: number
|
||||
}
|
||||
|
||||
export type CreateSandboxParams = {
|
||||
name: string
|
||||
snapshotId: string
|
||||
@@ -50,6 +66,8 @@ export type CreateSandboxParams = {
|
||||
ports: number[]
|
||||
networkPolicy: NetworkPolicy
|
||||
tags: Record<string, string>
|
||||
/** Provider-side snapshot expiry after last use (backstop behind Orca's own retention); 0 = never. */
|
||||
snapshotExpirationMs: number
|
||||
}
|
||||
|
||||
export type RunCommandParams = {
|
||||
@@ -76,6 +94,10 @@ export type SandboxProviderApi = {
|
||||
listSnapshots(name: string): Promise<ProviderSnapshot[]>
|
||||
/** Resolves when already gone. */
|
||||
deleteSnapshot(snapshotId: string): Promise<void>
|
||||
/** Every sandbox in Orca's project (all environments). `complete` is false when the listing was cut short. */
|
||||
listSandboxes(): Promise<{ sandboxes: ProviderSandboxSummary[]; complete: boolean }>
|
||||
/** Every snapshot in Orca's project. */
|
||||
listProjectSnapshots(): Promise<{ snapshots: ProviderProjectSnapshot[]; complete: boolean }>
|
||||
}
|
||||
|
||||
/** Provider failed or answered unexpectedly; the body is never surfaced to callers. */
|
||||
|
||||
@@ -36,6 +36,7 @@ export type SandboxRow = {
|
||||
github_token_expires_at: number | null
|
||||
/** Comma list of model API hosts whose key the egress proxy swaps in this session. */
|
||||
egress_secret_hosts: string | null
|
||||
retention_warned_at: number | null
|
||||
}
|
||||
|
||||
export type SessionFields = Pick<
|
||||
@@ -127,7 +128,7 @@ type Patch = Partial<Omit<SandboxRow, 'id' | 'account_id' | 'generation' | 'crea
|
||||
const PATCHABLE = new Set<string>([
|
||||
'status', 'session_id', 'session_started_at', 'session_deadline_at', 'lease_expires_at',
|
||||
'usage_settled_at', 'last_error', 'destroyed_at', 'github_token_expires_at',
|
||||
'egress_secret_hosts'
|
||||
'egress_secret_hosts', 'repo_url', 'retention_warned_at'
|
||||
])
|
||||
|
||||
/** Applies `patch` only if nobody moved the row since `expected` was read; null on a lost race. */
|
||||
|
||||
@@ -42,6 +42,14 @@ const Snapshot = z.looseObject({
|
||||
status: z.enum(['created', 'deleted', 'failed']),
|
||||
sourceSessionId: z.string()
|
||||
})
|
||||
const ProjectSnapshot = Snapshot.extend({ creationMethod: z.string().optional(), createdAt: z.number() })
|
||||
const SandboxSummary = z.looseObject({
|
||||
name: z.string(),
|
||||
status: Status,
|
||||
currentSessionId: z.string(),
|
||||
tags: z.record(z.string(), z.string()).optional()
|
||||
})
|
||||
const MAX_LISTED = 5000
|
||||
const ErrorBody = z.looseObject({ error: z.looseObject({ code: z.string().optional() }).optional() })
|
||||
const CommandLine = z.looseObject({ command: z.looseObject({ exitCode: z.number().nullable() }) })
|
||||
const OutputLine = z.looseObject({ stream: z.enum(['stdout', 'stderr']), data: z.string() })
|
||||
@@ -129,8 +137,8 @@ export function createVercelSandboxApi(
|
||||
timeout: params.timeoutMs,
|
||||
ports: params.ports,
|
||||
persistent: true,
|
||||
keepLastSnapshots: { count: 2 },
|
||||
snapshotExpiration: 0,
|
||||
keepLastSnapshots: { count: 2, expiration: params.snapshotExpirationMs },
|
||||
snapshotExpiration: params.snapshotExpirationMs,
|
||||
networkPolicy: params.networkPolicy,
|
||||
tags: params.tags
|
||||
}
|
||||
@@ -247,6 +255,46 @@ export function createVercelSandboxApi(
|
||||
return snapshots.map((s) => ({ id: s.id, status: s.status, sourceSessionId: s.sourceSessionId }))
|
||||
},
|
||||
|
||||
async listSandboxes() {
|
||||
const sandboxes: z.infer<typeof SandboxSummary>[] = []
|
||||
let cursor: string | undefined
|
||||
do {
|
||||
const response = await request('GET', '/v2/sandboxes', {
|
||||
query: { project: config.projectId, limit: '50', cursor }
|
||||
})
|
||||
const page = await json(
|
||||
z.looseObject({ sandboxes: z.array(SandboxSummary), pagination: z.looseObject({ next: z.string().nullable() }) }),
|
||||
response
|
||||
)
|
||||
sandboxes.push(...page.sandboxes)
|
||||
cursor = page.pagination.next ?? undefined
|
||||
} while (cursor && sandboxes.length < MAX_LISTED)
|
||||
return {
|
||||
complete: cursor === undefined,
|
||||
sandboxes: sandboxes.map((s) => ({ name: s.name, status: s.status, currentSessionId: s.currentSessionId, tags: s.tags ?? {} }))
|
||||
}
|
||||
},
|
||||
|
||||
async listProjectSnapshots() {
|
||||
const snapshots: z.infer<typeof ProjectSnapshot>[] = []
|
||||
let cursor: string | undefined
|
||||
do {
|
||||
const response = await request('GET', '/v2/sandboxes/snapshots', {
|
||||
query: { project: config.projectId, limit: '50', cursor }
|
||||
})
|
||||
const page = await json(
|
||||
z.looseObject({ snapshots: z.array(ProjectSnapshot), pagination: z.looseObject({ next: z.string().nullable() }) }),
|
||||
response
|
||||
)
|
||||
snapshots.push(...page.snapshots)
|
||||
cursor = page.pagination.next ?? undefined
|
||||
} while (cursor && snapshots.length < MAX_LISTED)
|
||||
return {
|
||||
complete: cursor === undefined,
|
||||
snapshots: snapshots.map((s) => ({ id: s.id, status: s.status, creationMethod: s.creationMethod ?? null, createdAt: s.createdAt }))
|
||||
}
|
||||
},
|
||||
|
||||
deleteSnapshot: (snapshotId) =>
|
||||
missingOk(() => request('DELETE', `/v2/sandboxes/snapshots/${encodeURIComponent(snapshotId)}`))
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@ type FakeBox = {
|
||||
|
||||
export type FakeVercel = SandboxProviderApi & {
|
||||
boxes: Map<string, FakeBox>
|
||||
snapshots: Map<string, ProviderSnapshot & { sandboxName: string }>
|
||||
snapshots: Map<string, ProviderSnapshot & { sandboxName: string; creationMethod: string; createdAt: number }>
|
||||
commands: { sessionId: string; command: string; env: Record<string, string> }[]
|
||||
failStart: boolean
|
||||
failCreate: boolean
|
||||
@@ -58,7 +58,7 @@ export function createFakeVercel(now: () => number): FakeVercel {
|
||||
const stop = (box: FakeBox): string => {
|
||||
box.status = 'stopped'
|
||||
const id = next('snap')
|
||||
snapshots.set(id, { id, status: 'created', sourceSessionId: box.sessionId, sandboxName: box.params.name })
|
||||
snapshots.set(id, { id, status: 'created', sourceSessionId: box.sessionId, sandboxName: box.params.name, creationMethod: 'automatic', createdAt: now() })
|
||||
return id
|
||||
}
|
||||
|
||||
@@ -150,6 +150,23 @@ export function createFakeVercel(now: () => number): FakeVercel {
|
||||
.filter((snapshot) => snapshot.sandboxName === name)
|
||||
.map(({ id, status, sourceSessionId }) => ({ id, status, sourceSessionId }))
|
||||
},
|
||||
async listSandboxes() {
|
||||
return {
|
||||
complete: true,
|
||||
sandboxes: [...boxes.values()].map((box) => ({
|
||||
name: box.params.name,
|
||||
status: box.status,
|
||||
currentSessionId: box.sessionId,
|
||||
tags: box.params.tags
|
||||
}))
|
||||
}
|
||||
},
|
||||
async listProjectSnapshots() {
|
||||
return {
|
||||
complete: true,
|
||||
snapshots: [...snapshots.values()].map(({ id, status, creationMethod, createdAt }) => ({ id, status, creationMethod, createdAt }))
|
||||
}
|
||||
},
|
||||
async deleteSnapshot(snapshotId) {
|
||||
const snapshot = snapshots.get(snapshotId)
|
||||
if (snapshot) {
|
||||
|
||||
@@ -1,53 +1,313 @@
|
||||
import { exportPKCS8, generateKeyPair, decodeJwt } from 'jose'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { createGitHubAppMinter } from '../src/github-app'
|
||||
import { bindInstallation, isInstallationBound, signState, verifyState } from '../src/github-connect'
|
||||
import { gitBasicAuthorization } from '../src/network-policy'
|
||||
import { START_COMMAND } from '../src/sandbox-vm-commands'
|
||||
import { createHarness } from './harness'
|
||||
|
||||
const MIN = 60_000
|
||||
const REPO = 'https://github.com/alice-gh/private-repo.git'
|
||||
const STATE_SECRET = 's'.repeat(40)
|
||||
const OPERATOR = 'o'.repeat(40)
|
||||
|
||||
type Call = { method: string; path: string; body: unknown; auth: string | null }
|
||||
|
||||
/** Fake api.github.com + github.com OAuth: alice-gh (id 1) owns installation 77 on her personal account. */
|
||||
function fakeGitHub(options: { now: () => number; users?: Record<string, { id: number; login: string; installations: unknown[]; memberships?: Record<string, unknown> }> }) {
|
||||
const calls: Call[] = []
|
||||
const users = options.users ?? {
|
||||
code_alice: { id: 1, login: 'alice-gh', installations: [{ id: 77, account: { id: 1, login: 'alice-gh', type: 'User' } }] },
|
||||
code_mallory: { id: 666, login: 'mallory-gh', installations: [] }
|
||||
}
|
||||
const tokens = new Map<string, keyof typeof users>()
|
||||
const fakeFetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
const url = new URL(String(input))
|
||||
const auth = new Headers(init?.headers).get('authorization')
|
||||
const body = init?.body ? JSON.parse(String(init.body)) : undefined
|
||||
calls.push({ method: init?.method ?? 'GET', path: url.pathname, body, auth })
|
||||
if (url.hostname === 'github.com' && url.pathname === '/login/oauth/access_token') {
|
||||
const user = users[body.code]
|
||||
if (!user) {
|
||||
return Response.json({ error: 'bad_verification_code' })
|
||||
}
|
||||
const token = `gho_${body.code}`
|
||||
tokens.set(token, body.code)
|
||||
return Response.json({ access_token: token })
|
||||
}
|
||||
const userToken = auth?.startsWith('Bearer gho_') ? users[tokens.get(auth.slice(7)) ?? ''] : undefined
|
||||
if (url.pathname === '/user') {
|
||||
return userToken ? Response.json({ id: userToken.id, login: userToken.login }) : new Response(null, { status: 401 })
|
||||
}
|
||||
if (url.pathname === '/user/installations') {
|
||||
return Response.json({ total_count: userToken!.installations.length, installations: userToken!.installations })
|
||||
}
|
||||
if (url.pathname.startsWith('/user/memberships/orgs/')) {
|
||||
const membership = userToken?.memberships?.[url.pathname.split('/').pop()!]
|
||||
return membership ? Response.json(membership) : new Response(null, { status: 404 })
|
||||
}
|
||||
// App-JWT routes: the repo is installed under installation 77 regardless of who asks.
|
||||
if (url.pathname === '/repos/alice-gh/private-repo/installation' || url.pathname === '/app/installations/77') {
|
||||
return Response.json({ id: 77, account: { id: 1, login: 'alice-gh', type: 'User' } })
|
||||
}
|
||||
if (url.pathname === '/app/installations/77/access_tokens') {
|
||||
return Response.json({
|
||||
token: `ghs_${calls.length}`,
|
||||
expires_at: new Date(options.now() + 60 * MIN).toISOString(),
|
||||
repositories: body.repositories.map((name: string) => ({ id: 9, name }))
|
||||
})
|
||||
}
|
||||
return new Response(null, { status: 404 })
|
||||
}) as typeof globalThis.fetch
|
||||
return { calls, fetch: fakeFetch }
|
||||
}
|
||||
|
||||
async function appKey() {
|
||||
const { privateKey } = await generateKeyPair('RS256', { extractable: true })
|
||||
return exportPKCS8(privateKey)
|
||||
}
|
||||
|
||||
async function harnessWithApp() {
|
||||
let clock = () => 0
|
||||
const github = fakeGitHub({ now: () => clock() })
|
||||
const pem = await appKey()
|
||||
let dbRef: D1Database | null = null
|
||||
const minter = createGitHubAppMinter({
|
||||
appId: '123',
|
||||
privateKeyPkcs8: pem,
|
||||
fetch: github.fetch,
|
||||
now: () => clock(),
|
||||
isBound: (id, account) => isInstallationBound(dbRef!, id, account),
|
||||
log: () => undefined
|
||||
})
|
||||
const h = await createHarness(
|
||||
{
|
||||
GITHUB_APP_SLUG: 'orca-cloud-test',
|
||||
GITHUB_APP_CLIENT_ID: 'Iv1.test',
|
||||
GITHUB_APP_CLIENT_SECRET: 'client-secret',
|
||||
GITHUB_STATE_SECRET: STATE_SECRET,
|
||||
OPERATOR_TOKEN: OPERATOR,
|
||||
EGRESS_PROXY_URL: 'https://sandbox.test/v1/egress/proxy'
|
||||
},
|
||||
{ github: minter, githubFetch: github.fetch }
|
||||
)
|
||||
clock = h.now
|
||||
dbRef = h.env.SANDBOX_DB
|
||||
return { h, github, minter }
|
||||
}
|
||||
|
||||
async function browser(h: Awaited<ReturnType<typeof createHarness>>, url: string) {
|
||||
const response = await h.send(new Request(url))
|
||||
return { status: response.status, location: response.headers.get('location'), text: await response.text() }
|
||||
}
|
||||
|
||||
/** Desktop asks for a connect URL, the browser installs + signs in on GitHub as `code`'s user. */
|
||||
async function connect(h: Awaited<ReturnType<typeof createHarness>>, as: string, code: string, installationId = 77) {
|
||||
const { body } = await h.call('POST', '/v1/github/connect', { as })
|
||||
const state = new URL(body.installUrl).searchParams.get('state')!
|
||||
const setup = await browser(h, `https://sandbox.test/v1/github/setup?installation_id=${installationId}&setup_action=install&state=${encodeURIComponent(state)}`)
|
||||
expect(setup.status).toBe(302)
|
||||
const authorize = new URL(setup.location!)
|
||||
expect(authorize.origin + authorize.pathname).toBe('https://github.com/login/oauth/authorize')
|
||||
expect(authorize.searchParams.get('redirect_uri')).toBe('https://sandbox.test/v1/github/callback')
|
||||
const carried = authorize.searchParams.get('state')!
|
||||
const callback = await browser(h, `https://sandbox.test/v1/github/callback?code=${code}&state=${encodeURIComponent(carried)}`)
|
||||
return { state, carried, callback, body }
|
||||
}
|
||||
|
||||
describe('GitHub App installation tokens', () => {
|
||||
it('mints a token scoped to the one workspace repo and reuses it until near expiry', async () => {
|
||||
const { privateKey } = await generateKeyPair('RS256', { extractable: true })
|
||||
const calls: { method: string; path: string; body: unknown; iss: unknown }[] = []
|
||||
it('mints a single-repo token only for a bound account, and re-checks the binding on cached tokens', async () => {
|
||||
let now = Date.parse('2026-11-09T00:00:00Z')
|
||||
const github = fakeGitHub({ now: () => now })
|
||||
const bound = new Set(['77:user:alice'])
|
||||
const minter = createGitHubAppMinter({
|
||||
appId: '123',
|
||||
privateKeyPkcs8: await exportPKCS8(privateKey),
|
||||
privateKeyPkcs8: await appKey(),
|
||||
now: () => now,
|
||||
fetch: async (input, init) => {
|
||||
const path = new URL(String(input)).pathname
|
||||
const jwt = new Headers(init?.headers).get('authorization')!.replace('Bearer ', '')
|
||||
calls.push({ method: init?.method ?? 'GET', path, body: init?.body ? JSON.parse(String(init.body)) : undefined, iss: decodeJwt(jwt).iss })
|
||||
return path.endsWith('/installation')
|
||||
? Response.json({ id: 77 })
|
||||
: Response.json({ token: 'ghs_minted', expires_at: new Date(now + 60 * MIN).toISOString() })
|
||||
fetch: github.fetch,
|
||||
isBound: async (id, account) => bound.has(`${id}:${account}`),
|
||||
log: () => undefined
|
||||
})
|
||||
const token = await minter.tokenForRepo(REPO, 'user:alice')
|
||||
expect(token).toMatchObject({ expiresAt: now + 60 * MIN })
|
||||
expect(github.calls.map((c) => [c.method, c.path, c.body])).toEqual([
|
||||
['GET', '/repos/alice-gh/private-repo/installation', undefined],
|
||||
[
|
||||
'POST',
|
||||
'/app/installations/77/access_tokens',
|
||||
{ repositories: ['private-repo'], permissions: { contents: 'write', pull_requests: 'write', metadata: 'read' } }
|
||||
]
|
||||
])
|
||||
expect(decodeJwt(github.calls[0]!.auth!.slice(7)).iss).toBe('123')
|
||||
|
||||
// Cross-tenant: another account naming the same repo gets nothing, and no token is minted.
|
||||
expect(await minter.tokenForRepo(REPO, 'user:bob')).toBeNull()
|
||||
expect(github.calls.filter((c) => c.path.endsWith('/access_tokens'))).toHaveLength(1)
|
||||
|
||||
now += 30 * MIN
|
||||
expect(await minter.tokenForRepo('https://github.com/alice-gh/private-repo', 'user:alice')).toEqual(token)
|
||||
bound.delete('77:user:alice')
|
||||
expect(await minter.tokenForRepo(REPO, 'user:alice')).toBeNull()
|
||||
|
||||
expect(await minter.tokenForRepo('https://gitlab.com/a/b', 'user:alice')).toBeNull()
|
||||
expect(await minter.tokenForRepo('https://x:y@github.com/alice-gh/private-repo', 'user:alice')).toBeNull()
|
||||
})
|
||||
|
||||
it('refuses a minted token that covers more than the one repository', async () => {
|
||||
const minter = createGitHubAppMinter({
|
||||
appId: '123',
|
||||
privateKeyPkcs8: await appKey(),
|
||||
isBound: async () => true,
|
||||
fetch: (async (input: RequestInfo | URL) =>
|
||||
String(input).endsWith('/installation')
|
||||
? Response.json({ id: 77, account: { id: 1, login: 'alice-gh', type: 'User' } })
|
||||
: Response.json({ token: 'ghs_wide', expires_at: new Date().toISOString(), repositories: [{ id: 1, name: 'private-repo' }, { id: 2, name: 'other' }] })) as typeof fetch
|
||||
})
|
||||
await expect(minter.tokenForRepo(REPO, 'user:alice')).rejects.toThrow('unexpected scope')
|
||||
})
|
||||
|
||||
it('cross-tenant attack: bob names alice’s repo and gets no credential and no token is minted', async () => {
|
||||
const { h, github } = await harnessWithApp()
|
||||
await bindInstallation(
|
||||
h.env.SANDBOX_DB,
|
||||
{
|
||||
installation_id: 77,
|
||||
account_id: 'user:alice',
|
||||
github_account_id: 1,
|
||||
github_account_login: 'alice-gh',
|
||||
github_account_type: 'User',
|
||||
bound_by_user_id: 'alice',
|
||||
bound_by_github_user_id: 1,
|
||||
method: 'oauth'
|
||||
},
|
||||
h.now()
|
||||
)
|
||||
const res = await h.call('POST', '/v1/sandboxes', { as: 'bob', body: { workspaceId: 'ws-steal', machineSize: 'standard', repo: { url: REPO } } })
|
||||
expect(res.status).toBe(200)
|
||||
const [box] = [...h.vercel.boxes.values()]
|
||||
expect(box!.policy.allow['github.com'] ?? []).toEqual([])
|
||||
expect(h.vercel.commands.find((c) => c.command === START_COMMAND)!.env.ORCA_GIT_EXTRAHEADER).toBeUndefined()
|
||||
expect(github.calls.filter((c) => c.path.endsWith('/access_tokens'))).toHaveLength(0)
|
||||
|
||||
// Alice, bound, gets her token.
|
||||
const mine = await h.call('POST', '/v1/sandboxes', { as: 'alice', body: { workspaceId: 'ws-mine', machineSize: 'standard', repo: { url: REPO } } })
|
||||
const aliceBox = h.vercel.boxes.get(`orca-test-${mine.body.sandbox.id}`)!
|
||||
expect(aliceBox.policy.allow['github.com']![0]).toMatchObject({
|
||||
transform: [{ headers: { authorization: expect.stringMatching(/^Basic /) } }]
|
||||
})
|
||||
expect(github.calls.filter((c) => c.path.endsWith('/access_tokens'))).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('binds an installation only after GitHub proves the installer owns it, single use', async () => {
|
||||
const { h, minter } = await harnessWithApp()
|
||||
const { state, carried, callback, body } = await connect(h, 'alice', 'code_alice')
|
||||
expect(body.installUrl).toMatch(/^https:\/\/github\.com\/apps\/orca-cloud-test\/installations\/new\?state=/)
|
||||
expect(callback.text).toContain('Connected GitHub <b>alice-gh</b>')
|
||||
expect(await minter.tokenForRepo(REPO, 'user:alice')).not.toBeNull()
|
||||
expect(await minter.tokenForRepo(REPO, 'user:bob')).toBeNull()
|
||||
expect((await h.call('GET', '/v1/github/installations', { as: 'alice' })).body.installations).toMatchObject([
|
||||
{ installationId: 77, account: 'alice-gh', accountType: 'User' }
|
||||
])
|
||||
expect((await h.call('GET', '/v1/github/installations', { as: 'bob' })).body.installations).toEqual([])
|
||||
|
||||
// Replaying the callback (or the original state) binds nothing more.
|
||||
const replay = await browser(h, `https://sandbox.test/v1/github/callback?code=code_alice&state=${encodeURIComponent(carried)}`)
|
||||
expect(replay.text).toContain('already used')
|
||||
expect(state).not.toBe(carried)
|
||||
})
|
||||
|
||||
it('attack: mallory starts her own connect and points the setup URL at alice’s installation id', async () => {
|
||||
const { h, minter } = await harnessWithApp()
|
||||
const { callback } = await connect(h, 'mallory', 'code_mallory', 77)
|
||||
expect(callback.text).toContain('Nothing was connected')
|
||||
expect(await minter.tokenForRepo(REPO, 'user:mallory')).toBeNull()
|
||||
expect(h.raw.prepare('SELECT COUNT(*) AS n FROM github_installations').get()).toEqual({ n: 0 })
|
||||
})
|
||||
|
||||
it('attack: a forged or tampered state (another account, no signature) is refused', async () => {
|
||||
const { h } = await harnessWithApp()
|
||||
const { body } = await h.call('POST', '/v1/github/connect', { as: 'mallory' })
|
||||
const state = new URL(body.installUrl).searchParams.get('state')!
|
||||
const payload = await verifyState(STATE_SECRET, state, h.now())
|
||||
const [bodyPart, mac] = state.split('.')
|
||||
const forged = `${btoa(JSON.stringify({ ...payload, a: 'user:alice' })).replace(/=+$/, '')}.${mac}`
|
||||
expect(bodyPart).not.toBe(forged.split('.')[0])
|
||||
for (const bad of [forged, await signState('x'.repeat(40), payload!), 'garbage']) {
|
||||
const res = await browser(h, `https://sandbox.test/v1/github/callback?code=code_alice&state=${encodeURIComponent(bad)}`)
|
||||
expect(res.text).toContain('not opened from Orca')
|
||||
}
|
||||
// A validly signed state whose nonce belongs to another account is refused too.
|
||||
const crossed = await signState(STATE_SECRET, { ...payload!, a: 'user:alice' })
|
||||
const res = await browser(h, `https://sandbox.test/v1/github/callback?code=code_alice&state=${encodeURIComponent(crossed)}`)
|
||||
expect(res.text).toContain('already used')
|
||||
expect(h.raw.prepare('SELECT COUNT(*) AS n FROM github_installations').get()).toEqual({ n: 0 })
|
||||
h.advance(11 * MIN)
|
||||
const expired = await browser(h, `https://sandbox.test/v1/github/callback?code=code_mallory&state=${encodeURIComponent(state)}`)
|
||||
expect(expired.text).toContain('not opened from Orca')
|
||||
})
|
||||
|
||||
it('org installations bind only for an active org admin', async () => {
|
||||
let now = 0
|
||||
const org = { id: 77, account: { id: 50, login: 'acme', type: 'Organization' } }
|
||||
const github = fakeGitHub({
|
||||
now: () => now,
|
||||
users: {
|
||||
code_member: { id: 2, login: 'member', installations: [org], memberships: { acme: { state: 'active', role: 'member' } } },
|
||||
code_admin: { id: 3, login: 'admin', installations: [org], memberships: { acme: { state: 'active', role: 'admin' } } },
|
||||
code_pending: { id: 4, login: 'pending', installations: [org], memberships: { acme: { state: 'pending', role: 'admin' } } }
|
||||
}
|
||||
})
|
||||
expect(await minter.tokenForRepo('https://github.com/nwparker/orca-sandbox-push-test.git')).toEqual({ token: 'ghs_minted', expiresAt: now + 60 * MIN })
|
||||
expect(calls).toEqual([
|
||||
{ method: 'GET', path: '/repos/nwparker/orca-sandbox-push-test/installation', body: undefined, iss: '123' },
|
||||
{
|
||||
const h = await createHarness(
|
||||
{ GITHUB_APP_SLUG: 'orca-cloud-test', GITHUB_APP_CLIENT_ID: 'Iv1.test', GITHUB_APP_CLIENT_SECRET: 'x', GITHUB_STATE_SECRET: STATE_SECRET },
|
||||
{ githubFetch: github.fetch }
|
||||
)
|
||||
now = h.now()
|
||||
for (const [code, expected] of [['code_member', 0], ['code_pending', 0], ['code_admin', 1]] as const) {
|
||||
const { callback } = await connect(h, 'carol', code)
|
||||
expect(callback.status).toBe(200)
|
||||
expect(h.raw.prepare(`SELECT COUNT(*) AS n FROM github_installations WHERE account_id = 'user:carol'`).get()).toEqual({ n: expected })
|
||||
}
|
||||
})
|
||||
|
||||
it('operator binding needs the operator token and is recorded as such', async () => {
|
||||
const { h, minter } = await harnessWithApp()
|
||||
const bind = (token: string) =>
|
||||
h.send(
|
||||
new Request('https://sandbox.test/v1/internal/github/bindings', {
|
||||
method: 'POST',
|
||||
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
|
||||
body: JSON.stringify({ installationId: 77, accountId: 'dev:push' })
|
||||
})
|
||||
)
|
||||
expect((await bind(await h.token('alice'))).status).toBe(404)
|
||||
expect((await bind('o'.repeat(39) + 'x')).status).toBe(404)
|
||||
const ok = await bind(OPERATOR)
|
||||
expect(await ok.json()).toEqual({ bound: { installationId: 77, accountId: 'dev:push', githubAccount: 'alice-gh' } })
|
||||
expect(h.raw.prepare('SELECT method, bound_by_user_id FROM github_installations').get()).toEqual({ method: 'operator', bound_by_user_id: 'operator' })
|
||||
expect(await minter.tokenForRepo(REPO, 'dev:push')).not.toBeNull()
|
||||
const byRepo = await h.send(
|
||||
new Request('https://sandbox.test/v1/internal/github/bindings', {
|
||||
method: 'POST',
|
||||
path: '/app/installations/77/access_tokens',
|
||||
body: { repositories: ['orca-sandbox-push-test'], permissions: { contents: 'write', pull_requests: 'write', metadata: 'read' } },
|
||||
iss: '123'
|
||||
}
|
||||
])
|
||||
now += 30 * MIN
|
||||
await minter.tokenForRepo('https://github.com/nwparker/orca-sandbox-push-test')
|
||||
expect(calls).toHaveLength(2)
|
||||
now += 26 * MIN
|
||||
await minter.tokenForRepo('https://github.com/nwparker/orca-sandbox-push-test')
|
||||
expect(calls).toHaveLength(4)
|
||||
expect(await minter.tokenForRepo('https://gitlab.com/a/b')).toBeNull()
|
||||
headers: { authorization: `Bearer ${OPERATOR}` },
|
||||
body: JSON.stringify({ repoUrl: REPO, accountId: 'dev:other' })
|
||||
})
|
||||
)
|
||||
expect(await byRepo.json()).toEqual({ bound: { installationId: 77, accountId: 'dev:other', githubAccount: 'alice-gh' } })
|
||||
const unbind = await h.send(
|
||||
new Request('https://sandbox.test/v1/internal/github/bindings', {
|
||||
method: 'DELETE',
|
||||
headers: { authorization: `Bearer ${OPERATOR}` },
|
||||
body: JSON.stringify({ installationId: 77, accountId: 'dev:push' })
|
||||
})
|
||||
)
|
||||
expect(await unbind.json()).toEqual({ revoked: 1 })
|
||||
expect(await minter.tokenForRepo(REPO, 'dev:push')).toBeNull()
|
||||
})
|
||||
|
||||
it('injects the repo token for git and refreshes it before it expires', async () => {
|
||||
let expiresAt = 0
|
||||
const h = await createHarness({}, {
|
||||
github: { tokenForRepo: async () => ({ token: `ghs_${expiresAt}`, expiresAt }) }
|
||||
github: { tokenForRepo: async () => ({ token: `ghs_${expiresAt}`, expiresAt }), getInstallation: async () => null, getRepoInstallation: async () => null }
|
||||
})
|
||||
expiresAt = h.now() + 60 * MIN
|
||||
const { body } = await h.call('POST', '/v1/sandboxes', {
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { createHarness } from './harness'
|
||||
|
||||
const OPERATOR = 'o'.repeat(40)
|
||||
const create = (workspaceId: string) => ({ body: { workspaceId, machineSize: 'standard' } })
|
||||
|
||||
async function operator(h: Awaited<ReturnType<typeof createHarness>>, method: string, path: string, body?: unknown) {
|
||||
const response = await h.send(
|
||||
new Request(`https://sandbox.test${path}`, {
|
||||
method,
|
||||
headers: { authorization: `Bearer ${OPERATOR}`, 'content-type': 'application/json' },
|
||||
body: body === undefined ? undefined : JSON.stringify(body)
|
||||
})
|
||||
)
|
||||
return { status: response.status, body: (await response.json()) as any }
|
||||
}
|
||||
|
||||
describe('kill switch and caps', () => {
|
||||
it('kill switch refuses creates and resumes, drains running sandboxes keeping their disk, and lifts', async () => {
|
||||
const h = await createHarness({ OPERATOR_TOKEN: OPERATOR })
|
||||
const a = await h.call('POST', '/v1/sandboxes', create('ws-a'))
|
||||
const b = await h.call('POST', '/v1/sandboxes', { ...create('ws-b'), as: 'bob' })
|
||||
|
||||
expect((await h.call('GET', '/v1/internal/status')).status).toBe(404)
|
||||
const on = await operator(h, 'POST', '/v1/internal/kill-switch', { enabled: true })
|
||||
expect(on.body).toMatchObject({ killSwitch: true, drained: { suspended: 2, failed: 0, remaining: 0 } })
|
||||
for (const box of h.vercel.boxes.values()) {
|
||||
expect(box.status).toBe('stopped')
|
||||
}
|
||||
expect([...h.vercel.snapshots.values()].filter((s) => s.status === 'created')).toHaveLength(2)
|
||||
|
||||
expect(await h.call('POST', '/v1/sandboxes', create('ws-c'))).toEqual({ status: 503, body: { error: 'maintenance' } })
|
||||
expect((await h.call('POST', `/v1/sandboxes/${a.body.sandbox.id}/resume`)).body).toEqual({ error: 'maintenance' })
|
||||
expect(h.billing.checks).toHaveLength(2)
|
||||
expect((await operator(h, 'GET', '/v1/internal/status')).body).toMatchObject({ killSwitch: true, active: 0 })
|
||||
|
||||
await operator(h, 'POST', '/v1/internal/kill-switch', { enabled: false })
|
||||
expect((await h.call('POST', `/v1/sandboxes/${b.body.sandbox.id}/resume`, { as: 'bob' })).status).toBe(200)
|
||||
})
|
||||
|
||||
it('the cron keeps draining while the kill switch is on; the static env switch also refuses', async () => {
|
||||
const h = await createHarness({ OPERATOR_TOKEN: OPERATOR })
|
||||
await operator(h, 'POST', '/v1/internal/kill-switch', { enabled: true, drain: false })
|
||||
expect((await h.call('POST', '/v1/sandboxes', create('ws-a'))).status).toBe(503)
|
||||
await operator(h, 'POST', '/v1/internal/kill-switch', { enabled: false })
|
||||
await h.call('POST', '/v1/sandboxes', create('ws-a'))
|
||||
await operator(h, 'POST', '/v1/internal/kill-switch', { enabled: true, drain: false })
|
||||
expect((await h.reconcile()).drained).toBe(1)
|
||||
|
||||
const fixed = await createHarness({ SANDBOX_CREATES_DISABLED: 'true' })
|
||||
expect((await fixed.call('POST', '/v1/sandboxes', create('ws-a'))).body).toEqual({ error: 'maintenance' })
|
||||
})
|
||||
|
||||
it('global running cap refuses the next start across accounts', async () => {
|
||||
const h = await createHarness({ SANDBOX_MAX_RUNNING: '1' })
|
||||
expect((await h.call('POST', '/v1/sandboxes', create('ws-a'))).status).toBe(200)
|
||||
const res = await h.call('POST', '/v1/sandboxes', { ...create('ws-b'), as: 'bob' })
|
||||
expect(res).toEqual({ status: 503, body: { error: 'capacity' } })
|
||||
expect(h.vercel.boxes.size).toBe(1)
|
||||
})
|
||||
|
||||
it('per-account backstop is 3 per seat even when billing would allow more', async () => {
|
||||
const h = await createHarness()
|
||||
for (const ws of ['ws-1', 'ws-2', 'ws-3']) {
|
||||
expect((await h.call('POST', '/v1/sandboxes', create(ws))).status).toBe(200)
|
||||
}
|
||||
expect(await h.call('POST', '/v1/sandboxes', create('ws-4'))).toEqual({
|
||||
status: 403,
|
||||
body: { error: 'entitlement_denied', reason: 'concurrency_limit' }
|
||||
})
|
||||
h.billing.decision = { allow: true, reason: null, suspendRunning: false, seats: 2 }
|
||||
expect((await h.call('POST', '/v1/sandboxes', create('ws-4'))).status).toBe(200)
|
||||
expect(h.vercel.boxes.size).toBe(4)
|
||||
})
|
||||
|
||||
it('rate limits creates per account per minute', async () => {
|
||||
const h = await createHarness({ CREATE_RATE_PER_MINUTE: '2' })
|
||||
expect((await h.call('POST', '/v1/sandboxes', create('ws-1'))).status).toBe(200)
|
||||
expect((await h.call('POST', '/v1/sandboxes', create('ws-1'))).status).toBe(200)
|
||||
expect(await h.call('POST', '/v1/sandboxes', create('ws-2'))).toEqual({ status: 429, body: { error: 'rate_limited' } })
|
||||
expect((await h.call('POST', '/v1/sandboxes', { ...create('ws-2'), as: 'bob' })).status).toBe(200)
|
||||
h.advance(60_000)
|
||||
expect((await h.call('POST', '/v1/sandboxes', create('ws-2'))).status).toBe(200)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,180 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { DEFAULT_RETENTION, housekeeping, type RetentionWarning } from '../src/retention'
|
||||
import type { LifecycleDeps } from '../src/lifecycle'
|
||||
import { GOLDEN, createHarness } from './harness'
|
||||
|
||||
const HOUR = 60 * 60_000
|
||||
const DAY = 24 * HOUR
|
||||
const create = (workspaceId: string) => ({ body: { workspaceId, machineSize: 'standard' } })
|
||||
|
||||
async function setup() {
|
||||
const h = await createHarness()
|
||||
const warnings: RetentionWarning[] = []
|
||||
const deps: LifecycleDeps = {
|
||||
db: h.env.SANDBOX_DB,
|
||||
provider: h.vercel,
|
||||
entitlements: {
|
||||
checkStart: async (request) => {
|
||||
h.billing.checks.push({ accountId: request.accountId, runningSandboxes: request.runningSandboxes })
|
||||
return h.billing.decision
|
||||
},
|
||||
reportUsage: async () => h.billing.usageAnswer,
|
||||
listSuspensions: async () => h.billing.suspensions
|
||||
},
|
||||
now: h.now,
|
||||
retention: DEFAULT_RETENTION,
|
||||
config: { environment: 'test', goldenSnapshotId: GOLDEN, defaultRegion: 'iad1' }
|
||||
}
|
||||
const sweep = () => housekeeping(deps, { force: true, notifier: { warn: async (w) => void warnings.push(w) } })
|
||||
return { h, deps, warnings, sweep }
|
||||
}
|
||||
|
||||
describe('orphan sweep', () => {
|
||||
it('suspends a provider sandbox with no live D1 row, deletes it after the grace period, leaves tracked ones alone', async () => {
|
||||
const { h, sweep } = await setup()
|
||||
const tracked = await h.call('POST', '/v1/sandboxes', create('ws-1'))
|
||||
await h.vercel.createSandbox({
|
||||
name: 'orca-test-sbx_ghost',
|
||||
snapshotId: GOLDEN,
|
||||
vcpus: 4,
|
||||
region: 'iad1',
|
||||
timeoutMs: 30 * 60_000,
|
||||
ports: [],
|
||||
networkPolicy: { allow: {} },
|
||||
tags: { orcaSandbox: 'sbx_ghost', orcaEnv: 'test' },
|
||||
snapshotExpirationMs: 0
|
||||
})
|
||||
// Another environment's sandbox is never judged by this one.
|
||||
await h.vercel.createSandbox({
|
||||
name: 'orca-prod-sbx_other',
|
||||
snapshotId: GOLDEN,
|
||||
vcpus: 4,
|
||||
region: 'iad1',
|
||||
timeoutMs: 30 * 60_000,
|
||||
ports: [],
|
||||
networkPolicy: { allow: {} },
|
||||
tags: { orcaSandbox: 'sbx_other', orcaEnv: 'prod' },
|
||||
snapshotExpirationMs: 0
|
||||
})
|
||||
const first = await sweep()
|
||||
expect(first).toMatchObject({ orphanSandboxesSuspended: 1, orphanSandboxesDeleted: 0 })
|
||||
expect(h.vercel.boxes.get('orca-test-sbx_ghost')!.status).toBe('stopped')
|
||||
expect(h.vercel.boxes.get('orca-prod-sbx_other')!.status).toBe('running')
|
||||
expect(h.vercel.boxes.get(`orca-test-${tracked.body.sandbox.id}`)!.status).toBe('running')
|
||||
|
||||
h.advance(23 * HOUR)
|
||||
expect((await sweep()).orphanSandboxesDeleted).toBe(0)
|
||||
h.advance(2 * HOUR)
|
||||
expect((await sweep()).orphanSandboxesDeleted).toBe(1)
|
||||
expect(h.vercel.boxes.has('orca-test-sbx_ghost')).toBe(false)
|
||||
const ghostSnaps = [...h.vercel.snapshots.values()].filter((s) => s.sandboxName === 'orca-test-sbx_ghost')
|
||||
expect(ghostSnaps.every((s) => s.status === 'deleted')).toBe(true)
|
||||
expect(h.vercel.boxes.has('orca-prod-sbx_other')).toBe(true)
|
||||
expect(h.vercel.boxes.has(`orca-test-${tracked.body.sandbox.id}`)).toBe(true)
|
||||
})
|
||||
|
||||
it('deletes automatic snapshots whose sandbox is gone after the grace period, never manual or owned ones', async () => {
|
||||
const { h, sweep } = await setup()
|
||||
const tracked = await h.call('POST', '/v1/sandboxes', create('ws-1'))
|
||||
await h.call('POST', `/v1/sandboxes/${tracked.body.sandbox.id}/suspend`)
|
||||
const add = (id: string, creationMethod: string, sandboxName: string) =>
|
||||
h.vercel.snapshots.set(id, { id, status: 'created', sourceSessionId: 's', sandboxName, creationMethod, createdAt: h.now() })
|
||||
add('snap_leftover', 'automatic', 'orca-test-sbx_deleted_long_ago')
|
||||
add('snap_old_golden', 'manual', 'golden-build')
|
||||
const owned = [...h.vercel.snapshots.values()].find((s) => s.sandboxName === `orca-test-${tracked.body.sandbox.id}`)!
|
||||
|
||||
expect((await sweep()).orphanSnapshotsSeen).toBe(1)
|
||||
h.advance(25 * HOUR)
|
||||
expect((await sweep()).orphanSnapshotsDeleted).toBe(1)
|
||||
expect(h.vercel.snapshots.get('snap_leftover')!.status).toBe('deleted')
|
||||
expect(h.vercel.snapshots.get('snap_old_golden')!.status).toBe('created')
|
||||
expect(h.vercel.snapshots.get(owned.id)!.status).toBe('created')
|
||||
expect((await h.call('POST', `/v1/sandboxes/${tracked.body.sandbox.id}/resume`)).status).toBe(200)
|
||||
})
|
||||
|
||||
it('a partial provider listing deletes nothing', async () => {
|
||||
const { h, sweep } = await setup()
|
||||
h.vercel.snapshots.set('snap_leftover', { id: 'snap_leftover', status: 'created', sourceSessionId: 's', sandboxName: 'gone', creationMethod: 'automatic', createdAt: 0 })
|
||||
const list = h.vercel.listSandboxes
|
||||
h.vercel.listSandboxes = async () => ({ ...(await list()), complete: false })
|
||||
await sweep()
|
||||
h.advance(48 * HOUR)
|
||||
expect((await sweep()).orphanSnapshotsDeleted).toBe(0)
|
||||
expect(h.vercel.snapshots.get('snap_leftover')!.status).toBe('created')
|
||||
})
|
||||
})
|
||||
|
||||
describe('data retention', () => {
|
||||
it('destroys a canceled account’s sandboxes 30 days after cancel, warning 7 days ahead', async () => {
|
||||
const { h, sweep, warnings } = await setup()
|
||||
const box = await h.call('POST', '/v1/sandboxes', create('ws-1'))
|
||||
h.billing.suspensions = ['user:alice']
|
||||
h.billing.decision = { allow: false, reason: 'payment_grace_expired', suspendRunning: true }
|
||||
await h.reconcile()
|
||||
expect(h.raw.prepare('SELECT reason FROM account_holds WHERE account_id = ?').get('user:alice')).toEqual({ reason: 'payment_grace_expired' })
|
||||
expect((await h.call('GET', `/v1/sandboxes/${box.body.sandbox.id}`)).body.sandbox.status).toBe('suspended')
|
||||
|
||||
h.advance(22 * DAY)
|
||||
expect((await sweep()).retentionWarnings).toBe(0)
|
||||
h.advance(1 * DAY)
|
||||
expect((await sweep()).retentionWarnings).toBe(1)
|
||||
expect(warnings[0]).toMatchObject({ kind: 'data_deletion', accountId: 'user:alice', sandboxIds: [box.body.sandbox.id] })
|
||||
h.advance(6 * DAY)
|
||||
expect((await sweep()).retentionDestroyed).toBe(0)
|
||||
h.advance(1 * DAY + HOUR)
|
||||
expect((await sweep()).retentionDestroyed).toBe(1)
|
||||
expect(h.vercel.boxes.size).toBe(0)
|
||||
expect([...h.vercel.snapshots.values()].every((s) => s.status === 'deleted')).toBe(true)
|
||||
expect(h.raw.prepare('SELECT repo_url, status FROM sandboxes').get()).toEqual({ repo_url: null, status: 'destroyed' })
|
||||
})
|
||||
|
||||
it('a canceled subscription found by the standing probe starts the clock; resubscribing clears it', async () => {
|
||||
const { h, sweep } = await setup()
|
||||
await h.call('POST', '/v1/sandboxes', create('ws-1'))
|
||||
h.billing.decision = { allow: false, reason: 'subscription_inactive', suspendRunning: false }
|
||||
expect((await sweep()).holdsOpened).toBe(1)
|
||||
h.advance(2 * DAY)
|
||||
h.billing.decision = { allow: true, reason: null, suspendRunning: false }
|
||||
expect((await sweep()).holdsCleared).toBe(1)
|
||||
h.advance(40 * DAY)
|
||||
expect((await sweep()).retentionDestroyed).toBe(0)
|
||||
expect(h.vercel.boxes.size).toBe(1)
|
||||
})
|
||||
|
||||
it('an exhausted-hours denial or a billing outage never starts a deletion clock', async () => {
|
||||
const { h, sweep, deps } = await setup()
|
||||
await h.call('POST', '/v1/sandboxes', create('ws-1'))
|
||||
h.billing.decision = { allow: false, reason: 'included_hours_exhausted', suspendRunning: false }
|
||||
expect((await sweep()).holdsOpened).toBe(0)
|
||||
deps.entitlements.checkStart = async () => {
|
||||
throw new Error('billing down')
|
||||
}
|
||||
h.advance(2 * DAY)
|
||||
expect((await sweep()).holdsOpened).toBe(0)
|
||||
expect(h.raw.prepare(`SELECT COUNT(*) AS n FROM account_holds WHERE reason <> 'active'`).get()).toEqual({ n: 0 })
|
||||
})
|
||||
|
||||
it('idle retention: a suspended sandbox untouched 90 days is warned at 83 and destroyed at 90; a resume resets it', async () => {
|
||||
const { h, sweep, warnings } = await setup()
|
||||
const idle = await h.call('POST', '/v1/sandboxes', create('ws-idle'))
|
||||
const used = await h.call('POST', '/v1/sandboxes', create('ws-used'))
|
||||
await h.call('POST', `/v1/sandboxes/${idle.body.sandbox.id}/suspend`)
|
||||
await h.call('POST', `/v1/sandboxes/${used.body.sandbox.id}/suspend`)
|
||||
h.advance(84 * DAY)
|
||||
expect((await sweep()).idleWarnings).toBe(2)
|
||||
expect(warnings.map((w) => w.kind)).toEqual(['idle_deletion', 'idle_deletion'])
|
||||
await h.call('POST', `/v1/sandboxes/${used.body.sandbox.id}/resume`)
|
||||
await h.call('POST', `/v1/sandboxes/${used.body.sandbox.id}/suspend`)
|
||||
h.advance(7 * DAY)
|
||||
expect((await sweep()).idleDestroyed).toBe(1)
|
||||
expect((await h.call('GET', `/v1/sandboxes/${idle.body.sandbox.id}`)).status).toBe(200)
|
||||
expect((await h.call('GET', `/v1/sandboxes/${idle.body.sandbox.id}`)).body.sandbox.status).toBe('destroyed')
|
||||
expect((await h.call('GET', `/v1/sandboxes/${used.body.sandbox.id}`)).body.sandbox.status).toBe('suspended')
|
||||
})
|
||||
|
||||
it('new sandboxes ask the provider for a snapshot expiry beyond idle retention', async () => {
|
||||
const { h } = await setup()
|
||||
await h.call('POST', '/v1/sandboxes', create('ws-1'))
|
||||
expect([...h.vercel.boxes.values()][0]!.params.snapshotExpirationMs).toBe(104 * DAY)
|
||||
})
|
||||
})
|
||||
@@ -41,7 +41,8 @@ describe('Vercel Sandbox REST client', () => {
|
||||
timeoutMs: 1_800_000,
|
||||
ports: [6768],
|
||||
networkPolicy: { allow: { 'registry.npmjs.org': [] } },
|
||||
tags: { orcaSandbox: 'sbx_1' }
|
||||
tags: { orcaSandbox: 'sbx_1' },
|
||||
snapshotExpirationMs: 104 * 86_400_000
|
||||
})
|
||||
expect(seen[0]!.method).toBe('POST')
|
||||
expect(`${seen[0]!.url.origin}${seen[0]!.url.pathname}${seen[0]!.url.search}`).toBe(
|
||||
@@ -53,8 +54,8 @@ describe('Vercel Sandbox REST client', () => {
|
||||
source: { type: 'snapshot', snapshotId: 'snap_golden' },
|
||||
resources: { vcpus: 4 },
|
||||
persistent: true,
|
||||
keepLastSnapshots: { count: 2 },
|
||||
snapshotExpiration: 0
|
||||
keepLastSnapshots: { count: 2, expiration: 104 * 86_400_000 },
|
||||
snapshotExpiration: 104 * 86_400_000
|
||||
})
|
||||
expect(state.routes).toEqual([{ port: 6768, url: 'https://sb-1.vercel.run' }])
|
||||
expect(state.session.timeoutMs).toBe(1_800_000)
|
||||
|
||||
@@ -4,7 +4,9 @@
|
||||
// wrangler deploy -c wrangler.dev.jsonc
|
||||
// Secrets (wrangler secret put -c wrangler.dev.jsonc, never here): VERCEL_TOKEN, SANDBOX_DEV_JWKS,
|
||||
// EGRESS_DEV_STATIC_AUTHORIZATION, CONTROL_PLANE_TOKEN (same value as on
|
||||
// orca-billing-dev), GITHUB_APP_ID, GITHUB_APP_PRIVATE_KEY (PKCS#8), SECRETS_KEK (base64 32 bytes).
|
||||
// orca-billing-dev), GITHUB_APP_ID, GITHUB_APP_PRIVATE_KEY (PKCS#8), SECRETS_KEK (base64 32 bytes),
|
||||
// OPERATOR_TOKEN (/v1/internal/*), GITHUB_STATE_SECRET; for the GitHub connect flow also
|
||||
// GITHUB_APP_CLIENT_ID + GITHUB_APP_CLIENT_SECRET (app settings → generate a client secret).
|
||||
"name": "orca-sandbox-control-dev",
|
||||
"main": "src/index.ts",
|
||||
"compatibility_date": "2026-11-01",
|
||||
@@ -29,7 +31,9 @@
|
||||
"VERCEL_PROJECT_ID": "prj_SkVtv6zJurF9dUHpfTRy4v9avwPq",
|
||||
"VERCEL_REGION": "iad1",
|
||||
"BILLING_URL": "https://orca-billing-dev.lovecast.workers.dev",
|
||||
// GOLDEN_SNAPSHOT_ID: unset until an image is built (image/build-golden-image.ts); lifecycle routes answer 503 meanwhile.
|
||||
// Golden image from image/build-golden-image.ts; lifecycle routes answer 503 without one.
|
||||
"GOLDEN_SNAPSHOT_ID": "snap_0fYmcpIUlc4OUC9viFN0Y4dVCDuD",
|
||||
"GITHUB_APP_SLUG": "orca-cloud-dev",
|
||||
"EGRESS_PROXY_URL": "https://orca-sandbox-control-dev.lovecast.workers.dev/v1/egress/proxy"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -230,6 +230,29 @@ Product decision: "API-key injection" in the Conductor parity matrix (form 2 = t
|
||||
in the same VM. Enable on dev after a live streaming test with the rebuilt image.
|
||||
- Needs: golden image rebuild (new `orca-sandbox-start`), and Orca Cloud auth minting `org_role` on sandbox tokens.
|
||||
|
||||
## GitHub installation binding, guardrails, retention (2026-11-09)
|
||||
|
||||
- Repo tokens need a binding: `github_installations(installation_id, account_id)`. `tokenForRepo(url, accountId)`
|
||||
looks up the repo's installation, refuses unless the caller's account holds a live binding (re-checked on cached
|
||||
tokens too), mints for that one repo (contents/pull_requests write, metadata read) and rejects a wider response.
|
||||
Binding: desktop `POST /v1/github/connect` → signed (HMAC), single-use, 10-min state carrying the Orca account →
|
||||
GitHub install / OAuth → `/v1/github/setup` + `/v1/github/callback` exchange the code for a user-to-server token and
|
||||
bind only installations on the GitHub user's own account or on an org where they are an active admin
|
||||
(`/user/memberships/orgs`, needs the app's Members:read). Installation ids in URLs are never trusted.
|
||||
Operator binding for support/dev: `POST /v1/internal/github/bindings {installationId|repoUrl, accountId}`.
|
||||
- Guardrails before billing: kill switch (D1 flag via `POST /v1/internal/kill-switch`, or `SANDBOX_CREATES_DISABLED`)
|
||||
refuses create/resume/rollover with 503 `maintenance` and drains running sandboxes (suspend, snapshot kept; cron
|
||||
keeps draining while on); global `SANDBOX_MAX_RUNNING` (50, 503 `capacity`); per-account 3 × seats backstop
|
||||
(403 `concurrency_limit`); `CREATE_RATE_PER_MINUTE` (10, 429).
|
||||
- Hourly housekeeping: provider sandboxes tagged with this env and no live row are stopped on first sight and
|
||||
deleted after `ORPHAN_GRACE_HOURS` (24); automatic snapshots whose sandbox no longer exists anywhere in the project
|
||||
are deleted after the same grace (manual/golden never; partial listings delete nothing). Canceled/unpaid accounts
|
||||
(suspensions list + daily standing probe; outages never start a clock) get a warning hook at T−7 days and are
|
||||
destroyed `DATA_RETENTION_DAYS` (30) after; suspended sandboxes idle `IDLE_RETENTION_DAYS` (90) likewise. New
|
||||
sandboxes set `snapshotExpiration` to 104 days (backstop). Destroyed rows drop `repo_url`.
|
||||
- Live on dev: unbound account → private clone refused (no token minted); after operator binding → clone + push +
|
||||
gh/Node via proxy OK; kill switch drill drained 1, refused create/resume, resume OK after lifting.
|
||||
|
||||
## What needs live access (not done)
|
||||
|
||||
Vercel Pro team id + project id + a Sandbox-scoped token (as Worker secrets), wrangler auth for a dev D1, a
|
||||
|
||||
Reference in New Issue
Block a user