Merge remote-tracking branch 'private/cloud/integration' into cloud/owner-transfer-fix

# Conflicts:
#	cloud/apps/sandbox-control/src/index.ts
This commit is contained in:
Neil
2026-11-13 06:14:23 -08:00
12 changed files with 513 additions and 31 deletions
+12 -1
View File
@@ -255,6 +255,8 @@ export type BootSource = {
region: string
/** An automatic environment (auto-environments.ts): the caller's machine size stands. */
auto?: boolean
/** When the build finished (ms): a just-made snapshot gets one retry on a provider 404 (lifecycle.ts). */
builtAt?: number | null
}
type BootBuildRow = {
@@ -264,6 +266,7 @@ type BootBuildRow = {
status: string
snapshot_id: string | null
snapshot_deleted_at: number | null
finished_at: number | null
result: string | null
config: string
}
@@ -286,7 +289,15 @@ function bootFrom(build: BootBuildRow, repoUrl: string | null | undefined): Boot
} catch {
// Default size.
}
return { snapshotId: build.snapshot_id, environmentId: build.environment_id, buildId: build.id, repoDir: dir, machineSize, region: build.region }
return {
snapshotId: build.snapshot_id,
environmentId: build.environment_id,
buildId: build.id,
repoDir: dir,
machineSize,
region: build.region,
builtAt: build.finished_at
}
}
/**
+16 -2
View File
@@ -146,7 +146,7 @@ import {
runningSandboxesForDefault
} from './agent-sources'
import { EgressHostsSchema, egressUserHosts, normalizeUserHosts, setEgressUserHosts } from './egress-hosts'
import { gitBasicAuthorization, parseEgressPreset } from './network-policy'
import { egressProxyMissing, gitBasicAuthorization, parseEgressPreset } from './network-policy'
import { cachedProviderEnv, forgetProviderEnv } from './session-env'
import { credentialTargets, type CredentialTarget } from './provider-env'
import type { Keyring } from './secrets-crypto'
@@ -1296,6 +1296,11 @@ async function routeRequest(request: Request, env: Env, deps: Deps): Promise<Res
if (!env.VERCEL_TEAM_ID || !env.VERCEL_PROJECT_ID) {
return fail('not_configured', 503)
}
// Second lock (EGRESS-SWAP-FAIL-CLOSED): with no proxy URL configured nothing should forward here, and rows
// may carry withheld-key routes; never swap a real credential in that state.
if (egressProxyMissing({ egressProxyUrl: env.EGRESS_PROXY_URL, environment: env.SANDBOX_ENV })) {
return fail('egress_proxy_missing', 503)
}
return await handleEgressProxy(request, {
teamId: env.VERCEL_TEAM_ID,
projectId: env.VERCEL_PROJECT_ID,
@@ -1438,7 +1443,16 @@ async function routeRequest(request: Request, env: Env, deps: Deps): Promise<Res
if (url.pathname === '/v1/health' && request.method === 'GET') {
// Post-deploy smoke: no auth, no DB. 503 names a config error a deploy must not ship.
const platform = egressAllowlistSetting(env.ORCA_PLATFORM_HOST_SUFFIXES)
return platform.error ? platformHostsMisconfigured(platform.error) : json({ ok: true }, 200)
if (platform.error) {
return platformHostsMisconfigured(platform.error)
}
// EGRESS-SWAP-FAIL-CLOSED: without the proxy no key reaches an agent and open egress is off.
if (egressProxyMissing({ egressProxyUrl: env.EGRESS_PROXY_URL, environment: env.SANDBOX_ENV })) {
console.error(JSON.stringify({ event: 'config_error', variable: 'EGRESS_PROXY_URL', environment: env.SANDBOX_ENV }))
return json({ error: 'egress_proxy_missing', keySwap: 'off' }, 503)
}
// keySwap: whether model keys can be swapped at egress (the launch smoke asserts "on").
return json({ ok: true, keySwap: env.EGRESS_PROXY_URL ? 'on' : 'off' }, 200)
}
if (url.pathname === '/v1/config' && request.method === 'GET') {
// Public product constants (no auth): every client reads these instead of hardcoding them.
+82 -12
View File
@@ -13,7 +13,7 @@ import {
killSwitchOn,
type GuardrailConfig
} from './guardrails'
import { DEFAULT_EGRESS_ALLOWLIST, DEFAULT_EGRESS_PRESET, compileNetworkPolicy, gitBasicAuthorization, type CredentialRoute, type EgressPreset, type PlatformHostSuffixesConfigError } from './network-policy'
import { DEFAULT_EGRESS_ALLOWLIST, DEFAULT_EGRESS_PRESET, KEYS_WITHHELD_BODY, compileNetworkPolicy, egressProxyMissing, gitBasicAuthorization, type CredentialRoute, type EgressPreset, type PlatformHostSuffixesConfigError } from './network-policy'
import { applyQueuedRevokes } from './grant-revokes'
import { ensureEntryPoints } from './golden-images'
import { RUNTIME_REFRESH_SCRIPT, type RuntimeBundle } from './runtime-bundles'
@@ -35,7 +35,8 @@ import {
runningOrcad,
startOrcad,
type OrcadReady,
type StartInputs
type StartInputs,
START_TIMEOUT_MS
} from './sandbox-vm-commands'
import { WORKSPACE_PRELUDE_SCRIPT } from '../image/vm-scripts.generated'
import { resolveEnvironmentBoot, type BootSource } from './environments'
@@ -206,17 +207,41 @@ export type Ready = {
/** Phase durations for one lifecycle call (logged + Server-Timing); counts only, no ids or secrets. */
export function phaseTimer(now: () => number = Date.now) {
const timings: Record<string, number> = {}
return {
const timer = {
timings,
/** The first phase that threw, and how long it ran: names a create's failure (createFailureReason). */
failed: null as { phase: string; ms: number } | null,
async time<T>(phase: string, work: () => Promise<T>): Promise<T> {
const started = now()
try {
return await work()
} catch (error) {
timer.failed ??= { phase, ms: now() - started }
throw error
} finally {
timings[phase] = now() - started
}
}
}
return timer
}
/**
* CREATE-BUILD-SNAPSHOT-404: a VM create from an environment build's snapshot can answer 404 just
* after the build (dev, 11-11 and 11-13: the same snapshot booted seconds before and after). Once,
* after a short wait, for a build under this age; never for goldens, old builds or other errors.
*/
const FRESH_BUILD_SNAPSHOT_MS = 15 * 60_000
const FRESH_BUILD_RETRY_WAIT_MS = 5_000
function freshBuildSnapshot404(error: unknown, source: BootSource | null, now: number): source is BootSource {
return (
error instanceof ProviderError &&
error.status === 404 &&
!!source &&
typeof source.builtAt === 'number' &&
now - source.builtAt < FRESH_BUILD_SNAPSHOT_MS
)
}
type BootPrep = {
@@ -262,13 +287,12 @@ export function githubPlaceholderHeaders(row: SandboxRow): string[] {
*/
export function effectiveEgressPreset(deps: Pick<LifecycleDeps, 'config'>, row: Pick<SandboxRow, 'id'>): EgressPreset {
const preset = deps.config.egressPreset ?? DEFAULT_EGRESS_PRESET
if (preset === 'open' && !deps.config.egressProxyUrl && !LOCAL_ENVIRONMENTS.has(deps.config.environment)) {
if (preset === 'open' && egressProxyMissing(deps.config)) {
console.error(JSON.stringify({ event: 'egress_open_without_proxy', sandboxId: row.id, environment: deps.config.environment, applied: 'limited' }))
return 'limited'
}
return preset
}
const LOCAL_ENVIRONMENTS = new Set(['local', 'test'])
/** Refuses up front (503 platform_hosts_misconfigured) when this create or boot would compile a limited policy. */
export function assertEgressAllowlistUsable(deps: Pick<LifecycleDeps, 'config'>, row: Pick<SandboxRow, 'id'>): void {
@@ -338,7 +362,28 @@ export function egressPolicy(deps: LifecycleDeps, row: SandboxRow, gitToken?: st
)
)
]
: []
: // No proxy (EGRESS-SWAP-FAIL-CLOSED): keys were withheld and only placeholders reached the VM. The firewall
// answers them itself with a plain error, so agents say what happened instead of a provider auth failure.
[
...egressAccountRoutes(row).flatMap(({ agent, host }): CredentialRoute[] => {
const provider = accountProvider(agent)
return provider
? [{ host, header: provider.header, placeholders: [provider.format(provider.placeholder(row))], via: 'respond', statusCode: 403, body: KEYS_WITHHELD_BODY }]
: []
}),
...egressSecretHosts(row).flatMap((entry) =>
Object.keys(entry.headers).map(
(header): CredentialRoute => ({
host: entry.host,
header,
placeholders: placeholderValues(entry, header, secretPlaceholder(row, entry.secret)),
via: 'respond',
statusCode: 403,
body: KEYS_WITHHELD_BODY
})
)
)
]
const all = [...git, ...credentials]
if (effectiveEgressPreset(deps, row) === 'open') {
// Every host is reachable; the credential claims (and their trailing 403s) still apply.
@@ -819,7 +864,7 @@ export async function createSandbox(
} catch (error) {
// A pooled VM that will not take the session is discarded; the caller still gets a workspace.
console.log(JSON.stringify({ event: 'pool_adopt_failed', error: String(error).slice(0, 200) }))
await failAndRelease(deps, (await getSandbox(deps.db, row.id)) ?? row, createFailureReason(error))
await failAndRelease(deps, (await getSandbox(deps.db, row.id)) ?? row, createFailureReason(error, timer))
// A boot that ran long (the start command timed out) would only time out again on a fresh VM,
// doubling the wait the user sees: only a quick failure, which points at the pooled VM, retries.
if (deps.now() - claimedAt > POOL_RETRY_WITHIN_MS) {
@@ -834,7 +879,7 @@ export async function createSandbox(
let state: ProviderSandboxState
try {
// The full policy (repo token, model-key routes) goes in at create: no second policy round trip.
state = await timer.time('vm', () =>
const createVm = () =>
deps.provider.createSandbox({
name: row.provider_name,
snapshotId: bootSnapshotId,
@@ -846,7 +891,25 @@ export async function createSandbox(
snapshotExpirationMs: deps.config.snapshotExpirationMs ?? DEFAULT_SNAPSHOT_EXPIRATION_MS,
tags: { orcaSandbox: row.id, orcaEnv: deps.config.environment }
})
)
state = await timer.time('vm', async () => {
try {
return await createVm()
} catch (error) {
if (!freshBuildSnapshot404(error, source, deps.now())) {
throw error
}
const sleep = deps.sleep ?? ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)))
await sleep(FRESH_BUILD_RETRY_WAIT_MS)
try {
const retried = await createVm()
console.log(JSON.stringify({ event: 'fresh_build_snapshot_retry', buildId: source.buildId, ok: true }))
return retried
} catch (again) {
console.log(JSON.stringify({ event: 'fresh_build_snapshot_retry', buildId: source.buildId, ok: false, error: String(again).slice(0, 200) }))
throw again
}
}
})
} catch (error) {
await failAndRelease(deps, row, createFailureReason(error))
// Only a create nothing pinned to a region moves on: no explicit region, no environment build.
@@ -867,13 +930,17 @@ export async function createSandbox(
console.log(JSON.stringify({ event: 'lifecycle_timing', op: 'create', fromBuild: source?.buildId ?? null, ms: timer.timings }))
return cloning ? { ...ready, cloning } : ready
} catch (error) {
await failAndRelease(deps, (await getSandbox(deps.db, row.id)) ?? row, createFailureReason(error))
await failAndRelease(deps, (await getSandbox(deps.db, row.id)) ?? row, createFailureReason(error, timer))
throw error
}
}
/** last_error for a create that failed with `error`: the step that broke, never a message. */
export function createFailureReason(error: unknown): string {
/**
* last_error for a create that failed with `error`: the step that broke, never a message. An error of
* no known kind is named by the phase it broke (`timer`): orcad's start, at or past its 120 s budget,
* is `orcad_start_timeout` (CREATE-FAIL-LABELS), so D1 alone tells a hung start from the rest.
*/
export function createFailureReason(error: unknown, timer?: Pick<ReturnType<typeof phaseTimer>, 'failed'>): string {
if (error instanceof VmCommandError) {
return error.code
}
@@ -883,6 +950,9 @@ export function createFailureReason(error: unknown): string {
if (error instanceof ProviderError) {
return `provider_${error.status}`
}
if (timer?.failed?.phase === 'orcad') {
return timer.failed.ms >= START_TIMEOUT_MS ? 'orcad_start_timeout' : 'orcad_start_error'
}
return 'create_failed'
}
@@ -15,7 +15,7 @@ export type NetworkPolicyRule =
transform: { headers: Record<string, string> }[]
}
| { match?: { headers?: { key: Matcher; value: Matcher }[] }; forwardURL: string }
| { response: { statusCode: number; body: string; contentType: string } }
| { match?: { headers?: { key: Matcher; value: Matcher }[] }; response: { statusCode: number; body: string; contentType: string } }
export type NetworkPolicy = { allow: Record<string, NetworkPolicyRule[]> }
@@ -166,6 +166,25 @@ export function parseEgressPreset(value: string | undefined): EgressPreset {
return preset === 'open' ? 'open' : 'limited'
}
/** Environments where running without the egress proxy is expected (dev loops, the unit harness). */
export function isLocalEnvironment(environment: string | undefined): boolean {
return environment === 'local' || environment === 'test'
}
/**
* A deployed env with no EGRESS_PROXY_URL: nothing can swap credentials at egress, so open egress falls back
* to limited (E3-PROD-GUARD) and swappable keys are withheld from the VM (EGRESS-SWAP-FAIL-CLOSED).
*/
export function egressProxyMissing(config: { egressProxyUrl?: string; environment?: string }): boolean {
return !config.egressProxyUrl && !isLocalEnvironment(config.environment)
}
/** What an agent sees when its model key was withheld (design-squad wording). */
export const KEYS_WITHHELD_MESSAGE =
"Orca Cloud didn't send your model API key or sign-in because of a problem on Orca's side, not with your key. Contact Orca support."
/** Provider-style error JSON: Anthropic clients read error.message, OpenAI-compatible ones too. */
export const KEYS_WITHHELD_BODY = JSON.stringify({ type: 'error', error: { type: 'permission_error', message: KEYS_WITHHELD_MESSAGE } })
export type CredentialRoute = {
host: string
/** Header the VM sends with a placeholder value, matched exactly (one claim rule per value). */
@@ -176,6 +195,8 @@ export type CredentialRoute = {
} & (
| { via: 'transform'; value: string }
| { via: 'forward'; forwardURL: string }
/** Answered by Vercel's firewall itself; nothing reaches the host (EGRESS-SWAP-FAIL-CLOSED). */
| { via: 'respond'; statusCode: number; body: string }
)
export type EgressPolicyInput = {
@@ -206,9 +227,13 @@ export function compileNetworkPolicy(input: EgressPolicyInput): NetworkPolicy {
const header = route.header.toLowerCase()
const claims = route.placeholders.map((placeholder): NetworkPolicyRule => {
const match = { headers: [{ key: { exact: header }, value: { exact: placeholder } }] }
return route.via === 'transform'
? { match, transform: [{ headers: { [header]: route.value } }] }
: { match, forwardURL: route.forwardURL }
if (route.via === 'transform') {
return { match, transform: [{ headers: { [header]: route.value } }] }
}
if (route.via === 'respond') {
return { match, response: { statusCode: route.statusCode, body: route.body, contentType: 'application/json' } }
}
return { match, forwardURL: route.forwardURL }
})
allow[host] = [...(allow[host] ?? []), ...claims]
// One route that wants the deny is enough: a shared host fails closed.
@@ -11,7 +11,7 @@ export const START_COMMAND = '/opt/orca/bin/orca-sandbox-start'
export const DRAIN_COMMAND = '/opt/orca/bin/orca-sandbox-drain'
export const ENV_COMMAND = '/opt/orca/bin/orca-sandbox-env'
const ENV_TIMEOUT_MS = 30_000
const START_TIMEOUT_MS = 120_000
export const START_TIMEOUT_MS = 120_000
/** Matches the parity plan: in-flight turns get up to 5 min to become durable restart offers. */
export const DRAIN_TIMEOUT_MS = 5 * 60_000
+4 -4
View File
@@ -63,8 +63,8 @@ function serializable(value: unknown): unknown {
return value
}
export type SecretLogSink = (line: string) => void
let sink: SecretLogSink = (line) => console.log(line)
export type SecretLogSink = (line: string, level?: 'error') => void
let sink: SecretLogSink = (line, level) => (level === 'error' ? console.error(line) : console.log(line))
/** Tests only: capture lines exactly as they would reach Workers logs. Returns a restore function. */
export function setSecretLogSink(next: SecretLogSink): () => void {
@@ -76,12 +76,12 @@ export function setSecretLogSink(next: SecretLogSink): () => void {
}
/** One structured log line. `known`: values in scope at the call site that must never be written. */
export function secretLog(event: string, fields: Record<string, unknown> = {}, known: Iterable<string> = []): void {
export function secretLog(event: string, fields: Record<string, unknown> = {}, known: Iterable<string> = [], level?: 'error'): void {
let line: string
try {
line = JSON.stringify({ event, ...fields }, (_key, value: unknown) => serializable(value))
} catch {
line = JSON.stringify({ event, unserializable: true })
}
sink(redactLine(line, known))
sink(redactLine(line, known), level)
}
+55 -5
View File
@@ -5,7 +5,8 @@ import type { LifecycleDeps } from './lifecycle'
import { agentModesFor, withoutSignInCredentials, type AgentMode } from './agent-sources'
import { accountProvider } from './account-providers'
import type { CredentialAgent } from './agent-credential-names'
import { deriveProviderEnv, deriveProviderScopes, providerHosts } from './provider-env'
import { SWAPPED_CREDENTIALS, deriveProviderEnv, deriveProviderScopes, providerHosts } from './provider-env'
import { egressProxyMissing } from './network-policy'
import { applySecretPlaceholders, encodeAccountRoute, encodeEgressSecretHosts } from './secret-egress'
import { resolveSandboxSecrets } from './secrets'
import type { Keyring } from './secrets-crypto'
@@ -70,15 +71,35 @@ export function forgetProviderEnv(sandboxId: string): void {
}
}
function egressSwapOn(deps: Pick<LifecycleDeps, 'secrets' | 'config'>, row: Pick<SandboxRow, 'account_id'>): boolean {
/** Placeholders are switched on for this account (SECRETS_EGRESS_PLACEHOLDERS, minus the per-team kill switch). */
function egressSwapWanted(deps: Pick<LifecycleDeps, 'secrets'>, row: Pick<SandboxRow, 'account_id'>): boolean {
const placeholders = deps.secrets?.egressPlaceholders
return (
!deps.secrets?.egressPlaceholdersOff?.has(row.account_id) &&
(placeholders === true || (placeholders instanceof Set && placeholders.has(row.account_id))) &&
Boolean(deps.config.egressProxyUrl)
(placeholders === true || (placeholders instanceof Set && placeholders.has(row.account_id)))
)
}
function egressSwapOn(deps: Pick<LifecycleDeps, 'secrets' | 'config'>, row: Pick<SandboxRow, 'account_id'>): boolean {
return egressSwapWanted(deps, row) && Boolean(deps.config.egressProxyUrl)
}
// One log line per session, not per env read (boot, refresh and the proxy's cache all land here). The boot read
// happens before the session exists, so its mark carries over to the session it starts.
const withheldLogged = new Set<string>()
function firstReadThisSession(kind: 'withheld' | 'plain', row: Pick<SandboxRow, 'id' | 'session_id'>): boolean {
const pending = `${kind}:${row.id}:pending`
if (!row.session_id) {
return withheldLogged.has(pending) ? false : (withheldLogged.add(pending), true)
}
const key = `${kind}:${row.id}:${row.session_id}`
if (withheldLogged.has(key)) {
return false
}
withheldLogged.add(key)
return !withheldLogged.delete(pending)
}
export type { AgentMode }
export async function sessionSecretEnv(
@@ -93,11 +114,15 @@ export async function sessionSecretEnv(
}
const { env: stored, scopes: storedScopes, skipped } = await resolveSandboxSecrets(deps.db, deps.secrets.keyring, row)
const swap = egressSwapOn(deps, row)
// EGRESS-SWAP-FAIL-CLOSED: a deployed env without the egress proxy can't swap keys at egress. Rather than
// fall back to real keys in the VM, keys become placeholders that the firewall answers with a plain error,
// and an Orca-account sign-in gets its placeholder answered the same way (the real token never leaves us).
const withhold = !swap && egressSwapWanted(deps, row) && egressProxyMissing(deps.config)
const derived = deriveProviderEnv(stored)
// D10: per agent, the owner's account (placeholder), the kill-switch fallback, or the keys.
const modes = await agentModesFor(deps.db, row, derived, {
providerEnabled: deps.secrets.accountProviders ?? { claude: false, codex: false },
accountsUsable: swap,
accountsUsable: swap || withhold,
now: Date.now(),
// Usable means decryptable now, not just stored: a token we can't open must not hold the session in account mode.
readable: async (agent) => {
@@ -111,7 +136,32 @@ export async function sessionSecretEnv(
const egressAllowHosts = providerHosts(env).join(',') || null
// Counts only.
secretLog('session_secrets', { sandboxId: row.id, injected: Object.keys(env).length, skipped })
if (withhold) {
const withheldSecrets = Object.fromEntries(SWAPPED_CREDENTIALS.filter((secret) => secret in env).map((secret) => [secret, true]))
if (firstReadThisSession('withheld', row)) {
// Counts only: no secret names or values.
secretLog('egress_swap_unavailable', { sandboxId: row.id, environment: deps.config.environment, withheld: Object.keys(withheldSecrets).length }, [], 'error')
}
const withheld = applySecretPlaceholders(row, env, scopes)
const accountEnv: Record<string, string> = {}
const accountRoutes: string[] = []
for (const agent of Object.keys(modes) as CredentialAgent[]) {
const provider = modes[agent] === 'account' ? accountProvider(agent) : undefined
if (provider) {
Object.assign(accountEnv, provider.sessionEnv(row))
accountRoutes.push(...provider.hosts.map((host) => encodeAccountRoute(agent, row.user_id, host)))
}
}
const hosts = [encodeEgressSecretHosts(withheld.targets), ...accountRoutes].filter(Boolean).join(',') || null
return { env: { ...withheld.env, ...accountEnv }, egressSecretHosts: hosts, egressAllowHosts }
}
if (!swap) {
// An operator turned placeholders off for this account (SECRETS_EGRESS_PLACEHOLDERS=off or accounts:<ids>
// without it, or SECRETS_EGRESS_PLACEHOLDERS_OFF): real keys in the VM are deliberate, but never silent.
const plain = SWAPPED_CREDENTIALS.filter((secret) => secret in env).length
if (plain > 0 && !egressSwapWanted(deps, row) && firstReadThisSession('plain', row)) {
secretLog('egress_swap_disabled', { sandboxId: row.id, plainKeys: plain })
}
return { env, egressSecretHosts: null, egressAllowHosts }
}
const swapped = applySecretPlaceholders(row, env, scopes)
+7 -1
View File
@@ -269,7 +269,13 @@ export async function updateRegistry(db: D1Database, id: string, patch: Registry
*/
export async function transferOwnership(db: D1Database, id: string, newOwnerId: string, now: number): Promise<SandboxRow | null> {
const result = await db
.prepare(`UPDATE sandboxes SET user_id = ?, ownership_transferred_at = ? WHERE id = ? AND status IN ('suspended', 'archived')`)
// Unlike other registry edits this bumps `generation` on purpose: a wake that read the row just
// before (still the previous owner's, unmarked) must lose its compare-and-set, not boot a
// private environment for them under the new owner.
.prepare(
`UPDATE sandboxes SET user_id = ?, ownership_transferred_at = ?, generation = generation + 1
WHERE id = ? AND status IN ('suspended', 'archived')`
)
.bind(newOwnerId, now, id)
.run()
return (result.meta.changes ?? 0) > 0 ? getSandbox(db, id) : null
@@ -0,0 +1,139 @@
// CREATE-BUILD-SNAPSHOT-404: on dev a VM create from an environment build's snapshot answered 404
// shortly after the build (the same snapshot booted seconds before and after). One retry, after a
// short wait, for a build under 15 min old; never for goldens, older builds or other errors.
// CREATE-FAIL-LABELS: a create that broke in orcad's start with an error of no known kind is named
// by that step (orcad_start_timeout past the 120 s budget), not the generic create_failed.
import { describe, expect, it } from 'vitest'
import { createFailureReason, phaseTimer } from '../src/lifecycle'
import { ProviderError } from '../src/sandbox-provider'
import { START_TIMEOUT_MS, VmCommandError } from '../src/sandbox-vm-commands'
import { GOLDEN, createHarness } from './harness'
const REPO = 'https://github.com/acme/web.git'
const RESULT = JSON.stringify({
durationSec: 3,
repos: [{ url: REPO, dir: 'acme/web', branch: 'main', commit: 'a'.repeat(40), lockfiles: {} }],
diskUsedBytes: 1,
diskTotalBytes: 2
})
async function setup() {
const sleeps: number[] = []
const h = await createHarness({}, { sleep: async (ms) => void sleeps.push(ms) })
h.vercel.buildResult = RESULT
// A fresh token per call: some tests move the clock past a token's 15 min age.
const call = async (method: string, path: string, body?: unknown) =>
h.call(method, path, { bearer: await h.token('alice', 'acme', 'sandbox', { orgRole: 'admin' }), body })
const env = await call('POST', '/v1/environments', { name: 'Default', repos: [{ url: 'acme/web', defaultBranch: 'main' }], installScript: 'true' })
const envId: string = env.body.environment.id
const started = await call('POST', `/v1/environments/${envId}/builds`)
const command = [...h.vercel.buildCommands.values()].at(-1)!
command.logs = `${JSON.stringify({ data: '::orca-step:: install\nok\n', stream: 'stdout' })}\n`
command.exitCode = 0
h.advance(1000)
const build = (await call('GET', `/v1/environments/${envId}/builds/${started.body.build.id}`)).body.build
expect(build.status).toBe('succeeded')
// The provider answers `failures` (in order) to the next VM creates, then creates normally.
const failures: number[] = []
const snapshotsTried: string[] = []
const createVm = h.vercel.createSandbox.bind(h.vercel)
h.vercel.createSandbox = async (params) => {
if (params.name.includes('smoke') || params.name.includes('envbuild')) {
return createVm(params)
}
snapshotsTried.push(params.snapshotId)
const status = failures.shift()
if (status) {
throw new ProviderError(status, 'provider_failed')
}
return createVm(params)
}
const create = (workspaceId: string, repo = true) =>
call('POST', '/v1/sandboxes', { workspaceId, machineSize: 'standard', ...(repo ? { repo: { url: REPO } } : {}) })
const lastError = (workspaceId: string) =>
(h.raw.prepare('SELECT last_error FROM sandboxes WHERE workspace_id = ? ORDER BY rowid DESC').get(workspaceId) as { last_error: string | null }).last_error
return { h, build, failures, snapshotsTried, sleeps, create, lastError }
}
describe('CREATE-BUILD-SNAPSHOT-404', () => {
it('a just-built snapshot that answers 404 is retried once after ~5 s, and the create succeeds', async () => {
const t = await setup()
t.failures.push(404)
const created = await t.create('ws-fresh')
expect(created.status).toBe(200)
expect(created.body.sandbox.environmentBuildId).toBe(t.build.id)
expect(t.sleeps).toEqual([5_000])
expect(t.snapshotsTried).toHaveLength(2)
expect(t.snapshotsTried[0]).toBe(t.snapshotsTried[1])
expect(t.snapshotsTried[0]).not.toBe(GOLDEN)
})
it('only once: a second 404 fails the create as before (provider_404)', async () => {
const t = await setup()
t.failures.push(404, 404)
expect((await t.create('ws-twice')).status).not.toBe(200)
expect(t.sleeps).toEqual([5_000])
expect(t.snapshotsTried).toHaveLength(2)
expect(t.lastError('ws-twice')).toBe('provider_404')
})
it('no retry for a build older than 15 min', async () => {
const t = await setup()
t.h.advance(16 * 60_000)
t.failures.push(404)
expect((await t.create('ws-old')).status).not.toBe(200)
expect(t.sleeps).toEqual([])
expect(t.snapshotsTried).toHaveLength(1)
})
it('no retry for other errors (403) on a fresh build', async () => {
const t = await setup()
t.failures.push(403)
expect((await t.create('ws-403')).status).not.toBe(200)
expect(t.sleeps).toEqual([])
expect(t.lastError('ws-403')).toBe('provider_403')
})
it('no retry for the golden image', async () => {
const t = await setup()
t.failures.push(404)
expect((await t.create('ws-golden', false)).status).not.toBe(200)
expect(t.snapshotsTried[0]).toBe(GOLDEN)
expect(t.sleeps).toEqual([])
expect(t.snapshotsTried).toHaveLength(1)
})
})
describe('CREATE-FAIL-LABELS', () => {
const failIn = async (phase: string, ms: number) => {
let clock = 0
const timer = phaseTimer(() => clock)
await timer
.time(phase, async () => {
clock += ms
throw new TypeError('Network connection lost.')
})
.catch(() => undefined)
return timer
}
it("names an unknown error in orcad's start by the step: a timeout at the 120 s budget, an error before it", async () => {
expect(createFailureReason(new TypeError('x'), await failIn('orcad', START_TIMEOUT_MS + 7_000))).toBe('orcad_start_timeout')
expect(createFailureReason(new TypeError('x'), await failIn('orcad', 2_000))).toBe('orcad_start_error')
})
it('keeps the known kinds and the generic label elsewhere', async () => {
const orcad = await failIn('orcad', START_TIMEOUT_MS)
expect(createFailureReason(new VmCommandError('orcad_start_failed'), orcad)).toBe('orcad_start_failed')
expect(createFailureReason(new ProviderError(404, 'gone'), orcad)).toBe('provider_404')
expect(createFailureReason(new TypeError('x'), await failIn('vm', 1_000))).toBe('create_failed')
expect(createFailureReason(new TypeError('x'))).toBe('create_failed')
})
it('the timer records only the first phase that threw, and keeps timings numeric (Server-Timing)', async () => {
const timer = await failIn('orcad', 5)
await timer.time('later', async () => Promise.reject(new Error('x'))).catch(() => undefined)
expect(timer.failed).toEqual({ phase: 'orcad', ms: 5 })
expect(Object.values(timer.timings).every((v) => typeof v === 'number')).toBe(true)
})
})
@@ -0,0 +1,161 @@
// EGRESS-SWAP-FAIL-CLOSED: a deployed env (not local/test) without EGRESS_PROXY_URL can't swap keys at egress.
// Model keys and Orca-account sign-ins are withheld (the VM gets placeholders, the firewall answers them with a
// plain error), the session logs egress_swap_unavailable once (error level, counts only), and /v1/health fails the
// post-deploy smoke with keySwap "off".
// Local and test keep today's behaviour.
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { KEYS_WITHHELD_MESSAGE, compileNetworkPolicy } from '../src/network-policy'
import { START_COMMAND } from '../src/sandbox-vm-commands'
import { createHarness } from './harness'
const KEK = Buffer.alloc(32, 7).toString('base64')
const KEY = 'sk-ant-api03-WITHHELD-fake-0000-aaaa'
const SETUP_TOKEN = 'sk-ant-oat01-FAKEsetupTOKEN_for_tests_only_000000000000'
let logs: string[] = []
beforeEach(() => {
logs = []
errors = []
for (const method of ['log', 'info', 'warn', 'error', 'debug'] as const) {
vi.spyOn(console, method).mockImplementation((...args: unknown[]) => {
const line = args.map((arg) => (typeof arg === 'string' ? arg : JSON.stringify(arg))).join(' ')
logs.push(line)
if (method === 'error') {
errors.push(line)
}
})
}
})
afterEach(() => {
expect(logs.join('\n')).not.toContain(KEY)
expect(logs.join('\n')).not.toContain(SETUP_TOKEN)
vi.restoreAllMocks()
})
async function setup(env: Record<string, string>) {
const h = await createHarness({ SECRETS_KEK: KEK, SECRETS_EGRESS_PLACEHOLDERS: 'on', AGENT_ACCOUNTS_CLAUDE: 'on', ...env })
const mia = await h.token('mia', undefined, 'sandbox')
return { h, mia }
}
function startEnv(h: Awaited<ReturnType<typeof createHarness>>): Record<string, string> {
const start = h.vercel.commands.filter((c) => c.command === START_COMMAND).at(-1)!
return Object.fromEntries(
(start.env.ORCA_SECRET_ENV ?? '')
.split('\n')
.filter(Boolean)
.map((line) => [line.slice(0, line.indexOf('=')), Buffer.from(line.slice(line.indexOf('=') + 1), 'base64').toString('utf8')])
)
}
let errors: string[] = []
const withheldEvents = () => errors.filter((line) => line.includes('"event":"egress_swap_unavailable"'))
describe('EGRESS-SWAP-FAIL-CLOSED', () => {
it('a deployed env without the proxy withholds model keys: placeholder in the VM, the firewall answers it with a plain error', async () => {
const { h, mia } = await setup({ SANDBOX_ENV: 'staging' })
await h.call('PUT', '/v1/secrets/personal/ANTHROPIC_API_KEY', { body: { value: KEY }, bearer: mia })
const created = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard' }, bearer: mia })
expect(created.status).toBe(200)
const id: string = created.body.sandbox.id
const env = startEnv(h)
expect(env.ANTHROPIC_API_KEY).toBeDefined()
expect(env.ANTHROPIC_API_KEY).not.toBe(KEY)
expect(JSON.stringify(h.vercel.commands)).not.toContain(KEY)
const box = [...h.vercel.boxes.values()][0]!
expect(JSON.stringify(box.policy)).not.toContain(KEY)
// Limited (E3-PROD-GUARD), and the key's host answers its placeholder itself.
expect(box.policy.allow['*']).toBeUndefined()
const rules = box.policy.allow['api.anthropic.com']!
const answer = rules.find((rule) => 'response' in rule && 'match' in rule && rule.match)
expect(answer).toMatchObject({
match: { headers: [{ key: { exact: 'x-api-key' }, value: { exact: env.ANTHROPIC_API_KEY } }] },
response: { statusCode: 403, contentType: 'application/json' }
})
expect(JSON.parse((answer as { response: { body: string } }).response.body).error.message).toBe(KEYS_WITHHELD_MESSAGE)
expect(rules.at(-1)).toMatchObject({ response: { statusCode: 403, body: 'orca-egress-denied' } })
// Logged once for the session at error level, however many times its env is read: a count, no names.
expect(withheldEvents()).toHaveLength(1)
expect(JSON.parse(withheldEvents()[0]!)).toEqual({ event: 'egress_swap_unavailable', sandboxId: id, environment: 'staging', withheld: 1 })
expect(withheldEvents()[0]).not.toContain('ANTHROPIC')
await h.call('POST', `/v1/sandboxes/${id}/env/refresh`, { bearer: mia })
expect(withheldEvents()).toHaveLength(1)
})
it('an Orca-account sign-in is withheld the same way: account placeholder only, answered by the firewall', async () => {
const { h, mia } = await setup({ SANDBOX_ENV: 'staging' })
expect((await h.call('PUT', '/v1/agent-credentials/claude', { bearer: mia, body: { token: SETUP_TOKEN } })).status).toBe(200)
await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard' }, bearer: mia })
const env = startEnv(h)
expect(env.CLAUDE_CODE_OAUTH_TOKEN).toMatch(/^orca-ph-claude-account-/)
const box = [...h.vercel.boxes.values()][0]!
expect(JSON.stringify(box.policy)).not.toContain(SETUP_TOKEN)
expect(box.policy.allow['api.anthropic.com']![0]).toMatchObject({
match: { headers: [{ key: { exact: 'authorization' }, value: { exact: `Bearer ${env.CLAUDE_CODE_OAUTH_TOKEN}` } }] },
response: { statusCode: 403 }
})
})
it('a deployed env with the proxy behaves as today: placeholder forwarded to the proxy, no event', async () => {
const { h, mia } = await setup({ SANDBOX_ENV: 'staging', EGRESS_PROXY_URL: 'https://sandbox.test/v1/egress/proxy' })
await h.call('PUT', '/v1/secrets/personal/ANTHROPIC_API_KEY', { body: { value: KEY }, bearer: mia })
await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard' }, bearer: mia })
const env = startEnv(h)
expect(env.ANTHROPIC_API_KEY).not.toBe(KEY)
const box = [...h.vercel.boxes.values()][0]!
expect(box.policy.allow['*']).toEqual([])
expect(box.policy.allow['api.anthropic.com']![0]).toMatchObject({ forwardURL: 'https://sandbox.test/v1/egress/proxy' })
expect(JSON.stringify(box.policy)).not.toContain(KEYS_WITHHELD_MESSAGE)
expect(withheldEvents()).toHaveLength(0)
})
it('test and local keep today’s behaviour without a proxy: plain env injection, no event', async () => {
const { h, mia } = await setup({ SANDBOX_ENV: 'test' })
await h.call('PUT', '/v1/secrets/personal/ANTHROPIC_API_KEY', { body: { value: KEY }, bearer: mia })
await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard' }, bearer: mia })
expect(startEnv(h).ANTHROPIC_API_KEY).toBe(KEY)
expect(withheldEvents()).toHaveLength(0)
// The plain key appears in the env on purpose here; keep it out of the afterEach log check.
logs = logs.filter((line) => !line.includes(KEY))
})
it('placeholders turned off by an operator: plain keys as before, but logged once per session (count only)', async () => {
const { h, mia } = await setup({ SANDBOX_ENV: 'staging', EGRESS_PROXY_URL: 'https://sandbox.test/v1/egress/proxy', SECRETS_EGRESS_PLACEHOLDERS: 'off' })
await h.call('PUT', '/v1/secrets/personal/ANTHROPIC_API_KEY', { body: { value: KEY }, bearer: mia })
const created = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard' }, bearer: mia })
expect(startEnv(h).ANTHROPIC_API_KEY).toBe(KEY)
await h.call('POST', `/v1/sandboxes/${created.body.sandbox.id}/env/refresh`, { bearer: mia })
const disabled = logs.filter((line) => line.includes('"event":"egress_swap_disabled"'))
expect(disabled).toHaveLength(1)
expect(JSON.parse(disabled[0]!)).toEqual({ event: 'egress_swap_disabled', sandboxId: created.body.sandbox.id, plainKeys: 1 })
logs = logs.filter((line) => !line.includes(KEY))
})
it('the egress proxy refuses outright in a deployed env without its URL (second lock)', async () => {
const { h } = await setup({ SANDBOX_ENV: 'staging', VERCEL_TEAM_ID: 'team_x', VERCEL_PROJECT_ID: 'prj_x' })
const res = await h.send(new Request('https://sandbox.test/v1/egress/proxy/v1/messages', { method: 'POST', headers: { 'vercel-forwarded-host': 'api.anthropic.com' } }))
expect(res.status).toBe(503)
expect(await res.json()).toMatchObject({ error: 'egress_proxy_missing' })
})
it('/v1/health is 503 egress_proxy_missing in a deployed env without the proxy, 200 with it or in local/test', async () => {
const missing = await setup({ SANDBOX_ENV: 'staging' })
expect(await missing.h.call('GET', '/v1/health')).toEqual({ status: 503, body: { error: 'egress_proxy_missing', keySwap: 'off' } })
const present = await setup({ SANDBOX_ENV: 'staging', EGRESS_PROXY_URL: 'https://sandbox.test/v1/egress/proxy' })
expect(await present.h.call('GET', '/v1/health')).toEqual({ status: 200, body: { ok: true, keySwap: 'on' } })
const local = await setup({ SANDBOX_ENV: 'test' })
expect(await local.h.call('GET', '/v1/health')).toEqual({ status: 200, body: { ok: true, keySwap: 'off' } })
})
it('a respond route compiles to a matched direct response, before the trailing deny', () => {
const policy = compileNetworkPolicy({
mode: 'allowlist',
allowlist: [],
credentials: [{ host: 'api.openai.com', header: 'Authorization', placeholders: ['Bearer orca-ph-x'], via: 'respond', statusCode: 403, body: '{"e":1}' }]
})
expect(policy.allow['api.openai.com']).toEqual([
{ match: { headers: [{ key: { exact: 'authorization' }, value: { exact: 'Bearer orca-ph-x' } }] }, response: { statusCode: 403, body: '{"e":1}', contentType: 'application/json' } },
{ response: { statusCode: 403, body: 'orca-egress-denied', contentType: 'text/plain' } }
])
})
})
@@ -408,7 +408,13 @@ describe('a hand-off to a teammate', () => {
expect(await transferOwnership(t.h.env.SANDBOX_DB, t.id, 'mia', t.h.now())).toBeNull()
expect(await t.row()).toMatchObject({ user_id: 'mallory', ownership_transferred_at: null })
t.h.raw.prepare("UPDATE sandboxes SET status = 'suspended' WHERE id = ?").run(t.id)
const before = await t.row()
expect(await transferOwnership(t.h.env.SANDBOX_DB, t.id, 'mia', t.h.now())).toMatchObject({ user_id: 'mia', ownership_transferred_at: t.h.now() })
// A wake that read the row before the hand-off loses its compare-and-set afterwards.
expect((await t.row()).generation).toBe(before.generation + 1)
const { transition } = await import('../src/store')
expect(await transition(t.h.env.SANDBOX_DB, before, { status: 'resuming' }, t.h.now())).toBeNull()
expect((await t.row()).status).toBe('suspended')
})
it('naming the current owner again is not a transfer', async () => {
@@ -18,7 +18,7 @@ async function limited() {
describe('a malformed ORCA_PLATFORM_HOST_SUFFIXES (PLATFORM-SUFFIX-GUARDS)', () => {
it('fails the health route a post-deploy smoke hits, with the code only; the log names the bad entry', async () => {
const h = await createHarness()
expect(await h.call('GET', '/v1/health')).toEqual({ status: 200, body: { ok: true } })
expect(await h.call('GET', '/v1/health')).toMatchObject({ status: 200, body: { ok: true } })
h.env.ORCA_PLATFORM_HOST_SUFFIXES = MALFORMED
const logged = vi.spyOn(console, 'error').mockImplementation(() => undefined)
try {