From a50afdc731c2b6a9dde58915712d6b8293a03eab Mon Sep 17 00:00:00 2001 From: Neil Date: Fri, 13 Nov 2026 05:51:24 -0800 Subject: [PATCH 1/4] fix(cloud): retry a fresh build snapshot's 404 once; name orcad start failures (CREATE-BUILD-SNAPSHOT-404, CREATE-FAIL-LABELS) --- .../apps/sandbox-control/src/environments.ts | 13 +- cloud/apps/sandbox-control/src/lifecycle.ts | 66 ++++++++- .../src/sandbox-vm-commands.ts | 2 +- .../test/create-build-snapshot-404.test.ts | 139 ++++++++++++++++++ 4 files changed, 210 insertions(+), 10 deletions(-) create mode 100644 cloud/apps/sandbox-control/test/create-build-snapshot-404.test.ts diff --git a/cloud/apps/sandbox-control/src/environments.ts b/cloud/apps/sandbox-control/src/environments.ts index 25a37008848..fa130c3f9a6 100644 --- a/cloud/apps/sandbox-control/src/environments.ts +++ b/cloud/apps/sandbox-control/src/environments.ts @@ -255,6 +255,8 @@ export type BootSource = { region: string /** An automatic environment (auto-environments.ts): the caller's machine size stands. */ auto?: boolean + /** When the build finished (ms): a just-made snapshot gets one retry on a provider 404 (lifecycle.ts). */ + builtAt?: number | null } type BootBuildRow = { @@ -264,6 +266,7 @@ type BootBuildRow = { status: string snapshot_id: string | null snapshot_deleted_at: number | null + finished_at: number | null result: string | null config: string } @@ -286,7 +289,15 @@ function bootFrom(build: BootBuildRow, repoUrl: string | null | undefined): Boot } catch { // Default size. } - return { snapshotId: build.snapshot_id, environmentId: build.environment_id, buildId: build.id, repoDir: dir, machineSize, region: build.region } + return { + snapshotId: build.snapshot_id, + environmentId: build.environment_id, + buildId: build.id, + repoDir: dir, + machineSize, + region: build.region, + builtAt: build.finished_at + } } /** diff --git a/cloud/apps/sandbox-control/src/lifecycle.ts b/cloud/apps/sandbox-control/src/lifecycle.ts index ed5426094a7..ba974703deb 100644 --- a/cloud/apps/sandbox-control/src/lifecycle.ts +++ b/cloud/apps/sandbox-control/src/lifecycle.ts @@ -35,7 +35,8 @@ import { runningOrcad, startOrcad, type OrcadReady, - type StartInputs + type StartInputs, + START_TIMEOUT_MS } from './sandbox-vm-commands' import { WORKSPACE_PRELUDE_SCRIPT } from '../image/vm-scripts.generated' import { resolveEnvironmentBoot, type BootSource } from './environments' @@ -206,17 +207,41 @@ export type Ready = { /** Phase durations for one lifecycle call (logged + Server-Timing); counts only, no ids or secrets. */ export function phaseTimer(now: () => number = Date.now) { const timings: Record = {} - return { + const timer = { timings, + /** The first phase that threw, and how long it ran: names a create's failure (createFailureReason). */ + failed: null as { phase: string; ms: number } | null, async time(phase: string, work: () => Promise): Promise { const started = now() try { return await work() + } catch (error) { + timer.failed ??= { phase, ms: now() - started } + throw error } finally { timings[phase] = now() - started } } } + return timer +} + +/** + * CREATE-BUILD-SNAPSHOT-404: a VM create from an environment build's snapshot can answer 404 just + * after the build (dev, 11-11 and 11-13: the same snapshot booted seconds before and after). Once, + * after a short wait, for a build under this age; never for goldens, old builds or other errors. + */ +const FRESH_BUILD_SNAPSHOT_MS = 15 * 60_000 +const FRESH_BUILD_RETRY_WAIT_MS = 5_000 + +function freshBuildSnapshot404(error: unknown, source: BootSource | null, now: number): source is BootSource { + return ( + error instanceof ProviderError && + error.status === 404 && + !!source && + typeof source.builtAt === 'number' && + now - source.builtAt < FRESH_BUILD_SNAPSHOT_MS + ) } type BootPrep = { @@ -816,7 +841,7 @@ export async function createSandbox( } catch (error) { // A pooled VM that will not take the session is discarded; the caller still gets a workspace. console.log(JSON.stringify({ event: 'pool_adopt_failed', error: String(error).slice(0, 200) })) - await failAndRelease(deps, (await getSandbox(deps.db, row.id)) ?? row, createFailureReason(error)) + await failAndRelease(deps, (await getSandbox(deps.db, row.id)) ?? row, createFailureReason(error, timer)) // A boot that ran long (the start command timed out) would only time out again on a fresh VM, // doubling the wait the user sees: only a quick failure, which points at the pooled VM, retries. if (deps.now() - claimedAt > POOL_RETRY_WITHIN_MS) { @@ -831,7 +856,7 @@ export async function createSandbox( let state: ProviderSandboxState try { // The full policy (repo token, model-key routes) goes in at create: no second policy round trip. - state = await timer.time('vm', () => + const createVm = () => deps.provider.createSandbox({ name: row.provider_name, snapshotId: bootSnapshotId, @@ -843,7 +868,25 @@ export async function createSandbox( snapshotExpirationMs: deps.config.snapshotExpirationMs ?? DEFAULT_SNAPSHOT_EXPIRATION_MS, tags: { orcaSandbox: row.id, orcaEnv: deps.config.environment } }) - ) + state = await timer.time('vm', async () => { + try { + return await createVm() + } catch (error) { + if (!freshBuildSnapshot404(error, source, deps.now())) { + throw error + } + const sleep = deps.sleep ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))) + await sleep(FRESH_BUILD_RETRY_WAIT_MS) + try { + const retried = await createVm() + console.log(JSON.stringify({ event: 'fresh_build_snapshot_retry', buildId: source.buildId, ok: true })) + return retried + } catch (again) { + console.log(JSON.stringify({ event: 'fresh_build_snapshot_retry', buildId: source.buildId, ok: false, error: String(again).slice(0, 200) })) + throw again + } + } + }) } catch (error) { await failAndRelease(deps, row, createFailureReason(error)) // Only a create nothing pinned to a region moves on: no explicit region, no environment build. @@ -864,13 +907,17 @@ export async function createSandbox( console.log(JSON.stringify({ event: 'lifecycle_timing', op: 'create', fromBuild: source?.buildId ?? null, ms: timer.timings })) return cloning ? { ...ready, cloning } : ready } catch (error) { - await failAndRelease(deps, (await getSandbox(deps.db, row.id)) ?? row, createFailureReason(error)) + await failAndRelease(deps, (await getSandbox(deps.db, row.id)) ?? row, createFailureReason(error, timer)) throw error } } -/** last_error for a create that failed with `error`: the step that broke, never a message. */ -export function createFailureReason(error: unknown): string { +/** + * last_error for a create that failed with `error`: the step that broke, never a message. An error of + * no known kind is named by the phase it broke (`timer`): orcad's start, at or past its 120 s budget, + * is `orcad_start_timeout` (CREATE-FAIL-LABELS), so D1 alone tells a hung start from the rest. + */ +export function createFailureReason(error: unknown, timer?: Pick, 'failed'>): string { if (error instanceof VmCommandError) { return error.code } @@ -880,6 +927,9 @@ export function createFailureReason(error: unknown): string { if (error instanceof ProviderError) { return `provider_${error.status}` } + if (timer?.failed?.phase === 'orcad') { + return timer.failed.ms >= START_TIMEOUT_MS ? 'orcad_start_timeout' : 'orcad_start_error' + } return 'create_failed' } diff --git a/cloud/apps/sandbox-control/src/sandbox-vm-commands.ts b/cloud/apps/sandbox-control/src/sandbox-vm-commands.ts index f827d0c7547..cfcd9c8e0fb 100644 --- a/cloud/apps/sandbox-control/src/sandbox-vm-commands.ts +++ b/cloud/apps/sandbox-control/src/sandbox-vm-commands.ts @@ -11,7 +11,7 @@ export const START_COMMAND = '/opt/orca/bin/orca-sandbox-start' export const DRAIN_COMMAND = '/opt/orca/bin/orca-sandbox-drain' export const ENV_COMMAND = '/opt/orca/bin/orca-sandbox-env' const ENV_TIMEOUT_MS = 30_000 -const START_TIMEOUT_MS = 120_000 +export const START_TIMEOUT_MS = 120_000 /** Matches the parity plan: in-flight turns get up to 5 min to become durable restart offers. */ export const DRAIN_TIMEOUT_MS = 5 * 60_000 diff --git a/cloud/apps/sandbox-control/test/create-build-snapshot-404.test.ts b/cloud/apps/sandbox-control/test/create-build-snapshot-404.test.ts new file mode 100644 index 00000000000..cf8d582208a --- /dev/null +++ b/cloud/apps/sandbox-control/test/create-build-snapshot-404.test.ts @@ -0,0 +1,139 @@ +// CREATE-BUILD-SNAPSHOT-404: on dev a VM create from an environment build's snapshot answered 404 +// shortly after the build (the same snapshot booted seconds before and after). One retry, after a +// short wait, for a build under 15 min old; never for goldens, older builds or other errors. +// CREATE-FAIL-LABELS: a create that broke in orcad's start with an error of no known kind is named +// by that step (orcad_start_timeout past the 120 s budget), not the generic create_failed. +import { describe, expect, it } from 'vitest' +import { createFailureReason, phaseTimer } from '../src/lifecycle' +import { ProviderError } from '../src/sandbox-provider' +import { START_TIMEOUT_MS, VmCommandError } from '../src/sandbox-vm-commands' +import { GOLDEN, createHarness } from './harness' + +const REPO = 'https://github.com/acme/web.git' +const RESULT = JSON.stringify({ + durationSec: 3, + repos: [{ url: REPO, dir: 'acme/web', branch: 'main', commit: 'a'.repeat(40), lockfiles: {} }], + diskUsedBytes: 1, + diskTotalBytes: 2 +}) + +async function setup() { + const sleeps: number[] = [] + const h = await createHarness({}, { sleep: async (ms) => void sleeps.push(ms) }) + h.vercel.buildResult = RESULT + // A fresh token per call: some tests move the clock past a token's 15 min age. + const call = async (method: string, path: string, body?: unknown) => + h.call(method, path, { bearer: await h.token('alice', 'acme', 'sandbox', { orgRole: 'admin' }), body }) + const env = await call('POST', '/v1/environments', { name: 'Default', repos: [{ url: 'acme/web', defaultBranch: 'main' }], installScript: 'true' }) + const envId: string = env.body.environment.id + const started = await call('POST', `/v1/environments/${envId}/builds`) + const command = [...h.vercel.buildCommands.values()].at(-1)! + command.logs = `${JSON.stringify({ data: '::orca-step:: install\nok\n', stream: 'stdout' })}\n` + command.exitCode = 0 + h.advance(1000) + const build = (await call('GET', `/v1/environments/${envId}/builds/${started.body.build.id}`)).body.build + expect(build.status).toBe('succeeded') + // The provider answers `failures` (in order) to the next VM creates, then creates normally. + const failures: number[] = [] + const snapshotsTried: string[] = [] + const createVm = h.vercel.createSandbox.bind(h.vercel) + h.vercel.createSandbox = async (params) => { + if (params.name.includes('smoke') || params.name.includes('envbuild')) { + return createVm(params) + } + snapshotsTried.push(params.snapshotId) + const status = failures.shift() + if (status) { + throw new ProviderError(status, 'provider_failed') + } + return createVm(params) + } + const create = (workspaceId: string, repo = true) => + call('POST', '/v1/sandboxes', { workspaceId, machineSize: 'standard', ...(repo ? { repo: { url: REPO } } : {}) }) + const lastError = (workspaceId: string) => + (h.raw.prepare('SELECT last_error FROM sandboxes WHERE workspace_id = ? ORDER BY rowid DESC').get(workspaceId) as { last_error: string | null }).last_error + return { h, build, failures, snapshotsTried, sleeps, create, lastError } +} + +describe('CREATE-BUILD-SNAPSHOT-404', () => { + it('a just-built snapshot that answers 404 is retried once after ~5 s, and the create succeeds', async () => { + const t = await setup() + t.failures.push(404) + const created = await t.create('ws-fresh') + expect(created.status).toBe(200) + expect(created.body.sandbox.environmentBuildId).toBe(t.build.id) + expect(t.sleeps).toEqual([5_000]) + expect(t.snapshotsTried).toHaveLength(2) + expect(t.snapshotsTried[0]).toBe(t.snapshotsTried[1]) + expect(t.snapshotsTried[0]).not.toBe(GOLDEN) + }) + + it('only once: a second 404 fails the create as before (provider_404)', async () => { + const t = await setup() + t.failures.push(404, 404) + expect((await t.create('ws-twice')).status).not.toBe(200) + expect(t.sleeps).toEqual([5_000]) + expect(t.snapshotsTried).toHaveLength(2) + expect(t.lastError('ws-twice')).toBe('provider_404') + }) + + it('no retry for a build older than 15 min', async () => { + const t = await setup() + t.h.advance(16 * 60_000) + t.failures.push(404) + expect((await t.create('ws-old')).status).not.toBe(200) + expect(t.sleeps).toEqual([]) + expect(t.snapshotsTried).toHaveLength(1) + }) + + it('no retry for other errors (403) on a fresh build', async () => { + const t = await setup() + t.failures.push(403) + expect((await t.create('ws-403')).status).not.toBe(200) + expect(t.sleeps).toEqual([]) + expect(t.lastError('ws-403')).toBe('provider_403') + }) + + it('no retry for the golden image', async () => { + const t = await setup() + t.failures.push(404) + expect((await t.create('ws-golden', false)).status).not.toBe(200) + expect(t.snapshotsTried[0]).toBe(GOLDEN) + expect(t.sleeps).toEqual([]) + expect(t.snapshotsTried).toHaveLength(1) + }) +}) + +describe('CREATE-FAIL-LABELS', () => { + const failIn = async (phase: string, ms: number) => { + let clock = 0 + const timer = phaseTimer(() => clock) + await timer + .time(phase, async () => { + clock += ms + throw new TypeError('Network connection lost.') + }) + .catch(() => undefined) + return timer + } + + it("names an unknown error in orcad's start by the step: a timeout at the 120 s budget, an error before it", async () => { + expect(createFailureReason(new TypeError('x'), await failIn('orcad', START_TIMEOUT_MS + 7_000))).toBe('orcad_start_timeout') + expect(createFailureReason(new TypeError('x'), await failIn('orcad', 2_000))).toBe('orcad_start_error') + }) + + it('keeps the known kinds and the generic label elsewhere', async () => { + const orcad = await failIn('orcad', START_TIMEOUT_MS) + expect(createFailureReason(new VmCommandError('orcad_start_failed'), orcad)).toBe('orcad_start_failed') + expect(createFailureReason(new ProviderError(404, 'gone'), orcad)).toBe('provider_404') + expect(createFailureReason(new TypeError('x'), await failIn('vm', 1_000))).toBe('create_failed') + expect(createFailureReason(new TypeError('x'))).toBe('create_failed') + }) + + it('the timer records only the first phase that threw, and keeps timings numeric (Server-Timing)', async () => { + const timer = await failIn('orcad', 5) + await timer.time('later', async () => Promise.reject(new Error('x'))).catch(() => undefined) + expect(timer.failed).toEqual({ phase: 'orcad', ms: 5 }) + expect(Object.values(timer.timings).every((v) => typeof v === 'number')).toBe(true) + }) +}) From ee19ee2684ede967f6f56387b6d27688244c921d Mon Sep 17 00:00:00 2001 From: Neil Date: Fri, 13 Nov 2026 06:03:03 -0800 Subject: [PATCH 2/4] fix(cloud): EGRESS-SWAP-FAIL-CLOSED a deployed env without EGRESS_PROXY_URL withholds model keys and account sign-ins (firewall answers the placeholder), logs once per session, /v1/health 503 egress_proxy_missing --- cloud/apps/sandbox-control/src/index.ts | 12 +- cloud/apps/sandbox-control/src/lifecycle.ts | 28 +++- .../sandbox-control/src/network-policy.ts | 33 ++++- cloud/apps/sandbox-control/src/session-env.ts | 50 +++++++- .../test/egress-swap-fail-closed.test.ts | 120 ++++++++++++++++++ 5 files changed, 229 insertions(+), 14 deletions(-) create mode 100644 cloud/apps/sandbox-control/test/egress-swap-fail-closed.test.ts diff --git a/cloud/apps/sandbox-control/src/index.ts b/cloud/apps/sandbox-control/src/index.ts index 366e7563231..822dd4de60f 100644 --- a/cloud/apps/sandbox-control/src/index.ts +++ b/cloud/apps/sandbox-control/src/index.ts @@ -146,7 +146,7 @@ import { runningSandboxesForDefault } from './agent-sources' import { EgressHostsSchema, egressUserHosts, normalizeUserHosts, setEgressUserHosts } from './egress-hosts' -import { gitBasicAuthorization, parseEgressPreset } from './network-policy' +import { egressProxyMissing, gitBasicAuthorization, parseEgressPreset } from './network-policy' import { cachedProviderEnv } from './session-env' import { credentialTargets, type CredentialTarget } from './provider-env' import type { Keyring } from './secrets-crypto' @@ -1435,7 +1435,15 @@ async function routeRequest(request: Request, env: Env, deps: Deps): Promise, row: Pick): EgressPreset { const preset = deps.config.egressPreset ?? DEFAULT_EGRESS_PRESET - if (preset === 'open' && !deps.config.egressProxyUrl && !LOCAL_ENVIRONMENTS.has(deps.config.environment)) { + if (preset === 'open' && egressProxyMissing(deps.config)) { console.error(JSON.stringify({ event: 'egress_open_without_proxy', sandboxId: row.id, environment: deps.config.environment, applied: 'limited' })) return 'limited' } return preset } -const LOCAL_ENVIRONMENTS = new Set(['local', 'test']) /** Refuses up front (503 platform_hosts_misconfigured) when this create or boot would compile a limited policy. */ export function assertEgressAllowlistUsable(deps: Pick, row: Pick): void { @@ -338,7 +337,28 @@ export function egressPolicy(deps: LifecycleDeps, row: SandboxRow, gitToken?: st ) ) ] - : [] + : // No proxy (EGRESS-SWAP-FAIL-CLOSED): keys were withheld and only placeholders reached the VM. The firewall + // answers them itself with a plain error, so agents say what happened instead of a provider auth failure. + [ + ...egressAccountRoutes(row).flatMap(({ agent, host }): CredentialRoute[] => { + const provider = accountProvider(agent) + return provider + ? [{ host, header: provider.header, placeholders: [provider.format(provider.placeholder(row))], via: 'respond', statusCode: 403, body: KEYS_WITHHELD_BODY }] + : [] + }), + ...egressSecretHosts(row).flatMap((entry) => + Object.keys(entry.headers).map( + (header): CredentialRoute => ({ + host: entry.host, + header, + placeholders: placeholderValues(entry, header, secretPlaceholder(row, entry.secret)), + via: 'respond', + statusCode: 403, + body: KEYS_WITHHELD_BODY + }) + ) + ) + ] const all = [...git, ...credentials] if (effectiveEgressPreset(deps, row) === 'open') { // Every host is reachable; the credential claims (and their trailing 403s) still apply. diff --git a/cloud/apps/sandbox-control/src/network-policy.ts b/cloud/apps/sandbox-control/src/network-policy.ts index 9e621db3afa..b0e1839eacd 100644 --- a/cloud/apps/sandbox-control/src/network-policy.ts +++ b/cloud/apps/sandbox-control/src/network-policy.ts @@ -15,7 +15,7 @@ export type NetworkPolicyRule = transform: { headers: Record }[] } | { match?: { headers?: { key: Matcher; value: Matcher }[] }; forwardURL: string } - | { response: { statusCode: number; body: string; contentType: string } } + | { match?: { headers?: { key: Matcher; value: Matcher }[] }; response: { statusCode: number; body: string; contentType: string } } export type NetworkPolicy = { allow: Record } @@ -166,6 +166,25 @@ export function parseEgressPreset(value: string | undefined): EgressPreset { return preset === 'open' ? 'open' : 'limited' } +/** Environments where running without the egress proxy is expected (dev loops, the unit harness). */ +export function isLocalEnvironment(environment: string | undefined): boolean { + return environment === 'local' || environment === 'test' +} + +/** + * A deployed env with no EGRESS_PROXY_URL: nothing can swap credentials at egress, so open egress falls back + * to limited (E3-PROD-GUARD) and swappable keys are withheld from the VM (EGRESS-SWAP-FAIL-CLOSED). + */ +export function egressProxyMissing(config: { egressProxyUrl?: string; environment?: string }): boolean { + return !config.egressProxyUrl && !isLocalEnvironment(config.environment) +} + +/** What an agent sees when its model key was withheld (design-squad wording). */ +export const KEYS_WITHHELD_MESSAGE = + "Orca Cloud couldn't use your model API key in this workspace because a service on Orca's side isn't set up, so the key was not sent. Contact Orca support or your team admin." +/** Provider-style error JSON: Anthropic clients read error.message, OpenAI-compatible ones too. */ +export const KEYS_WITHHELD_BODY = JSON.stringify({ type: 'error', error: { type: 'permission_error', message: KEYS_WITHHELD_MESSAGE } }) + export type CredentialRoute = { host: string /** Header the VM sends with a placeholder value, matched exactly (one claim rule per value). */ @@ -176,6 +195,8 @@ export type CredentialRoute = { } & ( | { via: 'transform'; value: string } | { via: 'forward'; forwardURL: string } + /** Answered by Vercel's firewall itself; nothing reaches the host (EGRESS-SWAP-FAIL-CLOSED). */ + | { via: 'respond'; statusCode: number; body: string } ) export type EgressPolicyInput = { @@ -206,9 +227,13 @@ export function compileNetworkPolicy(input: EgressPolicyInput): NetworkPolicy { const header = route.header.toLowerCase() const claims = route.placeholders.map((placeholder): NetworkPolicyRule => { const match = { headers: [{ key: { exact: header }, value: { exact: placeholder } }] } - return route.via === 'transform' - ? { match, transform: [{ headers: { [header]: route.value } }] } - : { match, forwardURL: route.forwardURL } + if (route.via === 'transform') { + return { match, transform: [{ headers: { [header]: route.value } }] } + } + if (route.via === 'respond') { + return { match, response: { statusCode: route.statusCode, body: route.body, contentType: 'application/json' } } + } + return { match, forwardURL: route.forwardURL } }) allow[host] = [...(allow[host] ?? []), ...claims] // One route that wants the deny is enough: a shared host fails closed. diff --git a/cloud/apps/sandbox-control/src/session-env.ts b/cloud/apps/sandbox-control/src/session-env.ts index c20fbebaa02..bbace2d72db 100644 --- a/cloud/apps/sandbox-control/src/session-env.ts +++ b/cloud/apps/sandbox-control/src/session-env.ts @@ -6,6 +6,7 @@ import { agentModesFor, withoutSignInCredentials, type AgentMode } from './agent import { accountProvider } from './account-providers' import type { CredentialAgent } from './agent-credential-names' import { deriveProviderEnv, deriveProviderScopes, providerHosts } from './provider-env' +import { egressProxyMissing } from './network-policy' import { applySecretPlaceholders, encodeAccountRoute, encodeEgressSecretHosts } from './secret-egress' import { resolveSandboxSecrets } from './secrets' import type { Keyring } from './secrets-crypto' @@ -70,15 +71,35 @@ export function forgetProviderEnv(sandboxId: string): void { } } -function egressSwapOn(deps: Pick, row: Pick): boolean { +/** Placeholders are switched on for this account (SECRETS_EGRESS_PLACEHOLDERS, minus the per-team kill switch). */ +function egressSwapWanted(deps: Pick, row: Pick): boolean { const placeholders = deps.secrets?.egressPlaceholders return ( !deps.secrets?.egressPlaceholdersOff?.has(row.account_id) && - (placeholders === true || (placeholders instanceof Set && placeholders.has(row.account_id))) && - Boolean(deps.config.egressProxyUrl) + (placeholders === true || (placeholders instanceof Set && placeholders.has(row.account_id))) ) } +function egressSwapOn(deps: Pick, row: Pick): boolean { + return egressSwapWanted(deps, row) && Boolean(deps.config.egressProxyUrl) +} + +// One log line per session, not per env read (boot, refresh and the proxy's cache all land here). The boot read +// happens before the session exists, so its mark carries over to the session it starts. +const withheldLogged = new Set() +function firstWithheldReadThisSession(row: Pick): boolean { + const pending = `${row.id}:pending` + if (!row.session_id) { + return withheldLogged.has(pending) ? false : (withheldLogged.add(pending), true) + } + const key = `${row.id}:${row.session_id}` + if (withheldLogged.has(key)) { + return false + } + withheldLogged.add(key) + return !withheldLogged.delete(pending) +} + export type { AgentMode } export async function sessionSecretEnv( @@ -93,11 +114,15 @@ export async function sessionSecretEnv( } const { env: stored, scopes: storedScopes, skipped } = await resolveSandboxSecrets(deps.db, deps.secrets.keyring, row) const swap = egressSwapOn(deps, row) + // EGRESS-SWAP-FAIL-CLOSED: a deployed env without the egress proxy can't swap keys at egress. Rather than + // fall back to real keys in the VM, keys become placeholders that the firewall answers with a plain error, + // and an Orca-account sign-in gets its placeholder answered the same way (the real token never leaves us). + const withhold = !swap && egressSwapWanted(deps, row) && egressProxyMissing(deps.config) const derived = deriveProviderEnv(stored) // D10: per agent, the owner's account (placeholder), the kill-switch fallback, or the keys. const modes = await agentModesFor(deps.db, row, derived, { providerEnabled: deps.secrets.accountProviders ?? { claude: false, codex: false }, - accountsUsable: swap, + accountsUsable: swap || withhold, now: Date.now(), // Usable means decryptable now, not just stored: a token we can't open must not hold the session in account mode. readable: async (agent) => { @@ -111,6 +136,23 @@ export async function sessionSecretEnv( const egressAllowHosts = providerHosts(env).join(',') || null // Counts only. secretLog('session_secrets', { sandboxId: row.id, injected: Object.keys(env).length, skipped }) + if (withhold) { + if (firstWithheldReadThisSession(row)) { + secretLog('egress_swap_off_no_proxy', { sandboxId: row.id, environment: deps.config.environment }) + } + const withheld = applySecretPlaceholders(row, env, scopes) + const accountEnv: Record = {} + const accountRoutes: string[] = [] + for (const agent of Object.keys(modes) as CredentialAgent[]) { + const provider = modes[agent] === 'account' ? accountProvider(agent) : undefined + if (provider) { + Object.assign(accountEnv, provider.sessionEnv(row)) + accountRoutes.push(...provider.hosts.map((host) => encodeAccountRoute(agent, row.user_id, host))) + } + } + const hosts = [encodeEgressSecretHosts(withheld.targets), ...accountRoutes].filter(Boolean).join(',') || null + return { env: { ...withheld.env, ...accountEnv }, egressSecretHosts: hosts, egressAllowHosts } + } if (!swap) { return { env, egressSecretHosts: null, egressAllowHosts } } diff --git a/cloud/apps/sandbox-control/test/egress-swap-fail-closed.test.ts b/cloud/apps/sandbox-control/test/egress-swap-fail-closed.test.ts new file mode 100644 index 00000000000..ada505be28f --- /dev/null +++ b/cloud/apps/sandbox-control/test/egress-swap-fail-closed.test.ts @@ -0,0 +1,120 @@ +// EGRESS-SWAP-FAIL-CLOSED: a deployed env (not local/test) without EGRESS_PROXY_URL can't swap keys at egress. +// Model keys and Orca-account sign-ins are withheld (the VM gets placeholders, the firewall answers them with a +// plain error), the session logs egress_swap_off_no_proxy once, and /v1/health fails the post-deploy smoke. +// Local and test keep today's behaviour. +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { KEYS_WITHHELD_MESSAGE, compileNetworkPolicy } from '../src/network-policy' +import { START_COMMAND } from '../src/sandbox-vm-commands' +import { createHarness } from './harness' + +const KEK = Buffer.alloc(32, 7).toString('base64') +const KEY = 'sk-ant-api03-WITHHELD-fake-0000-aaaa' +const SETUP_TOKEN = 'sk-ant-oat01-FAKEsetupTOKEN_for_tests_only_000000000000' + +let logs: string[] = [] +beforeEach(() => { + logs = [] + for (const method of ['log', 'info', 'warn', 'error', 'debug'] as const) { + vi.spyOn(console, method).mockImplementation((...args: unknown[]) => { + logs.push(args.map((arg) => (typeof arg === 'string' ? arg : JSON.stringify(arg))).join(' ')) + }) + } +}) +afterEach(() => { + expect(logs.join('\n')).not.toContain(KEY) + expect(logs.join('\n')).not.toContain(SETUP_TOKEN) + vi.restoreAllMocks() +}) + +async function setup(env: Record) { + const h = await createHarness({ SECRETS_KEK: KEK, SECRETS_EGRESS_PLACEHOLDERS: 'on', AGENT_ACCOUNTS_CLAUDE: 'on', ...env }) + const mia = await h.token('mia', undefined, 'sandbox') + return { h, mia } +} + +function startEnv(h: Awaited>): Record { + const start = h.vercel.commands.filter((c) => c.command === START_COMMAND).at(-1)! + return Object.fromEntries( + (start.env.ORCA_SECRET_ENV ?? '') + .split('\n') + .filter(Boolean) + .map((line) => [line.slice(0, line.indexOf('=')), Buffer.from(line.slice(line.indexOf('=') + 1), 'base64').toString('utf8')]) + ) +} + +const withheldEvents = () => logs.filter((line) => line.includes('"event":"egress_swap_off_no_proxy"')) + +describe('EGRESS-SWAP-FAIL-CLOSED', () => { + it('a deployed env without the proxy withholds model keys: placeholder in the VM, the firewall answers it with a plain error', async () => { + const { h, mia } = await setup({ SANDBOX_ENV: 'staging' }) + await h.call('PUT', '/v1/secrets/personal/ANTHROPIC_API_KEY', { body: { value: KEY }, bearer: mia }) + const created = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard' }, bearer: mia }) + expect(created.status).toBe(200) + const id: string = created.body.sandbox.id + const env = startEnv(h) + expect(env.ANTHROPIC_API_KEY).toBeDefined() + expect(env.ANTHROPIC_API_KEY).not.toBe(KEY) + expect(JSON.stringify(h.vercel.commands)).not.toContain(KEY) + const box = [...h.vercel.boxes.values()][0]! + expect(JSON.stringify(box.policy)).not.toContain(KEY) + // Limited (E3-PROD-GUARD), and the key's host answers its placeholder itself. + expect(box.policy.allow['*']).toBeUndefined() + const rules = box.policy.allow['api.anthropic.com']! + const answer = rules.find((rule) => 'response' in rule && 'match' in rule && rule.match) + expect(answer).toMatchObject({ + match: { headers: [{ key: { exact: 'x-api-key' }, value: { exact: env.ANTHROPIC_API_KEY } }] }, + response: { statusCode: 403, contentType: 'application/json' } + }) + expect(JSON.parse((answer as { response: { body: string } }).response.body).error.message).toBe(KEYS_WITHHELD_MESSAGE) + expect(rules.at(-1)).toMatchObject({ response: { statusCode: 403, body: 'orca-egress-denied' } }) + // Logged once for the session, however many times its env is read. + expect(withheldEvents()).toHaveLength(1) + await h.call('POST', `/v1/sandboxes/${id}/env/refresh`, { bearer: mia }) + expect(withheldEvents()).toHaveLength(1) + }) + + it('an Orca-account sign-in is withheld the same way: account placeholder only, answered by the firewall', async () => { + const { h, mia } = await setup({ SANDBOX_ENV: 'staging' }) + expect((await h.call('PUT', '/v1/agent-credentials/claude', { bearer: mia, body: { token: SETUP_TOKEN } })).status).toBe(200) + await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard' }, bearer: mia }) + const env = startEnv(h) + expect(env.CLAUDE_CODE_OAUTH_TOKEN).toMatch(/^orca-ph-claude-account-/) + const box = [...h.vercel.boxes.values()][0]! + expect(JSON.stringify(box.policy)).not.toContain(SETUP_TOKEN) + expect(box.policy.allow['api.anthropic.com']![0]).toMatchObject({ + match: { headers: [{ key: { exact: 'authorization' }, value: { exact: `Bearer ${env.CLAUDE_CODE_OAUTH_TOKEN}` } }] }, + response: { statusCode: 403 } + }) + }) + + it('test and local keep today’s behaviour without a proxy: plain env injection, no event', async () => { + const { h, mia } = await setup({ SANDBOX_ENV: 'test' }) + await h.call('PUT', '/v1/secrets/personal/ANTHROPIC_API_KEY', { body: { value: KEY }, bearer: mia }) + await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard' }, bearer: mia }) + expect(startEnv(h).ANTHROPIC_API_KEY).toBe(KEY) + expect(withheldEvents()).toHaveLength(0) + // The plain key appears in the env on purpose here; keep it out of the afterEach log check. + logs = logs.filter((line) => !line.includes(KEY)) + }) + + it('/v1/health is 503 egress_proxy_missing in a deployed env without the proxy, 200 with it or in local/test', async () => { + const missing = await setup({ SANDBOX_ENV: 'staging' }) + expect(await missing.h.call('GET', '/v1/health')).toMatchObject({ status: 503, body: { error: 'egress_proxy_missing' } }) + const present = await setup({ SANDBOX_ENV: 'staging', EGRESS_PROXY_URL: 'https://sandbox.test/v1/egress/proxy' }) + expect(await present.h.call('GET', '/v1/health')).toMatchObject({ status: 200, body: { ok: true } }) + const local = await setup({ SANDBOX_ENV: 'test' }) + expect((await local.h.call('GET', '/v1/health')).status).toBe(200) + }) + + it('a respond route compiles to a matched direct response, before the trailing deny', () => { + const policy = compileNetworkPolicy({ + mode: 'allowlist', + allowlist: [], + credentials: [{ host: 'api.openai.com', header: 'Authorization', placeholders: ['Bearer orca-ph-x'], via: 'respond', statusCode: 403, body: '{"e":1}' }] + }) + expect(policy.allow['api.openai.com']).toEqual([ + { match: { headers: [{ key: { exact: 'authorization' }, value: { exact: 'Bearer orca-ph-x' } }] }, response: { statusCode: 403, body: '{"e":1}', contentType: 'application/json' } }, + { response: { statusCode: 403, body: 'orca-egress-denied', contentType: 'text/plain' } } + ]) + }) +}) From c9f68dfa634af92933e08aa4a8a7db252ee8d677 Mon Sep 17 00:00:00 2001 From: Neil Date: Fri, 13 Nov 2026 06:04:30 -0800 Subject: [PATCH 3/4] fix(cloud): EGRESS-SWAP-FAIL-CLOSED merged spec: egress_swap_unavailable at error level (count only), health keySwap on/off, design-squad message, deployed-with-proxy test --- cloud/apps/sandbox-control/src/index.ts | 5 +-- .../sandbox-control/src/network-policy.ts | 2 +- cloud/apps/sandbox-control/src/secret-log.ts | 8 ++--- cloud/apps/sandbox-control/src/session-env.ts | 6 ++-- .../test/egress-swap-fail-closed.test.ts | 36 +++++++++++++++---- .../test/platform-host-guards.test.ts | 2 +- 6 files changed, 42 insertions(+), 17 deletions(-) diff --git a/cloud/apps/sandbox-control/src/index.ts b/cloud/apps/sandbox-control/src/index.ts index 822dd4de60f..cf162cd9c75 100644 --- a/cloud/apps/sandbox-control/src/index.ts +++ b/cloud/apps/sandbox-control/src/index.ts @@ -1441,9 +1441,10 @@ async function routeRequest(request: Request, env: Env, deps: Deps): Promise void -let sink: SecretLogSink = (line) => console.log(line) +export type SecretLogSink = (line: string, level?: 'error') => void +let sink: SecretLogSink = (line, level) => (level === 'error' ? console.error(line) : console.log(line)) /** Tests only: capture lines exactly as they would reach Workers logs. Returns a restore function. */ export function setSecretLogSink(next: SecretLogSink): () => void { @@ -76,12 +76,12 @@ export function setSecretLogSink(next: SecretLogSink): () => void { } /** One structured log line. `known`: values in scope at the call site that must never be written. */ -export function secretLog(event: string, fields: Record = {}, known: Iterable = []): void { +export function secretLog(event: string, fields: Record = {}, known: Iterable = [], level?: 'error'): void { let line: string try { line = JSON.stringify({ event, ...fields }, (_key, value: unknown) => serializable(value)) } catch { line = JSON.stringify({ event, unserializable: true }) } - sink(redactLine(line, known)) + sink(redactLine(line, known), level) } diff --git a/cloud/apps/sandbox-control/src/session-env.ts b/cloud/apps/sandbox-control/src/session-env.ts index bbace2d72db..4e91d35a581 100644 --- a/cloud/apps/sandbox-control/src/session-env.ts +++ b/cloud/apps/sandbox-control/src/session-env.ts @@ -5,7 +5,7 @@ import type { LifecycleDeps } from './lifecycle' import { agentModesFor, withoutSignInCredentials, type AgentMode } from './agent-sources' import { accountProvider } from './account-providers' import type { CredentialAgent } from './agent-credential-names' -import { deriveProviderEnv, deriveProviderScopes, providerHosts } from './provider-env' +import { SWAPPED_CREDENTIALS, deriveProviderEnv, deriveProviderScopes, providerHosts } from './provider-env' import { egressProxyMissing } from './network-policy' import { applySecretPlaceholders, encodeAccountRoute, encodeEgressSecretHosts } from './secret-egress' import { resolveSandboxSecrets } from './secrets' @@ -137,8 +137,10 @@ export async function sessionSecretEnv( // Counts only. secretLog('session_secrets', { sandboxId: row.id, injected: Object.keys(env).length, skipped }) if (withhold) { + const withheldSecrets = Object.fromEntries(SWAPPED_CREDENTIALS.filter((secret) => secret in env).map((secret) => [secret, true])) if (firstWithheldReadThisSession(row)) { - secretLog('egress_swap_off_no_proxy', { sandboxId: row.id, environment: deps.config.environment }) + // Counts only: no secret names or values. + secretLog('egress_swap_unavailable', { sandboxId: row.id, environment: deps.config.environment, withheld: Object.keys(withheldSecrets).length }, [], 'error') } const withheld = applySecretPlaceholders(row, env, scopes) const accountEnv: Record = {} diff --git a/cloud/apps/sandbox-control/test/egress-swap-fail-closed.test.ts b/cloud/apps/sandbox-control/test/egress-swap-fail-closed.test.ts index ada505be28f..2edce67a2d0 100644 --- a/cloud/apps/sandbox-control/test/egress-swap-fail-closed.test.ts +++ b/cloud/apps/sandbox-control/test/egress-swap-fail-closed.test.ts @@ -1,6 +1,7 @@ // EGRESS-SWAP-FAIL-CLOSED: a deployed env (not local/test) without EGRESS_PROXY_URL can't swap keys at egress. // Model keys and Orca-account sign-ins are withheld (the VM gets placeholders, the firewall answers them with a -// plain error), the session logs egress_swap_off_no_proxy once, and /v1/health fails the post-deploy smoke. +// plain error), the session logs egress_swap_unavailable once (error level, counts only), and /v1/health fails the +// post-deploy smoke with keySwap "off". // Local and test keep today's behaviour. import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { KEYS_WITHHELD_MESSAGE, compileNetworkPolicy } from '../src/network-policy' @@ -14,9 +15,14 @@ const SETUP_TOKEN = 'sk-ant-oat01-FAKEsetupTOKEN_for_tests_only_000000000000' let logs: string[] = [] beforeEach(() => { logs = [] + errors = [] for (const method of ['log', 'info', 'warn', 'error', 'debug'] as const) { vi.spyOn(console, method).mockImplementation((...args: unknown[]) => { - logs.push(args.map((arg) => (typeof arg === 'string' ? arg : JSON.stringify(arg))).join(' ')) + const line = args.map((arg) => (typeof arg === 'string' ? arg : JSON.stringify(arg))).join(' ') + logs.push(line) + if (method === 'error') { + errors.push(line) + } }) } }) @@ -42,7 +48,8 @@ function startEnv(h: Awaited>): Record logs.filter((line) => line.includes('"event":"egress_swap_off_no_proxy"')) +let errors: string[] = [] +const withheldEvents = () => errors.filter((line) => line.includes('"event":"egress_swap_unavailable"')) describe('EGRESS-SWAP-FAIL-CLOSED', () => { it('a deployed env without the proxy withholds model keys: placeholder in the VM, the firewall answers it with a plain error', async () => { @@ -67,8 +74,10 @@ describe('EGRESS-SWAP-FAIL-CLOSED', () => { }) expect(JSON.parse((answer as { response: { body: string } }).response.body).error.message).toBe(KEYS_WITHHELD_MESSAGE) expect(rules.at(-1)).toMatchObject({ response: { statusCode: 403, body: 'orca-egress-denied' } }) - // Logged once for the session, however many times its env is read. + // Logged once for the session at error level, however many times its env is read: a count, no names. expect(withheldEvents()).toHaveLength(1) + expect(JSON.parse(withheldEvents()[0]!)).toEqual({ event: 'egress_swap_unavailable', sandboxId: id, environment: 'staging', withheld: 1 }) + expect(withheldEvents()[0]).not.toContain('ANTHROPIC') await h.call('POST', `/v1/sandboxes/${id}/env/refresh`, { bearer: mia }) expect(withheldEvents()).toHaveLength(1) }) @@ -87,6 +96,19 @@ describe('EGRESS-SWAP-FAIL-CLOSED', () => { }) }) + it('a deployed env with the proxy behaves as today: placeholder forwarded to the proxy, no event', async () => { + const { h, mia } = await setup({ SANDBOX_ENV: 'staging', EGRESS_PROXY_URL: 'https://sandbox.test/v1/egress/proxy' }) + await h.call('PUT', '/v1/secrets/personal/ANTHROPIC_API_KEY', { body: { value: KEY }, bearer: mia }) + await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard' }, bearer: mia }) + const env = startEnv(h) + expect(env.ANTHROPIC_API_KEY).not.toBe(KEY) + const box = [...h.vercel.boxes.values()][0]! + expect(box.policy.allow['*']).toEqual([]) + expect(box.policy.allow['api.anthropic.com']![0]).toMatchObject({ forwardURL: 'https://sandbox.test/v1/egress/proxy' }) + expect(JSON.stringify(box.policy)).not.toContain(KEYS_WITHHELD_MESSAGE) + expect(withheldEvents()).toHaveLength(0) + }) + it('test and local keep today’s behaviour without a proxy: plain env injection, no event', async () => { const { h, mia } = await setup({ SANDBOX_ENV: 'test' }) await h.call('PUT', '/v1/secrets/personal/ANTHROPIC_API_KEY', { body: { value: KEY }, bearer: mia }) @@ -99,11 +121,11 @@ describe('EGRESS-SWAP-FAIL-CLOSED', () => { it('/v1/health is 503 egress_proxy_missing in a deployed env without the proxy, 200 with it or in local/test', async () => { const missing = await setup({ SANDBOX_ENV: 'staging' }) - expect(await missing.h.call('GET', '/v1/health')).toMatchObject({ status: 503, body: { error: 'egress_proxy_missing' } }) + expect(await missing.h.call('GET', '/v1/health')).toEqual({ status: 503, body: { error: 'egress_proxy_missing', keySwap: 'off' } }) const present = await setup({ SANDBOX_ENV: 'staging', EGRESS_PROXY_URL: 'https://sandbox.test/v1/egress/proxy' }) - expect(await present.h.call('GET', '/v1/health')).toMatchObject({ status: 200, body: { ok: true } }) + expect(await present.h.call('GET', '/v1/health')).toEqual({ status: 200, body: { ok: true, keySwap: 'on' } }) const local = await setup({ SANDBOX_ENV: 'test' }) - expect((await local.h.call('GET', '/v1/health')).status).toBe(200) + expect(await local.h.call('GET', '/v1/health')).toEqual({ status: 200, body: { ok: true, keySwap: 'off' } }) }) it('a respond route compiles to a matched direct response, before the trailing deny', () => { diff --git a/cloud/apps/sandbox-control/test/platform-host-guards.test.ts b/cloud/apps/sandbox-control/test/platform-host-guards.test.ts index c291175586f..82232944460 100644 --- a/cloud/apps/sandbox-control/test/platform-host-guards.test.ts +++ b/cloud/apps/sandbox-control/test/platform-host-guards.test.ts @@ -18,7 +18,7 @@ async function limited() { describe('a malformed ORCA_PLATFORM_HOST_SUFFIXES (PLATFORM-SUFFIX-GUARDS)', () => { it('fails the health route a post-deploy smoke hits, with the code only; the log names the bad entry', async () => { const h = await createHarness() - expect(await h.call('GET', '/v1/health')).toEqual({ status: 200, body: { ok: true } }) + expect(await h.call('GET', '/v1/health')).toMatchObject({ status: 200, body: { ok: true } }) h.env.ORCA_PLATFORM_HOST_SUFFIXES = MALFORMED const logged = vi.spyOn(console, 'error').mockImplementation(() => undefined) try { From 20c83d1412d5e04b6339da35e97a9dcdf1144148 Mon Sep 17 00:00:00 2001 From: Neil Date: Fri, 13 Nov 2026 06:05:44 -0800 Subject: [PATCH 4/4] fix(cloud): EGRESS-SWAP-FAIL-CLOSED review: egress_swap_disabled log when an operator turns placeholders off, proxy refuses without its URL --- cloud/apps/sandbox-control/src/index.ts | 5 +++++ cloud/apps/sandbox-control/src/session-env.ts | 14 ++++++++++---- .../test/egress-swap-fail-closed.test.ts | 19 +++++++++++++++++++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/cloud/apps/sandbox-control/src/index.ts b/cloud/apps/sandbox-control/src/index.ts index cf162cd9c75..36b5968c529 100644 --- a/cloud/apps/sandbox-control/src/index.ts +++ b/cloud/apps/sandbox-control/src/index.ts @@ -1293,6 +1293,11 @@ async function routeRequest(request: Request, env: Env, deps: Deps): Promise, row: Pick // One log line per session, not per env read (boot, refresh and the proxy's cache all land here). The boot read // happens before the session exists, so its mark carries over to the session it starts. const withheldLogged = new Set() -function firstWithheldReadThisSession(row: Pick): boolean { - const pending = `${row.id}:pending` +function firstReadThisSession(kind: 'withheld' | 'plain', row: Pick): boolean { + const pending = `${kind}:${row.id}:pending` if (!row.session_id) { return withheldLogged.has(pending) ? false : (withheldLogged.add(pending), true) } - const key = `${row.id}:${row.session_id}` + const key = `${kind}:${row.id}:${row.session_id}` if (withheldLogged.has(key)) { return false } @@ -138,7 +138,7 @@ export async function sessionSecretEnv( secretLog('session_secrets', { sandboxId: row.id, injected: Object.keys(env).length, skipped }) if (withhold) { const withheldSecrets = Object.fromEntries(SWAPPED_CREDENTIALS.filter((secret) => secret in env).map((secret) => [secret, true])) - if (firstWithheldReadThisSession(row)) { + if (firstReadThisSession('withheld', row)) { // Counts only: no secret names or values. secretLog('egress_swap_unavailable', { sandboxId: row.id, environment: deps.config.environment, withheld: Object.keys(withheldSecrets).length }, [], 'error') } @@ -156,6 +156,12 @@ export async function sessionSecretEnv( return { env: { ...withheld.env, ...accountEnv }, egressSecretHosts: hosts, egressAllowHosts } } if (!swap) { + // An operator turned placeholders off for this account (SECRETS_EGRESS_PLACEHOLDERS=off or accounts: + // without it, or SECRETS_EGRESS_PLACEHOLDERS_OFF): real keys in the VM are deliberate, but never silent. + const plain = SWAPPED_CREDENTIALS.filter((secret) => secret in env).length + if (plain > 0 && !egressSwapWanted(deps, row) && firstReadThisSession('plain', row)) { + secretLog('egress_swap_disabled', { sandboxId: row.id, plainKeys: plain }) + } return { env, egressSecretHosts: null, egressAllowHosts } } const swapped = applySecretPlaceholders(row, env, scopes) diff --git a/cloud/apps/sandbox-control/test/egress-swap-fail-closed.test.ts b/cloud/apps/sandbox-control/test/egress-swap-fail-closed.test.ts index 2edce67a2d0..ba3d696efae 100644 --- a/cloud/apps/sandbox-control/test/egress-swap-fail-closed.test.ts +++ b/cloud/apps/sandbox-control/test/egress-swap-fail-closed.test.ts @@ -119,6 +119,25 @@ describe('EGRESS-SWAP-FAIL-CLOSED', () => { logs = logs.filter((line) => !line.includes(KEY)) }) + it('placeholders turned off by an operator: plain keys as before, but logged once per session (count only)', async () => { + const { h, mia } = await setup({ SANDBOX_ENV: 'staging', EGRESS_PROXY_URL: 'https://sandbox.test/v1/egress/proxy', SECRETS_EGRESS_PLACEHOLDERS: 'off' }) + await h.call('PUT', '/v1/secrets/personal/ANTHROPIC_API_KEY', { body: { value: KEY }, bearer: mia }) + const created = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard' }, bearer: mia }) + expect(startEnv(h).ANTHROPIC_API_KEY).toBe(KEY) + await h.call('POST', `/v1/sandboxes/${created.body.sandbox.id}/env/refresh`, { bearer: mia }) + const disabled = logs.filter((line) => line.includes('"event":"egress_swap_disabled"')) + expect(disabled).toHaveLength(1) + expect(JSON.parse(disabled[0]!)).toEqual({ event: 'egress_swap_disabled', sandboxId: created.body.sandbox.id, plainKeys: 1 }) + logs = logs.filter((line) => !line.includes(KEY)) + }) + + it('the egress proxy refuses outright in a deployed env without its URL (second lock)', async () => { + const { h } = await setup({ SANDBOX_ENV: 'staging', VERCEL_TEAM_ID: 'team_x', VERCEL_PROJECT_ID: 'prj_x' }) + const res = await h.send(new Request('https://sandbox.test/v1/egress/proxy/v1/messages', { method: 'POST', headers: { 'vercel-forwarded-host': 'api.anthropic.com' } })) + expect(res.status).toBe(503) + expect(await res.json()).toMatchObject({ error: 'egress_proxy_missing' }) + }) + it('/v1/health is 503 egress_proxy_missing in a deployed env without the proxy, 200 with it or in local/test', async () => { const missing = await setup({ SANDBOX_ENV: 'staging' }) expect(await missing.h.call('GET', '/v1/health')).toEqual({ status: 503, body: { error: 'egress_proxy_missing', keySwap: 'off' } })