fix(cloud): EGRESS-SWAP-FAIL-CLOSED merged spec: egress_swap_unavailable at error level (count only), health keySwap on/off, design-squad message, deployed-with-proxy test

This commit is contained in:
Neil
2026-11-13 06:04:30 -08:00
parent ee19ee2684
commit c9f68dfa63
6 changed files with 42 additions and 17 deletions
+3 -2
View File
@@ -1441,9 +1441,10 @@ async function routeRequest(request: Request, env: Env, deps: Deps): Promise<Res
// EGRESS-SWAP-FAIL-CLOSED: without the proxy no key reaches an agent and open egress is off.
if (egressProxyMissing({ egressProxyUrl: env.EGRESS_PROXY_URL, environment: env.SANDBOX_ENV })) {
console.error(JSON.stringify({ event: 'config_error', variable: 'EGRESS_PROXY_URL', environment: env.SANDBOX_ENV }))
return json({ error: 'egress_proxy_missing' }, 503)
return json({ error: 'egress_proxy_missing', keySwap: 'off' }, 503)
}
return json({ ok: true }, 200)
// keySwap: whether model keys can be swapped at egress (the launch smoke asserts "on").
return json({ ok: true, keySwap: env.EGRESS_PROXY_URL ? 'on' : 'off' }, 200)
}
if (url.pathname === '/v1/config' && request.method === 'GET') {
// Public product constants (no auth): every client reads these instead of hardcoding them.
@@ -181,7 +181,7 @@ export function egressProxyMissing(config: { egressProxyUrl?: string; environmen
/** What an agent sees when its model key was withheld (design-squad wording). */
export const KEYS_WITHHELD_MESSAGE =
"Orca Cloud couldn't use your model API key in this workspace because a service on Orca's side isn't set up, so the key was not sent. Contact Orca support or your team admin."
"Orca Cloud didn't send your model API key or sign-in because of a problem on Orca's side, not with your key. Contact Orca support."
/** Provider-style error JSON: Anthropic clients read error.message, OpenAI-compatible ones too. */
export const KEYS_WITHHELD_BODY = JSON.stringify({ type: 'error', error: { type: 'permission_error', message: KEYS_WITHHELD_MESSAGE } })
+4 -4
View File
@@ -63,8 +63,8 @@ function serializable(value: unknown): unknown {
return value
}
export type SecretLogSink = (line: string) => void
let sink: SecretLogSink = (line) => console.log(line)
export type SecretLogSink = (line: string, level?: 'error') => void
let sink: SecretLogSink = (line, level) => (level === 'error' ? console.error(line) : console.log(line))
/** Tests only: capture lines exactly as they would reach Workers logs. Returns a restore function. */
export function setSecretLogSink(next: SecretLogSink): () => void {
@@ -76,12 +76,12 @@ export function setSecretLogSink(next: SecretLogSink): () => void {
}
/** One structured log line. `known`: values in scope at the call site that must never be written. */
export function secretLog(event: string, fields: Record<string, unknown> = {}, known: Iterable<string> = []): void {
export function secretLog(event: string, fields: Record<string, unknown> = {}, known: Iterable<string> = [], level?: 'error'): void {
let line: string
try {
line = JSON.stringify({ event, ...fields }, (_key, value: unknown) => serializable(value))
} catch {
line = JSON.stringify({ event, unserializable: true })
}
sink(redactLine(line, known))
sink(redactLine(line, known), level)
}
@@ -5,7 +5,7 @@ import type { LifecycleDeps } from './lifecycle'
import { agentModesFor, withoutSignInCredentials, type AgentMode } from './agent-sources'
import { accountProvider } from './account-providers'
import type { CredentialAgent } from './agent-credential-names'
import { deriveProviderEnv, deriveProviderScopes, providerHosts } from './provider-env'
import { SWAPPED_CREDENTIALS, deriveProviderEnv, deriveProviderScopes, providerHosts } from './provider-env'
import { egressProxyMissing } from './network-policy'
import { applySecretPlaceholders, encodeAccountRoute, encodeEgressSecretHosts } from './secret-egress'
import { resolveSandboxSecrets } from './secrets'
@@ -137,8 +137,10 @@ export async function sessionSecretEnv(
// Counts only.
secretLog('session_secrets', { sandboxId: row.id, injected: Object.keys(env).length, skipped })
if (withhold) {
const withheldSecrets = Object.fromEntries(SWAPPED_CREDENTIALS.filter((secret) => secret in env).map((secret) => [secret, true]))
if (firstWithheldReadThisSession(row)) {
secretLog('egress_swap_off_no_proxy', { sandboxId: row.id, environment: deps.config.environment })
// Counts only: no secret names or values.
secretLog('egress_swap_unavailable', { sandboxId: row.id, environment: deps.config.environment, withheld: Object.keys(withheldSecrets).length }, [], 'error')
}
const withheld = applySecretPlaceholders(row, env, scopes)
const accountEnv: Record<string, string> = {}
@@ -1,6 +1,7 @@
// EGRESS-SWAP-FAIL-CLOSED: a deployed env (not local/test) without EGRESS_PROXY_URL can't swap keys at egress.
// Model keys and Orca-account sign-ins are withheld (the VM gets placeholders, the firewall answers them with a
// plain error), the session logs egress_swap_off_no_proxy once, and /v1/health fails the post-deploy smoke.
// plain error), the session logs egress_swap_unavailable once (error level, counts only), and /v1/health fails the
// post-deploy smoke with keySwap "off".
// Local and test keep today's behaviour.
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { KEYS_WITHHELD_MESSAGE, compileNetworkPolicy } from '../src/network-policy'
@@ -14,9 +15,14 @@ const SETUP_TOKEN = 'sk-ant-oat01-FAKEsetupTOKEN_for_tests_only_000000000000'
let logs: string[] = []
beforeEach(() => {
logs = []
errors = []
for (const method of ['log', 'info', 'warn', 'error', 'debug'] as const) {
vi.spyOn(console, method).mockImplementation((...args: unknown[]) => {
logs.push(args.map((arg) => (typeof arg === 'string' ? arg : JSON.stringify(arg))).join(' '))
const line = args.map((arg) => (typeof arg === 'string' ? arg : JSON.stringify(arg))).join(' ')
logs.push(line)
if (method === 'error') {
errors.push(line)
}
})
}
})
@@ -42,7 +48,8 @@ function startEnv(h: Awaited<ReturnType<typeof createHarness>>): Record<string,
)
}
const withheldEvents = () => logs.filter((line) => line.includes('"event":"egress_swap_off_no_proxy"'))
let errors: string[] = []
const withheldEvents = () => errors.filter((line) => line.includes('"event":"egress_swap_unavailable"'))
describe('EGRESS-SWAP-FAIL-CLOSED', () => {
it('a deployed env without the proxy withholds model keys: placeholder in the VM, the firewall answers it with a plain error', async () => {
@@ -67,8 +74,10 @@ describe('EGRESS-SWAP-FAIL-CLOSED', () => {
})
expect(JSON.parse((answer as { response: { body: string } }).response.body).error.message).toBe(KEYS_WITHHELD_MESSAGE)
expect(rules.at(-1)).toMatchObject({ response: { statusCode: 403, body: 'orca-egress-denied' } })
// Logged once for the session, however many times its env is read.
// Logged once for the session at error level, however many times its env is read: a count, no names.
expect(withheldEvents()).toHaveLength(1)
expect(JSON.parse(withheldEvents()[0]!)).toEqual({ event: 'egress_swap_unavailable', sandboxId: id, environment: 'staging', withheld: 1 })
expect(withheldEvents()[0]).not.toContain('ANTHROPIC')
await h.call('POST', `/v1/sandboxes/${id}/env/refresh`, { bearer: mia })
expect(withheldEvents()).toHaveLength(1)
})
@@ -87,6 +96,19 @@ describe('EGRESS-SWAP-FAIL-CLOSED', () => {
})
})
it('a deployed env with the proxy behaves as today: placeholder forwarded to the proxy, no event', async () => {
const { h, mia } = await setup({ SANDBOX_ENV: 'staging', EGRESS_PROXY_URL: 'https://sandbox.test/v1/egress/proxy' })
await h.call('PUT', '/v1/secrets/personal/ANTHROPIC_API_KEY', { body: { value: KEY }, bearer: mia })
await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard' }, bearer: mia })
const env = startEnv(h)
expect(env.ANTHROPIC_API_KEY).not.toBe(KEY)
const box = [...h.vercel.boxes.values()][0]!
expect(box.policy.allow['*']).toEqual([])
expect(box.policy.allow['api.anthropic.com']![0]).toMatchObject({ forwardURL: 'https://sandbox.test/v1/egress/proxy' })
expect(JSON.stringify(box.policy)).not.toContain(KEYS_WITHHELD_MESSAGE)
expect(withheldEvents()).toHaveLength(0)
})
it('test and local keep today’s behaviour without a proxy: plain env injection, no event', async () => {
const { h, mia } = await setup({ SANDBOX_ENV: 'test' })
await h.call('PUT', '/v1/secrets/personal/ANTHROPIC_API_KEY', { body: { value: KEY }, bearer: mia })
@@ -99,11 +121,11 @@ describe('EGRESS-SWAP-FAIL-CLOSED', () => {
it('/v1/health is 503 egress_proxy_missing in a deployed env without the proxy, 200 with it or in local/test', async () => {
const missing = await setup({ SANDBOX_ENV: 'staging' })
expect(await missing.h.call('GET', '/v1/health')).toMatchObject({ status: 503, body: { error: 'egress_proxy_missing' } })
expect(await missing.h.call('GET', '/v1/health')).toEqual({ status: 503, body: { error: 'egress_proxy_missing', keySwap: 'off' } })
const present = await setup({ SANDBOX_ENV: 'staging', EGRESS_PROXY_URL: 'https://sandbox.test/v1/egress/proxy' })
expect(await present.h.call('GET', '/v1/health')).toMatchObject({ status: 200, body: { ok: true } })
expect(await present.h.call('GET', '/v1/health')).toEqual({ status: 200, body: { ok: true, keySwap: 'on' } })
const local = await setup({ SANDBOX_ENV: 'test' })
expect((await local.h.call('GET', '/v1/health')).status).toBe(200)
expect(await local.h.call('GET', '/v1/health')).toEqual({ status: 200, body: { ok: true, keySwap: 'off' } })
})
it('a respond route compiles to a matched direct response, before the trailing deny', () => {
@@ -18,7 +18,7 @@ async function limited() {
describe('a malformed ORCA_PLATFORM_HOST_SUFFIXES (PLATFORM-SUFFIX-GUARDS)', () => {
it('fails the health route a post-deploy smoke hits, with the code only; the log names the bad entry', async () => {
const h = await createHarness()
expect(await h.call('GET', '/v1/health')).toEqual({ status: 200, body: { ok: true } })
expect(await h.call('GET', '/v1/health')).toMatchObject({ status: 200, body: { ok: true } })
h.env.ORCA_PLATFORM_HOST_SUFFIXES = MALFORMED
const logged = vi.spyOn(console, 'error').mockImplementation(() => undefined)
try {