mirror of
https://github.com/stablyai/orca.git
synced 2026-11-17 19:34:29 +00:00
fix(cloud): EGRESS-SWAP-FAIL-CLOSED merged spec: egress_swap_unavailable at error level (count only), health keySwap on/off, design-squad message, deployed-with-proxy test
This commit is contained in:
@@ -1441,9 +1441,10 @@ async function routeRequest(request: Request, env: Env, deps: Deps): Promise<Res
|
||||
// EGRESS-SWAP-FAIL-CLOSED: without the proxy no key reaches an agent and open egress is off.
|
||||
if (egressProxyMissing({ egressProxyUrl: env.EGRESS_PROXY_URL, environment: env.SANDBOX_ENV })) {
|
||||
console.error(JSON.stringify({ event: 'config_error', variable: 'EGRESS_PROXY_URL', environment: env.SANDBOX_ENV }))
|
||||
return json({ error: 'egress_proxy_missing' }, 503)
|
||||
return json({ error: 'egress_proxy_missing', keySwap: 'off' }, 503)
|
||||
}
|
||||
return json({ ok: true }, 200)
|
||||
// keySwap: whether model keys can be swapped at egress (the launch smoke asserts "on").
|
||||
return json({ ok: true, keySwap: env.EGRESS_PROXY_URL ? 'on' : 'off' }, 200)
|
||||
}
|
||||
if (url.pathname === '/v1/config' && request.method === 'GET') {
|
||||
// Public product constants (no auth): every client reads these instead of hardcoding them.
|
||||
|
||||
@@ -181,7 +181,7 @@ export function egressProxyMissing(config: { egressProxyUrl?: string; environmen
|
||||
|
||||
/** What an agent sees when its model key was withheld (design-squad wording). */
|
||||
export const KEYS_WITHHELD_MESSAGE =
|
||||
"Orca Cloud couldn't use your model API key in this workspace because a service on Orca's side isn't set up, so the key was not sent. Contact Orca support or your team admin."
|
||||
"Orca Cloud didn't send your model API key or sign-in because of a problem on Orca's side, not with your key. Contact Orca support."
|
||||
/** Provider-style error JSON: Anthropic clients read error.message, OpenAI-compatible ones too. */
|
||||
export const KEYS_WITHHELD_BODY = JSON.stringify({ type: 'error', error: { type: 'permission_error', message: KEYS_WITHHELD_MESSAGE } })
|
||||
|
||||
|
||||
@@ -63,8 +63,8 @@ function serializable(value: unknown): unknown {
|
||||
return value
|
||||
}
|
||||
|
||||
export type SecretLogSink = (line: string) => void
|
||||
let sink: SecretLogSink = (line) => console.log(line)
|
||||
export type SecretLogSink = (line: string, level?: 'error') => void
|
||||
let sink: SecretLogSink = (line, level) => (level === 'error' ? console.error(line) : console.log(line))
|
||||
|
||||
/** Tests only: capture lines exactly as they would reach Workers logs. Returns a restore function. */
|
||||
export function setSecretLogSink(next: SecretLogSink): () => void {
|
||||
@@ -76,12 +76,12 @@ export function setSecretLogSink(next: SecretLogSink): () => void {
|
||||
}
|
||||
|
||||
/** One structured log line. `known`: values in scope at the call site that must never be written. */
|
||||
export function secretLog(event: string, fields: Record<string, unknown> = {}, known: Iterable<string> = []): void {
|
||||
export function secretLog(event: string, fields: Record<string, unknown> = {}, known: Iterable<string> = [], level?: 'error'): void {
|
||||
let line: string
|
||||
try {
|
||||
line = JSON.stringify({ event, ...fields }, (_key, value: unknown) => serializable(value))
|
||||
} catch {
|
||||
line = JSON.stringify({ event, unserializable: true })
|
||||
}
|
||||
sink(redactLine(line, known))
|
||||
sink(redactLine(line, known), level)
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ import type { LifecycleDeps } from './lifecycle'
|
||||
import { agentModesFor, withoutSignInCredentials, type AgentMode } from './agent-sources'
|
||||
import { accountProvider } from './account-providers'
|
||||
import type { CredentialAgent } from './agent-credential-names'
|
||||
import { deriveProviderEnv, deriveProviderScopes, providerHosts } from './provider-env'
|
||||
import { SWAPPED_CREDENTIALS, deriveProviderEnv, deriveProviderScopes, providerHosts } from './provider-env'
|
||||
import { egressProxyMissing } from './network-policy'
|
||||
import { applySecretPlaceholders, encodeAccountRoute, encodeEgressSecretHosts } from './secret-egress'
|
||||
import { resolveSandboxSecrets } from './secrets'
|
||||
@@ -137,8 +137,10 @@ export async function sessionSecretEnv(
|
||||
// Counts only.
|
||||
secretLog('session_secrets', { sandboxId: row.id, injected: Object.keys(env).length, skipped })
|
||||
if (withhold) {
|
||||
const withheldSecrets = Object.fromEntries(SWAPPED_CREDENTIALS.filter((secret) => secret in env).map((secret) => [secret, true]))
|
||||
if (firstWithheldReadThisSession(row)) {
|
||||
secretLog('egress_swap_off_no_proxy', { sandboxId: row.id, environment: deps.config.environment })
|
||||
// Counts only: no secret names or values.
|
||||
secretLog('egress_swap_unavailable', { sandboxId: row.id, environment: deps.config.environment, withheld: Object.keys(withheldSecrets).length }, [], 'error')
|
||||
}
|
||||
const withheld = applySecretPlaceholders(row, env, scopes)
|
||||
const accountEnv: Record<string, string> = {}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// EGRESS-SWAP-FAIL-CLOSED: a deployed env (not local/test) without EGRESS_PROXY_URL can't swap keys at egress.
|
||||
// Model keys and Orca-account sign-ins are withheld (the VM gets placeholders, the firewall answers them with a
|
||||
// plain error), the session logs egress_swap_off_no_proxy once, and /v1/health fails the post-deploy smoke.
|
||||
// plain error), the session logs egress_swap_unavailable once (error level, counts only), and /v1/health fails the
|
||||
// post-deploy smoke with keySwap "off".
|
||||
// Local and test keep today's behaviour.
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { KEYS_WITHHELD_MESSAGE, compileNetworkPolicy } from '../src/network-policy'
|
||||
@@ -14,9 +15,14 @@ const SETUP_TOKEN = 'sk-ant-oat01-FAKEsetupTOKEN_for_tests_only_000000000000'
|
||||
let logs: string[] = []
|
||||
beforeEach(() => {
|
||||
logs = []
|
||||
errors = []
|
||||
for (const method of ['log', 'info', 'warn', 'error', 'debug'] as const) {
|
||||
vi.spyOn(console, method).mockImplementation((...args: unknown[]) => {
|
||||
logs.push(args.map((arg) => (typeof arg === 'string' ? arg : JSON.stringify(arg))).join(' '))
|
||||
const line = args.map((arg) => (typeof arg === 'string' ? arg : JSON.stringify(arg))).join(' ')
|
||||
logs.push(line)
|
||||
if (method === 'error') {
|
||||
errors.push(line)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
@@ -42,7 +48,8 @@ function startEnv(h: Awaited<ReturnType<typeof createHarness>>): Record<string,
|
||||
)
|
||||
}
|
||||
|
||||
const withheldEvents = () => logs.filter((line) => line.includes('"event":"egress_swap_off_no_proxy"'))
|
||||
let errors: string[] = []
|
||||
const withheldEvents = () => errors.filter((line) => line.includes('"event":"egress_swap_unavailable"'))
|
||||
|
||||
describe('EGRESS-SWAP-FAIL-CLOSED', () => {
|
||||
it('a deployed env without the proxy withholds model keys: placeholder in the VM, the firewall answers it with a plain error', async () => {
|
||||
@@ -67,8 +74,10 @@ describe('EGRESS-SWAP-FAIL-CLOSED', () => {
|
||||
})
|
||||
expect(JSON.parse((answer as { response: { body: string } }).response.body).error.message).toBe(KEYS_WITHHELD_MESSAGE)
|
||||
expect(rules.at(-1)).toMatchObject({ response: { statusCode: 403, body: 'orca-egress-denied' } })
|
||||
// Logged once for the session, however many times its env is read.
|
||||
// Logged once for the session at error level, however many times its env is read: a count, no names.
|
||||
expect(withheldEvents()).toHaveLength(1)
|
||||
expect(JSON.parse(withheldEvents()[0]!)).toEqual({ event: 'egress_swap_unavailable', sandboxId: id, environment: 'staging', withheld: 1 })
|
||||
expect(withheldEvents()[0]).not.toContain('ANTHROPIC')
|
||||
await h.call('POST', `/v1/sandboxes/${id}/env/refresh`, { bearer: mia })
|
||||
expect(withheldEvents()).toHaveLength(1)
|
||||
})
|
||||
@@ -87,6 +96,19 @@ describe('EGRESS-SWAP-FAIL-CLOSED', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('a deployed env with the proxy behaves as today: placeholder forwarded to the proxy, no event', async () => {
|
||||
const { h, mia } = await setup({ SANDBOX_ENV: 'staging', EGRESS_PROXY_URL: 'https://sandbox.test/v1/egress/proxy' })
|
||||
await h.call('PUT', '/v1/secrets/personal/ANTHROPIC_API_KEY', { body: { value: KEY }, bearer: mia })
|
||||
await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard' }, bearer: mia })
|
||||
const env = startEnv(h)
|
||||
expect(env.ANTHROPIC_API_KEY).not.toBe(KEY)
|
||||
const box = [...h.vercel.boxes.values()][0]!
|
||||
expect(box.policy.allow['*']).toEqual([])
|
||||
expect(box.policy.allow['api.anthropic.com']![0]).toMatchObject({ forwardURL: 'https://sandbox.test/v1/egress/proxy' })
|
||||
expect(JSON.stringify(box.policy)).not.toContain(KEYS_WITHHELD_MESSAGE)
|
||||
expect(withheldEvents()).toHaveLength(0)
|
||||
})
|
||||
|
||||
it('test and local keep today’s behaviour without a proxy: plain env injection, no event', async () => {
|
||||
const { h, mia } = await setup({ SANDBOX_ENV: 'test' })
|
||||
await h.call('PUT', '/v1/secrets/personal/ANTHROPIC_API_KEY', { body: { value: KEY }, bearer: mia })
|
||||
@@ -99,11 +121,11 @@ describe('EGRESS-SWAP-FAIL-CLOSED', () => {
|
||||
|
||||
it('/v1/health is 503 egress_proxy_missing in a deployed env without the proxy, 200 with it or in local/test', async () => {
|
||||
const missing = await setup({ SANDBOX_ENV: 'staging' })
|
||||
expect(await missing.h.call('GET', '/v1/health')).toMatchObject({ status: 503, body: { error: 'egress_proxy_missing' } })
|
||||
expect(await missing.h.call('GET', '/v1/health')).toEqual({ status: 503, body: { error: 'egress_proxy_missing', keySwap: 'off' } })
|
||||
const present = await setup({ SANDBOX_ENV: 'staging', EGRESS_PROXY_URL: 'https://sandbox.test/v1/egress/proxy' })
|
||||
expect(await present.h.call('GET', '/v1/health')).toMatchObject({ status: 200, body: { ok: true } })
|
||||
expect(await present.h.call('GET', '/v1/health')).toEqual({ status: 200, body: { ok: true, keySwap: 'on' } })
|
||||
const local = await setup({ SANDBOX_ENV: 'test' })
|
||||
expect((await local.h.call('GET', '/v1/health')).status).toBe(200)
|
||||
expect(await local.h.call('GET', '/v1/health')).toEqual({ status: 200, body: { ok: true, keySwap: 'off' } })
|
||||
})
|
||||
|
||||
it('a respond route compiles to a matched direct response, before the trailing deny', () => {
|
||||
|
||||
@@ -18,7 +18,7 @@ async function limited() {
|
||||
describe('a malformed ORCA_PLATFORM_HOST_SUFFIXES (PLATFORM-SUFFIX-GUARDS)', () => {
|
||||
it('fails the health route a post-deploy smoke hits, with the code only; the log names the bad entry', async () => {
|
||||
const h = await createHarness()
|
||||
expect(await h.call('GET', '/v1/health')).toEqual({ status: 200, body: { ok: true } })
|
||||
expect(await h.call('GET', '/v1/health')).toMatchObject({ status: 200, body: { ok: true } })
|
||||
h.env.ORCA_PLATFORM_HOST_SUFFIXES = MALFORMED
|
||||
const logged = vi.spyOn(console, 'error').mockImplementation(() => undefined)
|
||||
try {
|
||||
|
||||
Reference in New Issue
Block a user