diff --git a/.github/workflows/cloud-egress-probe.yml b/.github/workflows/cloud-egress-probe.yml index 1b5d983ffea..28eba29ebc7 100644 --- a/.github/workflows/cloud-egress-probe.yml +++ b/.github/workflows/cloud-egress-probe.yml @@ -5,6 +5,13 @@ name: Cloud Egress Probe # Details and the local run: cloud/apps/sandbox-control/scripts/README.md on: workflow_dispatch: + inputs: + mode: + description: EGRESS_SWAP_MODE to check (both runs proxy, then firewall, on the same workspaces) + required: false + default: both + type: choice + options: [both, firewall, proxy] permissions: contents: read @@ -24,6 +31,7 @@ jobs: env: DEV_TEST_JWK: ${{ secrets.ORCA_DEV_TEST_JWK }} DEV_OPERATOR_TOKEN: ${{ secrets.ORCA_DEV_OPERATOR_TOKEN }} + MODE: ${{ inputs.mode || 'both' }} steps: - name: Check the dev credentials are configured id: creds @@ -71,7 +79,7 @@ jobs: printf '%s' "$DEV_TEST_JWK" > "$RUNNER_TEMP/dev-jwk.json" printf '%s' "$DEV_OPERATOR_TOKEN" > "$RUNNER_TEMP/dev-operator" ORCA_PROBE_DEV_JWK_FILE="$RUNNER_TEMP/dev-jwk.json" ORCA_PROBE_OPERATOR_TOKEN_FILE="$RUNNER_TEMP/dev-operator" \ - pnpm --filter @orca-cloud/sandbox-control exec tsx scripts/egress-probe.ts > "$RUNNER_TEMP/egress-probe.json" + pnpm --filter @orca-cloud/sandbox-control exec tsx scripts/egress-probe.ts --mode "$MODE" > "$RUNNER_TEMP/egress-probe.json" - uses: actions/upload-artifact@v4 if: always() && steps.creds.outputs.skip != '1' diff --git a/cloud/apps/sandbox-control/scripts/README.md b/cloud/apps/sandbox-control/scripts/README.md index b05ebfbc173..02f27ee6ea5 100644 --- a/cloud/apps/sandbox-control/scripts/README.md +++ b/cloud/apps/sandbox-control/scripts/README.md @@ -3,7 +3,8 @@ `egress-probe.ts` checks sandbox egress against **dev** before a release. It goes through Orca's own paths: real dev workspaces created with a dev-JWK user token, with commands run in a workspace terminal over the shipped `orca` CLI's pairing code (the desktop's E2EE path to orcad). Every -credential is bogus, and postman-echo.com plays the model provider. +credential is bogus. httpbin.org plays the model provider, ws.postman-echo.com is the WebSocket +host, and postman-echo.com is an unrelated host. Setup for a throwaway identity, all of it removed afterwards: @@ -11,23 +12,38 @@ Setup for a throwaway identity, all of it removed afterwards: - a stored GitHub identity, so member placeholders exist (the dev `member-github` route); - bogus personal secrets: - `ANTHROPIC_API_KEY`, with `ANTHROPIC_BASE_URL` set to the echo host; - - `OPENAI_API_KEY`, with `OPENAI_BASE_URL` set to `chatgpt.com/backend-api/codex`, so its egress - takes the ChatGPT relay. + - `OPENAI_API_KEY`, with `OPENAI_BASE_URL` set to `chatgpt.com/backend-api/codex`; + - `COPILOT_PROVIDER_API_KEY`, with `COPILOT_PROVIDER_BASE_URL` set to the WebSocket echo host. The probe creates two workspaces through the desktop's create path (clone after ready): - **public**: a public repo the App isn't installed on. Member placeholder, no App token (#758). - **app**: the App-installed test repo. App token, github.com fail-closed. -| Host | What must happen | -|---|---| -| echo host (custom base URL) | The provider sees the bogus key where the VM sent the placeholder. The other `authorization` header is unchanged, and no `vercel-forwarded-*` header or OIDC token arrives. A foreign key gets the firewall 403 `orca-egress-denied`. | -| chatgpt.com | The placeholder reaches ChatGPT: any provider answer counts, but not the firewall 403 and not Cloudflare's edge page. A foreign bearer gets the firewall 403. | -| api.github.com | No credential gets the firewall 403. With the workspace placeholder, GitHub answers 200. | -| github.com | Anonymous `git ls-remote` works on **public** (#758) and is refused on **app**. The app repo clones, and the workspace's own git works. | -| VM env | `ANTHROPIC_API_KEY` and `OPENAI_API_KEY` hold placeholders. This is checked by sha256, so the bogus values never cross the terminal. | +Each mode is checked on the same two workspaces. `--mode proxy|firewall|both` flips them with the +operator route `POST /v1/internal/sandboxes/:id/egress-mode`; without `--mode`, the environment's +default applies. Every request is a Node `fetch` (undici, like the agents) with a 30 s limit, so the +undici body hang shows up as a named failure. -The probe takes about 1–3 minutes, deletes what it created (`--keep` leaves it), prints one JSON +| Host or class | What must happen | +|---|---| +| echo host (custom base URL, a model key) | The provider sees the bogus key where the VM sent the placeholder. The other `authorization` header arrives exactly as sent, and no `vercel-forwarded-*` header or OIDC token arrives. A foreign key gets the firewall 403 `orca-egress-denied`. A chunked stream finishes. A 1 MB upload goes through: the public echo hosts refuse bigger bodies themselves. | +| chatgpt.com | The placeholder reaches ChatGPT: any provider answer counts, but not the firewall 403 and not Cloudflare's edge page. A foreign bearer gets the firewall 403. | +| WebSocket host | A WebSocket upgrade (101) through the swap. No credential gets the firewall 403. | +| an unrelated host | A placeholder sent there arrives unswapped: the real value never leaves for the wrong host. | +| api.github.com | No credential gets the firewall 403. With the workspace placeholder, GitHub answers. | +| github.com | Git smart-HTTP and the LFS batch API work with the workspace header, as do `git fetch`, a push of a probe branch and its delete. Anonymous `git ls-remote` works on **public** (#758) and is refused on **app**. Both repos clone through the desktop's create. | +| VM env | The keys hold placeholders. This is checked by sha256, so the bogus values never cross the terminal. | + +The report includes latency per class (median and p90 over 5 keep-alive samples where set): +model_key, model_key_stream, upload_1mb, codex, websocket, github_api, git_http, git_lfs. It +reports these per mode and per workspace. + +What it can't cover with bogus credentials: real provider SSE (Anthropic `/v1/messages`, Codex +`/responses` answer 401 before streaming). The chunked stream from the echo host exercises the same +transform path instead. + +The probe takes about 3 minutes per mode, deletes what it created (`--keep` leaves it), prints one JSON report and exits 1 on any failed expectation. It refuses any control plane other than dev. ## Run it locally @@ -43,7 +59,7 @@ gcloud --account secrets versions access latest --project onorca-cloud-sta gcloud --account secrets versions access latest --project onorca-cloud-staging \ --secret orca-sandbox-control-dev-operator-token > "$K/operator" ORCA_PROBE_DEV_JWK_FILE="$K/jwk.json" ORCA_PROBE_OPERATOR_TOKEN_FILE="$K/operator" \ - pnpm --filter @orca-cloud/sandbox-control exec tsx scripts/egress-probe.ts + pnpm --filter @orca-cloud/sandbox-control exec tsx scripts/egress-probe.ts --mode both rm -rf "$K" ``` diff --git a/cloud/apps/sandbox-control/scripts/egress-probe-checks.ts b/cloud/apps/sandbox-control/scripts/egress-probe-checks.ts index e917d92c7b2..eeb85310c96 100644 --- a/cloud/apps/sandbox-control/scripts/egress-probe-checks.ts +++ b/cloud/apps/sandbox-control/scripts/egress-probe-checks.ts @@ -7,63 +7,132 @@ import { EGRESS_DENIED_BODY } from '../src/network-policy' * One request from inside the workspace. A header value `$NAME` is read from the VM's env (orcad's * environment, as agents get it), so placeholders never cross the terminal. */ -export type ProbeRequest = { name: string; url: string; method?: 'GET' | 'POST'; headers?: Record } +export type ProbeRequest = { + name: string + url: string + method?: 'GET' | 'POST' + headers?: Record + /** POST a body of this many bytes (large uploads). */ + uploadBytes?: number + /** Read the body chunk by chunk (SSE / chunked streams): reports chunks and time to the first one. */ + stream?: boolean + /** An HTTP/1.1 WebSocket upgrade instead of a fetch: reports 101 on success. */ + websocket?: boolean + /** Samples for latency (sequential, one keep-alive client, like an agent's calls). Default 1. */ + repeat?: number + /** Latency class this request measures (report key). */ + latency?: string +} /** What the answer must be. */ export type Expectation = - /** The echo host answered as the provider: these headers (by sha256) arrived, and nothing from Vercel's hop. */ - | { kind: 'swapped'; headers: Record } + /** The echo host answered as the provider: `headers` arrived with these values (the swap), `keep` arrived as sent. */ + | { kind: 'swapped'; headers: Record; keep?: readonly string[] } + /** The headers arrived exactly as sent: a placeholder on a host it isn't for is never swapped. */ + | { kind: 'unswapped'; headers: readonly string[] } /** The firewall's own 403 (a credential host fails closed). */ | { kind: 'denied' } - /** The request reached the provider itself: not the firewall's 403, not an edge block page. */ + /** The provider itself answered: not the firewall's 403, not an edge block page, no hang. */ | { kind: 'upstream' } + /** A 200 that streamed at least this many chunks without hanging. */ + | { kind: 'streamed'; minChunks: number } + /** The WebSocket upgrade went through (101). */ + | { kind: 'upgraded' } export type ProbeCase = ProbeRequest & { expect: Expectation } -/** What the runner reports per request: never a header value, only sha256 prefixes of the echoed ones. */ +/** What the runner reports per request: never a header value, only sha256 prefixes. */ export type Summary = { status: number - /** Body is the firewall's deny body. */ denied: boolean - /** An HTML 403 from Cloudflare: the provider's edge refused the proxy. */ edgeBlocked: boolean - /** The echo host's answer: sha256 prefix of each echoed header, and Vercel hop headers seen. */ + /** The client gave up waiting for the answer or its body (the undici framing hang). */ + hang: boolean + /** Total ms per sample. */ + ms: number[] + chunks?: number + ttfbMs?: number + /** sha256 prefix of each header as the echo host received it, and as the VM sent it. */ echoed?: Record + sent?: Record leaked?: string[] } export const sha8 = (value: string): string => createHash('sha256').update(value).digest('hex').slice(0, 8) /** - * Runs in the workspace (node, from a terminal): fetches each request and prints one short line per - * fact, `PRB =`, ending with `PRB-END`. Short lines survive the terminal intact. - * The echo host (postman-echo.com) answers JSON with the request's headers under `headers`. + * Runs in the workspace (node, from a terminal). Prints one short line per fact, `PRB =`, + * then `PRB-END`. Short lines survive the terminal intact. Echo hosts answer JSON with the request's + * headers under `headers` (postman-echo.com, httpbin.org); header names are compared lowercased. */ export const VM_RUNNER = ` const { createHash } = require('node:crypto') +const https = require('node:https') const cases = JSON.parse(Buffer.from(process.argv[1], 'base64').toString()) const sha8 = (v) => createHash('sha256').update(String(v)).digest('hex').slice(0, 8) const out = (i, k, v) => console.log('PRB ' + i + ' ' + k + '=' + v) +const DENIED = ${JSON.stringify(EGRESS_DENIED_BODY)} +function upgrade(c, headers) { + return new Promise((resolve) => { + const u = new URL(c.url) + const req = https.request({ host: u.hostname, path: u.pathname + u.search, headers: { ...headers, Connection: 'Upgrade', Upgrade: 'websocket', 'Sec-WebSocket-Version': '13', 'Sec-WebSocket-Key': 'dGhlIHNhbXBsZSBub25jZQ==' }, timeout: 20000 }) + req.on('upgrade', (res, socket) => { socket.destroy(); resolve({ status: res.statusCode, body: '' }) }) + req.on('response', (res) => { let b = ''; res.on('data', (d) => (b += d)); res.on('end', () => resolve({ status: res.statusCode, body: b })) }) + req.on('timeout', () => { req.destroy(); resolve({ status: 0, hang: true, body: '' }) }) + req.on('error', () => resolve({ status: 0, body: '' })) + req.end() + }) +} ;(async () => { for (const [i, c] of cases.entries()) { const headers = {} - for (const [k, v] of Object.entries(c.headers || {})) headers[k] = v.startsWith('$') ? process.env[v.slice(1)] || '' : v - try { - const r = await fetch(c.url, { method: c.method || 'GET', headers, body: c.method === 'POST' ? '{}' : undefined, redirect: 'manual', signal: AbortSignal.timeout(25000) }) - const body = await r.text() + for (const [k, v] of Object.entries(c.headers || {})) { + headers[k] = v.startsWith('$') ? process.env[v.slice(1)] || '' : v + out(i, 'sent.' + k.toLowerCase(), sha8(headers[k])) + } + for (let n = 0; n < (c.repeat || 1); n++) { + const t0 = Date.now() + let r = { status: 0, body: '' } + if (c.websocket) { + r = await upgrade(c, headers) + } else { + try { + const body = c.uploadBytes ? Buffer.alloc(c.uploadBytes, 97) : c.method === 'POST' ? '{}' : undefined + const res = await fetch(c.url, { method: c.method || 'GET', headers, body, redirect: 'manual', signal: AbortSignal.timeout(30000) }) + let text = '' + if (c.stream) { + let chunks = 0 + const reader = res.body.getReader() + for (;;) { + const { done, value } = await reader.read() + if (done) break + if (chunks++ === 0) out(i, 'ttfb', Date.now() - t0) + if (text.length < 4000) text += Buffer.from(value).toString() + } + out(i, 'chunks', chunks) + } else { + text = await res.text() + } + r = { status: res.status, body: text, cf: res.status === 403 && /text\\/html/.test(res.headers.get('content-type') || '') && /cloudflare/i.test(res.headers.get('server') || '') } + } catch (e) { + r = { status: 0, body: '', hang: e && (e.name === 'TimeoutError' || e.name === 'AbortError') } + } + } + out(i, 'ms', Date.now() - t0) + if (n > 0) continue out(i, 's', r.status) - out(i, 'd', body.trim() === ${JSON.stringify(EGRESS_DENIED_BODY)} ? 1 : 0) - out(i, 'cf', r.status === 403 && /text\\/html/.test(r.headers.get('content-type') || '') && /cloudflare/i.test(r.headers.get('server') || '') ? 1 : 0) + out(i, 'd', r.body.trim() === DENIED ? 1 : 0) + out(i, 'cf', r.cf ? 1 : 0) + out(i, 'hang', r.hang ? 1 : 0) let echo = null - try { echo = JSON.parse(body).headers } catch {} + try { echo = JSON.parse(r.body).headers } catch {} if (echo && typeof echo === 'object') { for (const [k, v] of Object.entries(echo)) { - if (/^vercel-forwarded-|oidc/i.test(k)) out(i, 'leak', k) - else if (k in (c.headers || {})) out(i, 'h.' + k, sha8(v)) + const name = k.toLowerCase() + if (/^vercel-forwarded-|oidc/.test(name)) out(i, 'leak', name) + else if (Object.keys(c.headers || {}).some((h) => h.toLowerCase() === name)) out(i, 'h.' + name, sha8(v)) } } - } catch (e) { - out(i, 's', 0) } } console.log('PRB-END') @@ -71,18 +140,21 @@ const out = (i, k, v) => console.log('PRB ' + i + ' ' + k + '=' + v) ` export function parseRunnerOutput(text: string, count: number): Summary[] { - const summaries: Summary[] = Array.from({ length: count }, () => ({ status: 0, denied: false, edgeBlocked: false })) - for (const match of text.matchAll(/PRB (\d+) ([a-z.-]+)=(\S+)/g)) { + const summaries: Summary[] = Array.from({ length: count }, () => ({ status: 0, denied: false, edgeBlocked: false, hang: false, ms: [] })) + for (const match of text.matchAll(/PRB (\d+) ([a-z0-9.-]+)=(\S+)/g)) { const summary = summaries[Number(match[1])] const [key, value] = [match[2]!, match[3]!] - if (!summary) { - continue - } + if (!summary) continue if (key === 's') summary.status = Number(value) else if (key === 'd') summary.denied = value === '1' else if (key === 'cf') summary.edgeBlocked = value === '1' + else if (key === 'hang') summary.hang = value === '1' + else if (key === 'ms') summary.ms.push(Number(value)) + else if (key === 'chunks') summary.chunks = Number(value) + else if (key === 'ttfb') summary.ttfbMs = Number(value) else if (key === 'leak') summary.leaked = [...(summary.leaked ?? []), value] else if (key.startsWith('h.')) summary.echoed = { ...summary.echoed, [key.slice(2)]: value } + else if (key.startsWith('sent.')) summary.sent = { ...summary.sent, [key.slice(5)]: value } } return summaries } @@ -91,24 +163,60 @@ export function parseRunnerOutput(text: string, count: number): Summary[] { export function judge(probe: ProbeCase, answer: Summary): string[] { const { expect } = probe const denied = answer.status === 403 && answer.denied - if (expect.kind === 'denied') { - return denied ? [] : [`${probe.name}: expected the firewall's 403 ${EGRESS_DENIED_BODY}, got ${answer.status}`] + if (answer.hang) { + return [`${probe.name}: the client hung waiting for the answer (undici framing hang?)`] } - if (expect.kind === 'upstream') { - if (denied) return [`${probe.name}: refused by the firewall`] - if (answer.edgeBlocked) return [`${probe.name}: the provider's edge refused the egress proxy (Cloudflare 403 page)`] - return answer.status === 0 ? [`${probe.name}: no answer`] : [] - } - if (answer.status !== 200 || !answer.echoed) { - return [`${probe.name}: expected the echo host's answer, got ${answer.status}${denied ? ' (denied)' : ''}`] - } - const problems = (answer.leaked ?? []).map((name) => `${probe.name}: the provider saw ${name}`) - for (const [name, value] of Object.entries(expect.headers)) { - if (answer.echoed[name] !== sha8(value)) { - problems.push(`${probe.name}: header ${name} ${answer.echoed[name] ? 'arrived with another value' : 'did not arrive'}`) + switch (expect.kind) { + case 'denied': + return denied ? [] : [`${probe.name}: expected the firewall's 403 ${EGRESS_DENIED_BODY}, got ${answer.status}`] + case 'upstream': + if (denied) return [`${probe.name}: refused by the firewall`] + if (answer.edgeBlocked) return [`${probe.name}: the provider's edge refused our egress (Cloudflare 403 page)`] + return answer.status === 0 ? [`${probe.name}: no answer`] : [] + case 'upgraded': + return answer.status === 101 ? [] : [`${probe.name}: expected a WebSocket upgrade (101), got ${answer.status}${denied ? ' (denied)' : ''}`] + case 'streamed': + return answer.status === 200 && (answer.chunks ?? 0) >= expect.minChunks + ? [] + : [`${probe.name}: expected a 200 stream of ${expect.minChunks}+ chunks, got ${answer.status} with ${answer.chunks ?? 0}`] + case 'swapped': + case 'unswapped': { + if (answer.status !== 200 || !answer.echoed) { + return [`${probe.name}: expected the echo host's answer, got ${answer.status}${denied ? ' (denied)' : ''}`] + } + const problems = (answer.leaked ?? []).map((name) => `${probe.name}: the provider saw ${name}`) + const swapped = expect.kind === 'swapped' ? expect.headers : {} + for (const [name, value] of Object.entries(swapped)) { + if (answer.echoed[name] !== sha8(value)) { + problems.push(`${probe.name}: header ${name} ${answer.echoed[name] ? 'arrived with another value' : 'did not arrive'}`) + } + } + const unchanged = expect.kind === 'swapped' ? (expect.keep ?? []) : expect.headers + for (const name of unchanged) { + if (!answer.echoed[name] || answer.echoed[name] !== answer.sent?.[name]) { + problems.push(`${probe.name}: header ${name} ${answer.echoed[name] ? 'was changed on the way' : 'did not arrive'}`) + } + } + return problems } } - return problems +} + +/** Median and p90 of each latency class across these answers. */ +export function latencyByClass(cases: readonly ProbeCase[], answers: readonly Summary[]): Record { + const samples: Record = {} + cases.forEach((probe, i) => { + if (probe.latency && answers[i] && !answers[i]!.hang && answers[i]!.status !== 0) { + ;(samples[probe.latency] ??= []).push(...answers[i]!.ms) + } + }) + const pick = (sorted: number[], q: number) => sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))]! + return Object.fromEntries( + Object.entries(samples).map(([name, values]) => { + const sorted = [...values].sort((a, b) => a - b) + return [name, { n: sorted.length, medianMs: pick(sorted, 0.5), p90Ms: pick(sorted, 0.9) }] + }) + ) } /** The terminal line that runs the cases: the runner and its input both base64, one shell word each. */ diff --git a/cloud/apps/sandbox-control/scripts/egress-probe.ts b/cloud/apps/sandbox-control/scripts/egress-probe.ts index 0f3e61d3b57..371a108b025 100644 --- a/cloud/apps/sandbox-control/scripts/egress-probe.ts +++ b/cloud/apps/sandbox-control/scripts/egress-probe.ts @@ -1,6 +1,6 @@ // Release probe for sandbox egress, against DEV only, through Orca's own paths: real dev workspaces // created with a dev-JWK user token, driven from a terminal over the `orca` CLI's pairing code (the -// desktop's E2EE path). Bogus tokens only; postman-echo.com plays the model provider. +// desktop's E2EE path). Bogus tokens only; httpbin.org plays the model provider. // pnpm --filter @orca-cloud/sandbox-control exec tsx scripts/egress-probe.ts [--keep] // // A throwaway identity gets: the dev GitHub App installation bound for one test repo (operator), a @@ -29,7 +29,7 @@ import { execFile } from 'node:child_process' import { randomBytes } from 'node:crypto' import { readFileSync } from 'node:fs' import { SignJWT, importJWK, type JWK } from 'jose' -import { judge, parseRunnerOutput, runnerCommand, sha8, type ProbeCase } from './egress-probe-checks' +import { judge, latencyByClass, parseRunnerOutput, runnerCommand, sha8, type ProbeCase } from './egress-probe-checks' const CONTROL_URL = process.env.ORCA_PROBE_CONTROL_URL ?? 'https://orca-sandbox-control-dev.lovecast.workers.dev' const ORCA_CLI = process.env.ORCA_PROBE_CLI ?? '/Applications/Orca.app/Contents/Resources/bin/orca' @@ -37,13 +37,22 @@ const ORCA_CLI = process.env.ORCA_PROBE_CLI ?? '/Applications/Orca.app/Contents/ const APP_REPO = process.env.ORCA_PROBE_APP_REPO ?? 'https://github.com/nwparker/orca-sandbox-push-test' /** Public, and the dev App is not installed on its owner. */ const PUBLIC_REPO = 'https://github.com/octocat/Hello-World' -const ECHO = 'https://postman-echo.com' +/** The model provider stand-in (ANTHROPIC_BASE_URL): echoes headers, streams, takes bodies up to ~1 MB. */ +const ECHO = 'https://httpbin.org' +/** A WebSocket echo: COPILOT_PROVIDER_BASE_URL points here, so it carries a credential claim. */ +const WS_ECHO = 'https://ws.postman-echo.com' +/** A host no credential is for: a placeholder sent here must arrive unswapped. */ +const OTHER_ECHO = 'https://postman-echo.com' +type Mode = 'proxy' | 'firewall' if (!/^https:\/\/[a-z0-9-]+-dev\.lovecast\.workers\.dev$|^http:\/\/(127\.0\.0\.1|localhost)(:\d+)?$/.test(CONTROL_URL)) { throw new Error(`refusing a non-dev control plane: ${CONTROL_URL}`) } const keep = process.argv.includes('--keep') +const modeArg = process.argv.includes('--mode') ? process.argv[process.argv.indexOf('--mode') + 1] : 'default' +/** `default`: the environment's EGRESS_SWAP_MODE, unchanged. Otherwise each mode in turn, set per sandbox. */ +const MODES: readonly (Mode | 'default')[] = modeArg === 'both' ? ['proxy', 'firewall'] : modeArg === 'proxy' || modeArg === 'firewall' ? [modeArg] : ['default'] const runId = randomBytes(4).toString('hex') const sub = `egress-probe-${runId}` const accountId = `dev:${sub}` @@ -106,15 +115,24 @@ const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) /** Runs one line in a fresh workspace terminal and returns the output once `PRB-END` shows. */ async function inTerminal(pairingCode: string, projectRoot: string, line: string): Promise { - await orca(pairingCode, ['repo', 'add', '--path', projectRoot, '--kind', 'git']) - const created = await orca(pairingCode, ['terminal', 'create', '--worktree', `path:${projectRoot}`, '--title', 'egress-probe']) - const handle: string | undefined = created?.result?.terminal?.handle ?? created?.result?.handle ?? created?.result?.terminal?.id + // A project whose clone failed is not a git repo: register it as a folder then (the checks still run). + const repo = await orca(pairingCode, ['repo', 'add', '--path', projectRoot, '--kind', 'git']) + if (!repo?.ok) { + await orca(pairingCode, ['repo', 'add', '--path', projectRoot, '--kind', 'folder']) + } + let handle: string | undefined + for (let attempt = 0; attempt < 3 && !handle; attempt++) { + const created = await orca(pairingCode, ['terminal', 'create', '--worktree', `path:${projectRoot}`, '--title', 'egress-probe']) + handle = created?.result?.terminal?.handle ?? created?.result?.handle ?? created?.result?.terminal?.id + if (!handle) await sleep(2_000) + } if (!handle) { throw new Error('could not open a workspace terminal') } // `clear` first: the read must only see this run's lines. await orca(pairingCode, ['terminal', 'send', '--terminal', handle, '--text', `clear; ${line}`, '--enter']) - const deadline = Date.now() + 180_000 + // Generous: in a mode with hangs every request may take its full 30 s client timeout. + const deadline = Date.now() + 900_000 while (Date.now() < deadline) { const read = await orca(pairingCode, ['terminal', 'read', '--terminal', handle]) const text = textOf(read?.result) @@ -140,14 +158,24 @@ function factsScript(gitChecks: Record, bogus: Record { const failures: string[] = [] - const report: Record = { runId } + const report: Record = { runId, modes: MODES } const cleanup: (() => Promise)[] = [] const started = Date.now() - console.error(`[egress-probe] run ${runId}: throwaway identity ${sub}`) + console.error(`[egress-probe] run ${runId}: throwaway identity ${sub}, modes ${MODES.join(', ')}`) const user = client(await userToken()) const operator = client(fileSecret('ORCA_PROBE_OPERATOR_TOKEN_FILE')) - const bogusAnthropic = `sk-ant-api03-orcaprobe${randomBytes(16).toString('hex')}` - const bogusOpenAI = `sk-proj-orcaprobe${randomBytes(16).toString('hex')}` + const bogus = { + ANTHROPIC_API_KEY: `sk-ant-api03-orcaprobe${randomBytes(16).toString('hex')}`, + OPENAI_API_KEY: `sk-proj-orcaprobe${randomBytes(16).toString('hex')}`, + COPILOT_PROVIDER_API_KEY: `orcaprobe-${randomBytes(16).toString('hex')}` + } + const results: Record[] = [] + const check = (mode: string, workspace: string, name: string, ok: boolean, detail: string) => { + results.push({ mode, workspace, check: name, ok }) + if (!ok) { + failures.push(`[${mode}/${workspace}] ${name}: ${detail}`) + } + } try { // Identity: the App bound for the test repo, and a stored GitHub identity (member placeholders). const bound = await operator('POST', '/v1/internal/github/bindings', { repoUrl: APP_REPO, accountId }) @@ -161,10 +189,12 @@ async function main(): Promise { throw new Error(`store the probe's GitHub identity: ${member.status} ${member.body?.error ?? ''}`) } for (const [name, value] of [ - ['ANTHROPIC_API_KEY', bogusAnthropic], + ['ANTHROPIC_API_KEY', bogus.ANTHROPIC_API_KEY], ['ANTHROPIC_BASE_URL', ECHO], - ['OPENAI_API_KEY', bogusOpenAI], - ['OPENAI_BASE_URL', 'https://chatgpt.com/backend-api/codex'] + ['OPENAI_API_KEY', bogus.OPENAI_API_KEY], + ['OPENAI_BASE_URL', 'https://chatgpt.com/backend-api/codex'], + ['COPILOT_PROVIDER_API_KEY', bogus.COPILOT_PROVIDER_API_KEY], + ['COPILOT_PROVIDER_BASE_URL', WS_ECHO] ] as const) { const put = await user('PUT', `/v1/secrets/personal/${name}`, { value }) cleanup.push(() => user('DELETE', `/v1/secrets/personal/${name}`)) @@ -172,6 +202,7 @@ async function main(): Promise { throw new Error(`PUT secret ${name}: ${put.status} ${put.body?.error ?? ''}`) } } + const clones: Record = {} const create = async (label: string, repo: string) => { console.error(`[egress-probe] creating the ${label} workspace (${sub}-${label})`) const reply = await user('POST', '/v1/sandboxes', { workspaceId: `${sub}-${label}`, machineSize: 'standard', repo: { url: repo }, useEnvironment: false }) @@ -192,67 +223,111 @@ async function main(): Promise { clones[label] = clone return { id: id!, pairingCode: reply.body.connection.pairingCode as string, projectRoot: reply.body.connection.projectRoot as string } } - const clones: Record = {} const [publicBox, appBox] = await Promise.all([create('public', PUBLIC_REPO), create('app', APP_REPO)]) report.workspaces = { public: publicBox.id, app: appBox.id } + for (const [workspace, state] of Object.entries(clones)) { + check('create', workspace, 'github.com: the workspace repo clones (desktop create)', state === 'done', `clone ${state}`) + } - const anthropic: ProbeCase[] = [ + // Header values the runner reads from env (`$NAME`): the Bearer forms, and git's own header. + const prelude = [ + 'export PROBE_OPENAI_BEARER="Bearer $OPENAI_API_KEY" PROBE_COPILOT_BEARER="Bearer $COPILOT_PROVIDER_API_KEY" PROBE_GH_BEARER="Bearer $GH_TOKEN"', + 'PROBE_GIT_AUTH="$(git config --global --get-all http.https://github.com/.extraheader | tail -n 1)"; export PROBE_GIT_AUTH="${PROBE_GIT_AUTH#Authorization: }"' + ].join('; ') + const publicCases: ProbeCase[] = [ { - name: 'echo host (ANTHROPIC_BASE_URL): placeholder swapped, other auth untouched', - url: `${ECHO}/post`, + name: 'custom base URL (model key): placeholder swapped, other auth untouched', + url: `${ECHO}/anything`, method: 'POST', headers: { 'x-api-key': '$ANTHROPIC_API_KEY', authorization: 'Bearer probe-keep-bearer' }, - expect: { kind: 'swapped', headers: { 'x-api-key': bogusAnthropic, authorization: 'Bearer probe-keep-bearer' } } + expect: { kind: 'swapped', headers: { 'x-api-key': bogus.ANTHROPIC_API_KEY }, keep: ['authorization'] }, + repeat: 5, + latency: 'model_key' }, - { name: 'echo host: foreign key fails closed', url: `${ECHO}/post`, method: 'POST', headers: { 'x-api-key': 'sk-ant-probe-foreign' }, expect: { kind: 'denied' } }, + { name: 'custom base URL: foreign key fails closed', url: `${ECHO}/anything`, method: 'POST', headers: { 'x-api-key': 'sk-ant-probe-foreign' }, expect: { kind: 'denied' } }, + { name: 'custom base URL: chunked stream (SSE framing) through the swap', url: `${ECHO}/stream/5`, headers: { 'x-api-key': '$ANTHROPIC_API_KEY' }, stream: true, expect: { kind: 'streamed', minChunks: 1 }, latency: 'model_key_stream' }, { - name: 'chatgpt.com: placeholder reaches ChatGPT (relay)', - url: 'https://chatgpt.com/backend-api/codex/models', - headers: { authorization: '$PROBE_OPENAI_BEARER' }, - expect: { kind: 'upstream' } + // 1 MB: the public echo hosts refuse larger bodies themselves (httpbin 413 at 2 MB, postman-echo 500 at 1 MB). + name: 'custom base URL: 1 MB upload through the swap', + url: `${ECHO}/anything`, + method: 'POST', + uploadBytes: 1 << 20, + headers: { 'x-api-key': '$ANTHROPIC_API_KEY', 'content-type': 'application/octet-stream' }, + expect: { kind: 'swapped', headers: { 'x-api-key': bogus.ANTHROPIC_API_KEY } }, + latency: 'upload_1mb' }, + { name: 'chatgpt.com: placeholder reaches ChatGPT', url: 'https://chatgpt.com/backend-api/codex/models', headers: { authorization: '$PROBE_OPENAI_BEARER' }, expect: { kind: 'upstream' }, repeat: 5, latency: 'codex' }, { name: 'chatgpt.com: foreign bearer fails closed', url: 'https://chatgpt.com/backend-api/codex/models', headers: { authorization: 'Bearer probe-foreign' }, expect: { kind: 'denied' } }, + { name: 'WebSocket upgrade through the swap', url: `${WS_ECHO}/raw`, headers: { authorization: '$PROBE_COPILOT_BEARER' }, websocket: true, expect: { kind: 'upgraded' }, latency: 'websocket' }, + { name: 'WebSocket host: no credential fails closed', url: `${WS_ECHO}/raw`, websocket: true, expect: { kind: 'denied' } }, + { name: 'a placeholder sent to another host arrives unswapped', url: `${OTHER_ECHO}/get`, headers: { 'x-api-key': '$ANTHROPIC_API_KEY' }, expect: { kind: 'unswapped', headers: ['x-api-key'] } }, { name: 'api.github.com: no credential fails closed', url: 'https://api.github.com/rate_limit', expect: { kind: 'denied' } } ] const appCases: ProbeCase[] = [ - { name: 'api.github.com: workspace placeholder swapped (GitHub answers 200)', url: 'https://api.github.com/rate_limit', headers: { authorization: '$PROBE_GH_BEARER' }, expect: { kind: 'upstream' } }, + { name: 'api.github.com: workspace placeholder swapped (GitHub answers)', url: 'https://api.github.com/rate_limit', headers: { authorization: '$PROBE_GH_BEARER' }, expect: { kind: 'upstream' }, repeat: 5, latency: 'github_api' }, + { name: 'github.com: git smart-HTTP with the workspace header', url: `${APP_REPO}.git/info/refs?service=git-upload-pack`, headers: { authorization: '$PROBE_GIT_AUTH' }, expect: { kind: 'upstream' }, repeat: 5, latency: 'git_http' }, + { + name: 'github.com: LFS batch API with the workspace header', + url: `${APP_REPO}.git/info/lfs/objects/batch`, + method: 'POST', + headers: { authorization: '$PROBE_GIT_AUTH', accept: 'application/vnd.git-lfs+json', 'content-type': 'application/vnd.git-lfs+json' }, + expect: { kind: 'upstream' }, + latency: 'git_lfs' + }, { name: 'github.com: anonymous fails closed with an App token', url: `${APP_REPO}.git/info/refs?service=git-upload-pack`, expect: { kind: 'denied' } } ] - const digests = { ANTHROPIC_API_KEY: sha8(bogusAnthropic), OPENAI_API_KEY: sha8(bogusOpenAI) } - // The runner reads whole header values from env (`$NAME`), so the Bearer forms are built here. - const prelude = 'export PROBE_OPENAI_BEARER="Bearer $OPENAI_API_KEY" PROBE_GH_BEARER="Bearer $GH_TOKEN"; ' - console.error('[egress-probe] running the checks in both workspaces') - const runs = await Promise.all([ - inTerminal(publicBox.pairingCode, publicBox.projectRoot, `${prelude}${factsScript({ 'git.public-anonymous': `git ls-remote ${PUBLIC_REPO} HEAD` }, digests)}; ${runnerCommand(anthropic)}`), - inTerminal(appBox.pairingCode, appBox.projectRoot, `${prelude}${factsScript({ 'git.app-anonymous': `git -c http.extraheader= -c http.https://github.com/.extraheader= ls-remote ${APP_REPO} HEAD`, 'git.app-workspace': 'git ls-remote origin HEAD' }, {})}; ${runnerCommand(appCases)}`) - ]) - const results: Record[] = [] - const check = (workspace: string, name: string, ok: boolean, detail: string) => { - results.push({ workspace, check: name, ok }) - if (!ok) { - failures.push(`[${workspace}] ${name}: ${detail}`) + const digests = Object.fromEntries(Object.entries(bogus).map(([name, value]) => [name, sha8(value)])) + const latency: Record = {} + for (const mode of MODES) { + if (mode !== 'default') { + for (const box of [publicBox, appBox]) { + const set = await operator('POST', `/v1/internal/sandboxes/${box.id}/egress-mode`, { mode }) + if (set.status !== 200) { + throw new Error(`set egress mode ${mode} on ${box.id}: ${set.status} ${set.body?.error ?? ''} (is EGRESS_SWAP_MODE (A2) on dev?)`) + } + } + } + console.error(`[egress-probe] mode ${mode}: running the checks in both workspaces`) + const branch = `egress-probe-${runId}-${mode}` + const runs = await Promise.all([ + inTerminal(publicBox.pairingCode, publicBox.projectRoot, `${prelude}; ${factsScript({ 'git.public-anonymous': `git ls-remote ${PUBLIC_REPO} HEAD` }, digests)}; ${runnerCommand(publicCases)}`), + inTerminal( + appBox.pairingCode, + appBox.projectRoot, + `${prelude}; ${factsScript( + { + 'git.app-anonymous': `git -c http.extraheader= -c http.https://github.com/.extraheader= ls-remote ${APP_REPO} HEAD`, + 'git.app-fetch': 'git fetch -q origin', + 'git.app-push': `git push -q origin HEAD:refs/heads/${branch}`, + 'git.app-push-delete': `git push -q origin --delete ${branch}` + }, + {} + )}; ${runnerCommand(appCases)}` + ) + ]) + for (const [workspace, text, cases] of [['public', runs[0], publicCases], ['app', runs[1], appCases]] as const) { + const summaries = parseRunnerOutput(text, cases.length) + cases.forEach((probeCase, i) => { + const problems = judge(probeCase, summaries[i]!) + check(mode, workspace, probeCase.name, problems.length === 0, problems.join('; ')) + }) + latency[`${mode}/${workspace}`] = latencyByClass(cases, summaries) + } + const rc = (text: string, name: string) => new RegExp(`PRB ${name.replaceAll('.', '\\.')} rc=(\\d+)`).exec(text)?.[1] + check(mode, 'public', 'github.com: anonymous git ls-remote of a public repo works (#758)', rc(runs[0], 'git.public-anonymous') === '0', `rc=${rc(runs[0], 'git.public-anonymous')}`) + check(mode, 'app', 'github.com: anonymous git ls-remote is refused with an App token', !['0', undefined].includes(rc(runs[1], 'git.app-anonymous')), `rc=${rc(runs[1], 'git.app-anonymous')}`) + for (const step of ['fetch', 'push', 'push-delete']) { + check(mode, 'app', `github.com: git ${step} through the workspace's own header`, rc(runs[1], `git.app-${step}`) === '0', `rc=${rc(runs[1], `git.app-${step}`)}`) + } + for (const name of Object.keys(digests)) { + check(mode, 'public', `the VM holds a placeholder for ${name}`, new RegExp(`PRB env\\.${name} holds=placeholder`).test(runs[0]), 'the VM env holds the bogus value itself') } } - for (const [workspace, text, cases] of [['public', runs[0], anthropic], ['app', runs[1], appCases]] as const) { - const summaries = parseRunnerOutput(text, cases.length) - cases.forEach((probeCase, i) => { - const problems = judge(probeCase, summaries[i]!) - check(workspace, probeCase.name, problems.length === 0, problems.join('; ')) - }) - } - const rc = (text: string, name: string) => new RegExp(`PRB ${name.replaceAll('.', '\\.')} rc=(\\d+)`).exec(text)?.[1] - check('public', 'github.com: anonymous git ls-remote of a public repo works (#758)', rc(runs[0], 'git.public-anonymous') === '0', `rc=${rc(runs[0], 'git.public-anonymous')}`) - check('app', 'github.com: anonymous git ls-remote is refused with an App token', rc(runs[1], 'git.app-anonymous') !== '0' && rc(runs[1], 'git.app-anonymous') !== undefined, `rc=${rc(runs[1], 'git.app-anonymous')}`) - check('app', "github.com: the workspace's own git (placeholder) works", rc(runs[1], 'git.app-workspace') === '0', `rc=${rc(runs[1], 'git.app-workspace')}`) - for (const [workspace, state] of Object.entries(clones)) { - check(workspace, 'github.com: the workspace repo clones', state === 'done', `clone ${state}`) - } - for (const name of Object.keys(digests)) { - check('public', `the VM holds a placeholder for ${name}`, new RegExp(`PRB env\\.${name} holds=placeholder`).test(runs[0]), 'the VM env holds the bogus value itself') - } report.checks = results + report.latency = latency } catch (error) { failures.push(`probe error: ${error instanceof Error ? error.message : String(error)}`) + report.checks = results } finally { if (!keep) { for (const step of cleanup) { diff --git a/cloud/apps/sandbox-control/test/egress-probe-checks.test.ts b/cloud/apps/sandbox-control/test/egress-probe-checks.test.ts index 5701b156aa7..3182325b9c0 100644 --- a/cloud/apps/sandbox-control/test/egress-probe-checks.test.ts +++ b/cloud/apps/sandbox-control/test/egress-probe-checks.test.ts @@ -3,19 +3,20 @@ import { createServer } from 'node:http' import type { AddressInfo } from 'node:net' import { describe, expect, it } from 'vitest' import { EGRESS_DENIED_BODY } from '../src/network-policy' -import { judge, parseRunnerOutput, runnerCommand, sha8, type ProbeCase, type Summary } from '../scripts/egress-probe-checks' +import { judge, latencyByClass, parseRunnerOutput, runnerCommand, sha8, type ProbeCase, type Summary } from '../scripts/egress-probe-checks' -const answer = (summary: Partial): Summary => ({ status: 200, denied: false, edgeBlocked: false, ...summary }) +const answer = (summary: Partial): Summary => ({ status: 200, denied: false, edgeBlocked: false, hang: false, ms: [10], ...summary }) describe('egress probe judge', () => { - const swapped: ProbeCase = { name: 'echo', url: 'https://echo/post', expect: { kind: 'swapped', headers: { 'x-api-key': 'stand-in', authorization: 'Bearer keep' } } } + const swapped: ProbeCase = { name: 'echo', url: 'https://echo/post', expect: { kind: 'swapped', headers: { 'x-api-key': 'stand-in' }, keep: ['authorization'] } } + const sent = { 'x-api-key': sha8('orca-ph-placeholder'), authorization: sha8('Bearer keep') } it('accepts the provider seeing the swapped key and the other header unchanged', () => { - expect(judge(swapped, answer({ echoed: { 'x-api-key': sha8('stand-in'), authorization: sha8('Bearer keep') } }))).toEqual([]) + expect(judge(swapped, answer({ sent, echoed: { 'x-api-key': sha8('stand-in'), authorization: sha8('Bearer keep') } }))).toEqual([]) }) it('names a header that changed or never arrived, and anything leaked from the hop, never values', () => { - const problems = judge(swapped, answer({ echoed: { 'x-api-key': sha8('orca-ph-placeholder') }, leaked: ['vercel-sandbox-oidc-token'] })) + const problems = judge(swapped, answer({ sent, echoed: { 'x-api-key': sha8('orca-ph-placeholder') }, leaked: ['vercel-sandbox-oidc-token'] })) expect(problems).toEqual([ 'echo: the provider saw vercel-sandbox-oidc-token', 'echo: header x-api-key arrived with another value', @@ -40,6 +41,35 @@ describe('egress probe judge', () => { }) }) +describe('egress probe: hangs, streams, upgrades, leaks and latency', () => { + it('a hang fails every kind, even a fail-closed one', () => { + expect(judge({ name: 'x', url: 'u', expect: { kind: 'denied' } }, answer({ hang: true, status: 0 }))).toEqual(['x: the client hung waiting for the answer (undici framing hang?)']) + }) + + it('a placeholder on another host must arrive exactly as sent', () => { + const leak: ProbeCase = { name: 'leak', url: 'https://other/anything', expect: { kind: 'unswapped', headers: ['x-api-key'] } } + const sent = { 'x-api-key': sha8('orca-ph-x') } + expect(judge(leak, answer({ sent, echoed: sent }))).toEqual([]) + expect(judge(leak, answer({ sent, echoed: { 'x-api-key': sha8('the real key') } }))).toEqual(['leak: header x-api-key was changed on the way']) + }) + + it('streams need chunks and a 200; upgrades need a 101', () => { + expect(judge({ name: 's', url: 'u', expect: { kind: 'streamed', minChunks: 2 } }, answer({ chunks: 3 }))).toEqual([]) + expect(judge({ name: 's', url: 'u', expect: { kind: 'streamed', minChunks: 2 } }, answer({ chunks: 1 }))).toHaveLength(1) + expect(judge({ name: 'w', url: 'u', expect: { kind: 'upgraded' } }, answer({ status: 101 }))).toEqual([]) + expect(judge({ name: 'w', url: 'u', expect: { kind: 'upgraded' } }, answer({ status: 403, denied: true }))).toEqual(['w: expected a WebSocket upgrade (101), got 403 (denied)']) + }) + + it('latency per class: median and p90 over every sample, hangs left out', () => { + const cases: ProbeCase[] = [ + { name: 'a', url: 'u', latency: 'git', expect: { kind: 'upstream' } }, + { name: 'b', url: 'u', latency: 'git', expect: { kind: 'upstream' } }, + { name: 'c', url: 'u', latency: 'git', expect: { kind: 'upstream' } } + ] + expect(latencyByClass(cases, [answer({ ms: [10, 30, 20] }), answer({ ms: [40, 50] }), answer({ hang: true, ms: [30000] })])).toEqual({ git: { n: 5, medianMs: 30, p90Ms: 50 } }) + }) +}) + describe('the in-VM runner', () => { it('reports status, the deny body and echoed header digests from env-held values', async () => { // A local stand-in for postman-echo.com: echoes request headers under `headers`. @@ -48,13 +78,25 @@ describe('the in-VM runner', () => { res.writeHead(403, { 'content-type': 'text/plain' }).end(EGRESS_DENIED_BODY) return } - res.writeHead(200, { 'content-type': 'application/json' }).end(JSON.stringify({ headers: { ...req.headers, 'vercel-forwarded-host': 'x' } })) + if (req.url === '/stream') { + res.writeHead(200, { 'content-type': 'text/event-stream' }) + res.write('data: 1\n\n') + setTimeout(() => res.end('data: 2\n\n'), 50) + return + } + let bytes = 0 + req.on('data', (chunk: Buffer) => (bytes += chunk.length)) + req.on('end', () => + res.writeHead(200, { 'content-type': 'application/json' }).end(JSON.stringify({ bytes, headers: { ...req.headers, 'Vercel-Forwarded-Host': 'x' } })) + ) }) await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) const base = `http://127.0.0.1:${(server.address() as AddressInfo).port}` const cases = [ { name: 'echo', url: `${base}/post`, method: 'POST' as const, headers: { 'x-api-key': '$PH', authorization: 'Bearer keep' } }, - { name: 'denied', url: `${base}/denied` } + { name: 'denied', url: `${base}/denied` }, + { name: 'stream', url: `${base}/stream`, stream: true }, + { name: 'upload', url: `${base}/post`, method: 'POST' as const, uploadBytes: 1 << 20, repeat: 3 } ] const stdout = await new Promise((resolve, reject) => execFile('bash', ['-c', runnerCommand(cases)], { env: { ...process.env, PH: 'secret-ph' } }, (error, out) => (error ? reject(error) : resolve(out))) @@ -62,8 +104,18 @@ describe('the in-VM runner', () => { server.close() expect(stdout).toContain('PRB-END') expect(stdout).not.toContain('secret-ph') - const [echo, denied] = parseRunnerOutput(stdout, 2) - expect(echo).toEqual({ status: 200, denied: false, edgeBlocked: false, echoed: { 'x-api-key': sha8('secret-ph'), authorization: sha8('Bearer keep') }, leaked: ['vercel-forwarded-host'] }) + const [echo, denied, stream, upload] = parseRunnerOutput(stdout, 4) + expect(echo).toMatchObject({ + status: 200, + hang: false, + sent: { 'x-api-key': sha8('secret-ph'), authorization: sha8('Bearer keep') }, + echoed: { 'x-api-key': sha8('secret-ph'), authorization: sha8('Bearer keep') }, + leaked: ['vercel-forwarded-host'] + }) expect(denied).toMatchObject({ status: 403, denied: true }) + expect(stream).toMatchObject({ status: 200, hang: false }) + expect(stream!.chunks).toBeGreaterThanOrEqual(2) + expect(upload).toMatchObject({ status: 200 }) + expect(upload!.ms).toHaveLength(3) }) })