diff --git a/.github/workflows/cloud-egress-probe.yml b/.github/workflows/cloud-egress-probe.yml index 9ee8bb08be3..1b5d983ffea 100644 --- a/.github/workflows/cloud-egress-probe.yml +++ b/.github/workflows/cloud-egress-probe.yml @@ -1,20 +1,10 @@ name: Cloud Egress Probe -# Release check for sandbox egress, against DEV: placeholders are forwarded for the right hosts with -# other auth headers untouched, credential hosts fail closed, anonymous git works without an App -# token (#758), and the egress proxy swaps a bogus key in for its placeholder. Bogus tokens and -# throwaway echo sandboxes only. Details: cloud/apps/sandbox-control/scripts/README.md +# Release check for sandbox egress, against DEV, through Orca's own paths (dev workspaces driven +# over the shipped `orca` CLI's pairing code). Bogus tokens only; postman-echo.com is the echo host. +# Details and the local run: cloud/apps/sandbox-control/scripts/README.md on: workflow_dispatch: - inputs: - layers: - description: firewall (Vercel only) or all (also the dev control plane's swap) - required: false - default: all - type: choice - options: [all, firewall] - schedule: - - cron: '17 9 * * *' permissions: contents: read @@ -30,18 +20,17 @@ defaults: jobs: probe: runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 20 env: - VERCEL_TOKEN: ${{ secrets.ORCA_DEV_VERCEL_TOKEN }} DEV_TEST_JWK: ${{ secrets.ORCA_DEV_TEST_JWK }} - LAYERS: ${{ inputs.layers || 'all' }} + DEV_OPERATOR_TOKEN: ${{ secrets.ORCA_DEV_OPERATOR_TOKEN }} steps: - name: Check the dev credentials are configured id: creds working-directory: . run: | - if [ -z "$VERCEL_TOKEN" ] || { [ "$LAYERS" = all ] && [ -z "$DEV_TEST_JWK" ]; }; then - echo "::notice::ORCA_DEV_VERCEL_TOKEN / ORCA_DEV_TEST_JWK are not set: skipping the egress probe" + if [ -z "$DEV_TEST_JWK" ] || [ -z "$DEV_OPERATOR_TOKEN" ]; then + echo "::warning::ORCA_DEV_TEST_JWK / ORCA_DEV_OPERATOR_TOKEN are not set: the egress probe did not run" echo "skip=1" >> "$GITHUB_OUTPUT" fi @@ -62,17 +51,27 @@ jobs: - if: steps.creds.outputs.skip != '1' run: pnpm install --frozen-lockfile --filter @orca-cloud/sandbox-control... + # The shipped CLI from the latest release's Linux package: the same pairing path the desktop uses. + - name: Install the orca CLI + if: steps.creds.outputs.skip != '1' + working-directory: . + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release download --repo stablyai/orca --pattern 'orca-ide_*_amd64.deb' --dir "$RUNNER_TEMP/orca" + dpkg-deb -x "$RUNNER_TEMP"/orca/orca-ide_*_amd64.deb "$RUNNER_TEMP/orca/x" + cli="$(find "$RUNNER_TEMP/orca/x" -path '*/bin/orca' -type f | head -n 1)" + test -x "$cli" + echo "ORCA_PROBE_CLI=$cli" >> "$GITHUB_ENV" + - name: Probe dev egress if: steps.creds.outputs.skip != '1' run: | umask 077 - if [ "$LAYERS" = all ]; then - printf '%s' "$DEV_TEST_JWK" > "$RUNNER_TEMP/dev-jwk.json" - export ORCA_PROBE_DEV_JWK_FILE="$RUNNER_TEMP/dev-jwk.json" - fi - pnpm --filter @orca-cloud/sandbox-control exec tsx scripts/egress-probe.ts \ - $([ "$LAYERS" = firewall ] && echo --firewall-only) > "$RUNNER_TEMP/egress-probe.json" - rm -f "$RUNNER_TEMP/dev-jwk.json" + printf '%s' "$DEV_TEST_JWK" > "$RUNNER_TEMP/dev-jwk.json" + printf '%s' "$DEV_OPERATOR_TOKEN" > "$RUNNER_TEMP/dev-operator" + ORCA_PROBE_DEV_JWK_FILE="$RUNNER_TEMP/dev-jwk.json" ORCA_PROBE_OPERATOR_TOKEN_FILE="$RUNNER_TEMP/dev-operator" \ + pnpm --filter @orca-cloud/sandbox-control exec tsx scripts/egress-probe.ts > "$RUNNER_TEMP/egress-probe.json" - uses: actions/upload-artifact@v4 if: always() && steps.creds.outputs.skip != '1' diff --git a/cloud/apps/sandbox-control/image/boot-twice/README.md b/cloud/apps/sandbox-control/image/boot-twice/README.md index 89c5f54a3fd..d579493a65e 100644 --- a/cloud/apps/sandbox-control/image/boot-twice/README.md +++ b/cloud/apps/sandbox-control/image/boot-twice/README.md @@ -59,12 +59,20 @@ The options: What this check does not cover: - The runtime-bundle refresh: no bundle is published here. -- The Vercel firewall: that is the release probe, `cloud/apps/acceptance/src/egress-probe.ts`. +- The Vercel firewall and egress proxy: that is the release probe, `scripts/egress-probe.ts`. - Vercel's own VM behaviour, such as snapshot restore timing. ## CI -`.github/workflows/cloud-vm-boot-twice.yml` runs on pull requests that touch `image/**` or the -boot-step sources. It caches the provisioned layer in the GitHub Actions cache. On -nwparker/orca-private, where Actions is off, run the command above on omarchy before merging an -image change. +On nwparker/orca-private, where Actions is off, the premerge gate +(`config/scripts/cloud-integration/premerge-check.sh`) runs +`config/scripts/cloud-integration/boot-twice-check.sh` whenever a branch touches +`cloud/apps/sandbox-control/image/**`, which includes every VM script. The step: + +- takes its own omarchy slot, waiting at most 5 minutes for one; +- stops after 10 minutes; +- prints each failure; +- **fails open with a loud warning when Docker is unavailable**. + +`.github/workflows/cloud-vm-boot-twice.yml` runs the same check on pull requests once Actions is +on, with the provisioned layer cached in the GitHub Actions cache. diff --git a/cloud/apps/sandbox-control/scripts/README.md b/cloud/apps/sandbox-control/scripts/README.md index f386b146b8c..b05ebfbc173 100644 --- a/cloud/apps/sandbox-control/scripts/README.md +++ b/cloud/apps/sandbox-control/scripts/README.md @@ -1,51 +1,56 @@ # Egress release probe -`egress-probe.ts` checks sandbox egress against **dev** before a release. It uses bogus tokens, -and an echo server running in a throwaway sandbox of our own stands in for the egress proxy and -for a model provider. No real credential is sent anywhere. +`egress-probe.ts` checks sandbox egress against **dev** before a release. It goes through Orca's own +paths: real dev workspaces created with a dev-JWK user token, with commands run in a workspace +terminal over the shipped `orca` CLI's pairing code (the desktop's E2EE path to orcad). Every +credential is bogus, and postman-echo.com plays the model provider. -It tests two layers. +Setup for a throwaway identity, all of it removed afterwards: -**1. Firewall (Vercel only).** A throwaway VM gets the exact policy that `egressPolicy()` compiles -for a workspace with a Codex account, an Anthropic key and a member's GitHub identity. The forward -rules point at the echo sandbox. The policy is applied twice: once with the repo's App token and -once without (the GH-ANON-403 shape, #758). From the VM: +- the dev GitHub App's installation, bound for the test repo (operator route); +- a stored GitHub identity, so member placeholders exist (the dev `member-github` route); +- bogus personal secrets: + - `ANTHROPIC_API_KEY`, with `ANTHROPIC_BASE_URL` set to the echo host; + - `OPENAI_API_KEY`, with `OPENAI_BASE_URL` set to `chatgpt.com/backend-api/codex`, so its egress + takes the ChatGPT relay. -| Host | Placeholder | Foreign or no credential | -|---|---|---| -| chatgpt.com | forwarded for chatgpt.com; `chatgpt-account-id` and `x-api-key` unchanged | firewall 403 `orca-egress-denied` | -| api.anthropic.com | forwarded; the other `authorization` header unchanged | firewall 403 | -| api.github.com | forwarded | firewall 403 | -| github.com, App token | forwarded | anonymous: firewall 403; `git ls-remote` refused | -| github.com, member only (#758) | member placeholder forwarded | anonymous `git ls-remote` of a public repo works | +The probe creates two workspaces through the desktop's create path (clone after ready): -**2. Swap (the dev control plane).** This layer creates a real dev workspace for a throwaway -identity, with a bogus `ANTHROPIC_API_KEY` and `ANTHROPIC_BASE_URL` set to the echo sandbox. The VM -must hold only the placeholder. The echo (as the provider) must see the bogus key where the -placeholder was, with the other `authorization` header unchanged and no Vercel forwarding headers -or OIDC token. A foreign key sent to that host must get the firewall 403. +- **public**: a public repo the App isn't installed on. Member placeholder, no App token (#758). +- **app**: the App-installed test repo. App token, github.com fail-closed. -The probe takes about 1–2 minutes. It deletes everything it created (`--keep` leaves it in place), -prints one JSON report, and exits 1 on any failed expectation. +| Host | What must happen | +|---|---| +| echo host (custom base URL) | The provider sees the bogus key where the VM sent the placeholder. The other `authorization` header is unchanged, and no `vercel-forwarded-*` header or OIDC token arrives. A foreign key gets the firewall 403 `orca-egress-denied`. | +| chatgpt.com | The placeholder reaches ChatGPT: any provider answer counts, but not the firewall 403 and not Cloudflare's edge page. A foreign bearer gets the firewall 403. | +| api.github.com | No credential gets the firewall 403. With the workspace placeholder, GitHub answers 200. | +| github.com | Anonymous `git ls-remote` works on **public** (#758) and is refused on **app**. The app repo clones, and the workspace's own git works. | +| VM env | `ANTHROPIC_API_KEY` and `OPENAI_API_KEY` hold placeholders. This is checked by sha256, so the bogus values never cross the terminal. | + +The probe takes about 1–3 minutes, deletes what it created (`--keep` leaves it), prints one JSON +report and exits 1 on any failed expectation. It refuses any control plane other than dev. ## Run it locally +You need the Orca app installed (for its `orca` CLI) and `gcloud` access to the dev secrets. + ```sh cd cloud pnpm install --filter @orca-cloud/sandbox-control... -# The dev team's Vercel token, from a file (never pasted on the command line): -export ORCA_PROBE_VERCEL_TOKEN_FILE=/path/to/dev-vercel-token -# For the swap layer: the dev test signing key, escrowed in Secret Manager. +K=$(mktemp -d) # 0700; delete it afterwards gcloud --account secrets versions access latest --project onorca-cloud-staging \ - --secret orca-sandbox-control-dev-test-jwk > "$TMPDIR/dev-jwk.json" -export ORCA_PROBE_DEV_JWK_FILE="$TMPDIR/dev-jwk.json" -pnpm --filter @orca-cloud/sandbox-control exec tsx scripts/egress-probe.ts # --firewall-only skips layer 2 + --secret orca-sandbox-control-dev-test-jwk > "$K/jwk.json" +gcloud --account secrets versions access latest --project onorca-cloud-staging \ + --secret orca-sandbox-control-dev-operator-token > "$K/operator" +ORCA_PROBE_DEV_JWK_FILE="$K/jwk.json" ORCA_PROBE_OPERATOR_TOKEN_FILE="$K/operator" \ + pnpm --filter @orca-cloud/sandbox-control exec tsx scripts/egress-probe.ts +rm -rf "$K" ``` -The script refuses any control plane other than dev. +`ORCA_PROBE_CLI` points the probe at another `orca` CLI. The default is the installed app's. ## CI -`.github/workflows/cloud-egress-probe.yml` runs on manual dispatch and nightly. It needs the -`ORCA_DEV_VERCEL_TOKEN` and `ORCA_DEV_TEST_JWK` repository secrets, and skips with a notice when -they are missing. +`.github/workflows/cloud-egress-probe.yml` runs on manual dispatch. It installs the CLI from the +latest release's Linux package. It needs the `ORCA_DEV_TEST_JWK` and `ORCA_DEV_OPERATOR_TOKEN` +repository secrets, and warns and skips when they are missing. None are configured today. diff --git a/cloud/apps/sandbox-control/scripts/egress-probe-checks.ts b/cloud/apps/sandbox-control/scripts/egress-probe-checks.ts index 8e85b205eeb..e917d92c7b2 100644 --- a/cloud/apps/sandbox-control/scripts/egress-probe-checks.ts +++ b/cloud/apps/sandbox-control/scripts/egress-probe-checks.ts @@ -1,115 +1,119 @@ -// Pure parts of the egress release probe (egress-probe.ts): the echo server it runs, the requests -// it sends from the probe VM, and what each answer must look like. +// Pure parts of the egress release probe (egress-probe.ts): the runner it starts in a workspace +// terminal, the requests that runner sends, and what each answer must look like. +import { createHash } from 'node:crypto' import { EGRESS_DENIED_BODY } from '../src/network-policy' /** - * Runs in the echo sandbox: answers every request with its method, path and headers as JSON. Vercel's - * per-sandbox OIDC token (on forwarded requests) is reported as present, never echoed. + * One request from inside the workspace. A header value `$NAME` is read from the VM's env (orcad's + * environment, as agents get it), so placeholders never cross the terminal. */ -export const ECHO_SERVER = ` -require('node:http').createServer((req, res) => { - req.resume() - req.on('end', () => { - const headers = {} - for (const [name, value] of Object.entries(req.headers)) headers[name] = /oidc/i.test(name) ? '[present]' : value - const body = JSON.stringify({ probeEcho: true, method: req.method, url: req.url, headers }) - res.writeHead(200, { 'content-type': 'application/json', 'content-length': Buffer.byteLength(body) }) - res.end(body) - }) -}).listen(Number(process.argv[1] || 8080)) -` - -/** One request from the probe VM. `headers` values may name env vars (`$NAME`), expanded in the VM. */ export type ProbeRequest = { name: string; url: string; method?: 'GET' | 'POST'; headers?: Record } -/** What the answer must be: echoed through the forward rule, refused by the firewall, or passed upstream. */ +/** What the answer must be. */ export type Expectation = - /** Vercel's forward rule sent it on unchanged (the probe's echo stands in for the egress proxy). */ - | { kind: 'forwarded'; host: string; headers: Record } - /** The egress proxy swapped the placeholder and sent it to the host (the echo is that host). */ + /** The echo host answered as the provider: these headers (by sha256) arrived, and nothing from Vercel's hop. */ | { kind: 'swapped'; headers: Record } + /** The firewall's own 403 (a credential host fails closed). */ | { kind: 'denied' } - | { kind: 'upstream'; notDenied: true } + /** The request reached the provider itself: not the firewall's 403, not an edge block page. */ + | { kind: 'upstream' } export type ProbeCase = ProbeRequest & { expect: Expectation } -export type Answer = { status: number; body: string } - -type Echo = { probeEcho: true; method: string; url: string; headers: Record } - -function parseEcho(body: string): Echo | null { - try { - const value = JSON.parse(body) as Partial - return value.probeEcho === true && typeof value.headers === 'object' ? (value as Echo) : null - } catch { - return null - } +/** What the runner reports per request: never a header value, only sha256 prefixes of the echoed ones. */ +export type Summary = { + status: number + /** Body is the firewall's deny body. */ + denied: boolean + /** An HTML 403 from Cloudflare: the provider's edge refused the proxy. */ + edgeBlocked: boolean + /** The echo host's answer: sha256 prefix of each echoed header, and Vercel hop headers seen. */ + echoed?: Record + leaked?: string[] } -/** Problems with one answer; empty means it is what the case expects. Never quotes header values. */ -export function judge(probe: ProbeCase, answer: Answer): string[] { +export const sha8 = (value: string): string => createHash('sha256').update(value).digest('hex').slice(0, 8) + +/** + * Runs in the workspace (node, from a terminal): fetches each request and prints one short line per + * fact, `PRB =`, ending with `PRB-END`. Short lines survive the terminal intact. + * The echo host (postman-echo.com) answers JSON with the request's headers under `headers`. + */ +export const VM_RUNNER = ` +const { createHash } = require('node:crypto') +const cases = JSON.parse(Buffer.from(process.argv[1], 'base64').toString()) +const sha8 = (v) => createHash('sha256').update(String(v)).digest('hex').slice(0, 8) +const out = (i, k, v) => console.log('PRB ' + i + ' ' + k + '=' + v) +;(async () => { + for (const [i, c] of cases.entries()) { + const headers = {} + for (const [k, v] of Object.entries(c.headers || {})) headers[k] = v.startsWith('$') ? process.env[v.slice(1)] || '' : v + try { + const r = await fetch(c.url, { method: c.method || 'GET', headers, body: c.method === 'POST' ? '{}' : undefined, redirect: 'manual', signal: AbortSignal.timeout(25000) }) + const body = await r.text() + out(i, 's', r.status) + out(i, 'd', body.trim() === ${JSON.stringify(EGRESS_DENIED_BODY)} ? 1 : 0) + out(i, 'cf', r.status === 403 && /text\\/html/.test(r.headers.get('content-type') || '') && /cloudflare/i.test(r.headers.get('server') || '') ? 1 : 0) + let echo = null + try { echo = JSON.parse(body).headers } catch {} + if (echo && typeof echo === 'object') { + for (const [k, v] of Object.entries(echo)) { + if (/^vercel-forwarded-|oidc/i.test(k)) out(i, 'leak', k) + else if (k in (c.headers || {})) out(i, 'h.' + k, sha8(v)) + } + } + } catch (e) { + out(i, 's', 0) + } + } + console.log('PRB-END') +})() +` + +export function parseRunnerOutput(text: string, count: number): Summary[] { + const summaries: Summary[] = Array.from({ length: count }, () => ({ status: 0, denied: false, edgeBlocked: false })) + for (const match of text.matchAll(/PRB (\d+) ([a-z.-]+)=(\S+)/g)) { + const summary = summaries[Number(match[1])] + const [key, value] = [match[2]!, match[3]!] + if (!summary) { + continue + } + if (key === 's') summary.status = Number(value) + else if (key === 'd') summary.denied = value === '1' + else if (key === 'cf') summary.edgeBlocked = value === '1' + else if (key === 'leak') summary.leaked = [...(summary.leaked ?? []), value] + else if (key.startsWith('h.')) summary.echoed = { ...summary.echoed, [key.slice(2)]: value } + } + return summaries +} + +/** Problems with one answer; empty when it is what the case expects. Never quotes a header value. */ +export function judge(probe: ProbeCase, answer: Summary): string[] { const { expect } = probe - const denied = answer.status === 403 && answer.body.trim() === EGRESS_DENIED_BODY + const denied = answer.status === 403 && answer.denied if (expect.kind === 'denied') { return denied ? [] : [`${probe.name}: expected the firewall's 403 ${EGRESS_DENIED_BODY}, got ${answer.status}`] } if (expect.kind === 'upstream') { - return denied || answer.status === 0 ? [`${probe.name}: expected the request to reach the host, got ${answer.status}${denied ? ' (denied)' : ''}`] : [] + if (denied) return [`${probe.name}: refused by the firewall`] + if (answer.edgeBlocked) return [`${probe.name}: the provider's edge refused the egress proxy (Cloudflare 403 page)`] + return answer.status === 0 ? [`${probe.name}: no answer`] : [] } - const echo = parseEcho(answer.body) - if (!echo) { - return [`${probe.name}: expected the forward target's echo, got ${answer.status}${denied ? ' (denied)' : ''}`] - } - const problems: string[] = [] - if (expect.kind === 'swapped') { - // Nothing from Vercel's forward hop (routing headers, the sandbox's OIDC token) reaches the provider. - const leaked = Object.keys(echo.headers).filter((name) => name.startsWith('vercel-forwarded-') || /oidc/i.test(name)) - if (leaked.length > 0) { - problems.push(`${probe.name}: the provider saw ${leaked.join(', ')}`) - } - } else if (echo.headers['vercel-forwarded-host'] !== expect.host) { - problems.push(`${probe.name}: forwarded for ${JSON.stringify(echo.headers['vercel-forwarded-host'])}, expected ${expect.host}`) - } - if (expect.kind === 'forwarded' && echo.headers['vercel-sandbox-oidc-token'] !== '[present]') { - problems.push(`${probe.name}: forwarded without the sandbox OIDC token`) + if (answer.status !== 200 || !answer.echoed) { + return [`${probe.name}: expected the echo host's answer, got ${answer.status}${denied ? ' (denied)' : ''}`] } + const problems = (answer.leaked ?? []).map((name) => `${probe.name}: the provider saw ${name}`) for (const [name, value] of Object.entries(expect.headers)) { - if (echo.headers[name] !== value) { - problems.push(`${probe.name}: header ${name} was ${echo.headers[name] === undefined ? 'dropped' : 'changed'} on the way`) + if (answer.echoed[name] !== sha8(value)) { + problems.push(`${probe.name}: header ${name} ${answer.echoed[name] ? 'arrived with another value' : 'did not arrive'}`) } } return problems } -/** - * The probe VM's script: one `namestatusbase64(body)` line per request. Header values - * starting with `$` are read from the command's env, so placeholders never sit in argv. - */ -export function probeScript(cases: readonly ProbeRequest[]): string { - const lines = ['set -u', 'out=$(mktemp)'] - for (const probe of cases) { - const headers = Object.entries(probe.headers ?? {}).map(([name, value]) => - value.startsWith('$') ? `-H "${name}: \${${value.slice(1)}}"` : `-H ${shellQuote(`${name}: ${value}`)}` - ) - lines.push( - `code=$(curl -s -m 20 -o "$out" -w '%{http_code}' -X ${probe.method ?? 'GET'} ${headers.join(' ')} ${shellQuote(probe.url)} || true)`, - `printf '%s\\t%s\\t%s\\n' ${shellQuote(probe.name)} "\${code:-0}" "$(head -c 20000 "$out" | base64 -w0)"` - ) - } - return lines.join('\n') -} - -export function parseProbeOutput(stdout: string): Map { - const answers = new Map() - for (const line of stdout.split('\n')) { - const [name, status, body] = line.split('\t') - if (name && status !== undefined && body !== undefined) { - answers.set(name, { status: Number(status) || 0, body: Buffer.from(body, 'base64').toString('utf8') }) - } - } - return answers -} - -function shellQuote(value: string): string { - return `'${value.replaceAll("'", `'\\''`)}'` +/** The terminal line that runs the cases: the runner and its input both base64, one shell word each. */ +export function runnerCommand(cases: readonly ProbeRequest[]): string { + const runner = Buffer.from(VM_RUNNER).toString('base64') + const input = Buffer.from(JSON.stringify(cases.map(({ name: _name, ...request }) => request))).toString('base64') + return `node -e "$(echo ${runner} | base64 -d)" ${input}` } diff --git a/cloud/apps/sandbox-control/scripts/egress-probe.ts b/cloud/apps/sandbox-control/scripts/egress-probe.ts index d17b0e671e0..0f3e61d3b57 100644 --- a/cloud/apps/sandbox-control/scripts/egress-probe.ts +++ b/cloud/apps/sandbox-control/scripts/egress-probe.ts @@ -1,259 +1,141 @@ -// Release probe for sandbox egress, against DEV only. Bogus tokens and our own echo server; no real -// credential is sent anywhere. -// pnpm --filter @orca-cloud/sandbox-control exec tsx scripts/egress-probe.ts [--firewall-only] [--keep] +// Release probe for sandbox egress, against DEV only, through Orca's own paths: real dev workspaces +// created with a dev-JWK user token, driven from a terminal over the `orca` CLI's pairing code (the +// desktop's E2EE path). Bogus tokens only; postman-echo.com plays the model provider. +// pnpm --filter @orca-cloud/sandbox-control exec tsx scripts/egress-probe.ts [--keep] // -// 1. Firewall: a throwaway VM gets the policy egressPolicy() compiles for a workspace with a Codex -// account, an Anthropic key, a member's GitHub identity and (then without) the repo's App token. -// Its forward rules point at an echo sandbox instead of the egress proxy. From that VM: each -// placeholder must be forwarded for the right host with every other auth header unchanged; -// foreign or missing credentials on a credential host must get the firewall's own 403 -// (fail-closed); anonymous git to a public repo must work when there is no App token (#758). -// Hosts: chatgpt.com, api.anthropic.com, github.com, api.github.com. -// 2. Swap (unless --firewall-only): a real dev workspace with a bogus ANTHROPIC_API_KEY whose -// ANTHROPIC_BASE_URL is the echo sandbox. The echo (as the "provider") must see the bogus key in -// place of the placeholder, the other auth header unchanged, and nothing from Vercel's hop. -// An unclaimed request to that host must be refused. +// A throwaway identity gets: the dev GitHub App installation bound for one test repo (operator), a +// stored GitHub identity (the dev member-github route, so member placeholders exist), and bogus +// personal secrets ANTHROPIC_API_KEY (base URL: the echo host) and OPENAI_API_KEY (base URL: +// chatgpt.com, so its egress takes the ChatGPT relay). Two workspaces: +// public a public repo the App is not installed on: member placeholder, no App token (#758) +// app the App-installed test repo: App token, github.com fail-closed +// Checks (every one names the host it is about): +// - api.anthropic-compatible echo host: the provider sees the bogus key where the VM sent the +// placeholder, the other auth header unchanged, nothing from Vercel's hop; a foreign key 403s. +// - chatgpt.com: the placeholder reaches ChatGPT (any provider answer; not the firewall's 403 and +// not Cloudflare's edge page); a foreign bearer 403s. +// - api.github.com: no credential 403s; with the workspace placeholder GitHub answers 200. +// - github.com: anonymous `git ls-remote` works on `public` (#758) and 403s on `app`; the +// workspace's own git (placeholder extraheader) works on `app`. +// - the VM holds placeholders, never the bogus values. // -// Inputs (read from files or env, never printed): -// VERCEL_TOKEN or ORCA_PROBE_VERCEL_TOKEN_FILE Vercel token for the dev team -// ORCA_PROBE_DEV_JWK_FILE dev test signing key (swap layer only) -// ORCA_PROBE_CONTROL_URL default: the dev sandbox-control Worker -// ORCA_PROBE_VERCEL_TEAM_ID / _PROJECT_ID default: Orca's dev team and project -// Prints one JSON report and exits 1 on any failed expectation. See scripts/README.md. +// Inputs (files, never printed): +// ORCA_PROBE_DEV_JWK_FILE dev test signing key (SM: orca-sandbox-control-dev-test-jwk) +// ORCA_PROBE_OPERATOR_TOKEN_FILE dev operator token (SM: orca-sandbox-control-dev-operator-token) +// ORCA_PROBE_CONTROL_URL default: the dev sandbox-control Worker (anything else is refused) +// ORCA_PROBE_CLI default: the installed Orca app's `orca` CLI +// Prints one JSON report; exits 1 on any failed expectation. See scripts/README.md. +import { execFile } from 'node:child_process' import { randomBytes } from 'node:crypto' import { readFileSync } from 'node:fs' import { SignJWT, importJWK, type JWK } from 'jose' -import { accountProvider } from '../src/account-providers' -import { memberPlaceholder } from '../src/github-user-tokens' -import { egressPolicy, githubPlaceholderToken, type LifecycleDeps } from '../src/lifecycle' -import { gitBasicAuthorization } from '../src/network-policy' -import { secretPlaceholder } from '../src/secret-egress' -import type { SandboxRow } from '../src/store' -import { createVercelSandboxApi } from '../src/vercel-sandbox-api' -import { ECHO_SERVER, judge, parseProbeOutput, probeScript, type ProbeCase } from './egress-probe-checks' +import { judge, parseRunnerOutput, runnerCommand, sha8, type ProbeCase } from './egress-probe-checks' const CONTROL_URL = process.env.ORCA_PROBE_CONTROL_URL ?? 'https://orca-sandbox-control-dev.lovecast.workers.dev' -const TEAM_ID = process.env.ORCA_PROBE_VERCEL_TEAM_ID ?? 'team_c5IO9DfEYAxKEUgMxrbEtO0a' -const PROJECT_ID = process.env.ORCA_PROBE_VERCEL_PROJECT_ID ?? 'prj_SkVtv6zJurF9dUHpfTRy4v9avwPq' -const IMAGE = 'vercel/sandbox/node:24' -const REGION = 'iad1' -const ECHO_PORT = 8080 -/** A public repository: anonymous git must reach it when the workspace has no App token. */ -const PUBLIC_REPO = 'https://github.com/octocat/Hello-World.git' +const ORCA_CLI = process.env.ORCA_PROBE_CLI ?? '/Applications/Orca.app/Contents/Resources/bin/orca' +/** The dev GitHub App is installed on this repo. */ +const APP_REPO = process.env.ORCA_PROBE_APP_REPO ?? 'https://github.com/nwparker/orca-sandbox-push-test' +/** Public, and the dev App is not installed on its owner. */ +const PUBLIC_REPO = 'https://github.com/octocat/Hello-World' +const ECHO = 'https://postman-echo.com' -if (!/(-dev\.lovecast\.workers\.dev|127\.0\.0\.1|localhost)(\/|$)/.test(CONTROL_URL)) { +if (!/^https:\/\/[a-z0-9-]+-dev\.lovecast\.workers\.dev$|^http:\/\/(127\.0\.0\.1|localhost)(:\d+)?$/.test(CONTROL_URL)) { throw new Error(`refusing a non-dev control plane: ${CONTROL_URL}`) } -const args = process.argv.slice(2) -const firewallOnly = args.includes('--firewall-only') -const keep = args.includes('--keep') +const keep = process.argv.includes('--keep') const runId = randomBytes(4).toString('hex') - -function secretFrom(envName: string, fileEnv: string): string | null { - if (process.env[envName]) { - return process.env[envName]!.trim() - } - const file = process.env[fileEnv] - return file ? readFileSync(file, 'utf8').trim() : null -} - -const vercelToken = secretFrom('VERCEL_TOKEN', 'ORCA_PROBE_VERCEL_TOKEN_FILE') -if (!vercelToken) { - throw new Error('set VERCEL_TOKEN or ORCA_PROBE_VERCEL_TOKEN_FILE (dev team token)') -} -const vercel = createVercelSandboxApi({ token: vercelToken, teamId: TEAM_ID, projectId: PROJECT_ID }) - -type Box = { name: string; sessionId: string } - -async function bootBox(role: string): Promise { - const name = `orca-probe-${runId}-${role}` - const state = await vercel.createBuildSandbox({ name, image: IMAGE, vcpus: 2, region: REGION, timeoutMs: 20 * 60_000, tags: { orcaEgressProbe: runId } }) - console.error(`[egress-probe] ${role} sandbox ${name}`) - return { name, sessionId: state.session.id } -} - -async function run(box: Box, script: string, env: Record = {}): Promise<{ exitCode: number; stdout: string; stderr: string }> { - return vercel.runCommand(box.sessionId, { command: 'bash', args: ['-c', script], env, timeoutMs: 120_000 }) -} - -/** The echo server on its public route: the probe's stand-in for the egress proxy and for a model provider. */ -async function startEcho(box: Box): Promise { - const [route] = await vercel.updatePorts(box.name, [ECHO_PORT]) - if (!route) { - throw new Error('echo sandbox got no route') - } - const started = await run(box, `setsid nohup node -e "$ECHO" ${ECHO_PORT} >/tmp/echo.log 2>&1 ): Promise { - const row = syntheticRow(`probe${runId}`) - const deps = { config: { egressProxyUrl: `${echoUrl}/egress/forward`, environment: 'dev', egressPreset: 'open' } } as unknown as LifecycleDeps - const codex = accountProvider('codex')! - const member = await memberPlaceholder(row.id, 'egress-probe-member') - const env = { - CODEX_PH: codex.format(codex.placeholder(row)), - ANTHROPIC_PH: secretPlaceholder(row, 'ANTHROPIC_API_KEY'), - GIT_PH: gitBasicAuthorization(githubPlaceholderToken(row)), - GH_API_PH: `Bearer ${githubPlaceholderToken(row)}`, - MEMBER_PH: gitBasicAuthorization(member) - } - const refs = `${PUBLIC_REPO}/info/refs?service=git-upload-pack` - // Shared by both policies: model and account hosts. - const modelCases: ProbeCase[] = [ - { - name: 'chatgpt.com placeholder', - url: 'https://chatgpt.com/backend-api/codex/models', - headers: { authorization: '$CODEX_PH', 'chatgpt-account-id': 'probe-keep-account', 'x-api-key': 'probe-keep-other' }, - expect: { kind: 'forwarded', host: 'chatgpt.com', headers: { authorization: env.CODEX_PH, 'chatgpt-account-id': 'probe-keep-account', 'x-api-key': 'probe-keep-other' } } - }, - { name: 'chatgpt.com foreign bearer', url: 'https://chatgpt.com/backend-api/codex/models', headers: { authorization: 'Bearer probe-foreign' }, expect: { kind: 'denied' } }, - { name: 'chatgpt.com no credential', url: 'https://chatgpt.com/backend-api/codex/models', expect: { kind: 'denied' } }, - { - name: 'api.anthropic.com placeholder', - url: 'https://api.anthropic.com/v1/messages', - method: 'POST', - headers: { 'x-api-key': '$ANTHROPIC_PH', authorization: 'Bearer probe-keep-bearer' }, - expect: { kind: 'forwarded', host: 'api.anthropic.com', headers: { 'x-api-key': env.ANTHROPIC_PH, authorization: 'Bearer probe-keep-bearer' } } - }, - { name: 'api.anthropic.com foreign key', url: 'https://api.anthropic.com/v1/messages', method: 'POST', headers: { 'x-api-key': 'sk-ant-probe-foreign' }, expect: { kind: 'denied' } }, - { - name: 'api.github.com placeholder', - url: 'https://api.github.com/rate_limit', - headers: { authorization: '$GH_API_PH' }, - expect: { kind: 'forwarded', host: 'api.github.com', headers: { authorization: env.GH_API_PH } } - }, - { name: 'api.github.com no credential', url: 'https://api.github.com/rate_limit', expect: { kind: 'denied' } } - ] - const policies: { name: string; gitToken?: string; cases: ProbeCase[]; anonymousGit: boolean }[] = [ - { - name: 'with App token', - gitToken: 'ghs_egress_probe_bogus', - anonymousGit: false, - cases: [ - ...modelCases, - { name: 'github.com placeholder', url: refs, headers: { authorization: '$GIT_PH' }, expect: { kind: 'forwarded', host: 'github.com', headers: { authorization: env.GIT_PH } } }, - { name: 'github.com anonymous (fail-closed with a token)', url: refs, expect: { kind: 'denied' } } - ] - }, - { - name: 'member identity, no App token (#758)', - anonymousGit: true, - cases: [ - ...modelCases, - { name: 'github.com member placeholder', url: refs, headers: { authorization: '$MEMBER_PH' }, expect: { kind: 'forwarded', host: 'github.com', headers: { authorization: env.MEMBER_PH } } }, - { name: 'github.com anonymous', url: refs, expect: { kind: 'upstream', notDenied: true } } - ] - } - ] - const results: Record[] = [] - for (const policy of policies) { - await vercel.updateNetworkPolicy(probe.sessionId, egressPolicy(deps, row, policy.gitToken, [member])) - const answers = parseProbeOutput((await run(probe, probeScript(policy.cases), env)).stdout) - for (const probeCase of policy.cases) { - const answer = answers.get(probeCase.name) ?? { status: 0, body: '' } - const problems = judge(probeCase, answer) - failures.push(...problems.map((problem) => `[${policy.name}] ${problem}`)) - results.push({ policy: policy.name, case: probeCase.name, status: answer.status, ok: problems.length === 0 }) - } - // git itself, as the workspace doctor runs it. - const git = await run(probe, `GIT_TERMINAL_PROMPT=0 timeout 60 git ls-remote ${PUBLIC_REPO} HEAD >/tmp/ls 2>/tmp/ls.err; echo "rc=$?"; grep -c . /tmp/ls; grep -o 'returned error: [0-9]*' /tmp/ls.err | head -1`) - const ok = /rc=0/.test(git.stdout) - if (ok !== policy.anonymousGit) { - failures.push(`[${policy.name}] anonymous git ls-remote ${ok ? 'worked' : 'failed'} (${git.stdout.trim().replace(/\n/g, ' ')}), expected it to ${policy.anonymousGit ? 'work' : 'be refused'}`) - } - results.push({ policy: policy.name, case: 'git ls-remote (anonymous)', ok: ok === policy.anonymousGit, gitWorked: ok }) - } - report.firewall = results -} - -async function devToken(sub: string): Promise { - const file = process.env.ORCA_PROBE_DEV_JWK_FILE +const sub = `egress-probe-${runId}` +const accountId = `dev:${sub}` +const fileSecret = (env: string): string => { + const file = process.env[env] if (!file) { - throw new Error('the swap layer needs ORCA_PROBE_DEV_JWK_FILE (or pass --firewall-only)') + throw new Error(`set ${env}`) } - const jwk = JSON.parse(readFileSync(file, 'utf8')) as { kid: string; private: JWK; issuers?: { sandbox?: string }; audiences?: { sandbox?: string } } + return readFileSync(file, 'utf8').trim() +} + +async function userToken(): Promise { + const jwk = JSON.parse(fileSecret('ORCA_PROBE_DEV_JWK_FILE')) as { kid: string; private: JWK; issuers?: { sandbox?: string }; audiences?: { sandbox?: string } } return new SignJWT({ sub, purpose: 'sandbox' }) .setProtectedHeader({ alg: 'ES256', kid: jwk.kid }) .setIssuer(jwk.issuers?.sandbox ?? 'orca-sandbox-dev-test') .setAudience(jwk.audiences?.sandbox ?? 'orca-sandbox') .setIssuedAt() - .setExpirationTime('30m') + .setExpirationTime('45m') .sign(await importJWK(jwk.private, 'ES256')) } -async function swapLayer(echoUrl: string, failures: string[], report: Record, cleanup: (() => Promise)[]): Promise { - const token = await devToken(`egress-probe-${runId}`) - const call = async (method: string, path: string, body?: unknown) => { +type Reply = { status: number; body: Record | null } + +function client(bearer: string) { + return async (method: string, path: string, body?: unknown): Promise => { const response = await fetch(`${CONTROL_URL}${path}`, { method, - headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + // clone-progress: the desktop's create, whose clone runs after ready (GET /v1/sandboxes/:id/clone). + headers: { authorization: `Bearer ${bearer}`, 'content-type': 'application/json', 'x-orca-sandbox-features': 'clone-progress' }, body: body === undefined ? undefined : JSON.stringify(body), - signal: AbortSignal.timeout(150_000) + signal: AbortSignal.timeout(300_000) }) return { status: response.status, body: (await response.json().catch(() => null)) as Record | null } } - // A key shape no provider issues: only the probe's echo ever sees it. - const standIn = `sk-ant-api03-orcaprobe${randomBytes(16).toString('hex')}` - for (const [name, value] of [['ANTHROPIC_API_KEY', standIn], ['ANTHROPIC_BASE_URL', echoUrl]] as const) { - const put = await call('PUT', `/v1/secrets/personal/${name}`, { value }) - cleanup.push(() => call('DELETE', `/v1/secrets/personal/${name}`)) - if (put.status !== 200) { - throw new Error(`PUT secret ${name}: ${put.status} ${put.body?.error ?? ''}`) - } - } - const created = await call('POST', '/v1/sandboxes', { workspaceId: `egress-probe-${runId}`, machineSize: 'standard' }) - const sandboxId: string | undefined = created.body?.sandbox?.id - if (sandboxId) { - cleanup.unshift(() => call('DELETE', `/v1/sandboxes/${sandboxId}`)) - } - const sessionId: string | undefined = created.body?.sandbox?.session?.id - if (created.status !== 200 || !sessionId) { - throw new Error(`create workspace: ${created.status} ${created.body?.error ?? ''}`) - } - console.error(`[egress-probe] dev workspace ${sandboxId}`) - const box: Box = { name: '', sessionId } - // The VM's env: orcad's, as an agent sees it. - const envOf = `tr '\\0' '\\n' < /proc/$(cat ~/.orca-sandbox/orcad.pid)/environ | grep -E '^ANTHROPIC_(API_KEY|BASE_URL)=' | sed 's/^/export /'` - const cases: ProbeCase[] = [ - { - name: 'custom base URL, placeholder', - url: `${echoUrl}/v1/messages`, - method: 'POST', - headers: { 'x-api-key': '$ANTHROPIC_API_KEY', authorization: 'Bearer probe-keep-bearer' }, - expect: { kind: 'swapped', headers: { 'x-api-key': standIn, authorization: 'Bearer probe-keep-bearer' } } - }, - { name: 'custom base URL, foreign key', url: `${echoUrl}/v1/messages`, method: 'POST', headers: { 'x-api-key': 'sk-ant-probe-foreign' }, expect: { kind: 'denied' } } - ] - const result = await run(box, `eval "$(${envOf})"; [ "$ANTHROPIC_API_KEY" != "$STAND_IN_CHECK" ] && echo vm_holds_placeholder=1\n${probeScript(cases)}`, { STAND_IN_CHECK: standIn }) - if (!/^vm_holds_placeholder=1$/m.test(result.stdout)) { - failures.push('[swap] the VM does not hold a placeholder for ANTHROPIC_API_KEY') - } - const answers = parseProbeOutput(result.stdout) - const results = cases.map((probeCase) => { - const answer = answers.get(probeCase.name) ?? { status: 0, body: '' } - const problems = judge(probeCase, answer) - failures.push(...problems.map((problem) => `[swap] ${problem}`)) - return { case: probeCase.name, status: answer.status, ok: problems.length === 0 } +} + +/** The shipped CLI over a workspace's pairing code; the code never reaches logs or the report. */ +function orca(pairingCode: string, args: string[], timeoutMs = 60_000): Promise { + return new Promise((resolve) => { + execFile(ORCA_CLI, [...args, '--json', '--pairing-code', pairingCode], { timeout: timeoutMs, maxBuffer: 8 << 20, env: { ...process.env, ORCA_PAIRING_CODE: '' } }, (_error, stdout) => { + try { + resolve(JSON.parse(stdout)) + } catch { + resolve(null) + } + }) }) - report.swap = { sandboxId, results } +} + +/** Every string in a CLI result, one per line (terminal output may come as a string, lines or chunks). */ +function textOf(value: unknown): string { + if (typeof value === 'string') return value + if (Array.isArray(value)) return value.map(textOf).join('\n') + if (value && typeof value === 'object') return Object.values(value).map(textOf).join('\n') + return '' +} + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + +/** Runs one line in a fresh workspace terminal and returns the output once `PRB-END` shows. */ +async function inTerminal(pairingCode: string, projectRoot: string, line: string): Promise { + await orca(pairingCode, ['repo', 'add', '--path', projectRoot, '--kind', 'git']) + const created = await orca(pairingCode, ['terminal', 'create', '--worktree', `path:${projectRoot}`, '--title', 'egress-probe']) + const handle: string | undefined = created?.result?.terminal?.handle ?? created?.result?.handle ?? created?.result?.terminal?.id + if (!handle) { + throw new Error('could not open a workspace terminal') + } + // `clear` first: the read must only see this run's lines. + await orca(pairingCode, ['terminal', 'send', '--terminal', handle, '--text', `clear; ${line}`, '--enter']) + const deadline = Date.now() + 180_000 + while (Date.now() < deadline) { + const read = await orca(pairingCode, ['terminal', 'read', '--terminal', handle]) + const text = textOf(read?.result) + // The typed line is base64 past its first words, so PRB-END only shows once the runner prints it. + if (text.includes('PRB-END')) { + return text + } + await sleep(1500) + } + throw new Error('the probe did not finish in the workspace terminal') +} + +/** Git and env facts, one short line each, then the HTTP cases. */ +function factsScript(gitChecks: Record, bogus: Record): string { + const lines = Object.entries(gitChecks).map(([name, command]) => `GIT_TERMINAL_PROMPT=0 timeout 60 ${command} >/dev/null 2>&1; echo "PRB ${name} rc=$?"`) + // The VM must hold placeholders: compare by sha256 so the bogus values never cross the terminal. + for (const [name, digest] of Object.entries(bogus)) { + // The verdict word is a variable: the typed command (echoed by the terminal) must not contain it. + lines.push(`v=placeholder; [ "$(printf %s "\${${name}:-}" | sha256sum | cut -c1-8)" = ${digest} ] && v=bogus; echo "PRB env.${name} holds=$v"`) + } + return lines.join('; ') } async function main(): Promise { @@ -261,14 +143,114 @@ async function main(): Promise { const report: Record = { runId } const cleanup: (() => Promise)[] = [] const started = Date.now() + console.error(`[egress-probe] run ${runId}: throwaway identity ${sub}`) + const user = client(await userToken()) + const operator = client(fileSecret('ORCA_PROBE_OPERATOR_TOKEN_FILE')) + const bogusAnthropic = `sk-ant-api03-orcaprobe${randomBytes(16).toString('hex')}` + const bogusOpenAI = `sk-proj-orcaprobe${randomBytes(16).toString('hex')}` try { - const [echoBox, probeBox] = await Promise.all([bootBox('echo'), bootBox('probe')]) - cleanup.push(() => vercel.deleteBuildSandbox(echoBox.name), () => vercel.deleteBuildSandbox(probeBox.name)) - const echoUrl = await startEcho(echoBox) - await firewallLayer(probeBox, echoUrl, failures, report) - if (!firewallOnly) { - await swapLayer(echoUrl, failures, report, cleanup) + // Identity: the App bound for the test repo, and a stored GitHub identity (member placeholders). + const bound = await operator('POST', '/v1/internal/github/bindings', { repoUrl: APP_REPO, accountId }) + const installationId: number | undefined = bound.body?.bound?.installationId + if (bound.status !== 200 || !installationId) { + throw new Error(`bind the dev App: ${bound.status} ${bound.body?.error ?? ''}`) } + cleanup.push(() => operator('DELETE', '/v1/internal/github/bindings', { installationId, accountId })) + const member = await operator('POST', '/v1/internal/dev/member-github', { accountId, userId: sub, repoUrl: APP_REPO, login: 'egress-probe', githubUserId: 1, name: 'egress probe' }) + if (member.status !== 200) { + throw new Error(`store the probe's GitHub identity: ${member.status} ${member.body?.error ?? ''}`) + } + for (const [name, value] of [ + ['ANTHROPIC_API_KEY', bogusAnthropic], + ['ANTHROPIC_BASE_URL', ECHO], + ['OPENAI_API_KEY', bogusOpenAI], + ['OPENAI_BASE_URL', 'https://chatgpt.com/backend-api/codex'] + ] as const) { + const put = await user('PUT', `/v1/secrets/personal/${name}`, { value }) + cleanup.push(() => user('DELETE', `/v1/secrets/personal/${name}`)) + if (put.status !== 200) { + throw new Error(`PUT secret ${name}: ${put.status} ${put.body?.error ?? ''}`) + } + } + const create = async (label: string, repo: string) => { + console.error(`[egress-probe] creating the ${label} workspace (${sub}-${label})`) + const reply = await user('POST', '/v1/sandboxes', { workspaceId: `${sub}-${label}`, machineSize: 'standard', repo: { url: repo }, useEnvironment: false }) + const id: string | undefined = reply.body?.sandbox?.id + if (id) { + cleanup.unshift(() => user('DELETE', `/v1/sandboxes/${id}`)) + } + if (reply.status !== 200 || !reply.body?.connection?.pairingCode) { + throw new Error(`create the ${label} workspace: ${reply.status} ${reply.body?.error ?? ''}`) + } + console.error(`[egress-probe] ${label} workspace ${id}`) + // The desktop's path: the clone runs after ready, and the desktop polls it. + let clone = 'unknown' + for (const until = Date.now() + 240_000; Date.now() < until; await sleep(3_000)) { + clone = String((await user('GET', `/v1/sandboxes/${id}/clone`)).body?.clone?.state ?? 'unknown') + if (clone.startsWith('done') || clone.startsWith('failed')) break + } + clones[label] = clone + return { id: id!, pairingCode: reply.body.connection.pairingCode as string, projectRoot: reply.body.connection.projectRoot as string } + } + const clones: Record = {} + const [publicBox, appBox] = await Promise.all([create('public', PUBLIC_REPO), create('app', APP_REPO)]) + report.workspaces = { public: publicBox.id, app: appBox.id } + + const anthropic: ProbeCase[] = [ + { + name: 'echo host (ANTHROPIC_BASE_URL): placeholder swapped, other auth untouched', + url: `${ECHO}/post`, + method: 'POST', + headers: { 'x-api-key': '$ANTHROPIC_API_KEY', authorization: 'Bearer probe-keep-bearer' }, + expect: { kind: 'swapped', headers: { 'x-api-key': bogusAnthropic, authorization: 'Bearer probe-keep-bearer' } } + }, + { name: 'echo host: foreign key fails closed', url: `${ECHO}/post`, method: 'POST', headers: { 'x-api-key': 'sk-ant-probe-foreign' }, expect: { kind: 'denied' } }, + { + name: 'chatgpt.com: placeholder reaches ChatGPT (relay)', + url: 'https://chatgpt.com/backend-api/codex/models', + headers: { authorization: '$PROBE_OPENAI_BEARER' }, + expect: { kind: 'upstream' } + }, + { name: 'chatgpt.com: foreign bearer fails closed', url: 'https://chatgpt.com/backend-api/codex/models', headers: { authorization: 'Bearer probe-foreign' }, expect: { kind: 'denied' } }, + { name: 'api.github.com: no credential fails closed', url: 'https://api.github.com/rate_limit', expect: { kind: 'denied' } } + ] + const appCases: ProbeCase[] = [ + { name: 'api.github.com: workspace placeholder swapped (GitHub answers 200)', url: 'https://api.github.com/rate_limit', headers: { authorization: '$PROBE_GH_BEARER' }, expect: { kind: 'upstream' } }, + { name: 'github.com: anonymous fails closed with an App token', url: `${APP_REPO}.git/info/refs?service=git-upload-pack`, expect: { kind: 'denied' } } + ] + const digests = { ANTHROPIC_API_KEY: sha8(bogusAnthropic), OPENAI_API_KEY: sha8(bogusOpenAI) } + // The runner reads whole header values from env (`$NAME`), so the Bearer forms are built here. + const prelude = 'export PROBE_OPENAI_BEARER="Bearer $OPENAI_API_KEY" PROBE_GH_BEARER="Bearer $GH_TOKEN"; ' + console.error('[egress-probe] running the checks in both workspaces') + const runs = await Promise.all([ + inTerminal(publicBox.pairingCode, publicBox.projectRoot, `${prelude}${factsScript({ 'git.public-anonymous': `git ls-remote ${PUBLIC_REPO} HEAD` }, digests)}; ${runnerCommand(anthropic)}`), + inTerminal(appBox.pairingCode, appBox.projectRoot, `${prelude}${factsScript({ 'git.app-anonymous': `git -c http.extraheader= -c http.https://github.com/.extraheader= ls-remote ${APP_REPO} HEAD`, 'git.app-workspace': 'git ls-remote origin HEAD' }, {})}; ${runnerCommand(appCases)}`) + ]) + const results: Record[] = [] + const check = (workspace: string, name: string, ok: boolean, detail: string) => { + results.push({ workspace, check: name, ok }) + if (!ok) { + failures.push(`[${workspace}] ${name}: ${detail}`) + } + } + for (const [workspace, text, cases] of [['public', runs[0], anthropic], ['app', runs[1], appCases]] as const) { + const summaries = parseRunnerOutput(text, cases.length) + cases.forEach((probeCase, i) => { + const problems = judge(probeCase, summaries[i]!) + check(workspace, probeCase.name, problems.length === 0, problems.join('; ')) + }) + } + const rc = (text: string, name: string) => new RegExp(`PRB ${name.replaceAll('.', '\\.')} rc=(\\d+)`).exec(text)?.[1] + check('public', 'github.com: anonymous git ls-remote of a public repo works (#758)', rc(runs[0], 'git.public-anonymous') === '0', `rc=${rc(runs[0], 'git.public-anonymous')}`) + check('app', 'github.com: anonymous git ls-remote is refused with an App token', rc(runs[1], 'git.app-anonymous') !== '0' && rc(runs[1], 'git.app-anonymous') !== undefined, `rc=${rc(runs[1], 'git.app-anonymous')}`) + check('app', "github.com: the workspace's own git (placeholder) works", rc(runs[1], 'git.app-workspace') === '0', `rc=${rc(runs[1], 'git.app-workspace')}`) + for (const [workspace, state] of Object.entries(clones)) { + check(workspace, 'github.com: the workspace repo clones', state === 'done', `clone ${state}`) + } + for (const name of Object.keys(digests)) { + check('public', `the VM holds a placeholder for ${name}`, new RegExp(`PRB env\\.${name} holds=placeholder`).test(runs[0]), 'the VM env holds the bogus value itself') + } + report.checks = results } catch (error) { failures.push(`probe error: ${error instanceof Error ? error.message : String(error)}`) } finally { diff --git a/cloud/apps/sandbox-control/test/egress-probe-checks.test.ts b/cloud/apps/sandbox-control/test/egress-probe-checks.test.ts index 275a42b846c..5701b156aa7 100644 --- a/cloud/apps/sandbox-control/test/egress-probe-checks.test.ts +++ b/cloud/apps/sandbox-control/test/egress-probe-checks.test.ts @@ -1,62 +1,69 @@ -import { execFileSync } from 'node:child_process' +import { execFile } from 'node:child_process' +import { createServer } from 'node:http' +import type { AddressInfo } from 'node:net' import { describe, expect, it } from 'vitest' import { EGRESS_DENIED_BODY } from '../src/network-policy' -import { judge, parseProbeOutput, probeScript, type ProbeCase } from '../scripts/egress-probe-checks' +import { judge, parseRunnerOutput, runnerCommand, sha8, type ProbeCase, type Summary } from '../scripts/egress-probe-checks' -const echo = (headers: Record) => JSON.stringify({ probeEcho: true, method: 'GET', url: '/x', headers }) +const answer = (summary: Partial): Summary => ({ status: 200, denied: false, edgeBlocked: false, ...summary }) describe('egress probe judge', () => { - const forwarded: ProbeCase = { - name: 'chatgpt', - url: 'https://chatgpt.com/x', - expect: { kind: 'forwarded', host: 'chatgpt.com', headers: { authorization: 'Bearer ph', 'x-api-key': 'keep' } } - } + const swapped: ProbeCase = { name: 'echo', url: 'https://echo/post', expect: { kind: 'swapped', headers: { 'x-api-key': 'stand-in', authorization: 'Bearer keep' } } } - it('accepts a forward for the right host with every listed header unchanged', () => { - const body = echo({ 'vercel-forwarded-host': 'chatgpt.com', 'vercel-sandbox-oidc-token': '[present]', authorization: 'Bearer ph', 'x-api-key': 'keep' }) - expect(judge(forwarded, { status: 200, body })).toEqual([]) + it('accepts the provider seeing the swapped key and the other header unchanged', () => { + expect(judge(swapped, answer({ echoed: { 'x-api-key': sha8('stand-in'), authorization: sha8('Bearer keep') } }))).toEqual([]) }) - it('names a changed or dropped header and a wrong host, never the values', () => { - const body = echo({ 'vercel-forwarded-host': 'api.openai.com', 'vercel-sandbox-oidc-token': '[present]', authorization: 'Bearer real-secret' }) - const problems = judge(forwarded, { status: 200, body }) + it('names a header that changed or never arrived, and anything leaked from the hop, never values', () => { + const problems = judge(swapped, answer({ echoed: { 'x-api-key': sha8('orca-ph-placeholder') }, leaked: ['vercel-sandbox-oidc-token'] })) expect(problems).toEqual([ - 'chatgpt: forwarded for "api.openai.com", expected chatgpt.com', - 'chatgpt: header authorization was changed on the way', - 'chatgpt: header x-api-key was dropped on the way' + 'echo: the provider saw vercel-sandbox-oidc-token', + 'echo: header x-api-key arrived with another value', + 'echo: header authorization did not arrive' ]) - expect(problems.join(' ')).not.toContain('real-secret') + expect(problems.join(' ')).not.toContain('stand-in') }) - it("requires the firewall's own 403 body for a fail-closed case", () => { - const denied: ProbeCase = { name: 'foreign', url: 'https://chatgpt.com/x', expect: { kind: 'denied' } } - expect(judge(denied, { status: 403, body: `${EGRESS_DENIED_BODY}\n` })).toEqual([]) - expect(judge(denied, { status: 403, body: 'cloudflare' })).toHaveLength(1) - expect(judge(denied, { status: 200, body: echo({}) })).toHaveLength(1) + it("requires the firewall's own 403 for a fail-closed case", () => { + const denied: ProbeCase = { name: 'foreign', url: 'https://x', expect: { kind: 'denied' } } + expect(judge(denied, answer({ status: 403, denied: true }))).toEqual([]) + expect(judge(denied, answer({ status: 403, edgeBlocked: true }))).toHaveLength(1) + expect(judge(denied, answer({ status: 401 }))).toHaveLength(1) }) - it('treats a denial or no answer as failing an upstream case', () => { - const upstream: ProbeCase = { name: 'anon git', url: 'https://github.com/x', expect: { kind: 'upstream', notDenied: true } } - expect(judge(upstream, { status: 200, body: '001e# service=git-upload-pack' })).toEqual([]) - expect(judge(upstream, { status: 403, body: EGRESS_DENIED_BODY })).toHaveLength(1) - expect(judge(upstream, { status: 0, body: '' })).toHaveLength(1) - }) - - it('a swapped request must carry nothing from the forward hop', () => { - const swapped: ProbeCase = { name: 'swap', url: 'https://echo/x', expect: { kind: 'swapped', headers: { 'x-api-key': 'stand-in' } } } - expect(judge(swapped, { status: 200, body: echo({ 'x-api-key': 'stand-in' }) })).toEqual([]) - expect(judge(swapped, { status: 200, body: echo({ 'x-api-key': 'stand-in', 'vercel-sandbox-oidc-token': '[present]' }) })).toEqual([ - 'swap: the provider saw vercel-sandbox-oidc-token' - ]) + it('an upstream case fails on the firewall 403, an edge block page or no answer', () => { + const upstream: ProbeCase = { name: 'chatgpt', url: 'https://chatgpt.com/x', expect: { kind: 'upstream' } } + expect(judge(upstream, answer({ status: 401 }))).toEqual([]) + expect(judge(upstream, answer({ status: 403, denied: true }))).toEqual(['chatgpt: refused by the firewall']) + expect(judge(upstream, answer({ status: 403, edgeBlocked: true }))).toHaveLength(1) + expect(judge(upstream, answer({ status: 0 }))).toEqual(['chatgpt: no answer']) }) }) -describe('probeScript', () => { - it('reads $NAME header values from env and reports one line per request', () => { - const script = probeScript([{ name: "it's one", url: 'http://127.0.0.1:9/never', headers: { authorization: '$PH', 'x-other': 'literal' } }]) - expect(script).toContain('-H "authorization: ${PH}"') - expect(script).toContain("-H 'x-other: literal'") - const answers = parseProbeOutput(execFileSync('bash', ['-c', script], { env: { ...process.env, PH: 'Bearer x' }, encoding: 'utf8' })) - expect(answers.get("it's one")).toEqual({ status: 0, body: '' }) +describe('the in-VM runner', () => { + it('reports status, the deny body and echoed header digests from env-held values', async () => { + // A local stand-in for postman-echo.com: echoes request headers under `headers`. + const server = createServer((req, res) => { + if (req.url === '/denied') { + res.writeHead(403, { 'content-type': 'text/plain' }).end(EGRESS_DENIED_BODY) + return + } + res.writeHead(200, { 'content-type': 'application/json' }).end(JSON.stringify({ headers: { ...req.headers, 'vercel-forwarded-host': 'x' } })) + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + const base = `http://127.0.0.1:${(server.address() as AddressInfo).port}` + const cases = [ + { name: 'echo', url: `${base}/post`, method: 'POST' as const, headers: { 'x-api-key': '$PH', authorization: 'Bearer keep' } }, + { name: 'denied', url: `${base}/denied` } + ] + const stdout = await new Promise((resolve, reject) => + execFile('bash', ['-c', runnerCommand(cases)], { env: { ...process.env, PH: 'secret-ph' } }, (error, out) => (error ? reject(error) : resolve(out))) + ) + server.close() + expect(stdout).toContain('PRB-END') + expect(stdout).not.toContain('secret-ph') + const [echo, denied] = parseRunnerOutput(stdout, 2) + expect(echo).toEqual({ status: 200, denied: false, edgeBlocked: false, echoed: { 'x-api-key': sha8('secret-ph'), authorization: sha8('Bearer keep') }, leaked: ['vercel-forwarded-host'] }) + expect(denied).toMatchObject({ status: 403, denied: true }) }) }) diff --git a/config/scripts/cloud-integration/boot-twice-check.sh b/config/scripts/cloud-integration/boot-twice-check.sh new file mode 100755 index 00000000000..94c2c37b8e2 --- /dev/null +++ b/config/scripts/cloud-integration/boot-twice-check.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Premerge step for image changes (premerge-check.sh runs it only when cloud/apps/sandbox-control/image/** changed): +# provisions the golden image in Docker and boots it twice; the second boot must install, fetch and redo nothing. +# See cloud/apps/sandbox-control/image/boot-twice/README.md. Run from the merged worktree's root. +# - Docker unavailable: a loud warning and exit 0 (fail open), so a host without Docker never blocks every merge. +# - On omarchy it takes its own slot (~/orca-offload/bin/slot.sh), waiting at most BOOT_TWICE_SLOT_WAIT_S (300 s). +# - Hard limit BOOT_TWICE_TIMEOUT_S (600 s); a cold image build is ~5 min, a cached one ~15 s. +set -uo pipefail +timeout_s=${BOOT_TWICE_TIMEOUT_S:-600} +slot_wait_s=${BOOT_TWICE_SLOT_WAIT_S:-300} +if ! docker info >/dev/null 2>&1; then + printf '\npremerge: WARNING ==============================================================\n' >&2 + printf 'premerge: WARNING: Docker is unavailable here, so the boot-twice image check DID NOT RUN.\n' >&2 + printf 'premerge: WARNING: run it by hand before merging this image change (see image/boot-twice/README.md).\n' >&2 + printf 'premerge: WARNING ==============================================================\n\n' >&2 + exit 0 +fi +slot_tool="$HOME/orca-offload/bin/slot.sh" +tag="boot-twice-$$-$(date +%s)" +if [[ -x "$slot_tool" ]]; then + waited=0 + until "$slot_tool" acquire "$tag" >/dev/null 2>&1; do + if ((waited >= slot_wait_s)); then + echo "premerge: FAILED: boot-twice image check found no free omarchy slot in ${slot_wait_s}s; rerun the premerge" >&2 + exit 1 + fi + sleep 15 + waited=$((waited + 15)) + done + trap '"$slot_tool" release "$tag" >/dev/null 2>&1 || rm -rf "$HOME/orca-offload/wt/$tag"' EXIT +fi +echo "premerge: boot-twice image check (limit ${timeout_s}s)" +report="$(mktemp)" +(cd cloud && pnpm install --frozen-lockfile >/dev/null) +timeout --kill-after=30 "$timeout_s" pnpm --dir cloud --filter @orca-cloud/sandbox-control exec \ + tsx image/boot-twice/boot-twice.ts --release latest --report "$report" >/dev/null +status=$? +if ((status == 124 || status == 137)); then + echo "premerge: FAILED: boot-twice image check passed its ${timeout_s}s limit (cold image build? rerun once)" >&2 +elif ((status != 0)); then + echo "premerge: FAILED: boot-twice image check: the second boot of the image is not clean:" >&2 + jq -r '.failures[]? | "premerge: " + .' "$report" >&2 2>/dev/null || true + echo "premerge: details: cloud/apps/sandbox-control/image/boot-twice/README.md" >&2 +else + echo "premerge: boot-twice image check OK ($(jq -r '"boot 2 " + (.boots[1].ms|tostring) + " ms"' "$report" 2>/dev/null))" +fi +rm -f "$report" +exit "$status" diff --git a/config/scripts/cloud-integration/premerge-check.sh b/config/scripts/cloud-integration/premerge-check.sh index cac1740953c..9a2e1d40128 100755 --- a/config/scripts/cloud-integration/premerge-check.sh +++ b/config/scripts/cloud-integration/premerge-check.sh @@ -188,5 +188,9 @@ if ((${#tc_pkgs[@]})); then (cd "$pkg" && pnpm typecheck) done fi +# Image or VM-script changes: boot the image twice (Docker); the second boot must be clean. Fails open without Docker. +if [[ -n "$(git diff --name-only "$base"...HEAD -- cloud/apps/sandbox-control/image)" ]]; then + bash config/scripts/cloud-integration/boot-twice-check.sh +fi if [[ "${PREMERGE_FULL:-}" == 1 ]]; then scope="the full vitest suite"; else scope="${#tests[@]} targeted test file(s)"; fi echo "premerge OK: $branch merges cleanly into cloud/integration with typecheck, lint and $scope green."