Merge remote-tracking branch 'private/cloud/integration' into cloud/live-git-access

This commit is contained in:
Neil
2026-11-15 12:42:00 -08:00
198 changed files with 7669 additions and 395 deletions
+81
View File
@@ -0,0 +1,81 @@
name: Cloud Egress Probe
# Release check for sandbox egress, against DEV, through Orca's own paths (dev workspaces driven
# over the shipped `orca` CLI's pairing code). Bogus tokens only; postman-echo.com is the echo host.
# Details and the local run: cloud/apps/sandbox-control/scripts/README.md
on:
workflow_dispatch:
permissions:
contents: read
concurrency:
group: cloud-egress-probe
cancel-in-progress: false
defaults:
run:
working-directory: cloud
jobs:
probe:
runs-on: ubuntu-24.04
timeout-minutes: 20
env:
DEV_TEST_JWK: ${{ secrets.ORCA_DEV_TEST_JWK }}
DEV_OPERATOR_TOKEN: ${{ secrets.ORCA_DEV_OPERATOR_TOKEN }}
steps:
- name: Check the dev credentials are configured
id: creds
working-directory: .
run: |
if [ -z "$DEV_TEST_JWK" ] || [ -z "$DEV_OPERATOR_TOKEN" ]; then
echo "::warning::ORCA_DEV_TEST_JWK / ORCA_DEV_OPERATOR_TOKEN are not set: the egress probe did not run"
echo "skip=1" >> "$GITHUB_OUTPUT"
fi
- if: steps.creds.outputs.skip != '1'
uses: actions/checkout@v4
- if: steps.creds.outputs.skip != '1'
uses: ./.github/actions/setup-pnpm
with:
package-json-file: cloud/package.json
- if: steps.creds.outputs.skip != '1'
uses: actions/setup-node@v4
with:
node-version: 24
check-latest: true
- if: steps.creds.outputs.skip != '1'
run: pnpm install --frozen-lockfile --filter @orca-cloud/sandbox-control...
# The shipped CLI from the latest release's Linux package: the same pairing path the desktop uses.
- name: Install the orca CLI
if: steps.creds.outputs.skip != '1'
working-directory: .
env:
GH_TOKEN: ${{ github.token }}
run: |
gh release download --repo stablyai/orca --pattern 'orca-ide_*_amd64.deb' --dir "$RUNNER_TEMP/orca"
dpkg-deb -x "$RUNNER_TEMP"/orca/orca-ide_*_amd64.deb "$RUNNER_TEMP/orca/x"
cli="$(find "$RUNNER_TEMP/orca/x" -path '*/bin/orca' -type f | head -n 1)"
test -x "$cli"
echo "ORCA_PROBE_CLI=$cli" >> "$GITHUB_ENV"
- name: Probe dev egress
if: steps.creds.outputs.skip != '1'
run: |
umask 077
printf '%s' "$DEV_TEST_JWK" > "$RUNNER_TEMP/dev-jwk.json"
printf '%s' "$DEV_OPERATOR_TOKEN" > "$RUNNER_TEMP/dev-operator"
ORCA_PROBE_DEV_JWK_FILE="$RUNNER_TEMP/dev-jwk.json" ORCA_PROBE_OPERATOR_TOKEN_FILE="$RUNNER_TEMP/dev-operator" \
pnpm --filter @orca-cloud/sandbox-control exec tsx scripts/egress-probe.ts > "$RUNNER_TEMP/egress-probe.json"
- uses: actions/upload-artifact@v4
if: always() && steps.creds.outputs.skip != '1'
with:
name: cloud-egress-probe-report
path: ${{ runner.temp }}/egress-probe.json
retention-days: 14
+83
View File
@@ -0,0 +1,83 @@
name: Cloud VM Boot Twice
# Provisions the golden image in a container and boots the same disk twice through the control
# plane's boot steps. The second boot runs offline and must install, fetch and redo nothing.
# Local run and details: cloud/apps/sandbox-control/image/boot-twice/README.md
on:
pull_request:
types: [opened, synchronize, reopened, ready_for_review]
paths:
- cloud/apps/sandbox-control/image/**
- cloud/apps/sandbox-control/src/golden-images.ts
- cloud/apps/sandbox-control/src/sandbox-vm-commands.ts
- cloud/apps/sandbox-control/src/orcad-ready.ts
- .github/workflows/cloud-vm-boot-twice.yml
workflow_dispatch:
inputs:
release:
description: Orca release whose orcad bundle the image carries (vX.Y.Z or latest)
required: false
default: latest
type: string
permissions:
contents: read
concurrency:
group: cloud-vm-boot-twice-${{ github.ref }}
cancel-in-progress: true
defaults:
run:
working-directory: cloud
jobs:
boot-twice:
if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
runs-on: ubuntu-24.04
timeout-minutes: 25
env:
RELEASE: ${{ inputs.release || 'latest' }}
CONTEXT: ${{ runner.temp }}/boot-twice
# Raises the GitHub API limit for the latest-release lookup; read-only, no secret involved.
GITHUB_TOKEN: ${{ github.token }}
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/setup-pnpm
with:
package-json-file: cloud/package.json
- uses: actions/setup-node@v4
with:
node-version: 24
check-latest: true
- run: pnpm install --frozen-lockfile --filter @orca-cloud/sandbox-control...
# The context holds exactly what provisioning reads (script + env), so its bytes key the
# layer cache: an unchanged image reuses the provisioned layer and only boots.
- name: Write the image build context
run: pnpm --filter @orca-cloud/sandbox-control exec tsx image/boot-twice/boot-twice.ts --release "$RELEASE" --context-out "$CONTEXT"
- uses: docker/setup-buildx-action@v3
- name: Provision the image (cached)
uses: docker/build-push-action@v6
with:
context: ${{ env.CONTEXT }}
target: harness
tags: orca-boot-twice:ci
load: true
cache-from: type=gha,scope=cloud-vm-boot-twice
cache-to: type=gha,mode=max,scope=cloud-vm-boot-twice
- name: Boot twice
run: pnpm --filter @orca-cloud/sandbox-control exec tsx image/boot-twice/boot-twice.ts --release "$RELEASE" --image orca-boot-twice:ci --report "$RUNNER_TEMP/boot-twice-report.json"
- uses: actions/upload-artifact@v4
if: always()
with:
name: cloud-vm-boot-twice-report
path: ${{ runner.temp }}/boot-twice-report.json
retention-days: 14
+10
View File
@@ -0,0 +1,10 @@
# Build context: cloud/apps (the relay imports sandbox-control's codex-paths.ts, the shared path pinning).
FROM node:24-slim
WORKDIR /app
COPY sandbox-control/src/codex-paths.ts sandbox-control/src/codex-paths.ts
COPY chatgpt-relay/package.json chatgpt-relay/package.json
COPY chatgpt-relay/src chatgpt-relay/src
WORKDIR /app/chatgpt-relay
ENV NODE_ENV=production
USER node
CMD ["node", "src/index.ts"]
+19
View File
@@ -0,0 +1,19 @@
#!/usr/bin/env bash
# Builds the relay image with Cloud Build (context: cloud/apps) and prints its digest-pinned name for
# `terraform apply -var image=...` in cloud/infra/chatgpt-relay.
# Usage: build-image.sh <project> [region]
set -euo pipefail
project="${1:?project}"
region="${2:-us-central1}"
apps="$(cd "$(dirname "$0")/.." && pwd)"
repo="$region-docker.pkg.dev/$project/orca-chatgpt-relay/relay"
tag="$(git -C "$apps" rev-parse --short=12 HEAD)"
gcloud builds submit "$apps" --project "$project" --region "$region" --quiet \
--config /dev/stdin <<YAML >&2
steps:
- name: gcr.io/cloud-builders/docker
args: [build, -f, chatgpt-relay/Dockerfile, -t, "$repo:$tag", .]
images: ["$repo:$tag"]
YAML
digest="$(gcloud artifacts docker images describe "$repo:$tag" --project "$project" --format='value(image_summary.digest)')"
echo "$repo@$digest"
+17
View File
@@ -0,0 +1,17 @@
{
"name": "@orca-cloud/chatgpt-relay",
"private": true,
"version": "0.0.0",
"type": "module",
"scripts": {
"start": "node src/index.ts",
"test": "vitest run",
"typecheck": "tsc -p tsconfig.json --noEmit",
"lint": "tsc -p tsconfig.json --noEmit"
},
"devDependencies": {
"@types/node": "^24.19.2",
"typescript": "^7.0.2",
"vitest": "^5.0.5"
}
}
+8
View File
@@ -0,0 +1,8 @@
import { createServer } from 'node:http'
import { createRelay } from './relay.ts'
const server = createServer(createRelay())
// Long Codex turns stream for minutes: no Node-side request or header timeout (Cloud Run caps it).
server.requestTimeout = 0
server.headersTimeout = 0
server.listen(Number(process.env.PORT ?? 8080), () => console.log(JSON.stringify({ event: 'relay_listening' })))
+144
View File
@@ -0,0 +1,144 @@
// CODEX-RELAY: chatgpt.com refuses Cloudflare Workers' egress, so the sandbox-control egress proxy
// sends its chatgpt.com requests here (Cloud Run, which ChatGPT answers). The Worker already chose the
// credential and pinned the path; this relay pins the path again, forwards to chatgpt.com only, and
// streams the answer back as it arrives (Codex reads Server-Sent Events).
// Cloud Run IAM authenticates the Worker (an ID token in X-Serverless-Authorization, run.invoker on
// one service account): nothing unauthenticated reaches this code. No header or body is ever logged.
import type { IncomingMessage, ServerResponse } from 'node:http'
import { Readable } from 'node:stream'
import { codexPathAllowed } from '../../sandbox-control/src/codex-paths.ts'
export const UPSTREAM_ORIGIN = 'https://chatgpt.com'
/** Never forwarded: hop-by-hop, Cloud Run's own auth and tracing, and proxy breadcrumbs. */
const DROP_REQUEST = new Set([
'host',
'connection',
'keep-alive',
'proxy-authorization',
'proxy-connection',
'te',
'trailer',
'transfer-encoding',
'upgrade',
'content-length',
'x-serverless-authorization',
'forwarded',
'traceparent',
'tracestate',
'x-cloud-trace-context'
])
const DROP_REQUEST_PREFIXES = ['x-forwarded-', 'x-goog-', 'cf-']
/** fetch already decoded the body, so its encoding and length no longer describe what we send. */
const DROP_RESPONSE = new Set([
'connection',
'keep-alive',
'transfer-encoding',
'content-encoding',
'content-length',
'trailer',
'upgrade'
])
export type RelayDeps = {
fetch?: typeof fetch
log?: (line: Record<string, unknown>) => void
now?: () => number
}
function refuse(res: ServerResponse, status: number, reason: string): void {
res.writeHead(status, { 'content-type': 'text/plain' })
res.end(reason)
}
/** The first two path segments only (e.g. /backend-api/codex): never ids or queries. */
function pathLabel(path: string): string {
return (path.split('?')[0] ?? '').split('/').slice(0, 3).join('/')
}
export function forwardHeaders(req: IncomingMessage): Headers {
const headers = new Headers()
for (const [name, value] of Object.entries(req.headers)) {
const key = name.toLowerCase()
if (value === undefined || DROP_REQUEST.has(key) || DROP_REQUEST_PREFIXES.some((p) => key.startsWith(p))) {
continue
}
for (const one of Array.isArray(value) ? value : [value]) {
headers.append(key, one)
}
}
return headers
}
export function createRelay(deps: RelayDeps = {}) {
const doFetch = deps.fetch ?? fetch
const log = deps.log ?? ((line) => console.log(JSON.stringify(line)))
const now = deps.now ?? Date.now
return async function handle(req: IncomingMessage, res: ServerResponse): Promise<void> {
const started = now()
const method = (req.method ?? 'GET').toUpperCase()
const path = req.url ?? '/'
if (path === '/health' && method === 'GET') {
res.writeHead(200, { 'content-type': 'text/plain' })
res.end('ok')
return
}
// Codex is configured for HTTPS (supports_websockets = false); the Worker proxy can't upgrade either.
if (req.headers.upgrade) {
refuse(res, 426, 'upgrade_not_supported')
return
}
if (!path.startsWith('/') || !codexPathAllowed(method, path)) {
log({ event: 'relay_refused', reason: 'path_not_allowed', method, path: pathLabel(path) })
refuse(res, 403, 'path_not_allowed')
return
}
const hasBody = method !== 'GET' && method !== 'HEAD'
const abort = new AbortController()
// The VM's client went away: stop reading from chatgpt.com too.
res.on('close', () => {
if (!res.writableFinished) {
abort.abort()
}
})
let upstream: Response
try {
upstream = await doFetch(`${UPSTREAM_ORIGIN}${path}`, {
method,
headers: forwardHeaders(req),
body: hasBody ? (Readable.toWeb(req) as ReadableStream<Uint8Array>) : undefined,
// Node needs this for a streamed request body.
...(hasBody ? { duplex: 'half' } : {}),
redirect: 'manual',
signal: abort.signal
} as RequestInit)
} catch {
log({ event: 'relay_upstream_failed', method, path: pathLabel(path), ms: now() - started })
if (!res.headersSent) {
refuse(res, 502, 'upstream_unreachable')
}
return
}
const headers: Record<string, string[]> = {}
upstream.headers.forEach((value, name) => {
if (!DROP_RESPONSE.has(name.toLowerCase())) {
;(headers[name] ??= []).push(value)
}
})
res.writeHead(upstream.status, headers)
// Each chunk is written as it arrives: SSE events reach Codex without buffering.
res.flushHeaders()
if (upstream.body) {
try {
for await (const chunk of upstream.body as unknown as AsyncIterable<Uint8Array>) {
res.write(chunk)
}
} catch {
// The client left (abort) or chatgpt.com cut the stream: end what was sent.
}
}
res.end()
log({ event: 'relay', method, path: pathLabel(path), status: upstream.status, ms: now() - started })
}
}
+116
View File
@@ -0,0 +1,116 @@
import { createServer, request, type Server } from 'node:http'
import type { AddressInfo } from 'node:net'
import { afterEach, describe, expect, it } from 'vitest'
import { createRelay } from '../src/relay.ts'
type Seen = { url: string; headers: Headers; body: string | null }
let server: Server | null = null
afterEach(() => new Promise<void>((done) => (server ? server.close(() => done()) : done())))
async function start(reply: (seen: Seen) => Response) {
const seen: Seen[] = []
const logs: Record<string, unknown>[] = []
const relay = createRelay({
log: (line) => logs.push(line),
fetch: async (input, init) => {
const body = init?.body ? await new Response(init.body as BodyInit).text() : null
const one = { url: String(input), headers: new Headers(init?.headers), body }
seen.push(one)
return reply(one)
}
})
server = createServer((req, res) => void relay(req, res))
await new Promise<void>((done) => server!.listen(0, '127.0.0.1', done))
const port = (server.address() as AddressInfo).port
const call = (method: string, path: string, headers: Record<string, string> = {}, body?: string) =>
new Promise<{ status: number; headers: Record<string, unknown>; body: string; chunks: string[] }>((resolve, reject) => {
const req = request({ host: '127.0.0.1', port, method, path, headers }, (res) => {
const chunks: string[] = []
res.setEncoding('utf8')
res.on('data', (c: string) => chunks.push(c))
res.on('end', () => resolve({ status: res.statusCode ?? 0, headers: res.headers, body: chunks.join(''), chunks }))
})
req.on('error', reject)
req.end(body)
})
return { seen, logs, call }
}
describe('ChatGPT relay (CODEX-RELAY)', () => {
it('forwards an allowed Codex call to chatgpt.com with its credential, minus Cloud Run auth and hop headers', async () => {
const t = await start(() => new Response('{"ok":true}', { status: 200, headers: { 'content-type': 'application/json', 'content-encoding': 'gzip' } }))
const res = await t.call(
'POST',
'/backend-api/codex/responses',
{
authorization: 'Bearer real-token',
'chatgpt-account-id': 'acct-1',
'x-serverless-authorization': 'Bearer google-id-token',
'x-forwarded-for': '1.2.3.4',
'content-type': 'application/json'
},
'{"input":"hi"}'
)
expect(res.status).toBe(200)
expect(res.body).toBe('{"ok":true}')
expect(res.headers['content-encoding']).toBeUndefined()
const sent = t.seen[0]!
expect(sent.url).toBe('https://chatgpt.com/backend-api/codex/responses')
expect(sent.body).toBe('{"input":"hi"}')
expect(sent.headers.get('authorization')).toBe('Bearer real-token')
expect(sent.headers.get('chatgpt-account-id')).toBe('acct-1')
expect(sent.headers.get('x-serverless-authorization')).toBeNull()
expect(sent.headers.get('x-forwarded-for')).toBeNull()
expect(sent.headers.get('host')).toBeNull()
})
it('pins the same paths as the Worker: nothing else on chatgpt.com', async () => {
const t = await start(() => new Response('{}'))
expect((await t.call('GET', '/backend-api/conversations')).status).toBe(403)
expect((await t.call('POST', '/backend-api/wham/usage')).status).toBe(403)
expect((await t.call('GET', '/backend-api/codex/../conversations')).status).toBe(403)
expect((await t.call('GET', '/backend-api/wham/usage')).status).toBe(200)
expect(t.seen.map((s) => s.url)).toEqual(['https://chatgpt.com/backend-api/wham/usage'])
})
it('streams Server-Sent Events as they arrive, not after the upstream ends', async () => {
let push: (text: string) => void = () => {}
let close: () => void = () => {}
const stream = new ReadableStream<Uint8Array>({
start(controller) {
push = (text) => controller.enqueue(new TextEncoder().encode(text))
close = () => controller.close()
}
})
const t = await start(() => new Response(stream, { status: 200, headers: { 'content-type': 'text/event-stream' } }))
const pending = t.call('POST', '/backend-api/codex/responses', { 'content-type': 'application/json' }, '{}')
// The first event is out before the second exists.
await new Promise((r) => setTimeout(r, 20))
push('event: a\ndata: 1\n\n')
await new Promise((r) => setTimeout(r, 50))
push('event: b\ndata: 2\n\n')
close()
const res = await pending
expect(res.headers['content-type']).toBe('text/event-stream')
expect(res.body).toBe('event: a\ndata: 1\n\nevent: b\ndata: 2\n\n')
expect(res.chunks.length).toBeGreaterThanOrEqual(2)
})
it('logs method, a two-segment path and the status only: never headers, bodies or queries', async () => {
const t = await start(() => new Response('secret-body'))
await t.call('GET', '/backend-api/codex/models?client_version=0.166.0', { authorization: 'Bearer real-token' })
const text = JSON.stringify(t.logs)
expect(t.logs.at(-1)).toMatchObject({ event: 'relay', method: 'GET', path: '/backend-api/codex', status: 200 })
for (const leak of ['real-token', 'secret-body', 'client_version']) {
expect(text).not.toContain(leak)
}
})
it('refuses WebSocket upgrades and answers 502 when chatgpt.com is unreachable', async () => {
const t = await start(() => {
throw new TypeError('fetch failed')
})
expect((await t.call('GET', '/backend-api/codex/responses', { upgrade: 'websocket', connection: 'Upgrade' })).status).toBe(426)
expect((await t.call('GET', '/backend-api/wham/usage')).status).toBe(502)
})
})
+21
View File
@@ -0,0 +1,21 @@
{
"extends": "../../tsconfig.base.json",
"compilerOptions": {
"noEmit": true,
"allowImportingTsExtensions": true,
"erasableSyntaxOnly": true,
"types": [
"node"
],
"rootDir": "..",
"lib": [
"ES2024",
"DOM"
]
},
"include": [
"src",
"test",
"../sandbox-control/src/codex-paths.ts"
]
}
@@ -0,0 +1,7 @@
import { defineConfig } from 'vitest/config'
export default defineConfig({
// A string tsconfig skips the lookup, which otherwise climbs to the desktop app's tsconfig.
oxc: { tsconfig: false },
test: { include: ['test/**/*.test.ts'] }
})
@@ -0,0 +1,41 @@
# syntax=docker/dockerfile:1.7
# Boot-twice harness image: the golden image's provisioning, run in a container instead of a Vercel
# build sandbox. boot-twice.ts writes the build context (this file, provision-golden.sh and
# provision.env, the same env build-golden-image.ts passes) and then boots the result twice.
# Base: Ubuntu 26.04 (what vercel/sandbox/node:24 is) with Node 24 and passwordless sudo for `ubuntu`, which is what the provision
# script expects from the managed image it normally runs on.
FROM node:24-bookworm-slim@sha256:dc26ae92c9fb15cf6b1d63e20f0944b2efe0f115b20e00be48fb23fceb061c54 AS node
FROM ubuntu:26.04@sha256:065aa8626841f16057b78b027b464cd59c4fac36c204c72dcbf48d66f5fa8a96 AS base
COPY --from=node /usr/local/bin/node /usr/local/bin/node
COPY --from=node /usr/local/lib/node_modules /usr/local/lib/node_modules
RUN ln -s ../lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm \
&& ln -s ../lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx \
&& apt-get update -qq \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends ca-certificates sudo >/dev/null \
&& rm -rf /var/lib/apt/lists/* \
&& printf 'ubuntu ALL=(ALL) NOPASSWD:ALL\n' >/etc/sudoers.d/ubuntu && chmod 0440 /etc/sudoers.d/ubuntu
# The golden build: provision, then wipe identity, exactly as buildGoldenImage runs them.
FROM base AS golden
USER ubuntu
WORKDIR /home/ubuntu
COPY --chown=ubuntu:ubuntu provision-golden.sh provision.env /tmp/golden/
RUN set -a && . /tmp/golden/provision.env && set +a \
&& { bash /tmp/golden/provision-golden.sh >/tmp/golden/provision.out || { tail -n 40 /tmp/golden/provision.out; exit 1; }; } \
&& sudo install -D -m 0644 /tmp/golden/provision.out /var/lib/orca-boot-twice/provision-report.json \
&& /opt/orca/bin/orca-image-wipe
# Harness scaffolding only (outside /opt/orca): the fake per-boot kernel boot id, and a local repo
# for the workspace prelude to clone so the first boot needs no GitHub.
FROM golden AS harness
USER root
COPY boot-twice-init /usr/local/sbin/boot-twice-init
RUN chmod 0755 /usr/local/sbin/boot-twice-init \
&& git init -q --bare -b main /srv/fixture.git \
&& tmp="$(mktemp -d)" && git -C "$tmp" init -q -b main \
&& printf 'boot twice fixture\n' >"$tmp/README.md" && git -C "$tmp" add README.md \
&& git -C "$tmp" -c user.name=fixture -c user.email=fixture@orca.invalid commit -q -m fixture \
&& git -C "$tmp" push -q /srv/fixture.git main && rm -rf "$tmp" \
&& chown -R ubuntu:ubuntu /srv/fixture.git
CMD ["/usr/local/sbin/boot-twice-init"]
@@ -0,0 +1,78 @@
# Boot-twice check
This check catches the case where the image boots but a wake reinstalls or refetches things.
It provisions the golden image in a container, using the real `provision-golden.sh` and the same
env that `build-golden-image.ts` passes. Then it boots the same disk twice through the control
plane's own boot code:
1. `ENTRY_POINTS_PROBE` plus the install when the image's scripts are behind (the `refreshRuntime` step).
2. `startOrcad`, with the workspace prelude on the first boot only.
3. `ensureEntryPoints`, the once-per-image entry-point sync.
Each container start counts as one VM boot. The container gets a fresh kernel boot id, a tmpfs
`/run`, and the disk the previous boot left (`docker commit`).
The second boot must:
- **Come up.** It needs a pairing offer, and orcad must be alive and listening on `:6768`.
- **Install nothing.** `docker diff` against the first boot's disk may only touch boot state:
`~/.orca-sandbox`, `~/.orcad-data`, the per-boot agent config, the compile cache, `/tmp` and
`/run`. See `bootStatePaths` in `boot-twice-checks.ts`. Any `node_modules`, or any change under
`/opt/orca/bin`, `/usr`, `/etc` or a package cache, fails the check.
- **Fetch nothing.** The boot runs on an internal Docker network (no route out). Its only DNS
server is a sink that logs every name it is asked for. A lookup fails the check unless it is in
`RUNTIME_LOOKUPS`. Today that is only orcad's agent-state-rules refresh on github.com.
- **Redo nothing.** The container's init samples every process's command line every 50 ms. Any
installer, fetch or first-run command (apt, npm install, curl, git fetch, `opencode models`)
fails the check.
- **Skip the entry-point install.** No `installEntryPoints` call, and the first boot's
`.entry-points-<hash>` marker is present.
- **Carry the right files.** Every `/opt/orca/bin` entry point must be byte-identical to `image/vm/`,
`.scripts-stamp` must equal `VM_SCRIPTS_STAMP`, and every manifest pin must match the installed
version. The npm packages are compared exactly, by their `package.json`. gh, cursor-agent and
Chrome are compared by `--version`.
## Run it locally
You need Docker that can run `--privileged` containers: Linux, or Docker Desktop on macOS.
```sh
cd cloud
pnpm install --filter @orca-cloud/sandbox-control...
pnpm --filter @orca-cloud/sandbox-control exec tsx image/boot-twice/boot-twice.ts --release latest
```
A cold run takes about 4 minutes, almost all of it provisioning (apt, agent CLIs, Chrome).
Docker's layer cache keys on the provisioning inputs: `provision-golden.sh`, the manifest and the
entry points. A rerun with those unchanged takes about 15 seconds. The command exits non-zero and
lists each failure.
The options:
- `--report <file>` writes the full JSON report: boot timings, every command, the disk diff, lookups and versions.
- `--keep` leaves the two containers, the disk image and the DNS sink for inspection.
- `--release vX.Y.Z` boots a specific orca-server release bundle. The default is the latest
release, because the manifest pins no bundle: production goldens build orcad from source.
- `--image <tag>` boots an image you built yourself. `--context-out <dir>` writes the build context for one.
What this check does not cover:
- The runtime-bundle refresh: no bundle is published here.
- The Vercel firewall and egress proxy: that is the release probe, `scripts/egress-probe.ts`.
- Vercel's own VM behaviour, such as snapshot restore timing.
## CI
On nwparker/orca-private, where Actions is off, the premerge gate
(`config/scripts/cloud-integration/premerge-check.sh`) runs
`config/scripts/cloud-integration/boot-twice-check.sh` whenever a branch touches
`cloud/apps/sandbox-control/image/**`, which includes every VM script. The step:
- takes its own omarchy slot, waiting at most 5 minutes for one;
- stops after 10 minutes;
- prints each failure;
- **fails open with a loud warning when Docker is unavailable**.
`.github/workflows/cloud-vm-boot-twice.yml` runs the same check on pull requests once Actions is
on, with the provisioned layer cached in the GitHub Actions cache.
@@ -0,0 +1,111 @@
import { describe, expect, it } from 'vitest'
import { MANIFEST } from '../vm-scripts.generated'
import { entryPointMismatches, installerCommands, parseDockerDiff, pinnedTools, unexpectedChanges, versionMismatches } from './boot-twice-checks'
const HOME = '/home/ubuntu'
describe('unexpectedChanges', () => {
it('allows boot state, per-boot config and the parent directories they live in', () => {
const diff = parseDockerDiff(
[
'C /home',
'C /home/ubuntu',
'C /home/ubuntu/.orca-sandbox',
'A /home/ubuntu/.orca-sandbox/opencode-first-run',
'C /home/ubuntu/.orcad-data/agent-hooks/endpoint.env',
'C /home/ubuntu/.codex/config.toml',
'C /opt',
'C /opt/orca',
'C /opt/orca/compile-cache/v24-x64/712488c2',
'A /tmp/x',
'C /run'
].join('\n')
)
expect(unexpectedChanges(diff, HOME)).toEqual([])
})
it('flags installs: image files, package caches and node_modules anywhere', () => {
const diff = parseDockerDiff(
[
'C /opt/orca/bin/orca-sandbox-start',
'A /opt/orca/tools/lib/node_modules/x/package.json',
'A /home/ubuntu/.npm/_cacache/index',
'A /home/ubuntu/.config/opencode/node_modules/@opencode-ai/plugin',
'A /home/ubuntu/.orcad-data/node_modules/x',
'C /usr/lib/x',
'C /etc/apt/sources.list.d/ubuntu.sources',
'C /home/ubuntu/.config/opencode'
].join('\n')
)
expect(unexpectedChanges(diff, HOME).map((entry) => entry.path)).toEqual([
'/opt/orca/bin/orca-sandbox-start',
'/opt/orca/tools/lib/node_modules/x/package.json',
'/home/ubuntu/.npm/_cacache/index',
'/home/ubuntu/.config/opencode/node_modules/@opencode-ai/plugin',
'/home/ubuntu/.orcad-data/node_modules/x',
'/usr/lib/x',
'/etc/apt/sources.list.d/ubuntu.sources'
])
})
it('does not take an added directory above an allowed path as a parent change', () => {
expect(unexpectedChanges([{ kind: 'A', path: '/opt/orca' }], HOME)).toHaveLength(1)
})
})
describe('pinned versions', () => {
const tools = pinnedTools(MANIFEST)
it('covers every agent CLI package and the pinned binaries', () => {
expect(tools.filter((tool) => tool.source === 'package')).toHaveLength(MANIFEST.agentCliPackages.length)
expect(tools.map((tool) => tool.name)).toEqual(expect.arrayContaining(['@anthropic-ai/claude-code', 'gh', 'google-chrome']))
})
it('matches exact package versions and version lines that contain the pin', () => {
const observed: Record<string, string | null> = {}
for (const tool of tools) {
observed[tool.name] = tool.source === 'package' ? tool.expect : `prefix ${tool.expect} (suffix)`
}
expect(versionMismatches(tools, observed)).toEqual([])
const claude = tools.find((tool) => tool.name === '@anthropic-ai/claude-code')!
expect(versionMismatches([claude], { [claude.name]: '0.0.1' })).toEqual([`${claude.name}: "0.0.1" does not match the pin ${claude.expect}`])
expect(versionMismatches([claude], {})).toEqual([`${claude.name}: missing (pinned ${claude.expect})`])
})
it("reads Chrome's .deb revision off the version it reports", () => {
const chrome = tools.find((tool) => tool.name === 'google-chrome')!
expect(chrome.expect).not.toMatch(/-\d+$/)
})
})
describe('entryPointMismatches', () => {
it('names missing and changed entry points', () => {
expect(entryPointMismatches({ a: '1', b: '2', c: '3' }, { a: '1', b: '9' })).toEqual([
'b: differs from image/vm/b',
'c: missing from /opt/orca/bin'
])
})
})
describe('installerCommands', () => {
it('finds installs, fetches and first-run work among sampled command lines', () => {
expect(
installerCommands([
'bash -c timeout 120 "$1" models && : >"$2" opencode-first-run /opt/orca/tools/bin/opencode /home/ubuntu/.orca-sandbox/opencode-first-run',
'timeout 120 /opt/orca/tools/bin/opencode models ',
'/opt/orca/runtime/runtimes/node-x/bin/node /opt/orca/bin/orcad-boot --code-cache x',
'curl -fsS --max-time 25 -o /tmp/b.tgz https://example.invalid/b.tgz',
'git -C /vercel/sandbox/project fetch --progress origin',
'git fetch origin',
'sleep 0.05',
'tr \\0 ',
''
])
).toEqual([
'timeout 120 /opt/orca/tools/bin/opencode models',
'curl -fsS --max-time 25 -o /tmp/b.tgz https://example.invalid/b.tgz',
'git -C /vercel/sandbox/project fetch --progress origin',
'git fetch origin'
])
})
})
@@ -0,0 +1,125 @@
// Pure checks for the boot-twice harness (boot-twice.ts): what a second boot of the same disk may
// change, and whether the image carries the pinned tools and this tree's entry points.
import type { GoldenImageManifest } from '../golden-image'
export type DiffEntry = { kind: 'A' | 'C' | 'D'; path: string }
/** `docker diff` output: one `<A|C|D> <path>` line per changed path. */
export function parseDockerDiff(text: string): DiffEntry[] {
const entries: DiffEntry[] = []
for (const line of text.split('\n')) {
const match = /^([ACD]) (\/.*)$/.exec(line.trim())
if (match) {
entries.push({ kind: match[1] as DiffEntry['kind'], path: match[2]! })
}
}
return entries
}
/**
* What any boot may write: runtime state, logs and the per-boot config the start script rewrites.
* Everything else a second boot touches (/opt, /usr, /etc, package caches, node_modules) is an
* install that should have happened once, in the image or on the first boot.
*/
export function bootStatePaths(home: string): readonly string[] {
return [
// orca-sandbox-start: boot id, pids, ready file, logs, start lock, git-setup marks.
`${home}/.orca-sandbox`,
// orcad's own data directory (device identity, sessions, logs).
`${home}/.orcad-data`,
// The start script's git identity fallbacks (written only when missing or on an old default).
`${home}/.gitconfig`,
// orca-sandbox-env rewrites Codex's managed config block and OpenCode's managed provider list on every start.
`${home}/.codex/config.toml`,
`${home}/.config/opencode/.orca-managed-disabled-providers`,
// orcad reads OpenCode's session database at start; SQLite readers recreate its WAL and index files.
`${home}/.local/share/opencode/opencode.db-shm`,
`${home}/.local/share/opencode/opencode.db-wal`,
// Node's compile cache: orcad's node adds or refreshes entries for code it compiles. A cache, not an install.
'/opt/orca/compile-cache',
'/tmp',
'/var/tmp',
'/run'
]
}
function under(path: string, prefix: string): boolean {
return path === prefix || path.startsWith(`${prefix}/`)
}
/** Changes outside bootStatePaths. A changed directory that only contains allowed paths is fine. */
export function unexpectedChanges(entries: readonly DiffEntry[], home: string): DiffEntry[] {
const allowed = bootStatePaths(home)
return entries.filter(({ kind, path }) => {
if (/(^|\/)node_modules(\/|$)/.test(path)) {
return true
}
if (allowed.some((prefix) => under(path, prefix))) {
return false
}
// `C /home/ubuntu`: a parent directory of an allowed path changed its mtime.
return !(kind === 'C' && allowed.some((prefix) => prefix.startsWith(`${path}/`)))
})
}
/** One pinned tool: where its version is read in the image and the string it must contain. */
export type PinnedTool = { name: string; expect: string; source: 'package' | 'binary' }
/** Every version the manifest pins that the image can report. */
export function pinnedTools(manifest: GoldenImageManifest): PinnedTool[] {
const tools: PinnedTool[] = manifest.agentCliPackages.map((spec) => {
const at = spec.lastIndexOf('@')
return { name: spec.slice(0, at), expect: spec.slice(at + 1), source: 'package' }
})
if (manifest.gh.version) {
tools.push({ name: 'gh', expect: `gh version ${manifest.gh.version}`, source: 'binary' })
}
if (manifest.cursorAgent) {
tools.push({ name: 'cursor-agent', expect: manifest.cursorAgent.version, source: 'binary' })
}
if (manifest.chrome) {
// The .deb's "157.0.8103.38-1" reports itself as "Google Chrome 157.0.8103.38".
tools.push({ name: 'google-chrome', expect: manifest.chrome.version.replace(/-\d+$/, ''), source: 'binary' })
}
return tools
}
/** Observed versions: npm package.json `version` by package name, else a binary's `--version` line. */
export type ObservedVersions = Record<string, string | null>
export function versionMismatches(tools: readonly PinnedTool[], observed: ObservedVersions): string[] {
const problems: string[] = []
for (const tool of tools) {
const seen = observed[tool.name] ?? null
if (seen === null) {
problems.push(`${tool.name}: missing (pinned ${tool.expect})`)
} else if (tool.source === 'package' ? seen !== tool.expect : !seen.includes(tool.expect)) {
problems.push(`${tool.name}: ${JSON.stringify(seen)} does not match the pin ${tool.expect}`)
}
}
return problems
}
/** Entry points in the image whose bytes differ from this tree's image/vm sources. */
export function entryPointMismatches(expected: Record<string, string>, installed: Record<string, string | null>): string[] {
return Object.entries(expected)
.filter(([name, sha]) => installed[name] !== sha)
.map(([name]) => (installed[name] ? `${name}: differs from image/vm/${name}` : `${name}: missing from /opt/orca/bin`))
}
/**
* Commands that install, fetch or redo first-run work. Any of them in a second boot is a step that
* should have run once. Offline they fail fast and quietly, so the command line is the evidence.
*/
const INSTALLERS = [
/(^|\/)(apt|apt-get|dpkg|pip3?|curl|wget)( |$)/,
/(^|\/)npm (install|i|ci|add) /,
/(^|\/)git( -[Cc] \S+)* (clone|fetch|pull)( |$)/,
/(^|\/)opencode models( |$)/,
/(^|\/)tar (-?x|--extract)/
]
/** The installer command lines among a boot's sampled `/proc/<pid>/cmdline`s. */
export function installerCommands(commands: readonly string[]): string[] {
return [...new Set(commands.map((line) => line.trim()).filter((line) => line && INSTALLERS.some((pattern) => pattern.test(line))))]
}
@@ -0,0 +1,31 @@
#!/bin/sh
# PID 1's child (under `docker run --init`) in a boot-twice container. Each container start is one
# VM boot: a fresh kernel boot id, as a restored Vercel VM gets. A container otherwise sees the
# host's boot id on every start, and the start script would take a resumed disk for the same boot.
set -eu
cat /proc/sys/kernel/random/uuid >/run/boot-twice-boot-id
mount --bind /run/boot-twice-boot-id /proc/sys/kernel/random/boot_id
# Boot 2 resolves through the harness's DNS sink, which logs every name looked up (an offline boot
# that tries to fetch something fails quietly; the lookup is what shows it).
if [ -n "${BOOT_TWICE_DNS:-}" ]; then
printf 'nameserver %s\noptions attempts:1 timeout:1\n' "$BOOT_TWICE_DNS" >/run/boot-twice-resolv.conf
mount --bind /run/boot-twice-resolv.conf /etc/resolv.conf
fi
if [ -n "${BOOT_TWICE_TRACE:-}" ]; then
# Every 50 ms, the command line of each process not seen yet: a boot's installs and first-run work
# last far longer than that.
(
seen=' '
while :; do
for dir in /proc/[0-9]*; do
pid="${dir#/proc/}"
case "$seen" in *" $pid "*) continue ;; esac
seen="$seen$pid "
{ tr '\0' ' ' <"$dir/cmdline"; echo; } >>/run/boot-twice-procs.log 2>/dev/null || true
done
sleep 0.05
done
) &
fi
: >/run/boot-twice-ready
exec sleep infinity
@@ -0,0 +1,485 @@
// Boot-twice check for the golden image: provision it in a container, then boot the same disk twice
// through the control plane's own boot steps (entry-point probe, workspace prelude + start script,
// entry-point sync). The second boot runs with no network and must install and fetch nothing.
// pnpm --filter @orca-cloud/sandbox-control exec tsx image/boot-twice/boot-twice.ts [options]
// --release <vX.Y.Z|latest> orcad bundle (default: the manifest's bundleUrl, else the latest release)
// --image <tag> boot an image built elsewhere (CI builds it with the layer cache)
// --context-out <dir> only write the docker build context and exit
// --report <file> write the JSON report there too
// --keep leave the containers and the disk image for inspection
// Needs Docker that can run --privileged containers (Linux, Docker Desktop). See README.md.
import { spawn } from 'node:child_process'
import { createHash, randomBytes } from 'node:crypto'
import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { ENTRY_POINTS_PROBE, ensureEntryPoints, entryPointsBehind, installEntryPoints, parseEntryPointsProbe } from '../../src/golden-images'
import { startOrcad, type OrcadReady } from '../../src/sandbox-vm-commands'
import type { CommandResult, ProviderSandboxState, RunCommandParams, SandboxProviderApi } from '../../src/sandbox-provider'
import { ManifestSchema, VM_SCRIPTS, provisionEnv, resolveReleaseBundle, type BuildInputs, type GoldenImageManifest } from '../golden-image'
import { VM_SCRIPTS_HASH, VM_SCRIPTS_STAMP, WORKSPACE_PRELUDE_SCRIPT } from '../vm-scripts.generated'
import { entryPointMismatches, installerCommands, parseDockerDiff, pinnedTools, unexpectedChanges, versionMismatches, type DiffEntry, type ObservedVersions } from './boot-twice-checks'
const here = (path: string): string => fileURLToPath(new URL(path, import.meta.url).href)
const HOME = '/home/ubuntu'
const FIXTURE_REPO = 'file:///srv/fixture.git'
const INSTALL_MARK = 'echo synced'
type Options = { release?: string; image?: string; contextOut?: string; report?: string; keep: boolean }
function parseArgs(argv: string[]): Options {
const value = (flag: string): string | undefined => {
const at = argv.indexOf(flag)
return at === -1 ? undefined : argv[at + 1]
}
return { release: value('--release'), image: value('--image'), contextOut: value('--context-out'), report: value('--report'), keep: argv.includes('--keep') }
}
type Run = { code: number; stdout: string; stderr: string }
function docker(args: string[], options: { input?: string; timeoutMs?: number } = {}): Promise<Run> {
return new Promise((resolve, reject) => {
const child = spawn('docker', args, { stdio: ['pipe', 'pipe', 'pipe'] })
let stdout = ''
let stderr = ''
child.stdout.on('data', (chunk: Buffer) => (stdout += chunk.toString()))
child.stderr.on('data', (chunk: Buffer) => (stderr += chunk.toString()))
const timer = options.timeoutMs ? setTimeout(() => child.kill('SIGKILL'), options.timeoutMs) : null
child.on('error', reject)
child.on('close', (code) => {
if (timer) {
clearTimeout(timer)
}
resolve({ code: code ?? 124, stdout, stderr })
})
child.stdin.end(options.input ?? '')
})
}
async function dockerOk(args: string[], options: { input?: string; timeoutMs?: number } = {}): Promise<string> {
const run = await docker(args, options)
if (run.code !== 0) {
throw new Error(`docker ${args[0]} failed (${run.code}): ${run.stderr.slice(-1500)}`)
}
return run.stdout
}
/** Docker in buildx/build output mode: stream it, it takes minutes on a cold cache. */
function dockerStreamed(args: string[]): Promise<void> {
return new Promise((resolve, reject) => {
const child = spawn('docker', args, { stdio: ['ignore', 'inherit', 'inherit'] })
child.on('error', reject)
child.on('close', (code) => (code === 0 ? resolve() : reject(new Error(`docker ${args[0]} failed (${code})`))))
})
}
// Env arrives on stdin as NUL-separated NAME=value pairs: values can be multi-line (the prelude
// script) and large (every entry point, base64), which argv `-e` handles badly.
const ENV_LOADER = 'while IFS= read -r -d "" kv; do export "$kv"; done; exec "$@"'
type CommandRecord = { boot: number; command: string; exitCode: number; ms: number; installed: boolean }
/** Just enough of the provider for the boot steps: runCommand and readFile, as `docker exec`. */
function dockerProvider(boot: () => number, commands: CommandRecord[]): SandboxProviderApi {
const unsupported = (name: string) => () => Promise.reject(new Error(`boot-twice provider does not emulate ${name}`))
return {
async runCommand(container: string, params: RunCommandParams): Promise<CommandResult> {
const started = Date.now()
const input = Object.entries(params.env ?? {})
.map(([key, val]) => `${key}=${val}\0`)
.join('')
const user = params.sudo ? 'root' : 'ubuntu'
const run = await docker(
['exec', '-i', '-u', user, '-w', params.cwd ?? HOME, container, 'bash', '-c', ENV_LOADER, 'boot-twice-env', params.command, ...(params.args ?? [])],
{ input, timeoutMs: params.timeoutMs }
)
const script = params.args?.at(-1) ?? ''
commands.push({
boot: boot(),
command: params.command === 'bash' ? `bash -c ${JSON.stringify(script.split('\n')[0]!.slice(0, 80))}` : params.command,
exitCode: run.code,
ms: Date.now() - started,
installed: script.includes(INSTALL_MARK)
})
return { exitCode: run.code, stdout: run.stdout, stderr: run.stderr }
},
async readFile(container: string, path: string): Promise<string | null> {
const run = await docker(['exec', '-u', 'ubuntu', container, 'cat', '--', path])
return run.code === 0 ? run.stdout : null
},
createSandbox: unsupported('createSandbox'),
getSandbox: unsupported('getSandbox'),
stopSession: unsupported('stopSession'),
extendTimeout: unsupported('extendTimeout'),
updateNetworkPolicy: unsupported('updateNetworkPolicy'),
updatePorts: unsupported('updatePorts'),
updateSnapshotExpiration: unsupported('updateSnapshotExpiration'),
deleteSandbox: unsupported('deleteSandbox'),
listSnapshots: unsupported('listSnapshots'),
deleteSnapshot: unsupported('deleteSnapshot'),
listSandboxes: unsupported('listSandboxes'),
listProjectSnapshots: unsupported('listProjectSnapshots')
}
}
async function latestReleaseTag(): Promise<string> {
const headers: Record<string, string> = { accept: 'application/vnd.github+json', 'user-agent': 'orca-boot-twice' }
if (process.env.GITHUB_TOKEN) {
headers.authorization = `Bearer ${process.env.GITHUB_TOKEN}`
}
const response = await fetch('https://api.github.com/repos/stablyai/orca/releases/latest', { headers })
if (!response.ok) {
throw new Error(`latest release lookup failed (HTTP ${response.status}); pass --release vX.Y.Z`)
}
return ((await response.json()) as { tag_name: string }).tag_name
}
function loadInputs(): BuildInputs {
const manifest = ManifestSchema.parse(JSON.parse(readFileSync(here('../golden-image.manifest.json'), 'utf8')))
const vmScripts = Object.fromEntries(VM_SCRIPTS.map((name) => [name, readFileSync(here(`../vm/${name}`), 'utf8')])) as BuildInputs['vmScripts']
return { manifest, provisionScript: readFileSync(here('../provision-golden.sh'), 'utf8'), vmScripts, vmScriptsStamp: VM_SCRIPTS_STAMP, buildId: 'boot-twice' }
}
async function withBundle(inputs: BuildInputs, release: string | undefined): Promise<BuildInputs> {
if (!release && inputs.manifest.orcad.bundleUrl && inputs.manifest.orcad.sha256) {
return inputs
}
const tag = !release || release === 'latest' ? await latestReleaseTag() : release
const pinned = await resolveReleaseBundle(tag)
return { ...inputs, manifest: { ...inputs.manifest, orcad: { bundleUrl: pinned.bundleUrl, sha256: pinned.sha256 } } }
}
/** The docker build context; its bytes are the layer cache key, so nothing in it varies per run. */
function writeContext(dir: string, inputs: BuildInputs): void {
mkdirSync(dir, { recursive: true })
const lines = Object.entries(provisionEnv(inputs)).map(([key, val]) => {
if (val.includes("'")) {
throw new Error(`provision env ${key} has a quote`)
}
return `${key}='${val}'`
})
writeFileSync(join(dir, 'provision.env'), `${lines.join('\n')}\n`)
copyFileSync(here('../provision-golden.sh'), join(dir, 'provision-golden.sh'))
copyFileSync(here('./Dockerfile'), join(dir, 'Dockerfile'))
copyFileSync(here('./boot-twice-init'), join(dir, 'boot-twice-init'))
}
type BootReport = {
boot: number
network: boolean
ms: number
steps: Record<string, number>
probe: { stamp: number; marker: boolean } | null
ready: OrcadReady | null
error?: string
}
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))
async function waitFor(container: string, test: string, timeoutMs: number): Promise<boolean> {
const deadline = Date.now() + timeoutMs
while (Date.now() < deadline) {
if ((await docker(['exec', container, 'sh', '-c', test])).code === 0) {
return true
}
await sleep(250)
}
return false
}
// Quiet: no first-run work still running, and nothing outside the boot state paths written in the
// last 3 s. orcad installs agent hooks and OpenCode's first run ends a few seconds after ready.
const QUIET = [
'pgrep -f "opencode [m]odels|agent-[p]refetch" >/dev/null && exit 1',
`find ${HOME} /opt/orca -xdev \\( -path ${HOME}/.orca-sandbox -o -path ${HOME}/.orcad-data \\) -prune -o -newermt "-3 seconds" -print | grep -q . && exit 1`,
'exit 0'
].join('\n')
async function settle(container: string): Promise<void> {
await sleep(3_000)
await waitFor(container, QUIET, 150_000)
}
// The DNS sink: answers every query NXDOMAIN and prints each name once ("lookup <name>").
const DNS_SINK = `
const seen = new Set()
const socket = require('node:dgram').createSocket('udp4')
socket.on('message', (msg, peer) => {
if (msg.length < 12) return
const labels = []
let at = 12
while (at < msg.length && msg[at] !== 0) { labels.push(msg.subarray(at + 1, at + 1 + msg[at]).toString()); at += msg[at] + 1 }
const name = labels.join('.').toLowerCase()
if (!seen.has(name)) { seen.add(name); console.log('lookup ' + name) }
const reply = Buffer.from(msg.subarray(0, at + 5))
reply[2] = 0x81; reply[3] = 0x83; reply.writeUInt16BE(0, 6); reply.writeUInt16BE(0, 8); reply.writeUInt16BE(0, 10)
socket.send(reply, peer.port, peer.address)
})
socket.bind(53, () => console.log('sink ready'))
`
/** Names the harness itself makes up; anything else boot 2 looks up is a fetch attempt. */
const LOCAL_NAMES = /(^|\.)(invalid|localhost|local)$|^boot-twice(-\d+)?$/
type Sink = { network: string; ip: string; container: string }
async function startSink(image: string, runId: string): Promise<Sink> {
const network = `orca-boot-twice-${runId}`
const container = `orca-boot-twice-${runId}-dns`
await dockerOk(['network', 'create', '--internal', network])
await dockerOk(['run', '-d', '--name', container, '--network', network, '--user', 'root', image, 'node', '-e', DNS_SINK])
const ip = (await dockerOk(['inspect', '-f', `{{(index .NetworkSettings.Networks "${network}").IPAddress}}`, container])).trim()
const deadline = Date.now() + 15_000
while (!(await docker(['logs', container])).stdout.includes('sink ready')) {
if (Date.now() > deadline) {
throw new Error('DNS sink did not start')
}
await sleep(200)
}
return { network, ip, container }
}
/**
* Lookups every orcad start makes, by design: reported, not failed. orcad refreshes its agent-state
* rules from the orca GitHub releases at start (and keeps working on its cached copy when offline).
* The cost: a git fetch to github.com on boot 2 would hide behind this; the disk diff still catches
* one that wrote anything.
*/
const RUNTIME_LOOKUPS: Readonly<Record<string, string>> = {
'github.com': "orcad's agent-state-rules refresh at start"
}
async function sinkLookups(sink: Sink): Promise<{ unexpected: string[]; expected: string[] }> {
const out = (await docker(['logs', sink.container])).stdout
const names = [...out.matchAll(/^lookup (.+)$/gm)].map((match) => match[1]!).filter((name) => !LOCAL_NAMES.test(name))
return { unexpected: names.filter((name) => !(name in RUNTIME_LOOKUPS)), expected: names.filter((name) => name in RUNTIME_LOOKUPS) }
}
/** One VM boot: a new container on the disk image, then the steps lifecycle.ts runs on create/wake. */
async function bootOnce(
provider: SandboxProviderApi,
image: string,
container: string,
boot: number,
options: { network: boolean; prelude: boolean; sink?: { network: string; ip: string } }
): Promise<BootReport> {
const steps: Record<string, number> = {}
const time = async <T>(name: string, step: () => Promise<T>): Promise<T> => {
const started = Date.now()
try {
return await step()
} finally {
steps[name] = Date.now() - started
}
}
const report: BootReport = { boot, network: options.network, ms: 0, steps, probe: null, ready: null }
// Offline: an internal network (no route out) whose only resolver is the DNS sink.
const network = options.network ? [] : options.sink ? ['--network', options.sink.network, '--env', `BOOT_TWICE_DNS=${options.sink.ip}`] : ['--network', 'none']
// Every boot records the commands it runs (/run/boot-twice-procs.log, sampled by the init).
network.push('--env', 'BOOT_TWICE_TRACE=1')
await dockerOk(['run', '-d', '--init', '--privileged', '--name', container, '--hostname', `boot-twice-${boot}`, '--tmpfs', '/run', ...network, image])
if (!(await waitFor(container, 'test -e /run/boot-twice-ready', 30_000))) {
throw new Error(`boot ${boot}: container init did not come up`)
}
const started = Date.now()
try {
// refreshRuntime (no runtime bundle published here): the probe, and an install only when behind.
const probe = await time('probe', () => provider.runCommand(container, { command: 'bash', args: ['-c', ENTRY_POINTS_PROBE], sudo: true, timeoutMs: 40_000 }))
report.probe = parseEntryPointsProbe(probe.stdout)
if (entryPointsBehind(report.probe)) {
await time('entryPointsBeforeStart', () => installEntryPoints({ provider }, container))
}
const state: ProviderSandboxState = {
sandbox: { name: container, status: 'running', currentSessionId: container, currentSnapshotId: null, tags: {} },
session: { id: container, status: 'running', startedAt: started, timeoutMs: 3_600_000 },
routes: [{ port: 6768, url: 'https://boot-twice.invalid' }]
}
report.ready = await time('orcad', () =>
startOrcad(provider, state, {
sandboxId: `boot-twice-${boot}`,
hostname: 'boot-twice',
repo: { url: FIXTURE_REPO, ref: 'main' },
docker: true,
prelude: options.prelude ? { script: WORKSPACE_PRELUDE_SCRIPT } : undefined
})
)
// settleMemberGit after ready: the per-boot entry-point check (an install at most once per image).
await time('entryPointsAfterStart', () => ensureEntryPoints({ provider }, container))
} catch (error) {
report.error = error instanceof Error ? error.message : String(error)
}
report.ms = Date.now() - started
return report
}
const VERSIONS_SCRIPT = `
const { execFileSync } = require('node:child_process')
const { readFileSync } = require('node:fs')
const [packages, binaries] = JSON.parse(process.argv[1])
const out = {}
for (const name of packages) {
try { out[name] = JSON.parse(readFileSync('/opt/orca/tools/lib/node_modules/' + name + '/package.json', 'utf8')).version } catch { out[name] = null }
}
for (const name of binaries) {
try {
const text = execFileSync(name, ['--version'], { encoding: 'utf8', timeout: 30000, stdio: ['ignore', 'pipe', 'ignore'] })
out[name] = text.trim().split('\\n')[0] || null
} catch { out[name] = null }
}
console.log(JSON.stringify(out))
`
// What a booted image carries: entry points (sha256), the scripts stamp and markers, orcad's
// version and whether it is listening on 6768 (0x1A70) on all interfaces.
const IMAGE_PROBE = [
'cd /opt/orca/bin',
'for f in * .scripts-stamp .entry-points-*; do [ -f "$f" ] && echo "file $(sha256sum "$f" | cut -d" " -f1) $f"; done',
'echo "stamp $(cat .scripts-stamp 2>/dev/null || echo none)"',
'echo "orcad_version $(cat /opt/orca/runtime/orcad/.version 2>/dev/null || echo none)"',
'kill -0 "$(cat ~/.orca-sandbox/orcad.pid 2>/dev/null || echo 0)" 2>/dev/null && echo orcad_alive=1 || echo orcad_alive=0',
'grep -qi " 00000000:1A70 00000000:0000 0A" /proc/net/tcp && echo orcad_listening=1 || echo orcad_listening=0',
// Installers or fetchers still running after the boot answered.
'echo "busy $(ps -eo args= | grep -E "^(apt|apt-get|dpkg|npm|npx|pip|curl|wget) |opencode [m]odels|git ([c]lone|[f]etch)" | head -3 | tr "\\n" ";")"'
].join('\n')
type Failure = string
async function checkImage(provider: SandboxProviderApi, container: string, manifest: GoldenImageManifest, failures: Failure[]) {
const sources = Object.fromEntries(VM_SCRIPTS.map((name) => [name, createHash('sha256').update(readFileSync(here(`../vm/${name}`))).digest('hex')]))
const probe = await provider.runCommand(container, { command: 'bash', args: ['-c', IMAGE_PROBE], timeoutMs: 30_000 })
const installed: Record<string, string> = {}
for (const match of probe.stdout.matchAll(/^file ([0-9a-f]{64}) (.+)$/gm)) {
installed[match[2]!] = match[1]!
}
failures.push(...entryPointMismatches(sources, installed).map((problem) => `entry point ${problem}`))
const stamp = /^stamp (.+)$/m.exec(probe.stdout)?.[1]
if (stamp !== String(VM_SCRIPTS_STAMP)) {
failures.push(`/opt/orca/bin/.scripts-stamp is ${stamp ?? 'unreadable'}, expected ${VM_SCRIPTS_STAMP}`)
}
if (!(`.entry-points-${VM_SCRIPTS_HASH}` in installed)) {
failures.push(`no .entry-points-${VM_SCRIPTS_HASH} marker after two boots`)
}
if (!/^orcad_alive=1$/m.test(probe.stdout) || !/^orcad_listening=1$/m.test(probe.stdout)) {
failures.push('orcad is not running and listening on :6768 after the second boot')
}
const busy = /^busy (.*)$/m.exec(probe.stdout)?.[1]?.trim()
if (busy) {
failures.push(`still installing or fetching after the second boot: ${busy}`)
}
const tools = pinnedTools(manifest)
const run = await provider.runCommand(container, {
command: 'bash',
args: ['-lc', 'node -e "$1" "$2"', 'versions', VERSIONS_SCRIPT, JSON.stringify([tools.filter((t) => t.source === 'package').map((t) => t.name), tools.filter((t) => t.source === 'binary').map((t) => t.name)])],
timeoutMs: 180_000
})
let observed: ObservedVersions = {}
try {
observed = JSON.parse(run.stdout.trim().split('\n').at(-1) ?? '{}') as ObservedVersions
} catch {
failures.push(`version probe failed (exit ${run.exitCode}): ${run.stderr.slice(-300)}`)
}
failures.push(...versionMismatches(tools, observed))
return { orcadVersion: /^orcad_version (.+)$/m.exec(probe.stdout)?.[1] ?? null, versions: observed, entryPoints: Object.keys(installed).length }
}
async function main(): Promise<void> {
const options = parseArgs(process.argv.slice(2))
const inputs = await withBundle(loadInputs(), options.release)
if (options.contextOut) {
writeContext(options.contextOut, inputs)
console.log(JSON.stringify({ context: options.contextOut, bundleUrl: inputs.manifest.orcad.bundleUrl }))
return
}
const runId = randomBytes(3).toString('hex')
const started = Date.now()
let image = options.image
if (!image) {
const context = mkdtempSync(join(tmpdir(), 'orca-boot-twice-'))
writeContext(context, inputs)
image = 'orca-boot-twice:local'
console.error(`[boot-twice] building ${image} from ${context} (orcad ${inputs.manifest.orcad.bundleUrl})`)
await dockerStreamed(['build', '--target', 'harness', '-t', image, context])
}
const buildMs = Date.now() - started
const disk = `orca-boot-twice-disk:${runId}`
const containers = [`orca-boot-twice-${runId}-1`, `orca-boot-twice-${runId}-2`] as const
const commands: CommandRecord[] = []
let current = 1
const provider = dockerProvider(() => current, commands)
const failures: Failure[] = []
let diff: DiffEntry[] = []
let unexpected: DiffEntry[] = []
let image2: Awaited<ReturnType<typeof checkImage>> | null = null
const boots: BootReport[] = []
let sink: Sink | null = null
let lookups: { unexpected: string[]; expected: string[] } = { unexpected: [], expected: [] }
try {
console.error('[boot-twice] boot 1 (first boot of the image, network on)')
boots.push(await bootOnce(provider, image, containers[0], 1, { network: true, prelude: true }))
// A VM lives a while after ready: let the first boot's background work finish before the disk is kept.
await settle(containers[0])
await dockerOk(['stop', '-t', '20', containers[0]])
await dockerOk(['commit', containers[0], disk])
current = 2
console.error('[boot-twice] boot 2 (same disk, new boot id, no network)')
sink = await startSink(image, runId)
boots.push(await bootOnce(provider, disk, containers[1], 2, { network: false, prelude: false, sink }))
for (const boot of boots) {
if (boot.error || !boot.ready?.pairingCode.startsWith('orca://')) {
failures.push(`boot ${boot.boot}: orcad did not come up (${boot.error ?? 'no pairing offer'})`)
}
}
const reinstalls = commands.filter((command) => command.boot === 2 && command.installed)
if (reinstalls.length > 0) {
failures.push(`boot 2 reinstalled the entry points (${reinstalls.length}x): the first boot's marker did not stick`)
}
if (boots[1]?.probe && !boots[1].probe.marker) {
failures.push('boot 2 probe: no entry-points marker for this tree')
}
// Give the second boot the same time to do (and show) anything it starts in the background.
await settle(containers[1])
const procs = await docker(['exec', containers[1], 'cat', '/run/boot-twice-procs.log'])
failures.push(...installerCommands(procs.stdout.split('\n')).map((command) => `boot 2 ran ${JSON.stringify(command.slice(0, 160))}`))
lookups = await sinkLookups(sink)
failures.push(...lookups.unexpected.map((name) => `boot 2 tried to reach ${name}`))
// The disk diff comes first: the version probe runs the agent CLIs, which write their own caches.
diff = parseDockerDiff(await dockerOk(['diff', containers[1]]))
unexpected = unexpectedChanges(diff, HOME)
failures.push(...unexpected.map((entry) => `boot 2 changed ${entry.kind} ${entry.path}`))
image2 = await checkImage(provider, containers[1], inputs.manifest, failures)
} finally {
if (!options.keep) {
await docker(['rm', '-f', ...containers, ...(sink ? [sink.container] : [])])
await docker(['rmi', '-f', disk])
if (sink) {
await docker(['network', 'rm', sink.network])
}
}
}
const report = {
ok: failures.length === 0,
failures,
image,
orcadBundle: inputs.manifest.orcad.bundleUrl,
buildMs,
boots: boots.map(({ ready, ...rest }) => ({ ...rest, ready: ready ? { projectRoot: ready.projectRoot } : null })),
commands,
boot2: { diff: diff.length, unexpected, lookups, ...image2 },
totalMs: Date.now() - started
}
if (options.report) {
writeFileSync(options.report, `${JSON.stringify(report, null, 2)}\n`)
}
console.log(JSON.stringify(report, null, 2))
for (const boot of boots) {
console.error(`[boot-twice] boot ${boot.boot}: ${boot.ms} ms ${JSON.stringify(boot.steps)}`)
}
console.error(report.ok ? '[boot-twice] OK: the second boot installed nothing' : `[boot-twice] FAILED:\n ${failures.join('\n ')}`)
process.exitCode = report.ok ? 0 : 1
}
main().catch((error: unknown) => {
console.error(error instanceof Error ? error.stack ?? error.message : String(error))
process.exitCode = 1
})
@@ -7,6 +7,7 @@ import type { SandboxProviderApi } from '../src/sandbox-provider'
export const VM_SCRIPTS = [
'orca-sandbox-start',
'orca-sandbox-drain',
'orca-memguard',
'orca-sandbox-env',
'orca-session-env.sh',
'orca-env',
@@ -55,8 +55,11 @@ if [ ! -s "$ORCA_HOME/runtime/orcad/orcad-server.js" ]; then
server="$(find "$ORCA_HOME/runtime" -name orcad-server.js -type f | head -n 1)"
[ -n "$server" ] || { log 'bundle has no orcad-server.js'; exit 1; }
root="$(dirname "$(dirname "$server")")"
ln -sfn "$(dirname "$server")" "$ORCA_HOME/runtime/orcad"
[ -d "$ORCA_HOME/runtime/runtimes" ] || ln -sfn "$root/runtimes" "$ORCA_HOME/runtime/runtimes"
# Move the tree up rather than linking to it: every VM script finds node with
# `find .../runtime/runtimes -type f`, and find doesn't descend into a symlinked start point
# (a linked release bundle left the build with no node). orcad's ../runtimes still resolves.
for entry in "$root"/*; do mv "$entry" "$ORCA_HOME/runtime/"; done
rmdir "$root"
fi
test -s "$ORCA_HOME/runtime/orcad/orcad-server.js"
node="$(find "$ORCA_HOME/runtime/runtimes" -type f -name node | head -n 1)"
@@ -98,6 +101,16 @@ if [ -x "$opencode_pkg/bin/opencode.exe" ]; then
[ "$(stat -c %i "$variant/bin/opencode" 2>/dev/null)" = "$picked" ] || rm -rf "$variant"
done
"$opencode_pkg/bin/opencode.exe" --version >/dev/null
# OpenCode installs @opencode-ai/plugin (its own version) into ~/.config/opencode on every start
# until that dir has node_modules and a lock listing it, and draws nothing until that install ends
# once a plugin is configured (Orca's is): with a slow or unreachable registry the first launch
# sat blank ~85 s (n73). Install it here once; orca-sandbox-start copies it into a fresh home.
oc_seed="$ORCA_HOME/opencode-config-seed"
mkdir -p "$oc_seed"
printf '{\n "dependencies": {\n "@opencode-ai/plugin": "%s"\n }\n}\n' \
"$("$opencode_pkg/bin/opencode.exe" --version)" >"$oc_seed/package.json"
(cd "$oc_seed" && npm install --no-fund --no-audit --ignore-scripts >/dev/null)
test -s "$oc_seed/package-lock.json" && test -d "$oc_seed/node_modules/@opencode-ai/plugin"
fi
# Copilot CLI prefers GH_TOKEN/GITHUB_TOKEN over the user's own /login. In Orca Cloud those hold the
@@ -210,7 +223,7 @@ if [ -n "${NOVNC_VERSION:-}" ]; then
fi
log 'Orca entry points'
for name in orca-sandbox-start orca-sandbox-drain orca-sandbox-env orca-session-env.sh orca-env orca-egress-probe orca-image-wipe orca-agent-signout orca-share-proxy orca-sshd orca-ssh-keys orca-desktop-start orca-desktop-proxy orca-desktop-page.js orca-sandbox-grant orca-sandbox-revoke orca-sandbox-members orca-sandbox-read-snapshot orca-read-snapshot.cjs orca-git-as orca-git-identity orcad-boot git docker; do
for name in orca-sandbox-start orca-sandbox-drain orca-memguard orca-sandbox-env orca-session-env.sh orca-env orca-egress-probe orca-image-wipe orca-agent-signout orca-share-proxy orca-sshd orca-ssh-keys orca-desktop-start orca-desktop-proxy orca-desktop-page.js orca-sandbox-grant orca-sandbox-revoke orca-sandbox-members orca-sandbox-read-snapshot orca-read-snapshot.cjs orca-git-as orca-git-identity orcad-boot git docker; do
var="ORCA_VM_SCRIPT_${name//[-.]/_}"
printf '%s' "${!var}" | base64 -d >"$ORCA_HOME/bin/$name"
chmod 0755 "$ORCA_HOME/bin/$name"
File diff suppressed because one or more lines are too long
+308
View File
@@ -0,0 +1,308 @@
#!/usr/bin/env bash
# Memory guard for the workspace VM (runs as root: `sudo -n orca-memguard start`). A build or test
# that eats the VM's memory must cost that process, never the workspace: without this the kernel
# thrashes for minutes before its OOM killer picks something, and the VM looks dead meanwhile.
# start start the guard for this boot unless it already runs (returns at once)
# run the guard loop itself (start runs it detached)
# release-swap turn off and delete on-demand swap (before a stop, so the snapshot stays small)
# sweep apply OOM-killer priorities once and exit (tests, diagnostics)
# victim print "<pid> <rss KiB> <name>" of the process the guard would stop now (no kill)
# Every few seconds it ranks processes for the kernel's OOM killer (oom_score_adj):
# -900 Orca's own runtime: orcad, its terminal daemon, the share proxy (Orca's bundled Node)
# -500 terminal shells (children of the terminal daemon) and agent CLIs
# 500 everything else (builds, tests, dev servers, browsers); never lowered
# When MemAvailable falls under KILL_PCT (8%) of RAM it kills the largest process ranked 0 or more,
# records it in $RUN/kills and $RUN/state, and waits for memory to return. The kernel's OOM killer
# stays the backstop for allocations faster than the guard's 200 ms look, with the same ranks.
# Swap is added on demand (2 GiB files up to 1.5x RAM) when memory runs low, if the kernel takes it,
# and dropped after 10 quiet minutes. It also sizes /dev/shm to half of RAM (the provider mounts
# 64 MB, which crashes Chrome and Electron renderers).
# $RUN/state (key=value, world-readable) is what the control plane's status probe reads.
set -uo pipefail
RUN=/run/orca-memguard
# Tests point it at a fake process table.
PROC="${ORCA_MEMGUARD_PROC:-/proc}"
SWAP_DIR=/var/lib/orca-memguard
ORCA_HOME=/opt/orca
KILL_PCT="${ORCA_MEMGUARD_KILL_PCT:-8}"
HIGH_PCT=15
SWAP_ADD_PCT=20
QUIET_PCT=50
QUIET_SECONDS=600
TICK=1
FAST_TICK=0.2
SWEEP_EVERY=3
STATE_EVERY=15
SWAP_CHUNK_KIB=$((2 * 1024 * 1024))
boot_id() { local id; read -r id </proc/sys/kernel/random/boot_id && printf '%s' "$id"; }
now_ms() { local t="${EPOCHREALTIME//[.,]/}"; printf '%s' "${t:0:13}"; }
log() { printf '%s %s\n' "$(date -u +%FT%TZ)" "$*" >>"$RUN/memguard.log"; }
# Name of a process for people: the script a node/python/... runs, else its command name.
proc_name() {
local pid="$1" comm arg name='' skip=0 module=0 args=()
read -r comm 2>/dev/null <"$PROC/$pid/comm" || comm=''
mapfile -d '' -t args 2>/dev/null <"$PROC/$pid/cmdline" || true
case "${args[0]##*/}" in
node | nodejs | bun | deno | python* | ruby | perl | java | sh | bash | dash | zsh)
for arg in "${args[@]:1}"; do
if [ "$skip" -eq 1 ]; then skip=0; continue; fi
if [ "$module" -eq 1 ]; then name="${arg##*/}"; break; fi
case "$arg" in
# Inline code is no name (and may hold anything): the interpreter's name it is.
-e | -p | -c | --eval | --print) break ;;
-m | -jar) module=1 ;; # python -m pytest, java -jar gradle.jar: the module or jar
-r | --require | --import | --loader | -X | -W | -cp | -classpath) skip=1 ;;
-*) ;;
*) name="${arg##*/}"; break ;;
esac
done ;;
esac
[ -n "$name" ] || name="${args[0]##*/}"
[ -n "$name" ] || name="$comm"
name="${name%.js}"
name="${name%.mjs}"
name="${name%.cjs}"
name="${name%.jar}"
name="${name//[^A-Za-z0-9._+-]/}"
printf '%s' "${name:0:40}"
}
# The terminal daemon's pid(s): Orca-runtime node processes running the daemon entry.
daemon_pids=' '
# Sets RANK to the oom_score_adj a process should have; returns 1 for a kernel thread or a gone pid.
RANK=500
rank() {
local pid="$1" args=() stat ppid
mapfile -d '' -t -n 3 args 2>/dev/null <"$PROC/$pid/cmdline" || return 1
[ "${#args[@]}" -gt 0 ] || return 1
RANK=-900
case "${args[0]}" in "$ORCA_HOME"/runtime/runtimes/*) return 0 ;; esac
RANK=-500
read -r stat 2>/dev/null <"$PROC/$pid/stat" || return 1
stat="${stat##*) }"
ppid="${stat#* }"
ppid="${ppid%% *}"
case "$daemon_pids" in *" $ppid "*) return 0 ;; esac
case "${args[0]}/${args[1]:-}/${args[2]:-}" in "$ORCA_HOME"/tools/* | */"$ORCA_HOME"/tools/*) return 0 ;; esac
case "${args[0]##*/}" in claude | codex | opencode | gemini | cursor-agent | copilot | amp | droid) return 0 ;; esac
RANK=500
}
sweep() {
local p pid want adj args=()
daemon_pids=' '
for p in "$PROC"/[0-9]*; do
pid="${p#"$PROC"/}"
mapfile -d '' -t -n 3 args 2>/dev/null <"$p/cmdline" || continue
[ "${#args[@]}" -ge 2 ] || continue
case "${args[0]}" in "$ORCA_HOME"/runtime/runtimes/*) ;; *) continue ;; esac
case "${args[*]:1}" in *"$ORCA_HOME"/runtime/orcad/daemon*) daemon_pids+="$pid " ;; esac
done
for p in "$PROC"/[0-9]*; do
pid="${p#"$PROC"/}"
[ "$pid" -gt 1 ] && [ "$pid" != "$$" ] || continue
read -r adj 2>/dev/null <"$p/oom_score_adj" || continue
# -1000 is a process that asked for it as root (sshd): leave it alone.
[ "$adj" -gt -1000 ] || continue
rank "$pid" || continue
want="$RANK"
# Never lower what a process raised for itself (Chrome ranks its renderers above its browser).
if [ "$want" -eq 500 ]; then
[ "$adj" -lt 500 ] || continue
fi
[ "$adj" -ne "$want" ] || continue
printf '%s\n' "$want" 2>/dev/null >"$p/oom_score_adj" || true
done
}
mem_total=0 mem_avail=0 swap_total=0 swap_free=0
read_mem() {
local key value _
while read -r key value _; do
case "$key" in
MemTotal:) mem_total="$value" ;;
MemAvailable:) mem_avail="$value" ;;
SwapTotal:) swap_total="$value" ;;
SwapFree:) swap_free="$value" ;;
esac
done <"$PROC/meminfo"
}
# PSI percentages (avg10, integer part); 0 when the kernel doesn't report them.
psi_some=0 psi_full=0
read_psi() {
local kind avg _
psi_some=0 psi_full=0
[ -r "$PROC/pressure/memory" ] || return 0
while read -r kind avg _; do
avg="${avg#avg10=}"
avg="${avg%%.*}"
case "$kind" in some) psi_some="$avg" ;; full) psi_full="$avg" ;; esac
done 2>/dev/null <"$PROC/pressure/memory" || true
}
kills=0 last_kill_name='' last_kill_at=0 swap_mode=on-demand level=ok
write_state() {
local pct="$1"
{
printf 'bootId=%s\nat=%s\npid=%s\nlevel=%s\navailPct=%s\nmemTotalKiB=%s\nmemAvailKiB=%s\n' \
"$BOOT" "$(now_ms)" "$$" "$level" "$pct" "$mem_total" "$mem_avail"
printf 'swapTotalKiB=%s\nswapFreeKiB=%s\nswap=%s\npsiSome10=%s\npsiFull10=%s\n' \
"$swap_total" "$swap_free" "$swap_mode" "$psi_some" "$psi_full"
printf 'kills=%s\nlastKillName=%s\nlastKillAt=%s\n' "$kills" "$last_kill_name" "$last_kill_at"
} >"$RUN/state.tmp" && mv -f "$RUN/state.tmp" "$RUN/state"
}
# Sets VICTIM/VICTIM_RSS (pages) to the largest process ranked 0 or more; returns 1 when there is none.
VICTIM=0 VICTIM_RSS=0
choose_victim() {
local p pid adj rss _ best=0 best_rss=0 args=()
sweep
for p in "$PROC"/[0-9]*; do
pid="${p#"$PROC"/}"
[ "$pid" -gt 1 ] && [ "$pid" != "$$" ] || continue
read -r adj 2>/dev/null <"$p/oom_score_adj" || continue
[ "$adj" -ge 0 ] || continue
read -r _ rss _ 2>/dev/null <"$p/statm" || continue
# Kernel threads have no command line (and /proc sizes read 0, so `-s` can't tell).
mapfile -d '' -t -n 1 args 2>/dev/null <"$p/cmdline" || continue
[ "${#args[@]}" -gt 0 ] || continue
if [ "$rss" -gt "$best_rss" ]; then
best="$pid" best_rss="$rss"
fi
done
# Nothing big enough to matter (64 MiB): what fills memory is protected, so leave it to the kernel.
[ "$best" -gt 1 ] && [ "$best_rss" -ge 16384 ] || return 1
VICTIM="$best" VICTIM_RSS="$best_rss"
}
# Kills the largest process ranked 0 or more ($1: MemAvailable %). Returns 1 when there is none.
kill_largest() {
choose_victim || return 1
local best="$VICTIM" best_rss="$VICTIM_RSS" name at
name="$(proc_name "$best")"
kill -KILL "$best" 2>/dev/null || return 1
at="$(now_ms)"
kills=$((kills + 1)) last_kill_name="${name:-process}" last_kill_at="$at"
printf '%s %s %s %s %s\n' "$at" "$best" "$((best_rss * 4))" "$1" "$last_kill_name" >>"$RUN/kills"
tail -n 50 "$RUN/kills" >"$RUN/kills.tmp" && mv -f "$RUN/kills.tmp" "$RUN/kills"
log "killed pid=$best name=$last_kill_name rssKiB=$((best_rss * 4)) availPct=$1"
# Shells hear it like a kernel OOM kill (exit 137); this says why where someone may look.
logger -t orca-memguard "stopped $last_kill_name (pid $best, $((best_rss * 4)) KiB) at $1% memory available" 2>/dev/null || true
return 0
}
swap_files() { compgen -G "$SWAP_DIR/swap-*" || true; }
add_swap() {
[ "$swap_mode" = on-demand ] || return 1
[ $((swap_total + SWAP_CHUNK_KIB)) -le $((mem_total * 3 / 2)) ] || return 1
local free_kib file
free_kib="$(df -k --output=avail "$SWAP_DIR" 2>/dev/null | tail -n 1 | tr -d ' ')"
# Keep 4 GiB of the disk for the workspace itself.
[ "${free_kib:-0}" -gt $((SWAP_CHUNK_KIB + 4 * 1024 * 1024)) ] || return 1
file="$SWAP_DIR/swap-$(now_ms)"
if fallocate -l "$((SWAP_CHUNK_KIB))K" "$file" 2>/dev/null && chmod 600 "$file" \
&& mkswap -q "$file" >/dev/null 2>&1 && swapon "$file" 2>/dev/null; then
log "swap on $file"
return 0
fi
rm -f "$file"
# One refusal means this kernel or disk won't swap: don't retry every tick.
swap_mode=unsupported
log 'swap unavailable; guard kills only'
return 1
}
release_swap() {
local file
for file in $(swap_files | sort -r); do
swapoff "$file" 2>/dev/null && rm -f "$file" && log "swap off $file"
done
}
run() {
BOOT="$(boot_id)"
mkdir -p "$RUN" "$SWAP_DIR"
chmod 0755 "$RUN"
chmod 0700 "$SWAP_DIR"
# A resumed disk carries the last session's files: its kills and swap belong to that boot.
rm -f "$RUN/kills" "$RUN/state" "$RUN"/*.tmp
for file in $(swap_files); do
grep -q "^$file " /proc/swaps 2>/dev/null || rm -f "$file"
done
printf '%s %s\n' "$BOOT" "$$" >"$RUN/pid"
echo -1000 >/proc/self/oom_score_adj
read_mem
# Half of RAM for /dev/shm (tmpfs only takes pages that are written).
mount -o remount,size=50% /dev/shm 2>/dev/null && log "/dev/shm $(df -h --output=size /dev/shm | tail -n 1 | tr -d ' ')"
log "start mem=${mem_total}KiB"
local tick=0 pct quiet_since=0 prev='' now
while :; do
read_mem
read_psi
pct=$((mem_total > 0 ? mem_avail * 100 / mem_total : 100))
if [ $((tick % SWEEP_EVERY)) -eq 0 ]; then
sweep
fi
if [ "$pct" -lt "$SWAP_ADD_PCT" ] && { [ "$swap_total" -eq 0 ] || [ $((swap_free * 4)) -lt "$swap_total" ]; }; then
add_swap && read_mem
fi
# Swap only raises MemAvailable by paging idle memory out: a fast allocator outruns it (the
# kernel OOM-killed a 300 MB/s hog with 1.7 GB of swap free), so the threshold ignores swap.
if [ "$pct" -lt "$KILL_PCT" ]; then
level=critical
if kill_largest "$pct"; then
write_state "$pct"
prev="$level:$kills"
sleep 1 # let the kernel reclaim the victim's memory before judging again
tick=$((tick + 1))
continue
fi
elif [ "$psi_full" -ge 10 ]; then
level=critical
elif [ "$pct" -lt "$HIGH_PCT" ] || [ "$psi_some" -ge 20 ]; then
level=high
else
level=ok
fi
now="${EPOCHSECONDS}"
if [ "$pct" -ge "$QUIET_PCT" ] && [ "$swap_total" -gt 0 ]; then
[ "$quiet_since" -gt 0 ] || quiet_since="$now"
if [ $((now - quiet_since)) -ge "$QUIET_SECONDS" ] && [ $(((swap_total - swap_free) * 2)) -lt "$mem_avail" ]; then
release_swap
quiet_since=0
fi
else
quiet_since=0
fi
if [ "$level:$kills" != "$prev" ] || [ $((tick % STATE_EVERY)) -eq 0 ]; then
write_state "$pct"
prev="$level:$kills"
fi
tick=$((tick + 1))
# Memory can go from 15% to nothing in two seconds: look five times as often when it is low.
if [ "$level" = ok ]; then sleep "$TICK"; else sleep "$FAST_TICK"; fi
done
}
running() {
local boot pid
read -r boot pid <"$RUN/pid" 2>/dev/null || return 1
[ "$boot" = "$(boot_id)" ] && [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null \
&& grep -qa orca-memguard "/proc/$pid/cmdline" 2>/dev/null
}
case "${1:-}" in
start)
[ "$(id -u)" -eq 0 ] || { echo 'orca-memguard: needs root' >&2; exit 1; }
running && exit 0
mkdir -p "$RUN"
setsid nohup "$0" run </dev/null >/dev/null 2>>"$RUN/memguard.log" &
exit 0 ;;
run) run ;;
sweep) sweep ;;
victim) choose_victim && printf '%s %s %s\n' "$VICTIM" "$((VICTIM_RSS * 4))" "$(proc_name "$VICTIM")" ;;
release-swap) release_swap ;;
*) echo 'usage: orca-memguard start|run|sweep|victim|release-swap' >&2; exit 2 ;;
esac
@@ -12,6 +12,8 @@ kill -TERM "$pid"
for _ in $(seq 1 $((WAIT_SECONDS * 10))); do
if ! kill -0 "$pid" 2>/dev/null; then
rm -f "$STATE/orcad.pid" "$STATE/ready.json"
# On-demand swap files would ride the stop snapshot (2 GiB each). Best effort, bounded.
[ -x /opt/orca/bin/orca-memguard ] && sudo -n timeout 20 /opt/orca/bin/orca-memguard release-swap >/dev/null 2>&1 || true
exit 0
fi
sleep 0.1
@@ -36,9 +36,15 @@ anthropic_key_scrub=''
gateway_key=''
gemini_key=''
codex_placeholder=''
sign_in_inside=''
while IFS= read -r line; do
name="${line%%=*}" b64="${line#*=}"
# Not for shells: the Codex account placeholder becomes ~/.codex/auth.json below.
# Not for shells: the agents the person signs in inside this workspace (e.g. `claude,codex`).
if [ "$name" = ORCA_SIGN_IN_INSIDE ]; then
sign_in_inside="$(printf '%s' "$b64" | base64 -d)"
case "$sign_in_inside" in *[!a-z,]*) sign_in_inside='' ;; esac
fi
if [ "$name" = ORCA_CODEX_AUTH_PLACEHOLDER ]; then
codex_placeholder="$(printf '%s' "$b64" | base64 -d)"
case "$codex_placeholder" in *[!A-Za-z0-9._-]*) codex_placeholder='' ;; esac
@@ -164,7 +170,9 @@ fi
# through them, and the agent would sit there doing nothing. Merges; never touches credentials.
node_bin="$(command -v node || find /opt/orca/runtime/runtimes -type f -name node 2>/dev/null | head -n 1)"
if [ -n "$node_bin" ]; then
ORCA_PROJECT="$PROJECT" ORCA_KEY_TAIL="$anthropic_key_tail" ORCA_KEY_SCRUB="$anthropic_key_scrub" ORCA_GEMINI_KEY="$gemini_key" \
claude_inside=''
case ",$sign_in_inside," in *,claude,*) claude_inside=1 ;; esac
ORCA_CLAUDE_INSIDE="$claude_inside" ORCA_PROJECT="$PROJECT" ORCA_KEY_TAIL="$anthropic_key_tail" ORCA_KEY_SCRUB="$anthropic_key_scrub" ORCA_GEMINI_KEY="$gemini_key" \
ORCA_GATEWAY_MODEL="${gateway_key:+${anthropic_model:-$openai_model}}" "$node_bin" -e '
const fs = require("fs"), path = require("path"), os = require("os")
const home = os.homedir()
@@ -178,7 +186,11 @@ const write = (file, value) => {
const claudeFile = path.join(process.env.CLAUDE_CONFIG_DIR || home, ".claude.json")
const c = readJson(claudeFile)
const before = JSON.stringify(c)
c.hasCompletedOnboarding = true
// Signed in inside this workspace and no login here yet: leave first-run onboarding to Claude, so it opens
// on its own sign-in screens instead of "Not logged in". Once a login exists, onboarding is done as usual.
const claudeCredentials = path.join(process.env.CLAUDE_CONFIG_DIR || path.join(home, ".claude"), ".credentials.json")
if (process.env.ORCA_CLAUDE_INSIDE === "1" && !fs.existsSync(claudeCredentials)) delete c.hasCompletedOnboarding
else c.hasCompletedOnboarding = true
c.theme = c.theme || "dark"
c.bypassPermissionsModeAccepted = true
c.projects = c.projects && typeof c.projects === "object" ? c.projects : {}
@@ -110,6 +110,10 @@ case "${ORCA_DOCKER:-}" in
esac
start_share_proxy
unset ORCA_SHARE_PROXY_KEY
# MEMGUARD: OOM-killer ranks that spare orcad and terminals, a guard that stops the largest build or
# test before the VM stalls, on-demand swap and a /dev/shm sized for browsers. Before the
# already-started exit so a repair start brings a dead guard back; it returns at once.
sudo -n "$ORCA_HOME/bin/orca-memguard" start </dev/null >/dev/null 2>&1 9>&- || true
if [ "$(cat "$STATE/boot-id" 2>/dev/null || true)" = "$boot_id" ] && [ -s "$STATE/ready.json" ] \
&& kill -0 "$(cat "$STATE/orcad.pid" 2>/dev/null || echo 0)" 2>/dev/null; then
print_ready
@@ -191,16 +195,35 @@ fi
"$ORCA_HOME/bin/orca-sandbox-env" >/dev/null || echo 'session env file unavailable; secrets apply at boot only' >&2
unset ORCA_SECRET_ENV
# OpenCode's first run in a VM installs its plugin SDK into ~/.config/opencode, creates its database
# and fetches its model list: ~2.5 s more on the first launch (5.2 s vs 2.7 s warm, n72). orca-sandbox-env
# just wrote that config, so do the first run now, in the background at low priority, with no
# credentials (`opencode models` does all of it and exits). Skipped once it has run on this disk.
if [ -x "$ORCA_HOME/tools/bin/opencode" ] && [ ! -d "$HOME/.config/opencode/node_modules" ]; then
# OpenCode installs @opencode-ai/plugin into ~/.config/opencode and, with Orca's plugin configured,
# draws nothing until that install ends (blank ~85 s on a slow registry, n73). The image carries the
# install (provision-golden.sh): copy it into a home that has neither node_modules nor a package.json
# of its own, so the first launch never fetches.
oc_seed="$ORCA_HOME/opencode-config-seed"
oc_dir="$HOME/.config/opencode"
if [ -d "$oc_seed/node_modules" ] && [ ! -e "$oc_dir/node_modules" ] && [ ! -e "$oc_dir/package.json" ]; then
mkdir -p "$oc_dir" \
&& cp -R "$oc_seed/node_modules" "$oc_dir/node_modules" \
&& cp "$oc_seed/package.json" "$oc_seed/package-lock.json" "$oc_dir/" \
|| echo 'opencode config seed not copied' >&2
fi
# The rest of OpenCode's first run (its database, the model list): ~2.5 s on the first launch
# (5.2 s vs 2.7 s warm, n72). Done now in the background at low priority with no credentials
# (`opencode models` does it and exits). Skipped once it has run on this disk. An image without the
# seed (an environment snapshot from an older golden) still needs the install: then the first run
# waits for orcad to put Orca's plugin in ~/.config/opencode/plugins (~10 s), since OpenCode only
# finishes that install when a plugin is configured.
if [ -x "$ORCA_HOME/tools/bin/opencode" ] && { [ ! -e "$HOME/.local/share/opencode/opencode.db" ] || [ ! -e "$oc_dir/node_modules" ]; }; then
(
cd "$HOME" || exit 0
# shellcheck disable=SC2016 # expanded by the inner shell
env -i HOME="$HOME" USER="$(id -un)" LANG=C.UTF-8 PATH="$ORCA_HOME/tools/bin:/usr/local/bin:/usr/bin:/bin" \
OPENCODE_DISABLE_AUTOUPDATE=1 setsid nohup nice -n 19 timeout 120 \
"$ORCA_HOME/tools/bin/opencode" models </dev/null >/dev/null 2>&1 9>&- &
OPENCODE_DISABLE_AUTOUPDATE=1 setsid nohup nice -n 19 bash -c '
if [ ! -e "$HOME/.config/opencode/node_modules" ]; then
for _ in $(seq 1 120); do [ -d "$HOME/.config/opencode/plugins" ] && break; sleep 0.5; done
fi
exec timeout 120 "$1" models
' opencode-first-run "$ORCA_HOME/tools/bin/opencode" </dev/null >/dev/null 2>&1 9>&- &
)
fi
@@ -0,0 +1,18 @@
-- CREATE-GH-STEP: the create wizard's "GitHub access" step asks repo-access for every create and
-- polls it every 2 s while the person installs the App. GitHub's installation lookups (app JWT, one
-- per repository and owner) are cached here for a short TTL, shared by every isolate, and dropped by
-- the installation webhooks and the setup return so a fresh install answers on the next poll.
-- No credential or token is stored: installation ids and the owner's login only. Additive.
CREATE TABLE github_installation_cache (
-- '<owner>/<repo>' for a repository lookup, '<owner>/' for an owner lookup; lowercased.
cache_key TEXT PRIMARY KEY,
owner_key TEXT NOT NULL,
-- NULL: GitHub answered 404 (the App isn't installed there, or not on this repository).
installation_id INTEGER,
github_account_id INTEGER,
github_account_login TEXT,
github_account_type TEXT,
checked_at INTEGER NOT NULL
);
CREATE INDEX github_installation_cache_owner ON github_installation_cache (owner_key);
CREATE INDEX github_installation_cache_installation ON github_installation_cache (installation_id);
@@ -0,0 +1,4 @@
-- Nearest region: an environment records the region it lives in (the served region nearest whoever
-- created it), so its builds and the workspaces booted from it stay there (a snapshot only boots in
-- its own region). NULL: made before this was recorded, lives in the default region. Additive.
ALTER TABLE environments ADD COLUMN region TEXT;
@@ -0,0 +1,17 @@
-- "Sign in inside this workspace" (agent-sources contract §2): a per-person default Source of
-- 'in-workspace' stores nothing and injects nothing for that agent, so its own login inside each
-- sandbox wins. SQLite can't widen a CHECK in place, so the table is rebuilt with every row kept.
CREATE TABLE agent_sources_next (
user_id TEXT NOT NULL,
account_id TEXT NOT NULL,
scope TEXT NOT NULL,
agent TEXT NOT NULL CHECK (agent IN ('claude', 'codex')),
source TEXT NOT NULL CHECK (source IN ('sign-in', 'key', 'in-workspace', 'accepted', 'declined')),
updated_at INTEGER NOT NULL,
PRIMARY KEY (user_id, scope, agent)
);
INSERT INTO agent_sources_next (user_id, account_id, scope, agent, source, updated_at)
SELECT user_id, account_id, scope, agent, source, updated_at FROM agent_sources;
DROP TABLE agent_sources;
ALTER TABLE agent_sources_next RENAME TO agent_sources;
CREATE INDEX agent_sources_scope ON agent_sources (scope);
@@ -0,0 +1,56 @@
# Egress release probe
`egress-probe.ts` checks sandbox egress against **dev** before a release. It goes through Orca's own
paths: real dev workspaces created with a dev-JWK user token, with commands run in a workspace
terminal over the shipped `orca` CLI's pairing code (the desktop's E2EE path to orcad). Every
credential is bogus, and postman-echo.com plays the model provider.
Setup for a throwaway identity, all of it removed afterwards:
- the dev GitHub App's installation, bound for the test repo (operator route);
- a stored GitHub identity, so member placeholders exist (the dev `member-github` route);
- bogus personal secrets:
- `ANTHROPIC_API_KEY`, with `ANTHROPIC_BASE_URL` set to the echo host;
- `OPENAI_API_KEY`, with `OPENAI_BASE_URL` set to `chatgpt.com/backend-api/codex`, so its egress
takes the ChatGPT relay.
The probe creates two workspaces through the desktop's create path (clone after ready):
- **public**: a public repo the App isn't installed on. Member placeholder, no App token (#758).
- **app**: the App-installed test repo. App token, github.com fail-closed.
| Host | What must happen |
|---|---|
| echo host (custom base URL) | The provider sees the bogus key where the VM sent the placeholder. The other `authorization` header is unchanged, and no `vercel-forwarded-*` header or OIDC token arrives. A foreign key gets the firewall 403 `orca-egress-denied`. |
| chatgpt.com | The placeholder reaches ChatGPT: any provider answer counts, but not the firewall 403 and not Cloudflare's edge page. A foreign bearer gets the firewall 403. |
| api.github.com | No credential gets the firewall 403. With the workspace placeholder, GitHub answers 200. |
| github.com | Anonymous `git ls-remote` works on **public** (#758) and is refused on **app**. The app repo clones, and the workspace's own git works. |
| VM env | `ANTHROPIC_API_KEY` and `OPENAI_API_KEY` hold placeholders. This is checked by sha256, so the bogus values never cross the terminal. |
The probe takes about 1–3 minutes, deletes what it created (`--keep` leaves it), prints one JSON
report and exits 1 on any failed expectation. It refuses any control plane other than dev.
## Run it locally
You need the Orca app installed (for its `orca` CLI) and `gcloud` access to the dev secrets.
```sh
cd cloud
pnpm install --filter @orca-cloud/sandbox-control...
K=$(mktemp -d) # 0700; delete it afterwards
gcloud --account <you> secrets versions access latest --project onorca-cloud-staging \
--secret orca-sandbox-control-dev-test-jwk > "$K/jwk.json"
gcloud --account <you> secrets versions access latest --project onorca-cloud-staging \
--secret orca-sandbox-control-dev-operator-token > "$K/operator"
ORCA_PROBE_DEV_JWK_FILE="$K/jwk.json" ORCA_PROBE_OPERATOR_TOKEN_FILE="$K/operator" \
pnpm --filter @orca-cloud/sandbox-control exec tsx scripts/egress-probe.ts
rm -rf "$K"
```
`ORCA_PROBE_CLI` points the probe at another `orca` CLI. The default is the installed app's.
## CI
`.github/workflows/cloud-egress-probe.yml` runs on manual dispatch. It installs the CLI from the
latest release's Linux package. It needs the `ORCA_DEV_TEST_JWK` and `ORCA_DEV_OPERATOR_TOKEN`
repository secrets, and warns and skips when they are missing. None are configured today.
@@ -0,0 +1,119 @@
// Pure parts of the egress release probe (egress-probe.ts): the runner it starts in a workspace
// terminal, the requests that runner sends, and what each answer must look like.
import { createHash } from 'node:crypto'
import { EGRESS_DENIED_BODY } from '../src/network-policy'
/**
* One request from inside the workspace. A header value `$NAME` is read from the VM's env (orcad's
* environment, as agents get it), so placeholders never cross the terminal.
*/
export type ProbeRequest = { name: string; url: string; method?: 'GET' | 'POST'; headers?: Record<string, string> }
/** What the answer must be. */
export type Expectation =
/** The echo host answered as the provider: these headers (by sha256) arrived, and nothing from Vercel's hop. */
| { kind: 'swapped'; headers: Record<string, string> }
/** The firewall's own 403 (a credential host fails closed). */
| { kind: 'denied' }
/** The request reached the provider itself: not the firewall's 403, not an edge block page. */
| { kind: 'upstream' }
export type ProbeCase = ProbeRequest & { expect: Expectation }
/** What the runner reports per request: never a header value, only sha256 prefixes of the echoed ones. */
export type Summary = {
status: number
/** Body is the firewall's deny body. */
denied: boolean
/** An HTML 403 from Cloudflare: the provider's edge refused the proxy. */
edgeBlocked: boolean
/** The echo host's answer: sha256 prefix of each echoed header, and Vercel hop headers seen. */
echoed?: Record<string, string>
leaked?: string[]
}
export const sha8 = (value: string): string => createHash('sha256').update(value).digest('hex').slice(0, 8)
/**
* Runs in the workspace (node, from a terminal): fetches each request and prints one short line per
* fact, `PRB <case> <key>=<value>`, ending with `PRB-END`. Short lines survive the terminal intact.
* The echo host (postman-echo.com) answers JSON with the request's headers under `headers`.
*/
export const VM_RUNNER = `
const { createHash } = require('node:crypto')
const cases = JSON.parse(Buffer.from(process.argv[1], 'base64').toString())
const sha8 = (v) => createHash('sha256').update(String(v)).digest('hex').slice(0, 8)
const out = (i, k, v) => console.log('PRB ' + i + ' ' + k + '=' + v)
;(async () => {
for (const [i, c] of cases.entries()) {
const headers = {}
for (const [k, v] of Object.entries(c.headers || {})) headers[k] = v.startsWith('$') ? process.env[v.slice(1)] || '' : v
try {
const r = await fetch(c.url, { method: c.method || 'GET', headers, body: c.method === 'POST' ? '{}' : undefined, redirect: 'manual', signal: AbortSignal.timeout(25000) })
const body = await r.text()
out(i, 's', r.status)
out(i, 'd', body.trim() === ${JSON.stringify(EGRESS_DENIED_BODY)} ? 1 : 0)
out(i, 'cf', r.status === 403 && /text\\/html/.test(r.headers.get('content-type') || '') && /cloudflare/i.test(r.headers.get('server') || '') ? 1 : 0)
let echo = null
try { echo = JSON.parse(body).headers } catch {}
if (echo && typeof echo === 'object') {
for (const [k, v] of Object.entries(echo)) {
if (/^vercel-forwarded-|oidc/i.test(k)) out(i, 'leak', k)
else if (k in (c.headers || {})) out(i, 'h.' + k, sha8(v))
}
}
} catch (e) {
out(i, 's', 0)
}
}
console.log('PRB-END')
})()
`
export function parseRunnerOutput(text: string, count: number): Summary[] {
const summaries: Summary[] = Array.from({ length: count }, () => ({ status: 0, denied: false, edgeBlocked: false }))
for (const match of text.matchAll(/PRB (\d+) ([a-z.-]+)=(\S+)/g)) {
const summary = summaries[Number(match[1])]
const [key, value] = [match[2]!, match[3]!]
if (!summary) {
continue
}
if (key === 's') summary.status = Number(value)
else if (key === 'd') summary.denied = value === '1'
else if (key === 'cf') summary.edgeBlocked = value === '1'
else if (key === 'leak') summary.leaked = [...(summary.leaked ?? []), value]
else if (key.startsWith('h.')) summary.echoed = { ...summary.echoed, [key.slice(2)]: value }
}
return summaries
}
/** Problems with one answer; empty when it is what the case expects. Never quotes a header value. */
export function judge(probe: ProbeCase, answer: Summary): string[] {
const { expect } = probe
const denied = answer.status === 403 && answer.denied
if (expect.kind === 'denied') {
return denied ? [] : [`${probe.name}: expected the firewall's 403 ${EGRESS_DENIED_BODY}, got ${answer.status}`]
}
if (expect.kind === 'upstream') {
if (denied) return [`${probe.name}: refused by the firewall`]
if (answer.edgeBlocked) return [`${probe.name}: the provider's edge refused the egress proxy (Cloudflare 403 page)`]
return answer.status === 0 ? [`${probe.name}: no answer`] : []
}
if (answer.status !== 200 || !answer.echoed) {
return [`${probe.name}: expected the echo host's answer, got ${answer.status}${denied ? ' (denied)' : ''}`]
}
const problems = (answer.leaked ?? []).map((name) => `${probe.name}: the provider saw ${name}`)
for (const [name, value] of Object.entries(expect.headers)) {
if (answer.echoed[name] !== sha8(value)) {
problems.push(`${probe.name}: header ${name} ${answer.echoed[name] ? 'arrived with another value' : 'did not arrive'}`)
}
}
return problems
}
/** The terminal line that runs the cases: the runner and its input both base64, one shell word each. */
export function runnerCommand(cases: readonly ProbeRequest[]): string {
const runner = Buffer.from(VM_RUNNER).toString('base64')
const input = Buffer.from(JSON.stringify(cases.map(({ name: _name, ...request }) => request))).toString('base64')
return `node -e "$(echo ${runner} | base64 -d)" ${input}`
}
@@ -0,0 +1,268 @@
// Release probe for sandbox egress, against DEV only, through Orca's own paths: real dev workspaces
// created with a dev-JWK user token, driven from a terminal over the `orca` CLI's pairing code (the
// desktop's E2EE path). Bogus tokens only; postman-echo.com plays the model provider.
// pnpm --filter @orca-cloud/sandbox-control exec tsx scripts/egress-probe.ts [--keep]
//
// A throwaway identity gets: the dev GitHub App installation bound for one test repo (operator), a
// stored GitHub identity (the dev member-github route, so member placeholders exist), and bogus
// personal secrets ANTHROPIC_API_KEY (base URL: the echo host) and OPENAI_API_KEY (base URL:
// chatgpt.com, so its egress takes the ChatGPT relay). Two workspaces:
// public a public repo the App is not installed on: member placeholder, no App token (#758)
// app the App-installed test repo: App token, github.com fail-closed
// Checks (every one names the host it is about):
// - api.anthropic-compatible echo host: the provider sees the bogus key where the VM sent the
// placeholder, the other auth header unchanged, nothing from Vercel's hop; a foreign key 403s.
// - chatgpt.com: the placeholder reaches ChatGPT (any provider answer; not the firewall's 403 and
// not Cloudflare's edge page); a foreign bearer 403s.
// - api.github.com: no credential 403s; with the workspace placeholder GitHub answers 200.
// - github.com: anonymous `git ls-remote` works on `public` (#758) and 403s on `app`; the
// workspace's own git (placeholder extraheader) works on `app`.
// - the VM holds placeholders, never the bogus values.
//
// Inputs (files, never printed):
// ORCA_PROBE_DEV_JWK_FILE dev test signing key (SM: orca-sandbox-control-dev-test-jwk)
// ORCA_PROBE_OPERATOR_TOKEN_FILE dev operator token (SM: orca-sandbox-control-dev-operator-token)
// ORCA_PROBE_CONTROL_URL default: the dev sandbox-control Worker (anything else is refused)
// ORCA_PROBE_CLI default: the installed Orca app's `orca` CLI
// Prints one JSON report; exits 1 on any failed expectation. See scripts/README.md.
import { execFile } from 'node:child_process'
import { randomBytes } from 'node:crypto'
import { readFileSync } from 'node:fs'
import { SignJWT, importJWK, type JWK } from 'jose'
import { judge, parseRunnerOutput, runnerCommand, sha8, type ProbeCase } from './egress-probe-checks'
const CONTROL_URL = process.env.ORCA_PROBE_CONTROL_URL ?? 'https://orca-sandbox-control-dev.lovecast.workers.dev'
const ORCA_CLI = process.env.ORCA_PROBE_CLI ?? '/Applications/Orca.app/Contents/Resources/bin/orca'
/** The dev GitHub App is installed on this repo. */
const APP_REPO = process.env.ORCA_PROBE_APP_REPO ?? 'https://github.com/nwparker/orca-sandbox-push-test'
/** Public, and the dev App is not installed on its owner. */
const PUBLIC_REPO = 'https://github.com/octocat/Hello-World'
const ECHO = 'https://postman-echo.com'
if (!/^https:\/\/[a-z0-9-]+-dev\.lovecast\.workers\.dev$|^http:\/\/(127\.0\.0\.1|localhost)(:\d+)?$/.test(CONTROL_URL)) {
throw new Error(`refusing a non-dev control plane: ${CONTROL_URL}`)
}
const keep = process.argv.includes('--keep')
const runId = randomBytes(4).toString('hex')
const sub = `egress-probe-${runId}`
const accountId = `dev:${sub}`
const fileSecret = (env: string): string => {
const file = process.env[env]
if (!file) {
throw new Error(`set ${env}`)
}
return readFileSync(file, 'utf8').trim()
}
async function userToken(): Promise<string> {
const jwk = JSON.parse(fileSecret('ORCA_PROBE_DEV_JWK_FILE')) as { kid: string; private: JWK; issuers?: { sandbox?: string }; audiences?: { sandbox?: string } }
return new SignJWT({ sub, purpose: 'sandbox' })
.setProtectedHeader({ alg: 'ES256', kid: jwk.kid })
.setIssuer(jwk.issuers?.sandbox ?? 'orca-sandbox-dev-test')
.setAudience(jwk.audiences?.sandbox ?? 'orca-sandbox')
.setIssuedAt()
.setExpirationTime('45m')
.sign(await importJWK(jwk.private, 'ES256'))
}
type Reply = { status: number; body: Record<string, any> | null }
function client(bearer: string) {
return async (method: string, path: string, body?: unknown): Promise<Reply> => {
const response = await fetch(`${CONTROL_URL}${path}`, {
method,
// clone-progress: the desktop's create, whose clone runs after ready (GET /v1/sandboxes/:id/clone).
headers: { authorization: `Bearer ${bearer}`, 'content-type': 'application/json', 'x-orca-sandbox-features': 'clone-progress' },
body: body === undefined ? undefined : JSON.stringify(body),
signal: AbortSignal.timeout(300_000)
})
return { status: response.status, body: (await response.json().catch(() => null)) as Record<string, any> | null }
}
}
/** The shipped CLI over a workspace's pairing code; the code never reaches logs or the report. */
function orca(pairingCode: string, args: string[], timeoutMs = 60_000): Promise<any> {
return new Promise((resolve) => {
execFile(ORCA_CLI, [...args, '--json', '--pairing-code', pairingCode], { timeout: timeoutMs, maxBuffer: 8 << 20, env: { ...process.env, ORCA_PAIRING_CODE: '' } }, (_error, stdout) => {
try {
resolve(JSON.parse(stdout))
} catch {
resolve(null)
}
})
})
}
/** Every string in a CLI result, one per line (terminal output may come as a string, lines or chunks). */
function textOf(value: unknown): string {
if (typeof value === 'string') return value
if (Array.isArray(value)) return value.map(textOf).join('\n')
if (value && typeof value === 'object') return Object.values(value).map(textOf).join('\n')
return ''
}
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))
/** Runs one line in a fresh workspace terminal and returns the output once `PRB-END` shows. */
async function inTerminal(pairingCode: string, projectRoot: string, line: string): Promise<string> {
await orca(pairingCode, ['repo', 'add', '--path', projectRoot, '--kind', 'git'])
const created = await orca(pairingCode, ['terminal', 'create', '--worktree', `path:${projectRoot}`, '--title', 'egress-probe'])
const handle: string | undefined = created?.result?.terminal?.handle ?? created?.result?.handle ?? created?.result?.terminal?.id
if (!handle) {
throw new Error('could not open a workspace terminal')
}
// `clear` first: the read must only see this run's lines.
await orca(pairingCode, ['terminal', 'send', '--terminal', handle, '--text', `clear; ${line}`, '--enter'])
const deadline = Date.now() + 180_000
while (Date.now() < deadline) {
const read = await orca(pairingCode, ['terminal', 'read', '--terminal', handle])
const text = textOf(read?.result)
// The typed line is base64 past its first words, so PRB-END only shows once the runner prints it.
if (text.includes('PRB-END')) {
return text
}
await sleep(1500)
}
throw new Error('the probe did not finish in the workspace terminal')
}
/** Git and env facts, one short line each, then the HTTP cases. */
function factsScript(gitChecks: Record<string, string>, bogus: Record<string, string>): string {
const lines = Object.entries(gitChecks).map(([name, command]) => `GIT_TERMINAL_PROMPT=0 timeout 60 ${command} >/dev/null 2>&1; echo "PRB ${name} rc=$?"`)
// The VM must hold placeholders: compare by sha256 so the bogus values never cross the terminal.
for (const [name, digest] of Object.entries(bogus)) {
// The verdict word is a variable: the typed command (echoed by the terminal) must not contain it.
lines.push(`v=placeholder; [ "$(printf %s "\${${name}:-}" | sha256sum | cut -c1-8)" = ${digest} ] && v=bogus; echo "PRB env.${name} holds=$v"`)
}
return lines.join('; ')
}
async function main(): Promise<void> {
const failures: string[] = []
const report: Record<string, unknown> = { runId }
const cleanup: (() => Promise<unknown>)[] = []
const started = Date.now()
console.error(`[egress-probe] run ${runId}: throwaway identity ${sub}`)
const user = client(await userToken())
const operator = client(fileSecret('ORCA_PROBE_OPERATOR_TOKEN_FILE'))
const bogusAnthropic = `sk-ant-api03-orcaprobe${randomBytes(16).toString('hex')}`
const bogusOpenAI = `sk-proj-orcaprobe${randomBytes(16).toString('hex')}`
try {
// Identity: the App bound for the test repo, and a stored GitHub identity (member placeholders).
const bound = await operator('POST', '/v1/internal/github/bindings', { repoUrl: APP_REPO, accountId })
const installationId: number | undefined = bound.body?.bound?.installationId
if (bound.status !== 200 || !installationId) {
throw new Error(`bind the dev App: ${bound.status} ${bound.body?.error ?? ''}`)
}
cleanup.push(() => operator('DELETE', '/v1/internal/github/bindings', { installationId, accountId }))
const member = await operator('POST', '/v1/internal/dev/member-github', { accountId, userId: sub, repoUrl: APP_REPO, login: 'egress-probe', githubUserId: 1, name: 'egress probe' })
if (member.status !== 200) {
throw new Error(`store the probe's GitHub identity: ${member.status} ${member.body?.error ?? ''}`)
}
for (const [name, value] of [
['ANTHROPIC_API_KEY', bogusAnthropic],
['ANTHROPIC_BASE_URL', ECHO],
['OPENAI_API_KEY', bogusOpenAI],
['OPENAI_BASE_URL', 'https://chatgpt.com/backend-api/codex']
] as const) {
const put = await user('PUT', `/v1/secrets/personal/${name}`, { value })
cleanup.push(() => user('DELETE', `/v1/secrets/personal/${name}`))
if (put.status !== 200) {
throw new Error(`PUT secret ${name}: ${put.status} ${put.body?.error ?? ''}`)
}
}
const create = async (label: string, repo: string) => {
console.error(`[egress-probe] creating the ${label} workspace (${sub}-${label})`)
const reply = await user('POST', '/v1/sandboxes', { workspaceId: `${sub}-${label}`, machineSize: 'standard', repo: { url: repo }, useEnvironment: false })
const id: string | undefined = reply.body?.sandbox?.id
if (id) {
cleanup.unshift(() => user('DELETE', `/v1/sandboxes/${id}`))
}
if (reply.status !== 200 || !reply.body?.connection?.pairingCode) {
throw new Error(`create the ${label} workspace: ${reply.status} ${reply.body?.error ?? ''}`)
}
console.error(`[egress-probe] ${label} workspace ${id}`)
// The desktop's path: the clone runs after ready, and the desktop polls it.
let clone = 'unknown'
for (const until = Date.now() + 240_000; Date.now() < until; await sleep(3_000)) {
clone = String((await user('GET', `/v1/sandboxes/${id}/clone`)).body?.clone?.state ?? 'unknown')
if (clone.startsWith('done') || clone.startsWith('failed')) break
}
clones[label] = clone
return { id: id!, pairingCode: reply.body.connection.pairingCode as string, projectRoot: reply.body.connection.projectRoot as string }
}
const clones: Record<string, string> = {}
const [publicBox, appBox] = await Promise.all([create('public', PUBLIC_REPO), create('app', APP_REPO)])
report.workspaces = { public: publicBox.id, app: appBox.id }
const anthropic: ProbeCase[] = [
{
name: 'echo host (ANTHROPIC_BASE_URL): placeholder swapped, other auth untouched',
url: `${ECHO}/post`,
method: 'POST',
headers: { 'x-api-key': '$ANTHROPIC_API_KEY', authorization: 'Bearer probe-keep-bearer' },
expect: { kind: 'swapped', headers: { 'x-api-key': bogusAnthropic, authorization: 'Bearer probe-keep-bearer' } }
},
{ name: 'echo host: foreign key fails closed', url: `${ECHO}/post`, method: 'POST', headers: { 'x-api-key': 'sk-ant-probe-foreign' }, expect: { kind: 'denied' } },
{
name: 'chatgpt.com: placeholder reaches ChatGPT (relay)',
url: 'https://chatgpt.com/backend-api/codex/models',
headers: { authorization: '$PROBE_OPENAI_BEARER' },
expect: { kind: 'upstream' }
},
{ name: 'chatgpt.com: foreign bearer fails closed', url: 'https://chatgpt.com/backend-api/codex/models', headers: { authorization: 'Bearer probe-foreign' }, expect: { kind: 'denied' } },
{ name: 'api.github.com: no credential fails closed', url: 'https://api.github.com/rate_limit', expect: { kind: 'denied' } }
]
const appCases: ProbeCase[] = [
{ name: 'api.github.com: workspace placeholder swapped (GitHub answers 200)', url: 'https://api.github.com/rate_limit', headers: { authorization: '$PROBE_GH_BEARER' }, expect: { kind: 'upstream' } },
{ name: 'github.com: anonymous fails closed with an App token', url: `${APP_REPO}.git/info/refs?service=git-upload-pack`, expect: { kind: 'denied' } }
]
const digests = { ANTHROPIC_API_KEY: sha8(bogusAnthropic), OPENAI_API_KEY: sha8(bogusOpenAI) }
// The runner reads whole header values from env (`$NAME`), so the Bearer forms are built here.
const prelude = 'export PROBE_OPENAI_BEARER="Bearer $OPENAI_API_KEY" PROBE_GH_BEARER="Bearer $GH_TOKEN"; '
console.error('[egress-probe] running the checks in both workspaces')
const runs = await Promise.all([
inTerminal(publicBox.pairingCode, publicBox.projectRoot, `${prelude}${factsScript({ 'git.public-anonymous': `git ls-remote ${PUBLIC_REPO} HEAD` }, digests)}; ${runnerCommand(anthropic)}`),
inTerminal(appBox.pairingCode, appBox.projectRoot, `${prelude}${factsScript({ 'git.app-anonymous': `git -c http.extraheader= -c http.https://github.com/.extraheader= ls-remote ${APP_REPO} HEAD`, 'git.app-workspace': 'git ls-remote origin HEAD' }, {})}; ${runnerCommand(appCases)}`)
])
const results: Record<string, unknown>[] = []
const check = (workspace: string, name: string, ok: boolean, detail: string) => {
results.push({ workspace, check: name, ok })
if (!ok) {
failures.push(`[${workspace}] ${name}: ${detail}`)
}
}
for (const [workspace, text, cases] of [['public', runs[0], anthropic], ['app', runs[1], appCases]] as const) {
const summaries = parseRunnerOutput(text, cases.length)
cases.forEach((probeCase, i) => {
const problems = judge(probeCase, summaries[i]!)
check(workspace, probeCase.name, problems.length === 0, problems.join('; '))
})
}
const rc = (text: string, name: string) => new RegExp(`PRB ${name.replaceAll('.', '\\.')} rc=(\\d+)`).exec(text)?.[1]
check('public', 'github.com: anonymous git ls-remote of a public repo works (#758)', rc(runs[0], 'git.public-anonymous') === '0', `rc=${rc(runs[0], 'git.public-anonymous')}`)
check('app', 'github.com: anonymous git ls-remote is refused with an App token', rc(runs[1], 'git.app-anonymous') !== '0' && rc(runs[1], 'git.app-anonymous') !== undefined, `rc=${rc(runs[1], 'git.app-anonymous')}`)
check('app', "github.com: the workspace's own git (placeholder) works", rc(runs[1], 'git.app-workspace') === '0', `rc=${rc(runs[1], 'git.app-workspace')}`)
for (const [workspace, state] of Object.entries(clones)) {
check(workspace, 'github.com: the workspace repo clones', state === 'done', `clone ${state}`)
}
for (const name of Object.keys(digests)) {
check('public', `the VM holds a placeholder for ${name}`, new RegExp(`PRB env\\.${name} holds=placeholder`).test(runs[0]), 'the VM env holds the bogus value itself')
}
report.checks = results
} catch (error) {
failures.push(`probe error: ${error instanceof Error ? error.message : String(error)}`)
} finally {
if (!keep) {
for (const step of cleanup) {
await step().catch((error: unknown) => console.error(`[egress-probe] cleanup: ${String(error)}`))
}
}
}
console.log(JSON.stringify({ ok: failures.length === 0, failures, ...report, ms: Date.now() - started }, null, 2))
console.error(failures.length === 0 ? '[egress-probe] OK' : `[egress-probe] FAILED:\n ${failures.join('\n ')}`)
process.exitCode = failures.length === 0 ? 0 : 1
}
void main()
@@ -100,7 +100,7 @@ export const claudeAccount: AccountProvider = {
type: 'error',
error: {
type: 'authentication_error',
message: 'Your Claude sign-in for Orca Cloud expired. Renew it in Orca: Settings → Agent accounts.'
message: 'Your Claude sign-in for Orca Cloud expired. Renew it in Orca: Settings → Cloud agent sign-ins.'
}
})
}
+31 -10
View File
@@ -1,7 +1,9 @@
// Agent Source per person (agent-sources contract §2): "Your sign-in" or "API key", per agent, as a
// default plus an optional per-workspace override, and the one-time "Use my sign-in" offer answer (D8).
// A default can also be "Sign in inside this workspace" ('in-workspace'): Orca stores and injects nothing
// for that agent (no account placeholder, no key), and its own login inside each sandbox wins.
// GET /v1/agent-sources[?workspaceId=<sandbox id>]
// PUT /v1/agent-sources {agent, source: 'sign-in'|'key'|null, workspaceId?}
// PUT /v1/agent-sources {agent, source: 'sign-in'|'key'|'in-workspace'|null, workspaceId?}
// PUT /v1/agent-sources/offer {agent, answer: 'accepted'|'declined'}
// A sandbox applies its OWNER's effective source: with 'sign-in', that agent's credential names are left
// out of the session env (and so out of egress routes, the allowlist and gateway derivation), letting
@@ -23,13 +25,16 @@ const OFFER_SCOPE = 'offer'
type SourceRow = { scope: string; agent: CredentialAgent; source: string }
const AgentSchema = z.enum(['claude', 'codex'])
const IN_WORKSPACE = 'in-workspace'
export const PutSourceSchema = z
.object({
agent: AgentSchema,
source: z.enum(['sign-in', 'key']).nullable(),
source: z.enum(['sign-in', 'key', IN_WORKSPACE]).nullable(),
workspaceId: z.string().min(1).max(128).optional()
})
.strict()
// Signing in inside the workspace is a per-person default, never a one-workspace override.
.refine((body) => !(body.source === IN_WORKSPACE && body.workspaceId))
export const PutOfferSchema = z.object({ agent: AgentSchema, answer: z.enum(['accepted', 'declined']) }).strict()
export class AgentSourceError extends Error {
@@ -60,12 +65,15 @@ function byAgent(rows: SourceRow[], scope: string): Record<CredentialAgent, Agen
return out
}
/** Agents whose owner chose "Your sign-in" for this sandbox (override first, then default). */
export async function signInAgentsFor(db: D1Database, row: Pick<SandboxRow, 'id' | 'user_id'>): Promise<Set<CredentialAgent>> {
const rows = await rowsFor(db, row.user_id, [row.id, DEFAULT_SCOPE])
const workspace = byAgent(rows, row.id)
const defaults = byAgent(rows, DEFAULT_SCOPE)
return new Set(CREDENTIAL_AGENTS.filter((agent) => (workspace[agent] ?? defaults[agent]) === 'sign-in'))
/** Agents this person signs in inside each workspace (their default is 'in-workspace'). */
function inWorkspaceAgents(rows: SourceRow[]): Record<CredentialAgent, boolean> {
const out = { claude: false, codex: false } as Record<CredentialAgent, boolean>
for (const row of rows) {
if (row.scope === DEFAULT_SCOPE && row.source === IN_WORKSPACE) {
out[row.agent] = true
}
}
return out
}
/** Drops the credential names of agents on "Your sign-in" (after preset derivation, so derived ones go too). */
@@ -111,6 +119,7 @@ export async function agentSourcesView(
const subject = row ? row.user_id : caller.userId
const rows = await rowsFor(db, subject, [DEFAULT_SCOPE, OFFER_SCOPE, ...(row ? [row.id] : [])])
const defaults = byAgent(rows, DEFAULT_SCOPE)
const inWorkspace = inWorkspaceAgents(rows)
const workspace = row ? byAgent(rows, row.id) : null
const effective = await effectiveSecrets(db, caller, row?.repo_url ?? null)
const covered = coveredByKey(new Set(effective.map((secret) => secret.name)))
@@ -135,6 +144,10 @@ export async function agentSourcesView(
if (defaults[agent]) {
return [agent, { source: defaults[agent], origin: 'default' }]
}
// `source` keeps its two values for older desktops; `inWorkspace` says what this one is.
if (inWorkspace[agent]) {
return [agent, { source: null, origin: 'default' }]
}
return [agent, { source: covered[agent] ? 'key' : usable[agent] ? 'sign-in' : null, origin: 'implicit' }]
})
)
@@ -145,7 +158,7 @@ export async function agentSourcesView(
}
// D13: personal accounts never apply in a workspace shared with teammates.
const sharedWorkspace = row ? isSharedWorkspace(row) : false
return { defaults, workspace, effective: resolved, coveredByKey: covered, signInOffer, owner, sharedWorkspace }
return { defaults, inWorkspace, workspace, effective: resolved, coveredByKey: covered, signInOffer, owner, sharedWorkspace }
}
async function upsert(db: D1Database, caller: SandboxCaller, scope: string, agent: CredentialAgent, source: string | null, now: number) {
@@ -200,11 +213,12 @@ export async function deleteAgentSourcesForSandbox(db: D1Database, sandboxId: st
/**
* How each agent authenticates in this session (D10):
* - 'account': the owner's Orca-account credential through an egress placeholder (keys for that agent dropped);
* - 'inside': the owner signs in inside each workspace ('in-workspace'): keys dropped, the agent's own login;
* - 'omit': kill-switch fallback (Source sign-in, provider off): keys dropped, the agent's own login in the sandbox;
* - 'keep': API keys as resolved (precedence), the default.
* Source: explicit (workspace, then default), else implicit: a covering key wins, else a usable account.
*/
export type AgentMode = 'account' | 'omit' | 'keep'
export type AgentMode = 'account' | 'inside' | 'omit' | 'keep'
export async function agentModesFor(
db: D1Database,
row: Pick<SandboxRow, 'id' | 'user_id' | 'visibility' | 'ownership_transferred_at'>,
@@ -219,9 +233,16 @@ export async function agentModesFor(
const rows = await rowsFor(db, row.user_id, [row.id, DEFAULT_SCOPE])
const workspace = byAgent(rows, row.id)
const defaults = byAgent(rows, DEFAULT_SCOPE)
const inWorkspace = inWorkspaceAgents(rows)
const covered = coveredByKey(new Set(Object.keys(env)))
const modes = {} as Record<CredentialAgent, AgentMode>
for (const agent of CREDENTIAL_AGENTS) {
if (!workspace[agent] && inWorkspace[agent]) {
// Nothing of Orca's for this agent: no placeholder, no key, no egress route, so the login the person
// makes inside the sandbox (stored in its home, kept across sleep) reaches the vendor untouched.
modes[agent] = 'inside'
continue
}
const explicit = workspace[agent] ?? defaults[agent]
const provider = accountProvider(agent)
const enabled = options.providerEnabled[agent] && provider !== undefined
@@ -7,7 +7,7 @@
// it on the current image. Unused for 14 days it is deleted with its snapshots, unless an admin edited
// it (that makes it an ordinary 'org' environment).
import type { SandboxCaller } from './auth'
import { listBuilds, startBuild, TERMINAL, type BuildConfig, type BuildDeps } from './environment-builds'
import { environmentRegion, listBuilds, startBuild, TERMINAL, type BuildConfig, type BuildDeps } from './environment-builds'
import { canonicalRepoUrl, parseRepos, repoDir, type EnvironmentRow } from './environments'
import { normalizeRepo } from './secrets'
import type { SandboxProviderApi } from './sandbox-provider'
@@ -52,16 +52,22 @@ export type AutoEnvironmentOutcome = 'created' | 'rebuilt' | 'exists' | 'skipped
* A create of `repoUrl` booted without an environment: make (or retry building) the account's automatic
* environment for it. Never throws; the create it rides behind already succeeded.
*/
export async function ensureAutoEnvironment(deps: BuildDeps, accountId: string, repoUrl: string): Promise<AutoEnvironmentOutcome> {
export async function ensureAutoEnvironment(
deps: BuildDeps,
accountId: string,
repoUrl: string,
/** The create's region: the environment lives (and builds) there, so the next create of the repo lands there too. */
region?: string
): Promise<AutoEnvironmentOutcome> {
try {
return await ensure(deps, accountId, repoUrl)
return await ensure(deps, accountId, repoUrl, region)
} catch (error) {
log('auto_environment_failed', { error: String(error).slice(0, 200) })
return 'skipped'
}
}
async function ensure(deps: BuildDeps, accountId: string, repoUrl: string): Promise<AutoEnvironmentOutcome> {
async function ensure(deps: BuildDeps, accountId: string, repoUrl: string, region?: string): Promise<AutoEnvironmentOutcome> {
const url = canonicalRepoUrl(repoUrl)
const wanted = normalizeRepo(url)
if (!url || !wanted) {
@@ -90,11 +96,11 @@ async function ensure(deps: BuildDeps, accountId: string, repoUrl: string): Prom
await deps.db
.prepare(
`INSERT OR IGNORE INTO environments (id, account_id, name, repos, install_script, env_var_names, machine_size,
revision, next_build_number, created_by, updated_by, created_at, updated_at, kind, last_used_at)
VALUES (?, ?, ?, ?, '', '[]', 'standard', 1, 1, ?, ?, ?, ?, 'auto', ?)`
revision, next_build_number, created_by, updated_by, created_at, updated_at, kind, last_used_at, region)
VALUES (?, ?, ?, ?, '', '[]', 'standard', 1, 1, ?, ?, ?, ?, 'auto', ?, ?)`
)
.bind(id, accountId, repoDir(url), JSON.stringify([{ url, defaultBranch: null }]), AUTO_ENVIRONMENT_USER,
AUTO_ENVIRONMENT_USER, now, now, now)
AUTO_ENVIRONMENT_USER, now, now, now, environmentRegion(deps.config, region))
.run()
const environment = await deps.db
.prepare('SELECT * FROM environments WHERE id = ? AND deleted_at IS NULL')
@@ -0,0 +1,94 @@
// CODEX-RELAY: chatgpt.com refuses Cloudflare Workers' egress (a 403 "Unable to load site" page), so
// the egress proxy sends its chatgpt.com requests through the ChatGPT relay on Cloud Run
// (cloud/apps/chatgpt-relay), which ChatGPT answers. The Worker still decides everything: which
// credential, which paths. The relay pins the paths again and only ever talks to chatgpt.com.
// Auth: Cloud Run IAM. The Worker signs in as one service account that holds only run.invoker on the
// relay, and sends a Google ID token for the relay's URL in X-Serverless-Authorization, so the
// request's own Authorization (the ChatGPT token) passes through untouched.
// CHATGPT_RELAY_URL https://<service>.run.app (unset: chatgpt.com directly, as before)
// CHATGPT_RELAY_SERVICE_ACCOUNT (secret) service-account JSON with run.invoker on that service only
import { SignJWT, importPKCS8 } from 'jose'
import { z } from 'zod'
const TOKEN_URL = 'https://oauth2.googleapis.com/token'
export const RELAYED_HOSTS: ReadonlySet<string> = new Set(['chatgpt.com'])
const ServiceAccount = z.looseObject({ client_email: z.string().min(3).max(320), private_key: z.string().min(100) })
const IdTokenResponse = z.looseObject({ id_token: z.string().min(20) })
export class RelayAuthError extends Error {}
/** A Google ID token for `audience`, minted from the service account, cached until 5 minutes before expiry. */
export function createIdTokenSource(config: { serviceAccountJson: string; audience: string; fetch?: typeof fetch; now?: () => number }) {
const doFetch = config.fetch ?? fetch
const now = config.now ?? Date.now
let cached: { value: string; expiresAt: number } | null = null
let pending: Promise<string> | null = null
let signingKey: CryptoKey | null = null
async function mint(): Promise<string> {
let account: z.infer<typeof ServiceAccount>
try {
account = ServiceAccount.parse(JSON.parse(config.serviceAccountJson))
signingKey ??= await importPKCS8(account.private_key, 'RS256')
} catch {
throw new RelayAuthError('relay service account is malformed')
}
const iat = Math.floor(now() / 1000) - 30
const assertion = await new SignJWT({ target_audience: config.audience })
.setProtectedHeader({ alg: 'RS256', typ: 'JWT' })
.setIssuer(account.client_email)
.setSubject(account.client_email)
.setAudience(TOKEN_URL)
.setIssuedAt(iat)
.setExpirationTime(iat + 600)
.sign(signingKey)
const response = await doFetch(TOKEN_URL, {
method: 'POST',
headers: { 'content-type': 'application/x-www-form-urlencoded' },
body: new URLSearchParams({ grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer', assertion }).toString(),
signal: AbortSignal.timeout(10_000)
})
const parsed = IdTokenResponse.safeParse(await response.json().catch(() => null))
if (!response.ok || !parsed.success) {
throw new RelayAuthError(`relay id token mint failed (${response.status})`)
}
// Google ID tokens last an hour; read exp rather than assume it.
const payload = JSON.parse(atob(parsed.data.id_token.split('.')[1]!.replace(/-/g, '+').replace(/_/g, '/'))) as { exp?: number }
cached = { value: parsed.data.id_token, expiresAt: (payload.exp ?? Math.floor(now() / 1000) + 3600) * 1000 }
return cached.value
}
return async (): Promise<string> => {
if (cached && cached.expiresAt - now() > 5 * 60_000) {
return cached.value
}
pending ??= mint().finally(() => {
pending = null
})
return pending
}
}
/**
* The egress proxy's upstream fetch: chatgpt.com through the relay when one is configured, every other host
* (and chatgpt.com without a relay) directly. Same request, same path and query; only the origin changes.
*/
export function relayingFetch(config: { relayUrl?: string; idToken?: () => Promise<string>; fetch?: typeof fetch }): typeof fetch {
const doFetch = config.fetch ?? fetch
const relay = config.relayUrl ? new URL(config.relayUrl) : null
return (async (input: RequestInfo | URL, init?: RequestInit) => {
const target = new URL(typeof input === 'string' || input instanceof URL ? input : input.url)
if (!relay || !config.idToken || target.protocol !== 'https:' || !RELAYED_HOSTS.has(target.hostname)) {
return doFetch(input, init)
}
let idToken: string
try {
idToken = await config.idToken()
} catch {
// In ChatGPT's error shape (a JSON detail), so Codex prints it instead of retrying a bare 5xx.
console.error(JSON.stringify({ event: 'chatgpt_relay_auth_failed' }))
return Response.json({ detail: 'Orca Cloud couldn’t reach ChatGPT just now. Try again in a minute.' }, { status: 503 })
}
const headers = new Headers(init?.headers)
headers.set('x-serverless-authorization', `Bearer ${idToken}`)
return doFetch(`${relay.origin}${target.pathname}${target.search}`, { ...init, headers })
}) as typeof fetch
}
@@ -11,6 +11,9 @@
// (that is what `refresh_token_reused` punishes).
import { decodeJwt } from 'jose'
import type { AccountProvider } from './account-providers'
import { codexPathAllowed } from './codex-paths'
export { codexPathAllowed }
import { credentialBinding, putAgentCredential, readUsableAgentCredential } from './agent-credentials'
import type { Keyring } from './secrets-crypto'
import type { SandboxRow } from './store'
@@ -118,32 +121,6 @@ function formatCodex(credential: string): string {
return `Bearer ${stored ? stored.access_token : credential}`
}
/**
* The ChatGPT backend calls Codex makes with its login (seen live, Codex 0.166): the Codex API itself, its
* plugin catalog and usage. Nothing else on chatgpt.com (conversations, memories, connectors) gets the token.
*/
const CODEX_PREFIXES = ['/backend-api/codex/']
const CODEX_EXACT_GETS = new Set([
'/backend-api/ps/plugins/list',
'/backend-api/ps/plugins/installed',
'/backend-api/ps/plugins/suggested/codex',
'/backend-api/plugins/featured',
'/backend-api/plugins/export/curated',
'/backend-api/wham/usage'
])
export function codexPathAllowed(method: string, path: string): boolean {
const pathname = path.split('?')[0] ?? ''
// Plain segments only: no encoded or dot segments that could walk out of an allowed prefix.
if (!/^(\/[A-Za-z0-9_~-][A-Za-z0-9._~-]*)+\/?$/.test(pathname) || /(^|\/)\.\.?(\/|$)/.test(pathname)) {
return false
}
if (CODEX_PREFIXES.some((prefix) => pathname.startsWith(prefix))) {
return true
}
return method.toUpperCase() === 'GET' && CODEX_EXACT_GETS.has(pathname)
}
export const codexAccount: AccountProvider = {
agent: 'codex',
hosts: CODEX_ACCOUNT_HOSTS,
@@ -183,7 +160,7 @@ export const codexAccount: AccountProvider = {
status: 403,
contentType: 'application/json',
body: JSON.stringify({
detail: 'Orca Cloud can’t use your ChatGPT sign-in here. Check it in Orca: Settings → Agent accounts.'
detail: 'Orca Cloud can’t use your ChatGPT sign-in here. Check it in Orca: Settings → Cloud agent sign-ins.'
})
},
// CODEX-EGRESS-BLOCKED: chatgpt.com answers Cloudflare Workers' egress with a 403 "Unable to load
@@ -0,0 +1,29 @@
// The chatgpt.com paths Codex's login may be used on. Shared, unchanged, by the Worker's egress proxy
// (codex-credentials.ts) and the ChatGPT relay (cloud/apps/chatgpt-relay), which pins the same paths a
// second time. Erasable TypeScript only: the relay runs this file with Node's type stripping.
/**
* The ChatGPT backend calls Codex makes with its login (seen live, Codex 0.166): the Codex API itself, its
* plugin catalog and usage. Nothing else on chatgpt.com (conversations, memories, connectors) gets the token.
*/
const CODEX_PREFIXES = ['/backend-api/codex/']
const CODEX_EXACT_GETS = new Set([
'/backend-api/ps/plugins/list',
'/backend-api/ps/plugins/installed',
'/backend-api/ps/plugins/suggested/codex',
'/backend-api/plugins/featured',
'/backend-api/plugins/export/curated',
'/backend-api/wham/usage'
])
export function codexPathAllowed(method: string, path: string): boolean {
const pathname = path.split('?')[0] ?? ''
// Plain segments only: no encoded or dot segments that could walk out of an allowed prefix.
if (!/^(\/[A-Za-z0-9_~-][A-Za-z0-9._~-]*)+\/?$/.test(pathname) || /(^|\/)\.\.?(\/|$)/.test(pathname)) {
return false
}
if (CODEX_PREFIXES.some((prefix) => pathname.startsWith(prefix))) {
return true
}
return method.toUpperCase() === 'GET' && CODEX_EXACT_GETS.has(pathname)
}
+6
View File
@@ -15,6 +15,8 @@ export type Env = {
VERCEL_REGION?: string
/** Comma list tried in order when the default region's create fails on capacity (default `iad1`; "" = none). */
VERCEL_FALLBACK_REGIONS?: string
/** 'off': new workspaces and environments always start in the default region, not the one nearest the caller. */
VERCEL_NEAREST_REGION?: string
/** Overrides the Vercel API origin (tests only). */
VERCEL_API_BASE?: string
/** Seed golden image (orcad + agent CLIs), adopted when its region has none yet; see image/. */
@@ -90,6 +92,10 @@ export type Env = {
RUNTIME_REFRESH?: string
/** Public URL of this Worker's /v1/egress/proxy; Vercel forwardURL rules point here. */
EGRESS_PROXY_URL?: string
/** CODEX-RELAY (src/chatgpt-relay.ts): the Cloud Run relay chatgpt.com egress goes through. Unset: direct. */
CHATGPT_RELAY_URL?: string
/** (secret) service-account JSON holding run.invoker on CHATGPT_RELAY_URL's service only. */
CHATGPT_RELAY_SERVICE_ACCOUNT?: string
/** Shortened session deadline (ms) for staging rollover rehearsals; refused when SANDBOX_ENV=prod. */
SESSION_DEADLINE_MS?: string
/** Dev-only: Authorization the egress proxy injects for api.github.com until the GitHub App minter exists. */
@@ -162,8 +162,12 @@ export type BuildDeps = {
goldenSnapshotId: string
/** Every region's golden image (staleness compares a build with its own region's). */
goldenSnapshots?: Record<string, string>
/** The default region: new builds run here. */
/** The default region: builds of an environment without a served region of its own run here. */
region: string
/** Served regions, default first (each has a golden image); an environment may live in any of them. */
regions?: readonly string[]
/** A new environment lives in the served region nearest the admin who creates it (off: VERCEL_NEAREST_REGION=off). */
nearestRegion?: boolean
guardrails?: GuardrailConfig
/** The workspace base allowlist (ORCA_PLATFORM_HOST_SUFFIXES included); the smoke boot uses it too. */
egressAllowlist?: readonly string[]
@@ -174,6 +178,24 @@ export type BuildDeps = {
const log = (event: string, fields: Record<string, unknown>) => console.log(JSON.stringify({ event, ...fields }))
type RegionConfig = Pick<BuildDeps['config'], 'region' | 'regions' | 'goldenSnapshotId' | 'goldenSnapshots'>
/**
* Where an environment's builds run: the region it records, while that region is still served (has a
* golden image); else the default. A null region (an environment from before regions were recorded) is the default.
*/
export function environmentRegion(config: RegionConfig, region: string | null | undefined): string {
if (region && region !== config.region && config.regions?.includes(region) && config.goldenSnapshots?.[region]) {
return region
}
return config.region
}
/** The golden image a new build of an environment living in `region` (environmentRegion) starts from. */
export function regionGolden(config: RegionConfig, region: string): string {
return region === config.region ? config.goldenSnapshotId : (config.goldenSnapshots?.[region] ?? config.goldenSnapshotId)
}
function randomId(): string {
const bytes = crypto.getRandomValues(new Uint8Array(10))
return `bld_${Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join('')}`
@@ -303,6 +325,8 @@ export async function startBuild(
}
const now = deps.now()
const id = randomId()
// A snapshot only boots in its own region: the build runs where the environment lives.
const region = environmentRegion(deps.config, environment.region)
const base: BuildConfig = options.config ?? {
revision: environment.revision,
repos,
@@ -336,7 +360,7 @@ export async function startBuild(
WHERE changes() = 1`
)
.bind(id, environment.id, environment.account_id, environment.next_build_number, JSON.stringify(config),
deps.config.goldenSnapshotId, deps.config.region, `orca-${deps.config.environment}-envbuild-${id}`, caller.userId, now, now)
regionGolden(deps.config, region), region, `orca-${deps.config.environment}-envbuild-${id}`, caller.userId, now, now)
])
}
if (bumped[0]?.meta.changes !== 1) {
@@ -15,12 +15,14 @@
// - an environment below the security floor (environment-security.ts) is rebuilt regardless;
// - refresh builds are metered internally, never charged to the team (reportBuildUsage).
// Staleness is exact: environment_builds.base_snapshot_id is the golden snapshot each build started from.
// Regions (U17): an environment lives in the default region, so a build is compared with the default
// region's golden. An older build from another region (iad1 before pdx1 became the default) is stale
// by that rule and is rebuilt in the default region; its workspaces boot in the build's own region
// until the new build is active.
// Regions (U17): an environment lives in its recorded region while that region is served, else the
// default (environmentRegion), so a build is compared with that region's golden. A build from another
// region (iad1 before pdx1 became the default, for an environment that records none) is stale by that
// rule and is rebuilt where the environment lives; its workspaces boot in the build's own region until
// the new build is active.
import type { SandboxCaller } from './auth'
import {
environmentRegion,
GOLDEN_REFRESH_USER,
parseConfig,
startBuild,
@@ -53,13 +55,42 @@ type StaleRow = EnvironmentRow & {
active_created_at: number
}
// The golden a new build of e starts from: its region's (?1, region -> golden JSON of the served
// regions), else the default region's (?2). The same rule as environmentRegion + regionGolden.
const TARGET_GOLDEN = `coalesce(json_extract(?1, '$.' || e.region), ?2)`
const STALE_SQL = `SELECT e.*, b.config AS active_config, b.base_snapshot_id AS active_base, b.created_at AS active_created_at
FROM environments e JOIN environment_builds b ON b.id = e.active_build_id
WHERE e.deleted_at IS NULL AND b.base_snapshot_id != ?
WHERE e.deleted_at IS NULL AND b.base_snapshot_id != ${TARGET_GOLDEN}
AND NOT EXISTS (
SELECT 1 FROM environment_builds x WHERE x.environment_id = e.id
AND (x.status IN ('starting', 'running', 'snapshotting')
OR (x.base_snapshot_id = ? AND x.status != 'canceled')))`
OR (x.base_snapshot_id = ${TARGET_GOLDEN} AND x.status != 'canceled')))`
/** ?1 of TARGET_GOLDEN: each served region other than the default, with its golden. */
function servedGoldens(config: BuildDeps['config']): string {
const goldens: Record<string, string> = {}
for (const region of config.regions ?? []) {
const golden = config.goldenSnapshots?.[region]
if (region !== config.region && golden) {
goldens[region] = golden
}
}
return JSON.stringify(goldens)
}
/** The security floor of the region each environment builds in, read once per region. */
function floors(deps: BuildDeps): (environment: EnvironmentRow) => Promise<number> {
const cache = new Map<string, Promise<number>>()
return (environment) => {
const region = environmentRegion(deps.config, environment.region)
let floor = cache.get(region)
if (!floor) {
floor = securityFloor(deps.db, region)
cache.set(region, floor)
}
return floor
}
}
/** Why a stale environment rebuilds now ('security' | 'due'), or null to wait. */
async function refreshReason(
@@ -95,13 +126,15 @@ export async function refreshStaleEnvironments(
deps: BuildDeps,
options: GoldenRefreshOptions = {}
): Promise<GoldenRefreshTick> {
const golden = deps.config.goldenSnapshotId
const maxStarts = options.maxStartsPerTick ?? 2
const maxInFlight = options.maxInFlight ?? 3
const minIntervalMs = options.minIntervalMs ?? DEFAULT_REFRESH_MIN_INTERVAL_MS
const recentMs = options.recentMs ?? DEFAULT_REFRESH_RECENT_MS
const { results: stale } = await deps.db.prepare(`${STALE_SQL} ORDER BY e.updated_at DESC`).bind(golden, golden).all<StaleRow>()
const floor = await securityFloor(deps.db, deps.config.region)
const { results: stale } = await deps.db
.prepare(`${STALE_SQL} ORDER BY e.updated_at DESC`)
.bind(servedGoldens(deps.config), deps.config.goldenSnapshotId)
.all<StaleRow>()
const floorOf = floors(deps)
const inFlight =
(await deps.db
.prepare(`SELECT count(*) AS n FROM environment_builds WHERE status IN ('starting', 'running', 'snapshotting')`)
@@ -110,7 +143,7 @@ export async function refreshStaleEnvironments(
const ranked: { environment: StaleRow; reason: 'security' | 'due' }[] = []
let deferred = 0
for (const environment of stale) {
const reason = await refreshReason(deps, environment, floor, { minIntervalMs, recentMs })
const reason = await refreshReason(deps, environment, await floorOf(environment), { minIntervalMs, recentMs })
if (reason) {
ranked.push({ environment, reason })
} else {
@@ -147,15 +180,14 @@ export async function refreshEnvironment(
options: Pick<GoldenRefreshOptions, 'minIntervalMs'> = {}
): Promise<'started' | 'current' | 'deferred' | 'skipped'> {
try {
const golden = deps.config.goldenSnapshotId
const environment = await deps.db
.prepare(`${STALE_SQL} AND e.id = ?`)
.bind(golden, golden, environmentId)
.prepare(`${STALE_SQL} AND e.id = ?3`)
.bind(servedGoldens(deps.config), deps.config.goldenSnapshotId, environmentId)
.first<StaleRow>()
if (!environment) {
return 'current'
}
const floor = await securityFloor(deps.db, deps.config.region)
const floor = await securityFloor(deps.db, environmentRegion(deps.config, environment.region))
// Used right now, so no recency test: only the interval (or the floor) decides.
const reason = await refreshReason(deps, environment, floor, {
minIntervalMs: options.minIntervalMs ?? DEFAULT_REFRESH_MIN_INTERVAL_MS
@@ -26,6 +26,7 @@ import {
type BuildRow
} from './environment-builds'
import { checkStaleness } from './environment-staleness'
import { nearestRegions } from './regions'
import {
CreateEnvironmentSchema,
EnvironmentError,
@@ -135,7 +136,9 @@ export async function handleEnvironmentRoute(
if (!body.success) {
return json({ error: 'invalid_request', reason: body.error.issues[0]?.path.join('.') }, 400)
}
return json({ environment: environmentView(await createEnvironment(db, caller, body.data, deps.now())) })
// Its builds run, and its workspaces boot, where it lives: the served region nearest whoever creates it.
const region = nearestRegions(request, deps.config.regions ?? [], deps.config.nearestRegion !== false)?.[0] ?? null
return json({ environment: environmentView(await createEnvironment(db, caller, body.data, deps.now(), region)) })
}
return json({ error: 'not_found' }, 404)
}
@@ -2,7 +2,7 @@
// call per repo answers both "the branch moved" and "a lockfile changed" (installed deps are stale).
// Uses the GitHub App token when the repo is bound, anonymous otherwise (public repos). Cached briefly
// per isolate so a settings page polling it costs GitHub nothing extra.
import { LOCKFILES, getBuild, parseConfig, parseResult, type BuildDeps } from './environment-builds'
import { LOCKFILES, environmentRegion, getBuild, parseConfig, parseResult, regionGolden, type BuildDeps } from './environment-builds'
import { parseRepos, type EnvironmentRow } from './environments'
import { parseGitHubRepo } from './github-app'
@@ -37,7 +37,7 @@ export type Staleness = {
const LOCKFILE_NAMES = new Set(LOCKFILES)
export async function checkStaleness(
deps: Pick<BuildDeps, 'db' | 'github' | 'now'> & { githubFetch?: typeof fetch; config?: { goldenSnapshotId: string } },
deps: Pick<BuildDeps, 'db' | 'github' | 'now'> & { githubFetch?: typeof fetch; config?: Parameters<typeof regionGolden>[0] },
environment: EnvironmentRow,
refresh = false
): Promise<Staleness> {
@@ -54,7 +54,9 @@ export async function checkStaleness(
const config = parseConfig(build)
const result = parseResult(build)
const configChanged = config.revision !== environment.revision
const imageUpdated = !!deps.config && build.base_snapshot_id !== deps.config.goldenSnapshotId
// Compared with the golden of the region the environment builds in (its own, or the default).
const imageUpdated =
!!deps.config && build.base_snapshot_id !== regionGolden(deps.config, environmentRegion(deps.config, environment.region))
const repos: RepoStaleness[] = []
for (const repo of parseRepos(environment)) {
const built = result?.repos?.find((r) => r.url === repo.url)
@@ -37,6 +37,8 @@ export type EnvironmentRow = {
*/
kind?: 'org' | 'auto'
last_used_at?: number | null
/** Where it builds and its workspaces boot (environmentRegion); null for environments made before this was recorded. */
region?: string | null
}
export class EnvironmentError extends Error {
@@ -173,7 +175,9 @@ export async function createEnvironment(
db: D1Database,
caller: SandboxCaller,
input: z.infer<typeof CreateEnvironmentSchema>,
now: number
now: number,
/** The served region nearest the admin creating it; null leaves it to the default region. */
region: string | null = null
): Promise<EnvironmentRow> {
requireAdmin(caller)
const repos = checkedRepos(input.repos)
@@ -188,11 +192,11 @@ export async function createEnvironment(
await db
.prepare(
`INSERT INTO environments (id, account_id, name, repos, install_script, env_var_names, machine_size,
revision, next_build_number, created_by, updated_by, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, 1, 1, ?, ?, ?, ?)`
revision, next_build_number, created_by, updated_by, created_at, updated_at, region)
VALUES (?, ?, ?, ?, ?, ?, ?, 1, 1, ?, ?, ?, ?, ?)`
)
.bind(id, caller.accountId, input.name, JSON.stringify(repos), input.installScript,
JSON.stringify([...new Set(input.envVarNames)]), input.machineSize, caller.userId, caller.userId, now, now)
JSON.stringify([...new Set(input.envVarNames)]), input.machineSize, caller.userId, caller.userId, now, now, region)
.run()
return (await getEnvironment(db, caller.accountId, id))!
}
@@ -26,6 +26,11 @@ export type GitHubAppMinter = {
getInstallation(installationId: number): Promise<GitHubInstallation | null>
/** The installation covering a github.com repo URL, or null. */
getRepoInstallation(repoUrl: string): Promise<GitHubInstallation | null>
/**
* The App's installation on a GitHub org or user (whatever repositories it selected), or null.
* Tells "not installed on the owner" from "installed, but this repository isn't selected".
*/
getOwnerInstallation?(owner: string): Promise<GitHubInstallation | null>
/** App-JWT request to an app-level endpoint (operator: /app, /app/hook/config). */
appRequest?(method: string, path: '/app' | '/app/hook/config', body?: unknown): Promise<Response>
/**
@@ -262,6 +267,14 @@ export function createGitHubAppMinter(config: {
return github(method, path, body)
},
async getOwnerInstallation(owner) {
if (!/^[A-Za-z0-9-]{1,39}$/.test(owner)) {
return null
}
// Orgs and users have separate lookups; an org login 404s on /users/{login}/installation.
return (await installationFor(`/orgs/${owner}/installation`)) ?? installationFor(`/users/${owner}/installation`)
},
async getRepoInstallation(repoUrl) {
const parsed = parseGitHubRepo(repoUrl)
return parsed ? installationFor(`/repos/${parsed.owner}/${parsed.repo}/installation`) : null
@@ -0,0 +1,109 @@
// CREATE-GH-STEP: GitHub's installation lookups for repo-access, cached in D1 so every isolate shares
// them and the create wizard's 2 s poll doesn't spend an app-JWT call each time. A found installation
// is kept a minute, a 404 for 15 s. The installation webhooks and the setup return drop the owner's
// entries (forgetOwner/forgetInstallation), so an install is seen on the next poll, not after the TTL.
// Only ids and the owner's login are stored. Any D1 failure falls through to GitHub.
import type { GitHubAppMinter, GitHubInstallation } from './github-app'
export const FOUND_TTL_MS = 60_000
export const MISSING_TTL_MS = 15_000
type Row = {
installation_id: number | null
github_account_id: number | null
github_account_login: string | null
github_account_type: string | null
checked_at: number
}
function fromRow(row: Row): GitHubInstallation | null {
if (row.installation_id === null || row.github_account_login === null) {
return null
}
return {
id: row.installation_id,
account: {
id: row.github_account_id ?? 0,
login: row.github_account_login,
type: row.github_account_type === 'User' ? 'User' : 'Organization'
}
}
}
async function cached(
db: D1Database,
key: string,
ownerKey: string,
now: number,
lookup: () => Promise<GitHubInstallation | null>
): Promise<GitHubInstallation | null> {
const row = await db
.prepare(
'SELECT installation_id, github_account_id, github_account_login, github_account_type, checked_at FROM github_installation_cache WHERE cache_key = ?'
)
.bind(key)
.first<Row>()
.catch(() => null)
if (row && now - row.checked_at < (row.installation_id === null ? MISSING_TTL_MS : FOUND_TTL_MS)) {
return fromRow(row)
}
// GitHub errors propagate uncached (repo-access answers 502; the next poll asks again).
const found = await lookup()
await db
.prepare(
`INSERT INTO github_installation_cache (cache_key, owner_key, installation_id, github_account_id, github_account_login, github_account_type, checked_at)
VALUES (?, ?, ?, ?, ?, ?, ?)
ON CONFLICT (cache_key) DO UPDATE SET installation_id = excluded.installation_id, github_account_id = excluded.github_account_id,
github_account_login = excluded.github_account_login, github_account_type = excluded.github_account_type, checked_at = excluded.checked_at`
)
.bind(key, ownerKey, found?.id ?? null, found?.account.id ?? null, found?.account.login ?? null, found?.account.type ?? null, now)
.run()
.catch(() => undefined)
return found
}
/** GET /repos/{owner}/{repo}/installation, cached. */
export function cachedRepoInstallation(
db: D1Database,
github: GitHubAppMinter,
parsed: { owner: string; repo: string },
now: number
): Promise<GitHubInstallation | null> {
const owner = parsed.owner.toLowerCase()
return cached(db, `${owner}/${parsed.repo.toLowerCase()}`, owner, now, () =>
github.getRepoInstallation(`https://github.com/${parsed.owner}/${parsed.repo}`)
)
}
/** The App's installation on the owner (org or user), cached; null when the minter can't ask. */
export function cachedOwnerInstallation(
db: D1Database,
github: GitHubAppMinter,
owner: string,
now: number
): Promise<GitHubInstallation | null> {
const lookup = github.getOwnerInstallation
if (!lookup) {
return Promise.resolve(null)
}
const key = owner.toLowerCase()
return cached(db, `${key}/`, key, now, () => lookup.call(github, owner))
}
/** An install, uninstall, or repository selection change on this owner: ask GitHub next time. */
export async function forgetOwner(db: D1Database, owner: string): Promise<void> {
await db
.prepare('DELETE FROM github_installation_cache WHERE owner_key = ?')
.bind(owner.toLowerCase())
.run()
.catch(() => undefined)
}
/** Same, by installation id (a webhook whose account login is missing). */
export async function forgetInstallation(db: D1Database, installationId: number): Promise<void> {
await db
.prepare('DELETE FROM github_installation_cache WHERE installation_id = ?')
.bind(installationId)
.run()
.catch(() => undefined)
}
@@ -6,6 +6,12 @@
// installation): true = the caller can install it on the owner (it is their own GitHub account, or
// they are an active admin of the owner org); false = another user's account, or a non-admin org
// membership; null = unknown (no connected GitHub user token, membership unreadable, GitHub down).
// push (CREATE-GH-STEP): can a workspace on this repo push? 'ok' = granted; 'not_installed' = the
// App isn't on the owner; 'repo_not_selected' = it is, but this repository isn't one it selected
// (configureUrl: the installation's settings page); 'not_connected' = the App reaches the repo
// but through an installation this Orca account isn't bound to. installUrl is present for every
// push reason but 'ok' and 'repo_not_selected', public repositories included.
// GitHub's installation lookups are cached in D1 (github-installation-cache.ts).
// GET /v1/github/repositories[?cursor=] -> {repositories, nextCursor} (start from GitHub, D4/N12)
// GET /v1/internal/github/install-url?accountId&userId&repo&session (billing's setup chain,
// Bearer CONTROL_PLANE_TOKEN) -> {access, url}
@@ -16,6 +22,7 @@
// its own: POST <billing>/v1/internal/cloud-setup/github {sessionId?, accountId, repo, state}.
import type { GitHubAppMinter, GitHubListedRepo } from './github-app'
import { parseGitHubRepo } from './github-app'
import { cachedOwnerInstallation, cachedRepoInstallation, forgetInstallation, forgetOwner } from './github-installation-cache'
import {
bindApprovedRequests,
boundAccounts,
@@ -30,6 +37,9 @@ import {
export type RepoAccess = 'public' | 'granted' | 'missing' | 'requested'
/** Whether a workspace on the repository can push, and if not, what's missing (CREATE-GH-STEP). */
export type PushAccess = 'ok' | 'not_installed' | 'repo_not_selected' | 'not_connected'
export type RepoAccessDeps = {
db: D1Database
github: GitHubAppMinter | undefined
@@ -52,25 +62,93 @@ export async function repoAccess(
accountId: string,
repoUrl: string
): Promise<{ access: RepoAccess; owner: string; installationId: number | null }> {
const { access, owner, installationId } = await repoAccessWithPush(deps, accountId, repoUrl)
return { access, owner, installationId }
}
/** repoAccess plus whether a workspace can push, and the owner installation when that's the gap. */
export async function repoAccessWithPush(
deps: RepoAccessDeps,
accountId: string,
repoUrl: string
): Promise<{
access: RepoAccess
owner: string
installationId: number | null
push: PushAccess
ownerInstallation?: { id: number; type: 'User' | 'Organization' }
}> {
const parsed = parseGitHubRepo(repoUrl)
if (!parsed) {
throw new RepoAccessError('not_github')
}
const installation = deps.github ? await deps.github.getRepoInstallation(repoUrl) : null
const installation = deps.github ? await cachedRepoInstallation(deps.db, deps.github, parsed, deps.now) : null
if (installation && (await isInstallationBound(deps.db, installation.id, accountId))) {
return { access: 'granted', owner: parsed.owner, installationId: installation.id }
return { access: 'granted', owner: parsed.owner, installationId: installation.id, push: 'ok' }
}
if (await deps.publicRepoCheck(repoUrl)) {
return { access: 'public', owner: parsed.owner, installationId: installation?.id ?? null }
const [gap, isPublic] = await Promise.all([pushGap(deps, accountId, parsed.owner, installation), deps.publicRepoCheck(repoUrl)])
if (isPublic) {
return { access: 'public', owner: parsed.owner, installationId: installation?.id ?? null, ...gap }
}
if (!installation && (await openInstallRequest(deps.db, accountId, parsed.owner, deps.now))) {
// Ours, and still open on GitHub (an owner who declines leaves no webhook, only an empty list).
const pending = await deps.github?.pendingRequestOwners?.().catch(() => null)
if (!pending || pending.has(parsed.owner.toLowerCase())) {
return { access: 'requested', owner: parsed.owner, installationId: null }
return { access: 'requested', owner: parsed.owner, installationId: null, ...gap }
}
}
return { access: 'missing', owner: parsed.owner, installationId: installation?.id ?? null }
return { access: 'missing', owner: parsed.owner, installationId: installation?.id ?? null, ...gap }
}
/**
* Not granted: the App reaches the repo through someone else's binding, or it's on the owner without
* this repository, or it isn't on the owner. Only an installation bound to this account counts as
* "repo not selected" without asking GitHub; otherwise one cached owner lookup tells.
*/
async function pushGap(
deps: RepoAccessDeps,
accountId: string,
owner: string,
repoInstallation: { id: number } | null
): Promise<{ push: PushAccess; ownerInstallation?: { id: number; type: 'User' | 'Organization' } }> {
if (repoInstallation) {
return { push: 'not_connected' }
}
const bound = (await listAccountInstallations(deps.db, accountId)).find(
(row) => row.github_account_login.toLowerCase() === owner.toLowerCase()
)
if (bound) {
return {
push: 'repo_not_selected',
ownerInstallation: { id: bound.installation_id, type: bound.github_account_type === 'User' ? 'User' : 'Organization' }
}
}
// Never holds the answer up: GitHub erroring or slow reads as "not installed", whose install page
// GitHub turns into the installation's settings when the App is already there.
const onOwner = deps.github
? await withTimeout(cachedOwnerInstallation(deps.db, deps.github, owner, deps.now), OWNER_LOOKUP_TIMEOUT_MS).catch(() => null)
: null
if (!onOwner) {
return { push: 'not_installed' }
}
// On the owner, but bound to another Orca account (or none): binding it is the install round trip.
return (await isInstallationBound(deps.db, onOwner.id, accountId))
? { push: 'repo_not_selected', ownerInstallation: { id: onOwner.id, type: onOwner.account.type } }
: { push: 'not_connected' }
}
const OWNER_LOOKUP_TIMEOUT_MS = 3_000
function withTimeout<T>(work: Promise<T>, ms: number): Promise<T | null> {
let timer: ReturnType<typeof setTimeout> | undefined
return Promise.race([work, new Promise<null>((resolve) => (timer = setTimeout(() => resolve(null), ms)))]).finally(() => clearTimeout(timer))
}
/** GitHub's page where the installation's repository selection is changed. */
export function installationSettingsUrl(owner: string, installation: { id: number; type: 'User' | 'Organization' }): string {
return installation.type === 'User'
? `https://github.com/settings/installations/${installation.id}`
: `https://github.com/organizations/${encodeURIComponent(owner)}/settings/installations/${installation.id}`
}
export class RepoAccessError extends Error {
@@ -150,12 +228,17 @@ export async function repoAccessView(
if (!parsed) {
return { status: 400, body: { error: 'invalid_request', reason: 'not_github' } }
}
const result = await repoAccess(deps, caller.accountId, repo)
const body: Record<string, unknown> = { access: result.access, owner: parsed.owner, repo: canonicalRepoUrl(parsed) }
const result = await repoAccessWithPush(deps, caller.accountId, repo)
const body: Record<string, unknown> = { access: result.access, owner: parsed.owner, repo: canonicalRepoUrl(parsed), push: result.push }
// Only while the App isn't installed for this owner: that is when the desktop picks "Install" vs "ask an owner".
const askCanInstall = result.access === 'missing' || (result.access === 'public' && result.installationId === null)
// A public repository clones anonymously but can't push: the wizard's GitHub step installs from here too.
const wantsInstall = result.access === 'missing' || (result.access === 'public' && (result.push === 'not_installed' || result.push === 'not_connected'))
if (result.push === 'repo_not_selected' && result.ownerInstallation) {
body.configureUrl = installationSettingsUrl(parsed.owner, result.ownerInstallation)
}
const [install, canInstall] = await Promise.all([
result.access === 'missing' && deps.connect
wantsInstall && deps.connect
? repoInstallUrl(deps, caller, repo, {
setupSessionId: url.searchParams.get('session') ?? undefined,
repoId: positiveInt(url.searchParams.get('repoId')),
@@ -342,6 +425,13 @@ export async function handleInstallationEvent(
if (!id || !Number.isSafeInteger(id)) {
return { handled: 'ignored' }
}
if (eventName === 'installation' || eventName === 'installation_repositories') {
// Any install, uninstall, suspend or selection change: repo-access asks GitHub on the next poll.
await forgetInstallation(deps.db, id)
if (installation?.account?.login) {
await forgetOwner(deps.db, installation.account.login)
}
}
if (eventName === 'installation' && payload.action === 'deleted') {
return { handled: 'deleted', revoked: await revokeInstallation(deps.db, id, null, deps.now) }
}
@@ -399,6 +489,11 @@ export async function noticeAfterReturn(
}
const repo = `https://github.com/${state.r}`
const base = { ...(state.s ? { sessionId: state.s } : {}), accountId: state.a, repo }
// Back from GitHub's install page: its webhook may not have landed yet.
const owner = parseGitHubRepo(repo)?.owner
if (owner) {
await forgetOwner(deps.db, owner)
}
if (requested) {
return { ...base, state: 'pending' }
}
@@ -362,13 +362,13 @@ export function entryPointsVersion(): string {
// One round trip: the image's scripts stamp (missing = 0: images from before stamps) and whether
// this Worker's entry points are already installed.
const ENTRY_POINTS_PROBE = [
export const ENTRY_POINTS_PROBE = [
'stamp="$(cat /opt/orca/bin/.scripts-stamp 2>/dev/null || true)"',
'echo "stamp=${stamp:-0}"',
`if [ -e /opt/orca/bin/.entry-points-${VM_SCRIPTS_HASH} ]; then echo marker=1; else echo marker=0; fi`
].join('\n')
function parseEntryPointsProbe(stdout: string): { stamp: number; marker: boolean } | null {
export function parseEntryPointsProbe(stdout: string): { stamp: number; marker: boolean } | null {
const marker = /^marker=([01])$/m.exec(stdout)?.[1]
if (marker === undefined) {
return null
@@ -399,6 +399,16 @@ export async function ensureEntryPoints(deps: { provider: Pick<SandboxProviderAp
await installEntryPoints(deps, sessionId)
}
/**
* ENTRY-POINTS-BEFORE-START: a boot from an environment build made on an older golden would run that
* golden's orca-sandbox-start (no agent prefetch, no OpenCode first run) until the environment is
* rebuilt, hours later. True when the image's scripts are older than this Worker's: no marker for this
* set and a lower stamp (an equal stamp is the same scripts: the stamp only moves when they change).
*/
export function entryPointsBehind(image: { stamp: number; marker: boolean } | null): boolean {
return image !== null && !image.marker && image.stamp < VM_SCRIPTS_STAMP
}
export async function installEntryPoints(deps: { provider: Pick<SandboxProviderApi, 'runCommand'> }, sessionId: string): Promise<void> {
const env = provisionEnv({ manifest: MANIFEST, provisionScript: PROVISION_SCRIPT, vmScripts: VM_SCRIPT_SOURCES, vmScriptsStamp: VM_SCRIPTS_STAMP, buildId: 'sync' })
const scriptEnv = Object.fromEntries(Object.entries(env).filter(([key]) => key.startsWith('ORCA_VM_SCRIPT_')))
+67 -8
View File
@@ -64,8 +64,9 @@ import { createRoutinesApi } from './routines/api'
import { handleRoutineHook, isRoutineHookPath } from './routines/hooks'
import { tickRoutines } from './routines/runs'
import { activeGoldenSnapshots, advanceGoldenImages } from './golden-images'
import { sandboxRegions, seedRegion } from './regions'
import { nearestRegionEnabled, nearestRegions, sandboxRegions, seedRegion, servedRegions } from './regions'
import { handleRuntimeBundleRoute, runtimeBundleFor, runtimeBundleUrl } from './runtime-bundles'
import { createIdTokenSource, relayingFetch } from './chatgpt-relay'
import { NO_CREDENTIALS, handleEgressProxy, type CredentialBroker, type ProxiedRoute } from './egress-proxy'
import {
BillingUnavailableError,
@@ -891,6 +892,34 @@ function fail(error: string, status: number, reason?: string): Response {
return json(reason ? { error, reason } : { error }, status)
}
/** CREATE-STEP-WATCH: logs a step still pending after 10 s and, once the create starts, every step's time. */
export function createStepWatch(enabled: boolean, now: () => number = Date.now) {
const started = now()
const steps: Record<string, number> = {}
let reported = false
return {
async step<T>(name: string, work: Promise<T>): Promise<T> {
if (!enabled) {
return work
}
const at = now()
const slow = setTimeout(() => console.log(JSON.stringify({ event: 'create_step_slow', step: name, ms: now() - at })), 10_000)
try {
return await work
} finally {
clearTimeout(slow)
steps[name] = now() - at
}
},
done() {
if (enabled && !reported) {
reported = true
console.log(JSON.stringify({ event: 'create_steps', ms: now() - started, steps }))
}
}
}
}
async function readJson(request: Request, maxBytes = MAX_JSON_BYTES): Promise<unknown> {
const text = await request.text()
if (text.length > maxBytes) {
@@ -982,7 +1011,7 @@ export async function lifecycleDeps(env: Env, deps: Deps): Promise<LifecycleDeps
// AUTO_ENVIRONMENTS=off: creates of repos without an environment keep cloning from scratch.
if (env.AUTO_ENVIRONMENTS !== 'off') {
life.autoEnvironment = {
ensure: (accountId, repoUrl) => ensureAutoEnvironment(buildDeps(life, deps, env), accountId, repoUrl),
ensure: (accountId, repoUrl, region) => ensureAutoEnvironment(buildDeps(life, deps, env), accountId, repoUrl, region),
touch: (environmentId) => touchEnvironment(env.SANDBOX_DB, environmentId, life.now())
}
}
@@ -1051,6 +1080,8 @@ function buildDeps(life: LifecycleDeps, deps: Deps, env?: Env): BuildDeps {
goldenSnapshotId: life.config.goldenSnapshotId,
goldenSnapshots: life.config.goldenSnapshots,
region: life.config.defaultRegion,
regions: servedRegions(life.config),
nearestRegion: env ? nearestRegionEnabled(env) : true,
guardrails: life.config.guardrails,
egressAllowlist: life.config.egressAllowlist,
egressAllowlistError: life.config.egressAllowlistError
@@ -1318,7 +1349,7 @@ async function routeRequest(request: Request, env: Env, deps: Deps): Promise<Res
projectId: env.VERCEL_PROJECT_ID,
findSandbox: (name) => getSandboxByProviderName(env.SANDBOX_DB, name),
routesFor: egressRoutes(env, deps),
fetch: deps.egressFetch,
fetch: chatgptRelayFetch(env, deps.egressFetch),
keysForIssuer: deps.egressKeysForIssuer
})
}
@@ -1807,11 +1838,13 @@ async function routeRequest(request: Request, env: Env, deps: Deps): Promise<Res
if (!match) {
return fail('not_found', 404)
}
const caller = await authenticateDesktop(env, request, (deps.now ?? Date.now)())
// CREATE-STEP-WATCH: creates hung 180 s with no log and no row (n73). Name the step a create waits in.
const watch = createStepWatch(request.method === 'POST' && !match[1])
const caller = await watch.step('auth', authenticateDesktop(env, request, (deps.now ?? Date.now)()))
if (!caller) {
return fail('unauthorized', 401)
}
const life = await lifecycleDeps(env, deps)
const life = await watch.step('deps', lifecycleDeps(env, deps))
if (!life) {
return fail('not_configured', 503)
}
@@ -1834,12 +1867,12 @@ async function routeRequest(request: Request, env: Env, deps: Deps): Promise<Res
if (request.method !== 'POST') {
return fail('not_found', 404)
}
if (await tokenMintedBeforeMemberChange(life.db, caller)) {
if (await watch.step('stale_token', tokenMintedBeforeMemberChange(life.db, caller))) {
// ORG-TOKEN-STALE: never create (and bill) with a token minted before a role change.
return fail('unauthorized', 401, 'stale_token')
}
await consumeCreateToken(life.db, life.config.guardrails!, caller.accountId, life.now())
const raw = await readJson(request)
await watch.step('create_token', consumeCreateToken(life.db, life.config.guardrails!, caller.accountId, life.now()))
const raw = await watch.step('body', readJson(request))
const body = CreateRequestSchema.safeParse(raw)
if (!body.success) {
return fail('invalid_request', 400)
@@ -1857,12 +1890,18 @@ async function routeRequest(request: Request, env: Env, deps: Deps): Promise<Res
life.db,
{ accountId: caller.accountId, scope: `sandbox.create:${caller.userId}`, key: readIdempotencyKey(request), body: raw, now: life.now() },
async () => {
watch.done()
// Desktops that show the clone as its own step ask for it in the background (GET .../clone).
// Member identities install after the response; the VM's git holds commits and pushes until then.
let ready: Ready
try {
// Nothing pinned: start in the served region nearest the caller (an environment's own region still wins).
const nearRegions = body.data.region ? undefined : nearestRegions(request, servedRegions(life.config), nearestRegionEnabled(env))
// Region order only (never the caller's location): which region a create preferred, and why.
console.log(JSON.stringify({ event: 'create_region_order', explicit: body.data.region ?? null, order: nearRegions ?? null }))
ready = await createSandbox(life, caller, {
...body.data,
...(nearRegions ? { nearRegions } : {}),
cloneInBackground: features.includes('clone-progress'),
deferMemberGit: true
})
@@ -2863,3 +2902,23 @@ async function agentSourcesRoute(request: Request, url: URL, env: Env, deps: Dep
throw error
}
}
// CODEX-RELAY: one ID-token source per isolate and relay config (the token is cached for its hour).
let relayIdTokens: { key: string; source: () => Promise<string> } | null = null
function chatgptRelayFetch(env: Env, egressFetch: typeof fetch | undefined): typeof fetch | undefined {
if (!env.CHATGPT_RELAY_URL || !env.CHATGPT_RELAY_SERVICE_ACCOUNT) {
return egressFetch
}
const key = `${env.CHATGPT_RELAY_URL}\n${env.CHATGPT_RELAY_SERVICE_ACCOUNT.length}`
if (relayIdTokens?.key !== key) {
relayIdTokens = {
key,
source: createIdTokenSource({
serviceAccountJson: env.CHATGPT_RELAY_SERVICE_ACCOUNT,
audience: env.CHATGPT_RELAY_URL,
...(egressFetch ? { fetch: egressFetch } : {})
})
}
}
return relayingFetch({ relayUrl: env.CHATGPT_RELAY_URL, idToken: relayIdTokens.source, ...(egressFetch ? { fetch: egressFetch } : {}) })
}
+73 -16
View File
@@ -15,7 +15,7 @@ import {
} from './guardrails'
import { DEFAULT_EGRESS_ALLOWLIST, DEFAULT_EGRESS_PRESET, KEYS_WITHHELD_BODY, compileNetworkPolicy, egressProxyMissing, gitBasicAuthorization, type CredentialRoute, type EgressPreset, type PlatformHostSuffixesConfigError } from './network-policy'
import { applyQueuedRevokes } from './grant-revokes'
import { ensureEntryPoints } from './golden-images'
import { ENTRY_POINTS_PROBE, ensureEntryPoints, entryPointsBehind, installEntryPoints, parseEntryPointsProbe } from './golden-images'
import { RUNTIME_REFRESH_SCRIPT, type RuntimeBundle } from './runtime-bundles'
import { memberPlaceholder, type MemberGitIdentity } from './github-user-tokens'
import { connectedMembers, memberGitConfigs } from './member-git'
@@ -120,7 +120,8 @@ export type LifecycleDeps = {
* lists, `touch` when a create booted from an automatic one. Absent: neither happens.
*/
autoEnvironment?: {
ensure(accountId: string, repoUrl: string): Promise<unknown>
/** `region`: where the create that asked runs; the environment builds there (a snapshot is region-bound). */
ensure(accountId: string, repoUrl: string, region: string): Promise<unknown>
touch(environmentId: string): Promise<unknown>
}
/**
@@ -224,12 +225,31 @@ export function phaseTimer(now: () => number = Date.now) {
failed: null as { phase: string; ms: number } | null,
async time<T>(phase: string, work: () => Promise<T>): Promise<T> {
const started = now()
const limit = PHASE_LIMITS[phase]
let watch: ReturnType<typeof setTimeout> | undefined
try {
return await work()
if (limit === undefined) {
return await work()
}
// A phase with nothing slow in it that still hasn't answered is stuck (a create sat 180 s
// before its write-ahead insert with no provider call stalled). Name it in the log; fail it
// unless abandoning the work could strand something (the pool claim holds a VM).
return await Promise.race([
work(),
new Promise<never>((_, reject) => {
watch = setTimeout(() => {
console.log(JSON.stringify({ event: 'phase_timeout', phase, ms: limit.ms, failed: limit.fail }))
if (limit.fail) {
reject(new LifecycleError('provider_unavailable', `phase_timeout_${phase}`))
}
}, limit.ms)
})
])
} catch (error) {
timer.failed ??= { phase, ms: now() - started }
throw error
} finally {
clearTimeout(watch)
timings[phase] = now() - started
}
}
@@ -237,6 +257,17 @@ export function phaseTimer(now: () => number = Date.now) {
return timer
}
/**
* Bounds for the phases before a create's write-ahead insert (D1 reads, billing, the pool claim).
* The VM, policy and orcad phases are bounded by the provider calls inside them.
*/
export const PHASE_DEADLINES_MS = {
environment: { ms: 20_000, fail: true },
entitlement: { ms: 20_000, fail: true },
pool: { ms: 45_000, fail: false }
}
const PHASE_LIMITS: Partial<Record<string, { ms: number; fail: boolean }>> = PHASE_DEADLINES_MS
/**
* CREATE-BUILD-SNAPSHOT-404: a VM create from an environment build's snapshot can answer 404 just
* after the build (dev, 11-11 and 11-13: the same snapshot booted seconds before and after). Once,
@@ -645,7 +676,9 @@ export const MEMBERS_COMMAND = '/opt/orca/bin/orca-sandbox-members'
/**
* ENV-STALE boot step: a sandbox whose orcad is behind its region's active golden (booted from an environment
* build snapshot, or resumed from an older image) installs the golden's runtime bundle before orcad starts.
* Best effort and quiet: any failure keeps the old orcad, and every outcome is logged with its timing.
* The same command probes the VM's entry points (ENTRY-POINTS-BEFORE-START): scripts older than this
* Worker's are installed before the start script runs, so the common case adds no round trip.
* Best effort and quiet: any failure keeps the old orcad and scripts, and every outcome is logged with its timing.
*/
export async function refreshRuntime(deps: LifecycleDeps, row: SandboxRow, sessionId: string): Promise<string | null> {
if (!deps.runtimeRefresh) {
@@ -654,28 +687,41 @@ export async function refreshRuntime(deps: LifecycleDeps, row: SandboxRow, sessi
const started = deps.now()
let result = 'error'
let target: string | null = null
let entryPoints = 'unknown'
try {
const plan = await deps.runtimeRefresh(row.region, started)
if (!plan) {
result = 'no_bundle'
return result
}
target = plan.bundle.commit
target = plan?.bundle.commit ?? null
const run = await deps.provider.runCommand(sessionId, {
command: 'bash',
args: ['-c', RUNTIME_REFRESH_SCRIPT],
env: { WANT: plan.bundle.commit, SHA: plan.bundle.sha256, RT: plan.bundle.runtime, URL: plan.url },
args: ['-c', plan ? `${ENTRY_POINTS_PROBE}\n${RUNTIME_REFRESH_SCRIPT}` : ENTRY_POINTS_PROBE],
env: plan ? { WANT: plan.bundle.commit, SHA: plan.bundle.sha256, RT: plan.bundle.runtime, URL: plan.url } : {},
sudo: true,
timeoutMs: 40_000
})
result = run.stdout.trim().split('\n').pop() ?? 'error'
result = plan ? (run.stdout.trim().split('\n').pop() ?? 'error') : 'no_bundle'
const image = parseEntryPointsProbe(run.stdout)
entryPoints = entryPointsBehind(image) ? 'behind' : image ? 'current' : 'unknown'
if (entryPoints === 'behind') {
entryPoints = await installEntryPoints(deps, sessionId).then(
() => 'installed',
() => 'install_failed'
)
}
return result
} catch {
return result
} finally {
const [outcome, from] = result.split(' ')
console.log(
JSON.stringify({ event: 'runtime_refresh', sandboxId: row.id, outcome, from: from ?? null, to: target, ms: deps.now() - started })
JSON.stringify({
event: 'runtime_refresh',
sandboxId: row.id,
outcome,
from: from ?? null,
to: target,
entryPoints,
ms: deps.now() - started
})
)
}
}
@@ -768,7 +814,13 @@ export async function createSandbox(
request: {
workspaceId: string
machineSize: MachineSize
/** Explicit (the caller's or its project's): authoritative, and never moved by the capacity fallback. */
region?: string
/**
* Served regions nearest the caller first (regions.ts nearestRegions): where a create with nothing
* pinned starts, and the capacity fallback's order. Absent: the default region, then its fallbacks.
*/
nearRegions?: readonly string[]
repo?: { url: string; ref?: string }
/** Boot from exactly this environment build (bld_...). */
environmentBuildId?: string
@@ -829,7 +881,12 @@ export async function createSandbox(
const machineSize = source && !source.auto ? source.machineSize : request.machineSize
await timer.time('entitlement', () => requireStart(deps, caller.accountId, caller.userId, machineSize))
// A snapshot only boots in its own region: an environment build pins the region it was built in.
const region = source?.region ?? request.region ?? deps.config.defaultRegion
// Then an explicit region; else the nearest served one (each has a golden image).
const region =
source?.region ??
request.region ??
request.nearRegions?.find((candidate) => goldenFor(deps, candidate)) ??
deps.config.defaultRegion
const bootSnapshotId = source?.snapshotId ?? goldenFor(deps, region)
if (!bootSnapshotId) {
throw new LifecycleError('conflict', 'region_unavailable')
@@ -919,7 +976,7 @@ export async function createSandbox(
const work = source?.auto
? deps.autoEnvironment.touch(source.environmentId)
: !source
? deps.autoEnvironment.ensure(caller.accountId, request.repo.url)
? deps.autoEnvironment.ensure(caller.accountId, request.repo.url, region)
: null
if (work) {
const settled = work.catch(() => undefined)
@@ -1005,7 +1062,7 @@ export async function createSandbox(
// Only a create nothing pinned to a region moves on: no explicit region, no environment build.
const fallback =
options.fallback !== false && !request.region && !source && isRegionalFailure(error)
? deps.config.fallbackRegions?.find((candidate) => candidate !== region && goldenFor(deps, candidate))
? (request.nearRegions ?? deps.config.fallbackRegions)?.find((candidate) => candidate !== region && goldenFor(deps, candidate))
: undefined
if (fallback) {
console.log(JSON.stringify({ event: 'region_fallback', from: region, to: fallback, error: String(error).slice(0, 200) }))
+44 -23
View File
@@ -2,15 +2,18 @@
// A create claims one instead of booting (saves the provider create, ~2 s p50 / ~4 s p95 measured),
// then applies the caller's policy and starts orcad exactly as a fresh VM would. The pooled id becomes
// the sandbox id, so the provider name and ownership tag line up without renaming anything.
// SANDBOX_POOL_SIZE=0 (the default) turns it off; the kill switch drains it.
// SANDBOX_POOL_SIZE=0 (the default) turns it off; the kill switch drains it. Every served region (the
// default and its fallbacks, each with a golden image) keeps its own: a create starts in the region
// nearest the caller (regions.ts), and a snapshot only boots in its own region.
import type { MachineSize } from './entitlements'
import { killSwitchOn, type GuardrailConfig } from './guardrails'
import type { SandboxProviderApi } from './sandbox-provider'
import { ORCAD_PORT, SHARE_PROXY_PORT } from './sandbox-vm-commands'
import { SANDBOX_VCPUS } from './machine-size'
import { servedRegions } from './regions'
export type PoolConfig = {
/** Ready VMs kept per (machine size, region); only `standard` in the default region is pooled. */
/** Ready VMs kept per (machine size, region); only `standard` is pooled, in every served region. */
size: number
/** Recycle a pooled VM after this long so a claimed session keeps most of its 24 h budget. */
maxAgeMs: number
@@ -36,11 +39,13 @@ export type PoolDeps = {
now: () => number
config: {
environment: string
/** The default region's golden image: the pool warms VMs in the default region only. */
/** The default region's golden image. */
goldenSnapshotId: string
/** Every region's golden image; never deleted with a pooled VM. */
goldenSnapshots?: Record<string, string>
defaultRegion: string
/** Served besides the default (each has a golden image): pooled too. */
fallbackRegions?: readonly string[]
guardrails?: GuardrailConfig
snapshotExpirationMs?: number
pool?: PoolConfig
@@ -91,7 +96,7 @@ export async function claimPooledVm(
snapshotId: string = deps.config.goldenSnapshotId
): Promise<PooledVm | null> {
const pool = deps.config.pool
if (!pool || machineSize !== POOLED_SIZE || region !== deps.config.defaultRegion) {
if (!pool || machineSize !== POOLED_SIZE || !servedRegions(deps.config).includes(region)) {
return null
}
return deps.db
@@ -141,29 +146,43 @@ async function takeLease(deps: PoolDeps): Promise<boolean> {
return taken !== null
}
/** How many warm VMs of one image to keep, and the region it boots in (the image's own). */
type PoolTarget = { want: number; region: string }
/** Each served region's golden image, by region. */
function servedGoldens(config: PoolDeps['config']): [region: string, snapshotId: string][] {
return servedRegions(config).flatMap((region) => {
const golden = region === config.defaultRegion ? config.goldenSnapshotId : config.goldenSnapshots?.[region]
return golden ? [[region, golden] as [string, string]] : []
})
}
/**
* Images worth a warm VM: the golden image (`size`), plus the active build of every environment in
* use lately (BUILD_POOL_SIZE each), so a create from an org environment skips the provider boot too.
* Images worth a warm VM: each served region's golden image (`size`), plus the active build of every
* environment in use lately (BUILD_POOL_SIZE each, in its build's region), so a create from an org
* environment skips the provider boot too.
*/
async function poolTargets(deps: PoolDeps, size: number): Promise<Map<string, number>> {
const targets = new Map<string, number>([[deps.config.goldenSnapshotId, size]])
async function poolTargets(deps: PoolDeps, size: number): Promise<Map<string, PoolTarget>> {
const goldens = servedGoldens(deps.config)
const targets = new Map<string, PoolTarget>(goldens.map(([region, snapshotId]) => [snapshotId, { want: size, region }]))
const regions = goldens.map(([region]) => region)
const since = deps.now() - HOT_BUILD_WINDOW_MS
// Hot = a workspace was created from the environment lately (not merely built). Most recent first,
// so the chosen set is stable tick to tick; capped per account and overall.
const hot = await deps.db
.prepare(
`SELECT b.snapshot_id AS snapshot_id, e.account_id AS account_id,
`SELECT b.snapshot_id AS snapshot_id, b.region AS region, e.account_id AS account_id,
(SELECT MAX(s.created_at) FROM sandboxes s WHERE s.environment_id = e.id) AS used_at
FROM environments e JOIN environment_builds b ON b.id = e.active_build_id
WHERE e.deleted_at IS NULL AND b.status = 'succeeded' AND b.snapshot_id IS NOT NULL
AND b.snapshot_deleted_at IS NULL AND json_extract(b.config, '$.machineSize') = ?
AND b.region = ? AND used_at > ?
AND b.region IN (${regions.map(() => '?').join(', ') || 'NULL'}) AND used_at > ?
ORDER BY used_at DESC LIMIT ?`
)
// Pooled VMs boot in the default region; a build snapshot from another region cannot boot there.
.bind(POOLED_SIZE, deps.config.defaultRegion, since, MAX_HOT_BUILDS * 4)
.all<{ snapshot_id: string; account_id: string }>()
.catch(() => ({ results: [] as { snapshot_id: string; account_id: string }[] }))
// A build's VM boots in the build's own region, which must be served.
.bind(POOLED_SIZE, ...regions, since, MAX_HOT_BUILDS * 4)
.all<{ snapshot_id: string; region: string; account_id: string }>()
.catch(() => ({ results: [] as { snapshot_id: string; region: string; account_id: string }[] }))
const perAccount = new Map<string, number>()
let hotCount = 0
for (const row of hot.results) {
@@ -173,7 +192,7 @@ async function poolTargets(deps: PoolDeps, size: number): Promise<Map<string, nu
}
perAccount.set(row.account_id, count + 1)
hotCount++
targets.set(row.snapshot_id, Math.max(targets.get(row.snapshot_id) ?? 0, BUILD_POOL_SIZE))
targets.set(row.snapshot_id, { want: BUILD_POOL_SIZE, region: row.region })
}
return targets
}
@@ -184,7 +203,7 @@ async function maintain(deps: PoolDeps): Promise<PoolTick> {
const draining = !pool || (deps.config.guardrails ? await killSwitchOn(deps.db, deps.config.guardrails) : false)
const all = (await deps.db.prepare(`SELECT * FROM sandbox_pool WHERE status != 'lease'`).all<PooledVm>()).results
const maxAge = pool?.maxAgeMs ?? DEFAULT_POOL_MAX_AGE_MS
const targets = draining || !pool ? new Map<string, number>() : await poolTargets(deps, pool.size)
const targets = draining || !pool ? new Map<string, PoolTarget>() : await poolTargets(deps, pool.size)
const expired = (vm: PooledVm) =>
draining ||
!targets.has(vm.snapshot_id) ||
@@ -197,7 +216,7 @@ async function maintain(deps: PoolDeps): Promise<PoolTick> {
for (const vm of keep) {
const count = (kept.get(vm.snapshot_id) ?? 0) + 1
kept.set(vm.snapshot_id, count)
if (count > (targets.get(vm.snapshot_id) ?? 0)) {
if (count > (targets.get(vm.snapshot_id)?.want ?? 0)) {
extras.push(vm)
}
}
@@ -222,17 +241,19 @@ async function maintain(deps: PoolDeps): Promise<PoolTick> {
if (draining || !pool) {
return { created: 0, recycled, ready: Math.max(0, live) }
}
const boots: string[] = []
for (const [snapshotId, want] of targets) {
const boots: { snapshotId: string; region: string }[] = []
for (const [snapshotId, { want, region }] of targets) {
const have = Math.min(kept.get(snapshotId) ?? 0, want)
for (let i = have; i < want; i++) {
boots.push(snapshotId)
boots.push({ snapshotId, region })
}
}
if (boots.length === 0) {
return { created: 0, recycled, ready: Math.max(0, live) }
}
const results = await Promise.allSettled(boots.slice(0, MAX_BOOTS_PER_TICK).map((snapshotId) => bootPooledVm(deps, snapshotId)))
const results = await Promise.allSettled(
boots.slice(0, MAX_BOOTS_PER_TICK).map((boot) => bootPooledVm(deps, boot.snapshotId, boot.region))
)
const created = results.filter((r) => r.status === 'fulfilled').length
for (const failure of results.filter((r) => r.status === 'rejected')) {
console.log(JSON.stringify({ event: 'pool_boot_failed', error: String(failure.reason).slice(0, 200) }))
@@ -240,14 +261,14 @@ async function maintain(deps: PoolDeps): Promise<PoolTick> {
return { created, recycled, ready: live + created }
}
async function bootPooledVm(deps: PoolDeps, snapshotId: string): Promise<void> {
async function bootPooledVm(deps: PoolDeps, snapshotId: string, region: string): Promise<void> {
const id = poolId()
const now = deps.now()
const row: PooledVm = {
id,
provider_name: `orca-${deps.config.environment}-${id}`,
machine_size: POOLED_SIZE,
region: deps.config.defaultRegion,
region,
snapshot_id: snapshotId,
status: 'warming',
session_id: null,
+111
View File
@@ -7,6 +7,11 @@
// provider error. Automatic, not operator-driven: the create has no state yet and a person is
// waiting on it, and a regional capacity blip is over long before anyone could flip a var.
// VERCEL_FALLBACK_REGIONS="" turns it off.
// - a desktop create or a new environment with nothing pinned starts in the served region nearest the
// caller (Cloudflare's request geolocation), not always the default: every frame of a remote session
// pays the round trip. Served = the default and its fallbacks that have an active golden (and so a
// warm pool, pool.ts). The environment records its region, so its builds and workspaces stay there.
// VERCEL_NEAREST_REGION=off turns it off.
import { ProviderError } from './sandbox-provider'
export const DEFAULT_REGION = 'pdx1'
@@ -39,3 +44,109 @@ export function isRegionalFailure(error: unknown): boolean {
}
return error instanceof TypeError
}
/** Where each Vercel region is (approximate city coordinates). A region missing here is never "nearest". */
const REGION_SITES: Record<string, readonly [latitude: number, longitude: number]> = {
iad1: [38.9, -77.0], // Washington, D.C.
cle1: [41.5, -81.7], // Cleveland
pdx1: [45.5, -122.7], // Portland
sfo1: [37.8, -122.4], // San Francisco
yul1: [45.5, -73.6], // Montréal
gru1: [-23.6, -46.6], // São Paulo
dub1: [53.3, -6.3], // Dublin
lhr1: [51.5, -0.1], // London
cdg1: [48.9, 2.4], // Paris
fra1: [50.1, 8.7], // Frankfurt
arn1: [59.3, 18.1], // Stockholm
cpt1: [-33.9, 18.4], // Cape Town
dxb1: [25.2, 55.3], // Dubai
bom1: [19.1, 72.9], // Mumbai
sin1: [1.4, 103.8], // Singapore
hkg1: [22.3, 114.2], // Hong Kong
icn1: [37.6, 127.0], // Seoul
kix1: [34.7, 135.5], // Osaka
hnd1: [35.7, 139.7], // Tokyo
syd1: [-33.9, 151.2] // Sydney
}
/** A rough middle of each continent, for a request Cloudflare located only that far. */
const CONTINENT_SITES: Record<string, readonly [number, number]> = {
NA: [40, -95],
SA: [-15, -60],
EU: [50, 10],
AF: [5, 20],
AS: [30, 100],
OC: [-25, 140]
}
export type RequestGeo = { latitude?: number; longitude?: number; continent?: string }
/** Cloudflare's geolocation of the caller (`request.cf`); null outside the Workers runtime or when it has none. */
export function requestGeo(request: Request): RequestGeo | null {
const cf = (request as { cf?: { latitude?: unknown; longitude?: unknown; continent?: unknown } }).cf
if (!cf) {
return null
}
const latitude = Number(cf.latitude)
const longitude = Number(cf.longitude)
const continent = typeof cf.continent === 'string' ? cf.continent : undefined
const located = cf.latitude != null && cf.longitude != null && Number.isFinite(latitude) && Number.isFinite(longitude)
if (!located && !continent) {
return null
}
return { ...(located ? { latitude, longitude } : {}), ...(continent ? { continent } : {}) }
}
/** Great-circle distance in km. */
function distanceKm(a: readonly [number, number], b: readonly [number, number]): number {
const rad = Math.PI / 180
const dLat = (b[0] - a[0]) * rad
const dLon = (b[1] - a[1]) * rad
const h = Math.sin(dLat / 2) ** 2 + Math.cos(a[0] * rad) * Math.cos(b[0] * rad) * Math.sin(dLon / 2) ** 2
return 2 * 6371 * Math.asin(Math.min(1, Math.sqrt(h)))
}
/**
* `served` (default region first) reordered nearest-first for a caller at `geo`. Unchanged when the
* caller can't be located; a region with no known site keeps its place after the located ones.
*/
export function regionsByDistance(geo: RequestGeo | null, served: readonly string[]): string[] {
const at: readonly [number, number] | undefined =
geo?.latitude !== undefined && geo.longitude !== undefined
? [geo.latitude, geo.longitude]
: geo?.continent
? CONTINENT_SITES[geo.continent]
: undefined
if (!at) {
return [...served]
}
const ranked = served.map((region, index) => {
const site = REGION_SITES[region]
return { region, index, km: site ? distanceKm(at, site) : Number.POSITIVE_INFINITY }
})
// Stable: equal distances (and unknown sites) keep the configured order, so the default wins ties.
ranked.sort((a, b) => a.km - b.km || a.index - b.index)
return ranked.map((entry) => entry.region)
}
/** False when VERCEL_NEAREST_REGION=off: new workspaces and environments start in the default region. */
export function nearestRegionEnabled(env: { VERCEL_NEAREST_REGION?: string }): boolean {
return env.VERCEL_NEAREST_REGION?.trim() !== 'off'
}
/**
* The served regions (default first) nearest-first for this request, or undefined (the default region
* decides) when the caller can't be located, there is nothing to choose, or the choice is turned off.
*/
export function nearestRegions(request: Request, served: readonly string[], enabled = true): string[] | undefined {
if (!enabled || served.length < 2) {
return undefined
}
const geo = requestGeo(request)
return geo ? regionsByDistance(geo, served) : undefined
}
/** The regions new sandboxes may use, default first: each has an active golden image. */
export function servedRegions(config: { defaultRegion: string; fallbackRegions?: readonly string[] }): string[] {
return [...new Set([config.defaultRegion, ...(config.fallbackRegions ?? [])])]
}
@@ -6,6 +6,8 @@ import { z } from 'zod'
import { ProviderError, type SandboxProviderApi } from './sandbox-provider'
const PROJECT = '/vercel/sandbox/project'
/** Written by the VM's memory guard (image/vm/orca-memguard). */
const GUARD_STATE = '/run/orca-memguard/state'
/**
* CPU from two /proc/stat samples 250 ms apart, memory from /proc/meminfo, disk for the project
@@ -28,9 +30,22 @@ victim=$(sudo -n dmesg 2>/dev/null | grep -oE 'Killed process [0-9]+ \([^)]{1,64
[ "$victim" = MainThread ] && victim=node
orcad=false; pid=$(cat "$HOME/.orca-sandbox/orcad.pid" 2>/dev/null || echo 0); [ "$pid" -gt 1 ] 2>/dev/null && kill -0 "$pid" 2>/dev/null && orcad=true
load=$(awk '{print $1}' /proc/loadavg)
printf '{"cpuPercent":%d,"cpus":%d,"load1":%s,"memTotalBytes":%d,"memUsedBytes":%d,"diskTotalBytes":%d,"diskUsedBytes":%d,"uptimeSeconds":%d,"oomKills":%d,"lastOomVictim":%s,"orcadAlive":%s}\n' \
# MEMGUARD: the memory guard's state (orca-memguard), when it runs this boot and wrote in the last minute.
guard=null
if [ -r ${GUARD_STATE} ]; then
g() { sed -n "s/^$1=//p" ${GUARD_STATE} | head -n 1; }
g_at=$(g at); g_level=$(g level); g_kills=$(g kills); g_name=$(g lastKillName | tr -cd 'A-Za-z0-9._+-' | head -c 40); g_kill_at=$(g lastKillAt)
g_swap=$(g swapTotalKiB); [[ "$g_swap" =~ ^[0-9]+$ ]] && g_swap=$((g_swap * 1024)) || g_swap=0
now_ms=$(( $(date +%s%3N) ))
if [ "$(g bootId)" = "$(cat /proc/sys/kernel/random/boot_id)" ] && [[ "$g_at" =~ ^[0-9]+$ ]] && [ $((now_ms - g_at)) -lt 60000 ] \
&& [[ "$g_level" =~ ^(ok|high|critical)$ ]] && [[ "$g_kills" =~ ^[0-9]+$ ]] && [[ "$g_kill_at" =~ ^[0-9]+$ ]]; then
guard=$(printf '{"level":"%s","kills":%d,"lastKillName":%s,"lastKillAt":%s,"swapBytes":%d}' "$g_level" "$g_kills" \
"$( [ -n "$g_name" ] && printf '"%s"' "$g_name" || printf null )" "$( [ "$g_kill_at" -gt 0 ] && printf '%s' "$g_kill_at" || printf null )" "$g_swap")
fi
fi
printf '{"cpuPercent":%d,"cpus":%d,"load1":%s,"memTotalBytes":%d,"memUsedBytes":%d,"diskTotalBytes":%d,"diskUsedBytes":%d,"uptimeSeconds":%d,"oomKills":%d,"lastOomVictim":%s,"orcadAlive":%s,"guard":%s}\n' \
"$cpu_pct" "$(nproc)" "$load" "$mem_total" "$((mem_total-mem_avail))" "\${disk_total:-0}" "\${disk_used:-0}" "$uptime_s" "$oom" \
"$( [ -n "$victim" ] && printf '"%s"' "$victim" || printf null )" "$orcad"
"$( [ -n "$victim" ] && printf '"%s"' "$victim" || printf null )" "$orcad" "$guard"
`.replaceAll('\\${', '${')
export const VmMetricsSchema = z
@@ -51,6 +66,41 @@ export const VmMetricsSchema = z
export type VmMetrics = z.infer<typeof VmMetricsSchema>
/** The memory guard's last word: its pressure level, the processes it stopped this boot, its swap. */
export const VmGuardSchema = z
.object({
level: z.enum(['ok', 'high', 'critical']),
kills: z.number().int().min(0),
lastKillName: z.string().max(40).nullable(),
lastKillAt: z.number().int().positive().nullable(),
swapBytes: z.number().int().min(0)
})
.strict()
export type VmGuard = z.infer<typeof VmGuardSchema>
// The probe's line: the metrics plus the guard (null when no guard ran this boot: an older image).
const VmProbeSchema = VmMetricsSchema.extend({ guard: VmGuardSchema.nullable().optional() }).strict()
/** Memory pressure for the status read: the guard's level, else one from MemAvailable alone. */
export type WorkspaceMemory = {
pressure: 'ok' | 'high' | 'critical'
/** Processes the guard stopped this boot (the kernel's own OOM kills are `metrics.oomKills`). */
guardKills: number
lastGuardVictim: string | null
lastGuardKillAt: number | null
/** False when the VM runs no guard: pressure is then MemAvailable's, and nothing is stopped early. */
guarded: boolean
}
export function workspaceMemory(metrics: VmMetrics, guard: VmGuard | null): WorkspaceMemory {
if (guard) {
return { pressure: guard.level, guardKills: guard.kills, lastGuardVictim: guard.lastKillName, lastGuardKillAt: guard.lastKillAt, guarded: true }
}
const availPct = metrics.memTotalBytes > 0 ? ((metrics.memTotalBytes - metrics.memUsedBytes) * 100) / metrics.memTotalBytes : 100
const pressure = availPct < 8 ? 'critical' : availPct < 15 ? 'high' : 'ok'
return { pressure, guardKills: 0, lastGuardVictim: null, lastGuardKillAt: null, guarded: false }
}
const MAX_TREE = 2000
const MAX_CHANGED = 300
const MAX_COMMITS = 20
@@ -164,8 +214,17 @@ async function runJson<T>(
}
/** Null when the VM did not answer in time (the status panel shows "Unresponsive"). */
export function readVmMetrics(provider: SandboxProviderApi, sessionId: string): Promise<VmMetrics | null> {
return runJson(provider, sessionId, METRICS_SCRIPT, VmMetricsSchema, 8_000)
export async function readVmMetrics(
provider: SandboxProviderApi,
sessionId: string
): Promise<{ metrics: VmMetrics; memory: WorkspaceMemory } | null> {
const probe = await runJson(provider, sessionId, METRICS_SCRIPT, VmProbeSchema, 8_000)
if (!probe) {
return null
}
// `metrics` keeps its exact shape: desktops parse it strictly.
const { guard, ...metrics } = probe
return { metrics, memory: workspaceMemory(metrics, guard ?? null) }
}
export function captureArchiveMetadata(provider: SandboxProviderApi, sessionId: string): Promise<ArchiveMetadata | null> {
+10 -3
View File
@@ -14,6 +14,9 @@ import { SecretsKmsError } from './secrets-kms'
import type { SandboxRow } from './store'
import { secretLog } from './secret-log'
/** Session env name the VM env script reads for the agents signed in inside the workspace ('claude,codex'). */
export const SIGN_IN_INSIDE_ENV = 'ORCA_SIGN_IN_INSIDE'
export type SessionSecrets = {
env: Record<string, string>
/** Row value for `egress_secret_hosts`. */
@@ -134,6 +137,10 @@ export async function sessionSecretEnv(
const env = withoutSignInCredentials(derived, dropped)
const scopes = deriveProviderScopes(stored, env, storedScopes)
const egressAllowHosts = providerHosts(env).join(',') || null
// Not a secret and never in shells (ORCA_*): tells the VM env script which agents sign in there, so a
// first run with no login of its own shows the agent's sign-in screens instead of skipping them.
const inside = (Object.keys(modes) as CredentialAgent[]).filter((agent) => modes[agent] === 'inside')
const insideEnv: Record<string, string> = inside.length > 0 ? { [SIGN_IN_INSIDE_ENV]: inside.join(',') } : {}
// Counts only.
secretLog('session_secrets', { sandboxId: row.id, injected: Object.keys(env).length, skipped })
if (withhold) {
@@ -153,7 +160,7 @@ export async function sessionSecretEnv(
}
}
const hosts = [encodeEgressSecretHosts(withheld.targets), ...accountRoutes].filter(Boolean).join(',') || null
return { env: { ...withheld.env, ...accountEnv }, egressSecretHosts: hosts, egressAllowHosts }
return { env: { ...withheld.env, ...accountEnv, ...insideEnv }, egressSecretHosts: hosts, egressAllowHosts }
}
if (!swap) {
// An operator turned placeholders off for this account (SECRETS_EGRESS_PLACEHOLDERS=off or accounts:<ids>
@@ -162,7 +169,7 @@ export async function sessionSecretEnv(
if (plain > 0 && !egressSwapWanted(deps, row) && firstReadThisSession('plain', row)) {
secretLog('egress_swap_disabled', { sandboxId: row.id, plainKeys: plain })
}
return { env, egressSecretHosts: null, egressAllowHosts }
return { env: { ...env, ...insideEnv }, egressSecretHosts: null, egressAllowHosts }
}
const swapped = applySecretPlaceholders(row, env, scopes)
const accountEnv: Record<string, string> = {}
@@ -175,7 +182,7 @@ export async function sessionSecretEnv(
}
}
const secretHosts = [encodeEgressSecretHosts(swapped.targets), ...accountRoutes].filter(Boolean).join(',') || null
return { env: { ...swapped.env, ...accountEnv }, egressSecretHosts: secretHosts, egressAllowHosts }
return { env: { ...swapped.env, ...accountEnv, ...insideEnv }, egressSecretHosts: secretHosts, egressAllowHosts }
}
/** Provider hosts from the session's secrets plus the workspace's declared tool hosts (MCP). */
@@ -22,6 +22,7 @@ import {
ArchiveMetadataSchema,
captureArchiveMetadata,
readVmMetrics,
type WorkspaceMemory,
type ArchiveMetadata,
type VmMetrics
} from './sandbox-vm-inspect'
@@ -359,6 +360,8 @@ export async function expiredArchives(db: D1Database, now: number): Promise<Sand
export type WorkspaceStatus = {
state: WorkspaceState
metrics: VmMetrics | null
/** MEMGUARD: memory pressure and the guard's kills; null when the VM was not probed. */
memory: WorkspaceMemory | null
/** Since orcad last became ready (this session). */
uptimeMs: number | null
usage: { totalMs: number; centsPerHour: number; costCents: number }
@@ -393,13 +396,15 @@ export async function workspaceStatus(deps: LifecycleDeps, input: SandboxRow, co
// A lapsed lease: the provider may have stopped it, which is "sleeping", not "unresponsive".
const row = input.status === 'running' ? await confirmRunning(deps, input) : input
const probed = row.status === 'running' && row.session_id !== null
const metrics = probed ? await readVmMetrics(deps.provider, row.session_id!) : null
const probe = probed ? await readVmMetrics(deps.provider, row.session_id!) : null
const metrics = probe?.metrics ?? null
const now = deps.now()
const totalMs = await sandboxUsageMs(deps.db, row, now)
const centsPerHour = config.centsPerHour?.[row.machine_size] ?? DEFAULT_CENTS_PER_HOUR[row.machine_size]
return {
state: stateOf(row, metrics, probed),
metrics,
memory: probe?.memory ?? null,
uptimeMs: row.status === 'running' && row.ready_at !== null ? Math.max(0, now - row.ready_at) : null,
usage: { totalMs, centsPerHour, costCents: Math.round((totalMs / 3_600_000) * centsPerHour) },
archive: row.archived_at !== null ? { archivedAt: row.archived_at, expiresAt: row.archive_expires_at } : null,
@@ -42,7 +42,7 @@ function decode(encoded: string | undefined): Record<string, string> {
function expectSignedOut(answer: { status: number; body: string }): void {
expect(answer.status).toBe(401)
expect(JSON.parse(answer.body)).toMatchObject({ type: 'error', error: { type: 'authentication_error' } })
expect(JSON.parse(answer.body).error.message).toBe('Your Claude sign-in for Orca Cloud expired. Renew it in Orca: Settings → Agent accounts.')
expect(JSON.parse(answer.body).error.message).toBe('Your Claude sign-in for Orca Cloud expired. Renew it in Orca: Settings → Cloud agent sign-ins.')
}
async function setup(overrides: Record<string, string> = {}) {
@@ -0,0 +1,165 @@
// "Sign in inside this workspace": a per-person default Source under which Orca stores and injects
// nothing for that agent, so the agent's own login inside the sandbox reaches the vendor untouched.
import { readFileSync, readdirSync } from 'node:fs'
import { DatabaseSync } from 'node:sqlite'
import { fileURLToPath } from 'node:url'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { ENV_COMMAND, START_COMMAND } from '../src/sandbox-vm-commands'
import { createHarness } from './harness'
const KEK = Buffer.alloc(32, 7).toString('base64')
const PROXY_URL = 'https://sandbox.test/v1/egress/proxy'
const SETUP_TOKEN = 'sk-ant-oat01-FAKEsetupTOKEN_for_tests_only_000000000000'
const TEAM_KEY = 'sk-ant-api03-TEAM-fake-0000-aaaa'
const OPENAI_KEY = 'sk-proj-FAKE-openai-0000-bbbb'
let logs: string[] = []
beforeEach(() => {
logs = []
for (const method of ['log', 'info', 'warn', 'error', 'debug'] as const) {
vi.spyOn(console, method).mockImplementation((...args: unknown[]) => {
logs.push(args.map((arg) => (typeof arg === 'string' ? arg : JSON.stringify(arg))).join(' '))
})
}
})
afterEach(() => {
for (const value of [SETUP_TOKEN, TEAM_KEY, OPENAI_KEY]) {
expect(logs.join('\n')).not.toContain(value)
}
vi.restoreAllMocks()
})
function decode(encoded: string | undefined): Record<string, string> {
return Object.fromEntries(
(encoded ?? '')
.split('\n')
.filter(Boolean)
.map((line) => {
const at = line.indexOf('=')
return [line.slice(0, at), Buffer.from(line.slice(at + 1), 'base64').toString('utf8')]
})
)
}
/** Vendor hosts the agents' own sign-in and model calls use. */
const VENDOR_HOSTS = ['api.anthropic.com', 'claude.ai', 'platform.claude.com', 'console.anthropic.com', 'chatgpt.com', 'auth.openai.com', 'api.openai.com']
async function setup(overrides: Record<string, string> = {}) {
const h = await createHarness({
SECRETS_KEK: KEK,
EGRESS_PROXY_URL: PROXY_URL,
SECRETS_EGRESS_PLACEHOLDERS: 'on',
AGENT_ACCOUNTS_CLAUDE: 'on',
...overrides
})
const as = (sub: string, org?: string, role?: string) => h.token(sub, org, 'sandbox', role ? { orgRole: role } : {})
const lastEnv = (command: string) => decode(h.vercel.commands.filter((c) => c.command === command).at(-1)?.env.ORCA_SECRET_ENV)
const policy = () => [...h.vercel.boxes.values()].at(-1)!.policy.allow
return { h, as, lastEnv, policy }
}
describe('Source "in-workspace" (sign in inside this workspace)', () => {
it('injects nothing for that agent even with a stored account and a team key, and leaves its hosts unruled', async () => {
const { h, as, lastEnv, policy } = await setup()
const ada = await as('ada', 'acme', 'admin')
const mia = await as('mia', 'acme', 'member')
await h.call('PUT', '/v1/secrets/org/ANTHROPIC_API_KEY', { bearer: ada, body: { value: TEAM_KEY } })
await h.call('PUT', '/v1/secrets/personal/OPENAI_API_KEY', { bearer: mia, body: { value: OPENAI_KEY } })
await h.call('PUT', '/v1/agent-credentials/claude', { bearer: mia, body: { token: SETUP_TOKEN } })
for (const agent of ['claude', 'codex']) {
const put = await h.call('PUT', '/v1/agent-sources', { bearer: mia, body: { agent, source: 'in-workspace' } })
expect(put.status).toBe(200)
}
await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard' }, bearer: mia })
const env = lastEnv(START_COMMAND)
for (const name of ['CLAUDE_CODE_OAUTH_TOKEN', 'ANTHROPIC_API_KEY', 'OPENAI_API_KEY', 'ORCA_CODEX_AUTH_PLACEHOLDER']) {
expect(env[name]).toBeUndefined()
}
expect(Object.values(env).some((value) => value.startsWith('orca-ph-'))).toBe(false)
// The VM env script's cue (never a shell variable): these agents sign in inside the workspace.
expect(env.ORCA_SIGN_IN_INSIDE).toBe('claude,codex')
// Open egress: "*" plus only hosts that carry rules. No vendor host carries one (no claim, no 403).
const allow = policy()
expect(allow['*']).toEqual([])
for (const host of VENDOR_HOSTS) {
expect(allow[host]).toBeUndefined()
}
const view = (await h.call('GET', '/v1/agent-sources', { bearer: mia })).body
expect(view.inWorkspace).toEqual({ claude: true, codex: true })
// Older desktops read `defaults` and `effective.source` with two values only: neither ever says 'in-workspace'.
expect(view.defaults).toEqual({ claude: null, codex: null })
expect(view.effective.claude).toEqual({ source: null, origin: 'default' })
})
it('limited egress still reaches every vendor sign-in and API host unmodified', async () => {
const { h, as, policy } = await setup({ SANDBOX_EGRESS_PRESET: 'limited' })
const mia = await as('mia')
await h.call('PUT', '/v1/agent-credentials/claude', { bearer: mia, body: { token: SETUP_TOKEN } })
await h.call('PUT', '/v1/agent-sources', { bearer: mia, body: { agent: 'claude', source: 'in-workspace' } })
await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard' }, bearer: mia })
const allow = policy()
expect(allow['*']).toBeUndefined()
const reachable = (host: string) => {
const parent = `*.${host.split('.').slice(1).join('.')}`
return allow[host] ?? allow[parent]
}
for (const host of VENDOR_HOSTS) {
expect(reachable(host), host).toEqual([])
}
})
it('switching a running workspace\'s owner to it drops the account placeholder live; choosing the account again brings it back', async () => {
const { h, as, lastEnv } = await setup()
const mia = await as('mia')
await h.call('PUT', '/v1/agent-credentials/claude', { bearer: mia, body: { token: SETUP_TOKEN } })
const created = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard' }, bearer: mia })
const id: string = created.body.sandbox.id
expect(lastEnv(START_COMMAND).CLAUDE_CODE_OAUTH_TOKEN).toBe(`orca-ph-claude-account-${id}`)
await h.call('PUT', '/v1/agent-sources', { bearer: mia, body: { agent: 'claude', source: 'in-workspace' } })
expect(lastEnv(ENV_COMMAND).CLAUDE_CODE_OAUTH_TOKEN).toBeUndefined()
expect([...h.vercel.boxes.values()][0]!.policy.allow['api.anthropic.com']).toBeUndefined()
expect(lastEnv(ENV_COMMAND).ORCA_SIGN_IN_INSIDE).toBe('claude')
await h.call('PUT', '/v1/agent-sources', { bearer: mia, body: { agent: 'claude', source: null } })
expect(lastEnv(ENV_COMMAND).CLAUDE_CODE_OAUTH_TOKEN).toBe(`orca-ph-claude-account-${id}`)
expect(lastEnv(ENV_COMMAND).ORCA_SIGN_IN_INSIDE).toBeUndefined()
})
it('is a per-person default only: a workspace override is refused, and an explicit workspace Source still wins', async () => {
const { h, as, lastEnv } = await setup()
const mia = await as('mia')
await h.call('PUT', '/v1/agent-credentials/claude', { bearer: mia, body: { token: SETUP_TOKEN } })
const created = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard' }, bearer: mia })
const id: string = created.body.sandbox.id
const refused = await h.call('PUT', '/v1/agent-sources', { bearer: mia, body: { agent: 'claude', source: 'in-workspace', workspaceId: id } })
expect(refused.status).toBe(400)
await h.call('PUT', '/v1/agent-sources', { bearer: mia, body: { agent: 'claude', source: 'sign-in', workspaceId: id } })
await h.call('PUT', '/v1/agent-sources', { bearer: mia, body: { agent: 'claude', source: 'in-workspace' } })
expect(lastEnv(ENV_COMMAND).CLAUDE_CODE_OAUTH_TOKEN).toBe(`orca-ph-claude-account-${id}`)
})
it('the migration keeps every row written before the table was widened', () => {
const dir = fileURLToPath(new URL('../migrations/', import.meta.url).href)
const files = readdirSync(dir).filter((name) => name.endsWith('.sql')).sort()
const widen = files.indexOf('0063_agent_sources_in_workspace.sql')
expect(widen).toBeGreaterThan(0)
const raw = new DatabaseSync(':memory:')
for (const file of files.slice(0, widen)) {
raw.exec(readFileSync(`${dir}${file}`, 'utf8'))
}
const insert = raw.prepare('INSERT INTO agent_sources (user_id, account_id, scope, agent, source, updated_at) VALUES (?, ?, ?, ?, ?, ?)')
insert.run('mia', 'acme', 'default', 'codex', 'key', 1)
insert.run('mia', 'acme', 'offer', 'claude', 'declined', 2)
insert.run('mia', 'acme', 'sbx-1', 'claude', 'sign-in', 3)
expect(() => insert.run('mia', 'acme', 'default', 'claude', 'in-workspace', 4)).toThrow(/CHECK/)
raw.exec(readFileSync(`${dir}${files[widen]}`, 'utf8'))
insert.run('mia', 'acme', 'default', 'claude', 'in-workspace', 4)
expect(raw.prepare('SELECT scope, agent, source, updated_at FROM agent_sources ORDER BY updated_at').all()).toEqual([
{ scope: 'default', agent: 'codex', source: 'key', updated_at: 1 },
{ scope: 'offer', agent: 'claude', source: 'declined', updated_at: 2 },
{ scope: 'sbx-1', agent: 'claude', source: 'sign-in', updated_at: 3 },
{ scope: 'default', agent: 'claude', source: 'in-workspace', updated_at: 4 }
])
expect(() => insert.run('mia', 'acme', 'default', 'claude', 'bogus', 5)).toThrow(/CHECK/)
expect(raw.prepare(`SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = 'agent_sources'`).all()).toContainEqual({ name: 'agent_sources_scope' })
})
})
@@ -0,0 +1,85 @@
import { exportPKCS8, generateKeyPair, decodeJwt } from 'jose'
import { describe, expect, it } from 'vitest'
import { createIdTokenSource, relayingFetch } from '../src/chatgpt-relay'
const RELAY = 'https://orca-chatgpt-relay-abc.us-central1.run.app'
function idTokenFor(exp: number): string {
const enc = (o: unknown) => Buffer.from(JSON.stringify(o)).toString('base64url')
return `${enc({ alg: 'RS256' })}.${enc({ exp, aud: RELAY })}.sig`
}
async function serviceAccount() {
const { privateKey } = await generateKeyPair('RS256', { extractable: true })
return JSON.stringify({ client_email: 'relay-invoker@p.iam.gserviceaccount.com', private_key: await exportPKCS8(privateKey) })
}
describe('relayingFetch (CODEX-RELAY)', () => {
it('sends chatgpt.com through the relay with an ID token beside, never instead of, the ChatGPT credential', async () => {
const seen: { url: string; headers: Headers }[] = []
const fetchImpl = (async (input: RequestInfo | URL, init?: RequestInit) => {
seen.push({ url: String(input), headers: new Headers(init?.headers) })
return new Response('ok')
}) as typeof fetch
const relayed = relayingFetch({ relayUrl: RELAY, idToken: async () => 'google-id', fetch: fetchImpl })
await relayed('https://chatgpt.com/backend-api/codex/models?client_version=0.166.0', { headers: { authorization: 'Bearer chatgpt-real' } })
await relayed('https://api.anthropic.com/v1/messages', { headers: { authorization: 'Bearer other' } })
expect(seen[0]!.url).toBe(`${RELAY}/backend-api/codex/models?client_version=0.166.0`)
expect(seen[0]!.headers.get('authorization')).toBe('Bearer chatgpt-real')
expect(seen[0]!.headers.get('x-serverless-authorization')).toBe('Bearer google-id')
// Every other host goes direct, with no relay token.
expect(seen[1]!.url).toBe('https://api.anthropic.com/v1/messages')
expect(seen[1]!.headers.get('x-serverless-authorization')).toBeNull()
})
it('without a relay URL chatgpt.com goes direct, as before', async () => {
const urls: string[] = []
const fetchImpl = (async (input: RequestInfo | URL) => (urls.push(String(input)), new Response('ok'))) as typeof fetch
await relayingFetch({ fetch: fetchImpl })('https://chatgpt.com/backend-api/wham/usage')
expect(urls).toEqual(['https://chatgpt.com/backend-api/wham/usage'])
})
it('a relay sign-in failure answers in ChatGPT’s error shape (Codex prints it), never reaching chatgpt.com', async () => {
let calls = 0
const fetchImpl = (async () => (calls++, new Response('ok'))) as typeof fetch
const res = await relayingFetch({
relayUrl: RELAY,
idToken: async () => {
throw new Error('no')
},
fetch: fetchImpl
})('https://chatgpt.com/backend-api/codex/responses', { method: 'POST' })
expect(res.status).toBe(503)
expect(((await res.json()) as { detail: string }).detail).toContain('ChatGPT')
expect(calls).toBe(0)
})
})
describe('createIdTokenSource', () => {
it('mints with target_audience = the relay URL, and reuses the token until 5 minutes before it expires', async () => {
let now = 1_800_000_000_000
const assertions: string[] = []
const fetchImpl = (async (_input: RequestInfo | URL, init?: RequestInit) => {
assertions.push(new URLSearchParams(String(init?.body)).get('assertion')!)
return Response.json({ id_token: idTokenFor(Math.floor(now / 1000) + 3600) })
}) as typeof fetch
const source = createIdTokenSource({ serviceAccountJson: await serviceAccount(), audience: RELAY, fetch: fetchImpl, now: () => now })
const first = await source()
expect(await source()).toBe(first)
expect(assertions).toHaveLength(1)
const claims = decodeJwt(assertions[0]!)
expect(claims).toMatchObject({ target_audience: RELAY, aud: 'https://oauth2.googleapis.com/token', iss: 'relay-invoker@p.iam.gserviceaccount.com' })
now += 56 * 60_000
await source()
expect(assertions).toHaveLength(2)
})
it('a refused mint throws (the caller answers 503) and the next call tries again', async () => {
let ok = false
const fetchImpl = (async () => (ok ? Response.json({ id_token: idTokenFor(4e9) }) : new Response('{}', { status: 400 }))) as typeof fetch
const source = createIdTokenSource({ serviceAccountJson: await serviceAccount(), audience: RELAY, fetch: fetchImpl })
await expect(source()).rejects.toThrow()
ok = true
await expect(source()).resolves.toContain('.')
})
})
@@ -1,6 +1,6 @@
// Codex via the person's Orca account (contract D11, flag S2): the broker sandbox login, the lease refresher,
// the placeholder auth.json and the chatgpt.com egress swap. Fake provider; no real OpenAI call.
import { SignJWT, createLocalJWKSet, exportJWK, generateKeyPair } from 'jose'
import { SignJWT, createLocalJWKSet, exportJWK, exportPKCS8, generateKeyPair } from 'jose'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { BROKER_AUTH_PATH, BROKER_EXIT_PATH, BROKER_START_MARKER, parseDeviceCode } from '../src/codex-broker'
import { CODEX_TOKEN_URL, codexPlaceholder, refreshCodexCredential, refreshCodexLeases, setCodexTokenFetch } from '../src/codex-credentials'
@@ -83,6 +83,11 @@ async function setup(overrides: Record<string, string> = {}) {
egressKeysForIssuer: () => createLocalJWKSet(jwks),
credentials: { authorizationFor: async () => null },
egressFetch: async (input, init) => {
if (String(input) === 'https://oauth2.googleapis.com/token') {
// CODEX-RELAY: Google's ID token for the relay (only asked when a relay is configured).
const enc = (o: unknown) => Buffer.from(JSON.stringify(o)).toString('base64url')
return Response.json({ id_token: `${enc({ alg: 'RS256' })}.${enc({ exp: 4e9 })}.sig` })
}
upstream.push({ url: String(input), headers: new Headers(init?.headers) })
if (edgeBlock.on) {
return new Response('<html><body>Unable to load site</body></html>', {
@@ -381,7 +386,7 @@ describe('Codex account in workspaces (D10, D13)', () => {
// RS-4: still a 403 (a 401 would start Codex's own token refresh), now with words Codex prints.
expect(refused.status).toBe(403)
expect(JSON.parse(refused.body)).toEqual({
detail: 'Orca Cloud can’t use your ChatGPT sign-in here. Check it in Orca: Settings → Agent accounts.'
detail: 'Orca Cloud can’t use your ChatGPT sign-in here. Check it in Orca: Settings → Cloud agent sign-ins.'
})
expect(lastEnv('/opt/orca/bin/orca-sandbox-env').ORCA_CODEX_AUTH_PLACEHOLDER).toBeUndefined()
})
@@ -491,3 +496,25 @@ describe('CODEX-EGRESS-BLOCKED: chatgpt.com refuses the proxy itself', () => {
expect((await proxied(id, 'chatgpt.com', auth)).status).toBe(200)
})
})
describe('CODEX-RELAY: chatgpt.com through the Cloud Run relay', () => {
it('sends the swapped request to the relay with a Google ID token; the path and the ChatGPT credential are unchanged', async () => {
const { privateKey } = await generateKeyPair('RS256', { extractable: true })
const RELAY = 'https://orca-chatgpt-relay-x.us-central1.run.app'
const { h, as, signIn, proxied, upstream } = await setup({
CHATGPT_RELAY_URL: RELAY,
CHATGPT_RELAY_SERVICE_ACCOUNT: JSON.stringify({ client_email: 'invoker@p.iam.gserviceaccount.com', private_key: await exportPKCS8(privateKey) })
})
const mia = await as('mia', 'acme', 'member')
await signIn(mia)
const created = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard' }, bearer: mia })
const id: string = created.body.sandbox.id
const res = await proxied(id, 'chatgpt.com', { authorization: `Bearer ${codexPlaceholder({ id })}` }, '/backend-api/codex/models?client_version=0.166.0', 'GET')
expect(res.status).toBe(200)
const sent = upstream.at(-1)!
expect(sent.url).toBe(`${RELAY}/backend-api/codex/models?client_version=0.166.0`)
expect(sent.headers.get('authorization')).toBe(`Bearer ${ACCESS}`)
expect(sent.headers.get('chatgpt-account-id')).toBe('acct-1')
expect(sent.headers.get('x-serverless-authorization')).toMatch(/^Bearer [\w-]+\.[\w-]+\.sig$/)
})
})
@@ -0,0 +1,38 @@
// CREATE-STEP-WATCH: creates hung 180 s with no log and no row (n73); the steps before the create log.
import { afterEach, describe, expect, it, vi } from 'vitest'
import { createStepWatch } from '../src/index'
describe('CREATE-STEP-WATCH', () => {
afterEach(() => {
vi.useRealTimers()
vi.restoreAllMocks()
})
it('names a step still pending after 10 s, then reports every step once the create starts', async () => {
vi.useFakeTimers()
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
const watch = createStepWatch(true, () => Date.now())
let finish: (value: string) => void = () => undefined
const body = watch.step('body', new Promise<string>((resolve) => (finish = resolve)))
await vi.advanceTimersByTimeAsync(10_000)
expect(log).toHaveBeenCalledWith(expect.stringContaining('"step":"body"'))
finish('{}')
await expect(body).resolves.toBe('{}')
watch.done()
watch.done()
const reports = log.mock.calls.filter(([line]) => String(line).includes('"create_steps"'))
expect(reports).toHaveLength(1)
expect(String(reports[0]![0])).toContain('"body":10000')
})
it('stays quiet for other routes', async () => {
vi.useFakeTimers()
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
const watch = createStepWatch(false)
const pending = watch.step('auth', new Promise(() => undefined))
await vi.advanceTimersByTimeAsync(20_000)
watch.done()
expect(log).not.toHaveBeenCalled()
void pending
})
})
@@ -0,0 +1,69 @@
import { execFile } from 'node:child_process'
import { createServer } from 'node:http'
import type { AddressInfo } from 'node:net'
import { describe, expect, it } from 'vitest'
import { EGRESS_DENIED_BODY } from '../src/network-policy'
import { judge, parseRunnerOutput, runnerCommand, sha8, type ProbeCase, type Summary } from '../scripts/egress-probe-checks'
const answer = (summary: Partial<Summary>): Summary => ({ status: 200, denied: false, edgeBlocked: false, ...summary })
describe('egress probe judge', () => {
const swapped: ProbeCase = { name: 'echo', url: 'https://echo/post', expect: { kind: 'swapped', headers: { 'x-api-key': 'stand-in', authorization: 'Bearer keep' } } }
it('accepts the provider seeing the swapped key and the other header unchanged', () => {
expect(judge(swapped, answer({ echoed: { 'x-api-key': sha8('stand-in'), authorization: sha8('Bearer keep') } }))).toEqual([])
})
it('names a header that changed or never arrived, and anything leaked from the hop, never values', () => {
const problems = judge(swapped, answer({ echoed: { 'x-api-key': sha8('orca-ph-placeholder') }, leaked: ['vercel-sandbox-oidc-token'] }))
expect(problems).toEqual([
'echo: the provider saw vercel-sandbox-oidc-token',
'echo: header x-api-key arrived with another value',
'echo: header authorization did not arrive'
])
expect(problems.join(' ')).not.toContain('stand-in')
})
it("requires the firewall's own 403 for a fail-closed case", () => {
const denied: ProbeCase = { name: 'foreign', url: 'https://x', expect: { kind: 'denied' } }
expect(judge(denied, answer({ status: 403, denied: true }))).toEqual([])
expect(judge(denied, answer({ status: 403, edgeBlocked: true }))).toHaveLength(1)
expect(judge(denied, answer({ status: 401 }))).toHaveLength(1)
})
it('an upstream case fails on the firewall 403, an edge block page or no answer', () => {
const upstream: ProbeCase = { name: 'chatgpt', url: 'https://chatgpt.com/x', expect: { kind: 'upstream' } }
expect(judge(upstream, answer({ status: 401 }))).toEqual([])
expect(judge(upstream, answer({ status: 403, denied: true }))).toEqual(['chatgpt: refused by the firewall'])
expect(judge(upstream, answer({ status: 403, edgeBlocked: true }))).toHaveLength(1)
expect(judge(upstream, answer({ status: 0 }))).toEqual(['chatgpt: no answer'])
})
})
describe('the in-VM runner', () => {
it('reports status, the deny body and echoed header digests from env-held values', async () => {
// A local stand-in for postman-echo.com: echoes request headers under `headers`.
const server = createServer((req, res) => {
if (req.url === '/denied') {
res.writeHead(403, { 'content-type': 'text/plain' }).end(EGRESS_DENIED_BODY)
return
}
res.writeHead(200, { 'content-type': 'application/json' }).end(JSON.stringify({ headers: { ...req.headers, 'vercel-forwarded-host': 'x' } }))
})
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
const base = `http://127.0.0.1:${(server.address() as AddressInfo).port}`
const cases = [
{ name: 'echo', url: `${base}/post`, method: 'POST' as const, headers: { 'x-api-key': '$PH', authorization: 'Bearer keep' } },
{ name: 'denied', url: `${base}/denied` }
]
const stdout = await new Promise<string>((resolve, reject) =>
execFile('bash', ['-c', runnerCommand(cases)], { env: { ...process.env, PH: 'secret-ph' } }, (error, out) => (error ? reject(error) : resolve(out)))
)
server.close()
expect(stdout).toContain('PRB-END')
expect(stdout).not.toContain('secret-ph')
const [echo, denied] = parseRunnerOutput(stdout, 2)
expect(echo).toEqual({ status: 200, denied: false, edgeBlocked: false, echoed: { 'x-api-key': sha8('secret-ph'), authorization: sha8('Bearer keep') }, leaked: ['vercel-forwarded-host'] })
expect(denied).toMatchObject({ status: 403, denied: true })
})
})
@@ -0,0 +1,52 @@
// ENTRY-POINTS-BEFORE-START: a boot from an environment build made on an older golden ran that golden's
// orca-sandbox-start (no agent prefetch, no OpenCode first run) until the environment was rebuilt.
// refreshRuntime's one command also probes the entry points; older ones are installed before the start.
import { describe, expect, it, vi } from 'vitest'
import { VM_SCRIPTS_STAMP } from '../image/vm-scripts.generated'
import { entryPointsBehind } from '../src/golden-images'
import { refreshRuntime, type LifecycleDeps } from '../src/lifecycle'
import type { SandboxRow } from '../src/store'
type Call = { args?: string[]; env?: Record<string, string> }
function setup(probe: string, plan: boolean) {
const calls: Call[] = []
const runCommand = vi.fn(async (_session: string, call: Call) => {
calls.push(call)
return calls.length === 1 ? { exitCode: 0, stdout: `${probe}\n${plan ? 'current abc' : ''}`, stderr: '' } : { exitCode: 0, stdout: 'synced\n', stderr: '' }
})
const deps = {
provider: { runCommand },
now: () => 0,
runtimeRefresh: async () => (plan ? { bundle: { commit: 'abc', sha256: 'f', runtime: '22' }, url: 'https://x' } : null)
} as unknown as LifecycleDeps
const row = { id: 'sbx_1', region: 'iad1' } as unknown as SandboxRow
return { calls, deps, row }
}
describe('ENTRY-POINTS-BEFORE-START', () => {
it('installs older entry points before the start, in the same round trip as the runtime check', async () => {
const t = setup(`stamp=${VM_SCRIPTS_STAMP - 1}\nmarker=0`, true)
await expect(refreshRuntime(t.deps, t.row, 'sess')).resolves.toBe('current abc')
expect(t.calls).toHaveLength(2)
expect(t.calls[0]!.args!.at(-1)).toContain('stamp=')
expect(t.calls[0]!.args!.at(-1)).toContain('SOURCE_COMMIT')
expect(t.calls[1]!.args!.at(-1)).toContain('install -o root')
})
it('adds nothing when the image carries this Worker scripts (same stamp or marker)', async () => {
for (const probe of [`stamp=${VM_SCRIPTS_STAMP}\nmarker=0`, 'stamp=0\nmarker=1']) {
const t = setup(probe, true)
await refreshRuntime(t.deps, t.row, 'sess')
expect(t.calls).toHaveLength(1)
}
})
it('still probes without a runtime bundle, and never downgrades a newer image', async () => {
const t = setup(`stamp=${VM_SCRIPTS_STAMP - 1}\nmarker=0`, false)
await expect(refreshRuntime(t.deps, t.row, 'sess')).resolves.toBe('no_bundle')
expect(t.calls).toHaveLength(2)
expect(entryPointsBehind({ stamp: VM_SCRIPTS_STAMP + 1, marker: false })).toBe(false)
expect(entryPointsBehind(null)).toBe(false)
})
})
@@ -3,6 +3,7 @@ import { describe, expect, it } from 'vitest'
import { createGitHubAppMinter } from '../src/github-app'
import { installUrl, isInstallationBound } from '../src/github-connect'
import type { SetupNotice } from '../src/github-repo-access'
import { FOUND_TTL_MS, MISSING_TTL_MS } from '../src/github-installation-cache'
import { storeUserToken } from '../src/github-user-tokens'
import { keyringFor } from '../src/secrets'
import { createHarness } from './harness'
@@ -62,10 +63,11 @@ function fakeGitHub() {
if (url.pathname.startsWith('/user/memberships/orgs/') && faults.membership === 'forbidden') {
return new Response(null, { status: 403 })
}
if (url.pathname.startsWith('/users/') && faults.users === 'error') {
const anonymousProfile = /^\/users\/[^/]+$/.test(url.pathname)
if (anonymousProfile && faults.users === 'error') {
return new Response(null, { status: 502 })
}
if ((url.pathname.startsWith('/user/memberships/orgs/') && faults.membership === 'hang') || (url.pathname.startsWith('/users/') && faults.users === 'hang')) {
if ((url.pathname.startsWith('/user/memberships/orgs/') && faults.membership === 'hang') || (anonymousProfile && faults.users === 'hang')) {
return new Promise<Response>((_, reject) => init?.signal?.addEventListener('abort', () => reject(init.signal!.reason)))
}
if (url.pathname === '/user') {
@@ -100,6 +102,14 @@ function fakeGitHub() {
? Response.json({ id: person.id, login: person.login, type: 'User' })
: new Response(null, { status: 404 })
}
const ownerInstall = /^\/(orgs|users)\/([^/]+)\/installation$/.exec(url.pathname)
if (ownerInstall) {
const login = decodeURIComponent(ownerInstall[2]!).toLowerCase()
const found = installations.find(
(i) => i.account.login.toLowerCase() === login && (ownerInstall[1] === 'orgs') === (i.account.type === 'Organization')
)
return found ? Response.json({ id: found.id, account: found.account }) : new Response(null, { status: 404 })
}
const repoInstall = /^\/repos\/([^/]+)\/([^/]+)\/installation$/.exec(url.pathname)
if (repoInstall) {
const found = installations.find((i) => i.repos.includes(`${repoInstall[1]}/${repoInstall[2]}`.toLowerCase()))
@@ -203,13 +213,18 @@ const stateOf = (url: string) => new URL(url).searchParams.get('state')!
describe('GitHub repo access (cloud-start N2/N3, D2)', () => {
it('answers missing with an install URL, public, then granted once an admin installs and binds', async () => {
const { h, github, notices, trialNotices, browser } = await setup({ publicRepos: ['https://github.com/acme/site'] })
expect((await h.call('GET', `/v1/github/repo-access?repo=${encodeURIComponent('https://github.com/acme/site')}`)).body).toEqual({
const site = await h.call('GET', `/v1/github/repo-access?repo=${encodeURIComponent('https://github.com/acme/site')}`)
expect(site.body).toEqual({
access: 'public',
owner: 'acme',
repo: 'https://github.com/acme/site',
// Clones anonymously, but can't push until the App is on acme (CREATE-GH-STEP).
push: 'not_installed',
installUrl: expect.stringMatching(/^https:\/\/github\.com\/apps\/orca-cloud-test\/installations\/new\?state=/),
// The App isn't on acme yet and alice hasn't connected GitHub: unknown.
canInstall: null
})
expect(HostedGithubRepoAccessResponseSchema.parse(site.body)).toEqual(site.body)
const missing = await h.call('GET', `/v1/github/repo-access?repo=${encodeURIComponent(`${REPO}.git`)}&session=${SESSION}`)
expect(missing.body).toMatchObject({ access: 'missing', owner: 'acme', repo: REPO })
expect(HostedGithubRepoAccessResponseSchema.parse(missing.body)).toEqual(missing.body)
@@ -226,9 +241,10 @@ describe('GitHub repo access (cloud-start N2/N3, D2)', () => {
expect(notices).toEqual([{ sessionId: SESSION, accountId: 'user:alice', repo: REPO, state: 'done' }])
// Billing's trial abuse keys: numeric GitHub owner and user ids of the bind.
expect(trialNotices).toEqual([{ accountId: 'user:alice', githubOwnerId: 50, githubUserId: 3 }])
expect((await h.call('GET', `/v1/github/repo-access?repo=${encodeURIComponent(REPO)}`)).body).toEqual({ access: 'granted', owner: 'acme', repo: REPO })
expect((await h.call('GET', `/v1/github/repo-access?repo=${encodeURIComponent(REPO)}`)).body).toEqual({ access: 'granted', owner: 'acme', repo: REPO, push: 'ok' })
// Another Orca account is not granted by alice's binding.
expect((await h.call('GET', `/v1/github/repo-access?repo=${encodeURIComponent(REPO)}`, { as: 'bob' })).body.access).toBe('missing')
const bob = await h.call('GET', `/v1/github/repo-access?repo=${encodeURIComponent(REPO)}`, { as: 'bob' })
expect(bob.body).toMatchObject({ access: 'missing', push: 'not_connected' })
expect((await h.call('GET', '/v1/github/repo-access?repo=https%3A%2F%2Fgitlab.com%2Facme%2Fweb')).status).toBe(400)
})
@@ -378,6 +394,8 @@ describe('GitHub repo access (cloud-start N2/N3, D2)', () => {
`INSERT INTO github_installations VALUES (900, 'org:acme-team', 50, 'acme', 'Organization', 'u1', 3, 'oauth', 0, NULL)`
)
.run()
// Bound straight in the database (no webhook, no setup return): the cached 404 lasts its TTL.
h.advance(MISSING_TTL_MS)
expect((await h.call('GET', path, { bearer: CONTROL_PLANE_TOKEN })).body).toEqual({ access: 'granted', url: null })
expect((await h.call('GET', path.replace(SESSION, 'bad'), { bearer: CONTROL_PLANE_TOKEN })).status).toBe(400)
})
@@ -440,7 +458,9 @@ describe('repo-access canInstall (REPO-CAN-INSTALL)', () => {
const access = (h: Ctx['h'], repo = REPO, as?: string) => h.call('GET', `/v1/github/repo-access?repo=${encodeURIComponent(repo)}`, as ? { as } : {})
const membershipCalls = (ctx: Ctx) => ctx.github.calls.filter((c) => c.startsWith('GET /user/memberships/orgs/'))
/** canInstall's reads: owner type and membership. */
const github = (ctx: Ctx) => ctx.github.calls.filter((c) => c.startsWith('GET /user/memberships/orgs/') || c.startsWith('GET /users/'))
// The anonymous owner read and the membership read (not the app-JWT installation lookups).
const github = (ctx: Ctx) =>
ctx.github.calls.filter((c) => c.startsWith('GET /user/memberships/orgs/') || (c.startsWith('GET /users/') && !c.endsWith('/installation')))
/** Connects alice (account user:alice, user alice) as the fake GitHub user behind `code`. */
async function connect(ctx: Ctx, code: string, user: { id: number; login: string }, tokens: { accessExpiresAt?: number | null; refresh?: string | null } = {}) {
@@ -541,13 +561,23 @@ describe('repo-access canInstall (REPO-CAN-INSTALL)', () => {
access: 'public',
owner: 'acme',
repo: 'https://github.com/acme/site',
push: 'not_installed',
installUrl: expect.any(String),
canInstall: false
})
await connect(ctx, 'code_admin', { id: 3, login: 'admin-gh' })
expect((await access(ctx.h, 'https://github.com/acme/site')).body).toMatchObject({ access: 'public', canInstall: true })
// Installed on acme (bound to some other Orca account): the App is there, so no canInstall.
ctx.github.installations.push({ id: 900, account: ACME, repos: ['acme/site'] })
expect((await access(ctx.h, 'https://github.com/acme/site')).body).toEqual({ access: 'public', owner: 'acme', repo: 'https://github.com/acme/site' })
ctx.h.advance(MISSING_TTL_MS)
// Pushing still needs alice's account bound to it: the install round trip binds it.
expect((await access(ctx.h, 'https://github.com/acme/site')).body).toEqual({
access: 'public',
owner: 'acme',
repo: 'https://github.com/acme/site',
push: 'not_connected',
installUrl: expect.any(String)
})
})
it('granted and requested: absent', async () => {
@@ -556,7 +586,7 @@ describe('repo-access canInstall (REPO-CAN-INSTALL)', () => {
ctx.github.installations.push({ id: 900, account: ACME, repos: ['acme/web'] })
const setupPage = await ctx.browser(`https://sandbox.test/v1/github/setup?installation_id=900&setup_action=install&state=${encodeURIComponent(stateOf(missing.body.installUrl))}`)
await ctx.browser(`https://sandbox.test/v1/github/callback?code=code_admin&state=${encodeURIComponent(stateOf(setupPage.location!))}`)
expect((await access(ctx.h)).body).toEqual({ access: 'granted', owner: 'acme', repo: REPO })
expect((await access(ctx.h)).body).toEqual({ access: 'granted', owner: 'acme', repo: REPO, push: 'ok' })
// Sam (a member, connected by the request's OAuth round trip) asks an owner: requested.
const other = await setup()
@@ -564,7 +594,7 @@ describe('repo-access canInstall (REPO-CAN-INSTALL)', () => {
const page = await other.browser(`https://sandbox.test/v1/github/setup?setup_action=request&state=${encodeURIComponent(stateOf(samMissing.body.installUrl))}`)
other.github.requests.push({ account: { login: 'acme' }, requester: { id: 2 } })
await other.browser(`https://sandbox.test/v1/github/callback?code=code_sam&state=${encodeURIComponent(stateOf(page.location!))}`)
expect((await access(other.h, REPO, 'sam')).body).toEqual({ access: 'requested', owner: 'acme', repo: REPO })
expect((await access(other.h, REPO, 'sam')).body).toEqual({ access: 'requested', owner: 'acme', repo: REPO, push: 'not_installed' })
})
it('a token near expiry is refreshed through the member-git helper before the membership read', async () => {
@@ -582,3 +612,129 @@ describe('repo-access canInstall (REPO-CAN-INSTALL)', () => {
expect(membershipCalls(ctx)).toEqual([])
})
})
describe('repo-access push reason and installation cache (CREATE-GH-STEP)', () => {
type Ctx = Awaited<ReturnType<typeof setup>>
const access = (h: Ctx['h'], repo = REPO, as?: string) => h.call('GET', `/v1/github/repo-access?repo=${encodeURIComponent(repo)}`, as ? { as } : {})
const installLookups = (ctx: Ctx) => ctx.github.calls.filter((c) => c.endsWith('/installation'))
const bindAcme = async (ctx: Ctx, repos: string[]) => {
const missing = await access(ctx.h)
ctx.github.installations.push({ id: 900, account: ACME, repos })
const setupPage = await ctx.browser(`https://sandbox.test/v1/github/setup?installation_id=900&setup_action=install&state=${encodeURIComponent(stateOf(missing.body.installUrl))}`)
await ctx.browser(`https://sandbox.test/v1/github/callback?code=code_admin&state=${encodeURIComponent(stateOf(setupPage.location!))}`)
}
it('installed on the owner without this repository: repo_not_selected with the installation’s settings page', async () => {
const ctx = await setup({ publicRepos: ['https://github.com/acme/site'] })
await bindAcme(ctx, ['acme/web'])
const site = await access(ctx.h, 'https://github.com/acme/site')
expect(site.body).toEqual({
access: 'public',
owner: 'acme',
repo: 'https://github.com/acme/site',
push: 'repo_not_selected',
configureUrl: 'https://github.com/organizations/acme/settings/installations/900',
// The admin who bound it may change the selection.
canInstall: true
})
expect(HostedGithubRepoAccessResponseSchema.parse(site.body)).toEqual(site.body)
// A private one the installation doesn't select: still missing for cloning, same reason.
expect((await access(ctx.h, 'https://github.com/acme/api')).body).toMatchObject({ access: 'missing', push: 'repo_not_selected', installUrl: expect.any(String) })
})
it('a personal account’s installation links to the user settings page', async () => {
const ctx = await setup({ publicRepos: ['https://github.com/Alice-GH/dotfiles'] })
await connectAlice(ctx)
expect((await access(ctx.h, 'https://github.com/Alice-GH/dotfiles')).body).toMatchObject({
push: 'repo_not_selected',
configureUrl: 'https://github.com/settings/installations/901'
})
})
async function connectAlice(ctx: Ctx) {
const missing = await access(ctx.h, 'https://github.com/Alice-GH/notes')
ctx.github.installations.push({ id: 901, account: { id: 7, login: 'Alice-GH', type: 'User' }, repos: ['alice-gh/notes'] })
const setupPage = await ctx.browser(`https://sandbox.test/v1/github/setup?installation_id=901&setup_action=install&state=${encodeURIComponent(stateOf(missing.body.installUrl))}`)
await ctx.browser(`https://sandbox.test/v1/github/callback?code=code_alice&state=${encodeURIComponent(stateOf(setupPage.location!))}`)
}
it('an installation on the owner bound to another Orca account: not_connected, with an install page', async () => {
const ctx = await setup({ publicRepos: ['https://github.com/acme/site'] })
await bindAcme(ctx, ['acme/web'])
expect((await access(ctx.h, 'https://github.com/acme/site', 'bob')).body).toMatchObject({ access: 'public', push: 'not_connected', installUrl: expect.any(String) })
})
it('polls answer from the cache; installation webhooks drop it so the next poll sees the change', async () => {
const ctx = await setup({ publicRepos: ['https://github.com/acme/site'] })
await bindAcme(ctx, ['acme/web'])
ctx.github.calls.length = 0
expect((await access(ctx.h, 'https://github.com/acme/site')).body.push).toBe('repo_not_selected')
expect((await access(ctx.h, 'https://github.com/acme/site')).body.push).toBe('repo_not_selected')
// One repository lookup for both polls; the bound installation answers the owner without GitHub.
expect(installLookups(ctx)).toEqual(['GET /repos/acme/site/installation'])
// The owner selects acme/site on GitHub: without the webhook the cached 404 would stand for its TTL.
ctx.github.installations[0]!.repos.push('acme/site')
expect((await access(ctx.h, 'https://github.com/acme/site')).body.push).toBe('repo_not_selected')
const added = await ctx.webhook('installation_repositories', {
action: 'added',
installation: { id: 900, account: ACME },
repository_selection: 'selected',
repositories_added: [{ full_name: 'acme/site' }],
repositories_removed: [],
requester: null,
sender: { id: 3, type: 'User' }
})
expect(added.status).toBe(202)
expect((await access(ctx.h, 'https://github.com/acme/site')).body).toMatchObject({ access: 'granted', push: 'ok' })
// Removed again: found installations are cached too, and dropped the same way.
ctx.github.installations[0]!.repos = ['acme/web']
await ctx.webhook('installation_repositories', {
action: 'removed',
installation: { id: 900, account: ACME },
repository_selection: 'selected',
repositories_added: [],
repositories_removed: [{ full_name: 'acme/site' }],
requester: null,
sender: { id: 3, type: 'User' }
})
expect((await access(ctx.h, 'https://github.com/acme/site')).body.push).toBe('repo_not_selected')
})
it('a cached 404 lapses after its TTL; a found installation after a minute', async () => {
const ctx = await setup({ publicRepos: ['https://github.com/acme/site'] })
expect((await access(ctx.h, 'https://github.com/acme/site')).body.push).toBe('not_installed')
ctx.github.installations.push({ id: 900, account: ACME, repos: ['acme/site'] })
expect((await access(ctx.h, 'https://github.com/acme/site')).body.push).toBe('not_installed')
ctx.h.advance(MISSING_TTL_MS)
expect((await access(ctx.h, 'https://github.com/acme/site')).body.push).toBe('not_connected')
ctx.github.calls.length = 0
ctx.h.advance(FOUND_TTL_MS - 1)
await access(ctx.h, 'https://github.com/acme/site')
expect(installLookups(ctx)).toEqual([])
ctx.h.advance(1)
await access(ctx.h, 'https://github.com/acme/site')
expect(installLookups(ctx)).toEqual(['GET /repos/acme/site/installation'])
})
it('the owner lookup failing never fails the answer: not_installed', async () => {
const ctx = await setup({ publicRepos: ['https://github.com/acme/site'] })
const failing = ctx.github.fetch
const h = await createHarness({}, {
github: createGitHubAppMinter({
appId: '1',
privateKeyPkcs8: await exportPKCS8((await generateKeyPair('RS256', { extractable: true })).privateKey),
isBound: async () => false,
log: () => undefined,
fetch: (async (input: RequestInfo | URL, init?: RequestInit) =>
/\/(orgs|users)\/[^/]+\/installation$/.test(new URL(String(input)).pathname) ? new Response(null, { status: 502 }) : failing(input, init)) as typeof fetch
}),
githubFetch: failing,
publicRepoCheck: async () => true
})
const answer = await h.call('GET', `/v1/github/repo-access?repo=${encodeURIComponent('https://github.com/acme/site')}`)
expect(answer.status).toBe(200)
expect(answer.body).toMatchObject({ access: 'public', push: 'not_installed' })
})
})
+5 -1
View File
@@ -88,7 +88,8 @@ export async function createHarness(overrides: Partial<Env> = {}, extraDeps: Par
async function call(
method: string,
path: string,
init: { body?: unknown; as?: string; bearer?: string; headers?: Record<string, string> } = {}
/** `cf`: Cloudflare's request properties (geolocation), which the Workers runtime sets on a real request. */
init: { body?: unknown; as?: string; bearer?: string; headers?: Record<string, string>; cf?: Record<string, unknown> } = {}
) {
const bearer = init.bearer ?? (await token(init.as ?? 'alice'))
const request = new Request(`https://sandbox.test${path}`, {
@@ -96,6 +97,9 @@ export async function createHarness(overrides: Partial<Env> = {}, extraDeps: Par
headers: { authorization: `Bearer ${bearer}`, 'content-type': 'application/json', ...init.headers },
body: init.body === undefined ? undefined : JSON.stringify(init.body)
})
if (init.cf) {
Object.defineProperty(request, 'cf', { value: init.cf })
}
const response = await handleRequest(request, env, deps)
return { status: response.status, body: (await response.json()) as any }
}
@@ -0,0 +1,85 @@
// MEMGUARD: the VM's memory guard (image/vm/orca-memguard) ranks processes for the kernel's OOM killer
// so Orca's runtime and terminals outlive a build that eats the VM's memory, and names the process it
// would stop. Run against a fake process table (ORCA_MEMGUARD_PROC); the kill itself is verified in a VM.
import { spawnSync } from 'node:child_process'
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { fileURLToPath } from 'node:url'
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
const GUARD = fileURLToPath(new URL('../image/vm/orca-memguard', import.meta.url).href)
const START = fileURLToPath(new URL('../image/vm/orca-sandbox-start', import.meta.url).href)
// mapfile -d needs bash 4.4+ (the VM has 5.x; macOS's /bin/bash is 3.2).
const bash = ['/opt/homebrew/bin/bash', '/usr/local/bin/bash', '/usr/bin/bash', '/bin/bash'].find(
(path) => spawnSync(path, ['-c', '[ "${BASH_VERSINFO[0]}${BASH_VERSINFO[1]}" -ge 44 ]']).status === 0
)
const NODE = '/opt/orca/runtime/runtimes/node-abc/bin/node'
let proc = ''
function fake(pid: number, opts: { ppid: number; adj: number; rssPages: number; argv: string[]; comm?: string }): void {
const dir = join(proc, String(pid))
mkdirSync(dir, { recursive: true })
writeFileSync(join(dir, 'cmdline'), opts.argv.length ? `${opts.argv.join('\0')}\0` : '')
writeFileSync(join(dir, 'comm'), `${opts.comm ?? 'x'}\n`)
writeFileSync(join(dir, 'stat'), `${pid} (${opts.comm ?? 'x'}) S ${opts.ppid} ${pid} ${pid} 0 -1\n`)
writeFileSync(join(dir, 'statm'), `${opts.rssPages * 2} ${opts.rssPages} 100 1 0 1 0\n`)
writeFileSync(join(dir, 'oom_score_adj'), `${opts.adj}\n`)
}
const adj = (pid: number): number => Number(readFileSync(join(proc, String(pid), 'oom_score_adj'), 'utf8').trim())
const guard = (command: string) => spawnSync(bash!, [GUARD, command], { env: { PATH: process.env.PATH, ORCA_MEMGUARD_PROC: proc }, encoding: 'utf8' })
beforeEach(() => {
proc = mkdtempSync(join(tmpdir(), 'orca-memguard-proc-'))
fake(1, { ppid: 0, adj: 500, rssPages: 3000, argv: ['/run/vercel/share/sandbox-init'] })
fake(100, { ppid: 1, adj: 500, rssPages: 2_000_000, argv: [NODE, '/opt/orca/bin/orcad-boot', '--code-cache'], comm: 'MainThread' })
fake(101, { ppid: 100, adj: 500, rssPages: 20_000, argv: [NODE, '--max-semi-space-size=4', '/opt/orca/runtime/orcad/daemon-entry.js'] })
// A terminal's shell, and a typecheck it started (inherited the shell's rank).
fake(102, { ppid: 101, adj: 500, rssPages: 1_000, argv: ['/bin/bash', '-l'], comm: 'bash' })
fake(103, { ppid: 102, adj: -500, rssPages: 900_000, argv: ['node', '--max-old-space-size=8192', '/w/node_modules/typescript/bin/tsc', '-p', '.'], comm: 'MainThread' })
fake(104, { ppid: 102, adj: 300, rssPages: 40_000, argv: ['/opt/google/chrome/chrome', '--type=renderer'], comm: 'chrome' })
fake(105, { ppid: 102, adj: 800, rssPages: 40_000, argv: ['/usr/bin/some-tool'] })
fake(106, { ppid: 1, adj: -1000, rssPages: 600, argv: ['sshd: /opt/orca/openssh/sshd'] })
fake(107, { ppid: 102, adj: 500, rssPages: 400_000, argv: ['/opt/orca/tools/bin/claude'], comm: 'claude' })
fake(108, { ppid: 2, adj: 0, rssPages: 0, argv: [] }) // a kernel thread
})
afterEach(() => rmSync(proc, { recursive: true, force: true }))
describe.skipIf(!bash)('orca-memguard', () => {
it('protects the runtime, terminals and agents, and makes everything else the first victim', () => {
expect(guard('sweep').status).toBe(0)
expect([100, 101].map(adj)).toEqual([-900, -900])
expect(adj(102)).toBe(-500) // the terminal's shell
expect(adj(107)).toBe(-500) // an agent CLI
expect(adj(103)).toBe(500) // the shell's typecheck no longer inherits its rank
expect(adj(104)).toBe(500)
expect(adj(105)).toBe(800) // a process that raised its own rank keeps it
expect(adj(106)).toBe(-1000) // what root pinned stays pinned
expect(adj(108)).toBe(0) // kernel threads are not touched
expect(adj(1)).toBe(500)
})
it('would stop the largest unprotected process and names it by its script', () => {
const result = guard('victim')
expect(result.status).toBe(0)
expect(result.stdout.trim()).toBe(`103 ${900_000 * 4} tsc`)
})
it('names inline code by its interpreter, and leaves small processes to the kernel', () => {
fake(103, { ppid: 102, adj: 500, rssPages: 900_000, argv: ['node', '-e', 'const secret = 1'], comm: 'MainThread' })
expect(guard('victim').stdout.trim()).toBe(`103 ${900_000 * 4} node`)
rmSync(join(proc, '103'), { recursive: true })
rmSync(join(proc, '104'), { recursive: true })
rmSync(join(proc, '105'), { recursive: true })
// Only protected memory is large: nothing worth stopping.
expect(guard('victim').status).toBe(1)
})
it('starts from orca-sandbox-start before its already-running exit', () => {
const start = readFileSync(START, 'utf8')
const launch = start.indexOf('orca-memguard" start')
expect(launch).toBeGreaterThan(0)
expect(launch).toBeLessThan(start.indexOf('print_ready\n exit 0'))
})
})
@@ -0,0 +1,219 @@
// Nearest region: a create or a new environment with nothing pinned starts in the served region
// nearest the caller (Cloudflare's request geolocation); explicit regions and environments win.
import { describe, expect, it } from 'vitest'
import { refreshStaleEnvironments } from '../src/environment-golden-refresh'
import { maintainPool, poolConfigFromEnv, type PoolDeps } from '../src/pool'
import { nearestRegions, regionsByDistance, requestGeo, servedRegions } from '../src/regions'
import { GOLDEN, createHarness } from './harness'
const SAN_FRANCISCO = { latitude: '37.7749', longitude: '-122.4194', continent: 'NA', country: 'US' }
const NEW_YORK = { latitude: '40.7128', longitude: '-74.0060', continent: 'NA', country: 'US' }
const LONDON = { latitude: '51.5072', longitude: '-0.1276', continent: 'EU', country: 'GB' }
const TOKYO = { latitude: '35.6762', longitude: '139.6503', continent: 'AS', country: 'JP' }
const PDX1_GOLDEN = 'snap_pdx1'
const REPO = 'https://github.com/acme/web.git'
function located(cf: Record<string, unknown>): Request {
const request = new Request('https://sandbox.test/v1/sandboxes')
Object.defineProperty(request, 'cf', { value: cf })
return request
}
describe('regionsByDistance', () => {
const served = ['pdx1', 'iad1']
it('orders the served regions nearest the caller first', () => {
expect(regionsByDistance(requestGeo(located(SAN_FRANCISCO)), served)).toEqual(['pdx1', 'iad1'])
expect(regionsByDistance(requestGeo(located(NEW_YORK)), served)).toEqual(['iad1', 'pdx1'])
expect(regionsByDistance(requestGeo(located(LONDON)), served)).toEqual(['iad1', 'pdx1'])
expect(regionsByDistance(requestGeo(located(TOKYO)), served)).toEqual(['pdx1', 'iad1'])
expect(regionsByDistance(requestGeo(located(LONDON)), ['pdx1', 'iad1', 'fra1'])).toEqual(['fra1', 'iad1', 'pdx1'])
})
it('uses the continent when Cloudflare has no coordinates, and keeps the configured order without either', () => {
expect(regionsByDistance({ continent: 'EU' }, served)).toEqual(['iad1', 'pdx1'])
expect(regionsByDistance({ continent: 'XX' }, served)).toEqual(served)
expect(regionsByDistance(null, served)).toEqual(served)
})
it('puts a region with no known site after the located ones, in its configured place', () => {
expect(regionsByDistance(requestGeo(located(NEW_YORK)), ['zzz1', 'pdx1', 'iad1'])).toEqual(['iad1', 'pdx1', 'zzz1'])
})
it('reads request.cf, and nothing outside the Workers runtime', () => {
expect(requestGeo(located(NEW_YORK))).toEqual({ latitude: 40.7128, longitude: -74.006, continent: 'NA' })
expect(requestGeo(located({ continent: 'EU' }))).toEqual({ continent: 'EU' })
expect(requestGeo(located({ latitude: 'x', longitude: '1' }))).toBeNull()
expect(requestGeo(new Request('https://sandbox.test/'))).toBeNull()
})
it('decides nothing when off, unlocated, or with a single served region', () => {
expect(nearestRegions(located(NEW_YORK), served)).toEqual(['iad1', 'pdx1'])
expect(nearestRegions(located(NEW_YORK), served, false)).toBeUndefined()
expect(nearestRegions(new Request('https://sandbox.test/'), served)).toBeUndefined()
expect(nearestRegions(located(NEW_YORK), ['pdx1'])).toBeUndefined()
expect(servedRegions({ defaultRegion: 'pdx1', fallbackRegions: ['iad1', 'pdx1'] })).toEqual(['pdx1', 'iad1'])
})
})
/** pdx1 the default and iad1 its fallback, each with an active golden (iad1's is the seed, GOLDEN). */
async function twoRegions(overrides: Record<string, string> = {}, waitUntil?: (work: Promise<unknown>) => void) {
const h = await createHarness({ VERCEL_REGION: 'pdx1', VERCEL_FALLBACK_REGIONS: 'iad1', ...overrides }, waitUntil ? { waitUntil } : {})
h.raw
.prepare(
`INSERT INTO golden_images (id, region, source, status, snapshot_id, created_at, updated_at, activated_at)
VALUES ('g1', 'pdx1', 'seed', 'active', ?, 1, 1, 1)`
)
.run(PDX1_GOLDEN)
const box = (id: string) => h.vercel.boxes.get(`orca-test-${id}`)!.params
return { h, box }
}
describe('nearest region on create', () => {
it('creates in the served region nearest the caller, on that region’s golden', async () => {
const { h, box } = await twoRegions()
const east = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-ny', machineSize: 'standard' }, cf: NEW_YORK })
expect(east.status).toBe(200)
expect(east.body.sandbox.region).toBe('iad1')
expect(box(east.body.sandbox.id)).toMatchObject({ region: 'iad1', snapshotId: GOLDEN })
const west = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-sf', machineSize: 'standard' }, cf: SAN_FRANCISCO })
expect(west.body.sandbox.region).toBe('pdx1')
expect(box(west.body.sandbox.id)).toMatchObject({ region: 'pdx1', snapshotId: PDX1_GOLDEN })
// Unlocated (no request.cf): the default region, as before.
const unknown = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-x', machineSize: 'standard' } })
expect(unknown.body.sandbox.region).toBe('pdx1')
})
it('keeps an explicit region authoritative', async () => {
const { h } = await twoRegions()
const pinned = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard', region: 'pdx1' }, cf: NEW_YORK })
expect(pinned.body.sandbox.region).toBe('pdx1')
// Pinned means pinned: a capacity failure there is not moved to the nearer region.
h.vercel.failCreateRegions.add('pdx1')
const refused = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-2', machineSize: 'standard', region: 'pdx1' }, cf: NEW_YORK })
expect(refused.status).not.toBe(200)
})
it('never picks a region without an active golden, and falls back to the next nearest on capacity', async () => {
// iad1 is the default here and pdx1 has no image: a caller nearest pdx1 still lands in iad1.
const h = await createHarness({ VERCEL_REGION: 'iad1', VERCEL_FALLBACK_REGIONS: 'pdx1' })
const west = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-sf', machineSize: 'standard' }, cf: SAN_FRANCISCO })
expect(west.body.sandbox.region).toBe('iad1')
const { h: two } = await twoRegions()
two.vercel.failCreateRegions.add('iad1')
const moved = await two.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-ny', machineSize: 'standard' }, cf: NEW_YORK })
expect(moved.status).toBe(200)
expect(moved.body.sandbox.region).toBe('pdx1')
})
it('VERCEL_NEAREST_REGION=off keeps every create in the default region', async () => {
const { h } = await twoRegions({ VERCEL_NEAREST_REGION: 'off' })
const east = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-ny', machineSize: 'standard' }, cf: NEW_YORK })
expect(east.body.sandbox.region).toBe('pdx1')
})
it('resumes a sandbox in the region it was created in, wherever the caller is now', async () => {
const { h, box } = await twoRegions()
const created = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-ny', machineSize: 'standard' }, cf: NEW_YORK })
const id = created.body.sandbox.id
expect((await h.call('POST', `/v1/sandboxes/${id}/suspend`, { cf: SAN_FRANCISCO })).status).toBe(200)
const resumed = await h.call('POST', `/v1/sandboxes/${id}/resume`, { cf: SAN_FRANCISCO })
expect(resumed.status).toBe(200)
expect(resumed.body.sandbox.region).toBe('iad1')
expect(h.vercel.boxes.size).toBe(1)
expect(box(id).region).toBe('iad1')
expect((h.raw.prepare('SELECT region FROM sandboxes WHERE id = ?').get(id) as { region: string }).region).toBe('iad1')
})
})
describe('nearest region for environments', () => {
async function environmentIn(cf: Record<string, unknown>) {
const { h, box } = await twoRegions()
h.vercel.buildResult = JSON.stringify({ repos: [{ url: REPO, dir: 'acme/web', branch: 'main', commit: 'a'.repeat(40), lockfiles: {} }] })
const env = await h.call('POST', '/v1/environments', { body: { name: 'Default', repos: [{ url: 'acme/web' }] }, cf })
expect(env.status).toBe(200)
const envId = env.body.environment.id as string
const started = await h.call('POST', `/v1/environments/${envId}/builds`, { cf: SAN_FRANCISCO })
;[...h.vercel.buildCommands.values()].at(-1)!.exitCode = 0
const build = (await h.call('GET', `/v1/environments/${envId}/builds/${started.body.build.id}`)).body.build
expect(build.status).toBe('succeeded')
return { h, box, envId, buildId: build.id as string }
}
it('records the region nearest the admin who creates it, builds there, and boots its workspaces there', async () => {
const { h, box, envId, buildId } = await environmentIn(NEW_YORK)
expect(h.raw.prepare('SELECT region FROM environments WHERE id = ?').get(envId)).toEqual({ region: 'iad1' })
// Built in iad1 from iad1's golden, whoever pressed Build and wherever they are.
expect(h.raw.prepare('SELECT region, base_snapshot_id FROM environment_builds WHERE id = ?').get(buildId)).toEqual({ region: 'iad1', base_snapshot_id: GOLDEN })
// A teammate on the west coast boots from it in iad1: the snapshot only boots there.
const created = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-sf', machineSize: 'standard', repo: { url: REPO } }, cf: SAN_FRANCISCO })
expect(created.status).toBe(200)
expect(created.body.sandbox).toMatchObject({ region: 'iad1', environmentBuildId: buildId })
expect(box(created.body.sandbox.id).region).toBe('iad1')
})
it('compares an environment with its own region’s golden: no refresh churn for an iad1 environment', async () => {
const { h, envId } = await environmentIn(NEW_YORK)
const deps = {
db: h.env.SANDBOX_DB,
provider: h.vercel,
entitlements: { checkStart: async () => h.billing.decision, reportUsage: async () => h.billing.usageAnswer, listSuspensions: async () => [] },
now: h.now,
config: { environment: 'test', goldenSnapshotId: PDX1_GOLDEN, goldenSnapshots: { pdx1: PDX1_GOLDEN, iad1: GOLDEN }, region: 'pdx1', regions: ['pdx1', 'iad1'] }
}
expect(await refreshStaleEnvironments(deps as never, { minIntervalMs: 0 })).toMatchObject({ stale: 0, started: 0 })
// An environment from before regions were recorded lives in the default region: its iad1 build is stale.
h.raw.prepare('UPDATE environments SET region = NULL WHERE id = ?').run(envId)
expect(await refreshStaleEnvironments(deps as never, { minIntervalMs: 0 })).toMatchObject({ stale: 1, started: 1 })
expect(h.raw.prepare('SELECT region, base_snapshot_id FROM environment_builds WHERE environment_id = ? ORDER BY number DESC').get(envId)).toEqual({
region: 'pdx1',
base_snapshot_id: PDX1_GOLDEN
})
})
it('an environment made in the default region (west coast admin) stays there', async () => {
const { h, envId, buildId } = await environmentIn(SAN_FRANCISCO)
expect(h.raw.prepare('SELECT region FROM environments WHERE id = ?').get(envId)).toEqual({ region: 'pdx1' })
expect(h.raw.prepare('SELECT region, base_snapshot_id FROM environment_builds WHERE id = ?').get(buildId)).toEqual({ region: 'pdx1', base_snapshot_id: PDX1_GOLDEN })
})
it('an automatic environment lives where the create that made it ran', async () => {
const pending: Promise<unknown>[] = []
const { h } = await twoRegions({ AUTO_ENVIRONMENTS: 'on' }, (work) => void pending.push(work))
const created = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'w1', machineSize: 'standard', repo: { url: REPO } }, cf: NEW_YORK })
expect(created.body.sandbox.region).toBe('iad1')
while (pending.length > 0) {
await Promise.all(pending.splice(0))
}
expect(h.raw.prepare(`SELECT region FROM environments WHERE kind = 'auto'`).all()).toEqual([{ region: 'iad1' }])
expect(h.raw.prepare('SELECT region, base_snapshot_id FROM environment_builds').all()).toEqual([{ region: 'iad1', base_snapshot_id: GOLDEN }])
})
})
describe('warm pool per served region', () => {
it('keeps warm VMs in every served region, and a create claims one in its nearest region', async () => {
const { h } = await twoRegions({ SANDBOX_POOL_SIZE: '1' })
const deps: PoolDeps = {
db: h.env.SANDBOX_DB,
provider: h.vercel,
now: h.now,
config: {
environment: 'test',
goldenSnapshotId: PDX1_GOLDEN,
goldenSnapshots: { pdx1: PDX1_GOLDEN, iad1: GOLDEN },
defaultRegion: 'pdx1',
fallbackRegions: ['iad1'],
pool: poolConfigFromEnv(h.env)
}
}
expect(await maintainPool(deps)).toMatchObject({ created: 2 })
const pool = () => h.raw.prepare(`SELECT id, region, snapshot_id FROM sandbox_pool WHERE status = 'ready' ORDER BY region`).all() as { id: string; region: string; snapshot_id: string }[]
expect(pool().map(({ region, snapshot_id }) => ({ region, snapshot_id }))).toEqual([
{ region: 'iad1', snapshot_id: GOLDEN },
{ region: 'pdx1', snapshot_id: PDX1_GOLDEN }
])
const warmEast = pool().find((vm) => vm.region === 'iad1')!.id
const east = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-ny', machineSize: 'standard' }, cf: NEW_YORK })
expect(east.body.sandbox).toMatchObject({ id: warmEast, region: 'iad1' })
})
})
@@ -135,3 +135,37 @@ describe('orca-sandbox-env: Claude Code API-key approval (G9)', () => {
expect(claude(file).hasCompletedOnboarding).toBe(true)
})
})
describe('orca-sandbox-env: Claude signed in inside the workspace', () => {
const claudeJson = (file: (path: string) => string | null) => JSON.parse(file('.claude.json') ?? '{}')
it('with no login yet, leaves onboarding to Claude so it opens on its own sign-in screens; never a shell variable', () => {
const { run, file, sessionEnv } = scratch()
run({ ORCA_SIGN_IN_INSIDE: 'claude,codex', FOO: 'bar' })
const c = claudeJson(file)
expect(c.hasCompletedOnboarding).toBeUndefined()
// The other first-run answers still apply.
expect(c.bypassPermissionsModeAccepted).toBe(true)
expect(sessionEnv()).not.toContain('ORCA_SIGN_IN_INSIDE')
})
it('clears a mark an earlier session made, and marks onboarding done once a login exists here', () => {
const { home, run, file } = scratch()
run({})
expect(claudeJson(file).hasCompletedOnboarding).toBe(true)
run({ ORCA_SIGN_IN_INSIDE: 'claude' })
expect(claudeJson(file).hasCompletedOnboarding).toBeUndefined()
mkdirSync(join(home, '.claude'), { recursive: true })
writeFileSync(join(home, '.claude', '.credentials.json'), '{}')
run({ ORCA_SIGN_IN_INSIDE: 'claude' })
expect(claudeJson(file).hasCompletedOnboarding).toBe(true)
})
it('only Codex inside, or a malformed value, keeps Claude onboarding done', () => {
const { run, file } = scratch()
run({ ORCA_SIGN_IN_INSIDE: 'codex' })
expect(claudeJson(file).hasCompletedOnboarding).toBe(true)
run({ ORCA_SIGN_IN_INSIDE: 'claude;rm -rf /' })
expect(claudeJson(file).hasCompletedOnboarding).toBe(true)
})
})
@@ -0,0 +1,51 @@
// PHASE-DEADLINES: a create sat 180 s at "Start sandbox" before its write-ahead insert, with no
// provider call stalled. The pre-insert phases are bounded and a stuck one is named in the log.
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { LifecycleError, PHASE_DEADLINES_MS, phaseTimer } from '../src/lifecycle'
const never = () => new Promise<never>(() => undefined)
describe('PHASE-DEADLINES', () => {
beforeEach(() => {
vi.useFakeTimers()
vi.spyOn(console, 'log').mockImplementation(() => undefined)
})
afterEach(() => {
vi.useRealTimers()
vi.restoreAllMocks()
})
it('fails a stuck entitlement phase by name and logs it', async () => {
const timer = phaseTimer(() => Date.now())
const run = timer.time('entitlement', never)
const caught = run.catch((error: unknown) => error)
await vi.advanceTimersByTimeAsync(PHASE_DEADLINES_MS.entitlement.ms)
const error = await caught
expect(error).toBeInstanceOf(LifecycleError)
expect((error as LifecycleError).code).toBe('provider_unavailable')
expect((error as LifecycleError).reason).toBe('phase_timeout_entitlement')
expect(timer.failed?.phase).toBe('entitlement')
expect(console.log).toHaveBeenCalledWith(expect.stringContaining('"event":"phase_timeout"'))
})
it('only logs a slow pool claim: abandoning it could strand the claimed VM', async () => {
const timer = phaseTimer(() => Date.now())
let finish: (value: string) => void = () => undefined
const run = timer.time('pool', () => new Promise<string>((resolve) => (finish = resolve)))
await vi.advanceTimersByTimeAsync(PHASE_DEADLINES_MS.pool.ms + 1_000)
expect(console.log).toHaveBeenCalledWith(expect.stringContaining('"phase":"pool"'))
finish('vm')
await expect(run).resolves.toBe('vm')
expect(timer.failed).toBeNull()
})
it('leaves a fast phase and the provider-bounded phases alone', async () => {
const timer = phaseTimer(() => Date.now())
await expect(timer.time('environment', async () => 1)).resolves.toBe(1)
const vm = timer.time('vm', () => new Promise<number>((resolve) => setTimeout(() => resolve(2), 90_000)))
await vi.advanceTimersByTimeAsync(90_000)
await expect(vm).resolves.toBe(2)
expect(console.log).not.toHaveBeenCalled()
expect(vi.getTimerCount()).toBe(0)
})
})
@@ -154,6 +154,28 @@ describe('status panel', () => {
expect(status.metrics).toMatchObject({ oomKills: 1, lastOomVictim: 'node' })
})
it('reports memory pressure and guard kills beside the metrics, which keep their strict shape', async () => {
const { h, id } = await running()
const guard = { level: 'high', kills: 2, lastKillName: 'tsc', lastKillAt: 1_800_000_000_000, swapBytes: 2 ** 31 }
h.vercel.onScript = (script) => (script === METRICS_SCRIPT ? { exitCode: 0, stdout: `${metricsLine({ guard })}\n`, stderr: '' } : undefined)
const body = (await h.call('GET', `/v1/sandboxes/${id}/status`)).body
expect(body.status.metrics).not.toHaveProperty('guard')
const { status } = HostedWorkspaceStatusResponseSchema.parse(body)
expect(status.state).toBe('ready')
expect(status.memory).toEqual({ pressure: 'high', guardKills: 2, lastGuardVictim: 'tsc', lastGuardKillAt: 1_800_000_000_000, guarded: true })
})
it('derives pressure from MemAvailable when the VM runs no guard (an older image)', async () => {
const { h, id } = await running()
h.vercel.onScript = (script) =>
script === METRICS_SCRIPT ? { exitCode: 0, stdout: `${metricsLine({ memUsedBytes: 7.5e9, guard: null })}\n`, stderr: '' } : undefined
const { status } = HostedWorkspaceStatusResponseSchema.parse((await h.call('GET', `/v1/sandboxes/${id}/status`)).body)
expect(status.memory).toEqual({ pressure: 'critical', guardKills: 0, lastGuardVictim: null, lastGuardKillAt: null, guarded: false })
h.vercel.onScript = (script) => (script === METRICS_SCRIPT ? { exitCode: 0, stdout: `${metricsLine()}\n`, stderr: '' } : undefined)
const again = HostedWorkspaceStatusResponseSchema.parse((await h.call('GET', `/v1/sandboxes/${id}/status`)).body)
expect(again.status.memory?.pressure).toBe('ok')
})
it('counts settled sessions in the cost and does not probe a sleeping VM', async () => {
const { h, id } = await running()
h.advance(60 * 60_000)
+2 -1
View File
@@ -23,6 +23,7 @@
"include": [
"src",
"test",
"image"
"image",
"scripts"
]
}
@@ -78,6 +78,9 @@
// person in the team" check before a workspace changes owner. Unset = hand-offs refused.
"AUTH_INTERNAL_ORIGIN": "https://auth-staging.onorca.dev",
"EGRESS_PROXY_URL": "https://orca-sandbox-control-dev.lovecast.workers.dev/v1/egress/proxy",
// CODEX-RELAY: chatgpt.com refuses Workers' egress; its requests go through this Cloud Run relay
// (cloud/infra/chatgpt-relay, staging). Secret CHATGPT_RELAY_SERVICE_ACCOUNT holds its invoker key.
"CHATGPT_RELAY_URL": "https://orca-chatgpt-relay-gjzz5mc7ka-uc.a.run.app",
// SEC-3 phase 1: the secret store's KEK comes out of Cloud KMS (src/secrets-kms.ts); the Worker
// holds only the KMS-wrapped KEK (secret SECRETS_KEK_WRAPPED, AAD orca-secrets-kek:<SECRETS_KEK_ID>)
// and a service account that can only decrypt with this key (secret SECRETS_KMS_SERVICE_ACCOUNT).
+1
View File
@@ -0,0 +1 @@
.terraform/
+22
View File
@@ -0,0 +1,22 @@
# This file is maintained automatically by "terraform init".
# Manual edits may be lost in future updates.
provider "registry.terraform.io/hashicorp/google" {
version = "6.50.0"
constraints = "~> 6.0"
hashes = [
"h1:79CwMTsp3Ud1nOl5hFS5mxQHyT0fGVye7pqpU0PPlHI=",
"zh:1f3513fcfcbf7ca53d667a168c5067a4dd91a4d4cccd19743e248ff31065503c",
"zh:3da7db8fc2c51a77dd958ea8baaa05c29cd7f829bd8941c26e2ea9cb3aadc1e5",
"zh:3e09ac3f6ca8111cbb659d38c251771829f4347ab159a12db195e211c76068bb",
"zh:7bb9e41c568df15ccf1a8946037355eefb4dfb4e35e3b190808bb7c4abae547d",
"zh:81e5d78bdec7778e6d67b5c3544777505db40a826b6eb5abe9b86d4ba396866b",
"zh:8d309d020fb321525883f5c4ea864df3d5942b6087f6656d6d8b3a1377f340fc",
"zh:93e112559655ab95a523193158f4a4ac0f2bfed7eeaa712010b85ebb551d5071",
"zh:d3efe589ffd625b300cef5917c4629513f77e3a7b111c9df65075f76a46a63c7",
"zh:d4a4d672bbef756a870d8f32b35925f8ce2ef4f6bbd5b71a3cb764f1b6c85421",
"zh:e13a86bca299ba8a118e80d5f84fbdd708fe600ecdceea1a13d4919c068379fe",
"zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
"zh:fec30c095647b583a246c39d557704947195a1b7d41f81e369ba377d997faef6",
]
}
+31
View File
@@ -0,0 +1,31 @@
# ChatGPT relay (CODEX-RELAY)
chatgpt.com answers Cloudflare Workers' egress with a 403 "Unable to load site" page, so Codex via the
Orca account can't reach it through the sandbox-control egress proxy directly. The proxy sends its
chatgpt.com requests to this Cloud Run service instead (`cloud/apps/chatgpt-relay`), which:
- forwards to `https://chatgpt.com` only, on the Codex paths in `sandbox-control/src/codex-paths.ts`;
- streams responses as they arrive (SSE), refuses WebSocket upgrades (Codex is configured for HTTPS);
- logs method, a two-segment path, status and duration: never headers, bodies or queries;
- is callable only by `orca-chatgpt-relay-invoker` (Cloud Run IAM, ID token in `X-Serverless-Authorization`).
## Staging (serves orca-sandbox-control-dev)
```
terraform init -backend-config=backend/staging.hcl
terraform apply -var-file=environments/staging.tfvars # registry + accounts
IMG=$(CLOUDSDK_CORE_ACCOUNT=<you> ../../apps/chatgpt-relay/build-image.sh onorca-cloud-staging)
terraform apply -var-file=environments/staging.tfvars -var image=$IMG # the service
```
Worker: `CHATGPT_RELAY_URL` (wrangler var) = the `relay_url` output; secret `CHATGPT_RELAY_SERVICE_ACCOUNT` =
a key of `invoker_service_account`, created with `gcloud iam service-accounts keys create` into a 0600 temp
file, piped into `wrangler secret put`, then deleted. Without both, chatgpt.com goes direct, as before.
## Production (runbook, at Neil's go)
1. Add `backend/production.hcl` (bucket `onorca-cloud-terraform-state`, prefix `terraform/chatgpt-relay`) and
`environments/production.tfvars` (`project_id = "onorca-cloud"`, `environment = "production"`).
2. The same three steps against `onorca-cloud`.
3. Set `CHATGPT_RELAY_URL` in `wrangler.prod.jsonc` and the secret on `orca-sandbox-control-prod`; deploy.
4. Check: a Codex turn via the Orca account answers; the relay's logs show `relay ... status 200`.
@@ -0,0 +1,2 @@
bucket = "onorca-cloud-staging-terraform-state"
prefix = "terraform/chatgpt-relay"
@@ -0,0 +1,3 @@
# Serves orca-sandbox-control-dev (CHATGPT_RELAY_URL). Prod: a runbook step, not applied from here yet.
project_id = "onorca-cloud-staging"
environment = "staging"
+104
View File
@@ -0,0 +1,104 @@
# CODEX-RELAY: the ChatGPT relay. chatgpt.com refuses Cloudflare Workers' egress, so the sandbox-control
# Worker's egress proxy sends chatgpt.com requests through this Cloud Run service (cloud/apps/chatgpt-relay).
# It forwards to chatgpt.com only, on the Codex paths the Worker allows, and streams the answers back.
# Only the invoker service account (held by the Worker as CHATGPT_RELAY_SERVICE_ACCOUNT) may call it.
# Own root and state (backend prefix terraform/chatgpt-relay). The invoker's key is created with gcloud and
# piped straight into the Worker secret: no key material in this root or its state.
locals {
labels = {
app = "orca-cloud"
component = "chatgpt-relay"
environment = var.environment
managed_by = "terraform"
}
}
resource "google_project_service" "run" {
project = var.project_id
service = "run.googleapis.com"
disable_on_destroy = false
}
resource "google_project_service" "artifactregistry" {
project = var.project_id
service = "artifactregistry.googleapis.com"
disable_on_destroy = false
}
resource "google_artifact_registry_repository" "relay" {
project = var.project_id
location = var.region
repository_id = "orca-chatgpt-relay"
format = "DOCKER"
labels = local.labels
depends_on = [google_project_service.artifactregistry]
}
# The service runs as an account with no grants: it needs nothing in Google Cloud.
resource "google_service_account" "runtime" {
project = var.project_id
account_id = "orca-chatgpt-relay"
display_name = "ChatGPT relay runtime (no grants)"
}
resource "google_service_account" "invoker" {
project = var.project_id
account_id = "orca-chatgpt-relay-invoker"
display_name = "sandbox-control Worker: run.invoker on the ChatGPT relay only"
description = "Held by the sandbox-control Worker as CHATGPT_RELAY_SERVICE_ACCOUNT. Its only grant is run.invoker on orca-chatgpt-relay."
}
resource "google_cloud_run_v2_service" "relay" {
count = var.image == "" ? 0 : 1
project = var.project_id
name = "orca-chatgpt-relay"
location = var.region
ingress = "INGRESS_TRAFFIC_ALL"
labels = local.labels
deletion_protection = false
template {
service_account = google_service_account.runtime.email
# Codex turns stream for minutes.
timeout = "3600s"
max_instance_request_concurrency = 80
scaling {
min_instance_count = 0
max_instance_count = 4
}
containers {
image = var.image
resources {
limits = {
cpu = "1"
memory = "512Mi"
}
# SSE streams: keep CPU while a response is still being written.
cpu_idle = false
}
startup_probe {
http_get {
path = "/health"
}
}
}
}
depends_on = [google_project_service.run]
}
resource "google_cloud_run_v2_service_iam_member" "invoker" {
count = var.image == "" ? 0 : 1
project = var.project_id
location = var.region
name = google_cloud_run_v2_service.relay[0].name
role = "roles/run.invoker"
member = "serviceAccount:${google_service_account.invoker.email}"
}
+12
View File
@@ -0,0 +1,12 @@
output "relay_url" {
description = "CHATGPT_RELAY_URL for the sandbox-control Worker."
value = var.image == "" ? null : google_cloud_run_v2_service.relay[0].uri
}
output "image_repository" {
value = "${var.region}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.relay.repository_id}"
}
output "invoker_service_account" {
value = google_service_account.invoker.email
}
+18
View File
@@ -0,0 +1,18 @@
variable "project_id" {
type = string
}
variable "environment" {
type = string
}
variable "region" {
type = string
default = "us-central1"
}
variable "image" {
description = "Relay image (cloud/apps/chatgpt-relay/build-image.sh prints it). Empty: the root creates only the registry and accounts."
type = string
default = ""
}
+16
View File
@@ -0,0 +1,16 @@
terraform {
required_version = ">= 1.7.0"
required_providers {
google = {
source = "hashicorp/google"
version = "~> 6.0"
}
}
backend "gcs" {}
}
provider "google" {
project = var.project_id
}
+12
View File
@@ -235,6 +235,18 @@ importers:
specifier: ^4.157.0
version: 4.157.0(@cloudflare/workers-types@5.20261104.1)(@types/node@24.19.2)
apps/chatgpt-relay:
devDependencies:
'@types/node':
specifier: ^24.19.2
version: 24.19.2
typescript:
specifier: ^7.0.2
version: 7.0.2
vitest:
specifier: ^5.0.5
version: 5.0.5(@types/node@24.19.2)(vite@8.3.4(@types/node@24.19.2)(esbuild@0.28.2)(tsx@4.23.15))
apps/push:
dependencies:
'@hono/node-server':
@@ -49,6 +49,12 @@ const NON_UNIT_TEST_RUNNERS = [
sources: [`cloud/apps/${app}/vitest.config.ts`],
mirrors: ["include: ['src/**/*.test.ts']"]
})),
{
name: 'cloud chatgpt-relay',
include: ['cloud/apps/chatgpt-relay/test/**/*.test.ts'],
sources: ['cloud/apps/chatgpt-relay/vitest.config.ts'],
mirrors: ["include: ['test/**/*.test.ts']"]
},
{
name: 'cloud relay-edge',
include: ['cloud/apps/relay-edge/test/**/*.test.ts'],
+48
View File
@@ -0,0 +1,48 @@
#!/usr/bin/env bash
# Premerge step for image changes (premerge-check.sh runs it only when cloud/apps/sandbox-control/image/** changed):
# provisions the golden image in Docker and boots it twice; the second boot must install, fetch and redo nothing.
# See cloud/apps/sandbox-control/image/boot-twice/README.md. Run from the merged worktree's root.
# - Docker unavailable: a loud warning and exit 0 (fail open), so a host without Docker never blocks every merge.
# - On omarchy it takes its own slot (~/orca-offload/bin/slot.sh), waiting at most BOOT_TWICE_SLOT_WAIT_S (300 s).
# - Hard limit BOOT_TWICE_TIMEOUT_S (600 s); a cold image build is ~5 min, a cached one ~15 s.
set -uo pipefail
timeout_s=${BOOT_TWICE_TIMEOUT_S:-600}
slot_wait_s=${BOOT_TWICE_SLOT_WAIT_S:-300}
if ! docker info >/dev/null 2>&1; then
printf '\npremerge: WARNING ==============================================================\n' >&2
printf 'premerge: WARNING: Docker is unavailable here, so the boot-twice image check DID NOT RUN.\n' >&2
printf 'premerge: WARNING: run it by hand before merging this image change (see image/boot-twice/README.md).\n' >&2
printf 'premerge: WARNING ==============================================================\n\n' >&2
exit 0
fi
slot_tool="$HOME/orca-offload/bin/slot.sh"
tag="boot-twice-$$-$(date +%s)"
if [[ -x "$slot_tool" ]]; then
waited=0
until "$slot_tool" acquire "$tag" >/dev/null 2>&1; do
if ((waited >= slot_wait_s)); then
echo "premerge: FAILED: boot-twice image check found no free omarchy slot in ${slot_wait_s}s; rerun the premerge" >&2
exit 1
fi
sleep 15
waited=$((waited + 15))
done
trap '"$slot_tool" release "$tag" >/dev/null 2>&1 || rm -rf "$HOME/orca-offload/wt/$tag"' EXIT
fi
echo "premerge: boot-twice image check (limit ${timeout_s}s)"
report="$(mktemp)"
(cd cloud && pnpm install --frozen-lockfile >/dev/null)
timeout --kill-after=30 "$timeout_s" pnpm --dir cloud --filter @orca-cloud/sandbox-control exec \
tsx image/boot-twice/boot-twice.ts --release latest --report "$report" >/dev/null
status=$?
if ((status == 124 || status == 137)); then
echo "premerge: FAILED: boot-twice image check passed its ${timeout_s}s limit (cold image build? rerun once)" >&2
elif ((status != 0)); then
echo "premerge: FAILED: boot-twice image check: the second boot of the image is not clean:" >&2
jq -r '.failures[]? | "premerge: " + .' "$report" >&2 2>/dev/null || true
echo "premerge: details: cloud/apps/sandbox-control/image/boot-twice/README.md" >&2
else
echo "premerge: boot-twice image check OK ($(jq -r '"boot 2 " + (.boots[1].ms|tostring) + " ms"' "$report" 2>/dev/null))"
fi
rm -f "$report"
exit "$status"
@@ -188,5 +188,9 @@ if ((${#tc_pkgs[@]})); then
(cd "$pkg" && pnpm typecheck)
done
fi
# Image or VM-script changes: boot the image twice (Docker); the second boot must be clean. Fails open without Docker.
if [[ -n "$(git diff --name-only "$base"...HEAD -- cloud/apps/sandbox-control/image)" ]]; then
bash config/scripts/cloud-integration/boot-twice-check.sh
fi
if [[ "${PREMERGE_FULL:-}" == 1 ]]; then scope="the full vitest suite"; else scope="${#tests[@]} targeted test file(s)"; fi
echo "premerge OK: $branch merges cleanly into cloud/integration with typecheck, lint and $scope green."
@@ -131,6 +131,63 @@ it('a GitHub-only create keeps the typed name as the workspace title (NAME-DROPP
})
const SESSION = 'cs_abcdefghijklmnopqrstuv'
it('CREATE-GH-STEP: a public repo that can’t push asks with ids too; settings pages pass, others drop', async () => {
const githubRepoAccess = vi.fn(async (query: { repo: string; repoId?: number }) =>
query.repo.endsWith('/site')
? {
access: 'public' as const,
owner: 'acme',
repo: query.repo,
push: 'not_installed' as const,
installUrl: query.repoId
? 'https://github.com/apps/x/installations/new/permissions?suggested_target_id=50&repository_ids[]=7&state=s'
: 'https://github.com/apps/x/installations/new?state=s'
}
: {
access: 'public' as const,
owner: 'acme',
repo: query.repo,
push: 'repo_not_selected' as const,
configureUrl: query.repo.endsWith('/evil')
? 'https://evil.test/settings'
: 'https://github.com/organizations/acme/settings/installations/900'
}
)
const repoIds = vi.fn(async () => ({ repoId: 7, ownerId: 50 }))
registerHostedAccountHandlers(
{
config: async () => ({ idleSleepMinutes: 15 }),
connectGitHub: async () => ({ installUrl: REPO, authorizeUrl: REPO }),
githubRepoAccess,
githubRepositories: async () => ({ repositories: [], nextCursor: null }),
create: unusedCreate,
...githubUser
},
{ repoIds }
)
expect(
await handlers.get(CH.getRepoAccess)?.({}, { repoUrl: 'https://github.com/acme/site' })
).toMatchObject({
ok: true,
value: { push: 'not_installed', installUrl: expect.stringContaining('repository_ids[]=7') }
})
repoIds.mockClear()
expect(
await handlers.get(CH.getRepoAccess)?.({}, { repoUrl: 'https://github.com/acme/web' })
).toMatchObject({
ok: true,
value: { configureUrl: 'https://github.com/organizations/acme/settings/installations/900' }
})
// The installation's settings page needs no install ids.
expect(repoIds).not.toHaveBeenCalled()
const evil = await handlers.get(CH.getRepoAccess)?.(
{},
{ repoUrl: 'https://github.com/acme/evil' }
)
expect(evil).toMatchObject({ ok: true })
expect(evil).not.toHaveProperty('value.configureUrl')
})
it('asks again with the repo and owner ids when access is missing, so GitHub preselects the repo', async () => {
const githubRepoAccess = vi.fn(async (query: { repoId?: number }) => ({
access: 'missing' as const,
+20 -8
View File
@@ -3,6 +3,7 @@
import { ipcMain, shell } from 'electron'
import { ORCA_CLOUD_WORKSPACE_CHANNELS as CH } from '../../shared/hosted-sandbox/cloud-workspace-api'
import type { HostedSandboxConfigResponse } from '../../shared/hosted-sandbox/hosted-sandbox-wire'
import type { HostedGithubRepoAccessResponse } from '../../shared/hosted-sandbox/hosted-github-wire'
import type { HostedSandboxControlPlaneClient } from './hosted-sandbox-control-plane-client'
import { failure } from './hosted-workspace-runtime'
import {
@@ -53,6 +54,18 @@ function repoTarget(args: RepoArgs): { repo: string; setupSessionId?: string } |
return { repo: `https://github.com/${slug}`, ...(session ? { setupSessionId: session } : {}) }
}
/** Links the window opens must be GitHub's own pages. */
function githubPagesOnly(value: HostedGithubRepoAccessResponse): HostedGithubRepoAccessResponse {
const onGitHub = (url: string | undefined): boolean =>
url !== undefined && new URL(url).origin === 'https://github.com'
const { installUrl, configureUrl, ...rest } = value
return {
...rest,
...(onGitHub(installUrl) ? { installUrl } : {}),
...(onGitHub(configureUrl) ? { configureUrl } : {})
}
}
export function registerHostedAccountHandlers(
client: Pick<
HostedSandboxControlPlaneClient,
@@ -127,16 +140,15 @@ export function registerHostedAccountHandlers(
}
try {
const answer = await client.githubRepoAccess(target)
if (answer.access !== 'missing') {
return { ok: true, value: answer }
}
// A public repository that can't push gets an install page too (CREATE-GH-STEP).
const installs =
answer.access === 'missing' ||
(answer.access === 'public' &&
(answer.push === 'not_installed' || answer.push === 'not_connected'))
// Missing: ask again with the ids GitHub's install page needs to preselect only this repo.
const ids = await repoIds(target.repo)
const ids = installs ? await repoIds(target.repo) : null
const value = ids ? await client.githubRepoAccess({ ...target, ...ids }) : answer
if (value.installUrl && new URL(value.installUrl).origin !== 'https://github.com') {
return { ok: true, value: { ...value, installUrl: undefined } }
}
return { ok: true, value }
return { ok: true, value: githubPagesOnly(value) }
} catch (error) {
return failure(error)
}
@@ -6,7 +6,7 @@ import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { noteCloudPairingBindings } from '../cloud-workspaces/cloud-paired-environment-ids'
import { createOomTracker, withGitSetup } from './hosted-workspace-runtime'
import { createOomTracker, withGitSetup, withSandboxFacts } from './hosted-workspace-runtime'
import { findCloudSandbox } from './cloud-workspace-target'
function status(checkedAt: number, uptimeSeconds: number, oomKills: number): HostedWorkspaceStatus {
@@ -57,6 +57,30 @@ describe('OOM tracker', () => {
expect(events).toHaveLength(3)
})
it('announces guard kills as the guard stopping a named process, counted with kernel kills', () => {
const events: CloudOomEvent[] = []
const tracker = createOomTracker((event) => events.push(event))
const guarded = (checkedAt: number, uptime: number, kernel: number, guardKills: number) => ({
...status(checkedAt, uptime, kernel),
memory: {
pressure: 'high' as const,
guardKills,
lastGuardVictim: guardKills ? 'tsc' : null,
lastGuardKillAt: guardKills ? checkedAt : null,
guarded: true
}
})
const t0 = 1_000_000_000
tracker.observe('ws', 'sbx', guarded(t0, 100, 0, 0))
tracker.observe('ws', 'sbx', guarded(t0 + 60_000, 160, 0, 1))
tracker.observe('ws', 'sbx', guarded(t0 + 120_000, 220, 0, 1))
tracker.observe('ws', 'sbx', guarded(t0 + 180_000, 280, 1, 1))
expect(events).toEqual([
{ workspaceId: 'ws', oomKills: 1, victim: 'tsc', stoppedBy: 'guard', at: expect.any(Number) },
{ workspaceId: 'ws', oomKills: 2, victim: 'node', at: expect.any(Number) }
])
})
it('ignores a status without metrics', () => {
const events: CloudOomEvent[] = []
createOomTracker((event) => events.push(event)).observe('ws', 'sbx', {
@@ -96,6 +120,12 @@ describe('GIT-SETUP on the status read', () => {
expect(withGitSetup(read, {})).toBe(read)
})
it('carries the sandbox region for the card details', () => {
const read = status(1_000, 10, 0)
expect(withSandboxFacts(read, { region: 'iad1' })).toEqual({ ...read, region: 'iad1' })
expect(withSandboxFacts(read, { region: 'pdx1', gitSetup: 'failed' })).toEqual({ ...read, gitSetupFailed: true, region: 'pdx1' })
})
it('the strict sandbox view accepts the field the server sends with feature git-setup', () => {
const view = {
id: 'sb_1',
@@ -115,12 +115,13 @@ export function trimCheckouts(input: HostedArchiveCheckout[]): HostedArchiveChec
}
/**
* Announces each new kernel OOM kill once. The counter resets per VM boot; two readings belong to
* one boot when their implied boot times agree within two minutes (uptime and checkedAt come from
* Announces each new OOM kill once: the kernel's, and the memory guard's (MEMGUARD: processes the
* VM stopped before the kernel had to). Both counters reset per VM boot; two readings belong to one
* boot when their implied boot times agree within two minutes (uptime and checkedAt come from
* different clocks).
*/
export function createOomTracker(announce: (event: CloudOomEvent) => void) {
const seen = new Map<string, { bootAt: number; kills: number }>()
const seen = new Map<string, { bootAt: number; kernel: number; guard: number }>()
return {
observe(
workspaceId: string,
@@ -132,19 +133,26 @@ export function createOomTracker(announce: (event: CloudOomEvent) => void) {
if (!metrics) {
return
}
const guardKills = status.memory?.guardKills ?? 0
const bootAt = status.checkedAt - metrics.uptimeSeconds * 1000
const previous = seen.get(sandboxId)
const sameBoot = previous !== undefined && Math.abs(previous.bootAt - bootAt) < 120_000
const announced = sameBoot ? previous.kills : 0
const kernel = sameBoot ? previous.kernel : 0
const guard = sameBoot ? previous.guard : 0
seen.set(sandboxId, {
bootAt: sameBoot ? previous.bootAt : bootAt,
kills: Math.max(announced, metrics.oomKills)
kernel: Math.max(kernel, metrics.oomKills),
guard: Math.max(guard, guardKills)
})
if (metrics.oomKills > announced) {
const byGuard = guardKills > guard
if (byGuard || metrics.oomKills > kernel) {
announce({
workspaceId,
oomKills: metrics.oomKills,
victim: metrics.lastOomVictim,
// One count for dismissal: either kind of new kill shows the notice again.
oomKills: Math.max(kernel, metrics.oomKills) + Math.max(guard, guardKills),
// The guard names the process it chose; the kernel's name comes from its log.
victim: byGuard ? (status.memory?.lastGuardVictim ?? null) : metrics.lastOomVictim,
...(byGuard ? { stoppedBy: 'guard' as const } : {}),
at: now
})
}
@@ -168,6 +176,14 @@ export function withGitSetup(
return sandbox.gitSetup === 'failed' ? { ...status, gitSetupFailed: true } : status
}
/** withGitSetup plus the region the sandbox runs in, for the card's details. */
export function withSandboxFacts(
status: HostedWorkspaceStatus,
sandbox: Pick<HostedSandboxView, 'gitSetup' | 'region'>
): CloudWorkspaceStatus {
return { ...withGitSetup(status, sandbox), region: sandbox.region }
}
let current: HostedWorkspaceService | null = null
export function setHostedWorkspaceService(next: HostedWorkspaceService | null): void {
current = next
@@ -41,7 +41,7 @@ import {
findCloudRuntime,
readCachedCheckouts,
announceWakeUsable,
withGitSetup,
withSandboxFacts,
type CloudRuntime
} from './hosted-workspace-runtime'
@@ -204,7 +204,7 @@ export function installHostedWorkspaceService(deps: {
try {
const { sandbox, status } = await deps.client.workspaceStatus(found.sandboxId)
oom.observe(args.workspaceId, found.sandboxId, status)
return { ok: true, value: withGitSetup(status, sandbox) }
return { ok: true, value: withSandboxFacts(status, sandbox) }
} catch (error) {
// Gone server-side: prune it now rather than show "Couldn't start" until the next relist.
if (isEnvironmentProviderError(error, 'not_found')) {
@@ -7,6 +7,7 @@ import {
CodexBrokerStartResponseSchema,
CodexBrokerStatusResponseSchema,
type AgentCredentialsView,
type AgentDefaultSourceChoice,
type AgentSource,
type AgentSourcesView,
type CodexBrokerStart,
@@ -28,7 +29,10 @@ export type AgentCredentialsClient = {
/** Main only, right after setup-token on this computer or a paste. */
putClaude(token: string, label?: string): Promise<AgentCredentialsView>
remove(agent: OrcaCloudAccountAgent): Promise<AgentCredentialsView>
setDefaultSource(agent: OrcaCloudAccountAgent, source: AgentSource): Promise<AgentSourcesView>
setDefaultSource(
agent: OrcaCloudAccountAgent,
source: AgentDefaultSourceChoice
): Promise<AgentSourcesView>
/** The Source for one workspace (sandbox id); the reply describes that workspace. */
setWorkspaceSource(
agent: OrcaCloudAccountAgent,
@@ -3,6 +3,7 @@
import { BrowserWindow, app, ipcMain, shell } from 'electron'
import { z } from 'zod'
import {
IN_WORKSPACE_SOURCE,
ORCA_CLOUD_ACCOUNT_AGENTS,
type OrcaCloudAgentSetups
} from '../../shared/orca-cloud-agents/orca-cloud-agents-wire'
@@ -27,6 +28,10 @@ const PasteRequest = z.object({ token: z.string().min(1).max(1024) })
// The code (or callback URL) from Claude's sign-in page; main normalizes it.
const CodeRequest = z.object({ code: z.string().min(1).max(2048) })
const SourceRequest = z.object({ agent: Agent, source: z.enum(['sign-in', 'key']).nullable() })
const DefaultSourceRequest = z.object({
agent: Agent,
source: z.enum(['sign-in', 'key', IN_WORKSPACE_SOURCE]).nullable()
})
// A runtime environment, never a raw sandbox id: main resolves only sandboxes this desktop made.
const RuntimeEnvironment = z.string().min(1).max(256)
const WorkspaceRequest = z.object({ runtimeEnvironmentId: RuntimeEnvironment })
@@ -153,7 +158,7 @@ export function registerOrcaCloudAgentsHandlers(userDataPath: string): void {
return request.success ? service.remove(request.data.agent) : invalid
})
ipcMain.handle('orcaCloudAgents:setDefaultSource', (_event, raw: unknown) => {
const request = SourceRequest.safeParse(raw)
const request = DefaultSourceRequest.safeParse(raw)
return request.success
? service.setDefaultSource(request.data.agent, request.data.source)
: invalid
@@ -5,6 +5,7 @@ import {
CLAUDE_SETUP_TOKEN_PATTERN,
IDLE_ORCA_CLOUD_AGENT_SETUPS,
type AgentCredentialsView,
type AgentDefaultSourceChoice,
type AgentSource,
type AgentSourcesView,
type OrcaCloudAccountAgent,
@@ -339,7 +340,7 @@ export function createOrcaCloudAgentsService(deps: OrcaCloudAgentsServiceDeps) {
setDefaultSource: (
agent: OrcaCloudAccountAgent,
source: AgentSource
source: AgentDefaultSourceChoice
): Promise<OrcaCloudAgentsResult<AgentSourcesView>> =>
attempt((client) => client.setDefaultSource(agent, source)),
+2 -1
View File
@@ -1,6 +1,7 @@
import { ipcRenderer } from 'electron'
import type {
AgentCredentialsView,
AgentDefaultSourceChoice,
AgentSource,
AgentSourcesView,
OrcaCloudAccountAgent,
@@ -29,7 +30,7 @@ export type OrcaCloudAgentsApi = {
}) => Promise<OrcaCloudAgentsResult<AgentCredentialsView>>
setDefaultSource: (request: {
agent: OrcaCloudAccountAgent
source: AgentSource
source: AgentDefaultSourceChoice
}) => Promise<OrcaCloudAgentsResult<AgentSourcesView>>
/** Main's answer (runtime store + pairing bindings): is this runtime environment Orca Cloud? */
isCloudEnvironment: (request: { runtimeEnvironmentId: string }) => Promise<boolean>
+9
View File
@@ -637,6 +637,15 @@ body[data-modal-overlay-open] [data-toasts-raised] [data-sonner-toaster] {
gap: 6px;
}
/* Hovering the stack must never move the toast under the pointer. Sonner sizes an expanded toast to
the height it measured when the toast was issued; a toast whose content grew since then shrank
from the top on hover, sank out from under the pointer, collapsed and popped back up (a loop).
The front toast keeps its natural height in both states, as it already has when collapsed. */
[data-sonner-toaster]
[data-sonner-toast][data-mounted='true'][data-expanded='true'][data-front='true'][data-removed='false'] {
height: auto;
}
/* Clip folded toast overflow to prevent hover/re-expansion loops; hidden would create a focus-scrollable container. */
[data-sonner-toaster] [data-sonner-toast][data-expanded='false'][data-front='false'] {
overflow: clip;
@@ -47,6 +47,7 @@ export function agentsSummary(
break
case 'killSwitch':
case 'unavailable':
case 'inWorkspace':
rest.push(
t('agentHostPanel.summary.agentSignsInHere', '{{agent}} signs in for this workspace', {
agent
@@ -29,7 +29,7 @@ export type CloudAgentsPanelLoad =
export type CloudAgentsPanelActions = {
chooseSource: (agent: OrcaCloudAccountAgent, source: Exclude<AgentSource, null>) => void
/** Settings › Agent accounts (S0): set up once, or renew. */
/** Settings › Orca Cloud › Cloud agent sign-ins (S0): set up once, or renew. */
manage: () => void
signIn: () => void
retry: () => void
@@ -63,7 +63,10 @@ function statusCopy(row: CloudAgentPanelRow, t: Translate): { line: string; deta
line: t('agentHostPanel.cloud.expired', 'Your {{agent}} sign-in for Orca Cloud expired', {
agent
}),
detail: t('agentHostPanel.cloud.expiredDetail', 'Renew it once in Agent accounts.')
detail: t(
'agentHostPanel.cloud.expiredDetailCloudSignIns',
'Renew it once in Cloud agent sign-ins.'
)
}
case 'sharedUnavailable':
return {
@@ -82,7 +85,9 @@ function statusCopy(row: CloudAgentPanelRow, t: Translate): { line: string; deta
detail:
row.status.reason === 'paused'
? t('vmBuildAgents.paused', 'Orca account sign-in is paused')
: t('vmBuildAgents.notOffered', 'Signs in for each cloud workspace for now')
: row.status.reason === 'chosen'
? t('agentHostPanel.cloud.signsInHereChosen', 'Orca doesn’t store this sign-in')
: t('vmBuildAgents.notOffered', 'Signs in for each cloud workspace for now')
}
}
}
@@ -19,7 +19,7 @@ export type CloudAgentPanelStatus =
| { kind: 'notSetUp' }
| { kind: 'expired' }
/** No account path and no key: the agent signs in inside this workspace (§5c). */
| { kind: 'signsInHere'; reason: 'paused' | 'notOffered' }
| { kind: 'signsInHere'; reason: 'paused' | 'notOffered' | 'chosen' }
/** D13: a shared workspace never uses personal accounts or keys, and no team key covers it. */
| { kind: 'sharedUnavailable' }
@@ -96,6 +96,9 @@ export function cloudAgentPanelRows(
return { agent, status: { kind: 'signsInHere', reason: 'paused' }, source }
case 'unavailable':
return { agent, status: { kind: 'signsInHere', reason: 'notOffered' }, source }
case 'inWorkspace':
// Nothing of Orca's applies, so there is no Source to switch here either.
return { agent, status: { kind: 'signsInHere', reason: 'chosen' }, source: null }
}
})
}
@@ -7,7 +7,8 @@ import { closeAgentHostPanel } from './agent-host-panel-store'
const TARGET = {
accounts: { pane: 'accounts' },
agentAccounts: { pane: 'accounts', sectionId: 'orca-cloud' },
// Opens the sign-in sheet (below); its page is Settings › Orca Cloud › Cloud agent sign-ins.
agentAccounts: { pane: 'cloud-agent-sign-ins' },
agentApiKeys: { pane: 'agent-api-keys' },
cloudSecrets: { pane: 'cloud-secrets' },
customEnvironments: { pane: 'servers' },
@@ -239,6 +239,21 @@ export const runningRestartFailedNarrow = fixture({
/** PC-3b: the launch holds; Set up once runs the one-time Orca Cloud setup in the pane. */
export const notSetUp = fixture({ setUp: { setup: { phase: 'idle' } } })
export const notSetUpCompact = fixture({ setUp: { setup: { phase: 'idle' } }, width: 320 })
/** The quiet second choice on "Set up once": the agent signs in inside this workspace instead. */
export const notSetUpSignInInside = defineFixture({
width: 940,
render: () => (
<div className="pb-24">
<Pane setUp={{ setup: { phase: 'idle' } }} agent="codex" />
</div>
),
interact: async (frame) => {
frame
.querySelector<HTMLButtonElement>('button[aria-label="More choices"]')
?.dispatchEvent(new PointerEvent('pointerdown', { bubbles: true, button: 0 }))
await new Promise((resolve) => setTimeout(resolve, 200))
}
})
/** A half-width split: the held strip wraps its detail rather than cutting it off. */
export const notSetUpNarrow = fixture({ setUp: { setup: { phase: 'idle' } }, width: 440 })
export const notSetUpBrowser = fixture({
@@ -340,6 +340,32 @@ export function CloudAgentSignInStrip({
</DropdownMenuContent>
</DropdownMenu>
) : null}
{model.alternative && !model.otherWays ? (
<DropdownMenu>
<DropdownMenuTrigger asChild>
<Button
type="button"
variant="ghost"
size="icon-xs"
aria-label={t('agentSignInStrip.moreChoices', 'More choices')}
>
<MoreHorizontal aria-hidden />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end" className="max-w-72">
<DropdownMenuItem
onSelect={() => model.alternative && actions.primary(model.alternative.action)}
>
<span className="flex flex-col gap-0.5">
<span>{model.alternative.label}</span>
<span className="text-xs leading-4 text-muted-foreground">
{model.alternative.detail}
</span>
</span>
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
) : null}
{model.settingsLink && !stacked ? (
<CloudStripSettingsLink label={model.settingsLink} onOpen={actions.openSettings} />
) : null}
@@ -112,4 +112,39 @@ describe('cloudAgentSetupStripModel (PC-3b)', () => {
primary: { label: 'Start Claude', action: { kind: 'start' } }
})
})
it('offers signing in inside this workspace as a quiet second choice, only while nothing runs', () => {
const idle = cloudAgentSetupStripModel({
agent: 'codex',
setup: { phase: 'idle' },
ready: false,
t
})
expect(idle.primary?.action).toEqual({ kind: 'setUpOnce' })
expect(idle.alternative).toEqual({
label: 'Sign in inside this workspace instead',
detail:
'Orca won’t store your sign-in. You’ll sign in again in each new workspace. Team API keys won’t be used for this agent.',
action: { kind: 'signInInsideWorkspace' }
})
const failed = cloudAgentSetupStripModel({
agent: 'claude',
setup: { phase: 'failed', error: 'denied' },
ready: false,
t
})
expect(failed.alternative?.action).toEqual({ kind: 'signInInsideWorkspace' })
for (const setup of [
{ phase: 'browser', startedAt: 1, canReopen: true },
{ phase: 'saving' }
] as const) {
expect(
cloudAgentSetupStripModel({ agent: 'claude', setup, ready: false, t }).alternative
).toBeUndefined()
}
expect(
cloudAgentSetupStripModel({ agent: 'claude', setup: { phase: 'idle' }, ready: true, t })
.alternative
).toBeUndefined()
})
})
@@ -1,5 +1,5 @@
// PC-3b: the Orca Cloud pane strip for an agent that isn't set up yet. The launch holds; "Set up
// once" runs the same one-time setup as Settings › Accounts › Orca Cloud (S0) right in the pane,
// once" runs the same one-time setup as Settings › Orca Cloud › Cloud agent sign-ins (S0) right in the pane,
// and once it's saved the strip offers "Start {{agent}}". Pure: the strip renders what this returns.
import type { Translate } from '@/i18n/use-translate'
import {
@@ -30,6 +30,20 @@ function base(agent: string, t: Translate): CloudStripModel {
}
}
/** "Sign in inside this workspace instead": the in-VM login, as a quiet second choice to Set up once. */
export function signInInsideWorkspaceAlternative(
t: Translate
): NonNullable<CloudStripModel['alternative']> {
return {
label: t('agentSignInStrip.signInInsideWorkspace', 'Sign in inside this workspace instead'),
detail: t(
'agentSignInStrip.signInInsideWorkspaceDetail',
'Orca won’t store your sign-in. You’ll sign in again in each new workspace. Team API keys won’t be used for this agent.'
),
action: { kind: 'signInInsideWorkspace' }
}
}
/** A detail with `inline code` marks, as one sentence for a screen reader. */
function plainText(text: string): string {
return text.replaceAll('`', '')
@@ -100,7 +114,7 @@ export function cloudAgentSetupStripModel(args: {
switch (setup.phase) {
case 'idle':
case 'saved':
return need
return { ...need, alternative: signInInsideWorkspaceAlternative(t) }
case 'browser':
return {
...need,
@@ -223,6 +237,7 @@ export function cloudAgentSetupStripModel(args: {
action: { kind: 'reopenSetup' }
}
: { label: t('agentSignInStrip.tryAgain', 'Try again'), action: { kind: 'setUpOnce' } },
alternative: signInInsideWorkspaceAlternative(t),
announcement: copy.title
}
}
@@ -25,6 +25,8 @@ export type CloudStripPrimary =
| { kind: 'reopenSetup' }
/** PC-3b: the setup finished; start the held agent. */
| { kind: 'start' }
/** The quiet alternative to "Set up once": the agent signs in inside each workspace, Orca stores nothing. */
| { kind: 'signInInsideWorkspace' }
export type CloudStripModel = {
status: CloudStripStatus
@@ -57,6 +59,11 @@ export type CloudStripModel = {
} | null
/** "Other ways" under ⋯: the code fallback and the key path. */
otherWays: boolean
/**
* A quiet second choice under ⋯ (the strip has no "Other ways"): a label and one sentence on what
* it means, e.g. "Sign in inside this workspace instead" on "Set up once".
*/
alternative?: { label: string; detail: string; action: CloudStripPrimary }
/** Already on a code: ⋯ offers the paste fallback and Cancel instead of "Use a code". */
codeStep?: boolean
/**

Some files were not shown because too many files have changed in this diff Show More