mirror of
https://github.com/stablyai/orca.git
synced 2026-11-17 19:34:29 +00:00
Merge remote-tracking branch 'private/cloud/integration' into cloud/live-git-access
This commit is contained in:
@@ -0,0 +1,81 @@
|
||||
name: Cloud Egress Probe
|
||||
|
||||
# Release check for sandbox egress, against DEV, through Orca's own paths (dev workspaces driven
|
||||
# over the shipped `orca` CLI's pairing code). Bogus tokens only; postman-echo.com is the echo host.
|
||||
# Details and the local run: cloud/apps/sandbox-control/scripts/README.md
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: cloud-egress-probe
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
probe:
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
DEV_TEST_JWK: ${{ secrets.ORCA_DEV_TEST_JWK }}
|
||||
DEV_OPERATOR_TOKEN: ${{ secrets.ORCA_DEV_OPERATOR_TOKEN }}
|
||||
steps:
|
||||
- name: Check the dev credentials are configured
|
||||
id: creds
|
||||
working-directory: .
|
||||
run: |
|
||||
if [ -z "$DEV_TEST_JWK" ] || [ -z "$DEV_OPERATOR_TOKEN" ]; then
|
||||
echo "::warning::ORCA_DEV_TEST_JWK / ORCA_DEV_OPERATOR_TOKEN are not set: the egress probe did not run"
|
||||
echo "skip=1" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- if: steps.creds.outputs.skip != '1'
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- if: steps.creds.outputs.skip != '1'
|
||||
uses: ./.github/actions/setup-pnpm
|
||||
with:
|
||||
package-json-file: cloud/package.json
|
||||
|
||||
- if: steps.creds.outputs.skip != '1'
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- if: steps.creds.outputs.skip != '1'
|
||||
run: pnpm install --frozen-lockfile --filter @orca-cloud/sandbox-control...
|
||||
|
||||
# The shipped CLI from the latest release's Linux package: the same pairing path the desktop uses.
|
||||
- name: Install the orca CLI
|
||||
if: steps.creds.outputs.skip != '1'
|
||||
working-directory: .
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
gh release download --repo stablyai/orca --pattern 'orca-ide_*_amd64.deb' --dir "$RUNNER_TEMP/orca"
|
||||
dpkg-deb -x "$RUNNER_TEMP"/orca/orca-ide_*_amd64.deb "$RUNNER_TEMP/orca/x"
|
||||
cli="$(find "$RUNNER_TEMP/orca/x" -path '*/bin/orca' -type f | head -n 1)"
|
||||
test -x "$cli"
|
||||
echo "ORCA_PROBE_CLI=$cli" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Probe dev egress
|
||||
if: steps.creds.outputs.skip != '1'
|
||||
run: |
|
||||
umask 077
|
||||
printf '%s' "$DEV_TEST_JWK" > "$RUNNER_TEMP/dev-jwk.json"
|
||||
printf '%s' "$DEV_OPERATOR_TOKEN" > "$RUNNER_TEMP/dev-operator"
|
||||
ORCA_PROBE_DEV_JWK_FILE="$RUNNER_TEMP/dev-jwk.json" ORCA_PROBE_OPERATOR_TOKEN_FILE="$RUNNER_TEMP/dev-operator" \
|
||||
pnpm --filter @orca-cloud/sandbox-control exec tsx scripts/egress-probe.ts > "$RUNNER_TEMP/egress-probe.json"
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always() && steps.creds.outputs.skip != '1'
|
||||
with:
|
||||
name: cloud-egress-probe-report
|
||||
path: ${{ runner.temp }}/egress-probe.json
|
||||
retention-days: 14
|
||||
@@ -0,0 +1,83 @@
|
||||
name: Cloud VM Boot Twice
|
||||
|
||||
# Provisions the golden image in a container and boots the same disk twice through the control
|
||||
# plane's boot steps. The second boot runs offline and must install, fetch and redo nothing.
|
||||
# Local run and details: cloud/apps/sandbox-control/image/boot-twice/README.md
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened, ready_for_review]
|
||||
paths:
|
||||
- cloud/apps/sandbox-control/image/**
|
||||
- cloud/apps/sandbox-control/src/golden-images.ts
|
||||
- cloud/apps/sandbox-control/src/sandbox-vm-commands.ts
|
||||
- cloud/apps/sandbox-control/src/orcad-ready.ts
|
||||
- .github/workflows/cloud-vm-boot-twice.yml
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
release:
|
||||
description: Orca release whose orcad bundle the image carries (vX.Y.Z or latest)
|
||||
required: false
|
||||
default: latest
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: cloud-vm-boot-twice-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
defaults:
|
||||
run:
|
||||
working-directory: cloud
|
||||
|
||||
jobs:
|
||||
boot-twice:
|
||||
if: github.event_name != 'pull_request' || github.event.pull_request.draft != true
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 25
|
||||
env:
|
||||
RELEASE: ${{ inputs.release || 'latest' }}
|
||||
CONTEXT: ${{ runner.temp }}/boot-twice
|
||||
# Raises the GitHub API limit for the latest-release lookup; read-only, no secret involved.
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: ./.github/actions/setup-pnpm
|
||||
with:
|
||||
package-json-file: cloud/package.json
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
check-latest: true
|
||||
|
||||
- run: pnpm install --frozen-lockfile --filter @orca-cloud/sandbox-control...
|
||||
|
||||
# The context holds exactly what provisioning reads (script + env), so its bytes key the
|
||||
# layer cache: an unchanged image reuses the provisioned layer and only boots.
|
||||
- name: Write the image build context
|
||||
run: pnpm --filter @orca-cloud/sandbox-control exec tsx image/boot-twice/boot-twice.ts --release "$RELEASE" --context-out "$CONTEXT"
|
||||
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Provision the image (cached)
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: ${{ env.CONTEXT }}
|
||||
target: harness
|
||||
tags: orca-boot-twice:ci
|
||||
load: true
|
||||
cache-from: type=gha,scope=cloud-vm-boot-twice
|
||||
cache-to: type=gha,mode=max,scope=cloud-vm-boot-twice
|
||||
|
||||
- name: Boot twice
|
||||
run: pnpm --filter @orca-cloud/sandbox-control exec tsx image/boot-twice/boot-twice.ts --release "$RELEASE" --image orca-boot-twice:ci --report "$RUNNER_TEMP/boot-twice-report.json"
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: cloud-vm-boot-twice-report
|
||||
path: ${{ runner.temp }}/boot-twice-report.json
|
||||
retention-days: 14
|
||||
@@ -0,0 +1,10 @@
|
||||
# Build context: cloud/apps (the relay imports sandbox-control's codex-paths.ts, the shared path pinning).
|
||||
FROM node:24-slim
|
||||
WORKDIR /app
|
||||
COPY sandbox-control/src/codex-paths.ts sandbox-control/src/codex-paths.ts
|
||||
COPY chatgpt-relay/package.json chatgpt-relay/package.json
|
||||
COPY chatgpt-relay/src chatgpt-relay/src
|
||||
WORKDIR /app/chatgpt-relay
|
||||
ENV NODE_ENV=production
|
||||
USER node
|
||||
CMD ["node", "src/index.ts"]
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/usr/bin/env bash
|
||||
# Builds the relay image with Cloud Build (context: cloud/apps) and prints its digest-pinned name for
|
||||
# `terraform apply -var image=...` in cloud/infra/chatgpt-relay.
|
||||
# Usage: build-image.sh <project> [region]
|
||||
set -euo pipefail
|
||||
project="${1:?project}"
|
||||
region="${2:-us-central1}"
|
||||
apps="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
repo="$region-docker.pkg.dev/$project/orca-chatgpt-relay/relay"
|
||||
tag="$(git -C "$apps" rev-parse --short=12 HEAD)"
|
||||
gcloud builds submit "$apps" --project "$project" --region "$region" --quiet \
|
||||
--config /dev/stdin <<YAML >&2
|
||||
steps:
|
||||
- name: gcr.io/cloud-builders/docker
|
||||
args: [build, -f, chatgpt-relay/Dockerfile, -t, "$repo:$tag", .]
|
||||
images: ["$repo:$tag"]
|
||||
YAML
|
||||
digest="$(gcloud artifacts docker images describe "$repo:$tag" --project "$project" --format='value(image_summary.digest)')"
|
||||
echo "$repo@$digest"
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"name": "@orca-cloud/chatgpt-relay",
|
||||
"private": true,
|
||||
"version": "0.0.0",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"start": "node src/index.ts",
|
||||
"test": "vitest run",
|
||||
"typecheck": "tsc -p tsconfig.json --noEmit",
|
||||
"lint": "tsc -p tsconfig.json --noEmit"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.19.2",
|
||||
"typescript": "^7.0.2",
|
||||
"vitest": "^5.0.5"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
import { createServer } from 'node:http'
|
||||
import { createRelay } from './relay.ts'
|
||||
|
||||
const server = createServer(createRelay())
|
||||
// Long Codex turns stream for minutes: no Node-side request or header timeout (Cloud Run caps it).
|
||||
server.requestTimeout = 0
|
||||
server.headersTimeout = 0
|
||||
server.listen(Number(process.env.PORT ?? 8080), () => console.log(JSON.stringify({ event: 'relay_listening' })))
|
||||
@@ -0,0 +1,144 @@
|
||||
// CODEX-RELAY: chatgpt.com refuses Cloudflare Workers' egress, so the sandbox-control egress proxy
|
||||
// sends its chatgpt.com requests here (Cloud Run, which ChatGPT answers). The Worker already chose the
|
||||
// credential and pinned the path; this relay pins the path again, forwards to chatgpt.com only, and
|
||||
// streams the answer back as it arrives (Codex reads Server-Sent Events).
|
||||
// Cloud Run IAM authenticates the Worker (an ID token in X-Serverless-Authorization, run.invoker on
|
||||
// one service account): nothing unauthenticated reaches this code. No header or body is ever logged.
|
||||
import type { IncomingMessage, ServerResponse } from 'node:http'
|
||||
import { Readable } from 'node:stream'
|
||||
import { codexPathAllowed } from '../../sandbox-control/src/codex-paths.ts'
|
||||
|
||||
export const UPSTREAM_ORIGIN = 'https://chatgpt.com'
|
||||
|
||||
/** Never forwarded: hop-by-hop, Cloud Run's own auth and tracing, and proxy breadcrumbs. */
|
||||
const DROP_REQUEST = new Set([
|
||||
'host',
|
||||
'connection',
|
||||
'keep-alive',
|
||||
'proxy-authorization',
|
||||
'proxy-connection',
|
||||
'te',
|
||||
'trailer',
|
||||
'transfer-encoding',
|
||||
'upgrade',
|
||||
'content-length',
|
||||
'x-serverless-authorization',
|
||||
'forwarded',
|
||||
'traceparent',
|
||||
'tracestate',
|
||||
'x-cloud-trace-context'
|
||||
])
|
||||
const DROP_REQUEST_PREFIXES = ['x-forwarded-', 'x-goog-', 'cf-']
|
||||
|
||||
/** fetch already decoded the body, so its encoding and length no longer describe what we send. */
|
||||
const DROP_RESPONSE = new Set([
|
||||
'connection',
|
||||
'keep-alive',
|
||||
'transfer-encoding',
|
||||
'content-encoding',
|
||||
'content-length',
|
||||
'trailer',
|
||||
'upgrade'
|
||||
])
|
||||
|
||||
export type RelayDeps = {
|
||||
fetch?: typeof fetch
|
||||
log?: (line: Record<string, unknown>) => void
|
||||
now?: () => number
|
||||
}
|
||||
|
||||
function refuse(res: ServerResponse, status: number, reason: string): void {
|
||||
res.writeHead(status, { 'content-type': 'text/plain' })
|
||||
res.end(reason)
|
||||
}
|
||||
|
||||
/** The first two path segments only (e.g. /backend-api/codex): never ids or queries. */
|
||||
function pathLabel(path: string): string {
|
||||
return (path.split('?')[0] ?? '').split('/').slice(0, 3).join('/')
|
||||
}
|
||||
|
||||
export function forwardHeaders(req: IncomingMessage): Headers {
|
||||
const headers = new Headers()
|
||||
for (const [name, value] of Object.entries(req.headers)) {
|
||||
const key = name.toLowerCase()
|
||||
if (value === undefined || DROP_REQUEST.has(key) || DROP_REQUEST_PREFIXES.some((p) => key.startsWith(p))) {
|
||||
continue
|
||||
}
|
||||
for (const one of Array.isArray(value) ? value : [value]) {
|
||||
headers.append(key, one)
|
||||
}
|
||||
}
|
||||
return headers
|
||||
}
|
||||
|
||||
export function createRelay(deps: RelayDeps = {}) {
|
||||
const doFetch = deps.fetch ?? fetch
|
||||
const log = deps.log ?? ((line) => console.log(JSON.stringify(line)))
|
||||
const now = deps.now ?? Date.now
|
||||
return async function handle(req: IncomingMessage, res: ServerResponse): Promise<void> {
|
||||
const started = now()
|
||||
const method = (req.method ?? 'GET').toUpperCase()
|
||||
const path = req.url ?? '/'
|
||||
if (path === '/health' && method === 'GET') {
|
||||
res.writeHead(200, { 'content-type': 'text/plain' })
|
||||
res.end('ok')
|
||||
return
|
||||
}
|
||||
// Codex is configured for HTTPS (supports_websockets = false); the Worker proxy can't upgrade either.
|
||||
if (req.headers.upgrade) {
|
||||
refuse(res, 426, 'upgrade_not_supported')
|
||||
return
|
||||
}
|
||||
if (!path.startsWith('/') || !codexPathAllowed(method, path)) {
|
||||
log({ event: 'relay_refused', reason: 'path_not_allowed', method, path: pathLabel(path) })
|
||||
refuse(res, 403, 'path_not_allowed')
|
||||
return
|
||||
}
|
||||
const hasBody = method !== 'GET' && method !== 'HEAD'
|
||||
const abort = new AbortController()
|
||||
// The VM's client went away: stop reading from chatgpt.com too.
|
||||
res.on('close', () => {
|
||||
if (!res.writableFinished) {
|
||||
abort.abort()
|
||||
}
|
||||
})
|
||||
let upstream: Response
|
||||
try {
|
||||
upstream = await doFetch(`${UPSTREAM_ORIGIN}${path}`, {
|
||||
method,
|
||||
headers: forwardHeaders(req),
|
||||
body: hasBody ? (Readable.toWeb(req) as ReadableStream<Uint8Array>) : undefined,
|
||||
// Node needs this for a streamed request body.
|
||||
...(hasBody ? { duplex: 'half' } : {}),
|
||||
redirect: 'manual',
|
||||
signal: abort.signal
|
||||
} as RequestInit)
|
||||
} catch {
|
||||
log({ event: 'relay_upstream_failed', method, path: pathLabel(path), ms: now() - started })
|
||||
if (!res.headersSent) {
|
||||
refuse(res, 502, 'upstream_unreachable')
|
||||
}
|
||||
return
|
||||
}
|
||||
const headers: Record<string, string[]> = {}
|
||||
upstream.headers.forEach((value, name) => {
|
||||
if (!DROP_RESPONSE.has(name.toLowerCase())) {
|
||||
;(headers[name] ??= []).push(value)
|
||||
}
|
||||
})
|
||||
res.writeHead(upstream.status, headers)
|
||||
// Each chunk is written as it arrives: SSE events reach Codex without buffering.
|
||||
res.flushHeaders()
|
||||
if (upstream.body) {
|
||||
try {
|
||||
for await (const chunk of upstream.body as unknown as AsyncIterable<Uint8Array>) {
|
||||
res.write(chunk)
|
||||
}
|
||||
} catch {
|
||||
// The client left (abort) or chatgpt.com cut the stream: end what was sent.
|
||||
}
|
||||
}
|
||||
res.end()
|
||||
log({ event: 'relay', method, path: pathLabel(path), status: upstream.status, ms: now() - started })
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,116 @@
|
||||
import { createServer, request, type Server } from 'node:http'
|
||||
import type { AddressInfo } from 'node:net'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { createRelay } from '../src/relay.ts'
|
||||
|
||||
type Seen = { url: string; headers: Headers; body: string | null }
|
||||
let server: Server | null = null
|
||||
afterEach(() => new Promise<void>((done) => (server ? server.close(() => done()) : done())))
|
||||
|
||||
async function start(reply: (seen: Seen) => Response) {
|
||||
const seen: Seen[] = []
|
||||
const logs: Record<string, unknown>[] = []
|
||||
const relay = createRelay({
|
||||
log: (line) => logs.push(line),
|
||||
fetch: async (input, init) => {
|
||||
const body = init?.body ? await new Response(init.body as BodyInit).text() : null
|
||||
const one = { url: String(input), headers: new Headers(init?.headers), body }
|
||||
seen.push(one)
|
||||
return reply(one)
|
||||
}
|
||||
})
|
||||
server = createServer((req, res) => void relay(req, res))
|
||||
await new Promise<void>((done) => server!.listen(0, '127.0.0.1', done))
|
||||
const port = (server.address() as AddressInfo).port
|
||||
const call = (method: string, path: string, headers: Record<string, string> = {}, body?: string) =>
|
||||
new Promise<{ status: number; headers: Record<string, unknown>; body: string; chunks: string[] }>((resolve, reject) => {
|
||||
const req = request({ host: '127.0.0.1', port, method, path, headers }, (res) => {
|
||||
const chunks: string[] = []
|
||||
res.setEncoding('utf8')
|
||||
res.on('data', (c: string) => chunks.push(c))
|
||||
res.on('end', () => resolve({ status: res.statusCode ?? 0, headers: res.headers, body: chunks.join(''), chunks }))
|
||||
})
|
||||
req.on('error', reject)
|
||||
req.end(body)
|
||||
})
|
||||
return { seen, logs, call }
|
||||
}
|
||||
|
||||
describe('ChatGPT relay (CODEX-RELAY)', () => {
|
||||
it('forwards an allowed Codex call to chatgpt.com with its credential, minus Cloud Run auth and hop headers', async () => {
|
||||
const t = await start(() => new Response('{"ok":true}', { status: 200, headers: { 'content-type': 'application/json', 'content-encoding': 'gzip' } }))
|
||||
const res = await t.call(
|
||||
'POST',
|
||||
'/backend-api/codex/responses',
|
||||
{
|
||||
authorization: 'Bearer real-token',
|
||||
'chatgpt-account-id': 'acct-1',
|
||||
'x-serverless-authorization': 'Bearer google-id-token',
|
||||
'x-forwarded-for': '1.2.3.4',
|
||||
'content-type': 'application/json'
|
||||
},
|
||||
'{"input":"hi"}'
|
||||
)
|
||||
expect(res.status).toBe(200)
|
||||
expect(res.body).toBe('{"ok":true}')
|
||||
expect(res.headers['content-encoding']).toBeUndefined()
|
||||
const sent = t.seen[0]!
|
||||
expect(sent.url).toBe('https://chatgpt.com/backend-api/codex/responses')
|
||||
expect(sent.body).toBe('{"input":"hi"}')
|
||||
expect(sent.headers.get('authorization')).toBe('Bearer real-token')
|
||||
expect(sent.headers.get('chatgpt-account-id')).toBe('acct-1')
|
||||
expect(sent.headers.get('x-serverless-authorization')).toBeNull()
|
||||
expect(sent.headers.get('x-forwarded-for')).toBeNull()
|
||||
expect(sent.headers.get('host')).toBeNull()
|
||||
})
|
||||
|
||||
it('pins the same paths as the Worker: nothing else on chatgpt.com', async () => {
|
||||
const t = await start(() => new Response('{}'))
|
||||
expect((await t.call('GET', '/backend-api/conversations')).status).toBe(403)
|
||||
expect((await t.call('POST', '/backend-api/wham/usage')).status).toBe(403)
|
||||
expect((await t.call('GET', '/backend-api/codex/../conversations')).status).toBe(403)
|
||||
expect((await t.call('GET', '/backend-api/wham/usage')).status).toBe(200)
|
||||
expect(t.seen.map((s) => s.url)).toEqual(['https://chatgpt.com/backend-api/wham/usage'])
|
||||
})
|
||||
|
||||
it('streams Server-Sent Events as they arrive, not after the upstream ends', async () => {
|
||||
let push: (text: string) => void = () => {}
|
||||
let close: () => void = () => {}
|
||||
const stream = new ReadableStream<Uint8Array>({
|
||||
start(controller) {
|
||||
push = (text) => controller.enqueue(new TextEncoder().encode(text))
|
||||
close = () => controller.close()
|
||||
}
|
||||
})
|
||||
const t = await start(() => new Response(stream, { status: 200, headers: { 'content-type': 'text/event-stream' } }))
|
||||
const pending = t.call('POST', '/backend-api/codex/responses', { 'content-type': 'application/json' }, '{}')
|
||||
// The first event is out before the second exists.
|
||||
await new Promise((r) => setTimeout(r, 20))
|
||||
push('event: a\ndata: 1\n\n')
|
||||
await new Promise((r) => setTimeout(r, 50))
|
||||
push('event: b\ndata: 2\n\n')
|
||||
close()
|
||||
const res = await pending
|
||||
expect(res.headers['content-type']).toBe('text/event-stream')
|
||||
expect(res.body).toBe('event: a\ndata: 1\n\nevent: b\ndata: 2\n\n')
|
||||
expect(res.chunks.length).toBeGreaterThanOrEqual(2)
|
||||
})
|
||||
|
||||
it('logs method, a two-segment path and the status only: never headers, bodies or queries', async () => {
|
||||
const t = await start(() => new Response('secret-body'))
|
||||
await t.call('GET', '/backend-api/codex/models?client_version=0.166.0', { authorization: 'Bearer real-token' })
|
||||
const text = JSON.stringify(t.logs)
|
||||
expect(t.logs.at(-1)).toMatchObject({ event: 'relay', method: 'GET', path: '/backend-api/codex', status: 200 })
|
||||
for (const leak of ['real-token', 'secret-body', 'client_version']) {
|
||||
expect(text).not.toContain(leak)
|
||||
}
|
||||
})
|
||||
|
||||
it('refuses WebSocket upgrades and answers 502 when chatgpt.com is unreachable', async () => {
|
||||
const t = await start(() => {
|
||||
throw new TypeError('fetch failed')
|
||||
})
|
||||
expect((await t.call('GET', '/backend-api/codex/responses', { upgrade: 'websocket', connection: 'Upgrade' })).status).toBe(426)
|
||||
expect((await t.call('GET', '/backend-api/wham/usage')).status).toBe(502)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"extends": "../../tsconfig.base.json",
|
||||
"compilerOptions": {
|
||||
"noEmit": true,
|
||||
"allowImportingTsExtensions": true,
|
||||
"erasableSyntaxOnly": true,
|
||||
"types": [
|
||||
"node"
|
||||
],
|
||||
"rootDir": "..",
|
||||
"lib": [
|
||||
"ES2024",
|
||||
"DOM"
|
||||
]
|
||||
},
|
||||
"include": [
|
||||
"src",
|
||||
"test",
|
||||
"../sandbox-control/src/codex-paths.ts"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
import { defineConfig } from 'vitest/config'
|
||||
|
||||
export default defineConfig({
|
||||
// A string tsconfig skips the lookup, which otherwise climbs to the desktop app's tsconfig.
|
||||
oxc: { tsconfig: false },
|
||||
test: { include: ['test/**/*.test.ts'] }
|
||||
})
|
||||
@@ -0,0 +1,41 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
# Boot-twice harness image: the golden image's provisioning, run in a container instead of a Vercel
|
||||
# build sandbox. boot-twice.ts writes the build context (this file, provision-golden.sh and
|
||||
# provision.env, the same env build-golden-image.ts passes) and then boots the result twice.
|
||||
# Base: Ubuntu 26.04 (what vercel/sandbox/node:24 is) with Node 24 and passwordless sudo for `ubuntu`, which is what the provision
|
||||
# script expects from the managed image it normally runs on.
|
||||
FROM node:24-bookworm-slim@sha256:dc26ae92c9fb15cf6b1d63e20f0944b2efe0f115b20e00be48fb23fceb061c54 AS node
|
||||
|
||||
FROM ubuntu:26.04@sha256:065aa8626841f16057b78b027b464cd59c4fac36c204c72dcbf48d66f5fa8a96 AS base
|
||||
COPY --from=node /usr/local/bin/node /usr/local/bin/node
|
||||
COPY --from=node /usr/local/lib/node_modules /usr/local/lib/node_modules
|
||||
RUN ln -s ../lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm \
|
||||
&& ln -s ../lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx \
|
||||
&& apt-get update -qq \
|
||||
&& DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends ca-certificates sudo >/dev/null \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& printf 'ubuntu ALL=(ALL) NOPASSWD:ALL\n' >/etc/sudoers.d/ubuntu && chmod 0440 /etc/sudoers.d/ubuntu
|
||||
|
||||
# The golden build: provision, then wipe identity, exactly as buildGoldenImage runs them.
|
||||
FROM base AS golden
|
||||
USER ubuntu
|
||||
WORKDIR /home/ubuntu
|
||||
COPY --chown=ubuntu:ubuntu provision-golden.sh provision.env /tmp/golden/
|
||||
RUN set -a && . /tmp/golden/provision.env && set +a \
|
||||
&& { bash /tmp/golden/provision-golden.sh >/tmp/golden/provision.out || { tail -n 40 /tmp/golden/provision.out; exit 1; }; } \
|
||||
&& sudo install -D -m 0644 /tmp/golden/provision.out /var/lib/orca-boot-twice/provision-report.json \
|
||||
&& /opt/orca/bin/orca-image-wipe
|
||||
|
||||
# Harness scaffolding only (outside /opt/orca): the fake per-boot kernel boot id, and a local repo
|
||||
# for the workspace prelude to clone so the first boot needs no GitHub.
|
||||
FROM golden AS harness
|
||||
USER root
|
||||
COPY boot-twice-init /usr/local/sbin/boot-twice-init
|
||||
RUN chmod 0755 /usr/local/sbin/boot-twice-init \
|
||||
&& git init -q --bare -b main /srv/fixture.git \
|
||||
&& tmp="$(mktemp -d)" && git -C "$tmp" init -q -b main \
|
||||
&& printf 'boot twice fixture\n' >"$tmp/README.md" && git -C "$tmp" add README.md \
|
||||
&& git -C "$tmp" -c user.name=fixture -c user.email=fixture@orca.invalid commit -q -m fixture \
|
||||
&& git -C "$tmp" push -q /srv/fixture.git main && rm -rf "$tmp" \
|
||||
&& chown -R ubuntu:ubuntu /srv/fixture.git
|
||||
CMD ["/usr/local/sbin/boot-twice-init"]
|
||||
@@ -0,0 +1,78 @@
|
||||
# Boot-twice check
|
||||
|
||||
This check catches the case where the image boots but a wake reinstalls or refetches things.
|
||||
|
||||
It provisions the golden image in a container, using the real `provision-golden.sh` and the same
|
||||
env that `build-golden-image.ts` passes. Then it boots the same disk twice through the control
|
||||
plane's own boot code:
|
||||
|
||||
1. `ENTRY_POINTS_PROBE` plus the install when the image's scripts are behind (the `refreshRuntime` step).
|
||||
2. `startOrcad`, with the workspace prelude on the first boot only.
|
||||
3. `ensureEntryPoints`, the once-per-image entry-point sync.
|
||||
|
||||
Each container start counts as one VM boot. The container gets a fresh kernel boot id, a tmpfs
|
||||
`/run`, and the disk the previous boot left (`docker commit`).
|
||||
|
||||
The second boot must:
|
||||
|
||||
- **Come up.** It needs a pairing offer, and orcad must be alive and listening on `:6768`.
|
||||
- **Install nothing.** `docker diff` against the first boot's disk may only touch boot state:
|
||||
`~/.orca-sandbox`, `~/.orcad-data`, the per-boot agent config, the compile cache, `/tmp` and
|
||||
`/run`. See `bootStatePaths` in `boot-twice-checks.ts`. Any `node_modules`, or any change under
|
||||
`/opt/orca/bin`, `/usr`, `/etc` or a package cache, fails the check.
|
||||
- **Fetch nothing.** The boot runs on an internal Docker network (no route out). Its only DNS
|
||||
server is a sink that logs every name it is asked for. A lookup fails the check unless it is in
|
||||
`RUNTIME_LOOKUPS`. Today that is only orcad's agent-state-rules refresh on github.com.
|
||||
- **Redo nothing.** The container's init samples every process's command line every 50 ms. Any
|
||||
installer, fetch or first-run command (apt, npm install, curl, git fetch, `opencode models`)
|
||||
fails the check.
|
||||
- **Skip the entry-point install.** No `installEntryPoints` call, and the first boot's
|
||||
`.entry-points-<hash>` marker is present.
|
||||
- **Carry the right files.** Every `/opt/orca/bin` entry point must be byte-identical to `image/vm/`,
|
||||
`.scripts-stamp` must equal `VM_SCRIPTS_STAMP`, and every manifest pin must match the installed
|
||||
version. The npm packages are compared exactly, by their `package.json`. gh, cursor-agent and
|
||||
Chrome are compared by `--version`.
|
||||
|
||||
## Run it locally
|
||||
|
||||
You need Docker that can run `--privileged` containers: Linux, or Docker Desktop on macOS.
|
||||
|
||||
```sh
|
||||
cd cloud
|
||||
pnpm install --filter @orca-cloud/sandbox-control...
|
||||
pnpm --filter @orca-cloud/sandbox-control exec tsx image/boot-twice/boot-twice.ts --release latest
|
||||
```
|
||||
|
||||
A cold run takes about 4 minutes, almost all of it provisioning (apt, agent CLIs, Chrome).
|
||||
Docker's layer cache keys on the provisioning inputs: `provision-golden.sh`, the manifest and the
|
||||
entry points. A rerun with those unchanged takes about 15 seconds. The command exits non-zero and
|
||||
lists each failure.
|
||||
|
||||
The options:
|
||||
|
||||
- `--report <file>` writes the full JSON report: boot timings, every command, the disk diff, lookups and versions.
|
||||
- `--keep` leaves the two containers, the disk image and the DNS sink for inspection.
|
||||
- `--release vX.Y.Z` boots a specific orca-server release bundle. The default is the latest
|
||||
release, because the manifest pins no bundle: production goldens build orcad from source.
|
||||
- `--image <tag>` boots an image you built yourself. `--context-out <dir>` writes the build context for one.
|
||||
|
||||
What this check does not cover:
|
||||
|
||||
- The runtime-bundle refresh: no bundle is published here.
|
||||
- The Vercel firewall and egress proxy: that is the release probe, `scripts/egress-probe.ts`.
|
||||
- Vercel's own VM behaviour, such as snapshot restore timing.
|
||||
|
||||
## CI
|
||||
|
||||
On nwparker/orca-private, where Actions is off, the premerge gate
|
||||
(`config/scripts/cloud-integration/premerge-check.sh`) runs
|
||||
`config/scripts/cloud-integration/boot-twice-check.sh` whenever a branch touches
|
||||
`cloud/apps/sandbox-control/image/**`, which includes every VM script. The step:
|
||||
|
||||
- takes its own omarchy slot, waiting at most 5 minutes for one;
|
||||
- stops after 10 minutes;
|
||||
- prints each failure;
|
||||
- **fails open with a loud warning when Docker is unavailable**.
|
||||
|
||||
`.github/workflows/cloud-vm-boot-twice.yml` runs the same check on pull requests once Actions is
|
||||
on, with the provisioned layer cached in the GitHub Actions cache.
|
||||
@@ -0,0 +1,111 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { MANIFEST } from '../vm-scripts.generated'
|
||||
import { entryPointMismatches, installerCommands, parseDockerDiff, pinnedTools, unexpectedChanges, versionMismatches } from './boot-twice-checks'
|
||||
|
||||
const HOME = '/home/ubuntu'
|
||||
|
||||
describe('unexpectedChanges', () => {
|
||||
it('allows boot state, per-boot config and the parent directories they live in', () => {
|
||||
const diff = parseDockerDiff(
|
||||
[
|
||||
'C /home',
|
||||
'C /home/ubuntu',
|
||||
'C /home/ubuntu/.orca-sandbox',
|
||||
'A /home/ubuntu/.orca-sandbox/opencode-first-run',
|
||||
'C /home/ubuntu/.orcad-data/agent-hooks/endpoint.env',
|
||||
'C /home/ubuntu/.codex/config.toml',
|
||||
'C /opt',
|
||||
'C /opt/orca',
|
||||
'C /opt/orca/compile-cache/v24-x64/712488c2',
|
||||
'A /tmp/x',
|
||||
'C /run'
|
||||
].join('\n')
|
||||
)
|
||||
expect(unexpectedChanges(diff, HOME)).toEqual([])
|
||||
})
|
||||
|
||||
it('flags installs: image files, package caches and node_modules anywhere', () => {
|
||||
const diff = parseDockerDiff(
|
||||
[
|
||||
'C /opt/orca/bin/orca-sandbox-start',
|
||||
'A /opt/orca/tools/lib/node_modules/x/package.json',
|
||||
'A /home/ubuntu/.npm/_cacache/index',
|
||||
'A /home/ubuntu/.config/opencode/node_modules/@opencode-ai/plugin',
|
||||
'A /home/ubuntu/.orcad-data/node_modules/x',
|
||||
'C /usr/lib/x',
|
||||
'C /etc/apt/sources.list.d/ubuntu.sources',
|
||||
'C /home/ubuntu/.config/opencode'
|
||||
].join('\n')
|
||||
)
|
||||
expect(unexpectedChanges(diff, HOME).map((entry) => entry.path)).toEqual([
|
||||
'/opt/orca/bin/orca-sandbox-start',
|
||||
'/opt/orca/tools/lib/node_modules/x/package.json',
|
||||
'/home/ubuntu/.npm/_cacache/index',
|
||||
'/home/ubuntu/.config/opencode/node_modules/@opencode-ai/plugin',
|
||||
'/home/ubuntu/.orcad-data/node_modules/x',
|
||||
'/usr/lib/x',
|
||||
'/etc/apt/sources.list.d/ubuntu.sources'
|
||||
])
|
||||
})
|
||||
|
||||
it('does not take an added directory above an allowed path as a parent change', () => {
|
||||
expect(unexpectedChanges([{ kind: 'A', path: '/opt/orca' }], HOME)).toHaveLength(1)
|
||||
})
|
||||
})
|
||||
|
||||
describe('pinned versions', () => {
|
||||
const tools = pinnedTools(MANIFEST)
|
||||
|
||||
it('covers every agent CLI package and the pinned binaries', () => {
|
||||
expect(tools.filter((tool) => tool.source === 'package')).toHaveLength(MANIFEST.agentCliPackages.length)
|
||||
expect(tools.map((tool) => tool.name)).toEqual(expect.arrayContaining(['@anthropic-ai/claude-code', 'gh', 'google-chrome']))
|
||||
})
|
||||
|
||||
it('matches exact package versions and version lines that contain the pin', () => {
|
||||
const observed: Record<string, string | null> = {}
|
||||
for (const tool of tools) {
|
||||
observed[tool.name] = tool.source === 'package' ? tool.expect : `prefix ${tool.expect} (suffix)`
|
||||
}
|
||||
expect(versionMismatches(tools, observed)).toEqual([])
|
||||
const claude = tools.find((tool) => tool.name === '@anthropic-ai/claude-code')!
|
||||
expect(versionMismatches([claude], { [claude.name]: '0.0.1' })).toEqual([`${claude.name}: "0.0.1" does not match the pin ${claude.expect}`])
|
||||
expect(versionMismatches([claude], {})).toEqual([`${claude.name}: missing (pinned ${claude.expect})`])
|
||||
})
|
||||
|
||||
it("reads Chrome's .deb revision off the version it reports", () => {
|
||||
const chrome = tools.find((tool) => tool.name === 'google-chrome')!
|
||||
expect(chrome.expect).not.toMatch(/-\d+$/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('entryPointMismatches', () => {
|
||||
it('names missing and changed entry points', () => {
|
||||
expect(entryPointMismatches({ a: '1', b: '2', c: '3' }, { a: '1', b: '9' })).toEqual([
|
||||
'b: differs from image/vm/b',
|
||||
'c: missing from /opt/orca/bin'
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe('installerCommands', () => {
|
||||
it('finds installs, fetches and first-run work among sampled command lines', () => {
|
||||
expect(
|
||||
installerCommands([
|
||||
'bash -c timeout 120 "$1" models && : >"$2" opencode-first-run /opt/orca/tools/bin/opencode /home/ubuntu/.orca-sandbox/opencode-first-run',
|
||||
'timeout 120 /opt/orca/tools/bin/opencode models ',
|
||||
'/opt/orca/runtime/runtimes/node-x/bin/node /opt/orca/bin/orcad-boot --code-cache x',
|
||||
'curl -fsS --max-time 25 -o /tmp/b.tgz https://example.invalid/b.tgz',
|
||||
'git -C /vercel/sandbox/project fetch --progress origin',
|
||||
'git fetch origin',
|
||||
'sleep 0.05',
|
||||
'tr \\0 ',
|
||||
''
|
||||
])
|
||||
).toEqual([
|
||||
'timeout 120 /opt/orca/tools/bin/opencode models',
|
||||
'curl -fsS --max-time 25 -o /tmp/b.tgz https://example.invalid/b.tgz',
|
||||
'git -C /vercel/sandbox/project fetch --progress origin',
|
||||
'git fetch origin'
|
||||
])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,125 @@
|
||||
// Pure checks for the boot-twice harness (boot-twice.ts): what a second boot of the same disk may
|
||||
// change, and whether the image carries the pinned tools and this tree's entry points.
|
||||
import type { GoldenImageManifest } from '../golden-image'
|
||||
|
||||
export type DiffEntry = { kind: 'A' | 'C' | 'D'; path: string }
|
||||
|
||||
/** `docker diff` output: one `<A|C|D> <path>` line per changed path. */
|
||||
export function parseDockerDiff(text: string): DiffEntry[] {
|
||||
const entries: DiffEntry[] = []
|
||||
for (const line of text.split('\n')) {
|
||||
const match = /^([ACD]) (\/.*)$/.exec(line.trim())
|
||||
if (match) {
|
||||
entries.push({ kind: match[1] as DiffEntry['kind'], path: match[2]! })
|
||||
}
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
/**
|
||||
* What any boot may write: runtime state, logs and the per-boot config the start script rewrites.
|
||||
* Everything else a second boot touches (/opt, /usr, /etc, package caches, node_modules) is an
|
||||
* install that should have happened once, in the image or on the first boot.
|
||||
*/
|
||||
export function bootStatePaths(home: string): readonly string[] {
|
||||
return [
|
||||
// orca-sandbox-start: boot id, pids, ready file, logs, start lock, git-setup marks.
|
||||
`${home}/.orca-sandbox`,
|
||||
// orcad's own data directory (device identity, sessions, logs).
|
||||
`${home}/.orcad-data`,
|
||||
// The start script's git identity fallbacks (written only when missing or on an old default).
|
||||
`${home}/.gitconfig`,
|
||||
// orca-sandbox-env rewrites Codex's managed config block and OpenCode's managed provider list on every start.
|
||||
`${home}/.codex/config.toml`,
|
||||
`${home}/.config/opencode/.orca-managed-disabled-providers`,
|
||||
// orcad reads OpenCode's session database at start; SQLite readers recreate its WAL and index files.
|
||||
`${home}/.local/share/opencode/opencode.db-shm`,
|
||||
`${home}/.local/share/opencode/opencode.db-wal`,
|
||||
// Node's compile cache: orcad's node adds or refreshes entries for code it compiles. A cache, not an install.
|
||||
'/opt/orca/compile-cache',
|
||||
'/tmp',
|
||||
'/var/tmp',
|
||||
'/run'
|
||||
]
|
||||
}
|
||||
|
||||
function under(path: string, prefix: string): boolean {
|
||||
return path === prefix || path.startsWith(`${prefix}/`)
|
||||
}
|
||||
|
||||
/** Changes outside bootStatePaths. A changed directory that only contains allowed paths is fine. */
|
||||
export function unexpectedChanges(entries: readonly DiffEntry[], home: string): DiffEntry[] {
|
||||
const allowed = bootStatePaths(home)
|
||||
return entries.filter(({ kind, path }) => {
|
||||
if (/(^|\/)node_modules(\/|$)/.test(path)) {
|
||||
return true
|
||||
}
|
||||
if (allowed.some((prefix) => under(path, prefix))) {
|
||||
return false
|
||||
}
|
||||
// `C /home/ubuntu`: a parent directory of an allowed path changed its mtime.
|
||||
return !(kind === 'C' && allowed.some((prefix) => prefix.startsWith(`${path}/`)))
|
||||
})
|
||||
}
|
||||
|
||||
/** One pinned tool: where its version is read in the image and the string it must contain. */
|
||||
export type PinnedTool = { name: string; expect: string; source: 'package' | 'binary' }
|
||||
|
||||
/** Every version the manifest pins that the image can report. */
|
||||
export function pinnedTools(manifest: GoldenImageManifest): PinnedTool[] {
|
||||
const tools: PinnedTool[] = manifest.agentCliPackages.map((spec) => {
|
||||
const at = spec.lastIndexOf('@')
|
||||
return { name: spec.slice(0, at), expect: spec.slice(at + 1), source: 'package' }
|
||||
})
|
||||
if (manifest.gh.version) {
|
||||
tools.push({ name: 'gh', expect: `gh version ${manifest.gh.version}`, source: 'binary' })
|
||||
}
|
||||
if (manifest.cursorAgent) {
|
||||
tools.push({ name: 'cursor-agent', expect: manifest.cursorAgent.version, source: 'binary' })
|
||||
}
|
||||
if (manifest.chrome) {
|
||||
// The .deb's "157.0.8103.38-1" reports itself as "Google Chrome 157.0.8103.38".
|
||||
tools.push({ name: 'google-chrome', expect: manifest.chrome.version.replace(/-\d+$/, ''), source: 'binary' })
|
||||
}
|
||||
return tools
|
||||
}
|
||||
|
||||
/** Observed versions: npm package.json `version` by package name, else a binary's `--version` line. */
|
||||
export type ObservedVersions = Record<string, string | null>
|
||||
|
||||
export function versionMismatches(tools: readonly PinnedTool[], observed: ObservedVersions): string[] {
|
||||
const problems: string[] = []
|
||||
for (const tool of tools) {
|
||||
const seen = observed[tool.name] ?? null
|
||||
if (seen === null) {
|
||||
problems.push(`${tool.name}: missing (pinned ${tool.expect})`)
|
||||
} else if (tool.source === 'package' ? seen !== tool.expect : !seen.includes(tool.expect)) {
|
||||
problems.push(`${tool.name}: ${JSON.stringify(seen)} does not match the pin ${tool.expect}`)
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
/** Entry points in the image whose bytes differ from this tree's image/vm sources. */
|
||||
export function entryPointMismatches(expected: Record<string, string>, installed: Record<string, string | null>): string[] {
|
||||
return Object.entries(expected)
|
||||
.filter(([name, sha]) => installed[name] !== sha)
|
||||
.map(([name]) => (installed[name] ? `${name}: differs from image/vm/${name}` : `${name}: missing from /opt/orca/bin`))
|
||||
}
|
||||
|
||||
/**
|
||||
* Commands that install, fetch or redo first-run work. Any of them in a second boot is a step that
|
||||
* should have run once. Offline they fail fast and quietly, so the command line is the evidence.
|
||||
*/
|
||||
const INSTALLERS = [
|
||||
/(^|\/)(apt|apt-get|dpkg|pip3?|curl|wget)( |$)/,
|
||||
/(^|\/)npm (install|i|ci|add) /,
|
||||
/(^|\/)git( -[Cc] \S+)* (clone|fetch|pull)( |$)/,
|
||||
/(^|\/)opencode models( |$)/,
|
||||
/(^|\/)tar (-?x|--extract)/
|
||||
]
|
||||
|
||||
/** The installer command lines among a boot's sampled `/proc/<pid>/cmdline`s. */
|
||||
export function installerCommands(commands: readonly string[]): string[] {
|
||||
return [...new Set(commands.map((line) => line.trim()).filter((line) => line && INSTALLERS.some((pattern) => pattern.test(line))))]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
#!/bin/sh
|
||||
# PID 1's child (under `docker run --init`) in a boot-twice container. Each container start is one
|
||||
# VM boot: a fresh kernel boot id, as a restored Vercel VM gets. A container otherwise sees the
|
||||
# host's boot id on every start, and the start script would take a resumed disk for the same boot.
|
||||
set -eu
|
||||
cat /proc/sys/kernel/random/uuid >/run/boot-twice-boot-id
|
||||
mount --bind /run/boot-twice-boot-id /proc/sys/kernel/random/boot_id
|
||||
# Boot 2 resolves through the harness's DNS sink, which logs every name looked up (an offline boot
|
||||
# that tries to fetch something fails quietly; the lookup is what shows it).
|
||||
if [ -n "${BOOT_TWICE_DNS:-}" ]; then
|
||||
printf 'nameserver %s\noptions attempts:1 timeout:1\n' "$BOOT_TWICE_DNS" >/run/boot-twice-resolv.conf
|
||||
mount --bind /run/boot-twice-resolv.conf /etc/resolv.conf
|
||||
fi
|
||||
if [ -n "${BOOT_TWICE_TRACE:-}" ]; then
|
||||
# Every 50 ms, the command line of each process not seen yet: a boot's installs and first-run work
|
||||
# last far longer than that.
|
||||
(
|
||||
seen=' '
|
||||
while :; do
|
||||
for dir in /proc/[0-9]*; do
|
||||
pid="${dir#/proc/}"
|
||||
case "$seen" in *" $pid "*) continue ;; esac
|
||||
seen="$seen$pid "
|
||||
{ tr '\0' ' ' <"$dir/cmdline"; echo; } >>/run/boot-twice-procs.log 2>/dev/null || true
|
||||
done
|
||||
sleep 0.05
|
||||
done
|
||||
) &
|
||||
fi
|
||||
: >/run/boot-twice-ready
|
||||
exec sleep infinity
|
||||
@@ -0,0 +1,485 @@
|
||||
// Boot-twice check for the golden image: provision it in a container, then boot the same disk twice
|
||||
// through the control plane's own boot steps (entry-point probe, workspace prelude + start script,
|
||||
// entry-point sync). The second boot runs with no network and must install and fetch nothing.
|
||||
// pnpm --filter @orca-cloud/sandbox-control exec tsx image/boot-twice/boot-twice.ts [options]
|
||||
// --release <vX.Y.Z|latest> orcad bundle (default: the manifest's bundleUrl, else the latest release)
|
||||
// --image <tag> boot an image built elsewhere (CI builds it with the layer cache)
|
||||
// --context-out <dir> only write the docker build context and exit
|
||||
// --report <file> write the JSON report there too
|
||||
// --keep leave the containers and the disk image for inspection
|
||||
// Needs Docker that can run --privileged containers (Linux, Docker Desktop). See README.md.
|
||||
import { spawn } from 'node:child_process'
|
||||
import { createHash, randomBytes } from 'node:crypto'
|
||||
import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { ENTRY_POINTS_PROBE, ensureEntryPoints, entryPointsBehind, installEntryPoints, parseEntryPointsProbe } from '../../src/golden-images'
|
||||
import { startOrcad, type OrcadReady } from '../../src/sandbox-vm-commands'
|
||||
import type { CommandResult, ProviderSandboxState, RunCommandParams, SandboxProviderApi } from '../../src/sandbox-provider'
|
||||
import { ManifestSchema, VM_SCRIPTS, provisionEnv, resolveReleaseBundle, type BuildInputs, type GoldenImageManifest } from '../golden-image'
|
||||
import { VM_SCRIPTS_HASH, VM_SCRIPTS_STAMP, WORKSPACE_PRELUDE_SCRIPT } from '../vm-scripts.generated'
|
||||
import { entryPointMismatches, installerCommands, parseDockerDiff, pinnedTools, unexpectedChanges, versionMismatches, type DiffEntry, type ObservedVersions } from './boot-twice-checks'
|
||||
|
||||
const here = (path: string): string => fileURLToPath(new URL(path, import.meta.url).href)
|
||||
const HOME = '/home/ubuntu'
|
||||
const FIXTURE_REPO = 'file:///srv/fixture.git'
|
||||
const INSTALL_MARK = 'echo synced'
|
||||
|
||||
type Options = { release?: string; image?: string; contextOut?: string; report?: string; keep: boolean }
|
||||
|
||||
function parseArgs(argv: string[]): Options {
|
||||
const value = (flag: string): string | undefined => {
|
||||
const at = argv.indexOf(flag)
|
||||
return at === -1 ? undefined : argv[at + 1]
|
||||
}
|
||||
return { release: value('--release'), image: value('--image'), contextOut: value('--context-out'), report: value('--report'), keep: argv.includes('--keep') }
|
||||
}
|
||||
|
||||
type Run = { code: number; stdout: string; stderr: string }
|
||||
|
||||
function docker(args: string[], options: { input?: string; timeoutMs?: number } = {}): Promise<Run> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn('docker', args, { stdio: ['pipe', 'pipe', 'pipe'] })
|
||||
let stdout = ''
|
||||
let stderr = ''
|
||||
child.stdout.on('data', (chunk: Buffer) => (stdout += chunk.toString()))
|
||||
child.stderr.on('data', (chunk: Buffer) => (stderr += chunk.toString()))
|
||||
const timer = options.timeoutMs ? setTimeout(() => child.kill('SIGKILL'), options.timeoutMs) : null
|
||||
child.on('error', reject)
|
||||
child.on('close', (code) => {
|
||||
if (timer) {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
resolve({ code: code ?? 124, stdout, stderr })
|
||||
})
|
||||
child.stdin.end(options.input ?? '')
|
||||
})
|
||||
}
|
||||
|
||||
async function dockerOk(args: string[], options: { input?: string; timeoutMs?: number } = {}): Promise<string> {
|
||||
const run = await docker(args, options)
|
||||
if (run.code !== 0) {
|
||||
throw new Error(`docker ${args[0]} failed (${run.code}): ${run.stderr.slice(-1500)}`)
|
||||
}
|
||||
return run.stdout
|
||||
}
|
||||
|
||||
/** Docker in buildx/build output mode: stream it, it takes minutes on a cold cache. */
|
||||
function dockerStreamed(args: string[]): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn('docker', args, { stdio: ['ignore', 'inherit', 'inherit'] })
|
||||
child.on('error', reject)
|
||||
child.on('close', (code) => (code === 0 ? resolve() : reject(new Error(`docker ${args[0]} failed (${code})`))))
|
||||
})
|
||||
}
|
||||
|
||||
// Env arrives on stdin as NUL-separated NAME=value pairs: values can be multi-line (the prelude
|
||||
// script) and large (every entry point, base64), which argv `-e` handles badly.
|
||||
const ENV_LOADER = 'while IFS= read -r -d "" kv; do export "$kv"; done; exec "$@"'
|
||||
|
||||
type CommandRecord = { boot: number; command: string; exitCode: number; ms: number; installed: boolean }
|
||||
|
||||
/** Just enough of the provider for the boot steps: runCommand and readFile, as `docker exec`. */
|
||||
function dockerProvider(boot: () => number, commands: CommandRecord[]): SandboxProviderApi {
|
||||
const unsupported = (name: string) => () => Promise.reject(new Error(`boot-twice provider does not emulate ${name}`))
|
||||
return {
|
||||
async runCommand(container: string, params: RunCommandParams): Promise<CommandResult> {
|
||||
const started = Date.now()
|
||||
const input = Object.entries(params.env ?? {})
|
||||
.map(([key, val]) => `${key}=${val}\0`)
|
||||
.join('')
|
||||
const user = params.sudo ? 'root' : 'ubuntu'
|
||||
const run = await docker(
|
||||
['exec', '-i', '-u', user, '-w', params.cwd ?? HOME, container, 'bash', '-c', ENV_LOADER, 'boot-twice-env', params.command, ...(params.args ?? [])],
|
||||
{ input, timeoutMs: params.timeoutMs }
|
||||
)
|
||||
const script = params.args?.at(-1) ?? ''
|
||||
commands.push({
|
||||
boot: boot(),
|
||||
command: params.command === 'bash' ? `bash -c ${JSON.stringify(script.split('\n')[0]!.slice(0, 80))}` : params.command,
|
||||
exitCode: run.code,
|
||||
ms: Date.now() - started,
|
||||
installed: script.includes(INSTALL_MARK)
|
||||
})
|
||||
return { exitCode: run.code, stdout: run.stdout, stderr: run.stderr }
|
||||
},
|
||||
async readFile(container: string, path: string): Promise<string | null> {
|
||||
const run = await docker(['exec', '-u', 'ubuntu', container, 'cat', '--', path])
|
||||
return run.code === 0 ? run.stdout : null
|
||||
},
|
||||
createSandbox: unsupported('createSandbox'),
|
||||
getSandbox: unsupported('getSandbox'),
|
||||
stopSession: unsupported('stopSession'),
|
||||
extendTimeout: unsupported('extendTimeout'),
|
||||
updateNetworkPolicy: unsupported('updateNetworkPolicy'),
|
||||
updatePorts: unsupported('updatePorts'),
|
||||
updateSnapshotExpiration: unsupported('updateSnapshotExpiration'),
|
||||
deleteSandbox: unsupported('deleteSandbox'),
|
||||
listSnapshots: unsupported('listSnapshots'),
|
||||
deleteSnapshot: unsupported('deleteSnapshot'),
|
||||
listSandboxes: unsupported('listSandboxes'),
|
||||
listProjectSnapshots: unsupported('listProjectSnapshots')
|
||||
}
|
||||
}
|
||||
|
||||
async function latestReleaseTag(): Promise<string> {
|
||||
const headers: Record<string, string> = { accept: 'application/vnd.github+json', 'user-agent': 'orca-boot-twice' }
|
||||
if (process.env.GITHUB_TOKEN) {
|
||||
headers.authorization = `Bearer ${process.env.GITHUB_TOKEN}`
|
||||
}
|
||||
const response = await fetch('https://api.github.com/repos/stablyai/orca/releases/latest', { headers })
|
||||
if (!response.ok) {
|
||||
throw new Error(`latest release lookup failed (HTTP ${response.status}); pass --release vX.Y.Z`)
|
||||
}
|
||||
return ((await response.json()) as { tag_name: string }).tag_name
|
||||
}
|
||||
|
||||
function loadInputs(): BuildInputs {
|
||||
const manifest = ManifestSchema.parse(JSON.parse(readFileSync(here('../golden-image.manifest.json'), 'utf8')))
|
||||
const vmScripts = Object.fromEntries(VM_SCRIPTS.map((name) => [name, readFileSync(here(`../vm/${name}`), 'utf8')])) as BuildInputs['vmScripts']
|
||||
return { manifest, provisionScript: readFileSync(here('../provision-golden.sh'), 'utf8'), vmScripts, vmScriptsStamp: VM_SCRIPTS_STAMP, buildId: 'boot-twice' }
|
||||
}
|
||||
|
||||
async function withBundle(inputs: BuildInputs, release: string | undefined): Promise<BuildInputs> {
|
||||
if (!release && inputs.manifest.orcad.bundleUrl && inputs.manifest.orcad.sha256) {
|
||||
return inputs
|
||||
}
|
||||
const tag = !release || release === 'latest' ? await latestReleaseTag() : release
|
||||
const pinned = await resolveReleaseBundle(tag)
|
||||
return { ...inputs, manifest: { ...inputs.manifest, orcad: { bundleUrl: pinned.bundleUrl, sha256: pinned.sha256 } } }
|
||||
}
|
||||
|
||||
/** The docker build context; its bytes are the layer cache key, so nothing in it varies per run. */
|
||||
function writeContext(dir: string, inputs: BuildInputs): void {
|
||||
mkdirSync(dir, { recursive: true })
|
||||
const lines = Object.entries(provisionEnv(inputs)).map(([key, val]) => {
|
||||
if (val.includes("'")) {
|
||||
throw new Error(`provision env ${key} has a quote`)
|
||||
}
|
||||
return `${key}='${val}'`
|
||||
})
|
||||
writeFileSync(join(dir, 'provision.env'), `${lines.join('\n')}\n`)
|
||||
copyFileSync(here('../provision-golden.sh'), join(dir, 'provision-golden.sh'))
|
||||
copyFileSync(here('./Dockerfile'), join(dir, 'Dockerfile'))
|
||||
copyFileSync(here('./boot-twice-init'), join(dir, 'boot-twice-init'))
|
||||
}
|
||||
|
||||
type BootReport = {
|
||||
boot: number
|
||||
network: boolean
|
||||
ms: number
|
||||
steps: Record<string, number>
|
||||
probe: { stamp: number; marker: boolean } | null
|
||||
ready: OrcadReady | null
|
||||
error?: string
|
||||
}
|
||||
|
||||
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))
|
||||
|
||||
async function waitFor(container: string, test: string, timeoutMs: number): Promise<boolean> {
|
||||
const deadline = Date.now() + timeoutMs
|
||||
while (Date.now() < deadline) {
|
||||
if ((await docker(['exec', container, 'sh', '-c', test])).code === 0) {
|
||||
return true
|
||||
}
|
||||
await sleep(250)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Quiet: no first-run work still running, and nothing outside the boot state paths written in the
|
||||
// last 3 s. orcad installs agent hooks and OpenCode's first run ends a few seconds after ready.
|
||||
const QUIET = [
|
||||
'pgrep -f "opencode [m]odels|agent-[p]refetch" >/dev/null && exit 1',
|
||||
`find ${HOME} /opt/orca -xdev \\( -path ${HOME}/.orca-sandbox -o -path ${HOME}/.orcad-data \\) -prune -o -newermt "-3 seconds" -print | grep -q . && exit 1`,
|
||||
'exit 0'
|
||||
].join('\n')
|
||||
|
||||
async function settle(container: string): Promise<void> {
|
||||
await sleep(3_000)
|
||||
await waitFor(container, QUIET, 150_000)
|
||||
}
|
||||
|
||||
// The DNS sink: answers every query NXDOMAIN and prints each name once ("lookup <name>").
|
||||
const DNS_SINK = `
|
||||
const seen = new Set()
|
||||
const socket = require('node:dgram').createSocket('udp4')
|
||||
socket.on('message', (msg, peer) => {
|
||||
if (msg.length < 12) return
|
||||
const labels = []
|
||||
let at = 12
|
||||
while (at < msg.length && msg[at] !== 0) { labels.push(msg.subarray(at + 1, at + 1 + msg[at]).toString()); at += msg[at] + 1 }
|
||||
const name = labels.join('.').toLowerCase()
|
||||
if (!seen.has(name)) { seen.add(name); console.log('lookup ' + name) }
|
||||
const reply = Buffer.from(msg.subarray(0, at + 5))
|
||||
reply[2] = 0x81; reply[3] = 0x83; reply.writeUInt16BE(0, 6); reply.writeUInt16BE(0, 8); reply.writeUInt16BE(0, 10)
|
||||
socket.send(reply, peer.port, peer.address)
|
||||
})
|
||||
socket.bind(53, () => console.log('sink ready'))
|
||||
`
|
||||
|
||||
/** Names the harness itself makes up; anything else boot 2 looks up is a fetch attempt. */
|
||||
const LOCAL_NAMES = /(^|\.)(invalid|localhost|local)$|^boot-twice(-\d+)?$/
|
||||
|
||||
type Sink = { network: string; ip: string; container: string }
|
||||
|
||||
async function startSink(image: string, runId: string): Promise<Sink> {
|
||||
const network = `orca-boot-twice-${runId}`
|
||||
const container = `orca-boot-twice-${runId}-dns`
|
||||
await dockerOk(['network', 'create', '--internal', network])
|
||||
await dockerOk(['run', '-d', '--name', container, '--network', network, '--user', 'root', image, 'node', '-e', DNS_SINK])
|
||||
const ip = (await dockerOk(['inspect', '-f', `{{(index .NetworkSettings.Networks "${network}").IPAddress}}`, container])).trim()
|
||||
const deadline = Date.now() + 15_000
|
||||
while (!(await docker(['logs', container])).stdout.includes('sink ready')) {
|
||||
if (Date.now() > deadline) {
|
||||
throw new Error('DNS sink did not start')
|
||||
}
|
||||
await sleep(200)
|
||||
}
|
||||
return { network, ip, container }
|
||||
}
|
||||
|
||||
/**
|
||||
* Lookups every orcad start makes, by design: reported, not failed. orcad refreshes its agent-state
|
||||
* rules from the orca GitHub releases at start (and keeps working on its cached copy when offline).
|
||||
* The cost: a git fetch to github.com on boot 2 would hide behind this; the disk diff still catches
|
||||
* one that wrote anything.
|
||||
*/
|
||||
const RUNTIME_LOOKUPS: Readonly<Record<string, string>> = {
|
||||
'github.com': "orcad's agent-state-rules refresh at start"
|
||||
}
|
||||
|
||||
async function sinkLookups(sink: Sink): Promise<{ unexpected: string[]; expected: string[] }> {
|
||||
const out = (await docker(['logs', sink.container])).stdout
|
||||
const names = [...out.matchAll(/^lookup (.+)$/gm)].map((match) => match[1]!).filter((name) => !LOCAL_NAMES.test(name))
|
||||
return { unexpected: names.filter((name) => !(name in RUNTIME_LOOKUPS)), expected: names.filter((name) => name in RUNTIME_LOOKUPS) }
|
||||
}
|
||||
|
||||
/** One VM boot: a new container on the disk image, then the steps lifecycle.ts runs on create/wake. */
|
||||
async function bootOnce(
|
||||
provider: SandboxProviderApi,
|
||||
image: string,
|
||||
container: string,
|
||||
boot: number,
|
||||
options: { network: boolean; prelude: boolean; sink?: { network: string; ip: string } }
|
||||
): Promise<BootReport> {
|
||||
const steps: Record<string, number> = {}
|
||||
const time = async <T>(name: string, step: () => Promise<T>): Promise<T> => {
|
||||
const started = Date.now()
|
||||
try {
|
||||
return await step()
|
||||
} finally {
|
||||
steps[name] = Date.now() - started
|
||||
}
|
||||
}
|
||||
const report: BootReport = { boot, network: options.network, ms: 0, steps, probe: null, ready: null }
|
||||
// Offline: an internal network (no route out) whose only resolver is the DNS sink.
|
||||
const network = options.network ? [] : options.sink ? ['--network', options.sink.network, '--env', `BOOT_TWICE_DNS=${options.sink.ip}`] : ['--network', 'none']
|
||||
// Every boot records the commands it runs (/run/boot-twice-procs.log, sampled by the init).
|
||||
network.push('--env', 'BOOT_TWICE_TRACE=1')
|
||||
await dockerOk(['run', '-d', '--init', '--privileged', '--name', container, '--hostname', `boot-twice-${boot}`, '--tmpfs', '/run', ...network, image])
|
||||
if (!(await waitFor(container, 'test -e /run/boot-twice-ready', 30_000))) {
|
||||
throw new Error(`boot ${boot}: container init did not come up`)
|
||||
}
|
||||
const started = Date.now()
|
||||
try {
|
||||
// refreshRuntime (no runtime bundle published here): the probe, and an install only when behind.
|
||||
const probe = await time('probe', () => provider.runCommand(container, { command: 'bash', args: ['-c', ENTRY_POINTS_PROBE], sudo: true, timeoutMs: 40_000 }))
|
||||
report.probe = parseEntryPointsProbe(probe.stdout)
|
||||
if (entryPointsBehind(report.probe)) {
|
||||
await time('entryPointsBeforeStart', () => installEntryPoints({ provider }, container))
|
||||
}
|
||||
const state: ProviderSandboxState = {
|
||||
sandbox: { name: container, status: 'running', currentSessionId: container, currentSnapshotId: null, tags: {} },
|
||||
session: { id: container, status: 'running', startedAt: started, timeoutMs: 3_600_000 },
|
||||
routes: [{ port: 6768, url: 'https://boot-twice.invalid' }]
|
||||
}
|
||||
report.ready = await time('orcad', () =>
|
||||
startOrcad(provider, state, {
|
||||
sandboxId: `boot-twice-${boot}`,
|
||||
hostname: 'boot-twice',
|
||||
repo: { url: FIXTURE_REPO, ref: 'main' },
|
||||
docker: true,
|
||||
prelude: options.prelude ? { script: WORKSPACE_PRELUDE_SCRIPT } : undefined
|
||||
})
|
||||
)
|
||||
// settleMemberGit after ready: the per-boot entry-point check (an install at most once per image).
|
||||
await time('entryPointsAfterStart', () => ensureEntryPoints({ provider }, container))
|
||||
} catch (error) {
|
||||
report.error = error instanceof Error ? error.message : String(error)
|
||||
}
|
||||
report.ms = Date.now() - started
|
||||
return report
|
||||
}
|
||||
|
||||
const VERSIONS_SCRIPT = `
|
||||
const { execFileSync } = require('node:child_process')
|
||||
const { readFileSync } = require('node:fs')
|
||||
const [packages, binaries] = JSON.parse(process.argv[1])
|
||||
const out = {}
|
||||
for (const name of packages) {
|
||||
try { out[name] = JSON.parse(readFileSync('/opt/orca/tools/lib/node_modules/' + name + '/package.json', 'utf8')).version } catch { out[name] = null }
|
||||
}
|
||||
for (const name of binaries) {
|
||||
try {
|
||||
const text = execFileSync(name, ['--version'], { encoding: 'utf8', timeout: 30000, stdio: ['ignore', 'pipe', 'ignore'] })
|
||||
out[name] = text.trim().split('\\n')[0] || null
|
||||
} catch { out[name] = null }
|
||||
}
|
||||
console.log(JSON.stringify(out))
|
||||
`
|
||||
|
||||
// What a booted image carries: entry points (sha256), the scripts stamp and markers, orcad's
|
||||
// version and whether it is listening on 6768 (0x1A70) on all interfaces.
|
||||
const IMAGE_PROBE = [
|
||||
'cd /opt/orca/bin',
|
||||
'for f in * .scripts-stamp .entry-points-*; do [ -f "$f" ] && echo "file $(sha256sum "$f" | cut -d" " -f1) $f"; done',
|
||||
'echo "stamp $(cat .scripts-stamp 2>/dev/null || echo none)"',
|
||||
'echo "orcad_version $(cat /opt/orca/runtime/orcad/.version 2>/dev/null || echo none)"',
|
||||
'kill -0 "$(cat ~/.orca-sandbox/orcad.pid 2>/dev/null || echo 0)" 2>/dev/null && echo orcad_alive=1 || echo orcad_alive=0',
|
||||
'grep -qi " 00000000:1A70 00000000:0000 0A" /proc/net/tcp && echo orcad_listening=1 || echo orcad_listening=0',
|
||||
// Installers or fetchers still running after the boot answered.
|
||||
'echo "busy $(ps -eo args= | grep -E "^(apt|apt-get|dpkg|npm|npx|pip|curl|wget) |opencode [m]odels|git ([c]lone|[f]etch)" | head -3 | tr "\\n" ";")"'
|
||||
].join('\n')
|
||||
|
||||
type Failure = string
|
||||
|
||||
async function checkImage(provider: SandboxProviderApi, container: string, manifest: GoldenImageManifest, failures: Failure[]) {
|
||||
const sources = Object.fromEntries(VM_SCRIPTS.map((name) => [name, createHash('sha256').update(readFileSync(here(`../vm/${name}`))).digest('hex')]))
|
||||
const probe = await provider.runCommand(container, { command: 'bash', args: ['-c', IMAGE_PROBE], timeoutMs: 30_000 })
|
||||
const installed: Record<string, string> = {}
|
||||
for (const match of probe.stdout.matchAll(/^file ([0-9a-f]{64}) (.+)$/gm)) {
|
||||
installed[match[2]!] = match[1]!
|
||||
}
|
||||
failures.push(...entryPointMismatches(sources, installed).map((problem) => `entry point ${problem}`))
|
||||
const stamp = /^stamp (.+)$/m.exec(probe.stdout)?.[1]
|
||||
if (stamp !== String(VM_SCRIPTS_STAMP)) {
|
||||
failures.push(`/opt/orca/bin/.scripts-stamp is ${stamp ?? 'unreadable'}, expected ${VM_SCRIPTS_STAMP}`)
|
||||
}
|
||||
if (!(`.entry-points-${VM_SCRIPTS_HASH}` in installed)) {
|
||||
failures.push(`no .entry-points-${VM_SCRIPTS_HASH} marker after two boots`)
|
||||
}
|
||||
if (!/^orcad_alive=1$/m.test(probe.stdout) || !/^orcad_listening=1$/m.test(probe.stdout)) {
|
||||
failures.push('orcad is not running and listening on :6768 after the second boot')
|
||||
}
|
||||
const busy = /^busy (.*)$/m.exec(probe.stdout)?.[1]?.trim()
|
||||
if (busy) {
|
||||
failures.push(`still installing or fetching after the second boot: ${busy}`)
|
||||
}
|
||||
const tools = pinnedTools(manifest)
|
||||
const run = await provider.runCommand(container, {
|
||||
command: 'bash',
|
||||
args: ['-lc', 'node -e "$1" "$2"', 'versions', VERSIONS_SCRIPT, JSON.stringify([tools.filter((t) => t.source === 'package').map((t) => t.name), tools.filter((t) => t.source === 'binary').map((t) => t.name)])],
|
||||
timeoutMs: 180_000
|
||||
})
|
||||
let observed: ObservedVersions = {}
|
||||
try {
|
||||
observed = JSON.parse(run.stdout.trim().split('\n').at(-1) ?? '{}') as ObservedVersions
|
||||
} catch {
|
||||
failures.push(`version probe failed (exit ${run.exitCode}): ${run.stderr.slice(-300)}`)
|
||||
}
|
||||
failures.push(...versionMismatches(tools, observed))
|
||||
return { orcadVersion: /^orcad_version (.+)$/m.exec(probe.stdout)?.[1] ?? null, versions: observed, entryPoints: Object.keys(installed).length }
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const options = parseArgs(process.argv.slice(2))
|
||||
const inputs = await withBundle(loadInputs(), options.release)
|
||||
if (options.contextOut) {
|
||||
writeContext(options.contextOut, inputs)
|
||||
console.log(JSON.stringify({ context: options.contextOut, bundleUrl: inputs.manifest.orcad.bundleUrl }))
|
||||
return
|
||||
}
|
||||
const runId = randomBytes(3).toString('hex')
|
||||
const started = Date.now()
|
||||
let image = options.image
|
||||
if (!image) {
|
||||
const context = mkdtempSync(join(tmpdir(), 'orca-boot-twice-'))
|
||||
writeContext(context, inputs)
|
||||
image = 'orca-boot-twice:local'
|
||||
console.error(`[boot-twice] building ${image} from ${context} (orcad ${inputs.manifest.orcad.bundleUrl})`)
|
||||
await dockerStreamed(['build', '--target', 'harness', '-t', image, context])
|
||||
}
|
||||
const buildMs = Date.now() - started
|
||||
const disk = `orca-boot-twice-disk:${runId}`
|
||||
const containers = [`orca-boot-twice-${runId}-1`, `orca-boot-twice-${runId}-2`] as const
|
||||
const commands: CommandRecord[] = []
|
||||
let current = 1
|
||||
const provider = dockerProvider(() => current, commands)
|
||||
const failures: Failure[] = []
|
||||
let diff: DiffEntry[] = []
|
||||
let unexpected: DiffEntry[] = []
|
||||
let image2: Awaited<ReturnType<typeof checkImage>> | null = null
|
||||
const boots: BootReport[] = []
|
||||
let sink: Sink | null = null
|
||||
let lookups: { unexpected: string[]; expected: string[] } = { unexpected: [], expected: [] }
|
||||
try {
|
||||
console.error('[boot-twice] boot 1 (first boot of the image, network on)')
|
||||
boots.push(await bootOnce(provider, image, containers[0], 1, { network: true, prelude: true }))
|
||||
// A VM lives a while after ready: let the first boot's background work finish before the disk is kept.
|
||||
await settle(containers[0])
|
||||
await dockerOk(['stop', '-t', '20', containers[0]])
|
||||
await dockerOk(['commit', containers[0], disk])
|
||||
current = 2
|
||||
console.error('[boot-twice] boot 2 (same disk, new boot id, no network)')
|
||||
sink = await startSink(image, runId)
|
||||
boots.push(await bootOnce(provider, disk, containers[1], 2, { network: false, prelude: false, sink }))
|
||||
for (const boot of boots) {
|
||||
if (boot.error || !boot.ready?.pairingCode.startsWith('orca://')) {
|
||||
failures.push(`boot ${boot.boot}: orcad did not come up (${boot.error ?? 'no pairing offer'})`)
|
||||
}
|
||||
}
|
||||
const reinstalls = commands.filter((command) => command.boot === 2 && command.installed)
|
||||
if (reinstalls.length > 0) {
|
||||
failures.push(`boot 2 reinstalled the entry points (${reinstalls.length}x): the first boot's marker did not stick`)
|
||||
}
|
||||
if (boots[1]?.probe && !boots[1].probe.marker) {
|
||||
failures.push('boot 2 probe: no entry-points marker for this tree')
|
||||
}
|
||||
// Give the second boot the same time to do (and show) anything it starts in the background.
|
||||
await settle(containers[1])
|
||||
const procs = await docker(['exec', containers[1], 'cat', '/run/boot-twice-procs.log'])
|
||||
failures.push(...installerCommands(procs.stdout.split('\n')).map((command) => `boot 2 ran ${JSON.stringify(command.slice(0, 160))}`))
|
||||
lookups = await sinkLookups(sink)
|
||||
failures.push(...lookups.unexpected.map((name) => `boot 2 tried to reach ${name}`))
|
||||
// The disk diff comes first: the version probe runs the agent CLIs, which write their own caches.
|
||||
diff = parseDockerDiff(await dockerOk(['diff', containers[1]]))
|
||||
unexpected = unexpectedChanges(diff, HOME)
|
||||
failures.push(...unexpected.map((entry) => `boot 2 changed ${entry.kind} ${entry.path}`))
|
||||
image2 = await checkImage(provider, containers[1], inputs.manifest, failures)
|
||||
} finally {
|
||||
if (!options.keep) {
|
||||
await docker(['rm', '-f', ...containers, ...(sink ? [sink.container] : [])])
|
||||
await docker(['rmi', '-f', disk])
|
||||
if (sink) {
|
||||
await docker(['network', 'rm', sink.network])
|
||||
}
|
||||
}
|
||||
}
|
||||
const report = {
|
||||
ok: failures.length === 0,
|
||||
failures,
|
||||
image,
|
||||
orcadBundle: inputs.manifest.orcad.bundleUrl,
|
||||
buildMs,
|
||||
boots: boots.map(({ ready, ...rest }) => ({ ...rest, ready: ready ? { projectRoot: ready.projectRoot } : null })),
|
||||
commands,
|
||||
boot2: { diff: diff.length, unexpected, lookups, ...image2 },
|
||||
totalMs: Date.now() - started
|
||||
}
|
||||
if (options.report) {
|
||||
writeFileSync(options.report, `${JSON.stringify(report, null, 2)}\n`)
|
||||
}
|
||||
console.log(JSON.stringify(report, null, 2))
|
||||
for (const boot of boots) {
|
||||
console.error(`[boot-twice] boot ${boot.boot}: ${boot.ms} ms ${JSON.stringify(boot.steps)}`)
|
||||
}
|
||||
console.error(report.ok ? '[boot-twice] OK: the second boot installed nothing' : `[boot-twice] FAILED:\n ${failures.join('\n ')}`)
|
||||
process.exitCode = report.ok ? 0 : 1
|
||||
}
|
||||
|
||||
main().catch((error: unknown) => {
|
||||
console.error(error instanceof Error ? error.stack ?? error.message : String(error))
|
||||
process.exitCode = 1
|
||||
})
|
||||
@@ -7,6 +7,7 @@ import type { SandboxProviderApi } from '../src/sandbox-provider'
|
||||
export const VM_SCRIPTS = [
|
||||
'orca-sandbox-start',
|
||||
'orca-sandbox-drain',
|
||||
'orca-memguard',
|
||||
'orca-sandbox-env',
|
||||
'orca-session-env.sh',
|
||||
'orca-env',
|
||||
|
||||
@@ -55,8 +55,11 @@ if [ ! -s "$ORCA_HOME/runtime/orcad/orcad-server.js" ]; then
|
||||
server="$(find "$ORCA_HOME/runtime" -name orcad-server.js -type f | head -n 1)"
|
||||
[ -n "$server" ] || { log 'bundle has no orcad-server.js'; exit 1; }
|
||||
root="$(dirname "$(dirname "$server")")"
|
||||
ln -sfn "$(dirname "$server")" "$ORCA_HOME/runtime/orcad"
|
||||
[ -d "$ORCA_HOME/runtime/runtimes" ] || ln -sfn "$root/runtimes" "$ORCA_HOME/runtime/runtimes"
|
||||
# Move the tree up rather than linking to it: every VM script finds node with
|
||||
# `find .../runtime/runtimes -type f`, and find doesn't descend into a symlinked start point
|
||||
# (a linked release bundle left the build with no node). orcad's ../runtimes still resolves.
|
||||
for entry in "$root"/*; do mv "$entry" "$ORCA_HOME/runtime/"; done
|
||||
rmdir "$root"
|
||||
fi
|
||||
test -s "$ORCA_HOME/runtime/orcad/orcad-server.js"
|
||||
node="$(find "$ORCA_HOME/runtime/runtimes" -type f -name node | head -n 1)"
|
||||
@@ -98,6 +101,16 @@ if [ -x "$opencode_pkg/bin/opencode.exe" ]; then
|
||||
[ "$(stat -c %i "$variant/bin/opencode" 2>/dev/null)" = "$picked" ] || rm -rf "$variant"
|
||||
done
|
||||
"$opencode_pkg/bin/opencode.exe" --version >/dev/null
|
||||
# OpenCode installs @opencode-ai/plugin (its own version) into ~/.config/opencode on every start
|
||||
# until that dir has node_modules and a lock listing it, and draws nothing until that install ends
|
||||
# once a plugin is configured (Orca's is): with a slow or unreachable registry the first launch
|
||||
# sat blank ~85 s (n73). Install it here once; orca-sandbox-start copies it into a fresh home.
|
||||
oc_seed="$ORCA_HOME/opencode-config-seed"
|
||||
mkdir -p "$oc_seed"
|
||||
printf '{\n "dependencies": {\n "@opencode-ai/plugin": "%s"\n }\n}\n' \
|
||||
"$("$opencode_pkg/bin/opencode.exe" --version)" >"$oc_seed/package.json"
|
||||
(cd "$oc_seed" && npm install --no-fund --no-audit --ignore-scripts >/dev/null)
|
||||
test -s "$oc_seed/package-lock.json" && test -d "$oc_seed/node_modules/@opencode-ai/plugin"
|
||||
fi
|
||||
|
||||
# Copilot CLI prefers GH_TOKEN/GITHUB_TOKEN over the user's own /login. In Orca Cloud those hold the
|
||||
@@ -210,7 +223,7 @@ if [ -n "${NOVNC_VERSION:-}" ]; then
|
||||
fi
|
||||
|
||||
log 'Orca entry points'
|
||||
for name in orca-sandbox-start orca-sandbox-drain orca-sandbox-env orca-session-env.sh orca-env orca-egress-probe orca-image-wipe orca-agent-signout orca-share-proxy orca-sshd orca-ssh-keys orca-desktop-start orca-desktop-proxy orca-desktop-page.js orca-sandbox-grant orca-sandbox-revoke orca-sandbox-members orca-sandbox-read-snapshot orca-read-snapshot.cjs orca-git-as orca-git-identity orcad-boot git docker; do
|
||||
for name in orca-sandbox-start orca-sandbox-drain orca-memguard orca-sandbox-env orca-session-env.sh orca-env orca-egress-probe orca-image-wipe orca-agent-signout orca-share-proxy orca-sshd orca-ssh-keys orca-desktop-start orca-desktop-proxy orca-desktop-page.js orca-sandbox-grant orca-sandbox-revoke orca-sandbox-members orca-sandbox-read-snapshot orca-read-snapshot.cjs orca-git-as orca-git-identity orcad-boot git docker; do
|
||||
var="ORCA_VM_SCRIPT_${name//[-.]/_}"
|
||||
printf '%s' "${!var}" | base64 -d >"$ORCA_HOME/bin/$name"
|
||||
chmod 0755 "$ORCA_HOME/bin/$name"
|
||||
|
||||
File diff suppressed because one or more lines are too long
+308
@@ -0,0 +1,308 @@
|
||||
#!/usr/bin/env bash
|
||||
# Memory guard for the workspace VM (runs as root: `sudo -n orca-memguard start`). A build or test
|
||||
# that eats the VM's memory must cost that process, never the workspace: without this the kernel
|
||||
# thrashes for minutes before its OOM killer picks something, and the VM looks dead meanwhile.
|
||||
# start start the guard for this boot unless it already runs (returns at once)
|
||||
# run the guard loop itself (start runs it detached)
|
||||
# release-swap turn off and delete on-demand swap (before a stop, so the snapshot stays small)
|
||||
# sweep apply OOM-killer priorities once and exit (tests, diagnostics)
|
||||
# victim print "<pid> <rss KiB> <name>" of the process the guard would stop now (no kill)
|
||||
# Every few seconds it ranks processes for the kernel's OOM killer (oom_score_adj):
|
||||
# -900 Orca's own runtime: orcad, its terminal daemon, the share proxy (Orca's bundled Node)
|
||||
# -500 terminal shells (children of the terminal daemon) and agent CLIs
|
||||
# 500 everything else (builds, tests, dev servers, browsers); never lowered
|
||||
# When MemAvailable falls under KILL_PCT (8%) of RAM it kills the largest process ranked 0 or more,
|
||||
# records it in $RUN/kills and $RUN/state, and waits for memory to return. The kernel's OOM killer
|
||||
# stays the backstop for allocations faster than the guard's 200 ms look, with the same ranks.
|
||||
# Swap is added on demand (2 GiB files up to 1.5x RAM) when memory runs low, if the kernel takes it,
|
||||
# and dropped after 10 quiet minutes. It also sizes /dev/shm to half of RAM (the provider mounts
|
||||
# 64 MB, which crashes Chrome and Electron renderers).
|
||||
# $RUN/state (key=value, world-readable) is what the control plane's status probe reads.
|
||||
set -uo pipefail
|
||||
RUN=/run/orca-memguard
|
||||
# Tests point it at a fake process table.
|
||||
PROC="${ORCA_MEMGUARD_PROC:-/proc}"
|
||||
SWAP_DIR=/var/lib/orca-memguard
|
||||
ORCA_HOME=/opt/orca
|
||||
KILL_PCT="${ORCA_MEMGUARD_KILL_PCT:-8}"
|
||||
HIGH_PCT=15
|
||||
SWAP_ADD_PCT=20
|
||||
QUIET_PCT=50
|
||||
QUIET_SECONDS=600
|
||||
TICK=1
|
||||
FAST_TICK=0.2
|
||||
SWEEP_EVERY=3
|
||||
STATE_EVERY=15
|
||||
SWAP_CHUNK_KIB=$((2 * 1024 * 1024))
|
||||
|
||||
boot_id() { local id; read -r id </proc/sys/kernel/random/boot_id && printf '%s' "$id"; }
|
||||
now_ms() { local t="${EPOCHREALTIME//[.,]/}"; printf '%s' "${t:0:13}"; }
|
||||
log() { printf '%s %s\n' "$(date -u +%FT%TZ)" "$*" >>"$RUN/memguard.log"; }
|
||||
|
||||
# Name of a process for people: the script a node/python/... runs, else its command name.
|
||||
proc_name() {
|
||||
local pid="$1" comm arg name='' skip=0 module=0 args=()
|
||||
read -r comm 2>/dev/null <"$PROC/$pid/comm" || comm=''
|
||||
mapfile -d '' -t args 2>/dev/null <"$PROC/$pid/cmdline" || true
|
||||
case "${args[0]##*/}" in
|
||||
node | nodejs | bun | deno | python* | ruby | perl | java | sh | bash | dash | zsh)
|
||||
for arg in "${args[@]:1}"; do
|
||||
if [ "$skip" -eq 1 ]; then skip=0; continue; fi
|
||||
if [ "$module" -eq 1 ]; then name="${arg##*/}"; break; fi
|
||||
case "$arg" in
|
||||
# Inline code is no name (and may hold anything): the interpreter's name it is.
|
||||
-e | -p | -c | --eval | --print) break ;;
|
||||
-m | -jar) module=1 ;; # python -m pytest, java -jar gradle.jar: the module or jar
|
||||
-r | --require | --import | --loader | -X | -W | -cp | -classpath) skip=1 ;;
|
||||
-*) ;;
|
||||
*) name="${arg##*/}"; break ;;
|
||||
esac
|
||||
done ;;
|
||||
esac
|
||||
[ -n "$name" ] || name="${args[0]##*/}"
|
||||
[ -n "$name" ] || name="$comm"
|
||||
name="${name%.js}"
|
||||
name="${name%.mjs}"
|
||||
name="${name%.cjs}"
|
||||
name="${name%.jar}"
|
||||
name="${name//[^A-Za-z0-9._+-]/}"
|
||||
printf '%s' "${name:0:40}"
|
||||
}
|
||||
|
||||
# The terminal daemon's pid(s): Orca-runtime node processes running the daemon entry.
|
||||
daemon_pids=' '
|
||||
# Sets RANK to the oom_score_adj a process should have; returns 1 for a kernel thread or a gone pid.
|
||||
RANK=500
|
||||
rank() {
|
||||
local pid="$1" args=() stat ppid
|
||||
mapfile -d '' -t -n 3 args 2>/dev/null <"$PROC/$pid/cmdline" || return 1
|
||||
[ "${#args[@]}" -gt 0 ] || return 1
|
||||
RANK=-900
|
||||
case "${args[0]}" in "$ORCA_HOME"/runtime/runtimes/*) return 0 ;; esac
|
||||
RANK=-500
|
||||
read -r stat 2>/dev/null <"$PROC/$pid/stat" || return 1
|
||||
stat="${stat##*) }"
|
||||
ppid="${stat#* }"
|
||||
ppid="${ppid%% *}"
|
||||
case "$daemon_pids" in *" $ppid "*) return 0 ;; esac
|
||||
case "${args[0]}/${args[1]:-}/${args[2]:-}" in "$ORCA_HOME"/tools/* | */"$ORCA_HOME"/tools/*) return 0 ;; esac
|
||||
case "${args[0]##*/}" in claude | codex | opencode | gemini | cursor-agent | copilot | amp | droid) return 0 ;; esac
|
||||
RANK=500
|
||||
}
|
||||
|
||||
sweep() {
|
||||
local p pid want adj args=()
|
||||
daemon_pids=' '
|
||||
for p in "$PROC"/[0-9]*; do
|
||||
pid="${p#"$PROC"/}"
|
||||
mapfile -d '' -t -n 3 args 2>/dev/null <"$p/cmdline" || continue
|
||||
[ "${#args[@]}" -ge 2 ] || continue
|
||||
case "${args[0]}" in "$ORCA_HOME"/runtime/runtimes/*) ;; *) continue ;; esac
|
||||
case "${args[*]:1}" in *"$ORCA_HOME"/runtime/orcad/daemon*) daemon_pids+="$pid " ;; esac
|
||||
done
|
||||
for p in "$PROC"/[0-9]*; do
|
||||
pid="${p#"$PROC"/}"
|
||||
[ "$pid" -gt 1 ] && [ "$pid" != "$$" ] || continue
|
||||
read -r adj 2>/dev/null <"$p/oom_score_adj" || continue
|
||||
# -1000 is a process that asked for it as root (sshd): leave it alone.
|
||||
[ "$adj" -gt -1000 ] || continue
|
||||
rank "$pid" || continue
|
||||
want="$RANK"
|
||||
# Never lower what a process raised for itself (Chrome ranks its renderers above its browser).
|
||||
if [ "$want" -eq 500 ]; then
|
||||
[ "$adj" -lt 500 ] || continue
|
||||
fi
|
||||
[ "$adj" -ne "$want" ] || continue
|
||||
printf '%s\n' "$want" 2>/dev/null >"$p/oom_score_adj" || true
|
||||
done
|
||||
}
|
||||
|
||||
mem_total=0 mem_avail=0 swap_total=0 swap_free=0
|
||||
read_mem() {
|
||||
local key value _
|
||||
while read -r key value _; do
|
||||
case "$key" in
|
||||
MemTotal:) mem_total="$value" ;;
|
||||
MemAvailable:) mem_avail="$value" ;;
|
||||
SwapTotal:) swap_total="$value" ;;
|
||||
SwapFree:) swap_free="$value" ;;
|
||||
esac
|
||||
done <"$PROC/meminfo"
|
||||
}
|
||||
|
||||
# PSI percentages (avg10, integer part); 0 when the kernel doesn't report them.
|
||||
psi_some=0 psi_full=0
|
||||
read_psi() {
|
||||
local kind avg _
|
||||
psi_some=0 psi_full=0
|
||||
[ -r "$PROC/pressure/memory" ] || return 0
|
||||
while read -r kind avg _; do
|
||||
avg="${avg#avg10=}"
|
||||
avg="${avg%%.*}"
|
||||
case "$kind" in some) psi_some="$avg" ;; full) psi_full="$avg" ;; esac
|
||||
done 2>/dev/null <"$PROC/pressure/memory" || true
|
||||
}
|
||||
|
||||
kills=0 last_kill_name='' last_kill_at=0 swap_mode=on-demand level=ok
|
||||
write_state() {
|
||||
local pct="$1"
|
||||
{
|
||||
printf 'bootId=%s\nat=%s\npid=%s\nlevel=%s\navailPct=%s\nmemTotalKiB=%s\nmemAvailKiB=%s\n' \
|
||||
"$BOOT" "$(now_ms)" "$$" "$level" "$pct" "$mem_total" "$mem_avail"
|
||||
printf 'swapTotalKiB=%s\nswapFreeKiB=%s\nswap=%s\npsiSome10=%s\npsiFull10=%s\n' \
|
||||
"$swap_total" "$swap_free" "$swap_mode" "$psi_some" "$psi_full"
|
||||
printf 'kills=%s\nlastKillName=%s\nlastKillAt=%s\n' "$kills" "$last_kill_name" "$last_kill_at"
|
||||
} >"$RUN/state.tmp" && mv -f "$RUN/state.tmp" "$RUN/state"
|
||||
}
|
||||
|
||||
# Sets VICTIM/VICTIM_RSS (pages) to the largest process ranked 0 or more; returns 1 when there is none.
|
||||
VICTIM=0 VICTIM_RSS=0
|
||||
choose_victim() {
|
||||
local p pid adj rss _ best=0 best_rss=0 args=()
|
||||
sweep
|
||||
for p in "$PROC"/[0-9]*; do
|
||||
pid="${p#"$PROC"/}"
|
||||
[ "$pid" -gt 1 ] && [ "$pid" != "$$" ] || continue
|
||||
read -r adj 2>/dev/null <"$p/oom_score_adj" || continue
|
||||
[ "$adj" -ge 0 ] || continue
|
||||
read -r _ rss _ 2>/dev/null <"$p/statm" || continue
|
||||
# Kernel threads have no command line (and /proc sizes read 0, so `-s` can't tell).
|
||||
mapfile -d '' -t -n 1 args 2>/dev/null <"$p/cmdline" || continue
|
||||
[ "${#args[@]}" -gt 0 ] || continue
|
||||
if [ "$rss" -gt "$best_rss" ]; then
|
||||
best="$pid" best_rss="$rss"
|
||||
fi
|
||||
done
|
||||
# Nothing big enough to matter (64 MiB): what fills memory is protected, so leave it to the kernel.
|
||||
[ "$best" -gt 1 ] && [ "$best_rss" -ge 16384 ] || return 1
|
||||
VICTIM="$best" VICTIM_RSS="$best_rss"
|
||||
}
|
||||
|
||||
# Kills the largest process ranked 0 or more ($1: MemAvailable %). Returns 1 when there is none.
|
||||
kill_largest() {
|
||||
choose_victim || return 1
|
||||
local best="$VICTIM" best_rss="$VICTIM_RSS" name at
|
||||
name="$(proc_name "$best")"
|
||||
kill -KILL "$best" 2>/dev/null || return 1
|
||||
at="$(now_ms)"
|
||||
kills=$((kills + 1)) last_kill_name="${name:-process}" last_kill_at="$at"
|
||||
printf '%s %s %s %s %s\n' "$at" "$best" "$((best_rss * 4))" "$1" "$last_kill_name" >>"$RUN/kills"
|
||||
tail -n 50 "$RUN/kills" >"$RUN/kills.tmp" && mv -f "$RUN/kills.tmp" "$RUN/kills"
|
||||
log "killed pid=$best name=$last_kill_name rssKiB=$((best_rss * 4)) availPct=$1"
|
||||
# Shells hear it like a kernel OOM kill (exit 137); this says why where someone may look.
|
||||
logger -t orca-memguard "stopped $last_kill_name (pid $best, $((best_rss * 4)) KiB) at $1% memory available" 2>/dev/null || true
|
||||
return 0
|
||||
}
|
||||
|
||||
swap_files() { compgen -G "$SWAP_DIR/swap-*" || true; }
|
||||
add_swap() {
|
||||
[ "$swap_mode" = on-demand ] || return 1
|
||||
[ $((swap_total + SWAP_CHUNK_KIB)) -le $((mem_total * 3 / 2)) ] || return 1
|
||||
local free_kib file
|
||||
free_kib="$(df -k --output=avail "$SWAP_DIR" 2>/dev/null | tail -n 1 | tr -d ' ')"
|
||||
# Keep 4 GiB of the disk for the workspace itself.
|
||||
[ "${free_kib:-0}" -gt $((SWAP_CHUNK_KIB + 4 * 1024 * 1024)) ] || return 1
|
||||
file="$SWAP_DIR/swap-$(now_ms)"
|
||||
if fallocate -l "$((SWAP_CHUNK_KIB))K" "$file" 2>/dev/null && chmod 600 "$file" \
|
||||
&& mkswap -q "$file" >/dev/null 2>&1 && swapon "$file" 2>/dev/null; then
|
||||
log "swap on $file"
|
||||
return 0
|
||||
fi
|
||||
rm -f "$file"
|
||||
# One refusal means this kernel or disk won't swap: don't retry every tick.
|
||||
swap_mode=unsupported
|
||||
log 'swap unavailable; guard kills only'
|
||||
return 1
|
||||
}
|
||||
release_swap() {
|
||||
local file
|
||||
for file in $(swap_files | sort -r); do
|
||||
swapoff "$file" 2>/dev/null && rm -f "$file" && log "swap off $file"
|
||||
done
|
||||
}
|
||||
|
||||
run() {
|
||||
BOOT="$(boot_id)"
|
||||
mkdir -p "$RUN" "$SWAP_DIR"
|
||||
chmod 0755 "$RUN"
|
||||
chmod 0700 "$SWAP_DIR"
|
||||
# A resumed disk carries the last session's files: its kills and swap belong to that boot.
|
||||
rm -f "$RUN/kills" "$RUN/state" "$RUN"/*.tmp
|
||||
for file in $(swap_files); do
|
||||
grep -q "^$file " /proc/swaps 2>/dev/null || rm -f "$file"
|
||||
done
|
||||
printf '%s %s\n' "$BOOT" "$$" >"$RUN/pid"
|
||||
echo -1000 >/proc/self/oom_score_adj
|
||||
read_mem
|
||||
# Half of RAM for /dev/shm (tmpfs only takes pages that are written).
|
||||
mount -o remount,size=50% /dev/shm 2>/dev/null && log "/dev/shm $(df -h --output=size /dev/shm | tail -n 1 | tr -d ' ')"
|
||||
log "start mem=${mem_total}KiB"
|
||||
local tick=0 pct quiet_since=0 prev='' now
|
||||
while :; do
|
||||
read_mem
|
||||
read_psi
|
||||
pct=$((mem_total > 0 ? mem_avail * 100 / mem_total : 100))
|
||||
if [ $((tick % SWEEP_EVERY)) -eq 0 ]; then
|
||||
sweep
|
||||
fi
|
||||
if [ "$pct" -lt "$SWAP_ADD_PCT" ] && { [ "$swap_total" -eq 0 ] || [ $((swap_free * 4)) -lt "$swap_total" ]; }; then
|
||||
add_swap && read_mem
|
||||
fi
|
||||
# Swap only raises MemAvailable by paging idle memory out: a fast allocator outruns it (the
|
||||
# kernel OOM-killed a 300 MB/s hog with 1.7 GB of swap free), so the threshold ignores swap.
|
||||
if [ "$pct" -lt "$KILL_PCT" ]; then
|
||||
level=critical
|
||||
if kill_largest "$pct"; then
|
||||
write_state "$pct"
|
||||
prev="$level:$kills"
|
||||
sleep 1 # let the kernel reclaim the victim's memory before judging again
|
||||
tick=$((tick + 1))
|
||||
continue
|
||||
fi
|
||||
elif [ "$psi_full" -ge 10 ]; then
|
||||
level=critical
|
||||
elif [ "$pct" -lt "$HIGH_PCT" ] || [ "$psi_some" -ge 20 ]; then
|
||||
level=high
|
||||
else
|
||||
level=ok
|
||||
fi
|
||||
now="${EPOCHSECONDS}"
|
||||
if [ "$pct" -ge "$QUIET_PCT" ] && [ "$swap_total" -gt 0 ]; then
|
||||
[ "$quiet_since" -gt 0 ] || quiet_since="$now"
|
||||
if [ $((now - quiet_since)) -ge "$QUIET_SECONDS" ] && [ $(((swap_total - swap_free) * 2)) -lt "$mem_avail" ]; then
|
||||
release_swap
|
||||
quiet_since=0
|
||||
fi
|
||||
else
|
||||
quiet_since=0
|
||||
fi
|
||||
if [ "$level:$kills" != "$prev" ] || [ $((tick % STATE_EVERY)) -eq 0 ]; then
|
||||
write_state "$pct"
|
||||
prev="$level:$kills"
|
||||
fi
|
||||
tick=$((tick + 1))
|
||||
# Memory can go from 15% to nothing in two seconds: look five times as often when it is low.
|
||||
if [ "$level" = ok ]; then sleep "$TICK"; else sleep "$FAST_TICK"; fi
|
||||
done
|
||||
}
|
||||
|
||||
running() {
|
||||
local boot pid
|
||||
read -r boot pid <"$RUN/pid" 2>/dev/null || return 1
|
||||
[ "$boot" = "$(boot_id)" ] && [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null \
|
||||
&& grep -qa orca-memguard "/proc/$pid/cmdline" 2>/dev/null
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
start)
|
||||
[ "$(id -u)" -eq 0 ] || { echo 'orca-memguard: needs root' >&2; exit 1; }
|
||||
running && exit 0
|
||||
mkdir -p "$RUN"
|
||||
setsid nohup "$0" run </dev/null >/dev/null 2>>"$RUN/memguard.log" &
|
||||
exit 0 ;;
|
||||
run) run ;;
|
||||
sweep) sweep ;;
|
||||
victim) choose_victim && printf '%s %s %s\n' "$VICTIM" "$((VICTIM_RSS * 4))" "$(proc_name "$VICTIM")" ;;
|
||||
release-swap) release_swap ;;
|
||||
*) echo 'usage: orca-memguard start|run|sweep|victim|release-swap' >&2; exit 2 ;;
|
||||
esac
|
||||
@@ -12,6 +12,8 @@ kill -TERM "$pid"
|
||||
for _ in $(seq 1 $((WAIT_SECONDS * 10))); do
|
||||
if ! kill -0 "$pid" 2>/dev/null; then
|
||||
rm -f "$STATE/orcad.pid" "$STATE/ready.json"
|
||||
# On-demand swap files would ride the stop snapshot (2 GiB each). Best effort, bounded.
|
||||
[ -x /opt/orca/bin/orca-memguard ] && sudo -n timeout 20 /opt/orca/bin/orca-memguard release-swap >/dev/null 2>&1 || true
|
||||
exit 0
|
||||
fi
|
||||
sleep 0.1
|
||||
|
||||
@@ -36,9 +36,15 @@ anthropic_key_scrub=''
|
||||
gateway_key=''
|
||||
gemini_key=''
|
||||
codex_placeholder=''
|
||||
sign_in_inside=''
|
||||
while IFS= read -r line; do
|
||||
name="${line%%=*}" b64="${line#*=}"
|
||||
# Not for shells: the Codex account placeholder becomes ~/.codex/auth.json below.
|
||||
# Not for shells: the agents the person signs in inside this workspace (e.g. `claude,codex`).
|
||||
if [ "$name" = ORCA_SIGN_IN_INSIDE ]; then
|
||||
sign_in_inside="$(printf '%s' "$b64" | base64 -d)"
|
||||
case "$sign_in_inside" in *[!a-z,]*) sign_in_inside='' ;; esac
|
||||
fi
|
||||
if [ "$name" = ORCA_CODEX_AUTH_PLACEHOLDER ]; then
|
||||
codex_placeholder="$(printf '%s' "$b64" | base64 -d)"
|
||||
case "$codex_placeholder" in *[!A-Za-z0-9._-]*) codex_placeholder='' ;; esac
|
||||
@@ -164,7 +170,9 @@ fi
|
||||
# through them, and the agent would sit there doing nothing. Merges; never touches credentials.
|
||||
node_bin="$(command -v node || find /opt/orca/runtime/runtimes -type f -name node 2>/dev/null | head -n 1)"
|
||||
if [ -n "$node_bin" ]; then
|
||||
ORCA_PROJECT="$PROJECT" ORCA_KEY_TAIL="$anthropic_key_tail" ORCA_KEY_SCRUB="$anthropic_key_scrub" ORCA_GEMINI_KEY="$gemini_key" \
|
||||
claude_inside=''
|
||||
case ",$sign_in_inside," in *,claude,*) claude_inside=1 ;; esac
|
||||
ORCA_CLAUDE_INSIDE="$claude_inside" ORCA_PROJECT="$PROJECT" ORCA_KEY_TAIL="$anthropic_key_tail" ORCA_KEY_SCRUB="$anthropic_key_scrub" ORCA_GEMINI_KEY="$gemini_key" \
|
||||
ORCA_GATEWAY_MODEL="${gateway_key:+${anthropic_model:-$openai_model}}" "$node_bin" -e '
|
||||
const fs = require("fs"), path = require("path"), os = require("os")
|
||||
const home = os.homedir()
|
||||
@@ -178,7 +186,11 @@ const write = (file, value) => {
|
||||
const claudeFile = path.join(process.env.CLAUDE_CONFIG_DIR || home, ".claude.json")
|
||||
const c = readJson(claudeFile)
|
||||
const before = JSON.stringify(c)
|
||||
c.hasCompletedOnboarding = true
|
||||
// Signed in inside this workspace and no login here yet: leave first-run onboarding to Claude, so it opens
|
||||
// on its own sign-in screens instead of "Not logged in". Once a login exists, onboarding is done as usual.
|
||||
const claudeCredentials = path.join(process.env.CLAUDE_CONFIG_DIR || path.join(home, ".claude"), ".credentials.json")
|
||||
if (process.env.ORCA_CLAUDE_INSIDE === "1" && !fs.existsSync(claudeCredentials)) delete c.hasCompletedOnboarding
|
||||
else c.hasCompletedOnboarding = true
|
||||
c.theme = c.theme || "dark"
|
||||
c.bypassPermissionsModeAccepted = true
|
||||
c.projects = c.projects && typeof c.projects === "object" ? c.projects : {}
|
||||
|
||||
@@ -110,6 +110,10 @@ case "${ORCA_DOCKER:-}" in
|
||||
esac
|
||||
start_share_proxy
|
||||
unset ORCA_SHARE_PROXY_KEY
|
||||
# MEMGUARD: OOM-killer ranks that spare orcad and terminals, a guard that stops the largest build or
|
||||
# test before the VM stalls, on-demand swap and a /dev/shm sized for browsers. Before the
|
||||
# already-started exit so a repair start brings a dead guard back; it returns at once.
|
||||
sudo -n "$ORCA_HOME/bin/orca-memguard" start </dev/null >/dev/null 2>&1 9>&- || true
|
||||
if [ "$(cat "$STATE/boot-id" 2>/dev/null || true)" = "$boot_id" ] && [ -s "$STATE/ready.json" ] \
|
||||
&& kill -0 "$(cat "$STATE/orcad.pid" 2>/dev/null || echo 0)" 2>/dev/null; then
|
||||
print_ready
|
||||
@@ -191,16 +195,35 @@ fi
|
||||
"$ORCA_HOME/bin/orca-sandbox-env" >/dev/null || echo 'session env file unavailable; secrets apply at boot only' >&2
|
||||
unset ORCA_SECRET_ENV
|
||||
|
||||
# OpenCode's first run in a VM installs its plugin SDK into ~/.config/opencode, creates its database
|
||||
# and fetches its model list: ~2.5 s more on the first launch (5.2 s vs 2.7 s warm, n72). orca-sandbox-env
|
||||
# just wrote that config, so do the first run now, in the background at low priority, with no
|
||||
# credentials (`opencode models` does all of it and exits). Skipped once it has run on this disk.
|
||||
if [ -x "$ORCA_HOME/tools/bin/opencode" ] && [ ! -d "$HOME/.config/opencode/node_modules" ]; then
|
||||
# OpenCode installs @opencode-ai/plugin into ~/.config/opencode and, with Orca's plugin configured,
|
||||
# draws nothing until that install ends (blank ~85 s on a slow registry, n73). The image carries the
|
||||
# install (provision-golden.sh): copy it into a home that has neither node_modules nor a package.json
|
||||
# of its own, so the first launch never fetches.
|
||||
oc_seed="$ORCA_HOME/opencode-config-seed"
|
||||
oc_dir="$HOME/.config/opencode"
|
||||
if [ -d "$oc_seed/node_modules" ] && [ ! -e "$oc_dir/node_modules" ] && [ ! -e "$oc_dir/package.json" ]; then
|
||||
mkdir -p "$oc_dir" \
|
||||
&& cp -R "$oc_seed/node_modules" "$oc_dir/node_modules" \
|
||||
&& cp "$oc_seed/package.json" "$oc_seed/package-lock.json" "$oc_dir/" \
|
||||
|| echo 'opencode config seed not copied' >&2
|
||||
fi
|
||||
# The rest of OpenCode's first run (its database, the model list): ~2.5 s on the first launch
|
||||
# (5.2 s vs 2.7 s warm, n72). Done now in the background at low priority with no credentials
|
||||
# (`opencode models` does it and exits). Skipped once it has run on this disk. An image without the
|
||||
# seed (an environment snapshot from an older golden) still needs the install: then the first run
|
||||
# waits for orcad to put Orca's plugin in ~/.config/opencode/plugins (~10 s), since OpenCode only
|
||||
# finishes that install when a plugin is configured.
|
||||
if [ -x "$ORCA_HOME/tools/bin/opencode" ] && { [ ! -e "$HOME/.local/share/opencode/opencode.db" ] || [ ! -e "$oc_dir/node_modules" ]; }; then
|
||||
(
|
||||
cd "$HOME" || exit 0
|
||||
# shellcheck disable=SC2016 # expanded by the inner shell
|
||||
env -i HOME="$HOME" USER="$(id -un)" LANG=C.UTF-8 PATH="$ORCA_HOME/tools/bin:/usr/local/bin:/usr/bin:/bin" \
|
||||
OPENCODE_DISABLE_AUTOUPDATE=1 setsid nohup nice -n 19 timeout 120 \
|
||||
"$ORCA_HOME/tools/bin/opencode" models </dev/null >/dev/null 2>&1 9>&- &
|
||||
OPENCODE_DISABLE_AUTOUPDATE=1 setsid nohup nice -n 19 bash -c '
|
||||
if [ ! -e "$HOME/.config/opencode/node_modules" ]; then
|
||||
for _ in $(seq 1 120); do [ -d "$HOME/.config/opencode/plugins" ] && break; sleep 0.5; done
|
||||
fi
|
||||
exec timeout 120 "$1" models
|
||||
' opencode-first-run "$ORCA_HOME/tools/bin/opencode" </dev/null >/dev/null 2>&1 9>&- &
|
||||
)
|
||||
fi
|
||||
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
-- CREATE-GH-STEP: the create wizard's "GitHub access" step asks repo-access for every create and
|
||||
-- polls it every 2 s while the person installs the App. GitHub's installation lookups (app JWT, one
|
||||
-- per repository and owner) are cached here for a short TTL, shared by every isolate, and dropped by
|
||||
-- the installation webhooks and the setup return so a fresh install answers on the next poll.
|
||||
-- No credential or token is stored: installation ids and the owner's login only. Additive.
|
||||
CREATE TABLE github_installation_cache (
|
||||
-- '<owner>/<repo>' for a repository lookup, '<owner>/' for an owner lookup; lowercased.
|
||||
cache_key TEXT PRIMARY KEY,
|
||||
owner_key TEXT NOT NULL,
|
||||
-- NULL: GitHub answered 404 (the App isn't installed there, or not on this repository).
|
||||
installation_id INTEGER,
|
||||
github_account_id INTEGER,
|
||||
github_account_login TEXT,
|
||||
github_account_type TEXT,
|
||||
checked_at INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX github_installation_cache_owner ON github_installation_cache (owner_key);
|
||||
CREATE INDEX github_installation_cache_installation ON github_installation_cache (installation_id);
|
||||
@@ -0,0 +1,4 @@
|
||||
-- Nearest region: an environment records the region it lives in (the served region nearest whoever
|
||||
-- created it), so its builds and the workspaces booted from it stay there (a snapshot only boots in
|
||||
-- its own region). NULL: made before this was recorded, lives in the default region. Additive.
|
||||
ALTER TABLE environments ADD COLUMN region TEXT;
|
||||
@@ -0,0 +1,17 @@
|
||||
-- "Sign in inside this workspace" (agent-sources contract §2): a per-person default Source of
|
||||
-- 'in-workspace' stores nothing and injects nothing for that agent, so its own login inside each
|
||||
-- sandbox wins. SQLite can't widen a CHECK in place, so the table is rebuilt with every row kept.
|
||||
CREATE TABLE agent_sources_next (
|
||||
user_id TEXT NOT NULL,
|
||||
account_id TEXT NOT NULL,
|
||||
scope TEXT NOT NULL,
|
||||
agent TEXT NOT NULL CHECK (agent IN ('claude', 'codex')),
|
||||
source TEXT NOT NULL CHECK (source IN ('sign-in', 'key', 'in-workspace', 'accepted', 'declined')),
|
||||
updated_at INTEGER NOT NULL,
|
||||
PRIMARY KEY (user_id, scope, agent)
|
||||
);
|
||||
INSERT INTO agent_sources_next (user_id, account_id, scope, agent, source, updated_at)
|
||||
SELECT user_id, account_id, scope, agent, source, updated_at FROM agent_sources;
|
||||
DROP TABLE agent_sources;
|
||||
ALTER TABLE agent_sources_next RENAME TO agent_sources;
|
||||
CREATE INDEX agent_sources_scope ON agent_sources (scope);
|
||||
@@ -0,0 +1,56 @@
|
||||
# Egress release probe
|
||||
|
||||
`egress-probe.ts` checks sandbox egress against **dev** before a release. It goes through Orca's own
|
||||
paths: real dev workspaces created with a dev-JWK user token, with commands run in a workspace
|
||||
terminal over the shipped `orca` CLI's pairing code (the desktop's E2EE path to orcad). Every
|
||||
credential is bogus, and postman-echo.com plays the model provider.
|
||||
|
||||
Setup for a throwaway identity, all of it removed afterwards:
|
||||
|
||||
- the dev GitHub App's installation, bound for the test repo (operator route);
|
||||
- a stored GitHub identity, so member placeholders exist (the dev `member-github` route);
|
||||
- bogus personal secrets:
|
||||
- `ANTHROPIC_API_KEY`, with `ANTHROPIC_BASE_URL` set to the echo host;
|
||||
- `OPENAI_API_KEY`, with `OPENAI_BASE_URL` set to `chatgpt.com/backend-api/codex`, so its egress
|
||||
takes the ChatGPT relay.
|
||||
|
||||
The probe creates two workspaces through the desktop's create path (clone after ready):
|
||||
|
||||
- **public**: a public repo the App isn't installed on. Member placeholder, no App token (#758).
|
||||
- **app**: the App-installed test repo. App token, github.com fail-closed.
|
||||
|
||||
| Host | What must happen |
|
||||
|---|---|
|
||||
| echo host (custom base URL) | The provider sees the bogus key where the VM sent the placeholder. The other `authorization` header is unchanged, and no `vercel-forwarded-*` header or OIDC token arrives. A foreign key gets the firewall 403 `orca-egress-denied`. |
|
||||
| chatgpt.com | The placeholder reaches ChatGPT: any provider answer counts, but not the firewall 403 and not Cloudflare's edge page. A foreign bearer gets the firewall 403. |
|
||||
| api.github.com | No credential gets the firewall 403. With the workspace placeholder, GitHub answers 200. |
|
||||
| github.com | Anonymous `git ls-remote` works on **public** (#758) and is refused on **app**. The app repo clones, and the workspace's own git works. |
|
||||
| VM env | `ANTHROPIC_API_KEY` and `OPENAI_API_KEY` hold placeholders. This is checked by sha256, so the bogus values never cross the terminal. |
|
||||
|
||||
The probe takes about 1–3 minutes, deletes what it created (`--keep` leaves it), prints one JSON
|
||||
report and exits 1 on any failed expectation. It refuses any control plane other than dev.
|
||||
|
||||
## Run it locally
|
||||
|
||||
You need the Orca app installed (for its `orca` CLI) and `gcloud` access to the dev secrets.
|
||||
|
||||
```sh
|
||||
cd cloud
|
||||
pnpm install --filter @orca-cloud/sandbox-control...
|
||||
K=$(mktemp -d) # 0700; delete it afterwards
|
||||
gcloud --account <you> secrets versions access latest --project onorca-cloud-staging \
|
||||
--secret orca-sandbox-control-dev-test-jwk > "$K/jwk.json"
|
||||
gcloud --account <you> secrets versions access latest --project onorca-cloud-staging \
|
||||
--secret orca-sandbox-control-dev-operator-token > "$K/operator"
|
||||
ORCA_PROBE_DEV_JWK_FILE="$K/jwk.json" ORCA_PROBE_OPERATOR_TOKEN_FILE="$K/operator" \
|
||||
pnpm --filter @orca-cloud/sandbox-control exec tsx scripts/egress-probe.ts
|
||||
rm -rf "$K"
|
||||
```
|
||||
|
||||
`ORCA_PROBE_CLI` points the probe at another `orca` CLI. The default is the installed app's.
|
||||
|
||||
## CI
|
||||
|
||||
`.github/workflows/cloud-egress-probe.yml` runs on manual dispatch. It installs the CLI from the
|
||||
latest release's Linux package. It needs the `ORCA_DEV_TEST_JWK` and `ORCA_DEV_OPERATOR_TOKEN`
|
||||
repository secrets, and warns and skips when they are missing. None are configured today.
|
||||
@@ -0,0 +1,119 @@
|
||||
// Pure parts of the egress release probe (egress-probe.ts): the runner it starts in a workspace
|
||||
// terminal, the requests that runner sends, and what each answer must look like.
|
||||
import { createHash } from 'node:crypto'
|
||||
import { EGRESS_DENIED_BODY } from '../src/network-policy'
|
||||
|
||||
/**
|
||||
* One request from inside the workspace. A header value `$NAME` is read from the VM's env (orcad's
|
||||
* environment, as agents get it), so placeholders never cross the terminal.
|
||||
*/
|
||||
export type ProbeRequest = { name: string; url: string; method?: 'GET' | 'POST'; headers?: Record<string, string> }
|
||||
|
||||
/** What the answer must be. */
|
||||
export type Expectation =
|
||||
/** The echo host answered as the provider: these headers (by sha256) arrived, and nothing from Vercel's hop. */
|
||||
| { kind: 'swapped'; headers: Record<string, string> }
|
||||
/** The firewall's own 403 (a credential host fails closed). */
|
||||
| { kind: 'denied' }
|
||||
/** The request reached the provider itself: not the firewall's 403, not an edge block page. */
|
||||
| { kind: 'upstream' }
|
||||
|
||||
export type ProbeCase = ProbeRequest & { expect: Expectation }
|
||||
|
||||
/** What the runner reports per request: never a header value, only sha256 prefixes of the echoed ones. */
|
||||
export type Summary = {
|
||||
status: number
|
||||
/** Body is the firewall's deny body. */
|
||||
denied: boolean
|
||||
/** An HTML 403 from Cloudflare: the provider's edge refused the proxy. */
|
||||
edgeBlocked: boolean
|
||||
/** The echo host's answer: sha256 prefix of each echoed header, and Vercel hop headers seen. */
|
||||
echoed?: Record<string, string>
|
||||
leaked?: string[]
|
||||
}
|
||||
|
||||
export const sha8 = (value: string): string => createHash('sha256').update(value).digest('hex').slice(0, 8)
|
||||
|
||||
/**
|
||||
* Runs in the workspace (node, from a terminal): fetches each request and prints one short line per
|
||||
* fact, `PRB <case> <key>=<value>`, ending with `PRB-END`. Short lines survive the terminal intact.
|
||||
* The echo host (postman-echo.com) answers JSON with the request's headers under `headers`.
|
||||
*/
|
||||
export const VM_RUNNER = `
|
||||
const { createHash } = require('node:crypto')
|
||||
const cases = JSON.parse(Buffer.from(process.argv[1], 'base64').toString())
|
||||
const sha8 = (v) => createHash('sha256').update(String(v)).digest('hex').slice(0, 8)
|
||||
const out = (i, k, v) => console.log('PRB ' + i + ' ' + k + '=' + v)
|
||||
;(async () => {
|
||||
for (const [i, c] of cases.entries()) {
|
||||
const headers = {}
|
||||
for (const [k, v] of Object.entries(c.headers || {})) headers[k] = v.startsWith('$') ? process.env[v.slice(1)] || '' : v
|
||||
try {
|
||||
const r = await fetch(c.url, { method: c.method || 'GET', headers, body: c.method === 'POST' ? '{}' : undefined, redirect: 'manual', signal: AbortSignal.timeout(25000) })
|
||||
const body = await r.text()
|
||||
out(i, 's', r.status)
|
||||
out(i, 'd', body.trim() === ${JSON.stringify(EGRESS_DENIED_BODY)} ? 1 : 0)
|
||||
out(i, 'cf', r.status === 403 && /text\\/html/.test(r.headers.get('content-type') || '') && /cloudflare/i.test(r.headers.get('server') || '') ? 1 : 0)
|
||||
let echo = null
|
||||
try { echo = JSON.parse(body).headers } catch {}
|
||||
if (echo && typeof echo === 'object') {
|
||||
for (const [k, v] of Object.entries(echo)) {
|
||||
if (/^vercel-forwarded-|oidc/i.test(k)) out(i, 'leak', k)
|
||||
else if (k in (c.headers || {})) out(i, 'h.' + k, sha8(v))
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
out(i, 's', 0)
|
||||
}
|
||||
}
|
||||
console.log('PRB-END')
|
||||
})()
|
||||
`
|
||||
|
||||
export function parseRunnerOutput(text: string, count: number): Summary[] {
|
||||
const summaries: Summary[] = Array.from({ length: count }, () => ({ status: 0, denied: false, edgeBlocked: false }))
|
||||
for (const match of text.matchAll(/PRB (\d+) ([a-z.-]+)=(\S+)/g)) {
|
||||
const summary = summaries[Number(match[1])]
|
||||
const [key, value] = [match[2]!, match[3]!]
|
||||
if (!summary) {
|
||||
continue
|
||||
}
|
||||
if (key === 's') summary.status = Number(value)
|
||||
else if (key === 'd') summary.denied = value === '1'
|
||||
else if (key === 'cf') summary.edgeBlocked = value === '1'
|
||||
else if (key === 'leak') summary.leaked = [...(summary.leaked ?? []), value]
|
||||
else if (key.startsWith('h.')) summary.echoed = { ...summary.echoed, [key.slice(2)]: value }
|
||||
}
|
||||
return summaries
|
||||
}
|
||||
|
||||
/** Problems with one answer; empty when it is what the case expects. Never quotes a header value. */
|
||||
export function judge(probe: ProbeCase, answer: Summary): string[] {
|
||||
const { expect } = probe
|
||||
const denied = answer.status === 403 && answer.denied
|
||||
if (expect.kind === 'denied') {
|
||||
return denied ? [] : [`${probe.name}: expected the firewall's 403 ${EGRESS_DENIED_BODY}, got ${answer.status}`]
|
||||
}
|
||||
if (expect.kind === 'upstream') {
|
||||
if (denied) return [`${probe.name}: refused by the firewall`]
|
||||
if (answer.edgeBlocked) return [`${probe.name}: the provider's edge refused the egress proxy (Cloudflare 403 page)`]
|
||||
return answer.status === 0 ? [`${probe.name}: no answer`] : []
|
||||
}
|
||||
if (answer.status !== 200 || !answer.echoed) {
|
||||
return [`${probe.name}: expected the echo host's answer, got ${answer.status}${denied ? ' (denied)' : ''}`]
|
||||
}
|
||||
const problems = (answer.leaked ?? []).map((name) => `${probe.name}: the provider saw ${name}`)
|
||||
for (const [name, value] of Object.entries(expect.headers)) {
|
||||
if (answer.echoed[name] !== sha8(value)) {
|
||||
problems.push(`${probe.name}: header ${name} ${answer.echoed[name] ? 'arrived with another value' : 'did not arrive'}`)
|
||||
}
|
||||
}
|
||||
return problems
|
||||
}
|
||||
|
||||
/** The terminal line that runs the cases: the runner and its input both base64, one shell word each. */
|
||||
export function runnerCommand(cases: readonly ProbeRequest[]): string {
|
||||
const runner = Buffer.from(VM_RUNNER).toString('base64')
|
||||
const input = Buffer.from(JSON.stringify(cases.map(({ name: _name, ...request }) => request))).toString('base64')
|
||||
return `node -e "$(echo ${runner} | base64 -d)" ${input}`
|
||||
}
|
||||
@@ -0,0 +1,268 @@
|
||||
// Release probe for sandbox egress, against DEV only, through Orca's own paths: real dev workspaces
|
||||
// created with a dev-JWK user token, driven from a terminal over the `orca` CLI's pairing code (the
|
||||
// desktop's E2EE path). Bogus tokens only; postman-echo.com plays the model provider.
|
||||
// pnpm --filter @orca-cloud/sandbox-control exec tsx scripts/egress-probe.ts [--keep]
|
||||
//
|
||||
// A throwaway identity gets: the dev GitHub App installation bound for one test repo (operator), a
|
||||
// stored GitHub identity (the dev member-github route, so member placeholders exist), and bogus
|
||||
// personal secrets ANTHROPIC_API_KEY (base URL: the echo host) and OPENAI_API_KEY (base URL:
|
||||
// chatgpt.com, so its egress takes the ChatGPT relay). Two workspaces:
|
||||
// public a public repo the App is not installed on: member placeholder, no App token (#758)
|
||||
// app the App-installed test repo: App token, github.com fail-closed
|
||||
// Checks (every one names the host it is about):
|
||||
// - api.anthropic-compatible echo host: the provider sees the bogus key where the VM sent the
|
||||
// placeholder, the other auth header unchanged, nothing from Vercel's hop; a foreign key 403s.
|
||||
// - chatgpt.com: the placeholder reaches ChatGPT (any provider answer; not the firewall's 403 and
|
||||
// not Cloudflare's edge page); a foreign bearer 403s.
|
||||
// - api.github.com: no credential 403s; with the workspace placeholder GitHub answers 200.
|
||||
// - github.com: anonymous `git ls-remote` works on `public` (#758) and 403s on `app`; the
|
||||
// workspace's own git (placeholder extraheader) works on `app`.
|
||||
// - the VM holds placeholders, never the bogus values.
|
||||
//
|
||||
// Inputs (files, never printed):
|
||||
// ORCA_PROBE_DEV_JWK_FILE dev test signing key (SM: orca-sandbox-control-dev-test-jwk)
|
||||
// ORCA_PROBE_OPERATOR_TOKEN_FILE dev operator token (SM: orca-sandbox-control-dev-operator-token)
|
||||
// ORCA_PROBE_CONTROL_URL default: the dev sandbox-control Worker (anything else is refused)
|
||||
// ORCA_PROBE_CLI default: the installed Orca app's `orca` CLI
|
||||
// Prints one JSON report; exits 1 on any failed expectation. See scripts/README.md.
|
||||
import { execFile } from 'node:child_process'
|
||||
import { randomBytes } from 'node:crypto'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { SignJWT, importJWK, type JWK } from 'jose'
|
||||
import { judge, parseRunnerOutput, runnerCommand, sha8, type ProbeCase } from './egress-probe-checks'
|
||||
|
||||
const CONTROL_URL = process.env.ORCA_PROBE_CONTROL_URL ?? 'https://orca-sandbox-control-dev.lovecast.workers.dev'
|
||||
const ORCA_CLI = process.env.ORCA_PROBE_CLI ?? '/Applications/Orca.app/Contents/Resources/bin/orca'
|
||||
/** The dev GitHub App is installed on this repo. */
|
||||
const APP_REPO = process.env.ORCA_PROBE_APP_REPO ?? 'https://github.com/nwparker/orca-sandbox-push-test'
|
||||
/** Public, and the dev App is not installed on its owner. */
|
||||
const PUBLIC_REPO = 'https://github.com/octocat/Hello-World'
|
||||
const ECHO = 'https://postman-echo.com'
|
||||
|
||||
if (!/^https:\/\/[a-z0-9-]+-dev\.lovecast\.workers\.dev$|^http:\/\/(127\.0\.0\.1|localhost)(:\d+)?$/.test(CONTROL_URL)) {
|
||||
throw new Error(`refusing a non-dev control plane: ${CONTROL_URL}`)
|
||||
}
|
||||
|
||||
const keep = process.argv.includes('--keep')
|
||||
const runId = randomBytes(4).toString('hex')
|
||||
const sub = `egress-probe-${runId}`
|
||||
const accountId = `dev:${sub}`
|
||||
const fileSecret = (env: string): string => {
|
||||
const file = process.env[env]
|
||||
if (!file) {
|
||||
throw new Error(`set ${env}`)
|
||||
}
|
||||
return readFileSync(file, 'utf8').trim()
|
||||
}
|
||||
|
||||
async function userToken(): Promise<string> {
|
||||
const jwk = JSON.parse(fileSecret('ORCA_PROBE_DEV_JWK_FILE')) as { kid: string; private: JWK; issuers?: { sandbox?: string }; audiences?: { sandbox?: string } }
|
||||
return new SignJWT({ sub, purpose: 'sandbox' })
|
||||
.setProtectedHeader({ alg: 'ES256', kid: jwk.kid })
|
||||
.setIssuer(jwk.issuers?.sandbox ?? 'orca-sandbox-dev-test')
|
||||
.setAudience(jwk.audiences?.sandbox ?? 'orca-sandbox')
|
||||
.setIssuedAt()
|
||||
.setExpirationTime('45m')
|
||||
.sign(await importJWK(jwk.private, 'ES256'))
|
||||
}
|
||||
|
||||
type Reply = { status: number; body: Record<string, any> | null }
|
||||
|
||||
function client(bearer: string) {
|
||||
return async (method: string, path: string, body?: unknown): Promise<Reply> => {
|
||||
const response = await fetch(`${CONTROL_URL}${path}`, {
|
||||
method,
|
||||
// clone-progress: the desktop's create, whose clone runs after ready (GET /v1/sandboxes/:id/clone).
|
||||
headers: { authorization: `Bearer ${bearer}`, 'content-type': 'application/json', 'x-orca-sandbox-features': 'clone-progress' },
|
||||
body: body === undefined ? undefined : JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(300_000)
|
||||
})
|
||||
return { status: response.status, body: (await response.json().catch(() => null)) as Record<string, any> | null }
|
||||
}
|
||||
}
|
||||
|
||||
/** The shipped CLI over a workspace's pairing code; the code never reaches logs or the report. */
|
||||
function orca(pairingCode: string, args: string[], timeoutMs = 60_000): Promise<any> {
|
||||
return new Promise((resolve) => {
|
||||
execFile(ORCA_CLI, [...args, '--json', '--pairing-code', pairingCode], { timeout: timeoutMs, maxBuffer: 8 << 20, env: { ...process.env, ORCA_PAIRING_CODE: '' } }, (_error, stdout) => {
|
||||
try {
|
||||
resolve(JSON.parse(stdout))
|
||||
} catch {
|
||||
resolve(null)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/** Every string in a CLI result, one per line (terminal output may come as a string, lines or chunks). */
|
||||
function textOf(value: unknown): string {
|
||||
if (typeof value === 'string') return value
|
||||
if (Array.isArray(value)) return value.map(textOf).join('\n')
|
||||
if (value && typeof value === 'object') return Object.values(value).map(textOf).join('\n')
|
||||
return ''
|
||||
}
|
||||
|
||||
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))
|
||||
|
||||
/** Runs one line in a fresh workspace terminal and returns the output once `PRB-END` shows. */
|
||||
async function inTerminal(pairingCode: string, projectRoot: string, line: string): Promise<string> {
|
||||
await orca(pairingCode, ['repo', 'add', '--path', projectRoot, '--kind', 'git'])
|
||||
const created = await orca(pairingCode, ['terminal', 'create', '--worktree', `path:${projectRoot}`, '--title', 'egress-probe'])
|
||||
const handle: string | undefined = created?.result?.terminal?.handle ?? created?.result?.handle ?? created?.result?.terminal?.id
|
||||
if (!handle) {
|
||||
throw new Error('could not open a workspace terminal')
|
||||
}
|
||||
// `clear` first: the read must only see this run's lines.
|
||||
await orca(pairingCode, ['terminal', 'send', '--terminal', handle, '--text', `clear; ${line}`, '--enter'])
|
||||
const deadline = Date.now() + 180_000
|
||||
while (Date.now() < deadline) {
|
||||
const read = await orca(pairingCode, ['terminal', 'read', '--terminal', handle])
|
||||
const text = textOf(read?.result)
|
||||
// The typed line is base64 past its first words, so PRB-END only shows once the runner prints it.
|
||||
if (text.includes('PRB-END')) {
|
||||
return text
|
||||
}
|
||||
await sleep(1500)
|
||||
}
|
||||
throw new Error('the probe did not finish in the workspace terminal')
|
||||
}
|
||||
|
||||
/** Git and env facts, one short line each, then the HTTP cases. */
|
||||
function factsScript(gitChecks: Record<string, string>, bogus: Record<string, string>): string {
|
||||
const lines = Object.entries(gitChecks).map(([name, command]) => `GIT_TERMINAL_PROMPT=0 timeout 60 ${command} >/dev/null 2>&1; echo "PRB ${name} rc=$?"`)
|
||||
// The VM must hold placeholders: compare by sha256 so the bogus values never cross the terminal.
|
||||
for (const [name, digest] of Object.entries(bogus)) {
|
||||
// The verdict word is a variable: the typed command (echoed by the terminal) must not contain it.
|
||||
lines.push(`v=placeholder; [ "$(printf %s "\${${name}:-}" | sha256sum | cut -c1-8)" = ${digest} ] && v=bogus; echo "PRB env.${name} holds=$v"`)
|
||||
}
|
||||
return lines.join('; ')
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const failures: string[] = []
|
||||
const report: Record<string, unknown> = { runId }
|
||||
const cleanup: (() => Promise<unknown>)[] = []
|
||||
const started = Date.now()
|
||||
console.error(`[egress-probe] run ${runId}: throwaway identity ${sub}`)
|
||||
const user = client(await userToken())
|
||||
const operator = client(fileSecret('ORCA_PROBE_OPERATOR_TOKEN_FILE'))
|
||||
const bogusAnthropic = `sk-ant-api03-orcaprobe${randomBytes(16).toString('hex')}`
|
||||
const bogusOpenAI = `sk-proj-orcaprobe${randomBytes(16).toString('hex')}`
|
||||
try {
|
||||
// Identity: the App bound for the test repo, and a stored GitHub identity (member placeholders).
|
||||
const bound = await operator('POST', '/v1/internal/github/bindings', { repoUrl: APP_REPO, accountId })
|
||||
const installationId: number | undefined = bound.body?.bound?.installationId
|
||||
if (bound.status !== 200 || !installationId) {
|
||||
throw new Error(`bind the dev App: ${bound.status} ${bound.body?.error ?? ''}`)
|
||||
}
|
||||
cleanup.push(() => operator('DELETE', '/v1/internal/github/bindings', { installationId, accountId }))
|
||||
const member = await operator('POST', '/v1/internal/dev/member-github', { accountId, userId: sub, repoUrl: APP_REPO, login: 'egress-probe', githubUserId: 1, name: 'egress probe' })
|
||||
if (member.status !== 200) {
|
||||
throw new Error(`store the probe's GitHub identity: ${member.status} ${member.body?.error ?? ''}`)
|
||||
}
|
||||
for (const [name, value] of [
|
||||
['ANTHROPIC_API_KEY', bogusAnthropic],
|
||||
['ANTHROPIC_BASE_URL', ECHO],
|
||||
['OPENAI_API_KEY', bogusOpenAI],
|
||||
['OPENAI_BASE_URL', 'https://chatgpt.com/backend-api/codex']
|
||||
] as const) {
|
||||
const put = await user('PUT', `/v1/secrets/personal/${name}`, { value })
|
||||
cleanup.push(() => user('DELETE', `/v1/secrets/personal/${name}`))
|
||||
if (put.status !== 200) {
|
||||
throw new Error(`PUT secret ${name}: ${put.status} ${put.body?.error ?? ''}`)
|
||||
}
|
||||
}
|
||||
const create = async (label: string, repo: string) => {
|
||||
console.error(`[egress-probe] creating the ${label} workspace (${sub}-${label})`)
|
||||
const reply = await user('POST', '/v1/sandboxes', { workspaceId: `${sub}-${label}`, machineSize: 'standard', repo: { url: repo }, useEnvironment: false })
|
||||
const id: string | undefined = reply.body?.sandbox?.id
|
||||
if (id) {
|
||||
cleanup.unshift(() => user('DELETE', `/v1/sandboxes/${id}`))
|
||||
}
|
||||
if (reply.status !== 200 || !reply.body?.connection?.pairingCode) {
|
||||
throw new Error(`create the ${label} workspace: ${reply.status} ${reply.body?.error ?? ''}`)
|
||||
}
|
||||
console.error(`[egress-probe] ${label} workspace ${id}`)
|
||||
// The desktop's path: the clone runs after ready, and the desktop polls it.
|
||||
let clone = 'unknown'
|
||||
for (const until = Date.now() + 240_000; Date.now() < until; await sleep(3_000)) {
|
||||
clone = String((await user('GET', `/v1/sandboxes/${id}/clone`)).body?.clone?.state ?? 'unknown')
|
||||
if (clone.startsWith('done') || clone.startsWith('failed')) break
|
||||
}
|
||||
clones[label] = clone
|
||||
return { id: id!, pairingCode: reply.body.connection.pairingCode as string, projectRoot: reply.body.connection.projectRoot as string }
|
||||
}
|
||||
const clones: Record<string, string> = {}
|
||||
const [publicBox, appBox] = await Promise.all([create('public', PUBLIC_REPO), create('app', APP_REPO)])
|
||||
report.workspaces = { public: publicBox.id, app: appBox.id }
|
||||
|
||||
const anthropic: ProbeCase[] = [
|
||||
{
|
||||
name: 'echo host (ANTHROPIC_BASE_URL): placeholder swapped, other auth untouched',
|
||||
url: `${ECHO}/post`,
|
||||
method: 'POST',
|
||||
headers: { 'x-api-key': '$ANTHROPIC_API_KEY', authorization: 'Bearer probe-keep-bearer' },
|
||||
expect: { kind: 'swapped', headers: { 'x-api-key': bogusAnthropic, authorization: 'Bearer probe-keep-bearer' } }
|
||||
},
|
||||
{ name: 'echo host: foreign key fails closed', url: `${ECHO}/post`, method: 'POST', headers: { 'x-api-key': 'sk-ant-probe-foreign' }, expect: { kind: 'denied' } },
|
||||
{
|
||||
name: 'chatgpt.com: placeholder reaches ChatGPT (relay)',
|
||||
url: 'https://chatgpt.com/backend-api/codex/models',
|
||||
headers: { authorization: '$PROBE_OPENAI_BEARER' },
|
||||
expect: { kind: 'upstream' }
|
||||
},
|
||||
{ name: 'chatgpt.com: foreign bearer fails closed', url: 'https://chatgpt.com/backend-api/codex/models', headers: { authorization: 'Bearer probe-foreign' }, expect: { kind: 'denied' } },
|
||||
{ name: 'api.github.com: no credential fails closed', url: 'https://api.github.com/rate_limit', expect: { kind: 'denied' } }
|
||||
]
|
||||
const appCases: ProbeCase[] = [
|
||||
{ name: 'api.github.com: workspace placeholder swapped (GitHub answers 200)', url: 'https://api.github.com/rate_limit', headers: { authorization: '$PROBE_GH_BEARER' }, expect: { kind: 'upstream' } },
|
||||
{ name: 'github.com: anonymous fails closed with an App token', url: `${APP_REPO}.git/info/refs?service=git-upload-pack`, expect: { kind: 'denied' } }
|
||||
]
|
||||
const digests = { ANTHROPIC_API_KEY: sha8(bogusAnthropic), OPENAI_API_KEY: sha8(bogusOpenAI) }
|
||||
// The runner reads whole header values from env (`$NAME`), so the Bearer forms are built here.
|
||||
const prelude = 'export PROBE_OPENAI_BEARER="Bearer $OPENAI_API_KEY" PROBE_GH_BEARER="Bearer $GH_TOKEN"; '
|
||||
console.error('[egress-probe] running the checks in both workspaces')
|
||||
const runs = await Promise.all([
|
||||
inTerminal(publicBox.pairingCode, publicBox.projectRoot, `${prelude}${factsScript({ 'git.public-anonymous': `git ls-remote ${PUBLIC_REPO} HEAD` }, digests)}; ${runnerCommand(anthropic)}`),
|
||||
inTerminal(appBox.pairingCode, appBox.projectRoot, `${prelude}${factsScript({ 'git.app-anonymous': `git -c http.extraheader= -c http.https://github.com/.extraheader= ls-remote ${APP_REPO} HEAD`, 'git.app-workspace': 'git ls-remote origin HEAD' }, {})}; ${runnerCommand(appCases)}`)
|
||||
])
|
||||
const results: Record<string, unknown>[] = []
|
||||
const check = (workspace: string, name: string, ok: boolean, detail: string) => {
|
||||
results.push({ workspace, check: name, ok })
|
||||
if (!ok) {
|
||||
failures.push(`[${workspace}] ${name}: ${detail}`)
|
||||
}
|
||||
}
|
||||
for (const [workspace, text, cases] of [['public', runs[0], anthropic], ['app', runs[1], appCases]] as const) {
|
||||
const summaries = parseRunnerOutput(text, cases.length)
|
||||
cases.forEach((probeCase, i) => {
|
||||
const problems = judge(probeCase, summaries[i]!)
|
||||
check(workspace, probeCase.name, problems.length === 0, problems.join('; '))
|
||||
})
|
||||
}
|
||||
const rc = (text: string, name: string) => new RegExp(`PRB ${name.replaceAll('.', '\\.')} rc=(\\d+)`).exec(text)?.[1]
|
||||
check('public', 'github.com: anonymous git ls-remote of a public repo works (#758)', rc(runs[0], 'git.public-anonymous') === '0', `rc=${rc(runs[0], 'git.public-anonymous')}`)
|
||||
check('app', 'github.com: anonymous git ls-remote is refused with an App token', rc(runs[1], 'git.app-anonymous') !== '0' && rc(runs[1], 'git.app-anonymous') !== undefined, `rc=${rc(runs[1], 'git.app-anonymous')}`)
|
||||
check('app', "github.com: the workspace's own git (placeholder) works", rc(runs[1], 'git.app-workspace') === '0', `rc=${rc(runs[1], 'git.app-workspace')}`)
|
||||
for (const [workspace, state] of Object.entries(clones)) {
|
||||
check(workspace, 'github.com: the workspace repo clones', state === 'done', `clone ${state}`)
|
||||
}
|
||||
for (const name of Object.keys(digests)) {
|
||||
check('public', `the VM holds a placeholder for ${name}`, new RegExp(`PRB env\\.${name} holds=placeholder`).test(runs[0]), 'the VM env holds the bogus value itself')
|
||||
}
|
||||
report.checks = results
|
||||
} catch (error) {
|
||||
failures.push(`probe error: ${error instanceof Error ? error.message : String(error)}`)
|
||||
} finally {
|
||||
if (!keep) {
|
||||
for (const step of cleanup) {
|
||||
await step().catch((error: unknown) => console.error(`[egress-probe] cleanup: ${String(error)}`))
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log(JSON.stringify({ ok: failures.length === 0, failures, ...report, ms: Date.now() - started }, null, 2))
|
||||
console.error(failures.length === 0 ? '[egress-probe] OK' : `[egress-probe] FAILED:\n ${failures.join('\n ')}`)
|
||||
process.exitCode = failures.length === 0 ? 0 : 1
|
||||
}
|
||||
|
||||
void main()
|
||||
@@ -100,7 +100,7 @@ export const claudeAccount: AccountProvider = {
|
||||
type: 'error',
|
||||
error: {
|
||||
type: 'authentication_error',
|
||||
message: 'Your Claude sign-in for Orca Cloud expired. Renew it in Orca: Settings → Agent accounts.'
|
||||
message: 'Your Claude sign-in for Orca Cloud expired. Renew it in Orca: Settings → Cloud agent sign-ins.'
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
// Agent Source per person (agent-sources contract §2): "Your sign-in" or "API key", per agent, as a
|
||||
// default plus an optional per-workspace override, and the one-time "Use my sign-in" offer answer (D8).
|
||||
// A default can also be "Sign in inside this workspace" ('in-workspace'): Orca stores and injects nothing
|
||||
// for that agent (no account placeholder, no key), and its own login inside each sandbox wins.
|
||||
// GET /v1/agent-sources[?workspaceId=<sandbox id>]
|
||||
// PUT /v1/agent-sources {agent, source: 'sign-in'|'key'|null, workspaceId?}
|
||||
// PUT /v1/agent-sources {agent, source: 'sign-in'|'key'|'in-workspace'|null, workspaceId?}
|
||||
// PUT /v1/agent-sources/offer {agent, answer: 'accepted'|'declined'}
|
||||
// A sandbox applies its OWNER's effective source: with 'sign-in', that agent's credential names are left
|
||||
// out of the session env (and so out of egress routes, the allowlist and gateway derivation), letting
|
||||
@@ -23,13 +25,16 @@ const OFFER_SCOPE = 'offer'
|
||||
type SourceRow = { scope: string; agent: CredentialAgent; source: string }
|
||||
|
||||
const AgentSchema = z.enum(['claude', 'codex'])
|
||||
const IN_WORKSPACE = 'in-workspace'
|
||||
export const PutSourceSchema = z
|
||||
.object({
|
||||
agent: AgentSchema,
|
||||
source: z.enum(['sign-in', 'key']).nullable(),
|
||||
source: z.enum(['sign-in', 'key', IN_WORKSPACE]).nullable(),
|
||||
workspaceId: z.string().min(1).max(128).optional()
|
||||
})
|
||||
.strict()
|
||||
// Signing in inside the workspace is a per-person default, never a one-workspace override.
|
||||
.refine((body) => !(body.source === IN_WORKSPACE && body.workspaceId))
|
||||
export const PutOfferSchema = z.object({ agent: AgentSchema, answer: z.enum(['accepted', 'declined']) }).strict()
|
||||
|
||||
export class AgentSourceError extends Error {
|
||||
@@ -60,12 +65,15 @@ function byAgent(rows: SourceRow[], scope: string): Record<CredentialAgent, Agen
|
||||
return out
|
||||
}
|
||||
|
||||
/** Agents whose owner chose "Your sign-in" for this sandbox (override first, then default). */
|
||||
export async function signInAgentsFor(db: D1Database, row: Pick<SandboxRow, 'id' | 'user_id'>): Promise<Set<CredentialAgent>> {
|
||||
const rows = await rowsFor(db, row.user_id, [row.id, DEFAULT_SCOPE])
|
||||
const workspace = byAgent(rows, row.id)
|
||||
const defaults = byAgent(rows, DEFAULT_SCOPE)
|
||||
return new Set(CREDENTIAL_AGENTS.filter((agent) => (workspace[agent] ?? defaults[agent]) === 'sign-in'))
|
||||
/** Agents this person signs in inside each workspace (their default is 'in-workspace'). */
|
||||
function inWorkspaceAgents(rows: SourceRow[]): Record<CredentialAgent, boolean> {
|
||||
const out = { claude: false, codex: false } as Record<CredentialAgent, boolean>
|
||||
for (const row of rows) {
|
||||
if (row.scope === DEFAULT_SCOPE && row.source === IN_WORKSPACE) {
|
||||
out[row.agent] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** Drops the credential names of agents on "Your sign-in" (after preset derivation, so derived ones go too). */
|
||||
@@ -111,6 +119,7 @@ export async function agentSourcesView(
|
||||
const subject = row ? row.user_id : caller.userId
|
||||
const rows = await rowsFor(db, subject, [DEFAULT_SCOPE, OFFER_SCOPE, ...(row ? [row.id] : [])])
|
||||
const defaults = byAgent(rows, DEFAULT_SCOPE)
|
||||
const inWorkspace = inWorkspaceAgents(rows)
|
||||
const workspace = row ? byAgent(rows, row.id) : null
|
||||
const effective = await effectiveSecrets(db, caller, row?.repo_url ?? null)
|
||||
const covered = coveredByKey(new Set(effective.map((secret) => secret.name)))
|
||||
@@ -135,6 +144,10 @@ export async function agentSourcesView(
|
||||
if (defaults[agent]) {
|
||||
return [agent, { source: defaults[agent], origin: 'default' }]
|
||||
}
|
||||
// `source` keeps its two values for older desktops; `inWorkspace` says what this one is.
|
||||
if (inWorkspace[agent]) {
|
||||
return [agent, { source: null, origin: 'default' }]
|
||||
}
|
||||
return [agent, { source: covered[agent] ? 'key' : usable[agent] ? 'sign-in' : null, origin: 'implicit' }]
|
||||
})
|
||||
)
|
||||
@@ -145,7 +158,7 @@ export async function agentSourcesView(
|
||||
}
|
||||
// D13: personal accounts never apply in a workspace shared with teammates.
|
||||
const sharedWorkspace = row ? isSharedWorkspace(row) : false
|
||||
return { defaults, workspace, effective: resolved, coveredByKey: covered, signInOffer, owner, sharedWorkspace }
|
||||
return { defaults, inWorkspace, workspace, effective: resolved, coveredByKey: covered, signInOffer, owner, sharedWorkspace }
|
||||
}
|
||||
|
||||
async function upsert(db: D1Database, caller: SandboxCaller, scope: string, agent: CredentialAgent, source: string | null, now: number) {
|
||||
@@ -200,11 +213,12 @@ export async function deleteAgentSourcesForSandbox(db: D1Database, sandboxId: st
|
||||
/**
|
||||
* How each agent authenticates in this session (D10):
|
||||
* - 'account': the owner's Orca-account credential through an egress placeholder (keys for that agent dropped);
|
||||
* - 'inside': the owner signs in inside each workspace ('in-workspace'): keys dropped, the agent's own login;
|
||||
* - 'omit': kill-switch fallback (Source sign-in, provider off): keys dropped, the agent's own login in the sandbox;
|
||||
* - 'keep': API keys as resolved (precedence), the default.
|
||||
* Source: explicit (workspace, then default), else implicit: a covering key wins, else a usable account.
|
||||
*/
|
||||
export type AgentMode = 'account' | 'omit' | 'keep'
|
||||
export type AgentMode = 'account' | 'inside' | 'omit' | 'keep'
|
||||
export async function agentModesFor(
|
||||
db: D1Database,
|
||||
row: Pick<SandboxRow, 'id' | 'user_id' | 'visibility' | 'ownership_transferred_at'>,
|
||||
@@ -219,9 +233,16 @@ export async function agentModesFor(
|
||||
const rows = await rowsFor(db, row.user_id, [row.id, DEFAULT_SCOPE])
|
||||
const workspace = byAgent(rows, row.id)
|
||||
const defaults = byAgent(rows, DEFAULT_SCOPE)
|
||||
const inWorkspace = inWorkspaceAgents(rows)
|
||||
const covered = coveredByKey(new Set(Object.keys(env)))
|
||||
const modes = {} as Record<CredentialAgent, AgentMode>
|
||||
for (const agent of CREDENTIAL_AGENTS) {
|
||||
if (!workspace[agent] && inWorkspace[agent]) {
|
||||
// Nothing of Orca's for this agent: no placeholder, no key, no egress route, so the login the person
|
||||
// makes inside the sandbox (stored in its home, kept across sleep) reaches the vendor untouched.
|
||||
modes[agent] = 'inside'
|
||||
continue
|
||||
}
|
||||
const explicit = workspace[agent] ?? defaults[agent]
|
||||
const provider = accountProvider(agent)
|
||||
const enabled = options.providerEnabled[agent] && provider !== undefined
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
// it on the current image. Unused for 14 days it is deleted with its snapshots, unless an admin edited
|
||||
// it (that makes it an ordinary 'org' environment).
|
||||
import type { SandboxCaller } from './auth'
|
||||
import { listBuilds, startBuild, TERMINAL, type BuildConfig, type BuildDeps } from './environment-builds'
|
||||
import { environmentRegion, listBuilds, startBuild, TERMINAL, type BuildConfig, type BuildDeps } from './environment-builds'
|
||||
import { canonicalRepoUrl, parseRepos, repoDir, type EnvironmentRow } from './environments'
|
||||
import { normalizeRepo } from './secrets'
|
||||
import type { SandboxProviderApi } from './sandbox-provider'
|
||||
@@ -52,16 +52,22 @@ export type AutoEnvironmentOutcome = 'created' | 'rebuilt' | 'exists' | 'skipped
|
||||
* A create of `repoUrl` booted without an environment: make (or retry building) the account's automatic
|
||||
* environment for it. Never throws; the create it rides behind already succeeded.
|
||||
*/
|
||||
export async function ensureAutoEnvironment(deps: BuildDeps, accountId: string, repoUrl: string): Promise<AutoEnvironmentOutcome> {
|
||||
export async function ensureAutoEnvironment(
|
||||
deps: BuildDeps,
|
||||
accountId: string,
|
||||
repoUrl: string,
|
||||
/** The create's region: the environment lives (and builds) there, so the next create of the repo lands there too. */
|
||||
region?: string
|
||||
): Promise<AutoEnvironmentOutcome> {
|
||||
try {
|
||||
return await ensure(deps, accountId, repoUrl)
|
||||
return await ensure(deps, accountId, repoUrl, region)
|
||||
} catch (error) {
|
||||
log('auto_environment_failed', { error: String(error).slice(0, 200) })
|
||||
return 'skipped'
|
||||
}
|
||||
}
|
||||
|
||||
async function ensure(deps: BuildDeps, accountId: string, repoUrl: string): Promise<AutoEnvironmentOutcome> {
|
||||
async function ensure(deps: BuildDeps, accountId: string, repoUrl: string, region?: string): Promise<AutoEnvironmentOutcome> {
|
||||
const url = canonicalRepoUrl(repoUrl)
|
||||
const wanted = normalizeRepo(url)
|
||||
if (!url || !wanted) {
|
||||
@@ -90,11 +96,11 @@ async function ensure(deps: BuildDeps, accountId: string, repoUrl: string): Prom
|
||||
await deps.db
|
||||
.prepare(
|
||||
`INSERT OR IGNORE INTO environments (id, account_id, name, repos, install_script, env_var_names, machine_size,
|
||||
revision, next_build_number, created_by, updated_by, created_at, updated_at, kind, last_used_at)
|
||||
VALUES (?, ?, ?, ?, '', '[]', 'standard', 1, 1, ?, ?, ?, ?, 'auto', ?)`
|
||||
revision, next_build_number, created_by, updated_by, created_at, updated_at, kind, last_used_at, region)
|
||||
VALUES (?, ?, ?, ?, '', '[]', 'standard', 1, 1, ?, ?, ?, ?, 'auto', ?, ?)`
|
||||
)
|
||||
.bind(id, accountId, repoDir(url), JSON.stringify([{ url, defaultBranch: null }]), AUTO_ENVIRONMENT_USER,
|
||||
AUTO_ENVIRONMENT_USER, now, now, now)
|
||||
AUTO_ENVIRONMENT_USER, now, now, now, environmentRegion(deps.config, region))
|
||||
.run()
|
||||
const environment = await deps.db
|
||||
.prepare('SELECT * FROM environments WHERE id = ? AND deleted_at IS NULL')
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
// CODEX-RELAY: chatgpt.com refuses Cloudflare Workers' egress (a 403 "Unable to load site" page), so
|
||||
// the egress proxy sends its chatgpt.com requests through the ChatGPT relay on Cloud Run
|
||||
// (cloud/apps/chatgpt-relay), which ChatGPT answers. The Worker still decides everything: which
|
||||
// credential, which paths. The relay pins the paths again and only ever talks to chatgpt.com.
|
||||
// Auth: Cloud Run IAM. The Worker signs in as one service account that holds only run.invoker on the
|
||||
// relay, and sends a Google ID token for the relay's URL in X-Serverless-Authorization, so the
|
||||
// request's own Authorization (the ChatGPT token) passes through untouched.
|
||||
// CHATGPT_RELAY_URL https://<service>.run.app (unset: chatgpt.com directly, as before)
|
||||
// CHATGPT_RELAY_SERVICE_ACCOUNT (secret) service-account JSON with run.invoker on that service only
|
||||
import { SignJWT, importPKCS8 } from 'jose'
|
||||
import { z } from 'zod'
|
||||
|
||||
const TOKEN_URL = 'https://oauth2.googleapis.com/token'
|
||||
export const RELAYED_HOSTS: ReadonlySet<string> = new Set(['chatgpt.com'])
|
||||
const ServiceAccount = z.looseObject({ client_email: z.string().min(3).max(320), private_key: z.string().min(100) })
|
||||
const IdTokenResponse = z.looseObject({ id_token: z.string().min(20) })
|
||||
|
||||
export class RelayAuthError extends Error {}
|
||||
|
||||
/** A Google ID token for `audience`, minted from the service account, cached until 5 minutes before expiry. */
|
||||
export function createIdTokenSource(config: { serviceAccountJson: string; audience: string; fetch?: typeof fetch; now?: () => number }) {
|
||||
const doFetch = config.fetch ?? fetch
|
||||
const now = config.now ?? Date.now
|
||||
let cached: { value: string; expiresAt: number } | null = null
|
||||
let pending: Promise<string> | null = null
|
||||
let signingKey: CryptoKey | null = null
|
||||
async function mint(): Promise<string> {
|
||||
let account: z.infer<typeof ServiceAccount>
|
||||
try {
|
||||
account = ServiceAccount.parse(JSON.parse(config.serviceAccountJson))
|
||||
signingKey ??= await importPKCS8(account.private_key, 'RS256')
|
||||
} catch {
|
||||
throw new RelayAuthError('relay service account is malformed')
|
||||
}
|
||||
const iat = Math.floor(now() / 1000) - 30
|
||||
const assertion = await new SignJWT({ target_audience: config.audience })
|
||||
.setProtectedHeader({ alg: 'RS256', typ: 'JWT' })
|
||||
.setIssuer(account.client_email)
|
||||
.setSubject(account.client_email)
|
||||
.setAudience(TOKEN_URL)
|
||||
.setIssuedAt(iat)
|
||||
.setExpirationTime(iat + 600)
|
||||
.sign(signingKey)
|
||||
const response = await doFetch(TOKEN_URL, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/x-www-form-urlencoded' },
|
||||
body: new URLSearchParams({ grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer', assertion }).toString(),
|
||||
signal: AbortSignal.timeout(10_000)
|
||||
})
|
||||
const parsed = IdTokenResponse.safeParse(await response.json().catch(() => null))
|
||||
if (!response.ok || !parsed.success) {
|
||||
throw new RelayAuthError(`relay id token mint failed (${response.status})`)
|
||||
}
|
||||
// Google ID tokens last an hour; read exp rather than assume it.
|
||||
const payload = JSON.parse(atob(parsed.data.id_token.split('.')[1]!.replace(/-/g, '+').replace(/_/g, '/'))) as { exp?: number }
|
||||
cached = { value: parsed.data.id_token, expiresAt: (payload.exp ?? Math.floor(now() / 1000) + 3600) * 1000 }
|
||||
return cached.value
|
||||
}
|
||||
return async (): Promise<string> => {
|
||||
if (cached && cached.expiresAt - now() > 5 * 60_000) {
|
||||
return cached.value
|
||||
}
|
||||
pending ??= mint().finally(() => {
|
||||
pending = null
|
||||
})
|
||||
return pending
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The egress proxy's upstream fetch: chatgpt.com through the relay when one is configured, every other host
|
||||
* (and chatgpt.com without a relay) directly. Same request, same path and query; only the origin changes.
|
||||
*/
|
||||
export function relayingFetch(config: { relayUrl?: string; idToken?: () => Promise<string>; fetch?: typeof fetch }): typeof fetch {
|
||||
const doFetch = config.fetch ?? fetch
|
||||
const relay = config.relayUrl ? new URL(config.relayUrl) : null
|
||||
return (async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
const target = new URL(typeof input === 'string' || input instanceof URL ? input : input.url)
|
||||
if (!relay || !config.idToken || target.protocol !== 'https:' || !RELAYED_HOSTS.has(target.hostname)) {
|
||||
return doFetch(input, init)
|
||||
}
|
||||
let idToken: string
|
||||
try {
|
||||
idToken = await config.idToken()
|
||||
} catch {
|
||||
// In ChatGPT's error shape (a JSON detail), so Codex prints it instead of retrying a bare 5xx.
|
||||
console.error(JSON.stringify({ event: 'chatgpt_relay_auth_failed' }))
|
||||
return Response.json({ detail: 'Orca Cloud couldn’t reach ChatGPT just now. Try again in a minute.' }, { status: 503 })
|
||||
}
|
||||
const headers = new Headers(init?.headers)
|
||||
headers.set('x-serverless-authorization', `Bearer ${idToken}`)
|
||||
return doFetch(`${relay.origin}${target.pathname}${target.search}`, { ...init, headers })
|
||||
}) as typeof fetch
|
||||
}
|
||||
@@ -11,6 +11,9 @@
|
||||
// (that is what `refresh_token_reused` punishes).
|
||||
import { decodeJwt } from 'jose'
|
||||
import type { AccountProvider } from './account-providers'
|
||||
import { codexPathAllowed } from './codex-paths'
|
||||
|
||||
export { codexPathAllowed }
|
||||
import { credentialBinding, putAgentCredential, readUsableAgentCredential } from './agent-credentials'
|
||||
import type { Keyring } from './secrets-crypto'
|
||||
import type { SandboxRow } from './store'
|
||||
@@ -118,32 +121,6 @@ function formatCodex(credential: string): string {
|
||||
return `Bearer ${stored ? stored.access_token : credential}`
|
||||
}
|
||||
|
||||
/**
|
||||
* The ChatGPT backend calls Codex makes with its login (seen live, Codex 0.166): the Codex API itself, its
|
||||
* plugin catalog and usage. Nothing else on chatgpt.com (conversations, memories, connectors) gets the token.
|
||||
*/
|
||||
const CODEX_PREFIXES = ['/backend-api/codex/']
|
||||
const CODEX_EXACT_GETS = new Set([
|
||||
'/backend-api/ps/plugins/list',
|
||||
'/backend-api/ps/plugins/installed',
|
||||
'/backend-api/ps/plugins/suggested/codex',
|
||||
'/backend-api/plugins/featured',
|
||||
'/backend-api/plugins/export/curated',
|
||||
'/backend-api/wham/usage'
|
||||
])
|
||||
|
||||
export function codexPathAllowed(method: string, path: string): boolean {
|
||||
const pathname = path.split('?')[0] ?? ''
|
||||
// Plain segments only: no encoded or dot segments that could walk out of an allowed prefix.
|
||||
if (!/^(\/[A-Za-z0-9_~-][A-Za-z0-9._~-]*)+\/?$/.test(pathname) || /(^|\/)\.\.?(\/|$)/.test(pathname)) {
|
||||
return false
|
||||
}
|
||||
if (CODEX_PREFIXES.some((prefix) => pathname.startsWith(prefix))) {
|
||||
return true
|
||||
}
|
||||
return method.toUpperCase() === 'GET' && CODEX_EXACT_GETS.has(pathname)
|
||||
}
|
||||
|
||||
export const codexAccount: AccountProvider = {
|
||||
agent: 'codex',
|
||||
hosts: CODEX_ACCOUNT_HOSTS,
|
||||
@@ -183,7 +160,7 @@ export const codexAccount: AccountProvider = {
|
||||
status: 403,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
detail: 'Orca Cloud can’t use your ChatGPT sign-in here. Check it in Orca: Settings → Agent accounts.'
|
||||
detail: 'Orca Cloud can’t use your ChatGPT sign-in here. Check it in Orca: Settings → Cloud agent sign-ins.'
|
||||
})
|
||||
},
|
||||
// CODEX-EGRESS-BLOCKED: chatgpt.com answers Cloudflare Workers' egress with a 403 "Unable to load
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
// The chatgpt.com paths Codex's login may be used on. Shared, unchanged, by the Worker's egress proxy
|
||||
// (codex-credentials.ts) and the ChatGPT relay (cloud/apps/chatgpt-relay), which pins the same paths a
|
||||
// second time. Erasable TypeScript only: the relay runs this file with Node's type stripping.
|
||||
|
||||
/**
|
||||
* The ChatGPT backend calls Codex makes with its login (seen live, Codex 0.166): the Codex API itself, its
|
||||
* plugin catalog and usage. Nothing else on chatgpt.com (conversations, memories, connectors) gets the token.
|
||||
*/
|
||||
const CODEX_PREFIXES = ['/backend-api/codex/']
|
||||
const CODEX_EXACT_GETS = new Set([
|
||||
'/backend-api/ps/plugins/list',
|
||||
'/backend-api/ps/plugins/installed',
|
||||
'/backend-api/ps/plugins/suggested/codex',
|
||||
'/backend-api/plugins/featured',
|
||||
'/backend-api/plugins/export/curated',
|
||||
'/backend-api/wham/usage'
|
||||
])
|
||||
|
||||
export function codexPathAllowed(method: string, path: string): boolean {
|
||||
const pathname = path.split('?')[0] ?? ''
|
||||
// Plain segments only: no encoded or dot segments that could walk out of an allowed prefix.
|
||||
if (!/^(\/[A-Za-z0-9_~-][A-Za-z0-9._~-]*)+\/?$/.test(pathname) || /(^|\/)\.\.?(\/|$)/.test(pathname)) {
|
||||
return false
|
||||
}
|
||||
if (CODEX_PREFIXES.some((prefix) => pathname.startsWith(prefix))) {
|
||||
return true
|
||||
}
|
||||
return method.toUpperCase() === 'GET' && CODEX_EXACT_GETS.has(pathname)
|
||||
}
|
||||
@@ -15,6 +15,8 @@ export type Env = {
|
||||
VERCEL_REGION?: string
|
||||
/** Comma list tried in order when the default region's create fails on capacity (default `iad1`; "" = none). */
|
||||
VERCEL_FALLBACK_REGIONS?: string
|
||||
/** 'off': new workspaces and environments always start in the default region, not the one nearest the caller. */
|
||||
VERCEL_NEAREST_REGION?: string
|
||||
/** Overrides the Vercel API origin (tests only). */
|
||||
VERCEL_API_BASE?: string
|
||||
/** Seed golden image (orcad + agent CLIs), adopted when its region has none yet; see image/. */
|
||||
@@ -90,6 +92,10 @@ export type Env = {
|
||||
RUNTIME_REFRESH?: string
|
||||
/** Public URL of this Worker's /v1/egress/proxy; Vercel forwardURL rules point here. */
|
||||
EGRESS_PROXY_URL?: string
|
||||
/** CODEX-RELAY (src/chatgpt-relay.ts): the Cloud Run relay chatgpt.com egress goes through. Unset: direct. */
|
||||
CHATGPT_RELAY_URL?: string
|
||||
/** (secret) service-account JSON holding run.invoker on CHATGPT_RELAY_URL's service only. */
|
||||
CHATGPT_RELAY_SERVICE_ACCOUNT?: string
|
||||
/** Shortened session deadline (ms) for staging rollover rehearsals; refused when SANDBOX_ENV=prod. */
|
||||
SESSION_DEADLINE_MS?: string
|
||||
/** Dev-only: Authorization the egress proxy injects for api.github.com until the GitHub App minter exists. */
|
||||
|
||||
@@ -162,8 +162,12 @@ export type BuildDeps = {
|
||||
goldenSnapshotId: string
|
||||
/** Every region's golden image (staleness compares a build with its own region's). */
|
||||
goldenSnapshots?: Record<string, string>
|
||||
/** The default region: new builds run here. */
|
||||
/** The default region: builds of an environment without a served region of its own run here. */
|
||||
region: string
|
||||
/** Served regions, default first (each has a golden image); an environment may live in any of them. */
|
||||
regions?: readonly string[]
|
||||
/** A new environment lives in the served region nearest the admin who creates it (off: VERCEL_NEAREST_REGION=off). */
|
||||
nearestRegion?: boolean
|
||||
guardrails?: GuardrailConfig
|
||||
/** The workspace base allowlist (ORCA_PLATFORM_HOST_SUFFIXES included); the smoke boot uses it too. */
|
||||
egressAllowlist?: readonly string[]
|
||||
@@ -174,6 +178,24 @@ export type BuildDeps = {
|
||||
|
||||
const log = (event: string, fields: Record<string, unknown>) => console.log(JSON.stringify({ event, ...fields }))
|
||||
|
||||
type RegionConfig = Pick<BuildDeps['config'], 'region' | 'regions' | 'goldenSnapshotId' | 'goldenSnapshots'>
|
||||
|
||||
/**
|
||||
* Where an environment's builds run: the region it records, while that region is still served (has a
|
||||
* golden image); else the default. A null region (an environment from before regions were recorded) is the default.
|
||||
*/
|
||||
export function environmentRegion(config: RegionConfig, region: string | null | undefined): string {
|
||||
if (region && region !== config.region && config.regions?.includes(region) && config.goldenSnapshots?.[region]) {
|
||||
return region
|
||||
}
|
||||
return config.region
|
||||
}
|
||||
|
||||
/** The golden image a new build of an environment living in `region` (environmentRegion) starts from. */
|
||||
export function regionGolden(config: RegionConfig, region: string): string {
|
||||
return region === config.region ? config.goldenSnapshotId : (config.goldenSnapshots?.[region] ?? config.goldenSnapshotId)
|
||||
}
|
||||
|
||||
function randomId(): string {
|
||||
const bytes = crypto.getRandomValues(new Uint8Array(10))
|
||||
return `bld_${Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join('')}`
|
||||
@@ -303,6 +325,8 @@ export async function startBuild(
|
||||
}
|
||||
const now = deps.now()
|
||||
const id = randomId()
|
||||
// A snapshot only boots in its own region: the build runs where the environment lives.
|
||||
const region = environmentRegion(deps.config, environment.region)
|
||||
const base: BuildConfig = options.config ?? {
|
||||
revision: environment.revision,
|
||||
repos,
|
||||
@@ -336,7 +360,7 @@ export async function startBuild(
|
||||
WHERE changes() = 1`
|
||||
)
|
||||
.bind(id, environment.id, environment.account_id, environment.next_build_number, JSON.stringify(config),
|
||||
deps.config.goldenSnapshotId, deps.config.region, `orca-${deps.config.environment}-envbuild-${id}`, caller.userId, now, now)
|
||||
regionGolden(deps.config, region), region, `orca-${deps.config.environment}-envbuild-${id}`, caller.userId, now, now)
|
||||
])
|
||||
}
|
||||
if (bumped[0]?.meta.changes !== 1) {
|
||||
|
||||
@@ -15,12 +15,14 @@
|
||||
// - an environment below the security floor (environment-security.ts) is rebuilt regardless;
|
||||
// - refresh builds are metered internally, never charged to the team (reportBuildUsage).
|
||||
// Staleness is exact: environment_builds.base_snapshot_id is the golden snapshot each build started from.
|
||||
// Regions (U17): an environment lives in the default region, so a build is compared with the default
|
||||
// region's golden. An older build from another region (iad1 before pdx1 became the default) is stale
|
||||
// by that rule and is rebuilt in the default region; its workspaces boot in the build's own region
|
||||
// until the new build is active.
|
||||
// Regions (U17): an environment lives in its recorded region while that region is served, else the
|
||||
// default (environmentRegion), so a build is compared with that region's golden. A build from another
|
||||
// region (iad1 before pdx1 became the default, for an environment that records none) is stale by that
|
||||
// rule and is rebuilt where the environment lives; its workspaces boot in the build's own region until
|
||||
// the new build is active.
|
||||
import type { SandboxCaller } from './auth'
|
||||
import {
|
||||
environmentRegion,
|
||||
GOLDEN_REFRESH_USER,
|
||||
parseConfig,
|
||||
startBuild,
|
||||
@@ -53,13 +55,42 @@ type StaleRow = EnvironmentRow & {
|
||||
active_created_at: number
|
||||
}
|
||||
|
||||
// The golden a new build of e starts from: its region's (?1, region -> golden JSON of the served
|
||||
// regions), else the default region's (?2). The same rule as environmentRegion + regionGolden.
|
||||
const TARGET_GOLDEN = `coalesce(json_extract(?1, '$.' || e.region), ?2)`
|
||||
const STALE_SQL = `SELECT e.*, b.config AS active_config, b.base_snapshot_id AS active_base, b.created_at AS active_created_at
|
||||
FROM environments e JOIN environment_builds b ON b.id = e.active_build_id
|
||||
WHERE e.deleted_at IS NULL AND b.base_snapshot_id != ?
|
||||
WHERE e.deleted_at IS NULL AND b.base_snapshot_id != ${TARGET_GOLDEN}
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM environment_builds x WHERE x.environment_id = e.id
|
||||
AND (x.status IN ('starting', 'running', 'snapshotting')
|
||||
OR (x.base_snapshot_id = ? AND x.status != 'canceled')))`
|
||||
OR (x.base_snapshot_id = ${TARGET_GOLDEN} AND x.status != 'canceled')))`
|
||||
|
||||
/** ?1 of TARGET_GOLDEN: each served region other than the default, with its golden. */
|
||||
function servedGoldens(config: BuildDeps['config']): string {
|
||||
const goldens: Record<string, string> = {}
|
||||
for (const region of config.regions ?? []) {
|
||||
const golden = config.goldenSnapshots?.[region]
|
||||
if (region !== config.region && golden) {
|
||||
goldens[region] = golden
|
||||
}
|
||||
}
|
||||
return JSON.stringify(goldens)
|
||||
}
|
||||
|
||||
/** The security floor of the region each environment builds in, read once per region. */
|
||||
function floors(deps: BuildDeps): (environment: EnvironmentRow) => Promise<number> {
|
||||
const cache = new Map<string, Promise<number>>()
|
||||
return (environment) => {
|
||||
const region = environmentRegion(deps.config, environment.region)
|
||||
let floor = cache.get(region)
|
||||
if (!floor) {
|
||||
floor = securityFloor(deps.db, region)
|
||||
cache.set(region, floor)
|
||||
}
|
||||
return floor
|
||||
}
|
||||
}
|
||||
|
||||
/** Why a stale environment rebuilds now ('security' | 'due'), or null to wait. */
|
||||
async function refreshReason(
|
||||
@@ -95,13 +126,15 @@ export async function refreshStaleEnvironments(
|
||||
deps: BuildDeps,
|
||||
options: GoldenRefreshOptions = {}
|
||||
): Promise<GoldenRefreshTick> {
|
||||
const golden = deps.config.goldenSnapshotId
|
||||
const maxStarts = options.maxStartsPerTick ?? 2
|
||||
const maxInFlight = options.maxInFlight ?? 3
|
||||
const minIntervalMs = options.minIntervalMs ?? DEFAULT_REFRESH_MIN_INTERVAL_MS
|
||||
const recentMs = options.recentMs ?? DEFAULT_REFRESH_RECENT_MS
|
||||
const { results: stale } = await deps.db.prepare(`${STALE_SQL} ORDER BY e.updated_at DESC`).bind(golden, golden).all<StaleRow>()
|
||||
const floor = await securityFloor(deps.db, deps.config.region)
|
||||
const { results: stale } = await deps.db
|
||||
.prepare(`${STALE_SQL} ORDER BY e.updated_at DESC`)
|
||||
.bind(servedGoldens(deps.config), deps.config.goldenSnapshotId)
|
||||
.all<StaleRow>()
|
||||
const floorOf = floors(deps)
|
||||
const inFlight =
|
||||
(await deps.db
|
||||
.prepare(`SELECT count(*) AS n FROM environment_builds WHERE status IN ('starting', 'running', 'snapshotting')`)
|
||||
@@ -110,7 +143,7 @@ export async function refreshStaleEnvironments(
|
||||
const ranked: { environment: StaleRow; reason: 'security' | 'due' }[] = []
|
||||
let deferred = 0
|
||||
for (const environment of stale) {
|
||||
const reason = await refreshReason(deps, environment, floor, { minIntervalMs, recentMs })
|
||||
const reason = await refreshReason(deps, environment, await floorOf(environment), { minIntervalMs, recentMs })
|
||||
if (reason) {
|
||||
ranked.push({ environment, reason })
|
||||
} else {
|
||||
@@ -147,15 +180,14 @@ export async function refreshEnvironment(
|
||||
options: Pick<GoldenRefreshOptions, 'minIntervalMs'> = {}
|
||||
): Promise<'started' | 'current' | 'deferred' | 'skipped'> {
|
||||
try {
|
||||
const golden = deps.config.goldenSnapshotId
|
||||
const environment = await deps.db
|
||||
.prepare(`${STALE_SQL} AND e.id = ?`)
|
||||
.bind(golden, golden, environmentId)
|
||||
.prepare(`${STALE_SQL} AND e.id = ?3`)
|
||||
.bind(servedGoldens(deps.config), deps.config.goldenSnapshotId, environmentId)
|
||||
.first<StaleRow>()
|
||||
if (!environment) {
|
||||
return 'current'
|
||||
}
|
||||
const floor = await securityFloor(deps.db, deps.config.region)
|
||||
const floor = await securityFloor(deps.db, environmentRegion(deps.config, environment.region))
|
||||
// Used right now, so no recency test: only the interval (or the floor) decides.
|
||||
const reason = await refreshReason(deps, environment, floor, {
|
||||
minIntervalMs: options.minIntervalMs ?? DEFAULT_REFRESH_MIN_INTERVAL_MS
|
||||
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
type BuildRow
|
||||
} from './environment-builds'
|
||||
import { checkStaleness } from './environment-staleness'
|
||||
import { nearestRegions } from './regions'
|
||||
import {
|
||||
CreateEnvironmentSchema,
|
||||
EnvironmentError,
|
||||
@@ -135,7 +136,9 @@ export async function handleEnvironmentRoute(
|
||||
if (!body.success) {
|
||||
return json({ error: 'invalid_request', reason: body.error.issues[0]?.path.join('.') }, 400)
|
||||
}
|
||||
return json({ environment: environmentView(await createEnvironment(db, caller, body.data, deps.now())) })
|
||||
// Its builds run, and its workspaces boot, where it lives: the served region nearest whoever creates it.
|
||||
const region = nearestRegions(request, deps.config.regions ?? [], deps.config.nearestRegion !== false)?.[0] ?? null
|
||||
return json({ environment: environmentView(await createEnvironment(db, caller, body.data, deps.now(), region)) })
|
||||
}
|
||||
return json({ error: 'not_found' }, 404)
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// call per repo answers both "the branch moved" and "a lockfile changed" (installed deps are stale).
|
||||
// Uses the GitHub App token when the repo is bound, anonymous otherwise (public repos). Cached briefly
|
||||
// per isolate so a settings page polling it costs GitHub nothing extra.
|
||||
import { LOCKFILES, getBuild, parseConfig, parseResult, type BuildDeps } from './environment-builds'
|
||||
import { LOCKFILES, environmentRegion, getBuild, parseConfig, parseResult, regionGolden, type BuildDeps } from './environment-builds'
|
||||
import { parseRepos, type EnvironmentRow } from './environments'
|
||||
import { parseGitHubRepo } from './github-app'
|
||||
|
||||
@@ -37,7 +37,7 @@ export type Staleness = {
|
||||
const LOCKFILE_NAMES = new Set(LOCKFILES)
|
||||
|
||||
export async function checkStaleness(
|
||||
deps: Pick<BuildDeps, 'db' | 'github' | 'now'> & { githubFetch?: typeof fetch; config?: { goldenSnapshotId: string } },
|
||||
deps: Pick<BuildDeps, 'db' | 'github' | 'now'> & { githubFetch?: typeof fetch; config?: Parameters<typeof regionGolden>[0] },
|
||||
environment: EnvironmentRow,
|
||||
refresh = false
|
||||
): Promise<Staleness> {
|
||||
@@ -54,7 +54,9 @@ export async function checkStaleness(
|
||||
const config = parseConfig(build)
|
||||
const result = parseResult(build)
|
||||
const configChanged = config.revision !== environment.revision
|
||||
const imageUpdated = !!deps.config && build.base_snapshot_id !== deps.config.goldenSnapshotId
|
||||
// Compared with the golden of the region the environment builds in (its own, or the default).
|
||||
const imageUpdated =
|
||||
!!deps.config && build.base_snapshot_id !== regionGolden(deps.config, environmentRegion(deps.config, environment.region))
|
||||
const repos: RepoStaleness[] = []
|
||||
for (const repo of parseRepos(environment)) {
|
||||
const built = result?.repos?.find((r) => r.url === repo.url)
|
||||
|
||||
@@ -37,6 +37,8 @@ export type EnvironmentRow = {
|
||||
*/
|
||||
kind?: 'org' | 'auto'
|
||||
last_used_at?: number | null
|
||||
/** Where it builds and its workspaces boot (environmentRegion); null for environments made before this was recorded. */
|
||||
region?: string | null
|
||||
}
|
||||
|
||||
export class EnvironmentError extends Error {
|
||||
@@ -173,7 +175,9 @@ export async function createEnvironment(
|
||||
db: D1Database,
|
||||
caller: SandboxCaller,
|
||||
input: z.infer<typeof CreateEnvironmentSchema>,
|
||||
now: number
|
||||
now: number,
|
||||
/** The served region nearest the admin creating it; null leaves it to the default region. */
|
||||
region: string | null = null
|
||||
): Promise<EnvironmentRow> {
|
||||
requireAdmin(caller)
|
||||
const repos = checkedRepos(input.repos)
|
||||
@@ -188,11 +192,11 @@ export async function createEnvironment(
|
||||
await db
|
||||
.prepare(
|
||||
`INSERT INTO environments (id, account_id, name, repos, install_script, env_var_names, machine_size,
|
||||
revision, next_build_number, created_by, updated_by, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, 1, 1, ?, ?, ?, ?)`
|
||||
revision, next_build_number, created_by, updated_by, created_at, updated_at, region)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, 1, 1, ?, ?, ?, ?, ?)`
|
||||
)
|
||||
.bind(id, caller.accountId, input.name, JSON.stringify(repos), input.installScript,
|
||||
JSON.stringify([...new Set(input.envVarNames)]), input.machineSize, caller.userId, caller.userId, now, now)
|
||||
JSON.stringify([...new Set(input.envVarNames)]), input.machineSize, caller.userId, caller.userId, now, now, region)
|
||||
.run()
|
||||
return (await getEnvironment(db, caller.accountId, id))!
|
||||
}
|
||||
|
||||
@@ -26,6 +26,11 @@ export type GitHubAppMinter = {
|
||||
getInstallation(installationId: number): Promise<GitHubInstallation | null>
|
||||
/** The installation covering a github.com repo URL, or null. */
|
||||
getRepoInstallation(repoUrl: string): Promise<GitHubInstallation | null>
|
||||
/**
|
||||
* The App's installation on a GitHub org or user (whatever repositories it selected), or null.
|
||||
* Tells "not installed on the owner" from "installed, but this repository isn't selected".
|
||||
*/
|
||||
getOwnerInstallation?(owner: string): Promise<GitHubInstallation | null>
|
||||
/** App-JWT request to an app-level endpoint (operator: /app, /app/hook/config). */
|
||||
appRequest?(method: string, path: '/app' | '/app/hook/config', body?: unknown): Promise<Response>
|
||||
/**
|
||||
@@ -262,6 +267,14 @@ export function createGitHubAppMinter(config: {
|
||||
return github(method, path, body)
|
||||
},
|
||||
|
||||
async getOwnerInstallation(owner) {
|
||||
if (!/^[A-Za-z0-9-]{1,39}$/.test(owner)) {
|
||||
return null
|
||||
}
|
||||
// Orgs and users have separate lookups; an org login 404s on /users/{login}/installation.
|
||||
return (await installationFor(`/orgs/${owner}/installation`)) ?? installationFor(`/users/${owner}/installation`)
|
||||
},
|
||||
|
||||
async getRepoInstallation(repoUrl) {
|
||||
const parsed = parseGitHubRepo(repoUrl)
|
||||
return parsed ? installationFor(`/repos/${parsed.owner}/${parsed.repo}/installation`) : null
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
// CREATE-GH-STEP: GitHub's installation lookups for repo-access, cached in D1 so every isolate shares
|
||||
// them and the create wizard's 2 s poll doesn't spend an app-JWT call each time. A found installation
|
||||
// is kept a minute, a 404 for 15 s. The installation webhooks and the setup return drop the owner's
|
||||
// entries (forgetOwner/forgetInstallation), so an install is seen on the next poll, not after the TTL.
|
||||
// Only ids and the owner's login are stored. Any D1 failure falls through to GitHub.
|
||||
import type { GitHubAppMinter, GitHubInstallation } from './github-app'
|
||||
|
||||
export const FOUND_TTL_MS = 60_000
|
||||
export const MISSING_TTL_MS = 15_000
|
||||
|
||||
type Row = {
|
||||
installation_id: number | null
|
||||
github_account_id: number | null
|
||||
github_account_login: string | null
|
||||
github_account_type: string | null
|
||||
checked_at: number
|
||||
}
|
||||
|
||||
function fromRow(row: Row): GitHubInstallation | null {
|
||||
if (row.installation_id === null || row.github_account_login === null) {
|
||||
return null
|
||||
}
|
||||
return {
|
||||
id: row.installation_id,
|
||||
account: {
|
||||
id: row.github_account_id ?? 0,
|
||||
login: row.github_account_login,
|
||||
type: row.github_account_type === 'User' ? 'User' : 'Organization'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function cached(
|
||||
db: D1Database,
|
||||
key: string,
|
||||
ownerKey: string,
|
||||
now: number,
|
||||
lookup: () => Promise<GitHubInstallation | null>
|
||||
): Promise<GitHubInstallation | null> {
|
||||
const row = await db
|
||||
.prepare(
|
||||
'SELECT installation_id, github_account_id, github_account_login, github_account_type, checked_at FROM github_installation_cache WHERE cache_key = ?'
|
||||
)
|
||||
.bind(key)
|
||||
.first<Row>()
|
||||
.catch(() => null)
|
||||
if (row && now - row.checked_at < (row.installation_id === null ? MISSING_TTL_MS : FOUND_TTL_MS)) {
|
||||
return fromRow(row)
|
||||
}
|
||||
// GitHub errors propagate uncached (repo-access answers 502; the next poll asks again).
|
||||
const found = await lookup()
|
||||
await db
|
||||
.prepare(
|
||||
`INSERT INTO github_installation_cache (cache_key, owner_key, installation_id, github_account_id, github_account_login, github_account_type, checked_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (cache_key) DO UPDATE SET installation_id = excluded.installation_id, github_account_id = excluded.github_account_id,
|
||||
github_account_login = excluded.github_account_login, github_account_type = excluded.github_account_type, checked_at = excluded.checked_at`
|
||||
)
|
||||
.bind(key, ownerKey, found?.id ?? null, found?.account.id ?? null, found?.account.login ?? null, found?.account.type ?? null, now)
|
||||
.run()
|
||||
.catch(() => undefined)
|
||||
return found
|
||||
}
|
||||
|
||||
/** GET /repos/{owner}/{repo}/installation, cached. */
|
||||
export function cachedRepoInstallation(
|
||||
db: D1Database,
|
||||
github: GitHubAppMinter,
|
||||
parsed: { owner: string; repo: string },
|
||||
now: number
|
||||
): Promise<GitHubInstallation | null> {
|
||||
const owner = parsed.owner.toLowerCase()
|
||||
return cached(db, `${owner}/${parsed.repo.toLowerCase()}`, owner, now, () =>
|
||||
github.getRepoInstallation(`https://github.com/${parsed.owner}/${parsed.repo}`)
|
||||
)
|
||||
}
|
||||
|
||||
/** The App's installation on the owner (org or user), cached; null when the minter can't ask. */
|
||||
export function cachedOwnerInstallation(
|
||||
db: D1Database,
|
||||
github: GitHubAppMinter,
|
||||
owner: string,
|
||||
now: number
|
||||
): Promise<GitHubInstallation | null> {
|
||||
const lookup = github.getOwnerInstallation
|
||||
if (!lookup) {
|
||||
return Promise.resolve(null)
|
||||
}
|
||||
const key = owner.toLowerCase()
|
||||
return cached(db, `${key}/`, key, now, () => lookup.call(github, owner))
|
||||
}
|
||||
|
||||
/** An install, uninstall, or repository selection change on this owner: ask GitHub next time. */
|
||||
export async function forgetOwner(db: D1Database, owner: string): Promise<void> {
|
||||
await db
|
||||
.prepare('DELETE FROM github_installation_cache WHERE owner_key = ?')
|
||||
.bind(owner.toLowerCase())
|
||||
.run()
|
||||
.catch(() => undefined)
|
||||
}
|
||||
|
||||
/** Same, by installation id (a webhook whose account login is missing). */
|
||||
export async function forgetInstallation(db: D1Database, installationId: number): Promise<void> {
|
||||
await db
|
||||
.prepare('DELETE FROM github_installation_cache WHERE installation_id = ?')
|
||||
.bind(installationId)
|
||||
.run()
|
||||
.catch(() => undefined)
|
||||
}
|
||||
@@ -6,6 +6,12 @@
|
||||
// installation): true = the caller can install it on the owner (it is their own GitHub account, or
|
||||
// they are an active admin of the owner org); false = another user's account, or a non-admin org
|
||||
// membership; null = unknown (no connected GitHub user token, membership unreadable, GitHub down).
|
||||
// push (CREATE-GH-STEP): can a workspace on this repo push? 'ok' = granted; 'not_installed' = the
|
||||
// App isn't on the owner; 'repo_not_selected' = it is, but this repository isn't one it selected
|
||||
// (configureUrl: the installation's settings page); 'not_connected' = the App reaches the repo
|
||||
// but through an installation this Orca account isn't bound to. installUrl is present for every
|
||||
// push reason but 'ok' and 'repo_not_selected', public repositories included.
|
||||
// GitHub's installation lookups are cached in D1 (github-installation-cache.ts).
|
||||
// GET /v1/github/repositories[?cursor=] -> {repositories, nextCursor} (start from GitHub, D4/N12)
|
||||
// GET /v1/internal/github/install-url?accountId&userId&repo&session (billing's setup chain,
|
||||
// Bearer CONTROL_PLANE_TOKEN) -> {access, url}
|
||||
@@ -16,6 +22,7 @@
|
||||
// its own: POST <billing>/v1/internal/cloud-setup/github {sessionId?, accountId, repo, state}.
|
||||
import type { GitHubAppMinter, GitHubListedRepo } from './github-app'
|
||||
import { parseGitHubRepo } from './github-app'
|
||||
import { cachedOwnerInstallation, cachedRepoInstallation, forgetInstallation, forgetOwner } from './github-installation-cache'
|
||||
import {
|
||||
bindApprovedRequests,
|
||||
boundAccounts,
|
||||
@@ -30,6 +37,9 @@ import {
|
||||
|
||||
export type RepoAccess = 'public' | 'granted' | 'missing' | 'requested'
|
||||
|
||||
/** Whether a workspace on the repository can push, and if not, what's missing (CREATE-GH-STEP). */
|
||||
export type PushAccess = 'ok' | 'not_installed' | 'repo_not_selected' | 'not_connected'
|
||||
|
||||
export type RepoAccessDeps = {
|
||||
db: D1Database
|
||||
github: GitHubAppMinter | undefined
|
||||
@@ -52,25 +62,93 @@ export async function repoAccess(
|
||||
accountId: string,
|
||||
repoUrl: string
|
||||
): Promise<{ access: RepoAccess; owner: string; installationId: number | null }> {
|
||||
const { access, owner, installationId } = await repoAccessWithPush(deps, accountId, repoUrl)
|
||||
return { access, owner, installationId }
|
||||
}
|
||||
|
||||
/** repoAccess plus whether a workspace can push, and the owner installation when that's the gap. */
|
||||
export async function repoAccessWithPush(
|
||||
deps: RepoAccessDeps,
|
||||
accountId: string,
|
||||
repoUrl: string
|
||||
): Promise<{
|
||||
access: RepoAccess
|
||||
owner: string
|
||||
installationId: number | null
|
||||
push: PushAccess
|
||||
ownerInstallation?: { id: number; type: 'User' | 'Organization' }
|
||||
}> {
|
||||
const parsed = parseGitHubRepo(repoUrl)
|
||||
if (!parsed) {
|
||||
throw new RepoAccessError('not_github')
|
||||
}
|
||||
const installation = deps.github ? await deps.github.getRepoInstallation(repoUrl) : null
|
||||
const installation = deps.github ? await cachedRepoInstallation(deps.db, deps.github, parsed, deps.now) : null
|
||||
if (installation && (await isInstallationBound(deps.db, installation.id, accountId))) {
|
||||
return { access: 'granted', owner: parsed.owner, installationId: installation.id }
|
||||
return { access: 'granted', owner: parsed.owner, installationId: installation.id, push: 'ok' }
|
||||
}
|
||||
if (await deps.publicRepoCheck(repoUrl)) {
|
||||
return { access: 'public', owner: parsed.owner, installationId: installation?.id ?? null }
|
||||
const [gap, isPublic] = await Promise.all([pushGap(deps, accountId, parsed.owner, installation), deps.publicRepoCheck(repoUrl)])
|
||||
if (isPublic) {
|
||||
return { access: 'public', owner: parsed.owner, installationId: installation?.id ?? null, ...gap }
|
||||
}
|
||||
if (!installation && (await openInstallRequest(deps.db, accountId, parsed.owner, deps.now))) {
|
||||
// Ours, and still open on GitHub (an owner who declines leaves no webhook, only an empty list).
|
||||
const pending = await deps.github?.pendingRequestOwners?.().catch(() => null)
|
||||
if (!pending || pending.has(parsed.owner.toLowerCase())) {
|
||||
return { access: 'requested', owner: parsed.owner, installationId: null }
|
||||
return { access: 'requested', owner: parsed.owner, installationId: null, ...gap }
|
||||
}
|
||||
}
|
||||
return { access: 'missing', owner: parsed.owner, installationId: installation?.id ?? null }
|
||||
return { access: 'missing', owner: parsed.owner, installationId: installation?.id ?? null, ...gap }
|
||||
}
|
||||
|
||||
/**
|
||||
* Not granted: the App reaches the repo through someone else's binding, or it's on the owner without
|
||||
* this repository, or it isn't on the owner. Only an installation bound to this account counts as
|
||||
* "repo not selected" without asking GitHub; otherwise one cached owner lookup tells.
|
||||
*/
|
||||
async function pushGap(
|
||||
deps: RepoAccessDeps,
|
||||
accountId: string,
|
||||
owner: string,
|
||||
repoInstallation: { id: number } | null
|
||||
): Promise<{ push: PushAccess; ownerInstallation?: { id: number; type: 'User' | 'Organization' } }> {
|
||||
if (repoInstallation) {
|
||||
return { push: 'not_connected' }
|
||||
}
|
||||
const bound = (await listAccountInstallations(deps.db, accountId)).find(
|
||||
(row) => row.github_account_login.toLowerCase() === owner.toLowerCase()
|
||||
)
|
||||
if (bound) {
|
||||
return {
|
||||
push: 'repo_not_selected',
|
||||
ownerInstallation: { id: bound.installation_id, type: bound.github_account_type === 'User' ? 'User' : 'Organization' }
|
||||
}
|
||||
}
|
||||
// Never holds the answer up: GitHub erroring or slow reads as "not installed", whose install page
|
||||
// GitHub turns into the installation's settings when the App is already there.
|
||||
const onOwner = deps.github
|
||||
? await withTimeout(cachedOwnerInstallation(deps.db, deps.github, owner, deps.now), OWNER_LOOKUP_TIMEOUT_MS).catch(() => null)
|
||||
: null
|
||||
if (!onOwner) {
|
||||
return { push: 'not_installed' }
|
||||
}
|
||||
// On the owner, but bound to another Orca account (or none): binding it is the install round trip.
|
||||
return (await isInstallationBound(deps.db, onOwner.id, accountId))
|
||||
? { push: 'repo_not_selected', ownerInstallation: { id: onOwner.id, type: onOwner.account.type } }
|
||||
: { push: 'not_connected' }
|
||||
}
|
||||
|
||||
const OWNER_LOOKUP_TIMEOUT_MS = 3_000
|
||||
|
||||
function withTimeout<T>(work: Promise<T>, ms: number): Promise<T | null> {
|
||||
let timer: ReturnType<typeof setTimeout> | undefined
|
||||
return Promise.race([work, new Promise<null>((resolve) => (timer = setTimeout(() => resolve(null), ms)))]).finally(() => clearTimeout(timer))
|
||||
}
|
||||
|
||||
/** GitHub's page where the installation's repository selection is changed. */
|
||||
export function installationSettingsUrl(owner: string, installation: { id: number; type: 'User' | 'Organization' }): string {
|
||||
return installation.type === 'User'
|
||||
? `https://github.com/settings/installations/${installation.id}`
|
||||
: `https://github.com/organizations/${encodeURIComponent(owner)}/settings/installations/${installation.id}`
|
||||
}
|
||||
|
||||
export class RepoAccessError extends Error {
|
||||
@@ -150,12 +228,17 @@ export async function repoAccessView(
|
||||
if (!parsed) {
|
||||
return { status: 400, body: { error: 'invalid_request', reason: 'not_github' } }
|
||||
}
|
||||
const result = await repoAccess(deps, caller.accountId, repo)
|
||||
const body: Record<string, unknown> = { access: result.access, owner: parsed.owner, repo: canonicalRepoUrl(parsed) }
|
||||
const result = await repoAccessWithPush(deps, caller.accountId, repo)
|
||||
const body: Record<string, unknown> = { access: result.access, owner: parsed.owner, repo: canonicalRepoUrl(parsed), push: result.push }
|
||||
// Only while the App isn't installed for this owner: that is when the desktop picks "Install" vs "ask an owner".
|
||||
const askCanInstall = result.access === 'missing' || (result.access === 'public' && result.installationId === null)
|
||||
// A public repository clones anonymously but can't push: the wizard's GitHub step installs from here too.
|
||||
const wantsInstall = result.access === 'missing' || (result.access === 'public' && (result.push === 'not_installed' || result.push === 'not_connected'))
|
||||
if (result.push === 'repo_not_selected' && result.ownerInstallation) {
|
||||
body.configureUrl = installationSettingsUrl(parsed.owner, result.ownerInstallation)
|
||||
}
|
||||
const [install, canInstall] = await Promise.all([
|
||||
result.access === 'missing' && deps.connect
|
||||
wantsInstall && deps.connect
|
||||
? repoInstallUrl(deps, caller, repo, {
|
||||
setupSessionId: url.searchParams.get('session') ?? undefined,
|
||||
repoId: positiveInt(url.searchParams.get('repoId')),
|
||||
@@ -342,6 +425,13 @@ export async function handleInstallationEvent(
|
||||
if (!id || !Number.isSafeInteger(id)) {
|
||||
return { handled: 'ignored' }
|
||||
}
|
||||
if (eventName === 'installation' || eventName === 'installation_repositories') {
|
||||
// Any install, uninstall, suspend or selection change: repo-access asks GitHub on the next poll.
|
||||
await forgetInstallation(deps.db, id)
|
||||
if (installation?.account?.login) {
|
||||
await forgetOwner(deps.db, installation.account.login)
|
||||
}
|
||||
}
|
||||
if (eventName === 'installation' && payload.action === 'deleted') {
|
||||
return { handled: 'deleted', revoked: await revokeInstallation(deps.db, id, null, deps.now) }
|
||||
}
|
||||
@@ -399,6 +489,11 @@ export async function noticeAfterReturn(
|
||||
}
|
||||
const repo = `https://github.com/${state.r}`
|
||||
const base = { ...(state.s ? { sessionId: state.s } : {}), accountId: state.a, repo }
|
||||
// Back from GitHub's install page: its webhook may not have landed yet.
|
||||
const owner = parseGitHubRepo(repo)?.owner
|
||||
if (owner) {
|
||||
await forgetOwner(deps.db, owner)
|
||||
}
|
||||
if (requested) {
|
||||
return { ...base, state: 'pending' }
|
||||
}
|
||||
|
||||
@@ -362,13 +362,13 @@ export function entryPointsVersion(): string {
|
||||
|
||||
// One round trip: the image's scripts stamp (missing = 0: images from before stamps) and whether
|
||||
// this Worker's entry points are already installed.
|
||||
const ENTRY_POINTS_PROBE = [
|
||||
export const ENTRY_POINTS_PROBE = [
|
||||
'stamp="$(cat /opt/orca/bin/.scripts-stamp 2>/dev/null || true)"',
|
||||
'echo "stamp=${stamp:-0}"',
|
||||
`if [ -e /opt/orca/bin/.entry-points-${VM_SCRIPTS_HASH} ]; then echo marker=1; else echo marker=0; fi`
|
||||
].join('\n')
|
||||
|
||||
function parseEntryPointsProbe(stdout: string): { stamp: number; marker: boolean } | null {
|
||||
export function parseEntryPointsProbe(stdout: string): { stamp: number; marker: boolean } | null {
|
||||
const marker = /^marker=([01])$/m.exec(stdout)?.[1]
|
||||
if (marker === undefined) {
|
||||
return null
|
||||
@@ -399,6 +399,16 @@ export async function ensureEntryPoints(deps: { provider: Pick<SandboxProviderAp
|
||||
await installEntryPoints(deps, sessionId)
|
||||
}
|
||||
|
||||
/**
|
||||
* ENTRY-POINTS-BEFORE-START: a boot from an environment build made on an older golden would run that
|
||||
* golden's orca-sandbox-start (no agent prefetch, no OpenCode first run) until the environment is
|
||||
* rebuilt, hours later. True when the image's scripts are older than this Worker's: no marker for this
|
||||
* set and a lower stamp (an equal stamp is the same scripts: the stamp only moves when they change).
|
||||
*/
|
||||
export function entryPointsBehind(image: { stamp: number; marker: boolean } | null): boolean {
|
||||
return image !== null && !image.marker && image.stamp < VM_SCRIPTS_STAMP
|
||||
}
|
||||
|
||||
export async function installEntryPoints(deps: { provider: Pick<SandboxProviderApi, 'runCommand'> }, sessionId: string): Promise<void> {
|
||||
const env = provisionEnv({ manifest: MANIFEST, provisionScript: PROVISION_SCRIPT, vmScripts: VM_SCRIPT_SOURCES, vmScriptsStamp: VM_SCRIPTS_STAMP, buildId: 'sync' })
|
||||
const scriptEnv = Object.fromEntries(Object.entries(env).filter(([key]) => key.startsWith('ORCA_VM_SCRIPT_')))
|
||||
|
||||
@@ -64,8 +64,9 @@ import { createRoutinesApi } from './routines/api'
|
||||
import { handleRoutineHook, isRoutineHookPath } from './routines/hooks'
|
||||
import { tickRoutines } from './routines/runs'
|
||||
import { activeGoldenSnapshots, advanceGoldenImages } from './golden-images'
|
||||
import { sandboxRegions, seedRegion } from './regions'
|
||||
import { nearestRegionEnabled, nearestRegions, sandboxRegions, seedRegion, servedRegions } from './regions'
|
||||
import { handleRuntimeBundleRoute, runtimeBundleFor, runtimeBundleUrl } from './runtime-bundles'
|
||||
import { createIdTokenSource, relayingFetch } from './chatgpt-relay'
|
||||
import { NO_CREDENTIALS, handleEgressProxy, type CredentialBroker, type ProxiedRoute } from './egress-proxy'
|
||||
import {
|
||||
BillingUnavailableError,
|
||||
@@ -891,6 +892,34 @@ function fail(error: string, status: number, reason?: string): Response {
|
||||
return json(reason ? { error, reason } : { error }, status)
|
||||
}
|
||||
|
||||
/** CREATE-STEP-WATCH: logs a step still pending after 10 s and, once the create starts, every step's time. */
|
||||
export function createStepWatch(enabled: boolean, now: () => number = Date.now) {
|
||||
const started = now()
|
||||
const steps: Record<string, number> = {}
|
||||
let reported = false
|
||||
return {
|
||||
async step<T>(name: string, work: Promise<T>): Promise<T> {
|
||||
if (!enabled) {
|
||||
return work
|
||||
}
|
||||
const at = now()
|
||||
const slow = setTimeout(() => console.log(JSON.stringify({ event: 'create_step_slow', step: name, ms: now() - at })), 10_000)
|
||||
try {
|
||||
return await work
|
||||
} finally {
|
||||
clearTimeout(slow)
|
||||
steps[name] = now() - at
|
||||
}
|
||||
},
|
||||
done() {
|
||||
if (enabled && !reported) {
|
||||
reported = true
|
||||
console.log(JSON.stringify({ event: 'create_steps', ms: now() - started, steps }))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function readJson(request: Request, maxBytes = MAX_JSON_BYTES): Promise<unknown> {
|
||||
const text = await request.text()
|
||||
if (text.length > maxBytes) {
|
||||
@@ -982,7 +1011,7 @@ export async function lifecycleDeps(env: Env, deps: Deps): Promise<LifecycleDeps
|
||||
// AUTO_ENVIRONMENTS=off: creates of repos without an environment keep cloning from scratch.
|
||||
if (env.AUTO_ENVIRONMENTS !== 'off') {
|
||||
life.autoEnvironment = {
|
||||
ensure: (accountId, repoUrl) => ensureAutoEnvironment(buildDeps(life, deps, env), accountId, repoUrl),
|
||||
ensure: (accountId, repoUrl, region) => ensureAutoEnvironment(buildDeps(life, deps, env), accountId, repoUrl, region),
|
||||
touch: (environmentId) => touchEnvironment(env.SANDBOX_DB, environmentId, life.now())
|
||||
}
|
||||
}
|
||||
@@ -1051,6 +1080,8 @@ function buildDeps(life: LifecycleDeps, deps: Deps, env?: Env): BuildDeps {
|
||||
goldenSnapshotId: life.config.goldenSnapshotId,
|
||||
goldenSnapshots: life.config.goldenSnapshots,
|
||||
region: life.config.defaultRegion,
|
||||
regions: servedRegions(life.config),
|
||||
nearestRegion: env ? nearestRegionEnabled(env) : true,
|
||||
guardrails: life.config.guardrails,
|
||||
egressAllowlist: life.config.egressAllowlist,
|
||||
egressAllowlistError: life.config.egressAllowlistError
|
||||
@@ -1318,7 +1349,7 @@ async function routeRequest(request: Request, env: Env, deps: Deps): Promise<Res
|
||||
projectId: env.VERCEL_PROJECT_ID,
|
||||
findSandbox: (name) => getSandboxByProviderName(env.SANDBOX_DB, name),
|
||||
routesFor: egressRoutes(env, deps),
|
||||
fetch: deps.egressFetch,
|
||||
fetch: chatgptRelayFetch(env, deps.egressFetch),
|
||||
keysForIssuer: deps.egressKeysForIssuer
|
||||
})
|
||||
}
|
||||
@@ -1807,11 +1838,13 @@ async function routeRequest(request: Request, env: Env, deps: Deps): Promise<Res
|
||||
if (!match) {
|
||||
return fail('not_found', 404)
|
||||
}
|
||||
const caller = await authenticateDesktop(env, request, (deps.now ?? Date.now)())
|
||||
// CREATE-STEP-WATCH: creates hung 180 s with no log and no row (n73). Name the step a create waits in.
|
||||
const watch = createStepWatch(request.method === 'POST' && !match[1])
|
||||
const caller = await watch.step('auth', authenticateDesktop(env, request, (deps.now ?? Date.now)()))
|
||||
if (!caller) {
|
||||
return fail('unauthorized', 401)
|
||||
}
|
||||
const life = await lifecycleDeps(env, deps)
|
||||
const life = await watch.step('deps', lifecycleDeps(env, deps))
|
||||
if (!life) {
|
||||
return fail('not_configured', 503)
|
||||
}
|
||||
@@ -1834,12 +1867,12 @@ async function routeRequest(request: Request, env: Env, deps: Deps): Promise<Res
|
||||
if (request.method !== 'POST') {
|
||||
return fail('not_found', 404)
|
||||
}
|
||||
if (await tokenMintedBeforeMemberChange(life.db, caller)) {
|
||||
if (await watch.step('stale_token', tokenMintedBeforeMemberChange(life.db, caller))) {
|
||||
// ORG-TOKEN-STALE: never create (and bill) with a token minted before a role change.
|
||||
return fail('unauthorized', 401, 'stale_token')
|
||||
}
|
||||
await consumeCreateToken(life.db, life.config.guardrails!, caller.accountId, life.now())
|
||||
const raw = await readJson(request)
|
||||
await watch.step('create_token', consumeCreateToken(life.db, life.config.guardrails!, caller.accountId, life.now()))
|
||||
const raw = await watch.step('body', readJson(request))
|
||||
const body = CreateRequestSchema.safeParse(raw)
|
||||
if (!body.success) {
|
||||
return fail('invalid_request', 400)
|
||||
@@ -1857,12 +1890,18 @@ async function routeRequest(request: Request, env: Env, deps: Deps): Promise<Res
|
||||
life.db,
|
||||
{ accountId: caller.accountId, scope: `sandbox.create:${caller.userId}`, key: readIdempotencyKey(request), body: raw, now: life.now() },
|
||||
async () => {
|
||||
watch.done()
|
||||
// Desktops that show the clone as its own step ask for it in the background (GET .../clone).
|
||||
// Member identities install after the response; the VM's git holds commits and pushes until then.
|
||||
let ready: Ready
|
||||
try {
|
||||
// Nothing pinned: start in the served region nearest the caller (an environment's own region still wins).
|
||||
const nearRegions = body.data.region ? undefined : nearestRegions(request, servedRegions(life.config), nearestRegionEnabled(env))
|
||||
// Region order only (never the caller's location): which region a create preferred, and why.
|
||||
console.log(JSON.stringify({ event: 'create_region_order', explicit: body.data.region ?? null, order: nearRegions ?? null }))
|
||||
ready = await createSandbox(life, caller, {
|
||||
...body.data,
|
||||
...(nearRegions ? { nearRegions } : {}),
|
||||
cloneInBackground: features.includes('clone-progress'),
|
||||
deferMemberGit: true
|
||||
})
|
||||
@@ -2863,3 +2902,23 @@ async function agentSourcesRoute(request: Request, url: URL, env: Env, deps: Dep
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
// CODEX-RELAY: one ID-token source per isolate and relay config (the token is cached for its hour).
|
||||
let relayIdTokens: { key: string; source: () => Promise<string> } | null = null
|
||||
function chatgptRelayFetch(env: Env, egressFetch: typeof fetch | undefined): typeof fetch | undefined {
|
||||
if (!env.CHATGPT_RELAY_URL || !env.CHATGPT_RELAY_SERVICE_ACCOUNT) {
|
||||
return egressFetch
|
||||
}
|
||||
const key = `${env.CHATGPT_RELAY_URL}\n${env.CHATGPT_RELAY_SERVICE_ACCOUNT.length}`
|
||||
if (relayIdTokens?.key !== key) {
|
||||
relayIdTokens = {
|
||||
key,
|
||||
source: createIdTokenSource({
|
||||
serviceAccountJson: env.CHATGPT_RELAY_SERVICE_ACCOUNT,
|
||||
audience: env.CHATGPT_RELAY_URL,
|
||||
...(egressFetch ? { fetch: egressFetch } : {})
|
||||
})
|
||||
}
|
||||
}
|
||||
return relayingFetch({ relayUrl: env.CHATGPT_RELAY_URL, idToken: relayIdTokens.source, ...(egressFetch ? { fetch: egressFetch } : {}) })
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ import {
|
||||
} from './guardrails'
|
||||
import { DEFAULT_EGRESS_ALLOWLIST, DEFAULT_EGRESS_PRESET, KEYS_WITHHELD_BODY, compileNetworkPolicy, egressProxyMissing, gitBasicAuthorization, type CredentialRoute, type EgressPreset, type PlatformHostSuffixesConfigError } from './network-policy'
|
||||
import { applyQueuedRevokes } from './grant-revokes'
|
||||
import { ensureEntryPoints } from './golden-images'
|
||||
import { ENTRY_POINTS_PROBE, ensureEntryPoints, entryPointsBehind, installEntryPoints, parseEntryPointsProbe } from './golden-images'
|
||||
import { RUNTIME_REFRESH_SCRIPT, type RuntimeBundle } from './runtime-bundles'
|
||||
import { memberPlaceholder, type MemberGitIdentity } from './github-user-tokens'
|
||||
import { connectedMembers, memberGitConfigs } from './member-git'
|
||||
@@ -120,7 +120,8 @@ export type LifecycleDeps = {
|
||||
* lists, `touch` when a create booted from an automatic one. Absent: neither happens.
|
||||
*/
|
||||
autoEnvironment?: {
|
||||
ensure(accountId: string, repoUrl: string): Promise<unknown>
|
||||
/** `region`: where the create that asked runs; the environment builds there (a snapshot is region-bound). */
|
||||
ensure(accountId: string, repoUrl: string, region: string): Promise<unknown>
|
||||
touch(environmentId: string): Promise<unknown>
|
||||
}
|
||||
/**
|
||||
@@ -224,12 +225,31 @@ export function phaseTimer(now: () => number = Date.now) {
|
||||
failed: null as { phase: string; ms: number } | null,
|
||||
async time<T>(phase: string, work: () => Promise<T>): Promise<T> {
|
||||
const started = now()
|
||||
const limit = PHASE_LIMITS[phase]
|
||||
let watch: ReturnType<typeof setTimeout> | undefined
|
||||
try {
|
||||
return await work()
|
||||
if (limit === undefined) {
|
||||
return await work()
|
||||
}
|
||||
// A phase with nothing slow in it that still hasn't answered is stuck (a create sat 180 s
|
||||
// before its write-ahead insert with no provider call stalled). Name it in the log; fail it
|
||||
// unless abandoning the work could strand something (the pool claim holds a VM).
|
||||
return await Promise.race([
|
||||
work(),
|
||||
new Promise<never>((_, reject) => {
|
||||
watch = setTimeout(() => {
|
||||
console.log(JSON.stringify({ event: 'phase_timeout', phase, ms: limit.ms, failed: limit.fail }))
|
||||
if (limit.fail) {
|
||||
reject(new LifecycleError('provider_unavailable', `phase_timeout_${phase}`))
|
||||
}
|
||||
}, limit.ms)
|
||||
})
|
||||
])
|
||||
} catch (error) {
|
||||
timer.failed ??= { phase, ms: now() - started }
|
||||
throw error
|
||||
} finally {
|
||||
clearTimeout(watch)
|
||||
timings[phase] = now() - started
|
||||
}
|
||||
}
|
||||
@@ -237,6 +257,17 @@ export function phaseTimer(now: () => number = Date.now) {
|
||||
return timer
|
||||
}
|
||||
|
||||
/**
|
||||
* Bounds for the phases before a create's write-ahead insert (D1 reads, billing, the pool claim).
|
||||
* The VM, policy and orcad phases are bounded by the provider calls inside them.
|
||||
*/
|
||||
export const PHASE_DEADLINES_MS = {
|
||||
environment: { ms: 20_000, fail: true },
|
||||
entitlement: { ms: 20_000, fail: true },
|
||||
pool: { ms: 45_000, fail: false }
|
||||
}
|
||||
const PHASE_LIMITS: Partial<Record<string, { ms: number; fail: boolean }>> = PHASE_DEADLINES_MS
|
||||
|
||||
/**
|
||||
* CREATE-BUILD-SNAPSHOT-404: a VM create from an environment build's snapshot can answer 404 just
|
||||
* after the build (dev, 11-11 and 11-13: the same snapshot booted seconds before and after). Once,
|
||||
@@ -645,7 +676,9 @@ export const MEMBERS_COMMAND = '/opt/orca/bin/orca-sandbox-members'
|
||||
/**
|
||||
* ENV-STALE boot step: a sandbox whose orcad is behind its region's active golden (booted from an environment
|
||||
* build snapshot, or resumed from an older image) installs the golden's runtime bundle before orcad starts.
|
||||
* Best effort and quiet: any failure keeps the old orcad, and every outcome is logged with its timing.
|
||||
* The same command probes the VM's entry points (ENTRY-POINTS-BEFORE-START): scripts older than this
|
||||
* Worker's are installed before the start script runs, so the common case adds no round trip.
|
||||
* Best effort and quiet: any failure keeps the old orcad and scripts, and every outcome is logged with its timing.
|
||||
*/
|
||||
export async function refreshRuntime(deps: LifecycleDeps, row: SandboxRow, sessionId: string): Promise<string | null> {
|
||||
if (!deps.runtimeRefresh) {
|
||||
@@ -654,28 +687,41 @@ export async function refreshRuntime(deps: LifecycleDeps, row: SandboxRow, sessi
|
||||
const started = deps.now()
|
||||
let result = 'error'
|
||||
let target: string | null = null
|
||||
let entryPoints = 'unknown'
|
||||
try {
|
||||
const plan = await deps.runtimeRefresh(row.region, started)
|
||||
if (!plan) {
|
||||
result = 'no_bundle'
|
||||
return result
|
||||
}
|
||||
target = plan.bundle.commit
|
||||
target = plan?.bundle.commit ?? null
|
||||
const run = await deps.provider.runCommand(sessionId, {
|
||||
command: 'bash',
|
||||
args: ['-c', RUNTIME_REFRESH_SCRIPT],
|
||||
env: { WANT: plan.bundle.commit, SHA: plan.bundle.sha256, RT: plan.bundle.runtime, URL: plan.url },
|
||||
args: ['-c', plan ? `${ENTRY_POINTS_PROBE}\n${RUNTIME_REFRESH_SCRIPT}` : ENTRY_POINTS_PROBE],
|
||||
env: plan ? { WANT: plan.bundle.commit, SHA: plan.bundle.sha256, RT: plan.bundle.runtime, URL: plan.url } : {},
|
||||
sudo: true,
|
||||
timeoutMs: 40_000
|
||||
})
|
||||
result = run.stdout.trim().split('\n').pop() ?? 'error'
|
||||
result = plan ? (run.stdout.trim().split('\n').pop() ?? 'error') : 'no_bundle'
|
||||
const image = parseEntryPointsProbe(run.stdout)
|
||||
entryPoints = entryPointsBehind(image) ? 'behind' : image ? 'current' : 'unknown'
|
||||
if (entryPoints === 'behind') {
|
||||
entryPoints = await installEntryPoints(deps, sessionId).then(
|
||||
() => 'installed',
|
||||
() => 'install_failed'
|
||||
)
|
||||
}
|
||||
return result
|
||||
} catch {
|
||||
return result
|
||||
} finally {
|
||||
const [outcome, from] = result.split(' ')
|
||||
console.log(
|
||||
JSON.stringify({ event: 'runtime_refresh', sandboxId: row.id, outcome, from: from ?? null, to: target, ms: deps.now() - started })
|
||||
JSON.stringify({
|
||||
event: 'runtime_refresh',
|
||||
sandboxId: row.id,
|
||||
outcome,
|
||||
from: from ?? null,
|
||||
to: target,
|
||||
entryPoints,
|
||||
ms: deps.now() - started
|
||||
})
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -768,7 +814,13 @@ export async function createSandbox(
|
||||
request: {
|
||||
workspaceId: string
|
||||
machineSize: MachineSize
|
||||
/** Explicit (the caller's or its project's): authoritative, and never moved by the capacity fallback. */
|
||||
region?: string
|
||||
/**
|
||||
* Served regions nearest the caller first (regions.ts nearestRegions): where a create with nothing
|
||||
* pinned starts, and the capacity fallback's order. Absent: the default region, then its fallbacks.
|
||||
*/
|
||||
nearRegions?: readonly string[]
|
||||
repo?: { url: string; ref?: string }
|
||||
/** Boot from exactly this environment build (bld_...). */
|
||||
environmentBuildId?: string
|
||||
@@ -829,7 +881,12 @@ export async function createSandbox(
|
||||
const machineSize = source && !source.auto ? source.machineSize : request.machineSize
|
||||
await timer.time('entitlement', () => requireStart(deps, caller.accountId, caller.userId, machineSize))
|
||||
// A snapshot only boots in its own region: an environment build pins the region it was built in.
|
||||
const region = source?.region ?? request.region ?? deps.config.defaultRegion
|
||||
// Then an explicit region; else the nearest served one (each has a golden image).
|
||||
const region =
|
||||
source?.region ??
|
||||
request.region ??
|
||||
request.nearRegions?.find((candidate) => goldenFor(deps, candidate)) ??
|
||||
deps.config.defaultRegion
|
||||
const bootSnapshotId = source?.snapshotId ?? goldenFor(deps, region)
|
||||
if (!bootSnapshotId) {
|
||||
throw new LifecycleError('conflict', 'region_unavailable')
|
||||
@@ -919,7 +976,7 @@ export async function createSandbox(
|
||||
const work = source?.auto
|
||||
? deps.autoEnvironment.touch(source.environmentId)
|
||||
: !source
|
||||
? deps.autoEnvironment.ensure(caller.accountId, request.repo.url)
|
||||
? deps.autoEnvironment.ensure(caller.accountId, request.repo.url, region)
|
||||
: null
|
||||
if (work) {
|
||||
const settled = work.catch(() => undefined)
|
||||
@@ -1005,7 +1062,7 @@ export async function createSandbox(
|
||||
// Only a create nothing pinned to a region moves on: no explicit region, no environment build.
|
||||
const fallback =
|
||||
options.fallback !== false && !request.region && !source && isRegionalFailure(error)
|
||||
? deps.config.fallbackRegions?.find((candidate) => candidate !== region && goldenFor(deps, candidate))
|
||||
? (request.nearRegions ?? deps.config.fallbackRegions)?.find((candidate) => candidate !== region && goldenFor(deps, candidate))
|
||||
: undefined
|
||||
if (fallback) {
|
||||
console.log(JSON.stringify({ event: 'region_fallback', from: region, to: fallback, error: String(error).slice(0, 200) }))
|
||||
|
||||
@@ -2,15 +2,18 @@
|
||||
// A create claims one instead of booting (saves the provider create, ~2 s p50 / ~4 s p95 measured),
|
||||
// then applies the caller's policy and starts orcad exactly as a fresh VM would. The pooled id becomes
|
||||
// the sandbox id, so the provider name and ownership tag line up without renaming anything.
|
||||
// SANDBOX_POOL_SIZE=0 (the default) turns it off; the kill switch drains it.
|
||||
// SANDBOX_POOL_SIZE=0 (the default) turns it off; the kill switch drains it. Every served region (the
|
||||
// default and its fallbacks, each with a golden image) keeps its own: a create starts in the region
|
||||
// nearest the caller (regions.ts), and a snapshot only boots in its own region.
|
||||
import type { MachineSize } from './entitlements'
|
||||
import { killSwitchOn, type GuardrailConfig } from './guardrails'
|
||||
import type { SandboxProviderApi } from './sandbox-provider'
|
||||
import { ORCAD_PORT, SHARE_PROXY_PORT } from './sandbox-vm-commands'
|
||||
import { SANDBOX_VCPUS } from './machine-size'
|
||||
import { servedRegions } from './regions'
|
||||
|
||||
export type PoolConfig = {
|
||||
/** Ready VMs kept per (machine size, region); only `standard` in the default region is pooled. */
|
||||
/** Ready VMs kept per (machine size, region); only `standard` is pooled, in every served region. */
|
||||
size: number
|
||||
/** Recycle a pooled VM after this long so a claimed session keeps most of its 24 h budget. */
|
||||
maxAgeMs: number
|
||||
@@ -36,11 +39,13 @@ export type PoolDeps = {
|
||||
now: () => number
|
||||
config: {
|
||||
environment: string
|
||||
/** The default region's golden image: the pool warms VMs in the default region only. */
|
||||
/** The default region's golden image. */
|
||||
goldenSnapshotId: string
|
||||
/** Every region's golden image; never deleted with a pooled VM. */
|
||||
goldenSnapshots?: Record<string, string>
|
||||
defaultRegion: string
|
||||
/** Served besides the default (each has a golden image): pooled too. */
|
||||
fallbackRegions?: readonly string[]
|
||||
guardrails?: GuardrailConfig
|
||||
snapshotExpirationMs?: number
|
||||
pool?: PoolConfig
|
||||
@@ -91,7 +96,7 @@ export async function claimPooledVm(
|
||||
snapshotId: string = deps.config.goldenSnapshotId
|
||||
): Promise<PooledVm | null> {
|
||||
const pool = deps.config.pool
|
||||
if (!pool || machineSize !== POOLED_SIZE || region !== deps.config.defaultRegion) {
|
||||
if (!pool || machineSize !== POOLED_SIZE || !servedRegions(deps.config).includes(region)) {
|
||||
return null
|
||||
}
|
||||
return deps.db
|
||||
@@ -141,29 +146,43 @@ async function takeLease(deps: PoolDeps): Promise<boolean> {
|
||||
return taken !== null
|
||||
}
|
||||
|
||||
/** How many warm VMs of one image to keep, and the region it boots in (the image's own). */
|
||||
type PoolTarget = { want: number; region: string }
|
||||
|
||||
/** Each served region's golden image, by region. */
|
||||
function servedGoldens(config: PoolDeps['config']): [region: string, snapshotId: string][] {
|
||||
return servedRegions(config).flatMap((region) => {
|
||||
const golden = region === config.defaultRegion ? config.goldenSnapshotId : config.goldenSnapshots?.[region]
|
||||
return golden ? [[region, golden] as [string, string]] : []
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Images worth a warm VM: the golden image (`size`), plus the active build of every environment in
|
||||
* use lately (BUILD_POOL_SIZE each), so a create from an org environment skips the provider boot too.
|
||||
* Images worth a warm VM: each served region's golden image (`size`), plus the active build of every
|
||||
* environment in use lately (BUILD_POOL_SIZE each, in its build's region), so a create from an org
|
||||
* environment skips the provider boot too.
|
||||
*/
|
||||
async function poolTargets(deps: PoolDeps, size: number): Promise<Map<string, number>> {
|
||||
const targets = new Map<string, number>([[deps.config.goldenSnapshotId, size]])
|
||||
async function poolTargets(deps: PoolDeps, size: number): Promise<Map<string, PoolTarget>> {
|
||||
const goldens = servedGoldens(deps.config)
|
||||
const targets = new Map<string, PoolTarget>(goldens.map(([region, snapshotId]) => [snapshotId, { want: size, region }]))
|
||||
const regions = goldens.map(([region]) => region)
|
||||
const since = deps.now() - HOT_BUILD_WINDOW_MS
|
||||
// Hot = a workspace was created from the environment lately (not merely built). Most recent first,
|
||||
// so the chosen set is stable tick to tick; capped per account and overall.
|
||||
const hot = await deps.db
|
||||
.prepare(
|
||||
`SELECT b.snapshot_id AS snapshot_id, e.account_id AS account_id,
|
||||
`SELECT b.snapshot_id AS snapshot_id, b.region AS region, e.account_id AS account_id,
|
||||
(SELECT MAX(s.created_at) FROM sandboxes s WHERE s.environment_id = e.id) AS used_at
|
||||
FROM environments e JOIN environment_builds b ON b.id = e.active_build_id
|
||||
WHERE e.deleted_at IS NULL AND b.status = 'succeeded' AND b.snapshot_id IS NOT NULL
|
||||
AND b.snapshot_deleted_at IS NULL AND json_extract(b.config, '$.machineSize') = ?
|
||||
AND b.region = ? AND used_at > ?
|
||||
AND b.region IN (${regions.map(() => '?').join(', ') || 'NULL'}) AND used_at > ?
|
||||
ORDER BY used_at DESC LIMIT ?`
|
||||
)
|
||||
// Pooled VMs boot in the default region; a build snapshot from another region cannot boot there.
|
||||
.bind(POOLED_SIZE, deps.config.defaultRegion, since, MAX_HOT_BUILDS * 4)
|
||||
.all<{ snapshot_id: string; account_id: string }>()
|
||||
.catch(() => ({ results: [] as { snapshot_id: string; account_id: string }[] }))
|
||||
// A build's VM boots in the build's own region, which must be served.
|
||||
.bind(POOLED_SIZE, ...regions, since, MAX_HOT_BUILDS * 4)
|
||||
.all<{ snapshot_id: string; region: string; account_id: string }>()
|
||||
.catch(() => ({ results: [] as { snapshot_id: string; region: string; account_id: string }[] }))
|
||||
const perAccount = new Map<string, number>()
|
||||
let hotCount = 0
|
||||
for (const row of hot.results) {
|
||||
@@ -173,7 +192,7 @@ async function poolTargets(deps: PoolDeps, size: number): Promise<Map<string, nu
|
||||
}
|
||||
perAccount.set(row.account_id, count + 1)
|
||||
hotCount++
|
||||
targets.set(row.snapshot_id, Math.max(targets.get(row.snapshot_id) ?? 0, BUILD_POOL_SIZE))
|
||||
targets.set(row.snapshot_id, { want: BUILD_POOL_SIZE, region: row.region })
|
||||
}
|
||||
return targets
|
||||
}
|
||||
@@ -184,7 +203,7 @@ async function maintain(deps: PoolDeps): Promise<PoolTick> {
|
||||
const draining = !pool || (deps.config.guardrails ? await killSwitchOn(deps.db, deps.config.guardrails) : false)
|
||||
const all = (await deps.db.prepare(`SELECT * FROM sandbox_pool WHERE status != 'lease'`).all<PooledVm>()).results
|
||||
const maxAge = pool?.maxAgeMs ?? DEFAULT_POOL_MAX_AGE_MS
|
||||
const targets = draining || !pool ? new Map<string, number>() : await poolTargets(deps, pool.size)
|
||||
const targets = draining || !pool ? new Map<string, PoolTarget>() : await poolTargets(deps, pool.size)
|
||||
const expired = (vm: PooledVm) =>
|
||||
draining ||
|
||||
!targets.has(vm.snapshot_id) ||
|
||||
@@ -197,7 +216,7 @@ async function maintain(deps: PoolDeps): Promise<PoolTick> {
|
||||
for (const vm of keep) {
|
||||
const count = (kept.get(vm.snapshot_id) ?? 0) + 1
|
||||
kept.set(vm.snapshot_id, count)
|
||||
if (count > (targets.get(vm.snapshot_id) ?? 0)) {
|
||||
if (count > (targets.get(vm.snapshot_id)?.want ?? 0)) {
|
||||
extras.push(vm)
|
||||
}
|
||||
}
|
||||
@@ -222,17 +241,19 @@ async function maintain(deps: PoolDeps): Promise<PoolTick> {
|
||||
if (draining || !pool) {
|
||||
return { created: 0, recycled, ready: Math.max(0, live) }
|
||||
}
|
||||
const boots: string[] = []
|
||||
for (const [snapshotId, want] of targets) {
|
||||
const boots: { snapshotId: string; region: string }[] = []
|
||||
for (const [snapshotId, { want, region }] of targets) {
|
||||
const have = Math.min(kept.get(snapshotId) ?? 0, want)
|
||||
for (let i = have; i < want; i++) {
|
||||
boots.push(snapshotId)
|
||||
boots.push({ snapshotId, region })
|
||||
}
|
||||
}
|
||||
if (boots.length === 0) {
|
||||
return { created: 0, recycled, ready: Math.max(0, live) }
|
||||
}
|
||||
const results = await Promise.allSettled(boots.slice(0, MAX_BOOTS_PER_TICK).map((snapshotId) => bootPooledVm(deps, snapshotId)))
|
||||
const results = await Promise.allSettled(
|
||||
boots.slice(0, MAX_BOOTS_PER_TICK).map((boot) => bootPooledVm(deps, boot.snapshotId, boot.region))
|
||||
)
|
||||
const created = results.filter((r) => r.status === 'fulfilled').length
|
||||
for (const failure of results.filter((r) => r.status === 'rejected')) {
|
||||
console.log(JSON.stringify({ event: 'pool_boot_failed', error: String(failure.reason).slice(0, 200) }))
|
||||
@@ -240,14 +261,14 @@ async function maintain(deps: PoolDeps): Promise<PoolTick> {
|
||||
return { created, recycled, ready: live + created }
|
||||
}
|
||||
|
||||
async function bootPooledVm(deps: PoolDeps, snapshotId: string): Promise<void> {
|
||||
async function bootPooledVm(deps: PoolDeps, snapshotId: string, region: string): Promise<void> {
|
||||
const id = poolId()
|
||||
const now = deps.now()
|
||||
const row: PooledVm = {
|
||||
id,
|
||||
provider_name: `orca-${deps.config.environment}-${id}`,
|
||||
machine_size: POOLED_SIZE,
|
||||
region: deps.config.defaultRegion,
|
||||
region,
|
||||
snapshot_id: snapshotId,
|
||||
status: 'warming',
|
||||
session_id: null,
|
||||
|
||||
@@ -7,6 +7,11 @@
|
||||
// provider error. Automatic, not operator-driven: the create has no state yet and a person is
|
||||
// waiting on it, and a regional capacity blip is over long before anyone could flip a var.
|
||||
// VERCEL_FALLBACK_REGIONS="" turns it off.
|
||||
// - a desktop create or a new environment with nothing pinned starts in the served region nearest the
|
||||
// caller (Cloudflare's request geolocation), not always the default: every frame of a remote session
|
||||
// pays the round trip. Served = the default and its fallbacks that have an active golden (and so a
|
||||
// warm pool, pool.ts). The environment records its region, so its builds and workspaces stay there.
|
||||
// VERCEL_NEAREST_REGION=off turns it off.
|
||||
import { ProviderError } from './sandbox-provider'
|
||||
|
||||
export const DEFAULT_REGION = 'pdx1'
|
||||
@@ -39,3 +44,109 @@ export function isRegionalFailure(error: unknown): boolean {
|
||||
}
|
||||
return error instanceof TypeError
|
||||
}
|
||||
|
||||
/** Where each Vercel region is (approximate city coordinates). A region missing here is never "nearest". */
|
||||
const REGION_SITES: Record<string, readonly [latitude: number, longitude: number]> = {
|
||||
iad1: [38.9, -77.0], // Washington, D.C.
|
||||
cle1: [41.5, -81.7], // Cleveland
|
||||
pdx1: [45.5, -122.7], // Portland
|
||||
sfo1: [37.8, -122.4], // San Francisco
|
||||
yul1: [45.5, -73.6], // Montréal
|
||||
gru1: [-23.6, -46.6], // São Paulo
|
||||
dub1: [53.3, -6.3], // Dublin
|
||||
lhr1: [51.5, -0.1], // London
|
||||
cdg1: [48.9, 2.4], // Paris
|
||||
fra1: [50.1, 8.7], // Frankfurt
|
||||
arn1: [59.3, 18.1], // Stockholm
|
||||
cpt1: [-33.9, 18.4], // Cape Town
|
||||
dxb1: [25.2, 55.3], // Dubai
|
||||
bom1: [19.1, 72.9], // Mumbai
|
||||
sin1: [1.4, 103.8], // Singapore
|
||||
hkg1: [22.3, 114.2], // Hong Kong
|
||||
icn1: [37.6, 127.0], // Seoul
|
||||
kix1: [34.7, 135.5], // Osaka
|
||||
hnd1: [35.7, 139.7], // Tokyo
|
||||
syd1: [-33.9, 151.2] // Sydney
|
||||
}
|
||||
|
||||
/** A rough middle of each continent, for a request Cloudflare located only that far. */
|
||||
const CONTINENT_SITES: Record<string, readonly [number, number]> = {
|
||||
NA: [40, -95],
|
||||
SA: [-15, -60],
|
||||
EU: [50, 10],
|
||||
AF: [5, 20],
|
||||
AS: [30, 100],
|
||||
OC: [-25, 140]
|
||||
}
|
||||
|
||||
export type RequestGeo = { latitude?: number; longitude?: number; continent?: string }
|
||||
|
||||
/** Cloudflare's geolocation of the caller (`request.cf`); null outside the Workers runtime or when it has none. */
|
||||
export function requestGeo(request: Request): RequestGeo | null {
|
||||
const cf = (request as { cf?: { latitude?: unknown; longitude?: unknown; continent?: unknown } }).cf
|
||||
if (!cf) {
|
||||
return null
|
||||
}
|
||||
const latitude = Number(cf.latitude)
|
||||
const longitude = Number(cf.longitude)
|
||||
const continent = typeof cf.continent === 'string' ? cf.continent : undefined
|
||||
const located = cf.latitude != null && cf.longitude != null && Number.isFinite(latitude) && Number.isFinite(longitude)
|
||||
if (!located && !continent) {
|
||||
return null
|
||||
}
|
||||
return { ...(located ? { latitude, longitude } : {}), ...(continent ? { continent } : {}) }
|
||||
}
|
||||
|
||||
/** Great-circle distance in km. */
|
||||
function distanceKm(a: readonly [number, number], b: readonly [number, number]): number {
|
||||
const rad = Math.PI / 180
|
||||
const dLat = (b[0] - a[0]) * rad
|
||||
const dLon = (b[1] - a[1]) * rad
|
||||
const h = Math.sin(dLat / 2) ** 2 + Math.cos(a[0] * rad) * Math.cos(b[0] * rad) * Math.sin(dLon / 2) ** 2
|
||||
return 2 * 6371 * Math.asin(Math.min(1, Math.sqrt(h)))
|
||||
}
|
||||
|
||||
/**
|
||||
* `served` (default region first) reordered nearest-first for a caller at `geo`. Unchanged when the
|
||||
* caller can't be located; a region with no known site keeps its place after the located ones.
|
||||
*/
|
||||
export function regionsByDistance(geo: RequestGeo | null, served: readonly string[]): string[] {
|
||||
const at: readonly [number, number] | undefined =
|
||||
geo?.latitude !== undefined && geo.longitude !== undefined
|
||||
? [geo.latitude, geo.longitude]
|
||||
: geo?.continent
|
||||
? CONTINENT_SITES[geo.continent]
|
||||
: undefined
|
||||
if (!at) {
|
||||
return [...served]
|
||||
}
|
||||
const ranked = served.map((region, index) => {
|
||||
const site = REGION_SITES[region]
|
||||
return { region, index, km: site ? distanceKm(at, site) : Number.POSITIVE_INFINITY }
|
||||
})
|
||||
// Stable: equal distances (and unknown sites) keep the configured order, so the default wins ties.
|
||||
ranked.sort((a, b) => a.km - b.km || a.index - b.index)
|
||||
return ranked.map((entry) => entry.region)
|
||||
}
|
||||
|
||||
/** False when VERCEL_NEAREST_REGION=off: new workspaces and environments start in the default region. */
|
||||
export function nearestRegionEnabled(env: { VERCEL_NEAREST_REGION?: string }): boolean {
|
||||
return env.VERCEL_NEAREST_REGION?.trim() !== 'off'
|
||||
}
|
||||
|
||||
/**
|
||||
* The served regions (default first) nearest-first for this request, or undefined (the default region
|
||||
* decides) when the caller can't be located, there is nothing to choose, or the choice is turned off.
|
||||
*/
|
||||
export function nearestRegions(request: Request, served: readonly string[], enabled = true): string[] | undefined {
|
||||
if (!enabled || served.length < 2) {
|
||||
return undefined
|
||||
}
|
||||
const geo = requestGeo(request)
|
||||
return geo ? regionsByDistance(geo, served) : undefined
|
||||
}
|
||||
|
||||
/** The regions new sandboxes may use, default first: each has an active golden image. */
|
||||
export function servedRegions(config: { defaultRegion: string; fallbackRegions?: readonly string[] }): string[] {
|
||||
return [...new Set([config.defaultRegion, ...(config.fallbackRegions ?? [])])]
|
||||
}
|
||||
|
||||
@@ -6,6 +6,8 @@ import { z } from 'zod'
|
||||
import { ProviderError, type SandboxProviderApi } from './sandbox-provider'
|
||||
|
||||
const PROJECT = '/vercel/sandbox/project'
|
||||
/** Written by the VM's memory guard (image/vm/orca-memguard). */
|
||||
const GUARD_STATE = '/run/orca-memguard/state'
|
||||
|
||||
/**
|
||||
* CPU from two /proc/stat samples 250 ms apart, memory from /proc/meminfo, disk for the project
|
||||
@@ -28,9 +30,22 @@ victim=$(sudo -n dmesg 2>/dev/null | grep -oE 'Killed process [0-9]+ \([^)]{1,64
|
||||
[ "$victim" = MainThread ] && victim=node
|
||||
orcad=false; pid=$(cat "$HOME/.orca-sandbox/orcad.pid" 2>/dev/null || echo 0); [ "$pid" -gt 1 ] 2>/dev/null && kill -0 "$pid" 2>/dev/null && orcad=true
|
||||
load=$(awk '{print $1}' /proc/loadavg)
|
||||
printf '{"cpuPercent":%d,"cpus":%d,"load1":%s,"memTotalBytes":%d,"memUsedBytes":%d,"diskTotalBytes":%d,"diskUsedBytes":%d,"uptimeSeconds":%d,"oomKills":%d,"lastOomVictim":%s,"orcadAlive":%s}\n' \
|
||||
# MEMGUARD: the memory guard's state (orca-memguard), when it runs this boot and wrote in the last minute.
|
||||
guard=null
|
||||
if [ -r ${GUARD_STATE} ]; then
|
||||
g() { sed -n "s/^$1=//p" ${GUARD_STATE} | head -n 1; }
|
||||
g_at=$(g at); g_level=$(g level); g_kills=$(g kills); g_name=$(g lastKillName | tr -cd 'A-Za-z0-9._+-' | head -c 40); g_kill_at=$(g lastKillAt)
|
||||
g_swap=$(g swapTotalKiB); [[ "$g_swap" =~ ^[0-9]+$ ]] && g_swap=$((g_swap * 1024)) || g_swap=0
|
||||
now_ms=$(( $(date +%s%3N) ))
|
||||
if [ "$(g bootId)" = "$(cat /proc/sys/kernel/random/boot_id)" ] && [[ "$g_at" =~ ^[0-9]+$ ]] && [ $((now_ms - g_at)) -lt 60000 ] \
|
||||
&& [[ "$g_level" =~ ^(ok|high|critical)$ ]] && [[ "$g_kills" =~ ^[0-9]+$ ]] && [[ "$g_kill_at" =~ ^[0-9]+$ ]]; then
|
||||
guard=$(printf '{"level":"%s","kills":%d,"lastKillName":%s,"lastKillAt":%s,"swapBytes":%d}' "$g_level" "$g_kills" \
|
||||
"$( [ -n "$g_name" ] && printf '"%s"' "$g_name" || printf null )" "$( [ "$g_kill_at" -gt 0 ] && printf '%s' "$g_kill_at" || printf null )" "$g_swap")
|
||||
fi
|
||||
fi
|
||||
printf '{"cpuPercent":%d,"cpus":%d,"load1":%s,"memTotalBytes":%d,"memUsedBytes":%d,"diskTotalBytes":%d,"diskUsedBytes":%d,"uptimeSeconds":%d,"oomKills":%d,"lastOomVictim":%s,"orcadAlive":%s,"guard":%s}\n' \
|
||||
"$cpu_pct" "$(nproc)" "$load" "$mem_total" "$((mem_total-mem_avail))" "\${disk_total:-0}" "\${disk_used:-0}" "$uptime_s" "$oom" \
|
||||
"$( [ -n "$victim" ] && printf '"%s"' "$victim" || printf null )" "$orcad"
|
||||
"$( [ -n "$victim" ] && printf '"%s"' "$victim" || printf null )" "$orcad" "$guard"
|
||||
`.replaceAll('\\${', '${')
|
||||
|
||||
export const VmMetricsSchema = z
|
||||
@@ -51,6 +66,41 @@ export const VmMetricsSchema = z
|
||||
|
||||
export type VmMetrics = z.infer<typeof VmMetricsSchema>
|
||||
|
||||
/** The memory guard's last word: its pressure level, the processes it stopped this boot, its swap. */
|
||||
export const VmGuardSchema = z
|
||||
.object({
|
||||
level: z.enum(['ok', 'high', 'critical']),
|
||||
kills: z.number().int().min(0),
|
||||
lastKillName: z.string().max(40).nullable(),
|
||||
lastKillAt: z.number().int().positive().nullable(),
|
||||
swapBytes: z.number().int().min(0)
|
||||
})
|
||||
.strict()
|
||||
export type VmGuard = z.infer<typeof VmGuardSchema>
|
||||
|
||||
// The probe's line: the metrics plus the guard (null when no guard ran this boot: an older image).
|
||||
const VmProbeSchema = VmMetricsSchema.extend({ guard: VmGuardSchema.nullable().optional() }).strict()
|
||||
|
||||
/** Memory pressure for the status read: the guard's level, else one from MemAvailable alone. */
|
||||
export type WorkspaceMemory = {
|
||||
pressure: 'ok' | 'high' | 'critical'
|
||||
/** Processes the guard stopped this boot (the kernel's own OOM kills are `metrics.oomKills`). */
|
||||
guardKills: number
|
||||
lastGuardVictim: string | null
|
||||
lastGuardKillAt: number | null
|
||||
/** False when the VM runs no guard: pressure is then MemAvailable's, and nothing is stopped early. */
|
||||
guarded: boolean
|
||||
}
|
||||
|
||||
export function workspaceMemory(metrics: VmMetrics, guard: VmGuard | null): WorkspaceMemory {
|
||||
if (guard) {
|
||||
return { pressure: guard.level, guardKills: guard.kills, lastGuardVictim: guard.lastKillName, lastGuardKillAt: guard.lastKillAt, guarded: true }
|
||||
}
|
||||
const availPct = metrics.memTotalBytes > 0 ? ((metrics.memTotalBytes - metrics.memUsedBytes) * 100) / metrics.memTotalBytes : 100
|
||||
const pressure = availPct < 8 ? 'critical' : availPct < 15 ? 'high' : 'ok'
|
||||
return { pressure, guardKills: 0, lastGuardVictim: null, lastGuardKillAt: null, guarded: false }
|
||||
}
|
||||
|
||||
const MAX_TREE = 2000
|
||||
const MAX_CHANGED = 300
|
||||
const MAX_COMMITS = 20
|
||||
@@ -164,8 +214,17 @@ async function runJson<T>(
|
||||
}
|
||||
|
||||
/** Null when the VM did not answer in time (the status panel shows "Unresponsive"). */
|
||||
export function readVmMetrics(provider: SandboxProviderApi, sessionId: string): Promise<VmMetrics | null> {
|
||||
return runJson(provider, sessionId, METRICS_SCRIPT, VmMetricsSchema, 8_000)
|
||||
export async function readVmMetrics(
|
||||
provider: SandboxProviderApi,
|
||||
sessionId: string
|
||||
): Promise<{ metrics: VmMetrics; memory: WorkspaceMemory } | null> {
|
||||
const probe = await runJson(provider, sessionId, METRICS_SCRIPT, VmProbeSchema, 8_000)
|
||||
if (!probe) {
|
||||
return null
|
||||
}
|
||||
// `metrics` keeps its exact shape: desktops parse it strictly.
|
||||
const { guard, ...metrics } = probe
|
||||
return { metrics, memory: workspaceMemory(metrics, guard ?? null) }
|
||||
}
|
||||
|
||||
export function captureArchiveMetadata(provider: SandboxProviderApi, sessionId: string): Promise<ArchiveMetadata | null> {
|
||||
|
||||
@@ -14,6 +14,9 @@ import { SecretsKmsError } from './secrets-kms'
|
||||
import type { SandboxRow } from './store'
|
||||
import { secretLog } from './secret-log'
|
||||
|
||||
/** Session env name the VM env script reads for the agents signed in inside the workspace ('claude,codex'). */
|
||||
export const SIGN_IN_INSIDE_ENV = 'ORCA_SIGN_IN_INSIDE'
|
||||
|
||||
export type SessionSecrets = {
|
||||
env: Record<string, string>
|
||||
/** Row value for `egress_secret_hosts`. */
|
||||
@@ -134,6 +137,10 @@ export async function sessionSecretEnv(
|
||||
const env = withoutSignInCredentials(derived, dropped)
|
||||
const scopes = deriveProviderScopes(stored, env, storedScopes)
|
||||
const egressAllowHosts = providerHosts(env).join(',') || null
|
||||
// Not a secret and never in shells (ORCA_*): tells the VM env script which agents sign in there, so a
|
||||
// first run with no login of its own shows the agent's sign-in screens instead of skipping them.
|
||||
const inside = (Object.keys(modes) as CredentialAgent[]).filter((agent) => modes[agent] === 'inside')
|
||||
const insideEnv: Record<string, string> = inside.length > 0 ? { [SIGN_IN_INSIDE_ENV]: inside.join(',') } : {}
|
||||
// Counts only.
|
||||
secretLog('session_secrets', { sandboxId: row.id, injected: Object.keys(env).length, skipped })
|
||||
if (withhold) {
|
||||
@@ -153,7 +160,7 @@ export async function sessionSecretEnv(
|
||||
}
|
||||
}
|
||||
const hosts = [encodeEgressSecretHosts(withheld.targets), ...accountRoutes].filter(Boolean).join(',') || null
|
||||
return { env: { ...withheld.env, ...accountEnv }, egressSecretHosts: hosts, egressAllowHosts }
|
||||
return { env: { ...withheld.env, ...accountEnv, ...insideEnv }, egressSecretHosts: hosts, egressAllowHosts }
|
||||
}
|
||||
if (!swap) {
|
||||
// An operator turned placeholders off for this account (SECRETS_EGRESS_PLACEHOLDERS=off or accounts:<ids>
|
||||
@@ -162,7 +169,7 @@ export async function sessionSecretEnv(
|
||||
if (plain > 0 && !egressSwapWanted(deps, row) && firstReadThisSession('plain', row)) {
|
||||
secretLog('egress_swap_disabled', { sandboxId: row.id, plainKeys: plain })
|
||||
}
|
||||
return { env, egressSecretHosts: null, egressAllowHosts }
|
||||
return { env: { ...env, ...insideEnv }, egressSecretHosts: null, egressAllowHosts }
|
||||
}
|
||||
const swapped = applySecretPlaceholders(row, env, scopes)
|
||||
const accountEnv: Record<string, string> = {}
|
||||
@@ -175,7 +182,7 @@ export async function sessionSecretEnv(
|
||||
}
|
||||
}
|
||||
const secretHosts = [encodeEgressSecretHosts(swapped.targets), ...accountRoutes].filter(Boolean).join(',') || null
|
||||
return { env: { ...swapped.env, ...accountEnv }, egressSecretHosts: secretHosts, egressAllowHosts }
|
||||
return { env: { ...swapped.env, ...accountEnv, ...insideEnv }, egressSecretHosts: secretHosts, egressAllowHosts }
|
||||
}
|
||||
|
||||
/** Provider hosts from the session's secrets plus the workspace's declared tool hosts (MCP). */
|
||||
|
||||
@@ -22,6 +22,7 @@ import {
|
||||
ArchiveMetadataSchema,
|
||||
captureArchiveMetadata,
|
||||
readVmMetrics,
|
||||
type WorkspaceMemory,
|
||||
type ArchiveMetadata,
|
||||
type VmMetrics
|
||||
} from './sandbox-vm-inspect'
|
||||
@@ -359,6 +360,8 @@ export async function expiredArchives(db: D1Database, now: number): Promise<Sand
|
||||
export type WorkspaceStatus = {
|
||||
state: WorkspaceState
|
||||
metrics: VmMetrics | null
|
||||
/** MEMGUARD: memory pressure and the guard's kills; null when the VM was not probed. */
|
||||
memory: WorkspaceMemory | null
|
||||
/** Since orcad last became ready (this session). */
|
||||
uptimeMs: number | null
|
||||
usage: { totalMs: number; centsPerHour: number; costCents: number }
|
||||
@@ -393,13 +396,15 @@ export async function workspaceStatus(deps: LifecycleDeps, input: SandboxRow, co
|
||||
// A lapsed lease: the provider may have stopped it, which is "sleeping", not "unresponsive".
|
||||
const row = input.status === 'running' ? await confirmRunning(deps, input) : input
|
||||
const probed = row.status === 'running' && row.session_id !== null
|
||||
const metrics = probed ? await readVmMetrics(deps.provider, row.session_id!) : null
|
||||
const probe = probed ? await readVmMetrics(deps.provider, row.session_id!) : null
|
||||
const metrics = probe?.metrics ?? null
|
||||
const now = deps.now()
|
||||
const totalMs = await sandboxUsageMs(deps.db, row, now)
|
||||
const centsPerHour = config.centsPerHour?.[row.machine_size] ?? DEFAULT_CENTS_PER_HOUR[row.machine_size]
|
||||
return {
|
||||
state: stateOf(row, metrics, probed),
|
||||
metrics,
|
||||
memory: probe?.memory ?? null,
|
||||
uptimeMs: row.status === 'running' && row.ready_at !== null ? Math.max(0, now - row.ready_at) : null,
|
||||
usage: { totalMs, centsPerHour, costCents: Math.round((totalMs / 3_600_000) * centsPerHour) },
|
||||
archive: row.archived_at !== null ? { archivedAt: row.archived_at, expiresAt: row.archive_expires_at } : null,
|
||||
|
||||
@@ -42,7 +42,7 @@ function decode(encoded: string | undefined): Record<string, string> {
|
||||
function expectSignedOut(answer: { status: number; body: string }): void {
|
||||
expect(answer.status).toBe(401)
|
||||
expect(JSON.parse(answer.body)).toMatchObject({ type: 'error', error: { type: 'authentication_error' } })
|
||||
expect(JSON.parse(answer.body).error.message).toBe('Your Claude sign-in for Orca Cloud expired. Renew it in Orca: Settings → Agent accounts.')
|
||||
expect(JSON.parse(answer.body).error.message).toBe('Your Claude sign-in for Orca Cloud expired. Renew it in Orca: Settings → Cloud agent sign-ins.')
|
||||
}
|
||||
|
||||
async function setup(overrides: Record<string, string> = {}) {
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
// "Sign in inside this workspace": a per-person default Source under which Orca stores and injects
|
||||
// nothing for that agent, so the agent's own login inside the sandbox reaches the vendor untouched.
|
||||
import { readFileSync, readdirSync } from 'node:fs'
|
||||
import { DatabaseSync } from 'node:sqlite'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { ENV_COMMAND, START_COMMAND } from '../src/sandbox-vm-commands'
|
||||
import { createHarness } from './harness'
|
||||
|
||||
const KEK = Buffer.alloc(32, 7).toString('base64')
|
||||
const PROXY_URL = 'https://sandbox.test/v1/egress/proxy'
|
||||
const SETUP_TOKEN = 'sk-ant-oat01-FAKEsetupTOKEN_for_tests_only_000000000000'
|
||||
const TEAM_KEY = 'sk-ant-api03-TEAM-fake-0000-aaaa'
|
||||
const OPENAI_KEY = 'sk-proj-FAKE-openai-0000-bbbb'
|
||||
|
||||
let logs: string[] = []
|
||||
beforeEach(() => {
|
||||
logs = []
|
||||
for (const method of ['log', 'info', 'warn', 'error', 'debug'] as const) {
|
||||
vi.spyOn(console, method).mockImplementation((...args: unknown[]) => {
|
||||
logs.push(args.map((arg) => (typeof arg === 'string' ? arg : JSON.stringify(arg))).join(' '))
|
||||
})
|
||||
}
|
||||
})
|
||||
afterEach(() => {
|
||||
for (const value of [SETUP_TOKEN, TEAM_KEY, OPENAI_KEY]) {
|
||||
expect(logs.join('\n')).not.toContain(value)
|
||||
}
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
function decode(encoded: string | undefined): Record<string, string> {
|
||||
return Object.fromEntries(
|
||||
(encoded ?? '')
|
||||
.split('\n')
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
const at = line.indexOf('=')
|
||||
return [line.slice(0, at), Buffer.from(line.slice(at + 1), 'base64').toString('utf8')]
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
/** Vendor hosts the agents' own sign-in and model calls use. */
|
||||
const VENDOR_HOSTS = ['api.anthropic.com', 'claude.ai', 'platform.claude.com', 'console.anthropic.com', 'chatgpt.com', 'auth.openai.com', 'api.openai.com']
|
||||
|
||||
async function setup(overrides: Record<string, string> = {}) {
|
||||
const h = await createHarness({
|
||||
SECRETS_KEK: KEK,
|
||||
EGRESS_PROXY_URL: PROXY_URL,
|
||||
SECRETS_EGRESS_PLACEHOLDERS: 'on',
|
||||
AGENT_ACCOUNTS_CLAUDE: 'on',
|
||||
...overrides
|
||||
})
|
||||
const as = (sub: string, org?: string, role?: string) => h.token(sub, org, 'sandbox', role ? { orgRole: role } : {})
|
||||
const lastEnv = (command: string) => decode(h.vercel.commands.filter((c) => c.command === command).at(-1)?.env.ORCA_SECRET_ENV)
|
||||
const policy = () => [...h.vercel.boxes.values()].at(-1)!.policy.allow
|
||||
return { h, as, lastEnv, policy }
|
||||
}
|
||||
|
||||
describe('Source "in-workspace" (sign in inside this workspace)', () => {
|
||||
it('injects nothing for that agent even with a stored account and a team key, and leaves its hosts unruled', async () => {
|
||||
const { h, as, lastEnv, policy } = await setup()
|
||||
const ada = await as('ada', 'acme', 'admin')
|
||||
const mia = await as('mia', 'acme', 'member')
|
||||
await h.call('PUT', '/v1/secrets/org/ANTHROPIC_API_KEY', { bearer: ada, body: { value: TEAM_KEY } })
|
||||
await h.call('PUT', '/v1/secrets/personal/OPENAI_API_KEY', { bearer: mia, body: { value: OPENAI_KEY } })
|
||||
await h.call('PUT', '/v1/agent-credentials/claude', { bearer: mia, body: { token: SETUP_TOKEN } })
|
||||
for (const agent of ['claude', 'codex']) {
|
||||
const put = await h.call('PUT', '/v1/agent-sources', { bearer: mia, body: { agent, source: 'in-workspace' } })
|
||||
expect(put.status).toBe(200)
|
||||
}
|
||||
await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard' }, bearer: mia })
|
||||
const env = lastEnv(START_COMMAND)
|
||||
for (const name of ['CLAUDE_CODE_OAUTH_TOKEN', 'ANTHROPIC_API_KEY', 'OPENAI_API_KEY', 'ORCA_CODEX_AUTH_PLACEHOLDER']) {
|
||||
expect(env[name]).toBeUndefined()
|
||||
}
|
||||
expect(Object.values(env).some((value) => value.startsWith('orca-ph-'))).toBe(false)
|
||||
// The VM env script's cue (never a shell variable): these agents sign in inside the workspace.
|
||||
expect(env.ORCA_SIGN_IN_INSIDE).toBe('claude,codex')
|
||||
// Open egress: "*" plus only hosts that carry rules. No vendor host carries one (no claim, no 403).
|
||||
const allow = policy()
|
||||
expect(allow['*']).toEqual([])
|
||||
for (const host of VENDOR_HOSTS) {
|
||||
expect(allow[host]).toBeUndefined()
|
||||
}
|
||||
const view = (await h.call('GET', '/v1/agent-sources', { bearer: mia })).body
|
||||
expect(view.inWorkspace).toEqual({ claude: true, codex: true })
|
||||
// Older desktops read `defaults` and `effective.source` with two values only: neither ever says 'in-workspace'.
|
||||
expect(view.defaults).toEqual({ claude: null, codex: null })
|
||||
expect(view.effective.claude).toEqual({ source: null, origin: 'default' })
|
||||
})
|
||||
|
||||
it('limited egress still reaches every vendor sign-in and API host unmodified', async () => {
|
||||
const { h, as, policy } = await setup({ SANDBOX_EGRESS_PRESET: 'limited' })
|
||||
const mia = await as('mia')
|
||||
await h.call('PUT', '/v1/agent-credentials/claude', { bearer: mia, body: { token: SETUP_TOKEN } })
|
||||
await h.call('PUT', '/v1/agent-sources', { bearer: mia, body: { agent: 'claude', source: 'in-workspace' } })
|
||||
await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard' }, bearer: mia })
|
||||
const allow = policy()
|
||||
expect(allow['*']).toBeUndefined()
|
||||
const reachable = (host: string) => {
|
||||
const parent = `*.${host.split('.').slice(1).join('.')}`
|
||||
return allow[host] ?? allow[parent]
|
||||
}
|
||||
for (const host of VENDOR_HOSTS) {
|
||||
expect(reachable(host), host).toEqual([])
|
||||
}
|
||||
})
|
||||
|
||||
it('switching a running workspace\'s owner to it drops the account placeholder live; choosing the account again brings it back', async () => {
|
||||
const { h, as, lastEnv } = await setup()
|
||||
const mia = await as('mia')
|
||||
await h.call('PUT', '/v1/agent-credentials/claude', { bearer: mia, body: { token: SETUP_TOKEN } })
|
||||
const created = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard' }, bearer: mia })
|
||||
const id: string = created.body.sandbox.id
|
||||
expect(lastEnv(START_COMMAND).CLAUDE_CODE_OAUTH_TOKEN).toBe(`orca-ph-claude-account-${id}`)
|
||||
await h.call('PUT', '/v1/agent-sources', { bearer: mia, body: { agent: 'claude', source: 'in-workspace' } })
|
||||
expect(lastEnv(ENV_COMMAND).CLAUDE_CODE_OAUTH_TOKEN).toBeUndefined()
|
||||
expect([...h.vercel.boxes.values()][0]!.policy.allow['api.anthropic.com']).toBeUndefined()
|
||||
expect(lastEnv(ENV_COMMAND).ORCA_SIGN_IN_INSIDE).toBe('claude')
|
||||
await h.call('PUT', '/v1/agent-sources', { bearer: mia, body: { agent: 'claude', source: null } })
|
||||
expect(lastEnv(ENV_COMMAND).CLAUDE_CODE_OAUTH_TOKEN).toBe(`orca-ph-claude-account-${id}`)
|
||||
expect(lastEnv(ENV_COMMAND).ORCA_SIGN_IN_INSIDE).toBeUndefined()
|
||||
})
|
||||
|
||||
it('is a per-person default only: a workspace override is refused, and an explicit workspace Source still wins', async () => {
|
||||
const { h, as, lastEnv } = await setup()
|
||||
const mia = await as('mia')
|
||||
await h.call('PUT', '/v1/agent-credentials/claude', { bearer: mia, body: { token: SETUP_TOKEN } })
|
||||
const created = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard' }, bearer: mia })
|
||||
const id: string = created.body.sandbox.id
|
||||
const refused = await h.call('PUT', '/v1/agent-sources', { bearer: mia, body: { agent: 'claude', source: 'in-workspace', workspaceId: id } })
|
||||
expect(refused.status).toBe(400)
|
||||
await h.call('PUT', '/v1/agent-sources', { bearer: mia, body: { agent: 'claude', source: 'sign-in', workspaceId: id } })
|
||||
await h.call('PUT', '/v1/agent-sources', { bearer: mia, body: { agent: 'claude', source: 'in-workspace' } })
|
||||
expect(lastEnv(ENV_COMMAND).CLAUDE_CODE_OAUTH_TOKEN).toBe(`orca-ph-claude-account-${id}`)
|
||||
})
|
||||
|
||||
it('the migration keeps every row written before the table was widened', () => {
|
||||
const dir = fileURLToPath(new URL('../migrations/', import.meta.url).href)
|
||||
const files = readdirSync(dir).filter((name) => name.endsWith('.sql')).sort()
|
||||
const widen = files.indexOf('0063_agent_sources_in_workspace.sql')
|
||||
expect(widen).toBeGreaterThan(0)
|
||||
const raw = new DatabaseSync(':memory:')
|
||||
for (const file of files.slice(0, widen)) {
|
||||
raw.exec(readFileSync(`${dir}${file}`, 'utf8'))
|
||||
}
|
||||
const insert = raw.prepare('INSERT INTO agent_sources (user_id, account_id, scope, agent, source, updated_at) VALUES (?, ?, ?, ?, ?, ?)')
|
||||
insert.run('mia', 'acme', 'default', 'codex', 'key', 1)
|
||||
insert.run('mia', 'acme', 'offer', 'claude', 'declined', 2)
|
||||
insert.run('mia', 'acme', 'sbx-1', 'claude', 'sign-in', 3)
|
||||
expect(() => insert.run('mia', 'acme', 'default', 'claude', 'in-workspace', 4)).toThrow(/CHECK/)
|
||||
raw.exec(readFileSync(`${dir}${files[widen]}`, 'utf8'))
|
||||
insert.run('mia', 'acme', 'default', 'claude', 'in-workspace', 4)
|
||||
expect(raw.prepare('SELECT scope, agent, source, updated_at FROM agent_sources ORDER BY updated_at').all()).toEqual([
|
||||
{ scope: 'default', agent: 'codex', source: 'key', updated_at: 1 },
|
||||
{ scope: 'offer', agent: 'claude', source: 'declined', updated_at: 2 },
|
||||
{ scope: 'sbx-1', agent: 'claude', source: 'sign-in', updated_at: 3 },
|
||||
{ scope: 'default', agent: 'claude', source: 'in-workspace', updated_at: 4 }
|
||||
])
|
||||
expect(() => insert.run('mia', 'acme', 'default', 'claude', 'bogus', 5)).toThrow(/CHECK/)
|
||||
expect(raw.prepare(`SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = 'agent_sources'`).all()).toContainEqual({ name: 'agent_sources_scope' })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,85 @@
|
||||
import { exportPKCS8, generateKeyPair, decodeJwt } from 'jose'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { createIdTokenSource, relayingFetch } from '../src/chatgpt-relay'
|
||||
|
||||
const RELAY = 'https://orca-chatgpt-relay-abc.us-central1.run.app'
|
||||
|
||||
function idTokenFor(exp: number): string {
|
||||
const enc = (o: unknown) => Buffer.from(JSON.stringify(o)).toString('base64url')
|
||||
return `${enc({ alg: 'RS256' })}.${enc({ exp, aud: RELAY })}.sig`
|
||||
}
|
||||
|
||||
async function serviceAccount() {
|
||||
const { privateKey } = await generateKeyPair('RS256', { extractable: true })
|
||||
return JSON.stringify({ client_email: 'relay-invoker@p.iam.gserviceaccount.com', private_key: await exportPKCS8(privateKey) })
|
||||
}
|
||||
|
||||
describe('relayingFetch (CODEX-RELAY)', () => {
|
||||
it('sends chatgpt.com through the relay with an ID token beside, never instead of, the ChatGPT credential', async () => {
|
||||
const seen: { url: string; headers: Headers }[] = []
|
||||
const fetchImpl = (async (input: RequestInfo | URL, init?: RequestInit) => {
|
||||
seen.push({ url: String(input), headers: new Headers(init?.headers) })
|
||||
return new Response('ok')
|
||||
}) as typeof fetch
|
||||
const relayed = relayingFetch({ relayUrl: RELAY, idToken: async () => 'google-id', fetch: fetchImpl })
|
||||
await relayed('https://chatgpt.com/backend-api/codex/models?client_version=0.166.0', { headers: { authorization: 'Bearer chatgpt-real' } })
|
||||
await relayed('https://api.anthropic.com/v1/messages', { headers: { authorization: 'Bearer other' } })
|
||||
expect(seen[0]!.url).toBe(`${RELAY}/backend-api/codex/models?client_version=0.166.0`)
|
||||
expect(seen[0]!.headers.get('authorization')).toBe('Bearer chatgpt-real')
|
||||
expect(seen[0]!.headers.get('x-serverless-authorization')).toBe('Bearer google-id')
|
||||
// Every other host goes direct, with no relay token.
|
||||
expect(seen[1]!.url).toBe('https://api.anthropic.com/v1/messages')
|
||||
expect(seen[1]!.headers.get('x-serverless-authorization')).toBeNull()
|
||||
})
|
||||
|
||||
it('without a relay URL chatgpt.com goes direct, as before', async () => {
|
||||
const urls: string[] = []
|
||||
const fetchImpl = (async (input: RequestInfo | URL) => (urls.push(String(input)), new Response('ok'))) as typeof fetch
|
||||
await relayingFetch({ fetch: fetchImpl })('https://chatgpt.com/backend-api/wham/usage')
|
||||
expect(urls).toEqual(['https://chatgpt.com/backend-api/wham/usage'])
|
||||
})
|
||||
|
||||
it('a relay sign-in failure answers in ChatGPT’s error shape (Codex prints it), never reaching chatgpt.com', async () => {
|
||||
let calls = 0
|
||||
const fetchImpl = (async () => (calls++, new Response('ok'))) as typeof fetch
|
||||
const res = await relayingFetch({
|
||||
relayUrl: RELAY,
|
||||
idToken: async () => {
|
||||
throw new Error('no')
|
||||
},
|
||||
fetch: fetchImpl
|
||||
})('https://chatgpt.com/backend-api/codex/responses', { method: 'POST' })
|
||||
expect(res.status).toBe(503)
|
||||
expect(((await res.json()) as { detail: string }).detail).toContain('ChatGPT')
|
||||
expect(calls).toBe(0)
|
||||
})
|
||||
})
|
||||
|
||||
describe('createIdTokenSource', () => {
|
||||
it('mints with target_audience = the relay URL, and reuses the token until 5 minutes before it expires', async () => {
|
||||
let now = 1_800_000_000_000
|
||||
const assertions: string[] = []
|
||||
const fetchImpl = (async (_input: RequestInfo | URL, init?: RequestInit) => {
|
||||
assertions.push(new URLSearchParams(String(init?.body)).get('assertion')!)
|
||||
return Response.json({ id_token: idTokenFor(Math.floor(now / 1000) + 3600) })
|
||||
}) as typeof fetch
|
||||
const source = createIdTokenSource({ serviceAccountJson: await serviceAccount(), audience: RELAY, fetch: fetchImpl, now: () => now })
|
||||
const first = await source()
|
||||
expect(await source()).toBe(first)
|
||||
expect(assertions).toHaveLength(1)
|
||||
const claims = decodeJwt(assertions[0]!)
|
||||
expect(claims).toMatchObject({ target_audience: RELAY, aud: 'https://oauth2.googleapis.com/token', iss: 'relay-invoker@p.iam.gserviceaccount.com' })
|
||||
now += 56 * 60_000
|
||||
await source()
|
||||
expect(assertions).toHaveLength(2)
|
||||
})
|
||||
|
||||
it('a refused mint throws (the caller answers 503) and the next call tries again', async () => {
|
||||
let ok = false
|
||||
const fetchImpl = (async () => (ok ? Response.json({ id_token: idTokenFor(4e9) }) : new Response('{}', { status: 400 }))) as typeof fetch
|
||||
const source = createIdTokenSource({ serviceAccountJson: await serviceAccount(), audience: RELAY, fetch: fetchImpl })
|
||||
await expect(source()).rejects.toThrow()
|
||||
ok = true
|
||||
await expect(source()).resolves.toContain('.')
|
||||
})
|
||||
})
|
||||
@@ -1,6 +1,6 @@
|
||||
// Codex via the person's Orca account (contract D11, flag S2): the broker sandbox login, the lease refresher,
|
||||
// the placeholder auth.json and the chatgpt.com egress swap. Fake provider; no real OpenAI call.
|
||||
import { SignJWT, createLocalJWKSet, exportJWK, generateKeyPair } from 'jose'
|
||||
import { SignJWT, createLocalJWKSet, exportJWK, exportPKCS8, generateKeyPair } from 'jose'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { BROKER_AUTH_PATH, BROKER_EXIT_PATH, BROKER_START_MARKER, parseDeviceCode } from '../src/codex-broker'
|
||||
import { CODEX_TOKEN_URL, codexPlaceholder, refreshCodexCredential, refreshCodexLeases, setCodexTokenFetch } from '../src/codex-credentials'
|
||||
@@ -83,6 +83,11 @@ async function setup(overrides: Record<string, string> = {}) {
|
||||
egressKeysForIssuer: () => createLocalJWKSet(jwks),
|
||||
credentials: { authorizationFor: async () => null },
|
||||
egressFetch: async (input, init) => {
|
||||
if (String(input) === 'https://oauth2.googleapis.com/token') {
|
||||
// CODEX-RELAY: Google's ID token for the relay (only asked when a relay is configured).
|
||||
const enc = (o: unknown) => Buffer.from(JSON.stringify(o)).toString('base64url')
|
||||
return Response.json({ id_token: `${enc({ alg: 'RS256' })}.${enc({ exp: 4e9 })}.sig` })
|
||||
}
|
||||
upstream.push({ url: String(input), headers: new Headers(init?.headers) })
|
||||
if (edgeBlock.on) {
|
||||
return new Response('<html><body>Unable to load site</body></html>', {
|
||||
@@ -381,7 +386,7 @@ describe('Codex account in workspaces (D10, D13)', () => {
|
||||
// RS-4: still a 403 (a 401 would start Codex's own token refresh), now with words Codex prints.
|
||||
expect(refused.status).toBe(403)
|
||||
expect(JSON.parse(refused.body)).toEqual({
|
||||
detail: 'Orca Cloud can’t use your ChatGPT sign-in here. Check it in Orca: Settings → Agent accounts.'
|
||||
detail: 'Orca Cloud can’t use your ChatGPT sign-in here. Check it in Orca: Settings → Cloud agent sign-ins.'
|
||||
})
|
||||
expect(lastEnv('/opt/orca/bin/orca-sandbox-env').ORCA_CODEX_AUTH_PLACEHOLDER).toBeUndefined()
|
||||
})
|
||||
@@ -491,3 +496,25 @@ describe('CODEX-EGRESS-BLOCKED: chatgpt.com refuses the proxy itself', () => {
|
||||
expect((await proxied(id, 'chatgpt.com', auth)).status).toBe(200)
|
||||
})
|
||||
})
|
||||
|
||||
describe('CODEX-RELAY: chatgpt.com through the Cloud Run relay', () => {
|
||||
it('sends the swapped request to the relay with a Google ID token; the path and the ChatGPT credential are unchanged', async () => {
|
||||
const { privateKey } = await generateKeyPair('RS256', { extractable: true })
|
||||
const RELAY = 'https://orca-chatgpt-relay-x.us-central1.run.app'
|
||||
const { h, as, signIn, proxied, upstream } = await setup({
|
||||
CHATGPT_RELAY_URL: RELAY,
|
||||
CHATGPT_RELAY_SERVICE_ACCOUNT: JSON.stringify({ client_email: 'invoker@p.iam.gserviceaccount.com', private_key: await exportPKCS8(privateKey) })
|
||||
})
|
||||
const mia = await as('mia', 'acme', 'member')
|
||||
await signIn(mia)
|
||||
const created = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard' }, bearer: mia })
|
||||
const id: string = created.body.sandbox.id
|
||||
const res = await proxied(id, 'chatgpt.com', { authorization: `Bearer ${codexPlaceholder({ id })}` }, '/backend-api/codex/models?client_version=0.166.0', 'GET')
|
||||
expect(res.status).toBe(200)
|
||||
const sent = upstream.at(-1)!
|
||||
expect(sent.url).toBe(`${RELAY}/backend-api/codex/models?client_version=0.166.0`)
|
||||
expect(sent.headers.get('authorization')).toBe(`Bearer ${ACCESS}`)
|
||||
expect(sent.headers.get('chatgpt-account-id')).toBe('acct-1')
|
||||
expect(sent.headers.get('x-serverless-authorization')).toMatch(/^Bearer [\w-]+\.[\w-]+\.sig$/)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
// CREATE-STEP-WATCH: creates hung 180 s with no log and no row (n73); the steps before the create log.
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { createStepWatch } from '../src/index'
|
||||
|
||||
describe('CREATE-STEP-WATCH', () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
it('names a step still pending after 10 s, then reports every step once the create starts', async () => {
|
||||
vi.useFakeTimers()
|
||||
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
|
||||
const watch = createStepWatch(true, () => Date.now())
|
||||
let finish: (value: string) => void = () => undefined
|
||||
const body = watch.step('body', new Promise<string>((resolve) => (finish = resolve)))
|
||||
await vi.advanceTimersByTimeAsync(10_000)
|
||||
expect(log).toHaveBeenCalledWith(expect.stringContaining('"step":"body"'))
|
||||
finish('{}')
|
||||
await expect(body).resolves.toBe('{}')
|
||||
watch.done()
|
||||
watch.done()
|
||||
const reports = log.mock.calls.filter(([line]) => String(line).includes('"create_steps"'))
|
||||
expect(reports).toHaveLength(1)
|
||||
expect(String(reports[0]![0])).toContain('"body":10000')
|
||||
})
|
||||
|
||||
it('stays quiet for other routes', async () => {
|
||||
vi.useFakeTimers()
|
||||
const log = vi.spyOn(console, 'log').mockImplementation(() => undefined)
|
||||
const watch = createStepWatch(false)
|
||||
const pending = watch.step('auth', new Promise(() => undefined))
|
||||
await vi.advanceTimersByTimeAsync(20_000)
|
||||
watch.done()
|
||||
expect(log).not.toHaveBeenCalled()
|
||||
void pending
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,69 @@
|
||||
import { execFile } from 'node:child_process'
|
||||
import { createServer } from 'node:http'
|
||||
import type { AddressInfo } from 'node:net'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { EGRESS_DENIED_BODY } from '../src/network-policy'
|
||||
import { judge, parseRunnerOutput, runnerCommand, sha8, type ProbeCase, type Summary } from '../scripts/egress-probe-checks'
|
||||
|
||||
const answer = (summary: Partial<Summary>): Summary => ({ status: 200, denied: false, edgeBlocked: false, ...summary })
|
||||
|
||||
describe('egress probe judge', () => {
|
||||
const swapped: ProbeCase = { name: 'echo', url: 'https://echo/post', expect: { kind: 'swapped', headers: { 'x-api-key': 'stand-in', authorization: 'Bearer keep' } } }
|
||||
|
||||
it('accepts the provider seeing the swapped key and the other header unchanged', () => {
|
||||
expect(judge(swapped, answer({ echoed: { 'x-api-key': sha8('stand-in'), authorization: sha8('Bearer keep') } }))).toEqual([])
|
||||
})
|
||||
|
||||
it('names a header that changed or never arrived, and anything leaked from the hop, never values', () => {
|
||||
const problems = judge(swapped, answer({ echoed: { 'x-api-key': sha8('orca-ph-placeholder') }, leaked: ['vercel-sandbox-oidc-token'] }))
|
||||
expect(problems).toEqual([
|
||||
'echo: the provider saw vercel-sandbox-oidc-token',
|
||||
'echo: header x-api-key arrived with another value',
|
||||
'echo: header authorization did not arrive'
|
||||
])
|
||||
expect(problems.join(' ')).not.toContain('stand-in')
|
||||
})
|
||||
|
||||
it("requires the firewall's own 403 for a fail-closed case", () => {
|
||||
const denied: ProbeCase = { name: 'foreign', url: 'https://x', expect: { kind: 'denied' } }
|
||||
expect(judge(denied, answer({ status: 403, denied: true }))).toEqual([])
|
||||
expect(judge(denied, answer({ status: 403, edgeBlocked: true }))).toHaveLength(1)
|
||||
expect(judge(denied, answer({ status: 401 }))).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('an upstream case fails on the firewall 403, an edge block page or no answer', () => {
|
||||
const upstream: ProbeCase = { name: 'chatgpt', url: 'https://chatgpt.com/x', expect: { kind: 'upstream' } }
|
||||
expect(judge(upstream, answer({ status: 401 }))).toEqual([])
|
||||
expect(judge(upstream, answer({ status: 403, denied: true }))).toEqual(['chatgpt: refused by the firewall'])
|
||||
expect(judge(upstream, answer({ status: 403, edgeBlocked: true }))).toHaveLength(1)
|
||||
expect(judge(upstream, answer({ status: 0 }))).toEqual(['chatgpt: no answer'])
|
||||
})
|
||||
})
|
||||
|
||||
describe('the in-VM runner', () => {
|
||||
it('reports status, the deny body and echoed header digests from env-held values', async () => {
|
||||
// A local stand-in for postman-echo.com: echoes request headers under `headers`.
|
||||
const server = createServer((req, res) => {
|
||||
if (req.url === '/denied') {
|
||||
res.writeHead(403, { 'content-type': 'text/plain' }).end(EGRESS_DENIED_BODY)
|
||||
return
|
||||
}
|
||||
res.writeHead(200, { 'content-type': 'application/json' }).end(JSON.stringify({ headers: { ...req.headers, 'vercel-forwarded-host': 'x' } }))
|
||||
})
|
||||
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
|
||||
const base = `http://127.0.0.1:${(server.address() as AddressInfo).port}`
|
||||
const cases = [
|
||||
{ name: 'echo', url: `${base}/post`, method: 'POST' as const, headers: { 'x-api-key': '$PH', authorization: 'Bearer keep' } },
|
||||
{ name: 'denied', url: `${base}/denied` }
|
||||
]
|
||||
const stdout = await new Promise<string>((resolve, reject) =>
|
||||
execFile('bash', ['-c', runnerCommand(cases)], { env: { ...process.env, PH: 'secret-ph' } }, (error, out) => (error ? reject(error) : resolve(out)))
|
||||
)
|
||||
server.close()
|
||||
expect(stdout).toContain('PRB-END')
|
||||
expect(stdout).not.toContain('secret-ph')
|
||||
const [echo, denied] = parseRunnerOutput(stdout, 2)
|
||||
expect(echo).toEqual({ status: 200, denied: false, edgeBlocked: false, echoed: { 'x-api-key': sha8('secret-ph'), authorization: sha8('Bearer keep') }, leaked: ['vercel-forwarded-host'] })
|
||||
expect(denied).toMatchObject({ status: 403, denied: true })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,52 @@
|
||||
// ENTRY-POINTS-BEFORE-START: a boot from an environment build made on an older golden ran that golden's
|
||||
// orca-sandbox-start (no agent prefetch, no OpenCode first run) until the environment was rebuilt.
|
||||
// refreshRuntime's one command also probes the entry points; older ones are installed before the start.
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { VM_SCRIPTS_STAMP } from '../image/vm-scripts.generated'
|
||||
import { entryPointsBehind } from '../src/golden-images'
|
||||
import { refreshRuntime, type LifecycleDeps } from '../src/lifecycle'
|
||||
import type { SandboxRow } from '../src/store'
|
||||
|
||||
type Call = { args?: string[]; env?: Record<string, string> }
|
||||
|
||||
function setup(probe: string, plan: boolean) {
|
||||
const calls: Call[] = []
|
||||
const runCommand = vi.fn(async (_session: string, call: Call) => {
|
||||
calls.push(call)
|
||||
return calls.length === 1 ? { exitCode: 0, stdout: `${probe}\n${plan ? 'current abc' : ''}`, stderr: '' } : { exitCode: 0, stdout: 'synced\n', stderr: '' }
|
||||
})
|
||||
const deps = {
|
||||
provider: { runCommand },
|
||||
now: () => 0,
|
||||
runtimeRefresh: async () => (plan ? { bundle: { commit: 'abc', sha256: 'f', runtime: '22' }, url: 'https://x' } : null)
|
||||
} as unknown as LifecycleDeps
|
||||
const row = { id: 'sbx_1', region: 'iad1' } as unknown as SandboxRow
|
||||
return { calls, deps, row }
|
||||
}
|
||||
|
||||
describe('ENTRY-POINTS-BEFORE-START', () => {
|
||||
it('installs older entry points before the start, in the same round trip as the runtime check', async () => {
|
||||
const t = setup(`stamp=${VM_SCRIPTS_STAMP - 1}\nmarker=0`, true)
|
||||
await expect(refreshRuntime(t.deps, t.row, 'sess')).resolves.toBe('current abc')
|
||||
expect(t.calls).toHaveLength(2)
|
||||
expect(t.calls[0]!.args!.at(-1)).toContain('stamp=')
|
||||
expect(t.calls[0]!.args!.at(-1)).toContain('SOURCE_COMMIT')
|
||||
expect(t.calls[1]!.args!.at(-1)).toContain('install -o root')
|
||||
})
|
||||
|
||||
it('adds nothing when the image carries this Worker scripts (same stamp or marker)', async () => {
|
||||
for (const probe of [`stamp=${VM_SCRIPTS_STAMP}\nmarker=0`, 'stamp=0\nmarker=1']) {
|
||||
const t = setup(probe, true)
|
||||
await refreshRuntime(t.deps, t.row, 'sess')
|
||||
expect(t.calls).toHaveLength(1)
|
||||
}
|
||||
})
|
||||
|
||||
it('still probes without a runtime bundle, and never downgrades a newer image', async () => {
|
||||
const t = setup(`stamp=${VM_SCRIPTS_STAMP - 1}\nmarker=0`, false)
|
||||
await expect(refreshRuntime(t.deps, t.row, 'sess')).resolves.toBe('no_bundle')
|
||||
expect(t.calls).toHaveLength(2)
|
||||
expect(entryPointsBehind({ stamp: VM_SCRIPTS_STAMP + 1, marker: false })).toBe(false)
|
||||
expect(entryPointsBehind(null)).toBe(false)
|
||||
})
|
||||
})
|
||||
@@ -3,6 +3,7 @@ import { describe, expect, it } from 'vitest'
|
||||
import { createGitHubAppMinter } from '../src/github-app'
|
||||
import { installUrl, isInstallationBound } from '../src/github-connect'
|
||||
import type { SetupNotice } from '../src/github-repo-access'
|
||||
import { FOUND_TTL_MS, MISSING_TTL_MS } from '../src/github-installation-cache'
|
||||
import { storeUserToken } from '../src/github-user-tokens'
|
||||
import { keyringFor } from '../src/secrets'
|
||||
import { createHarness } from './harness'
|
||||
@@ -62,10 +63,11 @@ function fakeGitHub() {
|
||||
if (url.pathname.startsWith('/user/memberships/orgs/') && faults.membership === 'forbidden') {
|
||||
return new Response(null, { status: 403 })
|
||||
}
|
||||
if (url.pathname.startsWith('/users/') && faults.users === 'error') {
|
||||
const anonymousProfile = /^\/users\/[^/]+$/.test(url.pathname)
|
||||
if (anonymousProfile && faults.users === 'error') {
|
||||
return new Response(null, { status: 502 })
|
||||
}
|
||||
if ((url.pathname.startsWith('/user/memberships/orgs/') && faults.membership === 'hang') || (url.pathname.startsWith('/users/') && faults.users === 'hang')) {
|
||||
if ((url.pathname.startsWith('/user/memberships/orgs/') && faults.membership === 'hang') || (anonymousProfile && faults.users === 'hang')) {
|
||||
return new Promise<Response>((_, reject) => init?.signal?.addEventListener('abort', () => reject(init.signal!.reason)))
|
||||
}
|
||||
if (url.pathname === '/user') {
|
||||
@@ -100,6 +102,14 @@ function fakeGitHub() {
|
||||
? Response.json({ id: person.id, login: person.login, type: 'User' })
|
||||
: new Response(null, { status: 404 })
|
||||
}
|
||||
const ownerInstall = /^\/(orgs|users)\/([^/]+)\/installation$/.exec(url.pathname)
|
||||
if (ownerInstall) {
|
||||
const login = decodeURIComponent(ownerInstall[2]!).toLowerCase()
|
||||
const found = installations.find(
|
||||
(i) => i.account.login.toLowerCase() === login && (ownerInstall[1] === 'orgs') === (i.account.type === 'Organization')
|
||||
)
|
||||
return found ? Response.json({ id: found.id, account: found.account }) : new Response(null, { status: 404 })
|
||||
}
|
||||
const repoInstall = /^\/repos\/([^/]+)\/([^/]+)\/installation$/.exec(url.pathname)
|
||||
if (repoInstall) {
|
||||
const found = installations.find((i) => i.repos.includes(`${repoInstall[1]}/${repoInstall[2]}`.toLowerCase()))
|
||||
@@ -203,13 +213,18 @@ const stateOf = (url: string) => new URL(url).searchParams.get('state')!
|
||||
describe('GitHub repo access (cloud-start N2/N3, D2)', () => {
|
||||
it('answers missing with an install URL, public, then granted once an admin installs and binds', async () => {
|
||||
const { h, github, notices, trialNotices, browser } = await setup({ publicRepos: ['https://github.com/acme/site'] })
|
||||
expect((await h.call('GET', `/v1/github/repo-access?repo=${encodeURIComponent('https://github.com/acme/site')}`)).body).toEqual({
|
||||
const site = await h.call('GET', `/v1/github/repo-access?repo=${encodeURIComponent('https://github.com/acme/site')}`)
|
||||
expect(site.body).toEqual({
|
||||
access: 'public',
|
||||
owner: 'acme',
|
||||
repo: 'https://github.com/acme/site',
|
||||
// Clones anonymously, but can't push until the App is on acme (CREATE-GH-STEP).
|
||||
push: 'not_installed',
|
||||
installUrl: expect.stringMatching(/^https:\/\/github\.com\/apps\/orca-cloud-test\/installations\/new\?state=/),
|
||||
// The App isn't on acme yet and alice hasn't connected GitHub: unknown.
|
||||
canInstall: null
|
||||
})
|
||||
expect(HostedGithubRepoAccessResponseSchema.parse(site.body)).toEqual(site.body)
|
||||
const missing = await h.call('GET', `/v1/github/repo-access?repo=${encodeURIComponent(`${REPO}.git`)}&session=${SESSION}`)
|
||||
expect(missing.body).toMatchObject({ access: 'missing', owner: 'acme', repo: REPO })
|
||||
expect(HostedGithubRepoAccessResponseSchema.parse(missing.body)).toEqual(missing.body)
|
||||
@@ -226,9 +241,10 @@ describe('GitHub repo access (cloud-start N2/N3, D2)', () => {
|
||||
expect(notices).toEqual([{ sessionId: SESSION, accountId: 'user:alice', repo: REPO, state: 'done' }])
|
||||
// Billing's trial abuse keys: numeric GitHub owner and user ids of the bind.
|
||||
expect(trialNotices).toEqual([{ accountId: 'user:alice', githubOwnerId: 50, githubUserId: 3 }])
|
||||
expect((await h.call('GET', `/v1/github/repo-access?repo=${encodeURIComponent(REPO)}`)).body).toEqual({ access: 'granted', owner: 'acme', repo: REPO })
|
||||
expect((await h.call('GET', `/v1/github/repo-access?repo=${encodeURIComponent(REPO)}`)).body).toEqual({ access: 'granted', owner: 'acme', repo: REPO, push: 'ok' })
|
||||
// Another Orca account is not granted by alice's binding.
|
||||
expect((await h.call('GET', `/v1/github/repo-access?repo=${encodeURIComponent(REPO)}`, { as: 'bob' })).body.access).toBe('missing')
|
||||
const bob = await h.call('GET', `/v1/github/repo-access?repo=${encodeURIComponent(REPO)}`, { as: 'bob' })
|
||||
expect(bob.body).toMatchObject({ access: 'missing', push: 'not_connected' })
|
||||
expect((await h.call('GET', '/v1/github/repo-access?repo=https%3A%2F%2Fgitlab.com%2Facme%2Fweb')).status).toBe(400)
|
||||
})
|
||||
|
||||
@@ -378,6 +394,8 @@ describe('GitHub repo access (cloud-start N2/N3, D2)', () => {
|
||||
`INSERT INTO github_installations VALUES (900, 'org:acme-team', 50, 'acme', 'Organization', 'u1', 3, 'oauth', 0, NULL)`
|
||||
)
|
||||
.run()
|
||||
// Bound straight in the database (no webhook, no setup return): the cached 404 lasts its TTL.
|
||||
h.advance(MISSING_TTL_MS)
|
||||
expect((await h.call('GET', path, { bearer: CONTROL_PLANE_TOKEN })).body).toEqual({ access: 'granted', url: null })
|
||||
expect((await h.call('GET', path.replace(SESSION, 'bad'), { bearer: CONTROL_PLANE_TOKEN })).status).toBe(400)
|
||||
})
|
||||
@@ -440,7 +458,9 @@ describe('repo-access canInstall (REPO-CAN-INSTALL)', () => {
|
||||
const access = (h: Ctx['h'], repo = REPO, as?: string) => h.call('GET', `/v1/github/repo-access?repo=${encodeURIComponent(repo)}`, as ? { as } : {})
|
||||
const membershipCalls = (ctx: Ctx) => ctx.github.calls.filter((c) => c.startsWith('GET /user/memberships/orgs/'))
|
||||
/** canInstall's reads: owner type and membership. */
|
||||
const github = (ctx: Ctx) => ctx.github.calls.filter((c) => c.startsWith('GET /user/memberships/orgs/') || c.startsWith('GET /users/'))
|
||||
// The anonymous owner read and the membership read (not the app-JWT installation lookups).
|
||||
const github = (ctx: Ctx) =>
|
||||
ctx.github.calls.filter((c) => c.startsWith('GET /user/memberships/orgs/') || (c.startsWith('GET /users/') && !c.endsWith('/installation')))
|
||||
|
||||
/** Connects alice (account user:alice, user alice) as the fake GitHub user behind `code`. */
|
||||
async function connect(ctx: Ctx, code: string, user: { id: number; login: string }, tokens: { accessExpiresAt?: number | null; refresh?: string | null } = {}) {
|
||||
@@ -541,13 +561,23 @@ describe('repo-access canInstall (REPO-CAN-INSTALL)', () => {
|
||||
access: 'public',
|
||||
owner: 'acme',
|
||||
repo: 'https://github.com/acme/site',
|
||||
push: 'not_installed',
|
||||
installUrl: expect.any(String),
|
||||
canInstall: false
|
||||
})
|
||||
await connect(ctx, 'code_admin', { id: 3, login: 'admin-gh' })
|
||||
expect((await access(ctx.h, 'https://github.com/acme/site')).body).toMatchObject({ access: 'public', canInstall: true })
|
||||
// Installed on acme (bound to some other Orca account): the App is there, so no canInstall.
|
||||
ctx.github.installations.push({ id: 900, account: ACME, repos: ['acme/site'] })
|
||||
expect((await access(ctx.h, 'https://github.com/acme/site')).body).toEqual({ access: 'public', owner: 'acme', repo: 'https://github.com/acme/site' })
|
||||
ctx.h.advance(MISSING_TTL_MS)
|
||||
// Pushing still needs alice's account bound to it: the install round trip binds it.
|
||||
expect((await access(ctx.h, 'https://github.com/acme/site')).body).toEqual({
|
||||
access: 'public',
|
||||
owner: 'acme',
|
||||
repo: 'https://github.com/acme/site',
|
||||
push: 'not_connected',
|
||||
installUrl: expect.any(String)
|
||||
})
|
||||
})
|
||||
|
||||
it('granted and requested: absent', async () => {
|
||||
@@ -556,7 +586,7 @@ describe('repo-access canInstall (REPO-CAN-INSTALL)', () => {
|
||||
ctx.github.installations.push({ id: 900, account: ACME, repos: ['acme/web'] })
|
||||
const setupPage = await ctx.browser(`https://sandbox.test/v1/github/setup?installation_id=900&setup_action=install&state=${encodeURIComponent(stateOf(missing.body.installUrl))}`)
|
||||
await ctx.browser(`https://sandbox.test/v1/github/callback?code=code_admin&state=${encodeURIComponent(stateOf(setupPage.location!))}`)
|
||||
expect((await access(ctx.h)).body).toEqual({ access: 'granted', owner: 'acme', repo: REPO })
|
||||
expect((await access(ctx.h)).body).toEqual({ access: 'granted', owner: 'acme', repo: REPO, push: 'ok' })
|
||||
|
||||
// Sam (a member, connected by the request's OAuth round trip) asks an owner: requested.
|
||||
const other = await setup()
|
||||
@@ -564,7 +594,7 @@ describe('repo-access canInstall (REPO-CAN-INSTALL)', () => {
|
||||
const page = await other.browser(`https://sandbox.test/v1/github/setup?setup_action=request&state=${encodeURIComponent(stateOf(samMissing.body.installUrl))}`)
|
||||
other.github.requests.push({ account: { login: 'acme' }, requester: { id: 2 } })
|
||||
await other.browser(`https://sandbox.test/v1/github/callback?code=code_sam&state=${encodeURIComponent(stateOf(page.location!))}`)
|
||||
expect((await access(other.h, REPO, 'sam')).body).toEqual({ access: 'requested', owner: 'acme', repo: REPO })
|
||||
expect((await access(other.h, REPO, 'sam')).body).toEqual({ access: 'requested', owner: 'acme', repo: REPO, push: 'not_installed' })
|
||||
})
|
||||
|
||||
it('a token near expiry is refreshed through the member-git helper before the membership read', async () => {
|
||||
@@ -582,3 +612,129 @@ describe('repo-access canInstall (REPO-CAN-INSTALL)', () => {
|
||||
expect(membershipCalls(ctx)).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
describe('repo-access push reason and installation cache (CREATE-GH-STEP)', () => {
|
||||
type Ctx = Awaited<ReturnType<typeof setup>>
|
||||
const access = (h: Ctx['h'], repo = REPO, as?: string) => h.call('GET', `/v1/github/repo-access?repo=${encodeURIComponent(repo)}`, as ? { as } : {})
|
||||
const installLookups = (ctx: Ctx) => ctx.github.calls.filter((c) => c.endsWith('/installation'))
|
||||
const bindAcme = async (ctx: Ctx, repos: string[]) => {
|
||||
const missing = await access(ctx.h)
|
||||
ctx.github.installations.push({ id: 900, account: ACME, repos })
|
||||
const setupPage = await ctx.browser(`https://sandbox.test/v1/github/setup?installation_id=900&setup_action=install&state=${encodeURIComponent(stateOf(missing.body.installUrl))}`)
|
||||
await ctx.browser(`https://sandbox.test/v1/github/callback?code=code_admin&state=${encodeURIComponent(stateOf(setupPage.location!))}`)
|
||||
}
|
||||
|
||||
it('installed on the owner without this repository: repo_not_selected with the installation’s settings page', async () => {
|
||||
const ctx = await setup({ publicRepos: ['https://github.com/acme/site'] })
|
||||
await bindAcme(ctx, ['acme/web'])
|
||||
const site = await access(ctx.h, 'https://github.com/acme/site')
|
||||
expect(site.body).toEqual({
|
||||
access: 'public',
|
||||
owner: 'acme',
|
||||
repo: 'https://github.com/acme/site',
|
||||
push: 'repo_not_selected',
|
||||
configureUrl: 'https://github.com/organizations/acme/settings/installations/900',
|
||||
// The admin who bound it may change the selection.
|
||||
canInstall: true
|
||||
})
|
||||
expect(HostedGithubRepoAccessResponseSchema.parse(site.body)).toEqual(site.body)
|
||||
// A private one the installation doesn't select: still missing for cloning, same reason.
|
||||
expect((await access(ctx.h, 'https://github.com/acme/api')).body).toMatchObject({ access: 'missing', push: 'repo_not_selected', installUrl: expect.any(String) })
|
||||
})
|
||||
|
||||
it('a personal account’s installation links to the user settings page', async () => {
|
||||
const ctx = await setup({ publicRepos: ['https://github.com/Alice-GH/dotfiles'] })
|
||||
await connectAlice(ctx)
|
||||
expect((await access(ctx.h, 'https://github.com/Alice-GH/dotfiles')).body).toMatchObject({
|
||||
push: 'repo_not_selected',
|
||||
configureUrl: 'https://github.com/settings/installations/901'
|
||||
})
|
||||
})
|
||||
|
||||
async function connectAlice(ctx: Ctx) {
|
||||
const missing = await access(ctx.h, 'https://github.com/Alice-GH/notes')
|
||||
ctx.github.installations.push({ id: 901, account: { id: 7, login: 'Alice-GH', type: 'User' }, repos: ['alice-gh/notes'] })
|
||||
const setupPage = await ctx.browser(`https://sandbox.test/v1/github/setup?installation_id=901&setup_action=install&state=${encodeURIComponent(stateOf(missing.body.installUrl))}`)
|
||||
await ctx.browser(`https://sandbox.test/v1/github/callback?code=code_alice&state=${encodeURIComponent(stateOf(setupPage.location!))}`)
|
||||
}
|
||||
|
||||
it('an installation on the owner bound to another Orca account: not_connected, with an install page', async () => {
|
||||
const ctx = await setup({ publicRepos: ['https://github.com/acme/site'] })
|
||||
await bindAcme(ctx, ['acme/web'])
|
||||
expect((await access(ctx.h, 'https://github.com/acme/site', 'bob')).body).toMatchObject({ access: 'public', push: 'not_connected', installUrl: expect.any(String) })
|
||||
})
|
||||
|
||||
it('polls answer from the cache; installation webhooks drop it so the next poll sees the change', async () => {
|
||||
const ctx = await setup({ publicRepos: ['https://github.com/acme/site'] })
|
||||
await bindAcme(ctx, ['acme/web'])
|
||||
ctx.github.calls.length = 0
|
||||
expect((await access(ctx.h, 'https://github.com/acme/site')).body.push).toBe('repo_not_selected')
|
||||
expect((await access(ctx.h, 'https://github.com/acme/site')).body.push).toBe('repo_not_selected')
|
||||
// One repository lookup for both polls; the bound installation answers the owner without GitHub.
|
||||
expect(installLookups(ctx)).toEqual(['GET /repos/acme/site/installation'])
|
||||
|
||||
// The owner selects acme/site on GitHub: without the webhook the cached 404 would stand for its TTL.
|
||||
ctx.github.installations[0]!.repos.push('acme/site')
|
||||
expect((await access(ctx.h, 'https://github.com/acme/site')).body.push).toBe('repo_not_selected')
|
||||
const added = await ctx.webhook('installation_repositories', {
|
||||
action: 'added',
|
||||
installation: { id: 900, account: ACME },
|
||||
repository_selection: 'selected',
|
||||
repositories_added: [{ full_name: 'acme/site' }],
|
||||
repositories_removed: [],
|
||||
requester: null,
|
||||
sender: { id: 3, type: 'User' }
|
||||
})
|
||||
expect(added.status).toBe(202)
|
||||
expect((await access(ctx.h, 'https://github.com/acme/site')).body).toMatchObject({ access: 'granted', push: 'ok' })
|
||||
|
||||
// Removed again: found installations are cached too, and dropped the same way.
|
||||
ctx.github.installations[0]!.repos = ['acme/web']
|
||||
await ctx.webhook('installation_repositories', {
|
||||
action: 'removed',
|
||||
installation: { id: 900, account: ACME },
|
||||
repository_selection: 'selected',
|
||||
repositories_added: [],
|
||||
repositories_removed: [{ full_name: 'acme/site' }],
|
||||
requester: null,
|
||||
sender: { id: 3, type: 'User' }
|
||||
})
|
||||
expect((await access(ctx.h, 'https://github.com/acme/site')).body.push).toBe('repo_not_selected')
|
||||
})
|
||||
|
||||
it('a cached 404 lapses after its TTL; a found installation after a minute', async () => {
|
||||
const ctx = await setup({ publicRepos: ['https://github.com/acme/site'] })
|
||||
expect((await access(ctx.h, 'https://github.com/acme/site')).body.push).toBe('not_installed')
|
||||
ctx.github.installations.push({ id: 900, account: ACME, repos: ['acme/site'] })
|
||||
expect((await access(ctx.h, 'https://github.com/acme/site')).body.push).toBe('not_installed')
|
||||
ctx.h.advance(MISSING_TTL_MS)
|
||||
expect((await access(ctx.h, 'https://github.com/acme/site')).body.push).toBe('not_connected')
|
||||
ctx.github.calls.length = 0
|
||||
ctx.h.advance(FOUND_TTL_MS - 1)
|
||||
await access(ctx.h, 'https://github.com/acme/site')
|
||||
expect(installLookups(ctx)).toEqual([])
|
||||
ctx.h.advance(1)
|
||||
await access(ctx.h, 'https://github.com/acme/site')
|
||||
expect(installLookups(ctx)).toEqual(['GET /repos/acme/site/installation'])
|
||||
})
|
||||
|
||||
it('the owner lookup failing never fails the answer: not_installed', async () => {
|
||||
const ctx = await setup({ publicRepos: ['https://github.com/acme/site'] })
|
||||
const failing = ctx.github.fetch
|
||||
const h = await createHarness({}, {
|
||||
github: createGitHubAppMinter({
|
||||
appId: '1',
|
||||
privateKeyPkcs8: await exportPKCS8((await generateKeyPair('RS256', { extractable: true })).privateKey),
|
||||
isBound: async () => false,
|
||||
log: () => undefined,
|
||||
fetch: (async (input: RequestInfo | URL, init?: RequestInit) =>
|
||||
/\/(orgs|users)\/[^/]+\/installation$/.test(new URL(String(input)).pathname) ? new Response(null, { status: 502 }) : failing(input, init)) as typeof fetch
|
||||
}),
|
||||
githubFetch: failing,
|
||||
publicRepoCheck: async () => true
|
||||
})
|
||||
const answer = await h.call('GET', `/v1/github/repo-access?repo=${encodeURIComponent('https://github.com/acme/site')}`)
|
||||
expect(answer.status).toBe(200)
|
||||
expect(answer.body).toMatchObject({ access: 'public', push: 'not_installed' })
|
||||
})
|
||||
})
|
||||
|
||||
@@ -88,7 +88,8 @@ export async function createHarness(overrides: Partial<Env> = {}, extraDeps: Par
|
||||
async function call(
|
||||
method: string,
|
||||
path: string,
|
||||
init: { body?: unknown; as?: string; bearer?: string; headers?: Record<string, string> } = {}
|
||||
/** `cf`: Cloudflare's request properties (geolocation), which the Workers runtime sets on a real request. */
|
||||
init: { body?: unknown; as?: string; bearer?: string; headers?: Record<string, string>; cf?: Record<string, unknown> } = {}
|
||||
) {
|
||||
const bearer = init.bearer ?? (await token(init.as ?? 'alice'))
|
||||
const request = new Request(`https://sandbox.test${path}`, {
|
||||
@@ -96,6 +97,9 @@ export async function createHarness(overrides: Partial<Env> = {}, extraDeps: Par
|
||||
headers: { authorization: `Bearer ${bearer}`, 'content-type': 'application/json', ...init.headers },
|
||||
body: init.body === undefined ? undefined : JSON.stringify(init.body)
|
||||
})
|
||||
if (init.cf) {
|
||||
Object.defineProperty(request, 'cf', { value: init.cf })
|
||||
}
|
||||
const response = await handleRequest(request, env, deps)
|
||||
return { status: response.status, body: (await response.json()) as any }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
// MEMGUARD: the VM's memory guard (image/vm/orca-memguard) ranks processes for the kernel's OOM killer
|
||||
// so Orca's runtime and terminals outlive a build that eats the VM's memory, and names the process it
|
||||
// would stop. Run against a fake process table (ORCA_MEMGUARD_PROC); the kill itself is verified in a VM.
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest'
|
||||
|
||||
const GUARD = fileURLToPath(new URL('../image/vm/orca-memguard', import.meta.url).href)
|
||||
const START = fileURLToPath(new URL('../image/vm/orca-sandbox-start', import.meta.url).href)
|
||||
// mapfile -d needs bash 4.4+ (the VM has 5.x; macOS's /bin/bash is 3.2).
|
||||
const bash = ['/opt/homebrew/bin/bash', '/usr/local/bin/bash', '/usr/bin/bash', '/bin/bash'].find(
|
||||
(path) => spawnSync(path, ['-c', '[ "${BASH_VERSINFO[0]}${BASH_VERSINFO[1]}" -ge 44 ]']).status === 0
|
||||
)
|
||||
|
||||
const NODE = '/opt/orca/runtime/runtimes/node-abc/bin/node'
|
||||
let proc = ''
|
||||
|
||||
function fake(pid: number, opts: { ppid: number; adj: number; rssPages: number; argv: string[]; comm?: string }): void {
|
||||
const dir = join(proc, String(pid))
|
||||
mkdirSync(dir, { recursive: true })
|
||||
writeFileSync(join(dir, 'cmdline'), opts.argv.length ? `${opts.argv.join('\0')}\0` : '')
|
||||
writeFileSync(join(dir, 'comm'), `${opts.comm ?? 'x'}\n`)
|
||||
writeFileSync(join(dir, 'stat'), `${pid} (${opts.comm ?? 'x'}) S ${opts.ppid} ${pid} ${pid} 0 -1\n`)
|
||||
writeFileSync(join(dir, 'statm'), `${opts.rssPages * 2} ${opts.rssPages} 100 1 0 1 0\n`)
|
||||
writeFileSync(join(dir, 'oom_score_adj'), `${opts.adj}\n`)
|
||||
}
|
||||
const adj = (pid: number): number => Number(readFileSync(join(proc, String(pid), 'oom_score_adj'), 'utf8').trim())
|
||||
const guard = (command: string) => spawnSync(bash!, [GUARD, command], { env: { PATH: process.env.PATH, ORCA_MEMGUARD_PROC: proc }, encoding: 'utf8' })
|
||||
|
||||
beforeEach(() => {
|
||||
proc = mkdtempSync(join(tmpdir(), 'orca-memguard-proc-'))
|
||||
fake(1, { ppid: 0, adj: 500, rssPages: 3000, argv: ['/run/vercel/share/sandbox-init'] })
|
||||
fake(100, { ppid: 1, adj: 500, rssPages: 2_000_000, argv: [NODE, '/opt/orca/bin/orcad-boot', '--code-cache'], comm: 'MainThread' })
|
||||
fake(101, { ppid: 100, adj: 500, rssPages: 20_000, argv: [NODE, '--max-semi-space-size=4', '/opt/orca/runtime/orcad/daemon-entry.js'] })
|
||||
// A terminal's shell, and a typecheck it started (inherited the shell's rank).
|
||||
fake(102, { ppid: 101, adj: 500, rssPages: 1_000, argv: ['/bin/bash', '-l'], comm: 'bash' })
|
||||
fake(103, { ppid: 102, adj: -500, rssPages: 900_000, argv: ['node', '--max-old-space-size=8192', '/w/node_modules/typescript/bin/tsc', '-p', '.'], comm: 'MainThread' })
|
||||
fake(104, { ppid: 102, adj: 300, rssPages: 40_000, argv: ['/opt/google/chrome/chrome', '--type=renderer'], comm: 'chrome' })
|
||||
fake(105, { ppid: 102, adj: 800, rssPages: 40_000, argv: ['/usr/bin/some-tool'] })
|
||||
fake(106, { ppid: 1, adj: -1000, rssPages: 600, argv: ['sshd: /opt/orca/openssh/sshd'] })
|
||||
fake(107, { ppid: 102, adj: 500, rssPages: 400_000, argv: ['/opt/orca/tools/bin/claude'], comm: 'claude' })
|
||||
fake(108, { ppid: 2, adj: 0, rssPages: 0, argv: [] }) // a kernel thread
|
||||
})
|
||||
afterEach(() => rmSync(proc, { recursive: true, force: true }))
|
||||
|
||||
describe.skipIf(!bash)('orca-memguard', () => {
|
||||
it('protects the runtime, terminals and agents, and makes everything else the first victim', () => {
|
||||
expect(guard('sweep').status).toBe(0)
|
||||
expect([100, 101].map(adj)).toEqual([-900, -900])
|
||||
expect(adj(102)).toBe(-500) // the terminal's shell
|
||||
expect(adj(107)).toBe(-500) // an agent CLI
|
||||
expect(adj(103)).toBe(500) // the shell's typecheck no longer inherits its rank
|
||||
expect(adj(104)).toBe(500)
|
||||
expect(adj(105)).toBe(800) // a process that raised its own rank keeps it
|
||||
expect(adj(106)).toBe(-1000) // what root pinned stays pinned
|
||||
expect(adj(108)).toBe(0) // kernel threads are not touched
|
||||
expect(adj(1)).toBe(500)
|
||||
})
|
||||
|
||||
it('would stop the largest unprotected process and names it by its script', () => {
|
||||
const result = guard('victim')
|
||||
expect(result.status).toBe(0)
|
||||
expect(result.stdout.trim()).toBe(`103 ${900_000 * 4} tsc`)
|
||||
})
|
||||
|
||||
it('names inline code by its interpreter, and leaves small processes to the kernel', () => {
|
||||
fake(103, { ppid: 102, adj: 500, rssPages: 900_000, argv: ['node', '-e', 'const secret = 1'], comm: 'MainThread' })
|
||||
expect(guard('victim').stdout.trim()).toBe(`103 ${900_000 * 4} node`)
|
||||
rmSync(join(proc, '103'), { recursive: true })
|
||||
rmSync(join(proc, '104'), { recursive: true })
|
||||
rmSync(join(proc, '105'), { recursive: true })
|
||||
// Only protected memory is large: nothing worth stopping.
|
||||
expect(guard('victim').status).toBe(1)
|
||||
})
|
||||
|
||||
it('starts from orca-sandbox-start before its already-running exit', () => {
|
||||
const start = readFileSync(START, 'utf8')
|
||||
const launch = start.indexOf('orca-memguard" start')
|
||||
expect(launch).toBeGreaterThan(0)
|
||||
expect(launch).toBeLessThan(start.indexOf('print_ready\n exit 0'))
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,219 @@
|
||||
// Nearest region: a create or a new environment with nothing pinned starts in the served region
|
||||
// nearest the caller (Cloudflare's request geolocation); explicit regions and environments win.
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { refreshStaleEnvironments } from '../src/environment-golden-refresh'
|
||||
import { maintainPool, poolConfigFromEnv, type PoolDeps } from '../src/pool'
|
||||
import { nearestRegions, regionsByDistance, requestGeo, servedRegions } from '../src/regions'
|
||||
import { GOLDEN, createHarness } from './harness'
|
||||
|
||||
const SAN_FRANCISCO = { latitude: '37.7749', longitude: '-122.4194', continent: 'NA', country: 'US' }
|
||||
const NEW_YORK = { latitude: '40.7128', longitude: '-74.0060', continent: 'NA', country: 'US' }
|
||||
const LONDON = { latitude: '51.5072', longitude: '-0.1276', continent: 'EU', country: 'GB' }
|
||||
const TOKYO = { latitude: '35.6762', longitude: '139.6503', continent: 'AS', country: 'JP' }
|
||||
const PDX1_GOLDEN = 'snap_pdx1'
|
||||
const REPO = 'https://github.com/acme/web.git'
|
||||
|
||||
function located(cf: Record<string, unknown>): Request {
|
||||
const request = new Request('https://sandbox.test/v1/sandboxes')
|
||||
Object.defineProperty(request, 'cf', { value: cf })
|
||||
return request
|
||||
}
|
||||
|
||||
describe('regionsByDistance', () => {
|
||||
const served = ['pdx1', 'iad1']
|
||||
it('orders the served regions nearest the caller first', () => {
|
||||
expect(regionsByDistance(requestGeo(located(SAN_FRANCISCO)), served)).toEqual(['pdx1', 'iad1'])
|
||||
expect(regionsByDistance(requestGeo(located(NEW_YORK)), served)).toEqual(['iad1', 'pdx1'])
|
||||
expect(regionsByDistance(requestGeo(located(LONDON)), served)).toEqual(['iad1', 'pdx1'])
|
||||
expect(regionsByDistance(requestGeo(located(TOKYO)), served)).toEqual(['pdx1', 'iad1'])
|
||||
expect(regionsByDistance(requestGeo(located(LONDON)), ['pdx1', 'iad1', 'fra1'])).toEqual(['fra1', 'iad1', 'pdx1'])
|
||||
})
|
||||
|
||||
it('uses the continent when Cloudflare has no coordinates, and keeps the configured order without either', () => {
|
||||
expect(regionsByDistance({ continent: 'EU' }, served)).toEqual(['iad1', 'pdx1'])
|
||||
expect(regionsByDistance({ continent: 'XX' }, served)).toEqual(served)
|
||||
expect(regionsByDistance(null, served)).toEqual(served)
|
||||
})
|
||||
|
||||
it('puts a region with no known site after the located ones, in its configured place', () => {
|
||||
expect(regionsByDistance(requestGeo(located(NEW_YORK)), ['zzz1', 'pdx1', 'iad1'])).toEqual(['iad1', 'pdx1', 'zzz1'])
|
||||
})
|
||||
|
||||
it('reads request.cf, and nothing outside the Workers runtime', () => {
|
||||
expect(requestGeo(located(NEW_YORK))).toEqual({ latitude: 40.7128, longitude: -74.006, continent: 'NA' })
|
||||
expect(requestGeo(located({ continent: 'EU' }))).toEqual({ continent: 'EU' })
|
||||
expect(requestGeo(located({ latitude: 'x', longitude: '1' }))).toBeNull()
|
||||
expect(requestGeo(new Request('https://sandbox.test/'))).toBeNull()
|
||||
})
|
||||
|
||||
it('decides nothing when off, unlocated, or with a single served region', () => {
|
||||
expect(nearestRegions(located(NEW_YORK), served)).toEqual(['iad1', 'pdx1'])
|
||||
expect(nearestRegions(located(NEW_YORK), served, false)).toBeUndefined()
|
||||
expect(nearestRegions(new Request('https://sandbox.test/'), served)).toBeUndefined()
|
||||
expect(nearestRegions(located(NEW_YORK), ['pdx1'])).toBeUndefined()
|
||||
expect(servedRegions({ defaultRegion: 'pdx1', fallbackRegions: ['iad1', 'pdx1'] })).toEqual(['pdx1', 'iad1'])
|
||||
})
|
||||
})
|
||||
|
||||
/** pdx1 the default and iad1 its fallback, each with an active golden (iad1's is the seed, GOLDEN). */
|
||||
async function twoRegions(overrides: Record<string, string> = {}, waitUntil?: (work: Promise<unknown>) => void) {
|
||||
const h = await createHarness({ VERCEL_REGION: 'pdx1', VERCEL_FALLBACK_REGIONS: 'iad1', ...overrides }, waitUntil ? { waitUntil } : {})
|
||||
h.raw
|
||||
.prepare(
|
||||
`INSERT INTO golden_images (id, region, source, status, snapshot_id, created_at, updated_at, activated_at)
|
||||
VALUES ('g1', 'pdx1', 'seed', 'active', ?, 1, 1, 1)`
|
||||
)
|
||||
.run(PDX1_GOLDEN)
|
||||
const box = (id: string) => h.vercel.boxes.get(`orca-test-${id}`)!.params
|
||||
return { h, box }
|
||||
}
|
||||
|
||||
describe('nearest region on create', () => {
|
||||
it('creates in the served region nearest the caller, on that region’s golden', async () => {
|
||||
const { h, box } = await twoRegions()
|
||||
const east = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-ny', machineSize: 'standard' }, cf: NEW_YORK })
|
||||
expect(east.status).toBe(200)
|
||||
expect(east.body.sandbox.region).toBe('iad1')
|
||||
expect(box(east.body.sandbox.id)).toMatchObject({ region: 'iad1', snapshotId: GOLDEN })
|
||||
const west = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-sf', machineSize: 'standard' }, cf: SAN_FRANCISCO })
|
||||
expect(west.body.sandbox.region).toBe('pdx1')
|
||||
expect(box(west.body.sandbox.id)).toMatchObject({ region: 'pdx1', snapshotId: PDX1_GOLDEN })
|
||||
// Unlocated (no request.cf): the default region, as before.
|
||||
const unknown = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-x', machineSize: 'standard' } })
|
||||
expect(unknown.body.sandbox.region).toBe('pdx1')
|
||||
})
|
||||
|
||||
it('keeps an explicit region authoritative', async () => {
|
||||
const { h } = await twoRegions()
|
||||
const pinned = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-1', machineSize: 'standard', region: 'pdx1' }, cf: NEW_YORK })
|
||||
expect(pinned.body.sandbox.region).toBe('pdx1')
|
||||
// Pinned means pinned: a capacity failure there is not moved to the nearer region.
|
||||
h.vercel.failCreateRegions.add('pdx1')
|
||||
const refused = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-2', machineSize: 'standard', region: 'pdx1' }, cf: NEW_YORK })
|
||||
expect(refused.status).not.toBe(200)
|
||||
})
|
||||
|
||||
it('never picks a region without an active golden, and falls back to the next nearest on capacity', async () => {
|
||||
// iad1 is the default here and pdx1 has no image: a caller nearest pdx1 still lands in iad1.
|
||||
const h = await createHarness({ VERCEL_REGION: 'iad1', VERCEL_FALLBACK_REGIONS: 'pdx1' })
|
||||
const west = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-sf', machineSize: 'standard' }, cf: SAN_FRANCISCO })
|
||||
expect(west.body.sandbox.region).toBe('iad1')
|
||||
|
||||
const { h: two } = await twoRegions()
|
||||
two.vercel.failCreateRegions.add('iad1')
|
||||
const moved = await two.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-ny', machineSize: 'standard' }, cf: NEW_YORK })
|
||||
expect(moved.status).toBe(200)
|
||||
expect(moved.body.sandbox.region).toBe('pdx1')
|
||||
})
|
||||
|
||||
it('VERCEL_NEAREST_REGION=off keeps every create in the default region', async () => {
|
||||
const { h } = await twoRegions({ VERCEL_NEAREST_REGION: 'off' })
|
||||
const east = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-ny', machineSize: 'standard' }, cf: NEW_YORK })
|
||||
expect(east.body.sandbox.region).toBe('pdx1')
|
||||
})
|
||||
|
||||
it('resumes a sandbox in the region it was created in, wherever the caller is now', async () => {
|
||||
const { h, box } = await twoRegions()
|
||||
const created = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-ny', machineSize: 'standard' }, cf: NEW_YORK })
|
||||
const id = created.body.sandbox.id
|
||||
expect((await h.call('POST', `/v1/sandboxes/${id}/suspend`, { cf: SAN_FRANCISCO })).status).toBe(200)
|
||||
const resumed = await h.call('POST', `/v1/sandboxes/${id}/resume`, { cf: SAN_FRANCISCO })
|
||||
expect(resumed.status).toBe(200)
|
||||
expect(resumed.body.sandbox.region).toBe('iad1')
|
||||
expect(h.vercel.boxes.size).toBe(1)
|
||||
expect(box(id).region).toBe('iad1')
|
||||
expect((h.raw.prepare('SELECT region FROM sandboxes WHERE id = ?').get(id) as { region: string }).region).toBe('iad1')
|
||||
})
|
||||
})
|
||||
|
||||
describe('nearest region for environments', () => {
|
||||
async function environmentIn(cf: Record<string, unknown>) {
|
||||
const { h, box } = await twoRegions()
|
||||
h.vercel.buildResult = JSON.stringify({ repos: [{ url: REPO, dir: 'acme/web', branch: 'main', commit: 'a'.repeat(40), lockfiles: {} }] })
|
||||
const env = await h.call('POST', '/v1/environments', { body: { name: 'Default', repos: [{ url: 'acme/web' }] }, cf })
|
||||
expect(env.status).toBe(200)
|
||||
const envId = env.body.environment.id as string
|
||||
const started = await h.call('POST', `/v1/environments/${envId}/builds`, { cf: SAN_FRANCISCO })
|
||||
;[...h.vercel.buildCommands.values()].at(-1)!.exitCode = 0
|
||||
const build = (await h.call('GET', `/v1/environments/${envId}/builds/${started.body.build.id}`)).body.build
|
||||
expect(build.status).toBe('succeeded')
|
||||
return { h, box, envId, buildId: build.id as string }
|
||||
}
|
||||
|
||||
it('records the region nearest the admin who creates it, builds there, and boots its workspaces there', async () => {
|
||||
const { h, box, envId, buildId } = await environmentIn(NEW_YORK)
|
||||
expect(h.raw.prepare('SELECT region FROM environments WHERE id = ?').get(envId)).toEqual({ region: 'iad1' })
|
||||
// Built in iad1 from iad1's golden, whoever pressed Build and wherever they are.
|
||||
expect(h.raw.prepare('SELECT region, base_snapshot_id FROM environment_builds WHERE id = ?').get(buildId)).toEqual({ region: 'iad1', base_snapshot_id: GOLDEN })
|
||||
// A teammate on the west coast boots from it in iad1: the snapshot only boots there.
|
||||
const created = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-sf', machineSize: 'standard', repo: { url: REPO } }, cf: SAN_FRANCISCO })
|
||||
expect(created.status).toBe(200)
|
||||
expect(created.body.sandbox).toMatchObject({ region: 'iad1', environmentBuildId: buildId })
|
||||
expect(box(created.body.sandbox.id).region).toBe('iad1')
|
||||
})
|
||||
|
||||
it('compares an environment with its own region’s golden: no refresh churn for an iad1 environment', async () => {
|
||||
const { h, envId } = await environmentIn(NEW_YORK)
|
||||
const deps = {
|
||||
db: h.env.SANDBOX_DB,
|
||||
provider: h.vercel,
|
||||
entitlements: { checkStart: async () => h.billing.decision, reportUsage: async () => h.billing.usageAnswer, listSuspensions: async () => [] },
|
||||
now: h.now,
|
||||
config: { environment: 'test', goldenSnapshotId: PDX1_GOLDEN, goldenSnapshots: { pdx1: PDX1_GOLDEN, iad1: GOLDEN }, region: 'pdx1', regions: ['pdx1', 'iad1'] }
|
||||
}
|
||||
expect(await refreshStaleEnvironments(deps as never, { minIntervalMs: 0 })).toMatchObject({ stale: 0, started: 0 })
|
||||
// An environment from before regions were recorded lives in the default region: its iad1 build is stale.
|
||||
h.raw.prepare('UPDATE environments SET region = NULL WHERE id = ?').run(envId)
|
||||
expect(await refreshStaleEnvironments(deps as never, { minIntervalMs: 0 })).toMatchObject({ stale: 1, started: 1 })
|
||||
expect(h.raw.prepare('SELECT region, base_snapshot_id FROM environment_builds WHERE environment_id = ? ORDER BY number DESC').get(envId)).toEqual({
|
||||
region: 'pdx1',
|
||||
base_snapshot_id: PDX1_GOLDEN
|
||||
})
|
||||
})
|
||||
|
||||
it('an environment made in the default region (west coast admin) stays there', async () => {
|
||||
const { h, envId, buildId } = await environmentIn(SAN_FRANCISCO)
|
||||
expect(h.raw.prepare('SELECT region FROM environments WHERE id = ?').get(envId)).toEqual({ region: 'pdx1' })
|
||||
expect(h.raw.prepare('SELECT region, base_snapshot_id FROM environment_builds WHERE id = ?').get(buildId)).toEqual({ region: 'pdx1', base_snapshot_id: PDX1_GOLDEN })
|
||||
})
|
||||
|
||||
it('an automatic environment lives where the create that made it ran', async () => {
|
||||
const pending: Promise<unknown>[] = []
|
||||
const { h } = await twoRegions({ AUTO_ENVIRONMENTS: 'on' }, (work) => void pending.push(work))
|
||||
const created = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'w1', machineSize: 'standard', repo: { url: REPO } }, cf: NEW_YORK })
|
||||
expect(created.body.sandbox.region).toBe('iad1')
|
||||
while (pending.length > 0) {
|
||||
await Promise.all(pending.splice(0))
|
||||
}
|
||||
expect(h.raw.prepare(`SELECT region FROM environments WHERE kind = 'auto'`).all()).toEqual([{ region: 'iad1' }])
|
||||
expect(h.raw.prepare('SELECT region, base_snapshot_id FROM environment_builds').all()).toEqual([{ region: 'iad1', base_snapshot_id: GOLDEN }])
|
||||
})
|
||||
})
|
||||
|
||||
describe('warm pool per served region', () => {
|
||||
it('keeps warm VMs in every served region, and a create claims one in its nearest region', async () => {
|
||||
const { h } = await twoRegions({ SANDBOX_POOL_SIZE: '1' })
|
||||
const deps: PoolDeps = {
|
||||
db: h.env.SANDBOX_DB,
|
||||
provider: h.vercel,
|
||||
now: h.now,
|
||||
config: {
|
||||
environment: 'test',
|
||||
goldenSnapshotId: PDX1_GOLDEN,
|
||||
goldenSnapshots: { pdx1: PDX1_GOLDEN, iad1: GOLDEN },
|
||||
defaultRegion: 'pdx1',
|
||||
fallbackRegions: ['iad1'],
|
||||
pool: poolConfigFromEnv(h.env)
|
||||
}
|
||||
}
|
||||
expect(await maintainPool(deps)).toMatchObject({ created: 2 })
|
||||
const pool = () => h.raw.prepare(`SELECT id, region, snapshot_id FROM sandbox_pool WHERE status = 'ready' ORDER BY region`).all() as { id: string; region: string; snapshot_id: string }[]
|
||||
expect(pool().map(({ region, snapshot_id }) => ({ region, snapshot_id }))).toEqual([
|
||||
{ region: 'iad1', snapshot_id: GOLDEN },
|
||||
{ region: 'pdx1', snapshot_id: PDX1_GOLDEN }
|
||||
])
|
||||
const warmEast = pool().find((vm) => vm.region === 'iad1')!.id
|
||||
const east = await h.call('POST', '/v1/sandboxes', { body: { workspaceId: 'ws-ny', machineSize: 'standard' }, cf: NEW_YORK })
|
||||
expect(east.body.sandbox).toMatchObject({ id: warmEast, region: 'iad1' })
|
||||
})
|
||||
})
|
||||
@@ -135,3 +135,37 @@ describe('orca-sandbox-env: Claude Code API-key approval (G9)', () => {
|
||||
expect(claude(file).hasCompletedOnboarding).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('orca-sandbox-env: Claude signed in inside the workspace', () => {
|
||||
const claudeJson = (file: (path: string) => string | null) => JSON.parse(file('.claude.json') ?? '{}')
|
||||
|
||||
it('with no login yet, leaves onboarding to Claude so it opens on its own sign-in screens; never a shell variable', () => {
|
||||
const { run, file, sessionEnv } = scratch()
|
||||
run({ ORCA_SIGN_IN_INSIDE: 'claude,codex', FOO: 'bar' })
|
||||
const c = claudeJson(file)
|
||||
expect(c.hasCompletedOnboarding).toBeUndefined()
|
||||
// The other first-run answers still apply.
|
||||
expect(c.bypassPermissionsModeAccepted).toBe(true)
|
||||
expect(sessionEnv()).not.toContain('ORCA_SIGN_IN_INSIDE')
|
||||
})
|
||||
|
||||
it('clears a mark an earlier session made, and marks onboarding done once a login exists here', () => {
|
||||
const { home, run, file } = scratch()
|
||||
run({})
|
||||
expect(claudeJson(file).hasCompletedOnboarding).toBe(true)
|
||||
run({ ORCA_SIGN_IN_INSIDE: 'claude' })
|
||||
expect(claudeJson(file).hasCompletedOnboarding).toBeUndefined()
|
||||
mkdirSync(join(home, '.claude'), { recursive: true })
|
||||
writeFileSync(join(home, '.claude', '.credentials.json'), '{}')
|
||||
run({ ORCA_SIGN_IN_INSIDE: 'claude' })
|
||||
expect(claudeJson(file).hasCompletedOnboarding).toBe(true)
|
||||
})
|
||||
|
||||
it('only Codex inside, or a malformed value, keeps Claude onboarding done', () => {
|
||||
const { run, file } = scratch()
|
||||
run({ ORCA_SIGN_IN_INSIDE: 'codex' })
|
||||
expect(claudeJson(file).hasCompletedOnboarding).toBe(true)
|
||||
run({ ORCA_SIGN_IN_INSIDE: 'claude;rm -rf /' })
|
||||
expect(claudeJson(file).hasCompletedOnboarding).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
// PHASE-DEADLINES: a create sat 180 s at "Start sandbox" before its write-ahead insert, with no
|
||||
// provider call stalled. The pre-insert phases are bounded and a stuck one is named in the log.
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { LifecycleError, PHASE_DEADLINES_MS, phaseTimer } from '../src/lifecycle'
|
||||
|
||||
const never = () => new Promise<never>(() => undefined)
|
||||
|
||||
describe('PHASE-DEADLINES', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
vi.spyOn(console, 'log').mockImplementation(() => undefined)
|
||||
})
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
it('fails a stuck entitlement phase by name and logs it', async () => {
|
||||
const timer = phaseTimer(() => Date.now())
|
||||
const run = timer.time('entitlement', never)
|
||||
const caught = run.catch((error: unknown) => error)
|
||||
await vi.advanceTimersByTimeAsync(PHASE_DEADLINES_MS.entitlement.ms)
|
||||
const error = await caught
|
||||
expect(error).toBeInstanceOf(LifecycleError)
|
||||
expect((error as LifecycleError).code).toBe('provider_unavailable')
|
||||
expect((error as LifecycleError).reason).toBe('phase_timeout_entitlement')
|
||||
expect(timer.failed?.phase).toBe('entitlement')
|
||||
expect(console.log).toHaveBeenCalledWith(expect.stringContaining('"event":"phase_timeout"'))
|
||||
})
|
||||
|
||||
it('only logs a slow pool claim: abandoning it could strand the claimed VM', async () => {
|
||||
const timer = phaseTimer(() => Date.now())
|
||||
let finish: (value: string) => void = () => undefined
|
||||
const run = timer.time('pool', () => new Promise<string>((resolve) => (finish = resolve)))
|
||||
await vi.advanceTimersByTimeAsync(PHASE_DEADLINES_MS.pool.ms + 1_000)
|
||||
expect(console.log).toHaveBeenCalledWith(expect.stringContaining('"phase":"pool"'))
|
||||
finish('vm')
|
||||
await expect(run).resolves.toBe('vm')
|
||||
expect(timer.failed).toBeNull()
|
||||
})
|
||||
|
||||
it('leaves a fast phase and the provider-bounded phases alone', async () => {
|
||||
const timer = phaseTimer(() => Date.now())
|
||||
await expect(timer.time('environment', async () => 1)).resolves.toBe(1)
|
||||
const vm = timer.time('vm', () => new Promise<number>((resolve) => setTimeout(() => resolve(2), 90_000)))
|
||||
await vi.advanceTimersByTimeAsync(90_000)
|
||||
await expect(vm).resolves.toBe(2)
|
||||
expect(console.log).not.toHaveBeenCalled()
|
||||
expect(vi.getTimerCount()).toBe(0)
|
||||
})
|
||||
})
|
||||
@@ -154,6 +154,28 @@ describe('status panel', () => {
|
||||
expect(status.metrics).toMatchObject({ oomKills: 1, lastOomVictim: 'node' })
|
||||
})
|
||||
|
||||
it('reports memory pressure and guard kills beside the metrics, which keep their strict shape', async () => {
|
||||
const { h, id } = await running()
|
||||
const guard = { level: 'high', kills: 2, lastKillName: 'tsc', lastKillAt: 1_800_000_000_000, swapBytes: 2 ** 31 }
|
||||
h.vercel.onScript = (script) => (script === METRICS_SCRIPT ? { exitCode: 0, stdout: `${metricsLine({ guard })}\n`, stderr: '' } : undefined)
|
||||
const body = (await h.call('GET', `/v1/sandboxes/${id}/status`)).body
|
||||
expect(body.status.metrics).not.toHaveProperty('guard')
|
||||
const { status } = HostedWorkspaceStatusResponseSchema.parse(body)
|
||||
expect(status.state).toBe('ready')
|
||||
expect(status.memory).toEqual({ pressure: 'high', guardKills: 2, lastGuardVictim: 'tsc', lastGuardKillAt: 1_800_000_000_000, guarded: true })
|
||||
})
|
||||
|
||||
it('derives pressure from MemAvailable when the VM runs no guard (an older image)', async () => {
|
||||
const { h, id } = await running()
|
||||
h.vercel.onScript = (script) =>
|
||||
script === METRICS_SCRIPT ? { exitCode: 0, stdout: `${metricsLine({ memUsedBytes: 7.5e9, guard: null })}\n`, stderr: '' } : undefined
|
||||
const { status } = HostedWorkspaceStatusResponseSchema.parse((await h.call('GET', `/v1/sandboxes/${id}/status`)).body)
|
||||
expect(status.memory).toEqual({ pressure: 'critical', guardKills: 0, lastGuardVictim: null, lastGuardKillAt: null, guarded: false })
|
||||
h.vercel.onScript = (script) => (script === METRICS_SCRIPT ? { exitCode: 0, stdout: `${metricsLine()}\n`, stderr: '' } : undefined)
|
||||
const again = HostedWorkspaceStatusResponseSchema.parse((await h.call('GET', `/v1/sandboxes/${id}/status`)).body)
|
||||
expect(again.status.memory?.pressure).toBe('ok')
|
||||
})
|
||||
|
||||
it('counts settled sessions in the cost and does not probe a sleeping VM', async () => {
|
||||
const { h, id } = await running()
|
||||
h.advance(60 * 60_000)
|
||||
|
||||
@@ -23,6 +23,7 @@
|
||||
"include": [
|
||||
"src",
|
||||
"test",
|
||||
"image"
|
||||
"image",
|
||||
"scripts"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -78,6 +78,9 @@
|
||||
// person in the team" check before a workspace changes owner. Unset = hand-offs refused.
|
||||
"AUTH_INTERNAL_ORIGIN": "https://auth-staging.onorca.dev",
|
||||
"EGRESS_PROXY_URL": "https://orca-sandbox-control-dev.lovecast.workers.dev/v1/egress/proxy",
|
||||
// CODEX-RELAY: chatgpt.com refuses Workers' egress; its requests go through this Cloud Run relay
|
||||
// (cloud/infra/chatgpt-relay, staging). Secret CHATGPT_RELAY_SERVICE_ACCOUNT holds its invoker key.
|
||||
"CHATGPT_RELAY_URL": "https://orca-chatgpt-relay-gjzz5mc7ka-uc.a.run.app",
|
||||
// SEC-3 phase 1: the secret store's KEK comes out of Cloud KMS (src/secrets-kms.ts); the Worker
|
||||
// holds only the KMS-wrapped KEK (secret SECRETS_KEK_WRAPPED, AAD orca-secrets-kek:<SECRETS_KEK_ID>)
|
||||
// and a service account that can only decrypt with this key (secret SECRETS_KMS_SERVICE_ACCOUNT).
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
.terraform/
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
# This file is maintained automatically by "terraform init".
|
||||
# Manual edits may be lost in future updates.
|
||||
|
||||
provider "registry.terraform.io/hashicorp/google" {
|
||||
version = "6.50.0"
|
||||
constraints = "~> 6.0"
|
||||
hashes = [
|
||||
"h1:79CwMTsp3Ud1nOl5hFS5mxQHyT0fGVye7pqpU0PPlHI=",
|
||||
"zh:1f3513fcfcbf7ca53d667a168c5067a4dd91a4d4cccd19743e248ff31065503c",
|
||||
"zh:3da7db8fc2c51a77dd958ea8baaa05c29cd7f829bd8941c26e2ea9cb3aadc1e5",
|
||||
"zh:3e09ac3f6ca8111cbb659d38c251771829f4347ab159a12db195e211c76068bb",
|
||||
"zh:7bb9e41c568df15ccf1a8946037355eefb4dfb4e35e3b190808bb7c4abae547d",
|
||||
"zh:81e5d78bdec7778e6d67b5c3544777505db40a826b6eb5abe9b86d4ba396866b",
|
||||
"zh:8d309d020fb321525883f5c4ea864df3d5942b6087f6656d6d8b3a1377f340fc",
|
||||
"zh:93e112559655ab95a523193158f4a4ac0f2bfed7eeaa712010b85ebb551d5071",
|
||||
"zh:d3efe589ffd625b300cef5917c4629513f77e3a7b111c9df65075f76a46a63c7",
|
||||
"zh:d4a4d672bbef756a870d8f32b35925f8ce2ef4f6bbd5b71a3cb764f1b6c85421",
|
||||
"zh:e13a86bca299ba8a118e80d5f84fbdd708fe600ecdceea1a13d4919c068379fe",
|
||||
"zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
|
||||
"zh:fec30c095647b583a246c39d557704947195a1b7d41f81e369ba377d997faef6",
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
# ChatGPT relay (CODEX-RELAY)
|
||||
|
||||
chatgpt.com answers Cloudflare Workers' egress with a 403 "Unable to load site" page, so Codex via the
|
||||
Orca account can't reach it through the sandbox-control egress proxy directly. The proxy sends its
|
||||
chatgpt.com requests to this Cloud Run service instead (`cloud/apps/chatgpt-relay`), which:
|
||||
|
||||
- forwards to `https://chatgpt.com` only, on the Codex paths in `sandbox-control/src/codex-paths.ts`;
|
||||
- streams responses as they arrive (SSE), refuses WebSocket upgrades (Codex is configured for HTTPS);
|
||||
- logs method, a two-segment path, status and duration: never headers, bodies or queries;
|
||||
- is callable only by `orca-chatgpt-relay-invoker` (Cloud Run IAM, ID token in `X-Serverless-Authorization`).
|
||||
|
||||
## Staging (serves orca-sandbox-control-dev)
|
||||
|
||||
```
|
||||
terraform init -backend-config=backend/staging.hcl
|
||||
terraform apply -var-file=environments/staging.tfvars # registry + accounts
|
||||
IMG=$(CLOUDSDK_CORE_ACCOUNT=<you> ../../apps/chatgpt-relay/build-image.sh onorca-cloud-staging)
|
||||
terraform apply -var-file=environments/staging.tfvars -var image=$IMG # the service
|
||||
```
|
||||
|
||||
Worker: `CHATGPT_RELAY_URL` (wrangler var) = the `relay_url` output; secret `CHATGPT_RELAY_SERVICE_ACCOUNT` =
|
||||
a key of `invoker_service_account`, created with `gcloud iam service-accounts keys create` into a 0600 temp
|
||||
file, piped into `wrangler secret put`, then deleted. Without both, chatgpt.com goes direct, as before.
|
||||
|
||||
## Production (runbook, at Neil's go)
|
||||
|
||||
1. Add `backend/production.hcl` (bucket `onorca-cloud-terraform-state`, prefix `terraform/chatgpt-relay`) and
|
||||
`environments/production.tfvars` (`project_id = "onorca-cloud"`, `environment = "production"`).
|
||||
2. The same three steps against `onorca-cloud`.
|
||||
3. Set `CHATGPT_RELAY_URL` in `wrangler.prod.jsonc` and the secret on `orca-sandbox-control-prod`; deploy.
|
||||
4. Check: a Codex turn via the Orca account answers; the relay's logs show `relay ... status 200`.
|
||||
@@ -0,0 +1,2 @@
|
||||
bucket = "onorca-cloud-staging-terraform-state"
|
||||
prefix = "terraform/chatgpt-relay"
|
||||
@@ -0,0 +1,3 @@
|
||||
# Serves orca-sandbox-control-dev (CHATGPT_RELAY_URL). Prod: a runbook step, not applied from here yet.
|
||||
project_id = "onorca-cloud-staging"
|
||||
environment = "staging"
|
||||
@@ -0,0 +1,104 @@
|
||||
# CODEX-RELAY: the ChatGPT relay. chatgpt.com refuses Cloudflare Workers' egress, so the sandbox-control
|
||||
# Worker's egress proxy sends chatgpt.com requests through this Cloud Run service (cloud/apps/chatgpt-relay).
|
||||
# It forwards to chatgpt.com only, on the Codex paths the Worker allows, and streams the answers back.
|
||||
# Only the invoker service account (held by the Worker as CHATGPT_RELAY_SERVICE_ACCOUNT) may call it.
|
||||
# Own root and state (backend prefix terraform/chatgpt-relay). The invoker's key is created with gcloud and
|
||||
# piped straight into the Worker secret: no key material in this root or its state.
|
||||
|
||||
locals {
|
||||
labels = {
|
||||
app = "orca-cloud"
|
||||
component = "chatgpt-relay"
|
||||
environment = var.environment
|
||||
managed_by = "terraform"
|
||||
}
|
||||
}
|
||||
|
||||
resource "google_project_service" "run" {
|
||||
project = var.project_id
|
||||
service = "run.googleapis.com"
|
||||
disable_on_destroy = false
|
||||
}
|
||||
|
||||
resource "google_project_service" "artifactregistry" {
|
||||
project = var.project_id
|
||||
service = "artifactregistry.googleapis.com"
|
||||
disable_on_destroy = false
|
||||
}
|
||||
|
||||
resource "google_artifact_registry_repository" "relay" {
|
||||
project = var.project_id
|
||||
location = var.region
|
||||
repository_id = "orca-chatgpt-relay"
|
||||
format = "DOCKER"
|
||||
labels = local.labels
|
||||
|
||||
depends_on = [google_project_service.artifactregistry]
|
||||
}
|
||||
|
||||
# The service runs as an account with no grants: it needs nothing in Google Cloud.
|
||||
resource "google_service_account" "runtime" {
|
||||
project = var.project_id
|
||||
account_id = "orca-chatgpt-relay"
|
||||
display_name = "ChatGPT relay runtime (no grants)"
|
||||
}
|
||||
|
||||
resource "google_service_account" "invoker" {
|
||||
project = var.project_id
|
||||
account_id = "orca-chatgpt-relay-invoker"
|
||||
display_name = "sandbox-control Worker: run.invoker on the ChatGPT relay only"
|
||||
description = "Held by the sandbox-control Worker as CHATGPT_RELAY_SERVICE_ACCOUNT. Its only grant is run.invoker on orca-chatgpt-relay."
|
||||
}
|
||||
|
||||
resource "google_cloud_run_v2_service" "relay" {
|
||||
count = var.image == "" ? 0 : 1
|
||||
project = var.project_id
|
||||
name = "orca-chatgpt-relay"
|
||||
location = var.region
|
||||
ingress = "INGRESS_TRAFFIC_ALL"
|
||||
labels = local.labels
|
||||
|
||||
deletion_protection = false
|
||||
|
||||
template {
|
||||
service_account = google_service_account.runtime.email
|
||||
# Codex turns stream for minutes.
|
||||
timeout = "3600s"
|
||||
max_instance_request_concurrency = 80
|
||||
|
||||
scaling {
|
||||
min_instance_count = 0
|
||||
max_instance_count = 4
|
||||
}
|
||||
|
||||
containers {
|
||||
image = var.image
|
||||
|
||||
resources {
|
||||
limits = {
|
||||
cpu = "1"
|
||||
memory = "512Mi"
|
||||
}
|
||||
# SSE streams: keep CPU while a response is still being written.
|
||||
cpu_idle = false
|
||||
}
|
||||
|
||||
startup_probe {
|
||||
http_get {
|
||||
path = "/health"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
depends_on = [google_project_service.run]
|
||||
}
|
||||
|
||||
resource "google_cloud_run_v2_service_iam_member" "invoker" {
|
||||
count = var.image == "" ? 0 : 1
|
||||
project = var.project_id
|
||||
location = var.region
|
||||
name = google_cloud_run_v2_service.relay[0].name
|
||||
role = "roles/run.invoker"
|
||||
member = "serviceAccount:${google_service_account.invoker.email}"
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
output "relay_url" {
|
||||
description = "CHATGPT_RELAY_URL for the sandbox-control Worker."
|
||||
value = var.image == "" ? null : google_cloud_run_v2_service.relay[0].uri
|
||||
}
|
||||
|
||||
output "image_repository" {
|
||||
value = "${var.region}-docker.pkg.dev/${var.project_id}/${google_artifact_registry_repository.relay.repository_id}"
|
||||
}
|
||||
|
||||
output "invoker_service_account" {
|
||||
value = google_service_account.invoker.email
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
variable "project_id" {
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "environment" {
|
||||
type = string
|
||||
}
|
||||
|
||||
variable "region" {
|
||||
type = string
|
||||
default = "us-central1"
|
||||
}
|
||||
|
||||
variable "image" {
|
||||
description = "Relay image (cloud/apps/chatgpt-relay/build-image.sh prints it). Empty: the root creates only the registry and accounts."
|
||||
type = string
|
||||
default = ""
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
terraform {
|
||||
required_version = ">= 1.7.0"
|
||||
|
||||
required_providers {
|
||||
google = {
|
||||
source = "hashicorp/google"
|
||||
version = "~> 6.0"
|
||||
}
|
||||
}
|
||||
|
||||
backend "gcs" {}
|
||||
}
|
||||
|
||||
provider "google" {
|
||||
project = var.project_id
|
||||
}
|
||||
Generated
+12
@@ -235,6 +235,18 @@ importers:
|
||||
specifier: ^4.157.0
|
||||
version: 4.157.0(@cloudflare/workers-types@5.20261104.1)(@types/node@24.19.2)
|
||||
|
||||
apps/chatgpt-relay:
|
||||
devDependencies:
|
||||
'@types/node':
|
||||
specifier: ^24.19.2
|
||||
version: 24.19.2
|
||||
typescript:
|
||||
specifier: ^7.0.2
|
||||
version: 7.0.2
|
||||
vitest:
|
||||
specifier: ^5.0.5
|
||||
version: 5.0.5(@types/node@24.19.2)(vite@8.3.4(@types/node@24.19.2)(esbuild@0.28.2)(tsx@4.23.15))
|
||||
|
||||
apps/push:
|
||||
dependencies:
|
||||
'@hono/node-server':
|
||||
|
||||
@@ -49,6 +49,12 @@ const NON_UNIT_TEST_RUNNERS = [
|
||||
sources: [`cloud/apps/${app}/vitest.config.ts`],
|
||||
mirrors: ["include: ['src/**/*.test.ts']"]
|
||||
})),
|
||||
{
|
||||
name: 'cloud chatgpt-relay',
|
||||
include: ['cloud/apps/chatgpt-relay/test/**/*.test.ts'],
|
||||
sources: ['cloud/apps/chatgpt-relay/vitest.config.ts'],
|
||||
mirrors: ["include: ['test/**/*.test.ts']"]
|
||||
},
|
||||
{
|
||||
name: 'cloud relay-edge',
|
||||
include: ['cloud/apps/relay-edge/test/**/*.test.ts'],
|
||||
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env bash
|
||||
# Premerge step for image changes (premerge-check.sh runs it only when cloud/apps/sandbox-control/image/** changed):
|
||||
# provisions the golden image in Docker and boots it twice; the second boot must install, fetch and redo nothing.
|
||||
# See cloud/apps/sandbox-control/image/boot-twice/README.md. Run from the merged worktree's root.
|
||||
# - Docker unavailable: a loud warning and exit 0 (fail open), so a host without Docker never blocks every merge.
|
||||
# - On omarchy it takes its own slot (~/orca-offload/bin/slot.sh), waiting at most BOOT_TWICE_SLOT_WAIT_S (300 s).
|
||||
# - Hard limit BOOT_TWICE_TIMEOUT_S (600 s); a cold image build is ~5 min, a cached one ~15 s.
|
||||
set -uo pipefail
|
||||
timeout_s=${BOOT_TWICE_TIMEOUT_S:-600}
|
||||
slot_wait_s=${BOOT_TWICE_SLOT_WAIT_S:-300}
|
||||
if ! docker info >/dev/null 2>&1; then
|
||||
printf '\npremerge: WARNING ==============================================================\n' >&2
|
||||
printf 'premerge: WARNING: Docker is unavailable here, so the boot-twice image check DID NOT RUN.\n' >&2
|
||||
printf 'premerge: WARNING: run it by hand before merging this image change (see image/boot-twice/README.md).\n' >&2
|
||||
printf 'premerge: WARNING ==============================================================\n\n' >&2
|
||||
exit 0
|
||||
fi
|
||||
slot_tool="$HOME/orca-offload/bin/slot.sh"
|
||||
tag="boot-twice-$$-$(date +%s)"
|
||||
if [[ -x "$slot_tool" ]]; then
|
||||
waited=0
|
||||
until "$slot_tool" acquire "$tag" >/dev/null 2>&1; do
|
||||
if ((waited >= slot_wait_s)); then
|
||||
echo "premerge: FAILED: boot-twice image check found no free omarchy slot in ${slot_wait_s}s; rerun the premerge" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 15
|
||||
waited=$((waited + 15))
|
||||
done
|
||||
trap '"$slot_tool" release "$tag" >/dev/null 2>&1 || rm -rf "$HOME/orca-offload/wt/$tag"' EXIT
|
||||
fi
|
||||
echo "premerge: boot-twice image check (limit ${timeout_s}s)"
|
||||
report="$(mktemp)"
|
||||
(cd cloud && pnpm install --frozen-lockfile >/dev/null)
|
||||
timeout --kill-after=30 "$timeout_s" pnpm --dir cloud --filter @orca-cloud/sandbox-control exec \
|
||||
tsx image/boot-twice/boot-twice.ts --release latest --report "$report" >/dev/null
|
||||
status=$?
|
||||
if ((status == 124 || status == 137)); then
|
||||
echo "premerge: FAILED: boot-twice image check passed its ${timeout_s}s limit (cold image build? rerun once)" >&2
|
||||
elif ((status != 0)); then
|
||||
echo "premerge: FAILED: boot-twice image check: the second boot of the image is not clean:" >&2
|
||||
jq -r '.failures[]? | "premerge: " + .' "$report" >&2 2>/dev/null || true
|
||||
echo "premerge: details: cloud/apps/sandbox-control/image/boot-twice/README.md" >&2
|
||||
else
|
||||
echo "premerge: boot-twice image check OK ($(jq -r '"boot 2 " + (.boots[1].ms|tostring) + " ms"' "$report" 2>/dev/null))"
|
||||
fi
|
||||
rm -f "$report"
|
||||
exit "$status"
|
||||
@@ -188,5 +188,9 @@ if ((${#tc_pkgs[@]})); then
|
||||
(cd "$pkg" && pnpm typecheck)
|
||||
done
|
||||
fi
|
||||
# Image or VM-script changes: boot the image twice (Docker); the second boot must be clean. Fails open without Docker.
|
||||
if [[ -n "$(git diff --name-only "$base"...HEAD -- cloud/apps/sandbox-control/image)" ]]; then
|
||||
bash config/scripts/cloud-integration/boot-twice-check.sh
|
||||
fi
|
||||
if [[ "${PREMERGE_FULL:-}" == 1 ]]; then scope="the full vitest suite"; else scope="${#tests[@]} targeted test file(s)"; fi
|
||||
echo "premerge OK: $branch merges cleanly into cloud/integration with typecheck, lint and $scope green."
|
||||
|
||||
@@ -131,6 +131,63 @@ it('a GitHub-only create keeps the typed name as the workspace title (NAME-DROPP
|
||||
})
|
||||
const SESSION = 'cs_abcdefghijklmnopqrstuv'
|
||||
|
||||
it('CREATE-GH-STEP: a public repo that can’t push asks with ids too; settings pages pass, others drop', async () => {
|
||||
const githubRepoAccess = vi.fn(async (query: { repo: string; repoId?: number }) =>
|
||||
query.repo.endsWith('/site')
|
||||
? {
|
||||
access: 'public' as const,
|
||||
owner: 'acme',
|
||||
repo: query.repo,
|
||||
push: 'not_installed' as const,
|
||||
installUrl: query.repoId
|
||||
? 'https://github.com/apps/x/installations/new/permissions?suggested_target_id=50&repository_ids[]=7&state=s'
|
||||
: 'https://github.com/apps/x/installations/new?state=s'
|
||||
}
|
||||
: {
|
||||
access: 'public' as const,
|
||||
owner: 'acme',
|
||||
repo: query.repo,
|
||||
push: 'repo_not_selected' as const,
|
||||
configureUrl: query.repo.endsWith('/evil')
|
||||
? 'https://evil.test/settings'
|
||||
: 'https://github.com/organizations/acme/settings/installations/900'
|
||||
}
|
||||
)
|
||||
const repoIds = vi.fn(async () => ({ repoId: 7, ownerId: 50 }))
|
||||
registerHostedAccountHandlers(
|
||||
{
|
||||
config: async () => ({ idleSleepMinutes: 15 }),
|
||||
connectGitHub: async () => ({ installUrl: REPO, authorizeUrl: REPO }),
|
||||
githubRepoAccess,
|
||||
githubRepositories: async () => ({ repositories: [], nextCursor: null }),
|
||||
create: unusedCreate,
|
||||
...githubUser
|
||||
},
|
||||
{ repoIds }
|
||||
)
|
||||
expect(
|
||||
await handlers.get(CH.getRepoAccess)?.({}, { repoUrl: 'https://github.com/acme/site' })
|
||||
).toMatchObject({
|
||||
ok: true,
|
||||
value: { push: 'not_installed', installUrl: expect.stringContaining('repository_ids[]=7') }
|
||||
})
|
||||
repoIds.mockClear()
|
||||
expect(
|
||||
await handlers.get(CH.getRepoAccess)?.({}, { repoUrl: 'https://github.com/acme/web' })
|
||||
).toMatchObject({
|
||||
ok: true,
|
||||
value: { configureUrl: 'https://github.com/organizations/acme/settings/installations/900' }
|
||||
})
|
||||
// The installation's settings page needs no install ids.
|
||||
expect(repoIds).not.toHaveBeenCalled()
|
||||
const evil = await handlers.get(CH.getRepoAccess)?.(
|
||||
{},
|
||||
{ repoUrl: 'https://github.com/acme/evil' }
|
||||
)
|
||||
expect(evil).toMatchObject({ ok: true })
|
||||
expect(evil).not.toHaveProperty('value.configureUrl')
|
||||
})
|
||||
|
||||
it('asks again with the repo and owner ids when access is missing, so GitHub preselects the repo', async () => {
|
||||
const githubRepoAccess = vi.fn(async (query: { repoId?: number }) => ({
|
||||
access: 'missing' as const,
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import { ipcMain, shell } from 'electron'
|
||||
import { ORCA_CLOUD_WORKSPACE_CHANNELS as CH } from '../../shared/hosted-sandbox/cloud-workspace-api'
|
||||
import type { HostedSandboxConfigResponse } from '../../shared/hosted-sandbox/hosted-sandbox-wire'
|
||||
import type { HostedGithubRepoAccessResponse } from '../../shared/hosted-sandbox/hosted-github-wire'
|
||||
import type { HostedSandboxControlPlaneClient } from './hosted-sandbox-control-plane-client'
|
||||
import { failure } from './hosted-workspace-runtime'
|
||||
import {
|
||||
@@ -53,6 +54,18 @@ function repoTarget(args: RepoArgs): { repo: string; setupSessionId?: string } |
|
||||
return { repo: `https://github.com/${slug}`, ...(session ? { setupSessionId: session } : {}) }
|
||||
}
|
||||
|
||||
/** Links the window opens must be GitHub's own pages. */
|
||||
function githubPagesOnly(value: HostedGithubRepoAccessResponse): HostedGithubRepoAccessResponse {
|
||||
const onGitHub = (url: string | undefined): boolean =>
|
||||
url !== undefined && new URL(url).origin === 'https://github.com'
|
||||
const { installUrl, configureUrl, ...rest } = value
|
||||
return {
|
||||
...rest,
|
||||
...(onGitHub(installUrl) ? { installUrl } : {}),
|
||||
...(onGitHub(configureUrl) ? { configureUrl } : {})
|
||||
}
|
||||
}
|
||||
|
||||
export function registerHostedAccountHandlers(
|
||||
client: Pick<
|
||||
HostedSandboxControlPlaneClient,
|
||||
@@ -127,16 +140,15 @@ export function registerHostedAccountHandlers(
|
||||
}
|
||||
try {
|
||||
const answer = await client.githubRepoAccess(target)
|
||||
if (answer.access !== 'missing') {
|
||||
return { ok: true, value: answer }
|
||||
}
|
||||
// A public repository that can't push gets an install page too (CREATE-GH-STEP).
|
||||
const installs =
|
||||
answer.access === 'missing' ||
|
||||
(answer.access === 'public' &&
|
||||
(answer.push === 'not_installed' || answer.push === 'not_connected'))
|
||||
// Missing: ask again with the ids GitHub's install page needs to preselect only this repo.
|
||||
const ids = await repoIds(target.repo)
|
||||
const ids = installs ? await repoIds(target.repo) : null
|
||||
const value = ids ? await client.githubRepoAccess({ ...target, ...ids }) : answer
|
||||
if (value.installUrl && new URL(value.installUrl).origin !== 'https://github.com') {
|
||||
return { ok: true, value: { ...value, installUrl: undefined } }
|
||||
}
|
||||
return { ok: true, value }
|
||||
return { ok: true, value: githubPagesOnly(value) }
|
||||
} catch (error) {
|
||||
return failure(error)
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { noteCloudPairingBindings } from '../cloud-workspaces/cloud-paired-environment-ids'
|
||||
import { createOomTracker, withGitSetup } from './hosted-workspace-runtime'
|
||||
import { createOomTracker, withGitSetup, withSandboxFacts } from './hosted-workspace-runtime'
|
||||
import { findCloudSandbox } from './cloud-workspace-target'
|
||||
|
||||
function status(checkedAt: number, uptimeSeconds: number, oomKills: number): HostedWorkspaceStatus {
|
||||
@@ -57,6 +57,30 @@ describe('OOM tracker', () => {
|
||||
expect(events).toHaveLength(3)
|
||||
})
|
||||
|
||||
it('announces guard kills as the guard stopping a named process, counted with kernel kills', () => {
|
||||
const events: CloudOomEvent[] = []
|
||||
const tracker = createOomTracker((event) => events.push(event))
|
||||
const guarded = (checkedAt: number, uptime: number, kernel: number, guardKills: number) => ({
|
||||
...status(checkedAt, uptime, kernel),
|
||||
memory: {
|
||||
pressure: 'high' as const,
|
||||
guardKills,
|
||||
lastGuardVictim: guardKills ? 'tsc' : null,
|
||||
lastGuardKillAt: guardKills ? checkedAt : null,
|
||||
guarded: true
|
||||
}
|
||||
})
|
||||
const t0 = 1_000_000_000
|
||||
tracker.observe('ws', 'sbx', guarded(t0, 100, 0, 0))
|
||||
tracker.observe('ws', 'sbx', guarded(t0 + 60_000, 160, 0, 1))
|
||||
tracker.observe('ws', 'sbx', guarded(t0 + 120_000, 220, 0, 1))
|
||||
tracker.observe('ws', 'sbx', guarded(t0 + 180_000, 280, 1, 1))
|
||||
expect(events).toEqual([
|
||||
{ workspaceId: 'ws', oomKills: 1, victim: 'tsc', stoppedBy: 'guard', at: expect.any(Number) },
|
||||
{ workspaceId: 'ws', oomKills: 2, victim: 'node', at: expect.any(Number) }
|
||||
])
|
||||
})
|
||||
|
||||
it('ignores a status without metrics', () => {
|
||||
const events: CloudOomEvent[] = []
|
||||
createOomTracker((event) => events.push(event)).observe('ws', 'sbx', {
|
||||
@@ -96,6 +120,12 @@ describe('GIT-SETUP on the status read', () => {
|
||||
expect(withGitSetup(read, {})).toBe(read)
|
||||
})
|
||||
|
||||
it('carries the sandbox region for the card details', () => {
|
||||
const read = status(1_000, 10, 0)
|
||||
expect(withSandboxFacts(read, { region: 'iad1' })).toEqual({ ...read, region: 'iad1' })
|
||||
expect(withSandboxFacts(read, { region: 'pdx1', gitSetup: 'failed' })).toEqual({ ...read, gitSetupFailed: true, region: 'pdx1' })
|
||||
})
|
||||
|
||||
it('the strict sandbox view accepts the field the server sends with feature git-setup', () => {
|
||||
const view = {
|
||||
id: 'sb_1',
|
||||
|
||||
@@ -115,12 +115,13 @@ export function trimCheckouts(input: HostedArchiveCheckout[]): HostedArchiveChec
|
||||
}
|
||||
|
||||
/**
|
||||
* Announces each new kernel OOM kill once. The counter resets per VM boot; two readings belong to
|
||||
* one boot when their implied boot times agree within two minutes (uptime and checkedAt come from
|
||||
* Announces each new OOM kill once: the kernel's, and the memory guard's (MEMGUARD: processes the
|
||||
* VM stopped before the kernel had to). Both counters reset per VM boot; two readings belong to one
|
||||
* boot when their implied boot times agree within two minutes (uptime and checkedAt come from
|
||||
* different clocks).
|
||||
*/
|
||||
export function createOomTracker(announce: (event: CloudOomEvent) => void) {
|
||||
const seen = new Map<string, { bootAt: number; kills: number }>()
|
||||
const seen = new Map<string, { bootAt: number; kernel: number; guard: number }>()
|
||||
return {
|
||||
observe(
|
||||
workspaceId: string,
|
||||
@@ -132,19 +133,26 @@ export function createOomTracker(announce: (event: CloudOomEvent) => void) {
|
||||
if (!metrics) {
|
||||
return
|
||||
}
|
||||
const guardKills = status.memory?.guardKills ?? 0
|
||||
const bootAt = status.checkedAt - metrics.uptimeSeconds * 1000
|
||||
const previous = seen.get(sandboxId)
|
||||
const sameBoot = previous !== undefined && Math.abs(previous.bootAt - bootAt) < 120_000
|
||||
const announced = sameBoot ? previous.kills : 0
|
||||
const kernel = sameBoot ? previous.kernel : 0
|
||||
const guard = sameBoot ? previous.guard : 0
|
||||
seen.set(sandboxId, {
|
||||
bootAt: sameBoot ? previous.bootAt : bootAt,
|
||||
kills: Math.max(announced, metrics.oomKills)
|
||||
kernel: Math.max(kernel, metrics.oomKills),
|
||||
guard: Math.max(guard, guardKills)
|
||||
})
|
||||
if (metrics.oomKills > announced) {
|
||||
const byGuard = guardKills > guard
|
||||
if (byGuard || metrics.oomKills > kernel) {
|
||||
announce({
|
||||
workspaceId,
|
||||
oomKills: metrics.oomKills,
|
||||
victim: metrics.lastOomVictim,
|
||||
// One count for dismissal: either kind of new kill shows the notice again.
|
||||
oomKills: Math.max(kernel, metrics.oomKills) + Math.max(guard, guardKills),
|
||||
// The guard names the process it chose; the kernel's name comes from its log.
|
||||
victim: byGuard ? (status.memory?.lastGuardVictim ?? null) : metrics.lastOomVictim,
|
||||
...(byGuard ? { stoppedBy: 'guard' as const } : {}),
|
||||
at: now
|
||||
})
|
||||
}
|
||||
@@ -168,6 +176,14 @@ export function withGitSetup(
|
||||
return sandbox.gitSetup === 'failed' ? { ...status, gitSetupFailed: true } : status
|
||||
}
|
||||
|
||||
/** withGitSetup plus the region the sandbox runs in, for the card's details. */
|
||||
export function withSandboxFacts(
|
||||
status: HostedWorkspaceStatus,
|
||||
sandbox: Pick<HostedSandboxView, 'gitSetup' | 'region'>
|
||||
): CloudWorkspaceStatus {
|
||||
return { ...withGitSetup(status, sandbox), region: sandbox.region }
|
||||
}
|
||||
|
||||
let current: HostedWorkspaceService | null = null
|
||||
export function setHostedWorkspaceService(next: HostedWorkspaceService | null): void {
|
||||
current = next
|
||||
|
||||
@@ -41,7 +41,7 @@ import {
|
||||
findCloudRuntime,
|
||||
readCachedCheckouts,
|
||||
announceWakeUsable,
|
||||
withGitSetup,
|
||||
withSandboxFacts,
|
||||
type CloudRuntime
|
||||
} from './hosted-workspace-runtime'
|
||||
|
||||
@@ -204,7 +204,7 @@ export function installHostedWorkspaceService(deps: {
|
||||
try {
|
||||
const { sandbox, status } = await deps.client.workspaceStatus(found.sandboxId)
|
||||
oom.observe(args.workspaceId, found.sandboxId, status)
|
||||
return { ok: true, value: withGitSetup(status, sandbox) }
|
||||
return { ok: true, value: withSandboxFacts(status, sandbox) }
|
||||
} catch (error) {
|
||||
// Gone server-side: prune it now rather than show "Couldn't start" until the next relist.
|
||||
if (isEnvironmentProviderError(error, 'not_found')) {
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
CodexBrokerStartResponseSchema,
|
||||
CodexBrokerStatusResponseSchema,
|
||||
type AgentCredentialsView,
|
||||
type AgentDefaultSourceChoice,
|
||||
type AgentSource,
|
||||
type AgentSourcesView,
|
||||
type CodexBrokerStart,
|
||||
@@ -28,7 +29,10 @@ export type AgentCredentialsClient = {
|
||||
/** Main only, right after setup-token on this computer or a paste. */
|
||||
putClaude(token: string, label?: string): Promise<AgentCredentialsView>
|
||||
remove(agent: OrcaCloudAccountAgent): Promise<AgentCredentialsView>
|
||||
setDefaultSource(agent: OrcaCloudAccountAgent, source: AgentSource): Promise<AgentSourcesView>
|
||||
setDefaultSource(
|
||||
agent: OrcaCloudAccountAgent,
|
||||
source: AgentDefaultSourceChoice
|
||||
): Promise<AgentSourcesView>
|
||||
/** The Source for one workspace (sandbox id); the reply describes that workspace. */
|
||||
setWorkspaceSource(
|
||||
agent: OrcaCloudAccountAgent,
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import { BrowserWindow, app, ipcMain, shell } from 'electron'
|
||||
import { z } from 'zod'
|
||||
import {
|
||||
IN_WORKSPACE_SOURCE,
|
||||
ORCA_CLOUD_ACCOUNT_AGENTS,
|
||||
type OrcaCloudAgentSetups
|
||||
} from '../../shared/orca-cloud-agents/orca-cloud-agents-wire'
|
||||
@@ -27,6 +28,10 @@ const PasteRequest = z.object({ token: z.string().min(1).max(1024) })
|
||||
// The code (or callback URL) from Claude's sign-in page; main normalizes it.
|
||||
const CodeRequest = z.object({ code: z.string().min(1).max(2048) })
|
||||
const SourceRequest = z.object({ agent: Agent, source: z.enum(['sign-in', 'key']).nullable() })
|
||||
const DefaultSourceRequest = z.object({
|
||||
agent: Agent,
|
||||
source: z.enum(['sign-in', 'key', IN_WORKSPACE_SOURCE]).nullable()
|
||||
})
|
||||
// A runtime environment, never a raw sandbox id: main resolves only sandboxes this desktop made.
|
||||
const RuntimeEnvironment = z.string().min(1).max(256)
|
||||
const WorkspaceRequest = z.object({ runtimeEnvironmentId: RuntimeEnvironment })
|
||||
@@ -153,7 +158,7 @@ export function registerOrcaCloudAgentsHandlers(userDataPath: string): void {
|
||||
return request.success ? service.remove(request.data.agent) : invalid
|
||||
})
|
||||
ipcMain.handle('orcaCloudAgents:setDefaultSource', (_event, raw: unknown) => {
|
||||
const request = SourceRequest.safeParse(raw)
|
||||
const request = DefaultSourceRequest.safeParse(raw)
|
||||
return request.success
|
||||
? service.setDefaultSource(request.data.agent, request.data.source)
|
||||
: invalid
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
CLAUDE_SETUP_TOKEN_PATTERN,
|
||||
IDLE_ORCA_CLOUD_AGENT_SETUPS,
|
||||
type AgentCredentialsView,
|
||||
type AgentDefaultSourceChoice,
|
||||
type AgentSource,
|
||||
type AgentSourcesView,
|
||||
type OrcaCloudAccountAgent,
|
||||
@@ -339,7 +340,7 @@ export function createOrcaCloudAgentsService(deps: OrcaCloudAgentsServiceDeps) {
|
||||
|
||||
setDefaultSource: (
|
||||
agent: OrcaCloudAccountAgent,
|
||||
source: AgentSource
|
||||
source: AgentDefaultSourceChoice
|
||||
): Promise<OrcaCloudAgentsResult<AgentSourcesView>> =>
|
||||
attempt((client) => client.setDefaultSource(agent, source)),
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { ipcRenderer } from 'electron'
|
||||
import type {
|
||||
AgentCredentialsView,
|
||||
AgentDefaultSourceChoice,
|
||||
AgentSource,
|
||||
AgentSourcesView,
|
||||
OrcaCloudAccountAgent,
|
||||
@@ -29,7 +30,7 @@ export type OrcaCloudAgentsApi = {
|
||||
}) => Promise<OrcaCloudAgentsResult<AgentCredentialsView>>
|
||||
setDefaultSource: (request: {
|
||||
agent: OrcaCloudAccountAgent
|
||||
source: AgentSource
|
||||
source: AgentDefaultSourceChoice
|
||||
}) => Promise<OrcaCloudAgentsResult<AgentSourcesView>>
|
||||
/** Main's answer (runtime store + pairing bindings): is this runtime environment Orca Cloud? */
|
||||
isCloudEnvironment: (request: { runtimeEnvironmentId: string }) => Promise<boolean>
|
||||
|
||||
@@ -637,6 +637,15 @@ body[data-modal-overlay-open] [data-toasts-raised] [data-sonner-toaster] {
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
/* Hovering the stack must never move the toast under the pointer. Sonner sizes an expanded toast to
|
||||
the height it measured when the toast was issued; a toast whose content grew since then shrank
|
||||
from the top on hover, sank out from under the pointer, collapsed and popped back up (a loop).
|
||||
The front toast keeps its natural height in both states, as it already has when collapsed. */
|
||||
[data-sonner-toaster]
|
||||
[data-sonner-toast][data-mounted='true'][data-expanded='true'][data-front='true'][data-removed='false'] {
|
||||
height: auto;
|
||||
}
|
||||
|
||||
/* Clip folded toast overflow to prevent hover/re-expansion loops; hidden would create a focus-scrollable container. */
|
||||
[data-sonner-toaster] [data-sonner-toast][data-expanded='false'][data-front='false'] {
|
||||
overflow: clip;
|
||||
|
||||
@@ -47,6 +47,7 @@ export function agentsSummary(
|
||||
break
|
||||
case 'killSwitch':
|
||||
case 'unavailable':
|
||||
case 'inWorkspace':
|
||||
rest.push(
|
||||
t('agentHostPanel.summary.agentSignsInHere', '{{agent}} signs in for this workspace', {
|
||||
agent
|
||||
|
||||
@@ -29,7 +29,7 @@ export type CloudAgentsPanelLoad =
|
||||
|
||||
export type CloudAgentsPanelActions = {
|
||||
chooseSource: (agent: OrcaCloudAccountAgent, source: Exclude<AgentSource, null>) => void
|
||||
/** Settings › Agent accounts (S0): set up once, or renew. */
|
||||
/** Settings › Orca Cloud › Cloud agent sign-ins (S0): set up once, or renew. */
|
||||
manage: () => void
|
||||
signIn: () => void
|
||||
retry: () => void
|
||||
@@ -63,7 +63,10 @@ function statusCopy(row: CloudAgentPanelRow, t: Translate): { line: string; deta
|
||||
line: t('agentHostPanel.cloud.expired', 'Your {{agent}} sign-in for Orca Cloud expired', {
|
||||
agent
|
||||
}),
|
||||
detail: t('agentHostPanel.cloud.expiredDetail', 'Renew it once in Agent accounts.')
|
||||
detail: t(
|
||||
'agentHostPanel.cloud.expiredDetailCloudSignIns',
|
||||
'Renew it once in Cloud agent sign-ins.'
|
||||
)
|
||||
}
|
||||
case 'sharedUnavailable':
|
||||
return {
|
||||
@@ -82,7 +85,9 @@ function statusCopy(row: CloudAgentPanelRow, t: Translate): { line: string; deta
|
||||
detail:
|
||||
row.status.reason === 'paused'
|
||||
? t('vmBuildAgents.paused', 'Orca account sign-in is paused')
|
||||
: t('vmBuildAgents.notOffered', 'Signs in for each cloud workspace for now')
|
||||
: row.status.reason === 'chosen'
|
||||
? t('agentHostPanel.cloud.signsInHereChosen', 'Orca doesn’t store this sign-in')
|
||||
: t('vmBuildAgents.notOffered', 'Signs in for each cloud workspace for now')
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,7 @@ export type CloudAgentPanelStatus =
|
||||
| { kind: 'notSetUp' }
|
||||
| { kind: 'expired' }
|
||||
/** No account path and no key: the agent signs in inside this workspace (§5c). */
|
||||
| { kind: 'signsInHere'; reason: 'paused' | 'notOffered' }
|
||||
| { kind: 'signsInHere'; reason: 'paused' | 'notOffered' | 'chosen' }
|
||||
/** D13: a shared workspace never uses personal accounts or keys, and no team key covers it. */
|
||||
| { kind: 'sharedUnavailable' }
|
||||
|
||||
@@ -96,6 +96,9 @@ export function cloudAgentPanelRows(
|
||||
return { agent, status: { kind: 'signsInHere', reason: 'paused' }, source }
|
||||
case 'unavailable':
|
||||
return { agent, status: { kind: 'signsInHere', reason: 'notOffered' }, source }
|
||||
case 'inWorkspace':
|
||||
// Nothing of Orca's applies, so there is no Source to switch here either.
|
||||
return { agent, status: { kind: 'signsInHere', reason: 'chosen' }, source: null }
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -7,7 +7,8 @@ import { closeAgentHostPanel } from './agent-host-panel-store'
|
||||
|
||||
const TARGET = {
|
||||
accounts: { pane: 'accounts' },
|
||||
agentAccounts: { pane: 'accounts', sectionId: 'orca-cloud' },
|
||||
// Opens the sign-in sheet (below); its page is Settings › Orca Cloud › Cloud agent sign-ins.
|
||||
agentAccounts: { pane: 'cloud-agent-sign-ins' },
|
||||
agentApiKeys: { pane: 'agent-api-keys' },
|
||||
cloudSecrets: { pane: 'cloud-secrets' },
|
||||
customEnvironments: { pane: 'servers' },
|
||||
|
||||
@@ -239,6 +239,21 @@ export const runningRestartFailedNarrow = fixture({
|
||||
/** PC-3b: the launch holds; Set up once runs the one-time Orca Cloud setup in the pane. */
|
||||
export const notSetUp = fixture({ setUp: { setup: { phase: 'idle' } } })
|
||||
export const notSetUpCompact = fixture({ setUp: { setup: { phase: 'idle' } }, width: 320 })
|
||||
/** The quiet second choice on "Set up once": the agent signs in inside this workspace instead. */
|
||||
export const notSetUpSignInInside = defineFixture({
|
||||
width: 940,
|
||||
render: () => (
|
||||
<div className="pb-24">
|
||||
<Pane setUp={{ setup: { phase: 'idle' } }} agent="codex" />
|
||||
</div>
|
||||
),
|
||||
interact: async (frame) => {
|
||||
frame
|
||||
.querySelector<HTMLButtonElement>('button[aria-label="More choices"]')
|
||||
?.dispatchEvent(new PointerEvent('pointerdown', { bubbles: true, button: 0 }))
|
||||
await new Promise((resolve) => setTimeout(resolve, 200))
|
||||
}
|
||||
})
|
||||
/** A half-width split: the held strip wraps its detail rather than cutting it off. */
|
||||
export const notSetUpNarrow = fixture({ setUp: { setup: { phase: 'idle' } }, width: 440 })
|
||||
export const notSetUpBrowser = fixture({
|
||||
|
||||
@@ -340,6 +340,32 @@ export function CloudAgentSignInStrip({
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
) : null}
|
||||
{model.alternative && !model.otherWays ? (
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger asChild>
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon-xs"
|
||||
aria-label={t('agentSignInStrip.moreChoices', 'More choices')}
|
||||
>
|
||||
<MoreHorizontal aria-hidden />
|
||||
</Button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent align="end" className="max-w-72">
|
||||
<DropdownMenuItem
|
||||
onSelect={() => model.alternative && actions.primary(model.alternative.action)}
|
||||
>
|
||||
<span className="flex flex-col gap-0.5">
|
||||
<span>{model.alternative.label}</span>
|
||||
<span className="text-xs leading-4 text-muted-foreground">
|
||||
{model.alternative.detail}
|
||||
</span>
|
||||
</span>
|
||||
</DropdownMenuItem>
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
) : null}
|
||||
{model.settingsLink && !stacked ? (
|
||||
<CloudStripSettingsLink label={model.settingsLink} onOpen={actions.openSettings} />
|
||||
) : null}
|
||||
|
||||
@@ -112,4 +112,39 @@ describe('cloudAgentSetupStripModel (PC-3b)', () => {
|
||||
primary: { label: 'Start Claude', action: { kind: 'start' } }
|
||||
})
|
||||
})
|
||||
|
||||
it('offers signing in inside this workspace as a quiet second choice, only while nothing runs', () => {
|
||||
const idle = cloudAgentSetupStripModel({
|
||||
agent: 'codex',
|
||||
setup: { phase: 'idle' },
|
||||
ready: false,
|
||||
t
|
||||
})
|
||||
expect(idle.primary?.action).toEqual({ kind: 'setUpOnce' })
|
||||
expect(idle.alternative).toEqual({
|
||||
label: 'Sign in inside this workspace instead',
|
||||
detail:
|
||||
'Orca won’t store your sign-in. You’ll sign in again in each new workspace. Team API keys won’t be used for this agent.',
|
||||
action: { kind: 'signInInsideWorkspace' }
|
||||
})
|
||||
const failed = cloudAgentSetupStripModel({
|
||||
agent: 'claude',
|
||||
setup: { phase: 'failed', error: 'denied' },
|
||||
ready: false,
|
||||
t
|
||||
})
|
||||
expect(failed.alternative?.action).toEqual({ kind: 'signInInsideWorkspace' })
|
||||
for (const setup of [
|
||||
{ phase: 'browser', startedAt: 1, canReopen: true },
|
||||
{ phase: 'saving' }
|
||||
] as const) {
|
||||
expect(
|
||||
cloudAgentSetupStripModel({ agent: 'claude', setup, ready: false, t }).alternative
|
||||
).toBeUndefined()
|
||||
}
|
||||
expect(
|
||||
cloudAgentSetupStripModel({ agent: 'claude', setup: { phase: 'idle' }, ready: true, t })
|
||||
.alternative
|
||||
).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// PC-3b: the Orca Cloud pane strip for an agent that isn't set up yet. The launch holds; "Set up
|
||||
// once" runs the same one-time setup as Settings › Accounts › Orca Cloud (S0) right in the pane,
|
||||
// once" runs the same one-time setup as Settings › Orca Cloud › Cloud agent sign-ins (S0) right in the pane,
|
||||
// and once it's saved the strip offers "Start {{agent}}". Pure: the strip renders what this returns.
|
||||
import type { Translate } from '@/i18n/use-translate'
|
||||
import {
|
||||
@@ -30,6 +30,20 @@ function base(agent: string, t: Translate): CloudStripModel {
|
||||
}
|
||||
}
|
||||
|
||||
/** "Sign in inside this workspace instead": the in-VM login, as a quiet second choice to Set up once. */
|
||||
export function signInInsideWorkspaceAlternative(
|
||||
t: Translate
|
||||
): NonNullable<CloudStripModel['alternative']> {
|
||||
return {
|
||||
label: t('agentSignInStrip.signInInsideWorkspace', 'Sign in inside this workspace instead'),
|
||||
detail: t(
|
||||
'agentSignInStrip.signInInsideWorkspaceDetail',
|
||||
'Orca won’t store your sign-in. You’ll sign in again in each new workspace. Team API keys won’t be used for this agent.'
|
||||
),
|
||||
action: { kind: 'signInInsideWorkspace' }
|
||||
}
|
||||
}
|
||||
|
||||
/** A detail with `inline code` marks, as one sentence for a screen reader. */
|
||||
function plainText(text: string): string {
|
||||
return text.replaceAll('`', '')
|
||||
@@ -100,7 +114,7 @@ export function cloudAgentSetupStripModel(args: {
|
||||
switch (setup.phase) {
|
||||
case 'idle':
|
||||
case 'saved':
|
||||
return need
|
||||
return { ...need, alternative: signInInsideWorkspaceAlternative(t) }
|
||||
case 'browser':
|
||||
return {
|
||||
...need,
|
||||
@@ -223,6 +237,7 @@ export function cloudAgentSetupStripModel(args: {
|
||||
action: { kind: 'reopenSetup' }
|
||||
}
|
||||
: { label: t('agentSignInStrip.tryAgain', 'Try again'), action: { kind: 'setUpOnce' } },
|
||||
alternative: signInInsideWorkspaceAlternative(t),
|
||||
announcement: copy.title
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,6 +25,8 @@ export type CloudStripPrimary =
|
||||
| { kind: 'reopenSetup' }
|
||||
/** PC-3b: the setup finished; start the held agent. */
|
||||
| { kind: 'start' }
|
||||
/** The quiet alternative to "Set up once": the agent signs in inside each workspace, Orca stores nothing. */
|
||||
| { kind: 'signInInsideWorkspace' }
|
||||
|
||||
export type CloudStripModel = {
|
||||
status: CloudStripStatus
|
||||
@@ -57,6 +59,11 @@ export type CloudStripModel = {
|
||||
} | null
|
||||
/** "Other ways" under ⋯: the code fallback and the key path. */
|
||||
otherWays: boolean
|
||||
/**
|
||||
* A quiet second choice under ⋯ (the strip has no "Other ways"): a label and one sentence on what
|
||||
* it means, e.g. "Sign in inside this workspace instead" on "Set up once".
|
||||
*/
|
||||
alternative?: { label: string; detail: string; action: CloudStripPrimary }
|
||||
/** Already on a code: ⋯ offers the paste fallback and Cancel instead of "Use a code". */
|
||||
codeStep?: boolean
|
||||
/**
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user