From 947813d4fca3edeee7d03aeb10b5e608342673cd Mon Sep 17 00:00:00 2001
From: Neil
Date: Mon, 9 Nov 2026 15:46:10 -0800
Subject: [PATCH] fix(cloud): acceptance check fixes (seat true-down, repo
object, ssh probe, ports bound, kid from key file, load notes)
---
cloud/apps/acceptance/desktop/run-desktop.mjs | 786 ++++++++++++++++++
cloud/apps/acceptance/desktop/sandbox-api.mjs | 72 ++
cloud/apps/acceptance/src/checks/billing.ts | 4 +-
cloud/apps/acceptance/src/checks/core.ts | 6 +-
cloud/apps/acceptance/src/checks/lifecycle.ts | 11 +-
cloud/apps/acceptance/src/checks/platform.ts | 8 +-
cloud/apps/acceptance/src/checks/secrets.ts | 12 +-
cloud/apps/acceptance/src/inspect-build.ts | 12 +
cloud/apps/acceptance/src/list-provider.ts | 8 +
cloud/apps/acceptance/src/probe-build.ts | 49 ++
cloud/apps/acceptance/src/report.ts | 1 +
cloud/apps/acceptance/src/run.ts | 17 +
cloud/apps/acceptance/src/tokens.ts | 6 +-
13 files changed, 972 insertions(+), 20 deletions(-)
create mode 100644 cloud/apps/acceptance/desktop/run-desktop.mjs
create mode 100644 cloud/apps/acceptance/desktop/sandbox-api.mjs
create mode 100644 cloud/apps/acceptance/src/inspect-build.ts
create mode 100644 cloud/apps/acceptance/src/list-provider.ts
create mode 100644 cloud/apps/acceptance/src/probe-build.ts
diff --git a/cloud/apps/acceptance/desktop/run-desktop.mjs b/cloud/apps/acceptance/desktop/run-desktop.mjs
new file mode 100644
index 00000000000..90aca0003e4
--- /dev/null
+++ b/cloud/apps/acceptance/desktop/run-desktop.mjs
@@ -0,0 +1,786 @@
+#!/usr/bin/env node
+// Desktop (Electron) acceptance run for Orca Cloud: launches the dev Orca built from this checkout
+// against the DEV cloud backends, drives the real UI over CDP (playwright-core), and writes
+// results.json + screenshots into the artifacts dir given as argv[2].
+//
+// node cloud/apps/acceptance/desktop/run-desktop.mjs
+//
+// Env: ORCA_CLOUD_SANDBOX_URL / ORCA_CLOUD_SANDBOX_DEV_KEY_FILE / ORCA_BILLING_API_URL override the
+// dev defaults; W9D_USER (default w9-desktop) is the dev account; W9D_WAKE_CYCLES (default 5);
+// W9D_SKIP_BUILD=1 skips the workspace-package/native/CLI prebuild.
+import { execFileSync, spawn, spawnSync } from 'node:child_process'
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync, createWriteStream, readdirSync, rmdirSync } from 'node:fs'
+import os from 'node:os'
+import path from 'node:path'
+import { chromium } from 'playwright-core'
+import { DEV_KEY_FILE, SANDBOX_URL, deleteAllSandboxes, listSandboxes, sandboxApi } from './sandbox-api.mjs'
+
+const repoRoot = path.resolve(import.meta.dirname, '../../../..')
+const artifactsDir = path.resolve(process.argv[2] ?? `/tmp/orca-desktop-acceptance-${Date.now()}`)
+const shotsDir = path.join(artifactsDir, 'screenshots')
+mkdirSync(shotsDir, { recursive: true })
+
+const USER = process.env.W9D_USER ?? 'w9-desktop'
+const REPO_SLUG = 'nwparker/orca-sandbox-push-test'
+const WAKE_CYCLES = Number(process.env.W9D_WAKE_CYCLES ?? 5)
+const KEY_SAMPLES = 10
+const HTTP_SAMPLES = 10
+const PORT = 4173
+const runTag = new Date().toISOString().replace(/[:.]/g, '-')
+const wsName = `w9d-${Date.now().toString(36)}`
+const HEAVY = '/tmp/vdrv/heavy.sh'
+
+const results = {
+ runTag,
+ startedAt: new Date().toISOString(),
+ repoRoot,
+ gitHead: git('rev-parse', 'HEAD'),
+ branch: git('rev-parse', '--abbrev-ref', 'HEAD'),
+ sandboxUrl: SANDBOX_URL,
+ account: `dev:${USER}`,
+ workspaceName: wsName,
+ notes: [],
+ app: {},
+ steps: [],
+ bugs: [],
+ cleanup: {}
+}
+const created = { sandboxIds: new Set(), localPaths: [] }
+let runner = null
+let browser = null
+let page = null
+let ctx = null
+
+function git(...args) {
+ try {
+ return execFileSync('git', ['-C', repoRoot, ...args], { encoding: 'utf8' }).trim()
+ } catch {
+ return null
+ }
+}
+function log(...a) {
+ console.error(`[desktop ${new Date().toISOString().slice(11, 19)}]`, ...a)
+}
+function save() {
+ results.updatedAt = new Date().toISOString()
+ writeFileSync(path.join(artifactsDir, 'results.json'), JSON.stringify(results, null, 2))
+}
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms))
+function pct(samples, p) {
+ const s = [...samples].sort((a, b) => a - b)
+ if (!s.length) return null
+ return s[Math.min(s.length - 1, Math.ceil((p / 100) * s.length) - 1)]
+}
+function stats(samples) {
+ return samples.length
+ ? { n: samples.length, p50: pct(samples, 50), p95: pct(samples, 95), min: Math.min(...samples), max: Math.max(...samples), samples }
+ : { n: 0 }
+}
+const redact = (s) => String(s ?? '').replace(/[A-Za-z0-9_-]{32,}/g, '')
+
+async function shot(step, name) {
+ const file = path.join(shotsDir, `${step.id.replace(/[^a-z0-9.-]/gi, '_')}-${name}.png`)
+ try {
+ await page.screenshot({ path: file })
+ step.screenshots.push(path.relative(artifactsDir, file))
+ } catch (e) {
+ step.screenshots.push(`(screenshot failed: ${e.message})`)
+ }
+ return file
+}
+
+async function runStep(id, title, fn) {
+ const step = { id, title, status: 'fail', detail: '', screenshots: [], timings: {}, startedAt: new Date().toISOString() }
+ results.steps.push(step)
+ log('step', id, title)
+ const t0 = Date.now()
+ try {
+ await fn(step)
+ } catch (e) {
+ step.status = 'fail'
+ step.detail = `${step.detail ? step.detail + ' | ' : ''}error: ${redact(e.message).slice(0, 600)}`
+ await shot(step, 'error')
+ }
+ step.durationMs = Date.now() - t0
+ log(' ->', step.status, step.detail.slice(0, 200))
+ save()
+ return step
+}
+function bug(step, { title, expected, actual, repro, screenshot }) {
+ results.bugs.push({ step, title, expected, actual, repro, screenshot })
+ save()
+}
+
+// ---------- app lifecycle ----------
+
+function prebuild() {
+ if (process.env.W9D_SKIP_BUILD === '1') return
+ const cmd = [
+ `cd ${JSON.stringify(repoRoot)}`,
+ 'node config/scripts/workspace-package-build/build-workspace-packages.mjs',
+ 'node config/scripts/native-build/node-addons/ensure-native-runtime.mjs --runtime=electron',
+ // `orca ssh` / Copy SSH Command tunnel through the dev CLI; `pnpm dev` does not build it.
+ ...(existsSync(path.join(repoRoot, 'out/cli/index.js')) ? [] : ['node config/scripts/cli-build/build-cli.mjs'])
+ ].join(' && ')
+ log('prebuild (heavy lock)')
+ const t0 = Date.now()
+ const r = existsSync(HEAVY)
+ ? spawnSync(HEAVY, [cmd], { stdio: ['ignore', 'pipe', 'pipe'], encoding: 'utf8' })
+ : spawnSync('bash', ['-c', cmd], { stdio: ['ignore', 'pipe', 'pipe'], encoding: 'utf8' })
+ writeFileSync(path.join(artifactsDir, 'prebuild.log'), `${r.stdout}\n${r.stderr}`)
+ results.app.prebuildMs = Date.now() - t0
+ if (r.status !== 0) throw new Error(`prebuild failed (${r.status}); see prebuild.log`)
+}
+
+const appEnv = (userData) => ({
+ ...process.env,
+ ORCA_BACKGROUND_LAUNCH: '1',
+ ORCA_DEV_USER_DATA_PATH: userData,
+ ORCA_CLOUD_SANDBOX_URL: SANDBOX_URL,
+ ORCA_CLOUD_SANDBOX_DEV_KEY_FILE: DEV_KEY_FILE,
+ ORCA_BILLING_API_URL: process.env.ORCA_BILLING_API_URL ?? 'https://orca-billing-dev.lovecast.workers.dev',
+ ORCA_CLOUD_DEV_AUTH: '1',
+ ORCA_CLOUD_DEV_USER_ID: USER
+})
+
+async function launchApp(userData, label) {
+ const logFile = path.join(artifactsDir, `app-${label}.log`)
+ const out = createWriteStream(logFile)
+ const t0 = Date.now()
+ const child = spawn(process.execPath, ['config/scripts/development/launch/run-electron-vite-dev.mjs'], {
+ cwd: repoRoot,
+ env: appEnv(userData),
+ stdio: ['ignore', 'pipe', 'pipe']
+ })
+ runner = child
+ let buf = ''
+ const instance = await new Promise((resolve, reject) => {
+ const timer = setTimeout(() => reject(new Error('no ORCA_DEV_INSTANCE line within 180s')), 180_000)
+ const onData = (d) => {
+ out.write(d)
+ buf += d.toString()
+ const m = buf.match(/^ORCA_DEV_INSTANCE (\{.*\})$/m)
+ if (m) {
+ clearTimeout(timer)
+ resolve(JSON.parse(m[1]))
+ }
+ }
+ child.stdout.on('data', onData)
+ child.stderr.on('data', onData)
+ child.once('exit', (code) => reject(new Error(`dev runner exited early (${code}); see ${logFile}`)))
+ })
+ child.stdout.removeAllListeners('data')
+ child.stderr.removeAllListeners('data')
+ child.stdout.pipe(out)
+ child.stderr.pipe(out)
+ log('instance', instance.cdpUrl, instance.rendererUrl, 'runner pid', child.pid)
+ const deadline = Date.now() + 300_000
+ for (;;) {
+ try {
+ const targets = await (await fetch(`${instance.cdpUrl}/json`)).json()
+ if (targets.some((t) => t.type === 'page' && t.url.startsWith(instance.rendererUrl))) break
+ } catch {}
+ if (Date.now() > deadline) throw new Error('renderer never appeared on CDP')
+ await sleep(1000)
+ }
+ browser = await chromium.connectOverCDP(instance.cdpUrl)
+ page = browser.contexts().flatMap((c) => c.pages()).find((p) => p.url().startsWith(instance.rendererUrl))
+ if (!page) throw new Error('renderer page not found after connect')
+ await page.waitForFunction(() => Boolean(window.__store?.getState?.().repos), null, { timeout: 120_000 })
+ results.app[label] = { cdpUrl: instance.cdpUrl, rendererUrl: instance.rendererUrl, runnerPid: child.pid, launchMs: Date.now() - t0, log: path.relative(artifactsDir, logFile) }
+ save()
+ return instance
+}
+
+async function stopApp() {
+ if (!runner?.pid) return
+ const pid = runner.pid
+ const cmd = spawnSync('ps', ['-p', String(pid), '-o', 'command='], { encoding: 'utf8' }).stdout
+ if (!cmd.includes('run-electron-vite-dev.mjs')) {
+ log('refusing to kill pid', pid, 'cmd:', cmd.trim())
+ return
+ }
+ // Leave the CDP socket without closing the app's windows, then stop only our runner tree.
+ browser = null
+ runner.kill('SIGTERM')
+ for (let i = 0; i < 30 && spawnSync('ps', ['-p', String(pid)]).status === 0; i++) await sleep(500)
+ if (spawnSync('ps', ['-p', String(pid)]).status === 0) runner.kill('SIGKILL')
+ runner = null
+}
+
+// ---------- UI helpers ----------
+
+const store = (fn, arg) => page.evaluate(fn, arg)
+
+async function installToastRecorder() {
+ await page.evaluate(() => {
+ window.__w9dToasts = []
+ if (window.__w9dToastObs) return
+ window.__w9dToastObs = new MutationObserver(() => {
+ document.querySelectorAll('[data-sonner-toast]').forEach((t) => {
+ const s = t.innerText
+ if (s && !window.__w9dToasts.includes(s)) window.__w9dToasts.push(s)
+ })
+ })
+ window.__w9dToastObs.observe(document.body, { subtree: true, childList: true, characterData: true })
+ })
+}
+const toasts = () => page.evaluate(() => window.__w9dToasts ?? [])
+const clearToasts = () => page.evaluate(() => { window.__w9dToasts = [] })
+
+function rowTitle(name) {
+ const esc = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+ return page.locator('[data-worktree-title-inline-rename]', { hasText: new RegExp(`^${esc}$`) }).first()
+}
+async function openRowMenu(name) {
+ await page.keyboard.press('Escape').catch(() => {})
+ await rowTitle(name).click({ button: 'right' })
+ await page.getByRole('menuitem').first().waitFor({ timeout: 5000 })
+ await sleep(250)
+}
+const menuItems = () => page.getByRole('menuitem').allInnerTexts()
+
+async function dismissNoise() {
+ for (const name of ['No thanks', 'Later']) {
+ await page.getByRole('button', { name, exact: true }).first().click({ timeout: 500 }).catch(() => {})
+ }
+}
+
+async function visibleTerminals() {
+ return page.evaluate(() =>
+ [...document.querySelectorAll('.xterm-rows')]
+ .filter((r) => r.getBoundingClientRect().width > 0)
+ .map((r) => r.innerText.replace(/ /g, ' ').trimEnd())
+ )
+}
+async function focusTerminal() {
+ await page.locator('.xterm-screen').filter({ visible: true }).last().click()
+}
+const lastLine = (t) => t.split('\n').filter((l) => l.trim()).pop() ?? ''
+const isShellPrompt = (t) => /\$\s*$/.test(lastLine(t))
+
+async function openNewTerminalTab() {
+ await page.locator('button[aria-label="New tab"]:visible').first().click()
+ await page.getByRole('menuitem', { name: /New Terminal/ }).click()
+}
+
+/** Keydown (capture listener, in-page clock) -> the char shows up in the visible xterm DOM. */
+async function measureKeystroke(marker) {
+ await page.keyboard.type(marker.slice(0, -1))
+ await sleep(350)
+ await page.evaluate((marker) => {
+ const rows = [...document.querySelectorAll('.xterm-rows')].filter((r) => r.getBoundingClientRect().width > 0).pop()
+ window.__w9dLat = { down: 0, echo: 0 }
+ window.addEventListener('keydown', () => { if (!window.__w9dLat.down) window.__w9dLat.down = performance.now() }, { capture: true, once: true })
+ const obs = new MutationObserver(() => {
+ if (!window.__w9dLat.echo && rows.innerText.includes(marker)) {
+ window.__w9dLat.echo = performance.now()
+ obs.disconnect()
+ }
+ })
+ obs.observe(rows, { subtree: true, childList: true, characterData: true })
+ }, marker)
+ await page.keyboard.press(marker.slice(-1))
+ await page.waitForFunction(() => window.__w9dLat.echo > 0, null, { timeout: 15_000 })
+ const l = await page.evaluate(() => window.__w9dLat)
+ await page.keyboard.press('Control+U')
+ await sleep(250)
+ return Math.round(l.echo - l.down)
+}
+
+/** The sandbox this run created: listed under the dev account, created after Create was clicked. */
+async function sandboxForWorkspace(since) {
+ const { items } = await listSandboxes(USER)
+ const mine = items.filter((s) => (s.createdAt ?? 0) >= since - 5000)
+ return { items, newest: mine.sort((a, b) => (b.createdAt ?? 0) - (a.createdAt ?? 0))[0] ?? null }
+}
+async function sandboxStatus(id) {
+ const r = await sandboxApi(USER, 'GET', `/v1/sandboxes/${id}`)
+ return { http: r.status, status: r.json?.sandbox?.status ?? r.json?.error ?? null }
+}
+async function waitSandboxStatus(id, want, timeoutMs) {
+ const t0 = Date.now()
+ let last = null
+ while (Date.now() - t0 < timeoutMs) {
+ last = await sandboxStatus(id)
+ if (want.includes(last.status)) return { ...last, ms: Date.now() - t0 }
+ await sleep(1000)
+ }
+ return { ...last, ms: Date.now() - t0, timedOut: true }
+}
+
+// ---------- flow ----------
+
+async function main() {
+ save()
+ if (!existsSync(DEV_KEY_FILE)) throw new Error(`dev key file missing: ${DEV_KEY_FILE}`)
+ const before = await listSandboxes(USER)
+ results.preexistingSandboxes = before.items.map((s) => ({ id: s.id, status: s.status ?? s.state }))
+ prebuild()
+
+ const tmp = mkdtempSync(path.join(os.tmpdir(), 'orca-w9d-'))
+ const userData = path.join(tmp, 'profile')
+ const repoDir = path.join(tmp, 'repos', 'orca-sandbox-push-test')
+ mkdirSync(userData, { recursive: true })
+ results.app.userData = userData
+ execFileSync('gh', ['repo', 'clone', REPO_SLUG, repoDir, '--', '--quiet'], { stdio: 'ignore' })
+ created.tmp = tmp
+
+ await runStep('setup', 'Launch dev Orca, sign in (dev auth), add repo', async (step) => {
+ await launchApp(userData, 'first')
+ // Sign in before any cloud call, then relaunch: the sandbox token cache is not keyed on the
+ // signed-in identity (see bugs), so a fresh process guarantees the dev: owner.
+ await store(async () => {
+ const s = window.__store.getState()
+ await s.connectCurrentOrcaProfile?.()
+ await s.fetchOrcaProfiles?.()
+ })
+ const who = await store(() => window.__store.getState().orcaProfiles?.map((p) => p.cloud?.userId ?? null))
+ step.timings.firstLaunchMs = results.app.first.launchMs
+ await stopApp()
+ await launchApp(userData, 'main')
+ step.timings.relaunchMs = results.app.main.launchMs
+ await store(() => window.__store.getState().updateSettings({ terminalGpuAcceleration: 'off' }))
+ await store(async (p) => {
+ await window.api.repos.add({ path: p })
+ await window.__store.getState().fetchRepos({})
+ }, repoDir)
+ const repos = await store(() => window.__store.getState().repos.map((r) => r.displayName))
+ await installToastRecorder()
+ step.status = who?.includes(USER) && repos.includes('orca-sandbox-push-test') ? 'pass' : 'partial'
+ step.detail = `signed in as ${JSON.stringify(who)}; repos ${JSON.stringify(repos)}; terminal GPU off so xterm renders to DOM`
+ await shot(step, 'home')
+ })
+
+ let sandboxId = null
+
+ await runStep('cli.desktop', 'Create an Orca Cloud workspace from the composer', async (step) => {
+ await page.getByRole('button', { name: 'New workspace', exact: true }).first().click()
+ const d = page.getByRole('dialog')
+ await d.waitFor()
+ await d.getByRole('combobox', { name: 'Project' }).click()
+ await page.getByRole('option', { name: /orca-sandbox-push-test/ }).first().click()
+ await d.getByRole('combobox', { name: 'Run on' }).click()
+ const targets = page.getByRole('listbox', { name: 'Run targets' })
+ await targets.waitFor()
+ step.runTargets = (await targets.getByRole('option').allInnerTexts()).map((t) => t.split('\n')[0])
+ await shot(step, 'picker')
+ await targets.getByRole('option', { name: /Orca Cloud/ }).click()
+ await sleep(800)
+ await d.getByRole('button', { name: 'Later', exact: true }).click({ timeout: 1500 }).catch(() => {})
+ step.composerHint = (await d.innerText()).match(/Hosted sandbox[\s\S]*?idle/)?.[0]?.replace(/\s+/g, ' ')
+ // Smart mode offers "Use … as workspace name"; the Name tab takes the text as is.
+ await d.getByPlaceholder(/Type a name|Workspace name/).fill(wsName)
+ await page.getByRole('option', { name: /as workspace name/ }).click({ timeout: 2000 }).catch(() => {})
+ await shot(step, 'composer-orca-cloud')
+ await clearToasts()
+ const t0 = Date.now()
+ await d.getByRole('button', { name: /Create workspace/ }).click()
+ let provisioningShot = false
+ let firstOutput = null
+ const phases = []
+ for (;;) {
+ const s = await page.evaluate(() => {
+ const body = document.body.innerText
+ const rows = [...document.querySelectorAll('.xterm-rows')].filter((r) => r.getBoundingClientRect().width > 0)
+ return {
+ starting: /Starting your cloud workspace/.test(body),
+ failed: /Couldn.t start your cloud workspace/.test(body),
+ text: (body.match(/Starting your cloud workspace[\s\S]{0,400}/)?.[0] ?? '').replace(/\s+/g, ' '),
+ xterm: rows.map((r) => r.innerText.trim().length)
+ }
+ })
+ const t = Date.now() - t0
+ const phase = s.text.match(/(Start sandbox|Prepare environment|Open workspace) In progress/)?.[1] ?? null
+ if (phase && phases.at(-1)?.phase !== phase) phases.push({ phase, t })
+ if (s.starting && !provisioningShot && t > 1500) {
+ await shot(step, 'provisioning')
+ provisioningShot = true
+ }
+ if (s.failed) {
+ step.detail = `create failed: ${redact((await page.locator('main').innerText().catch(() => '')).slice(0, 400))}`
+ throw new Error(step.detail)
+ }
+ if (!s.starting && s.xterm.some((n) => n > 0)) {
+ firstOutput = t
+ break
+ }
+ if (t > 300_000) throw new Error('workspace not ready in 300s')
+ await sleep(100)
+ }
+ step.timings.createToFirstTerminalOutputMs = firstOutput
+ step.timings.phases = phases
+ await dismissNoise()
+ await shot(step, 'ready-agent-terminal')
+ // A plain shell next to the agent the composer launches: usable once it prints a prompt.
+ const t1 = Date.now()
+ await openNewTerminalTab()
+ await page.waitForFunction(
+ () => [...document.querySelectorAll('.xterm-rows')].some((r) => r.getBoundingClientRect().width > 0 && /\$\s*$/.test(r.innerText.trim())),
+ null,
+ { timeout: 60_000 }
+ )
+ step.timings.newTerminalToPromptMs = Date.now() - t1
+ step.timings.createToShellPromptMs = Date.now() - t0
+ const prompt = lastLine((await visibleTerminals()).pop() ?? '')
+ step.prompt = prompt
+ await shot(step, 'ready-shell')
+ const sb = await sandboxForWorkspace(t0)
+ sandboxId = sb.newest?.id ?? null
+ if (sandboxId) created.sandboxIds.add(sandboxId)
+ step.sandboxId = sandboxId
+ step.sandboxOwnerOk = Boolean(sandboxId)
+ // The cloud copy of the repo registers as its own project; its name should stay the repo's.
+ const repoNames = await store(() => window.__store.getState().repos.map((r) => ({ name: r.displayName, path: r.path })))
+ step.repos = repoNames
+ const renamed = repoNames.find((r) => r.path.startsWith('/vercel/') && r.name !== 'orca-sandbox-push-test')
+ if (renamed) {
+ const projectHeader = await page.getByText(renamed.name, { exact: true }).first().isVisible().catch(() => false)
+ bug('cli.desktop', {
+ title: `Cloud repo registers as "${renamed.name}" (basename of ${renamed.path}) and the sidebar project group / composer project list show that name instead of orca-sandbox-push-test`,
+ expected: 'project keeps the repo name orca-sandbox-push-test',
+ actual: `repo ${renamed.path} named "${renamed.name}"; sidebar header shows it: ${projectHeader}`,
+ repro: 'Create an Orca Cloud workspace for a repo whose cloud checkout lands in /vercel/sandbox/project; look at the sidebar project header',
+ screenshot: step.screenshots.at(-1)
+ })
+ }
+ const cloudy = /^ubuntu@/.test(prompt) && /orca\/workspaces/.test(prompt)
+ step.status = cloudy && sandboxId ? 'pass' : 'partial'
+ step.detail = `prompt "${prompt}"; sandbox ${sandboxId ?? 'NOT listed for ' + USER}; create->first terminal output ${firstOutput}ms, create->shell prompt ${step.timings.createToShellPromptMs}ms`
+ if (!sandboxId) {
+ bug('cli.desktop', {
+ title: 'Created sandbox is not owned by the signed-in dev account',
+ expected: `GET /v1/sandboxes as ${USER} lists the new sandbox`,
+ actual: 'not listed',
+ screenshot: step.screenshots.at(-1)
+ })
+ }
+ })
+ if (!sandboxId) results.notes.push('No sandbox id resolved for the workspace; lifecycle checks below rely on the UI only.')
+ const workspaceRow = await rowTitle(wsName).isVisible().catch(() => false)
+ if (!workspaceRow) {
+ for (const id of ['term.keystroke', 'acc.port-forward', 'acc.ssh', 'life.sleep-wake', 'life.transfer-local', 'life.destroy']) {
+ results.steps.push({ id, status: 'skipped', detail: 'no cloud workspace row (create failed)', screenshots: [], timings: {} })
+ }
+ save()
+ return
+ }
+
+ await runStep('term.keystroke', `First-keystroke latency in the cloud xterm (n=${KEY_SAMPLES})`, async (step) => {
+ await focusTerminal()
+ const samples = []
+ for (let i = 0; i < KEY_SAMPLES; i++) samples.push(await measureKeystroke(`k${i}${String.fromCharCode(97 + i)}q`))
+ step.timings.keystrokeToEchoMs = stats(samples)
+ step.status = pct(samples, 95) < 250 ? 'pass' : 'partial'
+ step.detail = `keydown -> char in xterm DOM: p50 ${pct(samples, 50)}ms, p95 ${pct(samples, 95)}ms (DOM renderer)`
+ await shot(step, 'typed')
+ })
+
+ await runStep('acc.port-forward', `Ports panel detects :${PORT}, forward to localhost, ${HTTP_SAMPLES} GETs`, async (step) => {
+ const preLocal = await fetch(`http://localhost:${PORT}/`).then((r) => r.status, () => 'closed')
+ step.localPortBefore = preLocal
+ await focusTerminal()
+ await page.keyboard.type(`python3 -m http.server ${PORT}\n`)
+ await page.waitForFunction((p) => [...document.querySelectorAll('.xterm-rows')].some((r) => r.innerText.includes(`port ${p}`)), PORT, { timeout: 20_000 })
+ const tServe = Date.now()
+ await page.locator('[aria-label^="Ports ("]').first().click()
+ const portRow = page.getByText(`:${PORT}`, { exact: true }).first()
+ await portRow.waitFor({ timeout: 60_000 })
+ step.timings.serveToPanelRowMs = Date.now() - tServe
+ let forwardedMs = null
+ for (let i = 0; i < 120; i++) {
+ const ok = await page.getByText(`localhost:${PORT}`).first().isVisible().catch(() => false)
+ const open = await fetch(`http://localhost:${PORT}/`).then((r) => r.ok, () => false)
+ if (ok && open) {
+ forwardedMs = Date.now() - tServe
+ break
+ }
+ await sleep(250)
+ }
+ step.timings.serveToLocalhostReachableMs = forwardedMs
+ step.autoForwardToast = (await toasts()).find((t) => /Forwarded/.test(t)) ?? null
+ await shot(step, 'ports-panel')
+ if (forwardedMs === null) {
+ step.status = 'partial'
+ step.detail = 'port row shown but localhost not reachable within 30s'
+ return
+ }
+ const samples = []
+ let server = null
+ let body = ''
+ for (let i = 0; i < HTTP_SAMPLES; i++) {
+ const s = performance.now()
+ const r = await fetch(`http://localhost:${PORT}/`)
+ body = await r.text()
+ samples.push(Math.round(performance.now() - s))
+ server = r.headers.get('server')
+ }
+ step.timings.httpGetMs = stats(samples)
+ const fromVm = /SimpleHTTP/i.test(server ?? '') && /README\.md/.test(body)
+ step.status = fromVm && preLocal === 'closed' ? 'pass' : 'partial'
+ step.detail = `auto-forwarded ${step.autoForwardToast ? `("${step.autoForwardToast.split('\n')[0]}")` : ''}; server "${server}"; listing has README.md: ${/README\.md/.test(body)}; GET p50 ${pct(samples, 50)}ms p95 ${pct(samples, 95)}ms`
+ await focusTerminal()
+ await page.keyboard.press('Control+C')
+ await sleep(500)
+ })
+
+ await runStep('acc.ssh', 'Copy SSH Command / Open in editor / orca ssh', async (step) => {
+ await openRowMenu(wsName)
+ step.contextMenu = (await menuItems()).map((t) => t.split('\n')[0])
+ await page.getByRole('menuitem', { name: 'Open in', exact: true }).hover()
+ await sleep(600)
+ const afterOpenIn = (await menuItems()).map((t) => t.replace(/\n/g, ' '))
+ step.openInOptions = afterOpenIn.slice(step.contextMenu.length)
+ await shot(step, 'open-in-menu')
+ await page.getByRole('menuitem', { name: 'SSH', exact: true }).hover()
+ await sleep(600)
+ const clipBefore = spawnSync('pbpaste', { encoding: 'utf8' }).stdout
+ await clearToasts()
+ const t0 = Date.now()
+ await page.getByRole('menuitem', { name: 'Copy SSH Command' }).click()
+ await page.waitForFunction(() => (window.__w9dToasts ?? []).some((t) => /copied|Couldn/.test(t)), null, { timeout: 90_000 })
+ step.timings.copySshCommandMs = Date.now() - t0
+ const tl = await toasts()
+ await shot(step, 'ssh-copied')
+ const clip = spawnSync('pbpaste', { encoding: 'utf8' }).stdout.trim()
+ spawnSync('pbcopy', { input: clipBefore })
+ step.toasts = tl.map(redact)
+ const copied = /^ssh /.test(clip) ? clip : null
+ step.sshCommand = redact(copied ?? '(not copied)')
+ let sshOk = false
+ if (copied) {
+ const parts = copied.split(/\s+/).slice(1)
+ const t1 = Date.now()
+ const r = spawnSync('ssh', ['-o', 'BatchMode=yes', '-o', 'ConnectTimeout=30', ...parts, 'echo ssh-ok $(hostname)'], { encoding: 'utf8', timeout: 90_000 })
+ step.timings.sshRoundTripMs = Date.now() - t1
+ step.sshResult = redact((r.stdout + r.stderr).trim().split('\n').slice(-3).join(' | '))
+ sshOk = /ssh-ok/.test(r.stdout)
+ }
+ const cli = path.join(repoRoot, 'out/cli/index.js')
+ let cliOk = false
+ if (existsSync(cli)) {
+ const t2 = Date.now()
+ const r = spawnSync(process.execPath, [cli, 'ssh', wsName, '--command', 'echo orca-ssh-ok $(hostname)'], {
+ encoding: 'utf8',
+ timeout: 120_000,
+ env: { ...process.env, ORCA_USER_DATA_PATH: results.app.userData }
+ })
+ step.timings.orcaSshMs = Date.now() - t2
+ step.orcaSshResult = redact((r.stdout + r.stderr).trim().split('\n').slice(-3).join(' | '))
+ cliOk = /orca-ssh-ok/.test(r.stdout)
+ } else {
+ step.orcaSshResult = 'out/cli/index.js missing'
+ }
+ step.status = sshOk && cliOk ? 'pass' : sshOk || cliOk ? 'partial' : 'fail'
+ step.detail = `Copy SSH Command -> ${step.sshCommand}; ssh: ${sshOk ? 'ok' : 'FAILED'} (${step.timings.sshRoundTripMs ?? '-'}ms); orca ssh: ${cliOk ? 'ok' : 'FAILED'} (${step.timings.orcaSshMs ?? '-'}ms); Open in offers ${JSON.stringify(step.openInOptions)} (not launched)`
+ })
+
+ await runStep('life.sleep-wake', `Suspend VM via UI, wake by opening the workspace, wake -> first echo (n=${WAKE_CYCLES})`, async (step) => {
+ const suspendMs = []
+ const wakeEchoMs = []
+ const cycles = []
+ for (let c = 0; c < WAKE_CYCLES; c++) {
+ const cyc = { cycle: c + 1 }
+ cycles.push(cyc)
+ await openRowMenu(wsName)
+ const ts = Date.now()
+ await page.getByRole('menuitem', { name: /^(Suspend VM|Sleep)$/ }).first().click()
+ await page.getByRole('dialog').getByRole('button', { name: /^(Sleep|Suspend)/ }).click({ timeout: 1500 }).catch(() => {})
+ if (sandboxId) {
+ const w = await waitSandboxStatus(sandboxId, ['suspended', 'stopped'], 120_000)
+ cyc.suspend = w
+ if (w.timedOut) throw new Error(`sandbox not suspended after 120s (last ${w.status})`)
+ } else {
+ await sleep(20_000)
+ }
+ cyc.suspendMs = Date.now() - ts
+ suspendMs.push(cyc.suspendMs)
+ await sleep(1500)
+ if (c === 0) await shot(step, 'asleep')
+ // Wake: open the workspace from the sidebar, keep typing until an echo makes it back.
+ await clearToasts()
+ const t0 = Date.now()
+ await rowTitle(wsName).click()
+ let wakingShot = c !== 0
+ let echo = null
+ let attempts = 0
+ let lastType = 0
+ const marker = `echo wake${c + 1}`
+ while (Date.now() - t0 < 180_000) {
+ if (!wakingShot && Date.now() - t0 > 1500) {
+ await shot(step, 'waking')
+ wakingShot = true
+ }
+ const terms = await visibleTerminals()
+ if (terms.some((t) => t.includes(marker))) {
+ echo = Date.now() - t0
+ break
+ }
+ if (terms.length && Date.now() - lastType > 1500) {
+ await focusTerminal().catch(() => {})
+ await page.keyboard.press('Control+U')
+ await page.keyboard.type(marker)
+ attempts++
+ lastType = Date.now()
+ }
+ await sleep(50)
+ }
+ if (echo === null) throw new Error(`no echo within 180s of wake (cycle ${c + 1})`)
+ cyc.wakeToEchoMs = echo
+ cyc.typeAttempts = attempts
+ wakeEchoMs.push(echo)
+ await page.keyboard.press('Control+U')
+ if (sandboxId) cyc.statusAfterWake = (await sandboxStatus(sandboxId)).status
+ cyc.prompt = lastLine((await visibleTerminals()).pop() ?? '')
+ if (c === 0) await shot(step, 'woken')
+ await sleep(2000)
+ }
+ step.cycles = cycles
+ step.timings.suspendToSuspendedMs = stats(suspendMs)
+ step.timings.wakeToFirstEchoMs = stats(wakeEchoMs)
+ step.status = wakeEchoMs.length === WAKE_CYCLES ? 'pass' : 'partial'
+ step.detail = `${wakeEchoMs.length}/${WAKE_CYCLES} cycles; suspend p50 ${pct(suspendMs, 50)}ms; wake->echo p50 ${pct(wakeEchoMs, 50)}ms p95 ${pct(wakeEchoMs, 95)}ms`
+ })
+
+ await runStep('life.transfer-local', 'Move to this computer', async (step) => {
+ await openRowMenu(wsName)
+ const items = (await menuItems()).map((t) => t.split('\n')[0])
+ if (!items.includes('Move to this computer')) {
+ step.status = 'partial'
+ step.detail = `absent; menu: ${JSON.stringify(items)}`
+ await page.keyboard.press('Escape')
+ return
+ }
+ await clearToasts()
+ const t0 = Date.now()
+ await page.getByRole('menuitem', { name: 'Move to this computer' }).click()
+ await sleep(800)
+ const dlg = await page.getByRole('dialog').innerText().catch(() => null)
+ if (dlg) {
+ step.confirmDialog = dlg.replace(/\s+/g, ' ')
+ await page.getByRole('dialog').getByRole('button', { name: /Move|Transfer|Continue/ }).first().click()
+ }
+ await shot(step, 'started')
+ let local = null
+ while (Date.now() - t0 < 180_000) {
+ local = await store((n) => Object.values(window.__store.getState().worktreesByRepo ?? {}).flat().find((w) => !w.path.startsWith('/vercel') && w.path.includes(n)), wsName)
+ if (local) break
+ await sleep(500)
+ }
+ step.timings.transferMs = Date.now() - t0
+ step.toasts = (await toasts()).map(redact)
+ await sleep(1500)
+ await shot(step, 'done')
+ if (!local) throw new Error('no local worktree appeared within 180s')
+ created.localPaths.push(local.path)
+ step.localWorktree = { name: local.displayName, path: local.path, branch: local.branch }
+ const files = existsSync(local.path) ? readdirSync(local.path) : []
+ const cloudStill = sandboxId ? (await sandboxStatus(sandboxId)).status : null
+ step.cloudStatusAfter = cloudStill
+ step.status = files.includes('README.md') ? 'pass' : 'partial'
+ step.detail = `local worktree "${local.displayName}" at ${local.path} (files: ${files.join(',')}); ${step.timings.transferMs}ms; cloud sandbox after: ${cloudStill} (copy, not move); no confirmation dialog: ${!dlg}`
+ // Remove the local copy through the UI so the repo stays clean.
+ await openRowMenu(local.displayName)
+ await page.getByRole('menuitem', { name: /^Delete/ }).click()
+ await page.getByRole('dialog').getByRole('button', { name: /^Delete( Workspace)?$/ }).click()
+ await rowTitle(local.displayName).waitFor({ state: 'detached', timeout: 60_000 })
+ step.localCopyDeleted = !existsSync(local.path)
+ })
+
+ await runStep('life.destroy', 'Delete the cloud workspace through the UI and confirm the VM is gone', async (step) => {
+ await openRowMenu(wsName)
+ await page.getByRole('menuitem', { name: /^Delete/ }).click()
+ const d = page.getByRole('dialog')
+ step.dialog = (await d.innerText()).replace(/\s+/g, ' ')
+ await shot(step, 'confirm')
+ const t0 = Date.now()
+ await d.getByRole('button', { name: /^Delete( Workspace)?$/ }).click()
+ await rowTitle(wsName).waitFor({ state: 'detached', timeout: 60_000 })
+ step.timings.uiRowGoneMs = Date.now() - t0
+ await shot(step, 'after')
+ if (!sandboxId) {
+ step.status = 'partial'
+ step.detail = 'row removed; no sandbox id to confirm against the API'
+ return
+ }
+ const w = await waitSandboxStatus(sandboxId, ['destroyed', 'deleted', 'not_found'], 120_000)
+ step.timings.deleteToDestroyedMs = w.ms
+ step.finalSandboxStatus = w.status
+ const listed = (await listSandboxes(USER)).items.some((s) => s.id === sandboxId)
+ step.status = !w.timedOut ? 'pass' : 'fail'
+ step.detail = `row gone in ${step.timings.uiRowGoneMs}ms; sandbox ${sandboxId} -> ${w.status} after ${w.ms}ms; still listed: ${listed}`
+ if (w.timedOut) {
+ bug('life.destroy', {
+ title: 'Deleting a cloud workspace in the UI leaves the sandbox running',
+ expected: 'sandbox destroyed',
+ actual: `status ${w.status} 120s after delete`,
+ screenshot: step.screenshots.at(-1)
+ })
+ }
+ })
+}
+
+async function cleanup() {
+ const c = results.cleanup
+ try {
+ c.sandboxes = await deleteAllSandboxes(USER)
+ for (const id of created.sandboxIds) {
+ const s = await sandboxStatus(id)
+ if (!['destroyed', 'deleted', 'not_found'].includes(s.status)) {
+ const r = await sandboxApi(USER, 'DELETE', `/v1/sandboxes/${id}`)
+ c.sandboxes.deleted.push({ id, forced: true, deleteStatus: r.status })
+ }
+ }
+ c.remaining = (await listSandboxes(USER)).items.map((s) => ({ id: s.id, status: s.status ?? s.state }))
+ } catch (e) {
+ c.sandboxError = e.message
+ }
+ try {
+ await stopApp()
+ c.appStopped = true
+ } catch (e) {
+ c.appStopError = e.message
+ }
+ // Orca's terminal daemon (and its shells) outlive the app by design; stop the ones bound to this
+ // run's throwaway profile — nothing else matches that unique temp path.
+ if (created.tmp) {
+ await sleep(2000)
+ const ps = spawnSync('ps', ['-axo', 'pid=,command='], { encoding: 'utf8' }).stdout
+ const pids = ps
+ .split('\n')
+ .filter((l) => l.includes(created.tmp))
+ .map((l) => Number(l.trim().split(/\s+/)[0]))
+ .filter((pid) => pid && pid !== process.pid)
+ for (const pid of pids) {
+ try {
+ process.kill(pid, 'SIGTERM')
+ } catch {}
+ }
+ c.profileProcessesStopped = pids.length
+ }
+ for (const p of created.localPaths) {
+ if (existsSync(p)) {
+ c.leftoverLocal = [...(c.leftoverLocal ?? []), p]
+ }
+ try {
+ rmdirSync(path.dirname(p)) // only succeeds when empty
+ } catch {}
+ }
+ if (created.tmp && process.env.W9D_KEEP_PROFILE !== '1') rmSync(created.tmp, { recursive: true, force: true })
+ save()
+}
+
+let exitCode = 0
+try {
+ await main()
+} catch (e) {
+ results.fatal = redact(e.stack ?? e.message)
+ exitCode = 1
+} finally {
+ await cleanup()
+ results.finishedAt = new Date().toISOString()
+ results.summary = results.steps.map((s) => `${s.id}: ${s.status}`)
+ save()
+ log('results', path.join(artifactsDir, 'results.json'))
+ process.exit(exitCode || (results.steps.some((s) => s.status === 'fail') ? 2 : 0))
+}
diff --git a/cloud/apps/acceptance/desktop/sandbox-api.mjs b/cloud/apps/acceptance/desktop/sandbox-api.mjs
new file mode 100644
index 00000000000..2b0ecc2b607
--- /dev/null
+++ b/cloud/apps/acceptance/desktop/sandbox-api.mjs
@@ -0,0 +1,72 @@
+// Direct dev control-plane calls for the desktop acceptance run: list/delete leftovers and probe
+// endpoints the UI depends on. Tokens are dev-signed (ES256) with the kid from the key file.
+import { readFileSync } from 'node:fs'
+import { SignJWT, importJWK } from 'jose'
+
+export const SANDBOX_URL =
+ process.env.ORCA_CLOUD_SANDBOX_URL ?? 'https://orca-sandbox-control-dev.lovecast.workers.dev'
+export const DEV_KEY_FILE = process.env.ORCA_CLOUD_SANDBOX_DEV_KEY_FILE ?? '/tmp/vdrv/secrets/dev-jwk.json'
+
+export async function sandboxToken(sub) {
+ const file = JSON.parse(readFileSync(DEV_KEY_FILE, 'utf8'))
+ const kid = file.private.kid ?? file.kid ?? 'dev1'
+ const key = await importJWK(file.private, 'ES256')
+ return new SignJWT({ sub, purpose: 'sandbox' })
+ .setProtectedHeader({ alg: 'ES256', kid })
+ .setIssuer('orca-sandbox-dev-test')
+ .setAudience('orca-sandbox')
+ .setIssuedAt()
+ .setExpirationTime('5m')
+ .sign(key)
+}
+
+export async function sandboxApi(sub, method, path, body) {
+ const res = await fetch(SANDBOX_URL + path, {
+ method,
+ headers: {
+ authorization: `Bearer ${await sandboxToken(sub)}`,
+ ...(body ? { 'content-type': 'application/json' } : {})
+ },
+ body: body ? JSON.stringify(body) : undefined
+ })
+ const text = await res.text()
+ let json = null
+ try {
+ json = JSON.parse(text)
+ } catch {}
+ return { status: res.status, json, text: json ? undefined : text.slice(0, 300) }
+}
+
+export async function listSandboxes(sub) {
+ const r = await sandboxApi(sub, 'GET', '/v1/sandboxes')
+ const items = r.json?.sandboxes ?? r.json?.items ?? (Array.isArray(r.json) ? r.json : [])
+ return { status: r.status, items }
+}
+
+/** Deletes every sandbox owned by `sub` that is not already gone; returns what it did. */
+export async function deleteAllSandboxes(sub) {
+ const { status, items } = await listSandboxes(sub)
+ const live = items.filter((s) => !/^(deleted|destroyed|deleting)$/i.test(String(s.state ?? s.status ?? '')))
+ const deleted = []
+ for (const s of live) {
+ const r = await sandboxApi(sub, 'DELETE', `/v1/sandboxes/${encodeURIComponent(s.id)}`)
+ deleted.push({ id: s.id, state: s.state ?? s.status, deleteStatus: r.status })
+ }
+ return { listStatus: status, seen: items.length, deleted }
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+ const [cmd = 'list', sub = 'w9-desktop'] = process.argv.slice(2)
+ if (cmd === 'list') {
+ const { status, items } = await listSandboxes(sub)
+ console.log(
+ JSON.stringify(
+ { status, items: items.map((s) => ({ id: s.id, state: s.state ?? s.status, name: s.name, createdAt: s.createdAt })) },
+ null,
+ 1
+ )
+ )
+ } else if (cmd === 'cleanup') {
+ console.log(JSON.stringify(await deleteAllSandboxes(sub), null, 1))
+ }
+}
diff --git a/cloud/apps/acceptance/src/checks/billing.ts b/cloud/apps/acceptance/src/checks/billing.ts
index da2cea3c465..ed4bddfc762 100644
--- a/cloud/apps/acceptance/src/checks/billing.ts
+++ b/cloud/apps/acceptance/src/checks/billing.ts
@@ -170,10 +170,10 @@ export const billingChecks: Record = {
const same = await push(2)
const ok =
added.body?.status === 'updated' && qtyAfterAdd === 3 && invoice?.billing_reason === 'subscription_update' && invoice?.status === 'paid' &&
- removed.body?.status === 'updated' && qtyAfterRemove === 2 && ent?.seats === 2 && same.body?.status === 'unchanged'
+ removed.body?.status === 'deferred' && qtyAfterRemove === 3 && removed.body?.effectiveAt > Date.now() && same.body?.status === 'deferred'
return {
status: ok ? 'pass' : 'fail',
- detail: `member added -> ${qtyAfterAdd} seats, proration invoice ${invoice?.billing_reason}/${invoice?.status} $${((invoice?.amount_paid ?? 0) / 100).toFixed(2)} charged now; member removed -> ${qtyAfterRemove} seats (credit next invoice), entitlements ${ent?.seats} seats; repeat push -> ${same.body?.status}`,
+ detail: `member added -> ${qtyAfterAdd} seats, proration invoice ${invoice?.billing_reason}/${invoice?.status} $${((invoice?.amount_paid ?? 0) / 100).toFixed(2)} charged now; member removed -> ${removed.body?.status} until ${removed.body?.effectiveAt ? new Date(removed.body.effectiveAt).toISOString().slice(0, 10) : '?'} (peak-seat billing: paid seat kept to period end, no mid-period refund abuse), still ${qtyAfterRemove} seats / entitlements ${ent?.seats}; repeat push -> ${same.body?.status}`,
evidence: [log('seat pushes', JSON.stringify({ added: added.body, removed: removed.body, same: same.body }))]
}
},
diff --git a/cloud/apps/acceptance/src/checks/core.ts b/cloud/apps/acceptance/src/checks/core.ts
index 929a89860be..76d41ea0f12 100644
--- a/cloud/apps/acceptance/src/checks/core.ts
+++ b/cloud/apps/acceptance/src/checks/core.ts
@@ -66,15 +66,15 @@ export const coreChecks: Record = {
const r = await ctx.exec(p.sandbox.session!.id, 'df -BG --output=size,avail /vercel/sandbox | tail -1')
const [size, avail] = r.stdout.trim().split(/\s+/).map((v) => Number.parseInt(v, 10))
return {
- status: (size ?? 0) >= 30 ? 'partial' : 'fail',
- detail: `disk ${size} GB (${avail} GB free). Conductor: 64 GB. Low-disk alert not yet probed (W1/W2).`,
+ status: (size ?? 0) >= 60 ? 'pass' : (size ?? 0) >= 30 ? 'partial' : 'fail',
+ detail: `disk ${size} GB (${avail} GB free). Conductor: 64 GB. Low-disk banner is desktop UI (W1 #47), driven by status.metrics.diskUsedBytes.`,
evidence: [log('df', r.stdout)]
}
},
'sub.tools': async (ctx) => {
const p = await ctx.primary()
- const tools = ['node', 'npm', 'pnpm', 'python3', 'git', 'gh', 'jq', 'rg', 'git-lfs', 'tmux', 'vim', 'claude', 'codex', 'orca', 'chromium', 'google-chrome']
+ const tools = ['node', 'npm', 'pnpm', 'python3', 'git', 'gh', 'jq', 'rg', 'git-lfs', 'tmux', 'vim', 'claude', 'codex', 'orca', 'google-chrome']
const r = await ctx.exec(p.sandbox.session!.id, LOGIN(`for t in ${tools.join(' ')}; do command -v $t >/dev/null 2>&1 && echo "$t=yes" || echo "$t=no"; done; node --version; claude --version 2>/dev/null | head -1; codex --version 2>/dev/null | head -1`))
const have = new Set(r.stdout.split('\n').filter((l) => l.endsWith('=yes')).map((l) => l.slice(0, -4)))
const missing = tools.filter((t) => !have.has(t))
diff --git a/cloud/apps/acceptance/src/checks/lifecycle.ts b/cloud/apps/acceptance/src/checks/lifecycle.ts
index 238f88b6635..912aaf6d1bc 100644
--- a/cloud/apps/acceptance/src/checks/lifecycle.ts
+++ b/cloud/apps/acceptance/src/checks/lifecycle.ts
@@ -116,8 +116,10 @@ export const lifecycleChecks: Record = {
await serve(ctx, b.sessionId, 4173)
await openTerminal(b.code, b.root)
let found: any = null
- for (let i = 0; i < 6 && !found; i++) {
- const r = await orca(b.code, ['ports', '--all'])
+ let last = ''
+ for (let i = 0; i < 3 && !found; i++) {
+ const r = await orca(b.code, ['ports', '--all'], 30_000)
+ last = `${r.ms} ms ${r.stderr.slice(0, 200) || JSON.stringify(r.json?.error ?? r.json).slice(0, 200)}`
found = JSON.stringify(r.json ?? '').includes('4173') ? r.json : null
if (!found) {
await sleep(2000)
@@ -125,7 +127,7 @@ export const lifecycleChecks: Record = {
}
return {
status: found ? 'pass' : 'fail',
- detail: found ? 'orca ports over the pairing sees port 4173 in the VM' : 'port 4173 not listed by orca ports',
+ detail: found ? 'orca ports over the pairing sees port 4173 in the VM' : `port 4173 not listed by orca ports over the pairing code (last: ${last})`,
evidence: found ? [log('orca ports', JSON.stringify(found).slice(0, 1200))] : []
}
},
@@ -174,7 +176,8 @@ export const lifecycleChecks: Record = {
const raw = Buffer.from(publicKey.export({ format: 'jwk' }).x!, 'base64url')
const blob = Buffer.concat([Buffer.from([0, 0, 0, 11]), Buffer.from('ssh-ed25519'), Buffer.from([0, 0, 0, 32]), raw]).toString('base64')
const r = await control.as(ctx.solo, 'POST', `/v1/sandboxes/${b.id}/ssh-keys`, { publicKey: `ssh-ed25519 ${blob} w9-acceptance` })
- const listening = (await ctx.exec(b.sessionId, `ss -ltn 2>/dev/null | grep -c ":${r.body?.ssh?.port ?? 2222} " ; grep -c w9-acceptance ~/.ssh/authorized_keys 2>/dev/null`)).stdout.trim().split('\n')
+ const portHex = (r.body?.ssh?.port ?? 2222).toString(16).toUpperCase().padStart(4, '0')
+ const listening = (await ctx.exec(b.sessionId, `cat /proc/net/tcp /proc/net/tcp6 2>/dev/null | awk '$4=="0A"' | grep -c ":${portHex} "; sudo sh -c 'cat /home/*/.ssh/authorized_keys /root/.ssh/authorized_keys 2>/dev/null' | grep -c w9-acceptance`)).stdout.trim().split('\n')
const ok = r.status === 200 && Number(listening[0]) > 0 && Number(listening[1]) > 0
return {
status: ok ? 'partial' : 'fail',
diff --git a/cloud/apps/acceptance/src/checks/platform.ts b/cloud/apps/acceptance/src/checks/platform.ts
index 6f899f1184e..3e291f6997b 100644
--- a/cloud/apps/acceptance/src/checks/platform.ts
+++ b/cloud/apps/acceptance/src/checks/platform.ts
@@ -251,7 +251,7 @@ export const platformChecks: Record = {
// ---------------- W5 ----------------
'api.rest': async (ctx) => {
- const created = await api(ctx, 'POST', '/v1/workspaces', { name: `w9-api-${tag(ctx)}`, repo: config.testRepo })
+ const created = await api(ctx, 'POST', '/v1/workspaces', { name: `w9-api-${tag(ctx)}`, repo: { url: config.testRepo } })
const id = created.body?.workspace?.id ?? created.body?.id
if (!id) {
return { status: 'fail', detail: `POST /v1/workspaces ${created.status} ${JSON.stringify(created.body).slice(0, 300)}` }
@@ -295,7 +295,7 @@ export const platformChecks: Record = {
'api.routine-webhook': async (ctx) => {
const r = await api(ctx, 'POST', '/v1/routines', {
name: `w9-hook-${tag(ctx)}`,
- repo: config.testRepo,
+ repo: { url: config.testRepo },
trigger: { kind: 'webhook' },
agent: 'claude',
prompt: 'Reply with the single word ok. Do not change any files.',
@@ -334,7 +334,7 @@ export const platformChecks: Record = {
'api.routine-cron': async (ctx) => {
const r = await api(ctx, 'POST', '/v1/routines', {
name: `w9-cron-${tag(ctx)}`,
- repo: config.testRepo,
+ repo: { url: config.testRepo },
trigger: { kind: 'schedule', cron: '0 9 * * 1-5', timezone: 'America/Los_Angeles' },
agent: 'claude',
prompt: 'Reply ok.',
@@ -345,7 +345,7 @@ export const platformChecks: Record = {
const id = r.body?.routine?.id
const gh = await api(ctx, 'POST', '/v1/routines', {
name: `w9-gh-${tag(ctx)}`,
- repo: config.testRepo,
+ repo: { url: config.testRepo },
trigger: { kind: 'github', events: ['pull_request.opened'], labels: ['orca'] },
agent: 'codex',
prompt: 'Review.',
diff --git a/cloud/apps/acceptance/src/checks/secrets.ts b/cloud/apps/acceptance/src/checks/secrets.ts
index b7c932e8f80..9fed85fd724 100644
--- a/cloud/apps/acceptance/src/checks/secrets.ts
+++ b/cloud/apps/acceptance/src/checks/secrets.ts
@@ -23,9 +23,9 @@ function teamSandbox(ctx: Ctx) {
const tag = ctx.run.file.runId
await put(ctx, ctx.owner, 'org', 'W9_SHARED', `org-${tag}`)
await put(ctx, ctx.owner, 'org', 'W9_OVERRIDE', `org-${tag}`)
- await put(ctx, ctx.owner, 'org', 'OPENAI_API_KEY', `org-fake-${tag}`)
+ await put(ctx, ctx.owner, 'org', 'AWS_REGION', `org-fake-${tag}`)
await put(ctx, ctx.member, 'personal', 'W9_OVERRIDE', `personal-${tag}`)
- await put(ctx, ctx.member, 'personal', 'OPENAI_API_KEY', `personal-fake-${tag}`)
+ await put(ctx, ctx.member, 'personal', 'AWS_REGION', `personal-fake-${tag}`)
const r = await ctx.sandboxes.create(ctx.member, { workspaceId: `w9-team-${tag}` })
if (r.status !== 200) {
throw new Error(`team sandbox create ${r.status} ${JSON.stringify(r.body)}`)
@@ -62,9 +62,9 @@ export const secretChecks: Record = {
'sec.precedence': async (ctx) => {
const tag = ctx.run.file.runId
const box = await teamSandbox(ctx)
- const env = await orcadEnv(ctx, box.sessionId, ['W9_OVERRIDE', 'OPENAI_API_KEY'])
+ const env = await orcadEnv(ctx, box.sessionId, ['W9_OVERRIDE', 'AWS_REGION'])
const personalWinsEnv = env.W9_OVERRIDE === `personal-${tag}`
- const orgWinsAgentKey = env.OPENAI_API_KEY === `org-fake-${tag}`
+ const orgWinsAgentKey = env.AWS_REGION === `org-fake-${tag}`
return {
status: personalWinsEnv && orgWinsAgentKey ? 'pass' : 'fail',
detail: `env: personal beats org = ${personalWinsEnv}; agent key: org beats personal = ${orgWinsAgentKey}`
@@ -84,9 +84,9 @@ export const secretChecks: Record = {
for (const [scope, who, name] of [
['org', ctx.owner, 'W9_SHARED'],
['org', ctx.owner, 'W9_OVERRIDE'],
- ['org', ctx.owner, 'OPENAI_API_KEY'],
+ ['org', ctx.owner, 'AWS_REGION'],
['personal', ctx.member, 'W9_OVERRIDE'],
- ['personal', ctx.member, 'OPENAI_API_KEY']
+ ['personal', ctx.member, 'AWS_REGION']
] as const) {
await control.as(who, 'DELETE', `/v1/secrets/${scope}/${name}`)
}
diff --git a/cloud/apps/acceptance/src/inspect-build.ts b/cloud/apps/acceptance/src/inspect-build.ts
new file mode 100644
index 00000000000..a3a038c9ffd
--- /dev/null
+++ b/cloud/apps/acceptance/src/inspect-build.ts
@@ -0,0 +1,12 @@
+// Debug helper: prints an environment build (as the team owner) without secrets.
+// tsx src/inspect-build.ts
+import { control } from './clients'
+
+const [envId, buildId] = process.argv.slice(2)
+const owner = { sub: 'w9-owner', org: 'w9-eval-team', orgRole: 'owner' as const }
+const envs = await control.as(owner, 'GET', '/v1/environments')
+console.log('environments', JSON.stringify(envs.body).slice(0, 1500))
+if (envId && buildId) {
+ const b = await control.as(owner, 'GET', `/v1/environments/${envId}/builds/${buildId}`)
+ console.log('build', JSON.stringify(b.body, null, 1).slice(0, 4000))
+}
diff --git a/cloud/apps/acceptance/src/list-provider.ts b/cloud/apps/acceptance/src/list-provider.ts
new file mode 100644
index 00000000000..cc1cb2ce498
--- /dev/null
+++ b/cloud/apps/acceptance/src/list-provider.ts
@@ -0,0 +1,8 @@
+// Lists every sandbox in Orca's Vercel project (status + tags) so stray eval resources are visible.
+// tsx src/list-provider.ts
+import { vercel } from './clients'
+
+const { sandboxes } = await vercel().listSandboxes()
+for (const s of sandboxes) {
+ console.log(`${s.name}\t${s.status}\t${JSON.stringify(s.tags)}`)
+}
diff --git a/cloud/apps/acceptance/src/probe-build.ts b/cloud/apps/acceptance/src/probe-build.ts
new file mode 100644
index 00000000000..4bf1a79270e
--- /dev/null
+++ b/cloud/apps/acceptance/src/probe-build.ts
@@ -0,0 +1,49 @@
+// Debug: minimal environment build, streams events for a few minutes, then deletes the environment.
+// tsx src/probe-build.ts [script]
+import { sandboxToken } from './tokens'
+import { control } from './clients'
+import { config } from './env'
+
+const owner = { sub: 'w9-owner', org: 'w9-eval-team', orgRole: 'owner' as const }
+const env = await control.as(owner, 'POST', '/v1/environments', { name: `w9-probe-${Date.now() % 100000}`, repos: [{ url: config.testRepo }], installScript: process.argv[2] ?? 'echo hello-build' })
+const envId = env.body?.environment?.id
+console.log('env', env.status, envId)
+try {
+ const b = await control.as(owner, 'POST', `/v1/environments/${envId}/builds`)
+ const id = b.body?.build?.id
+ console.log('build', b.status, id, b.body?.build?.status)
+ if (process.argv[3] === 'poll') {
+ const t0 = Date.now()
+ let s: any = null
+ while (Date.now() - t0 < 240_000) {
+ s = (await control.as(owner, 'GET', `/v1/environments/${envId}/builds/${id}`)).body?.build
+ console.log(Math.round((Date.now() - t0) / 1000), s?.status, s?.logBytes)
+ if (['succeeded', 'failed', 'canceled'].includes(s?.status)) break
+ await new Promise((r) => setTimeout(r, 10_000))
+ }
+ process.exitCode = 0
+ }
+ const ctl = new AbortController()
+ setTimeout(() => ctl.abort(), 240_000)
+ const r = await fetch(`${config.controlPlaneUrl}/v1/environments/${envId}/builds/${id}/events`, { headers: { authorization: `Bearer ${await sandboxToken(owner)}` }, signal: ctl.signal }).catch((e) => e)
+ if (r instanceof Response && r.body) {
+ const t0 = Date.now()
+ const dec = new TextDecoder()
+ try {
+ for await (const chunk of r.body as any) {
+ for (const line of dec.decode(chunk).split('\n').filter(Boolean)) {
+ const e = JSON.parse(line)
+ console.log(Math.round((Date.now() - t0) / 1000), e.type, e.type === 'log' ? String(e.text ?? e.line ?? JSON.stringify(e)).slice(0, 160) : JSON.stringify(e.build ?? e).slice(0, 200))
+ }
+ }
+ } catch (e) {
+ console.log('stream ended', String(e).slice(0, 80))
+ }
+ } else {
+ console.log('events', r instanceof Response ? r.status : String(r))
+ }
+ const final = await control.as(owner, 'GET', `/v1/environments/${envId}/builds/${id}`)
+ console.log('final', JSON.stringify(final.body?.build).slice(0, 600))
+} finally {
+ console.log('delete', (await control.as(owner, 'DELETE', `/v1/environments/${envId}`)).status)
+}
diff --git a/cloud/apps/acceptance/src/report.ts b/cloud/apps/acceptance/src/report.ts
index 09c391215f5..d1b84778355 100644
--- a/cloud/apps/acceptance/src/report.ts
+++ b/cloud/apps/acceptance/src/report.ts
@@ -162,6 +162,7 @@ Parity bar: conductor-parity-matrix.html<
create.usable adds the user's client connecting (orca CLI, E2EE handshake); *.firstEcho adds registering the project, opening a PTY and seeing a typed command echoed
(each CLI call reconnects, so this is an upper bound on what the desktop's warm connection sees); rollover.gap = drain + stop + boot + orcad ready.
${benchTable(runs)}
+${runs.filter((r) => r.benchmarks.length).slice(-1).flatMap((r) => r.notes.filter((n) => n.startsWith('load'))).map(esc).join(' · ')}
Known gaps and findings
${[...(extra.findings ?? []), ...(extra.gaps ?? [])].map((g) => `- ${g}
`).join('') || '- none recorded
'}
Cost of the eval and every cloud resource
diff --git a/cloud/apps/acceptance/src/run.ts b/cloud/apps/acceptance/src/run.ts
index f5d8ee785b2..f0fbde0a72f 100644
--- a/cloud/apps/acceptance/src/run.ts
+++ b/cloud/apps/acceptance/src/run.ts
@@ -12,6 +12,7 @@ import { opsChecks } from './checks/ops'
import { pendingChecks } from './checks/pending'
import { secretChecks } from './checks/secrets'
import { Ctx, runCheck, type Check } from './context'
+import { control } from './clients'
import { config } from './env'
import { renderReport } from './report'
import { RunState } from './run-state'
@@ -48,7 +49,21 @@ process.on('SIGINT', () => {
console.log('\ninterrupted: cleaning up (Ctrl-C again to abort cleanup)')
})
+/** Load on the shared dev stack while we measure (other workstreams run VMs too). */
+async function noteLoad(when: string): Promise {
+ const s = await control.operator('GET', '/v1/internal/status').catch(() => null)
+ run.note(`load ${when}: ${s?.body?.active ?? '?'} active / ${s?.body?.running ?? '?'} running sandboxes on the dev control plane (all accounts)`)
+}
+
try {
+ await noteLoad('at start')
+ // Idempotent: the eval accounts need the dev GitHub App installation to clone the private test repo.
+ for (const accountId of ['dev:w9-eval', 'dev:org:w9-eval-team', 'dev:w9-desktop']) {
+ const b = await control.operator('POST', '/v1/internal/github/bindings', { repoUrl: config.testRepo, accountId })
+ if (b.status !== 200) {
+ run.note(`github binding for ${accountId} -> ${b.status}`)
+ }
+ }
if (!flag('no-checks')) {
const ids = CATALOG.map((f) => f.id).filter((id) => (!only || only.includes(id)) && !skip.has(id) && !LATE.includes(id))
const late = LATE.filter((id) => (!only || only.includes(id)) && !skip.has(id))
@@ -63,8 +78,10 @@ try {
}
}
if (flag('bench') && !interrupted) {
+ await noteLoad('before benchmarks')
await benchCreate(ctx, n)
await benchWakeAndRollover(ctx, n)
+ await noteLoad('after benchmarks')
}
} finally {
const { failed } = await ctx.sandboxes.destroyAll()
diff --git a/cloud/apps/acceptance/src/tokens.ts b/cloud/apps/acceptance/src/tokens.ts
index bfcac2b4aba..25d0ca20351 100644
--- a/cloud/apps/acceptance/src/tokens.ts
+++ b/cloud/apps/acceptance/src/tokens.ts
@@ -14,10 +14,14 @@ function signingKey() {
key ??= importJWK(jwk.private, 'ES256')
return key
}
+function kid(): string {
+ const jwk = secrets.devJwk() as { kid?: string; private: { kid?: string } } | null
+ return jwk?.kid ?? jwk?.private.kid ?? 'dev1'
+}
async function sign(claims: Record, issuer: string, audience: string): Promise {
return new SignJWT(claims)
- .setProtectedHeader({ alg: 'ES256', kid: 'dev1' })
+ .setProtectedHeader({ alg: 'ES256', kid: kid() })
.setIssuer(issuer)
.setAudience(audience)
.setIssuedAt()