From 322ba5eabbebd3c192c006539c1cafb282cf5eb6 Mon Sep 17 00:00:00 2001 From: Neil Date: Tue, 10 Nov 2026 03:10:02 -0800 Subject: [PATCH 1/3] feat(cloud): acceptance catalog rows for run actions and per-workspace ports (gap 4) --- cloud/apps/acceptance/src/catalog.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cloud/apps/acceptance/src/catalog.ts b/cloud/apps/acceptance/src/catalog.ts index cad54adbc80..4c004a43373 100644 --- a/cloud/apps/acceptance/src/catalog.ts +++ b/cloud/apps/acceptance/src/catalog.ts @@ -40,7 +40,11 @@ export const CATALOG: Feature[] = [ { id: 'billing.entitlement-gate', section: S.acct, feature: 'Account without a plan is refused a sandbox (fail closed)', conductor: 'Pro/org required', owner: 'core', priority: 'P0' }, // Conductor's local core workflow (added after the independent audit; the cloud matrix left it out) { id: 'local.core-workflow', section: S.acct, feature: 'Local core workflow: create a local worktree, run an agent, see the diff, commit (desktop)', conductor: 'Core local app (free)', owner: 'w9-eval (m4air desktop run)', priority: 'P0' }, - { id: 'local.run-actions', section: S.acct, feature: 'Run actions: named run scripts (several, run_mode, auto-run after setup) with a per-workspace port block; Conductor import keeps them', conductor: '0.70 multiple run scripts, CONDUCTOR_PORT 10-port block', owner: 'gap-run-actions', priority: 'P0' }, + { id: 'local.run-actions', section: S.acct, feature: 'Run actions: orca.yaml scripts.run on the Run button, "Run: " terminal, trust prompt once, Stop (Ctrl+C → Stopped), Restart, ⌥⌘↩, autoRunAfterSetup only after a successful setup', conductor: '0.70 multiple run scripts, ⌘R, auto-run after setup', owner: 'gap-run-actions', priority: 'P0' }, + { id: 'run.ports', section: S.acct, feature: 'Per-workspace port block: ORCA_PORT / ORCA_PORT_COUNT=10 / CONDUCTOR_PORT, distinct per workspace, stable across restarts; Vite via Run listens on $ORCA_PORT locally and in cloud (auto-forward)', conductor: 'CONDUCTOR_PORT 10-port block', owner: 'gap-run-actions', priority: 'P0' }, + { id: 'run.nonconcurrent', section: S.acct, feature: 'runMode nonconcurrent stops the same action in another workspace first', conductor: 'run_mode', owner: 'gap-run-actions', priority: 'P1' }, + { id: 'run.cloud-resume', section: S.life, feature: 'A running action comes back after wake and after rollover in a cloud workspace', conductor: 'Restart run scripts after wake (manual)', owner: 'gap-run-actions', priority: 'P0', better: 'Comes back by itself' }, + { id: 'run.conductor-import', section: S.acct, feature: 'Conductor importer keeps run / run_mode / auto_run_after_setup (settings.toml, settings.local.toml, conductor.json)', conductor: 'n/a (migration)', owner: 'gap-run-actions', priority: 'P0' }, { id: 'life.read-asleep', section: S.life, feature: 'Chats, PR comments and check logs readable while asleep from any device (no wake)', conductor: '0.80/0.85, server-held', owner: 'gap-read-asleep', priority: 'P0' }, { id: 'local.checkpoints', section: S.acct, feature: 'Per-turn code checkpoints: revert restores files and chat together', conductor: '0.19 (Claude), 0.44 (Codex) checkpoints', owner: 'gap-checkpoints', priority: 'P0' }, // 1. Substrate From 29cdd2cb99ed64bc5707ab4290a73d382c35fd14 Mon Sep 17 00:00:00 2001 From: Neil Date: Tue, 10 Nov 2026 03:36:16 -0800 Subject: [PATCH 2/3] feat(cloud): acceptance read-while-asleep (live, E2EE, no wake) plus access/phone rows (owner-reported) --- cloud/apps/acceptance/findings.json | 5 +- cloud/apps/acceptance/src/catalog.ts | 2 + cloud/apps/acceptance/src/checks/billing.ts | 2 +- cloud/apps/acceptance/src/checks/platform.ts | 84 +++++++++++++++++++- cloud/apps/acceptance/src/import-desktop.ts | 6 +- 5 files changed, 95 insertions(+), 4 deletions(-) diff --git a/cloud/apps/acceptance/findings.json b/cloud/apps/acceptance/findings.json index 7d7520c0200..a883892c3a1 100644 --- a/cloud/apps/acceptance/findings.json +++ b/cloud/apps/acceptance/findings.json @@ -1,6 +1,7 @@ { "findings": [ "Wake delivery (target p95 < 6 s) — MET in pdx1: after core U17/U18 (default region pdx1, Standard 8 vCPU/16 GB, golden 63b6999), n=20, persistent connection. Wake to echo p50 5.04 s, p95 5.85 s, p99 5.94 s; 0/20 slow hosts. Pinned to iad1 at 8/16: p50 5.31, p95 10.59, p99 11.80 s, with 4/20 slow hosts. BEFORE (iad1 4/8): 5.28 / 8.99 / 9.40 s. Region, not size, removes the tail. From this Mac the first terminal list (0.18 vs 0.44 s) and echo (0.07 vs 0.15 s) are also faster in pdx1. Create p50: pdx1 2.03 s, iad1-pinned 4.26 s (pinned creates seem to skip the warm pool).", + "Read while asleep (gap 5, #189/#218), verified by W9 live: a teammate decrypted a sleeping team workspace (git incl. a new file, PR, sessions) with no wake: the Vercel session stayed stopped. 20 reads p50 89 ms, max 131 ms, vs about 5-6 s to wake. An outsider gets 404, and the plaintext marker is not in D1. Removal/private access control and the phone view are owner-reported (capped).", "Independent audit corrections (2026-11-10, reports/conductor-parity-audit.html): five rows are capped at partial because they passed on a weaker bar than the Conductor column: real-model agent run, subscription sign-in, default machine size, and the download and remote-desktop UIs. Four rows Conductor has that our bar left out are added as not landed: local core workflow, Run actions with per-workspace ports, read-while-asleep from any device, and per-turn code checkpoints. Our 'pass' counts measure the team's own bar. Every cloud result is on dev Workers, Stripe TEST mode and dev/staging tokens; nothing is shipped (mvp-tracker §5 launch items are all open).", "Agents and machine size (2026-11-10): image.agents: Claude Code, Codex, Cursor, OpenCode, Gemini and Copilot all answer --version on a fresh VM. agent.keys-swap: stand-in Gemini/Cursor keys exist on the VM only as placeholders; the provider sees the swapped key (Gemini 400 API_KEY_INVALID, Cursor 401); a foreign key or a disallowed path is refused (403); the stand-in values appear in 0 files and 0 process environments. sub.size cap lifted: Standard is now 8 vCPU / 16 GB (core U18, #194), and machineSize large maps to standard.", "Audit caps lifted by real-UI runs (m4air, 855d8ed34b6, desktop-20261109-234651): acc.download, file-tree Download: 20 MiB file in 2.1 s and a folder as zip in 0.6 s, with real progress and success toasts and matching sha256. Caveat: only the macOS Save panel answer was stubbed in main, because CDP cannot drive a native panel. acc.remote-desktop: Open remote desktop tab connected in 3.4 s; noVNC canvas streams non-blank pixels, and a red xterm launched in the VM appeared on the canvas. local.core-workflow passes (local worktree in 0.6 s, terminal edit, diff in Source Control, stage, commit).", @@ -26,6 +27,8 @@ "auditCaps": { "agent.real-task": "Proven only with moonshotai/kimi-k2 through the Claude Code CLI. Per the user decision (2026-11-10) there are no AI Gateway real-model runs; lifted by the user's own e2e session with their own credentials, and on the coordinator's e2e checklist. Image now also has Cursor, OpenCode, Gemini and Copilot (image.agents); W9 has shown Cursor and Gemini launch with their keys swapped at egress (agent.keys-swap), but no real-model run for those two.", "agent.signin": "Auth folders persist across sleep/wake and rollover, but subscription OAuth on a VM was never exercised. This cap is lifted by the user's own e2e session with their own credentials (after their review of #40228). It is on the coordinator's e2e checklist.", - "agent.more-agents": "Owner-reported (gap-agents) real-task runs on kimi-k2 via the gateway. W9 is not re-running them (no gateway spend per user decision). Lifted by the user e2e session with their own credentials." + "agent.more-agents": "Owner-reported (gap-agents) real-task runs on kimi-k2 via the gateway. W9 is not re-running them (no gateway spend per user decision). Lifted by the user e2e session with their own credentials.", + "life.read-asleep-access": "Owner-reported (gap-read-asleep). W9 has not re-run the member-removal push or the private prune, which need the auth membership push. W9 did verify independently that an outsider gets 404 (life.read-asleep).", + "life.read-asleep-phone": "Owner-reported, headless phone simulator. W9 has not re-run the mobile UI." } } \ No newline at end of file diff --git a/cloud/apps/acceptance/src/catalog.ts b/cloud/apps/acceptance/src/catalog.ts index 4c004a43373..2ac4c349064 100644 --- a/cloud/apps/acceptance/src/catalog.ts +++ b/cloud/apps/acceptance/src/catalog.ts @@ -46,6 +46,8 @@ export const CATALOG: Feature[] = [ { id: 'run.cloud-resume', section: S.life, feature: 'A running action comes back after wake and after rollover in a cloud workspace', conductor: 'Restart run scripts after wake (manual)', owner: 'gap-run-actions', priority: 'P0', better: 'Comes back by itself' }, { id: 'run.conductor-import', section: S.acct, feature: 'Conductor importer keeps run / run_mode / auto_run_after_setup (settings.toml, settings.local.toml, conductor.json)', conductor: 'n/a (migration)', owner: 'gap-run-actions', priority: 'P0' }, { id: 'life.read-asleep', section: S.life, feature: 'Chats, PR comments and check logs readable while asleep from any device (no wake)', conductor: '0.80/0.85, server-held', owner: 'gap-read-asleep', priority: 'P0' }, + { id: 'life.read-asleep-access', section: S.life, feature: 'Read-while-asleep access control: removed member loses access at once (keys deleted), private workspace is owner-only', conductor: 'Org access to server-held chats', owner: 'gap-read-asleep', priority: 'P0', better: 'End-to-end encrypted; server holds only ciphertext' }, + { id: 'life.read-asleep-phone', section: S.life, feature: 'Phone shows "Latest from this workspace" for a sleeping workspace without waking it', conductor: 'iOS app reads server-held chats', owner: 'gap-read-asleep', priority: 'P1' }, { id: 'local.checkpoints', section: S.acct, feature: 'Per-turn code checkpoints: revert restores files and chat together', conductor: '0.19 (Claude), 0.44 (Codex) checkpoints', owner: 'gap-checkpoints', priority: 'P0' }, // 1. Substrate { id: 'sub.create', section: S.sub, feature: 'Hosted microVM per workspace, one call / one click', conductor: '0.78 GA', owner: 'core', priority: 'P0' }, diff --git a/cloud/apps/acceptance/src/checks/billing.ts b/cloud/apps/acceptance/src/checks/billing.ts index 43e7be21b75..b245d691073 100644 --- a/cloud/apps/acceptance/src/checks/billing.ts +++ b/cloud/apps/acceptance/src/checks/billing.ts @@ -36,7 +36,7 @@ function d1(dir: string, binding: string, sql: string): any[] { return (JSON.parse(out.slice(out.indexOf('['))) as Array<{ results: any[] }>)[0]?.results ?? [] } const d1Billing = (sql: string) => d1(BILLING_DIR, 'BILLING_DB', sql) -const d1Control = (sql: string) => d1(new URL('../../../sandbox-control/', import.meta.url).pathname, 'SANDBOX_DB', sql) +export const d1Control = (sql: string) => d1(new URL('../../../sandbox-control/', import.meta.url).pathname, 'SANDBOX_DB', sql) async function advanceClock(clock: string, to: number): Promise { await stripe('POST', `/v1/test_helpers/test_clocks/${clock}/advance`, { frozen_time: Math.floor(to) }) diff --git a/cloud/apps/acceptance/src/checks/platform.ts b/cloud/apps/acceptance/src/checks/platform.ts index 644f87bd324..95e4ed59fce 100644 --- a/cloud/apps/acceptance/src/checks/platform.ts +++ b/cloud/apps/acceptance/src/checks/platform.ts @@ -5,7 +5,8 @@ import { existsSync, mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { SignJWT } from 'jose' -import { control } from '../clients' +import { control, vercelRest } from '../clients' +import { d1Control } from './billing' import { config, secrets } from '../env' import { log, type Check, type Ctx } from '../context' import { orca } from '../orca-cli' @@ -119,6 +120,24 @@ function openGrant(sealed: any, grantId: string, pair: ReturnType): any { + const aesOpen = (key: Buffer, nonceB64: string, ctB64: string, aad: string) => { + const ct = Buffer.from(ctB64, 'base64') + const d = createDecipheriv('aes-256-gcm', key, Buffer.from(nonceB64, 'base64')) + d.setAAD(Buffer.from(aad)) + d.setAuthTag(ct.subarray(ct.length - 16)) + return Buffer.concat([d.update(ct.subarray(0, ct.length - 16)), d.final()]) + } + const r = s.recipient + const eph = Buffer.from(r.ephemeralPublicKeyB64, 'base64') + const ephKey = createPublicKey({ key: { kty: 'OKP', crv: 'X25519', x: eph.toString('base64url') }, format: 'jwk' }) + const shared = diffieHellman({ privateKey: pair.privateKey, publicKey: ephKey }) + const kek = Buffer.from(hkdfSync('sha256', shared, Buffer.concat([eph, pair.publicRaw]), Buffer.from('orca-cloud-read-wrap-v1'), 32)) + const dek = aesOpen(kek, r.nonceB64, r.wrappedKeyB64, `orca-cloud-read-wrap-v1:${s.workspaceId}:${s.snapshotId}:${r.keyId}`) + return JSON.parse(aesOpen(dek, s.nonceB64, s.ciphertextB64, `orca-cloud-read-v1:${s.workspaceId}:${s.snapshotId}`).toString('utf8')) +} + let teamWs: Promise<{ shared: string; priv: string }> | null = null function teamWorkspaces(ctx: Ctx) { teamWs ??= (async () => { @@ -340,6 +359,69 @@ export const platformChecks: Record = { return { status: ok ? 'pass' : 'fail', detail: `cost in list: compute ${w?.cost?.computeMs} ms, $${((w?.cost?.totalUsdMicros ?? 0) / 1e6).toFixed(4)}` } }, + + 'life.read-asleep': async (ctx) => { + // Gap 5: a teammate's device reads a SLEEPING team workspace (git state, agents, chats) without + // waking it; end-to-end encrypted to the device's X25519 reader key. Also checks that an outsider + // is refused and that the control-plane DB holds no plaintext (a marker file name never appears). + const tag = ctx.run.file.runId.replace(/[^0-9]/g, '').slice(-8) + const marker = `w9readmarker${tag}` + const pair = x25519Pair() + const reg = await control.as(ctx.member, 'PUT', '/v1/reader-keys', { publicKeyB64: pair.publicRaw.toString('base64'), deviceLabel: 'w9 reader', deviceKind: 'desktop' }) + const keyId = reg.body?.key?.id ?? reg.body?.readerKey?.id ?? reg.body?.id + if (!keyId) { + return { status: 'fail', detail: `reader key ${reg.status} ${JSON.stringify(reg.body).slice(0, 200)}` } + } + const box = await ctx.sandboxes.create(ctx.owner, { workspaceId: `w9-read-${tag}`, title: 'W9 read asleep', visibility: 'team', repo: { url: config.testRepo }, useEnvironment: false }) + try { + if (box.status !== 200) { + return { status: 'fail', detail: `create ${box.status} ${JSON.stringify(box.body).slice(0, 200)}` } + } + const id = box.body.sandbox.id + await ctx.exec(box.body.sandbox.session!.id, `cd ${box.body.connection.projectRoot} && echo hi > ${marker}.txt`) + const sus = await ctx.sandboxes.suspend(id) + const provider = async () => (await vercelRest('GET', `/v2/sandboxes/orca-dev-${id}`, { projectId: config.vercelProjectId })).json + const before = await provider() + const samples: number[] = [] + let payload: any = null + let err = '' + for (let i = 0; i < 20; i++) { + const t0 = performance.now() + const r = await control.as(ctx.member, 'GET', `/v1/team/workspaces/${id}/read-snapshot?keyId=${keyId}`) + const s = r.body?.snapshot + if (!s) { + err = `${r.status} ${JSON.stringify(r.body).slice(0, 160)}` + break + } + payload = openReadSnapshot(s, pair) + samples.push(Math.round(performance.now() - t0)) + } + const after = await provider() + const stillStopped = !['running', 'pending'].includes(after?.session?.status ?? after?.sandbox?.status ?? '') + const outsider = await control.as(ctx.outsider, 'GET', `/v1/team/workspaces/${id}/read-snapshot?keyId=${keyId}`) + const text = JSON.stringify(payload ?? '') + const sawMarker = text.includes(marker) + const dbRows = d1Control(`SELECT COUNT(*) AS n FROM read_snapshots WHERE sandbox_id = '${id}' AND (ciphertext LIKE '%${marker}%')`) + const plaintextInDb = Number(dbRows[0]?.n ?? 0) > 0 + if (samples.length) { + ctx.run.bench('read.asleep', samples, 'teammate fetch + decrypt of a sleeping workspace snapshot, from this Mac') + } + const sorted = [...samples].sort((a, b) => a - b) + const ok = sus.status === 200 && samples.length === 20 && payload?.workspaceId === id && sawMarker && stillStopped && outsider.status >= 400 && !plaintextInDb + return { + status: ok ? 'pass' : 'fail', + detail: err + ? `read-snapshot: ${err}` + : `teammate decrypted the sleeping workspace's snapshot (sections: ${Object.keys(payload ?? {}).join(', ')}; git shows the new file: ${sawMarker}); 20 reads p50 ${sorted[9]} ms, max ${sorted.at(-1)} ms; Vercel session ${before?.session?.status ?? before?.sandbox?.status} -> ${after?.session?.status ?? after?.sandbox?.status} (no wake: ${stillStopped}); outsider -> ${outsider.status}; plaintext marker in D1: ${plaintextInDb}`, + ...(sorted.length ? { timingsMs: { readP50: sorted[9]!, readMax: sorted.at(-1)! } } : {}) + } + } finally { + await control.as(ctx.member, 'DELETE', `/v1/reader-keys/${keyId}`) + if (box.status === 200) { + await ctx.sandboxes.destroy(box.body.sandbox.id) + } + } + }, // ---------------- W5 ---------------- 'api.rest': async (ctx) => { const created = await api(ctx, 'POST', '/v1/workspaces', { name: `w9-api-${tag(ctx)}`, repo: { url: config.testRepo } }) diff --git a/cloud/apps/acceptance/src/import-desktop.ts b/cloud/apps/acceptance/src/import-desktop.ts index 637987035b8..8d03f348017 100644 --- a/cloud/apps/acceptance/src/import-desktop.ts +++ b/cloud/apps/acceptance/src/import-desktop.ts @@ -27,7 +27,11 @@ const MAP: Record = { 'life.auto-resume': 'life.auto-resume', 'acc.download': 'acc.download', 'acc.remote-desktop': 'acc.remote-desktop', - 'local.core-workflow': 'local.core-workflow' + 'local.core-workflow': 'local.core-workflow', + 'local.run-actions': 'local.run-actions', + 'run.ports': 'run.ports', + 'run.nonconcurrent': 'run.nonconcurrent', + 'run.cloud-resume': 'run.cloud-resume' } const checks: CheckResult[] = [] const benchmarks: RunFile['benchmarks'] = [] From b199587d7e3f46d49364499739d954c54ba9d781 Mon Sep 17 00:00:00 2001 From: Neil Date: Tue, 10 Nov 2026 03:42:02 -0800 Subject: [PATCH 3/3] feat(cloud): desktop run actions + ports checks (W9D_ONLY=run-actions) - local.run-actions: autoRunAfterSetup after a good setup and not after a failed one, Run button, one trust prompt, Stop -> Stopped, menu Restart replaces the tab, Cmd+Opt+Enter start/restart - run.nonconcurrent: echo-loop started in B stops A's - run.ports: ORCA_PORT/ORCA_PORT_COUNT/CONDUCTOR_PORT per terminal, distinct blocks, stable across an app restart, served locally; cloud half: the action in the VM reachable here via auto-forward - run.cloud-resume: action back after Sleep/wake and after Restart sandbox --- cloud/apps/acceptance/desktop/run-desktop.mjs | 393 ++++++++++++++++++ 1 file changed, 393 insertions(+) diff --git a/cloud/apps/acceptance/desktop/run-desktop.mjs b/cloud/apps/acceptance/desktop/run-desktop.mjs index 20b215c2aa6..dd0b5d787a3 100644 --- a/cloud/apps/acceptance/desktop/run-desktop.mjs +++ b/cloud/apps/acceptance/desktop/run-desktop.mjs @@ -1029,6 +1029,12 @@ async function main() { results.notes.push('test repo not clonable here; used a local repo with the same origin URL') } created.tmp = tmp + if (process.env.W9D_ONLY === 'run-actions') { + writeFileSync(path.join(repoDir, 'orca.yaml'), runActionsYaml(tmp)) + execFileSync('git', ['-C', repoDir, 'add', 'orca.yaml'], { stdio: 'ignore' }) + execFileSync('git', ['-C', repoDir, '-c', 'user.email=w9@orca.dev', '-c', 'user.name=W9', 'commit', '-qm', 'orca.yaml run actions'], { stdio: 'ignore' }) + results.app.repoDir = repoDir + } const setup = await runStep('setup', 'Launch dev Orca on a fresh profile (signed out), add repo', async (step) => { // #127 keys the sandbox token on the signed-in identity, so no sign-in-then-relaunch dance: the @@ -1914,12 +1920,399 @@ async function asleepMovedStep() { }) } +// ---------- run actions + per-workspace ports (#187/#216) ---------- + +/** Setup fails while /fail-setup exists; web serves on $ORCA_PORT; echo-loop is nonconcurrent. */ +function runActionsYaml(tmp) { + const marker = path.join(tmp, 'fail-setup') + return [ + 'scripts:', + ` setup: "test ! -e '${marker}' || exit 3; echo W9-SETUP-OK"`, + ' autoRunAfterSetup: web', + ' run:', + ' web:', + ' command: "python3 -m http.server $ORCA_PORT"', + ' default: true', + ' echo-loop:', + ' command: "while true; do echo loop-$ORCA_PORT; sleep 1; done"', + ' runMode: nonconcurrent', + '' + ].join('\n') +} + +const runRecords = () => page.evaluate(() => { try { return JSON.parse(localStorage.getItem('orca.runActions.v1') ?? '{}') } catch { return {} } }) +const runOf = async (worktreeId, actionId) => (await runRecords())?.[worktreeId]?.[actionId] ?? null + +/** Approve the orca.yaml trust prompt when it is up; counts prompts per kind. */ +async function approveTrust(step) { + const m = await store(() => { + const s = window.__store.getState() + return s.activeModal === 'confirm-orca-yaml-hooks' ? { kind: s.modalData?.scriptKind ?? null } : null + }).catch(() => null) + if (!m) return false + step.trustPrompts = [...(step.trustPrompts ?? []), { kind: m.kind, at: new Date().toISOString() }] + results.trustPrompts = [...(results.trustPrompts ?? []), { step: step.id, kind: m.kind }] + await shot(step, `trust-${m.kind}`) + await page.getByRole('button', { name: /^Run hooks$/ }).first().click({ timeout: 5000 }) + await sleep(500) + return true +} +async function waitFor(step, fn, timeoutMs) { + const t0 = Date.now() + while (Date.now() - t0 < timeoutMs) { + await approveTrust(step) + const v = await fn() + if (v) return { v, ms: Date.now() - t0 } + await sleep(200) + } + return { v: null, ms: Date.now() - t0, timedOut: true } +} +const httpOk = (port) => fetch(`http://127.0.0.1:${port}/`, { signal: AbortSignal.timeout(2000) }).then((r) => r.status, () => null) + +async function worktreeByName(name) { + return store((n) => Object.values(window.__store.getState().worktreesByRepo ?? {}).flat().find((w) => w.displayName === n) ?? null, name) +} +/** Create a local worktree from the composer (Blank Terminal); approve setup trust when asked. */ +async function createLocalWorktree(step, name) { + const d = await openComposer(step, `${name}-composer`) + await d.getByRole('combobox', { name: 'Run on' }).click() + const targets = page.getByRole('listbox', { name: 'Run targets' }) + await targets.waitFor() + await targets.getByRole('option', { name: /^Local Mac/ }).first().click() + await sleep(400) + if (!(await selectBlankAgent(d))) throw new Error('could not choose Blank Terminal') + await fillName(d, name) + const t0 = Date.now() + await d.getByRole('button', { name: /^Create worktree/ }).click() + const w = await waitFor(step, () => worktreeByName(name), 60_000) + if (!w.v) throw new Error(`no worktree ${name}`) + created.localPaths.push(w.v.path) + return { name, id: w.v.id, path: w.v.path, createdAt: t0 } +} +/** Open the workspace and read its env from a fresh plain terminal. */ +async function terminalEnv(step, wsRow, kind = 'local') { + const row = await rowOf(wsRow, kind) + await (row ?? page.getByText(wsRow, { exact: true }).first()).click() + await sleep(800) + await openNewTerminalTab() + await waitShellPrompt(60_000) + await focusTerminal() + await page.keyboard.type('echo W9ENV P=$ORCA_PORT C=$ORCA_PORT_COUNT K=$CONDUCTOR_PORT\n') + const r = await waitFor(step, async () => { + const m = (await visibleTerminals()).join('\n').match(/W9ENV P=(\d*) C=(\d*) K=(\d*)\s*$/m) + return m ? { ORCA_PORT: m[1], ORCA_PORT_COUNT: m[2], CONDUCTOR_PORT: m[3] } : null + }, 20_000) + return r.v +} +/** The Run button scoped to the visible tab strip; its chevron menu. */ +const runButton = () => page.getByTestId('run-action-button').filter({ visible: true }).first() +async function runMenu(step) { + await page.keyboard.press('Escape').catch(() => {}) + await runButton().getByRole('button', { name: 'Run actions' }).click() + await page.getByRole('menuitem').first().waitFor({ timeout: 5000 }) + await sleep(200) + step.lastRunMenu = (await page.getByRole('menu').filter({ visible: true }).last().innerText().catch(() => '')).replace(/\s+/g, ' ').slice(0, 400) +} +/** Click the menu item `item` listed under the action `actionName` (menu order: header, its items). */ +async function runMenuItem(step, actionName, item) { + await runMenu(step) + const found = await page.evaluate(({ actionName, item }) => { + const menu = [...document.querySelectorAll('[role="menu"]')].filter((m) => m.getBoundingClientRect().width > 0).pop() + if (!menu) return 'no menu' + let under = false + for (const el of menu.querySelectorAll('*')) { + const isItem = el.getAttribute('role') === 'menuitem' + const text = (el.textContent ?? '').replace(/\s+/g, ' ').trim() + if (!isItem && el.children.length <= 3 && text.startsWith(actionName) && !text.startsWith(`${actionName} `.repeat(2))) under = true + else if (!isItem && under && el.children.length <= 3 && /^(Web|Echo loop)\b/.test(text) && !text.startsWith(actionName)) under = false + if (under && isItem && text === item) { + el.setAttribute('data-w9d-run-item', '1') + return 'ok' + } + } + return 'not found' + }, { actionName, item }) + if (found !== 'ok') throw new Error(`menu item "${item}" under ${actionName}: ${found}; menu ${step.lastRunMenu}`) + await page.locator('[data-w9d-run-item="1"]').first().click() +} + +async function runActionsFlow() { + const A = `${wsName}-a` + const B = `${wsName}-b` + const C = `${wsName}-c` + const wsA = {} + const wsB = {} + const portsBefore = {} + await runStep('local.run-actions', 'Run actions: autoRunAfterSetup, Run button, trust once, Stop/Restart, ⌥⌘↩; no auto-run after a failed setup', async (step) => { + const sub = {} + step.substeps = sub + // A: setup succeeds -> web auto-runs. + Object.assign(wsA, await createLocalWorktree(step, A)) + const auto = await waitFor(step, async () => { + const r = await runOf(wsA.id, 'web') + return r?.status === 'running' ? r : null + }, 90_000) + sub.autoRunAfterSetup = Boolean(auto.v) + step.timings.createToAutoRunMs = auto.v ? Date.now() - wsA.createdAt : null + await shot(step, 'a-autorun') + const env = await terminalEnv(step, A) + wsA.port = Number(env?.ORCA_PORT) + const serve = await waitFor(step, async () => (await httpOk(wsA.port)) === 200, 30_000) + sub.serverOnOrcaPort = Boolean(serve.v) + step.webPortA = wsA.port + const tabs = await store((id) => (window.__store.getState().tabsByWorktree?.[id] ?? []).map((t) => t.customTitle ?? t.title), wsA.id) + step.tabsA = tabs + sub.runTab = tabs.some((t) => /^Run: Web$/.test(t ?? '')) + sub.buttonShowsAction = /Web/.test(await runButton().innerText().catch(() => '')) + await shot(step, 'a-run-button') + // Stop. + const before = await runOf(wsA.id, 'web') + const t0 = Date.now() + await runMenuItem(step, 'Web', 'Stop') + const stopped = await waitFor(step, async () => (await runOf(wsA.id, 'web'))?.status === 'stopped', 15_000) + step.timings.stopToStoppedMs = stopped.v ? Date.now() - t0 : null + await runMenu(step) + sub.stoppedLabel = /Stopped/.test(step.lastRunMenu) + await shot(step, 'a-stopped-menu') + await page.keyboard.press('Escape') + const freed = await waitFor(step, async () => (await httpOk(wsA.port)) === null, 10_000) + sub.stopFreesPort = Boolean(freed.v) + sub.stop = Boolean(stopped.v) + // ⌥⌘↩ starts it again. + await focusTerminal() + const t1 = Date.now() + await page.keyboard.press('Meta+Alt+Enter') + const started = await waitFor(step, async () => (await runOf(wsA.id, 'web'))?.status === 'running' && (await httpOk(wsA.port)) === 200, 30_000) + step.timings.shortcutStartToServingMs = started.v ? Date.now() - t1 : null + sub.shortcutStarts = Boolean(started.v) + // ⌥⌘↩ while running restarts (new tab). + const tabBefore = (await runOf(wsA.id, 'web'))?.tabId + const t2 = Date.now() + await page.keyboard.press('Meta+Alt+Enter') + const restarted = await waitFor(step, async () => { + const r = await runOf(wsA.id, 'web') + return r?.status === 'running' && r.tabId !== tabBefore && (await httpOk(wsA.port)) === 200 + }, 30_000) + step.timings.shortcutRestartToServingMs = restarted.v ? Date.now() - t2 : null + sub.shortcutRestarts = Boolean(restarted.v) + // Menu Restart replaces the tab. + const tabBefore2 = (await runOf(wsA.id, 'web'))?.tabId + const t3 = Date.now() + await runMenuItem(step, 'Web', 'Restart') + const rr = await waitFor(step, async () => { + const r = await runOf(wsA.id, 'web') + return r?.status === 'running' && r.tabId !== tabBefore2 && (await httpOk(wsA.port)) === 200 + }, 30_000) + step.timings.menuRestartToServingMs = rr.v ? Date.now() - t3 : null + const runTabs = (await store((id) => (window.__store.getState().tabsByWorktree?.[id] ?? []).map((t) => t.customTitle ?? t.title), wsA.id)).filter((t) => /^Run: Web$/.test(t ?? '')) + sub.restartReplacesTab = Boolean(rr.v) && runTabs.length === 1 + step.runWebTabsAfterRestart = runTabs.length + await shot(step, 'a-restarted') + // C: setup fails -> no auto-run. + writeFileSync(path.join(created.tmp, 'fail-setup'), 'x') + const wsC = await createLocalWorktree(step, C) + const setupTab = await waitFor(step, async () => (await store((id) => (window.__store.getState().tabsByWorktree?.[id] ?? []).some((t) => /Setup/.test(t.customTitle ?? t.title ?? '')), wsC.id)) || null, 30_000) + await sleep(15_000) + await approveTrust(step) + const cRun = await runOf(wsC.id, 'web') + sub.noAutoRunAfterFailedSetup = Boolean(setupTab.v) && !cRun + step.failedSetup = { setupTabSeen: Boolean(setupTab.v), runRecord: cRun } + await shot(step, 'c-setup-failed') + rmSync(path.join(created.tmp, 'fail-setup'), { force: true }) + step.wsC = wsC + // Trust prompt for run actions shown exactly once in this run so far. + const runPrompts = (results.trustPrompts ?? []).filter((x) => x.kind === 'run') + sub.trustOnce = runPrompts.length === 1 + step.trustPromptsByKind = (results.trustPrompts ?? []).reduce((a, x) => ({ ...a, [x.kind]: (a[x.kind] ?? 0) + 1 }), {}) + step.status = Object.values(sub).every(Boolean) ? 'pass' : 'fail' + step.detail = `substeps ${JSON.stringify(sub)}; trust prompts ${JSON.stringify(step.trustPromptsByKind)}; create->auto-run ${step.timings.createToAutoRunMs}ms; Stop->"Stopped" ${step.timings.stopToStoppedMs}ms; ⌥⌘↩ start->serving ${step.timings.shortcutStartToServingMs}ms, restart->serving ${step.timings.shortcutRestartToServingMs}ms; menu Restart->serving ${step.timings.menuRestartToServingMs}ms (Run: Web tabs after: ${step.runWebTabsAfterRestart}); failed setup: ${JSON.stringify(step.failedSetup)}` + }) + + await runStep('run.nonconcurrent', 'echo-loop (nonconcurrent): starting it in workspace B stops it in workspace A', async (step) => { + if (!wsA.id) throw new Error('no workspace A') + Object.assign(wsB, await createLocalWorktree(step, B)) + await waitFor(step, async () => (await runOf(wsB.id, 'web'))?.status === 'running', 60_000) + // Start echo-loop in A. + await (await rowOf(A, 'local')).click() + await sleep(800) + await runMenuItem(step, 'Echo loop', 'Run') + const aOn = await waitFor(step, async () => (await runOf(wsA.id, 'echo-loop'))?.status === 'running', 20_000) + // Start echo-loop in B. + await (await rowOf(B, 'local')).click() + await sleep(800) + const t0 = Date.now() + await runMenuItem(step, 'Echo loop', 'Run') + const res = await waitFor(step, async () => { + const a = await runOf(wsA.id, 'echo-loop') + const b = await runOf(wsB.id, 'echo-loop') + return b?.status === 'running' && (!a || a.status === 'stopped' || a.status === 'exited') ? { a: a?.status ?? 'gone', b: b.status } : null + }, 20_000) + step.timings.bStartToAStoppedMs = res.v ? Date.now() - t0 : null + step.after = { a: (await runOf(wsA.id, 'echo-loop'))?.status ?? 'gone', b: (await runOf(wsB.id, 'echo-loop'))?.status ?? 'gone', webA: (await runOf(wsA.id, 'web'))?.status, webB: (await runOf(wsB.id, 'web'))?.status } + await runMenu(step) + await shot(step, 'b-menu') + await page.keyboard.press('Escape') + step.status = aOn.v && res.v ? 'pass' : 'fail' + step.detail = `A echo-loop running: ${Boolean(aOn.v)}; after starting it in B (${step.timings.bStartToAStoppedMs}ms): ${JSON.stringify(step.after)}. Note: a nonconcurrent start stops every other run of the repo in any workspace, so web (concurrent) in A and in B was stopped too` + }) + + await runStep('run.ports', 'ORCA_PORT/ORCA_PORT_COUNT/CONDUCTOR_PORT in every terminal; distinct blocks per workspace; stable across an app restart; cloud: served in the VM and auto-forwarded here', async (step) => { + const sub = {} + step.substeps = sub + const envA = await terminalEnv(step, A) + const envB = await terminalEnv(step, B) + step.env = { A: envA, B: envB } + sub.envVars = [envA, envB].every((e) => e && e.ORCA_PORT && e.ORCA_PORT_COUNT === '10' && e.CONDUCTOR_PORT === e.ORCA_PORT) + sub.distinctBlocks = envA?.ORCA_PORT !== envB?.ORCA_PORT && Math.abs(Number(envA?.ORCA_PORT) - Number(envB?.ORCA_PORT)) >= 10 + // The nonconcurrent start above stopped web everywhere; start it again in B from the Run button. + // echo-loop is B's primary now, so start Web from the menu ("Run again" after a past run). + await runMenuItem(step, 'Web', 'Run again').catch(async () => { + await page.keyboard.press('Escape').catch(() => {}) + await runMenuItem(step, 'Web', 'Run') + }) + const served = await waitFor(step, async () => (await runOf(wsB.id, 'web'))?.status === 'running' && (await httpOk(Number(envB?.ORCA_PORT))) === 200, 30_000) + step.timings.runButtonToServingMs = served.v ? served.ms : null + sub.localServe = Boolean(served.v) + const regFile = path.join(results.app.userData, 'workspace-port-blocks.json') + step.registry = existsSync(regFile) ? JSON.parse(readFileSync(regFile, 'utf8')) : null + portsBefore.A = envA?.ORCA_PORT + portsBefore.B = envB?.ORCA_PORT + // App restart, same profile. + const t0 = Date.now() + await stopApp() + await launchApp(results.app.userData, 'restart') + await installToastRecorder() + await page.getByText(A, { exact: true }).first().waitFor({ timeout: 30_000 }) + await sleep(2000) + step.timings.relaunchMs = Date.now() - t0 + const envA2 = await terminalEnv(step, A) + const envB2 = await terminalEnv(step, B) + step.envAfterRestart = { A: envA2, B: envB2 } + sub.stableAfterRestart = envA2?.ORCA_PORT === envA?.ORCA_PORT && envB2?.ORCA_PORT === envB?.ORCA_PORT + step.runsAfterRestart = { webA: (await runOf(wsA.id, 'web'))?.status ?? null, webB: (await runOf(wsB.id, 'web'))?.status ?? null, serveA: await httpOk(Number(envA?.ORCA_PORT)), serveB: await httpOk(Number(envB?.ORCA_PORT)) } + await shot(step, 'after-restart') + step.localSubstepsPass = Object.values(sub).every(Boolean) + step.status = step.localSubstepsPass ? 'pass' : 'fail' + step.detail = `local: ${JSON.stringify(sub)}; A ${JSON.stringify(envA)}, B ${JSON.stringify(envB)}; after app restart (${step.timings.relaunchMs}ms) A ${envA2?.ORCA_PORT}, B ${envB2?.ORCA_PORT}; runs after restart ${JSON.stringify(step.runsAfterRestart)}` + }) + + // ---- cloud: the same web action in a VM workspace, forwarded here; Sleep/wake and Restart bring it back. + await store(() => window.__store.getState().connectCurrentOrcaProfile()) + await sleep(1500) + let cws = null + let cloudPort = null + const cloudWebRunning = async () => (await runOf(cws.id, 'web'))?.status === 'running' + await runStep('run.ports.cloud', 'Cloud workspace: the web action listens on $ORCA_PORT in the VM and is reachable here through the auto-forward', async (step) => { + const w = await createExtraWorkspace(step, `${wsName}-cl`, { blankAgent: true }) + const wt = await worktreeByName(w.name) + cws = { ...w, id: wt?.id, path: wt?.path } + step.sandboxId = w.sandboxId + // The run actions come from the repo's primary checkout orca.yaml; put it there in the VM. + const repo = await store((id) => { const s = window.__store.getState(); const r = s.repos.find((x) => (s.worktreesByRepo?.[x.id] ?? []).some((t) => t.id === id)); return r ? { id: r.id, path: r.path } : null }, cws.id) + step.cloudRepo = repo + await openNewTerminalTab() + await waitShellPrompt(90_000) + await focusTerminal() + const yaml = runActionsYaml('/nonexistent-w9').replace(/^ setup:.*\n/m, '').replace(/^ autoRunAfterSetup:.*\n/m, '') + const b64 = Buffer.from(yaml).toString('base64') + await page.keyboard.type(`echo ${b64} | base64 -d > ${repo?.path ?? '/vercel/sandbox/project'}/orca.yaml && cp ${repo?.path ?? '/vercel/sandbox/project'}/orca.yaml ./orca.yaml && echo YAMLOK; echo W9ENV P=$ORCA_PORT C=$ORCA_PORT_COUNT K=$CONDUCTOR_PORT\n`) + const env = await waitFor(step, async () => { + const m = (await visibleTerminals()).join('\n').match(/W9ENV P=(\d*) C=(\d*) K=(\d*)\s*$/m) + return m ? { ORCA_PORT: m[1], ORCA_PORT_COUNT: m[2], CONDUCTOR_PORT: m[3] } : null + }, 30_000) + step.vmEnv = env.v + cloudPort = Number(env.v?.ORCA_PORT) + // Re-read run actions: switch away and back, plus a window focus event. + await (await rowOf(A, 'local')).click() + await sleep(800) + await (await rowOf(cws.name)).click() + await page.evaluate(() => window.dispatchEvent(new Event('focus'))) + const btn = await waitFor(step, async () => /Web/.test(await runButton().innerText().catch(() => '')), 30_000) + step.runButtonShown = Boolean(btn.v) + const t0 = Date.now() + await runButton().getByRole('button', { name: /^Run Web/ }).click() + const running = await waitFor(step, cloudWebRunning, 60_000) + step.timings.clickToRunningMs = running.v ? Date.now() - t0 : null + const fwd = await waitFor(step, async () => (await httpOk(cloudPort)) === 200, 60_000) + step.timings.clickToForwardedServingMs = fwd.v ? Date.now() - t0 : null + step.forwardToast = (await toasts()).find((x) => /Forwarded/.test(x)) ?? null + await shot(step, 'cloud-running') + step.status = env.v && env.v.ORCA_PORT_COUNT === '10' && env.v.CONDUCTOR_PORT === env.v.ORCA_PORT && running.v && fwd.v ? 'pass' : 'fail' + // Fold the cloud half into run.ports (one catalog row). + const rp = results.steps.find((x) => x.id === 'run.ports') + if (rp) { + rp.cloud = { status: step.status, vmEnv: env.v, timings: step.timings } + rp.status = rp.localSubstepsPass && step.status === 'pass' ? 'pass' : 'fail' + rp.screenshots.push(...step.screenshots.filter((x) => /cloud-running/.test(x))) + } + step.detail = `VM terminal ${JSON.stringify(env.v)}; Run Web -> running ${step.timings.clickToRunningMs}ms, reachable on this Mac at localhost:${cloudPort} ${step.timings.clickToForwardedServingMs}ms after the click (${step.forwardToast ? step.forwardToast.split('\n')[0] : 'no forward toast seen'}); repo ${JSON.stringify(step.cloudRepo)}` + if (rp) rp.detail += `; cloud: ${step.detail}` + }) + + await runStep('run.cloud-resume', 'Cloud: Sleep then click to wake brings the action back; Restart sandbox (rollover) brings it back', async (step) => { + if (!cws?.id) throw new Error('no cloud workspace') + step.sandboxId = cws.sandboxId + const sub = {} + step.substeps = sub + // Sleep. + await openRowMenu(cws.name) + await page.getByRole('menuitem', { name: /^(Suspend VM|Sleep)$/ }).first().click() + await page.getByRole('dialog').getByRole('button', { name: /^(Sleep|Suspend)/ }).click({ timeout: 1500 }).catch(() => {}) + const asleep = await waitSandboxStatus(cws.sandboxId, ['suspended', 'sleeping', 'stopped'], 180_000) + step.timings.sleepMs = asleep.ms + await sleep(2000) + const t0 = Date.now() + await (await rowOf(cws.name)).click() + const back = await waitFor(step, async () => (await cloudWebRunning()) && (await httpOk(cloudPort)) === 200, 180_000) + step.timings.wakeClickToServingMs = back.v ? Date.now() - t0 : null + sub.wakeResume = Boolean(back.v) + await shot(step, 'after-wake') + // Restart sandbox (rollover) from the row menu. + await openRowMenu(cws.name) + const t1 = Date.now() + await page.getByRole('menuitem', { name: /^Restart sandbox$/ }).click() + const confirm = page.getByRole('button', { name: /^Restart sandbox$/ }).filter({ visible: true }) + if (await confirm.first().isVisible({ timeout: 2500 }).catch(() => false)) await confirm.first().click() + // Wait for the restart to take effect (server goes away), then for it to come back. + await waitFor(step, async () => (await httpOk(cloudPort)) === null, 60_000) + const back2 = await waitFor(step, async () => (await cloudWebRunning()) && (await httpOk(cloudPort)) === 200, 180_000) + step.timings.restartClickToServingMs = back2.v ? Date.now() - t1 : null + sub.restartResume = Boolean(back2.v) + await shot(step, 'after-restart') + step.status = Object.values(sub).every(Boolean) ? 'pass' : 'fail' + step.detail = `Sleep -> ${asleep.status} in ${asleep.ms}ms; click to wake -> action serving again through the forward ${step.timings.wakeClickToServingMs}ms after the click; Restart sandbox -> serving again ${step.timings.restartClickToServingMs}ms after the click; ${JSON.stringify(sub)}` + }) + + if (cws) { + await runStep('life.destroy', 'Delete the cloud workspace', async (step) => { + step.sandboxId = cws.sandboxId + const o = await uiDelete(step, cws.name, 'delete', 120_000) + const w = await waitSandboxStatus(cws.sandboxId, ['destroyed'], 120_000) + step.status = o.kind === 'removed' && w.status === 'destroyed' ? 'pass' : 'fail' + step.detail = `UI Delete -> ${o.kind} in ${o.ms}ms; sandbox -> ${w.status}` + }) + } + // Stop local servers so nothing keeps listening after the run. + for (const n of [A, B, C]) { + if (await rowShown(n, 'local')) { + await openRowMenu(n, 'local').catch(() => {}) + await page.getByRole('menuitem', { name: /^Delete/ }).click().catch(() => {}) + await page.getByRole('dialog').getByRole('button', { name: /^(Force )?Delete( Workspace)?$/ }).first().click().catch(() => {}) + await waitRowGone(n, 'local', 30_000).catch(() => {}) + } + } +} + /** * W9D_MODE=targeted: picker sign-in (twice per process: fresh profile, then after a sign-out), * create + keystroke, Move to this computer (Archive afterwards on) then Delete within seconds, * and a normal Delete. */ async function targetedFlow() { + if (process.env.W9D_ONLY === 'run-actions') { + await runActionsFlow() + return + } if (process.env.W9D_ONLY === 'ui-audit') { await uiAuditFlow() return