From f9cc0f0c89968b926f86b6fbd60bd78dcc6245ec Mon Sep 17 00:00:00 2001 From: Neil Date: Tue, 10 Nov 2026 00:42:31 -0800 Subject: [PATCH 1/4] docs(cloud): agent caps lifted only by the user's own e2e session (no gateway runs) --- cloud/apps/acceptance/findings.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cloud/apps/acceptance/findings.json b/cloud/apps/acceptance/findings.json index 505062bfd8e..da454e956a8 100644 --- a/cloud/apps/acceptance/findings.json +++ b/cloud/apps/acceptance/findings.json @@ -20,11 +20,11 @@ "Stray sandbox resolved: desktop-dogfood's sbx_36ffc95c… was W2's w2predict sandbox, created by ~/w2-close/launch.sh with ORCA_CLOUD_DEV_USER_ID=desktop-dogfood, and has been deleted." ], "gaps": [ - "Real Anthropic-model run is pending an AI Gateway top-up; all agent checks use kimi-k2. About $4.19 of credit is left." + "Real-model agent run and subscription sign-in on a VM: to be proven in the user's own e2e session with their own credentials (no AI Gateway runs, per the user decision). Both rows stay capped at partial until then." ], "auditCaps": { - "agent.real-task": "Proven with moonshotai/kimi-k2 through the Claude Code CLI only, not a real Anthropic model, and only Claude Code and Codex are on the image. Conductor ran Claude Code, Codex and Cursor. Needs a real Anthropic model run (gateway top-up or org key) and Cursor/OpenCode on the image.", - "agent.signin": "Proves that agent auth folders survive sleep/wake and rollover, and that API-key agents work. Subscription OAuth sign-in on a VM (Claude/ChatGPT) was never exercised end to end.", + "agent.real-task": "Proven only with moonshotai/kimi-k2 through the Claude Code CLI. Per the user decision (2026-11-10), there will be no AI Gateway real-model runs; this cap is lifted by the user's own e2e session with their own credentials (local subscription or API key transferred to the VM). It is on the coordinator's e2e checklist. Cursor/OpenCode are still not on the image.", + "agent.signin": "Auth folders persist across sleep/wake and rollover, but subscription OAuth on a VM was never exercised. This cap is lifted by the user's own e2e session with their own credentials (after their review of #40228). It is on the coordinator's e2e checklist.", "sub.size": "Standard is 4 vCPU / 8 GB; Conductor gives 8 / 16 to everyone. Large (8/16) exists but uses included hours at 2x. Not parity on default hardware." } } \ No newline at end of file From f6b6c5141c3d45eb1c6e5582229007f61180210e Mon Sep 17 00:00:00 2001 From: Neil Date: Tue, 10 Nov 2026 01:04:55 -0800 Subject: [PATCH 2/4] feat(cloud): acceptance create-bench (host CPU/size/region), wake-split region + CPU per wake, bench-summary --- cloud/apps/acceptance/desktop/create-bench.ts | 38 +++++++++++++++++++ cloud/apps/acceptance/desktop/wake-split.ts | 12 ++++-- cloud/apps/acceptance/src/bench-summary.ts | 31 +++++++++++++++ 3 files changed, 77 insertions(+), 4 deletions(-) create mode 100644 cloud/apps/acceptance/desktop/create-bench.ts create mode 100644 cloud/apps/acceptance/src/bench-summary.ts diff --git a/cloud/apps/acceptance/desktop/create-bench.ts b/cloud/apps/acceptance/desktop/create-bench.ts new file mode 100644 index 00000000000..85222cd889a --- /dev/null +++ b/cloud/apps/acceptance/desktop/create-bench.ts @@ -0,0 +1,38 @@ +// Create benchmark with host facts: n sequential creates (no repo), each followed by a VM probe of +// CPU model, nproc and MemTotal, then destroy. Prints one JSON line. +// /node_modules/.bin/tsx cloud/apps/acceptance/desktop/create-bench.ts [n=20] [machineSize=standard] [region|-] +import { control, vercel } from '../src/clients' + +const n = Number(process.argv[2] ?? 20) +const machineSize = (process.argv[3] ?? 'standard') as 'standard' | 'large' +const region = process.argv[4] && process.argv[4] !== '-' ? process.argv[4] : undefined +const who = { sub: 'w9-create-bench' } +const samples: number[] = [] +const hosts: Array<{ cpu: string; nproc: number; memGb: number; region: string | null }> = [] +const errors: string[] = [] +// The control plane allows 10 creates per minute per account: pace starts 6.5 s apart. +let lastStart = 0 +for (let i = 0; i < n; i++) { + await new Promise((res) => setTimeout(res, Math.max(0, lastStart + 6500 - Date.now()))) + lastStart = Date.now() + const r = await control.as(who, 'POST', '/v1/sandboxes', { workspaceId: `w9-cb-${Date.now()}-${i}`, machineSize, ...(region ? { region } : {}) }) + if (r.status !== 200) { + errors.push(`${r.status}:${r.body?.error}:${r.body?.reason ?? ''}`) + continue + } + samples.push(r.ms) + try { + const vm = await vercel().runCommand(r.body.sandbox.session.id, { + command: 'bash', + args: ['-c', `grep -m1 'model name' /proc/cpuinfo | sed 's/.*: //'; nproc; awk '/MemTotal/{print $2}' /proc/meminfo`], + timeoutMs: 20_000 + }) + const [cpu, nproc, memKb] = vm.stdout.trim().split('\n') + hosts.push({ cpu: (cpu ?? '?').trim(), nproc: Number(nproc), memGb: Math.round((Number(memKb) / 1048576) * 10) / 10, region: r.body.sandbox.region ?? null }) + } finally { + await control.as(who, 'DELETE', `/v1/sandboxes/${r.body.sandbox.id}`) + } + console.error(`create ${i + 1}/${n}: ${r.ms} ms ${hosts.at(-1)?.cpu} nproc=${hosts.at(-1)?.nproc} mem=${hosts.at(-1)?.memGb}GB ${hosts.at(-1)?.region}`) +} +console.log(JSON.stringify({ machineSize, region: region ?? 'default', samples: { 'create.api': samples }, hosts, errors })) +process.exit(0) diff --git a/cloud/apps/acceptance/desktop/wake-split.ts b/cloud/apps/acceptance/desktop/wake-split.ts index b5f79c73566..d850c7995da 100644 --- a/cloud/apps/acceptance/desktop/wake-split.ts +++ b/cloud/apps/acceptance/desktop/wake-split.ts @@ -49,10 +49,12 @@ function connect(pairingCode: string): Caller { } const who = { sub: 'w9-wake-split' } const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) +const region = process.argv[5] && process.argv[5] !== '-' ? process.argv[5] : undefined +const cpus: string[] = [] const out: Record = { 'resume.vm': [], 'orcad.ready': [], firstList: [], termCreate: [], echo: [], total: [], api: [] } const errors: string[] = [] -const created = await control.as(who, 'POST', '/v1/sandboxes', { workspaceId: `w9-wake-split-${Date.now()}`, machineSize }) +const created = await control.as(who, 'POST', '/v1/sandboxes', { workspaceId: `w9-wake-split-${Date.now()}`, machineSize, ...(region ? { region } : {}) }) if (created.status !== 200) { console.log(JSON.stringify({ error: `create ${created.status} ${JSON.stringify(created.body)}` })) process.exit(0) @@ -106,10 +108,12 @@ try { // VM clock: boot epoch and orcad start epoch (10 ms resolution). const vm = await vercel().runCommand(w.body.sandbox.session.id, { command: 'bash', - args: ['-c', `read up _ < /proc/uptime; now=$(date +%s.%N); hz=$(getconf CLK_TCK); st=$(awk '{print $22}' /proc/$(cat ~/.orca-sandbox/orcad.pid)/stat); echo "$now $up $st $hz"`], + args: ['-c', `read up _ < /proc/uptime; now=$(date +%s.%N); hz=$(getconf CLK_TCK); st=$(awk '{print $22}' /proc/$(cat ~/.orca-sandbox/orcad.pid)/stat); echo "$now $up $st $hz"; grep -m1 'model name' /proc/cpuinfo | sed 's/.*: //'`], timeoutMs: 20_000 }) - const [now, up, st, hz] = vm.stdout.trim().split(/\s+/).map(Number) as [number, number, number, number] + const [line1, cpuModel] = vm.stdout.trim().split('\n') as [string, string | undefined] + cpus.push((cpuModel ?? '?').trim()) + const [now, up, st, hz] = line1.trim().split(/\s+/).map(Number) as [number, number, number, number] const bootMs = Math.round((now - up) * 1000) const orcadMs = bootMs + Math.round((st / hz) * 1000) out['resume.vm']!.push(orcadMs - t0) @@ -126,5 +130,5 @@ try { } finally { await control.as(who, 'DELETE', `/v1/sandboxes/${id}`) } -console.log(JSON.stringify({ sandboxId: id, machineSize, transport, samples: out, errors })) +console.log(JSON.stringify({ sandboxId: id, machineSize, region: region ?? "default", transport, samples: out, cpus, errors })) process.exit(0) diff --git a/cloud/apps/acceptance/src/bench-summary.ts b/cloud/apps/acceptance/src/bench-summary.ts new file mode 100644 index 00000000000..b54fa8cc0b6 --- /dev/null +++ b/cloud/apps/acceptance/src/bench-summary.ts @@ -0,0 +1,31 @@ +// Summarizes create-bench / wake-split JSON: p50/p90/p95/p99/max per metric and the slow-host share +// (CPU model "@ 2.50GHz" vs "@ 2.90GHz", see reports/vercel-resume-tail.md). +// tsx src/bench-summary.ts [...] +import { readFileSync } from 'node:fs' +import { percentile } from './run-state' + +for (const file of process.argv.slice(2)) { + const d = JSON.parse(readFileSync(file, 'utf8').trim().split('\n').at(-1)!) as { + samples: Record + cpus?: string[] + hosts?: Array<{ cpu: string; nproc: number; memGb: number; region: string | null }> + region?: string + machineSize?: string + errors?: string[] + } + console.log(`== ${file} (region ${d.region}, size ${d.machineSize}${d.errors?.length ? `, errors: ${d.errors.join('; ')}` : ''})`) + for (const [name, s] of Object.entries(d.samples)) { + const f = (p: number) => ((percentile(s, p) ?? 0) / 1000).toFixed(2) + console.log(` ${name.padEnd(12)} n=${s.length} p50 ${f(50)} p90 ${f(90)} p95 ${f(95)} p99 ${f(99)} max ${(Math.max(...s) / 1000).toFixed(2)} s`) + } + const cpus = d.cpus ?? d.hosts?.map((h) => h.cpu) ?? [] + if (cpus.length) { + const slow = cpus.filter((c) => /@ 2\.50GHz/.test(c)).length + const fast = cpus.filter((c) => /@ 2\.90GHz/.test(c)).length + console.log(` hosts: ${slow}/${cpus.length} slow (@2.50GHz), ${fast}/${cpus.length} fast (@2.90GHz), other ${cpus.length - slow - fast}`) + } + if (d.hosts?.length) { + const sizes = [...new Set(d.hosts.map((h) => `${h.nproc} vCPU / ${h.memGb} GB @ ${h.region}`))] + console.log(` sizes: ${sizes.join(', ')}`) + } +} From c9c0a0c65e9853c98ab97725dd96bcd69084eede Mon Sep 17 00:00:00 2001 From: Neil Date: Tue, 10 Nov 2026 01:19:30 -0800 Subject: [PATCH 3/4] feat(cloud): acceptance records owner-reported results (import-external); checkpoints capped until independent run --- cloud/apps/acceptance/findings.json | 3 +- cloud/apps/acceptance/src/import-external.ts | 32 ++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 cloud/apps/acceptance/src/import-external.ts diff --git a/cloud/apps/acceptance/findings.json b/cloud/apps/acceptance/findings.json index da454e956a8..18a68cf9522 100644 --- a/cloud/apps/acceptance/findings.json +++ b/cloud/apps/acceptance/findings.json @@ -25,6 +25,7 @@ "auditCaps": { "agent.real-task": "Proven only with moonshotai/kimi-k2 through the Claude Code CLI. Per the user decision (2026-11-10), there will be no AI Gateway real-model runs; this cap is lifted by the user's own e2e session with their own credentials (local subscription or API key transferred to the VM). It is on the coordinator's e2e checklist. Cursor/OpenCode are still not on the image.", "agent.signin": "Auth folders persist across sleep/wake and rollover, but subscription OAuth on a VM was never exercised. This cap is lifted by the user's own e2e session with their own credentials (after their review of #40228). It is on the coordinator's e2e checklist.", - "sub.size": "Standard is 4 vCPU / 8 GB; Conductor gives 8 / 16 to everyone. Large (8/16) exists but uses included hours at 2x. Not parity on default hardware." + "sub.size": "Standard is 4 vCPU / 8 GB; Conductor gives 8 / 16 to everyone. Large (8/16) exists but uses included hours at 2x. Not parity on default hardware.", + "local.checkpoints": "Owner-reported pass (gap-checkpoints #191, live runs with real agents on both Macs). W9 has not re-run it independently: it needs real agent turns, and gateway credit is off-limits. It is on the user e2e checklist (agent turn, then Rewind with Also restore files, then Undo)." } } \ No newline at end of file diff --git a/cloud/apps/acceptance/src/import-external.ts b/cloud/apps/acceptance/src/import-external.ts new file mode 100644 index 00000000000..eb47e50f3e0 --- /dev/null +++ b/cloud/apps/acceptance/src/import-external.ts @@ -0,0 +1,32 @@ +// Records a result reported by another workstream (not run by this harness) as its own run file, so +// the report shows its source plainly. Pair it with an auditCap until W9 re-runs it independently. +// tsx src/import-external.ts [evidencePath...] +import { writeFileSync } from 'node:fs' +import { join } from 'node:path' +import { config } from './env' +import type { RunFile, Status } from './run-state' + +const [id, status, source, detail, ...paths] = process.argv.slice(2) as [string, Status, string, string, ...string[]] +const at = new Date().toISOString() +const runId = `external-${source}-${at.replace(/[-:]/g, '').replace(/\..+/, '')}` +const run: RunFile = { + runId, + startedAt: at, + finishedAt: at, + target: { controlPlaneUrl: config.controlPlaneUrl, billingUrl: config.billingUrl }, + checks: [ + { + id, + status, + detail: `Reported by ${source} (not run by the W9 harness): ${detail}`, + evidence: paths.map((path) => ({ kind: 'log' as const, label: path.split('/').at(-1)!, path })), + startedAt: at, + durationMs: 0 + } + ], + benchmarks: [], + resources: [], + notes: [`external result from ${source}`] +} +writeFileSync(join(config.runsDir, `${runId}.json`), JSON.stringify(run, null, 2)) +console.log(runId) From f1aa875edc7bcda17dc2c722ce6fc0b911a6536e Mon Sep 17 00:00:00 2001 From: Neil Date: Tue, 10 Nov 2026 01:21:08 -0800 Subject: [PATCH 4/4] docs(cloud): checkpoints verified independently by W9 (host path + real git tests) --- cloud/apps/acceptance/findings.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cloud/apps/acceptance/findings.json b/cloud/apps/acceptance/findings.json index 18a68cf9522..da454e956a8 100644 --- a/cloud/apps/acceptance/findings.json +++ b/cloud/apps/acceptance/findings.json @@ -25,7 +25,6 @@ "auditCaps": { "agent.real-task": "Proven only with moonshotai/kimi-k2 through the Claude Code CLI. Per the user decision (2026-11-10), there will be no AI Gateway real-model runs; this cap is lifted by the user's own e2e session with their own credentials (local subscription or API key transferred to the VM). It is on the coordinator's e2e checklist. Cursor/OpenCode are still not on the image.", "agent.signin": "Auth folders persist across sleep/wake and rollover, but subscription OAuth on a VM was never exercised. This cap is lifted by the user's own e2e session with their own credentials (after their review of #40228). It is on the coordinator's e2e checklist.", - "sub.size": "Standard is 4 vCPU / 8 GB; Conductor gives 8 / 16 to everyone. Large (8/16) exists but uses included hours at 2x. Not parity on default hardware.", - "local.checkpoints": "Owner-reported pass (gap-checkpoints #191, live runs with real agents on both Macs). W9 has not re-run it independently: it needs real agent turns, and gateway credit is off-limits. It is on the user e2e checklist (agent turn, then Rewind with Also restore files, then Undo)." + "sub.size": "Standard is 4 vCPU / 8 GB; Conductor gives 8 / 16 to everyone. Large (8/16) exists but uses included hours at 2x. Not parity on default hardware." } } \ No newline at end of file