From dd0cd5b0012e921c9510ca3cd079ef0ecae00507 Mon Sep 17 00:00:00 2001 From: Neil Date: Mon, 9 Nov 2026 21:27:41 -0800 Subject: [PATCH] feat(cloud): acceptance report p99 column; wake delivery on the #151/#162 golden (n=22) --- cloud/apps/acceptance/findings.json | 2 +- cloud/apps/acceptance/src/report.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cloud/apps/acceptance/findings.json b/cloud/apps/acceptance/findings.json index 31260fe1329..c97a87f4eb1 100644 --- a/cloud/apps/acceptance/findings.json +++ b/cloud/apps/acceptance/findings.json @@ -1,6 +1,6 @@ { "findings": [ - "Wake delivery (2026-11-10, coordinator target p95 < 6 s): NOT MET. The golden with #141 (5adc8fb, n=12) gives total p50 6.8 s, p95 11.0 s, against 7.7 / 9.5 s before #141 (n=10). #141 cut orcad ready from p50 2.4 to 1.5 s (p95 3.4 to 2.2 s). Vercel resume to orcad spawn stays at p50 3.0 s, and its tail (p95 6.4 s; 2 of 12 above 5.5 s) now dominates p95. After ready: first terminal list 0.5 s (includes the E2EE handshake), terminal create 0.5 s, typed command to echo 1.3 s (the new login shell's startup). Biggest remaining levers: the Vercel resume tail (core/Vercel), login-shell startup in the image (about 1.3 s), and reusing the warm connection the desktop already holds. Script: desktop/wake-split.ts.", + "Wake delivery (2026-11-10, target p95 < 6 s): on golden snap_aQmW (includes #141, #151, #162), one persistent connection per wake, n=22: total p50 5.13 s, p95 6.49 s, p99 6.72 s. Just over target at p95. Phases (p50 / p95): Vercel resume to orcad spawn 2.98 / 4.27 s; orcad ready 1.36 / 1.68 s; first terminal list incl. handshake 0.43 / 0.45 s; terminal create 0.14 / 0.17 s; echo 0.14 / 0.16 s. Before: 6.8 / 11.0 s (#141 only, reconnect per call) and 7.7 / 9.5 s (pre-#141). #151/#162 cut create+echo from 1.8 s to 0.29 s. What is left is Vercel resume (about 58% of p50) and orcad ready.", "Rows closed 2026-11-10: real 15-min idle sleep: an idle VM with no client stopped by itself after 19.2 min. Agent sign-in persistence: Claude answers, onboarding is pre-seeded, and the auth homes are kept fresh, after a wake and after a rollover. Auto-resume after Restart on m4air: one continue 29.4 s after the click, turn finished, no double continue. Wake on message: a message to a sleeping workspace was delivered in 6.1 s. Scheduled routine: fired by itself and succeeded on a fresh VM, which was then destroyed. File search: ripgrep over the pairing found both needles in 5,002 files, p50 88 ms. Usage metering: control-plane outbox, then billing hours, then the Stripe meter. Overage cap blocks new starts at $5. Dunning end to end: grace expiry (past_due_since backdated 97 h) leads the control-plane cron to suspend the running VM in 123 s; paying lets the account start again and the VM resumes. Desktop keystroke n=50: p50/p95/p99 84/94/144 ms; the 407 ms outlier did not recur. Web client is post-MVP.", "Control-plane billing gate (closed via core #166): a dev:billing-test-* account goes through the real billing Worker. No plan → 403 no_subscription; active plan → 200; 4th concurrent VM on 1 seat → 403; included hours used up → 403; failed renewal → 403 payment_past_due; grace expired → the running VM was suspended by the cron in 71 s and resume → 403; paid → resume 200. Staging team checks pass again (teams are deleted by the shared cleanup script).", "CLI create (closed via W5 #167): orca worktree create --on cloud --repo … with an API key, headless: running in 3.7 s; orca cloud workspace destroy works; signed out gives cloud_not_signed_in. Move, Sleep, then Delete: no wake, destroyed, Vercel 404, no misleading toasts. The row took 31 s to disappear, against about 2 s for other asleep deletes (sent to core-desktop as a perf note).", diff --git a/cloud/apps/acceptance/src/report.ts b/cloud/apps/acceptance/src/report.ts index 6193783f4e3..4fe152478f6 100644 --- a/cloud/apps/acceptance/src/report.ts +++ b/cloud/apps/acceptance/src/report.ts @@ -92,10 +92,10 @@ function benchTable(runs: RunFile[]): string { const b = latest.get(name) const n = b?.samples.length ?? 0 return `${esc(name)}${n}${b ? sec(percentile(b.samples, 50)) : 'pending'} -${b ? sec(percentile(b.samples, 95)) : ''}${b ? `${sec(Math.min(...b.samples))} – ${sec(Math.max(...b.samples))}` : ''} +${b ? sec(percentile(b.samples, 95)) : ''}${b && b.samples.length >= 20 ? sec(percentile(b.samples, 99)) : ''}${b ? `${sec(Math.min(...b.samples))} – ${sec(Math.max(...b.samples))}` : ''} ${esc(CONDUCTOR[name] ?? '')}${esc(EARLIER[name] ?? '')}${b ? `run ${esc(b.run.runId)}${b.note ? `
${esc(b.note)}` : ''}` : ''}` }) - return `
${rows.join('')}
Metricnp50p95min – maxConductor (documented)Our earlier numbersSource
` + return `
${rows.join('')}
Metricnp50p95p99 (n≥20)min – maxConductor (documented)Our earlier numbersSource
` } function ledgerTable(runs: RunFile[]): string {