diff --git a/cloud/apps/acceptance/findings.json b/cloud/apps/acceptance/findings.json index ae08eae3a71..f74290f9adb 100644 --- a/cloud/apps/acceptance/findings.json +++ b/cloud/apps/acceptance/findings.json @@ -1,5 +1,6 @@ { "findings": [ + "Independent audit corrections (2026-11-10, reports/conductor-parity-audit.html): five rows are capped at partial because they passed on a weaker bar than the Conductor column: real-model agent run, subscription sign-in, default machine size, and the download and remote-desktop UIs. Four rows Conductor has that our bar left out are added as not landed: local core workflow, Run actions with per-workspace ports, read-while-asleep from any device, and per-turn code checkpoints. Our 'pass' counts measure the team's own bar. Every cloud result is on dev Workers, Stripe TEST mode and dev/staging tokens; nothing is shipped (mvp-tracker §5 launch items are all open).", "Wake delivery (target p95 < 6 s): latest is golden snap_6kSW (543cff4: #173 V8 code cache, #175 sealed offer + 10 ms ready poll), n=22, persistent connection. Total p50 4.78 s, p95 6.31 s, p99 7.87 s. Phases p50 / p95 / p99: Vercel resume to orcad spawn 2.99 / 4.29 / 5.46 s; orcad ready 1.04 / 1.53 / 1.70 s; first list 0.44 / 0.48 / 0.56 s; terminal create 0.15 / 0.23 / 0.25 s; echo 0.15 / 0.16 / 0.16 s. History (p50 / p95): pre-#141 7.7 / 9.5 s; #141 6.8 / 11.0 s; #151/#162 5.13 / 6.49 s; #173/#175 4.78 / 6.31 s. The p95 gap is now almost all Vercel resume.", "Rows closed 2026-11-10: real 15-min idle sleep: an idle VM with no client stopped by itself after 19.2 min. Agent sign-in persistence: Claude answers, onboarding is pre-seeded, and the auth homes are kept fresh, after a wake and after a rollover. Auto-resume after Restart on m4air: one continue 29.4 s after the click, turn finished, no double continue. Wake on message: a message to a sleeping workspace was delivered in 6.1 s. Scheduled routine: fired by itself and succeeded on a fresh VM, which was then destroyed. File search: ripgrep over the pairing found both needles in 5,002 files, p50 88 ms. Usage metering: control-plane outbox, then billing hours, then the Stripe meter. Overage cap blocks new starts at $5. Dunning end to end: grace expiry (past_due_since backdated 97 h) leads the control-plane cron to suspend the running VM in 123 s; paying lets the account start again and the VM resumes. Desktop keystroke n=50: p50/p95/p99 84/94/144 ms; the 407 ms outlier did not recur. Web client is post-MVP.", "Control-plane billing gate (closed via core #166): a dev:billing-test-* account goes through the real billing Worker. No plan → 403 no_subscription; active plan → 200; 4th concurrent VM on 1 seat → 403; included hours used up → 403; failed renewal → 403 payment_past_due; grace expired → the running VM was suspended by the cron in 71 s and resume → 403; paid → resume 200. Staging team checks pass again (teams are deleted by the shared cleanup script).", @@ -19,5 +20,12 @@ ], "gaps": [ "Real Anthropic-model run is pending an AI Gateway top-up; all agent checks use kimi-k2. About $4.19 of credit is left." - ] + ], + "auditCaps": { + "agent.real-task": "Proven with moonshotai/kimi-k2 through the Claude Code CLI only, not a real Anthropic model, and only Claude Code and Codex are on the image. Conductor ran Claude Code, Codex and Cursor. Needs a real Anthropic model run (gateway top-up or org key) and Cursor/OpenCode on the image.", + "agent.signin": "Proves that agent auth folders survive sleep/wake and rollover, and that API-key agents work. Subscription OAuth sign-in on a VM (Claude/ChatGPT) was never exercised end to end.", + "sub.size": "Standard is 4 vCPU / 8 GB; Conductor gives 8 / 16 to everyone. Large (8/16) exists but uses included hours at 2x. Not parity on default hardware.", + "acc.download": "Transfer verified headlessly with the desktop download code, but the UI (file-tree Download action, Save dialog, progress/toasts) was not exercised. m4air UI run requested.", + "acc.remote-desktop": "Backend verified (Xvnc + ticketed WebSocket, 101/403), but streaming into a desktop tab in the app was not exercised. m4air UI run requested." + } } \ No newline at end of file diff --git a/cloud/apps/acceptance/src/catalog.ts b/cloud/apps/acceptance/src/catalog.ts index 9a5faf25e94..938a1d342a8 100644 --- a/cloud/apps/acceptance/src/catalog.ts +++ b/cloud/apps/acceptance/src/catalog.ts @@ -38,6 +38,11 @@ export const CATALOG: Feature[] = [ { id: 'billing.overage', section: S.acct, feature: 'Overage opt-in at $0.75/h with a monthly spend cap that blocks starts', conductor: 'No metering ("no extra charge today")', owner: 'team-billing-ux', priority: 'P0', better: 'Metered from day one with a hard cap' }, { id: 'billing.dunning', section: S.acct, feature: 'Payment failure -> grace -> suspend -> pay -> resume (test clocks)', conductor: 'not documented', owner: 'team-billing-ux', priority: 'P0' }, { id: 'billing.entitlement-gate', section: S.acct, feature: 'Account without a plan is refused a sandbox (fail closed)', conductor: 'Pro/org required', owner: 'core', priority: 'P0' }, + // Conductor's local core workflow (added after the independent audit; the cloud matrix left it out) + { id: 'local.core-workflow', section: S.acct, feature: 'Local core workflow: create a local worktree, run an agent, see the diff, commit (desktop)', conductor: 'Core local app (free)', owner: 'w9-eval (m4air desktop run)', priority: 'P0' }, + { id: 'local.run-actions', section: S.acct, feature: 'Run actions: named run scripts (several, run_mode, auto-run after setup) with a per-workspace port block; Conductor import keeps them', conductor: '0.70 multiple run scripts, CONDUCTOR_PORT 10-port block', owner: 'gap-run-actions', priority: 'P0' }, + { id: 'life.read-asleep', section: S.life, feature: 'Chats, PR comments and check logs readable while asleep from any device (no wake)', conductor: '0.80/0.85, server-held', owner: 'gap-read-asleep', priority: 'P0' }, + { id: 'local.checkpoints', section: S.acct, feature: 'Per-turn code checkpoints: revert restores files and chat together', conductor: '0.19 (Claude), 0.44 (Codex) checkpoints', owner: 'gap-checkpoints', priority: 'P0' }, // 1. Substrate { id: 'sub.create', section: S.sub, feature: 'Hosted microVM per workspace, one call / one click', conductor: '0.78 GA', owner: 'core', priority: 'P0' }, { id: 'sub.size', section: S.sub, feature: 'Machine sizes (standard 4 vCPU, large 8 vCPU)', conductor: 'Fixed 8 vCPU / 16 GB', owner: 'core', priority: 'P0', better: 'Choice of size' }, diff --git a/cloud/apps/acceptance/src/report.ts b/cloud/apps/acceptance/src/report.ts index 4fe152478f6..86e4b5a7824 100644 --- a/cloud/apps/acceptance/src/report.ts +++ b/cloud/apps/acceptance/src/report.ts @@ -69,10 +69,22 @@ function evidenceHtml(result: CheckResult, run: RunFile): string { return parts.join('
') } +/** Independent-audit caps (findings.json auditCaps): a check that passes on a weaker bar than the + * Conductor column is shown as partial, with the reason, until the stronger proof exists. */ +function auditCap(id: string): string | null { + return (loadFindings() as { auditCaps?: Record }).auditCaps?.[id] ?? null +} + +export function effectiveStatus(id: string, raw: string | undefined): string { + const status = raw ?? 'not_landed' + return status === 'pass' && auditCap(id) ? 'partial' : status +} + function featureRow(f: Feature, hit: { result: CheckResult; run: RunFile } | undefined): string { - const status = hit?.result.status ?? 'not_landed' + const status = effectiveStatus(f.id, hit?.result.status) const s = STATUS[status]! - const detail = hit ? esc(hit.result.detail) : `No check result yet; waiting on ${esc(f.owner)}.` + const cap = hit?.result.status === 'pass' ? auditCap(f.id) : null + const detail = (cap ? `Audit cap (partial): ${esc(cap)}
` : '') + (hit ? esc(hit.result.detail) : `No check result yet; waiting on ${esc(f.owner)}.`) return `${esc(f.feature)}
${esc(f.id)} · ${esc(f.priority)} · ${esc(f.owner)}
${s.label}${detail}${hit ? evidenceHtml(hit.result, hit.run) : ''} ${esc(f.conductor)}${f.better ? esc(f.better) : ''}` @@ -120,7 +132,7 @@ export function renderReport(): string { const latest = latestResults(runs) const counts = { pass: 0, partial: 0, fail: 0, not_landed: 0, skipped: 0, deferred: 0 } for (const f of CATALOG) { - counts[(latest.get(f.id)?.result.status ?? 'not_landed') as keyof typeof counts]++ + counts[effectiveStatus(f.id, latest.get(f.id)?.result.status) as keyof typeof counts]++ } const allResources = runs.flatMap((r) => r.resources) const cost = resourceCost(allResources)