mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 19:07:41 +00:00
fix(cloud): acceptance report applies independent-audit caps (partial) and adds local core workflow, run actions, read-while-asleep and checkpoints rows
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
{
|
||||
"findings": [
|
||||
"<b>Independent audit corrections (2026-11-10, reports/conductor-parity-audit.html):</b> five rows are capped at partial because they passed on a weaker bar than the Conductor column: real-model agent run, subscription sign-in, default machine size, and the download and remote-desktop UIs. Four rows Conductor has that our bar left out are added as not landed: local core workflow, Run actions with per-workspace ports, read-while-asleep from any device, and per-turn code checkpoints. Our 'pass' counts measure the team's own bar. Every cloud result is on dev Workers, Stripe TEST mode and dev/staging tokens; nothing is shipped (mvp-tracker §5 launch items are all open).",
|
||||
"<b>Wake delivery (target p95 < 6 s):</b> latest is golden snap_6kSW (543cff4: #173 V8 code cache, #175 sealed offer + 10 ms ready poll), n=22, persistent connection. Total p50 4.78 s, p95 6.31 s, p99 7.87 s. Phases p50 / p95 / p99: Vercel resume to orcad spawn 2.99 / 4.29 / 5.46 s; orcad ready 1.04 / 1.53 / 1.70 s; first list 0.44 / 0.48 / 0.56 s; terminal create 0.15 / 0.23 / 0.25 s; echo 0.15 / 0.16 / 0.16 s. History (p50 / p95): pre-#141 7.7 / 9.5 s; #141 6.8 / 11.0 s; #151/#162 5.13 / 6.49 s; #173/#175 4.78 / 6.31 s. The p95 gap is now almost all Vercel resume.",
|
||||
"<b>Rows closed 2026-11-10:</b> real 15-min idle sleep: an idle VM with no client stopped by itself after 19.2 min. Agent sign-in persistence: Claude answers, onboarding is pre-seeded, and the auth homes are kept fresh, after a wake and after a rollover. Auto-resume after Restart on m4air: one <code>continue</code> 29.4 s after the click, turn finished, no double continue. Wake on message: a message to a sleeping workspace was delivered in 6.1 s. Scheduled routine: fired by itself and succeeded on a fresh VM, which was then destroyed. File search: ripgrep over the pairing found both needles in 5,002 files, p50 88 ms. Usage metering: control-plane outbox, then billing hours, then the Stripe meter. Overage cap blocks new starts at $5. Dunning end to end: grace expiry (past_due_since backdated 97 h) leads the control-plane cron to suspend the running VM in 123 s; paying lets the account start again and the VM resumes. Desktop keystroke n=50: p50/p95/p99 84/94/144 ms; the 407 ms outlier did not recur. Web client is post-MVP.",
|
||||
"<b>Control-plane billing gate (closed via core #166):</b> a <code>dev:billing-test-*</code> account goes through the real billing Worker. No plan → 403 no_subscription; active plan → 200; 4th concurrent VM on 1 seat → 403; included hours used up → 403; failed renewal → 403 payment_past_due; grace expired → the running VM was suspended by the cron in 71 s and resume → 403; paid → resume 200. Staging team checks pass again (teams are deleted by the shared cleanup script).",
|
||||
@@ -19,5 +20,12 @@
|
||||
],
|
||||
"gaps": [
|
||||
"Real Anthropic-model run is pending an AI Gateway top-up; all agent checks use kimi-k2. About $4.19 of credit is left."
|
||||
]
|
||||
],
|
||||
"auditCaps": {
|
||||
"agent.real-task": "Proven with moonshotai/kimi-k2 through the Claude Code CLI only, not a real Anthropic model, and only Claude Code and Codex are on the image. Conductor ran Claude Code, Codex and Cursor. Needs a real Anthropic model run (gateway top-up or org key) and Cursor/OpenCode on the image.",
|
||||
"agent.signin": "Proves that agent auth folders survive sleep/wake and rollover, and that API-key agents work. Subscription OAuth sign-in on a VM (Claude/ChatGPT) was never exercised end to end.",
|
||||
"sub.size": "Standard is 4 vCPU / 8 GB; Conductor gives 8 / 16 to everyone. Large (8/16) exists but uses included hours at 2x. Not parity on default hardware.",
|
||||
"acc.download": "Transfer verified headlessly with the desktop download code, but the UI (file-tree Download action, Save dialog, progress/toasts) was not exercised. m4air UI run requested.",
|
||||
"acc.remote-desktop": "Backend verified (Xvnc + ticketed WebSocket, 101/403), but streaming into a desktop tab in the app was not exercised. m4air UI run requested."
|
||||
}
|
||||
}
|
||||
@@ -38,6 +38,11 @@ export const CATALOG: Feature[] = [
|
||||
{ id: 'billing.overage', section: S.acct, feature: 'Overage opt-in at $0.75/h with a monthly spend cap that blocks starts', conductor: 'No metering ("no extra charge today")', owner: 'team-billing-ux', priority: 'P0', better: 'Metered from day one with a hard cap' },
|
||||
{ id: 'billing.dunning', section: S.acct, feature: 'Payment failure -> grace -> suspend -> pay -> resume (test clocks)', conductor: 'not documented', owner: 'team-billing-ux', priority: 'P0' },
|
||||
{ id: 'billing.entitlement-gate', section: S.acct, feature: 'Account without a plan is refused a sandbox (fail closed)', conductor: 'Pro/org required', owner: 'core', priority: 'P0' },
|
||||
// Conductor's local core workflow (added after the independent audit; the cloud matrix left it out)
|
||||
{ id: 'local.core-workflow', section: S.acct, feature: 'Local core workflow: create a local worktree, run an agent, see the diff, commit (desktop)', conductor: 'Core local app (free)', owner: 'w9-eval (m4air desktop run)', priority: 'P0' },
|
||||
{ id: 'local.run-actions', section: S.acct, feature: 'Run actions: named run scripts (several, run_mode, auto-run after setup) with a per-workspace port block; Conductor import keeps them', conductor: '0.70 multiple run scripts, CONDUCTOR_PORT 10-port block', owner: 'gap-run-actions', priority: 'P0' },
|
||||
{ id: 'life.read-asleep', section: S.life, feature: 'Chats, PR comments and check logs readable while asleep from any device (no wake)', conductor: '0.80/0.85, server-held', owner: 'gap-read-asleep', priority: 'P0' },
|
||||
{ id: 'local.checkpoints', section: S.acct, feature: 'Per-turn code checkpoints: revert restores files and chat together', conductor: '0.19 (Claude), 0.44 (Codex) checkpoints', owner: 'gap-checkpoints', priority: 'P0' },
|
||||
// 1. Substrate
|
||||
{ id: 'sub.create', section: S.sub, feature: 'Hosted microVM per workspace, one call / one click', conductor: '0.78 GA', owner: 'core', priority: 'P0' },
|
||||
{ id: 'sub.size', section: S.sub, feature: 'Machine sizes (standard 4 vCPU, large 8 vCPU)', conductor: 'Fixed 8 vCPU / 16 GB', owner: 'core', priority: 'P0', better: 'Choice of size' },
|
||||
|
||||
@@ -69,10 +69,22 @@ function evidenceHtml(result: CheckResult, run: RunFile): string {
|
||||
return parts.join('<br>')
|
||||
}
|
||||
|
||||
/** Independent-audit caps (findings.json auditCaps): a check that passes on a weaker bar than the
|
||||
* Conductor column is shown as partial, with the reason, until the stronger proof exists. */
|
||||
function auditCap(id: string): string | null {
|
||||
return (loadFindings() as { auditCaps?: Record<string, string> }).auditCaps?.[id] ?? null
|
||||
}
|
||||
|
||||
export function effectiveStatus(id: string, raw: string | undefined): string {
|
||||
const status = raw ?? 'not_landed'
|
||||
return status === 'pass' && auditCap(id) ? 'partial' : status
|
||||
}
|
||||
|
||||
function featureRow(f: Feature, hit: { result: CheckResult; run: RunFile } | undefined): string {
|
||||
const status = hit?.result.status ?? 'not_landed'
|
||||
const status = effectiveStatus(f.id, hit?.result.status)
|
||||
const s = STATUS[status]!
|
||||
const detail = hit ? esc(hit.result.detail) : `No check result yet; waiting on <b>${esc(f.owner)}</b>.`
|
||||
const cap = hit?.result.status === 'pass' ? auditCap(f.id) : null
|
||||
const detail = (cap ? `<b>Audit cap (partial):</b> ${esc(cap)}<br>` : '') + (hit ? esc(hit.result.detail) : `No check result yet; waiting on <b>${esc(f.owner)}</b>.`)
|
||||
return `<tr><th>${esc(f.feature)}<div class="small muted"><code>${esc(f.id)}</code> · ${esc(f.priority)} · ${esc(f.owner)}</div></th>
|
||||
<td class="${s.cls} c">${s.label}</td><td>${detail}</td><td>${hit ? evidenceHtml(hit.result, hit.run) : ''}</td>
|
||||
<td>${esc(f.conductor)}</td><td>${f.better ? esc(f.better) : ''}</td></tr>`
|
||||
@@ -120,7 +132,7 @@ export function renderReport(): string {
|
||||
const latest = latestResults(runs)
|
||||
const counts = { pass: 0, partial: 0, fail: 0, not_landed: 0, skipped: 0, deferred: 0 }
|
||||
for (const f of CATALOG) {
|
||||
counts[(latest.get(f.id)?.result.status ?? 'not_landed') as keyof typeof counts]++
|
||||
counts[effectiveStatus(f.id, latest.get(f.id)?.result.status) as keyof typeof counts]++
|
||||
}
|
||||
const allResources = runs.flatMap((r) => r.resources)
|
||||
const cost = resourceCost(allResources)
|
||||
|
||||
Reference in New Issue
Block a user