mirror of
https://github.com/stablyai/orca.git
synced 2026-11-19 04:23:57 +00:00
Merge pull request #198 from nwparker/cloud/w9-eval
feat(cloud): acceptance region/size bench tools, external-result import, checkpoints verified, agent caps for user e2e
This commit is contained in:
@@ -0,0 +1,38 @@
|
||||
// Create benchmark with host facts: n sequential creates (no repo), each followed by a VM probe of
|
||||
// CPU model, nproc and MemTotal, then destroy. Prints one JSON line.
|
||||
// <repo>/node_modules/.bin/tsx cloud/apps/acceptance/desktop/create-bench.ts [n=20] [machineSize=standard] [region|-]
|
||||
import { control, vercel } from '../src/clients'
|
||||
|
||||
const n = Number(process.argv[2] ?? 20)
|
||||
const machineSize = (process.argv[3] ?? 'standard') as 'standard' | 'large'
|
||||
const region = process.argv[4] && process.argv[4] !== '-' ? process.argv[4] : undefined
|
||||
const who = { sub: 'w9-create-bench' }
|
||||
const samples: number[] = []
|
||||
const hosts: Array<{ cpu: string; nproc: number; memGb: number; region: string | null }> = []
|
||||
const errors: string[] = []
|
||||
// The control plane allows 10 creates per minute per account: pace starts 6.5 s apart.
|
||||
let lastStart = 0
|
||||
for (let i = 0; i < n; i++) {
|
||||
await new Promise((res) => setTimeout(res, Math.max(0, lastStart + 6500 - Date.now())))
|
||||
lastStart = Date.now()
|
||||
const r = await control.as(who, 'POST', '/v1/sandboxes', { workspaceId: `w9-cb-${Date.now()}-${i}`, machineSize, ...(region ? { region } : {}) })
|
||||
if (r.status !== 200) {
|
||||
errors.push(`${r.status}:${r.body?.error}:${r.body?.reason ?? ''}`)
|
||||
continue
|
||||
}
|
||||
samples.push(r.ms)
|
||||
try {
|
||||
const vm = await vercel().runCommand(r.body.sandbox.session.id, {
|
||||
command: 'bash',
|
||||
args: ['-c', `grep -m1 'model name' /proc/cpuinfo | sed 's/.*: //'; nproc; awk '/MemTotal/{print $2}' /proc/meminfo`],
|
||||
timeoutMs: 20_000
|
||||
})
|
||||
const [cpu, nproc, memKb] = vm.stdout.trim().split('\n')
|
||||
hosts.push({ cpu: (cpu ?? '?').trim(), nproc: Number(nproc), memGb: Math.round((Number(memKb) / 1048576) * 10) / 10, region: r.body.sandbox.region ?? null })
|
||||
} finally {
|
||||
await control.as(who, 'DELETE', `/v1/sandboxes/${r.body.sandbox.id}`)
|
||||
}
|
||||
console.error(`create ${i + 1}/${n}: ${r.ms} ms ${hosts.at(-1)?.cpu} nproc=${hosts.at(-1)?.nproc} mem=${hosts.at(-1)?.memGb}GB ${hosts.at(-1)?.region}`)
|
||||
}
|
||||
console.log(JSON.stringify({ machineSize, region: region ?? 'default', samples: { 'create.api': samples }, hosts, errors }))
|
||||
process.exit(0)
|
||||
@@ -49,10 +49,12 @@ function connect(pairingCode: string): Caller {
|
||||
}
|
||||
const who = { sub: 'w9-wake-split' }
|
||||
const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms))
|
||||
const region = process.argv[5] && process.argv[5] !== '-' ? process.argv[5] : undefined
|
||||
const cpus: string[] = []
|
||||
const out: Record<string, number[]> = { 'resume.vm': [], 'orcad.ready': [], firstList: [], termCreate: [], echo: [], total: [], api: [] }
|
||||
const errors: string[] = []
|
||||
|
||||
const created = await control.as(who, 'POST', '/v1/sandboxes', { workspaceId: `w9-wake-split-${Date.now()}`, machineSize })
|
||||
const created = await control.as(who, 'POST', '/v1/sandboxes', { workspaceId: `w9-wake-split-${Date.now()}`, machineSize, ...(region ? { region } : {}) })
|
||||
if (created.status !== 200) {
|
||||
console.log(JSON.stringify({ error: `create ${created.status} ${JSON.stringify(created.body)}` }))
|
||||
process.exit(0)
|
||||
@@ -106,10 +108,12 @@ try {
|
||||
// VM clock: boot epoch and orcad start epoch (10 ms resolution).
|
||||
const vm = await vercel().runCommand(w.body.sandbox.session.id, {
|
||||
command: 'bash',
|
||||
args: ['-c', `read up _ < /proc/uptime; now=$(date +%s.%N); hz=$(getconf CLK_TCK); st=$(awk '{print $22}' /proc/$(cat ~/.orca-sandbox/orcad.pid)/stat); echo "$now $up $st $hz"`],
|
||||
args: ['-c', `read up _ < /proc/uptime; now=$(date +%s.%N); hz=$(getconf CLK_TCK); st=$(awk '{print $22}' /proc/$(cat ~/.orca-sandbox/orcad.pid)/stat); echo "$now $up $st $hz"; grep -m1 'model name' /proc/cpuinfo | sed 's/.*: //'`],
|
||||
timeoutMs: 20_000
|
||||
})
|
||||
const [now, up, st, hz] = vm.stdout.trim().split(/\s+/).map(Number) as [number, number, number, number]
|
||||
const [line1, cpuModel] = vm.stdout.trim().split('\n') as [string, string | undefined]
|
||||
cpus.push((cpuModel ?? '?').trim())
|
||||
const [now, up, st, hz] = line1.trim().split(/\s+/).map(Number) as [number, number, number, number]
|
||||
const bootMs = Math.round((now - up) * 1000)
|
||||
const orcadMs = bootMs + Math.round((st / hz) * 1000)
|
||||
out['resume.vm']!.push(orcadMs - t0)
|
||||
@@ -126,5 +130,5 @@ try {
|
||||
} finally {
|
||||
await control.as(who, 'DELETE', `/v1/sandboxes/${id}`)
|
||||
}
|
||||
console.log(JSON.stringify({ sandboxId: id, machineSize, transport, samples: out, errors }))
|
||||
console.log(JSON.stringify({ sandboxId: id, machineSize, region: region ?? "default", transport, samples: out, cpus, errors }))
|
||||
process.exit(0)
|
||||
|
||||
@@ -20,11 +20,11 @@
|
||||
"<b>Stray sandbox resolved:</b> <code>desktop-dogfood</code>'s sbx_36ffc95c… was W2's w2predict sandbox, created by <code>~/w2-close/launch.sh</code> with ORCA_CLOUD_DEV_USER_ID=desktop-dogfood, and has been deleted."
|
||||
],
|
||||
"gaps": [
|
||||
"Real Anthropic-model run is pending an AI Gateway top-up; all agent checks use kimi-k2. About $4.19 of credit is left."
|
||||
"Real-model agent run and subscription sign-in on a VM: to be proven in the user's own e2e session with their own credentials (no AI Gateway runs, per the user decision). Both rows stay capped at partial until then."
|
||||
],
|
||||
"auditCaps": {
|
||||
"agent.real-task": "Proven with moonshotai/kimi-k2 through the Claude Code CLI only, not a real Anthropic model, and only Claude Code and Codex are on the image. Conductor ran Claude Code, Codex and Cursor. Needs a real Anthropic model run (gateway top-up or org key) and Cursor/OpenCode on the image.",
|
||||
"agent.signin": "Proves that agent auth folders survive sleep/wake and rollover, and that API-key agents work. Subscription OAuth sign-in on a VM (Claude/ChatGPT) was never exercised end to end.",
|
||||
"agent.real-task": "Proven only with moonshotai/kimi-k2 through the Claude Code CLI. Per the user decision (2026-11-10), there will be no AI Gateway real-model runs; this cap is lifted by the user's own e2e session with their own credentials (local subscription or API key transferred to the VM). It is on the coordinator's e2e checklist. Cursor/OpenCode are still not on the image.",
|
||||
"agent.signin": "Auth folders persist across sleep/wake and rollover, but subscription OAuth on a VM was never exercised. This cap is lifted by the user's own e2e session with their own credentials (after their review of #40228). It is on the coordinator's e2e checklist.",
|
||||
"sub.size": "Standard is 4 vCPU / 8 GB; Conductor gives 8 / 16 to everyone. Large (8/16) exists but uses included hours at 2x. Not parity on default hardware."
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
// Summarizes create-bench / wake-split JSON: p50/p90/p95/p99/max per metric and the slow-host share
|
||||
// (CPU model "@ 2.50GHz" vs "@ 2.90GHz", see reports/vercel-resume-tail.md).
|
||||
// tsx src/bench-summary.ts <file.json> [...]
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { percentile } from './run-state'
|
||||
|
||||
for (const file of process.argv.slice(2)) {
|
||||
const d = JSON.parse(readFileSync(file, 'utf8').trim().split('\n').at(-1)!) as {
|
||||
samples: Record<string, number[]>
|
||||
cpus?: string[]
|
||||
hosts?: Array<{ cpu: string; nproc: number; memGb: number; region: string | null }>
|
||||
region?: string
|
||||
machineSize?: string
|
||||
errors?: string[]
|
||||
}
|
||||
console.log(`== ${file} (region ${d.region}, size ${d.machineSize}${d.errors?.length ? `, errors: ${d.errors.join('; ')}` : ''})`)
|
||||
for (const [name, s] of Object.entries(d.samples)) {
|
||||
const f = (p: number) => ((percentile(s, p) ?? 0) / 1000).toFixed(2)
|
||||
console.log(` ${name.padEnd(12)} n=${s.length} p50 ${f(50)} p90 ${f(90)} p95 ${f(95)} p99 ${f(99)} max ${(Math.max(...s) / 1000).toFixed(2)} s`)
|
||||
}
|
||||
const cpus = d.cpus ?? d.hosts?.map((h) => h.cpu) ?? []
|
||||
if (cpus.length) {
|
||||
const slow = cpus.filter((c) => /@ 2\.50GHz/.test(c)).length
|
||||
const fast = cpus.filter((c) => /@ 2\.90GHz/.test(c)).length
|
||||
console.log(` hosts: ${slow}/${cpus.length} slow (@2.50GHz), ${fast}/${cpus.length} fast (@2.90GHz), other ${cpus.length - slow - fast}`)
|
||||
}
|
||||
if (d.hosts?.length) {
|
||||
const sizes = [...new Set(d.hosts.map((h) => `${h.nproc} vCPU / ${h.memGb} GB @ ${h.region}`))]
|
||||
console.log(` sizes: ${sizes.join(', ')}`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
// Records a result reported by another workstream (not run by this harness) as its own run file, so
|
||||
// the report shows its source plainly. Pair it with an auditCap until W9 re-runs it independently.
|
||||
// tsx src/import-external.ts <catalogId> <pass|partial|fail> <source> <detail> [evidencePath...]
|
||||
import { writeFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { config } from './env'
|
||||
import type { RunFile, Status } from './run-state'
|
||||
|
||||
const [id, status, source, detail, ...paths] = process.argv.slice(2) as [string, Status, string, string, ...string[]]
|
||||
const at = new Date().toISOString()
|
||||
const runId = `external-${source}-${at.replace(/[-:]/g, '').replace(/\..+/, '')}`
|
||||
const run: RunFile = {
|
||||
runId,
|
||||
startedAt: at,
|
||||
finishedAt: at,
|
||||
target: { controlPlaneUrl: config.controlPlaneUrl, billingUrl: config.billingUrl },
|
||||
checks: [
|
||||
{
|
||||
id,
|
||||
status,
|
||||
detail: `Reported by ${source} (not run by the W9 harness): ${detail}`,
|
||||
evidence: paths.map((path) => ({ kind: 'log' as const, label: path.split('/').at(-1)!, path })),
|
||||
startedAt: at,
|
||||
durationMs: 0
|
||||
}
|
||||
],
|
||||
benchmarks: [],
|
||||
resources: [],
|
||||
notes: [`external result from ${source}`]
|
||||
}
|
||||
writeFileSync(join(config.runsDir, `${runId}.json`), JSON.stringify(run, null, 2))
|
||||
console.log(runId)
|
||||
Reference in New Issue
Block a user