diff --git a/cloud/apps/acceptance/findings.json b/cloud/apps/acceptance/findings.json index a883892c3a1..738775c3c29 100644 --- a/cloud/apps/acceptance/findings.json +++ b/cloud/apps/acceptance/findings.json @@ -2,6 +2,7 @@ "findings": [ "Wake delivery (target p95 < 6 s) — MET in pdx1: after core U17/U18 (default region pdx1, Standard 8 vCPU/16 GB, golden 63b6999), n=20, persistent connection. Wake to echo p50 5.04 s, p95 5.85 s, p99 5.94 s; 0/20 slow hosts. Pinned to iad1 at 8/16: p50 5.31, p95 10.59, p99 11.80 s, with 4/20 slow hosts. BEFORE (iad1 4/8): 5.28 / 8.99 / 9.40 s. Region, not size, removes the tail. From this Mac the first terminal list (0.18 vs 0.44 s) and echo (0.07 vs 0.15 s) are also faster in pdx1. Create p50: pdx1 2.03 s, iad1-pinned 4.26 s (pinned creates seem to skip the warm pool).", "Read while asleep (gap 5, #189/#218), verified by W9 live: a teammate decrypted a sleeping team workspace (git incl. a new file, PR, sessions) with no wake: the Vercel session stayed stopped. 20 reads p50 89 ms, max 131 ms, vs about 5-6 s to wake. An outsider gets 404, and the plaintext marker is not in D1. Removal/private access control and the phone view are owner-reported (capped).", + "Read-asleep access, triggered by W9 itself: the operator members/removed route revokes the member; their pre-removal token and a fresh token both get 401, and their wrapped keys go 1 to 0, while the other teammate still reads. Making the workspace private gives that teammate 404 and 0 keys. Phone protocol: device token plus mobile reader key reads a sleeping private workspace in 125 ms, and unpairing revokes access.", "Independent audit corrections (2026-11-10, reports/conductor-parity-audit.html): five rows are capped at partial because they passed on a weaker bar than the Conductor column: real-model agent run, subscription sign-in, default machine size, and the download and remote-desktop UIs. Four rows Conductor has that our bar left out are added as not landed: local core workflow, Run actions with per-workspace ports, read-while-asleep from any device, and per-turn code checkpoints. Our 'pass' counts measure the team's own bar. Every cloud result is on dev Workers, Stripe TEST mode and dev/staging tokens; nothing is shipped (mvp-tracker §5 launch items are all open).", "Agents and machine size (2026-11-10): image.agents: Claude Code, Codex, Cursor, OpenCode, Gemini and Copilot all answer --version on a fresh VM. agent.keys-swap: stand-in Gemini/Cursor keys exist on the VM only as placeholders; the provider sees the swapped key (Gemini 400 API_KEY_INVALID, Cursor 401); a foreign key or a disallowed path is refused (403); the stand-in values appear in 0 files and 0 process environments. sub.size cap lifted: Standard is now 8 vCPU / 16 GB (core U18, #194), and machineSize large maps to standard.", "Audit caps lifted by real-UI runs (m4air, 855d8ed34b6, desktop-20261109-234651): acc.download, file-tree Download: 20 MiB file in 2.1 s and a folder as zip in 0.6 s, with real progress and success toasts and matching sha256. Caveat: only the macOS Save panel answer was stubbed in main, because CDP cannot drive a native panel. acc.remote-desktop: Open remote desktop tab connected in 3.4 s; noVNC canvas streams non-blank pixels, and a red xterm launched in the VM appeared on the canvas. local.core-workflow passes (local worktree in 0.6 s, terminal edit, diff in Source Control, stage, commit).", @@ -28,7 +29,6 @@ "agent.real-task": "Proven only with moonshotai/kimi-k2 through the Claude Code CLI. Per the user decision (2026-11-10) there are no AI Gateway real-model runs; lifted by the user's own e2e session with their own credentials, and on the coordinator's e2e checklist. Image now also has Cursor, OpenCode, Gemini and Copilot (image.agents); W9 has shown Cursor and Gemini launch with their keys swapped at egress (agent.keys-swap), but no real-model run for those two.", "agent.signin": "Auth folders persist across sleep/wake and rollover, but subscription OAuth on a VM was never exercised. This cap is lifted by the user's own e2e session with their own credentials (after their review of #40228). It is on the coordinator's e2e checklist.", "agent.more-agents": "Owner-reported (gap-agents) real-task runs on kimi-k2 via the gateway. W9 is not re-running them (no gateway spend per user decision). Lifted by the user e2e session with their own credentials.", - "life.read-asleep-access": "Owner-reported (gap-read-asleep). W9 has not re-run the member-removal push or the private prune, which need the auth membership push. W9 did verify independently that an outsider gets 404 (life.read-asleep).", - "life.read-asleep-phone": "Owner-reported, headless phone simulator. W9 has not re-run the mobile UI." + "life.read-asleep-phone": "Phone protocol verified by W9 (device token, mobile reader key, decrypt, no wake, revoked on unpair). The phone UI rendering (\"Latest from this workspace\") is still owner-reported (gap5 simulator screenshots); a W9 simulator UI run is pending." } } \ No newline at end of file diff --git a/cloud/apps/acceptance/src/checks/platform.ts b/cloud/apps/acceptance/src/checks/platform.ts index 95e4ed59fce..1779c162630 100644 --- a/cloud/apps/acceptance/src/checks/platform.ts +++ b/cloud/apps/acceptance/src/checks/platform.ts @@ -422,6 +422,107 @@ export const platformChecks: Record = { } } }, + + 'life.read-asleep-access': async (ctx) => { + // Access control for reading a sleeping workspace, triggered by W9 itself on dev: the operator + // member-removed route (what auth calls) revokes a member at once; making the workspace private + // prunes the other teammate. Checks the API answer and the wrapped keys left in D1. + const tag = ctx.run.file.runId.replace(/[^0-9]/g, '').slice(-8) + const org = `w9-rm-${tag}` + const owner = { sub: `w9-rm-owner-${tag}`, org, orgRole: 'owner' as const } + const m1 = { sub: `w9-rm-m1-${tag}`, org, orgRole: 'member' as const } + const m2 = { sub: `w9-rm-m2-${tag}`, org, orgRole: 'member' as const } + const k1 = x25519Pair() + const k2 = x25519Pair() + const reg = async (who: typeof m1, k: ReturnType) => { + const r = await control.as(who, 'PUT', '/v1/reader-keys', { publicKeyB64: k.publicRaw.toString('base64'), deviceLabel: 'w9', deviceKind: 'desktop' }) + return (r.body?.key?.id ?? r.body?.readerKey?.id ?? r.body?.id) as string + } + const id1 = await reg(m1, k1) + const id2 = await reg(m2, k2) + const box = await ctx.sandboxes.create(owner, { workspaceId: `w9-rm-${tag}`, title: 'W9 access', visibility: 'team', useEnvironment: false }) + try { + if (box.status !== 200 || !id1 || !id2) { + return { status: 'fail', detail: `setup: create ${box.status}, keys ${id1}/${id2}` } + } + const id = box.body.sandbox.id + await ctx.sandboxes.suspend(id) + const read = async (who: typeof m1, keyId: string, token?: string) => + token ? control.raw('GET', `/v1/team/workspaces/${id}/read-snapshot?keyId=${keyId}`, token) : control.as(who, 'GET', `/v1/team/workspaces/${id}/read-snapshot?keyId=${keyId}`) + const wrappedFor = (keyId: string) => Number(d1Control(`SELECT COUNT(*) AS n FROM read_snapshot_keys WHERE key_id = '${keyId}'`)[0]?.n ?? 0) + const before1 = (await read(m1, id1)).status + const before2 = (await read(m2, id2)).status + const m1Token = await sandboxToken(m1) // minted before the removal, like a device's live token + const wrapped1Before = wrappedFor(id1) + const removed = await control.operator('POST', '/v1/internal/members/removed', { accountId: `dev:org:${org}`, userId: m1.sub }) + const after1Old = (await read(m1, id1, m1Token)).status + const after1New = (await read(m1, id1)).status + const wrapped1After = wrappedFor(id1) + const after2 = (await read(m2, id2)).status + // Private: only the owner may read; the remaining teammate is pruned at once. + const priv = await control.as(owner, 'PATCH', `/v1/team/workspaces/${id}`, { visibility: 'private' }) + const after2Private = (await read(m2, id2)).status + const wrapped2After = wrappedFor(id2) + const ok = + before1 === 200 && before2 === 200 && wrapped1Before > 0 && removed.status === 200 && after1Old === 401 && after1New >= 400 && + wrapped1After === 0 && after2 === 200 && priv.status === 200 && after2Private >= 400 && wrapped2After === 0 + return { + status: ok ? 'pass' : 'fail', + detail: `team workspace asleep: m1 ${before1}, m2 ${before2}; operator members/removed(m1) -> ${removed.status} ${JSON.stringify(removed.body).slice(0, 80)}; m1 with pre-removal token -> ${after1Old}, fresh token -> ${after1New}; m1 wrapped keys ${wrapped1Before} -> ${wrapped1After}; m2 still reads -> ${after2}; made private (${priv.status}) -> m2 ${after2Private}, m2 wrapped keys -> ${wrapped2After}` + } + } finally { + await control.as(m1, 'DELETE', `/v1/reader-keys/${id1}`).catch(() => null) + await control.as(m2, 'DELETE', `/v1/reader-keys/${id2}`).catch(() => null) + if (box.status === 200) { + await ctx.sandboxes.destroy(box.body.sandbox.id) + } + } + }, + + 'life.read-asleep-phone': async (ctx) => { + // The phone's path, headless: a device session (Ed25519, like the app enrolls), a mobile reader + // key registered WITH the device token, then the sleeping workspace's snapshot fetched with the + // device token and decrypted. The phone UI rendering is a separate simulator run. + const tag = ctx.run.file.runId.replace(/[^0-9]/g, '').slice(-8) + const { publicKey, privateKey } = generateKeyPairSync('ed25519') + const pub = Buffer.from(publicKey.export({ format: 'jwk' }).x!, 'base64') + const enroll = await control.as(ctx.owner, 'POST', '/v1/device-sessions', { label: 'w9 phone', platform: 'ios', publicKeyB64: pub.toString('base64') }) + const ds = enroll.body?.deviceSession?.id + if (!ds) { + return { status: 'fail', detail: `enroll ${enroll.status} ${JSON.stringify(enroll.body).slice(0, 160)}` } + } + const deviceToken = async () => + new SignJWT({}).setProtectedHeader({ alg: 'EdDSA', kid: ds }).setIssuer('orca-device').setSubject(ds).setAudience('orca-sandbox').setIssuedAt().setExpirationTime('4m').sign(createPrivateKey(privateKey.export({ format: 'pem', type: 'pkcs8' }))) + const pair = x25519Pair() + const reg = await control.raw('PUT', '/v1/reader-keys', await deviceToken(), { publicKeyB64: pair.publicRaw.toString('base64'), deviceLabel: 'w9 phone', deviceKind: 'mobile' }) + const keyId = (reg.body?.key?.id ?? reg.body?.readerKey?.id ?? reg.body?.id) as string | undefined + const box = await ctx.sandboxes.create(ctx.owner, { workspaceId: `w9-phone-${tag}`, title: 'W9 phone read', visibility: 'private', useEnvironment: false }) + try { + if (!keyId || box.status !== 200) { + return { status: 'fail', detail: `reader key ${reg.status} ${JSON.stringify(reg.body).slice(0, 120)}; create ${box.status}` } + } + const id = box.body.sandbox.id + await ctx.sandboxes.suspend(id) + const t0 = performance.now() + const r = await control.raw('GET', `/v1/team/workspaces/${id}/read-snapshot?keyId=${keyId}`, await deviceToken()) + const payload = r.body?.snapshot ? openReadSnapshot(r.body.snapshot, pair) : null + const ms = Math.round(performance.now() - t0) + const after = (await vercelRest('GET', `/v2/sandboxes/orca-dev-${id}`, { projectId: config.vercelProjectId })).json + const stopped = !['running', 'pending'].includes(after?.session?.status ?? after?.sandbox?.status ?? '') + await control.as(ctx.owner, 'DELETE', `/v1/device-sessions/${ds}`) + const revoked = await control.raw('GET', `/v1/team/workspaces/${id}/read-snapshot?keyId=${keyId}`, await deviceToken()) + const ok = reg.status < 300 && payload?.workspaceId === id && stopped && revoked.status >= 400 + return { + status: ok ? 'pass' : 'fail', + detail: `phone path headless: device session ${enroll.status}; mobile reader key registered with the device token -> ${reg.status}; private sleeping workspace read with the device token + decrypted in ${ms} ms (sections ${Object.keys(payload ?? {}).join(', ')}); no wake=${stopped}; after unpairing the phone -> ${revoked.status}. The phone UI ("Latest from this workspace") is the simulator run.`, + timingsMs: { phoneRead: ms } + } + } finally { + if (box.status === 200) { + await ctx.sandboxes.destroy(box.body.sandbox.id) + } + } + }, // ---------------- W5 ---------------- 'api.rest': async (ctx) => { const created = await api(ctx, 'POST', '/v1/workspaces', { name: `w9-api-${tag(ctx)}`, repo: { url: config.testRepo } })