From 9caeede5300c08ba0aa352cc862d598a644c528f Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Fri, 14 Aug 2026 16:43:34 -0400 Subject: [PATCH] test(orchestration): pin STA-4325 delivery identity --- config/reliability-gates.jsonc | 34 +- ...stration-message-delivery-identity.test.ts | 400 ++++++++++++++++++ 2 files changed, 429 insertions(+), 5 deletions(-) create mode 100644 src/main/runtime/orchestration-message-delivery-identity.test.ts diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 1df65b955b5..8c29c88336f 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -8805,15 +8805,20 @@ "providers": ["local", "daemon", "ssh", "wsl", "remote-runtime"], "coveredPlatforms": ["macos"], "coveredProviders": ["local", "daemon", "ssh"], - "coverageNotes": "A deterministic in-process runtime/SQLite/RPC harness covers two remint-equivalent panes across Run and Dispatch ownership changes, captures mocked PTY-controller writes, executes the production check, reopens durable production-shape state under a fresh runtime, and verifies Run Delivery acknowledgment plus worker-check consumption. Controlled daemon-socket and SSH-multiplexer settlements prove a rejected pointer write stays undelivered and restart-retryable before durable staging; a fake clock proves a wedged daemon write disconnects at its bounded deadline. A 151-row backlog proves live-idle reconciliation and inactive-Dispatch transfer stay within 50-row pages and cooperative event-loop turns; explicit checks cooperatively drain their start-of-check sequence snapshot before filtering or waiting. Controlled setImmediate races prove Run/Dispatch consumers are fenced after ownership changes, split current/stored handles both drain, cancellation stops direct-mail paging, and an inactive-Dispatch cancellation notifies partial progress before a background continuation finishes only the captured snapshot. The authority decision adds no wire field, renderer state, polling, or provider call. Paired remote-runtime, live SSH, WSL, Linux, Windows, and headed restart evidence remain gaps.", - "motivatingLinks": ["internal incident evidence: improve-vps-setup, 2026-08-10"], + "coverageNotes": "A deterministic in-process runtime/SQLite/RPC harness covers two remint-equivalent panes across Run and Dispatch ownership changes, captures mocked PTY-controller writes, executes the production check, reopens durable production-shape state under a fresh runtime, and verifies Run Delivery acknowledgment plus worker-check consumption. The STA-4325 oracle separately fixes one status, dispatch, and worker_done message across direct-handle and canonical Run recipients, then proves notification count, concurrent Delivery identity, unread/peek/all/type filters, explicit acknowledgment, restart replay, and filtered-waiter wakeup against the same isolated SQLite rows. Controlled daemon-socket and SSH-multiplexer settlements prove a rejected pointer write stays undelivered and restart-retryable before durable staging; a fake clock proves a wedged daemon write disconnects at its bounded deadline. A 151-row backlog proves live-idle reconciliation and inactive-Dispatch transfer stay within 50-row pages and cooperative event-loop turns; explicit checks cooperatively drain their start-of-check sequence snapshot before filtering or waiting. Controlled setImmediate races prove Run/Dispatch consumers are fenced after ownership changes, split current/stored handles both drain, cancellation stops direct-mail paging, and an inactive-Dispatch cancellation notifies partial progress before a background continuation finishes only the captured snapshot. The authority decision adds no wire field, renderer state, polling, or provider call. Paired remote-runtime, live SSH, WSL, Linux, Windows, and headed restart evidence remain gaps.", + "motivatingLinks": [ + "https://linear.app/stably/issue/STA-4325", + "https://github.com/stablyai/orca/issues/14522", + "internal incident evidence: improve-vps-setup, 2026-08-10" + ], "invariant": "Every injected orchestration pointer must select the same mailbox as the exact check from that pane and expose at least the notified unread messages through that mailbox's acknowledgment semantics. Rebinding cannot let foreign direct mail wake a new owner, newer Run mail cannot be announced behind an outstanding fixed Delivery, and restart cannot replay pointer text already accepted by the PTY even if delayed Enter has not fired. Notification records only a durable pointer cursor; it never marks mail read or globally clears it. Direct and Dispatch mail without a mixed-version-compatible pinned consuming action must not synthesize a terminal turn.", "oracle": "Create Run A with unread current-delivery mail addressed directly to its coordinator handle, bind the same pane to Run B, drive a controlled working-to-idle event, and require zero PTY pointers while Run B check returns zero and Run A mail stays unread. Reopen a database already containing that mismatch and require live-idle reconciliation to make the row actionable to Run A without pointing Run B. For same-Run mail, require one Run-pinned pointer, a bound Delivery containing that row, explicit acknowledgment, and no replay after a fresh runtime, including restart between accepted pointer payload and delayed Enter. Require direct and Dispatch mail to remain explicitly checkable without an unpinned pointer, reminted active-worker mail to retain Dispatch ownership, newer Run mail to stay suppressed behind an outstanding fixed Delivery, waiter reservations to remain single-consumer across routing, absent local PTYs to retain mail, and a 151-row mismatch to reconcile in 50-row pages with one continuation per turn. During explicit paging, deterministically rebind the Run or complete the Dispatch and require a fenced response; require the inactive Dispatch's coordinator to consume and acknowledge the preserved rows through four fixed Run Deliveries while migration remains 50-row paged; split current and stored handles and require one check to route both; add mail beyond the captured sequence and require finite snapshot completion; abort between direct-mail pages and require paging to stop. Abort an inactive-Dispatch migration after its first page, require one partial wake and bounded background completion of the captured rows, and require a later row to remain outside that snapshot. Hold a daemon socket settlement past its fake-clock deadline and require disconnect plus a retryable undelivered pointer.", "commands": [ - "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orchestration-mailbox-routing-races.test.ts src/main/runtime/orchestration-mailbox-notification-consistency.test.ts src/main/runtime/orchestration-mailbox-transport-settlement.test.ts src/main/runtime/orchestration/orchestration-run-delivery-db.test.ts src/main/runtime/orchestration/orchestration-worker-dispatch-db.test.ts src/main/runtime/orchestration/formatter.test.ts src/main/providers/ssh-pty-provider.test.ts src/main/providers/ssh-pty-write.test.ts src/main/daemon/client.test.ts src/main/daemon/daemon-pty-router.test.ts src/main/daemon/degraded-daemon-pty-provider.test.ts", + "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orchestration-message-delivery-identity.test.ts src/main/runtime/orchestration-mailbox-routing-races.test.ts src/main/runtime/orchestration-mailbox-notification-consistency.test.ts src/main/runtime/orchestration-mailbox-transport-settlement.test.ts src/main/runtime/orchestration/orchestration-run-delivery-db.test.ts src/main/runtime/orchestration/orchestration-worker-dispatch-db.test.ts src/main/runtime/orchestration/formatter.test.ts src/main/providers/ssh-pty-provider.test.ts src/main/providers/ssh-pty-write.test.ts src/main/daemon/client.test.ts src/main/daemon/daemon-pty-router.test.ts src/main/daemon/degraded-daemon-pty-provider.test.ts", "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orca-runtime.test.ts src/main/runtime/terminal-send-stale-leaf-liveness.test.ts src/main/runtime/rpc/methods/orchestration.test.ts" ], "testFiles": [ + "src/main/runtime/orchestration-message-delivery-identity.test.ts", "src/main/runtime/orchestration-mailbox-notification-consistency.test.ts", "src/main/runtime/orchestration-mailbox-routing-races.test.ts", "src/main/runtime/orchestration-mailbox-transport-settlement.test.ts", @@ -8830,6 +8835,16 @@ "src/main/runtime/rpc/methods/orchestration.test.ts" ], "assertionRefs": [ + { + "file": "src/main/runtime/orchestration-message-delivery-identity.test.ts", + "assertions": [ + "status, dispatch, and worker_done rows keep stable message IDs across handle-to-Run routing", + "notification count equals the concurrently replayed fixed Delivery batch", + "unread, peek, all, and type filters preserve explicit acknowledgment semantics", + "restart replays one outstanding Delivery ID without inserting another Delivery row", + "a filtered check --wait wakes once without a duplicate terminal pointer" + ] + }, { "file": "src/main/runtime/orchestration-mailbox-notification-consistency.test.ts", "assertions": [ @@ -8897,7 +8912,16 @@ "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orchestration-mailbox-routing-races.test.ts src/main/runtime/orchestration-mailbox-notification-consistency.test.ts src/main/runtime/orchestration-mailbox-transport-settlement.test.ts src/main/runtime/orchestration/orchestration-run-delivery-db.test.ts src/main/runtime/orchestration/orchestration-worker-dispatch-db.test.ts src/main/runtime/orchestration/formatter.test.ts src/main/providers/ssh-pty-provider.test.ts src/main/providers/ssh-pty-write.test.ts src/main/daemon/client.test.ts src/main/daemon/daemon-pty-router.test.ts src/main/daemon/degraded-daemon-pty-provider.test.ts", "result": "passed", "durationSeconds": 6.86, - "summary": "228 tests passed across mailbox identity, direct and Dispatch pointer suppression, persisted reconciliation, 50-row paging and filtered waits, cross-PTY serialization, live ownership transfer, restart before delayed Enter, submit races, bounded daemon and SSH transport settlement, outstanding Deliveries, reminted Dispatch ownership, acknowledgment, remote pane remints, split-handle drains, ownership-change fencing, paged inactive-mailbox reachability, cancellation continuation, snapshot termination, reservation ordering, and bounded pane lookup. The original 10-scenario mailbox oracle failed 10/10 on origin/main plus the test commit and again with the implementation commit explicitly reverted, including pointerCount > 0 with bound check count 0." + "summary": "228 tests passed across mailbox identity, direct and Dispatch pointer suppression, persisted reconciliation, 50-row paging and filtered waits, cross-PTY serialization, live ownership transfer, restart before delayed Enter, submit races, bounded daemon and SSH transport settlement, outstanding Deliveries, reminted Dispatch ownership, acknowledgment, remote pane remints, split-handle drains, ownership-change fencing, paged inactive-mailbox reachability, cancellation continuation, snapshot termination, reservation ordering, and bounded pane lookup." + }, + { + "date": "2026-08-14", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/orchestration-message-delivery-identity.test.ts --reporter=dot --testTimeout=5000", + "result": "passed", + "durationSeconds": 5.58, + "summary": "The byte-identical two-case STA-4325 oracle failed 2/2 on reported v1.4.180 and latest origin/main 9bb8836bb6, passed 2/2 on the rebased candidate, and failed 2/2 with all candidate production changes disabled. The red path announced two direct rows while the bound check exposed only one canonical row; the second direct-only row was invisible to check before and after restart. The green path returned all three fixed message IDs in one Delivery, replayed that Delivery ID under concurrent pollers and restart, acknowledged only that batch, and woke one filtered worker_done waiter." }, { "date": "2026-08-13", @@ -8919,7 +8943,7 @@ }, "redGreenEvidence": { "status": "complete", - "evidence": "The identical original 10-scenario oracle failed 10/10 on origin/main plus the oracle commit, passed 10/10 on the candidate, failed 10/10 after explicitly reverting the implementation commit, and passed again after restoration. Added restart-upgrade, reminted-Dispatch, pointer-pinning, refusal, and bounded-backlog scenarios are candidate regressions; live topology coverage remains a separate promotion gap." + "evidence": "The byte-identical STA-4325 identity oracle failed 2/2 on v1.4.180 and latest origin/main 9bb8836bb6, passed 2/2 on the candidate, and failed 2/2 with the candidate production changes disabled. Added restart-upgrade, reminted-Dispatch, pointer-pinning, refusal, and bounded-backlog scenarios are candidate regressions; live topology coverage remains a separate promotion gap." }, "performanceBudget": { "required": true, diff --git a/src/main/runtime/orchestration-message-delivery-identity.test.ts b/src/main/runtime/orchestration-message-delivery-identity.test.ts new file mode 100644 index 00000000000..cb96071addc --- /dev/null +++ b/src/main/runtime/orchestration-message-delivery-identity.test.ts @@ -0,0 +1,400 @@ +import { spawn } from 'node:child_process' +import { existsSync, mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi } from 'vitest' +import { ORCHESTRATION_CONTRACT_VERSION } from '../../shared/protocol-version' +import { OrcaRuntimeService } from './orca-runtime' +import { OrchestrationDb } from './orchestration/db' +import { RpcDispatcher } from './rpc/dispatcher' +import { ORCHESTRATION_METHODS } from './rpc/methods/orchestration' +import { OrcaRuntimeRpcServer } from './runtime-rpc' + +vi.mock('electron', () => ({ + app: { getPath: vi.fn(() => tmpdir()), isPackaged: false }, + BrowserWindow: { fromId: vi.fn(() => null) }, + ipcMain: { on: vi.fn(), removeListener: vi.fn() }, + webContents: { fromId: vi.fn(() => null) } +})) + +const TAB_ID = '11111111-1111-4111-8111-111111111111' +const LEAF_ID = '22222222-2222-4222-8222-222222222222' +const PANE_KEY = `${TAB_ID}:${LEAF_ID}` +const PTY_ID = 'pty-sta-4325' +const TERMINAL_HANDLE = 'term_sta_4325' +const WORKTREE_ID = 'repo-sta-4325::/tmp/sta-4325' +const LAUNCH_TOKEN = 'sta-4325-launch' +const temporaryDirectories: string[] = [] +const CLI_PATH = join(process.cwd(), 'out', 'cli', 'index.js') + +type MessageResult = { id: string; type: string; read: number } +type CheckResult = { + runId: string + deliveryId: string | null + messages: MessageResult[] + count: number + replayed?: boolean + acknowledged?: string | null +} + +type Sqlite = { + prepare: (sql: string) => { + all: (...params: unknown[]) => unknown[] + } +} + +function sqliteFor(db: OrchestrationDb): Sqlite { + return (db as unknown as { db: Sqlite }).db +} + +function createDatabase(prefix: string): { db: OrchestrationDb; path: string } { + const directory = mkdtempSync(join(tmpdir(), prefix)) + temporaryDirectories.push(directory) + const path = join(directory, 'orchestration.db') + return { db: new OrchestrationDb(path), path } +} + +function createRuntime(db: OrchestrationDb): { + runtime: OrcaRuntimeService + write: ReturnType +} { + const runtime = new OrcaRuntimeService(null, undefined, { + attestAgentHookCompatibilityAuthority: ({ paneKey }) => + paneKey === PANE_KEY ? { paneKey, source: 'current_hook' } : null + }) + const write = vi.fn(() => true) + runtime.setOrchestrationDb(db) + runtime.setPtyController({ write, kill: vi.fn(), getForegroundProcess: async () => null }) + runtime.registerPty(PTY_ID, WORKTREE_ID, null, { + tabId: TAB_ID, + leafId: LEAF_ID, + incarnationId: 'sta-4325-incarnation', + agentLaunchAuthority: { launchToken: LAUNCH_TOKEN, launchAgent: 'codex' } + }) + runtime.registerPreAllocatedHandleForPty(PTY_ID, TERMINAL_HANDLE) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { + tabs: [ + { + tabId: TAB_ID, + worktreeId: WORKTREE_ID, + title: 'Codex', + activeLeafId: LEAF_ID, + layout: null + } + ], + leaves: [ + { + tabId: TAB_ID, + worktreeId: WORKTREE_ID, + leafId: LEAF_ID, + paneRuntimeId: 1, + ptyId: PTY_ID + } + ] + }) + return { runtime, write } +} + +async function driveToLiveIdle(runtime: OrcaRuntimeService): Promise { + await runtime.listTerminals() + runtime.onPtyData(PTY_ID, '\x1b]0;Codex working\x07', 1) + runtime.onPtyData(PTY_ID, '\x1b]0;Codex done\x07', 2) + await Promise.resolve() +} + +async function check( + runtime: OrcaRuntimeService, + params: Record = {} +): Promise { + const response = await new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }).dispatch({ + id: `req-sta-4325-${Math.random()}`, + authToken: 'test-auth-token', + method: 'orchestration.check', + orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION, + orchestrationCompatibilityEvidence: { + terminalHandle: TERMINAL_HANDLE, + paneKey: PANE_KEY, + launchToken: LAUNCH_TOKEN + }, + params: { terminal: TERMINAL_HANDLE, ...params } + }) + expect(response.ok).toBe(true) + if (!response.ok) { + throw new Error(response.error.message) + } + return response.result as CheckResult +} + +function pointerPayloads(write: ReturnType): string[] { + return write.mock.calls + .map(([, payload]) => String(payload)) + .filter((payload) => payload.includes('orca orchestration check')) +} + +async function runBuiltCli( + userDataPath: string, + args: string[] +): Promise<{ exitCode: number; stdout: string; stderr: string }> { + const child = spawn(process.execPath, [CLI_PATH, ...args], { + env: { + ...process.env, + ORCA_USER_DATA_PATH: userDataPath, + ORCA_TERMINAL_HANDLE: TERMINAL_HANDLE, + ORCA_PANE_KEY: PANE_KEY + }, + stdio: ['ignore', 'pipe', 'pipe'] + }) + const stdout: string[] = [] + const stderr: string[] = [] + child.stdout.setEncoding('utf8') + child.stderr.setEncoding('utf8') + child.stdout.on('data', (chunk) => stdout.push(chunk)) + child.stderr.on('data', (chunk) => stderr.push(chunk)) + const exitCode = await new Promise((resolve, reject) => { + child.once('exit', (code) => resolve(code ?? 1)) + child.once('error', reject) + }) + return { exitCode, stdout: stdout.join(''), stderr: stderr.join('') } +} + +describe('STA-4325 message and delivery identity', () => { + afterEach(() => { + vi.useRealTimers() + for (const directory of temporaryDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }) + } + }) + + it('keeps pointer counts, filters, message IDs, and one fixed Delivery aligned through ack', async () => { + vi.useFakeTimers() + const { db } = createDatabase('orca-sta-4325-identity-') + const { runtime, write } = createRuntime(db) + const run = db.createRun({ + objective: 'STA-4325 identity', + coordinatorHandle: TERMINAL_HANDLE, + coordinatorPaneKey: PANE_KEY + }) + const status = db.insertMessage({ + from: 'term_worker_a', + to: TERMINAL_HANDLE, + subject: 'stale status', + type: 'status', + runId: run.id, + deliveryContract: 'current_delivery' + }) + const dispatch = db.insertMessage({ + from: 'term_worker_b', + to: `run:${run.id}`, + subject: 'dispatch receipt', + type: 'dispatch', + runId: run.id, + deliveryContract: 'current_delivery' + }) + const done = db.insertMessage({ + from: 'term_worker_c', + to: TERMINAL_HANDLE, + subject: 'worker complete', + type: 'worker_done', + runId: run.id, + deliveryContract: 'current_delivery' + }) + + await driveToLiveIdle(runtime) + const [first, concurrent] = await Promise.all([check(runtime), check(runtime)]) + const expectedIds = [status.id, dispatch.id, done.id] + + expect(pointerPayloads(write)).toEqual([expect.stringContaining('3 orchestration messages')]) + expect(first).toMatchObject({ runId: run.id, count: 3, replayed: false }) + expect(concurrent).toMatchObject({ runId: run.id, count: 3, replayed: true }) + expect(first.deliveryId).toBeTruthy() + expect(concurrent.deliveryId).toBe(first.deliveryId) + expect(first.messages.map((message) => message.id)).toEqual(expectedIds) + expect(concurrent.messages.map((message) => message.id)).toEqual(expectedIds) + + const peeked = await check(runtime, { peek: true, unread: false }) + const filtered = await check(runtime, { peek: true, unread: false, types: 'worker_done' }) + const all = await check(runtime, { all: true }) + expect(peeked.messages.map((message) => message.id)).toEqual( + expect.arrayContaining(expectedIds) + ) + expect(peeked.count).toBe(3) + expect(filtered.messages.map((message) => message.id)).toEqual([done.id]) + expect(all.messages.map((message) => message.id)).toEqual(expect.arrayContaining(expectedIds)) + expect(all.count).toBe(3) + + const deliveryRowsBeforeAck = sqliteFor(db) + .prepare('SELECT id, status, message_ids FROM deliveries ORDER BY rowid') + .all() as { id: string; status: string; message_ids: string }[] + expect(deliveryRowsBeforeAck).toEqual([ + { id: first.deliveryId, status: 'outstanding', message_ids: JSON.stringify(expectedIds) } + ]) + for (const id of expectedIds) { + expect(db.getMessageById(id)).toMatchObject({ to_handle: `run:${run.id}`, read: 0 }) + } + + const acknowledged = await check(runtime, { ack: first.deliveryId }) + expect(acknowledged).toMatchObject({ + count: 0, + deliveryId: null, + acknowledged: first.deliveryId + }) + expect((await check(runtime, { unread: true })).count).toBe(0) + const acknowledgedHistory = await check(runtime, { all: true }) + expect(acknowledgedHistory.messages.map((message) => message.id)).toEqual( + expect.arrayContaining(expectedIds) + ) + expect(acknowledgedHistory.count).toBe(3) + expect(sqliteFor(db).prepare('SELECT id, status FROM deliveries ORDER BY rowid').all()).toEqual( + [{ id: first.deliveryId, status: 'acknowledged' }] + ) + for (const id of expectedIds) { + expect(db.getMessageById(id)?.read).toBe(1) + } + db.close() + }) + + it('replays one outstanding Delivery across restart and wakes a filtered waiter once', async () => { + vi.useFakeTimers() + const fixture = createDatabase('orca-sta-4325-restart-') + const firstRuntime = createRuntime(fixture.db) + const run = fixture.db.createRun({ + objective: 'STA-4325 restart', + coordinatorHandle: TERMINAL_HANDLE, + coordinatorPaneKey: PANE_KEY + }) + const status = fixture.db.insertMessage({ + from: 'term_worker', + to: TERMINAL_HANDLE, + subject: 'survive restart', + type: 'status', + runId: run.id, + deliveryContract: 'current_delivery' + }) + await driveToLiveIdle(firstRuntime.runtime) + const beforeRestart = await check(firstRuntime.runtime) + expect(beforeRestart.messages.map((message) => message.id)).toEqual([status.id]) + fixture.db.close() + + const reopened = new OrchestrationDb(fixture.path) + const restarted = createRuntime(reopened) + await driveToLiveIdle(restarted.runtime) + const afterRestart = await check(restarted.runtime) + expect(afterRestart).toMatchObject({ deliveryId: beforeRestart.deliveryId, replayed: true }) + expect(afterRestart.messages.map((message) => message.id)).toEqual([status.id]) + expect(sqliteFor(reopened).prepare('SELECT id FROM deliveries').all()).toEqual([ + { id: beforeRestart.deliveryId } + ]) + + await check(restarted.runtime, { ack: afterRestart.deliveryId }) + const waiting = check(restarted.runtime, { wait: true, types: 'worker_done', timeoutMs: 5_000 }) + const internals = restarted.runtime as unknown as { + messageWaitersByHandle: Map> + } + for (let attempt = 0; attempt < 20; attempt += 1) { + if (internals.messageWaitersByHandle.has(`run:${run.id}`)) { + break + } + await Promise.resolve() + } + expect(internals.messageWaitersByHandle.has(`run:${run.id}`)).toBe(true) + + const done = reopened.insertMessage({ + from: 'term_worker', + to: TERMINAL_HANDLE, + subject: 'done after restart', + type: 'worker_done', + runId: run.id, + deliveryContract: 'current_delivery' + }) + restarted.runtime.notifyMessageArrived(TERMINAL_HANDLE, done.type) + await vi.advanceTimersByTimeAsync(5_000) + + await expect(waiting).resolves.toMatchObject({ + runId: run.id, + count: 1, + messages: [expect.objectContaining({ id: done.id })] + }) + expect(pointerPayloads(restarted.write)).toHaveLength(0) + reopened.close() + }) + + it('keeps the built CLI count and Delivery output aligned with isolated SQLite state', async () => { + expect(existsSync(CLI_PATH)).toBe(true) + const userDataPath = mkdtempSync(join(tmpdir(), 'orca-sta-4325-cli-')) + temporaryDirectories.push(userDataPath) + const db = new OrchestrationDb(join(userDataPath, 'orchestration.db')) + const runtime = new OrcaRuntimeService() + runtime.setOrchestrationDb(db) + vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => + handle === TERMINAL_HANDLE ? PANE_KEY : null + ) + const run = db.createRun({ + objective: 'STA-4325 built CLI', + coordinatorHandle: TERMINAL_HANDLE, + coordinatorPaneKey: PANE_KEY + }) + const status = db.insertMessage({ + from: 'term_worker', + to: TERMINAL_HANDLE, + subject: 'direct status', + type: 'status', + runId: run.id, + deliveryContract: 'current_delivery' + }) + const done = db.insertMessage({ + from: 'term_worker', + to: `run:${run.id}`, + subject: 'canonical done', + type: 'worker_done', + runId: run.id, + deliveryContract: 'current_delivery' + }) + const server = new OrcaRuntimeRpcServer({ runtime, userDataPath }) + await server.start() + + try { + const first = await runBuiltCli(userDataPath, ['orchestration', 'check', '--json']) + expect(first.exitCode, first.stderr).toBe(0) + const firstPayload = JSON.parse(first.stdout) as { result: CheckResult } + expect(firstPayload.result).toMatchObject({ runId: run.id, count: 2, replayed: false }) + expect(firstPayload.result.messages.map((message) => message.id)).toEqual([ + status.id, + done.id + ]) + + const replay = await runBuiltCli(userDataPath, ['orchestration', 'check', '--json']) + expect(replay.exitCode, replay.stderr).toBe(0) + const replayPayload = JSON.parse(replay.stdout) as { result: CheckResult } + expect(replayPayload.result).toMatchObject({ + count: 2, + deliveryId: firstPayload.result.deliveryId, + replayed: true + }) + expect(sqliteFor(db).prepare('SELECT id, status FROM deliveries').all()).toEqual([ + { id: firstPayload.result.deliveryId, status: 'outstanding' } + ]) + expect(db.getMessageById(status.id)).toMatchObject({ + to_handle: `run:${run.id}`, + read: 0 + }) + + const acknowledged = await runBuiltCli(userDataPath, [ + 'orchestration', + 'check', + '--ack', + firstPayload.result.deliveryId!, + '--json' + ]) + expect(acknowledged.exitCode, acknowledged.stderr).toBe(0) + expect(JSON.parse(acknowledged.stdout)).toMatchObject({ + result: { count: 0, deliveryId: null, acknowledged: firstPayload.result.deliveryId } + }) + expect(db.getMessageById(status.id)?.read).toBe(1) + expect(db.getMessageById(done.id)?.read).toBe(1) + } finally { + await server.stop() + db.close() + } + }, 30_000) +})