test: cut ~70s of CI test time from agent-hooks helper and CLI restart-handoff suites (#39679)

* test(agent-hooks): drive Claude helper-kill transcript polls on a fake clock

The 1s transcript poll deadline made each waitFor sleep ~1s of real time.
Fake setTimeout/performance and advance the clock while file I/O stays real.

* test(orchestration): spawn the bundled CLI in the restart handoff test

Each of ~25 CLI subprocesses paid ~1s for a tsx loader start. Bundle the real
CLI once with the orcad CLI build and run it under plain Node.

---------

Co-authored-by: OrcaWin <293788423+OrcaWin@users.noreply.github.com>
This commit is contained in:
OrcaWin
2026-11-07 18:58:42 -08:00
committed by GitHub
co-authored by OrcaWin
parent 6757dd6758
commit 845eddc84e
3 changed files with 73 additions and 111 deletions
@@ -1,7 +1,7 @@
import { appendFile, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, expect, it, vi } from 'vitest'
import { afterEach, beforeEach, expect, it, vi } from 'vitest'
import { RelayAgentHookServer } from '../../wsl-guest/agent-hook-server'
import { selectFreshExplicitAgentStatus } from '../runtime/workspaces/process-listing/runtime-hook-agent-row-selection'
import { selectRuntimeHookAgentRowForPane } from '../runtime/session-lifecycle/projection/runtime-mobile-agent-status-projection'
@@ -15,15 +15,28 @@ vi.mock('../telemetry/cohort-classifier', () => ({
}))
const servers: { stop: () => void }[] = []
const folders: string[] = []
beforeEach(() => {
vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] })
})
afterEach(async () => {
for (const server of servers.splice(0)) {
server.stop()
}
vi.useRealTimers()
for (const folder of folders.splice(0)) {
await rm(folder, { recursive: true, force: true })
}
vi.restoreAllMocks()
})
// Transcript polls run on the fake clock while transcript I/O stays real.
const settle = (assertion: () => void) =>
vi.waitFor(
() => {
vi.advanceTimersByTime(250)
assertion()
},
{ timeout: 5_000, interval: 10 }
)
async function host(kind: string, initial: Record<string, unknown>[] = []) {
// All captured user/assistant/system records survive, including the child interruption after kill.
@@ -125,10 +138,7 @@ it.each(['local', 'relay', 'spool'])(
const before = p.row()
expect(before?.subagents?.[0]?.state).toBe('working')
await p.kill()
await vi.waitFor(() => expect(p.row()?.subagents ?? []).toEqual([]), {
timeout: 5_000,
interval: 30
})
await settle(() => expect(p.row()?.subagents ?? []).toEqual([]))
const after = p.row()!
expect(after.workingMode).toBe('monitoring')
expect(after.mainAgent).toEqual(before?.mainAgent)
@@ -201,10 +211,7 @@ it.each(['local', 'relay'])(
})
const before = p.row()!
await p.kill()
await vi.waitFor(() => expect(p.row()?.subagents ?? []).toEqual([]), {
timeout: 5_000,
interval: 30
})
await settle(() => expect(p.row()?.subagents ?? []).toEqual([]))
expect(p.row()?.mainAgent).toEqual(before.mainAgent)
expect(before.mainAgent?.outcome).toBe('cancellation')
expect(p.row()?.interrupted).toBe(true)
@@ -237,10 +244,7 @@ it.each(['hook', 'kill'])('matches child-only ending and wait ownership via %s',
await (ending === 'hook'
? p.post({ hook_event_name: 'SubagentStop', agent_id: 'a123' })
: p.kill())
await vi.waitFor(
() => expect(p.row()?.subagents?.some((c) => c.id === 'a123')).not.toBe(true),
{ timeout: 5_000, interval: 30 }
)
await settle(() => expect(p.row()?.subagents?.some((c) => c.id === 'a123')).not.toBe(true))
expect(p.row()?.state).toBe(owner === 'sibling' || owner === 'root' ? 'waiting' : 'working')
expect(p.row()?.toolName).toBe(owner === 'sibling' || owner === 'root' ? 'Bash' : undefined)
}
@@ -3,7 +3,7 @@ import { appendFile, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/pro
import { tmpdir } from 'node:os'
import { createServer } from 'node:http'
import { join } from 'node:path'
import { afterEach, expect, it, vi } from 'vitest'
import { afterEach, beforeEach, expect, it, vi } from 'vitest'
import { RelayAgentHookServer } from '../../wsl-guest/agent-hook-server'
import { AgentHookServer } from './server'
import { buildBody, PANE, postHookEvent } from './server.test-fixtures'
@@ -19,10 +19,15 @@ vi.mock('../telemetry/cohort-classifier', () => ({
}))
const servers: { stop: () => void }[] = []
const folders: string[] = []
const realSetTimeout = globalThis.setTimeout
beforeEach(() => {
vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'performance'] })
})
afterEach(async () => {
for (const server of servers.splice(0)) {
server.stop()
}
vi.useRealTimers()
// Only exact test-created mkdtemp paths; no external state is reachable.
for (const folder of folders.splice(0)) {
await rm(folder, { recursive: true, force: true })
@@ -32,7 +37,16 @@ afterEach(async () => {
const lines = (records: Record<string, unknown>[]) =>
records.map((r) => `${JSON.stringify(r)}\n`).join('')
// ast-grep-ignore: test-no-real-time-wait -- Real elapsed gaps separate hook R, provider K and restart S.
const pause = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))
const pause = (ms: number) => new Promise((resolve) => realSetTimeout(resolve, ms))
// Transcript polls run on the fake clock while transcript I/O and wall time stay real.
const settle = (assertion: () => void) =>
vi.waitFor(
() => {
vi.advanceTimersByTime(250)
assertion()
},
{ timeout: 3000, interval: 10 }
)
async function load(stream: string): Promise<Record<string, unknown>[]> {
return (
await readFile(
@@ -259,10 +273,7 @@ it.each([
advanceBeforeBind: () => clock?.mockReturnValue(acceptedAt + 10)
})
const before = p.row()!
await vi.waitFor(() => expect(p.row()?.subagents ?? []).toEqual([]), {
timeout: 3000,
interval: 30
})
await settle(() => expect(p.row()?.subagents ?? []).toEqual([]))
expect(p.row()?.workingMode).toBe('monitoring')
expect(p.row()?.mainAgent).toEqual(before.mainAgent)
expect(p.row()?.lastAssistantMessage).toBe('READY')
@@ -271,10 +282,7 @@ it.each([
const offset = owner.parent.offset
// A delayed hook is the legitimate parked wake. No second kill is appended.
await p.post(hooks[1])
await vi.waitFor(() => expect(p.row()?.subagents ?? []).toEqual([]), {
timeout: 3000,
interval: 30
})
await settle(() => expect(p.row()?.subagents ?? []).toEqual([]))
expect(p.entries.get(PANE)).toBe(owner)
expect(owner.parent.offset).toBe(offset)
if (when === 'accepted-before-bind') {
@@ -288,10 +296,7 @@ it.each(['desktop-http', 'relay-http', 'desktop-spool', 'relay-spool'])(
const p = await setup(kind, 'accepted-before-bind', { throwOnce: true })
expect(p.threw).toBe(true)
expect(p.entries.size).toBe(1)
await vi.waitFor(() => expect(p.row()?.subagents ?? []).toEqual([]), {
timeout: 3000,
interval: 30
})
await settle(() => expect(p.row()?.subagents ?? []).toEqual([]))
expect(p.row()?.state).toBe('working') // Unknown root remains live; no parent hook was supplied.
}
)
@@ -301,10 +306,7 @@ it.each(['relay-spool', 'desktop-spool'])(
async (kind) => {
const R = Math.floor(Date.now() / 1000) * 1000
const p = await setup(kind, 'before-start', { cutoff: R, killAt: R - 1001 })
await vi.waitFor(() => expect(p.entries.get(PANE)?.parent.offset).toBeGreaterThan(0), {
timeout: 3000,
interval: 30
})
await settle(() => expect(p.entries.get(PANE)?.parent.offset).toBeGreaterThan(0))
expect(p.entries.get(PANE)?.killedAt).toBeUndefined()
expect(p.row()?.subagents?.[0]?.state).toBe('working')
const child = join(p.transcript.slice(0, -6), 'subagents', 'agent-a123.jsonl')
@@ -322,20 +324,14 @@ it.each(['relay-spool', 'desktop-spool'])(
p.transcript,
lines([{ ...p.oldKill, timestamp: new Date(R + 500).toISOString() }])
)
await vi.waitFor(() => expect(p.entries.get(PANE)?.killedAt).toBe(R + 500), {
timeout: 3000,
interval: 30
})
await settle(() => expect(p.entries.get(PANE)?.killedAt).toBe(R + 500))
expect(p.row()?.subagents?.[0]?.state).toBe('working')
// R's precision admits K, but only provider generation decides whether this work ends.
await appendFile(
p.transcript,
lines([{ ...p.oldKill, timestamp: new Date(R + 600).toISOString() }])
)
await vi.waitFor(() => expect(p.row()?.subagents ?? []).toEqual([]), {
timeout: 3000,
interval: 30
})
await settle(() => expect(p.row()?.subagents ?? []).toEqual([]))
}
)
@@ -343,14 +339,11 @@ it('initialization of an already-listening desktop activates parked runtime spoo
const p = await setup('desktop-spool', 'before-start', { disabled: true })
expect(p.entries.size).toBe(0)
p.desktop.setStatusHooksEnabled(true)
await vi.waitFor(
() => {
expect(p.entries.size).toBe(1)
expect(p.row()?.subagents ?? []).toEqual([])
expect(p.row()?.workingMode).toBe('monitoring')
},
{ timeout: 3000, interval: 30 }
)
await settle(() => {
expect(p.entries.size).toBe(1)
expect(p.row()?.subagents ?? []).toEqual([])
expect(p.row()?.workingMode).toBe('monitoring')
})
})
it('caps future spool cutoffs and keeps HTTP cutoff provenance private', async () => {
@@ -358,10 +351,7 @@ it('caps future spool cutoffs and keeps HTTP cutoff provenance private', async (
const p = await setup('relay-spool', 'before-start', { cutoff: S + 60000 })
expect(p.entries.get(PANE)?.observedAt).toBeLessThan(S + 60000)
await appendFile(p.transcript, lines([{ ...p.oldKill, timestamp: new Date().toISOString() }]))
await vi.waitFor(() => expect(p.row()?.subagents ?? []).toEqual([]), {
timeout: 3000,
interval: 30
})
await settle(() => expect(p.row()?.subagents ?? []).toEqual([]))
const q = await setup('desktop-http', 'after-start')
await q.post(hooks[2], { receivedAt: 0, hookReceivedAt: 0 })
expect(q.entries.get(PANE)?.observedAt).toBeGreaterThan(S - 1000)
@@ -369,19 +359,13 @@ it('caps future spool cutoffs and keeps HTTP cutoff provenance private', async (
q.transcript,
lines([{ ...q.oldKill, timestamp: new Date(S - 10000).toISOString() }])
)
await vi.waitFor(() => expect(q.entries.get(PANE)?.parent.offset).toBeGreaterThan(0), {
timeout: 3000,
interval: 30
})
await settle(() => expect(q.entries.get(PANE)?.parent.offset).toBeGreaterThan(0))
expect(q.row()?.subagents?.[0]?.state).toBe('working')
})
it('a real failed bind parks without resources and explicit retry delivers its first fact', async () => {
const p = await setup('relay-spool', 'before-start', { retryBind: true })
await vi.waitFor(() => expect(p.row()?.subagents ?? []).toEqual([]), {
timeout: 3000,
interval: 30
})
await settle(() => expect(p.row()?.subagents ?? []).toEqual([]))
expect(p.row()?.workingMode).toBe('monitoring')
})
@@ -438,10 +422,7 @@ it('a held local child permission cannot reset the admitted owner', async () =>
expect(capture).not.toHaveBeenCalled()
expect(p.entries.get(PANE)).toBe(owner)
await appendFile(p.transcript, lines([{ ...p.oldKill, timestamp: new Date().toISOString() }]))
await vi.waitFor(() => expect(p.row()?.subagents ?? []).toEqual([]), {
timeout: 3000,
interval: 30
})
await settle(() => expect(p.row()?.subagents ?? []).toEqual([]))
})
it('preferred-port fallback activates the original admitted observation', async () => {
@@ -454,10 +435,7 @@ it('preferred-port fallback activates the original admitted observation', async
}
const p = await setup('relay-spool', 'before-start', { busyPort: address.port })
expect(p.relay.usedPortFallback).toBe(true)
await vi.waitFor(() => expect(p.row()?.subagents ?? []).toEqual([]), {
timeout: 3000,
interval: 30
})
await settle(() => expect(p.row()?.subagents ?? []).toEqual([]))
expect(p.row()?.workingMode).toBe('monitoring')
} finally {
await new Promise<void>((resolve) => busy.close(() => resolve()))
@@ -483,10 +461,7 @@ it.each(['desktop-http', 'relay-http'])(
p.transcript,
lines([{ ...p.oldKill, timestamp: new Date(S + 1).toISOString() }])
)
await vi.waitFor(() => expect(p.row()?.subagents ?? []).toEqual([]), {
timeout: 3000,
interval: 30
})
await settle(() => expect(p.row()?.subagents ?? []).toEqual([]))
}
)
@@ -517,8 +492,5 @@ it('reentrant mirror and OSC writes cannot borrow local hook admission', async (
expect(p.entries.size).toBe(1)
expect(p.desktop.getStatusSnapshotForPane(other)[0]?.state).toBe('working')
await appendFile(p.transcript, lines([{ ...p.oldKill, timestamp: new Date().toISOString() }]))
await vi.waitFor(() => expect(p.row()?.subagents ?? []).toEqual([]), {
timeout: 3000,
interval: 30
})
await settle(() => expect(p.row()?.subagents ?? []).toEqual([]))
})
@@ -1,20 +1,16 @@
import { spawn } from 'node:child_process'
import { mkdtempSync, rmSync } from 'node:fs'
import { createRequire } from 'node:module'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { pathToFileURL } from 'node:url'
import { afterEach, expect, it, vi } from 'vitest'
import { z } from 'zod'
import { buildOrcadCli } from '../../../config/scripts/orcad-distribution/server-bundle/orcad-entry-build.mjs'
import { OrchestrationDb } from './orchestration/db/orchestration-db'
import './orchestration/db/contract-constants'
import './orchestration/db/messages/mailbox-routing-page'
import './orchestration/db/dispatch-row-writer'
import './orchestration/db/attempt-outcome-projection'
const cliPath = join(process.cwd(), 'src/cli/index.ts')
const tsxLoader = pathToFileURL(createRequire(import.meta.url).resolve('tsx')).href
const entity = z.object({ id: z.string() })
const check = z.object({
result: z.object({
@@ -34,9 +30,12 @@ afterEach(() => {
vi.restoreAllMocks()
})
// Sequential source-CLI subprocesses and a disk-backed runtime restart exceed the default test budget.
// Sequential packaged-CLI subprocesses and a disk-backed runtime restart exceed the default test budget.
it("retains CLI work and mail across restart, refuses a worker's own Run, and completes a separate Run handoff", async () => {
const home = mkdtempSync(join(tmpdir(), 'orca-cli-restart-handoff-'))
// One bundle of the real CLI avoids a TypeScript loader start in each of ~25 subprocesses.
const cliPath = join(home, 'cli', 'index.js')
await buildOrcadCli(cliPath)
for (const name of [
'HOME',
'USERPROFILE',
@@ -93,36 +92,23 @@ it("retains CLI work and mail across restart, refuses a worker's own Run, and co
let runtime = runtimeForDatabase()
let server = new OrcaRuntimeRpcServer({ runtime, userDataPath: home })
async function cli(handle: string, args: string[], exitCode = 0): Promise<unknown> {
const child = spawn(
process.execPath,
[
'--import',
tsxLoader,
'--conditions=orca-source',
cliPath,
'orchestration',
...args,
'--json',
'--full'
],
{
cwd: home,
env: {
...process.env,
ORCA_TERMINAL_HANDLE: handle,
ORCA_PANE_KEY: runtime.getTerminalPaneKey(handle) ?? '',
ORCA_AGENT_LAUNCH_TOKEN:
handle === coord
? fixture.LAUNCH_TOKEN
: handle === worker
? fixture.SECOND_LAUNCH_TOKEN
: ''
},
stdio: ['ignore', 'pipe', 'pipe'],
windowsHide: true,
timeout: 15_000
}
)
const child = spawn(process.execPath, [cliPath, 'orchestration', ...args, '--json', '--full'], {
cwd: home,
env: {
...process.env,
ORCA_TERMINAL_HANDLE: handle,
ORCA_PANE_KEY: runtime.getTerminalPaneKey(handle) ?? '',
ORCA_AGENT_LAUNCH_TOKEN:
handle === coord
? fixture.LAUNCH_TOKEN
: handle === worker
? fixture.SECOND_LAUNCH_TOKEN
: ''
},
stdio: ['ignore', 'pipe', 'pipe'],
windowsHide: true,
timeout: 15_000
})
let stdout = ''
let stderr = ''
child.stdout.setEncoding('utf8').on('data', (chunk: string) => {