test(orchestration): diagnose surviving timers through public SQL reads

This commit is contained in:
Brennan Benson
2026-11-11 04:42:00 -08:00
parent b7d55d92f9
commit 7a5ffeb7c7
2 changed files with 70 additions and 34 deletions
@@ -59,6 +59,7 @@ export function createCoordinatorMailObservationClock(
await Promise.allSettled(pending)
}
}
const timers = new Map<Parameters<typeof clearTimeout>[0], { delay: number; stack: string }>()
return {
install(): void {
pending.clear()
@@ -86,6 +87,28 @@ export function createCoordinatorMailObservationClock(
drain,
start(): void {
vi.useFakeTimers({ toFake: ['Date', 'setTimeout', 'clearTimeout'] })
timers.clear()
const after = globalThis.setTimeout
const cancel = globalThis.clearTimeout
vi.spyOn(globalThis, 'setTimeout').mockImplementation((callback, delay, ...args) => {
const timer = after(
function (this: ReturnType<typeof setTimeout>, ...fired: Parameters<typeof callback>) {
timers.delete(timer)
callback.apply(this, fired)
},
delay,
...args
)
timers.set(timer, {
delay: delay ?? 0,
stack: new Error('fake timeout created').stack ?? ''
})
return timer
})
vi.spyOn(globalThis, 'clearTimeout').mockImplementation((timer) => {
timers.delete(timer)
return cancel(timer)
})
startFakeTimeAwakeClock()
active = true
},
@@ -115,7 +138,19 @@ export function createCoordinatorMailObservationClock(
await vi.advanceTimersByTimeAsync(2_000)
await waitForStructuredAgentSessionRecovery()
await drain()
expect(vi.getTimerCount()).toBe(fakeTimeAwakeClock.heartbeats)
const actual = vi.getTimerCount()
const expected = fakeTimeAwakeClock.heartbeats
if (actual !== expected) {
console.error('coordinator mail timers survived cleanup', {
test: expect.getState().currentTestName,
file: expect.getState().testPath,
actual,
expected,
pendingWork: pending.size,
timers: [...timers].map(([handle, metadata]) => ({ handle: Number(handle), ...metadata }))
})
}
expect(actual).toBe(expected)
},
restore(): void {
if (active) {
@@ -124,6 +159,7 @@ export function createCoordinatorMailObservationClock(
// useRealTimers dropped this clock's pending heartbeat along with the other fake timers.
fakeTimeAwakeClock.clock = null
fakeTimeAwakeClock.heartbeats = 0
timers.clear()
active = false
}
}
@@ -4,7 +4,7 @@ import { formatOrcaSessionAddress } from '../../../../../shared/orca-session-add
import { testOrcaSessionId } from '../../../../../shared/orca-session-address-test-fixture'
import { JournalSqlOwner } from '../../../../native-chat/agent-session-journal/journal-sql-owner'
import { JournalSqlReaderPool } from '../../../../native-chat/agent-session-journal/journal-sql-reader-pool'
import type { AgentSessionJournal } from '../../../../native-chat/agent-session-journal/journal-store'
import { journalSqlReads } from '../../../../native-chat/agent-session-journal/journal-sql-reads'
import { resultOf } from '../../../rpc/orchestration-session-caller-test-fixture'
import { operationId } from './structured-chat-coordinator-fake-codex-fixture'
import {
@@ -36,8 +36,24 @@ async function busyChat(sessionId: string) {
return chat
}
function journalOf(sessionId: string): AgentSessionJournal {
return host.collaboratorsForTests().sessions.get(sessionId)!.journal
function readsOf(sessionId: string) {
return journalSqlReads(host.deps.journalDatabase.readers, sessionId)
}
function holdMailRead(sessionId: string, hold: () => Promise<void>): void {
const read = JournalSqlReaderPool.prototype.point
let held = false
vi.spyOn(JournalSqlReaderPool.prototype, 'point').mockImplementation(async function (
this: JournalSqlReaderPool,
request
) {
const answer = await read.call(this, request)
if (!held && request.sessionId === sessionId && request.operation.kind === 'mail-observation') {
held = true
await hold()
}
return answer
})
}
describe('mail checks reconcile the currently owned source attempts', () => {
@@ -78,23 +94,9 @@ describe('mail checks reconcile the currently owned source attempts', () => {
await vi.waitFor(() => expect(chat.turns).toHaveLength(1), WAIT)
await settleTurn(PEER_CHAT, 0)
await accepted.promise
const read = JournalSqlReaderPool.prototype.point
let heldRead = false
vi.spyOn(JournalSqlReaderPool.prototype, 'point').mockImplementation(async function (
this: JournalSqlReaderPool,
request
) {
const answer = await read.call(this, request)
if (
!heldRead &&
request.sessionId === PEER_CHAT &&
request.operation.kind === 'mail-observation'
) {
heldRead = true
observed.resolve()
await releaseCheck.promise
}
return answer
holdMailRead(PEER_CHAT, async () => {
observed.resolve()
await releaseCheck.promise
})
const checking = call('orchestration.check', {}, { sessionId: PEER_CHAT })
try {
@@ -282,7 +284,11 @@ describe('mail checks reconcile the currently owned source attempts', () => {
return observe(attempt)
})
try {
const [card] = await journalOf(PEER_CHAT).queuedMessages.list()
const queue = await readsOf(PEER_CHAT).queue({ kind: 'list', facts: { reopenFloor: null } })
if (queue.kind !== 'list') {
throw new Error('journal reader returned another queue result')
}
const [card] = queue.cards
const edit = {
messageId: card!.messageId,
expectedBodyFingerprint: card!.fingerprint,
@@ -325,7 +331,7 @@ describe('mail checks reconcile the currently owned source attempts', () => {
}
)
).toMatchObject({ ok: true, value: { deleted: true } })
expect(await journalOf(PEER_CHAT).reads.mailObservation(original)).toMatchObject({
expect(await readsOf(PEER_CHAT).mailObservation(original)).toMatchObject({
withdrawal: 'person'
})
const hold = host.holdMailSources
@@ -447,18 +453,16 @@ describe('mail checks reconcile the currently owned source attempts', () => {
{ sessionId: COORDINATOR }
)
await vi.waitFor(async () => expect(await queuedCardTexts(PEER_CHAT)).toHaveLength(1), WAIT)
const journal = journalOf(PEER_CHAT)
const read = journal.reads.mailObservation
const reads = readsOf(PEER_CHAT)
const epoch = (await reads.cursor()).epoch
const reading = Promise.withResolvers<void>()
const release = Promise.withResolvers<void>()
onTestFinished(() => release.resolve())
const finished = Promise.withResolvers<void>()
vi.spyOn(journal.reads, 'mailObservation').mockImplementationOnce(async (...args) => {
const observed = await read(...args)
holdMailRead(PEER_CHAT, async () => {
reading.resolve()
await release.promise
finished.resolve()
return observed
})
const controller = new AbortController()
const checking = dispatcher.dispatch(
@@ -471,7 +475,7 @@ describe('mail checks reconcile the currently owned source attempts', () => {
expect(await checking).toMatchObject({ ok: false, error: { code: 'request_aborted' } })
// This actual source owner runs while the old read reply remains held.
await host.withForkSourceAdmission(PEER_CHAT, async () => {
expect((await journal.reads.cursor()).epoch).toBe(journal.cursor().epoch)
expect((await reads.cursor()).epoch).toBe(epoch)
})
expect(db.getUnreadMessages(address)).toMatchObject([{ subject: 'keep after disconnect' }])
} finally {
@@ -497,7 +501,6 @@ describe('mail checks reconcile the currently owned source attempts', () => {
)
await vi.waitFor(async () => expect(await queuedCardTexts(PEER_CHAT)).toHaveLength(1), WAIT)
const original = db.getStructuredMailSends(address)[0]!
const journal = journalOf(PEER_CHAT)
const automatic = Promise.withResolvers<void>()
const observe = host.mailObservation
vi.spyOn(host, 'mailObservation').mockImplementation(async (...args) => {
@@ -506,7 +509,6 @@ describe('mail checks reconcile the currently owned source attempts', () => {
}
return observe(...args)
})
const read = journal.reads.mailObservation
const reading = Promise.withResolvers<void>()
const release = Promise.withResolvers<void>()
const finished = Promise.withResolvers<void>()
@@ -514,12 +516,10 @@ describe('mail checks reconcile the currently owned source attempts', () => {
release.resolve()
automatic.resolve()
})
vi.spyOn(journal.reads, 'mailObservation').mockImplementationOnce(async (...args) => {
const observed = await read(...args)
holdMailRead(PEER_CHAT, async () => {
reading.resolve()
await release.promise
finished.resolve()
return observed
})
observationClock.start()
const invocation = request('orchestration.check', {}, { sessionId: PEER_CHAT })