starred/orca
1
0
Fork 0
mirror of https://github.com/stablyai/orca.git synced 2026-11-04 19:17:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(native-chat): write the Codex frame script's parent row out step by step

Browse Source 
Once every surface reads the child records, the provider tracker's roster is
no oracle: it and the records read the same child executions, so agreeing
with it cannot catch a defect in either. Each frame now states the child
liveness and the parent row it must fold to.
This commit is contained in:
Brennan Benson
2026-09-29 00:59:48 -07:00
parent abc964b924
commit bb1b369be0
1 changed files with 111 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/codex/codex-structured-child-work-producer.test.ts
+111 -36
View File
@@ -1,6 +1,7 @@
// A Codex session's frames, through the real adapter, into the host's child records: the order the
// host receives them in, and whether the parent row the records imply is today's row.
// host receives them in, and the parent row they fold to, written out step by step.
import { isDeepStrictEqual } from 'node:util'
import { describe, expect, it } from 'vitest'
import {
foldAgentLeadStatus,
@@ -42,8 +43,8 @@ const LINTER = 'thread-linter'
type Frame = { method: string; params: Record<string, unknown> }
type Delivery = { kind: 'journal' | 'publish' | 'evidence'; detail: string }
type Liveness = ReturnType<typeof agentChildWorkLiveness>
/** What the records and today's strip each said when the journal wrote or published a row. */
type JournalMoment = { recorded: Liveness; legacy: Liveness }
/** What the records said when the journal wrote or published a row. */
type JournalMoment = { recorded: Liveness; commandLive: boolean }
const turn = (
method: 'turn/started' | 'turn/completed',
@@ -99,7 +100,9 @@ async function producer() {
const moment = () =>
moments.push({
recorded: recordedLiveness(),
legacy: agentChildWorkLiveness(adapter.backgroundTaskState('session-1')?.tasks)
commandLive: records().some(
(record) => record.kind === 'command' && record.membership === 'live'
)
})
const adapter = new CodexStructuredSessionAdapter({
resolveLaunch: async () => ({
@@ -180,8 +183,6 @@ const fold = (
leadState: 'working' | 'done',
childWorkLiveness: Liveness
): AgentLeadStatusResolution => foldAgentLeadStatus({ leadState, childWorkLiveness })
/** The strip has no word for a child waiting on a human: to it, that child is working. */
const asStrip = (liveness: Liveness): Liveness => (liveness === 'waiting' ? 'working' : liveness)
const shellFrame = (
method: 'item/started' | 'item/completed',
threadId: string,
@@ -216,28 +217,43 @@ describe('Codex structured child-work producer', () => {
])
})
it('records the parent state today reads, at every journal write, while adding outcome and activity', async () => {
it("folds the parent from the records, at every journal write, with each child's outcome and activity", async () => {
const { adapter, momentsOf, records, recordedLiveness, byDescription, display } =
await producer()
const steps: {
frame: Frame
lead: 'working' | 'done'
childWaits?: true
live: Liveness
parent: AgentLeadStatusResolution
check?: () => void
}[] = [
{ frame: turn('turn/started', THREAD_ID, 'p1'), lead: 'working' },
{
frame: turn('turn/started', THREAD_ID, 'p1'),
lead: 'working',
live: null,
parent: { stateName: 'working' }
},
// Codex reports the child's turn before its announcement.
{
frame: turn('turn/started', REVIEWER, 'r1'),
lead: 'working',
live: null,
parent: { stateName: 'working' },
check: () => expect(records()).toEqual([])
},
{ frame: spawned(REVIEWER, 'review', 'p1'), lead: 'working' },
{
frame: spawned(REVIEWER, 'review', 'p1'),
lead: 'working',
live: 'working',
parent: { stateName: 'working' }
},
// An approved command: Codex starts it on the approval path and reports its exit from
// unified exec.
{
frame: shellFrame('item/started', REVIEWER, 'r1', 'cmd-1', 'npm test', 'agent'),
lead: 'working',
live: 'working',
parent: { stateName: 'working' },
check: () => {
expect(byDescription('review')?.operation).toMatchObject({
toolName: 'Bash',
@@ -254,6 +270,8 @@ describe('Codex structured child-work producer', () => {
{
frame: shellFrame('item/started', REVIEWER, 'r1', 'exec-1', 'npm run dev'),
lead: 'working',
live: 'working',
parent: { stateName: 'working' },
check: () =>
expect(byDescription('npm run dev')).toMatchObject({
membership: 'live',
@@ -263,6 +281,8 @@ describe('Codex structured child-work producer', () => {
{
frame: shellFrame('item/completed', REVIEWER, 'r1', 'cmd-1', 'npm test'),
lead: 'working',
live: 'working',
parent: { stateName: 'working' },
check: () => {
// A finished command leaves nothing behind.
expect(byDescription('npm test')).toBeUndefined()
@@ -279,32 +299,39 @@ describe('Codex structured child-work producer', () => {
id: 'msg-1',
text: 'Dev server is up'
}),
lead: 'working'
lead: 'working',
live: 'working',
parent: { stateName: 'working' }
},
// The parent's turn ends first; its child keeps running.
{
frame: turn('turn/completed', THREAD_ID, 'p1'),
lead: 'done',
live: 'working',
parent: { stateName: 'working' },
check: () =>
expect(byDescription('review')).toMatchObject({ membership: 'live', state: 'working' })
},
// The legacy task list carries no child state, so only the records can say a child waits,
// and the shared fold ranks that wait above the parent's own state.
// Only the records say a child waits, and the shared fold ranks that wait above the parent's
// own state.
{
frame: status(REVIEWER, ['waitingOnApproval']),
lead: 'done',
childWaits: true,
check: () => {
expect(byDescription('review')?.state).toBe('waiting')
expect(agentChildWorkLiveness(adapter.backgroundTaskState('session-1')?.tasks)).toBe(
'working'
)
}
live: 'waiting',
parent: { stateName: 'waiting' },
check: () => expect(byDescription('review')?.state).toBe('waiting')
},
{
frame: status(REVIEWER, []),
lead: 'done',
live: 'working',
parent: { stateName: 'working' }
},
{ frame: status(REVIEWER, []), lead: 'done' },
{
frame: turn('turn/completed', REVIEWER, 'r1'),
lead: 'done',
live: 'monitoring',
parent: { stateName: 'working', workingMode: 'monitoring' },
check: () => {
expect(byDescription('review')).toMatchObject({
membership: 'settled',
@@ -319,11 +346,18 @@ describe('Codex structured child-work producer', () => {
expect(display('review')).toBe('monitoring')
}
},
{ frame: turn('turn/started', THREAD_ID, 'p2'), lead: 'working' },
{
frame: turn('turn/started', THREAD_ID, 'p2'),
lead: 'working',
live: 'monitoring',
parent: { stateName: 'working' }
},
// The parent asks the finished child a follow-up: the same record, a new run.
{
frame: turn('turn/started', REVIEWER, 'r2'),
lead: 'working',
live: 'working',
parent: { stateName: 'working' },
check: () =>
expect(byDescription('review')).toMatchObject({
childWorkId: 'child-1',
@@ -332,10 +366,30 @@ describe('Codex structured child-work producer', () => {
previousInvocations: [expect.objectContaining({ outcome: 'succeeded' })]
})
},
{ frame: spawned(TESTER, 'test', 'p2'), lead: 'working' },
{ frame: turn('turn/started', TESTER, 't1'), lead: 'working' },
{ frame: spawned(LINTER, 'lint', 'p2'), lead: 'working' },
{ frame: turn('turn/started', LINTER, 'l1'), lead: 'working' },
{
frame: spawned(TESTER, 'test', 'p2'),
lead: 'working',
live: 'working',
parent: { stateName: 'working' }
},
{
frame: turn('turn/started', TESTER, 't1'),
lead: 'working',
live: 'working',
parent: { stateName: 'working' }
},
{
frame: spawned(LINTER, 'lint', 'p2'),
lead: 'working',
live: 'working',
parent: { stateName: 'working' }
},
{
frame: turn('turn/started', LINTER, 'l1'),
lead: 'working',
live: 'working',
parent: { stateName: 'working' }
},
// Codex ends this child's turn with an error it will not retry, and no turn/completed.
{
frame: {
@@ -343,28 +397,41 @@ describe('Codex structured child-work producer', () => {
params: { threadId: LINTER, turnId: 'l1', willRetry: false, error: { message: 'boom' } }
},
lead: 'working',
live: 'working',
parent: { stateName: 'working' },
check: () =>
expect(byDescription('lint')).toMatchObject({ membership: 'settled', outcome: 'failed' })
},
{
frame: turn('turn/completed', REVIEWER, 'r2', 'interrupted'),
lead: 'working',
live: 'working',
parent: { stateName: 'working' },
check: () =>
expect(byDescription('review')).toMatchObject({
membership: 'settled',
outcome: 'cancelled'
})
},
{ frame: turn('turn/completed', THREAD_ID, 'p2'), lead: 'done' },
{
frame: turn('turn/completed', THREAD_ID, 'p2'),
lead: 'done',
live: 'working',
parent: { stateName: 'working' }
},
{
frame: turn('turn/completed', TESTER, 't1', 'failed'),
lead: 'done',
live: 'monitoring',
parent: { stateName: 'working', workingMode: 'monitoring' },
check: () =>
expect(byDescription('test')).toMatchObject({ membership: 'settled', outcome: 'failed' })
},
{
frame: shellFrame('item/completed', REVIEWER, 'r1', 'exec-1', 'npm run dev'),
lead: 'done',
live: null,
parent: { stateName: 'done' },
check: () => {
expect(byDescription('npm run dev')).toBeUndefined()
expect(display('review')).toBe('interrupted')
@@ -372,24 +439,32 @@ describe('Codex structured child-work producer', () => {
}
]
let journalMoments = 0
let before = steps[0].parent
for (const [index, step] of steps.entries()) {
const frameMoments = momentsOf(step.frame)
journalMoments += frameMoments.length
for (const [at, { recorded, legacy }] of frameMoments.entries()) {
expect({ index, at, parent: fold(step.lead, asStrip(recorded)) }).toEqual({
// Every row the journal publishes mid-frame reads the parent from before the frame or after
// it, and never done while a command still runs.
for (const [at, { recorded, commandLive }] of frameMoments.entries()) {
const parent = fold(step.lead, recorded)
expect({
index,
at,
parent: fold(step.lead, legacy)
parent: [before, step.parent].some((p) => isDeepStrictEqual(p, parent))
}).toEqual({ index, at, parent: true })
expect({ index, at, done: commandLive && parent.stateName === 'done' }).toEqual({
index,
at,
done: false
})
}
const legacy = agentChildWorkLiveness(adapter.backgroundTaskState('session-1')?.tasks)
const recorded = recordedLiveness()
const expected = step.childWaits ? 'waiting' : legacy
expect({ index, parent: fold(step.lead, recorded) }).toEqual({
const live = recordedLiveness()
expect({ index, live, parent: fold(step.lead, live) }).toEqual({
index,
parent: fold(step.lead, expected)
live: step.live,
parent: step.parent
})
expect({ index, liveness: recorded }).toEqual({ index, liveness: expected })
before = step.parent
step.check?.()
}
expect(journalMoments).toBeGreaterThan(steps.length)