From 5fc6f7a1ccf04fa08f44a28acd92b3f768781db5 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Wed, 12 Aug 2026 17:35:57 -0700 Subject: [PATCH] test(native-chat): derive refusal retry state from host --- ...ctured-agent-session-refusal-retry.test.ts | 435 ++++++++++++++++++ .../agent-session-refusal-retry.test.ts | 79 ---- 2 files changed, 435 insertions(+), 79 deletions(-) create mode 100644 src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts delete mode 100644 src/shared/agent-session-refusal-retry.test.ts diff --git a/src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts b/src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts new file mode 100644 index 00000000000..3e12724513c --- /dev/null +++ b/src/main/native-chat/agent-session-wire/structured-agent-session-refusal-retry.test.ts @@ -0,0 +1,435 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it, vi, type Mock } from 'vitest' +import { AGENT_SESSION_MAX_NEW_OPERATION_AGE_MS } from '../../../shared/agent-session-host-authority' +import { AGENT_SESSION_DURABLE_OPERATION_PER_CLIENT_LIMIT } from '../../../shared/agent-session-operation-ledger' +import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope' +import type { AgentSessionRecord } from '../../../shared/agent-session-record' +import { + AGENT_SESSION_WIRE_REFUSAL_CODES, + type AgentSessionHandoffRequest, + type AgentSessionMutationEnvelope, + type AgentSessionMutationResult, + type AgentSessionWireRefusalCode +} from '../../../shared/agent-session-wire' +import { + agentSessionRefusalOperationState, + type AgentSessionRefusalOperationState +} from '../../../shared/agent-session-refusal-retry' +import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store' +import { AgentSessionJournal } from '../agent-session-journal/journal-store' +import type { StructuredAgentSessionAdapter } from './structured-agent-session-adapter' +import { StructuredAgentSessionHost } from './structured-agent-session-host' +import { + HOST_TEST_NOW as NOW, + HOST_TEST_SESSION as SESSION, + HOST_TEST_THREAD as THREAD, + hostTestAttachParams, + hostTestMessage +} from './structured-agent-session-host-test-data' +import type { StructuredAgentSessionHandoffTransport } from './structured-agent-session-handoff-types' + +const CALLER = { callerKey: 'client-1' } +const METHODS = [ + 'agentSession.requestHandoff', + 'agentSession.setOption', + 'agentSession.send' +] as const +type Method = (typeof METHODS)[number] +type Pair = `${Method}:${AgentSessionWireRefusalCode}` + +type CallSpec = { + method: Method + operationId: string + expectedRuntimeFence?: number + payloadFingerprint?: string + handoffAction?: 'start' | 'cancel-queued' + handoffDirection?: 'to-tui' | 'to-native' +} + +type Harness = { + root: string + store: AgentSessionRecordStore + host: StructuredAgentSessionHost + setOption: Mock +} + +const harnesses: Harness[] = [] +let operationSequence = 1_000 + +function operationId(timestamp = NOW): string { + operationSequence += 1 + return `${timestamp}-${operationSequence.toString(16).padStart(32, '0')}` +} + +function handoffTransport(): StructuredAgentSessionHandoffTransport { + const unused = async (): Promise => { + throw new Error('unused handoff transport') + } + return { + hostLabel: 'test-host', + launchTui: unused, + reproveTuiOwner: unused, + recoverTuiOwner: unused, + stopRecoveredOwner: async () => undefined, + waitForTuiExit: unused, + waitForTuiIdleOrExit: unused, + tuiStatus: () => 'idle' + } +} + +async function createHarness(options: { attached?: boolean; transport?: boolean } = {}) { + const root = await mkdtemp(join(tmpdir(), 'orca-refusal-oracle-')) + const store = await AgentSessionRecordStore.open({ + directory: join(root, 'store'), + hostId: 'local' + }) + const setOption = vi.fn(async () => undefined) + const adapter: StructuredAgentSessionAdapter = { + acquire: async ({ fence }) => ({ + process: { + hostId: 'local', + pid: 4242, + processStartTimeMs: NOW - 1_000, + spawnToken: store.getRecord(SESSION)?.lease.reservedSpawnToken ?? 'spawn-a' + }, + link: { + linkId: `link-${fence}`, + handle: { provider: 'codex', threadId: THREAD }, + origin: 'created', + mintedAtFence: fence, + observedAt: NOW + } + }), + dispatch: async () => ({ + state: 'accepted', + providerIdentity: { provider: 'codex', threadId: THREAD, turnId: 'turn-1', ordinal: 1 } + }), + cancelTurn: async () => ({ cancelled: true }), + answerPrompt: async () => undefined, + setOption + } + const host = new StructuredAgentSessionHost({ + store, + adapter, + journalRoot: root, + claimKeyId: 'key-1', + mintSpawnToken: () => 'spawn-a', + now: () => NOW, + ...(options.transport ? { handoffTransport: handoffTransport() } : {}) + }) + const harness = { root, store, host, setOption } + harnesses.push(harness) + if (options.attached !== false) { + expect(await host.attach(CALLER, hostTestAttachParams(null))).toMatchObject({ ok: true }) + } + return harness +} + +afterEach(async () => { + const completed = harnesses.splice(0) + await Promise.all(completed.map(async ({ host }) => host.flushAllStreamedEvents())) + await Promise.all(completed.map(async ({ root }) => rm(root, { recursive: true }))) +}) + +function callFields(spec: CallSpec): Record { + if (spec.method === 'agentSession.send') { + return { body: hostTestMessage('host oracle') } + } + if (spec.method === 'agentSession.setOption') { + return { key: 'model', value: 'gpt-5' } + } + return { + direction: spec.handoffDirection ?? 'to-tui', + mode: 'now', + action: spec.handoffAction ?? 'start' + } +} + +function envelope(harness: Harness, spec: CallSpec): AgentSessionMutationEnvelope { + const fields = callFields(spec) + return { + sessionId: SESSION, + clientOperationId: spec.operationId, + expectedRuntimeFence: + spec.expectedRuntimeFence ?? harness.store.getRecord(SESSION)?.lease.runtimeFence ?? 1, + payloadFingerprint: + spec.payloadFingerprint ?? + computeAgentSessionPayloadFingerprint({ method: spec.method, sessionId: SESSION, fields }) + } +} + +function invoke(harness: Harness, spec: CallSpec): Promise> { + const fields = callFields(spec) + const mutationEnvelope = envelope(harness, spec) + if (spec.method === 'agentSession.send') { + return harness.host.send(CALLER, { + envelope: mutationEnvelope, + body: fields.body as ReturnType + }) + } + if (spec.method === 'agentSession.setOption') { + return harness.host.setOption(CALLER, { + envelope: mutationEnvelope, + key: fields.key as string, + value: fields.value as string + }) + } + return harness.host.requestHandoff(CALLER, { + envelope: mutationEnvelope, + direction: fields.direction, + mode: fields.mode, + action: fields.action + } as AgentSessionHandoffRequest) +} + +function operationState(harness: Harness, operation: string) { + return harness.store + .listOperationRows() + .find((row) => row.callerKey === CALLER.callerKey && row.operationId === operation)?.outcome +} + +async function assertHostAgreement( + harness: Harness, + spec: CallSpec, + code: AgentSessionWireRefusalCode, + retryOnFreshHost = false +): Promise { + try { + const result = await invoke(harness, spec) + expect(result, `${spec.method}:${code}`).toMatchObject({ ok: false, refusal: { code } }) + } catch (error) { + expect(error).toMatchObject({ message: code }) + } + const outcome = operationState(harness, spec.operationId) + let oracle: AgentSessionRefusalOperationState + if (outcome?.status === 'failed') { + oracle = 'settled-rejected' + } else if (outcome?.status === 'unknown') { + oracle = 'unknown' + } else if (outcome?.status === 'pending') { + oracle = 'pending-admission' + } else { + const retryHarness = retryOnFreshHost ? await createHarness() : harness + await invoke(retryHarness, spec) + oracle = operationState(retryHarness, spec.operationId) + ? 'pending-admission' + : 'settled-rejected' + } + expect(agentSessionRefusalOperationState(spec.method, code), `${spec.method}:${code}`).toBe( + oracle + ) + return `${spec.method}:${code}` +} + +async function setLease( + harness: Harness, + update: (record: AgentSessionRecord) => AgentSessionRecord +): Promise { + await harness.store.transitionHandoff(SESSION, update) +} + +async function fillOperationLedger(harness: Harness): Promise { + while ( + harness.store.listOperationRows().filter((row) => row.callerKey === CALLER.callerKey).length < + AGENT_SESSION_DURABLE_OPERATION_PER_CLIENT_LIMIT + ) { + await harness.store.admitOperation({ + callerKey: CALLER.callerKey, + operationId: operationId(), + fingerprint: 'capacity-fixture', + now: NOW + }) + } +} + +// sendPlan and setOptionPlan have no unsupported branch; only handoff checks transport. +const UNREACHABLE = new Set([ + 'agentSession.send:structured_agent_session_unsupported', + 'agentSession.setOption:structured_agent_session_unsupported', + // replayedStructuredHandoffRefusal maps failed wire codes; other replay outcomes are successes. + 'agentSession.requestHandoff:agent_session_operation_unknown', + // performPrompt is the sole producer of prompt revision and resolution refusals. + 'agentSession.requestHandoff:agent_session_item_revision_stale', + 'agentSession.setOption:agent_session_item_revision_stale', + 'agentSession.send:agent_session_item_revision_stale', + 'agentSession.requestHandoff:agent_session_already_resolved', + 'agentSession.setOption:agent_session_already_resolved', + 'agentSession.send:agent_session_already_resolved', + // StructuredAgentSessionHost.mutate maps an absent record to AGENT_SESSION_NOT_ATTACHED. + 'agentSession.requestHandoff:agent_session_identity_required', + 'agentSession.setOption:agent_session_identity_required', + 'agentSession.send:agent_session_identity_required', + // No structured-agent-session host branch emits agent_session_journal_unreadable. + 'agentSession.requestHandoff:agent_session_journal_unreadable', + 'agentSession.setOption:agent_session_journal_unreadable', + 'agentSession.send:agent_session_journal_unreadable', + // admitStructuredHandoffRequest does not consult lease.unreconciled. + 'agentSession.requestHandoff:execution_owner_reconciling' +]) + +describe('agentSessionRefusalOperationState host oracle', () => { + it('agrees with every refusal the real host path can produce', async () => { + const produced = new Set() + const record = (pair: Pair) => produced.add(pair) + + const unsupported = await createHarness() + record( + await assertHostAgreement( + unsupported, + { + method: 'agentSession.requestHandoff', + operationId: operationId() + }, + 'structured_agent_session_unsupported' + ) + ) + + const stale = await createHarness() + for (const method of METHODS) { + record( + await assertHostAgreement( + stale, + { + method, + operationId: operationId(), + expectedRuntimeFence: 99 + }, + 'agent_session_checkpoint_stale' + ) + ) + } + + const conflict = await createHarness({ transport: true }) + record( + await assertHostAgreement( + conflict, + { + method: 'agentSession.requestHandoff', + operationId: operationId(), + handoffDirection: 'to-native' + }, + 'agent_session_conflict' + ) + ) + await setLease(conflict, (current) => ({ + ...current, + lease: { ...current.lease, runtimeKind: 'tui' } + })) + for (const method of ['agentSession.setOption', 'agentSession.send'] as const) { + record( + await assertHostAgreement( + conflict, + { method, operationId: operationId() }, + 'agent_session_conflict' + ) + ) + } + + const absent = await createHarness({ attached: false }) + for (const method of METHODS) { + record( + await assertHostAgreement( + absent, + { method, operationId: operationId() }, + 'agent_session_ownership_unknown', + true + ) + ) + } + + const operationConflict = await createHarness() + for (const method of ['agentSession.setOption', 'agentSession.send'] as const) { + record( + await assertHostAgreement( + operationConflict, + { + method, + operationId: operationId(), + payloadFingerprint: 'wrong' + }, + 'agent_session_operation_conflict' + ) + ) + } + record( + await assertHostAgreement( + operationConflict, + { + method: 'agentSession.requestHandoff', + operationId: operationId(), + handoffAction: 'cancel-queued' + }, + 'agent_session_operation_conflict' + ) + ) + + const ledgerRefusals = await createHarness() + for (const [code, timestamp] of [ + ['agent_session_operation_expired', NOW - AGENT_SESSION_MAX_NEW_OPERATION_AGE_MS - 1], + ['agent_session_operation_invalid', null] + ] as const) { + for (const method of METHODS) { + record( + await assertHostAgreement( + ledgerRefusals, + { + method, + operationId: timestamp === null ? 'invalid-operation-id' : operationId(timestamp) + }, + code + ) + ) + } + } + + const capacity = await createHarness() + await fillOperationLedger(capacity) + for (const method of METHODS) { + record( + await assertHostAgreement( + capacity, + { method, operationId: operationId() }, + 'agent_session_operation_capacity', + true + ) + ) + } + + const unknown = await createHarness() + unknown.setOption.mockRejectedValueOnce(new Error('reply lost')) + const optionUnknown = { method: 'agentSession.setOption' as const, operationId: operationId() } + await expect(invoke(unknown, optionUnknown)).rejects.toThrow('reply lost') + record(await assertHostAgreement(unknown, optionUnknown, 'agent_session_operation_unknown')) + + const appendFailure = vi + .spyOn(AgentSessionJournal.prototype, 'appendSubmission') + .mockRejectedValueOnce(new Error('journal write failed')) + const sendUnknown = { method: 'agentSession.send' as const, operationId: operationId() } + await expect(invoke(unknown, sendUnknown)).rejects.toThrow('journal write failed') + appendFailure.mockRestore() + record(await assertHostAgreement(unknown, sendUnknown, 'agent_session_operation_unknown')) + + const reconciling = await createHarness() + await setLease(reconciling, (current) => ({ + ...current, + lease: { ...current.lease, unreconciled: true } + })) + for (const method of ['agentSession.setOption', 'agentSession.send'] as const) { + record( + await assertHostAgreement( + reconciling, + { method, operationId: operationId() }, + 'execution_owner_reconciling' + ) + ) + } + + const allPairs = METHODS.flatMap((method) => + AGENT_SESSION_WIRE_REFUSAL_CODES.map((code) => `${method}:${code}` as Pair) + ) + expect(new Set([...produced, ...UNREACHABLE])).toEqual(new Set(allPairs)) + expect([...produced].filter((pair) => UNREACHABLE.has(pair))).toEqual([]) + }) +}) diff --git a/src/shared/agent-session-refusal-retry.test.ts b/src/shared/agent-session-refusal-retry.test.ts deleted file mode 100644 index 833bb7de84f..00000000000 --- a/src/shared/agent-session-refusal-retry.test.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { - AGENT_SESSION_WIRE_REFUSAL_CODES, - type AgentSessionWireRefusalCode -} from './agent-session-wire' -import { agentSessionRefusalOperationState } from './agent-session-refusal-retry' -import type { AgentSessionRefusalOperationState } from './agent-session-refusal-retry' - -const METHODS = [ - 'agentSession.requestHandoff', - 'agentSession.setOption', - 'agentSession.send' -] as const - -const PENDING: AgentSessionRefusalOperationState = 'pending-admission' -const SETTLED: AgentSessionRefusalOperationState = 'settled-rejected' -const UNKNOWN: AgentSessionRefusalOperationState = 'unknown' - -const EXPECTED = { - 'agentSession.requestHandoff': { - structured_agent_session_unsupported: SETTLED, - agent_session_checkpoint_stale: SETTLED, - agent_session_conflict: SETTLED, - agent_session_ownership_unknown: PENDING, - agent_session_operation_conflict: SETTLED, - agent_session_operation_expired: SETTLED, - agent_session_operation_capacity: PENDING, - agent_session_operation_invalid: SETTLED, - agent_session_operation_unknown: UNKNOWN, - agent_session_item_revision_stale: SETTLED, - agent_session_already_resolved: SETTLED, - agent_session_identity_required: PENDING, - agent_session_journal_unreadable: PENDING, - execution_owner_reconciling: PENDING - }, - 'agentSession.setOption': { - structured_agent_session_unsupported: PENDING, - agent_session_checkpoint_stale: PENDING, - agent_session_conflict: PENDING, - agent_session_ownership_unknown: PENDING, - agent_session_operation_conflict: SETTLED, - agent_session_operation_expired: SETTLED, - agent_session_operation_capacity: PENDING, - agent_session_operation_invalid: SETTLED, - agent_session_operation_unknown: UNKNOWN, - agent_session_item_revision_stale: SETTLED, - agent_session_already_resolved: SETTLED, - agent_session_identity_required: PENDING, - agent_session_journal_unreadable: PENDING, - execution_owner_reconciling: PENDING - }, - 'agentSession.send': { - structured_agent_session_unsupported: PENDING, - agent_session_checkpoint_stale: PENDING, - agent_session_conflict: PENDING, - agent_session_ownership_unknown: PENDING, - agent_session_operation_conflict: SETTLED, - agent_session_operation_expired: SETTLED, - agent_session_operation_capacity: PENDING, - agent_session_operation_invalid: SETTLED, - agent_session_operation_unknown: UNKNOWN, - agent_session_item_revision_stale: SETTLED, - agent_session_already_resolved: SETTLED, - agent_session_identity_required: PENDING, - agent_session_journal_unreadable: PENDING, - execution_owner_reconciling: PENDING - } -} as const satisfies Record< - (typeof METHODS)[number], - Record -> - -describe('agentSessionRefusalOperationState', () => { - it.each(METHODS)('matches the host ledger for every %s refusal', (method) => { - for (const code of AGENT_SESSION_WIRE_REFUSAL_CODES) { - expect(agentSessionRefusalOperationState(method, code), code).toBe(EXPECTED[method][code]) - } - }) -})