fix reverse structured session handoff

This commit is contained in:
Brennan Benson
2026-08-13 12:43:05 -07:00
parent ecdd5c8dd9
commit 84727e695a
10 changed files with 283 additions and 31 deletions
@@ -41,6 +41,7 @@ let transcriptPath: string
let optionFailure: Error | null
const dispatchedModels: string[] = []
const launchedOptions: (Readonly<Record<string, string>> | undefined)[] = []
const closedTuiOwners: StructuredTuiOwner[] = []
function envelope(method: string, fields: Record<string, unknown>): AgentSessionMutationEnvelope {
return {
@@ -94,6 +95,10 @@ function handoffTransport(): StructuredAgentSessionHandoffTransport {
record.lease.ownerProcess?.spawnToken ?? record.lease.reservedSpawnToken ?? 'recovered'
),
stopRecoveredOwner: async () => undefined,
closeTuiOwner: async (owner) => {
closedTuiOwners.push(owner)
return { transcriptPath: owner.transcriptPath }
},
waitForTuiExit: async (owner) => ({ transcriptPath: owner.transcriptPath }),
waitForTuiIdleOrExit: async () => 'idle',
tuiStatus: () => 'idle'
@@ -165,6 +170,7 @@ beforeEach(async () => {
optionFailure = null
dispatchedModels.length = 0
launchedOptions.length = 0
closedTuiOwners.length = 0
const accountHome = join(root, 'codex-home')
const sessionsDir = join(accountHome, 'sessions', '2026', '08', '12')
transcriptPath = join(sessionsDir, `rollout-2026-08-12T10-00-00-${THREAD}.jsonl`)
@@ -258,6 +264,7 @@ describe('structured session handoff options', () => {
)
expect(launchedOptions).toEqual([{ model: PICKED_MODEL, effort: PICKED_EFFORT }])
expect(closedTuiOwners).toHaveLength(1)
expect(acquire.mock.calls[1]?.[0].options).toEqual({
model: PICKED_MODEL,
effort: PICKED_EFFORT
@@ -89,6 +89,7 @@ export function enqueueStructuredHandoffAfterTurn(input: {
const { deps, params, queue, tuiOwner } = input
const sessionId = params.envelope.sessionId
let tuiReadiness: 'idle' | 'exited' | null = null
let observedTuiQueue = false
input.setStatus({
owner: params.direction === 'to-tui' ? 'native' : 'tui',
direction: params.direction,
@@ -103,10 +104,21 @@ export function enqueueStructuredHandoffAfterTurn(input: {
if (params.direction === 'to-tui') {
return !activeStructuredAgentSessionTurnId(deps.session(sessionId).journal.snapshot().items)
}
if (!observedTuiQueue) {
observedTuiQueue = true
return false
}
tuiReadiness = tuiOwner
? ((await deps.transport?.waitForTuiIdleOrExit(tuiOwner, signal)) ?? null)
: null
return tuiReadiness !== null
if (tuiReadiness === 'exited') {
return true
}
if (!activeStructuredAgentSessionTurnId(deps.session(sessionId).journal.snapshot().items)) {
tuiReadiness = 'idle'
return true
}
return false
},
() => {
const record = input.requireRecord()
@@ -115,7 +127,7 @@ export function enqueueStructuredHandoffAfterTurn(input: {
input.refuse(record)
return
}
input.begin({ ...params, mode: 'now' }, tuiReadiness === 'exited')
input.begin(params, tuiReadiness === 'exited')
}
)
}
@@ -42,7 +42,11 @@ export function assertScheduledStructuredHandoffIsAdmissible(input: {
}
return
}
if (!input.tuiAlreadyExited && input.tuiStatus !== 'idle') {
if (
!input.tuiAlreadyExited &&
input.tuiStatus !== 'idle' &&
(params.mode !== 'after-turn' || activeTurn !== null)
) {
throw new Error('The agent terminal became busy before the handoff started.')
}
}
@@ -43,7 +43,9 @@ export async function handoffStructuredSessionToNative(
terminal: owner.terminal,
hostLabel: deps.transport?.hostLabel
})
const exited = await deps.transport!.waitForTuiExit(owner)
const exited = deps.transport!.closeTuiOwner
? await deps.transport!.closeTuiOwner(owner)
: await deps.transport!.waitForTuiExit(owner)
transcriptPath = exited.transcriptPath ?? owner.transcriptPath
} catch (error) {
const current = context.requireRecord(sessionId)
@@ -38,6 +38,7 @@ export type StructuredAgentSessionHandoffTransport = {
}): Promise<StructuredTuiOwner>
recoverTuiOwner(record: AgentSessionRecord): Promise<StructuredTuiOwner>
stopRecoveredOwner(record: AgentSessionRecord): Promise<void>
closeTuiOwner?(owner: StructuredTuiOwner): Promise<{ transcriptPath?: string }>
waitForTuiExit(owner: StructuredTuiOwner): Promise<{ transcriptPath?: string }>
waitForTuiIdleOrExit(
owner: StructuredTuiOwner,
@@ -40,6 +40,9 @@ let launchTui: ReturnType<typeof vi.fn<StructuredAgentSessionHandoffTransport['l
let waitForTuiExit: ReturnType<
typeof vi.fn<StructuredAgentSessionHandoffTransport['waitForTuiExit']>
>
let closeTuiOwner: ReturnType<
typeof vi.fn<NonNullable<StructuredAgentSessionHandoffTransport['closeTuiOwner']>>
>
let waitForTuiIdleOrExit: ReturnType<
typeof vi.fn<StructuredAgentSessionHandoffTransport['waitForTuiIdleOrExit']>
>
@@ -208,6 +211,7 @@ function createCoordinator(): StructuredAgentSessionHandoffCoordinator {
return { ...owner, process: record.lease.ownerProcess ?? owner.process }
},
stopRecoveredOwner,
closeTuiOwner,
waitForTuiExit,
waitForTuiIdleOrExit,
tuiStatus: () => (tuiIdle ? 'idle' : 'busy'),
@@ -287,6 +291,7 @@ beforeEach(async () => {
})
launchTui = vi.fn(async ({ fence, spawnToken }) => makeTuiOwner(fence, spawnToken))
waitForTuiExit = vi.fn(async (owner) => ({ transcriptPath: owner.transcriptPath }))
closeTuiOwner = vi.fn(async (owner) => ({ transcriptPath: owner.transcriptPath }))
waitForTuiIdleOrExit = vi.fn(async () => tuiReadiness)
reproveTuiOwner = vi.fn(async ({ record, owner }) => {
expect(record.lease.ownerProcess).toEqual(owner.process)
@@ -335,6 +340,8 @@ describe('structured session ownership handoff', () => {
link: expect.objectContaining({ linkId: forwardRecord.lease.provenHandleLinkId })
})
})
expect(closeTuiOwner).toHaveBeenCalledOnce()
expect(waitForTuiExit).not.toHaveBeenCalled()
const messages = journal.snapshot().items.filter((item) => item.body.kind === 'message')
expect(messages).toHaveLength(2)
@@ -611,13 +618,13 @@ describe('structured session ownership handoff', () => {
tuiReadiness = null
expect(await submit(request('to-native', 'after-turn'))).toMatchObject({ ok: true })
expect(coordinator.status(SESSION).phase).toBe('queued')
expect(waitForTuiExit).not.toHaveBeenCalled()
expect(closeTuiOwner).not.toHaveBeenCalled()
tuiIdle = true
tuiReadiness = 'idle'
await vi.waitFor(() => expect(waitForTuiExit).toHaveBeenCalledOnce())
await vi.waitFor(() => expect(closeTuiOwner).toHaveBeenCalledOnce())
})
it('completes a mobile-originated queued reverse after the owning TUI exits', async () => {
it('closes the owning TUI and completes a mobile-originated queued reverse', async () => {
await moveToTui()
tuiIdle = false
tuiReadiness = null
@@ -625,11 +632,10 @@ describe('structured session ownership handoff', () => {
expect(await coordinator.request('mobile-device-a', reverse)).toMatchObject({ ok: true })
expect(coordinator.status(SESSION).phase).toBe('queued')
tuiReadiness = 'exited'
await vi.waitFor(() => expect(coordinator.status(SESSION).owner).toBe('native'))
expect(reproveTuiOwner).not.toHaveBeenCalled()
expect(waitForTuiExit).toHaveBeenCalledOnce()
expect(reproveTuiOwner).toHaveBeenCalledOnce()
expect(closeTuiOwner).toHaveBeenCalledOnce()
expect(store.getRecord(SESSION)?.lease).toMatchObject({
runtimeKind: 'native',
claimStatus: 'live',
@@ -645,7 +651,7 @@ describe('structured session ownership handoff', () => {
expect(reproveTuiOwner).toHaveBeenCalledOnce()
expect(reproveTuiOwner.mock.invocationCallOrder[0]).toBeLessThan(
waitForTuiExit.mock.invocationCallOrder[0] ?? Infinity
closeTuiOwner.mock.invocationCallOrder[0] ?? Infinity
)
})
@@ -655,7 +661,7 @@ describe('structured session ownership handoff', () => {
expect(await submit(request('to-native', 'now'))).toMatchObject({ ok: true })
await waitForPhase('failed')
expect(waitForTuiExit).not.toHaveBeenCalled()
expect(closeTuiOwner).not.toHaveBeenCalled()
expect(store.getRecord(SESSION)?.lease).toMatchObject({
runtimeKind: 'tui',
claimStatus: 'live',
@@ -665,7 +671,7 @@ describe('structured session ownership handoff', () => {
it('rolls a stale reverse handle back to one durable TUI owner and retries', async () => {
await moveToTui()
waitForTuiExit.mockRejectedValueOnce(new Error('terminal_handle_stale'))
closeTuiOwner.mockRejectedValueOnce(new Error('terminal_handle_stale'))
const retryOperation = operationId()
await expectAccepted(request('to-native', 'now', { operationId: retryOperation }))
@@ -687,14 +693,14 @@ describe('structured session ownership handoff', () => {
request('to-native', 'now', { action: 'retry', operationId: retryOperation })
)
await vi.waitFor(() => expect(coordinator.status(SESSION).owner).toBe('native'))
expect(waitForTuiExit).toHaveBeenCalledTimes(2)
expect(closeTuiOwner).toHaveBeenCalledTimes(2)
})
it('keeps one recoverable owner when a queued reverse auto-fire fails', async () => {
await moveToTui()
tuiIdle = false
tuiReadiness = null
waitForTuiExit.mockRejectedValueOnce(new Error('terminal_handle_stale'))
closeTuiOwner.mockRejectedValueOnce(new Error('terminal_handle_stale'))
expect(await submit(request('to-native', 'after-turn'))).toMatchObject({ ok: true })
expect(coordinator.status(SESSION).phase).toBe('queued')
@@ -820,7 +826,7 @@ describe('structured session ownership handoff', () => {
request('to-native', 'now', { action: 'retry', operationId: retryOperation })
)
await vi.waitFor(() => expect(coordinator.status(SESSION).owner).toBe('native'))
expect(waitForTuiExit).toHaveBeenCalledOnce()
expect(closeTuiOwner).toHaveBeenCalledOnce()
})
it('abandons a failed native acquisition and retries from the stopped TUI', async () => {
@@ -852,7 +858,7 @@ describe('structured session ownership handoff', () => {
request('to-native', 'now', { action: 'retry', operationId: retryOperation })
)
await vi.waitFor(() => expect(coordinator.status(SESSION).owner).toBe('native'))
expect(waitForTuiExit).toHaveBeenCalledOnce()
expect(closeTuiOwner).toHaveBeenCalledOnce()
expect(acquireNativeCalls).toBe(2)
})
@@ -200,9 +200,10 @@ export class StructuredAgentSessionHandoffCoordinator {
)
const tuiOwner = this.tuiOwners.get(record.sessionId)
const busy =
expectedOwner === 'native'
? turnId !== null
: structuredTuiStatus(tuiOwner, this.deps.transport) !== 'idle'
turnId !== null ||
(expectedOwner === 'tui' &&
(params.mode === 'after-turn' ||
structuredTuiStatus(tuiOwner, this.deps.transport) !== 'idle'))
if (busy && params.mode === 'now') {
return this.refuseAdmitted(
callerKey,
@@ -0,0 +1,170 @@
import { mkdtemp, rm } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, describe, expect, it, vi } from 'vitest'
import { computeAgentSessionPayloadFingerprint } from '../../../shared/agent-session-mutation-envelope'
import type { AgentSessionHandoffRequest } from '../../../shared/agent-session-wire'
import { AgentSessionRecordStore } from '../../runtime/agent-session-record-store'
import { recoverStoredDeadTuiOwnerForHandoff } from '../../runtime/agent-session-handoff-record-transitions'
import { openAgentSessionJournal } from '../agent-session-journal/journal-store'
import { StructuredAgentSessionHandoffCoordinator } from './structured-agent-session-handoff'
import type { StructuredAgentSessionHandoffTransport } from './structured-agent-session-handoff-types'
const NOW = 1_800_000_000_000
const SESSION = 'session-proven-dead-retry'
const THREAD = '019fd532-7c11-7a90-b6de-4e1a2c3d5f60'
const CREATE_OPERATION = `${NOW}-00000000000000000000000000000000`
const OPERATION = `${NOW}-00000000000000000000000000000001`
const roots: string[] = []
afterEach(async () => {
await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })))
})
describe('structured session proven-dead TUI retry', () => {
it('acquires native ownership without trying to close the dead TUI again', async () => {
const root = await mkdtemp(join(tmpdir(), 'orca-handoff-dead-retry-'))
roots.push(root)
const store = await AgentSessionRecordStore.open({
directory: join(root, 'store'),
hostId: 'local'
})
const reserved = await store.reserveOwner({
sessionId: SESSION,
location: {
executionHostId: 'local',
wslDistro: null,
workspaceId: 'workspace-1',
workspaceKind: 'folder'
},
provider: 'codex',
accountHome: { variable: 'CODEX_HOME', path: join(root, 'codex-home') },
runtimeKind: 'tui',
expectedFence: null,
spawnToken: 'tui-spawn',
claimKeyId: 'key-1',
handoffOperationId: null,
probe: { outcome: 'reservation-unused' },
operation: { callerKey: 'test', operationId: CREATE_OPERATION, fingerprint: 'create' },
now: NOW
})
const tuiFence = reserved.record.lease.runtimeFence
await store.commitProcessIdentity({
sessionId: SESSION,
fence: tuiFence,
process: {
hostId: 'local',
pid: 4200,
processStartTimeMs: NOW - 1_000,
spawnToken: 'tui-spawn'
},
now: NOW
})
await store.proveOwner({
sessionId: SESSION,
fence: tuiFence,
link: {
linkId: 'tui-link',
handle: { provider: 'codex', threadId: THREAD },
origin: 'created',
mintedAtFence: tuiFence,
observedAt: NOW
},
now: NOW
})
await recoverStoredDeadTuiOwnerForHandoff(store, {
sessionId: SESSION,
expectedFence: tuiFence,
operationId: OPERATION,
probe: { outcome: 'pid-absent' },
now: NOW
})
const journal = await openAgentSessionJournal({
identity: {
sessionId: SESSION,
workspaceId: 'workspace-1',
hostId: 'local',
agent: 'codex',
providerHandle: { kind: 'codex', threadId: THREAD }
},
journalDir: join(root, 'journal')
})
const closeTuiOwner =
vi.fn<NonNullable<StructuredAgentSessionHandoffTransport['closeTuiOwner']>>()
const coordinator = new StructuredAgentSessionHandoffCoordinator({
store,
claimKeyId: 'key-1',
transport: {
hostLabel: 'Test host',
launchTui: vi.fn(),
reproveTuiOwner: vi.fn(),
recoverTuiOwner: vi.fn(),
stopRecoveredOwner: vi.fn(),
closeTuiOwner,
waitForTuiExit: vi.fn(),
waitForTuiIdleOrExit: vi.fn(),
tuiStatus: () => 'busy'
},
session: () => ({ journal, fence: store.getRecord(SESSION)?.lease.runtimeFence ?? 1 }),
suspendNative: vi.fn(),
acquireNative: async ({ fence, spawnToken }) => {
await store.commitProcessIdentity({
sessionId: SESSION,
fence,
process: {
hostId: 'local',
pid: 4300,
processStartTimeMs: NOW,
spawnToken
},
now: NOW
})
return store.proveOwner({
sessionId: SESSION,
fence,
link: {
linkId: 'native-link',
handle: { provider: 'codex', threadId: THREAD },
origin: 'resumed',
mintedAtFence: fence,
observedAt: NOW
},
now: NOW
})
},
acquireNativeStop: vi.fn(async () => true),
importTuiHistory: vi.fn(),
publish: vi.fn(),
schedule: async (_sessionId, task) => task(),
now: () => NOW
})
const fields = {
direction: 'to-native' as const,
mode: 'now' as const,
action: 'retry' as const
}
const request: AgentSessionHandoffRequest = {
envelope: {
sessionId: SESSION,
clientOperationId: OPERATION,
expectedRuntimeFence: store.getRecord(SESSION)?.lease.runtimeFence ?? null,
payloadFingerprint: computeAgentSessionPayloadFingerprint({
method: 'agentSession.requestHandoff',
sessionId: SESSION,
fields
})
},
...fields
}
expect(coordinator.status(SESSION)).toMatchObject({ phase: 'failed', owner: 'tui' })
expect(await coordinator.request('client-1', request)).toMatchObject({ ok: true })
await vi.waitFor(() => expect(coordinator.status(SESSION).owner).toBe('native'))
expect(closeTuiOwner).not.toHaveBeenCalled()
expect(store.getRecord(SESSION)?.lease).toMatchObject({
runtimeKind: 'native',
claimStatus: 'live',
handoffStage: null
})
})
})
@@ -213,6 +213,7 @@ describe('structured TUI launch tab binding', () => {
waitForTerminal(): Promise<unknown>
waitForStructuredTuiProof(): Promise<{ transcriptPath?: string }>
waitForStructuredTuiPtyExit(): Promise<void>
closeTerminal(handle: string): Promise<unknown>
handles: Map<
string,
{
@@ -250,6 +251,8 @@ describe('structured TUI launch tab binding', () => {
internal.waitForStructuredTuiProof = waitForStructuredTuiProof
const waitForStructuredTuiPtyExit = vi.fn(async () => {})
internal.waitForStructuredTuiPtyExit = waitForStructuredTuiPtyExit
const closeTerminal = vi.fn(async () => undefined)
internal.closeTerminal = closeTerminal
readStructuredTuiProcessIdentity.mockResolvedValue({
hostId: 'local',
pid: 4243,
@@ -325,6 +328,37 @@ describe('structured TUI launch tab binding', () => {
'idle'
)
explicitStatus = null
const livePty = (
runtime as unknown as {
ptysById: Map<
string,
{
tailBuffer: string[]
tailPartialLine: string
preview: string
lastAgentStatus: null
lastAgentStatusObservedLive: boolean
}
>
}
).ptysById.get('pty-structured')!
Object.assign(livePty, {
tailBuffer: [
'OpenAI Codex (v0.147.0)',
'model: gpt-5.6-terra',
'directory: /tmp/structured-handoff'
],
tailPartialLine: '',
preview: '',
lastAgentStatus: null,
lastAgentStatusObservedLive: false
})
expect(transport.tuiStatus(owner)).toBe('idle')
await expect(transport.waitForTuiIdleOrExit(owner, new AbortController().signal)).resolves.toBe(
'idle'
)
const pty = (
runtime as unknown as {
ptysById: Map<string, { connected: boolean; launchToken: string | null }>
@@ -348,6 +382,11 @@ describe('structured TUI launch tab binding', () => {
expect(waitForStructuredTuiPtyExit).toHaveBeenCalledWith('pty-structured')
expect(waitForStructuredTuiProof).toHaveBeenCalledOnce()
await expect(transport.closeTuiOwner?.(rebound)).resolves.toEqual({
transcriptPath: '/tmp/rollout.jsonl'
})
expect(closeTerminal).toHaveBeenCalledWith(rebound.terminal.handle)
explicitStatus = null
pty.connected = false
await expect(
+21 -11
View File
@@ -9347,20 +9347,26 @@ export class OrcaRuntimeService {
},
stopRecoveredOwner: (record) => this.stopStructuredSessionProcess(record),
tuiStatus: (owner) => this.structuredTuiStatus(owner),
stopFailedTuiLaunch: async (owner) => {
closeTuiOwner: (owner) => this.closeStructuredTuiOwner(owner),
stopFailedTuiLaunch: async (owner) => void (await this.closeStructuredTuiOwner(owner))
}
}
private async closeStructuredTuiOwner(
owner: StructuredTuiOwner
): Promise<{ transcriptPath?: string }> {
if (this.ptysById.get(owner.terminal.ptyId)?.connected) {
const current = this.refreshStructuredTuiOwnerBinding(owner)
try {
await this.closeTerminal(current.terminal.handle)
} catch (error) {
if (this.ptysById.get(owner.terminal.ptyId)?.connected) {
const current = this.refreshStructuredTuiOwnerBinding(owner)
try {
await this.closeTerminal(current.terminal.handle)
} catch (error) {
if (this.ptysById.get(owner.terminal.ptyId)?.connected) {
throw error
}
}
throw error
}
await this.waitForStructuredTuiOwnerExit(owner)
}
}
await this.waitForStructuredTuiOwnerExit(owner)
return owner.transcriptPath ? { transcriptPath: owner.transcriptPath } : {}
}
private refreshStructuredTuiOwnerBinding(owner: StructuredTuiOwner): StructuredTuiOwner {
@@ -9495,8 +9501,12 @@ export class OrcaRuntimeService {
}
if (pty?.connected) {
const text = buildTerminalWaitText(pty.tailBuffer, pty.tailPartialLine, pty.preview)
const blocked = detectTerminalWaitBlockedReason(text) !== null
if (!blocked && isKnownReadyPromptPreview(text)) {
return 'idle'
}
return hasStructuredTuiIdleEvidence({
blocked: detectTerminalWaitBlockedReason(text) !== null,
blocked,
status: pty.lastAgentStatus,
statusObservedLive: pty.lastAgentStatusObservedLive
})