fix(orchestration): recover mail from current agent readiness

This commit is contained in:
Brennan Benson
2026-11-13 01:50:43 -08:00
parent 898adac720
commit 9c8ce9a35c
18 changed files with 594 additions and 119 deletions
@@ -6,17 +6,20 @@ import type { OrchestrationMailboxTerminal } from '../orchestration/mailbox/rout
export abstract class RuntimeMailboxDeliveryServices {
protected abstract resolveOrchestrationPointerSubmitTarget(
ptyId: string,
foreground?: MailboxForeground
foreground?: MailboxForeground,
stagedText?: string
): OrchestrationMailboxPointerSubmitTarget | null
protected abstract isAgentSettledForDelivery(
ptyId: string,
foreground?: MailboxForeground
foreground?: MailboxForeground,
stagedText?: string
): boolean
protected abstract isMailboxForegroundCandidate(terminal: OrchestrationMailboxTerminal): boolean
protected abstract mailboxTerminalAdmission(
terminal: OrchestrationMailboxTerminal,
foreground?: MailboxForeground
foreground?: MailboxForeground,
stagedText?: string
): OrchestrationMailboxTerminal
protected abstract deliverPendingMessagesForPty(ptyId: string): void
protected abstract needsMailboxForegroundEvaluation(ptyId: string): boolean
@@ -247,13 +247,13 @@ export abstract class OrcaRuntimeWithStopRequestedPtyIds extends OrcaRuntimeWith
isForegroundCandidate: (terminal) => this.isMailboxForegroundCandidate(terminal),
needsForegroundEvaluation: (ptyId) => this.needsMailboxForegroundEvaluation(ptyId),
onDeliveryDecision: (ptyId, decision) => this.armDeliveryRecheck(ptyId, decision),
isAgentSettledForDelivery: (ptyId, foreground) =>
this.isAgentSettledForDelivery(ptyId, foreground),
isAgentSettledForDelivery: (ptyId, foreground, stagedText) =>
this.isAgentSettledForDelivery(ptyId, foreground, stagedText),
isPtyAwaitingUserInput: (ptyId) => this.isPtyAwaitingUserInput(ptyId),
getMessageWaiters: (mailboxHandle) => this.messageWaiters.get(mailboxHandle),
getCliCommand: (terminalHandle) => this.getTerminalOrchestrationCliCommand(terminalHandle),
resolveSubmitTarget: (ptyId, foreground) =>
this.resolveOrchestrationPointerSubmitTarget(ptyId, foreground),
resolveSubmitTarget: (ptyId, foreground, stagedText) =>
this.resolveOrchestrationPointerSubmitTarget(ptyId, foreground, stagedText),
isLeafPtyProvenAbsent: (ptyId) => this.isLeafPtyProvenAbsent(ptyId),
redriveTerminal: (ptyId) => this.deliverPendingMessagesForPty(ptyId),
redriveMailbox: (mailboxHandle, reservedTypes) =>
@@ -4,6 +4,7 @@ import type {
} from './mailbox-pointer-delivery-contract'
import type { OrchestrationMessageWaiter } from './mailbox-pointer-eligibility'
import type { OrchestrationMailboxPointerDelivery } from './mailbox-pointer-delivery'
import { isMailboxTerminalTakingInput } from '../routing/mailbox-owner'
import type { OrchestrationMailboxPointerSubmitTarget } from './mailbox-pointer-submit'
type ForegroundMailboxBatch = Map<string, ReadonlySet<string> | undefined>
@@ -34,11 +35,18 @@ export class MailboxForegroundDelivery {
.probeForeground(ptyId, target.processIncarnation)
.then((foreground) => {
this.mailboxesByPty.delete(ptyId)
const current = deps.resolveSubmitTarget(ptyId)
const current = deps.resolveSubmitTarget(ptyId, foreground.foreground)
if (current?.processIncarnation !== target.processIncarnation) {
return
}
const applied = delivery.applyForeground(ptyId, foreground, foreground.foreground)
const applied = delivery.applyForeground(
ptyId,
foreground,
foreground.foreground,
foreground.foreground === 'agent' &&
isMailboxTerminalTakingInput(current.terminal) &&
deps.isAgentSettledForDelivery(ptyId, foreground.foreground)
)
if (applied && delivery.isForegroundHeld(ptyId)) {
return
}
@@ -76,7 +76,8 @@ export class MailboxForegroundEvidence {
applyForeground(
ptyId: string,
proof: OrchestrationMailboxForegroundObservation | undefined,
foreground: MailboxForeground = 'shell'
foreground: MailboxForeground = 'shell',
admitted = false
): boolean {
const observation = this.foregroundObservations.get(ptyId)
if (
@@ -93,7 +94,7 @@ export class MailboxForegroundEvidence {
if (foreground === 'shell') {
observation.held = true
observation.contradiction++
} else if (foreground !== 'unknown') {
} else if (foreground === 'agent' && admitted) {
observation.held = false
}
if (wasHeld && !observation.held) {
@@ -0,0 +1,179 @@
import { readRuntimeFixture } from '../../../terminal/wait/agent-transcript-replay-test-harness'
import { rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { afterEach, describe, expect, it, vi } from 'vitest'
import {
createDatabase,
createRuntime,
createBoundRun,
insertDirectRunMessage,
PTY_ID,
PANE_KEY,
temporaryDirectories
} from './orchestration-mailbox-notification-test-harness'
import { settledWriteStub } from '../../../../providers/testing/settled-pty-write-stub'
vi.mock('electron', () => ({
app: { getPath: vi.fn(() => tmpdir()), isPackaged: false },
BrowserWindow: { fromId: vi.fn(() => null) },
ipcMain: { on: vi.fn(), removeListener: vi.fn() },
webContents: { fromId: vi.fn(() => null) }
}))
const COMPOSER = `\x1b[2J\x1b[H${readRuntimeFixture('claude-2-1-316-default-unlabelled-empty').replaceAll('\x1b]0;✳ Claude Code\x07', '')}`
describe('mail-local foreground recovery readiness', () => {
afterEach(() => {
vi.restoreAllMocks()
vi.useRealTimers()
for (const directory of temporaryDirectories.splice(0)) {
rmSync(directory, { recursive: true, force: true })
}
})
it.each([
{
veto: 'login',
body: '\x1b[2J\x1b[HWelcome to Claude Code\r\nSelect login method:\r\n1. Claude account with subscription\r\n2. Anthropic Console account'
},
{
veto: 'dialog',
body: '\x1b[2J\x1b[HDo you want to create hello.txt?\r\n❯ 1. Yes\r\n 2. No\r\nEsc to cancel · Tab to amend'
},
{
veto: 'draft',
body: `\x1b[2J\x1b[H${readRuntimeFixture('claude-2-1-316-default-unlabelled-middle').replaceAll('\x1b]0;✳ Claude Code\x07', '')}`
},
{ veto: 'unhydrated', body: '' },
{ veto: 'restoring', body: '' },
{
veto: 'retained-idle dialog',
body: '\x1b[2J\x1b[HDo you want to create hello.txt?\r\n❯ 1. Yes\r\n 2. No\r\nEsc to cancel · Tab to amend',
retainIdle: true
}
])(
'requires a current ready screen after expired identity, preserving the $veto veto',
async ({
veto,
body,
retainIdle = false
}: {
veto: string
body: string
retainIdle?: boolean
}) => {
vi.useFakeTimers()
const db = createDatabase('orca-mail-expired-readiness-')
const row = {
paneKey: PANE_KEY,
connectionId: null,
agentType: 'claude' as const,
state: 'done' as 'done' | 'waiting',
prompt: '',
receivedAt: Date.now(),
stateStartedAt: 1
}
const originalRow = { ...row }
const { runtime, write } = createRuntime(db, {
launchAgent: null,
getAgentStatusSnapshot: () => [row],
getAgentStatusSnapshotForPane: () => [row]
})
let shell = true
let unavailable = false
runtime.setPtyController({
write,
writeWithSettlement: settledWriteStub(write),
kill: vi.fn(),
getSize: () => ({ cols: 100, rows: 30 }),
getForegroundProcess: async () => null,
supportsForegroundObservation: () => true,
inspectProcess: async () => {
if (unavailable) {
throw new Error('foreground inspection unavailable')
}
return {
foregroundProcess: shell ? 'zsh' : 'claude',
hasChildProcesses: !shell,
foregroundObservation: {
ptyId: PTY_ID,
ptyIncarnationId: 'mailbox-incarnation',
authorityGeneration: 'test',
observationEpoch: 1,
capturedAgeMs: 0,
verdict: 'observed',
source: 'pty-master',
foregroundKind: shell ? 'shell' : 'non-shell',
processName: shell ? 'zsh' : 'claude'
}
}
}
})
try {
await runtime.listTerminals()
runtime.onPtyData(PTY_ID, `\x1b]0;✳ Claude Code\x07${COMPOSER}`, 1)
await vi.advanceTimersByTimeAsync(1000)
if (!retainIdle) {
runtime.onPtyData(PTY_ID, '\x1b]0;workspace shell\x07', 2)
}
const run = createBoundRun(db, 'Current readiness after long hold')
const message = insertDirectRunMessage(db, run.id, 'Wait for the composer')
runtime.notifyMessageArrived(`run:${run.id}`, 'status')
await vi.advanceTimersByTimeAsync(31 * 60 * 1000)
expect(write).not.toHaveBeenCalled()
expect(runtime.mailTerminal?.lastAgentStatus).toBe(retainIdle ? 'idle' : null)
if (retainIdle) {
row.state = 'waiting'
row.receivedAt = Date.now()
}
shell = false
runtime.setMailScreenRestoring(veto === 'restoring')
const unreadable =
veto === 'unhydrated'
? vi.spyOn(runtime.idleEvidenceSource, 'readScreenLines').mockReturnValue(null)
: null
if (body) {
runtime.onPtyData(PTY_ID, body, 3)
}
await vi.advanceTimersByTimeAsync(65000)
expect(write).not.toHaveBeenCalled()
expect(db.getMessageById(message.id)).toMatchObject({
read: 0,
delivered_at: null,
pointer_enter_pending: 0
})
if (!retainIdle) {
expect(row).toEqual(originalRow)
}
unreadable?.mockRestore()
runtime.setMailScreenRestoring(false)
runtime.onPtyData(PTY_ID, COMPOSER, 4)
if (retainIdle) {
row.state = 'done'
unavailable = true
await vi.advanceTimersByTimeAsync(65000)
expect(write).not.toHaveBeenCalled()
unavailable = false
}
await vi.advanceTimersByTimeAsync(65000)
expect(write.mock.calls.map(([, data]) => data)).toEqual([
expect.stringContaining('You have 1 orchestration message'),
'\r'
])
expect(db.getMessageById(message.id)).toMatchObject({
read: 0,
delivered_at: expect.any(String),
pointer_enter_pending: 0
})
if (!retainIdle) {
expect(row).toEqual(originalRow)
}
expect(runtime.mailTerminal?.lastAgentStatus).toBe(retainIdle ? 'idle' : null)
await vi.advanceTimersByTimeAsync(65000)
expect(write).toHaveBeenCalledTimes(2)
} finally {
db.close()
}
}
)
})
@@ -1,3 +1,5 @@
import { resolvePtyForegroundMembers } from '../../../../pty/process-retirement/pty-master-foreground-observation'
import { readRuntimeFixture } from '../../../terminal/wait/agent-transcript-replay-test-harness'
import { rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { afterEach, describe, expect, it, vi } from 'vitest'
@@ -203,6 +205,7 @@ describe('owed mailbox foreground revalidation', () => {
write,
writeWithSettlement: settledWriteStub(write),
kill: vi.fn(),
getSize: () => ({ cols: 100, rows: 30 }),
getForegroundProcess: async () => null,
supportsForegroundObservation: () => true,
inspectProcess: async () => {
@@ -228,7 +231,7 @@ describe('owed mailbox foreground revalidation', () => {
}
})
await runtime.listTerminals()
runtime.onPtyData(PTY_ID, '\x1b]0;✳ Claude Code\x07', 1)
runtime.onPtyData(PTY_ID, readRuntimeFixture('claude-2-1-316-default-unlabelled-empty'), 1)
await vi.advanceTimersByTimeAsync(1000)
const run = createBoundRun(db, 'Hook boundary')
runtime.onPtyData(PTY_ID, '\x1b]0;workspace shell\x07', 2)
@@ -265,8 +268,6 @@ describe('owed mailbox foreground revalidation', () => {
expect(write).not.toHaveBeenCalled()
expect(db.getMessageById(message.id)).toMatchObject({ read: 0, delivered_at: null })
mode = 'agent'
// Refreshing the already-retained row's lease alone does not establish a new event.
row.receivedAt = Date.now()
await vi.advanceTimersByTimeAsync(65000)
expect(write.mock.calls.map(([, data]) => data)).toEqual([
expect.stringContaining('You have 1 orchestration message'),
@@ -294,6 +295,78 @@ describe('owed mailbox foreground revalidation', () => {
compatibilityDb.close()
})
it.each([
{ other: 'vim', command: '/usr/local/bin/codex-aarch64-apple-darwin' },
{ other: 'less', command: 'node /opt/node_modules/@openai/codex/bin/codex.js' },
{ other: 'node', command: '/usr/local/bin/codex' }
])(
'keeps retained idle mail out of $other and resumes with $command',
async ({ other, command }) => {
vi.useFakeTimers()
const db = createDatabase('orca-mail-other-program-')
const { runtime, write } = createRuntime(db)
let name = 'zsh'
runtime.setPtyController({
write,
writeWithSettlement: settledWriteStub(write),
kill: vi.fn(),
getForegroundProcess: async () => null,
supportsForegroundObservation: () => true,
inspectProcess: async () => ({
foregroundProcess: name,
hasChildProcesses: name !== 'zsh',
foregroundObservation: {
ptyId: PTY_ID,
ptyIncarnationId: 'mailbox-incarnation',
authorityGeneration: 'test',
observationEpoch: 1,
capturedAgeMs: 0,
verdict: 'observed',
source: 'pty-master',
foregroundKind: name === 'zsh' ? 'shell' : 'non-shell',
processName: name
}
})
})
try {
await driveToLiveIdle(runtime)
const run = createBoundRun(db, 'Other foreground program')
const message = insertDirectRunMessage(db, run.id, 'Held for intended agent')
runtime.notifyMessageArrived(`run:${run.id}`, 'status')
await vi.advanceTimersByTimeAsync(1000)
expect(write).not.toHaveBeenCalled()
name = other
await vi.advanceTimersByTimeAsync(65000)
expect(write).not.toHaveBeenCalled()
expect(db.getMessageById(message.id)).toMatchObject({ read: 0, delivered_at: null })
// The existing host reader canonicalizes an npm wrapper's argv to its agent identity.
const sample = resolvePtyForegroundMembers(
[
{ pid: 1, ppid: 0, pgid: 1, tpgid: 2, stat: 'S', command: '/bin/zsh' },
{ pid: 2, ppid: 1, pgid: 2, tpgid: 2, stat: 'S', command }
],
2
)
expect(sample).toMatchObject({ verdict: 'observed', foregroundKind: 'non-shell' })
name = sample.verdict === 'observed' ? (sample.processName ?? '') : ''
await vi.advanceTimersByTimeAsync(65000)
expect(write.mock.calls.map(([, data]) => data)).toEqual([
expect.stringContaining('You have 1 orchestration message'),
'\r'
])
expect(db.getMessageById(message.id)).toMatchObject({
read: 0,
delivered_at: expect.any(String),
pointer_enter_pending: 0
})
await vi.advanceTimersByTimeAsync(65000)
expect(write).toHaveBeenCalledTimes(2)
} finally {
db.close()
}
}
)
it('ends backed-off obligations on consumption, rerouting, retirement and disconnect, and recovers on reconnect', async () => {
vi.useFakeTimers()
for (const ending of ['consumed', 'rerouted', 'retired', 'disconnected'] as const) {
@@ -26,13 +26,18 @@ export type PointerDeliveryDependencies<TWaiter extends OrchestrationMessageWait
getDb: () => OrchestrationDb | null
getTerminal: (ptyId: string) => OrchestrationMailboxTerminal | null
/** Whether the terminal may take the pointer plus Enter now: at rest, or mid-turn (input queues). */
isAgentSettledForDelivery: (ptyId: string, foreground?: MailboxForeground) => boolean
isAgentSettledForDelivery: (
ptyId: string,
foreground?: MailboxForeground,
stagedText?: string
) => boolean
isPtyAwaitingUserInput: (ptyId: string) => boolean
getMessageWaiters: (mailboxHandle: string) => ReadonlySet<TWaiter> | undefined
getCliCommand: (terminalHandle: string) => OrchestrationCliCommand
resolveSubmitTarget: (
ptyId: string,
foreground?: MailboxForeground
foreground?: MailboxForeground,
stagedText?: string
) => OrchestrationMailboxPointerSubmitTarget | null
readForeground: (ptyId: string) => Promise<MailboxForeground>
isLeafPtyProvenAbsent: (ptyId: string) => Promise<boolean>
@@ -201,7 +201,7 @@ export class OrchestrationMailboxPointerDelivery<TWaiter extends OrchestrationMe
isForegroundHeld(ptyId: string): boolean {
const hold = this.state.foreground.foregroundHold(ptyId)
if (hold && hold !== this.deps.resolveSubmitTarget(ptyId)?.processIncarnation) {
this.state.retirePty(ptyId)
this.state.foreground.retirePty(ptyId)
return false
}
return !!hold
@@ -163,6 +163,68 @@ describe('retiring a pty mid-delivery', () => {
}
})
it('preserves staged and parked obligations when an obsolete hold is queried before retirement', async () => {
vi.useFakeTimers()
const db = new OrchestrationDb(':memory:')
const writePty = vi.fn(() => WRITE_ACCEPTED)
let incarnation = 'inc-1'
let resolveOldAbsence: (() => void) | undefined
let firstAbsence = true
const deps = {
...pointerDeps(db, writePty),
mailboxOwner: { resolve: (_terminal: unknown, handle: string) => handle },
isLeafPtyProvenAbsent: () => {
if (firstAbsence) {
firstAbsence = false
return new Promise<boolean>((resolve) => {
resolveOldAbsence = () => resolve(false)
})
}
return Promise.resolve(false)
},
resolveSubmitTarget: () => ({ terminal: TERMINAL, processIncarnation: incarnation })
}
const delivery = new OrchestrationMailboxPointerDelivery(deps as never)
try {
const first = db.insertMessage({ from: 'a', to: 'run:run-1', subject: 'staged' })
const second = db.insertMessage({ from: 'a', to: 'run:run-2', subject: 'parked' })
delivery.deliver(TERMINAL, { mailboxHandle: 'run:run-1' })
await vi.advanceTimersByTimeAsync(0)
delivery.deliver(TERMINAL, { mailboxHandle: 'run:run-2' })
expect(writePty).toHaveBeenCalledTimes(1)
await vi.advanceTimersByTimeAsync(500)
expect(resolveOldAbsence).toBeTypeOf('function')
const proof = delivery.captureForegroundObservation('pty-1')
delivery.applyForeground('pty-1', proof, 'shell')
incarnation = 'inc-2'
expect(delivery.isForegroundHeld('pty-1')).toBe(false)
delivery.retirePty('pty-1')
expect(db.getMessageById(first.id)).toMatchObject({
read: 0,
delivered_at: null,
pointer_enter_pending: 0
})
await vi.advanceTimersByTimeAsync(0)
expect(deps.redriveMailbox).toHaveBeenCalledWith('run:run-1', undefined)
expect(deps.redriveMailbox).toHaveBeenCalledWith('run:run-2', undefined)
delivery.deliver(TERMINAL, { mailboxHandle: 'run:run-2' })
await vi.advanceTimersByTimeAsync(0)
resolveOldAbsence?.()
await vi.advanceTimersByTimeAsync(1000)
expect(writePty).toHaveBeenCalledTimes(3)
expect(db.getMessageById(second.id)).toMatchObject({
read: 0,
delivered_at: expect.any(String),
pointer_enter_pending: 0
})
expect(db.getMessageById(first.id)?.pointer_enter_pending).toBe(0)
} finally {
delivery.retirePty('pty-1')
db.close()
vi.useRealTimers()
}
})
it('releases a pointer whose Enter never fired', async () => {
vi.useFakeTimers()
const db = new OrchestrationDb(':memory:')
@@ -68,6 +68,12 @@ export function stageOrchestrationMailboxPointer<TWaiter extends OrchestrationMe
) {
return
}
const pointerText = formatMessagePointer(
args.messages.length,
args.mailboxHandle,
args.deps.getCliCommand(expectedTarget.terminal.handle),
args.heldDeliveryId
)
const flight = args.state.beginFlight(ptyId)
flight.processIncarnation = expectedTarget.processIncarnation
flight.stagedMessageIds = args.messages.map((message) => message.id)
@@ -94,7 +100,7 @@ export function stageOrchestrationMailboxPointer<TWaiter extends OrchestrationMe
preserveAmbiguousWrite()
return
}
finishPointerWriteAndStageEnter(args, ptyId, flight, expectedTarget, settlement)
finishPointerWriteAndStageEnter(args, ptyId, flight, expectedTarget, settlement, pointerText)
}
const preserveAmbiguousWrite = (): void => {
if (!args.state.isCurrentFlight(ptyId, flight)) {
@@ -104,15 +110,7 @@ export function stageOrchestrationMailboxPointer<TWaiter extends OrchestrationMe
args.settle(ptyId, flight)
}
try {
const writeResult = args.deps.writePty(
ptyId,
formatMessagePointer(
args.messages.length,
args.mailboxHandle,
args.deps.getCliCommand(expectedTarget.terminal.handle),
args.heldDeliveryId
)
)
const writeResult = args.deps.writePty(ptyId, pointerText)
if (isSettledWrite(writeResult)) {
settlePointerWrite(writeResult)
return
@@ -128,7 +126,8 @@ function finishPointerWriteAndStageEnter<TWaiter extends OrchestrationMessageWai
ptyId: string,
flight: OrchestrationMailboxDeliveryFlight,
expectedTarget: OrchestrationMailboxPointerSubmitTarget,
settlement: Extract<WriteSettlement, { outcome: 'accepted' | 'refused' }>
settlement: Extract<WriteSettlement, { outcome: 'accepted' | 'refused' }>,
pointerText: string
): void {
let delayedSettle = false
try {
@@ -153,6 +152,7 @@ function finishPointerWriteAndStageEnter<TWaiter extends OrchestrationMessageWai
resolveSubmitTarget: args.deps.resolveSubmitTarget,
isLeafPtyProvenAbsent: args.deps.isLeafPtyProvenAbsent,
readForeground: args.deps.readForeground,
isAgentSettledForDelivery: args.deps.isAgentSettledForDelivery,
isPtyAwaitingUserInput: args.deps.isPtyAwaitingUserInput,
writePty: args.deps.writePty,
settle: args.settle,
@@ -165,7 +165,8 @@ function finishPointerWriteAndStageEnter<TWaiter extends OrchestrationMessageWai
newestSequence: args.newestSequence,
ptyId,
flight,
expectedTarget
expectedTarget,
stagedText: pointerText.trim()
}
)
flight.submitEnter = submitEnter
@@ -215,16 +215,16 @@ export class OrchestrationMailboxPointerState {
}
const flight = this.flightsByPtyId.get(ptyId)
this.flightsByPtyId.delete(ptyId)
const releasedMailboxes = new Set(this.parkedDeliveriesByPtyId.get(ptyId)?.keys())
this.parkedDeliveriesByPtyId.delete(ptyId)
const releasedMailboxes: string[] = []
for (const mailboxHandle of this.watermarkMailboxesByPtyId.get(ptyId) ?? []) {
const owner = this.watermarkByMailbox.get(mailboxHandle)
if (owner?.ptyId === ptyId && this.clearWatermark(mailboxHandle, owner.sequence, ptyId)) {
releasedMailboxes.push(mailboxHandle)
releasedMailboxes.add(mailboxHandle)
}
}
this.watermarkMailboxesByPtyId.delete(ptyId)
return { flight, releasedMailboxes }
return { flight, releasedMailboxes: [...releasedMailboxes] }
}
private removeWatermarkPtyIndex(mailbox: string, ptyId: string): void {
@@ -22,8 +22,14 @@ type PointerSubmitDependencies = {
getDb: () => OrchestrationDb | null
resolveSubmitTarget: (
ptyId: string,
foreground?: MailboxForeground
foreground?: MailboxForeground,
stagedText?: string
) => OrchestrationMailboxPointerSubmitTarget | null
isAgentSettledForDelivery?: (
ptyId: string,
foreground?: MailboxForeground,
stagedText?: string
) => boolean
readForeground: (ptyId: string) => Promise<MailboxForeground>
isLeafPtyProvenAbsent: (ptyId: string) => Promise<boolean>
isPtyAwaitingUserInput: (ptyId: string) => boolean
@@ -47,6 +53,7 @@ export function submitOrchestrationMailboxPointer(
ptyId: string
flight: OrchestrationMailboxDeliveryFlight
expectedTarget: OrchestrationMailboxPointerSubmitTarget
stagedText?: string
foregroundFollowup?: boolean
}
): void {
@@ -79,7 +86,7 @@ export function submitOrchestrationMailboxPointer(
finalizeReservation = false
return
}
const target = deps.resolveSubmitTarget(input.ptyId, foreground.foreground)
const target = deps.resolveSubmitTarget(input.ptyId, foreground.foreground, input.stagedText)
const terminal =
target?.terminal.handle === input.expectedTarget.terminal.handle &&
target.processIncarnation === input.expectedTarget.processIncarnation
@@ -88,10 +95,18 @@ export function submitOrchestrationMailboxPointer(
const sameMailbox =
terminal && deps.mailboxOwner.resolve(terminal, input.mailboxHandle) === input.mailboxHandle
const queueSafe = !!terminal && isMailboxTerminalTakingInput(terminal)
const settled =
deps.isAgentSettledForDelivery?.(input.ptyId, foreground.foreground, input.stagedText) !==
false
if (!terminal || !sameMailbox) {
clearAndRedrive = true
} else if (
!deps.state.foreground.applyForeground(input.ptyId, foreground, foreground.foreground)
!deps.state.foreground.applyForeground(
input.ptyId,
foreground,
foreground.foreground,
queueSafe && settled && !deps.isPtyAwaitingUserInput(input.ptyId)
)
) {
if (!input.foregroundFollowup) {
submitOrchestrationMailboxPointer(deps, { ...input, foregroundFollowup: true })
@@ -107,12 +122,14 @@ export function submitOrchestrationMailboxPointer(
redriveClearedPointer = false
} else if (
(terminal.lastAgentStatusObservedLive && terminal.lastAgentStatus === null) ||
(queueSafe && !settled) ||
deps.isPtyAwaitingUserInput(input.ptyId)
) {
// A neutral title can outlive the foreground check, and an open dialog would take the
// Enter as its answer; no Enter has been attempted yet.
deps.state.deferFlightUntilIdle(input.ptyId)
input.flight.submitEnter = () => submitOrchestrationMailboxPointer(deps, input)
deps.onDeliveryDecision?.(input.ptyId, { foreground: true })
preserveFlight = true
} else if (!queueSafe) {
releaseWithoutRedrive = true
@@ -81,6 +81,18 @@ class MailboxRuntime extends OrcaRuntimeService {
this.armDeliveryRecheck(ptyId)
}
setMailScreenRestoring(restoring: boolean): void {
if (restoring) {
this.headlessHydrationState.set(PTY_ID, 'pending')
} else {
this.headlessHydrationState.delete(PTY_ID)
}
}
get mailTerminal() {
return this.getOrchestrationMailboxTerminal(PTY_ID)
}
get idleEvidenceSource() {
return this.tuiIdleEvidenceSource
}
@@ -1,3 +1,4 @@
import { readRuntimeFixture } from '../../../terminal/wait/agent-transcript-replay-test-harness'
import { rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { afterEach, describe, expect, it, vi } from 'vitest'
@@ -229,14 +230,19 @@ describe('mailbox delivery into a pane awaiting the user', () => {
db.close()
})
it.each(['plain', 'title-setting'])(
'delivers after a silent foreground return in a %s shell and refreshes its receipt',
async (shellKind) => {
it.each([
{ shellKind: 'plain', launchAgent: null, longHold: false },
{ shellKind: 'title-setting', launchAgent: null, longHold: false },
{ shellKind: 'title-setting', launchAgent: 'claude' as const, longHold: true },
{ shellKind: 'title-setting', launchAgent: null, longHold: true }
])(
'delivers after a silent foreground return ($shellKind, launch=$launchAgent, expired=$longHold)',
async ({ shellKind, launchAgent, longHold }) => {
vi.useFakeTimers()
const db = createDatabase('orca-silent-mail-return-')
const receivedAt = Date.now()
const { runtime } = createRuntime(db, {
launchAgent: null,
launchAgent,
getAgentStatusSnapshot: () => [
{
paneKey: PANE_KEY,
@@ -250,37 +256,51 @@ describe('mailbox delivery into a pane awaiting the user', () => {
]
})
let shell = false
const write = vi.fn((_ptyId: string, _data: string) => true)
let inspections = 0
let echoSequence = 3
const write = vi.fn((_ptyId: string, data: string) => {
if (longHold) {
const composer =
data === '\r'
? readRuntimeFixture('claude-2-1-316-default-unlabelled-empty').replaceAll(
'\x1b]0;✳ Claude Code\x07',
''
)
: `\x1b[2J\x1b[H\x1b[?25h${'─'.repeat(40)}\r\n❯ ${data.trim()}\r\n${'─'.repeat(40)}\x1b[2;${data.trim().length + 3}H`
runtime.onPtyData(PTY_ID, composer, echoSequence++)
}
return true
})
runtime.setPtyController({
write,
writeWithSettlement: settledWriteStub(write),
kill: vi.fn(),
getSize: () => ({ cols: 200, rows: 30 }),
getForegroundProcess: async () => 'zsh',
confirmShellForeground: async () => shell,
supportsForegroundObservation: () => true,
inspectProcess: async () => ({
foregroundProcess: shell ? 'zsh' : 'claude',
hasChildProcesses: !shell,
foregroundObservation: {
ptyId: PTY_ID,
ptyIncarnationId: 'mailbox-incarnation',
authorityGeneration: 'test',
observationEpoch: 1,
capturedAgeMs: 0,
verdict: 'observed',
source: 'pty-master',
foregroundKind: shell ? 'shell' : 'non-shell',
processName: shell ? 'zsh' : 'claude'
inspectProcess: async () => {
inspections++
return {
foregroundProcess: shell ? 'zsh' : 'claude',
hasChildProcesses: !shell,
foregroundObservation: {
ptyId: PTY_ID,
ptyIncarnationId: 'mailbox-incarnation',
authorityGeneration: 'test',
observationEpoch: 1,
capturedAgeMs: 0,
verdict: 'observed',
source: 'pty-master',
foregroundKind: shell ? 'shell' : 'non-shell',
processName: shell ? 'zsh' : 'claude'
}
}
})
}
})
const run = createBoundRun(db, 'Silent foreground return')
await runtime.listTerminals()
runtime.onPtyData(
PTY_ID,
'\x1b]0;✳ Claude Code\x07\x1b[?2004h╭──────╮\r\n│ ❯  │\r\n╰──────╯',
1
)
runtime.onPtyData(PTY_ID, readRuntimeFixture('claude-2-1-316-default-unlabelled-empty'), 1)
await vi.advanceTimersByTimeAsync(1000)
shell = true
if (shellKind === 'title-setting') {
@@ -299,9 +319,14 @@ describe('mailbox delivery into a pane awaiting the user', () => {
delivered_at: null,
pointer_enter_pending: 0
})
// Only the physical foreground changes: no new observation or send can release the hold.
if (longHold) {
await vi.advanceTimersByTimeAsync(31 * 60 * 1000)
expect(write).not.toHaveBeenCalled()
expect(runtime.mailTerminal?.lastAgentStatus).toBeNull()
}
// Only the physical foreground changes; the hook timestamp remains expired and untouched.
shell = false
await vi.advanceTimersByTimeAsync(12000)
await vi.advanceTimersByTimeAsync(longHold ? 65000 : 12000)
expect(write.mock.calls.map(([, data]) => data)).toEqual([
expect.stringContaining('You have 1 orchestration message'),
'\r'
@@ -311,8 +336,13 @@ describe('mailbox delivery into a pane awaiting the user', () => {
delivered_at: expect.any(String),
pointer_enter_pending: 0
})
await vi.advanceTimersByTimeAsync(20000)
const completedInspections = inspections
await vi.advanceTimersByTimeAsync(65000)
expect(write).toHaveBeenCalledTimes(2)
expect(inspections).toBe(completedInspections)
if (longHold) {
expect(runtime.mailTerminal?.lastAgentStatus).toBeNull()
}
shell = true
const second = insertDirectRunMessage(db, run.id, 'Fresh receipt')
@@ -42,12 +42,16 @@ export abstract class OrcaRuntimeWithWriteOrchestrationPointerPty extends OrcaRu
/** The terminal a staged pointer's Enter goes to, re-read so a replaced process gets none. */
protected resolveOrchestrationPointerSubmitTarget(
ptyId: string,
foreground?: MailboxForeground
foreground?: MailboxForeground,
stagedText?: string
): OrchestrationMailboxPointerSubmitTarget | null {
const terminal = this.getOrchestrationMailboxTerminal(ptyId)
const processIncarnation = terminal && this.getTerminalProcessIncarnation(terminal.handle)
return processIncarnation
? { terminal: this.mailboxTerminalAdmission(terminal, foreground), processIncarnation }
? {
terminal: this.mailboxTerminalAdmission(terminal, foreground, stagedText),
processIncarnation
}
: null
}
@@ -1,5 +1,71 @@
import type { MailboxDeliveryHold } from '../../orchestration/mailbox/delivery-hold/mailbox-delivery-hold'
import { isTuiIdleReadyVerdict, type TuiIdleVerdict } from './tui-idle-evidence'
import {
evaluateTuiIdle,
hasQuietReadyScreen,
isTuiIdleReadyVerdict,
type TuiIdleVerdict
} from './tui-idle-evidence'
import { ptyTuiIdleEvidence, type TuiIdleEvidenceSource } from './tui-idle-evidence-source'
import type { RuntimePtyWorktreeRecord } from '../records/runtime-terminal-state-records'
export function mailboxDeliveryTuiVerdict(
source: TuiIdleEvidenceSource,
pty: RuntimePtyWorktreeRecord,
readWaitText: () => string,
recovering: boolean,
readComposerText: () => string | null,
stagedText?: string
) {
const admissionSource = source
const screen = recovering ? source.readScreenLines(pty.ptyId) : null
if (recovering) {
// An execution identity cannot renew an old hook's readiness or a tail's composer.
source = {
...source,
getAdoptedPtyIdleStatus: () => null,
getFirstPartyAgentStatus: () => null,
getHookTurn: undefined,
readScreenLines: () => screen
}
}
const evidence = ptyTuiIdleEvidence(
source,
recovering ? { ...pty, lastOscTitle: null } : pty,
recovering ? () => screen?.join('\n') ?? '' : readWaitText
)
let verdict: TuiIdleVerdict =
recovering && screen === null
? { kind: 'pending', quietForeground: 'closed' }
: evaluateTuiIdle({
...evidence,
readPositiveBodyEvidence: () =>
evidence.readPositiveBodyEvidence() ||
(recovering &&
evidence.readAgentRuleVerdict() === null &&
hasQuietReadyScreen(
evidence.record,
null,
() => {
const text = readComposerText()
return text === '' || text === stagedText
},
source.quiescenceMs
)),
// A current screen can veto recovery; old blocked tail text cannot veto ordinary delivery.
readTailBlockedReason: recovering ? evidence.readTailBlockedReason : () => null
})
const screenReady = isTuiIdleReadyVerdict(verdict)
if (recovering && screenReady) {
const current = ptyTuiIdleEvidence(admissionSource, pty, readWaitText)
verdict = evaluateTuiIdle({
...current,
readPositiveBodyEvidence: () => screenReady,
readTailBlockedReason: evidence.readTailBlockedReason,
readScreenInputVeto: evidence.readScreenInputVeto
})
}
return { verdict, readScreenInputVeto: evidence.readScreenInputVeto }
}
/** Why pointer plus Enter would not be typed into a settled-looking worker terminal now (dialogs and
* drafts are checked first); null means it would. Takes the waits' readiness ranking plus a working
@@ -9,7 +9,7 @@ import type { AgentStatus } from '../../../../shared/agent-status/title/agent-ti
import { detectExplicitIdleStatusFromTitle } from './terminal-wait-detection'
import { buildTerminalWaitText } from './terminal-wait-tail-state'
import { evaluateTuiIdle, type TuiIdleVerdict } from './tui-idle-evidence'
import { terminalDeliveryRefusal } from './terminal-delivery-refusal'
import { mailboxDeliveryTuiVerdict, terminalDeliveryRefusal } from './terminal-delivery-refusal'
import { ptyTuiIdleEvidence } from './tui-idle-evidence-source'
import { TUI_IDLE_QUIESCENCE_MS } from './terminal-idle-policy'
import { USER_INPUT_HOLD_CAP_MS } from './claude-open-dialog-hold'
@@ -22,14 +22,13 @@ import {
type MailboxDeliveryDecision
} from '../../orchestration/mailbox/notifications/mailbox-delivery-recheck'
import type { MailboxForeground } from './launched-agent-foreground'
import { mapExplicitAgentStateToRuntimeTerminalStatus } from '../../workspaces/process-listing/runtime-worktree-status-projection'
import {
isMailboxTerminalTakingInput,
type OrchestrationMailboxTerminal
} from '../../orchestration/mailbox/routing/mailbox-owner'
import { openCodeComposerHoldsDraft } from '../../../../shared/terminal/input/opencode-composer'
import { tuiAgentDisplayName } from '../../../../shared/agent-launch/catalog/tui-agent-display-names'
import { selectFreshExplicitAgentStatusRow } from '../../workspaces/process-listing/runtime-hook-agent-row-selection'
import { selectExplicitAgentIdentityRow } from '../../workspaces/process-listing/runtime-hook-agent-row-selection'
import { isTuiAgent } from '../../../../shared/agent-launch/catalog/tui-agent-config'
import type { MailboxDeliveryHold } from '../../orchestration/mailbox/delivery-hold/mailbox-delivery-hold'
@@ -128,31 +127,35 @@ export abstract class OrcaRuntimeWithResolveExitWaiters extends OrcaRuntimeWithB
}
protected isMailboxForegroundCandidate(terminal: OrchestrationMailboxTerminal): boolean {
if (terminal.lastAgentStatus !== null || !terminal.lastAgentStatusObservedLive) {
return false
}
const row = this.mailDeliveryHookRow(terminal.ptyId)
return !!row && (row.state === 'done' || row.state === 'working')
return (
terminal.lastAgentStatus === null &&
terminal.lastAgentStatusObservedLive &&
isTuiAgent(this.mailDeliveryAgent(terminal.ptyId))
)
}
protected mailboxTerminalAdmission(
terminal: OrchestrationMailboxTerminal,
foreground?: MailboxForeground
foreground?: MailboxForeground,
stagedText?: string
): OrchestrationMailboxTerminal {
if (terminal.lastAgentStatus !== null || foreground !== 'agent') {
return terminal
}
const row = this.mailDeliveryHookRow(terminal.ptyId)
return row
? { ...terminal, lastAgentStatus: mapExplicitAgentStateToRuntimeTerminalStatus(row.state) }
return terminal.lastAgentStatus === null &&
foreground === 'agent' &&
terminal.lastAgentStatusObservedLive &&
this.deliveryRefusal(terminal.ptyId, foreground, stagedText) === null
? { ...terminal, lastAgentStatus: 'idle' }
: terminal
}
/** Pointer plus Enter takes the waits' readiness ranking, plus a working verdict: agents queue or
* steer text sent mid-turn, so only dialogs, drafts and unsettled terminals hold mail. Callers
* separately require live status evidence. */
protected isAgentSettledForDelivery(ptyId: string, foreground?: MailboxForeground): boolean {
return this.deliveryRefusal(ptyId, foreground) === null
protected isAgentSettledForDelivery(
ptyId: string,
foreground?: MailboxForeground,
stagedText?: string
): boolean {
return this.deliveryRefusal(ptyId, foreground, stagedText) === null
}
/** Why delivery would hold mail for this worker terminal now, for a send receipt: the gate's own reason. */
@@ -175,26 +178,27 @@ export abstract class OrcaRuntimeWithResolveExitWaiters extends OrcaRuntimeWithB
) {
return null
}
const agentName = tuiAgentDisplayName(this.mailDeliveryAgent(ptyId) ?? '') ?? 'The agent'
const refusal = this.deliveryRefusal(ptyId, foreground?.foreground)
if (
foreground &&
!this.orchestrationMailboxPointerDelivery.applyForeground(
ptyId,
foreground,
foreground.foreground
foreground.foreground,
refusal === null
)
) {
if (!followup) {
return this.readTerminalMailHold(terminalHandle, true)
}
this.armDeliveryRecheck(ptyId, { foreground: true })
return { kind: 'not-live', agentName: this.mailDeliveryAgentName(ptyId) }
return { kind: 'not-live', agentName }
}
this.armDeliveryRecheck(
ptyId,
foreground?.foreground === 'shell' ? { foreground: true } : undefined
)
const agentName = this.mailDeliveryAgentName(ptyId)
const refusal = this.deliveryRefusal(ptyId, foreground?.foreground)
if (refusal === 'no-terminal') {
return null
}
@@ -212,67 +216,68 @@ export abstract class OrcaRuntimeWithResolveExitWaiters extends OrcaRuntimeWithB
private deliveryRefusal(
ptyId: string,
foreground?: MailboxForeground
foreground?: MailboxForeground,
stagedText?: string
): MailboxDeliveryHold | 'no-terminal' | null {
const pty = this.ptysById.get(ptyId)
if (!pty) {
return 'no-terminal'
}
const hold = this.ptyDeliveryHold(ptyId)
const hold = this.ptyDeliveryHold(ptyId, stagedText)
if (hold) {
return { kind: hold }
}
const { verdict, readScreenInputVeto } = this.deliveryTuiVerdict(pty, foreground)
return terminalDeliveryRefusal({
verdict,
recognizedAgent: !!this.mailDeliveryAgent(ptyId),
readScreenInputVeto
...this.deliveryTuiVerdict(pty, foreground, stagedText),
recognizedAgent: !!this.mailDeliveryAgent(ptyId)
})
}
private mailDeliveryHookRow(ptyId: string) {
private mailDeliveryHookAgent(ptyId: string) {
const join = this.readHookRowJoinForPty(ptyId)
const row = join ? selectFreshExplicitAgentStatusRow(join) : null
const row = join ? selectExplicitAgentIdentityRow(join) : null
const floor = this.agentPromptExplicitStatusFloorByPtyId.get(ptyId)
return row && (floor === undefined || row.receivedAt > floor) ? row : null
return row && (floor === undefined || row.receivedAt > floor) ? row.agentType : null
}
private mailDeliveryAgent(ptyId: string) {
return this.getPaneTerminalAgent(ptyId) ?? this.mailDeliveryHookRow(ptyId)?.agentType ?? null
return this.getPaneTerminalAgent(ptyId) ?? this.mailDeliveryHookAgent(ptyId) ?? null
}
private mailDeliveryAgentName(ptyId: string): string {
const agent = this.mailDeliveryAgent(ptyId)
return (isTuiAgent(agent) && tuiAgentDisplayName(agent)) || 'The agent'
}
private deliveryTuiVerdict(pty: RuntimePtyWorktreeRecord, foreground?: MailboxForeground) {
private deliveryTuiVerdict(
pty: RuntimePtyWorktreeRecord,
foreground?: MailboxForeground,
stagedText?: string
) {
const agent = this.mailDeliveryAgent(pty.ptyId)
const source = {
...this.tuiIdleEvidenceSource,
getPaneAgent: () => (isTuiAgent(agent) ? agent : null)
}
const raw = this.getOrchestrationMailboxTerminal(pty.ptyId)
const terminal = raw && this.mailboxTerminalAdmission(raw, foreground)
const record = { ...pty, lastAgentStatus: terminal?.lastAgentStatus ?? pty.lastAgentStatus }
const evidence = ptyTuiIdleEvidence(source, record, () =>
buildTerminalWaitText(pty.tailBuffer, pty.tailPartialLine, pty.preview)
return mailboxDeliveryTuiVerdict(
{ ...this.tuiIdleEvidenceSource, getPaneAgent: () => (isTuiAgent(agent) ? agent : null) },
pty,
() => buildTerminalWaitText(pty.tailBuffer, pty.tailPartialLine, pty.preview),
pty.lastAgentStatus === null && foreground === 'agent',
() =>
readTerminalComposerTypedText(
this.readWholeScreenModel(pty.ptyId)?.emulator.getCursorLineContext()
),
stagedText
)
// Why no blocked reader: a refusal here re-arms a recheck that a lingering tail prompt
// would spin, so delivery keeps its own, blocked-blind, reading of the same ranking.
const verdict = evaluateTuiIdle({ ...evidence, readTailBlockedReason: () => null })
return { verdict, readScreenInputVeto: evidence.readScreenInputVeto }
}
/** An unsent draft would be submitted with the injected text; the cap bounds an abandoned or misread one.
* Typed text only: an agent's dim prompt suggestion is not the user's draft (#38015). */
private ptyDeliveryHold(ptyId: string): 'prompt' | 'draft' | 'restoring' | null {
private ptyDeliveryHold(
ptyId: string,
stagedText?: string
): 'prompt' | 'draft' | 'restoring' | null {
if (this.isPtyAwaitingUserInput(ptyId)) {
return 'prompt'
}
const facts = this.terminalRunFacts
const state = this.readWholeScreenModel(ptyId)
const draft = !!state && this.hasComposerDraft(ptyId, state.emulator.getCursorLineContext())
const context = state?.emulator.getCursorLineContext()
const ownPointer =
stagedText !== undefined && readTerminalComposerTypedText(context) === stagedText
const draft = !!context && !ownPointer && this.hasComposerDraft(ptyId, context)
const drivingInputAt = facts.readLastDrivingInputAt(ptyId)
const unattributed = drivingInputAt === null && facts.readProcessStartedAt(ptyId) === null
// Why: main saw every input to a process it started, so with no typing there the draft is Orca's
@@ -59,6 +59,15 @@ function isHeldRestoredPermission(row: AgentStatusIpcPayload, join: HookRowJoin)
/** The freshest live hook row for a terminal, joined on any of its handles or pane keys. */
export function selectFreshExplicitAgentStatusRow(args: HookRowJoin): AgentStatusIpcPayload | null {
return selectExplicitAgentRow(args, true)
}
/** Historical live rows identify the intended agent, without renewing their status evidence. */
export function selectExplicitAgentIdentityRow(args: HookRowJoin): AgentStatusIpcPayload | null {
return selectExplicitAgentRow(args, false)
}
function selectExplicitAgentRow(args: HookRowJoin, fresh: boolean): AgentStatusIpcPayload | null {
const now = Date.now()
const handles = new Set(args.handles)
const paneKeys = new Set(args.paneKeys)
@@ -70,8 +79,8 @@ export function selectFreshExplicitAgentStatusRow(args: HookRowJoin): AgentStatu
if (
!row.state ||
typeof row.receivedAt !== 'number' ||
now - (row.evidenceObservedAt ?? row.receivedAt) > AGENT_STATUS_STALE_AFTER_MS ||
!(isLiveObservation(row) || isHeldRestoredPermission(row, args))
(fresh && now - (row.evidenceObservedAt ?? row.receivedAt) > AGENT_STATUS_STALE_AFTER_MS) ||
!(isLiveObservation(row) || (fresh && isHeldRestoredPermission(row, args)))
) {
continue
}