From 6bbc7e5f800e6a5f714d145c19dcba0bc97950b2 Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Tue, 10 Nov 2026 06:51:50 -0800 Subject: [PATCH] fix(chat): settle mail custody from writer outcomes --- .../structured-agent-session-queued-wiring.ts | 2 +- ...structured-agent-session-send-admission.ts | 3 +- .../structured-agent-session-host-types.ts | 9 +- .../routing/terminal-pane-mail-routing.ts | 14 +-- .../structured-mail-custody.ts | 12 +- ...tructured-mailbox-pointer-delivery.test.ts | 36 ++++-- .../structured-mailbox-pointer-delivery.ts | 29 +---- ...tructured-mailbox-pointer-host-contract.ts | 4 +- .../structured-mailbox-pointer-host.ts | 12 +- ...-chat-coordinator-mail-rig.test-fixture.ts | 3 +- .../structured-chat-mail-custody.test.ts | 103 ++++++++++++++++++ .../workspace-process-listing.ts | 3 +- 12 files changed, 155 insertions(+), 75 deletions(-) diff --git a/src/main/native-chat/agent-session-wire/delivery/structured-agent-session-queued-wiring.ts b/src/main/native-chat/agent-session-wire/delivery/structured-agent-session-queued-wiring.ts index bac6d58f4d8..2514fc29bfe 100644 --- a/src/main/native-chat/agent-session-wire/delivery/structured-agent-session-queued-wiring.ts +++ b/src/main/native-chat/agent-session-wire/delivery/structured-agent-session-queued-wiring.ts @@ -95,7 +95,7 @@ export function wireStructuredAgentSessionQueuedMessages( if (journal) { // Installed before anything reads the next card, so every reader applies the one mail rule. journal.queuedMessages.mailOwed = (source) => - context().deps.queuedAgentMessageOwed?.(sessionId, source, true) ?? 'owed' + context().deps.queuedAgentMessageOwed?.(sessionId, source) ?? 'owed' } journal?.queuedMessages.editLeases.onDeadline(() => { void context() diff --git a/src/main/native-chat/agent-session-wire/delivery/structured-agent-session-send-admission.ts b/src/main/native-chat/agent-session-wire/delivery/structured-agent-session-send-admission.ts index 1a618147b5b..11036ae9ee2 100644 --- a/src/main/native-chat/agent-session-wire/delivery/structured-agent-session-send-admission.ts +++ b/src/main/native-chat/agent-session-wire/delivery/structured-agent-session-send-admission.ts @@ -101,8 +101,7 @@ export async function admitStructuredAgentSessionSend( ) ) }, - custody: (sender) => - context.deps.queuedAgentMessageOwed?.(ctx.sessionId, sender, true) ?? 'owed', + custody: (sender) => context.deps.queuedAgentMessageOwed?.(ctx.sessionId, sender) ?? 'owed', ...(ctx.operationReceipt ? { receipt: ctx.operationReceipt } : {}) }) ) diff --git a/src/main/native-chat/agent-session-wire/host/structured-agent-session-host-types.ts b/src/main/native-chat/agent-session-wire/host/structured-agent-session-host-types.ts index c68c98c4f85..b1dfebca0e0 100644 --- a/src/main/native-chat/agent-session-wire/host/structured-agent-session-host-types.ts +++ b/src/main/native-chat/agent-session-wire/host/structured-agent-session-host-types.ts @@ -199,13 +199,8 @@ export type StructuredAgentSessionHostDeps = { ) => boolean /** Explicit switch admission also includes direct chat assignments, even at rest. */ hasOwedSwitchWork?: (record: AgentSessionRecord) => boolean - /** Whether an agent's queued message is still owed to this session at handover; absent: always. */ - queuedAgentMessageOwed?: ( - sessionId: string, - from: AgentMessageSource, - /** Only read, as the queue does choosing its next card; a handover also settles a stale send. */ - peek?: boolean - ) => QueuedMessageOwed + /** Read-only custody for an agent's queued mail; absent: always owed. */ + queuedAgentMessageOwed?: (sessionId: string, from: AgentMessageSource) => QueuedMessageOwed /** Advisory tab-hidden notice; failures are logged. */ onSessionTabHidden?: (sessionId: string, tabId?: string) => void /** Throws while a terminal may still write the record's head conversation; run before a spawn. */ diff --git a/src/main/runtime/orchestration/mailbox/routing/terminal-pane-mail-routing.ts b/src/main/runtime/orchestration/mailbox/routing/terminal-pane-mail-routing.ts index d253ae3a55d..eac7375b1ad 100644 --- a/src/main/runtime/orchestration/mailbox/routing/terminal-pane-mail-routing.ts +++ b/src/main/runtime/orchestration/mailbox/routing/terminal-pane-mail-routing.ts @@ -232,13 +232,8 @@ export abstract class OrcaRuntimeWithGetPtyRecordForPaneKey extends OrcaRuntimeW this.orchestrationStructuredMailboxPointerDelivery.withdrawStale(mailboxes) } - /** Asked as a chat's queue hands over an agent's card, or, with `peek`, as it chooses its next card; - * opening the database lets a restart's first handover decide. */ - structuredChatMailOwed( - sessionId: string, - from: AgentMessageSource, - peek = false - ): QueuedMessageOwed { + /** Read current mailbox custody without settling a card or its reservation. */ + structuredChatMailOwed(sessionId: string, from: AgentMessageSource): QueuedMessageOwed { if (!orchestrationMailNotice(from)) { return 'owed' } @@ -248,10 +243,7 @@ export abstract class OrcaRuntimeWithGetPtyRecordForPaneKey extends OrcaRuntimeW } catch { return 'unavailable' } - const delivery = this.orchestrationStructuredMailboxPointerDelivery - return peek - ? delivery.owedNow(sessionId, from, db) - : delivery.owedAtHandover(sessionId, from, db) + return this.orchestrationStructuredMailboxPointerDelivery.owedNow(sessionId, from, db) } /** Resolve every structured-owned mailbox here; declining Run or direct mail would leave both delivery lanes refusing it. */ diff --git a/src/main/runtime/orchestration/mailbox/structured-mail/structured-mail-custody.ts b/src/main/runtime/orchestration/mailbox/structured-mail/structured-mail-custody.ts index ece9005e896..1291173ee5b 100644 --- a/src/main/runtime/orchestration/mailbox/structured-mail/structured-mail-custody.ts +++ b/src/main/runtime/orchestration/mailbox/structured-mail/structured-mail-custody.ts @@ -160,7 +160,10 @@ export async function settleStructuredMailSends(input: { let released = false for (const row of db.getStructuredMailSends(mailboxHandle)) { const owed = input.owed(row.session_id, [row.message_id]) - const state = await host.observe(row.session_id, row.operation_id) + let state = await host.observe(row.session_id, row.operation_id) + if (owed === 'stale' && state.kind === 'waiting') { + state = await host.withdraw(row.session_id, row.operation_id) + } const message = db.getMessageById(row.message_id) const unread = message !== undefined && !message.read ? message : undefined const retire = (consumed: boolean) => db.retireStructuredMailSend(row, consumed) @@ -216,12 +219,7 @@ export async function settleStructuredMailSends(input: { continue case 'waiting': case 'unreadable': - if (owed === 'stale' && state.kind === 'waiting') { - if (await host.withdraw(row.session_id, row.operation_id)) { - release() - continue - } - } else if (unread && !unread.delivered_at) { + if (unread && !unread.delivered_at) { // A record saved before reservations: the chat holds it, so selection must skip it. db.markAsDelivered([row.message_id], mailboxHandle) } diff --git a/src/main/runtime/orchestration/mailbox/structured-pointer/structured-mailbox-pointer-delivery.test.ts b/src/main/runtime/orchestration/mailbox/structured-pointer/structured-mailbox-pointer-delivery.test.ts index e09e83c0236..160472f14e1 100644 --- a/src/main/runtime/orchestration/mailbox/structured-pointer/structured-mailbox-pointer-delivery.test.ts +++ b/src/main/runtime/orchestration/mailbox/structured-pointer/structured-mailbox-pointer-delivery.test.ts @@ -163,9 +163,9 @@ function harness(options: { busy?: boolean; senderNames?: RuntimeOrchestrationSe async (_sessionId, operationId) => chat.get(operationId) ?? { kind: 'absent' as const } ), wakeQueue: vi.fn(), - withdraw: vi.fn(async (_sessionId, operationId) => { + withdraw: vi.fn(async (_sessionId, operationId) => { chat.set(operationId, { kind: 'withdrawn' }) - return true + return { kind: 'withdrawn' } }) } const start = () => @@ -196,8 +196,8 @@ function harness(options: { busy?: boolean; senderNames?: RuntimeOrchestrationSe }, host, /** The queue's question at handover, for a card carrying message `id`. */ - handover: (sessionId: string, id: string) => - delivery.owedAtHandover( + owed: (sessionId: string, id: string) => + delivery.owedNow( sessionId, { kind: 'agent', @@ -397,13 +397,33 @@ describe('structured mailbox delivery: each message whole', () => { await h.deliver() const first = h.operation('m1') h.setOwner('session-2') - expect(h.handover('session-1', 'm1')).toBe('stale') + expect(h.owed('session-1', 'm1')).toBe('stale') + expect(h.records.get('m1')?.operation_id).toBe(first) h.chat.set(first, { kind: 'withdrawn' }) - await flush() + await h.journal('session-1') expect(h.sent.map((send) => send.sessionId)).toEqual(['session-1', 'session-2']) expect(h.mail.get('m1')?.read).toBe(0) }) + it('settles a handoff that wins stale-card withdrawal without sending it to the new owner', async () => { + const h = harness({ busy: true }) + h.receive('m1') + await h.deliver() + const first = h.operation('m1') + h.setOwner('session-2') + vi.mocked(h.host.withdraw).mockImplementationOnce(async (_sessionId, operationId) => { + h.chat.set(operationId, { kind: 'accepted' }) + return { kind: 'accepted' } + }) + h.delivery.withdrawStale() + await vi.waitFor(() => expect(h.mail.get('m1')?.read).toBe(1)) + expect(h.records.size).toBe(0) + expect(h.sent.map(({ sessionId, operationId }) => [sessionId, operationId])).toEqual([ + ['session-1', first] + ]) + expect(h.mail.get('m1')?.delivered_at).toBe('now') + }) + it('withdraws waiting cards of a Dispatch that ended, keeping the mail unread', async () => { const h = harness({ busy: true }) h.receive('m1') @@ -417,7 +437,7 @@ describe('structured mailbox delivery: each message whole', () => { expect(h.host.send).toHaveBeenCalledTimes(1) expect(h.mail.get('m1')).toMatchObject({ read: 0, delivered_at: null }) // The queue asks the same question at handover, should the card still be there. - expect(h.handover('session-1', 'm1')).toBe('stale') + expect(h.owed('session-1', 'm1')).toBe('stale') }) it('points a message too large for one chat message at check, and withdraws it when check takes it', async () => { @@ -528,7 +548,7 @@ describe('structured mailbox delivery across a real database close and reopen', watchJournal: () => () => {}, observe: async () => state, wakeQueue: () => {}, - withdraw: async () => true, + withdraw: async () => ({ kind: 'withdrawn' }), send } }) diff --git a/src/main/runtime/orchestration/mailbox/structured-pointer/structured-mailbox-pointer-delivery.ts b/src/main/runtime/orchestration/mailbox/structured-pointer/structured-mailbox-pointer-delivery.ts index 77b3b48d775..aa21281de46 100644 --- a/src/main/runtime/orchestration/mailbox/structured-pointer/structured-mailbox-pointer-delivery.ts +++ b/src/main/runtime/orchestration/mailbox/structured-pointer/structured-mailbox-pointer-delivery.ts @@ -13,10 +13,7 @@ import type { StructuredPointerDeliveryDependencies } from './structured-mailbox import { structuredMailMessage } from '../structured-mail/structured-mail-source' import type { StructuredMailSendRow } from '../../db/messages/structured-mail-send-store' import type { QueuedMessageOwed } from '../../../../native-chat/agent-session-journal/queued-messages/queued-message-consume-error' -import { - orchestrationMailNotice, - type AgentMessageSource -} from '../../../../../shared/agent-session/agent-session-message-source' +import type { AgentMessageSource } from '../../../../../shared/agent-session/agent-session-message-source' import { StructuredMailboxParking } from '../structured-mail/structured-mailbox-parking' import { StructuredMailSettledState } from '../structured-mail/structured-mail-settled-state' import * as mailCustody from '../structured-mail/structured-mail-custody' @@ -121,30 +118,6 @@ export class OrchestrationStructuredMailboxPointerDelivery< return this.authority.owedNow(sessionId, from, db) } - /** The queue asks this as it hands an agent's card over. A stale mail card is dropped without a - * turn, so its record is settled here: the deletion is never read later as the person's, and - * mail still in this mailbox goes to its new owner. */ - owedAtHandover( - sessionId: string, - from: AgentMessageSource, - db: OrchestrationDb | null - ): QueuedMessageOwed { - const owed = this.owedNow(sessionId, from, db) - const mail = orchestrationMailNotice(from) - if (owed === 'stale' && mail && db) { - for (const { messageId } of mail.messages) { - const row = db.getStructuredMailSend(mail.mailbox, messageId) - if (row?.session_id === sessionId) { - db.releaseStructuredMailSend(row) - } - } - // After the queue's write lane unwinds, through whichever lane owns the mailbox now. - this.settledState.invalidate(mail.mailbox) - queueMicrotask(() => this.redeliver(mail.mailbox)) - } - return owed - } - private async deliver(mailboxHandle: string, reservedTypes?: ReadonlySet): Promise { if (this.inFlight.has(mailboxHandle)) { const reserved = this.rerun.get(mailboxHandle) ?? new Set() diff --git a/src/main/runtime/orchestration/mailbox/structured-pointer/structured-mailbox-pointer-host-contract.ts b/src/main/runtime/orchestration/mailbox/structured-pointer/structured-mailbox-pointer-host-contract.ts index 3715f524269..3602affd63c 100644 --- a/src/main/runtime/orchestration/mailbox/structured-pointer/structured-mailbox-pointer-host-contract.ts +++ b/src/main/runtime/orchestration/mailbox/structured-pointer/structured-mailbox-pointer-host-contract.ts @@ -33,8 +33,8 @@ export type StructuredMailboxPointerHost = { observe: (sessionId: string, operationId: string) => Promise /** Have the chat's queue look again: a card it passed over may be decidable now. */ wakeQueue: (sessionId: string) => void - /** Delete a card still waiting in the queue; false when it could not be deleted now. */ - withdraw: (sessionId: string, operationId: string) => Promise + /** Delete a waiting card and report its actual withdrawal or handoff outcome. */ + withdraw: (sessionId: string, operationId: string) => Promise /** Current lease fence; `null` when no record backs the session any more. */ currentFence: (sessionId: string) => number | null /** Coalesced journal publications of a session; returns the unsubscribe. */ diff --git a/src/main/runtime/orchestration/mailbox/structured-pointer/structured-mailbox-pointer-host.ts b/src/main/runtime/orchestration/mailbox/structured-pointer/structured-mailbox-pointer-host.ts index ab4b776137a..d565ebb7e74 100644 --- a/src/main/runtime/orchestration/mailbox/structured-pointer/structured-mailbox-pointer-host.ts +++ b/src/main/runtime/orchestration/mailbox/structured-pointer/structured-mailbox-pointer-host.ts @@ -79,12 +79,11 @@ export function createStructuredMailboxPointerHost(): StructuredMailboxPointerHo async withdraw(sessionId, operationId) { const host = getStructuredAgentSessionHost() if (!host) { - return false + return { kind: 'unreadable' } } const fence = host.deps.store.getConversation(sessionId)?.currentFence if (fence === undefined) { - // No conversation is left to hand the card over. - return true + return { kind: 'unreadable' } } const fields = { messageId: operationId } const result = await host @@ -105,8 +104,11 @@ export function createStructuredMailboxPointerHost(): StructuredMailboxPointerHo } ) .catch(() => null) - // Already handed over, or never a card (sent at once), counts as done: nothing is left to hand over twice. - return result?.ok === true + if (!result?.ok) { + return { kind: 'unreadable' } + } + // A no-op may have lost to handoff; only the host's send answer can settle that attempt. + return result.value.deleted ? { kind: 'withdrawn' } : observeMailSend(sessionId, operationId) }, watchJournal(sessionId, onActivity) { diff --git a/src/main/runtime/orchestration/terminal-authority/structured-sessions/structured-chat-coordinator-mail-rig.test-fixture.ts b/src/main/runtime/orchestration/terminal-authority/structured-sessions/structured-chat-coordinator-mail-rig.test-fixture.ts index 3165c22028b..d0a5d984cdb 100644 --- a/src/main/runtime/orchestration/terminal-authority/structured-sessions/structured-chat-coordinator-mail-rig.test-fixture.ts +++ b/src/main/runtime/orchestration/terminal-authority/structured-sessions/structured-chat-coordinator-mail-rig.test-fixture.ts @@ -265,8 +265,7 @@ beforeEach(async () => { runtime.onStructuredSessionStatusForMail(summary) }, onSessionTabHidden: (sessionId) => runtime.onStructuredSessionTabHidden(sessionId), - queuedAgentMessageOwed: (sessionId, from, peek) => - runtime.structuredChatMailOwed(sessionId, from, peek) + queuedAgentMessageOwed: (sessionId, from) => runtime.structuredChatMailOwed(sessionId, from) }) dispatcher = new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS }) }) diff --git a/src/main/runtime/orchestration/terminal-authority/structured-sessions/structured-chat-mail-custody.test.ts b/src/main/runtime/orchestration/terminal-authority/structured-sessions/structured-chat-mail-custody.test.ts index f684f7dbfe2..0604ca51eee 100644 --- a/src/main/runtime/orchestration/terminal-authority/structured-sessions/structured-chat-mail-custody.test.ts +++ b/src/main/runtime/orchestration/terminal-authority/structured-sessions/structured-chat-mail-custody.test.ts @@ -14,6 +14,8 @@ import { captureHostStructuredAgentSessionWork } from '../../../../native-chat/a import { nextStructuredQueuedMessage } from '../../../../native-chat/agent-session-wire/delivery/structured-agent-session-queued-gate' import { idOf } from '../../../rpc/orchestration-session-caller-test-fixture' import { operationId } from './structured-chat-coordinator-fake-codex-fixture' +import { withTestAgentSessionDatabase } from '../../../agent-session/records/agent-session-record-store-test-harness' +import { createStructuredMailboxPointerHost } from '../../mailbox/structured-pointer/structured-mailbox-pointer-host' import { COORDINATOR, PEER_CHAT, @@ -26,6 +28,7 @@ import { isWholeMail, openChat, queuedCardTexts, + root, runtime, sendUserMessage, settleTurn, @@ -178,6 +181,106 @@ describe('a queued mail card runs only while its chat still owes it', () => { expect(chat.turns).toHaveLength(1) }) + it('answers a dispatched Delete from the actual send, never as a withdrawal', async () => { + const chat = await busyChat(COORDINATOR) + const { taskId } = await coordinatorRunAndTask() + await finishWorker(taskId) + await vi.waitFor(async () => expect(await queuedCardTexts()).toHaveLength(1), WAIT) + const [card] = await journalOf(COORDINATOR).queuedMessages.list() + const fields = { messageId: card!.messageId } + expect( + await host.queuedMessageSend( + { callerKey: 'test-surface' }, + { + envelope: { + sessionId: COORDINATOR, + clientOperationId: operationId(), + expectedRuntimeFence: host.deps.store.getConversation(COORDINATOR)!.currentFence, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.queuedMessageSend', + sessionId: COORDINATOR, + fields + }) + }, + ...fields + } + ) + ).toMatchObject({ ok: true }) + await vi.waitFor(() => expect(chat.steers).toHaveLength(1), WAIT) + const pointerHost = createStructuredMailboxPointerHost() + expect(await pointerHost.withdraw(COORDINATOR, fields.messageId)).toEqual({ kind: 'waiting' }) + expect(await journalOf(COORDINATOR).queuedMessages.get(fields.messageId)).toMatchObject({ + state: 'dispatched' + }) + chat.handlers.onNotification?.('item/completed', { + item: { + type: 'userMessage', + id: 'echo-claimed-mail', + clientId: chat.steers[0]!.clientUserMessageId, + content: [{ type: 'text', text: chat.steers[0]!.text }] + } + }) + await host.flushStreamedEvents(COORDINATOR) + expect(await pointerHost.withdraw(COORDINATOR, fields.messageId)).toEqual({ kind: 'accepted' }) + }) + + it('retains stale mail custody when the writer refuses its withdrawal', async () => { + const chat = await busyChat(COORDINATOR) + const { runId, taskId } = await coordinatorRunAndTask() + await finishWorker(taskId) + await vi.waitFor(async () => expect(await queuedCardTexts()).toHaveLength(1), WAIT) + const mailbox = `run:${runId}` + const [reserved] = db.getStructuredMailSends(mailbox) + expect(reserved).toBeDefined() + const [card] = await journalOf(COORDINATOR).queuedMessages.list() + const fields = { messageId: card!.messageId } + db.bindRun({ + runId, + coordinatorHandle: 'term_taker', + coordinatorPaneKey: 'tab_taker:cccccccc-cccc-4ccc-8ccc-cccccccccccc' + }) + await withTestAgentSessionDatabase(root, (database) => + database.exec(`CREATE TRIGGER refuse_mail_withdrawal BEFORE UPDATE ON journal_queue_cards + WHEN NEW.state = 'withdrawn' + BEGIN SELECT RAISE(ABORT, 'withdrawal unavailable'); END`) + ) + try { + const sending = host.queuedMessageSend( + { callerKey: 'test-surface' }, + { + envelope: { + sessionId: COORDINATOR, + clientOperationId: operationId(), + expectedRuntimeFence: host.deps.store.getConversation(COORDINATOR)!.currentFence, + payloadFingerprint: computeAgentSessionPayloadFingerprint({ + method: 'agentSession.queuedMessageSend', + sessionId: COORDINATOR, + fields + }) + }, + ...fields + } + ) + await expect(sending).rejects.toThrow('withdrawal unavailable') + expect(db.getStructuredMailSends(mailbox)).toEqual([reserved]) + expect(await journalOf(COORDINATOR).queuedMessages.get(fields.messageId)).toMatchObject({ + state: 'waiting' + }) + expect(db.getUnreadMessages(mailbox)).toMatchObject([ + { read: 0, delivered_at: expect.any(String) } + ]) + expect(chat.turns).toHaveLength(1) + } finally { + await withTestAgentSessionDatabase(root, (database) => + database.exec('DROP TRIGGER refuse_mail_withdrawal') + ) + } + runtime.withdrawStaleChatMail([mailbox]) + await vi.waitFor(() => expect(db.getStructuredMailSends(mailbox)).toEqual([]), WAIT) + expect(await queuedCardTexts()).toEqual([]) + expect(db.getUnreadMessages(mailbox)).toMatchObject([{ read: 0, delivered_at: null }]) + }) + it("lets the person's cards pass mail it cannot decide on now, then hands the mail over first once it can", async () => { const chat = await busyChat(COORDINATOR) const { taskId } = await coordinatorRunAndTask() diff --git a/src/main/runtime/workspaces/process-listing/workspace-process-listing.ts b/src/main/runtime/workspaces/process-listing/workspace-process-listing.ts index 6f5424779f2..a79119cb2a0 100644 --- a/src/main/runtime/workspaces/process-listing/workspace-process-listing.ts +++ b/src/main/runtime/workspaces/process-listing/workspace-process-listing.ts @@ -289,8 +289,7 @@ export abstract class OrcaRuntimeWithGetWorktreePs extends RuntimeChatWorkerEven ...(this.structuredAgentStatusSinkFn ? { statusSink: this.structuredAgentStatusSinkFn } : {}), conversationNaming: nameChat, onSessionTabHidden: (sessionId) => this.onStructuredSessionTabHidden(sessionId), - queuedAgentMessageOwed: (sessionId, from, peek) => - this.structuredChatMailOwed(sessionId, from, peek), + queuedAgentMessageOwed: (sessionId, from) => this.structuredChatMailOwed(sessionId, from), assertNoTerminalWriter: (record) => this.assertStructuredChatAcquireAdmissible(record), attentionDelivery: createStructuredAttentionMobileDelivery({ readNotificationSettings: () => this.requireStore().getSettings().notifications,