From 5384448ece00ec8fe2c026ed3a66612c4c6e924f Mon Sep 17 00:00:00 2001 From: Brennan Benson <79079362+brennanb2025@users.noreply.github.com> Date: Thu, 12 Nov 2026 18:56:50 -0800 Subject: [PATCH] fix(orchestration): retain mail wake custody through failed launches --- .../db/messages/message-inbox.ts | 61 +-- .../db/messages/undelivered-inbox-query.ts | 46 ++ .../delivery-hold/mailbox-delivery-hold.ts | 5 + .../routing/mail-wake-retry-admission.ts | 10 +- .../routing/runtime-mail-wake-admission.ts | 70 +-- .../routing/sleeping-mail-record-index.ts | 60 +++ .../mailbox/routing/slept-mailbox-runtime.ts | 191 ++++---- .../routing/slept-mailbox-wake.test.ts | 290 +++++++++++- .../mailbox/routing/slept-mailbox-wake.ts | 182 +++++++- .../routing/terminal-pane-mail-routing.ts | 2 +- .../mailbox-pointer-delivery.ts | 6 +- .../mailbox-pointer-eligibility.ts | 38 +- .../messaging/send-delivery-hold.ts | 2 +- .../projection/restored-pane-materialize.ts | 16 +- .../projection/retained-mail-wake-attempt.ts | 39 +- .../agent-identity/pane-agent-identity.ts | 2 +- .../lifecycle/transport-output-callbacks.ts | 15 + .../startup/pane-resume-record.ts | 23 +- .../startup/sleeping-record-access.ts | 9 +- .../agent-hibernation-coordinator.ts | 21 +- .../sleeping-agent-wake-requests.test.ts | 146 +++++- .../wake-sleeping-agents-in-background.ts | 22 +- .../terminals/terminal-pane-hibernation.ts | 433 +++++++++--------- .../lifecycle/mail-wake-obligation.ts | 12 +- 24 files changed, 1223 insertions(+), 478 deletions(-) create mode 100644 src/main/runtime/orchestration/db/messages/undelivered-inbox-query.ts create mode 100644 src/main/runtime/orchestration/mailbox/routing/sleeping-mail-record-index.ts diff --git a/src/main/runtime/orchestration/db/messages/message-inbox.ts b/src/main/runtime/orchestration/db/messages/message-inbox.ts index 172f609b7d9..5f177556e85 100644 --- a/src/main/runtime/orchestration/db/messages/message-inbox.ts +++ b/src/main/runtime/orchestration/db/messages/message-inbox.ts @@ -1,3 +1,4 @@ +import { undeliveredInboxQuery, type UndeliveredInboxOptions } from './undelivered-inbox-query' import type { MessageType, MessageRow } from '../orchestration-records' import { exposeMessageTimestamps, exposeMessageListTimestamps } from '../utc-timestamp' import { addLifecycleRejectionMarker } from '../lifecycle-rejection-marker' @@ -88,51 +89,29 @@ export function getUndeliveredUnreadMessages( this: OrchestrationDb, toHandle: string, types?: MessageType[], - options?: { - excludeTypes?: readonly string[] - excludeIds?: readonly string[] - excludeRunOwnedDirect?: boolean - newestFirst?: boolean - limit?: number - } + options?: UndeliveredInboxOptions ): MessageRow[] { - const conditions = [ - 'to_handle = ?', - 'read = 0', - 'delivered_at IS NULL', - 'pointer_enter_pending = 0', - "delivery_contract = 'current_delivery'", - ...(options?.excludeRunOwnedDirect ? [`NOT ${runOwnsDirectMessageSql('messages')}`] : []) - ] - const params: (string | number)[] = [toHandle] - if (types?.length) { - conditions.push(`type IN (${types.map(() => '?').join(',')})`) - params.push(...types) - } - if (options?.excludeTypes?.length) { - conditions.push(`type NOT IN (${options.excludeTypes.map(() => '?').join(',')})`) - params.push(...options.excludeTypes) - } - if (options?.excludeIds?.length) { - conditions.push(`id NOT IN (${options.excludeIds.map(() => '?').join(',')})`) - params.push(...options.excludeIds) - } - const limitSql = options?.limit === undefined ? '' : ' LIMIT ?' - if (options?.limit !== undefined) { - params.push(Math.max(1, Math.floor(options.limit))) - } + const query = undeliveredInboxQuery(toHandle, types, options) return exposeMessageListTimestamps( - // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: this SELECT returns only messages table rows, decoded at this database boundary. - this.db - .prepare( - `SELECT * FROM messages - WHERE ${conditions.join(' AND ')} - ORDER BY sequence${options?.newestFirst ? ' DESC' : ''}${limitSql}` - ) - .all(...params) as MessageRow[] + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the shared predicate changes selection only; SELECT * still returns the messages schema. + this.db.prepare(`SELECT * ${query.sql}`).all(...query.params) as MessageRow[] ) } +export function getUndeliveredUnreadSequence( + this: OrchestrationDb, + toHandle: string, + options: UndeliveredInboxOptions +): number | undefined { + const query = undeliveredInboxQuery(toHandle, undefined, { + ...options, + newestFirst: true, + limit: 1 + }) + const row = this.db.prepare(`SELECT sequence ${query.sql}`).get(...query.params) + return typeof row?.sequence === 'number' ? row.sequence : undefined +} + export function getUndeliveredUnreadMailboxHandles(this: OrchestrationDb): string[] { return ( this.db @@ -289,6 +268,7 @@ export function getThreadMessagesFor( export type MessageInboxMethods = { getUnreadMessages: typeof getUnreadMessages convertLifecycleMessageToRejection: typeof convertLifecycleMessageToRejection + getUndeliveredUnreadSequence: typeof getUndeliveredUnreadSequence getUndeliveredUnreadMessages: typeof getUndeliveredUnreadMessages getUndeliveredUnreadMailboxHandles: typeof getUndeliveredUnreadMailboxHandles getAllMessages: typeof getAllMessages @@ -307,6 +287,7 @@ export function attachMessageInbox(ctor: { prototype: object }): void { Object.assign(ctor.prototype, { getUnreadMessages, convertLifecycleMessageToRejection, + getUndeliveredUnreadSequence, getUndeliveredUnreadMessages, getUndeliveredUnreadMailboxHandles, getAllMessages, diff --git a/src/main/runtime/orchestration/db/messages/undelivered-inbox-query.ts b/src/main/runtime/orchestration/db/messages/undelivered-inbox-query.ts new file mode 100644 index 00000000000..911ef63540b --- /dev/null +++ b/src/main/runtime/orchestration/db/messages/undelivered-inbox-query.ts @@ -0,0 +1,46 @@ +import type { MessageType } from '../orchestration-records' +import { runOwnsDirectMessageSql } from '../runs/run-coordinator-mail-routing' + +export type UndeliveredInboxOptions = { + excludeTypes?: readonly string[] + excludeIds?: readonly string[] + excludeRunOwnedDirect?: boolean + newestFirst?: boolean + limit?: number +} + +export function undeliveredInboxQuery( + toHandle: string, + types?: MessageType[], + options?: UndeliveredInboxOptions +) { + const conditions = [ + 'to_handle = ?', + 'read = 0', + 'delivered_at IS NULL', + 'pointer_enter_pending = 0', + "delivery_contract = 'current_delivery'", + ...(options?.excludeRunOwnedDirect ? [`NOT ${runOwnsDirectMessageSql('messages')}`] : []) + ] + const params: (string | number)[] = [toHandle] + if (types?.length) { + conditions.push(`type IN (${types.map(() => '?').join(',')})`) + params.push(...types) + } + if (options?.excludeTypes?.length) { + conditions.push(`type NOT IN (${options.excludeTypes.map(() => '?').join(',')})`) + params.push(...options.excludeTypes) + } + if (options?.excludeIds?.length) { + conditions.push(`id NOT IN (${options.excludeIds.map(() => '?').join(',')})`) + params.push(...options.excludeIds) + } + const limitSql = options?.limit === undefined ? '' : ' LIMIT ?' + if (options?.limit !== undefined) { + params.push(Math.max(1, Math.floor(options.limit))) + } + return { + sql: `FROM messages WHERE ${conditions.join(' AND ')} ORDER BY sequence${options?.newestFirst ? ' DESC' : ''}${limitSql}`, + params + } +} diff --git a/src/main/runtime/orchestration/mailbox/delivery-hold/mailbox-delivery-hold.ts b/src/main/runtime/orchestration/mailbox/delivery-hold/mailbox-delivery-hold.ts index 172c128c899..9cd7fc5eae1 100644 --- a/src/main/runtime/orchestration/mailbox/delivery-hold/mailbox-delivery-hold.ts +++ b/src/main/runtime/orchestration/mailbox/delivery-hold/mailbox-delivery-hold.ts @@ -8,6 +8,8 @@ export type MailboxDeliveryHold = openTab: boolean failed?: boolean liveShell?: boolean + waking?: boolean + resuming?: boolean workspaceName?: string } | { kind: 'not-foreground'; agentName: string } @@ -78,6 +80,9 @@ export async function readMailboxDeliveryHold( export function describeMailboxDeliveryHold(recipient: string, hold: MailboxDeliveryHold): string { switch (hold.kind) { case 'asleep': + if (hold.waking || hold.resuming) { + return `Queued. Orca is ${hold.waking ? 'waking' : 'resuming'} ${hold.agentName}; the message will be delivered when it's back.` + } if (hold.liveShell) { return `Queued. Re-run ${hold.agentName} in that terminal; the message is waiting.` } diff --git a/src/main/runtime/orchestration/mailbox/routing/mail-wake-retry-admission.ts b/src/main/runtime/orchestration/mailbox/routing/mail-wake-retry-admission.ts index 9ae91f0c312..f206cd358b9 100644 --- a/src/main/runtime/orchestration/mailbox/routing/mail-wake-retry-admission.ts +++ b/src/main/runtime/orchestration/mailbox/routing/mail-wake-retry-admission.ts @@ -38,15 +38,7 @@ export async function admitsMailWakeRetry( return false } const inspection = await ports.inspect(attempt.ptyId, attempt.incarnationId) - const observation = inspection?.foregroundObservation - if ( - inspection?.childProcessEvidence !== 'no-children' || - inspection.hasChildProcesses || - observation?.verdict !== 'observed' || - observation.ptyId !== attempt.ptyId || - observation.ptyIncarnationId !== attempt.incarnationId || - observation.foregroundKind !== 'shell' - ) { + if (inspection?.childProcessEvidence !== 'no-children' || inspection.hasChildProcesses) { return false } return (await ports.shellInFront(attempt.ptyId)) && stillOwns() diff --git a/src/main/runtime/orchestration/mailbox/routing/runtime-mail-wake-admission.ts b/src/main/runtime/orchestration/mailbox/routing/runtime-mail-wake-admission.ts index e7a18622da7..f145c99b677 100644 --- a/src/main/runtime/orchestration/mailbox/routing/runtime-mail-wake-admission.ts +++ b/src/main/runtime/orchestration/mailbox/routing/runtime-mail-wake-admission.ts @@ -1,5 +1,8 @@ -import type { SleepingAgentSessionRecord } from '../../../../../shared/agent-session/lifecycle/agent-session-resume' +import type { SleptMailboxWake } from './slept-mailbox-wake' import { mailWakeObligation } from '../../../../../shared/agent-session/lifecycle/mail-wake-obligation' +import { TUI_AGENT_DISPLAY_NAMES } from '../../../../../shared/agent-launch/catalog/tui-agent-display-names' +import type { MailboxDeliveryHold } from '../delivery-hold/mailbox-delivery-hold' +import type { SleepingAgentSessionRecord } from '../../../../../shared/agent-session/lifecycle/agent-session-resume' import { OrcaRuntimeWithPruneMobileSessionTabGroupLayout } from '../../../session-lifecycle/projection/prune-mobile-session-tab-group-layout' import type { RestoredPane, @@ -13,10 +16,8 @@ export abstract class RuntimeMailWakeAdmission extends OrcaRuntimeWithPruneMobil protected abstract readonly mailWakeInvocations: MailWakeInvocations protected abstract hasSleepingMailForPane(paneKey: string): boolean protected abstract sleepingMailRecord(paneKey: string): SleepingAgentSessionRecord | null - protected abstract editMailWakeRecord( - record: SleepingAgentSessionRecord, - next: SleepingAgentSessionRecord - ): Promise + protected abstract readonly sleptMailboxWake: SleptMailboxWake + protected abstract watchSleepingMail(): void protected abstract getPtyRecordForPaneKey(paneKey: string): RuntimePtyWorktreeRecord | null protected async admitFailedMailWake( pane: RestoredPane, @@ -48,7 +49,7 @@ export abstract class RuntimeMailWakeAdmission extends OrcaRuntimeWithPruneMobil this.terminalRunFacts.readProcessStartedAt(ptyId, incarnationId) !== null ) } - const safe = await admitsMailWakeRetry(record, { + await admitsMailWakeRetry(record, { current: () => this.hasSleepingMailForPane(record.paneKey) ? this.sleepingMailRecord(record.paneKey) @@ -65,28 +66,43 @@ export abstract class RuntimeMailWakeAdmission extends OrcaRuntimeWithPruneMobil shellInFront: async (ptyId) => (await this.readLaunchedAgentForeground(ptyId, record.agent)) === 'shell' }) - if (!safe || !owns(attempt.ptyId, attempt.incarnationId) || !this.ptyController?.stopAndWait) { - return false + // Host-side input fencing is required before a proven failed invocation may be stopped. + return false + } + + async readSleepingMailHold(mailbox: string): Promise { + this.watchSleepingMail() + const record = this.sleptMailboxWake.batch(mailbox)?.record + if (!record || !mailWakeObligation(record)) { + return null } - if (!(await this.ptyController.stopAndWait(attempt.ptyId, { keepHistory: true }))) { - return false + const attempt = record.mailWake?.attempt + const pty = this.getPtyRecordForPaneKey(record.paneKey) + const liveShell = + !!attempt && + pty?.connected === true && + pty.ptyId === attempt.ptyId && + pty.incarnationId === attempt.incarnationId && + pty.launchToken === attempt.launchToken && + pty.launchIncarnationId === attempt.incarnationId && + pty.worktreeId === record.worktreeId && + pty.connectionId === (record.connectionId ?? null) && + this.mailWakeInvocations.ended(record) && + (await this.readLaunchedAgentForeground(attempt.ptyId, record.agent)) === 'shell' && + this.getPtyRecordForPaneKey(record.paneKey) === pty && + this.sleepingMailRecord(record.paneKey) === record + return { + kind: 'asleep', + waking: + mailWakeObligation(record) === 'automatic' && !this.sleptMailboxWake.failedToWake(record), + resuming: mailWakeObligation(record) === 'retained' && !liveShell, + agentName: TUI_AGENT_DISPLAY_NAMES[record.agent], + openTab: record.restoreOnTabOpenOnly === true, + failed: this.sleptMailboxWake.failedToWake(record), + liveShell, + workspaceName: this.resolvedWorktrees + .peek() + ?.worktrees.find((row) => row.id === record.worktreeId)?.displayName } - const current = this.sleepingMailRecord(record.paneKey) - if ( - current?.mailWake?.attempt?.launchToken !== attempt.launchToken || - mailWakeObligation(current) !== 'retained' || - !this.hasSleepingMailForPane(record.paneKey) || - (this.terminalRunFacts.readLastDrivingInputAt(attempt.ptyId) ?? -1) >= attempt.startedAt - ) { - return false - } - return this.editMailWakeRecord(current, { - ...current, - mailWake: { - id: current.mailWake!.id, - capturedAt: current.mailWake!.capturedAt, - startedAt: Date.now() - } - }) } } diff --git a/src/main/runtime/orchestration/mailbox/routing/sleeping-mail-record-index.ts b/src/main/runtime/orchestration/mailbox/routing/sleeping-mail-record-index.ts new file mode 100644 index 00000000000..8e3077156fb --- /dev/null +++ b/src/main/runtime/orchestration/mailbox/routing/sleeping-mail-record-index.ts @@ -0,0 +1,60 @@ +import type { SleepingAgentSessionRecord } from '../../../../../shared/agent-session/lifecycle/agent-session-resume' +import { + mailWakeEligibility, + mailWakeObligation +} from '../../../../../shared/agent-session/lifecycle/mail-wake-obligation' + +export class SleepingMailRecordIndex { + readonly records = new Map() + readonly custody = new Map() + readonly changed = new Set() + private readonly hostPanes = new Map>() + + launchTokens(): ReadonlySet { + return new Set( + [...this.custody.values()].flatMap((record) => + record.mailWake?.attempt ? [record.mailWake.attempt.launchToken] : [] + ) + ) + } + + refresh( + host: string, + records: Readonly>, + changed: (pane: string) => void + ): boolean { + const previous = this.hostPanes.get(host) ?? new Set() + const next = new Set(Object.keys(records)) + let eligibilityChanged = false + for (const pane of previous) { + if (!next.has(pane)) { + eligibilityChanged ||= this.records.has(pane) + this.records.delete(pane) + this.custody.delete(pane) + this.changed.add(pane) + changed(pane) + } + } + for (const record of Object.values(records)) { + const before = this.records.get(record.paneKey) + if (before !== record) { + this.changed.add(record.paneKey) + } + eligibilityChanged ||= + mailWakeEligibility(before ? [before] : []) !== mailWakeEligibility([record]) + if (mailWakeObligation(record)) { + this.records.set(record.paneKey, record) + } else { + this.records.delete(record.paneKey) + } + if (record.mailWake) { + this.custody.set(record.paneKey, record) + } else { + this.custody.delete(record.paneKey) + } + changed(record.paneKey) + } + this.hostPanes.set(host, next) + return eligibilityChanged + } +} diff --git a/src/main/runtime/orchestration/mailbox/routing/slept-mailbox-runtime.ts b/src/main/runtime/orchestration/mailbox/routing/slept-mailbox-runtime.ts index 5ad03240633..09e00f02aad 100644 --- a/src/main/runtime/orchestration/mailbox/routing/slept-mailbox-runtime.ts +++ b/src/main/runtime/orchestration/mailbox/routing/slept-mailbox-runtime.ts @@ -1,20 +1,19 @@ +import { replaceMailWakeRecord } from '../../../session-lifecycle/projection/retained-mail-wake-attempt' +import { SleepingMailRecordIndex } from './sleeping-mail-record-index' import { mailWakeObligation, - mailWakeEligibility, withoutMailWake } from '../../../../../shared/agent-session/lifecycle/mail-wake-obligation' import { queueTerminalWorkspaceCommand } from '../../../workspaces/lifecycle/workspace-lifecycle-commands' import { MailWakeInvocations } from './mail-wake-invocations' import { parsePaneKey } from '../../../../../shared/workspace-layout/stable-pane-id' import { RuntimeMailWakeAdmission } from './runtime-mail-wake-admission' -import { TUI_AGENT_DISPLAY_NAMES } from '../../../../../shared/agent-launch/catalog/tui-agent-display-names' import { SleptMailboxWake } from './slept-mailbox-wake' import type { SleepingAgentSessionRecord } from '../../../../../shared/agent-session/lifecycle/agent-session-resume' -import type { MailboxDeliveryHold } from '../delivery-hold/mailbox-delivery-hold' export abstract class RuntimeSleptMailboxWakes extends RuntimeMailWakeAdmission { protected readonly mailWakeInvocations = new MailWakeInvocations() - private readonly sleptMailboxWake = new SleptMailboxWake({ + protected readonly sleptMailboxWake = new SleptMailboxWake({ db: () => this._orchestrationDb, record: (paneKey) => this.sleepingMailRecord(paneKey), waiters: (mailbox) => this.messageWaiters.get(mailbox), @@ -87,15 +86,28 @@ export abstract class RuntimeSleptMailboxWakes extends RuntimeMailWakeAdmission }) private watchingSleepingMail = false + private readonly sleepingRecordIndex = new SleepingMailRecordIndex() + private readonly sleepingRecords = this.sleepingRecordIndex.records + private readonly custodyRecords = this.sleepingRecordIndex.custody + private readonly changedSleepingPanes = this.sleepingRecordIndex.changed - private watchSleepingMail(): void { + private refreshSleepingRecords(host: string): boolean { + const changed = this.sleepingRecordIndex.refresh( + host, + this.store?.getWorkspaceSession?.(host)?.sleepingAgentSessionsByPaneKey ?? {}, + (pane) => this.sleptMailboxWake.refreshPane(pane) + ) + this.mailWakeInvocations.retain(this.sleepingRecordIndex.launchTokens()) + return changed + } + + protected watchSleepingMail(): void { if (this.watchingSleepingMail) { return } this.watchingSleepingMail = true - const records = new Map() for (const host of this.store?.getWorkspaceSessionHostIds?.() ?? ['local']) { - records.set(host, this.sleepingMailEligibility(host)) + this.refreshSleepingRecords(host) } this.onClientEvent( (event) => { @@ -105,72 +117,61 @@ export abstract class RuntimeSleptMailboxWakes extends RuntimeMailWakeAdmission }, { consumesTerminalSideEffects: false } ) - this.subscribeAgentStatusRowChanges(() => this.watchMailWakeInvocations()) + this.subscribeAgentStatusRowChanges((paneKey) => this.watchMailWakeInvocations(paneKey)) this.crewChanges.subscribe((facets) => { if (facets.has('traffic') || facets.has('topology') || facets.has('records')) { - this.publishSleepingMail() + this.publishSleepingMail( + facets.has('topology') || facets.has('records') ? undefined : facets.trafficAddresses + ) } }) this.store?.onWorkspaceSessionWritten?.((hostId) => { const hosts = hostId === null ? (this.store?.getWorkspaceSessionHostIds?.() ?? ['local']) : [hostId] for (const host of hosts) { - const next = this.sleepingMailEligibility(host) + const changed = this.refreshSleepingRecords(host) this.watchMailWakeInvocations() - if (records.get(host) !== next) { - records.set(host, next) + if (changed) { queueMicrotask(() => this.retrySleptMailboxWakes()) + } else { + this.publishSleepingMail([]) } } }) } - private sleepingMailEligibility(host: string): string { - return mailWakeEligibility( - Object.values(this.store?.getWorkspaceSession?.(host)?.sleepingAgentSessionsByPaneKey ?? {}) - ) - } - - private allMailWakeRecords(): SleepingAgentSessionRecord[] { - return (this.store?.getWorkspaceSessionHostIds?.() ?? ['local']).flatMap((host) => - Object.values(this.store?.getWorkspaceSession?.(host)?.sleepingAgentSessionsByPaneKey ?? {}) - ) - } - - private watchMailWakeInvocations(): void { - const records = this.allMailWakeRecords() - const tokens = new Set() - for (const record of records) { - const attempt = record.mailWake?.attempt + private watchMailWakeInvocations(paneKey?: string): void { + const record = paneKey ? this.custodyRecords.get(paneKey) : undefined + for (const current of paneKey ? (record ? [record] : []) : this.custodyRecords.values()) { + const attempt = current.mailWake?.attempt if (!attempt) { continue } - tokens.add(attempt.launchToken) if ( this.terminalRunFacts.readProcessStartedAt(attempt.ptyId, attempt.incarnationId) !== null ) { - this.mailWakeInvocations.watch(record, this) + this.mailWakeInvocations.watch(current, this) } - const pty = this.getPtyRecordForPaneKey(record.paneKey) + const pty = this.getPtyRecordForPaneKey(current.paneKey) if ( pty?.connected === true && - pty.worktreeId === record.worktreeId && - pty.connectionId === (record.connectionId ?? null) && + pty.worktreeId === current.worktreeId && + pty.connectionId === (current.connectionId ?? null) && pty.incarnationId === attempt.incarnationId && pty.launchToken === attempt.launchToken && pty.launchIncarnationId === attempt.incarnationId && this.mailWakeInvocations.observe( - record, - this.getAgentProviderSessionRowsForPaneFn?.(record.paneKey) ?? [], + current, + this.getAgentProviderSessionRowsForPaneFn?.(current.paneKey) ?? [], pty.ptyId ) ) { - void this.editMailWakeRecord(record, withoutMailWake(record)).catch((error: unknown) => + this.sleptMailboxWake.confirmed(current) + void this.editMailWakeRecord(current, withoutMailWake(current)).catch((error: unknown) => console.warn('[orchestration] could not settle resumed mail custody', error) ) } } - this.mailWakeInvocations.retain(tokens) } protected async editMailWakeRecord( @@ -181,42 +182,20 @@ export abstract class RuntimeSleptMailboxWakes extends RuntimeMailWakeAdmission await this.commitWorkspaceSessionEdit( this.getWorkspaceSessionHostIdForWorktree(record.worktreeId), (session) => { - const current = session.sleepingAgentSessionsByPaneKey?.[record.paneKey] - if ( - current?.mailWake?.id !== record.mailWake?.id || - current?.capturedAt !== record.capturedAt || - current?.origin !== record.origin || - current?.restoreOnTabOpenOnly !== record.restoreOnTabOpenOnly || - current?.mailWake?.attempt?.launchToken !== record.mailWake?.attempt?.launchToken - ) { - return null - } - changed = true - return { - ...session, - sleepingAgentSessionsByPaneKey: { - ...session.sleepingAgentSessionsByPaneKey, - [record.paneKey]: next - } - } + const edited = replaceMailWakeRecord(session, record, next) + changed = edited !== null + return edited } ) if (changed && record.mailWake && !next.mailWake) { this.settledMailWakeIds.add(record.mailWake.id) - this.publishSleepingMail() + this.publishSleepingMail([]) } return changed } protected sleepingMailRecord(paneKey: string): SleepingAgentSessionRecord | null { - for (const host of this.store?.getWorkspaceSessionHostIds?.() ?? (['local'] as const)) { - const record = - this.store?.getWorkspaceSession?.(host)?.sleepingAgentSessionsByPaneKey?.[paneKey] - if (record) { - return record - } - } - return null + return this.sleepingRecords.get(paneKey) ?? null } protected hasSleepingMailForPane(paneKey: string): boolean { @@ -224,23 +203,32 @@ export abstract class RuntimeSleptMailboxWakes extends RuntimeMailWakeAdmission } protected requestSleptMailboxWake(mailbox: string, reservedTypes?: ReadonlySet): void { + this.watchSleepingMail() this.sleptMailboxWake.request(mailbox, reservedTypes) } retrySleptMailboxWakes(): void { this.watchSleepingMail() this.sleepingMailPublication = '' - this.publishSleepingMail() this.getExistingOrchestrationDb() - this.scheduleRestoredMessageRepoints() + this.publishSleepingMail() } private readonly settledMailWakeIds = new Set() private sleepingMailPublicationDirty = false + private globalSleepingMailInvalidation = false + private readonly sleepingMailAddresses = new Set() private sleepingMailPublicationQueued = false private sleepingMailPublication = '' - protected publishSleepingMail(): void { + protected publishSleepingMail(mailboxes?: Iterable): void { + if (mailboxes) { + for (const mailbox of mailboxes) { + this.sleepingMailAddresses.add(mailbox) + } + } else { + this.globalSleepingMailInvalidation = true + } if (this.sleepingMailPublicationQueued) { this.sleepingMailPublicationDirty = true return @@ -251,35 +239,43 @@ export abstract class RuntimeSleptMailboxWakes extends RuntimeMailWakeAdmission if (!this._orchestrationDb) { return } - const panes = new Map() - for (const mailbox of this._orchestrationDb?.getUndeliveredUnreadMailboxHandles() ?? []) { - const batch = this.sleptMailboxWake.batch(mailbox) - if (batch) { - panes.set(batch.record.paneKey, batch.record.worktreeId) + const global = this.globalSleepingMailInvalidation + const affected = new Set(this.changedSleepingPanes) + this.changedSleepingPanes.clear() + if (global && (this.sleepingRecords.size || this.sleptMailboxWake.hasTrackedMail)) { + this.sleptMailboxWake.reconcile() + for (const pane of this.custodyRecords.keys()) { + affected.add(pane) + } + } else { + for (const mailbox of this.sleepingMailAddresses) { + for (const pane of this.sleptMailboxWake.refresh(mailbox)) { + affected.add(pane) + } } } - for (const record of this.allMailWakeRecords()) { - if (record.mailWake && !panes.has(record.paneKey)) { + this.globalSleepingMailInvalidation = false + this.sleepingMailAddresses.clear() + for (const pane of affected) { + const record = this.custodyRecords.get(pane) + if (record && !this.sleptMailboxWake.owesPane(pane)) { await this.editMailWakeRecord(record, withoutMailWake(record)) } } + if (global) { + for (const pane of this.sleepingRecords.keys()) { + for (const mailbox of this.sleptMailboxWake.mailboxesForPane(pane)) { + this.sleptMailboxWake.request(mailbox) + } + } + } const settledIds = [...this.settledMailWakeIds] this.settledMailWakeIds.clear() - const ids = [...new Set(panes.values())].sort() - const key = JSON.stringify([ - ids, - this.allMailWakeRecords() - .filter((record) => record.mailWake) - .map((record) => record.mailWake!.id) - .sort() - ]) + const ids = this.sleptMailboxWake.heldWorktrees() + const key = JSON.stringify(ids) if (key !== this.sleepingMailPublication || settledIds.length) { this.sleepingMailPublication = key - if (settledIds.length) { - this.notifier?.sleepingMailChanged?.(ids, settledIds) - } else { - this.notifier?.sleepingMailChanged?.(ids) - } + this.notifier?.sleepingMailChanged?.(ids, ...(settledIds.length ? [settledIds] : [])) } } catch { /* The runtime can close its database before a queued publication. */ @@ -287,26 +283,9 @@ export abstract class RuntimeSleptMailboxWakes extends RuntimeMailWakeAdmission this.sleepingMailPublicationQueued = false if (this.sleepingMailPublicationDirty) { this.sleepingMailPublicationDirty = false - this.publishSleepingMail() + this.publishSleepingMail([]) } } }) } - - readSleepingMailHold(mailbox: string): MailboxDeliveryHold | null { - const record = this.sleptMailboxWake.batch(mailbox)?.record - if (!record || !mailWakeObligation(record)) { - return null - } - return { - kind: 'asleep', - agentName: TUI_AGENT_DISPLAY_NAMES[record.agent], - openTab: record.restoreOnTabOpenOnly === true, - failed: this.sleptMailboxWake.failedToWake(record), - liveShell: mailWakeObligation(record) === 'retained', - workspaceName: this.resolvedWorktrees - .peek() - ?.worktrees.find((row) => row.id === record.worktreeId)?.displayName - } - } } diff --git a/src/main/runtime/orchestration/mailbox/routing/slept-mailbox-wake.test.ts b/src/main/runtime/orchestration/mailbox/routing/slept-mailbox-wake.test.ts index 478860a1301..d5cd66a62b6 100644 --- a/src/main/runtime/orchestration/mailbox/routing/slept-mailbox-wake.test.ts +++ b/src/main/runtime/orchestration/mailbox/routing/slept-mailbox-wake.test.ts @@ -1,8 +1,12 @@ +import type { ExecutionHostId } from '../../../../../shared/execution-host' import type { WorkspaceSessionState } from '../../../../../shared/workspace-session/workspace-session-state-types' import { OrchestrationMailboxPointerDelivery } from '../terminal-pointer/mailbox-pointer-delivery' import { MailWakeInvocations } from './mail-wake-invocations' import { admitsMailWakeRetry } from './mail-wake-retry-admission' -import { retainMailWakeAttempt } from '../../../session-lifecycle/projection/retained-mail-wake-attempt' +import { + retainMailWakeAttempt, + restoreUndispatchedMailWake +} from '../../../session-lifecycle/projection/retained-mail-wake-attempt' import { RestoredPaneStarter } from '../../../terminal/recovery/restored-pane-start' import type { HostProcessInspection } from '../../../../../shared/terminal/pty/lifecycle/terminal-process-inspection' import type { RuntimeTerminalCreate } from '../../../../../shared/runtime-terminal-create-contract' @@ -185,7 +189,10 @@ it('holds manual Sleep through the sender receipt and keeps its bell until mail runtime.deliverPendingMessagesForHandle(mailbox) await settled() expect(resume).not.toHaveBeenCalled() - expect(runtime.readSleepingMailHold(mailbox)).toMatchObject({ kind: 'asleep', openTab: true }) + expect(await runtime.readSleepingMailHold(mailbox)).toMatchObject({ + kind: 'asleep', + openTab: true + }) expect(await sendDeliveryHoldWarning(runtime, db, mailbox)).toMatchObject({ code: 'recipient_delivery_held', recipient: mailbox @@ -263,6 +270,7 @@ it('retains a failed resume with a live shell, and restarts only a proven invoca let drivingInput: number | null = null let occupied = false let owned = true + let shellForeground = true const start = vi.fn(async () => ({ id: 'replacement' })) const starter = new RestoredPaneStarter({ boundPtyId: () => 'pty-shell', @@ -279,7 +287,7 @@ it('retains a failed resume with a live shell, and restarts only a proven invoca onInspect() return inspection }, - shellInFront: async () => true + shellInFront: async () => shellForeground }) }) const pane = { @@ -320,18 +328,7 @@ it('retains a failed resume with a live shell, and restarts only a proven invoca inspection = { foregroundProcess: 'zsh', hasChildProcesses: true, - childProcessEvidence: 'children', - foregroundObservation: { - verdict: 'observed', - ptyId: 'pty-shell', - ptyIncarnationId: 'inc-shell', - foregroundKind: 'shell', - source: 'pty-master', - processName: 'zsh', - authorityGeneration: 'host-1', - observationEpoch: 1, - capturedAgeMs: 0 - } + childProcessEvidence: 'children' } expect(await resumeFromMail()).toBeNull() inspection = { ...inspection, hasChildProcesses: false, childProcessEvidence: 'no-children' } @@ -357,6 +354,9 @@ it('retains a failed resume with a live shell, and restarts only a proven invoca expect(start).not.toHaveBeenCalled() session.sleepingAgentSessionsByPaneKey[paneKey] = retained onInspect = () => {} + shellForeground = false + expect(await resumeFromMail()).toBeNull() + shellForeground = true expect(await resumeFromMail()).toEqual({ id: 'replacement' }) expect(start).toHaveBeenCalledTimes(1) expect(write).not.toHaveBeenCalled() @@ -413,3 +413,263 @@ it('retains a failed resume with a live shell, and restarts only a proven invoca vi.useRealTimers() } }) + +it('suppresses a definitively confirmed late wake only for its obligation and releases consumed mailbox retention', async () => { + const db = new OrchestrationDb(':memory:') + let sleeping: SleepingAgentSessionRecord | null = record + const wake = vi.fn(async () => false) + const delivery = new SleptMailboxWake({ + db: () => db, + record: () => sleeping, + waiters: () => undefined, + prepare: async () => {}, + wake + }) + try { + const run = db.createRun({ + objective: 'late confirmation', + coordinatorHandle: 'term_old', + coordinatorPaneKey: paneKey + }) + const mailbox = `run:${run.id}` + const message = db.insertMessage({ + runId: run.id, + from: 'worker', + to: mailbox, + subject: 'done' + }) + delivery.request(mailbox) + await settled() + const retained: SleepingAgentSessionRecord = { + ...record, + origin: 'quit', + restoreOnTabOpenOnly: true, + capturedAt: 200, + mailWake: { id: 'obligation', capturedAt: 1, startedAt: 100 } + } + sleeping = retained + delivery.confirmed({ ...retained, mailWake: { ...retained.mailWake!, capturedAt: 2 } }) + expect(delivery.failedToWake(retained)).toBe(true) + delivery.confirmed(retained) + delivery.request(mailbox) + await settled() + expect(wake).toHaveBeenCalledTimes(1) + db.markAsRead([message.id]) + delivery.refresh(mailbox) + expect(Reflect.get(delivery, 'succeeded').size).toBe(0) + expect(Reflect.get(delivery, 'failed').size).toBe(0) + expect(Reflect.get(delivery, 'attempted').size).toBe(0) + + for (let i = 0; i < 20; i++) { + sleeping = record + wake.mockResolvedValueOnce(i % 2 === 0) + const nextRun = db.createRun({ + objective: 'retention', + coordinatorHandle: 'term_old', + coordinatorPaneKey: paneKey + }) + const handle = `run:${nextRun.id}` + const row = db.insertMessage({ + runId: nextRun.id, + from: 'worker', + to: handle, + subject: 'done' + }) + delivery.request(handle) + await settled() + db.markAsRead([row.id]) + delivery.refresh(handle) + } + expect(Reflect.get(delivery, 'succeeded').size).toBe(0) + expect(Reflect.get(delivery, 'failed').size).toBe(0) + expect(Reflect.get(delivery, 'attempted').size).toBe(0) + expect(delivery.heldWorktrees()).toEqual([]) + } finally { + db.close() + } +}) + +it.each([10, 100])( + 'keeps status visits and addressed mail queries bounded with %i saved sleepers', + async (count) => { + const db = new OrchestrationDb(':memory:') + let visits = 0 + let statusChanged: (key: string) => void = () => {} + let sessionWritten: (host: ExecutionHostId | null) => void = () => {} + const records: Record = {} + const mailboxes: string[] = [] + for (let i = 0; i < count; i++) { + const key = `tab-${i}:11111111-1111-4111-8111-111111111111` + records[key] = new Proxy( + { + ...record, + paneKey: key, + tabId: `tab-${i}`, + origin: 'quit', + restoreOnTabOpenOnly: true, + mailWake: { + id: `custody-${i}`, + capturedAt: 1, + startedAt: 100, + attempt: { + ptyId: `pty-${i}`, + incarnationId: `inc-${i}`, + launchToken: `token-${i}`, + startedAt: 100 + } + } + }, + { + get(target, property, receiver) { + visits++ + return Reflect.get(target, property, receiver) + } + } + ) + const run = db.createRun({ + objective: 'scale', + coordinatorHandle: `term_${i}`, + coordinatorPaneKey: key + }) + const mailbox = `run:${run.id}` + mailboxes.push(mailbox) + for (let j = 0; j < 5; j++) { + db.insertMessage({ runId: run.id, from: 'worker', to: mailbox, subject: 'done' }) + } + } + let session = { ...getDefaultWorkspaceSession(), sleepingAgentSessionsByPaneKey: records } + const readSession = vi.fn(() => session) + const runtime = new OrcaRuntimeService( + partialRuntimeStore({ + getWorkspaceSession: readSession, + getWorkspaceSessionHostIds: () => ['local'], + onWorkspaceSessionWritten: (listener) => { + sessionWritten = listener + return () => {} + } + }), + undefined, + { + subscribeAgentStatusRowChanges: (listener) => { + statusChanged = listener + return () => {} + } + } + ) + try { + runtime.setOrchestrationDb(db) + runtime.retrySleptMailboxWakes() + await settled() + visits = 0 + readSession.mockClear() + statusChanged('unrelated:11111111-1111-4111-8111-111111111111') + expect(visits).toBe(0) + statusChanged('tab-0:11111111-1111-4111-8111-111111111111') + // A status event needs only this pane's attempt and PTY identity; no session inventory. + expect(visits).toBeLessThanOrEqual(4) + expect(readSession).not.toHaveBeenCalled() + const inventory = vi.spyOn(db, 'getUndeliveredUnreadMailboxHandles') + const fullRows = vi.spyOn(db, 'getUndeliveredUnreadMessages') + const newest = vi.spyOn(db, 'getUndeliveredUnreadSequence') + runtime.crewChanges.notify( + Object.assign(new Set(['traffic'] as const), { trafficAddresses: new Set([mailboxes[0]!]) }) + ) + await settled() + expect(inventory).not.toHaveBeenCalled() + expect(fullRows).not.toHaveBeenCalled() + // Refresh and the custody check each need one scalar newest-sequence row for the affected mailbox. + expect(newest.mock.calls.length).toBeLessThanOrEqual(3) + expect( + newest.mock.results.filter((result) => result.value !== undefined).length + ).toBeLessThanOrEqual(3) + expect(newest).toHaveBeenCalled() + session = { ...session, sleepingAgentSessionsByPaneKey: {} } + sessionWritten('local') + await settled() + visits = 0 + readSession.mockClear() + statusChanged('tab-0:11111111-1111-4111-8111-111111111111') + expect(visits).toBe(0) + expect(readSession).not.toHaveBeenCalled() + } finally { + db.close() + } + } +) + +it('holds unknown dispatch custody but retries a positively undispatched wake on the record availability edge', async () => { + const db = new OrchestrationDb(':memory:') + const retained: SleepingAgentSessionRecord = { + ...record, + worktreeId: 'repo::/workspace', + origin: 'quit', + restoreOnTabOpenOnly: true, + mailWake: { id: 'not-dispatched', capturedAt: 1, startedAt: 100 } + } + let session: WorkspaceSessionState = { + ...getDefaultWorkspaceSession(), + sleepingAgentSessionsByPaneKey: { [paneKey]: retained } + } + let written: (host: ExecutionHostId | null) => void = () => {} + const runtime = new OrcaRuntimeService( + partialRuntimeStore({ + getWorkspaceSession: () => session, + getWorkspaceSessionHostIds: () => ['local'], + onWorkspaceSessionWritten: (listener) => { + written = listener + return () => {} + } + }) + ) + const resume = vi.fn( + async (_workspace: string, _pane?: string, prepare?: boolean) => prepare === true + ) + const notifier: RuntimeNotifier = { + worktreesChanged: vi.fn(), + reposChanged: vi.fn(), + activateWorktree: vi.fn(), + createTerminal: vi.fn(), + splitTerminal: vi.fn(), + renameTerminal: vi.fn(), + focusTerminal: vi.fn(), + closeTerminal: vi.fn(), + terminalFitOverrideChanged: vi.fn(), + terminalDriverChanged: vi.fn(), + resumeSleepingAgents: resume + } + try { + const run = db.createRun({ + objective: 'refused launch', + coordinatorHandle: 'term_old', + coordinatorPaneKey: paneKey + }) + const mailbox = `run:${run.id}` + db.insertMessage({ runId: run.id, from: 'worker', to: mailbox, subject: 'done' }) + runtime.setOrchestrationDb(db) + runtime.setNotifier(notifier) + runtime.syncWindowGraph(1, { tabs: [], leaves: [] }) + await settled() + expect(await runtime.readSleepingMailHold(mailbox)).toMatchObject({ + resuming: true, + liveShell: false + }) + runtime.retrySleptMailboxWakes() + await settled() + expect(resume.mock.calls.filter((call) => call[2] !== true)).toHaveLength(0) + await restoreUndispatchedMailWake(async (edit) => { + const next = edit(session) + if (next) { + session = next + written('local') + } + }, retained) + await settled() + expect(resume.mock.calls.filter((call) => call[2] !== true)).toEqual([ + ['repo::/workspace', 'tab:11111111-1111-4111-8111-111111111111'] + ]) + expect(session.sleepingAgentSessionsByPaneKey?.[paneKey]?.mailWake).toBeUndefined() + } finally { + runtime.setNotifier(null) + db.close() + } +}) diff --git a/src/main/runtime/orchestration/mailbox/routing/slept-mailbox-wake.ts b/src/main/runtime/orchestration/mailbox/routing/slept-mailbox-wake.ts index c7baa5a700b..24af66733b2 100644 --- a/src/main/runtime/orchestration/mailbox/routing/slept-mailbox-wake.ts +++ b/src/main/runtime/orchestration/mailbox/routing/slept-mailbox-wake.ts @@ -1,9 +1,10 @@ import { POTENTIALLY_LIVE_REMOTE_ATTACHMENT_STATES } from '../../db/federation/remote-attachment-liveness' import type { SleepingAgentSessionRecord } from '../../../../../shared/agent-session/lifecycle/agent-session-resume' +import { agentProviderSessionsEqual } from '../../../../../shared/agent-session/lifecycle/agent-session-resume' import { mailWakeObligation } from '../../../../../shared/agent-session/lifecycle/mail-wake-obligation' import type { OrchestrationDb } from '../../db/orchestration-db' import { - selectOrchestrationPointerBatch, + selectOrchestrationPointerSequence, type OrchestrationMessageWaiter } from '../terminal-pointer/mailbox-pointer-eligibility' @@ -45,12 +46,20 @@ export class SleptMailboxWake { private readonly inFlight = new Map() private readonly succeeded = new Map() private readonly failed = new Map() + private readonly owed = new Map() + private readonly byPane = new Map>() + private readonly attempted = new Map< + string, + { record: SleepingAgentSessionRecord; sequence: number } + >() + private readonly held = new Map() + private readonly workspaceCounts = new Map() private active = 0 constructor(private readonly deps: WakeDeps) {} failedToWake(record: SleepingAgentSessionRecord): boolean { - return this.failed.get(record.paneKey) === record.capturedAt + return this.failed.get(record.paneKey) === (record.mailWake?.capturedAt ?? record.capturedAt) } batch(mailbox: string, reservedTypes?: ReadonlySet) { @@ -60,24 +69,150 @@ export class SleptMailboxWake { if (!db || !record || !mailWakeObligation(record)) { return null } - const messages = selectOrchestrationPointerBatch({ - db, - mailboxHandle: mailbox, - waiters: this.deps.waiters(mailbox), - reservedTypes, - newestFirst: true - }).messages - const sequence = messages[0]?.sequence + const sequence = this.selection(mailbox, reservedTypes)?.sequence return sequence === undefined ? null : { record, sequence } } + private selection(mailbox: string, reservedTypes?: ReadonlySet) { + const db = this.deps.db() + const paneKey = db && mailboxOwnerPane(db, mailbox) + if (!db || !paneKey) { + return null + } + const sequence = selectOrchestrationPointerSequence({ + db, + mailboxHandle: mailbox, + waiters: this.deps.waiters(mailbox), + reservedTypes + }) + return sequence === undefined ? null : { paneKey, sequence } + } + + refresh(mailbox: string): string[] { + const before = this.owed.get(mailbox) + const next = this.selection(mailbox) + if (before) { + const handles = this.byPane.get(before.paneKey) + handles?.delete(mailbox) + if (!handles?.size) { + this.byPane.delete(before.paneKey) + } + } + this.owed.delete(mailbox) + if (next) { + this.owed.set(mailbox, next) + const handles = this.byPane.get(next.paneKey) ?? new Set() + handles.add(mailbox) + this.byPane.set(next.paneKey, handles) + } + const success = this.succeeded.get(mailbox) + if (!next || success?.paneKey !== next.paneKey || success.sequence !== next.sequence) { + this.succeeded.delete(mailbox) + } + const attempted = this.attempted.get(mailbox) + if (!next || (attempted && attempted.record.paneKey !== next.paneKey)) { + this.attempted.delete(mailbox) + } + if (before && !this.byPane.has(before.paneKey)) { + this.failed.delete(before.paneKey) + } + if (before) { + this.refreshPane(before.paneKey) + } + if (next) { + this.refreshPane(next.paneKey) + } + return [...new Set([...(before ? [before.paneKey] : []), ...(next ? [next.paneKey] : [])])] + } + + get hasTrackedMail(): boolean { + return this.owed.size > 0 || this.attempted.size > 0 + } + + reconcile(): void { + const handles = new Set(this.deps.db()?.getUndeliveredUnreadMailboxHandles() ?? []) + for (const handle of this.owed.keys()) { + handles.add(handle) + } + for (const handle of handles) { + this.refresh(handle) + } + } + + mailboxesForPane(paneKey: string): readonly string[] { + return [...(this.byPane.get(paneKey) ?? [])] + } + owesPane(paneKey: string): boolean { - return (this.deps.db()?.getUndeliveredUnreadMailboxHandles() ?? []).some( - (mailbox) => this.batch(mailbox)?.record.paneKey === paneKey - ) + for (const mailbox of this.mailboxesForPane(paneKey)) { + this.refresh(mailbox) + if (this.batch(mailbox)?.record.paneKey === paneKey) { + return true + } + } + return false + } + + refreshPane(paneKey: string): void { + const previous = this.held.get(paneKey) + const record = this.deps.record(paneKey) + const next = + this.byPane.has(paneKey) && mailWakeObligation(record) ? record?.worktreeId : undefined + if (previous === next) { + return + } + if (previous) { + const count = (this.workspaceCounts.get(previous) ?? 1) - 1 + if (count) { + this.workspaceCounts.set(previous, count) + } else { + this.workspaceCounts.delete(previous) + } + } + this.held.delete(paneKey) + if (next) { + this.held.set(paneKey, next) + this.workspaceCounts.set(next, (this.workspaceCounts.get(next) ?? 0) + 1) + } + if (!record) { + this.failed.delete(paneKey) + } + } + + heldWorktrees(): string[] { + return [...this.workspaceCounts.keys()].sort() + } + + confirmed(record: SleepingAgentSessionRecord): void { + for (const mailbox of this.mailboxesForPane(record.paneKey)) { + const attempt = this.attempted.get(mailbox) + if ( + !attempt || + attempt.record.worktreeId !== record.worktreeId || + attempt.record.agent !== record.agent || + (attempt.record.connectionId ?? null) !== (record.connectionId ?? null) || + (attempt.record.mailWake?.capturedAt ?? attempt.record.capturedAt) !== + record.mailWake?.capturedAt || + (attempt.record.mailWake && attempt.record.mailWake.id !== record.mailWake?.id) || + !agentProviderSessionsEqual( + record.agent, + record.providerSession, + attempt.record.providerSession + ) + ) { + continue + } + this.refresh(mailbox) + if (this.owed.get(mailbox)?.paneKey === record.paneKey) { + this.succeeded.set(mailbox, { paneKey: record.paneKey, sequence: attempt.sequence }) + this.failed.delete(record.paneKey) + } + this.attempted.delete(mailbox) + } } request(mailbox: string, reservedTypes?: ReadonlySet): void { + this.refresh(mailbox) const batch = this.batch(mailbox, reservedTypes) if ( !batch || @@ -125,6 +260,7 @@ export class SleptMailboxWake { ) { return } + this.failed.delete(before.record.paneKey) await this.deps.prepare(before.record) const current = this.batch(mailbox, reservedTypes) if ( @@ -134,11 +270,23 @@ export class SleptMailboxWake { ) { return } - if (await this.deps.wake(current.record).catch(() => false)) { + this.attempted.set(mailbox, { record: current.record, sequence: current.sequence }) + const succeeded = await this.deps.wake(current.record).catch(() => false) + this.refresh(mailbox) + if (this.owed.get(mailbox)?.paneKey !== current.record.paneKey) { + return + } + if (succeeded) { + if (this.owed.get(mailbox)?.sequence === current.sequence) { + this.succeeded.set(mailbox, { paneKey: current.record.paneKey, sequence: current.sequence }) + } this.failed.delete(current.record.paneKey) - this.succeeded.set(mailbox, { paneKey: current.record.paneKey, sequence: current.sequence }) - } else { - this.failed.set(current.record.paneKey, current.record.capturedAt) + this.attempted.delete(mailbox) + } else if (this.deps.record(current.record.paneKey)) { + this.failed.set( + current.record.paneKey, + current.record.mailWake?.capturedAt ?? current.record.capturedAt + ) } } } diff --git a/src/main/runtime/orchestration/mailbox/routing/terminal-pane-mail-routing.ts b/src/main/runtime/orchestration/mailbox/routing/terminal-pane-mail-routing.ts index 7e5feca15c7..3b0d0077cdc 100644 --- a/src/main/runtime/orchestration/mailbox/routing/terminal-pane-mail-routing.ts +++ b/src/main/runtime/orchestration/mailbox/routing/terminal-pane-mail-routing.ts @@ -235,7 +235,7 @@ export abstract class OrcaRuntimeWithGetPtyRecordForPaneKey extends RuntimeSlept * longer owes. Visible cleanup only: it never holds up the action that called it. `mailboxes`: * only those the action touched; without it, every mailbox (a reset, a store becoming readable). */ withdrawStaleChatMail(mailboxes?: Iterable): void { - this.publishSleepingMail() + this.publishSleepingMail(mailboxes) this.orchestrationStructuredMailboxPointerDelivery.withdrawStale(mailboxes) } diff --git a/src/main/runtime/orchestration/mailbox/terminal-pointer/mailbox-pointer-delivery.ts b/src/main/runtime/orchestration/mailbox/terminal-pointer/mailbox-pointer-delivery.ts index ea19000ae12..7260e9c3835 100644 --- a/src/main/runtime/orchestration/mailbox/terminal-pointer/mailbox-pointer-delivery.ts +++ b/src/main/runtime/orchestration/mailbox/terminal-pointer/mailbox-pointer-delivery.ts @@ -19,9 +19,13 @@ import { import { resumePendingOrchestrationMailboxPointer } from './mailbox-pointer-resume' import { stageOrchestrationMailboxPointer } from './mailbox-pointer-stage' +const DEFAULT_POINTER_ENTER_DELAY_MS = 500 + function pointerEnterDelayMs(): number { const configured = Number(process.env.ORCA_E2E_ORCHESTRATION_POINTER_ENTER_DELAY_MS) - return Number.isFinite(configured) && configured >= 1 && configured <= 60_000 ? configured : 500 + return Number.isFinite(configured) && configured >= 1 && configured <= 60_000 + ? configured + : DEFAULT_POINTER_ENTER_DELAY_MS } export class OrchestrationMailboxPointerDelivery { diff --git a/src/main/runtime/orchestration/mailbox/terminal-pointer/mailbox-pointer-eligibility.ts b/src/main/runtime/orchestration/mailbox/terminal-pointer/mailbox-pointer-eligibility.ts index afb2be75c60..cc4625a88f0 100644 --- a/src/main/runtime/orchestration/mailbox/terminal-pointer/mailbox-pointer-eligibility.ts +++ b/src/main/runtime/orchestration/mailbox/terminal-pointer/mailbox-pointer-eligibility.ts @@ -40,7 +40,7 @@ export type OrchestrationPointerBatch = { /** Both delivery lanes share this synchronous SQL selection; unfiltered waiters preempt all pointers. * Concrete waiter types are excluded byte-exactly in SQL; no JavaScript post-filter is needed. * An unacked batch defers rather than drops: only mail it does not hold is eligible, once each. */ -export function selectOrchestrationPointerBatch(input: { +type PointerSelection = { db: OrchestrationDb mailboxHandle: string waiters: ReadonlySet | undefined @@ -48,22 +48,44 @@ export function selectOrchestrationPointerBatch(input: { /** A terminal's own mailbox leaves rows a Run owns undelivered for routing to move. */ excludeRunOwnedDirect?: boolean newestFirst?: boolean -}): OrchestrationPointerBatch { - if (hasUnfilteredOrchestrationWaiter(input.waiters)) { - return { messages: [], heldDeliveryId: undefined } - } +} + +function pointerExclusions(input: PointerSelection, excludeIds?: readonly string[]) { const excludedTypes = new Set(input.reservedTypes) for (const waiter of input.waiters ?? []) { for (const type of waiter.typeFilter ?? []) { excludedTypes.add(type) } } + return { + excludeTypes: [...excludedTypes], + excludeIds, + excludeRunOwnedDirect: input.excludeRunOwnedDirect + } +} + +export function selectOrchestrationPointerSequence(input: PointerSelection): number | undefined { + return hasUnfilteredOrchestrationWaiter(input.waiters) + ? undefined + : input.db.getUndeliveredUnreadSequence( + input.mailboxHandle, + pointerExclusions( + input, + input.db.getOutstandingMailboxDelivery?.(input.mailboxHandle)?.messageIds + ) + ) +} + +export function selectOrchestrationPointerBatch( + input: PointerSelection +): OrchestrationPointerBatch { + if (hasUnfilteredOrchestrationWaiter(input.waiters)) { + return { messages: [], heldDeliveryId: undefined } + } const held = input.db.getOutstandingMailboxDelivery?.(input.mailboxHandle) return { messages: input.db.getUndeliveredUnreadMessages(input.mailboxHandle, undefined, { - excludeTypes: [...excludedTypes], - excludeIds: held?.messageIds, - excludeRunOwnedDirect: input.excludeRunOwnedDirect, + ...pointerExclusions(input, held?.messageIds), newestFirst: input.newestFirst, limit: ORCHESTRATION_DELIVERY_BATCH_LIMIT }), diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/send-delivery-hold.ts b/src/main/runtime/rpc/methods/orchestration/messaging/send-delivery-hold.ts index 6aa8faefd69..c9363e0ba1a 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/send-delivery-hold.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/send-delivery-hold.ts @@ -31,7 +31,7 @@ async function readDeliveryHoldWarning( recipient: string ): Promise { const hold = - runtime.readSleepingMailHold?.(recipient) ?? + (await runtime.readSleepingMailHold?.(recipient)) ?? (await readMailboxDeliveryHold(recipient, { chatSessionFor: (mailbox) => { const custody = structuredMailboxCustody(mailbox, db) diff --git a/src/main/runtime/session-lifecycle/projection/restored-pane-materialize.ts b/src/main/runtime/session-lifecycle/projection/restored-pane-materialize.ts index 33e7e2bd3ca..5152045969e 100644 --- a/src/main/runtime/session-lifecycle/projection/restored-pane-materialize.ts +++ b/src/main/runtime/session-lifecycle/projection/restored-pane-materialize.ts @@ -1,5 +1,5 @@ import { storedPaneBinding } from '../../terminal/recovery/restored-pane-binding' -import { retainMailWakeAttempt } from './retained-mail-wake-attempt' +import { retainMailWakeAttempt, restoreUndispatchedMailWake } from './retained-mail-wake-attempt' // Restore terminal-less saved panes for pending-row activation and workspace wakes; main-owned layouts also handle triggers outside desktop restore. import type { OrcaRuntimeWithAgentSessionResumeOwnership as AgentSessionResume } from './agent-session-resume-ownership' @@ -200,6 +200,7 @@ export abstract class OrcaRuntimeWithRestoredPaneStart extends OrcaRuntimeWithBu const resumeStartup = await this.resolvePaneOwnedResumeStartup(worktreeId, tab) // After a claimed-record resume starts, mark that record live. const paneKey = `${tab.parentTabId}:${tab.leafId}` + const hostId = this.getWorkspaceSessionHostIdForWorktree(worktreeId) const savedRecord = this.getWorkspaceSessionForWorktree(worktreeId)?.sleepingAgentSessionsByPaneKey?.[paneKey] const resumedRecord = @@ -272,11 +273,7 @@ export abstract class OrcaRuntimeWithRestoredPaneStart extends OrcaRuntimeWithBu onPtyCreated: (ptyId: string, result: RuntimeTerminalCreate) => { if (resumedRecord?.mailWake) { mailWakeWrite = retainMailWakeAttempt( - (edit) => - this.commitWorkspaceSessionEdit( - this.getWorkspaceSessionHostIdForWorktree(worktreeId), - edit - ), + (edit) => this.commitWorkspaceSessionEdit(hostId, edit), resumedRecord, ptyId, result @@ -312,7 +309,6 @@ export abstract class OrcaRuntimeWithRestoredPaneStart extends OrcaRuntimeWithBu await mailWakeWrite // A running resumed agent's record must not remain asleep. if (created.tab.ptyId && resumedRecord) { - const hostId = this.getWorkspaceSessionHostIdForWorktree(worktreeId) void markResumedRecordLive( (edit) => this.commitWorkspaceSessionEdit(hostId, edit), paneKey, @@ -322,6 +318,12 @@ export abstract class OrcaRuntimeWithRestoredPaneStart extends OrcaRuntimeWithBu return created } catch (err) { resumeClaim?.settleFailure(dispatched, err) + if (!dispatched && resumedRecord) { + await restoreUndispatchedMailWake( + (edit) => this.commitWorkspaceSessionEdit(hostId, edit), + resumedRecord + ) + } if ( !(automatic && isWorktreeTerminalsSleepingError(err)) && sessionId && diff --git a/src/main/runtime/session-lifecycle/projection/retained-mail-wake-attempt.ts b/src/main/runtime/session-lifecycle/projection/retained-mail-wake-attempt.ts index a21ba8e10c0..5cbf6ea2d37 100644 --- a/src/main/runtime/session-lifecycle/projection/retained-mail-wake-attempt.ts +++ b/src/main/runtime/session-lifecycle/projection/retained-mail-wake-attempt.ts @@ -1,4 +1,7 @@ -import { mailWakeObligation } from '../../../../shared/agent-session/lifecycle/mail-wake-obligation' +import { + mailWakeObligation, + undispatchedMailWakeRecord +} from '../../../../shared/agent-session/lifecycle/mail-wake-obligation' import type { SleepingAgentSessionRecord } from '../../../../shared/agent-session/lifecycle/agent-session-resume' import type { RuntimeTerminalCreate } from '../../../../shared/runtime-terminal-create-contract' import type { WorkspaceSessionState } from '../../../../shared/workspace-session/workspace-session-state-types' @@ -46,3 +49,37 @@ export async function retainMailWakeAttempt( } }) } + +export function replaceMailWakeRecord( + session: WorkspaceSessionState, + record: SleepingAgentSessionRecord, + next: SleepingAgentSessionRecord +): WorkspaceSessionState | null { + const current = session.sleepingAgentSessionsByPaneKey?.[record.paneKey] + if ( + current?.mailWake?.id !== record.mailWake?.id || + current?.capturedAt !== record.capturedAt || + current?.origin !== record.origin || + current?.restoreOnTabOpenOnly !== record.restoreOnTabOpenOnly || + current?.mailWake?.attempt?.launchToken !== record.mailWake?.attempt?.launchToken + ) { + return null + } + return { + ...session, + sleepingAgentSessionsByPaneKey: { + ...session.sleepingAgentSessionsByPaneKey, + [record.paneKey]: next + } + } +} + +export async function restoreUndispatchedMailWake( + commit: Commit, + record: SleepingAgentSessionRecord +): Promise { + const next = undispatchedMailWakeRecord(record) + if (next) { + await commit((session) => replaceMailWakeRecord(session, record, next)) + } +} diff --git a/src/renderer/src/components/terminal-pane/pty-connection/agent-identity/pane-agent-identity.ts b/src/renderer/src/components/terminal-pane/pty-connection/agent-identity/pane-agent-identity.ts index 1faac50ad67..2d263d59483 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection/agent-identity/pane-agent-identity.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection/agent-identity/pane-agent-identity.ts @@ -166,7 +166,7 @@ export function installPaneAgentIdentity(session: ConnectPanePtySession): void { const isHibernationWakeRecord = (ptyId && state.suppressedPtyExitIds[ptyId]) || (wake && wake.record === sleepingRecord?.record && (ptyId === null || wake.ptyId === ptyId)) - if (sleepingRecord && !isHibernationWakeRecord) { + if (sleepingRecord && !sleepingRecord.record.mailWake && !isHibernationWakeRecord) { session.clearSleepingRecordProviderDuplicates(state, sleepingRecord) } session.clearStaleAgentTabTitleOnConfirmedShell() diff --git a/src/renderer/src/components/terminal-pane/pty-connection/lifecycle/transport-output-callbacks.ts b/src/renderer/src/components/terminal-pane/pty-connection/lifecycle/transport-output-callbacks.ts index 64a5151a571..ad62d046d3d 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection/lifecycle/transport-output-callbacks.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection/lifecycle/transport-output-callbacks.ts @@ -1,3 +1,9 @@ +import { + currentMailWakeStartup, + isColdRestoreStartup, + restoreUndispatchedMailWake +} from '../startup/pane-resume-record' +import { useAppStore } from '@/store/index' import { applyTerminalLaunchResult, captureTerminalLaunchContext @@ -140,6 +146,15 @@ export function bindCaptureTransportOutputCallbacks(session: ConnectPanePtySessi } }, onSpawnDeferred: (reason: SpawnDeferralReason): void => { + if (isCurrent() && isColdRestoreStartup(savedStartup)) { + const record = currentMailWakeStartup( + savedStartup, + useAppStore.getState().sleepingAgentSessionsByPaneKey + ).sleepingRecordEntry?.record + if (record) { + restoreUndispatchedMailWake(record) + } + } if (isCurrent()) { session.pendingSpawnDeferral = reason } diff --git a/src/renderer/src/components/terminal-pane/pty-connection/startup/pane-resume-record.ts b/src/renderer/src/components/terminal-pane/pty-connection/startup/pane-resume-record.ts index 707de99c165..01e60d892a9 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection/startup/pane-resume-record.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection/startup/pane-resume-record.ts @@ -1,3 +1,4 @@ +import type { PtyPaneStartup } from './pty-pane-startup' import { useAppStore } from '@/store/index' import { recordPaneWakeThroughLayoutOwner } from '@/store/terminals/terminal-sleep-layout-owner' import { ORCA_LAYOUT_OWNER_IS_MAIN } from '../../../../../../shared/layout-owner-switch' @@ -8,10 +9,13 @@ import { } from '../../../../../../shared/agent-session/lifecycle/agent-session-resume' import type { ConnectPanePtySession } from '../connect-pane-pty-session' import type { ColdRestoreAgentResumeStartup, PendingStartupCommand } from './fresh-spawn-types' -import { mailWakeObligation } from '../../../../../../shared/agent-session/lifecycle/mail-wake-obligation' +import { + mailWakeObligation, + undispatchedMailWakeRecord +} from '../../../../../../shared/agent-session/lifecycle/mail-wake-obligation' export function isColdRestoreStartup( - startup: PendingStartupCommand | null | undefined + startup: PendingStartupCommand | PtyPaneStartup | undefined ): startup is ColdRestoreAgentResumeStartup { return Boolean( startup && @@ -114,7 +118,11 @@ export function installPaneResumeRecord(session: ConnectPanePtySession): void { } else { recordPaneWakeThroughLayoutOwner(state, session.deps.worktreeId, session.cacheKey) } - session.clearSleepingRecordProviderDuplicates(state, startup.sleepingRecordEntry) + session.clearSleepingRecordProviderDuplicates( + state, + startup.sleepingRecordEntry, + session.cacheKey + ) } return } @@ -150,3 +158,12 @@ export function installPaneResumeRecord(session: ConnectPanePtySession): void { } } } + +/** Called only after a definite refusal before dispatch, never after a lost spawn reply. */ +export function restoreUndispatchedMailWake(record: SleepingAgentSessionRecord): void { + const state = useAppStore.getState() + const next = undispatchedMailWakeRecord(record) + if (next && state.sleepingAgentSessionsByPaneKey[record.paneKey] === record) { + state.rememberSleepingAgentSession(next, record) + } +} diff --git a/src/renderer/src/components/terminal-pane/pty-connection/startup/sleeping-record-access.ts b/src/renderer/src/components/terminal-pane/pty-connection/startup/sleeping-record-access.ts index 9ff1154ada3..7a90433555b 100644 --- a/src/renderer/src/components/terminal-pane/pty-connection/startup/sleeping-record-access.ts +++ b/src/renderer/src/components/terminal-pane/pty-connection/startup/sleeping-record-access.ts @@ -47,7 +47,11 @@ export function installSleepingRecordAccess(session: ConnectPanePtySession): voi consumed: { paneKey: string; record: SleepingAgentSessionRecord }, keepPaneKey?: string ): void => { - if (consumed.paneKey !== keepPaneKey) { + if ( + consumed.paneKey !== keepPaneKey && + useAppStore.getState().sleepingAgentSessionsByPaneKey[consumed.paneKey] === consumed.record && + !consumed.record.mailWake + ) { state.clearSleepingAgentSession(consumed.paneKey) } for (const [paneKey, record] of sleepingAgentRecordIndex.entries( @@ -55,6 +59,9 @@ export function installSleepingRecordAccess(session: ConnectPanePtySession): voi consumed.record.worktreeId )) { if ( + useAppStore.getState().sleepingAgentSessionsByPaneKey[paneKey] === record && + !record.mailWake && + record.capturedAt <= consumed.record.capturedAt && paneKey !== consumed.paneKey && paneKey !== keepPaneKey && record.worktreeId === consumed.record.worktreeId && diff --git a/src/renderer/src/lib/agent-session/hibernation/agent-hibernation-coordinator.ts b/src/renderer/src/lib/agent-session/hibernation/agent-hibernation-coordinator.ts index caa584ae727..6918cdc1aa6 100644 --- a/src/renderer/src/lib/agent-session/hibernation/agent-hibernation-coordinator.ts +++ b/src/renderer/src/lib/agent-session/hibernation/agent-hibernation-coordinator.ts @@ -1,3 +1,4 @@ +import { trackAgentPaneSleep } from '@/store/terminals/agent-pane-sleep-in-flight' import { useAppStore } from '@/store/index' import { getEffectiveAgentHibernationIdleMs, @@ -145,15 +146,17 @@ async function hibernatePaneIfStillEligible( try { const state = useAppStore.getState() const owner = resolveAgentHibernationRuntimeOwner(state, worktreeId) - await state.shutdownCompletedAgentPaneForHibernation(worktreeId, { - paneKey: confirmedCandidate.paneKey, - tabId: confirmedCandidate.tabId, - leafId: confirmedCandidate.leafId, - ptyId: confirmedCandidate.targetPtyIds[0], - ...(owner.kind === 'runtime' - ? { expectedRuntimePtyId: confirmedCandidate.expectedRuntimePtyIds[0] } - : {}) - }) + await trackAgentPaneSleep(confirmedCandidate.paneKey, () => + state.shutdownCompletedAgentPaneForHibernation(worktreeId, { + paneKey: confirmedCandidate.paneKey, + tabId: confirmedCandidate.tabId, + leafId: confirmedCandidate.leafId, + ptyId: confirmedCandidate.targetPtyIds[0], + ...(owner.kind === 'runtime' + ? { expectedRuntimePtyId: confirmedCandidate.expectedRuntimePtyIds[0] } + : {}) + }) + ) } catch (err) { console.warn('[agent-hibernation] failed to hibernate agent pane:', id, err) } finally { diff --git a/src/renderer/src/lib/agent-session/resume/sleeping-agent-wake-requests.test.ts b/src/renderer/src/lib/agent-session/resume/sleeping-agent-wake-requests.test.ts index 56e93596301..6c6fa34cd49 100644 --- a/src/renderer/src/lib/agent-session/resume/sleeping-agent-wake-requests.test.ts +++ b/src/renderer/src/lib/agent-session/resume/sleeping-agent-wake-requests.test.ts @@ -1,6 +1,23 @@ +import { useAppStore } from '@/store/index' +import { installSleepingRecordAccess } from '@/components/terminal-pane/pty-connection/startup/sleeping-record-access' +import { installPaneAgentIdentity } from '@/components/terminal-pane/pty-connection/agent-identity/pane-agent-identity' +import { bindCaptureTransportOutputCallbacks } from '@/components/terminal-pane/pty-connection/lifecycle/transport-output-callbacks' +const foreground = vi.hoisted(() => ({ onShell: (_reason: 'process-exit') => {} })) +vi.mock( + '@/components/terminal-pane/pty-connection/agent-identity/foreground-process/pane-foreground-agent-tracker', + () => ({ + createPaneForegroundAgentTracker: (options: { + onConfirmedShellForeground: typeof foreground.onShell + }) => { + foreground.onShell = options.onConfirmedShellForeground + return {} + } + }) +) import { currentMailWakeStartup, - paneResumeRecord + paneResumeRecord, + installPaneResumeRecord } from '@/components/terminal-pane/pty-connection/startup/pane-resume-record' import { hasEmptyReattachRetireEvidence } from '@/components/terminal-pane/pty-connection/reattach/empty-reattach-retire-evidence' import type { ColdRestoreAgentResumeStartup } from '@/components/terminal-pane/pty-connection/startup/fresh-spawn-types' @@ -102,3 +119,130 @@ it('keeps pre-launch custody on its quit bookmark through an already armed wake currentMailWakeStartup(startup, { [original.paneKey]: manual }).sleepingRecordEntry?.record ).toBe(original) }) + +it('preserves custody through the installed spawn/alias/shell callbacks and restores only a definite prelaunch refusal', () => { + const original: SleepingAgentSessionRecord = { + paneKey: 'tab:11111111-1111-4111-8111-111111111111', + tabId: 'tab', + worktreeId: 'workspace', + agent: 'codex', + providerSession: { key: 'session_id', id: 'conversation' }, + prompt: '', + state: 'done', + origin: 'worktree-sleep', + capturedAt: 1, + updatedAt: 1 + } + const retained: SleepingAgentSessionRecord = { + ...original, + origin: 'quit', + restoreOnTabOpenOnly: true, + mailWake: { id: 'obligation', capturedAt: 1, startedAt: 100 } + } + const alias = { ...retained, paneKey: 'tab:22222222-2222-4222-8222-222222222222' } + const replacement = { + ...original, + paneKey: 'tab:33333333-3333-4333-8333-333333333333', + capturedAt: 2 + } + state.sleepingAgentSessionsByPaneKey = { + [original.paneKey]: retained, + [alias.paneKey]: alias, + [replacement.paneKey]: replacement + } + Object.assign(state, { + agentLaunchConfigByPaneKey: {}, + agentStatusByPaneKey: {}, + paneForegroundAgentByPaneKey: {}, + automaticAgentResumeClaimsByTabId: {}, + tabsByWorktree: {}, + suppressedPtyExitIds: {}, + rememberSleepingAgentSession: ( + record: SleepingAgentSessionRecord, + expected?: SleepingAgentSessionRecord + ) => { + if (expected && state.sleepingAgentSessionsByPaneKey[record.paneKey] !== expected) { + return + } + state.sleepingAgentSessionsByPaneKey = { + ...state.sleepingAgentSessionsByPaneKey, + [record.paneKey]: record + } + }, + clearSleepingAgentSession: (key: string) => { + const next = { ...state.sleepingAgentSessionsByPaneKey } + delete next[key] + state.sleepingAgentSessionsByPaneKey = next + }, + clearAgentLaunchConfig: vi.fn() + }) + const transport = { getPtyId: () => 'pty-shell' } + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the installers exercise the listed pane/store callbacks; unused transport methods are never called. + const session = { + cacheKey: original.paneKey, + paneStartup: null, + remotePtyIncarnationId: 'inc', + transport, + transportStreamGeneration: 0, + disposed: false, + pane: { + id: 1, + leafId: '11111111-1111-4111-8111-111111111111', + terminal: { + parser: { registerOscHandler: () => ({ dispose() {} }) }, + onRender: () => ({ dispose() {} }) + } + }, + deps: { + tabId: 'tab', + worktreeId: 'workspace', + isVisibleRef: { current: false }, + paneTransportsRef: { current: new Map([[1, transport]]) }, + onAgentExitedRef: { current: vi.fn() } + }, + writeInputModeGround: vi.fn(), + createProcessExitState: () => ({ detector: { observe() {} } }) + } as unknown as ConnectPanePtySession + installSleepingRecordAccess(session) + installPaneResumeRecord(session) + const startup: ColdRestoreAgentResumeStartup = { + agent: 'codex', + command: 'codex resume conversation', + resumeProviderSession: original.providerSession, + launchToken: 'invocation', + resumeOrigin: 'saved', + launchConfig: { agentArgs: '', agentEnv: {} }, + useLiveEntry: false, + hasSleepingRecord: true, + sleepingRecordEntry: { paneKey: original.paneKey, record: original } + } + session.keepResumeRecordAfterColdRestoreSpawn(startup) + expect(state.sleepingAgentSessionsByPaneKey[original.paneKey]?.mailWake?.attempt).toEqual({ + ptyId: 'pty-shell', + incarnationId: 'inc', + launchToken: 'invocation', + startedAt: 100 + }) + session.clearSleepingRecordProviderDuplicates(useAppStore.getState(), { + paneKey: original.paneKey, + record: original + }) + installPaneAgentIdentity(session) + foreground.onShell('process-exit') + expect(state.sleepingAgentSessionsByPaneKey[original.paneKey]?.mailWake?.id).toBe('obligation') + expect(state.sleepingAgentSessionsByPaneKey[alias.paneKey]?.mailWake?.id).toBe('obligation') + expect(state.sleepingAgentSessionsByPaneKey[replacement.paneKey]).toBe(replacement) + session.commandLifecycle.dispose() + + state.sleepingAgentSessionsByPaneKey = { [retained.paneKey]: retained } + bindCaptureTransportOutputCallbacks(session) + const capture = session.captureTransportOutputCallbacks(vi.fn(), null, startup) + expect(state.sleepingAgentSessionsByPaneKey[retained.paneKey]?.mailWake?.id).toBe('obligation') + capture.callbacks.onSpawnDeferred?.('paused') + expect(state.sleepingAgentSessionsByPaneKey[retained.paneKey]).toMatchObject({ + origin: 'worktree-sleep', + capturedAt: 1 + }) + expect(state.sleepingAgentSessionsByPaneKey[retained.paneKey]?.mailWake).toBeUndefined() + session.startupTiming?.finish('replaced') +}) diff --git a/src/renderer/src/lib/agent-session/resume/wake-sleeping-agents-in-background.ts b/src/renderer/src/lib/agent-session/resume/wake-sleeping-agents-in-background.ts index edde535f1c1..9bc14c21d61 100644 --- a/src/renderer/src/lib/agent-session/resume/wake-sleeping-agents-in-background.ts +++ b/src/renderer/src/lib/agent-session/resume/wake-sleeping-agents-in-background.ts @@ -1,3 +1,4 @@ +import { restoreUndispatchedMailWake } from '@/components/terminal-pane/pty-connection/startup/pane-resume-record' import { patchWorkspaceSessionByHost } from '../../workspace/persistence/workspace-session-host-persistence' import { createNonSecureContextUuid } from '../../../../../shared/non-secure-context-uuid' import { waitForAgentPaneSleep } from '@/store/terminals/agent-pane-sleep-in-flight' @@ -269,14 +270,19 @@ async function wakeSleepingPaneInBackground( } } useAppStore.getState().rememberSleepingAgentSession(captured, record) - await patchWorkspaceSessionByHost( - window.api.session, - { - sleepingAgentSessionsByPaneKey: useAppStore.getState().sleepingAgentSessionsByPaneKey - }, - useAppStore.getState() - ).written - await window.api.session.flush() + try { + await patchWorkspaceSessionByHost( + window.api.session, + { + sleepingAgentSessionsByPaneKey: useAppStore.getState().sleepingAgentSessionsByPaneKey + }, + useAppStore.getState() + ).written + await window.api.session.flush() + } catch (error) { + restoreUndispatchedMailWake(captured) + throw error + } if (useAppStore.getState().sleepingAgentSessionsByPaneKey[paneKey] !== captured) { return false } diff --git a/src/renderer/src/store/terminals/terminal-pane-hibernation.ts b/src/renderer/src/store/terminals/terminal-pane-hibernation.ts index 04500fdb53c..09ad04a392b 100644 --- a/src/renderer/src/store/terminals/terminal-pane-hibernation.ts +++ b/src/renderer/src/store/terminals/terminal-pane-hibernation.ts @@ -1,4 +1,3 @@ -import { trackAgentPaneSleep } from './agent-pane-sleep-in-flight' import { replaceRecordEntries, replaceTabBuckets } from '@/store/keyed-record-transitions' import { isRemoteRuntimePtyId } from '../../../../shared/remote-runtime/remote-runtime-pty-id' import { parsePaneKey } from '../../../../shared/workspace-layout/stable-pane-id' @@ -29,239 +28,231 @@ export function createTerminalPaneHibernationActions( get: TerminalStoreGet ): Pick { return { - shutdownCompletedAgentPaneForHibernation: (worktreeId, opts) => - trackAgentPaneSleep(opts.paneKey, async () => { - const paneKeys = [opts.paneKey] - const expectedRuntimePtyIds = sortedUniquePtyIds( - opts.expectedRuntimePtyId ? [opts.expectedRuntimePtyId] : [] - ) - const rendererShutdownPtyIds = [opts.ptyId] - const state = get() - const worktreeOwner = resolveTerminalPtyOwner(state, worktreeId) - const teardownOwner = resolveTerminalTeardownOwner(worktreeOwner, [opts.ptyId]) - // Use PTY owner for twins; losing bindings without an attributable host stop leaves unseen running agents. - if ( - teardownOwner.kind === 'refused' || - (worktreeOwner.kind === 'ambiguous' && - isRemoteRuntimePtyId(opts.ptyId) && - expectedRuntimePtyIds.length === 0) - ) { - console.warn( - '[agent-hibernation] refusing pane hibernation: terminal owner is ambiguous', - { - worktreeId, - paneKey: opts.paneKey - } - ) - throw new Error('ambiguous_terminal_owner') - } - const runtimeEnvironmentId = teardownOwner.runtimeEnvironmentId - // Why: pane transports emit renderer PTY ids, not raw exact-stop handles; guard only the identity that can deliver an exit callback. - const exitGuardPtyIds = [opts.ptyId] - const tab = (state.tabsByWorktree[worktreeId] ?? []).find( - (candidate) => candidate.id === opts.tabId - ) - const parsed = parsePaneKey(opts.paneKey) - const layout = state.terminalLayoutsByTabId[opts.tabId] - const liveTabPtyIds = state.ptyIdsByTabId[opts.tabId] ?? [] - if ( - !tab || - !parsed || - parsed.tabId !== opts.tabId || - parsed.leafId !== opts.leafId || - layout?.ptyIdsByLeafId?.[opts.leafId] !== opts.ptyId || - (expectedRuntimePtyIds.length === 0 && !liveTabPtyIds.includes(opts.ptyId)) - ) { - throw new Error('agent_hibernation_pane_binding_mismatch') - } - const sleepingAgentSessionRecords = collectSleepingAgentSessionRecordsForWorktree( - state, + shutdownCompletedAgentPaneForHibernation: async (worktreeId, opts) => { + const paneKeys = [opts.paneKey] + const expectedRuntimePtyIds = sortedUniquePtyIds( + opts.expectedRuntimePtyId ? [opts.expectedRuntimePtyId] : [] + ) + const rendererShutdownPtyIds = [opts.ptyId] + const state = get() + const worktreeOwner = resolveTerminalPtyOwner(state, worktreeId) + const teardownOwner = resolveTerminalTeardownOwner(worktreeOwner, [opts.ptyId]) + // Use PTY owner for twins; losing bindings without an attributable host stop leaves unseen running agents. + if ( + teardownOwner.kind === 'refused' || + (worktreeOwner.kind === 'ambiguous' && + isRemoteRuntimePtyId(opts.ptyId) && + expectedRuntimePtyIds.length === 0) + ) { + console.warn('[agent-hibernation] refusing pane hibernation: terminal owner is ambiguous', { worktreeId, - { - paneKeys, - captureMode: 'completed-agent-hibernation' + paneKey: opts.paneKey + }) + throw new Error('ambiguous_terminal_owner') + } + const runtimeEnvironmentId = teardownOwner.runtimeEnvironmentId + // Why: pane transports emit renderer PTY ids, not raw exact-stop handles; guard only the identity that can deliver an exit callback. + const exitGuardPtyIds = [opts.ptyId] + const tab = (state.tabsByWorktree[worktreeId] ?? []).find( + (candidate) => candidate.id === opts.tabId + ) + const parsed = parsePaneKey(opts.paneKey) + const layout = state.terminalLayoutsByTabId[opts.tabId] + const liveTabPtyIds = state.ptyIdsByTabId[opts.tabId] ?? [] + if ( + !tab || + !parsed || + parsed.tabId !== opts.tabId || + parsed.leafId !== opts.leafId || + layout?.ptyIdsByLeafId?.[opts.leafId] !== opts.ptyId || + (expectedRuntimePtyIds.length === 0 && !liveTabPtyIds.includes(opts.ptyId)) + ) { + throw new Error('agent_hibernation_pane_binding_mismatch') + } + const sleepingAgentSessionRecords = collectSleepingAgentSessionRecordsForWorktree( + state, + worktreeId, + { + paneKeys, + captureMode: 'completed-agent-hibernation' + } + ) + const retainedCompletionEvidence = collectHibernatedCompletionEvidenceForWorktree( + state, + worktreeId, + paneKeys + ) + if (!sleepingAgentSessionRecords[opts.paneKey]) { + // Why: killing the PTY with no persisted resume record strands the pane unwakeable; abort instead of hibernating unrecoverably. + throw new Error('agent_hibernation_capture_missing') + } + const capture = shutdownBufferCaptures.get(opts.tabId) + if (capture) { + try { + capture({ includeLocalBuffers: false }) + } catch { + // Don't let one tab's capture failure block the pane hibernation. + } + } + // Why: store sleeping records before kill, since pty:exit can arrive first. + const sleepingRecordKeys = Object.keys(sleepingAgentSessionRecords) + const replacedSleepingRecords: Record = + {} + for (const key of sleepingRecordKeys) { + const existing = state.sleepingAgentSessionsByPaneKey[key] + if (existing) { + replacedSleepingRecords[key] = existing + } + } + const rollbackTargetShutdownState = (): void => { + set((s) => { + const next = { ...s.suppressedPtyExitIds } + for (const ptyId of exitGuardPtyIds) { + delete next[ptyId] } + return { + suppressedPtyExitIds: next, + sleepingAgentSessionsByPaneKey: replaceRecordEntries( + s.sleepingAgentSessionsByPaneKey, + sleepingRecordKeys.map((key) => [key, replacedSleepingRecords[key] ?? null] as const) + ) + } + }) + } + set((s) => ({ + suppressedPtyExitIds: { + ...s.suppressedPtyExitIds, + ...Object.fromEntries(exitGuardPtyIds.map((ptyId) => [ptyId, true] as const)) + }, + sleepingAgentSessionsByPaneKey: replaceRecordEntries( + s.sleepingAgentSessionsByPaneKey, + Object.entries(sleepingAgentSessionRecords) ) - const retainedCompletionEvidence = collectHibernatedCompletionEvidenceForWorktree( - state, - worktreeId, - paneKeys - ) - if (!sleepingAgentSessionRecords[opts.paneKey]) { - // Why: killing the PTY with no persisted resume record strands the pane unwakeable; abort instead of hibernating unrecoverably. - throw new Error('agent_hibernation_capture_missing') + })) + if (expectedRuntimePtyIds.length > 0) { + if (!runtimeEnvironmentId) { + rollbackTargetShutdownState() + throw new Error('missing_runtime_for_exact_terminal_stop') } - const capture = shutdownBufferCaptures.get(opts.tabId) - if (capture) { - try { - capture({ includeLocalBuffers: false }) - } catch { - // Don't let one tab's capture failure block the pane hibernation. - } + let stopResult: { + stoppedPtyIds?: string[] + livePtyIds?: string[] + postStopVerified?: boolean + postStopFailure?: string } - // Why: store sleeping records before kill, since pty:exit can arrive first. - const sleepingRecordKeys = Object.keys(sleepingAgentSessionRecords) - const replacedSleepingRecords: Record< - string, - (typeof sleepingAgentSessionRecords)[string] - > = {} - for (const key of sleepingRecordKeys) { - const existing = state.sleepingAgentSessionsByPaneKey[key] - if (existing) { - replacedSleepingRecords[key] = existing - } - } - const rollbackTargetShutdownState = (): void => { - set((s) => { - const next = { ...s.suppressedPtyExitIds } - for (const ptyId of exitGuardPtyIds) { - delete next[ptyId] - } - return { - suppressedPtyExitIds: next, - sleepingAgentSessionsByPaneKey: replaceRecordEntries( - s.sleepingAgentSessionsByPaneKey, - sleepingRecordKeys.map( - (key) => [key, replacedSleepingRecords[key] ?? null] as const - ) - ) - } - }) - } - set((s) => ({ - suppressedPtyExitIds: { - ...s.suppressedPtyExitIds, - ...Object.fromEntries(exitGuardPtyIds.map((ptyId) => [ptyId, true] as const)) - }, - sleepingAgentSessionsByPaneKey: replaceRecordEntries( - s.sleepingAgentSessionsByPaneKey, - Object.entries(sleepingAgentSessionRecords) - ) - })) - if (expectedRuntimePtyIds.length > 0) { - if (!runtimeEnvironmentId) { - rollbackTargetShutdownState() - throw new Error('missing_runtime_for_exact_terminal_stop') - } - let stopResult: { + try { + stopResult = await callRuntimeRpc<{ stoppedPtyIds?: string[] livePtyIds?: string[] postStopVerified?: boolean postStopFailure?: string - } - try { - stopResult = await callRuntimeRpc<{ - stoppedPtyIds?: string[] - livePtyIds?: string[] - postStopVerified?: boolean - postStopFailure?: string - }>( - { kind: 'environment', environmentId: runtimeEnvironmentId }, - 'terminal.stopExact', - { - worktree: toRuntimeWorktreeSelector(worktreeId), - expectedPtyIds: expectedRuntimePtyIds, - keepHistory: true, - targetOnly: true - }, - { timeoutMs: 15000 } - ) - } catch (err) { - rollbackTargetShutdownState() - throw err - } - const stoppedPtyIds = sortedUniquePtyIds(stopResult.stoppedPtyIds) - const livePtyIds = sortedUniquePtyIds(stopResult.livePtyIds) - const targetWasLive = expectedRuntimePtyIds.every((ptyId) => livePtyIds.includes(ptyId)) - if (!equalStringSets(stoppedPtyIds, expectedRuntimePtyIds) || !targetWasLive) { - rollbackTargetShutdownState() - throw new Error('exact_terminal_stop_mismatch') - } - if (stopResult.postStopVerified !== true) { - rollbackTargetShutdownState() - throw new Error(stopResult.postStopFailure ?? 'exact_terminal_stop_unverified') - } - for (const snapshot of unregisterPtyDataHandlers(rendererShutdownPtyIds) ?? []) { - snapshot.commit?.() - } - } else if (!isRemoteRuntimePtyId(opts.ptyId)) { - // Why: pty.kill can flush final data before exit; unregister first so stale handlers can't fire phantom notifications during hibernation. - const handlerSnapshots = unregisterPtyDataHandlers(rendererShutdownPtyIds) ?? [] - try { - await window.api.pty.kill(opts.ptyId, { keepHistory: true }) - } catch (err) { - restorePtyDataHandlersAfterFailedShutdown(handlerSnapshots) - rollbackTargetShutdownState() - throw err - } - for (const snapshot of handlerSnapshots) { - snapshot.commit?.() - } + }>( + { kind: 'environment', environmentId: runtimeEnvironmentId }, + 'terminal.stopExact', + { + worktree: toRuntimeWorktreeSelector(worktreeId), + expectedPtyIds: expectedRuntimePtyIds, + keepHistory: true, + targetOnly: true + }, + { timeoutMs: 15000 } + ) + } catch (err) { + rollbackTargetShutdownState() + throw err } - // Switch point: main records the sleep this window decided, once the stop settled. - if (ORCA_LAYOUT_OWNER_IS_MAIN) { - recordPaneSleepThroughLayoutOwner(get(), worktreeId, sleepingAgentSessionRecords) + const stoppedPtyIds = sortedUniquePtyIds(stopResult.stoppedPtyIds) + const livePtyIds = sortedUniquePtyIds(stopResult.livePtyIds) + const targetWasLive = expectedRuntimePtyIds.every((ptyId) => livePtyIds.includes(ptyId)) + if (!equalStringSets(stoppedPtyIds, expectedRuntimePtyIds) || !targetWasLive) { + rollbackTargetShutdownState() + throw new Error('exact_terminal_stop_mismatch') } - settleAttentionRemovedBy(set, get, () => - set((s) => { - const existingPtyIds = s.ptyIdsByTabId[opts.tabId] ?? [] - const shutdownPtyIdSet = new Set(rendererShutdownPtyIds) - const remainingPtyIds = existingPtyIds.filter((ptyId) => !shutdownPtyIdSet.has(ptyId)) - const tabs = s.tabsByWorktree[worktreeId] ?? [] - const replacements: [string, typeof tabs][] = [] - const tabIndex = tabs.findIndex((candidate) => candidate.id === opts.tabId) - if (tabIndex !== -1) { - const nextTabs = [...tabs] - nextTabs[tabIndex] = { - ...nextTabs[tabIndex], - ptyId: remainingPtyIds.at(-1) ?? null - } - replacements.push([worktreeId, nextTabs]) + if (stopResult.postStopVerified !== true) { + rollbackTargetShutdownState() + throw new Error(stopResult.postStopFailure ?? 'exact_terminal_stop_unverified') + } + for (const snapshot of unregisterPtyDataHandlers(rendererShutdownPtyIds) ?? []) { + snapshot.commit?.() + } + } else if (!isRemoteRuntimePtyId(opts.ptyId)) { + // Why: pty.kill can flush final data before exit; unregister first so stale handlers can't fire phantom notifications during hibernation. + const handlerSnapshots = unregisterPtyDataHandlers(rendererShutdownPtyIds) ?? [] + try { + await window.api.pty.kill(opts.ptyId, { keepHistory: true }) + } catch (err) { + restorePtyDataHandlersAfterFailedShutdown(handlerSnapshots) + rollbackTargetShutdownState() + throw err + } + for (const snapshot of handlerSnapshots) { + snapshot.commit?.() + } + } + // Switch point: main records the sleep this window decided, once the stop settled. + if (ORCA_LAYOUT_OWNER_IS_MAIN) { + recordPaneSleepThroughLayoutOwner(get(), worktreeId, sleepingAgentSessionRecords) + } + settleAttentionRemovedBy(set, get, () => + set((s) => { + const existingPtyIds = s.ptyIdsByTabId[opts.tabId] ?? [] + const shutdownPtyIdSet = new Set(rendererShutdownPtyIds) + const remainingPtyIds = existingPtyIds.filter((ptyId) => !shutdownPtyIdSet.has(ptyId)) + const tabs = s.tabsByWorktree[worktreeId] ?? [] + const replacements: [string, typeof tabs][] = [] + const tabIndex = tabs.findIndex((candidate) => candidate.id === opts.tabId) + if (tabIndex !== -1) { + const nextTabs = [...tabs] + nextTabs[tabIndex] = { + ...nextTabs[tabIndex], + ptyId: remainingPtyIds.at(-1) ?? null } - const nextCodexRestartNoticeByPtyId = { ...s.codexRestartNoticeByPtyId } - for (const ptyId of exitGuardPtyIds) { - delete nextCodexRestartNoticeByPtyId[ptyId] - } - const nextLastKnownRelay = - remainingPtyIds.length === 0 - ? replaceRecordEntries(s.lastKnownRelayPtyIdByTabId, [[opts.tabId, null]]) - : s.lastKnownRelayPtyIdByTabId - let nextRuntimePaneTitlesByTabId = s.runtimePaneTitlesByTabId - const numericPaneId = Number(opts.leafId) - if ( - Number.isInteger(numericPaneId) && - s.runtimePaneTitlesByTabId[opts.tabId]?.[numericPaneId] - ) { - const nextByPane = { ...s.runtimePaneTitlesByTabId[opts.tabId] } - delete nextByPane[numericPaneId] - nextRuntimePaneTitlesByTabId = replaceRecordEntries(s.runtimePaneTitlesByTabId, [ - [opts.tabId, Object.keys(nextByPane).length > 0 ? nextByPane : null] - ]) - } - const nextLastTerminalInputAtByPaneKey = { ...s.lastTerminalInputAtByPaneKey } - delete nextLastTerminalInputAtByPaneKey[opts.paneKey] - return { - tabsByWorktree: replaceTabBuckets(s.tabsByWorktree, replacements), - ptyIdsByTabId: replaceRecordEntries(s.ptyIdsByTabId, [[opts.tabId, remainingPtyIds]]), - lastKnownRelayPtyIdByTabId: nextLastKnownRelay, - suppressedPtyExitIds: { - ...s.suppressedPtyExitIds, - ...Object.fromEntries(exitGuardPtyIds.map((ptyId) => [ptyId, true] as const)) - }, - codexRestartNoticeByPtyId: nextCodexRestartNoticeByPtyId, - ...(nextRuntimePaneTitlesByTabId !== s.runtimePaneTitlesByTabId - ? { - runtimePaneTitlesByTabId: nextRuntimePaneTitlesByTabId - } - : {}), - ...removePanesAttention(s, new Set([opts.paneKey])), - lastTerminalInputAtByPaneKey: nextLastTerminalInputAtByPaneKey - } - }) - ) - get().dropHibernatedAgentStatusPane(worktreeId, opts.paneKey, { - retainedCompletionEvidence + replacements.push([worktreeId, nextTabs]) + } + const nextCodexRestartNoticeByPtyId = { ...s.codexRestartNoticeByPtyId } + for (const ptyId of exitGuardPtyIds) { + delete nextCodexRestartNoticeByPtyId[ptyId] + } + const nextLastKnownRelay = + remainingPtyIds.length === 0 + ? replaceRecordEntries(s.lastKnownRelayPtyIdByTabId, [[opts.tabId, null]]) + : s.lastKnownRelayPtyIdByTabId + let nextRuntimePaneTitlesByTabId = s.runtimePaneTitlesByTabId + const numericPaneId = Number(opts.leafId) + if ( + Number.isInteger(numericPaneId) && + s.runtimePaneTitlesByTabId[opts.tabId]?.[numericPaneId] + ) { + const nextByPane = { ...s.runtimePaneTitlesByTabId[opts.tabId] } + delete nextByPane[numericPaneId] + nextRuntimePaneTitlesByTabId = replaceRecordEntries(s.runtimePaneTitlesByTabId, [ + [opts.tabId, Object.keys(nextByPane).length > 0 ? nextByPane : null] + ]) + } + const nextLastTerminalInputAtByPaneKey = { ...s.lastTerminalInputAtByPaneKey } + delete nextLastTerminalInputAtByPaneKey[opts.paneKey] + return { + tabsByWorktree: replaceTabBuckets(s.tabsByWorktree, replacements), + ptyIdsByTabId: replaceRecordEntries(s.ptyIdsByTabId, [[opts.tabId, remainingPtyIds]]), + lastKnownRelayPtyIdByTabId: nextLastKnownRelay, + suppressedPtyExitIds: { + ...s.suppressedPtyExitIds, + ...Object.fromEntries(exitGuardPtyIds.map((ptyId) => [ptyId, true] as const)) + }, + codexRestartNoticeByPtyId: nextCodexRestartNoticeByPtyId, + ...(nextRuntimePaneTitlesByTabId !== s.runtimePaneTitlesByTabId + ? { + runtimePaneTitlesByTabId: nextRuntimePaneTitlesByTabId + } + : {}), + ...removePanesAttention(s, new Set([opts.paneKey])), + lastTerminalInputAtByPaneKey: nextLastTerminalInputAtByPaneKey + } }) + ) + get().dropHibernatedAgentStatusPane(worktreeId, opts.paneKey, { + retainedCompletionEvidence }) + } } } diff --git a/src/shared/agent-session/lifecycle/mail-wake-obligation.ts b/src/shared/agent-session/lifecycle/mail-wake-obligation.ts index 5c2e5dbf81f..af09e2c0af0 100644 --- a/src/shared/agent-session/lifecycle/mail-wake-obligation.ts +++ b/src/shared/agent-session/lifecycle/mail-wake-obligation.ts @@ -52,8 +52,18 @@ export function mailWakeEligibility(records: readonly SleepingAgentSessionRecord .map((record) => [ record.paneKey, record.mailWake?.capturedAt ?? record.capturedAt, - mailWakeObligation(record) === 'manual' + mailWakeObligation(record) ]) .sort() ) } + +export function undispatchedMailWakeRecord( + record: SleepingAgentSessionRecord +): SleepingAgentSessionRecord | null { + if (mailWakeObligation(record) !== 'retained' || !record.mailWake || record.mailWake.attempt) { + return null + } + const { restoreOnTabOpenOnly: _tabOnly, ...resume } = withoutMailWake(record) + return { ...resume, origin: 'worktree-sleep', capturedAt: record.mailWake.capturedAt } +}