fix(orchestration): retain mail wake custody through failed launches

This commit is contained in:
Brennan Benson
2026-11-12 18:56:50 -08:00
parent ceba1839aa
commit 5384448ece
24 changed files with 1223 additions and 478 deletions
@@ -1,3 +1,4 @@
import { undeliveredInboxQuery, type UndeliveredInboxOptions } from './undelivered-inbox-query'
import type { MessageType, MessageRow } from '../orchestration-records'
import { exposeMessageTimestamps, exposeMessageListTimestamps } from '../utc-timestamp'
import { addLifecycleRejectionMarker } from '../lifecycle-rejection-marker'
@@ -88,51 +89,29 @@ export function getUndeliveredUnreadMessages(
this: OrchestrationDb,
toHandle: string,
types?: MessageType[],
options?: {
excludeTypes?: readonly string[]
excludeIds?: readonly string[]
excludeRunOwnedDirect?: boolean
newestFirst?: boolean
limit?: number
}
options?: UndeliveredInboxOptions
): MessageRow[] {
const conditions = [
'to_handle = ?',
'read = 0',
'delivered_at IS NULL',
'pointer_enter_pending = 0',
"delivery_contract = 'current_delivery'",
...(options?.excludeRunOwnedDirect ? [`NOT ${runOwnsDirectMessageSql('messages')}`] : [])
]
const params: (string | number)[] = [toHandle]
if (types?.length) {
conditions.push(`type IN (${types.map(() => '?').join(',')})`)
params.push(...types)
}
if (options?.excludeTypes?.length) {
conditions.push(`type NOT IN (${options.excludeTypes.map(() => '?').join(',')})`)
params.push(...options.excludeTypes)
}
if (options?.excludeIds?.length) {
conditions.push(`id NOT IN (${options.excludeIds.map(() => '?').join(',')})`)
params.push(...options.excludeIds)
}
const limitSql = options?.limit === undefined ? '' : ' LIMIT ?'
if (options?.limit !== undefined) {
params.push(Math.max(1, Math.floor(options.limit)))
}
const query = undeliveredInboxQuery(toHandle, types, options)
return exposeMessageListTimestamps(
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: this SELECT returns only messages table rows, decoded at this database boundary.
this.db
.prepare(
`SELECT * FROM messages
WHERE ${conditions.join(' AND ')}
ORDER BY sequence${options?.newestFirst ? ' DESC' : ''}${limitSql}`
)
.all(...params) as MessageRow[]
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the shared predicate changes selection only; SELECT * still returns the messages schema.
this.db.prepare(`SELECT * ${query.sql}`).all(...query.params) as MessageRow[]
)
}
export function getUndeliveredUnreadSequence(
this: OrchestrationDb,
toHandle: string,
options: UndeliveredInboxOptions
): number | undefined {
const query = undeliveredInboxQuery(toHandle, undefined, {
...options,
newestFirst: true,
limit: 1
})
const row = this.db.prepare(`SELECT sequence ${query.sql}`).get(...query.params)
return typeof row?.sequence === 'number' ? row.sequence : undefined
}
export function getUndeliveredUnreadMailboxHandles(this: OrchestrationDb): string[] {
return (
this.db
@@ -289,6 +268,7 @@ export function getThreadMessagesFor(
export type MessageInboxMethods = {
getUnreadMessages: typeof getUnreadMessages
convertLifecycleMessageToRejection: typeof convertLifecycleMessageToRejection
getUndeliveredUnreadSequence: typeof getUndeliveredUnreadSequence
getUndeliveredUnreadMessages: typeof getUndeliveredUnreadMessages
getUndeliveredUnreadMailboxHandles: typeof getUndeliveredUnreadMailboxHandles
getAllMessages: typeof getAllMessages
@@ -307,6 +287,7 @@ export function attachMessageInbox(ctor: { prototype: object }): void {
Object.assign(ctor.prototype, {
getUnreadMessages,
convertLifecycleMessageToRejection,
getUndeliveredUnreadSequence,
getUndeliveredUnreadMessages,
getUndeliveredUnreadMailboxHandles,
getAllMessages,
@@ -0,0 +1,46 @@
import type { MessageType } from '../orchestration-records'
import { runOwnsDirectMessageSql } from '../runs/run-coordinator-mail-routing'
export type UndeliveredInboxOptions = {
excludeTypes?: readonly string[]
excludeIds?: readonly string[]
excludeRunOwnedDirect?: boolean
newestFirst?: boolean
limit?: number
}
export function undeliveredInboxQuery(
toHandle: string,
types?: MessageType[],
options?: UndeliveredInboxOptions
) {
const conditions = [
'to_handle = ?',
'read = 0',
'delivered_at IS NULL',
'pointer_enter_pending = 0',
"delivery_contract = 'current_delivery'",
...(options?.excludeRunOwnedDirect ? [`NOT ${runOwnsDirectMessageSql('messages')}`] : [])
]
const params: (string | number)[] = [toHandle]
if (types?.length) {
conditions.push(`type IN (${types.map(() => '?').join(',')})`)
params.push(...types)
}
if (options?.excludeTypes?.length) {
conditions.push(`type NOT IN (${options.excludeTypes.map(() => '?').join(',')})`)
params.push(...options.excludeTypes)
}
if (options?.excludeIds?.length) {
conditions.push(`id NOT IN (${options.excludeIds.map(() => '?').join(',')})`)
params.push(...options.excludeIds)
}
const limitSql = options?.limit === undefined ? '' : ' LIMIT ?'
if (options?.limit !== undefined) {
params.push(Math.max(1, Math.floor(options.limit)))
}
return {
sql: `FROM messages WHERE ${conditions.join(' AND ')} ORDER BY sequence${options?.newestFirst ? ' DESC' : ''}${limitSql}`,
params
}
}
@@ -8,6 +8,8 @@ export type MailboxDeliveryHold =
openTab: boolean
failed?: boolean
liveShell?: boolean
waking?: boolean
resuming?: boolean
workspaceName?: string
}
| { kind: 'not-foreground'; agentName: string }
@@ -78,6 +80,9 @@ export async function readMailboxDeliveryHold(
export function describeMailboxDeliveryHold(recipient: string, hold: MailboxDeliveryHold): string {
switch (hold.kind) {
case 'asleep':
if (hold.waking || hold.resuming) {
return `Queued. Orca is ${hold.waking ? 'waking' : 'resuming'} ${hold.agentName}; the message will be delivered when it's back.`
}
if (hold.liveShell) {
return `Queued. Re-run ${hold.agentName} in that terminal; the message is waiting.`
}
@@ -38,15 +38,7 @@ export async function admitsMailWakeRetry(
return false
}
const inspection = await ports.inspect(attempt.ptyId, attempt.incarnationId)
const observation = inspection?.foregroundObservation
if (
inspection?.childProcessEvidence !== 'no-children' ||
inspection.hasChildProcesses ||
observation?.verdict !== 'observed' ||
observation.ptyId !== attempt.ptyId ||
observation.ptyIncarnationId !== attempt.incarnationId ||
observation.foregroundKind !== 'shell'
) {
if (inspection?.childProcessEvidence !== 'no-children' || inspection.hasChildProcesses) {
return false
}
return (await ports.shellInFront(attempt.ptyId)) && stillOwns()
@@ -1,5 +1,8 @@
import type { SleepingAgentSessionRecord } from '../../../../../shared/agent-session/lifecycle/agent-session-resume'
import type { SleptMailboxWake } from './slept-mailbox-wake'
import { mailWakeObligation } from '../../../../../shared/agent-session/lifecycle/mail-wake-obligation'
import { TUI_AGENT_DISPLAY_NAMES } from '../../../../../shared/agent-launch/catalog/tui-agent-display-names'
import type { MailboxDeliveryHold } from '../delivery-hold/mailbox-delivery-hold'
import type { SleepingAgentSessionRecord } from '../../../../../shared/agent-session/lifecycle/agent-session-resume'
import { OrcaRuntimeWithPruneMobileSessionTabGroupLayout } from '../../../session-lifecycle/projection/prune-mobile-session-tab-group-layout'
import type {
RestoredPane,
@@ -13,10 +16,8 @@ export abstract class RuntimeMailWakeAdmission extends OrcaRuntimeWithPruneMobil
protected abstract readonly mailWakeInvocations: MailWakeInvocations
protected abstract hasSleepingMailForPane(paneKey: string): boolean
protected abstract sleepingMailRecord(paneKey: string): SleepingAgentSessionRecord | null
protected abstract editMailWakeRecord(
record: SleepingAgentSessionRecord,
next: SleepingAgentSessionRecord
): Promise<boolean>
protected abstract readonly sleptMailboxWake: SleptMailboxWake
protected abstract watchSleepingMail(): void
protected abstract getPtyRecordForPaneKey(paneKey: string): RuntimePtyWorktreeRecord | null
protected async admitFailedMailWake(
pane: RestoredPane,
@@ -48,7 +49,7 @@ export abstract class RuntimeMailWakeAdmission extends OrcaRuntimeWithPruneMobil
this.terminalRunFacts.readProcessStartedAt(ptyId, incarnationId) !== null
)
}
const safe = await admitsMailWakeRetry(record, {
await admitsMailWakeRetry(record, {
current: () =>
this.hasSleepingMailForPane(record.paneKey)
? this.sleepingMailRecord(record.paneKey)
@@ -65,28 +66,43 @@ export abstract class RuntimeMailWakeAdmission extends OrcaRuntimeWithPruneMobil
shellInFront: async (ptyId) =>
(await this.readLaunchedAgentForeground(ptyId, record.agent)) === 'shell'
})
if (!safe || !owns(attempt.ptyId, attempt.incarnationId) || !this.ptyController?.stopAndWait) {
return false
// Host-side input fencing is required before a proven failed invocation may be stopped.
return false
}
async readSleepingMailHold(mailbox: string): Promise<MailboxDeliveryHold | null> {
this.watchSleepingMail()
const record = this.sleptMailboxWake.batch(mailbox)?.record
if (!record || !mailWakeObligation(record)) {
return null
}
if (!(await this.ptyController.stopAndWait(attempt.ptyId, { keepHistory: true }))) {
return false
const attempt = record.mailWake?.attempt
const pty = this.getPtyRecordForPaneKey(record.paneKey)
const liveShell =
!!attempt &&
pty?.connected === true &&
pty.ptyId === attempt.ptyId &&
pty.incarnationId === attempt.incarnationId &&
pty.launchToken === attempt.launchToken &&
pty.launchIncarnationId === attempt.incarnationId &&
pty.worktreeId === record.worktreeId &&
pty.connectionId === (record.connectionId ?? null) &&
this.mailWakeInvocations.ended(record) &&
(await this.readLaunchedAgentForeground(attempt.ptyId, record.agent)) === 'shell' &&
this.getPtyRecordForPaneKey(record.paneKey) === pty &&
this.sleepingMailRecord(record.paneKey) === record
return {
kind: 'asleep',
waking:
mailWakeObligation(record) === 'automatic' && !this.sleptMailboxWake.failedToWake(record),
resuming: mailWakeObligation(record) === 'retained' && !liveShell,
agentName: TUI_AGENT_DISPLAY_NAMES[record.agent],
openTab: record.restoreOnTabOpenOnly === true,
failed: this.sleptMailboxWake.failedToWake(record),
liveShell,
workspaceName: this.resolvedWorktrees
.peek()
?.worktrees.find((row) => row.id === record.worktreeId)?.displayName
}
const current = this.sleepingMailRecord(record.paneKey)
if (
current?.mailWake?.attempt?.launchToken !== attempt.launchToken ||
mailWakeObligation(current) !== 'retained' ||
!this.hasSleepingMailForPane(record.paneKey) ||
(this.terminalRunFacts.readLastDrivingInputAt(attempt.ptyId) ?? -1) >= attempt.startedAt
) {
return false
}
return this.editMailWakeRecord(current, {
...current,
mailWake: {
id: current.mailWake!.id,
capturedAt: current.mailWake!.capturedAt,
startedAt: Date.now()
}
})
}
}
@@ -0,0 +1,60 @@
import type { SleepingAgentSessionRecord } from '../../../../../shared/agent-session/lifecycle/agent-session-resume'
import {
mailWakeEligibility,
mailWakeObligation
} from '../../../../../shared/agent-session/lifecycle/mail-wake-obligation'
export class SleepingMailRecordIndex {
readonly records = new Map<string, SleepingAgentSessionRecord>()
readonly custody = new Map<string, SleepingAgentSessionRecord>()
readonly changed = new Set<string>()
private readonly hostPanes = new Map<string, Set<string>>()
launchTokens(): ReadonlySet<string> {
return new Set(
[...this.custody.values()].flatMap((record) =>
record.mailWake?.attempt ? [record.mailWake.attempt.launchToken] : []
)
)
}
refresh(
host: string,
records: Readonly<Record<string, SleepingAgentSessionRecord>>,
changed: (pane: string) => void
): boolean {
const previous = this.hostPanes.get(host) ?? new Set<string>()
const next = new Set(Object.keys(records))
let eligibilityChanged = false
for (const pane of previous) {
if (!next.has(pane)) {
eligibilityChanged ||= this.records.has(pane)
this.records.delete(pane)
this.custody.delete(pane)
this.changed.add(pane)
changed(pane)
}
}
for (const record of Object.values(records)) {
const before = this.records.get(record.paneKey)
if (before !== record) {
this.changed.add(record.paneKey)
}
eligibilityChanged ||=
mailWakeEligibility(before ? [before] : []) !== mailWakeEligibility([record])
if (mailWakeObligation(record)) {
this.records.set(record.paneKey, record)
} else {
this.records.delete(record.paneKey)
}
if (record.mailWake) {
this.custody.set(record.paneKey, record)
} else {
this.custody.delete(record.paneKey)
}
changed(record.paneKey)
}
this.hostPanes.set(host, next)
return eligibilityChanged
}
}
@@ -1,20 +1,19 @@
import { replaceMailWakeRecord } from '../../../session-lifecycle/projection/retained-mail-wake-attempt'
import { SleepingMailRecordIndex } from './sleeping-mail-record-index'
import {
mailWakeObligation,
mailWakeEligibility,
withoutMailWake
} from '../../../../../shared/agent-session/lifecycle/mail-wake-obligation'
import { queueTerminalWorkspaceCommand } from '../../../workspaces/lifecycle/workspace-lifecycle-commands'
import { MailWakeInvocations } from './mail-wake-invocations'
import { parsePaneKey } from '../../../../../shared/workspace-layout/stable-pane-id'
import { RuntimeMailWakeAdmission } from './runtime-mail-wake-admission'
import { TUI_AGENT_DISPLAY_NAMES } from '../../../../../shared/agent-launch/catalog/tui-agent-display-names'
import { SleptMailboxWake } from './slept-mailbox-wake'
import type { SleepingAgentSessionRecord } from '../../../../../shared/agent-session/lifecycle/agent-session-resume'
import type { MailboxDeliveryHold } from '../delivery-hold/mailbox-delivery-hold'
export abstract class RuntimeSleptMailboxWakes extends RuntimeMailWakeAdmission {
protected readonly mailWakeInvocations = new MailWakeInvocations()
private readonly sleptMailboxWake = new SleptMailboxWake({
protected readonly sleptMailboxWake = new SleptMailboxWake({
db: () => this._orchestrationDb,
record: (paneKey) => this.sleepingMailRecord(paneKey),
waiters: (mailbox) => this.messageWaiters.get(mailbox),
@@ -87,15 +86,28 @@ export abstract class RuntimeSleptMailboxWakes extends RuntimeMailWakeAdmission
})
private watchingSleepingMail = false
private readonly sleepingRecordIndex = new SleepingMailRecordIndex()
private readonly sleepingRecords = this.sleepingRecordIndex.records
private readonly custodyRecords = this.sleepingRecordIndex.custody
private readonly changedSleepingPanes = this.sleepingRecordIndex.changed
private watchSleepingMail(): void {
private refreshSleepingRecords(host: string): boolean {
const changed = this.sleepingRecordIndex.refresh(
host,
this.store?.getWorkspaceSession?.(host)?.sleepingAgentSessionsByPaneKey ?? {},
(pane) => this.sleptMailboxWake.refreshPane(pane)
)
this.mailWakeInvocations.retain(this.sleepingRecordIndex.launchTokens())
return changed
}
protected watchSleepingMail(): void {
if (this.watchingSleepingMail) {
return
}
this.watchingSleepingMail = true
const records = new Map<string, string>()
for (const host of this.store?.getWorkspaceSessionHostIds?.() ?? ['local']) {
records.set(host, this.sleepingMailEligibility(host))
this.refreshSleepingRecords(host)
}
this.onClientEvent(
(event) => {
@@ -105,72 +117,61 @@ export abstract class RuntimeSleptMailboxWakes extends RuntimeMailWakeAdmission
},
{ consumesTerminalSideEffects: false }
)
this.subscribeAgentStatusRowChanges(() => this.watchMailWakeInvocations())
this.subscribeAgentStatusRowChanges((paneKey) => this.watchMailWakeInvocations(paneKey))
this.crewChanges.subscribe((facets) => {
if (facets.has('traffic') || facets.has('topology') || facets.has('records')) {
this.publishSleepingMail()
this.publishSleepingMail(
facets.has('topology') || facets.has('records') ? undefined : facets.trafficAddresses
)
}
})
this.store?.onWorkspaceSessionWritten?.((hostId) => {
const hosts =
hostId === null ? (this.store?.getWorkspaceSessionHostIds?.() ?? ['local']) : [hostId]
for (const host of hosts) {
const next = this.sleepingMailEligibility(host)
const changed = this.refreshSleepingRecords(host)
this.watchMailWakeInvocations()
if (records.get(host) !== next) {
records.set(host, next)
if (changed) {
queueMicrotask(() => this.retrySleptMailboxWakes())
} else {
this.publishSleepingMail([])
}
}
})
}
private sleepingMailEligibility(host: string): string {
return mailWakeEligibility(
Object.values(this.store?.getWorkspaceSession?.(host)?.sleepingAgentSessionsByPaneKey ?? {})
)
}
private allMailWakeRecords(): SleepingAgentSessionRecord[] {
return (this.store?.getWorkspaceSessionHostIds?.() ?? ['local']).flatMap((host) =>
Object.values(this.store?.getWorkspaceSession?.(host)?.sleepingAgentSessionsByPaneKey ?? {})
)
}
private watchMailWakeInvocations(): void {
const records = this.allMailWakeRecords()
const tokens = new Set<string>()
for (const record of records) {
const attempt = record.mailWake?.attempt
private watchMailWakeInvocations(paneKey?: string): void {
const record = paneKey ? this.custodyRecords.get(paneKey) : undefined
for (const current of paneKey ? (record ? [record] : []) : this.custodyRecords.values()) {
const attempt = current.mailWake?.attempt
if (!attempt) {
continue
}
tokens.add(attempt.launchToken)
if (
this.terminalRunFacts.readProcessStartedAt(attempt.ptyId, attempt.incarnationId) !== null
) {
this.mailWakeInvocations.watch(record, this)
this.mailWakeInvocations.watch(current, this)
}
const pty = this.getPtyRecordForPaneKey(record.paneKey)
const pty = this.getPtyRecordForPaneKey(current.paneKey)
if (
pty?.connected === true &&
pty.worktreeId === record.worktreeId &&
pty.connectionId === (record.connectionId ?? null) &&
pty.worktreeId === current.worktreeId &&
pty.connectionId === (current.connectionId ?? null) &&
pty.incarnationId === attempt.incarnationId &&
pty.launchToken === attempt.launchToken &&
pty.launchIncarnationId === attempt.incarnationId &&
this.mailWakeInvocations.observe(
record,
this.getAgentProviderSessionRowsForPaneFn?.(record.paneKey) ?? [],
current,
this.getAgentProviderSessionRowsForPaneFn?.(current.paneKey) ?? [],
pty.ptyId
)
) {
void this.editMailWakeRecord(record, withoutMailWake(record)).catch((error: unknown) =>
this.sleptMailboxWake.confirmed(current)
void this.editMailWakeRecord(current, withoutMailWake(current)).catch((error: unknown) =>
console.warn('[orchestration] could not settle resumed mail custody', error)
)
}
}
this.mailWakeInvocations.retain(tokens)
}
protected async editMailWakeRecord(
@@ -181,42 +182,20 @@ export abstract class RuntimeSleptMailboxWakes extends RuntimeMailWakeAdmission
await this.commitWorkspaceSessionEdit(
this.getWorkspaceSessionHostIdForWorktree(record.worktreeId),
(session) => {
const current = session.sleepingAgentSessionsByPaneKey?.[record.paneKey]
if (
current?.mailWake?.id !== record.mailWake?.id ||
current?.capturedAt !== record.capturedAt ||
current?.origin !== record.origin ||
current?.restoreOnTabOpenOnly !== record.restoreOnTabOpenOnly ||
current?.mailWake?.attempt?.launchToken !== record.mailWake?.attempt?.launchToken
) {
return null
}
changed = true
return {
...session,
sleepingAgentSessionsByPaneKey: {
...session.sleepingAgentSessionsByPaneKey,
[record.paneKey]: next
}
}
const edited = replaceMailWakeRecord(session, record, next)
changed = edited !== null
return edited
}
)
if (changed && record.mailWake && !next.mailWake) {
this.settledMailWakeIds.add(record.mailWake.id)
this.publishSleepingMail()
this.publishSleepingMail([])
}
return changed
}
protected sleepingMailRecord(paneKey: string): SleepingAgentSessionRecord | null {
for (const host of this.store?.getWorkspaceSessionHostIds?.() ?? (['local'] as const)) {
const record =
this.store?.getWorkspaceSession?.(host)?.sleepingAgentSessionsByPaneKey?.[paneKey]
if (record) {
return record
}
}
return null
return this.sleepingRecords.get(paneKey) ?? null
}
protected hasSleepingMailForPane(paneKey: string): boolean {
@@ -224,23 +203,32 @@ export abstract class RuntimeSleptMailboxWakes extends RuntimeMailWakeAdmission
}
protected requestSleptMailboxWake(mailbox: string, reservedTypes?: ReadonlySet<string>): void {
this.watchSleepingMail()
this.sleptMailboxWake.request(mailbox, reservedTypes)
}
retrySleptMailboxWakes(): void {
this.watchSleepingMail()
this.sleepingMailPublication = ''
this.publishSleepingMail()
this.getExistingOrchestrationDb()
this.scheduleRestoredMessageRepoints()
this.publishSleepingMail()
}
private readonly settledMailWakeIds = new Set<string>()
private sleepingMailPublicationDirty = false
private globalSleepingMailInvalidation = false
private readonly sleepingMailAddresses = new Set<string>()
private sleepingMailPublicationQueued = false
private sleepingMailPublication = ''
protected publishSleepingMail(): void {
protected publishSleepingMail(mailboxes?: Iterable<string>): void {
if (mailboxes) {
for (const mailbox of mailboxes) {
this.sleepingMailAddresses.add(mailbox)
}
} else {
this.globalSleepingMailInvalidation = true
}
if (this.sleepingMailPublicationQueued) {
this.sleepingMailPublicationDirty = true
return
@@ -251,35 +239,43 @@ export abstract class RuntimeSleptMailboxWakes extends RuntimeMailWakeAdmission
if (!this._orchestrationDb) {
return
}
const panes = new Map<string, string>()
for (const mailbox of this._orchestrationDb?.getUndeliveredUnreadMailboxHandles() ?? []) {
const batch = this.sleptMailboxWake.batch(mailbox)
if (batch) {
panes.set(batch.record.paneKey, batch.record.worktreeId)
const global = this.globalSleepingMailInvalidation
const affected = new Set(this.changedSleepingPanes)
this.changedSleepingPanes.clear()
if (global && (this.sleepingRecords.size || this.sleptMailboxWake.hasTrackedMail)) {
this.sleptMailboxWake.reconcile()
for (const pane of this.custodyRecords.keys()) {
affected.add(pane)
}
} else {
for (const mailbox of this.sleepingMailAddresses) {
for (const pane of this.sleptMailboxWake.refresh(mailbox)) {
affected.add(pane)
}
}
}
for (const record of this.allMailWakeRecords()) {
if (record.mailWake && !panes.has(record.paneKey)) {
this.globalSleepingMailInvalidation = false
this.sleepingMailAddresses.clear()
for (const pane of affected) {
const record = this.custodyRecords.get(pane)
if (record && !this.sleptMailboxWake.owesPane(pane)) {
await this.editMailWakeRecord(record, withoutMailWake(record))
}
}
if (global) {
for (const pane of this.sleepingRecords.keys()) {
for (const mailbox of this.sleptMailboxWake.mailboxesForPane(pane)) {
this.sleptMailboxWake.request(mailbox)
}
}
}
const settledIds = [...this.settledMailWakeIds]
this.settledMailWakeIds.clear()
const ids = [...new Set(panes.values())].sort()
const key = JSON.stringify([
ids,
this.allMailWakeRecords()
.filter((record) => record.mailWake)
.map((record) => record.mailWake!.id)
.sort()
])
const ids = this.sleptMailboxWake.heldWorktrees()
const key = JSON.stringify(ids)
if (key !== this.sleepingMailPublication || settledIds.length) {
this.sleepingMailPublication = key
if (settledIds.length) {
this.notifier?.sleepingMailChanged?.(ids, settledIds)
} else {
this.notifier?.sleepingMailChanged?.(ids)
}
this.notifier?.sleepingMailChanged?.(ids, ...(settledIds.length ? [settledIds] : []))
}
} catch {
/* The runtime can close its database before a queued publication. */
@@ -287,26 +283,9 @@ export abstract class RuntimeSleptMailboxWakes extends RuntimeMailWakeAdmission
this.sleepingMailPublicationQueued = false
if (this.sleepingMailPublicationDirty) {
this.sleepingMailPublicationDirty = false
this.publishSleepingMail()
this.publishSleepingMail([])
}
}
})
}
readSleepingMailHold(mailbox: string): MailboxDeliveryHold | null {
const record = this.sleptMailboxWake.batch(mailbox)?.record
if (!record || !mailWakeObligation(record)) {
return null
}
return {
kind: 'asleep',
agentName: TUI_AGENT_DISPLAY_NAMES[record.agent],
openTab: record.restoreOnTabOpenOnly === true,
failed: this.sleptMailboxWake.failedToWake(record),
liveShell: mailWakeObligation(record) === 'retained',
workspaceName: this.resolvedWorktrees
.peek()
?.worktrees.find((row) => row.id === record.worktreeId)?.displayName
}
}
}
@@ -1,8 +1,12 @@
import type { ExecutionHostId } from '../../../../../shared/execution-host'
import type { WorkspaceSessionState } from '../../../../../shared/workspace-session/workspace-session-state-types'
import { OrchestrationMailboxPointerDelivery } from '../terminal-pointer/mailbox-pointer-delivery'
import { MailWakeInvocations } from './mail-wake-invocations'
import { admitsMailWakeRetry } from './mail-wake-retry-admission'
import { retainMailWakeAttempt } from '../../../session-lifecycle/projection/retained-mail-wake-attempt'
import {
retainMailWakeAttempt,
restoreUndispatchedMailWake
} from '../../../session-lifecycle/projection/retained-mail-wake-attempt'
import { RestoredPaneStarter } from '../../../terminal/recovery/restored-pane-start'
import type { HostProcessInspection } from '../../../../../shared/terminal/pty/lifecycle/terminal-process-inspection'
import type { RuntimeTerminalCreate } from '../../../../../shared/runtime-terminal-create-contract'
@@ -185,7 +189,10 @@ it('holds manual Sleep through the sender receipt and keeps its bell until mail
runtime.deliverPendingMessagesForHandle(mailbox)
await settled()
expect(resume).not.toHaveBeenCalled()
expect(runtime.readSleepingMailHold(mailbox)).toMatchObject({ kind: 'asleep', openTab: true })
expect(await runtime.readSleepingMailHold(mailbox)).toMatchObject({
kind: 'asleep',
openTab: true
})
expect(await sendDeliveryHoldWarning(runtime, db, mailbox)).toMatchObject({
code: 'recipient_delivery_held',
recipient: mailbox
@@ -263,6 +270,7 @@ it('retains a failed resume with a live shell, and restarts only a proven invoca
let drivingInput: number | null = null
let occupied = false
let owned = true
let shellForeground = true
const start = vi.fn(async () => ({ id: 'replacement' }))
const starter = new RestoredPaneStarter({
boundPtyId: () => 'pty-shell',
@@ -279,7 +287,7 @@ it('retains a failed resume with a live shell, and restarts only a proven invoca
onInspect()
return inspection
},
shellInFront: async () => true
shellInFront: async () => shellForeground
})
})
const pane = {
@@ -320,18 +328,7 @@ it('retains a failed resume with a live shell, and restarts only a proven invoca
inspection = {
foregroundProcess: 'zsh',
hasChildProcesses: true,
childProcessEvidence: 'children',
foregroundObservation: {
verdict: 'observed',
ptyId: 'pty-shell',
ptyIncarnationId: 'inc-shell',
foregroundKind: 'shell',
source: 'pty-master',
processName: 'zsh',
authorityGeneration: 'host-1',
observationEpoch: 1,
capturedAgeMs: 0
}
childProcessEvidence: 'children'
}
expect(await resumeFromMail()).toBeNull()
inspection = { ...inspection, hasChildProcesses: false, childProcessEvidence: 'no-children' }
@@ -357,6 +354,9 @@ it('retains a failed resume with a live shell, and restarts only a proven invoca
expect(start).not.toHaveBeenCalled()
session.sleepingAgentSessionsByPaneKey[paneKey] = retained
onInspect = () => {}
shellForeground = false
expect(await resumeFromMail()).toBeNull()
shellForeground = true
expect(await resumeFromMail()).toEqual({ id: 'replacement' })
expect(start).toHaveBeenCalledTimes(1)
expect(write).not.toHaveBeenCalled()
@@ -413,3 +413,263 @@ it('retains a failed resume with a live shell, and restarts only a proven invoca
vi.useRealTimers()
}
})
it('suppresses a definitively confirmed late wake only for its obligation and releases consumed mailbox retention', async () => {
const db = new OrchestrationDb(':memory:')
let sleeping: SleepingAgentSessionRecord | null = record
const wake = vi.fn(async () => false)
const delivery = new SleptMailboxWake({
db: () => db,
record: () => sleeping,
waiters: () => undefined,
prepare: async () => {},
wake
})
try {
const run = db.createRun({
objective: 'late confirmation',
coordinatorHandle: 'term_old',
coordinatorPaneKey: paneKey
})
const mailbox = `run:${run.id}`
const message = db.insertMessage({
runId: run.id,
from: 'worker',
to: mailbox,
subject: 'done'
})
delivery.request(mailbox)
await settled()
const retained: SleepingAgentSessionRecord = {
...record,
origin: 'quit',
restoreOnTabOpenOnly: true,
capturedAt: 200,
mailWake: { id: 'obligation', capturedAt: 1, startedAt: 100 }
}
sleeping = retained
delivery.confirmed({ ...retained, mailWake: { ...retained.mailWake!, capturedAt: 2 } })
expect(delivery.failedToWake(retained)).toBe(true)
delivery.confirmed(retained)
delivery.request(mailbox)
await settled()
expect(wake).toHaveBeenCalledTimes(1)
db.markAsRead([message.id])
delivery.refresh(mailbox)
expect(Reflect.get(delivery, 'succeeded').size).toBe(0)
expect(Reflect.get(delivery, 'failed').size).toBe(0)
expect(Reflect.get(delivery, 'attempted').size).toBe(0)
for (let i = 0; i < 20; i++) {
sleeping = record
wake.mockResolvedValueOnce(i % 2 === 0)
const nextRun = db.createRun({
objective: 'retention',
coordinatorHandle: 'term_old',
coordinatorPaneKey: paneKey
})
const handle = `run:${nextRun.id}`
const row = db.insertMessage({
runId: nextRun.id,
from: 'worker',
to: handle,
subject: 'done'
})
delivery.request(handle)
await settled()
db.markAsRead([row.id])
delivery.refresh(handle)
}
expect(Reflect.get(delivery, 'succeeded').size).toBe(0)
expect(Reflect.get(delivery, 'failed').size).toBe(0)
expect(Reflect.get(delivery, 'attempted').size).toBe(0)
expect(delivery.heldWorktrees()).toEqual([])
} finally {
db.close()
}
})
it.each([10, 100])(
'keeps status visits and addressed mail queries bounded with %i saved sleepers',
async (count) => {
const db = new OrchestrationDb(':memory:')
let visits = 0
let statusChanged: (key: string) => void = () => {}
let sessionWritten: (host: ExecutionHostId | null) => void = () => {}
const records: Record<string, SleepingAgentSessionRecord> = {}
const mailboxes: string[] = []
for (let i = 0; i < count; i++) {
const key = `tab-${i}:11111111-1111-4111-8111-111111111111`
records[key] = new Proxy(
{
...record,
paneKey: key,
tabId: `tab-${i}`,
origin: 'quit',
restoreOnTabOpenOnly: true,
mailWake: {
id: `custody-${i}`,
capturedAt: 1,
startedAt: 100,
attempt: {
ptyId: `pty-${i}`,
incarnationId: `inc-${i}`,
launchToken: `token-${i}`,
startedAt: 100
}
}
},
{
get(target, property, receiver) {
visits++
return Reflect.get(target, property, receiver)
}
}
)
const run = db.createRun({
objective: 'scale',
coordinatorHandle: `term_${i}`,
coordinatorPaneKey: key
})
const mailbox = `run:${run.id}`
mailboxes.push(mailbox)
for (let j = 0; j < 5; j++) {
db.insertMessage({ runId: run.id, from: 'worker', to: mailbox, subject: 'done' })
}
}
let session = { ...getDefaultWorkspaceSession(), sleepingAgentSessionsByPaneKey: records }
const readSession = vi.fn(() => session)
const runtime = new OrcaRuntimeService(
partialRuntimeStore({
getWorkspaceSession: readSession,
getWorkspaceSessionHostIds: () => ['local'],
onWorkspaceSessionWritten: (listener) => {
sessionWritten = listener
return () => {}
}
}),
undefined,
{
subscribeAgentStatusRowChanges: (listener) => {
statusChanged = listener
return () => {}
}
}
)
try {
runtime.setOrchestrationDb(db)
runtime.retrySleptMailboxWakes()
await settled()
visits = 0
readSession.mockClear()
statusChanged('unrelated:11111111-1111-4111-8111-111111111111')
expect(visits).toBe(0)
statusChanged('tab-0:11111111-1111-4111-8111-111111111111')
// A status event needs only this pane's attempt and PTY identity; no session inventory.
expect(visits).toBeLessThanOrEqual(4)
expect(readSession).not.toHaveBeenCalled()
const inventory = vi.spyOn(db, 'getUndeliveredUnreadMailboxHandles')
const fullRows = vi.spyOn(db, 'getUndeliveredUnreadMessages')
const newest = vi.spyOn(db, 'getUndeliveredUnreadSequence')
runtime.crewChanges.notify(
Object.assign(new Set(['traffic'] as const), { trafficAddresses: new Set([mailboxes[0]!]) })
)
await settled()
expect(inventory).not.toHaveBeenCalled()
expect(fullRows).not.toHaveBeenCalled()
// Refresh and the custody check each need one scalar newest-sequence row for the affected mailbox.
expect(newest.mock.calls.length).toBeLessThanOrEqual(3)
expect(
newest.mock.results.filter((result) => result.value !== undefined).length
).toBeLessThanOrEqual(3)
expect(newest).toHaveBeenCalled()
session = { ...session, sleepingAgentSessionsByPaneKey: {} }
sessionWritten('local')
await settled()
visits = 0
readSession.mockClear()
statusChanged('tab-0:11111111-1111-4111-8111-111111111111')
expect(visits).toBe(0)
expect(readSession).not.toHaveBeenCalled()
} finally {
db.close()
}
}
)
it('holds unknown dispatch custody but retries a positively undispatched wake on the record availability edge', async () => {
const db = new OrchestrationDb(':memory:')
const retained: SleepingAgentSessionRecord = {
...record,
worktreeId: 'repo::/workspace',
origin: 'quit',
restoreOnTabOpenOnly: true,
mailWake: { id: 'not-dispatched', capturedAt: 1, startedAt: 100 }
}
let session: WorkspaceSessionState = {
...getDefaultWorkspaceSession(),
sleepingAgentSessionsByPaneKey: { [paneKey]: retained }
}
let written: (host: ExecutionHostId | null) => void = () => {}
const runtime = new OrcaRuntimeService(
partialRuntimeStore({
getWorkspaceSession: () => session,
getWorkspaceSessionHostIds: () => ['local'],
onWorkspaceSessionWritten: (listener) => {
written = listener
return () => {}
}
})
)
const resume = vi.fn(
async (_workspace: string, _pane?: string, prepare?: boolean) => prepare === true
)
const notifier: RuntimeNotifier = {
worktreesChanged: vi.fn(),
reposChanged: vi.fn(),
activateWorktree: vi.fn(),
createTerminal: vi.fn(),
splitTerminal: vi.fn(),
renameTerminal: vi.fn(),
focusTerminal: vi.fn(),
closeTerminal: vi.fn(),
terminalFitOverrideChanged: vi.fn(),
terminalDriverChanged: vi.fn(),
resumeSleepingAgents: resume
}
try {
const run = db.createRun({
objective: 'refused launch',
coordinatorHandle: 'term_old',
coordinatorPaneKey: paneKey
})
const mailbox = `run:${run.id}`
db.insertMessage({ runId: run.id, from: 'worker', to: mailbox, subject: 'done' })
runtime.setOrchestrationDb(db)
runtime.setNotifier(notifier)
runtime.syncWindowGraph(1, { tabs: [], leaves: [] })
await settled()
expect(await runtime.readSleepingMailHold(mailbox)).toMatchObject({
resuming: true,
liveShell: false
})
runtime.retrySleptMailboxWakes()
await settled()
expect(resume.mock.calls.filter((call) => call[2] !== true)).toHaveLength(0)
await restoreUndispatchedMailWake(async (edit) => {
const next = edit(session)
if (next) {
session = next
written('local')
}
}, retained)
await settled()
expect(resume.mock.calls.filter((call) => call[2] !== true)).toEqual([
['repo::/workspace', 'tab:11111111-1111-4111-8111-111111111111']
])
expect(session.sleepingAgentSessionsByPaneKey?.[paneKey]?.mailWake).toBeUndefined()
} finally {
runtime.setNotifier(null)
db.close()
}
})
@@ -1,9 +1,10 @@
import { POTENTIALLY_LIVE_REMOTE_ATTACHMENT_STATES } from '../../db/federation/remote-attachment-liveness'
import type { SleepingAgentSessionRecord } from '../../../../../shared/agent-session/lifecycle/agent-session-resume'
import { agentProviderSessionsEqual } from '../../../../../shared/agent-session/lifecycle/agent-session-resume'
import { mailWakeObligation } from '../../../../../shared/agent-session/lifecycle/mail-wake-obligation'
import type { OrchestrationDb } from '../../db/orchestration-db'
import {
selectOrchestrationPointerBatch,
selectOrchestrationPointerSequence,
type OrchestrationMessageWaiter
} from '../terminal-pointer/mailbox-pointer-eligibility'
@@ -45,12 +46,20 @@ export class SleptMailboxWake {
private readonly inFlight = new Map<string, number>()
private readonly succeeded = new Map<string, { paneKey: string; sequence: number }>()
private readonly failed = new Map<string, number>()
private readonly owed = new Map<string, { paneKey: string; sequence: number }>()
private readonly byPane = new Map<string, Set<string>>()
private readonly attempted = new Map<
string,
{ record: SleepingAgentSessionRecord; sequence: number }
>()
private readonly held = new Map<string, string>()
private readonly workspaceCounts = new Map<string, number>()
private active = 0
constructor(private readonly deps: WakeDeps) {}
failedToWake(record: SleepingAgentSessionRecord): boolean {
return this.failed.get(record.paneKey) === record.capturedAt
return this.failed.get(record.paneKey) === (record.mailWake?.capturedAt ?? record.capturedAt)
}
batch(mailbox: string, reservedTypes?: ReadonlySet<string>) {
@@ -60,24 +69,150 @@ export class SleptMailboxWake {
if (!db || !record || !mailWakeObligation(record)) {
return null
}
const messages = selectOrchestrationPointerBatch({
db,
mailboxHandle: mailbox,
waiters: this.deps.waiters(mailbox),
reservedTypes,
newestFirst: true
}).messages
const sequence = messages[0]?.sequence
const sequence = this.selection(mailbox, reservedTypes)?.sequence
return sequence === undefined ? null : { record, sequence }
}
private selection(mailbox: string, reservedTypes?: ReadonlySet<string>) {
const db = this.deps.db()
const paneKey = db && mailboxOwnerPane(db, mailbox)
if (!db || !paneKey) {
return null
}
const sequence = selectOrchestrationPointerSequence({
db,
mailboxHandle: mailbox,
waiters: this.deps.waiters(mailbox),
reservedTypes
})
return sequence === undefined ? null : { paneKey, sequence }
}
refresh(mailbox: string): string[] {
const before = this.owed.get(mailbox)
const next = this.selection(mailbox)
if (before) {
const handles = this.byPane.get(before.paneKey)
handles?.delete(mailbox)
if (!handles?.size) {
this.byPane.delete(before.paneKey)
}
}
this.owed.delete(mailbox)
if (next) {
this.owed.set(mailbox, next)
const handles = this.byPane.get(next.paneKey) ?? new Set<string>()
handles.add(mailbox)
this.byPane.set(next.paneKey, handles)
}
const success = this.succeeded.get(mailbox)
if (!next || success?.paneKey !== next.paneKey || success.sequence !== next.sequence) {
this.succeeded.delete(mailbox)
}
const attempted = this.attempted.get(mailbox)
if (!next || (attempted && attempted.record.paneKey !== next.paneKey)) {
this.attempted.delete(mailbox)
}
if (before && !this.byPane.has(before.paneKey)) {
this.failed.delete(before.paneKey)
}
if (before) {
this.refreshPane(before.paneKey)
}
if (next) {
this.refreshPane(next.paneKey)
}
return [...new Set([...(before ? [before.paneKey] : []), ...(next ? [next.paneKey] : [])])]
}
get hasTrackedMail(): boolean {
return this.owed.size > 0 || this.attempted.size > 0
}
reconcile(): void {
const handles = new Set(this.deps.db()?.getUndeliveredUnreadMailboxHandles() ?? [])
for (const handle of this.owed.keys()) {
handles.add(handle)
}
for (const handle of handles) {
this.refresh(handle)
}
}
mailboxesForPane(paneKey: string): readonly string[] {
return [...(this.byPane.get(paneKey) ?? [])]
}
owesPane(paneKey: string): boolean {
return (this.deps.db()?.getUndeliveredUnreadMailboxHandles() ?? []).some(
(mailbox) => this.batch(mailbox)?.record.paneKey === paneKey
)
for (const mailbox of this.mailboxesForPane(paneKey)) {
this.refresh(mailbox)
if (this.batch(mailbox)?.record.paneKey === paneKey) {
return true
}
}
return false
}
refreshPane(paneKey: string): void {
const previous = this.held.get(paneKey)
const record = this.deps.record(paneKey)
const next =
this.byPane.has(paneKey) && mailWakeObligation(record) ? record?.worktreeId : undefined
if (previous === next) {
return
}
if (previous) {
const count = (this.workspaceCounts.get(previous) ?? 1) - 1
if (count) {
this.workspaceCounts.set(previous, count)
} else {
this.workspaceCounts.delete(previous)
}
}
this.held.delete(paneKey)
if (next) {
this.held.set(paneKey, next)
this.workspaceCounts.set(next, (this.workspaceCounts.get(next) ?? 0) + 1)
}
if (!record) {
this.failed.delete(paneKey)
}
}
heldWorktrees(): string[] {
return [...this.workspaceCounts.keys()].sort()
}
confirmed(record: SleepingAgentSessionRecord): void {
for (const mailbox of this.mailboxesForPane(record.paneKey)) {
const attempt = this.attempted.get(mailbox)
if (
!attempt ||
attempt.record.worktreeId !== record.worktreeId ||
attempt.record.agent !== record.agent ||
(attempt.record.connectionId ?? null) !== (record.connectionId ?? null) ||
(attempt.record.mailWake?.capturedAt ?? attempt.record.capturedAt) !==
record.mailWake?.capturedAt ||
(attempt.record.mailWake && attempt.record.mailWake.id !== record.mailWake?.id) ||
!agentProviderSessionsEqual(
record.agent,
record.providerSession,
attempt.record.providerSession
)
) {
continue
}
this.refresh(mailbox)
if (this.owed.get(mailbox)?.paneKey === record.paneKey) {
this.succeeded.set(mailbox, { paneKey: record.paneKey, sequence: attempt.sequence })
this.failed.delete(record.paneKey)
}
this.attempted.delete(mailbox)
}
}
request(mailbox: string, reservedTypes?: ReadonlySet<string>): void {
this.refresh(mailbox)
const batch = this.batch(mailbox, reservedTypes)
if (
!batch ||
@@ -125,6 +260,7 @@ export class SleptMailboxWake {
) {
return
}
this.failed.delete(before.record.paneKey)
await this.deps.prepare(before.record)
const current = this.batch(mailbox, reservedTypes)
if (
@@ -134,11 +270,23 @@ export class SleptMailboxWake {
) {
return
}
if (await this.deps.wake(current.record).catch(() => false)) {
this.attempted.set(mailbox, { record: current.record, sequence: current.sequence })
const succeeded = await this.deps.wake(current.record).catch(() => false)
this.refresh(mailbox)
if (this.owed.get(mailbox)?.paneKey !== current.record.paneKey) {
return
}
if (succeeded) {
if (this.owed.get(mailbox)?.sequence === current.sequence) {
this.succeeded.set(mailbox, { paneKey: current.record.paneKey, sequence: current.sequence })
}
this.failed.delete(current.record.paneKey)
this.succeeded.set(mailbox, { paneKey: current.record.paneKey, sequence: current.sequence })
} else {
this.failed.set(current.record.paneKey, current.record.capturedAt)
this.attempted.delete(mailbox)
} else if (this.deps.record(current.record.paneKey)) {
this.failed.set(
current.record.paneKey,
current.record.mailWake?.capturedAt ?? current.record.capturedAt
)
}
}
}
@@ -235,7 +235,7 @@ export abstract class OrcaRuntimeWithGetPtyRecordForPaneKey extends RuntimeSlept
* longer owes. Visible cleanup only: it never holds up the action that called it. `mailboxes`:
* only those the action touched; without it, every mailbox (a reset, a store becoming readable). */
withdrawStaleChatMail(mailboxes?: Iterable<string>): void {
this.publishSleepingMail()
this.publishSleepingMail(mailboxes)
this.orchestrationStructuredMailboxPointerDelivery.withdrawStale(mailboxes)
}
@@ -19,9 +19,13 @@ import {
import { resumePendingOrchestrationMailboxPointer } from './mailbox-pointer-resume'
import { stageOrchestrationMailboxPointer } from './mailbox-pointer-stage'
const DEFAULT_POINTER_ENTER_DELAY_MS = 500
function pointerEnterDelayMs(): number {
const configured = Number(process.env.ORCA_E2E_ORCHESTRATION_POINTER_ENTER_DELAY_MS)
return Number.isFinite(configured) && configured >= 1 && configured <= 60_000 ? configured : 500
return Number.isFinite(configured) && configured >= 1 && configured <= 60_000
? configured
: DEFAULT_POINTER_ENTER_DELAY_MS
}
export class OrchestrationMailboxPointerDelivery<TWaiter extends OrchestrationMessageWaiter> {
@@ -40,7 +40,7 @@ export type OrchestrationPointerBatch = {
/** Both delivery lanes share this synchronous SQL selection; unfiltered waiters preempt all pointers.
* Concrete waiter types are excluded byte-exactly in SQL; no JavaScript post-filter is needed.
* An unacked batch defers rather than drops: only mail it does not hold is eligible, once each. */
export function selectOrchestrationPointerBatch(input: {
type PointerSelection = {
db: OrchestrationDb
mailboxHandle: string
waiters: ReadonlySet<OrchestrationMessageWaiter> | undefined
@@ -48,22 +48,44 @@ export function selectOrchestrationPointerBatch(input: {
/** A terminal's own mailbox leaves rows a Run owns undelivered for routing to move. */
excludeRunOwnedDirect?: boolean
newestFirst?: boolean
}): OrchestrationPointerBatch {
if (hasUnfilteredOrchestrationWaiter(input.waiters)) {
return { messages: [], heldDeliveryId: undefined }
}
}
function pointerExclusions(input: PointerSelection, excludeIds?: readonly string[]) {
const excludedTypes = new Set(input.reservedTypes)
for (const waiter of input.waiters ?? []) {
for (const type of waiter.typeFilter ?? []) {
excludedTypes.add(type)
}
}
return {
excludeTypes: [...excludedTypes],
excludeIds,
excludeRunOwnedDirect: input.excludeRunOwnedDirect
}
}
export function selectOrchestrationPointerSequence(input: PointerSelection): number | undefined {
return hasUnfilteredOrchestrationWaiter(input.waiters)
? undefined
: input.db.getUndeliveredUnreadSequence(
input.mailboxHandle,
pointerExclusions(
input,
input.db.getOutstandingMailboxDelivery?.(input.mailboxHandle)?.messageIds
)
)
}
export function selectOrchestrationPointerBatch(
input: PointerSelection
): OrchestrationPointerBatch {
if (hasUnfilteredOrchestrationWaiter(input.waiters)) {
return { messages: [], heldDeliveryId: undefined }
}
const held = input.db.getOutstandingMailboxDelivery?.(input.mailboxHandle)
return {
messages: input.db.getUndeliveredUnreadMessages(input.mailboxHandle, undefined, {
excludeTypes: [...excludedTypes],
excludeIds: held?.messageIds,
excludeRunOwnedDirect: input.excludeRunOwnedDirect,
...pointerExclusions(input, held?.messageIds),
newestFirst: input.newestFirst,
limit: ORCHESTRATION_DELIVERY_BATCH_LIMIT
}),
@@ -31,7 +31,7 @@ async function readDeliveryHoldWarning(
recipient: string
): Promise<SendRecipientWarning | null> {
const hold =
runtime.readSleepingMailHold?.(recipient) ??
(await runtime.readSleepingMailHold?.(recipient)) ??
(await readMailboxDeliveryHold(recipient, {
chatSessionFor: (mailbox) => {
const custody = structuredMailboxCustody(mailbox, db)
@@ -1,5 +1,5 @@
import { storedPaneBinding } from '../../terminal/recovery/restored-pane-binding'
import { retainMailWakeAttempt } from './retained-mail-wake-attempt'
import { retainMailWakeAttempt, restoreUndispatchedMailWake } from './retained-mail-wake-attempt'
// Restore terminal-less saved panes for pending-row activation and workspace wakes; main-owned layouts also handle triggers outside desktop restore.
import type { OrcaRuntimeWithAgentSessionResumeOwnership as AgentSessionResume } from './agent-session-resume-ownership'
@@ -200,6 +200,7 @@ export abstract class OrcaRuntimeWithRestoredPaneStart extends OrcaRuntimeWithBu
const resumeStartup = await this.resolvePaneOwnedResumeStartup(worktreeId, tab)
// After a claimed-record resume starts, mark that record live.
const paneKey = `${tab.parentTabId}:${tab.leafId}`
const hostId = this.getWorkspaceSessionHostIdForWorktree(worktreeId)
const savedRecord =
this.getWorkspaceSessionForWorktree(worktreeId)?.sleepingAgentSessionsByPaneKey?.[paneKey]
const resumedRecord =
@@ -272,11 +273,7 @@ export abstract class OrcaRuntimeWithRestoredPaneStart extends OrcaRuntimeWithBu
onPtyCreated: (ptyId: string, result: RuntimeTerminalCreate) => {
if (resumedRecord?.mailWake) {
mailWakeWrite = retainMailWakeAttempt(
(edit) =>
this.commitWorkspaceSessionEdit(
this.getWorkspaceSessionHostIdForWorktree(worktreeId),
edit
),
(edit) => this.commitWorkspaceSessionEdit(hostId, edit),
resumedRecord,
ptyId,
result
@@ -312,7 +309,6 @@ export abstract class OrcaRuntimeWithRestoredPaneStart extends OrcaRuntimeWithBu
await mailWakeWrite
// A running resumed agent's record must not remain asleep.
if (created.tab.ptyId && resumedRecord) {
const hostId = this.getWorkspaceSessionHostIdForWorktree(worktreeId)
void markResumedRecordLive(
(edit) => this.commitWorkspaceSessionEdit(hostId, edit),
paneKey,
@@ -322,6 +318,12 @@ export abstract class OrcaRuntimeWithRestoredPaneStart extends OrcaRuntimeWithBu
return created
} catch (err) {
resumeClaim?.settleFailure(dispatched, err)
if (!dispatched && resumedRecord) {
await restoreUndispatchedMailWake(
(edit) => this.commitWorkspaceSessionEdit(hostId, edit),
resumedRecord
)
}
if (
!(automatic && isWorktreeTerminalsSleepingError(err)) &&
sessionId &&
@@ -1,4 +1,7 @@
import { mailWakeObligation } from '../../../../shared/agent-session/lifecycle/mail-wake-obligation'
import {
mailWakeObligation,
undispatchedMailWakeRecord
} from '../../../../shared/agent-session/lifecycle/mail-wake-obligation'
import type { SleepingAgentSessionRecord } from '../../../../shared/agent-session/lifecycle/agent-session-resume'
import type { RuntimeTerminalCreate } from '../../../../shared/runtime-terminal-create-contract'
import type { WorkspaceSessionState } from '../../../../shared/workspace-session/workspace-session-state-types'
@@ -46,3 +49,37 @@ export async function retainMailWakeAttempt(
}
})
}
export function replaceMailWakeRecord(
session: WorkspaceSessionState,
record: SleepingAgentSessionRecord,
next: SleepingAgentSessionRecord
): WorkspaceSessionState | null {
const current = session.sleepingAgentSessionsByPaneKey?.[record.paneKey]
if (
current?.mailWake?.id !== record.mailWake?.id ||
current?.capturedAt !== record.capturedAt ||
current?.origin !== record.origin ||
current?.restoreOnTabOpenOnly !== record.restoreOnTabOpenOnly ||
current?.mailWake?.attempt?.launchToken !== record.mailWake?.attempt?.launchToken
) {
return null
}
return {
...session,
sleepingAgentSessionsByPaneKey: {
...session.sleepingAgentSessionsByPaneKey,
[record.paneKey]: next
}
}
}
export async function restoreUndispatchedMailWake(
commit: Commit,
record: SleepingAgentSessionRecord
): Promise<void> {
const next = undispatchedMailWakeRecord(record)
if (next) {
await commit((session) => replaceMailWakeRecord(session, record, next))
}
}
@@ -166,7 +166,7 @@ export function installPaneAgentIdentity(session: ConnectPanePtySession): void {
const isHibernationWakeRecord =
(ptyId && state.suppressedPtyExitIds[ptyId]) ||
(wake && wake.record === sleepingRecord?.record && (ptyId === null || wake.ptyId === ptyId))
if (sleepingRecord && !isHibernationWakeRecord) {
if (sleepingRecord && !sleepingRecord.record.mailWake && !isHibernationWakeRecord) {
session.clearSleepingRecordProviderDuplicates(state, sleepingRecord)
}
session.clearStaleAgentTabTitleOnConfirmedShell()
@@ -1,3 +1,9 @@
import {
currentMailWakeStartup,
isColdRestoreStartup,
restoreUndispatchedMailWake
} from '../startup/pane-resume-record'
import { useAppStore } from '@/store/index'
import {
applyTerminalLaunchResult,
captureTerminalLaunchContext
@@ -140,6 +146,15 @@ export function bindCaptureTransportOutputCallbacks(session: ConnectPanePtySessi
}
},
onSpawnDeferred: (reason: SpawnDeferralReason): void => {
if (isCurrent() && isColdRestoreStartup(savedStartup)) {
const record = currentMailWakeStartup(
savedStartup,
useAppStore.getState().sleepingAgentSessionsByPaneKey
).sleepingRecordEntry?.record
if (record) {
restoreUndispatchedMailWake(record)
}
}
if (isCurrent()) {
session.pendingSpawnDeferral = reason
}
@@ -1,3 +1,4 @@
import type { PtyPaneStartup } from './pty-pane-startup'
import { useAppStore } from '@/store/index'
import { recordPaneWakeThroughLayoutOwner } from '@/store/terminals/terminal-sleep-layout-owner'
import { ORCA_LAYOUT_OWNER_IS_MAIN } from '../../../../../../shared/layout-owner-switch'
@@ -8,10 +9,13 @@ import {
} from '../../../../../../shared/agent-session/lifecycle/agent-session-resume'
import type { ConnectPanePtySession } from '../connect-pane-pty-session'
import type { ColdRestoreAgentResumeStartup, PendingStartupCommand } from './fresh-spawn-types'
import { mailWakeObligation } from '../../../../../../shared/agent-session/lifecycle/mail-wake-obligation'
import {
mailWakeObligation,
undispatchedMailWakeRecord
} from '../../../../../../shared/agent-session/lifecycle/mail-wake-obligation'
export function isColdRestoreStartup(
startup: PendingStartupCommand | null | undefined
startup: PendingStartupCommand | PtyPaneStartup | undefined
): startup is ColdRestoreAgentResumeStartup {
return Boolean(
startup &&
@@ -114,7 +118,11 @@ export function installPaneResumeRecord(session: ConnectPanePtySession): void {
} else {
recordPaneWakeThroughLayoutOwner(state, session.deps.worktreeId, session.cacheKey)
}
session.clearSleepingRecordProviderDuplicates(state, startup.sleepingRecordEntry)
session.clearSleepingRecordProviderDuplicates(
state,
startup.sleepingRecordEntry,
session.cacheKey
)
}
return
}
@@ -150,3 +158,12 @@ export function installPaneResumeRecord(session: ConnectPanePtySession): void {
}
}
}
/** Called only after a definite refusal before dispatch, never after a lost spawn reply. */
export function restoreUndispatchedMailWake(record: SleepingAgentSessionRecord): void {
const state = useAppStore.getState()
const next = undispatchedMailWakeRecord(record)
if (next && state.sleepingAgentSessionsByPaneKey[record.paneKey] === record) {
state.rememberSleepingAgentSession(next, record)
}
}
@@ -47,7 +47,11 @@ export function installSleepingRecordAccess(session: ConnectPanePtySession): voi
consumed: { paneKey: string; record: SleepingAgentSessionRecord },
keepPaneKey?: string
): void => {
if (consumed.paneKey !== keepPaneKey) {
if (
consumed.paneKey !== keepPaneKey &&
useAppStore.getState().sleepingAgentSessionsByPaneKey[consumed.paneKey] === consumed.record &&
!consumed.record.mailWake
) {
state.clearSleepingAgentSession(consumed.paneKey)
}
for (const [paneKey, record] of sleepingAgentRecordIndex.entries(
@@ -55,6 +59,9 @@ export function installSleepingRecordAccess(session: ConnectPanePtySession): voi
consumed.record.worktreeId
)) {
if (
useAppStore.getState().sleepingAgentSessionsByPaneKey[paneKey] === record &&
!record.mailWake &&
record.capturedAt <= consumed.record.capturedAt &&
paneKey !== consumed.paneKey &&
paneKey !== keepPaneKey &&
record.worktreeId === consumed.record.worktreeId &&
@@ -1,3 +1,4 @@
import { trackAgentPaneSleep } from '@/store/terminals/agent-pane-sleep-in-flight'
import { useAppStore } from '@/store/index'
import {
getEffectiveAgentHibernationIdleMs,
@@ -145,15 +146,17 @@ async function hibernatePaneIfStillEligible(
try {
const state = useAppStore.getState()
const owner = resolveAgentHibernationRuntimeOwner(state, worktreeId)
await state.shutdownCompletedAgentPaneForHibernation(worktreeId, {
paneKey: confirmedCandidate.paneKey,
tabId: confirmedCandidate.tabId,
leafId: confirmedCandidate.leafId,
ptyId: confirmedCandidate.targetPtyIds[0],
...(owner.kind === 'runtime'
? { expectedRuntimePtyId: confirmedCandidate.expectedRuntimePtyIds[0] }
: {})
})
await trackAgentPaneSleep(confirmedCandidate.paneKey, () =>
state.shutdownCompletedAgentPaneForHibernation(worktreeId, {
paneKey: confirmedCandidate.paneKey,
tabId: confirmedCandidate.tabId,
leafId: confirmedCandidate.leafId,
ptyId: confirmedCandidate.targetPtyIds[0],
...(owner.kind === 'runtime'
? { expectedRuntimePtyId: confirmedCandidate.expectedRuntimePtyIds[0] }
: {})
})
)
} catch (err) {
console.warn('[agent-hibernation] failed to hibernate agent pane:', id, err)
} finally {
@@ -1,6 +1,23 @@
import { useAppStore } from '@/store/index'
import { installSleepingRecordAccess } from '@/components/terminal-pane/pty-connection/startup/sleeping-record-access'
import { installPaneAgentIdentity } from '@/components/terminal-pane/pty-connection/agent-identity/pane-agent-identity'
import { bindCaptureTransportOutputCallbacks } from '@/components/terminal-pane/pty-connection/lifecycle/transport-output-callbacks'
const foreground = vi.hoisted(() => ({ onShell: (_reason: 'process-exit') => {} }))
vi.mock(
'@/components/terminal-pane/pty-connection/agent-identity/foreground-process/pane-foreground-agent-tracker',
() => ({
createPaneForegroundAgentTracker: (options: {
onConfirmedShellForeground: typeof foreground.onShell
}) => {
foreground.onShell = options.onConfirmedShellForeground
return {}
}
})
)
import {
currentMailWakeStartup,
paneResumeRecord
paneResumeRecord,
installPaneResumeRecord
} from '@/components/terminal-pane/pty-connection/startup/pane-resume-record'
import { hasEmptyReattachRetireEvidence } from '@/components/terminal-pane/pty-connection/reattach/empty-reattach-retire-evidence'
import type { ColdRestoreAgentResumeStartup } from '@/components/terminal-pane/pty-connection/startup/fresh-spawn-types'
@@ -102,3 +119,130 @@ it('keeps pre-launch custody on its quit bookmark through an already armed wake
currentMailWakeStartup(startup, { [original.paneKey]: manual }).sleepingRecordEntry?.record
).toBe(original)
})
it('preserves custody through the installed spawn/alias/shell callbacks and restores only a definite prelaunch refusal', () => {
const original: SleepingAgentSessionRecord = {
paneKey: 'tab:11111111-1111-4111-8111-111111111111',
tabId: 'tab',
worktreeId: 'workspace',
agent: 'codex',
providerSession: { key: 'session_id', id: 'conversation' },
prompt: '',
state: 'done',
origin: 'worktree-sleep',
capturedAt: 1,
updatedAt: 1
}
const retained: SleepingAgentSessionRecord = {
...original,
origin: 'quit',
restoreOnTabOpenOnly: true,
mailWake: { id: 'obligation', capturedAt: 1, startedAt: 100 }
}
const alias = { ...retained, paneKey: 'tab:22222222-2222-4222-8222-222222222222' }
const replacement = {
...original,
paneKey: 'tab:33333333-3333-4333-8333-333333333333',
capturedAt: 2
}
state.sleepingAgentSessionsByPaneKey = {
[original.paneKey]: retained,
[alias.paneKey]: alias,
[replacement.paneKey]: replacement
}
Object.assign(state, {
agentLaunchConfigByPaneKey: {},
agentStatusByPaneKey: {},
paneForegroundAgentByPaneKey: {},
automaticAgentResumeClaimsByTabId: {},
tabsByWorktree: {},
suppressedPtyExitIds: {},
rememberSleepingAgentSession: (
record: SleepingAgentSessionRecord,
expected?: SleepingAgentSessionRecord
) => {
if (expected && state.sleepingAgentSessionsByPaneKey[record.paneKey] !== expected) {
return
}
state.sleepingAgentSessionsByPaneKey = {
...state.sleepingAgentSessionsByPaneKey,
[record.paneKey]: record
}
},
clearSleepingAgentSession: (key: string) => {
const next = { ...state.sleepingAgentSessionsByPaneKey }
delete next[key]
state.sleepingAgentSessionsByPaneKey = next
},
clearAgentLaunchConfig: vi.fn()
})
const transport = { getPtyId: () => 'pty-shell' }
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the installers exercise the listed pane/store callbacks; unused transport methods are never called.
const session = {
cacheKey: original.paneKey,
paneStartup: null,
remotePtyIncarnationId: 'inc',
transport,
transportStreamGeneration: 0,
disposed: false,
pane: {
id: 1,
leafId: '11111111-1111-4111-8111-111111111111',
terminal: {
parser: { registerOscHandler: () => ({ dispose() {} }) },
onRender: () => ({ dispose() {} })
}
},
deps: {
tabId: 'tab',
worktreeId: 'workspace',
isVisibleRef: { current: false },
paneTransportsRef: { current: new Map([[1, transport]]) },
onAgentExitedRef: { current: vi.fn() }
},
writeInputModeGround: vi.fn(),
createProcessExitState: () => ({ detector: { observe() {} } })
} as unknown as ConnectPanePtySession
installSleepingRecordAccess(session)
installPaneResumeRecord(session)
const startup: ColdRestoreAgentResumeStartup = {
agent: 'codex',
command: 'codex resume conversation',
resumeProviderSession: original.providerSession,
launchToken: 'invocation',
resumeOrigin: 'saved',
launchConfig: { agentArgs: '', agentEnv: {} },
useLiveEntry: false,
hasSleepingRecord: true,
sleepingRecordEntry: { paneKey: original.paneKey, record: original }
}
session.keepResumeRecordAfterColdRestoreSpawn(startup)
expect(state.sleepingAgentSessionsByPaneKey[original.paneKey]?.mailWake?.attempt).toEqual({
ptyId: 'pty-shell',
incarnationId: 'inc',
launchToken: 'invocation',
startedAt: 100
})
session.clearSleepingRecordProviderDuplicates(useAppStore.getState(), {
paneKey: original.paneKey,
record: original
})
installPaneAgentIdentity(session)
foreground.onShell('process-exit')
expect(state.sleepingAgentSessionsByPaneKey[original.paneKey]?.mailWake?.id).toBe('obligation')
expect(state.sleepingAgentSessionsByPaneKey[alias.paneKey]?.mailWake?.id).toBe('obligation')
expect(state.sleepingAgentSessionsByPaneKey[replacement.paneKey]).toBe(replacement)
session.commandLifecycle.dispose()
state.sleepingAgentSessionsByPaneKey = { [retained.paneKey]: retained }
bindCaptureTransportOutputCallbacks(session)
const capture = session.captureTransportOutputCallbacks(vi.fn(), null, startup)
expect(state.sleepingAgentSessionsByPaneKey[retained.paneKey]?.mailWake?.id).toBe('obligation')
capture.callbacks.onSpawnDeferred?.('paused')
expect(state.sleepingAgentSessionsByPaneKey[retained.paneKey]).toMatchObject({
origin: 'worktree-sleep',
capturedAt: 1
})
expect(state.sleepingAgentSessionsByPaneKey[retained.paneKey]?.mailWake).toBeUndefined()
session.startupTiming?.finish('replaced')
})
@@ -1,3 +1,4 @@
import { restoreUndispatchedMailWake } from '@/components/terminal-pane/pty-connection/startup/pane-resume-record'
import { patchWorkspaceSessionByHost } from '../../workspace/persistence/workspace-session-host-persistence'
import { createNonSecureContextUuid } from '../../../../../shared/non-secure-context-uuid'
import { waitForAgentPaneSleep } from '@/store/terminals/agent-pane-sleep-in-flight'
@@ -269,14 +270,19 @@ async function wakeSleepingPaneInBackground(
}
}
useAppStore.getState().rememberSleepingAgentSession(captured, record)
await patchWorkspaceSessionByHost(
window.api.session,
{
sleepingAgentSessionsByPaneKey: useAppStore.getState().sleepingAgentSessionsByPaneKey
},
useAppStore.getState()
).written
await window.api.session.flush()
try {
await patchWorkspaceSessionByHost(
window.api.session,
{
sleepingAgentSessionsByPaneKey: useAppStore.getState().sleepingAgentSessionsByPaneKey
},
useAppStore.getState()
).written
await window.api.session.flush()
} catch (error) {
restoreUndispatchedMailWake(captured)
throw error
}
if (useAppStore.getState().sleepingAgentSessionsByPaneKey[paneKey] !== captured) {
return false
}
@@ -1,4 +1,3 @@
import { trackAgentPaneSleep } from './agent-pane-sleep-in-flight'
import { replaceRecordEntries, replaceTabBuckets } from '@/store/keyed-record-transitions'
import { isRemoteRuntimePtyId } from '../../../../shared/remote-runtime/remote-runtime-pty-id'
import { parsePaneKey } from '../../../../shared/workspace-layout/stable-pane-id'
@@ -29,239 +28,231 @@ export function createTerminalPaneHibernationActions(
get: TerminalStoreGet
): Pick<TerminalSlice, 'shutdownCompletedAgentPaneForHibernation'> {
return {
shutdownCompletedAgentPaneForHibernation: (worktreeId, opts) =>
trackAgentPaneSleep(opts.paneKey, async () => {
const paneKeys = [opts.paneKey]
const expectedRuntimePtyIds = sortedUniquePtyIds(
opts.expectedRuntimePtyId ? [opts.expectedRuntimePtyId] : []
)
const rendererShutdownPtyIds = [opts.ptyId]
const state = get()
const worktreeOwner = resolveTerminalPtyOwner(state, worktreeId)
const teardownOwner = resolveTerminalTeardownOwner(worktreeOwner, [opts.ptyId])
// Use PTY owner for twins; losing bindings without an attributable host stop leaves unseen running agents.
if (
teardownOwner.kind === 'refused' ||
(worktreeOwner.kind === 'ambiguous' &&
isRemoteRuntimePtyId(opts.ptyId) &&
expectedRuntimePtyIds.length === 0)
) {
console.warn(
'[agent-hibernation] refusing pane hibernation: terminal owner is ambiguous',
{
worktreeId,
paneKey: opts.paneKey
}
)
throw new Error('ambiguous_terminal_owner')
}
const runtimeEnvironmentId = teardownOwner.runtimeEnvironmentId
// Why: pane transports emit renderer PTY ids, not raw exact-stop handles; guard only the identity that can deliver an exit callback.
const exitGuardPtyIds = [opts.ptyId]
const tab = (state.tabsByWorktree[worktreeId] ?? []).find(
(candidate) => candidate.id === opts.tabId
)
const parsed = parsePaneKey(opts.paneKey)
const layout = state.terminalLayoutsByTabId[opts.tabId]
const liveTabPtyIds = state.ptyIdsByTabId[opts.tabId] ?? []
if (
!tab ||
!parsed ||
parsed.tabId !== opts.tabId ||
parsed.leafId !== opts.leafId ||
layout?.ptyIdsByLeafId?.[opts.leafId] !== opts.ptyId ||
(expectedRuntimePtyIds.length === 0 && !liveTabPtyIds.includes(opts.ptyId))
) {
throw new Error('agent_hibernation_pane_binding_mismatch')
}
const sleepingAgentSessionRecords = collectSleepingAgentSessionRecordsForWorktree(
state,
shutdownCompletedAgentPaneForHibernation: async (worktreeId, opts) => {
const paneKeys = [opts.paneKey]
const expectedRuntimePtyIds = sortedUniquePtyIds(
opts.expectedRuntimePtyId ? [opts.expectedRuntimePtyId] : []
)
const rendererShutdownPtyIds = [opts.ptyId]
const state = get()
const worktreeOwner = resolveTerminalPtyOwner(state, worktreeId)
const teardownOwner = resolveTerminalTeardownOwner(worktreeOwner, [opts.ptyId])
// Use PTY owner for twins; losing bindings without an attributable host stop leaves unseen running agents.
if (
teardownOwner.kind === 'refused' ||
(worktreeOwner.kind === 'ambiguous' &&
isRemoteRuntimePtyId(opts.ptyId) &&
expectedRuntimePtyIds.length === 0)
) {
console.warn('[agent-hibernation] refusing pane hibernation: terminal owner is ambiguous', {
worktreeId,
{
paneKeys,
captureMode: 'completed-agent-hibernation'
paneKey: opts.paneKey
})
throw new Error('ambiguous_terminal_owner')
}
const runtimeEnvironmentId = teardownOwner.runtimeEnvironmentId
// Why: pane transports emit renderer PTY ids, not raw exact-stop handles; guard only the identity that can deliver an exit callback.
const exitGuardPtyIds = [opts.ptyId]
const tab = (state.tabsByWorktree[worktreeId] ?? []).find(
(candidate) => candidate.id === opts.tabId
)
const parsed = parsePaneKey(opts.paneKey)
const layout = state.terminalLayoutsByTabId[opts.tabId]
const liveTabPtyIds = state.ptyIdsByTabId[opts.tabId] ?? []
if (
!tab ||
!parsed ||
parsed.tabId !== opts.tabId ||
parsed.leafId !== opts.leafId ||
layout?.ptyIdsByLeafId?.[opts.leafId] !== opts.ptyId ||
(expectedRuntimePtyIds.length === 0 && !liveTabPtyIds.includes(opts.ptyId))
) {
throw new Error('agent_hibernation_pane_binding_mismatch')
}
const sleepingAgentSessionRecords = collectSleepingAgentSessionRecordsForWorktree(
state,
worktreeId,
{
paneKeys,
captureMode: 'completed-agent-hibernation'
}
)
const retainedCompletionEvidence = collectHibernatedCompletionEvidenceForWorktree(
state,
worktreeId,
paneKeys
)
if (!sleepingAgentSessionRecords[opts.paneKey]) {
// Why: killing the PTY with no persisted resume record strands the pane unwakeable; abort instead of hibernating unrecoverably.
throw new Error('agent_hibernation_capture_missing')
}
const capture = shutdownBufferCaptures.get(opts.tabId)
if (capture) {
try {
capture({ includeLocalBuffers: false })
} catch {
// Don't let one tab's capture failure block the pane hibernation.
}
}
// Why: store sleeping records before kill, since pty:exit can arrive first.
const sleepingRecordKeys = Object.keys(sleepingAgentSessionRecords)
const replacedSleepingRecords: Record<string, (typeof sleepingAgentSessionRecords)[string]> =
{}
for (const key of sleepingRecordKeys) {
const existing = state.sleepingAgentSessionsByPaneKey[key]
if (existing) {
replacedSleepingRecords[key] = existing
}
}
const rollbackTargetShutdownState = (): void => {
set((s) => {
const next = { ...s.suppressedPtyExitIds }
for (const ptyId of exitGuardPtyIds) {
delete next[ptyId]
}
return {
suppressedPtyExitIds: next,
sleepingAgentSessionsByPaneKey: replaceRecordEntries(
s.sleepingAgentSessionsByPaneKey,
sleepingRecordKeys.map((key) => [key, replacedSleepingRecords[key] ?? null] as const)
)
}
})
}
set((s) => ({
suppressedPtyExitIds: {
...s.suppressedPtyExitIds,
...Object.fromEntries(exitGuardPtyIds.map((ptyId) => [ptyId, true] as const))
},
sleepingAgentSessionsByPaneKey: replaceRecordEntries(
s.sleepingAgentSessionsByPaneKey,
Object.entries(sleepingAgentSessionRecords)
)
const retainedCompletionEvidence = collectHibernatedCompletionEvidenceForWorktree(
state,
worktreeId,
paneKeys
)
if (!sleepingAgentSessionRecords[opts.paneKey]) {
// Why: killing the PTY with no persisted resume record strands the pane unwakeable; abort instead of hibernating unrecoverably.
throw new Error('agent_hibernation_capture_missing')
}))
if (expectedRuntimePtyIds.length > 0) {
if (!runtimeEnvironmentId) {
rollbackTargetShutdownState()
throw new Error('missing_runtime_for_exact_terminal_stop')
}
const capture = shutdownBufferCaptures.get(opts.tabId)
if (capture) {
try {
capture({ includeLocalBuffers: false })
} catch {
// Don't let one tab's capture failure block the pane hibernation.
}
let stopResult: {
stoppedPtyIds?: string[]
livePtyIds?: string[]
postStopVerified?: boolean
postStopFailure?: string
}
// Why: store sleeping records before kill, since pty:exit can arrive first.
const sleepingRecordKeys = Object.keys(sleepingAgentSessionRecords)
const replacedSleepingRecords: Record<
string,
(typeof sleepingAgentSessionRecords)[string]
> = {}
for (const key of sleepingRecordKeys) {
const existing = state.sleepingAgentSessionsByPaneKey[key]
if (existing) {
replacedSleepingRecords[key] = existing
}
}
const rollbackTargetShutdownState = (): void => {
set((s) => {
const next = { ...s.suppressedPtyExitIds }
for (const ptyId of exitGuardPtyIds) {
delete next[ptyId]
}
return {
suppressedPtyExitIds: next,
sleepingAgentSessionsByPaneKey: replaceRecordEntries(
s.sleepingAgentSessionsByPaneKey,
sleepingRecordKeys.map(
(key) => [key, replacedSleepingRecords[key] ?? null] as const
)
)
}
})
}
set((s) => ({
suppressedPtyExitIds: {
...s.suppressedPtyExitIds,
...Object.fromEntries(exitGuardPtyIds.map((ptyId) => [ptyId, true] as const))
},
sleepingAgentSessionsByPaneKey: replaceRecordEntries(
s.sleepingAgentSessionsByPaneKey,
Object.entries(sleepingAgentSessionRecords)
)
}))
if (expectedRuntimePtyIds.length > 0) {
if (!runtimeEnvironmentId) {
rollbackTargetShutdownState()
throw new Error('missing_runtime_for_exact_terminal_stop')
}
let stopResult: {
try {
stopResult = await callRuntimeRpc<{
stoppedPtyIds?: string[]
livePtyIds?: string[]
postStopVerified?: boolean
postStopFailure?: string
}
try {
stopResult = await callRuntimeRpc<{
stoppedPtyIds?: string[]
livePtyIds?: string[]
postStopVerified?: boolean
postStopFailure?: string
}>(
{ kind: 'environment', environmentId: runtimeEnvironmentId },
'terminal.stopExact',
{
worktree: toRuntimeWorktreeSelector(worktreeId),
expectedPtyIds: expectedRuntimePtyIds,
keepHistory: true,
targetOnly: true
},
{ timeoutMs: 15000 }
)
} catch (err) {
rollbackTargetShutdownState()
throw err
}
const stoppedPtyIds = sortedUniquePtyIds(stopResult.stoppedPtyIds)
const livePtyIds = sortedUniquePtyIds(stopResult.livePtyIds)
const targetWasLive = expectedRuntimePtyIds.every((ptyId) => livePtyIds.includes(ptyId))
if (!equalStringSets(stoppedPtyIds, expectedRuntimePtyIds) || !targetWasLive) {
rollbackTargetShutdownState()
throw new Error('exact_terminal_stop_mismatch')
}
if (stopResult.postStopVerified !== true) {
rollbackTargetShutdownState()
throw new Error(stopResult.postStopFailure ?? 'exact_terminal_stop_unverified')
}
for (const snapshot of unregisterPtyDataHandlers(rendererShutdownPtyIds) ?? []) {
snapshot.commit?.()
}
} else if (!isRemoteRuntimePtyId(opts.ptyId)) {
// Why: pty.kill can flush final data before exit; unregister first so stale handlers can't fire phantom notifications during hibernation.
const handlerSnapshots = unregisterPtyDataHandlers(rendererShutdownPtyIds) ?? []
try {
await window.api.pty.kill(opts.ptyId, { keepHistory: true })
} catch (err) {
restorePtyDataHandlersAfterFailedShutdown(handlerSnapshots)
rollbackTargetShutdownState()
throw err
}
for (const snapshot of handlerSnapshots) {
snapshot.commit?.()
}
}>(
{ kind: 'environment', environmentId: runtimeEnvironmentId },
'terminal.stopExact',
{
worktree: toRuntimeWorktreeSelector(worktreeId),
expectedPtyIds: expectedRuntimePtyIds,
keepHistory: true,
targetOnly: true
},
{ timeoutMs: 15000 }
)
} catch (err) {
rollbackTargetShutdownState()
throw err
}
// Switch point: main records the sleep this window decided, once the stop settled.
if (ORCA_LAYOUT_OWNER_IS_MAIN) {
recordPaneSleepThroughLayoutOwner(get(), worktreeId, sleepingAgentSessionRecords)
const stoppedPtyIds = sortedUniquePtyIds(stopResult.stoppedPtyIds)
const livePtyIds = sortedUniquePtyIds(stopResult.livePtyIds)
const targetWasLive = expectedRuntimePtyIds.every((ptyId) => livePtyIds.includes(ptyId))
if (!equalStringSets(stoppedPtyIds, expectedRuntimePtyIds) || !targetWasLive) {
rollbackTargetShutdownState()
throw new Error('exact_terminal_stop_mismatch')
}
settleAttentionRemovedBy(set, get, () =>
set((s) => {
const existingPtyIds = s.ptyIdsByTabId[opts.tabId] ?? []
const shutdownPtyIdSet = new Set(rendererShutdownPtyIds)
const remainingPtyIds = existingPtyIds.filter((ptyId) => !shutdownPtyIdSet.has(ptyId))
const tabs = s.tabsByWorktree[worktreeId] ?? []
const replacements: [string, typeof tabs][] = []
const tabIndex = tabs.findIndex((candidate) => candidate.id === opts.tabId)
if (tabIndex !== -1) {
const nextTabs = [...tabs]
nextTabs[tabIndex] = {
...nextTabs[tabIndex],
ptyId: remainingPtyIds.at(-1) ?? null
}
replacements.push([worktreeId, nextTabs])
if (stopResult.postStopVerified !== true) {
rollbackTargetShutdownState()
throw new Error(stopResult.postStopFailure ?? 'exact_terminal_stop_unverified')
}
for (const snapshot of unregisterPtyDataHandlers(rendererShutdownPtyIds) ?? []) {
snapshot.commit?.()
}
} else if (!isRemoteRuntimePtyId(opts.ptyId)) {
// Why: pty.kill can flush final data before exit; unregister first so stale handlers can't fire phantom notifications during hibernation.
const handlerSnapshots = unregisterPtyDataHandlers(rendererShutdownPtyIds) ?? []
try {
await window.api.pty.kill(opts.ptyId, { keepHistory: true })
} catch (err) {
restorePtyDataHandlersAfterFailedShutdown(handlerSnapshots)
rollbackTargetShutdownState()
throw err
}
for (const snapshot of handlerSnapshots) {
snapshot.commit?.()
}
}
// Switch point: main records the sleep this window decided, once the stop settled.
if (ORCA_LAYOUT_OWNER_IS_MAIN) {
recordPaneSleepThroughLayoutOwner(get(), worktreeId, sleepingAgentSessionRecords)
}
settleAttentionRemovedBy(set, get, () =>
set((s) => {
const existingPtyIds = s.ptyIdsByTabId[opts.tabId] ?? []
const shutdownPtyIdSet = new Set(rendererShutdownPtyIds)
const remainingPtyIds = existingPtyIds.filter((ptyId) => !shutdownPtyIdSet.has(ptyId))
const tabs = s.tabsByWorktree[worktreeId] ?? []
const replacements: [string, typeof tabs][] = []
const tabIndex = tabs.findIndex((candidate) => candidate.id === opts.tabId)
if (tabIndex !== -1) {
const nextTabs = [...tabs]
nextTabs[tabIndex] = {
...nextTabs[tabIndex],
ptyId: remainingPtyIds.at(-1) ?? null
}
const nextCodexRestartNoticeByPtyId = { ...s.codexRestartNoticeByPtyId }
for (const ptyId of exitGuardPtyIds) {
delete nextCodexRestartNoticeByPtyId[ptyId]
}
const nextLastKnownRelay =
remainingPtyIds.length === 0
? replaceRecordEntries<string>(s.lastKnownRelayPtyIdByTabId, [[opts.tabId, null]])
: s.lastKnownRelayPtyIdByTabId
let nextRuntimePaneTitlesByTabId = s.runtimePaneTitlesByTabId
const numericPaneId = Number(opts.leafId)
if (
Number.isInteger(numericPaneId) &&
s.runtimePaneTitlesByTabId[opts.tabId]?.[numericPaneId]
) {
const nextByPane = { ...s.runtimePaneTitlesByTabId[opts.tabId] }
delete nextByPane[numericPaneId]
nextRuntimePaneTitlesByTabId = replaceRecordEntries(s.runtimePaneTitlesByTabId, [
[opts.tabId, Object.keys(nextByPane).length > 0 ? nextByPane : null]
])
}
const nextLastTerminalInputAtByPaneKey = { ...s.lastTerminalInputAtByPaneKey }
delete nextLastTerminalInputAtByPaneKey[opts.paneKey]
return {
tabsByWorktree: replaceTabBuckets(s.tabsByWorktree, replacements),
ptyIdsByTabId: replaceRecordEntries(s.ptyIdsByTabId, [[opts.tabId, remainingPtyIds]]),
lastKnownRelayPtyIdByTabId: nextLastKnownRelay,
suppressedPtyExitIds: {
...s.suppressedPtyExitIds,
...Object.fromEntries(exitGuardPtyIds.map((ptyId) => [ptyId, true] as const))
},
codexRestartNoticeByPtyId: nextCodexRestartNoticeByPtyId,
...(nextRuntimePaneTitlesByTabId !== s.runtimePaneTitlesByTabId
? {
runtimePaneTitlesByTabId: nextRuntimePaneTitlesByTabId
}
: {}),
...removePanesAttention(s, new Set([opts.paneKey])),
lastTerminalInputAtByPaneKey: nextLastTerminalInputAtByPaneKey
}
})
)
get().dropHibernatedAgentStatusPane(worktreeId, opts.paneKey, {
retainedCompletionEvidence
replacements.push([worktreeId, nextTabs])
}
const nextCodexRestartNoticeByPtyId = { ...s.codexRestartNoticeByPtyId }
for (const ptyId of exitGuardPtyIds) {
delete nextCodexRestartNoticeByPtyId[ptyId]
}
const nextLastKnownRelay =
remainingPtyIds.length === 0
? replaceRecordEntries<string>(s.lastKnownRelayPtyIdByTabId, [[opts.tabId, null]])
: s.lastKnownRelayPtyIdByTabId
let nextRuntimePaneTitlesByTabId = s.runtimePaneTitlesByTabId
const numericPaneId = Number(opts.leafId)
if (
Number.isInteger(numericPaneId) &&
s.runtimePaneTitlesByTabId[opts.tabId]?.[numericPaneId]
) {
const nextByPane = { ...s.runtimePaneTitlesByTabId[opts.tabId] }
delete nextByPane[numericPaneId]
nextRuntimePaneTitlesByTabId = replaceRecordEntries(s.runtimePaneTitlesByTabId, [
[opts.tabId, Object.keys(nextByPane).length > 0 ? nextByPane : null]
])
}
const nextLastTerminalInputAtByPaneKey = { ...s.lastTerminalInputAtByPaneKey }
delete nextLastTerminalInputAtByPaneKey[opts.paneKey]
return {
tabsByWorktree: replaceTabBuckets(s.tabsByWorktree, replacements),
ptyIdsByTabId: replaceRecordEntries(s.ptyIdsByTabId, [[opts.tabId, remainingPtyIds]]),
lastKnownRelayPtyIdByTabId: nextLastKnownRelay,
suppressedPtyExitIds: {
...s.suppressedPtyExitIds,
...Object.fromEntries(exitGuardPtyIds.map((ptyId) => [ptyId, true] as const))
},
codexRestartNoticeByPtyId: nextCodexRestartNoticeByPtyId,
...(nextRuntimePaneTitlesByTabId !== s.runtimePaneTitlesByTabId
? {
runtimePaneTitlesByTabId: nextRuntimePaneTitlesByTabId
}
: {}),
...removePanesAttention(s, new Set([opts.paneKey])),
lastTerminalInputAtByPaneKey: nextLastTerminalInputAtByPaneKey
}
})
)
get().dropHibernatedAgentStatusPane(worktreeId, opts.paneKey, {
retainedCompletionEvidence
})
}
}
}
@@ -52,8 +52,18 @@ export function mailWakeEligibility(records: readonly SleepingAgentSessionRecord
.map((record) => [
record.paneKey,
record.mailWake?.capturedAt ?? record.capturedAt,
mailWakeObligation(record) === 'manual'
mailWakeObligation(record)
])
.sort()
)
}
export function undispatchedMailWakeRecord(
record: SleepingAgentSessionRecord
): SleepingAgentSessionRecord | null {
if (mailWakeObligation(record) !== 'retained' || !record.mailWake || record.mailWake.attempt) {
return null
}
const { restoreOnTabOpenOnly: _tabOnly, ...resume } = withoutMailWake(record)
return { ...resume, origin: 'worktree-sleep', capturedAt: record.mailWake.capturedAt }
}