mirror of
https://github.com/stablyai/orca.git
synced 2026-11-19 04:23:57 +00:00
fix(orchestration): retain mail wake custody through failed launches
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
import { undeliveredInboxQuery, type UndeliveredInboxOptions } from './undelivered-inbox-query'
|
||||
import type { MessageType, MessageRow } from '../orchestration-records'
|
||||
import { exposeMessageTimestamps, exposeMessageListTimestamps } from '../utc-timestamp'
|
||||
import { addLifecycleRejectionMarker } from '../lifecycle-rejection-marker'
|
||||
@@ -88,51 +89,29 @@ export function getUndeliveredUnreadMessages(
|
||||
this: OrchestrationDb,
|
||||
toHandle: string,
|
||||
types?: MessageType[],
|
||||
options?: {
|
||||
excludeTypes?: readonly string[]
|
||||
excludeIds?: readonly string[]
|
||||
excludeRunOwnedDirect?: boolean
|
||||
newestFirst?: boolean
|
||||
limit?: number
|
||||
}
|
||||
options?: UndeliveredInboxOptions
|
||||
): MessageRow[] {
|
||||
const conditions = [
|
||||
'to_handle = ?',
|
||||
'read = 0',
|
||||
'delivered_at IS NULL',
|
||||
'pointer_enter_pending = 0',
|
||||
"delivery_contract = 'current_delivery'",
|
||||
...(options?.excludeRunOwnedDirect ? [`NOT ${runOwnsDirectMessageSql('messages')}`] : [])
|
||||
]
|
||||
const params: (string | number)[] = [toHandle]
|
||||
if (types?.length) {
|
||||
conditions.push(`type IN (${types.map(() => '?').join(',')})`)
|
||||
params.push(...types)
|
||||
}
|
||||
if (options?.excludeTypes?.length) {
|
||||
conditions.push(`type NOT IN (${options.excludeTypes.map(() => '?').join(',')})`)
|
||||
params.push(...options.excludeTypes)
|
||||
}
|
||||
if (options?.excludeIds?.length) {
|
||||
conditions.push(`id NOT IN (${options.excludeIds.map(() => '?').join(',')})`)
|
||||
params.push(...options.excludeIds)
|
||||
}
|
||||
const limitSql = options?.limit === undefined ? '' : ' LIMIT ?'
|
||||
if (options?.limit !== undefined) {
|
||||
params.push(Math.max(1, Math.floor(options.limit)))
|
||||
}
|
||||
const query = undeliveredInboxQuery(toHandle, types, options)
|
||||
return exposeMessageListTimestamps(
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: this SELECT returns only messages table rows, decoded at this database boundary.
|
||||
this.db
|
||||
.prepare(
|
||||
`SELECT * FROM messages
|
||||
WHERE ${conditions.join(' AND ')}
|
||||
ORDER BY sequence${options?.newestFirst ? ' DESC' : ''}${limitSql}`
|
||||
)
|
||||
.all(...params) as MessageRow[]
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the shared predicate changes selection only; SELECT * still returns the messages schema.
|
||||
this.db.prepare(`SELECT * ${query.sql}`).all(...query.params) as MessageRow[]
|
||||
)
|
||||
}
|
||||
|
||||
export function getUndeliveredUnreadSequence(
|
||||
this: OrchestrationDb,
|
||||
toHandle: string,
|
||||
options: UndeliveredInboxOptions
|
||||
): number | undefined {
|
||||
const query = undeliveredInboxQuery(toHandle, undefined, {
|
||||
...options,
|
||||
newestFirst: true,
|
||||
limit: 1
|
||||
})
|
||||
const row = this.db.prepare(`SELECT sequence ${query.sql}`).get(...query.params)
|
||||
return typeof row?.sequence === 'number' ? row.sequence : undefined
|
||||
}
|
||||
|
||||
export function getUndeliveredUnreadMailboxHandles(this: OrchestrationDb): string[] {
|
||||
return (
|
||||
this.db
|
||||
@@ -289,6 +268,7 @@ export function getThreadMessagesFor(
|
||||
export type MessageInboxMethods = {
|
||||
getUnreadMessages: typeof getUnreadMessages
|
||||
convertLifecycleMessageToRejection: typeof convertLifecycleMessageToRejection
|
||||
getUndeliveredUnreadSequence: typeof getUndeliveredUnreadSequence
|
||||
getUndeliveredUnreadMessages: typeof getUndeliveredUnreadMessages
|
||||
getUndeliveredUnreadMailboxHandles: typeof getUndeliveredUnreadMailboxHandles
|
||||
getAllMessages: typeof getAllMessages
|
||||
@@ -307,6 +287,7 @@ export function attachMessageInbox(ctor: { prototype: object }): void {
|
||||
Object.assign(ctor.prototype, {
|
||||
getUnreadMessages,
|
||||
convertLifecycleMessageToRejection,
|
||||
getUndeliveredUnreadSequence,
|
||||
getUndeliveredUnreadMessages,
|
||||
getUndeliveredUnreadMailboxHandles,
|
||||
getAllMessages,
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import type { MessageType } from '../orchestration-records'
|
||||
import { runOwnsDirectMessageSql } from '../runs/run-coordinator-mail-routing'
|
||||
|
||||
export type UndeliveredInboxOptions = {
|
||||
excludeTypes?: readonly string[]
|
||||
excludeIds?: readonly string[]
|
||||
excludeRunOwnedDirect?: boolean
|
||||
newestFirst?: boolean
|
||||
limit?: number
|
||||
}
|
||||
|
||||
export function undeliveredInboxQuery(
|
||||
toHandle: string,
|
||||
types?: MessageType[],
|
||||
options?: UndeliveredInboxOptions
|
||||
) {
|
||||
const conditions = [
|
||||
'to_handle = ?',
|
||||
'read = 0',
|
||||
'delivered_at IS NULL',
|
||||
'pointer_enter_pending = 0',
|
||||
"delivery_contract = 'current_delivery'",
|
||||
...(options?.excludeRunOwnedDirect ? [`NOT ${runOwnsDirectMessageSql('messages')}`] : [])
|
||||
]
|
||||
const params: (string | number)[] = [toHandle]
|
||||
if (types?.length) {
|
||||
conditions.push(`type IN (${types.map(() => '?').join(',')})`)
|
||||
params.push(...types)
|
||||
}
|
||||
if (options?.excludeTypes?.length) {
|
||||
conditions.push(`type NOT IN (${options.excludeTypes.map(() => '?').join(',')})`)
|
||||
params.push(...options.excludeTypes)
|
||||
}
|
||||
if (options?.excludeIds?.length) {
|
||||
conditions.push(`id NOT IN (${options.excludeIds.map(() => '?').join(',')})`)
|
||||
params.push(...options.excludeIds)
|
||||
}
|
||||
const limitSql = options?.limit === undefined ? '' : ' LIMIT ?'
|
||||
if (options?.limit !== undefined) {
|
||||
params.push(Math.max(1, Math.floor(options.limit)))
|
||||
}
|
||||
return {
|
||||
sql: `FROM messages WHERE ${conditions.join(' AND ')} ORDER BY sequence${options?.newestFirst ? ' DESC' : ''}${limitSql}`,
|
||||
params
|
||||
}
|
||||
}
|
||||
@@ -8,6 +8,8 @@ export type MailboxDeliveryHold =
|
||||
openTab: boolean
|
||||
failed?: boolean
|
||||
liveShell?: boolean
|
||||
waking?: boolean
|
||||
resuming?: boolean
|
||||
workspaceName?: string
|
||||
}
|
||||
| { kind: 'not-foreground'; agentName: string }
|
||||
@@ -78,6 +80,9 @@ export async function readMailboxDeliveryHold(
|
||||
export function describeMailboxDeliveryHold(recipient: string, hold: MailboxDeliveryHold): string {
|
||||
switch (hold.kind) {
|
||||
case 'asleep':
|
||||
if (hold.waking || hold.resuming) {
|
||||
return `Queued. Orca is ${hold.waking ? 'waking' : 'resuming'} ${hold.agentName}; the message will be delivered when it's back.`
|
||||
}
|
||||
if (hold.liveShell) {
|
||||
return `Queued. Re-run ${hold.agentName} in that terminal; the message is waiting.`
|
||||
}
|
||||
|
||||
@@ -38,15 +38,7 @@ export async function admitsMailWakeRetry(
|
||||
return false
|
||||
}
|
||||
const inspection = await ports.inspect(attempt.ptyId, attempt.incarnationId)
|
||||
const observation = inspection?.foregroundObservation
|
||||
if (
|
||||
inspection?.childProcessEvidence !== 'no-children' ||
|
||||
inspection.hasChildProcesses ||
|
||||
observation?.verdict !== 'observed' ||
|
||||
observation.ptyId !== attempt.ptyId ||
|
||||
observation.ptyIncarnationId !== attempt.incarnationId ||
|
||||
observation.foregroundKind !== 'shell'
|
||||
) {
|
||||
if (inspection?.childProcessEvidence !== 'no-children' || inspection.hasChildProcesses) {
|
||||
return false
|
||||
}
|
||||
return (await ports.shellInFront(attempt.ptyId)) && stillOwns()
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import type { SleepingAgentSessionRecord } from '../../../../../shared/agent-session/lifecycle/agent-session-resume'
|
||||
import type { SleptMailboxWake } from './slept-mailbox-wake'
|
||||
import { mailWakeObligation } from '../../../../../shared/agent-session/lifecycle/mail-wake-obligation'
|
||||
import { TUI_AGENT_DISPLAY_NAMES } from '../../../../../shared/agent-launch/catalog/tui-agent-display-names'
|
||||
import type { MailboxDeliveryHold } from '../delivery-hold/mailbox-delivery-hold'
|
||||
import type { SleepingAgentSessionRecord } from '../../../../../shared/agent-session/lifecycle/agent-session-resume'
|
||||
import { OrcaRuntimeWithPruneMobileSessionTabGroupLayout } from '../../../session-lifecycle/projection/prune-mobile-session-tab-group-layout'
|
||||
import type {
|
||||
RestoredPane,
|
||||
@@ -13,10 +16,8 @@ export abstract class RuntimeMailWakeAdmission extends OrcaRuntimeWithPruneMobil
|
||||
protected abstract readonly mailWakeInvocations: MailWakeInvocations
|
||||
protected abstract hasSleepingMailForPane(paneKey: string): boolean
|
||||
protected abstract sleepingMailRecord(paneKey: string): SleepingAgentSessionRecord | null
|
||||
protected abstract editMailWakeRecord(
|
||||
record: SleepingAgentSessionRecord,
|
||||
next: SleepingAgentSessionRecord
|
||||
): Promise<boolean>
|
||||
protected abstract readonly sleptMailboxWake: SleptMailboxWake
|
||||
protected abstract watchSleepingMail(): void
|
||||
protected abstract getPtyRecordForPaneKey(paneKey: string): RuntimePtyWorktreeRecord | null
|
||||
protected async admitFailedMailWake(
|
||||
pane: RestoredPane,
|
||||
@@ -48,7 +49,7 @@ export abstract class RuntimeMailWakeAdmission extends OrcaRuntimeWithPruneMobil
|
||||
this.terminalRunFacts.readProcessStartedAt(ptyId, incarnationId) !== null
|
||||
)
|
||||
}
|
||||
const safe = await admitsMailWakeRetry(record, {
|
||||
await admitsMailWakeRetry(record, {
|
||||
current: () =>
|
||||
this.hasSleepingMailForPane(record.paneKey)
|
||||
? this.sleepingMailRecord(record.paneKey)
|
||||
@@ -65,28 +66,43 @@ export abstract class RuntimeMailWakeAdmission extends OrcaRuntimeWithPruneMobil
|
||||
shellInFront: async (ptyId) =>
|
||||
(await this.readLaunchedAgentForeground(ptyId, record.agent)) === 'shell'
|
||||
})
|
||||
if (!safe || !owns(attempt.ptyId, attempt.incarnationId) || !this.ptyController?.stopAndWait) {
|
||||
return false
|
||||
// Host-side input fencing is required before a proven failed invocation may be stopped.
|
||||
return false
|
||||
}
|
||||
|
||||
async readSleepingMailHold(mailbox: string): Promise<MailboxDeliveryHold | null> {
|
||||
this.watchSleepingMail()
|
||||
const record = this.sleptMailboxWake.batch(mailbox)?.record
|
||||
if (!record || !mailWakeObligation(record)) {
|
||||
return null
|
||||
}
|
||||
if (!(await this.ptyController.stopAndWait(attempt.ptyId, { keepHistory: true }))) {
|
||||
return false
|
||||
const attempt = record.mailWake?.attempt
|
||||
const pty = this.getPtyRecordForPaneKey(record.paneKey)
|
||||
const liveShell =
|
||||
!!attempt &&
|
||||
pty?.connected === true &&
|
||||
pty.ptyId === attempt.ptyId &&
|
||||
pty.incarnationId === attempt.incarnationId &&
|
||||
pty.launchToken === attempt.launchToken &&
|
||||
pty.launchIncarnationId === attempt.incarnationId &&
|
||||
pty.worktreeId === record.worktreeId &&
|
||||
pty.connectionId === (record.connectionId ?? null) &&
|
||||
this.mailWakeInvocations.ended(record) &&
|
||||
(await this.readLaunchedAgentForeground(attempt.ptyId, record.agent)) === 'shell' &&
|
||||
this.getPtyRecordForPaneKey(record.paneKey) === pty &&
|
||||
this.sleepingMailRecord(record.paneKey) === record
|
||||
return {
|
||||
kind: 'asleep',
|
||||
waking:
|
||||
mailWakeObligation(record) === 'automatic' && !this.sleptMailboxWake.failedToWake(record),
|
||||
resuming: mailWakeObligation(record) === 'retained' && !liveShell,
|
||||
agentName: TUI_AGENT_DISPLAY_NAMES[record.agent],
|
||||
openTab: record.restoreOnTabOpenOnly === true,
|
||||
failed: this.sleptMailboxWake.failedToWake(record),
|
||||
liveShell,
|
||||
workspaceName: this.resolvedWorktrees
|
||||
.peek()
|
||||
?.worktrees.find((row) => row.id === record.worktreeId)?.displayName
|
||||
}
|
||||
const current = this.sleepingMailRecord(record.paneKey)
|
||||
if (
|
||||
current?.mailWake?.attempt?.launchToken !== attempt.launchToken ||
|
||||
mailWakeObligation(current) !== 'retained' ||
|
||||
!this.hasSleepingMailForPane(record.paneKey) ||
|
||||
(this.terminalRunFacts.readLastDrivingInputAt(attempt.ptyId) ?? -1) >= attempt.startedAt
|
||||
) {
|
||||
return false
|
||||
}
|
||||
return this.editMailWakeRecord(current, {
|
||||
...current,
|
||||
mailWake: {
|
||||
id: current.mailWake!.id,
|
||||
capturedAt: current.mailWake!.capturedAt,
|
||||
startedAt: Date.now()
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
import type { SleepingAgentSessionRecord } from '../../../../../shared/agent-session/lifecycle/agent-session-resume'
|
||||
import {
|
||||
mailWakeEligibility,
|
||||
mailWakeObligation
|
||||
} from '../../../../../shared/agent-session/lifecycle/mail-wake-obligation'
|
||||
|
||||
export class SleepingMailRecordIndex {
|
||||
readonly records = new Map<string, SleepingAgentSessionRecord>()
|
||||
readonly custody = new Map<string, SleepingAgentSessionRecord>()
|
||||
readonly changed = new Set<string>()
|
||||
private readonly hostPanes = new Map<string, Set<string>>()
|
||||
|
||||
launchTokens(): ReadonlySet<string> {
|
||||
return new Set(
|
||||
[...this.custody.values()].flatMap((record) =>
|
||||
record.mailWake?.attempt ? [record.mailWake.attempt.launchToken] : []
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
refresh(
|
||||
host: string,
|
||||
records: Readonly<Record<string, SleepingAgentSessionRecord>>,
|
||||
changed: (pane: string) => void
|
||||
): boolean {
|
||||
const previous = this.hostPanes.get(host) ?? new Set<string>()
|
||||
const next = new Set(Object.keys(records))
|
||||
let eligibilityChanged = false
|
||||
for (const pane of previous) {
|
||||
if (!next.has(pane)) {
|
||||
eligibilityChanged ||= this.records.has(pane)
|
||||
this.records.delete(pane)
|
||||
this.custody.delete(pane)
|
||||
this.changed.add(pane)
|
||||
changed(pane)
|
||||
}
|
||||
}
|
||||
for (const record of Object.values(records)) {
|
||||
const before = this.records.get(record.paneKey)
|
||||
if (before !== record) {
|
||||
this.changed.add(record.paneKey)
|
||||
}
|
||||
eligibilityChanged ||=
|
||||
mailWakeEligibility(before ? [before] : []) !== mailWakeEligibility([record])
|
||||
if (mailWakeObligation(record)) {
|
||||
this.records.set(record.paneKey, record)
|
||||
} else {
|
||||
this.records.delete(record.paneKey)
|
||||
}
|
||||
if (record.mailWake) {
|
||||
this.custody.set(record.paneKey, record)
|
||||
} else {
|
||||
this.custody.delete(record.paneKey)
|
||||
}
|
||||
changed(record.paneKey)
|
||||
}
|
||||
this.hostPanes.set(host, next)
|
||||
return eligibilityChanged
|
||||
}
|
||||
}
|
||||
@@ -1,20 +1,19 @@
|
||||
import { replaceMailWakeRecord } from '../../../session-lifecycle/projection/retained-mail-wake-attempt'
|
||||
import { SleepingMailRecordIndex } from './sleeping-mail-record-index'
|
||||
import {
|
||||
mailWakeObligation,
|
||||
mailWakeEligibility,
|
||||
withoutMailWake
|
||||
} from '../../../../../shared/agent-session/lifecycle/mail-wake-obligation'
|
||||
import { queueTerminalWorkspaceCommand } from '../../../workspaces/lifecycle/workspace-lifecycle-commands'
|
||||
import { MailWakeInvocations } from './mail-wake-invocations'
|
||||
import { parsePaneKey } from '../../../../../shared/workspace-layout/stable-pane-id'
|
||||
import { RuntimeMailWakeAdmission } from './runtime-mail-wake-admission'
|
||||
import { TUI_AGENT_DISPLAY_NAMES } from '../../../../../shared/agent-launch/catalog/tui-agent-display-names'
|
||||
import { SleptMailboxWake } from './slept-mailbox-wake'
|
||||
import type { SleepingAgentSessionRecord } from '../../../../../shared/agent-session/lifecycle/agent-session-resume'
|
||||
import type { MailboxDeliveryHold } from '../delivery-hold/mailbox-delivery-hold'
|
||||
|
||||
export abstract class RuntimeSleptMailboxWakes extends RuntimeMailWakeAdmission {
|
||||
protected readonly mailWakeInvocations = new MailWakeInvocations()
|
||||
private readonly sleptMailboxWake = new SleptMailboxWake({
|
||||
protected readonly sleptMailboxWake = new SleptMailboxWake({
|
||||
db: () => this._orchestrationDb,
|
||||
record: (paneKey) => this.sleepingMailRecord(paneKey),
|
||||
waiters: (mailbox) => this.messageWaiters.get(mailbox),
|
||||
@@ -87,15 +86,28 @@ export abstract class RuntimeSleptMailboxWakes extends RuntimeMailWakeAdmission
|
||||
})
|
||||
|
||||
private watchingSleepingMail = false
|
||||
private readonly sleepingRecordIndex = new SleepingMailRecordIndex()
|
||||
private readonly sleepingRecords = this.sleepingRecordIndex.records
|
||||
private readonly custodyRecords = this.sleepingRecordIndex.custody
|
||||
private readonly changedSleepingPanes = this.sleepingRecordIndex.changed
|
||||
|
||||
private watchSleepingMail(): void {
|
||||
private refreshSleepingRecords(host: string): boolean {
|
||||
const changed = this.sleepingRecordIndex.refresh(
|
||||
host,
|
||||
this.store?.getWorkspaceSession?.(host)?.sleepingAgentSessionsByPaneKey ?? {},
|
||||
(pane) => this.sleptMailboxWake.refreshPane(pane)
|
||||
)
|
||||
this.mailWakeInvocations.retain(this.sleepingRecordIndex.launchTokens())
|
||||
return changed
|
||||
}
|
||||
|
||||
protected watchSleepingMail(): void {
|
||||
if (this.watchingSleepingMail) {
|
||||
return
|
||||
}
|
||||
this.watchingSleepingMail = true
|
||||
const records = new Map<string, string>()
|
||||
for (const host of this.store?.getWorkspaceSessionHostIds?.() ?? ['local']) {
|
||||
records.set(host, this.sleepingMailEligibility(host))
|
||||
this.refreshSleepingRecords(host)
|
||||
}
|
||||
this.onClientEvent(
|
||||
(event) => {
|
||||
@@ -105,72 +117,61 @@ export abstract class RuntimeSleptMailboxWakes extends RuntimeMailWakeAdmission
|
||||
},
|
||||
{ consumesTerminalSideEffects: false }
|
||||
)
|
||||
this.subscribeAgentStatusRowChanges(() => this.watchMailWakeInvocations())
|
||||
this.subscribeAgentStatusRowChanges((paneKey) => this.watchMailWakeInvocations(paneKey))
|
||||
this.crewChanges.subscribe((facets) => {
|
||||
if (facets.has('traffic') || facets.has('topology') || facets.has('records')) {
|
||||
this.publishSleepingMail()
|
||||
this.publishSleepingMail(
|
||||
facets.has('topology') || facets.has('records') ? undefined : facets.trafficAddresses
|
||||
)
|
||||
}
|
||||
})
|
||||
this.store?.onWorkspaceSessionWritten?.((hostId) => {
|
||||
const hosts =
|
||||
hostId === null ? (this.store?.getWorkspaceSessionHostIds?.() ?? ['local']) : [hostId]
|
||||
for (const host of hosts) {
|
||||
const next = this.sleepingMailEligibility(host)
|
||||
const changed = this.refreshSleepingRecords(host)
|
||||
this.watchMailWakeInvocations()
|
||||
if (records.get(host) !== next) {
|
||||
records.set(host, next)
|
||||
if (changed) {
|
||||
queueMicrotask(() => this.retrySleptMailboxWakes())
|
||||
} else {
|
||||
this.publishSleepingMail([])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
private sleepingMailEligibility(host: string): string {
|
||||
return mailWakeEligibility(
|
||||
Object.values(this.store?.getWorkspaceSession?.(host)?.sleepingAgentSessionsByPaneKey ?? {})
|
||||
)
|
||||
}
|
||||
|
||||
private allMailWakeRecords(): SleepingAgentSessionRecord[] {
|
||||
return (this.store?.getWorkspaceSessionHostIds?.() ?? ['local']).flatMap((host) =>
|
||||
Object.values(this.store?.getWorkspaceSession?.(host)?.sleepingAgentSessionsByPaneKey ?? {})
|
||||
)
|
||||
}
|
||||
|
||||
private watchMailWakeInvocations(): void {
|
||||
const records = this.allMailWakeRecords()
|
||||
const tokens = new Set<string>()
|
||||
for (const record of records) {
|
||||
const attempt = record.mailWake?.attempt
|
||||
private watchMailWakeInvocations(paneKey?: string): void {
|
||||
const record = paneKey ? this.custodyRecords.get(paneKey) : undefined
|
||||
for (const current of paneKey ? (record ? [record] : []) : this.custodyRecords.values()) {
|
||||
const attempt = current.mailWake?.attempt
|
||||
if (!attempt) {
|
||||
continue
|
||||
}
|
||||
tokens.add(attempt.launchToken)
|
||||
if (
|
||||
this.terminalRunFacts.readProcessStartedAt(attempt.ptyId, attempt.incarnationId) !== null
|
||||
) {
|
||||
this.mailWakeInvocations.watch(record, this)
|
||||
this.mailWakeInvocations.watch(current, this)
|
||||
}
|
||||
const pty = this.getPtyRecordForPaneKey(record.paneKey)
|
||||
const pty = this.getPtyRecordForPaneKey(current.paneKey)
|
||||
if (
|
||||
pty?.connected === true &&
|
||||
pty.worktreeId === record.worktreeId &&
|
||||
pty.connectionId === (record.connectionId ?? null) &&
|
||||
pty.worktreeId === current.worktreeId &&
|
||||
pty.connectionId === (current.connectionId ?? null) &&
|
||||
pty.incarnationId === attempt.incarnationId &&
|
||||
pty.launchToken === attempt.launchToken &&
|
||||
pty.launchIncarnationId === attempt.incarnationId &&
|
||||
this.mailWakeInvocations.observe(
|
||||
record,
|
||||
this.getAgentProviderSessionRowsForPaneFn?.(record.paneKey) ?? [],
|
||||
current,
|
||||
this.getAgentProviderSessionRowsForPaneFn?.(current.paneKey) ?? [],
|
||||
pty.ptyId
|
||||
)
|
||||
) {
|
||||
void this.editMailWakeRecord(record, withoutMailWake(record)).catch((error: unknown) =>
|
||||
this.sleptMailboxWake.confirmed(current)
|
||||
void this.editMailWakeRecord(current, withoutMailWake(current)).catch((error: unknown) =>
|
||||
console.warn('[orchestration] could not settle resumed mail custody', error)
|
||||
)
|
||||
}
|
||||
}
|
||||
this.mailWakeInvocations.retain(tokens)
|
||||
}
|
||||
|
||||
protected async editMailWakeRecord(
|
||||
@@ -181,42 +182,20 @@ export abstract class RuntimeSleptMailboxWakes extends RuntimeMailWakeAdmission
|
||||
await this.commitWorkspaceSessionEdit(
|
||||
this.getWorkspaceSessionHostIdForWorktree(record.worktreeId),
|
||||
(session) => {
|
||||
const current = session.sleepingAgentSessionsByPaneKey?.[record.paneKey]
|
||||
if (
|
||||
current?.mailWake?.id !== record.mailWake?.id ||
|
||||
current?.capturedAt !== record.capturedAt ||
|
||||
current?.origin !== record.origin ||
|
||||
current?.restoreOnTabOpenOnly !== record.restoreOnTabOpenOnly ||
|
||||
current?.mailWake?.attempt?.launchToken !== record.mailWake?.attempt?.launchToken
|
||||
) {
|
||||
return null
|
||||
}
|
||||
changed = true
|
||||
return {
|
||||
...session,
|
||||
sleepingAgentSessionsByPaneKey: {
|
||||
...session.sleepingAgentSessionsByPaneKey,
|
||||
[record.paneKey]: next
|
||||
}
|
||||
}
|
||||
const edited = replaceMailWakeRecord(session, record, next)
|
||||
changed = edited !== null
|
||||
return edited
|
||||
}
|
||||
)
|
||||
if (changed && record.mailWake && !next.mailWake) {
|
||||
this.settledMailWakeIds.add(record.mailWake.id)
|
||||
this.publishSleepingMail()
|
||||
this.publishSleepingMail([])
|
||||
}
|
||||
return changed
|
||||
}
|
||||
|
||||
protected sleepingMailRecord(paneKey: string): SleepingAgentSessionRecord | null {
|
||||
for (const host of this.store?.getWorkspaceSessionHostIds?.() ?? (['local'] as const)) {
|
||||
const record =
|
||||
this.store?.getWorkspaceSession?.(host)?.sleepingAgentSessionsByPaneKey?.[paneKey]
|
||||
if (record) {
|
||||
return record
|
||||
}
|
||||
}
|
||||
return null
|
||||
return this.sleepingRecords.get(paneKey) ?? null
|
||||
}
|
||||
|
||||
protected hasSleepingMailForPane(paneKey: string): boolean {
|
||||
@@ -224,23 +203,32 @@ export abstract class RuntimeSleptMailboxWakes extends RuntimeMailWakeAdmission
|
||||
}
|
||||
|
||||
protected requestSleptMailboxWake(mailbox: string, reservedTypes?: ReadonlySet<string>): void {
|
||||
this.watchSleepingMail()
|
||||
this.sleptMailboxWake.request(mailbox, reservedTypes)
|
||||
}
|
||||
|
||||
retrySleptMailboxWakes(): void {
|
||||
this.watchSleepingMail()
|
||||
this.sleepingMailPublication = ''
|
||||
this.publishSleepingMail()
|
||||
this.getExistingOrchestrationDb()
|
||||
this.scheduleRestoredMessageRepoints()
|
||||
this.publishSleepingMail()
|
||||
}
|
||||
|
||||
private readonly settledMailWakeIds = new Set<string>()
|
||||
private sleepingMailPublicationDirty = false
|
||||
private globalSleepingMailInvalidation = false
|
||||
private readonly sleepingMailAddresses = new Set<string>()
|
||||
private sleepingMailPublicationQueued = false
|
||||
private sleepingMailPublication = ''
|
||||
|
||||
protected publishSleepingMail(): void {
|
||||
protected publishSleepingMail(mailboxes?: Iterable<string>): void {
|
||||
if (mailboxes) {
|
||||
for (const mailbox of mailboxes) {
|
||||
this.sleepingMailAddresses.add(mailbox)
|
||||
}
|
||||
} else {
|
||||
this.globalSleepingMailInvalidation = true
|
||||
}
|
||||
if (this.sleepingMailPublicationQueued) {
|
||||
this.sleepingMailPublicationDirty = true
|
||||
return
|
||||
@@ -251,35 +239,43 @@ export abstract class RuntimeSleptMailboxWakes extends RuntimeMailWakeAdmission
|
||||
if (!this._orchestrationDb) {
|
||||
return
|
||||
}
|
||||
const panes = new Map<string, string>()
|
||||
for (const mailbox of this._orchestrationDb?.getUndeliveredUnreadMailboxHandles() ?? []) {
|
||||
const batch = this.sleptMailboxWake.batch(mailbox)
|
||||
if (batch) {
|
||||
panes.set(batch.record.paneKey, batch.record.worktreeId)
|
||||
const global = this.globalSleepingMailInvalidation
|
||||
const affected = new Set(this.changedSleepingPanes)
|
||||
this.changedSleepingPanes.clear()
|
||||
if (global && (this.sleepingRecords.size || this.sleptMailboxWake.hasTrackedMail)) {
|
||||
this.sleptMailboxWake.reconcile()
|
||||
for (const pane of this.custodyRecords.keys()) {
|
||||
affected.add(pane)
|
||||
}
|
||||
} else {
|
||||
for (const mailbox of this.sleepingMailAddresses) {
|
||||
for (const pane of this.sleptMailboxWake.refresh(mailbox)) {
|
||||
affected.add(pane)
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const record of this.allMailWakeRecords()) {
|
||||
if (record.mailWake && !panes.has(record.paneKey)) {
|
||||
this.globalSleepingMailInvalidation = false
|
||||
this.sleepingMailAddresses.clear()
|
||||
for (const pane of affected) {
|
||||
const record = this.custodyRecords.get(pane)
|
||||
if (record && !this.sleptMailboxWake.owesPane(pane)) {
|
||||
await this.editMailWakeRecord(record, withoutMailWake(record))
|
||||
}
|
||||
}
|
||||
if (global) {
|
||||
for (const pane of this.sleepingRecords.keys()) {
|
||||
for (const mailbox of this.sleptMailboxWake.mailboxesForPane(pane)) {
|
||||
this.sleptMailboxWake.request(mailbox)
|
||||
}
|
||||
}
|
||||
}
|
||||
const settledIds = [...this.settledMailWakeIds]
|
||||
this.settledMailWakeIds.clear()
|
||||
const ids = [...new Set(panes.values())].sort()
|
||||
const key = JSON.stringify([
|
||||
ids,
|
||||
this.allMailWakeRecords()
|
||||
.filter((record) => record.mailWake)
|
||||
.map((record) => record.mailWake!.id)
|
||||
.sort()
|
||||
])
|
||||
const ids = this.sleptMailboxWake.heldWorktrees()
|
||||
const key = JSON.stringify(ids)
|
||||
if (key !== this.sleepingMailPublication || settledIds.length) {
|
||||
this.sleepingMailPublication = key
|
||||
if (settledIds.length) {
|
||||
this.notifier?.sleepingMailChanged?.(ids, settledIds)
|
||||
} else {
|
||||
this.notifier?.sleepingMailChanged?.(ids)
|
||||
}
|
||||
this.notifier?.sleepingMailChanged?.(ids, ...(settledIds.length ? [settledIds] : []))
|
||||
}
|
||||
} catch {
|
||||
/* The runtime can close its database before a queued publication. */
|
||||
@@ -287,26 +283,9 @@ export abstract class RuntimeSleptMailboxWakes extends RuntimeMailWakeAdmission
|
||||
this.sleepingMailPublicationQueued = false
|
||||
if (this.sleepingMailPublicationDirty) {
|
||||
this.sleepingMailPublicationDirty = false
|
||||
this.publishSleepingMail()
|
||||
this.publishSleepingMail([])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
readSleepingMailHold(mailbox: string): MailboxDeliveryHold | null {
|
||||
const record = this.sleptMailboxWake.batch(mailbox)?.record
|
||||
if (!record || !mailWakeObligation(record)) {
|
||||
return null
|
||||
}
|
||||
return {
|
||||
kind: 'asleep',
|
||||
agentName: TUI_AGENT_DISPLAY_NAMES[record.agent],
|
||||
openTab: record.restoreOnTabOpenOnly === true,
|
||||
failed: this.sleptMailboxWake.failedToWake(record),
|
||||
liveShell: mailWakeObligation(record) === 'retained',
|
||||
workspaceName: this.resolvedWorktrees
|
||||
.peek()
|
||||
?.worktrees.find((row) => row.id === record.worktreeId)?.displayName
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
import type { ExecutionHostId } from '../../../../../shared/execution-host'
|
||||
import type { WorkspaceSessionState } from '../../../../../shared/workspace-session/workspace-session-state-types'
|
||||
import { OrchestrationMailboxPointerDelivery } from '../terminal-pointer/mailbox-pointer-delivery'
|
||||
import { MailWakeInvocations } from './mail-wake-invocations'
|
||||
import { admitsMailWakeRetry } from './mail-wake-retry-admission'
|
||||
import { retainMailWakeAttempt } from '../../../session-lifecycle/projection/retained-mail-wake-attempt'
|
||||
import {
|
||||
retainMailWakeAttempt,
|
||||
restoreUndispatchedMailWake
|
||||
} from '../../../session-lifecycle/projection/retained-mail-wake-attempt'
|
||||
import { RestoredPaneStarter } from '../../../terminal/recovery/restored-pane-start'
|
||||
import type { HostProcessInspection } from '../../../../../shared/terminal/pty/lifecycle/terminal-process-inspection'
|
||||
import type { RuntimeTerminalCreate } from '../../../../../shared/runtime-terminal-create-contract'
|
||||
@@ -185,7 +189,10 @@ it('holds manual Sleep through the sender receipt and keeps its bell until mail
|
||||
runtime.deliverPendingMessagesForHandle(mailbox)
|
||||
await settled()
|
||||
expect(resume).not.toHaveBeenCalled()
|
||||
expect(runtime.readSleepingMailHold(mailbox)).toMatchObject({ kind: 'asleep', openTab: true })
|
||||
expect(await runtime.readSleepingMailHold(mailbox)).toMatchObject({
|
||||
kind: 'asleep',
|
||||
openTab: true
|
||||
})
|
||||
expect(await sendDeliveryHoldWarning(runtime, db, mailbox)).toMatchObject({
|
||||
code: 'recipient_delivery_held',
|
||||
recipient: mailbox
|
||||
@@ -263,6 +270,7 @@ it('retains a failed resume with a live shell, and restarts only a proven invoca
|
||||
let drivingInput: number | null = null
|
||||
let occupied = false
|
||||
let owned = true
|
||||
let shellForeground = true
|
||||
const start = vi.fn(async () => ({ id: 'replacement' }))
|
||||
const starter = new RestoredPaneStarter({
|
||||
boundPtyId: () => 'pty-shell',
|
||||
@@ -279,7 +287,7 @@ it('retains a failed resume with a live shell, and restarts only a proven invoca
|
||||
onInspect()
|
||||
return inspection
|
||||
},
|
||||
shellInFront: async () => true
|
||||
shellInFront: async () => shellForeground
|
||||
})
|
||||
})
|
||||
const pane = {
|
||||
@@ -320,18 +328,7 @@ it('retains a failed resume with a live shell, and restarts only a proven invoca
|
||||
inspection = {
|
||||
foregroundProcess: 'zsh',
|
||||
hasChildProcesses: true,
|
||||
childProcessEvidence: 'children',
|
||||
foregroundObservation: {
|
||||
verdict: 'observed',
|
||||
ptyId: 'pty-shell',
|
||||
ptyIncarnationId: 'inc-shell',
|
||||
foregroundKind: 'shell',
|
||||
source: 'pty-master',
|
||||
processName: 'zsh',
|
||||
authorityGeneration: 'host-1',
|
||||
observationEpoch: 1,
|
||||
capturedAgeMs: 0
|
||||
}
|
||||
childProcessEvidence: 'children'
|
||||
}
|
||||
expect(await resumeFromMail()).toBeNull()
|
||||
inspection = { ...inspection, hasChildProcesses: false, childProcessEvidence: 'no-children' }
|
||||
@@ -357,6 +354,9 @@ it('retains a failed resume with a live shell, and restarts only a proven invoca
|
||||
expect(start).not.toHaveBeenCalled()
|
||||
session.sleepingAgentSessionsByPaneKey[paneKey] = retained
|
||||
onInspect = () => {}
|
||||
shellForeground = false
|
||||
expect(await resumeFromMail()).toBeNull()
|
||||
shellForeground = true
|
||||
expect(await resumeFromMail()).toEqual({ id: 'replacement' })
|
||||
expect(start).toHaveBeenCalledTimes(1)
|
||||
expect(write).not.toHaveBeenCalled()
|
||||
@@ -413,3 +413,263 @@ it('retains a failed resume with a live shell, and restarts only a proven invoca
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
|
||||
it('suppresses a definitively confirmed late wake only for its obligation and releases consumed mailbox retention', async () => {
|
||||
const db = new OrchestrationDb(':memory:')
|
||||
let sleeping: SleepingAgentSessionRecord | null = record
|
||||
const wake = vi.fn(async () => false)
|
||||
const delivery = new SleptMailboxWake({
|
||||
db: () => db,
|
||||
record: () => sleeping,
|
||||
waiters: () => undefined,
|
||||
prepare: async () => {},
|
||||
wake
|
||||
})
|
||||
try {
|
||||
const run = db.createRun({
|
||||
objective: 'late confirmation',
|
||||
coordinatorHandle: 'term_old',
|
||||
coordinatorPaneKey: paneKey
|
||||
})
|
||||
const mailbox = `run:${run.id}`
|
||||
const message = db.insertMessage({
|
||||
runId: run.id,
|
||||
from: 'worker',
|
||||
to: mailbox,
|
||||
subject: 'done'
|
||||
})
|
||||
delivery.request(mailbox)
|
||||
await settled()
|
||||
const retained: SleepingAgentSessionRecord = {
|
||||
...record,
|
||||
origin: 'quit',
|
||||
restoreOnTabOpenOnly: true,
|
||||
capturedAt: 200,
|
||||
mailWake: { id: 'obligation', capturedAt: 1, startedAt: 100 }
|
||||
}
|
||||
sleeping = retained
|
||||
delivery.confirmed({ ...retained, mailWake: { ...retained.mailWake!, capturedAt: 2 } })
|
||||
expect(delivery.failedToWake(retained)).toBe(true)
|
||||
delivery.confirmed(retained)
|
||||
delivery.request(mailbox)
|
||||
await settled()
|
||||
expect(wake).toHaveBeenCalledTimes(1)
|
||||
db.markAsRead([message.id])
|
||||
delivery.refresh(mailbox)
|
||||
expect(Reflect.get(delivery, 'succeeded').size).toBe(0)
|
||||
expect(Reflect.get(delivery, 'failed').size).toBe(0)
|
||||
expect(Reflect.get(delivery, 'attempted').size).toBe(0)
|
||||
|
||||
for (let i = 0; i < 20; i++) {
|
||||
sleeping = record
|
||||
wake.mockResolvedValueOnce(i % 2 === 0)
|
||||
const nextRun = db.createRun({
|
||||
objective: 'retention',
|
||||
coordinatorHandle: 'term_old',
|
||||
coordinatorPaneKey: paneKey
|
||||
})
|
||||
const handle = `run:${nextRun.id}`
|
||||
const row = db.insertMessage({
|
||||
runId: nextRun.id,
|
||||
from: 'worker',
|
||||
to: handle,
|
||||
subject: 'done'
|
||||
})
|
||||
delivery.request(handle)
|
||||
await settled()
|
||||
db.markAsRead([row.id])
|
||||
delivery.refresh(handle)
|
||||
}
|
||||
expect(Reflect.get(delivery, 'succeeded').size).toBe(0)
|
||||
expect(Reflect.get(delivery, 'failed').size).toBe(0)
|
||||
expect(Reflect.get(delivery, 'attempted').size).toBe(0)
|
||||
expect(delivery.heldWorktrees()).toEqual([])
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
})
|
||||
|
||||
it.each([10, 100])(
|
||||
'keeps status visits and addressed mail queries bounded with %i saved sleepers',
|
||||
async (count) => {
|
||||
const db = new OrchestrationDb(':memory:')
|
||||
let visits = 0
|
||||
let statusChanged: (key: string) => void = () => {}
|
||||
let sessionWritten: (host: ExecutionHostId | null) => void = () => {}
|
||||
const records: Record<string, SleepingAgentSessionRecord> = {}
|
||||
const mailboxes: string[] = []
|
||||
for (let i = 0; i < count; i++) {
|
||||
const key = `tab-${i}:11111111-1111-4111-8111-111111111111`
|
||||
records[key] = new Proxy(
|
||||
{
|
||||
...record,
|
||||
paneKey: key,
|
||||
tabId: `tab-${i}`,
|
||||
origin: 'quit',
|
||||
restoreOnTabOpenOnly: true,
|
||||
mailWake: {
|
||||
id: `custody-${i}`,
|
||||
capturedAt: 1,
|
||||
startedAt: 100,
|
||||
attempt: {
|
||||
ptyId: `pty-${i}`,
|
||||
incarnationId: `inc-${i}`,
|
||||
launchToken: `token-${i}`,
|
||||
startedAt: 100
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
get(target, property, receiver) {
|
||||
visits++
|
||||
return Reflect.get(target, property, receiver)
|
||||
}
|
||||
}
|
||||
)
|
||||
const run = db.createRun({
|
||||
objective: 'scale',
|
||||
coordinatorHandle: `term_${i}`,
|
||||
coordinatorPaneKey: key
|
||||
})
|
||||
const mailbox = `run:${run.id}`
|
||||
mailboxes.push(mailbox)
|
||||
for (let j = 0; j < 5; j++) {
|
||||
db.insertMessage({ runId: run.id, from: 'worker', to: mailbox, subject: 'done' })
|
||||
}
|
||||
}
|
||||
let session = { ...getDefaultWorkspaceSession(), sleepingAgentSessionsByPaneKey: records }
|
||||
const readSession = vi.fn(() => session)
|
||||
const runtime = new OrcaRuntimeService(
|
||||
partialRuntimeStore({
|
||||
getWorkspaceSession: readSession,
|
||||
getWorkspaceSessionHostIds: () => ['local'],
|
||||
onWorkspaceSessionWritten: (listener) => {
|
||||
sessionWritten = listener
|
||||
return () => {}
|
||||
}
|
||||
}),
|
||||
undefined,
|
||||
{
|
||||
subscribeAgentStatusRowChanges: (listener) => {
|
||||
statusChanged = listener
|
||||
return () => {}
|
||||
}
|
||||
}
|
||||
)
|
||||
try {
|
||||
runtime.setOrchestrationDb(db)
|
||||
runtime.retrySleptMailboxWakes()
|
||||
await settled()
|
||||
visits = 0
|
||||
readSession.mockClear()
|
||||
statusChanged('unrelated:11111111-1111-4111-8111-111111111111')
|
||||
expect(visits).toBe(0)
|
||||
statusChanged('tab-0:11111111-1111-4111-8111-111111111111')
|
||||
// A status event needs only this pane's attempt and PTY identity; no session inventory.
|
||||
expect(visits).toBeLessThanOrEqual(4)
|
||||
expect(readSession).not.toHaveBeenCalled()
|
||||
const inventory = vi.spyOn(db, 'getUndeliveredUnreadMailboxHandles')
|
||||
const fullRows = vi.spyOn(db, 'getUndeliveredUnreadMessages')
|
||||
const newest = vi.spyOn(db, 'getUndeliveredUnreadSequence')
|
||||
runtime.crewChanges.notify(
|
||||
Object.assign(new Set(['traffic'] as const), { trafficAddresses: new Set([mailboxes[0]!]) })
|
||||
)
|
||||
await settled()
|
||||
expect(inventory).not.toHaveBeenCalled()
|
||||
expect(fullRows).not.toHaveBeenCalled()
|
||||
// Refresh and the custody check each need one scalar newest-sequence row for the affected mailbox.
|
||||
expect(newest.mock.calls.length).toBeLessThanOrEqual(3)
|
||||
expect(
|
||||
newest.mock.results.filter((result) => result.value !== undefined).length
|
||||
).toBeLessThanOrEqual(3)
|
||||
expect(newest).toHaveBeenCalled()
|
||||
session = { ...session, sleepingAgentSessionsByPaneKey: {} }
|
||||
sessionWritten('local')
|
||||
await settled()
|
||||
visits = 0
|
||||
readSession.mockClear()
|
||||
statusChanged('tab-0:11111111-1111-4111-8111-111111111111')
|
||||
expect(visits).toBe(0)
|
||||
expect(readSession).not.toHaveBeenCalled()
|
||||
} finally {
|
||||
db.close()
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
it('holds unknown dispatch custody but retries a positively undispatched wake on the record availability edge', async () => {
|
||||
const db = new OrchestrationDb(':memory:')
|
||||
const retained: SleepingAgentSessionRecord = {
|
||||
...record,
|
||||
worktreeId: 'repo::/workspace',
|
||||
origin: 'quit',
|
||||
restoreOnTabOpenOnly: true,
|
||||
mailWake: { id: 'not-dispatched', capturedAt: 1, startedAt: 100 }
|
||||
}
|
||||
let session: WorkspaceSessionState = {
|
||||
...getDefaultWorkspaceSession(),
|
||||
sleepingAgentSessionsByPaneKey: { [paneKey]: retained }
|
||||
}
|
||||
let written: (host: ExecutionHostId | null) => void = () => {}
|
||||
const runtime = new OrcaRuntimeService(
|
||||
partialRuntimeStore({
|
||||
getWorkspaceSession: () => session,
|
||||
getWorkspaceSessionHostIds: () => ['local'],
|
||||
onWorkspaceSessionWritten: (listener) => {
|
||||
written = listener
|
||||
return () => {}
|
||||
}
|
||||
})
|
||||
)
|
||||
const resume = vi.fn(
|
||||
async (_workspace: string, _pane?: string, prepare?: boolean) => prepare === true
|
||||
)
|
||||
const notifier: RuntimeNotifier = {
|
||||
worktreesChanged: vi.fn(),
|
||||
reposChanged: vi.fn(),
|
||||
activateWorktree: vi.fn(),
|
||||
createTerminal: vi.fn(),
|
||||
splitTerminal: vi.fn(),
|
||||
renameTerminal: vi.fn(),
|
||||
focusTerminal: vi.fn(),
|
||||
closeTerminal: vi.fn(),
|
||||
terminalFitOverrideChanged: vi.fn(),
|
||||
terminalDriverChanged: vi.fn(),
|
||||
resumeSleepingAgents: resume
|
||||
}
|
||||
try {
|
||||
const run = db.createRun({
|
||||
objective: 'refused launch',
|
||||
coordinatorHandle: 'term_old',
|
||||
coordinatorPaneKey: paneKey
|
||||
})
|
||||
const mailbox = `run:${run.id}`
|
||||
db.insertMessage({ runId: run.id, from: 'worker', to: mailbox, subject: 'done' })
|
||||
runtime.setOrchestrationDb(db)
|
||||
runtime.setNotifier(notifier)
|
||||
runtime.syncWindowGraph(1, { tabs: [], leaves: [] })
|
||||
await settled()
|
||||
expect(await runtime.readSleepingMailHold(mailbox)).toMatchObject({
|
||||
resuming: true,
|
||||
liveShell: false
|
||||
})
|
||||
runtime.retrySleptMailboxWakes()
|
||||
await settled()
|
||||
expect(resume.mock.calls.filter((call) => call[2] !== true)).toHaveLength(0)
|
||||
await restoreUndispatchedMailWake(async (edit) => {
|
||||
const next = edit(session)
|
||||
if (next) {
|
||||
session = next
|
||||
written('local')
|
||||
}
|
||||
}, retained)
|
||||
await settled()
|
||||
expect(resume.mock.calls.filter((call) => call[2] !== true)).toEqual([
|
||||
['repo::/workspace', 'tab:11111111-1111-4111-8111-111111111111']
|
||||
])
|
||||
expect(session.sleepingAgentSessionsByPaneKey?.[paneKey]?.mailWake).toBeUndefined()
|
||||
} finally {
|
||||
runtime.setNotifier(null)
|
||||
db.close()
|
||||
}
|
||||
})
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import { POTENTIALLY_LIVE_REMOTE_ATTACHMENT_STATES } from '../../db/federation/remote-attachment-liveness'
|
||||
import type { SleepingAgentSessionRecord } from '../../../../../shared/agent-session/lifecycle/agent-session-resume'
|
||||
import { agentProviderSessionsEqual } from '../../../../../shared/agent-session/lifecycle/agent-session-resume'
|
||||
import { mailWakeObligation } from '../../../../../shared/agent-session/lifecycle/mail-wake-obligation'
|
||||
import type { OrchestrationDb } from '../../db/orchestration-db'
|
||||
import {
|
||||
selectOrchestrationPointerBatch,
|
||||
selectOrchestrationPointerSequence,
|
||||
type OrchestrationMessageWaiter
|
||||
} from '../terminal-pointer/mailbox-pointer-eligibility'
|
||||
|
||||
@@ -45,12 +46,20 @@ export class SleptMailboxWake {
|
||||
private readonly inFlight = new Map<string, number>()
|
||||
private readonly succeeded = new Map<string, { paneKey: string; sequence: number }>()
|
||||
private readonly failed = new Map<string, number>()
|
||||
private readonly owed = new Map<string, { paneKey: string; sequence: number }>()
|
||||
private readonly byPane = new Map<string, Set<string>>()
|
||||
private readonly attempted = new Map<
|
||||
string,
|
||||
{ record: SleepingAgentSessionRecord; sequence: number }
|
||||
>()
|
||||
private readonly held = new Map<string, string>()
|
||||
private readonly workspaceCounts = new Map<string, number>()
|
||||
private active = 0
|
||||
|
||||
constructor(private readonly deps: WakeDeps) {}
|
||||
|
||||
failedToWake(record: SleepingAgentSessionRecord): boolean {
|
||||
return this.failed.get(record.paneKey) === record.capturedAt
|
||||
return this.failed.get(record.paneKey) === (record.mailWake?.capturedAt ?? record.capturedAt)
|
||||
}
|
||||
|
||||
batch(mailbox: string, reservedTypes?: ReadonlySet<string>) {
|
||||
@@ -60,24 +69,150 @@ export class SleptMailboxWake {
|
||||
if (!db || !record || !mailWakeObligation(record)) {
|
||||
return null
|
||||
}
|
||||
const messages = selectOrchestrationPointerBatch({
|
||||
db,
|
||||
mailboxHandle: mailbox,
|
||||
waiters: this.deps.waiters(mailbox),
|
||||
reservedTypes,
|
||||
newestFirst: true
|
||||
}).messages
|
||||
const sequence = messages[0]?.sequence
|
||||
const sequence = this.selection(mailbox, reservedTypes)?.sequence
|
||||
return sequence === undefined ? null : { record, sequence }
|
||||
}
|
||||
|
||||
private selection(mailbox: string, reservedTypes?: ReadonlySet<string>) {
|
||||
const db = this.deps.db()
|
||||
const paneKey = db && mailboxOwnerPane(db, mailbox)
|
||||
if (!db || !paneKey) {
|
||||
return null
|
||||
}
|
||||
const sequence = selectOrchestrationPointerSequence({
|
||||
db,
|
||||
mailboxHandle: mailbox,
|
||||
waiters: this.deps.waiters(mailbox),
|
||||
reservedTypes
|
||||
})
|
||||
return sequence === undefined ? null : { paneKey, sequence }
|
||||
}
|
||||
|
||||
refresh(mailbox: string): string[] {
|
||||
const before = this.owed.get(mailbox)
|
||||
const next = this.selection(mailbox)
|
||||
if (before) {
|
||||
const handles = this.byPane.get(before.paneKey)
|
||||
handles?.delete(mailbox)
|
||||
if (!handles?.size) {
|
||||
this.byPane.delete(before.paneKey)
|
||||
}
|
||||
}
|
||||
this.owed.delete(mailbox)
|
||||
if (next) {
|
||||
this.owed.set(mailbox, next)
|
||||
const handles = this.byPane.get(next.paneKey) ?? new Set<string>()
|
||||
handles.add(mailbox)
|
||||
this.byPane.set(next.paneKey, handles)
|
||||
}
|
||||
const success = this.succeeded.get(mailbox)
|
||||
if (!next || success?.paneKey !== next.paneKey || success.sequence !== next.sequence) {
|
||||
this.succeeded.delete(mailbox)
|
||||
}
|
||||
const attempted = this.attempted.get(mailbox)
|
||||
if (!next || (attempted && attempted.record.paneKey !== next.paneKey)) {
|
||||
this.attempted.delete(mailbox)
|
||||
}
|
||||
if (before && !this.byPane.has(before.paneKey)) {
|
||||
this.failed.delete(before.paneKey)
|
||||
}
|
||||
if (before) {
|
||||
this.refreshPane(before.paneKey)
|
||||
}
|
||||
if (next) {
|
||||
this.refreshPane(next.paneKey)
|
||||
}
|
||||
return [...new Set([...(before ? [before.paneKey] : []), ...(next ? [next.paneKey] : [])])]
|
||||
}
|
||||
|
||||
get hasTrackedMail(): boolean {
|
||||
return this.owed.size > 0 || this.attempted.size > 0
|
||||
}
|
||||
|
||||
reconcile(): void {
|
||||
const handles = new Set(this.deps.db()?.getUndeliveredUnreadMailboxHandles() ?? [])
|
||||
for (const handle of this.owed.keys()) {
|
||||
handles.add(handle)
|
||||
}
|
||||
for (const handle of handles) {
|
||||
this.refresh(handle)
|
||||
}
|
||||
}
|
||||
|
||||
mailboxesForPane(paneKey: string): readonly string[] {
|
||||
return [...(this.byPane.get(paneKey) ?? [])]
|
||||
}
|
||||
|
||||
owesPane(paneKey: string): boolean {
|
||||
return (this.deps.db()?.getUndeliveredUnreadMailboxHandles() ?? []).some(
|
||||
(mailbox) => this.batch(mailbox)?.record.paneKey === paneKey
|
||||
)
|
||||
for (const mailbox of this.mailboxesForPane(paneKey)) {
|
||||
this.refresh(mailbox)
|
||||
if (this.batch(mailbox)?.record.paneKey === paneKey) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
refreshPane(paneKey: string): void {
|
||||
const previous = this.held.get(paneKey)
|
||||
const record = this.deps.record(paneKey)
|
||||
const next =
|
||||
this.byPane.has(paneKey) && mailWakeObligation(record) ? record?.worktreeId : undefined
|
||||
if (previous === next) {
|
||||
return
|
||||
}
|
||||
if (previous) {
|
||||
const count = (this.workspaceCounts.get(previous) ?? 1) - 1
|
||||
if (count) {
|
||||
this.workspaceCounts.set(previous, count)
|
||||
} else {
|
||||
this.workspaceCounts.delete(previous)
|
||||
}
|
||||
}
|
||||
this.held.delete(paneKey)
|
||||
if (next) {
|
||||
this.held.set(paneKey, next)
|
||||
this.workspaceCounts.set(next, (this.workspaceCounts.get(next) ?? 0) + 1)
|
||||
}
|
||||
if (!record) {
|
||||
this.failed.delete(paneKey)
|
||||
}
|
||||
}
|
||||
|
||||
heldWorktrees(): string[] {
|
||||
return [...this.workspaceCounts.keys()].sort()
|
||||
}
|
||||
|
||||
confirmed(record: SleepingAgentSessionRecord): void {
|
||||
for (const mailbox of this.mailboxesForPane(record.paneKey)) {
|
||||
const attempt = this.attempted.get(mailbox)
|
||||
if (
|
||||
!attempt ||
|
||||
attempt.record.worktreeId !== record.worktreeId ||
|
||||
attempt.record.agent !== record.agent ||
|
||||
(attempt.record.connectionId ?? null) !== (record.connectionId ?? null) ||
|
||||
(attempt.record.mailWake?.capturedAt ?? attempt.record.capturedAt) !==
|
||||
record.mailWake?.capturedAt ||
|
||||
(attempt.record.mailWake && attempt.record.mailWake.id !== record.mailWake?.id) ||
|
||||
!agentProviderSessionsEqual(
|
||||
record.agent,
|
||||
record.providerSession,
|
||||
attempt.record.providerSession
|
||||
)
|
||||
) {
|
||||
continue
|
||||
}
|
||||
this.refresh(mailbox)
|
||||
if (this.owed.get(mailbox)?.paneKey === record.paneKey) {
|
||||
this.succeeded.set(mailbox, { paneKey: record.paneKey, sequence: attempt.sequence })
|
||||
this.failed.delete(record.paneKey)
|
||||
}
|
||||
this.attempted.delete(mailbox)
|
||||
}
|
||||
}
|
||||
|
||||
request(mailbox: string, reservedTypes?: ReadonlySet<string>): void {
|
||||
this.refresh(mailbox)
|
||||
const batch = this.batch(mailbox, reservedTypes)
|
||||
if (
|
||||
!batch ||
|
||||
@@ -125,6 +260,7 @@ export class SleptMailboxWake {
|
||||
) {
|
||||
return
|
||||
}
|
||||
this.failed.delete(before.record.paneKey)
|
||||
await this.deps.prepare(before.record)
|
||||
const current = this.batch(mailbox, reservedTypes)
|
||||
if (
|
||||
@@ -134,11 +270,23 @@ export class SleptMailboxWake {
|
||||
) {
|
||||
return
|
||||
}
|
||||
if (await this.deps.wake(current.record).catch(() => false)) {
|
||||
this.attempted.set(mailbox, { record: current.record, sequence: current.sequence })
|
||||
const succeeded = await this.deps.wake(current.record).catch(() => false)
|
||||
this.refresh(mailbox)
|
||||
if (this.owed.get(mailbox)?.paneKey !== current.record.paneKey) {
|
||||
return
|
||||
}
|
||||
if (succeeded) {
|
||||
if (this.owed.get(mailbox)?.sequence === current.sequence) {
|
||||
this.succeeded.set(mailbox, { paneKey: current.record.paneKey, sequence: current.sequence })
|
||||
}
|
||||
this.failed.delete(current.record.paneKey)
|
||||
this.succeeded.set(mailbox, { paneKey: current.record.paneKey, sequence: current.sequence })
|
||||
} else {
|
||||
this.failed.set(current.record.paneKey, current.record.capturedAt)
|
||||
this.attempted.delete(mailbox)
|
||||
} else if (this.deps.record(current.record.paneKey)) {
|
||||
this.failed.set(
|
||||
current.record.paneKey,
|
||||
current.record.mailWake?.capturedAt ?? current.record.capturedAt
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -235,7 +235,7 @@ export abstract class OrcaRuntimeWithGetPtyRecordForPaneKey extends RuntimeSlept
|
||||
* longer owes. Visible cleanup only: it never holds up the action that called it. `mailboxes`:
|
||||
* only those the action touched; without it, every mailbox (a reset, a store becoming readable). */
|
||||
withdrawStaleChatMail(mailboxes?: Iterable<string>): void {
|
||||
this.publishSleepingMail()
|
||||
this.publishSleepingMail(mailboxes)
|
||||
this.orchestrationStructuredMailboxPointerDelivery.withdrawStale(mailboxes)
|
||||
}
|
||||
|
||||
|
||||
@@ -19,9 +19,13 @@ import {
|
||||
import { resumePendingOrchestrationMailboxPointer } from './mailbox-pointer-resume'
|
||||
import { stageOrchestrationMailboxPointer } from './mailbox-pointer-stage'
|
||||
|
||||
const DEFAULT_POINTER_ENTER_DELAY_MS = 500
|
||||
|
||||
function pointerEnterDelayMs(): number {
|
||||
const configured = Number(process.env.ORCA_E2E_ORCHESTRATION_POINTER_ENTER_DELAY_MS)
|
||||
return Number.isFinite(configured) && configured >= 1 && configured <= 60_000 ? configured : 500
|
||||
return Number.isFinite(configured) && configured >= 1 && configured <= 60_000
|
||||
? configured
|
||||
: DEFAULT_POINTER_ENTER_DELAY_MS
|
||||
}
|
||||
|
||||
export class OrchestrationMailboxPointerDelivery<TWaiter extends OrchestrationMessageWaiter> {
|
||||
|
||||
+30
-8
@@ -40,7 +40,7 @@ export type OrchestrationPointerBatch = {
|
||||
/** Both delivery lanes share this synchronous SQL selection; unfiltered waiters preempt all pointers.
|
||||
* Concrete waiter types are excluded byte-exactly in SQL; no JavaScript post-filter is needed.
|
||||
* An unacked batch defers rather than drops: only mail it does not hold is eligible, once each. */
|
||||
export function selectOrchestrationPointerBatch(input: {
|
||||
type PointerSelection = {
|
||||
db: OrchestrationDb
|
||||
mailboxHandle: string
|
||||
waiters: ReadonlySet<OrchestrationMessageWaiter> | undefined
|
||||
@@ -48,22 +48,44 @@ export function selectOrchestrationPointerBatch(input: {
|
||||
/** A terminal's own mailbox leaves rows a Run owns undelivered for routing to move. */
|
||||
excludeRunOwnedDirect?: boolean
|
||||
newestFirst?: boolean
|
||||
}): OrchestrationPointerBatch {
|
||||
if (hasUnfilteredOrchestrationWaiter(input.waiters)) {
|
||||
return { messages: [], heldDeliveryId: undefined }
|
||||
}
|
||||
}
|
||||
|
||||
function pointerExclusions(input: PointerSelection, excludeIds?: readonly string[]) {
|
||||
const excludedTypes = new Set(input.reservedTypes)
|
||||
for (const waiter of input.waiters ?? []) {
|
||||
for (const type of waiter.typeFilter ?? []) {
|
||||
excludedTypes.add(type)
|
||||
}
|
||||
}
|
||||
return {
|
||||
excludeTypes: [...excludedTypes],
|
||||
excludeIds,
|
||||
excludeRunOwnedDirect: input.excludeRunOwnedDirect
|
||||
}
|
||||
}
|
||||
|
||||
export function selectOrchestrationPointerSequence(input: PointerSelection): number | undefined {
|
||||
return hasUnfilteredOrchestrationWaiter(input.waiters)
|
||||
? undefined
|
||||
: input.db.getUndeliveredUnreadSequence(
|
||||
input.mailboxHandle,
|
||||
pointerExclusions(
|
||||
input,
|
||||
input.db.getOutstandingMailboxDelivery?.(input.mailboxHandle)?.messageIds
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
export function selectOrchestrationPointerBatch(
|
||||
input: PointerSelection
|
||||
): OrchestrationPointerBatch {
|
||||
if (hasUnfilteredOrchestrationWaiter(input.waiters)) {
|
||||
return { messages: [], heldDeliveryId: undefined }
|
||||
}
|
||||
const held = input.db.getOutstandingMailboxDelivery?.(input.mailboxHandle)
|
||||
return {
|
||||
messages: input.db.getUndeliveredUnreadMessages(input.mailboxHandle, undefined, {
|
||||
excludeTypes: [...excludedTypes],
|
||||
excludeIds: held?.messageIds,
|
||||
excludeRunOwnedDirect: input.excludeRunOwnedDirect,
|
||||
...pointerExclusions(input, held?.messageIds),
|
||||
newestFirst: input.newestFirst,
|
||||
limit: ORCHESTRATION_DELIVERY_BATCH_LIMIT
|
||||
}),
|
||||
|
||||
@@ -31,7 +31,7 @@ async function readDeliveryHoldWarning(
|
||||
recipient: string
|
||||
): Promise<SendRecipientWarning | null> {
|
||||
const hold =
|
||||
runtime.readSleepingMailHold?.(recipient) ??
|
||||
(await runtime.readSleepingMailHold?.(recipient)) ??
|
||||
(await readMailboxDeliveryHold(recipient, {
|
||||
chatSessionFor: (mailbox) => {
|
||||
const custody = structuredMailboxCustody(mailbox, db)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { storedPaneBinding } from '../../terminal/recovery/restored-pane-binding'
|
||||
import { retainMailWakeAttempt } from './retained-mail-wake-attempt'
|
||||
import { retainMailWakeAttempt, restoreUndispatchedMailWake } from './retained-mail-wake-attempt'
|
||||
// Restore terminal-less saved panes for pending-row activation and workspace wakes; main-owned layouts also handle triggers outside desktop restore.
|
||||
|
||||
import type { OrcaRuntimeWithAgentSessionResumeOwnership as AgentSessionResume } from './agent-session-resume-ownership'
|
||||
@@ -200,6 +200,7 @@ export abstract class OrcaRuntimeWithRestoredPaneStart extends OrcaRuntimeWithBu
|
||||
const resumeStartup = await this.resolvePaneOwnedResumeStartup(worktreeId, tab)
|
||||
// After a claimed-record resume starts, mark that record live.
|
||||
const paneKey = `${tab.parentTabId}:${tab.leafId}`
|
||||
const hostId = this.getWorkspaceSessionHostIdForWorktree(worktreeId)
|
||||
const savedRecord =
|
||||
this.getWorkspaceSessionForWorktree(worktreeId)?.sleepingAgentSessionsByPaneKey?.[paneKey]
|
||||
const resumedRecord =
|
||||
@@ -272,11 +273,7 @@ export abstract class OrcaRuntimeWithRestoredPaneStart extends OrcaRuntimeWithBu
|
||||
onPtyCreated: (ptyId: string, result: RuntimeTerminalCreate) => {
|
||||
if (resumedRecord?.mailWake) {
|
||||
mailWakeWrite = retainMailWakeAttempt(
|
||||
(edit) =>
|
||||
this.commitWorkspaceSessionEdit(
|
||||
this.getWorkspaceSessionHostIdForWorktree(worktreeId),
|
||||
edit
|
||||
),
|
||||
(edit) => this.commitWorkspaceSessionEdit(hostId, edit),
|
||||
resumedRecord,
|
||||
ptyId,
|
||||
result
|
||||
@@ -312,7 +309,6 @@ export abstract class OrcaRuntimeWithRestoredPaneStart extends OrcaRuntimeWithBu
|
||||
await mailWakeWrite
|
||||
// A running resumed agent's record must not remain asleep.
|
||||
if (created.tab.ptyId && resumedRecord) {
|
||||
const hostId = this.getWorkspaceSessionHostIdForWorktree(worktreeId)
|
||||
void markResumedRecordLive(
|
||||
(edit) => this.commitWorkspaceSessionEdit(hostId, edit),
|
||||
paneKey,
|
||||
@@ -322,6 +318,12 @@ export abstract class OrcaRuntimeWithRestoredPaneStart extends OrcaRuntimeWithBu
|
||||
return created
|
||||
} catch (err) {
|
||||
resumeClaim?.settleFailure(dispatched, err)
|
||||
if (!dispatched && resumedRecord) {
|
||||
await restoreUndispatchedMailWake(
|
||||
(edit) => this.commitWorkspaceSessionEdit(hostId, edit),
|
||||
resumedRecord
|
||||
)
|
||||
}
|
||||
if (
|
||||
!(automatic && isWorktreeTerminalsSleepingError(err)) &&
|
||||
sessionId &&
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
import { mailWakeObligation } from '../../../../shared/agent-session/lifecycle/mail-wake-obligation'
|
||||
import {
|
||||
mailWakeObligation,
|
||||
undispatchedMailWakeRecord
|
||||
} from '../../../../shared/agent-session/lifecycle/mail-wake-obligation'
|
||||
import type { SleepingAgentSessionRecord } from '../../../../shared/agent-session/lifecycle/agent-session-resume'
|
||||
import type { RuntimeTerminalCreate } from '../../../../shared/runtime-terminal-create-contract'
|
||||
import type { WorkspaceSessionState } from '../../../../shared/workspace-session/workspace-session-state-types'
|
||||
@@ -46,3 +49,37 @@ export async function retainMailWakeAttempt(
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
export function replaceMailWakeRecord(
|
||||
session: WorkspaceSessionState,
|
||||
record: SleepingAgentSessionRecord,
|
||||
next: SleepingAgentSessionRecord
|
||||
): WorkspaceSessionState | null {
|
||||
const current = session.sleepingAgentSessionsByPaneKey?.[record.paneKey]
|
||||
if (
|
||||
current?.mailWake?.id !== record.mailWake?.id ||
|
||||
current?.capturedAt !== record.capturedAt ||
|
||||
current?.origin !== record.origin ||
|
||||
current?.restoreOnTabOpenOnly !== record.restoreOnTabOpenOnly ||
|
||||
current?.mailWake?.attempt?.launchToken !== record.mailWake?.attempt?.launchToken
|
||||
) {
|
||||
return null
|
||||
}
|
||||
return {
|
||||
...session,
|
||||
sleepingAgentSessionsByPaneKey: {
|
||||
...session.sleepingAgentSessionsByPaneKey,
|
||||
[record.paneKey]: next
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function restoreUndispatchedMailWake(
|
||||
commit: Commit,
|
||||
record: SleepingAgentSessionRecord
|
||||
): Promise<void> {
|
||||
const next = undispatchedMailWakeRecord(record)
|
||||
if (next) {
|
||||
await commit((session) => replaceMailWakeRecord(session, record, next))
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -166,7 +166,7 @@ export function installPaneAgentIdentity(session: ConnectPanePtySession): void {
|
||||
const isHibernationWakeRecord =
|
||||
(ptyId && state.suppressedPtyExitIds[ptyId]) ||
|
||||
(wake && wake.record === sleepingRecord?.record && (ptyId === null || wake.ptyId === ptyId))
|
||||
if (sleepingRecord && !isHibernationWakeRecord) {
|
||||
if (sleepingRecord && !sleepingRecord.record.mailWake && !isHibernationWakeRecord) {
|
||||
session.clearSleepingRecordProviderDuplicates(state, sleepingRecord)
|
||||
}
|
||||
session.clearStaleAgentTabTitleOnConfirmedShell()
|
||||
|
||||
+15
@@ -1,3 +1,9 @@
|
||||
import {
|
||||
currentMailWakeStartup,
|
||||
isColdRestoreStartup,
|
||||
restoreUndispatchedMailWake
|
||||
} from '../startup/pane-resume-record'
|
||||
import { useAppStore } from '@/store/index'
|
||||
import {
|
||||
applyTerminalLaunchResult,
|
||||
captureTerminalLaunchContext
|
||||
@@ -140,6 +146,15 @@ export function bindCaptureTransportOutputCallbacks(session: ConnectPanePtySessi
|
||||
}
|
||||
},
|
||||
onSpawnDeferred: (reason: SpawnDeferralReason): void => {
|
||||
if (isCurrent() && isColdRestoreStartup(savedStartup)) {
|
||||
const record = currentMailWakeStartup(
|
||||
savedStartup,
|
||||
useAppStore.getState().sleepingAgentSessionsByPaneKey
|
||||
).sleepingRecordEntry?.record
|
||||
if (record) {
|
||||
restoreUndispatchedMailWake(record)
|
||||
}
|
||||
}
|
||||
if (isCurrent()) {
|
||||
session.pendingSpawnDeferral = reason
|
||||
}
|
||||
|
||||
+20
-3
@@ -1,3 +1,4 @@
|
||||
import type { PtyPaneStartup } from './pty-pane-startup'
|
||||
import { useAppStore } from '@/store/index'
|
||||
import { recordPaneWakeThroughLayoutOwner } from '@/store/terminals/terminal-sleep-layout-owner'
|
||||
import { ORCA_LAYOUT_OWNER_IS_MAIN } from '../../../../../../shared/layout-owner-switch'
|
||||
@@ -8,10 +9,13 @@ import {
|
||||
} from '../../../../../../shared/agent-session/lifecycle/agent-session-resume'
|
||||
import type { ConnectPanePtySession } from '../connect-pane-pty-session'
|
||||
import type { ColdRestoreAgentResumeStartup, PendingStartupCommand } from './fresh-spawn-types'
|
||||
import { mailWakeObligation } from '../../../../../../shared/agent-session/lifecycle/mail-wake-obligation'
|
||||
import {
|
||||
mailWakeObligation,
|
||||
undispatchedMailWakeRecord
|
||||
} from '../../../../../../shared/agent-session/lifecycle/mail-wake-obligation'
|
||||
|
||||
export function isColdRestoreStartup(
|
||||
startup: PendingStartupCommand | null | undefined
|
||||
startup: PendingStartupCommand | PtyPaneStartup | undefined
|
||||
): startup is ColdRestoreAgentResumeStartup {
|
||||
return Boolean(
|
||||
startup &&
|
||||
@@ -114,7 +118,11 @@ export function installPaneResumeRecord(session: ConnectPanePtySession): void {
|
||||
} else {
|
||||
recordPaneWakeThroughLayoutOwner(state, session.deps.worktreeId, session.cacheKey)
|
||||
}
|
||||
session.clearSleepingRecordProviderDuplicates(state, startup.sleepingRecordEntry)
|
||||
session.clearSleepingRecordProviderDuplicates(
|
||||
state,
|
||||
startup.sleepingRecordEntry,
|
||||
session.cacheKey
|
||||
)
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -150,3 +158,12 @@ export function installPaneResumeRecord(session: ConnectPanePtySession): void {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Called only after a definite refusal before dispatch, never after a lost spawn reply. */
|
||||
export function restoreUndispatchedMailWake(record: SleepingAgentSessionRecord): void {
|
||||
const state = useAppStore.getState()
|
||||
const next = undispatchedMailWakeRecord(record)
|
||||
if (next && state.sleepingAgentSessionsByPaneKey[record.paneKey] === record) {
|
||||
state.rememberSleepingAgentSession(next, record)
|
||||
}
|
||||
}
|
||||
|
||||
+8
-1
@@ -47,7 +47,11 @@ export function installSleepingRecordAccess(session: ConnectPanePtySession): voi
|
||||
consumed: { paneKey: string; record: SleepingAgentSessionRecord },
|
||||
keepPaneKey?: string
|
||||
): void => {
|
||||
if (consumed.paneKey !== keepPaneKey) {
|
||||
if (
|
||||
consumed.paneKey !== keepPaneKey &&
|
||||
useAppStore.getState().sleepingAgentSessionsByPaneKey[consumed.paneKey] === consumed.record &&
|
||||
!consumed.record.mailWake
|
||||
) {
|
||||
state.clearSleepingAgentSession(consumed.paneKey)
|
||||
}
|
||||
for (const [paneKey, record] of sleepingAgentRecordIndex.entries(
|
||||
@@ -55,6 +59,9 @@ export function installSleepingRecordAccess(session: ConnectPanePtySession): voi
|
||||
consumed.record.worktreeId
|
||||
)) {
|
||||
if (
|
||||
useAppStore.getState().sleepingAgentSessionsByPaneKey[paneKey] === record &&
|
||||
!record.mailWake &&
|
||||
record.capturedAt <= consumed.record.capturedAt &&
|
||||
paneKey !== consumed.paneKey &&
|
||||
paneKey !== keepPaneKey &&
|
||||
record.worktreeId === consumed.record.worktreeId &&
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { trackAgentPaneSleep } from '@/store/terminals/agent-pane-sleep-in-flight'
|
||||
import { useAppStore } from '@/store/index'
|
||||
import {
|
||||
getEffectiveAgentHibernationIdleMs,
|
||||
@@ -145,15 +146,17 @@ async function hibernatePaneIfStillEligible(
|
||||
try {
|
||||
const state = useAppStore.getState()
|
||||
const owner = resolveAgentHibernationRuntimeOwner(state, worktreeId)
|
||||
await state.shutdownCompletedAgentPaneForHibernation(worktreeId, {
|
||||
paneKey: confirmedCandidate.paneKey,
|
||||
tabId: confirmedCandidate.tabId,
|
||||
leafId: confirmedCandidate.leafId,
|
||||
ptyId: confirmedCandidate.targetPtyIds[0],
|
||||
...(owner.kind === 'runtime'
|
||||
? { expectedRuntimePtyId: confirmedCandidate.expectedRuntimePtyIds[0] }
|
||||
: {})
|
||||
})
|
||||
await trackAgentPaneSleep(confirmedCandidate.paneKey, () =>
|
||||
state.shutdownCompletedAgentPaneForHibernation(worktreeId, {
|
||||
paneKey: confirmedCandidate.paneKey,
|
||||
tabId: confirmedCandidate.tabId,
|
||||
leafId: confirmedCandidate.leafId,
|
||||
ptyId: confirmedCandidate.targetPtyIds[0],
|
||||
...(owner.kind === 'runtime'
|
||||
? { expectedRuntimePtyId: confirmedCandidate.expectedRuntimePtyIds[0] }
|
||||
: {})
|
||||
})
|
||||
)
|
||||
} catch (err) {
|
||||
console.warn('[agent-hibernation] failed to hibernate agent pane:', id, err)
|
||||
} finally {
|
||||
|
||||
@@ -1,6 +1,23 @@
|
||||
import { useAppStore } from '@/store/index'
|
||||
import { installSleepingRecordAccess } from '@/components/terminal-pane/pty-connection/startup/sleeping-record-access'
|
||||
import { installPaneAgentIdentity } from '@/components/terminal-pane/pty-connection/agent-identity/pane-agent-identity'
|
||||
import { bindCaptureTransportOutputCallbacks } from '@/components/terminal-pane/pty-connection/lifecycle/transport-output-callbacks'
|
||||
const foreground = vi.hoisted(() => ({ onShell: (_reason: 'process-exit') => {} }))
|
||||
vi.mock(
|
||||
'@/components/terminal-pane/pty-connection/agent-identity/foreground-process/pane-foreground-agent-tracker',
|
||||
() => ({
|
||||
createPaneForegroundAgentTracker: (options: {
|
||||
onConfirmedShellForeground: typeof foreground.onShell
|
||||
}) => {
|
||||
foreground.onShell = options.onConfirmedShellForeground
|
||||
return {}
|
||||
}
|
||||
})
|
||||
)
|
||||
import {
|
||||
currentMailWakeStartup,
|
||||
paneResumeRecord
|
||||
paneResumeRecord,
|
||||
installPaneResumeRecord
|
||||
} from '@/components/terminal-pane/pty-connection/startup/pane-resume-record'
|
||||
import { hasEmptyReattachRetireEvidence } from '@/components/terminal-pane/pty-connection/reattach/empty-reattach-retire-evidence'
|
||||
import type { ColdRestoreAgentResumeStartup } from '@/components/terminal-pane/pty-connection/startup/fresh-spawn-types'
|
||||
@@ -102,3 +119,130 @@ it('keeps pre-launch custody on its quit bookmark through an already armed wake
|
||||
currentMailWakeStartup(startup, { [original.paneKey]: manual }).sleepingRecordEntry?.record
|
||||
).toBe(original)
|
||||
})
|
||||
|
||||
it('preserves custody through the installed spawn/alias/shell callbacks and restores only a definite prelaunch refusal', () => {
|
||||
const original: SleepingAgentSessionRecord = {
|
||||
paneKey: 'tab:11111111-1111-4111-8111-111111111111',
|
||||
tabId: 'tab',
|
||||
worktreeId: 'workspace',
|
||||
agent: 'codex',
|
||||
providerSession: { key: 'session_id', id: 'conversation' },
|
||||
prompt: '',
|
||||
state: 'done',
|
||||
origin: 'worktree-sleep',
|
||||
capturedAt: 1,
|
||||
updatedAt: 1
|
||||
}
|
||||
const retained: SleepingAgentSessionRecord = {
|
||||
...original,
|
||||
origin: 'quit',
|
||||
restoreOnTabOpenOnly: true,
|
||||
mailWake: { id: 'obligation', capturedAt: 1, startedAt: 100 }
|
||||
}
|
||||
const alias = { ...retained, paneKey: 'tab:22222222-2222-4222-8222-222222222222' }
|
||||
const replacement = {
|
||||
...original,
|
||||
paneKey: 'tab:33333333-3333-4333-8333-333333333333',
|
||||
capturedAt: 2
|
||||
}
|
||||
state.sleepingAgentSessionsByPaneKey = {
|
||||
[original.paneKey]: retained,
|
||||
[alias.paneKey]: alias,
|
||||
[replacement.paneKey]: replacement
|
||||
}
|
||||
Object.assign(state, {
|
||||
agentLaunchConfigByPaneKey: {},
|
||||
agentStatusByPaneKey: {},
|
||||
paneForegroundAgentByPaneKey: {},
|
||||
automaticAgentResumeClaimsByTabId: {},
|
||||
tabsByWorktree: {},
|
||||
suppressedPtyExitIds: {},
|
||||
rememberSleepingAgentSession: (
|
||||
record: SleepingAgentSessionRecord,
|
||||
expected?: SleepingAgentSessionRecord
|
||||
) => {
|
||||
if (expected && state.sleepingAgentSessionsByPaneKey[record.paneKey] !== expected) {
|
||||
return
|
||||
}
|
||||
state.sleepingAgentSessionsByPaneKey = {
|
||||
...state.sleepingAgentSessionsByPaneKey,
|
||||
[record.paneKey]: record
|
||||
}
|
||||
},
|
||||
clearSleepingAgentSession: (key: string) => {
|
||||
const next = { ...state.sleepingAgentSessionsByPaneKey }
|
||||
delete next[key]
|
||||
state.sleepingAgentSessionsByPaneKey = next
|
||||
},
|
||||
clearAgentLaunchConfig: vi.fn()
|
||||
})
|
||||
const transport = { getPtyId: () => 'pty-shell' }
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: the installers exercise the listed pane/store callbacks; unused transport methods are never called.
|
||||
const session = {
|
||||
cacheKey: original.paneKey,
|
||||
paneStartup: null,
|
||||
remotePtyIncarnationId: 'inc',
|
||||
transport,
|
||||
transportStreamGeneration: 0,
|
||||
disposed: false,
|
||||
pane: {
|
||||
id: 1,
|
||||
leafId: '11111111-1111-4111-8111-111111111111',
|
||||
terminal: {
|
||||
parser: { registerOscHandler: () => ({ dispose() {} }) },
|
||||
onRender: () => ({ dispose() {} })
|
||||
}
|
||||
},
|
||||
deps: {
|
||||
tabId: 'tab',
|
||||
worktreeId: 'workspace',
|
||||
isVisibleRef: { current: false },
|
||||
paneTransportsRef: { current: new Map([[1, transport]]) },
|
||||
onAgentExitedRef: { current: vi.fn() }
|
||||
},
|
||||
writeInputModeGround: vi.fn(),
|
||||
createProcessExitState: () => ({ detector: { observe() {} } })
|
||||
} as unknown as ConnectPanePtySession
|
||||
installSleepingRecordAccess(session)
|
||||
installPaneResumeRecord(session)
|
||||
const startup: ColdRestoreAgentResumeStartup = {
|
||||
agent: 'codex',
|
||||
command: 'codex resume conversation',
|
||||
resumeProviderSession: original.providerSession,
|
||||
launchToken: 'invocation',
|
||||
resumeOrigin: 'saved',
|
||||
launchConfig: { agentArgs: '', agentEnv: {} },
|
||||
useLiveEntry: false,
|
||||
hasSleepingRecord: true,
|
||||
sleepingRecordEntry: { paneKey: original.paneKey, record: original }
|
||||
}
|
||||
session.keepResumeRecordAfterColdRestoreSpawn(startup)
|
||||
expect(state.sleepingAgentSessionsByPaneKey[original.paneKey]?.mailWake?.attempt).toEqual({
|
||||
ptyId: 'pty-shell',
|
||||
incarnationId: 'inc',
|
||||
launchToken: 'invocation',
|
||||
startedAt: 100
|
||||
})
|
||||
session.clearSleepingRecordProviderDuplicates(useAppStore.getState(), {
|
||||
paneKey: original.paneKey,
|
||||
record: original
|
||||
})
|
||||
installPaneAgentIdentity(session)
|
||||
foreground.onShell('process-exit')
|
||||
expect(state.sleepingAgentSessionsByPaneKey[original.paneKey]?.mailWake?.id).toBe('obligation')
|
||||
expect(state.sleepingAgentSessionsByPaneKey[alias.paneKey]?.mailWake?.id).toBe('obligation')
|
||||
expect(state.sleepingAgentSessionsByPaneKey[replacement.paneKey]).toBe(replacement)
|
||||
session.commandLifecycle.dispose()
|
||||
|
||||
state.sleepingAgentSessionsByPaneKey = { [retained.paneKey]: retained }
|
||||
bindCaptureTransportOutputCallbacks(session)
|
||||
const capture = session.captureTransportOutputCallbacks(vi.fn(), null, startup)
|
||||
expect(state.sleepingAgentSessionsByPaneKey[retained.paneKey]?.mailWake?.id).toBe('obligation')
|
||||
capture.callbacks.onSpawnDeferred?.('paused')
|
||||
expect(state.sleepingAgentSessionsByPaneKey[retained.paneKey]).toMatchObject({
|
||||
origin: 'worktree-sleep',
|
||||
capturedAt: 1
|
||||
})
|
||||
expect(state.sleepingAgentSessionsByPaneKey[retained.paneKey]?.mailWake).toBeUndefined()
|
||||
session.startupTiming?.finish('replaced')
|
||||
})
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { restoreUndispatchedMailWake } from '@/components/terminal-pane/pty-connection/startup/pane-resume-record'
|
||||
import { patchWorkspaceSessionByHost } from '../../workspace/persistence/workspace-session-host-persistence'
|
||||
import { createNonSecureContextUuid } from '../../../../../shared/non-secure-context-uuid'
|
||||
import { waitForAgentPaneSleep } from '@/store/terminals/agent-pane-sleep-in-flight'
|
||||
@@ -269,14 +270,19 @@ async function wakeSleepingPaneInBackground(
|
||||
}
|
||||
}
|
||||
useAppStore.getState().rememberSleepingAgentSession(captured, record)
|
||||
await patchWorkspaceSessionByHost(
|
||||
window.api.session,
|
||||
{
|
||||
sleepingAgentSessionsByPaneKey: useAppStore.getState().sleepingAgentSessionsByPaneKey
|
||||
},
|
||||
useAppStore.getState()
|
||||
).written
|
||||
await window.api.session.flush()
|
||||
try {
|
||||
await patchWorkspaceSessionByHost(
|
||||
window.api.session,
|
||||
{
|
||||
sleepingAgentSessionsByPaneKey: useAppStore.getState().sleepingAgentSessionsByPaneKey
|
||||
},
|
||||
useAppStore.getState()
|
||||
).written
|
||||
await window.api.session.flush()
|
||||
} catch (error) {
|
||||
restoreUndispatchedMailWake(captured)
|
||||
throw error
|
||||
}
|
||||
if (useAppStore.getState().sleepingAgentSessionsByPaneKey[paneKey] !== captured) {
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import { trackAgentPaneSleep } from './agent-pane-sleep-in-flight'
|
||||
import { replaceRecordEntries, replaceTabBuckets } from '@/store/keyed-record-transitions'
|
||||
import { isRemoteRuntimePtyId } from '../../../../shared/remote-runtime/remote-runtime-pty-id'
|
||||
import { parsePaneKey } from '../../../../shared/workspace-layout/stable-pane-id'
|
||||
@@ -29,239 +28,231 @@ export function createTerminalPaneHibernationActions(
|
||||
get: TerminalStoreGet
|
||||
): Pick<TerminalSlice, 'shutdownCompletedAgentPaneForHibernation'> {
|
||||
return {
|
||||
shutdownCompletedAgentPaneForHibernation: (worktreeId, opts) =>
|
||||
trackAgentPaneSleep(opts.paneKey, async () => {
|
||||
const paneKeys = [opts.paneKey]
|
||||
const expectedRuntimePtyIds = sortedUniquePtyIds(
|
||||
opts.expectedRuntimePtyId ? [opts.expectedRuntimePtyId] : []
|
||||
)
|
||||
const rendererShutdownPtyIds = [opts.ptyId]
|
||||
const state = get()
|
||||
const worktreeOwner = resolveTerminalPtyOwner(state, worktreeId)
|
||||
const teardownOwner = resolveTerminalTeardownOwner(worktreeOwner, [opts.ptyId])
|
||||
// Use PTY owner for twins; losing bindings without an attributable host stop leaves unseen running agents.
|
||||
if (
|
||||
teardownOwner.kind === 'refused' ||
|
||||
(worktreeOwner.kind === 'ambiguous' &&
|
||||
isRemoteRuntimePtyId(opts.ptyId) &&
|
||||
expectedRuntimePtyIds.length === 0)
|
||||
) {
|
||||
console.warn(
|
||||
'[agent-hibernation] refusing pane hibernation: terminal owner is ambiguous',
|
||||
{
|
||||
worktreeId,
|
||||
paneKey: opts.paneKey
|
||||
}
|
||||
)
|
||||
throw new Error('ambiguous_terminal_owner')
|
||||
}
|
||||
const runtimeEnvironmentId = teardownOwner.runtimeEnvironmentId
|
||||
// Why: pane transports emit renderer PTY ids, not raw exact-stop handles; guard only the identity that can deliver an exit callback.
|
||||
const exitGuardPtyIds = [opts.ptyId]
|
||||
const tab = (state.tabsByWorktree[worktreeId] ?? []).find(
|
||||
(candidate) => candidate.id === opts.tabId
|
||||
)
|
||||
const parsed = parsePaneKey(opts.paneKey)
|
||||
const layout = state.terminalLayoutsByTabId[opts.tabId]
|
||||
const liveTabPtyIds = state.ptyIdsByTabId[opts.tabId] ?? []
|
||||
if (
|
||||
!tab ||
|
||||
!parsed ||
|
||||
parsed.tabId !== opts.tabId ||
|
||||
parsed.leafId !== opts.leafId ||
|
||||
layout?.ptyIdsByLeafId?.[opts.leafId] !== opts.ptyId ||
|
||||
(expectedRuntimePtyIds.length === 0 && !liveTabPtyIds.includes(opts.ptyId))
|
||||
) {
|
||||
throw new Error('agent_hibernation_pane_binding_mismatch')
|
||||
}
|
||||
const sleepingAgentSessionRecords = collectSleepingAgentSessionRecordsForWorktree(
|
||||
state,
|
||||
shutdownCompletedAgentPaneForHibernation: async (worktreeId, opts) => {
|
||||
const paneKeys = [opts.paneKey]
|
||||
const expectedRuntimePtyIds = sortedUniquePtyIds(
|
||||
opts.expectedRuntimePtyId ? [opts.expectedRuntimePtyId] : []
|
||||
)
|
||||
const rendererShutdownPtyIds = [opts.ptyId]
|
||||
const state = get()
|
||||
const worktreeOwner = resolveTerminalPtyOwner(state, worktreeId)
|
||||
const teardownOwner = resolveTerminalTeardownOwner(worktreeOwner, [opts.ptyId])
|
||||
// Use PTY owner for twins; losing bindings without an attributable host stop leaves unseen running agents.
|
||||
if (
|
||||
teardownOwner.kind === 'refused' ||
|
||||
(worktreeOwner.kind === 'ambiguous' &&
|
||||
isRemoteRuntimePtyId(opts.ptyId) &&
|
||||
expectedRuntimePtyIds.length === 0)
|
||||
) {
|
||||
console.warn('[agent-hibernation] refusing pane hibernation: terminal owner is ambiguous', {
|
||||
worktreeId,
|
||||
{
|
||||
paneKeys,
|
||||
captureMode: 'completed-agent-hibernation'
|
||||
paneKey: opts.paneKey
|
||||
})
|
||||
throw new Error('ambiguous_terminal_owner')
|
||||
}
|
||||
const runtimeEnvironmentId = teardownOwner.runtimeEnvironmentId
|
||||
// Why: pane transports emit renderer PTY ids, not raw exact-stop handles; guard only the identity that can deliver an exit callback.
|
||||
const exitGuardPtyIds = [opts.ptyId]
|
||||
const tab = (state.tabsByWorktree[worktreeId] ?? []).find(
|
||||
(candidate) => candidate.id === opts.tabId
|
||||
)
|
||||
const parsed = parsePaneKey(opts.paneKey)
|
||||
const layout = state.terminalLayoutsByTabId[opts.tabId]
|
||||
const liveTabPtyIds = state.ptyIdsByTabId[opts.tabId] ?? []
|
||||
if (
|
||||
!tab ||
|
||||
!parsed ||
|
||||
parsed.tabId !== opts.tabId ||
|
||||
parsed.leafId !== opts.leafId ||
|
||||
layout?.ptyIdsByLeafId?.[opts.leafId] !== opts.ptyId ||
|
||||
(expectedRuntimePtyIds.length === 0 && !liveTabPtyIds.includes(opts.ptyId))
|
||||
) {
|
||||
throw new Error('agent_hibernation_pane_binding_mismatch')
|
||||
}
|
||||
const sleepingAgentSessionRecords = collectSleepingAgentSessionRecordsForWorktree(
|
||||
state,
|
||||
worktreeId,
|
||||
{
|
||||
paneKeys,
|
||||
captureMode: 'completed-agent-hibernation'
|
||||
}
|
||||
)
|
||||
const retainedCompletionEvidence = collectHibernatedCompletionEvidenceForWorktree(
|
||||
state,
|
||||
worktreeId,
|
||||
paneKeys
|
||||
)
|
||||
if (!sleepingAgentSessionRecords[opts.paneKey]) {
|
||||
// Why: killing the PTY with no persisted resume record strands the pane unwakeable; abort instead of hibernating unrecoverably.
|
||||
throw new Error('agent_hibernation_capture_missing')
|
||||
}
|
||||
const capture = shutdownBufferCaptures.get(opts.tabId)
|
||||
if (capture) {
|
||||
try {
|
||||
capture({ includeLocalBuffers: false })
|
||||
} catch {
|
||||
// Don't let one tab's capture failure block the pane hibernation.
|
||||
}
|
||||
}
|
||||
// Why: store sleeping records before kill, since pty:exit can arrive first.
|
||||
const sleepingRecordKeys = Object.keys(sleepingAgentSessionRecords)
|
||||
const replacedSleepingRecords: Record<string, (typeof sleepingAgentSessionRecords)[string]> =
|
||||
{}
|
||||
for (const key of sleepingRecordKeys) {
|
||||
const existing = state.sleepingAgentSessionsByPaneKey[key]
|
||||
if (existing) {
|
||||
replacedSleepingRecords[key] = existing
|
||||
}
|
||||
}
|
||||
const rollbackTargetShutdownState = (): void => {
|
||||
set((s) => {
|
||||
const next = { ...s.suppressedPtyExitIds }
|
||||
for (const ptyId of exitGuardPtyIds) {
|
||||
delete next[ptyId]
|
||||
}
|
||||
return {
|
||||
suppressedPtyExitIds: next,
|
||||
sleepingAgentSessionsByPaneKey: replaceRecordEntries(
|
||||
s.sleepingAgentSessionsByPaneKey,
|
||||
sleepingRecordKeys.map((key) => [key, replacedSleepingRecords[key] ?? null] as const)
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
set((s) => ({
|
||||
suppressedPtyExitIds: {
|
||||
...s.suppressedPtyExitIds,
|
||||
...Object.fromEntries(exitGuardPtyIds.map((ptyId) => [ptyId, true] as const))
|
||||
},
|
||||
sleepingAgentSessionsByPaneKey: replaceRecordEntries(
|
||||
s.sleepingAgentSessionsByPaneKey,
|
||||
Object.entries(sleepingAgentSessionRecords)
|
||||
)
|
||||
const retainedCompletionEvidence = collectHibernatedCompletionEvidenceForWorktree(
|
||||
state,
|
||||
worktreeId,
|
||||
paneKeys
|
||||
)
|
||||
if (!sleepingAgentSessionRecords[opts.paneKey]) {
|
||||
// Why: killing the PTY with no persisted resume record strands the pane unwakeable; abort instead of hibernating unrecoverably.
|
||||
throw new Error('agent_hibernation_capture_missing')
|
||||
}))
|
||||
if (expectedRuntimePtyIds.length > 0) {
|
||||
if (!runtimeEnvironmentId) {
|
||||
rollbackTargetShutdownState()
|
||||
throw new Error('missing_runtime_for_exact_terminal_stop')
|
||||
}
|
||||
const capture = shutdownBufferCaptures.get(opts.tabId)
|
||||
if (capture) {
|
||||
try {
|
||||
capture({ includeLocalBuffers: false })
|
||||
} catch {
|
||||
// Don't let one tab's capture failure block the pane hibernation.
|
||||
}
|
||||
let stopResult: {
|
||||
stoppedPtyIds?: string[]
|
||||
livePtyIds?: string[]
|
||||
postStopVerified?: boolean
|
||||
postStopFailure?: string
|
||||
}
|
||||
// Why: store sleeping records before kill, since pty:exit can arrive first.
|
||||
const sleepingRecordKeys = Object.keys(sleepingAgentSessionRecords)
|
||||
const replacedSleepingRecords: Record<
|
||||
string,
|
||||
(typeof sleepingAgentSessionRecords)[string]
|
||||
> = {}
|
||||
for (const key of sleepingRecordKeys) {
|
||||
const existing = state.sleepingAgentSessionsByPaneKey[key]
|
||||
if (existing) {
|
||||
replacedSleepingRecords[key] = existing
|
||||
}
|
||||
}
|
||||
const rollbackTargetShutdownState = (): void => {
|
||||
set((s) => {
|
||||
const next = { ...s.suppressedPtyExitIds }
|
||||
for (const ptyId of exitGuardPtyIds) {
|
||||
delete next[ptyId]
|
||||
}
|
||||
return {
|
||||
suppressedPtyExitIds: next,
|
||||
sleepingAgentSessionsByPaneKey: replaceRecordEntries(
|
||||
s.sleepingAgentSessionsByPaneKey,
|
||||
sleepingRecordKeys.map(
|
||||
(key) => [key, replacedSleepingRecords[key] ?? null] as const
|
||||
)
|
||||
)
|
||||
}
|
||||
})
|
||||
}
|
||||
set((s) => ({
|
||||
suppressedPtyExitIds: {
|
||||
...s.suppressedPtyExitIds,
|
||||
...Object.fromEntries(exitGuardPtyIds.map((ptyId) => [ptyId, true] as const))
|
||||
},
|
||||
sleepingAgentSessionsByPaneKey: replaceRecordEntries(
|
||||
s.sleepingAgentSessionsByPaneKey,
|
||||
Object.entries(sleepingAgentSessionRecords)
|
||||
)
|
||||
}))
|
||||
if (expectedRuntimePtyIds.length > 0) {
|
||||
if (!runtimeEnvironmentId) {
|
||||
rollbackTargetShutdownState()
|
||||
throw new Error('missing_runtime_for_exact_terminal_stop')
|
||||
}
|
||||
let stopResult: {
|
||||
try {
|
||||
stopResult = await callRuntimeRpc<{
|
||||
stoppedPtyIds?: string[]
|
||||
livePtyIds?: string[]
|
||||
postStopVerified?: boolean
|
||||
postStopFailure?: string
|
||||
}
|
||||
try {
|
||||
stopResult = await callRuntimeRpc<{
|
||||
stoppedPtyIds?: string[]
|
||||
livePtyIds?: string[]
|
||||
postStopVerified?: boolean
|
||||
postStopFailure?: string
|
||||
}>(
|
||||
{ kind: 'environment', environmentId: runtimeEnvironmentId },
|
||||
'terminal.stopExact',
|
||||
{
|
||||
worktree: toRuntimeWorktreeSelector(worktreeId),
|
||||
expectedPtyIds: expectedRuntimePtyIds,
|
||||
keepHistory: true,
|
||||
targetOnly: true
|
||||
},
|
||||
{ timeoutMs: 15000 }
|
||||
)
|
||||
} catch (err) {
|
||||
rollbackTargetShutdownState()
|
||||
throw err
|
||||
}
|
||||
const stoppedPtyIds = sortedUniquePtyIds(stopResult.stoppedPtyIds)
|
||||
const livePtyIds = sortedUniquePtyIds(stopResult.livePtyIds)
|
||||
const targetWasLive = expectedRuntimePtyIds.every((ptyId) => livePtyIds.includes(ptyId))
|
||||
if (!equalStringSets(stoppedPtyIds, expectedRuntimePtyIds) || !targetWasLive) {
|
||||
rollbackTargetShutdownState()
|
||||
throw new Error('exact_terminal_stop_mismatch')
|
||||
}
|
||||
if (stopResult.postStopVerified !== true) {
|
||||
rollbackTargetShutdownState()
|
||||
throw new Error(stopResult.postStopFailure ?? 'exact_terminal_stop_unverified')
|
||||
}
|
||||
for (const snapshot of unregisterPtyDataHandlers(rendererShutdownPtyIds) ?? []) {
|
||||
snapshot.commit?.()
|
||||
}
|
||||
} else if (!isRemoteRuntimePtyId(opts.ptyId)) {
|
||||
// Why: pty.kill can flush final data before exit; unregister first so stale handlers can't fire phantom notifications during hibernation.
|
||||
const handlerSnapshots = unregisterPtyDataHandlers(rendererShutdownPtyIds) ?? []
|
||||
try {
|
||||
await window.api.pty.kill(opts.ptyId, { keepHistory: true })
|
||||
} catch (err) {
|
||||
restorePtyDataHandlersAfterFailedShutdown(handlerSnapshots)
|
||||
rollbackTargetShutdownState()
|
||||
throw err
|
||||
}
|
||||
for (const snapshot of handlerSnapshots) {
|
||||
snapshot.commit?.()
|
||||
}
|
||||
}>(
|
||||
{ kind: 'environment', environmentId: runtimeEnvironmentId },
|
||||
'terminal.stopExact',
|
||||
{
|
||||
worktree: toRuntimeWorktreeSelector(worktreeId),
|
||||
expectedPtyIds: expectedRuntimePtyIds,
|
||||
keepHistory: true,
|
||||
targetOnly: true
|
||||
},
|
||||
{ timeoutMs: 15000 }
|
||||
)
|
||||
} catch (err) {
|
||||
rollbackTargetShutdownState()
|
||||
throw err
|
||||
}
|
||||
// Switch point: main records the sleep this window decided, once the stop settled.
|
||||
if (ORCA_LAYOUT_OWNER_IS_MAIN) {
|
||||
recordPaneSleepThroughLayoutOwner(get(), worktreeId, sleepingAgentSessionRecords)
|
||||
const stoppedPtyIds = sortedUniquePtyIds(stopResult.stoppedPtyIds)
|
||||
const livePtyIds = sortedUniquePtyIds(stopResult.livePtyIds)
|
||||
const targetWasLive = expectedRuntimePtyIds.every((ptyId) => livePtyIds.includes(ptyId))
|
||||
if (!equalStringSets(stoppedPtyIds, expectedRuntimePtyIds) || !targetWasLive) {
|
||||
rollbackTargetShutdownState()
|
||||
throw new Error('exact_terminal_stop_mismatch')
|
||||
}
|
||||
settleAttentionRemovedBy(set, get, () =>
|
||||
set((s) => {
|
||||
const existingPtyIds = s.ptyIdsByTabId[opts.tabId] ?? []
|
||||
const shutdownPtyIdSet = new Set(rendererShutdownPtyIds)
|
||||
const remainingPtyIds = existingPtyIds.filter((ptyId) => !shutdownPtyIdSet.has(ptyId))
|
||||
const tabs = s.tabsByWorktree[worktreeId] ?? []
|
||||
const replacements: [string, typeof tabs][] = []
|
||||
const tabIndex = tabs.findIndex((candidate) => candidate.id === opts.tabId)
|
||||
if (tabIndex !== -1) {
|
||||
const nextTabs = [...tabs]
|
||||
nextTabs[tabIndex] = {
|
||||
...nextTabs[tabIndex],
|
||||
ptyId: remainingPtyIds.at(-1) ?? null
|
||||
}
|
||||
replacements.push([worktreeId, nextTabs])
|
||||
if (stopResult.postStopVerified !== true) {
|
||||
rollbackTargetShutdownState()
|
||||
throw new Error(stopResult.postStopFailure ?? 'exact_terminal_stop_unverified')
|
||||
}
|
||||
for (const snapshot of unregisterPtyDataHandlers(rendererShutdownPtyIds) ?? []) {
|
||||
snapshot.commit?.()
|
||||
}
|
||||
} else if (!isRemoteRuntimePtyId(opts.ptyId)) {
|
||||
// Why: pty.kill can flush final data before exit; unregister first so stale handlers can't fire phantom notifications during hibernation.
|
||||
const handlerSnapshots = unregisterPtyDataHandlers(rendererShutdownPtyIds) ?? []
|
||||
try {
|
||||
await window.api.pty.kill(opts.ptyId, { keepHistory: true })
|
||||
} catch (err) {
|
||||
restorePtyDataHandlersAfterFailedShutdown(handlerSnapshots)
|
||||
rollbackTargetShutdownState()
|
||||
throw err
|
||||
}
|
||||
for (const snapshot of handlerSnapshots) {
|
||||
snapshot.commit?.()
|
||||
}
|
||||
}
|
||||
// Switch point: main records the sleep this window decided, once the stop settled.
|
||||
if (ORCA_LAYOUT_OWNER_IS_MAIN) {
|
||||
recordPaneSleepThroughLayoutOwner(get(), worktreeId, sleepingAgentSessionRecords)
|
||||
}
|
||||
settleAttentionRemovedBy(set, get, () =>
|
||||
set((s) => {
|
||||
const existingPtyIds = s.ptyIdsByTabId[opts.tabId] ?? []
|
||||
const shutdownPtyIdSet = new Set(rendererShutdownPtyIds)
|
||||
const remainingPtyIds = existingPtyIds.filter((ptyId) => !shutdownPtyIdSet.has(ptyId))
|
||||
const tabs = s.tabsByWorktree[worktreeId] ?? []
|
||||
const replacements: [string, typeof tabs][] = []
|
||||
const tabIndex = tabs.findIndex((candidate) => candidate.id === opts.tabId)
|
||||
if (tabIndex !== -1) {
|
||||
const nextTabs = [...tabs]
|
||||
nextTabs[tabIndex] = {
|
||||
...nextTabs[tabIndex],
|
||||
ptyId: remainingPtyIds.at(-1) ?? null
|
||||
}
|
||||
const nextCodexRestartNoticeByPtyId = { ...s.codexRestartNoticeByPtyId }
|
||||
for (const ptyId of exitGuardPtyIds) {
|
||||
delete nextCodexRestartNoticeByPtyId[ptyId]
|
||||
}
|
||||
const nextLastKnownRelay =
|
||||
remainingPtyIds.length === 0
|
||||
? replaceRecordEntries<string>(s.lastKnownRelayPtyIdByTabId, [[opts.tabId, null]])
|
||||
: s.lastKnownRelayPtyIdByTabId
|
||||
let nextRuntimePaneTitlesByTabId = s.runtimePaneTitlesByTabId
|
||||
const numericPaneId = Number(opts.leafId)
|
||||
if (
|
||||
Number.isInteger(numericPaneId) &&
|
||||
s.runtimePaneTitlesByTabId[opts.tabId]?.[numericPaneId]
|
||||
) {
|
||||
const nextByPane = { ...s.runtimePaneTitlesByTabId[opts.tabId] }
|
||||
delete nextByPane[numericPaneId]
|
||||
nextRuntimePaneTitlesByTabId = replaceRecordEntries(s.runtimePaneTitlesByTabId, [
|
||||
[opts.tabId, Object.keys(nextByPane).length > 0 ? nextByPane : null]
|
||||
])
|
||||
}
|
||||
const nextLastTerminalInputAtByPaneKey = { ...s.lastTerminalInputAtByPaneKey }
|
||||
delete nextLastTerminalInputAtByPaneKey[opts.paneKey]
|
||||
return {
|
||||
tabsByWorktree: replaceTabBuckets(s.tabsByWorktree, replacements),
|
||||
ptyIdsByTabId: replaceRecordEntries(s.ptyIdsByTabId, [[opts.tabId, remainingPtyIds]]),
|
||||
lastKnownRelayPtyIdByTabId: nextLastKnownRelay,
|
||||
suppressedPtyExitIds: {
|
||||
...s.suppressedPtyExitIds,
|
||||
...Object.fromEntries(exitGuardPtyIds.map((ptyId) => [ptyId, true] as const))
|
||||
},
|
||||
codexRestartNoticeByPtyId: nextCodexRestartNoticeByPtyId,
|
||||
...(nextRuntimePaneTitlesByTabId !== s.runtimePaneTitlesByTabId
|
||||
? {
|
||||
runtimePaneTitlesByTabId: nextRuntimePaneTitlesByTabId
|
||||
}
|
||||
: {}),
|
||||
...removePanesAttention(s, new Set([opts.paneKey])),
|
||||
lastTerminalInputAtByPaneKey: nextLastTerminalInputAtByPaneKey
|
||||
}
|
||||
})
|
||||
)
|
||||
get().dropHibernatedAgentStatusPane(worktreeId, opts.paneKey, {
|
||||
retainedCompletionEvidence
|
||||
replacements.push([worktreeId, nextTabs])
|
||||
}
|
||||
const nextCodexRestartNoticeByPtyId = { ...s.codexRestartNoticeByPtyId }
|
||||
for (const ptyId of exitGuardPtyIds) {
|
||||
delete nextCodexRestartNoticeByPtyId[ptyId]
|
||||
}
|
||||
const nextLastKnownRelay =
|
||||
remainingPtyIds.length === 0
|
||||
? replaceRecordEntries<string>(s.lastKnownRelayPtyIdByTabId, [[opts.tabId, null]])
|
||||
: s.lastKnownRelayPtyIdByTabId
|
||||
let nextRuntimePaneTitlesByTabId = s.runtimePaneTitlesByTabId
|
||||
const numericPaneId = Number(opts.leafId)
|
||||
if (
|
||||
Number.isInteger(numericPaneId) &&
|
||||
s.runtimePaneTitlesByTabId[opts.tabId]?.[numericPaneId]
|
||||
) {
|
||||
const nextByPane = { ...s.runtimePaneTitlesByTabId[opts.tabId] }
|
||||
delete nextByPane[numericPaneId]
|
||||
nextRuntimePaneTitlesByTabId = replaceRecordEntries(s.runtimePaneTitlesByTabId, [
|
||||
[opts.tabId, Object.keys(nextByPane).length > 0 ? nextByPane : null]
|
||||
])
|
||||
}
|
||||
const nextLastTerminalInputAtByPaneKey = { ...s.lastTerminalInputAtByPaneKey }
|
||||
delete nextLastTerminalInputAtByPaneKey[opts.paneKey]
|
||||
return {
|
||||
tabsByWorktree: replaceTabBuckets(s.tabsByWorktree, replacements),
|
||||
ptyIdsByTabId: replaceRecordEntries(s.ptyIdsByTabId, [[opts.tabId, remainingPtyIds]]),
|
||||
lastKnownRelayPtyIdByTabId: nextLastKnownRelay,
|
||||
suppressedPtyExitIds: {
|
||||
...s.suppressedPtyExitIds,
|
||||
...Object.fromEntries(exitGuardPtyIds.map((ptyId) => [ptyId, true] as const))
|
||||
},
|
||||
codexRestartNoticeByPtyId: nextCodexRestartNoticeByPtyId,
|
||||
...(nextRuntimePaneTitlesByTabId !== s.runtimePaneTitlesByTabId
|
||||
? {
|
||||
runtimePaneTitlesByTabId: nextRuntimePaneTitlesByTabId
|
||||
}
|
||||
: {}),
|
||||
...removePanesAttention(s, new Set([opts.paneKey])),
|
||||
lastTerminalInputAtByPaneKey: nextLastTerminalInputAtByPaneKey
|
||||
}
|
||||
})
|
||||
)
|
||||
get().dropHibernatedAgentStatusPane(worktreeId, opts.paneKey, {
|
||||
retainedCompletionEvidence
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,8 +52,18 @@ export function mailWakeEligibility(records: readonly SleepingAgentSessionRecord
|
||||
.map((record) => [
|
||||
record.paneKey,
|
||||
record.mailWake?.capturedAt ?? record.capturedAt,
|
||||
mailWakeObligation(record) === 'manual'
|
||||
mailWakeObligation(record)
|
||||
])
|
||||
.sort()
|
||||
)
|
||||
}
|
||||
|
||||
export function undispatchedMailWakeRecord(
|
||||
record: SleepingAgentSessionRecord
|
||||
): SleepingAgentSessionRecord | null {
|
||||
if (mailWakeObligation(record) !== 'retained' || !record.mailWake || record.mailWake.attempt) {
|
||||
return null
|
||||
}
|
||||
const { restoreOnTabOpenOnly: _tabOnly, ...resume } = withoutMailWake(record)
|
||||
return { ...resume, origin: 'worktree-sleep', capturedAt: record.mailWake.capturedAt }
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user