fix(orchestration): scope sleeping mail invalidation to affected owners

This commit is contained in:
Brennan Benson
2026-11-12 19:57:37 -08:00
parent bfb67daae2
commit 683289d853
16 changed files with 706 additions and 125 deletions
@@ -0,0 +1,52 @@
import { sessionCrewChanges } from '../../orchestration/crew/crew-changes'
import type { AgentSessionStoreState } from './agent-session-store-state'
import type { AgentSessionStoreRowWrites } from './agent-session-store-draft'
export function publishAgentSessionCrewChanges(
published: AgentSessionStoreState,
draft: AgentSessionStoreState,
writes: AgentSessionStoreRowWrites
): void {
const topologyChanged =
writes.sessionTabs !== null ||
writes.records.remove.length > 0 ||
writes.records.upsert.some(([id]) => {
const before = published.records.get(id)
const after = draft.records.get(id)
return (
!before ||
!after ||
before.lease.claimStatus !== after.lease.claimStatus ||
before.provider !== after.provider ||
before.location !== after.location ||
before.conversationCommand !== after.conversationCommand
)
})
const renamed = writes.records.upsert
.filter(
([id]) =>
published.records.get(id)?.conversationName !== draft.records.get(id)?.conversationName
)
.map(([id]) => id)
if (topologyChanged) {
const affected = new Set([...writes.records.remove, ...writes.records.upsert.map(([id]) => id)])
if (writes.sessionTabs) {
for (const id of [
...(published.sessionTabs?.sessionIds() ?? []),
...(draft.sessionTabs?.sessionIds() ?? [])
]) {
if (published.sessionTabs?.tabIdFor(id) !== draft.sessionTabs?.tabIdFor(id)) {
affected.add(id)
}
}
}
sessionCrewChanges.notify(
Object.assign(new Set(['records'] as const), {
recordIds: [...affected],
trafficAddresses: new Set([...affected].map((id) => `orca_session_id:${id}`))
})
)
} else if (renamed.length) {
sessionCrewChanges.notify(Object.assign(new Set(['names'] as const), { recordIds: renamed }))
}
}
@@ -61,7 +61,9 @@ export function openAgentSessionRecordStoreOnce(
(opened) => {
slot.store = opened.store
if (opening === slot) {
sessionCrewChanges.notify(new Set(['topology', 'records']))
sessionCrewChanges.notify(
Object.assign(new Set(['topology', 'records'] as const), { mailGlobal: true as const })
)
}
return opened
},
@@ -107,7 +109,9 @@ export async function releaseAgentSessionRecordStore(): Promise<OpenedAgentSessi
const released = opening
opening = null
if (released) {
sessionCrewChanges.notify(new Set(['topology', 'records']))
sessionCrewChanges.notify(
Object.assign(new Set(['topology', 'records'] as const), { mailGlobal: true as const })
)
}
return released ? released.opened.catch(() => null) : null
}
@@ -1,4 +1,4 @@
import { sessionCrewChanges } from '../../orchestration/crew/crew-changes'
import { publishAgentSessionCrewChanges } from './agent-session-crew-invalidation'
import type { JournalHostDatabase } from '../../../native-chat/agent-session-journal/database/journal-host-database'
import type { JournalOperationReceipt } from '../../../native-chat/agent-session-journal/writing/journal-row-writer'
import { journalOpenRefusalError } from '../../../native-chat/agent-session-journal/replay/journal-open-failure'
@@ -263,35 +263,7 @@ export class AgentSessionStoreTransactions {
if (lineageChanged) {
this.lineage = {}
}
const topologyChanged =
writes.sessionTabs !== null ||
writes.records.remove.length > 0 ||
writes.records.upsert.some(([id]) => {
const before = published.records.get(id)
const after = draft.records.get(id)
return (
!before ||
!after ||
before.lease.claimStatus !== after.lease.claimStatus ||
before.provider !== after.provider ||
before.location !== after.location ||
before.conversationCommand !== after.conversationCommand
)
})
const renamed = writes.records.upsert
.filter(
([id]) =>
published.records.get(id)?.conversationName !==
draft.records.get(id)?.conversationName
)
.map(([id]) => id)
if (topologyChanged) {
sessionCrewChanges.notify(new Set(['records']))
} else if (renamed.length) {
sessionCrewChanges.notify(
Object.assign(new Set(['names'] as const), { recordIds: renamed })
)
}
publishAgentSessionCrewChanges(published, draft, writes)
}
}
}
@@ -1,6 +1,8 @@
export type CrewFacet = 'topology' | 'records' | 'status' | 'names' | 'traffic'
export type CrewInvalidation = ReadonlySet<CrewFacet> & {
readonly recordIds?: readonly string[]
readonly mailboxOwners?: ReadonlySet<string>
readonly mailGlobal?: true
readonly trafficAddresses?: ReadonlySet<string>
}
@@ -0,0 +1,235 @@
import { isOrcaSessionId } from '../../../../shared/orca-session-address'
import type { OrchestrationDb } from '../db/orchestration-db'
const dispatchMutations = new Set([
'settleRemoteAttachmentInRelayTransaction',
'completeDispatch',
'failDispatch',
'closeQuestionsForDispatch',
'markWorkerDispatchReady',
'failWorkerStart',
'markWorkerStartUnknown',
'abandonWorkerDispatch',
'beginWorkerStop',
'settleWorkerStop',
'markWorkerStopUnknown',
'reconcileMissingWorkerTerminal',
'failRemoteAttachment',
'markRemoteAttachmentReady',
'beginRemoteAttachmentStop',
'settleRemoteAttachmentStop',
'markRemoteAttachmentStopUnknown',
'requestRemoteAttachmentTerminalRelease'
])
const taskMutations = new Set([
'completeActiveDispatchesForTask',
'failActiveDispatchForTask',
'promoteReadyTasks',
'updateTaskStatus'
])
function fields(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value)
}
/** Capture both sides of transfers; nested mutations contribute their own scope before publication. */
export class CrewMailMutationScope {
readonly trafficAddresses = new Set<string>()
readonly mailboxOwners = new Set<string>()
private readonly visited = new Set<string>()
get notice() {
return { trafficAddresses: this.trafficAddresses, mailboxOwners: this.mailboxOwners }
}
constructor(private readonly db: OrchestrationDb) {}
run(id: string): void {
this.trafficAddresses.add(`run:${id}`)
if (this.seen(`run:${id}`)) {
return
}
this.row(this.db.getRun(id))
}
dispatch(id: string): void {
this.trafficAddresses.add(`dispatch:${id}`)
if (this.seen(`dispatch:${id}`)) {
return
}
this.row(this.db.getDispatchContextById(id) ?? this.db.getRemoteDispatchAttachment(id))
}
private seen(id: string): boolean {
if (this.visited.has(id)) {
return true
}
this.visited.add(id)
return false
}
private task(id: string): void {
if (this.seen(`task:${id}`)) {
return
}
this.row(this.db.getTask(id))
for (const row of this.db.db
.prepare('SELECT id FROM dispatch_contexts WHERE task_id = ?')
.all(id)) {
if (typeof row.id === 'string') {
this.dispatch(row.id)
}
}
}
row(value: unknown): void {
if (Array.isArray(value)) {
for (const row of value) {
this.row(row)
}
} else if (fields(value)) {
for (const [key, item] of Object.entries(value)) {
if (typeof item === 'string') {
if (['runId', 'run_id', 'local_run_id'].includes(key)) {
this.run(item)
}
if (
[
'dispatchId',
'dispatch_id',
'owner_dispatch_id',
'toDispatchId',
'creatorDispatchId'
].includes(key)
) {
this.dispatch(item)
}
if (['taskId', 'task_id'].includes(key)) {
this.task(item)
}
if (
[
'paneKey',
'pane_key',
'fromPaneKey',
'toPaneKey',
'assigneePaneKey',
'assignee_pane_key',
'coordinatorPaneKey',
'coordinator_pane_key',
'creator_pane_key'
].includes(key)
) {
this.mailboxOwners.add(item)
}
if (
[
'mailboxHandle',
'from',
'to',
'from_handle',
'to_handle',
'assigneeHandle',
'assignee_handle',
'coordinatorHandle',
'coordinator_handle',
'terminalHandle',
'terminal_handle'
].includes(key)
) {
this.trafficAddresses.add(item)
}
if (
[
'coordinator_orca_session_id',
'assignee_orca_session_id',
'creator_orca_session_id'
].includes(key)
) {
this.trafficAddresses.add(`orca_session_id:${item}`)
}
if (['messageId', 'message_id'].includes(key)) {
this.row(this.db.getMessageById(item))
}
if (key === 'resourceId') {
this.row(this.db.getWorkerTerminalResource(item))
}
} else if (item && typeof item === 'object') {
this.row(item)
}
}
if (typeof value.id === 'string') {
if ('coordinator_pane_key' in value) {
this.trafficAddresses.add(`run:${value.id}`)
}
if ('assignee_pane_key' in value) {
this.trafficAddresses.add(`dispatch:${value.id}`)
}
}
}
}
capture(name: string, args: unknown[]): void {
const first = args[0]
this.row(args)
if (typeof first === 'string') {
if (dispatchMutations.has(name)) {
this.dispatch(first)
}
if (taskMutations.has(name)) {
this.task(first)
}
if (name === 'resolveGate' || name === 'timeoutGate') {
this.row(
this.db.db.prepare('SELECT run_id, task_id FROM decision_gates WHERE id = ?').get(first)
)
}
if (name === 'convertLifecycleMessageToRejection') {
this.row(this.db.getMessageById(first))
}
if (name === 'settleWorkerTerminalRelease') {
this.row(this.db.getWorkerTerminalResource(first))
}
if (name === 'routeDirectMessagePage') {
this.trafficAddresses.add(first)
if (typeof args[1] === 'string') {
this.run(args[1])
}
if (typeof args[2] === 'string') {
this.trafficAddresses.add(args[2])
}
}
if (name === 'routeUnreadDispatchMailboxToRunMailbox') {
this.dispatch(first)
if (typeof args[1] === 'string') {
this.run(args[1])
}
}
if (name === 'routeForeignDirectMessagesToOwnedMailboxes') {
this.trafficAddresses.add(first)
}
}
if (name === 'unbindOtherRunsForCoordinator' && fields(first)) {
const { paneKey, orcaSessionId, terminalHandle } = first
if (
(paneKey === null || typeof paneKey === 'string') &&
(orcaSessionId === null ||
(typeof orcaSessionId === 'string' && isOrcaSessionId(orcaSessionId))) &&
(terminalHandle === null || typeof terminalHandle === 'string')
) {
this.row(this.db.runsBoundToCoordinator({ paneKey, orcaSessionId, terminalHandle }))
}
}
if (name === 'transitionLifecycle' && fields(first) && typeof first.id === 'string') {
if (first.entity === 'run') {
this.run(first.id)
}
if (first.entity === 'dispatch' || first.entity === 'worker') {
this.dispatch(first.id)
}
if (first.entity === 'task') {
this.task(first.id)
}
}
}
}
@@ -1,3 +1,4 @@
import { CrewMailMutationScope } from './crew-mail-mutation-scope'
import type { CrewInvalidation } from './crew-changes'
import type { OrchestrationDb } from '../db/orchestration-db'
@@ -89,10 +90,13 @@ export function attachCrewMutationNotifications(ctor: { prototype: CrewMutationP
if (hasDomainMethod(prototype, 'settleWorkerTerminalRelease')) {
const release = prototype.settleWorkerTerminalRelease
prototype.settleWorkerTerminalRelease = function (this: OrchestrationDb, resourceId) {
const scope = new CrewMailMutationScope(this)
scope.capture('settleWorkerTerminalRelease', [resourceId])
const before = this.getWorkerTerminalResource(resourceId)?.release_state
const result = release.call(this, resourceId)
if (before !== result.release_state) {
this.crewChanges.notify(new Set(['topology']))
scope.row(result)
this.crewChanges.notify(Object.assign(new Set(['topology'] as const), scope.notice))
}
return result
}
@@ -100,10 +104,13 @@ export function attachCrewMutationNotifications(ctor: { prototype: CrewMutationP
if (hasDomainMethod(prototype, 'settleDeadWorkerTerminalRelease')) {
const release = prototype.settleDeadWorkerTerminalRelease
prototype.settleDeadWorkerTerminalRelease = function (this: OrchestrationDb, params) {
const scope = new CrewMailMutationScope(this)
scope.row(params)
const before = this.getWorkerTerminalResource(params.resourceId)?.release_state
const result = release.call(this, params)
if (before !== result.resource.release_state) {
this.crewChanges.notify(new Set(['topology']))
scope.row(result)
this.crewChanges.notify(Object.assign(new Set(['topology'] as const), scope.notice))
}
return result
}
@@ -111,9 +118,12 @@ export function attachCrewMutationNotifications(ctor: { prototype: CrewMutationP
if (hasDomainMethod(prototype, 'commitLegacyCompatibilityPrincipal')) {
const commit = prototype.commitLegacyCompatibilityPrincipal
prototype.commitLegacyCompatibilityPrincipal = function (this: OrchestrationDb, params) {
const scope = new CrewMailMutationScope(this)
scope.row(params)
const result = commit.call(this, params)
if (!result.duplicate) {
this.crewChanges.notify(new Set(['topology']))
scope.row(result)
this.crewChanges.notify(Object.assign(new Set(['topology'] as const), scope.notice))
}
return result
}
@@ -123,7 +133,9 @@ export function attachCrewMutationNotifications(ctor: { prototype: CrewMutationP
const known = this.localMutationCallerFingerprint
const result = sourceIdentity.call(this)
if (!known) {
this.crewChanges.notify()
this.crewChanges.notify(
Object.assign(new Set(['topology'] as const), { trafficAddresses: new Set<string>() })
)
}
return result
}
@@ -133,7 +145,9 @@ export function attachCrewMutationNotifications(ctor: { prototype: CrewMutationP
if (result.type !== 'heartbeat') {
const notice: CrewInvalidation =
result.delivery_contract === 'legacy_direct'
? new Set(['topology'])
? Object.assign(new Set(['topology'] as const), {
trafficAddresses: new Set([result.from_handle, result.to_handle])
})
: Object.assign(new Set(['traffic'] as const), {
trafficAddresses: new Set([result.from_handle, result.to_handle])
})
@@ -150,7 +164,18 @@ export function attachCrewMutationNotifications(ctor: { prototype: CrewMutationP
configurable: true,
writable: true,
value(this: OrchestrationDb, ...args: unknown[]) {
const before = new CrewMailMutationScope(this)
before.capture(name, args)
const result = original.apply(this, args)
const after = new CrewMailMutationScope(this)
after.capture(name, args)
after.row(result)
for (const address of before.trafficAddresses) {
after.trafficAddresses.add(address)
}
for (const owner of before.mailboxOwners) {
after.mailboxOwners.add(owner)
}
if (
typeof result === 'object' &&
result !== null &&
@@ -160,7 +185,13 @@ export function attachCrewMutationNotifications(ctor: { prototype: CrewMutationP
) {
return result
}
this.crewChanges.notify(new Set([trafficMutations.has(name) ? 'traffic' : 'topology']))
this.crewChanges.notify(
Object.assign(
new Set([trafficMutations.has(name) ? 'traffic' : 'topology'] as const),
after.notice,
name.startsWith('reset') ? { mailGlobal: true as const } : {}
)
)
return result
}
})
@@ -90,10 +90,15 @@ export function setLegacyCompatibilityPrincipalStatus(
WHERE id = ? AND status = 'committed'`
)
.run(status, id)
if (changed.changes) {
this.crewChanges.notify(new Set(['topology']))
const principal = this.getLegacyCompatibilityPrincipal(id)
if (changed.changes && principal) {
this.crewChanges.notify(
Object.assign(new Set(['topology'] as const), {
trafficAddresses: new Set([`run:${principal.run_id}`])
})
)
}
return this.getLegacyCompatibilityPrincipal(id)
return principal
}
export function getLegacyOperationReceipt(
@@ -1,3 +1,4 @@
import { captureMessageMutation } from './message-mutation-invalidation'
import { undeliveredInboxQuery, type UndeliveredInboxOptions } from './undelivered-inbox-query'
import type { MessageType, MessageRow } from '../orchestration-records'
import { exposeMessageTimestamps, exposeMessageListTimestamps } from '../utc-timestamp'
@@ -18,6 +19,7 @@ function runBatchedMessageMutation(
if (ids.length === 0) {
return
}
const changed = captureMessageMutation(db, ids)
db.db.exec(`SAVEPOINT ${MESSAGE_MUTATION_SAVEPOINT}`)
try {
for (let offset = 0; offset < ids.length; offset += MESSAGE_ID_UPDATE_BATCH_SIZE) {
@@ -31,6 +33,7 @@ function runBatchedMessageMutation(
db.db.exec(`RELEASE ${MESSAGE_MUTATION_SAVEPOINT}`)
throw error
}
changed()
}
export function getUnreadMessages(
@@ -0,0 +1,54 @@
import type { OrchestrationDb } from '../orchestration-db'
export function captureMessageMutation(db: OrchestrationDb, ids: string[]): () => void {
const read = () => {
const rows = new Map<string, { address: string; state: string }>()
for (let offset = 0; offset < ids.length; offset += 500) {
const batch = ids.slice(offset, offset + 500)
for (const row of db.db
.prepare(`
SELECT id, to_handle, read, delivered_at, pointer_enter_pending FROM messages
WHERE id IN (${batch.map(() => '?').join(',')})
`)
.all(...batch)) {
if (typeof row.id === 'string' && typeof row.to_handle === 'string') {
rows.set(row.id, { address: row.to_handle, state: JSON.stringify(row) })
}
}
}
return rows
}
const before = read()
return () => {
const publish = () => {
const after = read()
const addresses = new Set<string>()
for (const [id, row] of before) {
if (after.get(id)?.state !== row.state) {
addresses.add(row.address)
const next = after.get(id)
if (next) {
addresses.add(next.address)
}
}
}
if (addresses.size) {
db.crewChanges.notify(
Object.assign(new Set(['traffic'] as const), { trafficAddresses: addresses })
)
}
}
// An outer transaction can still roll back; compare committed rows on the next turn.
if (db.db.isTransaction) {
queueMicrotask(() => {
try {
publish()
} catch {
/* The owning database can close after the transaction commits. */
}
})
} else {
publish()
}
}
}
@@ -67,7 +67,11 @@ export function rememberRunCoordinatorHandle(
)
.run(runId, terminalHandle)
if (changed.changes) {
this.crewChanges.notify(new Set(['topology']))
this.crewChanges.notify(
Object.assign(new Set(['topology'] as const), {
trafficAddresses: new Set([`run:${runId}`, terminalHandle])
})
)
}
}
@@ -66,7 +66,7 @@ export abstract class RuntimeMailWakeAdmission extends OrcaRuntimeWithPruneMobil
shellInFront: async (ptyId) =>
(await this.readLaunchedAgentForeground(ptyId, record.agent)) === 'shell'
})
// Host-side input fencing is required before a proven failed invocation may be stopped.
// TEMPORARY: STA-15118 requires host input fencing before automatic surviving-shell replacement.
return false
}
@@ -77,25 +77,32 @@ export abstract class RuntimeMailWakeAdmission extends OrcaRuntimeWithPruneMobil
return null
}
const attempt = record.mailWake?.attempt
const pty = this.getPtyRecordForPaneKey(record.paneKey)
const liveShell =
!!attempt &&
pty?.connected === true &&
pty.ptyId === attempt.ptyId &&
pty.incarnationId === attempt.incarnationId &&
pty.launchToken === attempt.launchToken &&
pty.launchIncarnationId === attempt.incarnationId &&
pty.worktreeId === record.worktreeId &&
pty.connectionId === (record.connectionId ?? null) &&
this.mailWakeInvocations.ended(record) &&
(await this.readLaunchedAgentForeground(attempt.ptyId, record.agent)) === 'shell' &&
this.getPtyRecordForPaneKey(record.paneKey) === pty &&
this.sleepingMailRecord(record.paneKey) === record
const matchingAttempt = () => {
const pty = this.getPtyRecordForPaneKey(record.paneKey)
return (
!!attempt &&
pty?.connected === true &&
pty.ptyId === attempt.ptyId &&
pty.incarnationId === attempt.incarnationId &&
pty.launchToken === attempt.launchToken &&
pty.launchIncarnationId === attempt.incarnationId &&
pty.worktreeId === record.worktreeId &&
pty.connectionId === (record.connectionId ?? null) &&
this.terminalRunFacts.readProcessStartedAt(attempt.ptyId, attempt.incarnationId) !== null
)
}
const foreground =
attempt && matchingAttempt()
? await this.readLaunchedAgentForeground(attempt.ptyId, record.agent)
: 'unknown'
const current = matchingAttempt() && this.sleepingMailRecord(record.paneKey) === record
const ended = this.mailWakeInvocations.ended(record)
const liveShell = current && ended && foreground === 'shell'
return {
kind: 'asleep',
waking:
mailWakeObligation(record) === 'automatic' && !this.sleptMailboxWake.failedToWake(record),
resuming: mailWakeObligation(record) === 'retained' && !liveShell,
resuming: current && !ended && foreground === 'agent',
agentName: TUI_AGENT_DISPLAY_NAMES[record.agent],
openTab: record.restoreOnTabOpenOnly === true,
failed: this.sleptMailboxWake.failedToWake(record),
@@ -0,0 +1,74 @@
import type { SleptMailboxWake } from './slept-mailbox-wake'
import type { OrchestrationDb } from '../../db/orchestration-db'
import {
RUN_PANE_KEY_MATCH_SUFFIX_SQL,
DISPATCH_PANE_KEY_MATCH_SUFFIX_SQL,
REMOTE_ATTACHMENT_PANE_KEY_MATCH_SUFFIX_SQL,
paneKeyMatchSuffix,
isEquivalentPaneKey
} from '../../db/pane-key-match'
import { potentiallyLiveRemoteAttachmentSql } from '../../db/federation/remote-attachment-liveness'
import { parseOrcaSessionAddress } from '../../../../../shared/orca-session-address'
/** Owner edits discover mail without enumerating unrelated unread mailboxes. */
export function sleepingMailOwnerMailboxes(
db: OrchestrationDb,
pane: string | null,
address?: string
): string[] {
const suffix = pane === null ? null : paneKeyMatchSuffix(pane)
const sessionId = parseOrcaSessionAddress(address)
const rows = db.db
.prepare(`
SELECT 'run:' || id AS mailbox, coordinator_pane_key AS owner FROM runs
WHERE coordinator_pane_key IS NOT NULL AND ${RUN_PANE_KEY_MATCH_SUFFIX_SQL} = ?
UNION SELECT 'dispatch:' || id AS mailbox, assignee_pane_key AS owner FROM dispatch_contexts
WHERE assignee_pane_key IS NOT NULL AND ${DISPATCH_PANE_KEY_MATCH_SUFFIX_SQL} = ?
AND status IN ('pending', 'dispatched')
UNION SELECT 'dispatch:' || dispatch_id AS mailbox, pane_key AS owner FROM remote_dispatch_attachments
WHERE pane_key IS NOT NULL AND ${REMOTE_ATTACHMENT_PANE_KEY_MATCH_SUFFIX_SQL} = ?
AND ${potentiallyLiveRemoteAttachmentSql()}
UNION SELECT 'run:' || id AS mailbox, NULL AS owner FROM runs WHERE coordinator_orca_session_id = ?
UNION SELECT 'run:' || run_id AS mailbox, NULL AS owner FROM run_coordinator_handles WHERE terminal_handle = ?
UNION SELECT 'dispatch:' || id AS mailbox, NULL AS owner FROM dispatch_contexts
WHERE (assignee_handle = ? OR assignee_orca_session_id = ?)
AND status IN ('pending', 'dispatched')
`)
.all(suffix, suffix, suffix, sessionId, address ?? null, address ?? null, sessionId)
return rows.flatMap((row) =>
typeof row.mailbox === 'string' &&
(pane === null ||
row.owner === null ||
(typeof row.owner === 'string' && isEquivalentPaneKey(pane, row.owner)))
? [row.mailbox]
: []
)
}
export function refreshSleepingMailOwners(
wake: SleptMailboxWake,
db: OrchestrationDb,
owners: Iterable<string>,
addresses: Iterable<string>
): Set<string> {
const affected = new Set(owners)
const mailboxes = new Set(addresses)
for (const pane of affected) {
for (const mailbox of [...wake.mailboxesForPane(pane), ...wake.discover(pane)]) {
mailboxes.add(mailbox)
}
}
for (const address of addresses) {
if (!address.startsWith('run:') && !address.startsWith('dispatch:')) {
for (const mailbox of sleepingMailOwnerMailboxes(db, null, address)) {
mailboxes.add(mailbox)
}
}
}
for (const mailbox of mailboxes) {
for (const pane of wake.refresh(mailbox)) {
affected.add(pane)
}
}
return affected
}
@@ -1,3 +1,5 @@
import { refreshSleepingMailOwners } from './sleeping-mail-owner-mailboxes'
import { sessionCrewChanges, type CrewInvalidation } from '../../crew/crew-changes'
import { replaceMailWakeRecord } from '../../../session-lifecycle/projection/retained-mail-wake-attempt'
import { SleepingMailRecordIndex } from './sleeping-mail-record-index'
import {
@@ -118,13 +120,16 @@ export abstract class RuntimeSleptMailboxWakes extends RuntimeMailWakeAdmission
{ consumesTerminalSideEffects: false }
)
this.subscribeAgentStatusRowChanges((paneKey) => this.watchMailWakeInvocations(paneKey))
this.crewChanges.subscribe((facets) => {
const changed = (facets: CrewInvalidation) => {
if (facets.has('traffic') || facets.has('topology') || facets.has('records')) {
this.publishSleepingMail(
facets.has('topology') || facets.has('records') ? undefined : facets.trafficAddresses
facets.mailGlobal ? undefined : (facets.trafficAddresses ?? []),
facets.mailboxOwners
)
}
})
}
this.crewChanges.subscribe(changed)
sessionCrewChanges.subscribe(changed)
this.store?.onWorkspaceSessionWritten?.((hostId) => {
const hosts =
hostId === null ? (this.store?.getWorkspaceSessionHostIds?.() ?? ['local']) : [hostId]
@@ -132,7 +137,7 @@ export abstract class RuntimeSleptMailboxWakes extends RuntimeMailWakeAdmission
const changed = this.refreshSleepingRecords(host)
this.watchMailWakeInvocations()
if (changed) {
queueMicrotask(() => this.retrySleptMailboxWakes())
queueMicrotask(() => this.publishSleepingMail([]))
} else {
this.publishSleepingMail([])
}
@@ -221,7 +226,10 @@ export abstract class RuntimeSleptMailboxWakes extends RuntimeMailWakeAdmission
private sleepingMailPublicationQueued = false
private sleepingMailPublication = ''
protected publishSleepingMail(mailboxes?: Iterable<string>): void {
protected publishSleepingMail(mailboxes?: Iterable<string>, owners?: Iterable<string>): void {
for (const owner of owners ?? []) {
this.changedSleepingPanes.add(owner)
}
if (mailboxes) {
for (const mailbox of mailboxes) {
this.sleepingMailAddresses.add(mailbox)
@@ -240,7 +248,8 @@ export abstract class RuntimeSleptMailboxWakes extends RuntimeMailWakeAdmission
return
}
const global = this.globalSleepingMailInvalidation
const affected = new Set(this.changedSleepingPanes)
let affected = new Set(this.changedSleepingPanes)
const addresses = new Set(this.sleepingMailAddresses)
this.changedSleepingPanes.clear()
if (global && (this.sleepingRecords.size || this.sleptMailboxWake.hasTrackedMail)) {
this.sleptMailboxWake.reconcile()
@@ -248,11 +257,12 @@ export abstract class RuntimeSleptMailboxWakes extends RuntimeMailWakeAdmission
affected.add(pane)
}
} else {
for (const mailbox of this.sleepingMailAddresses) {
for (const pane of this.sleptMailboxWake.refresh(mailbox)) {
affected.add(pane)
}
}
affected = refreshSleepingMailOwners(
this.sleptMailboxWake,
this._orchestrationDb,
affected,
addresses
)
}
this.globalSleepingMailInvalidation = false
this.sleepingMailAddresses.clear()
@@ -262,11 +272,9 @@ export abstract class RuntimeSleptMailboxWakes extends RuntimeMailWakeAdmission
await this.editMailWakeRecord(record, withoutMailWake(record))
}
}
if (global) {
for (const pane of this.sleepingRecords.keys()) {
for (const mailbox of this.sleptMailboxWake.mailboxesForPane(pane)) {
this.sleptMailboxWake.request(mailbox)
}
for (const pane of global ? this.sleepingRecords.keys() : affected) {
for (const mailbox of this.sleptMailboxWake.mailboxesForPane(pane)) {
this.sleptMailboxWake.request(mailbox)
}
}
const settledIds = [...this.settledMailWakeIds]
@@ -1,6 +1,8 @@
import type { ExecutionHostId } from '../../../../../shared/execution-host'
import type { WorkspaceSessionState } from '../../../../../shared/workspace-session/workspace-session-state-types'
import { OrchestrationMailboxPointerDelivery } from '../terminal-pointer/mailbox-pointer-delivery'
import { createRootDispatch } from '../../db/root-dispatch-test-fixture'
import { ORCHESTRATION_REPLY_METHOD } from '../../../rpc/methods/orchestration/messaging/reply-method'
import { MailWakeInvocations } from './mail-wake-invocations'
import { admitsMailWakeRetry } from './mail-wake-retry-admission'
import {
@@ -37,6 +39,24 @@ const record: SleepingAgentSessionRecord = {
}
const settled = () => new Promise<void>((resolve) => setImmediate(resolve))
const mailNotifier = (
resume: RuntimeNotifier['resumeSleepingAgents'],
bell?: RuntimeNotifier['sleepingMailChanged']
): RuntimeNotifier => ({
worktreesChanged: vi.fn(),
reposChanged: vi.fn(),
activateWorktree: vi.fn(),
createTerminal: vi.fn(),
splitTerminal: vi.fn(),
renameTerminal: vi.fn(),
focusTerminal: vi.fn(),
closeTerminal: vi.fn(),
terminalFitOverrideChanged: vi.fn(),
terminalDriverChanged: vi.fn(),
resumeSleepingAgents: resume,
sleepingMailChanged: bell
})
it('wakes only the automatic sleeper owed deliverable mail, coalesces it, and retries a failed wake on availability', async () => {
const db = new OrchestrationDb(':memory:')
try {
@@ -156,20 +176,7 @@ it('holds manual Sleep through the sender receipt and keeps its bell until mail
)
const resume = vi.fn(async () => true)
const bell = vi.fn()
const notifier: RuntimeNotifier = {
worktreesChanged: vi.fn(),
reposChanged: vi.fn(),
activateWorktree: vi.fn(),
createTerminal: vi.fn(),
splitTerminal: vi.fn(),
renameTerminal: vi.fn(),
focusTerminal: vi.fn(),
closeTerminal: vi.fn(),
terminalFitOverrideChanged: vi.fn(),
terminalDriverChanged: vi.fn(),
resumeSleepingAgents: resume,
sleepingMailChanged: bell
}
const notifier = mailNotifier(resume, bell)
try {
runtime.setNotifier(notifier)
runtime.syncWindowGraph(1, { tabs: [], leaves: [] })
@@ -454,11 +461,16 @@ it('suppresses a definitively confirmed late wake only for its obligation and re
delivery.request(mailbox)
await settled()
expect(wake).toHaveBeenCalledTimes(1)
sleeping = null
delivery.refresh(mailbox)
sleeping = record
delivery.request(mailbox)
await settled()
expect(wake).toHaveBeenCalledTimes(1)
db.markAsRead([message.id])
delivery.refresh(mailbox)
expect(Reflect.get(delivery, 'succeeded').size).toBe(0)
expect(Reflect.get(delivery, 'failed').size).toBe(0)
expect(Reflect.get(delivery, 'attempted').size).toBe(0)
expect(delivery.failedToWake(retained)).toBe(false)
expect(delivery.heldWorktrees()).toEqual([])
for (let i = 0; i < 20; i++) {
sleeping = record
@@ -480,9 +492,8 @@ it('suppresses a definitively confirmed late wake only for its obligation and re
db.markAsRead([row.id])
delivery.refresh(handle)
}
expect(Reflect.get(delivery, 'succeeded').size).toBe(0)
expect(Reflect.get(delivery, 'failed').size).toBe(0)
expect(Reflect.get(delivery, 'attempted').size).toBe(0)
expect(wake).toHaveBeenCalledTimes(21)
expect(delivery.failedToWake(record)).toBe(false)
expect(delivery.heldWorktrees()).toEqual([])
} finally {
db.close()
@@ -499,7 +510,7 @@ it.each([10, 100])(
const records: Record<string, SleepingAgentSessionRecord> = {}
const mailboxes: string[] = []
for (let i = 0; i < count; i++) {
const key = `tab-${i}:11111111-1111-4111-8111-111111111111`
const key = `tab-${i}:11111111-1111-4111-8111-${String(i).padStart(12, '0')}`
records[key] = new Proxy(
{
...record,
@@ -537,6 +548,13 @@ it.each([10, 100])(
db.insertMessage({ runId: run.id, from: 'worker', to: mailbox, subject: 'done' })
}
}
const task = db.createTask({ spec: 'local completion', runId: mailboxes[0]!.slice(4) })
const dispatch = createRootDispatch(
db,
task.id,
'term_local_worker',
'tab-0:11111111-1111-4111-8111-000000000000'
)
let session = { ...getDefaultWorkspaceSession(), sleepingAgentSessionsByPaneKey: records }
const readSession = vi.fn(() => session)
const runtime = new OrcaRuntimeService(
@@ -564,24 +582,20 @@ it.each([10, 100])(
readSession.mockClear()
statusChanged('unrelated:11111111-1111-4111-8111-111111111111')
expect(visits).toBe(0)
statusChanged('tab-0:11111111-1111-4111-8111-111111111111')
statusChanged('tab-0:11111111-1111-4111-8111-000000000000')
// A status event needs only this pane's attempt and PTY identity; no session inventory.
expect(visits).toBeLessThanOrEqual(4)
expect(readSession).not.toHaveBeenCalled()
const inventory = vi.spyOn(db, 'getUndeliveredUnreadMailboxHandles')
const fullRows = vi.spyOn(db, 'getUndeliveredUnreadMessages')
const newest = vi.spyOn(db, 'getUndeliveredUnreadSequence')
runtime.crewChanges.notify(
Object.assign(new Set(['traffic'] as const), { trafficAddresses: new Set([mailboxes[0]!]) })
)
db.completeDispatch(dispatch.id)
await settled()
expect(inventory).not.toHaveBeenCalled()
expect(fullRows).not.toHaveBeenCalled()
// Refresh and the custody check each need one scalar newest-sequence row for the affected mailbox.
expect(newest.mock.calls.length).toBeLessThanOrEqual(3)
expect(
newest.mock.results.filter((result) => result.value !== undefined).length
).toBeLessThanOrEqual(3)
expect(newest.mock.calls.filter(([mailbox]) => mailbox !== mailboxes[0])).toEqual([])
// One affected Run: refresh, custody validation and each start-stage revalidation; no N-dependent work.
expect(newest.mock.calls.length).toBeLessThanOrEqual(8)
expect(newest).toHaveBeenCalled()
session = {
...session,
@@ -604,9 +618,18 @@ it.each([10, 100])(
await settled()
visits = 0
readSession.mockClear()
statusChanged('tab-0:11111111-1111-4111-8111-111111111111')
statusChanged('tab-0:11111111-1111-4111-8111-000000000000')
expect(visits).toBe(0)
expect(readSession).not.toHaveBeenCalled()
session = { ...session, sleepingAgentSessionsByPaneKey: {} }
sessionWritten('local')
await settled()
inventory.mockClear()
newest.mockClear()
db.completeDispatch(dispatch.id)
await settled()
expect(inventory).not.toHaveBeenCalled()
expect(newest).not.toHaveBeenCalled()
} finally {
db.close()
}
@@ -640,19 +663,7 @@ it('holds unknown dispatch custody but retries a positively undispatched wake on
const resume = vi.fn(
async (_workspace: string, _pane?: string, prepare?: boolean) => prepare === true
)
const notifier: RuntimeNotifier = {
worktreesChanged: vi.fn(),
reposChanged: vi.fn(),
activateWorktree: vi.fn(),
createTerminal: vi.fn(),
splitTerminal: vi.fn(),
renameTerminal: vi.fn(),
focusTerminal: vi.fn(),
closeTerminal: vi.fn(),
terminalFitOverrideChanged: vi.fn(),
terminalDriverChanged: vi.fn(),
resumeSleepingAgents: resume
}
const notifier = mailNotifier(resume)
try {
const run = db.createRun({
objective: 'refused launch',
@@ -666,7 +677,8 @@ it('holds unknown dispatch custody but retries a positively undispatched wake on
runtime.syncWindowGraph(1, { tabs: [], leaves: [] })
await settled()
expect(await runtime.readSleepingMailHold(mailbox)).toMatchObject({
resuming: true,
resuming: false,
openTab: true,
liveShell: false
})
runtime.retrySleptMailboxWakes()
@@ -689,3 +701,101 @@ it('holds unknown dispatch custody but retries a positively undispatched wake on
db.close()
}
})
it('clears the original workspace hold and custody when another terminal replies to its last mail', async () => {
const db = new OrchestrationDb(':memory:')
let session: WorkspaceSessionState = {
...getDefaultWorkspaceSession(),
sleepingAgentSessionsByPaneKey: {
[paneKey]: {
...record,
origin: 'quit',
restoreOnTabOpenOnly: true,
mailWake: { id: 'reply-custody', capturedAt: 1, startedAt: 100 }
}
}
}
let written: (host: ExecutionHostId | null) => void = () => {}
const runtime = new OrcaRuntimeService(
partialRuntimeStore({
getWorkspaceSession: () => session,
getWorkspaceSessionHostIds: () => ['local'],
setWorkspaceSession: (next) => {
session = next
written('local')
},
runDurableMutation: async (mutation) => mutation().value,
onWorkspaceSessionWritten: (listener) => {
written = listener
return () => {}
}
})
)
const bell = vi.fn()
runtime.setNotifier(
mailNotifier(
vi.fn(async () => false),
bell
)
)
try {
const run = db.createRun({
objective: 'reply consumes mail',
coordinatorHandle: 'term_asleep',
coordinatorPaneKey: paneKey
})
const mailbox = `run:${run.id}`
const sender = db.createRun({
objective: 'sender',
coordinatorHandle: 'term_sender',
coordinatorPaneKey: null
})
const original = db.insertMessage({
runId: run.id,
from: `run:${sender.id}`,
to: mailbox,
subject: 'question'
})
runtime.setOrchestrationDb(db)
runtime.retrySleptMailboxWakes()
await settled()
expect(await runtime.readSleepingMailHold(mailbox)).toMatchObject({
kind: 'asleep',
resuming: false
})
expect(bell).toHaveBeenLastCalledWith(['folder:workspace'])
const inventory = vi.spyOn(db, 'getUndeliveredUnreadMailboxHandles')
const originalInvalidation = vi.fn()
const unwatch = db.crewChanges.subscribe((event) => {
if (event.trafficAddresses?.has(mailbox)) {
originalInvalidation(db.getMessageById(original.id)?.read)
}
})
db.db.exec('BEGIN IMMEDIATE')
db.markAsRead([original.id])
expect(originalInvalidation).not.toHaveBeenCalled()
db.db.exec('ROLLBACK')
await settled()
expect(originalInvalidation).not.toHaveBeenCalled()
expect(await runtime.readSleepingMailHold(mailbox)).not.toBeNull()
await ORCHESTRATION_REPLY_METHOD.handler(
ORCHESTRATION_REPLY_METHOD.params.parse({
id: original.id,
from: 'term_other',
body: 'answer'
}),
{ runtime }
)
await settled()
unwatch()
expect(originalInvalidation).toHaveBeenCalledWith(1)
expect(db.getMessageById(original.id)?.read).toBe(1)
expect(await runtime.readSleepingMailHold(mailbox)).toBeNull()
expect(session.sleepingAgentSessionsByPaneKey?.[paneKey]?.mailWake).toBeUndefined()
expect(inventory).not.toHaveBeenCalled()
expect(bell).toHaveBeenLastCalledWith([], ['reply-custody'])
} finally {
runtime.setNotifier(null)
db.close()
}
})
@@ -1,3 +1,4 @@
import { sleepingMailOwnerMailboxes } from './sleeping-mail-owner-mailboxes'
import { POTENTIALLY_LIVE_REMOTE_ATTACHMENT_STATES } from '../../db/federation/remote-attachment-liveness'
import type { SleepingAgentSessionRecord } from '../../../../../shared/agent-session/lifecycle/agent-session-resume'
import { agentProviderSessionsEqual } from '../../../../../shared/agent-session/lifecycle/agent-session-resume'
@@ -76,7 +77,11 @@ export class SleptMailboxWake {
private selection(mailbox: string, reservedTypes?: ReadonlySet<string>) {
const db = this.deps.db()
const paneKey = db && mailboxOwnerPane(db, mailbox)
if (!db || !paneKey) {
if (
!db ||
!paneKey ||
(!mailWakeObligation(this.deps.record(paneKey)) && !this.succeeded.has(mailbox))
) {
return null
}
const sequence = selectOrchestrationPointerSequence({
@@ -90,7 +95,9 @@ export class SleptMailboxWake {
refresh(mailbox: string): string[] {
const before = this.owed.get(mailbox)
const next = this.selection(mailbox)
const selection = this.selection(mailbox)
const next =
selection && mailWakeObligation(this.deps.record(selection.paneKey)) ? selection : null
if (before) {
const handles = this.byPane.get(before.paneKey)
handles?.delete(mailbox)
@@ -106,7 +113,11 @@ export class SleptMailboxWake {
this.byPane.set(next.paneKey, handles)
}
const success = this.succeeded.get(mailbox)
if (!next || success?.paneKey !== next.paneKey || success.sequence !== next.sequence) {
if (
!selection ||
success?.paneKey !== selection.paneKey ||
success.sequence !== selection.sequence
) {
this.succeeded.delete(mailbox)
}
const attempted = this.attempted.get(mailbox)
@@ -126,12 +137,12 @@ export class SleptMailboxWake {
}
get hasTrackedMail(): boolean {
return this.owed.size > 0 || this.attempted.size > 0
return this.owed.size > 0 || this.attempted.size > 0 || this.succeeded.size > 0
}
reconcile(): void {
const handles = new Set(this.deps.db()?.getUndeliveredUnreadMailboxHandles() ?? [])
for (const handle of this.owed.keys()) {
for (const handle of [...this.owed.keys(), ...this.succeeded.keys()]) {
handles.add(handle)
}
for (const handle of handles) {
@@ -139,6 +150,13 @@ export class SleptMailboxWake {
}
}
discover(paneKey: string): readonly string[] {
const db = this.deps.db()
return db && mailWakeObligation(this.deps.record(paneKey))
? sleepingMailOwnerMailboxes(db, paneKey)
: []
}
mailboxesForPane(paneKey: string): readonly string[] {
return [...(this.byPane.get(paneKey) ?? [])]
}
@@ -32,7 +32,9 @@ export abstract class OrcaRuntimeWithCrewOperations extends OrcaRuntimeWithOrche
this.crewChanges.notify(facets)
})
this.orchestrationDbChanges.notify()
this.crewChanges.notify()
this.crewChanges.notify(
Object.assign(new Set(['topology'] as const), { mailGlobal: true as const })
)
}
private readonly crewReadModel = new CrewReadModel(