fix(orchestration): complete tasks on worker_done + coordinator UX fixes (#8030)

* fix(orchestration): complete worker tasks and improve coordinator UX

* Fix orchestration lifecycle sender resolution and peek/check compat hand

- Lifecycle sends (worker_done/heartbeat) now use ORCA_TERMINAL_HANDLE
  verbatim, skipping the liveness probe and pane remint that could
  block delivery during restarts or mismatch stale-runtime assignee
  handles.
- --peek now round-trips as {peek:true, unread:false} so older runtimes
  that strip unknown params degrade to non-destructive "all" instead of
  mark-read, with client-side filtering to restore peek semantics and a
  clear error when --peek --wait can't be honored.
- Reject combined read-mode flags (--unread/--peek/--all) before calling
  the runtime.
- Distinguish suppressed (already-consumed) lifecycle messages from
  ignored ones so send doesn't wake --wait waiters for stale heartbeats.
- Fix task summary truncation to avoid splitting UTF-16 surrogate pairs
  and to not misreport whitespace normalization as truncation.

* Add shared helper to abbreviate orchestration task specs for brief listi

- Normalizes whitespace and caps spec length at 160 chars, flagging
  truncation separately from whitespace-only changes
- Truncates on UTF-16 code point boundaries to avoid splitting
  surrogate pairs and emitting malformed strings

* Add pane-key identity to worker_done/heartbeat reconciliation and server

- Records the sender's pane key on messages and dispatch contexts so
  worker_done/heartbeat ownership can be verified by the remint-stable
  pane leaf instead of the terminal handle, which is reissued across
  restarts.
- Rejects lifecycle messages from a genuinely foreign pane while still
  tolerating handle remints, tab break-outs, and older CLIs that lack
  pane identity.
- Moves task-spec abbreviation server-side (orchestration.taskList
  --brief) so full specs no longer cross SSH/relay transports, with a
  client-side fallback for older runtimes; consolidates the shared
  abbreviation helper under src/shared.
- Adds a stderr warning when a pre-peek runtime's --peek response hits
  the 100-row cap, since older unread messages may be missing.

* Isolate ORCA_PANE_KEY in CLI test beforeEach to fix leaked senderPaneKey

Co-authored-by: Orca <help@stably.ai>

* Fix pane-key remint bypassing dispatch mutual-exclusion lock

- Dispatch locking only matched on assignee_handle, so a reminted
  terminal handle (tab break-out) could open a second concurrent
  dispatch on the same pane.
- Add leaf-UUID-based pane key comparison (parsePaneKey) as a
  secondary lock, falling back to exact handle match for legacy
  rows without pane keys.

* Update orchestration skill docs for lifecycle authority and CLI flag add

- Clarify that dispatch lifecycle is tied to taskId+dispatchId verified against
  the dispatched pane, not the terminal handle, since handles can be reminted
  after restart
- Document new `check --peek`/`--all` and `task-list --brief` flags, with
  fallback guidance for older CLIs that reject them
- Note that a valid worker_done auto-completes the task/dispatch, so workers
  shouldn't also call task-update manually

---------

Co-authored-by: Orca <help@stably.ai>
This commit is contained in:
Jinjing
2026-07-12 02:15:11 -07:00
committed by GitHub
co-authored by Orca
parent ee82d66a35
commit 26934b11bf
24 changed files with 1090 additions and 133 deletions
@@ -144,6 +144,31 @@ describe('Coordinator', () => {
expect(runtime.sentMessages.length).toBeGreaterThan(0)
})
it('records the assignee pane key when the runtime can resolve one', async () => {
db = new OrchestrationDb(':memory:')
const runtime = createMockRuntime()
runtime.terminals = [{ handle: 'term_a', worktreeId: 'wt1', connected: true, writable: true }]
const withPaneLookup = Object.assign(runtime, {
getTerminalPaneKey: (handle: string) => (handle === 'term_a' ? 'tab_a:leaf_a' : null)
})
const task = db.createTask({ spec: 'implement feature' })
const coordinator = new Coordinator(db, withPaneLookup, {
spec: 'build it',
coordinatorHandle: 'coord',
pollIntervalMs: 50
})
const runPromise = coordinator.run()
await new Promise((r) => {
setTimeout(r, 100)
})
expect(db.getDispatchContext(task.id)?.assignee_pane_key).toBe('tab_a:leaf_a')
insertWorkerDone(db, { taskId: task.id })
await runPromise
})
it('records completedTasks when send reconciled worker_done before coordinator read', async () => {
db = new OrchestrationDb(':memory:')
const runtime = createMockRuntime()
@@ -570,7 +595,7 @@ describe('Coordinator', () => {
expect(db.getDispatchContextById(activeCtx.id)?.status).toBe('completed')
})
it('ignores worker_done sent by a terminal that does not own the dispatch', async () => {
it('accepts worker_done payload provenance after an assignee handle changes', async () => {
db = new OrchestrationDb(':memory:')
const runtime = createMockRuntime()
const logs: string[] = []
@@ -579,9 +604,9 @@ describe('Coordinator', () => {
const ctx = db.createDispatchContext(task.id, 'term_owner')
db.insertMessage({
from: 'term_intruder',
from: 'term_reminted',
to: 'coord',
subject: 'Spoofed done',
subject: 'Done after restart',
type: 'worker_done',
payload: JSON.stringify({ taskId: task.id, dispatchId: ctx.id })
})
@@ -592,16 +617,12 @@ describe('Coordinator', () => {
pollIntervalMs: 20,
onLog: (m) => logs.push(m)
})
const runPromise = coordinator.run()
await new Promise((r) => {
setTimeout(r, 80)
})
coordinator.stop()
await runPromise
const result = await coordinator.run()
expect(db.getTask(task.id)?.status).toBe('dispatched')
expect(db.getDispatchContextById(ctx.id)?.status).toBe('dispatched')
expect(logs.some((m) => m.includes('expected term_owner'))).toBe(true)
expect(result.status).toBe('completed')
expect(db.getTask(task.id)?.status).toBe('completed')
expect(db.getDispatchContextById(ctx.id)?.status).toBe('completed')
expect(logs.some((m) => m.includes('accepting payload provenance'))).toBe(true)
})
it('can be stopped', async () => {
@@ -29,6 +29,9 @@ export type CoordinatorRuntime = {
behind: number
recentSubjects: string[]
} | null>
// Why: optional so lightweight runtime fakes keep compiling; when present,
// dispatch records the remint-stable pane identity of the assignee.
getTerminalPaneKey?(handle: string): string | null
}
// Why (§3.1): single threshold, no warn/refuse split. Coordinator picked 20
@@ -461,7 +464,11 @@ export class Coordinator {
}
}
const dispatch = this.db.createDispatchContext(task.id, targetHandle)
const dispatch = this.db.createDispatchContext(
task.id,
targetHandle,
this.runtime.getTerminalPaneKey?.(targetHandle) ?? undefined
)
// Why: agents dispatched by the coordinator must use orca-dev in dev mode
// so they talk to the dev runtime's socket, not production (Section 6.4).
+66
View File
@@ -339,6 +339,53 @@ describe('OrchestrationDb', () => {
)
})
// Real leaf UUIDs: pane keys are `${tabId}:${leafUuid}`; only the leaf is
// remint-stable identity (tab half can change on pane break-out).
const LEAF_A = '11111111-1111-1111-8111-111111111111'
const LEAF_B = '22222222-2222-4222-9222-222222222222'
it('rejects dispatch to a reminted handle on a pane with an active dispatch', () => {
const d = createDb()
const t1 = d.createTask({ spec: 'first' })
const t2 = d.createTask({ spec: 'second' })
d.createDispatchContext(t1.id, 'term_old', `tab_1:${LEAF_A}`)
expect(() => d.createDispatchContext(t2.id, 'term_new', `tab_1:${LEAF_A}`)).toThrow(
/already has an active dispatch/
)
})
it('rejects dispatch when pane keys share a leaf after break-out', () => {
const d = createDb()
const t1 = d.createTask({ spec: 'first' })
const t2 = d.createTask({ spec: 'second' })
d.createDispatchContext(t1.id, 'term_old', `tab_1:${LEAF_A}`)
expect(() => d.createDispatchContext(t2.id, 'term_new', `tab_2:${LEAF_A}`)).toThrow(
/already has an active dispatch/
)
})
it('allows concurrent dispatches to different panes', () => {
const d = createDb()
const t1 = d.createTask({ spec: 'first' })
const t2 = d.createTask({ spec: 'second' })
d.createDispatchContext(t1.id, 'term_a', `tab_1:${LEAF_A}`)
expect(() => d.createDispatchContext(t2.id, 'term_b', `tab_1:${LEAF_B}`)).not.toThrow()
})
it('falls back to handle lock when pane keys are missing', () => {
const d = createDb()
const t1 = d.createTask({ spec: 'first' })
const t2 = d.createTask({ spec: 'second' })
d.createDispatchContext(t1.id, 'term_worker')
// New dispatch has a pane key but the active row is legacy (no pane key):
// only handle identity can lock; a different handle is free.
expect(() => d.createDispatchContext(t2.id, 'term_other', `tab_1:${LEAF_A}`)).not.toThrow()
})
it('allows dispatch to a terminal after previous dispatch completes', () => {
const d = createDb()
const t1 = d.createTask({ spec: 'first' })
@@ -822,6 +869,25 @@ describe('OrchestrationDb', () => {
expect(d.getMessageById('msg_v1')?.subject).toBe('pre-migration')
})
it('adds pane-identity columns (v6) and persists them', () => {
const path = createV1Snapshot()
const d = new OrchestrationDb(path)
db = d
const task = d.createTask({ spec: 'work' })
const ctx = d.createDispatchContext(task.id, 'term_a', 'tab_1:leaf_1')
expect(d.getDispatchContextById(ctx.id)?.assignee_pane_key).toBe('tab_1:leaf_1')
const msg = d.insertMessage({
from: 'w',
to: 'c',
subject: 'done',
type: 'worker_done',
senderPaneKey: 'tab_1:leaf_1'
})
expect(d.getMessageById(msg.id)?.sender_pane_key).toBe('tab_1:leaf_1')
})
it('is idempotent: opening an already-migrated DB is a no-op', () => {
const path = createV1Snapshot()
const first = new OrchestrationDb(path)
+91 -16
View File
@@ -15,6 +15,18 @@ import type {
CoordinatorRun
} from './types'
import { buildOrchestrationTaskDisplayMetadata } from '../../../shared/orchestration-task-display'
import { parsePaneKey } from '../../../shared/stable-pane-id'
// Why: leaf UUID is the remint-stable pane identity; the tab half changes on
// break-out. Exact string match covers legacy/unparseable keys.
function isEquivalentPaneKey(a: string, b: string): boolean {
if (a === b) {
return true
}
const aLeaf = parsePaneKey(a)?.leafId
const bLeaf = parsePaneKey(b)?.leafId
return Boolean(aLeaf && bLeaf && aLeaf === bLeaf)
}
export type {
MessageType,
@@ -41,7 +53,10 @@ function generateId(prefix: string): string {
// the terminal that created a task so task-record worktree creation can infer
// the parent workspace even when no dispatch context exists. v4 → v5 adds
// explicit task_title/display_name fields for orchestration worker UI labels.
const SCHEMA_VERSION = 5
// v5 → v6 adds pane-identity columns (dispatch_contexts.assignee_pane_key,
// messages.sender_pane_key) so worker_done ownership survives terminal handle
// remints without accepting completions from unrelated panes.
const SCHEMA_VERSION = 6
export class OrchestrationDb {
private db: Database.Database
@@ -75,7 +90,8 @@ export class OrchestrationDb {
read INTEGER NOT NULL DEFAULT 0,
sequence INTEGER PRIMARY KEY AUTOINCREMENT,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
delivered_at TEXT
delivered_at TEXT,
sender_pane_key TEXT
);
CREATE UNIQUE INDEX IF NOT EXISTS idx_messages_id ON messages(id);
@@ -107,6 +123,7 @@ export class OrchestrationDb {
id TEXT PRIMARY KEY,
task_id TEXT NOT NULL,
assignee_handle TEXT,
assignee_pane_key TEXT,
status TEXT NOT NULL DEFAULT 'pending'
CHECK(status IN ('pending', 'dispatched', 'completed', 'failed', 'circuit_broken')),
failure_count INTEGER NOT NULL DEFAULT 0,
@@ -246,6 +263,14 @@ export class OrchestrationDb {
this.db.exec(`ALTER TABLE tasks ADD COLUMN display_name TEXT`)
}
}
if (current < 6) {
if (!this.hasColumn('dispatch_contexts', 'assignee_pane_key')) {
this.db.exec(`ALTER TABLE dispatch_contexts ADD COLUMN assignee_pane_key TEXT`)
}
if (!this.hasColumn('messages', 'sender_pane_key')) {
this.db.exec(`ALTER TABLE messages ADD COLUMN sender_pane_key TEXT`)
}
}
this.createUndeliveredInboxIndexIfPossible()
this.db.pragma(`user_version = ${SCHEMA_VERSION}`)
@@ -293,11 +318,12 @@ export class OrchestrationDb {
priority?: MessagePriority
threadId?: string
payload?: string
senderPaneKey?: string
}): MessageRow {
const id = generateId('msg')
const stmt = this.db.prepare(`
INSERT INTO messages (id, from_handle, to_handle, subject, body, type, priority, thread_id, payload)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
INSERT INTO messages (id, from_handle, to_handle, subject, body, type, priority, thread_id, payload, sender_pane_key)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
`)
stmt.run(
id,
@@ -308,7 +334,8 @@ export class OrchestrationDb {
msg.type ?? 'status',
msg.priority ?? 'normal',
msg.threadId ?? null,
msg.payload ?? null
msg.payload ?? null,
msg.senderPaneKey ?? null
)
return this.db.prepare('SELECT * FROM messages WHERE id = ?').get(id) as MessageRow
}
@@ -382,6 +409,20 @@ export class OrchestrationDb {
.run(...ids)
}
markAsReadAndDelivered(ids: string[]): void {
if (ids.length === 0) {
return
}
const placeholders = ids.map(() => '?').join(',')
// Why: superseded lifecycle messages stay queryable through history but
// must not be consumed or injected after their dispatch has finished.
this.db
.prepare(
`UPDATE messages SET read = 1, delivered_at = COALESCE(delivered_at, datetime('now')) WHERE id IN (${placeholders})`
)
.run(...ids)
}
getInbox(limit = 20): MessageRow[] {
return this.db
.prepare('SELECT * FROM messages ORDER BY sequence DESC LIMIT ?')
@@ -568,7 +609,14 @@ export class OrchestrationDb {
// ── Dispatch Contexts ──
createDispatchContext(taskId: string, assigneeHandle: string): DispatchContextRow {
createDispatchContext(
taskId: string,
assigneeHandle: string,
// Why: the pane key is the remint-stable identity behind the handle;
// recording it at dispatch time lets worker_done ownership survive
// restarts that reissue the handle.
assigneePaneKey?: string
): DispatchContextRow {
const task = this.getTask(taskId)
if (!task) {
throw new Error(`Task not found: ${taskId}`)
@@ -577,11 +625,11 @@ export class OrchestrationDb {
throw new Error(`Task ${taskId} is ${task.status}; only ready tasks can be dispatched`)
}
const existing = this.db
.prepare(
"SELECT * FROM dispatch_contexts WHERE assignee_handle = ? AND status IN ('pending', 'dispatched')"
)
.get(assigneeHandle) as DispatchContextRow | undefined
// Why: handle match covers legacy rows without pane keys; when both the
// new assignee and an active row have usable pane keys, also lock on
// equivalent pane identity so a reminted handle cannot open a second
// concurrent dispatch on the same pane.
const existing = this.findActiveDispatchForAssignee(assigneeHandle, assigneePaneKey)
if (existing) {
throw new Error(
@@ -599,10 +647,10 @@ export class OrchestrationDb {
const id = generateId('ctx')
this.db
.prepare(
`INSERT INTO dispatch_contexts (id, task_id, assignee_handle, status, failure_count, dispatched_at)
VALUES (?, ?, ?, 'dispatched', ?, datetime('now'))`
`INSERT INTO dispatch_contexts (id, task_id, assignee_handle, assignee_pane_key, status, failure_count, dispatched_at)
VALUES (?, ?, ?, ?, 'dispatched', ?, datetime('now'))`
)
.run(id, taskId, assigneeHandle, priorFailures)
.run(id, taskId, assigneeHandle, assigneePaneKey ?? null, priorFailures)
this.db.prepare("UPDATE tasks SET status = 'dispatched' WHERE id = ?").run(taskId)
@@ -624,11 +672,38 @@ export class OrchestrationDb {
}
getActiveDispatchForTerminal(handle: string): DispatchContextRow | undefined {
return this.db
return this.findActiveDispatchForAssignee(handle)
}
private findActiveDispatchForAssignee(
assigneeHandle: string,
assigneePaneKey?: string
): DispatchContextRow | undefined {
const byHandle = this.db
.prepare(
"SELECT * FROM dispatch_contexts WHERE assignee_handle = ? AND status IN ('pending', 'dispatched') LIMIT 1"
)
.get(handle) as DispatchContextRow | undefined
.get(assigneeHandle) as DispatchContextRow | undefined
if (byHandle) {
return byHandle
}
if (!assigneePaneKey) {
return undefined
}
const actives = this.db
.prepare(
"SELECT * FROM dispatch_contexts WHERE assignee_pane_key IS NOT NULL AND status IN ('pending', 'dispatched')"
)
.all() as DispatchContextRow[]
for (const row of actives) {
if (row.assignee_pane_key && isEquivalentPaneKey(row.assignee_pane_key, assigneePaneKey)) {
return row
}
}
return undefined
}
getLatestDispatchForTerminal(handle: string): DispatchContextRow | undefined {
@@ -17,6 +17,7 @@ function makeMessage(overrides: Partial<MessageRow> = {}): MessageRow {
sequence: 1,
created_at: '2026-01-01T00:00:00Z',
delivered_at: null,
sender_pane_key: null,
...overrides
}
}
@@ -0,0 +1,191 @@
import { afterEach, describe, expect, it } from 'vitest'
import { OrchestrationDb } from './db'
import { reconcileLifecycleMessage } from './lifecycle-reconciliation'
describe('lifecycle reconciliation', () => {
let db: OrchestrationDb
afterEach(() => db?.close())
it('completes an active dispatch from payload IDs after its terminal handle is reminted', () => {
db = new OrchestrationDb(':memory:')
const task = db.createTask({ spec: 'work' })
const dispatch = db.createDispatchContext(task.id, 'term_before_restart')
const logs: string[] = []
const message = db.insertMessage({
from: 'term_after_restart',
to: 'term_coordinator',
subject: 'Done',
type: 'worker_done',
payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id })
})
expect(reconcileLifecycleMessage(db, message, (line) => logs.push(line))).toEqual({
action: 'completed',
taskId: task.id,
dispatchId: dispatch.id
})
expect(db.getTask(task.id)?.status).toBe('completed')
expect(db.getDispatchContextById(dispatch.id)?.status).toBe('completed')
expect(logs.some((line) => line.includes('accepting payload provenance'))).toBe(true)
})
// Real leaf UUIDs: pane keys are `${tabId}:${leafUuid}` and only the leaf
// half is identity (the tab half changes on pane break-out).
const LEAF_A = '11111111-1111-1111-8111-111111111111'
const LEAF_B = '22222222-2222-4222-9222-222222222222'
it('completes worker_done from the dispatched pane after a handle remint', () => {
db = new OrchestrationDb(':memory:')
const task = db.createTask({ spec: 'work' })
const dispatch = db.createDispatchContext(task.id, 'term_before_restart', `tab_w:${LEAF_A}`)
const message = db.insertMessage({
from: 'term_after_restart',
to: 'term_coordinator',
subject: 'Done',
type: 'worker_done',
payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id }),
senderPaneKey: `tab_w:${LEAF_A}`
})
expect(reconcileLifecycleMessage(db, message).action).toBe('completed')
expect(db.getTask(task.id)?.status).toBe('completed')
})
it('completes worker_done from the same leaf after a pane break-out changed the tab half', () => {
db = new OrchestrationDb(':memory:')
const task = db.createTask({ spec: 'work' })
// Dispatch recorded the post-break-out pane key; the worker shell still
// holds the spawn-time key with the old tab id.
const dispatch = db.createDispatchContext(task.id, 'term_before_restart', `tab_new:${LEAF_A}`)
const message = db.insertMessage({
from: 'term_after_restart',
to: 'term_coordinator',
subject: 'Done',
type: 'worker_done',
payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id }),
senderPaneKey: `tab_old:${LEAF_A}`
})
expect(reconcileLifecycleMessage(db, message).action).toBe('completed')
expect(db.getTask(task.id)?.status).toBe('completed')
})
it('completes worker_done when a pane key is unparseable (legacy format)', () => {
db = new OrchestrationDb(':memory:')
const task = db.createTask({ spec: 'work' })
const dispatch = db.createDispatchContext(task.id, 'term_owner', `tab_w:${LEAF_A}`)
const message = db.insertMessage({
from: 'term_reminted',
to: 'term_coordinator',
subject: 'Done',
type: 'worker_done',
payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id }),
senderPaneKey: 'tab_w:42'
})
expect(reconcileLifecycleMessage(db, message).action).toBe('completed')
})
it('ignores worker_done sent from a different pane', () => {
db = new OrchestrationDb(':memory:')
const task = db.createTask({ spec: 'work' })
const dispatch = db.createDispatchContext(task.id, 'term_owner', `tab_w1:${LEAF_A}`)
const logs: string[] = []
const message = db.insertMessage({
from: 'term_other_worker',
to: 'term_coordinator',
subject: 'Done',
type: 'worker_done',
payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id }),
senderPaneKey: `tab_w2:${LEAF_B}`
})
expect(reconcileLifecycleMessage(db, message, (line) => logs.push(line))).toEqual({
action: 'ignored'
})
expect(db.getTask(task.id)?.status).toBe('dispatched')
expect(db.getDispatchContextById(dispatch.id)?.status).toBe('dispatched')
expect(logs.some((line) => line.includes(`expected pane tab_w1:${LEAF_A}`))).toBe(true)
})
it('ignores a heartbeat sent from a different pane without recording liveness', () => {
db = new OrchestrationDb(':memory:')
const task = db.createTask({ spec: 'work' })
const dispatch = db.createDispatchContext(task.id, 'term_owner', `tab_w1:${LEAF_A}`)
const heartbeat = db.insertMessage({
from: 'term_other_worker',
to: 'term_coordinator',
subject: 'alive',
type: 'heartbeat',
payload: JSON.stringify({ dispatchId: dispatch.id }),
senderPaneKey: `tab_w2:${LEAF_B}`
})
expect(reconcileLifecycleMessage(db, heartbeat)).toEqual({ action: 'ignored' })
expect(db.getDispatchContextById(dispatch.id)?.last_heartbeat_at).toBeNull()
})
it('records a heartbeat whose pane key drifted only in the tab half', () => {
db = new OrchestrationDb(':memory:')
const task = db.createTask({ spec: 'work' })
const dispatch = db.createDispatchContext(task.id, 'term_owner', `tab_new:${LEAF_A}`)
const heartbeat = db.insertMessage({
from: 'term_owner',
to: 'term_coordinator',
subject: 'alive',
type: 'heartbeat',
payload: JSON.stringify({ dispatchId: dispatch.id }),
senderPaneKey: `tab_old:${LEAF_A}`
})
expect(reconcileLifecycleMessage(db, heartbeat)).toEqual({
action: 'heartbeat_recorded',
dispatchId: dispatch.id
})
expect(db.getDispatchContextById(dispatch.id)?.last_heartbeat_at).not.toBeNull()
})
it('suppresses same-dispatch heartbeats once worker_done is reconciled', () => {
db = new OrchestrationDb(':memory:')
const task = db.createTask({ spec: 'work' })
const dispatch = db.createDispatchContext(task.id, 'term_worker')
const otherTask = db.createTask({ spec: 'other work' })
const otherDispatch = db.createDispatchContext(otherTask.id, 'term_other')
const insertHeartbeat = (dispatchId: string, from: string) =>
db.insertMessage({
from,
to: 'term_coordinator',
subject: 'alive',
type: 'heartbeat',
payload: JSON.stringify({ dispatchId })
})
const staleHeartbeat = insertHeartbeat(dispatch.id, 'term_worker')
const otherHeartbeat = insertHeartbeat(otherDispatch.id, 'term_other')
reconcileLifecycleMessage(db, staleHeartbeat)
reconcileLifecycleMessage(db, otherHeartbeat)
const done = db.insertMessage({
from: 'term_worker',
to: 'term_coordinator',
subject: 'Done',
type: 'worker_done',
payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id })
})
reconcileLifecycleMessage(db, done)
expect(db.getUnreadMessages('term_coordinator', ['heartbeat']).map((row) => row.id)).toEqual([
otherHeartbeat.id
])
const archived = db
.getAllMessagesForHandle('term_coordinator')
.find((row) => row.id === staleHeartbeat.id)
expect(archived).toMatchObject({ read: 1 })
expect(archived?.delivered_at).not.toBeNull()
const lateHeartbeat = insertHeartbeat(dispatch.id, 'term_worker')
expect(reconcileLifecycleMessage(db, lateHeartbeat)).toEqual({ action: 'suppressed' })
expect(db.getMessageById(lateHeartbeat.id)).toMatchObject({ read: 1 })
expect(db.getMessageById(lateHeartbeat.id)?.delivered_at).not.toBeNull()
})
})
@@ -1,8 +1,26 @@
import type { OrchestrationDb } from './db'
import type { MessageRow } from './types'
import { parsePaneKey } from '../../../shared/stable-pane-id'
// Why: the tab half of a pane key changes when a pane is broken out into its
// own tab, so only the leaf UUID is identity. Reject only when both keys
// parse and name different leaves; an unparseable (e.g. legacy numeric) key
// degrades to payload authority rather than stranding a completion.
function isForeignPane(assigneePaneKey: string, senderPaneKey: string): boolean {
if (assigneePaneKey === senderPaneKey) {
return false
}
const assigneeLeaf = parsePaneKey(assigneePaneKey)?.leafId
const senderLeaf = parsePaneKey(senderPaneKey)?.leafId
return Boolean(assigneeLeaf && senderLeaf && assigneeLeaf !== senderLeaf)
}
export type LifecycleReconciliationResult =
| { action: 'ignored' }
// Why: `suppressed` means the message was consumed at reconcile time (marked
// read); senders must not wake waiters for it, unlike `ignored` rows that
// stay unread and still need delivery.
| { action: 'suppressed' }
| { action: 'completed'; taskId: string; dispatchId: string }
| { action: 'heartbeat_recorded'; dispatchId: string }
@@ -63,6 +81,28 @@ function reconcileHeartbeatMessage(
return { action: 'ignored' }
}
const dispatch = db.getDispatchContextById(dispatchId)
if (!dispatch || dispatch.status !== 'dispatched') {
// Why: an in-flight heartbeat can arrive after completion; retain it for
// audit history without surfacing obsolete liveness to the coordinator.
db.markAsReadAndDelivered([msg.id])
onLog(`Heartbeat for inactive dispatch ${dispatchId} suppressed`)
return { action: 'suppressed' }
}
if (
dispatch.assignee_pane_key &&
msg.sender_pane_key &&
isForeignPane(dispatch.assignee_pane_key, msg.sender_pane_key)
) {
// Why: a wrong-pane heartbeat must not refresh liveness — it would mask
// a hung assignee behind another agent's timer.
onLog(
`Heartbeat for dispatch ${dispatchId} came from pane ${msg.sender_pane_key}, expected pane ${dispatch.assignee_pane_key}; ignored`
)
return { action: 'ignored' }
}
// Why: dispatchId-specific writes let the DB ignore late heartbeats for
// completed/failed retries without masking a newer hung dispatch.
db.recordHeartbeat(dispatchId, msg.created_at)
@@ -112,10 +152,25 @@ function reconcileWorkerDoneMessage(
return { action: 'ignored' }
}
if (dispatch.assignee_handle !== msg.from_handle) {
// Why: pane leaves are the remint-stable identity behind handles. When
// both sides carry one, a foreign leaf is a different pane completing
// someone else's task — reject it; the same leaf is the same pane after
// a handle remint or tab break-out. Without pane data (older CLI,
// sessions without ORCA_PANE_KEY) payload IDs stay the completion
// authority.
if (
dispatch.assignee_pane_key &&
msg.sender_pane_key &&
isForeignPane(dispatch.assignee_pane_key, msg.sender_pane_key)
) {
onLog(
`Warning: worker_done for dispatch ${dispatchId} came from pane ${msg.sender_pane_key}, expected pane ${dispatch.assignee_pane_key}; ignored`
)
return { action: 'ignored' }
}
onLog(
`Warning: worker_done for dispatch ${dispatchId} came from ${msg.from_handle}, expected ${dispatch.assignee_handle ?? '<unknown>'}`
`Warning: worker_done for dispatch ${dispatchId} came from ${msg.from_handle}, expected ${dispatch.assignee_handle ?? '<unknown>'}; accepting payload provenance`
)
return { action: 'ignored' }
}
// Why: `orchestration.send` can release the DB lock before waking the
// coordinator; the later coordinator read still needs to observe completion.
@@ -143,7 +198,26 @@ function reconcileWorkerDoneMessage(
completedAt: new Date().toISOString()
})
db.updateTaskStatus(taskId, 'completed', result)
suppressEarlierHeartbeats(db, msg, dispatchId)
onLog(`Task ${taskId} completed`)
return { action: 'completed', taskId, dispatchId }
}
function suppressEarlierHeartbeats(
db: OrchestrationDb,
workerDone: MessageRow,
dispatchId: string
): void {
const heartbeatIds = db
.getUnreadMessages(workerDone.to_handle, ['heartbeat'])
.filter((message) => {
if (message.sequence >= workerDone.sequence) {
return false
}
const payload = parseObjectPayload(message, () => undefined)
return payload.dispatchId === dispatchId
})
.map((message) => message.id)
db.markAsReadAndDelivered(heartbeatIds)
}
+2
View File
@@ -32,6 +32,7 @@ export type MessageRow = {
sequence: number
created_at: string
delivered_at: string | null
sender_pane_key: string | null
}
export type TaskRow = {
@@ -52,6 +53,7 @@ export type DispatchContextRow = {
id: string
task_id: string
assignee_handle: string | null
assignee_pane_key: string | null
status: DispatchStatus
failure_count: number
last_failure: string | null