mirror of
https://github.com/stablyai/orca.git
synced 2026-10-22 08:03:37 +00:00
fix(orchestration): complete tasks on worker_done + coordinator UX fixes (#8030)
* fix(orchestration): complete worker tasks and improve coordinator UX
* Fix orchestration lifecycle sender resolution and peek/check compat hand
- Lifecycle sends (worker_done/heartbeat) now use ORCA_TERMINAL_HANDLE
verbatim, skipping the liveness probe and pane remint that could
block delivery during restarts or mismatch stale-runtime assignee
handles.
- --peek now round-trips as {peek:true, unread:false} so older runtimes
that strip unknown params degrade to non-destructive "all" instead of
mark-read, with client-side filtering to restore peek semantics and a
clear error when --peek --wait can't be honored.
- Reject combined read-mode flags (--unread/--peek/--all) before calling
the runtime.
- Distinguish suppressed (already-consumed) lifecycle messages from
ignored ones so send doesn't wake --wait waiters for stale heartbeats.
- Fix task summary truncation to avoid splitting UTF-16 surrogate pairs
and to not misreport whitespace normalization as truncation.
* Add shared helper to abbreviate orchestration task specs for brief listi
- Normalizes whitespace and caps spec length at 160 chars, flagging
truncation separately from whitespace-only changes
- Truncates on UTF-16 code point boundaries to avoid splitting
surrogate pairs and emitting malformed strings
* Add pane-key identity to worker_done/heartbeat reconciliation and server
- Records the sender's pane key on messages and dispatch contexts so
worker_done/heartbeat ownership can be verified by the remint-stable
pane leaf instead of the terminal handle, which is reissued across
restarts.
- Rejects lifecycle messages from a genuinely foreign pane while still
tolerating handle remints, tab break-outs, and older CLIs that lack
pane identity.
- Moves task-spec abbreviation server-side (orchestration.taskList
--brief) so full specs no longer cross SSH/relay transports, with a
client-side fallback for older runtimes; consolidates the shared
abbreviation helper under src/shared.
- Adds a stderr warning when a pre-peek runtime's --peek response hits
the 100-row cap, since older unread messages may be missing.
* Isolate ORCA_PANE_KEY in CLI test beforeEach to fix leaked senderPaneKey
Co-authored-by: Orca <help@stably.ai>
* Fix pane-key remint bypassing dispatch mutual-exclusion lock
- Dispatch locking only matched on assignee_handle, so a reminted
terminal handle (tab break-out) could open a second concurrent
dispatch on the same pane.
- Add leaf-UUID-based pane key comparison (parsePaneKey) as a
secondary lock, falling back to exact handle match for legacy
rows without pane keys.
* Update orchestration skill docs for lifecycle authority and CLI flag add
- Clarify that dispatch lifecycle is tied to taskId+dispatchId verified against
the dispatched pane, not the terminal handle, since handles can be reminted
after restart
- Document new `check --peek`/`--all` and `task-list --brief` flags, with
fallback guidance for older CLIs that reject them
- Note that a valid worker_done auto-completes the task/dispatch, so workers
shouldn't also call task-update manually
---------
Co-authored-by: Orca <help@stably.ai>
This commit is contained in:
@@ -144,6 +144,31 @@ describe('Coordinator', () => {
|
||||
expect(runtime.sentMessages.length).toBeGreaterThan(0)
|
||||
})
|
||||
|
||||
it('records the assignee pane key when the runtime can resolve one', async () => {
|
||||
db = new OrchestrationDb(':memory:')
|
||||
const runtime = createMockRuntime()
|
||||
runtime.terminals = [{ handle: 'term_a', worktreeId: 'wt1', connected: true, writable: true }]
|
||||
const withPaneLookup = Object.assign(runtime, {
|
||||
getTerminalPaneKey: (handle: string) => (handle === 'term_a' ? 'tab_a:leaf_a' : null)
|
||||
})
|
||||
|
||||
const task = db.createTask({ spec: 'implement feature' })
|
||||
const coordinator = new Coordinator(db, withPaneLookup, {
|
||||
spec: 'build it',
|
||||
coordinatorHandle: 'coord',
|
||||
pollIntervalMs: 50
|
||||
})
|
||||
const runPromise = coordinator.run()
|
||||
await new Promise((r) => {
|
||||
setTimeout(r, 100)
|
||||
})
|
||||
|
||||
expect(db.getDispatchContext(task.id)?.assignee_pane_key).toBe('tab_a:leaf_a')
|
||||
|
||||
insertWorkerDone(db, { taskId: task.id })
|
||||
await runPromise
|
||||
})
|
||||
|
||||
it('records completedTasks when send reconciled worker_done before coordinator read', async () => {
|
||||
db = new OrchestrationDb(':memory:')
|
||||
const runtime = createMockRuntime()
|
||||
@@ -570,7 +595,7 @@ describe('Coordinator', () => {
|
||||
expect(db.getDispatchContextById(activeCtx.id)?.status).toBe('completed')
|
||||
})
|
||||
|
||||
it('ignores worker_done sent by a terminal that does not own the dispatch', async () => {
|
||||
it('accepts worker_done payload provenance after an assignee handle changes', async () => {
|
||||
db = new OrchestrationDb(':memory:')
|
||||
const runtime = createMockRuntime()
|
||||
const logs: string[] = []
|
||||
@@ -579,9 +604,9 @@ describe('Coordinator', () => {
|
||||
const ctx = db.createDispatchContext(task.id, 'term_owner')
|
||||
|
||||
db.insertMessage({
|
||||
from: 'term_intruder',
|
||||
from: 'term_reminted',
|
||||
to: 'coord',
|
||||
subject: 'Spoofed done',
|
||||
subject: 'Done after restart',
|
||||
type: 'worker_done',
|
||||
payload: JSON.stringify({ taskId: task.id, dispatchId: ctx.id })
|
||||
})
|
||||
@@ -592,16 +617,12 @@ describe('Coordinator', () => {
|
||||
pollIntervalMs: 20,
|
||||
onLog: (m) => logs.push(m)
|
||||
})
|
||||
const runPromise = coordinator.run()
|
||||
await new Promise((r) => {
|
||||
setTimeout(r, 80)
|
||||
})
|
||||
coordinator.stop()
|
||||
await runPromise
|
||||
const result = await coordinator.run()
|
||||
|
||||
expect(db.getTask(task.id)?.status).toBe('dispatched')
|
||||
expect(db.getDispatchContextById(ctx.id)?.status).toBe('dispatched')
|
||||
expect(logs.some((m) => m.includes('expected term_owner'))).toBe(true)
|
||||
expect(result.status).toBe('completed')
|
||||
expect(db.getTask(task.id)?.status).toBe('completed')
|
||||
expect(db.getDispatchContextById(ctx.id)?.status).toBe('completed')
|
||||
expect(logs.some((m) => m.includes('accepting payload provenance'))).toBe(true)
|
||||
})
|
||||
|
||||
it('can be stopped', async () => {
|
||||
|
||||
@@ -29,6 +29,9 @@ export type CoordinatorRuntime = {
|
||||
behind: number
|
||||
recentSubjects: string[]
|
||||
} | null>
|
||||
// Why: optional so lightweight runtime fakes keep compiling; when present,
|
||||
// dispatch records the remint-stable pane identity of the assignee.
|
||||
getTerminalPaneKey?(handle: string): string | null
|
||||
}
|
||||
|
||||
// Why (§3.1): single threshold, no warn/refuse split. Coordinator picked 20
|
||||
@@ -461,7 +464,11 @@ export class Coordinator {
|
||||
}
|
||||
}
|
||||
|
||||
const dispatch = this.db.createDispatchContext(task.id, targetHandle)
|
||||
const dispatch = this.db.createDispatchContext(
|
||||
task.id,
|
||||
targetHandle,
|
||||
this.runtime.getTerminalPaneKey?.(targetHandle) ?? undefined
|
||||
)
|
||||
|
||||
// Why: agents dispatched by the coordinator must use orca-dev in dev mode
|
||||
// so they talk to the dev runtime's socket, not production (Section 6.4).
|
||||
|
||||
@@ -339,6 +339,53 @@ describe('OrchestrationDb', () => {
|
||||
)
|
||||
})
|
||||
|
||||
// Real leaf UUIDs: pane keys are `${tabId}:${leafUuid}`; only the leaf is
|
||||
// remint-stable identity (tab half can change on pane break-out).
|
||||
const LEAF_A = '11111111-1111-1111-8111-111111111111'
|
||||
const LEAF_B = '22222222-2222-4222-9222-222222222222'
|
||||
|
||||
it('rejects dispatch to a reminted handle on a pane with an active dispatch', () => {
|
||||
const d = createDb()
|
||||
const t1 = d.createTask({ spec: 'first' })
|
||||
const t2 = d.createTask({ spec: 'second' })
|
||||
d.createDispatchContext(t1.id, 'term_old', `tab_1:${LEAF_A}`)
|
||||
|
||||
expect(() => d.createDispatchContext(t2.id, 'term_new', `tab_1:${LEAF_A}`)).toThrow(
|
||||
/already has an active dispatch/
|
||||
)
|
||||
})
|
||||
|
||||
it('rejects dispatch when pane keys share a leaf after break-out', () => {
|
||||
const d = createDb()
|
||||
const t1 = d.createTask({ spec: 'first' })
|
||||
const t2 = d.createTask({ spec: 'second' })
|
||||
d.createDispatchContext(t1.id, 'term_old', `tab_1:${LEAF_A}`)
|
||||
|
||||
expect(() => d.createDispatchContext(t2.id, 'term_new', `tab_2:${LEAF_A}`)).toThrow(
|
||||
/already has an active dispatch/
|
||||
)
|
||||
})
|
||||
|
||||
it('allows concurrent dispatches to different panes', () => {
|
||||
const d = createDb()
|
||||
const t1 = d.createTask({ spec: 'first' })
|
||||
const t2 = d.createTask({ spec: 'second' })
|
||||
d.createDispatchContext(t1.id, 'term_a', `tab_1:${LEAF_A}`)
|
||||
|
||||
expect(() => d.createDispatchContext(t2.id, 'term_b', `tab_1:${LEAF_B}`)).not.toThrow()
|
||||
})
|
||||
|
||||
it('falls back to handle lock when pane keys are missing', () => {
|
||||
const d = createDb()
|
||||
const t1 = d.createTask({ spec: 'first' })
|
||||
const t2 = d.createTask({ spec: 'second' })
|
||||
d.createDispatchContext(t1.id, 'term_worker')
|
||||
|
||||
// New dispatch has a pane key but the active row is legacy (no pane key):
|
||||
// only handle identity can lock; a different handle is free.
|
||||
expect(() => d.createDispatchContext(t2.id, 'term_other', `tab_1:${LEAF_A}`)).not.toThrow()
|
||||
})
|
||||
|
||||
it('allows dispatch to a terminal after previous dispatch completes', () => {
|
||||
const d = createDb()
|
||||
const t1 = d.createTask({ spec: 'first' })
|
||||
@@ -822,6 +869,25 @@ describe('OrchestrationDb', () => {
|
||||
expect(d.getMessageById('msg_v1')?.subject).toBe('pre-migration')
|
||||
})
|
||||
|
||||
it('adds pane-identity columns (v6) and persists them', () => {
|
||||
const path = createV1Snapshot()
|
||||
const d = new OrchestrationDb(path)
|
||||
db = d
|
||||
|
||||
const task = d.createTask({ spec: 'work' })
|
||||
const ctx = d.createDispatchContext(task.id, 'term_a', 'tab_1:leaf_1')
|
||||
expect(d.getDispatchContextById(ctx.id)?.assignee_pane_key).toBe('tab_1:leaf_1')
|
||||
|
||||
const msg = d.insertMessage({
|
||||
from: 'w',
|
||||
to: 'c',
|
||||
subject: 'done',
|
||||
type: 'worker_done',
|
||||
senderPaneKey: 'tab_1:leaf_1'
|
||||
})
|
||||
expect(d.getMessageById(msg.id)?.sender_pane_key).toBe('tab_1:leaf_1')
|
||||
})
|
||||
|
||||
it('is idempotent: opening an already-migrated DB is a no-op', () => {
|
||||
const path = createV1Snapshot()
|
||||
const first = new OrchestrationDb(path)
|
||||
|
||||
@@ -15,6 +15,18 @@ import type {
|
||||
CoordinatorRun
|
||||
} from './types'
|
||||
import { buildOrchestrationTaskDisplayMetadata } from '../../../shared/orchestration-task-display'
|
||||
import { parsePaneKey } from '../../../shared/stable-pane-id'
|
||||
|
||||
// Why: leaf UUID is the remint-stable pane identity; the tab half changes on
|
||||
// break-out. Exact string match covers legacy/unparseable keys.
|
||||
function isEquivalentPaneKey(a: string, b: string): boolean {
|
||||
if (a === b) {
|
||||
return true
|
||||
}
|
||||
const aLeaf = parsePaneKey(a)?.leafId
|
||||
const bLeaf = parsePaneKey(b)?.leafId
|
||||
return Boolean(aLeaf && bLeaf && aLeaf === bLeaf)
|
||||
}
|
||||
|
||||
export type {
|
||||
MessageType,
|
||||
@@ -41,7 +53,10 @@ function generateId(prefix: string): string {
|
||||
// the terminal that created a task so task-record worktree creation can infer
|
||||
// the parent workspace even when no dispatch context exists. v4 → v5 adds
|
||||
// explicit task_title/display_name fields for orchestration worker UI labels.
|
||||
const SCHEMA_VERSION = 5
|
||||
// v5 → v6 adds pane-identity columns (dispatch_contexts.assignee_pane_key,
|
||||
// messages.sender_pane_key) so worker_done ownership survives terminal handle
|
||||
// remints without accepting completions from unrelated panes.
|
||||
const SCHEMA_VERSION = 6
|
||||
|
||||
export class OrchestrationDb {
|
||||
private db: Database.Database
|
||||
@@ -75,7 +90,8 @@ export class OrchestrationDb {
|
||||
read INTEGER NOT NULL DEFAULT 0,
|
||||
sequence INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
delivered_at TEXT
|
||||
delivered_at TEXT,
|
||||
sender_pane_key TEXT
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_messages_id ON messages(id);
|
||||
@@ -107,6 +123,7 @@ export class OrchestrationDb {
|
||||
id TEXT PRIMARY KEY,
|
||||
task_id TEXT NOT NULL,
|
||||
assignee_handle TEXT,
|
||||
assignee_pane_key TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'pending'
|
||||
CHECK(status IN ('pending', 'dispatched', 'completed', 'failed', 'circuit_broken')),
|
||||
failure_count INTEGER NOT NULL DEFAULT 0,
|
||||
@@ -246,6 +263,14 @@ export class OrchestrationDb {
|
||||
this.db.exec(`ALTER TABLE tasks ADD COLUMN display_name TEXT`)
|
||||
}
|
||||
}
|
||||
if (current < 6) {
|
||||
if (!this.hasColumn('dispatch_contexts', 'assignee_pane_key')) {
|
||||
this.db.exec(`ALTER TABLE dispatch_contexts ADD COLUMN assignee_pane_key TEXT`)
|
||||
}
|
||||
if (!this.hasColumn('messages', 'sender_pane_key')) {
|
||||
this.db.exec(`ALTER TABLE messages ADD COLUMN sender_pane_key TEXT`)
|
||||
}
|
||||
}
|
||||
this.createUndeliveredInboxIndexIfPossible()
|
||||
|
||||
this.db.pragma(`user_version = ${SCHEMA_VERSION}`)
|
||||
@@ -293,11 +318,12 @@ export class OrchestrationDb {
|
||||
priority?: MessagePriority
|
||||
threadId?: string
|
||||
payload?: string
|
||||
senderPaneKey?: string
|
||||
}): MessageRow {
|
||||
const id = generateId('msg')
|
||||
const stmt = this.db.prepare(`
|
||||
INSERT INTO messages (id, from_handle, to_handle, subject, body, type, priority, thread_id, payload)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
INSERT INTO messages (id, from_handle, to_handle, subject, body, type, priority, thread_id, payload, sender_pane_key)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`)
|
||||
stmt.run(
|
||||
id,
|
||||
@@ -308,7 +334,8 @@ export class OrchestrationDb {
|
||||
msg.type ?? 'status',
|
||||
msg.priority ?? 'normal',
|
||||
msg.threadId ?? null,
|
||||
msg.payload ?? null
|
||||
msg.payload ?? null,
|
||||
msg.senderPaneKey ?? null
|
||||
)
|
||||
return this.db.prepare('SELECT * FROM messages WHERE id = ?').get(id) as MessageRow
|
||||
}
|
||||
@@ -382,6 +409,20 @@ export class OrchestrationDb {
|
||||
.run(...ids)
|
||||
}
|
||||
|
||||
markAsReadAndDelivered(ids: string[]): void {
|
||||
if (ids.length === 0) {
|
||||
return
|
||||
}
|
||||
const placeholders = ids.map(() => '?').join(',')
|
||||
// Why: superseded lifecycle messages stay queryable through history but
|
||||
// must not be consumed or injected after their dispatch has finished.
|
||||
this.db
|
||||
.prepare(
|
||||
`UPDATE messages SET read = 1, delivered_at = COALESCE(delivered_at, datetime('now')) WHERE id IN (${placeholders})`
|
||||
)
|
||||
.run(...ids)
|
||||
}
|
||||
|
||||
getInbox(limit = 20): MessageRow[] {
|
||||
return this.db
|
||||
.prepare('SELECT * FROM messages ORDER BY sequence DESC LIMIT ?')
|
||||
@@ -568,7 +609,14 @@ export class OrchestrationDb {
|
||||
|
||||
// ── Dispatch Contexts ──
|
||||
|
||||
createDispatchContext(taskId: string, assigneeHandle: string): DispatchContextRow {
|
||||
createDispatchContext(
|
||||
taskId: string,
|
||||
assigneeHandle: string,
|
||||
// Why: the pane key is the remint-stable identity behind the handle;
|
||||
// recording it at dispatch time lets worker_done ownership survive
|
||||
// restarts that reissue the handle.
|
||||
assigneePaneKey?: string
|
||||
): DispatchContextRow {
|
||||
const task = this.getTask(taskId)
|
||||
if (!task) {
|
||||
throw new Error(`Task not found: ${taskId}`)
|
||||
@@ -577,11 +625,11 @@ export class OrchestrationDb {
|
||||
throw new Error(`Task ${taskId} is ${task.status}; only ready tasks can be dispatched`)
|
||||
}
|
||||
|
||||
const existing = this.db
|
||||
.prepare(
|
||||
"SELECT * FROM dispatch_contexts WHERE assignee_handle = ? AND status IN ('pending', 'dispatched')"
|
||||
)
|
||||
.get(assigneeHandle) as DispatchContextRow | undefined
|
||||
// Why: handle match covers legacy rows without pane keys; when both the
|
||||
// new assignee and an active row have usable pane keys, also lock on
|
||||
// equivalent pane identity so a reminted handle cannot open a second
|
||||
// concurrent dispatch on the same pane.
|
||||
const existing = this.findActiveDispatchForAssignee(assigneeHandle, assigneePaneKey)
|
||||
|
||||
if (existing) {
|
||||
throw new Error(
|
||||
@@ -599,10 +647,10 @@ export class OrchestrationDb {
|
||||
const id = generateId('ctx')
|
||||
this.db
|
||||
.prepare(
|
||||
`INSERT INTO dispatch_contexts (id, task_id, assignee_handle, status, failure_count, dispatched_at)
|
||||
VALUES (?, ?, ?, 'dispatched', ?, datetime('now'))`
|
||||
`INSERT INTO dispatch_contexts (id, task_id, assignee_handle, assignee_pane_key, status, failure_count, dispatched_at)
|
||||
VALUES (?, ?, ?, ?, 'dispatched', ?, datetime('now'))`
|
||||
)
|
||||
.run(id, taskId, assigneeHandle, priorFailures)
|
||||
.run(id, taskId, assigneeHandle, assigneePaneKey ?? null, priorFailures)
|
||||
|
||||
this.db.prepare("UPDATE tasks SET status = 'dispatched' WHERE id = ?").run(taskId)
|
||||
|
||||
@@ -624,11 +672,38 @@ export class OrchestrationDb {
|
||||
}
|
||||
|
||||
getActiveDispatchForTerminal(handle: string): DispatchContextRow | undefined {
|
||||
return this.db
|
||||
return this.findActiveDispatchForAssignee(handle)
|
||||
}
|
||||
|
||||
private findActiveDispatchForAssignee(
|
||||
assigneeHandle: string,
|
||||
assigneePaneKey?: string
|
||||
): DispatchContextRow | undefined {
|
||||
const byHandle = this.db
|
||||
.prepare(
|
||||
"SELECT * FROM dispatch_contexts WHERE assignee_handle = ? AND status IN ('pending', 'dispatched') LIMIT 1"
|
||||
)
|
||||
.get(handle) as DispatchContextRow | undefined
|
||||
.get(assigneeHandle) as DispatchContextRow | undefined
|
||||
if (byHandle) {
|
||||
return byHandle
|
||||
}
|
||||
|
||||
if (!assigneePaneKey) {
|
||||
return undefined
|
||||
}
|
||||
|
||||
const actives = this.db
|
||||
.prepare(
|
||||
"SELECT * FROM dispatch_contexts WHERE assignee_pane_key IS NOT NULL AND status IN ('pending', 'dispatched')"
|
||||
)
|
||||
.all() as DispatchContextRow[]
|
||||
|
||||
for (const row of actives) {
|
||||
if (row.assignee_pane_key && isEquivalentPaneKey(row.assignee_pane_key, assigneePaneKey)) {
|
||||
return row
|
||||
}
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
getLatestDispatchForTerminal(handle: string): DispatchContextRow | undefined {
|
||||
|
||||
@@ -17,6 +17,7 @@ function makeMessage(overrides: Partial<MessageRow> = {}): MessageRow {
|
||||
sequence: 1,
|
||||
created_at: '2026-01-01T00:00:00Z',
|
||||
delivered_at: null,
|
||||
sender_pane_key: null,
|
||||
...overrides
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import { OrchestrationDb } from './db'
|
||||
import { reconcileLifecycleMessage } from './lifecycle-reconciliation'
|
||||
|
||||
describe('lifecycle reconciliation', () => {
|
||||
let db: OrchestrationDb
|
||||
|
||||
afterEach(() => db?.close())
|
||||
|
||||
it('completes an active dispatch from payload IDs after its terminal handle is reminted', () => {
|
||||
db = new OrchestrationDb(':memory:')
|
||||
const task = db.createTask({ spec: 'work' })
|
||||
const dispatch = db.createDispatchContext(task.id, 'term_before_restart')
|
||||
const logs: string[] = []
|
||||
const message = db.insertMessage({
|
||||
from: 'term_after_restart',
|
||||
to: 'term_coordinator',
|
||||
subject: 'Done',
|
||||
type: 'worker_done',
|
||||
payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id })
|
||||
})
|
||||
|
||||
expect(reconcileLifecycleMessage(db, message, (line) => logs.push(line))).toEqual({
|
||||
action: 'completed',
|
||||
taskId: task.id,
|
||||
dispatchId: dispatch.id
|
||||
})
|
||||
expect(db.getTask(task.id)?.status).toBe('completed')
|
||||
expect(db.getDispatchContextById(dispatch.id)?.status).toBe('completed')
|
||||
expect(logs.some((line) => line.includes('accepting payload provenance'))).toBe(true)
|
||||
})
|
||||
|
||||
// Real leaf UUIDs: pane keys are `${tabId}:${leafUuid}` and only the leaf
|
||||
// half is identity (the tab half changes on pane break-out).
|
||||
const LEAF_A = '11111111-1111-1111-8111-111111111111'
|
||||
const LEAF_B = '22222222-2222-4222-9222-222222222222'
|
||||
|
||||
it('completes worker_done from the dispatched pane after a handle remint', () => {
|
||||
db = new OrchestrationDb(':memory:')
|
||||
const task = db.createTask({ spec: 'work' })
|
||||
const dispatch = db.createDispatchContext(task.id, 'term_before_restart', `tab_w:${LEAF_A}`)
|
||||
const message = db.insertMessage({
|
||||
from: 'term_after_restart',
|
||||
to: 'term_coordinator',
|
||||
subject: 'Done',
|
||||
type: 'worker_done',
|
||||
payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id }),
|
||||
senderPaneKey: `tab_w:${LEAF_A}`
|
||||
})
|
||||
|
||||
expect(reconcileLifecycleMessage(db, message).action).toBe('completed')
|
||||
expect(db.getTask(task.id)?.status).toBe('completed')
|
||||
})
|
||||
|
||||
it('completes worker_done from the same leaf after a pane break-out changed the tab half', () => {
|
||||
db = new OrchestrationDb(':memory:')
|
||||
const task = db.createTask({ spec: 'work' })
|
||||
// Dispatch recorded the post-break-out pane key; the worker shell still
|
||||
// holds the spawn-time key with the old tab id.
|
||||
const dispatch = db.createDispatchContext(task.id, 'term_before_restart', `tab_new:${LEAF_A}`)
|
||||
const message = db.insertMessage({
|
||||
from: 'term_after_restart',
|
||||
to: 'term_coordinator',
|
||||
subject: 'Done',
|
||||
type: 'worker_done',
|
||||
payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id }),
|
||||
senderPaneKey: `tab_old:${LEAF_A}`
|
||||
})
|
||||
|
||||
expect(reconcileLifecycleMessage(db, message).action).toBe('completed')
|
||||
expect(db.getTask(task.id)?.status).toBe('completed')
|
||||
})
|
||||
|
||||
it('completes worker_done when a pane key is unparseable (legacy format)', () => {
|
||||
db = new OrchestrationDb(':memory:')
|
||||
const task = db.createTask({ spec: 'work' })
|
||||
const dispatch = db.createDispatchContext(task.id, 'term_owner', `tab_w:${LEAF_A}`)
|
||||
const message = db.insertMessage({
|
||||
from: 'term_reminted',
|
||||
to: 'term_coordinator',
|
||||
subject: 'Done',
|
||||
type: 'worker_done',
|
||||
payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id }),
|
||||
senderPaneKey: 'tab_w:42'
|
||||
})
|
||||
|
||||
expect(reconcileLifecycleMessage(db, message).action).toBe('completed')
|
||||
})
|
||||
|
||||
it('ignores worker_done sent from a different pane', () => {
|
||||
db = new OrchestrationDb(':memory:')
|
||||
const task = db.createTask({ spec: 'work' })
|
||||
const dispatch = db.createDispatchContext(task.id, 'term_owner', `tab_w1:${LEAF_A}`)
|
||||
const logs: string[] = []
|
||||
const message = db.insertMessage({
|
||||
from: 'term_other_worker',
|
||||
to: 'term_coordinator',
|
||||
subject: 'Done',
|
||||
type: 'worker_done',
|
||||
payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id }),
|
||||
senderPaneKey: `tab_w2:${LEAF_B}`
|
||||
})
|
||||
|
||||
expect(reconcileLifecycleMessage(db, message, (line) => logs.push(line))).toEqual({
|
||||
action: 'ignored'
|
||||
})
|
||||
expect(db.getTask(task.id)?.status).toBe('dispatched')
|
||||
expect(db.getDispatchContextById(dispatch.id)?.status).toBe('dispatched')
|
||||
expect(logs.some((line) => line.includes(`expected pane tab_w1:${LEAF_A}`))).toBe(true)
|
||||
})
|
||||
|
||||
it('ignores a heartbeat sent from a different pane without recording liveness', () => {
|
||||
db = new OrchestrationDb(':memory:')
|
||||
const task = db.createTask({ spec: 'work' })
|
||||
const dispatch = db.createDispatchContext(task.id, 'term_owner', `tab_w1:${LEAF_A}`)
|
||||
const heartbeat = db.insertMessage({
|
||||
from: 'term_other_worker',
|
||||
to: 'term_coordinator',
|
||||
subject: 'alive',
|
||||
type: 'heartbeat',
|
||||
payload: JSON.stringify({ dispatchId: dispatch.id }),
|
||||
senderPaneKey: `tab_w2:${LEAF_B}`
|
||||
})
|
||||
|
||||
expect(reconcileLifecycleMessage(db, heartbeat)).toEqual({ action: 'ignored' })
|
||||
expect(db.getDispatchContextById(dispatch.id)?.last_heartbeat_at).toBeNull()
|
||||
})
|
||||
|
||||
it('records a heartbeat whose pane key drifted only in the tab half', () => {
|
||||
db = new OrchestrationDb(':memory:')
|
||||
const task = db.createTask({ spec: 'work' })
|
||||
const dispatch = db.createDispatchContext(task.id, 'term_owner', `tab_new:${LEAF_A}`)
|
||||
const heartbeat = db.insertMessage({
|
||||
from: 'term_owner',
|
||||
to: 'term_coordinator',
|
||||
subject: 'alive',
|
||||
type: 'heartbeat',
|
||||
payload: JSON.stringify({ dispatchId: dispatch.id }),
|
||||
senderPaneKey: `tab_old:${LEAF_A}`
|
||||
})
|
||||
|
||||
expect(reconcileLifecycleMessage(db, heartbeat)).toEqual({
|
||||
action: 'heartbeat_recorded',
|
||||
dispatchId: dispatch.id
|
||||
})
|
||||
expect(db.getDispatchContextById(dispatch.id)?.last_heartbeat_at).not.toBeNull()
|
||||
})
|
||||
|
||||
it('suppresses same-dispatch heartbeats once worker_done is reconciled', () => {
|
||||
db = new OrchestrationDb(':memory:')
|
||||
const task = db.createTask({ spec: 'work' })
|
||||
const dispatch = db.createDispatchContext(task.id, 'term_worker')
|
||||
const otherTask = db.createTask({ spec: 'other work' })
|
||||
const otherDispatch = db.createDispatchContext(otherTask.id, 'term_other')
|
||||
const insertHeartbeat = (dispatchId: string, from: string) =>
|
||||
db.insertMessage({
|
||||
from,
|
||||
to: 'term_coordinator',
|
||||
subject: 'alive',
|
||||
type: 'heartbeat',
|
||||
payload: JSON.stringify({ dispatchId })
|
||||
})
|
||||
const staleHeartbeat = insertHeartbeat(dispatch.id, 'term_worker')
|
||||
const otherHeartbeat = insertHeartbeat(otherDispatch.id, 'term_other')
|
||||
reconcileLifecycleMessage(db, staleHeartbeat)
|
||||
reconcileLifecycleMessage(db, otherHeartbeat)
|
||||
const done = db.insertMessage({
|
||||
from: 'term_worker',
|
||||
to: 'term_coordinator',
|
||||
subject: 'Done',
|
||||
type: 'worker_done',
|
||||
payload: JSON.stringify({ taskId: task.id, dispatchId: dispatch.id })
|
||||
})
|
||||
|
||||
reconcileLifecycleMessage(db, done)
|
||||
|
||||
expect(db.getUnreadMessages('term_coordinator', ['heartbeat']).map((row) => row.id)).toEqual([
|
||||
otherHeartbeat.id
|
||||
])
|
||||
const archived = db
|
||||
.getAllMessagesForHandle('term_coordinator')
|
||||
.find((row) => row.id === staleHeartbeat.id)
|
||||
expect(archived).toMatchObject({ read: 1 })
|
||||
expect(archived?.delivered_at).not.toBeNull()
|
||||
|
||||
const lateHeartbeat = insertHeartbeat(dispatch.id, 'term_worker')
|
||||
expect(reconcileLifecycleMessage(db, lateHeartbeat)).toEqual({ action: 'suppressed' })
|
||||
expect(db.getMessageById(lateHeartbeat.id)).toMatchObject({ read: 1 })
|
||||
expect(db.getMessageById(lateHeartbeat.id)?.delivered_at).not.toBeNull()
|
||||
})
|
||||
})
|
||||
@@ -1,8 +1,26 @@
|
||||
import type { OrchestrationDb } from './db'
|
||||
import type { MessageRow } from './types'
|
||||
import { parsePaneKey } from '../../../shared/stable-pane-id'
|
||||
|
||||
// Why: the tab half of a pane key changes when a pane is broken out into its
|
||||
// own tab, so only the leaf UUID is identity. Reject only when both keys
|
||||
// parse and name different leaves; an unparseable (e.g. legacy numeric) key
|
||||
// degrades to payload authority rather than stranding a completion.
|
||||
function isForeignPane(assigneePaneKey: string, senderPaneKey: string): boolean {
|
||||
if (assigneePaneKey === senderPaneKey) {
|
||||
return false
|
||||
}
|
||||
const assigneeLeaf = parsePaneKey(assigneePaneKey)?.leafId
|
||||
const senderLeaf = parsePaneKey(senderPaneKey)?.leafId
|
||||
return Boolean(assigneeLeaf && senderLeaf && assigneeLeaf !== senderLeaf)
|
||||
}
|
||||
|
||||
export type LifecycleReconciliationResult =
|
||||
| { action: 'ignored' }
|
||||
// Why: `suppressed` means the message was consumed at reconcile time (marked
|
||||
// read); senders must not wake waiters for it, unlike `ignored` rows that
|
||||
// stay unread and still need delivery.
|
||||
| { action: 'suppressed' }
|
||||
| { action: 'completed'; taskId: string; dispatchId: string }
|
||||
| { action: 'heartbeat_recorded'; dispatchId: string }
|
||||
|
||||
@@ -63,6 +81,28 @@ function reconcileHeartbeatMessage(
|
||||
return { action: 'ignored' }
|
||||
}
|
||||
|
||||
const dispatch = db.getDispatchContextById(dispatchId)
|
||||
if (!dispatch || dispatch.status !== 'dispatched') {
|
||||
// Why: an in-flight heartbeat can arrive after completion; retain it for
|
||||
// audit history without surfacing obsolete liveness to the coordinator.
|
||||
db.markAsReadAndDelivered([msg.id])
|
||||
onLog(`Heartbeat for inactive dispatch ${dispatchId} suppressed`)
|
||||
return { action: 'suppressed' }
|
||||
}
|
||||
|
||||
if (
|
||||
dispatch.assignee_pane_key &&
|
||||
msg.sender_pane_key &&
|
||||
isForeignPane(dispatch.assignee_pane_key, msg.sender_pane_key)
|
||||
) {
|
||||
// Why: a wrong-pane heartbeat must not refresh liveness — it would mask
|
||||
// a hung assignee behind another agent's timer.
|
||||
onLog(
|
||||
`Heartbeat for dispatch ${dispatchId} came from pane ${msg.sender_pane_key}, expected pane ${dispatch.assignee_pane_key}; ignored`
|
||||
)
|
||||
return { action: 'ignored' }
|
||||
}
|
||||
|
||||
// Why: dispatchId-specific writes let the DB ignore late heartbeats for
|
||||
// completed/failed retries without masking a newer hung dispatch.
|
||||
db.recordHeartbeat(dispatchId, msg.created_at)
|
||||
@@ -112,10 +152,25 @@ function reconcileWorkerDoneMessage(
|
||||
return { action: 'ignored' }
|
||||
}
|
||||
if (dispatch.assignee_handle !== msg.from_handle) {
|
||||
// Why: pane leaves are the remint-stable identity behind handles. When
|
||||
// both sides carry one, a foreign leaf is a different pane completing
|
||||
// someone else's task — reject it; the same leaf is the same pane after
|
||||
// a handle remint or tab break-out. Without pane data (older CLI,
|
||||
// sessions without ORCA_PANE_KEY) payload IDs stay the completion
|
||||
// authority.
|
||||
if (
|
||||
dispatch.assignee_pane_key &&
|
||||
msg.sender_pane_key &&
|
||||
isForeignPane(dispatch.assignee_pane_key, msg.sender_pane_key)
|
||||
) {
|
||||
onLog(
|
||||
`Warning: worker_done for dispatch ${dispatchId} came from pane ${msg.sender_pane_key}, expected pane ${dispatch.assignee_pane_key}; ignored`
|
||||
)
|
||||
return { action: 'ignored' }
|
||||
}
|
||||
onLog(
|
||||
`Warning: worker_done for dispatch ${dispatchId} came from ${msg.from_handle}, expected ${dispatch.assignee_handle ?? '<unknown>'}`
|
||||
`Warning: worker_done for dispatch ${dispatchId} came from ${msg.from_handle}, expected ${dispatch.assignee_handle ?? '<unknown>'}; accepting payload provenance`
|
||||
)
|
||||
return { action: 'ignored' }
|
||||
}
|
||||
// Why: `orchestration.send` can release the DB lock before waking the
|
||||
// coordinator; the later coordinator read still needs to observe completion.
|
||||
@@ -143,7 +198,26 @@ function reconcileWorkerDoneMessage(
|
||||
completedAt: new Date().toISOString()
|
||||
})
|
||||
db.updateTaskStatus(taskId, 'completed', result)
|
||||
suppressEarlierHeartbeats(db, msg, dispatchId)
|
||||
|
||||
onLog(`Task ${taskId} completed`)
|
||||
return { action: 'completed', taskId, dispatchId }
|
||||
}
|
||||
|
||||
function suppressEarlierHeartbeats(
|
||||
db: OrchestrationDb,
|
||||
workerDone: MessageRow,
|
||||
dispatchId: string
|
||||
): void {
|
||||
const heartbeatIds = db
|
||||
.getUnreadMessages(workerDone.to_handle, ['heartbeat'])
|
||||
.filter((message) => {
|
||||
if (message.sequence >= workerDone.sequence) {
|
||||
return false
|
||||
}
|
||||
const payload = parseObjectPayload(message, () => undefined)
|
||||
return payload.dispatchId === dispatchId
|
||||
})
|
||||
.map((message) => message.id)
|
||||
db.markAsReadAndDelivered(heartbeatIds)
|
||||
}
|
||||
|
||||
@@ -32,6 +32,7 @@ export type MessageRow = {
|
||||
sequence: number
|
||||
created_at: string
|
||||
delivered_at: string | null
|
||||
sender_pane_key: string | null
|
||||
}
|
||||
|
||||
export type TaskRow = {
|
||||
@@ -52,6 +53,7 @@ export type DispatchContextRow = {
|
||||
id: string
|
||||
task_id: string
|
||||
assignee_handle: string | null
|
||||
assignee_pane_key: string | null
|
||||
status: DispatchStatus
|
||||
failure_count: number
|
||||
last_failure: string | null
|
||||
|
||||
Reference in New Issue
Block a user