Persist orchestration parent-child lineage across handle remints

Store pane keys to rebuild sidebar relationships after restart, then self-heal
handle bindings via heartbeat. Schema v6→v8 including repair for version-reuse.
This commit is contained in:
Jinjing
2026-07-21 18:03:19 -07:00
parent 4d49b9342d
commit 2e37aa215c
8 changed files with 667 additions and 34 deletions
+251
View File
@@ -24332,6 +24332,257 @@ describe('OrcaRuntimeService', () => {
expect(summary).toMatchObject({ hasHostSidebarActivity: true, status: 'working' })
})
it('rebuilds the orchestration parent link by pane key after a restart remints handles', async () => {
// Why: sidebar lineage must rebuild from pane identity once stored handles remint.
const childLeafId = '44444444-4444-4444-8444-444444444444'
const parentLeafId = '55555555-5555-4555-8555-555555555555'
const childPaneKey = `tab-child:${childLeafId}`
const parentPaneKey = `tab-parent:${parentLeafId}`
const persistedParentPaneKey = `tab-before-breakout:${parentLeafId}`
const now = Date.now()
const runtime = new OrcaRuntimeService(store, undefined, {
getAgentStatusSnapshot: () => [
{
paneKey: parentPaneKey,
worktreeId: TEST_WORKTREE_ID,
tabId: 'tab-parent',
state: 'working',
prompt: 'coordinate',
agentType: 'claude',
connectionId: null,
receivedAt: now,
stateStartedAt: now - 200
},
{
paneKey: childPaneKey,
worktreeId: TEST_WORKTREE_ID,
tabId: 'tab-child',
state: 'working',
prompt: 'do the work',
agentType: 'claude',
connectionId: null,
receivedAt: now,
stateStartedAt: now - 100
}
]
})
// Simulate a restart: no stored handle matches, only the pane key resolves.
runtime.setOrchestrationDb({
getActiveDispatchForTerminal: vi.fn((_handle: string, paneKey?: string) =>
paneKey === childPaneKey
? {
id: 'ctx-1',
task_id: 'task-1',
assignee_handle: 'term_before_restart',
assignee_pane_key: childPaneKey,
status: 'dispatched'
}
: undefined
),
getLatestDispatchForTerminal: vi.fn(() => undefined),
getTask: vi.fn(() => ({
id: 'task-1',
// Stale handle no longer resolves; the pane key is the durable link.
created_by_terminal_handle: 'term_parent_before_restart',
created_by_pane_key: persistedParentPaneKey,
task_title: 'Ship it',
display_name: 'Ship the change',
spec: 'Ship the change'
})),
getActiveCoordinatorRun: vi.fn(() => undefined)
} as never)
runtime.attachWindow(1)
runtime.syncWindowGraph(1, {
tabs: [
{
tabId: 'tab-parent',
worktreeId: TEST_WORKTREE_ID,
title: 'Claude',
activeLeafId: parentLeafId,
layout: null
},
{
tabId: 'tab-child',
worktreeId: TEST_WORKTREE_ID,
title: 'Claude',
activeLeafId: childLeafId,
layout: null
}
],
leaves: [
{
tabId: 'tab-parent',
worktreeId: TEST_WORKTREE_ID,
leafId: parentLeafId,
paneRuntimeId: 1,
ptyId: 'pty-parent'
},
{
tabId: 'tab-child',
worktreeId: TEST_WORKTREE_ID,
leafId: childLeafId,
paneRuntimeId: 1,
ptyId: 'pty-child'
}
]
})
expect(runtime.getAgentStatusOrchestrationContextForPaneKey(childPaneKey)?.parentPaneKey).toBe(
parentPaneKey
)
const { worktrees } = await runtime.getWorktreePs()
const summary = worktrees.find((w) => w.worktreeId === TEST_WORKTREE_ID)
const childAgent = summary?.agents?.find((agent) => agent.paneKey === childPaneKey)
expect(childAgent?.parentPaneKey).toBe(parentPaneKey)
})
it('never fails a prior dispatch when a new terminal reusing the same pane exits', () => {
// Why: crash detection mutates persistent task state, so it must match the
// exiting terminal strictly by its own handle — a new terminal reusing the
// pane must not fail the previous worker's dispatch. Display grouping stays
// lenient (pane-based, cosmetic); only this mutating path is strict.
const childLeafId = '66666666-6666-4666-8666-666666666666'
const childPaneKey = `tab-reuse:${childLeafId}`
const runtime = new OrcaRuntimeService(store, undefined, {
getAgentStatusSnapshot: () => []
})
const graph = (ptyId: string) => ({
tabs: [
{
tabId: 'tab-reuse',
worktreeId: TEST_WORKTREE_ID,
title: 'Claude',
activeLeafId: childLeafId,
layout: null
}
],
leaves: [
{
tabId: 'tab-reuse',
worktreeId: TEST_WORKTREE_ID,
leafId: childLeafId,
paneRuntimeId: 1,
ptyId
}
]
})
runtime.attachWindow(1)
runtime.syncWindowGraph(1, graph('pty-old'))
const oldHandle = runtime.getAgentStatusTerminalHandleForPaneKey(childPaneKey)
expect(oldHandle).toBeTruthy()
const failDispatch = vi.fn()
runtime.setOrchestrationDb({
// Why: mirror the real DB — an exact handle hit resolves the dispatch, and
// the pane key only serves the lenient (display) fallback.
getActiveDispatchForTerminal: vi.fn((handle: string, paneKey?: string) =>
handle === oldHandle || paneKey === childPaneKey
? {
id: 'ctx-reuse',
task_id: 'task-reuse',
assignee_handle: oldHandle,
assignee_pane_key: childPaneKey,
status: 'dispatched'
}
: undefined
),
getLatestDispatchForTerminal: vi.fn(() => undefined),
getTask: vi.fn(() => ({
id: 'task-reuse',
task_title: 'Old work',
display_name: 'Old work',
spec: 'Old work'
})),
getActiveCoordinatorRun: vi.fn(() => undefined),
failDispatch
} as never)
// The genuine worker's terminal exiting DOES fail its dispatch (exact handle).
runtime.onPtyExit('pty-old', 1)
expect(failDispatch).toHaveBeenCalledWith('ctx-reuse', expect.any(String))
// A new, unrelated terminal reuses the pane; its exit must NOT touch the
// prior dispatch — its handle owns nothing.
failDispatch.mockClear()
runtime.syncWindowGraph(1, graph('pty-new'))
runtime.onPtyExit('pty-new', 1)
expect(failDispatch).not.toHaveBeenCalled()
})
it('resolves the parent via its live handle when the persisted pane key is a dead legacy key', () => {
const childLeafId = '77777777-7777-4777-8777-777777777777'
const parentLeafId = '88888888-8888-4888-8888-888888888888'
const childPaneKey = `tab-legacy-child:${childLeafId}`
const parentPaneKey = `tab-legacy-parent:${parentLeafId}`
const runtime = new OrcaRuntimeService(store, undefined, {
getAgentStatusSnapshot: () => []
})
runtime.attachWindow(1)
runtime.syncWindowGraph(1, {
tabs: [
{
tabId: 'tab-legacy-parent',
worktreeId: TEST_WORKTREE_ID,
title: 'Claude',
activeLeafId: parentLeafId,
layout: null
},
{
tabId: 'tab-legacy-child',
worktreeId: TEST_WORKTREE_ID,
title: 'Claude',
activeLeafId: childLeafId,
layout: null
}
],
leaves: [
{
tabId: 'tab-legacy-parent',
worktreeId: TEST_WORKTREE_ID,
leafId: parentLeafId,
paneRuntimeId: 1,
ptyId: 'pty-legacy-parent'
},
{
tabId: 'tab-legacy-child',
worktreeId: TEST_WORKTREE_ID,
leafId: childLeafId,
paneRuntimeId: 2,
ptyId: 'pty-legacy-child'
}
]
})
const liveParentHandle = runtime.getAgentStatusTerminalHandleForPaneKey(parentPaneKey)
expect(liveParentHandle).toBeTruthy()
runtime.setOrchestrationDb({
getActiveDispatchForTerminal: vi.fn((_handle: string, paneKey?: string) =>
paneKey === childPaneKey
? {
id: 'ctx-legacy',
task_id: 'task-legacy',
assignee_handle: 'term_child_before_restart',
assignee_pane_key: childPaneKey,
status: 'dispatched'
}
: undefined
),
getLatestDispatchForTerminal: vi.fn(() => undefined),
getTask: vi.fn(() => ({
id: 'task-legacy',
created_by_terminal_handle: liveParentHandle,
// Legacy numeric pane key: unparseable, and no live pane matches it.
created_by_pane_key: 'stale-tab:3',
task_title: 'Legacy parent',
display_name: 'Legacy parent',
spec: 'Legacy parent'
})),
getActiveCoordinatorRun: vi.fn(() => undefined)
} as never)
expect(runtime.getAgentStatusOrchestrationContextForPaneKey(childPaneKey)?.parentPaneKey).toBe(
parentPaneKey
)
})
it('uses mirrored tab ownership after a workspace rename instead of stale hook attribution', async () => {
const renamedPath = '/tmp/worktree-renamed'
const renamedWorktreeId = `${TEST_REPO_ID}::${renamedPath}`
+80 -14
View File
@@ -10960,6 +10960,13 @@ export class OrcaRuntimeService {
return
}
// Why: this path mutates persistent task state (failDispatch), so match
// strictly by the exiting terminal's current handle — never by fuzzy pane
// identity, which could fail an unrelated task that merely reused this
// pane. A heartbeat rebinds the assignee to the live handle (see
// rebindDispatchAssignee), so this resolves the genuine worker once it
// checks in; before that (the brief post-restart window) we no-op and let
// the coordinator's stale-heartbeat timeout be the backstop.
const dispatch = this._orchestrationDb.getActiveDispatchForTerminal(handle)
if (!dispatch) {
return
@@ -21080,8 +21087,13 @@ export class OrcaRuntimeService {
const terminalParent = await this.resolveWorkspaceParentSelector(
`id:${terminal.worktreeId}`
)
// Why: lineage-candidate attribution is advisory (parent selection for
// display), so resolve leniently by durable pane identity; a heartbeat
// rebinds the assignee to the live handle, so this is normally an exact
// handle hit.
const activeDispatch = this._orchestrationDb?.getActiveDispatchForTerminal(
input.callerTerminalHandle
input.callerTerminalHandle,
this.getPaneKeyForTerminalHandle(input.callerTerminalHandle) ?? undefined
)
const activeRun = this._orchestrationDb?.getActiveCoordinatorRun()
if (activeDispatch) {
@@ -22863,7 +22875,7 @@ export class OrcaRuntimeService {
if (!handle) {
return undefined
}
return this.getAgentStatusOrchestrationContextForHandle(handle)
return this.getAgentStatusOrchestrationContextForHandle(handle, paneKey)
}
getAgentStatusTerminalHandleForPaneKey(paneKey: string): string | undefined {
@@ -22897,9 +22909,10 @@ export class OrcaRuntimeService {
continue
}
const handle = this.issueHandle(leaf)
const context = this.getAgentStatusOrchestrationContextForHandle(handle, db)
const paneKey = this.makeRuntimePaneKey(leaf)
const context = this.getAgentStatusOrchestrationContextForHandle(handle, paneKey, db)
if (context) {
contexts[this.makeRuntimePaneKey(leaf)] = context
contexts[paneKey] = context
}
}
for (const pty of this.ptysById.values()) {
@@ -22907,7 +22920,7 @@ export class OrcaRuntimeService {
continue
}
const handle = this.issuePtyHandle(pty)
const context = this.getAgentStatusOrchestrationContextForHandle(handle, db)
const context = this.getAgentStatusOrchestrationContextForHandle(handle, pty.paneKey, db)
if (context) {
contexts[pty.paneKey] = context
}
@@ -22917,14 +22930,20 @@ export class OrcaRuntimeService {
private getAgentStatusOrchestrationContextForHandle(
handle: string,
paneKey?: string,
db = this.getOrchestrationDbIfAvailable()
): AgentStatusOrchestrationContext | undefined {
// Why: active dispatches are authoritative for reused terminals. Completed
// context is only useful while the corresponding done/recent row can still
// be visible; after that it would stale-group unrelated future work.
// Why: this is a read-only display path (sidebar lineage grouping), so
// resolve leniently by durable pane identity — handle first, then leaf-UUID
// equivalence for a reminted/broken-out pane. A heartbeat rebinds the
// assignee to the live handle (see rebindDispatchAssignee), so the common
// case is an exact handle hit; the rare pre-heartbeat pane match can only
// mis-group cosmetically and self-heals once the real worker checks in or
// the dispatch closes. State mutation lives on the strict handle-only path
// (failActiveDispatchOnExit), never here.
const dispatch =
db?.getActiveDispatchForTerminal?.(handle) ??
this.getRecentCompletedDispatchForTerminal(handle, db)
db?.getActiveDispatchForTerminal?.(handle, paneKey) ??
this.getRecentCompletedDispatchForTerminal(handle, paneKey, db)
if (!dispatch) {
return undefined
}
@@ -22943,9 +22962,16 @@ export class OrcaRuntimeService {
(activeRun?.coordinator_handle && activeRun.coordinator_handle !== handle
? activeRun.coordinator_handle
: undefined)
const parentPaneKey = parentTerminalHandle
? this.getPaneKeyForTerminalHandle(parentTerminalHandle)
: undefined
const persistedParentPaneKey = task?.created_by_pane_key
// Why: breakout changes a pane's tab prefix; emit its current key so the
// sidebar's exact parent-row lookup still succeeds after restart. A live
// resolution (persisted key or handle) beats the raw persisted key, which
// may be stale when the parent pane is gone or legacy-keyed.
const parentPaneKey =
(persistedParentPaneKey ? this.getLiveEquivalentPaneKey(persistedParentPaneKey) : null) ??
(parentTerminalHandle ? this.getPaneKeyForTerminalHandle(parentTerminalHandle) : null) ??
persistedParentPaneKey ??
undefined
return {
taskId: dispatch.task_id,
@@ -22961,9 +22987,10 @@ export class OrcaRuntimeService {
private getRecentCompletedDispatchForTerminal(
handle: string,
paneKey?: string,
db = this.getOrchestrationDbIfAvailable()
): ReturnType<OrchestrationDb['getLatestDispatchForTerminal']> {
const dispatch = db?.getLatestDispatchForTerminal?.(handle)
const dispatch = db?.getLatestDispatchForTerminal?.(handle, paneKey)
if (dispatch?.status !== 'completed' || !dispatch.completed_at) {
return undefined
}
@@ -23011,6 +23038,45 @@ export class OrcaRuntimeService {
return null
}
private getLiveEquivalentPaneKey(paneKey: string): string | null {
const leafId = parsePaneKey(paneKey)?.leafId
let equivalentLeafPaneKey: string | null = null
for (const leaf of this.leaves.values()) {
// Why: "live" means a pane that can host an agent row; skip empty leaves.
if (!leaf.ptyId) {
continue
}
const currentPaneKey = this.makeRuntimePaneKey(leaf)
if (currentPaneKey === paneKey) {
return currentPaneKey
}
if (!equivalentLeafPaneKey && leafId && leaf.leafId === leafId) {
equivalentLeafPaneKey = currentPaneKey
}
}
if (equivalentLeafPaneKey) {
return equivalentLeafPaneKey
}
let equivalentPtyPaneKey: string | null = null
for (const pty of this.ptysById.values()) {
if (pty.paneKey === paneKey) {
return paneKey
}
// Why: without the leafId guard, two unparseable legacy keys would match
// as `undefined === undefined` and adopt an unrelated pane as parent.
if (
!equivalentPtyPaneKey &&
leafId &&
pty.paneKey &&
parsePaneKey(pty.paneKey)?.leafId === leafId
) {
equivalentPtyPaneKey = pty.paneKey
}
}
return equivalentPtyPaneKey
}
private getPaneKeyForTerminalHandle(handle: string): string | null {
const livePty = this.getLivePtyForHandle(handle)
if (livePty?.pty.paneKey) {
+98 -18
View File
@@ -94,8 +94,13 @@ function exposeMessageListTimestamps(messages: MessageRow[]): MessageRow[] {
// explicit task_title/display_name fields for orchestration worker UI labels.
// v5 → v6 adds pane-identity columns (dispatch_contexts.assignee_pane_key,
// messages.sender_pane_key) so worker_done ownership survives terminal handle
// remints without accepting completions from unrelated panes.
const SCHEMA_VERSION = 6
// remints without accepting completions from unrelated panes. v6 → v7 adds
// tasks.created_by_pane_key so the parent/child lineage the sidebar renders can
// be rebuilt after a restart remints every terminal handle. v7 → v8 re-runs that
// same column ensure: some long-lived DBs stamped user_version=7 before the v7
// ALTER shipped (version reuse), which left CREATE/INSERT on created_by_pane_key
// failing while migrate short-circuited.
const SCHEMA_VERSION = 8
export class OrchestrationDb {
private db: Database.Database
@@ -141,6 +146,7 @@ export class OrchestrationDb {
id TEXT PRIMARY KEY,
parent_id TEXT,
created_by_terminal_handle TEXT,
created_by_pane_key TEXT,
task_title TEXT,
display_name TEXT,
spec TEXT NOT NULL,
@@ -203,6 +209,7 @@ export class OrchestrationDb {
);
`)
this.createUndeliveredInboxIndexIfPossible()
this.createDispatchPaneKeyIndexIfPossible()
}
// Why: `CREATE TABLE IF NOT EXISTS` is a no-op against an existing on-disk
@@ -210,10 +217,14 @@ export class OrchestrationDb {
// not reach an upgraded user unless we migrate explicitly. The transaction
// guarantees atomicity — a mid-migration crash leaves the DB at the prior
// version because `user_version` is bumped only on success. Idempotent
// re-invocation is a no-op (current >= SCHEMA_VERSION short-circuit).
// re-invocation is a no-op (current >= SCHEMA_VERSION short-circuit), except
// for lightweight column repairs that recover version-reuse mistakes.
private migrate(): void {
const current = this.db.pragma('user_version', { simple: true }) as number
if (current >= SCHEMA_VERSION) {
// Why: short-circuit skips versioned steps; still repair columns that may
// be missing when user_version was stamped without the matching ALTER.
this.ensureCreatedByPaneKeyColumn()
return
}
@@ -310,7 +321,12 @@ export class OrchestrationDb {
this.db.exec(`ALTER TABLE messages ADD COLUMN sender_pane_key TEXT`)
}
}
// v6→v7 and v7→v8 both ensure created_by_pane_key (v8 repairs version-reuse).
if (current < 8) {
this.ensureCreatedByPaneKeyColumn()
}
this.createUndeliveredInboxIndexIfPossible()
this.createDispatchPaneKeyIndexIfPossible()
this.db.pragma(`user_version = ${SCHEMA_VERSION}`)
this.db.exec('COMMIT')
@@ -320,6 +336,14 @@ export class OrchestrationDb {
}
}
// Why: INSERT always names created_by_pane_key; a missing column hard-fails
// task-create even when user_version claims the schema is current.
private ensureCreatedByPaneKeyColumn(): void {
if (!this.hasColumn('tasks', 'created_by_pane_key')) {
this.db.exec(`ALTER TABLE tasks ADD COLUMN created_by_pane_key TEXT`)
}
}
private hasColumn(table: string, column: string): boolean {
const rows = this.db.pragma(`table_info(${table})`) as { name: string }[]
return rows.some((r) => r.name === column)
@@ -335,6 +359,16 @@ export class OrchestrationDb {
`)
}
private createDispatchPaneKeyIndexIfPossible(): void {
if (!this.hasColumn('dispatch_contexts', 'assignee_pane_key')) {
return
}
this.db.exec(`
CREATE INDEX IF NOT EXISTS idx_dispatch_contexts_assignee_pane_key
ON dispatch_contexts(assignee_pane_key)
`)
}
// Why: sqlite_master stores the original CREATE TABLE SQL including the
// CHECK clause. Inspecting that text is the cheapest reliable way to tell
// whether the pre-rebuild schema already knows about 'heartbeat' without
@@ -562,6 +596,8 @@ export class OrchestrationDb {
deps?: string[]
parentId?: string
createdByTerminalHandle?: string
// Why: terminal handles remint on restart, but sidebar lineage must persist.
createdByPaneKey?: string
}): TaskRow {
const id = generateId('task')
const depsJson = JSON.stringify(task.deps ?? [])
@@ -574,12 +610,13 @@ export class OrchestrationDb {
})
this.db
.prepare(
'INSERT INTO tasks (id, parent_id, created_by_terminal_handle, task_title, display_name, spec, status, deps) VALUES (?, ?, ?, ?, ?, ?, ?, ?)'
'INSERT INTO tasks (id, parent_id, created_by_terminal_handle, created_by_pane_key, task_title, display_name, spec, status, deps) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)'
)
.run(
id,
task.parentId ?? null,
task.createdByTerminalHandle ?? null,
task.createdByPaneKey ?? null,
display.taskTitle || null,
display.displayName || null,
task.spec,
@@ -758,8 +795,9 @@ export class OrchestrationDb {
| undefined
}
getActiveDispatchForTerminal(handle: string): DispatchContextRow | undefined {
return this.findActiveDispatchForAssignee(handle)
getActiveDispatchForTerminal(handle: string, paneKey?: string): DispatchContextRow | undefined {
// Why: the pane key keeps the dispatch resolvable after handle remint.
return this.findActiveDispatchForAssignee(handle, paneKey)
}
private findActiveDispatchForAssignee(
@@ -774,31 +812,73 @@ export class OrchestrationDb {
if (byHandle) {
return byHandle
}
return assigneePaneKey ? this.resolveDispatchByPaneKey(assigneePaneKey, true) : undefined
}
if (!assigneePaneKey) {
return undefined
getLatestDispatchForTerminal(handle: string, paneKey?: string): DispatchContextRow | undefined {
const byHandle = this.db
.prepare(
'SELECT * FROM dispatch_contexts WHERE assignee_handle = ? ORDER BY rowid DESC LIMIT 1'
)
.get(handle) as DispatchContextRow | undefined
if (byHandle || !paneKey) {
return byHandle
}
return this.resolveDispatchByPaneKey(paneKey, false)
}
// Why: the handle remints on restart, so fall back to the durable pane
// identity. Exact `assignee_pane_key` is index-served; the `substr` leaf
// suffix is a rare bridge for pre-heartbeat break-out (a heartbeat rebinds
// `assignee_pane_key` to the current key, so exact match dominates once a
// worker checks in). `activeOnly` scopes to live dispatches.
private resolveDispatchByPaneKey(
paneKey: string,
activeOnly: boolean
): DispatchContextRow | undefined {
const statusFilter = activeOnly ? " AND status IN ('pending', 'dispatched')" : ''
const exactPane = this.db
.prepare(
`SELECT * FROM dispatch_contexts
WHERE assignee_pane_key = ?${statusFilter} ORDER BY rowid DESC LIMIT 1`
)
.get(paneKey) as DispatchContextRow | undefined
if (exactPane) {
return exactPane
}
const actives = this.db
const leafId = parsePaneKey(paneKey)?.leafId
if (!leafId) {
return undefined
}
const rows = this.db
.prepare(
"SELECT * FROM dispatch_contexts WHERE assignee_pane_key IS NOT NULL AND status IN ('pending', 'dispatched')"
`SELECT * FROM dispatch_contexts
WHERE substr(assignee_pane_key, -37) = ?${statusFilter} ORDER BY rowid DESC`
)
.all() as DispatchContextRow[]
for (const row of actives) {
if (row.assignee_pane_key && isEquivalentPaneKey(row.assignee_pane_key, assigneePaneKey)) {
.all(`:${leafId}`) as DispatchContextRow[]
for (const row of rows) {
if (row.assignee_pane_key && isEquivalentPaneKey(row.assignee_pane_key, paneKey)) {
return row
}
}
return undefined
}
getLatestDispatchForTerminal(handle: string): DispatchContextRow | undefined {
return this.db
// Why: a heartbeat/worker_done authorized by dispatchId + pane identity
// proves this handle+pane now drives the dispatch. Rebinding the assignee
// re-anchors handle-based resolution after a restart remints the handle (or
// a break-out changes the tab prefix) — durably, with no in-memory tracking.
// Scoped to `dispatched` rows so a late message can't resurrect a closed
// dispatch; COALESCE keeps a missing sender pane key from wiping the stored one.
rebindDispatchAssignee(dispatchId: string, handle: string, paneKey?: string): void {
this.db
.prepare(
'SELECT * FROM dispatch_contexts WHERE assignee_handle = ? ORDER BY rowid DESC LIMIT 1'
`UPDATE dispatch_contexts
SET assignee_handle = ?, assignee_pane_key = COALESCE(?, assignee_pane_key)
WHERE id = ? AND status = 'dispatched'`
)
.get(handle) as DispatchContextRow | undefined
.run(handle, paneKey ?? null, dispatchId)
}
completeDispatch(ctxId: string): void {
@@ -146,6 +146,13 @@ function reconcileHeartbeatMessage(
return { action: 'rejected', code: 'sender_not_assignee', reason }
}
// Why: the heartbeat's pane authority proves this handle+pane now drives the
// dispatch, so re-anchor the assignee here. This is the durable self-heal
// after a restart remints the worker's handle — handle-based resolution and
// strict crash detection work again from the first post-restart heartbeat,
// with no in-memory tracking.
db.rebindDispatchAssignee(dispatchId, msg.from_handle, msg.sender_pane_key ?? undefined)
// Why: dispatchId-specific writes let the DB ignore late heartbeats for
// completed/failed retries without masking a newer hung dispatch.
db.recordHeartbeat(dispatchId, msg.created_at)
@@ -0,0 +1,220 @@
import { mkdtempSync, rmSync } from 'node:fs'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { afterEach, describe, expect, it } from 'vitest'
import Database from '../../sqlite/sync-database'
import { OrchestrationDb } from './db'
import { reconcileLifecycleMessage } from './lifecycle-reconciliation'
describe('orchestration lineage survives a restart handle remint', () => {
let db: OrchestrationDb | undefined
let tempDir: string | undefined
afterEach(() => {
db?.close()
db = undefined
if (tempDir) {
rmSync(tempDir, { recursive: true, force: true })
tempDir = undefined
}
})
function createDb(): OrchestrationDb {
db = new OrchestrationDb(':memory:')
return db
}
// Real leaf UUIDs: pane keys are `${tabId}:${leafUuid}`.
const LEAF_A = '11111111-1111-1111-8111-111111111111'
const LEAF_B = '22222222-2222-4222-9222-222222222222'
function createFileDbPath(): string {
tempDir = mkdtempSync(join(tmpdir(), 'orca-restart-lineage-'))
return join(tempDir, 'orchestration.sqlite')
}
it('reopens persisted creator and active-dispatch pane identities', () => {
const dbPath = createFileDbPath()
const beforeRestart = new OrchestrationDb(dbPath)
const task = beforeRestart.createTask({
spec: 'work',
createdByTerminalHandle: 'term_parent_before_restart',
createdByPaneKey: `tab_parent_before_breakout:${LEAF_A}`
})
beforeRestart.createDispatchContext(
task.id,
'term_child_before_restart',
`tab_child_before_breakout:${LEAF_B}`
)
beforeRestart.close()
db = new OrchestrationDb(dbPath)
expect(db.getTask(task.id)?.created_by_pane_key).toBe(`tab_parent_before_breakout:${LEAF_A}`)
expect(db.getActiveDispatchForTerminal('term_child_after_restart')).toBeUndefined()
expect(
db.getActiveDispatchForTerminal(
'term_child_after_restart',
`tab_child_after_breakout:${LEAF_B}`
)?.task_id
).toBe(task.id)
})
it('rebinds the dispatch assignee to the resumed worker on the first post-restart heartbeat', () => {
// Why: the durable self-heal. A heartbeat authorized by dispatchId + pane
// identity re-anchors the assignee to the reminted handle + current pane
// key, so exact handle-based resolution recovers with no in-memory state.
const d = createDb()
const task = d.createTask({ spec: 'work' })
const ctx = d.createDispatchContext(task.id, 'term_before_restart', `tab_before:${LEAF_B}`)
// Pre-heartbeat, only the durable pane identity resolves the dispatch.
expect(d.getActiveDispatchForTerminal('term_after_restart')).toBeUndefined()
const heartbeat = d.insertMessage({
from: 'term_after_restart',
to: 'coordinator',
subject: 'hb',
type: 'heartbeat',
payload: JSON.stringify({ dispatchId: ctx.id }),
senderPaneKey: `tab_after:${LEAF_B}`
})
expect(reconcileLifecycleMessage(d, heartbeat).action).toBe('heartbeat_recorded')
const rebound = d.getActiveDispatchForTerminal('term_after_restart')
expect(rebound?.id).toBe(ctx.id)
expect(rebound?.assignee_pane_key).toBe(`tab_after:${LEAF_B}`)
// The stale pre-restart handle no longer owns it.
expect(d.getActiveDispatchForTerminal('term_before_restart')).toBeUndefined()
})
it('does not rebind a closed dispatch from a late heartbeat', () => {
// Why: rebind is scoped to live dispatches — a straggler heartbeat after
// completion must not resurrect assignee identity on a done row.
const d = createDb()
const task = d.createTask({ spec: 'work' })
const ctx = d.createDispatchContext(task.id, 'term_old', `tab_before:${LEAF_A}`)
d.completeDispatch(ctx.id)
d.rebindDispatchAssignee(ctx.id, 'term_new', `tab_after:${LEAF_A}`)
expect(d.getDispatchContextById(ctx.id)?.assignee_handle).toBe('term_old')
})
it('finds a completed dispatch by its current pane key after a remint and breakout', () => {
const d = createDb()
const task = d.createTask({ spec: 'work' })
const ctx = d.createDispatchContext(task.id, 'term_old', `tab_before:${LEAF_B}`)
d.completeDispatch(ctx.id)
expect(d.getLatestDispatchForTerminal('term_new')).toBeUndefined()
const remint = d.getLatestDispatchForTerminal('term_new', `tab_after:${LEAF_B}`)
expect(remint?.id).toBe(ctx.id)
expect(remint?.status).toBe('completed')
})
it('migrates a v6 task table before persisting creator pane identity', () => {
const dbPath = createFileDbPath()
const raw = new Database(dbPath)
raw.exec(`
CREATE TABLE tasks (
id TEXT PRIMARY KEY,
parent_id TEXT,
created_by_terminal_handle TEXT,
task_title TEXT,
display_name TEXT,
spec TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending',
deps TEXT NOT NULL DEFAULT '[]',
result TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
completed_at TEXT
);
INSERT INTO tasks (id, created_by_terminal_handle, spec, status)
VALUES ('task_v6', 'term_legacy', 'legacy work', 'ready');
`)
raw.pragma('user_version = 6')
raw.close()
db = new OrchestrationDb(dbPath)
expect(db.getTask('task_v6')?.created_by_pane_key).toBeNull()
const created = db.createTask({
spec: 'new work',
createdByPaneKey: `tab_parent:${LEAF_A}`
})
expect(created.created_by_pane_key).toBe(`tab_parent:${LEAF_A}`)
expect(readUserVersion(dbPath)).toBe(8)
})
// Why: real orca-dev DBs stamped user_version=7 before created_by_pane_key
// shipped; short-circuit alone left task-create INSERT failing forever.
it('repairs a v7-stamped task table missing created_by_pane_key', () => {
const dbPath = createFileDbPath()
const raw = new Database(dbPath)
raw.exec(`
CREATE TABLE tasks (
id TEXT PRIMARY KEY,
parent_id TEXT,
created_by_terminal_handle TEXT,
task_title TEXT,
display_name TEXT,
spec TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending',
deps TEXT NOT NULL DEFAULT '[]',
result TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
completed_at TEXT
);
INSERT INTO tasks (id, created_by_terminal_handle, spec, status)
VALUES ('task_stuck_v7', 'term_legacy', 'legacy work', 'ready');
`)
raw.pragma('user_version = 7')
raw.close()
db = new OrchestrationDb(dbPath)
expect(db.getTask('task_stuck_v7')?.created_by_pane_key).toBeNull()
const created = db.createTask({
spec: 'repaired work',
createdByPaneKey: `tab_parent:${LEAF_A}`
})
expect(created.created_by_pane_key).toBe(`tab_parent:${LEAF_A}`)
expect(readUserVersion(dbPath)).toBe(8)
})
// Why: even a current stamp can lie; ensureColumn must still heal.
it('repairs a v8-stamped task table missing created_by_pane_key', () => {
const dbPath = createFileDbPath()
const raw = new Database(dbPath)
raw.exec(`
CREATE TABLE tasks (
id TEXT PRIMARY KEY,
parent_id TEXT,
created_by_terminal_handle TEXT,
task_title TEXT,
display_name TEXT,
spec TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending',
deps TEXT NOT NULL DEFAULT '[]',
result TEXT,
created_at TEXT NOT NULL DEFAULT (datetime('now')),
completed_at TEXT
);
`)
raw.pragma('user_version = 8')
raw.close()
db = new OrchestrationDb(dbPath)
const created = db.createTask({
spec: 'healed work',
createdByPaneKey: `tab_parent:${LEAF_A}`
})
expect(created.created_by_pane_key).toBe(`tab_parent:${LEAF_A}`)
})
})
function readUserVersion(dbPath: string): number {
const raw = new Database(dbPath)
try {
return raw.pragma('user_version', { simple: true }) as number
} finally {
raw.close()
}
}
+1
View File
@@ -39,6 +39,7 @@ export type TaskRow = {
id: string
parent_id: string | null
created_by_terminal_handle: string | null
created_by_pane_key: string | null
task_title: string | null
display_name: string | null
spec: string
@@ -1158,14 +1158,17 @@ describe('orchestration RPC methods', () => {
expect(result.task.status).toBe('pending')
})
it('records the caller terminal handle when creating a task', async () => {
it('records the caller terminal handle and pane key when creating a task', async () => {
setup()
vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue('tab_creator:leaf_creator')
const result = (await call('orchestration.taskCreate', {
spec: 'spawn related workspace',
callerTerminalHandle: 'term_creator'
})) as { task: { id: string } }
expect(db.getTask(result.task.id)?.created_by_terminal_handle).toBe('term_creator')
expect(db.getTask(result.task.id)?.created_by_pane_key).toBe('tab_creator:leaf_creator')
expect(runtime.getTerminalPaneKey).toHaveBeenCalledWith('term_creator')
})
it('rejects invalid deps JSON', async () => {
@@ -425,7 +425,12 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
displayName: params.displayName,
deps,
parentId: params.parent,
createdByTerminalHandle: params.callerTerminalHandle
createdByTerminalHandle: params.callerTerminalHandle,
// Why: capture the creator's remint-stable pane key so the sidebar
// parent/child lineage survives a restart that re-mints the handle.
createdByPaneKey: params.callerTerminalHandle
? (runtime.getTerminalPaneKey(params.callerTerminalHandle) ?? undefined)
: undefined
})
return { task }
}