diff --git a/src/main/runtime/orca-runtime.test.ts b/src/main/runtime/orca-runtime.test.ts index df63ad52573..83af3e44eb4 100644 --- a/src/main/runtime/orca-runtime.test.ts +++ b/src/main/runtime/orca-runtime.test.ts @@ -24332,6 +24332,257 @@ describe('OrcaRuntimeService', () => { expect(summary).toMatchObject({ hasHostSidebarActivity: true, status: 'working' }) }) + it('rebuilds the orchestration parent link by pane key after a restart remints handles', async () => { + // Why: sidebar lineage must rebuild from pane identity once stored handles remint. + const childLeafId = '44444444-4444-4444-8444-444444444444' + const parentLeafId = '55555555-5555-4555-8555-555555555555' + const childPaneKey = `tab-child:${childLeafId}` + const parentPaneKey = `tab-parent:${parentLeafId}` + const persistedParentPaneKey = `tab-before-breakout:${parentLeafId}` + const now = Date.now() + const runtime = new OrcaRuntimeService(store, undefined, { + getAgentStatusSnapshot: () => [ + { + paneKey: parentPaneKey, + worktreeId: TEST_WORKTREE_ID, + tabId: 'tab-parent', + state: 'working', + prompt: 'coordinate', + agentType: 'claude', + connectionId: null, + receivedAt: now, + stateStartedAt: now - 200 + }, + { + paneKey: childPaneKey, + worktreeId: TEST_WORKTREE_ID, + tabId: 'tab-child', + state: 'working', + prompt: 'do the work', + agentType: 'claude', + connectionId: null, + receivedAt: now, + stateStartedAt: now - 100 + } + ] + }) + // Simulate a restart: no stored handle matches, only the pane key resolves. + runtime.setOrchestrationDb({ + getActiveDispatchForTerminal: vi.fn((_handle: string, paneKey?: string) => + paneKey === childPaneKey + ? { + id: 'ctx-1', + task_id: 'task-1', + assignee_handle: 'term_before_restart', + assignee_pane_key: childPaneKey, + status: 'dispatched' + } + : undefined + ), + getLatestDispatchForTerminal: vi.fn(() => undefined), + getTask: vi.fn(() => ({ + id: 'task-1', + // Stale handle no longer resolves; the pane key is the durable link. + created_by_terminal_handle: 'term_parent_before_restart', + created_by_pane_key: persistedParentPaneKey, + task_title: 'Ship it', + display_name: 'Ship the change', + spec: 'Ship the change' + })), + getActiveCoordinatorRun: vi.fn(() => undefined) + } as never) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { + tabs: [ + { + tabId: 'tab-parent', + worktreeId: TEST_WORKTREE_ID, + title: 'Claude', + activeLeafId: parentLeafId, + layout: null + }, + { + tabId: 'tab-child', + worktreeId: TEST_WORKTREE_ID, + title: 'Claude', + activeLeafId: childLeafId, + layout: null + } + ], + leaves: [ + { + tabId: 'tab-parent', + worktreeId: TEST_WORKTREE_ID, + leafId: parentLeafId, + paneRuntimeId: 1, + ptyId: 'pty-parent' + }, + { + tabId: 'tab-child', + worktreeId: TEST_WORKTREE_ID, + leafId: childLeafId, + paneRuntimeId: 1, + ptyId: 'pty-child' + } + ] + }) + + expect(runtime.getAgentStatusOrchestrationContextForPaneKey(childPaneKey)?.parentPaneKey).toBe( + parentPaneKey + ) + const { worktrees } = await runtime.getWorktreePs() + const summary = worktrees.find((w) => w.worktreeId === TEST_WORKTREE_ID) + const childAgent = summary?.agents?.find((agent) => agent.paneKey === childPaneKey) + expect(childAgent?.parentPaneKey).toBe(parentPaneKey) + }) + + it('never fails a prior dispatch when a new terminal reusing the same pane exits', () => { + // Why: crash detection mutates persistent task state, so it must match the + // exiting terminal strictly by its own handle — a new terminal reusing the + // pane must not fail the previous worker's dispatch. Display grouping stays + // lenient (pane-based, cosmetic); only this mutating path is strict. + const childLeafId = '66666666-6666-4666-8666-666666666666' + const childPaneKey = `tab-reuse:${childLeafId}` + const runtime = new OrcaRuntimeService(store, undefined, { + getAgentStatusSnapshot: () => [] + }) + const graph = (ptyId: string) => ({ + tabs: [ + { + tabId: 'tab-reuse', + worktreeId: TEST_WORKTREE_ID, + title: 'Claude', + activeLeafId: childLeafId, + layout: null + } + ], + leaves: [ + { + tabId: 'tab-reuse', + worktreeId: TEST_WORKTREE_ID, + leafId: childLeafId, + paneRuntimeId: 1, + ptyId + } + ] + }) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, graph('pty-old')) + const oldHandle = runtime.getAgentStatusTerminalHandleForPaneKey(childPaneKey) + expect(oldHandle).toBeTruthy() + const failDispatch = vi.fn() + runtime.setOrchestrationDb({ + // Why: mirror the real DB — an exact handle hit resolves the dispatch, and + // the pane key only serves the lenient (display) fallback. + getActiveDispatchForTerminal: vi.fn((handle: string, paneKey?: string) => + handle === oldHandle || paneKey === childPaneKey + ? { + id: 'ctx-reuse', + task_id: 'task-reuse', + assignee_handle: oldHandle, + assignee_pane_key: childPaneKey, + status: 'dispatched' + } + : undefined + ), + getLatestDispatchForTerminal: vi.fn(() => undefined), + getTask: vi.fn(() => ({ + id: 'task-reuse', + task_title: 'Old work', + display_name: 'Old work', + spec: 'Old work' + })), + getActiveCoordinatorRun: vi.fn(() => undefined), + failDispatch + } as never) + + // The genuine worker's terminal exiting DOES fail its dispatch (exact handle). + runtime.onPtyExit('pty-old', 1) + expect(failDispatch).toHaveBeenCalledWith('ctx-reuse', expect.any(String)) + + // A new, unrelated terminal reuses the pane; its exit must NOT touch the + // prior dispatch — its handle owns nothing. + failDispatch.mockClear() + runtime.syncWindowGraph(1, graph('pty-new')) + runtime.onPtyExit('pty-new', 1) + expect(failDispatch).not.toHaveBeenCalled() + }) + + it('resolves the parent via its live handle when the persisted pane key is a dead legacy key', () => { + const childLeafId = '77777777-7777-4777-8777-777777777777' + const parentLeafId = '88888888-8888-4888-8888-888888888888' + const childPaneKey = `tab-legacy-child:${childLeafId}` + const parentPaneKey = `tab-legacy-parent:${parentLeafId}` + const runtime = new OrcaRuntimeService(store, undefined, { + getAgentStatusSnapshot: () => [] + }) + runtime.attachWindow(1) + runtime.syncWindowGraph(1, { + tabs: [ + { + tabId: 'tab-legacy-parent', + worktreeId: TEST_WORKTREE_ID, + title: 'Claude', + activeLeafId: parentLeafId, + layout: null + }, + { + tabId: 'tab-legacy-child', + worktreeId: TEST_WORKTREE_ID, + title: 'Claude', + activeLeafId: childLeafId, + layout: null + } + ], + leaves: [ + { + tabId: 'tab-legacy-parent', + worktreeId: TEST_WORKTREE_ID, + leafId: parentLeafId, + paneRuntimeId: 1, + ptyId: 'pty-legacy-parent' + }, + { + tabId: 'tab-legacy-child', + worktreeId: TEST_WORKTREE_ID, + leafId: childLeafId, + paneRuntimeId: 2, + ptyId: 'pty-legacy-child' + } + ] + }) + const liveParentHandle = runtime.getAgentStatusTerminalHandleForPaneKey(parentPaneKey) + expect(liveParentHandle).toBeTruthy() + runtime.setOrchestrationDb({ + getActiveDispatchForTerminal: vi.fn((_handle: string, paneKey?: string) => + paneKey === childPaneKey + ? { + id: 'ctx-legacy', + task_id: 'task-legacy', + assignee_handle: 'term_child_before_restart', + assignee_pane_key: childPaneKey, + status: 'dispatched' + } + : undefined + ), + getLatestDispatchForTerminal: vi.fn(() => undefined), + getTask: vi.fn(() => ({ + id: 'task-legacy', + created_by_terminal_handle: liveParentHandle, + // Legacy numeric pane key: unparseable, and no live pane matches it. + created_by_pane_key: 'stale-tab:3', + task_title: 'Legacy parent', + display_name: 'Legacy parent', + spec: 'Legacy parent' + })), + getActiveCoordinatorRun: vi.fn(() => undefined) + } as never) + + expect(runtime.getAgentStatusOrchestrationContextForPaneKey(childPaneKey)?.parentPaneKey).toBe( + parentPaneKey + ) + }) + it('uses mirrored tab ownership after a workspace rename instead of stale hook attribution', async () => { const renamedPath = '/tmp/worktree-renamed' const renamedWorktreeId = `${TEST_REPO_ID}::${renamedPath}` diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 112e5388bfd..a6e26c97559 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -10960,6 +10960,13 @@ export class OrcaRuntimeService { return } + // Why: this path mutates persistent task state (failDispatch), so match + // strictly by the exiting terminal's current handle — never by fuzzy pane + // identity, which could fail an unrelated task that merely reused this + // pane. A heartbeat rebinds the assignee to the live handle (see + // rebindDispatchAssignee), so this resolves the genuine worker once it + // checks in; before that (the brief post-restart window) we no-op and let + // the coordinator's stale-heartbeat timeout be the backstop. const dispatch = this._orchestrationDb.getActiveDispatchForTerminal(handle) if (!dispatch) { return @@ -21080,8 +21087,13 @@ export class OrcaRuntimeService { const terminalParent = await this.resolveWorkspaceParentSelector( `id:${terminal.worktreeId}` ) + // Why: lineage-candidate attribution is advisory (parent selection for + // display), so resolve leniently by durable pane identity; a heartbeat + // rebinds the assignee to the live handle, so this is normally an exact + // handle hit. const activeDispatch = this._orchestrationDb?.getActiveDispatchForTerminal( - input.callerTerminalHandle + input.callerTerminalHandle, + this.getPaneKeyForTerminalHandle(input.callerTerminalHandle) ?? undefined ) const activeRun = this._orchestrationDb?.getActiveCoordinatorRun() if (activeDispatch) { @@ -22863,7 +22875,7 @@ export class OrcaRuntimeService { if (!handle) { return undefined } - return this.getAgentStatusOrchestrationContextForHandle(handle) + return this.getAgentStatusOrchestrationContextForHandle(handle, paneKey) } getAgentStatusTerminalHandleForPaneKey(paneKey: string): string | undefined { @@ -22897,9 +22909,10 @@ export class OrcaRuntimeService { continue } const handle = this.issueHandle(leaf) - const context = this.getAgentStatusOrchestrationContextForHandle(handle, db) + const paneKey = this.makeRuntimePaneKey(leaf) + const context = this.getAgentStatusOrchestrationContextForHandle(handle, paneKey, db) if (context) { - contexts[this.makeRuntimePaneKey(leaf)] = context + contexts[paneKey] = context } } for (const pty of this.ptysById.values()) { @@ -22907,7 +22920,7 @@ export class OrcaRuntimeService { continue } const handle = this.issuePtyHandle(pty) - const context = this.getAgentStatusOrchestrationContextForHandle(handle, db) + const context = this.getAgentStatusOrchestrationContextForHandle(handle, pty.paneKey, db) if (context) { contexts[pty.paneKey] = context } @@ -22917,14 +22930,20 @@ export class OrcaRuntimeService { private getAgentStatusOrchestrationContextForHandle( handle: string, + paneKey?: string, db = this.getOrchestrationDbIfAvailable() ): AgentStatusOrchestrationContext | undefined { - // Why: active dispatches are authoritative for reused terminals. Completed - // context is only useful while the corresponding done/recent row can still - // be visible; after that it would stale-group unrelated future work. + // Why: this is a read-only display path (sidebar lineage grouping), so + // resolve leniently by durable pane identity — handle first, then leaf-UUID + // equivalence for a reminted/broken-out pane. A heartbeat rebinds the + // assignee to the live handle (see rebindDispatchAssignee), so the common + // case is an exact handle hit; the rare pre-heartbeat pane match can only + // mis-group cosmetically and self-heals once the real worker checks in or + // the dispatch closes. State mutation lives on the strict handle-only path + // (failActiveDispatchOnExit), never here. const dispatch = - db?.getActiveDispatchForTerminal?.(handle) ?? - this.getRecentCompletedDispatchForTerminal(handle, db) + db?.getActiveDispatchForTerminal?.(handle, paneKey) ?? + this.getRecentCompletedDispatchForTerminal(handle, paneKey, db) if (!dispatch) { return undefined } @@ -22943,9 +22962,16 @@ export class OrcaRuntimeService { (activeRun?.coordinator_handle && activeRun.coordinator_handle !== handle ? activeRun.coordinator_handle : undefined) - const parentPaneKey = parentTerminalHandle - ? this.getPaneKeyForTerminalHandle(parentTerminalHandle) - : undefined + const persistedParentPaneKey = task?.created_by_pane_key + // Why: breakout changes a pane's tab prefix; emit its current key so the + // sidebar's exact parent-row lookup still succeeds after restart. A live + // resolution (persisted key or handle) beats the raw persisted key, which + // may be stale when the parent pane is gone or legacy-keyed. + const parentPaneKey = + (persistedParentPaneKey ? this.getLiveEquivalentPaneKey(persistedParentPaneKey) : null) ?? + (parentTerminalHandle ? this.getPaneKeyForTerminalHandle(parentTerminalHandle) : null) ?? + persistedParentPaneKey ?? + undefined return { taskId: dispatch.task_id, @@ -22961,9 +22987,10 @@ export class OrcaRuntimeService { private getRecentCompletedDispatchForTerminal( handle: string, + paneKey?: string, db = this.getOrchestrationDbIfAvailable() ): ReturnType { - const dispatch = db?.getLatestDispatchForTerminal?.(handle) + const dispatch = db?.getLatestDispatchForTerminal?.(handle, paneKey) if (dispatch?.status !== 'completed' || !dispatch.completed_at) { return undefined } @@ -23011,6 +23038,45 @@ export class OrcaRuntimeService { return null } + private getLiveEquivalentPaneKey(paneKey: string): string | null { + const leafId = parsePaneKey(paneKey)?.leafId + let equivalentLeafPaneKey: string | null = null + for (const leaf of this.leaves.values()) { + // Why: "live" means a pane that can host an agent row; skip empty leaves. + if (!leaf.ptyId) { + continue + } + const currentPaneKey = this.makeRuntimePaneKey(leaf) + if (currentPaneKey === paneKey) { + return currentPaneKey + } + if (!equivalentLeafPaneKey && leafId && leaf.leafId === leafId) { + equivalentLeafPaneKey = currentPaneKey + } + } + if (equivalentLeafPaneKey) { + return equivalentLeafPaneKey + } + + let equivalentPtyPaneKey: string | null = null + for (const pty of this.ptysById.values()) { + if (pty.paneKey === paneKey) { + return paneKey + } + // Why: without the leafId guard, two unparseable legacy keys would match + // as `undefined === undefined` and adopt an unrelated pane as parent. + if ( + !equivalentPtyPaneKey && + leafId && + pty.paneKey && + parsePaneKey(pty.paneKey)?.leafId === leafId + ) { + equivalentPtyPaneKey = pty.paneKey + } + } + return equivalentPtyPaneKey + } + private getPaneKeyForTerminalHandle(handle: string): string | null { const livePty = this.getLivePtyForHandle(handle) if (livePty?.pty.paneKey) { diff --git a/src/main/runtime/orchestration/db.ts b/src/main/runtime/orchestration/db.ts index 5e8a46738fa..2790913a8fc 100644 --- a/src/main/runtime/orchestration/db.ts +++ b/src/main/runtime/orchestration/db.ts @@ -94,8 +94,13 @@ function exposeMessageListTimestamps(messages: MessageRow[]): MessageRow[] { // explicit task_title/display_name fields for orchestration worker UI labels. // v5 → v6 adds pane-identity columns (dispatch_contexts.assignee_pane_key, // messages.sender_pane_key) so worker_done ownership survives terminal handle -// remints without accepting completions from unrelated panes. -const SCHEMA_VERSION = 6 +// remints without accepting completions from unrelated panes. v6 → v7 adds +// tasks.created_by_pane_key so the parent/child lineage the sidebar renders can +// be rebuilt after a restart remints every terminal handle. v7 → v8 re-runs that +// same column ensure: some long-lived DBs stamped user_version=7 before the v7 +// ALTER shipped (version reuse), which left CREATE/INSERT on created_by_pane_key +// failing while migrate short-circuited. +const SCHEMA_VERSION = 8 export class OrchestrationDb { private db: Database.Database @@ -141,6 +146,7 @@ export class OrchestrationDb { id TEXT PRIMARY KEY, parent_id TEXT, created_by_terminal_handle TEXT, + created_by_pane_key TEXT, task_title TEXT, display_name TEXT, spec TEXT NOT NULL, @@ -203,6 +209,7 @@ export class OrchestrationDb { ); `) this.createUndeliveredInboxIndexIfPossible() + this.createDispatchPaneKeyIndexIfPossible() } // Why: `CREATE TABLE IF NOT EXISTS` is a no-op against an existing on-disk @@ -210,10 +217,14 @@ export class OrchestrationDb { // not reach an upgraded user unless we migrate explicitly. The transaction // guarantees atomicity — a mid-migration crash leaves the DB at the prior // version because `user_version` is bumped only on success. Idempotent - // re-invocation is a no-op (current >= SCHEMA_VERSION short-circuit). + // re-invocation is a no-op (current >= SCHEMA_VERSION short-circuit), except + // for lightweight column repairs that recover version-reuse mistakes. private migrate(): void { const current = this.db.pragma('user_version', { simple: true }) as number if (current >= SCHEMA_VERSION) { + // Why: short-circuit skips versioned steps; still repair columns that may + // be missing when user_version was stamped without the matching ALTER. + this.ensureCreatedByPaneKeyColumn() return } @@ -310,7 +321,12 @@ export class OrchestrationDb { this.db.exec(`ALTER TABLE messages ADD COLUMN sender_pane_key TEXT`) } } + // v6→v7 and v7→v8 both ensure created_by_pane_key (v8 repairs version-reuse). + if (current < 8) { + this.ensureCreatedByPaneKeyColumn() + } this.createUndeliveredInboxIndexIfPossible() + this.createDispatchPaneKeyIndexIfPossible() this.db.pragma(`user_version = ${SCHEMA_VERSION}`) this.db.exec('COMMIT') @@ -320,6 +336,14 @@ export class OrchestrationDb { } } + // Why: INSERT always names created_by_pane_key; a missing column hard-fails + // task-create even when user_version claims the schema is current. + private ensureCreatedByPaneKeyColumn(): void { + if (!this.hasColumn('tasks', 'created_by_pane_key')) { + this.db.exec(`ALTER TABLE tasks ADD COLUMN created_by_pane_key TEXT`) + } + } + private hasColumn(table: string, column: string): boolean { const rows = this.db.pragma(`table_info(${table})`) as { name: string }[] return rows.some((r) => r.name === column) @@ -335,6 +359,16 @@ export class OrchestrationDb { `) } + private createDispatchPaneKeyIndexIfPossible(): void { + if (!this.hasColumn('dispatch_contexts', 'assignee_pane_key')) { + return + } + this.db.exec(` + CREATE INDEX IF NOT EXISTS idx_dispatch_contexts_assignee_pane_key + ON dispatch_contexts(assignee_pane_key) + `) + } + // Why: sqlite_master stores the original CREATE TABLE SQL including the // CHECK clause. Inspecting that text is the cheapest reliable way to tell // whether the pre-rebuild schema already knows about 'heartbeat' without @@ -562,6 +596,8 @@ export class OrchestrationDb { deps?: string[] parentId?: string createdByTerminalHandle?: string + // Why: terminal handles remint on restart, but sidebar lineage must persist. + createdByPaneKey?: string }): TaskRow { const id = generateId('task') const depsJson = JSON.stringify(task.deps ?? []) @@ -574,12 +610,13 @@ export class OrchestrationDb { }) this.db .prepare( - 'INSERT INTO tasks (id, parent_id, created_by_terminal_handle, task_title, display_name, spec, status, deps) VALUES (?, ?, ?, ?, ?, ?, ?, ?)' + 'INSERT INTO tasks (id, parent_id, created_by_terminal_handle, created_by_pane_key, task_title, display_name, spec, status, deps) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)' ) .run( id, task.parentId ?? null, task.createdByTerminalHandle ?? null, + task.createdByPaneKey ?? null, display.taskTitle || null, display.displayName || null, task.spec, @@ -758,8 +795,9 @@ export class OrchestrationDb { | undefined } - getActiveDispatchForTerminal(handle: string): DispatchContextRow | undefined { - return this.findActiveDispatchForAssignee(handle) + getActiveDispatchForTerminal(handle: string, paneKey?: string): DispatchContextRow | undefined { + // Why: the pane key keeps the dispatch resolvable after handle remint. + return this.findActiveDispatchForAssignee(handle, paneKey) } private findActiveDispatchForAssignee( @@ -774,31 +812,73 @@ export class OrchestrationDb { if (byHandle) { return byHandle } + return assigneePaneKey ? this.resolveDispatchByPaneKey(assigneePaneKey, true) : undefined + } - if (!assigneePaneKey) { - return undefined + getLatestDispatchForTerminal(handle: string, paneKey?: string): DispatchContextRow | undefined { + const byHandle = this.db + .prepare( + 'SELECT * FROM dispatch_contexts WHERE assignee_handle = ? ORDER BY rowid DESC LIMIT 1' + ) + .get(handle) as DispatchContextRow | undefined + if (byHandle || !paneKey) { + return byHandle + } + return this.resolveDispatchByPaneKey(paneKey, false) + } + + // Why: the handle remints on restart, so fall back to the durable pane + // identity. Exact `assignee_pane_key` is index-served; the `substr` leaf + // suffix is a rare bridge for pre-heartbeat break-out (a heartbeat rebinds + // `assignee_pane_key` to the current key, so exact match dominates once a + // worker checks in). `activeOnly` scopes to live dispatches. + private resolveDispatchByPaneKey( + paneKey: string, + activeOnly: boolean + ): DispatchContextRow | undefined { + const statusFilter = activeOnly ? " AND status IN ('pending', 'dispatched')" : '' + const exactPane = this.db + .prepare( + `SELECT * FROM dispatch_contexts + WHERE assignee_pane_key = ?${statusFilter} ORDER BY rowid DESC LIMIT 1` + ) + .get(paneKey) as DispatchContextRow | undefined + if (exactPane) { + return exactPane } - const actives = this.db + const leafId = parsePaneKey(paneKey)?.leafId + if (!leafId) { + return undefined + } + const rows = this.db .prepare( - "SELECT * FROM dispatch_contexts WHERE assignee_pane_key IS NOT NULL AND status IN ('pending', 'dispatched')" + `SELECT * FROM dispatch_contexts + WHERE substr(assignee_pane_key, -37) = ?${statusFilter} ORDER BY rowid DESC` ) - .all() as DispatchContextRow[] - - for (const row of actives) { - if (row.assignee_pane_key && isEquivalentPaneKey(row.assignee_pane_key, assigneePaneKey)) { + .all(`:${leafId}`) as DispatchContextRow[] + for (const row of rows) { + if (row.assignee_pane_key && isEquivalentPaneKey(row.assignee_pane_key, paneKey)) { return row } } return undefined } - getLatestDispatchForTerminal(handle: string): DispatchContextRow | undefined { - return this.db + // Why: a heartbeat/worker_done authorized by dispatchId + pane identity + // proves this handle+pane now drives the dispatch. Rebinding the assignee + // re-anchors handle-based resolution after a restart remints the handle (or + // a break-out changes the tab prefix) — durably, with no in-memory tracking. + // Scoped to `dispatched` rows so a late message can't resurrect a closed + // dispatch; COALESCE keeps a missing sender pane key from wiping the stored one. + rebindDispatchAssignee(dispatchId: string, handle: string, paneKey?: string): void { + this.db .prepare( - 'SELECT * FROM dispatch_contexts WHERE assignee_handle = ? ORDER BY rowid DESC LIMIT 1' + `UPDATE dispatch_contexts + SET assignee_handle = ?, assignee_pane_key = COALESCE(?, assignee_pane_key) + WHERE id = ? AND status = 'dispatched'` ) - .get(handle) as DispatchContextRow | undefined + .run(handle, paneKey ?? null, dispatchId) } completeDispatch(ctxId: string): void { diff --git a/src/main/runtime/orchestration/lifecycle-reconciliation.ts b/src/main/runtime/orchestration/lifecycle-reconciliation.ts index 96917babaf6..3a9751d2bcb 100644 --- a/src/main/runtime/orchestration/lifecycle-reconciliation.ts +++ b/src/main/runtime/orchestration/lifecycle-reconciliation.ts @@ -146,6 +146,13 @@ function reconcileHeartbeatMessage( return { action: 'rejected', code: 'sender_not_assignee', reason } } + // Why: the heartbeat's pane authority proves this handle+pane now drives the + // dispatch, so re-anchor the assignee here. This is the durable self-heal + // after a restart remints the worker's handle — handle-based resolution and + // strict crash detection work again from the first post-restart heartbeat, + // with no in-memory tracking. + db.rebindDispatchAssignee(dispatchId, msg.from_handle, msg.sender_pane_key ?? undefined) + // Why: dispatchId-specific writes let the DB ignore late heartbeats for // completed/failed retries without masking a newer hung dispatch. db.recordHeartbeat(dispatchId, msg.created_at) diff --git a/src/main/runtime/orchestration/restart-lineage-survival.test.ts b/src/main/runtime/orchestration/restart-lineage-survival.test.ts new file mode 100644 index 00000000000..bf08c69a7ce --- /dev/null +++ b/src/main/runtime/orchestration/restart-lineage-survival.test.ts @@ -0,0 +1,220 @@ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { afterEach, describe, expect, it } from 'vitest' +import Database from '../../sqlite/sync-database' +import { OrchestrationDb } from './db' +import { reconcileLifecycleMessage } from './lifecycle-reconciliation' + +describe('orchestration lineage survives a restart handle remint', () => { + let db: OrchestrationDb | undefined + let tempDir: string | undefined + + afterEach(() => { + db?.close() + db = undefined + if (tempDir) { + rmSync(tempDir, { recursive: true, force: true }) + tempDir = undefined + } + }) + + function createDb(): OrchestrationDb { + db = new OrchestrationDb(':memory:') + return db + } + + // Real leaf UUIDs: pane keys are `${tabId}:${leafUuid}`. + const LEAF_A = '11111111-1111-1111-8111-111111111111' + const LEAF_B = '22222222-2222-4222-9222-222222222222' + + function createFileDbPath(): string { + tempDir = mkdtempSync(join(tmpdir(), 'orca-restart-lineage-')) + return join(tempDir, 'orchestration.sqlite') + } + + it('reopens persisted creator and active-dispatch pane identities', () => { + const dbPath = createFileDbPath() + const beforeRestart = new OrchestrationDb(dbPath) + const task = beforeRestart.createTask({ + spec: 'work', + createdByTerminalHandle: 'term_parent_before_restart', + createdByPaneKey: `tab_parent_before_breakout:${LEAF_A}` + }) + beforeRestart.createDispatchContext( + task.id, + 'term_child_before_restart', + `tab_child_before_breakout:${LEAF_B}` + ) + beforeRestart.close() + + db = new OrchestrationDb(dbPath) + expect(db.getTask(task.id)?.created_by_pane_key).toBe(`tab_parent_before_breakout:${LEAF_A}`) + expect(db.getActiveDispatchForTerminal('term_child_after_restart')).toBeUndefined() + expect( + db.getActiveDispatchForTerminal( + 'term_child_after_restart', + `tab_child_after_breakout:${LEAF_B}` + )?.task_id + ).toBe(task.id) + }) + + it('rebinds the dispatch assignee to the resumed worker on the first post-restart heartbeat', () => { + // Why: the durable self-heal. A heartbeat authorized by dispatchId + pane + // identity re-anchors the assignee to the reminted handle + current pane + // key, so exact handle-based resolution recovers with no in-memory state. + const d = createDb() + const task = d.createTask({ spec: 'work' }) + const ctx = d.createDispatchContext(task.id, 'term_before_restart', `tab_before:${LEAF_B}`) + + // Pre-heartbeat, only the durable pane identity resolves the dispatch. + expect(d.getActiveDispatchForTerminal('term_after_restart')).toBeUndefined() + + const heartbeat = d.insertMessage({ + from: 'term_after_restart', + to: 'coordinator', + subject: 'hb', + type: 'heartbeat', + payload: JSON.stringify({ dispatchId: ctx.id }), + senderPaneKey: `tab_after:${LEAF_B}` + }) + expect(reconcileLifecycleMessage(d, heartbeat).action).toBe('heartbeat_recorded') + + const rebound = d.getActiveDispatchForTerminal('term_after_restart') + expect(rebound?.id).toBe(ctx.id) + expect(rebound?.assignee_pane_key).toBe(`tab_after:${LEAF_B}`) + // The stale pre-restart handle no longer owns it. + expect(d.getActiveDispatchForTerminal('term_before_restart')).toBeUndefined() + }) + + it('does not rebind a closed dispatch from a late heartbeat', () => { + // Why: rebind is scoped to live dispatches — a straggler heartbeat after + // completion must not resurrect assignee identity on a done row. + const d = createDb() + const task = d.createTask({ spec: 'work' }) + const ctx = d.createDispatchContext(task.id, 'term_old', `tab_before:${LEAF_A}`) + d.completeDispatch(ctx.id) + + d.rebindDispatchAssignee(ctx.id, 'term_new', `tab_after:${LEAF_A}`) + expect(d.getDispatchContextById(ctx.id)?.assignee_handle).toBe('term_old') + }) + + it('finds a completed dispatch by its current pane key after a remint and breakout', () => { + const d = createDb() + const task = d.createTask({ spec: 'work' }) + const ctx = d.createDispatchContext(task.id, 'term_old', `tab_before:${LEAF_B}`) + d.completeDispatch(ctx.id) + + expect(d.getLatestDispatchForTerminal('term_new')).toBeUndefined() + const remint = d.getLatestDispatchForTerminal('term_new', `tab_after:${LEAF_B}`) + expect(remint?.id).toBe(ctx.id) + expect(remint?.status).toBe('completed') + }) + + it('migrates a v6 task table before persisting creator pane identity', () => { + const dbPath = createFileDbPath() + const raw = new Database(dbPath) + raw.exec(` + CREATE TABLE tasks ( + id TEXT PRIMARY KEY, + parent_id TEXT, + created_by_terminal_handle TEXT, + task_title TEXT, + display_name TEXT, + spec TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + deps TEXT NOT NULL DEFAULT '[]', + result TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + completed_at TEXT + ); + INSERT INTO tasks (id, created_by_terminal_handle, spec, status) + VALUES ('task_v6', 'term_legacy', 'legacy work', 'ready'); + `) + raw.pragma('user_version = 6') + raw.close() + + db = new OrchestrationDb(dbPath) + expect(db.getTask('task_v6')?.created_by_pane_key).toBeNull() + const created = db.createTask({ + spec: 'new work', + createdByPaneKey: `tab_parent:${LEAF_A}` + }) + expect(created.created_by_pane_key).toBe(`tab_parent:${LEAF_A}`) + expect(readUserVersion(dbPath)).toBe(8) + }) + + // Why: real orca-dev DBs stamped user_version=7 before created_by_pane_key + // shipped; short-circuit alone left task-create INSERT failing forever. + it('repairs a v7-stamped task table missing created_by_pane_key', () => { + const dbPath = createFileDbPath() + const raw = new Database(dbPath) + raw.exec(` + CREATE TABLE tasks ( + id TEXT PRIMARY KEY, + parent_id TEXT, + created_by_terminal_handle TEXT, + task_title TEXT, + display_name TEXT, + spec TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + deps TEXT NOT NULL DEFAULT '[]', + result TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + completed_at TEXT + ); + INSERT INTO tasks (id, created_by_terminal_handle, spec, status) + VALUES ('task_stuck_v7', 'term_legacy', 'legacy work', 'ready'); + `) + raw.pragma('user_version = 7') + raw.close() + + db = new OrchestrationDb(dbPath) + expect(db.getTask('task_stuck_v7')?.created_by_pane_key).toBeNull() + const created = db.createTask({ + spec: 'repaired work', + createdByPaneKey: `tab_parent:${LEAF_A}` + }) + expect(created.created_by_pane_key).toBe(`tab_parent:${LEAF_A}`) + expect(readUserVersion(dbPath)).toBe(8) + }) + + // Why: even a current stamp can lie; ensureColumn must still heal. + it('repairs a v8-stamped task table missing created_by_pane_key', () => { + const dbPath = createFileDbPath() + const raw = new Database(dbPath) + raw.exec(` + CREATE TABLE tasks ( + id TEXT PRIMARY KEY, + parent_id TEXT, + created_by_terminal_handle TEXT, + task_title TEXT, + display_name TEXT, + spec TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + deps TEXT NOT NULL DEFAULT '[]', + result TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + completed_at TEXT + ); + `) + raw.pragma('user_version = 8') + raw.close() + + db = new OrchestrationDb(dbPath) + const created = db.createTask({ + spec: 'healed work', + createdByPaneKey: `tab_parent:${LEAF_A}` + }) + expect(created.created_by_pane_key).toBe(`tab_parent:${LEAF_A}`) + }) +}) + +function readUserVersion(dbPath: string): number { + const raw = new Database(dbPath) + try { + return raw.pragma('user_version', { simple: true }) as number + } finally { + raw.close() + } +} diff --git a/src/main/runtime/orchestration/types.ts b/src/main/runtime/orchestration/types.ts index 58b86e41b08..0384d882af0 100644 --- a/src/main/runtime/orchestration/types.ts +++ b/src/main/runtime/orchestration/types.ts @@ -39,6 +39,7 @@ export type TaskRow = { id: string parent_id: string | null created_by_terminal_handle: string | null + created_by_pane_key: string | null task_title: string | null display_name: string | null spec: string diff --git a/src/main/runtime/rpc/methods/orchestration.test.ts b/src/main/runtime/rpc/methods/orchestration.test.ts index 95339cb3190..d13127d0d79 100644 --- a/src/main/runtime/rpc/methods/orchestration.test.ts +++ b/src/main/runtime/rpc/methods/orchestration.test.ts @@ -1158,14 +1158,17 @@ describe('orchestration RPC methods', () => { expect(result.task.status).toBe('pending') }) - it('records the caller terminal handle when creating a task', async () => { + it('records the caller terminal handle and pane key when creating a task', async () => { setup() + vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue('tab_creator:leaf_creator') const result = (await call('orchestration.taskCreate', { spec: 'spawn related workspace', callerTerminalHandle: 'term_creator' })) as { task: { id: string } } expect(db.getTask(result.task.id)?.created_by_terminal_handle).toBe('term_creator') + expect(db.getTask(result.task.id)?.created_by_pane_key).toBe('tab_creator:leaf_creator') + expect(runtime.getTerminalPaneKey).toHaveBeenCalledWith('term_creator') }) it('rejects invalid deps JSON', async () => { diff --git a/src/main/runtime/rpc/methods/orchestration.ts b/src/main/runtime/rpc/methods/orchestration.ts index bd2721d1b99..97bcd2b3ca5 100644 --- a/src/main/runtime/rpc/methods/orchestration.ts +++ b/src/main/runtime/rpc/methods/orchestration.ts @@ -425,7 +425,12 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [ displayName: params.displayName, deps, parentId: params.parent, - createdByTerminalHandle: params.callerTerminalHandle + createdByTerminalHandle: params.callerTerminalHandle, + // Why: capture the creator's remint-stable pane key so the sidebar + // parent/child lineage survives a restart that re-mints the handle. + createdByPaneKey: params.callerTerminalHandle + ? (runtime.getTerminalPaneKey(params.callerTerminalHandle) ?? undefined) + : undefined }) return { task } }