mirror of
https://github.com/stablyai/orca.git
synced 2026-10-28 16:02:55 +00:00
Persist orchestration parent-child lineage across handle remints
Store pane keys to rebuild sidebar relationships after restart, then self-heal handle bindings via heartbeat. Schema v6→v8 including repair for version-reuse.
This commit is contained in:
@@ -24332,6 +24332,257 @@ describe('OrcaRuntimeService', () => {
|
||||
expect(summary).toMatchObject({ hasHostSidebarActivity: true, status: 'working' })
|
||||
})
|
||||
|
||||
it('rebuilds the orchestration parent link by pane key after a restart remints handles', async () => {
|
||||
// Why: sidebar lineage must rebuild from pane identity once stored handles remint.
|
||||
const childLeafId = '44444444-4444-4444-8444-444444444444'
|
||||
const parentLeafId = '55555555-5555-4555-8555-555555555555'
|
||||
const childPaneKey = `tab-child:${childLeafId}`
|
||||
const parentPaneKey = `tab-parent:${parentLeafId}`
|
||||
const persistedParentPaneKey = `tab-before-breakout:${parentLeafId}`
|
||||
const now = Date.now()
|
||||
const runtime = new OrcaRuntimeService(store, undefined, {
|
||||
getAgentStatusSnapshot: () => [
|
||||
{
|
||||
paneKey: parentPaneKey,
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
tabId: 'tab-parent',
|
||||
state: 'working',
|
||||
prompt: 'coordinate',
|
||||
agentType: 'claude',
|
||||
connectionId: null,
|
||||
receivedAt: now,
|
||||
stateStartedAt: now - 200
|
||||
},
|
||||
{
|
||||
paneKey: childPaneKey,
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
tabId: 'tab-child',
|
||||
state: 'working',
|
||||
prompt: 'do the work',
|
||||
agentType: 'claude',
|
||||
connectionId: null,
|
||||
receivedAt: now,
|
||||
stateStartedAt: now - 100
|
||||
}
|
||||
]
|
||||
})
|
||||
// Simulate a restart: no stored handle matches, only the pane key resolves.
|
||||
runtime.setOrchestrationDb({
|
||||
getActiveDispatchForTerminal: vi.fn((_handle: string, paneKey?: string) =>
|
||||
paneKey === childPaneKey
|
||||
? {
|
||||
id: 'ctx-1',
|
||||
task_id: 'task-1',
|
||||
assignee_handle: 'term_before_restart',
|
||||
assignee_pane_key: childPaneKey,
|
||||
status: 'dispatched'
|
||||
}
|
||||
: undefined
|
||||
),
|
||||
getLatestDispatchForTerminal: vi.fn(() => undefined),
|
||||
getTask: vi.fn(() => ({
|
||||
id: 'task-1',
|
||||
// Stale handle no longer resolves; the pane key is the durable link.
|
||||
created_by_terminal_handle: 'term_parent_before_restart',
|
||||
created_by_pane_key: persistedParentPaneKey,
|
||||
task_title: 'Ship it',
|
||||
display_name: 'Ship the change',
|
||||
spec: 'Ship the change'
|
||||
})),
|
||||
getActiveCoordinatorRun: vi.fn(() => undefined)
|
||||
} as never)
|
||||
runtime.attachWindow(1)
|
||||
runtime.syncWindowGraph(1, {
|
||||
tabs: [
|
||||
{
|
||||
tabId: 'tab-parent',
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
title: 'Claude',
|
||||
activeLeafId: parentLeafId,
|
||||
layout: null
|
||||
},
|
||||
{
|
||||
tabId: 'tab-child',
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
title: 'Claude',
|
||||
activeLeafId: childLeafId,
|
||||
layout: null
|
||||
}
|
||||
],
|
||||
leaves: [
|
||||
{
|
||||
tabId: 'tab-parent',
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
leafId: parentLeafId,
|
||||
paneRuntimeId: 1,
|
||||
ptyId: 'pty-parent'
|
||||
},
|
||||
{
|
||||
tabId: 'tab-child',
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
leafId: childLeafId,
|
||||
paneRuntimeId: 1,
|
||||
ptyId: 'pty-child'
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
expect(runtime.getAgentStatusOrchestrationContextForPaneKey(childPaneKey)?.parentPaneKey).toBe(
|
||||
parentPaneKey
|
||||
)
|
||||
const { worktrees } = await runtime.getWorktreePs()
|
||||
const summary = worktrees.find((w) => w.worktreeId === TEST_WORKTREE_ID)
|
||||
const childAgent = summary?.agents?.find((agent) => agent.paneKey === childPaneKey)
|
||||
expect(childAgent?.parentPaneKey).toBe(parentPaneKey)
|
||||
})
|
||||
|
||||
it('never fails a prior dispatch when a new terminal reusing the same pane exits', () => {
|
||||
// Why: crash detection mutates persistent task state, so it must match the
|
||||
// exiting terminal strictly by its own handle — a new terminal reusing the
|
||||
// pane must not fail the previous worker's dispatch. Display grouping stays
|
||||
// lenient (pane-based, cosmetic); only this mutating path is strict.
|
||||
const childLeafId = '66666666-6666-4666-8666-666666666666'
|
||||
const childPaneKey = `tab-reuse:${childLeafId}`
|
||||
const runtime = new OrcaRuntimeService(store, undefined, {
|
||||
getAgentStatusSnapshot: () => []
|
||||
})
|
||||
const graph = (ptyId: string) => ({
|
||||
tabs: [
|
||||
{
|
||||
tabId: 'tab-reuse',
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
title: 'Claude',
|
||||
activeLeafId: childLeafId,
|
||||
layout: null
|
||||
}
|
||||
],
|
||||
leaves: [
|
||||
{
|
||||
tabId: 'tab-reuse',
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
leafId: childLeafId,
|
||||
paneRuntimeId: 1,
|
||||
ptyId
|
||||
}
|
||||
]
|
||||
})
|
||||
runtime.attachWindow(1)
|
||||
runtime.syncWindowGraph(1, graph('pty-old'))
|
||||
const oldHandle = runtime.getAgentStatusTerminalHandleForPaneKey(childPaneKey)
|
||||
expect(oldHandle).toBeTruthy()
|
||||
const failDispatch = vi.fn()
|
||||
runtime.setOrchestrationDb({
|
||||
// Why: mirror the real DB — an exact handle hit resolves the dispatch, and
|
||||
// the pane key only serves the lenient (display) fallback.
|
||||
getActiveDispatchForTerminal: vi.fn((handle: string, paneKey?: string) =>
|
||||
handle === oldHandle || paneKey === childPaneKey
|
||||
? {
|
||||
id: 'ctx-reuse',
|
||||
task_id: 'task-reuse',
|
||||
assignee_handle: oldHandle,
|
||||
assignee_pane_key: childPaneKey,
|
||||
status: 'dispatched'
|
||||
}
|
||||
: undefined
|
||||
),
|
||||
getLatestDispatchForTerminal: vi.fn(() => undefined),
|
||||
getTask: vi.fn(() => ({
|
||||
id: 'task-reuse',
|
||||
task_title: 'Old work',
|
||||
display_name: 'Old work',
|
||||
spec: 'Old work'
|
||||
})),
|
||||
getActiveCoordinatorRun: vi.fn(() => undefined),
|
||||
failDispatch
|
||||
} as never)
|
||||
|
||||
// The genuine worker's terminal exiting DOES fail its dispatch (exact handle).
|
||||
runtime.onPtyExit('pty-old', 1)
|
||||
expect(failDispatch).toHaveBeenCalledWith('ctx-reuse', expect.any(String))
|
||||
|
||||
// A new, unrelated terminal reuses the pane; its exit must NOT touch the
|
||||
// prior dispatch — its handle owns nothing.
|
||||
failDispatch.mockClear()
|
||||
runtime.syncWindowGraph(1, graph('pty-new'))
|
||||
runtime.onPtyExit('pty-new', 1)
|
||||
expect(failDispatch).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('resolves the parent via its live handle when the persisted pane key is a dead legacy key', () => {
|
||||
const childLeafId = '77777777-7777-4777-8777-777777777777'
|
||||
const parentLeafId = '88888888-8888-4888-8888-888888888888'
|
||||
const childPaneKey = `tab-legacy-child:${childLeafId}`
|
||||
const parentPaneKey = `tab-legacy-parent:${parentLeafId}`
|
||||
const runtime = new OrcaRuntimeService(store, undefined, {
|
||||
getAgentStatusSnapshot: () => []
|
||||
})
|
||||
runtime.attachWindow(1)
|
||||
runtime.syncWindowGraph(1, {
|
||||
tabs: [
|
||||
{
|
||||
tabId: 'tab-legacy-parent',
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
title: 'Claude',
|
||||
activeLeafId: parentLeafId,
|
||||
layout: null
|
||||
},
|
||||
{
|
||||
tabId: 'tab-legacy-child',
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
title: 'Claude',
|
||||
activeLeafId: childLeafId,
|
||||
layout: null
|
||||
}
|
||||
],
|
||||
leaves: [
|
||||
{
|
||||
tabId: 'tab-legacy-parent',
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
leafId: parentLeafId,
|
||||
paneRuntimeId: 1,
|
||||
ptyId: 'pty-legacy-parent'
|
||||
},
|
||||
{
|
||||
tabId: 'tab-legacy-child',
|
||||
worktreeId: TEST_WORKTREE_ID,
|
||||
leafId: childLeafId,
|
||||
paneRuntimeId: 2,
|
||||
ptyId: 'pty-legacy-child'
|
||||
}
|
||||
]
|
||||
})
|
||||
const liveParentHandle = runtime.getAgentStatusTerminalHandleForPaneKey(parentPaneKey)
|
||||
expect(liveParentHandle).toBeTruthy()
|
||||
runtime.setOrchestrationDb({
|
||||
getActiveDispatchForTerminal: vi.fn((_handle: string, paneKey?: string) =>
|
||||
paneKey === childPaneKey
|
||||
? {
|
||||
id: 'ctx-legacy',
|
||||
task_id: 'task-legacy',
|
||||
assignee_handle: 'term_child_before_restart',
|
||||
assignee_pane_key: childPaneKey,
|
||||
status: 'dispatched'
|
||||
}
|
||||
: undefined
|
||||
),
|
||||
getLatestDispatchForTerminal: vi.fn(() => undefined),
|
||||
getTask: vi.fn(() => ({
|
||||
id: 'task-legacy',
|
||||
created_by_terminal_handle: liveParentHandle,
|
||||
// Legacy numeric pane key: unparseable, and no live pane matches it.
|
||||
created_by_pane_key: 'stale-tab:3',
|
||||
task_title: 'Legacy parent',
|
||||
display_name: 'Legacy parent',
|
||||
spec: 'Legacy parent'
|
||||
})),
|
||||
getActiveCoordinatorRun: vi.fn(() => undefined)
|
||||
} as never)
|
||||
|
||||
expect(runtime.getAgentStatusOrchestrationContextForPaneKey(childPaneKey)?.parentPaneKey).toBe(
|
||||
parentPaneKey
|
||||
)
|
||||
})
|
||||
|
||||
it('uses mirrored tab ownership after a workspace rename instead of stale hook attribution', async () => {
|
||||
const renamedPath = '/tmp/worktree-renamed'
|
||||
const renamedWorktreeId = `${TEST_REPO_ID}::${renamedPath}`
|
||||
|
||||
@@ -10960,6 +10960,13 @@ export class OrcaRuntimeService {
|
||||
return
|
||||
}
|
||||
|
||||
// Why: this path mutates persistent task state (failDispatch), so match
|
||||
// strictly by the exiting terminal's current handle — never by fuzzy pane
|
||||
// identity, which could fail an unrelated task that merely reused this
|
||||
// pane. A heartbeat rebinds the assignee to the live handle (see
|
||||
// rebindDispatchAssignee), so this resolves the genuine worker once it
|
||||
// checks in; before that (the brief post-restart window) we no-op and let
|
||||
// the coordinator's stale-heartbeat timeout be the backstop.
|
||||
const dispatch = this._orchestrationDb.getActiveDispatchForTerminal(handle)
|
||||
if (!dispatch) {
|
||||
return
|
||||
@@ -21080,8 +21087,13 @@ export class OrcaRuntimeService {
|
||||
const terminalParent = await this.resolveWorkspaceParentSelector(
|
||||
`id:${terminal.worktreeId}`
|
||||
)
|
||||
// Why: lineage-candidate attribution is advisory (parent selection for
|
||||
// display), so resolve leniently by durable pane identity; a heartbeat
|
||||
// rebinds the assignee to the live handle, so this is normally an exact
|
||||
// handle hit.
|
||||
const activeDispatch = this._orchestrationDb?.getActiveDispatchForTerminal(
|
||||
input.callerTerminalHandle
|
||||
input.callerTerminalHandle,
|
||||
this.getPaneKeyForTerminalHandle(input.callerTerminalHandle) ?? undefined
|
||||
)
|
||||
const activeRun = this._orchestrationDb?.getActiveCoordinatorRun()
|
||||
if (activeDispatch) {
|
||||
@@ -22863,7 +22875,7 @@ export class OrcaRuntimeService {
|
||||
if (!handle) {
|
||||
return undefined
|
||||
}
|
||||
return this.getAgentStatusOrchestrationContextForHandle(handle)
|
||||
return this.getAgentStatusOrchestrationContextForHandle(handle, paneKey)
|
||||
}
|
||||
|
||||
getAgentStatusTerminalHandleForPaneKey(paneKey: string): string | undefined {
|
||||
@@ -22897,9 +22909,10 @@ export class OrcaRuntimeService {
|
||||
continue
|
||||
}
|
||||
const handle = this.issueHandle(leaf)
|
||||
const context = this.getAgentStatusOrchestrationContextForHandle(handle, db)
|
||||
const paneKey = this.makeRuntimePaneKey(leaf)
|
||||
const context = this.getAgentStatusOrchestrationContextForHandle(handle, paneKey, db)
|
||||
if (context) {
|
||||
contexts[this.makeRuntimePaneKey(leaf)] = context
|
||||
contexts[paneKey] = context
|
||||
}
|
||||
}
|
||||
for (const pty of this.ptysById.values()) {
|
||||
@@ -22907,7 +22920,7 @@ export class OrcaRuntimeService {
|
||||
continue
|
||||
}
|
||||
const handle = this.issuePtyHandle(pty)
|
||||
const context = this.getAgentStatusOrchestrationContextForHandle(handle, db)
|
||||
const context = this.getAgentStatusOrchestrationContextForHandle(handle, pty.paneKey, db)
|
||||
if (context) {
|
||||
contexts[pty.paneKey] = context
|
||||
}
|
||||
@@ -22917,14 +22930,20 @@ export class OrcaRuntimeService {
|
||||
|
||||
private getAgentStatusOrchestrationContextForHandle(
|
||||
handle: string,
|
||||
paneKey?: string,
|
||||
db = this.getOrchestrationDbIfAvailable()
|
||||
): AgentStatusOrchestrationContext | undefined {
|
||||
// Why: active dispatches are authoritative for reused terminals. Completed
|
||||
// context is only useful while the corresponding done/recent row can still
|
||||
// be visible; after that it would stale-group unrelated future work.
|
||||
// Why: this is a read-only display path (sidebar lineage grouping), so
|
||||
// resolve leniently by durable pane identity — handle first, then leaf-UUID
|
||||
// equivalence for a reminted/broken-out pane. A heartbeat rebinds the
|
||||
// assignee to the live handle (see rebindDispatchAssignee), so the common
|
||||
// case is an exact handle hit; the rare pre-heartbeat pane match can only
|
||||
// mis-group cosmetically and self-heals once the real worker checks in or
|
||||
// the dispatch closes. State mutation lives on the strict handle-only path
|
||||
// (failActiveDispatchOnExit), never here.
|
||||
const dispatch =
|
||||
db?.getActiveDispatchForTerminal?.(handle) ??
|
||||
this.getRecentCompletedDispatchForTerminal(handle, db)
|
||||
db?.getActiveDispatchForTerminal?.(handle, paneKey) ??
|
||||
this.getRecentCompletedDispatchForTerminal(handle, paneKey, db)
|
||||
if (!dispatch) {
|
||||
return undefined
|
||||
}
|
||||
@@ -22943,9 +22962,16 @@ export class OrcaRuntimeService {
|
||||
(activeRun?.coordinator_handle && activeRun.coordinator_handle !== handle
|
||||
? activeRun.coordinator_handle
|
||||
: undefined)
|
||||
const parentPaneKey = parentTerminalHandle
|
||||
? this.getPaneKeyForTerminalHandle(parentTerminalHandle)
|
||||
: undefined
|
||||
const persistedParentPaneKey = task?.created_by_pane_key
|
||||
// Why: breakout changes a pane's tab prefix; emit its current key so the
|
||||
// sidebar's exact parent-row lookup still succeeds after restart. A live
|
||||
// resolution (persisted key or handle) beats the raw persisted key, which
|
||||
// may be stale when the parent pane is gone or legacy-keyed.
|
||||
const parentPaneKey =
|
||||
(persistedParentPaneKey ? this.getLiveEquivalentPaneKey(persistedParentPaneKey) : null) ??
|
||||
(parentTerminalHandle ? this.getPaneKeyForTerminalHandle(parentTerminalHandle) : null) ??
|
||||
persistedParentPaneKey ??
|
||||
undefined
|
||||
|
||||
return {
|
||||
taskId: dispatch.task_id,
|
||||
@@ -22961,9 +22987,10 @@ export class OrcaRuntimeService {
|
||||
|
||||
private getRecentCompletedDispatchForTerminal(
|
||||
handle: string,
|
||||
paneKey?: string,
|
||||
db = this.getOrchestrationDbIfAvailable()
|
||||
): ReturnType<OrchestrationDb['getLatestDispatchForTerminal']> {
|
||||
const dispatch = db?.getLatestDispatchForTerminal?.(handle)
|
||||
const dispatch = db?.getLatestDispatchForTerminal?.(handle, paneKey)
|
||||
if (dispatch?.status !== 'completed' || !dispatch.completed_at) {
|
||||
return undefined
|
||||
}
|
||||
@@ -23011,6 +23038,45 @@ export class OrcaRuntimeService {
|
||||
return null
|
||||
}
|
||||
|
||||
private getLiveEquivalentPaneKey(paneKey: string): string | null {
|
||||
const leafId = parsePaneKey(paneKey)?.leafId
|
||||
let equivalentLeafPaneKey: string | null = null
|
||||
for (const leaf of this.leaves.values()) {
|
||||
// Why: "live" means a pane that can host an agent row; skip empty leaves.
|
||||
if (!leaf.ptyId) {
|
||||
continue
|
||||
}
|
||||
const currentPaneKey = this.makeRuntimePaneKey(leaf)
|
||||
if (currentPaneKey === paneKey) {
|
||||
return currentPaneKey
|
||||
}
|
||||
if (!equivalentLeafPaneKey && leafId && leaf.leafId === leafId) {
|
||||
equivalentLeafPaneKey = currentPaneKey
|
||||
}
|
||||
}
|
||||
if (equivalentLeafPaneKey) {
|
||||
return equivalentLeafPaneKey
|
||||
}
|
||||
|
||||
let equivalentPtyPaneKey: string | null = null
|
||||
for (const pty of this.ptysById.values()) {
|
||||
if (pty.paneKey === paneKey) {
|
||||
return paneKey
|
||||
}
|
||||
// Why: without the leafId guard, two unparseable legacy keys would match
|
||||
// as `undefined === undefined` and adopt an unrelated pane as parent.
|
||||
if (
|
||||
!equivalentPtyPaneKey &&
|
||||
leafId &&
|
||||
pty.paneKey &&
|
||||
parsePaneKey(pty.paneKey)?.leafId === leafId
|
||||
) {
|
||||
equivalentPtyPaneKey = pty.paneKey
|
||||
}
|
||||
}
|
||||
return equivalentPtyPaneKey
|
||||
}
|
||||
|
||||
private getPaneKeyForTerminalHandle(handle: string): string | null {
|
||||
const livePty = this.getLivePtyForHandle(handle)
|
||||
if (livePty?.pty.paneKey) {
|
||||
|
||||
@@ -94,8 +94,13 @@ function exposeMessageListTimestamps(messages: MessageRow[]): MessageRow[] {
|
||||
// explicit task_title/display_name fields for orchestration worker UI labels.
|
||||
// v5 → v6 adds pane-identity columns (dispatch_contexts.assignee_pane_key,
|
||||
// messages.sender_pane_key) so worker_done ownership survives terminal handle
|
||||
// remints without accepting completions from unrelated panes.
|
||||
const SCHEMA_VERSION = 6
|
||||
// remints without accepting completions from unrelated panes. v6 → v7 adds
|
||||
// tasks.created_by_pane_key so the parent/child lineage the sidebar renders can
|
||||
// be rebuilt after a restart remints every terminal handle. v7 → v8 re-runs that
|
||||
// same column ensure: some long-lived DBs stamped user_version=7 before the v7
|
||||
// ALTER shipped (version reuse), which left CREATE/INSERT on created_by_pane_key
|
||||
// failing while migrate short-circuited.
|
||||
const SCHEMA_VERSION = 8
|
||||
|
||||
export class OrchestrationDb {
|
||||
private db: Database.Database
|
||||
@@ -141,6 +146,7 @@ export class OrchestrationDb {
|
||||
id TEXT PRIMARY KEY,
|
||||
parent_id TEXT,
|
||||
created_by_terminal_handle TEXT,
|
||||
created_by_pane_key TEXT,
|
||||
task_title TEXT,
|
||||
display_name TEXT,
|
||||
spec TEXT NOT NULL,
|
||||
@@ -203,6 +209,7 @@ export class OrchestrationDb {
|
||||
);
|
||||
`)
|
||||
this.createUndeliveredInboxIndexIfPossible()
|
||||
this.createDispatchPaneKeyIndexIfPossible()
|
||||
}
|
||||
|
||||
// Why: `CREATE TABLE IF NOT EXISTS` is a no-op against an existing on-disk
|
||||
@@ -210,10 +217,14 @@ export class OrchestrationDb {
|
||||
// not reach an upgraded user unless we migrate explicitly. The transaction
|
||||
// guarantees atomicity — a mid-migration crash leaves the DB at the prior
|
||||
// version because `user_version` is bumped only on success. Idempotent
|
||||
// re-invocation is a no-op (current >= SCHEMA_VERSION short-circuit).
|
||||
// re-invocation is a no-op (current >= SCHEMA_VERSION short-circuit), except
|
||||
// for lightweight column repairs that recover version-reuse mistakes.
|
||||
private migrate(): void {
|
||||
const current = this.db.pragma('user_version', { simple: true }) as number
|
||||
if (current >= SCHEMA_VERSION) {
|
||||
// Why: short-circuit skips versioned steps; still repair columns that may
|
||||
// be missing when user_version was stamped without the matching ALTER.
|
||||
this.ensureCreatedByPaneKeyColumn()
|
||||
return
|
||||
}
|
||||
|
||||
@@ -310,7 +321,12 @@ export class OrchestrationDb {
|
||||
this.db.exec(`ALTER TABLE messages ADD COLUMN sender_pane_key TEXT`)
|
||||
}
|
||||
}
|
||||
// v6→v7 and v7→v8 both ensure created_by_pane_key (v8 repairs version-reuse).
|
||||
if (current < 8) {
|
||||
this.ensureCreatedByPaneKeyColumn()
|
||||
}
|
||||
this.createUndeliveredInboxIndexIfPossible()
|
||||
this.createDispatchPaneKeyIndexIfPossible()
|
||||
|
||||
this.db.pragma(`user_version = ${SCHEMA_VERSION}`)
|
||||
this.db.exec('COMMIT')
|
||||
@@ -320,6 +336,14 @@ export class OrchestrationDb {
|
||||
}
|
||||
}
|
||||
|
||||
// Why: INSERT always names created_by_pane_key; a missing column hard-fails
|
||||
// task-create even when user_version claims the schema is current.
|
||||
private ensureCreatedByPaneKeyColumn(): void {
|
||||
if (!this.hasColumn('tasks', 'created_by_pane_key')) {
|
||||
this.db.exec(`ALTER TABLE tasks ADD COLUMN created_by_pane_key TEXT`)
|
||||
}
|
||||
}
|
||||
|
||||
private hasColumn(table: string, column: string): boolean {
|
||||
const rows = this.db.pragma(`table_info(${table})`) as { name: string }[]
|
||||
return rows.some((r) => r.name === column)
|
||||
@@ -335,6 +359,16 @@ export class OrchestrationDb {
|
||||
`)
|
||||
}
|
||||
|
||||
private createDispatchPaneKeyIndexIfPossible(): void {
|
||||
if (!this.hasColumn('dispatch_contexts', 'assignee_pane_key')) {
|
||||
return
|
||||
}
|
||||
this.db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_dispatch_contexts_assignee_pane_key
|
||||
ON dispatch_contexts(assignee_pane_key)
|
||||
`)
|
||||
}
|
||||
|
||||
// Why: sqlite_master stores the original CREATE TABLE SQL including the
|
||||
// CHECK clause. Inspecting that text is the cheapest reliable way to tell
|
||||
// whether the pre-rebuild schema already knows about 'heartbeat' without
|
||||
@@ -562,6 +596,8 @@ export class OrchestrationDb {
|
||||
deps?: string[]
|
||||
parentId?: string
|
||||
createdByTerminalHandle?: string
|
||||
// Why: terminal handles remint on restart, but sidebar lineage must persist.
|
||||
createdByPaneKey?: string
|
||||
}): TaskRow {
|
||||
const id = generateId('task')
|
||||
const depsJson = JSON.stringify(task.deps ?? [])
|
||||
@@ -574,12 +610,13 @@ export class OrchestrationDb {
|
||||
})
|
||||
this.db
|
||||
.prepare(
|
||||
'INSERT INTO tasks (id, parent_id, created_by_terminal_handle, task_title, display_name, spec, status, deps) VALUES (?, ?, ?, ?, ?, ?, ?, ?)'
|
||||
'INSERT INTO tasks (id, parent_id, created_by_terminal_handle, created_by_pane_key, task_title, display_name, spec, status, deps) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)'
|
||||
)
|
||||
.run(
|
||||
id,
|
||||
task.parentId ?? null,
|
||||
task.createdByTerminalHandle ?? null,
|
||||
task.createdByPaneKey ?? null,
|
||||
display.taskTitle || null,
|
||||
display.displayName || null,
|
||||
task.spec,
|
||||
@@ -758,8 +795,9 @@ export class OrchestrationDb {
|
||||
| undefined
|
||||
}
|
||||
|
||||
getActiveDispatchForTerminal(handle: string): DispatchContextRow | undefined {
|
||||
return this.findActiveDispatchForAssignee(handle)
|
||||
getActiveDispatchForTerminal(handle: string, paneKey?: string): DispatchContextRow | undefined {
|
||||
// Why: the pane key keeps the dispatch resolvable after handle remint.
|
||||
return this.findActiveDispatchForAssignee(handle, paneKey)
|
||||
}
|
||||
|
||||
private findActiveDispatchForAssignee(
|
||||
@@ -774,31 +812,73 @@ export class OrchestrationDb {
|
||||
if (byHandle) {
|
||||
return byHandle
|
||||
}
|
||||
return assigneePaneKey ? this.resolveDispatchByPaneKey(assigneePaneKey, true) : undefined
|
||||
}
|
||||
|
||||
if (!assigneePaneKey) {
|
||||
return undefined
|
||||
getLatestDispatchForTerminal(handle: string, paneKey?: string): DispatchContextRow | undefined {
|
||||
const byHandle = this.db
|
||||
.prepare(
|
||||
'SELECT * FROM dispatch_contexts WHERE assignee_handle = ? ORDER BY rowid DESC LIMIT 1'
|
||||
)
|
||||
.get(handle) as DispatchContextRow | undefined
|
||||
if (byHandle || !paneKey) {
|
||||
return byHandle
|
||||
}
|
||||
return this.resolveDispatchByPaneKey(paneKey, false)
|
||||
}
|
||||
|
||||
// Why: the handle remints on restart, so fall back to the durable pane
|
||||
// identity. Exact `assignee_pane_key` is index-served; the `substr` leaf
|
||||
// suffix is a rare bridge for pre-heartbeat break-out (a heartbeat rebinds
|
||||
// `assignee_pane_key` to the current key, so exact match dominates once a
|
||||
// worker checks in). `activeOnly` scopes to live dispatches.
|
||||
private resolveDispatchByPaneKey(
|
||||
paneKey: string,
|
||||
activeOnly: boolean
|
||||
): DispatchContextRow | undefined {
|
||||
const statusFilter = activeOnly ? " AND status IN ('pending', 'dispatched')" : ''
|
||||
const exactPane = this.db
|
||||
.prepare(
|
||||
`SELECT * FROM dispatch_contexts
|
||||
WHERE assignee_pane_key = ?${statusFilter} ORDER BY rowid DESC LIMIT 1`
|
||||
)
|
||||
.get(paneKey) as DispatchContextRow | undefined
|
||||
if (exactPane) {
|
||||
return exactPane
|
||||
}
|
||||
|
||||
const actives = this.db
|
||||
const leafId = parsePaneKey(paneKey)?.leafId
|
||||
if (!leafId) {
|
||||
return undefined
|
||||
}
|
||||
const rows = this.db
|
||||
.prepare(
|
||||
"SELECT * FROM dispatch_contexts WHERE assignee_pane_key IS NOT NULL AND status IN ('pending', 'dispatched')"
|
||||
`SELECT * FROM dispatch_contexts
|
||||
WHERE substr(assignee_pane_key, -37) = ?${statusFilter} ORDER BY rowid DESC`
|
||||
)
|
||||
.all() as DispatchContextRow[]
|
||||
|
||||
for (const row of actives) {
|
||||
if (row.assignee_pane_key && isEquivalentPaneKey(row.assignee_pane_key, assigneePaneKey)) {
|
||||
.all(`:${leafId}`) as DispatchContextRow[]
|
||||
for (const row of rows) {
|
||||
if (row.assignee_pane_key && isEquivalentPaneKey(row.assignee_pane_key, paneKey)) {
|
||||
return row
|
||||
}
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
getLatestDispatchForTerminal(handle: string): DispatchContextRow | undefined {
|
||||
return this.db
|
||||
// Why: a heartbeat/worker_done authorized by dispatchId + pane identity
|
||||
// proves this handle+pane now drives the dispatch. Rebinding the assignee
|
||||
// re-anchors handle-based resolution after a restart remints the handle (or
|
||||
// a break-out changes the tab prefix) — durably, with no in-memory tracking.
|
||||
// Scoped to `dispatched` rows so a late message can't resurrect a closed
|
||||
// dispatch; COALESCE keeps a missing sender pane key from wiping the stored one.
|
||||
rebindDispatchAssignee(dispatchId: string, handle: string, paneKey?: string): void {
|
||||
this.db
|
||||
.prepare(
|
||||
'SELECT * FROM dispatch_contexts WHERE assignee_handle = ? ORDER BY rowid DESC LIMIT 1'
|
||||
`UPDATE dispatch_contexts
|
||||
SET assignee_handle = ?, assignee_pane_key = COALESCE(?, assignee_pane_key)
|
||||
WHERE id = ? AND status = 'dispatched'`
|
||||
)
|
||||
.get(handle) as DispatchContextRow | undefined
|
||||
.run(handle, paneKey ?? null, dispatchId)
|
||||
}
|
||||
|
||||
completeDispatch(ctxId: string): void {
|
||||
|
||||
@@ -146,6 +146,13 @@ function reconcileHeartbeatMessage(
|
||||
return { action: 'rejected', code: 'sender_not_assignee', reason }
|
||||
}
|
||||
|
||||
// Why: the heartbeat's pane authority proves this handle+pane now drives the
|
||||
// dispatch, so re-anchor the assignee here. This is the durable self-heal
|
||||
// after a restart remints the worker's handle — handle-based resolution and
|
||||
// strict crash detection work again from the first post-restart heartbeat,
|
||||
// with no in-memory tracking.
|
||||
db.rebindDispatchAssignee(dispatchId, msg.from_handle, msg.sender_pane_key ?? undefined)
|
||||
|
||||
// Why: dispatchId-specific writes let the DB ignore late heartbeats for
|
||||
// completed/failed retries without masking a newer hung dispatch.
|
||||
db.recordHeartbeat(dispatchId, msg.created_at)
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import Database from '../../sqlite/sync-database'
|
||||
import { OrchestrationDb } from './db'
|
||||
import { reconcileLifecycleMessage } from './lifecycle-reconciliation'
|
||||
|
||||
describe('orchestration lineage survives a restart handle remint', () => {
|
||||
let db: OrchestrationDb | undefined
|
||||
let tempDir: string | undefined
|
||||
|
||||
afterEach(() => {
|
||||
db?.close()
|
||||
db = undefined
|
||||
if (tempDir) {
|
||||
rmSync(tempDir, { recursive: true, force: true })
|
||||
tempDir = undefined
|
||||
}
|
||||
})
|
||||
|
||||
function createDb(): OrchestrationDb {
|
||||
db = new OrchestrationDb(':memory:')
|
||||
return db
|
||||
}
|
||||
|
||||
// Real leaf UUIDs: pane keys are `${tabId}:${leafUuid}`.
|
||||
const LEAF_A = '11111111-1111-1111-8111-111111111111'
|
||||
const LEAF_B = '22222222-2222-4222-9222-222222222222'
|
||||
|
||||
function createFileDbPath(): string {
|
||||
tempDir = mkdtempSync(join(tmpdir(), 'orca-restart-lineage-'))
|
||||
return join(tempDir, 'orchestration.sqlite')
|
||||
}
|
||||
|
||||
it('reopens persisted creator and active-dispatch pane identities', () => {
|
||||
const dbPath = createFileDbPath()
|
||||
const beforeRestart = new OrchestrationDb(dbPath)
|
||||
const task = beforeRestart.createTask({
|
||||
spec: 'work',
|
||||
createdByTerminalHandle: 'term_parent_before_restart',
|
||||
createdByPaneKey: `tab_parent_before_breakout:${LEAF_A}`
|
||||
})
|
||||
beforeRestart.createDispatchContext(
|
||||
task.id,
|
||||
'term_child_before_restart',
|
||||
`tab_child_before_breakout:${LEAF_B}`
|
||||
)
|
||||
beforeRestart.close()
|
||||
|
||||
db = new OrchestrationDb(dbPath)
|
||||
expect(db.getTask(task.id)?.created_by_pane_key).toBe(`tab_parent_before_breakout:${LEAF_A}`)
|
||||
expect(db.getActiveDispatchForTerminal('term_child_after_restart')).toBeUndefined()
|
||||
expect(
|
||||
db.getActiveDispatchForTerminal(
|
||||
'term_child_after_restart',
|
||||
`tab_child_after_breakout:${LEAF_B}`
|
||||
)?.task_id
|
||||
).toBe(task.id)
|
||||
})
|
||||
|
||||
it('rebinds the dispatch assignee to the resumed worker on the first post-restart heartbeat', () => {
|
||||
// Why: the durable self-heal. A heartbeat authorized by dispatchId + pane
|
||||
// identity re-anchors the assignee to the reminted handle + current pane
|
||||
// key, so exact handle-based resolution recovers with no in-memory state.
|
||||
const d = createDb()
|
||||
const task = d.createTask({ spec: 'work' })
|
||||
const ctx = d.createDispatchContext(task.id, 'term_before_restart', `tab_before:${LEAF_B}`)
|
||||
|
||||
// Pre-heartbeat, only the durable pane identity resolves the dispatch.
|
||||
expect(d.getActiveDispatchForTerminal('term_after_restart')).toBeUndefined()
|
||||
|
||||
const heartbeat = d.insertMessage({
|
||||
from: 'term_after_restart',
|
||||
to: 'coordinator',
|
||||
subject: 'hb',
|
||||
type: 'heartbeat',
|
||||
payload: JSON.stringify({ dispatchId: ctx.id }),
|
||||
senderPaneKey: `tab_after:${LEAF_B}`
|
||||
})
|
||||
expect(reconcileLifecycleMessage(d, heartbeat).action).toBe('heartbeat_recorded')
|
||||
|
||||
const rebound = d.getActiveDispatchForTerminal('term_after_restart')
|
||||
expect(rebound?.id).toBe(ctx.id)
|
||||
expect(rebound?.assignee_pane_key).toBe(`tab_after:${LEAF_B}`)
|
||||
// The stale pre-restart handle no longer owns it.
|
||||
expect(d.getActiveDispatchForTerminal('term_before_restart')).toBeUndefined()
|
||||
})
|
||||
|
||||
it('does not rebind a closed dispatch from a late heartbeat', () => {
|
||||
// Why: rebind is scoped to live dispatches — a straggler heartbeat after
|
||||
// completion must not resurrect assignee identity on a done row.
|
||||
const d = createDb()
|
||||
const task = d.createTask({ spec: 'work' })
|
||||
const ctx = d.createDispatchContext(task.id, 'term_old', `tab_before:${LEAF_A}`)
|
||||
d.completeDispatch(ctx.id)
|
||||
|
||||
d.rebindDispatchAssignee(ctx.id, 'term_new', `tab_after:${LEAF_A}`)
|
||||
expect(d.getDispatchContextById(ctx.id)?.assignee_handle).toBe('term_old')
|
||||
})
|
||||
|
||||
it('finds a completed dispatch by its current pane key after a remint and breakout', () => {
|
||||
const d = createDb()
|
||||
const task = d.createTask({ spec: 'work' })
|
||||
const ctx = d.createDispatchContext(task.id, 'term_old', `tab_before:${LEAF_B}`)
|
||||
d.completeDispatch(ctx.id)
|
||||
|
||||
expect(d.getLatestDispatchForTerminal('term_new')).toBeUndefined()
|
||||
const remint = d.getLatestDispatchForTerminal('term_new', `tab_after:${LEAF_B}`)
|
||||
expect(remint?.id).toBe(ctx.id)
|
||||
expect(remint?.status).toBe('completed')
|
||||
})
|
||||
|
||||
it('migrates a v6 task table before persisting creator pane identity', () => {
|
||||
const dbPath = createFileDbPath()
|
||||
const raw = new Database(dbPath)
|
||||
raw.exec(`
|
||||
CREATE TABLE tasks (
|
||||
id TEXT PRIMARY KEY,
|
||||
parent_id TEXT,
|
||||
created_by_terminal_handle TEXT,
|
||||
task_title TEXT,
|
||||
display_name TEXT,
|
||||
spec TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
deps TEXT NOT NULL DEFAULT '[]',
|
||||
result TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
completed_at TEXT
|
||||
);
|
||||
INSERT INTO tasks (id, created_by_terminal_handle, spec, status)
|
||||
VALUES ('task_v6', 'term_legacy', 'legacy work', 'ready');
|
||||
`)
|
||||
raw.pragma('user_version = 6')
|
||||
raw.close()
|
||||
|
||||
db = new OrchestrationDb(dbPath)
|
||||
expect(db.getTask('task_v6')?.created_by_pane_key).toBeNull()
|
||||
const created = db.createTask({
|
||||
spec: 'new work',
|
||||
createdByPaneKey: `tab_parent:${LEAF_A}`
|
||||
})
|
||||
expect(created.created_by_pane_key).toBe(`tab_parent:${LEAF_A}`)
|
||||
expect(readUserVersion(dbPath)).toBe(8)
|
||||
})
|
||||
|
||||
// Why: real orca-dev DBs stamped user_version=7 before created_by_pane_key
|
||||
// shipped; short-circuit alone left task-create INSERT failing forever.
|
||||
it('repairs a v7-stamped task table missing created_by_pane_key', () => {
|
||||
const dbPath = createFileDbPath()
|
||||
const raw = new Database(dbPath)
|
||||
raw.exec(`
|
||||
CREATE TABLE tasks (
|
||||
id TEXT PRIMARY KEY,
|
||||
parent_id TEXT,
|
||||
created_by_terminal_handle TEXT,
|
||||
task_title TEXT,
|
||||
display_name TEXT,
|
||||
spec TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
deps TEXT NOT NULL DEFAULT '[]',
|
||||
result TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
completed_at TEXT
|
||||
);
|
||||
INSERT INTO tasks (id, created_by_terminal_handle, spec, status)
|
||||
VALUES ('task_stuck_v7', 'term_legacy', 'legacy work', 'ready');
|
||||
`)
|
||||
raw.pragma('user_version = 7')
|
||||
raw.close()
|
||||
|
||||
db = new OrchestrationDb(dbPath)
|
||||
expect(db.getTask('task_stuck_v7')?.created_by_pane_key).toBeNull()
|
||||
const created = db.createTask({
|
||||
spec: 'repaired work',
|
||||
createdByPaneKey: `tab_parent:${LEAF_A}`
|
||||
})
|
||||
expect(created.created_by_pane_key).toBe(`tab_parent:${LEAF_A}`)
|
||||
expect(readUserVersion(dbPath)).toBe(8)
|
||||
})
|
||||
|
||||
// Why: even a current stamp can lie; ensureColumn must still heal.
|
||||
it('repairs a v8-stamped task table missing created_by_pane_key', () => {
|
||||
const dbPath = createFileDbPath()
|
||||
const raw = new Database(dbPath)
|
||||
raw.exec(`
|
||||
CREATE TABLE tasks (
|
||||
id TEXT PRIMARY KEY,
|
||||
parent_id TEXT,
|
||||
created_by_terminal_handle TEXT,
|
||||
task_title TEXT,
|
||||
display_name TEXT,
|
||||
spec TEXT NOT NULL,
|
||||
status TEXT NOT NULL DEFAULT 'pending',
|
||||
deps TEXT NOT NULL DEFAULT '[]',
|
||||
result TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
completed_at TEXT
|
||||
);
|
||||
`)
|
||||
raw.pragma('user_version = 8')
|
||||
raw.close()
|
||||
|
||||
db = new OrchestrationDb(dbPath)
|
||||
const created = db.createTask({
|
||||
spec: 'healed work',
|
||||
createdByPaneKey: `tab_parent:${LEAF_A}`
|
||||
})
|
||||
expect(created.created_by_pane_key).toBe(`tab_parent:${LEAF_A}`)
|
||||
})
|
||||
})
|
||||
|
||||
function readUserVersion(dbPath: string): number {
|
||||
const raw = new Database(dbPath)
|
||||
try {
|
||||
return raw.pragma('user_version', { simple: true }) as number
|
||||
} finally {
|
||||
raw.close()
|
||||
}
|
||||
}
|
||||
@@ -39,6 +39,7 @@ export type TaskRow = {
|
||||
id: string
|
||||
parent_id: string | null
|
||||
created_by_terminal_handle: string | null
|
||||
created_by_pane_key: string | null
|
||||
task_title: string | null
|
||||
display_name: string | null
|
||||
spec: string
|
||||
|
||||
@@ -1158,14 +1158,17 @@ describe('orchestration RPC methods', () => {
|
||||
expect(result.task.status).toBe('pending')
|
||||
})
|
||||
|
||||
it('records the caller terminal handle when creating a task', async () => {
|
||||
it('records the caller terminal handle and pane key when creating a task', async () => {
|
||||
setup()
|
||||
vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue('tab_creator:leaf_creator')
|
||||
const result = (await call('orchestration.taskCreate', {
|
||||
spec: 'spawn related workspace',
|
||||
callerTerminalHandle: 'term_creator'
|
||||
})) as { task: { id: string } }
|
||||
|
||||
expect(db.getTask(result.task.id)?.created_by_terminal_handle).toBe('term_creator')
|
||||
expect(db.getTask(result.task.id)?.created_by_pane_key).toBe('tab_creator:leaf_creator')
|
||||
expect(runtime.getTerminalPaneKey).toHaveBeenCalledWith('term_creator')
|
||||
})
|
||||
|
||||
it('rejects invalid deps JSON', async () => {
|
||||
|
||||
@@ -425,7 +425,12 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
|
||||
displayName: params.displayName,
|
||||
deps,
|
||||
parentId: params.parent,
|
||||
createdByTerminalHandle: params.callerTerminalHandle
|
||||
createdByTerminalHandle: params.callerTerminalHandle,
|
||||
// Why: capture the creator's remint-stable pane key so the sidebar
|
||||
// parent/child lineage survives a restart that re-mints the handle.
|
||||
createdByPaneKey: params.callerTerminalHandle
|
||||
? (runtime.getTerminalPaneKey(params.callerTerminalHandle) ?? undefined)
|
||||
: undefined
|
||||
})
|
||||
return { task }
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user