fix(orchestration): harden coordinator takeover races

This commit is contained in:
Brennan Benson
2026-08-27 16:54:46 -07:00
parent e3c1411cfe
commit 5945478604
25 changed files with 308 additions and 62 deletions
@@ -59,6 +59,8 @@ describe('orchestration skill guidance', () => {
expect(authority).toContain('per-Run authority')
expect(authority).toContain('not an operating-system or global Orca privilege')
expect(authority).toContain('`run-use` is an authority claim, not a read-only selection')
expect(authority).toContain('fenced failures report only `live` or `unverifiable`')
expect(authority).toContain('a proven `exited` incumbent permits the claim to succeed')
expect(authority).toContain('`live`')
expect(authority).toContain('`unverifiable`')
expect(authority).toContain('`exited`')
@@ -74,6 +76,10 @@ describe('orchestration skill guidance', () => {
expect(authority).toContain('run-show --id <run_id> --json')
expect(authority).toContain('binding.currentConsumer')
expect(authority).toContain('stop or exit the owning coordinator process')
expect(skill).not.toContain('unless impersonating another terminal')
expect(skill).toContain(
'A declared handle is routing input, never proof of coordinator authority'
)
})
it('teaches attested adoption without reviving the retired scheduler', () => {
+2 -2
View File
@@ -105,7 +105,7 @@ Orca orchestration is agent-operated. A Run therefore has one current coordinato
The single-writer rule prevents two agents from changing Task, Dispatch, worker, or gate state concurrently and prevents two consumers from acknowledging different views of the same FIFO Delivery. The current coordinator may create and update Tasks (`task-create`, `task-update`), start or dispatch workers (`worker-start`, `dispatch`), create and resolve gates (`gate-create`, `gate-resolve`), answer worker questions (`reply`), and consume or acknowledge Run mail (`check`). Explicit `run-show`, `task-list --run`, inbox, and peek calls remain read-only for non-owners and headless callers.
`run-use` is an authority claim, not a read-only selection. Read `error.data.coordinatorStatus` and `error.data.nextSteps` from `--json` failures; never infer authority from a terminal handle alone.
`run-use` is an authority claim, not a read-only selection. Read `error.data.coordinatorStatus` and `error.data.nextSteps` from `--json` failures; fenced failures report only `live` or `unverifiable` because a proven `exited` incumbent permits the claim to succeed. Never infer authority from a terminal handle alone.
- Same coordinator process, reminted handle: authority and any outstanding Delivery are preserved without advancing the consumer generation.
- Different process and `live` incumbent: `consumer_fenced`; continue from the owning coordinator terminal. To transfer intentionally, stop or exit the owning coordinator process before retrying from its replacement.
@@ -159,7 +159,7 @@ orca orchestration inbox [--limit <n>] [--json]
Rules:
- Omit `--from` unless impersonating another terminal; Orca auto-resolves it from the current terminal.
- Omit `--from` unless an injected preamble or exact recovery command supplies it; Orca normally resolves the current terminal. A declared handle is routing input, never proof of coordinator authority or a transfer mechanism.
- A coordinator `check` returns the bound Run's oldest FIFO Delivery (up to 50 messages) and replays that exact batch until `--ack <delivery_id>`. Process every message before acknowledging; `check --ack <id> --wait` acknowledges, checks, and waits in one operation.
- Use `--peek` and `--all` only for read-only history/debugging. Type filters decide when a waiter wakes; the returned actionable Delivery is still the oldest full batch.
- Use `dispatch:<id>` for coordinator guidance to one supervised worker. Orca routes that stable address locally or through the connected-server relay; do not substitute a remote terminal handle.
File diff suppressed because one or more lines are too long
@@ -146,7 +146,8 @@ describe('orchestration detached mailbox routing', () => {
runId: currentRun!.id,
coordinatorHandle: 'term_rebound_coordinator',
coordinatorPaneKey:
'77777777-7777-4777-8777-777777777777:88888888-8888-4888-8888-888888888888'
'77777777-7777-4777-8777-777777777777:88888888-8888-4888-8888-888888888888',
authorityContinuity: true
})
const checked = await checkBoundMailbox(harness.runtime)
expect(checked).toMatchObject({ runId: workerRun.id, dispatchId: dispatch.id, count: 1 })
@@ -446,7 +447,8 @@ describe('orchestration detached mailbox routing', () => {
runId: run.id,
coordinatorHandle: 'term_new_coordinator',
coordinatorPaneKey:
'55555555-5555-4555-8555-555555555555:66666666-6666-4666-8666-666666666666'
'55555555-5555-4555-8555-555555555555:66666666-6666-4666-8666-666666666666',
authorityContinuity: true
})
expect(db.getMessageById(message.id)?.to_handle).toBe(TERMINAL_HANDLE)
@@ -80,7 +80,8 @@ describe('orchestration notification mailbox consistency', () => {
db.bindRun({
runId: runA.id,
coordinatorHandle: SECOND_TERMINAL_HANDLE,
coordinatorPaneKey: SECOND_PANE_KEY
coordinatorPaneKey: SECOND_PANE_KEY,
authorityContinuity: true
})
sqliteFor(db)
.prepare('UPDATE messages SET to_handle = ? WHERE id = ?')
@@ -359,7 +360,8 @@ describe('orchestration notification mailbox consistency', () => {
runId: runA.id,
coordinatorHandle: 'term_new_coordinator',
coordinatorPaneKey:
'55555555-5555-4555-8555-555555555555:66666666-6666-4666-8666-666666666666'
'55555555-5555-4555-8555-555555555555:66666666-6666-4666-8666-666666666666',
authorityContinuity: true
})
const runB = createBoundRun(db, 'Run B')
const response = await new RpcDispatcher({
@@ -22,6 +22,8 @@ export const SECOND_TERMINAL_HANDLE = 'term_mailbox_consistency_second'
export const SECOND_LAUNCH_TOKEN = 'mailbox-consistency-second-launch'
export const WORKTREE_ID = 'repo-mailbox::/tmp/mailbox'
export const LAUNCH_TOKEN = 'mailbox-consistency-launch'
export const COORDINATOR_PROCESS_INCARNATION = `${PTY_ID}:mailbox-incarnation`
export const COORDINATOR_HOST_SCOPE = JSON.stringify({ kind: 'local', hostId: 'local' })
export const temporaryDirectories: string[] = []
export function createDatabase(prefix: string): OrchestrationDb {
@@ -38,7 +40,9 @@ export function createBoundRun(db: OrchestrationDb, objective: string) {
return db.createRun({
objective,
coordinatorHandle: TERMINAL_HANDLE,
coordinatorPaneKey: PANE_KEY
coordinatorPaneKey: PANE_KEY,
coordinatorProcessIncarnation: COORDINATOR_PROCESS_INCARNATION,
coordinatorHostScope: COORDINATOR_HOST_SCOPE
})
}
@@ -264,9 +264,10 @@ describe('orchestration mailbox routing races', () => {
const harness = createRuntime(db)
const run = db.createRun({
objective: 'Split Run handles',
coordinatorHandle: 'term_previous_coordinator',
coordinatorHandle: TERMINAL_HANDLE,
coordinatorPaneKey: PANE_KEY
})
db.rememberRunCoordinatorHandle(run.id, 'term_previous_coordinator')
const current = insertDirectRunMessage(db, run.id, 'Current handle')
const previous = db.insertMessage({
from: 'term_worker',
@@ -25,6 +25,8 @@ const TERMINAL_HANDLE = 'term_sta_4325'
const REMINTED_TERMINAL_HANDLE = 'term_sta_4325_reminted'
const WORKTREE_ID = 'repo-sta-4325::/tmp/sta-4325'
const LAUNCH_TOKEN = 'sta-4325-launch'
const COORDINATOR_PROCESS_INCARNATION = `${PTY_ID}:sta-4325-incarnation`
const COORDINATOR_HOST_SCOPE = JSON.stringify({ kind: 'local', hostId: 'local' })
const temporaryDirectories: string[] = []
const CLI_PATH = join(process.cwd(), 'out', 'cli', 'index.js')
const itIfCliBuilt = existsSync(CLI_PATH) ? it : it.skip
@@ -268,7 +270,9 @@ describe('STA-4325 message and delivery identity', () => {
const run = fixture.db.createRun({
objective: 'STA-4325 restart',
coordinatorHandle: TERMINAL_HANDLE,
coordinatorPaneKey: PANE_KEY
coordinatorPaneKey: PANE_KEY,
coordinatorProcessIncarnation: COORDINATOR_PROCESS_INCARNATION,
coordinatorHostScope: COORDINATOR_HOST_SCOPE
})
const status = fixture.db.insertMessage({
from: 'term_worker',
@@ -389,12 +393,17 @@ describe('STA-4325 message and delivery identity', () => {
const run = fixture.db.createRun({
objective: 'Late old-handle arrival',
coordinatorHandle: TERMINAL_HANDLE,
coordinatorPaneKey: PANE_KEY
coordinatorPaneKey: PANE_KEY,
coordinatorProcessIncarnation: COORDINATOR_PROCESS_INCARNATION,
coordinatorHostScope: COORDINATOR_HOST_SCOPE
})
fixture.db.bindRun({
runId: run.id,
coordinatorHandle: REMINTED_TERMINAL_HANDLE,
coordinatorPaneKey: PANE_KEY
coordinatorPaneKey: PANE_KEY,
coordinatorProcessIncarnation: COORDINATOR_PROCESS_INCARNATION,
coordinatorHostScope: COORDINATOR_HOST_SCOPE,
authorityContinuity: true
})
const done = fixture.db.insertMessage({
from: 'term_worker',
@@ -65,9 +65,7 @@ export function bindRun(
run.coordinator_process_incarnation ||
run.coordinator_host_scope
)
const requiresAuthorityProof =
hasIncumbentAuthority &&
Boolean(observation || run.coordinator_process_incarnation || run.coordinator_host_scope)
const requiresAuthorityProof = hasIncumbentAuthority
const legacyAuthority = params.legacyCoordinatorAuthority
const legacyPrincipalId = legacyAuthority?.principalId
const legacyPrincipal = legacyPrincipalId
@@ -119,6 +117,10 @@ export function bindRun(
run.coordinator_handle === params.coordinatorHandle &&
coordinatorPrincipal?.status !== 'committed'
)
const appliesLegacyTakeover = Boolean(
params.takeoverLegacy &&
(coordinatorPrincipal?.status === 'committed' || !hasIncumbentAuthority)
)
const replacesLegacyCoordinator = Boolean(
adoptedRun &&
!provenLegacyBinding &&
@@ -135,7 +137,7 @@ export function bindRun(
}
// Why: only LIVE legacy work needs the flag — settled work has no competing authority left, and
// fencing it would strand the recovered graph behind an attestation the caller may not have.
if (activeLegacyAssignment && !sameBinding && !provenLegacyBinding && !params.takeoverLegacy) {
if (activeLegacyAssignment && !sameBinding && !provenLegacyBinding && !appliesLegacyTakeover) {
throw new OrchestrationError(
'consumer_fenced',
'This adopted Run still has live legacy work. Its attested coordinator may rebind it, or a current coordinator may explicitly use run-use --takeover-legacy.',
@@ -149,7 +151,7 @@ export function bindRun(
requiresAuthorityProof &&
!sameAuthority &&
!provenLegacyBinding &&
!params.takeoverLegacy &&
!appliesLegacyTakeover &&
(!observationMatches || observation.status !== 'exited')
) {
const coordinatorStatus = observationMatches ? observation.status : 'unverifiable'
@@ -177,7 +179,16 @@ export function bindRun(
{ effectsApplied: false, coordinatorStatus, nextSteps }
)
}
this.unbindOtherRunsForPane(params.coordinatorPaneKey, params.runId)
this.unbindOtherRunsForPane(
params.coordinatorPaneKey,
{
handle: params.coordinatorHandle,
paneKey: params.coordinatorPaneKey,
processIncarnation: params.coordinatorProcessIncarnation ?? null,
hostScope: params.coordinatorHostScope ?? null
},
params.runId
)
for (const handle of new Set(
[run.coordinator_handle, params.coordinatorHandle].filter((value): value is string =>
Boolean(value)
@@ -191,18 +202,18 @@ export function bindRun(
run.coordinator_handle !== params.coordinatorHandle ||
run.coordinator_process_incarnation !== (params.coordinatorProcessIncarnation ?? null) ||
run.coordinator_host_scope !== (params.coordinatorHostScope ?? null)
if ((params.takeoverLegacy && !takeoverAlreadyApplied) || bindingMetadataChanged) {
if (adoptedRun && (params.takeoverLegacy || !activeLegacyAssignment)) {
if ((appliesLegacyTakeover && !takeoverAlreadyApplied) || bindingMetadataChanged) {
if (adoptedRun && (appliesLegacyTakeover || !activeLegacyAssignment)) {
if (
coordinatorPrincipal?.status === 'committed' &&
(params.takeoverLegacy ||
(appliesLegacyTakeover ||
coordinatorPrincipal.terminal_handle !== params.coordinatorHandle ||
!isEquivalentPaneKey(coordinatorPrincipal.pane_key, params.coordinatorPaneKey))
) {
this.setLegacyCompatibilityPrincipalStatus(coordinatorPrincipal.id, 'revoked')
}
}
const changesAuthority = !sameAuthority || Boolean(params.takeoverLegacy)
const changesAuthority = !sameAuthority || appliesLegacyTakeover
this.db
.prepare(
`UPDATE runs
@@ -224,7 +235,7 @@ export function bindRun(
if (changesAuthority) {
this.fenceOutstandingDelivery(params.runId)
}
if (params.takeoverLegacy || replacesLegacyCoordinator) {
if (appliesLegacyTakeover || replacesLegacyCoordinator) {
this.promoteLegacyCoordinatorMailForTakeover(params.runId, retainedCoordinatorHandle)
}
}
@@ -17,7 +17,12 @@ export function createRun(
const id = generateId('run')
this.db.exec('BEGIN IMMEDIATE')
try {
this.unbindOtherRunsForPane(params.coordinatorPaneKey)
this.unbindOtherRunsForPane(params.coordinatorPaneKey, {
handle: params.coordinatorHandle,
paneKey: params.coordinatorPaneKey,
processIncarnation: params.coordinatorProcessIncarnation ?? null,
hostScope: params.coordinatorHostScope ?? null
})
this.db
.prepare(
`INSERT INTO runs (
@@ -1,4 +1,9 @@
import type { RunRow } from '../../types'
import { OrchestrationError } from '../../orchestration-error'
import {
isCurrentRunCoordinator,
type RunCoordinatorIdentity
} from '../../run-coordinator-authority'
import { ORCHESTRATION_RUN_PAGE_LIMIT } from '../../../../../shared/orchestration-run-pagination'
import {
isEquivalentPaneKey,
@@ -124,10 +129,25 @@ export function getRunRaw(this: OrchestrationDb, id: string): RunRow | undefined
export function unbindOtherRunsForPane(
this: OrchestrationDb,
paneKey: string,
identity: RunCoordinatorIdentity,
exceptRunId?: string
): void {
for (const run of this.runsBoundToPane(paneKey)) {
if (run.id !== exceptRunId) {
if (!isCurrentRunCoordinator(run, identity)) {
throw new OrchestrationError(
'consumer_fenced',
`Run ${run.id} is owned by another coordinator process in this pane. No effects were applied.`,
{
effectsApplied: false,
nextSteps: [
`Inspect current authority with orca orchestration run-show --id ${run.id} --json.`,
`To switch Runs from this replacement process, first claim the existing Run with orca orchestration run-use --id ${run.id} --json after its owning host proves the incumbent exited.`,
'Do not retry the create or bind command unchanged while another process owns the pane.'
]
}
)
}
if (run.coordinator_handle) {
this.routeAllUnreadDirectMessagesToRunMailbox(run.id, run.coordinator_handle)
}
@@ -12,7 +12,7 @@ export function applySchemaMigrationV31(this: OrchestrationDb, current: number):
}
if (!this.hasColumn('runs', 'coordinator_authority_revision')) {
this.db.exec(
'ALTER TABLE runs ADD COLUMN coordinator_authority_revision INTEGER NOT NULL DEFAULT 0'
'ALTER TABLE runs ADD COLUMN coordinator_authority_revision INTEGER NOT NULL DEFAULT -1'
)
}
this.db.exec(`
@@ -184,8 +184,8 @@ describe('adopted Run binding without --takeover-legacy', () => {
expect(db.getLegacyCompatibilityPrincipal(principalId)?.status).toBe('revoked')
})
// Why: the acknowledgement is per claimant, not one-shot — a prior takeover does not license the next.
it('keeps fencing new claimants while legacy work stays live', () => {
// Why: the flag authorizes only the legacy-to-current cutover, never a second current coordinator.
it('keeps fencing flagged claimants after legacy takeover', () => {
const { db, adoptedRunId } = createAdoptedFixture({ settleWork: false })
db.bindRun({
runId: adoptedRunId,
@@ -198,7 +198,8 @@ describe('adopted Run binding without --takeover-legacy', () => {
db.bindRun({
runId: adoptedRunId,
coordinatorHandle: 'term_second_coord',
coordinatorPaneKey: 'tab_second:22222222-2222-4222-8222-222222222222'
coordinatorPaneKey: 'tab_second:22222222-2222-4222-8222-222222222222',
takeoverLegacy: true
})
).toThrowError(expect.objectContaining({ code: 'consumer_fenced' }))
})
@@ -315,7 +316,9 @@ describe('pane-bound Run lookup', () => {
const rebound = db.createRun({
objective: 'second',
coordinatorHandle: 'term_d',
coordinatorPaneKey: 'tab_reminted:99999999-9999-4999-8999-999999999999'
coordinatorPaneKey: 'tab_reminted:99999999-9999-4999-8999-999999999999',
coordinatorProcessIncarnation: 'pty-d:incarnation-1',
coordinatorHostScope: JSON.stringify({ kind: 'local', hostId: 'local' })
})
// createRun binds at generation 1; the unbind fences it to 2.
@@ -267,7 +267,9 @@ describe('OrchestrationDb Run state', () => {
const first = d.createRun({
objective: 'First objective',
coordinatorHandle: 'term_first',
coordinatorPaneKey: 'tab_a:11111111-1111-4111-8111-111111111111'
coordinatorPaneKey: 'tab_a:11111111-1111-4111-8111-111111111111',
coordinatorProcessIncarnation: 'pty:incarnation-1',
coordinatorHostScope: JSON.stringify({ kind: 'local', hostId: 'local' })
})
expect(first).toMatchObject({ consumer_generation: 1, legacy: 0 })
expect(d.getCurrentRunForPane('tab_reminted:11111111-1111-4111-8111-111111111111')?.id).toBe(
@@ -277,7 +279,9 @@ describe('OrchestrationDb Run state', () => {
const second = d.createRun({
objective: 'Second objective',
coordinatorHandle: 'term_second',
coordinatorPaneKey: 'tab_b:11111111-1111-4111-8111-111111111111'
coordinatorPaneKey: 'tab_b:11111111-1111-4111-8111-111111111111',
coordinatorProcessIncarnation: 'pty:incarnation-1',
coordinatorHostScope: JSON.stringify({ kind: 'local', hostId: 'local' })
})
expect(d.getRun(first.id)).toMatchObject({
coordinator_handle: null,
@@ -5,6 +5,7 @@ import { afterEach, describe, expect, it } from 'vitest'
import Database from '../../sqlite/sync-database'
import { OrchestrationDb } from './db'
import { SCHEMA_VERSION } from './db/contract-constants'
import { isCurrentRunCoordinator } from './run-coordinator-authority'
describe('Run coordinator authority migration', () => {
let directory: string | undefined
@@ -40,8 +41,17 @@ describe('Run coordinator authority migration', () => {
expect(db.db.pragma('user_version', { simple: true })).toBe(SCHEMA_VERSION)
expect(db.getRun(run.id)).toMatchObject({
coordinator_process_incarnation: null,
coordinator_host_scope: null
coordinator_host_scope: null,
coordinator_authority_revision: -1
})
expect(
isCurrentRunCoordinator(db.getRun(run.id)!, {
handle: 'term_replacement',
paneKey: 'tab_replacement:11111111-1111-4111-8111-111111111111',
processIncarnation: 'replacement:incarnation-1',
hostScope: JSON.stringify({ kind: 'local', hostId: 'local' })
})
).toBe(false)
const rebound = db.bindRun({
runId: run.id,
@@ -20,13 +20,6 @@ export function isCurrentRunCoordinator(run: RunRow, identity: RunCoordinatorIde
run.coordinator_host_scope === identity.hostScope
)
}
if (run.coordinator_authority_revision === 0) {
return Boolean(
run.coordinator_pane_key &&
identity.paneKey &&
isEquivalentPaneKey(run.coordinator_pane_key, identity.paneKey)
)
}
return Boolean(
run.coordinator_handle === identity.handle &&
run.coordinator_pane_key &&
@@ -48,7 +48,8 @@ describe('Run coordinator handle history migration', () => {
db.bindRun({
runId: run.id,
coordinatorHandle: 'term_new',
coordinatorPaneKey: 'tab:leaf'
coordinatorPaneKey: 'tab:leaf',
authorityContinuity: true
})
const late = db.insertMessage({
runId: run.id,
@@ -120,6 +120,15 @@ describe('orchestration RPC methods', () => {
it('uses the stable pane identity when the coordinator handle was reminted', async () => {
setup()
db.bindRun({
runId: activeRunId!,
coordinatorHandle: 'term_coord',
coordinatorPaneKey,
coordinatorProcessIncarnation: 'pty-coordinator:incarnation-1'
})
vi.mocked(runtime.getTerminalProcessIncarnation).mockReturnValue(
'pty-coordinator:incarnation-1'
)
db.insertMessage({
from: 'term_worker',
to: `run:${activeRunId}`,
@@ -249,7 +249,8 @@ describe('orchestration recipient routing oracle', () => {
db.bindRun({
runId: senderRunId,
coordinatorHandle: 'term_current',
coordinatorPaneKey: 'tab_current:leaf_current'
coordinatorPaneKey: 'tab_current:leaf_current',
authorityContinuity: true
})
mockTerminalPaneKeys((handle) =>
handle === 'term_current' ? 'tab_current:leaf_current' : null
@@ -330,7 +331,8 @@ describe('orchestration recipient routing oracle', () => {
db.bindRun({
runId: foreignRun.id,
coordinatorHandle: 'term_foreign_second',
coordinatorPaneKey: 'tab_foreign_second:leaf_foreign_second'
coordinatorPaneKey: 'tab_foreign_second:leaf_foreign_second',
authorityContinuity: true
})
vi.spyOn(runtime, 'listTerminals').mockResolvedValue({
terminals: [
@@ -363,12 +365,14 @@ describe('orchestration recipient routing oracle', () => {
db.bindRun({
runId: senderRunId,
coordinatorHandle: 'term_middle',
coordinatorPaneKey: 'tab_middle:leaf_middle'
coordinatorPaneKey: 'tab_middle:leaf_middle',
authorityContinuity: true
})
db.bindRun({
runId: senderRunId,
coordinatorHandle: 'term_sender',
coordinatorPaneKey: 'tab_sender:leaf_sender'
coordinatorPaneKey: 'tab_sender:leaf_sender',
authorityContinuity: true
})
vi.spyOn(runtime, 'listTerminals').mockResolvedValue({
terminals: [
@@ -176,7 +176,9 @@ describe('orchestration RPC methods', () => {
setup(false)
const oldPane = 'tab_old:11111111-1111-4111-8111-111111111111'
vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue(oldPane)
vi.spyOn(runtime, 'getLiveTerminalPaneKey').mockReturnValue(null)
vi.spyOn(runtime, 'getLiveTerminalPaneKey').mockImplementation((handle) =>
handle === 'term_new' ? oldPane : null
)
const run = db.createRun({
objective: 'Migrated disconnected owner',
coordinatorHandle: 'term_old',
@@ -208,6 +210,42 @@ describe('orchestration RPC methods', () => {
})
})
it('recovers a migrated binding after the resolved incumbent process is proven exited', async () => {
setup(false)
const oldPane = 'tab_old:11111111-1111-4111-8111-111111111111'
const newPane = 'tab_new:22222222-2222-4222-9222-222222222222'
vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) =>
handle === 'term_old' ? oldPane : newPane
)
vi.spyOn(runtime, 'getOrchestrationDispatchAuthority').mockImplementation((handle) => ({
runtimeId: 'runtime_test',
terminalHandle: handle,
ptyId: handle,
worktreeId: 'folder:workspace',
processIncarnation: `${handle}:incarnation-1`,
paneKey: handle === 'term_old' ? oldPane : newPane,
launchTokenHash: null,
hostScope: { kind: 'local', hostId: 'local' }
}))
const run = db.createRun({
objective: 'Migrated exited owner',
coordinatorHandle: 'term_old',
coordinatorPaneKey: oldPane
})
db.db.prepare('UPDATE runs SET coordinator_authority_revision = -1 WHERE id = ?').run(run.id)
vi.spyOn(runtime, 'inspectTerminalProcessIncarnationLiveness').mockResolvedValue('exited')
const rebound = (await call('orchestration.runUse', {
id: run.id,
from: 'term_new'
})) as { run: { coordinator_handle: string; consumer_generation: number } }
expect(rebound.run).toMatchObject({
coordinator_handle: 'term_new',
consumer_generation: run.consumer_generation + 1
})
})
it('backfills a migrated binding when a reminted handle resolves to the same process', async () => {
setup(false)
const oldPane = 'tab_old:11111111-1111-4111-8111-111111111111'
@@ -309,6 +347,38 @@ describe('orchestration RPC methods', () => {
code: 'consumer_fenced',
data: { effectsApplied: false, coordinatorStatus: 'live' }
})
await expect(
call('orchestration.runCreate', {
objective: 'Must not evict the live owner',
from: 'term_new'
})
).rejects.toMatchObject({ code: 'consumer_fenced', data: { effectsApplied: false } })
expect(db.getRun(created.run.id)).toMatchObject({
coordinator_handle: 'term_old',
consumer_generation: 1
})
const unboundTarget = db.createRun({
objective: 'Unbound target',
coordinatorHandle: 'term_target',
coordinatorPaneKey: 'tab_target:33333333-3333-4333-8333-333333333333'
})
db.db
.prepare(
`UPDATE runs
SET coordinator_handle = NULL, coordinator_pane_key = NULL,
coordinator_process_incarnation = NULL, coordinator_host_scope = NULL
WHERE id = ?`
)
.run(unboundTarget.id)
await expect(
call('orchestration.runUse', { id: unboundTarget.id, from: 'term_new' })
).rejects.toMatchObject({ code: 'consumer_fenced', data: { effectsApplied: false } })
expect(db.getRun(created.run.id)).toMatchObject({
coordinator_handle: 'term_old',
consumer_generation: 1
})
})
it('allows a distinct coordinator only after the owning host proves the incumbent exited', async () => {
@@ -423,6 +493,48 @@ describe('orchestration RPC methods', () => {
expect(cancel).toHaveBeenCalledWith(`run:${overlapping.id}`)
})
it('rejects a claimant process that reincarnates during incumbent observation', async () => {
setup(false)
const oldPane = 'tab_old:11111111-1111-4111-8111-111111111111'
const newPane = 'tab_new:22222222-2222-4222-9222-222222222222'
let claimantIncarnation = 'term_new:incarnation-1'
vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) =>
handle === 'term_old' ? oldPane : newPane
)
vi.spyOn(runtime, 'getOrchestrationDispatchAuthority').mockImplementation((handle) => ({
runtimeId: 'runtime_test',
terminalHandle: handle,
ptyId: handle,
worktreeId: 'folder:workspace',
processIncarnation: handle === 'term_old' ? 'term_old:incarnation-1' : claimantIncarnation,
paneKey: handle === 'term_old' ? oldPane : newPane,
launchTokenHash: null,
hostScope: { kind: 'local', hostId: 'local' }
}))
const target = (await call('orchestration.runCreate', {
objective: 'Keep claimant exact',
from: 'term_old'
})) as { run: { id: string } }
let finishObservation: ((status: 'exited') => void) | undefined
vi.spyOn(runtime, 'inspectTerminalProcessIncarnationLiveness').mockImplementation(
() => new Promise((resolve) => (finishObservation = resolve))
)
const use = call('orchestration.runUse', { id: target.run.id, from: 'term_new' })
await vi.waitFor(() => expect(finishObservation).toBeTypeOf('function'))
claimantIncarnation = 'term_new:incarnation-2'
finishObservation?.('exited')
await expect(use).rejects.toMatchObject({
code: 'consumer_fenced',
data: { effectsApplied: false }
})
expect(db.getRun(target.run.id)).toMatchObject({
coordinator_handle: 'term_old',
consumer_generation: 1
})
})
it('requires an explicit binding before task mutation', async () => {
setup(false)
vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue(coordinatorPaneKey)
@@ -446,6 +558,7 @@ describe('orchestration RPC methods', () => {
setup(false)
const oldPane = 'tab_old:11111111-1111-4111-8111-111111111111'
const newPane = 'tab_new:22222222-2222-4222-9222-222222222222'
const otherPane = 'tab_other:33333333-3333-4333-8333-333333333333'
vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) =>
handle === 'term_old' ? oldPane : newPane
)
@@ -457,7 +570,7 @@ describe('orchestration RPC methods', () => {
const runB = db.createRun({
objective: 'B',
coordinatorHandle: 'term_other',
coordinatorPaneKey: newPane
coordinatorPaneKey: otherPane
})
const taskA = db.createTask({ spec: 'A work', runId: runA.id })
db.createTask({ spec: 'B work', runId: runB.id })
@@ -8,7 +8,10 @@ import {
resolveOrchestrationCaller
} from './orchestration-run-scope'
import type { RunRow } from '../../orchestration/types'
import { isCurrentRunCoordinator } from '../../orchestration/run-coordinator-authority'
import {
isCurrentRunCoordinator,
type RunCoordinatorIdentity
} from '../../orchestration/run-coordinator-authority'
import {
isCallerCurrentRunCoordinator,
resolveRunCoordinatorIdentity
@@ -16,14 +19,15 @@ import {
async function observeRunCoordinator(
runtime: Parameters<typeof resolveOrchestrationCaller>[0],
run: RunRow
run: RunRow,
resolvedIdentity?: RunCoordinatorIdentity | null
) {
let status: 'live' | 'unverifiable' | 'exited' = 'unverifiable'
if (run.coordinator_process_incarnation && run.coordinator_host_scope) {
status = await runtime.inspectTerminalProcessIncarnationLiveness(
run.coordinator_process_incarnation,
run.coordinator_host_scope
)
const processIncarnation =
run.coordinator_process_incarnation ?? resolvedIdentity?.processIncarnation
const hostScope = run.coordinator_host_scope ?? resolvedIdentity?.hostScope
if (processIncarnation && hostScope) {
status = await runtime.inspectTerminalProcessIncarnationLiveness(processIncarnation, hostScope)
} else if (run.coordinator_handle) {
const verdict = runtime.getTerminalLivenessVerdict(run.coordinator_handle)
if (runtime.getLiveTerminalPaneKey(run.coordinator_handle) || verdict?.status === 'live') {
@@ -142,7 +146,47 @@ export const ORCHESTRATION_RUN_METHODS: RpcMethod[] = [
? isCurrentRunCoordinator(targetRun, identity) || dynamicProcessContinuity
: false
const incumbentObservation =
targetRun && !sameAuthority ? await observeRunCoordinator(runtime, targetRun) : undefined
targetRun && !sameAuthority
? await observeRunCoordinator(runtime, targetRun, incumbentIdentity)
: undefined
const currentPaneKey = resolveOrchestrationCaller(runtime, {
callerTerminalHandle: params.from,
callerEvidence: orchestrationCompatibilityEvidence,
callerAuthority,
requireStablePane: true,
evidenceAssertedByCaller: true
})
assertCallerHandleMatchesEvidence(runtime, params.from, orchestrationCompatibilityEvidence)
const currentIdentity = resolveRunCoordinatorIdentity(runtime, params.from, currentPaneKey)
const revalidatedCaller = orchestrationCompatibilityEvidence
? runtime.verifyOrchestrationCompatibilityCaller(
orchestrationCompatibilityEvidence,
params.takeoverLegacy ? { currentRuntimeLaunchSufficient: true } : undefined
)
: null
const claimantStillLive = revalidatedCaller
? revalidatedCaller.terminalHandle === params.from && revalidatedCaller.paneKey === paneKey
: legacyCoordinatorAuthority || !orchestrationCompatibilityEvidence
? runtime.getLiveTerminalPaneKey(params.from) === paneKey
: false
if (
(incumbentObservation && !claimantStillLive) ||
currentPaneKey !== paneKey ||
currentIdentity.processIncarnation !== identity.processIncarnation ||
currentIdentity.hostScope !== identity.hostScope
) {
throw new OrchestrationError(
'consumer_fenced',
'The claiming coordinator process changed while Run authority was being checked. No effects were applied.',
{
effectsApplied: false,
nextSteps: [
`Inspect current authority with orca orchestration run-show --id ${params.id} --json.`,
`Retry orca orchestration run-use --id ${params.id} --json from the replacement coordinator process.`
]
}
)
}
const priorRun = db.getCurrentRunForPane(paneKey)
const run = db.bindRun({
runId: params.id,
@@ -76,7 +76,8 @@ describe('orchestration RPC methods', () => {
db.bindRun({
runId: activeRunId!,
coordinatorHandle: 'term_reminted',
coordinatorPaneKey: remintedPaneKey
coordinatorPaneKey: remintedPaneKey,
authorityContinuity: true
})
const waiting = runtime.waitForMessage(`run:${activeRunId}`, { timeoutMs: 5_000 })
@@ -53,7 +53,7 @@ describe('orchestration RPC methods', () => {
})
it('records the caller pane, process, and Run generation when creating a task', async () => {
setup()
setup(false)
vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) =>
handle === 'term_creator' ? coordinatorPaneKey : null
)
@@ -62,9 +62,14 @@ describe('orchestration RPC methods', () => {
paneKey: coordinatorPaneKey,
processIncarnation: 'pty-creator:incarnation-a'
} as never)
const created = (await call('orchestration.runCreate', {
objective: 'Creator authority',
from: 'term_creator'
})) as { run: { id: string } }
const result = (await call('orchestration.taskCreate', {
spec: 'spawn related workspace',
callerTerminalHandle: 'term_creator'
callerTerminalHandle: 'term_creator',
run: created.run.id
})) as { task: { id: string } }
expect(db.getTask(result.task.id)).toMatchObject({
@@ -303,7 +303,7 @@ describe('legacy coordinator delivery targets after takeover', () => {
})
// Why: bindRun only revokes a committed principal when it takes over or the legacy work is settled,
// so a coordinator restarting inside the legacy pane rebinds the Run while the principal stays
// so a coordinator reminting its handle inside the legacy pane rebinds while the principal stays
// committed. Delivery still routes legacy_direct to the addressed handle there, and no reader can
// see that mailbox, so the permit must refuse rather than accept mail nobody will ever read.
describe('legacy coordinator delivery targets without a takeover', () => {
@@ -329,7 +329,8 @@ describe('legacy coordinator delivery targets without a takeover', () => {
harness.db.bindRun({
runId: harness.adoptedRunId,
coordinatorHandle: REBOUND_COORDINATOR_HANDLE,
coordinatorPaneKey: COORDINATOR_PANE
coordinatorPaneKey: COORDINATOR_PANE,
authorityContinuity: true
})
).toMatchObject({ coordinator_handle: REBOUND_COORDINATOR_HANDLE })
expect(harness.db.getLegacyCoordinatorPrincipal(harness.adoptedRunId)?.status).toBe('committed')
@@ -140,9 +140,7 @@ function createUpdateHarness(): Harness {
runtime.setOrchestrationDb(db)
vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue(null)
const getProcessIncarnation = runtime.getTerminalProcessIncarnation.bind(runtime)
vi.spyOn(runtime, 'getTerminalProcessIncarnation')
.mockImplementationOnce(getProcessIncarnation)
.mockReturnValue(null)
vi.spyOn(runtime, 'getTerminalProcessIncarnation').mockImplementation(getProcessIncarnation)
vi.spyOn(runtime, 'notifyMessageArrived').mockImplementation(() => {})
return new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS })
}