mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 00:02:31 +00:00
* fix(orchestration): enforce nested worker depth instead of an accidental fence Orca documented that "dispatched workers cannot spawn their own sub-workers (worker-start is coordinator-fenced)". No such check existed. What existed was a single Run-binding check in the workerStart RPC: a worker's terminal is not bound to a Run, so worker-start happened to fail. The rule was emergent, asserted by no test, and written in no doc — and it leaked. A worker could run-create its own Run, task-create, and worker-start: now bound, the check passed. Replace it with a real, configurable depth cap. Depth is derived from the caller's own active Dispatch rather than from Run binding, which is what dissolves the run-create bypass: creating a Run does not stop you being a worker. Enforcement lives in a single dispatch-row writer that owns all three INSERTs that mint a live worker — the generic claim, the supervised worker-start path (including every retry), and the remote attachment. Two of those were missed by earlier drafts of this change, so `creator` and `maxDepth` are required parameters: a new spawn path cannot compile without deciding, and a boundary test refuses the SQL anywhere else. Schema v30 adds depth to dispatch_contexts and remote_dispatch_attachments, NOT NULL DEFAULT 1 and backfilled to 1 so an unstamped or pre-upgrade row fails closed rather than reading as a root coordinator. The attachment pane indexes widen to the five states in which a remote worker may still be running: loss of contact is not evidence of process death, so an unverifiable worker still counts as a nesting parent. Also adds the caller-evidence assertion that workerStart was the only Run-scoped verb to skip, so a declared --from cannot name another terminal's pane and inherit its depth. Default is 1, so behaviour is unchanged unless the new setting is raised. Two limitations are deliberate and documented rather than papered over: this is a guardrail and not a security boundary, since a caller whose launch evidence is unverifiable (any ordinary restored terminal) can declare another handle; and it is enforced at supervised dispatch creation, so a settled worker whose process is still alive counts as a root again. * fix(orchestration): share caller resolution and pin worker gaps * refactor(orchestration): make the caller resolver's pane contract explicit Overloads so requireStablePane callers get a non-null string instead of casting, and rename the attestation opt-out to say what it means: the caller asserts it itself. A flag called assertEvidence:false reads as "attestation optional", which is the hole this helper exists to close. * fix(orchestration): propagate dispatch depth to federated workers * chore(cli): refresh bundled orchestration guide
272 lines
9.2 KiB
TypeScript
272 lines
9.2 KiB
TypeScript
import { afterEach, describe, expect, it } from 'vitest'
|
|
import { OrchestrationDb } from '../db'
|
|
import { AmbiguousDispatchParentError } from './dispatch-depth'
|
|
|
|
/**
|
|
* These pin the fence Orca documented but never enforced: before this feature a
|
|
* dispatched worker could create its own Run and dispatch sub-workers freely.
|
|
* Every rejection case here passes on the pre-change tree.
|
|
*/
|
|
describe('nested worker depth', () => {
|
|
let db: OrchestrationDb
|
|
const SYSTEM = { kind: 'system' } as const
|
|
const UNCAPPED = Number.MAX_SAFE_INTEGER
|
|
|
|
afterEach(() => db?.close())
|
|
|
|
function coordinatorDispatchesWorker(maxDepth = UNCAPPED) {
|
|
db = new OrchestrationDb(':memory:')
|
|
const task = db.createTask({ spec: 'root task' })
|
|
const worker = db.createDispatchContext({
|
|
taskId: task.id,
|
|
assigneeHandle: 'term_worker',
|
|
assigneePaneKey: 'tab_worker:leaf_worker',
|
|
creator: SYSTEM,
|
|
maxDepth
|
|
})
|
|
return worker
|
|
}
|
|
|
|
it('stamps a root coordinator dispatch at depth 1', () => {
|
|
expect(coordinatorDispatchesWorker().depth).toBe(1)
|
|
})
|
|
|
|
it('refuses a worker dispatching a sub-worker at the default cap', () => {
|
|
coordinatorDispatchesWorker()
|
|
const nested = db.createTask({ spec: 'nested task' })
|
|
expect(() =>
|
|
db.createDispatchContext({
|
|
taskId: nested.id,
|
|
assigneeHandle: 'term_sub',
|
|
assigneePaneKey: 'tab_sub:leaf_sub',
|
|
creator: {
|
|
kind: 'terminal',
|
|
handle: 'term_worker',
|
|
paneKey: 'tab_worker:leaf_worker'
|
|
},
|
|
maxDepth: 1
|
|
})
|
|
).toThrow(/depth 2 \(max 1\)/)
|
|
})
|
|
|
|
it('tells the refused worker to complete the task itself', () => {
|
|
coordinatorDispatchesWorker()
|
|
const nested = db.createTask({ spec: 'nested task' })
|
|
expect(() =>
|
|
db.createDispatchContext({
|
|
taskId: nested.id,
|
|
assigneeHandle: 'term_sub',
|
|
creator: { kind: 'terminal', handle: 'term_worker', paneKey: 'tab_worker:leaf_worker' },
|
|
maxDepth: 1
|
|
})
|
|
).toThrow(/Complete this task yourself/)
|
|
})
|
|
|
|
it('permits one more generation when the cap is raised, and records depth 2', () => {
|
|
coordinatorDispatchesWorker()
|
|
const nested = db.createTask({ spec: 'nested task' })
|
|
const sub = db.createDispatchContext({
|
|
taskId: nested.id,
|
|
assigneeHandle: 'term_sub',
|
|
assigneePaneKey: 'tab_sub:leaf_sub',
|
|
creator: { kind: 'terminal', handle: 'term_worker', paneKey: 'tab_worker:leaf_worker' },
|
|
maxDepth: 2
|
|
})
|
|
expect(sub.depth).toBe(2)
|
|
})
|
|
|
|
it('closes the run-create bypass: a fresh Run does not reset the creator depth', () => {
|
|
// The old fence keyed off Run binding, so a worker that created its own Run
|
|
// walked straight through. Depth comes from the creator's dispatch instead.
|
|
coordinatorDispatchesWorker()
|
|
const ownRun = db.createRun({
|
|
objective: 'worker-owned run',
|
|
coordinatorHandle: 'term_worker',
|
|
coordinatorPaneKey: 'tab_worker:leaf_worker'
|
|
})
|
|
const nested = db.createTask({ spec: 'nested task', runId: ownRun.id })
|
|
expect(() =>
|
|
db.createDispatchContext({
|
|
taskId: nested.id,
|
|
assigneeHandle: 'term_sub',
|
|
creator: { kind: 'terminal', handle: 'term_worker', paneKey: 'tab_worker:leaf_worker' },
|
|
maxDepth: 1
|
|
})
|
|
).toThrow(/depth 2 \(max 1\)/)
|
|
})
|
|
|
|
it('treats the in-process coordinator loop as a root even from a worker pane', () => {
|
|
coordinatorDispatchesWorker()
|
|
expect(db.resolveCreatorDepth({ kind: 'system' })).toBe(0)
|
|
})
|
|
|
|
it('resolves an unknown terminal to root depth', () => {
|
|
db = new OrchestrationDb(':memory:')
|
|
expect(db.resolveCreatorDepth({ kind: 'terminal', handle: 'term_nobody' })).toBe(0)
|
|
})
|
|
|
|
describe('remote attachments as parents', () => {
|
|
const PANE = 'tab_remote:leaf_remote'
|
|
const INCARNATION = 'inc-1'
|
|
|
|
function attachRemoteWorker(state: string, depth: number, paneKey = PANE, inc = INCARNATION) {
|
|
db.db
|
|
.prepare(
|
|
`INSERT INTO remote_dispatch_attachments
|
|
(dispatch_id, task_id, home_peer_fingerprint, protocol_version, runtime_epoch,
|
|
pane_key, process_incarnation, state, depth)
|
|
VALUES (?, ?, 'peer', 1, 'epoch', ?, ?, ?, ?)`
|
|
)
|
|
.run(`ctx_${state}_${depth}_${paneKey}_${inc}`, 'task_remote', paneKey, inc, state, depth)
|
|
}
|
|
|
|
// Loss of contact is never evidence of process death: an unverifiable remote
|
|
// worker must still block nesting. See docs/reference/ssh-execution-boundary.md.
|
|
for (const state of ['starting', 'ready', 'start_unknown', 'stopping', 'stop_unknown']) {
|
|
it(`counts a '${state}' attachment as a live parent`, () => {
|
|
db = new OrchestrationDb(':memory:')
|
|
attachRemoteWorker(state, 1)
|
|
expect(
|
|
db.resolveCreatorDepth({
|
|
kind: 'terminal',
|
|
handle: 'term_remote',
|
|
paneKey: PANE,
|
|
processIncarnation: INCARNATION
|
|
})
|
|
).toBe(1)
|
|
})
|
|
}
|
|
|
|
for (const state of ['succeeded', 'failed', 'stopped', 'abandoned']) {
|
|
it(`does not count a settled '${state}' attachment`, () => {
|
|
db = new OrchestrationDb(':memory:')
|
|
attachRemoteWorker(state, 1)
|
|
expect(
|
|
db.resolveCreatorDepth({
|
|
kind: 'terminal',
|
|
handle: 'term_remote',
|
|
paneKey: PANE,
|
|
processIncarnation: INCARNATION
|
|
})
|
|
).toBe(0)
|
|
})
|
|
}
|
|
|
|
it('ignores an attachment whose pane was reused by a new process', () => {
|
|
db = new OrchestrationDb(':memory:')
|
|
attachRemoteWorker('ready', 2)
|
|
expect(
|
|
db.resolveCreatorDepth({
|
|
kind: 'terminal',
|
|
handle: 'term_remote',
|
|
paneKey: PANE,
|
|
processIncarnation: 'inc-2'
|
|
})
|
|
).toBe(0)
|
|
})
|
|
|
|
it('fails closed when one identity matches two live attachments', () => {
|
|
db = new OrchestrationDb(':memory:')
|
|
attachRemoteWorker('ready', 1)
|
|
attachRemoteWorker('starting', 2)
|
|
expect(() =>
|
|
db.resolveCreatorDepth({
|
|
kind: 'terminal',
|
|
handle: 'term_remote',
|
|
paneKey: PANE,
|
|
processIncarnation: INCARNATION
|
|
})
|
|
).toThrow(AmbiguousDispatchParentError)
|
|
})
|
|
|
|
it('takes the maximum when a process holds both a local and a remote role', () => {
|
|
// Query order must not decide the answer: the deeper role governs.
|
|
db = new OrchestrationDb(':memory:')
|
|
const task = db.createTask({ spec: 'local role' })
|
|
db.createDispatchContext({
|
|
taskId: task.id,
|
|
assigneeHandle: 'term_both',
|
|
assigneePaneKey: PANE,
|
|
creator: { kind: 'system' },
|
|
maxDepth: UNCAPPED
|
|
})
|
|
attachRemoteWorker('ready', 3)
|
|
expect(
|
|
db.resolveCreatorDepth({
|
|
kind: 'terminal',
|
|
handle: 'term_both',
|
|
paneKey: PANE,
|
|
processIncarnation: INCARNATION
|
|
})
|
|
).toBe(3)
|
|
})
|
|
})
|
|
|
|
describe('the supervised worker-start path', () => {
|
|
// r2 put enforcement in createDispatchContext and missed this entirely:
|
|
// worker-start has its own insert, and so does every retry through it.
|
|
function startWorker(
|
|
taskId: string,
|
|
creator: Parameters<OrchestrationDb['resolveCreatorDepth']>[0],
|
|
maxDepth: number
|
|
) {
|
|
return db.createStartingWorkerDispatch({
|
|
taskId,
|
|
startOptions: {},
|
|
creator,
|
|
maxDepth
|
|
})
|
|
}
|
|
|
|
it('stamps depth 1 for a root coordinator', () => {
|
|
db = new OrchestrationDb(':memory:')
|
|
const task = db.createTask({ spec: 'root work' })
|
|
expect(startWorker(task.id, SYSTEM, UNCAPPED).dispatch.depth).toBe(1)
|
|
})
|
|
|
|
it('refuses a worker starting a sub-worker at the default cap', () => {
|
|
coordinatorDispatchesWorker()
|
|
const nested = db.createTask({ spec: 'nested work' })
|
|
expect(() =>
|
|
startWorker(
|
|
nested.id,
|
|
{ kind: 'terminal', handle: 'term_worker', paneKey: 'tab_worker:leaf_worker' },
|
|
1
|
|
)
|
|
).toThrow(/depth 2 \(max 1\)/)
|
|
})
|
|
|
|
it('refuses a worker retrying into a sub-worker at the default cap', () => {
|
|
coordinatorDispatchesWorker()
|
|
const nested = db.createTask({ spec: 'nested retry work' })
|
|
const first = startWorker(nested.id, SYSTEM, UNCAPPED)
|
|
db.failWorkerStart(first.dispatch.id, 'accepted', 'first attempt failed')
|
|
expect(() =>
|
|
db.createStartingWorkerDispatch({
|
|
taskId: nested.id,
|
|
startOptions: {},
|
|
retryOf: first.dispatch.id,
|
|
creator: { kind: 'terminal', handle: 'term_worker', paneKey: 'tab_worker:leaf_worker' },
|
|
maxDepth: 1
|
|
})
|
|
).toThrow(/depth 2 \(max 1\)/)
|
|
})
|
|
})
|
|
|
|
it('keeps a local row with a null process incarnation eligible as a parent', () => {
|
|
// Context-only dispatch stores null on purpose; requiring an incarnation
|
|
// locally would silently drop real parents and fail open.
|
|
db = new OrchestrationDb(':memory:')
|
|
const task = db.createTask({ spec: 'context only' })
|
|
const row = db.createDispatchContext({
|
|
taskId: task.id,
|
|
assigneeHandle: 'term_ctx',
|
|
assigneePaneKey: 'tab_ctx:leaf_ctx',
|
|
creator: { kind: 'system' },
|
|
maxDepth: UNCAPPED
|
|
})
|
|
expect(row.process_incarnation).toBeNull()
|
|
expect(db.resolveCreatorDepth({ kind: 'terminal', handle: 'term_ctx' })).toBe(1)
|
|
})
|
|
})
|