mirror of
https://github.com/stablyai/orca.git
synced 2026-10-24 08:03:58 +00:00
feat(orchestration): add durable run capacity targets
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"updatedAt": "2026-08-23",
|
||||
"updatedAt": "2026-08-27",
|
||||
"policy": {
|
||||
"maturityLevels": ["experimental", "soak", "blocking", "accepted-gap", "deprecated"],
|
||||
"blockingPromotion": {
|
||||
@@ -2148,9 +2148,7 @@
|
||||
},
|
||||
{
|
||||
"file": "src/cli/runtime/serve-signal-exit-diagnostic.test.ts",
|
||||
"assertions": [
|
||||
"late update handoff failure cannot rearm termination after child exit"
|
||||
]
|
||||
"assertions": ["late update handoff failure cannot rearm termination after child exit"]
|
||||
},
|
||||
{
|
||||
"file": "src/main/serve-update-handoff.test.ts",
|
||||
@@ -16425,9 +16423,7 @@
|
||||
},
|
||||
{
|
||||
"file": "src/main/browser/agent-browser-bridge-tab-routing.test.ts",
|
||||
"assertions": [
|
||||
"closing a tab retires the exact named agent-browser session"
|
||||
]
|
||||
"assertions": ["closing a tab retires the exact named agent-browser session"]
|
||||
},
|
||||
{
|
||||
"file": "src/main/startup/serve-signal-handlers.test.ts",
|
||||
@@ -16570,6 +16566,103 @@
|
||||
"The sentinel changes a pane title within an existing layout; concurrent split and close conflicts remain separate coverage."
|
||||
],
|
||||
"demotionRule": "Demote if a failed push suppresses an identical retry, a successful equal write resumes redundant churn, or the routed observer journey flakes without a diagnosed cause."
|
||||
},
|
||||
{
|
||||
"id": "orchestration.run-capacity-target",
|
||||
"title": "Run capacity targets expose and enforce safe backfill slots",
|
||||
"maturity": "experimental",
|
||||
"protection": "partial",
|
||||
"owner": "orchestration-runtime",
|
||||
"layer": "run-home-database-and-cli",
|
||||
"surfaces": [
|
||||
"Run target concurrency",
|
||||
"explicit Task capacity enrollment",
|
||||
"local and federated worker start",
|
||||
"settled worker terminal accounting"
|
||||
],
|
||||
"platforms": ["macos", "linux", "windows"],
|
||||
"providers": ["local", "ssh", "wsl", "remote-runtime"],
|
||||
"coveredPlatforms": ["macos"],
|
||||
"coveredProviders": ["local"],
|
||||
"coverageNotes": "Deterministic Run-home database and RPC contracts cover exact atomic admission, simultaneous settlement, dependency and decision-gate promotion, failed and blocked Tasks, empty eligible queues, retry circuit breaking, restart persistence, federated home admission, idempotent mutation replay, and retained, user-owned, reclaimable, and release-unknown terminal resources. Remote provider parity is architectural because all starts claim the capacity slot in the authoritative Run-home transaction before placement; live remote host journeys remain a gap.",
|
||||
"motivatingLinks": [
|
||||
"https://linear.app/stably/issue/STA-3664",
|
||||
"https://linear.app/stably/issue/STA-4693"
|
||||
],
|
||||
"invariant": "A Run with target concurrency N never admits more than N pending or dispatched attempts; only ready Tasks explicitly enrolled in that Run's capacity pool are exposed for backfill, and settlement reopens the exact slot without treating retained, user-owned, or unverifiable terminals as active lanes or automatically closing them.",
|
||||
"oracle": "Configure a target, enroll ready and dependency-blocked Tasks, and start concurrent local and federated attempts through both composed worker-start and low-level dispatch. Require the Run-home transaction to reject attempt N+1, require simultaneous worker_done settlement and dependency promotion to expose the exact reopened slots, require restart and replay to preserve target/enrollment without duplicate Dispatches, and require settled terminal cleanup debt to report reclaimable, retained/user-takeover, and release-unknown resources without closing them.",
|
||||
"commands": [
|
||||
"pnpm test src/main/runtime/orchestration/run-capacity.test.ts src/main/runtime/rpc/methods/orchestration-capacity.test.ts src/cli/handlers/orchestration-capacity-cli.test.ts",
|
||||
"pnpm tc:node && pnpm tc:cli"
|
||||
],
|
||||
"testFiles": [
|
||||
"src/main/runtime/orchestration/run-capacity.test.ts",
|
||||
"src/main/runtime/rpc/methods/orchestration-capacity.test.ts",
|
||||
"src/cli/handlers/orchestration-capacity-cli.test.ts"
|
||||
],
|
||||
"assertionRefs": [
|
||||
{
|
||||
"file": "src/main/runtime/orchestration/run-capacity.test.ts",
|
||||
"assertions": [
|
||||
"capacity admission is atomic and exact across composed and low-level dispatch paths",
|
||||
"federated starts claim the exact slot at the authoritative Run home",
|
||||
"simultaneous settlement and dependency promotion reopen the exact launchable slots",
|
||||
"pending decision gates suppress launchability until resolution",
|
||||
"launchable observation is index-backed and bounded by the available slots",
|
||||
"ready Tasks with a pre-fix active split are never offered twice",
|
||||
"restart persistence retains the target and explicit Task enrollment",
|
||||
"settled terminal debt distinguishes reclaimable, user takeover, and release-unknown resources without closing them"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "src/main/runtime/rpc/methods/orchestration-capacity.test.ts",
|
||||
"assertions": ["capacity mutations stay scoped to the coordinator's authoritative Run"]
|
||||
},
|
||||
{
|
||||
"file": "src/cli/handlers/orchestration-capacity-cli.test.ts",
|
||||
"assertions": [
|
||||
"capacity mutations carry replay identity and worker-start claims a slot only when explicitly requested"
|
||||
]
|
||||
}
|
||||
],
|
||||
"evidenceRuns": [
|
||||
{
|
||||
"date": "2026-08-27",
|
||||
"runner": "local",
|
||||
"platform": "macos",
|
||||
"result": "passed",
|
||||
"durationSeconds": 19.45,
|
||||
"command": "pnpm test src/main/runtime/orchestration/run-capacity.test.ts src/main/runtime/rpc/methods/orchestration-capacity.test.ts src/cli/handlers/orchestration-capacity-cli.test.ts",
|
||||
"summary": "Focused database, RPC, and CLI contracts passed after the original four-test oracle failed because no Run capacity contract existed."
|
||||
}
|
||||
],
|
||||
"runtimeBudget": {
|
||||
"p95Seconds": 25,
|
||||
"scope": "focused in-memory database, RPC, and CLI contracts"
|
||||
},
|
||||
"flakeHistory": {
|
||||
"status": "not-started",
|
||||
"evidence": "Fresh deterministic coverage has local candidate evidence only."
|
||||
},
|
||||
"redGreenEvidence": {
|
||||
"status": "complete",
|
||||
"evidence": "The initial focused oracle failed because configureRunCapacity and the capacity CLI contract did not exist; the candidate passes the same behavior-level contracts."
|
||||
},
|
||||
"performanceBudget": {
|
||||
"required": true,
|
||||
"evidence": "Capacity observation and admission are caller-driven and event-aligned. They add no timer, polling, subprocess, provider fanout, or wire opcode; admission uses one indexed Run-home count inside the existing BEGIN IMMEDIATE transaction, the launchable query is capped at the 64-slot maximum, and pending/debt queries use indexed filters instead of full historical inventories."
|
||||
},
|
||||
"promotionCriteria": [
|
||||
"Collect 100 consecutive routed CI passes or 14 days without an unexplained flake.",
|
||||
"Add live compiled-CLI Run-home and federated-server journeys on Windows, WSL, Linux, and physical SSH.",
|
||||
"Keep exact capacity admission, dependency promotion, replay, restart, and terminal-debt assertions green."
|
||||
],
|
||||
"knownGaps": [
|
||||
"Orca does not infer Task placement, worktree conflicts, agent choice, or setup policy; a coordinator loop must select those explicitly for each reported eligible Task.",
|
||||
"Live federated Windows, WSL, Linux, and physical SSH journeys are not recorded; provider parity is currently proven at the shared Run-home admission boundary.",
|
||||
"Settled retained and release-unknown terminals are reported as cleanup debt and intentionally remain open until their ownership is proved."
|
||||
],
|
||||
"demotionRule": "Demote if any start path exceeds the target, an unenrolled or dependency-blocked Task appears launchable, restart or replay duplicates admission, or capacity accounting closes or hides an unproven terminal resource."
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -219,6 +219,30 @@ orca orchestration worker-start --task <task_b> --worktree current --agent claud
|
||||
|
||||
`current` and exact existing worktrees create a fresh agent terminal and do not rerun setup. Reuse an existing agent only with `--terminal <handle>`.
|
||||
|
||||
### Durable target concurrency and safe backfill
|
||||
|
||||
A coordinator can declare a Run-wide execution ceiling and explicitly enroll independent Tasks in its capacity pool:
|
||||
|
||||
```bash
|
||||
orca orchestration capacity-set --target 5 --json
|
||||
orca orchestration capacity-enroll --task <task_id> --json
|
||||
orca orchestration capacity-show --json
|
||||
```
|
||||
|
||||
`capacity-show` reports the target, every active Dispatch in the Run, open slots, enrolled ready Tasks in stable creation order, enrolled pending/blocked Tasks, and settled terminal cleanup debt. Only Tasks explicitly enrolled with `capacity-enroll` appear as launchable; dependency promotion and decision-gate resolution must first make the Task `ready`. Failed, blocked, already dispatched, and circuit-broken work never appears as launchable.
|
||||
|
||||
The runtime deliberately does not invent worker placement. For each Task returned under `launchableTasks`, the coordinator chooses the existing `worker-start` placement and claims one slot atomically:
|
||||
|
||||
```bash
|
||||
orca orchestration worker-start --task <task_id> --capacity-slot --worktree current --agent codex --json
|
||||
```
|
||||
|
||||
Once a target is nonzero, every new Dispatch in that Run—including low-level `dispatch` and Tasks outside the pool—must fit below it. `--capacity-slot` additionally requires explicit pool enrollment. Concurrent starts serialize at the Run home, so a stale `capacity-show` or idempotent coordinator replay can fail with `capacity_full` or `task_not_startable` but cannot exceed the target or duplicate a Task. A target of `0` disables capacity-slot launches without stopping existing work.
|
||||
|
||||
After each accepted `worker_done`, release or reuse the settled worker as below, then call `capacity-show` again and start up to `launchableCount`. This closes the settled-lane backfill gap without a scheduler: Runs remain authoritative, placement remains explicit, and the same loop works after coordinator or runtime restart because target and enrollment are durable. Retained, user-owned, external, and `release_unknown` terminals are reported as cleanup debt; they are never auto-closed and do not masquerade as live execution lanes.
|
||||
|
||||
Use `capacity-withdraw --task <task_id>` to remove a Task from future backfill. Enrollment changes neither Task status nor dependencies and never cancels an active Dispatch.
|
||||
|
||||
For a per-invocation Claude, Codex, or Cursor launch, pass an opaque provider model id with `--model`; add `--effort` only when that agent/model supports the level. These options apply only to fresh agent terminals, override general agent default arguments, and are reported under `launch.requested` and `launch.effective` in the receipt:
|
||||
|
||||
```bash
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -108,6 +108,10 @@ export const HANDLER_GROUPS: readonly HandlerGroup[] = [
|
||||
'orchestration run-current',
|
||||
'orchestration run-list',
|
||||
'orchestration run-show',
|
||||
'orchestration capacity-set',
|
||||
'orchestration capacity-show',
|
||||
'orchestration capacity-enroll',
|
||||
'orchestration capacity-withdraw',
|
||||
'orchestration send',
|
||||
'orchestration check',
|
||||
'orchestration reply',
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const callMock = vi.fn()
|
||||
|
||||
vi.mock('../format', () => ({ printResult: vi.fn() }))
|
||||
vi.mock('../selectors', () => ({ getTerminalHandle: vi.fn() }))
|
||||
|
||||
import { ORCHESTRATION_HANDLERS } from './orchestration'
|
||||
|
||||
describe('orchestration capacity CLI contract', () => {
|
||||
beforeEach(() => callMock.mockReset())
|
||||
|
||||
it('forwards target configuration with mutation replay identity', async () => {
|
||||
callMock.mockResolvedValue({ result: { capacity: { runId: 'run_1' } } })
|
||||
|
||||
await ORCHESTRATION_HANDLERS['orchestration capacity-set']({
|
||||
flags: new Map<string, string | boolean>([
|
||||
['target', '5'],
|
||||
['run', 'run_1'],
|
||||
['from', 'term_coord'],
|
||||
['retry-request', 'request_1']
|
||||
]),
|
||||
client: { call: callMock },
|
||||
cwd: '/tmp/repo',
|
||||
json: true
|
||||
} as never)
|
||||
|
||||
expect(callMock).toHaveBeenCalledWith(
|
||||
'orchestration.capacityConfigure',
|
||||
{ target: 5, run: 'run_1', from: 'term_coord' },
|
||||
{ orchestrationRequestId: 'request_1' }
|
||||
)
|
||||
})
|
||||
|
||||
it('marks a worker start as an enrolled capacity claim only when requested', async () => {
|
||||
callMock
|
||||
.mockResolvedValueOnce({
|
||||
result: { capabilities: ['orchestration.run-capacity.v1'] }
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
result: {
|
||||
runId: 'run_1',
|
||||
taskId: 'task_1',
|
||||
dispatchId: 'ctx_1',
|
||||
state: 'ready',
|
||||
effects: [],
|
||||
residualResources: []
|
||||
}
|
||||
})
|
||||
|
||||
await ORCHESTRATION_HANDLERS['orchestration worker-start']({
|
||||
flags: new Map<string, string | boolean>([
|
||||
['task', 'task_1'],
|
||||
['agent', 'codex'],
|
||||
['capacity-slot', true],
|
||||
['from', 'term_coord']
|
||||
]),
|
||||
client: { call: callMock },
|
||||
cwd: '/tmp/repo',
|
||||
json: true
|
||||
} as never)
|
||||
|
||||
expect(callMock).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
'orchestration.workerStart',
|
||||
expect.objectContaining({ task: 'task_1', capacitySlot: true })
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -8,7 +8,10 @@ vi.mock('../selectors', () => ({ getTerminalHandle: vi.fn() }))
|
||||
|
||||
import { ORCHESTRATION_HANDLERS } from './orchestration'
|
||||
import { printResult } from '../format'
|
||||
import { ORCHESTRATION_WORKER_LAUNCH_PREFERENCES_RUNTIME_CAPABILITY } from '../../shared/protocol-version'
|
||||
import {
|
||||
ORCHESTRATION_RUN_CAPACITY_RUNTIME_CAPABILITY,
|
||||
ORCHESTRATION_WORKER_LAUNCH_PREFERENCES_RUNTIME_CAPABILITY
|
||||
} from '../../shared/protocol-version'
|
||||
|
||||
describe('orchestration worker-start CLI contract', () => {
|
||||
beforeEach(() => {
|
||||
@@ -125,6 +128,56 @@ describe('orchestration worker-start CLI contract', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('fails closed when the runtime cannot enforce a requested capacity slot', async () => {
|
||||
callMock.mockResolvedValueOnce({ result: { capabilities: [] } })
|
||||
|
||||
await expect(
|
||||
invokeWorkerStart(
|
||||
new Map<string, string | boolean>([
|
||||
['task', 'task_1'],
|
||||
['agent', 'codex'],
|
||||
['capacity-slot', true],
|
||||
['from', 'term_coord']
|
||||
])
|
||||
)
|
||||
).rejects.toMatchObject({ code: 'incompatible_runtime' })
|
||||
expect(callMock).toHaveBeenCalledTimes(1)
|
||||
expect(callMock).toHaveBeenCalledWith('status.get')
|
||||
})
|
||||
|
||||
it('forwards a capacity slot only after the runtime advertises enforcement', async () => {
|
||||
callMock
|
||||
.mockResolvedValueOnce({
|
||||
result: { capabilities: [ORCHESTRATION_RUN_CAPACITY_RUNTIME_CAPABILITY] }
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
result: {
|
||||
runId: 'run_1',
|
||||
taskId: 'task_1',
|
||||
dispatchId: 'ctx_1',
|
||||
state: 'ready',
|
||||
effects: [],
|
||||
residualResources: []
|
||||
}
|
||||
})
|
||||
|
||||
await invokeWorkerStart(
|
||||
new Map<string, string | boolean>([
|
||||
['task', 'task_1'],
|
||||
['agent', 'codex'],
|
||||
['capacity-slot', true],
|
||||
['from', 'term_coord']
|
||||
])
|
||||
)
|
||||
|
||||
expect(callMock).toHaveBeenNthCalledWith(1, 'status.get')
|
||||
expect(callMock).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
'orchestration.workerStart',
|
||||
expect.objectContaining({ capacitySlot: true })
|
||||
)
|
||||
})
|
||||
|
||||
it('fails before worker-start when the runtime would strip launch preferences', async () => {
|
||||
callMock.mockResolvedValueOnce({ result: { capabilities: [] } })
|
||||
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { CommandHandler } from '../dispatch'
|
||||
import { ORCHESTRATION_CHECK_HANDLER } from './orchestration/message-check-handler'
|
||||
import { ORCHESTRATION_CAPACITY_HANDLERS } from './orchestration/capacity-handlers'
|
||||
import {
|
||||
ORCHESTRATION_DISPATCH_HANDLER,
|
||||
ORCHESTRATION_DISPATCH_INSPECTION_HANDLERS
|
||||
@@ -17,6 +18,7 @@ import { ORCHESTRATION_WORKER_TERMINAL_HANDLERS } from './orchestration/worker-t
|
||||
|
||||
export const ORCHESTRATION_HANDLERS: Record<string, CommandHandler> = {
|
||||
...ORCHESTRATION_RUN_HANDLERS,
|
||||
...ORCHESTRATION_CAPACITY_HANDLERS,
|
||||
...ORCHESTRATION_SEND_HANDLER,
|
||||
...ORCHESTRATION_CHECK_HANDLER,
|
||||
...ORCHESTRATION_INBOX_HANDLERS,
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
import type { CommandHandler } from '../../dispatch'
|
||||
import { printResult } from '../../format'
|
||||
import { getOptionalStringFlag, getRequiredFiniteNumber, getRequiredStringFlag } from '../../flags'
|
||||
import { callOrchestrationMutation } from './mutation-request'
|
||||
import { resolveCoordinatorTerminalHandle } from './terminal-identity'
|
||||
|
||||
type CapacitySnapshot = {
|
||||
runId: string
|
||||
targetConcurrency: number
|
||||
activeCount: number
|
||||
availableSlots: number
|
||||
launchableCount: number
|
||||
launchableTasks: { id: string; display_name?: string | null; task_title?: string | null }[]
|
||||
eligiblePendingTaskIds: string[]
|
||||
settledTerminalDebt: {
|
||||
dispatchId: string
|
||||
terminalState: string
|
||||
retainedReason: string | null
|
||||
}[]
|
||||
}
|
||||
|
||||
function formatCapacity(snapshot: CapacitySnapshot): string {
|
||||
const head =
|
||||
`Run ${snapshot.runId}: target ${snapshot.targetConcurrency}, active ${snapshot.activeCount}, ` +
|
||||
`open ${snapshot.availableSlots}, launchable ${snapshot.launchableCount}`
|
||||
const launchable = snapshot.launchableTasks.map((task) => {
|
||||
const label = task.display_name ?? task.task_title
|
||||
return label ? `${task.id} ${label}` : task.id
|
||||
})
|
||||
const debt = snapshot.settledTerminalDebt.map(
|
||||
(worker) =>
|
||||
`${worker.dispatchId} [${worker.terminalState}]${worker.retainedReason ? ` ${worker.retainedReason}` : ''}`
|
||||
)
|
||||
return [
|
||||
head,
|
||||
launchable.length > 0
|
||||
? `Eligible ready tasks:\n${launchable.join('\n')}`
|
||||
: 'No eligible ready tasks.',
|
||||
snapshot.eligiblePendingTaskIds.length > 0
|
||||
? `Eligible but blocked/pending: ${snapshot.eligiblePendingTaskIds.join(', ')}`
|
||||
: '',
|
||||
debt.length > 0 ? `Settled terminal cleanup:\n${debt.join('\n')}` : ''
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join('\n')
|
||||
}
|
||||
|
||||
export const ORCHESTRATION_CAPACITY_HANDLERS: Record<string, CommandHandler> = {
|
||||
'orchestration capacity-set': async ({ flags, client, cwd, json }) => {
|
||||
const result = await callOrchestrationMutation<{ capacity: CapacitySnapshot }>(
|
||||
client,
|
||||
flags,
|
||||
'orchestration.capacityConfigure',
|
||||
{
|
||||
target: getRequiredFiniteNumber(flags, 'target'),
|
||||
run: getOptionalStringFlag(flags, 'run'),
|
||||
from: await resolveCoordinatorTerminalHandle(flags, cwd, client)
|
||||
}
|
||||
)
|
||||
printResult(result, json, (value) => formatCapacity(value.capacity))
|
||||
},
|
||||
'orchestration capacity-show': async ({ flags, client, cwd, json }) => {
|
||||
const run = getOptionalStringFlag(flags, 'run')
|
||||
const result = await client.call<{ capacity: CapacitySnapshot }>('orchestration.capacityShow', {
|
||||
run,
|
||||
from: run ? undefined : await resolveCoordinatorTerminalHandle(flags, cwd, client)
|
||||
})
|
||||
printResult(result, json, (value) => formatCapacity(value.capacity))
|
||||
},
|
||||
'orchestration capacity-enroll': async ({ flags, client, cwd, json }) => {
|
||||
const task = getRequiredStringFlag(flags, 'task')
|
||||
const result = await callOrchestrationMutation<{ task: { id: string } }>(
|
||||
client,
|
||||
flags,
|
||||
'orchestration.capacityTaskSet',
|
||||
{
|
||||
task,
|
||||
eligible: true,
|
||||
run: getOptionalStringFlag(flags, 'run'),
|
||||
from: await resolveCoordinatorTerminalHandle(flags, cwd, client)
|
||||
}
|
||||
)
|
||||
printResult(result, json, () => `Task ${task} enrolled in the Run capacity pool.`)
|
||||
},
|
||||
'orchestration capacity-withdraw': async ({ flags, client, cwd, json }) => {
|
||||
const task = getRequiredStringFlag(flags, 'task')
|
||||
const result = await callOrchestrationMutation<{ task: { id: string } }>(
|
||||
client,
|
||||
flags,
|
||||
'orchestration.capacityTaskSet',
|
||||
{
|
||||
task,
|
||||
eligible: false,
|
||||
run: getOptionalStringFlag(flags, 'run'),
|
||||
from: await resolveCoordinatorTerminalHandle(flags, cwd, client)
|
||||
}
|
||||
)
|
||||
printResult(result, json, () => `Task ${task} withdrawn from the Run capacity pool.`)
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,10 @@ import { printResult } from '../../format'
|
||||
import { getOptionalStringFlag, getRequiredStringFlag } from '../../flags'
|
||||
import { RuntimeClientError } from '../../runtime-client'
|
||||
import type { RuntimeStatus } from '../../../shared/runtime-types'
|
||||
import { ORCHESTRATION_WORKER_LAUNCH_PREFERENCES_RUNTIME_CAPABILITY } from '../../../shared/protocol-version'
|
||||
import {
|
||||
ORCHESTRATION_RUN_CAPACITY_RUNTIME_CAPABILITY,
|
||||
ORCHESTRATION_WORKER_LAUNCH_PREFERENCES_RUNTIME_CAPABILITY
|
||||
} from '../../../shared/protocol-version'
|
||||
import { callOrchestrationMutation } from './mutation-request'
|
||||
import { getOptionalPositiveIntegerValueFlag } from './numeric-flags'
|
||||
import { isDevCliInvocation } from './runtime-compatibility'
|
||||
@@ -13,9 +16,11 @@ export const ORCHESTRATION_WORKER_LAUNCH_HANDLER: Record<string, CommandHandler>
|
||||
'orchestration worker-start': async ({ flags, client, cwd, json }) => {
|
||||
const model = getOptionalStringFlag(flags, 'model')
|
||||
const effort = getOptionalStringFlag(flags, 'effort')
|
||||
if (model || effort) {
|
||||
const capacitySlot = flags.has('capacity-slot')
|
||||
if (model || effort || capacitySlot) {
|
||||
const status = await client.call<RuntimeStatus>('status.get')
|
||||
if (
|
||||
(model || effort) &&
|
||||
!status.result.capabilities?.includes(
|
||||
ORCHESTRATION_WORKER_LAUNCH_PREFERENCES_RUNTIME_CAPABILITY
|
||||
)
|
||||
@@ -25,6 +30,15 @@ export const ORCHESTRATION_WORKER_LAUNCH_HANDLER: Record<string, CommandHandler>
|
||||
'The connected Orca runtime does not support worker model or effort overrides. Update or restart Orca and try again.'
|
||||
)
|
||||
}
|
||||
if (
|
||||
capacitySlot &&
|
||||
!status.result.capabilities?.includes(ORCHESTRATION_RUN_CAPACITY_RUNTIME_CAPABILITY)
|
||||
) {
|
||||
throw new RuntimeClientError(
|
||||
'incompatible_runtime',
|
||||
'The connected Orca runtime cannot enforce Run capacity slots. Update or restart Orca and try again.'
|
||||
)
|
||||
}
|
||||
}
|
||||
const result = await callOrchestrationMutation<{
|
||||
runId: string
|
||||
@@ -51,6 +65,7 @@ export const ORCHESTRATION_WORKER_LAUNCH_HANDLER: Record<string, CommandHandler>
|
||||
effort,
|
||||
terminal: getOptionalStringFlag(flags, 'terminal'),
|
||||
retryOf: getOptionalStringFlag(flags, 'retry-of'),
|
||||
...(capacitySlot ? { capacitySlot: true } : {}),
|
||||
timeoutMs: getOptionalPositiveIntegerValueFlag(flags, 'timeout-ms'),
|
||||
run: getOptionalStringFlag(flags, 'run'),
|
||||
from: await resolveCoordinatorTerminalHandle(flags, cwd, client),
|
||||
|
||||
@@ -5,10 +5,11 @@ export const ORCHESTRATION_WORKER_COMMAND_SPECS: CommandSpec[] = [
|
||||
path: ['orchestration', 'worker-start'],
|
||||
summary: 'Start one supervised worker on the Run home or a connected Orca server',
|
||||
usage:
|
||||
'orca orchestration worker-start --task <task_id> [--on <saved-environment>] [--worktree <current|selector|new-child|new-top-level>] (--agent <agent> | --terminal <handle>) [--model <id>] [--effort <level>] [--name <name>] [--repo <selector>] [--base-branch <ref>] [--display-name <text>] [--comment <text>] [--setup <run|skip|inherit>] [--retry-of <dispatch_id>] [--timeout-ms <n>] [--run <run_id>] [--from <handle>] [--retry-request <id>] [--json]',
|
||||
'orca orchestration worker-start --task <task_id> [--capacity-slot] [--on <saved-environment>] [--worktree <current|selector|new-child|new-top-level>] (--agent <agent> | --terminal <handle>) [--model <id>] [--effort <level>] [--name <name>] [--repo <selector>] [--base-branch <ref>] [--display-name <text>] [--comment <text>] [--setup <run|skip|inherit>] [--retry-of <dispatch_id>] [--timeout-ms <n>] [--run <run_id>] [--from <handle>] [--retry-request <id>] [--json]',
|
||||
allowedFlags: [
|
||||
...GLOBAL_FLAGS,
|
||||
'task',
|
||||
'capacity-slot',
|
||||
'on',
|
||||
'worktree',
|
||||
'name',
|
||||
@@ -35,7 +36,8 @@ export const ORCHESTRATION_WORKER_COMMAND_SPECS: CommandSpec[] = [
|
||||
'--on selects only the worker server; the Run and this command remain on the current Orca server.',
|
||||
'Remote current and new-child are invalid; discover an exact remote selector or use new-top-level.',
|
||||
'--retry-of links the replacement attempt but does not inherit placement; repeat the intended --on/worktree and --agent/terminal choices.',
|
||||
'The call exits 0 only for ready. Failed or outcome_unknown exits 1 and JSON includes stage/failedStage, setup, effects, residualResources, and recovery commands when needed.'
|
||||
'The call exits 0 only for ready. Failed or outcome_unknown exits 1 and JSON includes stage/failedStage, setup, effects, residualResources, and recovery commands when needed.',
|
||||
'--capacity-slot atomically enforces the Run target and requires this ready Task to be explicitly enrolled.'
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -42,6 +42,34 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [
|
||||
usage: 'orca orchestration run-show --id <run_id> [--json]',
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'id']
|
||||
},
|
||||
{
|
||||
path: ['orchestration', 'capacity-set'],
|
||||
summary: 'Declare the target concurrent worker count for the bound Run',
|
||||
usage:
|
||||
'orca orchestration capacity-set --target <0-64> [--run <run_id>] [--from <handle>] [--retry-request <id>] [--json]',
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'target', 'run', 'from', 'retry-request'],
|
||||
notes: ['A target of 0 disables capacity-slot launches without changing existing workers.']
|
||||
},
|
||||
{
|
||||
path: ['orchestration', 'capacity-show'],
|
||||
summary: 'Show target, active lanes, eligible ready tasks, and settled terminal debt',
|
||||
usage: 'orca orchestration capacity-show [--run <run_id>] [--from <handle>] [--json]',
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'run', 'from']
|
||||
},
|
||||
{
|
||||
path: ['orchestration', 'capacity-enroll'],
|
||||
summary: 'Explicitly enroll one Task in the Run capacity pool',
|
||||
usage:
|
||||
'orca orchestration capacity-enroll --task <task_id> [--run <run_id>] [--from <handle>] [--retry-request <id>] [--json]',
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'task', 'run', 'from', 'retry-request']
|
||||
},
|
||||
{
|
||||
path: ['orchestration', 'capacity-withdraw'],
|
||||
summary: 'Withdraw one Task from the Run capacity pool',
|
||||
usage:
|
||||
'orca orchestration capacity-withdraw --task <task_id> [--run <run_id>] [--from <handle>] [--retry-request <id>] [--json]',
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'task', 'run', 'from', 'retry-request']
|
||||
},
|
||||
{
|
||||
path: ['orchestration', 'send'],
|
||||
summary: 'Send an inter-agent message',
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
import { existsSync, mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { OrcaRuntimeService } from './orca-runtime'
|
||||
import { OrchestrationDb } from './orchestration/db'
|
||||
import { OrcaRuntimeRpcServer } from './runtime-rpc'
|
||||
import { spawnProcess } from '../../shared/child-process/run-process'
|
||||
|
||||
const CLI_PATH = join(process.cwd(), 'out', 'cli', 'index.js')
|
||||
const describeIfBuilt = existsSync(CLI_PATH) ? describe : describe.skip
|
||||
|
||||
async function runBuiltCli(
|
||||
userDataPath: string,
|
||||
args: string[]
|
||||
): Promise<{ exitCode: number; stdout: string; stderr: string }> {
|
||||
const child = spawnProcess({
|
||||
program: process.execPath,
|
||||
args: [CLI_PATH, ...args],
|
||||
env: {
|
||||
...process.env,
|
||||
ORCA_USER_DATA_PATH: userDataPath,
|
||||
ORCA_TERMINAL_HANDLE: 'term_cli'
|
||||
},
|
||||
stdio: ['ignore', 'pipe', 'pipe']
|
||||
})
|
||||
if (!child.stdout || !child.stderr) {
|
||||
throw new Error('Compiled CLI test requires piped stdout and stderr.')
|
||||
}
|
||||
const stdout: Buffer[] = []
|
||||
const stderr: Buffer[] = []
|
||||
child.stdout.on('data', (chunk) => stdout.push(chunk))
|
||||
child.stderr.on('data', (chunk) => stderr.push(chunk))
|
||||
const exitCode = await new Promise<number>((resolve, reject) => {
|
||||
child.once('exit', (code) => resolve(code ?? 1))
|
||||
child.once('error', reject)
|
||||
})
|
||||
return {
|
||||
exitCode,
|
||||
stdout: Buffer.concat(stdout).toString('utf8'),
|
||||
stderr: Buffer.concat(stderr).toString('utf8')
|
||||
}
|
||||
}
|
||||
|
||||
function parseResult<T>(result: { exitCode: number; stdout: string; stderr: string }): T {
|
||||
expect(result.exitCode, `${result.stderr}\n${result.stdout}`).toBe(0)
|
||||
return JSON.parse(result.stdout) as T
|
||||
}
|
||||
|
||||
describeIfBuilt('Run capacity compiled CLI coordinator loop', () => {
|
||||
it('reopens one slot immediately after settlement and exposes the next enrolled Task', async () => {
|
||||
const userDataPath = mkdtempSync(join(tmpdir(), 'orca-capacity-cli-'))
|
||||
const runtime = new OrcaRuntimeService()
|
||||
const db = new OrchestrationDb(':memory:')
|
||||
runtime.setOrchestrationDb(db)
|
||||
vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue('tab_cli:leaf_cli')
|
||||
vi.spyOn(runtime, 'getLiveTerminalPaneKey').mockReturnValue('tab_cli:leaf_cli')
|
||||
const server = new OrcaRuntimeRpcServer({ runtime, userDataPath })
|
||||
await server.start()
|
||||
|
||||
try {
|
||||
parseResult(
|
||||
await runBuiltCli(userDataPath, [
|
||||
'orchestration',
|
||||
'run-create',
|
||||
'--objective',
|
||||
'compiled capacity loop',
|
||||
'--from',
|
||||
'term_cli',
|
||||
'--json'
|
||||
])
|
||||
)
|
||||
parseResult(
|
||||
await runBuiltCli(userDataPath, [
|
||||
'orchestration',
|
||||
'capacity-set',
|
||||
'--target',
|
||||
'1',
|
||||
'--from',
|
||||
'term_cli',
|
||||
'--json'
|
||||
])
|
||||
)
|
||||
|
||||
const tasks: string[] = []
|
||||
for (const spec of ['first lane', 'backfill lane']) {
|
||||
const created = parseResult<{ result: { task: { id: string } } }>(
|
||||
await runBuiltCli(userDataPath, [
|
||||
'orchestration',
|
||||
'task-create',
|
||||
'--spec',
|
||||
spec,
|
||||
'--from',
|
||||
'term_cli',
|
||||
'--json'
|
||||
])
|
||||
)
|
||||
tasks.push(created.result.task.id)
|
||||
parseResult(
|
||||
await runBuiltCli(userDataPath, [
|
||||
'orchestration',
|
||||
'capacity-enroll',
|
||||
'--task',
|
||||
created.result.task.id,
|
||||
'--from',
|
||||
'term_cli',
|
||||
'--json'
|
||||
])
|
||||
)
|
||||
}
|
||||
|
||||
const before = parseResult<{
|
||||
result: { capacity: { availableSlots: number; launchableTasks: { id: string }[] } }
|
||||
}>(
|
||||
await runBuiltCli(userDataPath, [
|
||||
'orchestration',
|
||||
'capacity-show',
|
||||
'--from',
|
||||
'term_cli',
|
||||
'--json'
|
||||
])
|
||||
)
|
||||
expect(before.result.capacity).toMatchObject({
|
||||
availableSlots: 1,
|
||||
launchableTasks: [{ id: tasks[0] }]
|
||||
})
|
||||
|
||||
const started = db.createStartingWorkerDispatch({
|
||||
taskId: tasks[0]!,
|
||||
startOptions: {},
|
||||
creator: { kind: 'system' },
|
||||
maxDepth: 1,
|
||||
capacitySlot: true
|
||||
})
|
||||
db.prepareStartingWorkerAuthority({
|
||||
dispatchId: started.dispatch.id,
|
||||
handle: 'term_worker',
|
||||
paneKey: 'tab_worker:leaf_worker',
|
||||
processIncarnation: 'pid:worker',
|
||||
worktreeId: 'repo::worktree',
|
||||
effects: [],
|
||||
setupState: 'not_applicable',
|
||||
terminalOwnership: 'created'
|
||||
})
|
||||
db.markWorkerDispatchReady(started.dispatch.id)
|
||||
|
||||
const full = parseResult<{
|
||||
result: { capacity: { activeCount: number; availableSlots: number } }
|
||||
}>(
|
||||
await runBuiltCli(userDataPath, [
|
||||
'orchestration',
|
||||
'capacity-show',
|
||||
'--from',
|
||||
'term_cli',
|
||||
'--json'
|
||||
])
|
||||
)
|
||||
expect(full.result.capacity).toMatchObject({ activeCount: 1, availableSlots: 0 })
|
||||
|
||||
db.settleWorkerReport({
|
||||
taskId: tasks[0]!,
|
||||
dispatchId: started.dispatch.id,
|
||||
outcome: 'succeeded',
|
||||
result: '{}'
|
||||
})
|
||||
const reopened = parseResult<{
|
||||
result: {
|
||||
capacity: {
|
||||
activeCount: number
|
||||
availableSlots: number
|
||||
launchableTasks: { id: string }[]
|
||||
settledTerminalDebt: { dispatchId: string; terminalState: string }[]
|
||||
}
|
||||
}
|
||||
}>(
|
||||
await runBuiltCli(userDataPath, [
|
||||
'orchestration',
|
||||
'capacity-show',
|
||||
'--from',
|
||||
'term_cli',
|
||||
'--json'
|
||||
])
|
||||
)
|
||||
expect(reopened.result.capacity).toMatchObject({
|
||||
activeCount: 0,
|
||||
availableSlots: 1,
|
||||
launchableTasks: [{ id: tasks[1] }],
|
||||
settledTerminalDebt: [{ dispatchId: started.dispatch.id, terminalState: 'reclaimable' }]
|
||||
})
|
||||
} finally {
|
||||
db.close()
|
||||
await server.stop()
|
||||
rmSync(userDataPath, { recursive: true, force: true })
|
||||
}
|
||||
}, 30_000)
|
||||
})
|
||||
@@ -37,6 +37,7 @@ import { attachRunCoordinatorMailRouting } from './runs/run-coordinator-mail-rou
|
||||
import { attachRunCreate } from './runs/run-create'
|
||||
import { attachRunDelivery } from './runs/run-delivery'
|
||||
import { attachRunLookup } from './runs/run-lookup'
|
||||
import { attachRunCapacityStore } from './run-capacity/run-capacity-store'
|
||||
import { attachLegacyCoordinatorMailTakeover } from './runs/legacy-coordinator-mail-takeover'
|
||||
import { attachAdoptLegacyRun } from './schema/adopt-legacy-run'
|
||||
import { attachBackfillLegacyQuestionThreads } from './schema/backfill-legacy-question-threads'
|
||||
@@ -81,6 +82,7 @@ export function attachOrchestrationDbMethods(ctor: { prototype: object }): void
|
||||
attachRunCreate(ctor)
|
||||
attachRunBinding(ctor)
|
||||
attachRunLookup(ctor)
|
||||
attachRunCapacityStore(ctor)
|
||||
attachRunCoordinatorMailRouting(ctor)
|
||||
attachLegacyCoordinatorMailTakeover(ctor)
|
||||
attachRunDelivery(ctor)
|
||||
|
||||
@@ -6,5 +6,5 @@ export const LEGACY_RUN_ID = ORCHESTRATION_LEGACY_RUN_ID
|
||||
export const LEGACY_CONTRACT_VERSION = 0
|
||||
export const CURRENT_CONTRACT_VERSION = ORCHESTRATION_CONTRACT_VERSION
|
||||
|
||||
// Schema versions: v2 'heartbeat'+last_heartbeat_at, v3 delivered_at, v4 task-creator terminal, v5 task_title/display_name, v6 pane identity, v7 lightweight Runs, v8 crash-safe Run deliveries, v9 durable question threads, v10 Dispatch capabilities, v11 durable mutation receipts, v12 composed worker state, v18 post-v6 version-skew repair, v19 adopted legacy Runs and compatibility receipts, v20 legacy question backfill, v21 legacy scheduler-loss provenance, v22 dispatch assignee lookup, v23 worker terminal resource ownership, v24 creator-incarnation authority, v25 active Dispatch handle lookup, v26 indexed mutation receipt capacity, v27 durable federation acknowledgments, v28 durable local mutation caller identity.
|
||||
export const SCHEMA_VERSION = 30
|
||||
// Schema versions: v2 'heartbeat'+last_heartbeat_at, v3 delivered_at, v4 task-creator terminal, v5 task_title/display_name, v6 pane identity, v7 lightweight Runs, v8 crash-safe Run deliveries, v9 durable question threads, v10 Dispatch capabilities, v11 durable mutation receipts, v12 composed worker state, v18 post-v6 version-skew repair, v19 adopted legacy Runs and compatibility receipts, v20 legacy question backfill, v21 legacy scheduler-loss provenance, v22 dispatch assignee lookup, v23 worker terminal resource ownership, v24 creator-incarnation authority, v25 active Dispatch handle lookup, v26 indexed mutation receipt capacity, v27 durable federation acknowledgments, v28 durable local mutation caller identity, v29 termination provenance, v30 dispatch depth, v31 Run capacity targets.
|
||||
export const SCHEMA_VERSION = 31
|
||||
|
||||
@@ -72,6 +72,7 @@ export function createDispatchContext(
|
||||
`Terminal ${assigneeHandle} already has an active dispatch (${occupied.id} for task ${occupied.task_id})`
|
||||
)
|
||||
}
|
||||
this.assertRunCapacityAvailable(taskId)
|
||||
throw new Error(
|
||||
`Task ${taskId} is ${current?.status ?? 'missing'}; only ready tasks can be dispatched`
|
||||
)
|
||||
|
||||
@@ -20,6 +20,17 @@ export const DISPATCH_CONTEXT_CLAIM_SQL = `INSERT INTO dispatch_contexts (
|
||||
SELECT ?, run_id, id, ?, ?, ?, ?, ?, 'dispatched', ?, ?, datetime('now')
|
||||
FROM tasks
|
||||
WHERE id = ? AND status = 'ready'
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM runs
|
||||
WHERE runs.id = tasks.run_id
|
||||
AND runs.legacy = 0
|
||||
AND runs.target_concurrency > 0
|
||||
AND (
|
||||
SELECT COUNT(*) FROM dispatch_contexts active
|
||||
WHERE active.run_id = tasks.run_id
|
||||
AND active.status IN ('pending', 'dispatched')
|
||||
) >= runs.target_concurrency
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM dispatch_contexts active
|
||||
WHERE active.assignee_handle = ?
|
||||
@@ -44,7 +55,19 @@ WHERE id = ? AND status = 'ready'
|
||||
|
||||
const STARTING_DISPATCH_CONTEXT_SQL = `INSERT INTO dispatch_contexts (
|
||||
id, run_id, task_id, contract_version, launch_token_hash, depth, status, dispatched_at
|
||||
) VALUES (?, ?, ?, ?, ?, ?, 'pending', datetime('now'))`
|
||||
)
|
||||
SELECT ?, ?, ?, ?, ?, ?, 'pending', datetime('now')
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM runs
|
||||
WHERE runs.id = ?
|
||||
AND runs.legacy = 0
|
||||
AND runs.target_concurrency > 0
|
||||
AND (
|
||||
SELECT COUNT(*) FROM dispatch_contexts active
|
||||
WHERE active.run_id = ?
|
||||
AND active.status IN ('pending', 'dispatched')
|
||||
) >= runs.target_concurrency
|
||||
)`
|
||||
|
||||
const REMOTE_DISPATCH_ATTACHMENT_SQL = `INSERT INTO remote_dispatch_attachments (
|
||||
dispatch_id, task_id, home_peer_fingerprint, protocol_version, runtime_epoch, depth
|
||||
@@ -109,14 +132,21 @@ export function insertStartingDispatchContextRow(
|
||||
}
|
||||
): void {
|
||||
assertStampedDepth(params.depth)
|
||||
db.prepare(STARTING_DISPATCH_CONTEXT_SQL).run(
|
||||
params.id,
|
||||
params.runId,
|
||||
params.taskId,
|
||||
params.contractVersion,
|
||||
params.launchTokenHash,
|
||||
params.depth
|
||||
)
|
||||
const inserted = db
|
||||
.prepare(STARTING_DISPATCH_CONTEXT_SQL)
|
||||
.run(
|
||||
params.id,
|
||||
params.runId,
|
||||
params.taskId,
|
||||
params.contractVersion,
|
||||
params.launchTokenHash,
|
||||
params.depth,
|
||||
params.runId,
|
||||
params.runId
|
||||
)
|
||||
if (inserted.changes !== 1) {
|
||||
throw new Error(`Run capacity changed while starting Task ${params.taskId}.`)
|
||||
}
|
||||
}
|
||||
|
||||
/** The worker host's record of a live worker driven by a remote Run home. */
|
||||
|
||||
@@ -37,6 +37,7 @@ import type { RunCoordinatorMailRoutingMethods } from './runs/run-coordinator-ma
|
||||
import type { RunCreateMethods } from './runs/run-create'
|
||||
import type { RunDeliveryMethods } from './runs/run-delivery'
|
||||
import type { RunLookupMethods } from './runs/run-lookup'
|
||||
import type { RunCapacityStoreMethods } from './run-capacity/run-capacity-store'
|
||||
import type { LegacyCoordinatorMailTakeoverMethods } from './runs/legacy-coordinator-mail-takeover'
|
||||
import type { AdoptLegacyRunMethods } from './schema/adopt-legacy-run'
|
||||
import type { BackfillLegacyQuestionThreadsMethods } from './schema/backfill-legacy-question-threads'
|
||||
@@ -80,6 +81,7 @@ export type OrchestrationDbMethods = CreateTablesMethods &
|
||||
RunCreateMethods &
|
||||
RunBindingMethods &
|
||||
RunLookupMethods &
|
||||
RunCapacityStoreMethods &
|
||||
RunCoordinatorMailRoutingMethods &
|
||||
LegacyCoordinatorMailTakeoverMethods &
|
||||
RunDeliveryMethods &
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
import { OrchestrationError } from '../../orchestration-error'
|
||||
import type { TaskRow } from '../../types'
|
||||
import {
|
||||
deriveWorkerTerminalListState,
|
||||
type WorkerDispatchListState,
|
||||
type WorkerTerminalListState,
|
||||
type WorkerTerminalResourceRow
|
||||
} from '../../worker-terminal-ownership'
|
||||
import type { OrchestrationDb } from '../orchestration-db'
|
||||
|
||||
const MAX_TARGET_CONCURRENCY = 64
|
||||
|
||||
export type RunCapacitySnapshot = {
|
||||
runId: string
|
||||
targetConcurrency: number
|
||||
activeCount: number
|
||||
availableSlots: number
|
||||
launchableCount: number
|
||||
launchableTasks: TaskRow[]
|
||||
eligiblePendingTaskIds: string[]
|
||||
settledTerminalDebt: {
|
||||
dispatchId: string
|
||||
terminalState: WorkerTerminalListState
|
||||
retainedReason: string | null
|
||||
}[]
|
||||
}
|
||||
|
||||
export function configureRunCapacity(
|
||||
this: OrchestrationDb,
|
||||
runId: string,
|
||||
targetConcurrency: number
|
||||
): RunCapacitySnapshot {
|
||||
if (
|
||||
!Number.isInteger(targetConcurrency) ||
|
||||
targetConcurrency < 0 ||
|
||||
targetConcurrency > MAX_TARGET_CONCURRENCY
|
||||
) {
|
||||
throw new OrchestrationError(
|
||||
'invalid_argument',
|
||||
`Target concurrency must be an integer between 0 and ${MAX_TARGET_CONCURRENCY}.`
|
||||
)
|
||||
}
|
||||
const updated = this.db
|
||||
.prepare(
|
||||
"UPDATE runs SET target_concurrency = ?, updated_at = datetime('now') WHERE id = ? AND legacy = 0"
|
||||
)
|
||||
.run(targetConcurrency, runId)
|
||||
if (updated.changes !== 1) {
|
||||
throw new OrchestrationError('run_not_found', `Run ${runId} was not found or is inspect-only.`)
|
||||
}
|
||||
return this.getRunCapacity(runId)
|
||||
}
|
||||
|
||||
export function setTaskCapacityEligibility(
|
||||
this: OrchestrationDb,
|
||||
taskId: string,
|
||||
eligible: boolean
|
||||
): TaskRow {
|
||||
const updated = this.db
|
||||
.prepare('UPDATE tasks SET capacity_eligible = ? WHERE id = ?')
|
||||
.run(eligible ? 1 : 0, taskId)
|
||||
if (updated.changes !== 1) {
|
||||
throw new OrchestrationError('task_not_found', `Task ${taskId} was not found.`)
|
||||
}
|
||||
return this.getTask(taskId) as TaskRow
|
||||
}
|
||||
|
||||
export function assertRunCapacityAvailable(
|
||||
this: OrchestrationDb,
|
||||
taskId: string,
|
||||
requireEnrollment = false
|
||||
): void {
|
||||
const task = this.getTask(taskId)
|
||||
if (!task) {
|
||||
throw new OrchestrationError('task_not_found', `Task ${taskId} was not found.`)
|
||||
}
|
||||
if (requireEnrollment && task.capacity_eligible !== 1) {
|
||||
throw new OrchestrationError(
|
||||
'task_not_startable',
|
||||
`Task ${task.id} is not enrolled in its Run capacity pool.`
|
||||
)
|
||||
}
|
||||
const run = this.getRunRaw(task.run_id)
|
||||
if (!run || run.legacy === 1) {
|
||||
if (!requireEnrollment) {
|
||||
return
|
||||
}
|
||||
throw new OrchestrationError(
|
||||
'capacity_not_configured',
|
||||
`Run ${task.run_id} cannot claim a capacity slot.`
|
||||
)
|
||||
}
|
||||
if (run.target_concurrency < 1) {
|
||||
if (!requireEnrollment) {
|
||||
return
|
||||
}
|
||||
throw new OrchestrationError(
|
||||
'capacity_not_configured',
|
||||
`Run ${run.id} has no target concurrency; configure it before claiming a capacity slot.`
|
||||
)
|
||||
}
|
||||
const row = this.db
|
||||
.prepare(
|
||||
`SELECT COUNT(*) AS active_count
|
||||
FROM dispatch_contexts
|
||||
WHERE run_id = ? AND status IN ('pending', 'dispatched')`
|
||||
)
|
||||
.get(run.id) as { active_count: number }
|
||||
if (row.active_count >= run.target_concurrency) {
|
||||
throw new OrchestrationError(
|
||||
'capacity_full',
|
||||
`Run ${run.id} already has ${row.active_count} active lanes at target concurrency ${run.target_concurrency}.`,
|
||||
{ runId: run.id, activeCount: row.active_count, targetConcurrency: run.target_concurrency }
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export function assertRunCapacitySlot(this: OrchestrationDb, taskId: string): void {
|
||||
this.assertRunCapacityAvailable(taskId, true)
|
||||
}
|
||||
|
||||
export function getRunCapacity(this: OrchestrationDb, runId: string): RunCapacitySnapshot {
|
||||
const run = this.getRunRaw(runId)
|
||||
if (!run || run.legacy === 1) {
|
||||
throw new OrchestrationError('run_not_found', `Run ${runId} was not found or is inspect-only.`)
|
||||
}
|
||||
const active = this.db
|
||||
.prepare(
|
||||
`SELECT COUNT(*) AS active_count
|
||||
FROM dispatch_contexts
|
||||
WHERE run_id = ? AND status IN ('pending', 'dispatched')`
|
||||
)
|
||||
.get(runId) as { active_count: number }
|
||||
const availableSlots = Math.max(0, run.target_concurrency - active.active_count)
|
||||
const launchableTasks = this.db
|
||||
.prepare(
|
||||
`SELECT tasks.* FROM tasks
|
||||
WHERE run_id = ? AND capacity_eligible = 1 AND status = 'ready'
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM dispatch_contexts
|
||||
WHERE dispatch_contexts.task_id = tasks.id
|
||||
AND dispatch_contexts.status IN ('pending', 'dispatched')
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM decision_gates
|
||||
WHERE decision_gates.task_id = tasks.id AND decision_gates.status = 'pending'
|
||||
)
|
||||
ORDER BY created_at, rowid
|
||||
LIMIT ?`
|
||||
)
|
||||
.all(runId, availableSlots) as TaskRow[]
|
||||
const eligiblePendingTaskIds = (
|
||||
this.db
|
||||
.prepare(
|
||||
`SELECT id FROM tasks
|
||||
WHERE run_id = ? AND capacity_eligible = 1 AND status IN ('pending', 'blocked')
|
||||
ORDER BY created_at, rowid`
|
||||
)
|
||||
.all(runId) as { id: string }[]
|
||||
).map((task) => task.id)
|
||||
const debtRows = this.db
|
||||
.prepare(
|
||||
`SELECT r.*, d.id AS dispatch_id,
|
||||
COALESCE(w.state, 'unsupervised') AS worker_state,
|
||||
COALESCE(w.agent_terminal_handle, d.assignee_handle) AS agent_terminal_handle
|
||||
FROM dispatch_contexts d
|
||||
JOIN worker_terminal_resources r ON r.owner_dispatch_id = d.id
|
||||
LEFT JOIN worker_dispatches w ON w.dispatch_id = d.id
|
||||
WHERE d.run_id = ?
|
||||
AND d.status IN ('completed', 'failed', 'circuit_broken')
|
||||
AND r.release_state <> 'released'
|
||||
ORDER BY COALESCE(w.created_at, d.created_at), d.rowid`
|
||||
)
|
||||
.all(runId) as (WorkerTerminalResourceRow & {
|
||||
dispatch_id: string
|
||||
worker_state: WorkerDispatchListState
|
||||
agent_terminal_handle: string | null
|
||||
})[]
|
||||
const settledTerminalDebt = debtRows.flatMap((row) => {
|
||||
const terminalState = deriveWorkerTerminalListState({
|
||||
workerState: row.worker_state,
|
||||
agentTerminalHandle: row.agent_terminal_handle,
|
||||
resource: row
|
||||
})
|
||||
return terminalState && terminalState !== 'released'
|
||||
? [
|
||||
{
|
||||
dispatchId: row.dispatch_id,
|
||||
terminalState,
|
||||
retainedReason: row.retained_reason
|
||||
}
|
||||
]
|
||||
: []
|
||||
})
|
||||
return {
|
||||
runId,
|
||||
targetConcurrency: run.target_concurrency,
|
||||
activeCount: active.active_count,
|
||||
availableSlots,
|
||||
launchableCount: launchableTasks.length,
|
||||
launchableTasks,
|
||||
eligiblePendingTaskIds,
|
||||
settledTerminalDebt
|
||||
}
|
||||
}
|
||||
|
||||
export type RunCapacityStoreMethods = {
|
||||
configureRunCapacity: typeof configureRunCapacity
|
||||
setTaskCapacityEligibility: typeof setTaskCapacityEligibility
|
||||
assertRunCapacityAvailable: typeof assertRunCapacityAvailable
|
||||
assertRunCapacitySlot: typeof assertRunCapacitySlot
|
||||
getRunCapacity: typeof getRunCapacity
|
||||
}
|
||||
|
||||
export function attachRunCapacityStore(ctor: { prototype: object }): void {
|
||||
Object.assign(ctor.prototype, {
|
||||
configureRunCapacity,
|
||||
setTaskCapacityEligibility,
|
||||
assertRunCapacityAvailable,
|
||||
assertRunCapacitySlot,
|
||||
getRunCapacity
|
||||
})
|
||||
}
|
||||
@@ -10,6 +10,7 @@ CREATE TABLE IF NOT EXISTS runs (
|
||||
coordinator_pane_key TEXT,
|
||||
consumer_generation INTEGER NOT NULL DEFAULT 0,
|
||||
legacy INTEGER NOT NULL DEFAULT 0,
|
||||
target_concurrency INTEGER NOT NULL DEFAULT 0 CHECK(target_concurrency BETWEEN 0 AND 64),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
@@ -108,6 +108,7 @@ CREATE TABLE IF NOT EXISTS tasks (
|
||||
'pending', 'ready', 'dispatched',
|
||||
'completed', 'failed', 'blocked'
|
||||
)),
|
||||
capacity_eligible INTEGER NOT NULL DEFAULT 0 CHECK(capacity_eligible IN (0, 1)),
|
||||
deps TEXT NOT NULL DEFAULT '[]',
|
||||
result TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
import type { OrchestrationDb } from '../orchestration-db'
|
||||
|
||||
export function applySchemaMigrationV31(this: OrchestrationDb, current: number): void {
|
||||
if (current >= 31) {
|
||||
return
|
||||
}
|
||||
if (!this.hasColumn('runs', 'target_concurrency')) {
|
||||
this.db.exec(
|
||||
'ALTER TABLE runs ADD COLUMN target_concurrency INTEGER NOT NULL DEFAULT 0 CHECK(target_concurrency BETWEEN 0 AND 64)'
|
||||
)
|
||||
}
|
||||
if (!this.hasColumn('tasks', 'capacity_eligible')) {
|
||||
this.db.exec(
|
||||
'ALTER TABLE tasks ADD COLUMN capacity_eligible INTEGER NOT NULL DEFAULT 0 CHECK(capacity_eligible IN (0, 1))'
|
||||
)
|
||||
}
|
||||
this.db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_tasks_capacity_ready
|
||||
ON tasks(run_id, capacity_eligible, status, created_at);
|
||||
`)
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import { SCHEMA_VERSION } from '../contract-constants'
|
||||
import type { OrchestrationDb } from '../orchestration-db'
|
||||
import { applySchemaMigrationsV13ToV30 } from './migrate-v13-v30'
|
||||
import { applySchemaMigrationsV2ToV12 } from './migrate-v2-v12'
|
||||
import { applySchemaMigrationV31 } from './migrate-v31'
|
||||
|
||||
// Why: CREATE TABLE IF NOT EXISTS won't alter existing DBs; migrate in a txn that bumps user_version only on success (atomic all-or-nothing).
|
||||
export function migrate(this: OrchestrationDb): void {
|
||||
@@ -16,6 +17,7 @@ export function migrate(this: OrchestrationDb): void {
|
||||
try {
|
||||
applySchemaMigrationsV2ToV12.call(this, current)
|
||||
applySchemaMigrationsV13ToV30.call(this, current)
|
||||
applySchemaMigrationV31.call(this, current)
|
||||
this.db.pragma(`user_version = ${SCHEMA_VERSION}`)
|
||||
this.db.exec('COMMIT')
|
||||
} catch (err) {
|
||||
|
||||
@@ -21,6 +21,7 @@ export function createTask(
|
||||
createdByProcessIncarnation?: string
|
||||
createdByRunGeneration?: number
|
||||
runId?: string
|
||||
capacityEligible?: boolean
|
||||
}
|
||||
): TaskRow {
|
||||
const runId = task.runId ?? LEGACY_RUN_ID
|
||||
@@ -49,7 +50,7 @@ export function createTask(
|
||||
`INSERT INTO tasks (
|
||||
id, run_id, parent_id, created_by_terminal_handle, created_by_pane_key,
|
||||
created_by_process_incarnation, created_by_run_generation,
|
||||
task_title, display_name, spec, status, deps
|
||||
task_title, display_name, spec, status, capacity_eligible, deps
|
||||
) VALUES (
|
||||
?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
|
||||
CASE WHEN EXISTS (
|
||||
@@ -59,7 +60,7 @@ export function createTask(
|
||||
WHERE dependency.id IS NULL
|
||||
OR dependency.run_id <> ?
|
||||
OR dependency.status <> 'completed'
|
||||
) THEN 'pending' ELSE 'ready' END,
|
||||
) THEN 'pending' ELSE 'ready' END, ?,
|
||||
?
|
||||
)`
|
||||
)
|
||||
@@ -76,6 +77,7 @@ export function createTask(
|
||||
task.spec,
|
||||
depsJson,
|
||||
runId,
|
||||
task.capacityEligible ? 1 : 0,
|
||||
depsJson
|
||||
)
|
||||
return this.db.prepare('SELECT * FROM tasks WHERE id = ?').get(id) as TaskRow
|
||||
|
||||
@@ -30,6 +30,7 @@ export function createStartingWorkerDispatch(
|
||||
/** Who is dispatching, for nesting depth. Required so a new caller must decide. */
|
||||
creator: DispatchCreator
|
||||
maxDepth: number
|
||||
capacitySlot?: boolean
|
||||
}
|
||||
): { dispatch: DispatchContextRow; worker: WorkerDispatchRow } {
|
||||
this.db.exec('BEGIN IMMEDIATE')
|
||||
@@ -85,6 +86,7 @@ export function createStartingWorkerDispatch(
|
||||
`Task ${task.id} is ${task.status}; only a ready Task can start.`
|
||||
)
|
||||
}
|
||||
this.assertRunCapacityAvailable(task.id, params.capacitySlot === true)
|
||||
|
||||
const id = generateId('ctx')
|
||||
if (params.mutationReceipt) {
|
||||
|
||||
@@ -28,9 +28,13 @@ const POST_V6_COLUMNS = [
|
||||
const VERSIONED_POST_V6_COLUMNS = [
|
||||
{ version: 27, table: 'federated_dispatches', column: 'to_home_acknowledged_sequence' },
|
||||
{ version: 30, table: 'dispatch_contexts', column: 'depth' },
|
||||
{ version: 30, table: 'remote_dispatch_attachments', column: 'depth' }
|
||||
{ version: 30, table: 'remote_dispatch_attachments', column: 'depth' },
|
||||
{ version: 31, table: 'runs', column: 'target_concurrency' },
|
||||
{ version: 31, table: 'tasks', column: 'capacity_eligible' }
|
||||
] as const
|
||||
|
||||
const VERSIONED_POST_V6_INDEXES = [{ version: 31, index: 'idx_tasks_capacity_ready' }] as const
|
||||
|
||||
const POST_V6_INDEXES = [
|
||||
'idx_messages_run_sequence',
|
||||
'idx_messages_delivery_contract',
|
||||
@@ -95,6 +99,9 @@ function hasCompletePostV6Schema(db: Database.Database, storedVersion: number):
|
||||
({ version, table, column }) =>
|
||||
storedVersion < version || hasOrchestrationColumn(db, table, column)
|
||||
) &&
|
||||
VERSIONED_POST_V6_INDEXES.every(
|
||||
({ version, index }) => storedVersion < version || hasOrchestrationIndex(db, index)
|
||||
) &&
|
||||
POST_V6_INDEXES.every((index) => hasOrchestrationIndex(db, index)) &&
|
||||
messagesAllowQuestions(db) &&
|
||||
hasConsistentLegacyAdoption(db)
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import Database from '../../sqlite/sync-database'
|
||||
import { OrchestrationDb } from './db'
|
||||
import { SCHEMA_VERSION } from './db/contract-constants'
|
||||
import { resolveOrchestrationMigrationStartVersion } from './orchestration-schema-version-skew'
|
||||
|
||||
describe('Run capacity migration (v31)', () => {
|
||||
let db: OrchestrationDb | undefined
|
||||
let tempDir: string | undefined
|
||||
|
||||
afterEach(() => {
|
||||
db?.close()
|
||||
if (tempDir) {
|
||||
rmSync(tempDir, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
function createV30Database(): { dbPath: string; runId: string; taskId: string } {
|
||||
tempDir = mkdtempSync(join(tmpdir(), 'orca-run-capacity-migration-'))
|
||||
const dbPath = join(tempDir, 'orchestration.db')
|
||||
const current = new OrchestrationDb(dbPath)
|
||||
const run = current.createRun({
|
||||
objective: 'upgrade capacity defaults',
|
||||
coordinatorHandle: 'term_coord',
|
||||
coordinatorPaneKey: 'repo:worktree:tab:leaf'
|
||||
})
|
||||
const task = current.createTask({ spec: 'pre-capacity task', runId: run.id })
|
||||
current.close()
|
||||
|
||||
const oldDb = new Database(dbPath)
|
||||
oldDb.exec('DROP INDEX idx_tasks_capacity_ready')
|
||||
oldDb.exec('ALTER TABLE tasks DROP COLUMN capacity_eligible')
|
||||
oldDb.exec('ALTER TABLE runs DROP COLUMN target_concurrency')
|
||||
oldDb.pragma('user_version = 30')
|
||||
oldDb.close()
|
||||
return { dbPath, runId: run.id, taskId: task.id }
|
||||
}
|
||||
|
||||
it('adds disabled targets and unenrolled Tasks when upgrading v30', () => {
|
||||
const { dbPath, runId, taskId } = createV30Database()
|
||||
db = new OrchestrationDb(dbPath)
|
||||
const sqlite = (db as unknown as { db: Database.Database }).db
|
||||
|
||||
expect(sqlite.pragma('user_version', { simple: true })).toBe(SCHEMA_VERSION)
|
||||
expect(db.getRun(runId)).toMatchObject({ target_concurrency: 0 })
|
||||
expect(db.getTask(taskId)).toMatchObject({ capacity_eligible: 0 })
|
||||
expect(db.getRunCapacity(runId)).toMatchObject({
|
||||
targetConcurrency: 0,
|
||||
activeCount: 0,
|
||||
availableSlots: 0,
|
||||
launchableCount: 0
|
||||
})
|
||||
})
|
||||
|
||||
it('repairs a database that claims v31 without the v31 columns', () => {
|
||||
const { dbPath } = createV30Database()
|
||||
const malformed = new Database(dbPath)
|
||||
malformed.pragma('user_version = 31')
|
||||
|
||||
expect(resolveOrchestrationMigrationStartVersion(malformed, 31, SCHEMA_VERSION)).toBe(6)
|
||||
malformed.close()
|
||||
|
||||
db = new OrchestrationDb(dbPath)
|
||||
expect(db.db.pragma('user_version', { simple: true })).toBe(SCHEMA_VERSION)
|
||||
})
|
||||
|
||||
it('repairs a database that claims v31 without the capacity-ready index', () => {
|
||||
tempDir = mkdtempSync(join(tmpdir(), 'orca-run-capacity-index-repair-'))
|
||||
const dbPath = join(tempDir, 'orchestration.db')
|
||||
db = new OrchestrationDb(dbPath)
|
||||
db.close()
|
||||
db = undefined
|
||||
|
||||
const malformed = new Database(dbPath)
|
||||
malformed.exec('DROP INDEX idx_tasks_capacity_ready')
|
||||
expect(resolveOrchestrationMigrationStartVersion(malformed, 31, SCHEMA_VERSION)).toBe(6)
|
||||
malformed.close()
|
||||
|
||||
db = new OrchestrationDb(dbPath)
|
||||
expect(
|
||||
db.db
|
||||
.prepare("SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = ?")
|
||||
.get('idx_tasks_capacity_ready')
|
||||
).toBeDefined()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,391 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { OrchestrationDb } from './db'
|
||||
import { createRootDispatch } from './db/root-dispatch-test-fixture'
|
||||
|
||||
describe('orchestration run capacity', () => {
|
||||
let db: OrchestrationDb | undefined
|
||||
let tempDirectory: string | undefined
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks()
|
||||
db?.close()
|
||||
if (tempDirectory) {
|
||||
rmSync(tempDirectory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
function setup(targetConcurrency = 2): { runId: string } {
|
||||
db = new OrchestrationDb(':memory:')
|
||||
const run = db.createRun({
|
||||
objective: 'keep an explicitly enrolled wave full',
|
||||
coordinatorHandle: 'term_coordinator',
|
||||
coordinatorPaneKey: 'repo:worktree:tab:leaf'
|
||||
})
|
||||
db.configureRunCapacity(run.id, targetConcurrency)
|
||||
return { runId: run.id }
|
||||
}
|
||||
|
||||
function startAndSettle(taskId: string, terminalHandle: string): string {
|
||||
const started = db!.createStartingWorkerDispatch({
|
||||
taskId,
|
||||
startOptions: {},
|
||||
creator: { kind: 'system' },
|
||||
maxDepth: 1,
|
||||
capacitySlot: true
|
||||
})
|
||||
db!.prepareStartingWorkerAuthority({
|
||||
dispatchId: started.dispatch.id,
|
||||
handle: terminalHandle,
|
||||
paneKey: `repo:worktree:tab:${terminalHandle}`,
|
||||
processIncarnation: `pid:${terminalHandle}`,
|
||||
worktreeId: 'repo::worktree',
|
||||
effects: [],
|
||||
setupState: 'not_applicable',
|
||||
terminalOwnership: 'created'
|
||||
})
|
||||
db!.markWorkerDispatchReady(started.dispatch.id)
|
||||
db!.settleWorkerReport({
|
||||
taskId,
|
||||
dispatchId: started.dispatch.id,
|
||||
outcome: 'succeeded',
|
||||
result: '{}'
|
||||
})
|
||||
return started.dispatch.id
|
||||
}
|
||||
|
||||
it('exposes only enrolled ready tasks as launchable capacity', () => {
|
||||
const { runId } = setup(3)
|
||||
const active = db!.createTask({ spec: 'active', runId, capacityEligible: true })
|
||||
createRootDispatch(db!, active.id, 'term_active')
|
||||
const ready = db!.createTask({ spec: 'ready', runId, capacityEligible: true })
|
||||
const unproven = db!.createTask({ spec: 'not enrolled', runId })
|
||||
const dependency = db!.createTask({ spec: 'dependency', runId })
|
||||
const pending = db!.createTask({
|
||||
spec: 'blocked on dependency',
|
||||
runId,
|
||||
deps: [dependency.id],
|
||||
capacityEligible: true
|
||||
})
|
||||
|
||||
const snapshot = db!.getRunCapacity(runId)
|
||||
|
||||
expect(snapshot).toMatchObject({
|
||||
runId,
|
||||
targetConcurrency: 3,
|
||||
activeCount: 1,
|
||||
availableSlots: 2,
|
||||
launchableCount: 1
|
||||
})
|
||||
expect(snapshot.launchableTasks.map((task) => task.id)).toEqual([ready.id])
|
||||
expect(snapshot.eligiblePendingTaskIds).toEqual([pending.id])
|
||||
expect(snapshot.launchableTasks.map((task) => task.id)).not.toContain(unproven.id)
|
||||
})
|
||||
|
||||
it('does not offer a pre-fix split Task that still has an active Dispatch', () => {
|
||||
const { runId } = setup(2)
|
||||
const split = db!.createTask({ spec: 'legacy split', runId, capacityEligible: true })
|
||||
createRootDispatch(db!, split.id, 'term_active')
|
||||
db!.db.prepare("UPDATE tasks SET status = 'ready' WHERE id = ?").run(split.id)
|
||||
|
||||
expect(db!.getRunCapacity(runId).launchableTasks).toEqual([])
|
||||
})
|
||||
|
||||
it('does not offer a Task while a decision gate remains pending', () => {
|
||||
const { runId } = setup(1)
|
||||
const task = db!.createTask({ spec: 'gated work', runId, capacityEligible: true })
|
||||
const gate = db!.createGate({ taskId: task.id, question: 'Proceed?' })
|
||||
db!.db.prepare("UPDATE tasks SET status = 'ready' WHERE id = ?").run(task.id)
|
||||
|
||||
expect(db!.getRunCapacity(runId).launchableTasks).toEqual([])
|
||||
db!.resolveGate(gate.id, 'yes')
|
||||
expect(db!.getRunCapacity(runId).launchableTasks.map((candidate) => candidate.id)).toEqual([
|
||||
task.id
|
||||
])
|
||||
})
|
||||
|
||||
it('bounds the indexed launchable query by the available slots', () => {
|
||||
const { runId } = setup(3)
|
||||
for (let index = 0; index < 200; index += 1) {
|
||||
db!.createTask({ spec: `ready ${index}`, runId, capacityEligible: true })
|
||||
}
|
||||
const prepare = db!.db.prepare.bind(db!.db)
|
||||
let launchableSql = ''
|
||||
vi.spyOn(db!.db, 'prepare').mockImplementation((sql) => {
|
||||
if (sql.includes('SELECT tasks.* FROM tasks')) {
|
||||
launchableSql = sql
|
||||
}
|
||||
return prepare(sql)
|
||||
})
|
||||
|
||||
expect(db!.getRunCapacity(runId).launchableTasks).toHaveLength(3)
|
||||
expect(launchableSql).toContain('LIMIT ?')
|
||||
const plan = prepare(`EXPLAIN QUERY PLAN ${launchableSql}`).all(runId, 3) as {
|
||||
detail: string
|
||||
}[]
|
||||
expect(plan.some((row) => row.detail.includes('idx_tasks_capacity_ready'))).toBe(true)
|
||||
})
|
||||
|
||||
it('atomically refuses a capacity start above the exact target', () => {
|
||||
const { runId } = setup(1)
|
||||
const first = db!.createTask({ spec: 'first', runId, capacityEligible: true })
|
||||
const second = db!.createTask({ spec: 'second', runId, capacityEligible: true })
|
||||
|
||||
db!.createStartingWorkerDispatch({
|
||||
taskId: first.id,
|
||||
startOptions: {},
|
||||
creator: { kind: 'system' },
|
||||
maxDepth: 1,
|
||||
capacitySlot: true
|
||||
})
|
||||
|
||||
expect(() =>
|
||||
db!.createStartingWorkerDispatch({
|
||||
taskId: second.id,
|
||||
startOptions: {},
|
||||
creator: { kind: 'system' },
|
||||
maxDepth: 1,
|
||||
capacitySlot: true
|
||||
})
|
||||
).toThrowError(/target concurrency 1/)
|
||||
expect(db!.getTask(second.id)?.status).toBe('ready')
|
||||
expect(() => createRootDispatch(db!, second.id, 'term_bypass')).toThrowError(
|
||||
/target concurrency 1/
|
||||
)
|
||||
})
|
||||
|
||||
it('claims federated capacity at the authoritative Run home', () => {
|
||||
const { runId } = setup(1)
|
||||
const first = db!.createTask({ spec: 'first remote', runId, capacityEligible: true })
|
||||
const second = db!.createTask({ spec: 'second remote', runId, capacityEligible: true })
|
||||
const federation = {
|
||||
environmentId: 'environment_windows',
|
||||
environmentName: 'windows',
|
||||
peerFingerprint: 'windows_peer',
|
||||
protocolVersion: 3
|
||||
}
|
||||
|
||||
db!.createStartingWorkerDispatch({
|
||||
taskId: first.id,
|
||||
startOptions: {},
|
||||
creator: { kind: 'system' },
|
||||
maxDepth: 1,
|
||||
capacitySlot: true,
|
||||
federation
|
||||
})
|
||||
|
||||
expect(() =>
|
||||
db!.createStartingWorkerDispatch({
|
||||
taskId: second.id,
|
||||
startOptions: {},
|
||||
creator: { kind: 'system' },
|
||||
maxDepth: 1,
|
||||
capacitySlot: true,
|
||||
federation
|
||||
})
|
||||
).toThrowError(/target concurrency 1/)
|
||||
})
|
||||
|
||||
it('serializes simultaneous low-level Dispatch claims across database connections', () => {
|
||||
tempDirectory = mkdtempSync(join(tmpdir(), 'orca-run-capacity-race-'))
|
||||
const databasePath = join(tempDirectory, 'orchestration.db')
|
||||
db = new OrchestrationDb(databasePath)
|
||||
const concurrent = new OrchestrationDb(databasePath)
|
||||
try {
|
||||
const run = db.createRun({
|
||||
objective: 'serialize low-level claims',
|
||||
coordinatorHandle: 'term_coord',
|
||||
coordinatorPaneKey: 'repo:worktree:tab:leaf'
|
||||
})
|
||||
db.configureRunCapacity(run.id, 1)
|
||||
const loser = db.createTask({ spec: 'losing claim', runId: run.id, capacityEligible: true })
|
||||
const winner = db.createTask({ spec: 'winning claim', runId: run.id, capacityEligible: true })
|
||||
const prepare = db.db.prepare.bind(db.db)
|
||||
let winnerDispatchId: string | undefined
|
||||
vi.spyOn(db.db, 'prepare').mockImplementation((sql) => {
|
||||
if (!winnerDispatchId && sql.includes('INSERT INTO dispatch_contexts')) {
|
||||
winnerDispatchId = createRootDispatch(concurrent, winner.id, 'term_winner').id
|
||||
}
|
||||
return prepare(sql)
|
||||
})
|
||||
|
||||
expect(() => createRootDispatch(db!, loser.id, 'term_loser')).toThrowError(
|
||||
/target concurrency 1/
|
||||
)
|
||||
expect(winnerDispatchId).toBeDefined()
|
||||
expect(db.getTask(loser.id)?.status).toBe('ready')
|
||||
expect(db.getRunCapacity(run.id)).toMatchObject({ activeCount: 1, availableSlots: 0 })
|
||||
} finally {
|
||||
concurrent.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('requires explicit enrollment for capacity starts', () => {
|
||||
const { runId } = setup(1)
|
||||
const task = db!.createTask({ spec: 'ordinary ready task', runId })
|
||||
|
||||
expect(() =>
|
||||
db!.createStartingWorkerDispatch({
|
||||
taskId: task.id,
|
||||
startOptions: {},
|
||||
creator: { kind: 'system' },
|
||||
maxDepth: 1,
|
||||
capacitySlot: true
|
||||
})
|
||||
).toThrowError(/not enrolled/)
|
||||
})
|
||||
|
||||
it('reports settled terminal debt separately from live lane capacity', () => {
|
||||
const { runId } = setup(2)
|
||||
const task = db!.createTask({ spec: 'settled worker', runId, capacityEligible: true })
|
||||
const dispatchId = startAndSettle(task.id, 'term_worker')
|
||||
|
||||
const snapshot = db!.getRunCapacity(runId)
|
||||
|
||||
expect(snapshot.activeCount).toBe(0)
|
||||
expect(snapshot.availableSlots).toBe(2)
|
||||
expect(snapshot.settledTerminalDebt).toEqual([
|
||||
expect.objectContaining({ dispatchId, terminalState: 'reclaimable' })
|
||||
])
|
||||
})
|
||||
|
||||
it('recomputes slots after simultaneous settlements and dependency promotion', () => {
|
||||
const { runId } = setup(2)
|
||||
const first = db!.createTask({ spec: 'first active', runId, capacityEligible: true })
|
||||
const second = db!.createTask({ spec: 'second active', runId, capacityEligible: true })
|
||||
const dependent = db!.createTask({
|
||||
spec: 'newly ready replacement',
|
||||
runId,
|
||||
deps: [first.id],
|
||||
capacityEligible: true
|
||||
})
|
||||
const firstStarted = db!.createStartingWorkerDispatch({
|
||||
taskId: first.id,
|
||||
startOptions: {},
|
||||
creator: { kind: 'system' },
|
||||
maxDepth: 1,
|
||||
capacitySlot: true
|
||||
})
|
||||
const secondStarted = db!.createStartingWorkerDispatch({
|
||||
taskId: second.id,
|
||||
startOptions: {},
|
||||
creator: { kind: 'system' },
|
||||
maxDepth: 1,
|
||||
capacitySlot: true
|
||||
})
|
||||
for (const [task, started, terminal] of [
|
||||
[first, firstStarted, 'term_first'],
|
||||
[second, secondStarted, 'term_second']
|
||||
] as const) {
|
||||
db!.prepareStartingWorkerAuthority({
|
||||
dispatchId: started.dispatch.id,
|
||||
handle: terminal,
|
||||
paneKey: `repo:worktree:tab:${terminal}`,
|
||||
processIncarnation: `pid:${terminal}`,
|
||||
worktreeId: 'repo::worktree',
|
||||
effects: [],
|
||||
setupState: 'not_applicable',
|
||||
terminalOwnership: 'created'
|
||||
})
|
||||
db!.markWorkerDispatchReady(started.dispatch.id)
|
||||
db!.settleWorkerReport({
|
||||
taskId: task.id,
|
||||
dispatchId: started.dispatch.id,
|
||||
outcome: 'succeeded',
|
||||
result: '{}'
|
||||
})
|
||||
}
|
||||
|
||||
const snapshot = db!.getRunCapacity(runId)
|
||||
expect(snapshot.activeCount).toBe(0)
|
||||
expect(snapshot.availableSlots).toBe(2)
|
||||
expect(snapshot.launchableTasks.map((task) => task.id)).toEqual([dependent.id])
|
||||
})
|
||||
|
||||
it('excludes failed and blocked enrolled tasks when no eligible work remains', () => {
|
||||
const { runId } = setup(2)
|
||||
const failed = db!.createTask({ spec: 'failed', runId, capacityEligible: true })
|
||||
const blocked = db!.createTask({ spec: 'blocked', runId, capacityEligible: true })
|
||||
db!.updateTaskStatus(failed.id, 'failed')
|
||||
db!.updateTaskStatus(blocked.id, 'blocked')
|
||||
|
||||
const snapshot = db!.getRunCapacity(runId)
|
||||
expect(snapshot.launchableCount).toBe(0)
|
||||
expect(snapshot.eligiblePendingTaskIds).toEqual([blocked.id])
|
||||
})
|
||||
|
||||
it('stops offering an enrolled Task after its retry circuit breaks', () => {
|
||||
const { runId } = setup(1)
|
||||
const task = db!.createTask({
|
||||
spec: 'repeatedly failing capacity work',
|
||||
runId,
|
||||
capacityEligible: true
|
||||
})
|
||||
|
||||
for (let attempt = 0; attempt < 3; attempt += 1) {
|
||||
const dispatch = createRootDispatch(db!, task.id, `term_retry_${attempt}`)
|
||||
db!.failDispatch(dispatch.id, `attempt ${attempt + 1}`)
|
||||
}
|
||||
|
||||
expect(db!.getTask(task.id)?.status).toBe('failed')
|
||||
expect(db!.getRunCapacity(runId)).toMatchObject({
|
||||
activeCount: 0,
|
||||
availableSlots: 1,
|
||||
launchableCount: 0,
|
||||
launchableTasks: []
|
||||
})
|
||||
})
|
||||
|
||||
it('persists target and enrollment across a database restart', () => {
|
||||
tempDirectory = mkdtempSync(join(tmpdir(), 'orca-run-capacity-'))
|
||||
const databasePath = join(tempDirectory, 'orchestration.db')
|
||||
db = new OrchestrationDb(databasePath)
|
||||
const run = db.createRun({
|
||||
objective: 'restart durable capacity',
|
||||
coordinatorHandle: 'term_coordinator',
|
||||
coordinatorPaneKey: 'repo:worktree:tab:leaf'
|
||||
})
|
||||
const task = db.createTask({
|
||||
spec: 'durable enrollment',
|
||||
runId: run.id,
|
||||
capacityEligible: true
|
||||
})
|
||||
db.configureRunCapacity(run.id, 5)
|
||||
db.close()
|
||||
db = new OrchestrationDb(databasePath)
|
||||
|
||||
const snapshot = db.getRunCapacity(run.id)
|
||||
expect(snapshot.targetConcurrency).toBe(5)
|
||||
expect(snapshot.launchableTasks.map((candidate) => candidate.id)).toEqual([task.id])
|
||||
})
|
||||
|
||||
it('surfaces user takeover and release-unknown without closing either terminal', () => {
|
||||
const { runId } = setup(2)
|
||||
const unknownTask = db!.createTask({ spec: 'unknown release', runId, capacityEligible: true })
|
||||
const userTask = db!.createTask({ spec: 'user takeover', runId, capacityEligible: true })
|
||||
const unknownDispatch = startAndSettle(unknownTask.id, 'term_unknown')
|
||||
const userDispatch = startAndSettle(userTask.id, 'term_user')
|
||||
db!.db
|
||||
.prepare(
|
||||
"UPDATE worker_terminal_resources SET release_state = 'unknown' WHERE owner_dispatch_id = ?"
|
||||
)
|
||||
.run(unknownDispatch)
|
||||
db!.db
|
||||
.prepare(
|
||||
"UPDATE worker_terminal_resources SET ownership_state = 'user_owned', release_state = 'retained', retained_reason = 'user_takeover' WHERE owner_dispatch_id = ?"
|
||||
)
|
||||
.run(userDispatch)
|
||||
|
||||
const debt = db!.getRunCapacity(runId).settledTerminalDebt
|
||||
expect(debt).toEqual([
|
||||
expect.objectContaining({ dispatchId: unknownDispatch, terminalState: 'release_unknown' }),
|
||||
expect.objectContaining({
|
||||
dispatchId: userDispatch,
|
||||
terminalState: 'retained',
|
||||
retainedReason: 'user_takeover'
|
||||
})
|
||||
])
|
||||
})
|
||||
})
|
||||
@@ -48,6 +48,7 @@ export type RunRow = {
|
||||
coordinator_pane_key: string | null
|
||||
consumer_generation: number
|
||||
legacy: number
|
||||
target_concurrency: number
|
||||
created_at: string
|
||||
updated_at: string
|
||||
}
|
||||
@@ -257,6 +258,7 @@ export type TaskRow = {
|
||||
display_name: string | null
|
||||
spec: string
|
||||
status: TaskStatus
|
||||
capacity_eligible: number
|
||||
deps: string
|
||||
result: string | null
|
||||
created_at: string
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
import { afterEach, describe, expect, it } from 'vitest'
|
||||
import type { RpcContext } from '../core'
|
||||
import { createOrchestrationRpcHarness } from './orchestration-rpc-test-harness'
|
||||
import type { OrchestrationDb } from '../../orchestration/db'
|
||||
import type { OrcaRuntimeService } from '../../orca-runtime'
|
||||
import { RpcDispatcher } from '../dispatcher'
|
||||
import { ORCHESTRATION_METHODS } from './orchestration'
|
||||
import { ORCHESTRATION_CONTRACT_VERSION } from '../../../../shared/protocol-version'
|
||||
|
||||
describe('orchestration capacity RPC', () => {
|
||||
const h = createOrchestrationRpcHarness()
|
||||
let db: OrchestrationDb
|
||||
let runtime: OrcaRuntimeService
|
||||
let ctx: RpcContext
|
||||
let runId: string
|
||||
|
||||
afterEach(() => h.cleanup())
|
||||
|
||||
function setup(): void {
|
||||
const state = h.setup()
|
||||
db = state.db
|
||||
runtime = state.runtime
|
||||
ctx = state.ctx
|
||||
runId = state.activeRunId as string
|
||||
}
|
||||
|
||||
it('declares a target and exposes explicitly enrolled ready work', async () => {
|
||||
setup()
|
||||
const task = db.createTask({ spec: 'eligible backfill', runId })
|
||||
|
||||
await h.call(
|
||||
'orchestration.capacityConfigure',
|
||||
{ target: 2, run: runId, from: 'term_coord' },
|
||||
ctx
|
||||
)
|
||||
await h.call(
|
||||
'orchestration.capacityTaskSet',
|
||||
{ task: task.id, eligible: true, run: runId, from: 'term_coord' },
|
||||
ctx
|
||||
)
|
||||
const shown = (await h.call(
|
||||
'orchestration.capacityShow',
|
||||
{ run: runId, from: 'term_coord' },
|
||||
ctx
|
||||
)) as { capacity: { targetConcurrency: number; launchableTasks: { id: string }[] } }
|
||||
|
||||
expect(shown.capacity.targetConcurrency).toBe(2)
|
||||
expect(shown.capacity.launchableTasks.map((candidate) => candidate.id)).toEqual([task.id])
|
||||
})
|
||||
|
||||
it('rejects capacity enrollment outside the bound Run', async () => {
|
||||
setup()
|
||||
const foreign = db.createRun({
|
||||
objective: 'foreign',
|
||||
coordinatorHandle: 'term_foreign',
|
||||
coordinatorPaneKey: 'foreign:worktree:tab:leaf'
|
||||
})
|
||||
const task = db.createTask({ spec: 'foreign task', runId: foreign.id })
|
||||
|
||||
await expect(
|
||||
h.call(
|
||||
'orchestration.capacityTaskSet',
|
||||
{ task: task.id, eligible: true, run: runId, from: 'term_coord' },
|
||||
ctx
|
||||
)
|
||||
).rejects.toMatchObject({ code: 'task_not_found' })
|
||||
})
|
||||
|
||||
it('replays target configuration without applying the mutation twice', async () => {
|
||||
setup()
|
||||
const dispatcher = new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS })
|
||||
const request = {
|
||||
authToken: 'local-token',
|
||||
method: 'orchestration.capacityConfigure',
|
||||
params: { target: 2, run: runId, from: 'term_coord' },
|
||||
orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION,
|
||||
orchestrationRequestId: 'capacity_replay'
|
||||
}
|
||||
|
||||
const first = await dispatcher.dispatch({ ...request, id: 'rpc_first' })
|
||||
db.configureRunCapacity(runId, 3)
|
||||
const replay = await dispatcher.dispatch({ ...request, id: 'rpc_replay' })
|
||||
|
||||
expect(first).toMatchObject({
|
||||
ok: true,
|
||||
result: { capacity: { targetConcurrency: 2 }, mutation: { replayed: false } }
|
||||
})
|
||||
expect(replay).toMatchObject({
|
||||
ok: true,
|
||||
result: { capacity: { targetConcurrency: 2 }, mutation: { replayed: true } }
|
||||
})
|
||||
expect(db.getRunCapacity(runId).targetConcurrency).toBe(3)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,81 @@
|
||||
import { z } from 'zod'
|
||||
import { defineMethod, type RpcMethod } from '../core'
|
||||
import { OptionalString, requiredString } from '../schemas'
|
||||
import { OrchestrationError } from '../../orchestration/orchestration-error'
|
||||
import { resolveRunScope } from './orchestration-run-scope'
|
||||
|
||||
const CapacityConfigureParams = z.object({
|
||||
target: z.number().int().min(0).max(64),
|
||||
run: OptionalString,
|
||||
from: requiredString('Missing coordinator terminal')
|
||||
})
|
||||
|
||||
const CapacityTaskParams = z.object({
|
||||
task: requiredString('Missing --task'),
|
||||
eligible: z.boolean(),
|
||||
run: OptionalString,
|
||||
from: requiredString('Missing coordinator terminal')
|
||||
})
|
||||
|
||||
const CapacityShowParams = z.object({
|
||||
run: OptionalString,
|
||||
from: OptionalString
|
||||
})
|
||||
|
||||
export const ORCHESTRATION_CAPACITY_METHODS: RpcMethod[] = [
|
||||
defineMethod({
|
||||
name: 'orchestration.capacityConfigure',
|
||||
params: CapacityConfigureParams,
|
||||
handler: (params, { orchestrationCompatibilityEvidence, runtime, legacyCoordinatorRunId }) => {
|
||||
const run = resolveRunScope(runtime, {
|
||||
runId: params.run,
|
||||
callerTerminalHandle: params.from,
|
||||
requireCurrentConsumer: true,
|
||||
legacyCoordinatorRunId,
|
||||
callerEvidence: orchestrationCompatibilityEvidence
|
||||
})
|
||||
return { capacity: runtime.getOrchestrationDb().configureRunCapacity(run.id, params.target) }
|
||||
}
|
||||
}),
|
||||
defineMethod({
|
||||
name: 'orchestration.capacityTaskSet',
|
||||
params: CapacityTaskParams,
|
||||
handler: (params, { orchestrationCompatibilityEvidence, runtime, legacyCoordinatorRunId }) => {
|
||||
const db = runtime.getOrchestrationDb()
|
||||
const run = resolveRunScope(runtime, {
|
||||
runId: params.run,
|
||||
callerTerminalHandle: params.from,
|
||||
requireCurrentConsumer: true,
|
||||
legacyCoordinatorRunId,
|
||||
callerEvidence: orchestrationCompatibilityEvidence
|
||||
})
|
||||
const existing = db.getTask(params.task)
|
||||
if (!existing || existing.run_id !== run.id) {
|
||||
throw new OrchestrationError(
|
||||
'task_not_found',
|
||||
`Task ${params.task} was not found in Run ${run.id}.`
|
||||
)
|
||||
}
|
||||
return { task: db.setTaskCapacityEligibility(params.task, params.eligible) }
|
||||
}
|
||||
}),
|
||||
defineMethod({
|
||||
name: 'orchestration.capacityShow',
|
||||
params: CapacityShowParams,
|
||||
handler: (params, { orchestrationCompatibilityEvidence, runtime, legacyCoordinatorRunId }) => {
|
||||
const db = runtime.getOrchestrationDb()
|
||||
const explicitRun = params.run ? db.getRun(params.run) : undefined
|
||||
const run =
|
||||
explicitRun?.legacy === 0
|
||||
? explicitRun
|
||||
: resolveRunScope(runtime, {
|
||||
runId: params.run,
|
||||
callerTerminalHandle: params.from,
|
||||
requireCurrentConsumer: params.run === undefined,
|
||||
legacyCoordinatorRunId,
|
||||
callerEvidence: orchestrationCompatibilityEvidence
|
||||
})
|
||||
return { capacity: db.getRunCapacity(run.id) }
|
||||
}
|
||||
})
|
||||
]
|
||||
@@ -118,10 +118,12 @@ export async function startFederatedWorker(args: {
|
||||
? params.setup
|
||||
? 'explicit_request'
|
||||
: 'orchestration_default'
|
||||
: 'existing_worktree'
|
||||
: 'existing_worktree',
|
||||
capacitySlot: params.capacitySlot ?? false
|
||||
},
|
||||
runtimeEpoch: runtime.getRuntimeId(),
|
||||
mutationReceipt: orchestrationMutation,
|
||||
capacitySlot: params.capacitySlot,
|
||||
federation: {
|
||||
environmentId: server.environmentId,
|
||||
environmentName: server.name,
|
||||
|
||||
@@ -25,6 +25,7 @@ export const WorkerStartParams = z.object({
|
||||
model: OptionalWorkerLaunchPreference,
|
||||
effort: OptionalWorkerLaunchPreference,
|
||||
retryOf: OptionalString,
|
||||
capacitySlot: z.boolean().optional(),
|
||||
timeoutMs: OptionalFiniteNumber,
|
||||
devMode: z.boolean().optional()
|
||||
})
|
||||
|
||||
@@ -117,7 +117,8 @@ export const ORCHESTRATION_WORKER_START_METHODS: RpcMethod[] = [
|
||||
? params.setup
|
||||
? 'explicit_request'
|
||||
: 'orchestration_default'
|
||||
: 'existing_worktree'
|
||||
: 'existing_worktree',
|
||||
capacitySlot: params.capacitySlot ?? false
|
||||
}
|
||||
const started = db.createStartingWorkerDispatch({
|
||||
creator: resolveDispatchCreator(runtime, params.from),
|
||||
@@ -126,7 +127,8 @@ export const ORCHESTRATION_WORKER_START_METHODS: RpcMethod[] = [
|
||||
retryOf: params.retryOf,
|
||||
startOptions,
|
||||
runtimeEpoch: runtime.getRuntimeId(),
|
||||
mutationReceipt: orchestrationMutation
|
||||
mutationReceipt: orchestrationMutation,
|
||||
capacitySlot: params.capacitySlot
|
||||
})
|
||||
const effects: WorkerEffect[] = []
|
||||
if (resolvedWorktree) {
|
||||
|
||||
@@ -33,6 +33,7 @@ import { resolveRunScope } from './orchestration-run-scope'
|
||||
import { ORCHESTRATION_RUN_METHODS } from './orchestration-runs'
|
||||
import { ORCHESTRATION_WORKER_METHODS } from './orchestration-worker-methods'
|
||||
import { ORCHESTRATION_FEDERATION_METHODS } from './orchestration-federation-methods'
|
||||
import { ORCHESTRATION_CAPACITY_METHODS } from './orchestration-capacity'
|
||||
import { OrchestrationError } from '../../orchestration/orchestration-error'
|
||||
import type { OrcaRuntimeService } from '../../orca-runtime'
|
||||
import type { RunRow } from '../../orchestration/types'
|
||||
@@ -440,6 +441,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
|
||||
...ORCHESTRATION_RUN_METHODS,
|
||||
...ORCHESTRATION_WORKER_METHODS,
|
||||
...ORCHESTRATION_FEDERATION_METHODS,
|
||||
...ORCHESTRATION_CAPACITY_METHODS,
|
||||
defineMethod({
|
||||
name: 'orchestration.send',
|
||||
params: SendParams,
|
||||
|
||||
@@ -13,6 +13,8 @@ describe('orchestration RPC contract', () => {
|
||||
['orchestration.reply', {}],
|
||||
['orchestration.taskCreate', {}],
|
||||
['orchestration.taskUpdate', {}],
|
||||
['orchestration.capacityConfigure', {}],
|
||||
['orchestration.capacityTaskSet', {}],
|
||||
['orchestration.dispatch', {}],
|
||||
['orchestration.workerStart', {}],
|
||||
['orchestration.workerStop', {}],
|
||||
|
||||
@@ -22,6 +22,8 @@ const ORCHESTRATION_MUTATION_METHODS = new Set([
|
||||
'orchestration.reply',
|
||||
'orchestration.taskCreate',
|
||||
'orchestration.taskUpdate',
|
||||
'orchestration.capacityConfigure',
|
||||
'orchestration.capacityTaskSet',
|
||||
'orchestration.dispatch',
|
||||
'orchestration.workerStart',
|
||||
'orchestration.workerStop',
|
||||
|
||||
@@ -50,6 +50,8 @@ export const ORCHESTRATION_WORKER_STOP_VERDICT_RUNTIME_CAPABILITY =
|
||||
'orchestration.worker-stop-verdict.v1' as const
|
||||
export const ORCHESTRATION_WORKER_LAUNCH_PREFERENCES_RUNTIME_CAPABILITY =
|
||||
'orchestration.worker-launch-preferences.v1' as const
|
||||
export const ORCHESTRATION_RUN_CAPACITY_RUNTIME_CAPABILITY =
|
||||
'orchestration.run-capacity.v1' as const
|
||||
export const ORCHESTRATION_FEDERATION_CONTROL_MAIL_PROTOCOL_VERSION = 2 as const
|
||||
export const ORCHESTRATION_FEDERATION_LIFECYCLE_SETTLEMENT_PROTOCOL_VERSION = 3 as const
|
||||
export const ORCHESTRATION_CONTRACT_VERSION = 1 as const
|
||||
@@ -164,6 +166,7 @@ export const RUNTIME_CAPABILITIES = [
|
||||
ORCHESTRATION_FEDERATION_LIFECYCLE_SETTLEMENT_RUNTIME_CAPABILITY,
|
||||
ORCHESTRATION_WORKER_STOP_VERDICT_RUNTIME_CAPABILITY,
|
||||
ORCHESTRATION_WORKER_LAUNCH_PREFERENCES_RUNTIME_CAPABILITY,
|
||||
ORCHESTRATION_RUN_CAPACITY_RUNTIME_CAPABILITY,
|
||||
ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY,
|
||||
BROWSER_SCREENCAST_RUNTIME_CAPABILITY,
|
||||
BROWSER_TAB_CREATE_KNOWN_ID_RUNTIME_CAPABILITY,
|
||||
|
||||
Reference in New Issue
Block a user