mirror of
https://github.com/stablyai/orca.git
synced 2026-10-21 16:03:26 +00:00
fix(orchestration): recover dispatch completion capability
This commit is contained in:
Jinwoo-H
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"updatedAt": "2026-08-10",
|
||||
"updatedAt": "2026-08-12",
|
||||
"policy": {
|
||||
"maturityLevels": ["experimental", "soak", "blocking", "accepted-gap", "deprecated"],
|
||||
"blockingPromotion": {
|
||||
@@ -11044,6 +11044,137 @@
|
||||
"Platform-named bundles are loaded on macOS locally; physical Linux and Windows evidence is not yet recorded."
|
||||
],
|
||||
"demotionRule": "Demote if any shipped companion stops loading on Node 18, selected installers mutate outside the isolated home, or the CI lane no longer runs the built artifact with Node 18."
|
||||
},
|
||||
{
|
||||
"id": "orchestration.dispatch-preamble-capability-recovery",
|
||||
"title": "Assigned workers recover one usable lifecycle capability from Dispatch preambles",
|
||||
"maturity": "experimental",
|
||||
"protection": "partial",
|
||||
"owner": "orchestration",
|
||||
"layer": "cli-runtime-sqlite-lifecycle-contract",
|
||||
"surfaces": [
|
||||
"dispatch-show --preamble",
|
||||
"Dispatch capability rotation",
|
||||
"worker_done settlement",
|
||||
"supervised Hermes prompt delivery"
|
||||
],
|
||||
"platforms": ["macos", "linux", "windows"],
|
||||
"providers": ["local", "ssh", "wsl", "paired-runtime"],
|
||||
"coveredPlatforms": ["macos"],
|
||||
"coveredProviders": ["local"],
|
||||
"coverageNotes": "Deterministic CLI, runtime, and dispatcher tests cover the accepted Orca-to-Hermes prompt-write boundary, exact local assignee recovery through direct and streaming dispatcher transports, settlement, handle remint, normalized full-evidence secret-safe concurrent deduplication, repeat rotation, coordinator inspection, old-runtime and missing-attestation refusal including reminted workers, pre-rotation response preparation failure, claimed-handle mismatch, stale/pending/revoked/start-unknown/stopping authority, same-runtime legacy Dispatch inspection, and zero terminal reinjection. A synthetic stale SSH attachment test covers the deduplication identity contract, not a live SSH host. Hermes 0.15.2 was installed under an isolated temporary HOME, but the run stopped before prompt consumption because no provider credentials were configured. Live SSH, WSL, paired hosts, mixed-binary recovery, Linux, Windows, and authenticated Hermes consumption remain uncollected.",
|
||||
"motivatingLinks": [
|
||||
"https://github.com/stablyai/orca/issues/12769",
|
||||
"https://linear.app/stably/issue/STA-3539"
|
||||
],
|
||||
"invariant": "An accepted supervised start must deliver the complete non-placeholder task and lifecycle preamble, including completion authority, to the exact assigned process incarnation. An active current-contract assignee may derive a fresh lifecycle command from dispatch-show --preamble and settle once; coordinators, claimed handles, stale processes, and legacy Dispatches must not rotate authority. Recovery persists only a capability hash and never writes to the terminal.",
|
||||
"oracle": "Start a supervised Hermes worker through production runtime methods and require Orca to accept one agent-prompt write containing the task marker, Task, Dispatch, and capability, targeted to a Dispatch durably bound to the asserted pane and process incarnation, with no placeholder or raw terminal write; then settle worker_done. Independently create an active Dispatch, invoke dispatchShow as its exact attested worker through direct and streaming dispatch, extract the new capability, and settle Task and Dispatch exactly once. Require same-request concurrency to return one in-memory-deduplicated token without a durable secret, handle remint to emit the current sender and CLI, and repeat recovery to invalidate only the prior token while the newest remains valid. Require coordinator, claimed --from mismatch, stale/pending/revoked/start-unknown/stopping authority, and legacy Dispatch attempts to leave existing authority unchanged, with no prompt injection or raw terminal send.",
|
||||
"commands": [
|
||||
"pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/rpc/orchestration-dispatch-preamble-recovery.test.ts src/main/runtime/rpc/methods/orchestration.test.ts src/cli/handlers/orchestration-dispatch-cli.test.ts src/shared/stable-pane-id.test.ts src/cli/runtime-client.test.ts --reporter=dot"
|
||||
],
|
||||
"testFiles": [
|
||||
"src/main/runtime/rpc/orchestration-dispatch-preamble-recovery.test.ts",
|
||||
"src/main/runtime/rpc/methods/orchestration.test.ts",
|
||||
"src/cli/handlers/orchestration-dispatch-cli.test.ts",
|
||||
"src/shared/stable-pane-id.test.ts",
|
||||
"src/cli/runtime-client.test.ts"
|
||||
],
|
||||
"assertionRefs": [
|
||||
{
|
||||
"file": "src/main/runtime/rpc/methods/orchestration.test.ts",
|
||||
"assertions": [
|
||||
"Orca accepts one complete non-placeholder agent-prompt write targeted at the supervised Hermes terminal's exact pane and process",
|
||||
"a recovered lifecycle command contains fresh authority and settles its Task and Dispatch"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "src/main/runtime/rpc/orchestration-dispatch-preamble-recovery.test.ts",
|
||||
"assertions": [
|
||||
"direct and streaming requests recover only for the exact assigned worker and perform no terminal write",
|
||||
"same-request concurrency is deduplicated by normalized full caller evidence without durable plaintext while later recovery invalidates only the prior capability",
|
||||
"fallible response inputs resolve before rotation so preparation failure preserves existing authority",
|
||||
"handle remint emits the current attested sender and host CLI",
|
||||
"coordinator, claimed handle, stale/pending/revoked/start-unknown/stopping authority, and legacy Dispatch attempts cannot rotate authority"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "src/cli/handlers/orchestration-dispatch-cli.test.ts",
|
||||
"assertions": [
|
||||
"the CLI sends an explicit recovery mutation with reminted caller evidence",
|
||||
"tokenless active-worker responses, including a reminted worker against an old runtime, fail actionably instead of printing unusable lifecycle commands"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "src/shared/stable-pane-id.test.ts",
|
||||
"assertions": [
|
||||
"tab-reminted pane keys compare by stable leaf identity without matching a different leaf"
|
||||
]
|
||||
},
|
||||
{
|
||||
"file": "src/cli/runtime-client.test.ts",
|
||||
"assertions": [
|
||||
"additive recovery requests retain mutation IDs and reach an old local runtime without weakening other mutation preflights"
|
||||
]
|
||||
}
|
||||
],
|
||||
"evidenceRuns": [
|
||||
{
|
||||
"date": "2026-08-12",
|
||||
"runner": "local",
|
||||
"platform": "macos",
|
||||
"result": "failed",
|
||||
"command": "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/rpc/orchestration-dispatch-preamble-recovery.test.ts src/main/runtime/rpc/methods/orchestration.test.ts src/cli/handlers/orchestration-dispatch-cli.test.ts src/shared/stable-pane-id.test.ts src/cli/runtime-client.test.ts --reporter=dot",
|
||||
"durationSeconds": 3.53,
|
||||
"summary": "Latest main 09ec516ae50b failed 26 tests while 182 controls passed with identical final test bytes: dispatchShow emitted no lifecycle capability, exact-worker settlement failed, reminted handles stayed stale, and CLI recovery requests were absent."
|
||||
},
|
||||
{
|
||||
"date": "2026-08-12",
|
||||
"runner": "local",
|
||||
"platform": "macos",
|
||||
"result": "failed",
|
||||
"command": "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/rpc/orchestration-dispatch-preamble-recovery.test.ts src/main/runtime/rpc/methods/orchestration.test.ts src/cli/handlers/orchestration-dispatch-cli.test.ts src/shared/stable-pane-id.test.ts src/cli/runtime-client.test.ts --reporter=dot",
|
||||
"durationSeconds": 3.31,
|
||||
"summary": "With the final candidate's causal recovery hook disabled and identical test bytes, 11 tests failed and 197 passed: recovered preambles had no capability, reminted commands kept the stale handle, concurrent recovery never reached rotation, and worker_done could not settle."
|
||||
},
|
||||
{
|
||||
"date": "2026-08-12",
|
||||
"runner": "local",
|
||||
"platform": "macos",
|
||||
"result": "passed",
|
||||
"command": "pnpm exec vitest run --config config/vitest.config.ts src/main/runtime/rpc/orchestration-dispatch-preamble-recovery.test.ts src/main/runtime/rpc/methods/orchestration.test.ts src/cli/handlers/orchestration-dispatch-cli.test.ts src/shared/stable-pane-id.test.ts src/cli/runtime-client.test.ts --reporter=dot",
|
||||
"durationSeconds": 3.47,
|
||||
"summary": "The candidate passed all 208 tests across the five focused files, including prompt-write acceptance, actionable CLI refusal, old-runtime request delivery, stable-pane remint identity, scoped fresh and adopted-database preflights, pre-rotation fault handling, direct and streaming dispatch, settlement, host-bound secret-safe concurrent deduplication, handle remint, repeat rotation, and negative authority states."
|
||||
}
|
||||
],
|
||||
"runtimeBudget": {
|
||||
"p95Seconds": 10,
|
||||
"scope": "focused in-memory runtime, SQLite, dispatcher, and streaming contracts"
|
||||
},
|
||||
"flakeHistory": {
|
||||
"status": "not-started",
|
||||
"evidence": "The oracle uses in-memory state and exact identities with no sleeps, polling, or elapsed-time assertions; CI and soak history have not started."
|
||||
},
|
||||
"redGreenEvidence": {
|
||||
"status": "complete",
|
||||
"evidence": "Latest main 09ec516ae50b fails 26 tests while 182 controls pass with identical final test bytes. The final candidate passes all 208 focused tests. Disabling only the causal recovery hook fails 11 tests while 197 controls pass."
|
||||
},
|
||||
"performanceBudget": {
|
||||
"required": true,
|
||||
"evidence": "Recovery adds one existing indexed active-Dispatch authority preflight, one bounded SQLite transaction, and one constant-size random token/hash update only when an exact worker explicitly requests the preamble. It adds no polling, timer, subprocess, provider scan, network fanout, renderer work, retained plaintext secret, or terminal write."
|
||||
},
|
||||
"promotionCriteria": [
|
||||
"Collect byte-identical baseline, candidate, and candidate-disabled evidence on the final merge base.",
|
||||
"Collect authenticated real-Hermes consumption evidence without a literal placeholder.",
|
||||
"Collect live SSH or paired-host evidence plus Linux, Windows, and WSL CI coverage.",
|
||||
"Collect 100 consecutive focused CI passes or 14 days of soak history."
|
||||
],
|
||||
"knownGaps": [
|
||||
"Hermes 0.15.2 stopped before prompt consumption because the isolated profile had no API key or provider; literal placeholder consumption is unverified and not claimed fixed.",
|
||||
"Streaming dispatcher coverage is not a live paired host, SSH session, or remote version-skew deployment.",
|
||||
"Mixed-binary recovery is refused for exact and pane-reminted assignees when an older runtime omits the additive recovery outcome; no old runtime can mint the new capability.",
|
||||
"Physical Linux, Windows, and WSL execution remains uncollected."
|
||||
],
|
||||
"demotionRule": "Keep experimental or demote if recovery can rotate for any caller other than the exact assigned pane and process, expose plaintext authority durably, write terminal input, invalidate authority without returning a replacement, or fail to settle once."
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,301 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const callMock = vi.fn()
|
||||
const getTerminalHandleMock = vi.hoisted(() => vi.fn())
|
||||
const originalTerminalHandle = process.env.ORCA_TERMINAL_HANDLE
|
||||
const originalPaneKey = process.env.ORCA_PANE_KEY
|
||||
const originalLaunchToken = process.env.ORCA_AGENT_LAUNCH_TOKEN
|
||||
|
||||
vi.mock('../format', () => ({ printResult: vi.fn() }))
|
||||
vi.mock('../selectors', () => ({ getTerminalHandle: getTerminalHandleMock }))
|
||||
|
||||
import { ORCHESTRATION_HANDLERS } from './orchestration'
|
||||
import { RuntimeClientError } from '../runtime-client'
|
||||
|
||||
function staleHandleError(): RuntimeClientError {
|
||||
return new RuntimeClientError('terminal_handle_stale', 'terminal_handle_stale')
|
||||
}
|
||||
|
||||
function stubStaleHandleRemint(liveHandle: string, downstream: unknown): void {
|
||||
callMock
|
||||
.mockRejectedValueOnce(staleHandleError())
|
||||
.mockResolvedValueOnce({ result: { terminal: { handle: liveHandle } } })
|
||||
.mockResolvedValueOnce(downstream)
|
||||
}
|
||||
|
||||
function stubStaleHandleRemintFailure(error: RuntimeClientError): void {
|
||||
callMock.mockRejectedValueOnce(staleHandleError()).mockRejectedValueOnce(error)
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
restoreEnv('ORCA_TERMINAL_HANDLE', originalTerminalHandle)
|
||||
restoreEnv('ORCA_PANE_KEY', originalPaneKey)
|
||||
restoreEnv('ORCA_AGENT_LAUNCH_TOKEN', originalLaunchToken)
|
||||
})
|
||||
|
||||
describe('orchestration dispatch coordinator handle', () => {
|
||||
beforeEach(() => {
|
||||
callMock.mockReset()
|
||||
getTerminalHandleMock.mockReset()
|
||||
delete process.env.ORCA_TERMINAL_HANDLE
|
||||
delete process.env.ORCA_PANE_KEY
|
||||
})
|
||||
|
||||
const invoke = (
|
||||
command: 'orchestration dispatch' | 'orchestration dispatch-show',
|
||||
flags: Map<string, string | boolean>
|
||||
) =>
|
||||
ORCHESTRATION_HANDLERS[command]({
|
||||
flags,
|
||||
client: { call: callMock },
|
||||
cwd: '/tmp/repo',
|
||||
json: true
|
||||
} as never)
|
||||
|
||||
it('remints a stale coordinator env handle from the caller pane key', async () => {
|
||||
process.env.ORCA_TERMINAL_HANDLE = 'term_stale_coord'
|
||||
process.env.ORCA_PANE_KEY = 'tab_coord:leaf_coord'
|
||||
stubStaleHandleRemint('term_live_coord', {
|
||||
result: { dispatch: { id: 'ctx_1', task_id: 'task_1', status: 'dispatched' } }
|
||||
})
|
||||
getTerminalHandleMock.mockRejectedValue(new Error('active terminal fallback is unsafe'))
|
||||
|
||||
await invoke(
|
||||
'orchestration dispatch',
|
||||
new Map<string, string | boolean>([
|
||||
['task', 'task_1'],
|
||||
['to', 'term_worker'],
|
||||
['inject', true]
|
||||
])
|
||||
)
|
||||
|
||||
expect(callMock).toHaveBeenNthCalledWith(1, 'terminal.show', {
|
||||
terminal: 'term_stale_coord'
|
||||
})
|
||||
expect(callMock).toHaveBeenNthCalledWith(2, 'terminal.resolvePane', {
|
||||
paneKey: 'tab_coord:leaf_coord'
|
||||
})
|
||||
expect(getTerminalHandleMock).not.toHaveBeenCalled()
|
||||
expect(callMock).toHaveBeenNthCalledWith(3, 'orchestration.dispatch', {
|
||||
task: 'task_1',
|
||||
to: 'term_worker',
|
||||
from: 'term_live_coord',
|
||||
inject: true,
|
||||
dryRun: undefined,
|
||||
returnPreamble: undefined,
|
||||
devMode: false
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects stale coordinator env handles when the caller pane cannot be proven', async () => {
|
||||
process.env.ORCA_TERMINAL_HANDLE = 'term_stale_coord'
|
||||
callMock.mockRejectedValueOnce(staleHandleError())
|
||||
getTerminalHandleMock.mockResolvedValue('term_wrong_active')
|
||||
|
||||
await expect(
|
||||
invoke(
|
||||
'orchestration dispatch',
|
||||
new Map<string, string | boolean>([
|
||||
['task', 'task_1'],
|
||||
['to', 'term_worker']
|
||||
])
|
||||
)
|
||||
).rejects.toMatchObject({ code: 'no_active_sender_terminal' })
|
||||
expect(callMock).toHaveBeenCalledTimes(1)
|
||||
expect(getTerminalHandleMock).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('propagates unexpected caller pane remint failures', async () => {
|
||||
process.env.ORCA_TERMINAL_HANDLE = 'term_stale_coord'
|
||||
process.env.ORCA_PANE_KEY = 'tab_coord:leaf_coord'
|
||||
stubStaleHandleRemintFailure(
|
||||
new RuntimeClientError('runtime_unavailable', 'runtime_unavailable')
|
||||
)
|
||||
|
||||
await expect(
|
||||
invoke('orchestration dispatch', new Map([['task', 'task_1']]))
|
||||
).rejects.toMatchObject({ code: 'runtime_unavailable' })
|
||||
expect(callMock).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
|
||||
it('requests secret-safe recovery with the reminted caller evidence', async () => {
|
||||
process.env.ORCA_TERMINAL_HANDLE = 'term_stale_coord'
|
||||
process.env.ORCA_PANE_KEY = 'tab_coord:leaf_coord'
|
||||
process.env.ORCA_AGENT_LAUNCH_TOKEN = 'launch-token'
|
||||
stubStaleHandleRemint('term_live_coord', {
|
||||
result: { dispatch: null, preamble: 'preamble' }
|
||||
})
|
||||
|
||||
await invoke(
|
||||
'orchestration dispatch-show',
|
||||
new Map<string, string | boolean>([
|
||||
['task', 'task_1'],
|
||||
['preamble', true]
|
||||
])
|
||||
)
|
||||
|
||||
expect(callMock).toHaveBeenNthCalledWith(
|
||||
3,
|
||||
'orchestration.dispatchShow',
|
||||
{
|
||||
task: 'task_1',
|
||||
preamble: true,
|
||||
recoverCapability: true,
|
||||
from: 'term_live_coord',
|
||||
devMode: false
|
||||
},
|
||||
{
|
||||
orchestrationCompatibilityEvidence: {
|
||||
terminalHandle: 'term_live_coord',
|
||||
paneKey: 'tab_coord:leaf_coord',
|
||||
launchToken: 'launch-token'
|
||||
}
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
it('rejects an old runtime tokenless response for an active assigned worker', async () => {
|
||||
process.env.ORCA_PANE_KEY = 'tab_worker:leaf_worker'
|
||||
process.env.ORCA_AGENT_LAUNCH_TOKEN = 'launch-token'
|
||||
callMock.mockResolvedValueOnce({
|
||||
result: {
|
||||
dispatch: {
|
||||
id: 'ctx_1',
|
||||
task_id: 'task_1',
|
||||
status: 'dispatched',
|
||||
assignee_handle: 'term_worker'
|
||||
},
|
||||
preamble: 'tokenless old-runtime preamble'
|
||||
}
|
||||
})
|
||||
|
||||
await expect(
|
||||
invoke(
|
||||
'orchestration dispatch-show',
|
||||
new Map<string, string | boolean>([
|
||||
['task', 'task_1'],
|
||||
['preamble', true],
|
||||
['from', 'term_worker']
|
||||
])
|
||||
)
|
||||
).rejects.toMatchObject({
|
||||
code: 'dispatch_capability_unavailable',
|
||||
message: expect.stringContaining('ask the coordinator to redispatch')
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps old-runtime coordinator preamble inspection tokenless', async () => {
|
||||
callMock.mockResolvedValueOnce({
|
||||
result: {
|
||||
dispatch: {
|
||||
id: 'ctx_1',
|
||||
task_id: 'task_1',
|
||||
status: 'dispatched',
|
||||
assignee_handle: 'term_worker'
|
||||
},
|
||||
preamble: 'tokenless old-runtime coordinator inspection'
|
||||
}
|
||||
})
|
||||
|
||||
await expect(
|
||||
invoke(
|
||||
'orchestration dispatch-show',
|
||||
new Map<string, string | boolean>([
|
||||
['task', 'task_1'],
|
||||
['preamble', true],
|
||||
['from', 'term_coordinator']
|
||||
])
|
||||
)
|
||||
).resolves.toBeUndefined()
|
||||
})
|
||||
|
||||
it.each([undefined, 'inspection'] as const)(
|
||||
'rejects a reminted worker tokenless response with recovery %s',
|
||||
async (recovery) => {
|
||||
process.env.ORCA_TERMINAL_HANDLE = 'term_stale_worker'
|
||||
process.env.ORCA_PANE_KEY = 'tab_worker:leaf_worker'
|
||||
process.env.ORCA_AGENT_LAUNCH_TOKEN = 'launch-token'
|
||||
stubStaleHandleRemint('term_live_worker', {
|
||||
result: {
|
||||
dispatch: {
|
||||
id: 'ctx_1',
|
||||
task_id: 'task_1',
|
||||
status: 'dispatched',
|
||||
assignee_handle: 'term_stale_worker'
|
||||
},
|
||||
preamble: 'tokenless reminted-worker preamble',
|
||||
...(recovery ? { recovery } : {})
|
||||
}
|
||||
})
|
||||
|
||||
await expect(
|
||||
invoke(
|
||||
'orchestration dispatch-show',
|
||||
new Map<string, string | boolean>([
|
||||
['task', 'task_1'],
|
||||
['preamble', true]
|
||||
])
|
||||
)
|
||||
).rejects.toMatchObject({ code: 'dispatch_capability_unavailable' })
|
||||
}
|
||||
)
|
||||
|
||||
it.each([undefined, 'inspection'] as const)(
|
||||
'rejects explicit reminted worker tokenless response with recovery %s',
|
||||
async (recovery) => {
|
||||
process.env.ORCA_PANE_KEY = 'tab_reminted:11111111-1111-4111-8111-111111111111'
|
||||
process.env.ORCA_AGENT_LAUNCH_TOKEN = 'launch-token'
|
||||
callMock.mockResolvedValueOnce({
|
||||
result: {
|
||||
dispatch: {
|
||||
id: 'ctx_1',
|
||||
task_id: 'task_1',
|
||||
status: 'dispatched',
|
||||
assignee_handle: 'term_stale_worker',
|
||||
assignee_pane_key: 'tab_original:11111111-1111-4111-8111-111111111111'
|
||||
},
|
||||
preamble: 'tokenless explicit-reminted-worker preamble',
|
||||
...(recovery ? { recovery } : {})
|
||||
}
|
||||
})
|
||||
|
||||
await expect(
|
||||
invoke(
|
||||
'orchestration dispatch-show',
|
||||
new Map<string, string | boolean>([
|
||||
['task', 'task_1'],
|
||||
['preamble', true],
|
||||
['from', 'term_live_worker']
|
||||
])
|
||||
)
|
||||
).rejects.toMatchObject({ code: 'dispatch_capability_unavailable' })
|
||||
}
|
||||
)
|
||||
|
||||
it('retires the legacy coordinator command without runtime effects', async () => {
|
||||
await expect(
|
||||
ORCHESTRATION_HANDLERS['orchestration coordinator-start']({
|
||||
flags: new Map([['spec', 'run the plan']]),
|
||||
client: { call: callMock },
|
||||
cwd: '/tmp/repo',
|
||||
json: true
|
||||
} as never)
|
||||
).rejects.toMatchObject({
|
||||
code: 'orchestration_migration_required',
|
||||
data: {
|
||||
reason: 'command_retired',
|
||||
effectsApplied: false,
|
||||
nextCommandArgs: ['skills', 'get', 'orchestration', '--full']
|
||||
}
|
||||
})
|
||||
expect(callMock).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
function restoreEnv(name: string, value: string | undefined): void {
|
||||
if (value === undefined) {
|
||||
delete process.env[name]
|
||||
} else {
|
||||
process.env[name] = value
|
||||
}
|
||||
}
|
||||
@@ -329,165 +329,6 @@ describe('orchestration send structured payload flags', () => {
|
||||
)
|
||||
})
|
||||
|
||||
describe('orchestration dispatch coordinator handle', () => {
|
||||
beforeEach(() => {
|
||||
callMock.mockReset()
|
||||
getTerminalHandleMock.mockReset()
|
||||
delete process.env.ORCA_TERMINAL_HANDLE
|
||||
delete process.env.ORCA_PANE_KEY
|
||||
})
|
||||
|
||||
const invokeDispatch = (flags: Map<string, string | boolean>) =>
|
||||
ORCHESTRATION_HANDLERS['orchestration dispatch']({
|
||||
flags,
|
||||
client: { call: callMock },
|
||||
cwd: '/tmp/repo',
|
||||
json: true
|
||||
} as never)
|
||||
|
||||
const invokeDispatchShow = (flags: Map<string, string | boolean>) =>
|
||||
ORCHESTRATION_HANDLERS['orchestration dispatch-show']({
|
||||
flags,
|
||||
client: { call: callMock },
|
||||
cwd: '/tmp/repo',
|
||||
json: true
|
||||
} as never)
|
||||
|
||||
const invokeRun = (flags: Map<string, string | boolean>) =>
|
||||
ORCHESTRATION_HANDLERS['orchestration coordinator-start']({
|
||||
flags,
|
||||
client: { call: callMock },
|
||||
cwd: '/tmp/repo',
|
||||
json: true
|
||||
} as never)
|
||||
|
||||
it('remints a stale coordinator env handle from the caller pane key', async () => {
|
||||
process.env.ORCA_TERMINAL_HANDLE = 'term_stale_coord'
|
||||
process.env.ORCA_PANE_KEY = 'tab_coord:leaf_coord'
|
||||
stubStaleHandleRemint('term_live_coord', {
|
||||
result: { dispatch: { id: 'ctx_1', task_id: 'task_1', status: 'dispatched' } }
|
||||
})
|
||||
getTerminalHandleMock.mockRejectedValue(new Error('active terminal fallback is unsafe'))
|
||||
|
||||
await invokeDispatch(
|
||||
new Map<string, string | boolean>([
|
||||
['task', 'task_1'],
|
||||
['to', 'term_worker'],
|
||||
['inject', true]
|
||||
])
|
||||
)
|
||||
|
||||
expect(callMock).toHaveBeenNthCalledWith(1, 'terminal.show', {
|
||||
terminal: 'term_stale_coord'
|
||||
})
|
||||
expect(callMock).toHaveBeenNthCalledWith(2, 'terminal.resolvePane', {
|
||||
paneKey: 'tab_coord:leaf_coord'
|
||||
})
|
||||
expect(getTerminalHandleMock).not.toHaveBeenCalled()
|
||||
expect(callMock).toHaveBeenNthCalledWith(3, 'orchestration.dispatch', {
|
||||
task: 'task_1',
|
||||
to: 'term_worker',
|
||||
from: 'term_live_coord',
|
||||
inject: true,
|
||||
dryRun: undefined,
|
||||
returnPreamble: undefined,
|
||||
devMode: false
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects stale coordinator env handles when the caller pane cannot be proven', async () => {
|
||||
process.env.ORCA_TERMINAL_HANDLE = 'term_stale_coord'
|
||||
callMock.mockRejectedValueOnce(staleHandleError())
|
||||
getTerminalHandleMock.mockResolvedValue('term_wrong_active')
|
||||
|
||||
await expect(
|
||||
invokeDispatch(
|
||||
new Map<string, string | boolean>([
|
||||
['task', 'task_1'],
|
||||
['to', 'term_worker']
|
||||
])
|
||||
)
|
||||
).rejects.toMatchObject({
|
||||
code: 'no_active_sender_terminal'
|
||||
})
|
||||
|
||||
expect(callMock).toHaveBeenCalledTimes(1)
|
||||
expect(getTerminalHandleMock).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('propagates unexpected caller pane remint failures for coordinator commands', async () => {
|
||||
process.env.ORCA_TERMINAL_HANDLE = 'term_stale_coord'
|
||||
process.env.ORCA_PANE_KEY = 'tab_coord:leaf_coord'
|
||||
stubStaleHandleRemintFailure(
|
||||
new RuntimeClientError('runtime_unavailable', 'runtime_unavailable')
|
||||
)
|
||||
getTerminalHandleMock.mockResolvedValue('term_wrong_active')
|
||||
|
||||
await expect(
|
||||
invokeDispatch(
|
||||
new Map<string, string | boolean>([
|
||||
['task', 'task_1'],
|
||||
['to', 'term_worker']
|
||||
])
|
||||
)
|
||||
).rejects.toMatchObject({
|
||||
code: 'runtime_unavailable'
|
||||
})
|
||||
|
||||
expect(callMock).toHaveBeenNthCalledWith(1, 'terminal.show', {
|
||||
terminal: 'term_stale_coord'
|
||||
})
|
||||
expect(callMock).toHaveBeenNthCalledWith(2, 'terminal.resolvePane', {
|
||||
paneKey: 'tab_coord:leaf_coord'
|
||||
})
|
||||
expect(callMock).toHaveBeenCalledTimes(2)
|
||||
expect(getTerminalHandleMock).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('uses a live coordinator handle for dispatch-show preamble previews', async () => {
|
||||
process.env.ORCA_TERMINAL_HANDLE = 'term_stale_coord'
|
||||
process.env.ORCA_PANE_KEY = 'tab_coord:leaf_coord'
|
||||
stubStaleHandleRemint('term_live_coord', {
|
||||
result: { dispatch: null, preamble: 'preamble' }
|
||||
})
|
||||
getTerminalHandleMock.mockRejectedValue(new Error('active terminal fallback is unsafe'))
|
||||
|
||||
await invokeDispatchShow(
|
||||
new Map<string, string | boolean>([
|
||||
['task', 'task_1'],
|
||||
['preamble', true]
|
||||
])
|
||||
)
|
||||
|
||||
expect(callMock).toHaveBeenNthCalledWith(1, 'terminal.show', {
|
||||
terminal: 'term_stale_coord'
|
||||
})
|
||||
expect(callMock).toHaveBeenNthCalledWith(2, 'terminal.resolvePane', {
|
||||
paneKey: 'tab_coord:leaf_coord'
|
||||
})
|
||||
expect(callMock).toHaveBeenNthCalledWith(3, 'orchestration.dispatchShow', {
|
||||
task: 'task_1',
|
||||
preamble: true,
|
||||
from: 'term_live_coord',
|
||||
devMode: false
|
||||
})
|
||||
})
|
||||
|
||||
it('retires the legacy coordinator command without runtime effects', async () => {
|
||||
await expect(
|
||||
invokeRun(new Map<string, string | boolean>([['spec', 'run the plan']]))
|
||||
).rejects.toMatchObject({
|
||||
code: 'orchestration_migration_required',
|
||||
data: {
|
||||
reason: 'command_retired',
|
||||
effectsApplied: false,
|
||||
nextCommandArgs: ['skills', 'get', 'orchestration', '--full']
|
||||
}
|
||||
})
|
||||
expect(callMock).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
describe('orchestration task-create caller handle', () => {
|
||||
beforeEach(() => {
|
||||
callMock.mockReset()
|
||||
|
||||
@@ -24,6 +24,9 @@ import type { NativeChatMessage } from '../../shared/native-chat-types'
|
||||
import type { RuntimeStatus, RuntimeTerminalRead } from '../../shared/runtime-types'
|
||||
import { ORCHESTRATION_WORKER_LAUNCH_PREFERENCES_RUNTIME_CAPABILITY } from '../../shared/protocol-version'
|
||||
import { orchestrationMigrationData } from '../../shared/orchestration-rpc-contract'
|
||||
import { readOrchestrationCompatibilityEvidence } from '../../shared/orchestration-compatibility-evidence'
|
||||
import type { RuntimeOrchestrationEnvelope } from '../../shared/runtime-rpc-envelope'
|
||||
import { isEquivalentPaneKey } from '../../shared/stable-pane-id'
|
||||
import { ORCHESTRATION_RUN_PAGE_LIMIT } from '../../shared/orchestration-run-pagination'
|
||||
import {
|
||||
formatMessageReadOnlyTag,
|
||||
@@ -391,7 +394,7 @@ function callMutation<TResult>(
|
||||
flags: Map<string, string | boolean>,
|
||||
method: string,
|
||||
params: unknown,
|
||||
options?: { timeoutMs?: number; orchestrationCapability?: string }
|
||||
options?: { timeoutMs?: number } & RuntimeOrchestrationEnvelope
|
||||
) {
|
||||
const requestId = getOptionalStringFlag(flags, 'retry-request')
|
||||
if (!requestId) {
|
||||
@@ -1180,19 +1183,65 @@ export const ORCHESTRATION_HANDLERS: Record<string, CommandHandler> = {
|
||||
|
||||
'orchestration dispatch-show': async ({ flags, client, cwd, json }) => {
|
||||
const showPreamble = flags.has('preamble') ? true : undefined
|
||||
const claimedFrom = showPreamble
|
||||
? (getOptionalStringFlag(flags, 'from') ?? process.env.ORCA_TERMINAL_HANDLE)?.trim()
|
||||
: undefined
|
||||
// Why: resolve --from so the previewed preamble embeds a real coordinator handle like an actual dispatch.
|
||||
const from = showPreamble
|
||||
? await resolveCoordinatorTerminalHandle(flags, cwd, client)
|
||||
: undefined
|
||||
const result = await client.call<{
|
||||
dispatch: { id: string; task_id: string; status: string } | null
|
||||
const compatibilityEvidence = from
|
||||
? readOrchestrationCompatibilityEvidence({
|
||||
...process.env,
|
||||
ORCA_TERMINAL_HANDLE: from
|
||||
})
|
||||
: undefined
|
||||
const result = await callMutation<{
|
||||
dispatch: {
|
||||
id: string
|
||||
task_id: string
|
||||
status: string
|
||||
assignee_handle?: string | null
|
||||
assignee_pane_key?: string | null
|
||||
} | null
|
||||
preamble?: string
|
||||
}>('orchestration.dispatchShow', {
|
||||
task: getRequiredStringFlag(flags, 'task'),
|
||||
preamble: showPreamble,
|
||||
from,
|
||||
devMode: isDevCliInvocation()
|
||||
})
|
||||
recovery?: 'recovered' | 'unavailable' | 'inspection' | 'not_requested'
|
||||
}>(
|
||||
client,
|
||||
flags,
|
||||
'orchestration.dispatchShow',
|
||||
{
|
||||
task: getRequiredStringFlag(flags, 'task'),
|
||||
preamble: showPreamble,
|
||||
recoverCapability: showPreamble,
|
||||
from,
|
||||
devMode: isDevCliInvocation()
|
||||
},
|
||||
compatibilityEvidence
|
||||
? { orchestrationCompatibilityEvidence: compatibilityEvidence }
|
||||
: undefined
|
||||
)
|
||||
if (
|
||||
showPreamble &&
|
||||
result.result.dispatch?.status === 'dispatched' &&
|
||||
result.result.recovery !== 'recovered' &&
|
||||
(result.result.recovery === 'unavailable' ||
|
||||
from === result.result.dispatch.assignee_handle ||
|
||||
claimedFrom === result.result.dispatch.assignee_handle ||
|
||||
Boolean(
|
||||
compatibilityEvidence?.paneKey &&
|
||||
result.result.dispatch.assignee_pane_key &&
|
||||
isEquivalentPaneKey(
|
||||
compatibilityEvidence.paneKey,
|
||||
result.result.dispatch.assignee_pane_key
|
||||
)
|
||||
))
|
||||
) {
|
||||
throw new RuntimeClientError(
|
||||
'dispatch_capability_unavailable',
|
||||
'The active Dispatch preamble did not include usable lifecycle authority because this worker identity could not be verified. Run this command inside the exact assigned worker on a current Orca runtime; if it still fails there, ask the coordinator to redispatch the Task to a current process.'
|
||||
)
|
||||
}
|
||||
printResult(result, json, (r) => {
|
||||
if (r.preamble && showPreamble) {
|
||||
return r.preamble
|
||||
|
||||
@@ -176,6 +176,58 @@ describe.skipIf(process.platform === 'win32')('RuntimeClient', () => {
|
||||
expect(requests[0]?.method).toBe('status.get')
|
||||
})
|
||||
|
||||
it('sends additive Dispatch recovery requests to an old local runtime', async () => {
|
||||
const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-client-'))
|
||||
const endpoint = join(userDataPath, 'runtime.sock')
|
||||
const requests: Record<string, unknown>[] = []
|
||||
const server = createServer((socket) => {
|
||||
sockets.add(socket)
|
||||
socket.once('close', () => sockets.delete(socket))
|
||||
socket.once('data', (data) => {
|
||||
const request = JSON.parse(String(data).trim()) as Record<string, unknown>
|
||||
requests.push(request)
|
||||
socket.write(
|
||||
`${JSON.stringify({
|
||||
id: request.id,
|
||||
ok: true,
|
||||
result: {
|
||||
dispatch: {
|
||||
id: 'ctx_1',
|
||||
task_id: 'task_1',
|
||||
status: 'dispatched',
|
||||
assignee_handle: 'term_worker'
|
||||
},
|
||||
preamble: 'tokenless old-runtime preamble'
|
||||
},
|
||||
_meta: { runtimeId: 'runtime-1' }
|
||||
})}\n`
|
||||
)
|
||||
})
|
||||
})
|
||||
servers.add(server)
|
||||
await new Promise<void>((resolve) => server.listen(endpoint, resolve))
|
||||
writeMetadata(userDataPath, endpoint)
|
||||
|
||||
const client = new RuntimeClient(userDataPath, 500)
|
||||
for (const from of ['term_coordinator', 'term_worker']) {
|
||||
await client.call('orchestration.dispatchShow', {
|
||||
task: 'task_1',
|
||||
preamble: true,
|
||||
recoverCapability: true,
|
||||
from
|
||||
})
|
||||
}
|
||||
|
||||
expect(requests).toHaveLength(2)
|
||||
expect(requests.map((request) => request.method)).toEqual([
|
||||
'orchestration.dispatchShow',
|
||||
'orchestration.dispatchShow'
|
||||
])
|
||||
expect(requests.every((request) => typeof request.orchestrationRequestId === 'string')).toBe(
|
||||
true
|
||||
)
|
||||
})
|
||||
|
||||
it('returns the full RPC envelope for successful calls', async () => {
|
||||
const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-client-'))
|
||||
const endpoint = join(userDataPath, 'runtime.sock')
|
||||
|
||||
@@ -73,7 +73,7 @@ export class RuntimeClient {
|
||||
): Promise<RuntimeRpcSuccess<TResult>> {
|
||||
const effectiveTimeoutMs = options?.timeoutMs ?? this.resolveMethodTimeoutMs(method, params)
|
||||
const orchestrationMutation = isOrchestrationMutation(method, params)
|
||||
if (orchestrationMutation) {
|
||||
if (orchestrationMutation && !isAdditiveDispatchRecovery(method, params)) {
|
||||
await this.ensureOrchestrationContractCompatible(effectiveTimeoutMs)
|
||||
}
|
||||
const orchestrationRequestId = orchestrationMutation
|
||||
@@ -82,6 +82,11 @@ export class RuntimeClient {
|
||||
const compatibilityEnvelope = method.startsWith('orchestration.')
|
||||
? {
|
||||
...this.orchestrationCompatibility,
|
||||
...(options?.orchestrationCompatibilityEvidence
|
||||
? {
|
||||
orchestrationCompatibilityEvidence: options.orchestrationCompatibilityEvidence
|
||||
}
|
||||
: {}),
|
||||
compatibilityInvocationId:
|
||||
orchestrationRequestId ?? this.orchestrationCompatibility.compatibilityInvocationId
|
||||
}
|
||||
@@ -247,6 +252,15 @@ export class RuntimeClient {
|
||||
}
|
||||
}
|
||||
|
||||
function isAdditiveDispatchRecovery(method: string, params: unknown): boolean {
|
||||
return (
|
||||
method === 'orchestration.dispatchShow' &&
|
||||
typeof params === 'object' &&
|
||||
params !== null &&
|
||||
(params as { recoverCapability?: unknown }).recoverCapability === true
|
||||
)
|
||||
}
|
||||
|
||||
function attachMutationRecovery(error: unknown, requestId: string | undefined): unknown {
|
||||
if (!requestId || !(error instanceof RuntimeClientError)) {
|
||||
return error
|
||||
|
||||
@@ -179,7 +179,10 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [
|
||||
summary: 'Show dispatch context for a task',
|
||||
usage:
|
||||
'orca orchestration dispatch-show --task <task_id> [--preamble] [--from <handle>] [--json]',
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'task', 'preamble', 'from']
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'task', 'preamble', 'from', 'retry-request'],
|
||||
notes: [
|
||||
'Inside the exact active worker, --preamble rotates lifecycle authority and invalidates the prior capability. Coordinator inspection and inspection of completed or legacy Dispatches remain tokenless.'
|
||||
]
|
||||
},
|
||||
{
|
||||
path: ['orchestration', 'ask'],
|
||||
|
||||
@@ -39,7 +39,7 @@ import type {
|
||||
} from './types'
|
||||
import { buildOrchestrationTaskDisplayMetadata } from '../../../shared/orchestration-task-display'
|
||||
import { ORCHESTRATION_LEGACY_RUN_ID } from '../../../shared/orchestration-rpc-contract'
|
||||
import { parsePaneKey } from '../../../shared/stable-pane-id'
|
||||
import { isEquivalentPaneKey } from '../../../shared/stable-pane-id'
|
||||
import { OrchestrationError } from './orchestration-error'
|
||||
import { resolveOrchestrationMigrationStartVersion } from './orchestration-schema-version-skew'
|
||||
import {
|
||||
@@ -63,16 +63,6 @@ import {
|
||||
migrateMutationReceiptCapacity
|
||||
} from './mutation-receipt-capacity'
|
||||
|
||||
// Why: leaf UUID is the remint-stable pane identity (tab half changes on break-out); exact match covers legacy/unparseable keys.
|
||||
function isEquivalentPaneKey(a: string, b: string): boolean {
|
||||
if (a === b) {
|
||||
return true
|
||||
}
|
||||
const aLeaf = parsePaneKey(a)?.leafId
|
||||
const bLeaf = parsePaneKey(b)?.leafId
|
||||
return Boolean(aLeaf && bLeaf && aLeaf === bLeaf)
|
||||
}
|
||||
|
||||
function parseWorkerTerminalPriorOwnerIds(value: string): string[] | null {
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(value)
|
||||
@@ -6416,6 +6406,55 @@ export class OrchestrationDb {
|
||||
return capability
|
||||
}
|
||||
|
||||
recoverDispatchCapability(params: {
|
||||
dispatchId: string
|
||||
callerPaneKey: string
|
||||
callerProcessIncarnation: string
|
||||
}): string {
|
||||
this.db.exec('BEGIN IMMEDIATE')
|
||||
try {
|
||||
const dispatch = this.getDispatchContextById(params.dispatchId)
|
||||
const task = dispatch ? this.getTask(dispatch.task_id) : undefined
|
||||
const worker = dispatch ? this.getWorkerDispatch(dispatch.id) : undefined
|
||||
if (
|
||||
!dispatch ||
|
||||
dispatch.contract_version !== CURRENT_CONTRACT_VERSION ||
|
||||
dispatch.status !== 'dispatched' ||
|
||||
dispatch.capability_revoked_at !== null ||
|
||||
task?.status !== 'dispatched' ||
|
||||
(worker !== undefined && worker.state !== 'ready')
|
||||
) {
|
||||
throw new OrchestrationError(
|
||||
'dispatch_inactive',
|
||||
`Dispatch ${params.dispatchId} is not an active current-contract Dispatch.`
|
||||
)
|
||||
}
|
||||
const assignedWorker =
|
||||
dispatch.assignee_pane_key !== null &&
|
||||
isEquivalentPaneKey(dispatch.assignee_pane_key, params.callerPaneKey) &&
|
||||
dispatch.process_incarnation === params.callerProcessIncarnation
|
||||
if (!assignedWorker) {
|
||||
throw new OrchestrationError(
|
||||
'dispatch_capability_invalid',
|
||||
`Caller does not own lifecycle recovery for Dispatch ${params.dispatchId}.`
|
||||
)
|
||||
}
|
||||
const capability = `dcap_${randomBytes(32).toString('base64url')}`
|
||||
this.db
|
||||
.prepare(
|
||||
`UPDATE dispatch_contexts
|
||||
SET capability_hash = ?
|
||||
WHERE id = ?`
|
||||
)
|
||||
.run(hashDispatchCapability(capability), params.dispatchId)
|
||||
this.db.exec('COMMIT')
|
||||
return capability
|
||||
} catch (error) {
|
||||
this.db.exec('ROLLBACK')
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
verifyDispatchCapability(params: {
|
||||
dispatchId: string
|
||||
capability: string | undefined
|
||||
|
||||
@@ -2241,6 +2241,62 @@ describe('orchestration RPC methods', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('accepts one complete supervised Hermes prompt write for the exact process', async () => {
|
||||
setup()
|
||||
mockCurrentWorkerStart()
|
||||
const rawSend = vi.spyOn(runtime, 'sendTerminal')
|
||||
const taskSpec = 'HERMES_TASK_MARKER implement the lifecycle contract'
|
||||
const task = db.createTask({ spec: taskSpec })
|
||||
|
||||
const started = (await call('orchestration.workerStart', {
|
||||
task: task.id,
|
||||
from: 'term_coord',
|
||||
worktree: 'id:repo::worktree',
|
||||
agent: 'hermes'
|
||||
})) as { dispatchId: string; state: string }
|
||||
const prompt = vi.mocked(runtime.sendTerminalAgentPrompt).mock.calls[0]?.[1] ?? ''
|
||||
const capability = prompt.match(/--dispatch-capability (dcap_[A-Za-z0-9_-]+)/)?.[1]
|
||||
|
||||
expect(started.state).toBe('ready')
|
||||
expect(prompt).toContain(`=== TASK ===\n${taskSpec}`)
|
||||
expect(prompt).toContain(`--task-id ${task.id} --dispatch-id ${started.dispatchId}`)
|
||||
expect(prompt).not.toContain('[[…]]')
|
||||
expect(capability).toBeDefined()
|
||||
expect(runtime.createTerminal).toHaveBeenCalledWith(
|
||||
'id:repo::worktree',
|
||||
expect.objectContaining({ startupAgent: 'hermes' })
|
||||
)
|
||||
expect(runtime.sendTerminalAgentPrompt).toHaveBeenCalledOnce()
|
||||
expect(runtime.sendTerminalAgentPrompt).toHaveBeenCalledWith('term_worker', prompt)
|
||||
expect(rawSend).not.toHaveBeenCalled()
|
||||
expect(db.getDispatchContextById(started.dispatchId)).toMatchObject({
|
||||
assignee_handle: 'term_worker',
|
||||
assignee_pane_key: 'tab_worker:leaf_worker',
|
||||
process_incarnation: 'runtime_test:term_worker:1',
|
||||
status: 'dispatched'
|
||||
})
|
||||
|
||||
ctx = { runtime, orchestrationCapability: capability }
|
||||
const settled = (await call('orchestration.send', {
|
||||
from: 'term_worker',
|
||||
type: 'worker_done',
|
||||
subject: 'Hermes complete',
|
||||
body: 'Implemented the contract. Verified the lifecycle. Nothing remains.',
|
||||
payload: JSON.stringify({
|
||||
taskId: task.id,
|
||||
dispatchId: started.dispatchId,
|
||||
outcome: 'succeeded'
|
||||
})
|
||||
})) as { lifecycle: { action: string } }
|
||||
|
||||
expect(settled.lifecycle.action).toBe('completed')
|
||||
expect(db.getTask(task.id)?.status).toBe('completed')
|
||||
expect(db.getDispatchContextById(started.dispatchId)).toMatchObject({
|
||||
status: 'completed',
|
||||
capability_revoked_at: expect.any(String)
|
||||
})
|
||||
})
|
||||
|
||||
it('applies and reports opaque per-invocation model preferences', async () => {
|
||||
setup()
|
||||
mockCurrentWorkerStart()
|
||||
@@ -2650,6 +2706,87 @@ describe('orchestration RPC methods', () => {
|
||||
expect(result.dispatch?.task_id).toBe(task.id)
|
||||
})
|
||||
|
||||
it('recovers an exact worker lifecycle command that settles once', async () => {
|
||||
setup()
|
||||
const workerPaneKey = 'tab_worker:leaf_worker'
|
||||
vi.mocked(runtime.getTerminalPaneKey).mockImplementation((handle) =>
|
||||
handle === 'term_worker' ? workerPaneKey : coordinatorPaneKey
|
||||
)
|
||||
const task = db.createTask({ spec: 'RECOVERY_TASK_MARKER finish the assigned work' })
|
||||
const dispatch = db.createDispatchContext(
|
||||
task.id,
|
||||
'term_worker',
|
||||
workerPaneKey,
|
||||
undefined,
|
||||
'runtime_test:term_worker:1'
|
||||
)
|
||||
db.mintDispatchCapability({
|
||||
dispatchId: dispatch.id,
|
||||
paneKey: workerPaneKey,
|
||||
processIncarnation: 'runtime_test:term_worker:1'
|
||||
})
|
||||
const promptSend = vi.spyOn(runtime, 'sendTerminalAgentPrompt')
|
||||
const rawSend = vi.spyOn(runtime, 'sendTerminal')
|
||||
vi.spyOn(runtime, 'verifyOrchestrationCompatibilityCaller').mockReturnValue({
|
||||
hostScope: { kind: 'local', hostId: 'local' },
|
||||
terminalHandle: 'term_worker',
|
||||
paneKey: workerPaneKey,
|
||||
processIncarnation: 'runtime_test:term_worker:1',
|
||||
launchTokenHash: 'worker-launch-token-hash'
|
||||
})
|
||||
ctx = {
|
||||
runtime,
|
||||
orchestrationCompatibilityEvidence: {
|
||||
terminalHandle: 'term_worker',
|
||||
paneKey: workerPaneKey,
|
||||
launchToken: 'worker-launch-token'
|
||||
},
|
||||
orchestrationCompatibilityCallerAuthority: {
|
||||
hostScope: { kind: 'local', hostId: 'local' },
|
||||
terminalHandle: 'term_worker',
|
||||
paneKey: workerPaneKey,
|
||||
processIncarnation: 'runtime_test:term_worker:1',
|
||||
launchTokenHash: 'worker-launch-token-hash'
|
||||
}
|
||||
}
|
||||
|
||||
const recovered = (await call('orchestration.dispatchShow', {
|
||||
task: task.id,
|
||||
preamble: true,
|
||||
recoverCapability: true,
|
||||
from: 'term_worker'
|
||||
})) as { preamble: string }
|
||||
const capability = recovered.preamble.match(
|
||||
/--dispatch-capability (dcap_[A-Za-z0-9_-]+)/
|
||||
)?.[1]
|
||||
|
||||
expect(recovered.preamble).toContain('RECOVERY_TASK_MARKER finish the assigned work')
|
||||
expect(recovered.preamble).toContain(`--dispatch-id ${dispatch.id}`)
|
||||
expect(capability).toBeDefined()
|
||||
expect(promptSend).not.toHaveBeenCalled()
|
||||
expect(rawSend).not.toHaveBeenCalled()
|
||||
|
||||
ctx = { runtime, orchestrationCapability: capability }
|
||||
const settled = (await call('orchestration.send', {
|
||||
from: 'term_worker',
|
||||
type: 'worker_done',
|
||||
subject: 'Recovered completion',
|
||||
body: 'Recovered authority. Finished the work. Nothing remains.',
|
||||
payload: JSON.stringify({
|
||||
taskId: task.id,
|
||||
dispatchId: dispatch.id,
|
||||
outcome: 'succeeded'
|
||||
})
|
||||
})) as { lifecycle: { action: string } }
|
||||
|
||||
expect(settled.lifecycle.action).toBe('completed')
|
||||
expect(db.getTask(task.id)?.status).toBe('completed')
|
||||
expect(db.getDispatchContextById(dispatch.id)).toMatchObject({
|
||||
status: 'completed',
|
||||
capability_revoked_at: expect.any(String)
|
||||
})
|
||||
})
|
||||
|
||||
it('--preamble works when no dispatch exists yet', async () => {
|
||||
setup()
|
||||
const task = db.createTask({ spec: 'build feature' })
|
||||
|
||||
@@ -211,6 +211,7 @@ const DispatchParams = z.object({
|
||||
const DispatchShowParams = z.object({
|
||||
task: OptionalString,
|
||||
preamble: OptionalBoolean,
|
||||
recoverCapability: OptionalBoolean,
|
||||
from: OptionalString,
|
||||
devMode: OptionalBoolean
|
||||
})
|
||||
@@ -1358,7 +1359,7 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
|
||||
defineMethod({
|
||||
name: 'orchestration.dispatchShow',
|
||||
params: DispatchShowParams,
|
||||
handler: (params, { runtime }) => {
|
||||
handler: (params, { runtime, orchestrationCompatibilityCallerAuthority }) => {
|
||||
const db = runtime.getOrchestrationDb()
|
||||
if (!params.task) {
|
||||
throw new Error('Missing --task')
|
||||
@@ -1371,18 +1372,55 @@ export const ORCHESTRATION_METHODS: RpcMethod[] = [
|
||||
if (!task) {
|
||||
throw new Error(`Task not found: ${params.task}`)
|
||||
}
|
||||
const workerHandle = ctx?.assignee_handle ?? 'worker'
|
||||
const canRecover = Boolean(
|
||||
params.recoverCapability &&
|
||||
ctx &&
|
||||
orchestrationCompatibilityCallerAuthority &&
|
||||
db.isDispatchProcessCurrent({
|
||||
dispatchId: ctx.id,
|
||||
paneKey: orchestrationCompatibilityCallerAuthority.paneKey,
|
||||
processIncarnation: orchestrationCompatibilityCallerAuthority.processIncarnation
|
||||
})
|
||||
)
|
||||
const workerHandle =
|
||||
canRecover && orchestrationCompatibilityCallerAuthority
|
||||
? orchestrationCompatibilityCallerAuthority.terminalHandle
|
||||
: (ctx?.assignee_handle ?? 'worker')
|
||||
const coordinatorHandle =
|
||||
(ctx ? db.getRun(ctx.run_id)?.coordinator_handle : undefined) ??
|
||||
params.from ??
|
||||
'coordinator'
|
||||
const cliCommand = ctx
|
||||
? runtime.getTerminalOrchestrationCliCommand(workerHandle)
|
||||
: undefined
|
||||
const dispatchCapability =
|
||||
canRecover && ctx && orchestrationCompatibilityCallerAuthority
|
||||
? db.recoverDispatchCapability({
|
||||
dispatchId: ctx.id,
|
||||
callerPaneKey: orchestrationCompatibilityCallerAuthority.paneKey,
|
||||
callerProcessIncarnation:
|
||||
orchestrationCompatibilityCallerAuthority.processIncarnation
|
||||
})
|
||||
: undefined
|
||||
const recovery = params.recoverCapability
|
||||
? dispatchCapability
|
||||
? 'recovered'
|
||||
: ctx?.status === 'dispatched' && params.from === ctx.assignee_handle
|
||||
? 'unavailable'
|
||||
: 'inspection'
|
||||
: 'not_requested'
|
||||
const preamble = buildDispatchPreamble({
|
||||
taskId: task.id,
|
||||
// Why: use the real ctx.id when present so the preview matches what was injected; placeholder when no dispatch has occurred yet.
|
||||
dispatchId: ctx?.id ?? 'ctx_preview',
|
||||
taskSpec: task.spec,
|
||||
coordinatorHandle: params.from ?? 'coordinator',
|
||||
coordinatorHandle,
|
||||
workerHandle,
|
||||
dispatchCapability,
|
||||
devMode: params.devMode,
|
||||
...(ctx ? { cliCommand: runtime.getTerminalOrchestrationCliCommand(workerHandle) } : {})
|
||||
...(cliCommand ? { cliCommand } : {})
|
||||
})
|
||||
return { dispatch: ctx ?? null, preamble }
|
||||
return { dispatch: ctx ?? null, preamble, recovery }
|
||||
}
|
||||
|
||||
return { dispatch: ctx ?? null }
|
||||
|
||||
@@ -0,0 +1,752 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { OrchestrationDb } from '../orchestration/db'
|
||||
import { OrcaRuntimeService } from '../orca-runtime'
|
||||
import { RpcDispatcher } from './dispatcher'
|
||||
import { ORCHESTRATION_METHODS } from './methods/orchestration'
|
||||
import {
|
||||
cleanupLegacyCompatibilityDispatcherHarnesses,
|
||||
createHarness,
|
||||
currentEvidence,
|
||||
CURRENT_COORDINATOR_HANDLE,
|
||||
CURRENT_COORDINATOR_PANE,
|
||||
CURRENT_WORKER_HANDLE,
|
||||
CURRENT_WORKER_PANE,
|
||||
evidence,
|
||||
invoke,
|
||||
request,
|
||||
WORKER_HANDLE
|
||||
} from './orchestration-legacy-compatibility-dispatcher-test-fixture'
|
||||
|
||||
const freshDatabases: OrchestrationDb[] = []
|
||||
|
||||
afterEach(() => {
|
||||
cleanupLegacyCompatibilityDispatcherHarnesses()
|
||||
for (const db of freshDatabases.splice(0)) {
|
||||
db.close()
|
||||
}
|
||||
})
|
||||
|
||||
describe('Dispatch preamble capability recovery', () => {
|
||||
it.each(['dispatch', 'websocket'] as const)(
|
||||
'%s lets the exact assigned worker recover authority that settles once',
|
||||
async (transport) => {
|
||||
const harness = createHarness()
|
||||
const { taskId, dispatchId, originalCapability } = createCurrentDispatch(harness)
|
||||
const promptSend = vi.spyOn(harness.runtime, 'sendTerminalAgentPrompt')
|
||||
const rawSend = vi.spyOn(harness.runtime, 'sendTerminal')
|
||||
|
||||
const recovered = await invoke(
|
||||
harness.dispatcher,
|
||||
request(
|
||||
'orchestration.dispatchShow',
|
||||
{
|
||||
task: taskId,
|
||||
preamble: true,
|
||||
recoverCapability: true,
|
||||
from: CURRENT_WORKER_HANDLE
|
||||
},
|
||||
currentEvidence('worker'),
|
||||
`recover-${transport}`
|
||||
),
|
||||
transport
|
||||
)
|
||||
expect(recovered).toMatchObject({ ok: true, result: { dispatch: { id: dispatchId } } })
|
||||
const preamble = (recovered as { result: { preamble: string } }).result.preamble
|
||||
const capability = extractCapability(preamble)
|
||||
expect(preamble).toContain(`--from ${CURRENT_WORKER_HANDLE}`)
|
||||
expect(preamble).toContain(`coordinator's terminal handle is: ${CURRENT_COORDINATOR_HANDLE}`)
|
||||
expect(capability).toMatch(/^dcap_/)
|
||||
expect(capability).not.toBe(originalCapability)
|
||||
expect(promptSend).not.toHaveBeenCalled()
|
||||
expect(rawSend).not.toHaveBeenCalled()
|
||||
|
||||
const completed = await harness.dispatcher.dispatch({
|
||||
...request(
|
||||
'orchestration.send',
|
||||
{
|
||||
from: CURRENT_WORKER_HANDLE,
|
||||
type: 'worker_done',
|
||||
subject: 'Recovered completion',
|
||||
body: 'Recovered authority. Finished the work. Nothing remains.',
|
||||
payload: JSON.stringify({ taskId, dispatchId, outcome: 'succeeded' })
|
||||
},
|
||||
currentEvidence('worker'),
|
||||
`complete-${transport}`
|
||||
),
|
||||
orchestrationCapability: capability
|
||||
})
|
||||
expect(completed).toMatchObject({
|
||||
ok: true,
|
||||
result: { lifecycle: { action: 'completed' } }
|
||||
})
|
||||
expect(harness.db.getTask(taskId)?.status).toBe('completed')
|
||||
expect(harness.db.getDispatchContextById(dispatchId)).toMatchObject({
|
||||
status: 'completed',
|
||||
capability_revoked_at: expect.any(String)
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
it('repeated recovery invalidates only the prior recovered capability', async () => {
|
||||
const harness = createHarness()
|
||||
const { taskId, dispatchId, originalCapability } = createCurrentDispatch(harness)
|
||||
|
||||
const first = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.dispatchShow',
|
||||
{
|
||||
task: taskId,
|
||||
preamble: true,
|
||||
recoverCapability: true,
|
||||
from: CURRENT_WORKER_HANDLE
|
||||
},
|
||||
currentEvidence('worker'),
|
||||
'first-worker-recovery'
|
||||
)
|
||||
)
|
||||
const firstCapability = extractCapability(
|
||||
(first as { result: { preamble: string } }).result.preamble
|
||||
)
|
||||
const second = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.dispatchShow',
|
||||
{
|
||||
task: taskId,
|
||||
preamble: true,
|
||||
recoverCapability: true,
|
||||
from: CURRENT_WORKER_HANDLE
|
||||
},
|
||||
currentEvidence('worker'),
|
||||
'second-worker-recovery'
|
||||
)
|
||||
)
|
||||
const secondCapability = extractCapability(
|
||||
(second as { result: { preamble: string } }).result.preamble
|
||||
)
|
||||
expect(firstCapability).toMatch(/^dcap_/)
|
||||
expect(secondCapability).toMatch(/^dcap_/)
|
||||
expect(secondCapability).not.toBe(firstCapability)
|
||||
expect(
|
||||
harness.db.verifyDispatchCapability({
|
||||
dispatchId,
|
||||
capability: originalCapability,
|
||||
paneKey: CURRENT_WORKER_PANE,
|
||||
processIncarnation: 'process-1'
|
||||
}).valid
|
||||
).toBe(false)
|
||||
expect(
|
||||
harness.db.verifyDispatchCapability({
|
||||
dispatchId,
|
||||
capability: firstCapability,
|
||||
paneKey: CURRENT_WORKER_PANE,
|
||||
processIncarnation: 'process-1'
|
||||
}).valid
|
||||
).toBe(false)
|
||||
expect(
|
||||
harness.db.verifyDispatchCapability({
|
||||
dispatchId,
|
||||
capability: secondCapability,
|
||||
paneKey: CURRENT_WORKER_PANE,
|
||||
processIncarnation: 'process-1'
|
||||
}).valid
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
it.each(['dispatch', 'websocket'] as const)(
|
||||
'%s recovers on a fresh current-only database',
|
||||
async (transport) => {
|
||||
const harness = createFreshHarness()
|
||||
const { taskId, dispatchId } = createCurrentDispatch(harness)
|
||||
|
||||
const response = await invoke(
|
||||
harness.dispatcher,
|
||||
request(
|
||||
'orchestration.dispatchShow',
|
||||
{
|
||||
task: taskId,
|
||||
preamble: true,
|
||||
recoverCapability: true,
|
||||
from: CURRENT_WORKER_HANDLE
|
||||
},
|
||||
currentEvidence('worker'),
|
||||
`fresh-recovery-${transport}`
|
||||
),
|
||||
transport
|
||||
)
|
||||
|
||||
expect(response.ok).toBe(true)
|
||||
expect(
|
||||
extractCapability((response as { result: { preamble: string } }).result.preamble)
|
||||
).toMatch(/^dcap_/)
|
||||
expect(harness.db.getDispatchContextById(dispatchId)?.capability_hash).toMatch(/^[a-f0-9]+$/)
|
||||
expect(harness.db.getLegacyAdoption()).toBeUndefined()
|
||||
}
|
||||
)
|
||||
|
||||
it('keeps ordinary fresh-database calls outside recovery authority preflights', async () => {
|
||||
const harness = createFreshHarness()
|
||||
const { taskId } = createCurrentDispatch(harness)
|
||||
const activeDispatchLookup = vi.spyOn(harness.db, 'getActiveDispatchForIdentity')
|
||||
const remoteAttachmentLookup = vi.spyOn(harness.db, 'findActiveRemoteAttachmentForPane')
|
||||
|
||||
await harness.dispatcher.dispatch(
|
||||
request('orchestration.taskList', {}, currentEvidence('worker'), 'fresh-task-list')
|
||||
)
|
||||
await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.dispatchShow',
|
||||
{ task: taskId, preamble: true, from: CURRENT_WORKER_HANDLE },
|
||||
currentEvidence('worker'),
|
||||
'fresh-dispatch-inspection'
|
||||
)
|
||||
)
|
||||
|
||||
expect(activeDispatchLookup).not.toHaveBeenCalled()
|
||||
expect(remoteAttachmentLookup).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('keeps adopted-database Dispatch inspection outside recovery authority preflights', async () => {
|
||||
const harness = createHarness()
|
||||
const { taskId } = createCurrentDispatch(harness)
|
||||
const activeDispatchLookup = vi.spyOn(harness.db, 'getActiveDispatchForIdentity')
|
||||
const remoteAttachmentLookup = vi.spyOn(harness.db, 'findActiveRemoteAttachmentForPane')
|
||||
|
||||
expect(harness.db.getLegacyAdoption()).toBeDefined()
|
||||
await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.dispatchShow',
|
||||
{ task: taskId, preamble: true, from: CURRENT_WORKER_HANDLE },
|
||||
currentEvidence('worker'),
|
||||
'adopted-dispatch-inspection'
|
||||
)
|
||||
)
|
||||
|
||||
expect(activeDispatchLookup).not.toHaveBeenCalled()
|
||||
expect(remoteAttachmentLookup).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('deduplicates one concurrent retry without persisting the recovered secret', async () => {
|
||||
const harness = createFreshHarness()
|
||||
const { taskId } = createCurrentDispatch(harness)
|
||||
const recover = vi.spyOn(harness.db, 'recoverDispatchCapability')
|
||||
const recoveryRequest = request(
|
||||
'orchestration.dispatchShow',
|
||||
{
|
||||
task: taskId,
|
||||
preamble: true,
|
||||
recoverCapability: true,
|
||||
from: CURRENT_WORKER_HANDLE
|
||||
},
|
||||
currentEvidence('worker'),
|
||||
'concurrent-recovery'
|
||||
)
|
||||
|
||||
const [first, second] = await Promise.all([
|
||||
harness.dispatcher.dispatch(recoveryRequest),
|
||||
harness.dispatcher.dispatch(recoveryRequest)
|
||||
])
|
||||
const firstCapability = extractCapability(
|
||||
(first as { result: { preamble: string } }).result.preamble
|
||||
)
|
||||
const secondCapability = extractCapability(
|
||||
(second as { result: { preamble: string } }).result.preamble
|
||||
)
|
||||
expect(firstCapability).toBe(secondCapability)
|
||||
expect(recover).toHaveBeenCalledOnce()
|
||||
expect(
|
||||
harness.db.getMutationReceipt(
|
||||
createHash('sha256').update('caller-token').digest('hex'),
|
||||
'concurrent-recovery'
|
||||
)
|
||||
).toBeUndefined()
|
||||
})
|
||||
|
||||
it('never shares an in-flight recovered secret with different caller evidence', async () => {
|
||||
const harness = createFreshHarness()
|
||||
const { taskId } = createCurrentDispatch(harness)
|
||||
const params = {
|
||||
task: taskId,
|
||||
preamble: true,
|
||||
recoverCapability: true,
|
||||
from: CURRENT_WORKER_HANDLE
|
||||
}
|
||||
|
||||
const [authorized, spoofed] = await Promise.all([
|
||||
harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.dispatchShow',
|
||||
params,
|
||||
currentEvidence('worker'),
|
||||
'cross-caller-recovery'
|
||||
)
|
||||
),
|
||||
harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.dispatchShow',
|
||||
params,
|
||||
currentEvidence('coordinator'),
|
||||
'cross-caller-recovery'
|
||||
)
|
||||
)
|
||||
])
|
||||
|
||||
expect(
|
||||
extractCapability((authorized as { result: { preamble: string } }).result.preamble)
|
||||
).toMatch(/^dcap_/)
|
||||
expect((spoofed as { result: { preamble: string } }).result.preamble).not.toContain(
|
||||
'--dispatch-capability'
|
||||
)
|
||||
})
|
||||
|
||||
it('never shares an in-flight recovered secret across host evidence', async () => {
|
||||
const harness = createFreshHarness()
|
||||
const { taskId } = createCurrentDispatch(harness)
|
||||
vi.mocked(harness.runtime.verifyOrchestrationCompatibilityCaller).mockImplementation(
|
||||
(callerEvidence) =>
|
||||
callerEvidence?.host?.kind === 'ssh' && callerEvidence.host.attachmentId === 'attached'
|
||||
? {
|
||||
hostScope: { kind: 'ssh', targetId: 'target' },
|
||||
terminalHandle: CURRENT_WORKER_HANDLE,
|
||||
paneKey: CURRENT_WORKER_PANE,
|
||||
processIncarnation: 'process-1',
|
||||
launchTokenHash: 'worker-launch-hash'
|
||||
}
|
||||
: null
|
||||
)
|
||||
const params = {
|
||||
task: taskId,
|
||||
preamble: true,
|
||||
recoverCapability: true,
|
||||
from: CURRENT_WORKER_HANDLE
|
||||
}
|
||||
const validEvidence = {
|
||||
...currentEvidence('worker'),
|
||||
host: {
|
||||
kind: 'ssh' as const,
|
||||
targetId: 'target',
|
||||
connectionIncarnation: 'connection',
|
||||
attachmentId: 'attached'
|
||||
}
|
||||
}
|
||||
const staleEvidence = {
|
||||
...validEvidence,
|
||||
host: { ...validEvidence.host, attachmentId: 'stale' }
|
||||
}
|
||||
|
||||
const [authorized, stale] = await Promise.all([
|
||||
harness.dispatcher.dispatch(
|
||||
request('orchestration.dispatchShow', params, validEvidence, 'cross-host-recovery')
|
||||
),
|
||||
harness.dispatcher.dispatch(
|
||||
request('orchestration.dispatchShow', params, staleEvidence, 'cross-host-recovery')
|
||||
)
|
||||
])
|
||||
|
||||
expect(
|
||||
extractCapability((authorized as { result: { preamble: string } }).result.preamble)
|
||||
).toMatch(/^dcap_/)
|
||||
expect((stale as { result: { preamble: string } }).result.preamble).not.toContain(
|
||||
'--dispatch-capability'
|
||||
)
|
||||
})
|
||||
|
||||
it('deduplicates equivalent normalized caller evidence', async () => {
|
||||
const harness = createFreshHarness()
|
||||
const { taskId } = createCurrentDispatch(harness)
|
||||
const recover = vi.spyOn(harness.db, 'recoverDispatchCapability')
|
||||
const params = {
|
||||
task: taskId,
|
||||
preamble: true,
|
||||
recoverCapability: true,
|
||||
from: CURRENT_WORKER_HANDLE
|
||||
}
|
||||
const normalized = currentEvidence('worker')
|
||||
const padded = {
|
||||
terminalHandle: ` ${normalized.terminalHandle} `,
|
||||
paneKey: ` ${normalized.paneKey} `,
|
||||
launchToken: ` ${normalized.launchToken} `
|
||||
}
|
||||
|
||||
const [first, second] = await Promise.all([
|
||||
harness.dispatcher.dispatch(
|
||||
request('orchestration.dispatchShow', params, normalized, 'normalized-recovery')
|
||||
),
|
||||
harness.dispatcher.dispatch(
|
||||
request('orchestration.dispatchShow', params, padded, 'normalized-recovery')
|
||||
)
|
||||
])
|
||||
|
||||
expect(extractCapability((first as { result: { preamble: string } }).result.preamble)).toBe(
|
||||
extractCapability((second as { result: { preamble: string } }).result.preamble)
|
||||
)
|
||||
expect(recover).toHaveBeenCalledOnce()
|
||||
})
|
||||
|
||||
it('reports unavailable recovery for the apparent assignee without attestation', async () => {
|
||||
const harness = createFreshHarness()
|
||||
const { taskId } = createCurrentDispatch(harness)
|
||||
|
||||
const response = await harness.dispatcher.dispatch({
|
||||
...request(
|
||||
'orchestration.dispatchShow',
|
||||
{
|
||||
task: taskId,
|
||||
preamble: true,
|
||||
recoverCapability: true,
|
||||
from: CURRENT_WORKER_HANDLE
|
||||
},
|
||||
currentEvidence('worker'),
|
||||
'missing-attestation-recovery'
|
||||
),
|
||||
orchestrationCompatibilityEvidence: undefined
|
||||
})
|
||||
|
||||
expect(response).toMatchObject({ ok: true, result: { recovery: 'unavailable' } })
|
||||
expect((response as { result: { preamble: string } }).result.preamble).not.toContain(
|
||||
'--dispatch-capability'
|
||||
)
|
||||
})
|
||||
|
||||
it('preserves existing authority when CLI-command resolution fails', async () => {
|
||||
const harness = createFreshHarness()
|
||||
const { taskId, dispatchId, originalCapability } = createCurrentDispatch(harness)
|
||||
const recover = vi.spyOn(harness.db, 'recoverDispatchCapability')
|
||||
vi.mocked(harness.runtime.getTerminalOrchestrationCliCommand).mockImplementation(() => {
|
||||
throw new Error('cli command unavailable')
|
||||
})
|
||||
|
||||
const response = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.dispatchShow',
|
||||
{
|
||||
task: taskId,
|
||||
preamble: true,
|
||||
recoverCapability: true,
|
||||
from: CURRENT_WORKER_HANDLE
|
||||
},
|
||||
currentEvidence('worker'),
|
||||
'cli-resolution-failure'
|
||||
)
|
||||
)
|
||||
|
||||
expect(response).toMatchObject({ ok: false, error: { message: 'cli command unavailable' } })
|
||||
expect(recover).not.toHaveBeenCalled()
|
||||
expect(
|
||||
harness.db.verifyDispatchCapability({
|
||||
dispatchId,
|
||||
capability: originalCapability,
|
||||
paneKey: CURRENT_WORKER_PANE,
|
||||
processIncarnation: 'process-1'
|
||||
}).valid
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
it('recovers and settles after the assigned handle is reminted', async () => {
|
||||
const harness = createFreshHarness()
|
||||
const { taskId, dispatchId } = createCurrentDispatch(harness)
|
||||
const remintedHandle = 'term_worker_reminted'
|
||||
vi.mocked(harness.runtime.verifyOrchestrationCompatibilityCaller).mockReturnValue({
|
||||
hostScope: { kind: 'local', hostId: 'local' },
|
||||
terminalHandle: remintedHandle,
|
||||
paneKey: CURRENT_WORKER_PANE,
|
||||
processIncarnation: 'process-1',
|
||||
launchTokenHash: 'reminted-launch-hash'
|
||||
})
|
||||
const remintedEvidence = {
|
||||
terminalHandle: remintedHandle,
|
||||
paneKey: CURRENT_WORKER_PANE,
|
||||
launchToken: 'reminted-launch-token'
|
||||
}
|
||||
|
||||
const recovered = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.dispatchShow',
|
||||
{
|
||||
task: taskId,
|
||||
preamble: true,
|
||||
recoverCapability: true,
|
||||
from: remintedHandle
|
||||
},
|
||||
remintedEvidence,
|
||||
'reminted-recovery'
|
||||
)
|
||||
)
|
||||
const preamble = (recovered as { result: { preamble: string } }).result.preamble
|
||||
const capability = extractCapability(preamble)
|
||||
expect(preamble).toContain(`--from ${remintedHandle}`)
|
||||
expect(harness.runtime.getTerminalOrchestrationCliCommand).toHaveBeenCalledWith(remintedHandle)
|
||||
|
||||
const completed = await harness.dispatcher.dispatch({
|
||||
...request(
|
||||
'orchestration.send',
|
||||
{
|
||||
from: remintedHandle,
|
||||
type: 'worker_done',
|
||||
subject: 'Reminted completion',
|
||||
body: 'Recovered after remint. Finished the work. Nothing remains.',
|
||||
payload: JSON.stringify({ taskId, dispatchId, outcome: 'succeeded' })
|
||||
},
|
||||
remintedEvidence,
|
||||
'reminted-completion'
|
||||
),
|
||||
orchestrationCapability: capability
|
||||
})
|
||||
expect(completed).toMatchObject({
|
||||
ok: true,
|
||||
result: { lifecycle: { action: 'completed' } }
|
||||
})
|
||||
})
|
||||
|
||||
it('rejects capability rotation by the bound coordinator', async () => {
|
||||
const harness = createHarness()
|
||||
const { taskId, dispatchId, originalCapability } = createCurrentDispatch(harness)
|
||||
|
||||
const response = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.dispatchShow',
|
||||
{
|
||||
task: taskId,
|
||||
preamble: true,
|
||||
recoverCapability: true,
|
||||
from: CURRENT_COORDINATOR_HANDLE
|
||||
},
|
||||
currentEvidence('coordinator'),
|
||||
'coordinator-recovery'
|
||||
)
|
||||
)
|
||||
expect(response.ok).toBe(true)
|
||||
expect(response).toMatchObject({ result: { recovery: 'inspection' } })
|
||||
expect((response as { result: { preamble: string } }).result.preamble).not.toContain(
|
||||
'--dispatch-capability'
|
||||
)
|
||||
expect(
|
||||
harness.db.verifyDispatchCapability({
|
||||
dispatchId,
|
||||
capability: originalCapability,
|
||||
paneKey: CURRENT_WORKER_PANE,
|
||||
processIncarnation: 'process-1'
|
||||
}).valid
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
it('does not rotate authority for a claimed handle that disagrees with attestation', async () => {
|
||||
const harness = createHarness()
|
||||
const { taskId, dispatchId, originalCapability } = createCurrentDispatch(harness)
|
||||
const recover = vi.spyOn(harness.db, 'recoverDispatchCapability')
|
||||
|
||||
const response = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.dispatchShow',
|
||||
{
|
||||
task: taskId,
|
||||
preamble: true,
|
||||
recoverCapability: true,
|
||||
from: CURRENT_COORDINATOR_HANDLE
|
||||
},
|
||||
currentEvidence('worker'),
|
||||
'spoofed-recovery'
|
||||
)
|
||||
)
|
||||
expect(response.ok).toBe(true)
|
||||
expect((response as { result: { preamble: string } }).result.preamble).not.toContain(
|
||||
'--dispatch-capability'
|
||||
)
|
||||
expect(recover).not.toHaveBeenCalled()
|
||||
expect(
|
||||
harness.db.verifyDispatchCapability({
|
||||
dispatchId,
|
||||
capability: originalCapability,
|
||||
paneKey: CURRENT_WORKER_PANE,
|
||||
processIncarnation: 'process-1'
|
||||
}).valid
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
it('rejects recovery after the assigned process incarnation changes', async () => {
|
||||
const harness = createHarness()
|
||||
const { taskId, dispatchId, originalCapability } = createCurrentDispatch(harness, 'process-2')
|
||||
|
||||
const response = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.dispatchShow',
|
||||
{
|
||||
task: taskId,
|
||||
preamble: true,
|
||||
recoverCapability: true,
|
||||
from: CURRENT_WORKER_HANDLE
|
||||
},
|
||||
currentEvidence('worker'),
|
||||
'stale-process-recovery'
|
||||
)
|
||||
)
|
||||
expect(response.ok).toBe(true)
|
||||
expect((response as { result: { preamble: string } }).result.preamble).not.toContain(
|
||||
'--dispatch-capability'
|
||||
)
|
||||
expect(
|
||||
harness.db.verifyDispatchCapability({
|
||||
dispatchId,
|
||||
capability: originalCapability,
|
||||
paneKey: CURRENT_WORKER_PANE,
|
||||
processIncarnation: 'process-2'
|
||||
}).valid
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
it('does not rotate pending or start-unknown supervised authority', () => {
|
||||
const harness = createFreshHarness()
|
||||
const { dispatchId, originalCapability } = createStartingDispatch(harness.db)
|
||||
|
||||
expect(() =>
|
||||
harness.db.recoverDispatchCapability({
|
||||
dispatchId,
|
||||
callerPaneKey: CURRENT_WORKER_PANE,
|
||||
callerProcessIncarnation: 'process-1'
|
||||
})
|
||||
).toThrow('not an active current-contract Dispatch')
|
||||
harness.db.markWorkerStartUnknown(dispatchId, 'input_unknown', 'write outcome unknown')
|
||||
expect(() =>
|
||||
harness.db.recoverDispatchCapability({
|
||||
dispatchId,
|
||||
callerPaneKey: CURRENT_WORKER_PANE,
|
||||
callerProcessIncarnation: 'process-1'
|
||||
})
|
||||
).toThrow('not an active current-contract Dispatch')
|
||||
expect(
|
||||
harness.db.verifyDispatchCapability({
|
||||
dispatchId,
|
||||
capability: originalCapability,
|
||||
paneKey: CURRENT_WORKER_PANE,
|
||||
processIncarnation: 'process-1'
|
||||
}).valid
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it('does not rotate revoked or stopping authority', () => {
|
||||
const revokedHarness = createFreshHarness()
|
||||
const revoked = createCurrentDispatch(revokedHarness)
|
||||
revokedHarness.db.revokeDispatchCapability(revoked.dispatchId)
|
||||
expect(() =>
|
||||
revokedHarness.db.recoverDispatchCapability({
|
||||
dispatchId: revoked.dispatchId,
|
||||
callerPaneKey: CURRENT_WORKER_PANE,
|
||||
callerProcessIncarnation: 'process-1'
|
||||
})
|
||||
).toThrow('not an active current-contract Dispatch')
|
||||
|
||||
const stoppingHarness = createFreshHarness()
|
||||
const stopping = createStartingDispatch(stoppingHarness.db)
|
||||
stoppingHarness.db.markWorkerDispatchReady(stopping.dispatchId)
|
||||
stoppingHarness.db.beginWorkerStop(stopping.dispatchId)
|
||||
expect(() =>
|
||||
stoppingHarness.db.recoverDispatchCapability({
|
||||
dispatchId: stopping.dispatchId,
|
||||
callerPaneKey: CURRENT_WORKER_PANE,
|
||||
callerProcessIncarnation: 'process-1'
|
||||
})
|
||||
).toThrow('not an active current-contract Dispatch')
|
||||
})
|
||||
|
||||
it('keeps legacy Dispatch preamble inspection tokenless and non-mutating', async () => {
|
||||
const harness = createHarness()
|
||||
const recover = vi.spyOn(harness.db, 'recoverDispatchCapability')
|
||||
|
||||
const response = await harness.dispatcher.dispatch(
|
||||
request(
|
||||
'orchestration.dispatchShow',
|
||||
{
|
||||
task: harness.taskId,
|
||||
preamble: true,
|
||||
recoverCapability: true,
|
||||
from: WORKER_HANDLE
|
||||
},
|
||||
evidence('worker'),
|
||||
'legacy-dispatch-inspection'
|
||||
)
|
||||
)
|
||||
expect(response.ok).toBe(true)
|
||||
expect((response as { result: { preamble: string } }).result.preamble).not.toContain(
|
||||
'--dispatch-capability'
|
||||
)
|
||||
expect(recover).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
|
||||
function createCurrentDispatch(
|
||||
harness: { db: OrchestrationDb },
|
||||
processIncarnation = 'process-1'
|
||||
): { taskId: string; dispatchId: string; originalCapability: string } {
|
||||
const run = harness.db.createRun({
|
||||
objective: 'current recovery',
|
||||
coordinatorHandle: CURRENT_COORDINATOR_HANDLE,
|
||||
coordinatorPaneKey: CURRENT_COORDINATOR_PANE
|
||||
})
|
||||
const task = harness.db.createTask({ spec: 'finish recovered work', runId: run.id })
|
||||
const dispatch = harness.db.createDispatchContext(
|
||||
task.id,
|
||||
CURRENT_WORKER_HANDLE,
|
||||
CURRENT_WORKER_PANE,
|
||||
undefined,
|
||||
processIncarnation
|
||||
)
|
||||
const originalCapability = harness.db.mintDispatchCapability({
|
||||
dispatchId: dispatch.id,
|
||||
paneKey: CURRENT_WORKER_PANE,
|
||||
processIncarnation
|
||||
})
|
||||
return { taskId: task.id, dispatchId: dispatch.id, originalCapability }
|
||||
}
|
||||
|
||||
function extractCapability(preamble: string): string | undefined {
|
||||
return preamble.match(/--dispatch-capability (dcap_[A-Za-z0-9_-]+)/)?.[1]
|
||||
}
|
||||
|
||||
function createStartingDispatch(db: OrchestrationDb): {
|
||||
dispatchId: string
|
||||
originalCapability: string
|
||||
} {
|
||||
const run = db.createRun({
|
||||
objective: 'supervised recovery state',
|
||||
coordinatorHandle: CURRENT_COORDINATOR_HANDLE,
|
||||
coordinatorPaneKey: CURRENT_COORDINATOR_PANE
|
||||
})
|
||||
const task = db.createTask({ spec: 'supervised assignment', runId: run.id })
|
||||
const dispatch = db.createStartingWorkerDispatch({ taskId: task.id, startOptions: {} }).dispatch
|
||||
const originalCapability = db.prepareStartingWorkerAuthority({
|
||||
dispatchId: dispatch.id,
|
||||
handle: CURRENT_WORKER_HANDLE,
|
||||
paneKey: CURRENT_WORKER_PANE,
|
||||
processIncarnation: 'process-1',
|
||||
worktreeId: 'id:repo::worktree',
|
||||
effects: [],
|
||||
setupState: 'not_required'
|
||||
})
|
||||
return { dispatchId: dispatch.id, originalCapability }
|
||||
}
|
||||
|
||||
function createFreshHarness(): {
|
||||
db: OrchestrationDb
|
||||
runtime: OrcaRuntimeService
|
||||
dispatcher: RpcDispatcher
|
||||
} {
|
||||
const db = new OrchestrationDb(':memory:')
|
||||
freshDatabases.push(db)
|
||||
const runtime = new OrcaRuntimeService()
|
||||
runtime.setOrchestrationDb(db)
|
||||
vi.spyOn(runtime, 'verifyOrchestrationCompatibilityCaller').mockReturnValue({
|
||||
hostScope: { kind: 'local', hostId: 'local' },
|
||||
terminalHandle: CURRENT_WORKER_HANDLE,
|
||||
paneKey: CURRENT_WORKER_PANE,
|
||||
processIncarnation: 'process-1',
|
||||
launchTokenHash: 'worker-launch-hash'
|
||||
})
|
||||
vi.spyOn(runtime, 'getTerminalOrchestrationCliCommand').mockReturnValue('orca')
|
||||
vi.spyOn(runtime, 'getTerminalPaneKey').mockReturnValue(CURRENT_WORKER_PANE)
|
||||
vi.spyOn(runtime, 'getTerminalProcessIncarnation').mockReturnValue('process-1')
|
||||
return {
|
||||
db,
|
||||
runtime,
|
||||
dispatcher: new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS })
|
||||
}
|
||||
}
|
||||
@@ -29,7 +29,8 @@ const COORDINATOR_PREFLIGHT_METHODS = new Set([
|
||||
|
||||
const CURRENT_AUTHORITY_PREFLIGHT_METHODS = new Set([
|
||||
...COORDINATOR_PREFLIGHT_METHODS,
|
||||
'orchestration.ask'
|
||||
'orchestration.ask',
|
||||
'orchestration.dispatchShow'
|
||||
])
|
||||
|
||||
export type LegacyCompatibilityRoute =
|
||||
@@ -65,7 +66,10 @@ export class OrchestrationLegacyCompatibility {
|
||||
return { handled: false }
|
||||
}
|
||||
const values = params as Record<string, unknown>
|
||||
if (CURRENT_AUTHORITY_PREFLIGHT_METHODS.has(request.method)) {
|
||||
if (
|
||||
CURRENT_AUTHORITY_PREFLIGHT_METHODS.has(request.method) &&
|
||||
(request.method !== 'orchestration.dispatchShow' || values.recoverCapability === true)
|
||||
) {
|
||||
const callerAuthority = this.resolveCurrentAuthority(request, values)
|
||||
if (callerAuthority) {
|
||||
return {
|
||||
@@ -134,7 +138,13 @@ export class OrchestrationLegacyCompatibility {
|
||||
): OrchestrationCompatibilityCallerAuthority | undefined {
|
||||
const db = this.runtime.getOrchestrationDb()
|
||||
const adoption = db.getLegacyAdoption()
|
||||
if (!adoption || stringValue(params.run) || request.method === 'orchestration.runUse') {
|
||||
const isDispatchRecovery =
|
||||
request.method === 'orchestration.dispatchShow' && params.recoverCapability === true
|
||||
if (
|
||||
(!adoption && !isDispatchRecovery) ||
|
||||
stringValue(params.run) ||
|
||||
request.method === 'orchestration.runUse'
|
||||
) {
|
||||
return undefined
|
||||
}
|
||||
const evidence = request.orchestrationCompatibilityEvidence
|
||||
@@ -167,6 +177,9 @@ export class OrchestrationLegacyCompatibility {
|
||||
? caller
|
||||
: undefined
|
||||
}
|
||||
if (!adoption) {
|
||||
return undefined
|
||||
}
|
||||
const boundRun = db.getCurrentRunForPane(evidence.paneKey)
|
||||
if (!boundRun) {
|
||||
return undefined
|
||||
|
||||
@@ -16,6 +16,10 @@ export type DurableMutationInvocation = {
|
||||
|
||||
export class OrchestrationMutationExecutor {
|
||||
private readonly inFlight = new Map<string, Promise<unknown>>()
|
||||
private readonly secretRecoveries = new Map<
|
||||
string,
|
||||
{ method: string; payloadHash: string; promise: Promise<unknown> }
|
||||
>()
|
||||
|
||||
constructor(private readonly runtime: OrcaRuntimeService) {}
|
||||
|
||||
@@ -34,6 +38,30 @@ export class OrchestrationMutationExecutor {
|
||||
.update(JSON.stringify(canonicalize({ method: request.method, params })))
|
||||
.digest('hex')
|
||||
const key = `${callerFingerprint}:${requestId}`
|
||||
if (isSecretRecoveryMutation(request.method, params)) {
|
||||
const secretKey = `${key}:${secretRecoveryCallerFingerprint(request)}`
|
||||
const active = this.secretRecoveries.get(secretKey)
|
||||
if (active) {
|
||||
if (active.method !== request.method || active.payloadHash !== payloadHash) {
|
||||
throw new OrchestrationError(
|
||||
'request_mismatch',
|
||||
`Mutation request ${requestId} is already in flight with different input.`
|
||||
)
|
||||
}
|
||||
return await active.promise
|
||||
}
|
||||
const recovery = Promise.resolve().then(() => invoke())
|
||||
this.secretRecoveries.set(secretKey, {
|
||||
method: request.method,
|
||||
payloadHash,
|
||||
promise: recovery
|
||||
})
|
||||
try {
|
||||
return await recovery
|
||||
} finally {
|
||||
this.secretRecoveries.delete(secretKey)
|
||||
}
|
||||
}
|
||||
const db = this.runtime.getOrchestrationDb()
|
||||
const identity = { callerFingerprint, requestId, method: request.method, payloadHash }
|
||||
const atomicWorkerAcceptance =
|
||||
@@ -111,6 +139,31 @@ export class OrchestrationMutationExecutor {
|
||||
}
|
||||
}
|
||||
|
||||
function isSecretRecoveryMutation(method: string, params: unknown): boolean {
|
||||
return (
|
||||
method === 'orchestration.dispatchShow' &&
|
||||
typeof params === 'object' &&
|
||||
params !== null &&
|
||||
(params as { recoverCapability?: unknown }).recoverCapability === true
|
||||
)
|
||||
}
|
||||
|
||||
function secretRecoveryCallerFingerprint(request: RpcRequest): string {
|
||||
const evidence = request.orchestrationCompatibilityEvidence
|
||||
return createHash('sha256')
|
||||
.update(
|
||||
JSON.stringify(
|
||||
canonicalize({
|
||||
terminalHandle: evidence?.terminalHandle?.trim() || null,
|
||||
paneKey: evidence?.paneKey?.trim() || null,
|
||||
launchToken: evidence?.launchToken?.trim() || null,
|
||||
host: evidence?.host ?? null
|
||||
})
|
||||
)
|
||||
)
|
||||
.digest('hex')
|
||||
}
|
||||
|
||||
const executorsByRuntime = new WeakMap<OrcaRuntimeService, OrchestrationMutationExecutor>()
|
||||
|
||||
export function getOrchestrationMutationExecutor(
|
||||
|
||||
@@ -29,6 +29,7 @@ describe('orchestration RPC contract', () => {
|
||||
['orchestration.check', { wait: true }],
|
||||
['orchestration.check', { unread: true }],
|
||||
['orchestration.check', { peek: true, ack: 'delivery_1' }],
|
||||
['orchestration.dispatchShow', { preamble: true, recoverCapability: true }],
|
||||
['orchestration.run', {}],
|
||||
['orchestration.runStop', {}]
|
||||
])('classifies %s as a mutation', (method, params) => {
|
||||
@@ -42,6 +43,7 @@ describe('orchestration RPC contract', () => {
|
||||
['orchestration.inbox', {}],
|
||||
['orchestration.taskList', {}],
|
||||
['orchestration.dispatchShow', {}],
|
||||
['orchestration.dispatchShow', { preamble: true }],
|
||||
['orchestration.gateList', {}],
|
||||
['orchestration.workerShow', {}],
|
||||
['orchestration.workerRead', {}],
|
||||
|
||||
@@ -57,6 +57,9 @@ export function isOrchestrationMutation(method: string, params: unknown): boolea
|
||||
if (method === 'orchestration.dispatch') {
|
||||
return !hasTrueProperty(params, 'dryRun')
|
||||
}
|
||||
if (method === 'orchestration.dispatchShow') {
|
||||
return hasTrueProperty(params, 'recoverCapability')
|
||||
}
|
||||
return ORCHESTRATION_MUTATION_METHODS.has(method)
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
isStablePaneId,
|
||||
isTerminalLeafId,
|
||||
isEquivalentPaneKey,
|
||||
makePaneKey,
|
||||
parseLegacyNumericPaneKey,
|
||||
parsePaneKey
|
||||
@@ -46,6 +47,13 @@ describe('stable pane ids', () => {
|
||||
expect(parsePaneKey('tab-1:')).toBeNull()
|
||||
})
|
||||
|
||||
it('matches reminted pane keys by their stable leaf', () => {
|
||||
expect(isEquivalentPaneKey(`tab-old:${LEAF_ID}`, `tab-new:${LEAF_ID}`)).toBe(true)
|
||||
expect(
|
||||
isEquivalentPaneKey(`tab-old:${LEAF_ID}`, 'tab-new:22222222-2222-4222-8222-222222222222')
|
||||
).toBe(false)
|
||||
})
|
||||
|
||||
it('parses legacy numeric pane keys only for migration aliases', () => {
|
||||
expect(parseLegacyNumericPaneKey(' tab-1:12 ')).toEqual({
|
||||
tabId: 'tab-1',
|
||||
|
||||
@@ -44,6 +44,15 @@ export function parsePaneKey(
|
||||
return { tabId, leafId, stablePaneId: leafId }
|
||||
}
|
||||
|
||||
export function isEquivalentPaneKey(left: string, right: string): boolean {
|
||||
if (left === right) {
|
||||
return true
|
||||
}
|
||||
const leftLeaf = parsePaneKey(left)?.leafId
|
||||
const rightLeaf = parsePaneKey(right)?.leafId
|
||||
return Boolean(leftLeaf && rightLeaf && leftLeaf === rightLeaf)
|
||||
}
|
||||
|
||||
export function parseLegacyNumericPaneKey(
|
||||
paneKey: unknown
|
||||
): { tabId: string; numericPaneId: string; paneKey: string } | null {
|
||||
|
||||
Reference in New Issue
Block a user