mirror of
https://github.com/stablyai/orca.git
synced 2026-10-22 00:03:47 +00:00
fix(orchestration): harden mutation replay identity
This commit is contained in:
@@ -11,7 +11,7 @@ export function orchestrationMutationRecoveryError(error: unknown): unknown {
|
||||
}
|
||||
const message = [
|
||||
stripUnsafeRetryAdvice(error.message, requestId),
|
||||
'The orchestration mutation may already have taken effect; a worktree, terminal, or dispatch may already exist.',
|
||||
'The orchestration mutation may already have taken effect; do not assume it failed.',
|
||||
`Re-issue the same command with --retry-request ${requestId} to recover idempotently. Do not retry this mutation without --retry-request.`,
|
||||
typeof data?.failedStage === 'string' ? `Failed stage: ${data.failedStage}.` : undefined,
|
||||
Array.isArray(data?.residualResources)
|
||||
|
||||
@@ -6,7 +6,8 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [
|
||||
{
|
||||
path: ['orchestration', 'run-create'],
|
||||
summary: 'Create and bind a lightweight orchestration Run',
|
||||
usage: 'orca orchestration run-create --objective <text> [--from <handle>] [--json]',
|
||||
usage:
|
||||
'orca orchestration run-create --objective <text> [--from <handle>] [--retry-request <id>] [--json]',
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'objective', 'from', 'retry-request'],
|
||||
notes: [
|
||||
'A Run is a namespace and home inbox. It never schedules or places workers.',
|
||||
@@ -17,7 +18,7 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [
|
||||
path: ['orchestration', 'run-use'],
|
||||
summary: 'Bind this coordinator terminal to an existing Run',
|
||||
usage:
|
||||
'orca orchestration run-use --id <run_id> [--from <handle>] [--takeover-legacy] [--json]',
|
||||
'orca orchestration run-use --id <run_id> [--from <handle>] [--takeover-legacy] [--retry-request <id>] [--json]',
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'id', 'from', 'takeover-legacy', 'retry-request'],
|
||||
notes: [
|
||||
'--takeover-legacy must run in the live coordinator agent terminal it binds; it preserves existing worker assignments.'
|
||||
@@ -45,7 +46,7 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [
|
||||
path: ['orchestration', 'send'],
|
||||
summary: 'Send an inter-agent message',
|
||||
usage:
|
||||
'orca orchestration send --subject <text> [--to <run:id|dispatch:id|legacy_handle>] [--run <run_id>] [--from <handle>] [--body <text>] [--type <type>] [--priority <level>] [--thread-id <id>] [--payload <json>] [--task-id <id>] [--dispatch-id <id>] [--outcome <succeeded|failed>] [--files-modified <csv>] [--report-path <path>] [--phase <text>] [--json]',
|
||||
'orca orchestration send --subject <text> [--to <run:id|dispatch:id|legacy_handle>] [--run <run_id>] [--from <handle>] [--body <text>] [--type <type>] [--priority <level>] [--thread-id <id>] [--payload <json>] [--task-id <id>] [--dispatch-id <id>] [--outcome <succeeded|failed>] [--files-modified <csv>] [--report-path <path>] [--phase <text>] [--retry-request <id>] [--json]',
|
||||
allowedFlags: [
|
||||
...GLOBAL_FLAGS,
|
||||
'to',
|
||||
@@ -80,7 +81,7 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [
|
||||
path: ['orchestration', 'check'],
|
||||
summary: 'Check messages for a terminal',
|
||||
usage:
|
||||
'orca orchestration check [--terminal <handle>] [--run <run_id>] [--ack <delivery_id>] [--unread | --peek | --all] [--types <type,...>] [--format] [--wait] [--timeout-ms <n>] [--json]\n' +
|
||||
'orca orchestration check [--terminal <handle>] [--run <run_id>] [--ack <delivery_id>] [--unread | --peek | --all] [--types <type,...>] [--format] [--wait] [--timeout-ms <n>] [--retry-request <id>] [--json]\n' +
|
||||
" default: return the bound Run's oldest unacknowledged FIFO batch.\n" +
|
||||
' --ack: acknowledge the prior whole batch before checking/waiting.\n' +
|
||||
' --peek: return only unread messages without marking them read.\n' +
|
||||
@@ -114,7 +115,7 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [
|
||||
path: ['orchestration', 'reply'],
|
||||
summary: 'Reply to a message',
|
||||
usage:
|
||||
'orca orchestration reply --id <msg_id> --body <text> [--run <run_id>] [--from <handle>] [--json]',
|
||||
'orca orchestration reply --id <msg_id> --body <text> [--run <run_id>] [--from <handle>] [--retry-request <id>] [--json]',
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'id', 'body', 'run', 'from', 'retry-request']
|
||||
},
|
||||
{
|
||||
@@ -127,7 +128,7 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [
|
||||
path: ['orchestration', 'task-create'],
|
||||
summary: 'Create an orchestration task',
|
||||
usage:
|
||||
'orca orchestration task-create --spec <text> [--task-title <text>] [--display-name <text>] [--deps <json_array>] [--parent <task_id>] [--run <run_id>] [--from <handle>] [--json]',
|
||||
'orca orchestration task-create --spec <text> [--task-title <text>] [--display-name <text>] [--deps <json_array>] [--parent <task_id>] [--run <run_id>] [--from <handle>] [--retry-request <id>] [--json]',
|
||||
allowedFlags: [
|
||||
...GLOBAL_FLAGS,
|
||||
'spec',
|
||||
@@ -152,7 +153,7 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [
|
||||
path: ['orchestration', 'task-update'],
|
||||
summary: 'Update a task status',
|
||||
usage:
|
||||
'orca orchestration task-update --id <task_id> --status <status> [--result <json>] [--run <run_id>] [--from <handle>] [--json]',
|
||||
'orca orchestration task-update --id <task_id> --status <status> [--result <json>] [--run <run_id>] [--from <handle>] [--retry-request <id>] [--json]',
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'id', 'status', 'result', 'run', 'from', 'retry-request'],
|
||||
notes: ['Valid --status values: pending, ready, dispatched, completed, failed, blocked.']
|
||||
},
|
||||
@@ -161,7 +162,7 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [
|
||||
path: ['orchestration', 'dispatch'],
|
||||
summary: 'Dispatch a task to a terminal',
|
||||
usage:
|
||||
'orca orchestration dispatch --task <task_id> --to <handle> [--from <handle>] [--run <run_id>] [--inject] [--dry-run] [--return-preamble] [--json]',
|
||||
'orca orchestration dispatch --task <task_id> --to <handle> [--from <handle>] [--run <run_id>] [--inject] [--dry-run] [--return-preamble] [--retry-request <id>] [--json]',
|
||||
allowedFlags: [
|
||||
...GLOBAL_FLAGS,
|
||||
'task',
|
||||
@@ -185,7 +186,7 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [
|
||||
path: ['orchestration', 'ask'],
|
||||
summary: 'Ask the coordinator a question and block until answered',
|
||||
usage:
|
||||
'orca orchestration ask (--question <text> | --resume <message_id>) [--to <run:id>] [--run <run_id>] [--options <csv>] [--timeout-ms <n>] [--from <handle>] [--json]',
|
||||
'orca orchestration ask (--question <text> | --resume <message_id>) [--to <run:id>] [--run <run_id>] [--options <csv>] [--timeout-ms <n>] [--from <handle>] [--retry-request <id>] [--json]',
|
||||
allowedFlags: [
|
||||
...GLOBAL_FLAGS,
|
||||
'to',
|
||||
@@ -236,14 +237,14 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [
|
||||
path: ['orchestration', 'gate-create'],
|
||||
summary: 'Create a decision gate blocking a task',
|
||||
usage:
|
||||
'orca orchestration gate-create --task <task_id> --question <text> [--options <json_array>] [--from <handle>] [--json]',
|
||||
'orca orchestration gate-create --task <task_id> --question <text> [--options <json_array>] [--from <handle>] [--retry-request <id>] [--json]',
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'task', 'question', 'options', 'from', 'retry-request']
|
||||
},
|
||||
{
|
||||
path: ['orchestration', 'gate-resolve'],
|
||||
summary: 'Resolve a pending decision gate',
|
||||
usage:
|
||||
'orca orchestration gate-resolve --id <gate_id> --resolution <text> [--from <handle>] [--json]',
|
||||
'orca orchestration gate-resolve --id <gate_id> --resolution <text> [--from <handle>] [--retry-request <id>] [--json]',
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'id', 'resolution', 'from', 'retry-request']
|
||||
},
|
||||
{
|
||||
@@ -257,7 +258,8 @@ export const ORCHESTRATION_COMMAND_SPECS: CommandSpec[] = [
|
||||
{
|
||||
path: ['orchestration', 'reset'],
|
||||
summary: 'Reset one explicit orchestration state scope',
|
||||
usage: 'orca orchestration reset (--all | --tasks | --messages) [--json]',
|
||||
usage:
|
||||
'orca orchestration reset (--all | --tasks | --messages) [--retry-request <id>] [--json]',
|
||||
allowedFlags: [...GLOBAL_FLAGS, 'all', 'tasks', 'messages', 'retry-request']
|
||||
}
|
||||
]
|
||||
|
||||
@@ -297,8 +297,8 @@ type RunListCursor = {
|
||||
id: string
|
||||
}
|
||||
|
||||
// Schema versions: v2 'heartbeat'+last_heartbeat_at, v3 delivered_at, v4 task-creator terminal, v5 task_title/display_name, v6 pane identity, v7 lightweight Runs, v8 crash-safe Run deliveries, v9 durable question threads, v10 Dispatch capabilities, v11 durable mutation receipts, v12 composed worker state, v18 post-v6 version-skew repair, v19 adopted legacy Runs and compatibility receipts, v20 legacy question backfill, v21 legacy scheduler-loss provenance, v22 dispatch assignee lookup, v23 worker terminal resource ownership, v24 creator-incarnation authority, v25 active Dispatch handle lookup, v26 indexed mutation receipt capacity, v27 durable federation acknowledgments.
|
||||
const SCHEMA_VERSION = 27
|
||||
// Schema versions: v2 'heartbeat'+last_heartbeat_at, v3 delivered_at, v4 task-creator terminal, v5 task_title/display_name, v6 pane identity, v7 lightweight Runs, v8 crash-safe Run deliveries, v9 durable question threads, v10 Dispatch capabilities, v11 durable mutation receipts, v12 composed worker state, v18 post-v6 version-skew repair, v19 adopted legacy Runs and compatibility receipts, v20 legacy question backfill, v21 legacy scheduler-loss provenance, v22 dispatch assignee lookup, v23 worker terminal resource ownership, v24 creator-incarnation authority, v25 active Dispatch handle lookup, v26 indexed mutation receipt capacity, v27 durable federation acknowledgments, v28 durable local mutation caller identity.
|
||||
const SCHEMA_VERSION = 28
|
||||
|
||||
function hardenOrchestrationDatabaseFiles(dbPath: (string & {}) | ':memory:'): void {
|
||||
if (dbPath === ':memory:' || process.platform === 'win32') {
|
||||
@@ -322,6 +322,7 @@ export class OrchestrationDb {
|
||||
// emptiness so the non-orchestration majority short-circuits the whole
|
||||
// per-terminal fan-out. Only createDispatchContext flips this false→true.
|
||||
private hasAnyDispatchContextsCache: boolean | undefined
|
||||
private localMutationCallerFingerprint: string | undefined
|
||||
|
||||
constructor(dbPath: (string & {}) | ':memory:') {
|
||||
this.db = new Database(dbPath)
|
||||
@@ -405,6 +406,11 @@ export class OrchestrationDb {
|
||||
PRIMARY KEY (caller_fingerprint, request_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS mutation_caller_identities (
|
||||
transport TEXT PRIMARY KEY,
|
||||
caller_fingerprint TEXT NOT NULL UNIQUE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS worker_dispatches (
|
||||
dispatch_id TEXT PRIMARY KEY,
|
||||
runtime_epoch TEXT,
|
||||
@@ -1012,6 +1018,14 @@ export class OrchestrationDb {
|
||||
'ALTER TABLE federated_dispatches ADD COLUMN to_home_acknowledged_sequence INTEGER NOT NULL DEFAULT 0'
|
||||
)
|
||||
}
|
||||
if (current < 28) {
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS mutation_caller_identities (
|
||||
transport TEXT PRIMARY KEY,
|
||||
caller_fingerprint TEXT NOT NULL UNIQUE
|
||||
);
|
||||
`)
|
||||
}
|
||||
this.db.exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_dispatch_assignee_pane_leaf
|
||||
ON dispatch_contexts(${DISPATCH_PANE_KEY_MATCH_SUFFIX_SQL})
|
||||
@@ -1427,6 +1441,34 @@ export class OrchestrationDb {
|
||||
|
||||
// ── Durable mutation receipts ──
|
||||
|
||||
getOrCreateLocalMutationCallerFingerprint(): string {
|
||||
if (this.localMutationCallerFingerprint) {
|
||||
return this.localMutationCallerFingerprint
|
||||
}
|
||||
const transport = 'local_authenticated_transport'
|
||||
const existing = this.db
|
||||
.prepare('SELECT caller_fingerprint FROM mutation_caller_identities WHERE transport = ?')
|
||||
.get(transport) as { caller_fingerprint: string } | undefined
|
||||
if (existing) {
|
||||
this.localMutationCallerFingerprint = existing.caller_fingerprint
|
||||
return this.localMutationCallerFingerprint
|
||||
}
|
||||
this.db
|
||||
.prepare(
|
||||
`INSERT OR IGNORE INTO mutation_caller_identities (transport, caller_fingerprint)
|
||||
VALUES (?, ?)`
|
||||
)
|
||||
.run(transport, randomBytes(32).toString('hex'))
|
||||
const created = this.db
|
||||
.prepare('SELECT caller_fingerprint FROM mutation_caller_identities WHERE transport = ?')
|
||||
.get(transport) as { caller_fingerprint: string } | undefined
|
||||
if (!created) {
|
||||
throw new Error('Failed to create the local orchestration mutation caller identity.')
|
||||
}
|
||||
this.localMutationCallerFingerprint = created.caller_fingerprint
|
||||
return this.localMutationCallerFingerprint
|
||||
}
|
||||
|
||||
beginMutationReceipt(params: {
|
||||
callerFingerprint: string
|
||||
requestId: string
|
||||
|
||||
@@ -41,7 +41,7 @@ describe('federation acknowledgment migration', () => {
|
||||
db = new OrchestrationDb(dbPath)
|
||||
const sqlite = (db as unknown as { db: Database.Database }).db
|
||||
|
||||
expect(sqlite.pragma('user_version', { simple: true })).toBe(27)
|
||||
expect(sqlite.pragma('user_version', { simple: true })).toBe(28)
|
||||
expect(db.getFederatedDispatch('ctx_migrated')).toMatchObject({
|
||||
to_home_imported_sequence: 2,
|
||||
to_home_acknowledged_sequence: 0
|
||||
|
||||
@@ -103,7 +103,7 @@ describe('mutation receipt capacity schema', () => {
|
||||
|
||||
db = new OrchestrationDb(dbPath)
|
||||
const sqlite = sqliteFor(db)
|
||||
expect(sqlite.pragma('user_version', { simple: true })).toBe(27)
|
||||
expect(sqlite.pragma('user_version', { simple: true })).toBe(28)
|
||||
expect(sqlite.prepare('SELECT receipt_count FROM mutation_receipt_ledger').get()).toEqual({
|
||||
receipt_count: 20
|
||||
})
|
||||
|
||||
@@ -240,7 +240,7 @@ describe('OrchestrationDb dispatch assignee index migration', () => {
|
||||
|
||||
db = new OrchestrationDb(dbPath)
|
||||
const sqlite = sqliteFor(db)
|
||||
expect(sqlite.pragma('user_version', { simple: true })).toBe(27)
|
||||
expect(sqlite.pragma('user_version', { simple: true })).toBe(28)
|
||||
expect(db.getDispatchContextById(dispatch.id)).toMatchObject({ assignee_handle: 'term_worker' })
|
||||
expect(db.getTask(task.id)).toMatchObject({
|
||||
created_by_pane_key: null,
|
||||
@@ -277,7 +277,7 @@ describe('OrchestrationDb dispatch assignee index migration', () => {
|
||||
|
||||
db.close()
|
||||
db = new OrchestrationDb(dbPath)
|
||||
expect(sqliteFor(db).pragma('user_version', { simple: true })).toBe(27)
|
||||
expect(sqliteFor(db).pragma('user_version', { simple: true })).toBe(28)
|
||||
expect(db.getDispatchContextById(dispatch.id)).toBeDefined()
|
||||
})
|
||||
|
||||
@@ -309,7 +309,7 @@ describe('OrchestrationDb dispatch assignee index migration', () => {
|
||||
|
||||
db = new OrchestrationDb(dbPath)
|
||||
const sqlite = sqliteFor(db)
|
||||
expect(sqlite.pragma('user_version', { simple: true })).toBe(27)
|
||||
expect(sqlite.pragma('user_version', { simple: true })).toBe(28)
|
||||
expect(db.getTask(task.id)).toMatchObject({
|
||||
created_by_pane_key: 'tab_creator:leaf_creator',
|
||||
created_by_process_incarnation: 'pty_creator:incarnation-a',
|
||||
@@ -328,7 +328,7 @@ describe('OrchestrationDb dispatch assignee index migration', () => {
|
||||
|
||||
db.close()
|
||||
db = new OrchestrationDb(dbPath)
|
||||
expect(sqliteFor(db).pragma('user_version', { simple: true })).toBe(27)
|
||||
expect(sqliteFor(db).pragma('user_version', { simple: true })).toBe(28)
|
||||
expect(db.getTask(task.id)?.created_by_process_incarnation).toBe('pty_creator:incarnation-a')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -3,6 +3,7 @@ import type { TerminalStreamFrame } from '../../../shared/terminal-stream-protoc
|
||||
import type { PairingRpcContext } from './core'
|
||||
|
||||
export type RpcDispatchStreamingOptions = {
|
||||
authenticatedCallerFingerprint?: string
|
||||
connectionId?: string
|
||||
signal?: AbortSignal
|
||||
clientId?: string
|
||||
|
||||
@@ -15,7 +15,6 @@ import { ALL_RPC_METHODS } from './methods'
|
||||
import { emulatorProbe, emulatorProbeError } from '../../emulator/emulator-probe'
|
||||
import type { OrcaRuntimeService } from '../orca-runtime'
|
||||
import {
|
||||
authenticatedCallerFingerprint,
|
||||
getOrchestrationMutationExecutor,
|
||||
type OrchestrationMutationExecutor,
|
||||
type DurableMutationInvocation
|
||||
@@ -41,7 +40,10 @@ export class RpcDispatcher {
|
||||
this.legacyOrchestration = new OrchestrationLegacyCompatibility(runtime)
|
||||
}
|
||||
|
||||
async dispatch(request: RpcRequest, options?: { signal?: AbortSignal }): Promise<RpcResponse> {
|
||||
async dispatch(
|
||||
request: RpcRequest,
|
||||
options?: { signal?: AbortSignal; authenticatedCallerFingerprint?: string }
|
||||
): Promise<RpcResponse> {
|
||||
const meta = this.meta()
|
||||
const method = this.registry.get(request.method)
|
||||
if (!method) {
|
||||
@@ -89,6 +91,11 @@ export class RpcDispatcher {
|
||||
request,
|
||||
compatibility.legacyCoordinatorAuthority
|
||||
)
|
||||
const authenticatedCallerFingerprint =
|
||||
options?.authenticatedCallerFingerprint ??
|
||||
(request.method.startsWith('orchestration.')
|
||||
? this.orchestrationMutations.getLocalAuthenticatedCallerFingerprint()
|
||||
: undefined)
|
||||
const invoke = (mutation?: DurableMutationInvocation) => {
|
||||
const legacyCoordinatorRunId = legacyCoordinator?.revalidate()
|
||||
return method.handler(effectiveParams, {
|
||||
@@ -97,7 +104,7 @@ export class RpcDispatcher {
|
||||
requestId: request.id,
|
||||
orchestrationCapability: request.orchestrationCapability,
|
||||
authenticatedCallerFingerprint:
|
||||
mutation?.identity.callerFingerprint ?? authenticatedCallerFingerprint(request),
|
||||
mutation?.identity.callerFingerprint ?? authenticatedCallerFingerprint,
|
||||
recordMutationReceipt: mutation?.recordReceipt,
|
||||
orchestrationMutation: mutation?.identity,
|
||||
legacyCoordinatorRunId,
|
||||
@@ -112,7 +119,7 @@ export class RpcDispatcher {
|
||||
request,
|
||||
effectiveParams,
|
||||
invoke,
|
||||
legacyCoordinator?.mutationCallerFingerprint
|
||||
legacyCoordinator?.mutationCallerFingerprint ?? authenticatedCallerFingerprint
|
||||
)
|
||||
recordRuntimeFeatureInteraction(
|
||||
this.runtime,
|
||||
@@ -177,6 +184,11 @@ export class RpcDispatcher {
|
||||
request,
|
||||
compatibility.legacyCoordinatorAuthority
|
||||
)
|
||||
const authenticatedCallerFingerprint =
|
||||
options?.authenticatedCallerFingerprint ??
|
||||
(request.method.startsWith('orchestration.')
|
||||
? this.orchestrationMutations.getLocalAuthenticatedCallerFingerprint()
|
||||
: undefined)
|
||||
const invoke = (mutation?: DurableMutationInvocation) => {
|
||||
const legacyCoordinatorRunId = legacyCoordinator?.revalidate()
|
||||
return method.handler(effectiveParams, {
|
||||
@@ -190,7 +202,7 @@ export class RpcDispatcher {
|
||||
clientCapabilities: options?.clientCapabilities,
|
||||
orchestrationCapability: request.orchestrationCapability,
|
||||
authenticatedCallerFingerprint:
|
||||
mutation?.identity.callerFingerprint ?? authenticatedCallerFingerprint(request),
|
||||
mutation?.identity.callerFingerprint ?? authenticatedCallerFingerprint,
|
||||
recordMutationReceipt: mutation?.recordReceipt,
|
||||
orchestrationMutation: mutation?.identity,
|
||||
pairing: options?.pairing,
|
||||
@@ -208,7 +220,7 @@ export class RpcDispatcher {
|
||||
request,
|
||||
effectiveParams,
|
||||
invoke,
|
||||
legacyCoordinator?.mutationCallerFingerprint
|
||||
legacyCoordinator?.mutationCallerFingerprint ?? authenticatedCallerFingerprint
|
||||
)
|
||||
recordRuntimeFeatureInteraction(
|
||||
this.runtime,
|
||||
|
||||
@@ -6,11 +6,12 @@ import { OrchestrationDb } from '../../orchestration/db'
|
||||
import type { OrchestrationEnvironmentTransport } from '../../orchestration/environment-transport'
|
||||
import type { RpcRequest } from '../core'
|
||||
import { RpcDispatcher } from '../dispatcher'
|
||||
import { authenticatedCallerFingerprint } from '../orchestration-mutation-executor'
|
||||
import { fingerprintAuthenticatedPairingDevice } from '../orchestration-mutation-executor'
|
||||
import { ORCHESTRATION_METHODS } from './orchestration'
|
||||
|
||||
describe('orchestration federation control mail', () => {
|
||||
const homeToken = 'run-home-device-token'
|
||||
const homeFingerprint = fingerprintAuthenticatedPairingDevice('run-home-device')
|
||||
const workerToken = 'worker-local-token'
|
||||
const workerPeerFingerprint = 'worker-peer'
|
||||
const coordinatorPaneKey = 'tab_coord:aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa'
|
||||
@@ -55,14 +56,17 @@ describe('orchestration federation control mail', () => {
|
||||
_meta: { runtimeId: workerRuntime.getRuntimeId() }
|
||||
}
|
||||
}
|
||||
const response = (await workerDispatcher.dispatch({
|
||||
id: `remote_${method}`,
|
||||
authToken: homeToken,
|
||||
method,
|
||||
params,
|
||||
orchestrationContractVersion: envelope?.orchestrationContractVersion,
|
||||
orchestrationRequestId: envelope?.orchestrationRequestId
|
||||
})) as RuntimeRpcResponse<unknown>
|
||||
const response = (await workerDispatcher.dispatch(
|
||||
{
|
||||
id: `remote_${method}`,
|
||||
authToken: homeToken,
|
||||
method,
|
||||
params,
|
||||
orchestrationContractVersion: envelope?.orchestrationContractVersion,
|
||||
orchestrationRequestId: envelope?.orchestrationRequestId
|
||||
},
|
||||
{ authenticatedCallerFingerprint: homeFingerprint }
|
||||
)) as RuntimeRpcResponse<unknown>
|
||||
return response
|
||||
}
|
||||
}
|
||||
@@ -99,11 +103,6 @@ describe('orchestration federation control mail', () => {
|
||||
dispatchId = started.dispatch.id
|
||||
homeDb.markWorkerDispatchReady(dispatchId)
|
||||
|
||||
const homeFingerprint = authenticatedCallerFingerprint({
|
||||
id: 'home',
|
||||
authToken: homeToken,
|
||||
method: 'orchestration.federationImport'
|
||||
})
|
||||
workerDb.createRemoteDispatchAttachment({
|
||||
dispatchId,
|
||||
taskId: task.id,
|
||||
@@ -182,9 +181,7 @@ describe('orchestration federation control mail', () => {
|
||||
const waiting = workerDispatcher.dispatch(checkRequest('wait-for-control', true))
|
||||
await Promise.resolve()
|
||||
|
||||
const imported = await workerDispatcher.dispatch(
|
||||
importRequest('import-control', 1, 'relay-control')
|
||||
)
|
||||
const imported = await dispatchImport(importRequest('import-control', 1, 'relay-control'))
|
||||
|
||||
expect(imported).toMatchObject({
|
||||
ok: true,
|
||||
@@ -201,8 +198,8 @@ describe('orchestration federation control mail', () => {
|
||||
})
|
||||
|
||||
it('accepts a repeated import after a lost acknowledgment without duplicating mail', async () => {
|
||||
const first = await workerDispatcher.dispatch(importRequest('first-import', 1, 'relay-control'))
|
||||
const repeated = await workerDispatcher.dispatch(
|
||||
const first = await dispatchImport(importRequest('first-import', 1, 'relay-control'))
|
||||
const repeated = await dispatchImport(
|
||||
importRequest('repeated-import', 1, 'different-message-id')
|
||||
)
|
||||
|
||||
@@ -258,7 +255,7 @@ describe('orchestration federation control mail', () => {
|
||||
expect(workerDb.getUnreadMessages(`dispatch:${dispatchId}`)).toHaveLength(0)
|
||||
expect(homeDb.listPendingFederationRelay(dispatchId, 'to_worker')).toHaveLength(1)
|
||||
await expect(
|
||||
workerDispatcher.dispatch(importRequest('late-direct-import', 1, 'late-control'))
|
||||
dispatchImport(importRequest('late-direct-import', 1, 'late-control'))
|
||||
).resolves.toMatchObject({
|
||||
ok: false,
|
||||
error: { code: 'dispatch_inactive' }
|
||||
@@ -276,9 +273,7 @@ describe('orchestration federation control mail', () => {
|
||||
const statusWaiter = workerDispatcher.dispatch(checkRequest('wait-status', true, 30, 'status'))
|
||||
await Promise.resolve()
|
||||
|
||||
await workerDispatcher.dispatch(
|
||||
importRequest('import-escalation', 1, 'relay-escalation', 'escalation')
|
||||
)
|
||||
await dispatchImport(importRequest('import-escalation', 1, 'relay-escalation', 'escalation'))
|
||||
|
||||
await expect(escalationWaiter).resolves.toMatchObject({
|
||||
ok: true,
|
||||
@@ -342,4 +337,10 @@ describe('orchestration federation control mail', () => {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function dispatchImport(request: RpcRequest) {
|
||||
return workerDispatcher.dispatch(request, {
|
||||
authenticatedCallerFingerprint: homeFingerprint
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { mkdtempSync, rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { ORCHESTRATION_CONTRACT_VERSION } from '../../../../shared/protocol-version'
|
||||
import { OrcaRuntimeService } from '../../orca-runtime'
|
||||
@@ -13,6 +15,7 @@ describe('orchestration new-worktree workers', () => {
|
||||
let db: OrchestrationDb
|
||||
let runtime: OrcaRuntimeService
|
||||
let runId: string
|
||||
const paths: string[] = []
|
||||
|
||||
beforeEach(() => {
|
||||
db = new OrchestrationDb(':memory:')
|
||||
@@ -71,7 +74,12 @@ describe('orchestration new-worktree workers', () => {
|
||||
})
|
||||
})
|
||||
|
||||
afterEach(() => db.close())
|
||||
afterEach(() => {
|
||||
db.close()
|
||||
for (const path of paths.splice(0)) {
|
||||
rmSync(path, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
async function startWorker(overrides: Record<string, unknown> = {}) {
|
||||
const task = db.createTask({ spec: 'new-worktree task', runId })
|
||||
@@ -542,7 +550,7 @@ describe('orchestration new-worktree workers', () => {
|
||||
const pending = dispatcher.dispatch(request)
|
||||
await vi.waitFor(() => expect(db.getDispatchContext(task.id)).toBeDefined())
|
||||
const acceptedDispatch = db.getDispatchContext(task.id)!
|
||||
const callerFingerprint = createHash('sha256').update('caller-token').digest('hex')
|
||||
const callerFingerprint = db.getOrCreateLocalMutationCallerFingerprint()
|
||||
const receipt = db.getMutationReceipt(callerFingerprint, 'worker_start_request')
|
||||
|
||||
expect(receipt).toMatchObject({
|
||||
@@ -574,7 +582,17 @@ describe('orchestration new-worktree workers', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('replays a dispatch-input failure without creating another worker', async () => {
|
||||
it('replays a dispatch-input failure after restart without creating another worker', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'orca-worker-start-replay-'))
|
||||
paths.push(dir)
|
||||
db.close()
|
||||
db = new OrchestrationDb(join(dir, 'orchestration.db'))
|
||||
runtime.setOrchestrationDb(db)
|
||||
runId = db.createRun({
|
||||
objective: 'Recover dispatch input',
|
||||
coordinatorHandle: 'term_coord',
|
||||
coordinatorPaneKey
|
||||
}).id
|
||||
mockCreatedWorktree({ hookFound: false })
|
||||
vi.mocked(runtime.sendTerminalAgentPrompt).mockRejectedValueOnce(
|
||||
new Error('connection closed before dispatch input was accepted')
|
||||
@@ -597,7 +615,33 @@ describe('orchestration new-worktree workers', () => {
|
||||
}
|
||||
|
||||
const first = await dispatcher.dispatch(request)
|
||||
const replay = await dispatcher.dispatch({ ...request, id: 'rpc_worker_start_retry' })
|
||||
if (!first.ok) {
|
||||
throw new Error(`Initial worker start failed: ${first.error.code}`)
|
||||
}
|
||||
const firstReceipt = first.result as {
|
||||
dispatchId: string
|
||||
residualResources: unknown[]
|
||||
}
|
||||
db.close()
|
||||
|
||||
db = new OrchestrationDb(join(dir, 'orchestration.db'))
|
||||
const restartedRuntime = new OrcaRuntimeService()
|
||||
restartedRuntime.setOrchestrationDb(db)
|
||||
const recreateWorktree = vi
|
||||
.spyOn(restartedRuntime, 'createManagedWorktree')
|
||||
.mockRejectedValue(new Error('replay recreated the worktree'))
|
||||
const reinjectPrompt = vi
|
||||
.spyOn(restartedRuntime, 'sendTerminalAgentPrompt')
|
||||
.mockRejectedValue(new Error('replay reinjected the prompt'))
|
||||
const restartedDispatcher = new RpcDispatcher({
|
||||
runtime: restartedRuntime,
|
||||
methods: ORCHESTRATION_METHODS
|
||||
})
|
||||
const replay = await restartedDispatcher.dispatch({
|
||||
...request,
|
||||
id: 'rpc_worker_start_retry',
|
||||
authToken: 'caller-token-after-restart'
|
||||
})
|
||||
|
||||
expect(first).toMatchObject({
|
||||
ok: true,
|
||||
@@ -614,13 +658,17 @@ describe('orchestration new-worktree workers', () => {
|
||||
expect(replay).toMatchObject({
|
||||
ok: true,
|
||||
result: {
|
||||
dispatchId: firstReceipt.dispatchId,
|
||||
state: 'failed',
|
||||
failedStage: 'dispatch_input',
|
||||
residualResources: firstReceipt.residualResources,
|
||||
mutation: { requestId: 'worker_start_request', replayed: true }
|
||||
}
|
||||
})
|
||||
expect(runtime.createManagedWorktree).toHaveBeenCalledOnce()
|
||||
expect(runtime.sendTerminalAgentPrompt).toHaveBeenCalledOnce()
|
||||
expect(recreateWorktree).not.toHaveBeenCalled()
|
||||
expect(reinjectPrompt).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('persists pre-effect, post-effect, and post-input stages in order', async () => {
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { z } from 'zod'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { ORCHESTRATION_CONTRACT_VERSION } from '../../../shared/protocol-version'
|
||||
@@ -73,7 +72,7 @@ describe('orchestration contract fence', () => {
|
||||
}
|
||||
})
|
||||
expect(effect).not.toHaveBeenCalled()
|
||||
const callerFingerprint = createHash('sha256').update('caller-token').digest('hex')
|
||||
const callerFingerprint = database.getOrCreateLocalMutationCallerFingerprint()
|
||||
expect(database.getMutationReceipt(callerFingerprint, 'mutation_1')).toBeUndefined()
|
||||
}
|
||||
)
|
||||
|
||||
@@ -29,7 +29,8 @@ export class OrchestrationMutationExecutor {
|
||||
if (!requestId || !isOrchestrationMutation(request.method, params)) {
|
||||
return await invoke()
|
||||
}
|
||||
const callerFingerprint = callerFingerprintOverride ?? authenticatedCallerFingerprint(request)
|
||||
const callerFingerprint =
|
||||
callerFingerprintOverride ?? this.getLocalAuthenticatedCallerFingerprint()
|
||||
const payloadHash = createHash('sha256')
|
||||
.update(JSON.stringify(canonicalize({ method: request.method, params })))
|
||||
.digest('hex')
|
||||
@@ -109,6 +110,10 @@ export class OrchestrationMutationExecutor {
|
||||
this.inFlight.delete(key)
|
||||
}
|
||||
}
|
||||
|
||||
getLocalAuthenticatedCallerFingerprint(): string {
|
||||
return this.runtime.getOrchestrationDb().getOrCreateLocalMutationCallerFingerprint()
|
||||
}
|
||||
}
|
||||
|
||||
const executorsByRuntime = new WeakMap<OrcaRuntimeService, OrchestrationMutationExecutor>()
|
||||
@@ -125,12 +130,8 @@ export function getOrchestrationMutationExecutor(
|
||||
return executor
|
||||
}
|
||||
|
||||
export function authenticatedCallerFingerprint(request: RpcRequest): string {
|
||||
const callerToken =
|
||||
request.authToken ||
|
||||
(request as RpcRequest & { deviceToken?: string }).deviceToken ||
|
||||
'authenticated_transport'
|
||||
return createHash('sha256').update(callerToken).digest('hex')
|
||||
export function fingerprintAuthenticatedPairingDevice(deviceId: string): string {
|
||||
return createHash('sha256').update(`paired_device\0${deviceId}`).digest('hex')
|
||||
}
|
||||
|
||||
function canonicalize(value: unknown): unknown {
|
||||
|
||||
@@ -93,7 +93,7 @@ describe('durable orchestration mutation ledger', () => {
|
||||
db.close()
|
||||
})
|
||||
|
||||
it('applies the same ledger on authenticated WebSocket dispatch', async () => {
|
||||
it('keys WebSocket replay to the authenticated device across reconnects', async () => {
|
||||
const { db, dispatcher, effect } = createHarness()
|
||||
const replies: string[] = []
|
||||
const firstRequest = request({
|
||||
@@ -102,10 +102,23 @@ describe('durable orchestration mutation ledger', () => {
|
||||
subject: 'remote'
|
||||
}) as RpcRequest & { deviceToken?: string }
|
||||
firstRequest.authToken = ''
|
||||
firstRequest.deviceToken = 'paired-device'
|
||||
await dispatcher.dispatchStreaming(firstRequest, (reply) => replies.push(reply))
|
||||
const replayRequest = { ...firstRequest, id: 'rpc_2' }
|
||||
await dispatcher.dispatchStreaming(replayRequest, (reply) => replies.push(reply))
|
||||
firstRequest.deviceToken = 'untrusted-request-value-a'
|
||||
await dispatcher.dispatchStreaming(firstRequest, (reply) => replies.push(reply), {
|
||||
authenticatedCallerFingerprint: 'paired-device-a'
|
||||
})
|
||||
const replayRequest = {
|
||||
...firstRequest,
|
||||
id: 'rpc_2',
|
||||
deviceToken: 'untrusted-request-value-b'
|
||||
}
|
||||
await dispatcher.dispatchStreaming(replayRequest, (reply) => replies.push(reply), {
|
||||
authenticatedCallerFingerprint: 'paired-device-a'
|
||||
})
|
||||
await dispatcher.dispatchStreaming(
|
||||
{ ...replayRequest, id: 'rpc_3' },
|
||||
(reply) => replies.push(reply),
|
||||
{ authenticatedCallerFingerprint: 'paired-device-b' }
|
||||
)
|
||||
|
||||
expect(JSON.parse(replies[0] ?? '{}')).toMatchObject({
|
||||
ok: true,
|
||||
@@ -115,7 +128,12 @@ describe('durable orchestration mutation ledger', () => {
|
||||
ok: true,
|
||||
result: { mutation: { replayed: true } }
|
||||
})
|
||||
expect(effect).toHaveBeenCalledTimes(1)
|
||||
expect(JSON.parse(replies[2] ?? '{}')).toMatchObject({
|
||||
ok: true,
|
||||
result: { mutation: { replayed: false } }
|
||||
})
|
||||
expect(effect).toHaveBeenCalledTimes(2)
|
||||
expect(db.getInbox(10)).toHaveLength(2)
|
||||
db.close()
|
||||
})
|
||||
|
||||
@@ -178,10 +196,42 @@ describe('durable orchestration mutation ledger', () => {
|
||||
second.db.close()
|
||||
})
|
||||
|
||||
it('replays a local mutation after runtime authentication rotates', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'orca-mutation-ledger-'))
|
||||
paths.push(dir)
|
||||
const dbPath = join(dir, 'orchestration.db')
|
||||
const firstRuntime = createHarness(dbPath)
|
||||
const first = await firstRuntime.dispatcher.dispatch(
|
||||
request({
|
||||
rpcId: 'rpc_1',
|
||||
mutationId: 'mutation_restart',
|
||||
subject: 'once',
|
||||
authToken: 'before-restart'
|
||||
})
|
||||
)
|
||||
firstRuntime.db.close()
|
||||
|
||||
const restartedRuntime = createHarness(dbPath)
|
||||
const replay = await restartedRuntime.dispatcher.dispatch(
|
||||
request({
|
||||
rpcId: 'rpc_2',
|
||||
mutationId: 'mutation_restart',
|
||||
subject: 'once',
|
||||
authToken: 'after-restart'
|
||||
})
|
||||
)
|
||||
|
||||
expect(first).toMatchObject({ ok: true, result: { mutation: { replayed: false } } })
|
||||
expect(replay).toMatchObject({ ok: true, result: { mutation: { replayed: true } } })
|
||||
expect(firstRuntime.effect).toHaveBeenCalledOnce()
|
||||
expect(restartedRuntime.effect).not.toHaveBeenCalled()
|
||||
restartedRuntime.db.close()
|
||||
})
|
||||
|
||||
it('returns unknown for a pending receipt left by a previous process', async () => {
|
||||
const { db, dispatcher } = createHarness()
|
||||
db.beginMutationReceipt({
|
||||
callerFingerprint: createHash('sha256').update('caller-token').digest('hex'),
|
||||
callerFingerprint: db.getOrCreateLocalMutationCallerFingerprint(),
|
||||
requestId: 'mutation_1',
|
||||
method: 'orchestration.send',
|
||||
payloadHash: createHash('sha256')
|
||||
@@ -201,7 +251,7 @@ describe('durable orchestration mutation ledger', () => {
|
||||
const runtime = new OrcaRuntimeService()
|
||||
runtime.setOrchestrationDb(db)
|
||||
const params = { dispatch: 'ctx_release' }
|
||||
const callerFingerprint = createHash('sha256').update('caller-token').digest('hex')
|
||||
const callerFingerprint = db.getOrCreateLocalMutationCallerFingerprint()
|
||||
const payloadHash = createHash('sha256')
|
||||
.update(JSON.stringify({ method: 'orchestration.workerRelease', params }))
|
||||
.digest('hex')
|
||||
@@ -249,7 +299,7 @@ describe('durable orchestration mutation ledger', () => {
|
||||
const runtime = new OrcaRuntimeService()
|
||||
runtime.setOrchestrationDb(db)
|
||||
const params = { from: 'term_coord', task: db.createTask({ spec: 'restart' }).id }
|
||||
const callerFingerprint = createHash('sha256').update('caller-token').digest('hex')
|
||||
const callerFingerprint = db.getOrCreateLocalMutationCallerFingerprint()
|
||||
const payloadHash = createHash('sha256')
|
||||
.update(JSON.stringify({ method: 'orchestration.workerStart', params }))
|
||||
.digest('hex')
|
||||
|
||||
@@ -3483,6 +3483,64 @@ describe('OrcaRuntimeRpcServer', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('isolates mutation replay by the authenticated paired device across reconnects', async () => {
|
||||
const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-rpc-'))
|
||||
const runtime = new OrcaRuntimeService()
|
||||
const db = new OrchestrationDb(':memory:')
|
||||
runtime.setOrchestrationDb(db)
|
||||
const server = new OrcaRuntimeRpcServer({ runtime, userDataPath, enableWebSocket: false })
|
||||
server['deviceRegistry'] = new DeviceRegistry(userDataPath)
|
||||
const firstDevice = server['deviceRegistry']!.addDevice('first-cli', 'runtime')
|
||||
const secondDevice = server['deviceRegistry']!.addDevice('second-cli', 'runtime')
|
||||
|
||||
const resetMessages = async (id: string, authenticatedToken: string) => {
|
||||
const replies: Record<string, unknown>[] = []
|
||||
await server['handleWebSocketMessage'](
|
||||
JSON.stringify(
|
||||
withCurrentOrchestrationContract({
|
||||
id,
|
||||
method: 'orchestration.reset',
|
||||
orchestrationRequestId: 'paired-reset-request',
|
||||
params: { messages: true }
|
||||
})
|
||||
),
|
||||
(response) => replies.push(JSON.parse(response) as Record<string, unknown>),
|
||||
() => {},
|
||||
undefined,
|
||||
undefined,
|
||||
authenticatedToken
|
||||
)
|
||||
return replies[0]
|
||||
}
|
||||
|
||||
try {
|
||||
db.insertMessage({ from: 'worker', to: 'coordinator', subject: 'before reset' })
|
||||
const first = await resetMessages('reset-first', firstDevice.token)
|
||||
db.insertMessage({ from: 'worker', to: 'coordinator', subject: 'after reset' })
|
||||
const replay = await resetMessages('reset-replay', firstDevice.token)
|
||||
|
||||
expect(first).toMatchObject({
|
||||
ok: true,
|
||||
result: { reset: 'messages', mutation: { replayed: false } }
|
||||
})
|
||||
expect(replay).toMatchObject({
|
||||
ok: true,
|
||||
result: { reset: 'messages', mutation: { replayed: true } }
|
||||
})
|
||||
expect(db.getInbox()).toEqual([expect.objectContaining({ subject: 'after reset' })])
|
||||
|
||||
const isolated = await resetMessages('reset-second-device', secondDevice.token)
|
||||
expect(isolated).toMatchObject({
|
||||
ok: true,
|
||||
result: { reset: 'messages', mutation: { replayed: false } }
|
||||
})
|
||||
expect(db.getInbox()).toEqual([])
|
||||
} finally {
|
||||
db.close()
|
||||
await server.stop()
|
||||
}
|
||||
})
|
||||
|
||||
it('rejects unpaired terminal creates before runtime dispatch', async () => {
|
||||
const userDataPath = mkdtempSync(join(tmpdir(), 'orca-runtime-rpc-'))
|
||||
const createMobileSessionTerminal = vi.fn()
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
import { RpcDispatcher } from './rpc/dispatcher'
|
||||
import type { RpcRequest, RpcResponse } from './rpc/core'
|
||||
import { errorResponse } from './rpc/errors'
|
||||
import { fingerprintAuthenticatedPairingDevice } from './rpc/orchestration-mutation-executor'
|
||||
import type { RpcMessageContext, RpcTransport } from './rpc/transport'
|
||||
import { UnixSocketTransport } from './rpc/unix-socket-transport'
|
||||
import { WebSocketTransport } from './rpc/ws-transport'
|
||||
@@ -1711,6 +1712,7 @@ export class OrcaRuntimeRpcServer {
|
||||
: undefined
|
||||
try {
|
||||
await this.dispatcher.dispatchStreaming(request, replyForRequest, {
|
||||
authenticatedCallerFingerprint: fingerprintAuthenticatedPairingDevice(device.deviceId),
|
||||
connectionId,
|
||||
clientId: token,
|
||||
pairedDeviceId: device.deviceId,
|
||||
|
||||
Reference in New Issue
Block a user