mirror of
https://github.com/stablyai/orca.git
synced 2026-11-16 02:47:34 +00:00
fix(orchestration): a remote worker's question is answered only by its Run's coordinator; tests for the forwarder
This commit is contained in:
@@ -265,3 +265,22 @@ export function deferRemoteDispatchStop(
|
||||
record.stop_attempt
|
||||
)
|
||||
}
|
||||
|
||||
export type RemoteOwnedQuestionRow = {
|
||||
message_id: string
|
||||
conversation_id: string
|
||||
run_id: string
|
||||
environment_id: string
|
||||
dispatch_id: string
|
||||
}
|
||||
|
||||
/** A question a remote worker asked here, which its server answers. */
|
||||
export function getRemoteOwnedQuestion(
|
||||
db: OrchestrationDb,
|
||||
messageId: string
|
||||
): RemoteOwnedQuestionRow | undefined {
|
||||
// oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SELECT * over this table returns exactly its row shape.
|
||||
return db.db
|
||||
.prepare('SELECT * FROM remote_owned_questions WHERE message_id = ?')
|
||||
.get(messageId) as RemoteOwnedQuestionRow | undefined
|
||||
}
|
||||
|
||||
@@ -0,0 +1,307 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY,
|
||||
ORCHESTRATION_CONTRACT_VERSION,
|
||||
ORCHESTRATION_REMOTE_OWNED_WORKERS_RUNTIME_CAPABILITY
|
||||
} from '../../../../../shared/orchestration/orchestration-runtime-capabilities'
|
||||
import type {
|
||||
RuntimeOrchestrationEnvelope,
|
||||
RuntimeRpcResponse
|
||||
} from '../../../../../shared/runtime-rpc-envelope'
|
||||
import { OrcaRuntimeService } from '../../../orca-runtime'
|
||||
import { RpcDispatcher } from '../../../rpc/dispatcher'
|
||||
import type { RpcResponse } from '../../../rpc/core'
|
||||
import { ORCHESTRATION_METHODS } from '../../../rpc/methods/orchestration/orchestration'
|
||||
import { OrchestrationDb } from '../../db/orchestration-db'
|
||||
import { getRemoteDispatch } from '../../db/remote-owned/remote-dispatch-records'
|
||||
import { importRemoteOwnedItem } from '../../db/remote-owned/remote-owned-import'
|
||||
import { getRemotePlacement } from '../../db/remote-owned/remote-placements'
|
||||
import { remoteServerPin } from '../../db/remote-owned/remote-run-links'
|
||||
import type {
|
||||
OrchestrationEnvironmentTransport,
|
||||
OrchestrationWorkerServer
|
||||
} from '../environment-transport'
|
||||
|
||||
const COORDINATOR_PANE = 'tab_coord:11111111-1111-4111-8111-111111111111'
|
||||
const OTHER_PANE = 'tab_other:22222222-2222-4222-8222-222222222222'
|
||||
const REQUEST = '6f1c2f1e-3a7b-4d55-9c1e-0a1b2c3d4e5f'
|
||||
|
||||
type Sent = {
|
||||
method: string
|
||||
params: Record<string, unknown>
|
||||
envelope?: RuntimeOrchestrationEnvelope
|
||||
}
|
||||
type Answer = { result: unknown } | { error: string; data?: unknown } | 'unreachable'
|
||||
|
||||
describe('commands about a worker a paired server owns', () => {
|
||||
let db: OrchestrationDb
|
||||
let runtime: OrcaRuntimeService
|
||||
let dispatcher: RpcDispatcher
|
||||
let sent: Sent[]
|
||||
let answers: Record<string, Answer[]>
|
||||
let capabilities: string[]
|
||||
let sequence = 0
|
||||
const server: OrchestrationWorkerServer = {
|
||||
environmentId: 'env_b',
|
||||
name: 'devB',
|
||||
peerFingerprint: 'host_b',
|
||||
homePeerFingerprint: 'cred_a',
|
||||
pairingRevision: 1
|
||||
}
|
||||
|
||||
const transport = (): OrchestrationEnvironmentTransport => ({
|
||||
resolve: () => server,
|
||||
hasEnvironment: () => true,
|
||||
call: async (_selector, method, params, _timeout, envelope, _revision, send) => {
|
||||
const ok = (result: unknown): RuntimeRpcResponse<unknown> => ({
|
||||
id: method,
|
||||
ok: true,
|
||||
result,
|
||||
_meta: { runtimeId: 'runtime_b' }
|
||||
})
|
||||
if (method === 'status.get') {
|
||||
return ok({ runtimeId: 'runtime_b', capabilities })
|
||||
}
|
||||
if (method === 'orchestration.peerMailPull') {
|
||||
return ok({ items: [] })
|
||||
}
|
||||
const prepared = send?.prepareRequest?.({ params, envelope, measure: () => 0 })
|
||||
sent.push({
|
||||
method,
|
||||
params: (prepared?.params ?? params) as Record<string, unknown>,
|
||||
envelope: prepared?.envelope ?? envelope
|
||||
})
|
||||
const answer = answers[method]?.shift() ?? 'unreachable'
|
||||
if (answer === 'unreachable') {
|
||||
throw Object.assign(new Error('Could not connect to the remote Orca runtime.'), {
|
||||
code: 'remote_runtime_unavailable'
|
||||
})
|
||||
}
|
||||
return 'result' in answer
|
||||
? ok(answer.result)
|
||||
: {
|
||||
id: method,
|
||||
ok: false,
|
||||
error: { code: answer.error, message: answer.error, data: answer.data }
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
beforeEach(() => {
|
||||
sent = []
|
||||
answers = {}
|
||||
capabilities = [
|
||||
ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY,
|
||||
ORCHESTRATION_REMOTE_OWNED_WORKERS_RUNTIME_CAPABILITY
|
||||
]
|
||||
db = new OrchestrationDb(':memory:')
|
||||
runtime = new OrcaRuntimeService(null, undefined, {
|
||||
orchestrationEnvironmentTransport: transport()
|
||||
})
|
||||
vi.spyOn(runtime, 'readMachineName').mockReturnValue('Mac A')
|
||||
const panes: Record<string, string> = { term_coord: COORDINATOR_PANE, term_other: OTHER_PANE }
|
||||
vi.spyOn(runtime, 'getTerminalPaneKey').mockImplementation((handle) => panes[handle] ?? null)
|
||||
vi.spyOn(runtime, 'getLiveTerminalPaneKey').mockImplementation(
|
||||
(handle) => panes[handle] ?? null
|
||||
)
|
||||
runtime.setOrchestrationDb(db)
|
||||
dispatcher = new RpcDispatcher({ runtime, methods: ORCHESTRATION_METHODS })
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
runtime.stopOrchestrationFederationRelay()
|
||||
db.close()
|
||||
vi.restoreAllMocks()
|
||||
})
|
||||
|
||||
function call(
|
||||
method: string,
|
||||
params: Record<string, unknown>,
|
||||
{ pane = COORDINATOR_PANE, requestId }: { pane?: string; requestId?: string } = {}
|
||||
): Promise<RpcResponse> {
|
||||
sequence += 1
|
||||
return dispatcher.dispatch({
|
||||
id: `call_${sequence}`,
|
||||
authToken: 'token',
|
||||
method,
|
||||
params,
|
||||
orchestrationContractVersion: ORCHESTRATION_CONTRACT_VERSION,
|
||||
orchestrationCompatibilityEvidence: { paneKey: pane },
|
||||
...(requestId ? { orchestrationRequestId: requestId } : {})
|
||||
})
|
||||
}
|
||||
|
||||
const started = (dispatchId = 'ctx_b1'): Answer => ({
|
||||
result: { dispatchId, runId: 'run_on_b', run: { id: 'run_on_b' }, state: 'ready' }
|
||||
})
|
||||
|
||||
async function startOnB(requestId = REQUEST) {
|
||||
answers['orchestration.workerStart'] = [started()]
|
||||
const response = await call(
|
||||
'orchestration.workerStart',
|
||||
{ from: 'term_coord', on: 'devB', spec: 'build it', worktree: 'new-top-level' },
|
||||
{ requestId }
|
||||
)
|
||||
if (!response.ok) {
|
||||
throw new Error(response.error.message)
|
||||
}
|
||||
return response.result as { taskId: string; runId: string; dispatchId: string }
|
||||
}
|
||||
|
||||
it("starts the worker on a server that owns its workers, then records it under this Orca's Task", async () => {
|
||||
const receipt = await startOnB()
|
||||
|
||||
const start = sent.find((s) => s.method === 'orchestration.workerStart')!
|
||||
expect(start.params).toMatchObject({ spec: 'build it', adoptTaskId: receipt.taskId })
|
||||
expect(start.params).not.toHaveProperty('from')
|
||||
expect(start.envelope?.orchestrationPeerRun?.label).toBe('Mac A')
|
||||
expect(receipt.runId).not.toBe('run_on_b')
|
||||
expect(db.getTask(receipt.taskId)).toMatchObject({
|
||||
run_id: receipt.runId,
|
||||
status: 'dispatched'
|
||||
})
|
||||
expect(getRemoteDispatch(db, 'ctx_b1')).toMatchObject({ task_id: receipt.taskId })
|
||||
})
|
||||
|
||||
it("keeps today's remote attachment for a server without the capability", async () => {
|
||||
capabilities = [ORCHESTRATION_CONTRACT_RUNTIME_CAPABILITY]
|
||||
await call(
|
||||
'orchestration.workerStart',
|
||||
{ from: 'term_coord', on: 'devB', spec: 'build it', worktree: 'new-top-level' },
|
||||
{ requestId: REQUEST }
|
||||
)
|
||||
|
||||
expect(sent.some((s) => s.method === 'orchestration.workerStart')).toBe(false)
|
||||
expect(db.db.prepare('SELECT COUNT(*) AS n FROM remote_placements').get()).toEqual({ n: 0 })
|
||||
})
|
||||
|
||||
it('frees the Task when the server refuses the start, and keeps it fenced when the answer is lost', async () => {
|
||||
answers['orchestration.workerStart'] = [
|
||||
{ error: 'invalid_argument', data: { effectsApplied: false } }
|
||||
]
|
||||
await call(
|
||||
'orchestration.workerStart',
|
||||
{ from: 'term_coord', on: 'devB', spec: 'refused', worktree: 'current' },
|
||||
{ requestId: REQUEST }
|
||||
)
|
||||
const refusedTask = String((sent[0].params as { adoptTaskId: string }).adoptTaskId)
|
||||
expect(getRemotePlacement(db, refusedTask)).toBeUndefined()
|
||||
|
||||
const lost = await call(
|
||||
'orchestration.workerStart',
|
||||
{ from: 'term_coord', on: 'devB', spec: 'lost', worktree: 'new-top-level' },
|
||||
{ requestId: 'a3f1e9d2-5b6c-4d7e-8f90-1a2b3c4d5e6f' }
|
||||
)
|
||||
expect(lost.ok ? undefined : lost.error.code).toBe('remote_runtime_unavailable')
|
||||
const lostTask = String((sent[1].params as { adoptTaskId: string }).adoptTaskId)
|
||||
expect(getRemotePlacement(db, lostTask)).toMatchObject({
|
||||
state: 'reserved',
|
||||
outcome_unknown: 1
|
||||
})
|
||||
})
|
||||
|
||||
it('scopes the retry key to the caller: two terminals reusing an ID never collide, one retrying gets its own key', async () => {
|
||||
const receipt = await startOnB()
|
||||
answers['orchestration.workerRelease'] = [
|
||||
{ result: { state: 'released' } },
|
||||
{ result: { state: 'released' } },
|
||||
{ result: { state: 'released' } }
|
||||
]
|
||||
const release = (pane: string) =>
|
||||
call('orchestration.workerRelease', { dispatch: 'ctx_b1' }, { pane, requestId: REQUEST })
|
||||
await release(COORDINATOR_PANE)
|
||||
await release(COORDINATOR_PANE)
|
||||
await release(OTHER_PANE)
|
||||
|
||||
const keys = sent
|
||||
.filter((s) => s.method === 'orchestration.workerRelease')
|
||||
.map((s) => s.envelope?.orchestrationRequestId)
|
||||
expect(keys[0]).toBe(keys[1])
|
||||
expect(keys[2]).not.toBe(keys[0])
|
||||
expect(keys[0]).not.toBe(REQUEST)
|
||||
expect(receipt.dispatchId).toBe('ctx_b1')
|
||||
})
|
||||
|
||||
it("answers in the caller's terms: its own request ID and its own Run, never the server's", async () => {
|
||||
const receipt = await startOnB()
|
||||
answers['orchestration.workerShow'] = []
|
||||
const key = () =>
|
||||
sent.filter((s) => s.method === 'orchestration.workerRelease').at(-1)?.envelope
|
||||
?.orchestrationRequestId
|
||||
answers['orchestration.workerRelease'] = [{ result: { state: 'released' } }]
|
||||
await call('orchestration.workerRelease', { dispatch: 'ctx_b1' }, { requestId: REQUEST })
|
||||
answers['orchestration.workerRelease'] = [
|
||||
{
|
||||
error: 'operation_unknown',
|
||||
data: { requestId: key(), runId: 'run_on_b' }
|
||||
}
|
||||
]
|
||||
const unknown = await call(
|
||||
'orchestration.workerRelease',
|
||||
{ dispatch: 'ctx_b1' },
|
||||
{ requestId: REQUEST }
|
||||
)
|
||||
|
||||
expect(unknown.ok ? undefined : unknown.error.data).toMatchObject({
|
||||
requestId: REQUEST,
|
||||
runId: receipt.runId,
|
||||
server: 'devB'
|
||||
})
|
||||
})
|
||||
|
||||
it("answers a remote worker's question only from its Run's coordinator, as a local one is", async () => {
|
||||
const receipt = await startOnB()
|
||||
const link = db.db.prepare('SELECT * FROM remote_run_links').get() as {
|
||||
conversation_id: string
|
||||
}
|
||||
importRemoteOwnedItem(db, remoteServerPin(server), {
|
||||
conversationId: link.conversation_id,
|
||||
sequence: 1,
|
||||
messageId: 'msg_q1',
|
||||
kind: 'worker_mail',
|
||||
payload: JSON.stringify({
|
||||
v: 1,
|
||||
id: 'msg_q1',
|
||||
homeRunId: receipt.runId,
|
||||
peerRunId: 'run_on_b',
|
||||
message: {
|
||||
from: 'term_w',
|
||||
type: 'question',
|
||||
subject: 'Which color?',
|
||||
body: 'Which color?',
|
||||
payload: null,
|
||||
priority: 'normal',
|
||||
threadId: null
|
||||
},
|
||||
dispatchId: 'ctx_b1',
|
||||
taskId: receipt.taskId
|
||||
})
|
||||
})
|
||||
answers['orchestration.reply'] = [{ result: { messageId: 'msg_a1' } }]
|
||||
|
||||
const stranger = await call(
|
||||
'orchestration.reply',
|
||||
{ id: 'msg_q1', body: 'red', from: 'term_other' },
|
||||
{ pane: OTHER_PANE }
|
||||
)
|
||||
const coordinator = await call('orchestration.reply', {
|
||||
id: 'msg_q1',
|
||||
body: 'teal',
|
||||
from: 'term_coord'
|
||||
})
|
||||
|
||||
expect(stranger.ok).toBe(false)
|
||||
expect(coordinator.ok).toBe(true)
|
||||
expect(
|
||||
sent.filter((s) => s.method === 'orchestration.reply').map((s) => s.params.body)
|
||||
).toEqual(['teal'])
|
||||
})
|
||||
|
||||
it('owes a stop the server did not answer and says so', async () => {
|
||||
await startOnB()
|
||||
const stop = await call('orchestration.workerStop', { dispatch: 'ctx_b1' })
|
||||
|
||||
expect(stop.ok ? stop.result : undefined).toMatchObject({ state: 'stop_requested' })
|
||||
expect(getRemoteDispatch(db, 'ctx_b1')).toMatchObject({ stop_owed: 1 })
|
||||
})
|
||||
})
|
||||
@@ -12,6 +12,7 @@ import type { OrchestrationDb } from '../../db/orchestration-db'
|
||||
import { runLifecycleWriteTransaction } from '../../db/lifecycle-write-transaction-runner'
|
||||
import {
|
||||
getRemoteDispatch,
|
||||
getRemoteOwnedQuestion,
|
||||
oweRemoteDispatchStop,
|
||||
remoteStopKey
|
||||
} from '../../db/remote-owned/remote-dispatch-records'
|
||||
@@ -179,13 +180,7 @@ function remoteSubject(
|
||||
? value.slice('dispatch:'.length)
|
||||
: undefined
|
||||
const record = dispatchId ? getRemoteDispatch(db, dispatchId) : undefined
|
||||
const question =
|
||||
kind === 'message' && value
|
||||
? // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: SELECT of one column returns exactly that column.
|
||||
(db.db
|
||||
.prepare('SELECT conversation_id FROM remote_owned_questions WHERE message_id = ?')
|
||||
.get(value) as { conversation_id: string } | undefined)
|
||||
: undefined
|
||||
const question = kind === 'message' && value ? getRemoteOwnedQuestion(db, value) : undefined
|
||||
const conversationId = record?.conversation_id ?? question?.conversation_id
|
||||
const link = conversationId ? getRemoteRunLink(db, conversationId) : undefined
|
||||
if (link) {
|
||||
|
||||
@@ -10,6 +10,7 @@ import type { ReplyParams } from '../../../../../../shared/rpc-contract/orchestr
|
||||
import { resolveRunScope } from '../runs/run-scope'
|
||||
import type { OrchestrationAccessRuleInput } from '../access/orchestration-method-access'
|
||||
import { helperQuestionAnswerer } from './helper-question-answerer'
|
||||
import { getRemoteOwnedQuestion } from '../../../../orchestration/db/remote-owned/remote-dispatch-records'
|
||||
|
||||
/** Who may answer: the helper a question was asked of, the Run's current consumer, or anyone for plain mail. */
|
||||
export type ReplyAccess =
|
||||
@@ -21,6 +22,8 @@ export type ReplyAccess =
|
||||
}
|
||||
| { kind: 'run'; original: MessageRow; question: QuestionRow; run: RunRow }
|
||||
| { kind: 'message'; original: MessageRow }
|
||||
/** A question a worker on a paired server asked; the reply goes to that server. */
|
||||
| { kind: 'remote'; original: MessageRow; run: RunRow }
|
||||
|
||||
export function replyAccess({
|
||||
params,
|
||||
@@ -56,6 +59,22 @@ export function replyAccess({
|
||||
)
|
||||
}
|
||||
const question = db.getQuestion(params.id)
|
||||
const remote = question ? undefined : getRemoteOwnedQuestion(db, params.id)
|
||||
if (remote) {
|
||||
// Answered by its Run's coordinator, as a question from a worker here is.
|
||||
const run = resolveRunScope(runtime, {
|
||||
runId: params.run ?? remote.run_id,
|
||||
callerTerminalHandle: caller,
|
||||
requireCurrentConsumer: true,
|
||||
legacyCoordinatorRunId,
|
||||
callerEvidence: ctx.orchestrationCompatibilityEvidence,
|
||||
callerSession: ctx.orchestrationCaller
|
||||
})
|
||||
if (run.id !== remote.run_id) {
|
||||
throw new Error(`Message not found: ${params.id}`)
|
||||
}
|
||||
return { kind: 'remote', original, run }
|
||||
}
|
||||
if (!question) {
|
||||
return { kind: 'message', original }
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user