mirror of
https://github.com/stablyai/orca.git
synced 2026-11-17 03:15:01 +00:00
fix(orchestration): let a restart-reissued worker's plain check take its answer, and a working local owner outrank finished remote history
A recovered federated worker's default (nonblocking) ask delivered its answer to Dispatch mail, but the plain check its wake types with the stale env handle failed the attachment's process fence and left the answer unread. check now resolves the caller through the same exact-incarnation recovery as ask and the reports, at entry and at each revalidation, so replacement and pane turnover still refuse. A worker given local work after its remote Dispatch finished had its default worker_done refused as run_home_unreachable by the finished attachment. The refusal now yields to a working local Dispatch and still refuses mail that names the finished remote Dispatch. The recovered reporter's admission re-read also pins the entry incarnation.
This commit is contained in:
@@ -153,25 +153,33 @@ export function resolveWorkerReporterIdentity(
|
||||
db: OrchestrationDb,
|
||||
args: {
|
||||
from: string
|
||||
attested:
|
||||
| Pick<OrchestrationCompatibilityCallerAuthority, 'paneKey' | 'processIncarnation'>
|
||||
/** The hook-attested caller, which speaks for `from` only when it names that same handle. */
|
||||
authority:
|
||||
| Pick<
|
||||
OrchestrationCompatibilityCallerAuthority,
|
||||
'terminalHandle' | 'paneKey' | 'processIncarnation'
|
||||
>
|
||||
| undefined
|
||||
hasSessionCaller: boolean
|
||||
evidence: OrchestrationCompatibilityEvidence | undefined
|
||||
}
|
||||
): { paneKey: string | null; processIncarnation: () => string | null } {
|
||||
const { from, attested } = args
|
||||
const { from } = args
|
||||
// Why: attested hook identity survives graph remount; caller params never supply lifecycle authority.
|
||||
const attested = args.authority?.terminalHandle === from ? args.authority : undefined
|
||||
const declaredPaneKey = attested?.paneKey ?? runtime.getTerminalPaneKey(from)
|
||||
const remint = () =>
|
||||
args.hasSessionCaller || attested || declaredPaneKey
|
||||
? null
|
||||
: resolveRemintedWorkerReporter(runtime, db, { from, evidence: args.evidence })
|
||||
const reminted = remint()
|
||||
// The re-read must still speak for the same Dispatch and pane that routing already used.
|
||||
// The re-read must still speak for the same Dispatch, pane and process that routing already used, so a
|
||||
// record rewritten to another incarnation after entry cannot carry the request over to it.
|
||||
const remintedAgain = (entry: RemintedWorkerReporter): string | null => {
|
||||
const again = remint()
|
||||
return again?.dispatchId === entry.dispatchId &&
|
||||
isEquivalentPaneKey(again.paneKey, entry.paneKey)
|
||||
isEquivalentPaneKey(again.paneKey, entry.paneKey) &&
|
||||
again.processIncarnation === entry.processIncarnation
|
||||
? again.processIncarnation
|
||||
: null
|
||||
}
|
||||
|
||||
@@ -56,7 +56,7 @@ export const ORCHESTRATION_ASK_METHODS = [
|
||||
// A runtime restart can reissue a surviving worker PTY's handle; recover it by exact incarnation.
|
||||
const reporter = resolveWorkerReporterIdentity(runtime, db, {
|
||||
from,
|
||||
attested: undefined,
|
||||
authority: undefined,
|
||||
hasSessionCaller: Boolean(orchestrationCaller),
|
||||
evidence: orchestrationCompatibilityEvidence
|
||||
})
|
||||
@@ -101,7 +101,7 @@ export const ORCHESTRATION_ASK_METHODS = [
|
||||
'question'
|
||||
)
|
||||
if (!askingDispatch) {
|
||||
refuseFinishedRemoteWorker(db, paneKey, undefined)
|
||||
refuseFinishedRemoteWorker(db, { address: from, paneKey }, undefined)
|
||||
throw new OrchestrationError(
|
||||
'dispatch_inactive',
|
||||
'ask requires an active supervised Dispatch.'
|
||||
|
||||
@@ -10,6 +10,7 @@ import { withRunCheckNext } from '../receipt-next'
|
||||
import { orchestrationSkillRecoveryData } from '../../../../../../shared/orchestration/orchestration-rpc-contract'
|
||||
import { hasRunBindingKey } from '../../../../orchestration/addressing/orchestration-caller-identity'
|
||||
import { orchestrationCallerIdentity } from '../runs/run-scope'
|
||||
import { resolveWorkerReporterIdentity } from '../../../../orchestration/workers/reports/reminted-worker-reporter'
|
||||
import {
|
||||
callerHoldsDispatchPane,
|
||||
dispatchFenced,
|
||||
@@ -44,11 +45,19 @@ export const ORCHESTRATION_CHECK_METHODS = [
|
||||
const handle = params.terminal ?? 'unknown'
|
||||
const typeFilter = parseMessageTypes(params.types)
|
||||
|
||||
// A restart can reissue a surviving worker PTY's handle; the woken worker still checks with its env
|
||||
// handle. Recover it by the Dispatch's exact recorded incarnation, as its ask and reports were.
|
||||
const reporter = resolveWorkerReporterIdentity(runtime, db, {
|
||||
from: handle,
|
||||
authority: undefined,
|
||||
hasSessionCaller: Boolean(orchestrationCaller),
|
||||
evidence: orchestrationCompatibilityEvidence
|
||||
})
|
||||
const caller = orchestrationCallerIdentity(runtime, {
|
||||
handle,
|
||||
session: orchestrationCaller,
|
||||
// Why: a live runtime handle is authoritative; pane metadata is only the restart fallback.
|
||||
paneKey: runtime.getTerminalPaneKey(handle) ?? params.terminalPaneKey
|
||||
paneKey: reporter.paneKey ?? params.terminalPaneKey
|
||||
})
|
||||
const paneKey = caller.paneKey ?? undefined
|
||||
const boundRun = hasRunBindingKey(caller) ? db.getCurrentRunForCoordinator(caller) : undefined
|
||||
@@ -94,7 +103,7 @@ export const ORCHESTRATION_CHECK_METHODS = [
|
||||
!db.isRemoteAttachmentProcessCurrent({
|
||||
dispatchId: remoteAttachment.dispatch_id,
|
||||
paneKey: paneKey ?? null,
|
||||
processIncarnation: runtime.getTerminalProcessIncarnation(handle)
|
||||
processIncarnation: reporter.processIncarnation()
|
||||
})
|
||||
) {
|
||||
throw new OrchestrationError(
|
||||
@@ -114,6 +123,7 @@ export const ORCHESTRATION_CHECK_METHODS = [
|
||||
signal,
|
||||
activeDispatch,
|
||||
remoteAttachment,
|
||||
processIncarnation: reporter.processIncarnation,
|
||||
recordMutationReceipt,
|
||||
acknowledgeOnly: !params.wait
|
||||
})
|
||||
|
||||
@@ -28,6 +28,8 @@ export async function checkWorkerMailbox(args: {
|
||||
signal: AbortSignal | undefined
|
||||
activeDispatch: ActiveDispatch | undefined
|
||||
remoteAttachment: RemoteAttachment | undefined
|
||||
/** The caller's process, re-read at each fence; a restart-reissued handle supplies its recovered one. */
|
||||
processIncarnation?: () => string | null
|
||||
wakeTypes?: MessageType[]
|
||||
revalidateConsumer?: () => void
|
||||
deferDelivery?: () => boolean
|
||||
@@ -100,7 +102,9 @@ export async function checkWorkerMailbox(args: {
|
||||
db.isRemoteAttachmentProcessCurrent({
|
||||
dispatchId: current.dispatch_id,
|
||||
paneKey,
|
||||
processIncarnation: runtime.getTerminalProcessIncarnation(handle)
|
||||
processIncarnation: args.processIncarnation
|
||||
? args.processIncarnation()
|
||||
: runtime.getTerminalProcessIncarnation(handle)
|
||||
})
|
||||
) {
|
||||
return
|
||||
|
||||
@@ -85,17 +85,30 @@ export function finishedWorkerSend<P extends { to?: string; type?: string }>(
|
||||
}
|
||||
}
|
||||
|
||||
/** A finished worker on a paired host cannot reach its coordinator yet; say why instead of a bare refusal. */
|
||||
/** A finished worker on a paired host cannot reach its coordinator yet; say why instead of a bare refusal.
|
||||
* A worker since given local work is no longer that finished worker: its default mail goes to its
|
||||
* working Dispatch, so only mail naming the finished remote Dispatch is refused. */
|
||||
export function refuseFinishedRemoteWorker(
|
||||
db: OrchestrationDb,
|
||||
paneKey: string | undefined,
|
||||
sender: { address: string; paneKey: string | undefined },
|
||||
to: string | undefined
|
||||
): void {
|
||||
const { address, paneKey } = sender
|
||||
const attachment = paneKey ? db.findLatestRemoteAttachmentForPane(paneKey) : undefined
|
||||
if (attachment && (!to || to === `dispatch:${attachment.dispatch_id}`)) {
|
||||
throw new OrchestrationError(
|
||||
'run_home_unreachable',
|
||||
`Dispatch ${attachment.dispatch_id} has finished, and its coordinator runs on another computer that a finished worker cannot reach yet. Nothing was sent; give the message to the user instead.`
|
||||
)
|
||||
if (!attachment) {
|
||||
return
|
||||
}
|
||||
if (to === `dispatch:${attachment.dispatch_id}`) {
|
||||
throw finishedRemoteUnreachable(attachment.dispatch_id)
|
||||
}
|
||||
if (!to && !db.getActiveDispatchForIdentity(address, paneKey)) {
|
||||
throw finishedRemoteUnreachable(attachment.dispatch_id)
|
||||
}
|
||||
}
|
||||
|
||||
function finishedRemoteUnreachable(dispatchId: string): OrchestrationError {
|
||||
return new OrchestrationError(
|
||||
'run_home_unreachable',
|
||||
`Dispatch ${dispatchId} has finished, and its coordinator runs on another computer that a finished worker cannot reach yet. Nothing was sent; give the message to the user instead.`
|
||||
)
|
||||
}
|
||||
|
||||
+213
@@ -6,6 +6,7 @@ import type { OrchestrationDb } from '../../../../orchestration/db/orchestration
|
||||
import type { OrcaRuntimeService } from '../../../../orca-runtime'
|
||||
import { createOrchestrationRpcHarness } from '../rpc-test-harness'
|
||||
import { createRootDispatch } from '../../../../orchestration/db/root-dispatch-test-fixture'
|
||||
import { resolveWorkerReporterIdentity } from '../../../../orchestration/workers/reports/reminted-worker-reporter'
|
||||
|
||||
const STALE_HANDLE = 'term_exported-before-restart'
|
||||
const REMINTED_HANDLE = 'term_reissued-after-restart'
|
||||
@@ -116,6 +117,48 @@ describe('restart-reissued worker handle across finished and federated Dispatche
|
||||
workerEnv()
|
||||
)
|
||||
|
||||
/** The plain `orchestration check` an Orca wake types: the env handle and pane, nothing else. */
|
||||
const check = (from: string = STALE_HANDLE, env: RpcContext = workerEnv()) =>
|
||||
harness.call('orchestration.check', { terminal: from, terminalPaneKey: WORKER_PANE }, env)
|
||||
|
||||
const askedId = (asked: unknown): string =>
|
||||
asked && typeof asked === 'object' && 'messageId' in asked ? String(asked.messageId) : ''
|
||||
|
||||
const checkedBodies = (checked: unknown): string[] =>
|
||||
checked &&
|
||||
typeof checked === 'object' &&
|
||||
'messages' in checked &&
|
||||
Array.isArray(checked.messages)
|
||||
? checked.messages.map((message: { body?: unknown }) => String(message.body))
|
||||
: []
|
||||
|
||||
/** The Run home answers the worker's question; the answer lands in its Dispatch mail. */
|
||||
const importRemoteAnswer = (questionId: string) =>
|
||||
harness.call(
|
||||
'orchestration.federationImport',
|
||||
{
|
||||
dispatchId: REMOTE_DISPATCH,
|
||||
items: [
|
||||
{
|
||||
dispatch_id: REMOTE_DISPATCH,
|
||||
direction: 'to_worker',
|
||||
sequence: 1,
|
||||
message_id: 'relay_answer',
|
||||
kind: 'reply',
|
||||
payload: JSON.stringify({
|
||||
questionId,
|
||||
answerMessageId: 'msg_remote_answer',
|
||||
body: 'Approved'
|
||||
})
|
||||
}
|
||||
]
|
||||
},
|
||||
{ ...ctx, authenticatedCallerFingerprint: 'home_peer' }
|
||||
)
|
||||
|
||||
const remoteAnswerUnread = () =>
|
||||
db.getUnreadMessages(`dispatch:${REMOTE_DISPATCH}`).some((m) => m.body === 'Approved')
|
||||
|
||||
const remoteRelayKinds = () =>
|
||||
db
|
||||
.listFederationRelay({ dispatchId: REMOTE_DISPATCH, direction: 'to_home', afterSequence: 0 })
|
||||
@@ -158,6 +201,42 @@ describe('restart-reissued worker handle across finished and federated Dispatche
|
||||
})
|
||||
})
|
||||
|
||||
describe('working local worker', () => {
|
||||
it('consumes its default question answer with the plain check its wake types', async () => {
|
||||
setupLocalWorker()
|
||||
const asked = await ask({ question: 'Proceed?', wait: false })
|
||||
expect(asked).toMatchObject({ pending: true })
|
||||
await harness.call(
|
||||
'orchestration.reply',
|
||||
{ id: askedId(asked), body: 'Approved locally', from: 'term_coord' },
|
||||
ctx
|
||||
)
|
||||
|
||||
expect(checkedBodies(await check())).toContain('Approved locally')
|
||||
})
|
||||
})
|
||||
|
||||
describe('admission re-read', () => {
|
||||
it('refuses a Dispatch record rewritten to another incarnation after entry', () => {
|
||||
const { dispatchId } = setupLocalWorker()
|
||||
const reporter = resolveWorkerReporterIdentity(runtime, db, {
|
||||
from: STALE_HANDLE,
|
||||
authority: undefined,
|
||||
hasSessionCaller: false,
|
||||
evidence: workerEnv().orchestrationCompatibilityEvidence
|
||||
})
|
||||
expect(reporter.processIncarnation()).toBe(INCARNATION)
|
||||
|
||||
// Record and pane agree on a new process, but it is not the one this request entered as.
|
||||
db.db
|
||||
.prepare('UPDATE dispatch_contexts SET process_incarnation = ? WHERE id = ?')
|
||||
.run(REPLACEMENT, dispatchId)
|
||||
paneIncarnation = REPLACEMENT
|
||||
|
||||
expect(reporter.processIncarnation()).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
describe('worker for another computer', () => {
|
||||
it('relays an escalation and a question to the Run home through its stale handle', async () => {
|
||||
setupFederatedWorker()
|
||||
@@ -178,6 +257,140 @@ describe('restart-reissued worker handle across finished and federated Dispatche
|
||||
expect(remoteRelayKinds()).toEqual([])
|
||||
})
|
||||
|
||||
describe('answer to a default (nonblocking) question', () => {
|
||||
it('is consumed by the plain check its wake types with the stale env handle', async () => {
|
||||
setupFederatedWorker()
|
||||
const asked = await ask({ question: 'Proceed?', wait: false })
|
||||
expect(asked).toMatchObject({ pending: true })
|
||||
await importRemoteAnswer(askedId(asked))
|
||||
expect(remoteAnswerUnread()).toBe(true)
|
||||
|
||||
const checked = await check()
|
||||
|
||||
expect(checkedBodies(checked)).toContain('Approved')
|
||||
expect(remoteAnswerUnread()).toBe(false)
|
||||
})
|
||||
|
||||
it('stays unread when the pane now records a replacement incarnation', async () => {
|
||||
setupFederatedWorker()
|
||||
const asked = await ask({ question: 'Proceed?', wait: false })
|
||||
await importRemoteAnswer(askedId(asked))
|
||||
paneIncarnation = REPLACEMENT
|
||||
|
||||
await expect(check()).rejects.toMatchObject({ code: 'dispatch_inactive' })
|
||||
expect(remoteAnswerUnread()).toBe(true)
|
||||
})
|
||||
|
||||
it('stays unread when a check carries no env evidence for the stale handle', async () => {
|
||||
setupFederatedWorker()
|
||||
const asked = await ask({ question: 'Proceed?', wait: false })
|
||||
await importRemoteAnswer(askedId(asked))
|
||||
|
||||
await expect(check(STALE_HANDLE, ctx)).rejects.toMatchObject({ code: 'dispatch_inactive' })
|
||||
expect(remoteAnswerUnread()).toBe(true)
|
||||
})
|
||||
|
||||
it('is not handed to the stale handle once another Dispatch owns the pane', async () => {
|
||||
setupFederatedWorker()
|
||||
const asked = await ask({ question: 'Proceed?', wait: false })
|
||||
await importRemoteAnswer(askedId(asked))
|
||||
db.settleRemoteAttachmentInRelayTransaction(REMOTE_DISPATCH, 'succeeded')
|
||||
const next = db.createTask({ spec: 'next work in the same terminal' })
|
||||
createRootDispatch(db, next.id, REMINTED_HANDLE, WORKER_PANE, undefined, INCARNATION)
|
||||
|
||||
const checked = await check().catch((error: unknown) => error)
|
||||
|
||||
expect(checkedBodies(checked)).not.toContain('Approved')
|
||||
expect(remoteAnswerUnread()).toBe(true)
|
||||
})
|
||||
})
|
||||
|
||||
describe('after its remote Dispatch finished and local work began', () => {
|
||||
function setupLocalWorkAfterRemoteHistory(handle: string): {
|
||||
taskId: string
|
||||
dispatchId: string
|
||||
} {
|
||||
setupFederatedWorker()
|
||||
db.settleRemoteAttachmentInRelayTransaction(REMOTE_DISPATCH, 'succeeded')
|
||||
const task = db.createTask({ spec: 'local work after the remote Dispatch finished' })
|
||||
const dispatch = createRootDispatch(
|
||||
db,
|
||||
task.id,
|
||||
handle,
|
||||
WORKER_PANE,
|
||||
undefined,
|
||||
INCARNATION
|
||||
)
|
||||
db.db
|
||||
.prepare('UPDATE dispatch_contexts SET host_scope = ? WHERE id = ?')
|
||||
.run(HOST_SCOPE, dispatch.id)
|
||||
return { taskId: task.id, dispatchId: dispatch.id }
|
||||
}
|
||||
|
||||
const currentEnv = (): RpcContext => ({
|
||||
...ctx,
|
||||
orchestrationCompatibilityEvidence: {
|
||||
terminalHandle: REMINTED_HANDLE,
|
||||
paneKey: WORKER_PANE,
|
||||
launchToken: 'launch-token'
|
||||
}
|
||||
})
|
||||
|
||||
it.each([
|
||||
['its stale env handle', STALE_HANDLE],
|
||||
['its current handle', REMINTED_HANDLE]
|
||||
])('settles the local Task on a default worker_done from %s', async (_label, handle) => {
|
||||
const { taskId, dispatchId } = setupLocalWorkAfterRemoteHistory(handle)
|
||||
|
||||
await expect(
|
||||
harness.call(
|
||||
'orchestration.send',
|
||||
{
|
||||
from: handle,
|
||||
subject: 'done',
|
||||
type: 'worker_done',
|
||||
payload: JSON.stringify({ taskId, dispatchId, outcome: 'succeeded' })
|
||||
},
|
||||
handle === STALE_HANDLE ? workerEnv() : currentEnv()
|
||||
)
|
||||
).resolves.toBeDefined()
|
||||
|
||||
expect(db.getTask(taskId)?.status).toBe('completed')
|
||||
expect(db.getDispatchContextById(dispatchId)?.status).toBe('completed')
|
||||
expect(remoteRelayKinds()).toEqual([])
|
||||
})
|
||||
|
||||
it('still refuses mail that names the finished remote Dispatch', async () => {
|
||||
setupLocalWorkAfterRemoteHistory(STALE_HANDLE)
|
||||
|
||||
await expect(
|
||||
harness.call(
|
||||
'orchestration.send',
|
||||
{
|
||||
from: STALE_HANDLE,
|
||||
to: `dispatch:${REMOTE_DISPATCH}`,
|
||||
subject: 'late',
|
||||
type: 'status'
|
||||
},
|
||||
workerEnv()
|
||||
)
|
||||
).rejects.toMatchObject({ code: 'run_home_unreachable' })
|
||||
})
|
||||
})
|
||||
|
||||
it('tells a finished worker its default worker_done cannot reach its coordinator', async () => {
|
||||
setupFederatedWorker()
|
||||
db.settleRemoteAttachmentInRelayTransaction(REMOTE_DISPATCH, 'succeeded')
|
||||
|
||||
await expect(
|
||||
send('worker_done', {
|
||||
taskId: 'task_remote_restart',
|
||||
dispatchId: REMOTE_DISPATCH,
|
||||
outcome: 'succeeded'
|
||||
})
|
||||
).rejects.toMatchObject({ code: 'run_home_unreachable' })
|
||||
})
|
||||
|
||||
it('tells a finished worker its coordinator is unreachable instead of a bare refusal', async () => {
|
||||
setupFederatedWorker()
|
||||
db.db
|
||||
|
||||
@@ -67,14 +67,9 @@ export const ORCHESTRATION_SEND_METHODS = [
|
||||
) => {
|
||||
const db = runtime.getOrchestrationDb()
|
||||
const from = params.from ?? 'unknown'
|
||||
const attestedCaller =
|
||||
orchestrationCompatibilityCallerAuthority?.terminalHandle === from
|
||||
? orchestrationCompatibilityCallerAuthority
|
||||
: undefined
|
||||
// Why: attested hook identity survives graph remount; caller params never supply lifecycle authority.
|
||||
const reporter = resolveWorkerReporterIdentity(runtime, db, {
|
||||
from,
|
||||
attested: attestedCaller,
|
||||
authority: orchestrationCompatibilityCallerAuthority,
|
||||
hasSessionCaller: Boolean(orchestrationCaller),
|
||||
evidence: orchestrationCompatibilityEvidence
|
||||
})
|
||||
@@ -109,7 +104,7 @@ export const ORCHESTRATION_SEND_METHODS = [
|
||||
})
|
||||
}
|
||||
|
||||
refuseFinishedRemoteWorker(db, senderPaneKey, params.to)
|
||||
refuseFinishedRemoteWorker(db, { address: from, paneKey: senderPaneKey }, params.to)
|
||||
const runGroup =
|
||||
params.to &&
|
||||
isGroupAddress(params.to) &&
|
||||
|
||||
Reference in New Issue
Block a user