fix(orchestration): worker-show of a remote worker reads this Orca's record, never its server

Show answers at once whether the server is awake, asleep or off, and agrees with worker-list.
Its note says when the record is from and what the pull loop last heard from the server; a
prompt wait already arrives as the server's mail. worker-read stays forwarded with its 5 s cap.
This commit is contained in:
Jinwoo-H
2026-11-10 18:45:44 -05:00
parent 9a20bf0f8c
commit 5262ef9baf
8 changed files with 149 additions and 50 deletions
@@ -2,14 +2,14 @@ import { describe, expect, it } from 'vitest'
import { WORKER_LIST_VIEW, WORKER_SHOW_VIEW } from './worker-agent-views'
describe('worker views of a worker a paired server runs', () => {
it('labels an offline show with its server, record time, last report and note', () => {
it('labels a remote worker’s show with its server, record time, last report and note', () => {
expect(
WORKER_SHOW_VIEW.fields({
dispatch: { id: 'ctx_w1', taskId: 'task_1', runId: 'run_1', status: 'dispatched' },
worker: {
dispatchId: 'ctx_w1',
state: 'ready',
stage: 'server_unreachable',
stage: 'server_record',
agentTerminalHandle: null
},
projection: null,
@@ -20,7 +20,7 @@ describe('worker views of a worker a paired server runs', () => {
).toMatchObject({
dispatchId: 'ctx_w1',
state: 'ready',
stage: 'server_unreachable',
stage: 'server_record',
server: 'devB',
asOf: '2026-11-09T08:30:00Z',
lastReport: 'Halfway: tests written.',
@@ -40,8 +40,8 @@ export type WorkerShowResult = {
terminalHandle?: string | null
} | null
observation?: { agentWait?: { source: string; reason?: string } | null; reason?: string }
/** Present only when the worker's paired server could not be reached and this Orca's record answered. */
server?: RemoteWorkerServer & { reachable: false }
/** Present only for a worker a paired server runs, which this Orca's record answers. */
server?: RemoteWorkerServer & { reachable?: boolean }
lastReport?: string | null
autoReleaseAt?: string | null
/** The paired Orca whose Run here coordinates this worker. */
+9 -1
View File
@@ -61,7 +61,10 @@ import { HEADLESS_RUNTIME_WINDOW_ID } from '../../shared/runtime-session-contrac
import { getHostDisplayLabelOverrides } from '../../shared/host-setting-overrides'
import type { OrchestrationDb } from './orchestration/db/orchestration-db'
import { continueRestoredWorkerMailSoon } from './rpc/methods/orchestration/messaging/settled-worker-follow-through'
import { PeerMailRelay } from './orchestration/federation/peer-mail/peer-mail-relay'
import {
PeerMailRelay,
type PeerServerAnswerStatus
} from './orchestration/federation/peer-mail/peer-mail-relay'
import { structuredAgentSessionHostTabId } from '../../shared/agent-session/view/structured-agent-session-host-tab-id'
const OrcaRuntimeSidebarBase: typeof OrcaRuntimeWithAgentTransitions = ORCA_SIDEBAR_INPUTS_IS_MAIN
@@ -79,6 +82,11 @@ class OrcaRuntimeService extends OrcaRuntimeSidebarBase {
this.peerMailRelay.ensure()
}
/** What the pull loop last heard from a paired server, without asking it. */
getPeerServerAnswerStatus(environmentId: string): PeerServerAnswerStatus | undefined {
return this.peerMailRelay?.answerStatus(environmentId)
}
override stopOrchestrationFederationRelay(): void {
super.stopOrchestrationFederationRelay()
this.peerMailRelay?.stop()
@@ -16,6 +16,7 @@ import {
remoteServerPin
} from '../../db/remote-owned/remote-run-links'
import { resendOwedRemoteActions } from '../remote-owned/remote-owed-actions'
import { isRemoteServerUnreachable } from '../remote-owned/remote-server-unreachable'
// Under the relay edge's idle cutoff, so an idle pull ends by itself instead of being dropped.
const PULL_WAIT_MS = 20_000
@@ -33,6 +34,14 @@ const PulledPeerMail = z.object({
)
})
/** What this Orca's pull loop last heard from a paired server; nothing until its first pull settles. */
export type PeerServerAnswerStatus = {
/** When the server last answered a pull (an idle pull answers after its wait). */
answeredAt?: number
/** Set while pulls get no answer at all; cleared by the next answer. */
unreachableSince?: number
}
type Loop = {
db: OrchestrationDb
stopped: boolean
@@ -48,6 +57,7 @@ const ORPHAN_LOOKUP_INTERVAL_MS = 30_000
* them. Stops and cancels this Orca owes the server go out on the same loop. */
export class PeerMailRelay {
private readonly loops = new Map<string, Loop>()
private readonly answers = new Map<string, PeerServerAnswerStatus>()
constructor(
private readonly runtime: OrcaRuntimeService,
@@ -74,6 +84,10 @@ export class PeerMailRelay {
}
}
answerStatus(environmentId: string): PeerServerAnswerStatus | undefined {
return this.answers.get(environmentId)
}
stop(): void {
for (const loop of this.loops.values()) {
loop.stopped = true
@@ -122,12 +136,24 @@ export class PeerMailRelay {
private async pullOnce(environmentId: string, loop: Loop): Promise<void> {
this.resendOwed(environmentId, loop)
const response = await this.runtime.callOrchestrationWorkerServer(
environmentId,
'orchestration.peerMailPull',
{ waitMs: PULL_WAIT_MS },
PULL_TIMEOUT_MS
)
const response = await this.runtime
.callOrchestrationWorkerServer(
environmentId,
'orchestration.peerMailPull',
{ waitMs: PULL_WAIT_MS },
PULL_TIMEOUT_MS
)
.catch((error: unknown) => {
if (isRemoteServerUnreachable(error)) {
const known = this.answers.get(environmentId)
this.answers.set(environmentId, {
...known,
unreachableSince: known?.unreachableSince ?? Date.now()
})
}
throw error
})
this.answers.set(environmentId, { answeredAt: Date.now() })
if (!this.isCurrent(environmentId, loop)) {
return
}
@@ -17,7 +17,7 @@ import {
import { recordRemoteStart, reserveRemotePlacement } from '../../db/remote-owned/remote-placements'
import { closeRemoteRunLink, openRemoteRunLink } from '../../db/remote-owned/remote-run-links'
import { ORCHESTRATION_WORKER_LIST_METHOD } from '../../../rpc/methods/orchestration/worker/worker-list-method'
import { offlineRemoteWorkerShow } from './remote-owned-reads'
import { remoteWorkerShow } from './remote-owned-reads'
import { explainRemoteServerFailure } from './remote-server-unreachable'
const SERVER: OrchestrationWorkerServer = {
@@ -51,7 +51,10 @@ describe('reads of workers a paired server runs, from this Orca’s records', ()
})
})
afterEach(() => db.close())
afterEach(() => {
vi.useRealTimers()
db.close()
})
/** A worker devB started for Task `taskId`, as devB last reported it at LAST_HEARD. */
function remoteWorker(taskId: string, dispatchId: string, environmentId = SERVER.environmentId) {
@@ -95,23 +98,44 @@ describe('reads of workers a paired server runs, from this Orca’s records', ()
} as never)) as ListedWorkers
}
it('shows an unreachable server’s worker in the ordinary shape, labelled with its server and last report', () => {
const shown = offlineRemoteWorkerShow(runtime, remoteWorker('task_1', 'ctx_w1'))
it('shows a remote worker in the ordinary shape, labelled with its server, record time and last report', () => {
const shown = remoteWorkerShow(runtime, remoteWorker('task_1', 'ctx_w1'))
expect(shown).toMatchObject({
dispatch: { id: 'ctx_w1', taskId: 'task_1', runId: run.id, status: 'dispatched' },
worker: { dispatchId: 'ctx_w1', state: 'ready', stage: 'server_unreachable' },
server: { name: 'devB', reachable: false, asOf: '2026-11-09T08:30:00Z' },
worker: { dispatchId: 'ctx_w1', state: 'ready', stage: 'server_record' },
server: { name: 'devB', asOf: '2026-11-09T08:30:00Z' },
lastReport: 'Halfway: tests written.'
})
expect(shown.observation.reason).toContain('devB could not be reached')
expect(shown.server).not.toHaveProperty('reachable')
expect(shown.observation.reason).toBe(
"devB's state as of 2026-11-09T08:30:00Z. Its output is kept only on devB."
)
})
it('adds what the pull loop last heard from the server, and says unreachable only when it knows', () => {
const record = remoteWorker('task_1', 'ctx_w1')
const now = Date.parse('2026-11-10T12:00:00Z')
vi.useFakeTimers({ toFake: ['Date'], now })
const heard = vi.spyOn(runtime, 'getPeerServerAnswerStatus')
heard.mockReturnValue({ answeredAt: now - 12_000 })
const live = remoteWorkerShow(runtime, record)
expect(live.server).toMatchObject({ reachable: true })
expect(live.observation.reason).toContain('devB last answered 12 s ago.')
heard.mockReturnValue({ answeredAt: now - 40_000, unreachableSince: now - 5_000 })
const asleep = remoteWorkerShow(runtime, record)
expect(asleep.server).toMatchObject({ reachable: false })
expect(asleep.observation.reason).toContain(
'devB could not be reached; it last answered 40 s ago.'
)
})
it('keeps the time devB last reported when this Orca only owes it a stop', () => {
const record = remoteWorker('task_1', 'ctx_w1')
oweRemoteDispatchStop(db, record.dispatch_id, 'intent_1')
const shown = offlineRemoteWorkerShow(runtime, getRemoteDispatch(db, 'ctx_w1')!)
const shown = remoteWorkerShow(runtime, getRemoteDispatch(db, 'ctx_w1')!)
expect(shown.server.asOf).toBe('2026-11-09T08:30:00Z')
expect(shown.observation.reason).toContain('A stop is owed')
})
@@ -120,7 +144,7 @@ describe('reads of workers a paired server runs, from this Orca’s records', ()
const record = remoteWorker('task_1', 'ctx_w1')
closeRemoteRunLink(db, record.conversation_id, 'The pairing with this server was re-made.')
const shown = offlineRemoteWorkerShow(runtime, record)
const shown = remoteWorkerShow(runtime, record)
expect(shown.server).toMatchObject({ name: 'devB', repaired: true })
expect(shown.observation.reason).toContain('The pairing with devB was re-made')
const listed = await listWorkers({ run: run.id, paginate: true })
@@ -143,7 +167,7 @@ describe('reads of workers a paired server runs, from this Orca’s records', ()
})
it('names a server removed since by its environment ID', () => {
const shown = offlineRemoteWorkerShow(runtime, remoteWorker('task_1', 'ctx_w1', 'env_gone'))
const shown = remoteWorkerShow(runtime, remoteWorker('task_1', 'ctx_w1', 'env_gone'))
expect(shown.server.name).toBe('env_gone')
})
@@ -6,10 +6,11 @@ import {
type RemoteDispatchRow
} from '../../db/remote-owned/remote-dispatch-records'
import { getRemoteRunLink } from '../../db/remote-owned/remote-run-links'
import type { PeerServerAnswerStatus } from '../peer-mail/peer-mail-relay'
type ReadsRuntime = Pick<
OrcaRuntimeService,
'getOrchestrationDb' | 'resolveOrchestrationWorkerServer'
'getOrchestrationDb' | 'resolveOrchestrationWorkerServer' | 'getPeerServerAnswerStatus'
>
/**
@@ -47,16 +48,18 @@ export function remoteServerName(runtime: ReadsRuntime, environmentId: string):
}
/**
* worker-show for a remote worker whose server could not be reached: this Orca's record in the
* ordinary show shape, labelled with the server and the time of its last report. Metadata only;
* the worker's output stays on its server.
* worker-show for a worker a paired server runs: always this Orca's record, never a call there, so
* it answers at once whether the server is awake, asleep or off, and agrees with worker-list. The
* note says when the record is from and what the pull loop last heard from the server. Metadata
* only: the worker's output stays on its server (worker-read), and a prompt wait arrives as mail.
*/
export function offlineRemoteWorkerShow(runtime: ReadsRuntime, record: RemoteDispatchRow) {
export function remoteWorkerShow(runtime: ReadsRuntime, record: RemoteDispatchRow) {
const link = getRemoteRunLink(runtime.getOrchestrationDb(), record.conversation_id)
const server = remoteWorkerServer(record, link?.state, (id) => remoteServerName(runtime, id))
const why = server.repaired
? `The pairing with ${server.name} was re-made, so it no longer reports this worker here`
: `${server.name} could not be reached`
const heard = server.repaired
? undefined
: runtime.getPeerServerAnswerStatus(record.environment_id)
const reachable = heard?.unreachableSince ? false : heard?.answeredAt ? true : undefined
const owedStop = record.stop_owed
? ` A stop is owed; this Orca sends it when ${server.name} answers.`
: ''
@@ -73,7 +76,7 @@ export function offlineRemoteWorkerShow(runtime: ReadsRuntime, record: RemoteDis
worker: {
dispatchId: record.dispatch_id,
state: record.worker_state ?? record.status,
stage: 'server_unreachable',
stage: 'server_record',
agentTerminalHandle: null,
worktreeId: null,
lastError: record.last_failure,
@@ -85,14 +88,32 @@ export function offlineRemoteWorkerShow(runtime: ReadsRuntime, record: RemoteDis
observation: {
status: 'unverifiable' as const,
exactWorker: false,
reason: `${why}, so this is this Orca's record of the worker: its state as of ${server.asOf}. Its output is kept only on ${server.name}.${owedStop}`
reason: `${recordNote(server, heard)} Its output is kept only on ${server.name}.${owedStop}`
},
server: {
...server,
...(reachable === undefined ? {} : { reachable }),
...(heard?.answeredAt ? { lastAnsweredAt: new Date(heard.answeredAt).toISOString() } : {})
},
server: { ...server, reachable: false as const },
lastReport: record.last_report,
autoReleaseAt: null
}
}
function recordNote(server: RemoteWorkerServer, heard: PeerServerAnswerStatus | undefined): string {
if (server.repaired) {
return `The pairing with ${server.name} was re-made, so it no longer reports this worker here; this is ${server.name}'s state as of ${server.asOf}.`
}
const state = `${server.name}'s state as of ${server.asOf}.`
const ago = heard?.answeredAt
? `${Math.max(0, Math.round((Date.now() - heard.answeredAt) / 1000))} s ago`
: undefined
if (heard?.unreachableSince) {
return `${state} ${server.name} could not be reached${ago ? `; it last answered ${ago}` : ''}.`
}
return ago ? `${state} ${server.name} last answered ${ago}.` : state
}
export type RemoteWorkerListRow = ReturnType<typeof remoteWorkerListRow>
function remoteWorkerListRow(
@@ -366,18 +366,42 @@ describe('commands about a worker a paired server owns', () => {
expect(show.ok ? record(show.result).server : undefined).toMatchObject({ name: 'devB' })
})
it("answers worker-show from this Orca's record within seconds when the server sleeps", async () => {
it("shows a remote worker from this Orca's record without calling its server", async () => {
const { runId } = await startOnB()
// devB is live and would say something newer; show still reads the record list reads.
answers['orchestration.workerShow'] = [
{ result: { worker: { dispatchId: 'ctx_b1', state: 'running', stage: 'working' } } }
]
const calls = sent.length
const show = await call('orchestration.workerShow', { dispatch: 'ctx_b1' })
const list = await call('orchestration.workerList', { run: runId, paginate: true })
expect(sent.slice(calls).map((s) => s.method)).not.toContain('orchestration.workerShow')
// What the pull loop last heard from devB, which answers its pulls.
expect(record(record(show.ok ? show.result : undefined).server).reachable).toBe(true)
const shown = record(show.ok ? show.result : undefined)
const workers = record(list.ok ? list.result : undefined).workers
const listed = (Array.isArray(workers) ? workers : [])
.map(record)
.find((worker) => worker.dispatchId === 'ctx_b1')
expect({
state: record(shown.worker).state,
asOf: record(shown.server).asOf
}).toEqual({ state: listed?.workerState, asOf: record(listed?.server).asOf })
})
it("gives up a worker-read within seconds when the server sleeps, in the server's terms", async () => {
await startOnB()
vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] })
sleeping = true
let show: RpcResponse | undefined
void call('orchestration.workerShow', { dispatch: 'ctx_b1' }).then((answer) => {
show = answer
let read: RpcResponse | undefined
void call('orchestration.workerRead', { dispatch: 'ctx_b1' }).then((answer) => {
read = answer
})
await vi.advanceTimersByTimeAsync(10_000)
vi.useRealTimers()
expect(show?.ok ? record(show.result).server : show).toMatchObject({ name: 'devB' })
expect(read?.ok ? undefined : read?.error.code).toBe('runtime_timeout')
})
it('scopes the retry key to the caller: two terminals reusing an ID never collide, one retrying gets its own key', async () => {
@@ -16,7 +16,6 @@ import {
getRemoteDispatch,
getRemoteOwnedQuestion
} from '../../db/remote-owned/remote-dispatch-records'
import { OrchestrationError } from '../../orchestration-error'
import { adoptRemoteAttempt } from '../../db/remote-owned/remote-placements'
import {
getOpenRemoteRunLink,
@@ -26,8 +25,8 @@ import {
import { callRemoteOwnedServer } from './remote-owned-calls'
import { remoteForwardKey, type RemoteForwardKey } from './remote-forward-caller'
import { inServerTerms } from './remote-owned-answers'
import { offlineRemoteWorkerShow, remoteServerName } from './remote-owned-reads'
import { explainRemoteServerFailure, isRemoteServerUnreachable } from './remote-server-unreachable'
import { remoteWorkerShow, remoteServerName } from './remote-owned-reads'
import { explainRemoteServerFailure } from './remote-server-unreachable'
import { serverOwnsPlacedWorkers, startRemoteOwnedWorker } from './remote-owned-start'
import { stopRemoteWorker } from './remote-owned-stop'
@@ -36,7 +35,7 @@ type Routed = { handled: false } | { handled: true; result: unknown }
const LOCAL: Routed = { handled: false }
// Long enough for a send that starts a follow-up worker there and waits for it to be ready.
const FORWARD_TIMEOUT_MS = 180_000
// A read gives up soon on a sleeping server, whose socket stays open; show then answers from this Orca's record.
// A read gives up soon on a sleeping server, whose socket stays open.
const READ_TIMEOUT_MS = 5_000
// What `--on <server>` sends there; send and reply --on stay peer mail.
const SERVER_CALLS: ReadonlySet<string> = new Set([
@@ -74,6 +73,11 @@ export async function routeRemoteOwnedCall(
return on === undefined ? LOCAL : routeServerCall(runtime, request, access, values, ctx, on)
}
checkDeclaredOrchestrationAccess(access, params, ctx)
const record = subject.dispatchId ? getRemoteDispatch(db, subject.dispatchId) : undefined
if (request.method === 'orchestration.workerShow' && record) {
// A status read is this Orca's record, never a call there: instant whatever the server is doing.
return { handled: true, result: remoteWorkerShow(runtime, record) }
}
const forward = remoteForwardKey(
runtime,
ctx,
@@ -235,14 +239,6 @@ async function forwardToLink(
seedNamedDispatches(db, link, answer)
return inServerTerms(answer, { server, forward, link })
} catch (error) {
const record = dispatchId ? getRemoteDispatch(db, dispatchId) : undefined
const unanswerable =
isRemoteServerUnreachable(error) ||
(error instanceof OrchestrationError && error.code === 'remote_link_closed')
if (record && method === 'orchestration.workerShow' && unanswerable) {
// The server is off, or no longer answers for this link: what this Orca last heard, never output.
return offlineRemoteWorkerShow(runtime, record)
}
throw inServerTerms(explainRemoteServerFailure(error, { server, method, params, dispatchId }), {
server,
forward,