diff --git a/src/cli/handlers/orchestration/worker-agent-views.test.ts b/src/cli/handlers/orchestration/worker-agent-views.test.ts index f9d85c3fde4..56ebbe06ad1 100644 --- a/src/cli/handlers/orchestration/worker-agent-views.test.ts +++ b/src/cli/handlers/orchestration/worker-agent-views.test.ts @@ -2,14 +2,14 @@ import { describe, expect, it } from 'vitest' import { WORKER_LIST_VIEW, WORKER_SHOW_VIEW } from './worker-agent-views' describe('worker views of a worker a paired server runs', () => { - it('labels an offline show with its server, record time, last report and note', () => { + it('labels a remote worker’s show with its server, record time, last report and note', () => { expect( WORKER_SHOW_VIEW.fields({ dispatch: { id: 'ctx_w1', taskId: 'task_1', runId: 'run_1', status: 'dispatched' }, worker: { dispatchId: 'ctx_w1', state: 'ready', - stage: 'server_unreachable', + stage: 'server_record', agentTerminalHandle: null }, projection: null, @@ -20,7 +20,7 @@ describe('worker views of a worker a paired server runs', () => { ).toMatchObject({ dispatchId: 'ctx_w1', state: 'ready', - stage: 'server_unreachable', + stage: 'server_record', server: 'devB', asOf: '2026-11-09T08:30:00Z', lastReport: 'Halfway: tests written.', diff --git a/src/cli/handlers/orchestration/worker-agent-views.ts b/src/cli/handlers/orchestration/worker-agent-views.ts index 07a883c2ae6..f7a171bc131 100644 --- a/src/cli/handlers/orchestration/worker-agent-views.ts +++ b/src/cli/handlers/orchestration/worker-agent-views.ts @@ -40,8 +40,8 @@ export type WorkerShowResult = { terminalHandle?: string | null } | null observation?: { agentWait?: { source: string; reason?: string } | null; reason?: string } - /** Present only when the worker's paired server could not be reached and this Orca's record answered. */ - server?: RemoteWorkerServer & { reachable: false } + /** Present only for a worker a paired server runs, which this Orca's record answers. */ + server?: RemoteWorkerServer & { reachable?: boolean } lastReport?: string | null autoReleaseAt?: string | null /** The paired Orca whose Run here coordinates this worker. */ diff --git a/src/main/runtime/orca-runtime.ts b/src/main/runtime/orca-runtime.ts index 456f346a079..aecce3dc198 100644 --- a/src/main/runtime/orca-runtime.ts +++ b/src/main/runtime/orca-runtime.ts @@ -61,7 +61,10 @@ import { HEADLESS_RUNTIME_WINDOW_ID } from '../../shared/runtime-session-contrac import { getHostDisplayLabelOverrides } from '../../shared/host-setting-overrides' import type { OrchestrationDb } from './orchestration/db/orchestration-db' import { continueRestoredWorkerMailSoon } from './rpc/methods/orchestration/messaging/settled-worker-follow-through' -import { PeerMailRelay } from './orchestration/federation/peer-mail/peer-mail-relay' +import { + PeerMailRelay, + type PeerServerAnswerStatus +} from './orchestration/federation/peer-mail/peer-mail-relay' import { structuredAgentSessionHostTabId } from '../../shared/agent-session/view/structured-agent-session-host-tab-id' const OrcaRuntimeSidebarBase: typeof OrcaRuntimeWithAgentTransitions = ORCA_SIDEBAR_INPUTS_IS_MAIN @@ -79,6 +82,11 @@ class OrcaRuntimeService extends OrcaRuntimeSidebarBase { this.peerMailRelay.ensure() } + /** What the pull loop last heard from a paired server, without asking it. */ + getPeerServerAnswerStatus(environmentId: string): PeerServerAnswerStatus | undefined { + return this.peerMailRelay?.answerStatus(environmentId) + } + override stopOrchestrationFederationRelay(): void { super.stopOrchestrationFederationRelay() this.peerMailRelay?.stop() diff --git a/src/main/runtime/orchestration/federation/peer-mail/peer-mail-relay.ts b/src/main/runtime/orchestration/federation/peer-mail/peer-mail-relay.ts index c4df5e6f6db..b4acacfaa47 100644 --- a/src/main/runtime/orchestration/federation/peer-mail/peer-mail-relay.ts +++ b/src/main/runtime/orchestration/federation/peer-mail/peer-mail-relay.ts @@ -16,6 +16,7 @@ import { remoteServerPin } from '../../db/remote-owned/remote-run-links' import { resendOwedRemoteActions } from '../remote-owned/remote-owed-actions' +import { isRemoteServerUnreachable } from '../remote-owned/remote-server-unreachable' // Under the relay edge's idle cutoff, so an idle pull ends by itself instead of being dropped. const PULL_WAIT_MS = 20_000 @@ -33,6 +34,14 @@ const PulledPeerMail = z.object({ ) }) +/** What this Orca's pull loop last heard from a paired server; nothing until its first pull settles. */ +export type PeerServerAnswerStatus = { + /** When the server last answered a pull (an idle pull answers after its wait). */ + answeredAt?: number + /** Set while pulls get no answer at all; cleared by the next answer. */ + unreachableSince?: number +} + type Loop = { db: OrchestrationDb stopped: boolean @@ -48,6 +57,7 @@ const ORPHAN_LOOKUP_INTERVAL_MS = 30_000 * them. Stops and cancels this Orca owes the server go out on the same loop. */ export class PeerMailRelay { private readonly loops = new Map() + private readonly answers = new Map() constructor( private readonly runtime: OrcaRuntimeService, @@ -74,6 +84,10 @@ export class PeerMailRelay { } } + answerStatus(environmentId: string): PeerServerAnswerStatus | undefined { + return this.answers.get(environmentId) + } + stop(): void { for (const loop of this.loops.values()) { loop.stopped = true @@ -122,12 +136,24 @@ export class PeerMailRelay { private async pullOnce(environmentId: string, loop: Loop): Promise { this.resendOwed(environmentId, loop) - const response = await this.runtime.callOrchestrationWorkerServer( - environmentId, - 'orchestration.peerMailPull', - { waitMs: PULL_WAIT_MS }, - PULL_TIMEOUT_MS - ) + const response = await this.runtime + .callOrchestrationWorkerServer( + environmentId, + 'orchestration.peerMailPull', + { waitMs: PULL_WAIT_MS }, + PULL_TIMEOUT_MS + ) + .catch((error: unknown) => { + if (isRemoteServerUnreachable(error)) { + const known = this.answers.get(environmentId) + this.answers.set(environmentId, { + ...known, + unreachableSince: known?.unreachableSince ?? Date.now() + }) + } + throw error + }) + this.answers.set(environmentId, { answeredAt: Date.now() }) if (!this.isCurrent(environmentId, loop)) { return } diff --git a/src/main/runtime/orchestration/federation/remote-owned/remote-owned-reads.test.ts b/src/main/runtime/orchestration/federation/remote-owned/remote-owned-reads.test.ts index b2616dedc29..87f57d29cdd 100644 --- a/src/main/runtime/orchestration/federation/remote-owned/remote-owned-reads.test.ts +++ b/src/main/runtime/orchestration/federation/remote-owned/remote-owned-reads.test.ts @@ -17,7 +17,7 @@ import { import { recordRemoteStart, reserveRemotePlacement } from '../../db/remote-owned/remote-placements' import { closeRemoteRunLink, openRemoteRunLink } from '../../db/remote-owned/remote-run-links' import { ORCHESTRATION_WORKER_LIST_METHOD } from '../../../rpc/methods/orchestration/worker/worker-list-method' -import { offlineRemoteWorkerShow } from './remote-owned-reads' +import { remoteWorkerShow } from './remote-owned-reads' import { explainRemoteServerFailure } from './remote-server-unreachable' const SERVER: OrchestrationWorkerServer = { @@ -51,7 +51,10 @@ describe('reads of workers a paired server runs, from this Orca’s records', () }) }) - afterEach(() => db.close()) + afterEach(() => { + vi.useRealTimers() + db.close() + }) /** A worker devB started for Task `taskId`, as devB last reported it at LAST_HEARD. */ function remoteWorker(taskId: string, dispatchId: string, environmentId = SERVER.environmentId) { @@ -95,23 +98,44 @@ describe('reads of workers a paired server runs, from this Orca’s records', () } as never)) as ListedWorkers } - it('shows an unreachable server’s worker in the ordinary shape, labelled with its server and last report', () => { - const shown = offlineRemoteWorkerShow(runtime, remoteWorker('task_1', 'ctx_w1')) + it('shows a remote worker in the ordinary shape, labelled with its server, record time and last report', () => { + const shown = remoteWorkerShow(runtime, remoteWorker('task_1', 'ctx_w1')) expect(shown).toMatchObject({ dispatch: { id: 'ctx_w1', taskId: 'task_1', runId: run.id, status: 'dispatched' }, - worker: { dispatchId: 'ctx_w1', state: 'ready', stage: 'server_unreachable' }, - server: { name: 'devB', reachable: false, asOf: '2026-11-09T08:30:00Z' }, + worker: { dispatchId: 'ctx_w1', state: 'ready', stage: 'server_record' }, + server: { name: 'devB', asOf: '2026-11-09T08:30:00Z' }, lastReport: 'Halfway: tests written.' }) - expect(shown.observation.reason).toContain('devB could not be reached') + expect(shown.server).not.toHaveProperty('reachable') + expect(shown.observation.reason).toBe( + "devB's state as of 2026-11-09T08:30:00Z. Its output is kept only on devB." + ) + }) + + it('adds what the pull loop last heard from the server, and says unreachable only when it knows', () => { + const record = remoteWorker('task_1', 'ctx_w1') + const now = Date.parse('2026-11-10T12:00:00Z') + vi.useFakeTimers({ toFake: ['Date'], now }) + const heard = vi.spyOn(runtime, 'getPeerServerAnswerStatus') + heard.mockReturnValue({ answeredAt: now - 12_000 }) + const live = remoteWorkerShow(runtime, record) + expect(live.server).toMatchObject({ reachable: true }) + expect(live.observation.reason).toContain('devB last answered 12 s ago.') + + heard.mockReturnValue({ answeredAt: now - 40_000, unreachableSince: now - 5_000 }) + const asleep = remoteWorkerShow(runtime, record) + expect(asleep.server).toMatchObject({ reachable: false }) + expect(asleep.observation.reason).toContain( + 'devB could not be reached; it last answered 40 s ago.' + ) }) it('keeps the time devB last reported when this Orca only owes it a stop', () => { const record = remoteWorker('task_1', 'ctx_w1') oweRemoteDispatchStop(db, record.dispatch_id, 'intent_1') - const shown = offlineRemoteWorkerShow(runtime, getRemoteDispatch(db, 'ctx_w1')!) + const shown = remoteWorkerShow(runtime, getRemoteDispatch(db, 'ctx_w1')!) expect(shown.server.asOf).toBe('2026-11-09T08:30:00Z') expect(shown.observation.reason).toContain('A stop is owed') }) @@ -120,7 +144,7 @@ describe('reads of workers a paired server runs, from this Orca’s records', () const record = remoteWorker('task_1', 'ctx_w1') closeRemoteRunLink(db, record.conversation_id, 'The pairing with this server was re-made.') - const shown = offlineRemoteWorkerShow(runtime, record) + const shown = remoteWorkerShow(runtime, record) expect(shown.server).toMatchObject({ name: 'devB', repaired: true }) expect(shown.observation.reason).toContain('The pairing with devB was re-made') const listed = await listWorkers({ run: run.id, paginate: true }) @@ -143,7 +167,7 @@ describe('reads of workers a paired server runs, from this Orca’s records', () }) it('names a server removed since by its environment ID', () => { - const shown = offlineRemoteWorkerShow(runtime, remoteWorker('task_1', 'ctx_w1', 'env_gone')) + const shown = remoteWorkerShow(runtime, remoteWorker('task_1', 'ctx_w1', 'env_gone')) expect(shown.server.name).toBe('env_gone') }) diff --git a/src/main/runtime/orchestration/federation/remote-owned/remote-owned-reads.ts b/src/main/runtime/orchestration/federation/remote-owned/remote-owned-reads.ts index 62a2c0049a4..418ad95f885 100644 --- a/src/main/runtime/orchestration/federation/remote-owned/remote-owned-reads.ts +++ b/src/main/runtime/orchestration/federation/remote-owned/remote-owned-reads.ts @@ -6,10 +6,11 @@ import { type RemoteDispatchRow } from '../../db/remote-owned/remote-dispatch-records' import { getRemoteRunLink } from '../../db/remote-owned/remote-run-links' +import type { PeerServerAnswerStatus } from '../peer-mail/peer-mail-relay' type ReadsRuntime = Pick< OrcaRuntimeService, - 'getOrchestrationDb' | 'resolveOrchestrationWorkerServer' + 'getOrchestrationDb' | 'resolveOrchestrationWorkerServer' | 'getPeerServerAnswerStatus' > /** @@ -47,16 +48,18 @@ export function remoteServerName(runtime: ReadsRuntime, environmentId: string): } /** - * worker-show for a remote worker whose server could not be reached: this Orca's record in the - * ordinary show shape, labelled with the server and the time of its last report. Metadata only; - * the worker's output stays on its server. + * worker-show for a worker a paired server runs: always this Orca's record, never a call there, so + * it answers at once whether the server is awake, asleep or off, and agrees with worker-list. The + * note says when the record is from and what the pull loop last heard from the server. Metadata + * only: the worker's output stays on its server (worker-read), and a prompt wait arrives as mail. */ -export function offlineRemoteWorkerShow(runtime: ReadsRuntime, record: RemoteDispatchRow) { +export function remoteWorkerShow(runtime: ReadsRuntime, record: RemoteDispatchRow) { const link = getRemoteRunLink(runtime.getOrchestrationDb(), record.conversation_id) const server = remoteWorkerServer(record, link?.state, (id) => remoteServerName(runtime, id)) - const why = server.repaired - ? `The pairing with ${server.name} was re-made, so it no longer reports this worker here` - : `${server.name} could not be reached` + const heard = server.repaired + ? undefined + : runtime.getPeerServerAnswerStatus(record.environment_id) + const reachable = heard?.unreachableSince ? false : heard?.answeredAt ? true : undefined const owedStop = record.stop_owed ? ` A stop is owed; this Orca sends it when ${server.name} answers.` : '' @@ -73,7 +76,7 @@ export function offlineRemoteWorkerShow(runtime: ReadsRuntime, record: RemoteDis worker: { dispatchId: record.dispatch_id, state: record.worker_state ?? record.status, - stage: 'server_unreachable', + stage: 'server_record', agentTerminalHandle: null, worktreeId: null, lastError: record.last_failure, @@ -85,14 +88,32 @@ export function offlineRemoteWorkerShow(runtime: ReadsRuntime, record: RemoteDis observation: { status: 'unverifiable' as const, exactWorker: false, - reason: `${why}, so this is this Orca's record of the worker: its state as of ${server.asOf}. Its output is kept only on ${server.name}.${owedStop}` + reason: `${recordNote(server, heard)} Its output is kept only on ${server.name}.${owedStop}` + }, + server: { + ...server, + ...(reachable === undefined ? {} : { reachable }), + ...(heard?.answeredAt ? { lastAnsweredAt: new Date(heard.answeredAt).toISOString() } : {}) }, - server: { ...server, reachable: false as const }, lastReport: record.last_report, autoReleaseAt: null } } +function recordNote(server: RemoteWorkerServer, heard: PeerServerAnswerStatus | undefined): string { + if (server.repaired) { + return `The pairing with ${server.name} was re-made, so it no longer reports this worker here; this is ${server.name}'s state as of ${server.asOf}.` + } + const state = `${server.name}'s state as of ${server.asOf}.` + const ago = heard?.answeredAt + ? `${Math.max(0, Math.round((Date.now() - heard.answeredAt) / 1000))} s ago` + : undefined + if (heard?.unreachableSince) { + return `${state} ${server.name} could not be reached${ago ? `; it last answered ${ago}` : ''}.` + } + return ago ? `${state} ${server.name} last answered ${ago}.` : state +} + export type RemoteWorkerListRow = ReturnType function remoteWorkerListRow( diff --git a/src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.test.ts b/src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.test.ts index e48bda2f725..66df3e7d46e 100644 --- a/src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.test.ts +++ b/src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.test.ts @@ -366,18 +366,42 @@ describe('commands about a worker a paired server owns', () => { expect(show.ok ? record(show.result).server : undefined).toMatchObject({ name: 'devB' }) }) - it("answers worker-show from this Orca's record within seconds when the server sleeps", async () => { + it("shows a remote worker from this Orca's record without calling its server", async () => { + const { runId } = await startOnB() + // devB is live and would say something newer; show still reads the record list reads. + answers['orchestration.workerShow'] = [ + { result: { worker: { dispatchId: 'ctx_b1', state: 'running', stage: 'working' } } } + ] + const calls = sent.length + const show = await call('orchestration.workerShow', { dispatch: 'ctx_b1' }) + const list = await call('orchestration.workerList', { run: runId, paginate: true }) + + expect(sent.slice(calls).map((s) => s.method)).not.toContain('orchestration.workerShow') + // What the pull loop last heard from devB, which answers its pulls. + expect(record(record(show.ok ? show.result : undefined).server).reachable).toBe(true) + const shown = record(show.ok ? show.result : undefined) + const workers = record(list.ok ? list.result : undefined).workers + const listed = (Array.isArray(workers) ? workers : []) + .map(record) + .find((worker) => worker.dispatchId === 'ctx_b1') + expect({ + state: record(shown.worker).state, + asOf: record(shown.server).asOf + }).toEqual({ state: listed?.workerState, asOf: record(listed?.server).asOf }) + }) + + it("gives up a worker-read within seconds when the server sleeps, in the server's terms", async () => { await startOnB() vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) sleeping = true - let show: RpcResponse | undefined - void call('orchestration.workerShow', { dispatch: 'ctx_b1' }).then((answer) => { - show = answer + let read: RpcResponse | undefined + void call('orchestration.workerRead', { dispatch: 'ctx_b1' }).then((answer) => { + read = answer }) await vi.advanceTimersByTimeAsync(10_000) vi.useRealTimers() - expect(show?.ok ? record(show.result).server : show).toMatchObject({ name: 'devB' }) + expect(read?.ok ? undefined : read?.error.code).toBe('runtime_timeout') }) it('scopes the retry key to the caller: two terminals reusing an ID never collide, one retrying gets its own key', async () => { diff --git a/src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.ts b/src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.ts index 9ef4cadb840..67b77d02498 100644 --- a/src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.ts +++ b/src/main/runtime/orchestration/federation/remote-owned/remote-owned-route.ts @@ -16,7 +16,6 @@ import { getRemoteDispatch, getRemoteOwnedQuestion } from '../../db/remote-owned/remote-dispatch-records' -import { OrchestrationError } from '../../orchestration-error' import { adoptRemoteAttempt } from '../../db/remote-owned/remote-placements' import { getOpenRemoteRunLink, @@ -26,8 +25,8 @@ import { import { callRemoteOwnedServer } from './remote-owned-calls' import { remoteForwardKey, type RemoteForwardKey } from './remote-forward-caller' import { inServerTerms } from './remote-owned-answers' -import { offlineRemoteWorkerShow, remoteServerName } from './remote-owned-reads' -import { explainRemoteServerFailure, isRemoteServerUnreachable } from './remote-server-unreachable' +import { remoteWorkerShow, remoteServerName } from './remote-owned-reads' +import { explainRemoteServerFailure } from './remote-server-unreachable' import { serverOwnsPlacedWorkers, startRemoteOwnedWorker } from './remote-owned-start' import { stopRemoteWorker } from './remote-owned-stop' @@ -36,7 +35,7 @@ type Routed = { handled: false } | { handled: true; result: unknown } const LOCAL: Routed = { handled: false } // Long enough for a send that starts a follow-up worker there and waits for it to be ready. const FORWARD_TIMEOUT_MS = 180_000 -// A read gives up soon on a sleeping server, whose socket stays open; show then answers from this Orca's record. +// A read gives up soon on a sleeping server, whose socket stays open. const READ_TIMEOUT_MS = 5_000 // What `--on ` sends there; send and reply --on stay peer mail. const SERVER_CALLS: ReadonlySet = new Set([ @@ -74,6 +73,11 @@ export async function routeRemoteOwnedCall( return on === undefined ? LOCAL : routeServerCall(runtime, request, access, values, ctx, on) } checkDeclaredOrchestrationAccess(access, params, ctx) + const record = subject.dispatchId ? getRemoteDispatch(db, subject.dispatchId) : undefined + if (request.method === 'orchestration.workerShow' && record) { + // A status read is this Orca's record, never a call there: instant whatever the server is doing. + return { handled: true, result: remoteWorkerShow(runtime, record) } + } const forward = remoteForwardKey( runtime, ctx, @@ -235,14 +239,6 @@ async function forwardToLink( seedNamedDispatches(db, link, answer) return inServerTerms(answer, { server, forward, link }) } catch (error) { - const record = dispatchId ? getRemoteDispatch(db, dispatchId) : undefined - const unanswerable = - isRemoteServerUnreachable(error) || - (error instanceof OrchestrationError && error.code === 'remote_link_closed') - if (record && method === 'orchestration.workerShow' && unanswerable) { - // The server is off, or no longer answers for this link: what this Orca last heard, never output. - return offlineRemoteWorkerShow(runtime, record) - } throw inServerTerms(explainRemoteServerFailure(error, { server, method, params, dispatchId }), { server, forward,