From e357add5ec9f5f4ffbee1305e459bbc585270703 Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Mon, 9 Nov 2026 00:52:12 -0500 Subject: [PATCH 1/3] fix(orchestration): keep a paired server's worker list when one Dispatch is from an earlier pairing The server's fleet snapshot refused the whole batch when one requested Dispatch was not this home's (for example, one started before the home re-paired), so the home showed every worker on that server as host_unavailable. Now only that Dispatch reads as unverifiable. --- .../federation/federation-control.ts | 9 +++++- .../federation-liveness-verdict.test.ts | 29 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/src/main/runtime/rpc/methods/orchestration/federation/federation-control.ts b/src/main/runtime/rpc/methods/orchestration/federation/federation-control.ts index 4fbe9d527ae..aa3430ec5bf 100644 --- a/src/main/runtime/rpc/methods/orchestration/federation/federation-control.ts +++ b/src/main/runtime/rpc/methods/orchestration/federation/federation-control.ts @@ -28,7 +28,14 @@ export const ORCHESTRATION_FEDERATION_CONTROL_METHODS = [ params: FederationFleetSnapshotParams, handler: async (params, { runtime, authenticatedCallerFingerprint }) => { const items = await mapWithConcurrency(params.dispatchIds, 16, async (dispatchId) => { - requireHomeAttachment(runtime, dispatchId, authenticatedCallerFingerprint) + const attachment = runtime.getOrchestrationDb().getRemoteDispatchAttachment(dispatchId) + // A Dispatch this home no longer holds here, such as one from an earlier pairing, must not hide the rest. + if (!attachment || attachment.home_peer_fingerprint !== authenticatedCallerFingerprint) { + return { + dispatchId, + observation: { status: 'unverifiable' as const, exactWorker: false } + } + } const observation = await observeRemoteAttachmentLiveness(runtime, dispatchId) const db = runtime.getOrchestrationDb() const autoReleaseAt = db.getLiveWorkerTerminalAutoReleaseAt(dispatchId) diff --git a/src/main/runtime/rpc/methods/orchestration/federation/federation-liveness-verdict.test.ts b/src/main/runtime/rpc/methods/orchestration/federation/federation-liveness-verdict.test.ts index 65bd9c84da4..20f434ede46 100644 --- a/src/main/runtime/rpc/methods/orchestration/federation/federation-liveness-verdict.test.ts +++ b/src/main/runtime/rpc/methods/orchestration/federation/federation-liveness-verdict.test.ts @@ -234,6 +234,35 @@ describe('federation host liveness verdicts', () => { }) }) + it('still reports its own workers when one requested Dispatch belongs to an earlier pairing', async () => { + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: observation reads only these terminal fields. + vi.spyOn(runtime, 'showTerminal').mockResolvedValue({ + handle: HANDLE, + worktreeId: 'repo::remote-worktree', + connected: true, + status: 'running' + } as never) + vi.spyOn(runtime, 'getTerminalLivenessVerdict').mockReturnValue(null) + // oxlint-disable-next-line typescript/consistent-type-assertions -- SAFETY: observation reads only the host scope. + vi.spyOn(runtime, 'getOrchestrationDispatchAuthority').mockReturnValue({ + hostScope: { kind: 'local', hostId: 'local' } + } as never) + + await expect( + call('orchestration.federationFleetSnapshot', { + dispatchIds: ['ctx_from_an_earlier_pairing', DISPATCH_ID] + }) + ).resolves.toMatchObject({ + items: [ + { + dispatchId: 'ctx_from_an_earlier_pairing', + observation: { status: 'unverifiable', exactWorker: false } + }, + { dispatchId: DISPATCH_ID, observation: { status: 'live', exactWorker: true } } + ] + }) + }) + it('keeps a disconnected verdict-less pane unverifiable rather than exited', async () => { vi.spyOn(runtime, 'getTerminalLivenessVerdict').mockReturnValue(null) vi.spyOn(runtime, 'getOrchestrationDispatchAuthority').mockReturnValue(null) From 31034a02f2ae9bb93401a916abd4873b1acfec34 Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Mon, 9 Nov 2026 00:57:34 -0500 Subject: [PATCH 2/3] test(orchestration): cover a live Dispatch held for another home in the fleet snapshot --- .../federation-liveness-verdict.test.ts | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/main/runtime/rpc/methods/orchestration/federation/federation-liveness-verdict.test.ts b/src/main/runtime/rpc/methods/orchestration/federation/federation-liveness-verdict.test.ts index 20f434ede46..5270e3dc7f7 100644 --- a/src/main/runtime/rpc/methods/orchestration/federation/federation-liveness-verdict.test.ts +++ b/src/main/runtime/rpc/methods/orchestration/federation/federation-liveness-verdict.test.ts @@ -248,6 +248,40 @@ describe('federation host liveness verdicts', () => { hostScope: { kind: 'local', hostId: 'local' } } as never) + const earlierPane = 'tab_earlier:cccccccc-cccc-4ccc-8ccc-cccccccccccc' + vi.mocked(runtime.getTerminalPaneKey).mockImplementation((handle) => + handle === 'term_earlier_worker' ? earlierPane : PANE_KEY + ) + vi.mocked(runtime.getTerminalProcessIncarnation).mockImplementation((handle) => + handle === 'term_earlier_worker' ? 'runtime:pty:earlier' : INCARNATION + ) + // Started for this server's earlier pairing with the home, so it is held under another fingerprint. + db.createRemoteDispatchAttachment({ + runId: 'run-home', + dispatchId: 'ctx_from_an_earlier_pairing', + taskId: 'task_earlier', + homePeerFingerprint: 'earlier-pairing-fingerprint', + protocolVersion: ORCHESTRATION_CONTRACT_VERSION, + runtimeEpoch: runtime.getRuntimeId(), + mutationReceipt: { + callerFingerprint: 'earlier-pairing-fingerprint', + requestId: 'rpc_attach_earlier', + method: 'orchestration.federationStart', + payloadHash: 'hash' + } + }) + db.prepareRemoteAttachmentAuthority({ + dispatchId: 'ctx_from_an_earlier_pairing', + paneKey: earlierPane, + processIncarnation: 'runtime:pty:earlier', + worktreeId: 'repo::remote-worktree', + terminalHandle: 'term_earlier_worker', + setupState: 'not_applicable', + effects: [], + terminalOwnership: 'created' + }) + db.markRemoteAttachmentReady('ctx_from_an_earlier_pairing') + await expect( call('orchestration.federationFleetSnapshot', { dispatchIds: ['ctx_from_an_earlier_pairing', DISPATCH_ID] @@ -256,11 +290,23 @@ describe('federation host liveness verdicts', () => { items: [ { dispatchId: 'ctx_from_an_earlier_pairing', + // Nothing of another fingerprint's Dispatch is read or returned. observation: { status: 'unverifiable', exactWorker: false } }, { dispatchId: DISPATCH_ID, observation: { status: 'live', exactWorker: true } } ] }) + const earlier = ( + (await call('orchestration.federationFleetSnapshot', { + dispatchIds: ['ctx_from_an_earlier_pairing'] + })) as { items: unknown[] } + ).items + expect(earlier).toEqual([ + { + dispatchId: 'ctx_from_an_earlier_pairing', + observation: { status: 'unverifiable', exactWorker: false } + } + ]) }) it('keeps a disconnected verdict-less pane unverifiable rather than exited', async () => { From f443089675f8abddc3dc2a1120d67030bd54e5e5 Mon Sep 17 00:00:00 2001 From: Jinwoo-H Date: Mon, 9 Nov 2026 01:03:51 -0500 Subject: [PATCH 3/3] test(orchestration): assert the other home's fleet item without a cast --- .../federation-liveness-verdict.test.ts | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/main/runtime/rpc/methods/orchestration/federation/federation-liveness-verdict.test.ts b/src/main/runtime/rpc/methods/orchestration/federation/federation-liveness-verdict.test.ts index 5270e3dc7f7..fc8dd1dad5c 100644 --- a/src/main/runtime/rpc/methods/orchestration/federation/federation-liveness-verdict.test.ts +++ b/src/main/runtime/rpc/methods/orchestration/federation/federation-liveness-verdict.test.ts @@ -296,17 +296,19 @@ describe('federation host liveness verdicts', () => { { dispatchId: DISPATCH_ID, observation: { status: 'live', exactWorker: true } } ] }) - const earlier = ( - (await call('orchestration.federationFleetSnapshot', { + await expect( + call('orchestration.federationFleetSnapshot', { dispatchIds: ['ctx_from_an_earlier_pairing'] - })) as { items: unknown[] } - ).items - expect(earlier).toEqual([ - { - dispatchId: 'ctx_from_an_earlier_pairing', - observation: { status: 'unverifiable', exactWorker: false } - } - ]) + }) + ).resolves.toEqual({ + runtimeEpoch: runtime.getRuntimeId(), + items: [ + { + dispatchId: 'ctx_from_an_earlier_pairing', + observation: { status: 'unverifiable', exactWorker: false } + } + ] + }) }) it('keeps a disconnected verdict-less pane unverifiable rather than exited', async () => {