perf(orchestration): back off failing federation relays instead of polling every second (#39734)

Each federated dispatch relay now reschedules itself and doubles its wait
after consecutive failures (capped at 30 s) instead of syncing every second
forever. A server turning verified, or a sync completed by another caller,
returns the relay to the base cadence. A coordinator blocked in
`check --wait` re-arms its run's relays every 15 s, so a recovered worker is
seen within that window. Relays for a server removed from this host stop
instead of retrying; the next relay arm brings them back if it is re-added.
Stale in-flight syncs no longer record acknowledgments, epochs or home
auto-release after the relay stopped.

Replaces #39283.

Co-authored-by: m4air <m4air@m4airs-MacBook-Air.local>
Co-authored-by: brennanb2025 <79079362+brennanb2025@users.noreply.github.com>
(cherry picked from commit 0f6432319d)
This commit is contained in:
OrcaWin
2026-11-08 10:19:49 -08:00
committed by Brennan Benson
parent d61692bdcc
commit 2fdf870dfd
18 changed files with 713 additions and 136 deletions
@@ -1,6 +1,9 @@
import { getAppEnvironment } from '../../../shared/app-environment'
import { getPreferredPairedRuntimeConnection } from '../../../shared/remote-runtime/host-catalog/runtime-environments'
import { resolveEnvironment } from '../../../shared/remote-runtime/host-store/runtime-environment-store'
import {
listEnvironments,
resolveEnvironment
} from '../../../shared/remote-runtime/host-store/runtime-environment-store'
import {
pairedOrchestrationPeers,
type OrchestrationEnvironmentTransport
@@ -21,6 +24,8 @@ export function createOrchestrationEnvironmentTransport(): OrchestrationEnvironm
pairingRevision: environment.pairingRevision ?? environment.createdAt
}
},
hasEnvironment: (environmentId) =>
listEnvironments(registryRoot()).some((environment) => environment.id === environmentId),
// Loaded on first dial: the routing graph is large and most hosts never place remote workers.
call: async (selector, method, params, timeoutMs, envelope, expectedPairingRevision, send) =>
(await import('./runtime-environment-transport-routing')).callRuntimeEnvironment(
@@ -13,6 +13,7 @@ import { RuntimeHostStatusOwner } from '../../../shared/remote-runtime/host-stat
import type { RuntimeStatus } from '../../../shared/runtime-types'
import {
RUNTIME_HOST_STATUS_CHANNEL,
type RuntimeHostStatusSnapshot,
runtimeHostStatusError,
runtimeHostStatusFailure,
type RuntimeHostStatusResponse
@@ -27,9 +28,11 @@ import { isRuntimeEnvironmentManuallyDisconnected } from './runtime-environment-
import { runtimeEnvironmentChangedFailure } from './runtime-environment-revision-guard'
import { resolveManagedRuntimeEnvironment } from './runtime-environment-managed-tunnel'
const statusListeners = new Set<() => void>()
const statusListeners = new Set<(snapshot: RuntimeHostStatusSnapshot) => void>()
export function onRuntimeEnvironmentStatusPublished(listener: () => void): () => void {
export function onRuntimeEnvironmentStatusPublished(
listener: (snapshot: RuntimeHostStatusSnapshot) => void
): () => void {
statusListeners.add(listener)
return () => statusListeners.delete(listener)
}
@@ -144,7 +147,7 @@ export function createRuntimeEnvironmentStatusOwner(
publish: (snapshot) => {
getRuntimeDesktopSurface().sendToRenderers?.(RUNTIME_HOST_STATUS_CHANNEL, snapshot)
for (const listener of statusListeners) {
listener()
listener(snapshot)
}
}
})
@@ -0,0 +1,56 @@
import { afterEach, describe, expect, it, vi } from 'vitest'
import type { RuntimeHostStatusSnapshot } from '../../../shared/remote-runtime/host-status/runtime-host-status'
import { trackRuntimeEnvironmentVerifiedTransitions } from './runtime-environment-verified-transitions'
import { FederationRelayScheduler } from '../../runtime/orchestration/federation/relay/federation-relay-scheduler'
let sequence = 0
function snapshot(
environmentId: string,
verification: RuntimeHostStatusSnapshot['verification']
): RuntimeHostStatusSnapshot {
sequence += 1
return {
environmentId,
pairingRevision: 1,
sequence,
checkedAt: sequence,
status: null,
verification,
transport: 'ready'
}
}
describe('trackRuntimeEnvironmentVerifiedTransitions', () => {
afterEach(() => {
vi.useRealTimers()
})
it('resumes a backed-off relay at once when its server is verified again', async () => {
vi.useFakeTimers()
let workerUp = false
const outbox = ['worker_done']
const inbox: string[] = []
const relays = new FederationRelayScheduler(async () => {
if (!workerUp) {
return 'failed'
}
inbox.push(...outbox.splice(0))
return 'synced'
})
const publish = trackRuntimeEnvironmentVerifiedTransitions((environmentId) =>
relays.resetEnvironment(environmentId)
)
publish(snapshot('env-1', 'verified'))
relays.arm('dispatch-1', 'env-1')
// Four failures leave the relay in a 16 s backoff.
await vi.advanceTimersByTimeAsync(14_000)
publish(snapshot('env-1', 'unavailable'))
workerUp = true
publish(snapshot('env-1', 'verified'))
await vi.advanceTimersByTimeAsync(0)
expect(inbox).toEqual(['worker_done'])
relays.clear()
})
})
@@ -0,0 +1,17 @@
import type { RuntimeHostStatusSnapshot } from '../../../shared/remote-runtime/host-status/runtime-host-status'
/** Reports a server only when it newly turns verified; repeat publishes of a healthy server stay silent. */
export function trackRuntimeEnvironmentVerifiedTransitions(
listener: (environmentId: string) => void
): (snapshot: RuntimeHostStatusSnapshot) => void {
const verified = new Set<string>()
return (snapshot) => {
const { environmentId } = snapshot
if (snapshot.verification !== 'verified' || snapshot.retired) {
verified.delete(environmentId)
} else if (!verified.has(environmentId)) {
verified.add(environmentId)
listener(environmentId)
}
}
}
@@ -120,9 +120,9 @@ export function updateFederatedDispatchRuntimeEpoch(
.prepare(
`UPDATE federated_dispatches
SET remote_runtime_epoch = ?, updated_at = datetime('now')
WHERE dispatch_id = ?`
WHERE dispatch_id = ? AND remote_runtime_epoch IS NOT ?`
)
.run(remoteRuntimeEpoch, dispatchId)
.run(remoteRuntimeEpoch, dispatchId, remoteRuntimeEpoch)
}
export type FederatedDispatchStoreMethods = {
@@ -29,6 +29,8 @@ export type OrchestrationEnvironmentSendOptions = {
export type OrchestrationEnvironmentTransport = {
resolve(selector: string): OrchestrationWorkerServer
/** False once the server is removed from this host, so its relays stop instead of retrying forever. */
hasEnvironment?(environmentId: string): boolean
call(
selector: string,
method: string,
@@ -38,6 +40,8 @@ export type OrchestrationEnvironmentTransport = {
expectedEnvironmentPairingRevision?: number,
send?: OrchestrationEnvironmentSendOptions
): Promise<RuntimeRpcResponse<unknown>>
/** Fires when a server's status turns verified, so backed-off relays retry at once. */
onEnvironmentVerified?(listener: (environmentId: string) => void): () => void
}
export function fingerprintOrchestrationPeer(publicKeyB64: string): string {
@@ -0,0 +1,136 @@
import { federationRelayRetryDelayMs } from './federation-relay-scheduler'
type PendingAcknowledgmentRow = { dispatchId: string; rowId: number }
type RetryDeadline = { failures: number; dueAt: number }
/** Walks pending rows from the cursor once around, skipping rows still cooling after a failure. */
function nextDueAcknowledgment(
findNextAfter: (afterRowId: number) => PendingAcknowledgmentRow | undefined,
cursorRowId: number,
deadlines: Map<string, RetryDeadline>,
now: number
): PendingAcknowledgmentRow | { waitMs: number } | null {
const pending = new Set<string>()
let afterRowId = cursorRowId
let wrapped = afterRowId === 0
for (;;) {
const row = findNextAfter(afterRowId)
if (!row) {
if (wrapped) {
break
}
wrapped = true
afterRowId = 0
continue
}
if (pending.has(row.dispatchId)) {
break
}
pending.add(row.dispatchId)
if ((deadlines.get(row.dispatchId)?.dueAt ?? 0) <= now) {
return row
}
afterRowId = row.rowId
}
let earliest = Infinity
for (const [dispatchId, { dueAt }] of deadlines) {
if (pending.has(dispatchId)) {
earliest = Math.min(earliest, dueAt)
} else {
deadlines.delete(dispatchId)
}
}
return pending.size === 0 ? null : { waitMs: Math.max(0, earliest - now) }
}
/** Settles terminal dispatches' outstanding acknowledgments one row per tick; a failing row cools off without holding back the rest. */
export class FederationAcknowledgmentRecovery {
private timer: ReturnType<typeof setTimeout> | null = null
private inFlight: Promise<void> | null = null
private cursorRowId = 0
private generation = 0
private readonly deadlines = new Map<string, RetryDeadline>()
constructor(
private readonly deps: {
findNextAfter: (afterRowId: number) => PendingAcknowledgmentRow | undefined
sync: (dispatchId: string) => Promise<unknown>
acknowledgedThrough: (dispatchId: string) => number | null
}
) {}
ensure(): void {
if (this.timer || this.inFlight) {
return
}
const recovery = this.recoverNext(this.generation).catch((error) => {
console.warn('[orchestration] terminal federation acknowledgment recovery failed', error)
})
this.inFlight = recovery
void recovery.finally(() => {
if (this.inFlight === recovery) {
this.inFlight = null
}
})
}
reset(): void {
this.generation += 1
if (this.timer) {
clearTimeout(this.timer)
}
this.timer = null
this.inFlight = null
this.cursorRowId = 0
this.deadlines.clear()
}
private async recoverNext(generation: number): Promise<void> {
const next = nextDueAcknowledgment(
this.deps.findNextAfter,
this.cursorRowId,
this.deadlines,
Date.now()
)
if (!next) {
return
}
if ('waitMs' in next) {
this.schedule(next.waitMs)
return
}
this.cursorRowId = next.rowId
const acknowledgedBefore = this.deps.acknowledgedThrough(next.dispatchId)
const synced = await this.deps.sync(next.dispatchId).then(
() => true,
() => false
)
if (generation !== this.generation) {
return
}
// Large backlogs settle a few pages per sync; only a sync that left the acknowledgment where it was made no progress.
const progressed =
synced &&
(this.deps.findNextAfter(next.rowId - 1)?.dispatchId !== next.dispatchId ||
this.deps.acknowledgedThrough(next.dispatchId) !== acknowledgedBefore)
if (progressed) {
this.deadlines.delete(next.dispatchId)
} else {
const failures = (this.deadlines.get(next.dispatchId)?.failures ?? 0) + 1
this.deadlines.set(next.dispatchId, {
failures,
dueAt: Date.now() + federationRelayRetryDelayMs(failures)
})
}
this.schedule(federationRelayRetryDelayMs(0))
}
private schedule(delayMs: number): void {
this.timer = setTimeout(() => {
this.timer = null
this.ensure()
}, delayMs)
this.timer.unref?.()
}
}
@@ -0,0 +1,66 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import {
FederationRelayScheduler,
type FederationRelayTickOutcome
} from './federation-relay-scheduler'
/** A worker outbox relayed into the coordinator's inbox whenever the worker answers. */
function relayedMailbox() {
const outbox: string[] = []
const inbox: string[] = []
let workerUp = false
let pendingTick: { finish: () => void } | null = null
let holdNextTick = false
const deliver = (): FederationRelayTickOutcome => {
if (!workerUp) {
return 'failed'
}
inbox.push(...outbox.splice(0))
return 'synced'
}
const tick = async (): Promise<FederationRelayTickOutcome> => {
if (holdNextTick) {
holdNextTick = false
await new Promise<void>((resolve) => (pendingTick = { finish: resolve }))
return 'failed'
}
return deliver()
}
return {
outbox,
inbox,
tick,
holdNextTick: () => {
holdNextTick = true
},
finishHeldTick: () => pendingTick?.finish(),
recover: () => {
workerUp = true
}
}
}
describe('FederationRelayScheduler', () => {
beforeEach(() => {
vi.useFakeTimers()
})
afterEach(() => {
vi.useRealTimers()
})
it('delivers a worker message that lands while a failing sync is in flight', async () => {
const mailbox = relayedMailbox()
const relays = new FederationRelayScheduler(mailbox.tick)
mailbox.holdNextTick()
relays.arm('dispatch-1', 'env-1')
mailbox.recover()
mailbox.outbox.push('worker_done')
relays.arm('dispatch-1', 'env-1')
mailbox.finishHeldTick()
await vi.advanceTimersByTimeAsync(0)
expect(mailbox.inbox).toEqual(['worker_done'])
relays.clear()
})
})
@@ -0,0 +1,132 @@
const RELAY_BASE_DELAY_MS = 1_000
// 30 s keeps an offline host to two failed syncs a minute, and a recovered one at most 30 s behind.
const RELAY_MAX_DELAY_MS = 30_000
/** How often a coordinator blocked in `check --wait` re-arms its run's relays, so recovery is seen within this. */
export const FEDERATION_RELAY_WAIT_REARM_MS = 15_000
export type FederationRelayTickOutcome = 'synced' | 'failed' | 'ended'
/** Consecutive failures double the wait up to the cap; a failure never ends a relay, since an unreachable host or dispatch can come back. */
export function federationRelayRetryDelayMs(failures: number): number {
return failures === 0
? RELAY_BASE_DELAY_MS
: Math.min(RELAY_MAX_DELAY_MS, RELAY_BASE_DELAY_MS * 2 ** failures)
}
type RelayChain = {
environmentId: string
failures: number
timer: ReturnType<typeof setTimeout> | null
dueAt: number
running: boolean
runAgain: boolean
}
/** One self-rescheduling sync per dispatch, so a dead or unreachable dispatch backs off instead of polling every second. */
export class FederationRelayScheduler {
private readonly chains = new Map<string, RelayChain>()
private readonly dispatchesByEnvironment = new Map<string, Set<string>>()
constructor(private readonly tick: (dispatchId: string) => Promise<FederationRelayTickOutcome>) {}
/** Starts a relay, or asks an existing one to sync now without forgetting its failures. */
arm(dispatchId: string, environmentId: string): void {
const chain = this.chains.get(dispatchId)
if (chain) {
this.syncSoon(dispatchId, chain)
return
}
const created: RelayChain = {
environmentId,
failures: 0,
timer: null,
dueAt: 0,
running: false,
runAgain: false
}
this.chains.set(dispatchId, created)
const siblings = this.dispatchesByEnvironment.get(environmentId) ?? new Set<string>()
siblings.add(dispatchId)
this.dispatchesByEnvironment.set(environmentId, siblings)
void this.run(dispatchId, created)
}
/** A server that just verified is worth retrying at the base cadence. */
resetEnvironment(environmentId: string): void {
for (const dispatchId of this.dispatchesByEnvironment.get(environmentId) ?? []) {
const chain = this.chains.get(dispatchId)
if (chain) {
chain.failures = 0
this.syncSoon(dispatchId, chain)
}
}
}
/** A sync another caller completed proves the relay healthy again, so the next one returns to the base cadence. */
noteSynced(dispatchId: string): void {
const chain = this.chains.get(dispatchId)
if (!chain || chain.running || chain.failures === 0) {
return
}
chain.failures = 0
const baseDelayMs = federationRelayRetryDelayMs(0)
// Only pull the next sync earlier; an already-requested immediate sync must not be postponed.
if (!chain.timer || chain.dueAt > Date.now() + baseDelayMs) {
this.schedule(dispatchId, chain, baseDelayMs)
}
}
clear(): void {
for (const chain of this.chains.values()) {
if (chain.timer) {
clearTimeout(chain.timer)
}
}
this.chains.clear()
this.dispatchesByEnvironment.clear()
}
private syncSoon(dispatchId: string, chain: RelayChain): void {
if (chain.running) {
chain.runAgain = true
return
}
this.schedule(dispatchId, chain, 0)
}
private schedule(dispatchId: string, chain: RelayChain, delayMs: number): void {
if (chain.timer) {
clearTimeout(chain.timer)
}
chain.dueAt = Date.now() + delayMs
chain.timer = setTimeout(() => void this.run(dispatchId, chain), delayMs)
chain.timer.unref?.()
}
private async run(dispatchId: string, chain: RelayChain): Promise<void> {
if (this.chains.get(dispatchId) !== chain) {
return
}
chain.timer = null
chain.running = true
const outcome = await this.tick(dispatchId).catch((): FederationRelayTickOutcome => 'failed')
chain.running = false
// A cleared or replaced chain must not come back from a sync that outlived it.
if (this.chains.get(dispatchId) !== chain) {
return
}
if (outcome === 'ended') {
this.chains.delete(dispatchId)
const siblings = this.dispatchesByEnvironment.get(chain.environmentId)
siblings?.delete(dispatchId)
if (siblings?.size === 0) {
this.dispatchesByEnvironment.delete(chain.environmentId)
}
return
}
chain.failures = outcome === 'synced' ? 0 : chain.failures + 1
const delayMs = chain.runAgain ? 0 : federationRelayRetryDelayMs(chain.failures)
chain.runAgain = false
this.schedule(dispatchId, chain, delayMs)
}
}
@@ -0,0 +1,65 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { createIdleSyncHarness } from './synchronization/federation-sync-test-harness'
import { RuntimeOrchestrationFederation } from './runtime-orchestration-federation'
import type { OrchestrationEnvironmentTransport } from './environment-transport'
describe('RuntimeOrchestrationFederation relay', () => {
beforeEach(() => {
vi.useFakeTimers()
})
afterEach(() => {
vi.useRealTimers()
})
it('stops relaying to a removed server and delivers its pending mail after the next ensureRelay', async () => {
const { runtime, remoteCall } = createIdleSyncHarness()
const db = runtime.getOrchestrationDb()
const active = db.getFederatedDispatch('dispatch_remote')
db.listActiveFederatedDispatches = () => (active ? [active] : [])
let serverSaved = false
let noteRefused!: () => void
const refused = new Promise<void>((resolve) => (noteRefused = resolve))
let noteAcknowledged!: () => void
const acknowledgedOnce = new Promise<void>((resolve) => (noteAcknowledged = resolve))
const recordAcknowledgment = db.recordFederatedHomeAcknowledgment.bind(db)
db.recordFederatedHomeAcknowledgment = (params) => {
recordAcknowledgment(params)
noteAcknowledged()
}
const answer = remoteCall.getMockImplementation()!
remoteCall.mockImplementation(async (...args) => {
if (!serverSaved) {
noteRefused()
throw new Error('Unknown environment: environment_windows')
}
return answer(...args)
})
const transport: OrchestrationEnvironmentTransport = {
resolve: () => {
throw new Error('unused')
},
call: () => {
throw new Error('unused')
},
hasEnvironment: () => serverSaved
}
const federation = new RuntimeOrchestrationFederation(runtime, transport)
const acknowledged = () =>
runtime.getOrchestrationDb().getFederatedDispatch('dispatch_remote')
?.to_home_acknowledged_sequence
federation.ensureRelay()
await refused
await vi.advanceTimersByTimeAsync(0)
serverSaved = true
// A relay that ended leaves no timer behind; one still backing off would retry and deliver here.
await vi.runAllTimersAsync()
expect(acknowledged()).toBe(0)
federation.ensureRelay()
await acknowledgedOnce
expect(acknowledged()).toBeGreaterThan(0)
federation.stopRelay()
})
})
@@ -24,15 +24,28 @@ import {
import { syncFederatedDispatch } from './synchronization/federation-sync'
import type { OrchestrationDb } from '../db/orchestration-db'
import type { FederationRuntime } from './federation-runtime'
import {
FederationRelayScheduler,
type FederationRelayTickOutcome
} from './relay/federation-relay-scheduler'
import { FederationAcknowledgmentRecovery } from './relay/federation-acknowledgment-recovery'
export class RuntimeOrchestrationFederation {
private readonly timers = new Map<string, ReturnType<typeof setInterval>>()
private readonly relays = new FederationRelayScheduler((dispatchId) => this.relayTick(dispatchId))
private readonly acknowledgmentRecovery = new FederationAcknowledgmentRecovery({
findNextAfter: (afterRowId) =>
this.runtime
.getOrchestrationDb()
.findNextTerminalFederatedDispatchPendingAcknowledgment(afterRowId),
sync: (dispatchId) => this.runtime.syncOrchestrationFederatedDispatch(dispatchId),
acknowledgedThrough: (dispatchId) =>
this.runtime.getOrchestrationDb().getFederatedDispatch(dispatchId)
?.to_home_acknowledged_sequence ?? null
})
private readonly syncs = new Map<string, { db: OrchestrationDb; promise: Promise<void> }>()
private readonly warnings = new Set<string>()
private terminalRecoveryTimer: ReturnType<typeof setTimeout> | null = null
private terminalRecoveryInFlight: Promise<void> | null = null
private terminalRecoveryRowId = 0
private relayGeneration = 0
private stopVerifiedWatch: (() => void) | null = null
/** Worker server -> runtime incarnation that last proved the orchestration contract. */
private readonly contractVerifiedRuntimeIds = new Map<string, string>()
@@ -43,16 +56,9 @@ export class RuntimeOrchestrationFederation {
resetForDatabaseChange(): void {
this.relayGeneration += 1
for (const timer of this.timers.values()) {
clearInterval(timer)
}
this.timers.clear()
if (this.terminalRecoveryTimer) {
clearTimeout(this.terminalRecoveryTimer)
}
this.terminalRecoveryTimer = null
this.terminalRecoveryInFlight = null
this.terminalRecoveryRowId = 0
this.unwatchVerifiedEnvironments()
this.relays.clear()
this.acknowledgmentRecovery.reset()
clearFederationAckCheckpoints(this.runtime)
this.syncs.clear()
this.warnings.clear()
@@ -172,10 +178,14 @@ export class RuntimeOrchestrationFederation {
if (current?.db === db) {
return current.promise
}
const sync = syncFederatedDispatch(this.runtime, dispatchId)
const generation = this.relayGeneration
const isCurrent = (): boolean =>
generation === this.relayGeneration && this.runtime.getOrchestrationDb() === db
const sync = syncFederatedDispatch(this.runtime, dispatchId, isCurrent)
.then(() => {
if (this.syncs.get(dispatchId)?.promise === sync) {
this.warnings.delete(dispatchId)
this.relays.noteSynced(dispatchId)
}
})
.catch((error: unknown) => {
@@ -200,10 +210,14 @@ export class RuntimeOrchestrationFederation {
async syncDispatchAfterCurrent(dispatchId: string): Promise<void> {
const db = this.runtime.getOrchestrationDb()
const generation = this.relayGeneration
const current = this.syncs.get(dispatchId)
if (current?.db === db) {
await current.promise.catch(() => undefined)
}
if (generation !== this.relayGeneration || this.runtime.getOrchestrationDb() !== db) {
return
}
await this.syncDispatch(dispatchId)
}
@@ -211,87 +225,61 @@ export class RuntimeOrchestrationFederation {
if (!this.transport) {
return
}
this.stopVerifiedWatch ??=
this.transport.onEnvironmentVerified?.((environmentId) =>
this.relays.resetEnvironment(environmentId)
) ?? null
for (const dispatch of this.runtime.getOrchestrationDb().listActiveFederatedDispatches(runId)) {
if (this.timers.has(dispatch.dispatch_id)) {
continue
}
const tick = () => {
const db = this.runtime.getOrchestrationDb()
if (!db.isFederatedDispatchRelayEligible(dispatch.dispatch_id)) {
const activeTimer = this.timers.get(dispatch.dispatch_id)
if (activeTimer) {
clearInterval(activeTimer)
}
this.timers.delete(dispatch.dispatch_id)
this.warnings.delete(dispatch.dispatch_id)
return
this.relays.arm(dispatch.dispatch_id, dispatch.environment_id)
}
this.acknowledgmentRecovery.ensure()
}
private async relayTick(dispatchId: string): Promise<FederationRelayTickOutcome> {
if (!this.runtime.getOrchestrationDb().isFederatedDispatchRelayEligible(dispatchId)) {
this.warnings.delete(dispatchId)
return 'ended'
}
return this.syncDispatch(dispatchId).then(
(): FederationRelayTickOutcome => 'synced',
(): FederationRelayTickOutcome => {
if (!this.isEnvironmentRemoved(dispatchId)) {
return 'failed'
}
void this.syncDispatch(dispatch.dispatch_id).catch(() => undefined)
this.warnings.delete(dispatchId)
return 'ended'
}
const timer = setInterval(tick, 1_000)
timer.unref?.()
this.timers.set(dispatch.dispatch_id, timer)
tick()
}
this.ensureTerminalHistoryRecovery()
}
private ensureTerminalHistoryRecovery(): void {
if (this.terminalRecoveryTimer || this.terminalRecoveryInFlight) {
return
}
const generation = this.relayGeneration
const recovery = this.recoverNextTerminalHistoryAcknowledgment(generation).catch((error) => {
console.warn('[orchestration] terminal federation acknowledgment recovery failed', error)
})
this.terminalRecoveryInFlight = recovery
void recovery.finally(() => {
if (this.terminalRecoveryInFlight === recovery) {
this.terminalRecoveryInFlight = null
}
})
}
private async recoverNextTerminalHistoryAcknowledgment(generation: number): Promise<void> {
const db = this.runtime.getOrchestrationDb()
let historical = db.findNextTerminalFederatedDispatchPendingAcknowledgment(
this.terminalRecoveryRowId
)
if (!historical && this.terminalRecoveryRowId > 0) {
this.terminalRecoveryRowId = 0
historical = db.findNextTerminalFederatedDispatchPendingAcknowledgment(0)
}
/** Checked only after a failed sync; a store read error keeps the relay retrying. */
private isEnvironmentRemoved(dispatchId: string): boolean {
const environmentId = this.runtime
.getOrchestrationDb()
.getFederatedDispatch(dispatchId)?.environment_id
if (environmentId === undefined) {
return false
}
if (!historical) {
return
try {
// A removed server cannot answer; the next ensureRelay re-arms it if the server is added back.
return this.transport?.hasEnvironment?.(environmentId) === false
} catch {
return false
}
this.terminalRecoveryRowId = historical.rowId
await this.runtime
.syncOrchestrationFederatedDispatch(historical.dispatchId)
.catch(() => undefined)
if (generation !== this.relayGeneration) {
return
}
this.terminalRecoveryTimer = setTimeout(() => {
this.terminalRecoveryTimer = null
this.ensureTerminalHistoryRecovery()
}, 1_000)
this.terminalRecoveryTimer.unref?.()
}
private unwatchVerifiedEnvironments(): void {
this.stopVerifiedWatch?.()
this.stopVerifiedWatch = null
}
stopRelay(): void {
this.relayGeneration += 1
for (const timer of this.timers.values()) {
clearInterval(timer)
}
this.timers.clear()
this.unwatchVerifiedEnvironments()
this.relays.clear()
this.acknowledgmentRecovery.reset()
this.warnings.clear()
this.syncs.clear()
if (this.terminalRecoveryTimer) {
clearTimeout(this.terminalRecoveryTimer)
}
this.terminalRecoveryTimer = null
this.terminalRecoveryInFlight = null
this.terminalRecoveryRowId = 0
clearFederationAckCheckpoints(this.runtime)
}
}
@@ -0,0 +1,52 @@
import { describe, expect, it } from 'vitest'
import { ORCHESTRATION_FEDERATION_RELEASE_ARCHIVE_RUNTIME_CAPABILITY } from '../../../../../shared/orchestration-runtime-capabilities'
import { createIdleSyncHarness } from './federation-sync-test-harness'
import { armHomeAutoReleaseForOlderHost } from './federation-sync-capability'
function releasableTerminalWorker() {
const harness = createIdleSyncHarness()
harness.setRemoteCapabilities([ORCHESTRATION_FEDERATION_RELEASE_ARCHIVE_RUNTIME_CAPABILITY])
const db = harness.runtime.getOrchestrationDb()
const federated = db.getFederatedDispatch('dispatch_remote')!
db.getFederatedDispatch = () => ({
...federated,
remote_terminal_handle: 'term_remote_worker',
home_auto_release_state: null
})
const worker = db.getWorkerDispatch('dispatch_remote')!
db.getWorkerDispatch = () => ({ ...worker, state: 'succeeded', stage: 'reported' })
db.workerAutoReleaseEnabled = () => true
return {
runtime: harness.runtime,
remoteCall: harness.remoteCall,
dispatchId: federated.dispatch_id,
armedDeadlines: () => db.claimDueFederatedHomeAutoReleases(Date.UTC(2100, 0, 1))
}
}
describe('armHomeAutoReleaseForOlderHost', () => {
it('does not arm from a sync the relay stopped while it probed the host', async () => {
const { runtime, remoteCall, dispatchId, armedDeadlines } = releasableTerminalWorker()
const answer = remoteCall.getMockImplementation()!
let noteProbe!: () => void
let releaseProbe!: () => void
const probing = new Promise<void>((resolve) => (noteProbe = resolve))
const released = new Promise<void>((resolve) => (releaseProbe = resolve))
remoteCall.mockImplementation(async (...args) => {
if (args[1] === 'status.get') {
noteProbe()
await released
}
return answer(...args)
})
let current = true
const arming = armHomeAutoReleaseForOlderHost(runtime, dispatchId, 1, () => current)
await probing
current = false
releaseProbe()
await arming
expect(armedDeadlines()).toEqual([])
})
})
@@ -55,7 +55,8 @@ function resolveFederatedHostCapability(
export async function armHomeAutoReleaseForOlderHost(
runtime: FederationRuntime,
dispatchId: string,
pairingRevision: number | undefined
pairingRevision: number | undefined,
isCurrent: () => boolean = () => true
): Promise<void> {
const db = runtime.getOrchestrationDb()
const federated = db.getFederatedDispatch(dispatchId)
@@ -84,7 +85,7 @@ export async function armHomeAutoReleaseForOlderHost(
)
)
)
if (!hostReleases?.supported && releasable?.supported !== false) {
if (isCurrent() && !hostReleases?.supported && releasable?.supported !== false) {
db.armFederatedHomeAutoRelease(dispatchId)
}
}
@@ -1,3 +1,4 @@
import { z } from 'zod'
import {
MESSAGE_TYPES,
type MessagePriority,
@@ -102,3 +103,22 @@ export function parseFederatedLifecycle(
})
}
}
// Decode untrusted peer pages before import to report malformed input as an orchestration error.
export const PulledRelayPage = z
.object({
runtimeEpoch: z.string().min(1),
items: z.array(
z
.object({
dispatch_id: z.string(),
direction: z.literal('to_home'),
sequence: z.number(),
message_id: z.string(),
kind: z.string(),
payload: z.string()
})
.passthrough()
)
})
.passthrough()
@@ -538,6 +538,29 @@ describe('federation relay acknowledgments', () => {
remoteCall.mock.calls.filter(([, method]) => method === 'orchestration.federationAck')
).toHaveLength(2)
})
it.each(['stopRelay', 'database swap'] as const)(
'keeps a sync interrupted by %s from acknowledging into current state',
async (interruption) => {
const { runtime, blockAck, replaceDb } = createIdleSyncHarness()
const gate = blockAck()
const oldSync = runtime.syncOrchestrationFederatedDispatch('dispatch_remote')
await gate.reached
if (interruption === 'stopRelay') {
runtime.stopOrchestrationFederationRelay()
} else {
replaceDb()
}
gate.release()
await oldSync
expect(
runtime.getOrchestrationDb().getFederatedDispatch('dispatch_remote')
?.to_home_acknowledged_sequence
).toBe(0)
}
)
})
describe('bounded outgoing answer relay', () => {
@@ -1,6 +1,5 @@
import { federationRelayPrefix } from '../relay/federation-relay-prefix'
import { federationRelayWireItem } from '../relay/federation-relay-wire'
import { z } from 'zod'
import type { FederationRuntime } from '../federation-runtime'
import type { FederatedLifecycleSettlement } from '../federation-lifecycle-settlement'
import '../../../../../shared/protocol-version'
@@ -17,30 +16,15 @@ import {
resolveFederatedLifecycleSettlementCapability
} from './federation-sync-capability'
import { getOrchestrationPeerCapabilityCache } from '../orchestration-peer-capability-cache'
import { parseFederatedLifecycle, parseRelayedMessage } from './federation-sync-message'
import {
parseFederatedLifecycle,
parseRelayedMessage,
PulledRelayPage
} from './federation-sync-message'
const FEDERATION_PULL_PAGE_SIZE = 50
const MAX_FEDERATION_PULL_PAGES_PER_SYNC = 6
// Decode untrusted peer pages before import to report malformed input as an orchestration error.
const PulledRelayPage = z
.object({
runtimeEpoch: z.string().min(1),
items: z.array(
z
.object({
dispatch_id: z.string(),
direction: z.literal('to_home'),
sequence: z.number(),
message_id: z.string(),
kind: z.string(),
payload: z.string()
})
.passthrough()
)
})
.passthrough()
export async function syncFederatedDispatch(
runtime: FederationRuntime,
dispatchId: string,
@@ -85,6 +69,9 @@ async function syncFederatedDispatchPages(
federated,
currentServer.pairingRevision
)
if (!isCurrent()) {
return { imported: 0, acknowledgedThrough: federated.to_home_imported_sequence }
}
const supportsLifecycleSettlement =
federated.protocol_version >= ORCHESTRATION_FEDERATION_LIFECYCLE_SETTLEMENT_PROTOCOL_VERSION &&
capability?.supported === true
@@ -221,26 +208,31 @@ async function syncFederatedDispatchPages(
const locallyAcknowledgedThrough = keepRelayEligible
? Math.max(0, delivered.acknowledgedThrough - 1)
: delivered.acknowledgedThrough
db.recordFederatedHomeAcknowledgment({
dispatchId,
remoteRuntimeEpoch: pulled.runtimeEpoch,
sequence: locallyAcknowledgedThrough
})
recordFederationAckCheckpoint(runtime, ackLease, {
...ackIdentity,
throughSequence: locallyAcknowledgedThrough
})
if (isCurrent()) {
db.recordFederatedHomeAcknowledgment({
dispatchId,
remoteRuntimeEpoch: pulled.runtimeEpoch,
sequence: locallyAcknowledgedThrough
})
recordFederationAckCheckpoint(runtime, ackLease, {
...ackIdentity,
throughSequence: locallyAcknowledgedThrough
})
}
}
getOrchestrationPeerCapabilityCache(runtime).observeEpoch(
federated.peer_fingerprint,
pulled.runtimeEpoch
)
db.updateFederatedDispatchRuntimeEpoch(dispatchId, pulled.runtimeEpoch)
if (isCurrent()) {
// Best effort: without a deadline the worker keeps manual release, as before.
await armHomeAutoReleaseForOlderHost(runtime, dispatchId, currentServer.pairingRevision).catch(
() => undefined
getOrchestrationPeerCapabilityCache(runtime).observeEpoch(
federated.peer_fingerprint,
pulled.runtimeEpoch
)
db.updateFederatedDispatchRuntimeEpoch(dispatchId, pulled.runtimeEpoch)
// Best effort: without a deadline the worker keeps manual release, as before.
await armHomeAutoReleaseForOlderHost(
runtime,
dispatchId,
currentServer.pairingRevision,
isCurrent
).catch(() => undefined)
}
const toWorker =
db.getWorkerDispatch(dispatchId)?.state === 'ready'
@@ -13,6 +13,7 @@ import type { CheckParams } from '../../../../../../shared/rpc-contract/orchestr
import type { z } from 'zod'
import type { OrchestrationSessionCaller } from '../../../../orchestration/addressing/orchestration-caller-identity'
import { checkRunPendingMail } from './check-run-pending-mail'
import { FEDERATION_RELAY_WAIT_REARM_MS } from '../../../../orchestration/federation/relay/federation-relay-scheduler'
type CheckParamsInput = z.infer<typeof CheckParams>
@@ -164,11 +165,21 @@ export async function checkRunMailbox(args: {
}
}
const waitResult = await runtime.waitForMessage(address, {
typeFilter: typeFilter as string[] | undefined,
timeoutMs: params.timeoutMs ?? undefined,
signal
})
const rearmRelays = setInterval(() => {
try {
runtime.ensureOrchestrationFederationRelay(run.id)
} catch (error) {
console.warn('[orchestration] federation relay re-arm failed during check --wait', error)
}
}, FEDERATION_RELAY_WAIT_REARM_MS)
rearmRelays.unref?.()
const waitResult = await runtime
.waitForMessage(address, {
typeFilter: typeFilter as string[] | undefined,
timeoutMs: params.timeoutMs ?? undefined,
signal
})
.finally(() => clearInterval(rearmRelays))
try {
revalidateConsumer()
} catch (error) {
@@ -25,6 +25,8 @@ import type { TerminalSideEffectBatch } from '../../shared/terminal/output/termi
import { resolveTuiAgentLaunchEnv } from '../../shared/agent-launch/choices/tui-agent-launch-defaults'
import { callRuntimeEnvironment } from '../ipc/runtime-environments/runtime-environment-transport-routing'
import { createOrchestrationEnvironmentTransport } from '../ipc/runtime-environments/orchestration-environment-transport'
import { trackRuntimeEnvironmentVerifiedTransitions } from '../ipc/runtime-environments/runtime-environment-verified-transitions'
import { onRuntimeEnvironmentStatusPublished } from '../ipc/runtime-environments/runtime-environment-status-owner'
import { mainProcessState as state } from './main-process-state'
import { suspendEphemeralVmWorkspace } from '../ephemeral-vm/workspace-suspend'
import {
@@ -114,7 +116,11 @@ export function initializeMainProcessRuntime(): OrcaRuntimeService {
homePath,
resolveTuiAgentLaunchEnv('codex', store.getSettings())
),
orchestrationEnvironmentTransport: createOrchestrationEnvironmentTransport(),
orchestrationEnvironmentTransport: {
...createOrchestrationEnvironmentTransport(),
onEnvironmentVerified: (listener) =>
onRuntimeEnvironmentStatusPublished(trackRuntimeEnvironmentVerifiedTransitions(listener))
},
workspaceSleepPlatform: {
callEnvironment: (environmentId, method, params, timeoutMs) =>
callRuntimeEnvironment(