diff --git a/src/main/ipc/runtime-environments/orchestration-environment-transport.ts b/src/main/ipc/runtime-environments/orchestration-environment-transport.ts index 64aeaa95c35..721ccd156c3 100644 --- a/src/main/ipc/runtime-environments/orchestration-environment-transport.ts +++ b/src/main/ipc/runtime-environments/orchestration-environment-transport.ts @@ -1,6 +1,9 @@ import { getAppEnvironment } from '../../../shared/app-environment' import { getPreferredPairedRuntimeConnection } from '../../../shared/remote-runtime/host-catalog/runtime-environments' -import { resolveEnvironment } from '../../../shared/remote-runtime/host-store/runtime-environment-store' +import { + listEnvironments, + resolveEnvironment +} from '../../../shared/remote-runtime/host-store/runtime-environment-store' import { pairedOrchestrationPeers, type OrchestrationEnvironmentTransport @@ -21,6 +24,8 @@ export function createOrchestrationEnvironmentTransport(): OrchestrationEnvironm pairingRevision: environment.pairingRevision ?? environment.createdAt } }, + hasEnvironment: (environmentId) => + listEnvironments(registryRoot()).some((environment) => environment.id === environmentId), // Loaded on first dial: the routing graph is large and most hosts never place remote workers. call: async (selector, method, params, timeoutMs, envelope, expectedPairingRevision, send) => (await import('./runtime-environment-transport-routing')).callRuntimeEnvironment( diff --git a/src/main/ipc/runtime-environments/runtime-environment-status-owner.ts b/src/main/ipc/runtime-environments/runtime-environment-status-owner.ts index a914326b15e..e46538641c9 100644 --- a/src/main/ipc/runtime-environments/runtime-environment-status-owner.ts +++ b/src/main/ipc/runtime-environments/runtime-environment-status-owner.ts @@ -13,6 +13,7 @@ import { RuntimeHostStatusOwner } from '../../../shared/remote-runtime/host-stat import type { RuntimeStatus } from '../../../shared/runtime-types' import { RUNTIME_HOST_STATUS_CHANNEL, + type RuntimeHostStatusSnapshot, runtimeHostStatusError, runtimeHostStatusFailure, type RuntimeHostStatusResponse @@ -27,9 +28,11 @@ import { isRuntimeEnvironmentManuallyDisconnected } from './runtime-environment- import { runtimeEnvironmentChangedFailure } from './runtime-environment-revision-guard' import { resolveManagedRuntimeEnvironment } from './runtime-environment-managed-tunnel' -const statusListeners = new Set<() => void>() +const statusListeners = new Set<(snapshot: RuntimeHostStatusSnapshot) => void>() -export function onRuntimeEnvironmentStatusPublished(listener: () => void): () => void { +export function onRuntimeEnvironmentStatusPublished( + listener: (snapshot: RuntimeHostStatusSnapshot) => void +): () => void { statusListeners.add(listener) return () => statusListeners.delete(listener) } @@ -144,7 +147,7 @@ export function createRuntimeEnvironmentStatusOwner( publish: (snapshot) => { getRuntimeDesktopSurface().sendToRenderers?.(RUNTIME_HOST_STATUS_CHANNEL, snapshot) for (const listener of statusListeners) { - listener() + listener(snapshot) } } }) diff --git a/src/main/ipc/runtime-environments/runtime-environment-verified-transitions.test.ts b/src/main/ipc/runtime-environments/runtime-environment-verified-transitions.test.ts new file mode 100644 index 00000000000..24a33068951 --- /dev/null +++ b/src/main/ipc/runtime-environments/runtime-environment-verified-transitions.test.ts @@ -0,0 +1,56 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { RuntimeHostStatusSnapshot } from '../../../shared/remote-runtime/host-status/runtime-host-status' +import { trackRuntimeEnvironmentVerifiedTransitions } from './runtime-environment-verified-transitions' +import { FederationRelayScheduler } from '../../runtime/orchestration/federation/relay/federation-relay-scheduler' + +let sequence = 0 +function snapshot( + environmentId: string, + verification: RuntimeHostStatusSnapshot['verification'] +): RuntimeHostStatusSnapshot { + sequence += 1 + return { + environmentId, + pairingRevision: 1, + sequence, + checkedAt: sequence, + status: null, + verification, + transport: 'ready' + } +} + +describe('trackRuntimeEnvironmentVerifiedTransitions', () => { + afterEach(() => { + vi.useRealTimers() + }) + + it('resumes a backed-off relay at once when its server is verified again', async () => { + vi.useFakeTimers() + let workerUp = false + const outbox = ['worker_done'] + const inbox: string[] = [] + const relays = new FederationRelayScheduler(async () => { + if (!workerUp) { + return 'failed' + } + inbox.push(...outbox.splice(0)) + return 'synced' + }) + const publish = trackRuntimeEnvironmentVerifiedTransitions((environmentId) => + relays.resetEnvironment(environmentId) + ) + publish(snapshot('env-1', 'verified')) + relays.arm('dispatch-1', 'env-1') + // Four failures leave the relay in a 16 s backoff. + await vi.advanceTimersByTimeAsync(14_000) + + publish(snapshot('env-1', 'unavailable')) + workerUp = true + publish(snapshot('env-1', 'verified')) + await vi.advanceTimersByTimeAsync(0) + + expect(inbox).toEqual(['worker_done']) + relays.clear() + }) +}) diff --git a/src/main/ipc/runtime-environments/runtime-environment-verified-transitions.ts b/src/main/ipc/runtime-environments/runtime-environment-verified-transitions.ts new file mode 100644 index 00000000000..4f7cc8e5d8c --- /dev/null +++ b/src/main/ipc/runtime-environments/runtime-environment-verified-transitions.ts @@ -0,0 +1,17 @@ +import type { RuntimeHostStatusSnapshot } from '../../../shared/remote-runtime/host-status/runtime-host-status' + +/** Reports a server only when it newly turns verified; repeat publishes of a healthy server stay silent. */ +export function trackRuntimeEnvironmentVerifiedTransitions( + listener: (environmentId: string) => void +): (snapshot: RuntimeHostStatusSnapshot) => void { + const verified = new Set() + return (snapshot) => { + const { environmentId } = snapshot + if (snapshot.verification !== 'verified' || snapshot.retired) { + verified.delete(environmentId) + } else if (!verified.has(environmentId)) { + verified.add(environmentId) + listener(environmentId) + } + } +} diff --git a/src/main/runtime/orchestration/db/federation/federated-dispatch-store.ts b/src/main/runtime/orchestration/db/federation/federated-dispatch-store.ts index 871b415ce5d..4a0bf9c11f4 100644 --- a/src/main/runtime/orchestration/db/federation/federated-dispatch-store.ts +++ b/src/main/runtime/orchestration/db/federation/federated-dispatch-store.ts @@ -120,9 +120,9 @@ export function updateFederatedDispatchRuntimeEpoch( .prepare( `UPDATE federated_dispatches SET remote_runtime_epoch = ?, updated_at = datetime('now') - WHERE dispatch_id = ?` + WHERE dispatch_id = ? AND remote_runtime_epoch IS NOT ?` ) - .run(remoteRuntimeEpoch, dispatchId) + .run(remoteRuntimeEpoch, dispatchId, remoteRuntimeEpoch) } export type FederatedDispatchStoreMethods = { diff --git a/src/main/runtime/orchestration/federation/environment-transport.ts b/src/main/runtime/orchestration/federation/environment-transport.ts index 7ca00ad95fc..fa99b22a5e6 100644 --- a/src/main/runtime/orchestration/federation/environment-transport.ts +++ b/src/main/runtime/orchestration/federation/environment-transport.ts @@ -29,6 +29,8 @@ export type OrchestrationEnvironmentSendOptions = { export type OrchestrationEnvironmentTransport = { resolve(selector: string): OrchestrationWorkerServer + /** False once the server is removed from this host, so its relays stop instead of retrying forever. */ + hasEnvironment?(environmentId: string): boolean call( selector: string, method: string, @@ -38,6 +40,8 @@ export type OrchestrationEnvironmentTransport = { expectedEnvironmentPairingRevision?: number, send?: OrchestrationEnvironmentSendOptions ): Promise> + /** Fires when a server's status turns verified, so backed-off relays retry at once. */ + onEnvironmentVerified?(listener: (environmentId: string) => void): () => void } export function fingerprintOrchestrationPeer(publicKeyB64: string): string { diff --git a/src/main/runtime/orchestration/federation/relay/federation-acknowledgment-recovery.ts b/src/main/runtime/orchestration/federation/relay/federation-acknowledgment-recovery.ts new file mode 100644 index 00000000000..cbc5b5b3e03 --- /dev/null +++ b/src/main/runtime/orchestration/federation/relay/federation-acknowledgment-recovery.ts @@ -0,0 +1,136 @@ +import { federationRelayRetryDelayMs } from './federation-relay-scheduler' + +type PendingAcknowledgmentRow = { dispatchId: string; rowId: number } + +type RetryDeadline = { failures: number; dueAt: number } + +/** Walks pending rows from the cursor once around, skipping rows still cooling after a failure. */ +function nextDueAcknowledgment( + findNextAfter: (afterRowId: number) => PendingAcknowledgmentRow | undefined, + cursorRowId: number, + deadlines: Map, + now: number +): PendingAcknowledgmentRow | { waitMs: number } | null { + const pending = new Set() + let afterRowId = cursorRowId + let wrapped = afterRowId === 0 + for (;;) { + const row = findNextAfter(afterRowId) + if (!row) { + if (wrapped) { + break + } + wrapped = true + afterRowId = 0 + continue + } + if (pending.has(row.dispatchId)) { + break + } + pending.add(row.dispatchId) + if ((deadlines.get(row.dispatchId)?.dueAt ?? 0) <= now) { + return row + } + afterRowId = row.rowId + } + let earliest = Infinity + for (const [dispatchId, { dueAt }] of deadlines) { + if (pending.has(dispatchId)) { + earliest = Math.min(earliest, dueAt) + } else { + deadlines.delete(dispatchId) + } + } + return pending.size === 0 ? null : { waitMs: Math.max(0, earliest - now) } +} + +/** Settles terminal dispatches' outstanding acknowledgments one row per tick; a failing row cools off without holding back the rest. */ +export class FederationAcknowledgmentRecovery { + private timer: ReturnType | null = null + private inFlight: Promise | null = null + private cursorRowId = 0 + private generation = 0 + private readonly deadlines = new Map() + + constructor( + private readonly deps: { + findNextAfter: (afterRowId: number) => PendingAcknowledgmentRow | undefined + sync: (dispatchId: string) => Promise + acknowledgedThrough: (dispatchId: string) => number | null + } + ) {} + + ensure(): void { + if (this.timer || this.inFlight) { + return + } + const recovery = this.recoverNext(this.generation).catch((error) => { + console.warn('[orchestration] terminal federation acknowledgment recovery failed', error) + }) + this.inFlight = recovery + void recovery.finally(() => { + if (this.inFlight === recovery) { + this.inFlight = null + } + }) + } + + reset(): void { + this.generation += 1 + if (this.timer) { + clearTimeout(this.timer) + } + this.timer = null + this.inFlight = null + this.cursorRowId = 0 + this.deadlines.clear() + } + + private async recoverNext(generation: number): Promise { + const next = nextDueAcknowledgment( + this.deps.findNextAfter, + this.cursorRowId, + this.deadlines, + Date.now() + ) + if (!next) { + return + } + if ('waitMs' in next) { + this.schedule(next.waitMs) + return + } + this.cursorRowId = next.rowId + const acknowledgedBefore = this.deps.acknowledgedThrough(next.dispatchId) + const synced = await this.deps.sync(next.dispatchId).then( + () => true, + () => false + ) + if (generation !== this.generation) { + return + } + // Large backlogs settle a few pages per sync; only a sync that left the acknowledgment where it was made no progress. + const progressed = + synced && + (this.deps.findNextAfter(next.rowId - 1)?.dispatchId !== next.dispatchId || + this.deps.acknowledgedThrough(next.dispatchId) !== acknowledgedBefore) + if (progressed) { + this.deadlines.delete(next.dispatchId) + } else { + const failures = (this.deadlines.get(next.dispatchId)?.failures ?? 0) + 1 + this.deadlines.set(next.dispatchId, { + failures, + dueAt: Date.now() + federationRelayRetryDelayMs(failures) + }) + } + this.schedule(federationRelayRetryDelayMs(0)) + } + + private schedule(delayMs: number): void { + this.timer = setTimeout(() => { + this.timer = null + this.ensure() + }, delayMs) + this.timer.unref?.() + } +} diff --git a/src/main/runtime/orchestration/federation/relay/federation-relay-scheduler.test.ts b/src/main/runtime/orchestration/federation/relay/federation-relay-scheduler.test.ts new file mode 100644 index 00000000000..10b44071200 --- /dev/null +++ b/src/main/runtime/orchestration/federation/relay/federation-relay-scheduler.test.ts @@ -0,0 +1,66 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + FederationRelayScheduler, + type FederationRelayTickOutcome +} from './federation-relay-scheduler' + +/** A worker outbox relayed into the coordinator's inbox whenever the worker answers. */ +function relayedMailbox() { + const outbox: string[] = [] + const inbox: string[] = [] + let workerUp = false + let pendingTick: { finish: () => void } | null = null + let holdNextTick = false + const deliver = (): FederationRelayTickOutcome => { + if (!workerUp) { + return 'failed' + } + inbox.push(...outbox.splice(0)) + return 'synced' + } + const tick = async (): Promise => { + if (holdNextTick) { + holdNextTick = false + await new Promise((resolve) => (pendingTick = { finish: resolve })) + return 'failed' + } + return deliver() + } + return { + outbox, + inbox, + tick, + holdNextTick: () => { + holdNextTick = true + }, + finishHeldTick: () => pendingTick?.finish(), + recover: () => { + workerUp = true + } + } +} + +describe('FederationRelayScheduler', () => { + beforeEach(() => { + vi.useFakeTimers() + }) + afterEach(() => { + vi.useRealTimers() + }) + + it('delivers a worker message that lands while a failing sync is in flight', async () => { + const mailbox = relayedMailbox() + const relays = new FederationRelayScheduler(mailbox.tick) + mailbox.holdNextTick() + relays.arm('dispatch-1', 'env-1') + + mailbox.recover() + mailbox.outbox.push('worker_done') + relays.arm('dispatch-1', 'env-1') + mailbox.finishHeldTick() + await vi.advanceTimersByTimeAsync(0) + + expect(mailbox.inbox).toEqual(['worker_done']) + relays.clear() + }) +}) diff --git a/src/main/runtime/orchestration/federation/relay/federation-relay-scheduler.ts b/src/main/runtime/orchestration/federation/relay/federation-relay-scheduler.ts new file mode 100644 index 00000000000..902416986a3 --- /dev/null +++ b/src/main/runtime/orchestration/federation/relay/federation-relay-scheduler.ts @@ -0,0 +1,132 @@ +const RELAY_BASE_DELAY_MS = 1_000 +// 30 s keeps an offline host to two failed syncs a minute, and a recovered one at most 30 s behind. +const RELAY_MAX_DELAY_MS = 30_000 +/** How often a coordinator blocked in `check --wait` re-arms its run's relays, so recovery is seen within this. */ +export const FEDERATION_RELAY_WAIT_REARM_MS = 15_000 + +export type FederationRelayTickOutcome = 'synced' | 'failed' | 'ended' + +/** Consecutive failures double the wait up to the cap; a failure never ends a relay, since an unreachable host or dispatch can come back. */ +export function federationRelayRetryDelayMs(failures: number): number { + return failures === 0 + ? RELAY_BASE_DELAY_MS + : Math.min(RELAY_MAX_DELAY_MS, RELAY_BASE_DELAY_MS * 2 ** failures) +} + +type RelayChain = { + environmentId: string + failures: number + timer: ReturnType | null + dueAt: number + running: boolean + runAgain: boolean +} + +/** One self-rescheduling sync per dispatch, so a dead or unreachable dispatch backs off instead of polling every second. */ +export class FederationRelayScheduler { + private readonly chains = new Map() + private readonly dispatchesByEnvironment = new Map>() + + constructor(private readonly tick: (dispatchId: string) => Promise) {} + + /** Starts a relay, or asks an existing one to sync now without forgetting its failures. */ + arm(dispatchId: string, environmentId: string): void { + const chain = this.chains.get(dispatchId) + if (chain) { + this.syncSoon(dispatchId, chain) + return + } + const created: RelayChain = { + environmentId, + failures: 0, + timer: null, + dueAt: 0, + running: false, + runAgain: false + } + this.chains.set(dispatchId, created) + const siblings = this.dispatchesByEnvironment.get(environmentId) ?? new Set() + siblings.add(dispatchId) + this.dispatchesByEnvironment.set(environmentId, siblings) + void this.run(dispatchId, created) + } + + /** A server that just verified is worth retrying at the base cadence. */ + resetEnvironment(environmentId: string): void { + for (const dispatchId of this.dispatchesByEnvironment.get(environmentId) ?? []) { + const chain = this.chains.get(dispatchId) + if (chain) { + chain.failures = 0 + this.syncSoon(dispatchId, chain) + } + } + } + + /** A sync another caller completed proves the relay healthy again, so the next one returns to the base cadence. */ + noteSynced(dispatchId: string): void { + const chain = this.chains.get(dispatchId) + if (!chain || chain.running || chain.failures === 0) { + return + } + chain.failures = 0 + const baseDelayMs = federationRelayRetryDelayMs(0) + // Only pull the next sync earlier; an already-requested immediate sync must not be postponed. + if (!chain.timer || chain.dueAt > Date.now() + baseDelayMs) { + this.schedule(dispatchId, chain, baseDelayMs) + } + } + + clear(): void { + for (const chain of this.chains.values()) { + if (chain.timer) { + clearTimeout(chain.timer) + } + } + this.chains.clear() + this.dispatchesByEnvironment.clear() + } + + private syncSoon(dispatchId: string, chain: RelayChain): void { + if (chain.running) { + chain.runAgain = true + return + } + this.schedule(dispatchId, chain, 0) + } + + private schedule(dispatchId: string, chain: RelayChain, delayMs: number): void { + if (chain.timer) { + clearTimeout(chain.timer) + } + chain.dueAt = Date.now() + delayMs + chain.timer = setTimeout(() => void this.run(dispatchId, chain), delayMs) + chain.timer.unref?.() + } + + private async run(dispatchId: string, chain: RelayChain): Promise { + if (this.chains.get(dispatchId) !== chain) { + return + } + chain.timer = null + chain.running = true + const outcome = await this.tick(dispatchId).catch((): FederationRelayTickOutcome => 'failed') + chain.running = false + // A cleared or replaced chain must not come back from a sync that outlived it. + if (this.chains.get(dispatchId) !== chain) { + return + } + if (outcome === 'ended') { + this.chains.delete(dispatchId) + const siblings = this.dispatchesByEnvironment.get(chain.environmentId) + siblings?.delete(dispatchId) + if (siblings?.size === 0) { + this.dispatchesByEnvironment.delete(chain.environmentId) + } + return + } + chain.failures = outcome === 'synced' ? 0 : chain.failures + 1 + const delayMs = chain.runAgain ? 0 : federationRelayRetryDelayMs(chain.failures) + chain.runAgain = false + this.schedule(dispatchId, chain, delayMs) + } +} diff --git a/src/main/runtime/orchestration/federation/runtime-orchestration-federation.test.ts b/src/main/runtime/orchestration/federation/runtime-orchestration-federation.test.ts new file mode 100644 index 00000000000..c28bd83a940 --- /dev/null +++ b/src/main/runtime/orchestration/federation/runtime-orchestration-federation.test.ts @@ -0,0 +1,65 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { createIdleSyncHarness } from './synchronization/federation-sync-test-harness' +import { RuntimeOrchestrationFederation } from './runtime-orchestration-federation' +import type { OrchestrationEnvironmentTransport } from './environment-transport' + +describe('RuntimeOrchestrationFederation relay', () => { + beforeEach(() => { + vi.useFakeTimers() + }) + afterEach(() => { + vi.useRealTimers() + }) + + it('stops relaying to a removed server and delivers its pending mail after the next ensureRelay', async () => { + const { runtime, remoteCall } = createIdleSyncHarness() + const db = runtime.getOrchestrationDb() + const active = db.getFederatedDispatch('dispatch_remote') + db.listActiveFederatedDispatches = () => (active ? [active] : []) + let serverSaved = false + let noteRefused!: () => void + const refused = new Promise((resolve) => (noteRefused = resolve)) + let noteAcknowledged!: () => void + const acknowledgedOnce = new Promise((resolve) => (noteAcknowledged = resolve)) + const recordAcknowledgment = db.recordFederatedHomeAcknowledgment.bind(db) + db.recordFederatedHomeAcknowledgment = (params) => { + recordAcknowledgment(params) + noteAcknowledged() + } + const answer = remoteCall.getMockImplementation()! + remoteCall.mockImplementation(async (...args) => { + if (!serverSaved) { + noteRefused() + throw new Error('Unknown environment: environment_windows') + } + return answer(...args) + }) + const transport: OrchestrationEnvironmentTransport = { + resolve: () => { + throw new Error('unused') + }, + call: () => { + throw new Error('unused') + }, + hasEnvironment: () => serverSaved + } + const federation = new RuntimeOrchestrationFederation(runtime, transport) + const acknowledged = () => + runtime.getOrchestrationDb().getFederatedDispatch('dispatch_remote') + ?.to_home_acknowledged_sequence + + federation.ensureRelay() + await refused + await vi.advanceTimersByTimeAsync(0) + serverSaved = true + // A relay that ended leaves no timer behind; one still backing off would retry and deliver here. + await vi.runAllTimersAsync() + expect(acknowledged()).toBe(0) + + federation.ensureRelay() + + await acknowledgedOnce + expect(acknowledged()).toBeGreaterThan(0) + federation.stopRelay() + }) +}) diff --git a/src/main/runtime/orchestration/federation/runtime-orchestration-federation.ts b/src/main/runtime/orchestration/federation/runtime-orchestration-federation.ts index 3539e8ec627..b9ddf50f776 100644 --- a/src/main/runtime/orchestration/federation/runtime-orchestration-federation.ts +++ b/src/main/runtime/orchestration/federation/runtime-orchestration-federation.ts @@ -24,15 +24,28 @@ import { import { syncFederatedDispatch } from './synchronization/federation-sync' import type { OrchestrationDb } from '../db/orchestration-db' import type { FederationRuntime } from './federation-runtime' +import { + FederationRelayScheduler, + type FederationRelayTickOutcome +} from './relay/federation-relay-scheduler' +import { FederationAcknowledgmentRecovery } from './relay/federation-acknowledgment-recovery' export class RuntimeOrchestrationFederation { - private readonly timers = new Map>() + private readonly relays = new FederationRelayScheduler((dispatchId) => this.relayTick(dispatchId)) + private readonly acknowledgmentRecovery = new FederationAcknowledgmentRecovery({ + findNextAfter: (afterRowId) => + this.runtime + .getOrchestrationDb() + .findNextTerminalFederatedDispatchPendingAcknowledgment(afterRowId), + sync: (dispatchId) => this.runtime.syncOrchestrationFederatedDispatch(dispatchId), + acknowledgedThrough: (dispatchId) => + this.runtime.getOrchestrationDb().getFederatedDispatch(dispatchId) + ?.to_home_acknowledged_sequence ?? null + }) private readonly syncs = new Map }>() private readonly warnings = new Set() - private terminalRecoveryTimer: ReturnType | null = null - private terminalRecoveryInFlight: Promise | null = null - private terminalRecoveryRowId = 0 private relayGeneration = 0 + private stopVerifiedWatch: (() => void) | null = null /** Worker server -> runtime incarnation that last proved the orchestration contract. */ private readonly contractVerifiedRuntimeIds = new Map() @@ -43,16 +56,9 @@ export class RuntimeOrchestrationFederation { resetForDatabaseChange(): void { this.relayGeneration += 1 - for (const timer of this.timers.values()) { - clearInterval(timer) - } - this.timers.clear() - if (this.terminalRecoveryTimer) { - clearTimeout(this.terminalRecoveryTimer) - } - this.terminalRecoveryTimer = null - this.terminalRecoveryInFlight = null - this.terminalRecoveryRowId = 0 + this.unwatchVerifiedEnvironments() + this.relays.clear() + this.acknowledgmentRecovery.reset() clearFederationAckCheckpoints(this.runtime) this.syncs.clear() this.warnings.clear() @@ -172,10 +178,14 @@ export class RuntimeOrchestrationFederation { if (current?.db === db) { return current.promise } - const sync = syncFederatedDispatch(this.runtime, dispatchId) + const generation = this.relayGeneration + const isCurrent = (): boolean => + generation === this.relayGeneration && this.runtime.getOrchestrationDb() === db + const sync = syncFederatedDispatch(this.runtime, dispatchId, isCurrent) .then(() => { if (this.syncs.get(dispatchId)?.promise === sync) { this.warnings.delete(dispatchId) + this.relays.noteSynced(dispatchId) } }) .catch((error: unknown) => { @@ -200,10 +210,14 @@ export class RuntimeOrchestrationFederation { async syncDispatchAfterCurrent(dispatchId: string): Promise { const db = this.runtime.getOrchestrationDb() + const generation = this.relayGeneration const current = this.syncs.get(dispatchId) if (current?.db === db) { await current.promise.catch(() => undefined) } + if (generation !== this.relayGeneration || this.runtime.getOrchestrationDb() !== db) { + return + } await this.syncDispatch(dispatchId) } @@ -211,87 +225,61 @@ export class RuntimeOrchestrationFederation { if (!this.transport) { return } + this.stopVerifiedWatch ??= + this.transport.onEnvironmentVerified?.((environmentId) => + this.relays.resetEnvironment(environmentId) + ) ?? null for (const dispatch of this.runtime.getOrchestrationDb().listActiveFederatedDispatches(runId)) { - if (this.timers.has(dispatch.dispatch_id)) { - continue - } - const tick = () => { - const db = this.runtime.getOrchestrationDb() - if (!db.isFederatedDispatchRelayEligible(dispatch.dispatch_id)) { - const activeTimer = this.timers.get(dispatch.dispatch_id) - if (activeTimer) { - clearInterval(activeTimer) - } - this.timers.delete(dispatch.dispatch_id) - this.warnings.delete(dispatch.dispatch_id) - return + this.relays.arm(dispatch.dispatch_id, dispatch.environment_id) + } + this.acknowledgmentRecovery.ensure() + } + + private async relayTick(dispatchId: string): Promise { + if (!this.runtime.getOrchestrationDb().isFederatedDispatchRelayEligible(dispatchId)) { + this.warnings.delete(dispatchId) + return 'ended' + } + return this.syncDispatch(dispatchId).then( + (): FederationRelayTickOutcome => 'synced', + (): FederationRelayTickOutcome => { + if (!this.isEnvironmentRemoved(dispatchId)) { + return 'failed' } - void this.syncDispatch(dispatch.dispatch_id).catch(() => undefined) + this.warnings.delete(dispatchId) + return 'ended' } - const timer = setInterval(tick, 1_000) - timer.unref?.() - this.timers.set(dispatch.dispatch_id, timer) - tick() - } - this.ensureTerminalHistoryRecovery() - } - - private ensureTerminalHistoryRecovery(): void { - if (this.terminalRecoveryTimer || this.terminalRecoveryInFlight) { - return - } - const generation = this.relayGeneration - const recovery = this.recoverNextTerminalHistoryAcknowledgment(generation).catch((error) => { - console.warn('[orchestration] terminal federation acknowledgment recovery failed', error) - }) - this.terminalRecoveryInFlight = recovery - void recovery.finally(() => { - if (this.terminalRecoveryInFlight === recovery) { - this.terminalRecoveryInFlight = null - } - }) - } - - private async recoverNextTerminalHistoryAcknowledgment(generation: number): Promise { - const db = this.runtime.getOrchestrationDb() - let historical = db.findNextTerminalFederatedDispatchPendingAcknowledgment( - this.terminalRecoveryRowId ) - if (!historical && this.terminalRecoveryRowId > 0) { - this.terminalRecoveryRowId = 0 - historical = db.findNextTerminalFederatedDispatchPendingAcknowledgment(0) + } + + /** Checked only after a failed sync; a store read error keeps the relay retrying. */ + private isEnvironmentRemoved(dispatchId: string): boolean { + const environmentId = this.runtime + .getOrchestrationDb() + .getFederatedDispatch(dispatchId)?.environment_id + if (environmentId === undefined) { + return false } - if (!historical) { - return + try { + // A removed server cannot answer; the next ensureRelay re-arms it if the server is added back. + return this.transport?.hasEnvironment?.(environmentId) === false + } catch { + return false } - this.terminalRecoveryRowId = historical.rowId - await this.runtime - .syncOrchestrationFederatedDispatch(historical.dispatchId) - .catch(() => undefined) - if (generation !== this.relayGeneration) { - return - } - this.terminalRecoveryTimer = setTimeout(() => { - this.terminalRecoveryTimer = null - this.ensureTerminalHistoryRecovery() - }, 1_000) - this.terminalRecoveryTimer.unref?.() + } + + private unwatchVerifiedEnvironments(): void { + this.stopVerifiedWatch?.() + this.stopVerifiedWatch = null } stopRelay(): void { this.relayGeneration += 1 - for (const timer of this.timers.values()) { - clearInterval(timer) - } - this.timers.clear() + this.unwatchVerifiedEnvironments() + this.relays.clear() + this.acknowledgmentRecovery.reset() this.warnings.clear() this.syncs.clear() - if (this.terminalRecoveryTimer) { - clearTimeout(this.terminalRecoveryTimer) - } - this.terminalRecoveryTimer = null - this.terminalRecoveryInFlight = null - this.terminalRecoveryRowId = 0 clearFederationAckCheckpoints(this.runtime) } } diff --git a/src/main/runtime/orchestration/federation/synchronization/federation-sync-capability.test.ts b/src/main/runtime/orchestration/federation/synchronization/federation-sync-capability.test.ts new file mode 100644 index 00000000000..98e43ff575a --- /dev/null +++ b/src/main/runtime/orchestration/federation/synchronization/federation-sync-capability.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from 'vitest' +import { ORCHESTRATION_FEDERATION_RELEASE_ARCHIVE_RUNTIME_CAPABILITY } from '../../../../../shared/orchestration-runtime-capabilities' +import { createIdleSyncHarness } from './federation-sync-test-harness' +import { armHomeAutoReleaseForOlderHost } from './federation-sync-capability' + +function releasableTerminalWorker() { + const harness = createIdleSyncHarness() + harness.setRemoteCapabilities([ORCHESTRATION_FEDERATION_RELEASE_ARCHIVE_RUNTIME_CAPABILITY]) + const db = harness.runtime.getOrchestrationDb() + const federated = db.getFederatedDispatch('dispatch_remote')! + db.getFederatedDispatch = () => ({ + ...federated, + remote_terminal_handle: 'term_remote_worker', + home_auto_release_state: null + }) + const worker = db.getWorkerDispatch('dispatch_remote')! + db.getWorkerDispatch = () => ({ ...worker, state: 'succeeded', stage: 'reported' }) + db.workerAutoReleaseEnabled = () => true + return { + runtime: harness.runtime, + remoteCall: harness.remoteCall, + dispatchId: federated.dispatch_id, + armedDeadlines: () => db.claimDueFederatedHomeAutoReleases(Date.UTC(2100, 0, 1)) + } +} + +describe('armHomeAutoReleaseForOlderHost', () => { + it('does not arm from a sync the relay stopped while it probed the host', async () => { + const { runtime, remoteCall, dispatchId, armedDeadlines } = releasableTerminalWorker() + const answer = remoteCall.getMockImplementation()! + let noteProbe!: () => void + let releaseProbe!: () => void + const probing = new Promise((resolve) => (noteProbe = resolve)) + const released = new Promise((resolve) => (releaseProbe = resolve)) + remoteCall.mockImplementation(async (...args) => { + if (args[1] === 'status.get') { + noteProbe() + await released + } + return answer(...args) + }) + let current = true + + const arming = armHomeAutoReleaseForOlderHost(runtime, dispatchId, 1, () => current) + await probing + current = false + releaseProbe() + await arming + + expect(armedDeadlines()).toEqual([]) + }) +}) diff --git a/src/main/runtime/orchestration/federation/synchronization/federation-sync-capability.ts b/src/main/runtime/orchestration/federation/synchronization/federation-sync-capability.ts index 1135a52567b..20e607c8f8b 100644 --- a/src/main/runtime/orchestration/federation/synchronization/federation-sync-capability.ts +++ b/src/main/runtime/orchestration/federation/synchronization/federation-sync-capability.ts @@ -55,7 +55,8 @@ function resolveFederatedHostCapability( export async function armHomeAutoReleaseForOlderHost( runtime: FederationRuntime, dispatchId: string, - pairingRevision: number | undefined + pairingRevision: number | undefined, + isCurrent: () => boolean = () => true ): Promise { const db = runtime.getOrchestrationDb() const federated = db.getFederatedDispatch(dispatchId) @@ -84,7 +85,7 @@ export async function armHomeAutoReleaseForOlderHost( ) ) ) - if (!hostReleases?.supported && releasable?.supported !== false) { + if (isCurrent() && !hostReleases?.supported && releasable?.supported !== false) { db.armFederatedHomeAutoRelease(dispatchId) } } diff --git a/src/main/runtime/orchestration/federation/synchronization/federation-sync-message.ts b/src/main/runtime/orchestration/federation/synchronization/federation-sync-message.ts index 51559111c05..a23a0686e70 100644 --- a/src/main/runtime/orchestration/federation/synchronization/federation-sync-message.ts +++ b/src/main/runtime/orchestration/federation/synchronization/federation-sync-message.ts @@ -1,3 +1,4 @@ +import { z } from 'zod' import { MESSAGE_TYPES, type MessagePriority, @@ -102,3 +103,22 @@ export function parseFederatedLifecycle( }) } } + +// Decode untrusted peer pages before import to report malformed input as an orchestration error. +export const PulledRelayPage = z + .object({ + runtimeEpoch: z.string().min(1), + items: z.array( + z + .object({ + dispatch_id: z.string(), + direction: z.literal('to_home'), + sequence: z.number(), + message_id: z.string(), + kind: z.string(), + payload: z.string() + }) + .passthrough() + ) + }) + .passthrough() diff --git a/src/main/runtime/orchestration/federation/synchronization/federation-sync.test.ts b/src/main/runtime/orchestration/federation/synchronization/federation-sync.test.ts index 3f9c0432fb2..58fefce174f 100644 --- a/src/main/runtime/orchestration/federation/synchronization/federation-sync.test.ts +++ b/src/main/runtime/orchestration/federation/synchronization/federation-sync.test.ts @@ -538,6 +538,29 @@ describe('federation relay acknowledgments', () => { remoteCall.mock.calls.filter(([, method]) => method === 'orchestration.federationAck') ).toHaveLength(2) }) + + it.each(['stopRelay', 'database swap'] as const)( + 'keeps a sync interrupted by %s from acknowledging into current state', + async (interruption) => { + const { runtime, blockAck, replaceDb } = createIdleSyncHarness() + const gate = blockAck() + + const oldSync = runtime.syncOrchestrationFederatedDispatch('dispatch_remote') + await gate.reached + if (interruption === 'stopRelay') { + runtime.stopOrchestrationFederationRelay() + } else { + replaceDb() + } + gate.release() + await oldSync + + expect( + runtime.getOrchestrationDb().getFederatedDispatch('dispatch_remote') + ?.to_home_acknowledged_sequence + ).toBe(0) + } + ) }) describe('bounded outgoing answer relay', () => { diff --git a/src/main/runtime/orchestration/federation/synchronization/federation-sync.ts b/src/main/runtime/orchestration/federation/synchronization/federation-sync.ts index 7db94881699..fed13a9a5b7 100644 --- a/src/main/runtime/orchestration/federation/synchronization/federation-sync.ts +++ b/src/main/runtime/orchestration/federation/synchronization/federation-sync.ts @@ -1,6 +1,5 @@ import { federationRelayPrefix } from '../relay/federation-relay-prefix' import { federationRelayWireItem } from '../relay/federation-relay-wire' -import { z } from 'zod' import type { FederationRuntime } from '../federation-runtime' import type { FederatedLifecycleSettlement } from '../federation-lifecycle-settlement' import '../../../../../shared/protocol-version' @@ -17,30 +16,15 @@ import { resolveFederatedLifecycleSettlementCapability } from './federation-sync-capability' import { getOrchestrationPeerCapabilityCache } from '../orchestration-peer-capability-cache' -import { parseFederatedLifecycle, parseRelayedMessage } from './federation-sync-message' +import { + parseFederatedLifecycle, + parseRelayedMessage, + PulledRelayPage +} from './federation-sync-message' const FEDERATION_PULL_PAGE_SIZE = 50 const MAX_FEDERATION_PULL_PAGES_PER_SYNC = 6 -// Decode untrusted peer pages before import to report malformed input as an orchestration error. -const PulledRelayPage = z - .object({ - runtimeEpoch: z.string().min(1), - items: z.array( - z - .object({ - dispatch_id: z.string(), - direction: z.literal('to_home'), - sequence: z.number(), - message_id: z.string(), - kind: z.string(), - payload: z.string() - }) - .passthrough() - ) - }) - .passthrough() - export async function syncFederatedDispatch( runtime: FederationRuntime, dispatchId: string, @@ -85,6 +69,9 @@ async function syncFederatedDispatchPages( federated, currentServer.pairingRevision ) + if (!isCurrent()) { + return { imported: 0, acknowledgedThrough: federated.to_home_imported_sequence } + } const supportsLifecycleSettlement = federated.protocol_version >= ORCHESTRATION_FEDERATION_LIFECYCLE_SETTLEMENT_PROTOCOL_VERSION && capability?.supported === true @@ -221,26 +208,31 @@ async function syncFederatedDispatchPages( const locallyAcknowledgedThrough = keepRelayEligible ? Math.max(0, delivered.acknowledgedThrough - 1) : delivered.acknowledgedThrough - db.recordFederatedHomeAcknowledgment({ - dispatchId, - remoteRuntimeEpoch: pulled.runtimeEpoch, - sequence: locallyAcknowledgedThrough - }) - recordFederationAckCheckpoint(runtime, ackLease, { - ...ackIdentity, - throughSequence: locallyAcknowledgedThrough - }) + if (isCurrent()) { + db.recordFederatedHomeAcknowledgment({ + dispatchId, + remoteRuntimeEpoch: pulled.runtimeEpoch, + sequence: locallyAcknowledgedThrough + }) + recordFederationAckCheckpoint(runtime, ackLease, { + ...ackIdentity, + throughSequence: locallyAcknowledgedThrough + }) + } } - getOrchestrationPeerCapabilityCache(runtime).observeEpoch( - federated.peer_fingerprint, - pulled.runtimeEpoch - ) - db.updateFederatedDispatchRuntimeEpoch(dispatchId, pulled.runtimeEpoch) if (isCurrent()) { - // Best effort: without a deadline the worker keeps manual release, as before. - await armHomeAutoReleaseForOlderHost(runtime, dispatchId, currentServer.pairingRevision).catch( - () => undefined + getOrchestrationPeerCapabilityCache(runtime).observeEpoch( + federated.peer_fingerprint, + pulled.runtimeEpoch ) + db.updateFederatedDispatchRuntimeEpoch(dispatchId, pulled.runtimeEpoch) + // Best effort: without a deadline the worker keeps manual release, as before. + await armHomeAutoReleaseForOlderHost( + runtime, + dispatchId, + currentServer.pairingRevision, + isCurrent + ).catch(() => undefined) } const toWorker = db.getWorkerDispatch(dispatchId)?.state === 'ready' diff --git a/src/main/runtime/rpc/methods/orchestration/messaging/check-run.ts b/src/main/runtime/rpc/methods/orchestration/messaging/check-run.ts index af18327372e..49ef6a58dbe 100644 --- a/src/main/runtime/rpc/methods/orchestration/messaging/check-run.ts +++ b/src/main/runtime/rpc/methods/orchestration/messaging/check-run.ts @@ -13,6 +13,7 @@ import type { CheckParams } from '../../../../../../shared/rpc-contract/orchestr import type { z } from 'zod' import type { OrchestrationSessionCaller } from '../../../../orchestration/addressing/orchestration-caller-identity' import { checkRunPendingMail } from './check-run-pending-mail' +import { FEDERATION_RELAY_WAIT_REARM_MS } from '../../../../orchestration/federation/relay/federation-relay-scheduler' type CheckParamsInput = z.infer @@ -164,11 +165,21 @@ export async function checkRunMailbox(args: { } } - const waitResult = await runtime.waitForMessage(address, { - typeFilter: typeFilter as string[] | undefined, - timeoutMs: params.timeoutMs ?? undefined, - signal - }) + const rearmRelays = setInterval(() => { + try { + runtime.ensureOrchestrationFederationRelay(run.id) + } catch (error) { + console.warn('[orchestration] federation relay re-arm failed during check --wait', error) + } + }, FEDERATION_RELAY_WAIT_REARM_MS) + rearmRelays.unref?.() + const waitResult = await runtime + .waitForMessage(address, { + typeFilter: typeFilter as string[] | undefined, + timeoutMs: params.timeoutMs ?? undefined, + signal + }) + .finally(() => clearInterval(rearmRelays)) try { revalidateConsumer() } catch (error) { diff --git a/src/main/startup/main-process-runtime-service.ts b/src/main/startup/main-process-runtime-service.ts index 0fdc6b502a9..00a6af5f993 100644 --- a/src/main/startup/main-process-runtime-service.ts +++ b/src/main/startup/main-process-runtime-service.ts @@ -25,6 +25,8 @@ import type { TerminalSideEffectBatch } from '../../shared/terminal/output/termi import { resolveTuiAgentLaunchEnv } from '../../shared/agent-launch/choices/tui-agent-launch-defaults' import { callRuntimeEnvironment } from '../ipc/runtime-environments/runtime-environment-transport-routing' import { createOrchestrationEnvironmentTransport } from '../ipc/runtime-environments/orchestration-environment-transport' +import { trackRuntimeEnvironmentVerifiedTransitions } from '../ipc/runtime-environments/runtime-environment-verified-transitions' +import { onRuntimeEnvironmentStatusPublished } from '../ipc/runtime-environments/runtime-environment-status-owner' import { mainProcessState as state } from './main-process-state' import { suspendEphemeralVmWorkspace } from '../ephemeral-vm/workspace-suspend' import { @@ -114,7 +116,11 @@ export function initializeMainProcessRuntime(): OrcaRuntimeService { homePath, resolveTuiAgentLaunchEnv('codex', store.getSettings()) ), - orchestrationEnvironmentTransport: createOrchestrationEnvironmentTransport(), + orchestrationEnvironmentTransport: { + ...createOrchestrationEnvironmentTransport(), + onEnvironmentVerified: (listener) => + onRuntimeEnvironmentStatusPublished(trackRuntimeEnvironmentVerifiedTransitions(listener)) + }, workspaceSleepPlatform: { callEnvironment: (environmentId, method, params, timeoutMs) => callRuntimeEnvironment(