mirror of
https://github.com/stablyai/orca.git
synced 2026-11-18 20:06:10 +00:00
perf(orchestration): back off failing federation relays instead of polling every second (#39734)
Each federated dispatch relay now reschedules itself and doubles its wait
after consecutive failures (capped at 30 s) instead of syncing every second
forever. A server turning verified, or a sync completed by another caller,
returns the relay to the base cadence. A coordinator blocked in
`check --wait` re-arms its run's relays every 15 s, so a recovered worker is
seen within that window. Relays for a server removed from this host stop
instead of retrying; the next relay arm brings them back if it is re-added.
Stale in-flight syncs no longer record acknowledgments, epochs or home
auto-release after the relay stopped.
Replaces #39283.
Co-authored-by: m4air <m4air@m4airs-MacBook-Air.local>
Co-authored-by: brennanb2025 <79079362+brennanb2025@users.noreply.github.com>
(cherry picked from commit 0f6432319d)
This commit is contained in:
@@ -1,6 +1,9 @@
|
||||
import { getAppEnvironment } from '../../../shared/app-environment'
|
||||
import { getPreferredPairedRuntimeConnection } from '../../../shared/remote-runtime/host-catalog/runtime-environments'
|
||||
import { resolveEnvironment } from '../../../shared/remote-runtime/host-store/runtime-environment-store'
|
||||
import {
|
||||
listEnvironments,
|
||||
resolveEnvironment
|
||||
} from '../../../shared/remote-runtime/host-store/runtime-environment-store'
|
||||
import {
|
||||
pairedOrchestrationPeers,
|
||||
type OrchestrationEnvironmentTransport
|
||||
@@ -21,6 +24,8 @@ export function createOrchestrationEnvironmentTransport(): OrchestrationEnvironm
|
||||
pairingRevision: environment.pairingRevision ?? environment.createdAt
|
||||
}
|
||||
},
|
||||
hasEnvironment: (environmentId) =>
|
||||
listEnvironments(registryRoot()).some((environment) => environment.id === environmentId),
|
||||
// Loaded on first dial: the routing graph is large and most hosts never place remote workers.
|
||||
call: async (selector, method, params, timeoutMs, envelope, expectedPairingRevision, send) =>
|
||||
(await import('./runtime-environment-transport-routing')).callRuntimeEnvironment(
|
||||
|
||||
@@ -13,6 +13,7 @@ import { RuntimeHostStatusOwner } from '../../../shared/remote-runtime/host-stat
|
||||
import type { RuntimeStatus } from '../../../shared/runtime-types'
|
||||
import {
|
||||
RUNTIME_HOST_STATUS_CHANNEL,
|
||||
type RuntimeHostStatusSnapshot,
|
||||
runtimeHostStatusError,
|
||||
runtimeHostStatusFailure,
|
||||
type RuntimeHostStatusResponse
|
||||
@@ -27,9 +28,11 @@ import { isRuntimeEnvironmentManuallyDisconnected } from './runtime-environment-
|
||||
import { runtimeEnvironmentChangedFailure } from './runtime-environment-revision-guard'
|
||||
import { resolveManagedRuntimeEnvironment } from './runtime-environment-managed-tunnel'
|
||||
|
||||
const statusListeners = new Set<() => void>()
|
||||
const statusListeners = new Set<(snapshot: RuntimeHostStatusSnapshot) => void>()
|
||||
|
||||
export function onRuntimeEnvironmentStatusPublished(listener: () => void): () => void {
|
||||
export function onRuntimeEnvironmentStatusPublished(
|
||||
listener: (snapshot: RuntimeHostStatusSnapshot) => void
|
||||
): () => void {
|
||||
statusListeners.add(listener)
|
||||
return () => statusListeners.delete(listener)
|
||||
}
|
||||
@@ -144,7 +147,7 @@ export function createRuntimeEnvironmentStatusOwner(
|
||||
publish: (snapshot) => {
|
||||
getRuntimeDesktopSurface().sendToRenderers?.(RUNTIME_HOST_STATUS_CHANNEL, snapshot)
|
||||
for (const listener of statusListeners) {
|
||||
listener()
|
||||
listener(snapshot)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { RuntimeHostStatusSnapshot } from '../../../shared/remote-runtime/host-status/runtime-host-status'
|
||||
import { trackRuntimeEnvironmentVerifiedTransitions } from './runtime-environment-verified-transitions'
|
||||
import { FederationRelayScheduler } from '../../runtime/orchestration/federation/relay/federation-relay-scheduler'
|
||||
|
||||
let sequence = 0
|
||||
function snapshot(
|
||||
environmentId: string,
|
||||
verification: RuntimeHostStatusSnapshot['verification']
|
||||
): RuntimeHostStatusSnapshot {
|
||||
sequence += 1
|
||||
return {
|
||||
environmentId,
|
||||
pairingRevision: 1,
|
||||
sequence,
|
||||
checkedAt: sequence,
|
||||
status: null,
|
||||
verification,
|
||||
transport: 'ready'
|
||||
}
|
||||
}
|
||||
|
||||
describe('trackRuntimeEnvironmentVerifiedTransitions', () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('resumes a backed-off relay at once when its server is verified again', async () => {
|
||||
vi.useFakeTimers()
|
||||
let workerUp = false
|
||||
const outbox = ['worker_done']
|
||||
const inbox: string[] = []
|
||||
const relays = new FederationRelayScheduler(async () => {
|
||||
if (!workerUp) {
|
||||
return 'failed'
|
||||
}
|
||||
inbox.push(...outbox.splice(0))
|
||||
return 'synced'
|
||||
})
|
||||
const publish = trackRuntimeEnvironmentVerifiedTransitions((environmentId) =>
|
||||
relays.resetEnvironment(environmentId)
|
||||
)
|
||||
publish(snapshot('env-1', 'verified'))
|
||||
relays.arm('dispatch-1', 'env-1')
|
||||
// Four failures leave the relay in a 16 s backoff.
|
||||
await vi.advanceTimersByTimeAsync(14_000)
|
||||
|
||||
publish(snapshot('env-1', 'unavailable'))
|
||||
workerUp = true
|
||||
publish(snapshot('env-1', 'verified'))
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
|
||||
expect(inbox).toEqual(['worker_done'])
|
||||
relays.clear()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,17 @@
|
||||
import type { RuntimeHostStatusSnapshot } from '../../../shared/remote-runtime/host-status/runtime-host-status'
|
||||
|
||||
/** Reports a server only when it newly turns verified; repeat publishes of a healthy server stay silent. */
|
||||
export function trackRuntimeEnvironmentVerifiedTransitions(
|
||||
listener: (environmentId: string) => void
|
||||
): (snapshot: RuntimeHostStatusSnapshot) => void {
|
||||
const verified = new Set<string>()
|
||||
return (snapshot) => {
|
||||
const { environmentId } = snapshot
|
||||
if (snapshot.verification !== 'verified' || snapshot.retired) {
|
||||
verified.delete(environmentId)
|
||||
} else if (!verified.has(environmentId)) {
|
||||
verified.add(environmentId)
|
||||
listener(environmentId)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -120,9 +120,9 @@ export function updateFederatedDispatchRuntimeEpoch(
|
||||
.prepare(
|
||||
`UPDATE federated_dispatches
|
||||
SET remote_runtime_epoch = ?, updated_at = datetime('now')
|
||||
WHERE dispatch_id = ?`
|
||||
WHERE dispatch_id = ? AND remote_runtime_epoch IS NOT ?`
|
||||
)
|
||||
.run(remoteRuntimeEpoch, dispatchId)
|
||||
.run(remoteRuntimeEpoch, dispatchId, remoteRuntimeEpoch)
|
||||
}
|
||||
|
||||
export type FederatedDispatchStoreMethods = {
|
||||
|
||||
@@ -29,6 +29,8 @@ export type OrchestrationEnvironmentSendOptions = {
|
||||
|
||||
export type OrchestrationEnvironmentTransport = {
|
||||
resolve(selector: string): OrchestrationWorkerServer
|
||||
/** False once the server is removed from this host, so its relays stop instead of retrying forever. */
|
||||
hasEnvironment?(environmentId: string): boolean
|
||||
call(
|
||||
selector: string,
|
||||
method: string,
|
||||
@@ -38,6 +40,8 @@ export type OrchestrationEnvironmentTransport = {
|
||||
expectedEnvironmentPairingRevision?: number,
|
||||
send?: OrchestrationEnvironmentSendOptions
|
||||
): Promise<RuntimeRpcResponse<unknown>>
|
||||
/** Fires when a server's status turns verified, so backed-off relays retry at once. */
|
||||
onEnvironmentVerified?(listener: (environmentId: string) => void): () => void
|
||||
}
|
||||
|
||||
export function fingerprintOrchestrationPeer(publicKeyB64: string): string {
|
||||
|
||||
+136
@@ -0,0 +1,136 @@
|
||||
import { federationRelayRetryDelayMs } from './federation-relay-scheduler'
|
||||
|
||||
type PendingAcknowledgmentRow = { dispatchId: string; rowId: number }
|
||||
|
||||
type RetryDeadline = { failures: number; dueAt: number }
|
||||
|
||||
/** Walks pending rows from the cursor once around, skipping rows still cooling after a failure. */
|
||||
function nextDueAcknowledgment(
|
||||
findNextAfter: (afterRowId: number) => PendingAcknowledgmentRow | undefined,
|
||||
cursorRowId: number,
|
||||
deadlines: Map<string, RetryDeadline>,
|
||||
now: number
|
||||
): PendingAcknowledgmentRow | { waitMs: number } | null {
|
||||
const pending = new Set<string>()
|
||||
let afterRowId = cursorRowId
|
||||
let wrapped = afterRowId === 0
|
||||
for (;;) {
|
||||
const row = findNextAfter(afterRowId)
|
||||
if (!row) {
|
||||
if (wrapped) {
|
||||
break
|
||||
}
|
||||
wrapped = true
|
||||
afterRowId = 0
|
||||
continue
|
||||
}
|
||||
if (pending.has(row.dispatchId)) {
|
||||
break
|
||||
}
|
||||
pending.add(row.dispatchId)
|
||||
if ((deadlines.get(row.dispatchId)?.dueAt ?? 0) <= now) {
|
||||
return row
|
||||
}
|
||||
afterRowId = row.rowId
|
||||
}
|
||||
let earliest = Infinity
|
||||
for (const [dispatchId, { dueAt }] of deadlines) {
|
||||
if (pending.has(dispatchId)) {
|
||||
earliest = Math.min(earliest, dueAt)
|
||||
} else {
|
||||
deadlines.delete(dispatchId)
|
||||
}
|
||||
}
|
||||
return pending.size === 0 ? null : { waitMs: Math.max(0, earliest - now) }
|
||||
}
|
||||
|
||||
/** Settles terminal dispatches' outstanding acknowledgments one row per tick; a failing row cools off without holding back the rest. */
|
||||
export class FederationAcknowledgmentRecovery {
|
||||
private timer: ReturnType<typeof setTimeout> | null = null
|
||||
private inFlight: Promise<void> | null = null
|
||||
private cursorRowId = 0
|
||||
private generation = 0
|
||||
private readonly deadlines = new Map<string, RetryDeadline>()
|
||||
|
||||
constructor(
|
||||
private readonly deps: {
|
||||
findNextAfter: (afterRowId: number) => PendingAcknowledgmentRow | undefined
|
||||
sync: (dispatchId: string) => Promise<unknown>
|
||||
acknowledgedThrough: (dispatchId: string) => number | null
|
||||
}
|
||||
) {}
|
||||
|
||||
ensure(): void {
|
||||
if (this.timer || this.inFlight) {
|
||||
return
|
||||
}
|
||||
const recovery = this.recoverNext(this.generation).catch((error) => {
|
||||
console.warn('[orchestration] terminal federation acknowledgment recovery failed', error)
|
||||
})
|
||||
this.inFlight = recovery
|
||||
void recovery.finally(() => {
|
||||
if (this.inFlight === recovery) {
|
||||
this.inFlight = null
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
reset(): void {
|
||||
this.generation += 1
|
||||
if (this.timer) {
|
||||
clearTimeout(this.timer)
|
||||
}
|
||||
this.timer = null
|
||||
this.inFlight = null
|
||||
this.cursorRowId = 0
|
||||
this.deadlines.clear()
|
||||
}
|
||||
|
||||
private async recoverNext(generation: number): Promise<void> {
|
||||
const next = nextDueAcknowledgment(
|
||||
this.deps.findNextAfter,
|
||||
this.cursorRowId,
|
||||
this.deadlines,
|
||||
Date.now()
|
||||
)
|
||||
if (!next) {
|
||||
return
|
||||
}
|
||||
if ('waitMs' in next) {
|
||||
this.schedule(next.waitMs)
|
||||
return
|
||||
}
|
||||
this.cursorRowId = next.rowId
|
||||
const acknowledgedBefore = this.deps.acknowledgedThrough(next.dispatchId)
|
||||
const synced = await this.deps.sync(next.dispatchId).then(
|
||||
() => true,
|
||||
() => false
|
||||
)
|
||||
if (generation !== this.generation) {
|
||||
return
|
||||
}
|
||||
// Large backlogs settle a few pages per sync; only a sync that left the acknowledgment where it was made no progress.
|
||||
const progressed =
|
||||
synced &&
|
||||
(this.deps.findNextAfter(next.rowId - 1)?.dispatchId !== next.dispatchId ||
|
||||
this.deps.acknowledgedThrough(next.dispatchId) !== acknowledgedBefore)
|
||||
if (progressed) {
|
||||
this.deadlines.delete(next.dispatchId)
|
||||
} else {
|
||||
const failures = (this.deadlines.get(next.dispatchId)?.failures ?? 0) + 1
|
||||
this.deadlines.set(next.dispatchId, {
|
||||
failures,
|
||||
dueAt: Date.now() + federationRelayRetryDelayMs(failures)
|
||||
})
|
||||
}
|
||||
this.schedule(federationRelayRetryDelayMs(0))
|
||||
}
|
||||
|
||||
private schedule(delayMs: number): void {
|
||||
this.timer = setTimeout(() => {
|
||||
this.timer = null
|
||||
this.ensure()
|
||||
}, delayMs)
|
||||
this.timer.unref?.()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import {
|
||||
FederationRelayScheduler,
|
||||
type FederationRelayTickOutcome
|
||||
} from './federation-relay-scheduler'
|
||||
|
||||
/** A worker outbox relayed into the coordinator's inbox whenever the worker answers. */
|
||||
function relayedMailbox() {
|
||||
const outbox: string[] = []
|
||||
const inbox: string[] = []
|
||||
let workerUp = false
|
||||
let pendingTick: { finish: () => void } | null = null
|
||||
let holdNextTick = false
|
||||
const deliver = (): FederationRelayTickOutcome => {
|
||||
if (!workerUp) {
|
||||
return 'failed'
|
||||
}
|
||||
inbox.push(...outbox.splice(0))
|
||||
return 'synced'
|
||||
}
|
||||
const tick = async (): Promise<FederationRelayTickOutcome> => {
|
||||
if (holdNextTick) {
|
||||
holdNextTick = false
|
||||
await new Promise<void>((resolve) => (pendingTick = { finish: resolve }))
|
||||
return 'failed'
|
||||
}
|
||||
return deliver()
|
||||
}
|
||||
return {
|
||||
outbox,
|
||||
inbox,
|
||||
tick,
|
||||
holdNextTick: () => {
|
||||
holdNextTick = true
|
||||
},
|
||||
finishHeldTick: () => pendingTick?.finish(),
|
||||
recover: () => {
|
||||
workerUp = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
describe('FederationRelayScheduler', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
})
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('delivers a worker message that lands while a failing sync is in flight', async () => {
|
||||
const mailbox = relayedMailbox()
|
||||
const relays = new FederationRelayScheduler(mailbox.tick)
|
||||
mailbox.holdNextTick()
|
||||
relays.arm('dispatch-1', 'env-1')
|
||||
|
||||
mailbox.recover()
|
||||
mailbox.outbox.push('worker_done')
|
||||
relays.arm('dispatch-1', 'env-1')
|
||||
mailbox.finishHeldTick()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
|
||||
expect(mailbox.inbox).toEqual(['worker_done'])
|
||||
relays.clear()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,132 @@
|
||||
const RELAY_BASE_DELAY_MS = 1_000
|
||||
// 30 s keeps an offline host to two failed syncs a minute, and a recovered one at most 30 s behind.
|
||||
const RELAY_MAX_DELAY_MS = 30_000
|
||||
/** How often a coordinator blocked in `check --wait` re-arms its run's relays, so recovery is seen within this. */
|
||||
export const FEDERATION_RELAY_WAIT_REARM_MS = 15_000
|
||||
|
||||
export type FederationRelayTickOutcome = 'synced' | 'failed' | 'ended'
|
||||
|
||||
/** Consecutive failures double the wait up to the cap; a failure never ends a relay, since an unreachable host or dispatch can come back. */
|
||||
export function federationRelayRetryDelayMs(failures: number): number {
|
||||
return failures === 0
|
||||
? RELAY_BASE_DELAY_MS
|
||||
: Math.min(RELAY_MAX_DELAY_MS, RELAY_BASE_DELAY_MS * 2 ** failures)
|
||||
}
|
||||
|
||||
type RelayChain = {
|
||||
environmentId: string
|
||||
failures: number
|
||||
timer: ReturnType<typeof setTimeout> | null
|
||||
dueAt: number
|
||||
running: boolean
|
||||
runAgain: boolean
|
||||
}
|
||||
|
||||
/** One self-rescheduling sync per dispatch, so a dead or unreachable dispatch backs off instead of polling every second. */
|
||||
export class FederationRelayScheduler {
|
||||
private readonly chains = new Map<string, RelayChain>()
|
||||
private readonly dispatchesByEnvironment = new Map<string, Set<string>>()
|
||||
|
||||
constructor(private readonly tick: (dispatchId: string) => Promise<FederationRelayTickOutcome>) {}
|
||||
|
||||
/** Starts a relay, or asks an existing one to sync now without forgetting its failures. */
|
||||
arm(dispatchId: string, environmentId: string): void {
|
||||
const chain = this.chains.get(dispatchId)
|
||||
if (chain) {
|
||||
this.syncSoon(dispatchId, chain)
|
||||
return
|
||||
}
|
||||
const created: RelayChain = {
|
||||
environmentId,
|
||||
failures: 0,
|
||||
timer: null,
|
||||
dueAt: 0,
|
||||
running: false,
|
||||
runAgain: false
|
||||
}
|
||||
this.chains.set(dispatchId, created)
|
||||
const siblings = this.dispatchesByEnvironment.get(environmentId) ?? new Set<string>()
|
||||
siblings.add(dispatchId)
|
||||
this.dispatchesByEnvironment.set(environmentId, siblings)
|
||||
void this.run(dispatchId, created)
|
||||
}
|
||||
|
||||
/** A server that just verified is worth retrying at the base cadence. */
|
||||
resetEnvironment(environmentId: string): void {
|
||||
for (const dispatchId of this.dispatchesByEnvironment.get(environmentId) ?? []) {
|
||||
const chain = this.chains.get(dispatchId)
|
||||
if (chain) {
|
||||
chain.failures = 0
|
||||
this.syncSoon(dispatchId, chain)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** A sync another caller completed proves the relay healthy again, so the next one returns to the base cadence. */
|
||||
noteSynced(dispatchId: string): void {
|
||||
const chain = this.chains.get(dispatchId)
|
||||
if (!chain || chain.running || chain.failures === 0) {
|
||||
return
|
||||
}
|
||||
chain.failures = 0
|
||||
const baseDelayMs = federationRelayRetryDelayMs(0)
|
||||
// Only pull the next sync earlier; an already-requested immediate sync must not be postponed.
|
||||
if (!chain.timer || chain.dueAt > Date.now() + baseDelayMs) {
|
||||
this.schedule(dispatchId, chain, baseDelayMs)
|
||||
}
|
||||
}
|
||||
|
||||
clear(): void {
|
||||
for (const chain of this.chains.values()) {
|
||||
if (chain.timer) {
|
||||
clearTimeout(chain.timer)
|
||||
}
|
||||
}
|
||||
this.chains.clear()
|
||||
this.dispatchesByEnvironment.clear()
|
||||
}
|
||||
|
||||
private syncSoon(dispatchId: string, chain: RelayChain): void {
|
||||
if (chain.running) {
|
||||
chain.runAgain = true
|
||||
return
|
||||
}
|
||||
this.schedule(dispatchId, chain, 0)
|
||||
}
|
||||
|
||||
private schedule(dispatchId: string, chain: RelayChain, delayMs: number): void {
|
||||
if (chain.timer) {
|
||||
clearTimeout(chain.timer)
|
||||
}
|
||||
chain.dueAt = Date.now() + delayMs
|
||||
chain.timer = setTimeout(() => void this.run(dispatchId, chain), delayMs)
|
||||
chain.timer.unref?.()
|
||||
}
|
||||
|
||||
private async run(dispatchId: string, chain: RelayChain): Promise<void> {
|
||||
if (this.chains.get(dispatchId) !== chain) {
|
||||
return
|
||||
}
|
||||
chain.timer = null
|
||||
chain.running = true
|
||||
const outcome = await this.tick(dispatchId).catch((): FederationRelayTickOutcome => 'failed')
|
||||
chain.running = false
|
||||
// A cleared or replaced chain must not come back from a sync that outlived it.
|
||||
if (this.chains.get(dispatchId) !== chain) {
|
||||
return
|
||||
}
|
||||
if (outcome === 'ended') {
|
||||
this.chains.delete(dispatchId)
|
||||
const siblings = this.dispatchesByEnvironment.get(chain.environmentId)
|
||||
siblings?.delete(dispatchId)
|
||||
if (siblings?.size === 0) {
|
||||
this.dispatchesByEnvironment.delete(chain.environmentId)
|
||||
}
|
||||
return
|
||||
}
|
||||
chain.failures = outcome === 'synced' ? 0 : chain.failures + 1
|
||||
const delayMs = chain.runAgain ? 0 : federationRelayRetryDelayMs(chain.failures)
|
||||
chain.runAgain = false
|
||||
this.schedule(dispatchId, chain, delayMs)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { createIdleSyncHarness } from './synchronization/federation-sync-test-harness'
|
||||
import { RuntimeOrchestrationFederation } from './runtime-orchestration-federation'
|
||||
import type { OrchestrationEnvironmentTransport } from './environment-transport'
|
||||
|
||||
describe('RuntimeOrchestrationFederation relay', () => {
|
||||
beforeEach(() => {
|
||||
vi.useFakeTimers()
|
||||
})
|
||||
afterEach(() => {
|
||||
vi.useRealTimers()
|
||||
})
|
||||
|
||||
it('stops relaying to a removed server and delivers its pending mail after the next ensureRelay', async () => {
|
||||
const { runtime, remoteCall } = createIdleSyncHarness()
|
||||
const db = runtime.getOrchestrationDb()
|
||||
const active = db.getFederatedDispatch('dispatch_remote')
|
||||
db.listActiveFederatedDispatches = () => (active ? [active] : [])
|
||||
let serverSaved = false
|
||||
let noteRefused!: () => void
|
||||
const refused = new Promise<void>((resolve) => (noteRefused = resolve))
|
||||
let noteAcknowledged!: () => void
|
||||
const acknowledgedOnce = new Promise<void>((resolve) => (noteAcknowledged = resolve))
|
||||
const recordAcknowledgment = db.recordFederatedHomeAcknowledgment.bind(db)
|
||||
db.recordFederatedHomeAcknowledgment = (params) => {
|
||||
recordAcknowledgment(params)
|
||||
noteAcknowledged()
|
||||
}
|
||||
const answer = remoteCall.getMockImplementation()!
|
||||
remoteCall.mockImplementation(async (...args) => {
|
||||
if (!serverSaved) {
|
||||
noteRefused()
|
||||
throw new Error('Unknown environment: environment_windows')
|
||||
}
|
||||
return answer(...args)
|
||||
})
|
||||
const transport: OrchestrationEnvironmentTransport = {
|
||||
resolve: () => {
|
||||
throw new Error('unused')
|
||||
},
|
||||
call: () => {
|
||||
throw new Error('unused')
|
||||
},
|
||||
hasEnvironment: () => serverSaved
|
||||
}
|
||||
const federation = new RuntimeOrchestrationFederation(runtime, transport)
|
||||
const acknowledged = () =>
|
||||
runtime.getOrchestrationDb().getFederatedDispatch('dispatch_remote')
|
||||
?.to_home_acknowledged_sequence
|
||||
|
||||
federation.ensureRelay()
|
||||
await refused
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
serverSaved = true
|
||||
// A relay that ended leaves no timer behind; one still backing off would retry and deliver here.
|
||||
await vi.runAllTimersAsync()
|
||||
expect(acknowledged()).toBe(0)
|
||||
|
||||
federation.ensureRelay()
|
||||
|
||||
await acknowledgedOnce
|
||||
expect(acknowledged()).toBeGreaterThan(0)
|
||||
federation.stopRelay()
|
||||
})
|
||||
})
|
||||
@@ -24,15 +24,28 @@ import {
|
||||
import { syncFederatedDispatch } from './synchronization/federation-sync'
|
||||
import type { OrchestrationDb } from '../db/orchestration-db'
|
||||
import type { FederationRuntime } from './federation-runtime'
|
||||
import {
|
||||
FederationRelayScheduler,
|
||||
type FederationRelayTickOutcome
|
||||
} from './relay/federation-relay-scheduler'
|
||||
import { FederationAcknowledgmentRecovery } from './relay/federation-acknowledgment-recovery'
|
||||
|
||||
export class RuntimeOrchestrationFederation {
|
||||
private readonly timers = new Map<string, ReturnType<typeof setInterval>>()
|
||||
private readonly relays = new FederationRelayScheduler((dispatchId) => this.relayTick(dispatchId))
|
||||
private readonly acknowledgmentRecovery = new FederationAcknowledgmentRecovery({
|
||||
findNextAfter: (afterRowId) =>
|
||||
this.runtime
|
||||
.getOrchestrationDb()
|
||||
.findNextTerminalFederatedDispatchPendingAcknowledgment(afterRowId),
|
||||
sync: (dispatchId) => this.runtime.syncOrchestrationFederatedDispatch(dispatchId),
|
||||
acknowledgedThrough: (dispatchId) =>
|
||||
this.runtime.getOrchestrationDb().getFederatedDispatch(dispatchId)
|
||||
?.to_home_acknowledged_sequence ?? null
|
||||
})
|
||||
private readonly syncs = new Map<string, { db: OrchestrationDb; promise: Promise<void> }>()
|
||||
private readonly warnings = new Set<string>()
|
||||
private terminalRecoveryTimer: ReturnType<typeof setTimeout> | null = null
|
||||
private terminalRecoveryInFlight: Promise<void> | null = null
|
||||
private terminalRecoveryRowId = 0
|
||||
private relayGeneration = 0
|
||||
private stopVerifiedWatch: (() => void) | null = null
|
||||
/** Worker server -> runtime incarnation that last proved the orchestration contract. */
|
||||
private readonly contractVerifiedRuntimeIds = new Map<string, string>()
|
||||
|
||||
@@ -43,16 +56,9 @@ export class RuntimeOrchestrationFederation {
|
||||
|
||||
resetForDatabaseChange(): void {
|
||||
this.relayGeneration += 1
|
||||
for (const timer of this.timers.values()) {
|
||||
clearInterval(timer)
|
||||
}
|
||||
this.timers.clear()
|
||||
if (this.terminalRecoveryTimer) {
|
||||
clearTimeout(this.terminalRecoveryTimer)
|
||||
}
|
||||
this.terminalRecoveryTimer = null
|
||||
this.terminalRecoveryInFlight = null
|
||||
this.terminalRecoveryRowId = 0
|
||||
this.unwatchVerifiedEnvironments()
|
||||
this.relays.clear()
|
||||
this.acknowledgmentRecovery.reset()
|
||||
clearFederationAckCheckpoints(this.runtime)
|
||||
this.syncs.clear()
|
||||
this.warnings.clear()
|
||||
@@ -172,10 +178,14 @@ export class RuntimeOrchestrationFederation {
|
||||
if (current?.db === db) {
|
||||
return current.promise
|
||||
}
|
||||
const sync = syncFederatedDispatch(this.runtime, dispatchId)
|
||||
const generation = this.relayGeneration
|
||||
const isCurrent = (): boolean =>
|
||||
generation === this.relayGeneration && this.runtime.getOrchestrationDb() === db
|
||||
const sync = syncFederatedDispatch(this.runtime, dispatchId, isCurrent)
|
||||
.then(() => {
|
||||
if (this.syncs.get(dispatchId)?.promise === sync) {
|
||||
this.warnings.delete(dispatchId)
|
||||
this.relays.noteSynced(dispatchId)
|
||||
}
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
@@ -200,10 +210,14 @@ export class RuntimeOrchestrationFederation {
|
||||
|
||||
async syncDispatchAfterCurrent(dispatchId: string): Promise<void> {
|
||||
const db = this.runtime.getOrchestrationDb()
|
||||
const generation = this.relayGeneration
|
||||
const current = this.syncs.get(dispatchId)
|
||||
if (current?.db === db) {
|
||||
await current.promise.catch(() => undefined)
|
||||
}
|
||||
if (generation !== this.relayGeneration || this.runtime.getOrchestrationDb() !== db) {
|
||||
return
|
||||
}
|
||||
await this.syncDispatch(dispatchId)
|
||||
}
|
||||
|
||||
@@ -211,87 +225,61 @@ export class RuntimeOrchestrationFederation {
|
||||
if (!this.transport) {
|
||||
return
|
||||
}
|
||||
this.stopVerifiedWatch ??=
|
||||
this.transport.onEnvironmentVerified?.((environmentId) =>
|
||||
this.relays.resetEnvironment(environmentId)
|
||||
) ?? null
|
||||
for (const dispatch of this.runtime.getOrchestrationDb().listActiveFederatedDispatches(runId)) {
|
||||
if (this.timers.has(dispatch.dispatch_id)) {
|
||||
continue
|
||||
}
|
||||
const tick = () => {
|
||||
const db = this.runtime.getOrchestrationDb()
|
||||
if (!db.isFederatedDispatchRelayEligible(dispatch.dispatch_id)) {
|
||||
const activeTimer = this.timers.get(dispatch.dispatch_id)
|
||||
if (activeTimer) {
|
||||
clearInterval(activeTimer)
|
||||
}
|
||||
this.timers.delete(dispatch.dispatch_id)
|
||||
this.warnings.delete(dispatch.dispatch_id)
|
||||
return
|
||||
this.relays.arm(dispatch.dispatch_id, dispatch.environment_id)
|
||||
}
|
||||
this.acknowledgmentRecovery.ensure()
|
||||
}
|
||||
|
||||
private async relayTick(dispatchId: string): Promise<FederationRelayTickOutcome> {
|
||||
if (!this.runtime.getOrchestrationDb().isFederatedDispatchRelayEligible(dispatchId)) {
|
||||
this.warnings.delete(dispatchId)
|
||||
return 'ended'
|
||||
}
|
||||
return this.syncDispatch(dispatchId).then(
|
||||
(): FederationRelayTickOutcome => 'synced',
|
||||
(): FederationRelayTickOutcome => {
|
||||
if (!this.isEnvironmentRemoved(dispatchId)) {
|
||||
return 'failed'
|
||||
}
|
||||
void this.syncDispatch(dispatch.dispatch_id).catch(() => undefined)
|
||||
this.warnings.delete(dispatchId)
|
||||
return 'ended'
|
||||
}
|
||||
const timer = setInterval(tick, 1_000)
|
||||
timer.unref?.()
|
||||
this.timers.set(dispatch.dispatch_id, timer)
|
||||
tick()
|
||||
}
|
||||
this.ensureTerminalHistoryRecovery()
|
||||
}
|
||||
|
||||
private ensureTerminalHistoryRecovery(): void {
|
||||
if (this.terminalRecoveryTimer || this.terminalRecoveryInFlight) {
|
||||
return
|
||||
}
|
||||
const generation = this.relayGeneration
|
||||
const recovery = this.recoverNextTerminalHistoryAcknowledgment(generation).catch((error) => {
|
||||
console.warn('[orchestration] terminal federation acknowledgment recovery failed', error)
|
||||
})
|
||||
this.terminalRecoveryInFlight = recovery
|
||||
void recovery.finally(() => {
|
||||
if (this.terminalRecoveryInFlight === recovery) {
|
||||
this.terminalRecoveryInFlight = null
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
private async recoverNextTerminalHistoryAcknowledgment(generation: number): Promise<void> {
|
||||
const db = this.runtime.getOrchestrationDb()
|
||||
let historical = db.findNextTerminalFederatedDispatchPendingAcknowledgment(
|
||||
this.terminalRecoveryRowId
|
||||
)
|
||||
if (!historical && this.terminalRecoveryRowId > 0) {
|
||||
this.terminalRecoveryRowId = 0
|
||||
historical = db.findNextTerminalFederatedDispatchPendingAcknowledgment(0)
|
||||
}
|
||||
|
||||
/** Checked only after a failed sync; a store read error keeps the relay retrying. */
|
||||
private isEnvironmentRemoved(dispatchId: string): boolean {
|
||||
const environmentId = this.runtime
|
||||
.getOrchestrationDb()
|
||||
.getFederatedDispatch(dispatchId)?.environment_id
|
||||
if (environmentId === undefined) {
|
||||
return false
|
||||
}
|
||||
if (!historical) {
|
||||
return
|
||||
try {
|
||||
// A removed server cannot answer; the next ensureRelay re-arms it if the server is added back.
|
||||
return this.transport?.hasEnvironment?.(environmentId) === false
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
this.terminalRecoveryRowId = historical.rowId
|
||||
await this.runtime
|
||||
.syncOrchestrationFederatedDispatch(historical.dispatchId)
|
||||
.catch(() => undefined)
|
||||
if (generation !== this.relayGeneration) {
|
||||
return
|
||||
}
|
||||
this.terminalRecoveryTimer = setTimeout(() => {
|
||||
this.terminalRecoveryTimer = null
|
||||
this.ensureTerminalHistoryRecovery()
|
||||
}, 1_000)
|
||||
this.terminalRecoveryTimer.unref?.()
|
||||
}
|
||||
|
||||
private unwatchVerifiedEnvironments(): void {
|
||||
this.stopVerifiedWatch?.()
|
||||
this.stopVerifiedWatch = null
|
||||
}
|
||||
|
||||
stopRelay(): void {
|
||||
this.relayGeneration += 1
|
||||
for (const timer of this.timers.values()) {
|
||||
clearInterval(timer)
|
||||
}
|
||||
this.timers.clear()
|
||||
this.unwatchVerifiedEnvironments()
|
||||
this.relays.clear()
|
||||
this.acknowledgmentRecovery.reset()
|
||||
this.warnings.clear()
|
||||
this.syncs.clear()
|
||||
if (this.terminalRecoveryTimer) {
|
||||
clearTimeout(this.terminalRecoveryTimer)
|
||||
}
|
||||
this.terminalRecoveryTimer = null
|
||||
this.terminalRecoveryInFlight = null
|
||||
this.terminalRecoveryRowId = 0
|
||||
clearFederationAckCheckpoints(this.runtime)
|
||||
}
|
||||
}
|
||||
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { ORCHESTRATION_FEDERATION_RELEASE_ARCHIVE_RUNTIME_CAPABILITY } from '../../../../../shared/orchestration-runtime-capabilities'
|
||||
import { createIdleSyncHarness } from './federation-sync-test-harness'
|
||||
import { armHomeAutoReleaseForOlderHost } from './federation-sync-capability'
|
||||
|
||||
function releasableTerminalWorker() {
|
||||
const harness = createIdleSyncHarness()
|
||||
harness.setRemoteCapabilities([ORCHESTRATION_FEDERATION_RELEASE_ARCHIVE_RUNTIME_CAPABILITY])
|
||||
const db = harness.runtime.getOrchestrationDb()
|
||||
const federated = db.getFederatedDispatch('dispatch_remote')!
|
||||
db.getFederatedDispatch = () => ({
|
||||
...federated,
|
||||
remote_terminal_handle: 'term_remote_worker',
|
||||
home_auto_release_state: null
|
||||
})
|
||||
const worker = db.getWorkerDispatch('dispatch_remote')!
|
||||
db.getWorkerDispatch = () => ({ ...worker, state: 'succeeded', stage: 'reported' })
|
||||
db.workerAutoReleaseEnabled = () => true
|
||||
return {
|
||||
runtime: harness.runtime,
|
||||
remoteCall: harness.remoteCall,
|
||||
dispatchId: federated.dispatch_id,
|
||||
armedDeadlines: () => db.claimDueFederatedHomeAutoReleases(Date.UTC(2100, 0, 1))
|
||||
}
|
||||
}
|
||||
|
||||
describe('armHomeAutoReleaseForOlderHost', () => {
|
||||
it('does not arm from a sync the relay stopped while it probed the host', async () => {
|
||||
const { runtime, remoteCall, dispatchId, armedDeadlines } = releasableTerminalWorker()
|
||||
const answer = remoteCall.getMockImplementation()!
|
||||
let noteProbe!: () => void
|
||||
let releaseProbe!: () => void
|
||||
const probing = new Promise<void>((resolve) => (noteProbe = resolve))
|
||||
const released = new Promise<void>((resolve) => (releaseProbe = resolve))
|
||||
remoteCall.mockImplementation(async (...args) => {
|
||||
if (args[1] === 'status.get') {
|
||||
noteProbe()
|
||||
await released
|
||||
}
|
||||
return answer(...args)
|
||||
})
|
||||
let current = true
|
||||
|
||||
const arming = armHomeAutoReleaseForOlderHost(runtime, dispatchId, 1, () => current)
|
||||
await probing
|
||||
current = false
|
||||
releaseProbe()
|
||||
await arming
|
||||
|
||||
expect(armedDeadlines()).toEqual([])
|
||||
})
|
||||
})
|
||||
+3
-2
@@ -55,7 +55,8 @@ function resolveFederatedHostCapability(
|
||||
export async function armHomeAutoReleaseForOlderHost(
|
||||
runtime: FederationRuntime,
|
||||
dispatchId: string,
|
||||
pairingRevision: number | undefined
|
||||
pairingRevision: number | undefined,
|
||||
isCurrent: () => boolean = () => true
|
||||
): Promise<void> {
|
||||
const db = runtime.getOrchestrationDb()
|
||||
const federated = db.getFederatedDispatch(dispatchId)
|
||||
@@ -84,7 +85,7 @@ export async function armHomeAutoReleaseForOlderHost(
|
||||
)
|
||||
)
|
||||
)
|
||||
if (!hostReleases?.supported && releasable?.supported !== false) {
|
||||
if (isCurrent() && !hostReleases?.supported && releasable?.supported !== false) {
|
||||
db.armFederatedHomeAutoRelease(dispatchId)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { z } from 'zod'
|
||||
import {
|
||||
MESSAGE_TYPES,
|
||||
type MessagePriority,
|
||||
@@ -102,3 +103,22 @@ export function parseFederatedLifecycle(
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Decode untrusted peer pages before import to report malformed input as an orchestration error.
|
||||
export const PulledRelayPage = z
|
||||
.object({
|
||||
runtimeEpoch: z.string().min(1),
|
||||
items: z.array(
|
||||
z
|
||||
.object({
|
||||
dispatch_id: z.string(),
|
||||
direction: z.literal('to_home'),
|
||||
sequence: z.number(),
|
||||
message_id: z.string(),
|
||||
kind: z.string(),
|
||||
payload: z.string()
|
||||
})
|
||||
.passthrough()
|
||||
)
|
||||
})
|
||||
.passthrough()
|
||||
|
||||
@@ -538,6 +538,29 @@ describe('federation relay acknowledgments', () => {
|
||||
remoteCall.mock.calls.filter(([, method]) => method === 'orchestration.federationAck')
|
||||
).toHaveLength(2)
|
||||
})
|
||||
|
||||
it.each(['stopRelay', 'database swap'] as const)(
|
||||
'keeps a sync interrupted by %s from acknowledging into current state',
|
||||
async (interruption) => {
|
||||
const { runtime, blockAck, replaceDb } = createIdleSyncHarness()
|
||||
const gate = blockAck()
|
||||
|
||||
const oldSync = runtime.syncOrchestrationFederatedDispatch('dispatch_remote')
|
||||
await gate.reached
|
||||
if (interruption === 'stopRelay') {
|
||||
runtime.stopOrchestrationFederationRelay()
|
||||
} else {
|
||||
replaceDb()
|
||||
}
|
||||
gate.release()
|
||||
await oldSync
|
||||
|
||||
expect(
|
||||
runtime.getOrchestrationDb().getFederatedDispatch('dispatch_remote')
|
||||
?.to_home_acknowledged_sequence
|
||||
).toBe(0)
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
describe('bounded outgoing answer relay', () => {
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { federationRelayPrefix } from '../relay/federation-relay-prefix'
|
||||
import { federationRelayWireItem } from '../relay/federation-relay-wire'
|
||||
import { z } from 'zod'
|
||||
import type { FederationRuntime } from '../federation-runtime'
|
||||
import type { FederatedLifecycleSettlement } from '../federation-lifecycle-settlement'
|
||||
import '../../../../../shared/protocol-version'
|
||||
@@ -17,30 +16,15 @@ import {
|
||||
resolveFederatedLifecycleSettlementCapability
|
||||
} from './federation-sync-capability'
|
||||
import { getOrchestrationPeerCapabilityCache } from '../orchestration-peer-capability-cache'
|
||||
import { parseFederatedLifecycle, parseRelayedMessage } from './federation-sync-message'
|
||||
import {
|
||||
parseFederatedLifecycle,
|
||||
parseRelayedMessage,
|
||||
PulledRelayPage
|
||||
} from './federation-sync-message'
|
||||
|
||||
const FEDERATION_PULL_PAGE_SIZE = 50
|
||||
const MAX_FEDERATION_PULL_PAGES_PER_SYNC = 6
|
||||
|
||||
// Decode untrusted peer pages before import to report malformed input as an orchestration error.
|
||||
const PulledRelayPage = z
|
||||
.object({
|
||||
runtimeEpoch: z.string().min(1),
|
||||
items: z.array(
|
||||
z
|
||||
.object({
|
||||
dispatch_id: z.string(),
|
||||
direction: z.literal('to_home'),
|
||||
sequence: z.number(),
|
||||
message_id: z.string(),
|
||||
kind: z.string(),
|
||||
payload: z.string()
|
||||
})
|
||||
.passthrough()
|
||||
)
|
||||
})
|
||||
.passthrough()
|
||||
|
||||
export async function syncFederatedDispatch(
|
||||
runtime: FederationRuntime,
|
||||
dispatchId: string,
|
||||
@@ -85,6 +69,9 @@ async function syncFederatedDispatchPages(
|
||||
federated,
|
||||
currentServer.pairingRevision
|
||||
)
|
||||
if (!isCurrent()) {
|
||||
return { imported: 0, acknowledgedThrough: federated.to_home_imported_sequence }
|
||||
}
|
||||
const supportsLifecycleSettlement =
|
||||
federated.protocol_version >= ORCHESTRATION_FEDERATION_LIFECYCLE_SETTLEMENT_PROTOCOL_VERSION &&
|
||||
capability?.supported === true
|
||||
@@ -221,26 +208,31 @@ async function syncFederatedDispatchPages(
|
||||
const locallyAcknowledgedThrough = keepRelayEligible
|
||||
? Math.max(0, delivered.acknowledgedThrough - 1)
|
||||
: delivered.acknowledgedThrough
|
||||
db.recordFederatedHomeAcknowledgment({
|
||||
dispatchId,
|
||||
remoteRuntimeEpoch: pulled.runtimeEpoch,
|
||||
sequence: locallyAcknowledgedThrough
|
||||
})
|
||||
recordFederationAckCheckpoint(runtime, ackLease, {
|
||||
...ackIdentity,
|
||||
throughSequence: locallyAcknowledgedThrough
|
||||
})
|
||||
if (isCurrent()) {
|
||||
db.recordFederatedHomeAcknowledgment({
|
||||
dispatchId,
|
||||
remoteRuntimeEpoch: pulled.runtimeEpoch,
|
||||
sequence: locallyAcknowledgedThrough
|
||||
})
|
||||
recordFederationAckCheckpoint(runtime, ackLease, {
|
||||
...ackIdentity,
|
||||
throughSequence: locallyAcknowledgedThrough
|
||||
})
|
||||
}
|
||||
}
|
||||
getOrchestrationPeerCapabilityCache(runtime).observeEpoch(
|
||||
federated.peer_fingerprint,
|
||||
pulled.runtimeEpoch
|
||||
)
|
||||
db.updateFederatedDispatchRuntimeEpoch(dispatchId, pulled.runtimeEpoch)
|
||||
if (isCurrent()) {
|
||||
// Best effort: without a deadline the worker keeps manual release, as before.
|
||||
await armHomeAutoReleaseForOlderHost(runtime, dispatchId, currentServer.pairingRevision).catch(
|
||||
() => undefined
|
||||
getOrchestrationPeerCapabilityCache(runtime).observeEpoch(
|
||||
federated.peer_fingerprint,
|
||||
pulled.runtimeEpoch
|
||||
)
|
||||
db.updateFederatedDispatchRuntimeEpoch(dispatchId, pulled.runtimeEpoch)
|
||||
// Best effort: without a deadline the worker keeps manual release, as before.
|
||||
await armHomeAutoReleaseForOlderHost(
|
||||
runtime,
|
||||
dispatchId,
|
||||
currentServer.pairingRevision,
|
||||
isCurrent
|
||||
).catch(() => undefined)
|
||||
}
|
||||
const toWorker =
|
||||
db.getWorkerDispatch(dispatchId)?.state === 'ready'
|
||||
|
||||
@@ -13,6 +13,7 @@ import type { CheckParams } from '../../../../../../shared/rpc-contract/orchestr
|
||||
import type { z } from 'zod'
|
||||
import type { OrchestrationSessionCaller } from '../../../../orchestration/addressing/orchestration-caller-identity'
|
||||
import { checkRunPendingMail } from './check-run-pending-mail'
|
||||
import { FEDERATION_RELAY_WAIT_REARM_MS } from '../../../../orchestration/federation/relay/federation-relay-scheduler'
|
||||
|
||||
type CheckParamsInput = z.infer<typeof CheckParams>
|
||||
|
||||
@@ -164,11 +165,21 @@ export async function checkRunMailbox(args: {
|
||||
}
|
||||
}
|
||||
|
||||
const waitResult = await runtime.waitForMessage(address, {
|
||||
typeFilter: typeFilter as string[] | undefined,
|
||||
timeoutMs: params.timeoutMs ?? undefined,
|
||||
signal
|
||||
})
|
||||
const rearmRelays = setInterval(() => {
|
||||
try {
|
||||
runtime.ensureOrchestrationFederationRelay(run.id)
|
||||
} catch (error) {
|
||||
console.warn('[orchestration] federation relay re-arm failed during check --wait', error)
|
||||
}
|
||||
}, FEDERATION_RELAY_WAIT_REARM_MS)
|
||||
rearmRelays.unref?.()
|
||||
const waitResult = await runtime
|
||||
.waitForMessage(address, {
|
||||
typeFilter: typeFilter as string[] | undefined,
|
||||
timeoutMs: params.timeoutMs ?? undefined,
|
||||
signal
|
||||
})
|
||||
.finally(() => clearInterval(rearmRelays))
|
||||
try {
|
||||
revalidateConsumer()
|
||||
} catch (error) {
|
||||
|
||||
@@ -25,6 +25,8 @@ import type { TerminalSideEffectBatch } from '../../shared/terminal/output/termi
|
||||
import { resolveTuiAgentLaunchEnv } from '../../shared/agent-launch/choices/tui-agent-launch-defaults'
|
||||
import { callRuntimeEnvironment } from '../ipc/runtime-environments/runtime-environment-transport-routing'
|
||||
import { createOrchestrationEnvironmentTransport } from '../ipc/runtime-environments/orchestration-environment-transport'
|
||||
import { trackRuntimeEnvironmentVerifiedTransitions } from '../ipc/runtime-environments/runtime-environment-verified-transitions'
|
||||
import { onRuntimeEnvironmentStatusPublished } from '../ipc/runtime-environments/runtime-environment-status-owner'
|
||||
import { mainProcessState as state } from './main-process-state'
|
||||
import { suspendEphemeralVmWorkspace } from '../ephemeral-vm/workspace-suspend'
|
||||
import {
|
||||
@@ -114,7 +116,11 @@ export function initializeMainProcessRuntime(): OrcaRuntimeService {
|
||||
homePath,
|
||||
resolveTuiAgentLaunchEnv('codex', store.getSettings())
|
||||
),
|
||||
orchestrationEnvironmentTransport: createOrchestrationEnvironmentTransport(),
|
||||
orchestrationEnvironmentTransport: {
|
||||
...createOrchestrationEnvironmentTransport(),
|
||||
onEnvironmentVerified: (listener) =>
|
||||
onRuntimeEnvironmentStatusPublished(trackRuntimeEnvironmentVerifiedTransitions(listener))
|
||||
},
|
||||
workspaceSleepPlatform: {
|
||||
callEnvironment: (environmentId, method, params, timeoutMs) =>
|
||||
callRuntimeEnvironment(
|
||||
|
||||
Reference in New Issue
Block a user