mirror of
https://github.com/stablyai/orca.git
synced 2026-10-20 08:03:28 +00:00
fix(relay): revive a dead relay broker without user interaction (#12432)
Broker deaths that end with closeNow() — an auth refresh failing past token expiry (laptop sleep) or a transient context read at open — left no retry timer, so Relay stayed offline until the user clicked Retry or auth state changed. Adds a dead-man's switch: - RelayAuthCoordinator.ensureLive(): reconciles only when there is no live broker, no scheduled retry, and no open in flight - DesktopRelayService arms a 5-minute liveness interval and exposes ensureLive() for wake signals - powerMonitor 'resume' triggers ensureLive (sleep is the common cause) Co-authored-by: OrcaWin <293788423+OrcaWin@users.noreply.github.com>
This commit is contained in:
committed by
Jinjing
co-authored by
OrcaWin
parent
b477ede2dd
commit
7600e9a0e3
+4
-1
@@ -2,7 +2,7 @@
|
||||
import { existsSync, statSync } from 'node:fs'
|
||||
import { isAbsolute, join } from 'node:path'
|
||||
import os from 'node:os'
|
||||
import { app, BrowserWindow, dialog, ipcMain, nativeTheme, type Tray } from 'electron'
|
||||
import { app, BrowserWindow, dialog, ipcMain, nativeTheme, powerMonitor, type Tray } from 'electron'
|
||||
import { initTccPromptNotice, stopTccPromptNotice } from './macos-tcc-prompt-notice'
|
||||
import { electronApp, is } from '@electron-toolkit/utils'
|
||||
import {
|
||||
@@ -2896,6 +2896,9 @@ void app.whenReady().then(async () => {
|
||||
provisionRelay: (context, params) => relayService.provisionRelay(context, params)
|
||||
})
|
||||
relayService.start()
|
||||
// Why: sleeping past relay-token expiry kills the broker with no retry
|
||||
// timer; resume is the moment that state becomes recoverable.
|
||||
powerMonitor.on('resume', () => desktopRelayService?.ensureLive())
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
'[relay] Desktop relay startup unavailable:',
|
||||
|
||||
@@ -42,12 +42,18 @@ export function pairingAuthorizationForContext(
|
||||
: null
|
||||
}
|
||||
|
||||
// Why: a broker that died without arming a retry (sleep past token expiry,
|
||||
// transient auth read) must not stay dead until the user clicks Retry. The
|
||||
// cadence is slow because it is a safety net, not the primary retry path.
|
||||
const RELAY_LIVENESS_INTERVAL_MS = 5 * 60_000
|
||||
|
||||
export class DesktopRelayService {
|
||||
private readonly coordinator: RelayAuthCoordinator
|
||||
private readonly revokeOutbox: RelayRevokeOutbox
|
||||
private readonly runtimeRpc: OrcaRuntimeRpcServer
|
||||
private readonly demandLedger: RelayDemandLedger
|
||||
private demandExpiryTimer: ReturnType<typeof setTimeout> | null = null
|
||||
private livenessTimer: ReturnType<typeof setInterval> | null = null
|
||||
private stopped = false
|
||||
|
||||
constructor(options: DesktopRelayServiceOptions) {
|
||||
@@ -90,6 +96,16 @@ export class DesktopRelayService {
|
||||
|
||||
start(): void {
|
||||
this.refreshDemand()
|
||||
if (!this.livenessTimer) {
|
||||
this.livenessTimer = setInterval(() => this.ensureLive(), RELAY_LIVENESS_INTERVAL_MS)
|
||||
}
|
||||
}
|
||||
|
||||
// Safe to call from any wake signal (power resume, network change).
|
||||
ensureLive(): void {
|
||||
if (!this.stopped) {
|
||||
this.coordinator.ensureLive()
|
||||
}
|
||||
}
|
||||
|
||||
authMutated(): void {
|
||||
@@ -217,6 +233,10 @@ export class DesktopRelayService {
|
||||
clearTimeout(this.demandExpiryTimer)
|
||||
this.demandExpiryTimer = null
|
||||
}
|
||||
if (this.livenessTimer) {
|
||||
clearInterval(this.livenessTimer)
|
||||
this.livenessTimer = null
|
||||
}
|
||||
this.coordinator.stop()
|
||||
}
|
||||
|
||||
|
||||
@@ -257,3 +257,77 @@ describe('RelayAuthCoordinator transient recovery', () => {
|
||||
expect(openBroker).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
})
|
||||
|
||||
describe('RelayAuthCoordinator liveness safety net', () => {
|
||||
it('reopens a broker that died leaving no retry timer behind', async () => {
|
||||
// Why: an auth refresh failing past token expiry closes the broker with
|
||||
// no timer; only ensureLive (periodic/power-resume) can revive it.
|
||||
vi.useFakeTimers()
|
||||
const dead = { closeNow: vi.fn(), isLive: () => false }
|
||||
const live = { closeNow: vi.fn(), isLive: () => true }
|
||||
const openBroker = vi.fn().mockResolvedValueOnce(dead).mockResolvedValueOnce(live)
|
||||
const coordinator = new RelayAuthCoordinator({
|
||||
readContext: async () => context,
|
||||
openBroker,
|
||||
onStatus: vi.fn(),
|
||||
random: () => 0.5
|
||||
})
|
||||
|
||||
coordinator.reconcile()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(coordinator.getActiveBroker()).toBe(dead)
|
||||
await vi.advanceTimersByTimeAsync(600_000)
|
||||
expect(openBroker).toHaveBeenCalledOnce()
|
||||
|
||||
coordinator.ensureLive()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(openBroker).toHaveBeenCalledTimes(2)
|
||||
expect(dead.closeNow).toHaveBeenCalled()
|
||||
expect(coordinator.getActiveBroker()).toBe(live)
|
||||
})
|
||||
|
||||
it('does not disturb a live broker, a scheduled retry, or an open in flight', async () => {
|
||||
vi.useFakeTimers()
|
||||
const broker = { closeNow: vi.fn(), isLive: () => true }
|
||||
let releaseOpen: ((value: typeof broker) => void) | undefined
|
||||
const openBroker = vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new RelayHttpError('assignment', 503, 30_000))
|
||||
.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
releaseOpen = resolve
|
||||
})
|
||||
)
|
||||
const coordinator = new RelayAuthCoordinator({
|
||||
readContext: async () => context,
|
||||
openBroker,
|
||||
onStatus: vi.fn(),
|
||||
random: () => 0.5
|
||||
})
|
||||
|
||||
coordinator.reconcile()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(openBroker).toHaveBeenCalledOnce()
|
||||
|
||||
// A scheduled Retry-After timer owns recovery; ensureLive must not race it.
|
||||
coordinator.ensureLive()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(openBroker).toHaveBeenCalledOnce()
|
||||
|
||||
// The retry fires into a slow open; ensureLive must not preempt it.
|
||||
await vi.advanceTimersByTimeAsync(30_000)
|
||||
expect(openBroker).toHaveBeenCalledTimes(2)
|
||||
coordinator.ensureLive()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(openBroker).toHaveBeenCalledTimes(2)
|
||||
releaseOpen?.(broker)
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(coordinator.getActiveBroker()).toBe(broker)
|
||||
|
||||
// A live broker needs nothing.
|
||||
coordinator.ensureLive()
|
||||
await vi.advanceTimersByTimeAsync(0)
|
||||
expect(openBroker).toHaveBeenCalledTimes(2)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -92,6 +92,22 @@ export class RelayAuthCoordinator {
|
||||
return this.ownership?.valid ? this.ownership.broker : null
|
||||
}
|
||||
|
||||
// Why: some broker deaths end with no retry timer — an auth refresh that
|
||||
// fails past token expiry (laptop sleep), or a transient context read that
|
||||
// returned null at open. Periodic/power-resume callers use this as a
|
||||
// dead-man's switch; it never disturbs a live broker, a scheduled retry,
|
||||
// or an open already in flight.
|
||||
ensureLive(): void {
|
||||
if (this.stopped || this.retryTimer || this.pendingOwnerships.size > 0) {
|
||||
return
|
||||
}
|
||||
const ownership = this.ownership
|
||||
if (ownership?.valid && (ownership.broker?.isLive?.() ?? true)) {
|
||||
return
|
||||
}
|
||||
this.beginReconcile(false)
|
||||
}
|
||||
|
||||
async waitForActiveBroker(): Promise<CoordinatedRelayBroker | null> {
|
||||
while (!this.stopped) {
|
||||
const broker = this.getActiveBroker()
|
||||
|
||||
Reference in New Issue
Block a user