fix(relay): revive a dead relay broker without user interaction (#12432)

Broker deaths that end with closeNow() — an auth refresh failing past
token expiry (laptop sleep) or a transient context read at open — left
no retry timer, so Relay stayed offline until the user clicked Retry or
auth state changed. Adds a dead-man's switch:

- RelayAuthCoordinator.ensureLive(): reconciles only when there is no
  live broker, no scheduled retry, and no open in flight
- DesktopRelayService arms a 5-minute liveness interval and exposes
  ensureLive() for wake signals
- powerMonitor 'resume' triggers ensureLive (sleep is the common cause)

Co-authored-by: OrcaWin <293788423+OrcaWin@users.noreply.github.com>
This commit is contained in:
Jinwoo Hong
2026-08-04 02:19:36 -07:00
committed by Jinjing
co-authored by OrcaWin
parent b477ede2dd
commit 7600e9a0e3
4 changed files with 114 additions and 1 deletions
+4 -1
View File
@@ -2,7 +2,7 @@
import { existsSync, statSync } from 'node:fs'
import { isAbsolute, join } from 'node:path'
import os from 'node:os'
import { app, BrowserWindow, dialog, ipcMain, nativeTheme, type Tray } from 'electron'
import { app, BrowserWindow, dialog, ipcMain, nativeTheme, powerMonitor, type Tray } from 'electron'
import { initTccPromptNotice, stopTccPromptNotice } from './macos-tcc-prompt-notice'
import { electronApp, is } from '@electron-toolkit/utils'
import {
@@ -2896,6 +2896,9 @@ void app.whenReady().then(async () => {
provisionRelay: (context, params) => relayService.provisionRelay(context, params)
})
relayService.start()
// Why: sleeping past relay-token expiry kills the broker with no retry
// timer; resume is the moment that state becomes recoverable.
powerMonitor.on('resume', () => desktopRelayService?.ensureLive())
} catch (error) {
console.warn(
'[relay] Desktop relay startup unavailable:',
@@ -42,12 +42,18 @@ export function pairingAuthorizationForContext(
: null
}
// Why: a broker that died without arming a retry (sleep past token expiry,
// transient auth read) must not stay dead until the user clicks Retry. The
// cadence is slow because it is a safety net, not the primary retry path.
const RELAY_LIVENESS_INTERVAL_MS = 5 * 60_000
export class DesktopRelayService {
private readonly coordinator: RelayAuthCoordinator
private readonly revokeOutbox: RelayRevokeOutbox
private readonly runtimeRpc: OrcaRuntimeRpcServer
private readonly demandLedger: RelayDemandLedger
private demandExpiryTimer: ReturnType<typeof setTimeout> | null = null
private livenessTimer: ReturnType<typeof setInterval> | null = null
private stopped = false
constructor(options: DesktopRelayServiceOptions) {
@@ -90,6 +96,16 @@ export class DesktopRelayService {
start(): void {
this.refreshDemand()
if (!this.livenessTimer) {
this.livenessTimer = setInterval(() => this.ensureLive(), RELAY_LIVENESS_INTERVAL_MS)
}
}
// Safe to call from any wake signal (power resume, network change).
ensureLive(): void {
if (!this.stopped) {
this.coordinator.ensureLive()
}
}
authMutated(): void {
@@ -217,6 +233,10 @@ export class DesktopRelayService {
clearTimeout(this.demandExpiryTimer)
this.demandExpiryTimer = null
}
if (this.livenessTimer) {
clearInterval(this.livenessTimer)
this.livenessTimer = null
}
this.coordinator.stop()
}
@@ -257,3 +257,77 @@ describe('RelayAuthCoordinator transient recovery', () => {
expect(openBroker).toHaveBeenCalledTimes(2)
})
})
describe('RelayAuthCoordinator liveness safety net', () => {
it('reopens a broker that died leaving no retry timer behind', async () => {
// Why: an auth refresh failing past token expiry closes the broker with
// no timer; only ensureLive (periodic/power-resume) can revive it.
vi.useFakeTimers()
const dead = { closeNow: vi.fn(), isLive: () => false }
const live = { closeNow: vi.fn(), isLive: () => true }
const openBroker = vi.fn().mockResolvedValueOnce(dead).mockResolvedValueOnce(live)
const coordinator = new RelayAuthCoordinator({
readContext: async () => context,
openBroker,
onStatus: vi.fn(),
random: () => 0.5
})
coordinator.reconcile()
await vi.advanceTimersByTimeAsync(0)
expect(coordinator.getActiveBroker()).toBe(dead)
await vi.advanceTimersByTimeAsync(600_000)
expect(openBroker).toHaveBeenCalledOnce()
coordinator.ensureLive()
await vi.advanceTimersByTimeAsync(0)
expect(openBroker).toHaveBeenCalledTimes(2)
expect(dead.closeNow).toHaveBeenCalled()
expect(coordinator.getActiveBroker()).toBe(live)
})
it('does not disturb a live broker, a scheduled retry, or an open in flight', async () => {
vi.useFakeTimers()
const broker = { closeNow: vi.fn(), isLive: () => true }
let releaseOpen: ((value: typeof broker) => void) | undefined
const openBroker = vi
.fn()
.mockRejectedValueOnce(new RelayHttpError('assignment', 503, 30_000))
.mockImplementationOnce(
() =>
new Promise((resolve) => {
releaseOpen = resolve
})
)
const coordinator = new RelayAuthCoordinator({
readContext: async () => context,
openBroker,
onStatus: vi.fn(),
random: () => 0.5
})
coordinator.reconcile()
await vi.advanceTimersByTimeAsync(0)
expect(openBroker).toHaveBeenCalledOnce()
// A scheduled Retry-After timer owns recovery; ensureLive must not race it.
coordinator.ensureLive()
await vi.advanceTimersByTimeAsync(0)
expect(openBroker).toHaveBeenCalledOnce()
// The retry fires into a slow open; ensureLive must not preempt it.
await vi.advanceTimersByTimeAsync(30_000)
expect(openBroker).toHaveBeenCalledTimes(2)
coordinator.ensureLive()
await vi.advanceTimersByTimeAsync(0)
expect(openBroker).toHaveBeenCalledTimes(2)
releaseOpen?.(broker)
await vi.advanceTimersByTimeAsync(0)
expect(coordinator.getActiveBroker()).toBe(broker)
// A live broker needs nothing.
coordinator.ensureLive()
await vi.advanceTimersByTimeAsync(0)
expect(openBroker).toHaveBeenCalledTimes(2)
})
})
@@ -92,6 +92,22 @@ export class RelayAuthCoordinator {
return this.ownership?.valid ? this.ownership.broker : null
}
// Why: some broker deaths end with no retry timer — an auth refresh that
// fails past token expiry (laptop sleep), or a transient context read that
// returned null at open. Periodic/power-resume callers use this as a
// dead-man's switch; it never disturbs a live broker, a scheduled retry,
// or an open already in flight.
ensureLive(): void {
if (this.stopped || this.retryTimer || this.pendingOwnerships.size > 0) {
return
}
const ownership = this.ownership
if (ownership?.valid && (ownership.broker?.isLive?.() ?? true)) {
return
}
this.beginReconcile(false)
}
async waitForActiveBroker(): Promise<CoordinatedRelayBroker | null> {
while (!this.stopped) {
const broker = this.getActiveBroker()