fix(mobile-relay): back off relay reconnects to stop cellular connect/disconnect churn (#9460)

* fix(mobile-relay): back off relay reconnects to stop cellular connect/disconnect churn

On cellular, the relay path re-dialed instantly on every network flap: a
NAT rebind / Wi-Fi<->cellular handoff silently kills the socket, the
revival trigger treats it as 'link came back' and calls recoverRelay(),
and the relay cell answers the overlapping resume with PEER_DROPPED (4408)
or LIMIT_EXCEEDED (4429). The session collapsed every close to a plain
'disconnected' and re-dialed with no delay, so the phone ping-ponged
connect/disconnect. The documented recovery contract (mobileRelayRecoveryFor,
which prescribes fullJitter backoff) had no callers.

- Add RelayReconnectBackoff: full-jitter exponential backoff (250ms floor,
  30s ceiling) that debounces re-dials via a cooldown window and wires up
  mobileRelayRecoveryFor. Reset on a successful migrate and on a genuine
  background->foreground transition (not on repeat foreground nudges).
- Extract the lease-rotation timer into RelayLeaseRotationTimer so the
  supervisor stays under max-lines (the direct-probe path can't be split
  out — it shares the operationInFlight mutex with recoverRelay).
- Add a deterministic test: repeated network-flap nudges re-dial instantly
  before the fix and are suppressed by the backoff window after.

* fix(mobile-relay): recover drops during direct probes

* fix(mobile-relay): recover half-open relay sessions

* fix(mobile-relay): preserve direct handshakes

* fix(mobile-relay): keep recovery retries bounded

* fix(mobile-relay): avoid redundant recovery dials

* fix(mobile-relay): keep all retries inside cooldown

* fix(mobile-relay): close recovery lifecycle races

* fix(mobile-relay): preserve in-progress direct auth

* fix(mobile-relay): preserve fatal recovery gates

* fix(mobile-relay): preserve backoff across unstable resumes

* fix(mobile-relay): close remaining recovery lifecycle gaps

* fix(mobile-relay): reset backoff only after stable relay
This commit is contained in:
Brennan Benson
2026-07-21 18:28:16 -07:00
committed by GitHub
parent 51873780c5
commit 261a7b714c
6 changed files with 1278 additions and 95 deletions
@@ -0,0 +1,23 @@
import type { MobileRelayEndpoint } from '../../../src/shared/mobile-relay-credential-contract'
import type { MobileRelayCredentialBundle } from './mobile-relay-credential-bundle'
import type { MobileRelayRpcSession } from './mobile-relay-rpc-session'
import type { resolveMobileRelayEndpoint } from './mobile-relay-resume-director'
import type { RpcClient } from './rpc-client'
import type { HostProfile } from './types'
export type MobileEndpointSupervisorDependencies = {
openDirect: (endpoint: string) => RpcClient
openRelay: (
relay: MobileRelayEndpoint,
credential: { token: string; version: number },
confirmReqId: string
) => MobileRelayRpcSession
resolveRelay: typeof resolveMobileRelayEndpoint
readBundle: (hostId: string) => Promise<MobileRelayCredentialBundle | null>
writeBundle: (bundle: MobileRelayCredentialBundle) => Promise<void>
saveHost: (host: HostProfile) => Promise<void>
now: () => number
randomBytes: (length: number) => Uint8Array
setTimer: typeof setTimeout
clearTimer: typeof clearTimeout
}
@@ -1,5 +1,6 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type { MobileRelayCredentialBundle } from './mobile-relay-credential-bundle'
import { hashMobileRelayCredential } from './mobile-relay-credential-hash'
import { RelayOuterError } from './mobile-relay-e2ee-link'
import type { MobileRelayRpcSession } from './mobile-relay-rpc-session'
import {
@@ -16,7 +17,7 @@ vi.mock('expo-crypto', () => ({ getRandomBytes: (length: number) => new Uint8Arr
class FakeSession implements RpcClient {
readonly sendRequest = vi.fn(
async (): Promise<RpcResponse> => ({
async (_method: string, _params?: unknown): Promise<RpcResponse> => ({
id: 'rpc-1',
ok: true,
result: {},
@@ -83,6 +84,7 @@ class FakeLogicalClient extends FakeSession implements StableLogicalRpcClient {
}
this.path = path
this.generation += 1
this.publishState('connected')
})
suspendActiveSession = vi.fn(() => this.publishState('disconnected'))
getActivePath = () => this.path
@@ -141,6 +143,36 @@ function dependencies(
}
}
function mockCredentialRotation(logical: FakeLogicalClient): void {
let installResult: Record<string, unknown> | null = null
logical.sendRequest.mockImplementation(async (method, params) => {
const request = params as { installReqId?: string; reqId?: string }
if (method === 'pairing.provisionRelay') {
installResult = {
v: 1,
reqId: request.reqId,
authorizationMode: 'authenticated-direct',
currentVersion: 3,
resumeExpiresAt: Date.now() + 300_000,
graceExpiresAt: Date.now() + 60_000
}
return { id: 'rpc-2', ok: true, result: installResult, _meta: { runtimeId: 'runtime-1' } }
}
return {
id: 'rpc-1',
ok: true,
result: {
v: 1,
relay,
installStatus: installResult
? { v: 1, reqId: request.installReqId, state: 'committed', result: installResult }
: { v: 1, reqId: request.installReqId, state: 'not-found' }
},
_meta: { runtimeId: 'runtime-1' }
}
})
}
describe('mobile endpoint supervisor', () => {
beforeEach(() => {
vi.useFakeTimers()
@@ -172,6 +204,14 @@ describe('mobile endpoint supervisor', () => {
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
logical.publishState('handshaking')
await vi.advanceTimersByTimeAsync(0)
expect(deps.openRelay).not.toHaveBeenCalled()
supervisor.setForeground(true)
await vi.advanceTimersByTimeAsync(0)
expect(deps.openRelay).not.toHaveBeenCalled()
logical.publishState('reconnecting')
await vi.waitFor(() => expect(logical.getActivePath()).toBe('relay'))
@@ -191,6 +231,27 @@ describe('mobile endpoint supervisor', () => {
supervisor.stop()
})
it('does not spend a queued relay retry while direct authentication is progressing', async () => {
const logical = new FakeLogicalClient('disconnected', 'lan')
const openRelay = vi.fn(() => new FakeRelaySession('disconnected', new RelayOuterError(4408)))
const deps = dependencies({
openRelay,
randomBytes: () => new Uint8Array([128, 0])
})
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
expect(openRelay).toHaveBeenCalledOnce()
logical.publishState('handshaking')
await vi.advanceTimersByTimeAsync(250)
expect(openRelay).toHaveBeenCalledOnce()
logical.publishState('disconnected')
await vi.waitFor(() => expect(openRelay).toHaveBeenCalledTimes(2))
supervisor.stop()
})
it('uses POST resolve for wrong-cell recovery and persists the authoritative target', async () => {
const logical = new FakeLogicalClient('disconnected', 'lan')
const openRelay = vi
@@ -228,6 +289,613 @@ describe('mobile endpoint supervisor', () => {
supervisor.stop()
})
it('recovers the relay when it drops during an unavailable direct probe', async () => {
const logical = new FakeLogicalClient('connected', 'relay')
const direct = new FakeSession('connecting')
const openRelay = vi.fn(() => new FakeRelaySession('connected'))
const deps = dependencies({
openDirect: vi.fn(() => direct),
openRelay
})
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
// Start the probe, then drop the active relay while the probe owns the
// operation mutex. The failed probe must hand recovery back to the relay.
await vi.advanceTimersByTimeAsync(15_000)
expect(deps.openDirect).toHaveBeenCalledOnce()
logical.publishState('disconnected')
direct.publishState('disconnected')
await vi.waitFor(() => expect(openRelay).toHaveBeenCalledOnce())
supervisor.stop()
})
it('replaces a half-open relay on a network nudge, then backs off failed resumes', async () => {
const logical = new FakeLogicalClient('disconnected', 'lan')
const openRelay = vi
.fn()
.mockReturnValueOnce(new FakeRelaySession('connected'))
.mockImplementation(() => new FakeRelaySession('disconnected', new RelayOuterError(4408)))
const deps = dependencies({
openRelay,
// Keep direct unavailable so relay recovery stays the only path under test.
openDirect: vi.fn(() => new FakeSession('disconnected')),
// Deterministic full jitter: fraction 0.5 → half the backoff window.
randomBytes: () => new Uint8Array([128, 0])
})
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
expect(openRelay).toHaveBeenCalledOnce()
// The OS reports a network handoff, but the dead relay never published onclose.
supervisor.setForeground(true)
await vi.advanceTimersByTimeAsync(0)
expect(logical.suspendActiveSession).toHaveBeenCalledOnce()
expect(openRelay).toHaveBeenCalledTimes(2)
// The relay cell rejects the replacement with PEER_DROPPED; more flap nudges
// must share the existing cooldown rather than opening more sockets.
for (let i = 0; i < 5; i++) {
supervisor.setForeground(true)
await vi.advanceTimersByTimeAsync(0)
}
expect(openRelay).toHaveBeenCalledTimes(2)
// Exactly one retry fires at the 250 ms deterministic backoff boundary.
await vi.advanceTimersByTimeAsync(249)
expect(openRelay).toHaveBeenCalledTimes(2)
await vi.advanceTimersByTimeAsync(1)
expect(openRelay).toHaveBeenCalledTimes(3)
supervisor.stop()
})
it('backs off a close from the active relay before opening its replacement', async () => {
const logical = new FakeLogicalClient('disconnected', 'lan')
const openRelay = vi
.fn()
.mockReturnValueOnce(new FakeRelaySession('connected', new RelayOuterError(4429)))
.mockImplementation(() => new FakeRelaySession('connected'))
const deps = dependencies({
openRelay,
randomBytes: () => new Uint8Array([128, 0])
})
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
logical.publishState('disconnected')
expect(openRelay).toHaveBeenCalledOnce()
await vi.advanceTimersByTimeAsync(249)
expect(openRelay).toHaveBeenCalledOnce()
await vi.advanceTimersByTimeAsync(1)
expect(openRelay).toHaveBeenCalledTimes(2)
supervisor.stop()
})
it('escalates backoff when relay sessions connect and then drop repeatedly', async () => {
const logical = new FakeLogicalClient('disconnected', 'lan')
const openRelay = vi.fn(() => new FakeRelaySession('connected', new RelayOuterError(4408)))
const deps = dependencies({
openRelay,
randomBytes: () => new Uint8Array([128, 0])
})
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
logical.publishState('disconnected')
await vi.advanceTimersByTimeAsync(250)
expect(openRelay).toHaveBeenCalledTimes(2)
logical.publishState('disconnected')
await vi.advanceTimersByTimeAsync(499)
expect(openRelay).toHaveBeenCalledTimes(2)
await vi.advanceTimersByTimeAsync(1)
expect(openRelay).toHaveBeenCalledTimes(3)
logical.publishState('disconnected')
await vi.advanceTimersByTimeAsync(999)
expect(openRelay).toHaveBeenCalledTimes(3)
await vi.advanceTimersByTimeAsync(1)
expect(openRelay).toHaveBeenCalledTimes(4)
supervisor.stop()
})
it('does not try a grace credential for a capacity failure before backing off', async () => {
const logical = new FakeLogicalClient('disconnected', 'lan')
const openRelay = vi.fn(() => new FakeRelaySession('disconnected', new RelayOuterError(4429)))
const deps = dependencies({
readBundle: vi.fn(async () => ({
...bundle,
grace: { ...bundle.current, token: 'C'.repeat(43), hash: 'D'.repeat(43), version: 1 }
})),
openRelay,
randomBytes: () => new Uint8Array([128, 0])
})
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
expect(openRelay).toHaveBeenCalledOnce()
await vi.advanceTimersByTimeAsync(249)
expect(openRelay).toHaveBeenCalledOnce()
await vi.advanceTimersByTimeAsync(1)
expect(openRelay).toHaveBeenCalledTimes(2)
supervisor.stop()
})
it('does not redial a rejected current credential on grace cooldown retries', async () => {
const logical = new FakeLogicalClient('disconnected', 'lan')
const openRelay = vi.fn(
(_relay, credential: { version: number }) =>
new FakeRelaySession(
'disconnected',
new RelayOuterError(credential.version === bundle.current.version ? 4401 : 4429)
)
)
const deps = dependencies({
readBundle: vi.fn(async () => ({
...bundle,
grace: { ...bundle.current, token: 'C'.repeat(43), hash: 'D'.repeat(43), version: 1 }
})),
openRelay,
randomBytes: () => new Uint8Array([128, 0])
})
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
expect(openRelay).toHaveBeenCalledTimes(2)
await vi.advanceTimersByTimeAsync(250)
expect(openRelay).toHaveBeenCalledTimes(3)
expect(openRelay.mock.calls[2]?.[1]).toEqual(expect.objectContaining({ version: 1 }))
supervisor.stop()
})
it('rotates a rejected current credential after grace keeps relay recovery alive', async () => {
const logical = new FakeLogicalClient('disconnected', 'lan')
const current = {
...bundle.current,
hash: hashMobileRelayCredential(bundle.current.token)
}
const writeBundle = vi.fn(async () => {})
const deps = dependencies({
readBundle: vi.fn(async () => ({
...bundle,
current,
grace: { ...current, token: 'C'.repeat(43), hash: 'D'.repeat(43), version: 1 }
})),
openRelay: vi.fn(
(_relay, credential: { version: number }) =>
new FakeRelaySession(
credential.version === current.version ? 'disconnected' : 'connected',
credential.version === current.version ? new RelayOuterError(4401) : null
)
),
writeBundle
})
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
writeBundle.mockClear()
await vi.advanceTimersByTimeAsync(60_000)
expect(writeBundle).toHaveBeenCalledWith(
expect.objectContaining({ pending: expect.any(Object) })
)
supervisor.stop()
})
it('does not duplicate transport failures across current and grace credentials', async () => {
const logical = new FakeLogicalClient('disconnected', 'lan')
const openRelay = vi.fn(() => new FakeRelaySession('disconnected', new Error('network down')))
const resolveRelay = vi.fn(async () => {
throw new Error('director unreachable')
})
const deps = dependencies({
readBundle: vi.fn(async () => ({
...bundle,
grace: { ...bundle.current, token: 'C'.repeat(43), hash: 'D'.repeat(43), version: 1 }
})),
openRelay,
resolveRelay
})
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
expect(openRelay).toHaveBeenCalledOnce()
expect(resolveRelay).toHaveBeenCalledOnce()
supervisor.stop()
})
it('keeps an authenticated relay off the backoff path when persistence fails', async () => {
const logical = new FakeLogicalClient('disconnected', 'lan')
const openRelay = vi.fn(() => new FakeRelaySession('connected'))
const deps = dependencies({
readBundle: vi.fn(async () => ({
...bundle,
grace: { ...bundle.current, token: 'C'.repeat(43), hash: 'D'.repeat(43), version: 1 }
})),
openRelay,
writeBundle: vi.fn(async () => {
throw new Error('secure store unavailable')
})
})
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
expect(openRelay).toHaveBeenCalledOnce()
logical.publishState('disconnected')
await vi.waitFor(() => expect(openRelay).toHaveBeenCalledTimes(2))
supervisor.stop()
})
it('cancels a pending relay retry when the original direct path reconnects', async () => {
const logical = new FakeLogicalClient('disconnected', 'lan')
const deps = dependencies({
openRelay: vi.fn(() => new FakeRelaySession('disconnected', new RelayOuterError(4408))),
randomBytes: () => new Uint8Array([128, 0])
})
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
expect(vi.getTimerCount()).toBe(1)
logical.publishState('connected')
expect(vi.getTimerCount()).toBe(0)
supervisor.stop()
})
it('recovers a relay drop while post-migration persistence owns the mutex', async () => {
const logical = new FakeLogicalClient('disconnected', 'lan')
let finishWrite: (() => void) | undefined
const writePending = new Promise<void>((resolve) => {
finishWrite = resolve
})
const openRelay = vi
.fn()
.mockReturnValueOnce(new FakeRelaySession('connected', new RelayOuterError(4408)))
.mockImplementation(() => new FakeRelaySession('connected'))
const deps = dependencies({
openRelay,
writeBundle: vi.fn(() => writePending),
randomBytes: () => new Uint8Array([128, 0])
})
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
const starting = supervisor.start()
await vi.waitFor(() => expect(deps.writeBundle).toHaveBeenCalledOnce())
logical.publishState('disconnected')
finishWrite?.()
await starting
expect(openRelay).toHaveBeenCalledOnce()
await vi.advanceTimersByTimeAsync(250)
expect(openRelay).toHaveBeenCalledTimes(2)
supervisor.stop()
})
it('waits for an external signal instead of polling a host-offline relay', async () => {
const logical = new FakeLogicalClient('disconnected', 'lan')
const openRelay = vi.fn(() => new FakeRelaySession('disconnected', new RelayOuterError(4404)))
const deps = dependencies({
openRelay,
randomBytes: () => new Uint8Array([128, 0])
})
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
expect(openRelay).toHaveBeenCalledOnce()
logical.publishState('disconnected')
expect(vi.getTimerCount()).toBe(0)
supervisor.setForeground(true)
expect(vi.getTimerCount()).toBe(1)
await vi.advanceTimersByTimeAsync(250)
expect(openRelay).toHaveBeenCalledTimes(2)
expect(vi.getTimerCount()).toBe(0)
supervisor.stop()
})
it('waits for direct connectivity before replacing a rejected relay credential', async () => {
const logical = new FakeLogicalClient('disconnected', 'lan')
const openRelay = vi.fn(() => new FakeRelaySession('disconnected', new RelayOuterError(4401)))
const deps = dependencies({ openRelay })
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
supervisor.setForeground(true)
await vi.advanceTimersByTimeAsync(1000)
expect(openRelay).toHaveBeenCalledOnce()
expect(deps.writeBundle).not.toHaveBeenCalled()
logical.publishState('connected')
await vi.waitFor(() => expect(deps.writeBundle).toHaveBeenCalledOnce())
supervisor.stop()
})
it('keeps rejected relay credentials gated until their replacement is durable', async () => {
const logical = new FakeLogicalClient('disconnected', 'lan')
const openRelay = vi
.fn()
.mockReturnValueOnce(new FakeRelaySession('disconnected', new RelayOuterError(4401)))
.mockImplementation(() => new FakeRelaySession('connected'))
let finishCredentialWrite: (() => void) | undefined
const credentialWritePending = new Promise<void>((resolve) => {
finishCredentialWrite = resolve
})
const writeBundle = vi
.fn<(value: MobileRelayCredentialBundle) => Promise<void>>()
.mockResolvedValue()
.mockResolvedValueOnce()
.mockReturnValueOnce(credentialWritePending)
mockCredentialRotation(logical)
const deps = dependencies({ openRelay, writeBundle })
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
logical.publishState('connected')
await vi.waitFor(() => expect(writeBundle).toHaveBeenCalledTimes(2))
// The direct socket can disappear after the server commits but before the
// replacement credential finishes its durable write.
logical.publishState('disconnected')
await vi.advanceTimersByTimeAsync(0)
expect(openRelay).toHaveBeenCalledOnce()
finishCredentialWrite?.()
await vi.waitFor(() => expect(openRelay).toHaveBeenCalledTimes(2))
expect(openRelay).toHaveBeenLastCalledWith(
relay,
expect.objectContaining({ version: 3 }),
expect.any(String)
)
supervisor.stop()
})
it('uses a scheduled credential rotation that finishes after relay rejection', async () => {
const logical = new FakeLogicalClient('connected', 'lan')
let finishCredentialWrite: (() => void) | undefined
const credentialWritePending = new Promise<void>((resolve) => {
finishCredentialWrite = resolve
})
const writeBundle = vi
.fn<(value: MobileRelayCredentialBundle) => Promise<void>>()
.mockResolvedValue()
.mockResolvedValueOnce()
.mockReturnValueOnce(credentialWritePending)
mockCredentialRotation(logical)
const openRelay = vi.fn(
(_relay, credential: { version: number }) =>
new FakeRelaySession(
credential.version === bundle.current.version ? 'disconnected' : 'connected',
credential.version === bundle.current.version ? new RelayOuterError(4401) : null
)
)
const deps = dependencies({
readBundle: vi.fn(async () => ({
...bundle,
current: { ...bundle.current, expiresAt: Date.now() + 60_000 }
})),
openRelay,
writeBundle
})
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
logical.publishState('connected')
await vi.waitFor(() => expect(writeBundle).toHaveBeenCalledTimes(2))
// The expiring credential can be rejected while its replacement is waiting on SecureStore.
logical.publishState('disconnected')
await vi.waitFor(() => expect(openRelay).toHaveBeenCalledOnce())
finishCredentialWrite?.()
await vi.waitFor(() => expect(openRelay).toHaveBeenCalledTimes(2))
expect(openRelay).toHaveBeenLastCalledWith(
relay,
expect.objectContaining({ version: 3 }),
expect.any(String)
)
supervisor.stop()
})
it('does not open a resolved relay replacement after backgrounding', async () => {
const logical = new FakeLogicalClient('disconnected', 'lan')
let finishResolve: ((value: typeof relay) => void) | undefined
const resolvePending = new Promise<typeof relay>((resolve) => {
finishResolve = resolve
})
const openRelay = vi
.fn()
.mockReturnValueOnce(new FakeRelaySession('disconnected', new RelayOuterError(4409)))
.mockReturnValueOnce(new FakeRelaySession('disconnected', new RelayOuterError(4408)))
const deps = dependencies({
openRelay,
resolveRelay: vi.fn(() => resolvePending)
})
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
const starting = supervisor.start()
await vi.waitFor(() => expect(deps.resolveRelay).toHaveBeenCalledOnce())
supervisor.setForeground(false)
finishResolve?.(relay)
await starting
expect(openRelay).toHaveBeenCalledOnce()
expect(vi.getTimerCount()).toBe(0)
supervisor.stop()
})
it('does not recreate a lease retry after forced replacement is backgrounded', async () => {
const logical = new FakeLogicalClient('disconnected', 'lan')
let finishResolve: ((value: typeof relay) => void) | undefined
const resolvePending = new Promise<typeof relay>((resolve) => {
finishResolve = resolve
})
const openRelay = vi
.fn()
.mockReturnValueOnce(new FakeRelaySession('connected', null, Date.now() + 31_000))
.mockReturnValueOnce(new FakeRelaySession('disconnected', new RelayOuterError(4409)))
.mockReturnValueOnce(new FakeRelaySession('disconnected', new RelayOuterError(4408)))
const deps = dependencies({
openRelay,
resolveRelay: vi.fn(() => resolvePending)
})
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
await vi.advanceTimersByTimeAsync(1000)
await vi.waitFor(() => expect(deps.resolveRelay).toHaveBeenCalledOnce())
supervisor.setForeground(false)
finishResolve?.(relay)
await vi.waitFor(() => expect(deps.saveHost).toHaveBeenCalledOnce())
await vi.advanceTimersByTimeAsync(0)
expect(openRelay).toHaveBeenCalledTimes(2)
expect(vi.getTimerCount()).toBe(0)
supervisor.stop()
})
it('does not recreate a lease timer after stop races relay persistence', async () => {
const logical = new FakeLogicalClient('disconnected', 'lan')
let finishWrite: (() => void) | undefined
const writePending = new Promise<void>((resolve) => {
finishWrite = resolve
})
const deps = dependencies({ writeBundle: vi.fn(() => writePending) })
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
const starting = supervisor.start()
await vi.waitFor(() => expect(deps.writeBundle).toHaveBeenCalledOnce())
supervisor.stop()
finishWrite?.()
await starting
expect(vi.getTimerCount()).toBe(0)
})
it('does not poll a host-offline relay through forced lease retries', async () => {
const logical = new FakeLogicalClient('disconnected', 'lan')
const openRelay = vi
.fn()
.mockReturnValueOnce(new FakeRelaySession('connected', null, Date.now() + 31_000))
.mockImplementation(() => new FakeRelaySession('disconnected', new RelayOuterError(4404)))
const deps = dependencies({ openRelay })
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
await vi.advanceTimersByTimeAsync(1000)
expect(openRelay).toHaveBeenCalledTimes(2)
await vi.advanceTimersByTimeAsync(5000)
expect(openRelay).toHaveBeenCalledTimes(2)
supervisor.stop()
})
it('keeps a fatal lease-replacement gate after the active relay later drops', async () => {
const logical = new FakeLogicalClient('disconnected', 'lan')
const openRelay = vi
.fn()
.mockReturnValueOnce(
new FakeRelaySession('connected', new RelayOuterError(4408), Date.now() + 31_000)
)
.mockReturnValueOnce(new FakeRelaySession('disconnected', new RelayOuterError(4401)))
.mockImplementation(() => new FakeRelaySession('connected'))
const deps = dependencies({
openRelay,
randomBytes: () => new Uint8Array([128, 0])
})
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
await vi.advanceTimersByTimeAsync(1000)
expect(openRelay).toHaveBeenCalledTimes(2)
// The old relay can outlive its rejected lease replacement, then close separately.
logical.publishState('disconnected')
await vi.advanceTimersByTimeAsync(30_000)
expect(openRelay).toHaveBeenCalledTimes(2)
supervisor.stop()
})
it('keeps revival nudges inside a failed lease rotation cooldown', async () => {
const logical = new FakeLogicalClient('disconnected', 'lan')
const openRelay = vi
.fn()
.mockReturnValueOnce(new FakeRelaySession('connected', null, Date.now() + 31_000))
.mockReturnValueOnce(new FakeRelaySession('disconnected', new RelayOuterError(4429)))
.mockImplementation(() => new FakeRelaySession('connected'))
const deps = dependencies({
openRelay,
randomBytes: () => new Uint8Array([128, 0])
})
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
await vi.advanceTimersByTimeAsync(1000)
expect(openRelay).toHaveBeenCalledTimes(2)
supervisor.setForeground(true)
await vi.advanceTimersByTimeAsync(249)
expect(openRelay).toHaveBeenCalledTimes(2)
await vi.advanceTimersByTimeAsync(1)
expect(openRelay).toHaveBeenCalledTimes(3)
supervisor.stop()
})
it('keeps lease rotation inside an active relay failure cooldown', async () => {
const logical = new FakeLogicalClient('disconnected', 'lan')
const openRelay = vi
.fn()
.mockReturnValueOnce(
new FakeRelaySession('connected', new RelayOuterError(4429), Date.now() + 31_000)
)
.mockImplementation(() => new FakeRelaySession('connected'))
const deps = dependencies({
openRelay,
randomBytes: () => new Uint8Array([128, 0])
})
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
await vi.advanceTimersByTimeAsync(900)
logical.publishState('disconnected')
await vi.advanceTimersByTimeAsync(100)
expect(openRelay).toHaveBeenCalledOnce()
await vi.advanceTimersByTimeAsync(150)
expect(openRelay).toHaveBeenCalledTimes(2)
supervisor.stop()
})
it('clears relay backoff on a genuine foreground so the retry is immediate', async () => {
const logical = new FakeLogicalClient('disconnected', 'lan')
const openRelay = vi.fn(() => new FakeRelaySession('disconnected', new RelayOuterError(4408)))
const deps = dependencies({
openRelay,
openDirect: vi.fn(() => new FakeSession('disconnected')),
randomBytes: () => new Uint8Array([128, 0])
})
const supervisor = new MobileEndpointSupervisor(logical, host, deps)
await supervisor.start()
const afterStart = openRelay.mock.calls.length
// Background → foreground is a fresh signal: dial now, not after the cooldown.
supervisor.setForeground(false)
supervisor.setForeground(true)
await vi.advanceTimersByTimeAsync(0)
expect(openRelay.mock.calls.length).toBeGreaterThan(afterStart)
supervisor.stop()
})
it('releases a background relay session and reconnects it on foreground', async () => {
const logical = new FakeLogicalClient('connected', 'relay')
const deps = dependencies()
@@ -237,6 +905,7 @@ describe('mobile endpoint supervisor', () => {
supervisor.setForeground(false)
expect(logical.suspendActiveSession).toHaveBeenCalledOnce()
expect(logical.getState()).toBe('disconnected')
expect(vi.getTimerCount()).toBe(0)
supervisor.setForeground(true)
await vi.waitFor(() => expect(logical.migrateTo).toHaveBeenCalled())
@@ -1,5 +1,7 @@
import type { MobileRelayEndpoint } from '../../../src/shared/mobile-relay-credential-contract'
import { openAuthenticatedDirectEndpoint } from './mobile-direct-endpoint-probe'
import type { MobileEndpointSupervisorDependencies } from './mobile-endpoint-supervisor-contract'
import { RelayReconnectController } from './mobile-relay-reconnect-controller'
import { RelayLeaseRotationTimer } from './mobile-relay-lease-rotation-timer'
import { MobileEndpointHysteresis } from './mobile-endpoint-hysteresis'
import {
encodeBase64Url,
@@ -13,37 +15,17 @@ import {
rotateMobileRelayCredential
} from './mobile-relay-credential-rotation'
import type { MobileRelayCredentialBundle } from './mobile-relay-credential-bundle'
import type { MobileRelayRpcSession } from './mobile-relay-rpc-session'
import { resolveMobileRelayEndpoint } from './mobile-relay-resume-director'
import type { RpcClient } from './rpc-client'
import type { StableLogicalRpcClient } from './stable-logical-rpc-client'
import type { HostProfile } from './types'
export type { MobileEndpointSupervisorDependencies } from './mobile-endpoint-supervisor-contract'
const DIRECT_PROBE_INTERVAL_MS = 15_000
const DIRECT_OBSERVATION_MS = 30_000
const MINIMUM_DWELL_MS = 60_000
const FAILURE_COOLDOWN_MS = 60_000
const LEASE_ROTATION_MARGIN_MS = 30_000
export type MobileEndpointSupervisorDependencies = {
openDirect: (endpoint: string) => RpcClient
openRelay: (
relay: MobileRelayEndpoint,
credential: { token: string; version: number },
confirmReqId: string
) => MobileRelayRpcSession
resolveRelay: typeof resolveMobileRelayEndpoint
readBundle: (hostId: string) => Promise<MobileRelayCredentialBundle | null>
writeBundle: (bundle: MobileRelayCredentialBundle) => Promise<void>
saveHost: (host: HostProfile) => Promise<void>
now: () => number
randomBytes: (length: number) => Uint8Array
setTimer: typeof setTimeout
clearTimer: typeof clearTimeout
}
export class MobileEndpointSupervisor {
private host: HostProfile
private bundle: MobileRelayCredentialBundle | null = null
private stopped = false
private foreground = true
@@ -51,22 +33,27 @@ export class MobileEndpointSupervisor {
private credentialRotationInFlight = false
private relayRotationPending = false
private probeTimer: ReturnType<typeof setTimeout> | null = null
private leaseTimer: ReturnType<typeof setTimeout> | null = null
private unsubscribeState: (() => void) | null = null
private readonly hysteresis: MobileEndpointHysteresis
private readonly relayReconnect: RelayReconnectController
private readonly leaseRotation: RelayLeaseRotationTimer
constructor(
private readonly logical: StableLogicalRpcClient,
host: HostProfile,
private host: HostProfile,
private readonly dependencies: MobileEndpointSupervisorDependencies
) {
this.host = host
this.hysteresis = new MobileEndpointHysteresis(dependencies.now(), {
directSuccessesRequired: 3,
directObservationMs: DIRECT_OBSERVATION_MS,
failureCooldownMs: FAILURE_COOLDOWN_MS,
minimumDwellMs: MINIMUM_DWELL_MS
})
this.relayReconnect = new RelayReconnectController(dependencies, this.recoverRelay.bind(this))
this.leaseRotation = new RelayLeaseRotationTimer(dependencies, () => {
this.relayRotationPending = true
void this.recoverRelay(true)
})
}
async start(): Promise<void> {
@@ -77,21 +64,16 @@ export class MobileEndpointSupervisor {
this.unsubscribeState = this.logical.onStateChange((state) => {
if (state === 'connected') {
if (this.logical.getActivePath() !== 'relay') {
void this.rotateCredentialIfNeeded()
void this.rotateCredentialIfNeeded(this.relayReconnect.resetForDirectConnection())
}
this.scheduleDirectProbe()
} else if (state === 'reconnecting' || state === 'disconnected' || state === 'auth-failed') {
} else {
// Why: the direct client enters reconnecting after its first failed
// dial and may never publish disconnected while its retry loop lives.
void this.recoverRelay()
this.relayReconnect.handleStateFailure(this.logical, state)
}
})
const initialState = this.logical.getState()
if (
initialState === 'reconnecting' ||
initialState === 'disconnected' ||
initialState === 'auth-failed'
) {
if (this.relayReconnect.needsRecovery(this.logical.getState())) {
// Why: the first direct dial can fail while encrypted relay credentials
// are still loading, before the supervisor subscribes to state changes.
await this.recoverRelay()
@@ -101,20 +83,17 @@ export class MobileEndpointSupervisor {
}
setForeground(foreground: boolean): void {
const wasForeground = this.foreground
this.foreground = foreground
if (foreground) {
void this.recoverRelay(this.relayRotationPending)
this.relayReconnect.handleForeground(this.logical, wasForeground)
this.scheduleDirectProbe(0)
} else {
if (this.logical.getActivePath() === 'relay') {
// Why: background phones must not hold billed relay data splices; the
// stable client keeps subscriptions for authenticated foreground replay.
this.logical.suspendActiveSession()
}
if (this.probeTimer) {
this.dependencies.clearTimer(this.probeTimer)
this.probeTimer = null
}
// Why: background phones must not hold billed relay data splices.
this.relayReconnect.suspendActiveRelay(this.logical)
this.clearDirectProbeTimer()
this.relayReconnect.clear()
this.leaseRotation.clear()
}
}
@@ -122,42 +101,64 @@ export class MobileEndpointSupervisor {
this.stopped = true
this.unsubscribeState?.()
this.unsubscribeState = null
if (this.probeTimer) {
this.dependencies.clearTimer(this.probeTimer)
this.probeTimer = null
}
this.clearLeaseTimer()
this.clearDirectProbeTimer()
this.relayReconnect.clear()
this.leaseRotation.clear()
}
private async recoverRelay(forceReplacement = false): Promise<void> {
// Why: connecting/handshaking is live direct progress; a relay dial would race it.
if (
this.stopped ||
!this.foreground ||
this.operationInFlight ||
!this.bundle ||
!this.host.relay ||
(!forceReplacement && this.logical.getState() === 'connected')
(!forceReplacement && !this.relayReconnect.needsRecovery(this.logical.getState()))
) {
return
}
// Why: revival and lease timers can overlap resume failures; one shared cooldown
// prevents PEER_DROPPED/LIMIT_EXCEEDED reconnect churn.
if (this.relayReconnect.shouldDefer()) {
return
}
this.operationInFlight = true
let lastError: Error | null = null
let retryAfterOperation = false
try {
const credentials = [this.bundle.current, this.bundle.grace].filter(
(credential): credential is NonNullable<typeof credential> =>
Boolean(credential && credential.expiresAt > this.dependencies.now())
const credentials = this.relayReconnect.eligibleCredentials(
this.bundle.current,
this.bundle.grace
)
for (const credential of credentials) {
if (await this.tryRelayCredential(credential)) {
const result = await this.tryRelayCredential(credential)
if (result.ok) {
retryAfterOperation = this.logical.getState() !== 'connected'
return
}
lastError = result.error
if (this.relayReconnect.shouldTryGraceAfterRelayFailure(result.error)) {
// Why: a rejected version stays invalid; retry only the grace credential.
this.relayReconnect.recordRejectedCredential(credential.version)
} else {
break
}
}
if (credentials.length > 0) {
// Why: cleanup may happen while a relay dial is awaiting the network;
// record its outcome without recreating a foreground retry timer.
const scheduleRetry = !forceReplacement && this.foreground && !this.stopped
this.relayReconnect.registerFailure(lastError, scheduleRetry)
}
} finally {
this.operationInFlight = false
if (forceReplacement && this.relayRotationPending && !this.stopped && !this.leaseTimer) {
this.leaseTimer = this.dependencies.setTimer(() => {
this.leaseTimer = null
void this.recoverRelay(true)
}, 5000)
if (forceReplacement && this.relayRotationPending && !this.stopped && this.foreground) {
this.leaseRotation.armRetry(this.relayReconnect.retryDelayMs(5000))
}
// Why: the active relay can drop while migration follow-up still owns the mutex.
if (retryAfterOperation && !this.stopped && this.foreground) {
void this.recoverRelay()
}
}
}
@@ -165,13 +166,13 @@ export class MobileEndpointSupervisor {
private async tryRelayCredential(credential: {
token: string
version: number
}): Promise<boolean> {
}): Promise<{ ok: true } | { ok: false; error: Error }> {
const first = await this.openAndMigrateRelay(credential)
if (first.ok) {
return true
return first
}
if (!isDirectorResolutionFailure(first.error) || !this.host.relay) {
return false
return first
}
try {
const resolved = await this.dependencies.resolveRelay({
@@ -179,9 +180,9 @@ export class MobileEndpointSupervisor {
resumeToken: credential.token
})
this.host = await persistRelayHost(this.host, resolved, this.dependencies.saveHost)
return (await this.openAndMigrateRelay(credential)).ok
} catch {
return false
return await this.openAndMigrateRelay(credential)
} catch (error) {
return { ok: false, error: toError(error) }
}
}
@@ -189,7 +190,8 @@ export class MobileEndpointSupervisor {
token: string
version: number
}): Promise<{ ok: true } | { ok: false; error: Error }> {
if (!this.host.relay || !this.bundle) {
// Why: director resolution and grace fallback can finish after background/stop.
if (this.stopped || !this.foreground || !this.host.relay || !this.bundle) {
return { ok: false, error: new Error('relay state missing') }
}
const session = this.dependencies.openRelay(
@@ -199,17 +201,23 @@ export class MobileEndpointSupervisor {
)
try {
await this.logical.migrateTo(session, 'relay')
this.relayReconnect.setActiveSession(session)
if (!this.foreground) {
this.logical.suspendActiveSession()
this.relayReconnect.suspendActiveRelay(this.logical)
}
this.relayRotationPending = false
this.hysteresis.recordMigration(this.dependencies.now())
const confirmation = session.getResumeConfirmation()
if (confirmation) {
this.bundle = applyResumeConfirmation(this.bundle, credential.version, confirmation)
await this.dependencies.writeBundle(this.bundle)
// Why: the relay is already authenticated; a SecureStore failure must
// not open another socket or count against transport recovery backoff.
await this.dependencies.writeBundle(this.bundle).catch(() => {})
}
this.scheduleLeaseRotation(session)
// Why: async persistence can finish after stop/background; never recreate a stale timer.
this.leaseRotation.scheduleFromLease(
this.stopped || !this.foreground ? null : session.getLeaseExpiresAt()
)
this.scheduleDirectProbe()
return { ok: true }
} catch (error) {
@@ -258,30 +266,32 @@ export class MobileEndpointSupervisor {
await this.logical.migrateTo(successful.client, successful.path)
successful = null
this.hysteresis.recordMigration(this.dependencies.now())
this.clearLeaseTimer()
this.leaseRotation.clear()
this.relayRotationPending = false
await this.rotateCredentialIfNeeded()
await this.rotateCredentialIfNeeded(this.relayReconnect.resetForDirectConnection())
} finally {
successful?.client.close()
this.operationInFlight = false
if (this.relayRotationPending) {
void this.recoverRelay(true)
// Why: a relay drop or backoff timer can arrive while the direct probe owns the mutex.
if (this.relayRotationPending || this.logical.getState() !== 'connected') {
void this.recoverRelay(this.relayRotationPending)
}
this.scheduleDirectProbe()
}
}
private async rotateCredentialIfNeeded(): Promise<void> {
private async rotateCredentialIfNeeded(force = false): Promise<void> {
if (
this.stopped ||
this.credentialRotationInFlight ||
!this.bundle ||
this.logical.getActivePath() === 'relay' ||
!mobileRelayCredentialNeedsRotation(this.bundle, this.dependencies.now())
(!force && !mobileRelayCredentialNeedsRotation(this.bundle, this.dependencies.now()))
) {
return
}
this.credentialRotationInFlight = true
let credentialRefreshed = false
try {
const result = await rotateMobileRelayCredential({
client: this.logical,
@@ -290,33 +300,32 @@ export class MobileEndpointSupervisor {
randomBytes: this.dependencies.randomBytes
})
this.bundle = result.bundle
// Why: a scheduled rotation can finish after the old credential enters the rejection gate.
credentialRefreshed = true
this.host = await persistRelayHost(this.host, result.relay, this.dependencies.saveHost)
} catch {
// Why: pending material remains durable; the next authenticated direct
// opportunity must reconcile it before creating another install key.
} finally {
if (credentialRefreshed) {
this.relayReconnect.completeCredentialRefresh()
}
this.credentialRotationInFlight = false
if (
credentialRefreshed &&
!this.stopped &&
this.foreground &&
this.relayReconnect.needsRecovery(this.logical.getState())
) {
void this.recoverRelay()
}
}
}
private scheduleLeaseRotation(session: MobileRelayRpcSession): void {
this.clearLeaseTimer()
const deadline = session.getLeaseExpiresAt()
if (!deadline) {
return
}
const delay = Math.max(1000, deadline - this.dependencies.now() - LEASE_ROTATION_MARGIN_MS)
this.leaseTimer = this.dependencies.setTimer(() => {
this.leaseTimer = null
this.relayRotationPending = true
void this.recoverRelay(true)
}, delay)
}
private clearLeaseTimer(): void {
if (this.leaseTimer) {
this.dependencies.clearTimer(this.leaseTimer)
this.leaseTimer = null
private clearDirectProbeTimer(): void {
if (this.probeTimer) {
this.dependencies.clearTimer(this.probeTimer)
this.probeTimer = null
}
}
}
@@ -0,0 +1,59 @@
// Why: the relay resume lease expires; the phone must proactively re-resume a
// little before the deadline (and retry shortly if a forced rotation didn't land)
// so the session never lapses. Owns the single lease/rotation timer slot.
const LEASE_ROTATION_MARGIN_MS = 30_000
export type RelayLeaseRotationDependencies = {
now: () => number
setTimer: typeof setTimeout
clearTimer: typeof clearTimeout
}
export class RelayLeaseRotationTimer {
private timer: ReturnType<typeof setTimeout> | null = null
constructor(
private readonly dependencies: RelayLeaseRotationDependencies,
private readonly onRotate: () => void
) {}
// Arm rotation a margin before the lease deadline. No-op with no deadline.
scheduleFromLease(leaseExpiresAt: number | null): void {
this.clear()
if (!leaseExpiresAt) {
return
}
const delay = Math.max(
1000,
leaseExpiresAt - this.dependencies.now() - LEASE_ROTATION_MARGIN_MS
)
this.arm(delay)
}
// Re-arm a short retry when a forced rotation's recovery did not complete.
// Ignored while a timer is already pending so retries never stack.
armRetry(delayMs: number | null): void {
if (delayMs == null || this.timer) {
return
}
this.arm(delayMs)
}
get pending(): boolean {
return this.timer != null
}
clear(): void {
if (this.timer) {
this.dependencies.clearTimer(this.timer)
this.timer = null
}
}
private arm(delayMs: number): void {
this.timer = this.dependencies.setTimer(() => {
this.timer = null
this.onRotate()
}, delayMs)
}
}
@@ -0,0 +1,147 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import type { MobileRelayRpcSession } from './mobile-relay-rpc-session'
import { MobileE2EEAuthenticationError } from './mobile-e2ee-v2-physical-channel'
import { RelayOuterError } from './mobile-relay-e2ee-link'
import { RelayReconnectController } from './mobile-relay-reconnect-controller'
import type { StableLogicalRpcClient } from './stable-logical-rpc-client'
vi.mock('react-native', () => ({ Platform: { OS: 'ios' } }))
vi.mock('expo-crypto', () => ({ getRandomBytes: (length: number) => new Uint8Array(length) }))
describe('relay reconnect controller', () => {
beforeEach(() => {
vi.useFakeTimers()
})
afterEach(() => {
vi.useRealTimers()
})
it('keeps one retry timer and cancels it when recovery needs an external signal', () => {
const onRetry = vi.fn()
const reconnect = createController(onRetry)
reconnect.registerFailure(new RelayOuterError(4429))
reconnect.registerFailure(new RelayOuterError(4408))
expect(vi.getTimerCount()).toBe(1)
reconnect.registerFailure(new RelayOuterError(4404))
expect(vi.getTimerCount()).toBe(0)
expect(reconnect.shouldDefer()).toBe(true)
expect(vi.getTimerCount()).toBe(0)
vi.runAllTimers()
expect(onRetry).not.toHaveBeenCalled()
})
it('drops a pending relay retry after direct connectivity wins', () => {
const onRetry = vi.fn()
const reconnect = createController(onRetry)
reconnect.registerFailure(new RelayOuterError(4408))
expect(vi.getTimerCount()).toBe(1)
reconnect.resetForDirectConnection()
expect(vi.getTimerCount()).toBe(0)
vi.runAllTimers()
expect(onRetry).not.toHaveBeenCalled()
})
it('waits for an external signal after rejected E2EE authentication', () => {
const onRetry = vi.fn()
const reconnect = createController(onRetry)
reconnect.registerFailure(new MobileE2EEAuthenticationError())
expect(reconnect.shouldDefer()).toBe(true)
expect(vi.getTimerCount()).toBe(0)
vi.advanceTimersByTime(60_000)
expect(onRetry).not.toHaveBeenCalled()
})
it('upgrades host-revival gating to fresh credentials without later downgrading it', () => {
const reconnect = createController(vi.fn())
reconnect.registerFailure(new RelayOuterError(4404))
reconnect.registerFailure(new RelayOuterError(4401))
reconnect.registerFailure(new RelayOuterError(4408))
expect(reconnect.resetForDirectConnection()).toBe(true)
expect(vi.getTimerCount()).toBe(0)
})
it('extends forced-rotation retries to the exponential cooldown', () => {
const reconnect = createController(vi.fn())
for (let failure = 0; failure < 6; failure++) {
reconnect.registerFailure(new RelayOuterError(4429), false)
}
expect(reconnect.retryDelayMs(5000)).toBe(8000)
expect(vi.getTimerCount()).toBe(0)
})
it('does not reset backoff merely because a failed attempt took one ceiling', () => {
const onRetry = vi.fn()
const reconnect = createController(onRetry)
reconnect.registerFailure(new RelayOuterError(4429))
vi.advanceTimersByTime(250)
expect(onRetry).toHaveBeenCalledOnce()
vi.advanceTimersByTime(30_000)
reconnect.registerFailure(new RelayOuterError(4429))
vi.advanceTimersByTime(249)
expect(onRetry).toHaveBeenCalledOnce()
vi.advanceTimersByTime(1)
expect(onRetry).toHaveBeenCalledOnce()
vi.advanceTimersByTime(249)
expect(onRetry).toHaveBeenCalledOnce()
vi.advanceTimersByTime(1)
expect(onRetry).toHaveBeenCalledTimes(2)
})
it('resets backoff after an authenticated relay remains stable', () => {
const onRetry = vi.fn()
const reconnect = createController(onRetry)
const session = {
getFailure: () => new RelayOuterError(4408)
} as MobileRelayRpcSession
const logical = {
getActivePath: () => 'relay'
} as StableLogicalRpcClient
reconnect.registerFailure(new RelayOuterError(4429))
vi.advanceTimersByTime(250)
reconnect.setActiveSession(session)
vi.advanceTimersByTime(30_000)
reconnect.registerActiveFailure(logical)
vi.advanceTimersByTime(249)
expect(onRetry).toHaveBeenCalledOnce()
vi.advanceTimersByTime(1)
expect(onRetry).toHaveBeenCalledTimes(2)
})
it('uses grace only when the outer relay credential was rejected', () => {
const reconnect = createController(vi.fn())
expect(reconnect.shouldTryGraceAfterRelayFailure(new RelayOuterError(4401))).toBe(true)
expect(reconnect.shouldTryGraceAfterRelayFailure(new Error('relay transport error'))).toBe(
false
)
expect(reconnect.shouldTryGraceAfterRelayFailure(new RelayOuterError(4408))).toBe(false)
expect(reconnect.shouldTryGraceAfterRelayFailure(new RelayOuterError(4429))).toBe(false)
})
})
function createController(onRetry: () => void): RelayReconnectController {
return new RelayReconnectController(
{
now: Date.now,
randomBytes: () => new Uint8Array([128, 0]),
setTimer: setTimeout,
clearTimer: clearTimeout
},
onRetry
)
}
@@ -0,0 +1,276 @@
import {
isMobileRelayCloseCode,
MOBILE_RELAY_CLOSE_CODE,
mobileRelayRecoveryFor
} from '../../../src/shared/mobile-relay-close-codes'
import type { MobileRelayRpcSession } from './mobile-relay-rpc-session'
import { MobileE2EEAuthenticationError } from './mobile-e2ee-v2-physical-channel'
import { RelayOuterError } from './mobile-relay-e2ee-link'
import type { StableLogicalRpcClient } from './stable-logical-rpc-client'
import type { ConnectionState } from './types'
// Why: relay resume closes and silent cellular NAT rebinds otherwise cause
// immediate re-dials that ping-pong the phone between connected and disconnected.
const RELAY_BACKOFF_MIN_MS = 250
const RELAY_BACKOFF_BASE_MS = 500
const RELAY_BACKOFF_CEILING_MS = 30_000
const RELAY_STABLE_CONNECTION_MS = RELAY_BACKOFF_CEILING_MS
export type RelayReconnectDependencies = {
now: () => number
randomBytes: (length: number) => Uint8Array
setTimer: typeof setTimeout
clearTimer: typeof clearTimeout
}
type RecoveryGate = 'external-signal' | 'fresh-credential'
export class RelayReconnectController {
private consecutiveFailures = 0
private activeRelayConnectedAt: number | null = null
private nextAttemptAt = 0
private timer: ReturnType<typeof setTimeout> | null = null
private activeSession: MobileRelayRpcSession | null = null
private recoveryGate: RecoveryGate | null = null
private readonly rejectedCredentialVersions = new Set<number>()
constructor(
private readonly dependencies: RelayReconnectDependencies,
private readonly onRetry: (forceReplacement?: boolean) => void
) {}
handleForeground(logical: StableLogicalRpcClient, wasForeground: boolean): void {
if (!wasForeground) {
// Why: an app resume is a fresh signal, unlike repeated network-flap nudges.
if (this.recoveryGate !== 'fresh-credential') {
this.reset()
}
} else if (this.recoveryGate === 'external-signal') {
this.recoveryGate = null
}
if (
wasForeground &&
this.recoveryGate !== 'fresh-credential' &&
logical.getState() === 'connected'
) {
// Why: a network handoff can leave the relay half-open without publishing a close.
this.suspendActiveRelay(logical)
}
// Why: revival nudges must honor failure cooldowns even when lease rotation is pending.
this.onRetry()
}
handleStateFailure(logical: StableLogicalRpcClient, state: ConnectionState): void {
if (!this.needsRecovery(state)) {
return
}
this.registerActiveFailure(logical)
this.onRetry()
}
needsRecovery(state: ConnectionState): boolean {
return state !== 'connected' && state !== 'connecting' && state !== 'handshaking'
}
suspendActiveRelay(logical: StableLogicalRpcClient): void {
if (logical.getActivePath() !== 'relay') {
return
}
this.activeSession = null
this.activeRelayConnectedAt = null
logical.suspendActiveSession()
}
setActiveSession(session: MobileRelayRpcSession): void {
this.activeSession = session
this.activeRelayConnectedAt = this.dependencies.now()
this.nextAttemptAt = 0
this.recoveryGate = null
this.clearTimer()
}
resetForDirectConnection(): boolean {
const needsCredentialRefresh =
this.recoveryGate === 'fresh-credential' || this.rejectedCredentialVersions.size > 0
this.activeSession = null
this.activeRelayConnectedAt = null
if (needsCredentialRefresh) {
// Why: the rejected credential stays unusable until its replacement is durable.
this.consecutiveFailures = 0
this.nextAttemptAt = 0
this.recoveryGate = 'fresh-credential'
this.clearTimer()
} else {
this.reset()
}
return needsCredentialRefresh
}
completeCredentialRefresh(): void {
if (this.recoveryGate === 'fresh-credential') {
this.rejectedCredentialVersions.clear()
this.reset()
}
}
eligibleCredentials<T extends { expiresAt: number; version: number }>(
...credentials: Array<T | null | undefined>
): T[] {
const eligible = credentials.filter((credential): credential is T =>
Boolean(
credential &&
credential.expiresAt > this.dependencies.now() &&
!this.rejectedCredentialVersions.has(credential.version)
)
)
if (eligible.length === 0 && this.rejectedCredentialVersions.size > 0) {
this.recoveryGate = 'fresh-credential'
this.clearTimer()
}
return eligible
}
recordRejectedCredential(version: number): void {
this.rejectedCredentialVersions.add(version)
}
registerActiveFailure(logical: StableLogicalRpcClient): void {
if (logical.getActivePath() !== 'relay') {
return
}
const failure = this.activeSession?.getFailure()
this.activeSession = null
if (failure) {
// Why: active relay closes need the same cooldown as failed replacement dials.
this.registerFailure(failure)
} else {
this.activeRelayConnectedAt = null
}
}
// True when the caller is still inside the cooldown window and must not
// re-dial. Arms the self-scheduled retry so recovery still happens on its own.
shouldDefer(): boolean {
if (this.recoveryGate) {
return true
}
if (this.dependencies.now() < this.nextAttemptAt) {
this.scheduleRetry()
return true
}
return false
}
registerFailure(error: Error | null, scheduleRetry = true): void {
const code = error instanceof RelayOuterError ? error.code : null
const recovery =
code != null && isMobileRelayCloseCode(code)
? mobileRelayRecoveryFor(code, 'phone-resume')
: null
if (
this.recoveryGate === 'fresh-credential' ||
(this.recoveryGate === 'external-signal' && recovery?.kind !== 'disable-relay-credential')
) {
// Why: only the gate's external signal can make a known-fatal recovery retryable.
this.clearTimer()
return
}
const now = this.dependencies.now()
if (
this.activeRelayConnectedAt != null &&
now - this.activeRelayConnectedAt >= RELAY_STABLE_CONNECTION_MS
) {
this.consecutiveFailures = 0
}
// Why: elapsed time inside a slow failed dial is not evidence of recovery;
// only an authenticated relay that survived the stability window resets the streak.
this.activeRelayConnectedAt = null
this.consecutiveFailures += 1
const delay = this.delayMs()
this.nextAttemptAt = now + delay
if (error instanceof MobileE2EEAuthenticationError) {
// Why: pairing state cannot change on a timer; polling only wakes the radio.
this.recoveryGate = 'external-signal'
this.clearTimer()
return
}
if (recovery?.kind === 'wait-for-host-revival') {
// Why: retrying HOST_OFFLINE without a revival signal is polling a known-negative state.
this.recoveryGate = 'external-signal'
this.clearTimer()
return
}
if (recovery?.kind === 'disable-relay-credential') {
// Why: a rejected outer credential cannot recover until direct connectivity refreshes it.
this.recoveryGate = 'fresh-credential'
this.clearTimer()
return
}
this.recoveryGate = null
if (!scheduleRetry) {
this.clearTimer()
return
}
this.scheduleRetry(delay)
}
shouldTryGraceAfterRelayFailure(error: Error): boolean {
// Why: only a rejected outer credential can be repaired by the grace token;
// retrying session/capacity close codes immediately recreates relay churn.
return (
error instanceof RelayOuterError &&
error.code === MOBILE_RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL
)
}
retryDelayMs(minimumMs: number): number | null {
if (this.recoveryGate) {
return null
}
return Math.max(minimumMs, this.nextAttemptAt - this.dependencies.now())
}
reset(): void {
this.consecutiveFailures = 0
this.activeRelayConnectedAt = null
this.nextAttemptAt = 0
this.recoveryGate = null
this.clearTimer()
}
clear(): void {
this.clearTimer()
this.activeSession = null
this.activeRelayConnectedAt = null
}
private clearTimer(): void {
if (this.timer) {
this.dependencies.clearTimer(this.timer)
this.timer = null
}
}
private scheduleRetry(delayMs?: number): void {
if (this.timer) {
return
}
const delay = delayMs ?? Math.max(0, this.nextAttemptAt - this.dependencies.now())
this.timer = this.dependencies.setTimer(() => {
this.timer = null
this.onRetry()
}, delay)
}
private delayMs(): number {
const exponent = Math.max(0, this.consecutiveFailures - 1)
const cap = Math.min(RELAY_BACKOFF_CEILING_MS, RELAY_BACKOFF_BASE_MS * 2 ** exponent)
// Full jitter (uniform in [0, cap)), floored so retries never busy-loop.
return Math.max(RELAY_BACKOFF_MIN_MS, Math.floor(cap * this.jitterFraction()))
}
private jitterFraction(): number {
const [high, low] = this.dependencies.randomBytes(2)
return (((high ?? 0) << 8) | (low ?? 0)) / 0x1_00_00
}
}