diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 03e7bc23c0f..f5200abb1a5 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "updatedAt": "2026-07-22", + "updatedAt": "2026-07-25", "policy": { "maturityLevels": [ "experimental", @@ -16,6 +16,131 @@ } }, "gates": [ + { + "id": "mobile-relay.endpoint-recovery", + "title": "Mobile relay recovery retries offline hosts and races direct endpoints", + "maturity": "experimental", + "protection": "partial", + "owner": "mobile-runtime", + "layer": "shared-mobile-transport-contract", + "surfaces": [ + "paired mobile reconnect", + "cloud relay host-offline recovery", + "LAN direct endpoint", + "Tailscale direct endpoint" + ], + "platforms": [ + "ios", + "android", + "macos", + "linux", + "windows" + ], + "providers": [ + "lan", + "tailscale", + "cloud-relay" + ], + "coveredPlatforms": [ + "macos" + ], + "coveredProviders": [ + "lan", + "tailscale", + "cloud-relay" + ], + "coverageNotes": "Deterministic TypeScript tests cover shared close-code policy, foreground retry timers, direct-winner cancellation, and concurrent LAN/Tailscale authentication. Physical iOS/Android radios, GFE, and production relay recovery remain live-test gaps.", + "motivatingLinks": [ + "https://github.com/stablyai/orca-cloud/pull/96" + ], + "invariant": "A foregrounded paired phone must recover from relay HOST_OFFLINE without a foreground or network-change signal, while direct recovery must select the first authenticated configured LAN or Tailscale endpoint without serial timeout delays. Backgrounding, direct success, or stop must cancel pending work, and losing probes must close without affecting the winner.", + "oracle": "Inject deterministic relay close codes, random bytes, fake timers, and independently controlled direct clients. Require HOST_OFFLINE to replace any faster transport timer with one 5-15 second retry, require no retry before the selected delay, race all unique non-relay endpoints, select the first authenticated path, close every loser exactly once, and retain no retry after direct connectivity wins.", + "commands": [ + "pnpm --dir mobile exec vitest run --root .. mobile/src/transport/mobile-direct-endpoint-probe.test.ts mobile/src/transport/mobile-relay-reconnect-controller.test.ts mobile/src/transport/mobile-endpoint-supervisor.test.ts", + "pnpm exec vitest run --config config/vitest.config.ts src/shared/mobile-relay-close-codes.test.ts --reporter=dot" + ], + "testFiles": [ + "mobile/src/transport/mobile-direct-endpoint-probe.test.ts", + "mobile/src/transport/mobile-relay-reconnect-controller.test.ts", + "mobile/src/transport/mobile-endpoint-supervisor.test.ts", + "src/shared/mobile-relay-close-codes.test.ts" + ], + "assertionRefs": [ + { + "file": "mobile/src/transport/mobile-direct-endpoint-probe.test.ts", + "assertions": [ + "a reachable Tailscale endpoint authenticates without waiting for a stale primary LAN timeout", + "the stale direct candidate closes while the authenticated winner stays open" + ] + }, + { + "file": "mobile/src/transport/mobile-relay-reconnect-controller.test.ts", + "assertions": [ + "HOST_OFFLINE replaces a pending capacity retry with the bounded host-offline delay", + "direct connectivity cancels the pending relay retry" + ] + }, + { + "file": "mobile/src/transport/mobile-endpoint-supervisor.test.ts", + "assertions": [ + "a foregrounded supervisor retries HOST_OFFLINE without an external lifecycle signal" + ] + }, + { + "file": "src/shared/mobile-relay-close-codes.test.ts", + "assertions": [ + "HOST_OFFLINE maps to self-healing full-jitter recovery" + ] + } + ], + "evidenceRuns": [ + { + "date": "2026-07-25", + "runner": "local", + "platform": "macos", + "command": "pnpm --dir mobile exec vitest run --root .. mobile/src/transport/mobile-direct-endpoint-probe.test.ts mobile/src/transport/mobile-relay-reconnect-controller.test.ts mobile/src/transport/mobile-endpoint-supervisor.test.ts", + "result": "passed", + "durationSeconds": 0.89, + "summary": "Three focused mobile transport files passed with 39 assertions." + }, + { + "date": "2026-07-25", + "runner": "local", + "platform": "macos", + "command": "pnpm exec vitest run --config config/vitest.config.ts src/shared/mobile-relay-close-codes.test.ts --reporter=dot", + "result": "passed", + "durationSeconds": 0.19, + "summary": "The shared close-code contract passed with five assertions." + } + ], + "runtimeBudget": { + "p95Seconds": 5, + "scope": "focused shared and mobile transport unit tests" + }, + "flakeHistory": { + "status": "unknown", + "evidence": "Two deterministic local runs exist; CI and soak history are not yet available." + }, + "redGreenEvidence": { + "status": "complete", + "evidence": "The prior external-signal HOST_OFFLINE policy fails the retry oracle, and the prior serial direct probe fails the first-authenticated-endpoint timing oracle. Both pass with the candidate behavior." + }, + "performanceBudget": { + "required": true, + "evidence": "All configured direct candidates start in one turn, the first authenticated candidate wins after 100 ms in the deterministic test, and every losing client is closed. A physical-device radio and battery budget is still required before promotion." + }, + "promotionCriteria": [ + "Collect 100 consecutive CI passes or 14 days of soak history.", + "Run paired iOS and Android recovery through production-like GFE HOST_OFFLINE responses.", + "Measure reconnect radio and battery impact for the 5-15 second foreground retry window." + ], + "knownGaps": [ + "No physical iOS or Android device was exercised.", + "The deterministic transport seam does not measure production GFE, carrier NAT, DNS, TLS, or Cloud SQL latency.", + "Background-to-foreground recovery remains covered by existing supervisor tests but lacks a physical sleep/wake run." + ], + "demotionRule": "Keep experimental or demote if focused tests flake, HOST_OFFLINE can park indefinitely, direct probes serialize configured endpoints, loser cleanup leaks clients, or physical-device radio cost exceeds the measured budget." + }, { "id": "git-worktree.refresh-event-semantics", "title": "Index-only Git metadata cannot trigger structural worktree refresh fanout", diff --git a/mobile/src/files/mobile-diff-image-preview.test.ts b/mobile/src/files/mobile-diff-image-preview.test.ts index e63f532b28d..46edf681228 100644 --- a/mobile/src/files/mobile-diff-image-preview.test.ts +++ b/mobile/src/files/mobile-diff-image-preview.test.ts @@ -1,17 +1,20 @@ import { describe, expect, it } from 'vitest' import { mobileDiffImageDataUri } from './mobile-diff-image-preview' +const PNG_BASE64 = + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAFgwJ/l8sm7wAAAABJRU5ErkJggg==' + describe('mobileDiffImageDataUri', () => { it('renders a modified image diff from the post-change bytes', () => { expect( mobileDiffImageDataUri({ kind: 'binary', - originalContent: 'b2xk', - modifiedContent: 'bmV3', + originalContent: PNG_BASE64, + modifiedContent: PNG_BASE64, isImage: true, mimeType: 'image/png' }) - ).toBe('data:image/png;base64,bmV3') + ).toBe(`data:image/png;base64,${PNG_BASE64}`) }) it('renders an added image diff (no original) from the modified bytes', () => { @@ -19,26 +22,26 @@ describe('mobileDiffImageDataUri', () => { mobileDiffImageDataUri({ kind: 'binary', originalContent: '', - modifiedContent: 'bmV3', + modifiedContent: PNG_BASE64, isImage: true, mimeType: 'image/png' }) - ).toBe('data:image/png;base64,bmV3') + ).toBe(`data:image/png;base64,${PNG_BASE64}`) }) it('falls back to the original bytes for a proven deletion (modifiedDeleted)', () => { expect( mobileDiffImageDataUri({ kind: 'binary', - originalContent: 'b2xk', + originalContent: PNG_BASE64, originalIsBinary: true, modifiedContent: '', modifiedIsBinary: false, modifiedDeleted: true, isImage: true, - mimeType: 'image/jpeg' + mimeType: 'image/png' }) - ).toBe('data:image/jpeg;base64,b2xk') + ).toBe(`data:image/png;base64,${PNG_BASE64}`) }) // The reviewer's read-failure case: a relay/SSH read returns an empty modified diff --git a/mobile/src/files/mobile-file-tab-doc.test.ts b/mobile/src/files/mobile-file-tab-doc.test.ts index a9570e6cf44..06ee3510c05 100644 --- a/mobile/src/files/mobile-file-tab-doc.test.ts +++ b/mobile/src/files/mobile-file-tab-doc.test.ts @@ -2,6 +2,9 @@ import { describe, expect, it } from 'vitest' import type { RpcResponse } from '../transport/types' import { resolveMobileFileTabDoc } from './mobile-file-tab-doc' +const PNG_BASE64 = + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAFgwJ/l8sm7wAAAABJRU5ErkJggg==' + function ok(result: unknown): RpcResponse { return { id: 'x', ok: true, result, _meta: { runtimeId: 'r' } } } @@ -49,8 +52,8 @@ describe('resolveMobileFileTabDoc', () => { const client = clientOf({ 'git.diff': ok({ kind: 'binary', - originalContent: 'b2xk', - modifiedContent: 'bmV3', + originalContent: PNG_BASE64, + modifiedContent: PNG_BASE64, modifiedIsBinary: true, isImage: true, mimeType: 'image/png' @@ -61,7 +64,11 @@ describe('resolveMobileFileTabDoc', () => { relativePath: 'm1.png', diffSource: 'unstaged' }) - expect(doc).toEqual({ status: 'ready', kind: 'image', dataUri: 'data:image/png;base64,bmV3' }) + expect(doc).toEqual({ + status: 'ready', + kind: 'image', + dataUri: `data:image/png;base64,${PNG_BASE64}` + }) }) it('throws binary_file for an image modify whose bytes are empty (no stale fallback)', async () => { @@ -89,10 +96,14 @@ describe('resolveMobileFileTabDoc', () => { it('renders a live image preview via files.readPreview', async () => { const client = clientOf({ - 'files.readPreview': ok({ content: 'bmV3', isImage: true, mimeType: 'image/png' }) + 'files.readPreview': ok({ content: PNG_BASE64, isImage: true, mimeType: 'image/png' }) }) const doc = await resolveMobileFileTabDoc(client, { ...WT, relativePath: 'logo.png' }) - expect(doc).toEqual({ status: 'ready', kind: 'image', dataUri: 'data:image/png;base64,bmV3' }) + expect(doc).toEqual({ + status: 'ready', + kind: 'image', + dataUri: `data:image/png;base64,${PNG_BASE64}` + }) expect(client.calls).toEqual(['files.readPreview']) }) diff --git a/mobile/src/transport/mobile-direct-endpoint-probe.test.ts b/mobile/src/transport/mobile-direct-endpoint-probe.test.ts new file mode 100644 index 00000000000..3afa8c08aac --- /dev/null +++ b/mobile/src/transport/mobile-direct-endpoint-probe.test.ts @@ -0,0 +1,76 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { RpcClient } from './rpc-client' +import { openAuthenticatedDirectEndpoint } from './mobile-direct-endpoint-probe' +import type { ConnectionState, HostProfile, RpcResponse } from './types' + +class FakeClient implements RpcClient { + readonly sendRequest = vi.fn( + async (): Promise => ({ + id: 'rpc-1', + ok: true, + result: {}, + _meta: { runtimeId: 'runtime-1' } + }) + ) + readonly subscribe = vi.fn(() => () => {}) + readonly updateTerminalSubscriptionViewport = vi.fn() + readonly notifyForeground = vi.fn() + readonly close = vi.fn(() => this.publishState('disconnected')) + private readonly listeners = new Set<(state: ConnectionState) => void>() + + constructor(private state: ConnectionState) {} + + getState = () => this.state + getReconnectAttempt = () => 0 + getLastConnectedAt = () => null + onStateChange = (listener: (state: ConnectionState) => void) => { + this.listeners.add(listener) + return () => this.listeners.delete(listener) + } + + publishState(state: ConnectionState): void { + this.state = state + for (const listener of this.listeners) { + listener(state) + } + } +} + +const host: HostProfile = { + id: 'host-1', + name: 'Blue Whale', + endpoint: 'ws://192.168.1.10:6768', + deviceToken: 'device-token', + publicKeyB64: 'A'.repeat(44), + lastConnected: 1, + endpoints: [ + { id: 'lan', kind: 'lan', url: 'ws://192.168.1.10:6768' }, + { id: 'tailscale', kind: 'tailscale', url: 'ws://100.64.0.2:6768' } + ] +} + +describe('mobile direct endpoint probe', () => { + beforeEach(() => vi.useFakeTimers()) + afterEach(() => vi.useRealTimers()) + + it('uses the first authenticated candidate without waiting for a stale primary', async () => { + const clients = new Map() + const openDirect = vi.fn((endpoint: string) => { + const client = new FakeClient('connecting') + clients.set(endpoint, client) + if (endpoint.includes('100.64.0.2')) { + setTimeout(() => client.publishState('connected'), 100) + } + return client + }) + + const probing = openAuthenticatedDirectEndpoint(host, openDirect, 12_000) + await vi.advanceTimersByTimeAsync(100) + const result = await probing + + expect(result?.path).toBe('tailscale') + expect(openDirect).toHaveBeenCalledTimes(2) + expect(clients.get(host.endpoint)?.close).toHaveBeenCalledOnce() + expect(result?.client.close).not.toHaveBeenCalled() + }) +}) diff --git a/mobile/src/transport/mobile-direct-endpoint-probe.ts b/mobile/src/transport/mobile-direct-endpoint-probe.ts index 99bd4e858f6..114a4f29130 100644 --- a/mobile/src/transport/mobile-direct-endpoint-probe.ts +++ b/mobile/src/transport/mobile-direct-endpoint-probe.ts @@ -58,14 +58,49 @@ export async function openAuthenticatedDirectEndpoint( openDirect: (endpoint: string) => RpcClient, timeoutMs: number ): Promise<{ client: RpcClient; path: Exclude } | null> { - for (const endpoint of directEndpointUrls(host)) { - const client = openDirect(endpoint) - try { - await waitForAuthenticatedSession(client, timeoutMs) - return { client, path: directPathForEndpoint(host, endpoint) } - } catch { - client.close() + const endpoints = directEndpointUrls(host) + return await new Promise((resolve) => { + const clients = new Set() + let remaining = endpoints.length + let settled = false + const rejectCandidate = (): void => { + remaining-- + if (!settled && remaining === 0) { + settled = true + resolve(null) + } } - } - return null + for (const endpoint of endpoints) { + let client: RpcClient + try { + client = openDirect(endpoint) + } catch { + rejectCandidate() + continue + } + clients.add(client) + void waitForAuthenticatedSession(client, timeoutMs).then( + () => { + if (settled) { + client.close() + return + } + settled = true + for (const candidate of clients) { + if (candidate !== client) { + candidate.close() + } + } + resolve({ client, path: directPathForEndpoint(host, endpoint) }) + }, + () => { + if (settled) { + return + } + client.close() + rejectCandidate() + } + ) + } + }) } diff --git a/mobile/src/transport/mobile-endpoint-supervisor.test.ts b/mobile/src/transport/mobile-endpoint-supervisor.test.ts index 0bfe75d3b73..3ec05f5ef1e 100644 --- a/mobile/src/transport/mobile-endpoint-supervisor.test.ts +++ b/mobile/src/transport/mobile-endpoint-supervisor.test.ts @@ -578,7 +578,7 @@ describe('mobile endpoint supervisor', () => { supervisor.stop() }) - it('waits for an external signal instead of polling a host-offline relay', async () => { + it('retries a host-offline relay without requiring an external signal', async () => { const logical = new FakeLogicalClient('disconnected', 'lan') const openRelay = vi.fn(() => new FakeRelaySession('disconnected', new RelayOuterError(4404))) const deps = dependencies({ @@ -591,13 +591,12 @@ describe('mobile endpoint supervisor', () => { expect(openRelay).toHaveBeenCalledOnce() logical.publishState('disconnected') - expect(vi.getTimerCount()).toBe(0) - - supervisor.setForeground(true) expect(vi.getTimerCount()).toBe(1) - await vi.advanceTimersByTimeAsync(250) + await vi.advanceTimersByTimeAsync(9_999) + expect(openRelay).toHaveBeenCalledOnce() + await vi.advanceTimersByTimeAsync(1) expect(openRelay).toHaveBeenCalledTimes(2) - expect(vi.getTimerCount()).toBe(0) + expect(vi.getTimerCount()).toBe(1) supervisor.stop() }) diff --git a/mobile/src/transport/mobile-relay-reconnect-controller.test.ts b/mobile/src/transport/mobile-relay-reconnect-controller.test.ts index c68ed48a4f8..d630f0db68b 100644 --- a/mobile/src/transport/mobile-relay-reconnect-controller.test.ts +++ b/mobile/src/transport/mobile-relay-reconnect-controller.test.ts @@ -17,7 +17,7 @@ describe('relay reconnect controller', () => { vi.useRealTimers() }) - it('keeps one retry timer and cancels it when recovery needs an external signal', () => { + it('keeps one retry timer when recovery changes from capacity to host offline', () => { const onRetry = vi.fn() const reconnect = createController(onRetry) @@ -26,11 +26,12 @@ describe('relay reconnect controller', () => { expect(vi.getTimerCount()).toBe(1) reconnect.registerFailure(new RelayOuterError(4404)) - expect(vi.getTimerCount()).toBe(0) + expect(vi.getTimerCount()).toBe(1) expect(reconnect.shouldDefer()).toBe(true) - expect(vi.getTimerCount()).toBe(0) - vi.runAllTimers() + vi.advanceTimersByTime(9_999) expect(onRetry).not.toHaveBeenCalled() + vi.advanceTimersByTime(1) + expect(onRetry).toHaveBeenCalledOnce() }) it('drops a pending relay retry after direct connectivity wins', () => { diff --git a/mobile/src/transport/mobile-relay-reconnect-controller.ts b/mobile/src/transport/mobile-relay-reconnect-controller.ts index 7e3e1ec0e13..6f586c17499 100644 --- a/mobile/src/transport/mobile-relay-reconnect-controller.ts +++ b/mobile/src/transport/mobile-relay-reconnect-controller.ts @@ -15,6 +15,8 @@ const RELAY_BACKOFF_MIN_MS = 250 const RELAY_BACKOFF_BASE_MS = 500 const RELAY_BACKOFF_CEILING_MS = 30_000 const RELAY_STABLE_CONNECTION_MS = RELAY_BACKOFF_CEILING_MS +const RELAY_HOST_OFFLINE_RETRY_MIN_MS = 5_000 +const RELAY_HOST_OFFLINE_RETRY_MAX_MS = 15_000 export type RelayReconnectDependencies = { now: () => number @@ -186,7 +188,8 @@ export class RelayReconnectController { // only an authenticated relay that survived the stability window resets the streak. this.activeRelayConnectedAt = null this.consecutiveFailures += 1 - const delay = this.delayMs() + const delay = + recovery?.kind === 'retry-after-host-offline' ? this.hostOfflineDelayMs() : this.delayMs() this.nextAttemptAt = now + delay if (error instanceof MobileE2EEAuthenticationError) { // Why: pairing state cannot change on a timer; polling only wakes the radio. @@ -194,18 +197,16 @@ export class RelayReconnectController { this.clearTimer() return } - if (recovery?.kind === 'wait-for-host-revival') { - // Why: retrying HOST_OFFLINE without a revival signal is polling a known-negative state. - this.recoveryGate = 'external-signal' - this.clearTimer() - return - } if (recovery?.kind === 'disable-relay-credential') { // Why: a rejected outer credential cannot recover until direct connectivity refreshes it. this.recoveryGate = 'fresh-credential' this.clearTimer() return } + if (recovery?.kind === 'retry-after-host-offline') { + // A prior transport timer must not bypass the slower known-offline retry. + this.clearTimer() + } this.recoveryGate = null if (!scheduleRetry) { this.clearTimer() @@ -269,6 +270,11 @@ export class RelayReconnectController { return Math.max(RELAY_BACKOFF_MIN_MS, Math.floor(cap * this.jitterFraction())) } + private hostOfflineDelayMs(): number { + const range = RELAY_HOST_OFFLINE_RETRY_MAX_MS - RELAY_HOST_OFFLINE_RETRY_MIN_MS + return RELAY_HOST_OFFLINE_RETRY_MIN_MS + Math.floor(range * this.jitterFraction()) + } + private jitterFraction(): number { const [high, low] = this.dependencies.randomBytes(2) return (((high ?? 0) << 8) | (low ?? 0)) / 0x1_00_00 diff --git a/src/shared/mobile-relay-close-codes.test.ts b/src/shared/mobile-relay-close-codes.test.ts index 829f966c066..6605c30c864 100644 --- a/src/shared/mobile-relay-close-codes.test.ts +++ b/src/shared/mobile-relay-close-codes.test.ts @@ -42,4 +42,11 @@ describe('mobile relay close-code contract', () => { directUnaffected: true }) }) + + it('keeps host-offline recovery self-healing', () => { + expect(mobileRelayRecoveryFor(4404, 'phone-resume')).toEqual({ + kind: 'retry-after-host-offline', + fullJitter: true + }) + }) }) diff --git a/src/shared/mobile-relay-close-codes.ts b/src/shared/mobile-relay-close-codes.ts index 6eefb151e52..eaff0a3a735 100644 --- a/src/shared/mobile-relay-close-codes.ts +++ b/src/shared/mobile-relay-close-codes.ts @@ -13,7 +13,7 @@ export type MobileRelayLeg = 'host-control' | 'host-data' | 'phone-invite' | 'ph export type MobileRelayRecovery = | { kind: 'disable-relay-credential'; directUnaffected: true } - | { kind: 'wait-for-host-revival' } + | { kind: 'retry-after-host-offline'; fullJitter: true } | { kind: 'reconnect-fresh-e2ee'; fullJitter: true } | { kind: 'resolve-invite-through-director-ws'; requireStrictlyNewerEpoch: true } | { kind: 'resolve-resume-through-director-post' } @@ -29,7 +29,7 @@ export function mobileRelayRecoveryFor( case MOBILE_RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL: return { kind: 'disable-relay-credential', directUnaffected: true } case MOBILE_RELAY_CLOSE_CODE.HOST_OFFLINE: - return { kind: 'wait-for-host-revival' } + return { kind: 'retry-after-host-offline', fullJitter: true } case MOBILE_RELAY_CLOSE_CODE.PEER_DROPPED: return { kind: 'reconnect-fresh-e2ee', fullJitter: true } case MOBILE_RELAY_CLOSE_CODE.WRONG_CELL: