diff --git a/config/reliability-gates.jsonc b/config/reliability-gates.jsonc index 986c6ec9345..64cbf10da14 100644 --- a/config/reliability-gates.jsonc +++ b/config/reliability-gates.jsonc @@ -785,6 +785,143 @@ ], "demotionRule": "Keep experimental or demote if focused tests flake, HOST_OFFLINE can park indefinitely, direct probes serialize configured endpoints, loser cleanup leaks clients, or physical-device radio cost exceeds the measured budget." }, + { + "id": "mobile-transport.lifecycle-liveness", + "title": "Wanted mobile hosts self-heal without false physical-session death", + "maturity": "experimental", + "protection": "partial", + "owner": "mobile-runtime", + "layer": "mobile-provider-and-physical-session-contract", + "surfaces": [ + "mobile host acquisition", + "Keychain and Keystore catalog reads", + "direct LAN and Tailscale RPC", + "cloud relay RPC", + "foreground and network revival" + ], + "platforms": ["ios", "android", "macos", "linux", "windows"], + "providers": ["lan", "tailscale", "cloud-relay"], + "coveredPlatforms": ["macos"], + "coveredProviders": ["lan", "tailscale", "cloud-relay"], + "coverageNotes": "Deterministic React/provider tests cover catalog, missing-host, construction, cancellation, release, forget, refresh, backoff, and stale-generation outcomes. Direct sessions cover idle probing and three fair misses. Relay sessions cover zero idle polling, rate-limited foreground probes, two fair misses, authenticated activity before semantic decoding, delivery ambiguity, scheduler stalls, and write failure. Physical iOS/Android sleep, carrier/Tailscale loss, and production relay paths remain live-test gaps.", + "motivatingLinks": [ + "https://linear.app/stably/issue/STA-4153/mobile-app-disconnects-frequently" + ], + "invariant": "A provider-wanted host without a logical client retries every current-generation pre-client failure until success or explicit cancellation, while release, disconnect, forget, refresh, and stale async continuations preserve their distinct resurrection policies. Direct sessions probe after authenticated silence and terminate after three fair misses. Relay sessions emit no periodic idle or demand-driven probes, use rate-limited foreground probes, and terminate after two fair misses. Authenticated traffic, scheduler stalls, and replacement identities cannot manufacture death.", + "oracle": "Inject rejected and never-settling catalog reads, missing profiles, synchronous construction failure, release, refresh, forget, revival, and stale completion under fake time. Require bounded 1s through 60s retry, exact current-generation publication, and no zero-owner or forgotten-client resurrection. Inject direct and relay authenticated text/binary traffic, silent probe windows, stalled clocks, replacement identities, and failed writes. Require direct three-miss tolerance; Relay zero idle and demand-driven liveness traffic, one foreground probe sequence, two fair 4s misses, a 10s voluntary rate limit, no old-session network-change probe, and delivery-unknown mutations without replay.", + "commands": [ + "pnpm --dir mobile exec vitest run --root .. mobile/src/transport/client-context.test.ts mobile/src/transport/host-open-recovery.test.tsx mobile/src/transport/host-open-retry-scheduler.test.ts mobile/src/transport/rpc-session-liveness-watchdog.test.ts mobile/src/transport/rpc-session-liveness-integration.test.ts mobile/src/transport/mobile-relay-rpc-session.test.ts mobile/src/transport/mobile-relay-rpc-session-liveness.test.ts mobile/src/transport/mobile-endpoint-nudge-router.test.ts mobile/src/transport/mobile-relay-e2ee-link.test.ts mobile/src/transport/stable-logical-rpc-client.test.ts" + ], + "testFiles": [ + "mobile/src/transport/client-context.test.ts", + "mobile/src/transport/host-open-recovery.test.tsx", + "mobile/src/transport/host-open-retry-scheduler.test.ts", + "mobile/src/transport/rpc-session-liveness-watchdog.test.ts", + "mobile/src/transport/rpc-session-liveness-integration.test.ts", + "mobile/src/transport/mobile-relay-rpc-session.test.ts", + "mobile/src/transport/mobile-relay-rpc-session-liveness.test.ts", + "mobile/src/transport/mobile-endpoint-nudge-router.test.ts", + "mobile/src/transport/mobile-relay-e2ee-link.test.ts", + "mobile/src/transport/stable-logical-rpc-client.test.ts" + ], + "assertionRefs": [ + { + "file": "mobile/src/transport/client-context.test.ts", + "assertions": ["a retired close-on-release owner closes an ownerless manual reconnect"] + }, + { + "file": "mobile/src/transport/host-open-recovery.test.tsx", + "assertions": [ + "catalog, missing-host, and construction failures retry without remounting", + "final release and forget prevent later client publication", + "refresh bypasses a never-settling stale open and stale failure cannot clobber its replacement", + "revival expedites recovery without resetting the failure tier", + "a retired owner's release cannot cancel a newer owner's open", + "explicit reconnect demand survives a pre-client failure" + ] + }, + { + "file": "mobile/src/transport/rpc-session-liveness-watchdog.test.ts", + "assertions": [ + "authenticated traffic resets the idle and missed-probe clocks", + "two fair misses are tolerated and the third terminates exactly once", + "scheduler stalls and replacement identities cannot terminate the current session", + "continuous authenticated traffic does not churn idle timers per frame" + ] + }, + { + "file": "mobile/src/transport/rpc-session-liveness-integration.test.ts", + "assertions": [ + "direct handshake and probe write exceptions recover without escaping", + "one silent foreground window preserves the connected direct socket", + "authenticated terminal binary output restores direct-session liveness", + "a replay write failure retains every queued stream for the replacement session", + "liveness probe replies cannot consume the authentication retry budget" + ] + }, + { + "file": "mobile/src/transport/mobile-relay-rpc-session.test.ts", + "assertions": [ + "written Relay requests retain delivery-unknown failures", + "resume confirmation and stream routing remain authenticated" + ] + }, + { + "file": "mobile/src/transport/mobile-relay-rpc-session-liveness.test.ts", + "assertions": [ + "an idle Relay emits zero periodic traffic", + "RPCs and subscriptions after prolonged silence emit no liveness probe", + "foreground Relay liveness uses two fair 4-second misses and a 10-second sequence rate limit", + "network change does not probe the replaced Relay", + "a failed foreground probe write disconnects immediately" + ] + }, + { + "file": "mobile/src/transport/stable-logical-rpc-client.test.ts", + "assertions": ["a delivery-unknown mutation is not replayed onto a replacement Relay"] + } + ], + "evidenceRuns": [ + { + "date": "2026-08-13", + "runner": "local", + "platform": "macos", + "command": "pnpm --dir mobile exec vitest run --root .. mobile/src/transport/client-context.test.ts mobile/src/transport/host-open-recovery.test.tsx mobile/src/transport/host-open-retry-scheduler.test.ts mobile/src/transport/rpc-session-liveness-watchdog.test.ts mobile/src/transport/rpc-session-liveness-integration.test.ts mobile/src/transport/mobile-relay-rpc-session.test.ts mobile/src/transport/mobile-relay-rpc-session-liveness.test.ts mobile/src/transport/mobile-endpoint-nudge-router.test.ts mobile/src/transport/mobile-relay-e2ee-link.test.ts mobile/src/transport/stable-logical-rpc-client.test.ts", + "result": "passed", + "durationSeconds": 0.74, + "summary": "Ten focused files passed 66 deterministic provider lifecycle, direct/Relay liveness, traffic, and delivery-ambiguity tests." + } + ], + "runtimeBudget": { + "p95Seconds": 5, + "scope": "focused mobile provider and transport unit tests" + }, + "flakeHistory": { + "status": "unknown", + "evidence": "One deterministic local focused run exists; CI and soak history are not yet available." + }, + "redGreenEvidence": { + "status": "complete", + "evidence": "Current main failed six deterministic assertions: terminal catalog failure, zero-owner publication, stale failure clobber, initial direct handshake write throw, one-window direct death, and foreground probe write throw. The candidate passes the same assertions and adds relay parity, backoff, cancellation, and scheduler-fairness coverage." + }, + "performanceBudget": { + "required": true, + "evidence": "Each wanted clientless host owns at most one open and one retry timer with a 60-second capped tail. Direct sessions own one idle or probe timer. Idle Relay sessions own no liveness timer and emit no periodic or demand-driven liveness traffic; foreground sequences start at most once per 10 seconds. No subprocess, wire opcode, or server-side work is introduced." + }, + "promotionCriteria": [ + "Collect 100 consecutive CI passes or 14 days of soak history.", + "Capture the existing final-timeout log immediately before an affected-device drop.", + "Run physical iOS and Android direct LAN, Tailscale, relay, background/resume, and network-handoff journeys.", + "Measure reconnect radio and battery impact during the 60-second catalog retry tail, direct 44-second idle liveness budget, and Relay foreground probes." + ], + "knownGaps": [ + "No physical iOS or Android device was exercised.", + "No affected-device log yet proves the old one-window watchdog caused the reported live-session drop.", + "Carrier NAT, production Tailscale, production relay cells, and real scheduler suspension remain unmeasured.", + "Persistent pre-client failure metadata is not yet exposed in Home UI." + ], + "demotionRule": "Keep experimental or demote if a wanted host can park without retry, stale work can publish or notify, disconnect or forget resurrects a host, direct closes before three fair misses, Relay polls while idle or on demand, survives two fair foreground misses, authenticated traffic fails to restore health, timers leak across identity replacement, mutations replay after ambiguous delivery, or physical-device radio cost exceeds the measured budget." + }, { "id": "desktop-relay.assignment-backpressure", "title": "Desktop relay drain recovery cannot amplify a director outage", diff --git a/mobile/app/h/[hostId]/index.tsx b/mobile/app/h/[hostId]/index.tsx index a6660ffe721..5fd6fb39d43 100644 --- a/mobile/app/h/[hostId]/index.tsx +++ b/mobile/app/h/[hostId]/index.tsx @@ -25,7 +25,7 @@ import { loadHosts, updateLastConnected } from '../../../src/transport/host-stor import { removeHostAndCloseClient } from '../../../src/transport/host-removal-lifecycle' import { useHostClient, - useCloseHost, + useForgetHostClient, useForceReconnect } from '../../../src/transport/client-context' import { useWorktreeResync } from '../../../src/transport/use-worktree-resync' @@ -148,7 +148,7 @@ export function HostScreen({ const repoMetadataFetchedAtRef = useRef(0) const newWorktreeModalRef = useRef<{ open: () => void }>(null) const newWorktreeModalVisibleRef = useRef(false) - const closeHostClient = useCloseHost() + const forgetHostClient = useForgetHostClient() const forceReconnectHost = useForceReconnect() const [worktrees, setWorktrees] = useState(initialCache ?? []) const [worktreesLoaded, setWorktreesLoaded] = useState(initialCache != null) @@ -636,14 +636,14 @@ export function HostScreen({ return } try { - await removeHostAndCloseClient(hostId, closeHostClient) + await removeHostAndCloseClient(hostId, forgetHostClient) leaveHost() } catch { // Why: removal can fail while still paired; re-open confirm (ConfirmModal closes on confirm). setConfirmRemoveHost(true) Alert.alert('Could not remove host', 'Please try again.') } - }, [hostId, leaveHost, closeHostClient]) + }, [hostId, leaveHost, forgetHostClient]) const navigateFromHostList = useCallback( (target: string) => { diff --git a/mobile/app/index.tsx b/mobile/app/index.tsx index 3edd5cf42f8..66b4ebf4b62 100644 --- a/mobile/app/index.tsx +++ b/mobile/app/index.tsx @@ -25,7 +25,12 @@ import type { HomeWorktreeSummary, HostWorktreeInfo } from '../src/worktree/home import type { RpcClient } from '../src/transport/rpc-client' import { createHostConnectRefetchGate } from '../src/transport/host-connect-refetch-gate' import { sendSingleFlightRequest } from '../src/transport/request-single-flight' -import { useCloseHost, useForceReconnect, usePrimeHosts } from '../src/transport/client-context' +import { + useDisconnectHostClient, + useForceReconnect, + useForgetHostClient, + usePrimeHosts +} from '../src/transport/client-context' import { useAllHostClients } from '../src/transport/use-all-host-clients' import { resolveHomeHostConnectionState, @@ -254,7 +259,8 @@ export default function HomeScreen() { () => Object.fromEntries(allClients.map(({ hostId, path }) => [hostId, path])), [allClients] ) - const closeHostClient = useCloseHost() + const disconnectHostClient = useDisconnectHostClient() + const forgetHostClient = useForgetHostClient() const forceReconnectHost = useForceReconnect() const primeHosts = usePrimeHosts() // Why: prime the cache with loaded HostProfiles to avoid a second serialized Keychain pass (multi-second connect latency) on cold start. @@ -643,7 +649,7 @@ export default function HomeScreen() { } const hostToRemove = confirmRemove try { - await removeHostAndCloseClient(hostToRemove.id, closeHostClient) + await removeHostAndCloseClient(hostToRemove.id, forgetHostClient) setConfirmRemove(null) setHostCatalog(await loadHostCatalog()) } catch { @@ -949,7 +955,7 @@ export default function HomeScreen() { : false, onDismiss: () => setActionTarget(null), onReconnect: (hostId) => void forceReconnectHost(hostId), - onDisconnect: closeHostClient, + onDisconnect: disconnectHostClient, onEdit: openMobileHostEdit, onRemove: (host) => setConfirmRemove(host) })} diff --git a/mobile/app/pair-confirm.tsx b/mobile/app/pair-confirm.tsx index 0621caa4f01..be790427088 100644 --- a/mobile/app/pair-confirm.tsx +++ b/mobile/app/pair-confirm.tsx @@ -9,7 +9,7 @@ import { type PreProfilePairingAttempt } from '../src/transport/pre-profile-pairing-coordinator' import type { ConnectionLogEntry } from '../src/transport/types' -import { useCloseHost } from '../src/transport/client-context' +import { useRefreshHostClient } from '../src/transport/client-context' import { colors, spacing, radii, typography } from '../src/theme/mobile-theme' import { ConnectionLog } from '../src/components/ConnectionLog' import { @@ -28,7 +28,7 @@ const PAIRING_OVERALL_TIMEOUT_MS = 25_000 export default function PairConfirmScreen() { const router = useRouter() - const closeHost = useCloseHost() + const refreshHostClient = useRefreshHostClient() const insets = useSafeAreaInsets() const params = useLocalSearchParams<{ code?: string }>() const [status, setStatus] = useState('awaiting-confirm') @@ -112,10 +112,8 @@ export default function PairConfirmScreen() { // Why: re-pairing the same desktop now reuses its existing host id // (STA-1840 dedup), so a client cached under that id from an earlier // pairing would keep the stale endpoint/relay. Close it so the - // destination screen opens a fresh client with the newly-paired - // profile — the removeHost() path already refreshes on re-pair, and a - // brand-new host has no cached entry so this is a no-op. - closeHost(hostId) + // Refresh any cached client from the newly persisted pairing profile. + refreshHostClient(hostId) const onboardingSteps = await loadMobileOnboardingSteps() if (!mountedRef.current) { return diff --git a/mobile/app/pair-scan.tsx b/mobile/app/pair-scan.tsx index 892c1b359a1..c78d7a907df 100644 --- a/mobile/app/pair-scan.tsx +++ b/mobile/app/pair-scan.tsx @@ -18,7 +18,7 @@ import { type PreProfilePairingAttempt } from '../src/transport/pre-profile-pairing-coordinator' import type { ConnectionLogEntry, PairingOffer } from '../src/transport/types' -import { useCloseHost } from '../src/transport/client-context' +import { useRefreshHostClient } from '../src/transport/client-context' import { colors, spacing, radii, typography } from '../src/theme/mobile-theme' import { TextInputModal } from '../src/components/TextInputModal' import { ConnectionLog } from '../src/components/ConnectionLog' @@ -47,7 +47,7 @@ function Step({ number, text }: { number: number; text: string }) { export default function PairScanScreen() { const router = useRouter() - const closeHost = useCloseHost() + const refreshHostClient = useRefreshHostClient() const insets = useSafeAreaInsets() const [permission, requestPermission] = useCameraPermissions() const [status, setStatus] = useState<'scanning' | 'connecting' | 'error'>('scanning') @@ -156,10 +156,8 @@ export default function PairScanScreen() { // Why: re-pairing the same desktop now reuses its existing host id // (STA-1840 dedup), so a client cached under that id from an earlier // pairing would keep the stale endpoint/relay. Close it so the - // destination screen opens a fresh client with the newly-paired - // profile — the removeHost() path already refreshes on re-pair, and a - // brand-new host has no cached entry so this is a no-op. - closeHost(hostId) + // Refresh any cached client from the newly persisted pairing profile. + refreshHostClient(hostId) const onboardingSteps = await loadMobileOnboardingSteps() if (!mountedRef.current) { return diff --git a/mobile/src/transport/client-context.test.ts b/mobile/src/transport/client-context.test.ts index 59b0a7ac5dc..69f531ef45f 100644 --- a/mobile/src/transport/client-context.test.ts +++ b/mobile/src/transport/client-context.test.ts @@ -1,4 +1,4 @@ -import { createElement } from 'react' +import { createElement, type ReactElement } from 'react' import { act, create, type ReactTestRenderer } from 'react-test-renderer' import { beforeEach, describe, expect, it, vi } from 'vitest' import type { ConnectionState } from './types' @@ -20,7 +20,12 @@ vi.mock('./connection-revival-triggers', () => ({ subscribeConnectionRevivalTriggers: () => () => {} })) -import { RpcClientProvider, useCloseHost, useForceReconnect, useHostClient } from './client-context' +import { + RpcClientProvider, + useDisconnectHostClient, + useForceReconnect, + useHostClient +} from './client-context' import { useAllHostClients } from './use-all-host-clients' import { selectHomeAutoConnectHostIds } from './home-host-auto-connect' @@ -67,25 +72,25 @@ const HOST = { type Harness = { readonly hook: ReturnType - readonly closeHost: (hostId: string) => void + readonly disconnectHost: (hostId: string) => void readonly unmount: () => void } async function renderHarness(hostId: string): Promise { let hook: ReturnType | null = null - let closeHost: ((hostId: string) => void) | null = null + let disconnectHost: ((hostId: string) => void) | null = null let renderer: ReactTestRenderer | null = null function Probe(): null { hook = useHostClient(hostId) - closeHost = useCloseHost() + disconnectHost = useDisconnectHostClient() return null } await act(async () => { renderer = create(createElement(RpcClientProvider, null, createElement(Probe))) }) - if (!hook || !closeHost || !renderer) { + if (!hook || !disconnectHost || !renderer) { throw new Error('harness did not render') } const mounted = renderer as ReactTestRenderer @@ -96,11 +101,11 @@ async function renderHarness(hostId: string): Promise { } return hook }, - closeHost: (id) => { - if (!closeHost) { - throw new Error('closeHost not rendered') + disconnectHost: (id) => { + if (!disconnectHost) { + throw new Error('disconnectHost not rendered') } - closeHost(id) + disconnectHost(id) }, unmount: () => mounted.unmount() } @@ -203,11 +208,11 @@ describe('useHostClient', () => { expect(harness.hook.client).not.toBeNull() expect(harness.hook.state).toBe('connected') - // Regression (STA-1511): closeHost deletes the entry; before the fix the + // Regression (STA-1511): disconnect deletes the entry; before the fix the // hook kept handing out the closed client, so mounted screens kept // driving requests that could never resolve. await act(async () => { - harness.closeHost(HOST.id) + harness.disconnectHost(HOST.id) }) expect(fake.closeMock).toHaveBeenCalled() expect(harness.hook.client).toBeNull() @@ -216,6 +221,40 @@ describe('useHostClient', () => { harness.unmount() }) + it('closes an ownerless reconnect after disconnect retires a mounted owner', async () => { + const initialClient = makeFakeClient('connected') + const replacementClient = makeFakeClient('connected') + connectMock.mockReturnValueOnce(initialClient).mockReturnValueOnce(replacementClient) + loadHostsMock.mockResolvedValue([HOST]) + let disconnectHost: ((hostId: string) => void) | null = null + let reconnectHost: ((hostId: string) => Promise) | null = null + + function Probe(): null { + useAllHostClients([HOST.id], { closeUnusedOnRelease: true }) + disconnectHost = useDisconnectHostClient() + reconnectHost = useForceReconnect() + return null + } + function App({ visible }: { visible: boolean }) { + return createElement(RpcClientProvider, null, visible ? createElement(Probe) : null) + } + + let renderer: { update(element: ReactElement): void; unmount(): void } | null = null + await act(async () => { + renderer = create(createElement(App, { visible: true })) + await Promise.resolve() + }) + await act(async () => { + disconnectHost?.(HOST.id) + await reconnectHost?.(HOST.id) + }) + act(() => renderer?.update(createElement(App, { visible: false }))) + + expect(initialClient.closeMock).toHaveBeenCalledOnce() + expect(replacementClient.closeMock).toHaveBeenCalledOnce() + act(() => renderer?.unmount()) + }) + it('reports disconnected instead of hanging when the host id is unknown', async () => { loadHostsMock.mockResolvedValue([]) @@ -299,10 +338,10 @@ describe('useHostClient', () => { connectMock.mockReturnValue(fake) loadHostsMock.mockReturnValue(hostLookup) - let closeHost: ((hostId: string) => void) | null = null + let disconnectHost: ((hostId: string) => void) | null = null let renderer: ReactTestRenderer | null = null function Probe(): null { - closeHost = useCloseHost() + disconnectHost = useDisconnectHostClient() useHostClient(HOST.id) return null } @@ -311,11 +350,11 @@ describe('useHostClient', () => { renderer = create(createElement(RpcClientProvider, null, createElement(Probe))) }) expect(loadHostsMock).toHaveBeenCalledOnce() - if (!closeHost || !resolveHosts || !renderer) { + if (!disconnectHost || !resolveHosts || !renderer) { throw new Error('pending-open harness did not initialize') } - act(() => closeHost?.(HOST.id)) + act(() => disconnectHost?.(HOST.id)) await act(async () => { resolveHosts?.([HOST]) await hostLookup diff --git a/mobile/src/transport/client-context.tsx b/mobile/src/transport/client-context.tsx index 18ee0b7782a..3e241b71270 100644 --- a/mobile/src/transport/client-context.tsx +++ b/mobile/src/transport/client-context.tsx @@ -11,12 +11,16 @@ import { type ReactNode } from 'react' import type { RpcClient } from './rpc-client' -import { connectionLogStore } from './connection-log-buffer' import { subscribeConnectionRevivalTriggers } from './connection-revival-triggers' import { HostClientOpenRegistry } from './host-client-open-registry' -import { decrementPendingAcquisition } from './host-client-acquisition-count' import { - clientActivePath, + HostClientAcquisitionRegistry, + type HostClientAcquisition +} from './host-client-acquisition-registry' +import { HostOpenRetryScheduler } from './host-open-retry-scheduler' +import { openHostClientEntry, type HostClientStoreEntry } from './host-entry-opener' +import { + createHostClientSelectors, listHostClients, notifyAllHostListeners, notifyHostStateListeners, @@ -25,38 +29,12 @@ import { subscribeHostStateListener, type CloseEntryOptions } from './host-client-context-state' -import { loadHosts } from './host-store' -import { openHostLogicalClient } from './host-logical-client' -import type { MobileConnectionPath } from './stable-logical-rpc-client' import type { ConnectionState, HostProfile } from './types' +import type { RpcClientContextValue } from './rpc-client-context-contract' -type StoreEntry = { - client: RpcClient - state: ConnectionState - refCount: number - unsubState: () => void -} - -export type RpcClientContextValue = { - acquire: (hostId: string, host?: HostProfile) => RpcClient | null - release: (hostId: string) => void - releaseAndCloseIfUnused: (hostId: string) => void - closeIfUnused: (hostId: string) => void - forceReconnect: (hostId: string) => Promise - closeHost: (hostId: string) => void - getState: (hostId: string) => ConnectionState - // null = host has no client entry and no open in flight; callers pick the default. - getKnownState: (hostId: string) => ConnectionState | null - getReconnectAttempt: (hostId: string) => number - // Why: ms-epoch of the last 'connected' (null if never this session); UI escalates "Reconnecting…" into a re-pair prompt. - getLastConnectedAt: (hostId: string) => number | null - getActivePath: (hostId: string) => MobileConnectionPath - subscribeHostState: (hostId: string, listener: (state: ConnectionState) => void) => () => void - getAllClients: () => Array<{ hostId: string; client: RpcClient }> - subscribeAllHosts: (listener: () => void) => () => void - // Why: lets the home screen feed already-loaded HostProfiles so we don't pay loadHosts() latency twice. - primeHosts: (hosts: HostProfile[]) => void -} +type StoreEntry = HostClientStoreEntry +export type { HostClientAcquisition } from './host-client-acquisition-registry' +export type { RpcClientContextValue } from './rpc-client-context-contract' const Ctx = createContext(null) @@ -69,6 +47,9 @@ export function RpcClientProvider({ children }: { children: ReactNode }) { // Pending opens keyed by hostId so two acquire() callers in the same render don't race the host lookup. const pendingOpensRef = useRef(new HostClientOpenRegistry()) const pendingAcquisitionsRef = useRef>(new Map()) + const acquisitionsRef = useRef(new HostClientAcquisitionRegistry()) + const manualDemandRef = useRef>(new Set()) + const retrySchedulerRef = useRef(null) // Why: cache of already-loaded HostProfiles so openEntry can skip a second loadHosts()/Keychain pass on cold start. const primedHostsRef = useRef>(new Map()) @@ -79,13 +60,18 @@ export function RpcClientProvider({ children }: { children: ReactNode }) { const closeEntry = useCallback((hostId: string, options: CloseEntryOptions) => { const entry = storeRef.current.get(hostId) - const acquisitionCount = entry?.refCount ?? pendingAcquisitionsRef.current.get(hostId) ?? 0 + const acquisitionCount = acquisitionsRef.current.count(hostId) pendingOpensRef.current.cancel(hostId) + retrySchedulerRef.current?.cancel(hostId) if (options.preserveAcquisitions && acquisitionCount > 0) { pendingAcquisitionsRef.current.set(hostId, acquisitionCount) } else { + acquisitionsRef.current.clear(hostId) pendingAcquisitionsRef.current.delete(hostId) } + if (!options.preserveAcquisitions) { + manualDemandRef.current.delete(hostId) + } if (options.forgetPrimedHost) { primedHostsRef.current.delete(hostId) } @@ -96,107 +82,59 @@ export function RpcClientProvider({ children }: { children: ReactNode }) { notifyAllHosts() }, []) - const closeHost = useCallback( - (hostId: string) => { - closeEntry(hostId, { forgetPrimedHost: true, preserveAcquisitions: true }) + const openEntry = useCallback( + (hostId: string, allowUnowned = false): Promise => { + const retryScheduler = retrySchedulerRef.current + if (!retryScheduler) { + throw new Error('host retry scheduler not initialized') + } + return openHostClientEntry( + { + store: storeRef.current, + pendingOpens: pendingOpensRef.current, + pendingAcquisitions: pendingAcquisitionsRef.current, + primedHosts: primedHostsRef.current, + retryScheduler, + notifyHostState, + notifyAllHosts + }, + hostId, + allowUnowned + ).then((entry) => { + if (entry) { + manualDemandRef.current.delete(hostId) + } + return entry + }) }, - [closeEntry] + [] ) - const openEntry = useCallback(async (hostId: string): Promise => { - const existing = pendingOpensRef.current.getActivePromise(hostId) - if (existing) { - await existing - return storeRef.current.get(hostId) ?? null - } - let resolve: () => void = () => {} - const promise = new Promise((res) => { - resolve = res + if (!retrySchedulerRef.current) { + retrySchedulerRef.current = new HostOpenRetryScheduler({ + canRetry: (hostId, generation) => + pendingOpensRef.current.isGenerationCurrent(hostId, generation) && + ((pendingAcquisitionsRef.current.get(hostId) ?? 0) > 0 || + manualDemandRef.current.has(hostId)) && + !storeRef.current.has(hostId) && + pendingOpensRef.current.getActivePromise(hostId) === null, + open: (hostId) => void openEntry(hostId, manualDemandRef.current.has(hostId)) }) - const pendingOpen = pendingOpensRef.current.register(hostId, promise) - // Why: already-mounted subscribers must go amber for the Keychain read too. - notifyHostState(hostId, 'connecting') - - try { - // Why: prefer the primed cache so we don't serialize a second Keychain pass on cold start. - let host = primedHostsRef.current.get(hostId) - if (!host) { - try { - const hosts = await loadHosts() - host = hosts.find((h) => h.id === hostId) - } catch { - // Why: cold-start Keychain failure (iOS mid-unlock / Android Keystore race); surface 'disconnected' so the user can Reconnect. - notifyHostState(hostId, 'disconnected') - notifyAllHosts() - return null - } - if (!host) { - // Why: silent return leaves screens on a permanent spinner (STA-1511); surface 'disconnected' so they show a retry affordance. - notifyHostState(hostId, 'disconnected') - notifyAllHosts() - return null - } - } - - if (pendingOpen.cancelled) { - return null - } - - // Re-check after any await — another acquire() may have completed. - const after = storeRef.current.get(hostId) - if (after) { - return after - } - - let client: RpcClient - try { - client = openHostLogicalClient(host, (entry) => connectionLogStore.append(hostId, entry)) - } catch { - // Why: openHostLogicalClient can throw synchronously (bad public key / invalid URL); notify so the UI leaves 'connecting'. - notifyHostState(hostId, 'disconnected') - notifyAllHosts() - return null - } - const unsubState = client.onStateChange((state) => { - const cur = storeRef.current.get(hostId) - if (!cur) { - return - } - cur.state = state - notifyHostState(hostId, state) - }) - const entry: StoreEntry = { - client, - state: client.getState(), - refCount: pendingAcquisitionsRef.current.get(hostId) ?? 0, - unsubState - } - pendingAcquisitionsRef.current.delete(hostId) - storeRef.current.set(hostId, entry) - notifyHostState(hostId, entry.state) - notifyAllHosts() - return entry - } finally { - pendingOpensRef.current.deleteIfCurrent(hostId, pendingOpen) - resolve() - } - }, []) + } // Synchronous get-or-open: returns an existing client immediately, else kicks off an async open and returns null this tick. const acquire = useCallback( - (hostId: string, host?: HostProfile): RpcClient | null => { + (hostId: string, acquisition: HostClientAcquisition, host?: HostProfile): RpcClient | null => { if (host) { primedHostsRef.current.set(hostId, host) } + const acquisitionCount = acquisitionsRef.current.acquire(hostId, acquisition) const existing = storeRef.current.get(hostId) if (existing) { - existing.refCount += 1 + existing.refCount = acquisitionCount return existing.client } - pendingAcquisitionsRef.current.set( - hostId, - (pendingAcquisitionsRef.current.get(hostId) ?? 0) + 1 - ) + pendingAcquisitionsRef.current.set(hostId, acquisitionCount) // Trigger async open; returns null this tick — consumers re-call acquire() from an effect that re-runs on state changes. void openEntry(hostId) return null @@ -209,27 +147,72 @@ export function RpcClientProvider({ children }: { children: ReactNode }) { [] ) + const refreshHostClient = useCallback( + (hostId: string) => { + closeEntry(hostId, { forgetPrimedHost: true, preserveAcquisitions: true }) + if ((pendingAcquisitionsRef.current.get(hostId) ?? 0) > 0) { + void openEntry(hostId) + } + }, + [closeEntry, openEntry] + ) + + const forgetHostClient = useCallback( + (hostId: string) => { + closeEntry(hostId, { forgetPrimedHost: true, preserveAcquisitions: false }) + }, + [closeEntry] + ) + + const disconnectHostClient = useCallback( + (hostId: string) => { + closeEntry(hostId, { forgetPrimedHost: false, preserveAcquisitions: false }) + }, + [closeEntry] + ) + // Why: no idle-close on refcount→0 — transient nav gaps flashed false 'disconnected', so keep sockets alive while foregrounded. - const release = useCallback((hostId: string) => { - const entry = storeRef.current.get(hostId) - if (entry) { - entry.refCount = Math.max(0, entry.refCount - 1) + const release = useCallback((hostId: string, acquisition: HostClientAcquisition) => { + const acquisitionCount = acquisitionsRef.current.release(hostId, acquisition) + if (acquisitionCount === null) { return } - decrementPendingAcquisition(pendingAcquisitionsRef.current, hostId) + const entry = storeRef.current.get(hostId) + if (entry) { + entry.refCount = acquisitionCount + return + } + if (acquisitionCount > 0) { + pendingAcquisitionsRef.current.set(hostId, acquisitionCount) + } else { + pendingAcquisitionsRef.current.delete(hostId) + if (!manualDemandRef.current.has(hostId)) { + pendingOpensRef.current.cancel(hostId) + retrySchedulerRef.current?.cancel(hostId) + } + } }, []) const releaseAndCloseIfUnused = useCallback( - (hostId: string) => { + (hostId: string, acquisition: HostClientAcquisition) => { + const acquisitionCount = acquisitionsRef.current.release(hostId, acquisition) + if (acquisitionCount === null) { + if (acquisitionsRef.current.count(hostId) === 0) { + closeEntry(hostId, { forgetPrimedHost: false, preserveAcquisitions: false }) + } + return + } const entry = storeRef.current.get(hostId) if (entry) { - entry.refCount = Math.max(0, entry.refCount - 1) + entry.refCount = acquisitionCount if (entry.refCount === 0) { closeEntry(hostId, { forgetPrimedHost: false, preserveAcquisitions: false }) } return } - if (decrementPendingAcquisition(pendingAcquisitionsRef.current, hostId) === 0) { + if (acquisitionCount > 0) { + pendingAcquisitionsRef.current.set(hostId, acquisitionCount) + } else { closeEntry(hostId, { forgetPrimedHost: false, preserveAcquisitions: false }) } }, @@ -239,12 +222,12 @@ export function RpcClientProvider({ children }: { children: ReactNode }) { const closeIfUnused = useCallback( (hostId: string) => { const entry = storeRef.current.get(hostId) - const pendingCount = pendingAcquisitionsRef.current.get(hostId) + const acquisitionCount = acquisitionsRef.current.count(hostId) const hasPendingOpen = pendingOpensRef.current.getActivePromise(hostId) !== null - if (!entry && pendingCount === undefined && !hasPendingOpen) { + if (!entry && acquisitionCount === 0 && !hasPendingOpen) { return } - if ((entry?.refCount ?? pendingCount ?? 0) === 0) { + if (acquisitionCount === 0) { closeEntry(hostId, { forgetPrimedHost: false, preserveAcquisitions: false }) } }, @@ -255,12 +238,15 @@ export function RpcClientProvider({ children }: { children: ReactNode }) { async (hostId: string) => { const entry = storeRef.current.get(hostId) // Why: ownership survives explicit close/re-pair while observers never become synthetic owners. - const savedRefCount = entry?.refCount ?? pendingAcquisitionsRef.current.get(hostId) ?? 0 + const savedRefCount = acquisitionsRef.current.count(hostId) + manualDemandRef.current.add(hostId) if (entry) { entry.unsubState() entry.client.close() storeRef.current.delete(hostId) } + pendingOpensRef.current.cancel(hostId) + retrySchedulerRef.current?.cancel(hostId) if (savedRefCount > 0) { pendingAcquisitionsRef.current.set( hostId, @@ -269,38 +255,13 @@ export function RpcClientProvider({ children }: { children: ReactNode }) { } // Why: Retry must read amber for the whole reopen, not grey-then-amber. notifyHostState(hostId, 'connecting') - await openEntry(hostId) + await openEntry(hostId, true) }, [openEntry] ) - // null = no entry and no open in flight; callers pick their own default. - const getKnownState = useCallback((hostId: string): ConnectionState | null => { - const entry = storeRef.current.get(hostId) - if (entry) { - return entry.state - } - // Why: the async open (a Keychain pass) predates the store entry; reading that - // window as 'disconnected' made every host screen flash dead on mount (S2). - return pendingOpensRef.current.getActivePromise(hostId) ? 'connecting' : null - }, []) - - const getState = useCallback( - (hostId: string): ConnectionState => getKnownState(hostId) ?? 'disconnected', - [getKnownState] - ) - - const getReconnectAttempt = useCallback((hostId: string): number => { - return storeRef.current.get(hostId)?.client.getReconnectAttempt() ?? 0 - }, []) - - const getLastConnectedAt = useCallback((hostId: string): number | null => { - return storeRef.current.get(hostId)?.client.getLastConnectedAt() ?? null - }, []) - - const getActivePath = useCallback((hostId: string): MobileConnectionPath => { - return clientActivePath(storeRef.current.get(hostId)?.client) - }, []) + const { getKnownState, getState, getReconnectAttempt, getLastConnectedAt, getActivePath } = + useMemo(() => createHostClientSelectors(storeRef.current, pendingOpensRef.current), []) const subscribeHostState = useCallback( (hostId: string, listener: (state: ConnectionState) => void) => @@ -321,6 +282,9 @@ export function RpcClientProvider({ children }: { children: ReactNode }) { const store = storeRef.current return () => { pendingOpensRef.current.cancelAll() + retrySchedulerRef.current?.cancelAll() + acquisitionsRef.current.clearAll() + manualDemandRef.current.clear() pendingAcquisitionsRef.current.clear() for (const [hostId] of store) { closeEntry(hostId, { forgetPrimedHost: true, preserveAcquisitions: false }) @@ -331,8 +295,15 @@ export function RpcClientProvider({ children }: { children: ReactNode }) { // Why: nudge live clients when the OS signals the link may be back so sessions recover without a restart (issue #5049). useEffect(() => { return subscribeConnectionRevivalTriggers((reason) => { + for (const hostId of pendingAcquisitionsRef.current.keys()) { + retrySchedulerRef.current?.expedite(hostId) + } for (const entry of storeRef.current.values()) { - entry.client.notifyForeground(reason) + try { + entry.client.notifyForeground(reason) + } catch { + // One broken physical session must not block recovery for other hosts. + } } }) }, []) @@ -344,7 +315,9 @@ export function RpcClientProvider({ children }: { children: ReactNode }) { releaseAndCloseIfUnused, closeIfUnused, forceReconnect, - closeHost, + refreshHostClient, + forgetHostClient, + disconnectHostClient, getState, getKnownState, getReconnectAttempt, @@ -361,7 +334,9 @@ export function RpcClientProvider({ children }: { children: ReactNode }) { releaseAndCloseIfUnused, closeIfUnused, forceReconnect, - closeHost, + refreshHostClient, + forgetHostClient, + disconnectHostClient, getState, getKnownState, getReconnectAttempt, @@ -399,6 +374,7 @@ export function useHostClient(hostId: string | undefined): { ) const clientRef = useRef(null) const clientHostIdRef = useRef(hostId) + const acquisitionRef = useRef({}) useEffect(() => { if (!hostId) { @@ -421,12 +397,12 @@ export function useHostClient(hostId: string | undefined): { clientRef.current = found.client force((n) => n + 1) } else if (!found && clientRef.current) { - // Why: closeHost deletes the entry with no replacement; null it so screens don't drive a dead client (STA-1511). + // Why: disconnect/forget deletes the entry; never retain a dead client (STA-1511). clientRef.current = null force((n) => n + 1) } }) - const initial = ctx.acquire(hostId) + const initial = ctx.acquire(hostId, acquisitionRef.current) clientRef.current = initial setState(ctx.getKnownState(hostId) ?? 'connecting') if (initial) { @@ -436,7 +412,7 @@ export function useHostClient(hostId: string | undefined): { return () => { cancelled = true unsub() - ctx.release(hostId) + ctx.release(hostId, acquisitionRef.current) clientRef.current = null clientHostIdRef.current = undefined } @@ -453,9 +429,19 @@ export function useHostClient(hostId: string | undefined): { } // Why: host-store's removeHost() must close the live client but has no React-side handle; this hook bridges to it. -export function useCloseHost(): (hostId: string) => void { +export function useRefreshHostClient(): (hostId: string) => void { const ctx = useRpcClientContext() - return ctx.closeHost + return ctx.refreshHostClient +} + +export function useForgetHostClient(): (hostId: string) => void { + const ctx = useRpcClientContext() + return ctx.forgetHostClient +} + +export function useDisconnectHostClient(): (hostId: string) => void { + const ctx = useRpcClientContext() + return ctx.disconnectHostClient } // Why: future-proof "Connection issues — try again" affordance. diff --git a/mobile/src/transport/host-client-acquisition-count.ts b/mobile/src/transport/host-client-acquisition-count.ts deleted file mode 100644 index 1a1123e5c8b..00000000000 --- a/mobile/src/transport/host-client-acquisition-count.ts +++ /dev/null @@ -1,9 +0,0 @@ -export function decrementPendingAcquisition(pending: Map, hostId: string): number { - const next = Math.max(0, (pending.get(hostId) ?? 0) - 1) - if (next === 0) { - pending.delete(hostId) - } else { - pending.set(hostId, next) - } - return next -} diff --git a/mobile/src/transport/host-client-acquisition-registry.ts b/mobile/src/transport/host-client-acquisition-registry.ts new file mode 100644 index 00000000000..4e09ebdd9dc --- /dev/null +++ b/mobile/src/transport/host-client-acquisition-registry.ts @@ -0,0 +1,38 @@ +export type HostClientAcquisition = object + +export class HostClientAcquisitionRegistry { + private readonly acquisitions = new Map>() + + acquire(hostId: string, acquisition: HostClientAcquisition): number { + let active = this.acquisitions.get(hostId) + if (!active) { + active = new Set() + this.acquisitions.set(hostId, active) + } + active.add(acquisition) + return active.size + } + + release(hostId: string, acquisition: HostClientAcquisition): number | null { + const active = this.acquisitions.get(hostId) + if (!active?.delete(acquisition)) { + return null + } + if (active.size === 0) { + this.acquisitions.delete(hostId) + } + return active.size + } + + count(hostId: string): number { + return this.acquisitions.get(hostId)?.size ?? 0 + } + + clear(hostId: string): void { + this.acquisitions.delete(hostId) + } + + clearAll(): void { + this.acquisitions.clear() + } +} diff --git a/mobile/src/transport/host-client-context-state.ts b/mobile/src/transport/host-client-context-state.ts index e0f7243baa8..2aa7ca36400 100644 --- a/mobile/src/transport/host-client-context-state.ts +++ b/mobile/src/transport/host-client-context-state.ts @@ -1,4 +1,6 @@ import type { RpcClient } from './rpc-client' +import type { HostClientOpenRegistry } from './host-client-open-registry' +import type { HostClientStoreEntry } from './host-entry-opener' import type { MobileConnectionPath, StableLogicalRpcClient } from './stable-logical-rpc-client' import type { ConnectionState, HostProfile } from './types' @@ -62,6 +64,30 @@ export function primeHostProfiles(cache: Map, hosts: HostPr } } +export function createHostClientSelectors( + entries: ReadonlyMap, + pendingOpens: HostClientOpenRegistry +) { + const getKnownState = (hostId: string): ConnectionState | null => { + const entry = entries.get(hostId) + if (entry) { + return entry.state + } + // Why: the Keychain pass predates the store entry; this window is connecting. + return pendingOpens.getActivePromise(hostId) ? 'connecting' : null + } + return { + getKnownState, + getState: (hostId: string): ConnectionState => getKnownState(hostId) ?? 'disconnected', + getReconnectAttempt: (hostId: string): number => + entries.get(hostId)?.client.getReconnectAttempt() ?? 0, + getLastConnectedAt: (hostId: string): number | null => + entries.get(hostId)?.client.getLastConnectedAt() ?? null, + getActivePath: (hostId: string): MobileConnectionPath => + clientActivePath(entries.get(hostId)?.client) + } +} + export function clientActivePath(client: RpcClient | undefined): MobileConnectionPath { const logical = client as Partial | undefined if (typeof logical?.getActivePath !== 'function') { diff --git a/mobile/src/transport/host-client-open-registry.ts b/mobile/src/transport/host-client-open-registry.ts index a47ca5a231a..e662f037c5e 100644 --- a/mobile/src/transport/host-client-open-registry.ts +++ b/mobile/src/transport/host-client-open-registry.ts @@ -1,10 +1,12 @@ export type HostClientOpenTicket = { cancelled: boolean + generation: number promise: Promise } export class HostClientOpenRegistry { private readonly pending = new Map() + private readonly generations = new Map() getActivePromise(hostId: string): Promise | null { const ticket = this.pending.get(hostId) @@ -12,7 +14,11 @@ export class HostClientOpenRegistry { } register(hostId: string, promise: Promise): HostClientOpenTicket { - const ticket = { cancelled: false, promise } + const ticket = { + cancelled: false, + generation: this.advanceGeneration(hostId), + promise + } this.pending.set(hostId, ticket) return ticket } @@ -25,6 +31,15 @@ export class HostClientOpenRegistry { // reference immediately while the ticket still cancels its continuation. this.pending.delete(hostId) } + this.advanceGeneration(hostId) + } + + isCurrent(hostId: string, ticket: HostClientOpenTicket): boolean { + return !ticket.cancelled && this.isGenerationCurrent(hostId, ticket.generation) + } + + isGenerationCurrent(hostId: string, generation: number): boolean { + return this.generations.get(hostId) === generation } deleteIfCurrent(hostId: string, ticket: HostClientOpenTicket): void { @@ -34,9 +49,16 @@ export class HostClientOpenRegistry { } cancelAll(): void { - for (const ticket of this.pending.values()) { + for (const [hostId, ticket] of this.pending) { ticket.cancelled = true + this.advanceGeneration(hostId) } this.pending.clear() } + + private advanceGeneration(hostId: string): number { + const generation = (this.generations.get(hostId) ?? 0) + 1 + this.generations.set(hostId, generation) + return generation + } } diff --git a/mobile/src/transport/host-entry-opener.ts b/mobile/src/transport/host-entry-opener.ts new file mode 100644 index 00000000000..220b63abc64 --- /dev/null +++ b/mobile/src/transport/host-entry-opener.ts @@ -0,0 +1,141 @@ +import { connectionLogStore } from './connection-log-buffer' +import { loadHosts } from './host-store' +import { openHostLogicalClient } from './host-logical-client' +import type { HostClientOpenRegistry } from './host-client-open-registry' +import type { HostOpenRetryScheduler } from './host-open-retry-scheduler' +import type { RpcClient } from './rpc-client' +import type { ConnectionState, HostProfile } from './types' + +export type HostClientStoreEntry = { + client: RpcClient + state: ConnectionState + refCount: number + unsubState: () => void +} + +type HostEntryOpenerState = { + store: Map + pendingOpens: HostClientOpenRegistry + pendingAcquisitions: Map + primedHosts: Map + retryScheduler: HostOpenRetryScheduler + notifyHostState: (hostId: string, state: ConnectionState) => void + notifyAllHosts: () => void +} + +type HostOpenFailureCategory = 'catalog-unavailable' | 'host-not-found' | 'client-construction' + +export async function openHostClientEntry( + state: HostEntryOpenerState, + hostId: string, + allowUnowned = false +): Promise { + const existing = state.pendingOpens.getActivePromise(hostId) + if (existing) { + await existing + return state.store.get(hostId) ?? null + } + let resolve: () => void = () => {} + const promise = new Promise((res) => { + resolve = res + }) + const ticket = state.pendingOpens.register(hostId, promise) + let settled = false + const settle = () => { + if (settled) { + return + } + settled = true + state.pendingOpens.deleteIfCurrent(hostId, ticket) + resolve() + } + const isCurrent = () => state.pendingOpens.isCurrent(hostId, ticket) + const isWanted = () => allowUnowned || (state.pendingAcquisitions.get(hostId) ?? 0) > 0 + const failCurrentOpen = (category: HostOpenFailureCategory) => { + if (!isCurrent()) { + return + } + settle() + state.notifyHostState(hostId, 'disconnected') + state.notifyAllHosts() + const retry = state.retryScheduler.recordFailure(hostId, ticket.generation) + connectionLogStore.append(hostId, { + id: `host-open-${ticket.generation}-${Date.now()}`, + ts: Date.now(), + level: 'error', + message: 'Host client open failed', + detail: `${category}; retry ${retry.nextDelayMs}ms (failure ${retry.failureCount})` + }) + } + state.notifyHostState(hostId, 'connecting') + + try { + let host = state.primedHosts.get(hostId) + if (!host) { + try { + const hosts = await loadHosts() + host = hosts.find((candidate) => candidate.id === hostId) + } catch { + failCurrentOpen('catalog-unavailable') + return null + } + if (!host) { + failCurrentOpen('host-not-found') + return null + } + } + if (!isCurrent() || !isWanted()) { + return null + } + const published = state.store.get(hostId) + if (published) { + settle() + state.retryScheduler.recordSuccess(hostId) + return published + } + + let client: RpcClient + try { + client = openHostLogicalClient(host, (entry) => connectionLogStore.append(hostId, entry)) + } catch { + failCurrentOpen('client-construction') + return null + } + if (!isCurrent() || !isWanted() || state.store.has(hostId)) { + client.close() + return state.store.get(hostId) ?? null + } + const unsubState = client.onStateChange((next) => { + const current = state.store.get(hostId) + if (!current) { + return + } + current.state = next + state.notifyHostState(hostId, next) + }) + const entry: HostClientStoreEntry = { + client, + state: client.getState(), + refCount: state.pendingAcquisitions.get(hostId) ?? 0, + unsubState + } + state.pendingAcquisitions.delete(hostId) + state.store.set(hostId, entry) + settle() + const priorFailureCount = state.retryScheduler.recordSuccess(hostId) + if (priorFailureCount > 0) { + connectionLogStore.append(hostId, { + id: `host-open-recovered-${ticket.generation}-${Date.now()}`, + ts: Date.now(), + level: 'success', + message: 'Host client recovered', + detail: `after ${priorFailureCount} failed open${priorFailureCount === 1 ? '' : 's'}` + }) + } + state.notifyHostState(hostId, entry.state) + state.notifyAllHosts() + return entry + } finally { + settle() + } +} diff --git a/mobile/src/transport/host-open-recovery.test.tsx b/mobile/src/transport/host-open-recovery.test.tsx new file mode 100644 index 00000000000..70ebc131791 --- /dev/null +++ b/mobile/src/transport/host-open-recovery.test.tsx @@ -0,0 +1,388 @@ +import { createElement, type ReactElement } from 'react' +import { act, create } from 'react-test-renderer' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { RpcClient } from './rpc-client' +import type { ConnectionState } from './types' + +const openHostLogicalClientMock = vi.fn() +const loadHostsMock = vi.fn() +const revival = vi.hoisted(() => ({ callback: null as null | ((reason: 'focus') => void) })) + +vi.mock('./host-logical-client', () => ({ + openHostLogicalClient: (...args: unknown[]) => openHostLogicalClientMock(...args) +})) +vi.mock('./host-store', () => ({ + loadHosts: () => loadHostsMock() +})) +vi.mock('./connection-revival-triggers', () => ({ + subscribeConnectionRevivalTriggers: (callback: (reason: 'focus') => void) => { + revival.callback = callback + return () => { + revival.callback = null + } + } +})) + +import { + useDisconnectHostClient, + RpcClientProvider, + useForgetHostClient, + useForceReconnect, + useHostClient, + useRefreshHostClient +} from './client-context' + +const HOST = { + id: 'host-1', + name: 'Host 1', + endpoint: 'ws://127.0.0.1:1', + deviceToken: 'token', + publicKeyB64: 'key', + lastConnected: 0 +} + +type MountedRenderer = { + unmount(): void + update(element: ReactElement): void +} + +function fakeClient(): RpcClient { + return { + sendRequest: vi.fn(), + subscribe: vi.fn(() => () => {}), + updateTerminalSubscriptionViewport: vi.fn(), + getState: () => 'connected', + getReconnectAttempt: () => 0, + getLastConnectedAt: () => null, + onStateChange: () => () => {}, + notifyForeground: vi.fn(), + close: vi.fn() + } +} + +beforeEach(() => { + vi.useFakeTimers() + openHostLogicalClientMock.mockReset() + loadHostsMock.mockReset() + revival.callback = null +}) + +afterEach(() => vi.useRealTimers()) + +describe('wanted host open recovery', () => { + it('recovers after a transient catalog failure without remounting', async () => { + const client = fakeClient() + loadHostsMock.mockRejectedValueOnce(new Error('catalog unavailable')).mockResolvedValue([HOST]) + openHostLogicalClientMock.mockReturnValue(client) + + let observed: { client: RpcClient | null; state: ConnectionState } | null = null + let renderer: MountedRenderer | null = null + function Probe(): null { + observed = useHostClient(HOST.id) + return null + } + + try { + await act(async () => { + renderer = create(createElement(RpcClientProvider, null, createElement(Probe))) + await Promise.resolve() + }) + expect(observed).toMatchObject({ client: null, state: 'disconnected' }) + expect(loadHostsMock).toHaveBeenCalledOnce() + + await act(async () => { + await vi.advanceTimersByTimeAsync(1_000) + }) + + expect(loadHostsMock).toHaveBeenCalledTimes(2) + expect(openHostLogicalClientMock).toHaveBeenCalledOnce() + expect(observed).toMatchObject({ client, state: 'connected' }) + } finally { + act(() => renderer?.unmount()) + } + }) + + it('recovers when a missing host appears in the catalog', async () => { + const client = fakeClient() + loadHostsMock.mockResolvedValueOnce([]).mockResolvedValueOnce([HOST]) + openHostLogicalClientMock.mockReturnValue(client) + + let observed: { client: RpcClient | null; state: ConnectionState } | null = null + let renderer: MountedRenderer | null = null + function Probe(): null { + observed = useHostClient(HOST.id) + return null + } + + try { + await act(async () => { + renderer = create(createElement(RpcClientProvider, null, createElement(Probe))) + await Promise.resolve() + }) + await act(async () => { + await vi.advanceTimersByTimeAsync(1_000) + }) + expect(loadHostsMock).toHaveBeenCalledTimes(2) + expect(observed).toMatchObject({ client, state: 'connected' }) + } finally { + act(() => renderer?.unmount()) + } + }) + + it('recovers after synchronous client construction fails', async () => { + const client = fakeClient() + loadHostsMock.mockResolvedValue([HOST]) + openHostLogicalClientMock.mockImplementationOnce(() => { + throw new Error('bad transient profile') + }) + openHostLogicalClientMock.mockReturnValueOnce(client) + + let observed: { client: RpcClient | null; state: ConnectionState } | null = null + let renderer: MountedRenderer | null = null + function Probe(): null { + observed = useHostClient(HOST.id) + return null + } + + try { + await act(async () => { + renderer = create(createElement(RpcClientProvider, null, createElement(Probe))) + await Promise.resolve() + }) + await act(async () => { + await vi.advanceTimersByTimeAsync(1_000) + }) + expect(openHostLogicalClientMock).toHaveBeenCalledTimes(2) + expect(observed).toMatchObject({ client, state: 'connected' }) + } finally { + act(() => renderer?.unmount()) + } + }) + + it('expedites recovery without resetting the failure tier', async () => { + const client = fakeClient() + loadHostsMock + .mockRejectedValueOnce(new Error('first failure')) + .mockRejectedValueOnce(new Error('second failure')) + .mockResolvedValueOnce([HOST]) + openHostLogicalClientMock.mockReturnValue(client) + + let renderer: MountedRenderer | null = null + function Probe(): null { + useHostClient(HOST.id) + return null + } + + try { + await act(async () => { + renderer = create(createElement(RpcClientProvider, null, createElement(Probe))) + await Promise.resolve() + }) + act(() => revival.callback?.('focus')) + await act(async () => { + await Promise.resolve() + }) + expect(loadHostsMock).toHaveBeenCalledTimes(2) + + await vi.advanceTimersByTimeAsync(1_999) + expect(loadHostsMock).toHaveBeenCalledTimes(2) + await vi.advanceTimersByTimeAsync(1) + expect(loadHostsMock).toHaveBeenCalledTimes(3) + } finally { + act(() => renderer?.unmount()) + } + }) + + it('forgets demand while a retry is waiting', async () => { + loadHostsMock.mockRejectedValue(new Error('catalog unavailable')) + + let forgetHostClient: ((hostId: string) => void) | null = null + let renderer: MountedRenderer | null = null + function Probe(): null { + useHostClient(HOST.id) + forgetHostClient = useForgetHostClient() + return null + } + + try { + await act(async () => { + renderer = create(createElement(RpcClientProvider, null, createElement(Probe))) + await Promise.resolve() + }) + act(() => forgetHostClient?.(HOST.id)) + await vi.advanceTimersByTimeAsync(120_000) + expect(loadHostsMock).toHaveBeenCalledOnce() + } finally { + act(() => renderer?.unmount()) + } + }) + + it('does not let a retired owner cancel a newer acquisition', async () => { + let resolveReplacement: ((hosts: (typeof HOST)[]) => void) | null = null + const replacementLookup = new Promise<(typeof HOST)[]>((resolve) => { + resolveReplacement = resolve + }) + loadHostsMock.mockResolvedValueOnce([HOST]).mockReturnValueOnce(replacementLookup) + const firstClient = fakeClient() + const replacementClient = fakeClient() + openHostLogicalClientMock + .mockReturnValueOnce(firstClient) + .mockReturnValueOnce(replacementClient) + + let disconnectHostClient: ((hostId: string) => void) | null = null + function RetiredOwner(): null { + useHostClient(HOST.id) + disconnectHostClient = useDisconnectHostClient() + return null + } + function NewOwner(): null { + useHostClient(HOST.id) + return null + } + function App({ oldVisible, newVisible }: { oldVisible: boolean; newVisible: boolean }) { + return createElement( + RpcClientProvider, + null, + oldVisible ? createElement(RetiredOwner) : null, + newVisible ? createElement(NewOwner) : null + ) + } + + let renderer: MountedRenderer | null = null + try { + await act(async () => { + renderer = create(createElement(App, { oldVisible: true, newVisible: false })) + await Promise.resolve() + }) + act(() => disconnectHostClient?.(HOST.id)) + act(() => renderer?.update(createElement(App, { oldVisible: true, newVisible: true }))) + act(() => renderer?.update(createElement(App, { oldVisible: false, newVisible: true }))) + + await act(async () => { + resolveReplacement?.([HOST]) + await replacementLookup + }) + expect(openHostLogicalClientMock).toHaveBeenCalledTimes(2) + expect(replacementClient.close).not.toHaveBeenCalled() + } finally { + act(() => renderer?.unmount()) + } + }) + + it('keeps explicit reconnect demand through a pre-client failure', async () => { + loadHostsMock + .mockRejectedValueOnce(new Error('catalog unavailable')) + .mockResolvedValueOnce([HOST]) + openHostLogicalClientMock.mockReturnValue(fakeClient()) + + let forceReconnect: ((hostId: string) => Promise) | null = null + function Probe(): null { + forceReconnect = useForceReconnect() + return null + } + + let renderer: MountedRenderer | null = null + try { + act(() => { + renderer = create(createElement(RpcClientProvider, null, createElement(Probe))) + }) + await act(async () => { + await forceReconnect?.(HOST.id) + }) + await act(async () => { + await vi.advanceTimersByTimeAsync(1_000) + }) + expect(loadHostsMock).toHaveBeenCalledTimes(2) + expect(openHostLogicalClientMock).toHaveBeenCalledOnce() + } finally { + act(() => renderer?.unmount()) + } + }) + + it('cancels an in-flight open after the final owner releases', async () => { + let resolveHosts: ((hosts: (typeof HOST)[]) => void) | null = null + const lookup = new Promise<(typeof HOST)[]>((resolve) => { + resolveHosts = resolve + }) + loadHostsMock.mockReturnValue(lookup) + openHostLogicalClientMock.mockReturnValue(fakeClient()) + + function Probe(): null { + useHostClient(HOST.id) + return null + } + function App({ visible }: { visible: boolean }) { + return createElement(RpcClientProvider, null, visible ? createElement(Probe) : null) + } + + let renderer: MountedRenderer | null = null + try { + act(() => { + renderer = create(createElement(App, { visible: true })) + }) + expect(loadHostsMock).toHaveBeenCalledOnce() + + act(() => renderer?.update(createElement(App, { visible: false }))) + await act(async () => { + resolveHosts?.([HOST]) + await lookup + }) + + expect(openHostLogicalClientMock).not.toHaveBeenCalled() + } finally { + act(() => renderer?.unmount()) + } + }) + + it('ignores a canceled failure after a replacement client succeeds', async () => { + let rejectStale: ((error: Error) => void) | null = null + const staleLookup = new Promise<(typeof HOST)[]>((_, reject) => { + rejectStale = reject + }) + loadHostsMock.mockReturnValueOnce(staleLookup).mockResolvedValueOnce([HOST]) + const client = fakeClient() + openHostLogicalClientMock.mockReturnValue(client) + + let refreshHostClient: ((hostId: string) => void) | null = null + let replacement: { client: RpcClient | null; state: ConnectionState } | null = null + function Primary(): null { + useHostClient(HOST.id) + refreshHostClient = useRefreshHostClient() + return null + } + function Replacement(): null { + replacement = useHostClient(HOST.id) + return null + } + function App({ showReplacement }: { showReplacement: boolean }) { + return createElement( + RpcClientProvider, + null, + createElement(Primary), + showReplacement ? createElement(Replacement) : null + ) + } + + let renderer: MountedRenderer | null = null + try { + act(() => { + renderer = create(createElement(App, { showReplacement: false })) + }) + act(() => refreshHostClient?.(HOST.id)) + await act(async () => { + renderer?.update(createElement(App, { showReplacement: true })) + await Promise.resolve() + }) + expect(replacement).toMatchObject({ client, state: 'connected' }) + + await act(async () => { + rejectStale?.(new Error('stale catalog failure')) + await Promise.resolve() + }) + + expect(replacement).toMatchObject({ client, state: 'connected' }) + } finally { + act(() => renderer?.unmount()) + } + }) +}) diff --git a/mobile/src/transport/host-open-retry-scheduler.test.ts b/mobile/src/transport/host-open-retry-scheduler.test.ts new file mode 100644 index 00000000000..6ee2b18bea5 --- /dev/null +++ b/mobile/src/transport/host-open-retry-scheduler.test.ts @@ -0,0 +1,57 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { HostOpenRetryScheduler } from './host-open-retry-scheduler' + +describe('HostOpenRetryScheduler', () => { + beforeEach(() => vi.useFakeTimers()) + afterEach(() => vi.useRealTimers()) + + it('advances through bounded retry tiers', async () => { + let generation = 1 + const open = vi.fn() + const scheduler = new HostOpenRetryScheduler({ + canRetry: (_hostId, candidate) => candidate === generation, + open + }) + + scheduler.recordFailure('host-1', generation) + await vi.advanceTimersByTimeAsync(1_000) + expect(open).toHaveBeenCalledOnce() + + generation = 2 + scheduler.recordFailure('host-1', generation) + await vi.advanceTimersByTimeAsync(1_999) + expect(open).toHaveBeenCalledOnce() + await vi.advanceTimersByTimeAsync(1) + expect(open).toHaveBeenCalledTimes(2) + }) + + it('expedites without forgiving the failure streak', async () => { + let generation = 1 + const open = vi.fn() + const scheduler = new HostOpenRetryScheduler({ + canRetry: (_hostId, candidate) => candidate === generation, + open + }) + + scheduler.recordFailure('host-1', generation) + scheduler.expedite('host-1') + expect(open).toHaveBeenCalledOnce() + + generation = 2 + scheduler.recordFailure('host-1', generation) + await vi.advanceTimersByTimeAsync(1_999) + expect(open).toHaveBeenCalledOnce() + await vi.advanceTimersByTimeAsync(1) + expect(open).toHaveBeenCalledTimes(2) + }) + + it('cancels retry delivery', async () => { + const open = vi.fn() + const scheduler = new HostOpenRetryScheduler({ canRetry: () => true, open }) + scheduler.recordFailure('host-1', 1) + scheduler.cancel('host-1') + + await vi.advanceTimersByTimeAsync(60_000) + expect(open).not.toHaveBeenCalled() + }) +}) diff --git a/mobile/src/transport/host-open-retry-scheduler.ts b/mobile/src/transport/host-open-retry-scheduler.ts new file mode 100644 index 00000000000..94bf1cc5843 --- /dev/null +++ b/mobile/src/transport/host-open-retry-scheduler.ts @@ -0,0 +1,79 @@ +const RETRY_DELAYS_MS = [1_000, 2_000, 5_000, 15_000, 30_000, 60_000] as const + +type RetryState = { + failureCount: number + generation: number + timer: ReturnType | null +} + +type HostOpenRetrySchedulerOptions = { + canRetry: (hostId: string, generation: number) => boolean + open: (hostId: string) => void + setTimer?: typeof setTimeout + clearTimer?: typeof clearTimeout +} + +export class HostOpenRetryScheduler { + private readonly states = new Map() + private readonly setTimer: typeof setTimeout + private readonly clearTimer: typeof clearTimeout + + constructor(private readonly options: HostOpenRetrySchedulerOptions) { + this.setTimer = options.setTimer ?? setTimeout + this.clearTimer = options.clearTimer ?? clearTimeout + } + + recordFailure(hostId: string, generation: number): { failureCount: number; nextDelayMs: number } { + const previous = this.states.get(hostId) + this.clearStateTimer(previous) + const failureCount = (previous?.failureCount ?? 0) + 1 + const state: RetryState = { failureCount, generation, timer: null } + this.states.set(hostId, state) + const delayIndex = Math.min(failureCount - 1, RETRY_DELAYS_MS.length - 1) + const nextDelayMs = RETRY_DELAYS_MS[delayIndex] + if (!this.options.canRetry(hostId, generation)) { + return { failureCount, nextDelayMs } + } + state.timer = this.setTimer(() => { + state.timer = null + if (this.states.get(hostId) === state && this.options.canRetry(hostId, generation)) { + this.options.open(hostId) + } + }, nextDelayMs) + return { failureCount, nextDelayMs } + } + + expedite(hostId: string): void { + const state = this.states.get(hostId) + if (!state || !this.options.canRetry(hostId, state.generation)) { + return + } + this.clearStateTimer(state) + this.options.open(hostId) + } + + recordSuccess(hostId: string): number { + const priorFailureCount = this.states.get(hostId)?.failureCount ?? 0 + this.cancel(hostId) + return priorFailureCount + } + + cancel(hostId: string): void { + this.clearStateTimer(this.states.get(hostId)) + this.states.delete(hostId) + } + + cancelAll(): void { + for (const state of this.states.values()) { + this.clearStateTimer(state) + } + this.states.clear() + } + + private clearStateTimer(state: RetryState | undefined): void { + if (state?.timer !== null && state?.timer !== undefined) { + this.clearTimer(state.timer) + state.timer = null + } + } +} diff --git a/mobile/src/transport/host-removal-lifecycle.ts b/mobile/src/transport/host-removal-lifecycle.ts index 64007cb9552..cd0a09cb67e 100644 --- a/mobile/src/transport/host-removal-lifecycle.ts +++ b/mobile/src/transport/host-removal-lifecycle.ts @@ -6,12 +6,12 @@ import { removeHost } from './host-store' export async function removeHostAndCloseClient( hostId: string, - closeHostClient: (hostId: string) => void + forgetHostClient: (hostId: string) => void ): Promise { // Why: closing before the metadata commit can strand a still-paired host on // storage failure; closing immediately after success prevents socket leaks. await removeHost(hostId) - closeHostClient(hostId) + forgetHostClient(hostId) // Why: the notification session outlives the socket by design (it must survive // reconnects), so removal is the only thing that can retire it. Left behind, a // re-pair of the same host would inherit a watermark for a counter it never saw. diff --git a/mobile/src/transport/mobile-endpoint-nudge-router.test.ts b/mobile/src/transport/mobile-endpoint-nudge-router.test.ts index a8014a90228..808ad53f6c8 100644 --- a/mobile/src/transport/mobile-endpoint-nudge-router.test.ts +++ b/mobile/src/transport/mobile-endpoint-nudge-router.test.ts @@ -19,12 +19,10 @@ function routerFixture() { const router = new MobileEndpointNudgeRouter({ logical, controller: { handleActiveNudge } as unknown as RelayReconnectController, - now: () => 20_000, isStopped: () => false, isForeground: () => foreground, setForeground, replaceRelay: vi.fn(), - recoverAfterDeadProbe: vi.fn(), scheduleDirectProbe }) return { handleActiveNudge, logical, router, scheduleDirectProbe, setForeground } @@ -38,9 +36,7 @@ describe('MobileEndpointNudgeRouter', () => { expect(fixture.setForeground).toHaveBeenCalledWith(true) expect(fixture.handleActiveNudge).toHaveBeenCalledWith(fixture.logical, 'focus') - expect(fixture.logical.sendRequest).toHaveBeenCalledWith('status.get', null, { - timeoutMs: 4000 - }) + expect(fixture.logical.sendRequest).not.toHaveBeenCalled() expect(fixture.scheduleDirectProbe).toHaveBeenCalledOnce() }) diff --git a/mobile/src/transport/mobile-endpoint-nudge-router.ts b/mobile/src/transport/mobile-endpoint-nudge-router.ts index d59c99b7c83..589992629cf 100644 --- a/mobile/src/transport/mobile-endpoint-nudge-router.ts +++ b/mobile/src/transport/mobile-endpoint-nudge-router.ts @@ -1,4 +1,3 @@ -import { MobileRelayFocusProbe } from './mobile-relay-focus-probe' import type { RelayReconnectController } from './mobile-relay-reconnect-controller' import type { StableLogicalRpcClient } from './stable-logical-rpc-client' import type { ForegroundNudgeReason } from './types' @@ -6,28 +5,17 @@ import type { ForegroundNudgeReason } from './types' // Routes attention/network nudges: focus and app-resume probe a healthy relay, // a network change replaces it make-before-break, everything else re-enters recovery. export class MobileEndpointNudgeRouter { - private readonly focusProbe: MobileRelayFocusProbe - constructor( private readonly args: { logical: StableLogicalRpcClient controller: RelayReconnectController - now: () => number isStopped: () => boolean isForeground: () => boolean setForeground: (foreground: boolean) => void replaceRelay: () => void - recoverAfterDeadProbe: (detail: string) => void scheduleDirectProbe: () => void } - ) { - this.focusProbe = new MobileRelayFocusProbe({ - logical: args.logical, - now: args.now, - canProbe: () => !args.isStopped() && args.isForeground(), - onDead: args.recoverAfterDeadProbe - }) - } + ) {} nudge(reason: ForegroundNudgeReason): void { const { args } = this @@ -43,9 +31,7 @@ export class MobileEndpointNudgeRouter { args.setForeground(true) } const verdict = args.controller.handleActiveNudge(args.logical, reason) - if (verdict === 'probe') { - void this.focusProbe.probe() - } else if (verdict === 'replace') { + if (verdict === 'replace') { args.replaceRelay() } args.scheduleDirectProbe() diff --git a/mobile/src/transport/mobile-endpoint-supervisor-nudge.test.ts b/mobile/src/transport/mobile-endpoint-supervisor-nudge.test.ts index b839732cf77..704dc56e315 100644 --- a/mobile/src/transport/mobile-endpoint-supervisor-nudge.test.ts +++ b/mobile/src/transport/mobile-endpoint-supervisor-nudge.test.ts @@ -89,7 +89,7 @@ describe('mobile endpoint supervisor nudges', () => { supervisor.stop() }) - it('probes a healthy relay on a focus nudge and keeps it untouched', async () => { + it('leaves physical relay probing to the session watchdog', async () => { const logical = new FakeLogicalClient('connected', 'relay') const deps = dependencies() const supervisor = new MobileEndpointSupervisor(logical, host, deps) @@ -97,28 +97,27 @@ describe('mobile endpoint supervisor nudges', () => { supervisor.nudge('focus') await vi.advanceTimersByTimeAsync(0) - expect(logical.sendRequest).toHaveBeenCalledWith('status.get', null, { timeoutMs: 4000 }) + expect(logical.sendRequest).not.toHaveBeenCalled() expect(logical.suspendActiveSession).not.toHaveBeenCalled() expect(deps.openRelay).not.toHaveBeenCalled() - // Repeated focus events inside the probe window coalesce into one probe. + // Repeated focus events never create a second supervisor-owned liveness policy. supervisor.nudge('focus') await vi.advanceTimersByTimeAsync(0) - expect(logical.sendRequest).toHaveBeenCalledOnce() + expect(logical.sendRequest).not.toHaveBeenCalled() supervisor.stop() }) - it('suspends and re-dials when the focus probe fails', async () => { + it('does not suspend a relay from one focus RPC failure', async () => { const logical = new FakeLogicalClient('connected', 'relay') const deps = dependencies() const supervisor = new MobileEndpointSupervisor(logical, host, deps) await supervisor.start() - logical.sendRequest.mockRejectedValueOnce(new Error('relay RPC timed out: status.get')) supervisor.nudge('focus') await vi.advanceTimersByTimeAsync(0) - expect(logical.suspendActiveSession).toHaveBeenCalledOnce() - await vi.waitFor(() => expect(deps.openRelay).toHaveBeenCalledOnce()) + expect(logical.suspendActiveSession).not.toHaveBeenCalled() + expect(deps.openRelay).not.toHaveBeenCalled() expect(logical.getState()).toBe('connected') supervisor.stop() }) diff --git a/mobile/src/transport/mobile-endpoint-supervisor.ts b/mobile/src/transport/mobile-endpoint-supervisor.ts index 13cb1293e8e..84a3526d022 100644 --- a/mobile/src/transport/mobile-endpoint-supervisor.ts +++ b/mobile/src/transport/mobile-endpoint-supervisor.ts @@ -60,16 +60,10 @@ export class MobileEndpointSupervisor { this.nudgeRouter = new MobileEndpointNudgeRouter({ logical, controller: this.relayReconnect, - now: dependencies.now, isStopped: () => this.stopped, isForeground: () => this.foreground, setForeground: (foreground) => this.setForeground(foreground), replaceRelay: () => void this.recoverRelay(true, true), - recoverAfterDeadProbe: (detail) => { - this.logRelay('relay probe failed; recovering', detail) - suspendRelayIfStillConnected(this.relayReconnect, this.logical) - void this.recoverRelay() - }, scheduleDirectProbe: () => this.directProbe.schedule(0) }) this.leaseRotation = new RelayLeaseRotationTimer(dependencies, () => { diff --git a/mobile/src/transport/mobile-relay-e2ee-link.test.ts b/mobile/src/transport/mobile-relay-e2ee-link.test.ts new file mode 100644 index 00000000000..d9093568fc1 --- /dev/null +++ b/mobile/src/transport/mobile-relay-e2ee-link.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it, vi } from 'vitest' + +vi.mock('./mobile-e2ee-v2-client-session', () => ({ + MobileE2EEV2ClientSession: { + create: () => ({}) + } +})) + +vi.mock('./mobile-e2ee-v2-physical-channel', () => ({ + MobileE2EEV2PhysicalChannel: class { + start = vi.fn() + handleMessage = vi.fn(async () => {}) + sendText = vi.fn(() => true) + sendBinary = vi.fn(() => true) + dispose = vi.fn() + } +})) + +import { MobileRelayE2eeLink } from './mobile-relay-e2ee-link' + +class ThrowingSocket { + static readonly OPEN = 1 + readonly OPEN = ThrowingSocket.OPEN + readyState = ThrowingSocket.OPEN + bufferedAmount = 0 + onopen: (() => void) | null = null + onmessage: ((event: { data: unknown }) => void) | null = null + onerror: (() => void) | null = null + onclose: ((event: { code: number }) => void) | null = null + send = vi.fn(() => { + throw new Error('relay auth write failed') + }) + close = vi.fn() +} + +describe('MobileRelayE2eeLink', () => { + it('routes the initial relay-auth write exception through link failure', () => { + const socket = new ThrowingSocket() + const onError = vi.fn() + new MobileRelayE2eeLink({ + endpoint: { + cellUrl: 'https://relay-c1.onorca.dev', + relayHostId: 'AbCdEf0123_-xyZ9' + }, + credential: 'credential', + expectedCredentialKind: 'resume', + deviceToken: 'device-token', + desktopPublicKeyB64: 'desktop-key', + onAuthenticated: vi.fn(), + onText: vi.fn(), + onBinary: vi.fn(), + onError, + createSocket: () => socket as unknown as WebSocket + }) + + expect(() => socket.onopen?.()).not.toThrow() + expect(onError).toHaveBeenCalledWith( + expect.objectContaining({ message: 'relay auth write failed' }) + ) + expect(socket.close).toHaveBeenCalledOnce() + }) +}) diff --git a/mobile/src/transport/mobile-relay-e2ee-link.ts b/mobile/src/transport/mobile-relay-e2ee-link.ts index 183a8deea6d..80d91589c53 100644 --- a/mobile/src/transport/mobile-relay-e2ee-link.ts +++ b/mobile/src/transport/mobile-relay-e2ee-link.ts @@ -76,14 +76,18 @@ export class MobileRelayE2eeLink { private bindSocket(): void { this.socket.onopen = () => { - this.socket.send( - JSON.stringify({ - type: 'relay-auth', - v: 1, - mode: 'connect', - credential: this.options.credential - }) - ) + try { + this.socket.send( + JSON.stringify({ + type: 'relay-auth', + v: 1, + mode: 'connect', + credential: this.options.credential + }) + ) + } catch (error) { + this.fail(asError(error)) + } } this.socket.onmessage = (event) => { this.inboundChain = this.inboundChain diff --git a/mobile/src/transport/mobile-relay-focus-probe.ts b/mobile/src/transport/mobile-relay-focus-probe.ts deleted file mode 100644 index 2671a62cd67..00000000000 --- a/mobile/src/transport/mobile-relay-focus-probe.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { - isLogicalClientCutoverError, - type StableLogicalRpcClient -} from './stable-logical-rpc-client' - -// Why: a probe must fail well before the 12s connect ceiling so a dead socket -// still recovers in a few seconds, but outlast one slow cellular RTT. -const PROBE_TIMEOUT_MS = 4_000 -// Why: focus fires per navigation event; one liveness answer covers them all. -const PROBE_MIN_INTERVAL_MS = 10_000 - -// Cheap liveness check for a focus nudge on an active relay: a healthy session -// answers and nothing visible changes; only a dead one is handed to onDead. -export class MobileRelayFocusProbe { - private lastProbeAt = 0 - - constructor( - private readonly args: { - logical: StableLogicalRpcClient - now: () => number - canProbe: () => boolean - onDead: (detail: string) => void - } - ) {} - - async probe(): Promise { - const { logical, now } = this.args - const at = now() - if ( - !this.args.canProbe() || - at - this.lastProbeAt < PROBE_MIN_INTERVAL_MS || - logical.getActivePath() !== 'relay' || - logical.getState() !== 'connected' - ) { - return - } - this.lastProbeAt = at - const generation = logical.getGeneration() - try { - await logical.sendRequest('status.get', null, { timeoutMs: PROBE_TIMEOUT_MS }) - } catch (error) { - if ( - isLogicalClientCutoverError(error) || - !this.args.canProbe() || - logical.getGeneration() !== generation || - logical.getActivePath() !== 'relay' - ) { - return - } - this.args.onDead(String((error as Error).message).slice(0, 80)) - } - } -} diff --git a/mobile/src/transport/mobile-relay-rpc-session-liveness.test.ts b/mobile/src/transport/mobile-relay-rpc-session-liveness.test.ts new file mode 100644 index 00000000000..b7073075b2b --- /dev/null +++ b/mobile/src/transport/mobile-relay-rpc-session-liveness.test.ts @@ -0,0 +1,173 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +const fakes = vi.hoisted(() => ({ + linkOptions: null as null | { + onHello(value: unknown): void + onAuthenticated(): void + onText(value: string): void + onBinary(value: Uint8Array): void + }, + sendText: vi.fn(() => true), + close: vi.fn() +})) + +vi.mock('./mobile-relay-e2ee-link', () => ({ + MobileRelayE2eeLink: class { + constructor(options: NonNullable) { + fakes.linkOptions = options + } + sendText = fakes.sendText + close = fakes.close + } +})) + +import { connectMobileRelayRpcSession } from './mobile-relay-rpc-session' + +const relay = { + v: 1 as const, + directorUrl: 'https://relay.onorca.dev', + cellUrl: 'https://relay-c1.onorca.dev', + assignmentEpoch: 7, + relayHostId: 'AbCdEf0123_-xyZ9', + e2eeFraming: 2 as const +} + +async function authenticateSession() { + const session = connectMobileRelayRpcSession({ + relay, + resumeToken: 'resume-secret', + resumeCredentialVersion: 3, + resumeConfirmReqId: 'confirm-1', + deviceToken: 'device-token', + desktopPublicKeyB64: 'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=', + requestTimeoutMs: 30_000 + }) + fakes.linkOptions!.onHello({ + type: 'relay-hello', + ok: true, + credentialKind: 'resume', + leaseExpiresAt: Date.now() + 60_000, + acceptedCredentialVersion: 3, + acceptedAs: 'current', + resumeExpiresAt: Date.now() + 300_000 + }) + fakes.linkOptions!.onAuthenticated() + await vi.waitFor(() => expect(fakes.sendText).toHaveBeenCalledOnce()) + const confirmation = sentRequests()[0]! + fakes.linkOptions!.onText( + JSON.stringify({ + id: confirmation.id, + ok: true, + result: { + v: 1, + relay, + resumeConfirmation: { + v: 1, + reqId: 'confirm-1', + currentVersion: 3, + acceptedAs: 'current', + renewed: true, + resumeExpiresAt: Date.now() + 300_000 + } + }, + _meta: { runtimeId: 'runtime-1' } + }) + ) + await vi.waitFor(() => expect(session.getState()).toBe('connected')) + fakes.sendText.mockClear() + return session +} + +function sentRequests(): Array<{ id: string; method: string }> { + return fakes.sendText.mock.calls.map( + ([value]) => JSON.parse(value as string) as { id: string; method: string } + ) +} + +describe('mobile relay RPC session liveness', () => { + beforeEach(() => { + vi.useFakeTimers() + vi.clearAllMocks() + fakes.linkOptions = null + fakes.sendText.mockReturnValue(true) + }) + afterEach(() => vi.useRealTimers()) + + it('sends no periodic traffic while an authenticated relay is idle', async () => { + const session = await authenticateSession() + + await vi.advanceTimersByTimeAsync(60_000) + + expect(fakes.sendText).not.toHaveBeenCalled() + expect(session.getState()).toBe('connected') + session.close() + }) + + it('disconnects after two fair foreground misses', async () => { + const session = await authenticateSession() + + session.notifyForeground('focus') + expect(sentRequests().map(({ method }) => method)).toEqual(['status.get']) + await vi.advanceTimersByTimeAsync(4_000) + expect(session.getState()).toBe('connected') + expect(fakes.sendText).toHaveBeenCalledTimes(2) + await vi.advanceTimersByTimeAsync(4_000) + + expect(session.getState()).toBe('disconnected') + expect(fakes.close).toHaveBeenCalledOnce() + }) + + it('rate-limits foreground sequences without suppressing a retry', async () => { + const session = await authenticateSession() + session.notifyForeground('focus') + const firstProbe = sentRequests()[0]! + fakes.linkOptions!.onText( + JSON.stringify({ id: firstProbe.id, ok: true, result: {}, _meta: { runtimeId: 'r1' } }) + ) + + session.notifyForeground('focus') + await vi.advanceTimersByTimeAsync(9_999) + session.notifyForeground('app-resume') + expect(fakes.sendText).toHaveBeenCalledOnce() + await vi.advanceTimersByTimeAsync(1) + session.notifyForeground('focus') + + expect(fakes.sendText).toHaveBeenCalledTimes(2) + session.close() + }) + + it('does not probe the old relay on a network change', async () => { + const session = await authenticateSession() + + session.notifyForeground('network-change') + + expect(fakes.sendText).not.toHaveBeenCalled() + session.close() + }) + + it('does not probe when work follows prolonged inbound silence', async () => { + const session = await authenticateSession() + await vi.advanceTimersByTimeAsync(60_000) + + const pending = session.sendRequest('terminal.send', { terminal: 'term', text: 'hi' }) + const outcome = pending.catch(() => undefined) + session.subscribe('terminal.subscribe', { terminal: 'term' }, vi.fn()) + await vi.advanceTimersByTimeAsync(0) + + expect(sentRequests().map(({ method }) => method)).toEqual([ + 'terminal.send', + 'terminal.subscribe' + ]) + session.close() + await outcome + }) + + it('fails immediately when a liveness probe cannot be written', async () => { + const session = await authenticateSession() + fakes.sendText.mockReturnValue(false) + + session.notifyForeground('focus') + + expect(session.getState()).toBe('disconnected') + expect(fakes.close).toHaveBeenCalledOnce() + }) +}) diff --git a/mobile/src/transport/mobile-relay-rpc-session.test.ts b/mobile/src/transport/mobile-relay-rpc-session.test.ts index f5fa49a785e..d5b547885cf 100644 --- a/mobile/src/transport/mobile-relay-rpc-session.test.ts +++ b/mobile/src/transport/mobile-relay-rpc-session.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { BrowserScreencastOpcode, encodeBrowserScreencastFrame @@ -103,6 +103,7 @@ describe('mobile relay RPC session', () => { fakes.linkOptions = null fakes.sendText.mockReturnValue(true) }) + afterEach(() => vi.useRealTimers()) it('requires exact resume observations and confirms by request ID before becoming connected', async () => { const { session, confirmationRequest } = await authenticateSession() diff --git a/mobile/src/transport/mobile-relay-rpc-session.ts b/mobile/src/transport/mobile-relay-rpc-session.ts index ac065012203..753f6f8fe26 100644 --- a/mobile/src/transport/mobile-relay-rpc-session.ts +++ b/mobile/src/transport/mobile-relay-rpc-session.ts @@ -9,9 +9,14 @@ import { MobileE2EEAuthenticationError } from './mobile-e2ee-v2-physical-channel import { markRpcDeliveryUnknown } from './rpc-delivery-ambiguity' import { openRpcRequestBudget, resolvePostConnectRequestTimeout } from './rpc-request-budget' import { isRpcResponse } from './rpc-response-shape' +import { RpcSessionLivenessWatchdog } from './rpc-session-liveness-watchdog' import type { RpcClient } from './rpc-client' import type { ConnectionState, RpcResponse } from './types' +const RELAY_PROBE_TIMEOUT_MS = 4_000 +const RELAY_MISSED_PROBE_LIMIT = 2 +const RELAY_FOREGROUND_PROBE_MIN_INTERVAL_MS = 10_000 + type PendingRequest = { resolve: (response: RpcResponse) => void reject: (error: Error) => void @@ -48,6 +53,7 @@ export function connectMobileRelayRpcSession(args: { let resumeConfirmation: DeviceResumeConfirmed | null = null let failure: Error | null = null let closed = false + const livenessIdentity = {} const streams = new MobileRelayRpcStreams({ nextId, sendFrame, @@ -74,8 +80,14 @@ export function connectMobileRelayRpcSession(args: { publishState('handshaking') }, onAuthenticated: () => void confirmResume(), - onText: handleText, - onBinary: handleBinary, + onText: (plaintext) => { + livenessWatchdog.noteAuthenticatedInbound(livenessIdentity) + handleText(plaintext) + }, + onBinary: (plaintext) => { + livenessWatchdog.noteAuthenticatedInbound(livenessIdentity) + handleBinary(plaintext) + }, onError: fail }) @@ -103,12 +115,17 @@ export function connectMobileRelayRpcSession(args: { stateListeners.add(listener) return () => stateListeners.delete(listener) }, - notifyForeground: () => {}, + notifyForeground: (reason) => { + if (state === 'connected' && reason !== 'network-change') { + livenessWatchdog.probeNow(livenessIdentity) + } + }, close() { if (closed) { return } closed = true + livenessWatchdog.stop(livenessIdentity) link.close() rejectPending(new Error('Client closed')) streams.clear() @@ -119,6 +136,16 @@ export function connectMobileRelayRpcSession(args: { getResumeConfirmation: () => resumeConfirmation, getFailure: () => failure } + const livenessWatchdog = new RpcSessionLivenessWatchdog({ + transport: 'relay', + idleProbeMs: null, + probeTimeoutMs: RELAY_PROBE_TIMEOUT_MS, + missedProbeLimit: RELAY_MISSED_PROBE_LIMIT, + voluntaryProbeMinIntervalMs: RELAY_FOREGROUND_PROBE_MIN_INTERVAL_MS, + sendProbe: () => + state === 'connected' && sendFrame({ id: nextId(), method: 'status.get', params: undefined }), + terminate: () => fail(new Error('relay session liveness timeout')) + }) return client async function confirmResume(): Promise { @@ -139,6 +166,7 @@ export function connectMobileRelayRpcSession(args: { resumeConfirmation = result.resumeConfirmation resumeExpiresAt = result.resumeConfirmation.resumeExpiresAt lastConnectedAt = Date.now() + livenessWatchdog.start(livenessIdentity) publishState('connected') } catch (error) { fail(asError(error)) @@ -242,6 +270,7 @@ export function connectMobileRelayRpcSession(args: { } closed = true failure = error + livenessWatchdog.stop(livenessIdentity) link.close() rejectPending(error) publishState(error instanceof MobileE2EEAuthenticationError ? 'auth-failed' : 'disconnected') diff --git a/mobile/src/transport/mobile-relay-rpc-streams.ts b/mobile/src/transport/mobile-relay-rpc-streams.ts index 7c485b27e02..2eacbd2168f 100644 --- a/mobile/src/transport/mobile-relay-rpc-streams.ts +++ b/mobile/src/transport/mobile-relay-rpc-streams.ts @@ -57,8 +57,10 @@ export class MobileRelayRpcStreams { void this.options .waitForConnected() .then(() => { - if (!stream.cancelled && !this.options.sendFrame({ id, method, params: stream.params })) { - this.fail(id, stream, 'Connection interrupted') + if (!stream.cancelled) { + if (!this.options.sendFrame({ id, method, params: stream.params })) { + this.fail(id, stream, 'Connection interrupted') + } } }) .catch((error: unknown) => { @@ -114,8 +116,7 @@ export class MobileRelayRpcStreams { } handleTerminalBinaryFrame(bytes, { terminalSnapshots: this.terminalSnapshots, - getListener: (streamId) => this.terminalListeners.get(streamId), - recordValidatedInboundTraffic: () => {} + getListener: (streamId) => this.terminalListeners.get(streamId) }) } diff --git a/mobile/src/transport/rpc-client-activity-probe.ts b/mobile/src/transport/rpc-client-activity-probe.ts deleted file mode 100644 index 02dc4362fbd..00000000000 --- a/mobile/src/transport/rpc-client-activity-probe.ts +++ /dev/null @@ -1,87 +0,0 @@ -import type { ConnectionState } from './types' - -// Why: RN auto-pongs pings natively, so JS needs an app-level probe to detect half-open sockets. -const ACTIVITY_PROBE_INTERVAL_MS = 20_000 -const ACTIVITY_PROBE_TIMEOUT_MS = 8_000 - -export type RpcActivityProbeDependencies = { - getState: () => ConnectionState - getSocket: () => WebSocket | null - // Counts only validated inbound traffic, so a malformed frame cannot pass for liveness. - getInboundSequence: () => number - nextId: () => string - registerPending: (id: string, onSettled: () => void) => void - clearPending: (id: string) => void - sendProbe: (id: string) => boolean - forceReconnect: (socket: WebSocket) => void -} - -export type RpcActivityProbe = { - start: () => void - stop: () => void - run: () => void -} - -export function createRpcActivityProbe( - dependencies: RpcActivityProbeDependencies -): RpcActivityProbe { - let timer: ReturnType | null = null - let inFlight = false - - function stop(): void { - if (timer) { - clearInterval(timer) - timer = null - } - } - - function run(): void { - const probeWs = dependencies.getSocket() - if (dependencies.getState() !== 'connected' || !probeWs || inFlight) { - return - } - inFlight = true - const id = dependencies.nextId() - const probeInboundSequence = dependencies.getInboundSequence() - let timedOut = false - const timeout = setTimeout(() => { - timedOut = true - inFlight = false - dependencies.clearPending(id) - // Why: any validated frame answered for the socket, even if this probe did not. - if (dependencies.getInboundSequence() > probeInboundSequence) { - return - } - console.log('[net] activity-probe TIMEOUT — forcing reconnect', { - state: dependencies.getState() - }) - // Why: stale probe timers must not close a replacement socket. - if (probeWs === dependencies.getSocket() && probeWs.readyState === WebSocket.OPEN) { - dependencies.forceReconnect(probeWs) - } - }, ACTIVITY_PROBE_TIMEOUT_MS) - - dependencies.registerPending(id, () => { - if (timedOut) { - return - } - inFlight = false - clearTimeout(timeout) - }) - - if (!dependencies.sendProbe(id)) { - inFlight = false - clearTimeout(timeout) - dependencies.clearPending(id) - } - } - - return { - run, - stop, - start(): void { - stop() - timer = setInterval(run, ACTIVITY_PROBE_INTERVAL_MS) - } - } -} diff --git a/mobile/src/transport/rpc-client-context-contract.ts b/mobile/src/transport/rpc-client-context-contract.ts new file mode 100644 index 00000000000..77ef745eef0 --- /dev/null +++ b/mobile/src/transport/rpc-client-context-contract.ts @@ -0,0 +1,28 @@ +import type { HostClientAcquisition } from './host-client-acquisition-registry' +import type { RpcClient } from './rpc-client' +import type { MobileConnectionPath } from './stable-logical-rpc-client' +import type { ConnectionState, HostProfile } from './types' + +export type RpcClientContextValue = { + acquire: ( + hostId: string, + acquisition: HostClientAcquisition, + host?: HostProfile + ) => RpcClient | null + release: (hostId: string, acquisition: HostClientAcquisition) => void + releaseAndCloseIfUnused: (hostId: string, acquisition: HostClientAcquisition) => void + closeIfUnused: (hostId: string) => void + forceReconnect: (hostId: string) => Promise + refreshHostClient: (hostId: string) => void + forgetHostClient: (hostId: string) => void + disconnectHostClient: (hostId: string) => void + getState: (hostId: string) => ConnectionState + getKnownState: (hostId: string) => ConnectionState | null + getReconnectAttempt: (hostId: string) => number + getLastConnectedAt: (hostId: string) => number | null + getActivePath: (hostId: string) => MobileConnectionPath + subscribeHostState: (hostId: string, listener: (state: ConnectionState) => void) => () => void + getAllClients: () => { hostId: string; client: RpcClient }[] + subscribeAllHosts: (listener: () => void) => () => void + primeHosts: (hosts: HostProfile[]) => void +} diff --git a/mobile/src/transport/rpc-client-live-recovery.test.ts b/mobile/src/transport/rpc-client-live-recovery.test.ts index da73fc0b5fd..471a53be740 100644 --- a/mobile/src/transport/rpc-client-live-recovery.test.ts +++ b/mobile/src/transport/rpc-client-live-recovery.test.ts @@ -151,14 +151,13 @@ describe.runIf(RUN_LIVE)('live foreground recovery (issue #5049)', () => { // comes back to the foreground. blackhole = true c.notifyForeground() - // Foreground probe budget is 8s; the interval probe alone would take - // up to 28s. Allow scheduling slack but stay well under 28s. + // Three fair probe windows tolerate transient mobile/Tailscale stalls. const detectMs = await waitFor( 'half-open detected', - 15_000, + 32_000, () => c.getState() !== 'connected' ) - expect(detectMs).toBeLessThan(12_000) + expect(detectMs).toBeLessThan(30_000) blackhole = false await waitFor('recovered after link healed', 15_000, () => c.getState() === 'connected') diff --git a/mobile/src/transport/rpc-client-terminal-binary-frame.test.ts b/mobile/src/transport/rpc-client-terminal-binary-frame.test.ts index 216fd2aec56..973522ec400 100644 --- a/mobile/src/transport/rpc-client-terminal-binary-frame.test.ts +++ b/mobile/src/transport/rpc-client-terminal-binary-frame.test.ts @@ -18,18 +18,15 @@ function encodeFrame(opcode: TerminalStreamOpcode, streamId: number, payload: un describe('handleTerminalBinaryFrame', () => { it('routes terminal metadata frames to the stream listener', () => { const listener = vi.fn() - const recordValidatedInboundTraffic = vi.fn() handleTerminalBinaryFrame( encodeFrame(TerminalStreamOpcode.Metadata, 42, { cwd: '/repo/src' }), { terminalSnapshots: new Map(), - getListener: (streamId) => (streamId === 42 ? listener : undefined), - recordValidatedInboundTraffic + getListener: (streamId) => (streamId === 42 ? listener : undefined) } ) - expect(recordValidatedInboundTraffic).toHaveBeenCalledTimes(1) expect(listener).toHaveBeenCalledWith({ type: 'metadata', streamId: 42, cwd: '/repo/src' }) }) }) diff --git a/mobile/src/transport/rpc-client-terminal-binary-frame.ts b/mobile/src/transport/rpc-client-terminal-binary-frame.ts index ee7da35383e..8e252d7136f 100644 --- a/mobile/src/transport/rpc-client-terminal-binary-frame.ts +++ b/mobile/src/transport/rpc-client-terminal-binary-frame.ts @@ -16,7 +16,6 @@ type StreamingListener = (result: unknown) => void type TerminalBinaryFrameOptions = { terminalSnapshots: Map getListener: (streamId: number) => StreamingListener | undefined - recordValidatedInboundTraffic: () => void } export function handleTerminalBinaryFrame( @@ -29,11 +28,9 @@ export function handleTerminalBinaryFrame( } const listener = options.getListener(frame.streamId) if (!listener) { - options.recordValidatedInboundTraffic() return } if (frame.opcode === TerminalStreamOpcode.Output) { - options.recordValidatedInboundTraffic() listener({ type: 'data', streamId: frame.streamId, @@ -46,7 +43,6 @@ export function handleTerminalBinaryFrame( if (!meta) { return } - options.recordValidatedInboundTraffic() options.terminalSnapshots.set(frame.streamId, { streamId: frame.streamId, meta, @@ -55,7 +51,6 @@ export function handleTerminalBinaryFrame( return } if (frame.opcode === TerminalStreamOpcode.SnapshotChunk) { - options.recordValidatedInboundTraffic() const snapshot = options.terminalSnapshots.get(frame.streamId) if (!snapshot) { return @@ -64,7 +59,6 @@ export function handleTerminalBinaryFrame( return } if (frame.opcode === TerminalStreamOpcode.SnapshotEnd) { - options.recordValidatedInboundTraffic() const snapshot = options.terminalSnapshots.get(frame.streamId) if (!snapshot) { return @@ -84,7 +78,6 @@ export function handleTerminalBinaryFrame( if (!meta) { return } - options.recordValidatedInboundTraffic() listener({ ...meta, type: 'resized', @@ -97,7 +90,6 @@ export function handleTerminalBinaryFrame( if (!meta) { return } - options.recordValidatedInboundTraffic() listener({ ...meta, type: 'metadata', @@ -106,7 +98,6 @@ export function handleTerminalBinaryFrame( return } if (frame.opcode === TerminalStreamOpcode.Error) { - options.recordValidatedInboundTraffic() listener({ type: 'error', streamId: frame.streamId, diff --git a/mobile/src/transport/rpc-client.test.ts b/mobile/src/transport/rpc-client.test.ts index 67e920ff1af..a7f3bb9d82a 100644 --- a/mobile/src/transport/rpc-client.test.ts +++ b/mobile/src/transport/rpc-client.test.ts @@ -687,7 +687,7 @@ describe('mobile rpc-client connection timeout', () => { client.close() }) - it('reaps a half-open socket within 8s of foreground', async () => { + it('reaps a half-open socket after three fair foreground probe windows', async () => { const client = connect('ws://desktop.invalid', 'token', 'server-key') const socket = mockSockets[0]! openAndAuthenticate(socket) @@ -696,9 +696,8 @@ describe('mobile rpc-client connection timeout', () => { client.notifyForeground() expect(sentRequests(socket, 'status.get')).toHaveLength(1) - await vi.advanceTimersByTimeAsync(8_000) + await vi.advanceTimersByTimeAsync(24_000) expect(socket.close).toHaveBeenCalled() - expect(client.getState()).toBe('reconnecting') await vi.advanceTimersByTimeAsync(500) openAndAuthenticate(mockSockets[mockSockets.length - 1]!) @@ -714,10 +713,9 @@ describe('mobile rpc-client connection timeout', () => { socket.emitCloseOnClose = false client.notifyForeground() - await vi.advanceTimersByTimeAsync(8_000) + await vi.advanceTimersByTimeAsync(24_000) expect(socket.close).toHaveBeenCalledTimes(1) - expect(client.getState()).toBe('reconnecting') socket.onclose?.() expect(client.getState()).toBe('reconnecting') @@ -740,7 +738,8 @@ describe('mobile rpc-client connection timeout', () => { client.notifyForeground() client.notifyForeground() expect(sentRequests(socket, 'status.get')).toHaveLength(1) - await vi.advanceTimersByTimeAsync(8_000) + await vi.advanceTimersByTimeAsync(24_000) + expect(sentRequests(socket, 'status.get')).toHaveLength(3) expect(socket.close).toHaveBeenCalledTimes(1) expect(client.getState()).toBe('reconnecting') @@ -748,6 +747,7 @@ describe('mobile rpc-client connection timeout', () => { expect(mockSockets).toHaveLength(2) client.close() }) + it('keeps a healthy connection when the foreground probe is answered', async () => { const { client, socket } = connectAuthenticated() @@ -782,13 +782,14 @@ describe('mobile rpc-client connection timeout', () => { await vi.advanceTimersByTimeAsync(8_000) expect(socket.close).not.toHaveBeenCalled() - expect(client.getState()).toBe('connected') await vi.advanceTimersByTimeAsync(12_000) expect(sentRequests(socket, 'status.get')).toHaveLength(2) await vi.advanceTimersByTimeAsync(7_999) expect(socket.close).not.toHaveBeenCalled() await vi.advanceTimersByTimeAsync(1) + expect(socket.close).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(16_000) expect(socket.close).toHaveBeenCalled() expect(client.getState()).toBe('reconnecting') @@ -853,18 +854,29 @@ describe('mobile rpc-client connection timeout', () => { client.close() }) - it('does not count malformed or undecryptable inbound payloads as probe activity', async () => { + it('counts authenticated unknown payloads before semantic decoding', async () => { const { client, socket } = connectAuthenticated() client.notifyForeground() - socket.receive('undecryptable') socket.receive('encrypted:{"unexpected":true}') socket.receive('encrypted:{"id":"rpc-incomplete","ok":true}') socket.receive(new Uint8Array([0xff, 0x00, 0x01])) await vi.advanceTimersByTimeAsync(8_000) + expect(socket.close).not.toHaveBeenCalled() + expect(client.getState()).toBe('connected') + + client.close() + }) + + it('does not count undecryptable payloads as probe activity', async () => { + const { client, socket } = connectAuthenticated() + + client.notifyForeground() + socket.receive('undecryptable') + + await vi.advanceTimersByTimeAsync(24_000) expect(socket.close).toHaveBeenCalled() - expect(client.getState()).toBe('reconnecting') client.close() }) diff --git a/mobile/src/transport/rpc-client.ts b/mobile/src/transport/rpc-client.ts index 9b0afe937f3..e9a4dcb72c9 100644 --- a/mobile/src/transport/rpc-client.ts +++ b/mobile/src/transport/rpc-client.ts @@ -37,7 +37,14 @@ import { import { markRpcDeliveryUnknown } from './rpc-delivery-ambiguity' import { openRpcRequestBudget, resolvePostConnectRequestTimeout } from './rpc-request-budget' import { isRpcResponse } from './rpc-response-shape' -import { createRpcActivityProbe } from './rpc-client-activity-probe' +import { + isStreamingSubscriptionReadyResult, + isTerminalSubscribedResult +} from './rpc-subscription-result-shapes' +import { + RpcSessionLivenessWatchdog, + type RpcSessionIdentity +} from './rpc-session-liveness-watchdog' import { isStaleForegroundDial } from './rpc-stale-dial' import { websocketPayloadToUint8 } from './websocket-payload-bytes' @@ -129,6 +136,7 @@ const CONNECT_TIMEOUT_MS = 12_000 const HANDSHAKE_TIMEOUT_MS = 5_000 // Why: RN may not expose WebSocket.readyState constants, but the CONNECTING protocol value (0) is stable across runtimes. const WEBSOCKET_CONNECTING_STATE = 0 +const LIVENESS_REQUEST_ID_PREFIX = 'mobile-liveness-' export type ConnectOptions = { onStateChange?: (state: ConnectionState) => void @@ -177,7 +185,7 @@ export function connect( let lastConnectedAt: number | null = null // Why: cheap diagnostics for RN/OkHttp process-state poisoning (retry cadence, inbound traffic, close timing). let lastInboundAt: number | null = null - let inboundSequence = 0 + let livenessIdentity: (RpcSessionIdentity & { socket: WebSocket }) | null = null let lastWsClosedAt: number | null = null let wsConstructionCounter = 0 let dialStartedAt = 0 @@ -315,6 +323,7 @@ export function connect( const openingWs = ws let openingWsAuthenticated = false let openingWsLastInboundAt: number | null = null + let openingLivenessIdentity: (RpcSessionIdentity & { socket: WebSocket }) | null = null // Why: RN can leave opens pending forever on flaky handoffs — force reconnect if onopen never arrives. connectTimer = setTimeout(() => { @@ -351,7 +360,12 @@ export function connect( type: 'e2ee_hello', publicKeyB64: publicKeyToBase64(ephemeral.publicKey) }) - openingWs.send(hello) + try { + openingWs.send(hello) + } catch { + closeAndSynthesize(openingWs) + return + } emitLog('info', 'Sent e2ee_hello', 'Awaiting server e2ee_ready') sharedKey = deriveSharedKey(ephemeral.secretKey, serverPublicKey) @@ -419,9 +433,11 @@ export function connect( streamCount: streamListeners.size }) openingWsAuthenticated = true + openingLivenessIdentity = { socket: openingWs } + livenessIdentity = openingLivenessIdentity + livenessWatchdog.start(openingLivenessIdentity) setState('connected') emitLog('success', 'Authenticated', 'Channel ready for RPC') - activityProbe.start() for (const [id, stream] of streamListeners) { if (stream.cancelled) { removeStreamListener(id) @@ -441,8 +457,9 @@ export function connect( ) { stream.sent = true } else { - emitStreamError(stream, 'Connection interrupted') - removeStreamListener(id) + // The failed write already starts recovery; retain every stream for replay. + markStreamsForReplay() + break } } } else if (msg.type === 'e2ee_error' || (!msg.ok && msg.error?.code === 'unauthorized')) { @@ -473,6 +490,9 @@ export function connect( if (!plaintextBytes) { return } + if (openingLivenessIdentity) { + livenessWatchdog.noteAuthenticatedInbound(openingLivenessIdentity) + } handleBinaryFrame(plaintextBytes) return } @@ -481,6 +501,9 @@ export function connect( if (plaintext === null) { return } + if (openingLivenessIdentity) { + livenessWatchdog.noteAuthenticatedInbound(openingLivenessIdentity) + } let response: unknown try { @@ -491,8 +514,9 @@ export function connect( if (!isRpcResponse(response)) { return } - recordValidatedInboundTraffic() - + if (response.id.startsWith(LIVENESS_REQUEST_ID_PREFIX)) { + return + } // Why: a mid-session unauthorized may be transient (issue #5200) — handleAuthRejection retries before latching auth-failed. if (!response.ok && response.error.code === 'unauthorized') { handleAuthRejection('Unauthorized — pairing may be revoked') @@ -641,7 +665,10 @@ export function connect( pendingBrowserScreencastRequestId = null markStreamsForReplay() clearHandshakeTimer() - activityProbe.stop() + if (livenessIdentity?.socket === closedWs) { + livenessWatchdog.stop(livenessIdentity) + livenessIdentity = null + } if (intentionallyClosed) { console.log('[net] handleSocketClosed — intentional close') setState('disconnected') @@ -672,6 +699,10 @@ export function connect( // Why: an auth rejection may be transient (issue #5200) — retry up to AUTH_RETRY_BUDGET times before latching auth-failed. function handleAuthRejection(reason: string, preserveRecovery = false): void { + if (livenessIdentity) { + livenessWatchdog.stop(livenessIdentity) + livenessIdentity = null + } authRejectionCount++ if (authRejectionCount < AUTH_RETRY_BUDGET) { console.log('[net] auth rejected — retrying handshake', { @@ -877,21 +908,15 @@ export function connect( } } - function recordValidatedInboundTraffic(): void { - inboundSequence++ - } - function handleBinaryFrame(bytes: Uint8Array): void { const browserFrame = decodeBrowserScreencastFrame(bytes) if (browserFrame) { - recordValidatedInboundTraffic() handleBrowserBinaryFrame(browserFrame) return } handleTerminalBinaryFrame(bytes, { terminalSnapshots, - getListener: (streamId) => terminalStreamListeners.get(streamId), - recordValidatedInboundTraffic + getListener: (streamId) => terminalStreamListeners.get(streamId) }) } @@ -908,8 +933,16 @@ export function connect( function sendEncrypted(request: unknown): boolean { if (ws && ws.readyState === WebSocket.OPEN && sharedKey) { - ws.send(encrypt(JSON.stringify(request), sharedKey)) - return true + const sendingWs = ws + try { + sendingWs.send(encrypt(JSON.stringify(request), sharedKey)) + return true + } catch { + if (ws === sendingWs) { + closeAndSynthesize(sendingWs) + } + return false + } } console.log('[net] sendEncrypted FAILED — channel not ready', { hasWs: !!ws, @@ -955,15 +988,23 @@ export function connect( } } - const activityProbe = createRpcActivityProbe({ - getState: () => state, - getSocket: () => ws, - getInboundSequence: () => inboundSequence, - nextId, - registerPending: (id, onSettled) => pending.set(id, { resolve: onSettled, reject: onSettled }), - clearPending: (id) => pending.delete(id), - sendProbe: (id) => sendEncrypted({ id, deviceToken, method: 'status.get' }), - forceReconnect: closeAndSynthesize + const livenessWatchdog = new RpcSessionLivenessWatchdog({ + transport: 'direct', + sendProbe: (identity) => { + if (identity !== livenessIdentity || state !== 'connected') { + return false + } + return sendEncrypted({ + id: `${LIVENESS_REQUEST_ID_PREFIX}${nextId()}`, + deviceToken, + method: 'status.get' + }) + }, + terminate: (identity) => { + if (identity === livenessIdentity && livenessIdentity.socket === ws) { + closeAndSynthesize(livenessIdentity.socket) + } + } }) openConnection() @@ -1119,10 +1160,11 @@ export function connect( return } if (state === 'connected') { - // Why: OS can kill the TCP path while backgrounded without onclose; probe now to detect the half-open socket in ≤8s (issue #5049). + // Why: resume probes now; three fair misses detect a half-open socket within 24s. console.log('[net] foreground — probing live connection') - activityProbe.start() - activityProbe.run() + if (livenessIdentity) { + livenessWatchdog.probeNow(livenessIdentity) + } return } const dialing = ws @@ -1157,7 +1199,10 @@ export function connect( } clearConnectTimer() clearHandshakeTimer() - activityProbe.stop() + if (livenessIdentity) { + livenessWatchdog.stop(livenessIdentity) + livenessIdentity = null + } if (ws) { ws.close() ws = null @@ -1169,25 +1214,3 @@ export function connect( } } } - -function isTerminalSubscribedResult( - value: unknown -): value is { type: 'subscribed'; streamId: number } { - return ( - !!value && - typeof value === 'object' && - (value as { type?: unknown }).type === 'subscribed' && - typeof (value as { streamId?: unknown }).streamId === 'number' - ) -} - -function isStreamingSubscriptionReadyResult( - value: unknown -): value is { type: 'ready'; subscriptionId: string } { - return ( - !!value && - typeof value === 'object' && - (value as { type?: unknown }).type === 'ready' && - typeof (value as { subscriptionId?: unknown }).subscriptionId === 'string' - ) -} diff --git a/mobile/src/transport/rpc-session-liveness-integration.test.ts b/mobile/src/transport/rpc-session-liveness-integration.test.ts new file mode 100644 index 00000000000..c446477c365 --- /dev/null +++ b/mobile/src/transport/rpc-session-liveness-integration.test.ts @@ -0,0 +1,188 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { connect } from './rpc-client' +import { encodeTerminalStreamFrame, TerminalStreamOpcode } from './terminal-stream-protocol' + +vi.mock('./e2ee', () => ({ + generateKeyPair: () => ({ + publicKey: new Uint8Array(32), + secretKey: new Uint8Array(32) + }), + deriveSharedKey: () => new Uint8Array(32), + publicKeyFromBase64: () => new Uint8Array(32), + publicKeyToBase64: () => 'client-public-key', + encrypt: (plaintext: string) => `encrypted:${plaintext}`, + decrypt: (raw: string) => raw.replace(/^encrypted:/, ''), + decryptBytes: (bytes: Uint8Array) => bytes +})) + +class MockWebSocket { + static readonly CONNECTING = 0 + static readonly OPEN = 1 + static readonly CLOSED = 3 + + readonly CONNECTING = MockWebSocket.CONNECTING + readonly OPEN = MockWebSocket.OPEN + readonly CLOSED = MockWebSocket.CLOSED + readyState = MockWebSocket.CONNECTING + onopen: (() => void) | null = null + onclose: (() => void) | null = null + onmessage: ((event: { data: unknown }) => void) | null = null + onerror: (() => void) | null = null + throwOnSend = false + throwForMethod: string | null = null + readonly sent: string[] = [] + close = vi.fn(() => { + this.readyState = MockWebSocket.CLOSED + this.onclose?.() + }) + + constructor() { + sockets.push(this) + } + + send(payload: string): void { + if ( + this.throwOnSend || + (this.throwForMethod && payload.includes(`"method":"${this.throwForMethod}"`)) + ) { + throw new Error('socket send failed') + } + this.sent.push(payload) + } + + authenticate(): void { + this.readyState = MockWebSocket.OPEN + this.onopen?.() + this.onmessage?.({ data: JSON.stringify({ type: 'e2ee_ready' }) }) + this.onmessage?.({ data: 'encrypted:{"type":"e2ee_authenticated"}' }) + } +} + +const sockets: MockWebSocket[] = [] +const originalWebSocket = globalThis.WebSocket + +beforeEach(() => { + vi.useFakeTimers() + sockets.length = 0 + globalThis.WebSocket = MockWebSocket as unknown as typeof WebSocket +}) + +afterEach(() => { + vi.useRealTimers() + globalThis.WebSocket = originalWebSocket +}) + +describe('physical session liveness', () => { + it('recovers when the initial handshake write races socket teardown', () => { + const client = connect('ws://desktop.invalid', 'token', 'server-key') + const socket = sockets[0]! + socket.throwOnSend = true + + try { + expect(() => socket.authenticate()).not.toThrow() + expect(socket.close).toHaveBeenCalledOnce() + expect(client.getState()).toBe('reconnecting') + } finally { + client.close() + } + }) + + it('tolerates the first fair silent foreground-probe window', async () => { + const client = connect('ws://desktop.invalid', 'token', 'server-key') + const socket = sockets[0]! + socket.authenticate() + + try { + client.notifyForeground() + await vi.advanceTimersByTimeAsync(8_000) + + expect(socket.close).not.toHaveBeenCalled() + expect(client.getState()).toBe('connected') + } finally { + client.close() + } + }) + + it('counts authenticated terminal binary output as liveness', async () => { + const client = connect('ws://desktop.invalid', 'token', 'server-key') + const socket = sockets[0]! + socket.authenticate() + client.notifyForeground() + socket.onmessage?.({ + data: encodeTerminalStreamFrame({ + opcode: TerminalStreamOpcode.Output, + streamId: 42, + seq: 1, + payload: new TextEncoder().encode('hello') + }) + }) + await Promise.resolve() + await vi.advanceTimersByTimeAsync(8_000) + + expect(socket.close).not.toHaveBeenCalled() + expect(client.getState()).toBe('connected') + client.close() + }) + + it('turns a probe write exception into session recovery', () => { + const client = connect('ws://desktop.invalid', 'token', 'server-key') + const socket = sockets[0]! + socket.authenticate() + socket.throwOnSend = true + + try { + expect(() => client.notifyForeground()).not.toThrow() + expect(socket.close).toHaveBeenCalledOnce() + expect(client.getState()).toBe('reconnecting') + } finally { + client.close() + } + }) + + it('retains every queued stream when replay write fails', async () => { + const client = connect('ws://desktop.invalid', 'token', 'server-key') + const first = sockets[0]! + const disposeFirst = client.subscribe('terminal.subscribe', { terminal: 'term-1' }, vi.fn()) + const disposeSecond = client.subscribe('terminal.subscribe', { terminal: 'term-2' }, vi.fn()) + first.throwForMethod = 'terminal.subscribe' + + first.authenticate() + expect(client.getState()).toBe('reconnecting') + await vi.advanceTimersByTimeAsync(500) + + const replacement = sockets[1]! + replacement.authenticate() + expect( + replacement.sent.filter((payload) => payload.includes('terminal.subscribe')) + ).toHaveLength(2) + + disposeFirst() + disposeSecond() + client.close() + }) + + it('does not let probe replies consume the authentication retry budget', () => { + const client = connect('ws://desktop.invalid', 'token', 'server-key') + const socket = sockets[0]! + socket.authenticate() + + for (let index = 0; index < 3; index++) { + client.notifyForeground() + const request = socket.sent + .map((payload) => payload.replace(/^encrypted:/, '')) + .map((payload) => JSON.parse(payload) as { id?: string; method?: string }) + .findLast((payload) => payload.method === 'status.get') + socket.onmessage?.({ + data: `encrypted:${JSON.stringify({ + id: request?.id, + ok: false, + error: { code: 'unauthorized', message: 'Unauthorized' } + })}` + }) + } + + expect(client.getState()).toBe('connected') + expect(sockets).toHaveLength(1) + client.close() + }) +}) diff --git a/mobile/src/transport/rpc-session-liveness-watchdog.test.ts b/mobile/src/transport/rpc-session-liveness-watchdog.test.ts new file mode 100644 index 00000000000..c3aab2a243f --- /dev/null +++ b/mobile/src/transport/rpc-session-liveness-watchdog.test.ts @@ -0,0 +1,145 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + LIVENESS_IDLE_MS, + LIVENESS_PROBE_TIMEOUT_MS, + RpcSessionLivenessWatchdog +} from './rpc-session-liveness-watchdog' + +describe('RpcSessionLivenessWatchdog', () => { + beforeEach(() => vi.useFakeTimers()) + afterEach(() => vi.useRealTimers()) + + function fixture() { + const sendProbe = vi.fn(() => true) + const terminate = vi.fn() + const watchdog = new RpcSessionLivenessWatchdog({ + transport: 'direct', + sendProbe, + terminate, + now: Date.now + }) + const identity = {} + watchdog.start(identity) + return { identity, sendProbe, terminate, watchdog } + } + + it('probes only after authenticated-inbound idle', async () => { + const { identity, sendProbe, watchdog } = fixture() + await vi.advanceTimersByTimeAsync(LIVENESS_IDLE_MS - 1) + expect(sendProbe).not.toHaveBeenCalled() + + watchdog.noteAuthenticatedInbound(identity) + await vi.advanceTimersByTimeAsync(LIVENESS_IDLE_MS - 1) + expect(sendProbe).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(1) + expect(sendProbe).toHaveBeenCalledOnce() + }) + + it('requires three fair consecutive misses', async () => { + const { identity, terminate, watchdog } = fixture() + watchdog.probeNow(identity) + + await vi.advanceTimersByTimeAsync(LIVENESS_PROBE_TIMEOUT_MS * 2) + expect(terminate).not.toHaveBeenCalled() + await vi.advanceTimersByTimeAsync(LIVENESS_PROBE_TIMEOUT_MS) + expect(terminate).toHaveBeenCalledOnce() + expect(terminate).toHaveBeenCalledWith(identity) + }) + + it('authenticated activity resets suspicion', async () => { + const { identity, terminate, watchdog } = fixture() + watchdog.probeNow(identity) + await vi.advanceTimersByTimeAsync(LIVENESS_PROBE_TIMEOUT_MS) + watchdog.noteAuthenticatedInbound(identity) + await vi.advanceTimersByTimeAsync(LIVENESS_IDLE_MS + LIVENESS_PROBE_TIMEOUT_MS * 2) + expect(terminate).not.toHaveBeenCalled() + }) + + it('does not churn timers during continuous authenticated traffic', async () => { + const setTimer = vi.fn(setTimeout) + const sendProbe = vi.fn(() => true) + const watchdog = new RpcSessionLivenessWatchdog({ + transport: 'direct', + sendProbe, + terminate: vi.fn(), + now: Date.now, + setTimer + }) + const identity = {} + watchdog.start(identity) + + await vi.advanceTimersByTimeAsync(10_000) + for (let index = 0; index < 100; index++) { + watchdog.noteAuthenticatedInbound(identity) + } + expect(setTimer).toHaveBeenCalledOnce() + + await vi.advanceTimersByTimeAsync(10_000) + expect(sendProbe).not.toHaveBeenCalled() + expect(setTimer).toHaveBeenCalledTimes(2) + }) + + it('does not charge a scheduler-stalled probe window', () => { + let now = 0 + let callback: (() => void) | null = null + const terminate = vi.fn() + const watchdog = new RpcSessionLivenessWatchdog({ + transport: 'relay', + idleProbeMs: null, + probeTimeoutMs: 4_000, + missedProbeLimit: 2, + sendProbe: () => true, + terminate, + now: () => now, + setTimer: (next) => { + callback = next + return 1 as unknown as ReturnType + }, + clearTimer: () => {} + }) + const identity = {} + watchdog.start(identity) + watchdog.probeNow(identity) + now = 8_000 + callback?.() + now += 4_000 + callback?.() + expect(terminate).not.toHaveBeenCalled() + }) + + it('invalidates late callbacks after identity replacement', () => { + const callbacks: (() => void)[] = [] + const terminate = vi.fn() + const watchdog = new RpcSessionLivenessWatchdog({ + transport: 'direct', + sendProbe: () => true, + terminate, + now: () => 0, + setTimer: (callback) => { + callbacks.push(callback) + return callbacks.length as unknown as ReturnType + }, + clearTimer: () => {} + }) + const first = {} + const replacement = {} + watchdog.start(first) + watchdog.probeNow(first) + watchdog.start(replacement) + callbacks.forEach((callback) => callback()) + expect(terminate).not.toHaveBeenCalled() + }) + + it('terminates immediately when a probe cannot be written', () => { + const terminate = vi.fn() + const identity = {} + const watchdog = new RpcSessionLivenessWatchdog({ + transport: 'direct', + sendProbe: () => false, + terminate + }) + watchdog.start(identity) + watchdog.probeNow(identity) + expect(terminate).toHaveBeenCalledWith(identity) + }) +}) diff --git a/mobile/src/transport/rpc-session-liveness-watchdog.ts b/mobile/src/transport/rpc-session-liveness-watchdog.ts new file mode 100644 index 00000000000..2563f799819 --- /dev/null +++ b/mobile/src/transport/rpc-session-liveness-watchdog.ts @@ -0,0 +1,189 @@ +export const LIVENESS_IDLE_MS = 20_000 +export const LIVENESS_PROBE_TIMEOUT_MS = 8_000 +export const MISSED_PROBE_LIMIT = 3 + +export type RpcSessionIdentity = object + +type WatchdogOptions = { + transport: 'direct' | 'relay' + sendProbe: (identity: RpcSessionIdentity) => boolean + terminate: (identity: RpcSessionIdentity) => void + idleProbeMs?: number | null + probeTimeoutMs?: number + missedProbeLimit?: number + voluntaryProbeMinIntervalMs?: number + now?: () => number + setTimer?: typeof setTimeout + clearTimer?: typeof clearTimeout +} + +export class RpcSessionLivenessWatchdog { + private identity: RpcSessionIdentity | null = null + private timer: ReturnType | null = null + private probing = false + private missedProbes = 0 + private lastInboundAt = 0 + private lastVoluntaryProbeAt: number | null = null + private readonly idleProbeMs: number | null + private readonly probeTimeoutMs: number + private readonly missedProbeLimit: number + private readonly voluntaryProbeMinIntervalMs: number + private readonly now: () => number + private readonly setTimer: typeof setTimeout + private readonly clearTimer: typeof clearTimeout + + constructor(private readonly options: WatchdogOptions) { + this.idleProbeMs = options.idleProbeMs === undefined ? LIVENESS_IDLE_MS : options.idleProbeMs + this.probeTimeoutMs = options.probeTimeoutMs ?? LIVENESS_PROBE_TIMEOUT_MS + this.missedProbeLimit = options.missedProbeLimit ?? MISSED_PROBE_LIMIT + this.voluntaryProbeMinIntervalMs = options.voluntaryProbeMinIntervalMs ?? 0 + this.now = options.now ?? Date.now + this.setTimer = options.setTimer ?? setTimeout + this.clearTimer = options.clearTimer ?? clearTimeout + } + + start(identity: RpcSessionIdentity): void { + this.clearActiveTimer() + this.identity = identity + this.probing = false + this.missedProbes = 0 + this.lastInboundAt = this.now() + this.lastVoluntaryProbeAt = null + this.armIdle(identity) + } + + noteAuthenticatedInbound(identity: RpcSessionIdentity): void { + if (this.identity !== identity) { + return + } + this.lastInboundAt = this.now() + if (this.missedProbes > 0) { + console.log('[net] activity-probe recovered', { + transport: this.options.transport, + priorMissedProbes: this.missedProbes + }) + } + if (!this.probing && this.missedProbes === 0) { + return + } + this.missedProbes = 0 + this.probing = false + this.armIdle(identity) + } + + probeNow(identity: RpcSessionIdentity): void { + if (this.identity !== identity || this.probing) { + return + } + const now = this.now() + if ( + this.lastVoluntaryProbeAt !== null && + now - this.lastVoluntaryProbeAt < this.voluntaryProbeMinIntervalMs + ) { + return + } + this.lastVoluntaryProbeAt = now + this.startProbe(identity) + } + + stop(identity: RpcSessionIdentity): void { + if (this.identity !== identity) { + return + } + this.clearActiveTimer() + this.identity = null + this.probing = false + this.missedProbes = 0 + this.lastInboundAt = 0 + this.lastVoluntaryProbeAt = null + } + + private armIdle(identity: RpcSessionIdentity, delayMs = this.idleProbeMs): void { + this.clearActiveTimer() + if (delayMs === null) { + return + } + this.timer = this.setTimer(() => { + this.timer = null + if (this.identity !== identity) { + return + } + const idleMs = this.now() - this.lastInboundAt + if (this.idleProbeMs !== null && idleMs < this.idleProbeMs) { + this.armIdle(identity, Math.max(1, this.idleProbeMs - Math.max(0, idleMs))) + } else { + this.startProbe(identity) + } + }, delayMs) + } + + private startProbe(identity: RpcSessionIdentity): void { + if (this.identity !== identity) { + return + } + this.clearActiveTimer() + this.probing = true + const sentAt = this.now() + let sent = false + try { + sent = this.options.sendProbe(identity) + } catch { + sent = false + } + if (!sent) { + this.terminateCurrent(identity) + return + } + this.timer = this.setTimer(() => this.handleProbeTimeout(identity, sentAt), this.probeTimeoutMs) + } + + private handleProbeTimeout(identity: RpcSessionIdentity, sentAt: number): void { + this.timer = null + if (this.identity !== identity) { + return + } + const elapsedMs = this.now() - sentAt + if (elapsedMs < 0 || elapsedMs > this.probeTimeoutMs * 1.5) { + console.log('[net] activity-probe unfair window skipped', { + transport: this.options.transport, + elapsedMs, + timeoutMs: this.probeTimeoutMs + }) + this.startProbe(identity) + return + } + this.missedProbes += 1 + if (this.missedProbes >= this.missedProbeLimit) { + this.terminateCurrent(identity) + return + } + console.log('[net] activity-probe timeout tolerated', { + transport: this.options.transport, + missedProbes: this.missedProbes, + missedProbeLimit: this.missedProbeLimit + }) + this.startProbe(identity) + } + + private terminateCurrent(identity: RpcSessionIdentity): void { + if (this.identity !== identity) { + return + } + this.clearActiveTimer() + this.identity = null + this.probing = false + console.log('[net] activity-probe TIMEOUT — forcing reconnect', { + transport: this.options.transport, + missedProbes: this.missedProbes, + missedProbeLimit: this.missedProbeLimit + }) + this.options.terminate(identity) + } + + private clearActiveTimer(): void { + if (this.timer !== null) { + this.clearTimer(this.timer) + this.timer = null + } + } +} diff --git a/mobile/src/transport/rpc-subscription-result-shapes.ts b/mobile/src/transport/rpc-subscription-result-shapes.ts new file mode 100644 index 00000000000..cf09acdb6ca --- /dev/null +++ b/mobile/src/transport/rpc-subscription-result-shapes.ts @@ -0,0 +1,21 @@ +export function isTerminalSubscribedResult( + value: unknown +): value is { type: 'subscribed'; streamId: number } { + return ( + !!value && + typeof value === 'object' && + (value as { type?: unknown }).type === 'subscribed' && + typeof (value as { streamId?: unknown }).streamId === 'number' + ) +} + +export function isStreamingSubscriptionReadyResult( + value: unknown +): value is { type: 'ready'; subscriptionId: string } { + return ( + !!value && + typeof value === 'object' && + (value as { type?: unknown }).type === 'ready' && + typeof (value as { subscriptionId?: unknown }).subscriptionId === 'string' + ) +} diff --git a/mobile/src/transport/settings-host-client-lifecycle.test.ts b/mobile/src/transport/settings-host-client-lifecycle.test.ts index 2762e7e85a5..be7621017ba 100644 --- a/mobile/src/transport/settings-host-client-lifecycle.test.ts +++ b/mobile/src/transport/settings-host-client-lifecycle.test.ts @@ -554,7 +554,7 @@ describe('settings host client lifecycle', () => { } const originalClient = clients.get(replacedHostId)?.[0] - act(() => context?.closeHost(replacedHostId)) + act(() => context?.refreshHostClient(replacedHostId)) expect(originalClient?.closeMock).toHaveBeenCalledOnce() await act(async () => { diff --git a/mobile/src/transport/stable-logical-rpc-client.test.ts b/mobile/src/transport/stable-logical-rpc-client.test.ts index 2171a690b59..3081d0fa8a1 100644 --- a/mobile/src/transport/stable-logical-rpc-client.test.ts +++ b/mobile/src/transport/stable-logical-rpc-client.test.ts @@ -146,8 +146,9 @@ describe('stable logical RPC client', () => { await expect(client.sendRequest('status.get')).resolves.toEqual(success('next')) }) - it('lets the physical close settle in-flight requests on suspend, preserving delivery marks', async () => { + it('preserves delivery ambiguity without replaying a mutation after relay replacement', async () => { const session = new FakeSession('connected') + const replacement = new FakeSession('connected') const inFlight = deferred() session.sendRequest.mockReturnValue(inFlight.promise) // Mirror the real physical contract: close() rejects post-write pendings @@ -161,8 +162,10 @@ describe('stable logical RPC client', () => { await expect(request).rejects.toBe(closeError) await expect(request.catch((error: unknown) => isRpcDeliveryUnknown(error))).resolves.toBe(true) - // New requests while suspended still fail definitively before any write. - await expect(client.sendRequest('status.get')).rejects.toThrow('Client suspended') + await client.migrateTo(replacement, 'relay') + expect(replacement.sendRequest).not.toHaveBeenCalled() + replacement.sendRequest.mockResolvedValue(success('next')) + await expect(client.sendRequest('status.get')).resolves.toEqual(success('next')) }) it('lets the physical close settle in-flight requests on close, keeping pre-write failures definite', async () => { diff --git a/mobile/src/transport/use-all-host-clients.ts b/mobile/src/transport/use-all-host-clients.ts index 1000916113b..4f5d036c877 100644 --- a/mobile/src/transport/use-all-host-clients.ts +++ b/mobile/src/transport/use-all-host-clients.ts @@ -3,6 +3,7 @@ import type { RpcClient } from './rpc-client' import type { MobileConnectionPath } from './stable-logical-rpc-client' import type { ConnectionState } from './types' import { useRpcClientContext } from './client-context' +import type { HostClientAcquisition } from './client-context' type UseAllHostClientsOptions = { autoConnectHostIds?: readonly string[] @@ -24,6 +25,7 @@ export function useAllHostClients(hostIds: string[], options?: UseAllHostClients ) const [tick, setTick] = useState(0) const acquiredHostIdsRef = useRef>(new Set()) + const acquisitionsRef = useRef>(new Map()) const hostUnsubscribesRef = useRef void>>(new Map()) const closeUnusedRef = useRef(closeUnusedOnRelease) @@ -42,10 +44,17 @@ export function useAllHostClients(hostIds: string[], options?: UseAllHostClients } hostUnsubscribesRef.current.clear() for (const id of acquiredHostIds) { + const acquisition = acquisitionsRef.current.get(id) + if (!acquisition) { + if (closeUnusedRef.current) { + ctx.closeIfUnused(id) + } + continue + } if (closeUnusedRef.current) { - ctx.releaseAndCloseIfUnused(id) + ctx.releaseAndCloseIfUnused(id, acquisition) } else { - ctx.release(id) + ctx.release(id, acquisition) } } if (closeUnusedRef.current) { @@ -56,6 +65,7 @@ export function useAllHostClients(hostIds: string[], options?: UseAllHostClients } } acquiredHostIdsRef.current.clear() + acquisitionsRef.current.clear() } }, [ctx]) @@ -82,16 +92,26 @@ export function useAllHostClients(hostIds: string[], options?: UseAllHostClients for (const id of acquiredHostIdsRef.current) { if (!nextAcquiredHostIds.has(id)) { - if (closeUnusedOnRelease) { - ctx.releaseAndCloseIfUnused(id) - } else { - ctx.release(id) + const acquisition = acquisitionsRef.current.get(id) + if (!acquisition) { + if (closeUnusedOnRelease) { + ctx.closeIfUnused(id) + } + continue } + if (closeUnusedOnRelease) { + ctx.releaseAndCloseIfUnused(id, acquisition) + } else { + ctx.release(id, acquisition) + } + acquisitionsRef.current.delete(id) } } for (const id of nextAcquiredHostIds) { if (!acquiredHostIdsRef.current.has(id)) { - ctx.acquire(id) + const acquisition = {} + acquisitionsRef.current.set(id, acquisition) + ctx.acquire(id, acquisition) } } if (closeUnusedOnRelease) {