fix(mobile): notice a silent drop while dictating within seconds (#35834)

An overdue chunk reply now asks the connection for one decisive liveness
probe instead of waiting for the idle watchdog, so the resend lands inside
the host's grace. Native clients probe directly; the OTA page asks the
shell with a new notify that older shells refuse with a diagnostic only.
This commit is contained in:
Kelvin Amoaba
2026-10-29 14:32:59 +00:00
committed by GitHub
parent 770a1b8e6e
commit c5957e7d4d
18 changed files with 223 additions and 24 deletions
@@ -9,6 +9,7 @@ import { MOBILE_DICTATION_RECONNECT_GRACE_MS } from '../../../src/shared/rpc-con
import { isRpcDeliveryUnknown } from '../transport/rpc-delivery-ambiguity'
import { rpcPayloadMember } from '../transport/rpc-reader-payload'
import { waitForRpcClientReconnected } from '../transport/rpc-client-reconnect-wait'
import { DECISIVE_PROBE_TIMEOUT_MS } from '../transport/rpc-session-liveness-watchdog'
import { isLogicalClientCutoverError } from '../transport/stable-logical-rpc-client'
import { isLogicalClientSuspendedError } from '../transport/logical-client-suspended-error'
import type {
@@ -141,15 +142,21 @@ export function enqueueMobileDictationAudioChunk(
sampleRate: MOBILE_DICTATION_PCM_SAMPLE_RATE,
seq: queue.pendingAudioBudget.nextSeq()
}
const hostDedupesResends = queue.pendingAudioBudget.hostDedupesResends
const sendChunk = sendAcrossConnectionMove(
client,
() =>
() => {
// A reply this late, with nothing heard meanwhile, means a silently dead connection; probe now, not after the idle watchdog, and keep asking until it settles.
const overdue = hostDedupesResends
? setInterval(() => client.probeLivenessNow?.(), DECISIVE_PROBE_TIMEOUT_MS)
: undefined
// Wait for a reconnect only as long as the host keeps the session; past that, fail and keep the text so far.
dictationAudioChunkSend.request(client, params, {
timeoutMs: MOBILE_DICTATION_RECONNECT_GRACE_MS
}),
return dictationAudioChunkSend
.request(client, params, { timeoutMs: MOBILE_DICTATION_RECONNECT_GRACE_MS })
.finally(() => clearInterval(overdue))
},
// A recording cancelled during the outage must not reach the host when it comes back.
queue.pendingAudioBudget.hostDedupesResends ? () => queue.isActive(dictationId) : undefined
hostDedupesResends ? () => queue.isActive(dictationId) : undefined
)
.then((reply) => {
// Only newer hosts send the text, and only when it changed; any other result still acknowledges the chunk.
@@ -183,6 +183,24 @@ describe('dictation through a reconnect', () => {
})
})
async function startWith(rpc: FakeRpcClient, startResult: Record<string, unknown>) {
mount(rpc, [], [])
await act(async () => {
const started = dictation().start()
while (rpc.requests.length === 0) {
await Promise.resolve()
}
rpc.requests.splice(0)[0].resolve({ id: 'desktop', ok: true, result: startResult })
await started
})
}
function speak(): void {
for (const handler of seam.chunkHandlers) {
handler({ data: Uint8Array.from([1, 2]), droppedBytes: 0 })
}
}
describe('audio sent on a connection that died', () => {
async function dropMidChunk(
startResult: Record<string, unknown>,
@@ -191,20 +209,7 @@ describe('audio sent on a connection that died', () => {
let state: ConnectionState = 'connected'
const rpc = createFakeRpcClient({ getState: () => state })
const sent: SentRequest[] = []
mount(rpc, [], [])
await act(async () => {
const started = dictation().start()
while (rpc.requests.length === 0) {
await Promise.resolve()
}
rpc.requests.splice(0)[0].resolve({ id: 'desktop', ok: true, result: startResult })
await started
})
const speak = () => {
for (const handler of seam.chunkHandlers) {
handler({ data: Uint8Array.from([1, 2]), droppedBytes: 0 })
}
}
await startWith(rpc, startResult)
await act(async () => {
speak()
speak()
@@ -248,6 +253,44 @@ describe('audio sent on a connection that died', () => {
})
})
describe('a chunk reply that stops coming', () => {
async function probesAround(startResult: Record<string, unknown>, overdueMs: number) {
vi.useFakeTimers()
try {
const rpc = createFakeRpcClient()
let probes = 0
rpc.probeLivenessNow = () => {
probes += 1
}
await startWith(rpc, startResult)
await act(async () => {
speak()
vi.advanceTimersByTime(overdueMs - 1)
})
const early = probes
await act(async () => {
vi.advanceTimersByTime(1)
})
const overdue = probes
await act(async () => {
vi.advanceTimersByTime(overdueMs)
})
return [early, overdue, probes]
} finally {
vi.useRealTimers()
}
}
it('asks for a decisive probe every four seconds the reply is late', async () => {
await expect(probesAround({ dedupesResentChunks: true }, 4000)).resolves.toEqual([0, 1, 2])
})
// A false alarm would end a live connection, and an old host cannot take the lost audio again.
it('leaves an old host to the idle watchdog', async () => {
await expect(probesAround({}, 4000)).resolves.toEqual([0, 0, 0])
})
})
describe('the text so far', () => {
it('shows what the host heard while recording and clears it as the final text lands', async () => {
const rpc = createFakeRpcClient()
@@ -9,6 +9,7 @@ import {
import { BRIDGE_PAGE_PAINTED } from './bridge/bridge-page-painted'
import { BRIDGE_ROUTE_PARAM_CLEAR } from './bridge/bridge-route-update'
import { BRIDGE_HAPTICS_GRANT, BRIDGE_HAPTICS_NOTIFY } from './bridge/bridge-haptics-notify'
import { BRIDGE_LIVENESS_PROBE_NOTIFY } from './bridge/bridge-liveness-probe-notify'
import { MOBILE_WEB_SHELL_GRANTS } from './page-route-policy'
describe('notifications, refusals and the fence', () => {
@@ -365,6 +366,23 @@ describe('haptics', () => {
})
})
// A page that stops hearing back cannot probe the socket itself; native owns the connection it rides.
describe('liveness probe', () => {
const probe = clientFrame({ type: 'notify', name: BRIDGE_LIVENESS_PROBE_NOTIFY })
it('asks the native client for a decisive probe', () => {
const bridge = harness()
let probes = 0
bridge.client.probeLivenessNow = () => {
probes += 1
}
bridge.host.receive(clientFrame({ type: 'ready' }))
bridge.host.receive(probe)
expect(probes).toBe(1)
expect(bridge.diagnostics).toEqual([])
})
})
describe('the page erasing a one-shot route param', () => {
// Forward both one-shot key and value to the native route holder; only it can compare before clearing (ruling 34).
it('carries a page clear up with the param and the value it named', () => {
@@ -7,6 +7,7 @@ import {
type BridgeInitRoute
} from './bridge/bridge-envelope'
import { BRIDGE_HAPTICS_NOTIFY } from './bridge/bridge-haptics-notify'
import { BRIDGE_LIVENESS_PROBE_NOTIFY } from './bridge/bridge-liveness-probe-notify'
import { bridgeNotifyRefusal } from './bridge/bridge-notify-grants'
import { BRIDGE_BACK_CLAIM_NOTIFY } from './bridge/bridge-page-back'
import { BRIDGE_PAGE_LOCATION } from './bridge/bridge-page-location'
@@ -94,6 +95,10 @@ export function createBridgeNotifyForwarder(args: {
options.onStorageWrite(message.key, message.value)
return
}
if (message.name === BRIDGE_LIVENESS_PROBE_NOTIFY) {
client.probeLivenessNow?.()
return
}
if (message.name === BRIDGE_HAPTICS_NOTIFY) {
// Haptics are one-way so each row tap avoids spending a request slot.
options.onHaptic(message.kind)
@@ -7,6 +7,7 @@ import {
BRIDGE_PROTOCOL_VERSION
} from './bridge-envelope'
import { BRIDGE_HAPTICS_GRANT, BRIDGE_HAPTICS_NOTIFY } from './bridge-haptics-notify'
import { BRIDGE_LIVENESS_PROBE_NOTIFY } from './bridge-liveness-probe-notify'
import { GRANTS, INIT, createPageClient as createPage } from './bridge-page-client-test-harness'
const resources = new Set<ReturnType<typeof createPage>>()
@@ -246,3 +247,15 @@ describe('bridge client haptics', () => {
expect(page.frames().at(-1)).toEqual({ v: BRIDGE_PROTOCOL_VERSION, type: 'close' })
})
})
// No grant gates it: an older shell refuses the frame with one diagnostic and keeps its idle watchdog.
describe('bridge client liveness probe', () => {
it('posts a probe frame once the session is up', () => {
const page = createPageClient()
page.start()
page.client.probeLivenessNow?.()
expect(page.frames().slice(1)).toEqual([
{ v: BRIDGE_PROTOCOL_VERSION, type: 'notify', name: BRIDGE_LIVENESS_PROBE_NOTIFY }
])
})
})
@@ -13,6 +13,7 @@ import {
type BridgeHapticsKind
} from './bridge-haptics-notify'
import { captureBridgeError } from './bridge-error-capture'
import { BRIDGE_LIVENESS_PROBE_NOTIFY } from './bridge-liveness-probe-notify'
import { BRIDGE_BACK_CLAIM_NOTIFY } from './bridge-page-back'
import { BRIDGE_PAGE_LOCATION } from './bridge-page-location'
import { BRIDGE_PAGE_PAINTED } from './bridge-page-painted'
@@ -37,6 +38,7 @@ export type BridgeClientNotifications = {
notifyExternalLink: (url: string) => boolean
notifyStorageWrite: (key: string, value: string | null) => boolean
notifyHaptics: (kind: BridgeHapticsKind) => boolean
notifyLivenessProbe: () => void
notifyPageFault: (error: unknown) => boolean
notifyPagePainted: () => void
notifyBackClaim: (claimed: boolean) => void
@@ -109,6 +111,9 @@ export function createBridgeClientNotifications(
notifyHaptics: (kind) =>
deps.hasGrant(BRIDGE_HAPTICS_GRANT) &&
post({ v: BRIDGE_PROTOCOL_VERSION, type: 'notify', name: BRIDGE_HAPTICS_NOTIFY, kind }),
notifyLivenessProbe: () => {
post({ v: BRIDGE_PROTOCOL_VERSION, type: 'notify', name: BRIDGE_LIVENESS_PROBE_NOTIFY })
},
notifyPageFault: (error) => {
if (deps.isClosed() || !deps.hasGrant(BRIDGE_FAULT_GRANT)) {
return false
@@ -0,0 +1,9 @@
import { z } from 'zod'
// The page asks native for a decisive liveness probe when its replies stop coming. Older shells refuse the frame with a diagnostic only and keep the idle watchdog.
export const BRIDGE_LIVENESS_PROBE_NOTIFY = 'liveness-probe'
export const BRIDGE_LIVENESS_PROBE_NOTIFY_FIELDS = {
type: z.literal('notify'),
name: z.literal(BRIDGE_LIVENESS_PROBE_NOTIFY)
}
@@ -1,6 +1,7 @@
import { z } from 'zod'
import { BridgeErrorCaptureSchema } from './bridge-error-capture'
import { BRIDGE_HAPTICS_NOTIFY_FIELDS } from './bridge-haptics-notify'
import { BRIDGE_LIVENESS_PROBE_NOTIFY_FIELDS } from './bridge-liveness-probe-notify'
import { BRIDGE_BACK_CLAIM_NOTIFY } from './bridge-page-back'
import { BRIDGE_PAGE_LOCATION } from './bridge-page-location'
import { BRIDGE_PAGE_PAINTED } from './bridge-page-painted'
@@ -79,6 +80,7 @@ export const BridgeNotifySchema = z.discriminatedUnion('name', [
error: BridgeErrorCaptureSchema
}),
z.object({ v: versionSchema, ...BRIDGE_HAPTICS_NOTIFY_FIELDS }),
z.object({ v: versionSchema, ...BRIDGE_LIVENESS_PROBE_NOTIFY_FIELDS }),
// Allow only the shell-provided one-shot param; its holder ignores a clear naming an older value (ruling 34).
z.object({
v: versionSchema,
@@ -5,6 +5,7 @@ import {
type BridgeClientMessage
} from './bridge-envelope'
import { BRIDGE_HAPTICS_GRANT, BRIDGE_HAPTICS_NOTIFY } from './bridge-haptics-notify'
import { BRIDGE_LIVENESS_PROBE_NOTIFY } from './bridge-liveness-probe-notify'
import { BRIDGE_BACK_CLAIM_NOTIFY } from './bridge-page-back'
import { BRIDGE_PAGE_LOCATION } from './bridge-page-location'
import { BRIDGE_PAGE_PAINTED } from './bridge-page-painted'
@@ -26,7 +27,8 @@ const BRIDGE_NOTIFY_GRANTS: Readonly<Record<BridgeNotifyName, string | null>> =
storage: 'storage',
[BRIDGE_EXTERNAL_LINK_GRANT]: BRIDGE_EXTERNAL_LINK_GRANT,
[BRIDGE_FAULT_GRANT]: BRIDGE_FAULT_GRANT,
[BRIDGE_HAPTICS_NOTIFY]: BRIDGE_HAPTICS_GRANT
[BRIDGE_HAPTICS_NOTIFY]: BRIDGE_HAPTICS_GRANT,
[BRIDGE_LIVENESS_PROBE_NOTIFY]: null
}
export type BridgeNotifyRefusal = 'before-ready' | 'ungranted'
@@ -319,6 +319,7 @@ export function createBridgeRpcClient(options: BridgeRpcClientOptions): BridgeRp
// Allow listener registration before init so providers can observe the first state change; it reads no unprimed value.
onStateChange: (listener) => cache.onStateChange(listener),
notifyForeground: notifications.notifyForeground,
probeLivenessNow: notifications.notifyLivenessProbe,
notifyNavigate: notifications.notifyNavigate,
notifyNavigateBack: notifications.notifyNavigateBack,
notifyExternalLink: notifications.notifyExternalLink,
@@ -170,6 +170,10 @@ export class DirectRpcClient implements RpcClient {
return this.connectionState.addListener(listener)
}
// The watchdog starts before capability negotiation, when a probe cannot be written yet.
readonly probeLivenessNow = (): void =>
void (this.getState() === 'connected' && this.liveness.probeCurrentDecisively())
notifyForeground(_reason?: ForegroundNudgeReason): void {
if (this.intentionallyClosed) {
return
@@ -52,6 +52,7 @@ export function routeRpcClientCalls(
carriesRpcEnvelope: client.carriesRpcEnvelope && (() => client.carriesRpcEnvelope?.() ?? true),
onStateChange: (listener) => client.onStateChange(listener),
notifyForeground: (reason) => client.notifyForeground(reason),
probeLivenessNow: client.probeLivenessNow && (() => client.probeLivenessNow?.()),
close: () => {}
}
}
@@ -237,6 +237,25 @@ describe('mobile relay RPC session liveness', () => {
expect(fakes.close).toHaveBeenCalledOnce()
})
it('probes for a caller that stopped hearing back, even inside the foreground cooldown', async () => {
const session = await authenticateSession()
session.notifyForeground('focus')
const firstProbe = sentRequests()[0]!
fakes.linkOptions!.onText(
JSON.stringify({ id: firstProbe.id, ok: true, result: {}, _meta: { runtimeId: 'r1' } })
)
// Just heard from, so not suspect yet.
session.probeLivenessNow?.()
expect(fakes.sendText).toHaveBeenCalledOnce()
await vi.advanceTimersByTimeAsync(4_000)
session.probeLivenessNow?.()
expect(fakes.sendText).toHaveBeenCalledTimes(2)
await vi.advanceTimersByTimeAsync(4_000)
expect(session.getState()).toBe('disconnected')
})
it('redials after one 4s miss when a retained relay returns from the background', async () => {
const onLog = vi.fn<ConnectionLogSink>()
const session = await authenticateSession(onLog)
@@ -17,7 +17,10 @@ import { isRpcResponse } from './rpc-response-shape'
import { RelayDialStageTracker, type RelayDialStageSource } from './relay-dial-stage'
import { RelayPendingRequests } from './relay-pending-requests'
import { MobileRelayConnectWaiters } from './mobile-relay-connect-waiters'
import { RpcSessionLivenessWatchdog } from './rpc-session-liveness-watchdog'
import {
DECISIVE_PROBE_TIMEOUT_MS,
RpcSessionLivenessWatchdog
} from './rpc-session-liveness-watchdog'
import { settleMobileRuntimeCapabilities } from './mobile-runtime-capability-negotiation'
import type { RelayHostCloseReason } from '../../../src/shared/relay-host-close-reason'
import type { RpcClient } from './rpc-client'
@@ -26,8 +29,6 @@ import type { ConnectionLogSink, ConnectionState, RpcResponse } from './types'
const RELAY_PROBE_TIMEOUT_MS = 4_000
const RELAY_MISSED_PROBE_LIMIT = 2
const RELAY_FOREGROUND_PROBE_MIN_INTERVAL_MS = 10_000
// Use one decisive post-background probe, allowing radio wake/TCP retransmit latency before redial.
const RELAY_RESUME_PROBE_MS = 4_000
let relayRpcSessionSequence = 0
export type MobileRelayRpcSession = RpcClient &
@@ -157,7 +158,9 @@ export function connectMobileRelayRpcSession(args: {
getResumeConfirmation: () => resumeConfirmation,
getFailure: () => failure,
probeAfterBackground: () =>
state === 'connected' && livenessWatchdog.probeNow(livenessIdentity, RELAY_RESUME_PROBE_MS)
state === 'connected' &&
livenessWatchdog.probeNow(livenessIdentity, DECISIVE_PROBE_TIMEOUT_MS),
probeLivenessNow: () => state === 'connected' && livenessWatchdog.probeCurrentDecisively()
}
const connectWaiters = new MobileRelayConnectWaiters(client)
const livenessWatchdog = new RpcSessionLivenessWatchdog({
+2
View File
@@ -36,6 +36,8 @@ export type RpcClient = UnvalidatedRpcRequestPort & {
carriesRpcEnvelope?: () => boolean
onStateChange: (listener: (state: ConnectionState) => void) => () => void
notifyForeground: (reason?: ForegroundNudgeReason) => void
// A caller that stopped hearing back asks for one decisive probe, so a silently dead socket ends in seconds.
probeLivenessNow?: () => void
// Settle every pending request before returning; only the physical sender knows which reached the wire.
// Reject written requests with markRpcDeliveryUnknown: the host may already have executed them.
close: () => void
@@ -113,6 +113,49 @@ describe('physical session liveness', () => {
}
})
it('ends a silent socket after one 4s miss when a caller stopped hearing back', async () => {
const client = connect('ws://desktop.invalid', 'token', 'server-key')
const socket = sockets[0]!
socket.authenticate()
try {
// Just heard from, so not suspect yet.
const probes = () => socket.sent.filter((frame) => frame.includes('mobile-liveness-'))
client.probeLivenessNow?.()
await vi.advanceTimersByTimeAsync(4_000)
expect(probes()).toHaveLength(0)
client.probeLivenessNow?.()
expect(probes()).toHaveLength(1)
await vi.advanceTimersByTimeAsync(3_999)
expect(socket.close).not.toHaveBeenCalled()
await vi.advanceTimersByTimeAsync(1)
expect(socket.close).toHaveBeenCalledOnce()
expect(client.getState()).toBe('reconnecting')
} finally {
client.close()
}
})
it('turns a tolerant probe already running into a decisive one', async () => {
const client = connect('ws://desktop.invalid', 'token', 'server-key')
const socket = sockets[0]!
socket.authenticate()
try {
// A network switch starts the three-miss probe first.
client.notifyForeground('network-change')
await vi.advanceTimersByTimeAsync(4_000)
client.probeLivenessNow?.()
await vi.advanceTimersByTimeAsync(4_000)
expect(socket.close).toHaveBeenCalledOnce()
} finally {
client.close()
}
})
it('counts authenticated terminal binary output as liveness', async () => {
const client = connect('ws://desktop.invalid', 'token', 'server-key')
const socket = sockets[0]!
@@ -3,6 +3,8 @@ import { defaultCancelTimer, defaultScheduleTimer, type ScheduleTimer } from './
export const LIVENESS_IDLE_MS = 20_000
export const LIVENESS_PROBE_TIMEOUT_MS = 8_000
export const MISSED_PROBE_LIMIT = 3
// One miss ends a session already suspected dead; long enough for radio wake and TCP retransmit before redial.
export const DECISIVE_PROBE_TIMEOUT_MS = 4_000
declare const rpcSessionIdentityBrand: unique symbol
@@ -35,6 +37,7 @@ export class RpcSessionLivenessWatchdog {
private identity: RpcSessionIdentity | null = null
private timer: ReturnType<typeof setTimeout> | null = null
private probing = false
private probingDecisively = false
private missedProbes = 0
private lastInboundAt = 0
private lastVoluntaryProbeAt: number | null = null
@@ -106,6 +109,19 @@ export class RpcSessionLivenessWatchdog {
this.startProbe(identity, decisiveTimeoutMs)
}
// For a caller that stopped hearing back: a session heard from within the window is not suspect, no foreground rate limit applies, and a tolerant probe already running is cut short.
probeCurrentDecisively(): void {
const identity = this.identity
const decisiveRunning = this.probing && this.probingDecisively
if (
identity &&
!decisiveRunning &&
this.now() - this.lastInboundAt >= DECISIVE_PROBE_TIMEOUT_MS
) {
this.startProbe(identity, DECISIVE_PROBE_TIMEOUT_MS)
}
}
stop(identity: RpcSessionIdentity): void {
if (this.identity !== identity) {
return
@@ -143,6 +159,7 @@ export class RpcSessionLivenessWatchdog {
}
this.clearActiveTimer()
this.probing = true
this.probingDecisively = decisiveTimeoutMs !== null
const sentAt = this.now()
let sent = false
try {
@@ -173,6 +173,11 @@ export function createStableLogicalRpcClient(
activeSession.notifyForeground(reason)
}
},
probeLivenessNow: () => {
if (!suspended) {
activeSession.probeLivenessNow?.()
}
},
close() {
if (closed) {
return