From ba3f9918f1d549f496ce843d97528d02b165d485 Mon Sep 17 00:00:00 2001 From: Neil <4138956+nwparker@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:57:40 -0800 Subject: [PATCH] relay-edge: reconnect_gap metric (time from a host's last control close to its next activation) (#41034) Diagnostics only. Separates sleeping hosts (long gap) from awake hosts whose connection was dropped (short gap) after silence closes. Stores one small per-host key, cleared on the next activation and by erase. --- cloud/apps/relay-edge/src/host-relay-base.ts | 4 ++++ cloud/apps/relay-edge/src/host-relay-handshake.ts | 7 ++++++- cloud/apps/relay-edge/src/metrics.ts | 5 +++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/cloud/apps/relay-edge/src/host-relay-base.ts b/cloud/apps/relay-edge/src/host-relay-base.ts index d550b6ca840..da0b038d345 100644 --- a/cloud/apps/relay-edge/src/host-relay-base.ts +++ b/cloud/apps/relay-edge/src/host-relay-base.ts @@ -77,6 +77,9 @@ export const NO_DEADLINE = Number.MAX_SAFE_INTEGER // How long a silent keepalive control has to answer the relay ping before it is closed. export const KEEPALIVE_PROBE_MS = 10_000 export const SWEEP_AT_KEY = 'sweepAt' +// Diagnostics only: the host's last control close, read once by the next activation. +export const LAST_CONTROL_CLOSE_KEY = 'lastControlClose' +export type LastControlClose = { goneAt: number; closedAt: number; code: number } // 6 h ± 30 min, as the cells: desktops re-ask the director at least this often. export const LEASE_BASE_MS = 6 * 60 * 60 * 1000 export const LEASE_JITTER_MS = 30 * 60 * 1000 @@ -181,6 +184,7 @@ export abstract class HostRelayBase extends DurableObject { this.attach(ws, null) if (a.k === 'control' && a.phase === 'active') { metric(this.env, 'control_close', 1, String(code), [], '', a.place) } if (a.k === 'control' && a.phase === 'active' && !this.activeControl()) { + await this.ctx.storage.put(LAST_CONTROL_CLOSE_KEY, { goneAt, closedAt: Date.now(), code }) const session = await this.ctx.storage.get('session') if (session && session.generation === a.generation) { const hostReason = relayHostCloseReasonFrom(reason) diff --git a/cloud/apps/relay-edge/src/host-relay-handshake.ts b/cloud/apps/relay-edge/src/host-relay-handshake.ts index 1d9995d44bf..36c7a86a2af 100644 --- a/cloud/apps/relay-edge/src/host-relay-handshake.ts +++ b/cloud/apps/relay-edge/src/host-relay-handshake.ts @@ -7,7 +7,7 @@ import { RELAY_CLOSE_CODE } from '../../../packages/relay-contract/src/close-cod import { metric } from './metrics' import type { EdgeClaims } from './env' import { b64, b64url, fromB64, hmacSha256, sha256B64url, timingSafeEqualStr } from './bytes' -import { HostRelayBase, ASSIGNMENT_CHECK_DEADLINE_MS, NO_DEADLINE, LEASE_BASE_MS, LEASE_JITTER_MS, type ControlAttachment, type Session } from './host-relay-base' +import { HostRelayBase, ASSIGNMENT_CHECK_DEADLINE_MS, NO_DEADLINE, LEASE_BASE_MS, LEASE_JITTER_MS, type ControlAttachment, type Session, LAST_CONTROL_CLOSE_KEY, type LastControlClose } from './host-relay-base' export abstract class HostRelayHandshake extends HostRelayBase { protected async onHostHello(ws: WebSocket, a: ControlAttachment, type: unknown, body: Record): Promise { @@ -193,6 +193,11 @@ export abstract class HostRelayHandshake extends HostRelayBase { // After the ack, which strict shipped parsers must see unchanged; only hosts that asked get it. if (a.clientKeepalive) { this.send(ws, { type: 'keepalive-offer', v: 1, intervalMs: this.config.pingIntervalMs }) } metric(this.env, 'control_active', 1, rebind ? 'rebind' : 'new', [a.openedAt === undefined ? -1 : now - a.openedAt], a.clientKeepalive ? 'keepalive' : 'legacy', a.place) + const last = await this.ctx.storage.get(LAST_CONTROL_CLOSE_KEY) + if (last) { + await this.ctx.storage.delete(LAST_CONTROL_CLOSE_KEY) + metric(this.env, 'reconnect_gap', Math.round((now - last.goneAt) / 1000), String(last.code), [Math.round((last.closedAt - last.goneAt) / 1000)], a.clientKeepalive ? 'keepalive' : 'legacy', a.place) + } await this.reschedule() } } diff --git a/cloud/apps/relay-edge/src/metrics.ts b/cloud/apps/relay-edge/src/metrics.ts index ea76d7f7f2c..c49284a7b06 100644 --- a/cloud/apps/relay-edge/src/metrics.ts +++ b/cloud/apps/relay-edge/src/metrics.ts @@ -33,6 +33,11 @@ export type MetricEvent = | 'auth_expired_close' // Control closed for silence (deadControlMetric, host-relay-control.ts). | 'silence_close' + // A host's control came back after its last control closed: value = seconds from when that + // control was last known alive (silence closes) or closed (others) to this activation; label = + // that close code; double2 = seconds between last-known-alive and the actual close. + // reason 'keepalive' | 'legacy' of the new control. + | 'reconnect_gap' // Pre-auth socket refused by the per-source limiter: label 'phone' | 'data' | 'health'. Unpushed-host phones: 'not_on_edge'. | 'preauth_limited' | 'not_on_edge'