relay edge: handshake time on control_active, control age and silence on dead-control closes

At 1%, many new controls went silent at or right after activation and were closed minutes later,
with no rebound from a retry. To tell a desktop that gave up the connect (15 s deadline, e.g. a
slow first hello on a new object) from one that died later: control_active double2 = ms from the
upgrade to activation; new silence_close (seconds silent, where found, keepalive/legacy, age);
auth_expired_close double3 = control age.
This commit is contained in:
Neil
2026-11-10 16:13:38 -08:00
parent a434065fe8
commit 87c434ecc4
5 changed files with 25 additions and 6 deletions
@@ -45,6 +45,9 @@ export type ControlAttachment = {
clientKeepalive?: boolean
// A keepalive control found silent: one relay ping sent at this time (see probeSilentControls).
probeAt?: number
// Upgrade and activation times, for metrics (absent on attachments from before these fields).
openedAt?: number
activeAt?: number
}
export type Attachment = ControlAttachment
export type Conn = {
@@ -28,8 +28,16 @@ export abstract class HostRelayControl extends HostRelayHandshake {
// Where the host is going away anyway (export, erase, drain, a new control of the same host): a
// silent control is closed now. Its orphan time is the latest the ping alarm would have found it,
// so rebind and the orphan grace are never shorter than before.
// A control closed for silence: value = seconds silent, label = where it was found, reason =
// keepalive | legacy; double2 = seconds since it became active (-1 if unknown).
protected deadControlMetric(a: ControlAttachment, now: number, seen: number, found: 'alarm' | 'on_demand' | 'expired'): void {
const ageS = a.activeAt === undefined ? -1 : Math.round((now - a.activeAt) / 1000)
metric(this.env, 'silence_close', Math.round((now - seen) / 1000), found, [ageS], this.keepalive(a) ? 'keepalive' : 'legacy', a.place)
}
protected async closeSilentControls(now = Date.now()): Promise<void> {
for (const { ws, seen } of this.silentControls(now)) {
for (const { ws, a, seen } of this.silentControls(now)) {
this.deadControlMetric(a, now, seen, 'on_demand')
this.close(ws, RELAY_CLOSE_CODE.PEER_DROPPED, 'control silence timeout')
await this.onSocketGone(ws, RELAY_CLOSE_CODE.PEER_DROPPED, '', Math.min(now, seen + this.config.silenceTimeoutMs + this.config.pingIntervalMs))
}
@@ -157,6 +157,7 @@ export abstract class HostRelayHandshake extends HostRelayBase {
})
const leaseExpiresAt = now + LEASE_BASE_MS + Math.round((Math.random() * 2 - 1) * LEASE_JITTER_MS)
a.phase = 'active'
a.activeAt = now
a.generation = generation
a.lastPongAt = now
a.lastPingAt = now
@@ -191,7 +192,7 @@ export abstract class HostRelayHandshake extends HostRelayBase {
})
// After the ack, which strict shipped parsers must see unchanged; only hosts that asked get it.
if (a.clientKeepalive) { this.send(ws, { type: 'keepalive-offer', v: 1, intervalMs: this.config.pingIntervalMs }) }
metric(this.env, 'control_active', 1, rebind ? 'rebind' : 'new', [], a.clientKeepalive ? 'keepalive' : 'legacy', a.place)
metric(this.env, 'control_active', 1, rebind ? 'rebind' : 'new', [a.openedAt === undefined ? -1 : now - a.openedAt], a.clientKeepalive ? 'keepalive' : 'legacy', a.place)
await this.reschedule()
}
}
+5 -2
View File
@@ -145,7 +145,8 @@ export class HostRelay extends HostRelayPhones {
claims,
origin,
caps: [...parseRelayHostCapabilities(request.headers.get('x-orca-host-capabilities') ?? '')],
deadline: Date.now() + RELAY_PROTOCOL_LIMITS.firstFrameDeadlineMs
deadline: Date.now() + RELAY_PROTOCOL_LIMITS.firstFrameDeadlineMs,
openedAt: Date.now()
})
await this.reschedule()
return new Response(null, { status: 101, webSocket: client })
@@ -180,6 +181,7 @@ export class HostRelay extends HostRelayPhones {
continue
}
if (now - seen > this.config.silenceTimeoutMs && (!this.keepalive(a) || now - a.probeAt! >= KEEPALIVE_PROBE_MS - 50)) {
this.deadControlMetric(a, now, seen, 'alarm')
this.close(ws, RELAY_CLOSE_CODE.PEER_DROPPED, 'control silence timeout')
await this.onSocketGone(ws, RELAY_CLOSE_CODE.PEER_DROPPED, '')
continue
@@ -190,13 +192,14 @@ export class HostRelay extends HostRelayPhones {
// drop counts stay comparable with the cells, whose pings would have closed it earlier.
// Its probe is cut short: the expired authorization ends the socket either way.
if (now - seen > this.config.silenceTimeoutMs) {
this.deadControlMetric(a, now, seen, 'expired')
this.close(ws, RELAY_CLOSE_CODE.PEER_DROPPED, 'control silence timeout')
await this.onSocketGone(ws, RELAY_CLOSE_CODE.PEER_DROPPED, '', Math.min(now, seen + this.config.silenceTimeoutMs + this.config.pingIntervalMs))
continue
}
const sinceRefreshS = a.lastRefreshAt === undefined ? -1 : Math.round((now - a.lastRefreshAt) / 1000)
const seenAgoS = Math.round((now - seen) / 1000)
metric(this.env, 'auth_expired_close', sinceRefreshS, a.lastRefreshAt === undefined ? 'never' : 'refreshed', [seenAgoS], this.keepalive(a) ? 'keepalive' : 'legacy', a.place)
metric(this.env, 'auth_expired_close', sinceRefreshS, a.lastRefreshAt === undefined ? 'never' : 'refreshed', [seenAgoS, a.activeAt === undefined ? -1 : Math.round((now - a.activeAt) / 1000)], this.keepalive(a) ? 'keepalive' : 'legacy', a.place)
this.close(ws, RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, 'relay authorization expired')
await this.onSocketGone(ws, RELAY_CLOSE_CODE.BAD_OUTER_CREDENTIAL, '')
continue
+6 -2
View File
@@ -2,7 +2,8 @@
import type { Env } from './env'
export type MetricEvent =
// label 'new' | 'rebind'; reason 'keepalive' | 'legacy' (took the keepalive offer or not).
// label 'new' | 'rebind'; reason 'keepalive' | 'legacy' (took the keepalive offer or not);
// double2 = ms from the upgrade to activation (the desktop gives up a connect after 15 s).
| 'control_active'
| 'control_close'
| 'phone_open'
@@ -27,8 +28,11 @@ export type MetricEvent =
| 'control_upgrade'
// Control closed 4401 for expired authorization: value = seconds since its last extending
// refresh, label 'refreshed'; -1 and label 'never' when this socket never extended its token.
// double2 = seconds since the host's last inbound frame or keepalive; reason 'keepalive' | 'legacy'.
// double2 = seconds since the host's last inbound frame or keepalive, double3 = seconds since the
// control became active (-1 if unknown); reason 'keepalive' | 'legacy'.
| 'auth_expired_close'
// Control closed for silence (deadControlMetric, host-relay-control.ts).
| 'silence_close'
// Pre-auth socket refused by the per-source limiter: label 'phone' | 'data' | 'health'. Unpushed-host phones: 'not_on_edge'.
| 'preauth_limited'
| 'not_on_edge'