mirror of
https://github.com/stablyai/orca.git
synced 2026-09-30 00:03:15 +00:00
Jinwoo Hong
ef428d879e
On 2026-09-04 an auth outage signed ~21,600 desktops out of Orca Cloud and every paired phone showed the generic "Can't reach desktop" for hours. The desktop knew why, the cell watched it happen, and neither could say so. The desktop now names auth loss on its control close reason; the cell remembers that reason per (userId, relayHostId) and replays it as the close reason of the 4404 it already sends a phone whose host is absent; the phone turns it into "Desktop signed out — sign in to Orca on your desktop to reconnect". Retry cadence, close codes and every message body are untouched. The reason rides the WebSocket close reason because there is no additive JSON channel to a shipped phone: RelayPhoneHelloSchema, RelayAuthSchema and the director's ResolveResponseSchema are all zod .strict(), and /v1/connect rejects any query string outright. A new close code was also rejected — an old phone would fall out of mobileRelayRecoveryFor and off the 5-15s host-offline backoff onto the faster transport backoff. The cell keeps the reason in memory rather than Postgres: a phone reaches the cell its host's assignment row already names, which is the cell that saw the close, and losing it on a cell restart degrades to today's verdict rather than a wrong one.
73 lines
2.2 KiB
TypeScript
73 lines
2.2 KiB
TypeScript
import {
|
|
ASSIGNMENT_LIMITS,
|
|
relayHostCloseReasonFrom,
|
|
type RelayHostCloseReason
|
|
} from '@orca-cloud/relay-contract'
|
|
|
|
// Retention matches the dormant assignment TTL: past it the host may have been
|
|
// rebalanced onto another cell, so this cell is no longer the one a phone asks.
|
|
const RETENTION_MS = ASSIGNMENT_LIMITS.dormantTtlMs
|
|
// A fleet-wide auth outage signs out every host at once; the cap bounds that
|
|
// burst well above any single cell's host count without becoming a leak.
|
|
const MAX_ENTRIES = 50_000
|
|
|
|
// Why in-memory and not Postgres: a phone reaches the cell its host's assignment
|
|
// row already names, which is the same cell that watched the control socket
|
|
// close. Losing this on a cell restart degrades to the pre-existing generic
|
|
// verdict, so the failure mode is the old behaviour rather than a wrong one.
|
|
export class HostCloseReasonMemory {
|
|
private readonly entries = new Map<string, { reason: RelayHostCloseReason; expiresAt: number }>()
|
|
|
|
constructor(private readonly now: () => number = Date.now) {}
|
|
|
|
// Silently ignores anything that is not a known reason, which is every close
|
|
// from a host that predates the field and every abrupt 1006.
|
|
record(key: string, reason: unknown): void {
|
|
const parsed = relayHostCloseReasonFrom(reason)
|
|
if (!parsed) {
|
|
return
|
|
}
|
|
this.entries.delete(key)
|
|
this.entries.set(key, { reason: parsed, expiresAt: this.now() + RETENTION_MS })
|
|
this.evict()
|
|
}
|
|
|
|
forget(key: string): void {
|
|
this.entries.delete(key)
|
|
}
|
|
|
|
read(key: string): RelayHostCloseReason | null {
|
|
const entry = this.entries.get(key)
|
|
if (!entry) {
|
|
return null
|
|
}
|
|
if (entry.expiresAt <= this.now()) {
|
|
this.entries.delete(key)
|
|
return null
|
|
}
|
|
return entry.reason
|
|
}
|
|
|
|
size(): number {
|
|
return this.entries.size
|
|
}
|
|
|
|
private evict(): void {
|
|
const now = this.now()
|
|
for (const [key, entry] of this.entries) {
|
|
if (entry.expiresAt > now) {
|
|
break
|
|
}
|
|
this.entries.delete(key)
|
|
}
|
|
// Insertion order is recency order (record deletes before setting), so the
|
|
// head is always the oldest survivor.
|
|
for (const key of this.entries.keys()) {
|
|
if (this.entries.size <= MAX_ENTRIES) {
|
|
break
|
|
}
|
|
this.entries.delete(key)
|
|
}
|
|
}
|
|
}
|