mirror of
https://github.com/stablyai/orca.git
synced 2026-09-30 16:02:56 +00:00
fix(relay): stop dead accept work, spread control rotations, fail direct probes fast
Incident 2026-09-04 ~01:05Z: after a background/foreground cycle the phone's
relay dial timed out inside the cell's acceptClient DB phase while the fleet
was in a cell-inventory lock storm (55P03 retries ~7.5k/h vs a ~1k/h floor).
Relay (cloud/apps/relay)
- acceptClient checks socket.readyState after each serialized Postgres call
and abandons the accept once the phone has hung up, releasing the capacity
reservation, failing the credential reservation, and releasing the activity
lease it just acquired instead of leaking it to expiry cleanup and then
throwing host_data_reservation_already_bound at bind.
- New structured event orca_relay_client_accept_abandoned {stage, elapsedMs}
and runtime-metric fields clientAcceptsAbandonedByStageDelta /
clientAcceptAbandonedMsMax so the "phone gave up behind the lock" rate is
quantifiable per cell.
- Control lease grants are jittered: 55 min minus [0, 10 min). Every host
that (re)connected in the same minute rebound as one cohort every ~54 min
(c27 autoheal recreate at 23:23Z re-homed ~420 controls; ~1.1k 1006 +
~1k 4408 "control rebound" closes landed in a 3 s window at 00:50:14Z),
and each rebind is an activateControl transaction on the inventory lock.
No wire change: leaseExpiresAt was always a server-chosen absolute time.
Desktop (src/main/runtime/relay)
- Control rotation rebinds 1-6 min early instead of 1-2 min, so a re-homed
cohort spreads across cycles rather than pinning one phase for the life of
the process.
Phone (mobile/src/transport)
- openAuthenticatedDirectEndpoint treats 'reconnecting' as a failed probe. On
a dead LAN the foreground direct dial dies with an instant 1006 and the
direct client enters its own 500/1000/2000 ms backoff; the probe used to
wait out its full 12 s bound holding the supervisor mutex, so relay recovery
queued behind three doomed redials. The stage-aware bound from #18518 is
unaffected.
Not done here: rolling the 23 GCE cells onto the post-#18521 image (500 ms
lock_timeout) is a deploy owned by cloud-deploy-relay-production-same-cap.
This commit is contained in:
@@ -9,6 +9,15 @@ import { RelayHttpError, requestRelayAssignment, type RelayAssignment } from './
|
||||
import type { RelayBrokerStatus, RelayIdentity } from './relay-session-broker-contract'
|
||||
import type { RelayRegion } from './relay-region-preference'
|
||||
|
||||
// Why: every host whose lease expires in the same minute rebinds in the same
|
||||
// minute, and each rebind takes the relay's contended cell-inventory lock. A
|
||||
// cell recreate re-homes hundreds of hosts at once and pins that cohort to one
|
||||
// phase for the life of the process (observed 2026-09-04: ~1k rebinds in 3s
|
||||
// every ~54 min). A wide early window spreads each cycle; the floor keeps the
|
||||
// rebind clear of the relay's expiry sweep even under a slow director.
|
||||
const CONTROL_ROTATION_EARLY_MIN_MS = 60_000
|
||||
const CONTROL_ROTATION_EARLY_JITTER_MS = 5 * 60_000
|
||||
|
||||
type RelayOriginPoolOptions = {
|
||||
directorUrl: string
|
||||
relayHostId: string
|
||||
@@ -244,7 +253,8 @@ export class RelayOriginPool {
|
||||
}
|
||||
const now = (this.options.now ?? Date.now)()
|
||||
const random = this.options.random ?? Math.random
|
||||
const earlyMs = 60_000 + Math.floor(random() * 60_001)
|
||||
const earlyMs =
|
||||
CONTROL_ROTATION_EARLY_MIN_MS + Math.floor(random() * (CONTROL_ROTATION_EARLY_JITTER_MS + 1))
|
||||
const delay = Math.max(0, origin.controlLeaseExpiresAt - earlyMs - now)
|
||||
this.rotationTimer = setTimeout(() => void this.rebindActiveControl(origin), delay)
|
||||
}
|
||||
|
||||
@@ -486,6 +486,59 @@ describe('RelaySessionBroker lifecycle ownership', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('RelaySessionBroker control rotation spreading', () => {
|
||||
beforeEach(() => {
|
||||
fakes.controls.length = 0
|
||||
fakes.transports.length = 0
|
||||
fakes.controlConnect.mockReset()
|
||||
fakes.exchange.mockReset().mockResolvedValue({ relayToken: 'relay-jwt', expiresAt: 10_000_000 })
|
||||
fakes.assign.mockReset().mockResolvedValue({
|
||||
cellUrl: 'https://relay.example.test',
|
||||
assignmentEpoch: 1,
|
||||
leaseExpiresAt: 10_000_000
|
||||
})
|
||||
})
|
||||
|
||||
// Incident 2026-09-04 00:50Z: ~1k hosts re-homed by one cell recreate rebound
|
||||
// together every ~54 min, each rebind taking the relay's cell-inventory lock.
|
||||
it('spreads same-lease hosts across a multi-minute window instead of one minute', async () => {
|
||||
vi.useFakeTimers()
|
||||
try {
|
||||
const leaseExpiresAt = 55 * 60_000
|
||||
const ack: RelayHostHelloAckMessage = {
|
||||
type: 'host-hello-ack',
|
||||
v: 1,
|
||||
generation: 1,
|
||||
controlResumeSecret: 'R'.repeat(43),
|
||||
leaseExpiresAt,
|
||||
activeConnIds: [],
|
||||
pendingConns: []
|
||||
}
|
||||
fakes.controlConnect.mockResolvedValue(ack)
|
||||
const earliest = await RelaySessionBroker.connect(brokerOptions({ random: () => 0.999999 }))
|
||||
const latest = await RelaySessionBroker.connect(brokerOptions({ random: () => 0 }))
|
||||
expect(fakes.controls).toHaveLength(2)
|
||||
|
||||
// The widest early roll rebinds ~6 min before expiry; the narrowest at 1 min.
|
||||
await vi.advanceTimersByTimeAsync(leaseExpiresAt - 6 * 60_000 - 1)
|
||||
expect(fakes.controls).toHaveLength(2)
|
||||
await vi.advanceTimersByTimeAsync(2)
|
||||
expect(fakes.controls).toHaveLength(3)
|
||||
expect(fakes.controls[2]!.options.previousGeneration).toBe(1)
|
||||
|
||||
await vi.advanceTimersByTimeAsync(4 * 60_000 + 59_000)
|
||||
expect(fakes.controls).toHaveLength(3)
|
||||
await vi.advanceTimersByTimeAsync(1_000)
|
||||
expect(fakes.controls).toHaveLength(4)
|
||||
|
||||
earliest.closeNow()
|
||||
latest.closeNow()
|
||||
} finally {
|
||||
vi.useRealTimers()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
function brokerBasisIds(broker: RelaySessionBroker): string[] {
|
||||
const pool = (broker as unknown as { originPool: unknown }).originPool
|
||||
return [...(pool as { basisOrigins: Map<string, unknown> }).basisOrigins.keys()]
|
||||
|
||||
Reference in New Issue
Block a user