diff --git a/cloud/apps/relay/src/cell-phone-path.blackbox.test.ts b/cloud/apps/relay/src/cell-phone-path.blackbox.test.ts index d1ece6ab745..62e500a4427 100644 --- a/cloud/apps/relay/src/cell-phone-path.blackbox.test.ts +++ b/cloud/apps/relay/src/cell-phone-path.blackbox.test.ts @@ -2,7 +2,7 @@ import { mkdtempSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { afterEach, describe, expect, it, vi } from 'vitest' -import { reserveDeadManWindowMs } from './cell-reserve-dead-man.js' +import { RESERVE_DEAD_MAN_BOOT_GRACE_MS, reserveDeadManWindowMs } from './cell-reserve-dead-man.js' import { openRelayDatabase } from './database.js' import { nextFrame, RESERVE_MODE_CELL_ID, startReserveModeCell } from './test-fixtures/reserve-mode-cell.js' import { StatementRecordingDatabase } from './test-fixtures/statement-recording-database.js' @@ -108,7 +108,10 @@ describe('the memory phone path', () => { expect(reserve.relay.sessions.phonePathEffective()).toBe('db') reserve.setAdmitMode('reserve', { phonePath: 'memory' }) expect(reserve.relay.sessions.phonePathEffective()).toBe('memory') + // No director has polled this process yet: past its plain window it is still in its boot grace. now += reserveDeadManWindowMs(reserve.config.cellId) + 1_000 + expect(reserve.relay.sessions.inReserveMode()).toBe(true) + now += RESERVE_DEAD_MAN_BOOT_GRACE_MS expect(reserve.relay.sessions.inReserveMode()).toBe(false) expect(reserve.relay.sessions.phonePathEffective()).toBe('db') }) diff --git a/cloud/apps/relay/src/cell-reserve-dead-man.test.ts b/cloud/apps/relay/src/cell-reserve-dead-man.test.ts index 30ba0eaee57..3203b7263fc 100644 --- a/cloud/apps/relay/src/cell-reserve-dead-man.test.ts +++ b/cloud/apps/relay/src/cell-reserve-dead-man.test.ts @@ -5,14 +5,15 @@ import type { AppliedControlFlags } from './relay-control-flag-channel.js' const WINDOW_MS = 60_000 -function setup() { +function setup(bootGraceMs?: number) { const clock = { value: 1_000_000 } const logs: string[] = [] const deadMan = new ReserveDeadMan( 'production-gce-c30', () => clock.value, WINDOW_MS, - (line) => logs.push(line) + (line) => logs.push(line), + bootGraceMs ) const events = (name: string) => logs.map((line) => JSON.parse(line)).filter((line) => line.event === name) @@ -37,6 +38,40 @@ describe('ReserveDeadMan', () => { expect(events('orca_relay_cell_reserve_dead_man_tripped')).toHaveLength(1) }) + it('gives a fresh process the boot grace until its first contact, then the plain window', () => { + const grace = 3 * WINDOW_MS + const { clock, deadMan, events } = setup(grace) + expect(deadMan.mode(reserve(1))).toBe('reserve') + // No director has reached the new process yet: past its window, still inside the grace. + clock.value += WINDOW_MS + grace + expect(deadMan.mode(reserve(1))).toBe('reserve') + deadMan.contact() + // After the first contact, silence trips on the plain window again. + clock.value += WINDOW_MS + 1 + expect(deadMan.mode(reserve(1))).toBe('db') + expect(events('orca_relay_cell_reserve_dead_man_tripped')).toHaveLength(1) + }) + + it('still trips a fresh process that never hears from a placing director', () => { + const grace = 3 * WINDOW_MS + const { clock, deadMan, events } = setup(grace) + expect(deadMan.mode(reserve(1))).toBe('reserve') + clock.value += WINDOW_MS + grace + 1 + expect(deadMan.mode(reserve(1))).toBe('db') + expect(events('orca_relay_cell_reserve_dead_man_tripped')).toEqual([ + expect.objectContaining({ silentMs: WINDOW_MS + grace + 1 }) + ]) + }) + + it('keeps the plain window for a later reserve write once a director has been in touch', () => { + const { clock, deadMan } = setup(3 * WINDOW_MS) + deadMan.mode(reserve(1)) + deadMan.contact() + expect(deadMan.mode(reserve(2))).toBe('reserve') + clock.value += WINDOW_MS + 1 + expect(deadMan.mode(reserve(2))).toBe('db') + }) + it('with deadManTripSlots, holds reserve until its slot claim lands, then trips and latches', () => { const { clock, deadMan, events } = setup() const applied = reserve(1, { deadManTripSlots: 2 }) diff --git a/cloud/apps/relay/src/cell-reserve-dead-man.ts b/cloud/apps/relay/src/cell-reserve-dead-man.ts index c66110b46a3..a02a671fadd 100644 --- a/cloud/apps/relay/src/cell-reserve-dead-man.ts +++ b/cloud/apps/relay/src/cell-reserve-dead-man.ts @@ -5,6 +5,11 @@ import type { AppliedControlFlags } from './relay-control-flag-channel.js' // Without placing-director contact, memory-only reserve ownership cannot supersede hosts; flip back to db and re-register as a switch flip would. export const RESERVE_DEAD_MAN_MS = 60_000 const RESERVE_DEAD_MAN_SPREAD_MS = 60_000 +// A fresh process waits for its load balancer before any director poll can reach it. On the +// 2026-11-01 rolls that gap outlasted the 71-103 s windows of c10, c16 and c24 (inferred, not +// measured: roughly 70-125 s). Until its first contact a new process gets this much more +// (autohealing's 180 s initial delay). +export const RESERVE_DEAD_MAN_BOOT_GRACE_MS = 180_000 // Stable 60–120s per-cell windows stagger fleet-wide fallback DB re-registration over one minute. export function reserveDeadManWindowMs(cellId: string): number { @@ -19,12 +24,14 @@ export class ReserveDeadMan { // With deadManTripSlots set, a silent cell trips only once it holds one of the fleet's slots. private permittedGeneration: number | null = null private pending: { generation: number; slots: number } | null = null + private contacted = false constructor( private readonly cellId: string, private readonly now: () => number = Date.now, private readonly windowMs = RESERVE_DEAD_MAN_MS, - private readonly log: (line: string) => void = (line) => console.error(line) + private readonly log: (line: string) => void = (line) => console.error(line), + private readonly bootGraceMs = 0 ) { this.lastContactAt = now() } @@ -32,6 +39,7 @@ export class ReserveDeadMan { // A feed poll marked reserver=1, or a real (not dry-run) booking. contact(): void { this.lastContactAt = this.now() + this.contacted = true // A claim still in flight is for a silence that has ended. this.pending = null this.permittedGeneration = null @@ -53,7 +61,8 @@ export class ReserveDeadMan { this.lastContactAt = Math.max(this.lastContactAt, now) } if (this.trippedGeneration === applied.generation) return 'db' - if (now - this.lastContactAt <= this.windowMs) { + const windowMs = this.contacted ? this.windowMs : this.windowMs + this.bootGraceMs + if (now - this.lastContactAt <= windowMs) { this.pending = null return 'reserve' } diff --git a/cloud/apps/relay/src/relay-server.ts b/cloud/apps/relay/src/relay-server.ts index 1bfef28a483..7965c7a9919 100644 --- a/cloud/apps/relay/src/relay-server.ts +++ b/cloud/apps/relay/src/relay-server.ts @@ -19,7 +19,11 @@ import { WebSocketServer } from 'ws' import type WebSocket from 'ws' import type { RawData } from 'ws' import { CellAdmitEffectiveWriter } from './cell-admit-effective-writer.js' -import { ReserveDeadMan, reserveDeadManWindowMs } from './cell-reserve-dead-man.js' +import { + RESERVE_DEAD_MAN_BOOT_GRACE_MS, + ReserveDeadMan, + reserveDeadManWindowMs +} from './cell-reserve-dead-man.js' import { createRelayApp } from './app.js' import { classifyAssignmentLease } from './assignment/assignment-lease.js' import { defaultIntakePerSec, type CellFlags } from './cell-flags.js' @@ -347,7 +351,13 @@ export function createRelayServer( readRelayDatabasePoolPressure(database).databasePoolWaiting >= config.databasePoolMax && readRelayDatabasePoolOldestWaitMs(database) >= HOST_HELLO_SHED_OLDEST_WAIT_MS const reserveDeadMan = reserveBook - ? new ReserveDeadMan(config.cellId, options.now, reserveDeadManWindowMs(config.cellId)) + ? new ReserveDeadMan( + config.cellId, + options.now, + reserveDeadManWindowMs(config.cellId), + undefined, + RESERVE_DEAD_MAN_BOOT_GRACE_MS + ) : null const effectiveAdmitMode = (): 'db' | 'reserve' => { const applied = options.cellFlags?.()