fix(relay): dead-man boot grace for a freshly booted reserve cell (#37225)

* fix(relay): give a freshly booted reserve cell a dead-man boot grace until its first director contact

A reserve cell's dead-man starts its 60-120 s window when the switch file applies, but no
placing director can reach a new process until its load balancer routes to it. On the
2026-11-01 production same-cap rolls that took up to about 110 s after the switch file
applied; c10 (104 s window) and c16 (71 s) tripped 1-4 s before the first director poll
landed, latched into db, and failed their roll after a clean drain and restart.

Until the process's first contact (a reserver=1 feed poll or a real booking), the window is
the hashed window plus 180 s, matching autohealing's initial delay. After the first contact
nothing changes. A process that never hears from a placing director still falls back to db,
up to three minutes later than before.

* test(relay): assert the server wires the dead-man boot grace

* docs(relay): state the boot gap behind the dead-man grace as inferred, not measured

cloud/apps/relay/src/cell-phone-path.blackbox.test.ts

@@ -2,7 +2,7 @@ import { mkdtempSync, rmSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { afterEach, describe, expect, it, vi } from 'vitest'
-import { reserveDeadManWindowMs } from './cell-reserve-dead-man.js'
+import { RESERVE_DEAD_MAN_BOOT_GRACE_MS, reserveDeadManWindowMs } from './cell-reserve-dead-man.js'
 import { openRelayDatabase } from './database.js'
 import { nextFrame, RESERVE_MODE_CELL_ID, startReserveModeCell } from './test-fixtures/reserve-mode-cell.js'
 import { StatementRecordingDatabase } from './test-fixtures/statement-recording-database.js'
@@ -108,7 +108,10 @@ describe('the memory phone path', () => {
     expect(reserve.relay.sessions.phonePathEffective()).toBe('db')
     reserve.setAdmitMode('reserve', { phonePath: 'memory' })
     expect(reserve.relay.sessions.phonePathEffective()).toBe('memory')
+    // No director has polled this process yet: past its plain window it is still in its boot grace.
     now += reserveDeadManWindowMs(reserve.config.cellId) + 1_000
+    expect(reserve.relay.sessions.inReserveMode()).toBe(true)
+    now += RESERVE_DEAD_MAN_BOOT_GRACE_MS
     expect(reserve.relay.sessions.inReserveMode()).toBe(false)
     expect(reserve.relay.sessions.phonePathEffective()).toBe('db')
   })

cloud/apps/relay/src/cell-reserve-dead-man.test.ts

@@ -5,14 +5,15 @@ import type { AppliedControlFlags } from './relay-control-flag-channel.js'
 
 const WINDOW_MS = 60_000
 
-function setup() {
+function setup(bootGraceMs?: number) {
   const clock = { value: 1_000_000 }
   const logs: string[] = []
   const deadMan = new ReserveDeadMan(
     'production-gce-c30',
     () => clock.value,
     WINDOW_MS,
-    (line) => logs.push(line)
+    (line) => logs.push(line),
+    bootGraceMs
   )
   const events = (name: string) =>
     logs.map((line) => JSON.parse(line)).filter((line) => line.event === name)
@@ -37,6 +38,40 @@ describe('ReserveDeadMan', () => {
     expect(events('orca_relay_cell_reserve_dead_man_tripped')).toHaveLength(1)
   })
 
+  it('gives a fresh process the boot grace until its first contact, then the plain window', () => {
+    const grace = 3 * WINDOW_MS
+    const { clock, deadMan, events } = setup(grace)
+    expect(deadMan.mode(reserve(1))).toBe('reserve')
+    // No director has reached the new process yet: past its window, still inside the grace.
+    clock.value += WINDOW_MS + grace
+    expect(deadMan.mode(reserve(1))).toBe('reserve')
+    deadMan.contact()
+    // After the first contact, silence trips on the plain window again.
+    clock.value += WINDOW_MS + 1
+    expect(deadMan.mode(reserve(1))).toBe('db')
+    expect(events('orca_relay_cell_reserve_dead_man_tripped')).toHaveLength(1)
+  })
+
+  it('still trips a fresh process that never hears from a placing director', () => {
+    const grace = 3 * WINDOW_MS
+    const { clock, deadMan, events } = setup(grace)
+    expect(deadMan.mode(reserve(1))).toBe('reserve')
+    clock.value += WINDOW_MS + grace + 1
+    expect(deadMan.mode(reserve(1))).toBe('db')
+    expect(events('orca_relay_cell_reserve_dead_man_tripped')).toEqual([
+      expect.objectContaining({ silentMs: WINDOW_MS + grace + 1 })
+    ])
+  })
+
+  it('keeps the plain window for a later reserve write once a director has been in touch', () => {
+    const { clock, deadMan } = setup(3 * WINDOW_MS)
+    deadMan.mode(reserve(1))
+    deadMan.contact()
+    expect(deadMan.mode(reserve(2))).toBe('reserve')
+    clock.value += WINDOW_MS + 1
+    expect(deadMan.mode(reserve(2))).toBe('db')
+  })
+
   it('with deadManTripSlots, holds reserve until its slot claim lands, then trips and latches', () => {
     const { clock, deadMan, events } = setup()
     const applied = reserve(1, { deadManTripSlots: 2 })

cloud/apps/relay/src/cell-reserve-dead-man.ts

@@ -5,6 +5,11 @@ import type { AppliedControlFlags } from './relay-control-flag-channel.js'
 // Without placing-director contact, memory-only reserve ownership cannot supersede hosts; flip back to db and re-register as a switch flip would.
 export const RESERVE_DEAD_MAN_MS = 60_000
 const RESERVE_DEAD_MAN_SPREAD_MS = 60_000
+// A fresh process waits for its load balancer before any director poll can reach it. On the
+// 2026-11-01 rolls that gap outlasted the 71-103 s windows of c10, c16 and c24 (inferred, not
+// measured: roughly 70-125 s). Until its first contact a new process gets this much more
+// (autohealing's 180 s initial delay).
+export const RESERVE_DEAD_MAN_BOOT_GRACE_MS = 180_000
 
 // Stable 60–120s per-cell windows stagger fleet-wide fallback DB re-registration over one minute.
 export function reserveDeadManWindowMs(cellId: string): number {
@@ -19,12 +24,14 @@ export class ReserveDeadMan {
   // With deadManTripSlots set, a silent cell trips only once it holds one of the fleet's slots.
   private permittedGeneration: number | null = null
   private pending: { generation: number; slots: number } | null = null
+  private contacted = false
 
   constructor(
     private readonly cellId: string,
     private readonly now: () => number = Date.now,
     private readonly windowMs = RESERVE_DEAD_MAN_MS,
-    private readonly log: (line: string) => void = (line) => console.error(line)
+    private readonly log: (line: string) => void = (line) => console.error(line),
+    private readonly bootGraceMs = 0
   ) {
     this.lastContactAt = now()
   }
@@ -32,6 +39,7 @@ export class ReserveDeadMan {
   // A feed poll marked reserver=1, or a real (not dry-run) booking.
   contact(): void {
     this.lastContactAt = this.now()
+    this.contacted = true
     // A claim still in flight is for a silence that has ended.
     this.pending = null
     this.permittedGeneration = null
@@ -53,7 +61,8 @@ export class ReserveDeadMan {
       this.lastContactAt = Math.max(this.lastContactAt, now)
     }
     if (this.trippedGeneration === applied.generation) return 'db'
-    if (now - this.lastContactAt <= this.windowMs) {
+    const windowMs = this.contacted ? this.windowMs : this.windowMs + this.bootGraceMs
+    if (now - this.lastContactAt <= windowMs) {
       this.pending = null
       return 'reserve'
     }

cloud/apps/relay/src/relay-server.ts

@@ -19,7 +19,11 @@ import { WebSocketServer } from 'ws'
 import type WebSocket from 'ws'
 import type { RawData } from 'ws'
 import { CellAdmitEffectiveWriter } from './cell-admit-effective-writer.js'
-import { ReserveDeadMan, reserveDeadManWindowMs } from './cell-reserve-dead-man.js'
+import {
+  RESERVE_DEAD_MAN_BOOT_GRACE_MS,
+  ReserveDeadMan,
+  reserveDeadManWindowMs
+} from './cell-reserve-dead-man.js'
 import { createRelayApp } from './app.js'
 import { classifyAssignmentLease } from './assignment/assignment-lease.js'
 import { defaultIntakePerSec, type CellFlags } from './cell-flags.js'
@@ -347,7 +351,13 @@ export function createRelayServer(
     readRelayDatabasePoolPressure(database).databasePoolWaiting >= config.databasePoolMax &&
     readRelayDatabasePoolOldestWaitMs(database) >= HOST_HELLO_SHED_OLDEST_WAIT_MS
   const reserveDeadMan = reserveBook
-    ? new ReserveDeadMan(config.cellId, options.now, reserveDeadManWindowMs(config.cellId))
+    ? new ReserveDeadMan(
+        config.cellId,
+        options.now,
+        reserveDeadManWindowMs(config.cellId),
+        undefined,
+        RESERVE_DEAD_MAN_BOOT_GRACE_MS
+      )
     : null
   const effectiveAdmitMode = (): 'db' | 'reserve' => {
     const applied = options.cellFlags?.()