fix(relay): give each failing cell the short retry cap once per episode

A director alternating between failing cells (A, B, A, B) kept the 60 s cap
forever because only the previous cell was compared. Track every cell that
failed since the last success; a repeat of any of them uses the full backoff.
This commit is contained in:
Neil
2026-11-13 04:01:59 -08:00
parent 081cc4447b
commit 630bd89d4a
2 changed files with 26 additions and 6 deletions
@@ -121,6 +121,21 @@ describe('RelayOriginPool drain retry cap', () => {
expect(Math.max(...deadCell)).toBeGreaterThan(250_000)
})
it('falls back to the full backoff when the director alternates between failing cells', async () => {
const deadCell = await asksDuring(THIRTY_MINUTES, async () => assignment)
harness.origins.length = 0
harness.failOpens = false
let epoch = 1
const alternating = await asksDuring(THIRTY_MINUTES, async () => {
epoch += 1
const cell = epoch % 2 === 0 ? 'a' : 'b'
return { ...assignment, cellUrl: `https://relay-${cell}.example`, assignmentEpoch: epoch }
})
// Each of the two cells gets the short cap once; after that the rate matches a dead cell.
expect(alternating.length).toBeLessThanOrEqual(deadCell.length + 2)
expect(Math.max(...alternating)).toBeGreaterThan(250_000)
})
it('retries within 60s each time the director names a different cell that then fails', async () => {
let epoch = 1
const gaps = await asksDuring(THIRTY_MINUTES, async () => {
@@ -13,8 +13,9 @@ import {
import { RelayControlRotation } from './relay-control-rotation'
// The director answered and only a cell it newly named failed: retry that once sooner. Director
// failures, and the same cell failing again, keep the full backoff: every retry re-asks the
// director, so a short cap on a dead cell would multiply director load for every host on it.
// failures, and any cell failing again in the same episode (even alternating A, B, A), keep the
// full backoff: every retry re-asks the director, so a short cap on dead cells would multiply
// director load for every host on them.
const CELL_CONNECT_RETRY_MAX_MS = 60_000
export class RelayOriginPool {
@@ -30,7 +31,8 @@ export class RelayOriginPool {
private readonly basisOrigins = this.retirement.basis
private assignment: RelayAssignment | null = null
private deferredAssignment: RelayAssignment | null = null
private failedCellUrl: string | null = null
// Cells that failed since the last success: each gets the short cap once.
private readonly failedCells = new Set<string>()
private relayJwt: string | null = null
private readonly rotation: RelayControlRotation
private rotationPromise: Promise<void> | null = null
@@ -257,10 +259,13 @@ export class RelayOriginPool {
this.retirement.schedule(origin, graceMs)
this.retirement.maybeClose(origin)
}
// attempts is 0 right after a success, so a cell that worked before counts as new again.
// attempts is 0 only right after a success, which starts a new episode.
private isNewCellFailure(cellUrl: string): boolean {
const repeated = this.drainRetry.attempts > 0 && this.failedCellUrl === cellUrl
this.failedCellUrl = cellUrl
if (this.drainRetry.attempts === 0) {
this.failedCells.clear()
}
const repeated = this.failedCells.has(cellUrl)
this.failedCells.add(cellUrl)
return !repeated
}