mirror of
https://github.com/stablyai/orca.git
synced 2026-11-14 09:57:48 +00:00
fix(relay): give each failing cell the short retry cap once per episode
A director alternating between failing cells (A, B, A, B) kept the 60 s cap forever because only the previous cell was compared. Track every cell that failed since the last success; a repeat of any of them uses the full backoff.
This commit is contained in:
@@ -121,6 +121,21 @@ describe('RelayOriginPool drain retry cap', () => {
|
||||
expect(Math.max(...deadCell)).toBeGreaterThan(250_000)
|
||||
})
|
||||
|
||||
it('falls back to the full backoff when the director alternates between failing cells', async () => {
|
||||
const deadCell = await asksDuring(THIRTY_MINUTES, async () => assignment)
|
||||
harness.origins.length = 0
|
||||
harness.failOpens = false
|
||||
let epoch = 1
|
||||
const alternating = await asksDuring(THIRTY_MINUTES, async () => {
|
||||
epoch += 1
|
||||
const cell = epoch % 2 === 0 ? 'a' : 'b'
|
||||
return { ...assignment, cellUrl: `https://relay-${cell}.example`, assignmentEpoch: epoch }
|
||||
})
|
||||
// Each of the two cells gets the short cap once; after that the rate matches a dead cell.
|
||||
expect(alternating.length).toBeLessThanOrEqual(deadCell.length + 2)
|
||||
expect(Math.max(...alternating)).toBeGreaterThan(250_000)
|
||||
})
|
||||
|
||||
it('retries within 60s each time the director names a different cell that then fails', async () => {
|
||||
let epoch = 1
|
||||
const gaps = await asksDuring(THIRTY_MINUTES, async () => {
|
||||
|
||||
@@ -13,8 +13,9 @@ import {
|
||||
import { RelayControlRotation } from './relay-control-rotation'
|
||||
|
||||
// The director answered and only a cell it newly named failed: retry that once sooner. Director
|
||||
// failures, and the same cell failing again, keep the full backoff: every retry re-asks the
|
||||
// director, so a short cap on a dead cell would multiply director load for every host on it.
|
||||
// failures, and any cell failing again in the same episode (even alternating A, B, A), keep the
|
||||
// full backoff: every retry re-asks the director, so a short cap on dead cells would multiply
|
||||
// director load for every host on them.
|
||||
const CELL_CONNECT_RETRY_MAX_MS = 60_000
|
||||
|
||||
export class RelayOriginPool {
|
||||
@@ -30,7 +31,8 @@ export class RelayOriginPool {
|
||||
private readonly basisOrigins = this.retirement.basis
|
||||
private assignment: RelayAssignment | null = null
|
||||
private deferredAssignment: RelayAssignment | null = null
|
||||
private failedCellUrl: string | null = null
|
||||
// Cells that failed since the last success: each gets the short cap once.
|
||||
private readonly failedCells = new Set<string>()
|
||||
private relayJwt: string | null = null
|
||||
private readonly rotation: RelayControlRotation
|
||||
private rotationPromise: Promise<void> | null = null
|
||||
@@ -257,10 +259,13 @@ export class RelayOriginPool {
|
||||
this.retirement.schedule(origin, graceMs)
|
||||
this.retirement.maybeClose(origin)
|
||||
}
|
||||
// attempts is 0 right after a success, so a cell that worked before counts as new again.
|
||||
// attempts is 0 only right after a success, which starts a new episode.
|
||||
private isNewCellFailure(cellUrl: string): boolean {
|
||||
const repeated = this.drainRetry.attempts > 0 && this.failedCellUrl === cellUrl
|
||||
this.failedCellUrl = cellUrl
|
||||
if (this.drainRetry.attempts === 0) {
|
||||
this.failedCells.clear()
|
||||
}
|
||||
const repeated = this.failedCells.has(cellUrl)
|
||||
this.failedCells.add(cellUrl)
|
||||
return !repeated
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user