From 630bd89d4a705eba7ba95a7d0fbb9db7367d4a07 Mon Sep 17 00:00:00 2001
From: Neil <neil@stably.ai>
Date: Fri, 13 Nov 2026 04:01:59 -0800
Subject: [PATCH] fix(relay): give each failing cell the short retry cap once
 per episode

A director alternating between failing cells (A, B, A, B) kept the 60 s cap
forever because only the previous cell was compared. Track every cell that
failed since the last success; a repeat of any of them uses the full backoff.
---
 .../relay-origin-pool-retry.test.ts             | 15 +++++++++++++++
 .../origin-lifecycle/relay-origin-pool.ts       | 17 +++++++++++------
 2 files changed, 26 insertions(+), 6 deletions(-)

diff --git a/src/main/runtime/mobile-cloud-relay/origin-lifecycle/relay-origin-pool-retry.test.ts b/src/main/runtime/mobile-cloud-relay/origin-lifecycle/relay-origin-pool-retry.test.ts
index 336fe4bd882..6eab94666fc 100644
--- a/src/main/runtime/mobile-cloud-relay/origin-lifecycle/relay-origin-pool-retry.test.ts
+++ b/src/main/runtime/mobile-cloud-relay/origin-lifecycle/relay-origin-pool-retry.test.ts
@@ -121,6 +121,21 @@ describe('RelayOriginPool drain retry cap', () => {
     expect(Math.max(...deadCell)).toBeGreaterThan(250_000)
   })
 
+  it('falls back to the full backoff when the director alternates between failing cells', async () => {
+    const deadCell = await asksDuring(THIRTY_MINUTES, async () => assignment)
+    harness.origins.length = 0
+    harness.failOpens = false
+    let epoch = 1
+    const alternating = await asksDuring(THIRTY_MINUTES, async () => {
+      epoch += 1
+      const cell = epoch % 2 === 0 ? 'a' : 'b'
+      return { ...assignment, cellUrl: `https://relay-${cell}.example`, assignmentEpoch: epoch }
+    })
+    // Each of the two cells gets the short cap once; after that the rate matches a dead cell.
+    expect(alternating.length).toBeLessThanOrEqual(deadCell.length + 2)
+    expect(Math.max(...alternating)).toBeGreaterThan(250_000)
+  })
+
   it('retries within 60s each time the director names a different cell that then fails', async () => {
     let epoch = 1
     const gaps = await asksDuring(THIRTY_MINUTES, async () => {
diff --git a/src/main/runtime/mobile-cloud-relay/origin-lifecycle/relay-origin-pool.ts b/src/main/runtime/mobile-cloud-relay/origin-lifecycle/relay-origin-pool.ts
index 1860f545cfa..e72199289a3 100644
--- a/src/main/runtime/mobile-cloud-relay/origin-lifecycle/relay-origin-pool.ts
+++ b/src/main/runtime/mobile-cloud-relay/origin-lifecycle/relay-origin-pool.ts
@@ -13,8 +13,9 @@ import {
 import { RelayControlRotation } from './relay-control-rotation'
 
 // The director answered and only a cell it newly named failed: retry that once sooner. Director
-// failures, and the same cell failing again, keep the full backoff: every retry re-asks the
-// director, so a short cap on a dead cell would multiply director load for every host on it.
+// failures, and any cell failing again in the same episode (even alternating A, B, A), keep the
+// full backoff: every retry re-asks the director, so a short cap on dead cells would multiply
+// director load for every host on them.
 const CELL_CONNECT_RETRY_MAX_MS = 60_000
 
 export class RelayOriginPool {
@@ -30,7 +31,8 @@ export class RelayOriginPool {
   private readonly basisOrigins = this.retirement.basis
   private assignment: RelayAssignment | null = null
   private deferredAssignment: RelayAssignment | null = null
-  private failedCellUrl: string | null = null
+  // Cells that failed since the last success: each gets the short cap once.
+  private readonly failedCells = new Set<string>()
   private relayJwt: string | null = null
   private readonly rotation: RelayControlRotation
   private rotationPromise: Promise<void> | null = null
@@ -257,10 +259,13 @@ export class RelayOriginPool {
     this.retirement.schedule(origin, graceMs)
     this.retirement.maybeClose(origin)
   }
-  // attempts is 0 right after a success, so a cell that worked before counts as new again.
+  // attempts is 0 only right after a success, which starts a new episode.
   private isNewCellFailure(cellUrl: string): boolean {
-    const repeated = this.drainRetry.attempts > 0 && this.failedCellUrl === cellUrl
-    this.failedCellUrl = cellUrl
+    if (this.drainRetry.attempts === 0) {
+      this.failedCells.clear()
+    }
+    const repeated = this.failedCells.has(cellUrl)
+    this.failedCells.add(cellUrl)
     return !repeated
   }