From 30d32d3fcf8f748aafe87fa0091dfc18aff70b75 Mon Sep 17 00:00:00 2001
From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com>
Date: Tue, 20 Oct 2026 04:40:49 -0400
Subject: [PATCH] fix(relay): let the same-cap trust probe outlast
 load-balancer warm-up (#31067)

A freshly restarted cell can be ready while the global load balancer still
answers 503 for it. The probe retried once after 2 s, so c10's 2026-10-20
roll failed after its drain and was left isolated. Retry wrapped source
5xx every 10 s for up to 2 minutes.

Co-authored-by: E2E Test <e2e@test.local>
---
 cloud/dev/scripts/probe-relay-rehome-trust.mjs | 16 +++++++++++++---
 .../scripts/probe-relay-rehome-trust.test.mjs  | 18 ++++++++++++++++--
 2 files changed, 29 insertions(+), 5 deletions(-)

diff --git a/cloud/dev/scripts/probe-relay-rehome-trust.mjs b/cloud/dev/scripts/probe-relay-rehome-trust.mjs
index d4338494b5f..96c6ccf2f22 100644
--- a/cloud/dev/scripts/probe-relay-rehome-trust.mjs
+++ b/cloud/dev/scripts/probe-relay-rehome-trust.mjs
@@ -6,6 +6,8 @@ import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
 const PRODUCTION_CELL =
   /^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35)$/
 const DIRECTOR_ORIGIN = 'https://relay.onorca.dev'
+const SOURCE_TRANSIENT_RETRIES = 12
+const SOURCE_TRANSIENT_RETRY_DELAY_MS = 10_000
 
 export function parseRehomeTrustProbeArguments(argv, environment = process.env) {
   const values = {}
@@ -56,11 +58,19 @@ export async function probeRehomeTrust(config, dependencies = {}) {
     },
     { wait: dependencies.wait }
   )
+  const wait = dependencies.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)))
   let response = await request()
   let body = await response.json().catch(() => ({}))
-  // The director wraps source HTTP failures in 409; retry only explicit transient statuses.
-  if (response.status === 409 && /^regional_rehome_trust_probe_source_(500|502|503|504)$/.test(body?.error ?? '')) {
-    await (dependencies.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))))(2_000)
+  // The director wraps source HTTP failures in 409; retry only explicit transient statuses. The
+  // load balancer can answer 503 for a minute after a fresh instance is ready (c10, 2026-10-20).
+  for (
+    let retry = 0;
+    retry < SOURCE_TRANSIENT_RETRIES &&
+    response.status === 409 &&
+    /^regional_rehome_trust_probe_source_(500|502|503|504)$/.test(body?.error ?? '');
+    retry++
+  ) {
+    await wait(SOURCE_TRANSIENT_RETRY_DELAY_MS)
     response = await request()
     body = await response.json().catch(() => ({}))
   }
diff --git a/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs b/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs
index f70087a4aec..9e31864956d 100644
--- a/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs
+++ b/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs
@@ -165,13 +165,27 @@ test('reports safe trust reasons, keeps rejection final, and redacts arbitrary e
   }
 })
 
-test('stops after the second wrapped transient failure', async () => {
+test('outlasts a load-balancer warm-up of consecutive wrapped source 503s', async () => {
+  let calls = 0
+  const waits = []
+  const result = await probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), {
+    wait: async (ms) => { waits.push(ms) },
+    fetch: async () => ++calls <= 6
+      ? Response.json({ error: 'regional_rehome_trust_probe_source_503' }, { status: 409 })
+      : Response.json(provenProbe)
+  })
+  assert.equal(calls, 7)
+  assert.deepEqual(waits, Array(6).fill(10_000))
+  assert.equal(result.proven, true)
+})
+
+test('stops after two minutes of wrapped transient failures', async () => {
   let calls = 0
   await assert.rejects(probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), {
     wait: async () => {},
     fetch: async () => { calls++; return Response.json({ error: 'regional_rehome_trust_probe_source_503' }, { status: 409 }) }
   }), /returned 409.*source_503/)
-  assert.equal(calls, 2)
+  assert.equal(calls, 13)
 })
 
 // The probe allowlist is the committed rehome source list, so a declared source can always probe.