fix(relay-ops): retry transient admin-endpoint failures in same-cap verify and rehome jobs (#18769)

cloud/dev/scripts/operate-relay-regional-rehome.mjs

@@ -1,4 +1,5 @@
 import { pathToFileURL } from 'node:url'
+import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
 import { inspectAdmissionSelector } from './relay-admission-selector.mjs'
 
 const DIRECTOR_ORIGIN = 'https://relay.onorca.dev'
@@ -229,15 +230,20 @@ export async function recoverRegionalRehomeEnable(config, post) {
 export async function operateRegionalRehome(config, dependencies = {}) {
   const fetchImpl = dependencies.fetch ?? fetch
   const post = dependencies.post ?? (async (path, body) => await responseJson(
-    await fetchImpl(`${config.directorOrigin}${path}`, {
-      method: 'POST',
-      headers: {
-        authorization: `Bearer ${config.token}`,
-        'content-type': 'application/json'
+    // Generation-guarded writes make a retry a no-op or an explicit mismatch, never a double apply.
+    await fetchAdminOnceMore(
+      fetchImpl,
+      `${config.directorOrigin}${path}`,
+      {
+        method: 'POST',
+        headers: {
+          authorization: `Bearer ${config.token}`,
+          'content-type': 'application/json'
+        },
+        body: JSON.stringify(body)
       },
-      body: JSON.stringify(body),
-      signal: AbortSignal.timeout(30_000)
-    }),
+      { wait: dependencies.wait }
+    ),
     path
   ))
   if (config.mode === 'recover-enable') {

cloud/dev/scripts/operate-relay-regional-rehome.test.mjs

@@ -263,3 +263,55 @@ test('main executes recovery mode and emits verified disabled control', async ()
     control: control(6, false)
   })
 })
+
+test('retries a transient 503 on the director control endpoint', async () => {
+  const config = parseRegionalRehomeArguments(
+    argumentsFor('inspect'),
+    { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
+  )
+  const paths = []
+  let selectorCalls = 0
+  const result = await operateRegionalRehome(config, {
+    wait: async () => {},
+    fetch: async (url) => {
+      const path = new URL(url).pathname
+      paths.push(path)
+      if (path === '/v1/admin/admission-selector/status') {
+        selectorCalls += 1
+        // The first read of each admin path 503s the way a warming instance does.
+        if (selectorCalls === 1) return new Response('warming up', { status: 503 })
+        return Response.json({ selector: { generation: 11, membership } })
+      }
+      if (paths.filter((value) => value === path).length === 1) {
+        return new Response('warming up', { status: 503 })
+      }
+      return Response.json({ v: 1, control: control(4, false) })
+    }
+  })
+  assert.equal(result.control.generation, 4)
+  assert.deepEqual(paths, [
+    '/v1/admin/admission-selector/status',
+    '/v1/admin/admission-selector/status',
+    '/v1/admin/regional-rehome-control',
+    '/v1/admin/regional-rehome-control'
+  ])
+})
+
+test('fails when both attempts at the director control endpoint return 503', async () => {
+  const config = parseRegionalRehomeArguments(
+    argumentsFor('inspect'),
+    { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' }
+  )
+  let calls = 0
+  await assert.rejects(
+    operateRegionalRehome(config, {
+      wait: async () => {},
+      fetch: async () => {
+        calls += 1
+        return new Response('warming up', { status: 503 })
+      }
+    }),
+    /returned 503/
+  )
+  assert.equal(calls, 2)
+})

cloud/dev/scripts/prepare-relay-production-capacity-canary.mjs

@@ -1,4 +1,5 @@
 import { pathToFileURL } from 'node:url'
+import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
 import {
   applyExactAdmissionSelector,
   inspectAdmissionSelector,
@@ -72,12 +73,16 @@ export async function prepareProductionCapacityCell(config, overrides = {}) {
   if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable')
   const postAt = async (origin, path, body) =>
     await responseJson(
-      await fetchImpl(`${origin}${path}`, {
-        method: 'POST',
-        headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
-        body: JSON.stringify(body),
-        signal: AbortSignal.timeout(30_000)
-      }),
+      await fetchAdminOnceMore(
+        fetchImpl,
+        `${origin}${path}`,
+        {
+          method: 'POST',
+          headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
+          body: JSON.stringify(body)
+        },
+        { wait: overrides.wait }
+      ),
       path
     )
   const post = async (path, body) => await postAt(config.directorOrigin, path, body)

cloud/dev/scripts/prepare-relay-production-capacity-canary.test.mjs

@@ -170,4 +170,42 @@ describe('production Relay capacity cell admission', () => {
       /irreversible/
     )
   })
+
+  it('retries a transient 503 on the cell drain endpoint', async () => {
+    let calls = 0
+    const result = await prepareProductionCapacityCell(
+      { ...config, mode: 'drain' },
+      {
+        token: 'token',
+        wait: async () => {},
+        fetch: async (url) => {
+          assert.equal(new URL(url).pathname, '/v1/admin/drain')
+          calls += 1
+          if (calls === 1) return response({ error: 'warming up' }, 503)
+          return response({ v: 1, draining: true })
+        }
+      }
+    )
+    assert.equal(calls, 2)
+    assert.deepEqual(result, { changed: false, drained: true })
+  })
+
+  it('fails when both drain attempts return a transient 503', async () => {
+    let calls = 0
+    await assert.rejects(
+      prepareProductionCapacityCell(
+        { ...config, mode: 'drain' },
+        {
+          token: 'token',
+          wait: async () => {},
+          fetch: async () => {
+            calls += 1
+            return response({ error: 'warming up' }, 503)
+          }
+        }
+      ),
+      /returned 503/
+    )
+    assert.equal(calls, 2)
+  })
 })

cloud/dev/scripts/probe-relay-rehome-trust.mjs

@@ -1,4 +1,5 @@
 import { pathToFileURL } from 'node:url'
+import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
 
 const PRODUCTION_CELL = /^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26)$/
 const DIRECTOR_ORIGIN = 'https://relay.onorca.dev'
@@ -35,7 +36,8 @@ export function parseRehomeTrustProbeArguments(argv, environment = process.env)
 
 export async function probeRehomeTrust(config, dependencies = {}) {
   const fetchImpl = dependencies.fetch ?? fetch
-  const response = await fetchImpl(
+  const response = await fetchAdminOnceMore(
+    fetchImpl,
     `${config.directorOrigin}/v1/admin/regional-rehome-trust-probe`,
     {
       method: 'POST',
@@ -47,9 +49,9 @@ export async function probeRehomeTrust(config, dependencies = {}) {
         v: 1,
         sourceCellId: config.cellId,
         sourceCellIncarnation: config.cellIncarnation
-      }),
-      signal: AbortSignal.timeout(30_000)
-    }
+      })
+    },
+    { wait: dependencies.wait }
   )
   const body = await response.json().catch(() => ({}))
   if (!response.ok) {

cloud/dev/scripts/probe-relay-rehome-trust.test.mjs

@@ -68,3 +68,46 @@ test('rejects partial or mismatched proof', async () => {
     /incomplete/
   )
 })
+
+const provenProbe = {
+  v: 1,
+  dedicatedIdentity: {
+    firstOutcome: 'host-not-connected',
+    secondOutcome: 'host-not-connected',
+    accepted: true,
+    idempotent: true
+  },
+  sharedRuntimeIdentityRejected: true,
+  proven: true
+}
+
+test('retries a transient 503 on the trust probe and proves on the second answer', async () => {
+  const config = parseRehomeTrustProbeArguments(argv, environment)
+  let calls = 0
+  const result = await probeRehomeTrust(config, {
+    wait: async () => {},
+    fetch: async () => {
+      calls += 1
+      if (calls === 1) return new Response('warming up', { status: 503 })
+      return Response.json(provenProbe)
+    }
+  })
+  assert.equal(calls, 2)
+  assert.equal(result.proven, true)
+})
+
+test('fails when both trust-probe attempts return a transient 503', async () => {
+  const config = parseRehomeTrustProbeArguments(argv, environment)
+  let calls = 0
+  await assert.rejects(
+    probeRehomeTrust(config, {
+      wait: async () => {},
+      fetch: async () => {
+        calls += 1
+        return new Response('warming up', { status: 503 })
+      }
+    }),
+    /returned 503/
+  )
+  assert.equal(calls, 2)
+})

cloud/dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs

@@ -0,0 +1,43 @@
+import assert from 'node:assert/strict'
+import { readFileSync } from 'node:fs'
+import { test } from 'node:test'
+import { fileURLToPath } from 'node:url'
+import { relayWorkflowUrl } from './relay-repository.mjs'
+
+const WORKFLOWS = [
+  'deploy-relay-production-same-cap-job.yml',
+  'operate-relay-production-rehome-job.yml'
+]
+
+function workflow(name) {
+  return readFileSync(fileURLToPath(relayWorkflowUrl(name)), 'utf8')
+}
+
+// A single transient 5xx from a warming instance behind the global load balancer
+// must not fail a canary, so no admin endpoint may be read by a bare curl.
+test('no admin endpoint is reached by a curl without a bounded retry', () => {
+  for (const name of WORKFLOWS) {
+    for (const invocation of workflow(name).split(/\bcurl\b/).slice(1)) {
+      const flags = invocation.split('\n          }')[0]
+      assert.match(flags, /--retry 3 --retry-delay 2 --retry-connrefused/, name)
+      assert.match(flags, /--max-time 30/, name)
+      // --retry-all-errors would also retry 401, 403, and 409, which are final.
+      assert.doesNotMatch(flags, /--retry-all-errors/, name)
+    }
+  }
+})
+
+test('every retried admin request captures only the final attempt body', () => {
+  const job = workflow('deploy-relay-production-same-cap-job.yml')
+  // --fail-with-body writes every failed attempt to stdout, so a retried
+  // request must land in a file curl truncates per attempt.
+  assert.match(job, /--output "\$\{out\}"/)
+  assert.equal(job.split('admin_post() {').length - 1, 2)
+  for (const call of [
+    /CURRENT_RUNTIME="\$\(admin_post current-runtime/,
+    /CURRENT_DIRECTOR_STATUS="\$\(admin_post current-cell-status/,
+    /TARGET_RUNTIME="\$\(admin_post target-runtime/,
+    /TARGET_DIRECTOR_STATUS="\$\(admin_post target-cell-status/
+  ]) assert.match(job, call)
+  assert.doesNotMatch(job, /\$\(curl /)
+})

cloud/dev/scripts/relay-admin-transient-retry.mjs

@@ -0,0 +1,29 @@
+// A single transient 5xx (load-balancer warm-up behind a fresh instance) must not fail a
+// deploy step. 4xx is never retried: auth and generation-mismatch answers are final.
+const TRANSIENT_STATUSES = [500, 502, 503, 504]
+const RETRY_DELAY_MS = 2_000
+const REQUEST_TIMEOUT_MS = 30_000
+
+export function isTransientAdminStatus(status) {
+  return TRANSIENT_STATUSES.includes(status)
+}
+
+// Each attempt gets its own timeout budget, so a reused signal cannot abort the retry.
+export async function fetchAdminOnceMore(fetchImpl, url, init, overrides = {}) {
+  const wait = overrides.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)))
+  const timeoutMs = overrides.timeoutMs ?? REQUEST_TIMEOUT_MS
+  const retryDelayMs = overrides.retryDelayMs ?? RETRY_DELAY_MS
+  const attempt = async () =>
+    await fetchImpl(url, { ...init, signal: AbortSignal.timeout(timeoutMs) })
+  let response
+  try {
+    response = await attempt()
+  } catch {
+    await wait(retryDelayMs)
+    return await attempt()
+  }
+  if (!isTransientAdminStatus(response.status)) return response
+  await response.arrayBuffer?.().catch(() => undefined)
+  await wait(retryDelayMs)
+  return await attempt()
+}

cloud/dev/scripts/relay-admin-transient-retry.test.mjs

@@ -0,0 +1,130 @@
+import assert from 'node:assert/strict'
+import { test } from 'node:test'
+import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
+
+const url = 'https://relay.onorca.dev/v1/admin/cell-status'
+const init = { method: 'POST', body: '{"v":1}' }
+
+function recordingWait(waits) {
+  return async (ms) => { waits.push(ms) }
+}
+
+test('a single transient 5xx is retried and the second answer is returned', async () => {
+  const waits = []
+  const statuses = [503, 200]
+  let calls = 0
+  const response = await fetchAdminOnceMore(
+    async () => {
+      calls += 1
+      const status = statuses.shift()
+      return new Response(JSON.stringify({ ok: status === 200 }), { status })
+    },
+    url,
+    init,
+    { wait: recordingWait(waits) }
+  )
+  assert.equal(calls, 2)
+  assert.equal(response.status, 200)
+  assert.deepEqual(waits, [2_000])
+  assert.deepEqual(await response.json(), { ok: true })
+})
+
+test('a connection failure is retried and the second answer is returned', async () => {
+  const waits = []
+  let calls = 0
+  const response = await fetchAdminOnceMore(
+    async () => {
+      calls += 1
+      if (calls === 1) throw new TypeError('fetch failed')
+      return Response.json({ ok: true })
+    },
+    url,
+    init,
+    { wait: recordingWait(waits) }
+  )
+  assert.equal(calls, 2)
+  assert.equal(response.status, 200)
+  assert.deepEqual(waits, [2_000])
+})
+
+test('two transient failures surface the second answer without a third attempt', async () => {
+  let calls = 0
+  const response = await fetchAdminOnceMore(
+    async () => {
+      calls += 1
+      return new Response('down', { status: 503 })
+    },
+    url,
+    init,
+    { wait: async () => {} }
+  )
+  assert.equal(calls, 2)
+  assert.equal(response.status, 503)
+})
+
+test('two connection failures rethrow the second error', async () => {
+  let calls = 0
+  await assert.rejects(
+    fetchAdminOnceMore(
+      async () => {
+        calls += 1
+        throw new TypeError(`fetch failed ${calls}`)
+      },
+      url,
+      init,
+      { wait: async () => {} }
+    ),
+    /fetch failed 2/
+  )
+  assert.equal(calls, 2)
+})
+
+test('4xx is final: auth and generation-mismatch answers are never retried', async () => {
+  for (const status of [400, 401, 403, 404, 409, 429]) {
+    let calls = 0
+    const response = await fetchAdminOnceMore(
+      async () => {
+        calls += 1
+        return new Response('no', { status })
+      },
+      url,
+      init,
+      { wait: async () => { throw new Error('must not wait') } }
+    )
+    assert.equal(calls, 1, `status ${status} must not be retried`)
+    assert.equal(response.status, status)
+  }
+})
+
+test('each attempt carries its own unexpired timeout signal', async () => {
+  const signals = []
+  await fetchAdminOnceMore(
+    async (_url, attemptInit) => {
+      signals.push(attemptInit.signal)
+      return new Response('down', { status: 502 })
+    },
+    url,
+    init,
+    { wait: async () => {}, timeoutMs: 30_000 }
+  )
+  assert.equal(signals.length, 2)
+  assert.notEqual(signals[0], signals[1])
+  assert.equal(signals[1].aborted, false)
+})
+
+test('the caller init is forwarded unchanged apart from the signal', async () => {
+  let seen
+  await fetchAdminOnceMore(
+    async (seenUrl, attemptInit) => {
+      seen = { seenUrl, attemptInit }
+      return Response.json({})
+    },
+    url,
+    { method: 'POST', headers: { authorization: 'Bearer t' }, body: '{"v":1}' },
+    { wait: async () => {} }
+  )
+  assert.equal(seen.seenUrl, url)
+  assert.equal(seen.attemptInit.method, 'POST')
+  assert.deepEqual(seen.attemptInit.headers, { authorization: 'Bearer t' })
+  assert.equal(seen.attemptInit.body, '{"v":1}')
+})

cloud/dev/scripts/verify-relay-capacity-transition.mjs

@@ -1,4 +1,5 @@
 import { pathToFileURL } from 'node:url'
+import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
 
 const CAPACITY_PROTOCOL = 2
 
@@ -378,9 +379,12 @@ export async function verifyCapacityTransition(config, overrides = {}) {
   const token = overrides.token ?? process.env.ORCA_RELAY_ADMIN_ID_TOKEN
   if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable')
   const health = await responseJson(
-    await fetchImpl(`${config.directorOrigin}/health`, {
-      signal: AbortSignal.timeout(15_000)
-    }),
+    await fetchAdminOnceMore(
+      fetchImpl,
+      `${config.directorOrigin}/health`,
+      {},
+      { wait, timeoutMs: 15_000 }
+    ),
     'director health'
   )
   if (health.ok !== true || health.connectionCapacityProtocol !== CAPACITY_PROTOCOL) {
@@ -394,12 +398,16 @@ export async function verifyCapacityTransition(config, overrides = {}) {
     lastObservation = { runtimeAvailable: runtime !== null }
     if ((runtime === null) === (config.runtime === 'unavailable')) {
       const result = await responseJson(
-        await fetchImpl(`${config.directorOrigin}/v1/admin/cell-status`, {
-          method: 'POST',
-          headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
-          body: JSON.stringify({ v: 1, cellId: config.cellId }),
-          signal: AbortSignal.timeout(30_000)
-        }),
+        await fetchAdminOnceMore(
+          fetchImpl,
+          `${config.directorOrigin}/v1/admin/cell-status`,
+          {
+            method: 'POST',
+            headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
+            body: JSON.stringify({ v: 1, cellId: config.cellId })
+          },
+          { wait }
+        ),
         'cell status'
       )
       const status = result.status

cloud/dev/scripts/verify-relay-capacity-transition.test.mjs

@@ -1094,3 +1094,77 @@ test('does not retry a rejected cell admin token', async () => {
   )
   assert.equal(waits, 0)
 })
+
+test('retries a transient 503 on the director cell-status read', async () => {
+  const base = harness()
+  const statusCalls = []
+  const result = await verifyCapacityTransition(config, {
+    token: 'masked-token',
+    wait: async () => {},
+    fetch: async (url, options) => {
+      const path = new URL(url).pathname
+      if (path !== '/v1/admin/cell-status') return await base(url, options)
+      statusCalls.push(path)
+      if (statusCalls.length === 1) return new Response('warming up', { status: 503 })
+      return await base(url, options)
+    }
+  })
+  assert.equal(statusCalls.length, 2)
+  assert.equal(result.cellId, config.cellId)
+})
+
+test('fails when both director cell-status attempts return a transient 503', async () => {
+  const base = harness()
+  let statusCalls = 0
+  await assert.rejects(
+    verifyCapacityTransition(config, {
+      token: 'masked-token',
+      wait: async () => {},
+      fetch: async (url, options) => {
+        const path = new URL(url).pathname
+        if (path !== '/v1/admin/cell-status') return await base(url, options)
+        statusCalls += 1
+        return new Response('warming up', { status: 503 })
+      }
+    }),
+    /cell status returned 503/
+  )
+  assert.equal(statusCalls, 2)
+})
+
+test('retries a transient 503 on the director health preflight', async () => {
+  const base = harness()
+  let healthCalls = 0
+  const result = await verifyCapacityTransition(config, {
+    token: 'masked-token',
+    wait: async () => {},
+    fetch: async (url, options) => {
+      const path = new URL(url).pathname
+      if (path !== '/health') return await base(url, options)
+      healthCalls += 1
+      if (healthCalls === 1) return new Response('warming up', { status: 503 })
+      return await base(url, options)
+    }
+  })
+  assert.equal(healthCalls, 2)
+  assert.equal(result.cellId, config.cellId)
+})
+
+test('fails when both director health attempts return a transient 503', async () => {
+  const base = harness()
+  let healthCalls = 0
+  await assert.rejects(
+    verifyCapacityTransition(config, {
+      token: 'masked-token',
+      wait: async () => {},
+      fetch: async (url, options) => {
+        const path = new URL(url).pathname
+        if (path !== '/health') return await base(url, options)
+        healthCalls += 1
+        return new Response('warming up', { status: 503 })
+      }
+    }),
+    /director health returned 503/
+  )
+  assert.equal(healthCalls, 2)
+})