fix(relay-ops): retry transient admin-endpoint failures in same-cap verify and rehome jobs (#18769)

This commit is contained in:
Jinwoo Hong
2026-09-04 22:04:21 -04:00
committed by GitHub
parent df7b1028dd
commit e2b70a5eba
13 changed files with 496 additions and 46 deletions
@@ -1,4 +1,5 @@
import { pathToFileURL } from 'node:url'
import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
const CAPACITY_PROTOCOL = 2
@@ -378,9 +379,12 @@ export async function verifyCapacityTransition(config, overrides = {}) {
const token = overrides.token ?? process.env.ORCA_RELAY_ADMIN_ID_TOKEN
if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable')
const health = await responseJson(
await fetchImpl(`${config.directorOrigin}/health`, {
signal: AbortSignal.timeout(15_000)
}),
await fetchAdminOnceMore(
fetchImpl,
`${config.directorOrigin}/health`,
{},
{ wait, timeoutMs: 15_000 }
),
'director health'
)
if (health.ok !== true || health.connectionCapacityProtocol !== CAPACITY_PROTOCOL) {
@@ -394,12 +398,16 @@ export async function verifyCapacityTransition(config, overrides = {}) {
lastObservation = { runtimeAvailable: runtime !== null }
if ((runtime === null) === (config.runtime === 'unavailable')) {
const result = await responseJson(
await fetchImpl(`${config.directorOrigin}/v1/admin/cell-status`, {
method: 'POST',
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify({ v: 1, cellId: config.cellId }),
signal: AbortSignal.timeout(30_000)
}),
await fetchAdminOnceMore(
fetchImpl,
`${config.directorOrigin}/v1/admin/cell-status`,
{
method: 'POST',
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify({ v: 1, cellId: config.cellId })
},
{ wait }
),
'cell status'
)
const status = result.status