Files
orca/cloud/dev/scripts/probe-relay-rehome-trust.mjs
T
Jinwoo HongandE2E Test 30d32d3fcf fix(relay): let the same-cap trust probe outlast load-balancer warm-up (#31067)
A freshly restarted cell can be ready while the global load balancer still
answers 503 for it. The probe retried once after 2 s, so c10's 2026-10-20
roll failed after its drain and was left isolated. Retry wrapped source
5xx every 10 s for up to 2 minutes.

Co-authored-by: E2E Test <e2e@test.local>
2026-10-20 04:40:49 -04:00

113 lines
4.5 KiB
JavaScript

import { pathToFileURL } from 'node:url'
import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs'
// Every cell that carries the rehome identity: the eighteen US cells and the
// seven asia-east2 cells that drain mis-homed hosts back the other way.
const PRODUCTION_CELL =
/^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35)$/
const DIRECTOR_ORIGIN = 'https://relay.onorca.dev'
const SOURCE_TRANSIENT_RETRIES = 12
const SOURCE_TRANSIENT_RETRY_DELAY_MS = 10_000
export function parseRehomeTrustProbeArguments(argv, environment = process.env) {
const values = {}
for (let index = 0; index < argv.length; index += 2) {
const key = argv[index]
const value = argv[index + 1]
if (!key?.startsWith('--') || value === undefined) throw new Error('invalid arguments')
values[key.slice(2)] = value
}
for (const key of ['director-origin', 'cell-id', 'cell-incarnation']) {
if (!values[key]) throw new Error(`missing --${key}`)
}
if (values['director-origin'] !== DIRECTOR_ORIGIN) {
throw new Error('--director-origin must be the production Relay origin')
}
if (!PRODUCTION_CELL.test(values['cell-id'])) throw new Error('--cell-id is not approved')
if (!/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(
values['cell-incarnation']
)) throw new Error('--cell-incarnation is invalid')
const token = environment.ORCA_RELAY_ADMIN_ID_TOKEN
if (!token || token.length > 8_192 || !/^[^.]+\.[^.]+\.[^.]+$/.test(token)) {
throw new Error('admin identity token is unavailable')
}
return {
directorOrigin: DIRECTOR_ORIGIN,
cellId: values['cell-id'],
cellIncarnation: values['cell-incarnation'],
token
}
}
export async function probeRehomeTrust(config, dependencies = {}) {
const fetchImpl = dependencies.fetch ?? fetch
const request = () => fetchAdminOnceMore(
fetchImpl,
`${config.directorOrigin}/v1/admin/regional-rehome-trust-probe`,
{
method: 'POST',
headers: {
authorization: `Bearer ${config.token}`,
'content-type': 'application/json'
},
body: JSON.stringify({
v: 1,
sourceCellId: config.cellId,
sourceCellIncarnation: config.cellIncarnation
})
},
{ wait: dependencies.wait }
)
const wait = dependencies.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)))
let response = await request()
let body = await response.json().catch(() => ({}))
// The director wraps source HTTP failures in 409; retry only explicit transient statuses. The
// load balancer can answer 503 for a minute after a fresh instance is ready (c10, 2026-10-20).
for (
let retry = 0;
retry < SOURCE_TRANSIENT_RETRIES &&
response.status === 409 &&
/^regional_rehome_trust_probe_source_(500|502|503|504)$/.test(body?.error ?? '');
retry++
) {
await wait(SOURCE_TRANSIENT_RETRY_DELAY_MS)
response = await request()
body = await response.json().catch(() => ({}))
}
if (!response.ok) {
const safeReasons = new Set([
'invalid_token', 'director_only', 'invalid_request',
'regional_rehome_trust_not_configured',
'regional_rehome_trust_probe_source_unavailable',
'regional_rehome_trust_probe_source_invalid_response',
'regional_rehome_trust_probe_not_proven',
...[400, 401, 403, 404, 409, 429, 500, 502, 503, 504]
.map((status) => `regional_rehome_trust_probe_source_${status}`)
])
const reason = safeReasons.has(body?.error) ? body.error : 'unrecognized_error'
throw new Error(`application-mediated rehome trust probe returned ${response.status}: ${reason}`)
}
if (
body.v !== 1 ||
body.dedicatedIdentity?.firstOutcome !== 'host-not-connected' ||
body.dedicatedIdentity?.secondOutcome !== 'host-not-connected' ||
body.dedicatedIdentity?.accepted !== true ||
body.dedicatedIdentity?.idempotent !== true ||
body.sharedRuntimeIdentityRejected !== true ||
body.proven !== true
) throw new Error('application-mediated rehome trust proof is incomplete')
return body
}
export async function main(argv = process.argv.slice(2)) {
const result = await probeRehomeTrust(parseRehomeTrustProbeArguments(argv))
process.stdout.write(`${JSON.stringify({ event: 'relay_rehome_trust_verified', ...result })}\n`)
}
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
main().catch((error) => {
process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`)
process.exitCode = 1
})
}