diff --git a/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml b/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml index afcaf3c066e..c4eea80933e 100644 --- a/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml +++ b/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml @@ -275,10 +275,22 @@ jobs: env: ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.deploy-auth.outputs.id_token }} run: | - CURRENT_RUNTIME="$(curl --fail-with-body --max-time 30 \ - --request POST "${CELL_ORIGIN}/v1/admin/runtime-status" \ - --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ - --header 'Content-Type: application/json' --data '{"v":1}')" + # A single transient 5xx (LB warm-up behind a fresh instance) must not + # fail a canary; 4xx (auth, generation mismatch) still fails fast. + admin_post() { + local out="${RUNNER_TEMP}/$1.json" + if ! curl --fail-with-body --max-time 30 \ + --retry 3 --retry-delay 2 --retry-connrefused --output "${out}" \ + --request POST "$2" \ + --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ + --header 'Content-Type: application/json' --data "$3"; then + cat "${out}" >&2 + return 1 + fi + cat "${out}" + } + CURRENT_RUNTIME="$(admin_post current-runtime \ + "${CELL_ORIGIN}/v1/admin/runtime-status" '{"v":1}')" # A rollback that failed between template apply and admission restore # leaves the cell already on the rollback image; resume from that # state instead of demanding the pre-rollback predecessor. @@ -374,11 +386,9 @@ jobs: if .regionalRehomeProtocol == null then "regionalRehomeProtocol" else empty end ] | if length > 0 then "runtime predecessor normalized legacy fields=" + join(",") else empty end' \ <<< "${CURRENT_RUNTIME}" - CURRENT_DIRECTOR_STATUS="$(curl --fail-with-body --max-time 30 \ - --request POST "${DIRECTOR_ORIGIN}/v1/admin/cell-status" \ - --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ - --header 'Content-Type: application/json' \ - --data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")" + CURRENT_DIRECTOR_STATUS="$(admin_post current-cell-status \ + "${DIRECTOR_ORIGIN}/v1/admin/cell-status" \ + "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")" SOURCE_INCARNATION="$(jq -er '.status.runtime.cellIncarnation' \ <<< "${CURRENT_DIRECTOR_STATUS}")" if test "${ROLLBACK_RESUME}" = true && ! jq -e \ @@ -536,6 +546,20 @@ jobs: env: ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }} run: | + # A single transient 5xx (LB warm-up behind a fresh instance) must not + # fail a canary; 4xx (auth, generation mismatch) still fails fast. + admin_post() { + local out="${RUNNER_TEMP}/$1.json" + if ! curl --fail-with-body --max-time 30 \ + --retry 3 --retry-delay 2 --retry-connrefused --output "${out}" \ + --request POST "$2" \ + --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ + --header 'Content-Type: application/json' --data "$3"; then + cat "${out}" >&2 + return 1 + fi + cat "${out}" + } node dev/scripts/verify-relay-capacity-transition.mjs \ --director-origin "${DIRECTOR_ORIGIN}" --cell-origin "${CELL_ORIGIN}" \ --cell-id "${TARGET_CELL_ID}" --hard-cap "${EXPECTED_HARD_CAP}" \ @@ -543,19 +567,15 @@ jobs: --heartbeat fresh --admission migration-only --draining forbidden \ --activity allowed --expected-image-digests "${DESIRED_IMAGE_DIGEST}" \ --regional-rehome-protocol "${DESIRED_REHOME_PROTOCOL}" --timeout-ms 900000 - TARGET_RUNTIME="$(curl --fail-with-body --max-time 30 \ - --request POST "${CELL_ORIGIN}/v1/admin/runtime-status" \ - --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ - --header 'Content-Type: application/json' --data '{"v":1}')" + TARGET_RUNTIME="$(admin_post target-runtime \ + "${CELL_ORIGIN}/v1/admin/runtime-status" '{"v":1}')" jq -e --arg digest "${DESIRED_IMAGE_DIGEST}" \ --argjson protocol "${DESIRED_REHOME_PROTOCOL}" \ '.imageDigest == $digest and (.regionalRehomeProtocol // 0) == $protocol' \ <<< "${TARGET_RUNTIME}" >/dev/null - TARGET_DIRECTOR_STATUS="$(curl --fail-with-body --max-time 30 \ - --request POST "${DIRECTOR_ORIGIN}/v1/admin/cell-status" \ - --header "Authorization: Bearer ${ORCA_RELAY_ADMIN_ID_TOKEN}" \ - --header 'Content-Type: application/json' \ - --data "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")" + TARGET_DIRECTOR_STATUS="$(admin_post target-cell-status \ + "${DIRECTOR_ORIGIN}/v1/admin/cell-status" \ + "$(jq -cn --arg cell "${TARGET_CELL_ID}" '{v:1,cellId:$cell}')")" TARGET_INCARNATION="$(jq -er '.status.runtime.cellIncarnation' \ <<< "${TARGET_DIRECTOR_STATUS}")" if test "${ROLLBACK_RESUME}" = true; then diff --git a/cloud/dev/scripts/operate-relay-regional-rehome.mjs b/cloud/dev/scripts/operate-relay-regional-rehome.mjs index 2ccce39926d..3887408520f 100644 --- a/cloud/dev/scripts/operate-relay-regional-rehome.mjs +++ b/cloud/dev/scripts/operate-relay-regional-rehome.mjs @@ -1,4 +1,5 @@ import { pathToFileURL } from 'node:url' +import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs' import { inspectAdmissionSelector } from './relay-admission-selector.mjs' const DIRECTOR_ORIGIN = 'https://relay.onorca.dev' @@ -229,15 +230,20 @@ export async function recoverRegionalRehomeEnable(config, post) { export async function operateRegionalRehome(config, dependencies = {}) { const fetchImpl = dependencies.fetch ?? fetch const post = dependencies.post ?? (async (path, body) => await responseJson( - await fetchImpl(`${config.directorOrigin}${path}`, { - method: 'POST', - headers: { - authorization: `Bearer ${config.token}`, - 'content-type': 'application/json' + // Generation-guarded writes make a retry a no-op or an explicit mismatch, never a double apply. + await fetchAdminOnceMore( + fetchImpl, + `${config.directorOrigin}${path}`, + { + method: 'POST', + headers: { + authorization: `Bearer ${config.token}`, + 'content-type': 'application/json' + }, + body: JSON.stringify(body) }, - body: JSON.stringify(body), - signal: AbortSignal.timeout(30_000) - }), + { wait: dependencies.wait } + ), path )) if (config.mode === 'recover-enable') { diff --git a/cloud/dev/scripts/operate-relay-regional-rehome.test.mjs b/cloud/dev/scripts/operate-relay-regional-rehome.test.mjs index 8ffe38dfe09..bfea6769ec4 100644 --- a/cloud/dev/scripts/operate-relay-regional-rehome.test.mjs +++ b/cloud/dev/scripts/operate-relay-regional-rehome.test.mjs @@ -263,3 +263,55 @@ test('main executes recovery mode and emits verified disabled control', async () control: control(6, false) }) }) + +test('retries a transient 503 on the director control endpoint', async () => { + const config = parseRegionalRehomeArguments( + argumentsFor('inspect'), + { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' } + ) + const paths = [] + let selectorCalls = 0 + const result = await operateRegionalRehome(config, { + wait: async () => {}, + fetch: async (url) => { + const path = new URL(url).pathname + paths.push(path) + if (path === '/v1/admin/admission-selector/status') { + selectorCalls += 1 + // The first read of each admin path 503s the way a warming instance does. + if (selectorCalls === 1) return new Response('warming up', { status: 503 }) + return Response.json({ selector: { generation: 11, membership } }) + } + if (paths.filter((value) => value === path).length === 1) { + return new Response('warming up', { status: 503 }) + } + return Response.json({ v: 1, control: control(4, false) }) + } + }) + assert.equal(result.control.generation, 4) + assert.deepEqual(paths, [ + '/v1/admin/admission-selector/status', + '/v1/admin/admission-selector/status', + '/v1/admin/regional-rehome-control', + '/v1/admin/regional-rehome-control' + ]) +}) + +test('fails when both attempts at the director control endpoint return 503', async () => { + const config = parseRegionalRehomeArguments( + argumentsFor('inspect'), + { ORCA_RELAY_ADMIN_ID_TOKEN: 'token' } + ) + let calls = 0 + await assert.rejects( + operateRegionalRehome(config, { + wait: async () => {}, + fetch: async () => { + calls += 1 + return new Response('warming up', { status: 503 }) + } + }), + /returned 503/ + ) + assert.equal(calls, 2) +}) diff --git a/cloud/dev/scripts/prepare-relay-production-capacity-canary.mjs b/cloud/dev/scripts/prepare-relay-production-capacity-canary.mjs index 5791c9f20e6..e5c18237453 100644 --- a/cloud/dev/scripts/prepare-relay-production-capacity-canary.mjs +++ b/cloud/dev/scripts/prepare-relay-production-capacity-canary.mjs @@ -1,4 +1,5 @@ import { pathToFileURL } from 'node:url' +import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs' import { applyExactAdmissionSelector, inspectAdmissionSelector, @@ -72,12 +73,16 @@ export async function prepareProductionCapacityCell(config, overrides = {}) { if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable') const postAt = async (origin, path, body) => await responseJson( - await fetchImpl(`${origin}${path}`, { - method: 'POST', - headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, - body: JSON.stringify(body), - signal: AbortSignal.timeout(30_000) - }), + await fetchAdminOnceMore( + fetchImpl, + `${origin}${path}`, + { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify(body) + }, + { wait: overrides.wait } + ), path ) const post = async (path, body) => await postAt(config.directorOrigin, path, body) diff --git a/cloud/dev/scripts/prepare-relay-production-capacity-canary.test.mjs b/cloud/dev/scripts/prepare-relay-production-capacity-canary.test.mjs index c5d0a9db3bc..ff397802860 100644 --- a/cloud/dev/scripts/prepare-relay-production-capacity-canary.test.mjs +++ b/cloud/dev/scripts/prepare-relay-production-capacity-canary.test.mjs @@ -170,4 +170,42 @@ describe('production Relay capacity cell admission', () => { /irreversible/ ) }) + + it('retries a transient 503 on the cell drain endpoint', async () => { + let calls = 0 + const result = await prepareProductionCapacityCell( + { ...config, mode: 'drain' }, + { + token: 'token', + wait: async () => {}, + fetch: async (url) => { + assert.equal(new URL(url).pathname, '/v1/admin/drain') + calls += 1 + if (calls === 1) return response({ error: 'warming up' }, 503) + return response({ v: 1, draining: true }) + } + } + ) + assert.equal(calls, 2) + assert.deepEqual(result, { changed: false, drained: true }) + }) + + it('fails when both drain attempts return a transient 503', async () => { + let calls = 0 + await assert.rejects( + prepareProductionCapacityCell( + { ...config, mode: 'drain' }, + { + token: 'token', + wait: async () => {}, + fetch: async () => { + calls += 1 + return response({ error: 'warming up' }, 503) + } + } + ), + /returned 503/ + ) + assert.equal(calls, 2) + }) }) diff --git a/cloud/dev/scripts/probe-relay-rehome-trust.mjs b/cloud/dev/scripts/probe-relay-rehome-trust.mjs index 7500d8bd14c..9a7d505d4bb 100644 --- a/cloud/dev/scripts/probe-relay-rehome-trust.mjs +++ b/cloud/dev/scripts/probe-relay-rehome-trust.mjs @@ -1,4 +1,5 @@ import { pathToFileURL } from 'node:url' +import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs' const PRODUCTION_CELL = /^production-gce-c(?:7|8|9|10|13|14|15|16|19|20|21|22|23|24|25|26)$/ const DIRECTOR_ORIGIN = 'https://relay.onorca.dev' @@ -35,7 +36,8 @@ export function parseRehomeTrustProbeArguments(argv, environment = process.env) export async function probeRehomeTrust(config, dependencies = {}) { const fetchImpl = dependencies.fetch ?? fetch - const response = await fetchImpl( + const response = await fetchAdminOnceMore( + fetchImpl, `${config.directorOrigin}/v1/admin/regional-rehome-trust-probe`, { method: 'POST', @@ -47,9 +49,9 @@ export async function probeRehomeTrust(config, dependencies = {}) { v: 1, sourceCellId: config.cellId, sourceCellIncarnation: config.cellIncarnation - }), - signal: AbortSignal.timeout(30_000) - } + }) + }, + { wait: dependencies.wait } ) const body = await response.json().catch(() => ({})) if (!response.ok) { diff --git a/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs b/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs index 509e9d53c7d..7d7b2cd95ac 100644 --- a/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs +++ b/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs @@ -68,3 +68,46 @@ test('rejects partial or mismatched proof', async () => { /incomplete/ ) }) + +const provenProbe = { + v: 1, + dedicatedIdentity: { + firstOutcome: 'host-not-connected', + secondOutcome: 'host-not-connected', + accepted: true, + idempotent: true + }, + sharedRuntimeIdentityRejected: true, + proven: true +} + +test('retries a transient 503 on the trust probe and proves on the second answer', async () => { + const config = parseRehomeTrustProbeArguments(argv, environment) + let calls = 0 + const result = await probeRehomeTrust(config, { + wait: async () => {}, + fetch: async () => { + calls += 1 + if (calls === 1) return new Response('warming up', { status: 503 }) + return Response.json(provenProbe) + } + }) + assert.equal(calls, 2) + assert.equal(result.proven, true) +}) + +test('fails when both trust-probe attempts return a transient 503', async () => { + const config = parseRehomeTrustProbeArguments(argv, environment) + let calls = 0 + await assert.rejects( + probeRehomeTrust(config, { + wait: async () => {}, + fetch: async () => { + calls += 1 + return new Response('warming up', { status: 503 }) + } + }), + /returned 503/ + ) + assert.equal(calls, 2) +}) diff --git a/cloud/dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs b/cloud/dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs new file mode 100644 index 00000000000..196fff9edf3 --- /dev/null +++ b/cloud/dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs @@ -0,0 +1,43 @@ +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import { test } from 'node:test' +import { fileURLToPath } from 'node:url' +import { relayWorkflowUrl } from './relay-repository.mjs' + +const WORKFLOWS = [ + 'deploy-relay-production-same-cap-job.yml', + 'operate-relay-production-rehome-job.yml' +] + +function workflow(name) { + return readFileSync(fileURLToPath(relayWorkflowUrl(name)), 'utf8') +} + +// A single transient 5xx from a warming instance behind the global load balancer +// must not fail a canary, so no admin endpoint may be read by a bare curl. +test('no admin endpoint is reached by a curl without a bounded retry', () => { + for (const name of WORKFLOWS) { + for (const invocation of workflow(name).split(/\bcurl\b/).slice(1)) { + const flags = invocation.split('\n }')[0] + assert.match(flags, /--retry 3 --retry-delay 2 --retry-connrefused/, name) + assert.match(flags, /--max-time 30/, name) + // --retry-all-errors would also retry 401, 403, and 409, which are final. + assert.doesNotMatch(flags, /--retry-all-errors/, name) + } + } +}) + +test('every retried admin request captures only the final attempt body', () => { + const job = workflow('deploy-relay-production-same-cap-job.yml') + // --fail-with-body writes every failed attempt to stdout, so a retried + // request must land in a file curl truncates per attempt. + assert.match(job, /--output "\$\{out\}"/) + assert.equal(job.split('admin_post() {').length - 1, 2) + for (const call of [ + /CURRENT_RUNTIME="\$\(admin_post current-runtime/, + /CURRENT_DIRECTOR_STATUS="\$\(admin_post current-cell-status/, + /TARGET_RUNTIME="\$\(admin_post target-runtime/, + /TARGET_DIRECTOR_STATUS="\$\(admin_post target-cell-status/ + ]) assert.match(job, call) + assert.doesNotMatch(job, /\$\(curl /) +}) diff --git a/cloud/dev/scripts/relay-admin-transient-retry.mjs b/cloud/dev/scripts/relay-admin-transient-retry.mjs new file mode 100644 index 00000000000..9995be96a8f --- /dev/null +++ b/cloud/dev/scripts/relay-admin-transient-retry.mjs @@ -0,0 +1,29 @@ +// A single transient 5xx (load-balancer warm-up behind a fresh instance) must not fail a +// deploy step. 4xx is never retried: auth and generation-mismatch answers are final. +const TRANSIENT_STATUSES = [500, 502, 503, 504] +const RETRY_DELAY_MS = 2_000 +const REQUEST_TIMEOUT_MS = 30_000 + +export function isTransientAdminStatus(status) { + return TRANSIENT_STATUSES.includes(status) +} + +// Each attempt gets its own timeout budget, so a reused signal cannot abort the retry. +export async function fetchAdminOnceMore(fetchImpl, url, init, overrides = {}) { + const wait = overrides.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))) + const timeoutMs = overrides.timeoutMs ?? REQUEST_TIMEOUT_MS + const retryDelayMs = overrides.retryDelayMs ?? RETRY_DELAY_MS + const attempt = async () => + await fetchImpl(url, { ...init, signal: AbortSignal.timeout(timeoutMs) }) + let response + try { + response = await attempt() + } catch { + await wait(retryDelayMs) + return await attempt() + } + if (!isTransientAdminStatus(response.status)) return response + await response.arrayBuffer?.().catch(() => undefined) + await wait(retryDelayMs) + return await attempt() +} diff --git a/cloud/dev/scripts/relay-admin-transient-retry.test.mjs b/cloud/dev/scripts/relay-admin-transient-retry.test.mjs new file mode 100644 index 00000000000..ec041084344 --- /dev/null +++ b/cloud/dev/scripts/relay-admin-transient-retry.test.mjs @@ -0,0 +1,130 @@ +import assert from 'node:assert/strict' +import { test } from 'node:test' +import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs' + +const url = 'https://relay.onorca.dev/v1/admin/cell-status' +const init = { method: 'POST', body: '{"v":1}' } + +function recordingWait(waits) { + return async (ms) => { waits.push(ms) } +} + +test('a single transient 5xx is retried and the second answer is returned', async () => { + const waits = [] + const statuses = [503, 200] + let calls = 0 + const response = await fetchAdminOnceMore( + async () => { + calls += 1 + const status = statuses.shift() + return new Response(JSON.stringify({ ok: status === 200 }), { status }) + }, + url, + init, + { wait: recordingWait(waits) } + ) + assert.equal(calls, 2) + assert.equal(response.status, 200) + assert.deepEqual(waits, [2_000]) + assert.deepEqual(await response.json(), { ok: true }) +}) + +test('a connection failure is retried and the second answer is returned', async () => { + const waits = [] + let calls = 0 + const response = await fetchAdminOnceMore( + async () => { + calls += 1 + if (calls === 1) throw new TypeError('fetch failed') + return Response.json({ ok: true }) + }, + url, + init, + { wait: recordingWait(waits) } + ) + assert.equal(calls, 2) + assert.equal(response.status, 200) + assert.deepEqual(waits, [2_000]) +}) + +test('two transient failures surface the second answer without a third attempt', async () => { + let calls = 0 + const response = await fetchAdminOnceMore( + async () => { + calls += 1 + return new Response('down', { status: 503 }) + }, + url, + init, + { wait: async () => {} } + ) + assert.equal(calls, 2) + assert.equal(response.status, 503) +}) + +test('two connection failures rethrow the second error', async () => { + let calls = 0 + await assert.rejects( + fetchAdminOnceMore( + async () => { + calls += 1 + throw new TypeError(`fetch failed ${calls}`) + }, + url, + init, + { wait: async () => {} } + ), + /fetch failed 2/ + ) + assert.equal(calls, 2) +}) + +test('4xx is final: auth and generation-mismatch answers are never retried', async () => { + for (const status of [400, 401, 403, 404, 409, 429]) { + let calls = 0 + const response = await fetchAdminOnceMore( + async () => { + calls += 1 + return new Response('no', { status }) + }, + url, + init, + { wait: async () => { throw new Error('must not wait') } } + ) + assert.equal(calls, 1, `status ${status} must not be retried`) + assert.equal(response.status, status) + } +}) + +test('each attempt carries its own unexpired timeout signal', async () => { + const signals = [] + await fetchAdminOnceMore( + async (_url, attemptInit) => { + signals.push(attemptInit.signal) + return new Response('down', { status: 502 }) + }, + url, + init, + { wait: async () => {}, timeoutMs: 30_000 } + ) + assert.equal(signals.length, 2) + assert.notEqual(signals[0], signals[1]) + assert.equal(signals[1].aborted, false) +}) + +test('the caller init is forwarded unchanged apart from the signal', async () => { + let seen + await fetchAdminOnceMore( + async (seenUrl, attemptInit) => { + seen = { seenUrl, attemptInit } + return Response.json({}) + }, + url, + { method: 'POST', headers: { authorization: 'Bearer t' }, body: '{"v":1}' }, + { wait: async () => {} } + ) + assert.equal(seen.seenUrl, url) + assert.equal(seen.attemptInit.method, 'POST') + assert.deepEqual(seen.attemptInit.headers, { authorization: 'Bearer t' }) + assert.equal(seen.attemptInit.body, '{"v":1}') +}) diff --git a/cloud/dev/scripts/verify-relay-capacity-transition.mjs b/cloud/dev/scripts/verify-relay-capacity-transition.mjs index e5ebe77d45f..b81ea15afb3 100644 --- a/cloud/dev/scripts/verify-relay-capacity-transition.mjs +++ b/cloud/dev/scripts/verify-relay-capacity-transition.mjs @@ -1,4 +1,5 @@ import { pathToFileURL } from 'node:url' +import { fetchAdminOnceMore } from './relay-admin-transient-retry.mjs' const CAPACITY_PROTOCOL = 2 @@ -378,9 +379,12 @@ export async function verifyCapacityTransition(config, overrides = {}) { const token = overrides.token ?? process.env.ORCA_RELAY_ADMIN_ID_TOKEN if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable') const health = await responseJson( - await fetchImpl(`${config.directorOrigin}/health`, { - signal: AbortSignal.timeout(15_000) - }), + await fetchAdminOnceMore( + fetchImpl, + `${config.directorOrigin}/health`, + {}, + { wait, timeoutMs: 15_000 } + ), 'director health' ) if (health.ok !== true || health.connectionCapacityProtocol !== CAPACITY_PROTOCOL) { @@ -394,12 +398,16 @@ export async function verifyCapacityTransition(config, overrides = {}) { lastObservation = { runtimeAvailable: runtime !== null } if ((runtime === null) === (config.runtime === 'unavailable')) { const result = await responseJson( - await fetchImpl(`${config.directorOrigin}/v1/admin/cell-status`, { - method: 'POST', - headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, - body: JSON.stringify({ v: 1, cellId: config.cellId }), - signal: AbortSignal.timeout(30_000) - }), + await fetchAdminOnceMore( + fetchImpl, + `${config.directorOrigin}/v1/admin/cell-status`, + { + method: 'POST', + headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' }, + body: JSON.stringify({ v: 1, cellId: config.cellId }) + }, + { wait } + ), 'cell status' ) const status = result.status diff --git a/cloud/dev/scripts/verify-relay-capacity-transition.test.mjs b/cloud/dev/scripts/verify-relay-capacity-transition.test.mjs index fb865257c4a..e596ffbced7 100644 --- a/cloud/dev/scripts/verify-relay-capacity-transition.test.mjs +++ b/cloud/dev/scripts/verify-relay-capacity-transition.test.mjs @@ -1094,3 +1094,77 @@ test('does not retry a rejected cell admin token', async () => { ) assert.equal(waits, 0) }) + +test('retries a transient 503 on the director cell-status read', async () => { + const base = harness() + const statusCalls = [] + const result = await verifyCapacityTransition(config, { + token: 'masked-token', + wait: async () => {}, + fetch: async (url, options) => { + const path = new URL(url).pathname + if (path !== '/v1/admin/cell-status') return await base(url, options) + statusCalls.push(path) + if (statusCalls.length === 1) return new Response('warming up', { status: 503 }) + return await base(url, options) + } + }) + assert.equal(statusCalls.length, 2) + assert.equal(result.cellId, config.cellId) +}) + +test('fails when both director cell-status attempts return a transient 503', async () => { + const base = harness() + let statusCalls = 0 + await assert.rejects( + verifyCapacityTransition(config, { + token: 'masked-token', + wait: async () => {}, + fetch: async (url, options) => { + const path = new URL(url).pathname + if (path !== '/v1/admin/cell-status') return await base(url, options) + statusCalls += 1 + return new Response('warming up', { status: 503 }) + } + }), + /cell status returned 503/ + ) + assert.equal(statusCalls, 2) +}) + +test('retries a transient 503 on the director health preflight', async () => { + const base = harness() + let healthCalls = 0 + const result = await verifyCapacityTransition(config, { + token: 'masked-token', + wait: async () => {}, + fetch: async (url, options) => { + const path = new URL(url).pathname + if (path !== '/health') return await base(url, options) + healthCalls += 1 + if (healthCalls === 1) return new Response('warming up', { status: 503 }) + return await base(url, options) + } + }) + assert.equal(healthCalls, 2) + assert.equal(result.cellId, config.cellId) +}) + +test('fails when both director health attempts return a transient 503', async () => { + const base = harness() + let healthCalls = 0 + await assert.rejects( + verifyCapacityTransition(config, { + token: 'masked-token', + wait: async () => {}, + fetch: async (url, options) => { + const path = new URL(url).pathname + if (path !== '/health') return await base(url, options) + healthCalls += 1 + return new Response('warming up', { status: 503 }) + } + }), + /director health returned 503/ + ) + assert.equal(healthCalls, 2) +}) diff --git a/cloud/package.json b/cloud/package.json index c75d027d3d2..788b6ea2629 100644 --- a/cloud/package.json +++ b/cloud/package.json @@ -21,7 +21,7 @@ "load:relay:recovery-gate": "node dev/scripts/run-relay-recovery-wave-gate.mjs", "ops:relay": "pnpm --filter @orca-cloud/relay-ops dev", "pretest": "node --test dev/scripts/capture-terraform-plan-baseline.test.mjs dev/scripts/operate-relay-asia-admission.test.mjs dev/scripts/prepare-relay-asia-director-cells.test.mjs dev/scripts/prepare-relay-asia-topology-input.test.mjs dev/scripts/production-cloud-sql-rollout-lock.test.mjs dev/scripts/read-relay-serving-regional-placement-version.test.mjs dev/scripts/relay-asia-admission-workflow.test.mjs dev/scripts/relay-asia-rollout-evidence.test.mjs dev/scripts/relay-asia-topology-workflow.test.mjs dev/scripts/relay-cloud-sql-connection-budget.test.mjs dev/scripts/relay-load-reader-evidence.test.mjs dev/scripts/relay-staging-deploy-identity.test.mjs dev/scripts/sanitize-relay-asia-admission-result.test.mjs dev/scripts/terraform-root-partition.test.mjs dev/scripts/validate-relay-asia-topology-plan.test.mjs ../.github/actions/cloud-sql-rollout-lease/action-contract.test.mjs ../.github/actions/cloud-sql-rollout-lease/storage-lease.test.mjs", - "test": "pnpm -r test && node --test dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-regional-rehome-workflow.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs", + "test": "pnpm -r test && node --test dev/scripts/classify-relay-production-capacity-director.test.mjs dev/scripts/classify-relay-staging-bootstrap.test.mjs dev/scripts/deploy-relay-blue-green.test.mjs dev/scripts/deploy-relay-gce-candidate.test.mjs dev/scripts/deploy-relay-gce-multi-target.test.mjs dev/scripts/github-smoke-token.test.mjs dev/scripts/infra.test.mjs dev/scripts/operate-relay-regional-rehome.test.mjs dev/scripts/power-staging-relay.test.mjs dev/scripts/prepare-relay-capacity-canary.test.mjs dev/scripts/prepare-relay-production-capacity-canary.test.mjs dev/scripts/probe-relay-legacy-admission.test.mjs dev/scripts/probe-relay-rehome-trust.test.mjs dev/scripts/production-cell-image-digest-consistency.test.mjs dev/scripts/read-relay-production-capacity-identity.test.mjs dev/scripts/relay-admin-endpoint-retry-workflow.test.mjs dev/scripts/relay-admin-transient-retry.test.mjs dev/scripts/relay-admission-selector.test.mjs dev/scripts/relay-gce-terraform-fence.test.mjs dev/scripts/relay-load-connection-failure.test.mjs dev/scripts/relay-load-control-peer.test.mjs dev/scripts/relay-load-director-capacity-gate.test.mjs dev/scripts/relay-load-model.test.mjs dev/scripts/relay-load-phase-barrier.test.mjs dev/scripts/relay-load-placement-boundary.test.mjs dev/scripts/relay-load-profile.test.mjs dev/scripts/relay-load-rebind-boundary.test.mjs dev/scripts/relay-load-region-behavior.test.mjs dev/scripts/relay-load-request-unit-boundary.test.mjs dev/scripts/relay-load-run-lifecycle.test.mjs dev/scripts/relay-monitor-evidence.test.mjs dev/scripts/relay-production-capacity-wave.test.mjs dev/scripts/relay-production-capacity-workflow.test.mjs dev/scripts/relay-production-identity-boundaries.test.mjs dev/scripts/relay-production-same-cap-wave.test.mjs dev/scripts/relay-public-workflow-contract.test.mjs dev/scripts/relay-recovery-wave-gate.test.mjs dev/scripts/relay-region-observation-evidence.test.mjs dev/scripts/relay-regional-rehome-workflow.test.mjs dev/scripts/relay-rehome-aggregate-evidence.test.mjs dev/scripts/relay-repository.test.mjs dev/scripts/relay-staging-c4-refresh-workflow.test.mjs dev/scripts/relay-staging-capacity-identity.test.mjs dev/scripts/staging-relay-apply-guard.test.mjs dev/scripts/validate-relay-capacity-plan.test.mjs dev/scripts/verify-relay-capacity-transition.test.mjs dev/scripts/verify-relay-legacy-bootstrap.test.mjs dev/scripts/workload-identity-attribute-conditions.test.mjs", "typecheck": "pnpm -r typecheck" }, "devDependencies": {