diff --git a/cloud/dev/scripts/probe-relay-rehome-trust.mjs b/cloud/dev/scripts/probe-relay-rehome-trust.mjs index 2208131e58a..73ea44a40f7 100644 --- a/cloud/dev/scripts/probe-relay-rehome-trust.mjs +++ b/cloud/dev/scripts/probe-relay-rehome-trust.mjs @@ -38,7 +38,7 @@ export function parseRehomeTrustProbeArguments(argv, environment = process.env) export async function probeRehomeTrust(config, dependencies = {}) { const fetchImpl = dependencies.fetch ?? fetch - const response = await fetchAdminOnceMore( + const request = () => fetchAdminOnceMore( fetchImpl, `${config.directorOrigin}/v1/admin/regional-rehome-trust-probe`, { @@ -55,9 +55,26 @@ export async function probeRehomeTrust(config, dependencies = {}) { }, { wait: dependencies.wait } ) - const body = await response.json().catch(() => ({})) + let response = await request() + let body = await response.json().catch(() => ({})) + // The director wraps source HTTP failures in 409; retry only explicit transient statuses. + if (response.status === 409 && /^regional_rehome_trust_probe_source_(500|502|503|504)$/.test(body?.error ?? '')) { + await (dependencies.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))))(2_000) + response = await request() + body = await response.json().catch(() => ({})) + } if (!response.ok) { - throw new Error(`application-mediated rehome trust probe returned ${response.status}`) + const safeReasons = new Set([ + 'invalid_token', 'director_only', 'invalid_request', + 'regional_rehome_trust_not_configured', + 'regional_rehome_trust_probe_source_unavailable', + 'regional_rehome_trust_probe_source_invalid_response', + 'regional_rehome_trust_probe_not_proven', + ...[400, 401, 403, 404, 409, 429, 500, 502, 503, 504] + .map((status) => `regional_rehome_trust_probe_source_${status}`) + ]) + const reason = safeReasons.has(body?.error) ? body.error : 'unrecognized_error' + throw new Error(`application-mediated rehome trust probe returned ${response.status}: ${reason}`) } if ( body.v !== 1 || diff --git a/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs b/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs index 789d33c40b6..034fcd06c4f 100644 --- a/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs +++ b/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs @@ -131,3 +131,40 @@ test('approves the asia-east2 rehome sources and still rejects unlisted cells', ) } }) + +test('retries one director-wrapped source 503 without relaxing the proof', async () => { + let calls = 0 + const result = await probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), { + wait: async () => {}, + fetch: async () => ++calls === 1 + ? Response.json({ error: 'regional_rehome_trust_probe_source_503' }, { status: 409 }) + : Response.json(provenProbe) + }) + assert.equal(calls, 2) + assert.equal(result.proven, true) +}) + +test('reports safe trust reasons, keeps rejection final, and redacts arbitrary error text', async () => { + for (const reason of ['regional_rehome_trust_probe_source_403', 'secret-token-example']) { + let calls = 0 + await assert.rejects(probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), { + wait: async () => { throw new Error('must not retry') }, + fetch: async () => { calls++; return Response.json({ error: reason }, { status: 409 }) } + }), error => { + assert.match(error.message, /returned 409/) + assert.ok(!error.message.includes('secret-token-example')) + if (reason.endsWith('_403')) assert.match(error.message, /source_403/) + return true + }) + assert.equal(calls, 1) + } +}) + +test('stops after the second wrapped transient failure', async () => { + let calls = 0 + await assert.rejects(probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), { + wait: async () => {}, + fetch: async () => { calls++; return Response.json({ error: 'regional_rehome_trust_probe_source_503' }, { status: 409 }) } + }), /returned 409.*source_503/) + assert.equal(calls, 2) +})