fix(cloud): diagnose wrapped relay trust probe failures (#20403)

cloud/dev/scripts/probe-relay-rehome-trust.mjs

@@ -38,7 +38,7 @@ export function parseRehomeTrustProbeArguments(argv, environment = process.env)
 
 export async function probeRehomeTrust(config, dependencies = {}) {
   const fetchImpl = dependencies.fetch ?? fetch
-  const response = await fetchAdminOnceMore(
+  const request = () => fetchAdminOnceMore(
     fetchImpl,
     `${config.directorOrigin}/v1/admin/regional-rehome-trust-probe`,
     {
@@ -55,9 +55,26 @@ export async function probeRehomeTrust(config, dependencies = {}) {
     },
     { wait: dependencies.wait }
   )
-  const body = await response.json().catch(() => ({}))
+  let response = await request()
+  let body = await response.json().catch(() => ({}))
+  // The director wraps source HTTP failures in 409; retry only explicit transient statuses.
+  if (response.status === 409 && /^regional_rehome_trust_probe_source_(500|502|503|504)$/.test(body?.error ?? '')) {
+    await (dependencies.wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))))(2_000)
+    response = await request()
+    body = await response.json().catch(() => ({}))
+  }
   if (!response.ok) {
-    throw new Error(`application-mediated rehome trust probe returned ${response.status}`)
+    const safeReasons = new Set([
+      'invalid_token', 'director_only', 'invalid_request',
+      'regional_rehome_trust_not_configured',
+      'regional_rehome_trust_probe_source_unavailable',
+      'regional_rehome_trust_probe_source_invalid_response',
+      'regional_rehome_trust_probe_not_proven',
+      ...[400, 401, 403, 404, 409, 429, 500, 502, 503, 504]
+        .map((status) => `regional_rehome_trust_probe_source_${status}`)
+    ])
+    const reason = safeReasons.has(body?.error) ? body.error : 'unrecognized_error'
+    throw new Error(`application-mediated rehome trust probe returned ${response.status}: ${reason}`)
   }
   if (
     body.v !== 1 ||

cloud/dev/scripts/probe-relay-rehome-trust.test.mjs

@@ -131,3 +131,40 @@ test('approves the asia-east2 rehome sources and still rejects unlisted cells',
     )
   }
 })
+
+test('retries one director-wrapped source 503 without relaxing the proof', async () => {
+  let calls = 0
+  const result = await probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), {
+    wait: async () => {},
+    fetch: async () => ++calls === 1
+      ? Response.json({ error: 'regional_rehome_trust_probe_source_503' }, { status: 409 })
+      : Response.json(provenProbe)
+  })
+  assert.equal(calls, 2)
+  assert.equal(result.proven, true)
+})
+
+test('reports safe trust reasons, keeps rejection final, and redacts arbitrary error text', async () => {
+  for (const reason of ['regional_rehome_trust_probe_source_403', 'secret-token-example']) {
+    let calls = 0
+    await assert.rejects(probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), {
+      wait: async () => { throw new Error('must not retry') },
+      fetch: async () => { calls++; return Response.json({ error: reason }, { status: 409 }) }
+    }), error => {
+      assert.match(error.message, /returned 409/)
+      assert.ok(!error.message.includes('secret-token-example'))
+      if (reason.endsWith('_403')) assert.match(error.message, /source_403/)
+      return true
+    })
+    assert.equal(calls, 1)
+  }
+})
+
+test('stops after the second wrapped transient failure', async () => {
+  let calls = 0
+  await assert.rejects(probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), {
+    wait: async () => {},
+    fetch: async () => { calls++; return Response.json({ error: 'regional_rehome_trust_probe_source_503' }, { status: 409 }) }
+  }), /returned 409.*source_503/)
+  assert.equal(calls, 2)
+})