Files
orca/cloud/dev/scripts/probe-relay-rehome-trust.test.mjs
T
Jinwoo Hong bf3f95245c feat(relay): declare Asia cell c30 at the c27 shape (#22375)
* feat(relay): declare Asia cell c30 at the c27 shape

Adds production-gce-c30 in asia-east2-a at the reviewed Asia shape (6,000
request units, 3,000/60 connection limits, 16-connection pool, disabled) and
the rehome trust the other Asia cells carry.

Every Asia enumeration now knows C30. The topology, admission, and director
tools treat it as its own reviewed wave so its plan and registration never
touch the live launch cells. C30 promotion requires C27 general and fresh
staging evidence. The topology and director validators now pin the committed
production pool of 16 instead of the stale 10, which had made the topology
workflow reject the committed launch cells.

* fix(relay): plan C30 at live images and prove it with its own canary

The shared URL map pulls every cell into the C30 topology plan, so the workflow
now plans each non-target cell at the image its live template serves, and the
validator names any change to a cell outside the wave. C30 promotion runs the
same five-minute production canary and automatic rollback C27 used, with the
load report proving the canary control was placed on C30, instead of relying
on staging evidence. C30 leaves the shadow gate's fleet pool list until it
serves, rollback rejects mixed partial sets, and a budget test pins the
mixed-Asia-pool refusal.

* fix(relay): pin C30 to the production director's live image digest

C30 promotion requires the director and C30 to report one digest, so C30
takes the director's sha256:4158d8a2 (read 2026-09-22). C27-C29 keep their
committed lines; every Asia check compares only the cells named in a run.

* fix(relay): read the committed cell map from a plan, not console

terraform console evaluates every output against state, and the Relay
deployments output indexes each cell's MIG, so it fails with Invalid index
while C30 is declared but not created. Read the map from a no-refresh,
unlocked plan over the same targets instead, and refuse empty overlay input.

* fix(relay): keep console readers working and C30 migration-only until promotion

relay_gce_cell_deployments indexed each cell's MIG, backend, and template,
so once C30 is declared but not applied every production terraform console
reader printed a warning to stdout and broke its jq parse. Wrap those six
lookups in try(..., null).

Same-cap listed C30 as general, so a rollback dispatch on a migration-only
C30 would restore it with activate and skip its canary. List it with the
migration-only cells until the promotion follow-up moves it.
2026-09-22 23:41:09 -04:00

173 lines
5.6 KiB
JavaScript

import assert from 'node:assert/strict'
import { test } from 'node:test'
import {
parseRehomeTrustProbeArguments,
probeRehomeTrust
} from './probe-relay-rehome-trust.mjs'
const argv = [
'--director-origin', 'https://relay.onorca.dev',
'--cell-id', 'production-gce-c7',
'--cell-incarnation', '11111111-1111-4111-8111-111111111111'
]
const environment = { ORCA_RELAY_ADMIN_ID_TOKEN: 'aaa.bbb.ccc' }
test('binds the application-mediated probe to an exact approved cell incarnation', () => {
assert.equal(parseRehomeTrustProbeArguments(argv, environment).cellId, 'production-gce-c7')
assert.throws(() => parseRehomeTrustProbeArguments(
argv.with(1, 'https://other.example.test'),
environment
))
assert.throws(() => parseRehomeTrustProbeArguments(argv, {
ORCA_RELAY_ADMIN_ID_TOKEN: 'not-a-token'
}))
})
test('requires complete aggregate application-mediated trust proof', async () => {
const config = parseRehomeTrustProbeArguments(argv, environment)
const result = await probeRehomeTrust(config, {
fetch: async (url, init) => {
assert.equal(url, 'https://relay.onorca.dev/v1/admin/regional-rehome-trust-probe')
assert.deepEqual(JSON.parse(init.body), {
v: 1,
sourceCellId: 'production-gce-c7',
sourceCellIncarnation: '11111111-1111-4111-8111-111111111111'
})
return Response.json({
v: 1,
dedicatedIdentity: {
firstOutcome: 'host-not-connected',
secondOutcome: 'host-not-connected',
accepted: true,
idempotent: true
},
sharedRuntimeIdentityRejected: true,
proven: true
})
}
})
assert.equal(result.proven, true)
})
test('rejects partial or mismatched proof', async () => {
const config = parseRehomeTrustProbeArguments(argv, environment)
await assert.rejects(
probeRehomeTrust(config, {
fetch: async () => Response.json({
v: 1,
dedicatedIdentity: {
firstOutcome: 'host-not-connected',
secondOutcome: 'host-not-connected',
accepted: true,
idempotent: true
},
sharedRuntimeIdentityRejected: false,
proven: false
})
}),
/incomplete/
)
})
const provenProbe = {
v: 1,
dedicatedIdentity: {
firstOutcome: 'host-not-connected',
secondOutcome: 'host-not-connected',
accepted: true,
idempotent: true
},
sharedRuntimeIdentityRejected: true,
proven: true
}
test('retries a transient 503 on the trust probe and proves on the second answer', async () => {
const config = parseRehomeTrustProbeArguments(argv, environment)
let calls = 0
const result = await probeRehomeTrust(config, {
wait: async () => {},
fetch: async () => {
calls += 1
if (calls === 1) return new Response('warming up', { status: 503 })
return Response.json(provenProbe)
}
})
assert.equal(calls, 2)
assert.equal(result.proven, true)
})
test('fails when both trust-probe attempts return a transient 503', async () => {
const config = parseRehomeTrustProbeArguments(argv, environment)
let calls = 0
await assert.rejects(
probeRehomeTrust(config, {
wait: async () => {},
fetch: async () => {
calls += 1
return new Response('warming up', { status: 503 })
}
}),
/returned 503/
)
assert.equal(calls, 2)
})
test('approves the asia-east2 rehome sources and still rejects unlisted cells', () => {
for (const cellId of [
'production-gce-c27', 'production-gce-c28', 'production-gce-c29', 'production-gce-c30'
]) {
const parsed = parseRehomeTrustProbeArguments(
argv.map((value) => (value === 'production-gce-c7' ? cellId : value)),
environment
)
assert.equal(parsed.cellId, cellId)
}
for (const cellId of ['production-gce-c1', 'production-gce-c17', 'production-gce-c31']) {
assert.throws(
() =>
parseRehomeTrustProbeArguments(
argv.map((value) => (value === 'production-gce-c7' ? cellId : value)),
environment
),
/--cell-id is not approved/
)
}
})
test('retries one director-wrapped source 503 without relaxing the proof', async () => {
let calls = 0
const result = await probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), {
wait: async () => {},
fetch: async () => ++calls === 1
? Response.json({ error: 'regional_rehome_trust_probe_source_503' }, { status: 409 })
: Response.json(provenProbe)
})
assert.equal(calls, 2)
assert.equal(result.proven, true)
})
test('reports safe trust reasons, keeps rejection final, and redacts arbitrary error text', async () => {
for (const reason of ['regional_rehome_trust_probe_source_403', 'secret-token-example']) {
let calls = 0
await assert.rejects(probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), {
wait: async () => { throw new Error('must not retry') },
fetch: async () => { calls++; return Response.json({ error: reason }, { status: 409 }) }
}), error => {
assert.match(error.message, /returned 409/)
assert.ok(!error.message.includes('secret-token-example'))
if (reason.endsWith('_403')) assert.match(error.message, /source_403/)
return true
})
assert.equal(calls, 1)
}
})
test('stops after the second wrapped transient failure', async () => {
let calls = 0
await assert.rejects(probeRehomeTrust(parseRehomeTrustProbeArguments(argv, environment), {
wait: async () => {},
fetch: async () => { calls++; return Response.json({ error: 'regional_rehome_trust_probe_source_503' }, { status: 409 }) }
}), /returned 409.*source_503/)
assert.equal(calls, 2)
})