From 113e58f34e53d7496b0473346dbc209ff0a805be Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Fri, 11 Sep 2026 21:27:13 -0400 Subject: [PATCH] feat(relay): support protocol 3 in cell rollout gates (#20174) * feat(relay): support protocol 3 in cell rollout gates * fix(relay): validate and prove protocol-3 cell rollouts * docs(relay): clarify regional capability deployment prerequisite * test(relay): cover protocol-3 plans across rollout cells --- ...d-deploy-relay-production-same-cap-job.yml | 6 +++--- ...cloud-deploy-relay-production-same-cap.yml | 4 ++-- .../relay-same-cap-script-census.test.mjs | 20 ++++++++++++------- .../scripts/validate-relay-capacity-plan.mjs | 6 +++--- .../validate-relay-capacity-plan.test.mjs | 4 ++++ .../verify-relay-capacity-transition.mjs | 4 ++-- cloud/docs/orca-relay-operations.md | 15 +++++++++----- 7 files changed, 37 insertions(+), 22 deletions(-) diff --git a/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml b/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml index 8ef61507088..2b4bb3fa439 100644 --- a/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml +++ b/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml @@ -67,8 +67,8 @@ jobs: [[ "${TARGET_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]] [[ "${ROLLBACK_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]] test "${TARGET_IMAGE_DIGEST}" != "${ROLLBACK_IMAGE_DIGEST}" - [[ "${TARGET_REHOME_PROTOCOL}" =~ ^[01]$ ]] - [[ "${ROLLBACK_REHOME_PROTOCOL}" =~ ^[01]$ ]] + [[ "${TARGET_REHOME_PROTOCOL}" =~ ^(0|1|3)$ ]] + [[ "${ROLLBACK_REHOME_PROTOCOL}" =~ ^(0|1|3)$ ]] [[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]] [[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]] [[ "${WAVE_INDEX}" =~ ^[0-3]$ ]] @@ -599,7 +599,7 @@ jobs: | jq -e '.control.enabled == false' >/dev/null - name: Prove exact per-host trust and idempotent no-neighbor behavior - if: ${{ inputs.mode != 'verify' && ((inputs.mode == 'rollback' && inputs.rollback-rehome-protocol == '1') || (inputs.mode != 'rollback' && inputs.target-rehome-protocol == '1')) }} + if: ${{ inputs.mode != 'verify' && ((inputs.mode == 'rollback' && inputs.rollback-rehome-protocol != '0') || (inputs.mode != 'rollback' && inputs.target-rehome-protocol != '0')) }} env: ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }} run: | diff --git a/.github/workflows/cloud-deploy-relay-production-same-cap.yml b/.github/workflows/cloud-deploy-relay-production-same-cap.yml index fba5df0dcb9..50d609b30d3 100644 --- a/.github/workflows/cloud-deploy-relay-production-same-cap.yml +++ b/.github/workflows/cloud-deploy-relay-production-same-cap.yml @@ -26,13 +26,13 @@ on: required: true default: '1' type: choice - options: ['0', '1'] + options: ['0', '1', '3'] rollback-rehome-protocol: description: Exact rollback regional-rehome protocol required: true default: '0' type: choice - options: ['0', '1'] + options: ['0', '1', '3'] expected-selector-generation: description: Exact selector generation before the first cell required: true diff --git a/cloud/dev/scripts/relay-same-cap-script-census.test.mjs b/cloud/dev/scripts/relay-same-cap-script-census.test.mjs index 7d5e4fee73e..3e5f0758028 100644 --- a/cloud/dev/scripts/relay-same-cap-script-census.test.mjs +++ b/cloud/dev/scripts/relay-same-cap-script-census.test.mjs @@ -77,14 +77,14 @@ function rollPlan({ cellId, cap, protocol }) { metadata_startup_script: startupScript({ cap, image: ROLLBACK_IMAGE, - trusted: protocol === 1 + trusted: protocol >= 1 }) }, after: { metadata_startup_script: startupScript({ cap, image: TARGET_IMAGE, - trusted: protocol === 1 + trusted: protocol >= 1 }), self_link: null }, @@ -178,11 +178,9 @@ describe('same-cap roll scripts accept every same-cap cell', () => { }) it('validates a correct plan for every wave cell at that cell\'s rehome protocol', () => { - for (const cellId of SAME_CAP_CELLS) { + for (const [cellId, protocol] of SAME_CAP_CELLS.flatMap((cell) => [[cell, 1], [cell, 3]])) { const [, cap] = resolveCellShape(cellId).stdout.trim().split(' ') - const protocol = REHOME_SOURCE_CELLS.has(cellId) ? 1 : 0 - // Every reviewed serving cell carries rehome trust now, in either region. - assert.equal(protocol, 1, cellId) + assert.equal(REHOME_SOURCE_CELLS.has(cellId), true, cellId) const config = { mode: 'same-cap-cell', cellId, @@ -204,7 +202,7 @@ describe('same-cap roll scripts accept every same-cap cell', () => { assert.throws( () => validateCapacityPlan(plan, { ...config, - regionalRehomeProtocol: String(1 - protocol) + regionalRehomeProtocol: '0' }), /reviewed image and capacity/, cellId @@ -239,3 +237,11 @@ describe('same-cap roll scripts accept every same-cap cell', () => { assert.doesNotMatch(capacityWorkflow, /--approved-cells/) }) }) + +// Both trusted versions must prove the same authenticated drain boundary. +it('proves rehome trust for protocol 3 on forward and rollback rolls', () => { + const step = workflow.split('name: Prove exact per-host trust and idempotent no-neighbor behavior')[1].split('\n - name:')[0] + assert.match(step, /inputs\.rollback-rehome-protocol != '0'/) + assert.match(step, /inputs\.target-rehome-protocol != '0'/) + assert.match(step, /probe-relay-rehome-trust\.mjs/) +}) diff --git a/cloud/dev/scripts/validate-relay-capacity-plan.mjs b/cloud/dev/scripts/validate-relay-capacity-plan.mjs index 294e85ae31d..34e84d51ead 100644 --- a/cloud/dev/scripts/validate-relay-capacity-plan.mjs +++ b/cloud/dev/scripts/validate-relay-capacity-plan.mjs @@ -9,7 +9,7 @@ const REHOME_CONFIG = // Only cells listed as regional rehome sources get rehome trust lines in their startup script. function rehomeProtocol({ regionalRehomeProtocol }) { - if (![0, 1, '0', '1'].includes(regionalRehomeProtocol)) { + if (![0, 1, 3, '0', '1', '3'].includes(regionalRehomeProtocol)) { throw new Error('same-cap Terraform plan has an invalid regional rehome protocol') } return Number(regionalRehomeProtocol) @@ -43,7 +43,7 @@ export function parseCapacityPlanArguments(argv) { (!values['rollback-image'] || !values['rehome-director-service-account'] || !values['rehome-audience'] || - !['0', '1'].includes(values['regional-rehome-protocol'])) + !['0', '1', '3'].includes(values['regional-rehome-protocol'])) ) throw new Error('same-cap validation requires rollback image and rehome trust config') if (values.mode !== 'same-cap-cell' && values['regional-rehome-protocol'] !== undefined) { throw new Error('--regional-rehome-protocol applies only to same-cap-cell validation') @@ -227,7 +227,7 @@ function requireDesiredStartupScript(script, config) { ` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${config.capacityServiceAccount}'` ]) } - const rehomeTrusted = config.mode === 'same-cap-cell' && rehomeProtocol(config) === 1 + const rehomeTrusted = config.mode === 'same-cap-cell' && rehomeProtocol(config) >= 1 if (rehomeTrusted) { expected.push( [ diff --git a/cloud/dev/scripts/validate-relay-capacity-plan.test.mjs b/cloud/dev/scripts/validate-relay-capacity-plan.test.mjs index fb6ccb57e1c..d6886c58011 100644 --- a/cloud/dev/scripts/validate-relay-capacity-plan.test.mjs +++ b/cloud/dev/scripts/validate-relay-capacity-plan.test.mjs @@ -756,6 +756,10 @@ test('the rehome protocol argument is required by same-cap-cell mode alone', () .regionalRehomeProtocol, '0' ) + assert.equal( + parseCapacityPlanArguments(sameCapArguments('--regional-rehome-protocol', '3')).regionalRehomeProtocol, + '3' + ) assert.throws( () => parseCapacityPlanArguments(sameCapArguments()), /requires rollback image and rehome trust config/ diff --git a/cloud/dev/scripts/verify-relay-capacity-transition.mjs b/cloud/dev/scripts/verify-relay-capacity-transition.mjs index b81ea15afb3..0c297fdad53 100644 --- a/cloud/dev/scripts/verify-relay-capacity-transition.mjs +++ b/cloud/dev/scripts/verify-relay-capacity-transition.mjs @@ -108,8 +108,8 @@ export function parseCapacityTransitionArguments(argv) { const regionalRehomeProtocol = values['regional-rehome-protocol'] === undefined ? undefined : integer(values['regional-rehome-protocol'], '--regional-rehome-protocol') - if (regionalRehomeProtocol !== undefined && ![0, 1].includes(regionalRehomeProtocol)) { - throw new Error('--regional-rehome-protocol must be 0 or 1') + if (regionalRehomeProtocol !== undefined && ![0, 1, 3].includes(regionalRehomeProtocol)) { + throw new Error('--regional-rehome-protocol must be 0, 1, or 3') } if (runtime === 'unavailable' && regionalRehomeProtocol !== undefined) { throw new Error('unavailable runtime cannot prove the regional rehome protocol') diff --git a/cloud/docs/orca-relay-operations.md b/cloud/docs/orca-relay-operations.md index f27b437fff9..4515048b29b 100644 --- a/cloud/docs/orca-relay-operations.md +++ b/cloud/docs/orca-relay-operations.md @@ -466,11 +466,16 @@ After a deployment traffic shift, preserve the old revision/tag until metrics an ## Regional rehoming -Rehoming moves a host to a general cell in the region its desktop last reported, in either -direction. Both roles need the drain protocol: a cell without it can be neither a source nor a -target, and it is not part of the fleet whose telemetry gates the worker. Until the asia-east2 -cells run `regionalRehomeProtocol` 1 they are none of the three, so no host is moved into or out -of Asia and an Asia cell in distress does not pause the worker. +Idle regional correction requires both source and target cells to advertise +`regionalRehomeProtocol >= 3`. PR #20105 introduced this capability version with +the idle handoff implementation. With that runtime, both rehome trust environment +settings must be configured to advertise 3; otherwise the cell advertises 0. +An older trusted runtime can advertise 1: configuring trust alone does not upgrade +its implementation. The separate `connectionCapacityProtocol: 2` health field does +not establish regional-correction readiness. Verify the live runtime version and +image, not only instance-template configuration, before rollout or enablement. +Incompatible cells are excluded from correction selection; enabling the cohort +cannot override this check. Director and cell deployments are separate operations. `host-cooldown-ms` is the minimum gap between two rehomes of one host. It bounds the damage from a desktop whose region probe flips: without it the host would be dragged back across the ocean on