starred/orca
1
0
Fork 0
mirror of https://github.com/stablyai/orca.git synced 2026-09-22 00:02:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(relay): support protocol 3 in cell rollout gates (#20174)

Browse Source 
* feat(relay): support protocol 3 in cell rollout gates

* fix(relay): validate and prove protocol-3 cell rollouts

* docs(relay): clarify regional capability deployment prerequisite

* test(relay): cover protocol-3 plans across rollout cells
This commit is contained in:
Jinwoo Hong
2026-09-11 21:27:13 -04:00
committed by GitHub
parent 70a588c8bf
commit 113e58f34e
7 changed files with 37 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
+3 -3
View File
@@ -67,8 +67,8 @@ jobs:
[[ "${TARGET_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
[[ "${ROLLBACK_IMAGE_DIGEST}" =~ ^sha256:[a-f0-9]{64}$ ]]
test "${TARGET_IMAGE_DIGEST}" != "${ROLLBACK_IMAGE_DIGEST}"
[[ "${TARGET_REHOME_PROTOCOL}" =~ ^[01]$ ]]
[[ "${ROLLBACK_REHOME_PROTOCOL}" =~ ^[01]$ ]]
[[ "${TARGET_REHOME_PROTOCOL}" =~ ^(0|1|3)$ ]]
[[ "${ROLLBACK_REHOME_PROTOCOL}" =~ ^(0|1|3)$ ]]
[[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
[[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
[[ "${WAVE_INDEX}" =~ ^[0-3]$ ]]
@@ -599,7 +599,7 @@ jobs:
| jq -e '.control.enabled == false' >/dev/null
- name: Prove exact per-host trust and idempotent no-neighbor behavior
if: ${{ inputs.mode != 'verify' && ((inputs.mode == 'rollback' && inputs.rollback-rehome-protocol == '1') || (inputs.mode != 'rollback' && inputs.target-rehome-protocol == '1')) }}
if: ${{ inputs.mode != 'verify' && ((inputs.mode == 'rollback' && inputs.rollback-rehome-protocol != '0') || (inputs.mode != 'rollback' && inputs.target-rehome-protocol != '0')) }}
env:
ORCA_RELAY_ADMIN_ID_TOKEN: ${{ steps.post-auth.outputs.id_token }}
run: |
.github/workflows/cloud-deploy-relay-production-same-cap.yml
+2 -2
View File
@@ -26,13 +26,13 @@ on:
required: true
default: '1'
type: choice
options: ['0', '1']
options: ['0', '1', '3']
rollback-rehome-protocol:
description: Exact rollback regional-rehome protocol
required: true
default: '0'
type: choice
options: ['0', '1']
options: ['0', '1', '3']
expected-selector-generation:
description: Exact selector generation before the first cell
required: true
cloud/dev/scripts/relay-same-cap-script-census.test.mjs
+13 -7
View File
@@ -77,14 +77,14 @@ function rollPlan({ cellId, cap, protocol }) {
metadata_startup_script: startupScript({
cap,
image: ROLLBACK_IMAGE,
trusted: protocol === 1
trusted: protocol >= 1
})
},
after: {
metadata_startup_script: startupScript({
cap,
image: TARGET_IMAGE,
trusted: protocol === 1
trusted: protocol >= 1
}),
self_link: null
},
@@ -178,11 +178,9 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
})
it('validates a correct plan for every wave cell at that cell\'s rehome protocol', () => {
for (const cellId of SAME_CAP_CELLS) {
for (const [cellId, protocol] of SAME_CAP_CELLS.flatMap((cell) => [[cell, 1], [cell, 3]])) {
const [, cap] = resolveCellShape(cellId).stdout.trim().split(' ')
const protocol = REHOME_SOURCE_CELLS.has(cellId) ? 1 : 0
// Every reviewed serving cell carries rehome trust now, in either region.
assert.equal(protocol, 1, cellId)
assert.equal(REHOME_SOURCE_CELLS.has(cellId), true, cellId)
const config = {
mode: 'same-cap-cell',
cellId,
@@ -204,7 +202,7 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
assert.throws(
() => validateCapacityPlan(plan, {
...config,
regionalRehomeProtocol: String(1 - protocol)
regionalRehomeProtocol: '0'
}),
/reviewed image and capacity/,
cellId
@@ -239,3 +237,11 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
assert.doesNotMatch(capacityWorkflow, /--approved-cells/)
})
})
// Both trusted versions must prove the same authenticated drain boundary.
it('proves rehome trust for protocol 3 on forward and rollback rolls', () => {
const step = workflow.split('name: Prove exact per-host trust and idempotent no-neighbor behavior')[1].split('\n - name:')[0]
assert.match(step, /inputs\.rollback-rehome-protocol != '0'/)
assert.match(step, /inputs\.target-rehome-protocol != '0'/)
assert.match(step, /probe-relay-rehome-trust\.mjs/)
})
cloud/dev/scripts/validate-relay-capacity-plan.mjs
+3 -3
View File
@@ -9,7 +9,7 @@ const REHOME_CONFIG =
// Only cells listed as regional rehome sources get rehome trust lines in their startup script.
function rehomeProtocol({ regionalRehomeProtocol }) {
if (![0, 1, '0', '1'].includes(regionalRehomeProtocol)) {
if (![0, 1, 3, '0', '1', '3'].includes(regionalRehomeProtocol)) {
throw new Error('same-cap Terraform plan has an invalid regional rehome protocol')
}
return Number(regionalRehomeProtocol)
@@ -43,7 +43,7 @@ export function parseCapacityPlanArguments(argv) {
(!values['rollback-image'] ||
!values['rehome-director-service-account'] ||
!values['rehome-audience'] ||
!['0', '1'].includes(values['regional-rehome-protocol']))
!['0', '1', '3'].includes(values['regional-rehome-protocol']))
) throw new Error('same-cap validation requires rollback image and rehome trust config')
if (values.mode !== 'same-cap-cell' && values['regional-rehome-protocol'] !== undefined) {
throw new Error('--regional-rehome-protocol applies only to same-cap-cell validation')
@@ -227,7 +227,7 @@ function requireDesiredStartupScript(script, config) {
` printf 'ORCA_RELAY_CAPACITY_SERVICE_ACCOUNT=%s\\n' '${config.capacityServiceAccount}'`
])
}
const rehomeTrusted = config.mode === 'same-cap-cell' && rehomeProtocol(config) === 1
const rehomeTrusted = config.mode === 'same-cap-cell' && rehomeProtocol(config) >= 1
if (rehomeTrusted) {
expected.push(
[
cloud/dev/scripts/validate-relay-capacity-plan.test.mjs
+4
View File
@@ -756,6 +756,10 @@ test('the rehome protocol argument is required by same-cap-cell mode alone', ()
.regionalRehomeProtocol,
'0'
)
assert.equal(
parseCapacityPlanArguments(sameCapArguments('--regional-rehome-protocol', '3')).regionalRehomeProtocol,
'3'
)
assert.throws(
() => parseCapacityPlanArguments(sameCapArguments()),
/requires rollback image and rehome trust config/
cloud/dev/scripts/verify-relay-capacity-transition.mjs
+2 -2
View File
@@ -108,8 +108,8 @@ export function parseCapacityTransitionArguments(argv) {
const regionalRehomeProtocol = values['regional-rehome-protocol'] === undefined
? undefined
: integer(values['regional-rehome-protocol'], '--regional-rehome-protocol')
if (regionalRehomeProtocol !== undefined && ![0, 1].includes(regionalRehomeProtocol)) {
throw new Error('--regional-rehome-protocol must be 0 or 1')
if (regionalRehomeProtocol !== undefined && ![0, 1, 3].includes(regionalRehomeProtocol)) {
throw new Error('--regional-rehome-protocol must be 0, 1, or 3')
}
if (runtime === 'unavailable' && regionalRehomeProtocol !== undefined) {
throw new Error('unavailable runtime cannot prove the regional rehome protocol')
cloud/docs/orca-relay-operations.md
+10 -5
View File
@@ -466,11 +466,16 @@ After a deployment traffic shift, preserve the old revision/tag until metrics an
## Regional rehoming
Rehoming moves a host to a general cell in the region its desktop last reported, in either
direction. Both roles need the drain protocol: a cell without it can be neither a source nor a
target, and it is not part of the fleet whose telemetry gates the worker. Until the asia-east2
cells run `regionalRehomeProtocol` 1 they are none of the three, so no host is moved into or out
of Asia and an Asia cell in distress does not pause the worker.
Idle regional correction requires both source and target cells to advertise
`regionalRehomeProtocol >= 3`. PR #20105 introduced this capability version with
the idle handoff implementation. With that runtime, both rehome trust environment
settings must be configured to advertise 3; otherwise the cell advertises 0.
An older trusted runtime can advertise 1: configuring trust alone does not upgrade
its implementation. The separate `connectionCapacityProtocol: 2` health field does
not establish regional-correction readiness. Verify the live runtime version and
image, not only instance-template configuration, before rollout or enablement.
Incompatible cells are excluded from correction selection; enabling the cohort
cannot override this check. Director and cell deployments are separate operations.
`host-cooldown-ms` is the minimum gap between two rehomes of one host. It bounds the damage from
a desktop whose region probe flips: without it the host would be dragged back across the ocean on