mirror of
https://github.com/stablyai/orca.git
synced 2026-09-22 00:02:31 +00:00
chore(relay): cut the cell LB connection drain to 60 s and allow ten-cell same-cap batches (#21848)
* perf(relay): cut the cell LB drain to 60s and widen the same-cap batch to ten cells Two independent sources of relay roll wall clock, neither of which protects a host: 1. `connection_draining_timeout_sec` on the per-cell backend services was 300s. The same-cap job drains every host off the cell to a restart-safe condition before Terraform runs, so the LB drain only ever covers a host still mid-handshake. Measured 2026-09-16 over ten same-cap cell jobs, it sat as ~5m55s of dead time between `Apply complete` and the old VM powering off, inside an 8.5-minute `wait-until --stable` step. Now 60s, and pinned in the topology `check` block beside the other fixed-one invariants. 2. The same-cap wave capped a batch at four cells, so a 22-cell roll needed six batches, six single-use monitor gates, and a human handoff per batch. The wave workflow now declares cell_1..cell_10 with the identical serial shape and chaining, and the validator accepts two to ten. The shared wave-index rule (`relay-monitor-evidence.mjs` and the relay-ops preflight CLI) widens from 0-3 to 0-9 so the later cells can present the same evidence; each job workflow keeps its own narrower range, so the capacity wave stays at four. Cells remain strictly serial, one at a time behind the rollout lease, each with its own live preflight. Claude-Session: https://claude.ai/session/relay-roll-drain-timeout-and-batch-cap * fix(relay): align the Asia topology plan validator with the 60s cell drain `validate-relay-asia-topology-plan.mjs` rejected any Asia backend whose `connection_draining_timeout_sec` was not 300, and `cloud-deploy-relay-asia-topology.yml` targets `google_compute_backend_service.relay_gce_cell["<cell>"]` per cell. With the Terraform local at 60 that workflow would have failed its own plan review. The validator's two restated topology values are now named exports, and a new census test reads `relay-gce-cells.tf` and equates three statements of each: the `relay_gce_topology` local, the topology `check` assert that pins it, and the validator constant. Terraform cannot export a local to JS, so reading the source is the only way to stop them drifting; the test was confirmed to fail when the local alone is moved back to 300. Repo-wide grep finds no other pin of the drain value. Claude-Session: https://claude.ai/session/relay-roll-drain-timeout-and-batch-cap
This commit is contained in:
Jinwoo Hong
@@ -78,7 +78,8 @@ jobs:
|
||||
[[ "${ROLLBACK_REHOME_PROTOCOL}" =~ ^(0|1|3)$ ]]
|
||||
[[ "${EXPECTED_SELECTOR_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
|
||||
[[ "${EXPECTED_REHOME_GENERATION}" =~ ^(0|[1-9][0-9]*)$ ]]
|
||||
[[ "${WAVE_INDEX}" =~ ^[0-3]$ ]]
|
||||
# cell_1..cell_10 in the calling wave; the chain is static, so this range is too.
|
||||
[[ "${WAVE_INDEX}" =~ ^[0-9]$ ]]
|
||||
# The caller validated this too; re-check here so a malformed override
|
||||
# can never reach a mutation through this reusable workflow.
|
||||
if test -n "${GATE_OVERRIDE_REASON}${GATE_OVERRIDE_CONFIRMATION}"; then
|
||||
|
||||
@@ -10,7 +10,7 @@ on:
|
||||
type: choice
|
||||
options: [verify, canary-apply, batch-apply, rollback]
|
||||
cell-ids:
|
||||
description: Ordered comma-separated serving cells; one canary or two to four batch cells
|
||||
description: Ordered comma-separated serving cells; one canary or two to ten batch cells
|
||||
required: true
|
||||
type: string
|
||||
target-image-digest:
|
||||
@@ -313,6 +313,144 @@ jobs:
|
||||
wave-index: '3'
|
||||
secrets: inherit
|
||||
|
||||
cell_5:
|
||||
if: ${{ needs.cell_4.result == 'success' && fromJSON(needs.gate.outputs.cells)[4] != null }}
|
||||
needs: [gate, cell_4]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
||||
with:
|
||||
mode: ${{ needs.gate.outputs.job-mode }}
|
||||
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[4] }}
|
||||
target-image-digest: ${{ inputs.target-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
||||
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
monitor-run-id: ${{ inputs.monitor-run-id }}
|
||||
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
||||
gate-override-reason: ${{ inputs.gate-override-reason }}
|
||||
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
|
||||
wave-index: '4'
|
||||
secrets: inherit
|
||||
|
||||
cell_6:
|
||||
if: ${{ needs.cell_5.result == 'success' && fromJSON(needs.gate.outputs.cells)[5] != null }}
|
||||
needs: [gate, cell_5]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
||||
with:
|
||||
mode: ${{ needs.gate.outputs.job-mode }}
|
||||
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[5] }}
|
||||
target-image-digest: ${{ inputs.target-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
||||
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
monitor-run-id: ${{ inputs.monitor-run-id }}
|
||||
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
||||
gate-override-reason: ${{ inputs.gate-override-reason }}
|
||||
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
|
||||
wave-index: '5'
|
||||
secrets: inherit
|
||||
|
||||
cell_7:
|
||||
if: ${{ needs.cell_6.result == 'success' && fromJSON(needs.gate.outputs.cells)[6] != null }}
|
||||
needs: [gate, cell_6]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
||||
with:
|
||||
mode: ${{ needs.gate.outputs.job-mode }}
|
||||
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[6] }}
|
||||
target-image-digest: ${{ inputs.target-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
||||
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
monitor-run-id: ${{ inputs.monitor-run-id }}
|
||||
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
||||
gate-override-reason: ${{ inputs.gate-override-reason }}
|
||||
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
|
||||
wave-index: '6'
|
||||
secrets: inherit
|
||||
|
||||
cell_8:
|
||||
if: ${{ needs.cell_7.result == 'success' && fromJSON(needs.gate.outputs.cells)[7] != null }}
|
||||
needs: [gate, cell_7]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
||||
with:
|
||||
mode: ${{ needs.gate.outputs.job-mode }}
|
||||
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[7] }}
|
||||
target-image-digest: ${{ inputs.target-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
||||
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
monitor-run-id: ${{ inputs.monitor-run-id }}
|
||||
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
||||
gate-override-reason: ${{ inputs.gate-override-reason }}
|
||||
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
|
||||
wave-index: '7'
|
||||
secrets: inherit
|
||||
|
||||
cell_9:
|
||||
if: ${{ needs.cell_8.result == 'success' && fromJSON(needs.gate.outputs.cells)[8] != null }}
|
||||
needs: [gate, cell_8]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
||||
with:
|
||||
mode: ${{ needs.gate.outputs.job-mode }}
|
||||
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[8] }}
|
||||
target-image-digest: ${{ inputs.target-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
||||
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
monitor-run-id: ${{ inputs.monitor-run-id }}
|
||||
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
||||
gate-override-reason: ${{ inputs.gate-override-reason }}
|
||||
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
|
||||
wave-index: '8'
|
||||
secrets: inherit
|
||||
|
||||
cell_10:
|
||||
if: ${{ needs.cell_9.result == 'success' && fromJSON(needs.gate.outputs.cells)[9] != null }}
|
||||
needs: [gate, cell_9]
|
||||
uses: ./.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
|
||||
with:
|
||||
mode: ${{ needs.gate.outputs.job-mode }}
|
||||
target-cell-id: ${{ fromJSON(needs.gate.outputs.cells)[9] }}
|
||||
target-image-digest: ${{ inputs.target-image-digest }}
|
||||
rollback-image-digest: ${{ inputs.rollback-image-digest }}
|
||||
target-rehome-protocol: ${{ inputs.target-rehome-protocol }}
|
||||
rollback-rehome-protocol: ${{ inputs.rollback-rehome-protocol }}
|
||||
expected-selector-generation: ${{ inputs.expected-selector-generation }}
|
||||
expected-existing-only-cells: ${{ inputs.expected-existing-only-cells }}
|
||||
expected-migration-only-cells: ${{ inputs.expected-migration-only-cells }}
|
||||
expected-general-cells: ${{ inputs.expected-general-cells }}
|
||||
expected-rehome-generation: ${{ inputs.expected-rehome-generation }}
|
||||
monitor-run-id: ${{ inputs.monitor-run-id }}
|
||||
monitor-run-attempt: ${{ inputs.monitor-run-attempt }}
|
||||
gate-override-reason: ${{ inputs.gate-override-reason }}
|
||||
gate-override-confirmation: ${{ inputs.gate-override-confirmation }}
|
||||
wave-index: '9'
|
||||
secrets: inherit
|
||||
|
||||
seal_canary:
|
||||
if: ${{ inputs.mode == 'canary-apply' }}
|
||||
needs: [gate, cell_1]
|
||||
@@ -360,6 +498,12 @@ jobs:
|
||||
- cell_2
|
||||
- cell_3
|
||||
- cell_4
|
||||
- cell_5
|
||||
- cell_6
|
||||
- cell_7
|
||||
- cell_8
|
||||
- cell_9
|
||||
- cell_10
|
||||
- seal_canary
|
||||
runs-on: blacksmith-2vcpu-ubuntu-2204
|
||||
timeout-minutes: 10
|
||||
|
||||
Reference in New Issue
Block a user