diff --git a/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml b/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml index abe0f327430..78b1099744e 100644 --- a/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml +++ b/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml @@ -57,9 +57,14 @@ jobs: GATE_OVERRIDE_REASON: ${{ inputs.gate-override-reason }} GATE_OVERRIDE_CONFIRMATION: ${{ inputs.gate-override-confirmation }} OUTPUT_DIRECTORY: ${{ github.workspace }}/relay-monitor-evidence - # ~800 controls over 2 min is ~7 re-dials/s per cell, well under the director's - # 5 x 80 in-flight assign cap. A cell on an older image ignores it and drains at once. - DRAIN_PACE_WINDOW_MS: '120000' + # Each drained host re-dials /v1/assign and is re-placed through the director's + # single-slot sticky lane, which clears roughly 8 hosts/s fleet-wide. A 2,100-host + # cell over 2 min (~18/s) overran it on 2026-09-23 and browned out assign for every + # region for ~5 min. 5 min is the cell's maximum window (DRAIN_PACE_WINDOW_MAX_MS); a + # ~2,700-host cell still arrives at ~9/s, so expect a short tail, not zero rejects. + # The transition wait below is the 15-min migration lease plus this window. A cell on + # an older image drains at once. + DRAIN_PACE_WINDOW_MS: '300000' steps: - name: Require exact reusable-workflow configuration working-directory: . @@ -574,7 +579,7 @@ jobs: --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \ --heartbeat either --admission migration-only --draining required \ --activity restart-safe --expected-image-digests "${PREDECESSOR_IMAGE_DIGEST}" \ - --timeout-ms 1020000 + --timeout-ms 1200000 - id: capacity-auth if: ${{ inputs.mode != 'verify' }} diff --git a/cloud/dev/scripts/relay-same-cap-script-census.test.mjs b/cloud/dev/scripts/relay-same-cap-script-census.test.mjs index f78b5fa2232..1e023ce0c9d 100644 --- a/cloud/dev/scripts/relay-same-cap-script-census.test.mjs +++ b/cloud/dev/scripts/relay-same-cap-script-census.test.mjs @@ -311,9 +311,10 @@ describe('same-cap roll scripts accept every same-cap cell', () => { it('paces the drain it sends to the selected cell', () => { const drain = workflow.split('--mode drain')[1] ?? '' assert.match(drain.split('\n').slice(0, 2).join(' '), /--pace-window-ms "\$\{DRAIN_PACE_WINDOW_MS\}"/) - assert.match(workflow, /DRAIN_PACE_WINDOW_MS: '120000'/) + // 5 min is the cell's DRAIN_PACE_WINDOW_MAX_MS; a 2,700-host cell at 2 min overruns the director's sticky lane. + assert.match(workflow, /DRAIN_PACE_WINDOW_MS: '300000'/) // The transition wait has to outlast the pacing window on top of the leases it waits on. - assert.match(workflow, /--activity restart-safe[\s\S]*?--timeout-ms 1020000/) + assert.match(workflow, /--activity restart-safe[\s\S]*?--timeout-ms 1200000/) }) it('passes this cell\'s rehome protocol and pool on every plan validation the job runs', () => {