From 128e97ffca080d39d0afea194c4b3c7cad24046b Mon Sep 17 00:00:00 2001
From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com>
Date: Wed, 23 Sep 2026 23:38:15 -0400
Subject: [PATCH] fix(relay): drain a same-cap cell over 5 minutes, not 2
 (#22584)

* fix(relay): drain a same-cap cell over 5 minutes, not 2

The 2026-09-23 c27 roll drained 2,145 hosts over the 2-minute window,
about 18 re-dials/s, while the director re-places roughly 8/s through
its single-slot sticky lane. The overflow queued behind slow
re-placements and timed out, so /v1/assign returned 503 fleet-wide for
about 5 minutes. 5 minutes is the cell's maximum pace window and keeps
the remaining 2,650-host cells near lane capacity. The transition wait
already outlasts a 5-minute window (17 min).

Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010

* test(relay): pin the same-cap drain window contract at 5 minutes

Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010

* fix(relay): keep the drain wait at lease plus the 5-minute window

The transition wait after a drain was set to the 15-minute migration
lease plus the pace window. Widening the window to 5 minutes without
moving the wait left 12 minutes for a migration that can hold for 15,
so a late-window migration would time the wave out into rollback.

Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010
---
 .../cloud-deploy-relay-production-same-cap-job.yml  | 13 +++++++++----
 .../scripts/relay-same-cap-script-census.test.mjs   |  5 +++--
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml b/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
index abe0f327430..78b1099744e 100644
--- a/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
+++ b/.github/workflows/cloud-deploy-relay-production-same-cap-job.yml
@@ -57,9 +57,14 @@ jobs:
       GATE_OVERRIDE_REASON: ${{ inputs.gate-override-reason }}
       GATE_OVERRIDE_CONFIRMATION: ${{ inputs.gate-override-confirmation }}
       OUTPUT_DIRECTORY: ${{ github.workspace }}/relay-monitor-evidence
-      # ~800 controls over 2 min is ~7 re-dials/s per cell, well under the director's
-      # 5 x 80 in-flight assign cap. A cell on an older image ignores it and drains at once.
-      DRAIN_PACE_WINDOW_MS: '120000'
+      # Each drained host re-dials /v1/assign and is re-placed through the director's
+      # single-slot sticky lane, which clears roughly 8 hosts/s fleet-wide. A 2,100-host
+      # cell over 2 min (~18/s) overran it on 2026-09-23 and browned out assign for every
+      # region for ~5 min. 5 min is the cell's maximum window (DRAIN_PACE_WINDOW_MAX_MS); a
+      # ~2,700-host cell still arrives at ~9/s, so expect a short tail, not zero rejects.
+      # The transition wait below is the 15-min migration lease plus this window. A cell on
+      # an older image drains at once.
+      DRAIN_PACE_WINDOW_MS: '300000'
     steps:
       - name: Require exact reusable-workflow configuration
         working-directory: .
@@ -574,7 +579,7 @@ jobs:
             --unobserved-bound "${EXPECTED_UNOBSERVED_BOUND}" \
             --heartbeat either --admission migration-only --draining required \
             --activity restart-safe --expected-image-digests "${PREDECESSOR_IMAGE_DIGEST}" \
-            --timeout-ms 1020000
+            --timeout-ms 1200000
 
       - id: capacity-auth
         if: ${{ inputs.mode != 'verify' }}
diff --git a/cloud/dev/scripts/relay-same-cap-script-census.test.mjs b/cloud/dev/scripts/relay-same-cap-script-census.test.mjs
index f78b5fa2232..1e023ce0c9d 100644
--- a/cloud/dev/scripts/relay-same-cap-script-census.test.mjs
+++ b/cloud/dev/scripts/relay-same-cap-script-census.test.mjs
@@ -311,9 +311,10 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
   it('paces the drain it sends to the selected cell', () => {
     const drain = workflow.split('--mode drain')[1] ?? ''
     assert.match(drain.split('\n').slice(0, 2).join(' '), /--pace-window-ms "\$\{DRAIN_PACE_WINDOW_MS\}"/)
-    assert.match(workflow, /DRAIN_PACE_WINDOW_MS: '120000'/)
+    // 5 min is the cell's DRAIN_PACE_WINDOW_MAX_MS; a 2,700-host cell at 2 min overruns the director's sticky lane.
+    assert.match(workflow, /DRAIN_PACE_WINDOW_MS: '300000'/)
     // The transition wait has to outlast the pacing window on top of the leases it waits on.
-    assert.match(workflow, /--activity restart-safe[\s\S]*?--timeout-ms 1020000/)
+    assert.match(workflow, /--activity restart-safe[\s\S]*?--timeout-ms 1200000/)
   })
 
   it('passes this cell\'s rehome protocol and pool on every plan validation the job runs', () => {