feat(relay): pace the drain send during a same-cap cell roll (#21284)

* feat(relay): pace the drain send during a same-cap cell roll

A same-cap roll drains a cell with graceMs 0, which sends `drain` to all
~800 controls in one pass. Every desktop re-dials on receipt regardless of
graceMs, so the whole cell reconnects inside a second. On 2026-09-16 that
stampede hit a Cloud SQL stall: attaches timed out, each leaving 10 minutes
of late-arrival debt on connection headroom, and placement answered
relay_capacity_exhausted fleet-wide for ~13 minutes.

Spreading the sends spreads the re-dials. `HostSessionRegistry.drain` takes
an optional pacing window and schedules each session's send evenly across
it; admission is fenced for every session up front, and each host keeps its
own full grace after its own send. /v1/admin/drain accepts `paceWindowMs`
(<= 5 min) and echoes what it applied. The same-cap job asks for 120 s, and
the drain-completion wait grew by the same amount. A cell still on an older
image rejects the field, so the deploy script falls back to an unpaced
drain rather than failing the roll.

* fix(relay): scope the drain fence to the hosts already told

Review of the paced drain found two problems, both from treating "this cell
is draining" as one instant when pacing makes it a window.

Timers: the sends queued by a paced drain were neither cleared when a later
drain superseded them nor unref'd. A SIGTERM mid-window left up to 800 no-op
timers holding the event loop open until systemd escalated to SIGKILL. Drain
timers are now tracked, cleared on the next drain, and unref'd, so a retry
re-arms a session's teardown instead of stacking a second one.

Phones: the client fence read the global draining flag, so every phone was
refused for the whole window even though its own host had not been told yet
and was still serving. The director keeps pointing phones at this cell until
their host moves, so they would have looped for up to two minutes. A session
is now fenced when its drain is sent, not when the drain starts, and the
client paths key off that. New control connections and re-attaches stay
fenced globally: nothing new should land on a cell that is going away.

cloud/dev/scripts/prepare-relay-production-capacity-canary.mjs

@@ -35,6 +35,9 @@ function cellOrigin(cellId) {
 // The same-cap roll covers the Asia cells the US-only capacity rollout never touches.
 const APPROVED_CELL_LISTS = { 'same-cap': SAME_CAP_CELLS }
 
+// Matches the cell's own cap on /v1/admin/drain.
+const MAX_PACE_WINDOW_MS = 5 * 60 * 1_000
+
 export function parseProductionCapacityCellArguments(argv) {
   const values = {}
   for (let index = 0; index < argv.length; index += 2) {
@@ -64,11 +67,22 @@ export function parseProductionCapacityCellArguments(argv) {
   ) {
     throw new Error('production capacity target origin is not exact')
   }
+  const paceWindowMs = values['pace-window-ms'] === undefined
+    ? 0
+    : Number(values['pace-window-ms'])
+  if (
+    !Number.isSafeInteger(paceWindowMs) ||
+    paceWindowMs < 0 ||
+    paceWindowMs > MAX_PACE_WINDOW_MS
+  ) {
+    throw new Error('--pace-window-ms must be an integer between 0 and 300000')
+  }
   return {
     directorOrigin: DIRECTOR_ORIGIN,
     cellOrigin: expectedCellOrigin,
     cellId,
-    mode: values.mode
+    mode: values.mode,
+    paceWindowMs
   }
 }
 
@@ -82,24 +96,39 @@ export async function prepareProductionCapacityCell(config, overrides = {}) {
   const fetchImpl = overrides.fetch ?? fetch
   const token = overrides.token ?? process.env.ORCA_RELAY_ADMIN_ID_TOKEN
   if (!token || token.length > 8_192) throw new Error('admin identity token is unavailable')
-  const postAt = async (origin, path, body) =>
-    await responseJson(
-      await fetchAdminOnceMore(
-        fetchImpl,
-        `${origin}${path}`,
-        {
-          method: 'POST',
-          headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
-          body: JSON.stringify(body)
-        },
-        { wait: overrides.wait }
-      ),
-      path
+  const postRaw = async (origin, path, body) =>
+    await fetchAdminOnceMore(
+      fetchImpl,
+      `${origin}${path}`,
+      {
+        method: 'POST',
+        headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
+        body: JSON.stringify(body)
+      },
+      { wait: overrides.wait }
     )
+  const postAt = async (origin, path, body) =>
+    await responseJson(await postRaw(origin, path, body), path)
   const post = async (path, body) => await postAt(config.directorOrigin, path, body)
   if (config.mode === 'drain') {
+    const paceWindowMs = config.paceWindowMs ?? 0
+    if (paceWindowMs > 0) {
+      const paced = await postRaw(config.cellOrigin, '/v1/admin/drain', {
+        v: 1,
+        graceMs: 0,
+        paceWindowMs
+      })
+      if (paced.ok) {
+        await paced.json().catch(() => ({}))
+        return { changed: false, drained: true, paceWindowMs }
+      }
+      // A cell still on an image without paced drain rejects the unknown field outright.
+      // An unpaced drain is the behaviour that cell already has, so fall back to it.
+      if (paced.status !== 400) throw new Error(`/v1/admin/drain returned ${paced.status}`)
+      await paced.json().catch(() => ({}))
+    }
     await postAt(config.cellOrigin, '/v1/admin/drain', { v: 1, graceMs: 0 })
-    return { changed: false, drained: true }
+    return { changed: false, drained: true, paceWindowMs: 0 }
   }
   const before = await inspectAdmissionSelector(post)
   const state = selectorCellState(before.selector, config.cellId)

cloud/dev/scripts/prepare-relay-production-capacity-canary.test.mjs

@@ -90,7 +90,8 @@ describe('production Relay capacity cell admission', () => {
       directorOrigin: 'https://relay.onorca.dev',
       cellOrigin: 'https://c7.relay.onorca.dev',
       cellId: 'production-gce-c7',
-      mode: 'isolate'
+      mode: 'isolate',
+      paceWindowMs: 0
     })
     assert.throws(() => parseProductionCapacityCellArguments([
       '--director-origin', 'https://relay.onorca.dev',
@@ -125,7 +126,8 @@ describe('production Relay capacity cell admission', () => {
         directorOrigin: 'https://relay.onorca.dev',
         cellOrigin: `https://${hostname}.relay.onorca.dev`,
         cellId,
-        mode: 'isolate'
+        mode: 'isolate',
+        paceWindowMs: 0
       })
     }
     for (const cellId of ['production-gce-c17', 'production-gce-c18', 'production-gce-c30']) {
@@ -168,13 +170,77 @@ describe('production Relay capacity cell admission', () => {
       { ...config, mode: 'drain' },
       { fetch: fake.fetch, token: 'token' }
     )
-    assert.deepEqual(result, { changed: false, drained: true })
+    assert.deepEqual(result, { changed: false, drained: true, paceWindowMs: 0 })
     assert.deepEqual(fake.calls, [{
       path: '/v1/admin/drain',
       body: { v: 1, graceMs: 0 }
     }])
   })
 
+  it('paces the drain send when the roll asks for a window', async () => {
+    const fake = canaryFetch()
+    const result = await prepareProductionCapacityCell(
+      { ...config, mode: 'drain', paceWindowMs: 120_000 },
+      { fetch: fake.fetch, token: 'token' }
+    )
+    assert.deepEqual(result, { changed: false, drained: true, paceWindowMs: 120_000 })
+    assert.deepEqual(fake.calls, [{
+      path: '/v1/admin/drain',
+      body: { v: 1, graceMs: 0, paceWindowMs: 120_000 }
+    }])
+  })
+
+  it('drains unpaced when the cell image rejects the pacing field', async () => {
+    const bodies = []
+    const result = await prepareProductionCapacityCell(
+      { ...config, mode: 'drain', paceWindowMs: 120_000 },
+      {
+        token: 'token',
+        wait: async () => {},
+        fetch: async (url, init) => {
+          assert.equal(new URL(url).pathname, '/v1/admin/drain')
+          const body = JSON.parse(init.body)
+          bodies.push(body)
+          if (body.paceWindowMs !== undefined) return response({ error: 'invalid_request' }, 400)
+          return response({ v: 1, draining: true })
+        }
+      }
+    )
+    assert.deepEqual(result, { changed: false, drained: true, paceWindowMs: 0 })
+    assert.deepEqual(bodies, [
+      { v: 1, graceMs: 0, paceWindowMs: 120_000 },
+      { v: 1, graceMs: 0 }
+    ])
+  })
+
+  it('fails a paced drain that the cell rejects for any other reason', async () => {
+    await assert.rejects(
+      prepareProductionCapacityCell(
+        { ...config, mode: 'drain', paceWindowMs: 120_000 },
+        {
+          token: 'token',
+          wait: async () => {},
+          fetch: async () => response({ error: 'invalid_token' }, 401)
+        }
+      ),
+      /returned 401/
+    )
+  })
+
+  it('refuses a pacing window that is not a bounded integer', () => {
+    const argv = (value) => [
+      '--director-origin', 'https://relay.onorca.dev',
+      '--cell-origin', 'https://c26.relay.onorca.dev',
+      '--cell-id', 'production-gce-c26',
+      '--mode', 'drain',
+      '--pace-window-ms', value
+    ]
+    for (const value of ['-1', '300001', '1.5', 'soon']) {
+      assert.throws(() => parseProductionCapacityCellArguments(argv(value)), /pace-window-ms/)
+    }
+    assert.equal(parseProductionCapacityCellArguments(argv('300000')).paceWindowMs, 300_000)
+  })
+
   it('restores only the selected cell to general admission', async () => {
     const fake = canaryFetch()
     await prepareProductionCapacityCell(
@@ -228,7 +294,7 @@ describe('production Relay capacity cell admission', () => {
       }
     )
     assert.equal(calls, 2)
-    assert.deepEqual(result, { changed: false, drained: true })
+    assert.deepEqual(result, { changed: false, drained: true, paceWindowMs: 0 })
   })
 
   it('fails when both drain attempts return a transient 503', async () => {

cloud/dev/scripts/relay-same-cap-script-census.test.mjs

@@ -159,7 +159,8 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
           directorOrigin: 'https://relay.onorca.dev',
           cellOrigin: `https://${hostname(cellId)}.relay.onorca.dev`,
           cellId,
-          mode
+          mode,
+          paceWindowMs: 0
         })
       }
     }
@@ -192,6 +193,14 @@ describe('same-cap roll scripts accept every same-cap cell', () => {
     }
   })
 
+  it('paces the drain it sends to the selected cell', () => {
+    const drain = workflow.split('--mode drain')[1] ?? ''
+    assert.match(drain.split('\n').slice(0, 2).join(' '), /--pace-window-ms "\$\{DRAIN_PACE_WINDOW_MS\}"/)
+    assert.match(workflow, /DRAIN_PACE_WINDOW_MS: '120000'/)
+    // The transition wait has to outlast the pacing window on top of the leases it waits on.
+    assert.match(workflow, /--activity restart-safe[\s\S]*?--timeout-ms 1020000/)
+  })
+
   it('passes this cell\'s rehome protocol and pool on every plan validation the job runs', () => {
     const invocations = workflow.split('validate-relay-capacity-plan.mjs').slice(1)
     assert.equal(invocations.length, 2)