From 52110982ca96f3f5503d02a0414923745c4ac6fc Mon Sep 17 00:00:00 2001 From: Jinwoo Hong <73622457+Jinwoo-H@users.noreply.github.com> Date: Wed, 30 Sep 2026 02:43:28 -0400 Subject: [PATCH] fix(push): run twelve delivery drains instead of four (#24038) Four drains, each holding one provider round trip of ~100 ms plus its database statements, capped delivery near 30/s. Production inflow reached 35/s on 2026-09-30, so the backlog aged past the five-minute TTL and notifications expired. Twelve drains lift the ceiling to roughly 90/s; the pool is now six per instance, so the extra drains queue on connections instead of starving the request path. Claude-Session: ced32ebb-7155-4413-adad-1eccd14c2010 --- cloud/apps/push/src/durable-push-worker.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cloud/apps/push/src/durable-push-worker.ts b/cloud/apps/push/src/durable-push-worker.ts index 64020d3846e..2a1a22f7a33 100644 --- a/cloud/apps/push/src/durable-push-worker.ts +++ b/cloud/apps/push/src/durable-push-worker.ts @@ -2,6 +2,10 @@ import { buildPushDelivery } from './push-delivery-message.js' import type { PushDispatcher } from './push-dispatcher.js' import type { DurablePushStore } from './durable-push-store.js' +// Four drains capped delivery near 30/s at ~120 ms per item, below the 2026-09 inflow; each drain +// holds one delivery in flight, so this is the worker's concurrency, not its DB draw. +const WORKER_DRAINS = 12 + export class DurablePushWorker { private timer?: NodeJS.Timeout private running: Promise | null = null @@ -29,7 +33,7 @@ export class DurablePushWorker { return } if (this.stopped) return - const pending = Promise.allSettled(Array.from({ length: 4 }, () => this.drain())).then( + const pending = Promise.allSettled(Array.from({ length: WORKER_DRAINS }, () => this.drain())).then( (results) => { const failure = results.find((result) => result.status === 'rejected') if (failure?.status === 'rejected') throw failure.reason