From f5d974bfd2dd4d0d939a9ea50cf7894b930c6cd7 Mon Sep 17 00:00:00 2001 From: Matthew Meszaros Date: Thu, 5 Nov 2026 13:30:39 +0100 Subject: [PATCH 1/6] feat: run consumer worker events on per-mailbox queues that share a lane limit so a slow mailbox holds only its own lane, and size that limit from the consumer's database pool unless CONSUMER_EVENT_CONCURRENCY is set --- cmd/consumer/main.go | 1 + .../docs/development/configuration.mdx | 2 +- docs/content/docs/development/events.mdx | 2 +- internal/app/consumer/event_lanes.go | 12 ++- internal/app/consumer/event_lanes_test.go | 17 +++- internal/app/consumer/service.go | 4 +- .../infrastructure/eventbus/nats_lanes.go | 99 +++++++++++++------ .../eventbus/nats_lanes_test.go | 52 +++++++++- 8 files changed, 144 insertions(+), 45 deletions(-) diff --git a/cmd/consumer/main.go b/cmd/consumer/main.go index d3cdf4ba5..0f780450c 100644 --- a/cmd/consumer/main.go +++ b/cmd/consumer/main.go @@ -553,6 +553,7 @@ func main() { jobsService := &jobs.JobsService{ CampaignIsolationEnabled: config.CampaignSendIsolationEnabled(), + DBConns: primaryDB.Pool.Config().MaxConns, Bus: consumerBus, Codec: consumerCodec, UniboxRepository: uniboxRepo, diff --git a/docs/content/docs/development/configuration.mdx b/docs/content/docs/development/configuration.mdx index 12ae769a4..ecfab10c1 100644 --- a/docs/content/docs/development/configuration.mdx +++ b/docs/content/docs/development/configuration.mdx @@ -388,7 +388,7 @@ The attachment download proxy works with filesystem and S3-compatible stores and | `CODEC_PROVIDER` | `json` or `avro` | `json` under compose, `avro` for a bare binary | yes | | `CODEC_READ_BOTH` | Migration setting for Go services: read JSON and Confluent-framed Avro while writing only `CODEC_PROVIDER`. Requires Kafka builds and the old Schema Registry until all retained Avro has drained | `false` | yes | | `EVENTBUS_HANDLER_TIMEOUT` | How long one handler may take before the delivery is abandoned. A subscription that needs longer carries its own budget, never shorter than this one: a worker's command subscription runs under 195s, because a send is given 120s (the SMTP client's 90s conversation plus the checks before it) and then records its result. On NATS the ack wait is the subscription's budget plus 5s | `30s` | yes | -| `CONSUMER_EVENT_CONCURRENCY` | Consumer lanes for worker events, from `1` to `64`, on both Kafka and NATS. Events are grouped by the mailbox the worker published them for, so each mailbox's events stay in order while different mailboxes are handled at once. Busy lanes hold database connections at the same time, so keep it well below the consumer's `DB_MAX_CONNS`. Invalid values use the default | `8` | yes | +| `CONSUMER_EVENT_CONCURRENCY` | How many mailboxes' worker events one consumer handles at once, from `1` to `64`, on both Kafka and NATS. Each mailbox's events stay in order, and on NATS a slow mailbox holds only its own lane. Unset, it follows the consumer's database pool (`DB_MAX_CONNS`), so raising the pool raises it too. Invalid values use the default | one per pooled database connection, at most `64` | yes | | `PUBSUB_ENABLED` | `false` uses the Redis bridge for realtime fanout, `true` uses Google Pub/Sub | `false` | yes | | `GCP_PROJECT_ID` | Project when `PUBSUB_ENABLED=true` | unset | yes | diff --git a/docs/content/docs/development/events.mdx b/docs/content/docs/development/events.mdx index 60c0de5f7..abaf3f03d 100644 --- a/docs/content/docs/development/events.mdx +++ b/docs/content/docs/development/events.mdx @@ -89,7 +89,7 @@ Workers publish the same envelope shape back on `jobs.worker-events`: Types: `NEW_EMAIL`, `INBOUND_BOUNCE`, `REMOVE_EMAIL`, `FLAGS_ADD`, `FLAGS_REMOVE`, `UPDATE_EMAIL`, `UPDATE_FOLDER`, `UPDATE_MAILBOX`, `DELETE_MAILBOX`, `TOKEN_UPDATE`, `HISTORY_ID_UPDATE`, `GRAPH_DELTA_UPDATE`, `SYNC_STATE`, `PUSH_STATE`, `EMAIL_SENT`, `EMAIL_FAILED`, `EMAIL_AUTH_ERROR`, `EMAIL_DISABLED`, `EMAIL_RATE_LIMITED`, `EMAIL_SERVER_ERROR`, `WORKER_HEALTH`. -The consumer registers one handler per type (`internal/app/consumer/events.go`); unregistered types are logged and acknowledged rather than redelivered. It handles them on ordered per-mailbox lanes (`CONSUMER_EVENT_CONCURRENCY`), keyed by the mailbox id each worker publishes under, so one mailbox's events stay in order while a burst from another mailbox does not queue behind them. +The consumer registers one handler per type (`internal/app/consumer/events.go`); unregistered types are logged and acknowledged rather than redelivered. It handles them on ordered per-mailbox lanes (`CONSUMER_EVENT_CONCURRENCY`), keyed by the mailbox id each worker publishes under, so one mailbox's events stay in order while a burst or a slow handler on another mailbox does not queue behind them. Unset, the number of mailboxes handled at once follows the consumer's database pool. Every `SEND_EMAIL` is answered with exactly one per-task result: `EMAIL_SENT` (the provider accepted the message; the consumer records the wire Message-ID on the task) or `EMAIL_FAILED` (a `SendEmailResult` carrying the error code and message). The control plane stamps a campaign step sent when it hands the send to the worker, so `EMAIL_FAILED` is what walks that back: the consumer marks the task failed, clears the step's `sent_at` and counts the attempt on `campaign_contact_progress`, gives the send back to the campaign's daily counters, writes the failure to the campaign activity log, and reopens a campaign that completed while the send was in flight. After `CampaignSendMaxAttempts` (5) the lead is marked failed and routing drops it; a `RECIPIENT_REJECTED` result (refused at RCPT) skips the retries and is ingested as a bounce instead. Account-level conditions (`EMAIL_AUTH_ERROR`, `EMAIL_DISABLED`, `EMAIL_RATE_LIMITED`, `EMAIL_SERVER_ERROR`) are raised in addition to, never instead of, the per-task result; they carry an `EmailErrorEvent` and act on the mailbox. A worker that does not hold the mailbox yet leaves the send for a few redeliveries before reporting it failed, and the backend never publishes a send to a worker that is not heartbeating. diff --git a/internal/app/consumer/event_lanes.go b/internal/app/consumer/event_lanes.go index 7789aff63..b45a39e18 100644 --- a/internal/app/consumer/event_lanes.go +++ b/internal/app/consumer/event_lanes.go @@ -8,12 +8,16 @@ import ( "github.com/warmbly/warmbly/internal/infrastructure/eventbus" ) -// WorkerEventConcurrency is how many mailboxes' worker events one consumer handles at once. -func WorkerEventConcurrency() int { - if n, err := strconv.Atoi(os.Getenv("CONSUMER_EVENT_CONCURRENCY")); err == nil && n >= 1 && n <= 64 { +// maxEventLanes is the most mailboxes one consumer handles at once. +const maxEventLanes = 64 + +// WorkerEventConcurrency is how many mailboxes' worker events one consumer handles at once: +// CONSUMER_EVENT_CONCURRENCY when set, otherwise one per pooled database connection. +func WorkerEventConcurrency(poolConns int32) int { + if n, err := strconv.Atoi(os.Getenv("CONSUMER_EVENT_CONCURRENCY")); err == nil && n >= 1 && n <= maxEventLanes { return n } - return 8 + return min(max(int(poolConns), 1), maxEventLanes) } // workerEventKey keeps one mailbox's events in order: workers publish them under the mailbox id, diff --git a/internal/app/consumer/event_lanes_test.go b/internal/app/consumer/event_lanes_test.go index e222164ea..0c244431c 100644 --- a/internal/app/consumer/event_lanes_test.go +++ b/internal/app/consumer/event_lanes_test.go @@ -7,11 +7,18 @@ import ( "github.com/warmbly/warmbly/internal/infrastructure/eventbus" ) -func TestWorkerEventConcurrencyDefaultsAndBounds(t *testing.T) { - for raw, want := range map[string]int{"": 8, "1": 1, "64": 64, "0": 8, "65": 8, "many": 8} { - t.Setenv("CONSUMER_EVENT_CONCURRENCY", raw) - if got := WorkerEventConcurrency(); got != want { - t.Errorf("CONSUMER_EVENT_CONCURRENCY=%q: got %d lanes, want %d", raw, got, want) +func TestWorkerEventConcurrencyFollowsThePoolUnlessSet(t *testing.T) { + for _, c := range []struct { + env string + pool int32 + want int + }{ + {"", 25, 25}, {"", 4, 4}, {"", 0, 1}, {"", 200, 64}, + {"1", 25, 1}, {"64", 4, 64}, {"0", 25, 25}, {"65", 25, 25}, {"many", 10, 10}, + } { + t.Setenv("CONSUMER_EVENT_CONCURRENCY", c.env) + if got := WorkerEventConcurrency(c.pool); got != c.want { + t.Errorf("CONSUMER_EVENT_CONCURRENCY=%q pool=%d: got %d lanes, want %d", c.env, c.pool, got, c.want) } } } diff --git a/internal/app/consumer/service.go b/internal/app/consumer/service.go index c5c3b2ae2..c087b0f0e 100644 --- a/internal/app/consumer/service.go +++ b/internal/app/consumer/service.go @@ -43,6 +43,8 @@ type CloudExecution interface { type JobsService struct { CampaignIsolationEnabled bool + // DBConns is the database pool the event handlers share; it sizes how many mailboxes run at once. + DBConns int32 // Bus delivers the jobs.worker-events stream (Kafka or NATS). Bus eventbus.EventBus // Codec decodes bus payloads (jobs.worker-events); it must match the @@ -151,7 +153,7 @@ func (s *JobsService) Start(ctx context.Context) { go s.runStoredSendResults(ctx) go s.runArrivalOutbox(ctx) - lanes := WorkerEventConcurrency() + lanes := WorkerEventConcurrency(s.DBConns) log.Info().Int("lanes", lanes).Msg("consumer: worker-events ordered mailbox lanes") if err := eventbus.SubscribeKeyed(ctx, s.Bus, []string{kafka.TopicWorkerEvents}, "consumer-group", lanes, workerEventKey, s.receive); err != nil { log.Error().Err(err).Msg("consumer: worker-events subscription ended") diff --git a/internal/infrastructure/eventbus/nats_lanes.go b/internal/infrastructure/eventbus/nats_lanes.go index ec3ef8500..4cf531543 100644 --- a/internal/infrastructure/eventbus/nats_lanes.go +++ b/internal/infrastructure/eventbus/nats_lanes.go @@ -1,7 +1,6 @@ package eventbus import ( - "hash/fnv" "sync" "time" @@ -9,13 +8,16 @@ import ( "github.com/rs/zerolog/log" ) +// natsLaneBuffer is how many deliveries each lane may hold, running or queued, before fetching pauses. const natsLaneBuffer = 32 -// natsLanes runs a keyed subscription on ordered lanes. JetStream times a -// delivery from when it was handed out, so every message the lanes hold, -// queued or running, is kept alive with an in-progress heartbeat. +// natsLanes runs a keyed subscription: each key's messages in order, up to +// lanes keys at once, so a slow key holds one lane and never another key. +// JetStream times a delivery from when it was handed out, so every message +// held, queued or running, is kept alive with an in-progress heartbeat. type natsLanes struct { - queues []chan jetstream.Msg + slots chan struct{} + room chan struct{} deliver func(jetstream.Msg) halted chan struct{} haltOnce sync.Once @@ -24,54 +26,79 @@ type natsLanes struct { mu sync.Mutex held map[jetstream.Msg]struct{} + keys map[string][]jetstream.Msg } func newNATSLanes(lanes int, ackWait time.Duration, deliver func(jetstream.Msg)) *natsLanes { p := &natsLanes{ - queues: make([]chan jetstream.Msg, lanes), + slots: make(chan struct{}, lanes), + room: make(chan struct{}, lanes*natsLaneBuffer), deliver: deliver, halted: make(chan struct{}), done: make(chan struct{}), held: map[jetstream.Msg]struct{}{}, - } - for i := range p.queues { - p.queues[i] = make(chan jetstream.Msg, natsLaneBuffer) - p.wg.Add(1) - go p.run(p.queues[i]) + keys: map[string][]jetstream.Msg{}, } go p.heartbeat(max(ackWait/3, 100*time.Millisecond)) return p } -func natsLane(key string, lanes int) int { - h := fnv.New32a() - _, _ = h.Write([]byte(key)) - return int(h.Sum32() % uint32(lanes)) -} - -// dispatch queues m behind every earlier message with the same key. +// dispatch queues m behind every earlier message with the same key, waiting +// for room when the lanes already hold their share. func (p *natsLanes) dispatch(m jetstream.Msg, key string) { + select { + case p.room <- struct{}{}: + case <-p.halted: + nak(m) + return + } p.hold(m) if p.isHalted() { p.abandon(m) return } - select { - case p.queues[natsLane(key, len(p.queues))] <- m: - case <-p.halted: - p.abandon(m) + p.mu.Lock() + if queue, running := p.keys[key]; running { + p.keys[key] = append(queue, m) + p.mu.Unlock() + return } + p.keys[key] = nil + p.mu.Unlock() + p.wg.Add(1) + go p.run(key, m) } -func (p *natsLanes) run(queue <-chan jetstream.Msg) { +// run handles key's messages one after another, taking a lane for each. +func (p *natsLanes) run(key string, m jetstream.Msg) { defer p.wg.Done() - for m := range queue { + for { + acquired := false + select { + case p.slots <- struct{}{}: + acquired = true + case <-p.halted: + } if p.isHalted() { - p.abandon(m) - continue + if acquired { + <-p.slots + } + p.abandonKey(key, m) + return } p.deliver(m) + <-p.slots p.release(m) + + p.mu.Lock() + queue := p.keys[key] + if len(queue) == 0 { + delete(p.keys, key) + p.mu.Unlock() + return + } + m, p.keys[key] = queue[0], queue[1:] + p.mu.Unlock() } } @@ -85,12 +112,29 @@ func (p *natsLanes) release(m jetstream.Msg) { p.mu.Lock() delete(p.held, m) p.mu.Unlock() + <-p.room } // abandon hands back a message that will not be handled here, so it is // redelivered without waiting out its ack wait. func (p *natsLanes) abandon(m jetstream.Msg) { p.release(m) + nak(m) +} + +// abandonKey hands back m and everything queued behind it. +func (p *natsLanes) abandonKey(key string, m jetstream.Msg) { + p.mu.Lock() + queue := p.keys[key] + delete(p.keys, key) + p.mu.Unlock() + p.abandon(m) + for _, q := range queue { + p.abandon(q) + } +} + +func nak(m jetstream.Msg) { if err := m.Nak(); err != nil { log.Warn().Err(err).Msg("eventbus nats nak failed") } @@ -136,9 +180,6 @@ func (p *natsLanes) halt() { // stop waits for the running handlers once no more messages can be dispatched. func (p *natsLanes) stop() { - for _, q := range p.queues { - close(q) - } p.wg.Wait() close(p.done) } diff --git a/internal/infrastructure/eventbus/nats_lanes_test.go b/internal/infrastructure/eventbus/nats_lanes_test.go index 2c58f4914..1092517e0 100644 --- a/internal/infrastructure/eventbus/nats_lanes_test.go +++ b/internal/infrastructure/eventbus/nats_lanes_test.go @@ -116,10 +116,6 @@ func TestNATSBus_KeyedLanesKeepKeyOrderAndRunKeysConcurrently(t *testing.T) { } return nil }) - // a and b must not share a lane for the concurrency half to mean anything. - if natsLane("a", 4) == natsLane("b", 4) { - t.Fatal("test keys hash to the same lane") - } waitForConsumer(t, bus, bus.durable("lanes", []string{topic})) for _, p := range []string{"a1", "a2", "a3", "b1"} { if err := bus.Publish(t.Context(), topic, "", []byte(p)); err != nil { @@ -141,6 +137,54 @@ func TestNATSBus_KeyedLanesKeepKeyOrderAndRunKeysConcurrently(t *testing.T) { } } +// A slow key takes one lane and no more: with two lanes, every other key keeps +// moving through the second while the first is held. +func TestNATSBus_KeyedLanesNeverQueueOneKeyBehindAnother(t *testing.T) { + bus := newTestNATSBus(t) + topic := "w.blocking" + + releaseA := make(chan struct{}) + handled := make(chan string, 8) + subscribeInBackground(t, t.Context(), bus, topic, "blocking", 2, payloadKey, func(_ context.Context, msg Message) error { + p := string(msg.Payload) + if p == "a1" { + select { + case <-releaseA: + case <-time.After(5 * time.Second): + } + } + handled <- p + return nil + }) + waitForConsumer(t, bus, bus.durable("blocking", []string{topic})) + for _, p := range []string{"a1", "b1", "c1", "d1", "e1"} { + if err := bus.Publish(t.Context(), topic, "", []byte(p)); err != nil { + t.Fatalf("publish %s: %v", p, err) + } + } + others := map[string]bool{} + for len(others) < 4 { + select { + case p := <-handled: + if p == "a1" { + t.Fatalf("a1 finished before the other keys: %v", others) + } + others[p] = true + case <-time.After(5 * time.Second): + t.Fatalf("keys waited behind a held one: handled %v", others) + } + } + close(releaseA) + select { + case p := <-handled: + if p != "a1" { + t.Fatalf("handled %s, want a1", p) + } + case <-time.After(5 * time.Second): + t.Fatal("a1 never finished") + } +} + // A message queued behind a slow one on its lane outlives its ack wait; the // in-progress heartbeat must keep JetStream from handing it out again. func TestNATSBus_KeyedLanesKeepQueuedMessagesAlive(t *testing.T) { From 0ca3ed81b16161674adc4bc656a1682998650a5b Mon Sep 17 00:00:00 2001 From: Matthew Meszaros Date: Thu, 5 Nov 2026 14:01:24 +0100 Subject: [PATCH 2/6] feat: keep consumer deliveries that wait for lane room on the in-progress heartbeat and state per-mailbox lane isolation only for NATS in the events docs --- docs/content/docs/development/events.mdx | 2 +- .../infrastructure/eventbus/nats_lanes.go | 10 +++- .../eventbus/nats_lanes_test.go | 52 +++++++++++++++++++ 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/docs/content/docs/development/events.mdx b/docs/content/docs/development/events.mdx index abaf3f03d..0cbe7d562 100644 --- a/docs/content/docs/development/events.mdx +++ b/docs/content/docs/development/events.mdx @@ -89,7 +89,7 @@ Workers publish the same envelope shape back on `jobs.worker-events`: Types: `NEW_EMAIL`, `INBOUND_BOUNCE`, `REMOVE_EMAIL`, `FLAGS_ADD`, `FLAGS_REMOVE`, `UPDATE_EMAIL`, `UPDATE_FOLDER`, `UPDATE_MAILBOX`, `DELETE_MAILBOX`, `TOKEN_UPDATE`, `HISTORY_ID_UPDATE`, `GRAPH_DELTA_UPDATE`, `SYNC_STATE`, `PUSH_STATE`, `EMAIL_SENT`, `EMAIL_FAILED`, `EMAIL_AUTH_ERROR`, `EMAIL_DISABLED`, `EMAIL_RATE_LIMITED`, `EMAIL_SERVER_ERROR`, `WORKER_HEALTH`. -The consumer registers one handler per type (`internal/app/consumer/events.go`); unregistered types are logged and acknowledged rather than redelivered. It handles them on ordered per-mailbox lanes (`CONSUMER_EVENT_CONCURRENCY`), keyed by the mailbox id each worker publishes under, so one mailbox's events stay in order while a burst or a slow handler on another mailbox does not queue behind them. Unset, the number of mailboxes handled at once follows the consumer's database pool. +The consumer registers one handler per type (`internal/app/consumer/events.go`); unregistered types are logged and acknowledged rather than redelivered. It handles them on ordered per-mailbox lanes (`CONSUMER_EVENT_CONCURRENCY`), keyed by the mailbox id each worker publishes under, so one mailbox's events stay in order while a burst from another mailbox does not queue behind them. On NATS a slow handler holds only its own mailbox's lane; on Kafka a mailbox shares a lane with the others its key hashes to. Unset, the number of mailboxes handled at once follows the consumer's database pool. Every `SEND_EMAIL` is answered with exactly one per-task result: `EMAIL_SENT` (the provider accepted the message; the consumer records the wire Message-ID on the task) or `EMAIL_FAILED` (a `SendEmailResult` carrying the error code and message). The control plane stamps a campaign step sent when it hands the send to the worker, so `EMAIL_FAILED` is what walks that back: the consumer marks the task failed, clears the step's `sent_at` and counts the attempt on `campaign_contact_progress`, gives the send back to the campaign's daily counters, writes the failure to the campaign activity log, and reopens a campaign that completed while the send was in flight. After `CampaignSendMaxAttempts` (5) the lead is marked failed and routing drops it; a `RECIPIENT_REJECTED` result (refused at RCPT) skips the retries and is ingested as a bounce instead. Account-level conditions (`EMAIL_AUTH_ERROR`, `EMAIL_DISABLED`, `EMAIL_RATE_LIMITED`, `EMAIL_SERVER_ERROR`) are raised in addition to, never instead of, the per-task result; they carry an `EmailErrorEvent` and act on the mailbox. A worker that does not hold the mailbox yet leaves the send for a few redeliveries before reporting it failed, and the backend never publishes a send to a worker that is not heartbeating. diff --git a/internal/infrastructure/eventbus/nats_lanes.go b/internal/infrastructure/eventbus/nats_lanes.go index 4cf531543..ffe99756f 100644 --- a/internal/infrastructure/eventbus/nats_lanes.go +++ b/internal/infrastructure/eventbus/nats_lanes.go @@ -46,13 +46,15 @@ func newNATSLanes(lanes int, ackWait time.Duration, deliver func(jetstream.Msg)) // dispatch queues m behind every earlier message with the same key, waiting // for room when the lanes already hold their share. func (p *natsLanes) dispatch(m jetstream.Msg, key string) { + // Held first, so the heartbeat keeps it alive while it waits for room. + p.hold(m) select { case p.room <- struct{}{}: case <-p.halted: + p.unhold(m) nak(m) return } - p.hold(m) if p.isHalted() { p.abandon(m) return @@ -109,10 +111,14 @@ func (p *natsLanes) hold(m jetstream.Msg) { } func (p *natsLanes) release(m jetstream.Msg) { + p.unhold(m) + <-p.room +} + +func (p *natsLanes) unhold(m jetstream.Msg) { p.mu.Lock() delete(p.held, m) p.mu.Unlock() - <-p.room } // abandon hands back a message that will not be handled here, so it is diff --git a/internal/infrastructure/eventbus/nats_lanes_test.go b/internal/infrastructure/eventbus/nats_lanes_test.go index 1092517e0..bd3df84e9 100644 --- a/internal/infrastructure/eventbus/nats_lanes_test.go +++ b/internal/infrastructure/eventbus/nats_lanes_test.go @@ -293,3 +293,55 @@ func TestNATSBus_KeyedLanesKeepSaturatedMessagesAlive(t *testing.T) { t.Errorf("handled %d messages, want %d", len(order), messages) } } + +// A delivery waiting for room behind a full lane is heartbeated too, so a slow +// handler ahead of it does not let JetStream hand it out a second time. +func TestNATSBus_KeyedLanesKeepMessagesWaitingForRoomAlive(t *testing.T) { + grace := natsAckGrace + natsAckGrace = 200 * time.Millisecond + t.Cleanup(func() { natsAckGrace = grace }) + t.Setenv("EVENTBUS_HANDLER_TIMEOUT", "300ms") + + bus := newTestNATSBus(t) + topic := "w.room" + // Two lanes, since one runs without the pool; one key fills both lanes' room. + const messages = 2*natsLaneBuffer + 4 + var mu sync.Mutex + seen := map[string][]int{} + subscribeInBackground(t, t.Context(), bus, topic, "room", 2, payloadKey, func(_ context.Context, msg Message) error { + p := string(msg.Payload) + mu.Lock() + seen[p] = append(seen[p], msg.Attempt) + mu.Unlock() + if p == "a00" { + // Well past the 500ms ack wait while the lane's room is full. + time.Sleep(1500 * time.Millisecond) + } + return nil + }) + ackWait := waitForConsumer(t, bus, bus.durable("room", []string{topic})) + for i := 0; i < messages; i++ { + if err := bus.Publish(t.Context(), topic, "a", fmt.Appendf(nil, "a%02d", i)); err != nil { + t.Fatalf("publish %d: %v", i, err) + } + } + deadline := time.Now().Add(10 * time.Second) + for { + mu.Lock() + n := len(seen) + mu.Unlock() + if n == messages || time.Now().After(deadline) { + break + } + time.Sleep(50 * time.Millisecond) + } + time.Sleep(2 * ackWait) + mu.Lock() + defer mu.Unlock() + for i := 0; i < messages; i++ { + p := fmt.Sprintf("a%02d", i) + if got := seen[p]; len(got) != 1 || got[0] != 1 { + t.Errorf("%s handled with attempts %v, want exactly one first delivery", p, got) + } + } +} From 36d59d66b82e1e7b9cb65d481cec01d59a31f046 Mon Sep 17 00:00:00 2001 From: Matthew Meszaros Date: Thu, 5 Nov 2026 14:18:21 +0100 Subject: [PATCH 3/6] feat: report distinct, total and redelivered handlings when the saturated NATS lane test times out so a slow CI run shows whether redeliveries caused it --- internal/infrastructure/eventbus/nats_lanes_test.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/internal/infrastructure/eventbus/nats_lanes_test.go b/internal/infrastructure/eventbus/nats_lanes_test.go index bd3df84e9..58ffc93cd 100644 --- a/internal/infrastructure/eventbus/nats_lanes_test.go +++ b/internal/infrastructure/eventbus/nats_lanes_test.go @@ -274,7 +274,14 @@ func TestNATSBus_KeyedLanesKeepSaturatedMessagesAlive(t *testing.T) { select { case <-allHandled: case <-time.After(20 * time.Second): - t.Fatal("not every saturated-lane message was handled") + mu.Lock() + redelivered := 0 + for _, attempts := range seen { + redelivered += len(attempts) - 1 + } + handled, distinct := len(order), len(seen) + mu.Unlock() + t.Fatalf("not every saturated-lane message was handled: %d distinct of %d, %d handlings, %d redeliveries", distinct, messages, handled, redelivered) } // Give already-prefetched redeliveries time to reach the handler too. time.Sleep(2 * ackWait) From 38c085d9dd24205d801afcab17f5eeaa3fc36fcc Mon Sep 17 00:00:00 2001 From: Matthew Meszaros Date: Thu, 5 Nov 2026 17:48:08 +0100 Subject: [PATCH 4/6] feat: pull keyed NATS subscriptions with a fetch loop that asks for only as many messages as the mailbox lanes have room for, so the next pull always follows the last one and a reserved delivery is dispatched without blocking --- internal/infrastructure/eventbus/nats.go | 27 +---- .../infrastructure/eventbus/nats_lanes.go | 109 ++++++++++++++++-- 2 files changed, 106 insertions(+), 30 deletions(-) diff --git a/internal/infrastructure/eventbus/nats.go b/internal/infrastructure/eventbus/nats.go index eda416e86..57ec029de 100644 --- a/internal/infrastructure/eventbus/nats.go +++ b/internal/infrastructure/eventbus/nats.go @@ -346,30 +346,13 @@ func (b *NATSBus) SubscribeKeyed(ctx context.Context, topics []string, group str } var pool *natsLanes - callback := deliver - var opts []jetstream.PullConsumeOpt + var cc jetstream.ConsumeContext if lanes > 1 { pool = newNATSLanes(lanes, ackWait, deliver) - callback = func(m jetstream.Msg) { - laneKey, err := resolveKey(ctx, key, b.envelope(m)) - if err != nil { - log.Error().Err(err).Str("subject", m.Subject()).Msg("eventbus nats key resolution failed") - if nakErr := m.NakWithDelay(time.Second); nakErr != nil { - log.Warn().Err(nakErr).Msg("eventbus nats nak failed") - } - return - } - pool.dispatch(m, laneKey) - } - // Fetch again only after dispatch tracks this delivery, even when a lane is full. - opts = append(opts, jetstream.PullMaxMessages(1)) - } - - cc, err := cons.Consume(callback, opts...) - if err != nil { - if pool != nil { - pool.stop() - } + cc = pool.start(ctx, cons, func(m jetstream.Msg) (string, error) { + return resolveKey(ctx, key, b.envelope(m)) + }) + } else if cc, err = cons.Consume(deliver); err != nil { return fmt.Errorf("eventbus nats: consume: %w", err) } diff --git a/internal/infrastructure/eventbus/nats_lanes.go b/internal/infrastructure/eventbus/nats_lanes.go index ffe99756f..90f4c40c7 100644 --- a/internal/infrastructure/eventbus/nats_lanes.go +++ b/internal/infrastructure/eventbus/nats_lanes.go @@ -1,6 +1,8 @@ package eventbus import ( + "context" + "errors" "sync" "time" @@ -11,6 +13,9 @@ import ( // natsLaneBuffer is how many deliveries each lane may hold, running or queued, before fetching pauses. const natsLaneBuffer = 32 +// natsFetchWait bounds one pull request, so a stop is seen promptly while nothing arrives. +const natsFetchWait = 5 * time.Second + // natsLanes runs a keyed subscription: each key's messages in order, up to // lanes keys at once, so a slow key holds one lane and never another key. // JetStream times a delivery from when it was handed out, so every message @@ -43,18 +48,106 @@ func newNATSLanes(lanes int, ackWait time.Duration, deliver func(jetstream.Msg)) return p } -// dispatch queues m behind every earlier message with the same key, waiting -// for room when the lanes already hold their share. -func (p *natsLanes) dispatch(m jetstream.Msg, key string) { - // Held first, so the heartbeat keeps it alive while it waits for room. - p.hold(m) +// natsPull stops a lanes' pull loop, in the shape Consume hands back. +type natsPull struct { + cancel context.CancelFunc + closed chan struct{} +} + +func (h *natsPull) Stop() { h.cancel() } +func (h *natsPull) Drain() { h.cancel() } +func (h *natsPull) Closed() <-chan struct{} { return h.closed } + +// start runs pull until ctx ends or the returned handle is stopped. +func (p *natsLanes) start(ctx context.Context, cons jetstream.Consumer, key func(jetstream.Msg) (string, error)) jetstream.ConsumeContext { + ctx, cancel := context.WithCancel(ctx) + h := &natsPull{cancel: cancel, closed: make(chan struct{})} + go func() { + defer close(h.closed) + p.pull(ctx, cons, key) + }() + return h +} + +// pull fetches only as many messages as the lanes have room for and +// dispatches each by key until ctx ends or the lanes halt. Every fetch is +// asked for after the previous one ends, so none is skipped and nothing is +// fetched that the lanes could not hold and heartbeat at once. +func (p *natsLanes) pull(ctx context.Context, cons jetstream.Consumer, key func(jetstream.Msg) (string, error)) { + for ctx.Err() == nil { + n := p.reserve(cap(p.slots)) + if n == 0 { + return + } + p.unreserve(n - p.fetch(ctx, cons, n, key)) + } +} + +// fetch dispatches up to n messages and reports how many it dispatched. +func (p *natsLanes) fetch(ctx context.Context, cons jetstream.Consumer, n int, key func(jetstream.Msg) (string, error)) int { + fctx, cancel := context.WithTimeout(ctx, natsFetchWait) + defer cancel() + batch, err := cons.Fetch(n, jetstream.FetchContext(fctx)) + if err != nil { + fetchFailed(ctx, err) + return 0 + } + dispatched := 0 + for m := range batch.Messages() { + k, err := key(m) + if err != nil { + log.Error().Err(err).Str("subject", m.Subject()).Msg("eventbus nats key resolution failed") + if nakErr := m.NakWithDelay(time.Second); nakErr != nil { + log.Warn().Err(nakErr).Msg("eventbus nats nak failed") + } + continue + } + p.dispatch(m, k) + dispatched++ + } + if err := batch.Error(); err != nil && !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, context.Canceled) { + fetchFailed(ctx, err) + } + return dispatched +} + +// fetchFailed logs a failed pull and pauses briefly, so an unreachable server is not hot-looped. +func fetchFailed(ctx context.Context, err error) { + log.Warn().Err(err).Msg("eventbus nats fetch failed") + select { + case <-ctx.Done(): + case <-time.After(time.Second): + } +} + +// reserve waits for room for one more delivery and takes up to limit, or none once halted. +func (p *natsLanes) reserve(limit int) int { select { case p.room <- struct{}{}: case <-p.halted: - p.unhold(m) - nak(m) - return + return 0 } + n := 1 + for n < limit { + select { + case p.room <- struct{}{}: + n++ + default: + return n + } + } + return n +} + +func (p *natsLanes) unreserve(n int) { + for range n { + <-p.room + } +} + +// dispatch queues m, for which room is reserved, behind every earlier message with the same key. +func (p *natsLanes) dispatch(m jetstream.Msg, key string) { + p.hold(m) if p.isHalted() { p.abandon(m) return From 21dcfbbb0426a0150ede013af203fec875da7094 Mon Sep 17 00:00:00 2001 From: Matthew Meszaros Date: Thu, 5 Nov 2026 17:57:29 +0100 Subject: [PATCH 5/6] feat: end a keyed NATS pull's wait for lane room when the bus closes it, so closing never waits behind a slow mailbox handler, with a regression test that closes the bus while every lane is held --- .../infrastructure/eventbus/nats_lanes.go | 8 ++-- .../eventbus/nats_lanes_test.go | 38 +++++++++++++++++++ 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/internal/infrastructure/eventbus/nats_lanes.go b/internal/infrastructure/eventbus/nats_lanes.go index 90f4c40c7..356b27271 100644 --- a/internal/infrastructure/eventbus/nats_lanes.go +++ b/internal/infrastructure/eventbus/nats_lanes.go @@ -75,7 +75,7 @@ func (p *natsLanes) start(ctx context.Context, cons jetstream.Consumer, key func // fetched that the lanes could not hold and heartbeat at once. func (p *natsLanes) pull(ctx context.Context, cons jetstream.Consumer, key func(jetstream.Msg) (string, error)) { for ctx.Err() == nil { - n := p.reserve(cap(p.slots)) + n := p.reserve(ctx, cap(p.slots)) if n == 0 { return } @@ -120,12 +120,14 @@ func fetchFailed(ctx context.Context, err error) { } } -// reserve waits for room for one more delivery and takes up to limit, or none once halted. -func (p *natsLanes) reserve(limit int) int { +// reserve waits for room for one more delivery and takes up to limit, or none once halted or stopped. +func (p *natsLanes) reserve(ctx context.Context, limit int) int { select { case p.room <- struct{}{}: case <-p.halted: return 0 + case <-ctx.Done(): + return 0 } n := 1 for n < limit { diff --git a/internal/infrastructure/eventbus/nats_lanes_test.go b/internal/infrastructure/eventbus/nats_lanes_test.go index 58ffc93cd..260fce525 100644 --- a/internal/infrastructure/eventbus/nats_lanes_test.go +++ b/internal/infrastructure/eventbus/nats_lanes_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "sync" + "sync/atomic" "testing" "time" ) @@ -352,3 +353,40 @@ func TestNATSBus_KeyedLanesKeepMessagesWaitingForRoomAlive(t *testing.T) { } } } + +// Closing the bus ends a pull waiting for room behind handlers that have not finished. +func TestNATSBus_CloseDoesNotWaitForLaneRoom(t *testing.T) { + bus := newTestNATSBus(t) + topic := "w.close" + release := make(chan struct{}) + var started atomic.Int32 + subscribeInBackground(t, t.Context(), bus, topic, "close", 2, payloadKey, func(context.Context, Message) error { + started.Add(1) + <-release + return nil + }) + t.Cleanup(func() { close(release) }) + waitForConsumer(t, bus, bus.durable("close", []string{topic})) + for i := 0; i < 2*natsLaneBuffer+8; i++ { + if err := bus.Publish(t.Context(), topic, "k", fmt.Appendf(nil, "%c%02d", 'a'+i%2, i)); err != nil { + t.Fatalf("publish %d: %v", i, err) + } + } + deadline := time.Now().Add(5 * time.Second) + for started.Load() < 2 && time.Now().Before(deadline) { + time.Sleep(20 * time.Millisecond) + } + // Let the pull fill the lanes' room and wait for more. + time.Sleep(500 * time.Millisecond) + + closed := make(chan struct{}) + go func() { + _ = bus.Close() + close(closed) + }() + select { + case <-closed: + case <-time.After(3 * time.Second): + t.Fatal("Close waited for room in the lanes") + } +} From 5f6532a67efd8a3c343701ce3e29882dde821f39 Mon Sep 17 00:00:00 2001 From: Matthew Meszaros Date: Thu, 5 Nov 2026 18:06:22 +0100 Subject: [PATCH 6/6] feat: halt the mailbox lanes when a keyed NATS pull is stopped so closing the bus hands queued deliveries back instead of starting them while the connection drains, asserted by the close regression test --- internal/infrastructure/eventbus/nats_lanes.go | 10 ++++++---- internal/infrastructure/eventbus/nats_lanes_test.go | 13 +++++++++++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/internal/infrastructure/eventbus/nats_lanes.go b/internal/infrastructure/eventbus/nats_lanes.go index 356b27271..b95eadc56 100644 --- a/internal/infrastructure/eventbus/nats_lanes.go +++ b/internal/infrastructure/eventbus/nats_lanes.go @@ -48,20 +48,22 @@ func newNATSLanes(lanes int, ackWait time.Duration, deliver func(jetstream.Msg)) return p } -// natsPull stops a lanes' pull loop, in the shape Consume hands back. +// natsPull stops a lanes' pull loop, in the shape Consume hands back. Stopping +// halts the lanes too, so a queued delivery is handed back rather than started. type natsPull struct { cancel context.CancelFunc + halt func() closed chan struct{} } -func (h *natsPull) Stop() { h.cancel() } -func (h *natsPull) Drain() { h.cancel() } +func (h *natsPull) Stop() { h.halt(); h.cancel() } +func (h *natsPull) Drain() { h.Stop() } func (h *natsPull) Closed() <-chan struct{} { return h.closed } // start runs pull until ctx ends or the returned handle is stopped. func (p *natsLanes) start(ctx context.Context, cons jetstream.Consumer, key func(jetstream.Msg) (string, error)) jetstream.ConsumeContext { ctx, cancel := context.WithCancel(ctx) - h := &natsPull{cancel: cancel, closed: make(chan struct{})} + h := &natsPull{cancel: cancel, halt: p.halt, closed: make(chan struct{})} go func() { defer close(h.closed) p.pull(ctx, cons, key) diff --git a/internal/infrastructure/eventbus/nats_lanes_test.go b/internal/infrastructure/eventbus/nats_lanes_test.go index 260fce525..c68c3de92 100644 --- a/internal/infrastructure/eventbus/nats_lanes_test.go +++ b/internal/infrastructure/eventbus/nats_lanes_test.go @@ -354,18 +354,21 @@ func TestNATSBus_KeyedLanesKeepMessagesWaitingForRoomAlive(t *testing.T) { } } -// Closing the bus ends a pull waiting for room behind handlers that have not finished. +// Closing the bus ends a pull waiting for room behind handlers that have not +// finished, and hands back what is queued instead of starting it. func TestNATSBus_CloseDoesNotWaitForLaneRoom(t *testing.T) { bus := newTestNATSBus(t) topic := "w.close" release := make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } var started atomic.Int32 subscribeInBackground(t, t.Context(), bus, topic, "close", 2, payloadKey, func(context.Context, Message) error { started.Add(1) <-release return nil }) - t.Cleanup(func() { close(release) }) + t.Cleanup(unblock) waitForConsumer(t, bus, bus.durable("close", []string{topic})) for i := 0; i < 2*natsLaneBuffer+8; i++ { if err := bus.Publish(t.Context(), topic, "k", fmt.Appendf(nil, "%c%02d", 'a'+i%2, i)); err != nil { @@ -389,4 +392,10 @@ func TestNATSBus_CloseDoesNotWaitForLaneRoom(t *testing.T) { case <-time.After(3 * time.Second): t.Fatal("Close waited for room in the lanes") } + before := started.Load() + unblock() + time.Sleep(500 * time.Millisecond) + if after := started.Load(); after != before { + t.Fatalf("%d queued deliveries started after Close", after-before) + } }