Merge pull request #1162 from warmbly/fix/consumer-mailbox-queues

feat: per-mailbox consumer queues sized from the database pool
This commit is contained in:
Matthew Meszaros
2026-11-05 17:21:06 +00:00
committed by GitHub
9 changed files with 357 additions and 66 deletions
+1
View File
@@ -553,6 +553,7 @@ func main() {
jobsService := &jobs.JobsService{
CampaignIsolationEnabled: config.CampaignSendIsolationEnabled(),
DBConns: primaryDB.Pool.Config().MaxConns,
Bus: consumerBus,
Codec: consumerCodec,
UniboxRepository: uniboxRepo,
@@ -388,7 +388,7 @@ The attachment download proxy works with filesystem and S3-compatible stores and
| `CODEC_PROVIDER` | `json` or `avro` | `json` under compose, `avro` for a bare binary | yes |
| `CODEC_READ_BOTH` | Migration setting for Go services: read JSON and Confluent-framed Avro while writing only `CODEC_PROVIDER`. Requires Kafka builds and the old Schema Registry until all retained Avro has drained | `false` | yes |
| `EVENTBUS_HANDLER_TIMEOUT` | How long one handler may take before the delivery is abandoned. A subscription that needs longer carries its own budget, never shorter than this one: a worker's command subscription runs under 195s, because a send is given 120s (the SMTP client's 90s conversation plus the checks before it) and then records its result. On NATS the ack wait is the subscription's budget plus 5s | `30s` | yes |
| `CONSUMER_EVENT_CONCURRENCY` | Consumer lanes for worker events, from `1` to `64`, on both Kafka and NATS. Events are grouped by the mailbox the worker published them for, so each mailbox's events stay in order while different mailboxes are handled at once. Busy lanes hold database connections at the same time, so keep it well below the consumer's `DB_MAX_CONNS`. Invalid values use the default | `8` | yes |
| `CONSUMER_EVENT_CONCURRENCY` | How many mailboxes' worker events one consumer handles at once, from `1` to `64`, on both Kafka and NATS. Each mailbox's events stay in order, and on NATS a slow mailbox holds only its own lane. Unset, it follows the consumer's database pool (`DB_MAX_CONNS`), so raising the pool raises it too. Invalid values use the default | one per pooled database connection, at most `64` | yes |
| `PUBSUB_ENABLED` | `false` uses the Redis bridge for realtime fanout, `true` uses Google Pub/Sub | `false` | yes |
| `GCP_PROJECT_ID` | Project when `PUBSUB_ENABLED=true` | unset | yes |
+1 -1
View File
@@ -89,7 +89,7 @@ Workers publish the same envelope shape back on `jobs.worker-events`:
Types: `NEW_EMAIL`, `INBOUND_BOUNCE`, `REMOVE_EMAIL`, `FLAGS_ADD`, `FLAGS_REMOVE`, `UPDATE_EMAIL`, `UPDATE_FOLDER`, `UPDATE_MAILBOX`, `DELETE_MAILBOX`, `TOKEN_UPDATE`, `HISTORY_ID_UPDATE`, `GRAPH_DELTA_UPDATE`, `SYNC_STATE`, `PUSH_STATE`, `EMAIL_SENT`, `EMAIL_FAILED`, `EMAIL_AUTH_ERROR`, `EMAIL_DISABLED`, `EMAIL_RATE_LIMITED`, `EMAIL_SERVER_ERROR`, `WORKER_HEALTH`.
The consumer registers one handler per type (`internal/app/consumer/events.go`); unregistered types are logged and acknowledged rather than redelivered. It handles them on ordered per-mailbox lanes (`CONSUMER_EVENT_CONCURRENCY`), keyed by the mailbox id each worker publishes under, so one mailbox's events stay in order while a burst from another mailbox does not queue behind them.
The consumer registers one handler per type (`internal/app/consumer/events.go`); unregistered types are logged and acknowledged rather than redelivered. It handles them on ordered per-mailbox lanes (`CONSUMER_EVENT_CONCURRENCY`), keyed by the mailbox id each worker publishes under, so one mailbox's events stay in order while a burst from another mailbox does not queue behind them. On NATS a slow handler holds only its own mailbox's lane; on Kafka a mailbox shares a lane with the others its key hashes to. Unset, the number of mailboxes handled at once follows the consumer's database pool.
Every `SEND_EMAIL` is answered with exactly one per-task result: `EMAIL_SENT` (the provider accepted the message; the consumer records the wire Message-ID on the task) or `EMAIL_FAILED` (a `SendEmailResult` carrying the error code and message). The control plane stamps a campaign step sent when it hands the send to the worker, so `EMAIL_FAILED` is what walks that back: the consumer marks the task failed, clears the step's `sent_at` and counts the attempt on `campaign_contact_progress`, gives the send back to the campaign's daily counters, writes the failure to the campaign activity log, and reopens a campaign that completed while the send was in flight. After `CampaignSendMaxAttempts` (5) the lead is marked failed and routing drops it; a `RECIPIENT_REJECTED` result (refused at RCPT) skips the retries and is ingested as a bounce instead. Account-level conditions (`EMAIL_AUTH_ERROR`, `EMAIL_DISABLED`, `EMAIL_RATE_LIMITED`, `EMAIL_SERVER_ERROR`) are raised in addition to, never instead of, the per-task result; they carry an `EmailErrorEvent` and act on the mailbox. A worker that does not hold the mailbox yet leaves the send for a few redeliveries before reporting it failed, and the backend never publishes a send to a worker that is not heartbeating.
+8 -4
View File
@@ -8,12 +8,16 @@ import (
"github.com/warmbly/warmbly/internal/infrastructure/eventbus"
)
// WorkerEventConcurrency is how many mailboxes' worker events one consumer handles at once.
func WorkerEventConcurrency() int {
if n, err := strconv.Atoi(os.Getenv("CONSUMER_EVENT_CONCURRENCY")); err == nil && n >= 1 && n <= 64 {
// maxEventLanes is the most mailboxes one consumer handles at once.
const maxEventLanes = 64
// WorkerEventConcurrency is how many mailboxes' worker events one consumer handles at once:
// CONSUMER_EVENT_CONCURRENCY when set, otherwise one per pooled database connection.
func WorkerEventConcurrency(poolConns int32) int {
if n, err := strconv.Atoi(os.Getenv("CONSUMER_EVENT_CONCURRENCY")); err == nil && n >= 1 && n <= maxEventLanes {
return n
}
return 8
return min(max(int(poolConns), 1), maxEventLanes)
}
// workerEventKey keeps one mailbox's events in order: workers publish them under the mailbox id,
+12 -5
View File
@@ -7,11 +7,18 @@ import (
"github.com/warmbly/warmbly/internal/infrastructure/eventbus"
)
func TestWorkerEventConcurrencyDefaultsAndBounds(t *testing.T) {
for raw, want := range map[string]int{"": 8, "1": 1, "64": 64, "0": 8, "65": 8, "many": 8} {
t.Setenv("CONSUMER_EVENT_CONCURRENCY", raw)
if got := WorkerEventConcurrency(); got != want {
t.Errorf("CONSUMER_EVENT_CONCURRENCY=%q: got %d lanes, want %d", raw, got, want)
func TestWorkerEventConcurrencyFollowsThePoolUnlessSet(t *testing.T) {
for _, c := range []struct {
env string
pool int32
want int
}{
{"", 25, 25}, {"", 4, 4}, {"", 0, 1}, {"", 200, 64},
{"1", 25, 1}, {"64", 4, 64}, {"0", 25, 25}, {"65", 25, 25}, {"many", 10, 10},
} {
t.Setenv("CONSUMER_EVENT_CONCURRENCY", c.env)
if got := WorkerEventConcurrency(c.pool); got != c.want {
t.Errorf("CONSUMER_EVENT_CONCURRENCY=%q pool=%d: got %d lanes, want %d", c.env, c.pool, got, c.want)
}
}
}
+3 -1
View File
@@ -43,6 +43,8 @@ type CloudExecution interface {
type JobsService struct {
CampaignIsolationEnabled bool
// DBConns is the database pool the event handlers share; it sizes how many mailboxes run at once.
DBConns int32
// Bus delivers the jobs.worker-events stream (Kafka or NATS).
Bus eventbus.EventBus
// Codec decodes bus payloads (jobs.worker-events); it must match the
@@ -151,7 +153,7 @@ func (s *JobsService) Start(ctx context.Context) {
go s.runStoredSendResults(ctx)
go s.runArrivalOutbox(ctx)
lanes := WorkerEventConcurrency()
lanes := WorkerEventConcurrency(s.DBConns)
log.Info().Int("lanes", lanes).Msg("consumer: worker-events ordered mailbox lanes")
if err := eventbus.SubscribeKeyed(ctx, s.Bus, []string{kafka.TopicWorkerEvents}, "consumer-group", lanes, workerEventKey, s.receive); err != nil {
log.Error().Err(err).Msg("consumer: worker-events subscription ended")
+5 -22
View File
@@ -346,30 +346,13 @@ func (b *NATSBus) SubscribeKeyed(ctx context.Context, topics []string, group str
}
var pool *natsLanes
callback := deliver
var opts []jetstream.PullConsumeOpt
var cc jetstream.ConsumeContext
if lanes > 1 {
pool = newNATSLanes(lanes, ackWait, deliver)
callback = func(m jetstream.Msg) {
laneKey, err := resolveKey(ctx, key, b.envelope(m))
if err != nil {
log.Error().Err(err).Str("subject", m.Subject()).Msg("eventbus nats key resolution failed")
if nakErr := m.NakWithDelay(time.Second); nakErr != nil {
log.Warn().Err(nakErr).Msg("eventbus nats nak failed")
}
return
}
pool.dispatch(m, laneKey)
}
// Fetch again only after dispatch tracks this delivery, even when a lane is full.
opts = append(opts, jetstream.PullMaxMessages(1))
}
cc, err := cons.Consume(callback, opts...)
if err != nil {
if pool != nil {
pool.stop()
}
cc = pool.start(ctx, cons, func(m jetstream.Msg) (string, error) {
return resolveKey(ctx, key, b.envelope(m))
})
} else if cc, err = cons.Consume(deliver); err != nil {
return fmt.Errorf("eventbus nats: consume: %w", err)
}
+171 -27
View File
@@ -1,7 +1,8 @@
package eventbus
import (
"hash/fnv"
"context"
"errors"
"sync"
"time"
@@ -9,13 +10,19 @@ import (
"github.com/rs/zerolog/log"
)
// natsLaneBuffer is how many deliveries each lane may hold, running or queued, before fetching pauses.
const natsLaneBuffer = 32
// natsLanes runs a keyed subscription on ordered lanes. JetStream times a
// delivery from when it was handed out, so every message the lanes hold,
// queued or running, is kept alive with an in-progress heartbeat.
// natsFetchWait bounds one pull request, so a stop is seen promptly while nothing arrives.
const natsFetchWait = 5 * time.Second
// natsLanes runs a keyed subscription: each key's messages in order, up to
// lanes keys at once, so a slow key holds one lane and never another key.
// JetStream times a delivery from when it was handed out, so every message
// held, queued or running, is kept alive with an in-progress heartbeat.
type natsLanes struct {
queues []chan jetstream.Msg
slots chan struct{}
room chan struct{}
deliver func(jetstream.Msg)
halted chan struct{}
haltOnce sync.Once
@@ -24,54 +31,173 @@ type natsLanes struct {
mu sync.Mutex
held map[jetstream.Msg]struct{}
keys map[string][]jetstream.Msg
}
func newNATSLanes(lanes int, ackWait time.Duration, deliver func(jetstream.Msg)) *natsLanes {
p := &natsLanes{
queues: make([]chan jetstream.Msg, lanes),
slots: make(chan struct{}, lanes),
room: make(chan struct{}, lanes*natsLaneBuffer),
deliver: deliver,
halted: make(chan struct{}),
done: make(chan struct{}),
held: map[jetstream.Msg]struct{}{},
}
for i := range p.queues {
p.queues[i] = make(chan jetstream.Msg, natsLaneBuffer)
p.wg.Add(1)
go p.run(p.queues[i])
keys: map[string][]jetstream.Msg{},
}
go p.heartbeat(max(ackWait/3, 100*time.Millisecond))
return p
}
func natsLane(key string, lanes int) int {
h := fnv.New32a()
_, _ = h.Write([]byte(key))
return int(h.Sum32() % uint32(lanes))
// natsPull stops a lanes' pull loop, in the shape Consume hands back. Stopping
// halts the lanes too, so a queued delivery is handed back rather than started.
type natsPull struct {
cancel context.CancelFunc
halt func()
closed chan struct{}
}
// dispatch queues m behind every earlier message with the same key.
func (h *natsPull) Stop() { h.halt(); h.cancel() }
func (h *natsPull) Drain() { h.Stop() }
func (h *natsPull) Closed() <-chan struct{} { return h.closed }
// start runs pull until ctx ends or the returned handle is stopped.
func (p *natsLanes) start(ctx context.Context, cons jetstream.Consumer, key func(jetstream.Msg) (string, error)) jetstream.ConsumeContext {
ctx, cancel := context.WithCancel(ctx)
h := &natsPull{cancel: cancel, halt: p.halt, closed: make(chan struct{})}
go func() {
defer close(h.closed)
p.pull(ctx, cons, key)
}()
return h
}
// pull fetches only as many messages as the lanes have room for and
// dispatches each by key until ctx ends or the lanes halt. Every fetch is
// asked for after the previous one ends, so none is skipped and nothing is
// fetched that the lanes could not hold and heartbeat at once.
func (p *natsLanes) pull(ctx context.Context, cons jetstream.Consumer, key func(jetstream.Msg) (string, error)) {
for ctx.Err() == nil {
n := p.reserve(ctx, cap(p.slots))
if n == 0 {
return
}
p.unreserve(n - p.fetch(ctx, cons, n, key))
}
}
// fetch dispatches up to n messages and reports how many it dispatched.
func (p *natsLanes) fetch(ctx context.Context, cons jetstream.Consumer, n int, key func(jetstream.Msg) (string, error)) int {
fctx, cancel := context.WithTimeout(ctx, natsFetchWait)
defer cancel()
batch, err := cons.Fetch(n, jetstream.FetchContext(fctx))
if err != nil {
fetchFailed(ctx, err)
return 0
}
dispatched := 0
for m := range batch.Messages() {
k, err := key(m)
if err != nil {
log.Error().Err(err).Str("subject", m.Subject()).Msg("eventbus nats key resolution failed")
if nakErr := m.NakWithDelay(time.Second); nakErr != nil {
log.Warn().Err(nakErr).Msg("eventbus nats nak failed")
}
continue
}
p.dispatch(m, k)
dispatched++
}
if err := batch.Error(); err != nil && !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, context.Canceled) {
fetchFailed(ctx, err)
}
return dispatched
}
// fetchFailed logs a failed pull and pauses briefly, so an unreachable server is not hot-looped.
func fetchFailed(ctx context.Context, err error) {
log.Warn().Err(err).Msg("eventbus nats fetch failed")
select {
case <-ctx.Done():
case <-time.After(time.Second):
}
}
// reserve waits for room for one more delivery and takes up to limit, or none once halted or stopped.
func (p *natsLanes) reserve(ctx context.Context, limit int) int {
select {
case p.room <- struct{}{}:
case <-p.halted:
return 0
case <-ctx.Done():
return 0
}
n := 1
for n < limit {
select {
case p.room <- struct{}{}:
n++
default:
return n
}
}
return n
}
func (p *natsLanes) unreserve(n int) {
for range n {
<-p.room
}
}
// dispatch queues m, for which room is reserved, behind every earlier message with the same key.
func (p *natsLanes) dispatch(m jetstream.Msg, key string) {
p.hold(m)
if p.isHalted() {
p.abandon(m)
return
}
select {
case p.queues[natsLane(key, len(p.queues))] <- m:
case <-p.halted:
p.abandon(m)
p.mu.Lock()
if queue, running := p.keys[key]; running {
p.keys[key] = append(queue, m)
p.mu.Unlock()
return
}
p.keys[key] = nil
p.mu.Unlock()
p.wg.Add(1)
go p.run(key, m)
}
func (p *natsLanes) run(queue <-chan jetstream.Msg) {
// run handles key's messages one after another, taking a lane for each.
func (p *natsLanes) run(key string, m jetstream.Msg) {
defer p.wg.Done()
for m := range queue {
for {
acquired := false
select {
case p.slots <- struct{}{}:
acquired = true
case <-p.halted:
}
if p.isHalted() {
p.abandon(m)
continue
if acquired {
<-p.slots
}
p.abandonKey(key, m)
return
}
p.deliver(m)
<-p.slots
p.release(m)
p.mu.Lock()
queue := p.keys[key]
if len(queue) == 0 {
delete(p.keys, key)
p.mu.Unlock()
return
}
m, p.keys[key] = queue[0], queue[1:]
p.mu.Unlock()
}
}
@@ -82,6 +208,11 @@ func (p *natsLanes) hold(m jetstream.Msg) {
}
func (p *natsLanes) release(m jetstream.Msg) {
p.unhold(m)
<-p.room
}
func (p *natsLanes) unhold(m jetstream.Msg) {
p.mu.Lock()
delete(p.held, m)
p.mu.Unlock()
@@ -91,6 +222,22 @@ func (p *natsLanes) release(m jetstream.Msg) {
// redelivered without waiting out its ack wait.
func (p *natsLanes) abandon(m jetstream.Msg) {
p.release(m)
nak(m)
}
// abandonKey hands back m and everything queued behind it.
func (p *natsLanes) abandonKey(key string, m jetstream.Msg) {
p.mu.Lock()
queue := p.keys[key]
delete(p.keys, key)
p.mu.Unlock()
p.abandon(m)
for _, q := range queue {
p.abandon(q)
}
}
func nak(m jetstream.Msg) {
if err := m.Nak(); err != nil {
log.Warn().Err(err).Msg("eventbus nats nak failed")
}
@@ -136,9 +283,6 @@ func (p *natsLanes) halt() {
// stop waits for the running handlers once no more messages can be dispatched.
func (p *natsLanes) stop() {
for _, q := range p.queues {
close(q)
}
p.wg.Wait()
close(p.done)
}
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"sync"
"sync/atomic"
"testing"
"time"
)
@@ -116,10 +117,6 @@ func TestNATSBus_KeyedLanesKeepKeyOrderAndRunKeysConcurrently(t *testing.T) {
}
return nil
})
// a and b must not share a lane for the concurrency half to mean anything.
if natsLane("a", 4) == natsLane("b", 4) {
t.Fatal("test keys hash to the same lane")
}
waitForConsumer(t, bus, bus.durable("lanes", []string{topic}))
for _, p := range []string{"a1", "a2", "a3", "b1"} {
if err := bus.Publish(t.Context(), topic, "", []byte(p)); err != nil {
@@ -141,6 +138,54 @@ func TestNATSBus_KeyedLanesKeepKeyOrderAndRunKeysConcurrently(t *testing.T) {
}
}
// A slow key takes one lane and no more: with two lanes, every other key keeps
// moving through the second while the first is held.
func TestNATSBus_KeyedLanesNeverQueueOneKeyBehindAnother(t *testing.T) {
bus := newTestNATSBus(t)
topic := "w.blocking"
releaseA := make(chan struct{})
handled := make(chan string, 8)
subscribeInBackground(t, t.Context(), bus, topic, "blocking", 2, payloadKey, func(_ context.Context, msg Message) error {
p := string(msg.Payload)
if p == "a1" {
select {
case <-releaseA:
case <-time.After(5 * time.Second):
}
}
handled <- p
return nil
})
waitForConsumer(t, bus, bus.durable("blocking", []string{topic}))
for _, p := range []string{"a1", "b1", "c1", "d1", "e1"} {
if err := bus.Publish(t.Context(), topic, "", []byte(p)); err != nil {
t.Fatalf("publish %s: %v", p, err)
}
}
others := map[string]bool{}
for len(others) < 4 {
select {
case p := <-handled:
if p == "a1" {
t.Fatalf("a1 finished before the other keys: %v", others)
}
others[p] = true
case <-time.After(5 * time.Second):
t.Fatalf("keys waited behind a held one: handled %v", others)
}
}
close(releaseA)
select {
case p := <-handled:
if p != "a1" {
t.Fatalf("handled %s, want a1", p)
}
case <-time.After(5 * time.Second):
t.Fatal("a1 never finished")
}
}
// A message queued behind a slow one on its lane outlives its ack wait; the
// in-progress heartbeat must keep JetStream from handing it out again.
func TestNATSBus_KeyedLanesKeepQueuedMessagesAlive(t *testing.T) {
@@ -230,7 +275,14 @@ func TestNATSBus_KeyedLanesKeepSaturatedMessagesAlive(t *testing.T) {
select {
case <-allHandled:
case <-time.After(20 * time.Second):
t.Fatal("not every saturated-lane message was handled")
mu.Lock()
redelivered := 0
for _, attempts := range seen {
redelivered += len(attempts) - 1
}
handled, distinct := len(order), len(seen)
mu.Unlock()
t.Fatalf("not every saturated-lane message was handled: %d distinct of %d, %d handlings, %d redeliveries", distinct, messages, handled, redelivered)
}
// Give already-prefetched redeliveries time to reach the handler too.
time.Sleep(2 * ackWait)
@@ -249,3 +301,101 @@ func TestNATSBus_KeyedLanesKeepSaturatedMessagesAlive(t *testing.T) {
t.Errorf("handled %d messages, want %d", len(order), messages)
}
}
// A delivery waiting for room behind a full lane is heartbeated too, so a slow
// handler ahead of it does not let JetStream hand it out a second time.
func TestNATSBus_KeyedLanesKeepMessagesWaitingForRoomAlive(t *testing.T) {
grace := natsAckGrace
natsAckGrace = 200 * time.Millisecond
t.Cleanup(func() { natsAckGrace = grace })
t.Setenv("EVENTBUS_HANDLER_TIMEOUT", "300ms")
bus := newTestNATSBus(t)
topic := "w.room"
// Two lanes, since one runs without the pool; one key fills both lanes' room.
const messages = 2*natsLaneBuffer + 4
var mu sync.Mutex
seen := map[string][]int{}
subscribeInBackground(t, t.Context(), bus, topic, "room", 2, payloadKey, func(_ context.Context, msg Message) error {
p := string(msg.Payload)
mu.Lock()
seen[p] = append(seen[p], msg.Attempt)
mu.Unlock()
if p == "a00" {
// Well past the 500ms ack wait while the lane's room is full.
time.Sleep(1500 * time.Millisecond)
}
return nil
})
ackWait := waitForConsumer(t, bus, bus.durable("room", []string{topic}))
for i := 0; i < messages; i++ {
if err := bus.Publish(t.Context(), topic, "a", fmt.Appendf(nil, "a%02d", i)); err != nil {
t.Fatalf("publish %d: %v", i, err)
}
}
deadline := time.Now().Add(10 * time.Second)
for {
mu.Lock()
n := len(seen)
mu.Unlock()
if n == messages || time.Now().After(deadline) {
break
}
time.Sleep(50 * time.Millisecond)
}
time.Sleep(2 * ackWait)
mu.Lock()
defer mu.Unlock()
for i := 0; i < messages; i++ {
p := fmt.Sprintf("a%02d", i)
if got := seen[p]; len(got) != 1 || got[0] != 1 {
t.Errorf("%s handled with attempts %v, want exactly one first delivery", p, got)
}
}
}
// Closing the bus ends a pull waiting for room behind handlers that have not
// finished, and hands back what is queued instead of starting it.
func TestNATSBus_CloseDoesNotWaitForLaneRoom(t *testing.T) {
bus := newTestNATSBus(t)
topic := "w.close"
release := make(chan struct{})
var releaseOnce sync.Once
unblock := func() { releaseOnce.Do(func() { close(release) }) }
var started atomic.Int32
subscribeInBackground(t, t.Context(), bus, topic, "close", 2, payloadKey, func(context.Context, Message) error {
started.Add(1)
<-release
return nil
})
t.Cleanup(unblock)
waitForConsumer(t, bus, bus.durable("close", []string{topic}))
for i := 0; i < 2*natsLaneBuffer+8; i++ {
if err := bus.Publish(t.Context(), topic, "k", fmt.Appendf(nil, "%c%02d", 'a'+i%2, i)); err != nil {
t.Fatalf("publish %d: %v", i, err)
}
}
deadline := time.Now().Add(5 * time.Second)
for started.Load() < 2 && time.Now().Before(deadline) {
time.Sleep(20 * time.Millisecond)
}
// Let the pull fill the lanes' room and wait for more.
time.Sleep(500 * time.Millisecond)
closed := make(chan struct{})
go func() {
_ = bus.Close()
close(closed)
}()
select {
case <-closed:
case <-time.After(3 * time.Second):
t.Fatal("Close waited for room in the lanes")
}
before := started.Load()
unblock()
time.Sleep(500 * time.Millisecond)
if after := started.Load(); after != before {
t.Fatalf("%d queued deliveries started after Close", after-before)
}
}