From d1d43111d0a78bfd2da2b0dcbca0a76aeaf85e54 Mon Sep 17 00:00:00 2001 From: Matthew Meszaros Date: Tue, 3 Nov 2026 17:18:48 +0100 Subject: [PATCH] feat: key SMTP pacing on the normalized server host and worker address, skip loopback relays, hold reserved starts that land in a later cooldown, measure the spaced window from the cooldown's end, classify dropped STARTTLS by transport error, and cover a long server-pacing retry time live --- internal/client/smtpimap/smtp/client.go | 20 ++++++--- internal/client/smtpimap/smtp/host_pace.go | 39 ++++++++++++++--- .../client/smtpimap/smtp/host_pace_test.go | 43 +++++++++++++++---- .../send_failure_isolation_live_test.go | 24 +++++++++++ 4 files changed, 106 insertions(+), 20 deletions(-) diff --git a/internal/client/smtpimap/smtp/client.go b/internal/client/smtpimap/smtp/client.go index ab5366c31..e838571ac 100644 --- a/internal/client/smtpimap/smtp/client.go +++ b/internal/client/smtpimap/smtp/client.go @@ -44,6 +44,15 @@ func tlsFailure(base *errx.MailError, stage string, err error) *errx.MailError { return sendFailure(base, err, stage, "mailbox") } +// startTLSFailure marks a handshake the server closed, which is how some servers shed connections. +func startTLSFailure(err error) *errx.MailError { + failure := tlsFailure(errx.ErrMailServerUnreachable, "starttls", err) + if droppedConnection(err) && failure.Failure != nil { + failure.Failure.Cause = causeDroppedHandshake + } + return failure +} + // pinFailure reports a pinned certificate refusal as a retryable mailbox // connection error, never as a credentials failure. func pinFailure(perr *mailtls.PinError) *errx.MailError { @@ -359,11 +368,12 @@ func (c *Client) sendRaw(ctx context.Context, from string, to []string, data []b if resolved == models.MailSecurityNone && !models.CleartextMailAllowed(host) { return errx.ErrMailInsecureRemoteHost } - pace := paceFor(host, port, c.BindIP) - if held := pace.wait(ctx, time.Now()); held != nil { - return held + if pace := paceFor(host, c.BindIP); pace != nil { + if held := pace.wait(ctx, time.Now()); held != nil { + return held + } + defer func() { result = pace.observe(time.Now(), result) }() } - defer func() { result = pace.observe(time.Now(), result) }() // The socket may be 587 with STARTTLS when the mailbox's 465 never // answered; the mode to speak is the one the dial reports. dialed, err := DialSubmission(ctx, c.BindIP, host, port, security) @@ -431,7 +441,7 @@ func (c *Client) sendRaw(ctx context.Context, from string, to []string, data []b if perr := mailtls.AsPinError(err, capture); perr != nil { return pinFailure(perr) } - return tlsFailure(errx.ErrMailServerUnreachable, "starttls", err) + return startTLSFailure(err) } } else if !netbind.InsecureTLS() && !c.plaintext { return sendFailure(errx.ErrMailServerUnreachableAt("starttls", errNoSTARTTLS), errNoSTARTTLS, "starttls", "mailbox") diff --git a/internal/client/smtpimap/smtp/host_pace.go b/internal/client/smtpimap/smtp/host_pace.go index f0c68f0d6..a32f2cf79 100644 --- a/internal/client/smtpimap/smtp/host_pace.go +++ b/internal/client/smtpimap/smtp/host_pace.go @@ -2,13 +2,16 @@ package smtp import ( "context" + "errors" + "io" "net" - "strconv" "strings" "sync" + "syscall" "time" "github.com/warmbly/warmbly/internal/errx" + "github.com/warmbly/warmbly/internal/models" ) // A server limiting connections from one client address refuses every mailbox @@ -21,6 +24,8 @@ const ( paceMaxWait = 10 * time.Second // PaceStage marks a send held on the worker without contacting the server. PaceStage = "pacing" + // causeDroppedHandshake marks a STARTTLS the server closed mid-handshake, as opposed to one it refused with a reason. + causeDroppedHandshake = "dropped_handshake" ) // hostPace is one worker address's standing with one SMTP server. @@ -34,8 +39,12 @@ type hostPace struct { var hostPaces sync.Map -func paceFor(host string, port int, local *net.TCPAddr) *hostPace { - key := net.JoinHostPort(strings.ToLower(host), strconv.Itoa(port)) + "|" +// paceFor keys on the server, not the port, so a 587 fallback shares the 465 standing; a relay on this machine has none. +func paceFor(host string, local *net.TCPAddr) *hostPace { + if models.LoopbackMailHost(host) { + return nil + } + key := strings.ToLower(strings.TrimSuffix(models.NormalizeMailHost(host), ".")) + "|" if local != nil { key += local.IP.String() } @@ -77,12 +86,22 @@ func (p *hostPace) wait(ctx context.Context, now time.Time) *errx.MailError { defer timer.Stop() select { case <-timer.C: - return nil + return p.heldIfCooling(time.Now()) case <-ctx.Done(): return pacedHold(now, now.Add(wait)) } } +// heldIfCooling refuses a reserved start that a refusal since the reservation put inside a cooldown. +func (p *hostPace) heldIfCooling(now time.Time) *errx.MailError { + p.mu.Lock() + defer p.mu.Unlock() + if now.Before(p.coolUntil) { + return pacedHold(now, p.coolUntil) + } + return nil +} + // observe records a session's outcome; one burst of refusals counts as one strike. func (p *hostPace) observe(now time.Time, err *errx.MailError) *errx.MailError { throttled := serverThrottled(err) @@ -102,7 +121,7 @@ func (p *hostPace) observe(now time.Time, err *errx.MailError) *errx.MailError { } p.coolUntil = now.Add(cool) } - p.pacedUntil = now.Add(pacedWindow) + p.pacedUntil = p.coolUntil.Add(pacedWindow) return throttledFailure(err, p.coolUntil) } @@ -118,16 +137,22 @@ func serverThrottled(err *errx.MailError) bool { case f.Stage == "auth" && f.Status == 454: return true case f.Stage == "starttls" && f.Status == 0: - // A dropped handshake; a refused certificate or a port without STARTTLS carries its own reason. - return strings.HasPrefix(err.Message, errx.ErrMailServerUnreachable.Message) + return f.Cause == causeDroppedHandshake } return false } +// droppedConnection is the server closing or resetting the connection, never a TLS alert or a certificate refusal. +func droppedConnection(err error) bool { + return errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) || + errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.ECONNABORTED) || errors.Is(err, syscall.EPIPE) +} + func throttledFailure(err *errx.MailError, retryAt time.Time) *errx.MailError { out := *err f := *err.Failure f.RetryAt = &retryAt + f.Cause = "" switch f.Stage { case "greeting", "ehlo", "starttls", "auth", "mail from": // Refused before any recipient was offered, so the send costs the lead nothing. diff --git a/internal/client/smtpimap/smtp/host_pace_test.go b/internal/client/smtpimap/smtp/host_pace_test.go index caf8a1884..1c2196cd3 100644 --- a/internal/client/smtpimap/smtp/host_pace_test.go +++ b/internal/client/smtpimap/smtp/host_pace_test.go @@ -7,6 +7,7 @@ import ( "net" "net/textproto" "strings" + "syscall" "testing" "time" @@ -30,7 +31,9 @@ func TestServerThrottledNamesOnlyAddressRefusals(t *testing.T) { {"greeting 421", throttleReply("greeting", 421), true}, {"auth 454", throttleReply("auth", 454), true}, {"rcpt 421", throttleReply("rcpt to", 421), true}, - {"dropped handshake", tlsFailure(errx.ErrMailServerUnreachable, "starttls", io.EOF), true}, + {"dropped handshake", startTLSFailure(io.EOF), true}, + {"reset handshake", startTLSFailure(&net.OpError{Op: "read", Err: syscall.ECONNRESET}), true}, + {"handshake alert", startTLSFailure(errors.New("remote error: tls: protocol version not supported")), false}, {"refused certificate", sendFailure(errx.ErrMailServerUnreachableAt("starttls", errors.New(errx.MailConnectionReasonCertificate)), io.EOF, "starttls", "mailbox"), false}, {"no STARTTLS", sendFailure(errx.ErrMailServerUnreachableAt("starttls", errNoSTARTTLS), errNoSTARTTLS, "starttls", "mailbox"), false}, {"lost auth connection", throttleReply("auth", 0), false}, @@ -50,7 +53,7 @@ func TestHostPaceCoolsOncePerBurstAndBacksOff(t *testing.T) { p := &hostPace{} t0 := time.Date(2026, 11, 3, 18, 0, 0, 0, time.UTC) - first := p.observe(t0, throttleReply("starttls", 0)) + first := p.observe(t0, startTLSFailure(io.EOF)) if !first.Failure.RetryAt.Equal(t0.Add(paceCooldownBase)) || first.Failure.Cause != errx.SendCauseServerPacing { t.Fatalf("first refusal = %+v", first.Failure) } @@ -74,9 +77,14 @@ func TestHostPaceCoolsOncePerBurstAndBacksOff(t *testing.T) { for i := 0; i < 10; i++ { p.observe(p.coolUntil, throttleReply("greeting", 421)) } - if got := p.coolUntil.Sub(p.pacedUntil.Add(-pacedWindow)); got != paceCooldownMax { + last := p.coolUntil.Add(-paceCooldownMax) + p.observe(p.coolUntil, startTLSFailure(io.EOF)) + if got := p.coolUntil.Sub(last.Add(paceCooldownMax)); got != paceCooldownMax { t.Fatalf("cooldown not capped: %v", got) } + if !p.pacedUntil.Equal(p.coolUntil.Add(pacedWindow)) { + t.Fatalf("spacing not measured from the end of the cooldown: paced=%v cool=%v", p.pacedUntil, p.coolUntil) + } } func TestHostPaceSpacesSessionsAfterARefusalUntilItRecovers(t *testing.T) { @@ -113,7 +121,7 @@ func TestHostPaceSpacesSessionsAfterARefusalUntilItRecovers(t *testing.T) { func TestHostPaceHoldIsARetryThatNeverReachedTheServer(t *testing.T) { p := &hostPace{} t0 := time.Now() - p.observe(t0, throttleReply("starttls", 0)) + p.observe(t0, startTLSFailure(io.EOF)) held := p.wait(context.Background(), t0) if held == nil || held.Failure == nil { t.Fatal("cooling host admitted a session") @@ -135,11 +143,30 @@ func TestHostPaceRefusalAfterRecipientsCostsTheLeadAnAttempt(t *testing.T) { } func TestPaceForKeysOnServerAndWorkerAddress(t *testing.T) { - a := paceFor("SMTP.Example.test", 587, &net.TCPAddr{IP: net.IPv4(192, 0, 2, 1)}) - if paceFor("smtp.example.test", 587, &net.TCPAddr{IP: net.IPv4(192, 0, 2, 1)}) != a { - t.Fatal("host case split the pace") + a := paceFor("SMTP.Example.test", &net.TCPAddr{IP: net.IPv4(192, 0, 2, 1)}) + if paceFor("smtp.example.test.", &net.TCPAddr{IP: net.IPv4(192, 0, 2, 1)}) != a { + t.Fatal("host spelling split the pace") } - if paceFor("smtp.example.test", 587, &net.TCPAddr{IP: net.IPv4(192, 0, 2, 2)}) == a || paceFor("smtp.example.test", 587, nil) == a { + if paceFor("smtp.example.test", &net.TCPAddr{IP: net.IPv4(192, 0, 2, 2)}) == a || paceFor("smtp.example.test", nil) == a { t.Fatal("another worker address shared the pace") } + if paceFor("::1", nil) != nil || paceFor("localhost", nil) != nil { + t.Fatal("a relay on this machine was paced") + } +} + +func TestHostPaceReservationYieldsToALaterCooldown(t *testing.T) { + p := &hostPace{} + now := time.Now() + p.pacedUntil = now.Add(pacedWindow) + p.nextStart = now.Add(50 * time.Millisecond) + done := make(chan *errx.MailError, 1) + go func() { done <- p.wait(context.Background(), now) }() + p.mu.Lock() + p.coolUntil = now.Add(time.Minute) + p.mu.Unlock() + held := <-done + if held == nil || held.Failure.Stage != PaceStage || !held.Failure.RetryAt.Equal(now.Add(time.Minute)) { + t.Fatalf("reserved start ran inside a cooldown: %+v", held) + } } diff --git a/internal/repository/send_failure_isolation_live_test.go b/internal/repository/send_failure_isolation_live_test.go index a8c24c474..c3c91192d 100644 --- a/internal/repository/send_failure_isolation_live_test.go +++ b/internal/repository/send_failure_isolation_live_test.go @@ -65,3 +65,27 @@ func TestLiveSendFailureKeepsRecipientAndMailboxRecoverySeparate(t *testing.T) { }) } } + +func TestLiveServerPacingHoldWaitsForItsRetryTime(t *testing.T) { + f := newCampaignIsolationFixture(t) + retryAt := time.Now().UTC().Add(9 * time.Minute) + result := models.SendEmailResult{TaskID: f.request.TaskID, Error: &models.EmailSendError{ + Code: string(errx.MailErrorCodeServerUnreachable), Message: errx.MailServerPacedMessage, + Failure: &errx.SendFailure{Provider: "smtp", Protocol: "smtp", Stage: "pacing", Cause: errx.SendCauseServerPacing, + Disposition: errx.SendRetry, Scope: "mailbox", ObservedAt: time.Now().UTC(), RetryAt: &retryAt}, + }} + if err := f.r.FinishOutbound(t.Context(), result.TaskID, f.sender, f.worker, result); err != nil { + t.Fatal(err) + } + if err := f.r.ApplySendResult(t.Context(), result, func(context.Context) error { return nil }); err != nil { + t.Fatal(err) + } + early, err := f.r.GetSendAdmission(t.Context(), f.org, f.sender, models.InboxProviderSMTPIMAP, retryAt.Add(-time.Minute)) + if err != nil || early.Allowed || early.RecoveryHold || early.RetryAt == nil || early.RetryAt.Before(retryAt.Add(-time.Second)) { + t.Fatalf("held send admitted before the server's retry time: %+v %v", early, err) + } + late, err := f.r.GetSendAdmission(t.Context(), f.org, f.sender, models.InboxProviderSMTPIMAP, retryAt.Add(time.Second)) + if err != nil || !late.Allowed || late.RecoveryHold { + t.Fatalf("held send not admitted after the retry time: %+v %v", late, err) + } +}