feat: key SMTP pacing on the normalized server host and worker address, skip loopback relays, hold reserved starts that land in a later cooldown, measure the spaced window from the cooldown's end, classify dropped STARTTLS by transport error, and cover a long server-pacing retry time live

This commit is contained in:
Matthew Meszaros
2026-11-03 17:18:48 +01:00
parent 03cd541c62
commit d1d43111d0
4 changed files with 106 additions and 20 deletions
+15 -5
View File
@@ -44,6 +44,15 @@ func tlsFailure(base *errx.MailError, stage string, err error) *errx.MailError {
return sendFailure(base, err, stage, "mailbox")
}
// startTLSFailure marks a handshake the server closed, which is how some servers shed connections.
func startTLSFailure(err error) *errx.MailError {
failure := tlsFailure(errx.ErrMailServerUnreachable, "starttls", err)
if droppedConnection(err) && failure.Failure != nil {
failure.Failure.Cause = causeDroppedHandshake
}
return failure
}
// pinFailure reports a pinned certificate refusal as a retryable mailbox
// connection error, never as a credentials failure.
func pinFailure(perr *mailtls.PinError) *errx.MailError {
@@ -359,11 +368,12 @@ func (c *Client) sendRaw(ctx context.Context, from string, to []string, data []b
if resolved == models.MailSecurityNone && !models.CleartextMailAllowed(host) {
return errx.ErrMailInsecureRemoteHost
}
pace := paceFor(host, port, c.BindIP)
if held := pace.wait(ctx, time.Now()); held != nil {
return held
if pace := paceFor(host, c.BindIP); pace != nil {
if held := pace.wait(ctx, time.Now()); held != nil {
return held
}
defer func() { result = pace.observe(time.Now(), result) }()
}
defer func() { result = pace.observe(time.Now(), result) }()
// The socket may be 587 with STARTTLS when the mailbox's 465 never
// answered; the mode to speak is the one the dial reports.
dialed, err := DialSubmission(ctx, c.BindIP, host, port, security)
@@ -431,7 +441,7 @@ func (c *Client) sendRaw(ctx context.Context, from string, to []string, data []b
if perr := mailtls.AsPinError(err, capture); perr != nil {
return pinFailure(perr)
}
return tlsFailure(errx.ErrMailServerUnreachable, "starttls", err)
return startTLSFailure(err)
}
} else if !netbind.InsecureTLS() && !c.plaintext {
return sendFailure(errx.ErrMailServerUnreachableAt("starttls", errNoSTARTTLS), errNoSTARTTLS, "starttls", "mailbox")
+32 -7
View File
@@ -2,13 +2,16 @@ package smtp
import (
"context"
"errors"
"io"
"net"
"strconv"
"strings"
"sync"
"syscall"
"time"
"github.com/warmbly/warmbly/internal/errx"
"github.com/warmbly/warmbly/internal/models"
)
// A server limiting connections from one client address refuses every mailbox
@@ -21,6 +24,8 @@ const (
paceMaxWait = 10 * time.Second
// PaceStage marks a send held on the worker without contacting the server.
PaceStage = "pacing"
// causeDroppedHandshake marks a STARTTLS the server closed mid-handshake, as opposed to one it refused with a reason.
causeDroppedHandshake = "dropped_handshake"
)
// hostPace is one worker address's standing with one SMTP server.
@@ -34,8 +39,12 @@ type hostPace struct {
var hostPaces sync.Map
func paceFor(host string, port int, local *net.TCPAddr) *hostPace {
key := net.JoinHostPort(strings.ToLower(host), strconv.Itoa(port)) + "|"
// paceFor keys on the server, not the port, so a 587 fallback shares the 465 standing; a relay on this machine has none.
func paceFor(host string, local *net.TCPAddr) *hostPace {
if models.LoopbackMailHost(host) {
return nil
}
key := strings.ToLower(strings.TrimSuffix(models.NormalizeMailHost(host), ".")) + "|"
if local != nil {
key += local.IP.String()
}
@@ -77,12 +86,22 @@ func (p *hostPace) wait(ctx context.Context, now time.Time) *errx.MailError {
defer timer.Stop()
select {
case <-timer.C:
return nil
return p.heldIfCooling(time.Now())
case <-ctx.Done():
return pacedHold(now, now.Add(wait))
}
}
// heldIfCooling refuses a reserved start that a refusal since the reservation put inside a cooldown.
func (p *hostPace) heldIfCooling(now time.Time) *errx.MailError {
p.mu.Lock()
defer p.mu.Unlock()
if now.Before(p.coolUntil) {
return pacedHold(now, p.coolUntil)
}
return nil
}
// observe records a session's outcome; one burst of refusals counts as one strike.
func (p *hostPace) observe(now time.Time, err *errx.MailError) *errx.MailError {
throttled := serverThrottled(err)
@@ -102,7 +121,7 @@ func (p *hostPace) observe(now time.Time, err *errx.MailError) *errx.MailError {
}
p.coolUntil = now.Add(cool)
}
p.pacedUntil = now.Add(pacedWindow)
p.pacedUntil = p.coolUntil.Add(pacedWindow)
return throttledFailure(err, p.coolUntil)
}
@@ -118,16 +137,22 @@ func serverThrottled(err *errx.MailError) bool {
case f.Stage == "auth" && f.Status == 454:
return true
case f.Stage == "starttls" && f.Status == 0:
// A dropped handshake; a refused certificate or a port without STARTTLS carries its own reason.
return strings.HasPrefix(err.Message, errx.ErrMailServerUnreachable.Message)
return f.Cause == causeDroppedHandshake
}
return false
}
// droppedConnection is the server closing or resetting the connection, never a TLS alert or a certificate refusal.
func droppedConnection(err error) bool {
return errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) ||
errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.ECONNABORTED) || errors.Is(err, syscall.EPIPE)
}
func throttledFailure(err *errx.MailError, retryAt time.Time) *errx.MailError {
out := *err
f := *err.Failure
f.RetryAt = &retryAt
f.Cause = ""
switch f.Stage {
case "greeting", "ehlo", "starttls", "auth", "mail from":
// Refused before any recipient was offered, so the send costs the lead nothing.
@@ -7,6 +7,7 @@ import (
"net"
"net/textproto"
"strings"
"syscall"
"testing"
"time"
@@ -30,7 +31,9 @@ func TestServerThrottledNamesOnlyAddressRefusals(t *testing.T) {
{"greeting 421", throttleReply("greeting", 421), true},
{"auth 454", throttleReply("auth", 454), true},
{"rcpt 421", throttleReply("rcpt to", 421), true},
{"dropped handshake", tlsFailure(errx.ErrMailServerUnreachable, "starttls", io.EOF), true},
{"dropped handshake", startTLSFailure(io.EOF), true},
{"reset handshake", startTLSFailure(&net.OpError{Op: "read", Err: syscall.ECONNRESET}), true},
{"handshake alert", startTLSFailure(errors.New("remote error: tls: protocol version not supported")), false},
{"refused certificate", sendFailure(errx.ErrMailServerUnreachableAt("starttls", errors.New(errx.MailConnectionReasonCertificate)), io.EOF, "starttls", "mailbox"), false},
{"no STARTTLS", sendFailure(errx.ErrMailServerUnreachableAt("starttls", errNoSTARTTLS), errNoSTARTTLS, "starttls", "mailbox"), false},
{"lost auth connection", throttleReply("auth", 0), false},
@@ -50,7 +53,7 @@ func TestHostPaceCoolsOncePerBurstAndBacksOff(t *testing.T) {
p := &hostPace{}
t0 := time.Date(2026, 11, 3, 18, 0, 0, 0, time.UTC)
first := p.observe(t0, throttleReply("starttls", 0))
first := p.observe(t0, startTLSFailure(io.EOF))
if !first.Failure.RetryAt.Equal(t0.Add(paceCooldownBase)) || first.Failure.Cause != errx.SendCauseServerPacing {
t.Fatalf("first refusal = %+v", first.Failure)
}
@@ -74,9 +77,14 @@ func TestHostPaceCoolsOncePerBurstAndBacksOff(t *testing.T) {
for i := 0; i < 10; i++ {
p.observe(p.coolUntil, throttleReply("greeting", 421))
}
if got := p.coolUntil.Sub(p.pacedUntil.Add(-pacedWindow)); got != paceCooldownMax {
last := p.coolUntil.Add(-paceCooldownMax)
p.observe(p.coolUntil, startTLSFailure(io.EOF))
if got := p.coolUntil.Sub(last.Add(paceCooldownMax)); got != paceCooldownMax {
t.Fatalf("cooldown not capped: %v", got)
}
if !p.pacedUntil.Equal(p.coolUntil.Add(pacedWindow)) {
t.Fatalf("spacing not measured from the end of the cooldown: paced=%v cool=%v", p.pacedUntil, p.coolUntil)
}
}
func TestHostPaceSpacesSessionsAfterARefusalUntilItRecovers(t *testing.T) {
@@ -113,7 +121,7 @@ func TestHostPaceSpacesSessionsAfterARefusalUntilItRecovers(t *testing.T) {
func TestHostPaceHoldIsARetryThatNeverReachedTheServer(t *testing.T) {
p := &hostPace{}
t0 := time.Now()
p.observe(t0, throttleReply("starttls", 0))
p.observe(t0, startTLSFailure(io.EOF))
held := p.wait(context.Background(), t0)
if held == nil || held.Failure == nil {
t.Fatal("cooling host admitted a session")
@@ -135,11 +143,30 @@ func TestHostPaceRefusalAfterRecipientsCostsTheLeadAnAttempt(t *testing.T) {
}
func TestPaceForKeysOnServerAndWorkerAddress(t *testing.T) {
a := paceFor("SMTP.Example.test", 587, &net.TCPAddr{IP: net.IPv4(192, 0, 2, 1)})
if paceFor("smtp.example.test", 587, &net.TCPAddr{IP: net.IPv4(192, 0, 2, 1)}) != a {
t.Fatal("host case split the pace")
a := paceFor("SMTP.Example.test", &net.TCPAddr{IP: net.IPv4(192, 0, 2, 1)})
if paceFor("smtp.example.test.", &net.TCPAddr{IP: net.IPv4(192, 0, 2, 1)}) != a {
t.Fatal("host spelling split the pace")
}
if paceFor("smtp.example.test", 587, &net.TCPAddr{IP: net.IPv4(192, 0, 2, 2)}) == a || paceFor("smtp.example.test", 587, nil) == a {
if paceFor("smtp.example.test", &net.TCPAddr{IP: net.IPv4(192, 0, 2, 2)}) == a || paceFor("smtp.example.test", nil) == a {
t.Fatal("another worker address shared the pace")
}
if paceFor("::1", nil) != nil || paceFor("localhost", nil) != nil {
t.Fatal("a relay on this machine was paced")
}
}
func TestHostPaceReservationYieldsToALaterCooldown(t *testing.T) {
p := &hostPace{}
now := time.Now()
p.pacedUntil = now.Add(pacedWindow)
p.nextStart = now.Add(50 * time.Millisecond)
done := make(chan *errx.MailError, 1)
go func() { done <- p.wait(context.Background(), now) }()
p.mu.Lock()
p.coolUntil = now.Add(time.Minute)
p.mu.Unlock()
held := <-done
if held == nil || held.Failure.Stage != PaceStage || !held.Failure.RetryAt.Equal(now.Add(time.Minute)) {
t.Fatalf("reserved start ran inside a cooldown: %+v", held)
}
}
@@ -65,3 +65,27 @@ func TestLiveSendFailureKeepsRecipientAndMailboxRecoverySeparate(t *testing.T) {
})
}
}
func TestLiveServerPacingHoldWaitsForItsRetryTime(t *testing.T) {
f := newCampaignIsolationFixture(t)
retryAt := time.Now().UTC().Add(9 * time.Minute)
result := models.SendEmailResult{TaskID: f.request.TaskID, Error: &models.EmailSendError{
Code: string(errx.MailErrorCodeServerUnreachable), Message: errx.MailServerPacedMessage,
Failure: &errx.SendFailure{Provider: "smtp", Protocol: "smtp", Stage: "pacing", Cause: errx.SendCauseServerPacing,
Disposition: errx.SendRetry, Scope: "mailbox", ObservedAt: time.Now().UTC(), RetryAt: &retryAt},
}}
if err := f.r.FinishOutbound(t.Context(), result.TaskID, f.sender, f.worker, result); err != nil {
t.Fatal(err)
}
if err := f.r.ApplySendResult(t.Context(), result, func(context.Context) error { return nil }); err != nil {
t.Fatal(err)
}
early, err := f.r.GetSendAdmission(t.Context(), f.org, f.sender, models.InboxProviderSMTPIMAP, retryAt.Add(-time.Minute))
if err != nil || early.Allowed || early.RecoveryHold || early.RetryAt == nil || early.RetryAt.Before(retryAt.Add(-time.Second)) {
t.Fatalf("held send admitted before the server's retry time: %+v %v", early, err)
}
late, err := f.r.GetSendAdmission(t.Context(), f.org, f.sender, models.InboxProviderSMTPIMAP, retryAt.Add(time.Second))
if err != nil || !late.Allowed || late.RecoveryHold {
t.Fatalf("held send not admitted after the retry time: %+v %v", late, err)
}
}