mirror of
https://github.com/warmbly/warmbly.git
synced 2026-11-17 16:19:13 +00:00
feat: key SMTP pacing on the normalized server host and worker address, skip loopback relays, hold reserved starts that land in a later cooldown, measure the spaced window from the cooldown's end, classify dropped STARTTLS by transport error, and cover a long server-pacing retry time live
This commit is contained in:
@@ -44,6 +44,15 @@ func tlsFailure(base *errx.MailError, stage string, err error) *errx.MailError {
|
||||
return sendFailure(base, err, stage, "mailbox")
|
||||
}
|
||||
|
||||
// startTLSFailure marks a handshake the server closed, which is how some servers shed connections.
|
||||
func startTLSFailure(err error) *errx.MailError {
|
||||
failure := tlsFailure(errx.ErrMailServerUnreachable, "starttls", err)
|
||||
if droppedConnection(err) && failure.Failure != nil {
|
||||
failure.Failure.Cause = causeDroppedHandshake
|
||||
}
|
||||
return failure
|
||||
}
|
||||
|
||||
// pinFailure reports a pinned certificate refusal as a retryable mailbox
|
||||
// connection error, never as a credentials failure.
|
||||
func pinFailure(perr *mailtls.PinError) *errx.MailError {
|
||||
@@ -359,11 +368,12 @@ func (c *Client) sendRaw(ctx context.Context, from string, to []string, data []b
|
||||
if resolved == models.MailSecurityNone && !models.CleartextMailAllowed(host) {
|
||||
return errx.ErrMailInsecureRemoteHost
|
||||
}
|
||||
pace := paceFor(host, port, c.BindIP)
|
||||
if held := pace.wait(ctx, time.Now()); held != nil {
|
||||
return held
|
||||
if pace := paceFor(host, c.BindIP); pace != nil {
|
||||
if held := pace.wait(ctx, time.Now()); held != nil {
|
||||
return held
|
||||
}
|
||||
defer func() { result = pace.observe(time.Now(), result) }()
|
||||
}
|
||||
defer func() { result = pace.observe(time.Now(), result) }()
|
||||
// The socket may be 587 with STARTTLS when the mailbox's 465 never
|
||||
// answered; the mode to speak is the one the dial reports.
|
||||
dialed, err := DialSubmission(ctx, c.BindIP, host, port, security)
|
||||
@@ -431,7 +441,7 @@ func (c *Client) sendRaw(ctx context.Context, from string, to []string, data []b
|
||||
if perr := mailtls.AsPinError(err, capture); perr != nil {
|
||||
return pinFailure(perr)
|
||||
}
|
||||
return tlsFailure(errx.ErrMailServerUnreachable, "starttls", err)
|
||||
return startTLSFailure(err)
|
||||
}
|
||||
} else if !netbind.InsecureTLS() && !c.plaintext {
|
||||
return sendFailure(errx.ErrMailServerUnreachableAt("starttls", errNoSTARTTLS), errNoSTARTTLS, "starttls", "mailbox")
|
||||
|
||||
@@ -2,13 +2,16 @@ package smtp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/warmbly/warmbly/internal/errx"
|
||||
"github.com/warmbly/warmbly/internal/models"
|
||||
)
|
||||
|
||||
// A server limiting connections from one client address refuses every mailbox
|
||||
@@ -21,6 +24,8 @@ const (
|
||||
paceMaxWait = 10 * time.Second
|
||||
// PaceStage marks a send held on the worker without contacting the server.
|
||||
PaceStage = "pacing"
|
||||
// causeDroppedHandshake marks a STARTTLS the server closed mid-handshake, as opposed to one it refused with a reason.
|
||||
causeDroppedHandshake = "dropped_handshake"
|
||||
)
|
||||
|
||||
// hostPace is one worker address's standing with one SMTP server.
|
||||
@@ -34,8 +39,12 @@ type hostPace struct {
|
||||
|
||||
var hostPaces sync.Map
|
||||
|
||||
func paceFor(host string, port int, local *net.TCPAddr) *hostPace {
|
||||
key := net.JoinHostPort(strings.ToLower(host), strconv.Itoa(port)) + "|"
|
||||
// paceFor keys on the server, not the port, so a 587 fallback shares the 465 standing; a relay on this machine has none.
|
||||
func paceFor(host string, local *net.TCPAddr) *hostPace {
|
||||
if models.LoopbackMailHost(host) {
|
||||
return nil
|
||||
}
|
||||
key := strings.ToLower(strings.TrimSuffix(models.NormalizeMailHost(host), ".")) + "|"
|
||||
if local != nil {
|
||||
key += local.IP.String()
|
||||
}
|
||||
@@ -77,12 +86,22 @@ func (p *hostPace) wait(ctx context.Context, now time.Time) *errx.MailError {
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-timer.C:
|
||||
return nil
|
||||
return p.heldIfCooling(time.Now())
|
||||
case <-ctx.Done():
|
||||
return pacedHold(now, now.Add(wait))
|
||||
}
|
||||
}
|
||||
|
||||
// heldIfCooling refuses a reserved start that a refusal since the reservation put inside a cooldown.
|
||||
func (p *hostPace) heldIfCooling(now time.Time) *errx.MailError {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if now.Before(p.coolUntil) {
|
||||
return pacedHold(now, p.coolUntil)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// observe records a session's outcome; one burst of refusals counts as one strike.
|
||||
func (p *hostPace) observe(now time.Time, err *errx.MailError) *errx.MailError {
|
||||
throttled := serverThrottled(err)
|
||||
@@ -102,7 +121,7 @@ func (p *hostPace) observe(now time.Time, err *errx.MailError) *errx.MailError {
|
||||
}
|
||||
p.coolUntil = now.Add(cool)
|
||||
}
|
||||
p.pacedUntil = now.Add(pacedWindow)
|
||||
p.pacedUntil = p.coolUntil.Add(pacedWindow)
|
||||
return throttledFailure(err, p.coolUntil)
|
||||
}
|
||||
|
||||
@@ -118,16 +137,22 @@ func serverThrottled(err *errx.MailError) bool {
|
||||
case f.Stage == "auth" && f.Status == 454:
|
||||
return true
|
||||
case f.Stage == "starttls" && f.Status == 0:
|
||||
// A dropped handshake; a refused certificate or a port without STARTTLS carries its own reason.
|
||||
return strings.HasPrefix(err.Message, errx.ErrMailServerUnreachable.Message)
|
||||
return f.Cause == causeDroppedHandshake
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// droppedConnection is the server closing or resetting the connection, never a TLS alert or a certificate refusal.
|
||||
func droppedConnection(err error) bool {
|
||||
return errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) ||
|
||||
errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.ECONNABORTED) || errors.Is(err, syscall.EPIPE)
|
||||
}
|
||||
|
||||
func throttledFailure(err *errx.MailError, retryAt time.Time) *errx.MailError {
|
||||
out := *err
|
||||
f := *err.Failure
|
||||
f.RetryAt = &retryAt
|
||||
f.Cause = ""
|
||||
switch f.Stage {
|
||||
case "greeting", "ehlo", "starttls", "auth", "mail from":
|
||||
// Refused before any recipient was offered, so the send costs the lead nothing.
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"net"
|
||||
"net/textproto"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -30,7 +31,9 @@ func TestServerThrottledNamesOnlyAddressRefusals(t *testing.T) {
|
||||
{"greeting 421", throttleReply("greeting", 421), true},
|
||||
{"auth 454", throttleReply("auth", 454), true},
|
||||
{"rcpt 421", throttleReply("rcpt to", 421), true},
|
||||
{"dropped handshake", tlsFailure(errx.ErrMailServerUnreachable, "starttls", io.EOF), true},
|
||||
{"dropped handshake", startTLSFailure(io.EOF), true},
|
||||
{"reset handshake", startTLSFailure(&net.OpError{Op: "read", Err: syscall.ECONNRESET}), true},
|
||||
{"handshake alert", startTLSFailure(errors.New("remote error: tls: protocol version not supported")), false},
|
||||
{"refused certificate", sendFailure(errx.ErrMailServerUnreachableAt("starttls", errors.New(errx.MailConnectionReasonCertificate)), io.EOF, "starttls", "mailbox"), false},
|
||||
{"no STARTTLS", sendFailure(errx.ErrMailServerUnreachableAt("starttls", errNoSTARTTLS), errNoSTARTTLS, "starttls", "mailbox"), false},
|
||||
{"lost auth connection", throttleReply("auth", 0), false},
|
||||
@@ -50,7 +53,7 @@ func TestHostPaceCoolsOncePerBurstAndBacksOff(t *testing.T) {
|
||||
p := &hostPace{}
|
||||
t0 := time.Date(2026, 11, 3, 18, 0, 0, 0, time.UTC)
|
||||
|
||||
first := p.observe(t0, throttleReply("starttls", 0))
|
||||
first := p.observe(t0, startTLSFailure(io.EOF))
|
||||
if !first.Failure.RetryAt.Equal(t0.Add(paceCooldownBase)) || first.Failure.Cause != errx.SendCauseServerPacing {
|
||||
t.Fatalf("first refusal = %+v", first.Failure)
|
||||
}
|
||||
@@ -74,9 +77,14 @@ func TestHostPaceCoolsOncePerBurstAndBacksOff(t *testing.T) {
|
||||
for i := 0; i < 10; i++ {
|
||||
p.observe(p.coolUntil, throttleReply("greeting", 421))
|
||||
}
|
||||
if got := p.coolUntil.Sub(p.pacedUntil.Add(-pacedWindow)); got != paceCooldownMax {
|
||||
last := p.coolUntil.Add(-paceCooldownMax)
|
||||
p.observe(p.coolUntil, startTLSFailure(io.EOF))
|
||||
if got := p.coolUntil.Sub(last.Add(paceCooldownMax)); got != paceCooldownMax {
|
||||
t.Fatalf("cooldown not capped: %v", got)
|
||||
}
|
||||
if !p.pacedUntil.Equal(p.coolUntil.Add(pacedWindow)) {
|
||||
t.Fatalf("spacing not measured from the end of the cooldown: paced=%v cool=%v", p.pacedUntil, p.coolUntil)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostPaceSpacesSessionsAfterARefusalUntilItRecovers(t *testing.T) {
|
||||
@@ -113,7 +121,7 @@ func TestHostPaceSpacesSessionsAfterARefusalUntilItRecovers(t *testing.T) {
|
||||
func TestHostPaceHoldIsARetryThatNeverReachedTheServer(t *testing.T) {
|
||||
p := &hostPace{}
|
||||
t0 := time.Now()
|
||||
p.observe(t0, throttleReply("starttls", 0))
|
||||
p.observe(t0, startTLSFailure(io.EOF))
|
||||
held := p.wait(context.Background(), t0)
|
||||
if held == nil || held.Failure == nil {
|
||||
t.Fatal("cooling host admitted a session")
|
||||
@@ -135,11 +143,30 @@ func TestHostPaceRefusalAfterRecipientsCostsTheLeadAnAttempt(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPaceForKeysOnServerAndWorkerAddress(t *testing.T) {
|
||||
a := paceFor("SMTP.Example.test", 587, &net.TCPAddr{IP: net.IPv4(192, 0, 2, 1)})
|
||||
if paceFor("smtp.example.test", 587, &net.TCPAddr{IP: net.IPv4(192, 0, 2, 1)}) != a {
|
||||
t.Fatal("host case split the pace")
|
||||
a := paceFor("SMTP.Example.test", &net.TCPAddr{IP: net.IPv4(192, 0, 2, 1)})
|
||||
if paceFor("smtp.example.test.", &net.TCPAddr{IP: net.IPv4(192, 0, 2, 1)}) != a {
|
||||
t.Fatal("host spelling split the pace")
|
||||
}
|
||||
if paceFor("smtp.example.test", 587, &net.TCPAddr{IP: net.IPv4(192, 0, 2, 2)}) == a || paceFor("smtp.example.test", 587, nil) == a {
|
||||
if paceFor("smtp.example.test", &net.TCPAddr{IP: net.IPv4(192, 0, 2, 2)}) == a || paceFor("smtp.example.test", nil) == a {
|
||||
t.Fatal("another worker address shared the pace")
|
||||
}
|
||||
if paceFor("::1", nil) != nil || paceFor("localhost", nil) != nil {
|
||||
t.Fatal("a relay on this machine was paced")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHostPaceReservationYieldsToALaterCooldown(t *testing.T) {
|
||||
p := &hostPace{}
|
||||
now := time.Now()
|
||||
p.pacedUntil = now.Add(pacedWindow)
|
||||
p.nextStart = now.Add(50 * time.Millisecond)
|
||||
done := make(chan *errx.MailError, 1)
|
||||
go func() { done <- p.wait(context.Background(), now) }()
|
||||
p.mu.Lock()
|
||||
p.coolUntil = now.Add(time.Minute)
|
||||
p.mu.Unlock()
|
||||
held := <-done
|
||||
if held == nil || held.Failure.Stage != PaceStage || !held.Failure.RetryAt.Equal(now.Add(time.Minute)) {
|
||||
t.Fatalf("reserved start ran inside a cooldown: %+v", held)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,3 +65,27 @@ func TestLiveSendFailureKeepsRecipientAndMailboxRecoverySeparate(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLiveServerPacingHoldWaitsForItsRetryTime(t *testing.T) {
|
||||
f := newCampaignIsolationFixture(t)
|
||||
retryAt := time.Now().UTC().Add(9 * time.Minute)
|
||||
result := models.SendEmailResult{TaskID: f.request.TaskID, Error: &models.EmailSendError{
|
||||
Code: string(errx.MailErrorCodeServerUnreachable), Message: errx.MailServerPacedMessage,
|
||||
Failure: &errx.SendFailure{Provider: "smtp", Protocol: "smtp", Stage: "pacing", Cause: errx.SendCauseServerPacing,
|
||||
Disposition: errx.SendRetry, Scope: "mailbox", ObservedAt: time.Now().UTC(), RetryAt: &retryAt},
|
||||
}}
|
||||
if err := f.r.FinishOutbound(t.Context(), result.TaskID, f.sender, f.worker, result); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.r.ApplySendResult(t.Context(), result, func(context.Context) error { return nil }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
early, err := f.r.GetSendAdmission(t.Context(), f.org, f.sender, models.InboxProviderSMTPIMAP, retryAt.Add(-time.Minute))
|
||||
if err != nil || early.Allowed || early.RecoveryHold || early.RetryAt == nil || early.RetryAt.Before(retryAt.Add(-time.Second)) {
|
||||
t.Fatalf("held send admitted before the server's retry time: %+v %v", early, err)
|
||||
}
|
||||
late, err := f.r.GetSendAdmission(t.Context(), f.org, f.sender, models.InboxProviderSMTPIMAP, retryAt.Add(time.Second))
|
||||
if err != nil || !late.Allowed || late.RecoveryHold {
|
||||
t.Fatalf("held send not admitted after the retry time: %+v %v", late, err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user