Delivery safety-net: timeouts + bounce classification + visible drops

Combines all five production fixes rolled out on the RabbitMQ branch
into one patch against the NATS/Trove codebase.

1. STARTTLS handshake timeout. TlsConnector::connect in starttls_upgrade
   has no built-in timeout; wrapping it in tokio::time::timeout(
   command_timeout) at both call sites (relay path and try_mx_host)
   prevents a remote that stalls after ACKing 220 STARTTLS from parking
   the delivery future indefinitely and leaking a per-domain pool
   permit on every occurrence.

2. Per-MX attempt backstop. deliver_to_domain now wraps each
   try_mx_host call in a 5-minute tokio::time::timeout. On elapsed the
   next MX is tried; no single await can hang the task.

3. Top-level handler timeout. The MessageHandler impl wraps
   self.deliver() in a 10-minute timeout. On elapsed, logs an error
   and returns HandlerResult::RetryAfter so JetStream re-delivers the
   message after a backoff instead of leaving the task leaked.

4. classify_bounce scans the response text for DNSBL / reputation /
   policy keywords (Spamhaus, SORBS, PBL/SBL/XBL, "poor reputation",
   "blocked using", "temporarily blocked", etc.) and classifies those
   as Block instead of Hard. A 5xx without an enhanced code was
   previously treated as a bad address, so any Spamhaus-worded 550 or
   "poor reputation" 554 flagged recipients for permanent auto-
   suppression even though the problem was ours.

5. Suppressed recipients now go through record_outcome (writing a
   Dropped event and publishing to the events exchange) and, when
   every recipient is suppressed, the message transitions to
   MessageStatus::Dropped. Previously suppressions produced zero rows
   in message_events, zero webhooks, and left the message stuck as
   `queued` indefinitely. The handler log line for Suppressed is
   also promoted from debug to info.

Also adds explicit timeouts to the MTA-STS reqwest client — the
default client has no timeout, so an unresponsive mta-sts.<domain>
policy server would otherwise block the entire delivery task.
This commit is contained in:
Jens Nylander
2026-10-06 13:17:38 +00:00
parent 91112ffd56
commit 62b0208cd7
4 changed files with 202 additions and 31 deletions
+10 -2
View File
@@ -199,9 +199,17 @@ impl Authenticator {
});
};
// 2. HTTPS fetch of the policy file.
// 2. HTTPS fetch of the policy file. Both connect/read and body-read
// are time-bounded: the default reqwest client has no timeout, and a
// slow or unresponsive policy server would otherwise stall the entire
// outbound delivery task indefinitely.
let url = format!("https://mta-sts.{domain}/.well-known/mta-sts.txt");
let response = reqwest::get(&url).await.map_err(|e| {
let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(10))
.connect_timeout(std::time::Duration::from_secs(5))
.build()
.map_err(|e| SentioError::Auth(format!("MTA-STS client build failed: {e}")))?;
let response = client.get(&url).send().await.map_err(|e| {
SentioError::Auth(format!("MTA-STS policy fetch failed for {domain}: {e}"))
})?;
+116 -20
View File
@@ -388,13 +388,32 @@ where
for rcpt in recipients {
match self.suppression_repo.check(tenant_id, rcpt).await {
Ok(true) => {
debug!(rcpt, "recipient suppressed, skipping");
outcomes.push((
rcpt.clone(),
DeliveryOutcome::Suppressed {
reason: format!("{rcpt} is suppressed"),
},
));
info!(
message_id = %message_id,
rcpt,
"recipient suppressed, dropping"
);
let outcome = DeliveryOutcome::Suppressed {
reason: format!("{rcpt} is suppressed"),
};
// Record a Dropped event + webhook payload so the drop is
// visible in message_events (otherwise suppressed sends
// produce zero rows and the message appears "stuck").
self.record_outcome(
tenant_id,
message_id,
rcpt,
&outcome,
retry_count,
&msg.envelope_from,
first_queued_at,
msg.dsn_ret.as_deref(),
msg.dsn_envid.as_deref(),
&msg.dsn_orcpt,
msg,
)
.await;
outcomes.push((rcpt.clone(), outcome));
}
Ok(false) => active_rcpts.push(rcpt.as_str()),
Err(e) => {
@@ -467,6 +486,14 @@ where
.any(|(_, o)| matches!(o, DeliveryOutcome::Deferred { .. }))
{
Some(MessageStatus::Deferred)
} else if !outcomes.is_empty()
&& outcomes
.iter()
.all(|(_, o)| matches!(o, DeliveryOutcome::Suppressed { .. }))
{
// Every recipient suppressed — mark the message terminal as Dropped
// so it does not remain in `queued` indefinitely with no events.
Some(MessageStatus::Dropped)
} else {
None
};
@@ -569,16 +596,24 @@ where
// Try each MX host in preference order.
let mut last_error = None;
// Hard upper bound per MX attempt so no single await can hang the task.
// The individual SMTP command / handshake timeouts are stricter; this is
// a defensive backstop for any future await that lacks its own timeout.
let mx_attempt_timeout = std::time::Duration::from_secs(300);
for mx_host in &mx_result.hosts {
match self
.try_mx_host(
let attempt = tokio::time::timeout(
mx_attempt_timeout,
self.try_mx_host(
mx_host, sender, recipients, message, &tls_req, dsn_ret, dsn_envid, dsn_notify,
dsn_orcpt,
)
.await
{
Ok(outcome) => return outcome,
Err(e) => {
),
)
.await;
match attempt {
Ok(Ok(outcome)) => return outcome,
Ok(Err(e)) => {
warn!(
mx = %mx_host.hostname,
error = %e,
@@ -586,6 +621,18 @@ where
);
last_error = Some(e.to_string());
}
Err(_) => {
warn!(
mx = %mx_host.hostname,
timeout_secs = mx_attempt_timeout.as_secs(),
"MX host delivery attempt timed out, trying next"
);
last_error = Some(format!(
"MX {} attempt timed out after {}s",
mx_host.hostname,
mx_attempt_timeout.as_secs()
));
}
}
}
@@ -764,7 +811,25 @@ where
}
};
let (stream, _buf, conn_config, hostname) = conn.into_parts();
match starttls_upgrade(stream, Arc::new(tls_config), &hostname).await {
let tls_timeout = conn_config.command_timeout;
let upgrade = match tokio::time::timeout(
tls_timeout,
starttls_upgrade(stream, Arc::new(tls_config), &hostname),
)
.await
{
Ok(r) => r,
Err(_) => {
return DeliveryOutcome::Deferred {
response: "Relay STARTTLS handshake timeout".into(),
remote_mta: host.to_string(),
bounce_class: BounceClass::Soft,
retry_count,
next_retry_at: compute_next_retry(retry_count, &self.config),
};
}
};
match upgrade {
Ok((tls_stream, _version)) => {
let (mut tls_conn, _) = match SmtpConnection::new(
tls_stream,
@@ -941,7 +1006,14 @@ where
.map_err(|e| smtp_err(format!("TLS config error: {e}")))?;
let (stream, read_buf, conn_config, hostname) = conn.into_parts();
match starttls_upgrade(stream, Arc::new(tls_config), &hostname).await {
let tls_timeout = conn_config.command_timeout;
let upgrade = tokio::time::timeout(
tls_timeout,
starttls_upgrade(stream, Arc::new(tls_config), &hostname),
)
.await
.map_err(|_| smtp_err("STARTTLS handshake timeout"))?;
match upgrade {
Ok((tls_stream, _version)) => {
// RFC 3207: after STARTTLS, the server does NOT send a new greeting.
// Use from_upgraded to skip greeting read.
@@ -1519,7 +1591,29 @@ where
"processing outbound delivery"
);
let outcomes = self.deliver(&msg, retry_count, first_queued_at).await;
// Hard upper bound on the delivery attempt. If any await inside
// deliver() hangs (unresponsive peer stalling an unprotected network
// call), this prevents the task from leaking indefinitely and holding
// resources (pool permits, DB connections). On elapsed we return
// RetryAfter so JetStream re-delivers after a backoff.
const HANDLER_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(600);
let outcomes = match tokio::time::timeout(
HANDLER_TIMEOUT,
self.deliver(&msg, retry_count, first_queued_at),
)
.await
{
Ok(o) => o,
Err(_) => {
error!(
message_id = %msg.message_id,
timeout_secs = HANDLER_TIMEOUT.as_secs(),
"delivery handler exceeded hard timeout; re-queueing for retry"
);
return HandlerResult::RetryAfter(std::time::Duration::from_secs(300));
}
};
for (_, outcome) in &outcomes {
match outcome {
@@ -1533,7 +1627,7 @@ where
warn!(message_id = %msg.message_id, response, "bounced");
}
DeliveryOutcome::Suppressed { reason } => {
debug!(message_id = %msg.message_id, reason, "suppressed");
info!(message_id = %msg.message_id, reason, "suppressed");
}
}
}
@@ -1599,7 +1693,8 @@ fn classify_response(resp: &SmtpResponse, remote_mta: &str) -> DeliveryOutcome {
remote_mta: remote_mta.to_string(),
}
} else if resp.is_transient() {
let bounce_class = classify_bounce(resp.code, resp.enhanced.as_deref());
let bounce_class =
classify_bounce(resp.code, resp.enhanced.as_deref(), Some(&response_text));
DeliveryOutcome::Deferred {
response: response_text,
remote_mta: remote_mta.to_string(),
@@ -1608,7 +1703,8 @@ fn classify_response(resp: &SmtpResponse, remote_mta: &str) -> DeliveryOutcome {
next_retry_at: Utc::now(),
}
} else {
let bounce_class = classify_bounce(resp.code, resp.enhanced.as_deref());
let bounce_class =
classify_bounce(resp.code, resp.enhanced.as_deref(), Some(&response_text));
DeliveryOutcome::Bounced {
response: response_text,
remote_mta: remote_mta.to_string(),
+75 -8
View File
@@ -40,9 +40,27 @@ pub struct DsnParams {
// Bounce classification
// ──────────────────────────────────────────────────────────────────────────────
/// Classify a bounce from the SMTP response code.
pub fn classify_bounce(code: u16, enhanced: Option<&str>) -> BounceClass {
// Check enhanced status code first.
/// Classify a bounce from the SMTP response.
///
/// `response_text` is the full remote response body — scanned for
/// DNSBL / reputation / policy keywords so a 5xx without an explicit
/// enhanced code isn't mistaken for a "bad address" hard bounce and
/// then auto-suppressed. Any sender-infra reject (Spamhaus, SORBS,
/// "poor reputation", PBL/SBL/XBL, "blocked") must classify as Block.
pub fn classify_bounce(
code: u16,
enhanced: Option<&str>,
response_text: Option<&str>,
) -> BounceClass {
// Keyword scan first: sender-side infrastructure rejections should never
// be treated as recipient-fatal, regardless of code/enhanced.
if let Some(text) = response_text {
if is_sender_reputation_reject(text) {
return BounceClass::Block;
}
}
// Check enhanced status code next.
if let Some(esc) = enhanced {
if esc.starts_with("5.1.") {
return BounceClass::Hard; // Bad address
@@ -63,6 +81,25 @@ pub fn classify_bounce(code: u16, enhanced: Option<&str>) -> BounceClass {
}
}
/// True if the response text indicates the remote rejected us due to our
/// sending infrastructure (DNSBL listing, poor reputation, policy), not
/// due to anything about the recipient address. Such rejections are
/// Block, not Hard — the recipient must not be auto-suppressed.
fn is_sender_reputation_reject(text: &str) -> bool {
let t = text.to_ascii_lowercase();
const DNSBL_HINTS: &[&str] = &[
"spamhaus", "sorbs", "spamcop", "barracuda", "senderscore",
"pbl", "sbl", "xbl", "dnsbl", "rbl", "blacklist", "blocklist",
];
const REPUTATION_HINTS: &[&str] = &[
"poor reputation", "bad reputation", "sender reputation",
"ip reputation", "blocked using", "listed by", "temporarily blocked",
"deferred due to reputation", "mail from this ip",
];
DNSBL_HINTS.iter().any(|k| t.contains(k))
|| REPUTATION_HINTS.iter().any(|k| t.contains(k))
}
/// Map a [`BounceClass`] to an RFC 3464 DSN action value.
fn dsn_action(bounce_class: BounceClass) -> &'static str {
match bounce_class {
@@ -261,19 +298,49 @@ mod tests {
#[test]
fn classify_bounce_hard() {
assert_eq!(classify_bounce(550, None), BounceClass::Hard);
assert_eq!(classify_bounce(550, Some("5.1.1")), BounceClass::Hard);
assert_eq!(classify_bounce(550, None, None), BounceClass::Hard);
assert_eq!(
classify_bounce(550, Some("5.1.1"), None),
BounceClass::Hard
);
}
#[test]
fn classify_bounce_soft() {
assert_eq!(classify_bounce(450, None), BounceClass::Soft);
assert_eq!(classify_bounce(421, Some("4.7.0")), BounceClass::Soft);
assert_eq!(classify_bounce(450, None, None), BounceClass::Soft);
assert_eq!(
classify_bounce(421, Some("4.7.0"), None),
BounceClass::Soft
);
}
#[test]
fn classify_bounce_block() {
assert_eq!(classify_bounce(550, Some("5.7.1")), BounceClass::Block);
assert_eq!(
classify_bounce(550, Some("5.7.1"), None),
BounceClass::Block
);
}
#[test]
fn classify_bounce_dnsbl_text_overrides_hard() {
let resp =
"550 rejected 213.164.219.19 is temporarily blocked. Contact support [pbl]";
assert_eq!(classify_bounce(550, None, Some(resp)), BounceClass::Block);
let resp2 = "550 Client host blocked using Spamhaus";
assert_eq!(classify_bounce(550, None, Some(resp2)), BounceClass::Block);
let resp3 = "554 Your access to this mail system has been rejected due to \
the sending MTA's poor reputation.";
assert_eq!(classify_bounce(554, None, Some(resp3)), BounceClass::Block);
}
#[test]
fn classify_bounce_user_unknown_stays_hard() {
let resp = "550 5.1.1 User unknown";
assert_eq!(
classify_bounce(550, Some("5.1.1"), Some(resp)),
BounceClass::Hard
);
}
#[test]
+1 -1
View File
@@ -481,7 +481,7 @@ where
// Classify. Unparseable code → Hard so we stop retrying.
let class = match (dsn.status_code, dsn.enhanced_status.as_deref()) {
(Some(code), enh) => classify_bounce(code, enh),
(Some(code), enh) => classify_bounce(code, enh, dsn.diagnostic.as_deref()),
_ => BounceClass::Hard,
};