From 62b0208cd7693322abacaedb2362e72d90b00c7d Mon Sep 17 00:00:00 2001 From: Jens Nylander Date: Tue, 6 Oct 2026 13:17:38 +0000 Subject: [PATCH] Delivery safety-net: timeouts + bounce classification + visible drops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Combines all five production fixes rolled out on the RabbitMQ branch into one patch against the NATS/Trove codebase. 1. STARTTLS handshake timeout. TlsConnector::connect in starttls_upgrade has no built-in timeout; wrapping it in tokio::time::timeout( command_timeout) at both call sites (relay path and try_mx_host) prevents a remote that stalls after ACKing 220 STARTTLS from parking the delivery future indefinitely and leaking a per-domain pool permit on every occurrence. 2. Per-MX attempt backstop. deliver_to_domain now wraps each try_mx_host call in a 5-minute tokio::time::timeout. On elapsed the next MX is tried; no single await can hang the task. 3. Top-level handler timeout. The MessageHandler impl wraps self.deliver() in a 10-minute timeout. On elapsed, logs an error and returns HandlerResult::RetryAfter so JetStream re-delivers the message after a backoff instead of leaving the task leaked. 4. classify_bounce scans the response text for DNSBL / reputation / policy keywords (Spamhaus, SORBS, PBL/SBL/XBL, "poor reputation", "blocked using", "temporarily blocked", etc.) and classifies those as Block instead of Hard. A 5xx without an enhanced code was previously treated as a bad address, so any Spamhaus-worded 550 or "poor reputation" 554 flagged recipients for permanent auto- suppression even though the problem was ours. 5. Suppressed recipients now go through record_outcome (writing a Dropped event and publishing to the events exchange) and, when every recipient is suppressed, the message transitions to MessageStatus::Dropped. Previously suppressions produced zero rows in message_events, zero webhooks, and left the message stuck as `queued` indefinitely. The handler log line for Suppressed is also promoted from debug to info. Also adds explicit timeouts to the MTA-STS reqwest client — the default client has no timeout, so an unresponsive mta-sts. policy server would otherwise block the entire delivery task. --- crates/sentio-auth/src/mta_sts.rs | 12 +- crates/sentio-smtp-client/src/delivery.rs | 136 ++++++++++++++++++---- crates/sentio-smtp-client/src/headers.rs | 83 +++++++++++-- crates/sentio-smtp-server/src/pipeline.rs | 2 +- 4 files changed, 202 insertions(+), 31 deletions(-) diff --git a/crates/sentio-auth/src/mta_sts.rs b/crates/sentio-auth/src/mta_sts.rs index bcb7a5d..9e68b2c 100644 --- a/crates/sentio-auth/src/mta_sts.rs +++ b/crates/sentio-auth/src/mta_sts.rs @@ -199,9 +199,17 @@ impl Authenticator { }); }; - // 2. HTTPS fetch of the policy file. + // 2. HTTPS fetch of the policy file. Both connect/read and body-read + // are time-bounded: the default reqwest client has no timeout, and a + // slow or unresponsive policy server would otherwise stall the entire + // outbound delivery task indefinitely. let url = format!("https://mta-sts.{domain}/.well-known/mta-sts.txt"); - let response = reqwest::get(&url).await.map_err(|e| { + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(10)) + .connect_timeout(std::time::Duration::from_secs(5)) + .build() + .map_err(|e| SentioError::Auth(format!("MTA-STS client build failed: {e}")))?; + let response = client.get(&url).send().await.map_err(|e| { SentioError::Auth(format!("MTA-STS policy fetch failed for {domain}: {e}")) })?; diff --git a/crates/sentio-smtp-client/src/delivery.rs b/crates/sentio-smtp-client/src/delivery.rs index 3ffc548..5375219 100644 --- a/crates/sentio-smtp-client/src/delivery.rs +++ b/crates/sentio-smtp-client/src/delivery.rs @@ -388,13 +388,32 @@ where for rcpt in recipients { match self.suppression_repo.check(tenant_id, rcpt).await { Ok(true) => { - debug!(rcpt, "recipient suppressed, skipping"); - outcomes.push(( - rcpt.clone(), - DeliveryOutcome::Suppressed { - reason: format!("{rcpt} is suppressed"), - }, - )); + info!( + message_id = %message_id, + rcpt, + "recipient suppressed, dropping" + ); + let outcome = DeliveryOutcome::Suppressed { + reason: format!("{rcpt} is suppressed"), + }; + // Record a Dropped event + webhook payload so the drop is + // visible in message_events (otherwise suppressed sends + // produce zero rows and the message appears "stuck"). + self.record_outcome( + tenant_id, + message_id, + rcpt, + &outcome, + retry_count, + &msg.envelope_from, + first_queued_at, + msg.dsn_ret.as_deref(), + msg.dsn_envid.as_deref(), + &msg.dsn_orcpt, + msg, + ) + .await; + outcomes.push((rcpt.clone(), outcome)); } Ok(false) => active_rcpts.push(rcpt.as_str()), Err(e) => { @@ -467,6 +486,14 @@ where .any(|(_, o)| matches!(o, DeliveryOutcome::Deferred { .. })) { Some(MessageStatus::Deferred) + } else if !outcomes.is_empty() + && outcomes + .iter() + .all(|(_, o)| matches!(o, DeliveryOutcome::Suppressed { .. })) + { + // Every recipient suppressed — mark the message terminal as Dropped + // so it does not remain in `queued` indefinitely with no events. + Some(MessageStatus::Dropped) } else { None }; @@ -569,16 +596,24 @@ where // Try each MX host in preference order. let mut last_error = None; + // Hard upper bound per MX attempt so no single await can hang the task. + // The individual SMTP command / handshake timeouts are stricter; this is + // a defensive backstop for any future await that lacks its own timeout. + let mx_attempt_timeout = std::time::Duration::from_secs(300); + for mx_host in &mx_result.hosts { - match self - .try_mx_host( + let attempt = tokio::time::timeout( + mx_attempt_timeout, + self.try_mx_host( mx_host, sender, recipients, message, &tls_req, dsn_ret, dsn_envid, dsn_notify, dsn_orcpt, - ) - .await - { - Ok(outcome) => return outcome, - Err(e) => { + ), + ) + .await; + + match attempt { + Ok(Ok(outcome)) => return outcome, + Ok(Err(e)) => { warn!( mx = %mx_host.hostname, error = %e, @@ -586,6 +621,18 @@ where ); last_error = Some(e.to_string()); } + Err(_) => { + warn!( + mx = %mx_host.hostname, + timeout_secs = mx_attempt_timeout.as_secs(), + "MX host delivery attempt timed out, trying next" + ); + last_error = Some(format!( + "MX {} attempt timed out after {}s", + mx_host.hostname, + mx_attempt_timeout.as_secs() + )); + } } } @@ -764,7 +811,25 @@ where } }; let (stream, _buf, conn_config, hostname) = conn.into_parts(); - match starttls_upgrade(stream, Arc::new(tls_config), &hostname).await { + let tls_timeout = conn_config.command_timeout; + let upgrade = match tokio::time::timeout( + tls_timeout, + starttls_upgrade(stream, Arc::new(tls_config), &hostname), + ) + .await + { + Ok(r) => r, + Err(_) => { + return DeliveryOutcome::Deferred { + response: "Relay STARTTLS handshake timeout".into(), + remote_mta: host.to_string(), + bounce_class: BounceClass::Soft, + retry_count, + next_retry_at: compute_next_retry(retry_count, &self.config), + }; + } + }; + match upgrade { Ok((tls_stream, _version)) => { let (mut tls_conn, _) = match SmtpConnection::new( tls_stream, @@ -941,7 +1006,14 @@ where .map_err(|e| smtp_err(format!("TLS config error: {e}")))?; let (stream, read_buf, conn_config, hostname) = conn.into_parts(); - match starttls_upgrade(stream, Arc::new(tls_config), &hostname).await { + let tls_timeout = conn_config.command_timeout; + let upgrade = tokio::time::timeout( + tls_timeout, + starttls_upgrade(stream, Arc::new(tls_config), &hostname), + ) + .await + .map_err(|_| smtp_err("STARTTLS handshake timeout"))?; + match upgrade { Ok((tls_stream, _version)) => { // RFC 3207: after STARTTLS, the server does NOT send a new greeting. // Use from_upgraded to skip greeting read. @@ -1519,7 +1591,29 @@ where "processing outbound delivery" ); - let outcomes = self.deliver(&msg, retry_count, first_queued_at).await; + // Hard upper bound on the delivery attempt. If any await inside + // deliver() hangs (unresponsive peer stalling an unprotected network + // call), this prevents the task from leaking indefinitely and holding + // resources (pool permits, DB connections). On elapsed we return + // RetryAfter so JetStream re-delivers after a backoff. + const HANDLER_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(600); + + let outcomes = match tokio::time::timeout( + HANDLER_TIMEOUT, + self.deliver(&msg, retry_count, first_queued_at), + ) + .await + { + Ok(o) => o, + Err(_) => { + error!( + message_id = %msg.message_id, + timeout_secs = HANDLER_TIMEOUT.as_secs(), + "delivery handler exceeded hard timeout; re-queueing for retry" + ); + return HandlerResult::RetryAfter(std::time::Duration::from_secs(300)); + } + }; for (_, outcome) in &outcomes { match outcome { @@ -1533,7 +1627,7 @@ where warn!(message_id = %msg.message_id, response, "bounced"); } DeliveryOutcome::Suppressed { reason } => { - debug!(message_id = %msg.message_id, reason, "suppressed"); + info!(message_id = %msg.message_id, reason, "suppressed"); } } } @@ -1599,7 +1693,8 @@ fn classify_response(resp: &SmtpResponse, remote_mta: &str) -> DeliveryOutcome { remote_mta: remote_mta.to_string(), } } else if resp.is_transient() { - let bounce_class = classify_bounce(resp.code, resp.enhanced.as_deref()); + let bounce_class = + classify_bounce(resp.code, resp.enhanced.as_deref(), Some(&response_text)); DeliveryOutcome::Deferred { response: response_text, remote_mta: remote_mta.to_string(), @@ -1608,7 +1703,8 @@ fn classify_response(resp: &SmtpResponse, remote_mta: &str) -> DeliveryOutcome { next_retry_at: Utc::now(), } } else { - let bounce_class = classify_bounce(resp.code, resp.enhanced.as_deref()); + let bounce_class = + classify_bounce(resp.code, resp.enhanced.as_deref(), Some(&response_text)); DeliveryOutcome::Bounced { response: response_text, remote_mta: remote_mta.to_string(), diff --git a/crates/sentio-smtp-client/src/headers.rs b/crates/sentio-smtp-client/src/headers.rs index 964793f..e7db094 100644 --- a/crates/sentio-smtp-client/src/headers.rs +++ b/crates/sentio-smtp-client/src/headers.rs @@ -40,9 +40,27 @@ pub struct DsnParams { // Bounce classification // ────────────────────────────────────────────────────────────────────────────── -/// Classify a bounce from the SMTP response code. -pub fn classify_bounce(code: u16, enhanced: Option<&str>) -> BounceClass { - // Check enhanced status code first. +/// Classify a bounce from the SMTP response. +/// +/// `response_text` is the full remote response body — scanned for +/// DNSBL / reputation / policy keywords so a 5xx without an explicit +/// enhanced code isn't mistaken for a "bad address" hard bounce and +/// then auto-suppressed. Any sender-infra reject (Spamhaus, SORBS, +/// "poor reputation", PBL/SBL/XBL, "blocked") must classify as Block. +pub fn classify_bounce( + code: u16, + enhanced: Option<&str>, + response_text: Option<&str>, +) -> BounceClass { + // Keyword scan first: sender-side infrastructure rejections should never + // be treated as recipient-fatal, regardless of code/enhanced. + if let Some(text) = response_text { + if is_sender_reputation_reject(text) { + return BounceClass::Block; + } + } + + // Check enhanced status code next. if let Some(esc) = enhanced { if esc.starts_with("5.1.") { return BounceClass::Hard; // Bad address @@ -63,6 +81,25 @@ pub fn classify_bounce(code: u16, enhanced: Option<&str>) -> BounceClass { } } +/// True if the response text indicates the remote rejected us due to our +/// sending infrastructure (DNSBL listing, poor reputation, policy), not +/// due to anything about the recipient address. Such rejections are +/// Block, not Hard — the recipient must not be auto-suppressed. +fn is_sender_reputation_reject(text: &str) -> bool { + let t = text.to_ascii_lowercase(); + const DNSBL_HINTS: &[&str] = &[ + "spamhaus", "sorbs", "spamcop", "barracuda", "senderscore", + "pbl", "sbl", "xbl", "dnsbl", "rbl", "blacklist", "blocklist", + ]; + const REPUTATION_HINTS: &[&str] = &[ + "poor reputation", "bad reputation", "sender reputation", + "ip reputation", "blocked using", "listed by", "temporarily blocked", + "deferred due to reputation", "mail from this ip", + ]; + DNSBL_HINTS.iter().any(|k| t.contains(k)) + || REPUTATION_HINTS.iter().any(|k| t.contains(k)) +} + /// Map a [`BounceClass`] to an RFC 3464 DSN action value. fn dsn_action(bounce_class: BounceClass) -> &'static str { match bounce_class { @@ -261,19 +298,49 @@ mod tests { #[test] fn classify_bounce_hard() { - assert_eq!(classify_bounce(550, None), BounceClass::Hard); - assert_eq!(classify_bounce(550, Some("5.1.1")), BounceClass::Hard); + assert_eq!(classify_bounce(550, None, None), BounceClass::Hard); + assert_eq!( + classify_bounce(550, Some("5.1.1"), None), + BounceClass::Hard + ); } #[test] fn classify_bounce_soft() { - assert_eq!(classify_bounce(450, None), BounceClass::Soft); - assert_eq!(classify_bounce(421, Some("4.7.0")), BounceClass::Soft); + assert_eq!(classify_bounce(450, None, None), BounceClass::Soft); + assert_eq!( + classify_bounce(421, Some("4.7.0"), None), + BounceClass::Soft + ); } #[test] fn classify_bounce_block() { - assert_eq!(classify_bounce(550, Some("5.7.1")), BounceClass::Block); + assert_eq!( + classify_bounce(550, Some("5.7.1"), None), + BounceClass::Block + ); + } + + #[test] + fn classify_bounce_dnsbl_text_overrides_hard() { + let resp = + "550 rejected 213.164.219.19 is temporarily blocked. Contact support [pbl]"; + assert_eq!(classify_bounce(550, None, Some(resp)), BounceClass::Block); + let resp2 = "550 Client host blocked using Spamhaus"; + assert_eq!(classify_bounce(550, None, Some(resp2)), BounceClass::Block); + let resp3 = "554 Your access to this mail system has been rejected due to \ + the sending MTA's poor reputation."; + assert_eq!(classify_bounce(554, None, Some(resp3)), BounceClass::Block); + } + + #[test] + fn classify_bounce_user_unknown_stays_hard() { + let resp = "550 5.1.1 User unknown"; + assert_eq!( + classify_bounce(550, Some("5.1.1"), Some(resp)), + BounceClass::Hard + ); } #[test] diff --git a/crates/sentio-smtp-server/src/pipeline.rs b/crates/sentio-smtp-server/src/pipeline.rs index 0392d71..ed5ea89 100644 --- a/crates/sentio-smtp-server/src/pipeline.rs +++ b/crates/sentio-smtp-server/src/pipeline.rs @@ -481,7 +481,7 @@ where // Classify. Unparseable code → Hard so we stop retrying. let class = match (dsn.status_code, dsn.enhanced_status.as_deref()) { - (Some(code), enh) => classify_bounce(code, enh), + (Some(code), enh) => classify_bounce(code, enh, dsn.diagnostic.as_deref()), _ => BounceClass::Hard, };