mirror of
https://github.com/truespar/sentio.git
synced 2026-11-17 08:16:11 +00:00
Delivery safety-net: timeouts + bounce classification + visible drops
Combines all five production fixes rolled out on the RabbitMQ branch into one patch against the NATS/Trove codebase. 1. STARTTLS handshake timeout. TlsConnector::connect in starttls_upgrade has no built-in timeout; wrapping it in tokio::time::timeout( command_timeout) at both call sites (relay path and try_mx_host) prevents a remote that stalls after ACKing 220 STARTTLS from parking the delivery future indefinitely and leaking a per-domain pool permit on every occurrence. 2. Per-MX attempt backstop. deliver_to_domain now wraps each try_mx_host call in a 5-minute tokio::time::timeout. On elapsed the next MX is tried; no single await can hang the task. 3. Top-level handler timeout. The MessageHandler impl wraps self.deliver() in a 10-minute timeout. On elapsed, logs an error and returns HandlerResult::RetryAfter so JetStream re-delivers the message after a backoff instead of leaving the task leaked. 4. classify_bounce scans the response text for DNSBL / reputation / policy keywords (Spamhaus, SORBS, PBL/SBL/XBL, "poor reputation", "blocked using", "temporarily blocked", etc.) and classifies those as Block instead of Hard. A 5xx without an enhanced code was previously treated as a bad address, so any Spamhaus-worded 550 or "poor reputation" 554 flagged recipients for permanent auto- suppression even though the problem was ours. 5. Suppressed recipients now go through record_outcome (writing a Dropped event and publishing to the events exchange) and, when every recipient is suppressed, the message transitions to MessageStatus::Dropped. Previously suppressions produced zero rows in message_events, zero webhooks, and left the message stuck as `queued` indefinitely. The handler log line for Suppressed is also promoted from debug to info. Also adds explicit timeouts to the MTA-STS reqwest client — the default client has no timeout, so an unresponsive mta-sts.<domain> policy server would otherwise block the entire delivery task.
This commit is contained in:
@@ -199,9 +199,17 @@ impl Authenticator {
|
||||
});
|
||||
};
|
||||
|
||||
// 2. HTTPS fetch of the policy file.
|
||||
// 2. HTTPS fetch of the policy file. Both connect/read and body-read
|
||||
// are time-bounded: the default reqwest client has no timeout, and a
|
||||
// slow or unresponsive policy server would otherwise stall the entire
|
||||
// outbound delivery task indefinitely.
|
||||
let url = format!("https://mta-sts.{domain}/.well-known/mta-sts.txt");
|
||||
let response = reqwest::get(&url).await.map_err(|e| {
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(10))
|
||||
.connect_timeout(std::time::Duration::from_secs(5))
|
||||
.build()
|
||||
.map_err(|e| SentioError::Auth(format!("MTA-STS client build failed: {e}")))?;
|
||||
let response = client.get(&url).send().await.map_err(|e| {
|
||||
SentioError::Auth(format!("MTA-STS policy fetch failed for {domain}: {e}"))
|
||||
})?;
|
||||
|
||||
|
||||
@@ -388,13 +388,32 @@ where
|
||||
for rcpt in recipients {
|
||||
match self.suppression_repo.check(tenant_id, rcpt).await {
|
||||
Ok(true) => {
|
||||
debug!(rcpt, "recipient suppressed, skipping");
|
||||
outcomes.push((
|
||||
rcpt.clone(),
|
||||
DeliveryOutcome::Suppressed {
|
||||
reason: format!("{rcpt} is suppressed"),
|
||||
},
|
||||
));
|
||||
info!(
|
||||
message_id = %message_id,
|
||||
rcpt,
|
||||
"recipient suppressed, dropping"
|
||||
);
|
||||
let outcome = DeliveryOutcome::Suppressed {
|
||||
reason: format!("{rcpt} is suppressed"),
|
||||
};
|
||||
// Record a Dropped event + webhook payload so the drop is
|
||||
// visible in message_events (otherwise suppressed sends
|
||||
// produce zero rows and the message appears "stuck").
|
||||
self.record_outcome(
|
||||
tenant_id,
|
||||
message_id,
|
||||
rcpt,
|
||||
&outcome,
|
||||
retry_count,
|
||||
&msg.envelope_from,
|
||||
first_queued_at,
|
||||
msg.dsn_ret.as_deref(),
|
||||
msg.dsn_envid.as_deref(),
|
||||
&msg.dsn_orcpt,
|
||||
msg,
|
||||
)
|
||||
.await;
|
||||
outcomes.push((rcpt.clone(), outcome));
|
||||
}
|
||||
Ok(false) => active_rcpts.push(rcpt.as_str()),
|
||||
Err(e) => {
|
||||
@@ -467,6 +486,14 @@ where
|
||||
.any(|(_, o)| matches!(o, DeliveryOutcome::Deferred { .. }))
|
||||
{
|
||||
Some(MessageStatus::Deferred)
|
||||
} else if !outcomes.is_empty()
|
||||
&& outcomes
|
||||
.iter()
|
||||
.all(|(_, o)| matches!(o, DeliveryOutcome::Suppressed { .. }))
|
||||
{
|
||||
// Every recipient suppressed — mark the message terminal as Dropped
|
||||
// so it does not remain in `queued` indefinitely with no events.
|
||||
Some(MessageStatus::Dropped)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -569,16 +596,24 @@ where
|
||||
// Try each MX host in preference order.
|
||||
let mut last_error = None;
|
||||
|
||||
// Hard upper bound per MX attempt so no single await can hang the task.
|
||||
// The individual SMTP command / handshake timeouts are stricter; this is
|
||||
// a defensive backstop for any future await that lacks its own timeout.
|
||||
let mx_attempt_timeout = std::time::Duration::from_secs(300);
|
||||
|
||||
for mx_host in &mx_result.hosts {
|
||||
match self
|
||||
.try_mx_host(
|
||||
let attempt = tokio::time::timeout(
|
||||
mx_attempt_timeout,
|
||||
self.try_mx_host(
|
||||
mx_host, sender, recipients, message, &tls_req, dsn_ret, dsn_envid, dsn_notify,
|
||||
dsn_orcpt,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(outcome) => return outcome,
|
||||
Err(e) => {
|
||||
),
|
||||
)
|
||||
.await;
|
||||
|
||||
match attempt {
|
||||
Ok(Ok(outcome)) => return outcome,
|
||||
Ok(Err(e)) => {
|
||||
warn!(
|
||||
mx = %mx_host.hostname,
|
||||
error = %e,
|
||||
@@ -586,6 +621,18 @@ where
|
||||
);
|
||||
last_error = Some(e.to_string());
|
||||
}
|
||||
Err(_) => {
|
||||
warn!(
|
||||
mx = %mx_host.hostname,
|
||||
timeout_secs = mx_attempt_timeout.as_secs(),
|
||||
"MX host delivery attempt timed out, trying next"
|
||||
);
|
||||
last_error = Some(format!(
|
||||
"MX {} attempt timed out after {}s",
|
||||
mx_host.hostname,
|
||||
mx_attempt_timeout.as_secs()
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -764,7 +811,25 @@ where
|
||||
}
|
||||
};
|
||||
let (stream, _buf, conn_config, hostname) = conn.into_parts();
|
||||
match starttls_upgrade(stream, Arc::new(tls_config), &hostname).await {
|
||||
let tls_timeout = conn_config.command_timeout;
|
||||
let upgrade = match tokio::time::timeout(
|
||||
tls_timeout,
|
||||
starttls_upgrade(stream, Arc::new(tls_config), &hostname),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(r) => r,
|
||||
Err(_) => {
|
||||
return DeliveryOutcome::Deferred {
|
||||
response: "Relay STARTTLS handshake timeout".into(),
|
||||
remote_mta: host.to_string(),
|
||||
bounce_class: BounceClass::Soft,
|
||||
retry_count,
|
||||
next_retry_at: compute_next_retry(retry_count, &self.config),
|
||||
};
|
||||
}
|
||||
};
|
||||
match upgrade {
|
||||
Ok((tls_stream, _version)) => {
|
||||
let (mut tls_conn, _) = match SmtpConnection::new(
|
||||
tls_stream,
|
||||
@@ -941,7 +1006,14 @@ where
|
||||
.map_err(|e| smtp_err(format!("TLS config error: {e}")))?;
|
||||
let (stream, read_buf, conn_config, hostname) = conn.into_parts();
|
||||
|
||||
match starttls_upgrade(stream, Arc::new(tls_config), &hostname).await {
|
||||
let tls_timeout = conn_config.command_timeout;
|
||||
let upgrade = tokio::time::timeout(
|
||||
tls_timeout,
|
||||
starttls_upgrade(stream, Arc::new(tls_config), &hostname),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| smtp_err("STARTTLS handshake timeout"))?;
|
||||
match upgrade {
|
||||
Ok((tls_stream, _version)) => {
|
||||
// RFC 3207: after STARTTLS, the server does NOT send a new greeting.
|
||||
// Use from_upgraded to skip greeting read.
|
||||
@@ -1519,7 +1591,29 @@ where
|
||||
"processing outbound delivery"
|
||||
);
|
||||
|
||||
let outcomes = self.deliver(&msg, retry_count, first_queued_at).await;
|
||||
// Hard upper bound on the delivery attempt. If any await inside
|
||||
// deliver() hangs (unresponsive peer stalling an unprotected network
|
||||
// call), this prevents the task from leaking indefinitely and holding
|
||||
// resources (pool permits, DB connections). On elapsed we return
|
||||
// RetryAfter so JetStream re-delivers after a backoff.
|
||||
const HANDLER_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(600);
|
||||
|
||||
let outcomes = match tokio::time::timeout(
|
||||
HANDLER_TIMEOUT,
|
||||
self.deliver(&msg, retry_count, first_queued_at),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(o) => o,
|
||||
Err(_) => {
|
||||
error!(
|
||||
message_id = %msg.message_id,
|
||||
timeout_secs = HANDLER_TIMEOUT.as_secs(),
|
||||
"delivery handler exceeded hard timeout; re-queueing for retry"
|
||||
);
|
||||
return HandlerResult::RetryAfter(std::time::Duration::from_secs(300));
|
||||
}
|
||||
};
|
||||
|
||||
for (_, outcome) in &outcomes {
|
||||
match outcome {
|
||||
@@ -1533,7 +1627,7 @@ where
|
||||
warn!(message_id = %msg.message_id, response, "bounced");
|
||||
}
|
||||
DeliveryOutcome::Suppressed { reason } => {
|
||||
debug!(message_id = %msg.message_id, reason, "suppressed");
|
||||
info!(message_id = %msg.message_id, reason, "suppressed");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1599,7 +1693,8 @@ fn classify_response(resp: &SmtpResponse, remote_mta: &str) -> DeliveryOutcome {
|
||||
remote_mta: remote_mta.to_string(),
|
||||
}
|
||||
} else if resp.is_transient() {
|
||||
let bounce_class = classify_bounce(resp.code, resp.enhanced.as_deref());
|
||||
let bounce_class =
|
||||
classify_bounce(resp.code, resp.enhanced.as_deref(), Some(&response_text));
|
||||
DeliveryOutcome::Deferred {
|
||||
response: response_text,
|
||||
remote_mta: remote_mta.to_string(),
|
||||
@@ -1608,7 +1703,8 @@ fn classify_response(resp: &SmtpResponse, remote_mta: &str) -> DeliveryOutcome {
|
||||
next_retry_at: Utc::now(),
|
||||
}
|
||||
} else {
|
||||
let bounce_class = classify_bounce(resp.code, resp.enhanced.as_deref());
|
||||
let bounce_class =
|
||||
classify_bounce(resp.code, resp.enhanced.as_deref(), Some(&response_text));
|
||||
DeliveryOutcome::Bounced {
|
||||
response: response_text,
|
||||
remote_mta: remote_mta.to_string(),
|
||||
|
||||
@@ -40,9 +40,27 @@ pub struct DsnParams {
|
||||
// Bounce classification
|
||||
// ──────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// Classify a bounce from the SMTP response code.
|
||||
pub fn classify_bounce(code: u16, enhanced: Option<&str>) -> BounceClass {
|
||||
// Check enhanced status code first.
|
||||
/// Classify a bounce from the SMTP response.
|
||||
///
|
||||
/// `response_text` is the full remote response body — scanned for
|
||||
/// DNSBL / reputation / policy keywords so a 5xx without an explicit
|
||||
/// enhanced code isn't mistaken for a "bad address" hard bounce and
|
||||
/// then auto-suppressed. Any sender-infra reject (Spamhaus, SORBS,
|
||||
/// "poor reputation", PBL/SBL/XBL, "blocked") must classify as Block.
|
||||
pub fn classify_bounce(
|
||||
code: u16,
|
||||
enhanced: Option<&str>,
|
||||
response_text: Option<&str>,
|
||||
) -> BounceClass {
|
||||
// Keyword scan first: sender-side infrastructure rejections should never
|
||||
// be treated as recipient-fatal, regardless of code/enhanced.
|
||||
if let Some(text) = response_text {
|
||||
if is_sender_reputation_reject(text) {
|
||||
return BounceClass::Block;
|
||||
}
|
||||
}
|
||||
|
||||
// Check enhanced status code next.
|
||||
if let Some(esc) = enhanced {
|
||||
if esc.starts_with("5.1.") {
|
||||
return BounceClass::Hard; // Bad address
|
||||
@@ -63,6 +81,25 @@ pub fn classify_bounce(code: u16, enhanced: Option<&str>) -> BounceClass {
|
||||
}
|
||||
}
|
||||
|
||||
/// True if the response text indicates the remote rejected us due to our
|
||||
/// sending infrastructure (DNSBL listing, poor reputation, policy), not
|
||||
/// due to anything about the recipient address. Such rejections are
|
||||
/// Block, not Hard — the recipient must not be auto-suppressed.
|
||||
fn is_sender_reputation_reject(text: &str) -> bool {
|
||||
let t = text.to_ascii_lowercase();
|
||||
const DNSBL_HINTS: &[&str] = &[
|
||||
"spamhaus", "sorbs", "spamcop", "barracuda", "senderscore",
|
||||
"pbl", "sbl", "xbl", "dnsbl", "rbl", "blacklist", "blocklist",
|
||||
];
|
||||
const REPUTATION_HINTS: &[&str] = &[
|
||||
"poor reputation", "bad reputation", "sender reputation",
|
||||
"ip reputation", "blocked using", "listed by", "temporarily blocked",
|
||||
"deferred due to reputation", "mail from this ip",
|
||||
];
|
||||
DNSBL_HINTS.iter().any(|k| t.contains(k))
|
||||
|| REPUTATION_HINTS.iter().any(|k| t.contains(k))
|
||||
}
|
||||
|
||||
/// Map a [`BounceClass`] to an RFC 3464 DSN action value.
|
||||
fn dsn_action(bounce_class: BounceClass) -> &'static str {
|
||||
match bounce_class {
|
||||
@@ -261,19 +298,49 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn classify_bounce_hard() {
|
||||
assert_eq!(classify_bounce(550, None), BounceClass::Hard);
|
||||
assert_eq!(classify_bounce(550, Some("5.1.1")), BounceClass::Hard);
|
||||
assert_eq!(classify_bounce(550, None, None), BounceClass::Hard);
|
||||
assert_eq!(
|
||||
classify_bounce(550, Some("5.1.1"), None),
|
||||
BounceClass::Hard
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_bounce_soft() {
|
||||
assert_eq!(classify_bounce(450, None), BounceClass::Soft);
|
||||
assert_eq!(classify_bounce(421, Some("4.7.0")), BounceClass::Soft);
|
||||
assert_eq!(classify_bounce(450, None, None), BounceClass::Soft);
|
||||
assert_eq!(
|
||||
classify_bounce(421, Some("4.7.0"), None),
|
||||
BounceClass::Soft
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_bounce_block() {
|
||||
assert_eq!(classify_bounce(550, Some("5.7.1")), BounceClass::Block);
|
||||
assert_eq!(
|
||||
classify_bounce(550, Some("5.7.1"), None),
|
||||
BounceClass::Block
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_bounce_dnsbl_text_overrides_hard() {
|
||||
let resp =
|
||||
"550 rejected 213.164.219.19 is temporarily blocked. Contact support [pbl]";
|
||||
assert_eq!(classify_bounce(550, None, Some(resp)), BounceClass::Block);
|
||||
let resp2 = "550 Client host blocked using Spamhaus";
|
||||
assert_eq!(classify_bounce(550, None, Some(resp2)), BounceClass::Block);
|
||||
let resp3 = "554 Your access to this mail system has been rejected due to \
|
||||
the sending MTA's poor reputation.";
|
||||
assert_eq!(classify_bounce(554, None, Some(resp3)), BounceClass::Block);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_bounce_user_unknown_stays_hard() {
|
||||
let resp = "550 5.1.1 User unknown";
|
||||
assert_eq!(
|
||||
classify_bounce(550, Some("5.1.1"), Some(resp)),
|
||||
BounceClass::Hard
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -481,7 +481,7 @@ where
|
||||
|
||||
// Classify. Unparseable code → Hard so we stop retrying.
|
||||
let class = match (dsn.status_code, dsn.enhanced_status.as_deref()) {
|
||||
(Some(code), enh) => classify_bounce(code, enh),
|
||||
(Some(code), enh) => classify_bounce(code, enh, dsn.diagnostic.as_deref()),
|
||||
_ => BounceClass::Hard,
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user