From d5b13a263bf5250737c4dd354bda47f1c6f6b9e6 Mon Sep 17 00:00:00 2001 From: Wez Furlong Date: Thu, 2 Mar 2023 08:58:09 -0700 Subject: [PATCH] Add kumo.configure_bounce_classifier and connect it to the logger --- Cargo.lock | 1 + crates/bounce-classify/src/lib.rs | 15 +++- crates/kumod/Cargo.toml | 1 + crates/kumod/src/logging.rs | 44 +++++++++- crates/kumod/src/mod_kumo.rs | 12 ++- .../kumo/configure_bounce_classifier.md | 82 +++++++++++++++++++ 6 files changed, 149 insertions(+), 6 deletions(-) create mode 100644 docs/reference/kumo/configure_bounce_classifier.md diff --git a/Cargo.lock b/Cargo.lock index 3154c78e..d82005d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1281,6 +1281,7 @@ dependencies = [ "axum-client-ip", "axum-server", "base64 0.13.1", + "bounce-classify", "caps", "chrono", "cidr", diff --git a/crates/bounce-classify/src/lib.rs b/crates/bounce-classify/src/lib.rs index 11c0c0f3..c7401e31 100644 --- a/crates/bounce-classify/src/lib.rs +++ b/crates/bounce-classify/src/lib.rs @@ -173,9 +173,18 @@ mod test { ("552 4.2.2 mailbox is stuffed", BounceClass::QuotaIssues), ("552 4.2.2 mailbox is stuffed", BounceClass::QuotaIssues), ("352 5.2.2 mailbox is stuffed", BounceClass::Uncategorized), - ("525 4.7.13 user account is disabled", BounceClass::InactiveMailbox), - ("551 4.7.17 mailbox owner has changed", BounceClass::InvalidRecipient), - ("551 4.7.18 domain owner has changed", BounceClass::BadDomain), + ( + "525 4.7.13 user account is disabled", + BounceClass::InactiveMailbox, + ), + ( + "551 4.7.17 mailbox owner has changed", + BounceClass::InvalidRecipient, + ), + ( + "551 4.7.18 domain owner has changed", + BounceClass::BadDomain, + ), ]; for &(input, output) in corpus { diff --git a/crates/kumod/Cargo.toml b/crates/kumod/Cargo.toml index 6220df82..54a4f443 100644 --- a/crates/kumod/Cargo.toml +++ b/crates/kumod/Cargo.toml @@ -12,6 +12,7 @@ axum = "0.6" axum-client-ip = "0.4" axum-server = {version="0.4", features=["tls-rustls"]} base64 = "0.13" +bounce-classify = {path="../bounce-classify"} caps = "0.5" chrono = {version="0.4", features=["serde"]} cidr = {version="0.2", features=["serde"]} diff --git a/crates/kumod/src/logging.rs b/crates/kumod/src/logging.rs index 183b98b0..aa02b3f7 100644 --- a/crates/kumod/src/logging.rs +++ b/crates/kumod/src/logging.rs @@ -1,6 +1,7 @@ use crate::mx::ResolvedAddress; -use anyhow::Context; +use anyhow::{anyhow, Context}; use async_channel::{Receiver, Sender}; +use bounce_classify::{BounceClass, BounceClassifier, BounceClassifierBuilder}; use chrono::{DateTime, Utc}; use message::Message; use once_cell::sync::OnceCell; @@ -14,6 +15,39 @@ use std::thread::JoinHandle; use zstd::stream::write::{AutoFinishEncoder, Encoder}; static LOGGER: OnceCell = OnceCell::new(); +static CLASSIFY: OnceCell = OnceCell::new(); + +#[derive(Deserialize, Clone, Debug)] +pub struct ClassifierParams { + pub files: Vec, +} + +impl ClassifierParams { + pub fn register(&self) -> anyhow::Result<()> { + let mut builder = BounceClassifierBuilder::new(); + for file_name in &self.files { + if file_name.ends_with(".json") { + builder + .merge_json_file(file_name) + .map_err(|err| anyhow!("{err}"))?; + } else if file_name.ends_with(".toml") { + builder + .merge_toml_file(file_name) + .map_err(|err| anyhow!("{err}"))?; + } else { + anyhow::bail!("{file_name}: classifier files must have either .toml or .json filename extension"); + } + } + + let classifier = builder.build().map_err(|err| anyhow!("{err}"))?; + + CLASSIFY + .set(classifier) + .map_err(|_| anyhow::anyhow!("classifieer already initialized"))?; + + Ok(()) + } +} #[derive(Deserialize, Clone, Debug)] pub struct LogFileParams { @@ -98,8 +132,11 @@ impl Logger { fn do_record( params: &LogFileParams, file: &mut Option, - record: JsonLogRecord, + mut record: JsonLogRecord, ) -> anyhow::Result<()> { + if let Some(classifier) = CLASSIFY.get() { + record.bounce_classification = classifier.classify_response(&record.response); + } if file.is_none() { let now = Utc::now(); let name = params.log_dir.join(now.format("%Y%m%d-%H%M%S").to_string()); @@ -216,6 +253,8 @@ pub struct JsonLogRecord { /// number of logged events to determine the true number pub num_attempts: u16, + pub bounce_classification: BounceClass, + pub egress_pool: Option, pub egress_source: Option, } @@ -253,6 +292,7 @@ pub async fn log_disposition( num_attempts: msg.get_num_attempts(), egress_pool: egress_pool.map(|s| s.to_string()), egress_source: egress_source.map(|s| s.to_string()), + bounce_classification: BounceClass::Uncategorized, }; if let Err(err) = logger.log(record).await { tracing::error!("failed to log: {err:#}"); diff --git a/crates/kumod/src/mod_kumo.rs b/crates/kumod/src/mod_kumo.rs index 66d3deb3..b283b45e 100644 --- a/crates/kumod/src/mod_kumo.rs +++ b/crates/kumod/src/mod_kumo.rs @@ -2,7 +2,7 @@ use crate::egress_path::EgressPathConfig; use crate::egress_source::{EgressPool, EgressSource}; use crate::http_server::HttpListenerParams; use crate::lifecycle::LifeCycle; -use crate::logging::LogFileParams; +use crate::logging::{ClassifierParams, LogFileParams}; use crate::queue::QueueConfig; use crate::smtp_server::{EsmtpListenerParams, RejectError}; use config::get_or_create_module; @@ -22,6 +22,16 @@ pub fn register(lua: &Lua) -> anyhow::Result<()> { })?, )?; + kumo_mod.set( + "configure_bounce_classifier", + lua.create_function(move |lua, params: Value| { + let params: ClassifierParams = lua.from_value(params)?; + params + .register() + .map_err(|err| mlua::Error::external(format!("{err:#}"))) + })?, + )?; + kumo_mod.set( "configure_local_logs", lua.create_function(move |lua, params: Value| { diff --git a/docs/reference/kumo/configure_bounce_classifier.md b/docs/reference/kumo/configure_bounce_classifier.md new file mode 100644 index 00000000..6ed21aad --- /dev/null +++ b/docs/reference/kumo/configure_bounce_classifier.md @@ -0,0 +1,82 @@ +# `kumo.configure_bounce_classifier {PARAMS}` + +Configures the bounce classifier. The purpose of the classifier +is to attempt to digest complex and wide-ranging responses into +a smaller set of categories to help inform the sender how best +to respond and react to a delivery failure. + +This function should be called only from inside your [init](../events/init.md) +event handler. + +```admonish +The precise set of classifications are not yet finalized so are +not reproduced here. They can be found in the `BounceClass` enum +in `kumomta/crates/bounce-classify/src/lib.rs` +``` + +The classifier must be configured with a set of rules files +that provide mappings from a set of regular expressions to +the available clasification codes. + +`kumo.configure_bounce_classifier` will compile the merged +set of files and rules into an efficient regexset that can +quickly match the rule to the classification code. + +Once the classifier has been configured via this function, +the logging functions will automatically call into it to +populate the `bounce_classification` field. + +```lua +kumo.on('init', function() + kumo.configure_local_logs { + log_dir = '/var/log/kumo-logs', + } + kumo.configure_bounce_classifier { + files = { + '/etc/kumo/iana.toml', + }, + } +end) +``` + +The `iana.toml` file is provided with rules that map from +[IANA defined status +codes](https://www.iana.org/assignments/smtp-enhanced-status-codes/smtp-enhanced-status-codes.xhtml) +to an appropriate bounce class. + +You may create and maintain your own classifications and add then to the list +of files. + +Here's an excerpt of the `iana.toml`: + +```toml +# This file contains rules that match SMTP ENHANCEDSTATUSCODES +# codes as defined in the IANA registry: +# https://www.iana.org/assignments/smtp-enhanced-status-codes/smtp-enhanced-status-codes.xhtml +# to bounce classifications. +[rules] +InvalidRecipient = [ + "^(451|550) [45]\\.1\\.[1234] ", + "^45[02] [45]\\.2\\.4 ", # Mailing list expansion + "^5\\d{2} [45]\\.7\\.17 ", # RRVS: Mailbox owner has changed +] +BadDomain = [ + "^(451|550) [45]\\.1\\.10 ", # NULL MX + "^5\\d{2} [45]\\.7\\.18 ", # RRVS: domain owner has changed +] +InactiveMailbox = [ + "^(451|550) [45]\\.1\\.[6] ", + "^[45]\\d{2} [45]\\.2\\.1 ", + "^525 [45]\\.7\\.13 ", # User account disabled +] +InvalidSender = [ + "^(451|550) [45]\\.1\\.[78] ", + "^\\d{3} [45]\\.7\\.27 ", # Send address has NULL MX +] +QuotaIssues = [ + "^552 [45]\\.2\\.2 ", + "^552 [45]\\.2\\.3 ", + "^452 [45]\\.3\\.1 ", # Mail System Full + "^55[24] [45]\\.3\\.4 ", # Message too large for system +] +```