From cdfe11cf22ca0b5f6255a5af2b5a7bdaa28e8bf4 Mon Sep 17 00:00:00 2001 From: Wez Furlong Date: Mon, 4 Dec 2023 12:10:15 -0700 Subject: [PATCH] Allow simple wildcard suffixes for header names in logs refs: https://github.com/KumoCorp/kumomta/issues/74 --- crates/integration-tests/source.lua | 1 + crates/integration-tests/src/kumod.rs | 26 ++++++++++++++++++++ crates/integration-tests/src/main.rs | 7 ++++++ crates/kumod/src/logging.rs | 27 ++++++++++++++++++++- crates/mailparsing/src/headermap.rs | 5 ++++ docs/changelog/main.md | 3 +++ docs/reference/kumo/configure_local_logs.md | 5 ++++ 7 files changed, 73 insertions(+), 1 deletion(-) diff --git a/crates/integration-tests/source.lua b/crates/integration-tests/source.lua index 4514ca92..b7e51ee8 100644 --- a/crates/integration-tests/source.lua +++ b/crates/integration-tests/source.lua @@ -20,6 +20,7 @@ kumo.on('init', function() kumo.configure_local_logs { log_dir = TEST_DIR .. '/logs', max_segment_duration = '1s', + headers = { 'X-*', 'Y-*' }, } if WEBHOOK_PORT then diff --git a/crates/integration-tests/src/kumod.rs b/crates/integration-tests/src/kumod.rs index f93c750e..80911552 100644 --- a/crates/integration-tests/src/kumod.rs +++ b/crates/integration-tests/src/kumod.rs @@ -95,6 +95,8 @@ impl MailGenParams<'_> { message.set_to(recip); message.set_subject(self.subject.unwrap_or("Hello! This is a test")); message.text_plain(body); + message.prepend("X-Test1", "Test1"); + message.prepend("X-Another", "Another"); Ok(message.build()?.to_message_string()) } } @@ -410,6 +412,30 @@ impl KumoDaemon { Maildir::from(self.dir.path().join("maildir")) } + pub fn check_for_x_and_y_headers_in_logs(&self) -> anyhow::Result<()> { + let dir = self.dir.path().join("logs"); + + for entry in std::fs::read_dir(&dir)? { + let entry = entry?; + if entry.file_type()?.is_file() { + let f = std::fs::File::open(entry.path())?; + let data = zstd::stream::decode_all(f)?; + let text = String::from_utf8(data)?; + eprintln!("{text}"); + + for line in text.lines() { + let record: JsonLogRecord = serde_json::from_str(&line)?; + if record.kind == RecordType::Reception { + assert!(record.headers.contains_key("x-test1")); + assert!(record.headers.contains_key("x-another")); + assert!(!record.headers.contains_key("y-something")); + } + } + } + } + Ok(()) + } + pub fn dump_logs(&self) -> anyhow::Result> { let dir = self.dir.path().join("logs"); let mut counts = BTreeMap::new(); diff --git a/crates/integration-tests/src/main.rs b/crates/integration-tests/src/main.rs index 1f3373a7..540abac6 100644 --- a/crates/integration-tests/src/main.rs +++ b/crates/integration-tests/src/main.rs @@ -481,6 +481,8 @@ AccountingStats { daemon.stop_both().await.context("stop_both")?; println!("Stopped!"); + daemon.source.check_for_x_and_y_headers_in_logs()?; + let delivery_summary = daemon.dump_logs().context("dump_logs")?; k9::snapshot!( delivery_summary, @@ -515,6 +517,11 @@ AccountingStats { assert!(parsed.headers().get_first("Received").is_some()); assert!(parsed.headers().get_first("X-KumoRef").is_some()); + + // These two headers are added to all MailGenParams generated mail + assert!(parsed.headers().get_first("X-Test1").is_some()); + assert!(parsed.headers().get_first("X-Another").is_some()); + k9::snapshot!( parsed.headers().from().unwrap(), r#" diff --git a/crates/kumod/src/logging.rs b/crates/kumod/src/logging.rs index cc4ee400..66e3a0d8 100644 --- a/crates/kumod/src/logging.rs +++ b/crates/kumod/src/logging.rs @@ -363,7 +363,11 @@ impl Logger { .push(value.into()); } - for name in &self.headers { + fn capture_header( + headers: &mut HashMap, + name: &str, + all_headers: &mut HashMap>, + ) { match all_headers.remove(&name.to_ascii_lowercase()) { Some(mut values) if values.len() == 1 => { headers.insert(name.to_string(), values.remove(0)); @@ -374,6 +378,27 @@ impl Logger { None => {} } } + + for name in &self.headers { + if name.ends_with('*') { + let pattern = name[..name.len() - 1].to_ascii_lowercase(); + let matching_names: Vec = all_headers + .keys() + .filter_map(|candidate| { + if candidate.to_ascii_lowercase().starts_with(&pattern) { + Some(candidate.to_string()) + } else { + None + } + }) + .collect(); + for name in matching_names { + capture_header(&mut headers, &name, &mut all_headers); + } + } else { + capture_header(&mut headers, name, &mut all_headers); + } + } } for name in &self.meta { diff --git a/crates/mailparsing/src/headermap.rs b/crates/mailparsing/src/headermap.rs index 1270cf26..cbf6957b 100644 --- a/crates/mailparsing/src/headermap.rs +++ b/crates/mailparsing/src/headermap.rs @@ -93,6 +93,11 @@ impl<'a> HeaderMap<'a> { Self { headers } } + pub fn prepend>>(&mut self, name: &str, v: V) { + self.headers + .insert(0, Header::new_unstructured(name.to_string(), v)); + } + pub fn get_first(&'a self, name: &str) -> Option<&Header<'a>> { self.iter_named(name).next() } diff --git a/docs/changelog/main.md b/docs/changelog/main.md index 77caac13..9c510669 100644 --- a/docs/changelog/main.md +++ b/docs/changelog/main.md @@ -8,5 +8,8 @@ * queue helper: Added `setup_with_options` method that allows skipping the registration of the `get_queue_config` event handler. This helps when building a more complex configuration policy, such as using the rollup helper. Thanks to @cai-n! #101 +* You may now use simple suffix based wildcards like `X-*` to match header + names to capture in log records. See + [kumo.configure_local_logs](../reference/kumo/configure_local_logs.md). #74 ## Fixes diff --git a/docs/reference/kumo/configure_local_logs.md b/docs/reference/kumo/configure_local_logs.md index f92f5aa2..b99f2ec2 100644 --- a/docs/reference/kumo/configure_local_logs.md +++ b/docs/reference/kumo/configure_local_logs.md @@ -100,6 +100,11 @@ kumo.configure_local_logs { } ``` +{{since('dev', indent=True)}} + Header names can now use simple wildcard suffixes; if the last character + of the header name is `*` then it will match any string with that prefix. + For example `"X-*"` will match any header names that start with `"X-"`. + ## log_dir Specifies the directory into which log file segments will be written.