Allow simple wildcard suffixes for header names in logs

refs: https://github.com/KumoCorp/kumomta/issues/74

crates/integration-tests/source.lua

@@ -20,6 +20,7 @@ kumo.on('init', function()
   kumo.configure_local_logs {
     log_dir = TEST_DIR .. '/logs',
     max_segment_duration = '1s',
+    headers = { 'X-*', 'Y-*' },
   }
 
   if WEBHOOK_PORT then

crates/integration-tests/src/kumod.rs

@@ -95,6 +95,8 @@ impl MailGenParams<'_> {
         message.set_to(recip);
         message.set_subject(self.subject.unwrap_or("Hello! This is a test"));
         message.text_plain(body);
+        message.prepend("X-Test1", "Test1");
+        message.prepend("X-Another", "Another");
         Ok(message.build()?.to_message_string())
     }
 }
@@ -410,6 +412,30 @@ impl KumoDaemon {
         Maildir::from(self.dir.path().join("maildir"))
     }
 
+    pub fn check_for_x_and_y_headers_in_logs(&self) -> anyhow::Result<()> {
+        let dir = self.dir.path().join("logs");
+
+        for entry in std::fs::read_dir(&dir)? {
+            let entry = entry?;
+            if entry.file_type()?.is_file() {
+                let f = std::fs::File::open(entry.path())?;
+                let data = zstd::stream::decode_all(f)?;
+                let text = String::from_utf8(data)?;
+                eprintln!("{text}");
+
+                for line in text.lines() {
+                    let record: JsonLogRecord = serde_json::from_str(&line)?;
+                    if record.kind == RecordType::Reception {
+                        assert!(record.headers.contains_key("x-test1"));
+                        assert!(record.headers.contains_key("x-another"));
+                        assert!(!record.headers.contains_key("y-something"));
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+
     pub fn dump_logs(&self) -> anyhow::Result<BTreeMap<RecordType, usize>> {
         let dir = self.dir.path().join("logs");
         let mut counts = BTreeMap::new();

crates/integration-tests/src/main.rs

@@ -481,6 +481,8 @@ AccountingStats {
         daemon.stop_both().await.context("stop_both")?;
         println!("Stopped!");
 
+        daemon.source.check_for_x_and_y_headers_in_logs()?;
+
         let delivery_summary = daemon.dump_logs().context("dump_logs")?;
         k9::snapshot!(
             delivery_summary,
@@ -515,6 +517,11 @@ AccountingStats {
 
         assert!(parsed.headers().get_first("Received").is_some());
         assert!(parsed.headers().get_first("X-KumoRef").is_some());
+
+        // These two headers are added to all MailGenParams generated mail
+        assert!(parsed.headers().get_first("X-Test1").is_some());
+        assert!(parsed.headers().get_first("X-Another").is_some());
+
         k9::snapshot!(
             parsed.headers().from().unwrap(),
             r#"

crates/kumod/src/logging.rs

@@ -363,7 +363,11 @@ impl Logger {
                     .push(value.into());
             }
 
-            for name in &self.headers {
+            fn capture_header(
+                headers: &mut HashMap<String, Value>,
+                name: &str,
+                all_headers: &mut HashMap<String, Vec<Value>>,
+            ) {
                 match all_headers.remove(&name.to_ascii_lowercase()) {
                     Some(mut values) if values.len() == 1 => {
                         headers.insert(name.to_string(), values.remove(0));
@@ -374,6 +378,27 @@ impl Logger {
                     None => {}
                 }
             }
+
+            for name in &self.headers {
+                if name.ends_with('*') {
+                    let pattern = name[..name.len() - 1].to_ascii_lowercase();
+                    let matching_names: Vec<String> = all_headers
+                        .keys()
+                        .filter_map(|candidate| {
+                            if candidate.to_ascii_lowercase().starts_with(&pattern) {
+                                Some(candidate.to_string())
+                            } else {
+                                None
+                            }
+                        })
+                        .collect();
+                    for name in matching_names {
+                        capture_header(&mut headers, &name, &mut all_headers);
+                    }
+                } else {
+                    capture_header(&mut headers, name, &mut all_headers);
+                }
+            }
         }
 
         for name in &self.meta {

crates/mailparsing/src/headermap.rs

@@ -93,6 +93,11 @@ impl<'a> HeaderMap<'a> {
         Self { headers }
     }
 
+    pub fn prepend<V: Into<SharedString<'a>>>(&mut self, name: &str, v: V) {
+        self.headers
+            .insert(0, Header::new_unstructured(name.to_string(), v));
+    }
+
     pub fn get_first(&'a self, name: &str) -> Option<&Header<'a>> {
         self.iter_named(name).next()
     }

docs/changelog/main.md

@@ -8,5 +8,8 @@
 * queue helper: Added `setup_with_options` method that allows skipping the registration of the
   `get_queue_config` event handler. This helps when building a more complex configuration
   policy, such as using the rollup helper. Thanks to @cai-n! #101
+* You may now use simple suffix based wildcards like `X-*` to match header
+  names to capture in log records. See
+  [kumo.configure_local_logs](../reference/kumo/configure_local_logs.md). #74
 
 ## Fixes

docs/reference/kumo/configure_local_logs.md

@@ -100,6 +100,11 @@ kumo.configure_local_logs {
 }
 ```
 
+{{since('dev', indent=True)}}
+    Header names can now use simple wildcard suffixes; if the last character
+    of the header name is `*` then it will match any string with that prefix.
+    For example `"X-*"` will match any header names that start with `"X-"`.
+
 ## log_dir
 
 Specifies the directory into which log file segments will be written.