serialize bstring as utf8 string when possible

Add some tests to vet this in lua as well.

lua mime parsing functions now also accept bstring input.

Cargo.lock

@@ -4525,6 +4525,8 @@ dependencies = [
  "pastey",
  "quoted_printable",
  "serde",
+ "serde_json",
+ "serde_with",
  "thiserror 2.0.18",
  "uuid",
 ]

crates/mailparsing/Cargo.toml

@@ -18,8 +18,10 @@ nom-utils = {path="../nom-utils"}
 pastey = {workspace=true}
 quoted_printable = {workspace=true}
 serde = {workspace=true}
+serde_with = {workspace=true}
 thiserror = {workspace=true}
 uuid = {workspace=true, features=["v4", "fast-rng"]}
 
 [dev-dependencies]
 k9 = {workspace=true}
+serde_json = {workspace=true}

crates/mailparsing/src/rfc5322_parser.rs

@@ -13,9 +13,36 @@ use nom_utils::{
     explain_nom, make_context_error, make_span, tag, utf8_non_ascii, IResult, ParseError, Span,
 };
 use serde::{Deserialize, Serialize};
+use serde_with::{serde_as, DeserializeAs, SerializeAs};
 use std::collections::BTreeMap;
 use std::fmt::Debug;
 
+/// A `serde_with` adapter that serializes `BString` as a JSON string when
+/// the value is valid UTF-8, falling back to the default byte-array
+/// representation otherwise.
+pub struct BStringUtf8;
+
+impl SerializeAs<BString> for BStringUtf8 {
+    fn serialize_as<S>(value: &BString, serializer: S) -> std::result::Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        match std::str::from_utf8(value.as_bytes()) {
+            Ok(s) => serializer.serialize_str(s),
+            Err(_) => value.serialize(serializer),
+        }
+    }
+}
+
+impl<'de> DeserializeAs<'de, BString> for BStringUtf8 {
+    fn deserialize_as<D>(deserializer: D) -> std::result::Result<BString, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        BString::deserialize(deserializer)
+    }
+}
+
 impl MailParsingError {
     pub fn from_nom(input: Span, err: nom::Err<ParseError<Span<'_>>>) -> Self {
         MailParsingError::HeaderParse(explain_nom(input, err))
@@ -1741,10 +1768,12 @@ impl Parser {
     }
 }
 
+#[serde_as]
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub struct ARCAuthenticationResults {
     pub instance: u8,
+    #[serde_as(as = "BStringUtf8")]
     pub serv_id: BString,
     pub version: Option<u32>,
     pub results: Vec<AuthenticationResult>,
@@ -1785,9 +1814,11 @@ impl EncodeHeaderValue for ARCAuthenticationResults {
     }
 }
 
+#[serde_as]
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub struct AuthenticationResults {
+    #[serde_as(as = "BStringUtf8")]
     pub serv_id: BString,
     #[serde(default)]
     pub version: Option<u32>,
@@ -1844,6 +1875,7 @@ impl EncodeHeaderValue for AuthenticationResults {
     }
 }
 
+#[serde_as]
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
 pub struct AuthenticationResult {
@@ -1851,8 +1883,10 @@ pub struct AuthenticationResult {
     #[serde(default)]
     pub method_version: Option<u32>,
     pub result: String,
+    #[serde_as(as = "Option<BStringUtf8>")]
     #[serde(default)]
     pub reason: Option<BString>,
+    #[serde_as(as = "BTreeMap<_, BStringUtf8>")]
     #[serde(default)]
     pub props: BTreeMap<String, BString>,
 }
@@ -1961,9 +1995,10 @@ pub struct Mailbox {
     pub address: AddrSpec,
 }
 
+#[serde_as]
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(transparent)]
-pub struct MessageID(pub BString);
+pub struct MessageID(#[serde_as(as = "BStringUtf8")] pub BString);
 
 impl EncodeHeaderValue for MessageID {
     fn encode_value(&self) -> SharedString<'static> {
@@ -3853,4 +3888,156 @@ ARCAuthenticationResults {
 "#
         );
     }
+
+    #[test]
+    fn bstring_utf8_serializes_utf8_as_string() {
+        // A MessageID with pure ASCII content serializes as a JSON string
+        let mid = MessageID(BString::from("abc123@example.com"));
+        let json = serde_json::to_string(&mid).unwrap();
+        k9::assert_equal!(json, r#""abc123@example.com""#);
+    }
+
+    #[test]
+    fn bstring_utf8_serializes_non_utf8_as_array() {
+        // A MessageID with invalid UTF-8 falls back to byte array
+        let mid = MessageID(BString::from(&b"hello\x80world"[..]));
+        let json = serde_json::to_string(&mid).unwrap();
+        k9::assert_equal!(json, "[104,101,108,108,111,128,119,111,114,108,100]");
+    }
+
+    #[test]
+    fn bstring_utf8_round_trip_utf8() {
+        let mid = MessageID(BString::from("test@example.com"));
+        let json = serde_json::to_string(&mid).unwrap();
+        let restored: MessageID = serde_json::from_str(&json).unwrap();
+        k9::assert_equal!(restored, mid);
+    }
+
+    #[test]
+    fn bstring_utf8_round_trip_non_utf8() {
+        let mid = MessageID(BString::from(&b"\xff\xfe"[..]));
+        let json = serde_json::to_string(&mid).unwrap();
+        let restored: MessageID = serde_json::from_str(&json).unwrap();
+        k9::assert_equal!(restored, mid);
+    }
+
+    #[test]
+    fn authentication_results_serialize_as_strings() {
+        let ar = AuthenticationResults {
+            serv_id: BString::from("example.com"),
+            version: None,
+            results: vec![AuthenticationResult {
+                method: "dkim".into(),
+                method_version: None,
+                result: "pass".into(),
+                reason: Some(BString::from("good signature")),
+                props: BTreeMap::from([
+                    ("header.d".into(), BString::from("example.com")),
+                    ("header.s".into(), BString::from("selector1")),
+                ]),
+            }],
+        };
+        let json = serde_json::to_string_pretty(&ar).unwrap();
+        // All BString fields that are valid UTF-8 should appear as JSON strings
+        k9::assert_equal!(
+            json,
+            r#"{
+  "serv_id": "example.com",
+  "version": null,
+  "results": [
+    {
+      "method": "dkim",
+      "method_version": null,
+      "result": "pass",
+      "reason": "good signature",
+      "props": {
+        "header.d": "example.com",
+        "header.s": "selector1"
+      }
+    }
+  ]
+}"#
+        );
+    }
+
+    #[test]
+    fn authentication_results_round_trip() {
+        let ar = AuthenticationResults {
+            serv_id: BString::from("mx.example.org"),
+            version: Some(1),
+            results: vec![AuthenticationResult {
+                method: "spf".into(),
+                method_version: None,
+                result: "pass".into(),
+                reason: None,
+                props: BTreeMap::from([(
+                    "smtp.mailfrom".into(),
+                    BString::from("sender@example.com"),
+                )]),
+            }],
+        };
+        let json = serde_json::to_string(&ar).unwrap();
+        let restored: AuthenticationResults = serde_json::from_str(&json).unwrap();
+        k9::assert_equal!(restored, ar);
+    }
+
+    #[test]
+    fn authentication_result_non_utf8_reason() {
+        let ar = AuthenticationResult {
+            method: "dkim".into(),
+            method_version: None,
+            result: "temperror".into(),
+            reason: Some(BString::from(&b"bad\x80data"[..])),
+            props: BTreeMap::new(),
+        };
+        let json = serde_json::to_string(&ar).unwrap();
+        // reason should be a byte array since it contains invalid UTF-8
+        assert!(json.contains(r#""reason":[98,97,100,128,100,97,116,97]"#));
+        let restored: AuthenticationResult = serde_json::from_str(&json).unwrap();
+        k9::assert_equal!(restored, ar);
+    }
+
+    #[test]
+    fn authentication_results_encode_value_with_binary() {
+        // Construct AuthenticationResults with non-UTF-8 bytes in BString fields
+        // and capture the encode_value() output for use in a Lua test.
+        let ar = AuthenticationResults {
+            serv_id: BString::from(&b"mx.ex\x80mple.com"[..]),
+            version: None,
+            results: vec![AuthenticationResult {
+                method: "spf".into(),
+                method_version: None,
+                result: "pass".into(),
+                reason: Some(BString::from(&b"good\xffsig"[..])),
+                props: BTreeMap::from([(
+                    "smtp.mailfrom".into(),
+                    BString::from(&b"user@\xfehost"[..]),
+                )]),
+            }],
+        };
+        let encoded = ar.encode_value();
+        k9::snapshot!(
+            encoded,
+            r#"
+mx.ex\x80mple.com;\r
+\tspf=pass reason=good\xffsig\r
+\tsmtp.mailfrom=user@\xfehost
+"#
+        );
+    }
+
+    #[test]
+    fn arc_authentication_results_serialize_as_strings() {
+        let arc = ARCAuthenticationResults {
+            instance: 1,
+            serv_id: BString::from("mx.example.com"),
+            version: None,
+            results: vec![],
+        };
+        let json = serde_json::to_string(&arc).unwrap();
+        k9::assert_equal!(
+            json,
+            r#"{"instance":1,"serv_id":"mx.example.com","version":null,"results":[]}"#
+        );
+    }
 }

crates/message/src/message.rs

@@ -1397,7 +1397,7 @@ impl UserData for Message {
 
         methods.add_async_method("parse_mime", |_lua, this, _: ()| async move {
             let data = this.data().await.map_err(any_err)?;
-            let owned_data = String::from_utf8_lossy(data.as_ref().as_ref()).to_string();
+            let owned_data = BString::new(data.as_ref().to_vec());
             let part = MimePart::parse(owned_data).map_err(any_err)?;
             Ok(mod_mimepart::PartRef::new(part))
         });

crates/message/test_address_header.lua

@@ -6,7 +6,8 @@ local function new_msg(content)
 end
 
 -- HeaderAddressList: single simple address without display name
-local msg = new_msg 'From: user@example.com\r\nTo: someone@example.com\r\n\r\nBody'
+local msg =
+  new_msg 'From: user@example.com\r\nTo: someone@example.com\r\n\r\nBody'
 local from = msg:from_header()
 utils.assert_eq(from.user, 'user')
 utils.assert_eq(from.domain, 'example.com')

crates/message/test_binary_header_values.lua

@@ -0,0 +1,86 @@
+local kumo = require 'kumo'
+local utils = require 'policy-extras.policy_utils'
+
+local function new_msg(content)
+  return kumo.make_message('sender@example.com', 'recip@example.com', content)
+end
+
+-- Latin-1 bytes (non-UTF-8) pass through get_first_named_header_value
+-- without lossy encoding and without error.
+-- 0xA9 is latin-1 copyright, 0xE9 is latin-1 é
+local msg =
+  new_msg 'Subject: hello \xa9 world \xe9\r\nX-Custom: \xff\xfe\r\n\r\nBody'
+utils.assert_eq(
+  msg:get_first_named_header_value 'subject',
+  'hello \xa9 world \xe9'
+)
+utils.assert_eq(msg:get_first_named_header_value 'X-Custom', '\xff\xfe')
+
+-- get_all_named_header_values returns binary content from multiple headers
+local msg2 =
+  new_msg 'X-Bin: \x80\x81\r\nX-Bin: \x90\x91\r\nSubject: test\r\n\r\nBody'
+local values = msg2:get_all_named_header_values 'X-Bin'
+utils.assert_eq(#values, 2)
+utils.assert_eq(values[1], '\x80\x81')
+utils.assert_eq(values[2], '\x90\x91')
+
+-- get_all_headers preserves binary in both names and values
+-- (header names are always ASCII in practice, but values can be binary)
+local msg3 = new_msg 'X-Data: \xde\xad\xbe\xef\r\nSubject: ok\r\n\r\nBody'
+local all = msg3:get_all_headers()
+-- Find the X-Data header
+local found = false
+for _, pair in ipairs(all) do
+  if pair[1] == 'X-Data' then
+    utils.assert_eq(pair[2], '\xde\xad\xbe\xef')
+    found = true
+  end
+end
+utils.assert_eq(
+  found,
+  true,
+  'X-Data header should be found in get_all_headers'
+)
+
+-- Verify that pure ASCII values also work fine alongside binary
+utils.assert_eq(
+  msg:get_first_named_header_value 'subject',
+  'hello \xa9 world \xe9'
+)
+
+-- A header with all 256 byte values (0x01-0xff, skipping 0x00 which
+-- terminates C strings, and \r\n which are line terminators)
+local bytes = {}
+for i = 1, 255 do
+  -- Skip CR (13) and LF (10) as they would break the header
+  if i ~= 10 and i ~= 13 then
+    table.insert(bytes, string.char(i))
+  end
+end
+local all_bytes = table.concat(bytes)
+local msg4 = new_msg('X-AllBytes: ' .. all_bytes .. '\r\n\r\nBody')
+local result = msg4:get_first_named_header_value 'X-AllBytes'
+utils.assert_eq(result, all_bytes)
+
+-- Authentication-Results header with binary (non-UTF-8) content in serv_id
+-- and reason fields. Binary in propspec values is tested with a leading
+-- non-ASCII byte to avoid the parser's domain-before-value alternation.
+local ar_header = 'mx.ex\x80mple.com;'
+  .. ' spf=pass reason=good\xffsig'
+  .. ' smtp.mailfrom=\xfevalue'
+local msg5 =
+  new_msg('Authentication-Results: ' .. ar_header .. '\r\n\r\nBody')
+local ar_value = msg5:get_first_named_header_value 'Authentication-Results'
+utils.assert_eq(ar_value, ar_header)
+
+-- Use the structured authentication_results accessor to verify binary
+-- is preserved in individual parsed fields.
+local hdr = msg5:parse_mime().headers:get_first_named 'Authentication-Results'
+local ar = hdr.authentication_results
+utils.assert_eq(ar.serv_id, 'mx.ex\x80mple.com')
+utils.assert_eq(#ar.results, 1)
+local r = ar.results[1]
+utils.assert_eq(r.method, 'spf')
+utils.assert_eq(r.result, 'pass')
+utils.assert_eq(r.reason, 'good\xffsig')
+utils.assert_eq(r.props['smtp.mailfrom'], '\xfevalue')

crates/mod-mimepart/src/lib.rs

@@ -1,6 +1,7 @@
 pub use crate::mimepart::PartRef;
+use bstr::BString;
 use config::{SerdeWrappedValue, any_err, get_or_create_sub_module};
-use mailparsing::{AttachmentOptions, MimePart, SharedString};
+use mailparsing::{AttachmentOptions, MimePart};
 use mlua::{Lua, UserDataRef};
 
 pub mod builder;
@@ -60,8 +61,8 @@ fn new_multipart(
     Ok(PartRef::new(part))
 }
 
-fn parse_eml(_: &Lua, eml_contents: String) -> mlua::Result<PartRef> {
-    let eml_contents: SharedString = eml_contents.into();
+fn parse_eml(_: &Lua, eml_contents: mlua::String) -> mlua::Result<PartRef> {
+    let eml_contents = BString::new(eml_contents.as_bytes().to_vec());
     let part = MimePart::parse(eml_contents).map_err(any_err)?;
     Ok(PartRef::new(part))
 }