diff --git a/crates/integration-tests/src/kumod.rs b/crates/integration-tests/src/kumod.rs
index 9171bd64..2a44df75 100644
--- a/crates/integration-tests/src/kumod.rs
+++ b/crates/integration-tests/src/kumod.rs
@@ -158,7 +158,7 @@ impl MailGenParams<'_> {
message.prepend("X-Test1", "Test1");
message.prepend("X-Another", "Another");
message.set_stable_content(true);
- Ok(message.build()?.to_message_string())
+ Ok(String::from_utf8(message.build()?.to_message_bytes())?)
}
}
diff --git a/crates/kumo-log-types/src/rfc3464.rs b/crates/kumo-log-types/src/rfc3464.rs
index 1205b48a..07d65f39 100644
--- a/crates/kumo-log-types/src/rfc3464.rs
+++ b/crates/kumo-log-types/src/rfc3464.rs
@@ -783,7 +783,7 @@ mod test {
let report_msg = Report::generate(¶ms, Some(&original_msg), &log)
.unwrap()
.unwrap();
- let report_eml = report_msg.to_message_string();
+ let report_eml = BString::from(report_msg.to_message_bytes());
k9::snapshot!(
&report_eml,
r#"
@@ -914,7 +914,7 @@ Subject: Hello!
let report_msg = Report::generate(¶ms, Some(&original_msg), &log)
.unwrap()
.unwrap();
- let report_eml = report_msg.to_message_string();
+ let report_eml = BString::from(report_msg.to_message_bytes());
k9::snapshot!(
&report_eml,
r#"
@@ -1047,7 +1047,7 @@ Subject: Hello!
let report_msg = Report::generate(¶ms, Some(&original_msg), &log)
.unwrap()
.unwrap();
- let report_eml = report_msg.to_message_string();
+ let report_eml = BString::from(report_msg.to_message_bytes());
k9::snapshot!(
&report_eml,
r#"
@@ -1185,7 +1185,7 @@ hello there
let report_msg = Report::generate(¶ms, Some(&original_msg), &log)
.unwrap()
.unwrap();
- let report_eml = report_msg.to_message_string();
+ let report_eml = BString::from(report_msg.to_message_bytes());
k9::snapshot!(
&report_eml,
r#"
diff --git a/crates/kumod/src/http_server/inject_v1.rs b/crates/kumod/src/http_server/inject_v1.rs
index 775db3e7..0b322763 100644
--- a/crates/kumod/src/http_server/inject_v1.rs
+++ b/crates/kumod/src/http_server/inject_v1.rs
@@ -508,7 +508,7 @@ impl<'a> Compiled<'a> {
msg.headers_mut().set_mime_version("1.0")?;
}
- Ok(msg.to_message_string())
+ Ok(String::from_utf8(msg.to_message_bytes())?)
}
Content::Builder {
text_body,
@@ -563,7 +563,7 @@ impl<'a> Compiled<'a> {
builder.attach_part(part.clone());
}
- Ok(builder.build()?.to_message_string())
+ Ok(String::from_utf8(builder.build()?.to_message_bytes())?)
}
}
}
diff --git a/crates/kumod/src/mod_kumo.rs b/crates/kumod/src/mod_kumo.rs
index 37fa0284..1c9b0dfe 100644
--- a/crates/kumod/src/mod_kumo.rs
+++ b/crates/kumod/src/mod_kumo.rs
@@ -284,14 +284,14 @@ pub fn register(lua: &Lua) -> anyhow::Result<()> {
Some(report) => {
let recip = EnvelopeAddress::parse(&log_record.sender)
.context("log_record is somehow an invalid EnvelopeAddress")?;
- let body = report.to_message_string();
+ let body = report.to_message_bytes();
let msg = Message::new_dirty(
SpoolId::new(),
EnvelopeAddress::null_sender(),
vec![recip],
serde_json::json!({}),
- Arc::new(body.as_bytes().to_vec().into_boxed_slice()),
+ Arc::new(body.into_boxed_slice()),
)?;
Ok(Some(msg))
}
diff --git a/crates/mailparsing/src/builder.rs b/crates/mailparsing/src/builder.rs
index 2124c7e7..466a9045 100644
--- a/crates/mailparsing/src/builder.rs
+++ b/crates/mailparsing/src/builder.rs
@@ -191,6 +191,7 @@ impl<'a> std::ops::DerefMut for MessageBuilder<'a> {
#[cfg(test)]
mod test {
use super::*;
+ use bstr::BString;
#[test]
fn basic() {
@@ -201,7 +202,7 @@ mod test {
b.text_html("this is html 🚀");
let msg = b.build().unwrap();
k9::snapshot!(
- msg.to_message_string(),
+ BString::from(msg.to_message_bytes()),
r#"
Content-Type: multipart/alternative;\r
\tboundary="ma-boundary"\r
@@ -251,7 +252,7 @@ Hello World in AMP!
);
let msg = b.build().unwrap();
k9::snapshot!(
- msg.to_message_string(),
+ BString::from(msg.to_message_bytes()),
r#"
Content-Type: multipart/alternative;\r
\tboundary="ma-boundary"\r
@@ -311,7 +312,7 @@ Content-Transfer-Encoding: quoted-printable\r
.unwrap();
let msg = b.build().unwrap();
k9::snapshot!(
- msg.to_message_string(),
+ BString::from(msg.to_message_bytes()),
r#"
Content-Type: multipart/mixed;\r
\tboundary="mm-boundary"\r
diff --git a/crates/mailparsing/src/mimepart.rs b/crates/mailparsing/src/mimepart.rs
index 8dbdc03f..5fd8db39 100644
--- a/crates/mailparsing/src/mimepart.rs
+++ b/crates/mailparsing/src/mimepart.rs
@@ -647,10 +647,10 @@ impl<'a> MimePart<'a> {
/// Convenience method wrapping write_message that returns
/// the formatted message as a standalone string
- pub fn to_message_string(&self) -> String {
+ pub fn to_message_bytes(&self) -> Vec {
let mut out = vec![];
self.write_message(&mut out).unwrap();
- String::from_utf8_lossy(&out).to_string()
+ out
}
pub fn replace_text_body(
@@ -1306,7 +1306,7 @@ mod test {
);
let part = MimePart::parse(message).unwrap();
- k9::assert_equal!(message, part.to_message_string());
+ k9::assert_equal!(message.as_bytes(), part.to_message_bytes());
assert_eq!(part.raw_body(), "I am the body");
k9::snapshot!(
part.body(),
@@ -1320,7 +1320,7 @@ Ok(
);
k9::snapshot!(
- part.rebuild(None).unwrap().to_message_string(),
+ BString::from(part.rebuild(None).unwrap().to_message_bytes()),
r#"
Content-Type: text/plain;\r
\tcharset="us-ascii"\r
@@ -1367,7 +1367,7 @@ I am the body\r
);
let part = MimePart::parse(message).unwrap();
- k9::assert_equal!(message, part.to_message_string());
+ k9::assert_equal!(message.as_bytes(), part.to_message_bytes());
assert_eq!(part.raw_body(), "aGVsbG8K\n");
k9::snapshot!(
part.body(),
@@ -1382,7 +1382,7 @@ Ok(
);
k9::snapshot!(
- part.rebuild(None).unwrap().to_message_string(),
+ BString::from(part.rebuild(None).unwrap().to_message_bytes()),
r#"
Content-Type: text/plain;\r
\tcharset="us-ascii"\r
@@ -1422,7 +1422,7 @@ hello=0A\r
let part = MimePart::parse(message).unwrap();
- k9::assert_equal!(message, part.to_message_string());
+ k9::assert_equal!(message.as_bytes(), part.to_message_bytes());
let children = part.child_parts();
k9::assert_equal!(children.len(), 2);
@@ -1475,7 +1475,7 @@ Ok(
);
let mut part = MimePart::parse(message).unwrap();
- k9::assert_equal!(message, part.to_message_string());
+ k9::assert_equal!(message.as_bytes(), part.to_message_bytes());
fn munge(part: &mut MimePart) {
let headers = part.headers_mut();
headers.push(Header::with_name_value("X-Woot", "Hello"));
@@ -1484,7 +1484,7 @@ Ok(
}
munge(&mut part);
- let re_encoded = part.to_message_string();
+ let re_encoded = BString::from(part.to_message_bytes());
k9::snapshot!(
re_encoded,
r#"
@@ -1520,7 +1520,7 @@ After the final boundary stuff gets ignored.\r
eprintln!("part with html removed is:\n{part:#?}");
- let re_encoded = part.to_message_string();
+ let re_encoded = BString::from(part.to_message_bytes());
k9::snapshot!(
re_encoded,
r#"
@@ -1546,7 +1546,7 @@ After the final boundary stuff gets ignored.\r
#[test]
fn replace_text_body() {
let mut part = MimePart::new_text_plain("Hello 👻\r\n").unwrap();
- let encoded = part.to_message_string();
+ let encoded = BString::from(part.to_message_bytes());
k9::snapshot!(
&encoded,
r#"
@@ -1561,7 +1561,7 @@ SGVsbG8g8J+Ruw0K\r
part.replace_text_body("text/plain", "Hello 🚀\r\n")
.unwrap();
- let encoded = part.to_message_string();
+ let encoded = BString::from(part.to_message_bytes());
k9::snapshot!(
&encoded,
r#"
@@ -1581,7 +1581,7 @@ SGVsbG8g8J+agA0K\r
let part = MimePart::new_text_plain(input_text).unwrap();
- let encoded = part.to_message_string();
+ let encoded = BString::from(part.to_message_bytes());
k9::snapshot!(
&encoded,
r#"
@@ -1598,7 +1598,7 @@ t's see how that turns out!\r
);
let parsed_part = MimePart::parse(encoded.clone()).unwrap();
- k9::assert_equal!(encoded.as_str(), parsed_part.to_message_string().as_str());
+ k9::assert_equal!(encoded, parsed_part.to_message_bytes());
k9::assert_equal!(part.body().unwrap(), DecodedBody::Text(input_text.into()));
k9::snapshot!(
parsed_part.simplified_structure_pointers(),
@@ -1644,7 +1644,7 @@ Ok(
)
.unwrap();
k9::snapshot!(
- msg.to_message_string(),
+ BString::from(msg.to_message_bytes()),
r#"
Content-Type: multipart/mixed;\r
\tboundary="my-boundary"\r
@@ -1852,7 +1852,7 @@ Ok(
let rebuilt = part.rebuild(None).unwrap();
k9::snapshot!(
- rebuilt.to_message_string(),
+ BString::from(rebuilt.to_message_bytes()),
r#"
Content-Type: multipart/mixed;\r
\tboundary="8a54d64d7ad7c04a084478052b36cbe1609b33bf3a41203aaee8dd642cd3"\r
@@ -1912,8 +1912,8 @@ Hello";
"Message has conformance issues: MISSING_MESSAGE_ID_HEADER"
);
- let rebuilt = msg
- .check_fix_conformance(
+ let rebuilt = BString::from(
+ msg.check_fix_conformance(
MessageConformance::MISSING_MESSAGE_ID_HEADER,
MessageConformance::MISSING_MESSAGE_ID_HEADER,
CheckFixSettings {
@@ -1923,7 +1923,8 @@ Hello";
)
.unwrap()
.unwrap()
- .to_message_string();
+ .to_message_bytes(),
+ );
k9::snapshot!(
rebuilt,
@@ -1947,8 +1948,8 @@ Hello this is a really long line Hello this is a really long line \
Hello this is a really long line Hello this is a really long line
";
let msg = MimePart::parse(DOUBLE_ANGLE_AND_LONG_LINE).unwrap();
- let rebuilt = msg
- .check_fix_conformance(
+ let rebuilt = BString::from(
+ msg.check_fix_conformance(
MessageConformance::MISSING_COLON_VALUE,
MessageConformance::MISSING_MESSAGE_ID_HEADER | MessageConformance::LINE_TOO_LONG,
CheckFixSettings {
@@ -1958,7 +1959,8 @@ Hello this is a really long line Hello this is a really long line
)
.unwrap()
.unwrap()
- .to_message_string();
+ .to_message_bytes(),
+ );
k9::snapshot!(
rebuilt,
@@ -1987,15 +1989,16 @@ y long line=0A\r
"X-Hello: there\r\nX-Header: value\r\nSubject: Hello\r\nX-Header: another value\r\nFrom :Someone@somewhere\r\n\r\nBody";
let msg = MimePart::parse(MULTI_HEADER_CONTENT).unwrap();
- let rebuilt = msg
- .check_fix_conformance(
+ let rebuilt = BString::from(
+ msg.check_fix_conformance(
MessageConformance::default(),
MessageConformance::MISSING_MIME_VERSION,
CheckFixSettings::default(),
)
.unwrap()
.unwrap()
- .to_message_string();
+ .to_message_bytes(),
+ );
k9::snapshot!(
rebuilt,
r#"
@@ -2011,15 +2014,16 @@ Body
);
let msg = MimePart::parse(MULTI_HEADER_CONTENT).unwrap();
- let rebuilt = msg
- .check_fix_conformance(
+ let rebuilt = BString::from(
+ msg.check_fix_conformance(
MessageConformance::default(),
MessageConformance::MISSING_MIME_VERSION | MessageConformance::NAME_ENDS_WITH_SPACE,
CheckFixSettings::default(),
)
.unwrap()
.unwrap()
- .to_message_string();
+ .to_message_bytes(),
+ );
k9::snapshot!(
rebuilt,
r#"
diff --git a/crates/mailparsing/src/rfc5322_parser.rs b/crates/mailparsing/src/rfc5322_parser.rs
index 29094e64..ccbb2e4c 100644
--- a/crates/mailparsing/src/rfc5322_parser.rs
+++ b/crates/mailparsing/src/rfc5322_parser.rs
@@ -2767,7 +2767,7 @@ Some(
);
k9::snapshot!(
- msg.rebuild(None).unwrap().to_message_string(),
+ BString::from(msg.rebuild(None).unwrap().to_message_bytes()),
r#"
Content-Type: text/plain;\r
\tcharset="us-ascii"\r
diff --git a/crates/mailparsing/src/strings.rs b/crates/mailparsing/src/strings.rs
index ff965bd4..997c35e1 100644
--- a/crates/mailparsing/src/strings.rs
+++ b/crates/mailparsing/src/strings.rs
@@ -200,3 +200,13 @@ impl<'a> IntoSharedString<'a> for &'a [u8] {
}
}
}
+
+impl<'a> IntoSharedString<'a> for BString {
+ fn into_shared_string(self) -> (SharedString<'a>, MessageConformance) {
+ let bytes: Vec = self.into();
+ match std::str::from_utf8(&bytes) {
+ Ok(_) => (bytes.into(), MessageConformance::default()),
+ Err(_) => (bytes.into(), MessageConformance::NEEDS_TRANSFER_ENCODING),
+ }
+ }
+}
diff --git a/crates/message/src/message.rs b/crates/message/src/message.rs
index 4da60b67..2627764f 100644
--- a/crates/message/src/message.rs
+++ b/crates/message/src/message.rs
@@ -1238,8 +1238,8 @@ impl Message {
text.push_str(content);
p.replace_text_body("text/plain", &*text)?;
- let new_data = msg.to_message_string();
- self.assign_data(new_data.into_bytes());
+ let new_data = msg.to_message_bytes();
+ self.assign_data(new_data);
Ok(true)
}
DecodedBody::Binary(_) => {
@@ -1274,8 +1274,8 @@ impl Message {
p.replace_text_body("text/html", &*text)?;
- let new_data = msg.to_message_string();
- self.assign_data(new_data.into_bytes());
+ let new_data = msg.to_message_bytes();
+ self.assign_data(new_data);
Ok(true)
}
DecodedBody::Binary(_) => {
@@ -1316,8 +1316,8 @@ impl Message {
let opt_msg = msg.check_fix_conformance(check, fix, settings)?;
if let Some(msg) = opt_msg {
- let new_data = msg.to_message_string();
- self.assign_data(new_data.into_bytes());
+ let new_data = msg.to_message_bytes();
+ self.assign_data(new_data);
}
Ok(())
diff --git a/crates/message/test_legacy_jp_encodings.lua b/crates/message/test_legacy_jp_encodings.lua
index 585cf420..efe42c16 100644
--- a/crates/message/test_legacy_jp_encodings.lua
+++ b/crates/message/test_legacy_jp_encodings.lua
@@ -10,45 +10,62 @@ local JP_UTF8 =
local SHIFT_JIS = kumo.encode.charset_encode('shift-jis', JP_UTF8)
utils.assert_eq(kumo.encode.charset_decode('shift-jis', SHIFT_JIS), JP_UTF8)
-local BODY = 'From: someone@example.com\r\nSubject: Testing\r\n\r\n'
+local BODY = 'From: someone@example.com\r\nContent-Type: text/plain; charset="shift-jis"\r\nSubject: Testing\r\n\r\n'
.. SHIFT_JIS
-local msg = new_msg(BODY)
-local mime_before = msg:parse_mime()
--- We might hope that we get the SHIFT_JIS bytes back out for the body,
--- but we'll try to interpret it as UTF-8 for internationalize email's sake,
--- and this input just happens to look like plausible UTF-8, even though
--- it is complete nonsense
-utils.assert_ne(mime_before.body, SHIFT_JIS)
+local function test_basics()
+ local msg = new_msg(BODY)
--- Interesting to note that the charset-normalizer based utf-8 decoder
--- won't allow converting these shift-jis bytes to utf-8, but Rust's
--- own built in str::from_utf8 will.
--- utils.assert_eq(mime_before.body, kumo.encode.charset_decode('utf-8', SHIFT_JIS))
+ local mime_before = msg:parse_mime()
+ -- We might hope that we get the SHIFT_JIS bytes back out for the body,
+ -- but we'll try to interpret it as UTF-8 for internationalize email's sake,
+ -- and this input just happens to look like plausible UTF-8, even though
+ -- it is complete nonsense
+ utils.assert_ne(mime_before.body, SHIFT_JIS)
--- Can we dkim-sign this binary content?
-local signer = kumo.dkim.rsa_sha256_signer {
- domain = 'example.com',
- selector = 'default',
- headers = { 'From', 'Subject' },
- key = 'example-private-dkim-key.pem',
-}
-msg:dkim_sign(signer)
--- kumo.log_info(msg:get_data())
+ -- Interesting to note that the charset-normalizer based utf-8 decoder
+ -- won't allow converting these shift-jis bytes to utf-8, but Rust's
+ -- own built in str::from_utf8 will.
+ -- utils.assert_eq(mime_before.body, kumo.encode.charset_decode('utf-8', SHIFT_JIS))
--- Use encoding detection to fix the encoding up. Note that this
--- definitely breaks the dkim signature, but we don't care in the
--- context of this test; the above is testing whether can sign
--- the wonky encoding, the below is testing whether we can fix
--- it. In practice, a given node will be doing either one or
--- the other.
-msg:check_fix_conformance('', 'NEEDS_TRANSFER_ENCODING', {
- detect_encoding = true,
- include_encodings = {
- 'shift-jis',
- },
- exclude_encodings = {},
-})
+ -- Can we dkim-sign this binary content?
+ local signer = kumo.dkim.rsa_sha256_signer {
+ domain = 'example.com',
+ selector = 'default',
+ headers = { 'From', 'Subject' },
+ key = 'example-private-dkim-key.pem',
+ }
+ msg:dkim_sign(signer)
+ -- kumo.log_info(msg:get_data())
-local mime = msg:parse_mime()
-utils.assert_eq(mime.body, JP_UTF8)
+ -- Use encoding detection to fix the encoding up. Note that this
+ -- definitely breaks the dkim signature, but we don't care in the
+ -- context of this test; the above is testing whether can sign
+ -- the wonky encoding, the below is testing whether we can fix
+ -- it. In practice, a given node will be doing either one or
+ -- the other.
+ msg:check_fix_conformance('', 'NEEDS_TRANSFER_ENCODING', {
+ detect_encoding = true,
+ include_encodings = {
+ 'shift-jis',
+ },
+ exclude_encodings = {},
+ })
+
+ local mime = msg:parse_mime()
+ utils.assert_eq(mime.body, JP_UTF8)
+end
+
+local function test_fix_missing_headers()
+ local msg = new_msg(BODY)
+ local res = msg:check_fix_conformance(
+ '',
+ 'MISSING_DATE_HEADER|MISSING_MESSAGE_ID_HEADER|MISSING_MIME_VERSION'
+ )
+
+ local mime = msg:parse_mime()
+ utils.assert_ne(mime.body, SHIFT_JIS)
+end
+
+test_basics()
+test_fix_missing_headers()
diff --git a/crates/mod-mimepart/src/mimepart.rs b/crates/mod-mimepart/src/mimepart.rs
index fcc837bb..ca21aae1 100644
--- a/crates/mod-mimepart/src/mimepart.rs
+++ b/crates/mod-mimepart/src/mimepart.rs
@@ -169,9 +169,9 @@ impl UserData for PartRef {
Ok(result)
});
- methods.add_meta_method(MetaMethod::ToString, move |_lua, this, ()| {
+ methods.add_meta_method(MetaMethod::ToString, move |lua, this, ()| {
let root = this.root_part.lock();
- Ok(root.to_message_string())
+ lua.create_string(root.to_message_bytes())
});
methods.add_method("rebuild", move |_lua, this, ()| {