mailparsing: switch parser input to byte slices

There's a bit of fuss around migrating the `char` nom parsing function
to `tag`, which is required for operating on bytes.  We need to jump
through some hoops in order to be able to show what tag was expected
if there is a mismatch.  That improves the output of our parse error
type to have less redundant information, so overall I'd say that
that is a plus.

A number of 8-bit related tests now fail and some continue to fail
from an earlier commit in this refactoring series.  We can now
start to tackle whether those need to be adjusted to the amended
API surface, or whether we need to add some supporting context
for those, on a case by case basis in follow-up commits.

crates/kumod/src/http_server/inject_v1.rs

@@ -713,7 +713,7 @@ impl InjectV1Request {
                     let opts = mailparsing::AttachmentOptions {
                         file_name: a.file_name.clone().map(|s| s.into()),
                         inline: a.content_id.is_some(),
-                        content_id: a.content_id.clone(),
+                        content_id: a.content_id.clone().map(Into::into),
                     };
 
                     let decoded_data;

crates/mailparsing/src/header.rs

@@ -184,57 +184,57 @@ impl<'a> Header<'a> {
     }
 
     pub fn as_content_transfer_encoding(&self) -> Result<MimeParameters> {
-        Parser::parse_content_transfer_encoding_header(self.get_raw_value_string()?)
+        Parser::parse_content_transfer_encoding_header(self.get_raw_value())
     }
 
     pub fn as_content_disposition(&self) -> Result<MimeParameters> {
-        Parser::parse_content_transfer_encoding_header(self.get_raw_value_string()?)
+        Parser::parse_content_transfer_encoding_header(self.get_raw_value())
     }
 
     pub fn as_content_type(&self) -> Result<MimeParameters> {
-        Parser::parse_content_type_header(self.get_raw_value_string()?)
+        Parser::parse_content_type_header(self.get_raw_value())
     }
 
     /// Parse the header into a mailbox-list (as defined in
     /// RFC 5322), which is how the `From` and `Resent-From`,
     /// headers are defined.
     pub fn as_mailbox_list(&self) -> Result<MailboxList> {
-        Parser::parse_mailbox_list_header(self.get_raw_value_string()?)
+        Parser::parse_mailbox_list_header(self.get_raw_value())
     }
 
     /// Parse the header into a mailbox (as defined in
     /// RFC 5322), which is how the `Sender` and `Resent-Sender`
     /// headers are defined.
     pub fn as_mailbox(&self) -> Result<Mailbox> {
-        Parser::parse_mailbox_header(self.get_raw_value_string()?)
+        Parser::parse_mailbox_header(self.get_raw_value())
     }
 
     pub fn as_address_list(&self) -> Result<AddressList> {
-        Parser::parse_address_list_header(self.get_raw_value_string()?)
+        Parser::parse_address_list_header(self.get_raw_value())
     }
 
     pub fn as_message_id(&self) -> Result<MessageID> {
-        Parser::parse_msg_id_header(self.get_raw_value_string()?)
+        Parser::parse_msg_id_header(self.get_raw_value())
     }
 
     pub fn as_content_id(&self) -> Result<MessageID> {
-        Parser::parse_content_id_header(self.get_raw_value_string()?)
+        Parser::parse_content_id_header(self.get_raw_value())
     }
 
     pub fn as_message_id_list(&self) -> Result<Vec<MessageID>> {
-        Parser::parse_msg_id_header_list(self.get_raw_value_string()?)
+        Parser::parse_msg_id_header_list(self.get_raw_value())
     }
 
     pub fn as_unstructured(&self) -> Result<BString> {
-        Parser::parse_unstructured_header(self.get_raw_value_string()?)
+        Parser::parse_unstructured_header(self.get_raw_value())
     }
 
     pub fn as_authentication_results(&self) -> Result<AuthenticationResults> {
-        Parser::parse_authentication_results_header(self.get_raw_value_string()?)
+        Parser::parse_authentication_results_header(self.get_raw_value())
     }
 
     pub fn as_arc_authentication_results(&self) -> Result<ARCAuthenticationResults> {
-        Parser::parse_arc_authentication_results_header(self.get_raw_value_string()?)
+        Parser::parse_arc_authentication_results_header(self.get_raw_value())
     }
 
     pub fn as_date(&self) -> Result<DateTime<FixedOffset>> {

crates/mailparsing/src/mimepart.rs

@@ -834,7 +834,7 @@ impl<'a> MimePart<'a> {
             headers.set_content_disposition(cd)?;
 
             if let Some(id) = &opts.content_id {
-                headers.set_content_id(MessageID(id.to_string()))?;
+                headers.set_content_id(MessageID(id.clone()))?;
             }
         }
 
@@ -1142,7 +1142,7 @@ impl<'a> MimePart<'a> {
         if to_fix.contains(MessageConformance::MISSING_MESSAGE_ID_HEADER) {
             if let Some(message_id) = &settings.message_id {
                 msg.headers_mut()
-                    .set_message_id(MessageID(message_id.clone()))?;
+                    .set_message_id(MessageID(message_id.clone().into()))?;
             }
         }
 
@@ -1243,7 +1243,7 @@ pub struct AttachmentOptions {
     #[serde(default)]
     pub inline: bool,
     #[serde(default)]
-    pub content_id: Option<String>,
+    pub content_id: Option<BString>,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]

crates/mailparsing/src/nom_utils.rs

@@ -1,11 +1,12 @@
+use bstr::ByteSlice;
 use nom::error::{ContextError, ErrorKind};
 use nom_locate::LocatedSpan;
 use std::fmt::{Debug, Write};
 
-pub(crate) type Span<'a> = LocatedSpan<&'a str>;
+pub(crate) type Span<'a> = LocatedSpan<&'a [u8]>;
 pub(crate) type IResult<'a, A, B> = nom::IResult<A, B, ParseError<Span<'a>>>;
 
-pub fn make_span(s: &'_ str) -> Span<'_> {
+pub fn make_span(s: &'_ [u8]) -> Span<'_> {
     Span::new(s)
 }
 
@@ -81,20 +82,22 @@ pub fn explain_nom(input: Span, err: nom::Err<ParseError<Span<'_>>>) -> String {
     match err {
         nom::Err::Error(e) => {
             let mut result = String::new();
-            for (i, (span, kind)) in e.errors.iter().enumerate() {
+            let mut lines_shown = vec![];
+
+            for (span, kind) in e.errors.iter() {
                 if input.is_empty() {
                     match kind {
                         ParseErrorKind::Char(c) => {
-                            write!(&mut result, "{i}: expected '{c}', got empty input\n\n")
+                            write!(&mut result, "Error expected '{c}', got empty input\n\n")
                         }
                         ParseErrorKind::Context(s) => {
-                            write!(&mut result, "{i}: in {s}, got empty input\n\n")
+                            write!(&mut result, "Error in {s}, got empty input\n\n")
                         }
                         ParseErrorKind::External { kind, reason } => {
-                            write!(&mut result, "{i}: {reason} {kind:?}, got empty input\n\n")
+                            write!(&mut result, "Error {reason} {kind:?}, got empty input\n\n")
                         }
                         ParseErrorKind::Nom(e) => {
-                            write!(&mut result, "{i}: in {e:?}, got empty input\n\n")
+                            write!(&mut result, "Error in {e:?}, got empty input\n\n")
                         }
                     }
                     .ok();
@@ -102,21 +105,33 @@ pub fn explain_nom(input: Span, err: nom::Err<ParseError<Span<'_>>>) -> String {
                 }
 
                 let line_number = span.location_line();
-                let line = std::str::from_utf8(span.get_line_beginning())
-                    .unwrap_or("<INVALID: line slice is not utf8!>");
+                let input_line = span.get_line_beginning();
                 // Remap \t in particular, because it can render as multiple
                 // columns and defeat the column number calculation provided
                 // by the Span type
-                let line: String = line
-                    .chars()
-                    .map(|c| match c {
+                let mut line = String::new();
+                for (start, end, c) in input_line.char_indices() {
+                    let c = match c {
                         '\t' => '\u{2409}',
                         '\r' => '\u{240d}',
                         '\n' => '\u{240a}',
-                        _ => c,
-                    })
-                    .collect();
+                        c => c,
+                    };
+
+                    if c == std::char::REPLACEMENT_CHARACTER {
+                        let bytes = &input_line[start..end];
+                        for b in bytes.iter() {
+                            line.push_str(&format!("\\x{b:02X}"));
+                        }
+                    } else {
+                        line.push(c);
+                    }
+                }
+
                 let column = span.get_utf8_column();
+
+                lines_shown.push(line_number);
+
                 let mut caret = " ".repeat(column.saturating_sub(1));
                 caret.push('^');
                 for _ in 1..span.fragment().len() {
@@ -128,7 +143,7 @@ pub fn explain_nom(input: Span, err: nom::Err<ParseError<Span<'_>>>) -> String {
                         if let Some(actual) = span.fragment().chars().next() {
                             write!(
                                 &mut result,
-                                "{i}: at line {line_number}:\n\
+                                "Error at line {line_number}:\n\
                                     {line}\n\
                                     {caret}\n\
                                     expected '{expected}', found {actual}\n\n",
@@ -136,7 +151,7 @@ pub fn explain_nom(input: Span, err: nom::Err<ParseError<Span<'_>>>) -> String {
                         } else {
                             write!(
                                 &mut result,
-                                "{i}: at line {line_number}:\n\
+                                "Error at line {line_number}:\n\
                                     {line}\n\
                                     {caret}\n\
                                     expected '{expected}', got end of input\n\n",
@@ -144,17 +159,12 @@ pub fn explain_nom(input: Span, err: nom::Err<ParseError<Span<'_>>>) -> String {
                         }
                     }
                     ParseErrorKind::Context(context) => {
-                        write!(
-                            &mut result,
-                            "{i}: at line {line_number}, in {context}:\n\
-                                {line}\n\
-                                {caret}\n\n",
-                        )
+                        write!(&mut result, "while parsing {context}\n")
                     }
-                    ParseErrorKind::External { kind, reason } => {
+                    ParseErrorKind::External { kind: _, reason } => {
                         write!(
                             &mut result,
-                            "{i}: at line {line_number}, {reason} {kind:?}:\n\
+                            "Error at line {line_number}, {reason}:\n\
                                 {line}\n\
                                 {caret}\n\n",
                         )
@@ -162,7 +172,7 @@ pub fn explain_nom(input: Span, err: nom::Err<ParseError<Span<'_>>>) -> String {
                     ParseErrorKind::Nom(nom_err) => {
                         write!(
                             &mut result,
-                            "{i}: at line {line_number}, in {nom_err:?}:\n\
+                            "Error at line {line_number}, in {nom_err:?}:\n\
                                 {line}\n\
                                 {caret}\n\n",
                         )

crates/mailparsing/src/strings.rs

@@ -107,11 +107,6 @@ impl SharedString<'_> {
         String::from_utf8_lossy(self.as_bytes())
     }
 
-    #[deprecated = "use either as_bytes() or to_str() instead"]
-    pub fn as_str(&self) -> &str {
-        self.to_str().expect("SharedString is not UTF-8")
-    }
-
     pub fn len(&self) -> usize {
         match self {
             Self::Owned(s) => s.len(),