diff --git a/Cargo.lock b/Cargo.lock index caaf2276..1c8d13ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2831,7 +2831,12 @@ name = "mailparsing" version = "0.1.0" dependencies = [ "bitflags 2.3.3", + "charset", + "data-encoding", "k9", + "pest", + "pest_derive", + "quoted_printable", "thiserror", ] diff --git a/crates/mailparsing/Cargo.toml b/crates/mailparsing/Cargo.toml index 56055112..816254a3 100644 --- a/crates/mailparsing/Cargo.toml +++ b/crates/mailparsing/Cargo.toml @@ -7,6 +7,11 @@ edition = "2021" [dependencies] bitflags = "2.3" +charset = "0.1" +data-encoding = "2.4" +pest = "2.5" +pest_derive = "2.5" +quoted_printable = "0.4" thiserror = "1.0" [dev-dependencies] diff --git a/crates/mailparsing/src/error.rs b/crates/mailparsing/src/error.rs index 29efc950..5771912e 100644 --- a/crates/mailparsing/src/error.rs +++ b/crates/mailparsing/src/error.rs @@ -4,4 +4,6 @@ use thiserror::Error; pub enum MailParsingError { #[error("invalid header: {0}")] HeaderParse(String), + #[error("Header {0} not found in mime part")] + MissingHeader(String), } diff --git a/crates/mailparsing/src/header.rs b/crates/mailparsing/src/header.rs index 0886e171..be68e7e8 100644 --- a/crates/mailparsing/src/header.rs +++ b/crates/mailparsing/src/header.rs @@ -1,5 +1,6 @@ use crate::headermap::HeaderMap; -use crate::{MailParsingError, Result, SharedString}; +use crate::rfc5322_parser::Parser; +use crate::{MailParsingError, Mailbox, MailboxList, Result, SharedString}; bitflags::bitflags! { #[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] @@ -76,6 +77,20 @@ impl<'a> Header<'a> { &self.value } + /// Parse the header into a mailbox-list (as defined in + /// RFC 5322), which is how the `From` and `Resent-From`, + /// headers are defined. + pub fn as_mailbox_list(&self) -> Result { + Parser::parse_mailbox_list_header(self.get_raw_value()) + } + + /// Parse the header into a mailbox (as defined in + /// RFC 5322), which is how the `Sender` and `Resent-Sender` + /// headers are defined. + pub fn as_mailbox(&self) -> Result { + Parser::parse_mailbox_header(self.get_raw_value()) + } + pub fn parse_headers>>( header_block: S, ) -> Result> { diff --git a/crates/mailparsing/src/headermap.rs b/crates/mailparsing/src/headermap.rs index fd5c09cf..3dd50685 100644 --- a/crates/mailparsing/src/headermap.rs +++ b/crates/mailparsing/src/headermap.rs @@ -1,4 +1,4 @@ -use crate::Header; +use crate::{Header, MailParsingError, Mailbox, MailboxList, Result}; /// Represents an ordered list of headers. /// Note that there may be multiple headers with the same name. @@ -46,4 +46,32 @@ impl<'a> HeaderMap<'a> { .iter() .filter(|header| header.get_name().eq_ignore_ascii_case(name)) } + + pub fn from(&self) -> Result> { + match self.get_first("From") { + None => Ok(None), + Some(header) => Ok(Some(header.as_mailbox_list()?)), + } + } + + pub fn resent_from(&self) -> Result> { + match self.get_first("Resent-From") { + None => Ok(None), + Some(header) => Ok(Some(header.as_mailbox_list()?)), + } + } + + pub fn sender(&self) -> Result> { + match self.get_first("Sender") { + None => Ok(None), + Some(header) => Ok(Some(header.as_mailbox()?)), + } + } + + pub fn resent_sender(&self) -> Result> { + match self.get_first("Resent-Sender") { + None => Ok(None), + Some(header) => Ok(Some(header.as_mailbox()?)), + } + } } diff --git a/crates/mailparsing/src/lib.rs b/crates/mailparsing/src/lib.rs index 64ac38bc..06cc4e3b 100644 --- a/crates/mailparsing/src/lib.rs +++ b/crates/mailparsing/src/lib.rs @@ -2,6 +2,7 @@ mod error; mod header; mod headermap; mod mimepart; +mod rfc5322_parser; mod strings; pub use error::MailParsingError; @@ -10,4 +11,5 @@ pub type Result = std::result::Result; pub use header::Header; pub use headermap::HeaderMap; pub use mimepart::MimePart; +pub use rfc5322_parser::*; pub use strings::SharedString; diff --git a/crates/mailparsing/src/mimepart.rs b/crates/mailparsing/src/mimepart.rs index 7c60798d..f6975f81 100644 --- a/crates/mailparsing/src/mimepart.rs +++ b/crates/mailparsing/src/mimepart.rs @@ -28,7 +28,7 @@ impl<'a> MimePart<'a> { }) } - pub fn headers(&self) -> &Vec
{ + pub fn headers(&self) -> &HeaderMap { &self.headers } diff --git a/crates/mailparsing/src/rfc5322.pest b/crates/mailparsing/src/rfc5322.pest new file mode 100644 index 00000000..112b1097 --- /dev/null +++ b/crates/mailparsing/src/rfc5322.pest @@ -0,0 +1,100 @@ +path = { angle_addr | ( cfws? ~ "<" ~ cfws? ~ ">" ~ cfws? ) } +received_token = { word | angle_addr | addr_spec | domain } + +address = { mailbox | group } +mailbox = { name_addr | addr_spec } +name_addr = { display_name? ~ angle_addr } + +angle_addr = { cfws? ~ "<" ~ addr_spec ~ ">" ~ cfws? | obs_angle_addr } +obs_angle_addr = { cfws? ~ "<" ~ obs_route ~ addr_spec ~ ">" ~ cfws? } + +group = { display_name ~ ":" ~ group_list? ~ ";" ~ cfws? } +display_name = { phrase } + +mailbox_list = { (mailbox ~ ("," ~ mailbox)*) | obs_mbox_list } +obs_mbox_list = { ((cfws? ~ ",")* ~ mailbox ~ ("," ~ (mailbox | cfws))*)+ } + +address_list = { (address ~ ("," ~ address)*) | obs_addr_list } +obs_addr_list = { ((cfws? ~ ",")* ~ address ~ ("," ~ (address | cfws))*)+ } + +group_list = { mailbox_list | cfws | obs_group_list } +obs_group_list = { (cfws? ~ ",")+ ~ cfws? } + +addr_spec = { local_part ~ "@" ~ domain } + +local_part = { dot_atom | quoted_string | obs_local_part } +obs_local_part = { word ~ (dot ~ word)* } +dot = @{ "." } + +domain = { dot_atom | domain_literal | obs_domain } +obs_domain = { atom ~ ( dot ~ atom)* } + +domain_literal = { cfws? ~ "[" ~ (fws? ~ dtext)* ~ fws? ~ "]" ~ cfws? } + +dtext = { '\u{21}'..'\u{5a}' | '\u{5e}'..'\u{7e}' | obs_dtext } +obs_dtext = { obs_no_ws_ctl | quoted_pair } + +alpha = { 'a'..'z' | 'A'..'Z' } +digit = { '0'..'9' } +atext_char = { "!" | "#" | "$" | "%" | "&" | "'" | "*" | "+" | "-" | "/" | "=" | + "?" | "^" | "_" | "`" | "{" | "|" | "}" | "~" | alpha | digit } + +atext = @{ atext_char+ } + +atom = { cfws? ~ atext ~ cfws? } +dot_atom_text = @{ atext ~ ("." ~ atext)* } +dot_atom = { cfws? ~ dot_atom_text ~ cfws? } +specials = { "(" | ")" | "<" | ">" | "[" | "]" | ":" | ";" | "@" | "\\" | "," | "." | "\"" } + +qtext = { "\u{21}" | '\u{23}'..'\u{5b}' | '\u{5d}'..'\u{7e}' | obs_qtext } +obs_qtext = { obs_no_ws_ctl } + +qcontent = { qtext | quoted_pair } +quoted_string = { cfws? ~ "\"" ~ (fws? ~ qcontent)* ~ fws? ~ "\"" ~ cfws? } +quoted_pair = { ( "\\" ~ (vchar | wsp)) | obs_qp } +obs_qp = { "\\" ~ ( "\u{00}" | obs_no_ws_ctl | "\r" | "\n") } + +word = { atom | quoted_string } + +phrase = { encoded_word+ | word+ | obs_phrase } +obs_phrase = { word ~ (word | "." | cfws)* } + +unstructured = { + // This rule always matches; since obs_unstruct is a superset, + // and this is an unstructured field anyway, we just use the obsolete + // syntax to match the header content + // ((fws? ~ vchar)* ~ wsp*) | + obs_unstruct +} +obs_unstruct = { (( "\r"* ~ "\n"* ~ (obs_utext ~ "\r"* ~ "\n"*)+) | fws)+ } +obs_utext = { "\u{00}" | obs_no_ws_ctl | vchar } + +obs_no_ws_ctl = { '\u{01}'..'\u{08}' | '\u{0b}'..'\u{0c}' | '\u{0d}'..'\u{1f}' | "\u{7f}" } +obs_ctext = { obs_no_ws_ctl } +obs_phrase_list = { (phrase | cfws)? ~ ("," ~ (phrase | cfws)?)* } +obs_route = { obs_domain_list ~ ":" } +obs_domain_list = { (cfws | ",")* ~ "@" ~ domain ~ ("," ~ cfws? ~ ("@" ~ domain)?)* } + +obs_fws = { wsp+ ~ ("\r"? ~ "\n" ~ wsp+)* } + +fws = { ((wsp* ~ "\r"? ~ "\n")* ~ wsp+) | obs_fws } +ctext = { '\u{21}'..'\u{27}' | '\u{2a}'..'\u{5b}' | '\u{5d}'..'\u{7e}' | obs_ctext } +ccontent = { ctext | quoted_pair | comment | encoded_word } +comment = { "(" ~ (fws? ~ ccontent)* ~ fws? ~ ")" } +cfws = { ( (fws? ~ comment)+ ~ fws?) | fws } +wsp = { " " | "\t" } + +vchar = { '\u{21}'..'\u{7e}' } + +encoded_word = { "=?" ~ charset ~ "?" ~ encoding ~ "?" ~ encoded_text ~ "?=" } +encoded_text = @{ (!( " " | "?") ~ vchar)+ } +charset = @{ token+ } +encoding = @{ token+ } +ctl = { '\u{00}'..'\u{1f}' | "\u{7f}" } +char = { '\u{01}'..'\u{7f}' } +token = { !(" " | especials | ctl) ~ char } +especials = { "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | + "<" | ">" | "/" | "[" | "]" | "?" | "." | "=" } + +parse_mailbox_list = _{ SOI ~ mailbox_list ~ EOI } +parse_mailbox = _{ SOI ~ mailbox ~ EOI } diff --git a/crates/mailparsing/src/rfc5322_parser.rs b/crates/mailparsing/src/rfc5322_parser.rs new file mode 100644 index 00000000..f62b5fd4 --- /dev/null +++ b/crates/mailparsing/src/rfc5322_parser.rs @@ -0,0 +1,529 @@ +use crate::{MailParsingError, Result}; +use charset::Charset; +use pest::iterators::{Pair, Pairs}; +use pest::Parser as _; +use pest_derive::*; + +#[derive(Parser)] +#[grammar = "rfc5322.pest"] +pub struct Parser; + +impl Parser { + pub fn parse_mailbox_list_header(text: &str) -> Result { + let mut pairs = Self::parse(Rule::parse_mailbox_list, text) + .map_err(|err| MailParsingError::HeaderParse(format!("{err:#}")))? + .next() + .unwrap() + .into_inner(); + + let mut result: Vec = vec![]; + + while let Some(pair) = pairs.next() { + result.push(Self::parse_mailbox(pair.into_inner())?); + } + + Ok(MailboxList(result)) + } + + pub fn parse_mailbox_header(text: &str) -> Result { + let pairs = Self::parse(Rule::parse_mailbox, text) + .map_err(|err| MailParsingError::HeaderParse(format!("{err:#}")))? + .next() + .unwrap() + .into_inner(); + + Self::parse_mailbox(pairs) + } + + fn parse_mailbox(pairs: Pairs) -> Result { + for p in pairs { + match p.as_rule() { + Rule::name_addr => return Self::parse_name_addr(p), + Rule::addr_spec => { + return Ok(Mailbox { + name: None, + address: Self::parse_addr_spec(p)?, + }) + } + rule => { + return Err(MailParsingError::HeaderParse(format!( + "Expected name_addr or addr_spec, but had {rule:?} {p:?}" + ))) + } + }; + } + Err(MailParsingError::HeaderParse( + "unreachable end of loop in parse_mailbox".to_string(), + )) + } + + fn parse_dot_atom(pair: Pair) -> Result { + for p in pair.into_inner() { + match p.as_rule() { + Rule::cfws => {} + Rule::dot_atom_text => return Ok(p.as_str().to_string()), + rule => { + return Err(MailParsingError::HeaderParse(format!( + "invalid {rule:?} {p:#?} in parse_dot_atom" + ))) + } + } + } + + Err(MailParsingError::HeaderParse(format!( + "Unreachable end of loop in parse_dot_atom" + ))) + } + + fn parse_local_part(pair: Pair) -> Result { + for p in pair.into_inner() { + match p.as_rule() { + Rule::dot_atom => return Self::parse_dot_atom(p), + Rule::quoted_string => return Self::parse_quoted_string(p), + Rule::obs_local_part => return Self::parse_obs_local_part(p), + rule => { + return Err(MailParsingError::HeaderParse(format!( + "Invalid {rule:?} {p:#?} in parse_local_part" + ))) + } + } + } + Err(MailParsingError::HeaderParse(format!( + "Unreachable end of loop in parse_local_part" + ))) + } + + fn parse_obs_local_part(pair: Pair) -> Result { + let mut result = String::new(); + for p in pair.into_inner() { + match p.as_rule() { + Rule::word => { + result += &Self::parse_word(p)?; + } + Rule::dot => { + result += p.as_str(); + } + rule => { + return Err(MailParsingError::HeaderParse(format!( + "Invalid {rule:?} {p:#?} in parse_obs_local_part" + ))) + } + } + } + Ok(result) + } + + fn parse_obs_domain(pair: Pair) -> Result { + let mut result = String::new(); + for p in pair.into_inner() { + match p.as_rule() { + Rule::atom => { + result += &Self::parse_atom(p)?; + } + Rule::dot => { + result += p.as_str(); + } + rule => { + return Err(MailParsingError::HeaderParse(format!( + "Invalid {rule:?} {p:#?} in parse_obs_domain" + ))) + } + } + } + Ok(result) + } + + fn parse_domain(pair: Pair) -> Result { + for p in pair.into_inner() { + match p.as_rule() { + Rule::dot_atom => return Self::parse_dot_atom(p), + Rule::domain_literal => return Self::parse_domain_literal(p), + Rule::obs_domain => return Self::parse_obs_domain(p), + rule => { + return Err(MailParsingError::HeaderParse(format!( + "Unhandled {rule:?} {p:#?} in parse_domain" + ))) + } + } + } + + Err(MailParsingError::HeaderParse(format!( + "Unreachable end of loop in parse_domain" + ))) + } + + fn parse_domain_literal(pair: Pair) -> Result { + let mut result = "[".to_string(); + for p in pair.into_inner() { + match p.as_rule() { + Rule::fws | Rule::cfws => {} + Rule::dtext => { + let dtext = p.as_str(); + if dtext.len() == 2 { + // Must be quoted_pair + result.push_str(&dtext[1..]); + } else { + result.push_str(dtext); + } + } + rule => { + return Err(MailParsingError::HeaderParse(format!( + "Unhandled {rule:?} {p:#?} in parse_domain_literal" + ))) + } + } + } + + result.push(']'); + + Ok(result) + } + + fn parse_addr_spec(pair: Pair) -> Result { + let mut result = String::new(); + + for p in pair.into_inner() { + match p.as_rule() { + Rule::local_part => { + result = Self::parse_local_part(p)?; + result.push('@'); + } + Rule::domain => { + result += &Self::parse_domain(p)?; + } + rule => { + return Err(MailParsingError::HeaderParse(format!( + "Invalid {rule:?} {p:#?} in parse_addr_spec" + ))) + } + } + } + + Ok(result) + } + + fn parse_angle_addr(pair: Pair) -> Result { + for p in pair.into_inner() { + match p.as_rule() { + Rule::addr_spec => return Self::parse_addr_spec(p), + Rule::cfws => {} + rule => { + return Err(MailParsingError::HeaderParse(format!( + "Unhandled {rule:?} {p:#?} in parse_angle_addr" + ))) + } + } + } + Err(MailParsingError::HeaderParse( + "unreachable end of loop in parse_angle_addr".to_string(), + )) + } + + fn parse_word(pair: Pair) -> Result { + for p in pair.into_inner() { + match p.as_rule() { + Rule::atom => return Self::parse_atom(p), + Rule::quoted_string => return Self::parse_quoted_string(p), + rule => { + return Err(MailParsingError::HeaderParse(format!( + "Invalid {rule:?} {p:#?} in parse_word" + ))) + } + } + } + Err(MailParsingError::HeaderParse( + "unreachable end of loop in parse_word".to_string(), + )) + } + + fn parse_quoted_string(pair: Pair) -> Result { + let mut result = String::new(); + let mut fws = false; + + for p in pair.into_inner() { + match p.as_rule() { + Rule::fws | Rule::cfws => { + if !result.is_empty() && !fws { + result.push(' '); + } + fws = true; + } + Rule::qcontent => { + fws = false; + let content = p.into_inner().next().unwrap(); + match content.as_rule() { + Rule::qtext => result.push_str(content.as_str()), + Rule::quoted_pair => result.push_str(&content.as_str()[1..]), + rule => { + return Err(MailParsingError::HeaderParse(format!( + "Invalid {rule:?} {content:#?} in parse_quoted_string qcontent" + ))) + } + } + } + rule => { + return Err(MailParsingError::HeaderParse(format!( + "Invalid {rule:?} {p:#?} in parse_quoted_string" + ))) + } + } + } + + if fws { + result.pop(); + } + + Ok(result) + } + + fn parse_encoded_word(pair: Pair) -> Result { + let mut charset = String::new(); + let mut encoding = String::new(); + let mut text = String::new(); + + for p in pair.into_inner() { + match p.as_rule() { + Rule::charset => { + charset = p.as_str().to_string(); + } + Rule::encoding => { + encoding = p.as_str().to_string(); + } + Rule::encoded_text => { + text = p.as_str().to_string(); + } + rule => { + return Err(MailParsingError::HeaderParse(format!( + "Invalid {rule:?} {p:#?} in parse_encoded_word" + ))) + } + } + } + + let bytes = match encoding.as_str() { + "B" | "b" => data_encoding::BASE64_MIME + .decode(text.as_bytes()) + .map_err(|err| { + MailParsingError::HeaderParse(format!( + "Invalid base64 encoding: {err:#} {text:?}" + )) + })?, + "Q" | "q" => quoted_printable::decode( + text.replace("_", " "), + quoted_printable::ParseMode::Robust, + ) + .map_err(|err| { + MailParsingError::HeaderParse(format!( + "Invalid quoted printable encoding: {err:#} {text:?}" + )) + })?, + _ => { + return Err(MailParsingError::HeaderParse(format!( + "Invalid encoding {encoding} in parse_encoded_word" + ))) + } + }; + + let charset = Charset::for_label_no_replacement(charset.as_bytes()).ok_or_else(|| { + MailParsingError::HeaderParse(format!("unsupported charset: {charset:?}")) + })?; + + let (decoded, _malformed) = charset.decode_without_bom_handling(&bytes); + + Ok(decoded.into()) + } + + fn parse_atom(pair: Pair) -> Result { + for p in pair.into_inner() { + match p.as_rule() { + Rule::atext => return Ok(p.as_str().to_string()), + rule => { + return Err(MailParsingError::HeaderParse(format!( + "Unhandled {rule:?} {p:#?} in parse_atom" + ))) + } + } + } + Err(MailParsingError::HeaderParse( + "unreachable end of loop in parse_atom".to_string(), + )) + } + + fn parse_phrase(pair: Pair) -> Result> { + let mut words = vec![]; + for p in pair.into_inner() { + match p.as_rule() { + Rule::word => words.push(Self::parse_word(p)?), + Rule::encoded_word => words.push(Self::parse_encoded_word(p)?), + rule => { + return Err(MailParsingError::HeaderParse(format!( + "Unhandled {rule:?} in parse_phrase" + ))) + } + } + } + Ok(words) + } + + fn parse_display_name(pair: Pair) -> Result { + let words = Self::parse_phrase(pair.into_inner().next().unwrap())?; + Ok(words.join(" ")) + } + + fn parse_name_addr(name_addr: Pair) -> Result { + let name_addr = name_addr.into_inner(); + let mut name = None; + + for p in name_addr { + match p.as_rule() { + Rule::display_name => { + name.replace(Self::parse_display_name(p)?); + //name.replace(Self::text_ignoring_cfws(p, true)?); + } + Rule::angle_addr => { + let address = Self::parse_angle_addr(p)?; + return Ok(Mailbox { name, address }); + } + rule => { + return Err(MailParsingError::HeaderParse(format!( + "parse_name_addr: invalid {rule:?} for {p:?}" + ))) + } + }; + } + + Err(MailParsingError::HeaderParse(format!( + "Unreachable end of loop in parse_name_addr" + ))) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MailboxList(Vec); + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Mailbox { + pub name: Option, + pub address: String, +} + +#[cfg(test)] +mod test { + use crate::MimePart; + + #[test] + fn mailbox_list_singular() { + let message = concat!( + "From: Someone (hello) , other@example.com,\n", + " \"John \\\"Smith\\\"\" (comment) \"More Quotes\" (more comment) \n", + "\n", + "I am the body" + ); + let msg = MimePart::parse(message).unwrap(); + let list = match msg.headers().from() { + Err(err) => panic!("Doh.\n{err:#}"), + Ok(list) => list, + }; + + k9::snapshot!( + list, + r#" +Some( + MailboxList( + [ + Mailbox { + name: Some( + "Someone", + ), + address: "someone@example.com", + }, + Mailbox { + name: None, + address: "other@example.com", + }, + Mailbox { + name: Some( + "John "Smith" More Quotes", + ), + address: "someone@crazy.example.com", + }, + ], + ), +) +"# + ); + } + + #[test] + fn sender() { + let message = "Sender: someone@[127.0.0.1]\n\n\n"; + let msg = MimePart::parse(message).unwrap(); + let list = match msg.headers().sender() { + Err(err) => panic!("Doh.\n{err:#}"), + Ok(list) => list, + }; + k9::snapshot!( + list, + r#" +Some( + Mailbox { + name: None, + address: "someone@[127.0.0.1]", + }, +) +"# + ); + } + + #[test] + fn domain_literal() { + let message = "From: someone@[127.0.0.1]\n\n\n"; + let msg = MimePart::parse(message).unwrap(); + let list = match msg.headers().from() { + Err(err) => panic!("Doh.\n{err:#}"), + Ok(list) => list, + }; + k9::snapshot!( + list, + r#" +Some( + MailboxList( + [ + Mailbox { + name: None, + address: "someone@[127.0.0.1]", + }, + ], + ), +) +"# + ); + } + + #[test] + fn rfc2047() { + let message = "From: =?US-ASCII?Q?Keith_Moore?= \n\n\n"; + let msg = MimePart::parse(message).unwrap(); + let list = match msg.headers().from() { + Err(err) => panic!("Doh.\n{err:#}"), + Ok(list) => list, + }; + k9::snapshot!( + list, + r#" +Some( + MailboxList( + [ + Mailbox { + name: Some( + "Keith Moore", + ), + address: "moore@cs.utk.edu", + }, + ], + ), +) +"# + ); + } +}