mailparsing: add Mailbox and MailboxList header parsing

For eg: the Sender and From headers.

This commit introduces a parser for RFC 5322 message header syntax,
as modified by RFC 2047 for encoded header fields.
This commit is contained in:
Wez Furlong
2023-08-13 22:10:13 -07:00
parent 41d7bf406f
commit 6e52594de4
9 changed files with 689 additions and 3 deletions
Generated
+5
View File
@@ -2831,7 +2831,12 @@ name = "mailparsing"
version = "0.1.0"
dependencies = [
"bitflags 2.3.3",
"charset",
"data-encoding",
"k9",
"pest",
"pest_derive",
"quoted_printable",
"thiserror",
]
+5
View File
@@ -7,6 +7,11 @@ edition = "2021"
[dependencies]
bitflags = "2.3"
charset = "0.1"
data-encoding = "2.4"
pest = "2.5"
pest_derive = "2.5"
quoted_printable = "0.4"
thiserror = "1.0"
[dev-dependencies]
+2
View File
@@ -4,4 +4,6 @@ use thiserror::Error;
pub enum MailParsingError {
#[error("invalid header: {0}")]
HeaderParse(String),
#[error("Header {0} not found in mime part")]
MissingHeader(String),
}
+16 -1
View File
@@ -1,5 +1,6 @@
use crate::headermap::HeaderMap;
use crate::{MailParsingError, Result, SharedString};
use crate::rfc5322_parser::Parser;
use crate::{MailParsingError, Mailbox, MailboxList, Result, SharedString};
bitflags::bitflags! {
#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)]
@@ -76,6 +77,20 @@ impl<'a> Header<'a> {
&self.value
}
/// Parse the header into a mailbox-list (as defined in
/// RFC 5322), which is how the `From` and `Resent-From`,
/// headers are defined.
pub fn as_mailbox_list(&self) -> Result<MailboxList> {
Parser::parse_mailbox_list_header(self.get_raw_value())
}
/// Parse the header into a mailbox (as defined in
/// RFC 5322), which is how the `Sender` and `Resent-Sender`
/// headers are defined.
pub fn as_mailbox(&self) -> Result<Mailbox> {
Parser::parse_mailbox_header(self.get_raw_value())
}
pub fn parse_headers<S: Into<SharedString<'a>>>(
header_block: S,
) -> Result<HeaderParseResult<'a>> {
+29 -1
View File
@@ -1,4 +1,4 @@
use crate::Header;
use crate::{Header, MailParsingError, Mailbox, MailboxList, Result};
/// Represents an ordered list of headers.
/// Note that there may be multiple headers with the same name.
@@ -46,4 +46,32 @@ impl<'a> HeaderMap<'a> {
.iter()
.filter(|header| header.get_name().eq_ignore_ascii_case(name))
}
pub fn from(&self) -> Result<Option<MailboxList>> {
match self.get_first("From") {
None => Ok(None),
Some(header) => Ok(Some(header.as_mailbox_list()?)),
}
}
pub fn resent_from(&self) -> Result<Option<MailboxList>> {
match self.get_first("Resent-From") {
None => Ok(None),
Some(header) => Ok(Some(header.as_mailbox_list()?)),
}
}
pub fn sender(&self) -> Result<Option<Mailbox>> {
match self.get_first("Sender") {
None => Ok(None),
Some(header) => Ok(Some(header.as_mailbox()?)),
}
}
pub fn resent_sender(&self) -> Result<Option<Mailbox>> {
match self.get_first("Resent-Sender") {
None => Ok(None),
Some(header) => Ok(Some(header.as_mailbox()?)),
}
}
}
+2
View File
@@ -2,6 +2,7 @@ mod error;
mod header;
mod headermap;
mod mimepart;
mod rfc5322_parser;
mod strings;
pub use error::MailParsingError;
@@ -10,4 +11,5 @@ pub type Result<T> = std::result::Result<T, MailParsingError>;
pub use header::Header;
pub use headermap::HeaderMap;
pub use mimepart::MimePart;
pub use rfc5322_parser::*;
pub use strings::SharedString;
+1 -1
View File
@@ -28,7 +28,7 @@ impl<'a> MimePart<'a> {
})
}
pub fn headers(&self) -> &Vec<Header> {
pub fn headers(&self) -> &HeaderMap {
&self.headers
}
+100
View File
@@ -0,0 +1,100 @@
path = { angle_addr | ( cfws? ~ "<" ~ cfws? ~ ">" ~ cfws? ) }
received_token = { word | angle_addr | addr_spec | domain }
address = { mailbox | group }
mailbox = { name_addr | addr_spec }
name_addr = { display_name? ~ angle_addr }
angle_addr = { cfws? ~ "<" ~ addr_spec ~ ">" ~ cfws? | obs_angle_addr }
obs_angle_addr = { cfws? ~ "<" ~ obs_route ~ addr_spec ~ ">" ~ cfws? }
group = { display_name ~ ":" ~ group_list? ~ ";" ~ cfws? }
display_name = { phrase }
mailbox_list = { (mailbox ~ ("," ~ mailbox)*) | obs_mbox_list }
obs_mbox_list = { ((cfws? ~ ",")* ~ mailbox ~ ("," ~ (mailbox | cfws))*)+ }
address_list = { (address ~ ("," ~ address)*) | obs_addr_list }
obs_addr_list = { ((cfws? ~ ",")* ~ address ~ ("," ~ (address | cfws))*)+ }
group_list = { mailbox_list | cfws | obs_group_list }
obs_group_list = { (cfws? ~ ",")+ ~ cfws? }
addr_spec = { local_part ~ "@" ~ domain }
local_part = { dot_atom | quoted_string | obs_local_part }
obs_local_part = { word ~ (dot ~ word)* }
dot = @{ "." }
domain = { dot_atom | domain_literal | obs_domain }
obs_domain = { atom ~ ( dot ~ atom)* }
domain_literal = { cfws? ~ "[" ~ (fws? ~ dtext)* ~ fws? ~ "]" ~ cfws? }
dtext = { '\u{21}'..'\u{5a}' | '\u{5e}'..'\u{7e}' | obs_dtext }
obs_dtext = { obs_no_ws_ctl | quoted_pair }
alpha = { 'a'..'z' | 'A'..'Z' }
digit = { '0'..'9' }
atext_char = { "!" | "#" | "$" | "%" | "&" | "'" | "*" | "+" | "-" | "/" | "=" |
"?" | "^" | "_" | "`" | "{" | "|" | "}" | "~" | alpha | digit }
atext = @{ atext_char+ }
atom = { cfws? ~ atext ~ cfws? }
dot_atom_text = @{ atext ~ ("." ~ atext)* }
dot_atom = { cfws? ~ dot_atom_text ~ cfws? }
specials = { "(" | ")" | "<" | ">" | "[" | "]" | ":" | ";" | "@" | "\\" | "," | "." | "\"" }
qtext = { "\u{21}" | '\u{23}'..'\u{5b}' | '\u{5d}'..'\u{7e}' | obs_qtext }
obs_qtext = { obs_no_ws_ctl }
qcontent = { qtext | quoted_pair }
quoted_string = { cfws? ~ "\"" ~ (fws? ~ qcontent)* ~ fws? ~ "\"" ~ cfws? }
quoted_pair = { ( "\\" ~ (vchar | wsp)) | obs_qp }
obs_qp = { "\\" ~ ( "\u{00}" | obs_no_ws_ctl | "\r" | "\n") }
word = { atom | quoted_string }
phrase = { encoded_word+ | word+ | obs_phrase }
obs_phrase = { word ~ (word | "." | cfws)* }
unstructured = {
// This rule always matches; since obs_unstruct is a superset,
// and this is an unstructured field anyway, we just use the obsolete
// syntax to match the header content
// ((fws? ~ vchar)* ~ wsp*) |
obs_unstruct
}
obs_unstruct = { (( "\r"* ~ "\n"* ~ (obs_utext ~ "\r"* ~ "\n"*)+) | fws)+ }
obs_utext = { "\u{00}" | obs_no_ws_ctl | vchar }
obs_no_ws_ctl = { '\u{01}'..'\u{08}' | '\u{0b}'..'\u{0c}' | '\u{0d}'..'\u{1f}' | "\u{7f}" }
obs_ctext = { obs_no_ws_ctl }
obs_phrase_list = { (phrase | cfws)? ~ ("," ~ (phrase | cfws)?)* }
obs_route = { obs_domain_list ~ ":" }
obs_domain_list = { (cfws | ",")* ~ "@" ~ domain ~ ("," ~ cfws? ~ ("@" ~ domain)?)* }
obs_fws = { wsp+ ~ ("\r"? ~ "\n" ~ wsp+)* }
fws = { ((wsp* ~ "\r"? ~ "\n")* ~ wsp+) | obs_fws }
ctext = { '\u{21}'..'\u{27}' | '\u{2a}'..'\u{5b}' | '\u{5d}'..'\u{7e}' | obs_ctext }
ccontent = { ctext | quoted_pair | comment | encoded_word }
comment = { "(" ~ (fws? ~ ccontent)* ~ fws? ~ ")" }
cfws = { ( (fws? ~ comment)+ ~ fws?) | fws }
wsp = { " " | "\t" }
vchar = { '\u{21}'..'\u{7e}' }
encoded_word = { "=?" ~ charset ~ "?" ~ encoding ~ "?" ~ encoded_text ~ "?=" }
encoded_text = @{ (!( " " | "?") ~ vchar)+ }
charset = @{ token+ }
encoding = @{ token+ }
ctl = { '\u{00}'..'\u{1f}' | "\u{7f}" }
char = { '\u{01}'..'\u{7f}' }
token = { !(" " | especials | ctl) ~ char }
especials = { "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" |
"<" | ">" | "/" | "[" | "]" | "?" | "." | "=" }
parse_mailbox_list = _{ SOI ~ mailbox_list ~ EOI }
parse_mailbox = _{ SOI ~ mailbox ~ EOI }
+529
View File
@@ -0,0 +1,529 @@
use crate::{MailParsingError, Result};
use charset::Charset;
use pest::iterators::{Pair, Pairs};
use pest::Parser as _;
use pest_derive::*;
#[derive(Parser)]
#[grammar = "rfc5322.pest"]
pub struct Parser;
impl Parser {
pub fn parse_mailbox_list_header(text: &str) -> Result<MailboxList> {
let mut pairs = Self::parse(Rule::parse_mailbox_list, text)
.map_err(|err| MailParsingError::HeaderParse(format!("{err:#}")))?
.next()
.unwrap()
.into_inner();
let mut result: Vec<Mailbox> = vec![];
while let Some(pair) = pairs.next() {
result.push(Self::parse_mailbox(pair.into_inner())?);
}
Ok(MailboxList(result))
}
pub fn parse_mailbox_header(text: &str) -> Result<Mailbox> {
let pairs = Self::parse(Rule::parse_mailbox, text)
.map_err(|err| MailParsingError::HeaderParse(format!("{err:#}")))?
.next()
.unwrap()
.into_inner();
Self::parse_mailbox(pairs)
}
fn parse_mailbox(pairs: Pairs<Rule>) -> Result<Mailbox> {
for p in pairs {
match p.as_rule() {
Rule::name_addr => return Self::parse_name_addr(p),
Rule::addr_spec => {
return Ok(Mailbox {
name: None,
address: Self::parse_addr_spec(p)?,
})
}
rule => {
return Err(MailParsingError::HeaderParse(format!(
"Expected name_addr or addr_spec, but had {rule:?} {p:?}"
)))
}
};
}
Err(MailParsingError::HeaderParse(
"unreachable end of loop in parse_mailbox".to_string(),
))
}
fn parse_dot_atom(pair: Pair<Rule>) -> Result<String> {
for p in pair.into_inner() {
match p.as_rule() {
Rule::cfws => {}
Rule::dot_atom_text => return Ok(p.as_str().to_string()),
rule => {
return Err(MailParsingError::HeaderParse(format!(
"invalid {rule:?} {p:#?} in parse_dot_atom"
)))
}
}
}
Err(MailParsingError::HeaderParse(format!(
"Unreachable end of loop in parse_dot_atom"
)))
}
fn parse_local_part(pair: Pair<Rule>) -> Result<String> {
for p in pair.into_inner() {
match p.as_rule() {
Rule::dot_atom => return Self::parse_dot_atom(p),
Rule::quoted_string => return Self::parse_quoted_string(p),
Rule::obs_local_part => return Self::parse_obs_local_part(p),
rule => {
return Err(MailParsingError::HeaderParse(format!(
"Invalid {rule:?} {p:#?} in parse_local_part"
)))
}
}
}
Err(MailParsingError::HeaderParse(format!(
"Unreachable end of loop in parse_local_part"
)))
}
fn parse_obs_local_part(pair: Pair<Rule>) -> Result<String> {
let mut result = String::new();
for p in pair.into_inner() {
match p.as_rule() {
Rule::word => {
result += &Self::parse_word(p)?;
}
Rule::dot => {
result += p.as_str();
}
rule => {
return Err(MailParsingError::HeaderParse(format!(
"Invalid {rule:?} {p:#?} in parse_obs_local_part"
)))
}
}
}
Ok(result)
}
fn parse_obs_domain(pair: Pair<Rule>) -> Result<String> {
let mut result = String::new();
for p in pair.into_inner() {
match p.as_rule() {
Rule::atom => {
result += &Self::parse_atom(p)?;
}
Rule::dot => {
result += p.as_str();
}
rule => {
return Err(MailParsingError::HeaderParse(format!(
"Invalid {rule:?} {p:#?} in parse_obs_domain"
)))
}
}
}
Ok(result)
}
fn parse_domain(pair: Pair<Rule>) -> Result<String> {
for p in pair.into_inner() {
match p.as_rule() {
Rule::dot_atom => return Self::parse_dot_atom(p),
Rule::domain_literal => return Self::parse_domain_literal(p),
Rule::obs_domain => return Self::parse_obs_domain(p),
rule => {
return Err(MailParsingError::HeaderParse(format!(
"Unhandled {rule:?} {p:#?} in parse_domain"
)))
}
}
}
Err(MailParsingError::HeaderParse(format!(
"Unreachable end of loop in parse_domain"
)))
}
fn parse_domain_literal(pair: Pair<Rule>) -> Result<String> {
let mut result = "[".to_string();
for p in pair.into_inner() {
match p.as_rule() {
Rule::fws | Rule::cfws => {}
Rule::dtext => {
let dtext = p.as_str();
if dtext.len() == 2 {
// Must be quoted_pair
result.push_str(&dtext[1..]);
} else {
result.push_str(dtext);
}
}
rule => {
return Err(MailParsingError::HeaderParse(format!(
"Unhandled {rule:?} {p:#?} in parse_domain_literal"
)))
}
}
}
result.push(']');
Ok(result)
}
fn parse_addr_spec(pair: Pair<Rule>) -> Result<String> {
let mut result = String::new();
for p in pair.into_inner() {
match p.as_rule() {
Rule::local_part => {
result = Self::parse_local_part(p)?;
result.push('@');
}
Rule::domain => {
result += &Self::parse_domain(p)?;
}
rule => {
return Err(MailParsingError::HeaderParse(format!(
"Invalid {rule:?} {p:#?} in parse_addr_spec"
)))
}
}
}
Ok(result)
}
fn parse_angle_addr(pair: Pair<Rule>) -> Result<String> {
for p in pair.into_inner() {
match p.as_rule() {
Rule::addr_spec => return Self::parse_addr_spec(p),
Rule::cfws => {}
rule => {
return Err(MailParsingError::HeaderParse(format!(
"Unhandled {rule:?} {p:#?} in parse_angle_addr"
)))
}
}
}
Err(MailParsingError::HeaderParse(
"unreachable end of loop in parse_angle_addr".to_string(),
))
}
fn parse_word(pair: Pair<Rule>) -> Result<String> {
for p in pair.into_inner() {
match p.as_rule() {
Rule::atom => return Self::parse_atom(p),
Rule::quoted_string => return Self::parse_quoted_string(p),
rule => {
return Err(MailParsingError::HeaderParse(format!(
"Invalid {rule:?} {p:#?} in parse_word"
)))
}
}
}
Err(MailParsingError::HeaderParse(
"unreachable end of loop in parse_word".to_string(),
))
}
fn parse_quoted_string(pair: Pair<Rule>) -> Result<String> {
let mut result = String::new();
let mut fws = false;
for p in pair.into_inner() {
match p.as_rule() {
Rule::fws | Rule::cfws => {
if !result.is_empty() && !fws {
result.push(' ');
}
fws = true;
}
Rule::qcontent => {
fws = false;
let content = p.into_inner().next().unwrap();
match content.as_rule() {
Rule::qtext => result.push_str(content.as_str()),
Rule::quoted_pair => result.push_str(&content.as_str()[1..]),
rule => {
return Err(MailParsingError::HeaderParse(format!(
"Invalid {rule:?} {content:#?} in parse_quoted_string qcontent"
)))
}
}
}
rule => {
return Err(MailParsingError::HeaderParse(format!(
"Invalid {rule:?} {p:#?} in parse_quoted_string"
)))
}
}
}
if fws {
result.pop();
}
Ok(result)
}
fn parse_encoded_word(pair: Pair<Rule>) -> Result<String> {
let mut charset = String::new();
let mut encoding = String::new();
let mut text = String::new();
for p in pair.into_inner() {
match p.as_rule() {
Rule::charset => {
charset = p.as_str().to_string();
}
Rule::encoding => {
encoding = p.as_str().to_string();
}
Rule::encoded_text => {
text = p.as_str().to_string();
}
rule => {
return Err(MailParsingError::HeaderParse(format!(
"Invalid {rule:?} {p:#?} in parse_encoded_word"
)))
}
}
}
let bytes = match encoding.as_str() {
"B" | "b" => data_encoding::BASE64_MIME
.decode(text.as_bytes())
.map_err(|err| {
MailParsingError::HeaderParse(format!(
"Invalid base64 encoding: {err:#} {text:?}"
))
})?,
"Q" | "q" => quoted_printable::decode(
text.replace("_", " "),
quoted_printable::ParseMode::Robust,
)
.map_err(|err| {
MailParsingError::HeaderParse(format!(
"Invalid quoted printable encoding: {err:#} {text:?}"
))
})?,
_ => {
return Err(MailParsingError::HeaderParse(format!(
"Invalid encoding {encoding} in parse_encoded_word"
)))
}
};
let charset = Charset::for_label_no_replacement(charset.as_bytes()).ok_or_else(|| {
MailParsingError::HeaderParse(format!("unsupported charset: {charset:?}"))
})?;
let (decoded, _malformed) = charset.decode_without_bom_handling(&bytes);
Ok(decoded.into())
}
fn parse_atom(pair: Pair<Rule>) -> Result<String> {
for p in pair.into_inner() {
match p.as_rule() {
Rule::atext => return Ok(p.as_str().to_string()),
rule => {
return Err(MailParsingError::HeaderParse(format!(
"Unhandled {rule:?} {p:#?} in parse_atom"
)))
}
}
}
Err(MailParsingError::HeaderParse(
"unreachable end of loop in parse_atom".to_string(),
))
}
fn parse_phrase(pair: Pair<Rule>) -> Result<Vec<String>> {
let mut words = vec![];
for p in pair.into_inner() {
match p.as_rule() {
Rule::word => words.push(Self::parse_word(p)?),
Rule::encoded_word => words.push(Self::parse_encoded_word(p)?),
rule => {
return Err(MailParsingError::HeaderParse(format!(
"Unhandled {rule:?} in parse_phrase"
)))
}
}
}
Ok(words)
}
fn parse_display_name(pair: Pair<Rule>) -> Result<String> {
let words = Self::parse_phrase(pair.into_inner().next().unwrap())?;
Ok(words.join(" "))
}
fn parse_name_addr(name_addr: Pair<Rule>) -> Result<Mailbox> {
let name_addr = name_addr.into_inner();
let mut name = None;
for p in name_addr {
match p.as_rule() {
Rule::display_name => {
name.replace(Self::parse_display_name(p)?);
//name.replace(Self::text_ignoring_cfws(p, true)?);
}
Rule::angle_addr => {
let address = Self::parse_angle_addr(p)?;
return Ok(Mailbox { name, address });
}
rule => {
return Err(MailParsingError::HeaderParse(format!(
"parse_name_addr: invalid {rule:?} for {p:?}"
)))
}
};
}
Err(MailParsingError::HeaderParse(format!(
"Unreachable end of loop in parse_name_addr"
)))
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MailboxList(Vec<Mailbox>);
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Mailbox {
pub name: Option<String>,
pub address: String,
}
#[cfg(test)]
mod test {
use crate::MimePart;
#[test]
fn mailbox_list_singular() {
let message = concat!(
"From: Someone (hello) <someone@example.com>, other@example.com,\n",
" \"John \\\"Smith\\\"\" (comment) \"More Quotes\" (more comment) <someone(another comment)@crazy.example.com(woot)>\n",
"\n",
"I am the body"
);
let msg = MimePart::parse(message).unwrap();
let list = match msg.headers().from() {
Err(err) => panic!("Doh.\n{err:#}"),
Ok(list) => list,
};
k9::snapshot!(
list,
r#"
Some(
MailboxList(
[
Mailbox {
name: Some(
"Someone",
),
address: "someone@example.com",
},
Mailbox {
name: None,
address: "other@example.com",
},
Mailbox {
name: Some(
"John "Smith" More Quotes",
),
address: "someone@crazy.example.com",
},
],
),
)
"#
);
}
#[test]
fn sender() {
let message = "Sender: someone@[127.0.0.1]\n\n\n";
let msg = MimePart::parse(message).unwrap();
let list = match msg.headers().sender() {
Err(err) => panic!("Doh.\n{err:#}"),
Ok(list) => list,
};
k9::snapshot!(
list,
r#"
Some(
Mailbox {
name: None,
address: "someone@[127.0.0.1]",
},
)
"#
);
}
#[test]
fn domain_literal() {
let message = "From: someone@[127.0.0.1]\n\n\n";
let msg = MimePart::parse(message).unwrap();
let list = match msg.headers().from() {
Err(err) => panic!("Doh.\n{err:#}"),
Ok(list) => list,
};
k9::snapshot!(
list,
r#"
Some(
MailboxList(
[
Mailbox {
name: None,
address: "someone@[127.0.0.1]",
},
],
),
)
"#
);
}
#[test]
fn rfc2047() {
let message = "From: =?US-ASCII?Q?Keith_Moore?= <moore@cs.utk.edu>\n\n\n";
let msg = MimePart::parse(message).unwrap();
let list = match msg.headers().from() {
Err(err) => panic!("Doh.\n{err:#}"),
Ok(list) => list,
};
k9::snapshot!(
list,
r#"
Some(
MailboxList(
[
Mailbox {
name: Some(
"Keith Moore",
),
address: "moore@cs.utk.edu",
},
],
),
)
"#
);
}
}