dkim: enable signing legacy 8-bit content

This is really what all of the recent binary message commits
have been all about, allowing us to parse and sign legacy 8-bit japanese
content.

crates/dkim/src/arc.rs

@@ -539,7 +539,7 @@ mod test {
             .unwrap();
 
         for instance in 1..=5 {
-            let email = ParsedEmail::parse(email_content.as_str()).unwrap();
+            let email = ParsedEmail::parse(&*email_content).unwrap();
             let arc = ARC::verify(&email, &resolver).await;
             assert_eq!(
                 arc.chain_validation_status(),

crates/integration-tests/src/test/arc.rs

@@ -79,7 +79,7 @@ async fn arc() -> anyhow::Result<()> {
         yQIDAQAB",
     );
 
-    let email = ParsedEmail::parse(String::from_utf8(payload.to_vec()).unwrap()).unwrap();
+    let email = ParsedEmail::parse(payload).unwrap();
     let arc = ARC::verify(&email, &resolver).await;
     assert_eq!(arc.chain_validation_status(), ChainValidationStatus::Pass);
 

crates/message/src/dkim.rs

@@ -313,9 +313,7 @@ pub struct CFSigner {
 impl CFSigner {
     fn sign(&self, message: &[u8]) -> anyhow::Result<String> {
         let parse_timer = SIGNER_PARSE.start_timer();
-        let message_str =
-            std::str::from_utf8(message).context("DKIM signer: message is not ASCII or UTF-8")?;
-        let mail = kumo_dkim::ParsedEmail::parse(message_str)
+        let mail = kumo_dkim::ParsedEmail::parse(message)
             .context("failed to parse message to pass to dkim signer")?;
         parse_timer.stop_and_record();
 

crates/message/test_legacy_jp_encodings.lua

@@ -0,0 +1,54 @@
+local kumo = require 'kumo'
+local utils = require 'policy-extras.policy_utils'
+
+local function new_msg(content)
+  return kumo.make_message('sender@example.com', 'recip@example.com', content)
+end
+
+local JP_UTF8 =
+  '日本語のテストメールです。こんにちは、ありがとうございます。'
+local SHIFT_JIS = kumo.encode.charset_encode('shift-jis', JP_UTF8)
+utils.assert_eq(kumo.encode.charset_decode('shift-jis', SHIFT_JIS), JP_UTF8)
+
+local BODY = 'From: someone@example.com\r\nSubject: Testing\r\n\r\n'
+  .. SHIFT_JIS
+local msg = new_msg(BODY)
+
+local mime_before = msg:parse_mime()
+-- We might hope that we get the SHIFT_JIS bytes back out for the body,
+-- but we'll try to interpret it as UTF-8 for internationalize email's sake,
+-- and this input just happens to look like plausible UTF-8, even though
+-- it is complete nonsense
+utils.assert_ne(mime_before.body, SHIFT_JIS)
+
+-- Interesting to note that the charset-normalizer based utf-8 decoder
+-- won't allow converting these shift-jis bytes to utf-8, but Rust's
+-- own built in str::from_utf8 will.
+-- utils.assert_eq(mime_before.body, kumo.encode.charset_decode('utf-8', SHIFT_JIS))
+
+-- Can we dkim-sign this binary content?
+local signer = kumo.dkim.rsa_sha256_signer {
+  domain = 'example.com',
+  selector = 'default',
+  headers = { 'From', 'Subject' },
+  key = 'example-private-dkim-key.pem',
+}
+msg:dkim_sign(signer)
+-- kumo.log_info(msg:get_data())
+
+-- Use encoding detection to fix the encoding up. Note that this
+-- definitely breaks the dkim signature, but we don't care in the
+-- context of this test; the above is testing whether can sign
+-- the wonky encoding, the below is testing whether we can fix
+-- it.  In practice, a given node will be doing either one or
+-- the other.
+msg:check_fix_conformance('', 'NEEDS_TRANSFER_ENCODING', {
+  detect_encoding = true,
+  include_encodings = {
+    'shift-jis',
+  },
+  exclude_encodings = {},
+})
+
+local mime = msg:parse_mime()
+utils.assert_eq(mime.body, JP_UTF8)