fix: keep an mbox message's arrival time and leave its state headers out on import

lib/mail-import/__tests__/engine.test.ts

@@ -106,6 +106,21 @@ describe('runImport', () => {
     expect(fake.imported[0].text.startsWith('Subject: m0\n')).toBe(true);
   });
 
+  it('takes the arrival time from the envelope line and leaves the state headers out of the stored message', async () => {
+    const fake = fakeClient();
+    const raw = 'From a@example.org Sat Jan  2 03:04:05 2021\nStatus: RO\nX-Mozilla-Status: 0001\nSubject: s\nDate: Tue, 5 May 2020 10:00:00 +0000\n\nbody\n';
+    await runImport(options(fake.client, [file(raw)]));
+    expect(fake.imported[0].receivedAt).toBe('2021-01-02T03:04:05Z');
+    expect(fake.imported[0].text).toBe('Subject: s\nDate: Tue, 5 May 2020 10:00:00 +0000\n\nbody\n');
+    expect(fake.imported[0].keywords).toEqual({ $seen: true });
+  });
+
+  it('falls back to the Date header when the envelope line has no usable date', async () => {
+    const fake = fakeClient();
+    await runImport(options(fake.client, [file('From MAILER-DAEMON Thu Jan  1 00:00:00 1970\nSubject: s\nDate: Tue, 5 May 2020 10:00:00 +0000\n\nb\n')]));
+    expect(fake.imported[0].receivedAt).toBe('2020-05-05T10:00:00Z');
+  });
+
   it('puts several messages in one Email/import and keeps uploads within the server limit', async () => {
     const fake = fakeClient({ uploadConcurrency: 4 });
     await runImport(options(fake.client, [file(mbox(45))]));

lib/mail-import/__tests__/mbox-status.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect } from 'vitest';
-import { mboxState } from '@/lib/mail-import/mbox-status';
+import { mboxState, withoutStateHeaders } from '@/lib/mail-import/mbox-status';
 
 describe('mboxState', () => {
   it('imports a message without any state header as read, as before', () => {
@@ -40,3 +40,19 @@ describe('mboxState', () => {
     expect(mboxState('subject: a\r\nx-status:\r\n F\r\nSTATUS: O').keywords).toEqual({ $flagged: true });
   });
 });
+
+describe('withoutStateHeaders', () => {
+  const enc = new TextEncoder();
+  const text = (parts: Uint8Array[]) => parts.map((p) => new TextDecoder().decode(p)).join('');
+  it('drops the state headers, folded lines too, and keeps the rest byte for byte', () => {
+    const raw = 'Status: RO\r\nSubject: hi\r\nX-Keywords: a\r\n b\r\nX-Mozilla-Status: 0001\r\nFrom: x@example.org\r\n\r\nStatus: in the body\r\n';
+    // Split in odd places, as the mbox splitter hands them over.
+    const parts = [enc.encode(raw.slice(0, 7)), enc.encode(raw.slice(7, 40)), enc.encode(raw.slice(40))];
+    expect(text(withoutStateHeaders(parts))).toBe('Subject: hi\r\nFrom: x@example.org\r\n\r\nStatus: in the body\r\n');
+  });
+  it('leaves a message without them as it was', () => {
+    const parts = [enc.encode('Subject: hi\n\nbody\n')];
+    expect(withoutStateHeaders(parts)).toBe(parts);
+  });
+});
+

lib/mail-import/mbox-status.ts

@@ -86,3 +86,64 @@ export function mboxState(headerBlock: string): MboxState {
   addKeywords(keywords, fields.get("x-mozilla-keys"));
   return { keywords, expunged };
 }
+
+const STATE_HEADER = /^(status|x-status|x-keywords|x-mozilla-status|x-mozilla-status2|x-mozilla-keys)$/i;
+/** How far into a message the header block is looked for. */
+const HEADER_LIMIT = 256 * 1024;
+
+/**
+ * The message without the mbox state headers read above: their state is now
+ * in JMAP keywords, and a stale copy in the stored message would only
+ * contradict it (and differ from what was exported). Everything else,
+ * the body included, stays byte for byte.
+ */
+export function withoutStateHeaders(parts: Uint8Array[]): Uint8Array[] {
+  // Find the end of the header block (the empty line) within the first parts.
+  let end = -1;
+  let offset = 0;
+  let lineHasText = false;
+  outer: for (const p of parts) {
+    for (let i = 0; i < p.length; i++) {
+      const b = p[i];
+      if (b === 0x0a) {
+        if (!lineHasText) { end = offset + i + 1; break outer; }
+        lineHasText = false;
+      } else if (b !== 0x0d) {
+        lineHasText = true;
+      }
+    }
+    offset += p.length;
+    if (offset > HEADER_LIMIT) break;
+  }
+  const headerEnd = end === -1 ? Math.min(offset, parts.reduce((n, p) => n + p.length, 0)) : end;
+  // The header bytes, copied, and the rest as it is.
+  const head = new Uint8Array(headerEnd);
+  const rest: Uint8Array[] = [];
+  let at = 0;
+  for (const p of parts) {
+    if (at >= headerEnd) { rest.push(p); continue; }
+    const take = Math.min(p.length, headerEnd - at);
+    head.set(p.subarray(0, take), at);
+    if (take < p.length) rest.push(p.subarray(take));
+    at += take;
+  }
+  const kept: Uint8Array[] = [];
+  let skipping = false;
+  let found = false;
+  let start = 0;
+  for (let i = 0; i <= head.length; i++) {
+    if (i < head.length && head[i] !== 0x0a) continue;
+    const line = head.subarray(start, Math.min(head.length, i + 1));
+    start = i + 1;
+    if (line.length === 0) continue;
+    const folded = line[0] === 0x20 || line[0] === 0x09;
+    if (!folded) {
+      const colon = line.indexOf(0x3a);
+      const name = colon > 0 ? String.fromCharCode(...line.subarray(0, colon)).trim() : "";
+      skipping = STATE_HEADER.test(name);
+      if (skipping) found = true;
+    }
+    if (!skipping) kept.push(line);
+  }
+  return found ? [...kept, ...rest] : parts;
+}

lib/mail-import/sources.ts

@@ -1,6 +1,6 @@
 import { dateFromHeaderBlock } from "@/lib/eml-import";
 import { MboxSplitter, envelopeDate, readHeaderBlock, type MboxMessage } from "./mbox";
-import { mboxState } from "./mbox-status";
+import { mboxState, withoutStateHeaders } from "./mbox-status";
 import { listZipEntries, readZipEntry, streamZipEntry } from "./zip-reader";
 
 /**
@@ -143,6 +143,13 @@ export async function* emlSource(file: File, options: SourceOptions): ItemSource
   options.onProgress?.(1);
 }
 
+/** A date between 1980 and a day from now, else undefined. */
+function plausible(date: string | undefined, now = Date.now()): string | undefined {
+  if (!date) return undefined;
+  const t = Date.parse(date);
+  return t >= Date.UTC(1980, 0, 1) && t - now <= 24 * 60 * 60 * 1000 ? date : undefined;
+}
+
 function mboxItem(message: MboxMessage, index: number, folder: string[]): ImportItem {
   const label = message.tooLarge ? message.fromLine : "";
   if (message.tooLarge) return { kind: "skipped", index, reason: "too_large", label };
@@ -150,14 +157,17 @@ function mboxItem(message: MboxMessage, index: number, folder: string[]): Import
   const state = mboxState(headers);
   const subject = subjectOf(headers);
   if (state.expunged) return { kind: "skipped", index, reason: "deleted", label: subject };
+  const parts = withoutStateHeaders(message.parts);
   return {
     kind: "message",
     index,
     folder,
-    data: new Blob(message.parts as BlobPart[], { type: MIME }),
-    size: message.size,
+    data: new Blob(parts as BlobPart[], { type: MIME }),
+    size: parts.reduce((n, p) => n + p.length, 0),
     keywords: state.keywords,
-    receivedAt: dateFromHeaderBlock(headers) ?? envelopeDate(message.fromLine),
+    // The envelope line holds when the message arrived (Bulwark's export writes
+    // receivedAt there); the Date header, when it was written, comes second.
+    receivedAt: plausible(envelopeDate(message.fromLine)) ?? dateFromHeaderBlock(headers),
     label: subject,
   };
 }