fix: split mbox files only on envelope lines that end in a date and handle very long header lines

lib/mail-export/__tests__/mbox-writer.test.ts

@@ -1,7 +1,7 @@
 import { describe, it, expect } from 'vitest';
 import { mboxEntry, envelopeLine } from '@/lib/mail-export/mbox-writer';
 import { MboxSplitter } from '@/lib/mail-import/mbox';
-import { mboxState } from '@/lib/mail-import/mbox-status';
+import { mboxState, withoutStateHeaders } from '@/lib/mail-import/mbox-status';
 import { readHeaderBlock } from '@/lib/mail-import/mbox';
 
 const enc = new TextEncoder();
@@ -21,6 +21,14 @@ function reread(mbox: Uint8Array) {
 describe('mboxEntry (mboxrd)', () => {
   const raw = 'Subject: hi\r\nX-Mozilla-Status: 0001\r\nStatus: RO\r\nX-Keywords:\r\n old\r\nFrom: a@example.org\r\n\r\nFrom the top\r\n>From quoted\r\nend\r\n';
 
+  it('copes with a header line of hundreds of kilobytes, on export and on import', () => {
+    const long = `X-${'a'.repeat(300_000)}: v\r\nStatus: RO\r\nSubject: s\r\n\r\nbody\r\n`;
+    const out = dec.decode(join(mboxEntry(enc.encode(long), { $seen: true }, '2020-01-01T00:00:00Z')));
+    expect(out).toContain('aaa: v\r\nSubject: s\r\n');
+    const kept = dec.decode(join(withoutStateHeaders([enc.encode(`${'X'.repeat(300_000)}: v\r\nStatus: RO\r\n\r\nb\r\n`)])));
+    expect(kept.endsWith(': v\r\n\r\nb\r\n')).toBe(true);
+  });
+
   it('writes an envelope line in asctime UTC', () => {
     expect(envelopeLine('2020-05-05T10:11:12Z', '\n')).toBe('From bulwark-export Tue May  5 10:11:12 2020\n');
   });

lib/mail-export/mbox-writer.ts

@@ -54,6 +54,8 @@ function startsWithFrom(data: Uint8Array, at: number, end: number): boolean {
 }
 
 const encoder = new TextEncoder();
+// Header names are ASCII; decoding (not spreading the bytes into fromCharCode) copes with a 100 KB line.
+const latin1 = new TextDecoder("latin1");
 const GT_BYTE = new Uint8Array([GT]);
 
 /**
@@ -87,7 +89,7 @@ export function mboxEntry(raw: Uint8Array, keywords: Record<string, boolean> | u
       } else {
         const colon = raw.indexOf(0x3a, lineStart);
         const name = colon > lineStart && colon < lineEnd
-          ? String.fromCharCode(...raw.subarray(lineStart, colon)).trim()
+          ? latin1.decode(raw.subarray(lineStart, colon)).trim()
           : "";
         skipping = STATE_HEADERS.test(name);
         if (skipping) { flush(lineStart); runStart = lineEnd; }
@@ -103,6 +105,9 @@ export function mboxEntry(raw: Uint8Array, keywords: Record<string, boolean> | u
     lineStart = lineEnd;
   }
   flush(raw.length);
+  // The empty line that separates messages. A message without a final line
+  // ending gets one first, so its last line isn't joined to the next envelope
+  // line; the import then sees one line ending more than the server stored.
   const endsWithLf = raw.length > 0 && raw[raw.length - 1] === LF;
   parts.push(encoder.encode(endsWithLf ? eol : eol + eol));
   return parts;

lib/mail-import/__tests__/mbox.test.ts

@@ -58,6 +58,16 @@ describe('MboxSplitter', () => {
     expect(texts(split(mbox))).toHaveLength(1);
   });
 
+  it('does not split on a body line with a time but no date after it', () => {
+    const mbox = 'From a Tue May  5 10:11:12 2020\nSubject: x\n\nhello\n\nFrom now on we meet at 10:30 in room 2020b\n';
+    expect(texts(split(mbox))).toHaveLength(1);
+  });
+
+  it('splits on envelope lines with a zone or "remote from"', () => {
+    const mbox = 'From a Sat Apr 18 08:29:01 +0000 2020\nSubject: 1\n\nb\n\nFrom b Sat Apr 18 08:29:01 2020 remote from host\nSubject: 2\n\nb\n';
+    expect(texts(split(mbox))).toHaveLength(2);
+  });
+
   it('unquotes mboxrd ">From " lines by one ">"', () => {
     const mbox = 'From a Tue May  5 10:11:12 2020\nSubject: q\n\n>From the start\n>>From deeper\n> not a from\n>Fromage\n';
     expect(texts(split(mbox, 4))).toEqual(['Subject: q\n\nFrom the start\n>From deeper\n> not a from\n>Fromage\n']);

lib/mail-import/mbox-status.ts

@@ -139,7 +139,7 @@ export function withoutStateHeaders(parts: Uint8Array[]): Uint8Array[] {
     const folded = line[0] === 0x20 || line[0] === 0x09;
     if (!folded) {
       const colon = line.indexOf(0x3a);
-      const name = colon > 0 ? String.fromCharCode(...line.subarray(0, colon)).trim() : "";
+      const name = colon > 0 ? new TextDecoder("latin1").decode(line.subarray(0, colon)).trim() : "";
       skipping = STATE_HEADER.test(name);
       if (skipping) found = true;
     }

lib/mail-import/mbox.ts

@@ -80,7 +80,9 @@ function stripEol(line: Uint8Array): Uint8Array {
  * empty line in a file whose writer didn't quote it.
  */
 export function isEnvelopeLine(text: string): boolean {
-  return /^From \S*.*\d{1,2}:\d{2}/.test(text);
+  // Sender, then a date that ends in the year: "... Tue May  5 10:11:12 2020",
+  // optionally with a zone before the year or "remote from host" after it.
+  return /^From \S+\s.*\b\d{1,2}:\d{2}(?::\d{2})?\s+(?:[+-]\d{4}\s+|[A-Z]{2,5}\s+)?\d{4}(?:\s+remote from \S+)?\s*$/.test(text);
 }
 
 const CONTENT_LENGTH = /^content-length:[ \t]*(\d+)[ \t]*\r?$/im;