diff --git a/lib/mail-export/__tests__/mbox-writer.test.ts b/lib/mail-export/__tests__/mbox-writer.test.ts index 0f9826e73..f8a799415 100644 --- a/lib/mail-export/__tests__/mbox-writer.test.ts +++ b/lib/mail-export/__tests__/mbox-writer.test.ts @@ -1,7 +1,7 @@ import { describe, it, expect } from 'vitest'; import { mboxEntry, envelopeLine } from '@/lib/mail-export/mbox-writer'; import { MboxSplitter } from '@/lib/mail-import/mbox'; -import { mboxState } from '@/lib/mail-import/mbox-status'; +import { mboxState, withoutStateHeaders } from '@/lib/mail-import/mbox-status'; import { readHeaderBlock } from '@/lib/mail-import/mbox'; const enc = new TextEncoder(); @@ -21,6 +21,14 @@ function reread(mbox: Uint8Array) { describe('mboxEntry (mboxrd)', () => { const raw = 'Subject: hi\r\nX-Mozilla-Status: 0001\r\nStatus: RO\r\nX-Keywords:\r\n old\r\nFrom: a@example.org\r\n\r\nFrom the top\r\n>From quoted\r\nend\r\n'; + it('copes with a header line of hundreds of kilobytes, on export and on import', () => { + const long = `X-${'a'.repeat(300_000)}: v\r\nStatus: RO\r\nSubject: s\r\n\r\nbody\r\n`; + const out = dec.decode(join(mboxEntry(enc.encode(long), { $seen: true }, '2020-01-01T00:00:00Z'))); + expect(out).toContain('aaa: v\r\nSubject: s\r\n'); + const kept = dec.decode(join(withoutStateHeaders([enc.encode(`${'X'.repeat(300_000)}: v\r\nStatus: RO\r\n\r\nb\r\n`)]))); + expect(kept.endsWith(': v\r\n\r\nb\r\n')).toBe(true); + }); + it('writes an envelope line in asctime UTC', () => { expect(envelopeLine('2020-05-05T10:11:12Z', '\n')).toBe('From bulwark-export Tue May 5 10:11:12 2020\n'); }); diff --git a/lib/mail-export/mbox-writer.ts b/lib/mail-export/mbox-writer.ts index d319d651f..3fde6f06f 100644 --- a/lib/mail-export/mbox-writer.ts +++ b/lib/mail-export/mbox-writer.ts @@ -54,6 +54,8 @@ function startsWithFrom(data: Uint8Array, at: number, end: number): boolean { } const encoder = new TextEncoder(); +// Header names are ASCII; decoding (not spreading the bytes into fromCharCode) copes with a 100 KB line. +const latin1 = new TextDecoder("latin1"); const GT_BYTE = new Uint8Array([GT]); /** @@ -87,7 +89,7 @@ export function mboxEntry(raw: Uint8Array, keywords: Record | u } else { const colon = raw.indexOf(0x3a, lineStart); const name = colon > lineStart && colon < lineEnd - ? String.fromCharCode(...raw.subarray(lineStart, colon)).trim() + ? latin1.decode(raw.subarray(lineStart, colon)).trim() : ""; skipping = STATE_HEADERS.test(name); if (skipping) { flush(lineStart); runStart = lineEnd; } @@ -103,6 +105,9 @@ export function mboxEntry(raw: Uint8Array, keywords: Record | u lineStart = lineEnd; } flush(raw.length); + // The empty line that separates messages. A message without a final line + // ending gets one first, so its last line isn't joined to the next envelope + // line; the import then sees one line ending more than the server stored. const endsWithLf = raw.length > 0 && raw[raw.length - 1] === LF; parts.push(encoder.encode(endsWithLf ? eol : eol + eol)); return parts; diff --git a/lib/mail-import/__tests__/mbox.test.ts b/lib/mail-import/__tests__/mbox.test.ts index db335b015..1163a25f7 100644 --- a/lib/mail-import/__tests__/mbox.test.ts +++ b/lib/mail-import/__tests__/mbox.test.ts @@ -58,6 +58,16 @@ describe('MboxSplitter', () => { expect(texts(split(mbox))).toHaveLength(1); }); + it('does not split on a body line with a time but no date after it', () => { + const mbox = 'From a Tue May 5 10:11:12 2020\nSubject: x\n\nhello\n\nFrom now on we meet at 10:30 in room 2020b\n'; + expect(texts(split(mbox))).toHaveLength(1); + }); + + it('splits on envelope lines with a zone or "remote from"', () => { + const mbox = 'From a Sat Apr 18 08:29:01 +0000 2020\nSubject: 1\n\nb\n\nFrom b Sat Apr 18 08:29:01 2020 remote from host\nSubject: 2\n\nb\n'; + expect(texts(split(mbox))).toHaveLength(2); + }); + it('unquotes mboxrd ">From " lines by one ">"', () => { const mbox = 'From a Tue May 5 10:11:12 2020\nSubject: q\n\n>From the start\n>>From deeper\n> not a from\n>Fromage\n'; expect(texts(split(mbox, 4))).toEqual(['Subject: q\n\nFrom the start\n>From deeper\n> not a from\n>Fromage\n']); diff --git a/lib/mail-import/mbox-status.ts b/lib/mail-import/mbox-status.ts index f1e43de70..74b7ed1df 100644 --- a/lib/mail-import/mbox-status.ts +++ b/lib/mail-import/mbox-status.ts @@ -139,7 +139,7 @@ export function withoutStateHeaders(parts: Uint8Array[]): Uint8Array[] { const folded = line[0] === 0x20 || line[0] === 0x09; if (!folded) { const colon = line.indexOf(0x3a); - const name = colon > 0 ? String.fromCharCode(...line.subarray(0, colon)).trim() : ""; + const name = colon > 0 ? new TextDecoder("latin1").decode(line.subarray(0, colon)).trim() : ""; skipping = STATE_HEADER.test(name); if (skipping) found = true; } diff --git a/lib/mail-import/mbox.ts b/lib/mail-import/mbox.ts index 31ff91e76..95aaf8818 100644 --- a/lib/mail-import/mbox.ts +++ b/lib/mail-import/mbox.ts @@ -80,7 +80,9 @@ function stripEol(line: Uint8Array): Uint8Array { * empty line in a file whose writer didn't quote it. */ export function isEnvelopeLine(text: string): boolean { - return /^From \S*.*\d{1,2}:\d{2}/.test(text); + // Sender, then a date that ends in the year: "... Tue May 5 10:11:12 2020", + // optionally with a zone before the year or "remote from host" after it. + return /^From \S+\s.*\b\d{1,2}:\d{2}(?::\d{2})?\s+(?:[+-]\d{4}\s+|[A-Z]{2,5}\s+)?\d{4}(?:\s+remote from \S+)?\s*$/.test(text); } const CONTENT_LENGTH = /^content-length:[ \t]*(\d+)[ \t]*\r?$/im;