From c43d776b40421b5ddaff4acd2f12b8033ecefeab Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 05:26:55 +0100 Subject: [PATCH] fix: keep an mbox message's arrival time and leave its state headers out on import --- lib/mail-import/__tests__/engine.test.ts | 15 +++++ lib/mail-import/__tests__/mbox-status.test.ts | 18 +++++- lib/mail-import/mbox-status.ts | 61 +++++++++++++++++++ lib/mail-import/sources.ts | 18 ++++-- 4 files changed, 107 insertions(+), 5 deletions(-) diff --git a/lib/mail-import/__tests__/engine.test.ts b/lib/mail-import/__tests__/engine.test.ts index d21b4a4cf..302ea6738 100644 --- a/lib/mail-import/__tests__/engine.test.ts +++ b/lib/mail-import/__tests__/engine.test.ts @@ -106,6 +106,21 @@ describe('runImport', () => { expect(fake.imported[0].text.startsWith('Subject: m0\n')).toBe(true); }); + it('takes the arrival time from the envelope line and leaves the state headers out of the stored message', async () => { + const fake = fakeClient(); + const raw = 'From a@example.org Sat Jan 2 03:04:05 2021\nStatus: RO\nX-Mozilla-Status: 0001\nSubject: s\nDate: Tue, 5 May 2020 10:00:00 +0000\n\nbody\n'; + await runImport(options(fake.client, [file(raw)])); + expect(fake.imported[0].receivedAt).toBe('2021-01-02T03:04:05Z'); + expect(fake.imported[0].text).toBe('Subject: s\nDate: Tue, 5 May 2020 10:00:00 +0000\n\nbody\n'); + expect(fake.imported[0].keywords).toEqual({ $seen: true }); + }); + + it('falls back to the Date header when the envelope line has no usable date', async () => { + const fake = fakeClient(); + await runImport(options(fake.client, [file('From MAILER-DAEMON Thu Jan 1 00:00:00 1970\nSubject: s\nDate: Tue, 5 May 2020 10:00:00 +0000\n\nb\n')])); + expect(fake.imported[0].receivedAt).toBe('2020-05-05T10:00:00Z'); + }); + it('puts several messages in one Email/import and keeps uploads within the server limit', async () => { const fake = fakeClient({ uploadConcurrency: 4 }); await runImport(options(fake.client, [file(mbox(45))])); diff --git a/lib/mail-import/__tests__/mbox-status.test.ts b/lib/mail-import/__tests__/mbox-status.test.ts index 249eb8ff4..163e6250b 100644 --- a/lib/mail-import/__tests__/mbox-status.test.ts +++ b/lib/mail-import/__tests__/mbox-status.test.ts @@ -1,5 +1,5 @@ import { describe, it, expect } from 'vitest'; -import { mboxState } from '@/lib/mail-import/mbox-status'; +import { mboxState, withoutStateHeaders } from '@/lib/mail-import/mbox-status'; describe('mboxState', () => { it('imports a message without any state header as read, as before', () => { @@ -40,3 +40,19 @@ describe('mboxState', () => { expect(mboxState('subject: a\r\nx-status:\r\n F\r\nSTATUS: O').keywords).toEqual({ $flagged: true }); }); }); + +describe('withoutStateHeaders', () => { + const enc = new TextEncoder(); + const text = (parts: Uint8Array[]) => parts.map((p) => new TextDecoder().decode(p)).join(''); + it('drops the state headers, folded lines too, and keeps the rest byte for byte', () => { + const raw = 'Status: RO\r\nSubject: hi\r\nX-Keywords: a\r\n b\r\nX-Mozilla-Status: 0001\r\nFrom: x@example.org\r\n\r\nStatus: in the body\r\n'; + // Split in odd places, as the mbox splitter hands them over. + const parts = [enc.encode(raw.slice(0, 7)), enc.encode(raw.slice(7, 40)), enc.encode(raw.slice(40))]; + expect(text(withoutStateHeaders(parts))).toBe('Subject: hi\r\nFrom: x@example.org\r\n\r\nStatus: in the body\r\n'); + }); + it('leaves a message without them as it was', () => { + const parts = [enc.encode('Subject: hi\n\nbody\n')]; + expect(withoutStateHeaders(parts)).toBe(parts); + }); +}); + diff --git a/lib/mail-import/mbox-status.ts b/lib/mail-import/mbox-status.ts index 16cd892f1..f1e43de70 100644 --- a/lib/mail-import/mbox-status.ts +++ b/lib/mail-import/mbox-status.ts @@ -86,3 +86,64 @@ export function mboxState(headerBlock: string): MboxState { addKeywords(keywords, fields.get("x-mozilla-keys")); return { keywords, expunged }; } + +const STATE_HEADER = /^(status|x-status|x-keywords|x-mozilla-status|x-mozilla-status2|x-mozilla-keys)$/i; +/** How far into a message the header block is looked for. */ +const HEADER_LIMIT = 256 * 1024; + +/** + * The message without the mbox state headers read above: their state is now + * in JMAP keywords, and a stale copy in the stored message would only + * contradict it (and differ from what was exported). Everything else, + * the body included, stays byte for byte. + */ +export function withoutStateHeaders(parts: Uint8Array[]): Uint8Array[] { + // Find the end of the header block (the empty line) within the first parts. + let end = -1; + let offset = 0; + let lineHasText = false; + outer: for (const p of parts) { + for (let i = 0; i < p.length; i++) { + const b = p[i]; + if (b === 0x0a) { + if (!lineHasText) { end = offset + i + 1; break outer; } + lineHasText = false; + } else if (b !== 0x0d) { + lineHasText = true; + } + } + offset += p.length; + if (offset > HEADER_LIMIT) break; + } + const headerEnd = end === -1 ? Math.min(offset, parts.reduce((n, p) => n + p.length, 0)) : end; + // The header bytes, copied, and the rest as it is. + const head = new Uint8Array(headerEnd); + const rest: Uint8Array[] = []; + let at = 0; + for (const p of parts) { + if (at >= headerEnd) { rest.push(p); continue; } + const take = Math.min(p.length, headerEnd - at); + head.set(p.subarray(0, take), at); + if (take < p.length) rest.push(p.subarray(take)); + at += take; + } + const kept: Uint8Array[] = []; + let skipping = false; + let found = false; + let start = 0; + for (let i = 0; i <= head.length; i++) { + if (i < head.length && head[i] !== 0x0a) continue; + const line = head.subarray(start, Math.min(head.length, i + 1)); + start = i + 1; + if (line.length === 0) continue; + const folded = line[0] === 0x20 || line[0] === 0x09; + if (!folded) { + const colon = line.indexOf(0x3a); + const name = colon > 0 ? String.fromCharCode(...line.subarray(0, colon)).trim() : ""; + skipping = STATE_HEADER.test(name); + if (skipping) found = true; + } + if (!skipping) kept.push(line); + } + return found ? [...kept, ...rest] : parts; +} diff --git a/lib/mail-import/sources.ts b/lib/mail-import/sources.ts index 80c787c06..48c9c6ac6 100644 --- a/lib/mail-import/sources.ts +++ b/lib/mail-import/sources.ts @@ -1,6 +1,6 @@ import { dateFromHeaderBlock } from "@/lib/eml-import"; import { MboxSplitter, envelopeDate, readHeaderBlock, type MboxMessage } from "./mbox"; -import { mboxState } from "./mbox-status"; +import { mboxState, withoutStateHeaders } from "./mbox-status"; import { listZipEntries, readZipEntry, streamZipEntry } from "./zip-reader"; /** @@ -143,6 +143,13 @@ export async function* emlSource(file: File, options: SourceOptions): ItemSource options.onProgress?.(1); } +/** A date between 1980 and a day from now, else undefined. */ +function plausible(date: string | undefined, now = Date.now()): string | undefined { + if (!date) return undefined; + const t = Date.parse(date); + return t >= Date.UTC(1980, 0, 1) && t - now <= 24 * 60 * 60 * 1000 ? date : undefined; +} + function mboxItem(message: MboxMessage, index: number, folder: string[]): ImportItem { const label = message.tooLarge ? message.fromLine : ""; if (message.tooLarge) return { kind: "skipped", index, reason: "too_large", label }; @@ -150,14 +157,17 @@ function mboxItem(message: MboxMessage, index: number, folder: string[]): Import const state = mboxState(headers); const subject = subjectOf(headers); if (state.expunged) return { kind: "skipped", index, reason: "deleted", label: subject }; + const parts = withoutStateHeaders(message.parts); return { kind: "message", index, folder, - data: new Blob(message.parts as BlobPart[], { type: MIME }), - size: message.size, + data: new Blob(parts as BlobPart[], { type: MIME }), + size: parts.reduce((n, p) => n + p.length, 0), keywords: state.keywords, - receivedAt: dateFromHeaderBlock(headers) ?? envelopeDate(message.fromLine), + // The envelope line holds when the message arrived (Bulwark's export writes + // receivedAt there); the Date header, when it was written, comes second. + receivedAt: plausible(envelopeDate(message.fromLine)) ?? dateFromHeaderBlock(headers), label: subject, }; }