mirror of
https://github.com/bulwarkmail/webmail.git
synced 2026-11-03 08:15:38 +00:00
Linus Rath
7102a6f0cb
- A message whose attachments are bigger than the server takes is skipped before anything is read (a 1 GB attachment took several GB in the worker and could crash it), and a worker that stops answering ends the import with a message instead of a wait that never ends. - The items after a broken one in a PST folder are counted as unreadable in the summary instead of disappearing. - Sent Items and Deleted Items are known from the PST's own entry IDs, and the Inbox and the others by their names in more languages, so a PST imported at the top level fills the user's own folders whatever its language. The hint about that shows only for the top level. - Message-ID, In-Reply-To and Content-ID from the PST can't carry line breaks into the rebuilt headers.
382 lines
20 KiB
TypeScript
382 lines
20 KiB
TypeScript
import { describe, it, expect, vi } from 'vitest';
|
|
import { runImport, uploadWindowEnd, quotaFrom, FolderResolver, type ImportProgress, type ImportOptions } from '@/lib/mail-import/engine';
|
|
import { checkpointKey, deleteCheckpoint, readCheckpoint } from '@/lib/mail-import/checkpoints';
|
|
import type { IJMAPClient, UploadedEmailImport } from '@/lib/jmap/client-interface';
|
|
import type { Mailbox } from '@/lib/jmap/types';
|
|
|
|
const mbox = (n: number, opts: { status?: (i: number) => string } = {}) =>
|
|
Array.from({ length: n }, (_, i) =>
|
|
`From a@example.org Tue May 5 10:${String(i % 60).padStart(2, '0')}:00 2020\nSubject: m${i}\n${opts.status?.(i) ?? ''}Date: Tue, 5 May 2020 10:00:00 +0000\n\nbody ${i}\n\n`,
|
|
).join('');
|
|
|
|
// Checkpoints older than 30 days are dropped, so the fake clock starts now.
|
|
const T0 = Date.now();
|
|
|
|
const file = (content: string, name = 'mail.mbox') => new File([content], name, { lastModified: 1000 });
|
|
|
|
const mailbox = (id: string, name: string, extra: Partial<Mailbox> = {}): Mailbox => ({
|
|
id, name, sortOrder: 0, totalEmails: 0, unreadEmails: 0, totalThreads: 0, unreadThreads: 0, isSubscribed: true,
|
|
myRights: {} as Mailbox['myRights'], ...extra,
|
|
});
|
|
|
|
function fakeClient(opts: {
|
|
failWhen?: (text: string) => boolean;
|
|
quotaAfter?: number;
|
|
uploadConcurrency?: number;
|
|
/** Upload errors with no HTTP status (a dropped connection), for the first n uploads. */
|
|
networkErrors?: number;
|
|
/** Milliseconds an upload of a message takes. */
|
|
delay?: (text: string) => number;
|
|
/** An Email/import error for a message. */
|
|
importError?: (text: string) => { type: string; error: string } | undefined;
|
|
} = {}) {
|
|
let uploads = 0;
|
|
let inFlight = 0;
|
|
let maxInFlight = 0;
|
|
let quotaHit = false;
|
|
const imported: Array<UploadedEmailImport & { accountId?: string; text: string }> = [];
|
|
const blobs = new Map<string, string>();
|
|
const created: Array<{ name: string; parentId?: string; accountId?: string }> = [];
|
|
const mailboxes: Mailbox[] = [mailbox('inbox', 'Inbox', { role: 'inbox' }), mailbox('target', 'Target')];
|
|
const importCalls: number[] = [];
|
|
const client = {
|
|
getCapabilities: () => ({ 'urn:ietf:params:jmap:core': { maxConcurrentUpload: opts.uploadConcurrency ?? 4 } }),
|
|
getMaxConcurrentRequests: () => 4,
|
|
getMaxSizeUpload: () => 1000,
|
|
getMailboxes: vi.fn(async () => mailboxes.map((m) => ({ ...m }))),
|
|
createMailbox: vi.fn(async (name: string, parentId?: string, accountId?: string) => {
|
|
created.push({ name, parentId, accountId });
|
|
const box = mailbox(`new-${created.length}`, name, { parentId });
|
|
return box;
|
|
}),
|
|
uploadBlob: vi.fn(async (f: File, o: { accountId?: string; signal?: AbortSignal }) => {
|
|
const n = uploads++;
|
|
inFlight++;
|
|
maxInFlight = Math.max(maxInFlight, inFlight);
|
|
try {
|
|
const body = await f.text();
|
|
await new Promise((r) => setTimeout(r, opts.delay?.(body) ?? 1));
|
|
if (o.signal?.aborted) throw new DOMException('Upload aborted', 'AbortError');
|
|
if (opts.networkErrors && n < opts.networkErrors) throw new Error('Upload network error');
|
|
if (opts.quotaAfter !== undefined && n >= opts.quotaAfter && !quotaHit) {
|
|
quotaHit = true;
|
|
throw new Error('Failed to upload file: 429 - {"title":"Quota exceeded","detail":"You have exceeded the blob upload quota of 1000 files or 50000000 bytes."}');
|
|
}
|
|
const text = await f.text();
|
|
if (opts.failWhen?.(text)) throw new Error('Failed to upload file: 500 - oops');
|
|
const id = `blob-${n}`;
|
|
blobs.set(id, text);
|
|
return { blobId: id, size: f.size, type: f.type };
|
|
} finally {
|
|
inFlight--;
|
|
}
|
|
}),
|
|
importUploadedEmails: vi.fn(async (entries: UploadedEmailImport[], accountId?: string) => {
|
|
importCalls.push(entries.length);
|
|
return entries.map((e) => {
|
|
const text = blobs.get(e.blobId)!;
|
|
const refused = opts.importError?.(text);
|
|
if (refused) return refused;
|
|
imported.push({ ...e, accountId, text });
|
|
return { id: `email-${imported.length}` };
|
|
});
|
|
}),
|
|
} as unknown as IJMAPClient;
|
|
return { client, imported, created, importCalls, get maxInFlight() { return maxInFlight; }, get uploads() { return uploads; } };
|
|
}
|
|
|
|
function options(client: IJMAPClient, files: File[], extra: Partial<ImportOptions> = {}): ImportOptions {
|
|
let clock = T0;
|
|
return {
|
|
client,
|
|
accountId: 'acc-shared',
|
|
targetMailboxId: 'target',
|
|
keepFolders: false,
|
|
files: files.map((f) => ({ file: f, format: f.name.endsWith('.zip') ? 'zip' : f.name.endsWith('.eml') ? 'eml' : 'mbox' })),
|
|
scope: 'bulwark-1',
|
|
resume: false,
|
|
signal: new AbortController().signal,
|
|
onProgress: () => {},
|
|
now: () => clock,
|
|
sleep: async (ms) => { clock += ms; },
|
|
reportEveryMs: 0,
|
|
...extra,
|
|
};
|
|
}
|
|
|
|
describe('runImport', () => {
|
|
it('imports every message of an mbox into the target folder of its own account, with state and date', async () => {
|
|
const fake = fakeClient();
|
|
const status = (i: number) => (i === 1 ? 'Status: O\nX-Status: F\n' : i === 2 ? 'X-Mozilla-Status: 0009\n' : '');
|
|
const result = await runImport(options(fake.client, [file(mbox(4, { status }))]));
|
|
expect(result.phase).toBe('done');
|
|
expect(result.imported).toBe(3);
|
|
expect(result.skipped).toEqual({ deleted: 1 });
|
|
expect(fake.imported.map((e) => e.accountId)).toEqual(['acc-shared', 'acc-shared', 'acc-shared']);
|
|
expect(fake.imported.every((e) => e.mailboxIds.target)).toBe(true);
|
|
expect(fake.imported.map((e) => e.keywords)).toEqual([{ $seen: true }, { $flagged: true }, { $seen: true }]);
|
|
expect(fake.imported[0].receivedAt).toBe('2020-05-05T10:00:00Z');
|
|
// The envelope line is gone; Email/import would keep it as a broken header.
|
|
expect(fake.imported[0].text.startsWith('Subject: m0\n')).toBe(true);
|
|
});
|
|
|
|
it('takes the arrival time from Bulwark\'s own envelope line and leaves the state headers out of the stored message', async () => {
|
|
const fake = fakeClient();
|
|
const raw = 'From bulwark-export Sat Jan 2 03:04:05 2021\nStatus: RO\nX-Mozilla-Status: 0001\nSubject: s\nDate: Tue, 5 May 2020 10:00:00 +0000\n\nbody\n';
|
|
await runImport(options(fake.client, [file(raw)]));
|
|
expect(fake.imported[0].receivedAt).toBe('2021-01-02T03:04:05Z');
|
|
expect(fake.imported[0].text).toBe('Subject: s\nDate: Tue, 5 May 2020 10:00:00 +0000\n\nbody\n');
|
|
expect(fake.imported[0].keywords).toEqual({ $seen: true });
|
|
});
|
|
|
|
it('reads Thunderbird state headers in .eml files, and imports a plain .eml as read like before', async () => {
|
|
const fake = fakeClient();
|
|
const unread = new File(['X-Mozilla-Status: 0004\r\nSubject: u\r\n\r\nStatus: R in the body\r\n'], 'u.eml');
|
|
const plain = new File(['Subject: p\r\n\r\nbody\r\n'], 'p.eml');
|
|
await runImport(options(fake.client, [unread, plain]));
|
|
expect(fake.imported.map((e) => e.keywords)).toEqual([{ $flagged: true }, { $seen: true }]);
|
|
});
|
|
|
|
it('dates a Thunderbird message by its Date header, not by when it was copied into the folder', async () => {
|
|
const fake = fakeClient();
|
|
await runImport(options(fake.client, [file('From - Sat Nov 1 09:00:00 2025\nSubject: s\nDate: Mon, 2 Jan 2017 08:30:00 +0100\n\nb\n')]));
|
|
expect(fake.imported[0].receivedAt).toBe('2017-01-02T07:30:00Z');
|
|
});
|
|
|
|
it('trusts an envelope line with a time zone (Google Takeout)', async () => {
|
|
const fake = fakeClient();
|
|
await runImport(options(fake.client, [file('From 1234@xxx Sat Apr 18 08:29:01 +0200 2020\nSubject: s\nDate: Fri, 17 Apr 2020 22:00:00 +0000\n\nb\n')]));
|
|
expect(fake.imported[0].receivedAt).toBe('2020-04-18T06:29:01Z');
|
|
});
|
|
|
|
it('falls back to the Date header when the envelope line has no usable date', async () => {
|
|
const fake = fakeClient();
|
|
await runImport(options(fake.client, [file('From MAILER-DAEMON Thu Jan 1 00:00:00 1970\nSubject: s\nDate: Tue, 5 May 2020 10:00:00 +0000\n\nb\n')]));
|
|
expect(fake.imported[0].receivedAt).toBe('2020-05-05T10:00:00Z');
|
|
});
|
|
|
|
it('moves the progress message by message through a small mbox', async () => {
|
|
const fake = fakeClient();
|
|
const fractions: number[] = [];
|
|
await runImport(options(fake.client, [file(mbox(10))], { onProgress: (p) => { if (p.phase === 'running') fractions.push(p.fraction); } }));
|
|
const between = new Set(fractions.filter((f) => f > 0 && f < 1).map((f) => f.toFixed(2)));
|
|
expect(between.size).toBeGreaterThanOrEqual(5);
|
|
});
|
|
|
|
it('puts several messages in one Email/import and keeps uploads within the server limit', async () => {
|
|
const fake = fakeClient({ uploadConcurrency: 4 });
|
|
await runImport(options(fake.client, [file(mbox(45))]));
|
|
expect(fake.imported).toHaveLength(45);
|
|
expect(Math.max(...fake.importCalls)).toBeGreaterThan(1);
|
|
expect(fake.importCalls.length).toBeLessThan(45);
|
|
// maxConcurrentRequests 4 leaves two requests for the rest of the app.
|
|
expect(fake.maxInFlight).toBeLessThanOrEqual(2);
|
|
});
|
|
|
|
it('pauses on the hourly upload limit until the window ends, then goes on', async () => {
|
|
const fake = fakeClient({ quotaAfter: 3 });
|
|
const seen: ImportProgress[] = [];
|
|
const result = await runImport(options(fake.client, [file(mbox(6))], { onProgress: (p) => seen.push(p) }));
|
|
const paused = seen.find((p) => p.phase === 'paused');
|
|
expect(paused?.pausedUntil).toBe(uploadWindowEnd(T0) + 5000);
|
|
expect(paused?.quota).toEqual({ files: 1000, bytes: 50_000_000 });
|
|
expect(result.phase).toBe('done');
|
|
expect(result.imported).toBe(6);
|
|
expect(fake.imported.map((e) => e.text.split('\n')[0]).sort()).toEqual(['Subject: m0', 'Subject: m1', 'Subject: m2', 'Subject: m3', 'Subject: m4', 'Subject: m5']);
|
|
});
|
|
|
|
it('counts a message that keeps failing and carries on with the rest', async () => {
|
|
const fake = fakeClient({ failWhen: (text) => text.startsWith('Subject: m1\n') });
|
|
const result = await runImport(options(fake.client, [file(mbox(3))]));
|
|
expect(result.imported).toBe(2);
|
|
expect(result.failed).toBe(1);
|
|
expect(result.failures).toEqual([{ label: 'm1', reason: 'HTTP 500' }]);
|
|
// Tried three times before giving up.
|
|
expect(fake.uploads).toBe(5);
|
|
});
|
|
|
|
it('skips messages over the upload size', async () => {
|
|
const fake = fakeClient();
|
|
const big = `From a Tue May 5 10:11:12 2020\nSubject: big\n\n${'x'.repeat(2000)}\n\n`;
|
|
const result = await runImport(options(fake.client, [file(big + mbox(1))]));
|
|
expect(result.skipped).toEqual({ too_large: 1 });
|
|
expect(result.imported).toBe(1);
|
|
});
|
|
|
|
it('keeps a checkpoint when cancelled and continues there without importing twice', async () => {
|
|
const fake = fakeClient();
|
|
const controller = new AbortController();
|
|
const f = file(mbox(30));
|
|
let cancelledAt = 0;
|
|
const first = await runImport(options(fake.client, [f], {
|
|
signal: controller.signal,
|
|
onProgress: (p) => { if (p.imported >= 5 && !controller.signal.aborted) { cancelledAt = p.imported; controller.abort(); } },
|
|
}));
|
|
expect(first.phase).toBe('cancelled');
|
|
expect(cancelledAt).toBeGreaterThan(0);
|
|
const key = checkpointKey({ scope: 'bulwark-1', accountId: 'acc-shared', target: 'target', keepFolders: false, file: f });
|
|
const saved = await readCheckpoint(key);
|
|
expect(saved?.done).toBe(first.imported);
|
|
|
|
const second = await runImport(options(fake.client, [f], { resume: true }));
|
|
expect(second.phase).toBe('done');
|
|
expect(second.imported).toBe(30);
|
|
const subjects = fake.imported.map((e) => e.text.split('\n')[0]);
|
|
expect(new Set(subjects).size).toBe(30);
|
|
expect(subjects).toHaveLength(30);
|
|
await expect(readCheckpoint(key)).resolves.toBeNull();
|
|
});
|
|
|
|
it('tries a message that failed again on the next run, and only that one', async () => {
|
|
const fake = fakeClient({ failWhen: (text) => text.startsWith('Subject: m1\n') });
|
|
const f = file(mbox(4));
|
|
const first = await runImport(options(fake.client, [f]));
|
|
expect(first).toMatchObject({ phase: 'done', imported: 3, failed: 1, retryable: 1 });
|
|
const again = fakeClient();
|
|
const second = await runImport(options(again.client, [f], { resume: true }));
|
|
expect(again.imported.map((e) => e.text.split('\n')[0])).toEqual(['Subject: m1']);
|
|
expect(second).toMatchObject({ phase: 'done', imported: 4, failed: 0 });
|
|
});
|
|
|
|
it('stops after ten failures in a row instead of burning through the file, and keeps the checkpoint', async () => {
|
|
const fake = fakeClient({ failWhen: () => true });
|
|
const f = file(mbox(40));
|
|
const result = await runImport(options(fake.client, [f]));
|
|
expect(result).toMatchObject({ phase: 'failed', error: 'import:failures' });
|
|
expect(fake.uploads).toBeLessThan(40 * 3);
|
|
const key = checkpointKey({ scope: 'bulwark-1', accountId: 'acc-shared', target: 'target', keepFolders: false, file: f });
|
|
// Nothing was done, so nothing is skipped next time: the failures are tried again.
|
|
expect((await readCheckpoint(key))?.done ?? 0).toBe(0);
|
|
});
|
|
|
|
it('stops at once when the mailbox is over quota', async () => {
|
|
const fake = fakeClient({ importError: (text) => (text.includes('Subject: m3\n') ? { type: 'overQuota', error: 'over quota' } : undefined) });
|
|
const result = await runImport(options(fake.client, [file(mbox(60))]));
|
|
expect(result).toMatchObject({ phase: 'failed', error: 'import:over_quota' });
|
|
expect(fake.uploads).toBeLessThan(60);
|
|
});
|
|
|
|
it('pauses on an upload with no answer and goes on, instead of failing the message', async () => {
|
|
const fake = fakeClient({ networkErrors: 1 });
|
|
const seen: ImportProgress[] = [];
|
|
const result = await runImport(options(fake.client, [file(mbox(3))], { onProgress: (p) => seen.push(p) }));
|
|
expect(seen.some((p) => p.phase === 'paused' && p.pausedUntil === T0 + 30_000)).toBe(true);
|
|
expect(result).toMatchObject({ phase: 'done', imported: 3, failed: 0 });
|
|
});
|
|
|
|
it('resumes without importing again what came in after a slow message', async () => {
|
|
// m2 is a slow upload; the messages after it are imported first.
|
|
const fake = fakeClient({ delay: (text) => (text.startsWith('Subject: m2\n') ? 300 : 1) });
|
|
const controller = new AbortController();
|
|
const f = file(mbox(12));
|
|
await runImport(options(fake.client, [f], {
|
|
signal: controller.signal,
|
|
onProgress: () => { if (fake.uploads >= 8 && !controller.signal.aborted) controller.abort(); },
|
|
}));
|
|
const firstRun = fake.imported.map((e) => e.text.split('\n')[0]);
|
|
expect(firstRun).not.toContain('Subject: m2');
|
|
expect(firstRun.some((s) => Number(s.slice(10)) > 2)).toBe(true);
|
|
await runImport(options(fake.client, [f], { resume: true }));
|
|
const subjects = fake.imported.map((e) => e.text.split('\n')[0]);
|
|
expect(subjects).toHaveLength(12);
|
|
expect(new Set(subjects).size).toBe(12);
|
|
});
|
|
|
|
it('writes no checkpoint and imports nothing more once the account signs out', async () => {
|
|
const fake = fakeClient({ delay: () => 20 });
|
|
const controller = new AbortController();
|
|
const f = file(mbox(30));
|
|
const key = checkpointKey({ scope: 'bulwark-1', accountId: 'acc-shared', target: 'target', keepFolders: false, file: f });
|
|
let callsAtSignOut = -1;
|
|
const result = await runImport(options(fake.client, [f], {
|
|
signal: controller.signal,
|
|
onProgress: (p) => {
|
|
if (p.imported >= 2 && !controller.signal.aborted) {
|
|
callsAtSignOut = fake.importCalls.length;
|
|
controller.abort('signout');
|
|
// What the sign-out does next: clear the account's checkpoints.
|
|
void deleteCheckpoint(key);
|
|
}
|
|
},
|
|
}));
|
|
expect(result.phase).toBe('cancelled');
|
|
expect(fake.importCalls.length).toBe(callsAtSignOut);
|
|
await expect(readCheckpoint(key)).resolves.toBeNull();
|
|
});
|
|
|
|
it('refuses a file another tab is importing', async () => {
|
|
const locks = { request: vi.fn(async (_n: string, _o: unknown, cb: (lock: unknown) => unknown) => cb(null)) };
|
|
vi.stubGlobal('navigator', { ...navigator, locks });
|
|
try {
|
|
const fake = fakeClient();
|
|
const result = await runImport(options(fake.client, [file(mbox(2))]));
|
|
expect(result).toMatchObject({ phase: 'failed', error: 'import:busy' });
|
|
expect(fake.uploads).toBe(0);
|
|
} finally {
|
|
vi.unstubAllGlobals();
|
|
}
|
|
});
|
|
|
|
it('starts again from the top when not resuming', async () => {
|
|
const fake = fakeClient();
|
|
const f = file(mbox(3));
|
|
await runImport(options(fake.client, [f]));
|
|
await runImport(options(fake.client, [f]));
|
|
expect(fake.imported).toHaveLength(6);
|
|
});
|
|
|
|
it('recreates a file\'s folders under the target, once, reusing ones that exist', async () => {
|
|
const { ZipWriter } = await import('@/lib/mail-export/zip-writer');
|
|
const parts: Uint8Array[] = [];
|
|
const z = new ZipWriter(async (c) => { parts.push(c.slice()); });
|
|
const enc = new TextEncoder();
|
|
await z.addFile('Projects/2020/a.eml', enc.encode('Subject: a\r\n\r\nA'));
|
|
await z.addFile('Projects/2020/b.eml', enc.encode('Subject: b\r\n\r\nB'));
|
|
await z.addFile('Projects/c.eml', enc.encode('Subject: c\r\n\r\nC'));
|
|
await z.addFile('Target/d.eml', enc.encode('Subject: d\r\n\r\nD'));
|
|
await z.addFile('top.eml', enc.encode('Subject: t\r\n\r\nT'));
|
|
await z.finish();
|
|
const zip = new File(parts as BlobPart[], 'export.zip', { lastModified: 1 });
|
|
|
|
const fake = fakeClient();
|
|
const result = await runImport(options(fake.client, [zip], { keepFolders: true, targetMailboxId: null }));
|
|
expect(result.imported).toBe(5);
|
|
expect(fake.created).toEqual([
|
|
{ name: 'Projects', parentId: undefined, accountId: 'acc-shared' },
|
|
{ name: '2020', parentId: 'new-1', accountId: 'acc-shared' },
|
|
]);
|
|
const byText = Object.fromEntries(fake.imported.map((e) => [e.text.split('\r\n')[0], Object.keys(e.mailboxIds)[0]]));
|
|
// "Target" exists at the top; a message at the top of the file goes to the Inbox.
|
|
expect(byText).toEqual({ 'Subject: a': 'new-2', 'Subject: b': 'new-2', 'Subject: c': 'new-1', 'Subject: d': 'target', 'Subject: t': 'inbox' });
|
|
expect(result.foldersCreated).toBe(2);
|
|
});
|
|
|
|
it('puts everything into the target when folders are not kept', async () => {
|
|
const { ZipWriter } = await import('@/lib/mail-export/zip-writer');
|
|
const parts: Uint8Array[] = [];
|
|
const z = new ZipWriter(async (c) => { parts.push(c.slice()); });
|
|
await z.addFile('Deep/x.eml', new TextEncoder().encode('Subject: x\r\n\r\nX'));
|
|
await z.finish();
|
|
const fake = fakeClient();
|
|
await runImport(options(fake.client, [new File(parts as BlobPart[], 'e.zip')]));
|
|
expect(fake.created).toEqual([]);
|
|
expect(Object.keys(fake.imported[0].mailboxIds)).toEqual(['target']);
|
|
});
|
|
|
|
it('puts a top folder into the special folder the file names, in any language', async () => {
|
|
const mailboxes = [mailbox('in', 'Posteingang', { role: 'inbox' }), mailbox('sent', 'Gesendet', { role: 'sent' }), mailbox('bin', 'Papierkorb', { role: 'trash' })];
|
|
const client = { getMailboxes: async () => mailboxes, createMailbox: vi.fn() } as unknown as IJMAPClient;
|
|
const folders = new FolderResolver(client, 'acc', null);
|
|
// A Spanish PST whose store says this is Sent Items.
|
|
await expect(folders.resolve(['Elementos enviados'], 'sent')).resolves.toBe('sent');
|
|
// Names in the languages Outlook ships most often.
|
|
await expect(folders.resolve(['Éléments supprimés'])).resolves.toBe('bin');
|
|
await expect(folders.resolve(['Bandeja de entrada'])).resolves.toBe('in');
|
|
});
|
|
|
|
it('reads the hour window and the quota from the server', () => {
|
|
expect(uploadWindowEnd(Date.UTC(2026, 10, 1, 13, 20))).toBe(Date.UTC(2026, 10, 1, 14, 0));
|
|
expect(quotaFrom('You have exceeded the blob upload quota of 1000 files or 50000000 bytes.')).toEqual({ files: 1000, bytes: 50000000 });
|
|
});
|
|
});
|