From c74426ffca7ea2d76c2713dbba9cdfba40b4f074 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 05:45:21 +0100 Subject: [PATCH] test: cover mbox and PST import and folder export against the test servers --- .../tests/22-ent-import-export.spec.ts | 351 ++++++++++++++++++ 1 file changed, 351 insertions(+) create mode 100644 integration/tests/22-ent-import-export.spec.ts diff --git a/integration/tests/22-ent-import-export.spec.ts b/integration/tests/22-ent-import-export.spec.ts new file mode 100644 index 000000000..59a4bd162 --- /dev/null +++ b/integration/tests/22-ent-import-export.spec.ts @@ -0,0 +1,351 @@ +/** + * Package C: mail import (mbox, PST) and export (folder as mbox or ZIP), + * through the real UI, as ent-carol, against a VM 106 Stalwart. + * + * Run with enterprise-rig/run-spec.sh. Each run works in its own folders + * (C--...) and deletes them with their mail at the end; the upload + * quota (1000 messages or 50 MB an hour per user) easily holds a run. + */ +import { test, expect, type Page, type Download } from '@playwright/test'; +import { mkdtempSync, readFileSync, utimesSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { ENT, ENT_PASSWORD, jmapAs, requireEntEnv, shot, userPage } from './helpers/ent'; +import { JMAP_URL } from './helpers/config'; +import { folderRow } from './helpers/app'; + +requireEntEnv(); + +const USER = ENT.carol; +const STAMP = `C-${Date.now().toString(36)}`; +const PST = readFileSync(path.join(__dirname, '..', '..', 'lib', 'mail-import', '__tests__', 'fixtures', 'Outlook.pst')); +const created: string[] = []; + +const MBOX = [ + 'From alice@example.org Tue May 5 10:11:12 2020', + 'Status: RO', + 'From: Alice ', + `Subject: ${STAMP} mbox read`, + 'Date: Tue, 5 May 2020 10:11:00 +0000', + '', + '>From the start of a line', + 'body one', + '', + 'From bob@example.org Wed May 6 11:00:00 2020', + 'Status: O', + 'X-Status: F', + 'From: Bob ', + `Subject: ${STAMP} mbox flagged`, + 'Date: Wed, 6 May 2020 10:59:00 +0000', + '', + 'body two', + '', + 'From carol@example.org Thu May 7 12:00:00 2020', + 'X-Mozilla-Status: 0009', + `Subject: ${STAMP} mbox deleted`, + '', + 'deleted in Thunderbird', + '', + 'From dave@example.org Fri May 8 13:30:00 2020', + `Subject: ${STAMP} mbox plain`, + '', + 'no state header', + '', +].join('\n'); + +const auth = () => 'Basic ' + Buffer.from(`${USER}:${ENT_PASSWORD}`).toString('base64'); + +async function createFolder(name: string, parentId: string | null = null): Promise { + const client = await jmapAs(USER); + const id = await client.createMailbox(name, parentId); + created.push(id); + return id; +} + +interface Stored { id: string; subject: string; receivedAt: string; keywords: Record; blobId: string; hasAttachment: boolean; mailboxIds: Record } + +async function messagesIn(mailboxId: string): Promise { + const client = await jmapAs(USER); + const res = await client.request([ + ['Email/query', { accountId: client.accountId, filter: { inMailbox: mailboxId }, sort: [{ property: 'receivedAt', isAscending: true }], limit: 500 }, '0'], + ['Email/get', { accountId: client.accountId, '#ids': { resultOf: '0', name: 'Email/query', path: '/ids' }, properties: ['subject', 'receivedAt', 'keywords', 'blobId', 'hasAttachment', 'mailboxIds'] }, '1'], + ]); + return res.methodResponses[1][1].list as Stored[]; +} + +async function children(parentId: string): Promise> { + const client = await jmapAs(USER); + const all = await client.mailboxes(); + return Object.fromEntries(all.filter((m) => m.parentId === parentId).map((m) => [m.name, m.id])); +} + +/** Seeds a message with Email/import, as another client would have stored it. */ +async function seed(mailboxId: string, subject: string, receivedAt: string, keywords: Record): Promise { + const client = await jmapAs(USER); + const raw = `From: Seed \r\nTo: ${USER}\r\nSubject: ${subject}\r\nDate: Mon, 1 Jan 2018 08:00:00 +0000\r\nMessage-ID: <${subject.replace(/\W/g, '')}@example.org>\r\n\r\nFrom the seed: ${subject}\r\n`; + const up = await fetch(`${JMAP_URL}/jmap/upload/${client.accountId}/`, { method: 'POST', headers: { Authorization: auth(), 'Content-Type': 'message/rfc822' }, body: raw }); + expect(up.ok).toBe(true); + const { blobId } = await up.json(); + const res = await client.request([['Email/import', { accountId: client.accountId, emails: { e: { blobId, mailboxIds: { [mailboxId]: true }, keywords, receivedAt } } }, '0']]); + expect(res.methodResponses[0][1].created?.e?.id, JSON.stringify(res.methodResponses[0][1])).toBeTruthy(); +} + +async function openFolderMenu(page: Page, name: string): Promise { + const row = folderRow(page, { name }).first(); + await expect(row).toBeVisible({ timeout: 30_000 }); + await row.click({ button: 'right' }); +} + +async function importFromFolderMenu(page: Page, folder: string, file: { name: string; buffer: Buffer; mimeType?: string } | string, opts: { keepFolders?: boolean } = {}) { + await openFolderMenu(page, folder); + await page.getByRole('menuitem', { name: 'Import mail…' }).click(); + const dialog = page.getByTestId('mail-import-dialog'); + await expect(dialog).toBeVisible(); + // A path keeps the file's date, which the resume checkpoint is keyed on, like a file on the user's disk. + await page.getByTestId('mail-import-file').setInputFiles(typeof file === 'string' ? file : { name: file.name, mimeType: file.mimeType ?? 'application/octet-stream', buffer: file.buffer }); + await expect(dialog.getByText(typeof file === 'string' ? path.basename(file) : file.name)).toBeVisible(); + if (opts.keepFolders === false) await dialog.getByRole('checkbox', { name: /Recreate the folders/ }).uncheck(); + return dialog; +} + +/** Hides the save picker so the export goes through the private file and a download Playwright can read. */ +async function withoutSavePicker(page: Page): Promise { + await page.evaluate(() => { delete (window as { showSaveFilePicker?: unknown }).showSaveFilePicker; }); +} + +async function exportFolder(page: Page, folder: string, format: 'mbox' | 'eml'): Promise { + await withoutSavePicker(page); + await openFolderMenu(page, folder); + await page.getByTestId('mailbox-export').click(); + const dialog = page.getByTestId('mail-export-dialog'); + await expect(dialog).toBeVisible(); + await page.getByTestId(`mail-export-format-${format}`).check(); + const download = page.waitForEvent('download', { timeout: 60_000 }); + await page.getByTestId('mail-export-start').click(); + const file = await download; + await expect(dialog.getByRole('heading', { name: 'Export finished' })).toBeVisible({ timeout: 60_000 }); + return file; +} + +async function readDownload(download: Download): Promise { + return readFileSync(await download.path()); +} + +test.describe.configure({ mode: 'serial' }); + +test.afterAll(async () => { + const client = await jmapAs(USER); + // Children first: a folder with subfolders can't be destroyed. + const all = await client.mailboxes(); + const ours = all.filter((m) => m.name.startsWith(STAMP) || created.includes(m.id)); + const withDescendants = new Set(ours.map((m) => m.id)); + let grew = true; + while (grew) { + grew = false; + for (const m of all) if (m.parentId && withDescendants.has(m.parentId) && !withDescendants.has(m.id)) { withDescendants.add(m.id); grew = true; } + } + const depth = (id: string): number => { const m = all.find((x) => x.id === id); return m?.parentId ? 1 + depth(m.parentId) : 0; }; + const ids = [...withDescendants].sort((a, b) => depth(b) - depth(a)); + for (const id of ids) { + await client.request([['Mailbox/set', { accountId: client.accountId, destroy: [id], onDestroyRemoveEmails: true }, '0']]).catch(() => {}); + } +}); + +test('an mbox lands in the folder with its dates, read and flagged state, and the deleted message left out', async ({ browser }) => { + const folderName = `${STAMP} mbox`; + const folderId = await createFolder(folderName); + const page = await userPage(browser, USER); + const dialog = await importFromFolderMenu(page, folderName, { name: 'thunderbird.mbox', buffer: Buffer.from(MBOX) }); + await expect(dialog.getByText('mbox file')).toBeVisible(); + await shot(page, 'c-import-dialog'); + await page.getByTestId('mail-import-start').click(); + await expect(dialog.getByRole('heading', { name: 'Import finished' })).toBeVisible({ timeout: 60_000 }); + await expect(dialog.getByText('Messages imported: 3')).toBeVisible(); + await expect(dialog.getByText('Deleted in Thunderbird: 1')).toBeVisible(); + await shot(page, 'c-import-summary'); + + const stored = await messagesIn(folderId); + expect(stored.map((m) => m.subject)).toEqual([`${STAMP} mbox read`, `${STAMP} mbox flagged`, `${STAMP} mbox plain`]); + // The envelope line is the arrival time. + expect(stored.map((m) => m.receivedAt)).toEqual(['2020-05-05T10:11:12Z', '2020-05-06T11:00:00Z', '2020-05-08T13:30:00Z']); + expect(stored.map((m) => !!m.keywords.$seen)).toEqual([true, false, true]); + expect(stored.map((m) => !!m.keywords.$flagged)).toEqual([false, true, false]); + // The stored message starts with its own headers (no leftover "From " envelope line) and is unquoted. + const res = await fetch(`${JMAP_URL}/jmap/download/${(await jmapAs(USER)).accountId}/${stored[0].blobId}/m.eml?accept=message/rfc822`, { headers: { Authorization: auth() } }); + const raw = await res.text(); + expect(raw.startsWith('From: Alice')).toBe(true); + expect(raw).toContain('\nFrom the start of a line'); + expect(raw).not.toMatch(/^Status:/m); + await page.context().close(); +}); + +test('a PST lands in its own folders under the target with dates, read state and attachments', async ({ browser }) => { + const parentName = `${STAMP} pst`; + const parentId = await createFolder(parentName); + const page = await userPage(browser, USER); + const dialog = await importFromFolderMenu(page, parentName, { name: 'Outlook.pst', buffer: PST }); + await expect(dialog.getByText('Outlook data file')).toBeVisible(); + await expect(dialog.getByRole('checkbox', { name: /Recreate the folders/ })).toBeChecked(); + await page.getByTestId('mail-import-start').click(); + await expect(dialog.getByRole('heading', { name: 'Import finished' })).toBeVisible({ timeout: 90_000 }); + await expect(dialog.getByText('Messages imported: 14')).toBeVisible(); + await expect(dialog.getByText('Folders created: 2')).toBeVisible(); + await expect(dialog.getByText('Contacts: 1')).toBeVisible(); + await expect(dialog.getByText('Calendar items: 1')).toBeVisible(); + await expect(dialog.getByText('Messages attached to other messages: 4')).toBeVisible(); + + const folders = await children(parentId); + expect(Object.keys(folders).sort()).toEqual(['Inbox', 'Sent Items']); + const inbox = await messagesIn(folders.Inbox); + const sent = await messagesIn(folders['Sent Items']); + expect(inbox).toHaveLength(8); + expect(sent).toHaveLength(6); + expect(inbox.filter((m) => !m.keywords.$seen).map((m) => m.subject).sort()).toEqual(['message 1', 'message 2']); + expect(inbox.find((m) => m.subject === 'Multiple attachments')?.receivedAt).toBe('2011-06-21T23:51:07Z'); + expect(inbox.find((m) => m.subject === 'Multiple attachments')?.hasAttachment).toBe(true); + // A message from the Sent folder, rebuilt without original headers. + const client = await jmapAs(USER); + const detail = await client.request([['Email/get', { accountId: client.accountId, ids: [sent.find((m) => m.subject === 'HTML body')!.id], properties: ['from', 'to', 'sentAt', 'attachments', 'htmlBody'] }, '0']]); + const html = detail.methodResponses[0][1].list[0]; + expect(html.from[0].email).toBe('saqib.razzaq@xp.local'); + expect(html.attachments.length).toBeGreaterThanOrEqual(12); + await page.context().close(); +}); + +test('a folder exported as mbox from the UI imports back to the same messages, dates and read state', async ({ browser }) => { + const sourceName = `${STAMP} export`; + const sourceId = await createFolder(sourceName); + await seed(sourceId, `${STAMP} exp read`, '2019-03-01T08:00:00Z', { $seen: true }); + await seed(sourceId, `${STAMP} exp flagged`, '2019-03-02T09:30:00Z', { $flagged: true }); + await seed(sourceId, `${STAMP} exp answered`, '2019-03-03T10:45:00Z', { $seen: true, $answered: true }); + const backName = `${STAMP} reimport`; + const backId = await createFolder(backName); + const page = await userPage(browser, USER); + + const download = await exportFolder(page, sourceName, 'mbox'); + expect(download.suggestedFilename()).toBe(`${sourceName}.mbox`); + const mbox = await readDownload(download); + expect(mbox.toString('latin1')).toMatch(/^From MAILER-DAEMON Fri Mar 1 08:00:00 2019\r?\n/); + expect(mbox.toString('latin1')).toContain('>From the seed'); + await shot(page, 'c-export-done'); + await page.getByRole('button', { name: 'Close' }).last().click(); + + const dialog = await importFromFolderMenu(page, backName, { name: download.suggestedFilename(), buffer: mbox }); + await page.getByTestId('mail-import-start').click(); + await expect(dialog.getByRole('heading', { name: 'Import finished' })).toBeVisible({ timeout: 60_000 }); + + const original = await messagesIn(sourceId); + const back = await messagesIn(backId); + const view = (list: Stored[]) => list.map((m) => ({ subject: m.subject, receivedAt: m.receivedAt, seen: !!m.keywords.$seen, flagged: !!m.keywords.$flagged, answered: !!m.keywords.$answered })); + expect(back).toHaveLength(3); + expect(view(back)).toEqual(view(original)); + await page.context().close(); +}); + +test('a folder exported as a ZIP of .eml files imports back to the same messages', async ({ browser }) => { + const sourceName = `${STAMP} export`; + const client = await jmapAs(USER); + const sourceId = (await client.mailboxByName(sourceName))!.id; + const backName = `${STAMP} reimport zip`; + const backId = await createFolder(backName); + const page = await userPage(browser, USER); + const download = await exportFolder(page, sourceName, 'eml'); + expect(download.suggestedFilename()).toBe(`${sourceName}.zip`); + const zip = await readDownload(download); + expect(zip.subarray(0, 4).toString('latin1')).toBe('PK\u0003\u0004'); + await page.getByRole('button', { name: 'Close' }).last().click(); + + const dialog = await importFromFolderMenu(page, backName, { name: download.suggestedFilename(), buffer: zip }); + await expect(dialog.getByText('ZIP archive')).toBeVisible(); + await page.getByTestId('mail-import-start').click(); + await expect(dialog.getByRole('heading', { name: 'Import finished' })).toBeVisible({ timeout: 60_000 }); + const original = (await messagesIn(sourceId)).map((m) => m.subject).sort(); + const back = (await messagesIn(backId)).map((m) => m.subject).sort(); + expect(back).toEqual(original); + await page.context().close(); +}); + +test('the progress dialog cancels an import, keeps what came in, and continues later without duplicates', async ({ browser }) => { + const folderName = `${STAMP} cancel`; + const folderId = await createFolder(folderName); + const count = 40; + const big = Array.from({ length: count }, (_, i) => + `From x@example.org Mon Jun 1 10:${String(i).padStart(2, '0')}:00 2020\nSubject: ${STAMP} n${i}\n\nbody ${i}\n\n`).join(''); + const onDisk = path.join(mkdtempSync(path.join(tmpdir(), 'bulwark-c-')), 'many.mbox'); + writeFileSync(onDisk, big); + utimesSync(onDisk, new Date('2024-01-01T00:00:00Z'), new Date('2024-01-01T00:00:00Z')); + const page = await userPage(browser, USER); + // Slow uploads down so Cancel lands in the middle. + await page.route(/\/jmap\/upload\//, async (route) => { + await new Promise((r) => setTimeout(r, 500)); + await route.continue(); + }); + const dialog = await importFromFolderMenu(page, folderName, onDisk); + await page.getByTestId('mail-import-start').click(); + await expect(dialog.getByRole('heading', { name: 'Importing mail' })).toBeVisible(); + await expect(dialog.getByRole('progressbar', { name: 'Import progress' })).toBeVisible(); + await expect(dialog.getByTestId('mail-import-count')).toHaveText(/Messages imported: [1-9]/, { timeout: 30_000 }); + await shot(page, 'c-import-progress'); + await dialog.getByRole('button', { name: 'Cancel import' }).click(); + await expect(dialog.getByRole('heading', { name: 'Import cancelled' })).toBeVisible({ timeout: 30_000 }); + await expect(dialog.getByText('To go on later, import the same files into the same folder again.')).toBeVisible(); + const partial = (await messagesIn(folderId)).length; + expect(partial).toBeGreaterThan(0); + expect(partial).toBeLessThan(count); + await page.getByRole('button', { name: 'Close' }).last().click(); + await page.unroute(/\/jmap\/upload\//); + + const again = await importFromFolderMenu(page, folderName, onDisk); + const resume = again.getByRole('checkbox', { name: /Continue where the last import/ }); + await expect(resume).toBeChecked(); + await page.getByTestId('mail-import-start').click(); + await expect(again.getByRole('heading', { name: 'Import finished' })).toBeVisible({ timeout: 60_000 }); + const all = await messagesIn(folderId); + expect(all).toHaveLength(count); + expect(new Set(all.map((m) => m.subject)).size).toBe(count); + await page.context().close(); +}); + +async function openBackupSettings(page: Page): Promise { + await page.goto('/settings/backup'); + // On a phone a settings link opens the list of sections first. + const button = page.getByTestId('settings-mail-import'); + if (!(await button.isVisible().catch(() => false))) { + await page.getByText('Backup & reset', { exact: true }).first().click(); + } + await expect(button).toBeVisible(); +} + +test('import and export dialogs fill the screen at 390 px and import from Settings', async ({ browser }) => { + const targetName = `${STAMP} phone`; + const target = await createFolder(targetName); + const page = await userPage(browser, USER, true); + await openBackupSettings(page); + await page.getByTestId('settings-mail-import').click(); + const dialog = page.getByTestId('mail-import-dialog'); + await expect(dialog).toBeVisible(); + // Full screen once the opening animation is over. + await expect.poll(async () => (await dialog.boundingBox())?.width ?? 0).toBeGreaterThanOrEqual(389); + await page.getByTestId('mail-import-file').setInputFiles({ name: 'phone.mbox', mimeType: 'application/mbox', buffer: Buffer.from(MBOX.replaceAll(`${STAMP} mbox`, `${STAMP} phone`)) }); + // From Settings there is no folder to start from: pick one. + await page.getByTestId('mail-import-target').selectOption({ label: targetName }); + await page.waitForTimeout(300); + await shot(page, 'c-import-phone'); + await page.getByTestId('mail-import-start').click(); + await expect(dialog.getByRole('heading', { name: 'Import finished' })).toBeVisible({ timeout: 60_000 }); + await expect.poll(async () => (await dialog.boundingBox())?.width ?? 0).toBeGreaterThanOrEqual(389); + await shot(page, 'c-import-phone-summary'); + expect(await messagesIn(target)).toHaveLength(3); + await page.getByRole('button', { name: 'Close' }).last().click(); + + await page.getByTestId('settings-mail-export').click(); + const exportDialog = page.getByTestId('mail-export-dialog'); + await expect(exportDialog.getByRole('heading', { name: 'Export all mail' })).toBeVisible(); + await expect.poll(async () => (await exportDialog.boundingBox())?.width ?? 0).toBeGreaterThanOrEqual(389); + await shot(page, 'c-export-phone'); + // Escape closes it and the focus goes back to the button that opened it. + await page.keyboard.press('Escape'); + await expect(exportDialog).toBeHidden(); + await expect(page.getByTestId('settings-mail-export')).toBeFocused(); + await page.context().close(); +});