fix: keep failed messages for the next import run and stop an import that can't go on

A message that failed for a passing reason (a dropped connection, a
server error, an expired token) used to count as done, so continuing
the import skipped it for good, and a slow upload followed by a cancel
imported the messages after it twice.

- The checkpoint keeps every message that is done, also past a gap,
  and the next run of the same import tries only the rest again.
- Ten failures in a row, a full mailbox (overQuota) or a refusal
  (forbidden) stop the import with a message instead of going through
  the whole file.
- An upload without an answer pauses the import instead of failing the
  message: 30 seconds, then until the upload window ends.
- Email/import, the export's Blob/get and folder listing pass the
  client's own 429 pause by, and a 429 on them doesn't pause the rest
  of the app.
- Signing out stops the job of that account only, writes no checkpoint
  and imports nothing more afterwards; another account's sign-out
  leaves the job alone.
- One tab imports a file at a time (Web Locks), and Import waits until
  the dialog knows whether an earlier run can be continued.

The import dialog tests now wait for the Import button to be enabled
before clicking it, since it stays off during that lookup.

lib/__tests__/jmap-mail-transfer.test.ts

@@ -36,6 +36,28 @@ describe('importUploadedEmails', () => {
   });
 });
 
+describe('the transfer\'s own JMAP requests', () => {
+  const entry = { blobId: 'b1', mailboxIds: { inbox: true }, keywords: {} };
+
+  it('throw RateLimitError on a 429 without pausing the rest of the client', async () => {
+    vi.stubGlobal('fetch', vi.fn(async () => new Response('slow down', { status: 429, headers: { 'Retry-After': '20' } })));
+    const c = client();
+    Object.assign(c, { apiUrl: 'https://jmap.example.com/jmap/' });
+    await expect(c.importUploadedEmails([entry])).rejects.toMatchObject({ name: 'RateLimitError', retryAfterMs: 20_000 });
+    expect((c as unknown as { isRateLimited(): boolean }).isRateLimited()).toBe(false);
+  });
+
+  it('still go out while the client waits out a 429 of its own', async () => {
+    const fetch = vi.fn(async () => new Response(JSON.stringify({ methodResponses: [['Email/import', { created: { imp0: { id: 'e1' } } }, '0']] })));
+    vi.stubGlobal('fetch', fetch);
+    const c = client();
+    Object.assign(c, { apiUrl: 'https://jmap.example.com/jmap/' });
+    (c as unknown as { setRateLimited(ms: number): void }).setRateLimited(60_000);
+    await expect(c.importUploadedEmails([entry])).resolves.toEqual([{ id: 'e1' }]);
+    expect(fetch).toHaveBeenCalledTimes(1);
+  });
+});
+
 describe('fetchBlobForTransfer', () => {
   it('downloads with the account in the URL', async () => {
     const fetch = vi.fn(async () => new Response(new Uint8Array([1, 2, 3])));

lib/jmap/client-interface.ts

@@ -346,6 +346,8 @@ export interface IJMAPClient {
     properties: string[],
     accountId?: string,
     limit?: number,
+    /** `transfer`: a mail export's own request (a 429 doesn't pause the rest of the client). */
+    opts?: { transfer?: boolean },
   ): Promise<Array<Record<string, unknown>>>;
 
   // ── Email mutations ───────────────────────────────────────────

lib/jmap/client.ts

@@ -1752,10 +1752,10 @@ export class JMAPClient implements IJMAPClient {
   private async authenticatedFetch(
     url: string,
     init?: Parameters<typeof fetch>[1],
-    opts?: { timeoutMs?: number },
+    opts?: { timeoutMs?: number; transfer?: boolean },
   ): Promise<Response> {
     // Short-circuit: if rate-limited, reject immediately without sending a request
-    if (this.isRateLimited()) {
+    if (!opts?.transfer && this.isRateLimited()) {
       const remaining = this.rateLimitedUntil - Date.now();
       this.notifyRateLimitBlocked(remaining);
       throw new RateLimitError(remaining);
@@ -1786,7 +1786,7 @@ export class JMAPClient implements IJMAPClient {
     // Handle 429 rate limiting - stop immediately, do not retry
     if (response.status === 429) {
       const retryAfterMs = JMAPClient.parseRetryAfter(response);
-      this.setRateLimited(retryAfterMs);
+      if (!opts?.transfer) this.setRateLimited(retryAfterMs);
       throw new RateLimitError(retryAfterMs);
     }
 
@@ -2085,7 +2085,12 @@ export class JMAPClient implements IJMAPClient {
     }
   }
 
-  private async request(methodCalls: JMAPMethodCall[], using?: string[]): Promise<JMAPResponse> {
+  /**
+   * `transfer`: a mail import or export's own request. A 429 throws
+   * RateLimitError for the transfer to wait on, without pausing the rest of
+   * the client, and a pause the client is in doesn't hold the transfer up.
+   */
+  private async request(methodCalls: JMAPMethodCall[], using?: string[], opts?: { transfer?: boolean }): Promise<JMAPResponse> {
     if (!this.apiUrl) {
       throw new Error('Not connected. Call connect() first.');
     }
@@ -2105,7 +2110,7 @@ export class JMAPClient implements IJMAPClient {
             'Content-Type': 'application/json',
           },
           body,
-        }),
+        }, opts?.transfer ? { transfer: true } : undefined),
       );
 
       const responseText = await response.text();
@@ -4303,6 +4308,7 @@ export class JMAPClient implements IJMAPClient {
     properties: string[],
     accountId?: string,
     limit: number = 10000,
+    opts?: { transfer?: boolean },
   ): Promise<Array<Record<string, unknown>>> {
     const targetAccountId = accountId || this.accountId;
     const pageSize = Math.min(500, this.getMaxObjectsInGet());
@@ -4324,7 +4330,7 @@ export class JMAPClient implements IJMAPClient {
           "#ids": { resultOf: "0", name: "Email/query", path: "/ids" },
           properties: ["id", ...properties],
         }, "1"],
-      ]);
+      ], undefined, opts?.transfer ? { transfer: true } : undefined);
       assertQuerySucceeded(response, 'Email query');
       const [getName, getResult] = response.methodResponses?.[1] ?? [];
       if (getName !== "Email/get" || !getResult) {
@@ -6158,7 +6164,7 @@ export class JMAPClient implements IJMAPClient {
     const blobGet = async (chunk: string[], property: "size" | "data:asBase64") => {
       const call = () => this.request([
         ["Blob/get", { accountId: account, ids: chunk, properties: [property] }, "0"],
-      ], ["urn:ietf:params:jmap:core", BLOB_CAPABILITY]);
+      ], ["urn:ietf:params:jmap:core", BLOB_CAPABILITY], { transfer: options?.sized });
       let [name, result] = (await call()).methodResponses?.[0] ?? [];
       if (name === "error" && result?.type === "forbidden" && account !== this.accountId) {
         account = this.accountId;
@@ -11259,7 +11265,7 @@ export class JMAPClient implements IJMAPClient {
     });
     const response = await this.request([
       ['Email/import', { accountId: targetAccountId, emails }, '0'],
-    ]);
+    ], undefined, { transfer: true });
     const [name, result] = response.methodResponses?.[0] ?? [];
     if (name !== 'Email/import') {
       throw jmapError(`Email/import failed: ${result?.description || result?.type || 'unknown'}`, result);

lib/mail-export/engine.ts

@@ -268,6 +268,7 @@ export async function runExport(options: ExportOptions): Promise<ExportProgress>
         ["blobId", "keywords", "receivedAt", "size", "subject"],
         accountId,
         Number.MAX_SAFE_INTEGER,
+        { transfer: true },
       ));
       const refs = list
         .filter((e) => typeof e.blobId === "string")

lib/mail-import/__tests__/engine.test.ts

@@ -1,6 +1,6 @@
 import { describe, it, expect, vi } from 'vitest';
 import { runImport, uploadWindowEnd, quotaFrom, type ImportProgress, type ImportOptions } from '@/lib/mail-import/engine';
-import { checkpointKey, readCheckpoint } from '@/lib/mail-import/checkpoints';
+import { checkpointKey, deleteCheckpoint, readCheckpoint } from '@/lib/mail-import/checkpoints';
 import type { IJMAPClient, UploadedEmailImport } from '@/lib/jmap/client-interface';
 import type { Mailbox } from '@/lib/jmap/types';
 
@@ -19,7 +19,17 @@ const mailbox = (id: string, name: string, extra: Partial<Mailbox> = {}): Mailbo
   myRights: {} as Mailbox['myRights'], ...extra,
 });
 
-function fakeClient(opts: { failWhen?: (text: string) => boolean; quotaAfter?: number; uploadConcurrency?: number } = {}) {
+function fakeClient(opts: {
+  failWhen?: (text: string) => boolean;
+  quotaAfter?: number;
+  uploadConcurrency?: number;
+  /** Upload errors with no HTTP status (a dropped connection), for the first n uploads. */
+  networkErrors?: number;
+  /** Milliseconds an upload of a message takes. */
+  delay?: (text: string) => number;
+  /** An Email/import error for a message. */
+  importError?: (text: string) => { type: string; error: string } | undefined;
+} = {}) {
   let uploads = 0;
   let inFlight = 0;
   let maxInFlight = 0;
@@ -44,8 +54,10 @@ function fakeClient(opts: { failWhen?: (text: string) => boolean; quotaAfter?: n
       inFlight++;
       maxInFlight = Math.max(maxInFlight, inFlight);
       try {
-        await new Promise((r) => setTimeout(r, 1));
+        const body = await f.text();
+        await new Promise((r) => setTimeout(r, opts.delay?.(body) ?? 1));
         if (o.signal?.aborted) throw new DOMException('Upload aborted', 'AbortError');
+        if (opts.networkErrors && n < opts.networkErrors) throw new Error('Upload network error');
         if (opts.quotaAfter !== undefined && n >= opts.quotaAfter && !quotaHit) {
           quotaHit = true;
           throw new Error('Failed to upload file: 429 - {"title":"Quota exceeded","detail":"You have exceeded the blob upload quota of 1000 files or 50000000 bytes."}');
@@ -63,6 +75,8 @@ function fakeClient(opts: { failWhen?: (text: string) => boolean; quotaAfter?: n
       importCalls.push(entries.length);
       return entries.map((e) => {
         const text = blobs.get(e.blobId)!;
+        const refused = opts.importError?.(text);
+        if (refused) return refused;
         imported.push({ ...e, accountId, text });
         return { id: `email-${imported.length}` };
       });
@@ -201,6 +215,96 @@ describe('runImport', () => {
     await expect(readCheckpoint(key)).resolves.toBeNull();
   });
 
+  it('tries a message that failed again on the next run, and only that one', async () => {
+    const fake = fakeClient({ failWhen: (text) => text.startsWith('Subject: m1\n') });
+    const f = file(mbox(4));
+    const first = await runImport(options(fake.client, [f]));
+    expect(first).toMatchObject({ phase: 'done', imported: 3, failed: 1, retryable: 1 });
+    const again = fakeClient();
+    const second = await runImport(options(again.client, [f], { resume: true }));
+    expect(again.imported.map((e) => e.text.split('\n')[0])).toEqual(['Subject: m1']);
+    expect(second).toMatchObject({ phase: 'done', imported: 4, failed: 0 });
+  });
+
+  it('stops after ten failures in a row instead of burning through the file, and keeps the checkpoint', async () => {
+    const fake = fakeClient({ failWhen: () => true });
+    const f = file(mbox(40));
+    const result = await runImport(options(fake.client, [f]));
+    expect(result).toMatchObject({ phase: 'failed', error: 'import:failures' });
+    expect(fake.uploads).toBeLessThan(40 * 3);
+    const key = checkpointKey({ scope: 'bulwark-1', accountId: 'acc-shared', target: 'target', keepFolders: false, file: f });
+    // Nothing was done, so nothing is skipped next time: the failures are tried again.
+    expect((await readCheckpoint(key))?.done ?? 0).toBe(0);
+  });
+
+  it('stops at once when the mailbox is over quota', async () => {
+    const fake = fakeClient({ importError: (text) => (text.includes('Subject: m3\n') ? { type: 'overQuota', error: 'over quota' } : undefined) });
+    const result = await runImport(options(fake.client, [file(mbox(60))]));
+    expect(result).toMatchObject({ phase: 'failed', error: 'import:over_quota' });
+    expect(fake.uploads).toBeLessThan(60);
+  });
+
+  it('pauses on an upload with no answer and goes on, instead of failing the message', async () => {
+    const fake = fakeClient({ networkErrors: 1 });
+    const seen: ImportProgress[] = [];
+    const result = await runImport(options(fake.client, [file(mbox(3))], { onProgress: (p) => seen.push(p) }));
+    expect(seen.some((p) => p.phase === 'paused' && p.pausedUntil === T0 + 30_000)).toBe(true);
+    expect(result).toMatchObject({ phase: 'done', imported: 3, failed: 0 });
+  });
+
+  it('resumes without importing again what came in after a slow message', async () => {
+    // m2 is a slow upload; the messages after it are imported first.
+    const fake = fakeClient({ delay: (text) => (text.startsWith('Subject: m2\n') ? 300 : 1) });
+    const controller = new AbortController();
+    const f = file(mbox(12));
+    await runImport(options(fake.client, [f], {
+      signal: controller.signal,
+      onProgress: () => { if (fake.uploads >= 8 && !controller.signal.aborted) controller.abort(); },
+    }));
+    const firstRun = fake.imported.map((e) => e.text.split('\n')[0]);
+    expect(firstRun).not.toContain('Subject: m2');
+    expect(firstRun.some((s) => Number(s.slice(10)) > 2)).toBe(true);
+    await runImport(options(fake.client, [f], { resume: true }));
+    const subjects = fake.imported.map((e) => e.text.split('\n')[0]);
+    expect(subjects).toHaveLength(12);
+    expect(new Set(subjects).size).toBe(12);
+  });
+
+  it('writes no checkpoint and imports nothing more once the account signs out', async () => {
+    const fake = fakeClient({ delay: () => 20 });
+    const controller = new AbortController();
+    const f = file(mbox(30));
+    const key = checkpointKey({ scope: 'bulwark-1', accountId: 'acc-shared', target: 'target', keepFolders: false, file: f });
+    let callsAtSignOut = -1;
+    const result = await runImport(options(fake.client, [f], {
+      signal: controller.signal,
+      onProgress: (p) => {
+        if (p.imported >= 2 && !controller.signal.aborted) {
+          callsAtSignOut = fake.importCalls.length;
+          controller.abort('signout');
+          // What the sign-out does next: clear the account's checkpoints.
+          void deleteCheckpoint(key);
+        }
+      },
+    }));
+    expect(result.phase).toBe('cancelled');
+    expect(fake.importCalls.length).toBe(callsAtSignOut);
+    await expect(readCheckpoint(key)).resolves.toBeNull();
+  });
+
+  it('refuses a file another tab is importing', async () => {
+    const locks = { request: vi.fn(async (_n: string, _o: unknown, cb: (lock: unknown) => unknown) => cb(null)) };
+    vi.stubGlobal('navigator', { ...navigator, locks });
+    try {
+      const fake = fakeClient();
+      const result = await runImport(options(fake.client, [file(mbox(2))]));
+      expect(result).toMatchObject({ phase: 'failed', error: 'import:busy' });
+      expect(fake.uploads).toBe(0);
+    } finally {
+      vi.unstubAllGlobals();
+    }
+  });
+
   it('starts again from the top when not resuming', async () => {
     const fake = fakeClient();
     const f = file(mbox(3));

lib/mail-import/checkpoints.ts

@@ -12,8 +12,10 @@ export interface ImportCheckpoint {
   /** The Bulwark account that ran the import. */
   scope: string;
   fileName: string;
-  /** Items handled (imported, skipped or failed) from the start of the file. */
+  /** Items done for good (imported or skipped) from the start of the file, without a gap. */
   done: number;
+  /** Items done after the first gap (a message that failed and is tried again). */
+  ahead?: number[];
   imported: number;
   updatedAt: number;
 }

lib/mail-import/engine.ts

@@ -1,6 +1,7 @@
 import type { IJMAPClient, UploadedEmailImport, UploadedEmailImportResult } from "@/lib/jmap/client-interface";
 import type { Mailbox } from "@/lib/jmap/types";
 import { checkpointKey, deleteCheckpoint, readCheckpoint, writeCheckpoint } from "./checkpoints";
+import { holdLock } from "./locks";
 import { emlSource, mboxSource, zipSource, type ImportFormat, type ImportItem, type ItemSource, type SkipReason, type SourceOptions } from "./sources";
 
 /**
@@ -42,8 +43,10 @@ export interface ImportProgress {
   /** The limit the server named, when it did. */
   quota?: { files?: number; bytes?: number };
   foldersCreated: number;
-  /** Why the whole import stopped (phase failed). */
+  /** Why the whole import stopped (phase failed): `import:<code>`, `pst:<code>: ...` or the error's text. */
   error?: string;
+  /** Messages that failed for a passing reason; importing the same files again tries them again. */
+  retryable?: number;
 }
 
 export interface ImportFile {
@@ -138,17 +141,22 @@ function maxConcurrentUploads(client: IJMAPClient): number {
   return Math.max(1, Math.min(uploads, (client.getMaxConcurrentRequests?.() ?? 4) - 2));
 }
 
-/** Top-level folder names that mean a special folder, as Outlook and others name them. */
+/**
+ * Top-level folder names that mean a special folder, as Outlook, Thunderbird
+ * and others name them in the languages Outlook ships most often. Outlook
+ * files say which folder is which themselves (`folderRole`); for mbox and
+ * ZIP files the name is all there is.
+ */
 const ROLE_NAMES: Record<string, string[]> = {
-  inbox: ["inbox", "posteingang"],
-  sent: ["sent", "sent items", "sent mail", "sent messages", "gesendet", "gesendete elemente", "gesendete objekte"],
-  drafts: ["drafts", "entwürfe"],
-  trash: ["trash", "deleted items", "deleted messages", "papierkorb", "gelöschte elemente", "gelöschte objekte"],
-  junk: ["junk", "junk e-mail", "junk email", "junk mail", "spam", "junk-e-mail"],
-  archive: ["archive", "archiv"],
+  inbox: ["inbox", "posteingang", "boîte de réception", "bandeja de entrada", "posta in arrivo", "postvak in", "caixa de entrada", "skrzynka odbiorcza", "входящие", "doručená pošta", "inkorgen", "indbakke", "innboks", "saapuneet"],
+  sent: ["sent", "sent items", "sent mail", "sent messages", "gesendet", "gesendete elemente", "gesendete objekte", "éléments envoyés", "elementos enviados", "posta inviata", "verzonden items", "itens enviados", "elementy wysłane", "отправленные", "odeslaná pošta", "skickat", "sendt post", "sendte elementer", "lähetetyt"],
+  drafts: ["drafts", "entwürfe", "brouillons", "borradores", "bozze", "concepten", "rascunhos", "wersje robocze", "черновики", "koncepty", "utkast", "kladder", "luonnokset"],
+  trash: ["trash", "deleted items", "deleted messages", "papierkorb", "gelöschte elemente", "gelöschte objekte", "éléments supprimés", "elementos eliminados", "posta eliminata", "verwijderde items", "itens excluídos", "elementy usunięte", "удаленные", "удалённые", "odstraněná pošta", "borttaget", "slettet post", "slettede elementer", "poistetut"],
+  junk: ["junk", "junk e-mail", "junk email", "junk mail", "spam", "junk-e-mail", "courrier indésirable", "correo no deseado", "posta indesiderata", "ongewenste e-mail", "lixo eletrônico", "wiadomości-śmieci", "нежелательная почта", "nevyžádaná pošta", "skräppost", "uønsket e-mail", "søppelpost", "roskaposti"],
+  archive: ["archive", "archiv", "archives", "archivo", "archivio", "archief", "arquivo", "archiwum", "архив", "arkiv", "arkisto"],
 };
 
-function roleFor(name: string): string | undefined {
+export function roleFor(name: string): string | undefined {
   const key = name.trim().toLowerCase();
   return Object.keys(ROLE_NAMES).find((role) => ROLE_NAMES[role].includes(key));
 }
@@ -156,8 +164,9 @@ function roleFor(name: string): string | undefined {
 /**
  * Maps a file's folder paths to mailboxes in the account, creating what's
  * missing once. A folder with the same name under the same parent is used
- * again, so a resumed import doesn't make a second copy of the tree; at the
- * top level an "Inbox" or "Sent Items" means the account's own.
+ * again, so a resumed import doesn't make a second copy of the tree. At the
+ * top level a special folder (the file says so, or its name does) means the
+ * account's own Inbox, Sent, Drafts, Trash, Junk or Archive.
  */
 export class FolderResolver {
   private mailboxes: Mailbox[] | null = null;
@@ -185,11 +194,12 @@ export class FolderResolver {
     return FolderResolver.idOf(box);
   }
 
-  resolve(path: string[]): Promise<string> {
+  /** `topRole`: what the file says the first folder of `path` is. */
+  resolve(path: string[], topRole?: string): Promise<string> {
     const key = path.join("\u0000");
     let pending = this.cache.get(key);
     if (!pending) {
-      pending = this.resolveUncached(path);
+      pending = this.resolveUncached(path, topRole);
       // A failed creation may be tried again by the next message.
       pending.catch(() => this.cache.delete(key));
       this.cache.set(key, pending);
@@ -197,20 +207,20 @@ export class FolderResolver {
     return pending;
   }
 
-  private async resolveUncached(path: string[]): Promise<string> {
+  private async resolveUncached(path: string[], topRole?: string): Promise<string> {
     if (path.length === 0) return this.root ?? this.inbox();
-    const parentId = path.length === 1 ? this.root : await this.resolve(path.slice(0, -1));
+    const parentId = path.length === 1 ? this.root : await this.resolve(path.slice(0, -1), topRole);
     const name = path[path.length - 1].trim().slice(0, 255) || "Imported";
     const mailboxes = await this.list();
+    if (parentId === null) {
+      const role = (path.length === 1 && topRole) || roleFor(name);
+      const special = role && mailboxes.find((m) => m.role === role);
+      if (special) return FolderResolver.idOf(special);
+    }
     const same = (m: Mailbox) => m.name.trim().toLowerCase() === name.toLowerCase();
     const parentMatches = (m: Mailbox) => (parentId === null ? !m.parentId : m.parentId === parentId || m.parentId === mailboxes.find((p) => FolderResolver.idOf(p) === parentId)?.id);
     const existing = mailboxes.find((m) => parentMatches(m) && same(m));
     if (existing) return FolderResolver.idOf(existing);
-    if (parentId === null) {
-      const role = roleFor(name);
-      const special = role && mailboxes.find((m) => m.role === role);
-      if (special) return FolderResolver.idOf(special);
-    }
     const created = await this.client.createMailbox(name, parentId ?? undefined, this.accountId);
     this.created++;
     mailboxes.push({ ...created, parentId: parentId ?? undefined });
@@ -225,23 +235,66 @@ interface Uploaded {
   entry: UploadedEmailImport;
 }
 
-/** Counts handled items per file, so the checkpoint is the first index not handled yet. */
-class Watermark {
-  private next = 0;
-  private readonly ahead = new Set<number>();
-  constructor(start: number) { this.next = start; }
+/**
+ * Which items of a file are done for good (imported, or skipped for a reason
+ * that won't change): everything before `next`, and the ones in `ahead`
+ * past a gap. A message that failed for a passing reason leaves a gap, so
+ * the next run of the same import tries it again; nothing done is redone.
+ */
+export class Watermark {
+  private next: number;
+  private readonly ahead: Set<number>;
+  constructor(start: number, ahead: Iterable<number> = []) {
+    this.next = start;
+    this.ahead = new Set(ahead);
+    this.advance();
+  }
+  private advance(): void {
+    while (this.ahead.has(this.next)) { this.ahead.delete(this.next); this.next++; }
+  }
   mark(index: number): void {
     if (index < this.next) return;
     this.ahead.add(index);
-    while (this.ahead.has(this.next)) { this.ahead.delete(this.next); this.next++; }
+    this.advance();
+  }
+  has(index: number): boolean {
+    return index < this.next || this.ahead.has(index);
   }
   get value(): number { return this.next; }
+  get beyond(): number[] { return [...this.ahead].sort((a, b) => a - b); }
 }
 
+/** The whole import stops (phase failed); the checkpoint keeps what was done. */
+export class ImportStopped extends Error {
+  constructor(readonly code: "over_quota" | "forbidden" | "failures" | "network" | "busy") {
+    super(`import:${code}`);
+    this.name = "ImportStopped";
+  }
+}
+
+/** This many failures in a row stop the import instead of burning through the file. */
+const MAX_FAILURES_IN_A_ROW = 10;
+/** Pauses after uploads fail without an answer: a short one, then until the upload window ends. */
+const NETWORK_PAUSE_MS = 30_000;
+/** Email/import errors that stop the whole import: nothing more would go in. */
+const STOPPING_IMPORT_ERRORS: Record<string, ImportStopped["code"]> = { overQuota: "over_quota", forbidden: "forbidden" };
+
 export async function runImport(options: ImportOptions): Promise<ImportProgress> {
-  const { client, accountId, signal } = options;
+  const { client, accountId } = options;
   const now = options.now ?? Date.now;
   const sleep = options.sleep ?? defaultSleep;
+  // Our own signal: a cancel from outside, or a stop from inside.
+  const inner = new AbortController();
+  const outerAbort = () => inner.abort(options.signal.reason);
+  if (options.signal.aborted) outerAbort();
+  else options.signal.addEventListener("abort", outerAbort, { once: true });
+  const signal = inner.signal;
+  const state: { stopped: ImportStopped | null } = { stopped: null };
+  const stop = (error: ImportStopped) => {
+    if (!state.stopped) state.stopped = error;
+    inner.abort(error);
+  };
+
   const progress: ImportProgress = { phase: "running", imported: 0, failed: 0, skipped: {}, failures: [], fraction: 0, foldersCreated: 0 };
   const folders = new FolderResolver(client, accountId, options.targetMailboxId);
   const maxSize = client.getMaxSizeUpload() || Number.POSITIVE_INFINITY;
@@ -256,9 +309,15 @@ export async function runImport(options: ImportOptions): Promise<ImportProgress>
     options.onProgress({ ...progress, skipped: { ...progress.skipped }, failures: [...progress.failures] });
   };
   const skip = (reason: SkipReason, n = 1) => { progress.skipped[reason] = (progress.skipped[reason] ?? 0) + n; };
-  const fail = (label: string, reason: string) => {
+  let failuresInARow = 0;
+  let retryable = 0;
+  /** A failure; `permanent` ones (not a message at all) aren't tried again on the next run. */
+  const fail = (u: { fileIndex: number; index: number; label: string }, reason: string, permanent = false) => {
     progress.failed++;
-    if (progress.failures.length < MAX_FAILURES_LISTED) progress.failures.push({ label, reason });
+    if (progress.failures.length < MAX_FAILURES_LISTED) progress.failures.push({ label: u.label, reason });
+    if (permanent) { handled(u); return; }
+    progress.retryable = ++retryable;
+    if (++failuresInARow >= MAX_FAILURES_IN_A_ROW) stop(new ImportStopped("failures"));
   };
 
   // Shared pause: when the server refuses uploads, every lane waits.
@@ -292,10 +351,13 @@ export async function runImport(options: ImportOptions): Promise<ImportProgress>
   const importedPerFile: number[] = options.files.map(() => 0);
   let lastSaved = 0;
   const saveCheckpoints = async (force = false) => {
+    // After sign-out nothing about the account may be written again.
+    if (options.signal.reason === "signout") return;
     if (!force && now() - lastSaved < CHECKPOINT_EVERY_MS) return;
     lastSaved = now();
-    await Promise.all(marks.map((m, i) => m.value > 0 ? writeCheckpoint({
-      key: keys[i], scope: options.scope, fileName: options.files[i].file.name, done: m.value, imported: importedPerFile[i], updatedAt: now(),
+    await Promise.all(marks.map((m, i) => m.value > 0 || m.beyond.length > 0 ? writeCheckpoint({
+      key: keys[i], scope: options.scope, fileName: options.files[i].file.name,
+      done: m.value, ahead: m.beyond, imported: importedPerFile[i], updatedAt: now(),
     }) : Promise.resolve()));
   };
 
@@ -303,7 +365,9 @@ export async function runImport(options: ImportOptions): Promise<ImportProgress>
   let queue: Uploaded[] = [];
   let importing: Promise<void> = Promise.resolve();
   let flushTimer: ReturnType<typeof setTimeout> | null = null;
-  const handled = (u: { fileIndex: number; index: number }) => marks[u.fileIndex].mark(u.index);
+  function handled(u: { fileIndex: number; index: number }): void {
+    marks[u.fileIndex].mark(u.index);
+  }
 
   const importBatch = async (batch: Uploaded[]) => {
     let results: UploadedEmailImportResult[] | null = null;
@@ -318,22 +382,36 @@ export async function runImport(options: ImportOptions): Promise<ImportProgress>
       } catch (error) {
         const name = (error as { name?: string } | null)?.name;
         if (name === "RateLimitError" && attempt < 3) {
-          await sleep(Math.max(1000, (error as { retryAfterMs?: number }).retryAfterMs ?? 60_000), signal);
+          await sleep(Math.max(1000, (error as { retryAfterMs?: number }).retryAfterMs ?? 60_000), options.signal).catch(() => {});
+          if (options.signal.aborted) break;
+          continue;
+        }
+        if (attempt < 2 && !isAbort(error)) {
+          await sleep(2000 * (attempt + 1), options.signal).catch(() => {});
+          if (options.signal.aborted) break;
           continue;
         }
-        if (attempt < 1 && !isAbort(error)) { await sleep(2000, signal); continue; }
         results = batch.map(() => ({ error: error instanceof Error ? error.message : String(error) }));
       }
     }
+    if (!results) return;
     results.forEach((result, i) => {
       const item = batch[i];
       if ("id" in result) {
         progress.imported++;
         importedPerFile[item.fileIndex]++;
-      } else {
-        fail(item.label, result.type === "invalidEmail" ? "invalid" : result.error);
+        failuresInARow = 0;
+        handled(item);
+        return;
       }
-      handled(item);
+      const stopping = result.type ? STOPPING_IMPORT_ERRORS[result.type] : undefined;
+      if (stopping) {
+        progress.failed++;
+        stop(new ImportStopped(stopping));
+        return;
+      }
+      // A blob that isn't a message won't become one: don't try it again.
+      fail(item, result.type === "invalidEmail" ? "invalid" : result.error, result.type === "invalidEmail");
     });
     report();
     await saveCheckpoints();
@@ -355,14 +433,15 @@ export async function runImport(options: ImportOptions): Promise<ImportProgress>
 
   // ── Uploads, a few at a time ──
   const running = new Set<Promise<void>>();
+  let networkErrors = 0;
   const upload = async (item: Extract<ImportItem, { kind: "message" }>, fileIndex: number) => {
+    const u = { fileIndex, index: item.index, label: item.label };
     let mailboxId: string;
     try {
-      mailboxId = await folders.resolve(options.keepFolders ? item.folder : []);
+      mailboxId = await folders.resolve(options.keepFolders ? item.folder : [], item.folderRole);
     } catch (error) {
-      if (isAbort(error)) throw error;
-      fail(item.label, error instanceof Error ? error.message : String(error));
-      handled({ fileIndex, index: item.index });
+      if (isAbort(error) || signal.aborted) throw new ImportCancelled();
+      fail(u, error instanceof Error ? error.message : String(error));
       return;
     }
     const file = new File([item.data], "message.eml", { type: "message/rfc822" });
@@ -370,6 +449,7 @@ export async function runImport(options: ImportOptions): Promise<ImportProgress>
       await waitWhilePaused();
       try {
         const { blobId } = await client.uploadBlob(file, { accountId, signal });
+        networkErrors = 0;
         enqueue({
           index: item.index,
           fileIndex,
@@ -385,17 +465,25 @@ export async function runImport(options: ImportOptions): Promise<ImportProgress>
           progress.quota = quotaFrom(body);
           continue;
         }
+        if (status === undefined) {
+          // No answer at all: the connection dropped, or a refusal the browser
+          // couldn't read. Pause and try again rather than losing messages.
+          networkErrors++;
+          if (networkErrors > 2) { stop(new ImportStopped("network")); throw new ImportCancelled(); }
+          pausedUntil = Math.max(pausedUntil, networkErrors === 1 ? now() + NETWORK_PAUSE_MS : uploadWindowEnd(now()) + 5000);
+          continue;
+        }
         if (status === 413 || (status === 400 && /maxSizeUpload/.test(body))) {
           skip("too_large");
-          handled({ fileIndex, index: item.index });
+          handled(u);
           return;
         }
+        if (status === 403) { stop(new ImportStopped("forbidden")); throw new ImportCancelled(); }
         if (attempt < UPLOAD_RETRY_DELAYS_MS.length) {
           await sleep(UPLOAD_RETRY_DELAYS_MS[attempt], signal);
           continue;
         }
-        fail(item.label, status ? `HTTP ${status}` : body);
-        handled({ fileIndex, index: item.index });
+        fail(u, `HTTP ${status}`);
         return;
       }
     }
@@ -404,30 +492,36 @@ export async function runImport(options: ImportOptions): Promise<ImportProgress>
   const startUpload = async (item: Extract<ImportItem, { kind: "message" }>, fileIndex: number) => {
     while (running.size >= lanes) await Promise.race(running);
     if (signal.aborted) throw new ImportCancelled();
-    // A lane never rejects: a cancel shows in `signal`, anything else is a failure of this message.
+    // A lane never rejects: a cancel or a stop shows in `signal`, anything else is a failure of this message.
     const task: Promise<void> = upload(item, fileIndex)
       .catch((error) => {
-        if (!isAbort(error) && !signal.aborted) {
-          fail(item.label, error instanceof Error ? error.message : String(error));
-          handled({ fileIndex, index: item.index });
-        }
+        if (!isAbort(error) && !signal.aborted) fail({ fileIndex, index: item.index, label: item.label }, error instanceof Error ? error.message : String(error));
       })
       .finally(() => { running.delete(task); });
     running.add(task);
   };
 
+  // One import of a file at a time, across the tabs of this browser.
+  const releases: (() => void)[] = [];
+  let keepCheckpoints = false;
   try {
+    for (const key of keys) {
+      const release = await holdLock(`bulwark-mail-import:${key}`);
+      if (!release) throw new ImportStopped("busy");
+      releases.push(release);
+    }
     for (let f = 0; f < options.files.length; f++) {
       const { file, format } = options.files[f];
       const checkpoint = options.resume ? await readCheckpoint(keys[f]) : null;
       if (!options.resume) await deleteCheckpoint(keys[f]);
-      const skipBefore = checkpoint?.done ?? 0;
       if (checkpoint) { progress.imported += checkpoint.imported; importedPerFile[f] = checkpoint.imported; }
-      marks.push(new Watermark(skipBefore));
+      const mark = new Watermark(checkpoint?.done ?? 0, checkpoint?.ahead ?? []);
+      marks.push(mark);
       progress.fileName = file.name;
       const sourceOptions: SourceOptions = {
         maxMessageBytes: maxSize,
-        skipBefore,
+        skipBefore: mark.value,
+        skipAlso: new Set(mark.beyond),
         signal,
         onProgress: (fraction) => { fileFraction[f] = fraction; setFraction(); report(); },
         onTotal: (count) => {
@@ -446,7 +540,7 @@ export async function runImport(options: ImportOptions): Promise<ImportProgress>
 
       try {
         for await (const item of source) {
-          if (signal.aborted) throw new ImportCancelled();
+          if (signal.aborted) throw state.stopped ?? new ImportCancelled();
           if (item.kind === "skipped") {
             // Items an earlier run handled are already counted in its imported total.
             if (item.reason !== "resumed") skip(item.reason);
@@ -464,21 +558,32 @@ export async function runImport(options: ImportOptions): Promise<ImportProgress>
       setFraction();
     }
     while (running.size > 0) await Promise.race(running);
-    if (signal.aborted) throw new ImportCancelled();
+    if (signal.aborted) throw state.stopped ?? new ImportCancelled();
     await flush();
     await importing;
+    if (state.stopped) throw state.stopped;
     progress.phase = "done";
     progress.fraction = 1;
     progress.pausedUntil = undefined;
-    await Promise.all(keys.map((k) => deleteCheckpoint(k)));
+    // Messages that failed for a passing reason are tried again by the next run.
+    keepCheckpoints = retryable > 0;
+    if (keepCheckpoints) await saveCheckpoints(true);
+    else await Promise.all(keys.map((k) => deleteCheckpoint(k)));
   } catch (error) {
-    // Let the uploads in flight end (they were aborted) and keep what got imported.
+    const signedOut = options.signal.reason === "signout";
     await Promise.allSettled([...running]);
-    await flush().catch(() => {});
-    await importing.catch(() => {});
-    await saveCheckpoints(true).catch(() => {});
+    if (!signedOut) {
+      // Keep what was uploaded and where the import got to.
+      await flush().catch(() => {});
+      await importing.catch(() => {});
+      await saveCheckpoints(true).catch(() => {});
+    }
     progress.pausedUntil = undefined;
-    if (isAbort(error) || signal.aborted) {
+    const reason = state.stopped ?? error;
+    if ((reason as { name?: string } | null)?.name === "ImportStopped") {
+      progress.phase = "failed";
+      progress.error = (reason as ImportStopped).message;
+    } else if (isAbort(error) || options.signal.aborted) {
       progress.phase = "cancelled";
     } else {
       progress.phase = "failed";
@@ -486,6 +591,8 @@ export async function runImport(options: ImportOptions): Promise<ImportProgress>
     }
   } finally {
     if (flushTimer) clearTimeout(flushTimer);
+    options.signal.removeEventListener("abort", outerAbort);
+    for (const release of releases) release();
   }
   report(true);
   return progress;

lib/mail-import/locks.ts

@@ -0,0 +1,36 @@
+/**
+ * Web Locks across the tabs of this browser, held until released: one tab
+ * imports a file at a time, and an export's private file isn't swept while
+ * the tab that wrote it still uses it. Where the browser has no Web Locks,
+ * every request succeeds (one tab can't know about another then).
+ */
+
+interface LockManagerLike {
+  request(name: string, options: { ifAvailable: boolean }, callback: (lock: unknown) => Promise<void> | void): Promise<unknown>;
+}
+
+function lockManager(): LockManagerLike | null {
+  const locks = typeof navigator !== "undefined" ? (navigator as { locks?: LockManagerLike }).locks : undefined;
+  return locks && typeof locks.request === "function" ? locks : null;
+}
+
+export function hasWebLocks(): boolean {
+  return lockManager() !== null;
+}
+
+/** Takes the lock if nobody holds it: a release function, or null when another tab has it. */
+export function holdLock(name: string): Promise<(() => void) | null> {
+  const locks = lockManager();
+  if (!locks) return Promise.resolve(() => {});
+  return new Promise((resolve) => {
+    locks
+      .request(name, { ifAvailable: true }, (lock) => {
+        if (!lock) {
+          resolve(null);
+          return;
+        }
+        return new Promise<void>((release) => resolve(release));
+      })
+      .catch(() => resolve(() => {}));
+  });
+}

lib/mail-import/pst-source.ts

@@ -65,7 +65,7 @@ export async function* openPstSource(file: File, options: SourceOptions): ItemSo
   };
 
   try {
-    worker.postMessage({ type: "open", file, skipBefore: options.skipBefore, maxMessageBytes: options.maxMessageBytes });
+    worker.postMessage({ type: "open", file, skipBefore: options.skipBefore, skipAlso: [...(options.skipAlso ?? [])], maxMessageBytes: options.maxMessageBytes });
     const opened = await next();
     if (opened.type === "abort") throw new DOMException("Import cancelled", "AbortError");
     if (opened.type === "error") throw new PstReadError(opened.code, opened.message);

lib/mail-import/pst/items.ts

@@ -29,7 +29,7 @@ function utcDate(date: Date | undefined): string | undefined {
   return date.toISOString().replace(/\.\d{3}Z$/, "Z");
 }
 
-export function walkPst(pst: any, options: { skipBefore: number; maxMessageBytes: number; boundarySeed?: string }): PstWalk {
+export function walkPst(pst: any, options: { skipBefore: number; skipAlso?: Set<number>; maxMessageBytes: number; boundarySeed?: string }): PstWalk {
   const folders: PstFolderRef[] = [...pstFolders(pst)];
   const total = folders.reduce((n, f) => n + (Number(f.folder.contentCount) || 0), 0);
 
@@ -47,7 +47,7 @@ export function walkPst(pst: any, options: { skipBefore: number; maxMessageBytes
         }
         if (!child) break;
         const i = index++;
-        if (i < options.skipBefore) {
+        if (i < options.skipBefore || options.skipAlso?.has(i)) {
           yield { kind: "skipped", index: i, reason: "resumed" };
           continue;
         }

lib/mail-import/pst/worker.ts

@@ -4,7 +4,7 @@
  * folders and hands over one message at a time as MIME, only as many as the
  * page asked for, so a large file never piles up in memory.
  *
- *   page → worker  { type: "open", file, skipBefore, maxMessageBytes }
+ *   page → worker  { type: "open", file, skipBefore, skipAlso, maxMessageBytes }
  *                  { type: "pull", count }
  *   worker → page  { type: "opened", total }
  *                  { type: "item", item, fraction }   (item.mime is transferred)
@@ -37,12 +37,12 @@ function errorCode(error: unknown): string {
 }
 
 scope.onmessage = (event: MessageEvent) => {
-  const data = event.data as { type: string; file?: File; skipBefore?: number; maxMessageBytes?: number; count?: number };
+  const data = event.data as { type: string; file?: File; skipBefore?: number; skipAlso?: number[]; maxMessageBytes?: number; count?: number };
   try {
     if (data.type === "open" && data.file) {
       registerPstFile("import.pst", data.file);
       pst = new PSTFile("import.pst") as unknown as { close(): void };
-      const walk = walkPst(pst, { skipBefore: data.skipBefore ?? 0, maxMessageBytes: data.maxMessageBytes ?? Number.POSITIVE_INFINITY });
+      const walk = walkPst(pst, { skipBefore: data.skipBefore ?? 0, skipAlso: new Set(data.skipAlso ?? []), maxMessageBytes: data.maxMessageBytes ?? Number.POSITIVE_INFINITY });
       items = walk.items;
       total = walk.total;
       scope.postMessage({ type: "opened", total });

lib/mail-import/sources.ts

@@ -40,6 +40,8 @@ export interface ImportMessage {
   label: string;
   /** Messages attached to this one that couldn't be read. */
   embeddedSkipped?: number;
+  /** The special folder (inbox, sent, drafts, trash, junk) the top folder of `folder` is, when the file says so. */
+  folderRole?: string;
 }
 
 export interface ImportSkip {
@@ -56,6 +58,8 @@ export interface SourceOptions {
   maxMessageBytes: number;
   /** Items before this index were handled by an earlier run: report them as `resumed`. */
   skipBefore: number;
+  /** Items after `skipBefore` an earlier run handled too (it stopped with gaps). */
+  skipAlso?: Set<number>;
   /** Called as the file is read: the part done, 0 to 1. */
   onProgress?: (fraction: number) => void;
   /** Called once the number of messages in the file is known. */
@@ -65,6 +69,11 @@ export interface SourceOptions {
 
 export type ItemSource = AsyncGenerator<ImportItem, void, undefined>;
 
+/** Whether an earlier run of the same import handled item `index`. */
+export function isResumed(options: Pick<SourceOptions, "skipBefore" | "skipAlso">, index: number): boolean {
+  return index < options.skipBefore || !!options.skipAlso?.has(index);
+}
+
 const READ_CHUNK = 1 << 20;
 const MIME = "message/rfc822";
 
@@ -119,7 +128,7 @@ export function subjectOf(headerBlock: string): string {
 
 export async function* emlSource(file: File, options: SourceOptions): ItemSource {
   options.onTotal?.(1);
-  if (options.skipBefore > 0) {
+  if (isResumed(options, 0)) {
     yield { kind: "skipped", index: 0, reason: "resumed" };
     options.onProgress?.(1);
     return;
@@ -189,7 +198,7 @@ async function* mboxFromChunks(
     for (const m of messages) {
       onOffset?.(m.offset);
       const i = index++;
-      yield i < options.skipBefore ? { kind: "skipped", index: i, reason: "resumed" } : mboxItem(m, i, folder);
+      yield isResumed(options, i) ? { kind: "skipped", index: i, reason: "resumed" } : mboxItem(m, i, folder);
     }
   };
   for await (const chunk of chunks) {
@@ -249,7 +258,7 @@ export async function* zipSource(file: File, options: SourceOptions): ItemSource
       yield* mboxFromChunks(chunks, zipFolder(entry.name, true), options, index, (next) => { index = next; });
     } else if (isEml(entry.name)) {
       const i = index++;
-      if (i < options.skipBefore) {
+      if (isResumed(options, i)) {
         yield { kind: "skipped", index: i, reason: "resumed" };
       } else if (entry.size > options.maxMessageBytes) {
         yield { kind: "skipped", index: i, reason: "too_large", label: emlLabel(entry.name) };
@@ -269,7 +278,7 @@ export async function* zipSource(file: File, options: SourceOptions): ItemSource
       }
     } else {
       const i = index++;
-      yield i < options.skipBefore ? { kind: "skipped", index: i, reason: "resumed" } : { kind: "skipped", index: i, reason: "other", label: emlLabel(entry.name) };
+      yield isResumed(options, i) ? { kind: "skipped", index: i, reason: "resumed" } : { kind: "skipped", index: i, reason: "other", label: emlLabel(entry.name) };
     }
     doneBytes += entry.compressedSize;
     options.onProgress?.(doneBytes / totalBytes);