webmail/lib/mail-export/engine.ts

import type { IJMAPClient } from "@/lib/jmap/client-interface";
import type { Mailbox } from "@/lib/jmap/types";
import { mboxEntry } from "./mbox-writer";
import { ZipWriter, type ZipEntryStream } from "./zip-writer";
import type { ExportFile, ExportSink } from "./sinks";

/**
 * Exports folders as mbox (mboxrd) or as a ZIP of .eml files, streamed into a
 * sink so the mailbox never sits in memory.
 *
 * - One folder as mbox is a plain .mbox file; anything else is a ZIP: one
 *   `<path>.mbox` per folder, or `<path>/<n> <subject>.eml` per message, with
 *   empty folders kept as directories. A message in several folders is
 *   written once per folder, as IMAP programs do.
 * - Messages come in batches through Blob/get (many per request) where the
 *   server offers it, larger ones and the rest as downloads. At most
 *   maxConcurrentRequests - 2 requests run at once (Stalwart 1.0 counts
 *   downloads against it), and the export keeps itself well under the
 *   server's 1000 requests a minute. A 429 pauses the export alone, never
 *   the rest of the app (downloads don't go through the client-wide limit).
 */

export type ExportFormat = "mbox" | "eml";
export type ExportPhase = "running" | "paused" | "done" | "cancelled" | "failed";

export interface ExportFolder {
  /** The mailbox id in its account (not the store's namespaced id). */
  mailboxId: string;
  /** Path of names from the top, e.g. ["Archive", "2020"]. */
  path: string[];
}

export type NotKeptReason = "attachments_off" | "not_kept";

/**
 * A local-store export has no original of this message and can't reach the
 * server. Counted by reason in `notKept`, not listed one by one.
 */
export class MessageNotKeptError extends Error {
  constructor(readonly reason: NotKeptReason) {
    super("The original of this message is not on this device");
    this.name = "MessageNotKeptError";
  }
}

export interface ExportProgress {
  phase: ExportPhase;
  exported: number;
  failed: number;
  failures: { label: string; reason: string }[];
  /** The part of `failed` a local-store export had no original for, by reason. */
  notKept?: Partial<Record<NotKeptReason, number>>;
  total: number;
  fraction: number;
  pausedUntil?: number;
  file?: ExportFile;
  error?: string;
  /** What stopped it, as the error's name: a dialog says this in the user's language. */
  errorCode?: string;
  /** The limit an ExportTooLargeError ran into, in bytes. */
  errorLimit?: number;
  /** A file in the place the user picked that couldn't be removed after a run that wrote nothing whole. */
  leftover?: string;
}

/** What an export reads through: the real client, or lib/mail-export/local-store-client.ts. */
export type ExportClient = Pick<IJMAPClient, "queryEmailFields" | "getBlobsAsBase64" | "fetchBlobForTransfer" | "fetchBlob" | "getMaxConcurrentRequests">;

export interface ExportOptions {
  client: ExportClient;
  accountId: string;
  folders: ExportFolder[];
  format: ExportFormat;
  sink: ExportSink;
  signal: AbortSignal;
  onProgress: (progress: ExportProgress) => void;
  now?: () => number;
  sleep?: (ms: number, signal: AbortSignal) => Promise<void>;
  reportEveryMs?: number;
  /** No file when nothing was written, also when nothing failed: the person chose mail, not an empty file. */
  noEmptyFile?: boolean;
  /**
   * Requests per minute the export allows itself (the server's limit is 1000 for
   * everything). Infinity for a client that keeps to the limit itself.
   */
  requestsPerMinute?: number;
}

interface MessageRef {
  id: string;
  blobId: string;
  size: number;
  keywords: Record<string, boolean>;
  receivedAt?: string;
  subject: string;
}

/** Messages up to this size travel inside Blob/get; bigger ones are downloaded. */
const BATCH_MESSAGE_MAX = 1024 * 1024;
/** Bytes per Blob/get, like the client's BLOB_BATCH_BYTES. */
const BATCH_BYTES = 4 * 1024 * 1024;
const BATCH_COUNT = 100;
const MAX_FAILURES_LISTED = 20;
const RETRY_DELAYS_MS = [500, 2000, 5000];

function leftoverOf(aborted: { leftover?: string } | void | undefined): string | undefined {
  return aborted ? aborted.leftover : undefined;
}

function isAbort(error: unknown): boolean {
  return (error as { name?: string } | null)?.name === "AbortError";
}

function cancelled(): DOMException {
  return new DOMException("Export cancelled", "AbortError");
}

function defaultSleep(ms: number, signal: AbortSignal): Promise<void> {
  return new Promise((resolve, reject) => {
    if (signal.aborted) { reject(cancelled()); return; }
    const timer = setTimeout(() => { signal.removeEventListener("abort", onAbort); resolve(); }, ms);
    const onAbort = () => { clearTimeout(timer); reject(cancelled()); };
    signal.addEventListener("abort", onAbort, { once: true });
  });
}

function decodeBase64(base64: string): Uint8Array {
  const fromBase64 = (Uint8Array as unknown as { fromBase64?: (s: string) => Uint8Array }).fromBase64;
  if (fromBase64) return fromBase64(base64);
  const binary = atob(base64);
  const out = new Uint8Array(binary.length);
  for (let i = 0; i < binary.length; i++) out[i] = binary.charCodeAt(i);
  return out;
}

/** A name that is safe as a file or folder name on every system. */
export function safeName(name: string, fallback: string, max = 80): string {
  // eslint-disable-next-line no-control-regex
  const clean = name.replace(/[\u0000-\u001f\u007f<>:"/\\|?*]+/g, " ").replace(/\s+/g, " ").trim().replace(/^\.+|\.+$/g, "");
  const cut = Array.from(clean).slice(0, max).join("").trim();
  return cut || fallback;
}

/**
 * The account's folders as export paths, parents first in the order the app
 * shows them; two folders with the same name next to each other get " (2)".
 */
export function folderTree(mailboxes: Mailbox[]): ExportFolder[] {
  const idOf = (m: Mailbox) => m.originalId || m.id;
  const byParent = new Map<string, Mailbox[]>();
  const ids = new Set(mailboxes.map((m) => m.id));
  for (const m of mailboxes) {
    const parent = m.parentId && ids.has(m.parentId) ? m.parentId : "";
    if (!byParent.has(parent)) byParent.set(parent, []);
    byParent.get(parent)!.push(m);
  }
  const ROLE_ORDER = ["inbox", "drafts", "sent", "archive", "junk", "trash"];
  const rank = (m: Mailbox) => { const i = m.role ? ROLE_ORDER.indexOf(m.role) : -1; return i === -1 ? ROLE_ORDER.length : i; };
  const out: ExportFolder[] = [];
  const visit = (parent: string, path: string[]) => {
    const children = (byParent.get(parent) ?? []).slice().sort((a, b) => rank(a) - rank(b) || (a.sortOrder ?? 0) - (b.sortOrder ?? 0) || a.name.localeCompare(b.name));
    const used = new Set<string>();
    for (const m of children) {
      const base = safeName(m.name, "Folder");
      let name = base;
      for (let n = 2; used.has(name.toLowerCase()); n++) name = `${base} (${n})`;
      used.add(name.toLowerCase());
      out.push({ mailboxId: idOf(m), path: [...path, name] });
      visit(m.id, [...path, name]);
    }
  };
  visit("", []);
  return out;
}

/** Keeps the export under a number of requests per minute. */
export class RateGate {
  private times: number[] = [];
  constructor(private readonly perMinute: number, private readonly now: () => number, private readonly sleep: (ms: number) => Promise<void>) {}
  async take(): Promise<void> {
    if (this.perMinute === Infinity) return;
    for (;;) {
      const t = this.now();
      this.times = this.times.filter((x) => t - x < 60_000);
      if (this.times.length < this.perMinute) { this.times.push(t); return; }
      await this.sleep(60_000 - (t - this.times[0]) + 10);
    }
  }
}

export async function runExport(options: ExportOptions): Promise<ExportProgress> {
  const { client, accountId, signal, sink } = options;
  const now = options.now ?? Date.now;
  const sleep = (ms: number) => (options.sleep ?? defaultSleep)(ms, signal);
  const gate = new RateGate(options.requestsPerMinute ?? 600, now, sleep);
  const lanes = Math.max(1, (client.getMaxConcurrentRequests?.() ?? 4) - 2);
  const progress: ExportProgress = { phase: "running", exported: 0, failed: 0, failures: [], total: 0, fraction: 0 };
  let lastReport = 0;
  const report = (force = false) => {
    const t = now();
    if (!force && t - lastReport < (options.reportEveryMs ?? 150)) return;
    lastReport = t;
    progress.fraction = progress.total > 0 ? Math.min(1, (progress.exported + progress.failed) / progress.total) : 0;
    options.onProgress({ ...progress, failures: [...progress.failures], ...(progress.notKept ? { notKept: { ...progress.notKept } } : {}) });
  };
  const fail = (label: string, reason: string) => {
    progress.failed++;
    if (progress.failures.length < MAX_FAILURES_LISTED) progress.failures.push({ label, reason });
  };
  const pause = async (ms: number) => {
    progress.phase = "paused";
    progress.pausedUntil = now() + ms;
    report(true);
    await sleep(ms);
    progress.phase = "running";
    progress.pausedUntil = undefined;
    report(true);
  };

  /** Runs a request, waiting out the server's refusals. */
  const withRetry = async <T>(run: () => Promise<T>): Promise<T> => {
    for (let attempt = 0; ; attempt++) {
      if (signal.aborted) throw cancelled();
      await gate.take();
      try {
        return await run();
      } catch (error) {
        if (isAbort(error) || signal.aborted) throw cancelled();
        const name = (error as { name?: string } | null)?.name;
        if (name === "RateLimitError") {
          await pause(Math.max(1000, (error as { retryAfterMs?: number }).retryAfterMs ?? 60_000));
          continue;
        }
        const message = error instanceof Error ? error.message : String(error);
        const busy = /maxConcurrentRequests/.test(message);
        if ((busy || /network|Failed to fetch|timed out/i.test(message)) && attempt < RETRY_DELAYS_MS.length) {
          await sleep(RETRY_DELAYS_MS[attempt]);
          continue;
        }
        throw error;
      }
    }
  };

  const download = (ref: MessageRef): Promise<Uint8Array> => withRetry(async () => {
    if (client.fetchBlobForTransfer) return client.fetchBlobForTransfer(ref.blobId, accountId, signal);
    return new Uint8Array(await (await client.fetchBlob(ref.blobId, "message.eml", "message/rfc822", accountId)).arrayBuffer());
  });

  /** The raw messages of one batch, in order; null where a message couldn't be fetched. */
  const fetchBatch = async (batch: MessageRef[]): Promise<(Uint8Array | Error)[]> => {
    const out = new Map<string, Uint8Array>();
    if (batch.length > 1 || (batch.length === 1 && batch[0].size <= BATCH_MESSAGE_MAX)) {
      try {
        const result = client.getBlobsAsBase64
          ? await withRetry(() => client.getBlobsAsBase64!(batch.map((r) => r.blobId), accountId, { sized: true }))
          : null;
        if (result) for (const [blobId, base64] of result.data) out.set(blobId, decodeBase64(base64));
      } catch (error) {
        if (isAbort(error) || signal.aborted) throw cancelled();
        // Fall back to downloads for this batch.
      }
    }
    const results: (Uint8Array | Error)[] = [];
    for (const ref of batch) {
      const got = out.get(ref.blobId);
      if (got) { results.push(got); continue; }
      try {
        results.push(await download(ref));
      } catch (error) {
        if (isAbort(error) || signal.aborted) throw cancelled();
        results.push(error instanceof Error ? error : new Error(String(error)));
      }
    }
    return results;
  };

  const batchesOf = (refs: MessageRef[]): MessageRef[][] => {
    const batches: MessageRef[][] = [];
    let current: MessageRef[] = [];
    let bytes = 0;
    for (const ref of refs) {
      if (ref.size > BATCH_MESSAGE_MAX) { batches.push([ref]); continue; }
      if (current.length > 0 && (bytes + ref.size > BATCH_BYTES || current.length >= BATCH_COUNT)) {
        batches.push(current);
        current = [];
        bytes = 0;
      }
      current.push(ref);
      bytes += ref.size;
    }
    if (current.length > 0) batches.push(current);
    return batches;
  };

  try {
    // Every folder's messages, oldest first, so the totals are known up front.
    const lists: MessageRef[][] = [];
    for (const folder of options.folders) {
      const list = await withRetry(() => client.queryEmailFields(
        { inMailbox: folder.mailboxId },
        ["blobId", "keywords", "receivedAt", "size", "subject"],
        accountId,
        Number.MAX_SAFE_INTEGER,
        { transfer: true },
      ));
      const refs = list
        .filter((e) => typeof e.blobId === "string")
        .map((e) => ({
          id: String(e.id),
          blobId: String(e.blobId),
          size: typeof e.size === "number" ? e.size : 0,
          keywords: (e.keywords as Record<string, boolean> | undefined) ?? {},
          receivedAt: typeof e.receivedAt === "string" ? e.receivedAt : undefined,
          subject: typeof e.subject === "string" ? e.subject : "",
        }))
        .reverse();
      lists.push(refs);
      progress.total += refs.length;
      report();
    }

    const single = options.folders.length === 1 && options.format === "mbox";
    const zip = single ? null : new ZipWriter((chunk) => sink.write(chunk));

    for (let f = 0; f < options.folders.length; f++) {
      const folder = options.folders[f];
      const refs = lists[f];
      const dir = folder.path.join("/");
      let stream: ZipEntryStream | null = null;
      if (zip && options.format === "mbox" && refs.length > 0) stream = await zip.openStream(`${dir}.mbox`);
      if (zip && options.format === "eml" && refs.length === 0 && dir) await zip.addDirectory(dir);
      const write = async (bytes: Uint8Array) => (stream ? stream.write(bytes) : sink.write(bytes));

      const batches = batchesOf(refs);
      let seq = 0;
      // Fetch a few batches ahead, write them in order.
      const pending: Promise<(Uint8Array | Error)[]>[] = [];
      let next = 0;
      const fill = () => {
        while (pending.length < lanes && next < batches.length) {
          const p = fetchBatch(batches[next++]);
          p.catch(() => {});
          pending.push(p);
        }
      };
      fill();
      for (let b = 0; b < batches.length; b++) {
        const results = await pending.shift()!;
        fill();
        for (let i = 0; i < results.length; i++) {
          const ref = batches[b][i];
          const raw = results[i];
          seq++;
          if (raw instanceof Error) {
            if (raw.name === "MessageNotKeptError") {
              const { reason } = raw as MessageNotKeptError;
              progress.failed++;
              progress.notKept = { ...progress.notKept, [reason]: (progress.notKept?.[reason] ?? 0) + 1 };
            } else {
              fail(ref.subject, raw.message);
            }
            report();
            continue;
          }
          if (options.format === "mbox") {
            for (const part of mboxEntry(raw, ref.keywords, ref.receivedAt)) await write(part);
          } else {
            const name = `${String(seq).padStart(5, "0")} ${safeName(ref.subject, "message", 60)}.eml`;
            await zip!.addFile(dir ? `${dir}/${name}` : name, raw, ref.receivedAt ? new Date(ref.receivedAt) : new Date());
          }
          progress.exported++;
          report();
        }
        if (signal.aborted) throw cancelled();
      }
      if (stream) await stream.close();
    }
    // A cancel with nothing left to fetch must not close and hand out the file.
    if (signal.aborted) throw cancelled();
    if (progress.exported === 0 && (progress.failed > 0 || options.noEmptyFile)) {
      // Every message failed: an empty file would only look like a backup.
      progress.leftover = leftoverOf(await sink.abort());
    } else {
      if (zip) await zip.finish();
      const file = await sink.close(signal);
      // Stopped while the file was being finished: it goes, and nobody gets to download it.
      if (signal.aborted) {
        await file.discard(true).catch(() => {});
        throw cancelled();
      }
      progress.file = file;
    }
    progress.phase = "done";
    progress.fraction = 1;
  } catch (error) {
    progress.leftover = leftoverOf(await sink.abort().catch(() => undefined));
    progress.pausedUntil = undefined;
    if (isAbort(error) || signal.aborted) {
      progress.phase = "cancelled";
    } else {
      progress.phase = "failed";
      progress.error = error instanceof Error ? error.message : String(error);
      progress.errorCode = (error as { name?: string } | null)?.name;
      progress.errorLimit = (error as { limit?: number } | null)?.limit;
    }
  }
  report(true);
  return progress;
}