Merge branch 'main' into feat/enterprise-pack

# Conflicts: # components/admin/admin-shell.tsx # lib/admin/outgoing-hosts-registry.ts # lib/admin/reachable-hosts.ts # scripts/lite/build.mjs # stores/policy-store.ts
2026-11-04 08:15:41 +00:00 · 2026-11-01 13:46:59 +01:00
parent c32c78894a eb26219b3c
commit e9b235bab2
117 changed files with 5963 additions and 80 deletions
@@ -0,0 +1,15 @@
+// Docker HEALTHCHECK: exit 0 when /api/health/live answers 200 within 4 s.
+// Liveness only, so a Stalwart outage never marks the container unhealthy.
+// The base path is baked in at build time (BULWARK_BASE_PATH in the image).
+// https://bulwarkmail.org/docs/deployment/monitoring#health
+const port = process.env.PORT || '3000';
+const listen = process.env.HOSTNAME;
+const host = !listen || listen === '0.0.0.0' || listen === '::' ? '127.0.0.1' : listen.includes(':') ? `[${listen}]` : listen;
+const base = (process.env.BULWARK_BASE_PATH || '').replace(/\/+$/, '');
+
+try {
+  const res = await fetch(`http://${host}:${port}${base}/api/health/live`, { signal: AbortSignal.timeout(4000) });
+  process.exit(res.ok ? 0 : 1);
+} catch {
+  process.exit(1);
+}
@@ -8,6 +8,11 @@
 //              (Stalwart target: the entry document, then the zip)
 //   4. verify    fail on missing files or undocumented /api references
 //
+// With BULWARK_SOURCE_MAPS=1 (CI) the browser source maps move out of out/
+// right after the build, before anything is packed, into SOURCEMAPS_OUT
+// (default sourcemaps/lite-<target>) for the error tracker. Either way the
+// export and the Stalwart zip are checked for maps last.
+//
 // Inputs (env): LITE_TARGET (static | stalwart), LITE_JMAP_SERVER_URL,
 // LITE_APP_NAME, LITE_ALLOW_CUSTOM_ENDPOINT, LITE_REMEMBER_ME, LITE_DEMO_MODE,
 // LITE_NO_THIRD_PARTY_TRAFFIC,
@@ -16,13 +21,14 @@
 // --skip-prepare (tree already pruned), --dry-run (print the plan only).
 import { spawnSync } from "node:child_process";
 import { createRequire } from "node:module";
-import { readFileSync } from "node:fs";
+import { existsSync, readFileSync } from "node:fs";
 import { join } from "node:path";
-import { makeBuildId, normalizeBasePath, parseLiteTarget, resolveRepoRoot } from "./lib.mjs";
+import { makeBuildId, normalizeBasePath, parseLiteTarget, resolveRepoRoot, STALWART_ZIP_NAME } from "./lib.mjs";
 import { runPrepare } from "./prepare.mjs";
 import { runPostbuild } from "./postbuild.mjs";
 import { verifyExport } from "./verify.mjs";
 import { buildPstWorker } from "../build-pst-worker.mjs";
+import { check as checkSourceMaps, collect as collectSourceMaps } from "../sourcemaps.mjs";

 const repoRoot = resolveRepoRoot(import.meta.url);
 const argv = process.argv.slice(2);
@@ -86,6 +92,12 @@ if (build.status !== 0) {
  process.exit(build.status ?? 1);
 }

+if (env.BULWARK_SOURCE_MAPS === "1") {
+  const out = env.SOURCEMAPS_OUT?.trim() || join(repoRoot, "sourcemaps", `lite-${target}`);
+  const moved = collectSourceMaps([join(repoRoot, "out", "_next", "static")], out);
+  console.log(`[lite] moved ${moved.length} source maps to ${out}`);
+}
+
 runPostbuild({ env });

 const { problems, locales, apiStrings, zipBytes } = verifyExport({ target });
@@ -94,4 +106,11 @@ if (problems.length > 0) {
  for (const p of problems) console.error(`  - ${p}`);
  process.exit(1);
 }
+const shipped = [join(repoRoot, "out"), join(repoRoot, STALWART_ZIP_NAME)].filter((p) => existsSync(p));
+const mapProblems = shipped.flatMap((p) => checkSourceMaps(p).map((problem) => `${p}: ${problem}`));
+if (mapProblems.length > 0) {
+  console.error(`[lite] source maps would ship (${mapProblems.length}):`);
+  for (const p of mapProblems.slice(0, 20)) console.error(`  - ${p}`);
+  process.exit(1);
+}
 console.log(`[lite] done: out/ holds ${locales.length} locales; ${apiStrings.length} documented /api strings in chunks${zipBytes ? `; zip ${(zipBytes / 1024 / 1024).toFixed(1)} MB` : ""}`);
@@ -0,0 +1,205 @@
+#!/usr/bin/env node
+// Source maps: built in CI (BULWARK_SOURCE_MAPS=1), uploaded to the error
+// tracker, never shipped. https://bulwarkmail.org/docs/deployment/monitoring
+//
+//   node scripts/sourcemaps.mjs collect --out <dir> <root>...
+//     Moves every .map under each root into <dir>/<n>/... with a copy of the
+//     .js/.css file that names it (sentry-cli uploads the pairs), and removes
+//     the `sourceMappingURL` comment from the shipped file. The chunk keeps
+//     its `debugId` comment and registration, which is all the tracker needs.
+//
+//   node scripts/sourcemaps.mjs check <dir|.zip|.tar.gz>...
+//     Fails when anything that would ship holds a .map file, an inline
+//     (data:) source map, or a `sourceMappingURL` to a file that isn't there.
+import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { basename, dirname, join, relative, resolve, sep } from "node:path";
+import { gunzipSync, inflateRawSync } from "node:zlib";
+import { fileURLToPath } from "node:url";
+
+// At the start of a line, as bundlers write it; the same text inside a
+// string literal is not a reference.
+const COMMENT = /^[ \t]*\/[/*][#@][ \t]*sourceMappingURL=([^\s*'"`]+)[ \t]*(?:\*\/)?[ \t]*$/gm;
+const SCRIPT = /\.(?:js|mjs|cjs|css)$/;
+
+function walk(dir, out = []) {
+  for (const name of readdirSync(dir)) {
+    const path = join(dir, name);
+    const stat = statSync(path, { throwIfNoEntry: false });
+    if (!stat) continue;
+    if (stat.isDirectory()) walk(path, out);
+    else out.push(path);
+  }
+  return out;
+}
+
+/** The sourceMappingURL targets named in a file's text. */
+export function mapReferences(text) {
+  return [...text.matchAll(COMMENT)].map((m) => m[1]);
+}
+
+/** Remove the sourceMappingURL comments; everything else stays as it is. */
+export function stripReferences(text) {
+  return text.replace(COMMENT, "").replace(/\n{2,}$/, "\n");
+}
+
+export function collect(roots, outDir) {
+  const moved = [];
+  mkdirSync(outDir, { recursive: true });
+  roots.forEach((given, index) => {
+    // Absolute, so the maps a file names (resolved below) match the walked paths.
+    const root = resolve(given);
+    if (!existsSync(root)) return;
+    const files = walk(root);
+    const maps = new Set(files.filter((f) => f.endsWith(".map")));
+    const target = join(outDir, String(index));
+    for (const file of files.filter((f) => SCRIPT.test(f))) {
+      const text = readFileSync(file, "utf8");
+      const refs = mapReferences(text);
+      if (refs.length === 0) continue;
+      for (const ref of refs) {
+        if (ref.startsWith("data:")) continue;
+        const map = resolve(dirname(file), decodeURIComponent(ref));
+        if (!maps.has(map)) continue;
+        for (const source of [file, map]) {
+          const dest = join(target, relative(root, source));
+          mkdirSync(dirname(dest), { recursive: true });
+          copyFileSync(source, dest);
+        }
+      }
+      writeFileSync(file, stripReferences(text));
+    }
+    // Every map goes, named by a file or not.
+    for (const map of maps) {
+      const dest = join(target, relative(root, map));
+      if (!existsSync(dest)) {
+        mkdirSync(dirname(dest), { recursive: true });
+        copyFileSync(map, dest);
+      }
+      rmSync(map);
+      moved.push(map);
+    }
+  });
+  return moved;
+}
+
+// --- archives -------------------------------------------------------------
+
+function zipEntries(buffer) {
+  const entries = [];
+  let eocd = buffer.length - 22;
+  while (eocd >= 0 && buffer.readUInt32LE(eocd) !== 0x06054b50) eocd--;
+  if (eocd < 0) throw new Error("not a zip file");
+  const count = buffer.readUInt16LE(eocd + 10);
+  let at = buffer.readUInt32LE(eocd + 16);
+  for (let i = 0; i < count; i++) {
+    const method = buffer.readUInt16LE(at + 10);
+    const size = buffer.readUInt32LE(at + 20);
+    const nameLength = buffer.readUInt16LE(at + 28);
+    const extraLength = buffer.readUInt16LE(at + 30);
+    const commentLength = buffer.readUInt16LE(at + 32);
+    const local = buffer.readUInt32LE(at + 42);
+    const name = buffer.subarray(at + 46, at + 46 + nameLength).toString("utf8");
+    const dataStart = local + 30 + buffer.readUInt16LE(local + 26) + buffer.readUInt16LE(local + 28);
+    const raw = buffer.subarray(dataStart, dataStart + size);
+    entries.push({ name, read: () => (method === 0 ? raw : inflateRawSync(raw)) });
+    at += 46 + nameLength + extraLength + commentLength;
+  }
+  return entries;
+}
+
+function tarEntries(buffer) {
+  const entries = [];
+  let at = 0;
+  let longName = null;
+  while (at + 512 <= buffer.length) {
+    const header = buffer.subarray(at, at + 512);
+    if (header.every((b) => b === 0)) break;
+    const field = (start, length) => header.subarray(start, start + length).toString("utf8").replace(/\0.*$/s, "");
+    const size = parseInt(field(124, 12).trim() || "0", 8);
+    const type = field(156, 1);
+    const prefix = field(345, 155);
+    let name = longName ?? (prefix ? `${prefix}/${field(0, 100)}` : field(0, 100));
+    longName = null;
+    const data = buffer.subarray(at + 512, at + 512 + size);
+    if (type === "L") longName = data.toString("utf8").replace(/\0.*$/s, "");
+    else if (type === "0" || type === "") entries.push({ name, read: () => data });
+    else if (type === "x") {
+      const path = /\d+ path=([^\n]+)\n/.exec(data.toString("utf8"));
+      if (path) longName = path[1];
+    }
+    at += 512 + Math.ceil(size / 512) * 512;
+  }
+  return entries;
+}
+
+function filesOf(path) {
+  if (statSync(path).isDirectory()) {
+    return walk(path).map((file) => ({ name: relative(path, file).split(sep).join("/"), read: () => readFileSync(file) }));
+  }
+  const buffer = readFileSync(path);
+  if (path.endsWith(".zip")) return zipEntries(buffer);
+  if (path.endsWith(".tar.gz") || path.endsWith(".tgz")) return tarEntries(gunzipSync(buffer));
+  if (path.endsWith(".tar")) return tarEntries(buffer);
+  throw new Error(`don't know how to read ${path}`);
+}
+
+/** Problems with what would ship at `path`; empty when it is clean. */
+export function check(path) {
+  const problems = [];
+  const entries = filesOf(path);
+  const names = new Set(entries.map((e) => e.name));
+  for (const entry of entries) {
+    if (entry.name.endsWith(".map")) {
+      problems.push(`${entry.name}: source map file`);
+      continue;
+    }
+    if (!SCRIPT.test(entry.name)) continue;
+    for (const ref of mapReferences(entry.read().toString("utf8"))) {
+      if (ref.startsWith("data:")) {
+        problems.push(`${entry.name}: inline source map`);
+        continue;
+      }
+      const target = join(dirname(entry.name), decodeURIComponent(ref)).split(sep).join("/");
+      if (!names.has(target)) problems.push(`${entry.name}: sourceMappingURL to missing ${ref}`);
+    }
+  }
+  return problems;
+}
+
+function main(argv) {
+  const [command, ...rest] = argv;
+  if (command === "collect") {
+    const at = rest.indexOf("--out");
+    if (at < 0 || !rest[at + 1]) throw new Error("collect needs --out <dir>");
+    const out = rest[at + 1];
+    const roots = rest.filter((_, i) => i !== at && i !== at + 1);
+    const moved = collect(roots, out);
+    console.log(`[sourcemaps] moved ${moved.length} maps out of ${roots.join(", ")} into ${out}`);
+    return 0;
+  }
+  if (command === "check") {
+    let failed = false;
+    for (const path of rest) {
+      if (!existsSync(path)) {
+        console.error(`[sourcemaps] ${path}: not found`);
+        failed = true;
+        continue;
+      }
+      const problems = check(path);
+      if (problems.length) {
+        failed = true;
+        console.error(`[sourcemaps] ${basename(path)}: ${problems.length} problem(s)`);
+        for (const p of problems.slice(0, 20)) console.error(`  - ${p}`);
+      } else {
+        console.log(`[sourcemaps] ${path}: no source maps`);
+      }
+    }
+    return failed ? 1 : 0;
+  }
+  console.error("usage: sourcemaps.mjs collect --out <dir> <root>... | check <path>...");
+  return 2;
+}
+
+if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
+  process.exit(main(process.argv.slice(2)));
+}
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# Upload a folder of source maps (with their chunks, as scripts/sourcemaps.mjs
+# collect leaves them) to Bugsink. Bugsink finds a map by the debug ID alone,
+# so the release name doesn't matter, but it keeps maps per project: every
+# project that gets events for these chunks needs them.
+#
+#   BUGSINK_AUTH_TOKEN  upload token; without it the upload is skipped with a notice
+#   BUGSINK_URL         default https://bugsink.internal.bulwarkmail.org/
+#   BUGSINK_PROJECTS    default "bulwark-instances bulwark-telemetry"
+#   SENTRY_CLI_VERSION  default 3.8.2
+#
+# https://bulwarkmail.org/docs/deployment/monitoring
+set -euo pipefail
+
+dir="${1:?usage: upload-sourcemaps.sh <dir>}"
+if [ -z "${BUGSINK_AUTH_TOKEN:-}" ]; then
+  echo "::notice::BUGSINK_AUTH_TOKEN is not set: source maps in ${dir} were built but not uploaded"
+  exit 0
+fi
+if [ -z "$(find "$dir" -name '*.map' -print -quit 2>/dev/null)" ]; then
+  echo "::warning::no source maps in ${dir}"
+  exit 0
+fi
+
+url="${BUGSINK_URL:-https://bugsink.internal.bulwarkmail.org/}"
+cli="@sentry/cli@${SENTRY_CLI_VERSION:-3.8.2}"
+for project in ${BUGSINK_PROJECTS:-bulwark-instances bulwark-telemetry}; do
+  SENTRY_AUTH_TOKEN="$BUGSINK_AUTH_TOKEN" npx --yes "$cli" --url "$url" \
+    sourcemaps --org bugsinkhasnoorgs --project "$project" upload "$dir"
+done