From be5a5e88ec9fb073e7a72ab434a7e747cecb4872 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 03:31:48 +0100 Subject: [PATCH] ci: upload browser source maps to Bugsink and keep every map out of what ships Every image, the standalone tarballs and the Lite zips are built with source maps (`BULWARK_SOURCE_MAPS=1`). Turbopack builds give each chunk a debug ID themselves; the webpack build in the Docker image gets them from `sentry-cli sourcemaps inject`. `scripts/sourcemaps.mjs collect` then moves every map out of the build before anything is packed, and `scripts/sourcemaps.mjs check` fails the build when a `.map` file, an inline map or a `sourceMappingURL` to a missing file would ship. - Docker images: a `sourcemaps` stage holds only the maps; the amd64 job exports it (a cache hit) and uploads it, since both platforms build identical chunks - standalone tarballs, the Lite zips and the Lite image upload their own maps and check what they ship - `scripts/upload-sourcemaps.sh` uploads to `bulwark-instances` and `bulwark-telemetry` (Bugsink keeps maps per project) and skips with a notice when the `BUGSINK_AUTH_TOKEN` secret is missing --- .dockerignore | 2 + .github/workflows/docker-publish-lite.yml | 22 ++ .github/workflows/docker-publish-releases.yml | 49 +++++ .github/workflows/docker-publish.yml | 27 +++ .github/workflows/standalone-release.yml | 18 ++ .github/workflows/static-lite.yml | 13 ++ .gitignore | 2 + Dockerfile | 19 ++ Dockerfile.lite | 14 +- lib/__tests__/sourcemaps-script.test.ts | 96 +++++++++ next.config.ts | 8 + scripts/lite/build.mjs | 23 +- scripts/sourcemaps.mjs | 203 ++++++++++++++++++ scripts/upload-sourcemaps.sh | 30 +++ 14 files changed, 522 insertions(+), 4 deletions(-) create mode 100644 lib/__tests__/sourcemaps-script.test.ts create mode 100644 scripts/sourcemaps.mjs create mode 100644 scripts/upload-sourcemaps.sh diff --git a/.dockerignore b/.dockerignore index 0942bf880..5d576f432 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,7 @@ node_modules .next out +sourcemaps lite-image .git .gitignore @@ -11,6 +12,7 @@ scripts/ !scripts/lite/ !scripts/build-plugin-sandbox.mjs !scripts/healthcheck.mjs +!scripts/sourcemaps.mjs # Built in the image by scripts/build-plugin-sandbox.mjs public/plugin-sandbox-assets/ TODO.md diff --git a/.github/workflows/docker-publish-lite.yml b/.github/workflows/docker-publish-lite.yml index 723bd5753..3912bb270 100644 --- a/.github/workflows/docker-publish-lite.yml +++ b/.github/workflows/docker-publish-lite.yml @@ -178,9 +178,31 @@ jobs: labels: ${{ steps.meta.outputs.labels }} build-args: | GIT_COMMIT=${{ github.sha }} + BULWARK_SOURCE_MAPS=1 cache-from: type=gha,scope=lite cache-to: type=gha,mode=max,scope=lite + # The same build again, a cache hit, for the stage that holds only the + # source maps; the image never has them. One export for both platforms. + # https://bulwarkmail.org/docs/admin/monitoring + - name: Export source maps + uses: docker/build-push-action@v7 + with: + context: . + file: Dockerfile.lite + platforms: linux/amd64 + target: sourcemaps + build-args: | + GIT_COMMIT=${{ github.sha }} + BULWARK_SOURCE_MAPS=1 + outputs: type=local,dest=sourcemaps + cache-from: type=gha,scope=lite + + - name: Upload source maps to Bugsink + env: + BUGSINK_AUTH_TOKEN: ${{ secrets.BUGSINK_AUTH_TOKEN }} + run: bash scripts/upload-sourcemaps.sh sourcemaps + - name: Inspect image run: docker buildx imagetools inspect ${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }} diff --git a/.github/workflows/docker-publish-releases.yml b/.github/workflows/docker-publish-releases.yml index 62d7d56c8..e906e0c57 100644 --- a/.github/workflows/docker-publish-releases.yml +++ b/.github/workflows/docker-publish-releases.yml @@ -59,10 +59,34 @@ jobs: labels: ${{ steps.meta.outputs.labels }} build-args: | GIT_COMMIT=${{ github.sha }} + BULWARK_SOURCE_MAPS=1 outputs: type=image,name=${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true cache-from: type=gha,scope=${{ matrix.platform }} cache-to: type=gha,mode=max,scope=${{ matrix.platform }} + # The same build again, a cache hit on this builder, for the stage that + # holds only the source maps (Dockerfile, "sourcemaps"); the image never + # has them. Both platforms build identical chunks, so amd64 uploads. + # https://bulwarkmail.org/docs/admin/monitoring + - name: Export source maps + if: matrix.platform == 'linux/amd64' + uses: docker/build-push-action@v7 + with: + context: . + platforms: ${{ matrix.platform }} + target: sourcemaps + build-args: | + GIT_COMMIT=${{ github.sha }} + BULWARK_SOURCE_MAPS=1 + outputs: type=local,dest=sourcemaps + cache-from: type=gha,scope=${{ matrix.platform }} + + - name: Upload source maps to Bugsink + if: matrix.platform == 'linux/amd64' + env: + BUGSINK_AUTH_TOKEN: ${{ secrets.BUGSINK_AUTH_TOKEN }} + run: bash scripts/upload-sourcemaps.sh sourcemaps + - name: Export digest run: | mkdir -p /tmp/digests @@ -184,10 +208,35 @@ jobs: build-args: | GIT_COMMIT=${{ github.sha }} NEXT_PUBLIC_LOCALE_PREFIX=always + BULWARK_SOURCE_MAPS=1 outputs: type=image,name=${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true cache-from: type=gha,scope=always-${{ matrix.platform }} cache-to: type=gha,mode=max,scope=always-${{ matrix.platform }} + # The same build again, a cache hit on this builder, for the stage that + # holds only the source maps (Dockerfile, "sourcemaps"); the image never + # has them. Both platforms build identical chunks, so amd64 uploads. + # https://bulwarkmail.org/docs/admin/monitoring + - name: Export source maps + if: matrix.platform == 'linux/amd64' + uses: docker/build-push-action@v7 + with: + context: . + platforms: ${{ matrix.platform }} + target: sourcemaps + build-args: | + GIT_COMMIT=${{ github.sha }} + NEXT_PUBLIC_LOCALE_PREFIX=always + BULWARK_SOURCE_MAPS=1 + outputs: type=local,dest=sourcemaps + cache-from: type=gha,scope=always-${{ matrix.platform }} + + - name: Upload source maps to Bugsink + if: matrix.platform == 'linux/amd64' + env: + BUGSINK_AUTH_TOKEN: ${{ secrets.BUGSINK_AUTH_TOKEN }} + run: bash scripts/upload-sourcemaps.sh sourcemaps + - name: Export digest run: | mkdir -p /tmp/digests-always diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 96224784a..d10f6c423 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -18,6 +18,9 @@ on: - "public/**" - "package.json" - "package-lock.json" + - "scripts/healthcheck.mjs" + - "scripts/sourcemaps.mjs" + - "scripts/upload-sourcemaps.sh" - ".github/workflows/docker-publish.yml" workflow_dispatch: @@ -89,10 +92,34 @@ jobs: labels: ${{ steps.meta.outputs.labels }} build-args: | GIT_COMMIT=${{ github.sha }} + BULWARK_SOURCE_MAPS=1 outputs: type=image,name=${{ needs.prepare.outputs.image_name }},push-by-digest=true,name-canonical=true,push=true cache-from: type=gha,scope=${{ matrix.platform }} cache-to: type=gha,mode=max,scope=${{ matrix.platform }} + # The same build again, a cache hit on this builder, for the stage that + # holds only the source maps (Dockerfile, "sourcemaps"); the image never + # has them. Both platforms build identical chunks, so amd64 uploads. + # https://bulwarkmail.org/docs/admin/monitoring + - name: Export source maps + if: matrix.platform == 'linux/amd64' + uses: docker/build-push-action@v7 + with: + context: . + platforms: ${{ matrix.platform }} + target: sourcemaps + build-args: | + GIT_COMMIT=${{ github.sha }} + BULWARK_SOURCE_MAPS=1 + outputs: type=local,dest=sourcemaps + cache-from: type=gha,scope=${{ matrix.platform }} + + - name: Upload source maps to Bugsink + if: matrix.platform == 'linux/amd64' + env: + BUGSINK_AUTH_TOKEN: ${{ secrets.BUGSINK_AUTH_TOKEN }} + run: bash scripts/upload-sourcemaps.sh sourcemaps + - name: Export digest run: | mkdir -p /tmp/digests diff --git a/.github/workflows/standalone-release.yml b/.github/workflows/standalone-release.yml index 50ef82024..f50ec11b2 100644 --- a/.github/workflows/standalone-release.yml +++ b/.github/workflows/standalone-release.yml @@ -40,9 +40,24 @@ jobs: - name: Install dependencies run: npm ci + # With browser and server source maps; Turbopack gives every chunk a + # debug ID. The maps go to Bugsink and never into the tarball. + # https://bulwarkmail.org/docs/admin/monitoring - name: Build standalone + env: + BULWARK_SOURCE_MAPS: "1" run: npm run build + - name: Move source maps out of the build + run: node scripts/sourcemaps.mjs collect --out sourcemaps .next/static .next/standalone + + # Turbopack chunks were not checked for being the same on both + # platforms, so each one uploads its own. + - name: Upload source maps to Bugsink + env: + BUGSINK_AUTH_TOKEN: ${{ secrets.BUGSINK_AUTH_TOKEN }} + run: bash scripts/upload-sourcemaps.sh sourcemaps + - name: Package tarball env: REF_NAME: ${{ github.ref_name }} @@ -59,6 +74,9 @@ jobs: tar -czf "$TARBALL" bulwark-standalone/ echo "TARBALL=$TARBALL" >> "$GITHUB_ENV" + - name: Check the tarball holds no source maps + run: node scripts/sourcemaps.mjs check "$TARBALL" + - name: Upload artifact uses: actions/upload-artifact@v7 with: diff --git a/.github/workflows/static-lite.yml b/.github/workflows/static-lite.yml index d4528ec24..1c23a1e4a 100644 --- a/.github/workflows/static-lite.yml +++ b/.github/workflows/static-lite.yml @@ -79,8 +79,18 @@ jobs: LITE_APP_NAME: ${{ inputs.app_name }} LITE_LOCALES: ${{ inputs.locales }} LITE_DEMO_MODE: ${{ inputs.demo }} + # Browser source maps for Bugsink; build:lite moves them out of out/ + # before anything is packed. https://bulwarkmail.org/docs/admin/monitoring + BULWARK_SOURCE_MAPS: "1" + SOURCEMAPS_OUT: sourcemaps run: npm run build:lite + # Each target is its own build with its own chunks, so each uploads. + - name: Upload source maps to Bugsink + env: + BUGSINK_AUTH_TOKEN: ${{ secrets.BUGSINK_AUTH_TOKEN }} + run: bash scripts/upload-sourcemaps.sh sourcemaps + # Every host literal in the bundle must be a placeholder, the configured # JMAP server, a built-in host the no-third-party switch turns off, or a # string listed as never requested (#903). @@ -107,6 +117,9 @@ jobs: echo "ZIP=${ZIP}" >> "$GITHUB_ENV" ls -la "${ZIP}" + - name: Check the zip holds no source maps + run: node scripts/sourcemaps.mjs check "$ZIP" + - name: Upload artifact uses: actions/upload-artifact@v7 with: diff --git a/.gitignore b/.gitignore index a67d67b1c..c90b34d31 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,8 @@ # next.js /.next/ /out/ +# Source maps CI collects for the error tracker (scripts/sourcemaps.mjs) +/sourcemaps/ /lite-image/ # scripts/build-plugin-sandbox.mjs /public/plugin-sandbox-assets/ diff --git a/Dockerfile b/Dockerfile index fe9f6f986..d6ed0a17d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,10 @@ ENV NEXT_PUBLIC_DEFAULT_LOCALE=$NEXT_PUBLIC_DEFAULT_LOCALE # `git rev-parse` inside the build can't find it - CI must pass it in. ARG GIT_COMMIT=unknown ENV GIT_COMMIT=$GIT_COMMIT +# Browser source maps for the error tracker: CI passes 1, a plain build makes +# none (next.config.ts reads it). They never reach the image, see below. +ARG BULWARK_SOURCE_MAPS=0 +ENV BULWARK_SOURCE_MAPS=$BULWARK_SOURCE_MAPS # The mock JMAP server (DEV_MOCK_JMAP) accepts any password and is for # `npm run dev` only; images ship without it. # `next build` type-checks every .ts file, and some unit tests import the @@ -33,6 +37,21 @@ RUN rm -rf "app/api/dev-jmap" && \ node scripts/build-plugin-sandbox.mjs && \ npx next build --webpack +# webpack has no debug IDs of its own, so sentry-cli adds them to the +# chunks; then every map moves to /sourcemaps (exported with +# `--target sourcemaps`) and the check fails the build if one would ship. +# https://bulwarkmail.org/docs/admin/monitoring +ARG SENTRY_CLI_VERSION=3.8.2 +RUN if [ "$BULWARK_SOURCE_MAPS" = "1" ]; then \ + npx --yes "@sentry/cli@${SENTRY_CLI_VERSION}" sourcemaps inject .next/static; \ + fi && \ + node scripts/sourcemaps.mjs collect --out /sourcemaps .next/static .next/standalone && \ + node scripts/sourcemaps.mjs check .next/static .next/standalone public + +# Only the maps, for CI to upload: never part of the image. +FROM scratch AS sourcemaps +COPY --from=builder /sourcemaps / + FROM node:24-alpine AS runner LABEL org.opencontainers.image.title="Bulwark Webmail" diff --git a/Dockerfile.lite b/Dockerfile.lite index 8a71f6650..94ab47a9d 100644 --- a/Dockerfile.lite +++ b/Dockerfile.lite @@ -29,8 +29,18 @@ ARG NEXT_PUBLIC_DEFAULT_LOCALE= # Commit SHA shown in the About screen. .dockerignore excludes .git, so CI # must pass it in. ARG GIT_COMMIT=unknown -RUN LITE_TARGET=static NEXT_PUBLIC_BASE_PATH= npm run build:lite && \ - node scripts/lite/container.mjs /image +# Browser source maps for the error tracker: CI passes 1. build:lite moves +# them to /sourcemaps (exported with `--target sourcemaps`) and fails if one +# would ship. https://bulwarkmail.org/docs/admin/monitoring +ARG BULWARK_SOURCE_MAPS=0 +RUN mkdir -p /sourcemaps && \ + LITE_TARGET=static NEXT_PUBLIC_BASE_PATH= SOURCEMAPS_OUT=/sourcemaps npm run build:lite && \ + node scripts/lite/container.mjs /image && \ + node scripts/sourcemaps.mjs check /image/html + +# Only the maps, for CI to upload: never part of the image. +FROM scratch AS sourcemaps +COPY --from=builder /sourcemaps / FROM nginxinc/nginx-unprivileged:1.30-alpine-slim diff --git a/lib/__tests__/sourcemaps-script.test.ts b/lib/__tests__/sourcemaps-script.test.ts new file mode 100644 index 000000000..e9b8ac385 --- /dev/null +++ b/lib/__tests__/sourcemaps-script.test.ts @@ -0,0 +1,96 @@ +// @vitest-environment node +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { gzipSync } from 'node:zlib'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +// @ts-expect-error plain .mjs without types +import { check, collect, mapReferences } from '../../scripts/sourcemaps.mjs'; +// @ts-expect-error plain .mjs without types +import { buildZip } from '../../scripts/lite/lib.mjs'; + +// CI builds source maps for the error tracker; none may ship (images, the +// standalone tarball, Lite zips). + +let dir: string; +const put = (rel: string, text: string) => { + const file = path.join(dir, rel); + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync(file, text); +}; + +beforeEach(() => { dir = mkdtempSync(path.join(tmpdir(), 'bulwark-maps-')); }); +afterEach(() => { rmSync(dir, { recursive: true, force: true }); }); + +function tar(files: Record): Buffer { + const blocks: Buffer[] = []; + for (const [name, text] of Object.entries(files)) { + const data = Buffer.from(text); + const header = Buffer.alloc(512); + header.write(name, 0); + header.write('0000644\0', 100); + header.write(data.length.toString(8).padStart(11, '0') + '\0', 124); + header.write('0', 156); + header.write('ustar\0', 257); + blocks.push(header, data, Buffer.alloc((512 - (data.length % 512)) % 512)); + } + blocks.push(Buffer.alloc(1024)); + return Buffer.concat(blocks); +} + +describe('collect', () => { + it('moves maps out with the chunks that name them, and strips the comment from what ships', () => { + put('static/chunks/a.js', 'console.log(1);\n//# debugId=bc67f7db-7f3b-7be2-6f94-966cd7825cc2\n//# sourceMappingURL=a1b2.js.map\n'); + put('static/chunks/a1b2.js.map', '{"version":3}'); + put('static/css/x.css', 'a{}\n/*# sourceMappingURL=x.css.map */\n'); + put('static/css/x.css.map', '{"version":3}'); + put('static/chunks/orphan.js.map', '{"version":3}'); + const out = path.join(dir, 'maps'); + + const moved = collect([path.join(dir, 'static')], out); + + expect(moved).toHaveLength(3); + expect(existsSync(path.join(dir, 'static/chunks/a1b2.js.map'))).toBe(false); + const shipped = readFileSync(path.join(dir, 'static/chunks/a.js'), 'utf8'); + expect(mapReferences(shipped)).toEqual([]); + expect(shipped).toContain('//# debugId=bc67f7db-7f3b-7be2-6f94-966cd7825cc2'); + expect(readFileSync(path.join(out, '0/chunks/a.js'), 'utf8')).toContain('sourceMappingURL=a1b2.js.map'); + expect(existsSync(path.join(out, '0/chunks/a1b2.js.map'))).toBe(true); + expect(existsSync(path.join(out, '0/chunks/orphan.js.map'))).toBe(true); + expect(check(path.join(dir, 'static'))).toEqual([]); + }); +}); + +describe('check', () => { + it('finds map files, dangling references and inline maps in a folder', () => { + put('a/x.js.map', '{}'); + put('a/y.js', 'x\n//# sourceMappingURL=gone.js.map\n'); + put('a/z.js', 'x\n//# sourceMappingURL=data:application/json;base64,e30=\n'); + put('a/ok.js', 'const s = "//# sourceMappingURL=in-a-string.map";\n'); + expect(check(path.join(dir, 'a')).sort()).toEqual([ + 'x.js.map: source map file', + 'y.js: sourceMappingURL to missing gone.js.map', + 'z.js: inline source map', + ]); + }); + + it('reads zip archives', () => { + const zip = path.join(dir, 'lite.zip'); + writeFileSync(zip, buildZip([ + { name: '_next/static/chunks/a.js', data: Buffer.from('x\n//# sourceMappingURL=a.js.map\n') }, + { name: '_next/static/chunks/a.js.map', data: Buffer.from('{}') }, + ])); + expect(check(zip)).toEqual(['_next/static/chunks/a.js.map: source map file']); + writeFileSync(zip, buildZip([{ name: 'index.html', data: Buffer.from('') }])); + expect(check(zip)).toEqual([]); + }); + + it('reads gzipped tarballs', () => { + const tgz = path.join(dir, 'standalone.tar.gz'); + writeFileSync(tgz, gzipSync(tar({ + 'bulwark-standalone/.next/server/chunks/1.js': 'x\n//# sourceMappingURL=1.js.map\n', + 'bulwark-standalone/.next/server/chunks/1.js.map': '{}', + }))); + expect(check(tgz)).toEqual(['bulwark-standalone/.next/server/chunks/1.js.map: source map file']); + }); +}); diff --git a/next.config.ts b/next.config.ts index 2435bd968..b238b59c6 100644 --- a/next.config.ts +++ b/next.config.ts @@ -47,8 +47,15 @@ if (isLite && (process.env.NEXT_PUBLIC_LOCALE_PREFIX ?? "never") !== "always") { throw new Error("Bulwark Lite requires NEXT_PUBLIC_LOCALE_PREFIX=always (use `npm run build:lite`)"); } +// Browser source maps, for resolving the stack traces in error reports. +// Only CI turns them on (BULWARK_SOURCE_MAPS=1): it uploads them to the +// error tracker and removes every .map file before anything ships +// (scripts/sourcemaps.mjs). A plain build makes none. +const sourceMaps = process.env.BULWARK_SOURCE_MAPS === "1"; + const nextConfig: NextConfig = { output: isLite ? "export" : "standalone", + productionBrowserSourceMaps: sourceMaps, // No `X-Powered-By: Next.js`; headers() below sends // `X-Powered-By: bulwarkmail.org` instead, without a version. poweredByHeader: false, @@ -132,6 +139,7 @@ const nextConfig: NextConfig = { }, turbopack: { root: process.env.NEXT_TURBOPACK_ROOT || import.meta.dirname, + ...(sourceMaps ? { debugIds: true } : {}), }, env: { NEXT_PUBLIC_GIT_COMMIT: gitCommitHash, diff --git a/scripts/lite/build.mjs b/scripts/lite/build.mjs index cfa36568e..08430e59d 100644 --- a/scripts/lite/build.mjs +++ b/scripts/lite/build.mjs @@ -8,6 +8,11 @@ // (Stalwart target: the entry document, then the zip) // 4. verify fail on missing files or undocumented /api references // +// With BULWARK_SOURCE_MAPS=1 (CI) the browser source maps move out of out/ +// right after the build, before anything is packed, into SOURCEMAPS_OUT +// (default sourcemaps/lite-) for the error tracker. Either way the +// export and the Stalwart zip are checked for maps last. +// // Inputs (env): LITE_TARGET (static | stalwart), LITE_JMAP_SERVER_URL, // LITE_APP_NAME, LITE_ALLOW_CUSTOM_ENDPOINT, LITE_REMEMBER_ME, LITE_DEMO_MODE, // LITE_NO_THIRD_PARTY_TRAFFIC, @@ -16,12 +21,13 @@ // --skip-prepare (tree already pruned), --dry-run (print the plan only). import { spawnSync } from "node:child_process"; import { createRequire } from "node:module"; -import { readFileSync } from "node:fs"; +import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; -import { makeBuildId, normalizeBasePath, parseLiteTarget, resolveRepoRoot } from "./lib.mjs"; +import { makeBuildId, normalizeBasePath, parseLiteTarget, resolveRepoRoot, STALWART_ZIP_NAME } from "./lib.mjs"; import { runPrepare } from "./prepare.mjs"; import { runPostbuild } from "./postbuild.mjs"; import { verifyExport } from "./verify.mjs"; +import { check as checkSourceMaps, collect as collectSourceMaps } from "../sourcemaps.mjs"; const repoRoot = resolveRepoRoot(import.meta.url); const argv = process.argv.slice(2); @@ -81,6 +87,12 @@ if (build.status !== 0) { process.exit(build.status ?? 1); } +if (env.BULWARK_SOURCE_MAPS === "1") { + const out = env.SOURCEMAPS_OUT?.trim() || join(repoRoot, "sourcemaps", `lite-${target}`); + const moved = collectSourceMaps([join(repoRoot, "out", "_next", "static")], out); + console.log(`[lite] moved ${moved.length} source maps to ${out}`); +} + runPostbuild({ env }); const { problems, locales, apiStrings, zipBytes } = verifyExport({ target }); @@ -89,4 +101,11 @@ if (problems.length > 0) { for (const p of problems) console.error(` - ${p}`); process.exit(1); } +const shipped = [join(repoRoot, "out"), join(repoRoot, STALWART_ZIP_NAME)].filter((p) => existsSync(p)); +const mapProblems = shipped.flatMap((p) => checkSourceMaps(p).map((problem) => `${p}: ${problem}`)); +if (mapProblems.length > 0) { + console.error(`[lite] source maps would ship (${mapProblems.length}):`); + for (const p of mapProblems.slice(0, 20)) console.error(` - ${p}`); + process.exit(1); +} console.log(`[lite] done: out/ holds ${locales.length} locales; ${apiStrings.length} documented /api strings in chunks${zipBytes ? `; zip ${(zipBytes / 1024 / 1024).toFixed(1)} MB` : ""}`); diff --git a/scripts/sourcemaps.mjs b/scripts/sourcemaps.mjs new file mode 100644 index 000000000..4303506d3 --- /dev/null +++ b/scripts/sourcemaps.mjs @@ -0,0 +1,203 @@ +#!/usr/bin/env node +// Source maps: built in CI (BULWARK_SOURCE_MAPS=1), uploaded to the error +// tracker, never shipped. https://bulwarkmail.org/docs/admin/monitoring +// +// node scripts/sourcemaps.mjs collect --out ... +// Moves every .map under each root into //... with a copy of the +// .js/.css file that names it (sentry-cli uploads the pairs), and removes +// the `sourceMappingURL` comment from the shipped file. The chunk keeps +// its `debugId` comment and registration, which is all the tracker needs. +// +// node scripts/sourcemaps.mjs check ... +// Fails when anything that would ship holds a .map file, an inline +// (data:) source map, or a `sourceMappingURL` to a file that isn't there. +import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { basename, dirname, join, relative, resolve, sep } from "node:path"; +import { gunzipSync, inflateRawSync } from "node:zlib"; +import { fileURLToPath } from "node:url"; + +// At the start of a line, as bundlers write it; the same text inside a +// string literal is not a reference. +const COMMENT = /^[ \t]*\/[/*][#@][ \t]*sourceMappingURL=([^\s*'"`]+)[ \t]*(?:\*\/)?[ \t]*$/gm; +const SCRIPT = /\.(?:js|mjs|cjs|css)$/; + +function walk(dir, out = []) { + for (const name of readdirSync(dir)) { + const path = join(dir, name); + const stat = statSync(path, { throwIfNoEntry: false }); + if (!stat) continue; + if (stat.isDirectory()) walk(path, out); + else out.push(path); + } + return out; +} + +/** The sourceMappingURL targets named in a file's text. */ +export function mapReferences(text) { + return [...text.matchAll(COMMENT)].map((m) => m[1]); +} + +/** Remove the sourceMappingURL comments; everything else stays as it is. */ +export function stripReferences(text) { + return text.replace(COMMENT, "").replace(/\n{2,}$/, "\n"); +} + +export function collect(roots, outDir) { + const moved = []; + mkdirSync(outDir, { recursive: true }); + roots.forEach((root, index) => { + if (!existsSync(root)) return; + const files = walk(root); + const maps = new Set(files.filter((f) => f.endsWith(".map"))); + const target = join(outDir, String(index)); + for (const file of files.filter((f) => SCRIPT.test(f))) { + const text = readFileSync(file, "utf8"); + const refs = mapReferences(text); + if (refs.length === 0) continue; + for (const ref of refs) { + if (ref.startsWith("data:")) continue; + const map = resolve(dirname(file), decodeURIComponent(ref)); + if (!maps.has(map)) continue; + for (const source of [file, map]) { + const dest = join(target, relative(root, source)); + mkdirSync(dirname(dest), { recursive: true }); + copyFileSync(source, dest); + } + } + writeFileSync(file, stripReferences(text)); + } + // Every map goes, named by a file or not. + for (const map of maps) { + const dest = join(target, relative(root, map)); + if (!existsSync(dest)) { + mkdirSync(dirname(dest), { recursive: true }); + copyFileSync(map, dest); + } + rmSync(map); + moved.push(map); + } + }); + return moved; +} + +// --- archives ------------------------------------------------------------- + +function zipEntries(buffer) { + const entries = []; + let eocd = buffer.length - 22; + while (eocd >= 0 && buffer.readUInt32LE(eocd) !== 0x06054b50) eocd--; + if (eocd < 0) throw new Error("not a zip file"); + const count = buffer.readUInt16LE(eocd + 10); + let at = buffer.readUInt32LE(eocd + 16); + for (let i = 0; i < count; i++) { + const method = buffer.readUInt16LE(at + 10); + const size = buffer.readUInt32LE(at + 20); + const nameLength = buffer.readUInt16LE(at + 28); + const extraLength = buffer.readUInt16LE(at + 30); + const commentLength = buffer.readUInt16LE(at + 32); + const local = buffer.readUInt32LE(at + 42); + const name = buffer.subarray(at + 46, at + 46 + nameLength).toString("utf8"); + const dataStart = local + 30 + buffer.readUInt16LE(local + 26) + buffer.readUInt16LE(local + 28); + const raw = buffer.subarray(dataStart, dataStart + size); + entries.push({ name, read: () => (method === 0 ? raw : inflateRawSync(raw)) }); + at += 46 + nameLength + extraLength + commentLength; + } + return entries; +} + +function tarEntries(buffer) { + const entries = []; + let at = 0; + let longName = null; + while (at + 512 <= buffer.length) { + const header = buffer.subarray(at, at + 512); + if (header.every((b) => b === 0)) break; + const field = (start, length) => header.subarray(start, start + length).toString("utf8").replace(/\0.*$/s, ""); + const size = parseInt(field(124, 12).trim() || "0", 8); + const type = field(156, 1); + const prefix = field(345, 155); + let name = longName ?? (prefix ? `${prefix}/${field(0, 100)}` : field(0, 100)); + longName = null; + const data = buffer.subarray(at + 512, at + 512 + size); + if (type === "L") longName = data.toString("utf8").replace(/\0.*$/s, ""); + else if (type === "0" || type === "") entries.push({ name, read: () => data }); + else if (type === "x") { + const path = /\d+ path=([^\n]+)\n/.exec(data.toString("utf8")); + if (path) longName = path[1]; + } + at += 512 + Math.ceil(size / 512) * 512; + } + return entries; +} + +function filesOf(path) { + if (statSync(path).isDirectory()) { + return walk(path).map((file) => ({ name: relative(path, file).split(sep).join("/"), read: () => readFileSync(file) })); + } + const buffer = readFileSync(path); + if (path.endsWith(".zip")) return zipEntries(buffer); + if (path.endsWith(".tar.gz") || path.endsWith(".tgz")) return tarEntries(gunzipSync(buffer)); + if (path.endsWith(".tar")) return tarEntries(buffer); + throw new Error(`don't know how to read ${path}`); +} + +/** Problems with what would ship at `path`; empty when it is clean. */ +export function check(path) { + const problems = []; + const entries = filesOf(path); + const names = new Set(entries.map((e) => e.name)); + for (const entry of entries) { + if (entry.name.endsWith(".map")) { + problems.push(`${entry.name}: source map file`); + continue; + } + if (!SCRIPT.test(entry.name)) continue; + for (const ref of mapReferences(entry.read().toString("utf8"))) { + if (ref.startsWith("data:")) { + problems.push(`${entry.name}: inline source map`); + continue; + } + const target = join(dirname(entry.name), decodeURIComponent(ref)).split(sep).join("/"); + if (!names.has(target)) problems.push(`${entry.name}: sourceMappingURL to missing ${ref}`); + } + } + return problems; +} + +function main(argv) { + const [command, ...rest] = argv; + if (command === "collect") { + const at = rest.indexOf("--out"); + if (at < 0 || !rest[at + 1]) throw new Error("collect needs --out "); + const out = rest[at + 1]; + const roots = rest.filter((_, i) => i !== at && i !== at + 1); + const moved = collect(roots, out); + console.log(`[sourcemaps] moved ${moved.length} maps out of ${roots.join(", ")} into ${out}`); + return 0; + } + if (command === "check") { + let failed = false; + for (const path of rest) { + if (!existsSync(path)) { + console.error(`[sourcemaps] ${path}: not found`); + failed = true; + continue; + } + const problems = check(path); + if (problems.length) { + failed = true; + console.error(`[sourcemaps] ${basename(path)}: ${problems.length} problem(s)`); + for (const p of problems.slice(0, 20)) console.error(` - ${p}`); + } else { + console.log(`[sourcemaps] ${path}: no source maps`); + } + } + return failed ? 1 : 0; + } + console.error("usage: sourcemaps.mjs collect --out ... | check ..."); + return 2; +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + process.exit(main(process.argv.slice(2))); +} diff --git a/scripts/upload-sourcemaps.sh b/scripts/upload-sourcemaps.sh new file mode 100644 index 000000000..c191e7919 --- /dev/null +++ b/scripts/upload-sourcemaps.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Upload a folder of source maps (with their chunks, as scripts/sourcemaps.mjs +# collect leaves them) to Bugsink. Bugsink finds a map by the debug ID alone, +# so the release name doesn't matter, but it keeps maps per project: every +# project that gets events for these chunks needs them. +# +# BUGSINK_AUTH_TOKEN upload token; without it the upload is skipped with a notice +# BUGSINK_URL default https://bugsink.internal.bulwarkmail.org/ +# BUGSINK_PROJECTS default "bulwark-instances bulwark-telemetry" +# SENTRY_CLI_VERSION default 3.8.2 +# +# https://bulwarkmail.org/docs/admin/monitoring +set -euo pipefail + +dir="${1:?usage: upload-sourcemaps.sh }" +if [ -z "${BUGSINK_AUTH_TOKEN:-}" ]; then + echo "::notice::BUGSINK_AUTH_TOKEN is not set: source maps in ${dir} were built but not uploaded" + exit 0 +fi +if [ -z "$(find "$dir" -name '*.map' -print -quit 2>/dev/null)" ]; then + echo "::warning::no source maps in ${dir}" + exit 0 +fi + +url="${BUGSINK_URL:-https://bugsink.internal.bulwarkmail.org/}" +cli="@sentry/cli@${SENTRY_CLI_VERSION:-3.8.2}" +for project in ${BUGSINK_PROJECTS:-bulwark-instances bulwark-telemetry}; do + SENTRY_AUTH_TOKEN="$BUGSINK_AUTH_TOKEN" npx --yes "$cli" --url "$url" \ + sourcemaps --org bugsinkhasnoorgs --project "$project" upload "$dir" +done