diff --git a/.dockerignore b/.dockerignore index 68da62e5b..5d576f432 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,7 @@ node_modules .next out +sourcemaps lite-image .git .gitignore @@ -10,6 +11,8 @@ lite-image scripts/ !scripts/lite/ !scripts/build-plugin-sandbox.mjs +!scripts/healthcheck.mjs +!scripts/sourcemaps.mjs # Built in the image by scripts/build-plugin-sandbox.mjs public/plugin-sandbox-assets/ TODO.md diff --git a/.github/workflows/docker-publish-lite.yml b/.github/workflows/docker-publish-lite.yml index 723bd5753..0a22c5dc5 100644 --- a/.github/workflows/docker-publish-lite.yml +++ b/.github/workflows/docker-publish-lite.yml @@ -181,6 +181,11 @@ jobs: cache-from: type=gha,scope=lite cache-to: type=gha,mode=max,scope=lite + # No source maps for this image yet: with BULWARK_SOURCE_MAPS=1 the + # multi-platform build stalled on the runner after the Lite build step + # (twice, 2026-11-01), though it finishes locally. The static Lite zips + # (static-lite.yml) still upload theirs. + - name: Inspect image run: docker buildx imagetools inspect ${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }} diff --git a/.github/workflows/docker-publish-releases.yml b/.github/workflows/docker-publish-releases.yml index 62d7d56c8..86bf50d18 100644 --- a/.github/workflows/docker-publish-releases.yml +++ b/.github/workflows/docker-publish-releases.yml @@ -59,10 +59,34 @@ jobs: labels: ${{ steps.meta.outputs.labels }} build-args: | GIT_COMMIT=${{ github.sha }} + BULWARK_SOURCE_MAPS=1 outputs: type=image,name=${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true cache-from: type=gha,scope=${{ matrix.platform }} cache-to: type=gha,mode=max,scope=${{ matrix.platform }} + # The same build again, a cache hit on this builder, for the stage that + # holds only the source maps (Dockerfile, "sourcemaps"); the image never + # has them. Both platforms build identical chunks, so amd64 uploads. + # https://bulwarkmail.org/docs/deployment/monitoring + - name: Export source maps + if: matrix.platform == 'linux/amd64' + uses: docker/build-push-action@v7 + with: + context: . + platforms: ${{ matrix.platform }} + target: sourcemaps + build-args: | + GIT_COMMIT=${{ github.sha }} + BULWARK_SOURCE_MAPS=1 + outputs: type=local,dest=sourcemaps + cache-from: type=gha,scope=${{ matrix.platform }} + + - name: Upload source maps to Bugsink + if: matrix.platform == 'linux/amd64' + env: + BUGSINK_AUTH_TOKEN: ${{ secrets.BUGSINK_AUTH_TOKEN }} + run: bash scripts/upload-sourcemaps.sh sourcemaps + - name: Export digest run: | mkdir -p /tmp/digests @@ -184,10 +208,35 @@ jobs: build-args: | GIT_COMMIT=${{ github.sha }} NEXT_PUBLIC_LOCALE_PREFIX=always + BULWARK_SOURCE_MAPS=1 outputs: type=image,name=${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true cache-from: type=gha,scope=always-${{ matrix.platform }} cache-to: type=gha,mode=max,scope=always-${{ matrix.platform }} + # The same build again, a cache hit on this builder, for the stage that + # holds only the source maps (Dockerfile, "sourcemaps"); the image never + # has them. Both platforms build identical chunks, so amd64 uploads. + # https://bulwarkmail.org/docs/deployment/monitoring + - name: Export source maps + if: matrix.platform == 'linux/amd64' + uses: docker/build-push-action@v7 + with: + context: . + platforms: ${{ matrix.platform }} + target: sourcemaps + build-args: | + GIT_COMMIT=${{ github.sha }} + NEXT_PUBLIC_LOCALE_PREFIX=always + BULWARK_SOURCE_MAPS=1 + outputs: type=local,dest=sourcemaps + cache-from: type=gha,scope=always-${{ matrix.platform }} + + - name: Upload source maps to Bugsink + if: matrix.platform == 'linux/amd64' + env: + BUGSINK_AUTH_TOKEN: ${{ secrets.BUGSINK_AUTH_TOKEN }} + run: bash scripts/upload-sourcemaps.sh sourcemaps + - name: Export digest run: | mkdir -p /tmp/digests-always diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 96224784a..f5b6a70f5 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -18,6 +18,9 @@ on: - "public/**" - "package.json" - "package-lock.json" + - "scripts/healthcheck.mjs" + - "scripts/sourcemaps.mjs" + - "scripts/upload-sourcemaps.sh" - ".github/workflows/docker-publish.yml" workflow_dispatch: @@ -89,10 +92,34 @@ jobs: labels: ${{ steps.meta.outputs.labels }} build-args: | GIT_COMMIT=${{ github.sha }} + BULWARK_SOURCE_MAPS=1 outputs: type=image,name=${{ needs.prepare.outputs.image_name }},push-by-digest=true,name-canonical=true,push=true cache-from: type=gha,scope=${{ matrix.platform }} cache-to: type=gha,mode=max,scope=${{ matrix.platform }} + # The same build again, a cache hit on this builder, for the stage that + # holds only the source maps (Dockerfile, "sourcemaps"); the image never + # has them. Both platforms build identical chunks, so amd64 uploads. + # https://bulwarkmail.org/docs/deployment/monitoring + - name: Export source maps + if: matrix.platform == 'linux/amd64' + uses: docker/build-push-action@v7 + with: + context: . + platforms: ${{ matrix.platform }} + target: sourcemaps + build-args: | + GIT_COMMIT=${{ github.sha }} + BULWARK_SOURCE_MAPS=1 + outputs: type=local,dest=sourcemaps + cache-from: type=gha,scope=${{ matrix.platform }} + + - name: Upload source maps to Bugsink + if: matrix.platform == 'linux/amd64' + env: + BUGSINK_AUTH_TOKEN: ${{ secrets.BUGSINK_AUTH_TOKEN }} + run: bash scripts/upload-sourcemaps.sh sourcemaps + - name: Export digest run: | mkdir -p /tmp/digests diff --git a/.github/workflows/kubernetes.yml b/.github/workflows/kubernetes.yml new file mode 100644 index 000000000..e62d2d215 --- /dev/null +++ b/.github/workflows/kubernetes.yml @@ -0,0 +1,103 @@ +name: "CI: Kubernetes manifests" + +# Builds the image from this commit, deploys deploy/kubernetes to a kind +# cluster (three replicas and one Valkey) and runs +# deploy/kubernetes/test/smoke.sh against it: every replica ready on the +# shared store, one admin session valid everywhere, a config change reaching +# every replica, no failed requests while a pod is deleted and during a +# rolling restart. The config-in-git overlay runs next to it in its own +# namespace and must refuse config saves. +# +# It runs when the manifests change, weekly on main (an app change can break +# several replicas without touching the manifests), and on demand: +# gh workflow run kubernetes.yml + +on: + pull_request: + paths: + - deploy/kubernetes/** + - .github/workflows/kubernetes.yml + push: + branches: [main] + paths: + - deploy/kubernetes/** + - .github/workflows/kubernetes.yml + schedule: + - cron: "41 3 * * 1" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: kubernetes-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + kind: + name: kind + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + KIND_VERSION: v0.29.0 + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Build the image + run: docker build --build-arg "GIT_COMMIT=${GITHUB_SHA}" -t bulwark-local:ci . + + - name: Create the cluster + run: | + # The runner's own kind in /usr/local/bin belongs to root; use a pinned one. + mkdir -p "$RUNNER_TEMP/bin" + curl -fsSLo "$RUNNER_TEMP/bin/kind" "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64" + chmod +x "$RUNNER_TEMP/bin/kind" + echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH" + export PATH="$RUNNER_TEMP/bin:$PATH" + kind create cluster --name ci --wait 180s + kind load docker-image bulwark-local:ci --name ci + + - name: Deploy the base and the config-in-git overlay + run: | + umask 077 + admin="$(openssl rand -hex 16)" + echo "::add-mask::$admin" + { + echo "SESSION_SECRET=$(openssl rand -base64 32)" + echo "ADMIN_PASSWORD=$admin" + echo "VALKEY_PASSWORD=$(openssl rand -hex 24)" + } > deploy/kubernetes/base/secrets.env + echo "ADMIN_PASSWORD=$admin" >> "$GITHUB_ENV" + for target in base:bulwark overlays/config-in-git:bulwark-ro; do + src="${target%%:*}"; ns="${target##*:}" + mkdir -p "ci/$ns" + cat > "ci/$ns/kustomization.yaml" <> "$GITHUB_ENV" + - name: Check the tarball holds no source maps + run: node scripts/sourcemaps.mjs check "$TARBALL" + - name: Upload artifact uses: actions/upload-artifact@v7 with: diff --git a/.github/workflows/static-lite.yml b/.github/workflows/static-lite.yml index d4528ec24..7cd355c29 100644 --- a/.github/workflows/static-lite.yml +++ b/.github/workflows/static-lite.yml @@ -79,8 +79,18 @@ jobs: LITE_APP_NAME: ${{ inputs.app_name }} LITE_LOCALES: ${{ inputs.locales }} LITE_DEMO_MODE: ${{ inputs.demo }} + # Browser source maps for Bugsink; build:lite moves them out of out/ + # before anything is packed. https://bulwarkmail.org/docs/deployment/monitoring + BULWARK_SOURCE_MAPS: "1" + SOURCEMAPS_OUT: sourcemaps run: npm run build:lite + # Each target is its own build with its own chunks, so each uploads. + - name: Upload source maps to Bugsink + env: + BUGSINK_AUTH_TOKEN: ${{ secrets.BUGSINK_AUTH_TOKEN }} + run: bash scripts/upload-sourcemaps.sh sourcemaps + # Every host literal in the bundle must be a placeholder, the configured # JMAP server, a built-in host the no-third-party switch turns off, or a # string listed as never requested (#903). @@ -107,6 +117,9 @@ jobs: echo "ZIP=${ZIP}" >> "$GITHUB_ENV" ls -la "${ZIP}" + - name: Check the zip holds no source maps + run: node scripts/sourcemaps.mjs check "$ZIP" + - name: Upload artifact uses: actions/upload-artifact@v7 with: diff --git a/.gitignore b/.gitignore index 24d233b38..0914d4091 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,8 @@ # next.js /.next/ /out/ +# Source maps CI collects for the error tracker (scripts/sourcemaps.mjs) +/sourcemaps/ /lite-image/ # scripts/build-plugin-sandbox.mjs /public/plugin-sandbox-assets/ @@ -57,3 +59,6 @@ next-env.d.ts # Sibling repos /repos/ + +# Kubernetes reference manifests: the secrets users fill in +deploy/kubernetes/base/secrets.env diff --git a/Dockerfile b/Dockerfile index 7485ced40..add3eabd1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,10 @@ ENV NEXT_PUBLIC_DEFAULT_LOCALE=$NEXT_PUBLIC_DEFAULT_LOCALE # `git rev-parse` inside the build can't find it - CI must pass it in. ARG GIT_COMMIT=unknown ENV GIT_COMMIT=$GIT_COMMIT +# Browser source maps for the error tracker: CI passes 1, a plain build makes +# none (next.config.ts reads it). They never reach the image, see below. +ARG BULWARK_SOURCE_MAPS=0 +ENV BULWARK_SOURCE_MAPS=$BULWARK_SOURCE_MAPS # The mock JMAP server (DEV_MOCK_JMAP) accepts any password and is for # `npm run dev` only; images ship without it. # `next build` type-checks every .ts file, and some unit tests import the @@ -34,6 +38,21 @@ RUN rm -rf "app/api/dev-jmap" && \ node scripts/build-plugin-sandbox.mjs && \ npx next build --webpack +# webpack has no debug IDs of its own, so sentry-cli adds them to the +# chunks; then every map moves to /sourcemaps (exported with +# `--target sourcemaps`) and the check fails the build if one would ship. +# https://bulwarkmail.org/docs/deployment/monitoring +ARG SENTRY_CLI_VERSION=3.8.2 +RUN if [ "$BULWARK_SOURCE_MAPS" = "1" ]; then \ + npx --yes "@sentry/cli@${SENTRY_CLI_VERSION}" sourcemaps inject .next/static; \ + fi && \ + node scripts/sourcemaps.mjs collect --out /sourcemaps .next/static .next/standalone && \ + node scripts/sourcemaps.mjs check .next/static .next/standalone public + +# Only the maps, for CI to upload: never part of the image. +FROM scratch AS sourcemaps +COPY --from=builder /sourcemaps / + FROM node:24-alpine AS runner LABEL org.opencontainers.image.title="Bulwark Webmail" @@ -57,6 +76,7 @@ RUN apk upgrade --no-cache && \ COPY --from=builder /app/public ./public COPY --from=builder /app/.next/standalone ./ COPY --from=builder /app/.next/static ./.next/static +COPY --from=builder /app/scripts/healthcheck.mjs ./healthcheck.mjs # Every data dir exists up front and belongs to the runtime user, so a fresh # volume on /app/data (Docker copies these into it) or on any one of them is # writable. plugin-secrets holds users' plugin credentials, version-check the @@ -69,4 +89,9 @@ USER nextjs EXPOSE 3000 ENV PORT=3000 ENV HOSTNAME="0.0.0.0" +# The health check needs the base path the bundle was built with. +ARG NEXT_PUBLIC_BASE_PATH= +ENV BULWARK_BASE_PATH=$NEXT_PUBLIC_BASE_PATH +# Liveness only (/api/health/live); readiness is /api/health/ready. +HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 CMD ["node", "healthcheck.mjs"] CMD ["node", "server.js"] diff --git a/app/(main)/[locale]/error.tsx b/app/(main)/[locale]/error.tsx index f25e2ad46..421c4be6c 100644 --- a/app/(main)/[locale]/error.tsx +++ b/app/(main)/[locale]/error.tsx @@ -3,6 +3,7 @@ import { useEffect } from "react"; import { useTranslations } from "next-intl"; import { ErrorState } from "@/components/error/error-fallbacks"; +import { captureException } from "@/lib/error-reporting/client"; /** * Route-level error boundary for locale pages. @@ -19,6 +20,7 @@ export default function LocaleError({ useEffect(() => { console.error("Route error:", error); + captureException(error, "boundary", false); }, [error]); return ( diff --git a/app/(main)/admin/[[...segments]]/page.tsx b/app/(main)/admin/[[...segments]]/page.tsx index 9af174495..2a271e2c3 100644 --- a/app/(main)/admin/[[...segments]]/page.tsx +++ b/app/(main)/admin/[[...segments]]/page.tsx @@ -24,6 +24,7 @@ import { MarketplaceDetailPage } from '../_pages/marketplace-detail'; import { ExtensionDetailPage } from '../_pages/extension-detail'; import { VersionPage } from '../_pages/version'; import { TelemetryPage } from '../_pages/telemetry'; +import { MonitoringPage } from '../_pages/monitoring'; import { LogsPage } from '../_pages/logs'; /** @@ -64,6 +65,7 @@ export default function AdminPage() { case 'marketplace': return detail ? : ; case 'version': return ; case 'telemetry': return ; + case 'monitoring': return ; case 'logs': return ; } } diff --git a/app/(main)/admin/_pages/monitoring.tsx b/app/(main)/admin/_pages/monitoring.tsx new file mode 100644 index 000000000..c574cb790 --- /dev/null +++ b/app/(main)/admin/_pages/monitoring.tsx @@ -0,0 +1,211 @@ +'use client'; + +import { useCallback, useEffect, useState } from 'react'; +import { Loader2 } from '@/components/icons'; +import { apiFetch } from '@/lib/browser-navigation'; +import { adminPageDef } from '@/lib/admin/admin-nav'; +import { useAdminUnsavedChanges } from '@/stores/admin-unsaved-store'; +import { SettingsSection, SectionNav } from '@/components/settings/settings-section'; +import { AdminPageHeader } from '@/components/admin/page-header'; +import { AdminSaveBar } from '@/components/admin/save-bar'; +import { withEdit } from '@/lib/admin/edits'; +import { useAdminFeedback } from '@/components/admin/feedback'; +import { ConfigTextItem, ConfigNote } from '@/components/admin/config-items'; + +interface ConfigEntry { + value?: unknown; + source: 'admin' | 'env' | 'default'; +} + +interface ErrorEntry { + fingerprint: string; + name: string; + code?: string; + version: string; + runtime: 'browser' | 'server'; + count: number; + frames: Array<{ path: string; line?: number; column?: number }>; +} + +interface MonitoringState { + collecting: boolean; + day: string; + errorsToday: { total: number; top: ErrorEntry[] }; + metrics?: { enabled: boolean }; + tracing?: { configured: boolean; active: boolean; endpointHost: string | null }; +} + +const PAGE = adminPageDef('monitoring'); +const KEYS = ['errorReportingDsn', 'errorReportingSampleRate', 'errorReportingEnvironment'] as const; +const DOCS = 'https://bulwarkmail.org/docs/deployment/monitoring'; + +export function MonitoringPage() { + const [config, setConfig] = useState>({}); + const [state, setState] = useState(null); + const [edits, setEdits] = useState>({}); + const [loading, setLoading] = useState(true); + const [saving, setSaving] = useState(false); + const setMessage = useAdminFeedback(); + + const load = useCallback(async () => { + const [configRes, stateRes] = await Promise.all([apiFetch('/api/admin/config'), apiFetch('/api/admin/monitoring')]); + if (configRes.ok) setConfig(await configRes.json()); + if (stateRes.ok) setState(await stateRes.json()); + setLoading(false); + }, []); + + useEffect(() => { void load(); }, [load]); + + function handleChange(key: string, value: unknown) { + setEdits((prev) => withEdit(prev, key, value, config[key]?.value)); + setMessage(null); + } + + function currentValue(key: string): string { + const value = key in edits ? edits[key] : config[key]?.value; + return typeof value === 'string' ? value : value === undefined || value === null ? '' : String(value); + } + + async function handleSave() { + if (Object.keys(edits).length === 0) return; + setSaving(true); + setMessage(null); + const res = await apiFetch('/api/admin/config', { + method: 'PATCH', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(edits), + }); + if (res.ok) { + setMessage({ type: 'success', text: 'Saved. Browsers pick it up on their next page load.' }); + setEdits({}); + await load(); + } else { + const data = await res.json().catch(() => ({})); + setMessage({ type: 'error', text: data.error || 'Failed to save' }); + } + setSaving(false); + } + + async function handleRevert(key: string) { + const res = await apiFetch('/api/admin/config', { + method: 'DELETE', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ key }), + }); + if (res.ok) { + setEdits((prev) => { + const next = { ...prev }; + delete next[key]; + return next; + }); + await load(); + } + } + + const hasEdits = KEYS.some((key) => key in edits); + useAdminUnsavedChanges('monitoring', hasEdits); + + if (loading) { + return ( +
+ +
+ Loading +
+
+ ); + } + + const today = state?.errorsToday; + + return ( +
+ +
+ + + + + Send errors from browsers and from this server to your own error tracker: Bugsink, GlitchTip or + Sentry. Off while the address is empty. Message text, addresses, file names, tokens and URLs are + removed before anything is sent. What is sent + + + + + + + + {!state?.collecting ? ( + + Not counting errors. Counting starts when an error reporting address is set, the metrics + endpoint is on, or error counts are sent with the usage stats. + + ) : !today || today.total === 0 ? ( +

No errors so far today (UTC).

+ ) : ( +
+

{today.total} errors today (UTC), most frequent first.

+
    + {today.top.map((entry) => ( +
  • +
    +
    {entry.name}{entry.code ? `: ${entry.code}` : ''}
    +
    + {entry.frames[0] ? `${entry.frames[0].path}:${entry.frames[0].line ?? ''}:${entry.frames[0].column ?? ''}` : 'no stack'} + {' · '}{entry.runtime}{' · '}{entry.version} +
    +
    + {entry.count} +
    • + ))} +
+
+ )} + + + +

+ {state?.metrics?.enabled + ? <>On. Prometheus scrapes /api/metrics on each replica with the token as a bearer token. + : <>Off. Set METRICS_TOKEN (or METRICS_TOKEN_FILE) in the environment to turn on /api/metrics.} +

+ + Requests per route group, sign-ins, calls to Stalwart and other services, browser errors, memory and + event-loop lag. Metric list and scrape config + + + + +
    +
  • /api/health/live: the process answers. For liveness probes and the Docker health check.
    • +
  • /api/health/ready: config loaded, data dirs writable, Stalwart reachable. For readiness probes.
    • +
+ + + +

+ {state?.tracing?.active + ? <>On. Spans go to {state.tracing.endpointHost ?? 'the OTLP endpoint'} over OTLP/HTTP. + : state?.tracing?.configured + ? <>Configured, but not running in this process. Restart Bulwark after setting the OTEL_* variables. + : <>Off. Set OTEL_EXPORTER_OTLP_ENDPOINT (and the other standard OTEL_* variables you need) to send traces.} +

+ + Next's request spans plus spans for calls to Stalwart and other services. Stalwart gets the trace + context, so its spans join the trace. URLs, user agents and addresses are removed from spans.{' '} + Setup + + +
+ + setEdits({})} + note="Browsers pick it up on their next page load" + /> +
+ ); +} diff --git a/app/(main)/admin/_pages/telemetry.tsx b/app/(main)/admin/_pages/telemetry.tsx index aac209d68..74a84ed06 100644 --- a/app/(main)/admin/_pages/telemetry.tsx +++ b/app/(main)/admin/_pages/telemetry.tsx @@ -16,15 +16,22 @@ import { cn } from '@/lib/utils'; interface TelemetryStatus { consent: 'pending' | 'on' | 'off'; consentSource: 'no-third-party' | 'env' | 'file'; + /** 'on' sends them daily; 'instant' also sends today's as they change. */ + errorsConsent?: ErrorsMode; + errorsConsentSource?: 'telemetry' | 'no-third-party' | 'env' | 'file'; endpoint: string; defaultEndpoint: string; consentedAt: string | null; lastSentAt: string | null; nextScheduledAt: string | null; payloadPreview: Record; + errorsPreview?: unknown[]; + instantErrorsPreview?: unknown[]; accountCounts: { total: number; active7d: number }; } +type ErrorsMode = 'on' | 'off' | 'instant'; + const PAGE = adminPageDef('telemetry'); function timeAgo(iso: string | null): string { @@ -56,10 +63,12 @@ export function TelemetryPage() { // The heartbeat switch waits for Save like the endpoint (one page, one // save bar). const [consentDraft, setConsentDraft] = useState<'on' | 'off' | null>(null); + const [errorsDraft, setErrorsDraft] = useState(null); const endpointDirty = status !== null && endpointDraft !== status.endpoint; const consentDirty = status !== null && consentDraft !== null && consentDraft !== status.consent; - const dirty = endpointDirty || consentDirty; + const errorsDirty = status !== null && errorsDraft !== null && errorsDraft !== (status.errorsConsent ?? 'off'); + const dirty = endpointDirty || consentDirty || errorsDirty; useAdminUnsavedChanges('telemetry', dirty); async function refresh(): Promise { @@ -99,11 +108,17 @@ export function TelemetryPage() { const error = await post({ action: 'set-consent', consent: consentDraft }); if (error) errors.push(`Heartbeats: ${error}`); } + // After the heartbeat switch: error counts need it on. + if (errorsDirty && errorsDraft) { + const error = await post({ action: 'set-errors-consent', consent: errorsDraft }); + if (error) errors.push(`Error counts: ${error}`); + } if (endpointDirty) { const error = await post({ action: 'set-endpoint', endpoint: endpointDraft }); if (error) errors.push(`Endpoint: ${error}`); } setConsentDraft(null); + setErrorsDraft(null); setMessage(errors.length > 0 ? { type: 'error', text: `Couldn't save: ${errors.join('; ')}` } : { type: 'success', text: 'Telemetry settings saved.' }); @@ -146,6 +161,25 @@ export function TelemetryPage() { const isOn = status.consent === 'on'; const shownOn = (consentDraft ?? status.consent) === 'on'; + // Error counts follow the heartbeat switch as it is about to be saved. + const errorsLocked = noThirdParty || status.errorsConsentSource === 'env' || !shownOn; + const errorsMode: ErrorsMode = shownOn ? (errorsDraft ?? status.errorsConsent ?? 'off') : 'off'; + const errorsShownOn = errorsMode !== 'off'; + const errorsDescription = [ + 'Adds how often each error happened yesterday, the error name and where in the code it happened. No message text, no URLs, no instance id goes to the error tracker.', + !shownOn && 'Needs the heartbeat on.', + status.errorsConsentSource === 'env' && 'Locked by the BULWARK_TELEMETRY_ERRORS env var.', + ].filter(Boolean).join(' '); + const instantDescription = [ + 'Also sends today\'s counts within about a minute when an error is new or happens noticeably more often, so a broken release shows up the same day. The same fields as the daily counts; the collector learns roughly when errors happen.', + !errorsShownOn && 'Needs error counts on.', + ].filter(Boolean).join(' '); + + // The payload as it will be once the switches are saved. + const previewPayload = errorsShownOn + ? { ...status.payloadPreview, schema: '2', errors: status.errorsPreview ?? [] } + : (({ errors: _omit, ...v1 }) => ({ ...v1, schema: '1' }))(status.payloadPreview as Record & { errors?: unknown }); + const consentDescription = [ status.consent === 'pending' && 'Initialising - no heartbeats sent yet.', status.consent === 'on' && status.endpoint && 'Heartbeats are enabled. Thanks for helping us improve!', @@ -181,7 +215,25 @@ export function TelemetryPage() { { setConsentDraft(checked ? 'on' : 'off'); setMessage(null); }} + onChange={(checked) => { + setConsentDraft(checked ? 'on' : 'off'); + if (!checked) setErrorsDraft('off'); + setMessage(null); + }} + /> + + + { setErrorsDraft(checked ? 'on' : 'off'); setMessage(null); }} + /> + + + { setErrorsDraft(checked ? 'instant' : 'on'); setMessage(null); }} /> @@ -252,8 +304,24 @@ export function TelemetryPage() { 
-            {JSON.stringify(status.payloadPreview, null, 2)}
+            {JSON.stringify(previewPayload, null, 2)}
+ {errorsMode === 'instant' && ( + <> +

+ Sent right away: today's error counts as they would go out now. Each minute only + the errors that are new or reached a higher count range are sent. +

+ 
+                {JSON.stringify({
+                  schema: '1',
+                  instance_id: status.payloadPreview.instance_id,
+                  day: new Date().toISOString().slice(0, 10),
+                  errors: status.instantErrorsPreview ?? [],
+                }, null, 2)}
+
+ + )} @@ -261,7 +329,7 @@ export function TelemetryPage() { dirty={dirty} saving={busy === 'save'} onSave={() => void save()} - onDiscard={() => { setEndpointDraft(status.endpoint); setConsentDraft(null); }} + onDiscard={() => { setEndpointDraft(status.endpoint); setConsentDraft(null); setErrorsDraft(null); }} saveLabel="Save" /> diff --git a/app/(main)/layout.tsx b/app/(main)/layout.tsx index 96f57ab7c..208656678 100644 --- a/app/(main)/layout.tsx +++ b/app/(main)/layout.tsx @@ -5,6 +5,7 @@ import { headers } from "next/headers"; import { getLocale, getTranslations } from "next-intl/server"; import { ServiceWorkerRegistration } from "@/components/service-worker-registration"; import { FaviconBadge } from "@/components/favicon-badge"; +import { ErrorReportingRoot } from "@/components/error/error-reporting-root"; import { ThemeColorSync } from "@/components/theme-color-sync"; import { IosViewportHeal } from "@/components/ios-viewport-heal"; import { configManager } from "@/lib/admin/config-manager"; @@ -328,6 +329,7 @@ export default async function RootLayout({ {!themeColorConfigured && } + {children} diff --git a/app/api/admin/monitoring/route.ts b/app/api/admin/monitoring/route.ts new file mode 100644 index 000000000..57dd4823c --- /dev/null +++ b/app/api/admin/monitoring/route.ts @@ -0,0 +1,37 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { requireAdminAuth } from '@/lib/admin/session'; +import { logger } from '@/lib/logger'; +import { readErrors, summarizeDay, utcDay } from '@/lib/error-reporting/aggregate'; +import { shouldCollectErrors } from '@/lib/error-reporting/settings'; +import { storedDataUnavailableResponse } from '@/lib/stored-data-errors'; +import { metricsEnabled } from '@/lib/metrics'; +import { isTracingActive, tracingEndpoint } from '@/lib/tracing/config'; + +export const dynamic = 'force-dynamic'; + +/** + * GET /api/admin/monitoring: what System > Monitoring shows besides the + * config keys (those come from /api/admin/config): whether errors are being + * counted, today's counts, and whether metrics and tracing are on. + */ +export async function GET(request: NextRequest) { + try { + const auth = await requireAdminAuth(request); + if ('error' in auth) return auth.error; + + const collecting = await shouldCollectErrors(); + const today = utcDay(); + const errorsToday = summarizeDay(await readErrors(), today); + const metrics = { enabled: metricsEnabled() }; + const endpoint = tracingEndpoint(); + let endpointHost: string | null = null; + try { endpointHost = endpoint ? new URL(endpoint).host : null; } catch { endpointHost = null; } + const tracing = { configured: endpoint !== null, active: isTracingActive(), endpointHost }; + return NextResponse.json({ collecting, day: today, errorsToday, metrics, tracing }, { headers: { 'Cache-Control': 'no-store' } }); + } catch (err) { + logger.error('monitoring GET error', { error: err instanceof Error ? err.message : 'unknown' }); + const unavailable = storedDataUnavailableResponse(err); + if (unavailable) return unavailable; + return NextResponse.json({ error: 'Internal server error' }, { status: 500 }); + } +} diff --git a/app/api/admin/telemetry/route.ts b/app/api/admin/telemetry/route.ts index aac9d0263..2773879b2 100644 --- a/app/api/admin/telemetry/route.ts +++ b/app/api/admin/telemetry/route.ts @@ -4,6 +4,7 @@ import { auditLog } from '@/lib/admin/audit'; import { logger } from '@/lib/logger'; import { effectiveConsent, + effectiveErrorsConsent, updateState, buildPayload, sendOnce, @@ -13,6 +14,8 @@ import { resolveEndpointAllowed, } from '@/lib/telemetry'; import { storedDataUnavailableResponse } from '@/lib/stored-data-errors'; +import { errorsForDay, errorsForPayload } from '@/lib/telemetry/payload'; +import { readErrors, utcDay } from '@/lib/error-reporting/aggregate'; /** * GET /api/admin/telemetry @@ -25,21 +28,31 @@ export async function GET(request: NextRequest) { if ('error' in auth) return auth.error; const { consent, source, state } = await effectiveConsent(); - const [payload, accountCounts] = await Promise.all([ + const [payload, accountCounts, errors, counted] = await Promise.all([ buildPayload(), getLoginCounts(), + effectiveErrorsConsent(), + readErrors(), ]); return NextResponse.json( { consent, consentSource: source, + // The separate consent to add error counts (schema v2). + errorsConsent: errors.consent, + errorsConsentSource: errors.source, endpoint: state.endpoint ?? DEFAULT_ENDPOINT, consentedAt: state.consentedAt, lastSentAt: state.lastSentAt, nextScheduledAt: state.nextScheduledAt, defaultEndpoint: DEFAULT_ENDPOINT, payloadPreview: payload, + // What `errors` would hold, so the page can show the v2 payload as soon + // as the switch is flipped, before anything is saved or sent. + errorsPreview: errorsForPayload(counted), + // What instant mode would send now: today's counts so far. + instantErrorsPreview: errorsForDay(counted, utcDay()), accountCounts, }, { headers: { 'Cache-Control': 'no-store' } }, @@ -57,7 +70,8 @@ export async function GET(request: NextRequest) { /** * POST /api/admin/telemetry * Body: { action: 'set-consent' | 'set-endpoint' | 'send-now', ... } - * set-consent : { action, consent: 'on' | 'off' } + * set-consent : { action, consent: 'on' | 'off' } (off also switches error counts off) + * set-errors-consent : { action, consent: 'on' | 'instant' | 'off' } ('on' is daily) * set-endpoint : { action, endpoint: string } * send-now : { action } */ @@ -100,12 +114,39 @@ export async function POST(request: NextRequest) { before = state.consent; state.consent = consent; if (consent === 'on' && !state.consentedAt) state.consentedAt = now; + // Error counts ride on the heartbeat: no heartbeats, no error counts, + // and turning heartbeats on again doesn't bring them back by itself. + if (consent === 'off') state.errorsConsent = 'off'; }); await reschedule(); await auditLog('telemetry.set-consent', { from: before, to: body.consent }, ip); return NextResponse.json({ ok: true }); } + if (body.action === 'set-errors-consent') { + const errors = await effectiveErrorsConsent(); + if (errors.source === 'env') { + return NextResponse.json({ error: 'error counts are set by the BULWARK_TELEMETRY_ERRORS env var' }, { status: 409 }); + } + if (errors.source === 'no-third-party') { + return NextResponse.json({ error: 'telemetry is off while NO_THIRD_PARTY_TRAFFIC is on' }, { status: 409 }); + } + if (body.consent !== 'on' && body.consent !== 'off' && body.consent !== 'instant') { + return NextResponse.json({ error: 'consent must be "on", "instant" or "off"' }, { status: 400 }); + } + if (body.consent !== 'off' && errors.source === 'telemetry') { + return NextResponse.json({ error: 'turn on the heartbeat first' }, { status: 409 }); + } + const consent = body.consent; + let before: string | undefined; + await updateState((state) => { + before = state.errorsConsent ?? 'off'; + state.errorsConsent = consent; + }); + await auditLog('telemetry.set-errors-consent', { from: before, to: consent }, ip); + return NextResponse.json({ ok: true }); + } + if (body.action === 'set-endpoint') { if (typeof body.endpoint !== 'string') { return NextResponse.json({ error: 'endpoint required' }, { status: 400 }); diff --git a/app/api/auth/token/route.ts b/app/api/auth/token/route.ts index 389599afa..6c065e524 100644 --- a/app/api/auth/token/route.ts +++ b/app/api/auth/token/route.ts @@ -1,4 +1,5 @@ import { NextRequest, NextResponse } from 'next/server'; +import { observeUpstream } from '@/lib/metrics/upstream'; import { cookies } from 'next/headers'; import { logger } from '@/lib/logger'; import { @@ -57,7 +58,8 @@ function getSlot(request: NextRequest): number { const REVOCATION_TIMEOUT_MS = 3000; async function revokeRefreshToken(token: string, serverId: string | null, metadata: OAuthMetadata | null): Promise { - if (!metadata?.revocation_endpoint) return; + const revocationEndpoint = metadata?.revocation_endpoint; + if (!revocationEndpoint) return; const params = buildOAuthParams({ token, token_type_hint: 'refresh_token', @@ -66,13 +68,13 @@ async function revokeRefreshToken(token: string, serverId: string | null, metada try { // Never follow a redirect with the refresh token and client secret in // the body: the endpoint was validated, its redirect target was not. - const revocationResponse = await fetch(metadata.revocation_endpoint, { + const revocationResponse = await observeUpstream('oauth', () => fetch(revocationEndpoint, { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: params.toString(), signal: AbortSignal.timeout(REVOCATION_TIMEOUT_MS), redirect: 'error', - }); + })); if (!revocationResponse.ok) { logger.warn('Token revocation returned error', { status: revocationResponse.status }); } @@ -262,13 +264,13 @@ export async function PUT(request: NextRequest) { refresh_token: refreshToken, }, serverId, await defaultClientOptions(serverId)); - const tokenResponse = await fetch(tokenEndpoint, { + const tokenResponse = await observeUpstream('oauth', () => fetch(tokenEndpoint, { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: params.toString(), // See revokeRefreshToken: the body holds the refresh token. redirect: 'error', - }); + })); if (!tokenResponse.ok) { const errorText = await tokenResponse.text(); diff --git a/app/api/config/route.ts b/app/api/config/route.ts index 6e8208834..de9521a46 100644 --- a/app/api/config/route.ts +++ b/app/api/config/route.ts @@ -12,6 +12,7 @@ import { type BrandingOverrideKey, } from '@/lib/admin/domain-branding'; import { isNoThirdPartyTrafficOn } from '@/lib/admin/third-party-mode'; +import { browserErrorSettings } from '@/lib/error-reporting/settings'; /** * Runtime configuration endpoint @@ -109,6 +110,9 @@ export async function GET(request: NextRequest) { signLanguageUrl: httpUrlOrEmpty(branded('signLanguageUrl', '')), // Browser-side switches of lib/outgoing-hosts.ts read this (#903). noThirdPartyTraffic: await isNoThirdPartyTrafficOn(), + // Error reporting (lib/error-reporting/client.ts): off unless a DSN is + // set or this server counts errors. The DSN is public by design. + errorReporting: await browserErrorSettings(), }, { // Runtime settings can change without new asset URLs. Prevent stale diff --git a/app/api/errors/__tests__/route.test.ts b/app/api/errors/__tests__/route.test.ts new file mode 100644 index 000000000..d53e1dde6 --- /dev/null +++ b/app/api/errors/__tests__/route.test.ts @@ -0,0 +1,104 @@ +// @vitest-environment node +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { NextRequest } from 'next/server'; + +const config: Record = {}; +vi.mock('@/lib/admin/config-manager', () => ({ + configManager: { + ensureLoaded: async () => {}, + get: (key: string, fallback: unknown) => (key in config ? config[key] : fallback), + }, +})); + +import { POST } from '../route'; +import { flushErrors, readErrors, resetAggregateForTests, utcDay } from '@/lib/error-reporting/aggregate'; +import { metrics, resetMetricsForTests } from '@/lib/metrics'; + +const DSN = 'https://6f0b9c79ed3a4315b76595bc2536ac40@bugsink.example.org/3'; +const SUBJECT = 'Merger plans for Acme Q3'; + +let dir: string; +let ip = 0; + +function post(body: unknown): NextRequest { + ip += 1; + return new NextRequest('http://localhost/api/errors', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-forwarded-for': `203.0.113.${ip}` }, + body: typeof body === 'string' ? body : JSON.stringify(body), + }); +} + +const report = { + name: 'TypeError', + message: `Cannot open "${SUBJECT}"`, + frames: [{ path: `https://mail.example.org/_next/static/chunks/1.js?subject=${encodeURIComponent(SUBJECT)}`, line: 2, column: 30, fn: 'Ab' }], + version: '1.20.0', +}; + +beforeEach(() => { + dir = mkdtempSync(path.join(tmpdir(), 'bulwark-errors-route-')); + process.env.TELEMETRY_DATA_DIR = dir; + for (const key of Object.keys(config)) delete config[key]; + resetAggregateForTests(); +}); + +afterEach(() => { + resetAggregateForTests(); + delete process.env.TELEMETRY_DATA_DIR; + rmSync(dir, { recursive: true, force: true }); +}); + +describe('POST /api/errors', () => { + it('keeps nothing while error collection is off', async () => { + const res = await POST(post(report)); + expect(res.status).toBe(204); + await flushErrors(); + expect(await readErrors()).toEqual({}); + }); + + it('counts a report, scrubbed again, without its message', async () => { + config.errorReportingDsn = DSN; + expect((await POST(post(report))).status).toBe(204); + await flushErrors(); + const stored = JSON.stringify(await readErrors()); + expect(stored).not.toContain('Merger'); + expect(stored).not.toContain('subject='); + const [entry] = Object.values((await readErrors())[utcDay()]); + expect(entry).toMatchObject({ name: 'TypeError', runtime: 'browser', version: '1.20.0', count: 1 }); + expect(entry.frames).toEqual([{ path: '/_next/static/chunks/1.js', line: 2, column: 30, fn: 'Ab' }]); + }); + + it('counts while only the metrics endpoint is on, for the browser errors metric', async () => { + process.env.METRICS_TOKEN = 'scrape-secret-0123456789'; + resetMetricsForTests(); + try { + expect((await POST(post(report))).status).toBe(204); + await flushErrors(); + expect(Object.keys((await readErrors())[utcDay()] ?? {})).toHaveLength(1); + expect(await metrics().registry.metrics()).toContain('bulwark_browser_errors_total 1'); + } finally { + delete process.env.METRICS_TOKEN; + resetMetricsForTests(); + } + }); + + it('ignores a name or code that is not an identifier or fixed code', async () => { + config.errorReportingDsn = DSN; + await POST(post({ ...report, name: SUBJECT, code: SUBJECT })); + await flushErrors(); + const [entry] = Object.values((await readErrors())[utcDay()]); + expect(entry.name).toBe('Error'); + expect(entry.code).toBeUndefined(); + }); + + it('refuses bodies that are too large or not JSON objects', async () => { + config.errorReportingDsn = DSN; + expect((await POST(post('x'.repeat(17 * 1024)))).status).toBe(413); + expect((await POST(post('[1,2]'))).status).toBe(400); + expect((await POST(post('not json'))).status).toBe(400); + }); +}); diff --git a/app/api/errors/route.ts b/app/api/errors/route.ts new file mode 100644 index 000000000..b31a38952 --- /dev/null +++ b/app/api/errors/route.ts @@ -0,0 +1,72 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getClientIP } from '@/lib/admin/session'; +import { getStore } from '@/lib/shared-store'; +import { recordError } from '@/lib/error-reporting/aggregate'; +import { appVersion, isErrorCode } from '@/lib/error-reporting/envelope'; +import { scrubName } from '@/lib/error-reporting/scrub'; +import { shouldCollectErrors } from '@/lib/error-reporting/settings'; +import { sanitizeFrames } from '@/lib/error-reporting/stack'; +import { countBrowserError } from '@/lib/metrics'; + +export const dynamic = 'force-dynamic'; + +/** + * POST /api/errors: a browser of this instance reports an error it caught + * (lib/error-reporting/client.ts), for the per-day error counts. Only while + * collection is on (a DSN, the metrics endpoint or error-count consent); + * otherwise 204 and nothing is kept. + * + * Same-origin only (the proxy's origin gate), at most 16 KB, 30 a minute per + * address. Nothing a browser sends is trusted: every field is scrubbed again + * and the fingerprint is computed here. The DSN gets the browser's own + * report; this route only counts. + */ + +const MAX_BODY_BYTES = 16 * 1024; +const PER_MINUTE = 30; + +const NO_STORE = { 'Cache-Control': 'no-store' }; + +export async function POST(request: NextRequest) { + if (!(await shouldCollectErrors())) return new NextResponse(null, { status: 204, headers: NO_STORE }); + + const length = Number(request.headers.get('content-length') ?? '0'); + if (length > MAX_BODY_BYTES) return NextResponse.json({ error: 'too large' }, { status: 413, headers: NO_STORE }); + + const limit = await getStore().counters.reserve([ + { key: `error-reports:${getClientIP(request)}`, max: PER_MINUTE, windowMs: 60_000 }, + ]).catch(() => ({ ok: true })); + if (!limit.ok) return NextResponse.json({ error: 'rate limited' }, { status: 429, headers: NO_STORE }); + + let text: string; + try { + text = await request.text(); + } catch { + return NextResponse.json({ error: 'bad request' }, { status: 400, headers: NO_STORE }); + } + if (text.length > MAX_BODY_BYTES) return NextResponse.json({ error: 'too large' }, { status: 413, headers: NO_STORE }); + + let body: Record; + try { + const parsed = JSON.parse(text) as unknown; + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) throw new Error('not an object'); + body = parsed as Record; + } catch { + return NextResponse.json({ error: 'bad request' }, { status: 400, headers: NO_STORE }); + } + + const code = isErrorCode(body.code) ? body.code : undefined; + recordError( + { + name: scrubName(body.name), + frames: sanitizeFrames(body.frames), + ...(code ? { code } : {}), + }, + 'browser', + // The version the tab runs (a tab left open across an update runs the + // old one, and its frames point into that); the server's when absent. + typeof body.version === 'string' && /^\d+\.\d+\.\d+(?:[-+][\w.-]{1,40})?$/.test(body.version) ? body.version : appVersion(), + ); + countBrowserError(); + return new NextResponse(null, { status: 204, headers: NO_STORE }); +} diff --git a/app/api/health/live/route.ts b/app/api/health/live/route.ts new file mode 100644 index 000000000..e1f150f27 --- /dev/null +++ b/app/api/health/live/route.ts @@ -0,0 +1,18 @@ +import { NextResponse } from 'next/server'; + +export const dynamic = 'force-dynamic'; + +/** + * Liveness: 200 while the process answers. Nothing else is checked, so a + * Stalwart or store outage never makes an orchestrator restart Bulwark. + * Readiness is /api/health/ready. https://bulwarkmail.org/docs/deployment/monitoring#health + */ +const HEADERS = { 'Cache-Control': 'no-store' }; + +export function GET() { + return NextResponse.json({ status: 'ok' }, { headers: HEADERS }); +} + +export function HEAD() { + return new NextResponse(null, { status: 200, headers: HEADERS }); +} diff --git a/app/api/health/ready/route.ts b/app/api/health/ready/route.ts new file mode 100644 index 000000000..c352dee6b --- /dev/null +++ b/app/api/health/ready/route.ts @@ -0,0 +1,23 @@ +import { NextResponse } from 'next/server'; +import { readiness } from '@/lib/health/readiness'; + +export const dynamic = 'force-dynamic'; + +/** + * Readiness: 200 when this replica can serve, 503 otherwise. Checks that + * the config is loaded, the shared store (if any) answers, the data dirs are + * writable and Stalwart's session resource answers within 2 s. The body + * names the checks only. Results are reused for 5 s. + * https://bulwarkmail.org/docs/deployment/monitoring#health + */ +const HEADERS = { 'Cache-Control': 'no-store' }; + +export async function GET() { + const result = await readiness(); + return NextResponse.json(result, { status: result.ready ? 200 : 503, headers: HEADERS }); +} + +export async function HEAD() { + const result = await readiness(); + return new NextResponse(null, { status: result.ready ? 200 : 503, headers: HEADERS }); +} diff --git a/app/api/metrics/__tests__/route.test.ts b/app/api/metrics/__tests__/route.test.ts new file mode 100644 index 000000000..1fa22e8d3 --- /dev/null +++ b/app/api/metrics/__tests__/route.test.ts @@ -0,0 +1,55 @@ +// @vitest-environment node +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { NextRequest } from 'next/server'; +import { GET } from '../route'; +import { resetMetricsForTests } from '@/lib/metrics'; + +function scrape(authorization?: string): NextRequest { + return new NextRequest('http://localhost/api/metrics', { headers: authorization ? { authorization } : {} }); +} + +afterEach(() => { + delete process.env.METRICS_TOKEN; + delete process.env.METRICS_TOKEN_FILE; + resetMetricsForTests(); +}); + +describe('GET /api/metrics', () => { + it('does not exist without a token configured', async () => { + const res = await GET(scrape('Bearer anything')); + expect(res.status).toBe(404); + }); + + it('wants the bearer token', async () => { + process.env.METRICS_TOKEN = 'scrape-secret-0123456789'; + expect((await GET(scrape())).status).toBe(401); + expect((await GET(scrape('Bearer wrong'))).status).toBe(401); + expect((await GET(scrape('Basic scrape-secret-0123456789'))).status).toBe(401); + }); + + it('answers Prometheus text with the right token', async () => { + process.env.METRICS_TOKEN = 'scrape-secret-0123456789'; + const res = await GET(scrape('Bearer scrape-secret-0123456789')); + expect(res.status).toBe(200); + expect(res.headers.get('content-type')).toContain('text/plain'); + const body = await res.text(); + expect(body).toContain('# TYPE bulwark_http_requests_total counter'); + expect(body).toContain('bulwark_build_info'); + expect(body).toContain('bulwark_nodejs_eventloop_lag_seconds'); + expect(body).toContain('bulwark_signins_total{outcome="rate_limited"} 0'); + }); + + it('reads the token from METRICS_TOKEN_FILE', async () => { + const dir = mkdtempSync(path.join(tmpdir(), 'bulwark-metrics-')); + try { + writeFileSync(path.join(dir, 'token'), 'from-a-file-0123456789\n'); + process.env.METRICS_TOKEN_FILE = path.join(dir, 'token'); + expect((await GET(scrape('Bearer from-a-file-0123456789'))).status).toBe(200); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/app/api/metrics/route.ts b/app/api/metrics/route.ts new file mode 100644 index 000000000..79bf76f02 --- /dev/null +++ b/app/api/metrics/route.ts @@ -0,0 +1,41 @@ +import { timingSafeEqual } from 'node:crypto'; +import { NextRequest, NextResponse } from 'next/server'; +import { metrics, metricsToken } from '@/lib/metrics'; + +export const dynamic = 'force-dynamic'; + +/** + * GET /api/metrics: Prometheus metrics of this replica. + * + * Off by default: without METRICS_TOKEN (or METRICS_TOKEN_FILE) it answers + * 404 as if it didn't exist. With one, the scraper sends + * `Authorization: Bearer `; anything else gets 401. + * + * https://bulwarkmail.org/docs/deployment/monitoring#metrics + */ + +function sameToken(given: string, expected: string): boolean { + const a = Buffer.from(given); + const b = Buffer.from(expected); + return a.length === b.length && timingSafeEqual(a, b); +} + +export async function GET(request: NextRequest) { + const token = metricsToken(); + if (!token) return new NextResponse('Not Found', { status: 404, headers: { 'Cache-Control': 'no-store' } }); + + const header = request.headers.get('authorization') ?? ''; + const given = header.startsWith('Bearer ') ? header.slice('Bearer '.length).trim() : ''; + if (!given || !sameToken(given, token)) { + return new NextResponse('Unauthorized', { + status: 401, + headers: { 'Cache-Control': 'no-store', 'WWW-Authenticate': 'Bearer realm="metrics"' }, + }); + } + + const { registry } = metrics(); + return new NextResponse(await registry.metrics(), { + status: 200, + headers: { 'Content-Type': registry.contentType, 'Cache-Control': 'no-store' }, + }); +} diff --git a/app/api/translate/route.ts b/app/api/translate/route.ts index c53e0bcb0..232ee0703 100644 --- a/app/api/translate/route.ts +++ b/app/api/translate/route.ts @@ -1,4 +1,5 @@ import { NextRequest, NextResponse } from 'next/server'; +import { observeUpstream } from '@/lib/metrics/upstream'; import { rejectCrossOriginRequest } from '@/lib/security/same-origin'; import { readFileEnv } from "@/lib/read-file-env"; import { getStalwartCredentials } from '@/lib/stalwart/credentials'; @@ -174,10 +175,10 @@ async function mymemoryRequest( url.searchParams.set('q', q); url.searchParams.set('langpair', langpair); - const res = await fetch(url.toString(), { + const res = await observeUpstream('translate', () => fetch(url.toString(), { signal, headers: { 'User-Agent': 'Bulwark-Webmail-Translate (+https://bulwarkmail.org)' }, - }); + })); const data = (await res.json().catch(() => null)) as | { responseStatus?: number | string; responseData?: { translatedText?: string }; responseDetails?: string } | null; @@ -232,7 +233,7 @@ async function translateLibre( process.env.LIBRETRANSLATE_API_KEY || readFileEnv(process.env.LIBRETRANSLATE_API_KEY_FILE); const url = endpoint.replace(/\/+$/, '') + '/translate'; - const res = await fetch(url, { + const res = await observeUpstream('translate', () => fetch(url, { method: 'POST', signal, headers: { 'Content-Type': 'application/json' }, @@ -243,7 +244,7 @@ async function translateLibre( format: 'text', ...(apiKey ? { api_key: apiKey } : {}), }), - }); + })); const data = (await res.json().catch(() => null)) as | { translatedText?: string; detectedLanguage?: { language?: string }; error?: string } | null; diff --git a/app/global-error.tsx b/app/global-error.tsx index b45178af5..3801a6475 100644 --- a/app/global-error.tsx +++ b/app/global-error.tsx @@ -2,6 +2,8 @@ import { useEffect } from "react"; import { AlertTriangle, RefreshCw } from "@/components/icons"; +import { fetchConfig } from "@/hooks/use-config"; +import { captureException } from "@/lib/error-reporting/client"; import "./globals.css"; /** @@ -25,6 +27,10 @@ export default function GlobalError({ }) { useEffect(() => { console.error("Global error:", error); + // Waits for the config when the app crashed before it arrived; sent only + // when error reporting is configured. + captureException(error, "global-error", false); + void fetchConfig().catch(() => undefined); }, [error]); return ( diff --git a/components/admin/admin-shell.tsx b/components/admin/admin-shell.tsx index 9b721d17f..d2003ba8a 100644 --- a/components/admin/admin-shell.tsx +++ b/components/admin/admin-shell.tsx @@ -8,6 +8,7 @@ import { ArrowUpCircle, Activity, Bell, + HeartRateMonitor, Building2, CalendarDays, ChevronRight, @@ -67,6 +68,7 @@ const PAGE_ICONS: Record = { marketplace: Store, version: ArrowUpCircle, telemetry: Activity, + monitoring: HeartRateMonitor, logs: ScrollText, }; diff --git a/components/error/error-boundary.tsx b/components/error/error-boundary.tsx index cd088d3b5..b53e476b0 100644 --- a/components/error/error-boundary.tsx +++ b/components/error/error-boundary.tsx @@ -3,6 +3,7 @@ import React, { Component, ReactNode } from "react"; import { useTranslations } from "next-intl"; import { debug } from "@/lib/debug"; +import { captureException } from "@/lib/error-reporting/client"; export interface FallbackProps { error: Error; @@ -43,6 +44,10 @@ class ErrorBoundaryCore extends Component< componentStack: errorInfo.componentStack, }); + // Off unless error reporting is configured (lib/error-reporting/client.ts). + // The component stack stays here: it names components, nothing else. + captureException(error, "boundary", false); + // Call optional error handler this.props.onError?.(error, errorInfo); } diff --git a/components/error/error-reporting-root.tsx b/components/error/error-reporting-root.tsx new file mode 100644 index 000000000..9f74dfd44 --- /dev/null +++ b/components/error/error-reporting-root.tsx @@ -0,0 +1,30 @@ +"use client"; + +import { useEffect } from "react"; +import { usePathname } from "next/navigation"; +import { fetchConfig } from "@/hooks/use-config"; +import { installGlobalHandlers, recordNavigation } from "@/lib/error-reporting/client"; + +/** + * Starts browser error reporting: window.onerror and unhandled rejections, + * and a breadcrumb per route change (the route's area only, never the URL). + * Whether anything is sent is decided by the config (/api/config, or Lite's + * policy.json); until it arrives, reports wait, and with reporting off they + * are dropped. Renders nothing. + */ +export function ErrorReportingRoot() { + const pathname = usePathname(); + + useEffect(() => { + installGlobalHandlers(); + // Cached after the first fetch; this only makes sure it happens on pages + // that don't read the config themselves (admin, setup). + void fetchConfig().catch(() => undefined); + }, []); + + useEffect(() => { + if (pathname) recordNavigation(pathname); + }, [pathname]); + + return null; +} diff --git a/components/icons/index.ts b/components/icons/index.ts index ad4e220f4..4cbc456ab 100644 --- a/components/icons/index.ts +++ b/components/icons/index.ts @@ -123,6 +123,7 @@ export { IconH1 as Heading1, IconH2 as Heading2, IconHeart as Heart, + IconHeartRateMonitor as HeartRateMonitor, IconHelpCircle as HelpCircle, IconHighlight as Highlighter, IconHome as Home, diff --git a/deploy/demo-deploy.sh b/deploy/demo-deploy.sh index 00a8ea645..2cd8d7d3c 100644 --- a/deploy/demo-deploy.sh +++ b/deploy/demo-deploy.sh @@ -30,6 +30,9 @@ DATA=/opt/bulwark-demo-data # exists only so pruning cannot collect the image we may need to go back to. PREVIOUS=bulwark-demo:previous STATE=/opt/bulwark-demo-deploy +# Optional extra environment for the container (e.g. ERROR_REPORTING_DSN), +# one VAR=value per line. Not in a bind-mounted dir: the app never sees it. +ENV_FILE=/opt/bulwark-demo-data/demo.env log() { echo "[demo-deploy] $*"; } @@ -45,11 +48,14 @@ version_of() { docker_ image inspect -f '{{index .Config.Labels "org.opencontain # host port 3002, and the three bind mounts that keep the demo past the setup # wizard (admin/config.json holds setupComplete). run_container() { + local env_args=() + if [ -f "$ENV_FILE" ]; then env_args=(--env-file "$ENV_FILE"); fi docker_ rm -f "$NAME" >/dev/null 2>&1 || true docker_ run -d \ --name "$NAME" \ --restart unless-stopped \ -p "${PORT}:3000" \ + ${env_args[@]+"${env_args[@]}"} \ -e DEMO_MODE=true \ -v "$DATA/admin:/app/data/admin" \ -v "$DATA/settings:/app/data/settings" \ diff --git a/deploy/kubernetes/base/bulwark.env b/deploy/kubernetes/base/bulwark.env new file mode 100644 index 000000000..aa88c979d --- /dev/null +++ b/deploy/kubernetes/base/bulwark.env @@ -0,0 +1,13 @@ +# Settings for every Bulwark replica (no secrets here; those go in secrets.env). +# Any variable from https://bulwarkmail.org/docs/getting-started/configuration/environment-reference +# can be added. + +# The JMAP server users sign in to. Both the pods and users' browsers must reach it. +JMAP_SERVER_URL=https://mail.example.com + +# Proxies in front of the pods: 1 for an ingress controller alone, 2 if a cloud +# load balancer sits in front of it. Sign-in limits key on the address this picks. +TRUSTED_PROXY_DEPTH=1 + +SETTINGS_SYNC_ENABLED=true +LOG_LEVEL=info diff --git a/deploy/kubernetes/base/bulwark.yaml b/deploy/kubernetes/base/bulwark.yaml new file mode 100644 index 000000000..5d40e9c12 --- /dev/null +++ b/deploy/kubernetes/base/bulwark.yaml @@ -0,0 +1,144 @@ +# Bulwark itself: any number of replicas, no volumes. Everything they share +# (admin config, plugins, branding, synced settings, sign-in limits, sessions) +# lives in the Valkey store; see https://bulwarkmail.org/docs/deployment/multi-instance +apiVersion: v1 +kind: Service +metadata: + name: bulwark + labels: + app.kubernetes.io/name: bulwark +spec: + selector: + app.kubernetes.io/name: bulwark + ports: + - name: http + port: 80 + targetPort: http +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: bulwark + labels: + app.kubernetes.io/name: bulwark +spec: + replicas: 3 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + selector: + matchLabels: + app.kubernetes.io/name: bulwark + template: + metadata: + labels: + app.kubernetes.io/name: bulwark + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: 1001 + seccompProfile: + type: RuntimeDefault + # Spread the replicas over nodes where there are several. + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/name: bulwark + terminationGracePeriodSeconds: 30 + containers: + - name: bulwark + image: ghcr.io/bulwarkmail/webmail + ports: + - name: http + containerPort: 3000 + envFrom: + - configMapRef: + name: bulwark-env + env: + - name: SESSION_SECRET + valueFrom: + secretKeyRef: + name: bulwark-secrets + key: SESSION_SECRET + - name: ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: bulwark-secrets + key: ADMIN_PASSWORD + - name: VALKEY_PASSWORD + valueFrom: + secretKeyRef: + name: bulwark-secrets + key: VALKEY_PASSWORD + # Kubernetes fills in $(VALKEY_PASSWORD) from the variable above. + - name: REDIS_URL + value: redis://:$(VALKEY_PASSWORD)@valkey:6379/0 + # /api/health only says the process runs; /api/ready also says it + # finished starting and reaches the store. A store outage takes the + # pods out of the Service without restarting them. + startupProbe: + httpGet: + path: /api/health + port: http + periodSeconds: 2 + failureThreshold: 60 + readinessProbe: + httpGet: + path: /api/ready + port: http + periodSeconds: 5 + failureThreshold: 2 + livenessProbe: + httpGet: + path: /api/health + port: http + periodSeconds: 10 + failureThreshold: 3 + lifecycle: + # Let the Service drop the pod before it stops taking requests. + preStop: + exec: + command: ["sleep", "5"] + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 768Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + # Scratch space only; nothing here needs to survive the pod. + - name: data + mountPath: /app/data + - name: next-cache + mountPath: /app/.next/cache + - name: tmp + mountPath: /tmp + volumes: + - name: data + emptyDir: {} + - name: next-cache + emptyDir: {} + - name: tmp + emptyDir: {} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: bulwark +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: bulwark diff --git a/deploy/kubernetes/base/kustomization.yaml b/deploy/kubernetes/base/kustomization.yaml new file mode 100644 index 000000000..e7b76c1ce --- /dev/null +++ b/deploy/kubernetes/base/kustomization.yaml @@ -0,0 +1,32 @@ +# The base the top-level kustomization.yaml and the overlays use. Change the +# namespace, the image tag (pin a release, never latest in production) and +# the replica count here or, better, in an overlay. secrets.env must exist +# next to this file (copy secrets.env.example). +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: bulwark + +resources: + - namespace.yaml + - valkey.yaml + - bulwark.yaml + +configMapGenerator: + - name: bulwark-env + envs: + - bulwark.env + +secretGenerator: + - name: bulwark-secrets + envs: + - secrets.env + +images: + - name: ghcr.io/bulwarkmail/webmail + newTag: latest + +labels: + - pairs: + app.kubernetes.io/part-of: bulwark + includeSelectors: false diff --git a/deploy/kubernetes/base/namespace.yaml b/deploy/kubernetes/base/namespace.yaml new file mode 100644 index 000000000..ff5226df2 --- /dev/null +++ b/deploy/kubernetes/base/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: bulwark diff --git a/deploy/kubernetes/base/secrets.env.example b/deploy/kubernetes/base/secrets.env.example new file mode 100644 index 000000000..24cb08cf1 --- /dev/null +++ b/deploy/kubernetes/base/secrets.env.example @@ -0,0 +1,13 @@ +# Copy to secrets.env (kept out of git) and fill in. Generate each with: +# openssl rand -base64 32 +# Or delete the secretGenerator in kustomization.yaml and create the Secret +# `bulwark-secrets` with these three keys some other way (External Secrets, +# Sealed Secrets, your CI). + +# Encrypts sessions, synced settings and plugin credentials. The same on every +# replica; changing it signs everyone out. +SESSION_SECRET= +# First admin password; hashed into the store on the first start. +ADMIN_PASSWORD= +# Password of the Valkey store. +VALKEY_PASSWORD= diff --git a/deploy/kubernetes/base/valkey.yaml b/deploy/kubernetes/base/valkey.yaml new file mode 100644 index 000000000..debc8bb92 --- /dev/null +++ b/deploy/kubernetes/base/valkey.yaml @@ -0,0 +1,95 @@ +# The shared store every Bulwark replica uses. One Valkey with persistence +# (AOF) and noeviction: an evicting policy would silently drop settings and +# admin config under memory pressure. A managed Redis or Valkey works as well; +# then delete this file from kustomization.yaml and point REDIS_URL at it. +apiVersion: v1 +kind: Service +metadata: + name: valkey + labels: + app.kubernetes.io/name: valkey +spec: + clusterIP: None + selector: + app.kubernetes.io/name: valkey + ports: + - name: redis + port: 6379 + targetPort: redis +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: valkey + labels: + app.kubernetes.io/name: valkey +spec: + serviceName: valkey + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: valkey + template: + metadata: + labels: + app.kubernetes.io/name: valkey + spec: + securityContext: + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + fsGroup: 999 + seccompProfile: + type: RuntimeDefault + containers: + - name: valkey + image: valkey/valkey:8-alpine + command: ["sh", "-c"] + args: + - >- + exec valkey-server + --requirepass "$VALKEY_PASSWORD" + --appendonly yes + --maxmemory 512mb + --maxmemory-policy noeviction + --dir /data + env: + - name: VALKEY_PASSWORD + valueFrom: + secretKeyRef: + name: bulwark-secrets + key: VALKEY_PASSWORD + ports: + - name: redis + containerPort: 6379 + readinessProbe: + exec: + command: ["sh", "-c", "REDISCLI_AUTH=\"$VALKEY_PASSWORD\" valkey-cli --no-auth-warning ping | grep -q PONG"] + periodSeconds: 5 + livenessProbe: + exec: + command: ["sh", "-c", "REDISCLI_AUTH=\"$VALKEY_PASSWORD\" valkey-cli --no-auth-warning ping | grep -q PONG"] + periodSeconds: 10 + failureThreshold: 6 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + memory: 640Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: data + mountPath: /data + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi diff --git a/deploy/kubernetes/ingress.example.yaml b/deploy/kubernetes/ingress.example.yaml new file mode 100644 index 000000000..fa94692a3 --- /dev/null +++ b/deploy/kubernetes/ingress.example.yaml @@ -0,0 +1,28 @@ +# Not applied by kustomization.yaml: adapt and apply it yourself, or use a +# Gateway API HTTPRoute instead. Plain round robin is fine; Bulwark needs no +# sticky sessions. Set TRUSTED_PROXY_DEPTH in bulwark.env to the number of +# proxies in front of the pods (1 for the ingress controller alone). +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: bulwark + namespace: bulwark + annotations: + # Attachments and plugin packages can be large. + nginx.ingress.kubernetes.io/proxy-body-size: 100m +spec: + ingressClassName: nginx + tls: + - hosts: [webmail.example.com] + secretName: bulwark-tls + rules: + - host: webmail.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: bulwark + port: + name: http diff --git a/deploy/kubernetes/kustomization.yaml b/deploy/kubernetes/kustomization.yaml new file mode 100644 index 000000000..006ce308b --- /dev/null +++ b/deploy/kubernetes/kustomization.yaml @@ -0,0 +1,14 @@ +# Bulwark Webmail on Kubernetes: three replicas sharing one Valkey store. +# https://bulwarkmail.org/docs/deployment/kubernetes +# +# cp base/secrets.env.example base/secrets.env # fill in the three secrets +# $EDITOR base/bulwark.env # at least JMAP_SERVER_URL +# kubectl apply -k . # or: -k overlays/config-in-git +# +# For your own changes (namespace, image tag, replicas, resources) add an +# overlay next to overlays/config-in-git that uses ../../base. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - base diff --git a/deploy/kubernetes/overlays/config-in-git/config-from-files.yaml b/deploy/kubernetes/overlays/config-in-git/config-from-files.yaml new file mode 100644 index 000000000..fe91e62f7 --- /dev/null +++ b/deploy/kubernetes/overlays/config-in-git/config-from-files.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: bulwark +spec: + template: + spec: + containers: + - name: bulwark + env: + - name: ADMIN_CONFIG_READONLY + value: "true" + volumeMounts: + - name: admin-config + mountPath: /app/data/admin + readOnly: true + volumes: + - name: admin-config + configMap: + name: bulwark-admin-config diff --git a/deploy/kubernetes/overlays/config-in-git/config.json b/deploy/kubernetes/overlays/config-in-git/config.json new file mode 100644 index 000000000..e0754da49 --- /dev/null +++ b/deploy/kubernetes/overlays/config-in-git/config.json @@ -0,0 +1,4 @@ +{ + "appName": "Webmail", + "setupComplete": true +} diff --git a/deploy/kubernetes/overlays/config-in-git/kustomization.yaml b/deploy/kubernetes/overlays/config-in-git/kustomization.yaml new file mode 100644 index 000000000..9f604dba2 --- /dev/null +++ b/deploy/kubernetes/overlays/config-in-git/kustomization.yaml @@ -0,0 +1,23 @@ +# Admin config and user policy from files in git instead of the admin console. +# Every replica reads config.json and policy.json from the ConfigMap and picks +# up a change within a few seconds; the admin console shows them but can't +# save them. Sessions, settings and the audit log still live in the store. +# +# kubectl apply -k overlays/config-in-git +# +# To start from your current policy, use "Export policy.json" on the admin +# console's Domains and groups page. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +configMapGenerator: + - name: bulwark-admin-config + files: + - config.json + - policy.json + +patches: + - path: config-from-files.yaml diff --git a/deploy/kubernetes/overlays/config-in-git/policy.json b/deploy/kubernetes/overlays/config-in-git/policy.json new file mode 100644 index 000000000..d5a66620e --- /dev/null +++ b/deploy/kubernetes/overlays/config-in-git/policy.json @@ -0,0 +1,5 @@ +{ + "restrictions": {}, + "features": {}, + "defaults": {} +} diff --git a/deploy/kubernetes/test/smoke.sh b/deploy/kubernetes/test/smoke.sh new file mode 100644 index 000000000..9d097aaa4 --- /dev/null +++ b/deploy/kubernetes/test/smoke.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# Smoke test for a running deployment of deploy/kubernetes: +# - Valkey and every Bulwark replica become ready, on the shared store; +# - an admin session made on one replica is valid on every other one; +# - a config change saved on one replica reaches the others within 5 s; +# - deleting a pod and a rolling restart don't fail requests through the +# Service, and the admin session survives the new pods. +# +# Usage: ADMIN_PASSWORD=... deploy/kubernetes/test/smoke.sh [namespace] +# With READ_ONLY_CONFIG=1 (overlays/config-in-git) it checks that config saves +# are refused instead of spread. +# Uses the current kubectl context. The password goes to curl on stdin, never +# on a command line. +set -euo pipefail + +NS="${1:-bulwark}" +: "${ADMIN_PASSWORD:?set ADMIN_PASSWORD to the admin password of the deployment}" +k() { kubectl -n "$NS" "$@"; } +fail() { echo "FAIL: $*" >&2; exit 1; } +step() { echo "== $*"; } + +step "rollout" +k rollout status statefulset/valkey --timeout=300s +k rollout status deployment/bulwark --timeout=600s + +step "curl pod" +if ! k get pod smoke-curl >/dev/null 2>&1; then + k run smoke-curl --image=curlimages/curl:8.11.1 --restart=Never --labels=app.kubernetes.io/name=smoke-curl \ + --command -- sleep 1800 >/dev/null +fi +k wait pod/smoke-curl --for=condition=Ready --timeout=180s >/dev/null +c() { k exec -i smoke-curl -- curl -sS --max-time 10 "$@"; } +status() { k exec smoke-curl -- curl -s -o /dev/null -w '%{http_code}' --max-time 5 "$@" || echo 000; } + +pod_ips() { k get pods -l app.kubernetes.io/name=bulwark --field-selector=status.phase=Running -o jsonpath='{.items[*].status.podIP}'; } +read -r -a IPS <<<"$(pod_ips)" +[ "${#IPS[@]}" -ge 2 ] || fail "need at least 2 replicas, found ${#IPS[@]}" +echo "replicas: ${IPS[*]}" + +step "every replica is ready on the shared store" +for ip in "${IPS[@]}"; do + out="$(c "http://$ip:3000/api/ready")" + echo "$ip $out" + grep -q '"ready":true' <<<"$out" || fail "$ip not ready" + grep -q '"store":"redis"' <<<"$out" || fail "$ip not on the shared store" +done + +step "an admin session made on one replica works on every replica" +json_string() { + local s="${1//\\/\\\\}" + s="${s//\"/\\\"}" + printf '"%s"' "$s" +} +code="$(printf '{"password":%s}' "$(json_string "$ADMIN_PASSWORD")" \ + | c -o /dev/null -w '%{http_code}' -c /tmp/jar -H 'Content-Type: application/json' --data-binary @- \ + "http://${IPS[0]}:3000/api/admin/auth")" +[ "$code" = 200 ] || fail "admin sign-in on ${IPS[0]} answered $code" +# curl's jar keeps cookies per host and every pod has its own address, so the +# cookie goes in a header. It stays in the curl pod, off every command line. +k exec smoke-curl -- sh -c "awk '{ sub(/^#HttpOnly_/, \"\") } !/^#/ && NF >= 7 { printf \"%s=%s; \", \$6, \$7 }' /tmp/jar > /tmp/cookie" +ca() { k exec -i smoke-curl -- sh -c 'curl -sS --max-time 10 -H "Cookie: $(cat /tmp/cookie)" "$@"' sh "$@"; } +for ip in "${IPS[@]}"; do + ca "http://$ip:3000/api/admin/auth" /dev/null; do + total=$((total + 1)) + [ "$(status "$SVC")" = 200 ] || failures=$((failures + 1)) + sleep 1 + done + wait "$pid" + echo "requests: $total, failed: $failures" + [ "$failures" -eq 0 ] || fail "$failures requests failed" +} + +step "deleting a pod keeps the Service answering" +victim="$(k get pods -l app.kubernetes.io/name=bulwark -o jsonpath='{.items[0].metadata.name}')" +probe_while k delete pod "$victim" --wait=true +k rollout status deployment/bulwark --timeout=300s + +step "a rolling restart keeps the Service answering" +k rollout restart deployment/bulwark +probe_while k rollout status deployment/bulwark --timeout=600s + +step "the admin session survives the new pods" +ca "http://bulwark.$NS.svc/api/admin/auth" /dev/null +echo "OK" diff --git a/hooks/use-config.ts b/hooks/use-config.ts index 77329854a..d54a208d7 100644 --- a/hooks/use-config.ts +++ b/hooks/use-config.ts @@ -8,6 +8,7 @@ import { IS_LITE, IS_LITE_STALWART, LITE_CONFIG_PATH, withLiteBuildId } from '@/ import { applyLiteConfig, liteStalwartDefaults } from '@/lib/lite-config'; import { isConfigData } from '@/lib/config-validation'; import { setNoThirdPartyTraffic } from '@/lib/third-party-mode'; +import { configureErrorReporting, type ClientSettings } from '@/lib/error-reporting/client'; export interface ConfigData { appName: string; @@ -59,6 +60,8 @@ export interface ConfigData { noThirdPartyTraffic: boolean; /** Lite only: whether sign-out revokes the refresh token (#1388). The server build decides in its token route. */ oauthRevokeOnSignOut?: boolean; + /** Server build only; Lite reads it from policy.json (stores/policy-store.ts). */ + errorReporting?: ClientSettings; } interface AppConfig extends ConfigData { @@ -144,6 +147,8 @@ export async function fetchConfig(): Promise { .then((data) => { configCache = data; setNoThirdPartyTraffic(data.noThirdPartyTraffic === true); + // Lite configures error reporting from policy.json instead. + if (!IS_LITE) configureErrorReporting(data.errorReporting ?? null); // Fetch admin policy alongside config (non-blocking) usePolicyStore.getState().fetchPolicy(); return data; diff --git a/instrumentation.node.ts b/instrumentation.node.ts index 605628441..a1f36ff11 100644 --- a/instrumentation.node.ts +++ b/instrumentation.node.ts @@ -9,12 +9,21 @@ import { bootState, getStore, sharedStoreUrl } from "./lib/shared-store"; import { bootSharedMode } from "./lib/shared-store/boot"; import { importLocalData } from "./lib/shared-store/import"; import { exportSharedData } from "./lib/shared-store/export"; +import { logger, isJsonLogFormat } from "./lib/logger"; const pkg = JSON.parse( readFileSync(`${process.cwd()}/package.json`, "utf-8") ); const current: string = pkg.version ?? "0.0.0"; -console.info(`Bulwark Webmail v${current}`); +logger.info(`Bulwark Webmail v${current}`, { version: current }); + +// Prometheus metrics, only with METRICS_TOKEN(_FILE): count every request +// from the first one on. https://bulwarkmail.org/docs/deployment/monitoring#metrics +if (process.env.METRICS_TOKEN?.trim() || process.env.METRICS_TOKEN_FILE?.trim()) { + void import("./lib/metrics/http").then(({ installHttpMetrics }) => { + if (installHttpMetrics()) logger.info("Metrics endpoint on at /api/metrics"); + }); +} // With REDIS_URL every replica shares one store (#1066). Without it, // everything below runs exactly as it always has. @@ -36,12 +45,12 @@ async function exportAndExit(): Promise { await store.ping(2000); break; } catch { - console.warn("Export: waiting for the shared store"); + logger.warn("Export: waiting for the shared store"); await new Promise((resolve) => setTimeout(resolve, 2000)); } } const written = await exportSharedData(); - console.info("Export from the shared store done; start without REDIS_URL on this volume now.", written); + logger.info("Export from the shared store done; start without REDIS_URL on this volume now.", { written }); process.exit(0); } @@ -57,18 +66,23 @@ migrateLegacyAdminLayout() .then(() => migratePolicyUnifiedMailbox()) .then(() => (exportOnly ? exportAndExit() : shared ? bootShared() : loadAdmin())) .then(async () => { - console.info("Admin dashboard initialized"); + logger.info("Admin dashboard initialized"); + // Errors routes catch and log, and errors Next reports (instrumentation.ts), + // go to the error tracker when one is configured. Off otherwise. + // https://bulwarkmail.org/docs/deployment/monitoring + const { installServerErrorReporting } = await import("./lib/error-reporting/server"); + await installServerErrorReporting(); // With a shared store, boot already made or loaded the secret. if (!shared) { try { if (await ensureSessionSecret()) { - console.info( + logger.info( "Generated a session secret in the admin state dir. Keep that dir on a " + "persistent volume, or set SESSION_SECRET, so restarts don't sign everyone out.", ); } } catch (err) { - console.warn("Failed to generate a session secret:", err instanceof Error ? err.message : err); + logger.warn("Failed to generate a session secret", { error: err instanceof Error ? err.message : String(err) }); } } // If we're in bootstrap state (no JMAP_SERVER_URL env and no @@ -79,20 +93,22 @@ migrateLegacyAdminLayout() try { const token = await ensureSetupToken(); const port = process.env.PORT || "3000"; - console.info(""); - console.info("=============================================================="); - console.info(" SETUP REQUIRED"); - console.info(` Token: ${token}`); // In the fragment, so the token stays out of access logs and Referer. - console.info(` Open: http://:${port}/setup#token=${token}`); - console.info(" Token expires in 1 hour. Restart the container to reissue."); - console.info("=============================================================="); - console.info(""); + const open = `http://:${port}/setup#token=${token}`; + if (isJsonLogFormat()) { + logger.info("Setup required", { token, open, expiresIn: "1h" }); + } else { + for (const line of [ + "==============================================================", + " SETUP REQUIRED", + ` Token: ${token}`, + ` Open: ${open}`, + " Token expires in 1 hour. Restart the container to reissue.", + "==============================================================", + ]) logger.info(line); + } } catch (err) { - console.warn( - "Failed to issue setup token:", - err instanceof Error ? err.message : err, - ); + logger.warn("Failed to issue setup token", { error: err instanceof Error ? err.message : String(err) }); } } }) @@ -103,18 +119,22 @@ migrateLegacyAdminLayout() // See https://bulwarkmail.org/docs/legal/privacy/telemetry const { startScheduler, stopScheduler, markProcessStart } = await import("./lib/telemetry"); markProcessStart(); + // Instant error counts: a one-minute tick that sends only while the + // error-count consent is "instant". + const instantErrors = await import("./lib/telemetry/instant-errors"); // Hourly check against version.telemetry.bulwarkmail.org. Disable with // BULWARK_UPDATE_CHECK=off or override the endpoint with // BULWARK_UPDATE_CHECK_URL. const versionCheck = await import("./lib/version-check"); - // Both run on one replica of the deployment, the scheduler leader + // All run on one replica of the deployment, the scheduler leader // (#1066); a single instance always leads. const { startSchedulerLeader } = await import("./lib/scheduler-leader"); await startSchedulerLeader([ { name: "telemetry", start: startScheduler, stop: stopScheduler }, + { name: "telemetry-errors", start: instantErrors.startInstantErrors, stop: instantErrors.stopInstantErrors }, { name: "version-check", start: versionCheck.startScheduler, stop: versionCheck.stopScheduler }, ]); }) .catch((err) => { - console.warn("Admin dashboard init skipped:", err instanceof Error ? err.message : err); + logger.warn("Admin dashboard init skipped", { error: err instanceof Error ? err.message : String(err) }); }); diff --git a/instrumentation.ts b/instrumentation.ts index 41630756b..3f9bcba91 100644 --- a/instrumentation.ts +++ b/instrumentation.ts @@ -1,5 +1,28 @@ +import type { Instrumentation } from "next"; +import { tracingConfigured } from "./lib/tracing/config"; + export async function register() { if (process.env.NEXT_RUNTIME === "nodejs") { + // OpenTelemetry, only with an OTLP endpoint in the standard OTEL_* + // variables; otherwise the SDK is never loaded. First, so Next's own + // spans from the first request on go to it. + // https://bulwarkmail.org/docs/deployment/monitoring#tracing + if (tracingConfigured()) { + const { startTracing } = await import("./lib/tracing/start"); + startTracing(); + } await import("./instrumentation.node"); } } + +/** + * Errors Next caught while rendering a page or running a route handler, + * server action or the proxy. Counted, and sent to the error tracker when + * one is configured (lib/error-reporting/server.ts); nothing happens while + * error reporting is off. https://bulwarkmail.org/docs/deployment/monitoring + */ +export const onRequestError: Instrumentation.onRequestError = async (error, _request, context) => { + if (process.env.NEXT_RUNTIME !== "nodejs") return; + const { captureRequestError } = await import("./lib/error-reporting/server"); + await captureRequestError(error, context.routePath); +}; diff --git a/lib/__tests__/config-validation.test.ts b/lib/__tests__/config-validation.test.ts index b2737ee7e..626e575ce 100644 --- a/lib/__tests__/config-validation.test.ts +++ b/lib/__tests__/config-validation.test.ts @@ -107,3 +107,18 @@ describe('server configuration validation', () => { expect(isConfigData({ ...createConfig(), jmapServers: [entry] })).toBe(false); }); }); + +describe('error reporting settings in the server configuration', () => { + const settings = { dsn: '', sampleRate: 1, environment: 'production', collect: false }; + + it('are optional, for Lite and servers before they existed', () => { + expect(isConfigData({ ...createConfig(), errorReporting: settings })).toBe(true); + expect(isConfigData(createConfig())).toBe(true); + }); + + it('are refused when malformed', () => { + expect(isConfigData({ ...createConfig(), errorReporting: { ...settings, sampleRate: '1' } })).toBe(false); + expect(isConfigData({ ...createConfig(), errorReporting: { ...settings, collect: 'yes' } })).toBe(false); + expect(isConfigData({ ...createConfig(), errorReporting: 'https://key@x.example/1' })).toBe(false); + }); +}); diff --git a/lib/__tests__/jmap-error-reporting.test.ts b/lib/__tests__/jmap-error-reporting.test.ts new file mode 100644 index 000000000..6749e7083 --- /dev/null +++ b/lib/__tests__/jmap-error-reporting.test.ts @@ -0,0 +1,58 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const reported = vi.hoisted(() => ({ calls: [] as string[][], failures: [] as unknown[] })); +vi.mock('@/lib/error-reporting/client', () => ({ + recordJmapCall: (methods: string[]) => { reported.calls.push(methods); }, + reportJmapFailure: (failure: unknown) => { reported.failures.push(failure); }, +})); + +import { JMAPClient } from '../jmap/client'; + +type Call = [string, Record, string]; +const SUBJECT = 'Merger plans for Acme Q3'; + +function clientAnswering(response: Response) { + const client = new JMAPClient('https://jmap.example.com', 'user@example.com', 'pass'); + Object.assign(client, { apiUrl: 'https://jmap.example.com/jmap/', accountId: 'me' }); + vi.spyOn(client as unknown as { authenticatedFetch: () => Promise }, 'authenticatedFetch').mockResolvedValue(response); + const request = (calls: Call[]) => (client as unknown as { request: (c: Call[]) => Promise }).request(calls); + return request; +} + +beforeEach(() => { + reported.calls = []; + reported.failures = []; +}); + +describe('JMAP requests and error reporting', () => { + it('keeps the method names of each request as a breadcrumb', async () => { + const request = clientAnswering(new Response(JSON.stringify({ methodResponses: [] }))); + await request([['Email/query', { filter: { subject: SUBJECT } }, '0'], ['Email/get', {}, '1']]); + expect(reported.calls).toEqual([['Email/query', 'Email/get']]); + expect(reported.failures).toEqual([]); + }); + + it('reports a failed request by its status only', async () => { + const request = clientAnswering(new Response(`{"detail":"${SUBJECT}"}`, { status: 503 })); + await expect(request([['Email/get', {}, '0']])).rejects.toThrow(); + expect(reported.failures).toEqual([{ status: 503 }]); + }); + + it('does not report an expired token or the concurrency limit', async () => { + const request = clientAnswering(new Response('', { status: 401 })); + await expect(request([['Email/get', {}, '0']])).rejects.toThrow(); + expect(reported.failures).toEqual([]); + }); + + it('reports method errors by method and type, but not the ones that only mean "sync again"', async () => { + const request = clientAnswering(new Response(JSON.stringify({ + methodResponses: [ + ['error', { type: 'overQuota', description: SUBJECT }, 'a'], + ['error', { type: 'cannotCalculateChanges' }, 'b'], + ['Mailbox/get', { list: [] }, 'c'], + ], + }))); + await request([['Email/set', {}, 'a'], ['Email/changes', {}, 'b'], ['Mailbox/get', {}, 'c']]); + expect(reported.failures).toEqual([{ method: 'Email/set', type: 'overQuota' }]); + }); +}); diff --git a/lib/__tests__/route-groups.test.ts b/lib/__tests__/route-groups.test.ts new file mode 100644 index 000000000..fa54b744a --- /dev/null +++ b/lib/__tests__/route-groups.test.ts @@ -0,0 +1,21 @@ +// @vitest-environment node +import { readdirSync, statSync } from 'node:fs'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { API_GROUPS, PAGE_AREAS } from '@/lib/route-groups'; + +const ROOT = path.resolve(__dirname, '..', '..'); +const dirs = (rel: string) => readdirSync(path.join(ROOT, rel)).filter((name) => statSync(path.join(ROOT, rel, name)).isDirectory()); + +// The groups are a fixed list so metric labels and report routes stay +// bounded; a new API folder or page area has to be added to it. +describe('route groups', () => { + it('list every API folder', () => { + expect([...API_GROUPS].sort()).toEqual(dirs('app/api').filter((d) => d !== '__tests__').sort()); + }); + + it('list every page area', () => { + const areas = dirs('app/(main)/[locale]').filter((d) => !d.startsWith('[') && !d.startsWith('(')); + for (const area of areas) expect(PAGE_AREAS).toContain(area); + }); +}); diff --git a/lib/__tests__/sourcemaps-script.test.ts b/lib/__tests__/sourcemaps-script.test.ts new file mode 100644 index 000000000..16beb45bc --- /dev/null +++ b/lib/__tests__/sourcemaps-script.test.ts @@ -0,0 +1,111 @@ +// @vitest-environment node +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { gzipSync } from 'node:zlib'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { check, collect, mapReferences } from '../../scripts/sourcemaps.mjs'; +import { buildZip } from '../../scripts/lite/lib.mjs'; + +// CI builds source maps for the error tracker; none may ship (images, the +// standalone tarball, Lite zips). + +let dir: string; +const put = (rel: string, text: string) => { + const file = path.join(dir, rel); + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync(file, text); +}; + +beforeEach(() => { dir = mkdtempSync(path.join(tmpdir(), 'bulwark-maps-')); }); +afterEach(() => { rmSync(dir, { recursive: true, force: true }); }); + +function tar(files: Record): Buffer { + const blocks: Buffer[] = []; + for (const [name, text] of Object.entries(files)) { + const data = Buffer.from(text); + const header = Buffer.alloc(512); + header.write(name, 0); + header.write('0000644\0', 100); + header.write(data.length.toString(8).padStart(11, '0') + '\0', 124); + header.write('0', 156); + header.write('ustar\0', 257); + blocks.push(header, data, Buffer.alloc((512 - (data.length % 512)) % 512)); + } + blocks.push(Buffer.alloc(1024)); + return Buffer.concat(blocks); +} + +describe('collect', () => { + it('moves maps out with the chunks that name them, and strips the comment from what ships', () => { + put('static/chunks/a.js', 'console.log(1);\n//# debugId=bc67f7db-7f3b-7be2-6f94-966cd7825cc2\n//# sourceMappingURL=a1b2.js.map\n'); + put('static/chunks/a1b2.js.map', '{"version":3}'); + put('static/css/x.css', 'a{}\n/*# sourceMappingURL=x.css.map */\n'); + put('static/css/x.css.map', '{"version":3}'); + put('static/chunks/orphan.js.map', '{"version":3}'); + const out = path.join(dir, 'maps'); + + const moved = collect([path.join(dir, 'static')], out); + + expect(moved).toHaveLength(3); + expect(existsSync(path.join(dir, 'static/chunks/a1b2.js.map'))).toBe(false); + const shipped = readFileSync(path.join(dir, 'static/chunks/a.js'), 'utf8'); + expect(mapReferences(shipped)).toEqual([]); + expect(shipped).toContain('//# debugId=bc67f7db-7f3b-7be2-6f94-966cd7825cc2'); + expect(readFileSync(path.join(out, '0/chunks/a.js'), 'utf8')).toContain('sourceMappingURL=a1b2.js.map'); + expect(existsSync(path.join(out, '0/chunks/a1b2.js.map'))).toBe(true); + expect(existsSync(path.join(out, '0/chunks/orphan.js.map'))).toBe(true); + expect(check(path.join(dir, 'static'))).toEqual([]); + }); +}); + +describe('collect with a relative root', () => { + it('still copies the chunk that names each map', () => { + put('rel/chunks/b.js', 'x\n//# sourceMappingURL=b.js.map\n'); + put('rel/chunks/b.js.map', '{"version":3}'); + const cwd = process.cwd(); + process.chdir(dir); + try { + collect(['rel'], 'out'); + } finally { + process.chdir(cwd); + } + expect(existsSync(path.join(dir, 'out/0/chunks/b.js'))).toBe(true); + expect(existsSync(path.join(dir, 'out/0/chunks/b.js.map'))).toBe(true); + expect(mapReferences(readFileSync(path.join(dir, 'rel/chunks/b.js'), 'utf8'))).toEqual([]); + }); +}); + +describe('check', () => { + it('finds map files, dangling references and inline maps in a folder', () => { + put('a/x.js.map', '{}'); + put('a/y.js', 'x\n//# sourceMappingURL=gone.js.map\n'); + put('a/z.js', 'x\n//# sourceMappingURL=data:application/json;base64,e30=\n'); + put('a/ok.js', 'const s = "//# sourceMappingURL=in-a-string.map";\n'); + expect(check(path.join(dir, 'a')).sort()).toEqual([ + 'x.js.map: source map file', + 'y.js: sourceMappingURL to missing gone.js.map', + 'z.js: inline source map', + ]); + }); + + it('reads zip archives', () => { + const zip = path.join(dir, 'lite.zip'); + writeFileSync(zip, buildZip([ + { name: '_next/static/chunks/a.js', data: Buffer.from('x\n//# sourceMappingURL=a.js.map\n') }, + { name: '_next/static/chunks/a.js.map', data: Buffer.from('{}') }, + ])); + expect(check(zip)).toEqual(['_next/static/chunks/a.js.map: source map file']); + writeFileSync(zip, buildZip([{ name: 'index.html', data: Buffer.from('') }])); + expect(check(zip)).toEqual([]); + }); + + it('reads gzipped tarballs', () => { + const tgz = path.join(dir, 'standalone.tar.gz'); + writeFileSync(tgz, gzipSync(tar({ + 'bulwark-standalone/.next/server/chunks/1.js': 'x\n//# sourceMappingURL=1.js.map\n', + 'bulwark-standalone/.next/server/chunks/1.js.map': '{}', + }))); + expect(check(tgz)).toEqual(['bulwark-standalone/.next/server/chunks/1.js.map: source map file']); + }); +}); diff --git a/lib/__tests__/telemetry-error-counts.test.ts b/lib/__tests__/telemetry-error-counts.test.ts new file mode 100644 index 000000000..5868c0442 --- /dev/null +++ b/lib/__tests__/telemetry-error-counts.test.ts @@ -0,0 +1,166 @@ +// @vitest-environment node +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { NextRequest } from 'next/server'; + +// Error counts in the heartbeat (schema 2) have their own consent, off by +// default and off whenever telemetry is off. Without it the heartbeat is +// exactly schema 1. + +const thirdParty = vi.hoisted(() => ({ on: false })); +vi.mock('@/lib/admin/third-party-mode', () => ({ isNoThirdPartyTrafficOn: async () => thirdParty.on })); +vi.mock('@/lib/admin/session', () => ({ + requireAdminAuth: vi.fn(async () => ({ session: { sub: 'admin' } })), + getClientIP: () => '127.0.0.1', +})); +vi.mock('@/lib/admin/audit', () => ({ auditLog: vi.fn(async () => {}) })); +vi.mock('@/lib/admin/plugin-registry', () => ({ + getPluginRegistry: async () => ({ plugins: [] }), + getThemeRegistry: async () => ({ themes: [] }), +})); + +import { GET, POST } from '@/app/api/admin/telemetry/route'; +import { buildPayload, effectiveErrorsConsent, errorsForPayload, loadState, saveState } from '@/lib/telemetry'; +import { resetAggregateForTests, utcDay, type Aggregate } from '@/lib/error-reporting/aggregate'; + +let dir: string; +const DAY = 86_400_000; +const NOW = Date.parse('2026-11-10T12:00:00Z'); +const debugId = 'bc67f7db-7f3b-7be2-6f94-966cd7825cc2'; + +function post(body: Record) { + return POST(new NextRequest('http://localhost/api/admin/telemetry', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + })); +} + +function entry(count: number, extra: Partial = {}) { + return { + name: 'TypeError', + frames: [ + { path: '/_next/static/chunks/1.js', line: 2, column: 30, debugId }, + { path: 'node_modules/next/dist/x.js', line: 1, column: 1 }, + { path: '[other]' }, + ], + version: '1.20.0', + runtime: 'browser' as const, + count, + ...extra, + }; +} + +beforeAll(() => { + dir = mkdtempSync(path.join(tmpdir(), 'bulwark-telemetry-errors-')); + vi.stubEnv('ADMIN_CONFIG_DIR', dir); + vi.stubEnv('TELEMETRY_DATA_DIR', path.join(dir, 'telemetry')); +}); +afterAll(() => { + vi.unstubAllEnvs(); + rmSync(dir, { recursive: true, force: true }); +}); +beforeEach(async () => { + thirdParty.on = false; + vi.stubEnv('BULWARK_TELEMETRY', ''); + vi.stubEnv('BULWARK_TELEMETRY_ERRORS', ''); + resetAggregateForTests(); + await saveState({ ...(await loadState()), consent: 'on', errorsConsent: 'off' }); + const yesterday = utcDay(Date.now() - DAY); + mkdirSync(path.join(dir, 'telemetry'), { recursive: true }); + writeFileSync(path.join(dir, 'telemetry', 'errors.json'), JSON.stringify({ [yesterday]: { aaaa000011112222: entry(7) } })); +}); +afterEach(() => { + resetAggregateForTests(); +}); + +describe('error counts consent', () => { + it('is off by default and the heartbeat stays schema 1, without errors', async () => { + expect((await effectiveErrorsConsent()).consent).toBe('off'); + const payload = await buildPayload(); + expect(payload.schema).toBe('1'); + expect('errors' in payload).toBe(false); + }); + + it('turns the heartbeat into schema 2 with the counts when on', async () => { + expect((await post({ action: 'set-errors-consent', consent: 'on' })).status).toBe(200); + const payload = await buildPayload(); + expect(payload.schema).toBe('2'); + expect(payload.schema === '2' && payload.errors).toEqual([{ + name: 'TypeError', + fingerprint: 'aaaa000011112222', + frames: [{ path: '/_next/static/chunks/1.js', line: 2, col: 30, debug_id: debugId }], + version: '1.20.0', + count: '6-10', + days: 1, + }]); + }); + + it('can\'t be turned on while telemetry is off, and goes off with it', async () => { + await saveState({ ...(await loadState()), consent: 'off' }); + expect((await post({ action: 'set-errors-consent', consent: 'on' })).status).toBe(409); + + await saveState({ ...(await loadState()), consent: 'on', errorsConsent: 'on' }); + expect((await effectiveErrorsConsent()).consent).toBe('on'); + await post({ action: 'set-consent', consent: 'off' }); + expect((await loadState()).errorsConsent).toBe('off'); + await post({ action: 'set-consent', consent: 'on' }); + expect((await effectiveErrorsConsent()).consent).toBe('off'); + }); + + it('is off in no-third-party mode, whatever is stored or set in the env', async () => { + await saveState({ ...(await loadState()), errorsConsent: 'on' }); + vi.stubEnv('BULWARK_TELEMETRY_ERRORS', 'on'); + thirdParty.on = true; + expect(await effectiveErrorsConsent()).toEqual({ consent: 'off', source: 'no-third-party' }); + }); + + it('follows BULWARK_TELEMETRY_ERRORS and refuses changes from the panel then', async () => { + vi.stubEnv('BULWARK_TELEMETRY_ERRORS', 'on'); + expect(await effectiveErrorsConsent()).toEqual({ consent: 'on', source: 'env' }); + expect((await post({ action: 'set-errors-consent', consent: 'off' })).status).toBe(409); + }); + + it('shows the v2 errors in the admin preview before anything is turned on', async () => { + const body = await (await GET(new NextRequest('http://localhost/api/admin/telemetry'))).json(); + expect(body.payloadPreview.schema).toBe('1'); + expect(body.errorsConsent).toBe('off'); + expect(body.errorsPreview).toHaveLength(1); + }); +}); + +describe('errors in the payload', () => { + const yesterday = utcDay(NOW - DAY); + + it('come from the last complete day, most frequent first, at most 50', () => { + const day: Aggregate[string] = {}; + for (let i = 0; i < 60; i++) day[`f${String(i).padStart(15, '0')}`] = entry(i + 1); + const errors = errorsForPayload({ [yesterday]: day, [utcDay(NOW)]: { today: entry(999) } }, NOW); + expect(errors).toHaveLength(50); + expect(errors[0].fingerprint).toBe('f000000000000059'); + expect(errors.some((e) => e.fingerprint === 'today')).toBe(false); + }); + + it('count the days of the last week a fingerprint occurred on', () => { + const agg: Aggregate = { + [yesterday]: { f: entry(1) }, + [utcDay(NOW - 3 * DAY)]: { f: entry(1) }, + [utcDay(NOW - 9 * DAY)]: { f: entry(1) }, + }; + expect(errorsForPayload(agg, NOW)[0].days).toBe(2); + }); + + it('carry a message only when it is a fixed error code', () => { + const agg: Aggregate = { + [yesterday]: { + a: entry(1, { name: 'JmapMethodError', code: 'Email/set: overQuota' }), + b: entry(1, { code: 'Merger plans for Acme' as string }), + }, + }; + const [withCode, without] = errorsForPayload(agg, NOW); + expect(withCode.message).toBe('Email/set: overQuota'); + expect(without.message).toBeUndefined(); + }); +}); diff --git a/lib/__tests__/telemetry-instant-errors.test.ts b/lib/__tests__/telemetry-instant-errors.test.ts new file mode 100644 index 000000000..b3f7d79ff --- /dev/null +++ b/lib/__tests__/telemetry-instant-errors.test.ts @@ -0,0 +1,189 @@ +// @vitest-environment node +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { NextRequest } from 'next/server'; + +// Instant error counts: with the error-count consent set to "instant", the +// scheduler leader sends the current day's counts to the collector's +// /v1/errors once a minute, but only errors that are new or whose count +// bucket changed since the last send. Daily mode sends nothing extra. + +const thirdParty = vi.hoisted(() => ({ on: false })); +vi.mock('@/lib/admin/third-party-mode', () => ({ isNoThirdPartyTrafficOn: async () => thirdParty.on })); +vi.mock('@/lib/admin/session', () => ({ + requireAdminAuth: vi.fn(async () => ({ session: { sub: 'admin' } })), + getClientIP: () => '127.0.0.1', +})); +vi.mock('@/lib/admin/audit', () => ({ auditLog: vi.fn(async () => {}) })); +vi.mock('@/lib/admin/plugin-registry', () => ({ + getPluginRegistry: async () => ({ plugins: [] }), + getThemeRegistry: async () => ({ themes: [] }), +})); + +const sent = vi.fn(async (_url: string, _init?: { body?: string }) => new Response('{"ok":true}')); +vi.mock('@/lib/telemetry/endpoint-guard', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + resolveEndpointAllowed: vi.fn(async () => ({ ok: true })), + fetchTelemetryTarget: (url: string, init?: { body?: string }) => sent(url, init), + }; +}); + +import { POST } from '@/app/api/admin/telemetry/route'; +import { DEFAULT_ENDPOINT, buildPayload, effectiveErrorsConsent, loadState, saveState } from '@/lib/telemetry'; +import { resetInstantErrorsForTests, sendInstantErrors } from '@/lib/telemetry/instant-errors'; +import { resetAggregateForTests, utcDay } from '@/lib/error-reporting/aggregate'; + +let dir: string; +const debugId = 'bc67f7db-7f3b-7be2-6f94-966cd7825cc2'; + +function entry(count: number, name = 'TypeError') { + return { + name, + frames: [ + { path: '/_next/static/chunks/1.js', line: 2, column: 30, debugId }, + { path: 'node_modules/next/dist/x.js', line: 1, column: 1 }, + ], + version: '1.20.0', + runtime: 'browser' as const, + count, + }; +} + +function storeCounts(day: Record>): void { + writeFileSync(path.join(dir, 'telemetry', 'errors.json'), JSON.stringify({ [utcDay()]: day })); +} + +function bodies(): Array & { errors: Array> }> { + return sent.mock.calls.map(([, init]) => JSON.parse(String(init?.body))); +} + +function post(body: Record) { + return POST(new NextRequest('http://localhost/api/admin/telemetry', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + })); +} + +beforeAll(() => { + dir = mkdtempSync(path.join(tmpdir(), 'bulwark-telemetry-instant-')); + vi.stubEnv('ADMIN_CONFIG_DIR', dir); + vi.stubEnv('TELEMETRY_DATA_DIR', path.join(dir, 'telemetry')); + mkdirSync(path.join(dir, 'telemetry'), { recursive: true }); +}); +afterAll(() => { + vi.unstubAllEnvs(); + rmSync(dir, { recursive: true, force: true }); +}); +beforeEach(async () => { + thirdParty.on = false; + vi.stubEnv('BULWARK_TELEMETRY', ''); + vi.stubEnv('BULWARK_TELEMETRY_ERRORS', ''); + sent.mockClear(); + sent.mockImplementation(async () => new Response('{"ok":true}')); + resetAggregateForTests(); + resetInstantErrorsForTests(); + await saveState({ ...(await loadState()), consent: 'on', endpoint: DEFAULT_ENDPOINT, errorsConsent: 'instant' }); + storeCounts({ aaaa000011112222: entry(1) }); +}); +afterEach(() => { + resetAggregateForTests(); + resetInstantErrorsForTests(); +}); + +describe('instant error counts', () => { + it('sends today\'s counts to the collector\'s /v1/errors, nothing but the allowed fields', async () => { + await sendInstantErrors(); + expect(sent).toHaveBeenCalledTimes(1); + expect(sent.mock.calls[0][0]).toBe('https://telemetry.bulwarkmail.org/v1/errors'); + const [body] = bodies(); + expect(Object.keys(body).sort()).toEqual(['day', 'errors', 'instance_id', 'schema']); + expect(body.schema).toBe('1'); + expect(body.day).toBe(utcDay()); + expect(body.instance_id).toMatch(/^[0-9a-f-]{36}$/); + expect(body.errors).toEqual([{ + name: 'TypeError', + fingerprint: 'aaaa000011112222', + frames: [{ path: '/_next/static/chunks/1.js', line: 2, col: 30, debug_id: debugId }], + version: '1.20.0', + count: '1', + days: 1, + }]); + }); + + it('sends an error again only when its count bucket changes', async () => { + storeCounts({ aaaa000011112222: entry(1), bbbb000011112222: entry(1, 'RangeError') }); + await sendInstantErrors(); + expect(bodies()[0].errors).toHaveLength(2); + + await sendInstantErrors(); + expect(sent).toHaveBeenCalledTimes(1); + + // 1 -> 3 moves from '1' to '2-5'. + storeCounts({ aaaa000011112222: entry(3), bbbb000011112222: entry(1, 'RangeError') }); + await sendInstantErrors(); + expect(sent).toHaveBeenCalledTimes(2); + expect(bodies()[1].errors.map((e) => [e.fingerprint, e.count])).toEqual([['aaaa000011112222', '2-5']]); + + // 3 -> 4 is still '2-5': nothing new to say. + storeCounts({ aaaa000011112222: entry(4), bbbb000011112222: entry(1, 'RangeError') }); + await sendInstantErrors(); + expect(sent).toHaveBeenCalledTimes(2); + }); + + it('tries a failed send again on the next tick', async () => { + sent.mockImplementationOnce(async () => new Response('{}', { status: 503 })); + await sendInstantErrors(); + await sendInstantErrors(); + expect(sent).toHaveBeenCalledTimes(2); + expect(bodies()[1].errors).toHaveLength(1); + }); + + it('sends nothing in daily mode, with error counts off, with the heartbeat off or in no-third-party mode', async () => { + await saveState({ ...(await loadState()), errorsConsent: 'on' }); + await sendInstantErrors(); + await saveState({ ...(await loadState()), errorsConsent: 'off' }); + await sendInstantErrors(); + await saveState({ ...(await loadState()), consent: 'off', errorsConsent: 'instant' }); + await sendInstantErrors(); + await saveState({ ...(await loadState()), consent: 'on' }); + thirdParty.on = true; + await sendInstantErrors(); + expect(sent).not.toHaveBeenCalled(); + }); + + it('sends nothing when the collector URL is empty, and follows a custom one', async () => { + await saveState({ ...(await loadState()), endpoint: '' }); + await sendInstantErrors(); + expect(sent).not.toHaveBeenCalled(); + await saveState({ ...(await loadState()), endpoint: 'https://collector.example.org/v1/heartbeat' }); + await sendInstantErrors(); + expect(sent.mock.calls[0][0]).toBe('https://collector.example.org/v1/errors'); + }); + + it('also sends the daily heartbeat with error counts (schema 2)', async () => { + expect((await buildPayload()).schema).toBe('2'); + }); +}); + +describe('instant consent', () => { + it('can be chosen in the admin panel and needs the heartbeat', async () => { + await saveState({ ...(await loadState()), errorsConsent: 'off' }); + expect((await post({ action: 'set-errors-consent', consent: 'instant' })).status).toBe(200); + expect((await loadState()).errorsConsent).toBe('instant'); + expect((await effectiveErrorsConsent()).consent).toBe('instant'); + + await saveState({ ...(await loadState()), consent: 'off', errorsConsent: 'off' }); + expect((await post({ action: 'set-errors-consent', consent: 'instant' })).status).toBe(409); + }); + + it('can be set with BULWARK_TELEMETRY_ERRORS=instant', async () => { + await saveState({ ...(await loadState()), errorsConsent: 'off' }); + vi.stubEnv('BULWARK_TELEMETRY_ERRORS', 'instant'); + expect(await effectiveErrorsConsent()).toEqual({ consent: 'instant', source: 'env' }); + }); +}); diff --git a/lib/__tests__/telemetry-payload.test.ts b/lib/__tests__/telemetry-payload.test.ts index 2cc9b138c..399eb5bef 100644 --- a/lib/__tests__/telemetry-payload.test.ts +++ b/lib/__tests__/telemetry-payload.test.ts @@ -8,7 +8,10 @@ vi.mock('@/lib/admin/config-manager', () => ({ get: (key: string, fallback: unknown) => (key === 'jmapServerUrl' ? 'https://mail.example.org' : fallback), }, })); -vi.mock('@/lib/telemetry/state', () => ({ getInstanceId: async () => 'instance-1' })); +vi.mock('@/lib/telemetry/state', () => ({ + getInstanceId: async () => 'instance-1', + effectiveErrorsConsent: async () => ({ consent: 'off', source: 'file' }), +})); vi.mock('@/lib/telemetry/login-tracker', () => ({ getLoginCounts: async () => ({ total: 3, active7d: 1 }) })); vi.mock('@/lib/admin/plugin-registry', () => ({ getPluginRegistry: async () => ({ plugins: [] }), diff --git a/lib/admin/__tests__/admin-password-read-only.test.ts b/lib/admin/__tests__/admin-password-read-only.test.ts new file mode 100644 index 000000000..bcad786ad --- /dev/null +++ b/lib/admin/__tests__/admin-password-read-only.test.ts @@ -0,0 +1,59 @@ +// @vitest-environment node +import { afterAll, afterEach, describe, expect, it, vi } from 'vitest'; +import { existsSync, rmSync } from 'node:fs'; +import path from 'node:path'; + +// A config dir mounted read-only (Kubernetes ConfigMap, a :ro volume) without +// an admin.json, and ADMIN_PASSWORD in the environment: the admin password +// works from memory on every replica instead of failing the start, which +// kept every replica of a shared-store deployment from ever becoming ready. + +// Hoisted: isConfigReadOnly() reads the env once. +const dir = await vi.hoisted(async () => { + const { mkdtempSync, mkdirSync } = await import('node:fs'); + const { tmpdir } = await import('node:os'); + const { join } = await import('node:path'); + const root = mkdtempSync(join(tmpdir(), 'bw-admin-pw-ro-')); + mkdirSync(join(root, 'config')); + process.env.ADMIN_CONFIG_DIR = join(root, 'config'); + process.env.ADMIN_STATE_DIR = join(root, 'state'); + process.env.ADMIN_CONFIG_READONLY = 'true'; + process.env.ADMIN_PASSWORD = 'from-the-environment'; + return root; +}); + +import { initAdminPassword, isAdminEnabled, resetAdminCacheForTests, verifyAdminPassword } from '../password'; +import { setStoreForTests } from '@/lib/shared-store'; +import { resetLocalStoreForTests } from '@/lib/shared-store/local'; +import { separateFakeStore } from './shared-store-fake'; + +afterEach(() => { + resetAdminCacheForTests(); + setStoreForTests(null); + resetLocalStoreForTests(); +}); + +afterAll(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +describe('ADMIN_PASSWORD with a read-only config and no admin.json', () => { + it('works from memory on a single instance and writes nothing to the config dir', async () => { + await expect(initAdminPassword()).resolves.toBe(true); + expect(isAdminEnabled()).toBe(true); + expect(await verifyAdminPassword('from-the-environment')).toBe(true); + expect(await verifyAdminPassword('wrong')).toBe(false); + expect(existsSync(path.join(dir, 'config', 'admin.json'))).toBe(false); + }); + + it('lets every replica of a shared store start and sign in', async () => { + setStoreForTests(separateFakeStore(path.join(dir, 'store'))); + for (let replica = 0; replica < 3; replica++) { + resetAdminCacheForTests(); + await expect(initAdminPassword()).resolves.toBe(true); + expect(await verifyAdminPassword('from-the-environment')).toBe(true); + expect(await verifyAdminPassword('wrong')).toBe(false); + } + expect(existsSync(path.join(dir, 'config', 'admin.json'))).toBe(false); + }); +}); diff --git a/lib/admin/__tests__/reachable-hosts-observability.test.ts b/lib/admin/__tests__/reachable-hosts-observability.test.ts new file mode 100644 index 000000000..dd4ca025a --- /dev/null +++ b/lib/admin/__tests__/reachable-hosts-observability.test.ts @@ -0,0 +1,21 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest'; +import { listReachableHosts, noThirdPartyCspSources } from '@/lib/admin/reachable-hosts'; + +const policy = { pushRelays: [], pushRelayUrl: '', defaultSidebarApps: [], features: { sidebarAppsEnabled: false, calendarEnabled: false } }; + +function hosts(config: Record, env: Record = {}) { + return listReachableHosts({ get: (key) => config[key], policy, env, buildRelayUrl: '' }); +} + +// No third-party traffic (#903) still allows the hosts the admin set: the +// error tracker is one of them, and the page's CSP must let browsers reach it. +describe('error reporting in the no-third-party list', () => { + it('is listed, without the key, only when a DSN is set', () => { + expect(hosts({}).filter((h) => h.kind === 'Error reporting')).toEqual([]); + const listed = hosts({ errorReportingDsn: 'https://6f0b9c79ed3a4315b76595bc2536ac40@errors.example.org/3' }) + .filter((h) => h.kind === 'Error reporting'); + expect(listed).toEqual([{ kind: 'Error reporting', host: 'errors.example.org', origin: 'https://errors.example.org', setIn: 'ERROR_REPORTING_DSN', side: 'both' }]); + expect(noThirdPartyCspSources(listed, false).connect).toEqual(['https://errors.example.org']); + }); +}); diff --git a/lib/admin/admin-nav.ts b/lib/admin/admin-nav.ts index e0eb282bc..4a93f4f09 100644 --- a/lib/admin/admin-nav.ts +++ b/lib/admin/admin-nav.ts @@ -31,6 +31,7 @@ export const ADMIN_PAGE_IDS = [ 'marketplace', 'version', 'telemetry', + 'monitoring', 'logs', ] as const; @@ -287,12 +288,26 @@ export const ADMIN_PAGES: readonly AdminPageDef[] = [ group: 'system', keywords: 'telemetry heartbeat anonymous consent privacy', sections: [ - { id: 'status', title: 'Status', keywords: 'enable disable consent' }, + { id: 'status', title: 'Status', keywords: 'enable disable consent error counts' }, { id: 'accounts', title: 'Account activity', keywords: 'active total logins' }, { id: 'endpoint', title: 'Endpoint', keywords: 'collector url' }, { id: 'payload', title: 'Payload preview', keywords: 'send now json' }, ], }, + { + id: 'monitoring', + label: 'Monitoring', + description: 'Error reports, metrics, health checks and tracing for your own tools', + group: 'system', + keywords: 'monitoring errors sentry bugsink glitchtip dsn prometheus metrics health probes tracing opentelemetry otel', + sections: [ + { id: 'errors', title: 'Error reporting', keywords: 'dsn sentry bugsink glitchtip sample rate environment' }, + { id: 'errors-today', title: 'Errors today', keywords: 'count fingerprint top' }, + { id: 'metrics', title: 'Metrics', keywords: 'prometheus scrape token grafana' }, + { id: 'health', title: 'Health checks', keywords: 'liveness readiness probe kubernetes docker' }, + { id: 'tracing', title: 'Tracing', keywords: 'opentelemetry otel otlp jaeger tempo spans traceparent' }, + ], + }, { id: 'logs', label: 'Audit log', diff --git a/lib/admin/audit.ts b/lib/admin/audit.ts index 0c4e2b6ed..572def9cd 100644 --- a/lib/admin/audit.ts +++ b/lib/admin/audit.ts @@ -89,6 +89,10 @@ export async function auditLog(action: string, detail: Record, ip, }; + // Also in the server log, so LOG_FORMAT=json shippers see admin actions. + // The audit file below stays the record the admin panel reads. + logger.info(`audit: ${action}`, { audit: entry }); + try { // One line per entry; the store rotates the local file (audit.log.3 → // deleted, .2 → .3, .1 → .2, audit.log → .1) once it passes 10 MB. diff --git a/lib/admin/outgoing-hosts-registry.ts b/lib/admin/outgoing-hosts-registry.ts index f6c6f0819..33fc84611 100644 --- a/lib/admin/outgoing-hosts-registry.ts +++ b/lib/admin/outgoing-hosts-registry.ts @@ -39,7 +39,7 @@ export const OUTGOING_HOSTS: readonly OutgoingHost[] = [ host: 'telemetry.bulwarkmail.org', files: ['lib/telemetry/types.ts'], side: 'server', - usedBy: 'Anonymous daily heartbeat, sent only after an admin turned telemetry on.', + usedBy: 'Anonymous daily heartbeat, sent only after an admin turned telemetry on; with instant error counts on, also the current day\'s error counts as they change.', off: 'Off by default. BULWARK_TELEMETRY=off, or System > Usage stats (switch it off or clear the collector URL).', inMode: 'Never sent, whatever the telemetry setting says.', lite: false, @@ -157,9 +157,10 @@ export const OUTGOING_HOSTS: readonly OutgoingHost[] = [ * has to be looked at again. */ export const NON_REQUEST_HOSTS: readonly { host: string; files: readonly string[]; why: string }[] = [ - { host: 'bulwarkmail.org', why: 'docs links, comments and the contact URL in User-Agent headers', files: ['app/(main)/admin/_pages/telemetry.tsx', 'app/api/fetch-ical/route.ts', 'app/api/translate/route.ts','components/admin/org/sections/recovery-section.tsx', 'components/admin/org/sections/company-calendars-section.tsx', 'components/admin/org/sections/free-busy-section.tsx', 'components/admin/org/sections/labels-section.tsx', 'components/admin/org/sections/out-of-office-calendar-section.tsx', 'components/admin/org/sections/phishing-report-section.tsx', 'instrumentation.node.ts', 'lib/telemetry/state.ts', 'lib/telemetry/types.ts'] }, + { host: 'bulwarkmail.org', why: 'docs links, comments and the contact URL in User-Agent headers', files: ['app/(main)/admin/_pages/telemetry.tsx', 'app/api/fetch-ical/route.ts', 'app/api/translate/route.ts', 'components/admin/org/sections/recovery-section.tsx', 'components/admin/org/sections/company-calendars-section.tsx', 'components/admin/org/sections/free-busy-section.tsx', 'components/admin/org/sections/labels-section.tsx', 'components/admin/org/sections/out-of-office-calendar-section.tsx', 'components/admin/org/sections/phishing-report-section.tsx', 'instrumentation.node.ts', 'lib/telemetry/state.ts', 'lib/telemetry/types.ts', 'app/(main)/admin/_pages/monitoring.tsx', 'lib/admin/types.ts', 'lib/error-reporting/client.ts', 'lib/error-reporting/lite.ts', 'lib/error-reporting/scrub.ts', 'instrumentation.ts', 'app/api/health/live/route.ts', 'app/api/health/ready/route.ts', 'app/api/metrics/route.ts', 'lib/metrics/index.ts', 'lib/tracing/config.ts', 'lib/tracing/scrub.ts', 'lib/tracing/start.ts'] }, { host: 'bulwarkmail.org', why: 'docs links of the Notices, Devices and out-of-office template pages', files: ['components/admin/org/sections/announcement-section.tsx', 'components/admin/org/sections/devices-section.tsx', 'components/admin/org/sections/retention-section.tsx', 'components/admin/org/sections/vacation-templates-section.tsx'] }, { host: 'bulwarkmail.com', why: 'JMAP capability URI', files: ['lib/jmap/client.ts'] }, + { host: 'develop.sentry.dev', why: 'link to the envelope format in a comment', files: ['lib/error-reporting/envelope.ts'] }, { host: 'extensions.bulwarkmail.org', why: 'signed statement type URIs', files: ['lib/admin/directory-keys.ts'] }, { host: 'www.w3.org', why: 'XML namespaces', files: ['components/ui/flag-icons.tsx', 'lib/favicon-badge.ts'] }, { host: 'creativecommons.org', why: 'licence metadata inside an SVG', files: ['components/ui/flag-icons.tsx'] }, diff --git a/lib/admin/password.ts b/lib/admin/password.ts index 592991bc9..bae73d0b5 100644 --- a/lib/admin/password.ts +++ b/lib/admin/password.ts @@ -2,7 +2,7 @@ import { scrypt, randomBytes, timingSafeEqual } from 'node:crypto'; import { logger } from '@/lib/logger'; import { isSharedMode, type DocCollection, type PutOptions, type PutResult, type StoredDoc } from '@/lib/shared-store'; import { announceChange, onSharedChange } from '@/lib/shared-store/changes'; -import { assertWritable } from './paths'; +import { assertWritable, isConfigReadOnly } from './paths'; import { adminConfigDocs, adminStateDocs, isStoreUnavailable } from './admin-store'; import type { AdminConfigData, AdminStateData } from './types'; import { readFileEnv } from '../read-file-env'; @@ -131,6 +131,12 @@ interface AdminCache { initialized: boolean; /** Shared mode: one read of the state for checks that run together. */ stateRead?: Promise; + /** + * The password came from ADMIN_PASSWORD into a read-only config without an + * admin.json: it lives in this process only, and every replica makes it + * from the same variable on start. + */ + fromEnvOnly?: boolean; unlisten?: () => void; } @@ -149,6 +155,7 @@ export function resetAdminCacheForTests(): void { cache.initialized = false; cache.stateRead = undefined; cache.unlisten = undefined; + cache.fromEnvOnly = undefined; } function freshState(): AdminStateData { @@ -271,6 +278,24 @@ export async function initAdminPassword(): Promise { const hash = isHashed(envPassword) ? envPassword : await hashPassword(envPassword); const config: AdminConfigData = { passwordHash: hash, passwordHashFile: undefined }; + if (isConfigReadOnly()) { + // Nowhere to save admin.json (a ConfigMap, a :ro volume). Keep the + // password in memory instead of failing the start; each replica does the + // same from the same variable. Changing it in the admin console is + // refused, as every other write to read-only config is. + cache.config = config; + cache.fromEnvOnly = true; + cache.state = shared ? await readStateData() : (await readStateData()) ?? freshState(); + if (shared && !cache.state) { + const fresh = freshState(); + cache.state = (await writeStateData(fresh, { ifVersion: null })).ok ? fresh : (await readStateData()) ?? fresh; + } + if (!cache.initialized) { + logger.info('Admin dashboard enabled with ADMIN_PASSWORD (config is read-only, so it is not saved to admin.json)'); + } + cache.initialized = true; + return true; + } if (shared) { // Replicas booting together with ADMIN_PASSWORD: the first one writes // admin.json and moves the session generation on, the others use it. @@ -319,6 +344,11 @@ function verifyAgainst(config: AdminConfigData, password: string): Promise { + if (cache.fromEnvOnly && cache.config) { + // A mounted admin.json that appeared later still wins. + const mounted = (await readConfigDoc())?.data ?? null; + return verifyAgainst(mounted ?? cache.config, password); + } if (isSharedMode()) { cache.config = (await readConfigDoc())?.data ?? null; } else if (!cache.config) { diff --git a/lib/admin/reachable-hosts.ts b/lib/admin/reachable-hosts.ts index f801f0c1c..a9ed1bde7 100644 --- a/lib/admin/reachable-hosts.ts +++ b/lib/admin/reachable-hosts.ts @@ -3,6 +3,7 @@ import { parseJmapServers } from './jmap-servers'; import { isBuiltInHost } from '@/lib/outgoing-hosts'; import { DEFAULT_RELAY_BASE_URL } from '@/lib/push-relays'; import { allFeeds } from './company-feeds'; +import { tracingEndpoint } from '@/lib/tracing/config'; /** * What stays reachable with "no third-party traffic" on: the hosts the @@ -105,6 +106,11 @@ export function listReachableHosts({ get, policy, env, buildRelayUrl = DEFAULT_R for (const app of policy.defaultSidebarApps ?? []) add('Sidebar app', app?.url, `User policy > Sidebar apps (${app?.name ?? ''})`, 'browser'); add('Document editor (WOPI)', get('wopiClientUrl'), 'WOPI_CLIENT_URL', 'both'); add('Translation (LibreTranslate)', env.LIBRETRANSLATE_URL, 'LIBRETRANSLATE_URL', 'server'); + // Only when an admin set it; browsers and this server send error reports there. + add('Error reporting', get('errorReportingDsn'), 'ERROR_REPORTING_DSN', 'both'); + // OpenTelemetry's OTLP endpoint, only when set in the standard OTEL_* variables. + const otlp = tracingEndpoint(env); + if (otlp) add('Tracing (OTLP)', otlp, env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT ? 'OTEL_EXPORTER_OTLP_TRACES_ENDPOINT' : 'OTEL_EXPORTER_OTLP_ENDPOINT', 'server'); for (const key of BRANDING_KEYS) add('Branding image', get(key), key, 'browser'); // User-directed: what users type in themselves stays allowed. diff --git a/lib/admin/types.ts b/lib/admin/types.ts index fb35d3954..d6bb72231 100644 --- a/lib/admin/types.ts +++ b/lib/admin/types.ts @@ -363,6 +363,12 @@ export const CONFIG_ENV_MAP: Record; accounts: Record; @@ -361,9 +362,13 @@ async function fetchIdentityEmails( }; // No redirect handling here: a redirect on the POST would downgrade it to // a GET and can only come from a misconfigured proxy - treat as failure. - const response = options.trusted - ? await fetch(apiUrl, { ...requestInit, redirect: 'manual' }) - : await fetchPublicUrl(apiUrl, requestInit); + // Measured and traced like every call to Stalwart (lib/metrics/upstream.ts). + const response = await observeUpstream('stalwart', async (trace) => { + const init = { ...requestInit, headers: { ...requestInit.headers, ...trace } }; + return options.trusted + ? await fetch(apiUrl, { ...init, redirect: 'manual' }) + : await fetchPublicUrl(apiUrl, init); + }); if (!response.ok) { throw new JmapAuthVerificationError( response.status === 401 || response.status === 403 @@ -435,9 +440,13 @@ async function fetchVerifiedSession( // Untrusted (user-supplied) endpoints connect through the rebinding-safe // fetch so the address validated above is the one the socket uses. // Admin-configured servers may legitimately live on private addresses. - response = options.trusted - ? await fetch(currentUrl, { ...requestInit, redirect: 'manual' }) - : await fetchPublicUrl(currentUrl, requestInit); + const target = currentUrl; + response = await observeUpstream('stalwart', async (trace) => { + const init = { ...requestInit, headers: { ...requestInit.headers, ...trace } }; + return options.trusted + ? await fetch(target, { ...init, redirect: 'manual' }) + : await fetchPublicUrl(target, init); + }); if (response.status >= 300 && response.status < 400) { const location = response.headers.get('location'); diff --git a/lib/config-validation.ts b/lib/config-validation.ts index 5652fd89d..a00a638e1 100644 --- a/lib/config-validation.ts +++ b/lib/config-validation.ts @@ -74,6 +74,14 @@ const configValidators = { noThirdPartyTraffic: isBoolean, // Lite only (lib/lite-config.ts); the server's response leaves it out. oauthRevokeOnSignOut: (value): value is boolean | undefined => value === undefined || typeof value === 'boolean', + // Absent from Lite's config and from servers before 1.20. + errorReporting: (value): value is ConfigData['errorReporting'] => value === undefined || ( + isRecord(value) + && typeof value.dsn === 'string' + && typeof value.sampleRate === 'number' + && typeof value.environment === 'string' + && typeof value.collect === 'boolean' + ), } satisfies Validators; /** diff --git a/lib/error-reporting/__tests__/aggregate.test.ts b/lib/error-reporting/__tests__/aggregate.test.ts new file mode 100644 index 000000000..a9dfaf577 --- /dev/null +++ b/lib/error-reporting/__tests__/aggregate.test.ts @@ -0,0 +1,34 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest'; +import { bound, KEEP_DAYS, MAX_PER_DAY, merge, summarizeDay, type Aggregate, type AggregateEntry } from '../aggregate'; + +function entry(count: number): AggregateEntry { + return { name: 'Error', frames: [], version: '1.20.0', runtime: 'browser', count }; +} + +describe('error aggregate', () => { + it('adds counts from several replicas instead of overwriting them', () => { + const a: Aggregate = { '2026-11-01': { f1: entry(2) } }; + const b: Aggregate = { '2026-11-01': { f1: entry(3), f2: entry(1) } }; + expect(merge(a, b)['2026-11-01']).toEqual({ f1: entry(5), f2: entry(1) }); + }); + + it('keeps the most frequent fingerprints per day and only the last days', () => { + const many: Record = {}; + for (let i = 0; i < MAX_PER_DAY + 50; i++) many[`f${i}`] = entry(i + 1); + const old = '2026-10-01'; + const bounded = bound({ '2026-11-01': many, [old]: { x: entry(1) } }, '2026-11-01'); + expect(Object.keys(bounded['2026-11-01'])).toHaveLength(MAX_PER_DAY); + expect(bounded['2026-11-01'].f0).toBeUndefined(); + expect(bounded['2026-11-01'][`f${MAX_PER_DAY + 49}`]).toBeDefined(); + expect(bounded[old]).toBeUndefined(); + const lastKept = new Date(Date.parse('2026-11-01T00:00:00Z') - (KEEP_DAYS - 1) * 86_400_000).toISOString().slice(0, 10); + expect(Object.keys(bound({ [lastKept]: { x: entry(1) } }, '2026-11-01'))).toEqual([lastKept]); + }); + + it('summarizes a day, most frequent first', () => { + const summary = summarizeDay({ '2026-11-01': { a: entry(1), b: entry(5) } }, '2026-11-01'); + expect(summary.total).toBe(6); + expect(summary.top.map((e) => e.fingerprint)).toEqual(['b', 'a']); + }); +}); diff --git a/lib/error-reporting/__tests__/client-lite.test.ts b/lib/error-reporting/__tests__/client-lite.test.ts new file mode 100644 index 000000000..5e4ab5d04 --- /dev/null +++ b/lib/error-reporting/__tests__/client-lite.test.ts @@ -0,0 +1,34 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { captureException, configureErrorReporting, resetErrorReportingForTests } from '../client'; + +const DSN = 'https://6f0b9c79ed3a4315b76595bc2536ac40@bugsink.example.org/3'; +let fetchSpy: ReturnType; + +beforeEach(() => { + vi.stubEnv('NEXT_PUBLIC_BULWARK_LITE', '1'); + resetErrorReportingForTests(); + sessionStorage.clear(); + fetchSpy = vi.fn().mockResolvedValue(new Response(null, { status: 200 })); + vi.stubGlobal('fetch', fetchSpy); +}); + +afterEach(() => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); +}); + +// Bulwark Lite has no server, so it never posts to /api/errors, whatever +// the settings say; reports go to the DSN only. +describe('error reporting in Lite', () => { + it('sends to the DSN and never to /api/errors', () => { + configureErrorReporting({ dsn: DSN, sampleRate: 1, environment: 'lite', collect: true }); + const error = new TypeError('boom'); + error.stack = 'TypeError: boom\n at a (https://lite.example.org/_next/static/chunks/1.js:2:30)'; + captureException(error); + const urls = fetchSpy.mock.calls.map(([url]) => String(url)); + expect(urls).toHaveLength(1); + expect(urls[0]).toContain('/api/3/envelope/'); + expect(urls.some((url) => url.includes('/api/errors'))).toBe(false); + }); +}); diff --git a/lib/error-reporting/__tests__/client.test.ts b/lib/error-reporting/__tests__/client.test.ts new file mode 100644 index 000000000..e4fc2709e --- /dev/null +++ b/lib/error-reporting/__tests__/client.test.ts @@ -0,0 +1,120 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { + captureException, configureErrorReporting, recordJmapCall, recordNavigation, reportJmapFailure, + resetErrorReportingForTests, +} from '../client'; + +const DSN = 'https://6f0b9c79ed3a4315b76595bc2536ac40@bugsink.example.org/3'; +const SUBJECT = 'Merger plans for Acme Q3'; +const ADDRESS = 'ceo.private@acme-corp.example'; +const FOLDER = 'Acquisition Talks'; + +function errorWithStack(message: string): Error { + const error = new TypeError(message); + error.stack = `TypeError: ${message}\n at Ab (https://mail.example.org/_next/static/chunks/1.js:2:30)\n at https://mail.example.org/_next/static/chunks/2.js:4:5`; + return error; +} + +let fetchSpy: ReturnType; + +function calls(): Array<{ url: string; init: RequestInit }> { + return fetchSpy.mock.calls.map(([url, init]) => ({ url: String(url), init: init as RequestInit })); +} + +beforeEach(() => { + resetErrorReportingForTests(); + sessionStorage.clear(); + fetchSpy = vi.fn().mockResolvedValue(new Response(null, { status: 200 })); + vi.stubGlobal('fetch', fetchSpy); +}); + +afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); + +describe('browser error reporting', () => { + it('sends nothing while reporting is off', () => { + configureErrorReporting(null); + captureException(errorWithStack('boom')); + configureErrorReporting({ dsn: '', sampleRate: 1, environment: 'production', collect: false }); + captureException(errorWithStack('boom again')); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('holds reports until the config arrives, then sends them', () => { + captureException(errorWithStack('early')); + expect(fetchSpy).not.toHaveBeenCalled(); + configureErrorReporting({ dsn: DSN, sampleRate: 1, environment: 'production', collect: false }); + expect(calls().map((c) => c.url)).toEqual(['https://bugsink.example.org/api/3/envelope/?sentry_key=6f0b9c79ed3a4315b76595bc2536ac40&sentry_version=7']); + }); + + it('sends to the DSN without cookies or referrer, and with no mail content anywhere', () => { + configureErrorReporting({ dsn: DSN, sampleRate: 1, environment: 'staging', collect: true }); + window.history.pushState({}, '', `/en/mail/folder/${encodeURIComponent(FOLDER)}/Mabc`); + recordNavigation(window.location.pathname); + recordJmapCall(['Email/query', 'Email/get']); + captureException(errorWithStack(`Failed to open "${SUBJECT}" from ${ADDRESS} at https://mail.example.org/download?name=${encodeURIComponent(SUBJECT)}`)); + + const [dsnCall, ownCall] = calls(); + expect(dsnCall.init.credentials).toBe('omit'); + expect(dsnCall.init.referrerPolicy).toBe('no-referrer'); + expect(ownCall.url).toBe('/api/errors'); + for (const { init } of [dsnCall, ownCall]) { + const body = String(init.body); + for (const planted of [SUBJECT, ADDRESS, FOLDER, encodeURIComponent(FOLDER), 'Merger', 'acme-corp', 'download?name']) { + expect(body).not.toContain(planted); + } + } + const event = JSON.parse(String(dsnCall.init.body).trim().split('\n')[2]); + expect(event.exception.values[0].value).toBe('Failed to open "…" from [email] at [url]'); + expect(event.environment).toBe('staging'); + expect(event.breadcrumbs.values.map((b: { message: string }) => b.message)).toEqual(['/mail/…', 'Email/query, Email/get']); + expect(event.tags.route).toBe('/mail/…'); + }); + + it('sends each fingerprint once per session', () => { + configureErrorReporting({ dsn: DSN, sampleRate: 1, environment: 'production', collect: false }); + captureException(errorWithStack('same')); + captureException(errorWithStack('same')); + expect(fetchSpy).toHaveBeenCalledTimes(1); + }); + + it('stops at ten reports a minute', () => { + configureErrorReporting({ dsn: DSN, sampleRate: 1, environment: 'production', collect: false }); + for (let i = 0; i < 15; i++) { + const error = new Error(`e${i}`); + error.stack = `Error\n at https://m.example/_next/static/chunks/${i}.js:1:1`; + captureException(error); + } + expect(fetchSpy).toHaveBeenCalledTimes(10); + }); + + it('applies the sample rate to the DSN only', () => { + vi.spyOn(Math, 'random').mockReturnValue(0.5); + configureErrorReporting({ dsn: DSN, sampleRate: 0.1, environment: 'production', collect: true }); + captureException(errorWithStack('sampled out')); + expect(calls().map((c) => c.url)).toEqual(['/api/errors']); + }); + + it('ignores errors that come only from extensions or other scripts', () => { + configureErrorReporting({ dsn: DSN, sampleRate: 1, environment: 'production', collect: false }); + const error = new Error('from an extension'); + error.stack = 'Error\n at chrome-extension://abc/content.js:1:1'; + captureException(error); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('reports a failed JMAP request by status or method and type, never the answer', () => { + configureErrorReporting({ dsn: DSN, sampleRate: 1, environment: 'production', collect: true }); + reportJmapFailure({ status: 503 }); + reportJmapFailure({ method: 'Email/set', type: 'overQuota' }); + reportJmapFailure({ method: SUBJECT, type: ADDRESS }); + const own = calls().filter((c) => c.url === '/api/errors').map((c) => JSON.parse(String(c.init.body))); + expect(own.map((r) => [r.name, r.code])).toEqual([ + ['JmapRequestError', 'http:503'], + ['JmapMethodError', 'Email/set: overQuota'], + ['JmapMethodError', 'Unknown/method: unknown'], + ]); + }); +}); diff --git a/lib/error-reporting/__tests__/collect-consent.test.ts b/lib/error-reporting/__tests__/collect-consent.test.ts new file mode 100644 index 000000000..a1dbcfa6d --- /dev/null +++ b/lib/error-reporting/__tests__/collect-consent.test.ts @@ -0,0 +1,31 @@ +// @vitest-environment node +import { describe, expect, it, vi } from 'vitest'; + +const consent = vi.hoisted(() => ({ value: 'off' as 'on' | 'off' | 'instant' })); +vi.mock('@/lib/admin/config-manager', () => ({ + configManager: { ensureLoaded: async () => {}, get: (_key: string, fallback: unknown) => fallback }, +})); +vi.mock('@/lib/telemetry/state', () => ({ + effectiveErrorsConsent: async () => ({ consent: consent.value, source: 'file' }), +})); + +import { shouldCollectErrors } from '../settings'; + +// Error counts in the heartbeat need errors counted on this server, so the +// consent alone turns collection on; with nothing set it stays off. +describe('error collection', () => { + it('is off with no DSN, no metrics and no error-count consent', async () => { + consent.value = 'off'; + expect(await shouldCollectErrors()).toBe(false); + }); + + it('is on with the error-count consent alone', async () => { + consent.value = 'on'; + expect(await shouldCollectErrors()).toBe(true); + }); + + it('is on in instant mode too', async () => { + consent.value = 'instant'; + expect(await shouldCollectErrors()).toBe(true); + }); +}); diff --git a/lib/error-reporting/__tests__/envelope.test.ts b/lib/error-reporting/__tests__/envelope.test.ts new file mode 100644 index 000000000..41d45af57 --- /dev/null +++ b/lib/error-reporting/__tests__/envelope.test.ts @@ -0,0 +1,118 @@ +// @vitest-environment node +import { afterEach, describe, expect, it } from 'vitest'; +import { buildEnvelope, buildEvent, envelopeUrl, isErrorCode, parseDsn } from '../envelope'; +import { debugIdsByPath, resetDebugIdCache } from '../debug-ids'; +import { fingerprint } from '../fingerprint'; +import { parseStack } from '../stack'; + +const DSN = 'https://6f0b9c79ed3a4315b76595bc2536ac40@bugsink.example.org/3'; + +describe('parseDsn', () => { + it('reads key, origin and project', () => { + expect(parseDsn(DSN)).toEqual({ origin: 'https://bugsink.example.org', pathPrefix: '', publicKey: '6f0b9c79ed3a4315b76595bc2536ac40', projectId: '3' }); + expect(parseDsn('https://6f0b9c79ed3a4315b76595bc2536ac40@host.example:8443/sentry/12')?.pathPrefix).toBe('/sentry'); + }); + + it('refuses what is not a DSN', () => { + for (const bad of ['', 'not a url', 'https://bugsink.example.org/3', 'ftp://6f0b9c79ed3a4315b76595bc2536ac40@x.example/3', + 'https://6f0b9c79ed3a4315b76595bc2536ac40@x.example/abc', 'https://6f0b9c79ed3a4315b76595bc2536ac40:secret@x.example/3']) { + expect(parseDsn(bad), bad).toBeNull(); + } + }); + + it('puts the key in the query, so a browser sends no custom header', () => { + expect(envelopeUrl(parseDsn(DSN)!)).toBe('https://bugsink.example.org/api/3/envelope/?sentry_key=6f0b9c79ed3a4315b76595bc2536ac40&sentry_version=7'); + }); +}); + +describe('stack parsing', () => { + it('reads V8, Firefox and Safari stacks, top frame first', () => { + const chrome = 'TypeError: x\n at Ab (https://mail.example.org/_next/static/chunks/1.js:2:30)\n at async https://mail.example.org/_next/static/chunks/2.js:4:5'; + const firefox = 'Ab@https://mail.example.org/_next/static/chunks/1.js:2:30\n@https://mail.example.org/_next/static/chunks/2.js:4:5'; + for (const stack of [chrome, firefox]) { + expect(parseStack(stack)).toEqual([ + { path: '/_next/static/chunks/1.js', line: 2, column: 30, fn: 'Ab' }, + { path: '/_next/static/chunks/2.js', line: 4, column: 5, fn: undefined }, + ]); + } + expect(parseStack(' at Object.handler (/app/.next/server/chunks/77.js:1:900)')[0]).toEqual({ path: '/.next/server/chunks/77.js', line: 1, column: 900, fn: 'Object.handler' }); + }); +}); + +describe('debug ids', () => { + afterEach(() => { + delete (globalThis as Record)._debugIds; + delete (globalThis as Record)._sentryDebugIds; + resetDebugIdCache(); + }); + + it('maps each registered chunk to its id, from Turbopack and sentry-cli alike', () => { + (globalThis as Record)._debugIds = { + 'Error\n at https://mail.example.org/_next/static/chunks/1.js:1:10': 'bc67f7db-7f3b-7be2-6f94-966cd7825cc2', + }; + (globalThis as Record)._sentryDebugIds = { + 'Error\n at https://mail.example.org/_next/static/chunks/2.js:1:10': 'c40c4654-00a8-5efe-b3a3-42818d0ccc15', + }; + expect(Object.fromEntries(debugIdsByPath())).toEqual({ + '/_next/static/chunks/1.js': 'bc67f7db-7f3b-7be2-6f94-966cd7825cc2', + '/_next/static/chunks/2.js': 'c40c4654-00a8-5efe-b3a3-42818d0ccc15', + }); + }); +}); + +describe('buildEvent', () => { + const frames = [ + { path: '/_next/static/chunks/1.js', line: 2, column: 30, fn: 'Ab' }, + { path: '/_next/static/chunks/2.js', line: 4, column: 5 }, + ]; + const event = buildEvent( + { name: 'TypeError', message: 'x is undefined', frames, handled: false, mechanism: 'onerror', route: '/mail/…' }, + { + eventId: 'a'.repeat(32), timestamp: 1, release: '1.20.0+abc1234', environment: 'production', platform: 'javascript', + fingerprint: 'ffff0000ffff0000', debugIds: new Map([['/_next/static/chunks/1.js', 'bc67f7db-7f3b-7be2-6f94-966cd7825cc2']]), + }, + ); + + it('names a source map image for each frame file with a debug id, under the frame path', () => { + const exception = (event.exception as { values: Array<{ stacktrace: { frames: Array<{ abs_path: string; lineno: number }> } }> }).values[0]; + // Oldest first: the throwing frame is last. + expect(exception.stacktrace.frames.map((f) => f.abs_path)).toEqual(['/_next/static/chunks/2.js', '/_next/static/chunks/1.js']); + expect(event.debug_meta).toEqual({ images: [{ type: 'sourcemap', code_file: '/_next/static/chunks/1.js', debug_id: 'bc67f7db-7f3b-7be2-6f94-966cd7825cc2' }] }); + }); + + it('groups by Bulwark\'s fingerprint and carries the release', () => { + expect(event.fingerprint).toEqual(['ffff0000ffff0000']); + expect(event.release).toBe('1.20.0+abc1234'); + expect(event.tags).toEqual({ route: '/mail/…' }); + }); + + it('wraps it in a three-line envelope', () => { + const lines = buildEnvelope(event).trim().split('\n'); + expect(lines).toHaveLength(3); + expect(JSON.parse(lines[1])).toEqual({ type: 'event', content_type: 'application/json' }); + expect(JSON.parse(lines[2]).event_id).toBe('a'.repeat(32)); + }); +}); + +describe('fingerprint', () => { + const frames = [{ path: '/_next/static/chunks/1.js', line: 2, column: 30 }]; + + it('differs by name, frame, version and code, and ignores frames outside the bundle', () => { + const base = fingerprint('TypeError', frames, '1.20.0'); + expect(base).toMatch(/^[0-9a-f]{16}$/); + expect(fingerprint('TypeError', frames, '1.20.0')).toBe(base); + expect(fingerprint('RangeError', frames, '1.20.0')).not.toBe(base); + expect(fingerprint('TypeError', [{ ...frames[0], line: 3 }], '1.20.0')).not.toBe(base); + expect(fingerprint('TypeError', frames, '1.20.1')).not.toBe(base); + expect(fingerprint('TypeError', frames, '1.20.0', 'serverFail')).not.toBe(base); + expect(fingerprint('TypeError', [...frames, { path: '[other]', line: 1, column: 1 }], '1.20.0')).toBe(base); + }); + + it('accepts only fixed error codes', () => { + expect(isErrorCode('Email/set: overQuota')).toBe(true); + expect(isErrorCode('serverFail')).toBe(true); + expect(isErrorCode('http:503')).toBe(true); + expect(isErrorCode('Merger plans for Acme')).toBe(false); + expect(isErrorCode('Email/set: over quota now')).toBe(false); + }); +}); diff --git a/lib/error-reporting/__tests__/lite.test.ts b/lib/error-reporting/__tests__/lite.test.ts new file mode 100644 index 000000000..b73000005 --- /dev/null +++ b/lib/error-reporting/__tests__/lite.test.ts @@ -0,0 +1,21 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest'; +import { liteErrorReporting } from '../lite'; + +const DSN = 'https://6f0b9c79ed3a4315b76595bc2536ac40@errors.example.org/3'; + +// Bulwark Lite has no server: policy.json carries the error reporting address. +describe('Lite error reporting from policy.json', () => { + it('is off without a valid DSN', () => { + expect(liteErrorReporting({})).toBeNull(); + expect(liteErrorReporting(null)).toBeNull(); + expect(liteErrorReporting({ errorReporting: { dsn: 'not a dsn' } })).toBeNull(); + }); + + it('reads the DSN, sample rate and environment, and never asks for server-side counting', () => { + expect(liteErrorReporting({ errorReporting: { dsn: DSN, sampleRate: 0.25, environment: 'lite-prod' } })) + .toEqual({ dsn: DSN, sampleRate: 0.25, environment: 'lite-prod', collect: false }); + expect(liteErrorReporting({ errorReporting: { dsn: DSN, sampleRate: 7, environment: 'bad env!' } })) + .toEqual({ dsn: DSN, sampleRate: 1, environment: 'production', collect: false }); + }); +}); diff --git a/lib/error-reporting/__tests__/scrub.test.ts b/lib/error-reporting/__tests__/scrub.test.ts new file mode 100644 index 000000000..eaf2c708d --- /dev/null +++ b/lib/error-reporting/__tests__/scrub.test.ts @@ -0,0 +1,109 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest'; +import { scrubFramePath, scrubJmapErrorType, scrubMethodName, scrubName, scrubText } from '../scrub'; +import { routeGroup, scrubRoutePath } from '@/lib/route-groups'; + +// Mail content planted where it could end up in an error: none of it may +// survive scrubbing. +const SUBJECT = 'Merger plans for Acme Q3'; +const ADDRESS = 'ceo.private@acme-corp.example'; +const FOLDER = 'Acquisition Talks'; +const FILE = 'term-sheet-final.pdf'; +const TOKEN = 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1c2VyIn0.c2lnbmF0dXJlMTIzNDU2'; + +function expectClean(text: string) { + for (const planted of [SUBJECT, ADDRESS, FOLDER, FILE, TOKEN, 'Merger', 'acme-corp', 'Acquisition', 'term-sheet']) { + expect(text).not.toContain(planted); + } +} + +describe('scrubText', () => { + it('removes a quoted subject, an address and a file name from a message', () => { + const out = scrubText(`Failed to move "${SUBJECT}" from ${ADDRESS} with attachment ${FILE}`); + expectClean(out); + expect(out).toBe('Failed to move "…" from [email] with attachment [file]'); + }); + + it('removes text in every kind of quote, but keeps contractions', () => { + expect(scrubText(`can't open 'Merger' or “Merger” or ‘Merger’ or «Merger» or \`Merger\``)) + .toBe("can't open '…' or “…” or ‘…’ or «…» or `…`"); + }); + + it('removes URLs whole, with their query strings', () => { + const out = scrubText(`fetch https://mail.acme-corp.example/jmap/download/abc/${FILE}?token=${TOKEN} failed`); + expectClean(out); + expect(out).toBe('fetch [url] failed'); + }); + + it('removes credentials, JWTs, key=value pairs and random-looking ids', () => { + expect(scrubText(`Authorization: Bearer ${TOKEN}`)).toBe('Authorization: Bearer [token]'); + expect(scrubText(`token ${TOKEN} expired`)).toBe('token [token] expired'); + expect(scrubText('cookie session=abc123def; theme=dark')).toBe('cookie session=[redacted]; theme=[redacted]'); + expect(scrubText('blob Ma1b2c3d4e5f6a7b8c9d0e1f2a3b4 missing')).toBe('blob [id] missing'); + }); + + it('keeps only the first line and drops a server answer after the status', () => { + expect(scrubText(`Request failed: 400 - {"description":"${SUBJECT}"}`)).toBe('Request failed: 400'); + expect(scrubText(`Something broke\n${SUBJECT}\n${ADDRESS}`)).toBe('Something broke'); + }); + + it('keeps engine messages readable', () => { + expect(scrubText('Cannot read properties of undefined (reading \'id\')')).toBe('Cannot read properties of undefined (reading \'…\')'); + // Minified identifiers go too; the resolved stack names the code. + expect(scrubText('e.map is not a function')).toBe('… is not a function'); + }); + + it('removes unquoted free text such as a folder name or a subject', () => { + const out = scrubText('Cannot move message from ' + ADDRESS + ' to ' + FOLDER + ': ' + SUBJECT.toLowerCase()); + expectClean(out); + expect(out).toBe('Cannot move message from [email] to …: … for …'); + }); + + it('caps the length', () => { + expect(scrubText('x '.repeat(500)).length).toBeLessThanOrEqual(200); + }); +}); + +describe('route paths in breadcrumbs and reports', () => { + it('keeps the area only, never folder names or ids', () => { + const path = `/de/mail/folder/${encodeURIComponent(FOLDER)}/Mabc123?q=${encodeURIComponent(SUBJECT)}`; + const out = scrubRoutePath(path); + expectClean(decodeURIComponent(out)); + expect(out).toBe('/mail/…'); + }); + + it('removes the base path and maps unknown areas and API routes to fixed names', () => { + expect(scrubRoutePath('/webmail/en/calendar', '/webmail')).toBe('/calendar'); + expect(scrubRoutePath(`/en/${encodeURIComponent(FOLDER)}`)).toBe('/other'); + expect(scrubRoutePath('/api/wopi/files/abc/contents')).toBe('/api/wopi/…'); + expect(routeGroup(`/api/${FOLDER}`)).toBe('api:other'); + expect(routeGroup('/_next/static/chunks/a.js')).toBe('static'); + expect(routeGroup('/')).toBe('page:home'); + }); +}); + +describe('frame paths', () => { + it('drops the origin, base path, query and fragment', () => { + expect(scrubFramePath('https://mail.acme-corp.example/webmail/_next/static/chunks/12-ab.js?dpl=x#y')).toBe('/_next/static/chunks/12-ab.js'); + expect(scrubFramePath('/home/alice/bulwark/.next/server/chunks/ssr/a.js')).toBe('/.next/server/chunks/ssr/a.js'); + expect(scrubFramePath('/home/alice/bulwark/node_modules/next/dist/x.js')).toBe('node_modules/next/dist/x.js'); + expect(scrubFramePath('node:internal/process/task_queues')).toBe('node:internal/process/task_queues'); + }); + + it('turns anything else into [other]', () => { + expect(scrubFramePath('chrome-extension://abc/content.js')).toBe('[other]'); + expect(scrubFramePath(`https://acme-corp.example/${FILE}`)).toBe('[other]'); + expect(scrubFramePath('blob:https://x/1')).toBe('[other]'); + }); +}); + +describe('names', () => { + it('keeps identifiers only', () => { + expect(scrubName('TypeError')).toBe('TypeError'); + expect(scrubName(SUBJECT)).toBe('Error'); + expect(scrubMethodName('Email/get')).toBe('Email/get'); + expect(scrubMethodName(SUBJECT)).toBe('Unknown/method'); + expect(scrubJmapErrorType('overQuota')).toBe('overQuota'); + expect(scrubJmapErrorType(ADDRESS)).toBe('unknown'); + }); +}); diff --git a/lib/error-reporting/__tests__/server.test.ts b/lib/error-reporting/__tests__/server.test.ts new file mode 100644 index 000000000..e17afb52d --- /dev/null +++ b/lib/error-reporting/__tests__/server.test.ts @@ -0,0 +1,101 @@ +// @vitest-environment node +import { mkdtempSync, rmSync, existsSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const config: Record = {}; +vi.mock('@/lib/admin/config-manager', () => ({ + configManager: { + ensureLoaded: async () => {}, + get: (key: string, fallback: unknown) => (key in config ? config[key] : fallback), + }, +})); + +import { captureLoggedError, captureRequestError, installServerErrorReporting } from '../server'; +import { flushErrors, readErrors, resetAggregateForTests, utcDay } from '../aggregate'; +import { logger, setErrorSink } from '@/lib/logger'; + +const DSN = 'https://6f0b9c79ed3a4315b76595bc2536ac40@bugsink.example.org/3'; +const SUBJECT = 'Merger plans for Acme Q3'; +const ADDRESS = 'ceo.private@acme-corp.example'; + +let dir: string; +let fetchSpy: ReturnType; + +beforeEach(() => { + dir = mkdtempSync(path.join(tmpdir(), 'bulwark-errors-')); + process.env.TELEMETRY_DATA_DIR = dir; + for (const key of Object.keys(config)) delete config[key]; + resetAggregateForTests(); + fetchSpy = vi.fn().mockResolvedValue(new Response(null, { status: 200 })); + vi.stubGlobal('fetch', fetchSpy); +}); + +afterEach(() => { + setErrorSink(null); + resetAggregateForTests(); + vi.unstubAllGlobals(); + delete process.env.TELEMETRY_DATA_DIR; + rmSync(dir, { recursive: true, force: true }); +}); + +function serverError(message: string): Error { + const error = new Error(message); + error.stack = `Error: ${message}\n at handler (/app/.next/server/chunks/77.js:1:900)\n at /app/node_modules/next/dist/server.js:5:6`; + return error; +} + +describe('server error reporting', () => { + it('does nothing at all without a DSN or other reason to collect', async () => { + await captureRequestError(serverError('boom'), '/api/admin/config'); + await flushErrors(); + expect(fetchSpy).not.toHaveBeenCalled(); + expect(existsSync(path.join(dir, 'errors.json'))).toBe(false); + }); + + it('sends a request error to the DSN with the route pattern and no mail content', async () => { + config.errorReportingDsn = DSN; + config.errorReportingEnvironment = 'beta'; + await captureRequestError(serverError(`Could not file "${SUBJECT}" for ${ADDRESS}`), '/api/wopi/files/[fileId]'); + + expect(fetchSpy).toHaveBeenCalledTimes(1); + const [url, init] = fetchSpy.mock.calls[0]; + expect(url).toBe('https://bugsink.example.org/api/3/envelope/?sentry_key=6f0b9c79ed3a4315b76595bc2536ac40&sentry_version=7'); + const body = String((init as RequestInit).body); + expect(body).not.toContain('Merger'); + expect(body).not.toContain(ADDRESS); + const event = JSON.parse(body.trim().split('\n')[2]); + expect(event.platform).toBe('node'); + expect(event.environment).toBe('beta'); + expect(event.tags).toEqual({ runtime: 'server', route: '/api/wopi/files/[fileId]' }); + expect(event.exception.values[0].stacktrace.frames.at(-1).abs_path).toBe('/.next/server/chunks/77.js'); + }); + + it('counts the error per day', async () => { + config.errorReportingDsn = DSN; + await captureRequestError(serverError('one')); + await captureRequestError(serverError('one')); + await flushErrors(); + const day = (await readErrors())[utcDay()]; + expect(Object.values(day).map((e) => [e.name, e.runtime, e.count])).toEqual([['Error', 'server', 2]]); + }); + + it('reports logger.error calls with the caller\'s frames once installed', async () => { + config.errorReportingDsn = DSN; + await installServerErrorReporting(); + logger.error('telemetry GET error', { error: `failed for ${ADDRESS}` }); + await vi.waitFor(() => expect(fetchSpy).toHaveBeenCalledTimes(1)); + const event = JSON.parse(String((fetchSpy.mock.calls[0][1] as RequestInit).body).trim().split('\n')[2]); + expect(event.exception.values[0].type).toBe('LoggedError'); + expect(event.exception.values[0].value).toBe('telemetry GET error: failed for [email]'); + }); + + it('drops the logger\'s own frames from a logged error', async () => { + config.errorReportingDsn = DSN; + const stack = 'Error\n at log (/app/.next/server/chunks/1.js:1:1)\n at Object.error (/app/.next/server/chunks/1.js:2:2)\n at GET (/app/.next/server/chunks/route.js:9:9)'; + await captureLoggedError('x failed', undefined, stack); + const event = JSON.parse(String((fetchSpy.mock.calls[0][1] as RequestInit).body).trim().split('\n')[2]); + expect(event.exception.values[0].stacktrace.frames.map((f: { function: string }) => f.function)).toEqual(['GET']); + }); +}); diff --git a/lib/error-reporting/aggregate.ts b/lib/error-reporting/aggregate.ts new file mode 100644 index 000000000..75c6fb484 --- /dev/null +++ b/lib/error-reporting/aggregate.ts @@ -0,0 +1,186 @@ +import { logger } from '@/lib/logger'; +import { telemetryDocs } from '@/lib/telemetry/state'; +import { isErrorCode } from './envelope'; +import { fingerprint as fingerprintOf } from './fingerprint'; +import type { Frame } from './stack'; +import { debugIdsByPath } from './debug-ids'; + +/** + * Errors counted per UTC day: fingerprint -> name, top frames, version and + * how often. The admin's "errors today", the metrics endpoint and the + * telemetry error counts read it. Only what scrubbing leaves goes in. + * + * Bounded: MAX_PER_DAY fingerprints per day (the rarest go first) and + * KEEP_DAYS days. Counts gather in memory and are added to `errors.json` in + * telemetry's document collection every FLUSH_MS, so several replicas add up + * (an update, not an overwrite). + */ + +export const MAX_PER_DAY = 200; +export const KEEP_DAYS = 8; +export const MAX_FRAMES_KEPT = 5; +const FLUSH_MS = 60_000; +export const ERRORS_DOC = 'errors.json'; + +export interface AggregateEntry { + name: string; + frames: Frame[]; + version: string; + /** A fixed error code when the error has one. */ + code?: string; + runtime: 'browser' | 'server'; + count: number; +} + +/** day (YYYY-MM-DD) -> fingerprint -> entry */ +export type Aggregate = Record>; + +export interface Recordable { + name: string; + frames: Frame[]; + code?: string; +} + +interface AggregateState { + pending: Aggregate; + timer: ReturnType | null; + flushing: Promise | null; +} + +const KEY = Symbol.for('bulwark.error-reporting.aggregate'); +function state(): AggregateState { + const g = globalThis as typeof globalThis & { [KEY]?: AggregateState }; + return (g[KEY] ??= { pending: {}, timer: null, flushing: null }); +} + +export function utcDay(at = Date.now()): string { + return new Date(at).toISOString().slice(0, 10); +} + +/** Keep the newest KEEP_DAYS days and the MAX_PER_DAY most frequent per day. */ +export function bound(agg: Aggregate, today = utcDay()): Aggregate { + const cutoff = utcDay(Date.parse(`${today}T00:00:00Z`) - (KEEP_DAYS - 1) * 86_400_000); + const out: Aggregate = {}; + for (const [day, entries] of Object.entries(agg)) { + if (!/^\d{4}-\d{2}-\d{2}$/.test(day) || day < cutoff) continue; + const kept = Object.entries(entries) + .sort(([, a], [, b]) => b.count - a.count) + .slice(0, MAX_PER_DAY); + out[day] = Object.fromEntries(kept); + } + return out; +} + +/** Add `delta`'s counts to `base`. */ +export function merge(base: Aggregate, delta: Aggregate): Aggregate { + const out: Aggregate = {}; + for (const [day, entries] of Object.entries(base)) out[day] = { ...entries }; + for (const [day, entries] of Object.entries(delta)) { + const target = (out[day] ??= {}); + for (const [print, entry] of Object.entries(entries)) { + const existing = target[print]; + target[print] = existing ? { ...existing, count: existing.count + entry.count } : { ...entry }; + } + } + return out; +} + +function parseStored(data: Buffer | null): Aggregate { + if (!data) return {}; + try { + const parsed = JSON.parse(data.toString('utf8')) as unknown; + return parsed && typeof parsed === 'object' ? (parsed as Aggregate) : {}; + } catch { + return {}; + } +} + +/** Count one error. Returns its fingerprint. */ +export function recordError(report: Recordable, runtime: 'browser' | 'server', version: string, at = Date.now()): string { + const print = fingerprintOf(report.name, report.frames, version, report.code); + const s = state(); + const day = utcDay(at); + const entries = (s.pending[day] ??= {}); + const existing = entries[print]; + if (existing) { + existing.count += 1; + } else if (Object.keys(entries).length < MAX_PER_DAY) { + entries[print] = { + name: report.name, + // The server's own bundle files get this process's debug IDs; browser + // frames bring theirs along. + frames: report.frames.slice(0, MAX_FRAMES_KEPT).map((frame) => { + if (frame.debugId) return frame; + const id = runtime === 'server' ? debugIdsByPath().get(frame.path) : undefined; + return id ? { ...frame, debugId: id } : frame; + }), + version, + ...(isErrorCode(report.code) ? { code: report.code } : {}), + runtime, + count: 1, + }; + } + startTimer(); + return print; +} + +function startTimer(): void { + const s = state(); + if (s.timer) return; + s.timer = setInterval(() => { void flushErrors(); }, FLUSH_MS); + s.timer.unref?.(); +} + +/** Add what this process counted since the last flush to the stored aggregate. */ +export function flushErrors(): Promise { + const s = state(); + if (s.flushing) return s.flushing; + const delta = s.pending; + if (Object.keys(delta).length === 0) return Promise.resolve(); + s.pending = {}; + s.flushing = telemetryDocs() + .update(ERRORS_DOC, (current) => JSON.stringify(bound(merge(parseStored(current), delta)))) + .then(() => undefined) + .catch((error) => { + // Keep the counts for the next try. + s.pending = merge(delta, s.pending); + logger.warn('error counts not stored', { error: error instanceof Error ? error.message : String(error) }); + }) + .finally(() => { s.flushing = null; }); + return s.flushing; +} + +/** The stored aggregate with this process's unflushed counts added. */ +export async function readErrors(): Promise { + let stored: Aggregate = {}; + try { + stored = parseStored((await telemetryDocs().get(ERRORS_DOC))?.data ?? null); + } catch (error) { + logger.warn('error counts not readable', { error: error instanceof Error ? error.message : String(error) }); + } + return bound(merge(stored, state().pending)); +} + +export interface DaySummary { + total: number; + top: Array<{ fingerprint: string } & AggregateEntry>; +} + +export function summarizeDay(agg: Aggregate, day = utcDay(), limit = 20): DaySummary { + const entries = Object.entries(agg[day] ?? {}); + return { + total: entries.reduce((sum, [, e]) => sum + e.count, 0), + top: entries + .sort(([, a], [, b]) => b.count - a.count) + .slice(0, limit) + .map(([fingerprint, e]) => ({ fingerprint, ...e })), + }; +} + +/** For tests. */ +export function resetAggregateForTests(): void { + const s = state(); + if (s.timer) clearInterval(s.timer); + const g = globalThis as typeof globalThis & { [KEY]?: AggregateState }; + delete g[KEY]; +} diff --git a/lib/error-reporting/client.ts b/lib/error-reporting/client.ts new file mode 100644 index 000000000..23d6e69db --- /dev/null +++ b/lib/error-reporting/client.ts @@ -0,0 +1,253 @@ +import { apiFetch, getPathPrefix } from '@/lib/browser-navigation'; +import { scrubRoutePath } from '@/lib/route-groups'; +import { debugIdsByPath } from './debug-ids'; +import { + appVersion, buildEnvelope, buildEvent, envelopeUrl, isErrorCode, newEventId, parseDsn, releaseName, + type Breadcrumb, type ErrorReport, +} from './envelope'; +import { fingerprint } from './fingerprint'; +import { ReportLimiter } from './limiter'; +import { scrubJmapErrorType, scrubMethodName, scrubName, scrubText } from './scrub'; +import { parseStack } from './stack'; + +/** + * Error reporting in the browser, off until the config names a DSN or asks + * for errors to be collected (lib/error-reporting/settings.ts). Captures what + * the error boundaries, window.onerror, unhandled rejections and failed JMAP + * requests hand it, scrubs it (./scrub.ts) and sends it to the DSN and/or + * this server's /api/errors. Nothing is sent before the config arrived, and + * nothing at all when both are off. + * + * https://bulwarkmail.org/docs/deployment/monitoring + */ + +export interface ClientSettings { + /** The error tracker's DSN; '' for none. */ + dsn: string; + /** 0..1, applied to what goes to the DSN. */ + sampleRate: number; + environment: string; + /** Also post errors to this server's /api/errors (the per-day counts). */ + collect: boolean; +} + +const MAX_PENDING = 10; +const MAX_BREADCRUMBS = 20; + +interface ClientState { + /** undefined until configured; null when reporting is off. */ + settings: ClientSettings | null | undefined; + pending: ErrorReport[]; + breadcrumbs: Breadcrumb[]; + limiter: ReportLimiter | null; + installed: boolean; +} + +// On globalThis: the error boundary, the JMAP client and the root component +// may each end up with their own copy of this module. +const KEY = Symbol.for('bulwark.error-reporting.client'); +function state(): ClientState { + const g = globalThis as typeof globalThis & { [KEY]?: ClientState }; + return (g[KEY] ??= { settings: undefined, pending: [], breadcrumbs: [], limiter: null, installed: false }); +} + +function sessionStore(): Storage | null { + try { + return typeof sessionStorage === 'undefined' ? null : sessionStorage; + } catch { + return null; + } +} + +function limiter(): ReportLimiter { + const s = state(); + return (s.limiter ??= new ReportLimiter({ perMinute: 10, perSession: 30, dedupWindowMs: Infinity, storage: sessionStore() })); +} + +function basePath(): string { + return getPathPrefix(); +} + +function currentRoute(): string | undefined { + if (typeof location === 'undefined') return undefined; + return scrubRoutePath(location.pathname, basePath()); +} + +/** Apply the settings from /api/config (or Lite's policy.json); null switches reporting off. */ +export function configureErrorReporting(settings: ClientSettings | null): void { + const s = state(); + const on = settings && (parseDsn(settings.dsn) || settings.collect) ? settings : null; + s.settings = on; + const queued = s.pending; + s.pending = []; + if (on) for (const report of queued) deliver(report); +} + +export function isErrorReportingOn(): boolean { + return !!state().settings; +} + +/** A route change or a JMAP call, kept for the next report. */ +export function addBreadcrumb(category: Breadcrumb['category'], message: string): void { + const s = state(); + if (s.settings === null) return; + s.breadcrumbs.push({ timestamp: Date.now() / 1000, category, message }); + if (s.breadcrumbs.length > MAX_BREADCRUMBS) s.breadcrumbs.shift(); +} + +/** A route change: only the scrubbed route is kept. */ +export function recordNavigation(pathname: string): void { + addBreadcrumb('navigation', scrubRoutePath(pathname, basePath())); +} + +/** A JMAP request: only the method names. */ +export function recordJmapCall(methods: readonly string[]): void { + addBreadcrumb('jmap', methods.slice(0, 5).map(scrubMethodName).join(', ')); +} + +/** Turn anything thrown into a scrubbed report; null for what isn't worth one. */ +export function reportFromError(error: unknown, mechanism: string, handled: boolean): ErrorReport | null { + let name = 'Error'; + let message = ''; + let stack: string | undefined; + if (error instanceof Error || (error && typeof error === 'object' && 'message' in error)) { + const e = error as { name?: unknown; message?: unknown; stack?: unknown }; + name = scrubName(e.name); + message = scrubText(e.message); + stack = typeof e.stack === 'string' ? e.stack : undefined; + } else if (typeof error === 'string') { + name = mechanism === 'unhandledrejection' ? 'UnhandledRejection' : 'Error'; + message = scrubText(error); + } else { + return null; + } + const frames = parseStack(stack); + // Every frame outside our bundles: a browser extension or injected script. + if (frames.length > 0 && frames.every((frame) => frame.path === '[other]')) return null; + return { name, message, frames, handled, mechanism, route: currentRoute() }; +} + +/** Report an error caught by a boundary, a global handler or a caller. */ +export function captureException(error: unknown, mechanism = 'manual', handled = true): void { + try { + const report = reportFromError(error, mechanism, handled); + if (report) captureReport(report); + } catch { + // Reporting must never throw into the code that reported. + } +} + +/** Report a JMAP request that failed: the method names and the status or error type only. */ +export function reportJmapFailure(failure: { status: number } | { method: string; type: string }): void { + try { + const code = 'status' in failure + ? `http:${Number.isInteger(failure.status) ? failure.status : 0}` + : `${scrubMethodName(failure.method)}: ${scrubJmapErrorType(failure.type)}`; + if (!isErrorCode(code)) return; + const name = 'status' in failure ? 'JmapRequestError' : 'JmapMethodError'; + const frames = parseStack(new Error().stack).slice(1); + captureReport({ + name, + message: code, + code, + frames, + handled: true, + mechanism: 'jmap', + route: currentRoute(), + }); + } catch { + // never throws into the JMAP client + } +} + +export function captureReport(report: ErrorReport): void { + const s = state(); + if (s.settings === null) return; + const withCrumbs = { ...report, breadcrumbs: report.breadcrumbs ?? [...s.breadcrumbs] }; + if (s.settings === undefined) { + if (s.pending.length < MAX_PENDING) s.pending.push(withCrumbs); + return; + } + deliver(withCrumbs); +} + +function deliver(report: ErrorReport): void { + const s = state(); + const settings = s.settings; + if (!settings) return; + const version = appVersion(); + const print = fingerprint(report.name, report.frames, version, report.code); + if (!limiter().allow(print)) return; + + const dsn = parseDsn(settings.dsn); + if (dsn && Math.random() < settings.sampleRate) { + const event = buildEvent(report, { + eventId: newEventId(), + timestamp: Date.now() / 1000, + release: releaseName(), + environment: settings.environment, + platform: 'javascript', + fingerprint: print, + debugIds: debugIdsByPath(), + tags: { runtime: 'browser', ...(process.env.NEXT_PUBLIC_BULWARK_LITE === '1' ? { lite: 'yes' } : {}) }, + }); + send(envelopeUrl(dsn), buildEnvelope(event), { + // text/plain keeps it a simple request: no preflight, no custom header. + headers: { 'Content-Type': 'text/plain;charset=UTF-8' }, + credentials: 'omit', + // The page URL holds folder names and message ids. + referrerPolicy: 'no-referrer', + mode: 'cors', + }); + } + // Lite has no server. Read here, not through IS_LITE: the bundler inlines + // the variable only in this module, so the branch and the endpoint string + // leave the Lite bundle (scripts/lite/verify.mjs). + if (process.env.NEXT_PUBLIC_BULWARK_LITE !== '1' && settings.collect) { + send('/api/errors', JSON.stringify({ + name: report.name, + message: report.message, + code: report.code, + // With debug IDs, so the counts can point at the uploaded source maps. + frames: withDebugIds(report.frames.slice(0, 10)), + mechanism: report.mechanism, + handled: report.handled, + route: report.route, + version, + }), { headers: { 'Content-Type': 'application/json' }, credentials: 'same-origin' }, true); + } +} + +// A failed send is dropped quietly: it never turns into another report. +function send(url: string, body: string, init: RequestInit, sameOrigin = false): void { + const request = sameOrigin + ? apiFetch(url, { method: 'POST', body, keepalive: true, ...init }) + : fetch(url, { method: 'POST', body, keepalive: true, ...init }); + void Promise.resolve(request).catch(() => undefined); +} + +function withDebugIds(frames: ErrorReport['frames']): ErrorReport['frames'] { + const ids = debugIdsByPath(); + return frames.map((frame) => (ids.has(frame.path) ? { ...frame, debugId: ids.get(frame.path) } : frame)); +} + +/** window.onerror and unhandled rejections, once. */ +export function installGlobalHandlers(): void { + const s = state(); + if (s.installed || typeof window === 'undefined') return; + s.installed = true; + window.addEventListener('error', (event: ErrorEvent) => { + // "Script error." from another origin carries nothing to report. + if (!event.error) return; + captureException(event.error, 'onerror', false); + }); + window.addEventListener('unhandledrejection', (event: PromiseRejectionEvent) => { + captureException(event.reason, 'unhandledrejection', false); + }); +} + +/** For tests. */ +export function resetErrorReportingForTests(): void { + const g = globalThis as typeof globalThis & { [KEY]?: ClientState }; + delete g[KEY]; +} diff --git a/lib/error-reporting/debug-ids.ts b/lib/error-reporting/debug-ids.ts new file mode 100644 index 000000000..88ae156d9 --- /dev/null +++ b/lib/error-reporting/debug-ids.ts @@ -0,0 +1,47 @@ +import { parseStack } from './stack'; + +/** + * The debug ID of each loaded bundle file, by scrubbed path. + * + * Turbopack (`turbopack.debugIds`) and `sentry-cli sourcemaps inject` (the + * webpack build in Docker) start every chunk with a line that stores + * `globalThis[KEY][new Error().stack] = ""`. The first frame of that + * stack is the chunk itself, so its path maps to the id. The error tracker + * finds the uploaded source map by that id alone. + */ + +const KEYS = ['_debugIds', '_sentryDebugIds'] as const; +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +let cache: { size: number; ids: Map } | null = null; + +function registered(): Array<[string, string]> { + const g = globalThis as Record; + const out: Array<[string, string]> = []; + for (const key of KEYS) { + const map = g[key]; + if (!map || typeof map !== 'object') continue; + for (const [stack, id] of Object.entries(map as Record)) { + if (typeof id === 'string' && UUID.test(id)) out.push([stack, id.toLowerCase()]); + } + } + return out; +} + +/** Path -> debug ID for every chunk loaded so far. Chunks load lazily, so this is rebuilt when more have registered. */ +export function debugIdsByPath(): Map { + const entries = registered(); + if (cache && cache.size === entries.length) return cache.ids; + const ids = new Map(); + for (const [stack, id] of entries) { + const first = parseStack(stack).find((frame) => frame.path !== '[other]'); + if (first) ids.set(first.path, id); + } + cache = { size: entries.length, ids }; + return ids; +} + +/** For tests. */ +export function resetDebugIdCache(): void { + cache = null; +} diff --git a/lib/error-reporting/envelope.ts b/lib/error-reporting/envelope.ts new file mode 100644 index 000000000..aaad77e1b --- /dev/null +++ b/lib/error-reporting/envelope.ts @@ -0,0 +1,149 @@ +import type { Frame } from './stack'; + +/** + * The Sentry envelope format, which Bugsink, GlitchTip and Sentry accept. + * Only what Bulwark sends: one event item per envelope. + * https://develop.sentry.dev/sdk/data-model/envelopes/ + */ + +export interface Dsn { + /** scheme://host[:port] */ + origin: string; + /** Anything between the host and the project id, usually ''. */ + pathPrefix: string; + publicKey: string; + projectId: string; +} + +/** `https://@host[:port][/prefix]/`, or null. */ +export function parseDsn(raw: unknown): Dsn | null { + if (typeof raw !== 'string' || !raw.trim()) return null; + let url: URL; + try { + url = new URL(raw.trim()); + } catch { + return null; + } + if (url.protocol !== 'https:' && url.protocol !== 'http:') return null; + if (!url.username || url.password || url.search || url.hash) return null; + const parts = url.pathname.split('/').filter(Boolean); + const projectId = parts.pop(); + if (!projectId || !/^\d+$/.test(projectId) || !/^[0-9a-f]{16,64}$/i.test(url.username)) return null; + return { + origin: `${url.protocol}//${url.host}`, + pathPrefix: parts.length ? `/${parts.join('/')}` : '', + publicKey: url.username, + projectId, + }; +} + +/** Where envelopes go. The key travels in the query, so a browser needs no custom header. */ +export function envelopeUrl(dsn: Dsn): string { + return `${dsn.origin}${dsn.pathPrefix}/api/${dsn.projectId}/envelope/?sentry_key=${dsn.publicKey}&sentry_version=7`; +} + +/** A fixed error code: `Email/set: overQuota`, `serverFail`, `http:503`. */ +export const ERROR_CODE = /^(?:(?:[A-Z][A-Za-z]{0,40}\/[a-z][A-Za-z]{0,40}: )?[a-z][A-Za-z]{1,40}|http:\d{3})$/; + +export function isErrorCode(value: unknown): value is string { + return typeof value === 'string' && ERROR_CODE.test(value); +} + +export interface Breadcrumb { + /** Seconds since the epoch. */ + timestamp: number; + category: 'navigation' | 'jmap'; + /** A scrubbed route (`/mail/…`) or a JMAP method name. */ + message: string; +} + +/** An error, scrubbed, as both sides hand it to the sender. */ +export interface ErrorReport { + name: string; + message: string; + frames: Frame[]; + /** One of Bulwark's fixed codes, when the error has one. */ + code?: string; + handled: boolean; + /** How it was caught: onerror, unhandledrejection, boundary, global-error, jmap, request, logger. */ + mechanism: string; + breadcrumbs?: Breadcrumb[]; + /** The scrubbed route it happened on. */ + route?: string; +} + +export interface EventContext { + eventId: string; + /** Seconds since the epoch. */ + timestamp: number; + release: string; + environment: string; + platform: 'javascript' | 'node'; + fingerprint: string; + /** Path -> debug id, for the frames' files. */ + debugIds: Map; + tags?: Record; +} + +/** The event payload. Frames go oldest first, as the format wants. */ +export function buildEvent(report: ErrorReport, ctx: EventContext): Record { + const frames = [...report.frames].reverse().map((frame) => ({ + abs_path: frame.path, + filename: frame.path, + ...(frame.fn ? { function: frame.fn } : {}), + ...(frame.line !== undefined ? { lineno: frame.line } : {}), + ...(frame.column !== undefined ? { colno: frame.column } : {}), + in_app: frame.path.startsWith('/_next/') || frame.path.startsWith('/.next/'), + })); + const images = [...new Set(report.frames.map((frame) => frame.path))] + .filter((path) => ctx.debugIds.has(path)) + .map((path) => ({ type: 'sourcemap', code_file: path, debug_id: ctx.debugIds.get(path) })); + return { + event_id: ctx.eventId, + timestamp: ctx.timestamp, + platform: ctx.platform, + level: 'error', + release: ctx.release, + environment: ctx.environment, + // Bulwark's own grouping: the same key as the per-day counts. + fingerprint: [ctx.fingerprint], + exception: { + values: [{ + type: report.name, + value: report.code ?? report.message, + mechanism: { type: report.mechanism, handled: report.handled }, + ...(frames.length ? { stacktrace: { frames } } : {}), + }], + }, + ...(report.breadcrumbs?.length + ? { breadcrumbs: { values: report.breadcrumbs.map((b) => ({ timestamp: b.timestamp, category: b.category, message: b.message })) } } + : {}), + ...(images.length ? { debug_meta: { images } } : {}), + tags: { ...(ctx.tags ?? {}), ...(report.route ? { route: report.route } : {}) }, + sdk: { name: 'bulwark.error-reporting', version: '1' }, + }; +} + +/** One event in an envelope. */ +export function buildEnvelope(event: Record): string { + const header = { event_id: event.event_id, sent_at: new Date().toISOString() }; + return `${JSON.stringify(header)}\n${JSON.stringify({ type: 'event', content_type: 'application/json' })}\n${JSON.stringify(event)}\n`; +} + +/** 32 hex digits, as event ids are. */ +export function newEventId(): string { + const bytes = new Uint8Array(16); + globalThis.crypto.getRandomValues(bytes); + return Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join(''); +} + +/** `+`, the release the source maps were uploaded for. */ +export function releaseName(): string { + const version = process.env.NEXT_PUBLIC_APP_VERSION || '0.0.0'; + const commit = process.env.NEXT_PUBLIC_GIT_COMMIT; + return commit && commit !== 'unknown' ? `${version}+${commit}` : version; +} + +export function appVersion(): string { + return process.env.NEXT_PUBLIC_APP_VERSION || '0.0.0'; +} diff --git a/lib/error-reporting/fingerprint.ts b/lib/error-reporting/fingerprint.ts new file mode 100644 index 000000000..f57a1e537 --- /dev/null +++ b/lib/error-reporting/fingerprint.ts @@ -0,0 +1,34 @@ +import type { Frame } from './stack'; + +/** How many of the top frames make up a fingerprint. */ +export const FINGERPRINT_FRAMES = 5; + +/** + * 64-bit FNV-1a as 16 hex digits. Synchronous and the same in the browser and + * on the server; not for security, only for grouping. + */ +export function hash64(text: string): string { + let h1 = 0x811c9dc5; + let h2 = 0xcbf29ce4; + for (let i = 0; i < text.length; i++) { + const c = text.charCodeAt(i); + h1 = Math.imul(h1 ^ c, 0x01000193) >>> 0; + h2 = Math.imul(h2 ^ c ^ (h1 >>> 7), 0x01000193) >>> 0; + } + return h1.toString(16).padStart(8, '0') + h2.toString(16).padStart(8, '0'); +} + +/** + * The fingerprint of an error: its name, the top frames (file without origin, + * line, column) and the Bulwark version. The same error in another release is + * another fingerprint, because its frames point into other chunks anyway. + * `code` is one of Bulwark's fixed error codes (a JMAP error type), for + * errors that are all raised from the same place. + */ +export function fingerprint(name: string, frames: readonly Frame[], version: string, code?: string): string { + const top = frames + .filter((frame) => frame.path !== '[other]') + .slice(0, FINGERPRINT_FRAMES) + .map((frame) => `${frame.path}:${frame.line ?? ''}:${frame.column ?? ''}`); + return hash64([name, code ?? '', ...top, version].join('|')); +} diff --git a/lib/error-reporting/limiter.ts b/lib/error-reporting/limiter.ts new file mode 100644 index 000000000..69bbc332c --- /dev/null +++ b/lib/error-reporting/limiter.ts @@ -0,0 +1,81 @@ +/** + * Keeps error reporting from flooding anyone: each fingerprint once per + * window, at most `perMinute` reports in any minute and `perSession` in all. + * The browser keeps its state in sessionStorage, so a page that crashes on + * every load doesn't start over each time. + */ + +export interface LimiterOptions { + perMinute: number; + perSession: number; + /** How long a fingerprint stays "already sent"; Infinity for the whole session. */ + dedupWindowMs: number; + /** Persist across page loads (sessionStorage); absent on the server. */ + storage?: Pick | null; + storageKey?: string; + now?: () => number; +} + +interface State { + total: number; + sent: Record; +} + +const MAX_REMEMBERED = 500; + +export class ReportLimiter { + private readonly options: Required> & { storage: LimiterOptions['storage'] }; + private recent: number[] = []; + private state: State = { total: 0, sent: {} }; + + constructor(options: LimiterOptions) { + this.options = { storageKey: 'bulwark.error-reporting', now: () => Date.now(), storage: null, ...options }; + this.load(); + } + + private load(): void { + const { storage, storageKey } = this.options; + if (!storage) return; + try { + const raw = storage.getItem(storageKey); + if (!raw) return; + const parsed = JSON.parse(raw) as Partial; + if (typeof parsed.total === 'number' && parsed.sent && typeof parsed.sent === 'object') { + this.state = { total: parsed.total, sent: parsed.sent as Record }; + } + } catch { + // Unreadable or blocked storage: start fresh, in memory. + } + } + + private save(): void { + const { storage, storageKey } = this.options; + if (!storage) return; + try { + storage.setItem(storageKey, JSON.stringify(this.state)); + } catch { + // Quota or privacy mode: the in-memory state still applies. + } + } + + /** Whether a report with this fingerprint may go out now; counts it when it may. */ + allow(fingerprint: string): boolean { + const now = this.options.now(); + const last = this.state.sent[fingerprint]; + if (last !== undefined && now - last < this.options.dedupWindowMs) return false; + if (this.state.total >= this.options.perSession) return false; + this.recent = this.recent.filter((at) => now - at < 60_000); + if (this.recent.length >= this.options.perMinute) return false; + this.recent.push(now); + this.state.total += 1; + this.state.sent[fingerprint] = now; + const keys = Object.keys(this.state.sent); + if (keys.length > MAX_REMEMBERED) { + for (const key of keys.sort((a, b) => this.state.sent[a] - this.state.sent[b]).slice(0, keys.length - MAX_REMEMBERED)) { + delete this.state.sent[key]; + } + } + this.save(); + return true; + } +} diff --git a/lib/error-reporting/lite.ts b/lib/error-reporting/lite.ts new file mode 100644 index 000000000..3ab6bdad7 --- /dev/null +++ b/lib/error-reporting/lite.ts @@ -0,0 +1,22 @@ +import type { ClientSettings } from './client'; +import { parseDsn } from './envelope'; + +/** + * Bulwark Lite has no server: error reporting is set in policy.json and + * browsers send straight to the DSN. There are no per-day counts. + * + * "errorReporting": { "dsn": "https://@host/", "sampleRate": 1, "environment": "production" } + * + * The DSN's host must also be in the CSP's connect-src of the Lite host. + * https://bulwarkmail.org/docs/deployment/static#error-reporting + */ +export function liteErrorReporting(policy: unknown): ClientSettings | null { + const raw = policy && typeof policy === 'object' ? (policy as Record).errorReporting : undefined; + if (!raw || typeof raw !== 'object') return null; + const input = raw as Record; + const dsn = typeof input.dsn === 'string' && parseDsn(input.dsn) ? input.dsn.trim() : ''; + if (!dsn) return null; + const rate = typeof input.sampleRate === 'number' && Number.isFinite(input.sampleRate) ? Math.min(1, Math.max(0, input.sampleRate)) : 1; + const environment = typeof input.environment === 'string' && /^[\w.-]{1,64}$/.test(input.environment) ? input.environment : 'production'; + return { dsn, sampleRate: rate, environment, collect: false }; +} diff --git a/lib/error-reporting/scrub.ts b/lib/error-reporting/scrub.ts new file mode 100644 index 000000000..9c24601e4 --- /dev/null +++ b/lib/error-reporting/scrub.ts @@ -0,0 +1,163 @@ +/** + * What an error report may carry, applied in the browser before sending and + * again on the server for everything a browser posts (lib/error-reporting). + * + * Mail content must never leave the browser in an error, not even to the + * admin's own error tracker: no message bodies, subjects, addresses, file + * names, tokens, cookies or URLs. Error messages are free text, so the rules + * remove whatever could be such content: everything after the first line, + * quoted text, addresses, URLs, file names, credentials, long random-looking + * strings and key=value pairs, and then every word that is not on a fixed + * list of common words. What stays is the shape of the message + * ("Cannot read properties of undefined (reading "…")"). + * + * Documented at https://bulwarkmail.org/docs/deployment/monitoring#what-is-removed + */ + +export const MAX_MESSAGE_LENGTH = 200; + +const REDACTED = '…'; + +// Quoted text: "…", '…', `…`, “…”, „…“, ‘…’, «…», ‹…›. A single quote only +// counts when it starts after a space or bracket, so "can't" stays a word. +const QUOTED: RegExp[] = [ + /"[^"\n]*"/g, + /`[^`\n]*`/g, + /[“„][^”“\n]*[”“]/g, + /‘[^’\n]*’/g, + /«[^»\n]*»/g, + /‹[^›\n]*›/g, + /(^|[\s([{:,=])'[^'\n]*'(?=$|[\s)\]}.,:;!?])/g, +]; + +const EMAIL = /[^\s@<>"'`()[\]{},;:]+@[^\s@<>"'`()[\]{},;:]+\.[^\s@<>"'`()[\]{},;:]+/g; +const URL_LIKE = /\b[a-z][a-z0-9+.-]*:\/\/[^\s"'`<>()]*/gi; +const WWW = /\bwww\.[^\s"'`<>()]+/gi; +const AUTH = /\b(Bearer|Basic|Token)\s+[^\s,;]+/gi; +const JWT = /\beyJ[\w-]{5,}\.[\w-]{5,}\.[\w-]*/g; +const KEY_VALUE = /\b([A-Za-z_][\w.-]{0,40})=([^\s&;,]+)/g; +const FILE_NAME = /[^\s/\\"'`<>()]+\.(?:pdf|docx?|xlsx?|pptx?|odt|ods|odp|rtf|txt|csv|eml|msg|ics|vcf|zip|rar|7z|tar|gz|png|jpe?g|gif|webp|heic|svg|mp[34]|mov|wav|html?|xml|json|p7[msc]|asc|gpg|pgp|key|pem)\b/gi; +// Runs of letters and digits that look random: ids, hashes, keys, tokens. +const LONG_RUN = /[A-Za-z0-9+/_=-]{24,}/g; +const IPV4 = /\b\d{1,3}(?:\.\d{1,3}){3}\b/g; +const PHONE = /\+?\d[\d ()-]{7,}\d/g; + +function looksRandom(run: string): boolean { + // Long identifiers in code are words glued together: keep them when they + // are only letters. Anything with digits or symbols mixed in goes. + return /\d/.test(run) || /[+/=_-]/.test(run) || run.length > 40; +} + +/** Free text from an error, made safe to send. */ +export function scrubText(input: unknown): string { + if (typeof input !== 'string' || input === '') return ''; + // Only the first line: server answers, stack traces and multi-line content + // that ended up in a message all come after it. + let text = input.split(/\r?\n/, 1)[0]; + // "Request failed: 400 - ": the answer may echo a subject. + text = text.replace(/^(.*?\b\d{3})\s+-\s.*$/, '$1'); + text = text.replace(AUTH, '$1 [token]'); + text = text.replace(JWT, '[token]'); + text = text.replace(URL_LIKE, '[url]'); + text = text.replace(WWW, '[url]'); + text = text.replace(EMAIL, '[email]'); + for (const quoted of QUOTED) { + text = text.replace(quoted, (match, lead?: string) => { + const prefix = typeof lead === 'string' ? lead : ''; + const quote = match.slice(prefix.length, prefix.length + 1); + const close = match.slice(-1); + return `${prefix}${quote}${REDACTED}${close}`; + }); + } + text = text.replace(FILE_NAME, '[file]'); + text = text.replace(KEY_VALUE, '$1=[redacted]'); + text = text.replace(LONG_RUN, (run) => (looksRandom(run) ? '[id]' : run)); + text = text.replace(IPV4, '[ip]'); + text = text.replace(PHONE, '[number]'); + // Last, and the rule the others only make readable: a word stays only when + // it is on the list of common words. Free text such as an unquoted subject + // or folder name ("to Acquisition Talks") becomes "…". + text = text.replace(WORD_OR_PLACEHOLDER, (token) => ( + PLACEHOLDER.test(token) || COMMON_WORDS.has(token.toLowerCase()) ? token : REDACTED + )); + text = text.replace(/\d{5,}/g, REDACTED).replace(/…(?:[\s,;]+…)+/g, REDACTED); + if (text.length > MAX_MESSAGE_LENGTH) text = `${text.slice(0, MAX_MESSAGE_LENGTH - 1)}…`; + return text; +} + +const PLACEHOLDER = /^\[(?:email|url|token|file|id|ip|number|redacted)\]$/; +const WORD_OR_PLACEHOLDER = /\[(?:email|url|token|file|id|ip|number|redacted)\]|[\p{L}_$][\p{L}\p{N}_$'’.-]*/gu; + +/** + * Words an error message may keep: English function words and the words + * JavaScript engines, browsers, Next.js and Bulwark's own errors are made of. + * Anything else could be content and is replaced. + */ +const COMMON_WORDS = new Set(` +a about above access after again against all already an and another any are argument arguments array as at attempt +attempted available bad be been before being body both but by call called calling can can't cannot canceled cancelled +change changed changes child chunk class client close closed code component connect connected connection constructor +context could create created data date decode default defined delete deleted denied did does doesn't done during each +element empty end entry error errors event exceed exceeded exist exists expected failed fetch field file files find +first for found from function get given got had has have header headers here hook hydration if in index initial +instead internal invalid is isn't it item items its json key keys length limit list load loaded loading lost +mailbox mailboxes match maximum may maybe message messages method missing more must network new no node none not +null number object of offline on once one only open operation or out over parse parsing permission please property +properties provided read reading ready received recursion redirect refused rejected render rendered rendering request +requested requests required resource response result retry returned save saved script send sent server session set +should size some stack state status still string such support supported sync syntax than that the then there these +this thread timed timeout to token too type unable undefined unexpected unknown update updated upload uploaded use +used user using valid value values was were when which while with within without worker write wrong +abort aborted account accounts attachment attachments auth authentication authorization basic bearer blob broke broken check health heartbeat metrics something telemetry version blocked browser cache calendar callback contact contacts cookie +cors credentials decrypt denied download draft email emails encrypt encrypted expired failure folder folders +identity identities import keyword keywords login logout mail move moved oauth outbox plugin +plugins push quota rate reply sender settings signature submission subscription template theme upload websocket +`.split(/\s+/).filter(Boolean)); + +/** An error name: a code identifier, or "Error". */ +export function scrubName(input: unknown): string { + return typeof input === 'string' && /^[A-Za-z_$][\w$]{0,63}$/.test(input) ? input : 'Error'; +} + +/** A function name from a stack frame: identifiers and the usual decorations only. */ +export function scrubFunctionName(input: unknown): string | undefined { + if (typeof input !== 'string' || input === '') return undefined; + const trimmed = input.trim().slice(0, 120); + return /^[\w$.<>[\] /#-]+$/.test(trimmed) && !/@/.test(trimmed) ? trimmed : '?'; +} + +/** + * The path of a stack frame's file, without origin, base path, query or + * fragment: `/_next/static/chunks/x.js` for the browser bundle, + * `/.next/server/...` for the server's, `node:...` for Node's own modules, + * `node_modules/...` for dependencies; `[other]` for anything else + * (extensions, plugin code, evaluated code). + */ +export function scrubFramePath(input: unknown): string { + if (typeof input !== 'string' || input === '') return '[other]'; + const path = input.split(/[?#]/, 1)[0].replace(/\\/g, '/'); + for (const marker of ['/_next/static/', '/.next/server/', '/.next/static/']) { + const at = path.indexOf(marker); + if (at >= 0) { + const rest = path.slice(at); + return /^[\w./()[\]@+~-]+$/.test(rest) ? rest.replace('/.next/static/', '/_next/static/') : '[other]'; + } + } + if (/^node:[\w/.-]+$/.test(path)) return path; + const modules = path.lastIndexOf('/node_modules/'); + if (modules >= 0) { + const rest = path.slice(modules + 1); + return /^[\w./@+~-]+$/.test(rest) ? rest : '[other]'; + } + return '[other]'; +} + +/** A JMAP method name ("Email/get") or a fixed placeholder. */ +export function scrubMethodName(input: unknown): string { + return typeof input === 'string' && /^[A-Z][A-Za-z]{0,40}\/[a-z][A-Za-z]{0,40}$/.test(input) ? input : 'Unknown/method'; +} + +/** A JMAP error type ("serverFail", "overQuota") or a fixed placeholder. */ +export function scrubJmapErrorType(input: unknown): string { + return typeof input === 'string' && /^[a-z][A-Za-z]{1,40}$/.test(input) ? input : 'unknown'; +} diff --git a/lib/error-reporting/server.ts b/lib/error-reporting/server.ts new file mode 100644 index 000000000..d94e728c3 --- /dev/null +++ b/lib/error-reporting/server.ts @@ -0,0 +1,132 @@ +import { configManager } from '@/lib/admin/config-manager'; +import { recordError } from './aggregate'; +import { + appVersion, buildEnvelope, buildEvent, envelopeUrl, newEventId, parseDsn, releaseName, type ErrorReport, +} from './envelope'; +import { fingerprint } from './fingerprint'; +import { ReportLimiter } from './limiter'; +import { errorReportingSettings, shouldCollectErrors } from './settings'; +import { scrubName, scrubText } from './scrub'; +import { parseStack, type Frame } from './stack'; +import { debugIdsByPath } from './debug-ids'; + +/** + * Error reporting on the server: errors Next hands to onRequestError + * (instrumentation.ts) and errors routes catch and log with logger.error. + * While collection is on (a DSN, the metrics endpoint or error-count + * consent, lib/error-reporting/settings.ts) every one is counted in the + * per-day aggregate; with a DSN it is also sent there, at most once an hour + * per fingerprint and 60 a minute. Sending never blocks or fails the request + * that had the error. With everything off nothing happens at all. + */ + +const SEND_TIMEOUT_MS = 5000; +const ROUTE_PATTERN = /^\/[\w/[\]().@-]*$/; + +interface ServerState { + limiter: ReportLimiter; + sending: number; +} + +const KEY = Symbol.for('bulwark.error-reporting.server'); +function state(): ServerState { + const g = globalThis as typeof globalThis & { [KEY]?: ServerState }; + return (g[KEY] ??= { + limiter: new ReportLimiter({ perMinute: 60, perSession: Number.POSITIVE_INFINITY, dedupWindowMs: 60 * 60_000 }), + sending: 0, + }); +} + +async function deliver(report: ErrorReport): Promise { + if (!(await shouldCollectErrors())) return; + const version = appVersion(); + recordError(report, 'server', version); + const settings = errorReportingSettings(); + const dsn = parseDsn(settings.dsn); + if (!dsn) return; + const print = fingerprint(report.name, report.frames, version, report.code); + const s = state(); + if (s.sending > 4 || !s.limiter.allow(print)) return; + if (Math.random() >= settings.sampleRate) return; + const event = buildEvent(report, { + eventId: newEventId(), + timestamp: Date.now() / 1000, + release: releaseName(), + environment: settings.environment, + platform: 'node', + fingerprint: print, + debugIds: debugIdsByPath(), + tags: { runtime: 'server' }, + }); + s.sending++; + try { + await fetch(envelopeUrl(dsn), { + method: 'POST', + headers: { 'Content-Type': 'application/x-sentry-envelope' }, + body: buildEnvelope(event), + redirect: 'manual', + signal: AbortSignal.timeout(SEND_TIMEOUT_MS), + }); + } finally { + s.sending--; + } +} + +/** Count `report` and send it to the DSN when one is set. Never rejects. */ +export function handleServerReport(report: ErrorReport): Promise { + return deliver(report).catch(() => undefined); +} + +/** Next's onRequestError: an error a route, page or server action threw. */ +export function captureRequestError(error: unknown, routePattern?: string): Promise { + try { + const e = (error && typeof error === 'object' ? error : { message: String(error) }) as { name?: unknown; message?: unknown; stack?: unknown }; + return handleServerReport({ + name: scrubName(e.name), + message: scrubText(e.message), + frames: parseStack(e.stack), + handled: false, + mechanism: 'request', + // A route pattern from the code ("/api/admin/[id]"), never the request's URL. + ...(typeof routePattern === 'string' && ROUTE_PATTERN.test(routePattern) ? { route: routePattern } : {}), + }); + } catch { + return Promise.resolve(); + } +} + +/** + * logger.error: a route that caught an error and logged it. The frames are + * the logging call's, minus the logger's own `skip` frames. + */ +export function captureLoggedError( + message: string, + extra: Record | undefined, + stack: string | undefined, + skip = 2, +): Promise { + try { + const detail = typeof extra?.error === 'string' ? extra.error : extra?.error instanceof Error ? extra.error.message : ''; + const frames: Frame[] = parseStack(stack).slice(skip); + return handleServerReport({ + name: 'LoggedError', + message: scrubText(detail ? `${message}: ${detail}` : message), + frames, + handled: true, + mechanism: 'logger', + }); + } catch { + return Promise.resolve(); + } +} + +/** + * Hook the logger up (instrumentation.node.ts). The logger only knows a + * function on globalThis, so it stays free of config code and safe for any + * bundle that imports it. + */ +export async function installServerErrorReporting(): Promise { + await configManager.ensureLoaded(); + const { setErrorSink } = await import('@/lib/logger'); + setErrorSink((message, extra, stack) => { void captureLoggedError(message, extra, stack); }); +} diff --git a/lib/error-reporting/settings.ts b/lib/error-reporting/settings.ts new file mode 100644 index 000000000..0e4097e32 --- /dev/null +++ b/lib/error-reporting/settings.ts @@ -0,0 +1,56 @@ +import { configManager } from '@/lib/admin/config-manager'; +import { parseDsn } from './envelope'; +import { metricsEnabled } from '@/lib/metrics'; +import { effectiveErrorsConsent } from '@/lib/telemetry/state'; +import type { ClientSettings } from './client'; + +/** + * The error-reporting settings as the server resolves them: admin config, + * then ERROR_REPORTING_DSN(_FILE) / ERROR_REPORTING_SAMPLE_RATE / + * ERROR_REPORTING_ENVIRONMENT, then off. Bulwark Lite reads the same three + * from policy.json (lib/error-reporting/lite.ts). + */ + +export interface ErrorReportingSettings { + /** A valid DSN, or '' when none is set or the value doesn't parse. */ + dsn: string; + sampleRate: number; + environment: string; +} + +export function parseSampleRate(value: unknown): number { + const n = typeof value === 'number' ? value : typeof value === 'string' && value.trim() !== '' ? Number(value) : NaN; + if (!Number.isFinite(n)) return 1; + return Math.min(1, Math.max(0, n)); +} + +export function parseEnvironment(value: unknown): string { + const text = typeof value === 'string' ? value.trim() : ''; + return /^[\w.-]{1,64}$/.test(text) ? text : 'production'; +} + +export function errorReportingSettings(): ErrorReportingSettings { + const raw = configManager.get('errorReportingDsn', ''); + return { + dsn: parseDsn(raw) ? raw.trim() : '', + sampleRate: parseSampleRate(configManager.get('errorReportingSampleRate', '1')), + environment: parseEnvironment(configManager.get('errorReportingEnvironment', 'production')), + }; +} + +/** + * Whether browsers should post their errors to /api/errors: when there is a + * DSN to report to, a metrics endpoint that counts them, or consent to send + * error counts with the telemetry heartbeat. + */ +export async function shouldCollectErrors(): Promise { + await configManager.ensureLoaded(); + if (errorReportingSettings().dsn || metricsEnabled()) return true; + return (await effectiveErrorsConsent()).consent !== 'off'; +} + +/** What /api/config hands the browser. */ +export async function browserErrorSettings(): Promise { + const settings = errorReportingSettings(); + return { ...settings, collect: await shouldCollectErrors() }; +} diff --git a/lib/error-reporting/stack.ts b/lib/error-reporting/stack.ts new file mode 100644 index 000000000..6a6b83f00 --- /dev/null +++ b/lib/error-reporting/stack.ts @@ -0,0 +1,62 @@ +import { scrubFramePath, scrubFunctionName } from './scrub'; + +/** One stack frame, already scrubbed. */ +export interface Frame { + /** The file as scrubFramePath leaves it: `/_next/static/chunks/x.js`. */ + path: string; + line?: number; + column?: number; + fn?: string; + /** The debug ID of the file, when its bundle registered one (./debug-ids.ts). */ + debugId?: string; +} + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +export const MAX_FRAMES = 30; + +// V8 (Chrome, Edge, Node): " at fn (file:1:2)", " at file:1:2", +// " at async fn (file:1:2)", " at new Foo (file:1:2)". +const V8 = /^\s*at (?:(.+?) \()?(.+?):(\d+):(\d+)\)?\s*$/; +// SpiderMonkey and JavaScriptCore (Firefox, Safari): "fn@file:1:2", "@file:1:2". +const GECKO = /^\s*([^@]*)@(.+?):(\d+):(\d+)\s*$/; + +function number(value: string): number | undefined { + const n = Number(value); + return Number.isInteger(n) && n >= 0 && n < 10_000_000 ? n : undefined; +} + +/** The frames of a stack, the frame that threw first, at most MAX_FRAMES. */ +export function parseStack(stack: unknown): Frame[] { + if (typeof stack !== 'string') return []; + const frames: Frame[] = []; + for (const line of stack.split('\n')) { + const match = V8.exec(line) ?? GECKO.exec(line); + if (!match) continue; + const [, fn, file, lineNo, column] = match; + // "at fn (native)" and eval chains carry no file worth keeping. + frames.push({ + path: scrubFramePath(file), + line: number(lineNo), + column: number(column), + fn: scrubFunctionName(fn?.replace(/^async /, '')), + }); + if (frames.length >= MAX_FRAMES) break; + } + return frames; +} + +/** Frames handed in from outside (a browser's report), checked and scrubbed again. */ +export function sanitizeFrames(input: unknown): Frame[] { + if (!Array.isArray(input)) return []; + const out: Frame[] = []; + for (const raw of input.slice(0, MAX_FRAMES)) { + if (!raw || typeof raw !== 'object') continue; + const frame = raw as Record; + const line = typeof frame.line === 'number' ? number(String(frame.line)) : undefined; + const column = typeof frame.column === 'number' ? number(String(frame.column)) : undefined; + const debugId = typeof frame.debugId === 'string' && UUID.test(frame.debugId) ? frame.debugId.toLowerCase() : undefined; + out.push({ path: scrubFramePath(frame.path), line, column, fn: scrubFunctionName(frame.fn), ...(debugId ? { debugId } : {}) }); + } + return out; +} diff --git a/lib/health/__tests__/readiness.test.ts b/lib/health/__tests__/readiness.test.ts new file mode 100644 index 000000000..35871414b --- /dev/null +++ b/lib/health/__tests__/readiness.test.ts @@ -0,0 +1,86 @@ +// @vitest-environment node +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const config: Record = {}; +vi.mock('@/lib/admin/config-manager', () => ({ + configManager: { + ensureLoaded: async () => {}, + get: (key: string, fallback: unknown) => (key in config ? config[key] : fallback), + }, +})); + +import { readiness, resetReadinessCache } from '../readiness'; +import { GET as ready } from '@/app/api/health/ready/route'; +import { GET as live } from '@/app/api/health/live/route'; + +let dir: string; +let fetchSpy: ReturnType; + +beforeEach(() => { + dir = mkdtempSync(path.join(tmpdir(), 'bulwark-ready-')); + process.env.ADMIN_STATE_DIR = path.join(dir, 'state'); + process.env.TELEMETRY_DATA_DIR = path.join(dir, 'telemetry'); + for (const key of Object.keys(config)) delete config[key]; + config.jmapServerUrl = 'https://mail.example.org'; + resetReadinessCache(); + fetchSpy = vi.fn().mockResolvedValue(new Response(null, { status: 401 })); + vi.stubGlobal('fetch', fetchSpy); +}); + +afterEach(() => { + vi.unstubAllGlobals(); + delete process.env.ADMIN_STATE_DIR; + delete process.env.TELEMETRY_DATA_DIR; + rmSync(dir, { recursive: true, force: true }); +}); + +describe('health endpoints', () => { + it('live answers 200 whatever Stalwart does', async () => { + fetchSpy.mockRejectedValue(new Error('down')); + expect((await live()).status).toBe(200); + }); + + it('ready answers 200 when Stalwart answers, even with 401', async () => { + const res = await ready(); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ ready: true, checks: { config: 'ok', store: 'ok', dataDirs: 'ok', stalwart: 'ok' } }); + expect(String(fetchSpy.mock.calls[0][0])).toBe('https://mail.example.org/.well-known/jmap'); + }); + + it('ready answers 503 when Stalwart is unreachable or failing, and names no host', async () => { + fetchSpy.mockRejectedValue(new Error('ECONNREFUSED mail.example.org')); + const res = await ready(); + expect(res.status).toBe(503); + const text = await res.text(); + expect(JSON.parse(text).checks.stalwart).toBe('fail'); + expect(text).not.toContain('example.org'); + + resetReadinessCache(); + fetchSpy.mockResolvedValue(new Response(null, { status: 502 })); + expect((await ready()).status).toBe(503); + }); + + it('skips Stalwart before a server is configured', async () => { + delete config.jmapServerUrl; + delete process.env.JMAP_SERVER_URL; + expect((await readiness()).checks.stalwart).toBe('skipped'); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('fails when a data dir is not writable', async () => { + writeFileSync(path.join(dir, 'blocked'), ''); + process.env.TELEMETRY_DATA_DIR = path.join(dir, 'blocked', 'telemetry'); + const result = await readiness(); + expect(result.checks.dataDirs).toBe('fail'); + expect(result.ready).toBe(false); + }); + + it('reuses a result for a few seconds', async () => { + await readiness(1000); + await readiness(2000); + expect(fetchSpy).toHaveBeenCalledTimes(1); + }); +}); diff --git a/lib/health/readiness.ts b/lib/health/readiness.ts new file mode 100644 index 000000000..4543b4a2a --- /dev/null +++ b/lib/health/readiness.ts @@ -0,0 +1,131 @@ +import { randomBytes } from 'node:crypto'; +import { mkdir, unlink, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { configManager } from '@/lib/admin/config-manager'; +import { getStateDir } from '@/lib/admin/paths'; +import { observeUpstream } from '@/lib/metrics/upstream'; +import { bootState, getStore, isSharedMode } from '@/lib/shared-store'; +import { getSettingsDir } from '@/lib/settings-sync'; +import { getDir as getTelemetryDir } from '@/lib/telemetry/state'; + +/** + * Whether this replica can serve, for /api/health/ready (and, for the + * shared store, /api/ready). Each check answers ok, fail or skipped; the + * answer names checks, never hosts or paths, because anyone may ask. + */ + +export type CheckResult = 'ok' | 'fail' | 'skipped'; + +export interface Readiness { + ready: boolean; + checks: { + config: CheckResult; + store: CheckResult; + dataDirs: CheckResult; + stalwart: CheckResult; + }; +} + +const STALWART_TIMEOUT_MS = 2000; +const STORE_TIMEOUT_MS = 1000; +const CACHE_MS = 5000; + +const KEY = Symbol.for('bulwark.health.readiness'); +interface CacheState { at: number; result: Readiness | null; pending: Promise | null } +function cache(): CacheState { + const g = globalThis as typeof globalThis & { [KEY]?: CacheState }; + return (g[KEY] ??= { at: 0, result: null, pending: null }); +} + +async function checkConfig(): Promise { + try { + await configManager.ensureLoaded(); + return 'ok'; + } catch { + return 'fail'; + } +} + +/** Booted, and with a shared store, the store answered a ping. */ +export async function checkStore(): Promise { + if (bootState() !== 'ready') return 'fail'; + const store = getStore(); + if (store.mode === 'local') return 'ok'; + try { + await store.ping(STORE_TIMEOUT_MS); + return 'ok'; + } catch { + return 'fail'; + } +} + +async function writable(dir: string): Promise { + const probe = path.join(dir, `.ready-${process.pid}-${randomBytes(4).toString('hex')}`); + try { + await mkdir(dir, { recursive: true }); + await writeFile(probe, ''); + await unlink(probe); + return true; + } catch { + return false; + } +} + +/** The dirs this process writes to in local mode; a shared store holds that data instead. */ +async function checkDataDirs(): Promise { + if (isSharedMode()) return 'skipped'; + const dirs = [getStateDir(), getTelemetryDir()]; + if (configManager.get('settingsSyncEnabled', false)) dirs.push(getSettingsDir()); + const results = await Promise.all(dirs.map(writable)); + return results.every(Boolean) ? 'ok' : 'fail'; +} + +/** Stalwart's session resource answers, whatever the status below 500 (401 is expected without credentials). */ +async function checkStalwart(): Promise { + const base = configManager.get('jmapServerUrl', '') || process.env.JMAP_SERVER_URL || ''; + if (!base.trim()) return 'skipped'; + let url: URL; + try { + url = new URL('/.well-known/jmap', base.trim()); + } catch { + return 'fail'; + } + try { + const response = await observeUpstream('stalwart', () => fetch(url, { + method: 'GET', + redirect: 'manual', + signal: AbortSignal.timeout(STALWART_TIMEOUT_MS), + })); + await response.body?.cancel().catch(() => undefined); + return response.status < 500 ? 'ok' : 'fail'; + } catch { + return 'fail'; + } +} + +async function evaluate(): Promise { + const config = await checkConfig(); + const [store, dataDirs, stalwart] = await Promise.all([checkStore(), checkDataDirs(), checkStalwart()]); + const checks = { config, store, dataDirs, stalwart }; + return { ready: Object.values(checks).every((c) => c !== 'fail'), checks }; +} + +/** The readiness of this replica, reused for CACHE_MS so probes can't flood Stalwart or the store. */ +export async function readiness(now = Date.now()): Promise { + const c = cache(); + if (c.result && now - c.at < CACHE_MS) return c.result; + c.pending ??= evaluate() + .then((result) => { + c.result = result; + c.at = Date.now(); + return result; + }) + .finally(() => { c.pending = null; }); + return c.pending; +} + +/** For tests. */ +export function resetReadinessCache(): void { + const g = globalThis as typeof globalThis & { [KEY]?: CacheState }; + delete g[KEY]; +} diff --git a/lib/jmap/client.ts b/lib/jmap/client.ts index b893794c6..dad151f72 100644 --- a/lib/jmap/client.ts +++ b/lib/jmap/client.ts @@ -1,3 +1,4 @@ +import { recordJmapCall, reportJmapFailure } from "@/lib/error-reporting/client"; import { advertisesMailboxMetadata, bulwarkEntries, escapeMetadataPathSegment, MAX_METADATA_ENTRY_BYTES, METADATA_CAPABILITY, MetadataPatchError, MetadataRefusedError, MetadataTooLargeError, MetadataUnsupportedError, metadataValueBytes, namespaceFor } from "./bulwark-metadata"; import { IncompleteDeleteError } from "./delete-error"; import { downloadAllowed, saveBlobAs, type DownloadKind } from "@/lib/download-guard"; @@ -191,6 +192,29 @@ function mailboxIdsReplacement( return { mailboxIds }; } +/** + * Statuses that are part of normal operation: an expired token (refreshed + * by the caller) and Stalwart's concurrency limit (retried above). + */ +const UNREPORTED_STATUSES = new Set([401, 429]); + +/** + * Method errors that are answers, not failures: the client resyncs, pages + * again or asks again with the new state. + */ +const UNREPORTED_METHOD_ERRORS = new Set(['cannotCalculateChanges', 'anchorNotFound', 'stateMismatch', 'tooManyChanges']); + +/** Report each `["error", {type}]` answer by method name and type (lib/error-reporting/client.ts). */ +function reportMethodErrors(methodCalls: JMAPMethodCall[], response: JMAPResponse): void { + for (const [name, result, callId] of response?.methodResponses ?? []) { + if (name !== 'error') continue; + const type = (result as { type?: unknown } | undefined)?.type; + if (typeof type !== 'string' || UNREPORTED_METHOD_ERRORS.has(type)) continue; + const method = methodCalls.find((call) => call[2] === callId)?.[0] ?? 'Unknown/method'; + reportJmapFailure({ method, type }); + } +} + export class RateLimitError extends Error { retryAfterMs: number; constructor(retryAfterMs: number) { @@ -2107,6 +2131,8 @@ export class JMAPClient implements IJMAPClient { if (!this.apiUrl) { throw new Error('Not connected. Call connect() first.'); } + // Error reports keep the method names of the last calls, nothing else. + recordJmapCall(methodCalls.map(([name]) => name)); const requestBody = { using: using || (hasSubmissionMethod(methodCalls) ? [...SUBMISSION_USING] : ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"]), @@ -2141,15 +2167,20 @@ export class JMAPClient implements IJMAPClient { continue; } console.error('Request failed:', response.status, responseText); + // The status only: the answer may quote what was sent. + if (!UNREPORTED_STATUSES.has(response.status)) reportJmapFailure({ status: response.status }); throw new Error(`Request failed: ${response.status} - ${responseText.substring(0, 200)}`); } + let parsed: JMAPResponse; try { - return JSON.parse(responseText); + parsed = JSON.parse(responseText); } catch { console.error('Failed to parse response:', responseText); throw new Error('Invalid JSON response from server'); } + reportMethodErrors(methodCalls, parsed); + return parsed; } } diff --git a/lib/logger.ts b/lib/logger.ts index e704e87d1..b3c87994b 100644 --- a/lib/logger.ts +++ b/lib/logger.ts @@ -40,7 +40,38 @@ function isJson(): boolean { return configured === 'json'; } +/** Whether log lines go out as JSON (LOG_FORMAT=json or the admin setting). */ +export function isJsonLogFormat(): boolean { + return isJson(); +} + +/** + * Gets every logger.error call, with the stack of the call. The server's + * error reporting sets it (lib/error-reporting/server.ts); without it errors + * are only logged. + */ +export type ErrorSink = (message: string, extra: Record | undefined, stack: string | undefined) => void; +const SINK_KEY = Symbol.for('bulwark.logger.errorSink'); +type GlobalWithSink = typeof globalThis & { [SINK_KEY]?: ErrorSink }; + +export function setErrorSink(sink: ErrorSink | null): void { + const target = globalThis as GlobalWithSink; + if (sink) target[SINK_KEY] = sink; + else delete target[SINK_KEY]; +} + function log(level: LogLevel, message: string, extra?: Record): void { + if (level === 'error') { + const sink = (globalThis as GlobalWithSink)[SINK_KEY]; + if (sink) { + try { + // Frames: log, logger.error, then the caller. + sink(message, extra, new Error().stack); + } catch { + // reporting must not break logging + } + } + } if (LEVELS[level] > getLevel()) return; if (isJson()) { diff --git a/lib/metrics/__tests__/metrics.test.ts b/lib/metrics/__tests__/metrics.test.ts new file mode 100644 index 000000000..286dd4030 --- /dev/null +++ b/lib/metrics/__tests__/metrics.test.ts @@ -0,0 +1,93 @@ +// @vitest-environment node +import http from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { metrics, observeRequest, resetMetricsForTests, signinOutcome } from '../index'; +import { observeUpstream } from '../upstream'; +import { installHttpMetrics } from '../http'; + +async function value(name: string, labels: Record = {}): Promise { + const metric = (await metrics().registry.getMetricsAsJSON()).find((m) => m.name === name); + const found = (metric?.values as Array<{ labels: Record; value: number }> | undefined) + ?.find((v) => Object.entries(labels).every(([k, want]) => v.labels[k] === want)); + return found?.value; +} + +beforeEach(() => { + process.env.METRICS_TOKEN = 'test-token-1234567890'; + resetMetricsForTests(); +}); + +afterEach(() => { + delete process.env.METRICS_TOKEN; + resetMetricsForTests(); + vi.restoreAllMocks(); +}); + +describe('request metrics', () => { + it('label requests by route group and status class, never by the raw path', async () => { + observeRequest('/de/mail/folder/Acquisition%20Talks/Mabc?q=secret', 'GET', 200, 0.05); + observeRequest('/webmail/api/wopi/files/abc/contents', 'POST', 503, 1.2, '/webmail'); + observeRequest('/api/whatever-a-user-typed', 'BREW', 404, 0.01); + expect(await value('bulwark_http_requests_total', { group: 'page:mail', method: 'GET', status: '2xx' })).toBe(1); + expect(await value('bulwark_http_requests_total', { group: 'api:wopi', method: 'POST', status: '5xx' })).toBe(1); + expect(await value('bulwark_http_requests_total', { group: 'api:other', method: 'other', status: '4xx' })).toBe(1); + const text = await metrics().registry.metrics(); + expect(text).not.toContain('Acquisition'); + expect(text).not.toContain('whatever-a-user-typed'); + expect(text).not.toContain('secret'); + }); + + it('count sign-ins by outcome from the auth routes', async () => { + observeRequest('/api/auth/session', 'POST', 200, 0.1); + observeRequest('/api/auth/session', 'POST', 401, 0.1); + observeRequest('/api/auth/session', 'POST', 429, 0.1); + observeRequest('/api/auth/token', 'PUT', 401, 0.1); + observeRequest('/api/auth/token', 'PUT', 200, 0.1); + observeRequest('/api/auth/session', 'GET', 200, 0.1); + for (const [outcome, n] of [['success', 1], ['failure', 1], ['rate_limited', 1], ['refresh_failure', 1]] as const) { + expect(await value('bulwark_signins_total', { outcome }), outcome).toBe(n); + } + }); + + it('map outcomes only for sign-in and refresh calls', () => { + expect(signinOutcome('api:auth', '/api/auth/totp-token-exchange', 'POST', 200)).toBe('success'); + expect(signinOutcome('api:auth', '/api/auth/session', 'DELETE', 200)).toBeNull(); + expect(signinOutcome('api:admin', '/api/admin/auth', 'POST', 401)).toBeNull(); + }); + + it('do nothing while the endpoint is off', async () => { + delete process.env.METRICS_TOKEN; + observeRequest('/api/config', 'GET', 200, 0.01); + process.env.METRICS_TOKEN = 'test-token-1234567890'; + expect(await value('bulwark_http_requests_total', { group: 'api:config' })).toBeUndefined(); + }); + + it('come from Node\'s HTTP server itself once installed', async () => { + expect(installHttpMetrics('')).toBe(true); + const server = http.createServer((_req, res) => { res.statusCode = 204; res.end(); }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + try { + const { port } = server.address() as AddressInfo; + await fetch(`http://127.0.0.1:${port}/api/health/live`); + await vi.waitFor(async () => expect(await value('bulwark_http_requests_total', { group: 'api:health', status: '2xx' })).toBe(1)); + } finally { + server.close(); + } + }); +}); + +describe('upstream metrics', () => { + it('count answers below 500 as ok and 5xx or errors as error', async () => { + await observeUpstream('stalwart', async () => new Response(null, { status: 401 })); + await observeUpstream('stalwart', async () => new Response(null, { status: 502 })); + await expect(observeUpstream('wopi', async () => { throw new Error('down'); })).rejects.toThrow('down'); + expect(await value('bulwark_upstream_requests_total', { target: 'stalwart', outcome: 'ok' })).toBe(1); + expect(await value('bulwark_upstream_requests_total', { target: 'stalwart', outcome: 'error' })).toBe(1); + expect(await value('bulwark_upstream_requests_total', { target: 'wopi', outcome: 'error' })).toBe(1); + }); + + it('report build info', async () => { + expect(await metrics().registry.metrics()).toMatch(/bulwark_build_info\{version="[^"]+",build="[^"]+"\} 1/); + }); +}); diff --git a/lib/metrics/__tests__/upstream-wiring.test.ts b/lib/metrics/__tests__/upstream-wiring.test.ts new file mode 100644 index 000000000..b470ffd2a --- /dev/null +++ b/lib/metrics/__tests__/upstream-wiring.test.ts @@ -0,0 +1,29 @@ +// @vitest-environment node +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('@/lib/admin/config-manager', () => ({ + configManager: { ensureLoaded: async () => {}, get: (_key: string, fallback: unknown) => fallback }, +})); + +import { fetchJmapServer } from '@/lib/stalwart/server-fetch'; +import { metrics, resetMetricsForTests } from '@/lib/metrics'; + +beforeEach(() => { + process.env.METRICS_TOKEN = 'scrape-secret-0123456789'; + resetMetricsForTests(); +}); + +afterEach(() => { + delete process.env.METRICS_TOKEN; + resetMetricsForTests(); + vi.unstubAllGlobals(); +}); + +// The server's calls to Stalwart all go through fetchJmapServer. +describe('Stalwart calls', () => { + it('are counted under the stalwart target', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('{}', { status: 200 }))); + await fetchJmapServer('https://mail.example.org/jmap/session', {}, true); + expect(await metrics().registry.metrics()).toContain('bulwark_upstream_requests_total{target="stalwart",outcome="ok"} 1'); + }); +}); diff --git a/lib/metrics/http.ts b/lib/metrics/http.ts new file mode 100644 index 000000000..1bffbb9a8 --- /dev/null +++ b/lib/metrics/http.ts @@ -0,0 +1,39 @@ +import diagnostics from 'node:diagnostics_channel'; +import type { IncomingMessage, ServerResponse } from 'node:http'; +import { metricsEnabled, observeRequest } from './index'; + +/** + * Counts every request this process serves, through Node's own HTTP + * diagnostics channels: nothing is wrapped or patched, and the proxy's + * matcher doesn't matter. Installed once at startup when the metrics + * endpoint is on (instrumentation.node.ts). + */ + +interface HttpState { + installed: boolean; + started: WeakMap; +} + +const KEY = Symbol.for('bulwark.metrics.http'); +function state(): HttpState { + const g = globalThis as typeof globalThis & { [KEY]?: HttpState }; + return (g[KEY] ??= { installed: false, started: new WeakMap() }); +} + +export function installHttpMetrics(basePath = process.env.NEXT_PUBLIC_BASE_PATH || ''): boolean { + const s = state(); + if (s.installed || !metricsEnabled()) return false; + s.installed = true; + diagnostics.subscribe('http.server.request.start', (message) => { + const { request } = message as { request: IncomingMessage }; + s.started.set(request, performance.now()); + }); + diagnostics.subscribe('http.server.response.finish', (message) => { + const { request, response } = message as { request: IncomingMessage; response: ServerResponse }; + const start = s.started.get(request); + if (start === undefined) return; + s.started.delete(request); + observeRequest(request.url ?? '/', request.method, response.statusCode, (performance.now() - start) / 1000, basePath); + }); + return true; +} diff --git a/lib/metrics/index.ts b/lib/metrics/index.ts new file mode 100644 index 000000000..8dd6e498f --- /dev/null +++ b/lib/metrics/index.ts @@ -0,0 +1,147 @@ +import { readFileSync } from 'node:fs'; +import type { Counter, Gauge, Histogram, Registry } from '@prometheus-io/client'; +import client from '@prometheus-io/client'; +import { routeGroup } from '@/lib/route-groups'; + +/** + * Prometheus metrics for /api/metrics, off unless METRICS_TOKEN (or + * METRICS_TOKEN_FILE) is set. Every replica counts for itself; Prometheus + * scrapes each one. Labels come from fixed lists only (route groups, sign-in + * outcomes, upstream targets), never from user names or hosts users type. + * + * https://bulwarkmail.org/docs/deployment/monitoring#metrics + */ + +export const SIGNIN_OUTCOMES = ['success', 'failure', 'refresh_failure', 'rate_limited'] as const; +export type SigninOutcome = typeof SIGNIN_OUTCOMES[number]; + +export const UPSTREAM_TARGETS = ['stalwart', 'wopi', 'translate', 'plugin_proxy', 'oauth'] as const; +export type UpstreamTarget = typeof UPSTREAM_TARGETS[number]; + +const METHODS = new Set(['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS']); +const DURATION_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30]; + +/** The bearer token /api/metrics wants, or null when the endpoint is off. */ +export function metricsToken(): string | null { + const direct = process.env.METRICS_TOKEN?.trim(); + if (direct) return direct; + const file = process.env.METRICS_TOKEN_FILE?.trim(); + if (!file) return null; + try { + return readFileSync(file, 'utf8').trim() || null; + } catch { + return null; + } +} + +export function metricsEnabled(): boolean { + return metricsToken() !== null; +} + +interface Metrics { + registry: Registry; + httpRequests: Counter<'group' | 'method' | 'status'>; + httpDuration: Histogram<'group' | 'method'>; + signins: Counter<'outcome'>; + upstreamRequests: Counter<'target' | 'outcome'>; + upstreamDuration: Histogram<'target'>; + browserErrors: Counter; + errorsToday: Gauge; +} + +const KEY = Symbol.for('bulwark.metrics'); + +/** The registry and metrics, made once per process whatever bundle asks. */ +export function metrics(): Metrics { + const g = globalThis as typeof globalThis & { [KEY]?: Metrics }; + if (g[KEY]) return g[KEY]; + const registry = new client.Registry(); + client.collectDefaultMetrics({ register: registry, prefix: 'bulwark_' }); + const version = process.env.NEXT_PUBLIC_APP_VERSION || '0.0.0'; + const build = process.env.NEXT_PUBLIC_GIT_COMMIT || process.env.BULWARK_BUILD || 'unknown'; + new client.Gauge({ name: 'bulwark_build_info', help: 'Bulwark version and build; always 1', labelNames: ['version', 'build'], registers: [registry] }) + .set({ version, build }, 1); + new client.Gauge({ + name: 'bulwark_uptime_seconds', + help: 'Seconds since this process started', + registers: [registry], + collect() { this.set(process.uptime()); }, + }); + const created: Metrics = { + registry, + httpRequests: new client.Counter({ name: 'bulwark_http_requests_total', help: 'HTTP requests by route group, method and status class', labelNames: ['group', 'method', 'status'], registers: [registry] }), + httpDuration: new client.Histogram({ name: 'bulwark_http_request_duration_seconds', help: 'HTTP request duration by route group and method', labelNames: ['group', 'method'], buckets: DURATION_BUCKETS, registers: [registry] }), + signins: new client.Counter({ name: 'bulwark_signins_total', help: 'Sign-ins handled by this server, by outcome', labelNames: ['outcome'], registers: [registry] }), + upstreamRequests: new client.Counter({ name: 'bulwark_upstream_requests_total', help: 'Requests this server made to other services, by target and outcome', labelNames: ['target', 'outcome'], registers: [registry] }), + upstreamDuration: new client.Histogram({ name: 'bulwark_upstream_request_duration_seconds', help: 'Duration of requests to other services, by target', labelNames: ['target'], buckets: DURATION_BUCKETS, registers: [registry] }), + browserErrors: new client.Counter({ name: 'bulwark_browser_errors_total', help: 'Errors browsers reported to this server', registers: [registry] }), + errorsToday: new client.Gauge({ + name: 'bulwark_errors_today', + help: 'Errors counted today (UTC) by the whole deployment, browsers and servers', + registers: [registry], + async collect() { + const { readErrors, summarizeDay } = await import('@/lib/error-reporting/aggregate'); + this.set(summarizeDay(await readErrors()).total); + }, + }), + }; + // Pre-create the label sets, so a series exists at 0 before its first event. + for (const outcome of SIGNIN_OUTCOMES) created.signins.inc({ outcome }, 0); + for (const target of UPSTREAM_TARGETS) { + for (const outcome of ['ok', 'error']) created.upstreamRequests.inc({ target, outcome }, 0); + } + g[KEY] = created; + return created; +} + +export function methodLabel(method: string | undefined): string { + const upper = (method ?? '').toUpperCase(); + return METHODS.has(upper) ? upper : 'other'; +} + +export function statusClass(status: number): string { + return status >= 100 && status < 600 ? `${Math.floor(status / 100)}xx` : 'other'; +} + +/** What a finished request means for sign-ins, from its route, method and status. */ +export function signinOutcome(group: string, path: string, method: string, status: number): SigninOutcome | null { + if (group !== 'api:auth') return null; + const route = path.split(/[?#]/, 1)[0].replace(/\/+$/, ''); + const signIn = method === 'POST' && /\/api\/auth\/(?:session|stalwart-context|totp-token-exchange|token)$/.test(route); + const refresh = method === 'PUT' && /\/api\/auth\/token$/.test(route); + if (!signIn && !refresh) return null; + if (status === 429) return 'rate_limited'; + if (status >= 200 && status < 300) return refresh ? null : 'success'; + return refresh ? 'refresh_failure' : 'failure'; +} + +/** Count a finished HTTP request (lib/metrics/http.ts). */ +export function observeRequest(path: string, method: string | undefined, status: number, seconds: number, basePath = ''): void { + if (!metricsEnabled()) return; + const m = metrics(); + const group = routeGroup(path, basePath); + const verb = methodLabel(method); + m.httpRequests.inc({ group, method: verb, status: statusClass(status) }); + m.httpDuration.observe({ group, method: verb }, seconds); + const outcome = signinOutcome(group, basePath && path.startsWith(basePath) ? path.slice(basePath.length) : path, verb, status); + if (outcome) m.signins.inc({ outcome }); +} + +export function observeUpstreamResult(target: UpstreamTarget, ok: boolean, seconds: number): void { + if (!metricsEnabled()) return; + const m = metrics(); + m.upstreamRequests.inc({ target, outcome: ok ? 'ok' : 'error' }); + m.upstreamDuration.observe({ target }, seconds); +} + +export function countBrowserError(): void { + if (!metricsEnabled()) return; + metrics().browserErrors.inc(); +} + +/** For tests. */ +export function resetMetricsForTests(): void { + const g = globalThis as typeof globalThis & { [KEY]?: Metrics }; + g[KEY]?.registry.clear(); + delete g[KEY]; +} diff --git a/lib/metrics/upstream.ts b/lib/metrics/upstream.ts new file mode 100644 index 000000000..e7cd1493a --- /dev/null +++ b/lib/metrics/upstream.ts @@ -0,0 +1,62 @@ +import { observeUpstreamResult, type UpstreamTarget } from './index'; +import { isTracingActive } from '@/lib/tracing/config'; + +/** Headers that carry the trace context to the service called (`traceparent`). */ +export type TraceHeaders = Record; + +/** + * Run one request this server makes to another service, under a fixed + * target name: + * + * - metrics (when /api/metrics is on): duration and outcome. An answer + * below 500 counts as ok (a 401 from Stalwart is an answer); a 5xx or a + * thrown error as error. + * - tracing (when OTEL_* is set): a client span named after the target, + * with no URL, host or user in it, and the W3C trace context handed to + * `run` as headers, so the callee's spans join the trace. + * + * Costs nothing while both are off. + */ +export async function observeUpstream( + target: UpstreamTarget, + run: (traceHeaders: TraceHeaders) => Promise, +): Promise { + if (isTracingActive()) return traced(target, run); + return measured(target, () => run({})); +} + +async function measured(target: UpstreamTarget, run: () => Promise): Promise { + const start = performance.now(); + try { + const response = await run(); + observeUpstreamResult(target, response.status < 500, (performance.now() - start) / 1000); + return response; + } catch (error) { + observeUpstreamResult(target, false, (performance.now() - start) / 1000); + throw error; + } +} + +async function traced(target: UpstreamTarget, run: (traceHeaders: TraceHeaders) => Promise): Promise { + const api = await import('@opentelemetry/api'); + const tracer = api.trace.getTracer('bulwark-webmail'); + return tracer.startActiveSpan(`upstream ${target}`, { + kind: api.SpanKind.CLIENT, + attributes: { 'bulwark.upstream.target': target }, + }, async (span) => { + const headers: TraceHeaders = {}; + api.propagation.inject(api.context.active(), headers); + try { + const response = await measured(target, () => run(headers)); + span.setAttribute('http.response.status_code', response.status); + if (response.status >= 500) span.setStatus({ code: api.SpanStatusCode.ERROR }); + return response; + } catch (error) { + // The error's name only: messages can name hosts or paths. + span.setStatus({ code: api.SpanStatusCode.ERROR, message: error instanceof Error ? error.name : 'Error' }); + throw error; + } finally { + span.end(); + } + }); +} diff --git a/lib/oauth/token-exchange.ts b/lib/oauth/token-exchange.ts index e002f218d..a60031595 100644 --- a/lib/oauth/token-exchange.ts +++ b/lib/oauth/token-exchange.ts @@ -1,4 +1,5 @@ import { logger } from '@/lib/logger'; +import { observeUpstream } from '@/lib/metrics/upstream'; import { discoverOAuth } from '@/lib/oauth/discovery'; import type { EndpointValidator, OAuthMetadata } from '@/lib/oauth/discovery'; import { isPublicHttpUrl } from '@/lib/security/url-guard'; @@ -209,12 +210,12 @@ export async function exchangeCodeForTokens( // The body carries the code, the PKCE verifier and the client secret. The // endpoint was validated; a redirect would re-send all of it to a host // that was not, so none is followed. - const tokenResponse = await fetch(tokenEndpoint, { + const tokenResponse = await observeUpstream('oauth', () => fetch(tokenEndpoint, { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: params.toString(), redirect: 'error', - }); + })); if (!tokenResponse.ok) { const errorText = await tokenResponse.text(); diff --git a/lib/plugin-secrets/proxy.ts b/lib/plugin-secrets/proxy.ts index 409649e87..9a9bcbb0b 100644 --- a/lib/plugin-secrets/proxy.ts +++ b/lib/plugin-secrets/proxy.ts @@ -9,6 +9,7 @@ import { getPlugin } from '@/lib/admin/plugin-registry'; import { getPluginConfig } from '@/lib/admin/plugin-config'; import { logger } from '@/lib/logger'; import { DisallowedUrlError, fetchPublicUrl } from '@/lib/security/url-guard'; +import { observeUpstream } from '@/lib/metrics/upstream'; import { authHeaderFor, credentialOrigin, privateProxyOrigins, type CredentialInject, type CredentialSpec } from './credentials'; import { getUserCredential, setUserCredentialSecret, userKey, type VaultUser } from './vault'; import { declaredUserCredential } from './known-plugins'; @@ -278,8 +279,10 @@ async function send( signal: init.signal, ...(init.body ? { body: init.body } : {}), }; - if (trusted) return fetch(url, { ...request, redirect: 'manual' } as RequestInit); - return (await fetchPublicUrl(url, request as never)) as unknown as Response; + return observeUpstream('plugin_proxy', async () => { + if (trusted) return fetch(url, { ...request, redirect: 'manual' } as RequestInit); + return (await fetchPublicUrl(url, request as never)) as unknown as Response; + }); } function responseHeaders(upstream: Response): Headers { diff --git a/lib/route-groups.ts b/lib/route-groups.ts new file mode 100644 index 000000000..6c222f6a6 --- /dev/null +++ b/lib/route-groups.ts @@ -0,0 +1,75 @@ +/** + * A request path reduced to a fixed, small set of names: the area of the app + * or the API folder it belongs to. Error reports use it instead of the URL, + * and the request metrics use it as a label, so neither ever carries a folder + * name, a message id, a query string or anything else a user typed. + * + * lib/__tests__/route-groups.test.ts checks API_GROUPS against app/api and + * PAGE_AREAS against app/(main)/[locale]. + */ + +/** The folders under app/api. */ +export const API_GROUPS: readonly string[] = [ + 'account', 'account-archive', 'admin', 'auth', 'bimi', 'caldav', 'calendar-agenda', 'config', + 'connector', 'dev-jmap', 'errors', 'favicon', 'fetch-ical', 'health', 'metrics', 'og-image', + 'plugin-approval-status', 'plugin-proxy', 'plugin-secrets', 'plugin-signing-pubkey', 'plugins', + 'push', 'pwa-icon', 'pwa-screenshot', 'ready', 'settings', 'setup', 'system', 'translate', + 'webdav', 'wopi', +]; + +/** The page areas under app/(main)/[locale], plus the pages outside it. */ +export const PAGE_AREAS: readonly string[] = [ + 'apps', 'auth', 'calendar', 'contacts', 'files', 'login', 'mail', 'plugins', 'pro', 'settings', + 'admin', 'setup', 'connector', 'protocol', 'plugin-sandbox', 'plugin-sandbox-privileged', +]; + +const API = new Set(API_GROUPS); +const AREAS = new Set(PAGE_AREAS); +const ASSET_FOLDERS = new Set(['branding', 'notification', 'demo', 'plugin-sandbox-assets']); +/** en, de, pt-BR, zh-TW, sr-Latn: what a locale prefix looks like. */ +const LOCALE = /^[a-z]{2,3}(?:-[A-Za-z]{2,4})?$/; + +interface Parsed { + group: string; + /** Path segments below the group's own. */ + deeper: boolean; +} + +function parse(rawPath: string, basePath: string): Parsed { + let path = rawPath.split(/[?#]/, 1)[0] || '/'; + if (basePath && (path === basePath || path.startsWith(`${basePath}/`))) path = path.slice(basePath.length) || '/'; + const segments = path.split('/').filter(Boolean); + if (segments.length === 0) return { group: 'page:home', deeper: false }; + const [first, second] = segments; + if (first === '_next') { + return { group: second === 'static' ? 'static' : second === 'image' ? 'image' : 'other', deeper: false }; + } + if (first === 'api') { + return { group: second && API.has(second) ? `api:${second}` : 'api:other', deeper: segments.length > 2 }; + } + if ((segments.length === 1 && first.includes('.')) || ASSET_FOLDERS.has(first)) return { group: 'asset', deeper: false }; + const rest = LOCALE.test(first) && !AREAS.has(first) ? segments.slice(1) : segments; + if (rest.length === 0) return { group: 'page:home', deeper: false }; + return { group: AREAS.has(rest[0]) ? `page:${rest[0]}` : 'page:other', deeper: rest.length > 1 }; +} + +/** + * The group of a path: `api:`, `page:`, `page:home`, `static`, + * `image`, `asset` or `other`. `basePath` is removed first. + */ +export function routeGroup(rawPath: string, basePath = ''): string { + return parse(rawPath, basePath).group; +} + +/** + * A path for an error report: the group as a path, and `/…` when the path + * went deeper. `/de/mail/folder/Projects%20ACME/abc` -> `/mail/…`. + */ +export function scrubRoutePath(rawPath: string, basePath = ''): string { + const { group, deeper } = parse(rawPath, basePath); + const base = group === 'page:home' ? '/' + : group.startsWith('api:') ? `/api/${group.slice(4)}` + : group.startsWith('page:') ? `/${group.slice(5)}` + : `/${group}`; + return deeper ? `${base}/…` : base; +} diff --git a/lib/shared-store/server-state-allowlist.ts b/lib/shared-store/server-state-allowlist.ts index d81f0b8c8..a46473fd5 100644 --- a/lib/shared-store/server-state-allowlist.ts +++ b/lib/shared-store/server-state-allowlist.ts @@ -35,6 +35,8 @@ export const SERVER_STATE_ALLOWLIST: AllowedState[] = [ { site: 'app/api/favicon/route.ts#cache', class: 'cache', why: 'sender icons from a public service, 14 days' }, { site: 'app/api/favicon/route.ts#negativeCache', class: 'cache', why: 'sender icons that failed, 1 day' }, { site: 'app/api/ready/route.ts#last', class: 'cache', why: 'the last store ping, 1 s, so probes can\'t flood the store' }, + { site: 'lib/tracing/config.ts#globalThis:ACTIVE', class: 'per-process', why: 'whether this process registered the OpenTelemetry SDK at startup' }, + { site: 'lib/error-reporting/debug-ids.ts#cache', class: 'per-process', why: 'debug ids of the bundle files this process loaded, read from globalThis' }, { site: 'lib/admin/directory-keys.ts#keyringCache', class: 'cache', why: 'extension directory keys, 10 min; the anti-rollback record is in the store' }, { site: 'lib/admin/paths.ts#cachedReadOnly', class: 'per-process', why: 'parsed ADMIN_CONFIG_READONLY env' }, { site: 'lib/admin/plugin-approvals.ts#maxPackageBytes', class: 'per-process', why: 'a limit tests can lower' }, diff --git a/lib/stalwart/server-fetch.ts b/lib/stalwart/server-fetch.ts index d1adc2fe3..7e0296cf2 100644 --- a/lib/stalwart/server-fetch.ts +++ b/lib/stalwart/server-fetch.ts @@ -19,6 +19,7 @@ import { configManager } from '@/lib/admin/config-manager'; import { logger } from '@/lib/logger'; import { parseJmapServers, resolveTrustedJmapUrl } from '@/lib/admin/jmap-servers'; import { fetchPublicUrl, type PublicFetchResponse } from '@/lib/security/url-guard'; +import { observeUpstream } from '@/lib/metrics/upstream'; import { isStoredCredentialRefused, recordStoredCredentialStatus, @@ -72,7 +73,8 @@ export async function fetchJmapServer( ): Promise { const authHeader = new Headers(givenInit.headers).get('authorization'); if (authHeader && (await isStoredCredentialRefused(authHeader))) return refusedCredentialResponse(); - const response = await fetchUnbudgeted(url, givenInit, trusted); + // With tracing on, Stalwart gets the trace context and its spans join ours. + const response = await observeUpstream('stalwart', (trace) => fetchUnbudgeted(url, withTraceHeaders(givenInit, trace), trusted)); // Not awaited: this process's copy is updated at once, the shared write follows. if (authHeader) { void recordStoredCredentialStatus(authHeader, response.status).catch((error) => { @@ -82,6 +84,13 @@ export async function fetchJmapServer( return response; } +function withTraceHeaders(init: ServerFetchInit, trace: Record): ServerFetchInit { + if (Object.keys(trace).length === 0) return init; + const headers = new Headers(init.headers); + for (const [name, value] of Object.entries(trace)) headers.set(name, value); + return { ...init, headers }; +} + async function fetchUnbudgeted( url: string, givenInit: ServerFetchInit, diff --git a/lib/telemetry/index.ts b/lib/telemetry/index.ts index 2cc956d4c..19d15b1ea 100644 --- a/lib/telemetry/index.ts +++ b/lib/telemetry/index.ts @@ -1,7 +1,7 @@ export { startScheduler, stopScheduler, reschedule, sendOnce } from './sender'; -export { buildPayload, markProcessStart } from './payload'; +export { buildPayload, errorsForDay, errorsForPayload, markProcessStart } from './payload'; export { - loadState, saveState, updateState, getInstanceId, effectiveConsent, + loadState, saveState, updateState, getInstanceId, effectiveConsent, effectiveErrorsConsent, } from './state'; export { recordLogin, getLoginCounts } from './login-tracker'; export { @@ -9,7 +9,8 @@ export { } from './endpoint-guard'; export type { EndpointCheck } from './endpoint-guard'; export type { - TelemetryPayload, TelemetryStateFile, ConsentState, + TelemetryPayload, TelemetryPayloadV2, AnyTelemetryPayload, TelemetryError, TelemetryStateFile, ConsentState, + ErrorsConsent, TelemetryErrorBatch, Platform, OsFamily, CountBucket, TelemetryFeatures, } from './types'; export { DEFAULT_ENDPOINT } from './types'; diff --git a/lib/telemetry/instant-errors.ts b/lib/telemetry/instant-errors.ts new file mode 100644 index 000000000..bfc248153 --- /dev/null +++ b/lib/telemetry/instant-errors.ts @@ -0,0 +1,123 @@ +import { logger } from '@/lib/logger'; +import { readErrors, utcDay } from '@/lib/error-reporting/aggregate'; +import { effectiveConsent, effectiveErrorsConsent, endpointEnabled, getInstanceId } from './state'; +import { errorsForDay } from './payload'; +import { fetchTelemetryTarget, resolveEndpointAllowed } from './endpoint-guard'; +import { DEFAULT_ENDPOINT, errorsEndpointFor, type TelemetryErrorBatch } from './types'; + +/** + * Instant error counts. With the error-count consent on "instant", the + * scheduler leader posts the current UTC day's counts to the collector's + * /v1/errors every TICK_MS. Only errors that are new or whose count bucket + * changed since the last send go out, so a known error repeating doesn't + * cause traffic until it reaches the next bucket. The daily heartbeat still + * carries the complete day as before. + * + * Each tick reads the consent, so switching it in the admin takes effect on + * the next tick without restarting anything. Other replicas' counts reach + * the store with their own flush (every minute), so they go out a minute later. + */ + +const TICK_MS = 60_000; + +interface InstantState { + timer: NodeJS.Timeout | null; + /** The day `sent` is for. */ + day: string; + /** `fingerprint|version` -> the count bucket last sent. */ + sent: Map; + running: Promise | null; +} + +// On globalThis, like the heartbeat timer: one per process, whichever +// module copy starts or stops it. +const KEY = Symbol.for('bulwark.telemetry.instant-errors'); +function state(): InstantState { + const g = globalThis as typeof globalThis & { [KEY]?: InstantState }; + return (g[KEY] ??= { timer: null, day: '', sent: new Map(), running: null }); +} + +async function tick(now: number): Promise { + const s = state(); + const { consent } = await effectiveErrorsConsent(); + if (consent !== 'instant') { + s.sent.clear(); + return; + } + const { state: stored } = await effectiveConsent(); + const endpoint = stored.endpoint ?? DEFAULT_ENDPOINT; + if (!endpointEnabled(endpoint)) return; + let target: string; + try { + target = errorsEndpointFor(endpoint); + } catch { + return; + } + + const day = utcDay(now); + if (s.day !== day) { + s.day = day; + s.sent.clear(); + } + const changed = errorsForDay(await readErrors(), day) + .filter((error) => s.sent.get(`${error.fingerprint}|${error.version}`) !== error.count); + if (changed.length === 0) return; + + // Checked at send time, like the heartbeat: defeats DNS rebinding and a + // state file edited past the admin API. + const guard = await resolveEndpointAllowed(target); + if (!guard.ok) { + logger.warn('telemetry: errors endpoint blocked', { reason: guard.reason }); + return; + } + + const batch: TelemetryErrorBatch = { schema: '1', instance_id: await getInstanceId(), day, errors: changed }; + const res = await fetchTelemetryTarget(target, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(batch), + signal: AbortSignal.timeout(5000), + }); + if (!res.ok) { + // Not marked as sent: the next tick tries them again. + logger.warn('telemetry: instant error counts rejected', { status: res.status }); + return; + } + for (const error of changed) s.sent.set(`${error.fingerprint}|${error.version}`, error.count); +} + +/** One tick: send what changed, if instant mode is on. Never throws. */ +export function sendInstantErrors(now = Date.now()): Promise { + const s = state(); + if (s.running) return s.running; + s.running = tick(now) + .catch((error) => { + logger.warn('telemetry: instant error counts failed', { error: error instanceof Error ? error.message : String(error) }); + }) + .finally(() => { s.running = null; }); + return s.running; +} + +// Run by the scheduler leader (lib/scheduler-leader.ts). Idempotent. +export async function startInstantErrors(): Promise { + const s = state(); + if (s.timer) return; + s.timer = setInterval(() => { void sendInstantErrors(); }, TICK_MS); + s.timer.unref?.(); +} + +export async function stopInstantErrors(): Promise { + const s = state(); + if (s.timer) clearInterval(s.timer); + s.timer = null; + // A new leader starts from nothing and sends the day's counts once. + s.sent.clear(); +} + +/** For tests. */ +export function resetInstantErrorsForTests(): void { + const s = state(); + if (s.timer) clearInterval(s.timer); + const g = globalThis as typeof globalThis & { [KEY]?: InstantState }; + delete g[KEY]; +} diff --git a/lib/telemetry/payload.ts b/lib/telemetry/payload.ts index 5928d41cd..6c56c0e4f 100644 --- a/lib/telemetry/payload.ts +++ b/lib/telemetry/payload.ts @@ -1,14 +1,20 @@ import { readFileSync } from 'node:fs'; import path from 'node:path'; import { configManager } from '@/lib/admin/config-manager'; -import { getInstanceId } from './state'; +import { effectiveErrorsConsent, getInstanceId } from './state'; import { getLoginCounts } from './login-tracker'; -import type { - TelemetryPayload, - TelemetryFeatures, - Platform, - OsFamily, - CountBucket, +import { isErrorCode } from '@/lib/error-reporting/envelope'; +import { readErrors, utcDay, type Aggregate } from '@/lib/error-reporting/aggregate'; +import { + MAX_TELEMETRY_ERRORS, + MAX_TELEMETRY_FRAMES, + type AnyTelemetryPayload, + type TelemetryError, + type TelemetryPayload, + type TelemetryFeatures, + type Platform, + type OsFamily, + type CountBucket, } from './types'; let processStartedAt = Date.now(); @@ -99,7 +105,61 @@ async function countExtensions(): Promise<{ extensions: number; themes: number } } } -export async function buildPayload(): Promise { +const BUNDLE_PATH = /^\/(?:_next\/static|\.next\/server)\/[\w./()[\]@+~-]{1,200}$/; +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; + +/** + * The error counts of the last complete UTC day (the day before `now`), for + * schema v2: at most MAX_TELEMETRY_ERRORS, the most frequent first. Only + * frames in Bulwark's own bundles go (no extension, plugin or Node paths), + * and a message only when it is one of Bulwark's fixed error codes. + */ +export function errorsForPayload(agg: Aggregate, now = Date.now()): TelemetryError[] { + return errorsForDay(agg, utcDay(now - 86_400_000)); +} + +/** + * The error counts of one UTC day, shaped as in errorsForPayload. `days` + * counts the days of the week ending on `day` the error occurred on. + * Instant mode sends the current day's this way. + */ +export function errorsForDay(agg: Aggregate, day: string): TelemetryError[] { + const end = Date.parse(`${day}T00:00:00Z`); + const recentDays = Array.from({ length: 7 }, (_, i) => utcDay(end - i * 86_400_000)); + return Object.entries(agg[day] ?? {}) + .sort(([, a], [, b]) => b.count - a.count) + .slice(0, MAX_TELEMETRY_ERRORS) + .map(([fingerprint, entry]) => ({ + name: entry.name, + fingerprint, + frames: entry.frames + .filter((frame) => BUNDLE_PATH.test(frame.path)) + .slice(0, MAX_TELEMETRY_FRAMES) + .map((frame) => ({ + path: frame.path, + line: frame.line ?? null, + col: frame.column ?? null, + debug_id: frame.debugId && UUID.test(frame.debugId) ? frame.debugId : null, + })), + version: entry.version, + count: bucketCount(entry.count), + days: recentDays.filter((d) => agg[d]?.[fingerprint]).length, + ...(isErrorCode(entry.code) ? { message: entry.code } : {}), + })); +} + +/** + * The heartbeat. Schema 1, exactly as before, unless the admin also agreed + * to send error counts; then schema 2 with `errors` added. + */ +export async function buildPayload(): Promise { + const base = await buildPayloadV1(); + // Daily and instant mode both send the last complete day with the heartbeat. + if ((await effectiveErrorsConsent()).consent === 'off') return base; + return { ...base, schema: '2', errors: errorsForPayload(await readErrors()) }; +} + +async function buildPayloadV1(): Promise { const instance_id = await getInstanceId(); const { version, build } = readPackage(); const features = await readFeatures(); diff --git a/lib/telemetry/state.ts b/lib/telemetry/state.ts index 9a82f2c2a..ca9f8655b 100644 --- a/lib/telemetry/state.ts +++ b/lib/telemetry/state.ts @@ -3,7 +3,7 @@ import { randomUUID } from 'node:crypto'; import { logger } from '@/lib/logger'; import { getStore, type CollectionOptions } from '@/lib/shared-store'; import { isStoreUnavailable, StoredDataUnreadableError } from '@/lib/stored-data-errors'; -import type { TelemetryStateFile, ConsentState } from './types'; +import type { TelemetryStateFile, ConsentState, ErrorsConsent } from './types'; import { DEFAULT_ENDPOINT } from './types'; import { isNoThirdPartyTrafficOn } from '@/lib/admin/third-party-mode'; @@ -13,15 +13,16 @@ export function getDir(): string { } /** - * Telemetry's documents (#1066): `state.json`, `logins.json` and the - * instance id `.telemetry-id`. Local mode: today's files in the telemetry - * dir. Shared mode: one copy for the whole deployment, so every replica - * reports as the same instance with the same counts. + * Telemetry's documents (#1066): `state.json`, `logins.json`, the instance + * id `.telemetry-id` and the per-day error counts `errors.json` + * (lib/error-reporting/aggregate.ts). Local mode: today's files in the + * telemetry dir. Shared mode: one copy for the whole deployment, so every + * replica reports as the same instance with the same counts. */ export const TELEMETRY_COLLECTION: CollectionOptions = { local: { dir: () => getDir(), - idOf: (relative) => (['state.json', 'logins.json', '.telemetry-id'].includes(relative) ? relative : null), + idOf: (relative) => (['state.json', 'logins.json', '.telemetry-id', 'errors.json'].includes(relative) ? relative : null), }, }; @@ -173,6 +174,34 @@ export async function saveStateIfUnchanged(state: TelemetryStateFile, version: s return (await telemetryDocs().put(STATE_DOC, serialize(state), { ifVersion: version })).ok; } +function errorsEnvOverride(): ErrorsConsent | null { + const v = (process.env.BULWARK_TELEMETRY_ERRORS ?? '').toLowerCase(); + if (v === 'off' || v === 'false' || v === '0' || v === 'no') return 'off'; + if (v === 'on' || v === 'true' || v === '1' || v === 'yes' || v === 'daily') return 'on'; + if (v === 'instant') return 'instant'; + return null; +} + +/** + * Effective consent to send error counts (schema v2): off whenever + * telemetry itself is off (which covers the no-third-party mode), then + * BULWARK_TELEMETRY_ERRORS, then the stored choice, off by default. + * 'on' sends them daily with the heartbeat; 'instant' does that too and + * also sends the current day's counts as they change. + */ +export async function effectiveErrorsConsent(): Promise<{ + consent: ErrorsConsent; + source: 'telemetry' | 'no-third-party' | 'env' | 'file'; +}> { + const telemetry = await effectiveConsent(); + if (telemetry.source === 'no-third-party') return { consent: 'off', source: 'no-third-party' }; + if (telemetry.consent !== 'on') return { consent: 'off', source: 'telemetry' }; + const env = errorsEnvOverride(); + if (env) return { consent: env, source: 'env' }; + const stored = telemetry.state.errorsConsent; + return { consent: stored === 'on' || stored === 'instant' ? stored : 'off', source: 'file' }; +} + // Effective consent: the no-third-party mode wins over everything (#903), // then the env var, then the file. UI changes are blocked when either // override is active so the user knows where it's coming from. diff --git a/lib/telemetry/types.ts b/lib/telemetry/types.ts index 0e8fccfe2..21deb9eb4 100644 --- a/lib/telemetry/types.ts +++ b/lib/telemetry/types.ts @@ -1,4 +1,5 @@ -// Schema v1 of the anonymous heartbeat. Documented at +// Schema v1 of the anonymous heartbeat, and v2 (v1 plus error counts, sent +// only with the separate error-count consent). Documented at // https://bulwarkmail.org/docs/legal/privacy/telemetry export type ConsentState = 'pending' | 'on' | 'off'; @@ -36,6 +37,66 @@ export interface TelemetryPayload { uptime_days: number; } +/** At most this many errors in a v2 heartbeat, the most frequent first. */ +export const MAX_TELEMETRY_ERRORS = 50; +/** Top frames per error. */ +export const MAX_TELEMETRY_FRAMES = 5; + +export interface TelemetryErrorFrame { + /** The bundle file without origin: `/_next/static/chunks/x.js` or `/.next/server/...`. */ + path: string; + line: number | null; + col: number | null; + /** The file's debug ID, which finds its uploaded source map. */ + debug_id: string | null; +} + +/** One error fingerprint of the last complete UTC day. */ +export interface TelemetryError { + /** The error's class name (TypeError, JmapMethodError). */ + name: string; + fingerprint: string; + frames: TelemetryErrorFrame[]; + version: string; + /** How often that day, bucketed like the account counts. */ + count: CountBucket; + /** On how many of the last 7 days it occurred (1-7). */ + days: number; + /** Only one of Bulwark's fixed error codes (`Email/set: overQuota`), never free text. */ + message?: string; +} + +export interface TelemetryPayloadV2 extends Omit { + schema: '2'; + errors: TelemetryError[]; +} + +export type AnyTelemetryPayload = TelemetryPayload | TelemetryPayloadV2; + +/** + * Error counts: off, daily (with the heartbeat, for the day before) or + * instant (daily, plus the current day's counts within about a minute). + * 'on' is the daily mode; the stored value predates the instant one. + */ +export type ErrorsConsent = 'on' | 'off' | 'instant'; + +/** + * What instant mode posts to the collector's /v1/errors: the current UTC + * day's errors that are new or whose count bucket changed since the last send. + */ +export interface TelemetryErrorBatch { + schema: '1'; + instance_id: string; + /** The UTC day the counts are for (YYYY-MM-DD). */ + day: string; + errors: TelemetryError[]; +} + +/** The collector's errors URL next to its heartbeat URL: `.../v1/heartbeat` -> `.../v1/errors`. */ +export function errorsEndpointFor(heartbeatEndpoint: string): string { + return new URL('errors', heartbeatEndpoint).toString(); +} + export interface TelemetryStateFile { consent: ConsentState; endpoint: string; @@ -47,6 +108,12 @@ export interface TelemetryStateFile { */ lastScheduledSentAt?: string | null; nextScheduledAt: string | null; + /** + * The separate consent to add error counts (schema v2), and whether they + * also go out instantly. Off unless an admin turns it on; switching + * telemetry off switches it off too. + */ + errorsConsent?: ErrorsConsent; } export const DEFAULT_ENDPOINT = 'https://telemetry.bulwarkmail.org/v1/heartbeat'; diff --git a/lib/tracing/__tests__/config.test.ts b/lib/tracing/__tests__/config.test.ts new file mode 100644 index 000000000..49b6429fd --- /dev/null +++ b/lib/tracing/__tests__/config.test.ts @@ -0,0 +1,44 @@ +// @vitest-environment node +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { tracingConfigured, tracingEndpoint } from '../config'; +import { listReachableHosts } from '@/lib/admin/reachable-hosts'; + +const started = vi.hoisted(() => ({ count: 0 })); +vi.mock('@/lib/tracing/start', () => ({ startTracing: () => { started.count++; } })); +vi.mock('@/instrumentation.node', () => ({})); + +afterEach(() => { + vi.unstubAllEnvs(); + started.count = 0; +}); + +describe('tracing configuration', () => { + it('needs an OTLP endpoint from the standard variables', () => { + expect(tracingConfigured({})).toBe(false); + expect(tracingEndpoint({ OTEL_EXPORTER_OTLP_ENDPOINT: 'http://collector:4318' })).toBe('http://collector:4318'); + expect(tracingEndpoint({ OTEL_EXPORTER_OTLP_ENDPOINT: 'http://a:4318', OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: 'http://b:4318/v1/traces' })).toBe('http://b:4318/v1/traces'); + }); + + it('honours OTEL_SDK_DISABLED and OTEL_TRACES_EXPORTER=none', () => { + expect(tracingConfigured({ OTEL_EXPORTER_OTLP_ENDPOINT: 'http://c:4318', OTEL_SDK_DISABLED: 'true' })).toBe(false); + expect(tracingConfigured({ OTEL_EXPORTER_OTLP_ENDPOINT: 'http://c:4318', OTEL_TRACES_EXPORTER: 'none' })).toBe(false); + expect(tracingConfigured({ OTEL_EXPORTER_OTLP_ENDPOINT: 'http://c:4318', OTEL_TRACES_EXPORTER: 'otlp' })).toBe(true); + }); + + it('loads the SDK at startup only when configured', async () => { + vi.stubEnv('NEXT_RUNTIME', 'nodejs'); + const { register } = await import('@/instrumentation'); + await register(); + expect(started.count).toBe(0); + vi.stubEnv('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://collector.example.org:4318'); + await register(); + expect(started.count).toBe(1); + }); + + it('lists the OTLP endpoint as a host the admin set, only when set', () => { + const base = { get: () => undefined, policy: { pushRelays: [], pushRelayUrl: '', defaultSidebarApps: [], features: { sidebarAppsEnabled: false, calendarEnabled: false } }, buildRelayUrl: '' }; + expect(listReachableHosts({ ...base, env: {} }).filter((h) => h.kind === 'Tracing (OTLP)')).toEqual([]); + expect(listReachableHosts({ ...base, env: { OTEL_EXPORTER_OTLP_ENDPOINT: 'https://otel.example.org:4318' } }).filter((h) => h.kind === 'Tracing (OTLP)')) + .toEqual([{ kind: 'Tracing (OTLP)', host: 'otel.example.org:4318', origin: 'https://otel.example.org:4318', setIn: 'OTEL_EXPORTER_OTLP_ENDPOINT', side: 'server' }]); + }); +}); diff --git a/lib/tracing/__tests__/tracing.test.ts b/lib/tracing/__tests__/tracing.test.ts new file mode 100644 index 000000000..dd9b5c73c --- /dev/null +++ b/lib/tracing/__tests__/tracing.test.ts @@ -0,0 +1,115 @@ +// @vitest-environment node +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { InMemorySpanExporter } from '@opentelemetry/sdk-trace-node'; +import { trace } from '@opentelemetry/api'; +import { W3CTraceContextPropagator } from '@opentelemetry/core'; + +vi.mock('@/lib/admin/config-manager', () => ({ + configManager: { ensureLoaded: async () => {}, get: (_key: string, fallback: unknown) => fallback }, +})); + +import { createProvider } from '../start'; +import { markTracingActive } from '../config'; +import { scrubAttributes, scrubSpanName } from '../scrub'; +import { observeUpstream } from '@/lib/metrics/upstream'; +import { fetchJmapServer } from '@/lib/stalwart/server-fetch'; +import { verifyJmapAuth } from '@/lib/auth/verify-jmap-auth'; + +const exporter = new InMemorySpanExporter(); +const provider = createProvider(exporter, false); +const ADDRESS = 'ceo.private@acme-corp.example'; + +beforeAll(() => { + provider.register({ propagator: new W3CTraceContextPropagator() }); + markTracingActive(); +}); +afterAll(async () => { + await provider.shutdown(); + trace.disable(); +}); +beforeEach(() => exporter.reset()); + +describe('spans for upstream calls', () => { + it('are named after the target and carry no URL, host or user', async () => { + await observeUpstream('wopi', async () => new Response(null, { status: 200 })); + const [span] = exporter.getFinishedSpans(); + expect(span.name).toBe('upstream wopi'); + expect(span.attributes).toEqual({ 'bulwark.upstream.target': 'wopi', 'http.response.status_code': 200 }); + }); + + it('hand the trace context to the callee', async () => { + let seen: Record = {}; + await observeUpstream('stalwart', async (headers) => { seen = headers; return new Response(null, { status: 401 }); }); + const [span] = exporter.getFinishedSpans(); + expect(seen.traceparent).toMatch(new RegExp(`^00-${span.spanContext().traceId}-${span.spanContext().spanId}-01$`)); + }); + + it('reach Stalwart as a traceparent header', async () => { + const fetchSpy = vi.fn().mockResolvedValue(new Response('{}', { status: 200 })); + vi.stubGlobal('fetch', fetchSpy); + try { + await fetchJmapServer('https://mail.example.org/jmap/session', { headers: { Accept: 'application/json' } }, true); + const headers = new Headers((fetchSpy.mock.calls[0][1] as RequestInit).headers); + expect(headers.get('traceparent')).toMatch(/^00-[0-9a-f]{32}-[0-9a-f]{16}-01$/); + expect(headers.get('accept')).toBe('application/json'); + } finally { + vi.unstubAllGlobals(); + } + }); + + it('reach Stalwart from the sign-in check too', async () => { + const fetchSpy = vi.fn().mockResolvedValue(new Response(JSON.stringify({ apiUrl: 'https://mail.example.org/jmap/', username: 'u@example.org' }), { status: 200 })); + vi.stubGlobal('fetch', fetchSpy); + try { + await verifyJmapAuth('https://mail.example.org', 'Basic dTpw', { trusted: true }).catch(() => undefined); + const headers = new Headers((fetchSpy.mock.calls[0][1] as RequestInit).headers); + expect(String(fetchSpy.mock.calls[0][0])).toBe('https://mail.example.org/.well-known/jmap'); + expect(headers.get('traceparent')).toMatch(/^00-[0-9a-f]{32}-[0-9a-f]{16}-01$/); + expect(headers.get('authorization')).toBe('Basic dTpw'); + } finally { + vi.unstubAllGlobals(); + } + }); + + it('mark 5xx and thrown errors as errors, without the error message', async () => { + await expect(observeUpstream('translate', async () => { throw new Error(`refused for ${ADDRESS}`); })).rejects.toThrow(); + const [span] = exporter.getFinishedSpans(); + expect(span.status).toEqual({ code: 2, message: 'Error' }); + }); +}); + +describe('every exported span', () => { + it('is scrubbed before export', () => { + const tracer = trace.getTracer('test'); + const span = tracer.startSpan('fetch GET https://mail.example.org/jmap/download/a/b/term-sheet.pdf?accept=x'); + span.setAttributes({ + 'url.full': 'https://mail.example.org/jmap/download/a/b/term-sheet.pdf?accept=x', + 'http.target': '/en/mail/folder/Acquisition%20Talks/Mabc?q=merger', + 'url.query': 'q=merger', + 'http.user_agent': 'Mozilla/5.0', + 'client.address': '203.0.113.7', + 'enduser.id': ADDRESS, + 'next.route': '/[locale]/mail/[[...segments]]', + 'custom.note': `sent by ${ADDRESS} with Bearer abc.def`, + }); + span.end(); + const [exported] = exporter.getFinishedSpans(); + expect(exported.name).toBe('fetch GET https://mail.example.org'); + expect(exported.attributes).toEqual({ + 'url.full': 'https://mail.example.org', + 'http.target': '/mail/…', + 'next.route': '/[locale]/mail/[[...segments]]', + 'custom.note': 'sent by [email] with Bearer [token]', + }); + }); +}); + +describe('span names', () => { + it('keep route patterns and reduce raw paths', () => { + expect(scrubSpanName('GET /api/admin/plugins/[id]')).toBe('GET /api/admin/plugins/[id]'); + expect(scrubSpanName('GET /de/contacts/Mustermann')).toBe('GET /contacts/…'); + const attrs: Record = { 'url.path': '/api/wopi/files/abc' }; + scrubAttributes(attrs); + expect(attrs['url.path']).toBe('/api/wopi/…'); + }); +}); diff --git a/lib/tracing/config.ts b/lib/tracing/config.ts new file mode 100644 index 000000000..a23e40079 --- /dev/null +++ b/lib/tracing/config.ts @@ -0,0 +1,30 @@ +/** + * Whether OpenTelemetry tracing is configured, from the standard OTEL_* + * variables only: an OTLP endpoint (OTEL_EXPORTER_OTLP_TRACES_ENDPOINT or + * OTEL_EXPORTER_OTLP_ENDPOINT), unless OTEL_SDK_DISABLED=true or + * OTEL_TRACES_EXPORTER=none. Without it the SDK is never loaded. + * + * https://bulwarkmail.org/docs/deployment/monitoring#tracing + */ +export function tracingEndpoint(env: Record = process.env): string | null { + if ((env.OTEL_SDK_DISABLED ?? '').trim().toLowerCase() === 'true') return null; + const exporters = (env.OTEL_TRACES_EXPORTER ?? '').toLowerCase().split(',').map((s) => s.trim()).filter(Boolean); + if (exporters.length > 0 && !exporters.includes('otlp')) return null; + const endpoint = (env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT || env.OTEL_EXPORTER_OTLP_ENDPOINT || '').trim(); + return endpoint || null; +} + +export function tracingConfigured(env: Record = process.env): boolean { + return tracingEndpoint(env) !== null; +} + +const ACTIVE = Symbol.for('bulwark.tracing.active'); + +/** Set by lib/tracing/start.ts once the SDK is registered. */ +export function markTracingActive(): void { + (globalThis as typeof globalThis & { [ACTIVE]?: boolean })[ACTIVE] = true; +} + +export function isTracingActive(): boolean { + return (globalThis as typeof globalThis & { [ACTIVE]?: boolean })[ACTIVE] === true; +} diff --git a/lib/tracing/scrub.ts b/lib/tracing/scrub.ts new file mode 100644 index 000000000..e698457e0 --- /dev/null +++ b/lib/tracing/scrub.ts @@ -0,0 +1,60 @@ +import { scrubRoutePath } from '@/lib/route-groups'; +import { scrubText } from '@/lib/error-reporting/scrub'; + +/** + * What a span may carry before it leaves this server: no user names, mail + * addresses, tokens, query strings or paths that name folders, messages or + * files. Next's own spans and ours go through it. + * + * - absolute URLs keep their origin only (a Stalwart download URL ends in + * the attachment's file name) + * - request paths become their route group (`/mail/…`) + * - query, fragment, user agent and client address attributes are dropped + * - any other text with an address or a credential is scrubbed + * + * https://bulwarkmail.org/docs/deployment/monitoring#tracing + */ + +type AttributeValue = string | number | boolean | Array | undefined; +export type Attributes = Record; + +const DROP = /^(?:url\.query|url\.fragment|http\.user_agent|user_agent\.|enduser\.|client\.address|client\.port|http\.client_ip|net\.peer\.ip|net\.sock\.peer\.addr|network\.peer\.address|http\.request\.header\.|http\.response\.header\.|db\.statement)/; +const PATH_KEYS = new Set(['http.target', 'url.path', 'next.page.path']); +const URL_IN_TEXT = /\b(https?):\/\/([^/\s?#"'<>]+)[^\s"'<>]*/gi; +const METHOD_AND_PATH = /\b(GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS)\s+(\/[^\s]*)/g; + +/** A path from the request: keep route patterns (`/api/admin/[id]`), reduce anything else to its group. */ +function safePath(path: string): string { + const bare = path.split(/[?#]/, 1)[0]; + return bare.includes('[') && /^[\w/[\].()-]+$/.test(bare) ? bare : scrubRoutePath(bare); +} + +/** A span name: URLs reduced to their origin, request paths to their group. */ +export function scrubSpanName(name: string): string { + return name + .replace(URL_IN_TEXT, (_match, scheme: string, host: string) => `${scheme.toLowerCase()}://${host}`) + .replace(METHOD_AND_PATH, (_match, method: string, path: string) => `${method} ${safePath(path)}`); +} + +function scrubString(key: string, value: string): string { + if (PATH_KEYS.has(key)) return safePath(value); + if (/@|bearer |basic |eyJ|:\/\//i.test(value)) { + return value.replace(URL_IN_TEXT, (_m, scheme: string, host: string) => `${scheme.toLowerCase()}://${host}`) + .split(/\s+/).map((word) => (/@|eyJ/.test(word) ? scrubText(word) : word)).join(' ') + .replace(/\b(Bearer|Basic)\s+\S+/gi, '$1 [token]'); + } + return value; +} + +/** Scrub span attributes in place (the processor runs it before export). */ +export function scrubAttributes(attributes: Attributes): void { + for (const key of Object.keys(attributes)) { + if (DROP.test(key)) { + delete attributes[key]; + continue; + } + const value = attributes[key]; + if (typeof value === 'string') attributes[key] = scrubString(key, value); + else if (Array.isArray(value)) attributes[key] = value.map((v) => (typeof v === 'string' ? scrubString(key, v) : v)); + } +} diff --git a/lib/tracing/start.ts b/lib/tracing/start.ts new file mode 100644 index 000000000..12f1cb691 --- /dev/null +++ b/lib/tracing/start.ts @@ -0,0 +1,68 @@ +import { W3CTraceContextPropagator } from '@opentelemetry/core'; +import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'; +import { detectResources, envDetector, resourceFromAttributes } from '@opentelemetry/resources'; +import { + BatchSpanProcessor, + NodeTracerProvider, + SimpleSpanProcessor, + type ReadableSpan, + type Span, + type SpanExporter, + type SpanProcessor, +} from '@opentelemetry/sdk-trace-node'; +import type { Context } from '@opentelemetry/api'; +import { markTracingActive } from './config'; +import { scrubAttributes, scrubSpanName, type Attributes } from './scrub'; + +/** + * OpenTelemetry tracing, loaded only by a dynamic import from + * instrumentation.ts when an OTLP endpoint is configured (./config.ts). + * Everything else comes from the standard OTEL_* variables, which the OTLP + * exporter and the env resource detector read themselves. + * + * Next's own spans (requests, rendering, route handlers) and Bulwark's spans + * for the calls it makes (lib/metrics/upstream.ts) go through the same + * provider; a processor scrubs every span before export (./scrub.ts). + * + * https://bulwarkmail.org/docs/deployment/monitoring#tracing + */ + +/** Scrubs a span's name and attributes, then hands it on. */ +export class ScrubbingSpanProcessor implements SpanProcessor { + constructor(private readonly next: SpanProcessor) {} + + onStart(span: Span, parentContext: Context): void { + this.next.onStart(span, parentContext); + } + + onEnd(span: ReadableSpan): void { + scrubAttributes(span.attributes as Attributes); + const named = span as unknown as { name: string }; + named.name = scrubSpanName(span.name); + for (const event of span.events) if (event.attributes) scrubAttributes(event.attributes as Attributes); + this.next.onEnd(span); + } + + forceFlush(): Promise { + return this.next.forceFlush(); + } + + shutdown(): Promise { + return this.next.shutdown(); + } +} + +export function createProvider(exporter: SpanExporter, batch = true): NodeTracerProvider { + const resource = resourceFromAttributes({ + 'service.name': process.env.OTEL_SERVICE_NAME?.trim() || 'bulwark-webmail', + 'service.version': process.env.NEXT_PUBLIC_APP_VERSION || '0.0.0', + }).merge(detectResources({ detectors: [envDetector] })); + const inner: SpanProcessor = batch ? new BatchSpanProcessor(exporter) : new SimpleSpanProcessor(exporter); + return new NodeTracerProvider({ resource, spanProcessors: [new ScrubbingSpanProcessor(inner)] }); +} + +export function startTracing(): void { + const provider = createProvider(new OTLPTraceExporter()); + provider.register({ propagator: new W3CTraceContextPropagator() }); + markTracingActive(); +} diff --git a/lib/wopi/discovery.ts b/lib/wopi/discovery.ts index d2150567f..971285f26 100644 --- a/lib/wopi/discovery.ts +++ b/lib/wopi/discovery.ts @@ -1,4 +1,5 @@ import { configManager } from '@/lib/admin/config-manager'; +import { observeUpstream } from '@/lib/metrics/upstream'; import { logger } from '@/lib/logger'; /** @@ -123,10 +124,10 @@ export async function getWopiActions(): Promise { async function fetchWopiActions(discoveryUrl: string): Promise { try { - const res = await fetch(discoveryUrl, { + const res = await observeUpstream('wopi', () => fetch(discoveryUrl, { headers: { Accept: 'text/xml' }, signal: AbortSignal.timeout(DISCOVERY_TIMEOUT_MS), - }); + })); if (!res.ok) { logger.warn('WOPI discovery fetch failed', { discoveryUrl, status: res.status }); return null; diff --git a/next.config.ts b/next.config.ts index e8cfc98f7..91d183d57 100644 --- a/next.config.ts +++ b/next.config.ts @@ -47,8 +47,15 @@ if (isLite && (process.env.NEXT_PUBLIC_LOCALE_PREFIX ?? "never") !== "always") { throw new Error("Bulwark Lite requires NEXT_PUBLIC_LOCALE_PREFIX=always (use `npm run build:lite`)"); } +// Browser source maps, for resolving the stack traces in error reports. +// Only CI turns them on (BULWARK_SOURCE_MAPS=1): it uploads them to the +// error tracker and removes every .map file before anything ships +// (scripts/sourcemaps.mjs). A plain build makes none. +const sourceMaps = process.env.BULWARK_SOURCE_MAPS === "1"; + const nextConfig: NextConfig = { output: isLite ? "export" : "standalone", + productionBrowserSourceMaps: sourceMaps, // No `X-Powered-By: Next.js`; headers() below sends // `X-Powered-By: bulwarkmail.org` instead, without a version. poweredByHeader: false, @@ -70,9 +77,9 @@ const nextConfig: NextConfig = { // esbuild ships native binaries + a README the bundler can't parse; load // it from node_modules at runtime instead of trying to bundle it. Used by // PLUGIN_DEV_DIR's on-the-fly bundler. ioredis (the shared store, #1066) - // is loaded from node_modules too, so the proxy and the route bundles - // don't each carry a copy. - serverExternalPackages: ["esbuild", "ioredis"], + // and @prometheus-io/client (/api/metrics) are loaded from node_modules + // too, so the proxy and the route bundles don't each carry a copy. + serverExternalPackages: ["esbuild", "ioredis", "@prometheus-io/client"], // The build's type check leaves out tests, which are a third of the files // and push tsc past the default heap on a 4 GB machine (#1431). // `npm run typecheck` still checks them with tsconfig.json. @@ -141,6 +148,7 @@ const nextConfig: NextConfig = { }, turbopack: { root: process.env.NEXT_TURBOPACK_ROOT || import.meta.dirname, + ...(sourceMaps ? { debugIds: true } : {}), }, env: { NEXT_PUBLIC_GIT_COMMIT: gitCommitHash, diff --git a/package-lock.json b/package-lock.json index b4785ce20..0efeb25a4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,12 @@ "@dnd-kit/utilities": "^3.2.2", "@noble/curves": "^2.4.0", "@noble/hashes": "^2.4.0", + "@opentelemetry/api": "1.9.1", + "@opentelemetry/core": "2.8.0", + "@opentelemetry/exporter-trace-otlp-http": "0.219.0", + "@opentelemetry/resources": "2.8.0", + "@opentelemetry/sdk-trace-node": "2.8.0", + "@prometheus-io/client": "0.16.1", "@tabler/icons-react": "^3.50.0", "@tanstack/react-virtual": "^3.14.23", "@tiptap/extension-color": "^3.32.1", @@ -2392,6 +2398,204 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/api-logs": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.219.0.tgz", + "integrity": "sha512-FFx7YnaYJlIjqWW/AG/yAZ0L/NEY724PipXXXQLdtZPbLwBGbUMTGL1i/esI56TWfTUXxhLfpgrnWJCG8aUJyg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/context-async-hooks": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-2.8.0.tgz", + "integrity": "sha512-/3FIraneMcng67SUJCxvyInk/oxzwsxyadufk0wwfOBLf5wqtAGX4MoQASwSbndBPeARzBryUM9Azr5kHIdWLw==", + "license": "Apache-2.0", + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/core": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.8.0.tgz", + "integrity": "sha512-hd1Lfh8p545nNz+jq1Ejfz+Mn1hyLuxYn1YzTfFNrxr8urEWMNQLPf1Th8kjOH+HxwawCrtgBp8JpBUR4ZSgww==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-http": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-http/-/exporter-trace-otlp-http-0.219.0.tgz", + "integrity": "sha512-9t6SvBXXBEjOBcIzgozvBbd3jWrv3Gt3ngGhl1fhdZ/zRc7oZDVOFEqbi2zlBpW9BXhgDMKv422J0DL/3iQWfw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.8.0", + "@opentelemetry/otlp-exporter-base": "0.219.0", + "@opentelemetry/otlp-transformer": "0.219.0", + "@opentelemetry/resources": "2.8.0", + "@opentelemetry/sdk-trace-base": "2.8.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-exporter-base": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.219.0.tgz", + "integrity": "sha512-zvIxQX/AZUVKDU+hCuYx+7UkiP7GRdnk1ZbFQRYzHvYp47cAWR4j3IhoPhV9KaeXEv2xdGq3IA6PnpzDmLcmSA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.8.0", + "@opentelemetry/otlp-transformer": "0.219.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.219.0.tgz", + "integrity": "sha512-aaYKAyXhw9VchKZVGOopD3Gw/kPsyrX2c6IQ0AW32mTjqmZOh5Y6Gf5OYqTNqVktAeBjmFinhyFaCwW6GYK9YQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.219.0", + "@opentelemetry/core": "2.8.0", + "@opentelemetry/resources": "2.8.0", + "@opentelemetry/sdk-logs": "0.219.0", + "@opentelemetry/sdk-metrics": "2.8.0", + "@opentelemetry/sdk-trace-base": "2.8.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/resources": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.8.0.tgz", + "integrity": "sha512-qmXQ27ilDbUK/vGMqwL8D4/rhn76C+sherM4wTbjlfknR8Nvfc/hCxjRJPhkzZzUsPiNg16SA31NxMabwttRjg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.8.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-logs": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.219.0.tgz", + "integrity": "sha512-s6lTKRakaPClvKoWHRChxnXjDMkM/TQ30ff78jN6EBGf7MI7VzANE5PU3f4z9qDUudWjvZjOLHG0rBnBKYvoXA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.219.0", + "@opentelemetry/core": "2.8.0", + "@opentelemetry/resources": "2.8.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.4.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-metrics": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.8.0.tgz", + "integrity": "sha512-UDBGaj6W0Rgy5rTTaoxs8gVGF/aGkAKyjurJv7se6wjRxJu7FoquTLT/vt54DZfo4crbprYfhX/SOK9+BPw1qg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.8.0", + "@opentelemetry/resources": "2.8.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-base": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.8.0.tgz", + "integrity": "sha512-mhU4jp+vW0mGbFRd+GeXHvmfA4aDqWjBjLC3pE5XMpLs0IE2ryYb019Ts2AQrOq67gaTF25D91+fgvEHDZEnuQ==", + "deprecated": "Use @opentelemetry/sdk-trace package", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.8.0", + "@opentelemetry/resources": "2.8.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-node": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-node/-/sdk-trace-node-2.8.0.tgz", + "integrity": "sha512-nZt9OGufioAc3AfoLTqA9bsAeaMJAictYDdI2VcNQ+PmT+3rfKjAZDZvgPfd8VPX0O5Bw1hdQF6kDK8VSpZiWg==", + "deprecated": "Use @opentelemetry/sdk-trace package", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/context-async-hooks": "2.8.0", + "@opentelemetry/core": "2.8.0", + "@opentelemetry/sdk-trace-base": "2.8.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/semantic-conventions": { + "version": "1.44.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.44.0.tgz", + "integrity": "sha512-E3JkOlMBi3N8OGGkbl7tFyqk2MAS6/lySRZBK3Mu7AzjF1fT8oIChH9ChE48SUWzDn9Kdwta+G/p5LJ4xW4CLQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, "node_modules/@oxc-project/types": { "version": "0.155.0", "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.155.0.tgz", @@ -2721,6 +2925,19 @@ "dev": true, "license": "MIT" }, + "node_modules/@prometheus-io/client": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@prometheus-io/client/-/client-0.16.1.tgz", + "integrity": "sha512-XkFPsGFGoSs/UMg/vh0QLDpanl/tsrOV5PLHdpGe8Ho+ir/Llhytr15j6y8ZwLrpIGzK9gTVVpIalA6C4UjUZw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.4.0", + "tdigest": "^0.1.1" + }, + "engines": { + "node": "^22 || ^24 || >=26" + } + }, "node_modules/@rolldown/binding-android-arm-eabi": { "version": "1.2.15", "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.15.tgz", @@ -4966,6 +5183,12 @@ "require-from-string": "^2.0.2" } }, + "node_modules/bintrees": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", + "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==", + "license": "MIT" + }, "node_modules/brace-expansion": { "version": "5.0.12", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz", @@ -9463,6 +9686,15 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/tdigest": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.3.tgz", + "integrity": "sha512-zbRt+lT+/H4fRItHshczHErVCQnitJk8MfMT24MqFJf3YL7SJJPqGIGeuOdvxXxM/AHFzKBl7WoyaYwqO9s3Kw==", + "license": "MIT", + "dependencies": { + "bintrees": "1.0.2" + } + }, "node_modules/tinybench": { "version": "6.2.2", "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-6.2.2.tgz", diff --git a/package.json b/package.json index d7745c563..5adc7a774 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,12 @@ "@dnd-kit/utilities": "^3.2.2", "@noble/curves": "^2.4.0", "@noble/hashes": "^2.4.0", + "@opentelemetry/api": "1.9.1", + "@opentelemetry/core": "2.8.0", + "@opentelemetry/exporter-trace-otlp-http": "0.219.0", + "@opentelemetry/resources": "2.8.0", + "@opentelemetry/sdk-trace-node": "2.8.0", + "@prometheus-io/client": "0.16.1", "@tabler/icons-react": "^3.50.0", "@tanstack/react-virtual": "^3.14.23", "@tiptap/extension-color": "^3.32.1", diff --git a/proxy.ts b/proxy.ts index b539c4d5a..eb767eb03 100644 --- a/proxy.ts +++ b/proxy.ts @@ -179,6 +179,8 @@ export function mailFrameAncestors(allowed: string | undefined): string { return sources.join(" "); } +const PROBE_PATHS = new Set(["/api/health", "/api/health/live", "/api/health/ready", "/api/ready", "/api/metrics"]); + function isSetupPath(pathname: string): boolean { return ( pathname === "/setup" || @@ -200,9 +202,10 @@ function redirectTo(request: NextRequest, pathname: string, search: string): Nex export async function proxy(request: NextRequest) { // The probes answer without config or store, so a replica that is still - // starting (or lost its shared store) can say so (#1066). + // starting (or lost its shared store) can say so (#1066). The metrics + // endpoint, too: a scrape shouldn't fail while a replica boots. const probePath = routePathOf(request.nextUrl.pathname); - if (probePath === "/api/health" || probePath === "/api/ready") { + if (PROBE_PATHS.has(probePath)) { const probe = NextResponse.next(); probe.headers.set("X-Content-Type-Options", "nosniff"); probe.headers.set("X-Frame-Options", "DENY"); diff --git a/scripts/healthcheck.mjs b/scripts/healthcheck.mjs new file mode 100644 index 000000000..d78a46157 --- /dev/null +++ b/scripts/healthcheck.mjs @@ -0,0 +1,15 @@ +// Docker HEALTHCHECK: exit 0 when /api/health/live answers 200 within 4 s. +// Liveness only, so a Stalwart outage never marks the container unhealthy. +// The base path is baked in at build time (BULWARK_BASE_PATH in the image). +// https://bulwarkmail.org/docs/deployment/monitoring#health +const port = process.env.PORT || '3000'; +const listen = process.env.HOSTNAME; +const host = !listen || listen === '0.0.0.0' || listen === '::' ? '127.0.0.1' : listen.includes(':') ? `[${listen}]` : listen; +const base = (process.env.BULWARK_BASE_PATH || '').replace(/\/+$/, ''); + +try { + const res = await fetch(`http://${host}:${port}${base}/api/health/live`, { signal: AbortSignal.timeout(4000) }); + process.exit(res.ok ? 0 : 1); +} catch { + process.exit(1); +} diff --git a/scripts/lite/build.mjs b/scripts/lite/build.mjs index cd5684625..2de7fd7f6 100644 --- a/scripts/lite/build.mjs +++ b/scripts/lite/build.mjs @@ -8,6 +8,11 @@ // (Stalwart target: the entry document, then the zip) // 4. verify fail on missing files or undocumented /api references // +// With BULWARK_SOURCE_MAPS=1 (CI) the browser source maps move out of out/ +// right after the build, before anything is packed, into SOURCEMAPS_OUT +// (default sourcemaps/lite-) for the error tracker. Either way the +// export and the Stalwart zip are checked for maps last. +// // Inputs (env): LITE_TARGET (static | stalwart), LITE_JMAP_SERVER_URL, // LITE_APP_NAME, LITE_ALLOW_CUSTOM_ENDPOINT, LITE_REMEMBER_ME, LITE_DEMO_MODE, // LITE_NO_THIRD_PARTY_TRAFFIC, @@ -16,13 +21,14 @@ // --skip-prepare (tree already pruned), --dry-run (print the plan only). import { spawnSync } from "node:child_process"; import { createRequire } from "node:module"; -import { readFileSync } from "node:fs"; +import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; -import { makeBuildId, normalizeBasePath, parseLiteTarget, resolveRepoRoot } from "./lib.mjs"; +import { makeBuildId, normalizeBasePath, parseLiteTarget, resolveRepoRoot, STALWART_ZIP_NAME } from "./lib.mjs"; import { runPrepare } from "./prepare.mjs"; import { runPostbuild } from "./postbuild.mjs"; import { verifyExport } from "./verify.mjs"; import { buildPstWorker } from "../build-pst-worker.mjs"; +import { check as checkSourceMaps, collect as collectSourceMaps } from "../sourcemaps.mjs"; const repoRoot = resolveRepoRoot(import.meta.url); const argv = process.argv.slice(2); @@ -86,6 +92,12 @@ if (build.status !== 0) { process.exit(build.status ?? 1); } +if (env.BULWARK_SOURCE_MAPS === "1") { + const out = env.SOURCEMAPS_OUT?.trim() || join(repoRoot, "sourcemaps", `lite-${target}`); + const moved = collectSourceMaps([join(repoRoot, "out", "_next", "static")], out); + console.log(`[lite] moved ${moved.length} source maps to ${out}`); +} + runPostbuild({ env }); const { problems, locales, apiStrings, zipBytes } = verifyExport({ target }); @@ -94,4 +106,11 @@ if (problems.length > 0) { for (const p of problems) console.error(` - ${p}`); process.exit(1); } +const shipped = [join(repoRoot, "out"), join(repoRoot, STALWART_ZIP_NAME)].filter((p) => existsSync(p)); +const mapProblems = shipped.flatMap((p) => checkSourceMaps(p).map((problem) => `${p}: ${problem}`)); +if (mapProblems.length > 0) { + console.error(`[lite] source maps would ship (${mapProblems.length}):`); + for (const p of mapProblems.slice(0, 20)) console.error(` - ${p}`); + process.exit(1); +} console.log(`[lite] done: out/ holds ${locales.length} locales; ${apiStrings.length} documented /api strings in chunks${zipBytes ? `; zip ${(zipBytes / 1024 / 1024).toFixed(1)} MB` : ""}`); diff --git a/scripts/sourcemaps.mjs b/scripts/sourcemaps.mjs new file mode 100644 index 000000000..89b24fb1f --- /dev/null +++ b/scripts/sourcemaps.mjs @@ -0,0 +1,205 @@ +#!/usr/bin/env node +// Source maps: built in CI (BULWARK_SOURCE_MAPS=1), uploaded to the error +// tracker, never shipped. https://bulwarkmail.org/docs/deployment/monitoring +// +// node scripts/sourcemaps.mjs collect --out ... +// Moves every .map under each root into //... with a copy of the +// .js/.css file that names it (sentry-cli uploads the pairs), and removes +// the `sourceMappingURL` comment from the shipped file. The chunk keeps +// its `debugId` comment and registration, which is all the tracker needs. +// +// node scripts/sourcemaps.mjs check ... +// Fails when anything that would ship holds a .map file, an inline +// (data:) source map, or a `sourceMappingURL` to a file that isn't there. +import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { basename, dirname, join, relative, resolve, sep } from "node:path"; +import { gunzipSync, inflateRawSync } from "node:zlib"; +import { fileURLToPath } from "node:url"; + +// At the start of a line, as bundlers write it; the same text inside a +// string literal is not a reference. +const COMMENT = /^[ \t]*\/[/*][#@][ \t]*sourceMappingURL=([^\s*'"`]+)[ \t]*(?:\*\/)?[ \t]*$/gm; +const SCRIPT = /\.(?:js|mjs|cjs|css)$/; + +function walk(dir, out = []) { + for (const name of readdirSync(dir)) { + const path = join(dir, name); + const stat = statSync(path, { throwIfNoEntry: false }); + if (!stat) continue; + if (stat.isDirectory()) walk(path, out); + else out.push(path); + } + return out; +} + +/** The sourceMappingURL targets named in a file's text. */ +export function mapReferences(text) { + return [...text.matchAll(COMMENT)].map((m) => m[1]); +} + +/** Remove the sourceMappingURL comments; everything else stays as it is. */ +export function stripReferences(text) { + return text.replace(COMMENT, "").replace(/\n{2,}$/, "\n"); +} + +export function collect(roots, outDir) { + const moved = []; + mkdirSync(outDir, { recursive: true }); + roots.forEach((given, index) => { + // Absolute, so the maps a file names (resolved below) match the walked paths. + const root = resolve(given); + if (!existsSync(root)) return; + const files = walk(root); + const maps = new Set(files.filter((f) => f.endsWith(".map"))); + const target = join(outDir, String(index)); + for (const file of files.filter((f) => SCRIPT.test(f))) { + const text = readFileSync(file, "utf8"); + const refs = mapReferences(text); + if (refs.length === 0) continue; + for (const ref of refs) { + if (ref.startsWith("data:")) continue; + const map = resolve(dirname(file), decodeURIComponent(ref)); + if (!maps.has(map)) continue; + for (const source of [file, map]) { + const dest = join(target, relative(root, source)); + mkdirSync(dirname(dest), { recursive: true }); + copyFileSync(source, dest); + } + } + writeFileSync(file, stripReferences(text)); + } + // Every map goes, named by a file or not. + for (const map of maps) { + const dest = join(target, relative(root, map)); + if (!existsSync(dest)) { + mkdirSync(dirname(dest), { recursive: true }); + copyFileSync(map, dest); + } + rmSync(map); + moved.push(map); + } + }); + return moved; +} + +// --- archives ------------------------------------------------------------- + +function zipEntries(buffer) { + const entries = []; + let eocd = buffer.length - 22; + while (eocd >= 0 && buffer.readUInt32LE(eocd) !== 0x06054b50) eocd--; + if (eocd < 0) throw new Error("not a zip file"); + const count = buffer.readUInt16LE(eocd + 10); + let at = buffer.readUInt32LE(eocd + 16); + for (let i = 0; i < count; i++) { + const method = buffer.readUInt16LE(at + 10); + const size = buffer.readUInt32LE(at + 20); + const nameLength = buffer.readUInt16LE(at + 28); + const extraLength = buffer.readUInt16LE(at + 30); + const commentLength = buffer.readUInt16LE(at + 32); + const local = buffer.readUInt32LE(at + 42); + const name = buffer.subarray(at + 46, at + 46 + nameLength).toString("utf8"); + const dataStart = local + 30 + buffer.readUInt16LE(local + 26) + buffer.readUInt16LE(local + 28); + const raw = buffer.subarray(dataStart, dataStart + size); + entries.push({ name, read: () => (method === 0 ? raw : inflateRawSync(raw)) }); + at += 46 + nameLength + extraLength + commentLength; + } + return entries; +} + +function tarEntries(buffer) { + const entries = []; + let at = 0; + let longName = null; + while (at + 512 <= buffer.length) { + const header = buffer.subarray(at, at + 512); + if (header.every((b) => b === 0)) break; + const field = (start, length) => header.subarray(start, start + length).toString("utf8").replace(/\0.*$/s, ""); + const size = parseInt(field(124, 12).trim() || "0", 8); + const type = field(156, 1); + const prefix = field(345, 155); + let name = longName ?? (prefix ? `${prefix}/${field(0, 100)}` : field(0, 100)); + longName = null; + const data = buffer.subarray(at + 512, at + 512 + size); + if (type === "L") longName = data.toString("utf8").replace(/\0.*$/s, ""); + else if (type === "0" || type === "") entries.push({ name, read: () => data }); + else if (type === "x") { + const path = /\d+ path=([^\n]+)\n/.exec(data.toString("utf8")); + if (path) longName = path[1]; + } + at += 512 + Math.ceil(size / 512) * 512; + } + return entries; +} + +function filesOf(path) { + if (statSync(path).isDirectory()) { + return walk(path).map((file) => ({ name: relative(path, file).split(sep).join("/"), read: () => readFileSync(file) })); + } + const buffer = readFileSync(path); + if (path.endsWith(".zip")) return zipEntries(buffer); + if (path.endsWith(".tar.gz") || path.endsWith(".tgz")) return tarEntries(gunzipSync(buffer)); + if (path.endsWith(".tar")) return tarEntries(buffer); + throw new Error(`don't know how to read ${path}`); +} + +/** Problems with what would ship at `path`; empty when it is clean. */ +export function check(path) { + const problems = []; + const entries = filesOf(path); + const names = new Set(entries.map((e) => e.name)); + for (const entry of entries) { + if (entry.name.endsWith(".map")) { + problems.push(`${entry.name}: source map file`); + continue; + } + if (!SCRIPT.test(entry.name)) continue; + for (const ref of mapReferences(entry.read().toString("utf8"))) { + if (ref.startsWith("data:")) { + problems.push(`${entry.name}: inline source map`); + continue; + } + const target = join(dirname(entry.name), decodeURIComponent(ref)).split(sep).join("/"); + if (!names.has(target)) problems.push(`${entry.name}: sourceMappingURL to missing ${ref}`); + } + } + return problems; +} + +function main(argv) { + const [command, ...rest] = argv; + if (command === "collect") { + const at = rest.indexOf("--out"); + if (at < 0 || !rest[at + 1]) throw new Error("collect needs --out "); + const out = rest[at + 1]; + const roots = rest.filter((_, i) => i !== at && i !== at + 1); + const moved = collect(roots, out); + console.log(`[sourcemaps] moved ${moved.length} maps out of ${roots.join(", ")} into ${out}`); + return 0; + } + if (command === "check") { + let failed = false; + for (const path of rest) { + if (!existsSync(path)) { + console.error(`[sourcemaps] ${path}: not found`); + failed = true; + continue; + } + const problems = check(path); + if (problems.length) { + failed = true; + console.error(`[sourcemaps] ${basename(path)}: ${problems.length} problem(s)`); + for (const p of problems.slice(0, 20)) console.error(` - ${p}`); + } else { + console.log(`[sourcemaps] ${path}: no source maps`); + } + } + return failed ? 1 : 0; + } + console.error("usage: sourcemaps.mjs collect --out ... | check ..."); + return 2; +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + process.exit(main(process.argv.slice(2))); +} diff --git a/scripts/upload-sourcemaps.sh b/scripts/upload-sourcemaps.sh new file mode 100755 index 000000000..5c0c35976 --- /dev/null +++ b/scripts/upload-sourcemaps.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Upload a folder of source maps (with their chunks, as scripts/sourcemaps.mjs +# collect leaves them) to Bugsink. Bugsink finds a map by the debug ID alone, +# so the release name doesn't matter, but it keeps maps per project: every +# project that gets events for these chunks needs them. +# +# BUGSINK_AUTH_TOKEN upload token; without it the upload is skipped with a notice +# BUGSINK_URL default https://bugsink.internal.bulwarkmail.org/ +# BUGSINK_PROJECTS default "bulwark-instances bulwark-telemetry" +# SENTRY_CLI_VERSION default 3.8.2 +# +# https://bulwarkmail.org/docs/deployment/monitoring +set -euo pipefail + +dir="${1:?usage: upload-sourcemaps.sh }" +if [ -z "${BUGSINK_AUTH_TOKEN:-}" ]; then + echo "::notice::BUGSINK_AUTH_TOKEN is not set: source maps in ${dir} were built but not uploaded" + exit 0 +fi +if [ -z "$(find "$dir" -name '*.map' -print -quit 2>/dev/null)" ]; then + echo "::warning::no source maps in ${dir}" + exit 0 +fi + +url="${BUGSINK_URL:-https://bugsink.internal.bulwarkmail.org/}" +cli="@sentry/cli@${SENTRY_CLI_VERSION:-3.8.2}" +for project in ${BUGSINK_PROJECTS:-bulwark-instances bulwark-telemetry}; do + SENTRY_AUTH_TOKEN="$BUGSINK_AUTH_TOKEN" npx --yes "$cli" --url "$url" \ + sourcemaps --org bugsinkhasnoorgs --project "$project" upload "$dir" +done diff --git a/stores/policy-store.ts b/stores/policy-store.ts index 7eeac445a..ece536085 100644 --- a/stores/policy-store.ts +++ b/stores/policy-store.ts @@ -4,6 +4,8 @@ import { DEFAULT_POLICY, DEFAULT_THEME_POLICY, POLICY_DOMAINS_HEADER, POLICY_SCO import { apiFetch } from '@/lib/browser-navigation'; import { IS_LITE, IS_LITE_STALWART, LITE_POLICY_PATH, withLiteBuildId } from '@/lib/lite'; import { applyLitePolicy } from '@/lib/lite-config'; +import { configureErrorReporting } from '@/lib/error-reporting/client'; +import { liteErrorReporting } from '@/lib/error-reporting/lite'; import { resolvePolicy, type PolicySource, type PolicySubject } from '@/lib/admin/policy-scopes'; import { DEFAULT_ORGANISATION_POLICY, type OrganisationPolicy } from '@/lib/admin/organisation-policy'; @@ -203,6 +205,8 @@ export const usePolicyStore = create()((set, get) => ({ const partial = res.headers?.get?.(POLICY_SCOPE_HEADER) === 'public'; const domainsHeader = res.headers?.get?.(POLICY_DOMAINS_HEADER); const scopeDomains = typeof domainsHeader === 'string' ? domainsHeader.split(',').filter(Boolean) : null; + // Lite has no /api/config: error reporting is set in policy.json. + if (IS_LITE) configureErrorReporting(liteErrorReporting(data)); // A refetch that brings nothing new (the periodic one, hooks/use-policy-refresh.ts) // keeps the resolved objects, so nothing that reads the policy re-runs. const json = JSON.stringify(data); @@ -217,12 +221,15 @@ export const usePolicyStore = create()((set, get) => ({ if (!partial) writeCachedPolicy(data); } else if (IS_LITE && res.status === 404) { // No policy.json: the deployment set nothing, which is an answer. + if (IS_LITE) configureErrorReporting(null); set({ ...(fallback.policy ? resolved(fallback.policy, get().subject) : {}), loaded: true, source: 'server' }); writeCachedPolicy(null); } else { + if (IS_LITE) configureErrorReporting(null); settleWithoutServer(); } } catch { + if (IS_LITE && request === policyRequest) configureErrorReporting(null); if (request === policyRequest) settleWithoutServer(); } finally { if (request === policyRequest) set({ fetching: false });