From 692f965f2064df1224fc0517caa8251f6bb817d8 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:40:25 +0100 Subject: [PATCH 01/26] fix: start with ADMIN_PASSWORD when the admin config is mounted read-only #1066 --- .../admin-password-read-only.test.ts | 59 +++++++++++++++++++ lib/admin/password.ts | 32 +++++++++- 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 lib/admin/__tests__/admin-password-read-only.test.ts diff --git a/lib/admin/__tests__/admin-password-read-only.test.ts b/lib/admin/__tests__/admin-password-read-only.test.ts new file mode 100644 index 000000000..bcad786ad --- /dev/null +++ b/lib/admin/__tests__/admin-password-read-only.test.ts @@ -0,0 +1,59 @@ +// @vitest-environment node +import { afterAll, afterEach, describe, expect, it, vi } from 'vitest'; +import { existsSync, rmSync } from 'node:fs'; +import path from 'node:path'; + +// A config dir mounted read-only (Kubernetes ConfigMap, a :ro volume) without +// an admin.json, and ADMIN_PASSWORD in the environment: the admin password +// works from memory on every replica instead of failing the start, which +// kept every replica of a shared-store deployment from ever becoming ready. + +// Hoisted: isConfigReadOnly() reads the env once. +const dir = await vi.hoisted(async () => { + const { mkdtempSync, mkdirSync } = await import('node:fs'); + const { tmpdir } = await import('node:os'); + const { join } = await import('node:path'); + const root = mkdtempSync(join(tmpdir(), 'bw-admin-pw-ro-')); + mkdirSync(join(root, 'config')); + process.env.ADMIN_CONFIG_DIR = join(root, 'config'); + process.env.ADMIN_STATE_DIR = join(root, 'state'); + process.env.ADMIN_CONFIG_READONLY = 'true'; + process.env.ADMIN_PASSWORD = 'from-the-environment'; + return root; +}); + +import { initAdminPassword, isAdminEnabled, resetAdminCacheForTests, verifyAdminPassword } from '../password'; +import { setStoreForTests } from '@/lib/shared-store'; +import { resetLocalStoreForTests } from '@/lib/shared-store/local'; +import { separateFakeStore } from './shared-store-fake'; + +afterEach(() => { + resetAdminCacheForTests(); + setStoreForTests(null); + resetLocalStoreForTests(); +}); + +afterAll(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +describe('ADMIN_PASSWORD with a read-only config and no admin.json', () => { + it('works from memory on a single instance and writes nothing to the config dir', async () => { + await expect(initAdminPassword()).resolves.toBe(true); + expect(isAdminEnabled()).toBe(true); + expect(await verifyAdminPassword('from-the-environment')).toBe(true); + expect(await verifyAdminPassword('wrong')).toBe(false); + expect(existsSync(path.join(dir, 'config', 'admin.json'))).toBe(false); + }); + + it('lets every replica of a shared store start and sign in', async () => { + setStoreForTests(separateFakeStore(path.join(dir, 'store'))); + for (let replica = 0; replica < 3; replica++) { + resetAdminCacheForTests(); + await expect(initAdminPassword()).resolves.toBe(true); + expect(await verifyAdminPassword('from-the-environment')).toBe(true); + expect(await verifyAdminPassword('wrong')).toBe(false); + } + expect(existsSync(path.join(dir, 'config', 'admin.json'))).toBe(false); + }); +}); diff --git a/lib/admin/password.ts b/lib/admin/password.ts index 592991bc9..bae73d0b5 100644 --- a/lib/admin/password.ts +++ b/lib/admin/password.ts @@ -2,7 +2,7 @@ import { scrypt, randomBytes, timingSafeEqual } from 'node:crypto'; import { logger } from '@/lib/logger'; import { isSharedMode, type DocCollection, type PutOptions, type PutResult, type StoredDoc } from '@/lib/shared-store'; import { announceChange, onSharedChange } from '@/lib/shared-store/changes'; -import { assertWritable } from './paths'; +import { assertWritable, isConfigReadOnly } from './paths'; import { adminConfigDocs, adminStateDocs, isStoreUnavailable } from './admin-store'; import type { AdminConfigData, AdminStateData } from './types'; import { readFileEnv } from '../read-file-env'; @@ -131,6 +131,12 @@ interface AdminCache { initialized: boolean; /** Shared mode: one read of the state for checks that run together. */ stateRead?: Promise; + /** + * The password came from ADMIN_PASSWORD into a read-only config without an + * admin.json: it lives in this process only, and every replica makes it + * from the same variable on start. + */ + fromEnvOnly?: boolean; unlisten?: () => void; } @@ -149,6 +155,7 @@ export function resetAdminCacheForTests(): void { cache.initialized = false; cache.stateRead = undefined; cache.unlisten = undefined; + cache.fromEnvOnly = undefined; } function freshState(): AdminStateData { @@ -271,6 +278,24 @@ export async function initAdminPassword(): Promise { const hash = isHashed(envPassword) ? envPassword : await hashPassword(envPassword); const config: AdminConfigData = { passwordHash: hash, passwordHashFile: undefined }; + if (isConfigReadOnly()) { + // Nowhere to save admin.json (a ConfigMap, a :ro volume). Keep the + // password in memory instead of failing the start; each replica does the + // same from the same variable. Changing it in the admin console is + // refused, as every other write to read-only config is. + cache.config = config; + cache.fromEnvOnly = true; + cache.state = shared ? await readStateData() : (await readStateData()) ?? freshState(); + if (shared && !cache.state) { + const fresh = freshState(); + cache.state = (await writeStateData(fresh, { ifVersion: null })).ok ? fresh : (await readStateData()) ?? fresh; + } + if (!cache.initialized) { + logger.info('Admin dashboard enabled with ADMIN_PASSWORD (config is read-only, so it is not saved to admin.json)'); + } + cache.initialized = true; + return true; + } if (shared) { // Replicas booting together with ADMIN_PASSWORD: the first one writes // admin.json and moves the session generation on, the others use it. @@ -319,6 +344,11 @@ function verifyAgainst(config: AdminConfigData, password: string): Promise { + if (cache.fromEnvOnly && cache.config) { + // A mounted admin.json that appeared later still wins. + const mounted = (await readConfigDoc())?.data ?? null; + return verifyAgainst(mounted ?? cache.config, password); + } if (isSharedMode()) { cache.config = (await readConfigDoc())?.data ?? null; } else if (!cache.config) { From cba1109444da94d4c41c91aa1cb7b17e081ea4ca Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:50:19 +0100 Subject: [PATCH 02/26] feat: report browser and server errors to an error tracker the admin sets Admins had logs and nothing else: client errors only reached the browser console. An admin can now set a Sentry-protocol address (DSN) for Bugsink, GlitchTip or Sentry under System > Monitoring, or with `ERROR_REPORTING_DSN`, `ERROR_REPORTING_DSN_FILE`, `ERROR_REPORTING_SAMPLE_RATE` and `ERROR_REPORTING_ENVIRONMENT`. Bulwark Lite reads the same from `errorReporting` in policy.json. Everything stays off while no address is set. - browsers report errors from the error boundaries, window.onerror, unhandled rejections and failed JMAP requests (method and error type only), with route and JMAP method breadcrumbs and the chunks' debug IDs - the server reports what Next hands to onRequestError and what routes log with logger.error - every report is scrubbed first: quoted text, addresses, URLs, file names, tokens and the rest after the first line are removed, and the server scrubs what browsers post to /api/errors again - reports are deduplicated and rate-limited per session and process - errors are counted per day for "errors today" on the admin page - startup messages and audit entries go through the logger, so `LOG_FORMAT=json` covers them --- app/(main)/[locale]/error.tsx | 2 + app/(main)/admin/[[...segments]]/page.tsx | 2 + app/(main)/admin/_pages/monitoring.tsx | 175 +++++++++++++ app/(main)/layout.tsx | 2 + app/api/admin/monitoring/route.ts | 30 +++ app/api/config/route.ts | 4 + app/api/errors/__tests__/route.test.ts | 89 +++++++ app/api/errors/route.ts | 70 +++++ app/global-error.tsx | 6 + components/admin/admin-shell.tsx | 2 + components/error/error-boundary.tsx | 5 + components/error/error-reporting-root.tsx | 30 +++ components/icons/index.ts | 1 + hooks/use-config.ts | 5 + instrumentation.node.ts | 46 ++-- instrumentation.ts | 14 + lib/__tests__/config-validation.test.ts | 15 ++ lib/__tests__/jmap-error-reporting.test.ts | 58 +++++ lib/__tests__/route-groups.test.ts | 21 ++ .../reachable-hosts-observability.test.ts | 21 ++ lib/admin/admin-nav.ts | 12 + lib/admin/audit.ts | 4 + lib/admin/outgoing-hosts-registry.ts | 3 +- lib/admin/reachable-hosts.ts | 2 + lib/admin/types.ts | 6 + lib/config-validation.ts | 8 + .../__tests__/aggregate.test.ts | 34 +++ lib/error-reporting/__tests__/client.test.ts | 120 +++++++++ .../__tests__/envelope.test.ts | 118 +++++++++ lib/error-reporting/__tests__/lite.test.ts | 21 ++ lib/error-reporting/__tests__/scrub.test.ts | 102 ++++++++ lib/error-reporting/__tests__/server.test.ts | 101 ++++++++ lib/error-reporting/aggregate.ts | 179 +++++++++++++ lib/error-reporting/client.ts | 244 ++++++++++++++++++ lib/error-reporting/debug-ids.ts | 47 ++++ lib/error-reporting/envelope.ts | 149 +++++++++++ lib/error-reporting/fingerprint.ts | 34 +++ lib/error-reporting/limiter.ts | 81 ++++++ lib/error-reporting/lite.ts | 22 ++ lib/error-reporting/scrub.ts | 126 +++++++++ lib/error-reporting/server.ts | 132 ++++++++++ lib/error-reporting/settings.ts | 53 ++++ lib/error-reporting/stack.ts | 57 ++++ lib/jmap/client.ts | 33 ++- lib/logger.ts | 31 +++ lib/route-groups.ts | 75 ++++++ lib/shared-store/server-state-allowlist.ts | 1 + lib/telemetry/state.ts | 11 +- stores/policy-store.ts | 6 + 49 files changed, 2384 insertions(+), 26 deletions(-) create mode 100644 app/(main)/admin/_pages/monitoring.tsx create mode 100644 app/api/admin/monitoring/route.ts create mode 100644 app/api/errors/__tests__/route.test.ts create mode 100644 app/api/errors/route.ts create mode 100644 components/error/error-reporting-root.tsx create mode 100644 lib/__tests__/jmap-error-reporting.test.ts create mode 100644 lib/__tests__/route-groups.test.ts create mode 100644 lib/admin/__tests__/reachable-hosts-observability.test.ts create mode 100644 lib/error-reporting/__tests__/aggregate.test.ts create mode 100644 lib/error-reporting/__tests__/client.test.ts create mode 100644 lib/error-reporting/__tests__/envelope.test.ts create mode 100644 lib/error-reporting/__tests__/lite.test.ts create mode 100644 lib/error-reporting/__tests__/scrub.test.ts create mode 100644 lib/error-reporting/__tests__/server.test.ts create mode 100644 lib/error-reporting/aggregate.ts create mode 100644 lib/error-reporting/client.ts create mode 100644 lib/error-reporting/debug-ids.ts create mode 100644 lib/error-reporting/envelope.ts create mode 100644 lib/error-reporting/fingerprint.ts create mode 100644 lib/error-reporting/limiter.ts create mode 100644 lib/error-reporting/lite.ts create mode 100644 lib/error-reporting/scrub.ts create mode 100644 lib/error-reporting/server.ts create mode 100644 lib/error-reporting/settings.ts create mode 100644 lib/error-reporting/stack.ts create mode 100644 lib/route-groups.ts diff --git a/app/(main)/[locale]/error.tsx b/app/(main)/[locale]/error.tsx index f25e2ad46..421c4be6c 100644 --- a/app/(main)/[locale]/error.tsx +++ b/app/(main)/[locale]/error.tsx @@ -3,6 +3,7 @@ import { useEffect } from "react"; import { useTranslations } from "next-intl"; import { ErrorState } from "@/components/error/error-fallbacks"; +import { captureException } from "@/lib/error-reporting/client"; /** * Route-level error boundary for locale pages. @@ -19,6 +20,7 @@ export default function LocaleError({ useEffect(() => { console.error("Route error:", error); + captureException(error, "boundary", false); }, [error]); return ( diff --git a/app/(main)/admin/[[...segments]]/page.tsx b/app/(main)/admin/[[...segments]]/page.tsx index 4d71528bc..0c28af644 100644 --- a/app/(main)/admin/[[...segments]]/page.tsx +++ b/app/(main)/admin/[[...segments]]/page.tsx @@ -20,6 +20,7 @@ import { MarketplaceDetailPage } from '../_pages/marketplace-detail'; import { ExtensionDetailPage } from '../_pages/extension-detail'; import { VersionPage } from '../_pages/version'; import { TelemetryPage } from '../_pages/telemetry'; +import { MonitoringPage } from '../_pages/monitoring'; import { LogsPage } from '../_pages/logs'; /** @@ -56,6 +57,7 @@ export default function AdminPage() { case 'marketplace': return detail ? : ; case 'version': return ; case 'telemetry': return ; + case 'monitoring': return ; case 'logs': return ; } } diff --git a/app/(main)/admin/_pages/monitoring.tsx b/app/(main)/admin/_pages/monitoring.tsx new file mode 100644 index 000000000..268740b69 --- /dev/null +++ b/app/(main)/admin/_pages/monitoring.tsx @@ -0,0 +1,175 @@ +'use client'; + +import { useCallback, useEffect, useState } from 'react'; +import { Loader2 } from '@/components/icons'; +import { apiFetch } from '@/lib/browser-navigation'; +import { adminPageDef } from '@/lib/admin/admin-nav'; +import { useAdminUnsavedChanges } from '@/stores/admin-unsaved-store'; +import { SettingsSection, SectionNav } from '@/components/settings/settings-section'; +import { AdminPageHeader } from '@/components/admin/page-header'; +import { AdminSaveBar } from '@/components/admin/save-bar'; +import { withEdit } from '@/lib/admin/edits'; +import { useAdminFeedback } from '@/components/admin/feedback'; +import { ConfigTextItem, ConfigNote } from '@/components/admin/config-items'; + +interface ConfigEntry { + value?: unknown; + source: 'admin' | 'env' | 'default'; +} + +interface ErrorEntry { + fingerprint: string; + name: string; + code?: string; + version: string; + runtime: 'browser' | 'server'; + count: number; + frames: Array<{ path: string; line?: number; column?: number }>; +} + +interface MonitoringState { + collecting: boolean; + day: string; + errorsToday: { total: number; top: ErrorEntry[] }; +} + +const PAGE = adminPageDef('monitoring'); +const KEYS = ['errorReportingDsn', 'errorReportingSampleRate', 'errorReportingEnvironment'] as const; +const DOCS = 'https://bulwarkmail.org/docs/admin/monitoring'; + +export function MonitoringPage() { + const [config, setConfig] = useState>({}); + const [state, setState] = useState(null); + const [edits, setEdits] = useState>({}); + const [loading, setLoading] = useState(true); + const [saving, setSaving] = useState(false); + const setMessage = useAdminFeedback(); + + const load = useCallback(async () => { + const [configRes, stateRes] = await Promise.all([apiFetch('/api/admin/config'), apiFetch('/api/admin/monitoring')]); + if (configRes.ok) setConfig(await configRes.json()); + if (stateRes.ok) setState(await stateRes.json()); + setLoading(false); + }, []); + + useEffect(() => { void load(); }, [load]); + + function handleChange(key: string, value: unknown) { + setEdits((prev) => withEdit(prev, key, value, config[key]?.value)); + setMessage(null); + } + + function currentValue(key: string): string { + const value = key in edits ? edits[key] : config[key]?.value; + return typeof value === 'string' ? value : value === undefined || value === null ? '' : String(value); + } + + async function handleSave() { + if (Object.keys(edits).length === 0) return; + setSaving(true); + setMessage(null); + const res = await apiFetch('/api/admin/config', { + method: 'PATCH', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(edits), + }); + if (res.ok) { + setMessage({ type: 'success', text: 'Saved. Browsers pick it up on their next page load.' }); + setEdits({}); + await load(); + } else { + const data = await res.json().catch(() => ({})); + setMessage({ type: 'error', text: data.error || 'Failed to save' }); + } + setSaving(false); + } + + async function handleRevert(key: string) { + const res = await apiFetch('/api/admin/config', { + method: 'DELETE', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ key }), + }); + if (res.ok) { + setEdits((prev) => { + const next = { ...prev }; + delete next[key]; + return next; + }); + await load(); + } + } + + const hasEdits = KEYS.some((key) => key in edits); + useAdminUnsavedChanges('monitoring', hasEdits); + + if (loading) { + return ( +
+ +
+ Loading +
+
+ ); + } + + const today = state?.errorsToday; + + return ( +
+ +
+ + + + + Send errors from browsers and from this server to your own error tracker: Bugsink, GlitchTip or + Sentry. Off while the address is empty. Message text, addresses, file names, tokens and URLs are + removed before anything is sent. What is sent + + + + + + + + {!state?.collecting ? ( + + Not counting errors. Counting starts when an error reporting address is set, the metrics + endpoint is on, or error counts are sent with the usage stats. + + ) : !today || today.total === 0 ? ( +

No errors so far today (UTC).

+ ) : ( +
+

{today.total} errors today (UTC), most frequent first.

+
    + {today.top.map((entry) => ( +
  • +
    +
    {entry.name}{entry.code ? `: ${entry.code}` : ''}
    +
    + {entry.frames[0] ? `${entry.frames[0].path}:${entry.frames[0].line ?? ''}:${entry.frames[0].column ?? ''}` : 'no stack'} + {' · '}{entry.runtime}{' · '}{entry.version} +
    +
    + {entry.count} +
    • + ))} +
+
+ )} + +
+ + setEdits({})} + note="Browsers pick it up on their next page load" + /> +
+ ); +} diff --git a/app/(main)/layout.tsx b/app/(main)/layout.tsx index 96f57ab7c..208656678 100644 --- a/app/(main)/layout.tsx +++ b/app/(main)/layout.tsx @@ -5,6 +5,7 @@ import { headers } from "next/headers"; import { getLocale, getTranslations } from "next-intl/server"; import { ServiceWorkerRegistration } from "@/components/service-worker-registration"; import { FaviconBadge } from "@/components/favicon-badge"; +import { ErrorReportingRoot } from "@/components/error/error-reporting-root"; import { ThemeColorSync } from "@/components/theme-color-sync"; import { IosViewportHeal } from "@/components/ios-viewport-heal"; import { configManager } from "@/lib/admin/config-manager"; @@ -328,6 +329,7 @@ export default async function RootLayout({ {!themeColorConfigured && } + {children} diff --git a/app/api/admin/monitoring/route.ts b/app/api/admin/monitoring/route.ts new file mode 100644 index 000000000..a6f71b387 --- /dev/null +++ b/app/api/admin/monitoring/route.ts @@ -0,0 +1,30 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { requireAdminAuth } from '@/lib/admin/session'; +import { logger } from '@/lib/logger'; +import { readErrors, summarizeDay, utcDay } from '@/lib/error-reporting/aggregate'; +import { shouldCollectErrors } from '@/lib/error-reporting/settings'; +import { storedDataUnavailableResponse } from '@/lib/stored-data-errors'; + +export const dynamic = 'force-dynamic'; + +/** + * GET /api/admin/monitoring: what System > Monitoring shows besides the + * config keys (those come from /api/admin/config): whether errors are being + * counted and today's counts. + */ +export async function GET(request: NextRequest) { + try { + const auth = await requireAdminAuth(request); + if ('error' in auth) return auth.error; + + const collecting = await shouldCollectErrors(); + const today = utcDay(); + const errorsToday = summarizeDay(await readErrors(), today); + return NextResponse.json({ collecting, day: today, errorsToday }, { headers: { 'Cache-Control': 'no-store' } }); + } catch (err) { + logger.error('monitoring GET error', { error: err instanceof Error ? err.message : 'unknown' }); + const unavailable = storedDataUnavailableResponse(err); + if (unavailable) return unavailable; + return NextResponse.json({ error: 'Internal server error' }, { status: 500 }); + } +} diff --git a/app/api/config/route.ts b/app/api/config/route.ts index 6e8208834..de9521a46 100644 --- a/app/api/config/route.ts +++ b/app/api/config/route.ts @@ -12,6 +12,7 @@ import { type BrandingOverrideKey, } from '@/lib/admin/domain-branding'; import { isNoThirdPartyTrafficOn } from '@/lib/admin/third-party-mode'; +import { browserErrorSettings } from '@/lib/error-reporting/settings'; /** * Runtime configuration endpoint @@ -109,6 +110,9 @@ export async function GET(request: NextRequest) { signLanguageUrl: httpUrlOrEmpty(branded('signLanguageUrl', '')), // Browser-side switches of lib/outgoing-hosts.ts read this (#903). noThirdPartyTraffic: await isNoThirdPartyTrafficOn(), + // Error reporting (lib/error-reporting/client.ts): off unless a DSN is + // set or this server counts errors. The DSN is public by design. + errorReporting: await browserErrorSettings(), }, { // Runtime settings can change without new asset URLs. Prevent stale diff --git a/app/api/errors/__tests__/route.test.ts b/app/api/errors/__tests__/route.test.ts new file mode 100644 index 000000000..b6bddfa02 --- /dev/null +++ b/app/api/errors/__tests__/route.test.ts @@ -0,0 +1,89 @@ +// @vitest-environment node +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { NextRequest } from 'next/server'; + +const config: Record = {}; +vi.mock('@/lib/admin/config-manager', () => ({ + configManager: { + ensureLoaded: async () => {}, + get: (key: string, fallback: unknown) => (key in config ? config[key] : fallback), + }, +})); + +import { POST } from '../route'; +import { flushErrors, readErrors, resetAggregateForTests, utcDay } from '@/lib/error-reporting/aggregate'; + +const DSN = 'https://6f0b9c79ed3a4315b76595bc2536ac40@bugsink.example.org/3'; +const SUBJECT = 'Merger plans for Acme Q3'; + +let dir: string; +let ip = 0; + +function post(body: unknown): NextRequest { + ip += 1; + return new NextRequest('http://localhost/api/errors', { + method: 'POST', + headers: { 'content-type': 'application/json', 'x-forwarded-for': `203.0.113.${ip}` }, + body: typeof body === 'string' ? body : JSON.stringify(body), + }); +} + +const report = { + name: 'TypeError', + message: `Cannot open "${SUBJECT}"`, + frames: [{ path: `https://mail.example.org/_next/static/chunks/1.js?subject=${encodeURIComponent(SUBJECT)}`, line: 2, column: 30, fn: 'Ab' }], + version: '1.20.0', +}; + +beforeEach(() => { + dir = mkdtempSync(path.join(tmpdir(), 'bulwark-errors-route-')); + process.env.TELEMETRY_DATA_DIR = dir; + for (const key of Object.keys(config)) delete config[key]; + resetAggregateForTests(); +}); + +afterEach(() => { + resetAggregateForTests(); + delete process.env.TELEMETRY_DATA_DIR; + rmSync(dir, { recursive: true, force: true }); +}); + +describe('POST /api/errors', () => { + it('keeps nothing while error collection is off', async () => { + const res = await POST(post(report)); + expect(res.status).toBe(204); + await flushErrors(); + expect(await readErrors()).toEqual({}); + }); + + it('counts a report, scrubbed again, without its message', async () => { + config.errorReportingDsn = DSN; + expect((await POST(post(report))).status).toBe(204); + await flushErrors(); + const stored = JSON.stringify(await readErrors()); + expect(stored).not.toContain('Merger'); + expect(stored).not.toContain('subject='); + const [entry] = Object.values((await readErrors())[utcDay()]); + expect(entry).toMatchObject({ name: 'TypeError', runtime: 'browser', version: '1.20.0', count: 1 }); + expect(entry.frames).toEqual([{ path: '/_next/static/chunks/1.js', line: 2, column: 30, fn: 'Ab' }]); + }); + + it('ignores a name or code that is not an identifier or fixed code', async () => { + config.errorReportingDsn = DSN; + await POST(post({ ...report, name: SUBJECT, code: SUBJECT })); + await flushErrors(); + const [entry] = Object.values((await readErrors())[utcDay()]); + expect(entry.name).toBe('Error'); + expect(entry.code).toBeUndefined(); + }); + + it('refuses bodies that are too large or not JSON objects', async () => { + config.errorReportingDsn = DSN; + expect((await POST(post('x'.repeat(17 * 1024)))).status).toBe(413); + expect((await POST(post('[1,2]'))).status).toBe(400); + expect((await POST(post('not json'))).status).toBe(400); + }); +}); diff --git a/app/api/errors/route.ts b/app/api/errors/route.ts new file mode 100644 index 000000000..7e283f260 --- /dev/null +++ b/app/api/errors/route.ts @@ -0,0 +1,70 @@ +import { NextRequest, NextResponse } from 'next/server'; +import { getClientIP } from '@/lib/admin/session'; +import { getStore } from '@/lib/shared-store'; +import { recordError } from '@/lib/error-reporting/aggregate'; +import { appVersion, isErrorCode } from '@/lib/error-reporting/envelope'; +import { scrubName } from '@/lib/error-reporting/scrub'; +import { shouldCollectErrors } from '@/lib/error-reporting/settings'; +import { sanitizeFrames } from '@/lib/error-reporting/stack'; + +export const dynamic = 'force-dynamic'; + +/** + * POST /api/errors: a browser of this instance reports an error it caught + * (lib/error-reporting/client.ts), for the per-day error counts. Only while + * collection is on (a DSN, the metrics endpoint or error-count consent); + * otherwise 204 and nothing is kept. + * + * Same-origin only (the proxy's origin gate), at most 16 KB, 30 a minute per + * address. Nothing a browser sends is trusted: every field is scrubbed again + * and the fingerprint is computed here. The DSN gets the browser's own + * report; this route only counts. + */ + +const MAX_BODY_BYTES = 16 * 1024; +const PER_MINUTE = 30; + +const NO_STORE = { 'Cache-Control': 'no-store' }; + +export async function POST(request: NextRequest) { + if (!(await shouldCollectErrors())) return new NextResponse(null, { status: 204, headers: NO_STORE }); + + const length = Number(request.headers.get('content-length') ?? '0'); + if (length > MAX_BODY_BYTES) return NextResponse.json({ error: 'too large' }, { status: 413, headers: NO_STORE }); + + const limit = await getStore().counters.reserve([ + { key: `error-reports:${getClientIP(request)}`, max: PER_MINUTE, windowMs: 60_000 }, + ]).catch(() => ({ ok: true })); + if (!limit.ok) return NextResponse.json({ error: 'rate limited' }, { status: 429, headers: NO_STORE }); + + let text: string; + try { + text = await request.text(); + } catch { + return NextResponse.json({ error: 'bad request' }, { status: 400, headers: NO_STORE }); + } + if (text.length > MAX_BODY_BYTES) return NextResponse.json({ error: 'too large' }, { status: 413, headers: NO_STORE }); + + let body: Record; + try { + const parsed = JSON.parse(text) as unknown; + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) throw new Error('not an object'); + body = parsed as Record; + } catch { + return NextResponse.json({ error: 'bad request' }, { status: 400, headers: NO_STORE }); + } + + const code = isErrorCode(body.code) ? body.code : undefined; + recordError( + { + name: scrubName(body.name), + frames: sanitizeFrames(body.frames), + ...(code ? { code } : {}), + }, + 'browser', + // The version the tab runs (a tab left open across an update runs the + // old one, and its frames point into that); the server's when absent. + typeof body.version === 'string' && /^\d+\.\d+\.\d+(?:[-+][\w.-]{1,40})?$/.test(body.version) ? body.version : appVersion(), + ); + return new NextResponse(null, { status: 204, headers: NO_STORE }); +} diff --git a/app/global-error.tsx b/app/global-error.tsx index b45178af5..3801a6475 100644 --- a/app/global-error.tsx +++ b/app/global-error.tsx @@ -2,6 +2,8 @@ import { useEffect } from "react"; import { AlertTriangle, RefreshCw } from "@/components/icons"; +import { fetchConfig } from "@/hooks/use-config"; +import { captureException } from "@/lib/error-reporting/client"; import "./globals.css"; /** @@ -25,6 +27,10 @@ export default function GlobalError({ }) { useEffect(() => { console.error("Global error:", error); + // Waits for the config when the app crashed before it arrived; sent only + // when error reporting is configured. + captureException(error, "global-error", false); + void fetchConfig().catch(() => undefined); }, [error]); return ( diff --git a/components/admin/admin-shell.tsx b/components/admin/admin-shell.tsx index 8a5074ba3..3008828f8 100644 --- a/components/admin/admin-shell.tsx +++ b/components/admin/admin-shell.tsx @@ -7,6 +7,7 @@ import { ArrowLeft, ArrowUpCircle, Activity, + HeartRateMonitor, Building2, ChevronRight, KeyRound, @@ -59,6 +60,7 @@ const PAGE_ICONS: Record = { marketplace: Store, version: ArrowUpCircle, telemetry: Activity, + monitoring: HeartRateMonitor, logs: ScrollText, }; diff --git a/components/error/error-boundary.tsx b/components/error/error-boundary.tsx index cd088d3b5..b53e476b0 100644 --- a/components/error/error-boundary.tsx +++ b/components/error/error-boundary.tsx @@ -3,6 +3,7 @@ import React, { Component, ReactNode } from "react"; import { useTranslations } from "next-intl"; import { debug } from "@/lib/debug"; +import { captureException } from "@/lib/error-reporting/client"; export interface FallbackProps { error: Error; @@ -43,6 +44,10 @@ class ErrorBoundaryCore extends Component< componentStack: errorInfo.componentStack, }); + // Off unless error reporting is configured (lib/error-reporting/client.ts). + // The component stack stays here: it names components, nothing else. + captureException(error, "boundary", false); + // Call optional error handler this.props.onError?.(error, errorInfo); } diff --git a/components/error/error-reporting-root.tsx b/components/error/error-reporting-root.tsx new file mode 100644 index 000000000..9f74dfd44 --- /dev/null +++ b/components/error/error-reporting-root.tsx @@ -0,0 +1,30 @@ +"use client"; + +import { useEffect } from "react"; +import { usePathname } from "next/navigation"; +import { fetchConfig } from "@/hooks/use-config"; +import { installGlobalHandlers, recordNavigation } from "@/lib/error-reporting/client"; + +/** + * Starts browser error reporting: window.onerror and unhandled rejections, + * and a breadcrumb per route change (the route's area only, never the URL). + * Whether anything is sent is decided by the config (/api/config, or Lite's + * policy.json); until it arrives, reports wait, and with reporting off they + * are dropped. Renders nothing. + */ +export function ErrorReportingRoot() { + const pathname = usePathname(); + + useEffect(() => { + installGlobalHandlers(); + // Cached after the first fetch; this only makes sure it happens on pages + // that don't read the config themselves (admin, setup). + void fetchConfig().catch(() => undefined); + }, []); + + useEffect(() => { + if (pathname) recordNavigation(pathname); + }, [pathname]); + + return null; +} diff --git a/components/icons/index.ts b/components/icons/index.ts index a110750dd..b91c959b7 100644 --- a/components/icons/index.ts +++ b/components/icons/index.ts @@ -122,6 +122,7 @@ export { IconH1 as Heading1, IconH2 as Heading2, IconHeart as Heart, + IconHeartRateMonitor as HeartRateMonitor, IconHelpCircle as HelpCircle, IconHighlight as Highlighter, IconHome as Home, diff --git a/hooks/use-config.ts b/hooks/use-config.ts index 77329854a..d54a208d7 100644 --- a/hooks/use-config.ts +++ b/hooks/use-config.ts @@ -8,6 +8,7 @@ import { IS_LITE, IS_LITE_STALWART, LITE_CONFIG_PATH, withLiteBuildId } from '@/ import { applyLiteConfig, liteStalwartDefaults } from '@/lib/lite-config'; import { isConfigData } from '@/lib/config-validation'; import { setNoThirdPartyTraffic } from '@/lib/third-party-mode'; +import { configureErrorReporting, type ClientSettings } from '@/lib/error-reporting/client'; export interface ConfigData { appName: string; @@ -59,6 +60,8 @@ export interface ConfigData { noThirdPartyTraffic: boolean; /** Lite only: whether sign-out revokes the refresh token (#1388). The server build decides in its token route. */ oauthRevokeOnSignOut?: boolean; + /** Server build only; Lite reads it from policy.json (stores/policy-store.ts). */ + errorReporting?: ClientSettings; } interface AppConfig extends ConfigData { @@ -144,6 +147,8 @@ export async function fetchConfig(): Promise { .then((data) => { configCache = data; setNoThirdPartyTraffic(data.noThirdPartyTraffic === true); + // Lite configures error reporting from policy.json instead. + if (!IS_LITE) configureErrorReporting(data.errorReporting ?? null); // Fetch admin policy alongside config (non-blocking) usePolicyStore.getState().fetchPolicy(); return data; diff --git a/instrumentation.node.ts b/instrumentation.node.ts index 605628441..94f50b00b 100644 --- a/instrumentation.node.ts +++ b/instrumentation.node.ts @@ -9,12 +9,13 @@ import { bootState, getStore, sharedStoreUrl } from "./lib/shared-store"; import { bootSharedMode } from "./lib/shared-store/boot"; import { importLocalData } from "./lib/shared-store/import"; import { exportSharedData } from "./lib/shared-store/export"; +import { logger, isJsonLogFormat } from "./lib/logger"; const pkg = JSON.parse( readFileSync(`${process.cwd()}/package.json`, "utf-8") ); const current: string = pkg.version ?? "0.0.0"; -console.info(`Bulwark Webmail v${current}`); +logger.info(`Bulwark Webmail v${current}`, { version: current }); // With REDIS_URL every replica shares one store (#1066). Without it, // everything below runs exactly as it always has. @@ -36,12 +37,12 @@ async function exportAndExit(): Promise { await store.ping(2000); break; } catch { - console.warn("Export: waiting for the shared store"); + logger.warn("Export: waiting for the shared store"); await new Promise((resolve) => setTimeout(resolve, 2000)); } } const written = await exportSharedData(); - console.info("Export from the shared store done; start without REDIS_URL on this volume now.", written); + logger.info("Export from the shared store done; start without REDIS_URL on this volume now.", { written }); process.exit(0); } @@ -57,18 +58,23 @@ migrateLegacyAdminLayout() .then(() => migratePolicyUnifiedMailbox()) .then(() => (exportOnly ? exportAndExit() : shared ? bootShared() : loadAdmin())) .then(async () => { - console.info("Admin dashboard initialized"); + logger.info("Admin dashboard initialized"); + // Errors routes catch and log, and errors Next reports (instrumentation.ts), + // go to the error tracker when one is configured. Off otherwise. + // https://bulwarkmail.org/docs/admin/monitoring + const { installServerErrorReporting } = await import("./lib/error-reporting/server"); + await installServerErrorReporting(); // With a shared store, boot already made or loaded the secret. if (!shared) { try { if (await ensureSessionSecret()) { - console.info( + logger.info( "Generated a session secret in the admin state dir. Keep that dir on a " + "persistent volume, or set SESSION_SECRET, so restarts don't sign everyone out.", ); } } catch (err) { - console.warn("Failed to generate a session secret:", err instanceof Error ? err.message : err); + logger.warn("Failed to generate a session secret", { error: err instanceof Error ? err.message : String(err) }); } } // If we're in bootstrap state (no JMAP_SERVER_URL env and no @@ -79,20 +85,22 @@ migrateLegacyAdminLayout() try { const token = await ensureSetupToken(); const port = process.env.PORT || "3000"; - console.info(""); - console.info("=============================================================="); - console.info(" SETUP REQUIRED"); - console.info(` Token: ${token}`); // In the fragment, so the token stays out of access logs and Referer. - console.info(` Open: http://:${port}/setup#token=${token}`); - console.info(" Token expires in 1 hour. Restart the container to reissue."); - console.info("=============================================================="); - console.info(""); + const open = `http://:${port}/setup#token=${token}`; + if (isJsonLogFormat()) { + logger.info("Setup required", { token, open, expiresIn: "1h" }); + } else { + for (const line of [ + "==============================================================", + " SETUP REQUIRED", + ` Token: ${token}`, + ` Open: ${open}`, + " Token expires in 1 hour. Restart the container to reissue.", + "==============================================================", + ]) logger.info(line); + } } catch (err) { - console.warn( - "Failed to issue setup token:", - err instanceof Error ? err.message : err, - ); + logger.warn("Failed to issue setup token", { error: err instanceof Error ? err.message : String(err) }); } } }) @@ -116,5 +124,5 @@ migrateLegacyAdminLayout() ]); }) .catch((err) => { - console.warn("Admin dashboard init skipped:", err instanceof Error ? err.message : err); + logger.warn("Admin dashboard init skipped", { error: err instanceof Error ? err.message : String(err) }); }); diff --git a/instrumentation.ts b/instrumentation.ts index 41630756b..a11419dd4 100644 --- a/instrumentation.ts +++ b/instrumentation.ts @@ -1,5 +1,19 @@ +import type { Instrumentation } from "next"; + export async function register() { if (process.env.NEXT_RUNTIME === "nodejs") { await import("./instrumentation.node"); } } + +/** + * Errors Next caught while rendering a page or running a route handler, + * server action or the proxy. Counted, and sent to the error tracker when + * one is configured (lib/error-reporting/server.ts); nothing happens while + * error reporting is off. https://bulwarkmail.org/docs/admin/monitoring + */ +export const onRequestError: Instrumentation.onRequestError = async (error, _request, context) => { + if (process.env.NEXT_RUNTIME !== "nodejs") return; + const { captureRequestError } = await import("./lib/error-reporting/server"); + await captureRequestError(error, context.routePath); +}; diff --git a/lib/__tests__/config-validation.test.ts b/lib/__tests__/config-validation.test.ts index b2737ee7e..626e575ce 100644 --- a/lib/__tests__/config-validation.test.ts +++ b/lib/__tests__/config-validation.test.ts @@ -107,3 +107,18 @@ describe('server configuration validation', () => { expect(isConfigData({ ...createConfig(), jmapServers: [entry] })).toBe(false); }); }); + +describe('error reporting settings in the server configuration', () => { + const settings = { dsn: '', sampleRate: 1, environment: 'production', collect: false }; + + it('are optional, for Lite and servers before they existed', () => { + expect(isConfigData({ ...createConfig(), errorReporting: settings })).toBe(true); + expect(isConfigData(createConfig())).toBe(true); + }); + + it('are refused when malformed', () => { + expect(isConfigData({ ...createConfig(), errorReporting: { ...settings, sampleRate: '1' } })).toBe(false); + expect(isConfigData({ ...createConfig(), errorReporting: { ...settings, collect: 'yes' } })).toBe(false); + expect(isConfigData({ ...createConfig(), errorReporting: 'https://key@x.example/1' })).toBe(false); + }); +}); diff --git a/lib/__tests__/jmap-error-reporting.test.ts b/lib/__tests__/jmap-error-reporting.test.ts new file mode 100644 index 000000000..6749e7083 --- /dev/null +++ b/lib/__tests__/jmap-error-reporting.test.ts @@ -0,0 +1,58 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const reported = vi.hoisted(() => ({ calls: [] as string[][], failures: [] as unknown[] })); +vi.mock('@/lib/error-reporting/client', () => ({ + recordJmapCall: (methods: string[]) => { reported.calls.push(methods); }, + reportJmapFailure: (failure: unknown) => { reported.failures.push(failure); }, +})); + +import { JMAPClient } from '../jmap/client'; + +type Call = [string, Record, string]; +const SUBJECT = 'Merger plans for Acme Q3'; + +function clientAnswering(response: Response) { + const client = new JMAPClient('https://jmap.example.com', 'user@example.com', 'pass'); + Object.assign(client, { apiUrl: 'https://jmap.example.com/jmap/', accountId: 'me' }); + vi.spyOn(client as unknown as { authenticatedFetch: () => Promise }, 'authenticatedFetch').mockResolvedValue(response); + const request = (calls: Call[]) => (client as unknown as { request: (c: Call[]) => Promise }).request(calls); + return request; +} + +beforeEach(() => { + reported.calls = []; + reported.failures = []; +}); + +describe('JMAP requests and error reporting', () => { + it('keeps the method names of each request as a breadcrumb', async () => { + const request = clientAnswering(new Response(JSON.stringify({ methodResponses: [] }))); + await request([['Email/query', { filter: { subject: SUBJECT } }, '0'], ['Email/get', {}, '1']]); + expect(reported.calls).toEqual([['Email/query', 'Email/get']]); + expect(reported.failures).toEqual([]); + }); + + it('reports a failed request by its status only', async () => { + const request = clientAnswering(new Response(`{"detail":"${SUBJECT}"}`, { status: 503 })); + await expect(request([['Email/get', {}, '0']])).rejects.toThrow(); + expect(reported.failures).toEqual([{ status: 503 }]); + }); + + it('does not report an expired token or the concurrency limit', async () => { + const request = clientAnswering(new Response('', { status: 401 })); + await expect(request([['Email/get', {}, '0']])).rejects.toThrow(); + expect(reported.failures).toEqual([]); + }); + + it('reports method errors by method and type, but not the ones that only mean "sync again"', async () => { + const request = clientAnswering(new Response(JSON.stringify({ + methodResponses: [ + ['error', { type: 'overQuota', description: SUBJECT }, 'a'], + ['error', { type: 'cannotCalculateChanges' }, 'b'], + ['Mailbox/get', { list: [] }, 'c'], + ], + }))); + await request([['Email/set', {}, 'a'], ['Email/changes', {}, 'b'], ['Mailbox/get', {}, 'c']]); + expect(reported.failures).toEqual([{ method: 'Email/set', type: 'overQuota' }]); + }); +}); diff --git a/lib/__tests__/route-groups.test.ts b/lib/__tests__/route-groups.test.ts new file mode 100644 index 000000000..fa54b744a --- /dev/null +++ b/lib/__tests__/route-groups.test.ts @@ -0,0 +1,21 @@ +// @vitest-environment node +import { readdirSync, statSync } from 'node:fs'; +import path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { API_GROUPS, PAGE_AREAS } from '@/lib/route-groups'; + +const ROOT = path.resolve(__dirname, '..', '..'); +const dirs = (rel: string) => readdirSync(path.join(ROOT, rel)).filter((name) => statSync(path.join(ROOT, rel, name)).isDirectory()); + +// The groups are a fixed list so metric labels and report routes stay +// bounded; a new API folder or page area has to be added to it. +describe('route groups', () => { + it('list every API folder', () => { + expect([...API_GROUPS].sort()).toEqual(dirs('app/api').filter((d) => d !== '__tests__').sort()); + }); + + it('list every page area', () => { + const areas = dirs('app/(main)/[locale]').filter((d) => !d.startsWith('[') && !d.startsWith('(')); + for (const area of areas) expect(PAGE_AREAS).toContain(area); + }); +}); diff --git a/lib/admin/__tests__/reachable-hosts-observability.test.ts b/lib/admin/__tests__/reachable-hosts-observability.test.ts new file mode 100644 index 000000000..dd4ca025a --- /dev/null +++ b/lib/admin/__tests__/reachable-hosts-observability.test.ts @@ -0,0 +1,21 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest'; +import { listReachableHosts, noThirdPartyCspSources } from '@/lib/admin/reachable-hosts'; + +const policy = { pushRelays: [], pushRelayUrl: '', defaultSidebarApps: [], features: { sidebarAppsEnabled: false, calendarEnabled: false } }; + +function hosts(config: Record, env: Record = {}) { + return listReachableHosts({ get: (key) => config[key], policy, env, buildRelayUrl: '' }); +} + +// No third-party traffic (#903) still allows the hosts the admin set: the +// error tracker is one of them, and the page's CSP must let browsers reach it. +describe('error reporting in the no-third-party list', () => { + it('is listed, without the key, only when a DSN is set', () => { + expect(hosts({}).filter((h) => h.kind === 'Error reporting')).toEqual([]); + const listed = hosts({ errorReportingDsn: 'https://6f0b9c79ed3a4315b76595bc2536ac40@errors.example.org/3' }) + .filter((h) => h.kind === 'Error reporting'); + expect(listed).toEqual([{ kind: 'Error reporting', host: 'errors.example.org', origin: 'https://errors.example.org', setIn: 'ERROR_REPORTING_DSN', side: 'both' }]); + expect(noThirdPartyCspSources(listed, false).connect).toEqual(['https://errors.example.org']); + }); +}); diff --git a/lib/admin/admin-nav.ts b/lib/admin/admin-nav.ts index 5b336e453..8d3d938b5 100644 --- a/lib/admin/admin-nav.ts +++ b/lib/admin/admin-nav.ts @@ -27,6 +27,7 @@ export const ADMIN_PAGE_IDS = [ 'marketplace', 'version', 'telemetry', + 'monitoring', 'logs', ] as const; @@ -243,6 +244,17 @@ export const ADMIN_PAGES: readonly AdminPageDef[] = [ { id: 'payload', title: 'Payload preview', keywords: 'send now json' }, ], }, + { + id: 'monitoring', + label: 'Monitoring', + description: 'Error reports, metrics, health checks and tracing for your own tools', + group: 'system', + keywords: 'monitoring errors sentry bugsink glitchtip dsn prometheus metrics health probes tracing opentelemetry otel', + sections: [ + { id: 'errors', title: 'Error reporting', keywords: 'dsn sentry bugsink glitchtip sample rate environment' }, + { id: 'errors-today', title: 'Errors today', keywords: 'count fingerprint top' }, + ], + }, { id: 'logs', label: 'Audit log', diff --git a/lib/admin/audit.ts b/lib/admin/audit.ts index 0c4e2b6ed..572def9cd 100644 --- a/lib/admin/audit.ts +++ b/lib/admin/audit.ts @@ -89,6 +89,10 @@ export async function auditLog(action: string, detail: Record, ip, }; + // Also in the server log, so LOG_FORMAT=json shippers see admin actions. + // The audit file below stays the record the admin panel reads. + logger.info(`audit: ${action}`, { audit: entry }); + try { // One line per entry; the store rotates the local file (audit.log.3 → // deleted, .2 → .3, .1 → .2, audit.log → .1) once it passes 10 MB. diff --git a/lib/admin/outgoing-hosts-registry.ts b/lib/admin/outgoing-hosts-registry.ts index 24fa214d7..178ae3401 100644 --- a/lib/admin/outgoing-hosts-registry.ts +++ b/lib/admin/outgoing-hosts-registry.ts @@ -157,8 +157,9 @@ export const OUTGOING_HOSTS: readonly OutgoingHost[] = [ * has to be looked at again. */ export const NON_REQUEST_HOSTS: readonly { host: string; files: readonly string[]; why: string }[] = [ - { host: 'bulwarkmail.org', why: 'docs links, comments and the contact URL in User-Agent headers', files: ['app/(main)/admin/_pages/telemetry.tsx', 'app/api/fetch-ical/route.ts', 'app/api/translate/route.ts','components/admin/org/sections/recovery-section.tsx', 'instrumentation.node.ts', 'lib/telemetry/state.ts', 'lib/telemetry/types.ts'] }, + { host: 'bulwarkmail.org', why: 'docs links, comments and the contact URL in User-Agent headers', files: ['app/(main)/admin/_pages/telemetry.tsx', 'app/api/fetch-ical/route.ts', 'app/api/translate/route.ts','components/admin/org/sections/recovery-section.tsx', 'instrumentation.node.ts', 'lib/telemetry/state.ts', 'lib/telemetry/types.ts', 'app/(main)/admin/_pages/monitoring.tsx', 'lib/admin/types.ts', 'lib/error-reporting/client.ts', 'lib/error-reporting/lite.ts', 'lib/error-reporting/scrub.ts', 'instrumentation.ts'] }, { host: 'bulwarkmail.com', why: 'JMAP capability URI', files: ['lib/jmap/client.ts'] }, + { host: 'develop.sentry.dev', why: 'link to the envelope format in a comment', files: ['lib/error-reporting/envelope.ts'] }, { host: 'extensions.bulwarkmail.org', why: 'signed statement type URIs', files: ['lib/admin/directory-keys.ts'] }, { host: 'www.w3.org', why: 'XML namespaces', files: ['components/ui/flag-icons.tsx', 'lib/favicon-badge.ts'] }, { host: 'creativecommons.org', why: 'licence metadata inside an SVG', files: ['components/ui/flag-icons.tsx'] }, diff --git a/lib/admin/reachable-hosts.ts b/lib/admin/reachable-hosts.ts index cfae6020d..8d848815b 100644 --- a/lib/admin/reachable-hosts.ts +++ b/lib/admin/reachable-hosts.ts @@ -91,6 +91,8 @@ export function listReachableHosts({ get, policy, env, buildRelayUrl = DEFAULT_R for (const app of policy.defaultSidebarApps ?? []) add('Sidebar app', app?.url, `User policy > Sidebar apps (${app?.name ?? ''})`, 'browser'); add('Document editor (WOPI)', get('wopiClientUrl'), 'WOPI_CLIENT_URL', 'both'); add('Translation (LibreTranslate)', env.LIBRETRANSLATE_URL, 'LIBRETRANSLATE_URL', 'server'); + // Only when an admin set it; browsers and this server send error reports there. + add('Error reporting', get('errorReportingDsn'), 'ERROR_REPORTING_DSN', 'both'); for (const key of BRANDING_KEYS) add('Branding image', get(key), key, 'browser'); // User-directed: what users type in themselves stays allowed. diff --git a/lib/admin/types.ts b/lib/admin/types.ts index fb35d3954..521fb40dc 100644 --- a/lib/admin/types.ts +++ b/lib/admin/types.ts @@ -363,6 +363,12 @@ export const CONFIG_ENV_MAP: Record value === undefined || typeof value === 'boolean', + // Absent from Lite's config and from servers before 1.20. + errorReporting: (value): value is ConfigData['errorReporting'] => value === undefined || ( + isRecord(value) + && typeof value.dsn === 'string' + && typeof value.sampleRate === 'number' + && typeof value.environment === 'string' + && typeof value.collect === 'boolean' + ), } satisfies Validators; /** diff --git a/lib/error-reporting/__tests__/aggregate.test.ts b/lib/error-reporting/__tests__/aggregate.test.ts new file mode 100644 index 000000000..a9dfaf577 --- /dev/null +++ b/lib/error-reporting/__tests__/aggregate.test.ts @@ -0,0 +1,34 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest'; +import { bound, KEEP_DAYS, MAX_PER_DAY, merge, summarizeDay, type Aggregate, type AggregateEntry } from '../aggregate'; + +function entry(count: number): AggregateEntry { + return { name: 'Error', frames: [], version: '1.20.0', runtime: 'browser', count }; +} + +describe('error aggregate', () => { + it('adds counts from several replicas instead of overwriting them', () => { + const a: Aggregate = { '2026-11-01': { f1: entry(2) } }; + const b: Aggregate = { '2026-11-01': { f1: entry(3), f2: entry(1) } }; + expect(merge(a, b)['2026-11-01']).toEqual({ f1: entry(5), f2: entry(1) }); + }); + + it('keeps the most frequent fingerprints per day and only the last days', () => { + const many: Record = {}; + for (let i = 0; i < MAX_PER_DAY + 50; i++) many[`f${i}`] = entry(i + 1); + const old = '2026-10-01'; + const bounded = bound({ '2026-11-01': many, [old]: { x: entry(1) } }, '2026-11-01'); + expect(Object.keys(bounded['2026-11-01'])).toHaveLength(MAX_PER_DAY); + expect(bounded['2026-11-01'].f0).toBeUndefined(); + expect(bounded['2026-11-01'][`f${MAX_PER_DAY + 49}`]).toBeDefined(); + expect(bounded[old]).toBeUndefined(); + const lastKept = new Date(Date.parse('2026-11-01T00:00:00Z') - (KEEP_DAYS - 1) * 86_400_000).toISOString().slice(0, 10); + expect(Object.keys(bound({ [lastKept]: { x: entry(1) } }, '2026-11-01'))).toEqual([lastKept]); + }); + + it('summarizes a day, most frequent first', () => { + const summary = summarizeDay({ '2026-11-01': { a: entry(1), b: entry(5) } }, '2026-11-01'); + expect(summary.total).toBe(6); + expect(summary.top.map((e) => e.fingerprint)).toEqual(['b', 'a']); + }); +}); diff --git a/lib/error-reporting/__tests__/client.test.ts b/lib/error-reporting/__tests__/client.test.ts new file mode 100644 index 000000000..e4fc2709e --- /dev/null +++ b/lib/error-reporting/__tests__/client.test.ts @@ -0,0 +1,120 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { + captureException, configureErrorReporting, recordJmapCall, recordNavigation, reportJmapFailure, + resetErrorReportingForTests, +} from '../client'; + +const DSN = 'https://6f0b9c79ed3a4315b76595bc2536ac40@bugsink.example.org/3'; +const SUBJECT = 'Merger plans for Acme Q3'; +const ADDRESS = 'ceo.private@acme-corp.example'; +const FOLDER = 'Acquisition Talks'; + +function errorWithStack(message: string): Error { + const error = new TypeError(message); + error.stack = `TypeError: ${message}\n at Ab (https://mail.example.org/_next/static/chunks/1.js:2:30)\n at https://mail.example.org/_next/static/chunks/2.js:4:5`; + return error; +} + +let fetchSpy: ReturnType; + +function calls(): Array<{ url: string; init: RequestInit }> { + return fetchSpy.mock.calls.map(([url, init]) => ({ url: String(url), init: init as RequestInit })); +} + +beforeEach(() => { + resetErrorReportingForTests(); + sessionStorage.clear(); + fetchSpy = vi.fn().mockResolvedValue(new Response(null, { status: 200 })); + vi.stubGlobal('fetch', fetchSpy); +}); + +afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); +}); + +describe('browser error reporting', () => { + it('sends nothing while reporting is off', () => { + configureErrorReporting(null); + captureException(errorWithStack('boom')); + configureErrorReporting({ dsn: '', sampleRate: 1, environment: 'production', collect: false }); + captureException(errorWithStack('boom again')); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('holds reports until the config arrives, then sends them', () => { + captureException(errorWithStack('early')); + expect(fetchSpy).not.toHaveBeenCalled(); + configureErrorReporting({ dsn: DSN, sampleRate: 1, environment: 'production', collect: false }); + expect(calls().map((c) => c.url)).toEqual(['https://bugsink.example.org/api/3/envelope/?sentry_key=6f0b9c79ed3a4315b76595bc2536ac40&sentry_version=7']); + }); + + it('sends to the DSN without cookies or referrer, and with no mail content anywhere', () => { + configureErrorReporting({ dsn: DSN, sampleRate: 1, environment: 'staging', collect: true }); + window.history.pushState({}, '', `/en/mail/folder/${encodeURIComponent(FOLDER)}/Mabc`); + recordNavigation(window.location.pathname); + recordJmapCall(['Email/query', 'Email/get']); + captureException(errorWithStack(`Failed to open "${SUBJECT}" from ${ADDRESS} at https://mail.example.org/download?name=${encodeURIComponent(SUBJECT)}`)); + + const [dsnCall, ownCall] = calls(); + expect(dsnCall.init.credentials).toBe('omit'); + expect(dsnCall.init.referrerPolicy).toBe('no-referrer'); + expect(ownCall.url).toBe('/api/errors'); + for (const { init } of [dsnCall, ownCall]) { + const body = String(init.body); + for (const planted of [SUBJECT, ADDRESS, FOLDER, encodeURIComponent(FOLDER), 'Merger', 'acme-corp', 'download?name']) { + expect(body).not.toContain(planted); + } + } + const event = JSON.parse(String(dsnCall.init.body).trim().split('\n')[2]); + expect(event.exception.values[0].value).toBe('Failed to open "…" from [email] at [url]'); + expect(event.environment).toBe('staging'); + expect(event.breadcrumbs.values.map((b: { message: string }) => b.message)).toEqual(['/mail/…', 'Email/query, Email/get']); + expect(event.tags.route).toBe('/mail/…'); + }); + + it('sends each fingerprint once per session', () => { + configureErrorReporting({ dsn: DSN, sampleRate: 1, environment: 'production', collect: false }); + captureException(errorWithStack('same')); + captureException(errorWithStack('same')); + expect(fetchSpy).toHaveBeenCalledTimes(1); + }); + + it('stops at ten reports a minute', () => { + configureErrorReporting({ dsn: DSN, sampleRate: 1, environment: 'production', collect: false }); + for (let i = 0; i < 15; i++) { + const error = new Error(`e${i}`); + error.stack = `Error\n at https://m.example/_next/static/chunks/${i}.js:1:1`; + captureException(error); + } + expect(fetchSpy).toHaveBeenCalledTimes(10); + }); + + it('applies the sample rate to the DSN only', () => { + vi.spyOn(Math, 'random').mockReturnValue(0.5); + configureErrorReporting({ dsn: DSN, sampleRate: 0.1, environment: 'production', collect: true }); + captureException(errorWithStack('sampled out')); + expect(calls().map((c) => c.url)).toEqual(['/api/errors']); + }); + + it('ignores errors that come only from extensions or other scripts', () => { + configureErrorReporting({ dsn: DSN, sampleRate: 1, environment: 'production', collect: false }); + const error = new Error('from an extension'); + error.stack = 'Error\n at chrome-extension://abc/content.js:1:1'; + captureException(error); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('reports a failed JMAP request by status or method and type, never the answer', () => { + configureErrorReporting({ dsn: DSN, sampleRate: 1, environment: 'production', collect: true }); + reportJmapFailure({ status: 503 }); + reportJmapFailure({ method: 'Email/set', type: 'overQuota' }); + reportJmapFailure({ method: SUBJECT, type: ADDRESS }); + const own = calls().filter((c) => c.url === '/api/errors').map((c) => JSON.parse(String(c.init.body))); + expect(own.map((r) => [r.name, r.code])).toEqual([ + ['JmapRequestError', 'http:503'], + ['JmapMethodError', 'Email/set: overQuota'], + ['JmapMethodError', 'Unknown/method: unknown'], + ]); + }); +}); diff --git a/lib/error-reporting/__tests__/envelope.test.ts b/lib/error-reporting/__tests__/envelope.test.ts new file mode 100644 index 000000000..41d45af57 --- /dev/null +++ b/lib/error-reporting/__tests__/envelope.test.ts @@ -0,0 +1,118 @@ +// @vitest-environment node +import { afterEach, describe, expect, it } from 'vitest'; +import { buildEnvelope, buildEvent, envelopeUrl, isErrorCode, parseDsn } from '../envelope'; +import { debugIdsByPath, resetDebugIdCache } from '../debug-ids'; +import { fingerprint } from '../fingerprint'; +import { parseStack } from '../stack'; + +const DSN = 'https://6f0b9c79ed3a4315b76595bc2536ac40@bugsink.example.org/3'; + +describe('parseDsn', () => { + it('reads key, origin and project', () => { + expect(parseDsn(DSN)).toEqual({ origin: 'https://bugsink.example.org', pathPrefix: '', publicKey: '6f0b9c79ed3a4315b76595bc2536ac40', projectId: '3' }); + expect(parseDsn('https://6f0b9c79ed3a4315b76595bc2536ac40@host.example:8443/sentry/12')?.pathPrefix).toBe('/sentry'); + }); + + it('refuses what is not a DSN', () => { + for (const bad of ['', 'not a url', 'https://bugsink.example.org/3', 'ftp://6f0b9c79ed3a4315b76595bc2536ac40@x.example/3', + 'https://6f0b9c79ed3a4315b76595bc2536ac40@x.example/abc', 'https://6f0b9c79ed3a4315b76595bc2536ac40:secret@x.example/3']) { + expect(parseDsn(bad), bad).toBeNull(); + } + }); + + it('puts the key in the query, so a browser sends no custom header', () => { + expect(envelopeUrl(parseDsn(DSN)!)).toBe('https://bugsink.example.org/api/3/envelope/?sentry_key=6f0b9c79ed3a4315b76595bc2536ac40&sentry_version=7'); + }); +}); + +describe('stack parsing', () => { + it('reads V8, Firefox and Safari stacks, top frame first', () => { + const chrome = 'TypeError: x\n at Ab (https://mail.example.org/_next/static/chunks/1.js:2:30)\n at async https://mail.example.org/_next/static/chunks/2.js:4:5'; + const firefox = 'Ab@https://mail.example.org/_next/static/chunks/1.js:2:30\n@https://mail.example.org/_next/static/chunks/2.js:4:5'; + for (const stack of [chrome, firefox]) { + expect(parseStack(stack)).toEqual([ + { path: '/_next/static/chunks/1.js', line: 2, column: 30, fn: 'Ab' }, + { path: '/_next/static/chunks/2.js', line: 4, column: 5, fn: undefined }, + ]); + } + expect(parseStack(' at Object.handler (/app/.next/server/chunks/77.js:1:900)')[0]).toEqual({ path: '/.next/server/chunks/77.js', line: 1, column: 900, fn: 'Object.handler' }); + }); +}); + +describe('debug ids', () => { + afterEach(() => { + delete (globalThis as Record)._debugIds; + delete (globalThis as Record)._sentryDebugIds; + resetDebugIdCache(); + }); + + it('maps each registered chunk to its id, from Turbopack and sentry-cli alike', () => { + (globalThis as Record)._debugIds = { + 'Error\n at https://mail.example.org/_next/static/chunks/1.js:1:10': 'bc67f7db-7f3b-7be2-6f94-966cd7825cc2', + }; + (globalThis as Record)._sentryDebugIds = { + 'Error\n at https://mail.example.org/_next/static/chunks/2.js:1:10': 'c40c4654-00a8-5efe-b3a3-42818d0ccc15', + }; + expect(Object.fromEntries(debugIdsByPath())).toEqual({ + '/_next/static/chunks/1.js': 'bc67f7db-7f3b-7be2-6f94-966cd7825cc2', + '/_next/static/chunks/2.js': 'c40c4654-00a8-5efe-b3a3-42818d0ccc15', + }); + }); +}); + +describe('buildEvent', () => { + const frames = [ + { path: '/_next/static/chunks/1.js', line: 2, column: 30, fn: 'Ab' }, + { path: '/_next/static/chunks/2.js', line: 4, column: 5 }, + ]; + const event = buildEvent( + { name: 'TypeError', message: 'x is undefined', frames, handled: false, mechanism: 'onerror', route: '/mail/…' }, + { + eventId: 'a'.repeat(32), timestamp: 1, release: '1.20.0+abc1234', environment: 'production', platform: 'javascript', + fingerprint: 'ffff0000ffff0000', debugIds: new Map([['/_next/static/chunks/1.js', 'bc67f7db-7f3b-7be2-6f94-966cd7825cc2']]), + }, + ); + + it('names a source map image for each frame file with a debug id, under the frame path', () => { + const exception = (event.exception as { values: Array<{ stacktrace: { frames: Array<{ abs_path: string; lineno: number }> } }> }).values[0]; + // Oldest first: the throwing frame is last. + expect(exception.stacktrace.frames.map((f) => f.abs_path)).toEqual(['/_next/static/chunks/2.js', '/_next/static/chunks/1.js']); + expect(event.debug_meta).toEqual({ images: [{ type: 'sourcemap', code_file: '/_next/static/chunks/1.js', debug_id: 'bc67f7db-7f3b-7be2-6f94-966cd7825cc2' }] }); + }); + + it('groups by Bulwark\'s fingerprint and carries the release', () => { + expect(event.fingerprint).toEqual(['ffff0000ffff0000']); + expect(event.release).toBe('1.20.0+abc1234'); + expect(event.tags).toEqual({ route: '/mail/…' }); + }); + + it('wraps it in a three-line envelope', () => { + const lines = buildEnvelope(event).trim().split('\n'); + expect(lines).toHaveLength(3); + expect(JSON.parse(lines[1])).toEqual({ type: 'event', content_type: 'application/json' }); + expect(JSON.parse(lines[2]).event_id).toBe('a'.repeat(32)); + }); +}); + +describe('fingerprint', () => { + const frames = [{ path: '/_next/static/chunks/1.js', line: 2, column: 30 }]; + + it('differs by name, frame, version and code, and ignores frames outside the bundle', () => { + const base = fingerprint('TypeError', frames, '1.20.0'); + expect(base).toMatch(/^[0-9a-f]{16}$/); + expect(fingerprint('TypeError', frames, '1.20.0')).toBe(base); + expect(fingerprint('RangeError', frames, '1.20.0')).not.toBe(base); + expect(fingerprint('TypeError', [{ ...frames[0], line: 3 }], '1.20.0')).not.toBe(base); + expect(fingerprint('TypeError', frames, '1.20.1')).not.toBe(base); + expect(fingerprint('TypeError', frames, '1.20.0', 'serverFail')).not.toBe(base); + expect(fingerprint('TypeError', [...frames, { path: '[other]', line: 1, column: 1 }], '1.20.0')).toBe(base); + }); + + it('accepts only fixed error codes', () => { + expect(isErrorCode('Email/set: overQuota')).toBe(true); + expect(isErrorCode('serverFail')).toBe(true); + expect(isErrorCode('http:503')).toBe(true); + expect(isErrorCode('Merger plans for Acme')).toBe(false); + expect(isErrorCode('Email/set: over quota now')).toBe(false); + }); +}); diff --git a/lib/error-reporting/__tests__/lite.test.ts b/lib/error-reporting/__tests__/lite.test.ts new file mode 100644 index 000000000..b73000005 --- /dev/null +++ b/lib/error-reporting/__tests__/lite.test.ts @@ -0,0 +1,21 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest'; +import { liteErrorReporting } from '../lite'; + +const DSN = 'https://6f0b9c79ed3a4315b76595bc2536ac40@errors.example.org/3'; + +// Bulwark Lite has no server: policy.json carries the error reporting address. +describe('Lite error reporting from policy.json', () => { + it('is off without a valid DSN', () => { + expect(liteErrorReporting({})).toBeNull(); + expect(liteErrorReporting(null)).toBeNull(); + expect(liteErrorReporting({ errorReporting: { dsn: 'not a dsn' } })).toBeNull(); + }); + + it('reads the DSN, sample rate and environment, and never asks for server-side counting', () => { + expect(liteErrorReporting({ errorReporting: { dsn: DSN, sampleRate: 0.25, environment: 'lite-prod' } })) + .toEqual({ dsn: DSN, sampleRate: 0.25, environment: 'lite-prod', collect: false }); + expect(liteErrorReporting({ errorReporting: { dsn: DSN, sampleRate: 7, environment: 'bad env!' } })) + .toEqual({ dsn: DSN, sampleRate: 1, environment: 'production', collect: false }); + }); +}); diff --git a/lib/error-reporting/__tests__/scrub.test.ts b/lib/error-reporting/__tests__/scrub.test.ts new file mode 100644 index 000000000..65be662c0 --- /dev/null +++ b/lib/error-reporting/__tests__/scrub.test.ts @@ -0,0 +1,102 @@ +// @vitest-environment node +import { describe, expect, it } from 'vitest'; +import { scrubFramePath, scrubJmapErrorType, scrubMethodName, scrubName, scrubText } from '../scrub'; +import { routeGroup, scrubRoutePath } from '@/lib/route-groups'; + +// Mail content planted where it could end up in an error: none of it may +// survive scrubbing. +const SUBJECT = 'Merger plans for Acme Q3'; +const ADDRESS = 'ceo.private@acme-corp.example'; +const FOLDER = 'Acquisition Talks'; +const FILE = 'term-sheet-final.pdf'; +const TOKEN = 'eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1c2VyIn0.c2lnbmF0dXJlMTIzNDU2'; + +function expectClean(text: string) { + for (const planted of [SUBJECT, ADDRESS, FOLDER, FILE, TOKEN, 'Merger', 'acme-corp', 'Acquisition', 'term-sheet']) { + expect(text).not.toContain(planted); + } +} + +describe('scrubText', () => { + it('removes a quoted subject, an address and a file name from a message', () => { + const out = scrubText(`Failed to move "${SUBJECT}" from ${ADDRESS} with attachment ${FILE}`); + expectClean(out); + expect(out).toBe('Failed to move "…" from [email] with attachment [file]'); + }); + + it('removes text in every kind of quote, but keeps contractions', () => { + expect(scrubText(`can't open 'Merger' or “Merger” or ‘Merger’ or «Merger» or \`Merger\``)) + .toBe("can't open '…' or “…” or ‘…’ or «…» or `…`"); + }); + + it('removes URLs whole, with their query strings', () => { + const out = scrubText(`fetch https://mail.acme-corp.example/jmap/download/abc/${FILE}?token=${TOKEN} failed`); + expectClean(out); + expect(out).toBe('fetch [url] failed'); + }); + + it('removes credentials, JWTs, key=value pairs and random-looking ids', () => { + expect(scrubText(`Authorization: Bearer ${TOKEN}`)).toBe('Authorization: Bearer [token]'); + expect(scrubText(`token ${TOKEN} expired`)).toBe('token [token] expired'); + expect(scrubText('cookie session=abc123def; theme=dark')).toBe('cookie session=[redacted]; theme=[redacted]'); + expect(scrubText('blob Ma1b2c3d4e5f6a7b8c9d0e1f2a3b4 missing')).toBe('blob [id] missing'); + }); + + it('keeps only the first line and drops a server answer after the status', () => { + expect(scrubText(`Request failed: 400 - {"description":"${SUBJECT}"}`)).toBe('Request failed: 400'); + expect(scrubText(`Something broke\n${SUBJECT}\n${ADDRESS}`)).toBe('Something broke'); + }); + + it('keeps engine messages readable', () => { + expect(scrubText('Cannot read properties of undefined (reading \'id\')')).toBe('Cannot read properties of undefined (reading \'…\')'); + expect(scrubText('e.map is not a function')).toBe('e.map is not a function'); + }); + + it('caps the length', () => { + expect(scrubText('x '.repeat(500)).length).toBeLessThanOrEqual(200); + }); +}); + +describe('route paths in breadcrumbs and reports', () => { + it('keeps the area only, never folder names or ids', () => { + const path = `/de/mail/folder/${encodeURIComponent(FOLDER)}/Mabc123?q=${encodeURIComponent(SUBJECT)}`; + const out = scrubRoutePath(path); + expectClean(decodeURIComponent(out)); + expect(out).toBe('/mail/…'); + }); + + it('removes the base path and maps unknown areas and API routes to fixed names', () => { + expect(scrubRoutePath('/webmail/en/calendar', '/webmail')).toBe('/calendar'); + expect(scrubRoutePath(`/en/${encodeURIComponent(FOLDER)}`)).toBe('/other'); + expect(scrubRoutePath('/api/wopi/files/abc/contents')).toBe('/api/wopi/…'); + expect(routeGroup(`/api/${FOLDER}`)).toBe('api:other'); + expect(routeGroup('/_next/static/chunks/a.js')).toBe('static'); + expect(routeGroup('/')).toBe('page:home'); + }); +}); + +describe('frame paths', () => { + it('drops the origin, base path, query and fragment', () => { + expect(scrubFramePath('https://mail.acme-corp.example/webmail/_next/static/chunks/12-ab.js?dpl=x#y')).toBe('/_next/static/chunks/12-ab.js'); + expect(scrubFramePath('/home/alice/bulwark/.next/server/chunks/ssr/a.js')).toBe('/.next/server/chunks/ssr/a.js'); + expect(scrubFramePath('/home/alice/bulwark/node_modules/next/dist/x.js')).toBe('node_modules/next/dist/x.js'); + expect(scrubFramePath('node:internal/process/task_queues')).toBe('node:internal/process/task_queues'); + }); + + it('turns anything else into [other]', () => { + expect(scrubFramePath('chrome-extension://abc/content.js')).toBe('[other]'); + expect(scrubFramePath(`https://acme-corp.example/${FILE}`)).toBe('[other]'); + expect(scrubFramePath('blob:https://x/1')).toBe('[other]'); + }); +}); + +describe('names', () => { + it('keeps identifiers only', () => { + expect(scrubName('TypeError')).toBe('TypeError'); + expect(scrubName(SUBJECT)).toBe('Error'); + expect(scrubMethodName('Email/get')).toBe('Email/get'); + expect(scrubMethodName(SUBJECT)).toBe('Unknown/method'); + expect(scrubJmapErrorType('overQuota')).toBe('overQuota'); + expect(scrubJmapErrorType(ADDRESS)).toBe('unknown'); + }); +}); diff --git a/lib/error-reporting/__tests__/server.test.ts b/lib/error-reporting/__tests__/server.test.ts new file mode 100644 index 000000000..e17afb52d --- /dev/null +++ b/lib/error-reporting/__tests__/server.test.ts @@ -0,0 +1,101 @@ +// @vitest-environment node +import { mkdtempSync, rmSync, existsSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const config: Record = {}; +vi.mock('@/lib/admin/config-manager', () => ({ + configManager: { + ensureLoaded: async () => {}, + get: (key: string, fallback: unknown) => (key in config ? config[key] : fallback), + }, +})); + +import { captureLoggedError, captureRequestError, installServerErrorReporting } from '../server'; +import { flushErrors, readErrors, resetAggregateForTests, utcDay } from '../aggregate'; +import { logger, setErrorSink } from '@/lib/logger'; + +const DSN = 'https://6f0b9c79ed3a4315b76595bc2536ac40@bugsink.example.org/3'; +const SUBJECT = 'Merger plans for Acme Q3'; +const ADDRESS = 'ceo.private@acme-corp.example'; + +let dir: string; +let fetchSpy: ReturnType; + +beforeEach(() => { + dir = mkdtempSync(path.join(tmpdir(), 'bulwark-errors-')); + process.env.TELEMETRY_DATA_DIR = dir; + for (const key of Object.keys(config)) delete config[key]; + resetAggregateForTests(); + fetchSpy = vi.fn().mockResolvedValue(new Response(null, { status: 200 })); + vi.stubGlobal('fetch', fetchSpy); +}); + +afterEach(() => { + setErrorSink(null); + resetAggregateForTests(); + vi.unstubAllGlobals(); + delete process.env.TELEMETRY_DATA_DIR; + rmSync(dir, { recursive: true, force: true }); +}); + +function serverError(message: string): Error { + const error = new Error(message); + error.stack = `Error: ${message}\n at handler (/app/.next/server/chunks/77.js:1:900)\n at /app/node_modules/next/dist/server.js:5:6`; + return error; +} + +describe('server error reporting', () => { + it('does nothing at all without a DSN or other reason to collect', async () => { + await captureRequestError(serverError('boom'), '/api/admin/config'); + await flushErrors(); + expect(fetchSpy).not.toHaveBeenCalled(); + expect(existsSync(path.join(dir, 'errors.json'))).toBe(false); + }); + + it('sends a request error to the DSN with the route pattern and no mail content', async () => { + config.errorReportingDsn = DSN; + config.errorReportingEnvironment = 'beta'; + await captureRequestError(serverError(`Could not file "${SUBJECT}" for ${ADDRESS}`), '/api/wopi/files/[fileId]'); + + expect(fetchSpy).toHaveBeenCalledTimes(1); + const [url, init] = fetchSpy.mock.calls[0]; + expect(url).toBe('https://bugsink.example.org/api/3/envelope/?sentry_key=6f0b9c79ed3a4315b76595bc2536ac40&sentry_version=7'); + const body = String((init as RequestInit).body); + expect(body).not.toContain('Merger'); + expect(body).not.toContain(ADDRESS); + const event = JSON.parse(body.trim().split('\n')[2]); + expect(event.platform).toBe('node'); + expect(event.environment).toBe('beta'); + expect(event.tags).toEqual({ runtime: 'server', route: '/api/wopi/files/[fileId]' }); + expect(event.exception.values[0].stacktrace.frames.at(-1).abs_path).toBe('/.next/server/chunks/77.js'); + }); + + it('counts the error per day', async () => { + config.errorReportingDsn = DSN; + await captureRequestError(serverError('one')); + await captureRequestError(serverError('one')); + await flushErrors(); + const day = (await readErrors())[utcDay()]; + expect(Object.values(day).map((e) => [e.name, e.runtime, e.count])).toEqual([['Error', 'server', 2]]); + }); + + it('reports logger.error calls with the caller\'s frames once installed', async () => { + config.errorReportingDsn = DSN; + await installServerErrorReporting(); + logger.error('telemetry GET error', { error: `failed for ${ADDRESS}` }); + await vi.waitFor(() => expect(fetchSpy).toHaveBeenCalledTimes(1)); + const event = JSON.parse(String((fetchSpy.mock.calls[0][1] as RequestInit).body).trim().split('\n')[2]); + expect(event.exception.values[0].type).toBe('LoggedError'); + expect(event.exception.values[0].value).toBe('telemetry GET error: failed for [email]'); + }); + + it('drops the logger\'s own frames from a logged error', async () => { + config.errorReportingDsn = DSN; + const stack = 'Error\n at log (/app/.next/server/chunks/1.js:1:1)\n at Object.error (/app/.next/server/chunks/1.js:2:2)\n at GET (/app/.next/server/chunks/route.js:9:9)'; + await captureLoggedError('x failed', undefined, stack); + const event = JSON.parse(String((fetchSpy.mock.calls[0][1] as RequestInit).body).trim().split('\n')[2]); + expect(event.exception.values[0].stacktrace.frames.map((f: { function: string }) => f.function)).toEqual(['GET']); + }); +}); diff --git a/lib/error-reporting/aggregate.ts b/lib/error-reporting/aggregate.ts new file mode 100644 index 000000000..5d933ce34 --- /dev/null +++ b/lib/error-reporting/aggregate.ts @@ -0,0 +1,179 @@ +import { logger } from '@/lib/logger'; +import { telemetryDocs } from '@/lib/telemetry/state'; +import { isErrorCode } from './envelope'; +import { fingerprint as fingerprintOf } from './fingerprint'; +import type { Frame } from './stack'; + +/** + * Errors counted per UTC day: fingerprint -> name, top frames, version and + * how often. The admin's "errors today", the metrics endpoint and the + * telemetry error counts read it. Only what scrubbing leaves goes in. + * + * Bounded: MAX_PER_DAY fingerprints per day (the rarest go first) and + * KEEP_DAYS days. Counts gather in memory and are added to `errors.json` in + * telemetry's document collection every FLUSH_MS, so several replicas add up + * (an update, not an overwrite). + */ + +export const MAX_PER_DAY = 200; +export const KEEP_DAYS = 8; +export const MAX_FRAMES_KEPT = 5; +const FLUSH_MS = 60_000; +export const ERRORS_DOC = 'errors.json'; + +export interface AggregateEntry { + name: string; + frames: Frame[]; + version: string; + /** A fixed error code when the error has one. */ + code?: string; + runtime: 'browser' | 'server'; + count: number; +} + +/** day (YYYY-MM-DD) -> fingerprint -> entry */ +export type Aggregate = Record>; + +export interface Recordable { + name: string; + frames: Frame[]; + code?: string; +} + +interface AggregateState { + pending: Aggregate; + timer: ReturnType | null; + flushing: Promise | null; +} + +const KEY = Symbol.for('bulwark.error-reporting.aggregate'); +function state(): AggregateState { + const g = globalThis as typeof globalThis & { [KEY]?: AggregateState }; + return (g[KEY] ??= { pending: {}, timer: null, flushing: null }); +} + +export function utcDay(at = Date.now()): string { + return new Date(at).toISOString().slice(0, 10); +} + +/** Keep the newest KEEP_DAYS days and the MAX_PER_DAY most frequent per day. */ +export function bound(agg: Aggregate, today = utcDay()): Aggregate { + const cutoff = utcDay(Date.parse(`${today}T00:00:00Z`) - (KEEP_DAYS - 1) * 86_400_000); + const out: Aggregate = {}; + for (const [day, entries] of Object.entries(agg)) { + if (!/^\d{4}-\d{2}-\d{2}$/.test(day) || day < cutoff) continue; + const kept = Object.entries(entries) + .sort(([, a], [, b]) => b.count - a.count) + .slice(0, MAX_PER_DAY); + out[day] = Object.fromEntries(kept); + } + return out; +} + +/** Add `delta`'s counts to `base`. */ +export function merge(base: Aggregate, delta: Aggregate): Aggregate { + const out: Aggregate = {}; + for (const [day, entries] of Object.entries(base)) out[day] = { ...entries }; + for (const [day, entries] of Object.entries(delta)) { + const target = (out[day] ??= {}); + for (const [print, entry] of Object.entries(entries)) { + const existing = target[print]; + target[print] = existing ? { ...existing, count: existing.count + entry.count } : { ...entry }; + } + } + return out; +} + +function parseStored(data: Buffer | null): Aggregate { + if (!data) return {}; + try { + const parsed = JSON.parse(data.toString('utf8')) as unknown; + return parsed && typeof parsed === 'object' ? (parsed as Aggregate) : {}; + } catch { + return {}; + } +} + +/** Count one error. Returns its fingerprint. */ +export function recordError(report: Recordable, runtime: 'browser' | 'server', version: string, at = Date.now()): string { + const print = fingerprintOf(report.name, report.frames, version, report.code); + const s = state(); + const day = utcDay(at); + const entries = (s.pending[day] ??= {}); + const existing = entries[print]; + if (existing) { + existing.count += 1; + } else if (Object.keys(entries).length < MAX_PER_DAY) { + entries[print] = { + name: report.name, + frames: report.frames.slice(0, MAX_FRAMES_KEPT), + version, + ...(isErrorCode(report.code) ? { code: report.code } : {}), + runtime, + count: 1, + }; + } + startTimer(); + return print; +} + +function startTimer(): void { + const s = state(); + if (s.timer) return; + s.timer = setInterval(() => { void flushErrors(); }, FLUSH_MS); + s.timer.unref?.(); +} + +/** Add what this process counted since the last flush to the stored aggregate. */ +export function flushErrors(): Promise { + const s = state(); + if (s.flushing) return s.flushing; + const delta = s.pending; + if (Object.keys(delta).length === 0) return Promise.resolve(); + s.pending = {}; + s.flushing = telemetryDocs() + .update(ERRORS_DOC, (current) => JSON.stringify(bound(merge(parseStored(current), delta)))) + .then(() => undefined) + .catch((error) => { + // Keep the counts for the next try. + s.pending = merge(delta, s.pending); + logger.warn('error counts not stored', { error: error instanceof Error ? error.message : String(error) }); + }) + .finally(() => { s.flushing = null; }); + return s.flushing; +} + +/** The stored aggregate with this process's unflushed counts added. */ +export async function readErrors(): Promise { + let stored: Aggregate = {}; + try { + stored = parseStored((await telemetryDocs().get(ERRORS_DOC))?.data ?? null); + } catch (error) { + logger.warn('error counts not readable', { error: error instanceof Error ? error.message : String(error) }); + } + return bound(merge(stored, state().pending)); +} + +export interface DaySummary { + total: number; + top: Array<{ fingerprint: string } & AggregateEntry>; +} + +export function summarizeDay(agg: Aggregate, day = utcDay(), limit = 20): DaySummary { + const entries = Object.entries(agg[day] ?? {}); + return { + total: entries.reduce((sum, [, e]) => sum + e.count, 0), + top: entries + .sort(([, a], [, b]) => b.count - a.count) + .slice(0, limit) + .map(([fingerprint, e]) => ({ fingerprint, ...e })), + }; +} + +/** For tests. */ +export function resetAggregateForTests(): void { + const s = state(); + if (s.timer) clearInterval(s.timer); + const g = globalThis as typeof globalThis & { [KEY]?: AggregateState }; + delete g[KEY]; +} diff --git a/lib/error-reporting/client.ts b/lib/error-reporting/client.ts new file mode 100644 index 000000000..c329d6b06 --- /dev/null +++ b/lib/error-reporting/client.ts @@ -0,0 +1,244 @@ +import { apiFetch, getPathPrefix } from '@/lib/browser-navigation'; +import { scrubRoutePath } from '@/lib/route-groups'; +import { debugIdsByPath } from './debug-ids'; +import { + appVersion, buildEnvelope, buildEvent, envelopeUrl, isErrorCode, newEventId, parseDsn, releaseName, + type Breadcrumb, type ErrorReport, +} from './envelope'; +import { fingerprint } from './fingerprint'; +import { ReportLimiter } from './limiter'; +import { scrubJmapErrorType, scrubMethodName, scrubName, scrubText } from './scrub'; +import { parseStack } from './stack'; + +/** + * Error reporting in the browser, off until the config names a DSN or asks + * for errors to be collected (lib/error-reporting/settings.ts). Captures what + * the error boundaries, window.onerror, unhandled rejections and failed JMAP + * requests hand it, scrubs it (./scrub.ts) and sends it to the DSN and/or + * this server's /api/errors. Nothing is sent before the config arrived, and + * nothing at all when both are off. + * + * https://bulwarkmail.org/docs/admin/monitoring + */ + +export interface ClientSettings { + /** The error tracker's DSN; '' for none. */ + dsn: string; + /** 0..1, applied to what goes to the DSN. */ + sampleRate: number; + environment: string; + /** Also post errors to this server's /api/errors (the per-day counts). */ + collect: boolean; +} + +const MAX_PENDING = 10; +const MAX_BREADCRUMBS = 20; + +interface ClientState { + /** undefined until configured; null when reporting is off. */ + settings: ClientSettings | null | undefined; + pending: ErrorReport[]; + breadcrumbs: Breadcrumb[]; + limiter: ReportLimiter | null; + installed: boolean; +} + +// On globalThis: the error boundary, the JMAP client and the root component +// may each end up with their own copy of this module. +const KEY = Symbol.for('bulwark.error-reporting.client'); +function state(): ClientState { + const g = globalThis as typeof globalThis & { [KEY]?: ClientState }; + return (g[KEY] ??= { settings: undefined, pending: [], breadcrumbs: [], limiter: null, installed: false }); +} + +function sessionStore(): Storage | null { + try { + return typeof sessionStorage === 'undefined' ? null : sessionStorage; + } catch { + return null; + } +} + +function limiter(): ReportLimiter { + const s = state(); + return (s.limiter ??= new ReportLimiter({ perMinute: 10, perSession: 30, dedupWindowMs: Infinity, storage: sessionStore() })); +} + +function basePath(): string { + return getPathPrefix(); +} + +function currentRoute(): string | undefined { + if (typeof location === 'undefined') return undefined; + return scrubRoutePath(location.pathname, basePath()); +} + +/** Apply the settings from /api/config (or Lite's policy.json); null switches reporting off. */ +export function configureErrorReporting(settings: ClientSettings | null): void { + const s = state(); + const on = settings && (parseDsn(settings.dsn) || settings.collect) ? settings : null; + s.settings = on; + const queued = s.pending; + s.pending = []; + if (on) for (const report of queued) deliver(report); +} + +export function isErrorReportingOn(): boolean { + return !!state().settings; +} + +/** A route change or a JMAP call, kept for the next report. */ +export function addBreadcrumb(category: Breadcrumb['category'], message: string): void { + const s = state(); + if (s.settings === null) return; + s.breadcrumbs.push({ timestamp: Date.now() / 1000, category, message }); + if (s.breadcrumbs.length > MAX_BREADCRUMBS) s.breadcrumbs.shift(); +} + +/** A route change: only the scrubbed route is kept. */ +export function recordNavigation(pathname: string): void { + addBreadcrumb('navigation', scrubRoutePath(pathname, basePath())); +} + +/** A JMAP request: only the method names. */ +export function recordJmapCall(methods: readonly string[]): void { + addBreadcrumb('jmap', methods.slice(0, 5).map(scrubMethodName).join(', ')); +} + +/** Turn anything thrown into a scrubbed report; null for what isn't worth one. */ +export function reportFromError(error: unknown, mechanism: string, handled: boolean): ErrorReport | null { + let name = 'Error'; + let message = ''; + let stack: string | undefined; + if (error instanceof Error || (error && typeof error === 'object' && 'message' in error)) { + const e = error as { name?: unknown; message?: unknown; stack?: unknown }; + name = scrubName(e.name); + message = scrubText(e.message); + stack = typeof e.stack === 'string' ? e.stack : undefined; + } else if (typeof error === 'string') { + name = mechanism === 'unhandledrejection' ? 'UnhandledRejection' : 'Error'; + message = scrubText(error); + } else { + return null; + } + const frames = parseStack(stack); + // Every frame outside our bundles: a browser extension or injected script. + if (frames.length > 0 && frames.every((frame) => frame.path === '[other]')) return null; + return { name, message, frames, handled, mechanism, route: currentRoute() }; +} + +/** Report an error caught by a boundary, a global handler or a caller. */ +export function captureException(error: unknown, mechanism = 'manual', handled = true): void { + try { + const report = reportFromError(error, mechanism, handled); + if (report) captureReport(report); + } catch { + // Reporting must never throw into the code that reported. + } +} + +/** Report a JMAP request that failed: the method names and the status or error type only. */ +export function reportJmapFailure(failure: { status: number } | { method: string; type: string }): void { + try { + const code = 'status' in failure + ? `http:${Number.isInteger(failure.status) ? failure.status : 0}` + : `${scrubMethodName(failure.method)}: ${scrubJmapErrorType(failure.type)}`; + if (!isErrorCode(code)) return; + const name = 'status' in failure ? 'JmapRequestError' : 'JmapMethodError'; + const frames = parseStack(new Error().stack).slice(1); + captureReport({ + name, + message: code, + code, + frames, + handled: true, + mechanism: 'jmap', + route: currentRoute(), + }); + } catch { + // never throws into the JMAP client + } +} + +export function captureReport(report: ErrorReport): void { + const s = state(); + if (s.settings === null) return; + const withCrumbs = { ...report, breadcrumbs: report.breadcrumbs ?? [...s.breadcrumbs] }; + if (s.settings === undefined) { + if (s.pending.length < MAX_PENDING) s.pending.push(withCrumbs); + return; + } + deliver(withCrumbs); +} + +function deliver(report: ErrorReport): void { + const s = state(); + const settings = s.settings; + if (!settings) return; + const version = appVersion(); + const print = fingerprint(report.name, report.frames, version, report.code); + if (!limiter().allow(print)) return; + + const dsn = parseDsn(settings.dsn); + if (dsn && Math.random() < settings.sampleRate) { + const event = buildEvent(report, { + eventId: newEventId(), + timestamp: Date.now() / 1000, + release: releaseName(), + environment: settings.environment, + platform: 'javascript', + fingerprint: print, + debugIds: debugIdsByPath(), + tags: { runtime: 'browser', ...(process.env.NEXT_PUBLIC_BULWARK_LITE === '1' ? { lite: 'yes' } : {}) }, + }); + send(envelopeUrl(dsn), buildEnvelope(event), { + // text/plain keeps it a simple request: no preflight, no custom header. + headers: { 'Content-Type': 'text/plain;charset=UTF-8' }, + credentials: 'omit', + // The page URL holds folder names and message ids. + referrerPolicy: 'no-referrer', + mode: 'cors', + }); + } + if (settings.collect) { + send('/api/errors', JSON.stringify({ + name: report.name, + message: report.message, + code: report.code, + frames: report.frames.slice(0, 10), + mechanism: report.mechanism, + handled: report.handled, + route: report.route, + version, + }), { headers: { 'Content-Type': 'application/json' }, credentials: 'same-origin' }, true); + } +} + +// A failed send is dropped quietly: it never turns into another report. +function send(url: string, body: string, init: RequestInit, sameOrigin = false): void { + const request = sameOrigin + ? apiFetch(url, { method: 'POST', body, keepalive: true, ...init }) + : fetch(url, { method: 'POST', body, keepalive: true, ...init }); + void Promise.resolve(request).catch(() => undefined); +} + +/** window.onerror and unhandled rejections, once. */ +export function installGlobalHandlers(): void { + const s = state(); + if (s.installed || typeof window === 'undefined') return; + s.installed = true; + window.addEventListener('error', (event: ErrorEvent) => { + // "Script error." from another origin carries nothing to report. + if (!event.error) return; + captureException(event.error, 'onerror', false); + }); + window.addEventListener('unhandledrejection', (event: PromiseRejectionEvent) => { + captureException(event.reason, 'unhandledrejection', false); + }); +} + +/** For tests. */ +export function resetErrorReportingForTests(): void { + const g = globalThis as typeof globalThis & { [KEY]?: ClientState }; + delete g[KEY]; +} diff --git a/lib/error-reporting/debug-ids.ts b/lib/error-reporting/debug-ids.ts new file mode 100644 index 000000000..88ae156d9 --- /dev/null +++ b/lib/error-reporting/debug-ids.ts @@ -0,0 +1,47 @@ +import { parseStack } from './stack'; + +/** + * The debug ID of each loaded bundle file, by scrubbed path. + * + * Turbopack (`turbopack.debugIds`) and `sentry-cli sourcemaps inject` (the + * webpack build in Docker) start every chunk with a line that stores + * `globalThis[KEY][new Error().stack] = ""`. The first frame of that + * stack is the chunk itself, so its path maps to the id. The error tracker + * finds the uploaded source map by that id alone. + */ + +const KEYS = ['_debugIds', '_sentryDebugIds'] as const; +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +let cache: { size: number; ids: Map } | null = null; + +function registered(): Array<[string, string]> { + const g = globalThis as Record; + const out: Array<[string, string]> = []; + for (const key of KEYS) { + const map = g[key]; + if (!map || typeof map !== 'object') continue; + for (const [stack, id] of Object.entries(map as Record)) { + if (typeof id === 'string' && UUID.test(id)) out.push([stack, id.toLowerCase()]); + } + } + return out; +} + +/** Path -> debug ID for every chunk loaded so far. Chunks load lazily, so this is rebuilt when more have registered. */ +export function debugIdsByPath(): Map { + const entries = registered(); + if (cache && cache.size === entries.length) return cache.ids; + const ids = new Map(); + for (const [stack, id] of entries) { + const first = parseStack(stack).find((frame) => frame.path !== '[other]'); + if (first) ids.set(first.path, id); + } + cache = { size: entries.length, ids }; + return ids; +} + +/** For tests. */ +export function resetDebugIdCache(): void { + cache = null; +} diff --git a/lib/error-reporting/envelope.ts b/lib/error-reporting/envelope.ts new file mode 100644 index 000000000..aaad77e1b --- /dev/null +++ b/lib/error-reporting/envelope.ts @@ -0,0 +1,149 @@ +import type { Frame } from './stack'; + +/** + * The Sentry envelope format, which Bugsink, GlitchTip and Sentry accept. + * Only what Bulwark sends: one event item per envelope. + * https://develop.sentry.dev/sdk/data-model/envelopes/ + */ + +export interface Dsn { + /** scheme://host[:port] */ + origin: string; + /** Anything between the host and the project id, usually ''. */ + pathPrefix: string; + publicKey: string; + projectId: string; +} + +/** `https://@host[:port][/prefix]/`, or null. */ +export function parseDsn(raw: unknown): Dsn | null { + if (typeof raw !== 'string' || !raw.trim()) return null; + let url: URL; + try { + url = new URL(raw.trim()); + } catch { + return null; + } + if (url.protocol !== 'https:' && url.protocol !== 'http:') return null; + if (!url.username || url.password || url.search || url.hash) return null; + const parts = url.pathname.split('/').filter(Boolean); + const projectId = parts.pop(); + if (!projectId || !/^\d+$/.test(projectId) || !/^[0-9a-f]{16,64}$/i.test(url.username)) return null; + return { + origin: `${url.protocol}//${url.host}`, + pathPrefix: parts.length ? `/${parts.join('/')}` : '', + publicKey: url.username, + projectId, + }; +} + +/** Where envelopes go. The key travels in the query, so a browser needs no custom header. */ +export function envelopeUrl(dsn: Dsn): string { + return `${dsn.origin}${dsn.pathPrefix}/api/${dsn.projectId}/envelope/?sentry_key=${dsn.publicKey}&sentry_version=7`; +} + +/** A fixed error code: `Email/set: overQuota`, `serverFail`, `http:503`. */ +export const ERROR_CODE = /^(?:(?:[A-Z][A-Za-z]{0,40}\/[a-z][A-Za-z]{0,40}: )?[a-z][A-Za-z]{1,40}|http:\d{3})$/; + +export function isErrorCode(value: unknown): value is string { + return typeof value === 'string' && ERROR_CODE.test(value); +} + +export interface Breadcrumb { + /** Seconds since the epoch. */ + timestamp: number; + category: 'navigation' | 'jmap'; + /** A scrubbed route (`/mail/…`) or a JMAP method name. */ + message: string; +} + +/** An error, scrubbed, as both sides hand it to the sender. */ +export interface ErrorReport { + name: string; + message: string; + frames: Frame[]; + /** One of Bulwark's fixed codes, when the error has one. */ + code?: string; + handled: boolean; + /** How it was caught: onerror, unhandledrejection, boundary, global-error, jmap, request, logger. */ + mechanism: string; + breadcrumbs?: Breadcrumb[]; + /** The scrubbed route it happened on. */ + route?: string; +} + +export interface EventContext { + eventId: string; + /** Seconds since the epoch. */ + timestamp: number; + release: string; + environment: string; + platform: 'javascript' | 'node'; + fingerprint: string; + /** Path -> debug id, for the frames' files. */ + debugIds: Map; + tags?: Record; +} + +/** The event payload. Frames go oldest first, as the format wants. */ +export function buildEvent(report: ErrorReport, ctx: EventContext): Record { + const frames = [...report.frames].reverse().map((frame) => ({ + abs_path: frame.path, + filename: frame.path, + ...(frame.fn ? { function: frame.fn } : {}), + ...(frame.line !== undefined ? { lineno: frame.line } : {}), + ...(frame.column !== undefined ? { colno: frame.column } : {}), + in_app: frame.path.startsWith('/_next/') || frame.path.startsWith('/.next/'), + })); + const images = [...new Set(report.frames.map((frame) => frame.path))] + .filter((path) => ctx.debugIds.has(path)) + .map((path) => ({ type: 'sourcemap', code_file: path, debug_id: ctx.debugIds.get(path) })); + return { + event_id: ctx.eventId, + timestamp: ctx.timestamp, + platform: ctx.platform, + level: 'error', + release: ctx.release, + environment: ctx.environment, + // Bulwark's own grouping: the same key as the per-day counts. + fingerprint: [ctx.fingerprint], + exception: { + values: [{ + type: report.name, + value: report.code ?? report.message, + mechanism: { type: report.mechanism, handled: report.handled }, + ...(frames.length ? { stacktrace: { frames } } : {}), + }], + }, + ...(report.breadcrumbs?.length + ? { breadcrumbs: { values: report.breadcrumbs.map((b) => ({ timestamp: b.timestamp, category: b.category, message: b.message })) } } + : {}), + ...(images.length ? { debug_meta: { images } } : {}), + tags: { ...(ctx.tags ?? {}), ...(report.route ? { route: report.route } : {}) }, + sdk: { name: 'bulwark.error-reporting', version: '1' }, + }; +} + +/** One event in an envelope. */ +export function buildEnvelope(event: Record): string { + const header = { event_id: event.event_id, sent_at: new Date().toISOString() }; + return `${JSON.stringify(header)}\n${JSON.stringify({ type: 'event', content_type: 'application/json' })}\n${JSON.stringify(event)}\n`; +} + +/** 32 hex digits, as event ids are. */ +export function newEventId(): string { + const bytes = new Uint8Array(16); + globalThis.crypto.getRandomValues(bytes); + return Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join(''); +} + +/** `+`, the release the source maps were uploaded for. */ +export function releaseName(): string { + const version = process.env.NEXT_PUBLIC_APP_VERSION || '0.0.0'; + const commit = process.env.NEXT_PUBLIC_GIT_COMMIT; + return commit && commit !== 'unknown' ? `${version}+${commit}` : version; +} + +export function appVersion(): string { + return process.env.NEXT_PUBLIC_APP_VERSION || '0.0.0'; +} diff --git a/lib/error-reporting/fingerprint.ts b/lib/error-reporting/fingerprint.ts new file mode 100644 index 000000000..f57a1e537 --- /dev/null +++ b/lib/error-reporting/fingerprint.ts @@ -0,0 +1,34 @@ +import type { Frame } from './stack'; + +/** How many of the top frames make up a fingerprint. */ +export const FINGERPRINT_FRAMES = 5; + +/** + * 64-bit FNV-1a as 16 hex digits. Synchronous and the same in the browser and + * on the server; not for security, only for grouping. + */ +export function hash64(text: string): string { + let h1 = 0x811c9dc5; + let h2 = 0xcbf29ce4; + for (let i = 0; i < text.length; i++) { + const c = text.charCodeAt(i); + h1 = Math.imul(h1 ^ c, 0x01000193) >>> 0; + h2 = Math.imul(h2 ^ c ^ (h1 >>> 7), 0x01000193) >>> 0; + } + return h1.toString(16).padStart(8, '0') + h2.toString(16).padStart(8, '0'); +} + +/** + * The fingerprint of an error: its name, the top frames (file without origin, + * line, column) and the Bulwark version. The same error in another release is + * another fingerprint, because its frames point into other chunks anyway. + * `code` is one of Bulwark's fixed error codes (a JMAP error type), for + * errors that are all raised from the same place. + */ +export function fingerprint(name: string, frames: readonly Frame[], version: string, code?: string): string { + const top = frames + .filter((frame) => frame.path !== '[other]') + .slice(0, FINGERPRINT_FRAMES) + .map((frame) => `${frame.path}:${frame.line ?? ''}:${frame.column ?? ''}`); + return hash64([name, code ?? '', ...top, version].join('|')); +} diff --git a/lib/error-reporting/limiter.ts b/lib/error-reporting/limiter.ts new file mode 100644 index 000000000..69bbc332c --- /dev/null +++ b/lib/error-reporting/limiter.ts @@ -0,0 +1,81 @@ +/** + * Keeps error reporting from flooding anyone: each fingerprint once per + * window, at most `perMinute` reports in any minute and `perSession` in all. + * The browser keeps its state in sessionStorage, so a page that crashes on + * every load doesn't start over each time. + */ + +export interface LimiterOptions { + perMinute: number; + perSession: number; + /** How long a fingerprint stays "already sent"; Infinity for the whole session. */ + dedupWindowMs: number; + /** Persist across page loads (sessionStorage); absent on the server. */ + storage?: Pick | null; + storageKey?: string; + now?: () => number; +} + +interface State { + total: number; + sent: Record; +} + +const MAX_REMEMBERED = 500; + +export class ReportLimiter { + private readonly options: Required> & { storage: LimiterOptions['storage'] }; + private recent: number[] = []; + private state: State = { total: 0, sent: {} }; + + constructor(options: LimiterOptions) { + this.options = { storageKey: 'bulwark.error-reporting', now: () => Date.now(), storage: null, ...options }; + this.load(); + } + + private load(): void { + const { storage, storageKey } = this.options; + if (!storage) return; + try { + const raw = storage.getItem(storageKey); + if (!raw) return; + const parsed = JSON.parse(raw) as Partial; + if (typeof parsed.total === 'number' && parsed.sent && typeof parsed.sent === 'object') { + this.state = { total: parsed.total, sent: parsed.sent as Record }; + } + } catch { + // Unreadable or blocked storage: start fresh, in memory. + } + } + + private save(): void { + const { storage, storageKey } = this.options; + if (!storage) return; + try { + storage.setItem(storageKey, JSON.stringify(this.state)); + } catch { + // Quota or privacy mode: the in-memory state still applies. + } + } + + /** Whether a report with this fingerprint may go out now; counts it when it may. */ + allow(fingerprint: string): boolean { + const now = this.options.now(); + const last = this.state.sent[fingerprint]; + if (last !== undefined && now - last < this.options.dedupWindowMs) return false; + if (this.state.total >= this.options.perSession) return false; + this.recent = this.recent.filter((at) => now - at < 60_000); + if (this.recent.length >= this.options.perMinute) return false; + this.recent.push(now); + this.state.total += 1; + this.state.sent[fingerprint] = now; + const keys = Object.keys(this.state.sent); + if (keys.length > MAX_REMEMBERED) { + for (const key of keys.sort((a, b) => this.state.sent[a] - this.state.sent[b]).slice(0, keys.length - MAX_REMEMBERED)) { + delete this.state.sent[key]; + } + } + this.save(); + return true; + } +} diff --git a/lib/error-reporting/lite.ts b/lib/error-reporting/lite.ts new file mode 100644 index 000000000..f754f1cd5 --- /dev/null +++ b/lib/error-reporting/lite.ts @@ -0,0 +1,22 @@ +import type { ClientSettings } from './client'; +import { parseDsn } from './envelope'; + +/** + * Bulwark Lite has no server: error reporting is set in policy.json and + * browsers send straight to the DSN. There are no per-day counts. + * + * "errorReporting": { "dsn": "https://@host/", "sampleRate": 1, "environment": "production" } + * + * The DSN's host must also be in the CSP's connect-src of the Lite host. + * https://bulwarkmail.org/docs/lite/configuration + */ +export function liteErrorReporting(policy: unknown): ClientSettings | null { + const raw = policy && typeof policy === 'object' ? (policy as Record).errorReporting : undefined; + if (!raw || typeof raw !== 'object') return null; + const input = raw as Record; + const dsn = typeof input.dsn === 'string' && parseDsn(input.dsn) ? input.dsn.trim() : ''; + if (!dsn) return null; + const rate = typeof input.sampleRate === 'number' && Number.isFinite(input.sampleRate) ? Math.min(1, Math.max(0, input.sampleRate)) : 1; + const environment = typeof input.environment === 'string' && /^[\w.-]{1,64}$/.test(input.environment) ? input.environment : 'production'; + return { dsn, sampleRate: rate, environment, collect: false }; +} diff --git a/lib/error-reporting/scrub.ts b/lib/error-reporting/scrub.ts new file mode 100644 index 000000000..809d633ef --- /dev/null +++ b/lib/error-reporting/scrub.ts @@ -0,0 +1,126 @@ +/** + * What an error report may carry, applied in the browser before sending and + * again on the server for everything a browser posts (lib/error-reporting). + * + * Mail content must never leave the browser in an error, not even to the + * admin's own error tracker: no message bodies, subjects, addresses, file + * names, tokens, cookies or URLs. Error messages are free text, so the rules + * remove whatever could be such content: quoted text, addresses, URLs, file + * names, credentials, long random-looking strings, key=value pairs, and + * everything after the first line. What stays is the shape of the message + * ("Cannot read properties of undefined (reading "…")"). + * + * Documented at https://bulwarkmail.org/docs/admin/monitoring#what-is-removed + */ + +export const MAX_MESSAGE_LENGTH = 200; + +const REDACTED = '…'; + +// Quoted text: "…", '…', `…`, “…”, „…“, ‘…’, «…», ‹…›. A single quote only +// counts when it starts after a space or bracket, so "can't" stays a word. +const QUOTED: RegExp[] = [ + /"[^"\n]*"/g, + /`[^`\n]*`/g, + /[“„][^”“\n]*[”“]/g, + /‘[^’\n]*’/g, + /«[^»\n]*»/g, + /‹[^›\n]*›/g, + /(^|[\s([{:,=])'[^'\n]*'(?=$|[\s)\]}.,:;!?])/g, +]; + +const EMAIL = /[^\s@<>"'`()[\]{},;:]+@[^\s@<>"'`()[\]{},;:]+\.[^\s@<>"'`()[\]{},;:]+/g; +const URL_LIKE = /\b[a-z][a-z0-9+.-]*:\/\/[^\s"'`<>()]*/gi; +const WWW = /\bwww\.[^\s"'`<>()]+/gi; +const AUTH = /\b(Bearer|Basic|Token)\s+[^\s,;]+/gi; +const JWT = /\beyJ[\w-]{5,}\.[\w-]{5,}\.[\w-]*/g; +const KEY_VALUE = /\b([A-Za-z_][\w.-]{0,40})=([^\s&;,]+)/g; +const FILE_NAME = /[^\s/\\"'`<>()]+\.(?:pdf|docx?|xlsx?|pptx?|odt|ods|odp|rtf|txt|csv|eml|msg|ics|vcf|zip|rar|7z|tar|gz|png|jpe?g|gif|webp|heic|svg|mp[34]|mov|wav|html?|xml|json|p7[msc]|asc|gpg|pgp|key|pem)\b/gi; +// Runs of letters and digits that look random: ids, hashes, keys, tokens. +const LONG_RUN = /[A-Za-z0-9+/_=-]{24,}/g; +const IPV4 = /\b\d{1,3}(?:\.\d{1,3}){3}\b/g; +const PHONE = /\+?\d[\d ()-]{7,}\d/g; + +function looksRandom(run: string): boolean { + // Long identifiers in code are words glued together: keep them when they + // are only letters. Anything with digits or symbols mixed in goes. + return /\d/.test(run) || /[+/=_-]/.test(run) || run.length > 40; +} + +/** Free text from an error, made safe to send. */ +export function scrubText(input: unknown): string { + if (typeof input !== 'string' || input === '') return ''; + // Only the first line: server answers, stack traces and multi-line content + // that ended up in a message all come after it. + let text = input.split(/\r?\n/, 1)[0]; + // "Request failed: 400 - ": the answer may echo a subject. + text = text.replace(/^(.*?\b\d{3})\s+-\s.*$/, '$1'); + text = text.replace(AUTH, '$1 [token]'); + text = text.replace(JWT, '[token]'); + text = text.replace(URL_LIKE, '[url]'); + text = text.replace(WWW, '[url]'); + text = text.replace(EMAIL, '[email]'); + for (const quoted of QUOTED) { + text = text.replace(quoted, (match, lead?: string) => { + const prefix = typeof lead === 'string' ? lead : ''; + const quote = match.slice(prefix.length, prefix.length + 1); + const close = match.slice(-1); + return `${prefix}${quote}${REDACTED}${close}`; + }); + } + text = text.replace(FILE_NAME, '[file]'); + text = text.replace(KEY_VALUE, '$1=[redacted]'); + text = text.replace(LONG_RUN, (run) => (looksRandom(run) ? '[id]' : run)); + text = text.replace(IPV4, '[ip]'); + text = text.replace(PHONE, '[number]'); + if (text.length > MAX_MESSAGE_LENGTH) text = `${text.slice(0, MAX_MESSAGE_LENGTH - 1)}…`; + return text; +} + +/** An error name: a code identifier, or "Error". */ +export function scrubName(input: unknown): string { + return typeof input === 'string' && /^[A-Za-z_$][\w$]{0,63}$/.test(input) ? input : 'Error'; +} + +/** A function name from a stack frame: identifiers and the usual decorations only. */ +export function scrubFunctionName(input: unknown): string | undefined { + if (typeof input !== 'string' || input === '') return undefined; + const trimmed = input.trim().slice(0, 120); + return /^[\w$.<>[\] /#-]+$/.test(trimmed) && !/@/.test(trimmed) ? trimmed : '?'; +} + +/** + * The path of a stack frame's file, without origin, base path, query or + * fragment: `/_next/static/chunks/x.js` for the browser bundle, + * `/.next/server/...` for the server's, `node:...` for Node's own modules, + * `node_modules/...` for dependencies; `[other]` for anything else + * (extensions, plugin code, evaluated code). + */ +export function scrubFramePath(input: unknown): string { + if (typeof input !== 'string' || input === '') return '[other]'; + const path = input.split(/[?#]/, 1)[0].replace(/\\/g, '/'); + for (const marker of ['/_next/static/', '/.next/server/', '/.next/static/']) { + const at = path.indexOf(marker); + if (at >= 0) { + const rest = path.slice(at); + return /^[\w./()[\]@+~-]+$/.test(rest) ? rest.replace('/.next/static/', '/_next/static/') : '[other]'; + } + } + if (/^node:[\w/.-]+$/.test(path)) return path; + const modules = path.lastIndexOf('/node_modules/'); + if (modules >= 0) { + const rest = path.slice(modules + 1); + return /^[\w./@+~-]+$/.test(rest) ? rest : '[other]'; + } + return '[other]'; +} + +/** A JMAP method name ("Email/get") or a fixed placeholder. */ +export function scrubMethodName(input: unknown): string { + return typeof input === 'string' && /^[A-Z][A-Za-z]{0,40}\/[a-z][A-Za-z]{0,40}$/.test(input) ? input : 'Unknown/method'; +} + +/** A JMAP error type ("serverFail", "overQuota") or a fixed placeholder. */ +export function scrubJmapErrorType(input: unknown): string { + return typeof input === 'string' && /^[a-z][A-Za-z]{1,40}$/.test(input) ? input : 'unknown'; +} diff --git a/lib/error-reporting/server.ts b/lib/error-reporting/server.ts new file mode 100644 index 000000000..d94e728c3 --- /dev/null +++ b/lib/error-reporting/server.ts @@ -0,0 +1,132 @@ +import { configManager } from '@/lib/admin/config-manager'; +import { recordError } from './aggregate'; +import { + appVersion, buildEnvelope, buildEvent, envelopeUrl, newEventId, parseDsn, releaseName, type ErrorReport, +} from './envelope'; +import { fingerprint } from './fingerprint'; +import { ReportLimiter } from './limiter'; +import { errorReportingSettings, shouldCollectErrors } from './settings'; +import { scrubName, scrubText } from './scrub'; +import { parseStack, type Frame } from './stack'; +import { debugIdsByPath } from './debug-ids'; + +/** + * Error reporting on the server: errors Next hands to onRequestError + * (instrumentation.ts) and errors routes catch and log with logger.error. + * While collection is on (a DSN, the metrics endpoint or error-count + * consent, lib/error-reporting/settings.ts) every one is counted in the + * per-day aggregate; with a DSN it is also sent there, at most once an hour + * per fingerprint and 60 a minute. Sending never blocks or fails the request + * that had the error. With everything off nothing happens at all. + */ + +const SEND_TIMEOUT_MS = 5000; +const ROUTE_PATTERN = /^\/[\w/[\]().@-]*$/; + +interface ServerState { + limiter: ReportLimiter; + sending: number; +} + +const KEY = Symbol.for('bulwark.error-reporting.server'); +function state(): ServerState { + const g = globalThis as typeof globalThis & { [KEY]?: ServerState }; + return (g[KEY] ??= { + limiter: new ReportLimiter({ perMinute: 60, perSession: Number.POSITIVE_INFINITY, dedupWindowMs: 60 * 60_000 }), + sending: 0, + }); +} + +async function deliver(report: ErrorReport): Promise { + if (!(await shouldCollectErrors())) return; + const version = appVersion(); + recordError(report, 'server', version); + const settings = errorReportingSettings(); + const dsn = parseDsn(settings.dsn); + if (!dsn) return; + const print = fingerprint(report.name, report.frames, version, report.code); + const s = state(); + if (s.sending > 4 || !s.limiter.allow(print)) return; + if (Math.random() >= settings.sampleRate) return; + const event = buildEvent(report, { + eventId: newEventId(), + timestamp: Date.now() / 1000, + release: releaseName(), + environment: settings.environment, + platform: 'node', + fingerprint: print, + debugIds: debugIdsByPath(), + tags: { runtime: 'server' }, + }); + s.sending++; + try { + await fetch(envelopeUrl(dsn), { + method: 'POST', + headers: { 'Content-Type': 'application/x-sentry-envelope' }, + body: buildEnvelope(event), + redirect: 'manual', + signal: AbortSignal.timeout(SEND_TIMEOUT_MS), + }); + } finally { + s.sending--; + } +} + +/** Count `report` and send it to the DSN when one is set. Never rejects. */ +export function handleServerReport(report: ErrorReport): Promise { + return deliver(report).catch(() => undefined); +} + +/** Next's onRequestError: an error a route, page or server action threw. */ +export function captureRequestError(error: unknown, routePattern?: string): Promise { + try { + const e = (error && typeof error === 'object' ? error : { message: String(error) }) as { name?: unknown; message?: unknown; stack?: unknown }; + return handleServerReport({ + name: scrubName(e.name), + message: scrubText(e.message), + frames: parseStack(e.stack), + handled: false, + mechanism: 'request', + // A route pattern from the code ("/api/admin/[id]"), never the request's URL. + ...(typeof routePattern === 'string' && ROUTE_PATTERN.test(routePattern) ? { route: routePattern } : {}), + }); + } catch { + return Promise.resolve(); + } +} + +/** + * logger.error: a route that caught an error and logged it. The frames are + * the logging call's, minus the logger's own `skip` frames. + */ +export function captureLoggedError( + message: string, + extra: Record | undefined, + stack: string | undefined, + skip = 2, +): Promise { + try { + const detail = typeof extra?.error === 'string' ? extra.error : extra?.error instanceof Error ? extra.error.message : ''; + const frames: Frame[] = parseStack(stack).slice(skip); + return handleServerReport({ + name: 'LoggedError', + message: scrubText(detail ? `${message}: ${detail}` : message), + frames, + handled: true, + mechanism: 'logger', + }); + } catch { + return Promise.resolve(); + } +} + +/** + * Hook the logger up (instrumentation.node.ts). The logger only knows a + * function on globalThis, so it stays free of config code and safe for any + * bundle that imports it. + */ +export async function installServerErrorReporting(): Promise { + await configManager.ensureLoaded(); + const { setErrorSink } = await import('@/lib/logger'); + setErrorSink((message, extra, stack) => { void captureLoggedError(message, extra, stack); }); +} diff --git a/lib/error-reporting/settings.ts b/lib/error-reporting/settings.ts new file mode 100644 index 000000000..c29a31470 --- /dev/null +++ b/lib/error-reporting/settings.ts @@ -0,0 +1,53 @@ +import { configManager } from '@/lib/admin/config-manager'; +import { parseDsn } from './envelope'; +import type { ClientSettings } from './client'; + +/** + * The error-reporting settings as the server resolves them: admin config, + * then ERROR_REPORTING_DSN(_FILE) / ERROR_REPORTING_SAMPLE_RATE / + * ERROR_REPORTING_ENVIRONMENT, then off. Bulwark Lite reads the same three + * from policy.json (lib/error-reporting/lite.ts). + */ + +export interface ErrorReportingSettings { + /** A valid DSN, or '' when none is set or the value doesn't parse. */ + dsn: string; + sampleRate: number; + environment: string; +} + +export function parseSampleRate(value: unknown): number { + const n = typeof value === 'number' ? value : typeof value === 'string' && value.trim() !== '' ? Number(value) : NaN; + if (!Number.isFinite(n)) return 1; + return Math.min(1, Math.max(0, n)); +} + +export function parseEnvironment(value: unknown): string { + const text = typeof value === 'string' ? value.trim() : ''; + return /^[\w.-]{1,64}$/.test(text) ? text : 'production'; +} + +export function errorReportingSettings(): ErrorReportingSettings { + const raw = configManager.get('errorReportingDsn', ''); + return { + dsn: parseDsn(raw) ? raw.trim() : '', + sampleRate: parseSampleRate(configManager.get('errorReportingSampleRate', '1')), + environment: parseEnvironment(configManager.get('errorReportingEnvironment', 'production')), + }; +} + +/** + * Whether browsers should post their errors to /api/errors: when there is a + * DSN to report to, a metrics endpoint that counts them, or consent to send + * error counts with the telemetry heartbeat. + */ +export async function shouldCollectErrors(): Promise { + await configManager.ensureLoaded(); + return !!errorReportingSettings().dsn; +} + +/** What /api/config hands the browser. */ +export async function browserErrorSettings(): Promise { + const settings = errorReportingSettings(); + return { ...settings, collect: await shouldCollectErrors() }; +} diff --git a/lib/error-reporting/stack.ts b/lib/error-reporting/stack.ts new file mode 100644 index 000000000..1c228eb99 --- /dev/null +++ b/lib/error-reporting/stack.ts @@ -0,0 +1,57 @@ +import { scrubFramePath, scrubFunctionName } from './scrub'; + +/** One stack frame, already scrubbed. */ +export interface Frame { + /** The file as scrubFramePath leaves it: `/_next/static/chunks/x.js`. */ + path: string; + line?: number; + column?: number; + fn?: string; +} + +export const MAX_FRAMES = 30; + +// V8 (Chrome, Edge, Node): " at fn (file:1:2)", " at file:1:2", +// " at async fn (file:1:2)", " at new Foo (file:1:2)". +const V8 = /^\s*at (?:(.+?) \()?(.+?):(\d+):(\d+)\)?\s*$/; +// SpiderMonkey and JavaScriptCore (Firefox, Safari): "fn@file:1:2", "@file:1:2". +const GECKO = /^\s*([^@]*)@(.+?):(\d+):(\d+)\s*$/; + +function number(value: string): number | undefined { + const n = Number(value); + return Number.isInteger(n) && n >= 0 && n < 10_000_000 ? n : undefined; +} + +/** The frames of a stack, the frame that threw first, at most MAX_FRAMES. */ +export function parseStack(stack: unknown): Frame[] { + if (typeof stack !== 'string') return []; + const frames: Frame[] = []; + for (const line of stack.split('\n')) { + const match = V8.exec(line) ?? GECKO.exec(line); + if (!match) continue; + const [, fn, file, lineNo, column] = match; + // "at fn (native)" and eval chains carry no file worth keeping. + frames.push({ + path: scrubFramePath(file), + line: number(lineNo), + column: number(column), + fn: scrubFunctionName(fn?.replace(/^async /, '')), + }); + if (frames.length >= MAX_FRAMES) break; + } + return frames; +} + +/** Frames handed in from outside (a browser's report), checked and scrubbed again. */ +export function sanitizeFrames(input: unknown): Frame[] { + if (!Array.isArray(input)) return []; + const out: Frame[] = []; + for (const raw of input.slice(0, MAX_FRAMES)) { + if (!raw || typeof raw !== 'object') continue; + const frame = raw as Record; + const line = typeof frame.line === 'number' ? number(String(frame.line)) : undefined; + const column = typeof frame.column === 'number' ? number(String(frame.column)) : undefined; + out.push({ path: scrubFramePath(frame.path), line, column, fn: scrubFunctionName(frame.fn) }); + } + return out; +} diff --git a/lib/jmap/client.ts b/lib/jmap/client.ts index 285bd9569..79cfb94a9 100644 --- a/lib/jmap/client.ts +++ b/lib/jmap/client.ts @@ -1,3 +1,4 @@ +import { recordJmapCall, reportJmapFailure } from "@/lib/error-reporting/client"; import { advertisesMailboxMetadata, bulwarkEntries, escapeMetadataPathSegment, MAX_METADATA_ENTRY_BYTES, METADATA_CAPABILITY, MetadataPatchError, MetadataRefusedError, MetadataTooLargeError, MetadataUnsupportedError, metadataValueBytes, namespaceFor } from "./bulwark-metadata"; import { IncompleteDeleteError } from "./delete-error"; import { ArchiveEmailsError } from "./archive-error"; @@ -189,6 +190,29 @@ function mailboxIdsReplacement( return { mailboxIds }; } +/** + * Statuses that are part of normal operation: an expired token (refreshed + * by the caller) and Stalwart's concurrency limit (retried above). + */ +const UNREPORTED_STATUSES = new Set([401, 429]); + +/** + * Method errors that are answers, not failures: the client resyncs, pages + * again or asks again with the new state. + */ +const UNREPORTED_METHOD_ERRORS = new Set(['cannotCalculateChanges', 'anchorNotFound', 'stateMismatch', 'tooManyChanges']); + +/** Report each `["error", {type}]` answer by method name and type (lib/error-reporting/client.ts). */ +function reportMethodErrors(methodCalls: JMAPMethodCall[], response: JMAPResponse): void { + for (const [name, result, callId] of response?.methodResponses ?? []) { + if (name !== 'error') continue; + const type = (result as { type?: unknown } | undefined)?.type; + if (typeof type !== 'string' || UNREPORTED_METHOD_ERRORS.has(type)) continue; + const method = methodCalls.find((call) => call[2] === callId)?.[0] ?? 'Unknown/method'; + reportJmapFailure({ method, type }); + } +} + export class RateLimitError extends Error { retryAfterMs: number; constructor(retryAfterMs: number) { @@ -2089,6 +2113,8 @@ export class JMAPClient implements IJMAPClient { if (!this.apiUrl) { throw new Error('Not connected. Call connect() first.'); } + // Error reports keep the method names of the last calls, nothing else. + recordJmapCall(methodCalls.map(([name]) => name)); const requestBody = { using: using || (hasSubmissionMethod(methodCalls) ? [...SUBMISSION_USING] : ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"]), @@ -2123,15 +2149,20 @@ export class JMAPClient implements IJMAPClient { continue; } console.error('Request failed:', response.status, responseText); + // The status only: the answer may quote what was sent. + if (!UNREPORTED_STATUSES.has(response.status)) reportJmapFailure({ status: response.status }); throw new Error(`Request failed: ${response.status} - ${responseText.substring(0, 200)}`); } + let parsed: JMAPResponse; try { - return JSON.parse(responseText); + parsed = JSON.parse(responseText); } catch { console.error('Failed to parse response:', responseText); throw new Error('Invalid JSON response from server'); } + reportMethodErrors(methodCalls, parsed); + return parsed; } } diff --git a/lib/logger.ts b/lib/logger.ts index e704e87d1..b3c87994b 100644 --- a/lib/logger.ts +++ b/lib/logger.ts @@ -40,7 +40,38 @@ function isJson(): boolean { return configured === 'json'; } +/** Whether log lines go out as JSON (LOG_FORMAT=json or the admin setting). */ +export function isJsonLogFormat(): boolean { + return isJson(); +} + +/** + * Gets every logger.error call, with the stack of the call. The server's + * error reporting sets it (lib/error-reporting/server.ts); without it errors + * are only logged. + */ +export type ErrorSink = (message: string, extra: Record | undefined, stack: string | undefined) => void; +const SINK_KEY = Symbol.for('bulwark.logger.errorSink'); +type GlobalWithSink = typeof globalThis & { [SINK_KEY]?: ErrorSink }; + +export function setErrorSink(sink: ErrorSink | null): void { + const target = globalThis as GlobalWithSink; + if (sink) target[SINK_KEY] = sink; + else delete target[SINK_KEY]; +} + function log(level: LogLevel, message: string, extra?: Record): void { + if (level === 'error') { + const sink = (globalThis as GlobalWithSink)[SINK_KEY]; + if (sink) { + try { + // Frames: log, logger.error, then the caller. + sink(message, extra, new Error().stack); + } catch { + // reporting must not break logging + } + } + } if (LEVELS[level] > getLevel()) return; if (isJson()) { diff --git a/lib/route-groups.ts b/lib/route-groups.ts new file mode 100644 index 000000000..e10e6c092 --- /dev/null +++ b/lib/route-groups.ts @@ -0,0 +1,75 @@ +/** + * A request path reduced to a fixed, small set of names: the area of the app + * or the API folder it belongs to. Error reports use it instead of the URL, + * and the request metrics use it as a label, so neither ever carries a folder + * name, a message id, a query string or anything else a user typed. + * + * lib/__tests__/route-groups.test.ts checks API_GROUPS against app/api and + * PAGE_AREAS against app/(main)/[locale]. + */ + +/** The folders under app/api. */ +export const API_GROUPS: readonly string[] = [ + 'account', 'account-archive', 'admin', 'auth', 'bimi', 'caldav', 'calendar-agenda', 'config', + 'connector', 'dev-jmap', 'errors', 'favicon', 'fetch-ical', 'health', 'og-image', + 'plugin-approval-status', 'plugin-proxy', 'plugin-secrets', 'plugin-signing-pubkey', 'plugins', + 'push', 'pwa-icon', 'pwa-screenshot', 'ready', 'settings', 'setup', 'system', 'translate', + 'webdav', 'wopi', +]; + +/** The page areas under app/(main)/[locale], plus the pages outside it. */ +export const PAGE_AREAS: readonly string[] = [ + 'apps', 'auth', 'calendar', 'contacts', 'files', 'login', 'mail', 'plugins', 'pro', 'settings', + 'admin', 'setup', 'connector', 'protocol', 'plugin-sandbox', 'plugin-sandbox-privileged', +]; + +const API = new Set(API_GROUPS); +const AREAS = new Set(PAGE_AREAS); +const ASSET_FOLDERS = new Set(['branding', 'notification', 'demo', 'plugin-sandbox-assets']); +/** en, de, pt-BR, zh-TW, sr-Latn: what a locale prefix looks like. */ +const LOCALE = /^[a-z]{2,3}(?:-[A-Za-z]{2,4})?$/; + +interface Parsed { + group: string; + /** Path segments below the group's own. */ + deeper: boolean; +} + +function parse(rawPath: string, basePath: string): Parsed { + let path = rawPath.split(/[?#]/, 1)[0] || '/'; + if (basePath && (path === basePath || path.startsWith(`${basePath}/`))) path = path.slice(basePath.length) || '/'; + const segments = path.split('/').filter(Boolean); + if (segments.length === 0) return { group: 'page:home', deeper: false }; + const [first, second] = segments; + if (first === '_next') { + return { group: second === 'static' ? 'static' : second === 'image' ? 'image' : 'other', deeper: false }; + } + if (first === 'api') { + return { group: second && API.has(second) ? `api:${second}` : 'api:other', deeper: segments.length > 2 }; + } + if ((segments.length === 1 && first.includes('.')) || ASSET_FOLDERS.has(first)) return { group: 'asset', deeper: false }; + const rest = LOCALE.test(first) && !AREAS.has(first) ? segments.slice(1) : segments; + if (rest.length === 0) return { group: 'page:home', deeper: false }; + return { group: AREAS.has(rest[0]) ? `page:${rest[0]}` : 'page:other', deeper: rest.length > 1 }; +} + +/** + * The group of a path: `api:`, `page:`, `page:home`, `static`, + * `image`, `asset` or `other`. `basePath` is removed first. + */ +export function routeGroup(rawPath: string, basePath = ''): string { + return parse(rawPath, basePath).group; +} + +/** + * A path for an error report: the group as a path, and `/…` when the path + * went deeper. `/de/mail/folder/Projects%20ACME/abc` -> `/mail/…`. + */ +export function scrubRoutePath(rawPath: string, basePath = ''): string { + const { group, deeper } = parse(rawPath, basePath); + const base = group === 'page:home' ? '/' + : group.startsWith('api:') ? `/api/${group.slice(4)}` + : group.startsWith('page:') ? `/${group.slice(5)}` + : `/${group}`; + return deeper ? `${base}/…` : base; +} diff --git a/lib/shared-store/server-state-allowlist.ts b/lib/shared-store/server-state-allowlist.ts index 303984d17..2384bf964 100644 --- a/lib/shared-store/server-state-allowlist.ts +++ b/lib/shared-store/server-state-allowlist.ts @@ -35,6 +35,7 @@ export const SERVER_STATE_ALLOWLIST: AllowedState[] = [ { site: 'app/api/favicon/route.ts#cache', class: 'cache', why: 'sender icons from a public service, 14 days' }, { site: 'app/api/favicon/route.ts#negativeCache', class: 'cache', why: 'sender icons that failed, 1 day' }, { site: 'app/api/ready/route.ts#last', class: 'cache', why: 'the last store ping, 1 s, so probes can\'t flood the store' }, + { site: 'lib/error-reporting/debug-ids.ts#cache', class: 'per-process', why: 'debug ids of the bundle files this process loaded, read from globalThis' }, { site: 'lib/admin/directory-keys.ts#keyringCache', class: 'cache', why: 'extension directory keys, 10 min; the anti-rollback record is in the store' }, { site: 'lib/admin/paths.ts#cachedReadOnly', class: 'per-process', why: 'parsed ADMIN_CONFIG_READONLY env' }, { site: 'lib/admin/plugin-approvals.ts#maxPackageBytes', class: 'per-process', why: 'a limit tests can lower' }, diff --git a/lib/telemetry/state.ts b/lib/telemetry/state.ts index 9a82f2c2a..22a6e3645 100644 --- a/lib/telemetry/state.ts +++ b/lib/telemetry/state.ts @@ -13,15 +13,16 @@ export function getDir(): string { } /** - * Telemetry's documents (#1066): `state.json`, `logins.json` and the - * instance id `.telemetry-id`. Local mode: today's files in the telemetry - * dir. Shared mode: one copy for the whole deployment, so every replica - * reports as the same instance with the same counts. + * Telemetry's documents (#1066): `state.json`, `logins.json`, the instance + * id `.telemetry-id` and the per-day error counts `errors.json` + * (lib/error-reporting/aggregate.ts). Local mode: today's files in the + * telemetry dir. Shared mode: one copy for the whole deployment, so every + * replica reports as the same instance with the same counts. */ export const TELEMETRY_COLLECTION: CollectionOptions = { local: { dir: () => getDir(), - idOf: (relative) => (['state.json', 'logins.json', '.telemetry-id'].includes(relative) ? relative : null), + idOf: (relative) => (['state.json', 'logins.json', '.telemetry-id', 'errors.json'].includes(relative) ? relative : null), }, }; diff --git a/stores/policy-store.ts b/stores/policy-store.ts index 6c0a6945a..b2e188573 100644 --- a/stores/policy-store.ts +++ b/stores/policy-store.ts @@ -4,6 +4,8 @@ import { DEFAULT_POLICY, DEFAULT_THEME_POLICY, POLICY_DOMAINS_HEADER, POLICY_SCO import { apiFetch } from '@/lib/browser-navigation'; import { IS_LITE, IS_LITE_STALWART, LITE_POLICY_PATH, withLiteBuildId } from '@/lib/lite'; import { applyLitePolicy } from '@/lib/lite-config'; +import { configureErrorReporting } from '@/lib/error-reporting/client'; +import { liteErrorReporting } from '@/lib/error-reporting/lite'; import { resolvePolicy, type PolicySource, type PolicySubject } from '@/lib/admin/policy-scopes'; import { DEFAULT_ORGANISATION_POLICY, type OrganisationPolicy } from '@/lib/admin/organisation-policy'; @@ -115,11 +117,15 @@ export const usePolicyStore = create()((set, get) => ({ const partial = res.headers?.get?.(POLICY_SCOPE_HEADER) === 'public'; const domainsHeader = res.headers?.get?.(POLICY_DOMAINS_HEADER); const scopeDomains = typeof domainsHeader === 'string' ? domainsHeader.split(',').filter(Boolean) : null; + // Lite has no /api/config: error reporting is set in policy.json. + if (IS_LITE) configureErrorReporting(liteErrorReporting(data)); set({ ...resolved(IS_LITE ? applyLitePolicy(data) : data, get().subject), loaded: true, partial, scopeDomains }); } else { + if (IS_LITE) configureErrorReporting(null); set({ ...(fallback.policy ? resolved(fallback.policy, get().subject) : {}), loaded: true }); } } catch { + if (IS_LITE && request === policyRequest) configureErrorReporting(null); if (request === policyRequest) set({ ...(fallback.policy ? resolved(fallback.policy, get().subject) : {}), loaded: true }); } }, From 8308e5e858f05b3827a8f0c444e758a4d93cbeef Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:52:37 +0100 Subject: [PATCH 03/26] feat: add Kubernetes manifests for several replicas sharing one Valkey #1066 --- .gitignore | 3 + deploy/kubernetes/base/bulwark.env | 13 ++ deploy/kubernetes/base/bulwark.yaml | 144 ++++++++++++++++++ deploy/kubernetes/base/kustomization.yaml | 32 ++++ deploy/kubernetes/base/namespace.yaml | 4 + deploy/kubernetes/base/secrets.env.example | 13 ++ deploy/kubernetes/base/valkey.yaml | 95 ++++++++++++ deploy/kubernetes/ingress.example.yaml | 28 ++++ deploy/kubernetes/kustomization.yaml | 14 ++ .../config-in-git/config-from-files.yaml | 20 +++ .../overlays/config-in-git/config.json | 4 + .../overlays/config-in-git/kustomization.yaml | 23 +++ .../overlays/config-in-git/policy.json | 5 + deploy/kubernetes/test/smoke.sh | 120 +++++++++++++++ 14 files changed, 518 insertions(+) create mode 100644 deploy/kubernetes/base/bulwark.env create mode 100644 deploy/kubernetes/base/bulwark.yaml create mode 100644 deploy/kubernetes/base/kustomization.yaml create mode 100644 deploy/kubernetes/base/namespace.yaml create mode 100644 deploy/kubernetes/base/secrets.env.example create mode 100644 deploy/kubernetes/base/valkey.yaml create mode 100644 deploy/kubernetes/ingress.example.yaml create mode 100644 deploy/kubernetes/kustomization.yaml create mode 100644 deploy/kubernetes/overlays/config-in-git/config-from-files.yaml create mode 100644 deploy/kubernetes/overlays/config-in-git/config.json create mode 100644 deploy/kubernetes/overlays/config-in-git/kustomization.yaml create mode 100644 deploy/kubernetes/overlays/config-in-git/policy.json create mode 100644 deploy/kubernetes/test/smoke.sh diff --git a/.gitignore b/.gitignore index a67d67b1c..19e0f5fa8 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,6 @@ next-env.d.ts # Sibling repos /repos/ + +# Kubernetes reference manifests: the secrets users fill in +deploy/kubernetes/base/secrets.env diff --git a/deploy/kubernetes/base/bulwark.env b/deploy/kubernetes/base/bulwark.env new file mode 100644 index 000000000..aa88c979d --- /dev/null +++ b/deploy/kubernetes/base/bulwark.env @@ -0,0 +1,13 @@ +# Settings for every Bulwark replica (no secrets here; those go in secrets.env). +# Any variable from https://bulwarkmail.org/docs/getting-started/configuration/environment-reference +# can be added. + +# The JMAP server users sign in to. Both the pods and users' browsers must reach it. +JMAP_SERVER_URL=https://mail.example.com + +# Proxies in front of the pods: 1 for an ingress controller alone, 2 if a cloud +# load balancer sits in front of it. Sign-in limits key on the address this picks. +TRUSTED_PROXY_DEPTH=1 + +SETTINGS_SYNC_ENABLED=true +LOG_LEVEL=info diff --git a/deploy/kubernetes/base/bulwark.yaml b/deploy/kubernetes/base/bulwark.yaml new file mode 100644 index 000000000..5d40e9c12 --- /dev/null +++ b/deploy/kubernetes/base/bulwark.yaml @@ -0,0 +1,144 @@ +# Bulwark itself: any number of replicas, no volumes. Everything they share +# (admin config, plugins, branding, synced settings, sign-in limits, sessions) +# lives in the Valkey store; see https://bulwarkmail.org/docs/deployment/multi-instance +apiVersion: v1 +kind: Service +metadata: + name: bulwark + labels: + app.kubernetes.io/name: bulwark +spec: + selector: + app.kubernetes.io/name: bulwark + ports: + - name: http + port: 80 + targetPort: http +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: bulwark + labels: + app.kubernetes.io/name: bulwark +spec: + replicas: 3 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + selector: + matchLabels: + app.kubernetes.io/name: bulwark + template: + metadata: + labels: + app.kubernetes.io/name: bulwark + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: 1001 + seccompProfile: + type: RuntimeDefault + # Spread the replicas over nodes where there are several. + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/name: bulwark + terminationGracePeriodSeconds: 30 + containers: + - name: bulwark + image: ghcr.io/bulwarkmail/webmail + ports: + - name: http + containerPort: 3000 + envFrom: + - configMapRef: + name: bulwark-env + env: + - name: SESSION_SECRET + valueFrom: + secretKeyRef: + name: bulwark-secrets + key: SESSION_SECRET + - name: ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: bulwark-secrets + key: ADMIN_PASSWORD + - name: VALKEY_PASSWORD + valueFrom: + secretKeyRef: + name: bulwark-secrets + key: VALKEY_PASSWORD + # Kubernetes fills in $(VALKEY_PASSWORD) from the variable above. + - name: REDIS_URL + value: redis://:$(VALKEY_PASSWORD)@valkey:6379/0 + # /api/health only says the process runs; /api/ready also says it + # finished starting and reaches the store. A store outage takes the + # pods out of the Service without restarting them. + startupProbe: + httpGet: + path: /api/health + port: http + periodSeconds: 2 + failureThreshold: 60 + readinessProbe: + httpGet: + path: /api/ready + port: http + periodSeconds: 5 + failureThreshold: 2 + livenessProbe: + httpGet: + path: /api/health + port: http + periodSeconds: 10 + failureThreshold: 3 + lifecycle: + # Let the Service drop the pod before it stops taking requests. + preStop: + exec: + command: ["sleep", "5"] + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 768Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + # Scratch space only; nothing here needs to survive the pod. + - name: data + mountPath: /app/data + - name: next-cache + mountPath: /app/.next/cache + - name: tmp + mountPath: /tmp + volumes: + - name: data + emptyDir: {} + - name: next-cache + emptyDir: {} + - name: tmp + emptyDir: {} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: bulwark +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: bulwark diff --git a/deploy/kubernetes/base/kustomization.yaml b/deploy/kubernetes/base/kustomization.yaml new file mode 100644 index 000000000..e7b76c1ce --- /dev/null +++ b/deploy/kubernetes/base/kustomization.yaml @@ -0,0 +1,32 @@ +# The base the top-level kustomization.yaml and the overlays use. Change the +# namespace, the image tag (pin a release, never latest in production) and +# the replica count here or, better, in an overlay. secrets.env must exist +# next to this file (copy secrets.env.example). +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: bulwark + +resources: + - namespace.yaml + - valkey.yaml + - bulwark.yaml + +configMapGenerator: + - name: bulwark-env + envs: + - bulwark.env + +secretGenerator: + - name: bulwark-secrets + envs: + - secrets.env + +images: + - name: ghcr.io/bulwarkmail/webmail + newTag: latest + +labels: + - pairs: + app.kubernetes.io/part-of: bulwark + includeSelectors: false diff --git a/deploy/kubernetes/base/namespace.yaml b/deploy/kubernetes/base/namespace.yaml new file mode 100644 index 000000000..ff5226df2 --- /dev/null +++ b/deploy/kubernetes/base/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: bulwark diff --git a/deploy/kubernetes/base/secrets.env.example b/deploy/kubernetes/base/secrets.env.example new file mode 100644 index 000000000..24cb08cf1 --- /dev/null +++ b/deploy/kubernetes/base/secrets.env.example @@ -0,0 +1,13 @@ +# Copy to secrets.env (kept out of git) and fill in. Generate each with: +# openssl rand -base64 32 +# Or delete the secretGenerator in kustomization.yaml and create the Secret +# `bulwark-secrets` with these three keys some other way (External Secrets, +# Sealed Secrets, your CI). + +# Encrypts sessions, synced settings and plugin credentials. The same on every +# replica; changing it signs everyone out. +SESSION_SECRET= +# First admin password; hashed into the store on the first start. +ADMIN_PASSWORD= +# Password of the Valkey store. +VALKEY_PASSWORD= diff --git a/deploy/kubernetes/base/valkey.yaml b/deploy/kubernetes/base/valkey.yaml new file mode 100644 index 000000000..debc8bb92 --- /dev/null +++ b/deploy/kubernetes/base/valkey.yaml @@ -0,0 +1,95 @@ +# The shared store every Bulwark replica uses. One Valkey with persistence +# (AOF) and noeviction: an evicting policy would silently drop settings and +# admin config under memory pressure. A managed Redis or Valkey works as well; +# then delete this file from kustomization.yaml and point REDIS_URL at it. +apiVersion: v1 +kind: Service +metadata: + name: valkey + labels: + app.kubernetes.io/name: valkey +spec: + clusterIP: None + selector: + app.kubernetes.io/name: valkey + ports: + - name: redis + port: 6379 + targetPort: redis +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: valkey + labels: + app.kubernetes.io/name: valkey +spec: + serviceName: valkey + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: valkey + template: + metadata: + labels: + app.kubernetes.io/name: valkey + spec: + securityContext: + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + fsGroup: 999 + seccompProfile: + type: RuntimeDefault + containers: + - name: valkey + image: valkey/valkey:8-alpine + command: ["sh", "-c"] + args: + - >- + exec valkey-server + --requirepass "$VALKEY_PASSWORD" + --appendonly yes + --maxmemory 512mb + --maxmemory-policy noeviction + --dir /data + env: + - name: VALKEY_PASSWORD + valueFrom: + secretKeyRef: + name: bulwark-secrets + key: VALKEY_PASSWORD + ports: + - name: redis + containerPort: 6379 + readinessProbe: + exec: + command: ["sh", "-c", "REDISCLI_AUTH=\"$VALKEY_PASSWORD\" valkey-cli --no-auth-warning ping | grep -q PONG"] + periodSeconds: 5 + livenessProbe: + exec: + command: ["sh", "-c", "REDISCLI_AUTH=\"$VALKEY_PASSWORD\" valkey-cli --no-auth-warning ping | grep -q PONG"] + periodSeconds: 10 + failureThreshold: 6 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + memory: 640Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: data + mountPath: /data + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi diff --git a/deploy/kubernetes/ingress.example.yaml b/deploy/kubernetes/ingress.example.yaml new file mode 100644 index 000000000..fa94692a3 --- /dev/null +++ b/deploy/kubernetes/ingress.example.yaml @@ -0,0 +1,28 @@ +# Not applied by kustomization.yaml: adapt and apply it yourself, or use a +# Gateway API HTTPRoute instead. Plain round robin is fine; Bulwark needs no +# sticky sessions. Set TRUSTED_PROXY_DEPTH in bulwark.env to the number of +# proxies in front of the pods (1 for the ingress controller alone). +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: bulwark + namespace: bulwark + annotations: + # Attachments and plugin packages can be large. + nginx.ingress.kubernetes.io/proxy-body-size: 100m +spec: + ingressClassName: nginx + tls: + - hosts: [webmail.example.com] + secretName: bulwark-tls + rules: + - host: webmail.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: bulwark + port: + name: http diff --git a/deploy/kubernetes/kustomization.yaml b/deploy/kubernetes/kustomization.yaml new file mode 100644 index 000000000..006ce308b --- /dev/null +++ b/deploy/kubernetes/kustomization.yaml @@ -0,0 +1,14 @@ +# Bulwark Webmail on Kubernetes: three replicas sharing one Valkey store. +# https://bulwarkmail.org/docs/deployment/kubernetes +# +# cp base/secrets.env.example base/secrets.env # fill in the three secrets +# $EDITOR base/bulwark.env # at least JMAP_SERVER_URL +# kubectl apply -k . # or: -k overlays/config-in-git +# +# For your own changes (namespace, image tag, replicas, resources) add an +# overlay next to overlays/config-in-git that uses ../../base. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - base diff --git a/deploy/kubernetes/overlays/config-in-git/config-from-files.yaml b/deploy/kubernetes/overlays/config-in-git/config-from-files.yaml new file mode 100644 index 000000000..fe91e62f7 --- /dev/null +++ b/deploy/kubernetes/overlays/config-in-git/config-from-files.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: bulwark +spec: + template: + spec: + containers: + - name: bulwark + env: + - name: ADMIN_CONFIG_READONLY + value: "true" + volumeMounts: + - name: admin-config + mountPath: /app/data/admin + readOnly: true + volumes: + - name: admin-config + configMap: + name: bulwark-admin-config diff --git a/deploy/kubernetes/overlays/config-in-git/config.json b/deploy/kubernetes/overlays/config-in-git/config.json new file mode 100644 index 000000000..e0754da49 --- /dev/null +++ b/deploy/kubernetes/overlays/config-in-git/config.json @@ -0,0 +1,4 @@ +{ + "appName": "Webmail", + "setupComplete": true +} diff --git a/deploy/kubernetes/overlays/config-in-git/kustomization.yaml b/deploy/kubernetes/overlays/config-in-git/kustomization.yaml new file mode 100644 index 000000000..ab85f582b --- /dev/null +++ b/deploy/kubernetes/overlays/config-in-git/kustomization.yaml @@ -0,0 +1,23 @@ +# Admin config and user policy from files in git instead of the admin console. +# Every replica reads config.json and policy.json from the ConfigMap and picks +# up a change within a few seconds; the admin console shows them but can't +# save them. Sessions, settings and the audit log still live in the store. +# +# kubectl apply -k overlays/config-in-git +# +# Export your current policy from the admin console (Organisation > Export) +# into policy.json to start from it. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +configMapGenerator: + - name: bulwark-admin-config + files: + - config.json + - policy.json + +patches: + - path: config-from-files.yaml diff --git a/deploy/kubernetes/overlays/config-in-git/policy.json b/deploy/kubernetes/overlays/config-in-git/policy.json new file mode 100644 index 000000000..d5a66620e --- /dev/null +++ b/deploy/kubernetes/overlays/config-in-git/policy.json @@ -0,0 +1,5 @@ +{ + "restrictions": {}, + "features": {}, + "defaults": {} +} diff --git a/deploy/kubernetes/test/smoke.sh b/deploy/kubernetes/test/smoke.sh new file mode 100644 index 000000000..9d097aaa4 --- /dev/null +++ b/deploy/kubernetes/test/smoke.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# Smoke test for a running deployment of deploy/kubernetes: +# - Valkey and every Bulwark replica become ready, on the shared store; +# - an admin session made on one replica is valid on every other one; +# - a config change saved on one replica reaches the others within 5 s; +# - deleting a pod and a rolling restart don't fail requests through the +# Service, and the admin session survives the new pods. +# +# Usage: ADMIN_PASSWORD=... deploy/kubernetes/test/smoke.sh [namespace] +# With READ_ONLY_CONFIG=1 (overlays/config-in-git) it checks that config saves +# are refused instead of spread. +# Uses the current kubectl context. The password goes to curl on stdin, never +# on a command line. +set -euo pipefail + +NS="${1:-bulwark}" +: "${ADMIN_PASSWORD:?set ADMIN_PASSWORD to the admin password of the deployment}" +k() { kubectl -n "$NS" "$@"; } +fail() { echo "FAIL: $*" >&2; exit 1; } +step() { echo "== $*"; } + +step "rollout" +k rollout status statefulset/valkey --timeout=300s +k rollout status deployment/bulwark --timeout=600s + +step "curl pod" +if ! k get pod smoke-curl >/dev/null 2>&1; then + k run smoke-curl --image=curlimages/curl:8.11.1 --restart=Never --labels=app.kubernetes.io/name=smoke-curl \ + --command -- sleep 1800 >/dev/null +fi +k wait pod/smoke-curl --for=condition=Ready --timeout=180s >/dev/null +c() { k exec -i smoke-curl -- curl -sS --max-time 10 "$@"; } +status() { k exec smoke-curl -- curl -s -o /dev/null -w '%{http_code}' --max-time 5 "$@" || echo 000; } + +pod_ips() { k get pods -l app.kubernetes.io/name=bulwark --field-selector=status.phase=Running -o jsonpath='{.items[*].status.podIP}'; } +read -r -a IPS <<<"$(pod_ips)" +[ "${#IPS[@]}" -ge 2 ] || fail "need at least 2 replicas, found ${#IPS[@]}" +echo "replicas: ${IPS[*]}" + +step "every replica is ready on the shared store" +for ip in "${IPS[@]}"; do + out="$(c "http://$ip:3000/api/ready")" + echo "$ip $out" + grep -q '"ready":true' <<<"$out" || fail "$ip not ready" + grep -q '"store":"redis"' <<<"$out" || fail "$ip not on the shared store" +done + +step "an admin session made on one replica works on every replica" +json_string() { + local s="${1//\\/\\\\}" + s="${s//\"/\\\"}" + printf '"%s"' "$s" +} +code="$(printf '{"password":%s}' "$(json_string "$ADMIN_PASSWORD")" \ + | c -o /dev/null -w '%{http_code}' -c /tmp/jar -H 'Content-Type: application/json' --data-binary @- \ + "http://${IPS[0]}:3000/api/admin/auth")" +[ "$code" = 200 ] || fail "admin sign-in on ${IPS[0]} answered $code" +# curl's jar keeps cookies per host and every pod has its own address, so the +# cookie goes in a header. It stays in the curl pod, off every command line. +k exec smoke-curl -- sh -c "awk '{ sub(/^#HttpOnly_/, \"\") } !/^#/ && NF >= 7 { printf \"%s=%s; \", \$6, \$7 }' /tmp/jar > /tmp/cookie" +ca() { k exec -i smoke-curl -- sh -c 'curl -sS --max-time 10 -H "Cookie: $(cat /tmp/cookie)" "$@"' sh "$@"; } +for ip in "${IPS[@]}"; do + ca "http://$ip:3000/api/admin/auth" /dev/null; do + total=$((total + 1)) + [ "$(status "$SVC")" = 200 ] || failures=$((failures + 1)) + sleep 1 + done + wait "$pid" + echo "requests: $total, failed: $failures" + [ "$failures" -eq 0 ] || fail "$failures requests failed" +} + +step "deleting a pod keeps the Service answering" +victim="$(k get pods -l app.kubernetes.io/name=bulwark -o jsonpath='{.items[0].metadata.name}')" +probe_while k delete pod "$victim" --wait=true +k rollout status deployment/bulwark --timeout=300s + +step "a rolling restart keeps the Service answering" +k rollout restart deployment/bulwark +probe_while k rollout status deployment/bulwark --timeout=600s + +step "the admin session survives the new pods" +ca "http://bulwark.$NS.svc/api/admin/auth" /dev/null +echo "OK" From cffa8bb1942483dd16f3fbc6aa55dc19b62b09b5 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:52:37 +0100 Subject: [PATCH 04/26] ci: deploy the Kubernetes manifests to kind and smoke-test several replicas --- .github/workflows/kubernetes.yml | 99 ++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 .github/workflows/kubernetes.yml diff --git a/.github/workflows/kubernetes.yml b/.github/workflows/kubernetes.yml new file mode 100644 index 000000000..d521c4341 --- /dev/null +++ b/.github/workflows/kubernetes.yml @@ -0,0 +1,99 @@ +name: "CI: Kubernetes manifests" + +# Builds the image from this commit, deploys deploy/kubernetes to a kind +# cluster (three replicas and one Valkey) and runs +# deploy/kubernetes/test/smoke.sh against it: every replica ready on the +# shared store, one admin session valid everywhere, a config change reaching +# every replica, no failed requests while a pod is deleted and during a +# rolling restart. The config-in-git overlay runs next to it in its own +# namespace and must refuse config saves. +# +# It runs when the manifests change, weekly on main (an app change can break +# several replicas without touching the manifests), and on demand: +# gh workflow run kubernetes.yml + +on: + pull_request: + paths: + - deploy/kubernetes/** + - .github/workflows/kubernetes.yml + push: + branches: [main] + paths: + - deploy/kubernetes/** + - .github/workflows/kubernetes.yml + schedule: + - cron: "41 3 * * 1" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: kubernetes-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + kind: + name: kind + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + KIND_VERSION: v0.29.0 + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Build the image + run: docker build --build-arg "GIT_COMMIT=${GITHUB_SHA}" -t bulwark-local:ci . + + - name: Create the cluster + run: | + curl -fsSLo /usr/local/bin/kind "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64" + chmod +x /usr/local/bin/kind + kind create cluster --name ci --wait 180s + kind load docker-image bulwark-local:ci --name ci + + - name: Deploy the base and the config-in-git overlay + run: | + umask 077 + admin="$(openssl rand -hex 16)" + echo "::add-mask::$admin" + { + echo "SESSION_SECRET=$(openssl rand -base64 32)" + echo "ADMIN_PASSWORD=$admin" + echo "VALKEY_PASSWORD=$(openssl rand -hex 24)" + } > deploy/kubernetes/base/secrets.env + echo "ADMIN_PASSWORD=$admin" >> "$GITHUB_ENV" + for target in base:bulwark overlays/config-in-git:bulwark-ro; do + src="${target%%:*}"; ns="${target##*:}" + mkdir -p "ci/$ns" + cat > "ci/$ns/kustomization.yaml" < Date: Sun, 1 Nov 2026 02:53:49 +0100 Subject: [PATCH 05/26] docs: name the policy export button in the config-in-git overlay --- deploy/kubernetes/overlays/config-in-git/kustomization.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/kubernetes/overlays/config-in-git/kustomization.yaml b/deploy/kubernetes/overlays/config-in-git/kustomization.yaml index ab85f582b..9f604dba2 100644 --- a/deploy/kubernetes/overlays/config-in-git/kustomization.yaml +++ b/deploy/kubernetes/overlays/config-in-git/kustomization.yaml @@ -5,8 +5,8 @@ # # kubectl apply -k overlays/config-in-git # -# Export your current policy from the admin console (Organisation > Export) -# into policy.json to start from it. +# To start from your current policy, use "Export policy.json" on the admin +# console's Domains and groups page. apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization From 481c4b6030cdb0d0a9ae3915cdbf62ca21674142 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:59:40 +0100 Subject: [PATCH 06/26] feat: serve Prometheus metrics and split health into live and ready - `/api/metrics` in Prometheus text format, off until `METRICS_TOKEN` or `METRICS_TOKEN_FILE` is set (404 before, 401 without the bearer token): requests per route group with status class and latency, sign-in outcomes, calls to Stalwart, WOPI, translation, the plugin proxy and OAuth, browser errors, errors today, process memory, event-loop lag, uptime and build info. Labels come from fixed lists. Each replica serves its own. - `/api/health/live` answers while the process runs; `/api/health/ready` checks the config, the shared store, writable data dirs and Stalwart's session resource. `/api/health` and `/api/ready` are unchanged. - the Docker image gets a HEALTHCHECK on the live endpoint - adds prom-client, and the OpenTelemetry packages the tracing step loads on demand --- .dockerignore | 1 + Dockerfile | 6 + app/(main)/admin/_pages/monitoring.tsx | 20 ++ app/api/admin/monitoring/route.ts | 6 +- app/api/auth/token/route.ts | 12 +- app/api/errors/__tests__/route.test.ts | 15 ++ app/api/errors/route.ts | 2 + app/api/health/live/route.ts | 18 ++ app/api/health/ready/route.ts | 23 ++ app/api/metrics/__tests__/route.test.ts | 55 ++++ app/api/metrics/route.ts | 41 +++ app/api/translate/route.ts | 9 +- instrumentation.node.ts | 8 + lib/admin/admin-nav.ts | 2 + lib/admin/outgoing-hosts-registry.ts | 2 +- lib/error-reporting/settings.ts | 3 +- lib/health/__tests__/readiness.test.ts | 86 +++++++ lib/health/readiness.ts | 131 ++++++++++ lib/metrics/__tests__/metrics.test.ts | 93 +++++++ lib/metrics/__tests__/upstream-wiring.test.ts | 29 +++ lib/metrics/http.ts | 39 +++ lib/metrics/index.ts | 147 +++++++++++ lib/metrics/upstream.ts | 19 ++ lib/oauth/token-exchange.ts | 5 +- lib/plugin-secrets/proxy.ts | 7 +- lib/route-groups.ts | 2 +- lib/stalwart/server-fetch.ts | 3 +- lib/wopi/discovery.ts | 5 +- next.config.ts | 6 +- package-lock.json | 235 ++++++++++++++++++ package.json | 8 + proxy.ts | 7 +- scripts/healthcheck.mjs | 15 ++ 33 files changed, 1034 insertions(+), 26 deletions(-) create mode 100644 app/api/health/live/route.ts create mode 100644 app/api/health/ready/route.ts create mode 100644 app/api/metrics/__tests__/route.test.ts create mode 100644 app/api/metrics/route.ts create mode 100644 lib/health/__tests__/readiness.test.ts create mode 100644 lib/health/readiness.ts create mode 100644 lib/metrics/__tests__/metrics.test.ts create mode 100644 lib/metrics/__tests__/upstream-wiring.test.ts create mode 100644 lib/metrics/http.ts create mode 100644 lib/metrics/index.ts create mode 100644 lib/metrics/upstream.ts create mode 100644 scripts/healthcheck.mjs diff --git a/.dockerignore b/.dockerignore index 68da62e5b..0942bf880 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,6 +10,7 @@ lite-image scripts/ !scripts/lite/ !scripts/build-plugin-sandbox.mjs +!scripts/healthcheck.mjs # Built in the image by scripts/build-plugin-sandbox.mjs public/plugin-sandbox-assets/ TODO.md diff --git a/Dockerfile b/Dockerfile index 515c30937..fe9f6f986 100644 --- a/Dockerfile +++ b/Dockerfile @@ -56,6 +56,7 @@ RUN apk upgrade --no-cache && \ COPY --from=builder /app/public ./public COPY --from=builder /app/.next/standalone ./ COPY --from=builder /app/.next/static ./.next/static +COPY --from=builder /app/scripts/healthcheck.mjs ./healthcheck.mjs # Every data dir exists up front and belongs to the runtime user, so a fresh # volume on /app/data (Docker copies these into it) or on any one of them is # writable. plugin-secrets holds users' plugin credentials, version-check the @@ -68,4 +69,9 @@ USER nextjs EXPOSE 3000 ENV PORT=3000 ENV HOSTNAME="0.0.0.0" +# The health check needs the base path the bundle was built with. +ARG NEXT_PUBLIC_BASE_PATH= +ENV BULWARK_BASE_PATH=$NEXT_PUBLIC_BASE_PATH +# Liveness only (/api/health/live); readiness is /api/health/ready. +HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 CMD ["node", "healthcheck.mjs"] CMD ["node", "server.js"] diff --git a/app/(main)/admin/_pages/monitoring.tsx b/app/(main)/admin/_pages/monitoring.tsx index 268740b69..24c4f8044 100644 --- a/app/(main)/admin/_pages/monitoring.tsx +++ b/app/(main)/admin/_pages/monitoring.tsx @@ -31,6 +31,7 @@ interface MonitoringState { collecting: boolean; day: string; errorsToday: { total: number; top: ErrorEntry[] }; + metrics?: { enabled: boolean }; } const PAGE = adminPageDef('monitoring'); @@ -161,6 +162,25 @@ export function MonitoringPage() { )} + + +

+ {state?.metrics?.enabled + ? <>On. Prometheus scrapes /api/metrics on each replica with the token as a bearer token. + : <>Off. Set METRICS_TOKEN (or METRICS_TOKEN_FILE) in the environment to turn on /api/metrics.} +

+ + Requests per route group, sign-ins, calls to Stalwart and other services, browser errors, memory and + event-loop lag. Metric list and scrape config + + + + +
    +
  • /api/health/live: the process answers. For liveness probes and the Docker health check.
    • +
  • /api/health/ready: config loaded, data dirs writable, Stalwart reachable. For readiness probes.
    • +
+ Monitoring shows besides the * config keys (those come from /api/admin/config): whether errors are being - * counted and today's counts. + * counted, today's counts and whether the metrics endpoint is on. */ export async function GET(request: NextRequest) { try { @@ -20,7 +21,8 @@ export async function GET(request: NextRequest) { const collecting = await shouldCollectErrors(); const today = utcDay(); const errorsToday = summarizeDay(await readErrors(), today); - return NextResponse.json({ collecting, day: today, errorsToday }, { headers: { 'Cache-Control': 'no-store' } }); + const metrics = { enabled: metricsEnabled() }; + return NextResponse.json({ collecting, day: today, errorsToday, metrics }, { headers: { 'Cache-Control': 'no-store' } }); } catch (err) { logger.error('monitoring GET error', { error: err instanceof Error ? err.message : 'unknown' }); const unavailable = storedDataUnavailableResponse(err); diff --git a/app/api/auth/token/route.ts b/app/api/auth/token/route.ts index 389599afa..6c065e524 100644 --- a/app/api/auth/token/route.ts +++ b/app/api/auth/token/route.ts @@ -1,4 +1,5 @@ import { NextRequest, NextResponse } from 'next/server'; +import { observeUpstream } from '@/lib/metrics/upstream'; import { cookies } from 'next/headers'; import { logger } from '@/lib/logger'; import { @@ -57,7 +58,8 @@ function getSlot(request: NextRequest): number { const REVOCATION_TIMEOUT_MS = 3000; async function revokeRefreshToken(token: string, serverId: string | null, metadata: OAuthMetadata | null): Promise { - if (!metadata?.revocation_endpoint) return; + const revocationEndpoint = metadata?.revocation_endpoint; + if (!revocationEndpoint) return; const params = buildOAuthParams({ token, token_type_hint: 'refresh_token', @@ -66,13 +68,13 @@ async function revokeRefreshToken(token: string, serverId: string | null, metada try { // Never follow a redirect with the refresh token and client secret in // the body: the endpoint was validated, its redirect target was not. - const revocationResponse = await fetch(metadata.revocation_endpoint, { + const revocationResponse = await observeUpstream('oauth', () => fetch(revocationEndpoint, { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: params.toString(), signal: AbortSignal.timeout(REVOCATION_TIMEOUT_MS), redirect: 'error', - }); + })); if (!revocationResponse.ok) { logger.warn('Token revocation returned error', { status: revocationResponse.status }); } @@ -262,13 +264,13 @@ export async function PUT(request: NextRequest) { refresh_token: refreshToken, }, serverId, await defaultClientOptions(serverId)); - const tokenResponse = await fetch(tokenEndpoint, { + const tokenResponse = await observeUpstream('oauth', () => fetch(tokenEndpoint, { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: params.toString(), // See revokeRefreshToken: the body holds the refresh token. redirect: 'error', - }); + })); if (!tokenResponse.ok) { const errorText = await tokenResponse.text(); diff --git a/app/api/errors/__tests__/route.test.ts b/app/api/errors/__tests__/route.test.ts index b6bddfa02..d53e1dde6 100644 --- a/app/api/errors/__tests__/route.test.ts +++ b/app/api/errors/__tests__/route.test.ts @@ -15,6 +15,7 @@ vi.mock('@/lib/admin/config-manager', () => ({ import { POST } from '../route'; import { flushErrors, readErrors, resetAggregateForTests, utcDay } from '@/lib/error-reporting/aggregate'; +import { metrics, resetMetricsForTests } from '@/lib/metrics'; const DSN = 'https://6f0b9c79ed3a4315b76595bc2536ac40@bugsink.example.org/3'; const SUBJECT = 'Merger plans for Acme Q3'; @@ -71,6 +72,20 @@ describe('POST /api/errors', () => { expect(entry.frames).toEqual([{ path: '/_next/static/chunks/1.js', line: 2, column: 30, fn: 'Ab' }]); }); + it('counts while only the metrics endpoint is on, for the browser errors metric', async () => { + process.env.METRICS_TOKEN = 'scrape-secret-0123456789'; + resetMetricsForTests(); + try { + expect((await POST(post(report))).status).toBe(204); + await flushErrors(); + expect(Object.keys((await readErrors())[utcDay()] ?? {})).toHaveLength(1); + expect(await metrics().registry.metrics()).toContain('bulwark_browser_errors_total 1'); + } finally { + delete process.env.METRICS_TOKEN; + resetMetricsForTests(); + } + }); + it('ignores a name or code that is not an identifier or fixed code', async () => { config.errorReportingDsn = DSN; await POST(post({ ...report, name: SUBJECT, code: SUBJECT })); diff --git a/app/api/errors/route.ts b/app/api/errors/route.ts index 7e283f260..b31a38952 100644 --- a/app/api/errors/route.ts +++ b/app/api/errors/route.ts @@ -6,6 +6,7 @@ import { appVersion, isErrorCode } from '@/lib/error-reporting/envelope'; import { scrubName } from '@/lib/error-reporting/scrub'; import { shouldCollectErrors } from '@/lib/error-reporting/settings'; import { sanitizeFrames } from '@/lib/error-reporting/stack'; +import { countBrowserError } from '@/lib/metrics'; export const dynamic = 'force-dynamic'; @@ -66,5 +67,6 @@ export async function POST(request: NextRequest) { // old one, and its frames point into that); the server's when absent. typeof body.version === 'string' && /^\d+\.\d+\.\d+(?:[-+][\w.-]{1,40})?$/.test(body.version) ? body.version : appVersion(), ); + countBrowserError(); return new NextResponse(null, { status: 204, headers: NO_STORE }); } diff --git a/app/api/health/live/route.ts b/app/api/health/live/route.ts new file mode 100644 index 000000000..b5ac3a999 --- /dev/null +++ b/app/api/health/live/route.ts @@ -0,0 +1,18 @@ +import { NextResponse } from 'next/server'; + +export const dynamic = 'force-dynamic'; + +/** + * Liveness: 200 while the process answers. Nothing else is checked, so a + * Stalwart or store outage never makes an orchestrator restart Bulwark. + * Readiness is /api/health/ready. https://bulwarkmail.org/docs/admin/monitoring#health + */ +const HEADERS = { 'Cache-Control': 'no-store' }; + +export function GET() { + return NextResponse.json({ status: 'ok' }, { headers: HEADERS }); +} + +export function HEAD() { + return new NextResponse(null, { status: 200, headers: HEADERS }); +} diff --git a/app/api/health/ready/route.ts b/app/api/health/ready/route.ts new file mode 100644 index 000000000..fe562e724 --- /dev/null +++ b/app/api/health/ready/route.ts @@ -0,0 +1,23 @@ +import { NextResponse } from 'next/server'; +import { readiness } from '@/lib/health/readiness'; + +export const dynamic = 'force-dynamic'; + +/** + * Readiness: 200 when this replica can serve, 503 otherwise. Checks that + * the config is loaded, the shared store (if any) answers, the data dirs are + * writable and Stalwart's session resource answers within 2 s. The body + * names the checks only. Results are reused for 5 s. + * https://bulwarkmail.org/docs/admin/monitoring#health + */ +const HEADERS = { 'Cache-Control': 'no-store' }; + +export async function GET() { + const result = await readiness(); + return NextResponse.json(result, { status: result.ready ? 200 : 503, headers: HEADERS }); +} + +export async function HEAD() { + const result = await readiness(); + return new NextResponse(null, { status: result.ready ? 200 : 503, headers: HEADERS }); +} diff --git a/app/api/metrics/__tests__/route.test.ts b/app/api/metrics/__tests__/route.test.ts new file mode 100644 index 000000000..1fa22e8d3 --- /dev/null +++ b/app/api/metrics/__tests__/route.test.ts @@ -0,0 +1,55 @@ +// @vitest-environment node +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { NextRequest } from 'next/server'; +import { GET } from '../route'; +import { resetMetricsForTests } from '@/lib/metrics'; + +function scrape(authorization?: string): NextRequest { + return new NextRequest('http://localhost/api/metrics', { headers: authorization ? { authorization } : {} }); +} + +afterEach(() => { + delete process.env.METRICS_TOKEN; + delete process.env.METRICS_TOKEN_FILE; + resetMetricsForTests(); +}); + +describe('GET /api/metrics', () => { + it('does not exist without a token configured', async () => { + const res = await GET(scrape('Bearer anything')); + expect(res.status).toBe(404); + }); + + it('wants the bearer token', async () => { + process.env.METRICS_TOKEN = 'scrape-secret-0123456789'; + expect((await GET(scrape())).status).toBe(401); + expect((await GET(scrape('Bearer wrong'))).status).toBe(401); + expect((await GET(scrape('Basic scrape-secret-0123456789'))).status).toBe(401); + }); + + it('answers Prometheus text with the right token', async () => { + process.env.METRICS_TOKEN = 'scrape-secret-0123456789'; + const res = await GET(scrape('Bearer scrape-secret-0123456789')); + expect(res.status).toBe(200); + expect(res.headers.get('content-type')).toContain('text/plain'); + const body = await res.text(); + expect(body).toContain('# TYPE bulwark_http_requests_total counter'); + expect(body).toContain('bulwark_build_info'); + expect(body).toContain('bulwark_nodejs_eventloop_lag_seconds'); + expect(body).toContain('bulwark_signins_total{outcome="rate_limited"} 0'); + }); + + it('reads the token from METRICS_TOKEN_FILE', async () => { + const dir = mkdtempSync(path.join(tmpdir(), 'bulwark-metrics-')); + try { + writeFileSync(path.join(dir, 'token'), 'from-a-file-0123456789\n'); + process.env.METRICS_TOKEN_FILE = path.join(dir, 'token'); + expect((await GET(scrape('Bearer from-a-file-0123456789'))).status).toBe(200); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/app/api/metrics/route.ts b/app/api/metrics/route.ts new file mode 100644 index 000000000..16c8a98a5 --- /dev/null +++ b/app/api/metrics/route.ts @@ -0,0 +1,41 @@ +import { timingSafeEqual } from 'node:crypto'; +import { NextRequest, NextResponse } from 'next/server'; +import { metrics, metricsToken } from '@/lib/metrics'; + +export const dynamic = 'force-dynamic'; + +/** + * GET /api/metrics: Prometheus metrics of this replica. + * + * Off by default: without METRICS_TOKEN (or METRICS_TOKEN_FILE) it answers + * 404 as if it didn't exist. With one, the scraper sends + * `Authorization: Bearer `; anything else gets 401. + * + * https://bulwarkmail.org/docs/admin/monitoring#metrics + */ + +function sameToken(given: string, expected: string): boolean { + const a = Buffer.from(given); + const b = Buffer.from(expected); + return a.length === b.length && timingSafeEqual(a, b); +} + +export async function GET(request: NextRequest) { + const token = metricsToken(); + if (!token) return new NextResponse('Not Found', { status: 404, headers: { 'Cache-Control': 'no-store' } }); + + const header = request.headers.get('authorization') ?? ''; + const given = header.startsWith('Bearer ') ? header.slice('Bearer '.length).trim() : ''; + if (!given || !sameToken(given, token)) { + return new NextResponse('Unauthorized', { + status: 401, + headers: { 'Cache-Control': 'no-store', 'WWW-Authenticate': 'Bearer realm="metrics"' }, + }); + } + + const { registry } = metrics(); + return new NextResponse(await registry.metrics(), { + status: 200, + headers: { 'Content-Type': registry.contentType, 'Cache-Control': 'no-store' }, + }); +} diff --git a/app/api/translate/route.ts b/app/api/translate/route.ts index c53e0bcb0..232ee0703 100644 --- a/app/api/translate/route.ts +++ b/app/api/translate/route.ts @@ -1,4 +1,5 @@ import { NextRequest, NextResponse } from 'next/server'; +import { observeUpstream } from '@/lib/metrics/upstream'; import { rejectCrossOriginRequest } from '@/lib/security/same-origin'; import { readFileEnv } from "@/lib/read-file-env"; import { getStalwartCredentials } from '@/lib/stalwart/credentials'; @@ -174,10 +175,10 @@ async function mymemoryRequest( url.searchParams.set('q', q); url.searchParams.set('langpair', langpair); - const res = await fetch(url.toString(), { + const res = await observeUpstream('translate', () => fetch(url.toString(), { signal, headers: { 'User-Agent': 'Bulwark-Webmail-Translate (+https://bulwarkmail.org)' }, - }); + })); const data = (await res.json().catch(() => null)) as | { responseStatus?: number | string; responseData?: { translatedText?: string }; responseDetails?: string } | null; @@ -232,7 +233,7 @@ async function translateLibre( process.env.LIBRETRANSLATE_API_KEY || readFileEnv(process.env.LIBRETRANSLATE_API_KEY_FILE); const url = endpoint.replace(/\/+$/, '') + '/translate'; - const res = await fetch(url, { + const res = await observeUpstream('translate', () => fetch(url, { method: 'POST', signal, headers: { 'Content-Type': 'application/json' }, @@ -243,7 +244,7 @@ async function translateLibre( format: 'text', ...(apiKey ? { api_key: apiKey } : {}), }), - }); + })); const data = (await res.json().catch(() => null)) as | { translatedText?: string; detectedLanguage?: { language?: string }; error?: string } | null; diff --git a/instrumentation.node.ts b/instrumentation.node.ts index 94f50b00b..0d0f91907 100644 --- a/instrumentation.node.ts +++ b/instrumentation.node.ts @@ -17,6 +17,14 @@ const pkg = JSON.parse( const current: string = pkg.version ?? "0.0.0"; logger.info(`Bulwark Webmail v${current}`, { version: current }); +// Prometheus metrics, only with METRICS_TOKEN(_FILE): count every request +// from the first one on. https://bulwarkmail.org/docs/admin/monitoring#metrics +if (process.env.METRICS_TOKEN?.trim() || process.env.METRICS_TOKEN_FILE?.trim()) { + void import("./lib/metrics/http").then(({ installHttpMetrics }) => { + if (installHttpMetrics()) logger.info("Metrics endpoint on at /api/metrics"); + }); +} + // With REDIS_URL every replica shares one store (#1066). Without it, // everything below runs exactly as it always has. const shared = sharedStoreUrl() !== null; diff --git a/lib/admin/admin-nav.ts b/lib/admin/admin-nav.ts index 8d3d938b5..6352d28bb 100644 --- a/lib/admin/admin-nav.ts +++ b/lib/admin/admin-nav.ts @@ -253,6 +253,8 @@ export const ADMIN_PAGES: readonly AdminPageDef[] = [ sections: [ { id: 'errors', title: 'Error reporting', keywords: 'dsn sentry bugsink glitchtip sample rate environment' }, { id: 'errors-today', title: 'Errors today', keywords: 'count fingerprint top' }, + { id: 'metrics', title: 'Metrics', keywords: 'prometheus scrape token grafana' }, + { id: 'health', title: 'Health checks', keywords: 'liveness readiness probe kubernetes docker' }, ], }, { diff --git a/lib/admin/outgoing-hosts-registry.ts b/lib/admin/outgoing-hosts-registry.ts index 178ae3401..d32438294 100644 --- a/lib/admin/outgoing-hosts-registry.ts +++ b/lib/admin/outgoing-hosts-registry.ts @@ -157,7 +157,7 @@ export const OUTGOING_HOSTS: readonly OutgoingHost[] = [ * has to be looked at again. */ export const NON_REQUEST_HOSTS: readonly { host: string; files: readonly string[]; why: string }[] = [ - { host: 'bulwarkmail.org', why: 'docs links, comments and the contact URL in User-Agent headers', files: ['app/(main)/admin/_pages/telemetry.tsx', 'app/api/fetch-ical/route.ts', 'app/api/translate/route.ts','components/admin/org/sections/recovery-section.tsx', 'instrumentation.node.ts', 'lib/telemetry/state.ts', 'lib/telemetry/types.ts', 'app/(main)/admin/_pages/monitoring.tsx', 'lib/admin/types.ts', 'lib/error-reporting/client.ts', 'lib/error-reporting/lite.ts', 'lib/error-reporting/scrub.ts', 'instrumentation.ts'] }, + { host: 'bulwarkmail.org', why: 'docs links, comments and the contact URL in User-Agent headers', files: ['app/(main)/admin/_pages/telemetry.tsx', 'app/api/fetch-ical/route.ts', 'app/api/translate/route.ts','components/admin/org/sections/recovery-section.tsx', 'instrumentation.node.ts', 'lib/telemetry/state.ts', 'lib/telemetry/types.ts', 'app/(main)/admin/_pages/monitoring.tsx', 'lib/admin/types.ts', 'lib/error-reporting/client.ts', 'lib/error-reporting/lite.ts', 'lib/error-reporting/scrub.ts', 'instrumentation.ts', 'app/api/health/live/route.ts', 'app/api/health/ready/route.ts', 'app/api/metrics/route.ts', 'lib/metrics/index.ts'] }, { host: 'bulwarkmail.com', why: 'JMAP capability URI', files: ['lib/jmap/client.ts'] }, { host: 'develop.sentry.dev', why: 'link to the envelope format in a comment', files: ['lib/error-reporting/envelope.ts'] }, { host: 'extensions.bulwarkmail.org', why: 'signed statement type URIs', files: ['lib/admin/directory-keys.ts'] }, diff --git a/lib/error-reporting/settings.ts b/lib/error-reporting/settings.ts index c29a31470..a37cbb68c 100644 --- a/lib/error-reporting/settings.ts +++ b/lib/error-reporting/settings.ts @@ -1,5 +1,6 @@ import { configManager } from '@/lib/admin/config-manager'; import { parseDsn } from './envelope'; +import { metricsEnabled } from '@/lib/metrics'; import type { ClientSettings } from './client'; /** @@ -43,7 +44,7 @@ export function errorReportingSettings(): ErrorReportingSettings { */ export async function shouldCollectErrors(): Promise { await configManager.ensureLoaded(); - return !!errorReportingSettings().dsn; + return !!errorReportingSettings().dsn || metricsEnabled(); } /** What /api/config hands the browser. */ diff --git a/lib/health/__tests__/readiness.test.ts b/lib/health/__tests__/readiness.test.ts new file mode 100644 index 000000000..35871414b --- /dev/null +++ b/lib/health/__tests__/readiness.test.ts @@ -0,0 +1,86 @@ +// @vitest-environment node +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const config: Record = {}; +vi.mock('@/lib/admin/config-manager', () => ({ + configManager: { + ensureLoaded: async () => {}, + get: (key: string, fallback: unknown) => (key in config ? config[key] : fallback), + }, +})); + +import { readiness, resetReadinessCache } from '../readiness'; +import { GET as ready } from '@/app/api/health/ready/route'; +import { GET as live } from '@/app/api/health/live/route'; + +let dir: string; +let fetchSpy: ReturnType; + +beforeEach(() => { + dir = mkdtempSync(path.join(tmpdir(), 'bulwark-ready-')); + process.env.ADMIN_STATE_DIR = path.join(dir, 'state'); + process.env.TELEMETRY_DATA_DIR = path.join(dir, 'telemetry'); + for (const key of Object.keys(config)) delete config[key]; + config.jmapServerUrl = 'https://mail.example.org'; + resetReadinessCache(); + fetchSpy = vi.fn().mockResolvedValue(new Response(null, { status: 401 })); + vi.stubGlobal('fetch', fetchSpy); +}); + +afterEach(() => { + vi.unstubAllGlobals(); + delete process.env.ADMIN_STATE_DIR; + delete process.env.TELEMETRY_DATA_DIR; + rmSync(dir, { recursive: true, force: true }); +}); + +describe('health endpoints', () => { + it('live answers 200 whatever Stalwart does', async () => { + fetchSpy.mockRejectedValue(new Error('down')); + expect((await live()).status).toBe(200); + }); + + it('ready answers 200 when Stalwart answers, even with 401', async () => { + const res = await ready(); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ ready: true, checks: { config: 'ok', store: 'ok', dataDirs: 'ok', stalwart: 'ok' } }); + expect(String(fetchSpy.mock.calls[0][0])).toBe('https://mail.example.org/.well-known/jmap'); + }); + + it('ready answers 503 when Stalwart is unreachable or failing, and names no host', async () => { + fetchSpy.mockRejectedValue(new Error('ECONNREFUSED mail.example.org')); + const res = await ready(); + expect(res.status).toBe(503); + const text = await res.text(); + expect(JSON.parse(text).checks.stalwart).toBe('fail'); + expect(text).not.toContain('example.org'); + + resetReadinessCache(); + fetchSpy.mockResolvedValue(new Response(null, { status: 502 })); + expect((await ready()).status).toBe(503); + }); + + it('skips Stalwart before a server is configured', async () => { + delete config.jmapServerUrl; + delete process.env.JMAP_SERVER_URL; + expect((await readiness()).checks.stalwart).toBe('skipped'); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('fails when a data dir is not writable', async () => { + writeFileSync(path.join(dir, 'blocked'), ''); + process.env.TELEMETRY_DATA_DIR = path.join(dir, 'blocked', 'telemetry'); + const result = await readiness(); + expect(result.checks.dataDirs).toBe('fail'); + expect(result.ready).toBe(false); + }); + + it('reuses a result for a few seconds', async () => { + await readiness(1000); + await readiness(2000); + expect(fetchSpy).toHaveBeenCalledTimes(1); + }); +}); diff --git a/lib/health/readiness.ts b/lib/health/readiness.ts new file mode 100644 index 000000000..4543b4a2a --- /dev/null +++ b/lib/health/readiness.ts @@ -0,0 +1,131 @@ +import { randomBytes } from 'node:crypto'; +import { mkdir, unlink, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { configManager } from '@/lib/admin/config-manager'; +import { getStateDir } from '@/lib/admin/paths'; +import { observeUpstream } from '@/lib/metrics/upstream'; +import { bootState, getStore, isSharedMode } from '@/lib/shared-store'; +import { getSettingsDir } from '@/lib/settings-sync'; +import { getDir as getTelemetryDir } from '@/lib/telemetry/state'; + +/** + * Whether this replica can serve, for /api/health/ready (and, for the + * shared store, /api/ready). Each check answers ok, fail or skipped; the + * answer names checks, never hosts or paths, because anyone may ask. + */ + +export type CheckResult = 'ok' | 'fail' | 'skipped'; + +export interface Readiness { + ready: boolean; + checks: { + config: CheckResult; + store: CheckResult; + dataDirs: CheckResult; + stalwart: CheckResult; + }; +} + +const STALWART_TIMEOUT_MS = 2000; +const STORE_TIMEOUT_MS = 1000; +const CACHE_MS = 5000; + +const KEY = Symbol.for('bulwark.health.readiness'); +interface CacheState { at: number; result: Readiness | null; pending: Promise | null } +function cache(): CacheState { + const g = globalThis as typeof globalThis & { [KEY]?: CacheState }; + return (g[KEY] ??= { at: 0, result: null, pending: null }); +} + +async function checkConfig(): Promise { + try { + await configManager.ensureLoaded(); + return 'ok'; + } catch { + return 'fail'; + } +} + +/** Booted, and with a shared store, the store answered a ping. */ +export async function checkStore(): Promise { + if (bootState() !== 'ready') return 'fail'; + const store = getStore(); + if (store.mode === 'local') return 'ok'; + try { + await store.ping(STORE_TIMEOUT_MS); + return 'ok'; + } catch { + return 'fail'; + } +} + +async function writable(dir: string): Promise { + const probe = path.join(dir, `.ready-${process.pid}-${randomBytes(4).toString('hex')}`); + try { + await mkdir(dir, { recursive: true }); + await writeFile(probe, ''); + await unlink(probe); + return true; + } catch { + return false; + } +} + +/** The dirs this process writes to in local mode; a shared store holds that data instead. */ +async function checkDataDirs(): Promise { + if (isSharedMode()) return 'skipped'; + const dirs = [getStateDir(), getTelemetryDir()]; + if (configManager.get('settingsSyncEnabled', false)) dirs.push(getSettingsDir()); + const results = await Promise.all(dirs.map(writable)); + return results.every(Boolean) ? 'ok' : 'fail'; +} + +/** Stalwart's session resource answers, whatever the status below 500 (401 is expected without credentials). */ +async function checkStalwart(): Promise { + const base = configManager.get('jmapServerUrl', '') || process.env.JMAP_SERVER_URL || ''; + if (!base.trim()) return 'skipped'; + let url: URL; + try { + url = new URL('/.well-known/jmap', base.trim()); + } catch { + return 'fail'; + } + try { + const response = await observeUpstream('stalwart', () => fetch(url, { + method: 'GET', + redirect: 'manual', + signal: AbortSignal.timeout(STALWART_TIMEOUT_MS), + })); + await response.body?.cancel().catch(() => undefined); + return response.status < 500 ? 'ok' : 'fail'; + } catch { + return 'fail'; + } +} + +async function evaluate(): Promise { + const config = await checkConfig(); + const [store, dataDirs, stalwart] = await Promise.all([checkStore(), checkDataDirs(), checkStalwart()]); + const checks = { config, store, dataDirs, stalwart }; + return { ready: Object.values(checks).every((c) => c !== 'fail'), checks }; +} + +/** The readiness of this replica, reused for CACHE_MS so probes can't flood Stalwart or the store. */ +export async function readiness(now = Date.now()): Promise { + const c = cache(); + if (c.result && now - c.at < CACHE_MS) return c.result; + c.pending ??= evaluate() + .then((result) => { + c.result = result; + c.at = Date.now(); + return result; + }) + .finally(() => { c.pending = null; }); + return c.pending; +} + +/** For tests. */ +export function resetReadinessCache(): void { + const g = globalThis as typeof globalThis & { [KEY]?: CacheState }; + delete g[KEY]; +} diff --git a/lib/metrics/__tests__/metrics.test.ts b/lib/metrics/__tests__/metrics.test.ts new file mode 100644 index 000000000..286dd4030 --- /dev/null +++ b/lib/metrics/__tests__/metrics.test.ts @@ -0,0 +1,93 @@ +// @vitest-environment node +import http from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { metrics, observeRequest, resetMetricsForTests, signinOutcome } from '../index'; +import { observeUpstream } from '../upstream'; +import { installHttpMetrics } from '../http'; + +async function value(name: string, labels: Record = {}): Promise { + const metric = (await metrics().registry.getMetricsAsJSON()).find((m) => m.name === name); + const found = (metric?.values as Array<{ labels: Record; value: number }> | undefined) + ?.find((v) => Object.entries(labels).every(([k, want]) => v.labels[k] === want)); + return found?.value; +} + +beforeEach(() => { + process.env.METRICS_TOKEN = 'test-token-1234567890'; + resetMetricsForTests(); +}); + +afterEach(() => { + delete process.env.METRICS_TOKEN; + resetMetricsForTests(); + vi.restoreAllMocks(); +}); + +describe('request metrics', () => { + it('label requests by route group and status class, never by the raw path', async () => { + observeRequest('/de/mail/folder/Acquisition%20Talks/Mabc?q=secret', 'GET', 200, 0.05); + observeRequest('/webmail/api/wopi/files/abc/contents', 'POST', 503, 1.2, '/webmail'); + observeRequest('/api/whatever-a-user-typed', 'BREW', 404, 0.01); + expect(await value('bulwark_http_requests_total', { group: 'page:mail', method: 'GET', status: '2xx' })).toBe(1); + expect(await value('bulwark_http_requests_total', { group: 'api:wopi', method: 'POST', status: '5xx' })).toBe(1); + expect(await value('bulwark_http_requests_total', { group: 'api:other', method: 'other', status: '4xx' })).toBe(1); + const text = await metrics().registry.metrics(); + expect(text).not.toContain('Acquisition'); + expect(text).not.toContain('whatever-a-user-typed'); + expect(text).not.toContain('secret'); + }); + + it('count sign-ins by outcome from the auth routes', async () => { + observeRequest('/api/auth/session', 'POST', 200, 0.1); + observeRequest('/api/auth/session', 'POST', 401, 0.1); + observeRequest('/api/auth/session', 'POST', 429, 0.1); + observeRequest('/api/auth/token', 'PUT', 401, 0.1); + observeRequest('/api/auth/token', 'PUT', 200, 0.1); + observeRequest('/api/auth/session', 'GET', 200, 0.1); + for (const [outcome, n] of [['success', 1], ['failure', 1], ['rate_limited', 1], ['refresh_failure', 1]] as const) { + expect(await value('bulwark_signins_total', { outcome }), outcome).toBe(n); + } + }); + + it('map outcomes only for sign-in and refresh calls', () => { + expect(signinOutcome('api:auth', '/api/auth/totp-token-exchange', 'POST', 200)).toBe('success'); + expect(signinOutcome('api:auth', '/api/auth/session', 'DELETE', 200)).toBeNull(); + expect(signinOutcome('api:admin', '/api/admin/auth', 'POST', 401)).toBeNull(); + }); + + it('do nothing while the endpoint is off', async () => { + delete process.env.METRICS_TOKEN; + observeRequest('/api/config', 'GET', 200, 0.01); + process.env.METRICS_TOKEN = 'test-token-1234567890'; + expect(await value('bulwark_http_requests_total', { group: 'api:config' })).toBeUndefined(); + }); + + it('come from Node\'s HTTP server itself once installed', async () => { + expect(installHttpMetrics('')).toBe(true); + const server = http.createServer((_req, res) => { res.statusCode = 204; res.end(); }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + try { + const { port } = server.address() as AddressInfo; + await fetch(`http://127.0.0.1:${port}/api/health/live`); + await vi.waitFor(async () => expect(await value('bulwark_http_requests_total', { group: 'api:health', status: '2xx' })).toBe(1)); + } finally { + server.close(); + } + }); +}); + +describe('upstream metrics', () => { + it('count answers below 500 as ok and 5xx or errors as error', async () => { + await observeUpstream('stalwart', async () => new Response(null, { status: 401 })); + await observeUpstream('stalwart', async () => new Response(null, { status: 502 })); + await expect(observeUpstream('wopi', async () => { throw new Error('down'); })).rejects.toThrow('down'); + expect(await value('bulwark_upstream_requests_total', { target: 'stalwart', outcome: 'ok' })).toBe(1); + expect(await value('bulwark_upstream_requests_total', { target: 'stalwart', outcome: 'error' })).toBe(1); + expect(await value('bulwark_upstream_requests_total', { target: 'wopi', outcome: 'error' })).toBe(1); + }); + + it('report build info', async () => { + expect(await metrics().registry.metrics()).toMatch(/bulwark_build_info\{version="[^"]+",build="[^"]+"\} 1/); + }); +}); diff --git a/lib/metrics/__tests__/upstream-wiring.test.ts b/lib/metrics/__tests__/upstream-wiring.test.ts new file mode 100644 index 000000000..b470ffd2a --- /dev/null +++ b/lib/metrics/__tests__/upstream-wiring.test.ts @@ -0,0 +1,29 @@ +// @vitest-environment node +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('@/lib/admin/config-manager', () => ({ + configManager: { ensureLoaded: async () => {}, get: (_key: string, fallback: unknown) => fallback }, +})); + +import { fetchJmapServer } from '@/lib/stalwart/server-fetch'; +import { metrics, resetMetricsForTests } from '@/lib/metrics'; + +beforeEach(() => { + process.env.METRICS_TOKEN = 'scrape-secret-0123456789'; + resetMetricsForTests(); +}); + +afterEach(() => { + delete process.env.METRICS_TOKEN; + resetMetricsForTests(); + vi.unstubAllGlobals(); +}); + +// The server's calls to Stalwart all go through fetchJmapServer. +describe('Stalwart calls', () => { + it('are counted under the stalwart target', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('{}', { status: 200 }))); + await fetchJmapServer('https://mail.example.org/jmap/session', {}, true); + expect(await metrics().registry.metrics()).toContain('bulwark_upstream_requests_total{target="stalwart",outcome="ok"} 1'); + }); +}); diff --git a/lib/metrics/http.ts b/lib/metrics/http.ts new file mode 100644 index 000000000..1bffbb9a8 --- /dev/null +++ b/lib/metrics/http.ts @@ -0,0 +1,39 @@ +import diagnostics from 'node:diagnostics_channel'; +import type { IncomingMessage, ServerResponse } from 'node:http'; +import { metricsEnabled, observeRequest } from './index'; + +/** + * Counts every request this process serves, through Node's own HTTP + * diagnostics channels: nothing is wrapped or patched, and the proxy's + * matcher doesn't matter. Installed once at startup when the metrics + * endpoint is on (instrumentation.node.ts). + */ + +interface HttpState { + installed: boolean; + started: WeakMap; +} + +const KEY = Symbol.for('bulwark.metrics.http'); +function state(): HttpState { + const g = globalThis as typeof globalThis & { [KEY]?: HttpState }; + return (g[KEY] ??= { installed: false, started: new WeakMap() }); +} + +export function installHttpMetrics(basePath = process.env.NEXT_PUBLIC_BASE_PATH || ''): boolean { + const s = state(); + if (s.installed || !metricsEnabled()) return false; + s.installed = true; + diagnostics.subscribe('http.server.request.start', (message) => { + const { request } = message as { request: IncomingMessage }; + s.started.set(request, performance.now()); + }); + diagnostics.subscribe('http.server.response.finish', (message) => { + const { request, response } = message as { request: IncomingMessage; response: ServerResponse }; + const start = s.started.get(request); + if (start === undefined) return; + s.started.delete(request); + observeRequest(request.url ?? '/', request.method, response.statusCode, (performance.now() - start) / 1000, basePath); + }); + return true; +} diff --git a/lib/metrics/index.ts b/lib/metrics/index.ts new file mode 100644 index 000000000..dfdea1749 --- /dev/null +++ b/lib/metrics/index.ts @@ -0,0 +1,147 @@ +import { readFileSync } from 'node:fs'; +import type { Counter, Gauge, Histogram, Registry } from 'prom-client'; +import client from 'prom-client'; +import { routeGroup } from '@/lib/route-groups'; + +/** + * Prometheus metrics for /api/metrics, off unless METRICS_TOKEN (or + * METRICS_TOKEN_FILE) is set. Every replica counts for itself; Prometheus + * scrapes each one. Labels come from fixed lists only (route groups, sign-in + * outcomes, upstream targets), never from user names or hosts users type. + * + * https://bulwarkmail.org/docs/admin/monitoring#metrics + */ + +export const SIGNIN_OUTCOMES = ['success', 'failure', 'refresh_failure', 'rate_limited'] as const; +export type SigninOutcome = typeof SIGNIN_OUTCOMES[number]; + +export const UPSTREAM_TARGETS = ['stalwart', 'wopi', 'translate', 'plugin_proxy', 'oauth'] as const; +export type UpstreamTarget = typeof UPSTREAM_TARGETS[number]; + +const METHODS = new Set(['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS']); +const DURATION_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30]; + +/** The bearer token /api/metrics wants, or null when the endpoint is off. */ +export function metricsToken(): string | null { + const direct = process.env.METRICS_TOKEN?.trim(); + if (direct) return direct; + const file = process.env.METRICS_TOKEN_FILE?.trim(); + if (!file) return null; + try { + return readFileSync(file, 'utf8').trim() || null; + } catch { + return null; + } +} + +export function metricsEnabled(): boolean { + return metricsToken() !== null; +} + +interface Metrics { + registry: Registry; + httpRequests: Counter<'group' | 'method' | 'status'>; + httpDuration: Histogram<'group' | 'method'>; + signins: Counter<'outcome'>; + upstreamRequests: Counter<'target' | 'outcome'>; + upstreamDuration: Histogram<'target'>; + browserErrors: Counter; + errorsToday: Gauge; +} + +const KEY = Symbol.for('bulwark.metrics'); + +/** The registry and metrics, made once per process whatever bundle asks. */ +export function metrics(): Metrics { + const g = globalThis as typeof globalThis & { [KEY]?: Metrics }; + if (g[KEY]) return g[KEY]; + const registry = new client.Registry(); + client.collectDefaultMetrics({ register: registry, prefix: 'bulwark_' }); + const version = process.env.NEXT_PUBLIC_APP_VERSION || '0.0.0'; + const build = process.env.NEXT_PUBLIC_GIT_COMMIT || process.env.BULWARK_BUILD || 'unknown'; + new client.Gauge({ name: 'bulwark_build_info', help: 'Bulwark version and build; always 1', labelNames: ['version', 'build'], registers: [registry] }) + .set({ version, build }, 1); + new client.Gauge({ + name: 'bulwark_uptime_seconds', + help: 'Seconds since this process started', + registers: [registry], + collect() { this.set(process.uptime()); }, + }); + const created: Metrics = { + registry, + httpRequests: new client.Counter({ name: 'bulwark_http_requests_total', help: 'HTTP requests by route group, method and status class', labelNames: ['group', 'method', 'status'], registers: [registry] }), + httpDuration: new client.Histogram({ name: 'bulwark_http_request_duration_seconds', help: 'HTTP request duration by route group and method', labelNames: ['group', 'method'], buckets: DURATION_BUCKETS, registers: [registry] }), + signins: new client.Counter({ name: 'bulwark_signins_total', help: 'Sign-ins handled by this server, by outcome', labelNames: ['outcome'], registers: [registry] }), + upstreamRequests: new client.Counter({ name: 'bulwark_upstream_requests_total', help: 'Requests this server made to other services, by target and outcome', labelNames: ['target', 'outcome'], registers: [registry] }), + upstreamDuration: new client.Histogram({ name: 'bulwark_upstream_request_duration_seconds', help: 'Duration of requests to other services, by target', labelNames: ['target'], buckets: DURATION_BUCKETS, registers: [registry] }), + browserErrors: new client.Counter({ name: 'bulwark_browser_errors_total', help: 'Errors browsers reported to this server', registers: [registry] }), + errorsToday: new client.Gauge({ + name: 'bulwark_errors_today', + help: 'Errors counted today (UTC) by the whole deployment, browsers and servers', + registers: [registry], + async collect() { + const { readErrors, summarizeDay } = await import('@/lib/error-reporting/aggregate'); + this.set(summarizeDay(await readErrors()).total); + }, + }), + }; + // Pre-create the label sets, so a series exists at 0 before its first event. + for (const outcome of SIGNIN_OUTCOMES) created.signins.inc({ outcome }, 0); + for (const target of UPSTREAM_TARGETS) { + for (const outcome of ['ok', 'error']) created.upstreamRequests.inc({ target, outcome }, 0); + } + g[KEY] = created; + return created; +} + +export function methodLabel(method: string | undefined): string { + const upper = (method ?? '').toUpperCase(); + return METHODS.has(upper) ? upper : 'other'; +} + +export function statusClass(status: number): string { + return status >= 100 && status < 600 ? `${Math.floor(status / 100)}xx` : 'other'; +} + +/** What a finished request means for sign-ins, from its route, method and status. */ +export function signinOutcome(group: string, path: string, method: string, status: number): SigninOutcome | null { + if (group !== 'api:auth') return null; + const route = path.split(/[?#]/, 1)[0].replace(/\/+$/, ''); + const signIn = method === 'POST' && /\/api\/auth\/(?:session|stalwart-context|totp-token-exchange|token)$/.test(route); + const refresh = method === 'PUT' && /\/api\/auth\/token$/.test(route); + if (!signIn && !refresh) return null; + if (status === 429) return 'rate_limited'; + if (status >= 200 && status < 300) return refresh ? null : 'success'; + return refresh ? 'refresh_failure' : 'failure'; +} + +/** Count a finished HTTP request (lib/metrics/http.ts). */ +export function observeRequest(path: string, method: string | undefined, status: number, seconds: number, basePath = ''): void { + if (!metricsEnabled()) return; + const m = metrics(); + const group = routeGroup(path, basePath); + const verb = methodLabel(method); + m.httpRequests.inc({ group, method: verb, status: statusClass(status) }); + m.httpDuration.observe({ group, method: verb }, seconds); + const outcome = signinOutcome(group, basePath && path.startsWith(basePath) ? path.slice(basePath.length) : path, verb, status); + if (outcome) m.signins.inc({ outcome }); +} + +export function observeUpstreamResult(target: UpstreamTarget, ok: boolean, seconds: number): void { + if (!metricsEnabled()) return; + const m = metrics(); + m.upstreamRequests.inc({ target, outcome: ok ? 'ok' : 'error' }); + m.upstreamDuration.observe({ target }, seconds); +} + +export function countBrowserError(): void { + if (!metricsEnabled()) return; + metrics().browserErrors.inc(); +} + +/** For tests. */ +export function resetMetricsForTests(): void { + const g = globalThis as typeof globalThis & { [KEY]?: Metrics }; + g[KEY]?.registry.clear(); + delete g[KEY]; +} diff --git a/lib/metrics/upstream.ts b/lib/metrics/upstream.ts new file mode 100644 index 000000000..5aaf47fca --- /dev/null +++ b/lib/metrics/upstream.ts @@ -0,0 +1,19 @@ +import { observeUpstreamResult, type UpstreamTarget } from './index'; + +/** + * Run one request this server makes to another service and record its + * duration and outcome under a fixed target name. An answer below 500 counts + * as ok (a 401 from Stalwart is an answer); a 5xx or a thrown error as + * error. Costs nothing while metrics are off. + */ +export async function observeUpstream(target: UpstreamTarget, run: () => Promise): Promise { + const start = performance.now(); + try { + const response = await run(); + observeUpstreamResult(target, response.status < 500, (performance.now() - start) / 1000); + return response; + } catch (error) { + observeUpstreamResult(target, false, (performance.now() - start) / 1000); + throw error; + } +} diff --git a/lib/oauth/token-exchange.ts b/lib/oauth/token-exchange.ts index e002f218d..a60031595 100644 --- a/lib/oauth/token-exchange.ts +++ b/lib/oauth/token-exchange.ts @@ -1,4 +1,5 @@ import { logger } from '@/lib/logger'; +import { observeUpstream } from '@/lib/metrics/upstream'; import { discoverOAuth } from '@/lib/oauth/discovery'; import type { EndpointValidator, OAuthMetadata } from '@/lib/oauth/discovery'; import { isPublicHttpUrl } from '@/lib/security/url-guard'; @@ -209,12 +210,12 @@ export async function exchangeCodeForTokens( // The body carries the code, the PKCE verifier and the client secret. The // endpoint was validated; a redirect would re-send all of it to a host // that was not, so none is followed. - const tokenResponse = await fetch(tokenEndpoint, { + const tokenResponse = await observeUpstream('oauth', () => fetch(tokenEndpoint, { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: params.toString(), redirect: 'error', - }); + })); if (!tokenResponse.ok) { const errorText = await tokenResponse.text(); diff --git a/lib/plugin-secrets/proxy.ts b/lib/plugin-secrets/proxy.ts index 766b49dd7..6a84a3f8b 100644 --- a/lib/plugin-secrets/proxy.ts +++ b/lib/plugin-secrets/proxy.ts @@ -9,6 +9,7 @@ import { getPlugin } from '@/lib/admin/plugin-registry'; import { getPluginConfig } from '@/lib/admin/plugin-config'; import { logger } from '@/lib/logger'; import { DisallowedUrlError, fetchPublicUrl } from '@/lib/security/url-guard'; +import { observeUpstream } from '@/lib/metrics/upstream'; import { authHeaderFor, credentialOrigin, privateProxyOrigins, type CredentialInject, type CredentialSpec } from './credentials'; import { getUserCredential, setUserCredentialSecret, userKey, type VaultUser } from './vault'; import { declaredUserCredential } from './known-plugins'; @@ -239,8 +240,10 @@ async function send( signal: init.signal, ...(init.body ? { body: init.body } : {}), }; - if (trusted) return fetch(url, { ...request, redirect: 'manual' } as RequestInit); - return (await fetchPublicUrl(url, request as never)) as unknown as Response; + return observeUpstream('plugin_proxy', async () => { + if (trusted) return fetch(url, { ...request, redirect: 'manual' } as RequestInit); + return (await fetchPublicUrl(url, request as never)) as unknown as Response; + }); } function responseHeaders(upstream: Response): Headers { diff --git a/lib/route-groups.ts b/lib/route-groups.ts index e10e6c092..6c222f6a6 100644 --- a/lib/route-groups.ts +++ b/lib/route-groups.ts @@ -11,7 +11,7 @@ /** The folders under app/api. */ export const API_GROUPS: readonly string[] = [ 'account', 'account-archive', 'admin', 'auth', 'bimi', 'caldav', 'calendar-agenda', 'config', - 'connector', 'dev-jmap', 'errors', 'favicon', 'fetch-ical', 'health', 'og-image', + 'connector', 'dev-jmap', 'errors', 'favicon', 'fetch-ical', 'health', 'metrics', 'og-image', 'plugin-approval-status', 'plugin-proxy', 'plugin-secrets', 'plugin-signing-pubkey', 'plugins', 'push', 'pwa-icon', 'pwa-screenshot', 'ready', 'settings', 'setup', 'system', 'translate', 'webdav', 'wopi', diff --git a/lib/stalwart/server-fetch.ts b/lib/stalwart/server-fetch.ts index d1adc2fe3..710c2493c 100644 --- a/lib/stalwart/server-fetch.ts +++ b/lib/stalwart/server-fetch.ts @@ -19,6 +19,7 @@ import { configManager } from '@/lib/admin/config-manager'; import { logger } from '@/lib/logger'; import { parseJmapServers, resolveTrustedJmapUrl } from '@/lib/admin/jmap-servers'; import { fetchPublicUrl, type PublicFetchResponse } from '@/lib/security/url-guard'; +import { observeUpstream } from '@/lib/metrics/upstream'; import { isStoredCredentialRefused, recordStoredCredentialStatus, @@ -72,7 +73,7 @@ export async function fetchJmapServer( ): Promise { const authHeader = new Headers(givenInit.headers).get('authorization'); if (authHeader && (await isStoredCredentialRefused(authHeader))) return refusedCredentialResponse(); - const response = await fetchUnbudgeted(url, givenInit, trusted); + const response = await observeUpstream('stalwart', () => fetchUnbudgeted(url, givenInit, trusted)); // Not awaited: this process's copy is updated at once, the shared write follows. if (authHeader) { void recordStoredCredentialStatus(authHeader, response.status).catch((error) => { diff --git a/lib/wopi/discovery.ts b/lib/wopi/discovery.ts index d2150567f..971285f26 100644 --- a/lib/wopi/discovery.ts +++ b/lib/wopi/discovery.ts @@ -1,4 +1,5 @@ import { configManager } from '@/lib/admin/config-manager'; +import { observeUpstream } from '@/lib/metrics/upstream'; import { logger } from '@/lib/logger'; /** @@ -123,10 +124,10 @@ export async function getWopiActions(): Promise { async function fetchWopiActions(discoveryUrl: string): Promise { try { - const res = await fetch(discoveryUrl, { + const res = await observeUpstream('wopi', () => fetch(discoveryUrl, { headers: { Accept: 'text/xml' }, signal: AbortSignal.timeout(DISCOVERY_TIMEOUT_MS), - }); + })); if (!res.ok) { logger.warn('WOPI discovery fetch failed', { discoveryUrl, status: res.status }); return null; diff --git a/next.config.ts b/next.config.ts index d039caec1..cb20d9f8c 100644 --- a/next.config.ts +++ b/next.config.ts @@ -61,9 +61,9 @@ const nextConfig: NextConfig = { // esbuild ships native binaries + a README the bundler can't parse; load // it from node_modules at runtime instead of trying to bundle it. Used by // PLUGIN_DEV_DIR's on-the-fly bundler. ioredis (the shared store, #1066) - // is loaded from node_modules too, so the proxy and the route bundles - // don't each carry a copy. - serverExternalPackages: ["esbuild", "ioredis"], + // and prom-client (/api/metrics) are loaded from node_modules too, so the + // proxy and the route bundles don't each carry a copy. + serverExternalPackages: ["esbuild", "ioredis", "prom-client"], // The build's type check leaves out tests, which are a third of the files // and push tsc past the default heap on a 4 GB machine (#1431). // `npm run typecheck` still checks them with tsconfig.json. diff --git a/package-lock.json b/package-lock.json index 878d8cb72..055af593f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,13 @@ "@dnd-kit/utilities": "^3.2.2", "@noble/curves": "^2.4.0", "@noble/hashes": "^2.4.0", + "@opentelemetry/api": "1.9.1", + "@opentelemetry/core": "2.8.0", + "@opentelemetry/exporter-trace-otlp-http": "0.219.0", + "@opentelemetry/resources": "2.8.0", + "@opentelemetry/sdk-trace-base": "2.8.0", + "@opentelemetry/sdk-trace-node": "2.8.0", + "@opentelemetry/semantic-conventions": "1.44.0", "@tabler/icons-react": "^3.50.0", "@tanstack/react-virtual": "^3.14.23", "@tiptap/extension-color": "^3.32.1", @@ -43,6 +50,7 @@ "otpauth": "^9.5.2", "pdfjs-dist": "^6.4.299", "postal-mime": "^4.1.4", + "prom-client": "15.1.3", "qrcode": "^1.5.4", "react": "^19.3.1", "react-dom": "^19.3.1", @@ -2389,6 +2397,204 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/api-logs": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.219.0.tgz", + "integrity": "sha512-FFx7YnaYJlIjqWW/AG/yAZ0L/NEY724PipXXXQLdtZPbLwBGbUMTGL1i/esI56TWfTUXxhLfpgrnWJCG8aUJyg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/context-async-hooks": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-2.8.0.tgz", + "integrity": "sha512-/3FIraneMcng67SUJCxvyInk/oxzwsxyadufk0wwfOBLf5wqtAGX4MoQASwSbndBPeARzBryUM9Azr5kHIdWLw==", + "license": "Apache-2.0", + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/core": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.8.0.tgz", + "integrity": "sha512-hd1Lfh8p545nNz+jq1Ejfz+Mn1hyLuxYn1YzTfFNrxr8urEWMNQLPf1Th8kjOH+HxwawCrtgBp8JpBUR4ZSgww==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-http": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-http/-/exporter-trace-otlp-http-0.219.0.tgz", + "integrity": "sha512-9t6SvBXXBEjOBcIzgozvBbd3jWrv3Gt3ngGhl1fhdZ/zRc7oZDVOFEqbi2zlBpW9BXhgDMKv422J0DL/3iQWfw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.8.0", + "@opentelemetry/otlp-exporter-base": "0.219.0", + "@opentelemetry/otlp-transformer": "0.219.0", + "@opentelemetry/resources": "2.8.0", + "@opentelemetry/sdk-trace-base": "2.8.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-exporter-base": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.219.0.tgz", + "integrity": "sha512-zvIxQX/AZUVKDU+hCuYx+7UkiP7GRdnk1ZbFQRYzHvYp47cAWR4j3IhoPhV9KaeXEv2xdGq3IA6PnpzDmLcmSA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.8.0", + "@opentelemetry/otlp-transformer": "0.219.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.219.0.tgz", + "integrity": "sha512-aaYKAyXhw9VchKZVGOopD3Gw/kPsyrX2c6IQ0AW32mTjqmZOh5Y6Gf5OYqTNqVktAeBjmFinhyFaCwW6GYK9YQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.219.0", + "@opentelemetry/core": "2.8.0", + "@opentelemetry/resources": "2.8.0", + "@opentelemetry/sdk-logs": "0.219.0", + "@opentelemetry/sdk-metrics": "2.8.0", + "@opentelemetry/sdk-trace-base": "2.8.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/resources": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.8.0.tgz", + "integrity": "sha512-qmXQ27ilDbUK/vGMqwL8D4/rhn76C+sherM4wTbjlfknR8Nvfc/hCxjRJPhkzZzUsPiNg16SA31NxMabwttRjg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.8.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-logs": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.219.0.tgz", + "integrity": "sha512-s6lTKRakaPClvKoWHRChxnXjDMkM/TQ30ff78jN6EBGf7MI7VzANE5PU3f4z9qDUudWjvZjOLHG0rBnBKYvoXA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.219.0", + "@opentelemetry/core": "2.8.0", + "@opentelemetry/resources": "2.8.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.4.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-metrics": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.8.0.tgz", + "integrity": "sha512-UDBGaj6W0Rgy5rTTaoxs8gVGF/aGkAKyjurJv7se6wjRxJu7FoquTLT/vt54DZfo4crbprYfhX/SOK9+BPw1qg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.8.0", + "@opentelemetry/resources": "2.8.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-base": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.8.0.tgz", + "integrity": "sha512-mhU4jp+vW0mGbFRd+GeXHvmfA4aDqWjBjLC3pE5XMpLs0IE2ryYb019Ts2AQrOq67gaTF25D91+fgvEHDZEnuQ==", + "deprecated": "Use @opentelemetry/sdk-trace package", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.8.0", + "@opentelemetry/resources": "2.8.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-node": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-node/-/sdk-trace-node-2.8.0.tgz", + "integrity": "sha512-nZt9OGufioAc3AfoLTqA9bsAeaMJAictYDdI2VcNQ+PmT+3rfKjAZDZvgPfd8VPX0O5Bw1hdQF6kDK8VSpZiWg==", + "deprecated": "Use @opentelemetry/sdk-trace package", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/context-async-hooks": "2.8.0", + "@opentelemetry/core": "2.8.0", + "@opentelemetry/sdk-trace-base": "2.8.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/semantic-conventions": { + "version": "1.44.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.44.0.tgz", + "integrity": "sha512-E3JkOlMBi3N8OGGkbl7tFyqk2MAS6/lySRZBK3Mu7AzjF1fT8oIChH9ChE48SUWzDn9Kdwta+G/p5LJ4xW4CLQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, "node_modules/@oxc-project/types": { "version": "0.155.0", "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.155.0.tgz", @@ -4943,6 +5149,12 @@ "require-from-string": "^2.0.2" } }, + "node_modules/bintrees": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", + "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==", + "license": "MIT" + }, "node_modules/brace-expansion": { "version": "5.0.12", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz", @@ -8375,6 +8587,20 @@ "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", "license": "MIT" }, + "node_modules/prom-client": { + "version": "15.1.3", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.3.tgz", + "integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==", + "deprecated": "prom-client has been replaced by @prometheus-io/client", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.4.0", + "tdigest": "^0.1.1" + }, + "engines": { + "node": "^16 || ^18 || >=20" + } + }, "node_modules/prop-types": { "version": "15.8.1", "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", @@ -9355,6 +9581,15 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/tdigest": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.3.tgz", + "integrity": "sha512-zbRt+lT+/H4fRItHshczHErVCQnitJk8MfMT24MqFJf3YL7SJJPqGIGeuOdvxXxM/AHFzKBl7WoyaYwqO9s3Kw==", + "license": "MIT", + "dependencies": { + "bintrees": "1.0.2" + } + }, "node_modules/tinybench": { "version": "6.2.2", "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-6.2.2.tgz", diff --git a/package.json b/package.json index 6f78a0579..ae34dfde9 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,13 @@ "@dnd-kit/utilities": "^3.2.2", "@noble/curves": "^2.4.0", "@noble/hashes": "^2.4.0", + "@opentelemetry/api": "1.9.1", + "@opentelemetry/core": "2.8.0", + "@opentelemetry/exporter-trace-otlp-http": "0.219.0", + "@opentelemetry/resources": "2.8.0", + "@opentelemetry/sdk-trace-base": "2.8.0", + "@opentelemetry/sdk-trace-node": "2.8.0", + "@opentelemetry/semantic-conventions": "1.44.0", "@tabler/icons-react": "^3.50.0", "@tanstack/react-virtual": "^3.14.23", "@tiptap/extension-color": "^3.32.1", @@ -73,6 +80,7 @@ "otpauth": "^9.5.2", "pdfjs-dist": "^6.4.299", "postal-mime": "^4.1.4", + "prom-client": "15.1.3", "qrcode": "^1.5.4", "react": "^19.3.1", "react-dom": "^19.3.1", diff --git a/proxy.ts b/proxy.ts index 12f1423ed..0ba40028a 100644 --- a/proxy.ts +++ b/proxy.ts @@ -179,6 +179,8 @@ export function mailFrameAncestors(allowed: string | undefined): string { return sources.join(" "); } +const PROBE_PATHS = new Set(["/api/health", "/api/health/live", "/api/health/ready", "/api/ready", "/api/metrics"]); + function isSetupPath(pathname: string): boolean { return ( pathname === "/setup" || @@ -189,9 +191,10 @@ function isSetupPath(pathname: string): boolean { export async function proxy(request: NextRequest) { // The probes answer without config or store, so a replica that is still - // starting (or lost its shared store) can say so (#1066). + // starting (or lost its shared store) can say so (#1066). The metrics + // endpoint, too: a scrape shouldn't fail while a replica boots. const probePath = routePathOf(request.nextUrl.pathname); - if (probePath === "/api/health" || probePath === "/api/ready") { + if (PROBE_PATHS.has(probePath)) { const probe = NextResponse.next(); probe.headers.set("X-Content-Type-Options", "nosniff"); probe.headers.set("X-Frame-Options", "DENY"); diff --git a/scripts/healthcheck.mjs b/scripts/healthcheck.mjs new file mode 100644 index 000000000..26ebbc359 --- /dev/null +++ b/scripts/healthcheck.mjs @@ -0,0 +1,15 @@ +// Docker HEALTHCHECK: exit 0 when /api/health/live answers 200 within 4 s. +// Liveness only, so a Stalwart outage never marks the container unhealthy. +// The base path is baked in at build time (BULWARK_BASE_PATH in the image). +// https://bulwarkmail.org/docs/admin/monitoring#health +const port = process.env.PORT || '3000'; +const listen = process.env.HOSTNAME; +const host = !listen || listen === '0.0.0.0' || listen === '::' ? '127.0.0.1' : listen.includes(':') ? `[${listen}]` : listen; +const base = (process.env.BULWARK_BASE_PATH || '').replace(/\/+$/, ''); + +try { + const res = await fetch(`http://${host}:${port}${base}/api/health/live`, { signal: AbortSignal.timeout(4000) }); + process.exit(res.ok ? 0 : 1); +} catch { + process.exit(1); +} From 110f7fa96a051b9651e1ebb3fe9792bf41d4f470 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:59:41 +0100 Subject: [PATCH 07/26] ci: install kind where the runner can make it executable --- .github/workflows/kubernetes.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/kubernetes.yml b/.github/workflows/kubernetes.yml index d521c4341..e62d2d215 100644 --- a/.github/workflows/kubernetes.yml +++ b/.github/workflows/kubernetes.yml @@ -49,8 +49,12 @@ jobs: - name: Create the cluster run: | - curl -fsSLo /usr/local/bin/kind "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64" - chmod +x /usr/local/bin/kind + # The runner's own kind in /usr/local/bin belongs to root; use a pinned one. + mkdir -p "$RUNNER_TEMP/bin" + curl -fsSLo "$RUNNER_TEMP/bin/kind" "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64" + chmod +x "$RUNNER_TEMP/bin/kind" + echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH" + export PATH="$RUNNER_TEMP/bin:$PATH" kind create cluster --name ci --wait 180s kind load docker-image bulwark-local:ci --name ci From e580ed24da3754584accd2c071c32c34a8f60a1c Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 03:10:41 +0100 Subject: [PATCH 08/26] feat: send anonymous error counts with the usage stats when the admin also agrees to it A second switch under System > Usage stats, "Also send anonymous error counts", or `BULWARK_TELEMETRY_ERRORS=on|off`. Off by default, off whenever the heartbeat is off or no third-party traffic is on, and switching the heartbeat off switches it off too. With it on the heartbeat becomes schema 2: the v1 fields unchanged plus `errors`, up to 50 fingerprints of the last complete UTC day with the error name, the top frames in Bulwark's bundles (path, line, column, debug ID), the version, a bucketed count, on how many of the last 7 days it occurred, and a message only when it is a fixed error code. Without the consent the heartbeat stays exactly schema 1. The admin preview shows the v2 payload as soon as the switch is flipped. --- app/(main)/admin/_pages/telemetry.tsx | 44 ++++- app/api/admin/telemetry/route.ts | 43 ++++- lib/__tests__/telemetry-error-counts.test.ts | 166 ++++++++++++++++++ lib/__tests__/telemetry-payload.test.ts | 5 +- lib/admin/admin-nav.ts | 2 +- .../__tests__/collect-consent.test.ts | 26 +++ lib/error-reporting/aggregate.ts | 9 +- lib/error-reporting/client.ts | 8 +- lib/error-reporting/settings.ts | 4 +- lib/error-reporting/stack.ts | 7 +- lib/telemetry/index.ts | 6 +- lib/telemetry/payload.ts | 66 ++++++- lib/telemetry/state.ts | 24 +++ lib/telemetry/types.ts | 44 ++++- 14 files changed, 430 insertions(+), 24 deletions(-) create mode 100644 lib/__tests__/telemetry-error-counts.test.ts create mode 100644 lib/error-reporting/__tests__/collect-consent.test.ts diff --git a/app/(main)/admin/_pages/telemetry.tsx b/app/(main)/admin/_pages/telemetry.tsx index aac209d68..1ccdd94a1 100644 --- a/app/(main)/admin/_pages/telemetry.tsx +++ b/app/(main)/admin/_pages/telemetry.tsx @@ -16,12 +16,15 @@ import { cn } from '@/lib/utils'; interface TelemetryStatus { consent: 'pending' | 'on' | 'off'; consentSource: 'no-third-party' | 'env' | 'file'; + errorsConsent?: 'on' | 'off'; + errorsConsentSource?: 'telemetry' | 'no-third-party' | 'env' | 'file'; endpoint: string; defaultEndpoint: string; consentedAt: string | null; lastSentAt: string | null; nextScheduledAt: string | null; payloadPreview: Record; + errorsPreview?: unknown[]; accountCounts: { total: number; active7d: number }; } @@ -56,10 +59,12 @@ export function TelemetryPage() { // The heartbeat switch waits for Save like the endpoint (one page, one // save bar). const [consentDraft, setConsentDraft] = useState<'on' | 'off' | null>(null); + const [errorsDraft, setErrorsDraft] = useState<'on' | 'off' | null>(null); const endpointDirty = status !== null && endpointDraft !== status.endpoint; const consentDirty = status !== null && consentDraft !== null && consentDraft !== status.consent; - const dirty = endpointDirty || consentDirty; + const errorsDirty = status !== null && errorsDraft !== null && errorsDraft !== (status.errorsConsent ?? 'off'); + const dirty = endpointDirty || consentDirty || errorsDirty; useAdminUnsavedChanges('telemetry', dirty); async function refresh(): Promise { @@ -99,11 +104,17 @@ export function TelemetryPage() { const error = await post({ action: 'set-consent', consent: consentDraft }); if (error) errors.push(`Heartbeats: ${error}`); } + // After the heartbeat switch: error counts need it on. + if (errorsDirty && errorsDraft) { + const error = await post({ action: 'set-errors-consent', consent: errorsDraft }); + if (error) errors.push(`Error counts: ${error}`); + } if (endpointDirty) { const error = await post({ action: 'set-endpoint', endpoint: endpointDraft }); if (error) errors.push(`Endpoint: ${error}`); } setConsentDraft(null); + setErrorsDraft(null); setMessage(errors.length > 0 ? { type: 'error', text: `Couldn't save: ${errors.join('; ')}` } : { type: 'success', text: 'Telemetry settings saved.' }); @@ -146,6 +157,20 @@ export function TelemetryPage() { const isOn = status.consent === 'on'; const shownOn = (consentDraft ?? status.consent) === 'on'; + // Error counts follow the heartbeat switch as it is about to be saved. + const errorsLocked = noThirdParty || status.errorsConsentSource === 'env' || !shownOn; + const errorsShownOn = shownOn && (errorsDraft ?? status.errorsConsent ?? 'off') === 'on'; + const errorsDescription = [ + 'Adds how often each error happened yesterday, the error name and where in the code it happened. No message text, no URLs, no instance id goes to the error tracker.', + !shownOn && 'Needs the heartbeat on.', + status.errorsConsentSource === 'env' && 'Locked by the BULWARK_TELEMETRY_ERRORS env var.', + ].filter(Boolean).join(' '); + + // The payload as it will be once the switches are saved. + const previewPayload = errorsShownOn + ? { ...status.payloadPreview, schema: '2', errors: status.errorsPreview ?? [] } + : (({ errors: _omit, ...v1 }) => ({ ...v1, schema: '1' }))(status.payloadPreview as Record & { errors?: unknown }); + const consentDescription = [ status.consent === 'pending' && 'Initialising - no heartbeats sent yet.', status.consent === 'on' && status.endpoint && 'Heartbeats are enabled. Thanks for helping us improve!', @@ -181,7 +206,18 @@ export function TelemetryPage() { { setConsentDraft(checked ? 'on' : 'off'); setMessage(null); }} + onChange={(checked) => { + setConsentDraft(checked ? 'on' : 'off'); + if (!checked) setErrorsDraft('off'); + setMessage(null); + }} + /> + + + { setErrorsDraft(checked ? 'on' : 'off'); setMessage(null); }} /> @@ -252,7 +288,7 @@ export function TelemetryPage() { 
-            {JSON.stringify(status.payloadPreview, null, 2)}
+            {JSON.stringify(previewPayload, null, 2)}
@@ -261,7 +297,7 @@ export function TelemetryPage() { dirty={dirty} saving={busy === 'save'} onSave={() => void save()} - onDiscard={() => { setEndpointDraft(status.endpoint); setConsentDraft(null); }} + onDiscard={() => { setEndpointDraft(status.endpoint); setConsentDraft(null); setErrorsDraft(null); }} saveLabel="Save" /> diff --git a/app/api/admin/telemetry/route.ts b/app/api/admin/telemetry/route.ts index aac9d0263..9c8068e2e 100644 --- a/app/api/admin/telemetry/route.ts +++ b/app/api/admin/telemetry/route.ts @@ -4,6 +4,7 @@ import { auditLog } from '@/lib/admin/audit'; import { logger } from '@/lib/logger'; import { effectiveConsent, + effectiveErrorsConsent, updateState, buildPayload, sendOnce, @@ -13,6 +14,8 @@ import { resolveEndpointAllowed, } from '@/lib/telemetry'; import { storedDataUnavailableResponse } from '@/lib/stored-data-errors'; +import { errorsForPayload } from '@/lib/telemetry/payload'; +import { readErrors } from '@/lib/error-reporting/aggregate'; /** * GET /api/admin/telemetry @@ -25,21 +28,29 @@ export async function GET(request: NextRequest) { if ('error' in auth) return auth.error; const { consent, source, state } = await effectiveConsent(); - const [payload, accountCounts] = await Promise.all([ + const [payload, accountCounts, errors, counted] = await Promise.all([ buildPayload(), getLoginCounts(), + effectiveErrorsConsent(), + readErrors(), ]); return NextResponse.json( { consent, consentSource: source, + // The separate consent to add error counts (schema v2). + errorsConsent: errors.consent, + errorsConsentSource: errors.source, endpoint: state.endpoint ?? DEFAULT_ENDPOINT, consentedAt: state.consentedAt, lastSentAt: state.lastSentAt, nextScheduledAt: state.nextScheduledAt, defaultEndpoint: DEFAULT_ENDPOINT, payloadPreview: payload, + // What `errors` would hold, so the page can show the v2 payload as soon + // as the switch is flipped, before anything is saved or sent. + errorsPreview: errorsForPayload(counted), accountCounts, }, { headers: { 'Cache-Control': 'no-store' } }, @@ -57,7 +68,8 @@ export async function GET(request: NextRequest) { /** * POST /api/admin/telemetry * Body: { action: 'set-consent' | 'set-endpoint' | 'send-now', ... } - * set-consent : { action, consent: 'on' | 'off' } + * set-consent : { action, consent: 'on' | 'off' } (off also switches error counts off) + * set-errors-consent : { action, consent: 'on' | 'off' } * set-endpoint : { action, endpoint: string } * send-now : { action } */ @@ -100,12 +112,39 @@ export async function POST(request: NextRequest) { before = state.consent; state.consent = consent; if (consent === 'on' && !state.consentedAt) state.consentedAt = now; + // Error counts ride on the heartbeat: no heartbeats, no error counts, + // and turning heartbeats on again doesn't bring them back by itself. + if (consent === 'off') state.errorsConsent = 'off'; }); await reschedule(); await auditLog('telemetry.set-consent', { from: before, to: body.consent }, ip); return NextResponse.json({ ok: true }); } + if (body.action === 'set-errors-consent') { + const errors = await effectiveErrorsConsent(); + if (errors.source === 'env') { + return NextResponse.json({ error: 'error counts are set by the BULWARK_TELEMETRY_ERRORS env var' }, { status: 409 }); + } + if (errors.source === 'no-third-party') { + return NextResponse.json({ error: 'telemetry is off while NO_THIRD_PARTY_TRAFFIC is on' }, { status: 409 }); + } + if (body.consent !== 'on' && body.consent !== 'off') { + return NextResponse.json({ error: 'consent must be "on" or "off"' }, { status: 400 }); + } + if (body.consent === 'on' && errors.source === 'telemetry') { + return NextResponse.json({ error: 'turn on the heartbeat first' }, { status: 409 }); + } + const consent = body.consent; + let before: string | undefined; + await updateState((state) => { + before = state.errorsConsent ?? 'off'; + state.errorsConsent = consent; + }); + await auditLog('telemetry.set-errors-consent', { from: before, to: consent }, ip); + return NextResponse.json({ ok: true }); + } + if (body.action === 'set-endpoint') { if (typeof body.endpoint !== 'string') { return NextResponse.json({ error: 'endpoint required' }, { status: 400 }); diff --git a/lib/__tests__/telemetry-error-counts.test.ts b/lib/__tests__/telemetry-error-counts.test.ts new file mode 100644 index 000000000..5868c0442 --- /dev/null +++ b/lib/__tests__/telemetry-error-counts.test.ts @@ -0,0 +1,166 @@ +// @vitest-environment node +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { NextRequest } from 'next/server'; + +// Error counts in the heartbeat (schema 2) have their own consent, off by +// default and off whenever telemetry is off. Without it the heartbeat is +// exactly schema 1. + +const thirdParty = vi.hoisted(() => ({ on: false })); +vi.mock('@/lib/admin/third-party-mode', () => ({ isNoThirdPartyTrafficOn: async () => thirdParty.on })); +vi.mock('@/lib/admin/session', () => ({ + requireAdminAuth: vi.fn(async () => ({ session: { sub: 'admin' } })), + getClientIP: () => '127.0.0.1', +})); +vi.mock('@/lib/admin/audit', () => ({ auditLog: vi.fn(async () => {}) })); +vi.mock('@/lib/admin/plugin-registry', () => ({ + getPluginRegistry: async () => ({ plugins: [] }), + getThemeRegistry: async () => ({ themes: [] }), +})); + +import { GET, POST } from '@/app/api/admin/telemetry/route'; +import { buildPayload, effectiveErrorsConsent, errorsForPayload, loadState, saveState } from '@/lib/telemetry'; +import { resetAggregateForTests, utcDay, type Aggregate } from '@/lib/error-reporting/aggregate'; + +let dir: string; +const DAY = 86_400_000; +const NOW = Date.parse('2026-11-10T12:00:00Z'); +const debugId = 'bc67f7db-7f3b-7be2-6f94-966cd7825cc2'; + +function post(body: Record) { + return POST(new NextRequest('http://localhost/api/admin/telemetry', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + })); +} + +function entry(count: number, extra: Partial = {}) { + return { + name: 'TypeError', + frames: [ + { path: '/_next/static/chunks/1.js', line: 2, column: 30, debugId }, + { path: 'node_modules/next/dist/x.js', line: 1, column: 1 }, + { path: '[other]' }, + ], + version: '1.20.0', + runtime: 'browser' as const, + count, + ...extra, + }; +} + +beforeAll(() => { + dir = mkdtempSync(path.join(tmpdir(), 'bulwark-telemetry-errors-')); + vi.stubEnv('ADMIN_CONFIG_DIR', dir); + vi.stubEnv('TELEMETRY_DATA_DIR', path.join(dir, 'telemetry')); +}); +afterAll(() => { + vi.unstubAllEnvs(); + rmSync(dir, { recursive: true, force: true }); +}); +beforeEach(async () => { + thirdParty.on = false; + vi.stubEnv('BULWARK_TELEMETRY', ''); + vi.stubEnv('BULWARK_TELEMETRY_ERRORS', ''); + resetAggregateForTests(); + await saveState({ ...(await loadState()), consent: 'on', errorsConsent: 'off' }); + const yesterday = utcDay(Date.now() - DAY); + mkdirSync(path.join(dir, 'telemetry'), { recursive: true }); + writeFileSync(path.join(dir, 'telemetry', 'errors.json'), JSON.stringify({ [yesterday]: { aaaa000011112222: entry(7) } })); +}); +afterEach(() => { + resetAggregateForTests(); +}); + +describe('error counts consent', () => { + it('is off by default and the heartbeat stays schema 1, without errors', async () => { + expect((await effectiveErrorsConsent()).consent).toBe('off'); + const payload = await buildPayload(); + expect(payload.schema).toBe('1'); + expect('errors' in payload).toBe(false); + }); + + it('turns the heartbeat into schema 2 with the counts when on', async () => { + expect((await post({ action: 'set-errors-consent', consent: 'on' })).status).toBe(200); + const payload = await buildPayload(); + expect(payload.schema).toBe('2'); + expect(payload.schema === '2' && payload.errors).toEqual([{ + name: 'TypeError', + fingerprint: 'aaaa000011112222', + frames: [{ path: '/_next/static/chunks/1.js', line: 2, col: 30, debug_id: debugId }], + version: '1.20.0', + count: '6-10', + days: 1, + }]); + }); + + it('can\'t be turned on while telemetry is off, and goes off with it', async () => { + await saveState({ ...(await loadState()), consent: 'off' }); + expect((await post({ action: 'set-errors-consent', consent: 'on' })).status).toBe(409); + + await saveState({ ...(await loadState()), consent: 'on', errorsConsent: 'on' }); + expect((await effectiveErrorsConsent()).consent).toBe('on'); + await post({ action: 'set-consent', consent: 'off' }); + expect((await loadState()).errorsConsent).toBe('off'); + await post({ action: 'set-consent', consent: 'on' }); + expect((await effectiveErrorsConsent()).consent).toBe('off'); + }); + + it('is off in no-third-party mode, whatever is stored or set in the env', async () => { + await saveState({ ...(await loadState()), errorsConsent: 'on' }); + vi.stubEnv('BULWARK_TELEMETRY_ERRORS', 'on'); + thirdParty.on = true; + expect(await effectiveErrorsConsent()).toEqual({ consent: 'off', source: 'no-third-party' }); + }); + + it('follows BULWARK_TELEMETRY_ERRORS and refuses changes from the panel then', async () => { + vi.stubEnv('BULWARK_TELEMETRY_ERRORS', 'on'); + expect(await effectiveErrorsConsent()).toEqual({ consent: 'on', source: 'env' }); + expect((await post({ action: 'set-errors-consent', consent: 'off' })).status).toBe(409); + }); + + it('shows the v2 errors in the admin preview before anything is turned on', async () => { + const body = await (await GET(new NextRequest('http://localhost/api/admin/telemetry'))).json(); + expect(body.payloadPreview.schema).toBe('1'); + expect(body.errorsConsent).toBe('off'); + expect(body.errorsPreview).toHaveLength(1); + }); +}); + +describe('errors in the payload', () => { + const yesterday = utcDay(NOW - DAY); + + it('come from the last complete day, most frequent first, at most 50', () => { + const day: Aggregate[string] = {}; + for (let i = 0; i < 60; i++) day[`f${String(i).padStart(15, '0')}`] = entry(i + 1); + const errors = errorsForPayload({ [yesterday]: day, [utcDay(NOW)]: { today: entry(999) } }, NOW); + expect(errors).toHaveLength(50); + expect(errors[0].fingerprint).toBe('f000000000000059'); + expect(errors.some((e) => e.fingerprint === 'today')).toBe(false); + }); + + it('count the days of the last week a fingerprint occurred on', () => { + const agg: Aggregate = { + [yesterday]: { f: entry(1) }, + [utcDay(NOW - 3 * DAY)]: { f: entry(1) }, + [utcDay(NOW - 9 * DAY)]: { f: entry(1) }, + }; + expect(errorsForPayload(agg, NOW)[0].days).toBe(2); + }); + + it('carry a message only when it is a fixed error code', () => { + const agg: Aggregate = { + [yesterday]: { + a: entry(1, { name: 'JmapMethodError', code: 'Email/set: overQuota' }), + b: entry(1, { code: 'Merger plans for Acme' as string }), + }, + }; + const [withCode, without] = errorsForPayload(agg, NOW); + expect(withCode.message).toBe('Email/set: overQuota'); + expect(without.message).toBeUndefined(); + }); +}); diff --git a/lib/__tests__/telemetry-payload.test.ts b/lib/__tests__/telemetry-payload.test.ts index 2cc9b138c..399eb5bef 100644 --- a/lib/__tests__/telemetry-payload.test.ts +++ b/lib/__tests__/telemetry-payload.test.ts @@ -8,7 +8,10 @@ vi.mock('@/lib/admin/config-manager', () => ({ get: (key: string, fallback: unknown) => (key === 'jmapServerUrl' ? 'https://mail.example.org' : fallback), }, })); -vi.mock('@/lib/telemetry/state', () => ({ getInstanceId: async () => 'instance-1' })); +vi.mock('@/lib/telemetry/state', () => ({ + getInstanceId: async () => 'instance-1', + effectiveErrorsConsent: async () => ({ consent: 'off', source: 'file' }), +})); vi.mock('@/lib/telemetry/login-tracker', () => ({ getLoginCounts: async () => ({ total: 3, active7d: 1 }) })); vi.mock('@/lib/admin/plugin-registry', () => ({ getPluginRegistry: async () => ({ plugins: [] }), diff --git a/lib/admin/admin-nav.ts b/lib/admin/admin-nav.ts index 6352d28bb..a9637b163 100644 --- a/lib/admin/admin-nav.ts +++ b/lib/admin/admin-nav.ts @@ -238,7 +238,7 @@ export const ADMIN_PAGES: readonly AdminPageDef[] = [ group: 'system', keywords: 'telemetry heartbeat anonymous consent privacy', sections: [ - { id: 'status', title: 'Status', keywords: 'enable disable consent' }, + { id: 'status', title: 'Status', keywords: 'enable disable consent error counts' }, { id: 'accounts', title: 'Account activity', keywords: 'active total logins' }, { id: 'endpoint', title: 'Endpoint', keywords: 'collector url' }, { id: 'payload', title: 'Payload preview', keywords: 'send now json' }, diff --git a/lib/error-reporting/__tests__/collect-consent.test.ts b/lib/error-reporting/__tests__/collect-consent.test.ts new file mode 100644 index 000000000..93e82d937 --- /dev/null +++ b/lib/error-reporting/__tests__/collect-consent.test.ts @@ -0,0 +1,26 @@ +// @vitest-environment node +import { describe, expect, it, vi } from 'vitest'; + +const consent = vi.hoisted(() => ({ value: 'off' as 'on' | 'off' })); +vi.mock('@/lib/admin/config-manager', () => ({ + configManager: { ensureLoaded: async () => {}, get: (_key: string, fallback: unknown) => fallback }, +})); +vi.mock('@/lib/telemetry/state', () => ({ + effectiveErrorsConsent: async () => ({ consent: consent.value, source: 'file' }), +})); + +import { shouldCollectErrors } from '../settings'; + +// Error counts in the heartbeat need errors counted on this server, so the +// consent alone turns collection on; with nothing set it stays off. +describe('error collection', () => { + it('is off with no DSN, no metrics and no error-count consent', async () => { + consent.value = 'off'; + expect(await shouldCollectErrors()).toBe(false); + }); + + it('is on with the error-count consent alone', async () => { + consent.value = 'on'; + expect(await shouldCollectErrors()).toBe(true); + }); +}); diff --git a/lib/error-reporting/aggregate.ts b/lib/error-reporting/aggregate.ts index 5d933ce34..75c6fb484 100644 --- a/lib/error-reporting/aggregate.ts +++ b/lib/error-reporting/aggregate.ts @@ -3,6 +3,7 @@ import { telemetryDocs } from '@/lib/telemetry/state'; import { isErrorCode } from './envelope'; import { fingerprint as fingerprintOf } from './fingerprint'; import type { Frame } from './stack'; +import { debugIdsByPath } from './debug-ids'; /** * Errors counted per UTC day: fingerprint -> name, top frames, version and @@ -106,7 +107,13 @@ export function recordError(report: Recordable, runtime: 'browser' | 'server', v } else if (Object.keys(entries).length < MAX_PER_DAY) { entries[print] = { name: report.name, - frames: report.frames.slice(0, MAX_FRAMES_KEPT), + // The server's own bundle files get this process's debug IDs; browser + // frames bring theirs along. + frames: report.frames.slice(0, MAX_FRAMES_KEPT).map((frame) => { + if (frame.debugId) return frame; + const id = runtime === 'server' ? debugIdsByPath().get(frame.path) : undefined; + return id ? { ...frame, debugId: id } : frame; + }), version, ...(isErrorCode(report.code) ? { code: report.code } : {}), runtime, diff --git a/lib/error-reporting/client.ts b/lib/error-reporting/client.ts index c329d6b06..7debc1d8d 100644 --- a/lib/error-reporting/client.ts +++ b/lib/error-reporting/client.ts @@ -205,7 +205,8 @@ function deliver(report: ErrorReport): void { name: report.name, message: report.message, code: report.code, - frames: report.frames.slice(0, 10), + // With debug IDs, so the counts can point at the uploaded source maps. + frames: withDebugIds(report.frames.slice(0, 10)), mechanism: report.mechanism, handled: report.handled, route: report.route, @@ -222,6 +223,11 @@ function send(url: string, body: string, init: RequestInit, sameOrigin = false): void Promise.resolve(request).catch(() => undefined); } +function withDebugIds(frames: ErrorReport['frames']): ErrorReport['frames'] { + const ids = debugIdsByPath(); + return frames.map((frame) => (ids.has(frame.path) ? { ...frame, debugId: ids.get(frame.path) } : frame)); +} + /** window.onerror and unhandled rejections, once. */ export function installGlobalHandlers(): void { const s = state(); diff --git a/lib/error-reporting/settings.ts b/lib/error-reporting/settings.ts index a37cbb68c..64092419a 100644 --- a/lib/error-reporting/settings.ts +++ b/lib/error-reporting/settings.ts @@ -1,6 +1,7 @@ import { configManager } from '@/lib/admin/config-manager'; import { parseDsn } from './envelope'; import { metricsEnabled } from '@/lib/metrics'; +import { effectiveErrorsConsent } from '@/lib/telemetry/state'; import type { ClientSettings } from './client'; /** @@ -44,7 +45,8 @@ export function errorReportingSettings(): ErrorReportingSettings { */ export async function shouldCollectErrors(): Promise { await configManager.ensureLoaded(); - return !!errorReportingSettings().dsn || metricsEnabled(); + if (errorReportingSettings().dsn || metricsEnabled()) return true; + return (await effectiveErrorsConsent()).consent === 'on'; } /** What /api/config hands the browser. */ diff --git a/lib/error-reporting/stack.ts b/lib/error-reporting/stack.ts index 1c228eb99..6a6b83f00 100644 --- a/lib/error-reporting/stack.ts +++ b/lib/error-reporting/stack.ts @@ -7,8 +7,12 @@ export interface Frame { line?: number; column?: number; fn?: string; + /** The debug ID of the file, when its bundle registered one (./debug-ids.ts). */ + debugId?: string; } +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + export const MAX_FRAMES = 30; // V8 (Chrome, Edge, Node): " at fn (file:1:2)", " at file:1:2", @@ -51,7 +55,8 @@ export function sanitizeFrames(input: unknown): Frame[] { const frame = raw as Record; const line = typeof frame.line === 'number' ? number(String(frame.line)) : undefined; const column = typeof frame.column === 'number' ? number(String(frame.column)) : undefined; - out.push({ path: scrubFramePath(frame.path), line, column, fn: scrubFunctionName(frame.fn) }); + const debugId = typeof frame.debugId === 'string' && UUID.test(frame.debugId) ? frame.debugId.toLowerCase() : undefined; + out.push({ path: scrubFramePath(frame.path), line, column, fn: scrubFunctionName(frame.fn), ...(debugId ? { debugId } : {}) }); } return out; } diff --git a/lib/telemetry/index.ts b/lib/telemetry/index.ts index 2cc956d4c..9abd3e16f 100644 --- a/lib/telemetry/index.ts +++ b/lib/telemetry/index.ts @@ -1,7 +1,7 @@ export { startScheduler, stopScheduler, reschedule, sendOnce } from './sender'; -export { buildPayload, markProcessStart } from './payload'; +export { buildPayload, errorsForPayload, markProcessStart } from './payload'; export { - loadState, saveState, updateState, getInstanceId, effectiveConsent, + loadState, saveState, updateState, getInstanceId, effectiveConsent, effectiveErrorsConsent, } from './state'; export { recordLogin, getLoginCounts } from './login-tracker'; export { @@ -9,7 +9,7 @@ export { } from './endpoint-guard'; export type { EndpointCheck } from './endpoint-guard'; export type { - TelemetryPayload, TelemetryStateFile, ConsentState, + TelemetryPayload, TelemetryPayloadV2, AnyTelemetryPayload, TelemetryError, TelemetryStateFile, ConsentState, Platform, OsFamily, CountBucket, TelemetryFeatures, } from './types'; export { DEFAULT_ENDPOINT } from './types'; diff --git a/lib/telemetry/payload.ts b/lib/telemetry/payload.ts index 5928d41cd..fe4ecbce2 100644 --- a/lib/telemetry/payload.ts +++ b/lib/telemetry/payload.ts @@ -1,14 +1,20 @@ import { readFileSync } from 'node:fs'; import path from 'node:path'; import { configManager } from '@/lib/admin/config-manager'; -import { getInstanceId } from './state'; +import { effectiveErrorsConsent, getInstanceId } from './state'; import { getLoginCounts } from './login-tracker'; -import type { - TelemetryPayload, - TelemetryFeatures, - Platform, - OsFamily, - CountBucket, +import { isErrorCode } from '@/lib/error-reporting/envelope'; +import { readErrors, utcDay, type Aggregate } from '@/lib/error-reporting/aggregate'; +import { + MAX_TELEMETRY_ERRORS, + MAX_TELEMETRY_FRAMES, + type AnyTelemetryPayload, + type TelemetryError, + type TelemetryPayload, + type TelemetryFeatures, + type Platform, + type OsFamily, + type CountBucket, } from './types'; let processStartedAt = Date.now(); @@ -99,7 +105,51 @@ async function countExtensions(): Promise<{ extensions: number; themes: number } } } -export async function buildPayload(): Promise { +const BUNDLE_PATH = /^\/(?:_next\/static|\.next\/server)\/[\w./()[\]@+~-]{1,200}$/; +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; + +/** + * The error counts of the last complete UTC day (the day before `now`), for + * schema v2: at most MAX_TELEMETRY_ERRORS, the most frequent first. Only + * frames in Bulwark's own bundles go (no extension, plugin or Node paths), + * and a message only when it is one of Bulwark's fixed error codes. + */ +export function errorsForPayload(agg: Aggregate, now = Date.now()): TelemetryError[] { + const day = utcDay(now - 86_400_000); + const recentDays = Array.from({ length: 7 }, (_, i) => utcDay(now - (i + 1) * 86_400_000)); + return Object.entries(agg[day] ?? {}) + .sort(([, a], [, b]) => b.count - a.count) + .slice(0, MAX_TELEMETRY_ERRORS) + .map(([fingerprint, entry]) => ({ + name: entry.name, + fingerprint, + frames: entry.frames + .filter((frame) => BUNDLE_PATH.test(frame.path)) + .slice(0, MAX_TELEMETRY_FRAMES) + .map((frame) => ({ + path: frame.path, + line: frame.line ?? null, + col: frame.column ?? null, + debug_id: frame.debugId && UUID.test(frame.debugId) ? frame.debugId : null, + })), + version: entry.version, + count: bucketCount(entry.count), + days: recentDays.filter((d) => agg[d]?.[fingerprint]).length, + ...(isErrorCode(entry.code) ? { message: entry.code } : {}), + })); +} + +/** + * The heartbeat. Schema 1, exactly as before, unless the admin also agreed + * to send error counts; then schema 2 with `errors` added. + */ +export async function buildPayload(): Promise { + const base = await buildPayloadV1(); + if ((await effectiveErrorsConsent()).consent !== 'on') return base; + return { ...base, schema: '2', errors: errorsForPayload(await readErrors()) }; +} + +async function buildPayloadV1(): Promise { const instance_id = await getInstanceId(); const { version, build } = readPackage(); const features = await readFeatures(); diff --git a/lib/telemetry/state.ts b/lib/telemetry/state.ts index 22a6e3645..03e7d3548 100644 --- a/lib/telemetry/state.ts +++ b/lib/telemetry/state.ts @@ -174,6 +174,30 @@ export async function saveStateIfUnchanged(state: TelemetryStateFile, version: s return (await telemetryDocs().put(STATE_DOC, serialize(state), { ifVersion: version })).ok; } +function errorsEnvOverride(): 'on' | 'off' | null { + const v = (process.env.BULWARK_TELEMETRY_ERRORS ?? '').toLowerCase(); + if (v === 'off' || v === 'false' || v === '0' || v === 'no') return 'off'; + if (v === 'on' || v === 'true' || v === '1' || v === 'yes') return 'on'; + return null; +} + +/** + * Effective consent to send error counts (schema v2): off whenever + * telemetry itself is off (which covers the no-third-party mode), then + * BULWARK_TELEMETRY_ERRORS, then the stored choice, off by default. + */ +export async function effectiveErrorsConsent(): Promise<{ + consent: 'on' | 'off'; + source: 'telemetry' | 'no-third-party' | 'env' | 'file'; +}> { + const telemetry = await effectiveConsent(); + if (telemetry.source === 'no-third-party') return { consent: 'off', source: 'no-third-party' }; + if (telemetry.consent !== 'on') return { consent: 'off', source: 'telemetry' }; + const env = errorsEnvOverride(); + if (env) return { consent: env, source: 'env' }; + return { consent: telemetry.state.errorsConsent === 'on' ? 'on' : 'off', source: 'file' }; +} + // Effective consent: the no-third-party mode wins over everything (#903), // then the env var, then the file. UI changes are blocked when either // override is active so the user knows where it's coming from. diff --git a/lib/telemetry/types.ts b/lib/telemetry/types.ts index 0e8fccfe2..3e1bc091d 100644 --- a/lib/telemetry/types.ts +++ b/lib/telemetry/types.ts @@ -1,4 +1,5 @@ -// Schema v1 of the anonymous heartbeat. Documented at +// Schema v1 of the anonymous heartbeat, and v2 (v1 plus error counts, sent +// only with the separate error-count consent). Documented at // https://bulwarkmail.org/docs/legal/privacy/telemetry export type ConsentState = 'pending' | 'on' | 'off'; @@ -36,6 +37,42 @@ export interface TelemetryPayload { uptime_days: number; } +/** At most this many errors in a v2 heartbeat, the most frequent first. */ +export const MAX_TELEMETRY_ERRORS = 50; +/** Top frames per error. */ +export const MAX_TELEMETRY_FRAMES = 5; + +export interface TelemetryErrorFrame { + /** The bundle file without origin: `/_next/static/chunks/x.js` or `/.next/server/...`. */ + path: string; + line: number | null; + col: number | null; + /** The file's debug ID, which finds its uploaded source map. */ + debug_id: string | null; +} + +/** One error fingerprint of the last complete UTC day. */ +export interface TelemetryError { + /** The error's class name (TypeError, JmapMethodError). */ + name: string; + fingerprint: string; + frames: TelemetryErrorFrame[]; + version: string; + /** How often that day, bucketed like the account counts. */ + count: CountBucket; + /** On how many of the last 7 days it occurred (1-7). */ + days: number; + /** Only one of Bulwark's fixed error codes (`Email/set: overQuota`), never free text. */ + message?: string; +} + +export interface TelemetryPayloadV2 extends Omit { + schema: '2'; + errors: TelemetryError[]; +} + +export type AnyTelemetryPayload = TelemetryPayload | TelemetryPayloadV2; + export interface TelemetryStateFile { consent: ConsentState; endpoint: string; @@ -47,6 +84,11 @@ export interface TelemetryStateFile { */ lastScheduledSentAt?: string | null; nextScheduledAt: string | null; + /** + * The separate consent to add error counts (schema v2). Off unless an + * admin turns it on; switching telemetry off switches it off too. + */ + errorsConsent?: 'on' | 'off'; } export const DEFAULT_ENDPOINT = 'https://telemetry.bulwarkmail.org/v1/heartbeat'; From 7cb0c5f97294b2a308da2a1e5f4f74cec8e6f468 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 03:20:52 +0100 Subject: [PATCH 09/26] chore: use @prometheus-io/client, the maintained successor of the deprecated prom-client --- lib/metrics/index.ts | 4 ++-- next.config.ts | 6 +++--- package-lock.json | 29 ++++++++++++++--------------- package.json | 2 +- 4 files changed, 20 insertions(+), 21 deletions(-) diff --git a/lib/metrics/index.ts b/lib/metrics/index.ts index dfdea1749..2bf5a33b8 100644 --- a/lib/metrics/index.ts +++ b/lib/metrics/index.ts @@ -1,6 +1,6 @@ import { readFileSync } from 'node:fs'; -import type { Counter, Gauge, Histogram, Registry } from 'prom-client'; -import client from 'prom-client'; +import type { Counter, Gauge, Histogram, Registry } from '@prometheus-io/client'; +import client from '@prometheus-io/client'; import { routeGroup } from '@/lib/route-groups'; /** diff --git a/next.config.ts b/next.config.ts index cb20d9f8c..2435bd968 100644 --- a/next.config.ts +++ b/next.config.ts @@ -61,9 +61,9 @@ const nextConfig: NextConfig = { // esbuild ships native binaries + a README the bundler can't parse; load // it from node_modules at runtime instead of trying to bundle it. Used by // PLUGIN_DEV_DIR's on-the-fly bundler. ioredis (the shared store, #1066) - // and prom-client (/api/metrics) are loaded from node_modules too, so the - // proxy and the route bundles don't each carry a copy. - serverExternalPackages: ["esbuild", "ioredis", "prom-client"], + // and @prometheus-io/client (/api/metrics) are loaded from node_modules + // too, so the proxy and the route bundles don't each carry a copy. + serverExternalPackages: ["esbuild", "ioredis", "@prometheus-io/client"], // The build's type check leaves out tests, which are a third of the files // and push tsc past the default heap on a 4 GB machine (#1431). // `npm run typecheck` still checks them with tsconfig.json. diff --git a/package-lock.json b/package-lock.json index 055af593f..4f178ae26 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,6 +21,7 @@ "@opentelemetry/sdk-trace-base": "2.8.0", "@opentelemetry/sdk-trace-node": "2.8.0", "@opentelemetry/semantic-conventions": "1.44.0", + "@prometheus-io/client": "0.16.1", "@tabler/icons-react": "^3.50.0", "@tanstack/react-virtual": "^3.14.23", "@tiptap/extension-color": "^3.32.1", @@ -50,7 +51,6 @@ "otpauth": "^9.5.2", "pdfjs-dist": "^6.4.299", "postal-mime": "^4.1.4", - "prom-client": "15.1.3", "qrcode": "^1.5.4", "react": "^19.3.1", "react-dom": "^19.3.1", @@ -2924,6 +2924,19 @@ "dev": true, "license": "MIT" }, + "node_modules/@prometheus-io/client": { + "version": "0.16.1", + "resolved": "https://registry.npmjs.org/@prometheus-io/client/-/client-0.16.1.tgz", + "integrity": "sha512-XkFPsGFGoSs/UMg/vh0QLDpanl/tsrOV5PLHdpGe8Ho+ir/Llhytr15j6y8ZwLrpIGzK9gTVVpIalA6C4UjUZw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.4.0", + "tdigest": "^0.1.1" + }, + "engines": { + "node": "^22 || ^24 || >=26" + } + }, "node_modules/@rolldown/binding-android-arm-eabi": { "version": "1.2.15", "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.15.tgz", @@ -8587,20 +8600,6 @@ "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", "license": "MIT" }, - "node_modules/prom-client": { - "version": "15.1.3", - "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.3.tgz", - "integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==", - "deprecated": "prom-client has been replaced by @prometheus-io/client", - "license": "Apache-2.0", - "dependencies": { - "@opentelemetry/api": "^1.4.0", - "tdigest": "^0.1.1" - }, - "engines": { - "node": "^16 || ^18 || >=20" - } - }, "node_modules/prop-types": { "version": "15.8.1", "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", diff --git a/package.json b/package.json index ae34dfde9..eead11624 100644 --- a/package.json +++ b/package.json @@ -51,6 +51,7 @@ "@opentelemetry/sdk-trace-base": "2.8.0", "@opentelemetry/sdk-trace-node": "2.8.0", "@opentelemetry/semantic-conventions": "1.44.0", + "@prometheus-io/client": "0.16.1", "@tabler/icons-react": "^3.50.0", "@tanstack/react-virtual": "^3.14.23", "@tiptap/extension-color": "^3.32.1", @@ -80,7 +81,6 @@ "otpauth": "^9.5.2", "pdfjs-dist": "^6.4.299", "postal-mime": "^4.1.4", - "prom-client": "15.1.3", "qrcode": "^1.5.4", "react": "^19.3.1", "react-dom": "^19.3.1", From d49839435b2b024195806865183d500d11c61937 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 03:25:28 +0100 Subject: [PATCH 10/26] feat: send OpenTelemetry traces when an OTLP endpoint is configured With `OTEL_EXPORTER_OTLP_ENDPOINT` or `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` set (and neither `OTEL_SDK_DISABLED=true` nor `OTEL_TRACES_EXPORTER=none`), Bulwark loads the OpenTelemetry SDK at startup and exports Next's own spans plus a client span for every call to Stalwart, WOPI, translation, the plugin proxy and OAuth over OTLP/HTTP. Calls to Stalwart carry a `traceparent` header, so Stalwart's spans join the trace. All other settings come from the standard `OTEL_*` variables. Every span is scrubbed before export: URLs keep their origin only, request paths become their route group, query strings, user agents and client addresses are dropped, and addresses or credentials in other attributes are removed. Without an endpoint the SDK is never loaded. The endpoint is listed under the hosts that stay reachable with no third-party traffic, and System > Monitoring shows whether tracing runs. --- app/(main)/admin/_pages/monitoring.tsx | 16 ++++ app/api/admin/monitoring/route.ts | 9 +- instrumentation.ts | 9 ++ lib/admin/admin-nav.ts | 1 + lib/admin/outgoing-hosts-registry.ts | 2 +- lib/admin/reachable-hosts.ts | 4 + lib/metrics/upstream.ts | 53 +++++++++-- lib/shared-store/server-state-allowlist.ts | 1 + lib/stalwart/server-fetch.ts | 10 ++- lib/tracing/__tests__/config.test.ts | 44 +++++++++ lib/tracing/__tests__/tracing.test.ts | 100 +++++++++++++++++++++ lib/tracing/config.ts | 30 +++++++ lib/tracing/scrub.ts | 60 +++++++++++++ lib/tracing/start.ts | 68 ++++++++++++++ package-lock.json | 2 - package.json | 2 - 16 files changed, 398 insertions(+), 13 deletions(-) create mode 100644 lib/tracing/__tests__/config.test.ts create mode 100644 lib/tracing/__tests__/tracing.test.ts create mode 100644 lib/tracing/config.ts create mode 100644 lib/tracing/scrub.ts create mode 100644 lib/tracing/start.ts diff --git a/app/(main)/admin/_pages/monitoring.tsx b/app/(main)/admin/_pages/monitoring.tsx index 24c4f8044..feea399b7 100644 --- a/app/(main)/admin/_pages/monitoring.tsx +++ b/app/(main)/admin/_pages/monitoring.tsx @@ -32,6 +32,7 @@ interface MonitoringState { day: string; errorsToday: { total: number; top: ErrorEntry[] }; metrics?: { enabled: boolean }; + tracing?: { configured: boolean; active: boolean; endpointHost: string | null }; } const PAGE = adminPageDef('monitoring'); @@ -181,6 +182,21 @@ export function MonitoringPage() {
  • /api/health/ready: config loaded, data dirs writable, Stalwart reachable. For readiness probes.
    • + + +

    + {state?.tracing?.active + ? <>On. Spans go to {state.tracing.endpointHost ?? 'the OTLP endpoint'} over OTLP/HTTP. + : state?.tracing?.configured + ? <>Configured, but not running in this process. Restart Bulwark after setting the OTEL_* variables. + : <>Off. Set OTEL_EXPORTER_OTLP_ENDPOINT (and the other standard OTEL_* variables you need) to send traces.} +

    + + Next's request spans plus spans for calls to Stalwart and other services. Stalwart gets the trace + context, so its spans join the trace. URLs, user agents and addresses are removed from spans.{' '} + Setup + +     Monitoring shows besides the * config keys (those come from /api/admin/config): whether errors are being - * counted, today's counts and whether the metrics endpoint is on. + * counted, today's counts, and whether metrics and tracing are on. */ export async function GET(request: NextRequest) { try { @@ -22,7 +23,11 @@ export async function GET(request: NextRequest) { const today = utcDay(); const errorsToday = summarizeDay(await readErrors(), today); const metrics = { enabled: metricsEnabled() }; - return NextResponse.json({ collecting, day: today, errorsToday, metrics }, { headers: { 'Cache-Control': 'no-store' } }); + const endpoint = tracingEndpoint(); + let endpointHost: string | null = null; + try { endpointHost = endpoint ? new URL(endpoint).host : null; } catch { endpointHost = null; } + const tracing = { configured: endpoint !== null, active: isTracingActive(), endpointHost }; + return NextResponse.json({ collecting, day: today, errorsToday, metrics, tracing }, { headers: { 'Cache-Control': 'no-store' } }); } catch (err) { logger.error('monitoring GET error', { error: err instanceof Error ? err.message : 'unknown' }); const unavailable = storedDataUnavailableResponse(err); diff --git a/instrumentation.ts b/instrumentation.ts index a11419dd4..47c86b261 100644 --- a/instrumentation.ts +++ b/instrumentation.ts @@ -1,7 +1,16 @@ import type { Instrumentation } from "next"; +import { tracingConfigured } from "./lib/tracing/config"; export async function register() { if (process.env.NEXT_RUNTIME === "nodejs") { + // OpenTelemetry, only with an OTLP endpoint in the standard OTEL_* + // variables; otherwise the SDK is never loaded. First, so Next's own + // spans from the first request on go to it. + // https://bulwarkmail.org/docs/admin/monitoring#tracing + if (tracingConfigured()) { + const { startTracing } = await import("./lib/tracing/start"); + startTracing(); + } await import("./instrumentation.node"); } } diff --git a/lib/admin/admin-nav.ts b/lib/admin/admin-nav.ts index a9637b163..b110d106f 100644 --- a/lib/admin/admin-nav.ts +++ b/lib/admin/admin-nav.ts @@ -255,6 +255,7 @@ export const ADMIN_PAGES: readonly AdminPageDef[] = [ { id: 'errors-today', title: 'Errors today', keywords: 'count fingerprint top' }, { id: 'metrics', title: 'Metrics', keywords: 'prometheus scrape token grafana' }, { id: 'health', title: 'Health checks', keywords: 'liveness readiness probe kubernetes docker' }, + { id: 'tracing', title: 'Tracing', keywords: 'opentelemetry otel otlp jaeger tempo spans traceparent' }, ], }, { diff --git a/lib/admin/outgoing-hosts-registry.ts b/lib/admin/outgoing-hosts-registry.ts index d32438294..c469e9186 100644 --- a/lib/admin/outgoing-hosts-registry.ts +++ b/lib/admin/outgoing-hosts-registry.ts @@ -157,7 +157,7 @@ export const OUTGOING_HOSTS: readonly OutgoingHost[] = [ * has to be looked at again. */ export const NON_REQUEST_HOSTS: readonly { host: string; files: readonly string[]; why: string }[] = [ - { host: 'bulwarkmail.org', why: 'docs links, comments and the contact URL in User-Agent headers', files: ['app/(main)/admin/_pages/telemetry.tsx', 'app/api/fetch-ical/route.ts', 'app/api/translate/route.ts','components/admin/org/sections/recovery-section.tsx', 'instrumentation.node.ts', 'lib/telemetry/state.ts', 'lib/telemetry/types.ts', 'app/(main)/admin/_pages/monitoring.tsx', 'lib/admin/types.ts', 'lib/error-reporting/client.ts', 'lib/error-reporting/lite.ts', 'lib/error-reporting/scrub.ts', 'instrumentation.ts', 'app/api/health/live/route.ts', 'app/api/health/ready/route.ts', 'app/api/metrics/route.ts', 'lib/metrics/index.ts'] }, + { host: 'bulwarkmail.org', why: 'docs links, comments and the contact URL in User-Agent headers', files: ['app/(main)/admin/_pages/telemetry.tsx', 'app/api/fetch-ical/route.ts', 'app/api/translate/route.ts','components/admin/org/sections/recovery-section.tsx', 'instrumentation.node.ts', 'lib/telemetry/state.ts', 'lib/telemetry/types.ts', 'app/(main)/admin/_pages/monitoring.tsx', 'lib/admin/types.ts', 'lib/error-reporting/client.ts', 'lib/error-reporting/lite.ts', 'lib/error-reporting/scrub.ts', 'instrumentation.ts', 'app/api/health/live/route.ts', 'app/api/health/ready/route.ts', 'app/api/metrics/route.ts', 'lib/metrics/index.ts', 'lib/tracing/config.ts', 'lib/tracing/scrub.ts', 'lib/tracing/start.ts'] }, { host: 'bulwarkmail.com', why: 'JMAP capability URI', files: ['lib/jmap/client.ts'] }, { host: 'develop.sentry.dev', why: 'link to the envelope format in a comment', files: ['lib/error-reporting/envelope.ts'] }, { host: 'extensions.bulwarkmail.org', why: 'signed statement type URIs', files: ['lib/admin/directory-keys.ts'] }, diff --git a/lib/admin/reachable-hosts.ts b/lib/admin/reachable-hosts.ts index 8d848815b..143c8dc33 100644 --- a/lib/admin/reachable-hosts.ts +++ b/lib/admin/reachable-hosts.ts @@ -2,6 +2,7 @@ import type { SettingsPolicy } from './types'; import { parseJmapServers } from './jmap-servers'; import { isBuiltInHost } from '@/lib/outgoing-hosts'; import { DEFAULT_RELAY_BASE_URL } from '@/lib/push-relays'; +import { tracingEndpoint } from '@/lib/tracing/config'; /** * What stays reachable with "no third-party traffic" on: the hosts the @@ -93,6 +94,9 @@ export function listReachableHosts({ get, policy, env, buildRelayUrl = DEFAULT_R add('Translation (LibreTranslate)', env.LIBRETRANSLATE_URL, 'LIBRETRANSLATE_URL', 'server'); // Only when an admin set it; browsers and this server send error reports there. add('Error reporting', get('errorReportingDsn'), 'ERROR_REPORTING_DSN', 'both'); + // OpenTelemetry's OTLP endpoint, only when set in the standard OTEL_* variables. + const otlp = tracingEndpoint(env); + if (otlp) add('Tracing (OTLP)', otlp, env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT ? 'OTEL_EXPORTER_OTLP_TRACES_ENDPOINT' : 'OTEL_EXPORTER_OTLP_ENDPOINT', 'server'); for (const key of BRANDING_KEYS) add('Branding image', get(key), key, 'browser'); // User-directed: what users type in themselves stays allowed. diff --git a/lib/metrics/upstream.ts b/lib/metrics/upstream.ts index 5aaf47fca..e7cd1493a 100644 --- a/lib/metrics/upstream.ts +++ b/lib/metrics/upstream.ts @@ -1,12 +1,31 @@ import { observeUpstreamResult, type UpstreamTarget } from './index'; +import { isTracingActive } from '@/lib/tracing/config'; + +/** Headers that carry the trace context to the service called (`traceparent`). */ +export type TraceHeaders = Record; /** - * Run one request this server makes to another service and record its - * duration and outcome under a fixed target name. An answer below 500 counts - * as ok (a 401 from Stalwart is an answer); a 5xx or a thrown error as - * error. Costs nothing while metrics are off. + * Run one request this server makes to another service, under a fixed + * target name: + * + * - metrics (when /api/metrics is on): duration and outcome. An answer + * below 500 counts as ok (a 401 from Stalwart is an answer); a 5xx or a + * thrown error as error. + * - tracing (when OTEL_* is set): a client span named after the target, + * with no URL, host or user in it, and the W3C trace context handed to + * `run` as headers, so the callee's spans join the trace. + * + * Costs nothing while both are off. */ -export async function observeUpstream(target: UpstreamTarget, run: () => Promise): Promise { +export async function observeUpstream( + target: UpstreamTarget, + run: (traceHeaders: TraceHeaders) => Promise, +): Promise { + if (isTracingActive()) return traced(target, run); + return measured(target, () => run({})); +} + +async function measured(target: UpstreamTarget, run: () => Promise): Promise { const start = performance.now(); try { const response = await run(); @@ -17,3 +36,27 @@ export async function observeUpstream(target: Upst throw error; } } + +async function traced(target: UpstreamTarget, run: (traceHeaders: TraceHeaders) => Promise): Promise { + const api = await import('@opentelemetry/api'); + const tracer = api.trace.getTracer('bulwark-webmail'); + return tracer.startActiveSpan(`upstream ${target}`, { + kind: api.SpanKind.CLIENT, + attributes: { 'bulwark.upstream.target': target }, + }, async (span) => { + const headers: TraceHeaders = {}; + api.propagation.inject(api.context.active(), headers); + try { + const response = await measured(target, () => run(headers)); + span.setAttribute('http.response.status_code', response.status); + if (response.status >= 500) span.setStatus({ code: api.SpanStatusCode.ERROR }); + return response; + } catch (error) { + // The error's name only: messages can name hosts or paths. + span.setStatus({ code: api.SpanStatusCode.ERROR, message: error instanceof Error ? error.name : 'Error' }); + throw error; + } finally { + span.end(); + } + }); +} diff --git a/lib/shared-store/server-state-allowlist.ts b/lib/shared-store/server-state-allowlist.ts index 2384bf964..562b25d51 100644 --- a/lib/shared-store/server-state-allowlist.ts +++ b/lib/shared-store/server-state-allowlist.ts @@ -35,6 +35,7 @@ export const SERVER_STATE_ALLOWLIST: AllowedState[] = [ { site: 'app/api/favicon/route.ts#cache', class: 'cache', why: 'sender icons from a public service, 14 days' }, { site: 'app/api/favicon/route.ts#negativeCache', class: 'cache', why: 'sender icons that failed, 1 day' }, { site: 'app/api/ready/route.ts#last', class: 'cache', why: 'the last store ping, 1 s, so probes can\'t flood the store' }, + { site: 'lib/tracing/config.ts#globalThis:ACTIVE', class: 'per-process', why: 'whether this process registered the OpenTelemetry SDK at startup' }, { site: 'lib/error-reporting/debug-ids.ts#cache', class: 'per-process', why: 'debug ids of the bundle files this process loaded, read from globalThis' }, { site: 'lib/admin/directory-keys.ts#keyringCache', class: 'cache', why: 'extension directory keys, 10 min; the anti-rollback record is in the store' }, { site: 'lib/admin/paths.ts#cachedReadOnly', class: 'per-process', why: 'parsed ADMIN_CONFIG_READONLY env' }, diff --git a/lib/stalwart/server-fetch.ts b/lib/stalwart/server-fetch.ts index 710c2493c..7e0296cf2 100644 --- a/lib/stalwart/server-fetch.ts +++ b/lib/stalwart/server-fetch.ts @@ -73,7 +73,8 @@ export async function fetchJmapServer( ): Promise { const authHeader = new Headers(givenInit.headers).get('authorization'); if (authHeader && (await isStoredCredentialRefused(authHeader))) return refusedCredentialResponse(); - const response = await observeUpstream('stalwart', () => fetchUnbudgeted(url, givenInit, trusted)); + // With tracing on, Stalwart gets the trace context and its spans join ours. + const response = await observeUpstream('stalwart', (trace) => fetchUnbudgeted(url, withTraceHeaders(givenInit, trace), trusted)); // Not awaited: this process's copy is updated at once, the shared write follows. if (authHeader) { void recordStoredCredentialStatus(authHeader, response.status).catch((error) => { @@ -83,6 +84,13 @@ export async function fetchJmapServer( return response; } +function withTraceHeaders(init: ServerFetchInit, trace: Record): ServerFetchInit { + if (Object.keys(trace).length === 0) return init; + const headers = new Headers(init.headers); + for (const [name, value] of Object.entries(trace)) headers.set(name, value); + return { ...init, headers }; +} + async function fetchUnbudgeted( url: string, givenInit: ServerFetchInit, diff --git a/lib/tracing/__tests__/config.test.ts b/lib/tracing/__tests__/config.test.ts new file mode 100644 index 000000000..49b6429fd --- /dev/null +++ b/lib/tracing/__tests__/config.test.ts @@ -0,0 +1,44 @@ +// @vitest-environment node +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { tracingConfigured, tracingEndpoint } from '../config'; +import { listReachableHosts } from '@/lib/admin/reachable-hosts'; + +const started = vi.hoisted(() => ({ count: 0 })); +vi.mock('@/lib/tracing/start', () => ({ startTracing: () => { started.count++; } })); +vi.mock('@/instrumentation.node', () => ({})); + +afterEach(() => { + vi.unstubAllEnvs(); + started.count = 0; +}); + +describe('tracing configuration', () => { + it('needs an OTLP endpoint from the standard variables', () => { + expect(tracingConfigured({})).toBe(false); + expect(tracingEndpoint({ OTEL_EXPORTER_OTLP_ENDPOINT: 'http://collector:4318' })).toBe('http://collector:4318'); + expect(tracingEndpoint({ OTEL_EXPORTER_OTLP_ENDPOINT: 'http://a:4318', OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: 'http://b:4318/v1/traces' })).toBe('http://b:4318/v1/traces'); + }); + + it('honours OTEL_SDK_DISABLED and OTEL_TRACES_EXPORTER=none', () => { + expect(tracingConfigured({ OTEL_EXPORTER_OTLP_ENDPOINT: 'http://c:4318', OTEL_SDK_DISABLED: 'true' })).toBe(false); + expect(tracingConfigured({ OTEL_EXPORTER_OTLP_ENDPOINT: 'http://c:4318', OTEL_TRACES_EXPORTER: 'none' })).toBe(false); + expect(tracingConfigured({ OTEL_EXPORTER_OTLP_ENDPOINT: 'http://c:4318', OTEL_TRACES_EXPORTER: 'otlp' })).toBe(true); + }); + + it('loads the SDK at startup only when configured', async () => { + vi.stubEnv('NEXT_RUNTIME', 'nodejs'); + const { register } = await import('@/instrumentation'); + await register(); + expect(started.count).toBe(0); + vi.stubEnv('OTEL_EXPORTER_OTLP_ENDPOINT', 'http://collector.example.org:4318'); + await register(); + expect(started.count).toBe(1); + }); + + it('lists the OTLP endpoint as a host the admin set, only when set', () => { + const base = { get: () => undefined, policy: { pushRelays: [], pushRelayUrl: '', defaultSidebarApps: [], features: { sidebarAppsEnabled: false, calendarEnabled: false } }, buildRelayUrl: '' }; + expect(listReachableHosts({ ...base, env: {} }).filter((h) => h.kind === 'Tracing (OTLP)')).toEqual([]); + expect(listReachableHosts({ ...base, env: { OTEL_EXPORTER_OTLP_ENDPOINT: 'https://otel.example.org:4318' } }).filter((h) => h.kind === 'Tracing (OTLP)')) + .toEqual([{ kind: 'Tracing (OTLP)', host: 'otel.example.org:4318', origin: 'https://otel.example.org:4318', setIn: 'OTEL_EXPORTER_OTLP_ENDPOINT', side: 'server' }]); + }); +}); diff --git a/lib/tracing/__tests__/tracing.test.ts b/lib/tracing/__tests__/tracing.test.ts new file mode 100644 index 000000000..5134404b9 --- /dev/null +++ b/lib/tracing/__tests__/tracing.test.ts @@ -0,0 +1,100 @@ +// @vitest-environment node +import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { InMemorySpanExporter } from '@opentelemetry/sdk-trace-node'; +import { trace } from '@opentelemetry/api'; +import { W3CTraceContextPropagator } from '@opentelemetry/core'; + +vi.mock('@/lib/admin/config-manager', () => ({ + configManager: { ensureLoaded: async () => {}, get: (_key: string, fallback: unknown) => fallback }, +})); + +import { createProvider } from '../start'; +import { markTracingActive } from '../config'; +import { scrubAttributes, scrubSpanName } from '../scrub'; +import { observeUpstream } from '@/lib/metrics/upstream'; +import { fetchJmapServer } from '@/lib/stalwart/server-fetch'; + +const exporter = new InMemorySpanExporter(); +const provider = createProvider(exporter, false); +const ADDRESS = 'ceo.private@acme-corp.example'; + +beforeAll(() => { + provider.register({ propagator: new W3CTraceContextPropagator() }); + markTracingActive(); +}); +afterAll(async () => { + await provider.shutdown(); + trace.disable(); +}); +beforeEach(() => exporter.reset()); + +describe('spans for upstream calls', () => { + it('are named after the target and carry no URL, host or user', async () => { + await observeUpstream('wopi', async () => new Response(null, { status: 200 })); + const [span] = exporter.getFinishedSpans(); + expect(span.name).toBe('upstream wopi'); + expect(span.attributes).toEqual({ 'bulwark.upstream.target': 'wopi', 'http.response.status_code': 200 }); + }); + + it('hand the trace context to the callee', async () => { + let seen: Record = {}; + await observeUpstream('stalwart', async (headers) => { seen = headers; return new Response(null, { status: 401 }); }); + const [span] = exporter.getFinishedSpans(); + expect(seen.traceparent).toMatch(new RegExp(`^00-${span.spanContext().traceId}-${span.spanContext().spanId}-01$`)); + }); + + it('reach Stalwart as a traceparent header', async () => { + const fetchSpy = vi.fn().mockResolvedValue(new Response('{}', { status: 200 })); + vi.stubGlobal('fetch', fetchSpy); + try { + await fetchJmapServer('https://mail.example.org/jmap/session', { headers: { Accept: 'application/json' } }, true); + const headers = new Headers((fetchSpy.mock.calls[0][1] as RequestInit).headers); + expect(headers.get('traceparent')).toMatch(/^00-[0-9a-f]{32}-[0-9a-f]{16}-01$/); + expect(headers.get('accept')).toBe('application/json'); + } finally { + vi.unstubAllGlobals(); + } + }); + + it('mark 5xx and thrown errors as errors, without the error message', async () => { + await expect(observeUpstream('translate', async () => { throw new Error(`refused for ${ADDRESS}`); })).rejects.toThrow(); + const [span] = exporter.getFinishedSpans(); + expect(span.status).toEqual({ code: 2, message: 'Error' }); + }); +}); + +describe('every exported span', () => { + it('is scrubbed before export', () => { + const tracer = trace.getTracer('test'); + const span = tracer.startSpan('fetch GET https://mail.example.org/jmap/download/a/b/term-sheet.pdf?accept=x'); + span.setAttributes({ + 'url.full': 'https://mail.example.org/jmap/download/a/b/term-sheet.pdf?accept=x', + 'http.target': '/en/mail/folder/Acquisition%20Talks/Mabc?q=merger', + 'url.query': 'q=merger', + 'http.user_agent': 'Mozilla/5.0', + 'client.address': '203.0.113.7', + 'enduser.id': ADDRESS, + 'next.route': '/[locale]/mail/[[...segments]]', + 'custom.note': `sent by ${ADDRESS} with Bearer abc.def`, + }); + span.end(); + const [exported] = exporter.getFinishedSpans(); + expect(exported.name).toBe('fetch GET https://mail.example.org'); + expect(exported.attributes).toEqual({ + 'url.full': 'https://mail.example.org', + 'http.target': '/mail/…', + 'next.route': '/[locale]/mail/[[...segments]]', + 'custom.note': 'sent by [email] with Bearer [token]', + }); + }); +}); + +describe('span names', () => { + it('keep route patterns and reduce raw paths', () => { + expect(scrubSpanName('GET /api/admin/plugins/[id]')).toBe('GET /api/admin/plugins/[id]'); + expect(scrubSpanName('GET /de/contacts/Mustermann')).toBe('GET /contacts/…'); + const attrs: Record = { 'url.path': '/api/wopi/files/abc' }; + scrubAttributes(attrs); + expect(attrs['url.path']).toBe('/api/wopi/…'); + }); +}); diff --git a/lib/tracing/config.ts b/lib/tracing/config.ts new file mode 100644 index 000000000..13347804e --- /dev/null +++ b/lib/tracing/config.ts @@ -0,0 +1,30 @@ +/** + * Whether OpenTelemetry tracing is configured, from the standard OTEL_* + * variables only: an OTLP endpoint (OTEL_EXPORTER_OTLP_TRACES_ENDPOINT or + * OTEL_EXPORTER_OTLP_ENDPOINT), unless OTEL_SDK_DISABLED=true or + * OTEL_TRACES_EXPORTER=none. Without it the SDK is never loaded. + * + * https://bulwarkmail.org/docs/admin/monitoring#tracing + */ +export function tracingEndpoint(env: Record = process.env): string | null { + if ((env.OTEL_SDK_DISABLED ?? '').trim().toLowerCase() === 'true') return null; + const exporters = (env.OTEL_TRACES_EXPORTER ?? '').toLowerCase().split(',').map((s) => s.trim()).filter(Boolean); + if (exporters.length > 0 && !exporters.includes('otlp')) return null; + const endpoint = (env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT || env.OTEL_EXPORTER_OTLP_ENDPOINT || '').trim(); + return endpoint || null; +} + +export function tracingConfigured(env: Record = process.env): boolean { + return tracingEndpoint(env) !== null; +} + +const ACTIVE = Symbol.for('bulwark.tracing.active'); + +/** Set by lib/tracing/start.ts once the SDK is registered. */ +export function markTracingActive(): void { + (globalThis as typeof globalThis & { [ACTIVE]?: boolean })[ACTIVE] = true; +} + +export function isTracingActive(): boolean { + return (globalThis as typeof globalThis & { [ACTIVE]?: boolean })[ACTIVE] === true; +} diff --git a/lib/tracing/scrub.ts b/lib/tracing/scrub.ts new file mode 100644 index 000000000..ae86018a3 --- /dev/null +++ b/lib/tracing/scrub.ts @@ -0,0 +1,60 @@ +import { scrubRoutePath } from '@/lib/route-groups'; +import { scrubText } from '@/lib/error-reporting/scrub'; + +/** + * What a span may carry before it leaves this server: no user names, mail + * addresses, tokens, query strings or paths that name folders, messages or + * files. Next's own spans and ours go through it. + * + * - absolute URLs keep their origin only (a Stalwart download URL ends in + * the attachment's file name) + * - request paths become their route group (`/mail/…`) + * - query, fragment, user agent and client address attributes are dropped + * - any other text with an address or a credential is scrubbed + * + * https://bulwarkmail.org/docs/admin/monitoring#tracing + */ + +type AttributeValue = string | number | boolean | Array | undefined; +export type Attributes = Record; + +const DROP = /^(?:url\.query|url\.fragment|http\.user_agent|user_agent\.|enduser\.|client\.address|client\.port|http\.client_ip|net\.peer\.ip|net\.sock\.peer\.addr|network\.peer\.address|http\.request\.header\.|http\.response\.header\.|db\.statement)/; +const PATH_KEYS = new Set(['http.target', 'url.path', 'next.page.path']); +const URL_IN_TEXT = /\b(https?):\/\/([^/\s?#"'<>]+)[^\s"'<>]*/gi; +const METHOD_AND_PATH = /\b(GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS)\s+(\/[^\s]*)/g; + +/** A path from the request: keep route patterns (`/api/admin/[id]`), reduce anything else to its group. */ +function safePath(path: string): string { + const bare = path.split(/[?#]/, 1)[0]; + return bare.includes('[') && /^[\w/[\].()-]+$/.test(bare) ? bare : scrubRoutePath(bare); +} + +/** A span name: URLs reduced to their origin, request paths to their group. */ +export function scrubSpanName(name: string): string { + return name + .replace(URL_IN_TEXT, (_match, scheme: string, host: string) => `${scheme.toLowerCase()}://${host}`) + .replace(METHOD_AND_PATH, (_match, method: string, path: string) => `${method} ${safePath(path)}`); +} + +function scrubString(key: string, value: string): string { + if (PATH_KEYS.has(key)) return safePath(value); + if (/@|bearer |basic |eyJ|:\/\//i.test(value)) { + return value.replace(URL_IN_TEXT, (_m, scheme: string, host: string) => `${scheme.toLowerCase()}://${host}`) + .split(/\s+/).map((word) => (/@|eyJ/.test(word) ? scrubText(word) : word)).join(' ') + .replace(/\b(Bearer|Basic)\s+\S+/gi, '$1 [token]'); + } + return value; +} + +/** Scrub span attributes in place (the processor runs it before export). */ +export function scrubAttributes(attributes: Attributes): void { + for (const key of Object.keys(attributes)) { + if (DROP.test(key)) { + delete attributes[key]; + continue; + } + const value = attributes[key]; + if (typeof value === 'string') attributes[key] = scrubString(key, value); + else if (Array.isArray(value)) attributes[key] = value.map((v) => (typeof v === 'string' ? scrubString(key, v) : v)); + } +} diff --git a/lib/tracing/start.ts b/lib/tracing/start.ts new file mode 100644 index 000000000..037430377 --- /dev/null +++ b/lib/tracing/start.ts @@ -0,0 +1,68 @@ +import { W3CTraceContextPropagator } from '@opentelemetry/core'; +import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'; +import { detectResources, envDetector, resourceFromAttributes } from '@opentelemetry/resources'; +import { + BatchSpanProcessor, + NodeTracerProvider, + SimpleSpanProcessor, + type ReadableSpan, + type Span, + type SpanExporter, + type SpanProcessor, +} from '@opentelemetry/sdk-trace-node'; +import type { Context } from '@opentelemetry/api'; +import { markTracingActive } from './config'; +import { scrubAttributes, scrubSpanName, type Attributes } from './scrub'; + +/** + * OpenTelemetry tracing, loaded only by a dynamic import from + * instrumentation.ts when an OTLP endpoint is configured (./config.ts). + * Everything else comes from the standard OTEL_* variables, which the OTLP + * exporter and the env resource detector read themselves. + * + * Next's own spans (requests, rendering, route handlers) and Bulwark's spans + * for the calls it makes (lib/metrics/upstream.ts) go through the same + * provider; a processor scrubs every span before export (./scrub.ts). + * + * https://bulwarkmail.org/docs/admin/monitoring#tracing + */ + +/** Scrubs a span's name and attributes, then hands it on. */ +export class ScrubbingSpanProcessor implements SpanProcessor { + constructor(private readonly next: SpanProcessor) {} + + onStart(span: Span, parentContext: Context): void { + this.next.onStart(span, parentContext); + } + + onEnd(span: ReadableSpan): void { + scrubAttributes(span.attributes as Attributes); + const named = span as unknown as { name: string }; + named.name = scrubSpanName(span.name); + for (const event of span.events) if (event.attributes) scrubAttributes(event.attributes as Attributes); + this.next.onEnd(span); + } + + forceFlush(): Promise { + return this.next.forceFlush(); + } + + shutdown(): Promise { + return this.next.shutdown(); + } +} + +export function createProvider(exporter: SpanExporter, batch = true): NodeTracerProvider { + const resource = resourceFromAttributes({ + 'service.name': process.env.OTEL_SERVICE_NAME?.trim() || 'bulwark-webmail', + 'service.version': process.env.NEXT_PUBLIC_APP_VERSION || '0.0.0', + }).merge(detectResources({ detectors: [envDetector] })); + const inner: SpanProcessor = batch ? new BatchSpanProcessor(exporter) : new SimpleSpanProcessor(exporter); + return new NodeTracerProvider({ resource, spanProcessors: [new ScrubbingSpanProcessor(inner)] }); +} + +export function startTracing(): void { + const provider = createProvider(new OTLPTraceExporter()); + provider.register({ propagator: new W3CTraceContextPropagator() }); + markTracingActive(); +} diff --git a/package-lock.json b/package-lock.json index 4f178ae26..f3f276a37 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,9 +18,7 @@ "@opentelemetry/core": "2.8.0", "@opentelemetry/exporter-trace-otlp-http": "0.219.0", "@opentelemetry/resources": "2.8.0", - "@opentelemetry/sdk-trace-base": "2.8.0", "@opentelemetry/sdk-trace-node": "2.8.0", - "@opentelemetry/semantic-conventions": "1.44.0", "@prometheus-io/client": "0.16.1", "@tabler/icons-react": "^3.50.0", "@tanstack/react-virtual": "^3.14.23", diff --git a/package.json b/package.json index eead11624..474421da5 100644 --- a/package.json +++ b/package.json @@ -48,9 +48,7 @@ "@opentelemetry/core": "2.8.0", "@opentelemetry/exporter-trace-otlp-http": "0.219.0", "@opentelemetry/resources": "2.8.0", - "@opentelemetry/sdk-trace-base": "2.8.0", "@opentelemetry/sdk-trace-node": "2.8.0", - "@opentelemetry/semantic-conventions": "1.44.0", "@prometheus-io/client": "0.16.1", "@tabler/icons-react": "^3.50.0", "@tanstack/react-virtual": "^3.14.23", From be5a5e88ec9fb073e7a72ab434a7e747cecb4872 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 03:31:48 +0100 Subject: [PATCH 11/26] ci: upload browser source maps to Bugsink and keep every map out of what ships Every image, the standalone tarballs and the Lite zips are built with source maps (`BULWARK_SOURCE_MAPS=1`). Turbopack builds give each chunk a debug ID themselves; the webpack build in the Docker image gets them from `sentry-cli sourcemaps inject`. `scripts/sourcemaps.mjs collect` then moves every map out of the build before anything is packed, and `scripts/sourcemaps.mjs check` fails the build when a `.map` file, an inline map or a `sourceMappingURL` to a missing file would ship. - Docker images: a `sourcemaps` stage holds only the maps; the amd64 job exports it (a cache hit) and uploads it, since both platforms build identical chunks - standalone tarballs, the Lite zips and the Lite image upload their own maps and check what they ship - `scripts/upload-sourcemaps.sh` uploads to `bulwark-instances` and `bulwark-telemetry` (Bugsink keeps maps per project) and skips with a notice when the `BUGSINK_AUTH_TOKEN` secret is missing --- .dockerignore | 2 + .github/workflows/docker-publish-lite.yml | 22 ++ .github/workflows/docker-publish-releases.yml | 49 +++++ .github/workflows/docker-publish.yml | 27 +++ .github/workflows/standalone-release.yml | 18 ++ .github/workflows/static-lite.yml | 13 ++ .gitignore | 2 + Dockerfile | 19 ++ Dockerfile.lite | 14 +- lib/__tests__/sourcemaps-script.test.ts | 96 +++++++++ next.config.ts | 8 + scripts/lite/build.mjs | 23 +- scripts/sourcemaps.mjs | 203 ++++++++++++++++++ scripts/upload-sourcemaps.sh | 30 +++ 14 files changed, 522 insertions(+), 4 deletions(-) create mode 100644 lib/__tests__/sourcemaps-script.test.ts create mode 100644 scripts/sourcemaps.mjs create mode 100644 scripts/upload-sourcemaps.sh diff --git a/.dockerignore b/.dockerignore index 0942bf880..5d576f432 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,7 @@ node_modules .next out +sourcemaps lite-image .git .gitignore @@ -11,6 +12,7 @@ scripts/ !scripts/lite/ !scripts/build-plugin-sandbox.mjs !scripts/healthcheck.mjs +!scripts/sourcemaps.mjs # Built in the image by scripts/build-plugin-sandbox.mjs public/plugin-sandbox-assets/ TODO.md diff --git a/.github/workflows/docker-publish-lite.yml b/.github/workflows/docker-publish-lite.yml index 723bd5753..3912bb270 100644 --- a/.github/workflows/docker-publish-lite.yml +++ b/.github/workflows/docker-publish-lite.yml @@ -178,9 +178,31 @@ jobs: labels: ${{ steps.meta.outputs.labels }} build-args: | GIT_COMMIT=${{ github.sha }} + BULWARK_SOURCE_MAPS=1 cache-from: type=gha,scope=lite cache-to: type=gha,mode=max,scope=lite + # The same build again, a cache hit, for the stage that holds only the + # source maps; the image never has them. One export for both platforms. + # https://bulwarkmail.org/docs/admin/monitoring + - name: Export source maps + uses: docker/build-push-action@v7 + with: + context: . + file: Dockerfile.lite + platforms: linux/amd64 + target: sourcemaps + build-args: | + GIT_COMMIT=${{ github.sha }} + BULWARK_SOURCE_MAPS=1 + outputs: type=local,dest=sourcemaps + cache-from: type=gha,scope=lite + + - name: Upload source maps to Bugsink + env: + BUGSINK_AUTH_TOKEN: ${{ secrets.BUGSINK_AUTH_TOKEN }} + run: bash scripts/upload-sourcemaps.sh sourcemaps + - name: Inspect image run: docker buildx imagetools inspect ${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }} diff --git a/.github/workflows/docker-publish-releases.yml b/.github/workflows/docker-publish-releases.yml index 62d7d56c8..e906e0c57 100644 --- a/.github/workflows/docker-publish-releases.yml +++ b/.github/workflows/docker-publish-releases.yml @@ -59,10 +59,34 @@ jobs: labels: ${{ steps.meta.outputs.labels }} build-args: | GIT_COMMIT=${{ github.sha }} + BULWARK_SOURCE_MAPS=1 outputs: type=image,name=${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true cache-from: type=gha,scope=${{ matrix.platform }} cache-to: type=gha,mode=max,scope=${{ matrix.platform }} + # The same build again, a cache hit on this builder, for the stage that + # holds only the source maps (Dockerfile, "sourcemaps"); the image never + # has them. Both platforms build identical chunks, so amd64 uploads. + # https://bulwarkmail.org/docs/admin/monitoring + - name: Export source maps + if: matrix.platform == 'linux/amd64' + uses: docker/build-push-action@v7 + with: + context: . + platforms: ${{ matrix.platform }} + target: sourcemaps + build-args: | + GIT_COMMIT=${{ github.sha }} + BULWARK_SOURCE_MAPS=1 + outputs: type=local,dest=sourcemaps + cache-from: type=gha,scope=${{ matrix.platform }} + + - name: Upload source maps to Bugsink + if: matrix.platform == 'linux/amd64' + env: + BUGSINK_AUTH_TOKEN: ${{ secrets.BUGSINK_AUTH_TOKEN }} + run: bash scripts/upload-sourcemaps.sh sourcemaps + - name: Export digest run: | mkdir -p /tmp/digests @@ -184,10 +208,35 @@ jobs: build-args: | GIT_COMMIT=${{ github.sha }} NEXT_PUBLIC_LOCALE_PREFIX=always + BULWARK_SOURCE_MAPS=1 outputs: type=image,name=${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true cache-from: type=gha,scope=always-${{ matrix.platform }} cache-to: type=gha,mode=max,scope=always-${{ matrix.platform }} + # The same build again, a cache hit on this builder, for the stage that + # holds only the source maps (Dockerfile, "sourcemaps"); the image never + # has them. Both platforms build identical chunks, so amd64 uploads. + # https://bulwarkmail.org/docs/admin/monitoring + - name: Export source maps + if: matrix.platform == 'linux/amd64' + uses: docker/build-push-action@v7 + with: + context: . + platforms: ${{ matrix.platform }} + target: sourcemaps + build-args: | + GIT_COMMIT=${{ github.sha }} + NEXT_PUBLIC_LOCALE_PREFIX=always + BULWARK_SOURCE_MAPS=1 + outputs: type=local,dest=sourcemaps + cache-from: type=gha,scope=always-${{ matrix.platform }} + + - name: Upload source maps to Bugsink + if: matrix.platform == 'linux/amd64' + env: + BUGSINK_AUTH_TOKEN: ${{ secrets.BUGSINK_AUTH_TOKEN }} + run: bash scripts/upload-sourcemaps.sh sourcemaps + - name: Export digest run: | mkdir -p /tmp/digests-always diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 96224784a..d10f6c423 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -18,6 +18,9 @@ on: - "public/**" - "package.json" - "package-lock.json" + - "scripts/healthcheck.mjs" + - "scripts/sourcemaps.mjs" + - "scripts/upload-sourcemaps.sh" - ".github/workflows/docker-publish.yml" workflow_dispatch: @@ -89,10 +92,34 @@ jobs: labels: ${{ steps.meta.outputs.labels }} build-args: | GIT_COMMIT=${{ github.sha }} + BULWARK_SOURCE_MAPS=1 outputs: type=image,name=${{ needs.prepare.outputs.image_name }},push-by-digest=true,name-canonical=true,push=true cache-from: type=gha,scope=${{ matrix.platform }} cache-to: type=gha,mode=max,scope=${{ matrix.platform }} + # The same build again, a cache hit on this builder, for the stage that + # holds only the source maps (Dockerfile, "sourcemaps"); the image never + # has them. Both platforms build identical chunks, so amd64 uploads. + # https://bulwarkmail.org/docs/admin/monitoring + - name: Export source maps + if: matrix.platform == 'linux/amd64' + uses: docker/build-push-action@v7 + with: + context: . + platforms: ${{ matrix.platform }} + target: sourcemaps + build-args: | + GIT_COMMIT=${{ github.sha }} + BULWARK_SOURCE_MAPS=1 + outputs: type=local,dest=sourcemaps + cache-from: type=gha,scope=${{ matrix.platform }} + + - name: Upload source maps to Bugsink + if: matrix.platform == 'linux/amd64' + env: + BUGSINK_AUTH_TOKEN: ${{ secrets.BUGSINK_AUTH_TOKEN }} + run: bash scripts/upload-sourcemaps.sh sourcemaps + - name: Export digest run: | mkdir -p /tmp/digests diff --git a/.github/workflows/standalone-release.yml b/.github/workflows/standalone-release.yml index 50ef82024..f50ec11b2 100644 --- a/.github/workflows/standalone-release.yml +++ b/.github/workflows/standalone-release.yml @@ -40,9 +40,24 @@ jobs: - name: Install dependencies run: npm ci + # With browser and server source maps; Turbopack gives every chunk a + # debug ID. The maps go to Bugsink and never into the tarball. + # https://bulwarkmail.org/docs/admin/monitoring - name: Build standalone + env: + BULWARK_SOURCE_MAPS: "1" run: npm run build + - name: Move source maps out of the build + run: node scripts/sourcemaps.mjs collect --out sourcemaps .next/static .next/standalone + + # Turbopack chunks were not checked for being the same on both + # platforms, so each one uploads its own. + - name: Upload source maps to Bugsink + env: + BUGSINK_AUTH_TOKEN: ${{ secrets.BUGSINK_AUTH_TOKEN }} + run: bash scripts/upload-sourcemaps.sh sourcemaps + - name: Package tarball env: REF_NAME: ${{ github.ref_name }} @@ -59,6 +74,9 @@ jobs: tar -czf "$TARBALL" bulwark-standalone/ echo "TARBALL=$TARBALL" >> "$GITHUB_ENV" + - name: Check the tarball holds no source maps + run: node scripts/sourcemaps.mjs check "$TARBALL" + - name: Upload artifact uses: actions/upload-artifact@v7 with: diff --git a/.github/workflows/static-lite.yml b/.github/workflows/static-lite.yml index d4528ec24..1c23a1e4a 100644 --- a/.github/workflows/static-lite.yml +++ b/.github/workflows/static-lite.yml @@ -79,8 +79,18 @@ jobs: LITE_APP_NAME: ${{ inputs.app_name }} LITE_LOCALES: ${{ inputs.locales }} LITE_DEMO_MODE: ${{ inputs.demo }} + # Browser source maps for Bugsink; build:lite moves them out of out/ + # before anything is packed. https://bulwarkmail.org/docs/admin/monitoring + BULWARK_SOURCE_MAPS: "1" + SOURCEMAPS_OUT: sourcemaps run: npm run build:lite + # Each target is its own build with its own chunks, so each uploads. + - name: Upload source maps to Bugsink + env: + BUGSINK_AUTH_TOKEN: ${{ secrets.BUGSINK_AUTH_TOKEN }} + run: bash scripts/upload-sourcemaps.sh sourcemaps + # Every host literal in the bundle must be a placeholder, the configured # JMAP server, a built-in host the no-third-party switch turns off, or a # string listed as never requested (#903). @@ -107,6 +117,9 @@ jobs: echo "ZIP=${ZIP}" >> "$GITHUB_ENV" ls -la "${ZIP}" + - name: Check the zip holds no source maps + run: node scripts/sourcemaps.mjs check "$ZIP" + - name: Upload artifact uses: actions/upload-artifact@v7 with: diff --git a/.gitignore b/.gitignore index a67d67b1c..c90b34d31 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,8 @@ # next.js /.next/ /out/ +# Source maps CI collects for the error tracker (scripts/sourcemaps.mjs) +/sourcemaps/ /lite-image/ # scripts/build-plugin-sandbox.mjs /public/plugin-sandbox-assets/ diff --git a/Dockerfile b/Dockerfile index fe9f6f986..d6ed0a17d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,10 @@ ENV NEXT_PUBLIC_DEFAULT_LOCALE=$NEXT_PUBLIC_DEFAULT_LOCALE # `git rev-parse` inside the build can't find it - CI must pass it in. ARG GIT_COMMIT=unknown ENV GIT_COMMIT=$GIT_COMMIT +# Browser source maps for the error tracker: CI passes 1, a plain build makes +# none (next.config.ts reads it). They never reach the image, see below. +ARG BULWARK_SOURCE_MAPS=0 +ENV BULWARK_SOURCE_MAPS=$BULWARK_SOURCE_MAPS # The mock JMAP server (DEV_MOCK_JMAP) accepts any password and is for # `npm run dev` only; images ship without it. # `next build` type-checks every .ts file, and some unit tests import the @@ -33,6 +37,21 @@ RUN rm -rf "app/api/dev-jmap" && \ node scripts/build-plugin-sandbox.mjs && \ npx next build --webpack +# webpack has no debug IDs of its own, so sentry-cli adds them to the +# chunks; then every map moves to /sourcemaps (exported with +# `--target sourcemaps`) and the check fails the build if one would ship. +# https://bulwarkmail.org/docs/admin/monitoring +ARG SENTRY_CLI_VERSION=3.8.2 +RUN if [ "$BULWARK_SOURCE_MAPS" = "1" ]; then \ + npx --yes "@sentry/cli@${SENTRY_CLI_VERSION}" sourcemaps inject .next/static; \ + fi && \ + node scripts/sourcemaps.mjs collect --out /sourcemaps .next/static .next/standalone && \ + node scripts/sourcemaps.mjs check .next/static .next/standalone public + +# Only the maps, for CI to upload: never part of the image. +FROM scratch AS sourcemaps +COPY --from=builder /sourcemaps / + FROM node:24-alpine AS runner LABEL org.opencontainers.image.title="Bulwark Webmail" diff --git a/Dockerfile.lite b/Dockerfile.lite index 8a71f6650..94ab47a9d 100644 --- a/Dockerfile.lite +++ b/Dockerfile.lite @@ -29,8 +29,18 @@ ARG NEXT_PUBLIC_DEFAULT_LOCALE= # Commit SHA shown in the About screen. .dockerignore excludes .git, so CI # must pass it in. ARG GIT_COMMIT=unknown -RUN LITE_TARGET=static NEXT_PUBLIC_BASE_PATH= npm run build:lite && \ - node scripts/lite/container.mjs /image +# Browser source maps for the error tracker: CI passes 1. build:lite moves +# them to /sourcemaps (exported with `--target sourcemaps`) and fails if one +# would ship. https://bulwarkmail.org/docs/admin/monitoring +ARG BULWARK_SOURCE_MAPS=0 +RUN mkdir -p /sourcemaps && \ + LITE_TARGET=static NEXT_PUBLIC_BASE_PATH= SOURCEMAPS_OUT=/sourcemaps npm run build:lite && \ + node scripts/lite/container.mjs /image && \ + node scripts/sourcemaps.mjs check /image/html + +# Only the maps, for CI to upload: never part of the image. +FROM scratch AS sourcemaps +COPY --from=builder /sourcemaps / FROM nginxinc/nginx-unprivileged:1.30-alpine-slim diff --git a/lib/__tests__/sourcemaps-script.test.ts b/lib/__tests__/sourcemaps-script.test.ts new file mode 100644 index 000000000..e9b8ac385 --- /dev/null +++ b/lib/__tests__/sourcemaps-script.test.ts @@ -0,0 +1,96 @@ +// @vitest-environment node +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { gzipSync } from 'node:zlib'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +// @ts-expect-error plain .mjs without types +import { check, collect, mapReferences } from '../../scripts/sourcemaps.mjs'; +// @ts-expect-error plain .mjs without types +import { buildZip } from '../../scripts/lite/lib.mjs'; + +// CI builds source maps for the error tracker; none may ship (images, the +// standalone tarball, Lite zips). + +let dir: string; +const put = (rel: string, text: string) => { + const file = path.join(dir, rel); + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync(file, text); +}; + +beforeEach(() => { dir = mkdtempSync(path.join(tmpdir(), 'bulwark-maps-')); }); +afterEach(() => { rmSync(dir, { recursive: true, force: true }); }); + +function tar(files: Record): Buffer { + const blocks: Buffer[] = []; + for (const [name, text] of Object.entries(files)) { + const data = Buffer.from(text); + const header = Buffer.alloc(512); + header.write(name, 0); + header.write('0000644\0', 100); + header.write(data.length.toString(8).padStart(11, '0') + '\0', 124); + header.write('0', 156); + header.write('ustar\0', 257); + blocks.push(header, data, Buffer.alloc((512 - (data.length % 512)) % 512)); + } + blocks.push(Buffer.alloc(1024)); + return Buffer.concat(blocks); +} + +describe('collect', () => { + it('moves maps out with the chunks that name them, and strips the comment from what ships', () => { + put('static/chunks/a.js', 'console.log(1);\n//# debugId=bc67f7db-7f3b-7be2-6f94-966cd7825cc2\n//# sourceMappingURL=a1b2.js.map\n'); + put('static/chunks/a1b2.js.map', '{"version":3}'); + put('static/css/x.css', 'a{}\n/*# sourceMappingURL=x.css.map */\n'); + put('static/css/x.css.map', '{"version":3}'); + put('static/chunks/orphan.js.map', '{"version":3}'); + const out = path.join(dir, 'maps'); + + const moved = collect([path.join(dir, 'static')], out); + + expect(moved).toHaveLength(3); + expect(existsSync(path.join(dir, 'static/chunks/a1b2.js.map'))).toBe(false); + const shipped = readFileSync(path.join(dir, 'static/chunks/a.js'), 'utf8'); + expect(mapReferences(shipped)).toEqual([]); + expect(shipped).toContain('//# debugId=bc67f7db-7f3b-7be2-6f94-966cd7825cc2'); + expect(readFileSync(path.join(out, '0/chunks/a.js'), 'utf8')).toContain('sourceMappingURL=a1b2.js.map'); + expect(existsSync(path.join(out, '0/chunks/a1b2.js.map'))).toBe(true); + expect(existsSync(path.join(out, '0/chunks/orphan.js.map'))).toBe(true); + expect(check(path.join(dir, 'static'))).toEqual([]); + }); +}); + +describe('check', () => { + it('finds map files, dangling references and inline maps in a folder', () => { + put('a/x.js.map', '{}'); + put('a/y.js', 'x\n//# sourceMappingURL=gone.js.map\n'); + put('a/z.js', 'x\n//# sourceMappingURL=data:application/json;base64,e30=\n'); + put('a/ok.js', 'const s = "//# sourceMappingURL=in-a-string.map";\n'); + expect(check(path.join(dir, 'a')).sort()).toEqual([ + 'x.js.map: source map file', + 'y.js: sourceMappingURL to missing gone.js.map', + 'z.js: inline source map', + ]); + }); + + it('reads zip archives', () => { + const zip = path.join(dir, 'lite.zip'); + writeFileSync(zip, buildZip([ + { name: '_next/static/chunks/a.js', data: Buffer.from('x\n//# sourceMappingURL=a.js.map\n') }, + { name: '_next/static/chunks/a.js.map', data: Buffer.from('{}') }, + ])); + expect(check(zip)).toEqual(['_next/static/chunks/a.js.map: source map file']); + writeFileSync(zip, buildZip([{ name: 'index.html', data: Buffer.from('') }])); + expect(check(zip)).toEqual([]); + }); + + it('reads gzipped tarballs', () => { + const tgz = path.join(dir, 'standalone.tar.gz'); + writeFileSync(tgz, gzipSync(tar({ + 'bulwark-standalone/.next/server/chunks/1.js': 'x\n//# sourceMappingURL=1.js.map\n', + 'bulwark-standalone/.next/server/chunks/1.js.map': '{}', + }))); + expect(check(tgz)).toEqual(['bulwark-standalone/.next/server/chunks/1.js.map: source map file']); + }); +}); diff --git a/next.config.ts b/next.config.ts index 2435bd968..b238b59c6 100644 --- a/next.config.ts +++ b/next.config.ts @@ -47,8 +47,15 @@ if (isLite && (process.env.NEXT_PUBLIC_LOCALE_PREFIX ?? "never") !== "always") { throw new Error("Bulwark Lite requires NEXT_PUBLIC_LOCALE_PREFIX=always (use `npm run build:lite`)"); } +// Browser source maps, for resolving the stack traces in error reports. +// Only CI turns them on (BULWARK_SOURCE_MAPS=1): it uploads them to the +// error tracker and removes every .map file before anything ships +// (scripts/sourcemaps.mjs). A plain build makes none. +const sourceMaps = process.env.BULWARK_SOURCE_MAPS === "1"; + const nextConfig: NextConfig = { output: isLite ? "export" : "standalone", + productionBrowserSourceMaps: sourceMaps, // No `X-Powered-By: Next.js`; headers() below sends // `X-Powered-By: bulwarkmail.org` instead, without a version. poweredByHeader: false, @@ -132,6 +139,7 @@ const nextConfig: NextConfig = { }, turbopack: { root: process.env.NEXT_TURBOPACK_ROOT || import.meta.dirname, + ...(sourceMaps ? { debugIds: true } : {}), }, env: { NEXT_PUBLIC_GIT_COMMIT: gitCommitHash, diff --git a/scripts/lite/build.mjs b/scripts/lite/build.mjs index cfa36568e..08430e59d 100644 --- a/scripts/lite/build.mjs +++ b/scripts/lite/build.mjs @@ -8,6 +8,11 @@ // (Stalwart target: the entry document, then the zip) // 4. verify fail on missing files or undocumented /api references // +// With BULWARK_SOURCE_MAPS=1 (CI) the browser source maps move out of out/ +// right after the build, before anything is packed, into SOURCEMAPS_OUT +// (default sourcemaps/lite-) for the error tracker. Either way the +// export and the Stalwart zip are checked for maps last. +// // Inputs (env): LITE_TARGET (static | stalwart), LITE_JMAP_SERVER_URL, // LITE_APP_NAME, LITE_ALLOW_CUSTOM_ENDPOINT, LITE_REMEMBER_ME, LITE_DEMO_MODE, // LITE_NO_THIRD_PARTY_TRAFFIC, @@ -16,12 +21,13 @@ // --skip-prepare (tree already pruned), --dry-run (print the plan only). import { spawnSync } from "node:child_process"; import { createRequire } from "node:module"; -import { readFileSync } from "node:fs"; +import { existsSync, readFileSync } from "node:fs"; import { join } from "node:path"; -import { makeBuildId, normalizeBasePath, parseLiteTarget, resolveRepoRoot } from "./lib.mjs"; +import { makeBuildId, normalizeBasePath, parseLiteTarget, resolveRepoRoot, STALWART_ZIP_NAME } from "./lib.mjs"; import { runPrepare } from "./prepare.mjs"; import { runPostbuild } from "./postbuild.mjs"; import { verifyExport } from "./verify.mjs"; +import { check as checkSourceMaps, collect as collectSourceMaps } from "../sourcemaps.mjs"; const repoRoot = resolveRepoRoot(import.meta.url); const argv = process.argv.slice(2); @@ -81,6 +87,12 @@ if (build.status !== 0) { process.exit(build.status ?? 1); } +if (env.BULWARK_SOURCE_MAPS === "1") { + const out = env.SOURCEMAPS_OUT?.trim() || join(repoRoot, "sourcemaps", `lite-${target}`); + const moved = collectSourceMaps([join(repoRoot, "out", "_next", "static")], out); + console.log(`[lite] moved ${moved.length} source maps to ${out}`); +} + runPostbuild({ env }); const { problems, locales, apiStrings, zipBytes } = verifyExport({ target }); @@ -89,4 +101,11 @@ if (problems.length > 0) { for (const p of problems) console.error(` - ${p}`); process.exit(1); } +const shipped = [join(repoRoot, "out"), join(repoRoot, STALWART_ZIP_NAME)].filter((p) => existsSync(p)); +const mapProblems = shipped.flatMap((p) => checkSourceMaps(p).map((problem) => `${p}: ${problem}`)); +if (mapProblems.length > 0) { + console.error(`[lite] source maps would ship (${mapProblems.length}):`); + for (const p of mapProblems.slice(0, 20)) console.error(` - ${p}`); + process.exit(1); +} console.log(`[lite] done: out/ holds ${locales.length} locales; ${apiStrings.length} documented /api strings in chunks${zipBytes ? `; zip ${(zipBytes / 1024 / 1024).toFixed(1)} MB` : ""}`); diff --git a/scripts/sourcemaps.mjs b/scripts/sourcemaps.mjs new file mode 100644 index 000000000..4303506d3 --- /dev/null +++ b/scripts/sourcemaps.mjs @@ -0,0 +1,203 @@ +#!/usr/bin/env node +// Source maps: built in CI (BULWARK_SOURCE_MAPS=1), uploaded to the error +// tracker, never shipped. https://bulwarkmail.org/docs/admin/monitoring +// +// node scripts/sourcemaps.mjs collect --out ... +// Moves every .map under each root into //... with a copy of the +// .js/.css file that names it (sentry-cli uploads the pairs), and removes +// the `sourceMappingURL` comment from the shipped file. The chunk keeps +// its `debugId` comment and registration, which is all the tracker needs. +// +// node scripts/sourcemaps.mjs check ... +// Fails when anything that would ship holds a .map file, an inline +// (data:) source map, or a `sourceMappingURL` to a file that isn't there. +import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { basename, dirname, join, relative, resolve, sep } from "node:path"; +import { gunzipSync, inflateRawSync } from "node:zlib"; +import { fileURLToPath } from "node:url"; + +// At the start of a line, as bundlers write it; the same text inside a +// string literal is not a reference. +const COMMENT = /^[ \t]*\/[/*][#@][ \t]*sourceMappingURL=([^\s*'"`]+)[ \t]*(?:\*\/)?[ \t]*$/gm; +const SCRIPT = /\.(?:js|mjs|cjs|css)$/; + +function walk(dir, out = []) { + for (const name of readdirSync(dir)) { + const path = join(dir, name); + const stat = statSync(path, { throwIfNoEntry: false }); + if (!stat) continue; + if (stat.isDirectory()) walk(path, out); + else out.push(path); + } + return out; +} + +/** The sourceMappingURL targets named in a file's text. */ +export function mapReferences(text) { + return [...text.matchAll(COMMENT)].map((m) => m[1]); +} + +/** Remove the sourceMappingURL comments; everything else stays as it is. */ +export function stripReferences(text) { + return text.replace(COMMENT, "").replace(/\n{2,}$/, "\n"); +} + +export function collect(roots, outDir) { + const moved = []; + mkdirSync(outDir, { recursive: true }); + roots.forEach((root, index) => { + if (!existsSync(root)) return; + const files = walk(root); + const maps = new Set(files.filter((f) => f.endsWith(".map"))); + const target = join(outDir, String(index)); + for (const file of files.filter((f) => SCRIPT.test(f))) { + const text = readFileSync(file, "utf8"); + const refs = mapReferences(text); + if (refs.length === 0) continue; + for (const ref of refs) { + if (ref.startsWith("data:")) continue; + const map = resolve(dirname(file), decodeURIComponent(ref)); + if (!maps.has(map)) continue; + for (const source of [file, map]) { + const dest = join(target, relative(root, source)); + mkdirSync(dirname(dest), { recursive: true }); + copyFileSync(source, dest); + } + } + writeFileSync(file, stripReferences(text)); + } + // Every map goes, named by a file or not. + for (const map of maps) { + const dest = join(target, relative(root, map)); + if (!existsSync(dest)) { + mkdirSync(dirname(dest), { recursive: true }); + copyFileSync(map, dest); + } + rmSync(map); + moved.push(map); + } + }); + return moved; +} + +// --- archives ------------------------------------------------------------- + +function zipEntries(buffer) { + const entries = []; + let eocd = buffer.length - 22; + while (eocd >= 0 && buffer.readUInt32LE(eocd) !== 0x06054b50) eocd--; + if (eocd < 0) throw new Error("not a zip file"); + const count = buffer.readUInt16LE(eocd + 10); + let at = buffer.readUInt32LE(eocd + 16); + for (let i = 0; i < count; i++) { + const method = buffer.readUInt16LE(at + 10); + const size = buffer.readUInt32LE(at + 20); + const nameLength = buffer.readUInt16LE(at + 28); + const extraLength = buffer.readUInt16LE(at + 30); + const commentLength = buffer.readUInt16LE(at + 32); + const local = buffer.readUInt32LE(at + 42); + const name = buffer.subarray(at + 46, at + 46 + nameLength).toString("utf8"); + const dataStart = local + 30 + buffer.readUInt16LE(local + 26) + buffer.readUInt16LE(local + 28); + const raw = buffer.subarray(dataStart, dataStart + size); + entries.push({ name, read: () => (method === 0 ? raw : inflateRawSync(raw)) }); + at += 46 + nameLength + extraLength + commentLength; + } + return entries; +} + +function tarEntries(buffer) { + const entries = []; + let at = 0; + let longName = null; + while (at + 512 <= buffer.length) { + const header = buffer.subarray(at, at + 512); + if (header.every((b) => b === 0)) break; + const field = (start, length) => header.subarray(start, start + length).toString("utf8").replace(/\0.*$/s, ""); + const size = parseInt(field(124, 12).trim() || "0", 8); + const type = field(156, 1); + const prefix = field(345, 155); + let name = longName ?? (prefix ? `${prefix}/${field(0, 100)}` : field(0, 100)); + longName = null; + const data = buffer.subarray(at + 512, at + 512 + size); + if (type === "L") longName = data.toString("utf8").replace(/\0.*$/s, ""); + else if (type === "0" || type === "") entries.push({ name, read: () => data }); + else if (type === "x") { + const path = /\d+ path=([^\n]+)\n/.exec(data.toString("utf8")); + if (path) longName = path[1]; + } + at += 512 + Math.ceil(size / 512) * 512; + } + return entries; +} + +function filesOf(path) { + if (statSync(path).isDirectory()) { + return walk(path).map((file) => ({ name: relative(path, file).split(sep).join("/"), read: () => readFileSync(file) })); + } + const buffer = readFileSync(path); + if (path.endsWith(".zip")) return zipEntries(buffer); + if (path.endsWith(".tar.gz") || path.endsWith(".tgz")) return tarEntries(gunzipSync(buffer)); + if (path.endsWith(".tar")) return tarEntries(buffer); + throw new Error(`don't know how to read ${path}`); +} + +/** Problems with what would ship at `path`; empty when it is clean. */ +export function check(path) { + const problems = []; + const entries = filesOf(path); + const names = new Set(entries.map((e) => e.name)); + for (const entry of entries) { + if (entry.name.endsWith(".map")) { + problems.push(`${entry.name}: source map file`); + continue; + } + if (!SCRIPT.test(entry.name)) continue; + for (const ref of mapReferences(entry.read().toString("utf8"))) { + if (ref.startsWith("data:")) { + problems.push(`${entry.name}: inline source map`); + continue; + } + const target = join(dirname(entry.name), decodeURIComponent(ref)).split(sep).join("/"); + if (!names.has(target)) problems.push(`${entry.name}: sourceMappingURL to missing ${ref}`); + } + } + return problems; +} + +function main(argv) { + const [command, ...rest] = argv; + if (command === "collect") { + const at = rest.indexOf("--out"); + if (at < 0 || !rest[at + 1]) throw new Error("collect needs --out "); + const out = rest[at + 1]; + const roots = rest.filter((_, i) => i !== at && i !== at + 1); + const moved = collect(roots, out); + console.log(`[sourcemaps] moved ${moved.length} maps out of ${roots.join(", ")} into ${out}`); + return 0; + } + if (command === "check") { + let failed = false; + for (const path of rest) { + if (!existsSync(path)) { + console.error(`[sourcemaps] ${path}: not found`); + failed = true; + continue; + } + const problems = check(path); + if (problems.length) { + failed = true; + console.error(`[sourcemaps] ${basename(path)}: ${problems.length} problem(s)`); + for (const p of problems.slice(0, 20)) console.error(` - ${p}`); + } else { + console.log(`[sourcemaps] ${path}: no source maps`); + } + } + return failed ? 1 : 0; + } + console.error("usage: sourcemaps.mjs collect --out ... | check ..."); + return 2; +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + process.exit(main(process.argv.slice(2))); +} diff --git a/scripts/upload-sourcemaps.sh b/scripts/upload-sourcemaps.sh new file mode 100644 index 000000000..c191e7919 --- /dev/null +++ b/scripts/upload-sourcemaps.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Upload a folder of source maps (with their chunks, as scripts/sourcemaps.mjs +# collect leaves them) to Bugsink. Bugsink finds a map by the debug ID alone, +# so the release name doesn't matter, but it keeps maps per project: every +# project that gets events for these chunks needs them. +# +# BUGSINK_AUTH_TOKEN upload token; without it the upload is skipped with a notice +# BUGSINK_URL default https://bugsink.internal.bulwarkmail.org/ +# BUGSINK_PROJECTS default "bulwark-instances bulwark-telemetry" +# SENTRY_CLI_VERSION default 3.8.2 +# +# https://bulwarkmail.org/docs/admin/monitoring +set -euo pipefail + +dir="${1:?usage: upload-sourcemaps.sh }" +if [ -z "${BUGSINK_AUTH_TOKEN:-}" ]; then + echo "::notice::BUGSINK_AUTH_TOKEN is not set: source maps in ${dir} were built but not uploaded" + exit 0 +fi +if [ -z "$(find "$dir" -name '*.map' -print -quit 2>/dev/null)" ]; then + echo "::warning::no source maps in ${dir}" + exit 0 +fi + +url="${BUGSINK_URL:-https://bugsink.internal.bulwarkmail.org/}" +cli="@sentry/cli@${SENTRY_CLI_VERSION:-3.8.2}" +for project in ${BUGSINK_PROJECTS:-bulwark-instances bulwark-telemetry}; do + SENTRY_AUTH_TOKEN="$BUGSINK_AUTH_TOKEN" npx --yes "$cli" --url "$url" \ + sourcemaps --org bugsinkhasnoorgs --project "$project" upload "$dir" +done From 00e67de0474e4183b294f287a8b580d4ff837f89 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 03:38:30 +0100 Subject: [PATCH 12/26] ci: mark the source map upload script executable --- scripts/upload-sourcemaps.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/upload-sourcemaps.sh diff --git a/scripts/upload-sourcemaps.sh b/scripts/upload-sourcemaps.sh old mode 100644 new mode 100755 From 77d4114d4a52c8588ea35281a0083c565f4c042c Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 03:49:53 +0100 Subject: [PATCH 13/26] fix: remove unquoted free text such as folder names from error report messages --- lib/error-reporting/__tests__/scrub.test.ts | 9 ++++- lib/error-reporting/scrub.ts | 45 +++++++++++++++++++-- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/lib/error-reporting/__tests__/scrub.test.ts b/lib/error-reporting/__tests__/scrub.test.ts index 65be662c0..eaf2c708d 100644 --- a/lib/error-reporting/__tests__/scrub.test.ts +++ b/lib/error-reporting/__tests__/scrub.test.ts @@ -49,7 +49,14 @@ describe('scrubText', () => { it('keeps engine messages readable', () => { expect(scrubText('Cannot read properties of undefined (reading \'id\')')).toBe('Cannot read properties of undefined (reading \'…\')'); - expect(scrubText('e.map is not a function')).toBe('e.map is not a function'); + // Minified identifiers go too; the resolved stack names the code. + expect(scrubText('e.map is not a function')).toBe('… is not a function'); + }); + + it('removes unquoted free text such as a folder name or a subject', () => { + const out = scrubText('Cannot move message from ' + ADDRESS + ' to ' + FOLDER + ': ' + SUBJECT.toLowerCase()); + expectClean(out); + expect(out).toBe('Cannot move message from [email] to …: … for …'); }); it('caps the length', () => { diff --git a/lib/error-reporting/scrub.ts b/lib/error-reporting/scrub.ts index 809d633ef..9c24601e4 100644 --- a/lib/error-reporting/scrub.ts +++ b/lib/error-reporting/scrub.ts @@ -5,12 +5,13 @@ * Mail content must never leave the browser in an error, not even to the * admin's own error tracker: no message bodies, subjects, addresses, file * names, tokens, cookies or URLs. Error messages are free text, so the rules - * remove whatever could be such content: quoted text, addresses, URLs, file - * names, credentials, long random-looking strings, key=value pairs, and - * everything after the first line. What stays is the shape of the message + * remove whatever could be such content: everything after the first line, + * quoted text, addresses, URLs, file names, credentials, long random-looking + * strings and key=value pairs, and then every word that is not on a fixed + * list of common words. What stays is the shape of the message * ("Cannot read properties of undefined (reading "…")"). * - * Documented at https://bulwarkmail.org/docs/admin/monitoring#what-is-removed + * Documented at https://bulwarkmail.org/docs/deployment/monitoring#what-is-removed */ export const MAX_MESSAGE_LENGTH = 200; @@ -73,10 +74,46 @@ export function scrubText(input: unknown): string { text = text.replace(LONG_RUN, (run) => (looksRandom(run) ? '[id]' : run)); text = text.replace(IPV4, '[ip]'); text = text.replace(PHONE, '[number]'); + // Last, and the rule the others only make readable: a word stays only when + // it is on the list of common words. Free text such as an unquoted subject + // or folder name ("to Acquisition Talks") becomes "…". + text = text.replace(WORD_OR_PLACEHOLDER, (token) => ( + PLACEHOLDER.test(token) || COMMON_WORDS.has(token.toLowerCase()) ? token : REDACTED + )); + text = text.replace(/\d{5,}/g, REDACTED).replace(/…(?:[\s,;]+…)+/g, REDACTED); if (text.length > MAX_MESSAGE_LENGTH) text = `${text.slice(0, MAX_MESSAGE_LENGTH - 1)}…`; return text; } +const PLACEHOLDER = /^\[(?:email|url|token|file|id|ip|number|redacted)\]$/; +const WORD_OR_PLACEHOLDER = /\[(?:email|url|token|file|id|ip|number|redacted)\]|[\p{L}_$][\p{L}\p{N}_$'’.-]*/gu; + +/** + * Words an error message may keep: English function words and the words + * JavaScript engines, browsers, Next.js and Bulwark's own errors are made of. + * Anything else could be content and is replaced. + */ +const COMMON_WORDS = new Set(` +a about above access after again against all already an and another any are argument arguments array as at attempt +attempted available bad be been before being body both but by call called calling can can't cannot canceled cancelled +change changed changes child chunk class client close closed code component connect connected connection constructor +context could create created data date decode default defined delete deleted denied did does doesn't done during each +element empty end entry error errors event exceed exceeded exist exists expected failed fetch field file files find +first for found from function get given got had has have header headers here hook hydration if in index initial +instead internal invalid is isn't it item items its json key keys length limit list load loaded loading lost +mailbox mailboxes match maximum may maybe message messages method missing more must network new no node none not +null number object of offline on once one only open operation or out over parse parsing permission please property +properties provided read reading ready received recursion redirect refused rejected render rendered rendering request +requested requests required resource response result retry returned save saved script send sent server session set +should size some stack state status still string such support supported sync syntax than that the then there these +this thread timed timeout to token too type unable undefined unexpected unknown update updated upload uploaded use +used user using valid value values was were when which while with within without worker write wrong +abort aborted account accounts attachment attachments auth authentication authorization basic bearer blob broke broken check health heartbeat metrics something telemetry version blocked browser cache calendar callback contact contacts cookie +cors credentials decrypt denied download draft email emails encrypt encrypted expired failure folder folders +identity identities import keyword keywords login logout mail move moved oauth outbox plugin +plugins push quota rate reply sender settings signature submission subscription template theme upload websocket +`.split(/\s+/).filter(Boolean)); + /** An error name: a code identifier, or "Error". */ export function scrubName(input: unknown): string { return typeof input === 'string' && /^[A-Za-z_$][\w$]{0,63}$/.test(input) ? input : 'Error'; From 75cb0f62fe793316535b93afc8f41c4a91042b87 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 03:49:53 +0100 Subject: [PATCH 14/26] fix: measure and trace the sign-in check's calls to Stalwart --- lib/auth/verify-jmap-auth.ts | 23 ++++++++++++++++------- lib/tracing/__tests__/tracing.test.ts | 15 +++++++++++++++ 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/lib/auth/verify-jmap-auth.ts b/lib/auth/verify-jmap-auth.ts index de6edaf88..6c455cdf4 100644 --- a/lib/auth/verify-jmap-auth.ts +++ b/lib/auth/verify-jmap-auth.ts @@ -5,6 +5,7 @@ import { type PublicFetchResponse, } from '@/lib/security/url-guard'; import { forwardedClientHeaders } from '@/lib/auth/client-address'; +import { observeUpstream } from '@/lib/metrics/upstream'; const VERIFY_TIMEOUT_MS = 10000; const MAX_REDIRECTS = 3; @@ -150,7 +151,7 @@ export function assertBasicAuthMatchesUsername(authHeader: string, claimedUserna export interface VerifiedJmapSession { apiUrl: string; - /** Canonical login name the server reports for the credential (RFC 8620 §2). */ + /** Canonical login name the server reports for the credential (RFC 8620 §2). */ username?: string; primaryAccounts?: Record; accounts: Record; @@ -361,9 +362,13 @@ async function fetchIdentityEmails( }; // No redirect handling here: a redirect on the POST would downgrade it to // a GET and can only come from a misconfigured proxy - treat as failure. - const response = options.trusted - ? await fetch(apiUrl, { ...requestInit, redirect: 'manual' }) - : await fetchPublicUrl(apiUrl, requestInit); + // Measured and traced like every call to Stalwart (lib/metrics/upstream.ts). + const response = await observeUpstream('stalwart', async (trace) => { + const init = { ...requestInit, headers: { ...requestInit.headers, ...trace } }; + return options.trusted + ? await fetch(apiUrl, { ...init, redirect: 'manual' }) + : await fetchPublicUrl(apiUrl, init); + }); if (!response.ok) { throw new JmapAuthVerificationError( response.status === 401 || response.status === 403 @@ -435,9 +440,13 @@ async function fetchVerifiedSession( // Untrusted (user-supplied) endpoints connect through the rebinding-safe // fetch so the address validated above is the one the socket uses. // Admin-configured servers may legitimately live on private addresses. - response = options.trusted - ? await fetch(currentUrl, { ...requestInit, redirect: 'manual' }) - : await fetchPublicUrl(currentUrl, requestInit); + const target = currentUrl; + response = await observeUpstream('stalwart', async (trace) => { + const init = { ...requestInit, headers: { ...requestInit.headers, ...trace } }; + return options.trusted + ? await fetch(target, { ...init, redirect: 'manual' }) + : await fetchPublicUrl(target, init); + }); if (response.status >= 300 && response.status < 400) { const location = response.headers.get('location'); diff --git a/lib/tracing/__tests__/tracing.test.ts b/lib/tracing/__tests__/tracing.test.ts index 5134404b9..dd9b5c73c 100644 --- a/lib/tracing/__tests__/tracing.test.ts +++ b/lib/tracing/__tests__/tracing.test.ts @@ -13,6 +13,7 @@ import { markTracingActive } from '../config'; import { scrubAttributes, scrubSpanName } from '../scrub'; import { observeUpstream } from '@/lib/metrics/upstream'; import { fetchJmapServer } from '@/lib/stalwart/server-fetch'; +import { verifyJmapAuth } from '@/lib/auth/verify-jmap-auth'; const exporter = new InMemorySpanExporter(); const provider = createProvider(exporter, false); @@ -56,6 +57,20 @@ describe('spans for upstream calls', () => { } }); + it('reach Stalwart from the sign-in check too', async () => { + const fetchSpy = vi.fn().mockResolvedValue(new Response(JSON.stringify({ apiUrl: 'https://mail.example.org/jmap/', username: 'u@example.org' }), { status: 200 })); + vi.stubGlobal('fetch', fetchSpy); + try { + await verifyJmapAuth('https://mail.example.org', 'Basic dTpw', { trusted: true }).catch(() => undefined); + const headers = new Headers((fetchSpy.mock.calls[0][1] as RequestInit).headers); + expect(String(fetchSpy.mock.calls[0][0])).toBe('https://mail.example.org/.well-known/jmap'); + expect(headers.get('traceparent')).toMatch(/^00-[0-9a-f]{32}-[0-9a-f]{16}-01$/); + expect(headers.get('authorization')).toBe('Basic dTpw'); + } finally { + vi.unstubAllGlobals(); + } + }); + it('mark 5xx and thrown errors as errors, without the error message', async () => { await expect(observeUpstream('translate', async () => { throw new Error(`refused for ${ADDRESS}`); })).rejects.toThrow(); const [span] = exporter.getFinishedSpans(); From e02389b0791d858b6b6d22b0625bbb4f04422eec Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 03:49:54 +0100 Subject: [PATCH 15/26] test: drop type directives the source map script test doesn't need --- lib/__tests__/sourcemaps-script.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/__tests__/sourcemaps-script.test.ts b/lib/__tests__/sourcemaps-script.test.ts index e9b8ac385..cb21352d4 100644 --- a/lib/__tests__/sourcemaps-script.test.ts +++ b/lib/__tests__/sourcemaps-script.test.ts @@ -4,9 +4,7 @@ import { tmpdir } from 'node:os'; import path from 'node:path'; import { gzipSync } from 'node:zlib'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -// @ts-expect-error plain .mjs without types import { check, collect, mapReferences } from '../../scripts/sourcemaps.mjs'; -// @ts-expect-error plain .mjs without types import { buildZip } from '../../scripts/lite/lib.mjs'; // CI builds source maps for the error tracker; none may ship (images, the From 83a19f33296db61943481fd481113ea45170d050 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 03:49:54 +0100 Subject: [PATCH 16/26] docs: point code comments at the monitoring guide where the website has it --- .github/workflows/docker-publish-lite.yml | 2 +- .github/workflows/docker-publish-releases.yml | 4 ++-- .github/workflows/docker-publish.yml | 2 +- .github/workflows/standalone-release.yml | 2 +- .github/workflows/static-lite.yml | 2 +- Dockerfile | 2 +- Dockerfile.lite | 2 +- app/(main)/admin/_pages/monitoring.tsx | 2 +- app/api/health/live/route.ts | 2 +- app/api/health/ready/route.ts | 2 +- app/api/metrics/route.ts | 2 +- instrumentation.node.ts | 4 ++-- instrumentation.ts | 4 ++-- lib/admin/types.ts | 2 +- lib/error-reporting/client.ts | 2 +- lib/error-reporting/lite.ts | 2 +- lib/metrics/index.ts | 2 +- lib/tracing/config.ts | 2 +- lib/tracing/scrub.ts | 2 +- lib/tracing/start.ts | 2 +- scripts/healthcheck.mjs | 2 +- scripts/sourcemaps.mjs | 2 +- scripts/upload-sourcemaps.sh | 2 +- 23 files changed, 26 insertions(+), 26 deletions(-) diff --git a/.github/workflows/docker-publish-lite.yml b/.github/workflows/docker-publish-lite.yml index 3912bb270..854a3e35f 100644 --- a/.github/workflows/docker-publish-lite.yml +++ b/.github/workflows/docker-publish-lite.yml @@ -184,7 +184,7 @@ jobs: # The same build again, a cache hit, for the stage that holds only the # source maps; the image never has them. One export for both platforms. - # https://bulwarkmail.org/docs/admin/monitoring + # https://bulwarkmail.org/docs/deployment/monitoring - name: Export source maps uses: docker/build-push-action@v7 with: diff --git a/.github/workflows/docker-publish-releases.yml b/.github/workflows/docker-publish-releases.yml index e906e0c57..86bf50d18 100644 --- a/.github/workflows/docker-publish-releases.yml +++ b/.github/workflows/docker-publish-releases.yml @@ -67,7 +67,7 @@ jobs: # The same build again, a cache hit on this builder, for the stage that # holds only the source maps (Dockerfile, "sourcemaps"); the image never # has them. Both platforms build identical chunks, so amd64 uploads. - # https://bulwarkmail.org/docs/admin/monitoring + # https://bulwarkmail.org/docs/deployment/monitoring - name: Export source maps if: matrix.platform == 'linux/amd64' uses: docker/build-push-action@v7 @@ -216,7 +216,7 @@ jobs: # The same build again, a cache hit on this builder, for the stage that # holds only the source maps (Dockerfile, "sourcemaps"); the image never # has them. Both platforms build identical chunks, so amd64 uploads. - # https://bulwarkmail.org/docs/admin/monitoring + # https://bulwarkmail.org/docs/deployment/monitoring - name: Export source maps if: matrix.platform == 'linux/amd64' uses: docker/build-push-action@v7 diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index d10f6c423..f5b6a70f5 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -100,7 +100,7 @@ jobs: # The same build again, a cache hit on this builder, for the stage that # holds only the source maps (Dockerfile, "sourcemaps"); the image never # has them. Both platforms build identical chunks, so amd64 uploads. - # https://bulwarkmail.org/docs/admin/monitoring + # https://bulwarkmail.org/docs/deployment/monitoring - name: Export source maps if: matrix.platform == 'linux/amd64' uses: docker/build-push-action@v7 diff --git a/.github/workflows/standalone-release.yml b/.github/workflows/standalone-release.yml index f50ec11b2..164b9e81e 100644 --- a/.github/workflows/standalone-release.yml +++ b/.github/workflows/standalone-release.yml @@ -42,7 +42,7 @@ jobs: # With browser and server source maps; Turbopack gives every chunk a # debug ID. The maps go to Bugsink and never into the tarball. - # https://bulwarkmail.org/docs/admin/monitoring + # https://bulwarkmail.org/docs/deployment/monitoring - name: Build standalone env: BULWARK_SOURCE_MAPS: "1" diff --git a/.github/workflows/static-lite.yml b/.github/workflows/static-lite.yml index 1c23a1e4a..7cd355c29 100644 --- a/.github/workflows/static-lite.yml +++ b/.github/workflows/static-lite.yml @@ -80,7 +80,7 @@ jobs: LITE_LOCALES: ${{ inputs.locales }} LITE_DEMO_MODE: ${{ inputs.demo }} # Browser source maps for Bugsink; build:lite moves them out of out/ - # before anything is packed. https://bulwarkmail.org/docs/admin/monitoring + # before anything is packed. https://bulwarkmail.org/docs/deployment/monitoring BULWARK_SOURCE_MAPS: "1" SOURCEMAPS_OUT: sourcemaps run: npm run build:lite diff --git a/Dockerfile b/Dockerfile index d6ed0a17d..b089f6539 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,7 +40,7 @@ RUN rm -rf "app/api/dev-jmap" && \ # webpack has no debug IDs of its own, so sentry-cli adds them to the # chunks; then every map moves to /sourcemaps (exported with # `--target sourcemaps`) and the check fails the build if one would ship. -# https://bulwarkmail.org/docs/admin/monitoring +# https://bulwarkmail.org/docs/deployment/monitoring ARG SENTRY_CLI_VERSION=3.8.2 RUN if [ "$BULWARK_SOURCE_MAPS" = "1" ]; then \ npx --yes "@sentry/cli@${SENTRY_CLI_VERSION}" sourcemaps inject .next/static; \ diff --git a/Dockerfile.lite b/Dockerfile.lite index 94ab47a9d..931fc5b55 100644 --- a/Dockerfile.lite +++ b/Dockerfile.lite @@ -31,7 +31,7 @@ ARG NEXT_PUBLIC_DEFAULT_LOCALE= ARG GIT_COMMIT=unknown # Browser source maps for the error tracker: CI passes 1. build:lite moves # them to /sourcemaps (exported with `--target sourcemaps`) and fails if one -# would ship. https://bulwarkmail.org/docs/admin/monitoring +# would ship. https://bulwarkmail.org/docs/deployment/monitoring ARG BULWARK_SOURCE_MAPS=0 RUN mkdir -p /sourcemaps && \ LITE_TARGET=static NEXT_PUBLIC_BASE_PATH= SOURCEMAPS_OUT=/sourcemaps npm run build:lite && \ diff --git a/app/(main)/admin/_pages/monitoring.tsx b/app/(main)/admin/_pages/monitoring.tsx index feea399b7..c574cb790 100644 --- a/app/(main)/admin/_pages/monitoring.tsx +++ b/app/(main)/admin/_pages/monitoring.tsx @@ -37,7 +37,7 @@ interface MonitoringState { const PAGE = adminPageDef('monitoring'); const KEYS = ['errorReportingDsn', 'errorReportingSampleRate', 'errorReportingEnvironment'] as const; -const DOCS = 'https://bulwarkmail.org/docs/admin/monitoring'; +const DOCS = 'https://bulwarkmail.org/docs/deployment/monitoring'; export function MonitoringPage() { const [config, setConfig] = useState>({}); diff --git a/app/api/health/live/route.ts b/app/api/health/live/route.ts index b5ac3a999..e1f150f27 100644 --- a/app/api/health/live/route.ts +++ b/app/api/health/live/route.ts @@ -5,7 +5,7 @@ export const dynamic = 'force-dynamic'; /** * Liveness: 200 while the process answers. Nothing else is checked, so a * Stalwart or store outage never makes an orchestrator restart Bulwark. - * Readiness is /api/health/ready. https://bulwarkmail.org/docs/admin/monitoring#health + * Readiness is /api/health/ready. https://bulwarkmail.org/docs/deployment/monitoring#health */ const HEADERS = { 'Cache-Control': 'no-store' }; diff --git a/app/api/health/ready/route.ts b/app/api/health/ready/route.ts index fe562e724..c352dee6b 100644 --- a/app/api/health/ready/route.ts +++ b/app/api/health/ready/route.ts @@ -8,7 +8,7 @@ export const dynamic = 'force-dynamic'; * the config is loaded, the shared store (if any) answers, the data dirs are * writable and Stalwart's session resource answers within 2 s. The body * names the checks only. Results are reused for 5 s. - * https://bulwarkmail.org/docs/admin/monitoring#health + * https://bulwarkmail.org/docs/deployment/monitoring#health */ const HEADERS = { 'Cache-Control': 'no-store' }; diff --git a/app/api/metrics/route.ts b/app/api/metrics/route.ts index 16c8a98a5..79bf76f02 100644 --- a/app/api/metrics/route.ts +++ b/app/api/metrics/route.ts @@ -11,7 +11,7 @@ export const dynamic = 'force-dynamic'; * 404 as if it didn't exist. With one, the scraper sends * `Authorization: Bearer `; anything else gets 401. * - * https://bulwarkmail.org/docs/admin/monitoring#metrics + * https://bulwarkmail.org/docs/deployment/monitoring#metrics */ function sameToken(given: string, expected: string): boolean { diff --git a/instrumentation.node.ts b/instrumentation.node.ts index 0d0f91907..581235674 100644 --- a/instrumentation.node.ts +++ b/instrumentation.node.ts @@ -18,7 +18,7 @@ const current: string = pkg.version ?? "0.0.0"; logger.info(`Bulwark Webmail v${current}`, { version: current }); // Prometheus metrics, only with METRICS_TOKEN(_FILE): count every request -// from the first one on. https://bulwarkmail.org/docs/admin/monitoring#metrics +// from the first one on. https://bulwarkmail.org/docs/deployment/monitoring#metrics if (process.env.METRICS_TOKEN?.trim() || process.env.METRICS_TOKEN_FILE?.trim()) { void import("./lib/metrics/http").then(({ installHttpMetrics }) => { if (installHttpMetrics()) logger.info("Metrics endpoint on at /api/metrics"); @@ -69,7 +69,7 @@ migrateLegacyAdminLayout() logger.info("Admin dashboard initialized"); // Errors routes catch and log, and errors Next reports (instrumentation.ts), // go to the error tracker when one is configured. Off otherwise. - // https://bulwarkmail.org/docs/admin/monitoring + // https://bulwarkmail.org/docs/deployment/monitoring const { installServerErrorReporting } = await import("./lib/error-reporting/server"); await installServerErrorReporting(); // With a shared store, boot already made or loaded the secret. diff --git a/instrumentation.ts b/instrumentation.ts index 47c86b261..3f9bcba91 100644 --- a/instrumentation.ts +++ b/instrumentation.ts @@ -6,7 +6,7 @@ export async function register() { // OpenTelemetry, only with an OTLP endpoint in the standard OTEL_* // variables; otherwise the SDK is never loaded. First, so Next's own // spans from the first request on go to it. - // https://bulwarkmail.org/docs/admin/monitoring#tracing + // https://bulwarkmail.org/docs/deployment/monitoring#tracing if (tracingConfigured()) { const { startTracing } = await import("./lib/tracing/start"); startTracing(); @@ -19,7 +19,7 @@ export async function register() { * Errors Next caught while rendering a page or running a route handler, * server action or the proxy. Counted, and sent to the error tracker when * one is configured (lib/error-reporting/server.ts); nothing happens while - * error reporting is off. https://bulwarkmail.org/docs/admin/monitoring + * error reporting is off. https://bulwarkmail.org/docs/deployment/monitoring */ export const onRequestError: Instrumentation.onRequestError = async (error, _request, context) => { if (process.env.NEXT_RUNTIME !== "nodejs") return; diff --git a/lib/admin/types.ts b/lib/admin/types.ts index 521fb40dc..d6bb72231 100644 --- a/lib/admin/types.ts +++ b/lib/admin/types.ts @@ -365,7 +365,7 @@ export const CONFIG_ENV_MAP: Record@host/", "sampleRate": 1, "environment": "production" } * * The DSN's host must also be in the CSP's connect-src of the Lite host. - * https://bulwarkmail.org/docs/lite/configuration + * https://bulwarkmail.org/docs/deployment/static#error-reporting */ export function liteErrorReporting(policy: unknown): ClientSettings | null { const raw = policy && typeof policy === 'object' ? (policy as Record).errorReporting : undefined; diff --git a/lib/metrics/index.ts b/lib/metrics/index.ts index 2bf5a33b8..8dd6e498f 100644 --- a/lib/metrics/index.ts +++ b/lib/metrics/index.ts @@ -9,7 +9,7 @@ import { routeGroup } from '@/lib/route-groups'; * scrapes each one. Labels come from fixed lists only (route groups, sign-in * outcomes, upstream targets), never from user names or hosts users type. * - * https://bulwarkmail.org/docs/admin/monitoring#metrics + * https://bulwarkmail.org/docs/deployment/monitoring#metrics */ export const SIGNIN_OUTCOMES = ['success', 'failure', 'refresh_failure', 'rate_limited'] as const; diff --git a/lib/tracing/config.ts b/lib/tracing/config.ts index 13347804e..a23e40079 100644 --- a/lib/tracing/config.ts +++ b/lib/tracing/config.ts @@ -4,7 +4,7 @@ * OTEL_EXPORTER_OTLP_ENDPOINT), unless OTEL_SDK_DISABLED=true or * OTEL_TRACES_EXPORTER=none. Without it the SDK is never loaded. * - * https://bulwarkmail.org/docs/admin/monitoring#tracing + * https://bulwarkmail.org/docs/deployment/monitoring#tracing */ export function tracingEndpoint(env: Record = process.env): string | null { if ((env.OTEL_SDK_DISABLED ?? '').trim().toLowerCase() === 'true') return null; diff --git a/lib/tracing/scrub.ts b/lib/tracing/scrub.ts index ae86018a3..e698457e0 100644 --- a/lib/tracing/scrub.ts +++ b/lib/tracing/scrub.ts @@ -12,7 +12,7 @@ import { scrubText } from '@/lib/error-reporting/scrub'; * - query, fragment, user agent and client address attributes are dropped * - any other text with an address or a credential is scrubbed * - * https://bulwarkmail.org/docs/admin/monitoring#tracing + * https://bulwarkmail.org/docs/deployment/monitoring#tracing */ type AttributeValue = string | number | boolean | Array | undefined; diff --git a/lib/tracing/start.ts b/lib/tracing/start.ts index 037430377..12f1cb691 100644 --- a/lib/tracing/start.ts +++ b/lib/tracing/start.ts @@ -24,7 +24,7 @@ import { scrubAttributes, scrubSpanName, type Attributes } from './scrub'; * for the calls it makes (lib/metrics/upstream.ts) go through the same * provider; a processor scrubs every span before export (./scrub.ts). * - * https://bulwarkmail.org/docs/admin/monitoring#tracing + * https://bulwarkmail.org/docs/deployment/monitoring#tracing */ /** Scrubs a span's name and attributes, then hands it on. */ diff --git a/scripts/healthcheck.mjs b/scripts/healthcheck.mjs index 26ebbc359..d78a46157 100644 --- a/scripts/healthcheck.mjs +++ b/scripts/healthcheck.mjs @@ -1,7 +1,7 @@ // Docker HEALTHCHECK: exit 0 when /api/health/live answers 200 within 4 s. // Liveness only, so a Stalwart outage never marks the container unhealthy. // The base path is baked in at build time (BULWARK_BASE_PATH in the image). -// https://bulwarkmail.org/docs/admin/monitoring#health +// https://bulwarkmail.org/docs/deployment/monitoring#health const port = process.env.PORT || '3000'; const listen = process.env.HOSTNAME; const host = !listen || listen === '0.0.0.0' || listen === '::' ? '127.0.0.1' : listen.includes(':') ? `[${listen}]` : listen; diff --git a/scripts/sourcemaps.mjs b/scripts/sourcemaps.mjs index 4303506d3..4f332b3aa 100644 --- a/scripts/sourcemaps.mjs +++ b/scripts/sourcemaps.mjs @@ -1,6 +1,6 @@ #!/usr/bin/env node // Source maps: built in CI (BULWARK_SOURCE_MAPS=1), uploaded to the error -// tracker, never shipped. https://bulwarkmail.org/docs/admin/monitoring +// tracker, never shipped. https://bulwarkmail.org/docs/deployment/monitoring // // node scripts/sourcemaps.mjs collect --out ... // Moves every .map under each root into //... with a copy of the diff --git a/scripts/upload-sourcemaps.sh b/scripts/upload-sourcemaps.sh index c191e7919..5c0c35976 100755 --- a/scripts/upload-sourcemaps.sh +++ b/scripts/upload-sourcemaps.sh @@ -9,7 +9,7 @@ # BUGSINK_PROJECTS default "bulwark-instances bulwark-telemetry" # SENTRY_CLI_VERSION default 3.8.2 # -# https://bulwarkmail.org/docs/admin/monitoring +# https://bulwarkmail.org/docs/deployment/monitoring set -euo pipefail dir="${1:?usage: upload-sourcemaps.sh }" From b92717921e7eb53fb2f278c1bd792171a68aeab8 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 03:57:47 +0100 Subject: [PATCH 17/26] fix: upload each source map with the chunk that names it when the build path is relative --- lib/__tests__/sourcemaps-script.test.ts | 17 +++++++++++++++++ scripts/sourcemaps.mjs | 4 +++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/lib/__tests__/sourcemaps-script.test.ts b/lib/__tests__/sourcemaps-script.test.ts index cb21352d4..16beb45bc 100644 --- a/lib/__tests__/sourcemaps-script.test.ts +++ b/lib/__tests__/sourcemaps-script.test.ts @@ -59,6 +59,23 @@ describe('collect', () => { }); }); +describe('collect with a relative root', () => { + it('still copies the chunk that names each map', () => { + put('rel/chunks/b.js', 'x\n//# sourceMappingURL=b.js.map\n'); + put('rel/chunks/b.js.map', '{"version":3}'); + const cwd = process.cwd(); + process.chdir(dir); + try { + collect(['rel'], 'out'); + } finally { + process.chdir(cwd); + } + expect(existsSync(path.join(dir, 'out/0/chunks/b.js'))).toBe(true); + expect(existsSync(path.join(dir, 'out/0/chunks/b.js.map'))).toBe(true); + expect(mapReferences(readFileSync(path.join(dir, 'rel/chunks/b.js'), 'utf8'))).toEqual([]); + }); +}); + describe('check', () => { it('finds map files, dangling references and inline maps in a folder', () => { put('a/x.js.map', '{}'); diff --git a/scripts/sourcemaps.mjs b/scripts/sourcemaps.mjs index 4f332b3aa..89b24fb1f 100644 --- a/scripts/sourcemaps.mjs +++ b/scripts/sourcemaps.mjs @@ -45,7 +45,9 @@ export function stripReferences(text) { export function collect(roots, outDir) { const moved = []; mkdirSync(outDir, { recursive: true }); - roots.forEach((root, index) => { + roots.forEach((given, index) => { + // Absolute, so the maps a file names (resolved below) match the walked paths. + const root = resolve(given); if (!existsSync(root)) return; const files = walk(root); const maps = new Set(files.filter((f) => f.endsWith(".map"))); From 4fe0b6b611793e20348316655cbec7b81007f5fe Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 04:04:15 +0100 Subject: [PATCH 18/26] ci: upload the standalone build's server source maps too --- .github/workflows/standalone-release.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/standalone-release.yml b/.github/workflows/standalone-release.yml index 164b9e81e..bd4d55126 100644 --- a/.github/workflows/standalone-release.yml +++ b/.github/workflows/standalone-release.yml @@ -48,8 +48,10 @@ jobs: BULWARK_SOURCE_MAPS: "1" run: npm run build + # .next/server holds the server chunks' maps; standalone copies the + # chunks only. It doesn't ship, but its maps are needed for server frames. - name: Move source maps out of the build - run: node scripts/sourcemaps.mjs collect --out sourcemaps .next/static .next/standalone + run: node scripts/sourcemaps.mjs collect --out sourcemaps .next/static .next/standalone .next/server # Turbopack chunks were not checked for being the same on both # platforms, so each one uploads its own. From e7f3b21c0600069c6797e85df8479eddb26b76dc Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 04:32:45 +0100 Subject: [PATCH 19/26] chore: let the demo deploy pass extra environment from an env file on the server --- deploy/demo-deploy.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/deploy/demo-deploy.sh b/deploy/demo-deploy.sh index 00a8ea645..2cd8d7d3c 100644 --- a/deploy/demo-deploy.sh +++ b/deploy/demo-deploy.sh @@ -30,6 +30,9 @@ DATA=/opt/bulwark-demo-data # exists only so pruning cannot collect the image we may need to go back to. PREVIOUS=bulwark-demo:previous STATE=/opt/bulwark-demo-deploy +# Optional extra environment for the container (e.g. ERROR_REPORTING_DSN), +# one VAR=value per line. Not in a bind-mounted dir: the app never sees it. +ENV_FILE=/opt/bulwark-demo-data/demo.env log() { echo "[demo-deploy] $*"; } @@ -45,11 +48,14 @@ version_of() { docker_ image inspect -f '{{index .Config.Labels "org.opencontain # host port 3002, and the three bind mounts that keep the demo past the setup # wizard (admin/config.json holds setupComplete). run_container() { + local env_args=() + if [ -f "$ENV_FILE" ]; then env_args=(--env-file "$ENV_FILE"); fi docker_ rm -f "$NAME" >/dev/null 2>&1 || true docker_ run -d \ --name "$NAME" \ --restart unless-stopped \ -p "${PORT}:3000" \ + ${env_args[@]+"${env_args[@]}"} \ -e DEMO_MODE=true \ -v "$DATA/admin:/app/data/admin" \ -v "$DATA/settings:/app/data/settings" \ From 79f73c5c846d4e17fce7a4527fedfb3d0b895937 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 04:35:04 +0100 Subject: [PATCH 20/26] fix: keep the errors endpoint out of the Bulwark Lite bundle --- .../__tests__/client-lite.test.ts | 37 +++++++++++++++++++ lib/error-reporting/client.ts | 5 ++- 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 lib/error-reporting/__tests__/client-lite.test.ts diff --git a/lib/error-reporting/__tests__/client-lite.test.ts b/lib/error-reporting/__tests__/client-lite.test.ts new file mode 100644 index 000000000..3c6064839 --- /dev/null +++ b/lib/error-reporting/__tests__/client-lite.test.ts @@ -0,0 +1,37 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('@/lib/lite', async (importOriginal) => ({ + ...(await importOriginal()), + IS_LITE: true, +})); + +import { captureException, configureErrorReporting, resetErrorReportingForTests } from '../client'; + +const DSN = 'https://6f0b9c79ed3a4315b76595bc2536ac40@bugsink.example.org/3'; +let fetchSpy: ReturnType; + +beforeEach(() => { + resetErrorReportingForTests(); + sessionStorage.clear(); + fetchSpy = vi.fn().mockResolvedValue(new Response(null, { status: 200 })); + vi.stubGlobal('fetch', fetchSpy); +}); + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +// Bulwark Lite has no server, so it never posts to /api/errors, whatever +// the settings say; reports go to the DSN only. +describe('error reporting in Lite', () => { + it('sends to the DSN and never to /api/errors', () => { + configureErrorReporting({ dsn: DSN, sampleRate: 1, environment: 'lite', collect: true }); + const error = new TypeError('boom'); + error.stack = 'TypeError: boom\n at a (https://lite.example.org/_next/static/chunks/1.js:2:30)'; + captureException(error); + const urls = fetchSpy.mock.calls.map(([url]) => String(url)); + expect(urls).toHaveLength(1); + expect(urls[0]).toContain('/api/3/envelope/'); + expect(urls.some((url) => url.includes('/api/errors'))).toBe(false); + }); +}); diff --git a/lib/error-reporting/client.ts b/lib/error-reporting/client.ts index 57f27ca8c..712a21f14 100644 --- a/lib/error-reporting/client.ts +++ b/lib/error-reporting/client.ts @@ -1,4 +1,5 @@ import { apiFetch, getPathPrefix } from '@/lib/browser-navigation'; +import { IS_LITE } from '@/lib/lite'; import { scrubRoutePath } from '@/lib/route-groups'; import { debugIdsByPath } from './debug-ids'; import { @@ -200,7 +201,9 @@ function deliver(report: ErrorReport): void { mode: 'cors', }); } - if (settings.collect) { + // Lite has no server: the bundler drops this branch, and the endpoint + // with it (scripts/lite/verify.mjs fails on undocumented /api strings). + if (!IS_LITE && settings.collect) { send('/api/errors', JSON.stringify({ name: report.name, message: report.message, From 0ca60ee5d7cd1819d26706998db992e26c627667 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 04:40:16 +0100 Subject: [PATCH 21/26] fix: drop the errors endpoint from the Bulwark Lite bundle for real --- lib/error-reporting/__tests__/client-lite.test.ts | 7 ++----- lib/error-reporting/client.ts | 8 ++++---- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/lib/error-reporting/__tests__/client-lite.test.ts b/lib/error-reporting/__tests__/client-lite.test.ts index 3c6064839..5e4ab5d04 100644 --- a/lib/error-reporting/__tests__/client-lite.test.ts +++ b/lib/error-reporting/__tests__/client-lite.test.ts @@ -1,16 +1,12 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -vi.mock('@/lib/lite', async (importOriginal) => ({ - ...(await importOriginal()), - IS_LITE: true, -})); - import { captureException, configureErrorReporting, resetErrorReportingForTests } from '../client'; const DSN = 'https://6f0b9c79ed3a4315b76595bc2536ac40@bugsink.example.org/3'; let fetchSpy: ReturnType; beforeEach(() => { + vi.stubEnv('NEXT_PUBLIC_BULWARK_LITE', '1'); resetErrorReportingForTests(); sessionStorage.clear(); fetchSpy = vi.fn().mockResolvedValue(new Response(null, { status: 200 })); @@ -18,6 +14,7 @@ beforeEach(() => { }); afterEach(() => { + vi.unstubAllEnvs(); vi.unstubAllGlobals(); }); diff --git a/lib/error-reporting/client.ts b/lib/error-reporting/client.ts index 712a21f14..23d6e69db 100644 --- a/lib/error-reporting/client.ts +++ b/lib/error-reporting/client.ts @@ -1,5 +1,4 @@ import { apiFetch, getPathPrefix } from '@/lib/browser-navigation'; -import { IS_LITE } from '@/lib/lite'; import { scrubRoutePath } from '@/lib/route-groups'; import { debugIdsByPath } from './debug-ids'; import { @@ -201,9 +200,10 @@ function deliver(report: ErrorReport): void { mode: 'cors', }); } - // Lite has no server: the bundler drops this branch, and the endpoint - // with it (scripts/lite/verify.mjs fails on undocumented /api strings). - if (!IS_LITE && settings.collect) { + // Lite has no server. Read here, not through IS_LITE: the bundler inlines + // the variable only in this module, so the branch and the endpoint string + // leave the Lite bundle (scripts/lite/verify.mjs). + if (process.env.NEXT_PUBLIC_BULWARK_LITE !== '1' && settings.collect) { send('/api/errors', JSON.stringify({ name: report.name, message: report.message, From 867c734ca5ce321027d06c53eb6e1deee7584d13 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 05:52:27 +0100 Subject: [PATCH 22/26] ci: publish the Lite image without source maps until its build stops stalling on the runner --- .github/workflows/docker-publish-lite.yml | 25 ++++------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/.github/workflows/docker-publish-lite.yml b/.github/workflows/docker-publish-lite.yml index 854a3e35f..0a22c5dc5 100644 --- a/.github/workflows/docker-publish-lite.yml +++ b/.github/workflows/docker-publish-lite.yml @@ -178,30 +178,13 @@ jobs: labels: ${{ steps.meta.outputs.labels }} build-args: | GIT_COMMIT=${{ github.sha }} - BULWARK_SOURCE_MAPS=1 cache-from: type=gha,scope=lite cache-to: type=gha,mode=max,scope=lite - # The same build again, a cache hit, for the stage that holds only the - # source maps; the image never has them. One export for both platforms. - # https://bulwarkmail.org/docs/deployment/monitoring - - name: Export source maps - uses: docker/build-push-action@v7 - with: - context: . - file: Dockerfile.lite - platforms: linux/amd64 - target: sourcemaps - build-args: | - GIT_COMMIT=${{ github.sha }} - BULWARK_SOURCE_MAPS=1 - outputs: type=local,dest=sourcemaps - cache-from: type=gha,scope=lite - - - name: Upload source maps to Bugsink - env: - BUGSINK_AUTH_TOKEN: ${{ secrets.BUGSINK_AUTH_TOKEN }} - run: bash scripts/upload-sourcemaps.sh sourcemaps + # No source maps for this image yet: with BULWARK_SOURCE_MAPS=1 the + # multi-platform build stalled on the runner after the Lite build step + # (twice, 2026-11-01), though it finishes locally. The static Lite zips + # (static-lite.yml) still upload theirs. - name: Inspect image run: docker buildx imagetools inspect ${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }} From 158d87842baa953cf577d3e8fb726a70409522bd Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 06:24:38 +0100 Subject: [PATCH 23/26] ci: restore the Lite image build step as it was before the source map work --- Dockerfile.lite | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/Dockerfile.lite b/Dockerfile.lite index 931fc5b55..8a71f6650 100644 --- a/Dockerfile.lite +++ b/Dockerfile.lite @@ -29,18 +29,8 @@ ARG NEXT_PUBLIC_DEFAULT_LOCALE= # Commit SHA shown in the About screen. .dockerignore excludes .git, so CI # must pass it in. ARG GIT_COMMIT=unknown -# Browser source maps for the error tracker: CI passes 1. build:lite moves -# them to /sourcemaps (exported with `--target sourcemaps`) and fails if one -# would ship. https://bulwarkmail.org/docs/deployment/monitoring -ARG BULWARK_SOURCE_MAPS=0 -RUN mkdir -p /sourcemaps && \ - LITE_TARGET=static NEXT_PUBLIC_BASE_PATH= SOURCEMAPS_OUT=/sourcemaps npm run build:lite && \ - node scripts/lite/container.mjs /image && \ - node scripts/sourcemaps.mjs check /image/html - -# Only the maps, for CI to upload: never part of the image. -FROM scratch AS sourcemaps -COPY --from=builder /sourcemaps / +RUN LITE_TARGET=static NEXT_PUBLIC_BASE_PATH= npm run build:lite && \ + node scripts/lite/container.mjs /image FROM nginxinc/nginx-unprivileged:1.30-alpine-slim From eb26219b3c87dea40608f4231a8d889e5436a5ba Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 12:55:58 +0100 Subject: [PATCH 24/26] feat: send telemetry error counts within a minute when an admin turns on instant mode --- app/(main)/admin/_pages/telemetry.tsx | 38 +++- app/api/admin/telemetry/route.ts | 14 +- instrumentation.node.ts | 6 +- .../telemetry-instant-errors.test.ts | 189 ++++++++++++++++++ lib/admin/outgoing-hosts-registry.ts | 2 +- .../__tests__/collect-consent.test.ts | 7 +- lib/error-reporting/settings.ts | 2 +- lib/telemetry/index.ts | 3 +- lib/telemetry/instant-errors.ts | 123 ++++++++++++ lib/telemetry/payload.ts | 16 +- lib/telemetry/state.ts | 14 +- lib/telemetry/types.ts | 31 ++- 12 files changed, 420 insertions(+), 25 deletions(-) create mode 100644 lib/__tests__/telemetry-instant-errors.test.ts create mode 100644 lib/telemetry/instant-errors.ts diff --git a/app/(main)/admin/_pages/telemetry.tsx b/app/(main)/admin/_pages/telemetry.tsx index 1ccdd94a1..74a84ed06 100644 --- a/app/(main)/admin/_pages/telemetry.tsx +++ b/app/(main)/admin/_pages/telemetry.tsx @@ -16,7 +16,8 @@ import { cn } from '@/lib/utils'; interface TelemetryStatus { consent: 'pending' | 'on' | 'off'; consentSource: 'no-third-party' | 'env' | 'file'; - errorsConsent?: 'on' | 'off'; + /** 'on' sends them daily; 'instant' also sends today's as they change. */ + errorsConsent?: ErrorsMode; errorsConsentSource?: 'telemetry' | 'no-third-party' | 'env' | 'file'; endpoint: string; defaultEndpoint: string; @@ -25,9 +26,12 @@ interface TelemetryStatus { nextScheduledAt: string | null; payloadPreview: Record; errorsPreview?: unknown[]; + instantErrorsPreview?: unknown[]; accountCounts: { total: number; active7d: number }; } +type ErrorsMode = 'on' | 'off' | 'instant'; + const PAGE = adminPageDef('telemetry'); function timeAgo(iso: string | null): string { @@ -59,7 +63,7 @@ export function TelemetryPage() { // The heartbeat switch waits for Save like the endpoint (one page, one // save bar). const [consentDraft, setConsentDraft] = useState<'on' | 'off' | null>(null); - const [errorsDraft, setErrorsDraft] = useState<'on' | 'off' | null>(null); + const [errorsDraft, setErrorsDraft] = useState(null); const endpointDirty = status !== null && endpointDraft !== status.endpoint; const consentDirty = status !== null && consentDraft !== null && consentDraft !== status.consent; @@ -159,12 +163,17 @@ export function TelemetryPage() { // Error counts follow the heartbeat switch as it is about to be saved. const errorsLocked = noThirdParty || status.errorsConsentSource === 'env' || !shownOn; - const errorsShownOn = shownOn && (errorsDraft ?? status.errorsConsent ?? 'off') === 'on'; + const errorsMode: ErrorsMode = shownOn ? (errorsDraft ?? status.errorsConsent ?? 'off') : 'off'; + const errorsShownOn = errorsMode !== 'off'; const errorsDescription = [ 'Adds how often each error happened yesterday, the error name and where in the code it happened. No message text, no URLs, no instance id goes to the error tracker.', !shownOn && 'Needs the heartbeat on.', status.errorsConsentSource === 'env' && 'Locked by the BULWARK_TELEMETRY_ERRORS env var.', ].filter(Boolean).join(' '); + const instantDescription = [ + 'Also sends today\'s counts within about a minute when an error is new or happens noticeably more often, so a broken release shows up the same day. The same fields as the daily counts; the collector learns roughly when errors happen.', + !errorsShownOn && 'Needs error counts on.', + ].filter(Boolean).join(' '); // The payload as it will be once the switches are saved. const previewPayload = errorsShownOn @@ -220,6 +229,13 @@ export function TelemetryPage() { onChange={(checked) => { setErrorsDraft(checked ? 'on' : 'off'); setMessage(null); }} /> + + { setErrorsDraft(checked ? 'instant' : 'on'); setMessage(null); }} + /> + {timeAgo(status.lastSentAt)} @@ -290,6 +306,22 @@ export function TelemetryPage() { 
                 {JSON.stringify(previewPayload, null, 2)}
    + {errorsMode === 'instant' && ( + <> +

    + Sent right away: today's error counts as they would go out now. Each minute only + the errors that are new or reached a higher count range are sent. +

    + 
    +                {JSON.stringify({
+                  schema: '1',
+                  instance_id: status.payloadPreview.instance_id,
+                  day: new Date().toISOString().slice(0, 10),
+                  errors: status.instantErrorsPreview ?? [],
+                }, null, 2)}
+
    + + )} diff --git a/app/api/admin/telemetry/route.ts b/app/api/admin/telemetry/route.ts index 9c8068e2e..2773879b2 100644 --- a/app/api/admin/telemetry/route.ts +++ b/app/api/admin/telemetry/route.ts @@ -14,8 +14,8 @@ import { resolveEndpointAllowed, } from '@/lib/telemetry'; import { storedDataUnavailableResponse } from '@/lib/stored-data-errors'; -import { errorsForPayload } from '@/lib/telemetry/payload'; -import { readErrors } from '@/lib/error-reporting/aggregate'; +import { errorsForDay, errorsForPayload } from '@/lib/telemetry/payload'; +import { readErrors, utcDay } from '@/lib/error-reporting/aggregate'; /** * GET /api/admin/telemetry @@ -51,6 +51,8 @@ export async function GET(request: NextRequest) { // What `errors` would hold, so the page can show the v2 payload as soon // as the switch is flipped, before anything is saved or sent. errorsPreview: errorsForPayload(counted), + // What instant mode would send now: today's counts so far. + instantErrorsPreview: errorsForDay(counted, utcDay()), accountCounts, }, { headers: { 'Cache-Control': 'no-store' } }, @@ -69,7 +71,7 @@ export async function GET(request: NextRequest) { * POST /api/admin/telemetry * Body: { action: 'set-consent' | 'set-endpoint' | 'send-now', ... } * set-consent : { action, consent: 'on' | 'off' } (off also switches error counts off) - * set-errors-consent : { action, consent: 'on' | 'off' } + * set-errors-consent : { action, consent: 'on' | 'instant' | 'off' } ('on' is daily) * set-endpoint : { action, endpoint: string } * send-now : { action } */ @@ -129,10 +131,10 @@ export async function POST(request: NextRequest) { if (errors.source === 'no-third-party') { return NextResponse.json({ error: 'telemetry is off while NO_THIRD_PARTY_TRAFFIC is on' }, { status: 409 }); } - if (body.consent !== 'on' && body.consent !== 'off') { - return NextResponse.json({ error: 'consent must be "on" or "off"' }, { status: 400 }); + if (body.consent !== 'on' && body.consent !== 'off' && body.consent !== 'instant') { + return NextResponse.json({ error: 'consent must be "on", "instant" or "off"' }, { status: 400 }); } - if (body.consent === 'on' && errors.source === 'telemetry') { + if (body.consent !== 'off' && errors.source === 'telemetry') { return NextResponse.json({ error: 'turn on the heartbeat first' }, { status: 409 }); } const consent = body.consent; diff --git a/instrumentation.node.ts b/instrumentation.node.ts index 581235674..a1f36ff11 100644 --- a/instrumentation.node.ts +++ b/instrumentation.node.ts @@ -119,15 +119,19 @@ migrateLegacyAdminLayout() // See https://bulwarkmail.org/docs/legal/privacy/telemetry const { startScheduler, stopScheduler, markProcessStart } = await import("./lib/telemetry"); markProcessStart(); + // Instant error counts: a one-minute tick that sends only while the + // error-count consent is "instant". + const instantErrors = await import("./lib/telemetry/instant-errors"); // Hourly check against version.telemetry.bulwarkmail.org. Disable with // BULWARK_UPDATE_CHECK=off or override the endpoint with // BULWARK_UPDATE_CHECK_URL. const versionCheck = await import("./lib/version-check"); - // Both run on one replica of the deployment, the scheduler leader + // All run on one replica of the deployment, the scheduler leader // (#1066); a single instance always leads. const { startSchedulerLeader } = await import("./lib/scheduler-leader"); await startSchedulerLeader([ { name: "telemetry", start: startScheduler, stop: stopScheduler }, + { name: "telemetry-errors", start: instantErrors.startInstantErrors, stop: instantErrors.stopInstantErrors }, { name: "version-check", start: versionCheck.startScheduler, stop: versionCheck.stopScheduler }, ]); }) diff --git a/lib/__tests__/telemetry-instant-errors.test.ts b/lib/__tests__/telemetry-instant-errors.test.ts new file mode 100644 index 000000000..b3f7d79ff --- /dev/null +++ b/lib/__tests__/telemetry-instant-errors.test.ts @@ -0,0 +1,189 @@ +// @vitest-environment node +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { NextRequest } from 'next/server'; + +// Instant error counts: with the error-count consent set to "instant", the +// scheduler leader sends the current day's counts to the collector's +// /v1/errors once a minute, but only errors that are new or whose count +// bucket changed since the last send. Daily mode sends nothing extra. + +const thirdParty = vi.hoisted(() => ({ on: false })); +vi.mock('@/lib/admin/third-party-mode', () => ({ isNoThirdPartyTrafficOn: async () => thirdParty.on })); +vi.mock('@/lib/admin/session', () => ({ + requireAdminAuth: vi.fn(async () => ({ session: { sub: 'admin' } })), + getClientIP: () => '127.0.0.1', +})); +vi.mock('@/lib/admin/audit', () => ({ auditLog: vi.fn(async () => {}) })); +vi.mock('@/lib/admin/plugin-registry', () => ({ + getPluginRegistry: async () => ({ plugins: [] }), + getThemeRegistry: async () => ({ themes: [] }), +})); + +const sent = vi.fn(async (_url: string, _init?: { body?: string }) => new Response('{"ok":true}')); +vi.mock('@/lib/telemetry/endpoint-guard', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + resolveEndpointAllowed: vi.fn(async () => ({ ok: true })), + fetchTelemetryTarget: (url: string, init?: { body?: string }) => sent(url, init), + }; +}); + +import { POST } from '@/app/api/admin/telemetry/route'; +import { DEFAULT_ENDPOINT, buildPayload, effectiveErrorsConsent, loadState, saveState } from '@/lib/telemetry'; +import { resetInstantErrorsForTests, sendInstantErrors } from '@/lib/telemetry/instant-errors'; +import { resetAggregateForTests, utcDay } from '@/lib/error-reporting/aggregate'; + +let dir: string; +const debugId = 'bc67f7db-7f3b-7be2-6f94-966cd7825cc2'; + +function entry(count: number, name = 'TypeError') { + return { + name, + frames: [ + { path: '/_next/static/chunks/1.js', line: 2, column: 30, debugId }, + { path: 'node_modules/next/dist/x.js', line: 1, column: 1 }, + ], + version: '1.20.0', + runtime: 'browser' as const, + count, + }; +} + +function storeCounts(day: Record>): void { + writeFileSync(path.join(dir, 'telemetry', 'errors.json'), JSON.stringify({ [utcDay()]: day })); +} + +function bodies(): Array & { errors: Array> }> { + return sent.mock.calls.map(([, init]) => JSON.parse(String(init?.body))); +} + +function post(body: Record) { + return POST(new NextRequest('http://localhost/api/admin/telemetry', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body), + })); +} + +beforeAll(() => { + dir = mkdtempSync(path.join(tmpdir(), 'bulwark-telemetry-instant-')); + vi.stubEnv('ADMIN_CONFIG_DIR', dir); + vi.stubEnv('TELEMETRY_DATA_DIR', path.join(dir, 'telemetry')); + mkdirSync(path.join(dir, 'telemetry'), { recursive: true }); +}); +afterAll(() => { + vi.unstubAllEnvs(); + rmSync(dir, { recursive: true, force: true }); +}); +beforeEach(async () => { + thirdParty.on = false; + vi.stubEnv('BULWARK_TELEMETRY', ''); + vi.stubEnv('BULWARK_TELEMETRY_ERRORS', ''); + sent.mockClear(); + sent.mockImplementation(async () => new Response('{"ok":true}')); + resetAggregateForTests(); + resetInstantErrorsForTests(); + await saveState({ ...(await loadState()), consent: 'on', endpoint: DEFAULT_ENDPOINT, errorsConsent: 'instant' }); + storeCounts({ aaaa000011112222: entry(1) }); +}); +afterEach(() => { + resetAggregateForTests(); + resetInstantErrorsForTests(); +}); + +describe('instant error counts', () => { + it('sends today\'s counts to the collector\'s /v1/errors, nothing but the allowed fields', async () => { + await sendInstantErrors(); + expect(sent).toHaveBeenCalledTimes(1); + expect(sent.mock.calls[0][0]).toBe('https://telemetry.bulwarkmail.org/v1/errors'); + const [body] = bodies(); + expect(Object.keys(body).sort()).toEqual(['day', 'errors', 'instance_id', 'schema']); + expect(body.schema).toBe('1'); + expect(body.day).toBe(utcDay()); + expect(body.instance_id).toMatch(/^[0-9a-f-]{36}$/); + expect(body.errors).toEqual([{ + name: 'TypeError', + fingerprint: 'aaaa000011112222', + frames: [{ path: '/_next/static/chunks/1.js', line: 2, col: 30, debug_id: debugId }], + version: '1.20.0', + count: '1', + days: 1, + }]); + }); + + it('sends an error again only when its count bucket changes', async () => { + storeCounts({ aaaa000011112222: entry(1), bbbb000011112222: entry(1, 'RangeError') }); + await sendInstantErrors(); + expect(bodies()[0].errors).toHaveLength(2); + + await sendInstantErrors(); + expect(sent).toHaveBeenCalledTimes(1); + + // 1 -> 3 moves from '1' to '2-5'. + storeCounts({ aaaa000011112222: entry(3), bbbb000011112222: entry(1, 'RangeError') }); + await sendInstantErrors(); + expect(sent).toHaveBeenCalledTimes(2); + expect(bodies()[1].errors.map((e) => [e.fingerprint, e.count])).toEqual([['aaaa000011112222', '2-5']]); + + // 3 -> 4 is still '2-5': nothing new to say. + storeCounts({ aaaa000011112222: entry(4), bbbb000011112222: entry(1, 'RangeError') }); + await sendInstantErrors(); + expect(sent).toHaveBeenCalledTimes(2); + }); + + it('tries a failed send again on the next tick', async () => { + sent.mockImplementationOnce(async () => new Response('{}', { status: 503 })); + await sendInstantErrors(); + await sendInstantErrors(); + expect(sent).toHaveBeenCalledTimes(2); + expect(bodies()[1].errors).toHaveLength(1); + }); + + it('sends nothing in daily mode, with error counts off, with the heartbeat off or in no-third-party mode', async () => { + await saveState({ ...(await loadState()), errorsConsent: 'on' }); + await sendInstantErrors(); + await saveState({ ...(await loadState()), errorsConsent: 'off' }); + await sendInstantErrors(); + await saveState({ ...(await loadState()), consent: 'off', errorsConsent: 'instant' }); + await sendInstantErrors(); + await saveState({ ...(await loadState()), consent: 'on' }); + thirdParty.on = true; + await sendInstantErrors(); + expect(sent).not.toHaveBeenCalled(); + }); + + it('sends nothing when the collector URL is empty, and follows a custom one', async () => { + await saveState({ ...(await loadState()), endpoint: '' }); + await sendInstantErrors(); + expect(sent).not.toHaveBeenCalled(); + await saveState({ ...(await loadState()), endpoint: 'https://collector.example.org/v1/heartbeat' }); + await sendInstantErrors(); + expect(sent.mock.calls[0][0]).toBe('https://collector.example.org/v1/errors'); + }); + + it('also sends the daily heartbeat with error counts (schema 2)', async () => { + expect((await buildPayload()).schema).toBe('2'); + }); +}); + +describe('instant consent', () => { + it('can be chosen in the admin panel and needs the heartbeat', async () => { + await saveState({ ...(await loadState()), errorsConsent: 'off' }); + expect((await post({ action: 'set-errors-consent', consent: 'instant' })).status).toBe(200); + expect((await loadState()).errorsConsent).toBe('instant'); + expect((await effectiveErrorsConsent()).consent).toBe('instant'); + + await saveState({ ...(await loadState()), consent: 'off', errorsConsent: 'off' }); + expect((await post({ action: 'set-errors-consent', consent: 'instant' })).status).toBe(409); + }); + + it('can be set with BULWARK_TELEMETRY_ERRORS=instant', async () => { + await saveState({ ...(await loadState()), errorsConsent: 'off' }); + vi.stubEnv('BULWARK_TELEMETRY_ERRORS', 'instant'); + expect(await effectiveErrorsConsent()).toEqual({ consent: 'instant', source: 'env' }); + }); +}); diff --git a/lib/admin/outgoing-hosts-registry.ts b/lib/admin/outgoing-hosts-registry.ts index c469e9186..394f467be 100644 --- a/lib/admin/outgoing-hosts-registry.ts +++ b/lib/admin/outgoing-hosts-registry.ts @@ -39,7 +39,7 @@ export const OUTGOING_HOSTS: readonly OutgoingHost[] = [ host: 'telemetry.bulwarkmail.org', files: ['lib/telemetry/types.ts'], side: 'server', - usedBy: 'Anonymous daily heartbeat, sent only after an admin turned telemetry on.', + usedBy: 'Anonymous daily heartbeat, sent only after an admin turned telemetry on; with instant error counts on, also the current day\'s error counts as they change.', off: 'Off by default. BULWARK_TELEMETRY=off, or System > Usage stats (switch it off or clear the collector URL).', inMode: 'Never sent, whatever the telemetry setting says.', lite: false, diff --git a/lib/error-reporting/__tests__/collect-consent.test.ts b/lib/error-reporting/__tests__/collect-consent.test.ts index 93e82d937..a1dbcfa6d 100644 --- a/lib/error-reporting/__tests__/collect-consent.test.ts +++ b/lib/error-reporting/__tests__/collect-consent.test.ts @@ -1,7 +1,7 @@ // @vitest-environment node import { describe, expect, it, vi } from 'vitest'; -const consent = vi.hoisted(() => ({ value: 'off' as 'on' | 'off' })); +const consent = vi.hoisted(() => ({ value: 'off' as 'on' | 'off' | 'instant' })); vi.mock('@/lib/admin/config-manager', () => ({ configManager: { ensureLoaded: async () => {}, get: (_key: string, fallback: unknown) => fallback }, })); @@ -23,4 +23,9 @@ describe('error collection', () => { consent.value = 'on'; expect(await shouldCollectErrors()).toBe(true); }); + + it('is on in instant mode too', async () => { + consent.value = 'instant'; + expect(await shouldCollectErrors()).toBe(true); + }); }); diff --git a/lib/error-reporting/settings.ts b/lib/error-reporting/settings.ts index 64092419a..0e4097e32 100644 --- a/lib/error-reporting/settings.ts +++ b/lib/error-reporting/settings.ts @@ -46,7 +46,7 @@ export function errorReportingSettings(): ErrorReportingSettings { export async function shouldCollectErrors(): Promise { await configManager.ensureLoaded(); if (errorReportingSettings().dsn || metricsEnabled()) return true; - return (await effectiveErrorsConsent()).consent === 'on'; + return (await effectiveErrorsConsent()).consent !== 'off'; } /** What /api/config hands the browser. */ diff --git a/lib/telemetry/index.ts b/lib/telemetry/index.ts index 9abd3e16f..19d15b1ea 100644 --- a/lib/telemetry/index.ts +++ b/lib/telemetry/index.ts @@ -1,5 +1,5 @@ export { startScheduler, stopScheduler, reschedule, sendOnce } from './sender'; -export { buildPayload, errorsForPayload, markProcessStart } from './payload'; +export { buildPayload, errorsForDay, errorsForPayload, markProcessStart } from './payload'; export { loadState, saveState, updateState, getInstanceId, effectiveConsent, effectiveErrorsConsent, } from './state'; @@ -10,6 +10,7 @@ export { export type { EndpointCheck } from './endpoint-guard'; export type { TelemetryPayload, TelemetryPayloadV2, AnyTelemetryPayload, TelemetryError, TelemetryStateFile, ConsentState, + ErrorsConsent, TelemetryErrorBatch, Platform, OsFamily, CountBucket, TelemetryFeatures, } from './types'; export { DEFAULT_ENDPOINT } from './types'; diff --git a/lib/telemetry/instant-errors.ts b/lib/telemetry/instant-errors.ts new file mode 100644 index 000000000..bfc248153 --- /dev/null +++ b/lib/telemetry/instant-errors.ts @@ -0,0 +1,123 @@ +import { logger } from '@/lib/logger'; +import { readErrors, utcDay } from '@/lib/error-reporting/aggregate'; +import { effectiveConsent, effectiveErrorsConsent, endpointEnabled, getInstanceId } from './state'; +import { errorsForDay } from './payload'; +import { fetchTelemetryTarget, resolveEndpointAllowed } from './endpoint-guard'; +import { DEFAULT_ENDPOINT, errorsEndpointFor, type TelemetryErrorBatch } from './types'; + +/** + * Instant error counts. With the error-count consent on "instant", the + * scheduler leader posts the current UTC day's counts to the collector's + * /v1/errors every TICK_MS. Only errors that are new or whose count bucket + * changed since the last send go out, so a known error repeating doesn't + * cause traffic until it reaches the next bucket. The daily heartbeat still + * carries the complete day as before. + * + * Each tick reads the consent, so switching it in the admin takes effect on + * the next tick without restarting anything. Other replicas' counts reach + * the store with their own flush (every minute), so they go out a minute later. + */ + +const TICK_MS = 60_000; + +interface InstantState { + timer: NodeJS.Timeout | null; + /** The day `sent` is for. */ + day: string; + /** `fingerprint|version` -> the count bucket last sent. */ + sent: Map; + running: Promise | null; +} + +// On globalThis, like the heartbeat timer: one per process, whichever +// module copy starts or stops it. +const KEY = Symbol.for('bulwark.telemetry.instant-errors'); +function state(): InstantState { + const g = globalThis as typeof globalThis & { [KEY]?: InstantState }; + return (g[KEY] ??= { timer: null, day: '', sent: new Map(), running: null }); +} + +async function tick(now: number): Promise { + const s = state(); + const { consent } = await effectiveErrorsConsent(); + if (consent !== 'instant') { + s.sent.clear(); + return; + } + const { state: stored } = await effectiveConsent(); + const endpoint = stored.endpoint ?? DEFAULT_ENDPOINT; + if (!endpointEnabled(endpoint)) return; + let target: string; + try { + target = errorsEndpointFor(endpoint); + } catch { + return; + } + + const day = utcDay(now); + if (s.day !== day) { + s.day = day; + s.sent.clear(); + } + const changed = errorsForDay(await readErrors(), day) + .filter((error) => s.sent.get(`${error.fingerprint}|${error.version}`) !== error.count); + if (changed.length === 0) return; + + // Checked at send time, like the heartbeat: defeats DNS rebinding and a + // state file edited past the admin API. + const guard = await resolveEndpointAllowed(target); + if (!guard.ok) { + logger.warn('telemetry: errors endpoint blocked', { reason: guard.reason }); + return; + } + + const batch: TelemetryErrorBatch = { schema: '1', instance_id: await getInstanceId(), day, errors: changed }; + const res = await fetchTelemetryTarget(target, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(batch), + signal: AbortSignal.timeout(5000), + }); + if (!res.ok) { + // Not marked as sent: the next tick tries them again. + logger.warn('telemetry: instant error counts rejected', { status: res.status }); + return; + } + for (const error of changed) s.sent.set(`${error.fingerprint}|${error.version}`, error.count); +} + +/** One tick: send what changed, if instant mode is on. Never throws. */ +export function sendInstantErrors(now = Date.now()): Promise { + const s = state(); + if (s.running) return s.running; + s.running = tick(now) + .catch((error) => { + logger.warn('telemetry: instant error counts failed', { error: error instanceof Error ? error.message : String(error) }); + }) + .finally(() => { s.running = null; }); + return s.running; +} + +// Run by the scheduler leader (lib/scheduler-leader.ts). Idempotent. +export async function startInstantErrors(): Promise { + const s = state(); + if (s.timer) return; + s.timer = setInterval(() => { void sendInstantErrors(); }, TICK_MS); + s.timer.unref?.(); +} + +export async function stopInstantErrors(): Promise { + const s = state(); + if (s.timer) clearInterval(s.timer); + s.timer = null; + // A new leader starts from nothing and sends the day's counts once. + s.sent.clear(); +} + +/** For tests. */ +export function resetInstantErrorsForTests(): void { + const s = state(); + if (s.timer) clearInterval(s.timer); + const g = globalThis as typeof globalThis & { [KEY]?: InstantState }; + delete g[KEY]; +} diff --git a/lib/telemetry/payload.ts b/lib/telemetry/payload.ts index fe4ecbce2..6c56c0e4f 100644 --- a/lib/telemetry/payload.ts +++ b/lib/telemetry/payload.ts @@ -115,8 +115,17 @@ const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; * and a message only when it is one of Bulwark's fixed error codes. */ export function errorsForPayload(agg: Aggregate, now = Date.now()): TelemetryError[] { - const day = utcDay(now - 86_400_000); - const recentDays = Array.from({ length: 7 }, (_, i) => utcDay(now - (i + 1) * 86_400_000)); + return errorsForDay(agg, utcDay(now - 86_400_000)); +} + +/** + * The error counts of one UTC day, shaped as in errorsForPayload. `days` + * counts the days of the week ending on `day` the error occurred on. + * Instant mode sends the current day's this way. + */ +export function errorsForDay(agg: Aggregate, day: string): TelemetryError[] { + const end = Date.parse(`${day}T00:00:00Z`); + const recentDays = Array.from({ length: 7 }, (_, i) => utcDay(end - i * 86_400_000)); return Object.entries(agg[day] ?? {}) .sort(([, a], [, b]) => b.count - a.count) .slice(0, MAX_TELEMETRY_ERRORS) @@ -145,7 +154,8 @@ export function errorsForPayload(agg: Aggregate, now = Date.now()): TelemetryErr */ export async function buildPayload(): Promise { const base = await buildPayloadV1(); - if ((await effectiveErrorsConsent()).consent !== 'on') return base; + // Daily and instant mode both send the last complete day with the heartbeat. + if ((await effectiveErrorsConsent()).consent === 'off') return base; return { ...base, schema: '2', errors: errorsForPayload(await readErrors()) }; } diff --git a/lib/telemetry/state.ts b/lib/telemetry/state.ts index 03e7d3548..ca9f8655b 100644 --- a/lib/telemetry/state.ts +++ b/lib/telemetry/state.ts @@ -3,7 +3,7 @@ import { randomUUID } from 'node:crypto'; import { logger } from '@/lib/logger'; import { getStore, type CollectionOptions } from '@/lib/shared-store'; import { isStoreUnavailable, StoredDataUnreadableError } from '@/lib/stored-data-errors'; -import type { TelemetryStateFile, ConsentState } from './types'; +import type { TelemetryStateFile, ConsentState, ErrorsConsent } from './types'; import { DEFAULT_ENDPOINT } from './types'; import { isNoThirdPartyTrafficOn } from '@/lib/admin/third-party-mode'; @@ -174,10 +174,11 @@ export async function saveStateIfUnchanged(state: TelemetryStateFile, version: s return (await telemetryDocs().put(STATE_DOC, serialize(state), { ifVersion: version })).ok; } -function errorsEnvOverride(): 'on' | 'off' | null { +function errorsEnvOverride(): ErrorsConsent | null { const v = (process.env.BULWARK_TELEMETRY_ERRORS ?? '').toLowerCase(); if (v === 'off' || v === 'false' || v === '0' || v === 'no') return 'off'; - if (v === 'on' || v === 'true' || v === '1' || v === 'yes') return 'on'; + if (v === 'on' || v === 'true' || v === '1' || v === 'yes' || v === 'daily') return 'on'; + if (v === 'instant') return 'instant'; return null; } @@ -185,9 +186,11 @@ function errorsEnvOverride(): 'on' | 'off' | null { * Effective consent to send error counts (schema v2): off whenever * telemetry itself is off (which covers the no-third-party mode), then * BULWARK_TELEMETRY_ERRORS, then the stored choice, off by default. + * 'on' sends them daily with the heartbeat; 'instant' does that too and + * also sends the current day's counts as they change. */ export async function effectiveErrorsConsent(): Promise<{ - consent: 'on' | 'off'; + consent: ErrorsConsent; source: 'telemetry' | 'no-third-party' | 'env' | 'file'; }> { const telemetry = await effectiveConsent(); @@ -195,7 +198,8 @@ export async function effectiveErrorsConsent(): Promise<{ if (telemetry.consent !== 'on') return { consent: 'off', source: 'telemetry' }; const env = errorsEnvOverride(); if (env) return { consent: env, source: 'env' }; - return { consent: telemetry.state.errorsConsent === 'on' ? 'on' : 'off', source: 'file' }; + const stored = telemetry.state.errorsConsent; + return { consent: stored === 'on' || stored === 'instant' ? stored : 'off', source: 'file' }; } // Effective consent: the no-third-party mode wins over everything (#903), diff --git a/lib/telemetry/types.ts b/lib/telemetry/types.ts index 3e1bc091d..21deb9eb4 100644 --- a/lib/telemetry/types.ts +++ b/lib/telemetry/types.ts @@ -73,6 +73,30 @@ export interface TelemetryPayloadV2 extends Omit { export type AnyTelemetryPayload = TelemetryPayload | TelemetryPayloadV2; +/** + * Error counts: off, daily (with the heartbeat, for the day before) or + * instant (daily, plus the current day's counts within about a minute). + * 'on' is the daily mode; the stored value predates the instant one. + */ +export type ErrorsConsent = 'on' | 'off' | 'instant'; + +/** + * What instant mode posts to the collector's /v1/errors: the current UTC + * day's errors that are new or whose count bucket changed since the last send. + */ +export interface TelemetryErrorBatch { + schema: '1'; + instance_id: string; + /** The UTC day the counts are for (YYYY-MM-DD). */ + day: string; + errors: TelemetryError[]; +} + +/** The collector's errors URL next to its heartbeat URL: `.../v1/heartbeat` -> `.../v1/errors`. */ +export function errorsEndpointFor(heartbeatEndpoint: string): string { + return new URL('errors', heartbeatEndpoint).toString(); +} + export interface TelemetryStateFile { consent: ConsentState; endpoint: string; @@ -85,10 +109,11 @@ export interface TelemetryStateFile { lastScheduledSentAt?: string | null; nextScheduledAt: string | null; /** - * The separate consent to add error counts (schema v2). Off unless an - * admin turns it on; switching telemetry off switches it off too. + * The separate consent to add error counts (schema v2), and whether they + * also go out instantly. Off unless an admin turns it on; switching + * telemetry off switches it off too. */ - errorsConsent?: 'on' | 'off'; + errorsConsent?: ErrorsConsent; } export const DEFAULT_ENDPOINT = 'https://telemetry.bulwarkmail.org/v1/heartbeat'; From b196455720b5139ae5318130722bf6d7bb0bc83e Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 13:25:01 +0100 Subject: [PATCH 25/26] test: give the plugin vacation API test the client methods the out-of-office calendar sync reads The out-of-office calendar block (package B) syncs from the vacation store, so the plugin vacation API now reaches client.getAccountId and supportsCalendars. The real client has both; the test's mock client didn't, so two cases threw. The mock gains them; no assertion changed. --- lib/__tests__/plugin-host-settings-apis.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/__tests__/plugin-host-settings-apis.test.ts b/lib/__tests__/plugin-host-settings-apis.test.ts index c22f93e6c..7cacda476 100644 --- a/lib/__tests__/plugin-host-settings-apis.test.ts +++ b/lib/__tests__/plugin-host-settings-apis.test.ts @@ -47,6 +47,10 @@ beforeEach(() => { subject: 'Away', textBody: 'Back in June', htmlBody: null, })), setVacationResponse: vi.fn(async () => undefined), + // The out-of-office calendar sync (lib/vacation-calendar.ts) reads the + // account and skips servers without calendars. + getAccountId: vi.fn(() => 'acc1'), + supportsCalendars: vi.fn(() => false), }; useAuthStore.setState({ client: client as unknown as IJMAPClient }); useTemplateStore.setState({ templates: [greeting], recentTemplateIds: [], deletedTemplateIds: {} }); From c32c78894af8fe53ea1138e5f3b224eb7422b6dd Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 13:33:42 +0100 Subject: [PATCH 26/26] fix: keep saving an out-of-office reply working when its calendar block can't be synced --- lib/vacation-calendar.ts | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/lib/vacation-calendar.ts b/lib/vacation-calendar.ts index cbbd44e19..1a142ac0e 100644 --- a/lib/vacation-calendar.ts +++ b/lib/vacation-calendar.ts @@ -345,6 +345,9 @@ function run(key: string): void { if (state.running.has(key) || !job) return; state.pending.delete(key); state.running.set(key, syncVacationBlock(job.client, job.accountId, job.reply, job.trigger, new Date(), job.previous) + // The calendar block is a side effect of the reply: a failed sync is + // logged and never surfaces as a failed save. + .catch((error) => { debug.error('Out-of-office calendar sync failed:', error); }) .finally(() => { state.running.delete(key); if (state.pending.has(key) && !state.timers.has(key)) run(key); @@ -367,21 +370,32 @@ function schedule(job: Pending): void { /** After a saved reply: sync when the switch, the dates or the subject changed. */ export function vacationReplySaved(client: SyncClient, accountId: string | undefined, before: ReplyState, after: ReplyState): void { if (relevant(before) === relevant(after)) return; - schedule({ client, accountId, reply: after, previous: { fromDate: before.fromDate, toDate: before.toDate }, trigger: 'save' }); + safely(() => schedule({ client, accountId, reply: after, previous: { fromDate: before.fromDate, toDate: before.toDate }, trigger: 'save' })); } /** The user changed the block setting: sync with the reply as it is. */ export function vacationBlockChanged(client: SyncClient, accountId: string | undefined, reply: ReplyState): void { - schedule({ client, accountId, reply, previous: { fromDate: reply.fromDate, toDate: reply.toDate }, trigger: 'save' }); + safely(() => schedule({ client, accountId, reply, previous: { fromDate: reply.fromDate, toDate: reply.toDate }, trigger: 'save' })); } /** After the reply was loaded: once per page load and account. */ export function vacationReplyLoaded(client: SyncClient, accountId: string | undefined, reply: ReplyState): void { - const key = accountId ?? client.getAccountId(); - const state = scheduler(); - if (state.loaded.has(key)) return; - state.loaded.add(key); - schedule({ client, accountId, reply, previous: null, trigger: 'load' }); + safely(() => { + const key = accountId ?? client.getAccountId(); + const state = scheduler(); + if (state.loaded.has(key)) return; + state.loaded.add(key); + schedule({ client, accountId, reply, previous: null, trigger: 'load' }); + }); +} + +/** Scheduling the calendar sync must never break saving or loading the reply. */ +function safely(fn: () => void): void { + try { + fn(); + } catch (error) { + debug.error('Out-of-office calendar sync not scheduled:', error); + } } /** For tests: drop what the scheduler holds. */