feat: serve Prometheus metrics and split health into live and ready

- `/api/metrics` in Prometheus text format, off until `METRICS_TOKEN`
  or `METRICS_TOKEN_FILE` is set (404 before, 401 without the bearer
  token): requests per route group with status class and latency,
  sign-in outcomes, calls to Stalwart, WOPI, translation, the plugin
  proxy and OAuth, browser errors, errors today, process memory,
  event-loop lag, uptime and build info. Labels come from fixed lists.
  Each replica serves its own.
- `/api/health/live` answers while the process runs;
  `/api/health/ready` checks the config, the shared store, writable
  data dirs and Stalwart's session resource. `/api/health` and
  `/api/ready` are unchanged.
- the Docker image gets a HEALTHCHECK on the live endpoint
- adds prom-client, and the OpenTelemetry packages the tracing step
  loads on demand

.dockerignore

@@ -10,6 +10,7 @@ lite-image
 scripts/
 !scripts/lite/
 !scripts/build-plugin-sandbox.mjs
+!scripts/healthcheck.mjs
 # Built in the image by scripts/build-plugin-sandbox.mjs
 public/plugin-sandbox-assets/
 TODO.md

Dockerfile

@@ -56,6 +56,7 @@ RUN apk upgrade --no-cache && \
 COPY --from=builder /app/public ./public
 COPY --from=builder /app/.next/standalone ./
 COPY --from=builder /app/.next/static ./.next/static
+COPY --from=builder /app/scripts/healthcheck.mjs ./healthcheck.mjs
 # Every data dir exists up front and belongs to the runtime user, so a fresh
 # volume on /app/data (Docker copies these into it) or on any one of them is
 # writable. plugin-secrets holds users' plugin credentials, version-check the
@@ -68,4 +69,9 @@ USER nextjs
 EXPOSE 3000
 ENV PORT=3000
 ENV HOSTNAME="0.0.0.0"
+# The health check needs the base path the bundle was built with.
+ARG NEXT_PUBLIC_BASE_PATH=
+ENV BULWARK_BASE_PATH=$NEXT_PUBLIC_BASE_PATH
+# Liveness only (/api/health/live); readiness is /api/health/ready.
+HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 CMD ["node", "healthcheck.mjs"]
 CMD ["node", "server.js"]

app/(main)/admin/_pages/monitoring.tsx

@@ -31,6 +31,7 @@ interface MonitoringState {
   collecting: boolean;
   day: string;
   errorsToday: { total: number; top: ErrorEntry[] };
+  metrics?: { enabled: boolean };
 }
 
 const PAGE = adminPageDef('monitoring');
@@ -161,6 +162,25 @@ export function MonitoringPage() {
             </div>
           )}
         </SettingsSection>
+
+        <SettingsSection id="metrics" title="Metrics">
+          <p className="text-sm">
+            {state?.metrics?.enabled
+              ? <>On. Prometheus scrapes <code className="font-mono">/api/metrics</code> on each replica with the token as a bearer token.</>
+              : <>Off. Set <code className="font-mono">METRICS_TOKEN</code> (or <code className="font-mono">METRICS_TOKEN_FILE</code>) in the environment to turn on <code className="font-mono">/api/metrics</code>.</>}
+          </p>
+          <ConfigNote>
+            Requests per route group, sign-ins, calls to Stalwart and other services, browser errors, memory and
+            event-loop lag. <a className="underline" href={`${DOCS}#metrics`} target="_blank" rel="noreferrer">Metric list and scrape config</a>
+          </ConfigNote>
+        </SettingsSection>
+
+        <SettingsSection id="health" title="Health checks">
+          <ul className="space-y-1 text-sm">
+            <li><code className="font-mono">/api/health/live</code>: the process answers. For liveness probes and the Docker health check.</li>
+            <li><code className="font-mono">/api/health/ready</code>: config loaded, data dirs writable, Stalwart reachable. For readiness probes.</li>
+          </ul>
+        </SettingsSection>
       </div>
 
       <AdminSaveBar

app/api/admin/monitoring/route.ts

@@ -4,13 +4,14 @@ import { logger } from '@/lib/logger';
 import { readErrors, summarizeDay, utcDay } from '@/lib/error-reporting/aggregate';
 import { shouldCollectErrors } from '@/lib/error-reporting/settings';
 import { storedDataUnavailableResponse } from '@/lib/stored-data-errors';
+import { metricsEnabled } from '@/lib/metrics';
 
 export const dynamic = 'force-dynamic';
 
 /**
  * GET /api/admin/monitoring: what System > Monitoring shows besides the
  * config keys (those come from /api/admin/config): whether errors are being
- * counted and today's counts.
+ * counted, today's counts and whether the metrics endpoint is on.
  */
 export async function GET(request: NextRequest) {
   try {
@@ -20,7 +21,8 @@ export async function GET(request: NextRequest) {
     const collecting = await shouldCollectErrors();
     const today = utcDay();
     const errorsToday = summarizeDay(await readErrors(), today);
-    return NextResponse.json({ collecting, day: today, errorsToday }, { headers: { 'Cache-Control': 'no-store' } });
+    const metrics = { enabled: metricsEnabled() };
+    return NextResponse.json({ collecting, day: today, errorsToday, metrics }, { headers: { 'Cache-Control': 'no-store' } });
   } catch (err) {
     logger.error('monitoring GET error', { error: err instanceof Error ? err.message : 'unknown' });
     const unavailable = storedDataUnavailableResponse(err);

app/api/auth/token/route.ts

@@ -1,4 +1,5 @@
 import { NextRequest, NextResponse } from 'next/server';
+import { observeUpstream } from '@/lib/metrics/upstream';
 import { cookies } from 'next/headers';
 import { logger } from '@/lib/logger';
 import {
@@ -57,7 +58,8 @@ function getSlot(request: NextRequest): number {
 const REVOCATION_TIMEOUT_MS = 3000;
 
 async function revokeRefreshToken(token: string, serverId: string | null, metadata: OAuthMetadata | null): Promise<void> {
-  if (!metadata?.revocation_endpoint) return;
+  const revocationEndpoint = metadata?.revocation_endpoint;
+  if (!revocationEndpoint) return;
   const params = buildOAuthParams({
     token,
     token_type_hint: 'refresh_token',
@@ -66,13 +68,13 @@ async function revokeRefreshToken(token: string, serverId: string | null, metada
   try {
     // Never follow a redirect with the refresh token and client secret in
     // the body: the endpoint was validated, its redirect target was not.
-    const revocationResponse = await fetch(metadata.revocation_endpoint, {
+    const revocationResponse = await observeUpstream('oauth', () => fetch(revocationEndpoint, {
       method: 'POST',
       headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
       body: params.toString(),
       signal: AbortSignal.timeout(REVOCATION_TIMEOUT_MS),
       redirect: 'error',
-    });
+    }));
     if (!revocationResponse.ok) {
       logger.warn('Token revocation returned error', { status: revocationResponse.status });
     }
@@ -262,13 +264,13 @@ export async function PUT(request: NextRequest) {
       refresh_token: refreshToken,
     }, serverId, await defaultClientOptions(serverId));
 
-    const tokenResponse = await fetch(tokenEndpoint, {
+    const tokenResponse = await observeUpstream('oauth', () => fetch(tokenEndpoint, {
       method: 'POST',
       headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
       body: params.toString(),
       // See revokeRefreshToken: the body holds the refresh token.
       redirect: 'error',
-    });
+    }));
 
     if (!tokenResponse.ok) {
       const errorText = await tokenResponse.text();

app/api/errors/__tests__/route.test.ts

@@ -15,6 +15,7 @@ vi.mock('@/lib/admin/config-manager', () => ({
 
 import { POST } from '../route';
 import { flushErrors, readErrors, resetAggregateForTests, utcDay } from '@/lib/error-reporting/aggregate';
+import { metrics, resetMetricsForTests } from '@/lib/metrics';
 
 const DSN = 'https://6f0b9c79ed3a4315b76595bc2536ac40@bugsink.example.org/3';
 const SUBJECT = 'Merger plans for Acme Q3';
@@ -71,6 +72,20 @@ describe('POST /api/errors', () => {
     expect(entry.frames).toEqual([{ path: '/_next/static/chunks/1.js', line: 2, column: 30, fn: 'Ab' }]);
   });
 
+  it('counts while only the metrics endpoint is on, for the browser errors metric', async () => {
+    process.env.METRICS_TOKEN = 'scrape-secret-0123456789';
+    resetMetricsForTests();
+    try {
+      expect((await POST(post(report))).status).toBe(204);
+      await flushErrors();
+      expect(Object.keys((await readErrors())[utcDay()] ?? {})).toHaveLength(1);
+      expect(await metrics().registry.metrics()).toContain('bulwark_browser_errors_total 1');
+    } finally {
+      delete process.env.METRICS_TOKEN;
+      resetMetricsForTests();
+    }
+  });
+
   it('ignores a name or code that is not an identifier or fixed code', async () => {
     config.errorReportingDsn = DSN;
     await POST(post({ ...report, name: SUBJECT, code: SUBJECT }));

app/api/errors/route.ts

@@ -6,6 +6,7 @@ import { appVersion, isErrorCode } from '@/lib/error-reporting/envelope';
 import { scrubName } from '@/lib/error-reporting/scrub';
 import { shouldCollectErrors } from '@/lib/error-reporting/settings';
 import { sanitizeFrames } from '@/lib/error-reporting/stack';
+import { countBrowserError } from '@/lib/metrics';
 
 export const dynamic = 'force-dynamic';
 
@@ -66,5 +67,6 @@ export async function POST(request: NextRequest) {
     // old one, and its frames point into that); the server's when absent.
     typeof body.version === 'string' && /^\d+\.\d+\.\d+(?:[-+][\w.-]{1,40})?$/.test(body.version) ? body.version : appVersion(),
   );
+  countBrowserError();
   return new NextResponse(null, { status: 204, headers: NO_STORE });
 }

app/api/health/live/route.ts

@@ -0,0 +1,18 @@
+import { NextResponse } from 'next/server';
+
+export const dynamic = 'force-dynamic';
+
+/**
+ * Liveness: 200 while the process answers. Nothing else is checked, so a
+ * Stalwart or store outage never makes an orchestrator restart Bulwark.
+ * Readiness is /api/health/ready. https://bulwarkmail.org/docs/admin/monitoring#health
+ */
+const HEADERS = { 'Cache-Control': 'no-store' };
+
+export function GET() {
+  return NextResponse.json({ status: 'ok' }, { headers: HEADERS });
+}
+
+export function HEAD() {
+  return new NextResponse(null, { status: 200, headers: HEADERS });
+}

app/api/health/ready/route.ts

@@ -0,0 +1,23 @@
+import { NextResponse } from 'next/server';
+import { readiness } from '@/lib/health/readiness';
+
+export const dynamic = 'force-dynamic';
+
+/**
+ * Readiness: 200 when this replica can serve, 503 otherwise. Checks that
+ * the config is loaded, the shared store (if any) answers, the data dirs are
+ * writable and Stalwart's session resource answers within 2 s. The body
+ * names the checks only. Results are reused for 5 s.
+ * https://bulwarkmail.org/docs/admin/monitoring#health
+ */
+const HEADERS = { 'Cache-Control': 'no-store' };
+
+export async function GET() {
+  const result = await readiness();
+  return NextResponse.json(result, { status: result.ready ? 200 : 503, headers: HEADERS });
+}
+
+export async function HEAD() {
+  const result = await readiness();
+  return new NextResponse(null, { status: result.ready ? 200 : 503, headers: HEADERS });
+}

app/api/metrics/__tests__/route.test.ts

@@ -0,0 +1,55 @@
+// @vitest-environment node
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { afterEach, describe, expect, it } from 'vitest';
+import { NextRequest } from 'next/server';
+import { GET } from '../route';
+import { resetMetricsForTests } from '@/lib/metrics';
+
+function scrape(authorization?: string): NextRequest {
+  return new NextRequest('http://localhost/api/metrics', { headers: authorization ? { authorization } : {} });
+}
+
+afterEach(() => {
+  delete process.env.METRICS_TOKEN;
+  delete process.env.METRICS_TOKEN_FILE;
+  resetMetricsForTests();
+});
+
+describe('GET /api/metrics', () => {
+  it('does not exist without a token configured', async () => {
+    const res = await GET(scrape('Bearer anything'));
+    expect(res.status).toBe(404);
+  });
+
+  it('wants the bearer token', async () => {
+    process.env.METRICS_TOKEN = 'scrape-secret-0123456789';
+    expect((await GET(scrape())).status).toBe(401);
+    expect((await GET(scrape('Bearer wrong'))).status).toBe(401);
+    expect((await GET(scrape('Basic scrape-secret-0123456789'))).status).toBe(401);
+  });
+
+  it('answers Prometheus text with the right token', async () => {
+    process.env.METRICS_TOKEN = 'scrape-secret-0123456789';
+    const res = await GET(scrape('Bearer scrape-secret-0123456789'));
+    expect(res.status).toBe(200);
+    expect(res.headers.get('content-type')).toContain('text/plain');
+    const body = await res.text();
+    expect(body).toContain('# TYPE bulwark_http_requests_total counter');
+    expect(body).toContain('bulwark_build_info');
+    expect(body).toContain('bulwark_nodejs_eventloop_lag_seconds');
+    expect(body).toContain('bulwark_signins_total{outcome="rate_limited"} 0');
+  });
+
+  it('reads the token from METRICS_TOKEN_FILE', async () => {
+    const dir = mkdtempSync(path.join(tmpdir(), 'bulwark-metrics-'));
+    try {
+      writeFileSync(path.join(dir, 'token'), 'from-a-file-0123456789\n');
+      process.env.METRICS_TOKEN_FILE = path.join(dir, 'token');
+      expect((await GET(scrape('Bearer from-a-file-0123456789'))).status).toBe(200);
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+});

app/api/metrics/route.ts

@@ -0,0 +1,41 @@
+import { timingSafeEqual } from 'node:crypto';
+import { NextRequest, NextResponse } from 'next/server';
+import { metrics, metricsToken } from '@/lib/metrics';
+
+export const dynamic = 'force-dynamic';
+
+/**
+ * GET /api/metrics: Prometheus metrics of this replica.
+ *
+ * Off by default: without METRICS_TOKEN (or METRICS_TOKEN_FILE) it answers
+ * 404 as if it didn't exist. With one, the scraper sends
+ * `Authorization: Bearer <token>`; anything else gets 401.
+ *
+ * https://bulwarkmail.org/docs/admin/monitoring#metrics
+ */
+
+function sameToken(given: string, expected: string): boolean {
+  const a = Buffer.from(given);
+  const b = Buffer.from(expected);
+  return a.length === b.length && timingSafeEqual(a, b);
+}
+
+export async function GET(request: NextRequest) {
+  const token = metricsToken();
+  if (!token) return new NextResponse('Not Found', { status: 404, headers: { 'Cache-Control': 'no-store' } });
+
+  const header = request.headers.get('authorization') ?? '';
+  const given = header.startsWith('Bearer ') ? header.slice('Bearer '.length).trim() : '';
+  if (!given || !sameToken(given, token)) {
+    return new NextResponse('Unauthorized', {
+      status: 401,
+      headers: { 'Cache-Control': 'no-store', 'WWW-Authenticate': 'Bearer realm="metrics"' },
+    });
+  }
+
+  const { registry } = metrics();
+  return new NextResponse(await registry.metrics(), {
+    status: 200,
+    headers: { 'Content-Type': registry.contentType, 'Cache-Control': 'no-store' },
+  });
+}

app/api/translate/route.ts

@@ -1,4 +1,5 @@
 import { NextRequest, NextResponse } from 'next/server';
+import { observeUpstream } from '@/lib/metrics/upstream';
 import { rejectCrossOriginRequest } from '@/lib/security/same-origin';
 import { readFileEnv } from "@/lib/read-file-env";
 import { getStalwartCredentials } from '@/lib/stalwart/credentials';
@@ -174,10 +175,10 @@ async function mymemoryRequest(
   url.searchParams.set('q', q);
   url.searchParams.set('langpair', langpair);
 
-  const res = await fetch(url.toString(), {
+  const res = await observeUpstream('translate', () => fetch(url.toString(), {
     signal,
     headers: { 'User-Agent': 'Bulwark-Webmail-Translate (+https://bulwarkmail.org)' },
-  });
+  }));
   const data = (await res.json().catch(() => null)) as
     | { responseStatus?: number | string; responseData?: { translatedText?: string }; responseDetails?: string }
     | null;
@@ -232,7 +233,7 @@ async function translateLibre(
     process.env.LIBRETRANSLATE_API_KEY ||
     readFileEnv(process.env.LIBRETRANSLATE_API_KEY_FILE);
   const url = endpoint.replace(/\/+$/, '') + '/translate';
-  const res = await fetch(url, {
+  const res = await observeUpstream('translate', () => fetch(url, {
     method: 'POST',
     signal,
     headers: { 'Content-Type': 'application/json' },
@@ -243,7 +244,7 @@ async function translateLibre(
       format: 'text',
       ...(apiKey ? { api_key: apiKey } : {}),
     }),
-  });
+  }));
   const data = (await res.json().catch(() => null)) as
     | { translatedText?: string; detectedLanguage?: { language?: string }; error?: string }
     | null;

instrumentation.node.ts

@@ -17,6 +17,14 @@ const pkg = JSON.parse(
 const current: string = pkg.version ?? "0.0.0";
 logger.info(`Bulwark Webmail v${current}`, { version: current });
 
+// Prometheus metrics, only with METRICS_TOKEN(_FILE): count every request
+// from the first one on. https://bulwarkmail.org/docs/admin/monitoring#metrics
+if (process.env.METRICS_TOKEN?.trim() || process.env.METRICS_TOKEN_FILE?.trim()) {
+  void import("./lib/metrics/http").then(({ installHttpMetrics }) => {
+    if (installHttpMetrics()) logger.info("Metrics endpoint on at /api/metrics");
+  });
+}
+
 // With REDIS_URL every replica shares one store (#1066). Without it,
 // everything below runs exactly as it always has.
 const shared = sharedStoreUrl() !== null;

lib/admin/admin-nav.ts

@@ -253,6 +253,8 @@ export const ADMIN_PAGES: readonly AdminPageDef[] = [
     sections: [
       { id: 'errors', title: 'Error reporting', keywords: 'dsn sentry bugsink glitchtip sample rate environment' },
       { id: 'errors-today', title: 'Errors today', keywords: 'count fingerprint top' },
+      { id: 'metrics', title: 'Metrics', keywords: 'prometheus scrape token grafana' },
+      { id: 'health', title: 'Health checks', keywords: 'liveness readiness probe kubernetes docker' },
     ],
   },
   {

lib/admin/outgoing-hosts-registry.ts

@@ -157,7 +157,7 @@ export const OUTGOING_HOSTS: readonly OutgoingHost[] = [
  * has to be looked at again.
  */
 export const NON_REQUEST_HOSTS: readonly { host: string; files: readonly string[]; why: string }[] = [
-  { host: 'bulwarkmail.org', why: 'docs links, comments and the contact URL in User-Agent headers', files: ['app/(main)/admin/_pages/telemetry.tsx', 'app/api/fetch-ical/route.ts', 'app/api/translate/route.ts','components/admin/org/sections/recovery-section.tsx', 'instrumentation.node.ts', 'lib/telemetry/state.ts', 'lib/telemetry/types.ts', 'app/(main)/admin/_pages/monitoring.tsx', 'lib/admin/types.ts', 'lib/error-reporting/client.ts', 'lib/error-reporting/lite.ts', 'lib/error-reporting/scrub.ts', 'instrumentation.ts'] },
+  { host: 'bulwarkmail.org', why: 'docs links, comments and the contact URL in User-Agent headers', files: ['app/(main)/admin/_pages/telemetry.tsx', 'app/api/fetch-ical/route.ts', 'app/api/translate/route.ts','components/admin/org/sections/recovery-section.tsx', 'instrumentation.node.ts', 'lib/telemetry/state.ts', 'lib/telemetry/types.ts', 'app/(main)/admin/_pages/monitoring.tsx', 'lib/admin/types.ts', 'lib/error-reporting/client.ts', 'lib/error-reporting/lite.ts', 'lib/error-reporting/scrub.ts', 'instrumentation.ts', 'app/api/health/live/route.ts', 'app/api/health/ready/route.ts', 'app/api/metrics/route.ts', 'lib/metrics/index.ts'] },
   { host: 'bulwarkmail.com', why: 'JMAP capability URI', files: ['lib/jmap/client.ts'] },
   { host: 'develop.sentry.dev', why: 'link to the envelope format in a comment', files: ['lib/error-reporting/envelope.ts'] },
   { host: 'extensions.bulwarkmail.org', why: 'signed statement type URIs', files: ['lib/admin/directory-keys.ts'] },

lib/error-reporting/settings.ts

@@ -1,5 +1,6 @@
 import { configManager } from '@/lib/admin/config-manager';
 import { parseDsn } from './envelope';
+import { metricsEnabled } from '@/lib/metrics';
 import type { ClientSettings } from './client';
 
 /**
@@ -43,7 +44,7 @@ export function errorReportingSettings(): ErrorReportingSettings {
  */
 export async function shouldCollectErrors(): Promise<boolean> {
   await configManager.ensureLoaded();
-  return !!errorReportingSettings().dsn;
+  return !!errorReportingSettings().dsn || metricsEnabled();
 }
 
 /** What /api/config hands the browser. */

lib/health/__tests__/readiness.test.ts

@@ -0,0 +1,86 @@
+// @vitest-environment node
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+const config: Record<string, unknown> = {};
+vi.mock('@/lib/admin/config-manager', () => ({
+  configManager: {
+    ensureLoaded: async () => {},
+    get: (key: string, fallback: unknown) => (key in config ? config[key] : fallback),
+  },
+}));
+
+import { readiness, resetReadinessCache } from '../readiness';
+import { GET as ready } from '@/app/api/health/ready/route';
+import { GET as live } from '@/app/api/health/live/route';
+
+let dir: string;
+let fetchSpy: ReturnType<typeof vi.fn>;
+
+beforeEach(() => {
+  dir = mkdtempSync(path.join(tmpdir(), 'bulwark-ready-'));
+  process.env.ADMIN_STATE_DIR = path.join(dir, 'state');
+  process.env.TELEMETRY_DATA_DIR = path.join(dir, 'telemetry');
+  for (const key of Object.keys(config)) delete config[key];
+  config.jmapServerUrl = 'https://mail.example.org';
+  resetReadinessCache();
+  fetchSpy = vi.fn().mockResolvedValue(new Response(null, { status: 401 }));
+  vi.stubGlobal('fetch', fetchSpy);
+});
+
+afterEach(() => {
+  vi.unstubAllGlobals();
+  delete process.env.ADMIN_STATE_DIR;
+  delete process.env.TELEMETRY_DATA_DIR;
+  rmSync(dir, { recursive: true, force: true });
+});
+
+describe('health endpoints', () => {
+  it('live answers 200 whatever Stalwart does', async () => {
+    fetchSpy.mockRejectedValue(new Error('down'));
+    expect((await live()).status).toBe(200);
+  });
+
+  it('ready answers 200 when Stalwart answers, even with 401', async () => {
+    const res = await ready();
+    expect(res.status).toBe(200);
+    expect(await res.json()).toEqual({ ready: true, checks: { config: 'ok', store: 'ok', dataDirs: 'ok', stalwart: 'ok' } });
+    expect(String(fetchSpy.mock.calls[0][0])).toBe('https://mail.example.org/.well-known/jmap');
+  });
+
+  it('ready answers 503 when Stalwart is unreachable or failing, and names no host', async () => {
+    fetchSpy.mockRejectedValue(new Error('ECONNREFUSED mail.example.org'));
+    const res = await ready();
+    expect(res.status).toBe(503);
+    const text = await res.text();
+    expect(JSON.parse(text).checks.stalwart).toBe('fail');
+    expect(text).not.toContain('example.org');
+
+    resetReadinessCache();
+    fetchSpy.mockResolvedValue(new Response(null, { status: 502 }));
+    expect((await ready()).status).toBe(503);
+  });
+
+  it('skips Stalwart before a server is configured', async () => {
+    delete config.jmapServerUrl;
+    delete process.env.JMAP_SERVER_URL;
+    expect((await readiness()).checks.stalwart).toBe('skipped');
+    expect(fetchSpy).not.toHaveBeenCalled();
+  });
+
+  it('fails when a data dir is not writable', async () => {
+    writeFileSync(path.join(dir, 'blocked'), '');
+    process.env.TELEMETRY_DATA_DIR = path.join(dir, 'blocked', 'telemetry');
+    const result = await readiness();
+    expect(result.checks.dataDirs).toBe('fail');
+    expect(result.ready).toBe(false);
+  });
+
+  it('reuses a result for a few seconds', async () => {
+    await readiness(1000);
+    await readiness(2000);
+    expect(fetchSpy).toHaveBeenCalledTimes(1);
+  });
+});

lib/health/readiness.ts

@@ -0,0 +1,131 @@
+import { randomBytes } from 'node:crypto';
+import { mkdir, unlink, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { configManager } from '@/lib/admin/config-manager';
+import { getStateDir } from '@/lib/admin/paths';
+import { observeUpstream } from '@/lib/metrics/upstream';
+import { bootState, getStore, isSharedMode } from '@/lib/shared-store';
+import { getSettingsDir } from '@/lib/settings-sync';
+import { getDir as getTelemetryDir } from '@/lib/telemetry/state';
+
+/**
+ * Whether this replica can serve, for /api/health/ready (and, for the
+ * shared store, /api/ready). Each check answers ok, fail or skipped; the
+ * answer names checks, never hosts or paths, because anyone may ask.
+ */
+
+export type CheckResult = 'ok' | 'fail' | 'skipped';
+
+export interface Readiness {
+  ready: boolean;
+  checks: {
+    config: CheckResult;
+    store: CheckResult;
+    dataDirs: CheckResult;
+    stalwart: CheckResult;
+  };
+}
+
+const STALWART_TIMEOUT_MS = 2000;
+const STORE_TIMEOUT_MS = 1000;
+const CACHE_MS = 5000;
+
+const KEY = Symbol.for('bulwark.health.readiness');
+interface CacheState { at: number; result: Readiness | null; pending: Promise<Readiness> | null }
+function cache(): CacheState {
+  const g = globalThis as typeof globalThis & { [KEY]?: CacheState };
+  return (g[KEY] ??= { at: 0, result: null, pending: null });
+}
+
+async function checkConfig(): Promise<CheckResult> {
+  try {
+    await configManager.ensureLoaded();
+    return 'ok';
+  } catch {
+    return 'fail';
+  }
+}
+
+/** Booted, and with a shared store, the store answered a ping. */
+export async function checkStore(): Promise<CheckResult> {
+  if (bootState() !== 'ready') return 'fail';
+  const store = getStore();
+  if (store.mode === 'local') return 'ok';
+  try {
+    await store.ping(STORE_TIMEOUT_MS);
+    return 'ok';
+  } catch {
+    return 'fail';
+  }
+}
+
+async function writable(dir: string): Promise<boolean> {
+  const probe = path.join(dir, `.ready-${process.pid}-${randomBytes(4).toString('hex')}`);
+  try {
+    await mkdir(dir, { recursive: true });
+    await writeFile(probe, '');
+    await unlink(probe);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/** The dirs this process writes to in local mode; a shared store holds that data instead. */
+async function checkDataDirs(): Promise<CheckResult> {
+  if (isSharedMode()) return 'skipped';
+  const dirs = [getStateDir(), getTelemetryDir()];
+  if (configManager.get<boolean>('settingsSyncEnabled', false)) dirs.push(getSettingsDir());
+  const results = await Promise.all(dirs.map(writable));
+  return results.every(Boolean) ? 'ok' : 'fail';
+}
+
+/** Stalwart's session resource answers, whatever the status below 500 (401 is expected without credentials). */
+async function checkStalwart(): Promise<CheckResult> {
+  const base = configManager.get<string>('jmapServerUrl', '') || process.env.JMAP_SERVER_URL || '';
+  if (!base.trim()) return 'skipped';
+  let url: URL;
+  try {
+    url = new URL('/.well-known/jmap', base.trim());
+  } catch {
+    return 'fail';
+  }
+  try {
+    const response = await observeUpstream('stalwart', () => fetch(url, {
+      method: 'GET',
+      redirect: 'manual',
+      signal: AbortSignal.timeout(STALWART_TIMEOUT_MS),
+    }));
+    await response.body?.cancel().catch(() => undefined);
+    return response.status < 500 ? 'ok' : 'fail';
+  } catch {
+    return 'fail';
+  }
+}
+
+async function evaluate(): Promise<Readiness> {
+  const config = await checkConfig();
+  const [store, dataDirs, stalwart] = await Promise.all([checkStore(), checkDataDirs(), checkStalwart()]);
+  const checks = { config, store, dataDirs, stalwart };
+  return { ready: Object.values(checks).every((c) => c !== 'fail'), checks };
+}
+
+/** The readiness of this replica, reused for CACHE_MS so probes can't flood Stalwart or the store. */
+export async function readiness(now = Date.now()): Promise<Readiness> {
+  const c = cache();
+  if (c.result && now - c.at < CACHE_MS) return c.result;
+  c.pending ??= evaluate()
+    .then((result) => {
+      c.result = result;
+      c.at = Date.now();
+      return result;
+    })
+    .finally(() => { c.pending = null; });
+  return c.pending;
+}
+
+/** For tests. */
+export function resetReadinessCache(): void {
+  const g = globalThis as typeof globalThis & { [KEY]?: CacheState };
+  delete g[KEY];
+}

lib/metrics/__tests__/metrics.test.ts

@@ -0,0 +1,93 @@
+// @vitest-environment node
+import http from 'node:http';
+import type { AddressInfo } from 'node:net';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { metrics, observeRequest, resetMetricsForTests, signinOutcome } from '../index';
+import { observeUpstream } from '../upstream';
+import { installHttpMetrics } from '../http';
+
+async function value(name: string, labels: Record<string, string> = {}): Promise<number | undefined> {
+  const metric = (await metrics().registry.getMetricsAsJSON()).find((m) => m.name === name);
+  const found = (metric?.values as Array<{ labels: Record<string, string | number>; value: number }> | undefined)
+    ?.find((v) => Object.entries(labels).every(([k, want]) => v.labels[k] === want));
+  return found?.value;
+}
+
+beforeEach(() => {
+  process.env.METRICS_TOKEN = 'test-token-1234567890';
+  resetMetricsForTests();
+});
+
+afterEach(() => {
+  delete process.env.METRICS_TOKEN;
+  resetMetricsForTests();
+  vi.restoreAllMocks();
+});
+
+describe('request metrics', () => {
+  it('label requests by route group and status class, never by the raw path', async () => {
+    observeRequest('/de/mail/folder/Acquisition%20Talks/Mabc?q=secret', 'GET', 200, 0.05);
+    observeRequest('/webmail/api/wopi/files/abc/contents', 'POST', 503, 1.2, '/webmail');
+    observeRequest('/api/whatever-a-user-typed', 'BREW', 404, 0.01);
+    expect(await value('bulwark_http_requests_total', { group: 'page:mail', method: 'GET', status: '2xx' })).toBe(1);
+    expect(await value('bulwark_http_requests_total', { group: 'api:wopi', method: 'POST', status: '5xx' })).toBe(1);
+    expect(await value('bulwark_http_requests_total', { group: 'api:other', method: 'other', status: '4xx' })).toBe(1);
+    const text = await metrics().registry.metrics();
+    expect(text).not.toContain('Acquisition');
+    expect(text).not.toContain('whatever-a-user-typed');
+    expect(text).not.toContain('secret');
+  });
+
+  it('count sign-ins by outcome from the auth routes', async () => {
+    observeRequest('/api/auth/session', 'POST', 200, 0.1);
+    observeRequest('/api/auth/session', 'POST', 401, 0.1);
+    observeRequest('/api/auth/session', 'POST', 429, 0.1);
+    observeRequest('/api/auth/token', 'PUT', 401, 0.1);
+    observeRequest('/api/auth/token', 'PUT', 200, 0.1);
+    observeRequest('/api/auth/session', 'GET', 200, 0.1);
+    for (const [outcome, n] of [['success', 1], ['failure', 1], ['rate_limited', 1], ['refresh_failure', 1]] as const) {
+      expect(await value('bulwark_signins_total', { outcome }), outcome).toBe(n);
+    }
+  });
+
+  it('map outcomes only for sign-in and refresh calls', () => {
+    expect(signinOutcome('api:auth', '/api/auth/totp-token-exchange', 'POST', 200)).toBe('success');
+    expect(signinOutcome('api:auth', '/api/auth/session', 'DELETE', 200)).toBeNull();
+    expect(signinOutcome('api:admin', '/api/admin/auth', 'POST', 401)).toBeNull();
+  });
+
+  it('do nothing while the endpoint is off', async () => {
+    delete process.env.METRICS_TOKEN;
+    observeRequest('/api/config', 'GET', 200, 0.01);
+    process.env.METRICS_TOKEN = 'test-token-1234567890';
+    expect(await value('bulwark_http_requests_total', { group: 'api:config' })).toBeUndefined();
+  });
+
+  it('come from Node\'s HTTP server itself once installed', async () => {
+    expect(installHttpMetrics('')).toBe(true);
+    const server = http.createServer((_req, res) => { res.statusCode = 204; res.end(); });
+    await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
+    try {
+      const { port } = server.address() as AddressInfo;
+      await fetch(`http://127.0.0.1:${port}/api/health/live`);
+      await vi.waitFor(async () => expect(await value('bulwark_http_requests_total', { group: 'api:health', status: '2xx' })).toBe(1));
+    } finally {
+      server.close();
+    }
+  });
+});
+
+describe('upstream metrics', () => {
+  it('count answers below 500 as ok and 5xx or errors as error', async () => {
+    await observeUpstream('stalwart', async () => new Response(null, { status: 401 }));
+    await observeUpstream('stalwart', async () => new Response(null, { status: 502 }));
+    await expect(observeUpstream('wopi', async () => { throw new Error('down'); })).rejects.toThrow('down');
+    expect(await value('bulwark_upstream_requests_total', { target: 'stalwart', outcome: 'ok' })).toBe(1);
+    expect(await value('bulwark_upstream_requests_total', { target: 'stalwart', outcome: 'error' })).toBe(1);
+    expect(await value('bulwark_upstream_requests_total', { target: 'wopi', outcome: 'error' })).toBe(1);
+  });
+
+  it('report build info', async () => {
+    expect(await metrics().registry.metrics()).toMatch(/bulwark_build_info\{version="[^"]+",build="[^"]+"\} 1/);
+  });
+});

lib/metrics/__tests__/upstream-wiring.test.ts

@@ -0,0 +1,29 @@
+// @vitest-environment node
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+vi.mock('@/lib/admin/config-manager', () => ({
+  configManager: { ensureLoaded: async () => {}, get: (_key: string, fallback: unknown) => fallback },
+}));
+
+import { fetchJmapServer } from '@/lib/stalwart/server-fetch';
+import { metrics, resetMetricsForTests } from '@/lib/metrics';
+
+beforeEach(() => {
+  process.env.METRICS_TOKEN = 'scrape-secret-0123456789';
+  resetMetricsForTests();
+});
+
+afterEach(() => {
+  delete process.env.METRICS_TOKEN;
+  resetMetricsForTests();
+  vi.unstubAllGlobals();
+});
+
+// The server's calls to Stalwart all go through fetchJmapServer.
+describe('Stalwart calls', () => {
+  it('are counted under the stalwart target', async () => {
+    vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('{}', { status: 200 })));
+    await fetchJmapServer('https://mail.example.org/jmap/session', {}, true);
+    expect(await metrics().registry.metrics()).toContain('bulwark_upstream_requests_total{target="stalwart",outcome="ok"} 1');
+  });
+});

lib/metrics/http.ts

@@ -0,0 +1,39 @@
+import diagnostics from 'node:diagnostics_channel';
+import type { IncomingMessage, ServerResponse } from 'node:http';
+import { metricsEnabled, observeRequest } from './index';
+
+/**
+ * Counts every request this process serves, through Node's own HTTP
+ * diagnostics channels: nothing is wrapped or patched, and the proxy's
+ * matcher doesn't matter. Installed once at startup when the metrics
+ * endpoint is on (instrumentation.node.ts).
+ */
+
+interface HttpState {
+  installed: boolean;
+  started: WeakMap<IncomingMessage, number>;
+}
+
+const KEY = Symbol.for('bulwark.metrics.http');
+function state(): HttpState {
+  const g = globalThis as typeof globalThis & { [KEY]?: HttpState };
+  return (g[KEY] ??= { installed: false, started: new WeakMap() });
+}
+
+export function installHttpMetrics(basePath = process.env.NEXT_PUBLIC_BASE_PATH || ''): boolean {
+  const s = state();
+  if (s.installed || !metricsEnabled()) return false;
+  s.installed = true;
+  diagnostics.subscribe('http.server.request.start', (message) => {
+    const { request } = message as { request: IncomingMessage };
+    s.started.set(request, performance.now());
+  });
+  diagnostics.subscribe('http.server.response.finish', (message) => {
+    const { request, response } = message as { request: IncomingMessage; response: ServerResponse };
+    const start = s.started.get(request);
+    if (start === undefined) return;
+    s.started.delete(request);
+    observeRequest(request.url ?? '/', request.method, response.statusCode, (performance.now() - start) / 1000, basePath);
+  });
+  return true;
+}

lib/metrics/index.ts

@@ -0,0 +1,147 @@
+import { readFileSync } from 'node:fs';
+import type { Counter, Gauge, Histogram, Registry } from 'prom-client';
+import client from 'prom-client';
+import { routeGroup } from '@/lib/route-groups';
+
+/**
+ * Prometheus metrics for /api/metrics, off unless METRICS_TOKEN (or
+ * METRICS_TOKEN_FILE) is set. Every replica counts for itself; Prometheus
+ * scrapes each one. Labels come from fixed lists only (route groups, sign-in
+ * outcomes, upstream targets), never from user names or hosts users type.
+ *
+ * https://bulwarkmail.org/docs/admin/monitoring#metrics
+ */
+
+export const SIGNIN_OUTCOMES = ['success', 'failure', 'refresh_failure', 'rate_limited'] as const;
+export type SigninOutcome = typeof SIGNIN_OUTCOMES[number];
+
+export const UPSTREAM_TARGETS = ['stalwart', 'wopi', 'translate', 'plugin_proxy', 'oauth'] as const;
+export type UpstreamTarget = typeof UPSTREAM_TARGETS[number];
+
+const METHODS = new Set(['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS']);
+const DURATION_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30];
+
+/** The bearer token /api/metrics wants, or null when the endpoint is off. */
+export function metricsToken(): string | null {
+  const direct = process.env.METRICS_TOKEN?.trim();
+  if (direct) return direct;
+  const file = process.env.METRICS_TOKEN_FILE?.trim();
+  if (!file) return null;
+  try {
+    return readFileSync(file, 'utf8').trim() || null;
+  } catch {
+    return null;
+  }
+}
+
+export function metricsEnabled(): boolean {
+  return metricsToken() !== null;
+}
+
+interface Metrics {
+  registry: Registry;
+  httpRequests: Counter<'group' | 'method' | 'status'>;
+  httpDuration: Histogram<'group' | 'method'>;
+  signins: Counter<'outcome'>;
+  upstreamRequests: Counter<'target' | 'outcome'>;
+  upstreamDuration: Histogram<'target'>;
+  browserErrors: Counter;
+  errorsToday: Gauge;
+}
+
+const KEY = Symbol.for('bulwark.metrics');
+
+/** The registry and metrics, made once per process whatever bundle asks. */
+export function metrics(): Metrics {
+  const g = globalThis as typeof globalThis & { [KEY]?: Metrics };
+  if (g[KEY]) return g[KEY];
+  const registry = new client.Registry();
+  client.collectDefaultMetrics({ register: registry, prefix: 'bulwark_' });
+  const version = process.env.NEXT_PUBLIC_APP_VERSION || '0.0.0';
+  const build = process.env.NEXT_PUBLIC_GIT_COMMIT || process.env.BULWARK_BUILD || 'unknown';
+  new client.Gauge({ name: 'bulwark_build_info', help: 'Bulwark version and build; always 1', labelNames: ['version', 'build'], registers: [registry] })
+    .set({ version, build }, 1);
+  new client.Gauge({
+    name: 'bulwark_uptime_seconds',
+    help: 'Seconds since this process started',
+    registers: [registry],
+    collect() { this.set(process.uptime()); },
+  });
+  const created: Metrics = {
+    registry,
+    httpRequests: new client.Counter({ name: 'bulwark_http_requests_total', help: 'HTTP requests by route group, method and status class', labelNames: ['group', 'method', 'status'], registers: [registry] }),
+    httpDuration: new client.Histogram({ name: 'bulwark_http_request_duration_seconds', help: 'HTTP request duration by route group and method', labelNames: ['group', 'method'], buckets: DURATION_BUCKETS, registers: [registry] }),
+    signins: new client.Counter({ name: 'bulwark_signins_total', help: 'Sign-ins handled by this server, by outcome', labelNames: ['outcome'], registers: [registry] }),
+    upstreamRequests: new client.Counter({ name: 'bulwark_upstream_requests_total', help: 'Requests this server made to other services, by target and outcome', labelNames: ['target', 'outcome'], registers: [registry] }),
+    upstreamDuration: new client.Histogram({ name: 'bulwark_upstream_request_duration_seconds', help: 'Duration of requests to other services, by target', labelNames: ['target'], buckets: DURATION_BUCKETS, registers: [registry] }),
+    browserErrors: new client.Counter({ name: 'bulwark_browser_errors_total', help: 'Errors browsers reported to this server', registers: [registry] }),
+    errorsToday: new client.Gauge({
+      name: 'bulwark_errors_today',
+      help: 'Errors counted today (UTC) by the whole deployment, browsers and servers',
+      registers: [registry],
+      async collect() {
+        const { readErrors, summarizeDay } = await import('@/lib/error-reporting/aggregate');
+        this.set(summarizeDay(await readErrors()).total);
+      },
+    }),
+  };
+  // Pre-create the label sets, so a series exists at 0 before its first event.
+  for (const outcome of SIGNIN_OUTCOMES) created.signins.inc({ outcome }, 0);
+  for (const target of UPSTREAM_TARGETS) {
+    for (const outcome of ['ok', 'error']) created.upstreamRequests.inc({ target, outcome }, 0);
+  }
+  g[KEY] = created;
+  return created;
+}
+
+export function methodLabel(method: string | undefined): string {
+  const upper = (method ?? '').toUpperCase();
+  return METHODS.has(upper) ? upper : 'other';
+}
+
+export function statusClass(status: number): string {
+  return status >= 100 && status < 600 ? `${Math.floor(status / 100)}xx` : 'other';
+}
+
+/** What a finished request means for sign-ins, from its route, method and status. */
+export function signinOutcome(group: string, path: string, method: string, status: number): SigninOutcome | null {
+  if (group !== 'api:auth') return null;
+  const route = path.split(/[?#]/, 1)[0].replace(/\/+$/, '');
+  const signIn = method === 'POST' && /\/api\/auth\/(?:session|stalwart-context|totp-token-exchange|token)$/.test(route);
+  const refresh = method === 'PUT' && /\/api\/auth\/token$/.test(route);
+  if (!signIn && !refresh) return null;
+  if (status === 429) return 'rate_limited';
+  if (status >= 200 && status < 300) return refresh ? null : 'success';
+  return refresh ? 'refresh_failure' : 'failure';
+}
+
+/** Count a finished HTTP request (lib/metrics/http.ts). */
+export function observeRequest(path: string, method: string | undefined, status: number, seconds: number, basePath = ''): void {
+  if (!metricsEnabled()) return;
+  const m = metrics();
+  const group = routeGroup(path, basePath);
+  const verb = methodLabel(method);
+  m.httpRequests.inc({ group, method: verb, status: statusClass(status) });
+  m.httpDuration.observe({ group, method: verb }, seconds);
+  const outcome = signinOutcome(group, basePath && path.startsWith(basePath) ? path.slice(basePath.length) : path, verb, status);
+  if (outcome) m.signins.inc({ outcome });
+}
+
+export function observeUpstreamResult(target: UpstreamTarget, ok: boolean, seconds: number): void {
+  if (!metricsEnabled()) return;
+  const m = metrics();
+  m.upstreamRequests.inc({ target, outcome: ok ? 'ok' : 'error' });
+  m.upstreamDuration.observe({ target }, seconds);
+}
+
+export function countBrowserError(): void {
+  if (!metricsEnabled()) return;
+  metrics().browserErrors.inc();
+}
+
+/** For tests. */
+export function resetMetricsForTests(): void {
+  const g = globalThis as typeof globalThis & { [KEY]?: Metrics };
+  g[KEY]?.registry.clear();
+  delete g[KEY];
+}

lib/metrics/upstream.ts

@@ -0,0 +1,19 @@
+import { observeUpstreamResult, type UpstreamTarget } from './index';
+
+/**
+ * Run one request this server makes to another service and record its
+ * duration and outcome under a fixed target name. An answer below 500 counts
+ * as ok (a 401 from Stalwart is an answer); a 5xx or a thrown error as
+ * error. Costs nothing while metrics are off.
+ */
+export async function observeUpstream<T extends { status: number }>(target: UpstreamTarget, run: () => Promise<T>): Promise<T> {
+  const start = performance.now();
+  try {
+    const response = await run();
+    observeUpstreamResult(target, response.status < 500, (performance.now() - start) / 1000);
+    return response;
+  } catch (error) {
+    observeUpstreamResult(target, false, (performance.now() - start) / 1000);
+    throw error;
+  }
+}

lib/oauth/token-exchange.ts

@@ -1,4 +1,5 @@
 import { logger } from '@/lib/logger';
+import { observeUpstream } from '@/lib/metrics/upstream';
 import { discoverOAuth } from '@/lib/oauth/discovery';
 import type { EndpointValidator, OAuthMetadata } from '@/lib/oauth/discovery';
 import { isPublicHttpUrl } from '@/lib/security/url-guard';
@@ -209,12 +210,12 @@ export async function exchangeCodeForTokens(
   // The body carries the code, the PKCE verifier and the client secret. The
   // endpoint was validated; a redirect would re-send all of it to a host
   // that was not, so none is followed.
-  const tokenResponse = await fetch(tokenEndpoint, {
+  const tokenResponse = await observeUpstream('oauth', () => fetch(tokenEndpoint, {
     method: 'POST',
     headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
     body: params.toString(),
     redirect: 'error',
-  });
+  }));
 
   if (!tokenResponse.ok) {
     const errorText = await tokenResponse.text();

lib/plugin-secrets/proxy.ts

@@ -9,6 +9,7 @@ import { getPlugin } from '@/lib/admin/plugin-registry';
 import { getPluginConfig } from '@/lib/admin/plugin-config';
 import { logger } from '@/lib/logger';
 import { DisallowedUrlError, fetchPublicUrl } from '@/lib/security/url-guard';
+import { observeUpstream } from '@/lib/metrics/upstream';
 import { authHeaderFor, credentialOrigin, privateProxyOrigins, type CredentialInject, type CredentialSpec } from './credentials';
 import { getUserCredential, setUserCredentialSecret, userKey, type VaultUser } from './vault';
 import { declaredUserCredential } from './known-plugins';
@@ -239,8 +240,10 @@ async function send(
     signal: init.signal,
     ...(init.body ? { body: init.body } : {}),
   };
-  if (trusted) return fetch(url, { ...request, redirect: 'manual' } as RequestInit);
-  return (await fetchPublicUrl(url, request as never)) as unknown as Response;
+  return observeUpstream('plugin_proxy', async () => {
+    if (trusted) return fetch(url, { ...request, redirect: 'manual' } as RequestInit);
+    return (await fetchPublicUrl(url, request as never)) as unknown as Response;
+  });
 }
 
 function responseHeaders(upstream: Response): Headers {

lib/route-groups.ts

@@ -11,7 +11,7 @@
 /** The folders under app/api. */
 export const API_GROUPS: readonly string[] = [
   'account', 'account-archive', 'admin', 'auth', 'bimi', 'caldav', 'calendar-agenda', 'config',
-  'connector', 'dev-jmap', 'errors', 'favicon', 'fetch-ical', 'health', 'og-image',
+  'connector', 'dev-jmap', 'errors', 'favicon', 'fetch-ical', 'health', 'metrics', 'og-image',
   'plugin-approval-status', 'plugin-proxy', 'plugin-secrets', 'plugin-signing-pubkey', 'plugins',
   'push', 'pwa-icon', 'pwa-screenshot', 'ready', 'settings', 'setup', 'system', 'translate',
   'webdav', 'wopi',

lib/stalwart/server-fetch.ts

@@ -19,6 +19,7 @@ import { configManager } from '@/lib/admin/config-manager';
 import { logger } from '@/lib/logger';
 import { parseJmapServers, resolveTrustedJmapUrl } from '@/lib/admin/jmap-servers';
 import { fetchPublicUrl, type PublicFetchResponse } from '@/lib/security/url-guard';
+import { observeUpstream } from '@/lib/metrics/upstream';
 import {
   isStoredCredentialRefused,
   recordStoredCredentialStatus,
@@ -72,7 +73,7 @@ export async function fetchJmapServer(
 ): Promise<Response> {
   const authHeader = new Headers(givenInit.headers).get('authorization');
   if (authHeader && (await isStoredCredentialRefused(authHeader))) return refusedCredentialResponse();
-  const response = await fetchUnbudgeted(url, givenInit, trusted);
+  const response = await observeUpstream('stalwart', () => fetchUnbudgeted(url, givenInit, trusted));
   // Not awaited: this process's copy is updated at once, the shared write follows.
   if (authHeader) {
     void recordStoredCredentialStatus(authHeader, response.status).catch((error) => {

lib/wopi/discovery.ts

@@ -1,4 +1,5 @@
 import { configManager } from '@/lib/admin/config-manager';
+import { observeUpstream } from '@/lib/metrics/upstream';
 import { logger } from '@/lib/logger';
 
 /**
@@ -123,10 +124,10 @@ export async function getWopiActions(): Promise<WopiActions | null> {
 
 async function fetchWopiActions(discoveryUrl: string): Promise<WopiActions | null> {
   try {
-    const res = await fetch(discoveryUrl, {
+    const res = await observeUpstream('wopi', () => fetch(discoveryUrl, {
       headers: { Accept: 'text/xml' },
       signal: AbortSignal.timeout(DISCOVERY_TIMEOUT_MS),
-    });
+    }));
     if (!res.ok) {
       logger.warn('WOPI discovery fetch failed', { discoveryUrl, status: res.status });
       return null;

next.config.ts

@@ -61,9 +61,9 @@ const nextConfig: NextConfig = {
   // esbuild ships native binaries + a README the bundler can't parse; load
   // it from node_modules at runtime instead of trying to bundle it. Used by
   // PLUGIN_DEV_DIR's on-the-fly bundler. ioredis (the shared store, #1066)
-  // is loaded from node_modules too, so the proxy and the route bundles
-  // don't each carry a copy.
-  serverExternalPackages: ["esbuild", "ioredis"],
+  // and prom-client (/api/metrics) are loaded from node_modules too, so the
+  // proxy and the route bundles don't each carry a copy.
+  serverExternalPackages: ["esbuild", "ioredis", "prom-client"],
   // The build's type check leaves out tests, which are a third of the files
   // and push tsc past the default heap on a 4 GB machine (#1431).
   // `npm run typecheck` still checks them with tsconfig.json.

package-lock.json

@@ -14,6 +14,13 @@
         "@dnd-kit/utilities": "^3.2.2",
         "@noble/curves": "^2.4.0",
         "@noble/hashes": "^2.4.0",
+        "@opentelemetry/api": "1.9.1",
+        "@opentelemetry/core": "2.8.0",
+        "@opentelemetry/exporter-trace-otlp-http": "0.219.0",
+        "@opentelemetry/resources": "2.8.0",
+        "@opentelemetry/sdk-trace-base": "2.8.0",
+        "@opentelemetry/sdk-trace-node": "2.8.0",
+        "@opentelemetry/semantic-conventions": "1.44.0",
         "@tabler/icons-react": "^3.50.0",
         "@tanstack/react-virtual": "^3.14.23",
         "@tiptap/extension-color": "^3.32.1",
@@ -43,6 +50,7 @@
         "otpauth": "^9.5.2",
         "pdfjs-dist": "^6.4.299",
         "postal-mime": "^4.1.4",
+        "prom-client": "15.1.3",
         "qrcode": "^1.5.4",
         "react": "^19.3.1",
         "react-dom": "^19.3.1",
@@ -2389,6 +2397,204 @@
         "url": "https://paulmillr.com/funding/"
       }
     },
+    "node_modules/@opentelemetry/api": {
+      "version": "1.9.1",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz",
+      "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": ">=8.0.0"
+      }
+    },
+    "node_modules/@opentelemetry/api-logs": {
+      "version": "0.219.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.219.0.tgz",
+      "integrity": "sha512-FFx7YnaYJlIjqWW/AG/yAZ0L/NEY724PipXXXQLdtZPbLwBGbUMTGL1i/esI56TWfTUXxhLfpgrnWJCG8aUJyg==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@opentelemetry/api": "^1.3.0"
+      },
+      "engines": {
+        "node": ">=8.0.0"
+      }
+    },
+    "node_modules/@opentelemetry/context-async-hooks": {
+      "version": "2.8.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-2.8.0.tgz",
+      "integrity": "sha512-/3FIraneMcng67SUJCxvyInk/oxzwsxyadufk0wwfOBLf5wqtAGX4MoQASwSbndBPeARzBryUM9Azr5kHIdWLw==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": "^18.19.0 || >=20.6.0"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": ">=1.0.0 <1.10.0"
+      }
+    },
+    "node_modules/@opentelemetry/core": {
+      "version": "2.8.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.8.0.tgz",
+      "integrity": "sha512-hd1Lfh8p545nNz+jq1Ejfz+Mn1hyLuxYn1YzTfFNrxr8urEWMNQLPf1Th8kjOH+HxwawCrtgBp8JpBUR4ZSgww==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@opentelemetry/semantic-conventions": "^1.29.0"
+      },
+      "engines": {
+        "node": "^18.19.0 || >=20.6.0"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": ">=1.0.0 <1.10.0"
+      }
+    },
+    "node_modules/@opentelemetry/exporter-trace-otlp-http": {
+      "version": "0.219.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-http/-/exporter-trace-otlp-http-0.219.0.tgz",
+      "integrity": "sha512-9t6SvBXXBEjOBcIzgozvBbd3jWrv3Gt3ngGhl1fhdZ/zRc7oZDVOFEqbi2zlBpW9BXhgDMKv422J0DL/3iQWfw==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@opentelemetry/core": "2.8.0",
+        "@opentelemetry/otlp-exporter-base": "0.219.0",
+        "@opentelemetry/otlp-transformer": "0.219.0",
+        "@opentelemetry/resources": "2.8.0",
+        "@opentelemetry/sdk-trace-base": "2.8.0"
+      },
+      "engines": {
+        "node": "^18.19.0 || >=20.6.0"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": "^1.3.0"
+      }
+    },
+    "node_modules/@opentelemetry/otlp-exporter-base": {
+      "version": "0.219.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.219.0.tgz",
+      "integrity": "sha512-zvIxQX/AZUVKDU+hCuYx+7UkiP7GRdnk1ZbFQRYzHvYp47cAWR4j3IhoPhV9KaeXEv2xdGq3IA6PnpzDmLcmSA==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@opentelemetry/core": "2.8.0",
+        "@opentelemetry/otlp-transformer": "0.219.0"
+      },
+      "engines": {
+        "node": "^18.19.0 || >=20.6.0"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": "^1.3.0"
+      }
+    },
+    "node_modules/@opentelemetry/otlp-transformer": {
+      "version": "0.219.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.219.0.tgz",
+      "integrity": "sha512-aaYKAyXhw9VchKZVGOopD3Gw/kPsyrX2c6IQ0AW32mTjqmZOh5Y6Gf5OYqTNqVktAeBjmFinhyFaCwW6GYK9YQ==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@opentelemetry/api-logs": "0.219.0",
+        "@opentelemetry/core": "2.8.0",
+        "@opentelemetry/resources": "2.8.0",
+        "@opentelemetry/sdk-logs": "0.219.0",
+        "@opentelemetry/sdk-metrics": "2.8.0",
+        "@opentelemetry/sdk-trace-base": "2.8.0"
+      },
+      "engines": {
+        "node": "^18.19.0 || >=20.6.0"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": "^1.3.0"
+      }
+    },
+    "node_modules/@opentelemetry/resources": {
+      "version": "2.8.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.8.0.tgz",
+      "integrity": "sha512-qmXQ27ilDbUK/vGMqwL8D4/rhn76C+sherM4wTbjlfknR8Nvfc/hCxjRJPhkzZzUsPiNg16SA31NxMabwttRjg==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@opentelemetry/core": "2.8.0",
+        "@opentelemetry/semantic-conventions": "^1.29.0"
+      },
+      "engines": {
+        "node": "^18.19.0 || >=20.6.0"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": ">=1.3.0 <1.10.0"
+      }
+    },
+    "node_modules/@opentelemetry/sdk-logs": {
+      "version": "0.219.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.219.0.tgz",
+      "integrity": "sha512-s6lTKRakaPClvKoWHRChxnXjDMkM/TQ30ff78jN6EBGf7MI7VzANE5PU3f4z9qDUudWjvZjOLHG0rBnBKYvoXA==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@opentelemetry/api-logs": "0.219.0",
+        "@opentelemetry/core": "2.8.0",
+        "@opentelemetry/resources": "2.8.0",
+        "@opentelemetry/semantic-conventions": "^1.29.0"
+      },
+      "engines": {
+        "node": "^18.19.0 || >=20.6.0"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": ">=1.4.0 <1.10.0"
+      }
+    },
+    "node_modules/@opentelemetry/sdk-metrics": {
+      "version": "2.8.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.8.0.tgz",
+      "integrity": "sha512-UDBGaj6W0Rgy5rTTaoxs8gVGF/aGkAKyjurJv7se6wjRxJu7FoquTLT/vt54DZfo4crbprYfhX/SOK9+BPw1qg==",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@opentelemetry/core": "2.8.0",
+        "@opentelemetry/resources": "2.8.0"
+      },
+      "engines": {
+        "node": "^18.19.0 || >=20.6.0"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": ">=1.9.0 <1.10.0"
+      }
+    },
+    "node_modules/@opentelemetry/sdk-trace-base": {
+      "version": "2.8.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.8.0.tgz",
+      "integrity": "sha512-mhU4jp+vW0mGbFRd+GeXHvmfA4aDqWjBjLC3pE5XMpLs0IE2ryYb019Ts2AQrOq67gaTF25D91+fgvEHDZEnuQ==",
+      "deprecated": "Use @opentelemetry/sdk-trace package",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@opentelemetry/core": "2.8.0",
+        "@opentelemetry/resources": "2.8.0",
+        "@opentelemetry/semantic-conventions": "^1.29.0"
+      },
+      "engines": {
+        "node": "^18.19.0 || >=20.6.0"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": ">=1.3.0 <1.10.0"
+      }
+    },
+    "node_modules/@opentelemetry/sdk-trace-node": {
+      "version": "2.8.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-node/-/sdk-trace-node-2.8.0.tgz",
+      "integrity": "sha512-nZt9OGufioAc3AfoLTqA9bsAeaMJAictYDdI2VcNQ+PmT+3rfKjAZDZvgPfd8VPX0O5Bw1hdQF6kDK8VSpZiWg==",
+      "deprecated": "Use @opentelemetry/sdk-trace package",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@opentelemetry/context-async-hooks": "2.8.0",
+        "@opentelemetry/core": "2.8.0",
+        "@opentelemetry/sdk-trace-base": "2.8.0"
+      },
+      "engines": {
+        "node": "^18.19.0 || >=20.6.0"
+      },
+      "peerDependencies": {
+        "@opentelemetry/api": ">=1.0.0 <1.10.0"
+      }
+    },
+    "node_modules/@opentelemetry/semantic-conventions": {
+      "version": "1.44.0",
+      "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.44.0.tgz",
+      "integrity": "sha512-E3JkOlMBi3N8OGGkbl7tFyqk2MAS6/lySRZBK3Mu7AzjF1fT8oIChH9ChE48SUWzDn9Kdwta+G/p5LJ4xW4CLQ==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": ">=14"
+      }
+    },
     "node_modules/@oxc-project/types": {
       "version": "0.155.0",
       "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.155.0.tgz",
@@ -4943,6 +5149,12 @@
         "require-from-string": "^2.0.2"
       }
     },
+    "node_modules/bintrees": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz",
+      "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==",
+      "license": "MIT"
+    },
     "node_modules/brace-expansion": {
       "version": "5.0.12",
       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz",
@@ -8375,6 +8587,20 @@
       "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==",
       "license": "MIT"
     },
+    "node_modules/prom-client": {
+      "version": "15.1.3",
+      "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.3.tgz",
+      "integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==",
+      "deprecated": "prom-client has been replaced by @prometheus-io/client",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "@opentelemetry/api": "^1.4.0",
+        "tdigest": "^0.1.1"
+      },
+      "engines": {
+        "node": "^16 || ^18 || >=20"
+      }
+    },
     "node_modules/prop-types": {
       "version": "15.8.1",
       "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz",
@@ -9355,6 +9581,15 @@
         "url": "https://opencollective.com/webpack"
       }
     },
+    "node_modules/tdigest": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.3.tgz",
+      "integrity": "sha512-zbRt+lT+/H4fRItHshczHErVCQnitJk8MfMT24MqFJf3YL7SJJPqGIGeuOdvxXxM/AHFzKBl7WoyaYwqO9s3Kw==",
+      "license": "MIT",
+      "dependencies": {
+        "bintrees": "1.0.2"
+      }
+    },
     "node_modules/tinybench": {
       "version": "6.2.2",
       "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-6.2.2.tgz",

package.json

@@ -44,6 +44,13 @@
     "@dnd-kit/utilities": "^3.2.2",
     "@noble/curves": "^2.4.0",
     "@noble/hashes": "^2.4.0",
+    "@opentelemetry/api": "1.9.1",
+    "@opentelemetry/core": "2.8.0",
+    "@opentelemetry/exporter-trace-otlp-http": "0.219.0",
+    "@opentelemetry/resources": "2.8.0",
+    "@opentelemetry/sdk-trace-base": "2.8.0",
+    "@opentelemetry/sdk-trace-node": "2.8.0",
+    "@opentelemetry/semantic-conventions": "1.44.0",
     "@tabler/icons-react": "^3.50.0",
     "@tanstack/react-virtual": "^3.14.23",
     "@tiptap/extension-color": "^3.32.1",
@@ -73,6 +80,7 @@
     "otpauth": "^9.5.2",
     "pdfjs-dist": "^6.4.299",
     "postal-mime": "^4.1.4",
+    "prom-client": "15.1.3",
     "qrcode": "^1.5.4",
     "react": "^19.3.1",
     "react-dom": "^19.3.1",

proxy.ts

@@ -179,6 +179,8 @@ export function mailFrameAncestors(allowed: string | undefined): string {
   return sources.join(" ");
 }
 
+const PROBE_PATHS = new Set(["/api/health", "/api/health/live", "/api/health/ready", "/api/ready", "/api/metrics"]);
+
 function isSetupPath(pathname: string): boolean {
   return (
     pathname === "/setup" ||
@@ -189,9 +191,10 @@ function isSetupPath(pathname: string): boolean {
 
 export async function proxy(request: NextRequest) {
   // The probes answer without config or store, so a replica that is still
-  // starting (or lost its shared store) can say so (#1066).
+  // starting (or lost its shared store) can say so (#1066). The metrics
+  // endpoint, too: a scrape shouldn't fail while a replica boots.
   const probePath = routePathOf(request.nextUrl.pathname);
-  if (probePath === "/api/health" || probePath === "/api/ready") {
+  if (PROBE_PATHS.has(probePath)) {
     const probe = NextResponse.next();
     probe.headers.set("X-Content-Type-Options", "nosniff");
     probe.headers.set("X-Frame-Options", "DENY");

scripts/healthcheck.mjs

@@ -0,0 +1,15 @@
+// Docker HEALTHCHECK: exit 0 when /api/health/live answers 200 within 4 s.
+// Liveness only, so a Stalwart outage never marks the container unhealthy.
+// The base path is baked in at build time (BULWARK_BASE_PATH in the image).
+// https://bulwarkmail.org/docs/admin/monitoring#health
+const port = process.env.PORT || '3000';
+const listen = process.env.HOSTNAME;
+const host = !listen || listen === '0.0.0.0' || listen === '::' ? '127.0.0.1' : listen.includes(':') ? `[${listen}]` : listen;
+const base = (process.env.BULWARK_BASE_PATH || '').replace(/\/+$/, '');
+
+try {
+  const res = await fetch(`http://${host}:${port}${base}/api/health/live`, { signal: AbortSignal.timeout(4000) });
+  process.exit(res.ok ? 0 : 1);
+} catch {
+  process.exit(1);
+}