diff --git a/.dockerignore b/.dockerignore index 68da62e5b..0942bf880 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,6 +10,7 @@ lite-image scripts/ !scripts/lite/ !scripts/build-plugin-sandbox.mjs +!scripts/healthcheck.mjs # Built in the image by scripts/build-plugin-sandbox.mjs public/plugin-sandbox-assets/ TODO.md diff --git a/Dockerfile b/Dockerfile index 515c30937..fe9f6f986 100644 --- a/Dockerfile +++ b/Dockerfile @@ -56,6 +56,7 @@ RUN apk upgrade --no-cache && \ COPY --from=builder /app/public ./public COPY --from=builder /app/.next/standalone ./ COPY --from=builder /app/.next/static ./.next/static +COPY --from=builder /app/scripts/healthcheck.mjs ./healthcheck.mjs # Every data dir exists up front and belongs to the runtime user, so a fresh # volume on /app/data (Docker copies these into it) or on any one of them is # writable. plugin-secrets holds users' plugin credentials, version-check the @@ -68,4 +69,9 @@ USER nextjs EXPOSE 3000 ENV PORT=3000 ENV HOSTNAME="0.0.0.0" +# The health check needs the base path the bundle was built with. +ARG NEXT_PUBLIC_BASE_PATH= +ENV BULWARK_BASE_PATH=$NEXT_PUBLIC_BASE_PATH +# Liveness only (/api/health/live); readiness is /api/health/ready. +HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 CMD ["node", "healthcheck.mjs"] CMD ["node", "server.js"] diff --git a/app/(main)/admin/_pages/monitoring.tsx b/app/(main)/admin/_pages/monitoring.tsx index 268740b69..24c4f8044 100644 --- a/app/(main)/admin/_pages/monitoring.tsx +++ b/app/(main)/admin/_pages/monitoring.tsx @@ -31,6 +31,7 @@ interface MonitoringState { collecting: boolean; day: string; errorsToday: { total: number; top: ErrorEntry[] }; + metrics?: { enabled: boolean }; } const PAGE = adminPageDef('monitoring'); @@ -161,6 +162,25 @@ export function MonitoringPage() { )} + + +

+ {state?.metrics?.enabled + ? <>On. Prometheus scrapes /api/metrics on each replica with the token as a bearer token. + : <>Off. Set METRICS_TOKEN (or METRICS_TOKEN_FILE) in the environment to turn on /api/metrics.} +

+ + Requests per route group, sign-ins, calls to Stalwart and other services, browser errors, memory and + event-loop lag. Metric list and scrape config + + + + + + Monitoring shows besides the * config keys (those come from /api/admin/config): whether errors are being - * counted and today's counts. + * counted, today's counts and whether the metrics endpoint is on. */ export async function GET(request: NextRequest) { try { @@ -20,7 +21,8 @@ export async function GET(request: NextRequest) { const collecting = await shouldCollectErrors(); const today = utcDay(); const errorsToday = summarizeDay(await readErrors(), today); - return NextResponse.json({ collecting, day: today, errorsToday }, { headers: { 'Cache-Control': 'no-store' } }); + const metrics = { enabled: metricsEnabled() }; + return NextResponse.json({ collecting, day: today, errorsToday, metrics }, { headers: { 'Cache-Control': 'no-store' } }); } catch (err) { logger.error('monitoring GET error', { error: err instanceof Error ? err.message : 'unknown' }); const unavailable = storedDataUnavailableResponse(err); diff --git a/app/api/auth/token/route.ts b/app/api/auth/token/route.ts index 389599afa..6c065e524 100644 --- a/app/api/auth/token/route.ts +++ b/app/api/auth/token/route.ts @@ -1,4 +1,5 @@ import { NextRequest, NextResponse } from 'next/server'; +import { observeUpstream } from '@/lib/metrics/upstream'; import { cookies } from 'next/headers'; import { logger } from '@/lib/logger'; import { @@ -57,7 +58,8 @@ function getSlot(request: NextRequest): number { const REVOCATION_TIMEOUT_MS = 3000; async function revokeRefreshToken(token: string, serverId: string | null, metadata: OAuthMetadata | null): Promise { - if (!metadata?.revocation_endpoint) return; + const revocationEndpoint = metadata?.revocation_endpoint; + if (!revocationEndpoint) return; const params = buildOAuthParams({ token, token_type_hint: 'refresh_token', @@ -66,13 +68,13 @@ async function revokeRefreshToken(token: string, serverId: string | null, metada try { // Never follow a redirect with the refresh token and client secret in // the body: the endpoint was validated, its redirect target was not. - const revocationResponse = await fetch(metadata.revocation_endpoint, { + const revocationResponse = await observeUpstream('oauth', () => fetch(revocationEndpoint, { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: params.toString(), signal: AbortSignal.timeout(REVOCATION_TIMEOUT_MS), redirect: 'error', - }); + })); if (!revocationResponse.ok) { logger.warn('Token revocation returned error', { status: revocationResponse.status }); } @@ -262,13 +264,13 @@ export async function PUT(request: NextRequest) { refresh_token: refreshToken, }, serverId, await defaultClientOptions(serverId)); - const tokenResponse = await fetch(tokenEndpoint, { + const tokenResponse = await observeUpstream('oauth', () => fetch(tokenEndpoint, { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: params.toString(), // See revokeRefreshToken: the body holds the refresh token. redirect: 'error', - }); + })); if (!tokenResponse.ok) { const errorText = await tokenResponse.text(); diff --git a/app/api/errors/__tests__/route.test.ts b/app/api/errors/__tests__/route.test.ts index b6bddfa02..d53e1dde6 100644 --- a/app/api/errors/__tests__/route.test.ts +++ b/app/api/errors/__tests__/route.test.ts @@ -15,6 +15,7 @@ vi.mock('@/lib/admin/config-manager', () => ({ import { POST } from '../route'; import { flushErrors, readErrors, resetAggregateForTests, utcDay } from '@/lib/error-reporting/aggregate'; +import { metrics, resetMetricsForTests } from '@/lib/metrics'; const DSN = 'https://6f0b9c79ed3a4315b76595bc2536ac40@bugsink.example.org/3'; const SUBJECT = 'Merger plans for Acme Q3'; @@ -71,6 +72,20 @@ describe('POST /api/errors', () => { expect(entry.frames).toEqual([{ path: '/_next/static/chunks/1.js', line: 2, column: 30, fn: 'Ab' }]); }); + it('counts while only the metrics endpoint is on, for the browser errors metric', async () => { + process.env.METRICS_TOKEN = 'scrape-secret-0123456789'; + resetMetricsForTests(); + try { + expect((await POST(post(report))).status).toBe(204); + await flushErrors(); + expect(Object.keys((await readErrors())[utcDay()] ?? {})).toHaveLength(1); + expect(await metrics().registry.metrics()).toContain('bulwark_browser_errors_total 1'); + } finally { + delete process.env.METRICS_TOKEN; + resetMetricsForTests(); + } + }); + it('ignores a name or code that is not an identifier or fixed code', async () => { config.errorReportingDsn = DSN; await POST(post({ ...report, name: SUBJECT, code: SUBJECT })); diff --git a/app/api/errors/route.ts b/app/api/errors/route.ts index 7e283f260..b31a38952 100644 --- a/app/api/errors/route.ts +++ b/app/api/errors/route.ts @@ -6,6 +6,7 @@ import { appVersion, isErrorCode } from '@/lib/error-reporting/envelope'; import { scrubName } from '@/lib/error-reporting/scrub'; import { shouldCollectErrors } from '@/lib/error-reporting/settings'; import { sanitizeFrames } from '@/lib/error-reporting/stack'; +import { countBrowserError } from '@/lib/metrics'; export const dynamic = 'force-dynamic'; @@ -66,5 +67,6 @@ export async function POST(request: NextRequest) { // old one, and its frames point into that); the server's when absent. typeof body.version === 'string' && /^\d+\.\d+\.\d+(?:[-+][\w.-]{1,40})?$/.test(body.version) ? body.version : appVersion(), ); + countBrowserError(); return new NextResponse(null, { status: 204, headers: NO_STORE }); } diff --git a/app/api/health/live/route.ts b/app/api/health/live/route.ts new file mode 100644 index 000000000..b5ac3a999 --- /dev/null +++ b/app/api/health/live/route.ts @@ -0,0 +1,18 @@ +import { NextResponse } from 'next/server'; + +export const dynamic = 'force-dynamic'; + +/** + * Liveness: 200 while the process answers. Nothing else is checked, so a + * Stalwart or store outage never makes an orchestrator restart Bulwark. + * Readiness is /api/health/ready. https://bulwarkmail.org/docs/admin/monitoring#health + */ +const HEADERS = { 'Cache-Control': 'no-store' }; + +export function GET() { + return NextResponse.json({ status: 'ok' }, { headers: HEADERS }); +} + +export function HEAD() { + return new NextResponse(null, { status: 200, headers: HEADERS }); +} diff --git a/app/api/health/ready/route.ts b/app/api/health/ready/route.ts new file mode 100644 index 000000000..fe562e724 --- /dev/null +++ b/app/api/health/ready/route.ts @@ -0,0 +1,23 @@ +import { NextResponse } from 'next/server'; +import { readiness } from '@/lib/health/readiness'; + +export const dynamic = 'force-dynamic'; + +/** + * Readiness: 200 when this replica can serve, 503 otherwise. Checks that + * the config is loaded, the shared store (if any) answers, the data dirs are + * writable and Stalwart's session resource answers within 2 s. The body + * names the checks only. Results are reused for 5 s. + * https://bulwarkmail.org/docs/admin/monitoring#health + */ +const HEADERS = { 'Cache-Control': 'no-store' }; + +export async function GET() { + const result = await readiness(); + return NextResponse.json(result, { status: result.ready ? 200 : 503, headers: HEADERS }); +} + +export async function HEAD() { + const result = await readiness(); + return new NextResponse(null, { status: result.ready ? 200 : 503, headers: HEADERS }); +} diff --git a/app/api/metrics/__tests__/route.test.ts b/app/api/metrics/__tests__/route.test.ts new file mode 100644 index 000000000..1fa22e8d3 --- /dev/null +++ b/app/api/metrics/__tests__/route.test.ts @@ -0,0 +1,55 @@ +// @vitest-environment node +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { NextRequest } from 'next/server'; +import { GET } from '../route'; +import { resetMetricsForTests } from '@/lib/metrics'; + +function scrape(authorization?: string): NextRequest { + return new NextRequest('http://localhost/api/metrics', { headers: authorization ? { authorization } : {} }); +} + +afterEach(() => { + delete process.env.METRICS_TOKEN; + delete process.env.METRICS_TOKEN_FILE; + resetMetricsForTests(); +}); + +describe('GET /api/metrics', () => { + it('does not exist without a token configured', async () => { + const res = await GET(scrape('Bearer anything')); + expect(res.status).toBe(404); + }); + + it('wants the bearer token', async () => { + process.env.METRICS_TOKEN = 'scrape-secret-0123456789'; + expect((await GET(scrape())).status).toBe(401); + expect((await GET(scrape('Bearer wrong'))).status).toBe(401); + expect((await GET(scrape('Basic scrape-secret-0123456789'))).status).toBe(401); + }); + + it('answers Prometheus text with the right token', async () => { + process.env.METRICS_TOKEN = 'scrape-secret-0123456789'; + const res = await GET(scrape('Bearer scrape-secret-0123456789')); + expect(res.status).toBe(200); + expect(res.headers.get('content-type')).toContain('text/plain'); + const body = await res.text(); + expect(body).toContain('# TYPE bulwark_http_requests_total counter'); + expect(body).toContain('bulwark_build_info'); + expect(body).toContain('bulwark_nodejs_eventloop_lag_seconds'); + expect(body).toContain('bulwark_signins_total{outcome="rate_limited"} 0'); + }); + + it('reads the token from METRICS_TOKEN_FILE', async () => { + const dir = mkdtempSync(path.join(tmpdir(), 'bulwark-metrics-')); + try { + writeFileSync(path.join(dir, 'token'), 'from-a-file-0123456789\n'); + process.env.METRICS_TOKEN_FILE = path.join(dir, 'token'); + expect((await GET(scrape('Bearer from-a-file-0123456789'))).status).toBe(200); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/app/api/metrics/route.ts b/app/api/metrics/route.ts new file mode 100644 index 000000000..16c8a98a5 --- /dev/null +++ b/app/api/metrics/route.ts @@ -0,0 +1,41 @@ +import { timingSafeEqual } from 'node:crypto'; +import { NextRequest, NextResponse } from 'next/server'; +import { metrics, metricsToken } from '@/lib/metrics'; + +export const dynamic = 'force-dynamic'; + +/** + * GET /api/metrics: Prometheus metrics of this replica. + * + * Off by default: without METRICS_TOKEN (or METRICS_TOKEN_FILE) it answers + * 404 as if it didn't exist. With one, the scraper sends + * `Authorization: Bearer `; anything else gets 401. + * + * https://bulwarkmail.org/docs/admin/monitoring#metrics + */ + +function sameToken(given: string, expected: string): boolean { + const a = Buffer.from(given); + const b = Buffer.from(expected); + return a.length === b.length && timingSafeEqual(a, b); +} + +export async function GET(request: NextRequest) { + const token = metricsToken(); + if (!token) return new NextResponse('Not Found', { status: 404, headers: { 'Cache-Control': 'no-store' } }); + + const header = request.headers.get('authorization') ?? ''; + const given = header.startsWith('Bearer ') ? header.slice('Bearer '.length).trim() : ''; + if (!given || !sameToken(given, token)) { + return new NextResponse('Unauthorized', { + status: 401, + headers: { 'Cache-Control': 'no-store', 'WWW-Authenticate': 'Bearer realm="metrics"' }, + }); + } + + const { registry } = metrics(); + return new NextResponse(await registry.metrics(), { + status: 200, + headers: { 'Content-Type': registry.contentType, 'Cache-Control': 'no-store' }, + }); +} diff --git a/app/api/translate/route.ts b/app/api/translate/route.ts index c53e0bcb0..232ee0703 100644 --- a/app/api/translate/route.ts +++ b/app/api/translate/route.ts @@ -1,4 +1,5 @@ import { NextRequest, NextResponse } from 'next/server'; +import { observeUpstream } from '@/lib/metrics/upstream'; import { rejectCrossOriginRequest } from '@/lib/security/same-origin'; import { readFileEnv } from "@/lib/read-file-env"; import { getStalwartCredentials } from '@/lib/stalwart/credentials'; @@ -174,10 +175,10 @@ async function mymemoryRequest( url.searchParams.set('q', q); url.searchParams.set('langpair', langpair); - const res = await fetch(url.toString(), { + const res = await observeUpstream('translate', () => fetch(url.toString(), { signal, headers: { 'User-Agent': 'Bulwark-Webmail-Translate (+https://bulwarkmail.org)' }, - }); + })); const data = (await res.json().catch(() => null)) as | { responseStatus?: number | string; responseData?: { translatedText?: string }; responseDetails?: string } | null; @@ -232,7 +233,7 @@ async function translateLibre( process.env.LIBRETRANSLATE_API_KEY || readFileEnv(process.env.LIBRETRANSLATE_API_KEY_FILE); const url = endpoint.replace(/\/+$/, '') + '/translate'; - const res = await fetch(url, { + const res = await observeUpstream('translate', () => fetch(url, { method: 'POST', signal, headers: { 'Content-Type': 'application/json' }, @@ -243,7 +244,7 @@ async function translateLibre( format: 'text', ...(apiKey ? { api_key: apiKey } : {}), }), - }); + })); const data = (await res.json().catch(() => null)) as | { translatedText?: string; detectedLanguage?: { language?: string }; error?: string } | null; diff --git a/instrumentation.node.ts b/instrumentation.node.ts index 94f50b00b..0d0f91907 100644 --- a/instrumentation.node.ts +++ b/instrumentation.node.ts @@ -17,6 +17,14 @@ const pkg = JSON.parse( const current: string = pkg.version ?? "0.0.0"; logger.info(`Bulwark Webmail v${current}`, { version: current }); +// Prometheus metrics, only with METRICS_TOKEN(_FILE): count every request +// from the first one on. https://bulwarkmail.org/docs/admin/monitoring#metrics +if (process.env.METRICS_TOKEN?.trim() || process.env.METRICS_TOKEN_FILE?.trim()) { + void import("./lib/metrics/http").then(({ installHttpMetrics }) => { + if (installHttpMetrics()) logger.info("Metrics endpoint on at /api/metrics"); + }); +} + // With REDIS_URL every replica shares one store (#1066). Without it, // everything below runs exactly as it always has. const shared = sharedStoreUrl() !== null; diff --git a/lib/admin/admin-nav.ts b/lib/admin/admin-nav.ts index 8d3d938b5..6352d28bb 100644 --- a/lib/admin/admin-nav.ts +++ b/lib/admin/admin-nav.ts @@ -253,6 +253,8 @@ export const ADMIN_PAGES: readonly AdminPageDef[] = [ sections: [ { id: 'errors', title: 'Error reporting', keywords: 'dsn sentry bugsink glitchtip sample rate environment' }, { id: 'errors-today', title: 'Errors today', keywords: 'count fingerprint top' }, + { id: 'metrics', title: 'Metrics', keywords: 'prometheus scrape token grafana' }, + { id: 'health', title: 'Health checks', keywords: 'liveness readiness probe kubernetes docker' }, ], }, { diff --git a/lib/admin/outgoing-hosts-registry.ts b/lib/admin/outgoing-hosts-registry.ts index 178ae3401..d32438294 100644 --- a/lib/admin/outgoing-hosts-registry.ts +++ b/lib/admin/outgoing-hosts-registry.ts @@ -157,7 +157,7 @@ export const OUTGOING_HOSTS: readonly OutgoingHost[] = [ * has to be looked at again. */ export const NON_REQUEST_HOSTS: readonly { host: string; files: readonly string[]; why: string }[] = [ - { host: 'bulwarkmail.org', why: 'docs links, comments and the contact URL in User-Agent headers', files: ['app/(main)/admin/_pages/telemetry.tsx', 'app/api/fetch-ical/route.ts', 'app/api/translate/route.ts','components/admin/org/sections/recovery-section.tsx', 'instrumentation.node.ts', 'lib/telemetry/state.ts', 'lib/telemetry/types.ts', 'app/(main)/admin/_pages/monitoring.tsx', 'lib/admin/types.ts', 'lib/error-reporting/client.ts', 'lib/error-reporting/lite.ts', 'lib/error-reporting/scrub.ts', 'instrumentation.ts'] }, + { host: 'bulwarkmail.org', why: 'docs links, comments and the contact URL in User-Agent headers', files: ['app/(main)/admin/_pages/telemetry.tsx', 'app/api/fetch-ical/route.ts', 'app/api/translate/route.ts','components/admin/org/sections/recovery-section.tsx', 'instrumentation.node.ts', 'lib/telemetry/state.ts', 'lib/telemetry/types.ts', 'app/(main)/admin/_pages/monitoring.tsx', 'lib/admin/types.ts', 'lib/error-reporting/client.ts', 'lib/error-reporting/lite.ts', 'lib/error-reporting/scrub.ts', 'instrumentation.ts', 'app/api/health/live/route.ts', 'app/api/health/ready/route.ts', 'app/api/metrics/route.ts', 'lib/metrics/index.ts'] }, { host: 'bulwarkmail.com', why: 'JMAP capability URI', files: ['lib/jmap/client.ts'] }, { host: 'develop.sentry.dev', why: 'link to the envelope format in a comment', files: ['lib/error-reporting/envelope.ts'] }, { host: 'extensions.bulwarkmail.org', why: 'signed statement type URIs', files: ['lib/admin/directory-keys.ts'] }, diff --git a/lib/error-reporting/settings.ts b/lib/error-reporting/settings.ts index c29a31470..a37cbb68c 100644 --- a/lib/error-reporting/settings.ts +++ b/lib/error-reporting/settings.ts @@ -1,5 +1,6 @@ import { configManager } from '@/lib/admin/config-manager'; import { parseDsn } from './envelope'; +import { metricsEnabled } from '@/lib/metrics'; import type { ClientSettings } from './client'; /** @@ -43,7 +44,7 @@ export function errorReportingSettings(): ErrorReportingSettings { */ export async function shouldCollectErrors(): Promise { await configManager.ensureLoaded(); - return !!errorReportingSettings().dsn; + return !!errorReportingSettings().dsn || metricsEnabled(); } /** What /api/config hands the browser. */ diff --git a/lib/health/__tests__/readiness.test.ts b/lib/health/__tests__/readiness.test.ts new file mode 100644 index 000000000..35871414b --- /dev/null +++ b/lib/health/__tests__/readiness.test.ts @@ -0,0 +1,86 @@ +// @vitest-environment node +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +const config: Record = {}; +vi.mock('@/lib/admin/config-manager', () => ({ + configManager: { + ensureLoaded: async () => {}, + get: (key: string, fallback: unknown) => (key in config ? config[key] : fallback), + }, +})); + +import { readiness, resetReadinessCache } from '../readiness'; +import { GET as ready } from '@/app/api/health/ready/route'; +import { GET as live } from '@/app/api/health/live/route'; + +let dir: string; +let fetchSpy: ReturnType; + +beforeEach(() => { + dir = mkdtempSync(path.join(tmpdir(), 'bulwark-ready-')); + process.env.ADMIN_STATE_DIR = path.join(dir, 'state'); + process.env.TELEMETRY_DATA_DIR = path.join(dir, 'telemetry'); + for (const key of Object.keys(config)) delete config[key]; + config.jmapServerUrl = 'https://mail.example.org'; + resetReadinessCache(); + fetchSpy = vi.fn().mockResolvedValue(new Response(null, { status: 401 })); + vi.stubGlobal('fetch', fetchSpy); +}); + +afterEach(() => { + vi.unstubAllGlobals(); + delete process.env.ADMIN_STATE_DIR; + delete process.env.TELEMETRY_DATA_DIR; + rmSync(dir, { recursive: true, force: true }); +}); + +describe('health endpoints', () => { + it('live answers 200 whatever Stalwart does', async () => { + fetchSpy.mockRejectedValue(new Error('down')); + expect((await live()).status).toBe(200); + }); + + it('ready answers 200 when Stalwart answers, even with 401', async () => { + const res = await ready(); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ ready: true, checks: { config: 'ok', store: 'ok', dataDirs: 'ok', stalwart: 'ok' } }); + expect(String(fetchSpy.mock.calls[0][0])).toBe('https://mail.example.org/.well-known/jmap'); + }); + + it('ready answers 503 when Stalwart is unreachable or failing, and names no host', async () => { + fetchSpy.mockRejectedValue(new Error('ECONNREFUSED mail.example.org')); + const res = await ready(); + expect(res.status).toBe(503); + const text = await res.text(); + expect(JSON.parse(text).checks.stalwart).toBe('fail'); + expect(text).not.toContain('example.org'); + + resetReadinessCache(); + fetchSpy.mockResolvedValue(new Response(null, { status: 502 })); + expect((await ready()).status).toBe(503); + }); + + it('skips Stalwart before a server is configured', async () => { + delete config.jmapServerUrl; + delete process.env.JMAP_SERVER_URL; + expect((await readiness()).checks.stalwart).toBe('skipped'); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('fails when a data dir is not writable', async () => { + writeFileSync(path.join(dir, 'blocked'), ''); + process.env.TELEMETRY_DATA_DIR = path.join(dir, 'blocked', 'telemetry'); + const result = await readiness(); + expect(result.checks.dataDirs).toBe('fail'); + expect(result.ready).toBe(false); + }); + + it('reuses a result for a few seconds', async () => { + await readiness(1000); + await readiness(2000); + expect(fetchSpy).toHaveBeenCalledTimes(1); + }); +}); diff --git a/lib/health/readiness.ts b/lib/health/readiness.ts new file mode 100644 index 000000000..4543b4a2a --- /dev/null +++ b/lib/health/readiness.ts @@ -0,0 +1,131 @@ +import { randomBytes } from 'node:crypto'; +import { mkdir, unlink, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { configManager } from '@/lib/admin/config-manager'; +import { getStateDir } from '@/lib/admin/paths'; +import { observeUpstream } from '@/lib/metrics/upstream'; +import { bootState, getStore, isSharedMode } from '@/lib/shared-store'; +import { getSettingsDir } from '@/lib/settings-sync'; +import { getDir as getTelemetryDir } from '@/lib/telemetry/state'; + +/** + * Whether this replica can serve, for /api/health/ready (and, for the + * shared store, /api/ready). Each check answers ok, fail or skipped; the + * answer names checks, never hosts or paths, because anyone may ask. + */ + +export type CheckResult = 'ok' | 'fail' | 'skipped'; + +export interface Readiness { + ready: boolean; + checks: { + config: CheckResult; + store: CheckResult; + dataDirs: CheckResult; + stalwart: CheckResult; + }; +} + +const STALWART_TIMEOUT_MS = 2000; +const STORE_TIMEOUT_MS = 1000; +const CACHE_MS = 5000; + +const KEY = Symbol.for('bulwark.health.readiness'); +interface CacheState { at: number; result: Readiness | null; pending: Promise | null } +function cache(): CacheState { + const g = globalThis as typeof globalThis & { [KEY]?: CacheState }; + return (g[KEY] ??= { at: 0, result: null, pending: null }); +} + +async function checkConfig(): Promise { + try { + await configManager.ensureLoaded(); + return 'ok'; + } catch { + return 'fail'; + } +} + +/** Booted, and with a shared store, the store answered a ping. */ +export async function checkStore(): Promise { + if (bootState() !== 'ready') return 'fail'; + const store = getStore(); + if (store.mode === 'local') return 'ok'; + try { + await store.ping(STORE_TIMEOUT_MS); + return 'ok'; + } catch { + return 'fail'; + } +} + +async function writable(dir: string): Promise { + const probe = path.join(dir, `.ready-${process.pid}-${randomBytes(4).toString('hex')}`); + try { + await mkdir(dir, { recursive: true }); + await writeFile(probe, ''); + await unlink(probe); + return true; + } catch { + return false; + } +} + +/** The dirs this process writes to in local mode; a shared store holds that data instead. */ +async function checkDataDirs(): Promise { + if (isSharedMode()) return 'skipped'; + const dirs = [getStateDir(), getTelemetryDir()]; + if (configManager.get('settingsSyncEnabled', false)) dirs.push(getSettingsDir()); + const results = await Promise.all(dirs.map(writable)); + return results.every(Boolean) ? 'ok' : 'fail'; +} + +/** Stalwart's session resource answers, whatever the status below 500 (401 is expected without credentials). */ +async function checkStalwart(): Promise { + const base = configManager.get('jmapServerUrl', '') || process.env.JMAP_SERVER_URL || ''; + if (!base.trim()) return 'skipped'; + let url: URL; + try { + url = new URL('/.well-known/jmap', base.trim()); + } catch { + return 'fail'; + } + try { + const response = await observeUpstream('stalwart', () => fetch(url, { + method: 'GET', + redirect: 'manual', + signal: AbortSignal.timeout(STALWART_TIMEOUT_MS), + })); + await response.body?.cancel().catch(() => undefined); + return response.status < 500 ? 'ok' : 'fail'; + } catch { + return 'fail'; + } +} + +async function evaluate(): Promise { + const config = await checkConfig(); + const [store, dataDirs, stalwart] = await Promise.all([checkStore(), checkDataDirs(), checkStalwart()]); + const checks = { config, store, dataDirs, stalwart }; + return { ready: Object.values(checks).every((c) => c !== 'fail'), checks }; +} + +/** The readiness of this replica, reused for CACHE_MS so probes can't flood Stalwart or the store. */ +export async function readiness(now = Date.now()): Promise { + const c = cache(); + if (c.result && now - c.at < CACHE_MS) return c.result; + c.pending ??= evaluate() + .then((result) => { + c.result = result; + c.at = Date.now(); + return result; + }) + .finally(() => { c.pending = null; }); + return c.pending; +} + +/** For tests. */ +export function resetReadinessCache(): void { + const g = globalThis as typeof globalThis & { [KEY]?: CacheState }; + delete g[KEY]; +} diff --git a/lib/metrics/__tests__/metrics.test.ts b/lib/metrics/__tests__/metrics.test.ts new file mode 100644 index 000000000..286dd4030 --- /dev/null +++ b/lib/metrics/__tests__/metrics.test.ts @@ -0,0 +1,93 @@ +// @vitest-environment node +import http from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { metrics, observeRequest, resetMetricsForTests, signinOutcome } from '../index'; +import { observeUpstream } from '../upstream'; +import { installHttpMetrics } from '../http'; + +async function value(name: string, labels: Record = {}): Promise { + const metric = (await metrics().registry.getMetricsAsJSON()).find((m) => m.name === name); + const found = (metric?.values as Array<{ labels: Record; value: number }> | undefined) + ?.find((v) => Object.entries(labels).every(([k, want]) => v.labels[k] === want)); + return found?.value; +} + +beforeEach(() => { + process.env.METRICS_TOKEN = 'test-token-1234567890'; + resetMetricsForTests(); +}); + +afterEach(() => { + delete process.env.METRICS_TOKEN; + resetMetricsForTests(); + vi.restoreAllMocks(); +}); + +describe('request metrics', () => { + it('label requests by route group and status class, never by the raw path', async () => { + observeRequest('/de/mail/folder/Acquisition%20Talks/Mabc?q=secret', 'GET', 200, 0.05); + observeRequest('/webmail/api/wopi/files/abc/contents', 'POST', 503, 1.2, '/webmail'); + observeRequest('/api/whatever-a-user-typed', 'BREW', 404, 0.01); + expect(await value('bulwark_http_requests_total', { group: 'page:mail', method: 'GET', status: '2xx' })).toBe(1); + expect(await value('bulwark_http_requests_total', { group: 'api:wopi', method: 'POST', status: '5xx' })).toBe(1); + expect(await value('bulwark_http_requests_total', { group: 'api:other', method: 'other', status: '4xx' })).toBe(1); + const text = await metrics().registry.metrics(); + expect(text).not.toContain('Acquisition'); + expect(text).not.toContain('whatever-a-user-typed'); + expect(text).not.toContain('secret'); + }); + + it('count sign-ins by outcome from the auth routes', async () => { + observeRequest('/api/auth/session', 'POST', 200, 0.1); + observeRequest('/api/auth/session', 'POST', 401, 0.1); + observeRequest('/api/auth/session', 'POST', 429, 0.1); + observeRequest('/api/auth/token', 'PUT', 401, 0.1); + observeRequest('/api/auth/token', 'PUT', 200, 0.1); + observeRequest('/api/auth/session', 'GET', 200, 0.1); + for (const [outcome, n] of [['success', 1], ['failure', 1], ['rate_limited', 1], ['refresh_failure', 1]] as const) { + expect(await value('bulwark_signins_total', { outcome }), outcome).toBe(n); + } + }); + + it('map outcomes only for sign-in and refresh calls', () => { + expect(signinOutcome('api:auth', '/api/auth/totp-token-exchange', 'POST', 200)).toBe('success'); + expect(signinOutcome('api:auth', '/api/auth/session', 'DELETE', 200)).toBeNull(); + expect(signinOutcome('api:admin', '/api/admin/auth', 'POST', 401)).toBeNull(); + }); + + it('do nothing while the endpoint is off', async () => { + delete process.env.METRICS_TOKEN; + observeRequest('/api/config', 'GET', 200, 0.01); + process.env.METRICS_TOKEN = 'test-token-1234567890'; + expect(await value('bulwark_http_requests_total', { group: 'api:config' })).toBeUndefined(); + }); + + it('come from Node\'s HTTP server itself once installed', async () => { + expect(installHttpMetrics('')).toBe(true); + const server = http.createServer((_req, res) => { res.statusCode = 204; res.end(); }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + try { + const { port } = server.address() as AddressInfo; + await fetch(`http://127.0.0.1:${port}/api/health/live`); + await vi.waitFor(async () => expect(await value('bulwark_http_requests_total', { group: 'api:health', status: '2xx' })).toBe(1)); + } finally { + server.close(); + } + }); +}); + +describe('upstream metrics', () => { + it('count answers below 500 as ok and 5xx or errors as error', async () => { + await observeUpstream('stalwart', async () => new Response(null, { status: 401 })); + await observeUpstream('stalwart', async () => new Response(null, { status: 502 })); + await expect(observeUpstream('wopi', async () => { throw new Error('down'); })).rejects.toThrow('down'); + expect(await value('bulwark_upstream_requests_total', { target: 'stalwart', outcome: 'ok' })).toBe(1); + expect(await value('bulwark_upstream_requests_total', { target: 'stalwart', outcome: 'error' })).toBe(1); + expect(await value('bulwark_upstream_requests_total', { target: 'wopi', outcome: 'error' })).toBe(1); + }); + + it('report build info', async () => { + expect(await metrics().registry.metrics()).toMatch(/bulwark_build_info\{version="[^"]+",build="[^"]+"\} 1/); + }); +}); diff --git a/lib/metrics/__tests__/upstream-wiring.test.ts b/lib/metrics/__tests__/upstream-wiring.test.ts new file mode 100644 index 000000000..b470ffd2a --- /dev/null +++ b/lib/metrics/__tests__/upstream-wiring.test.ts @@ -0,0 +1,29 @@ +// @vitest-environment node +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +vi.mock('@/lib/admin/config-manager', () => ({ + configManager: { ensureLoaded: async () => {}, get: (_key: string, fallback: unknown) => fallback }, +})); + +import { fetchJmapServer } from '@/lib/stalwart/server-fetch'; +import { metrics, resetMetricsForTests } from '@/lib/metrics'; + +beforeEach(() => { + process.env.METRICS_TOKEN = 'scrape-secret-0123456789'; + resetMetricsForTests(); +}); + +afterEach(() => { + delete process.env.METRICS_TOKEN; + resetMetricsForTests(); + vi.unstubAllGlobals(); +}); + +// The server's calls to Stalwart all go through fetchJmapServer. +describe('Stalwart calls', () => { + it('are counted under the stalwart target', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(new Response('{}', { status: 200 }))); + await fetchJmapServer('https://mail.example.org/jmap/session', {}, true); + expect(await metrics().registry.metrics()).toContain('bulwark_upstream_requests_total{target="stalwart",outcome="ok"} 1'); + }); +}); diff --git a/lib/metrics/http.ts b/lib/metrics/http.ts new file mode 100644 index 000000000..1bffbb9a8 --- /dev/null +++ b/lib/metrics/http.ts @@ -0,0 +1,39 @@ +import diagnostics from 'node:diagnostics_channel'; +import type { IncomingMessage, ServerResponse } from 'node:http'; +import { metricsEnabled, observeRequest } from './index'; + +/** + * Counts every request this process serves, through Node's own HTTP + * diagnostics channels: nothing is wrapped or patched, and the proxy's + * matcher doesn't matter. Installed once at startup when the metrics + * endpoint is on (instrumentation.node.ts). + */ + +interface HttpState { + installed: boolean; + started: WeakMap; +} + +const KEY = Symbol.for('bulwark.metrics.http'); +function state(): HttpState { + const g = globalThis as typeof globalThis & { [KEY]?: HttpState }; + return (g[KEY] ??= { installed: false, started: new WeakMap() }); +} + +export function installHttpMetrics(basePath = process.env.NEXT_PUBLIC_BASE_PATH || ''): boolean { + const s = state(); + if (s.installed || !metricsEnabled()) return false; + s.installed = true; + diagnostics.subscribe('http.server.request.start', (message) => { + const { request } = message as { request: IncomingMessage }; + s.started.set(request, performance.now()); + }); + diagnostics.subscribe('http.server.response.finish', (message) => { + const { request, response } = message as { request: IncomingMessage; response: ServerResponse }; + const start = s.started.get(request); + if (start === undefined) return; + s.started.delete(request); + observeRequest(request.url ?? '/', request.method, response.statusCode, (performance.now() - start) / 1000, basePath); + }); + return true; +} diff --git a/lib/metrics/index.ts b/lib/metrics/index.ts new file mode 100644 index 000000000..dfdea1749 --- /dev/null +++ b/lib/metrics/index.ts @@ -0,0 +1,147 @@ +import { readFileSync } from 'node:fs'; +import type { Counter, Gauge, Histogram, Registry } from 'prom-client'; +import client from 'prom-client'; +import { routeGroup } from '@/lib/route-groups'; + +/** + * Prometheus metrics for /api/metrics, off unless METRICS_TOKEN (or + * METRICS_TOKEN_FILE) is set. Every replica counts for itself; Prometheus + * scrapes each one. Labels come from fixed lists only (route groups, sign-in + * outcomes, upstream targets), never from user names or hosts users type. + * + * https://bulwarkmail.org/docs/admin/monitoring#metrics + */ + +export const SIGNIN_OUTCOMES = ['success', 'failure', 'refresh_failure', 'rate_limited'] as const; +export type SigninOutcome = typeof SIGNIN_OUTCOMES[number]; + +export const UPSTREAM_TARGETS = ['stalwart', 'wopi', 'translate', 'plugin_proxy', 'oauth'] as const; +export type UpstreamTarget = typeof UPSTREAM_TARGETS[number]; + +const METHODS = new Set(['GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS']); +const DURATION_BUCKETS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30]; + +/** The bearer token /api/metrics wants, or null when the endpoint is off. */ +export function metricsToken(): string | null { + const direct = process.env.METRICS_TOKEN?.trim(); + if (direct) return direct; + const file = process.env.METRICS_TOKEN_FILE?.trim(); + if (!file) return null; + try { + return readFileSync(file, 'utf8').trim() || null; + } catch { + return null; + } +} + +export function metricsEnabled(): boolean { + return metricsToken() !== null; +} + +interface Metrics { + registry: Registry; + httpRequests: Counter<'group' | 'method' | 'status'>; + httpDuration: Histogram<'group' | 'method'>; + signins: Counter<'outcome'>; + upstreamRequests: Counter<'target' | 'outcome'>; + upstreamDuration: Histogram<'target'>; + browserErrors: Counter; + errorsToday: Gauge; +} + +const KEY = Symbol.for('bulwark.metrics'); + +/** The registry and metrics, made once per process whatever bundle asks. */ +export function metrics(): Metrics { + const g = globalThis as typeof globalThis & { [KEY]?: Metrics }; + if (g[KEY]) return g[KEY]; + const registry = new client.Registry(); + client.collectDefaultMetrics({ register: registry, prefix: 'bulwark_' }); + const version = process.env.NEXT_PUBLIC_APP_VERSION || '0.0.0'; + const build = process.env.NEXT_PUBLIC_GIT_COMMIT || process.env.BULWARK_BUILD || 'unknown'; + new client.Gauge({ name: 'bulwark_build_info', help: 'Bulwark version and build; always 1', labelNames: ['version', 'build'], registers: [registry] }) + .set({ version, build }, 1); + new client.Gauge({ + name: 'bulwark_uptime_seconds', + help: 'Seconds since this process started', + registers: [registry], + collect() { this.set(process.uptime()); }, + }); + const created: Metrics = { + registry, + httpRequests: new client.Counter({ name: 'bulwark_http_requests_total', help: 'HTTP requests by route group, method and status class', labelNames: ['group', 'method', 'status'], registers: [registry] }), + httpDuration: new client.Histogram({ name: 'bulwark_http_request_duration_seconds', help: 'HTTP request duration by route group and method', labelNames: ['group', 'method'], buckets: DURATION_BUCKETS, registers: [registry] }), + signins: new client.Counter({ name: 'bulwark_signins_total', help: 'Sign-ins handled by this server, by outcome', labelNames: ['outcome'], registers: [registry] }), + upstreamRequests: new client.Counter({ name: 'bulwark_upstream_requests_total', help: 'Requests this server made to other services, by target and outcome', labelNames: ['target', 'outcome'], registers: [registry] }), + upstreamDuration: new client.Histogram({ name: 'bulwark_upstream_request_duration_seconds', help: 'Duration of requests to other services, by target', labelNames: ['target'], buckets: DURATION_BUCKETS, registers: [registry] }), + browserErrors: new client.Counter({ name: 'bulwark_browser_errors_total', help: 'Errors browsers reported to this server', registers: [registry] }), + errorsToday: new client.Gauge({ + name: 'bulwark_errors_today', + help: 'Errors counted today (UTC) by the whole deployment, browsers and servers', + registers: [registry], + async collect() { + const { readErrors, summarizeDay } = await import('@/lib/error-reporting/aggregate'); + this.set(summarizeDay(await readErrors()).total); + }, + }), + }; + // Pre-create the label sets, so a series exists at 0 before its first event. + for (const outcome of SIGNIN_OUTCOMES) created.signins.inc({ outcome }, 0); + for (const target of UPSTREAM_TARGETS) { + for (const outcome of ['ok', 'error']) created.upstreamRequests.inc({ target, outcome }, 0); + } + g[KEY] = created; + return created; +} + +export function methodLabel(method: string | undefined): string { + const upper = (method ?? '').toUpperCase(); + return METHODS.has(upper) ? upper : 'other'; +} + +export function statusClass(status: number): string { + return status >= 100 && status < 600 ? `${Math.floor(status / 100)}xx` : 'other'; +} + +/** What a finished request means for sign-ins, from its route, method and status. */ +export function signinOutcome(group: string, path: string, method: string, status: number): SigninOutcome | null { + if (group !== 'api:auth') return null; + const route = path.split(/[?#]/, 1)[0].replace(/\/+$/, ''); + const signIn = method === 'POST' && /\/api\/auth\/(?:session|stalwart-context|totp-token-exchange|token)$/.test(route); + const refresh = method === 'PUT' && /\/api\/auth\/token$/.test(route); + if (!signIn && !refresh) return null; + if (status === 429) return 'rate_limited'; + if (status >= 200 && status < 300) return refresh ? null : 'success'; + return refresh ? 'refresh_failure' : 'failure'; +} + +/** Count a finished HTTP request (lib/metrics/http.ts). */ +export function observeRequest(path: string, method: string | undefined, status: number, seconds: number, basePath = ''): void { + if (!metricsEnabled()) return; + const m = metrics(); + const group = routeGroup(path, basePath); + const verb = methodLabel(method); + m.httpRequests.inc({ group, method: verb, status: statusClass(status) }); + m.httpDuration.observe({ group, method: verb }, seconds); + const outcome = signinOutcome(group, basePath && path.startsWith(basePath) ? path.slice(basePath.length) : path, verb, status); + if (outcome) m.signins.inc({ outcome }); +} + +export function observeUpstreamResult(target: UpstreamTarget, ok: boolean, seconds: number): void { + if (!metricsEnabled()) return; + const m = metrics(); + m.upstreamRequests.inc({ target, outcome: ok ? 'ok' : 'error' }); + m.upstreamDuration.observe({ target }, seconds); +} + +export function countBrowserError(): void { + if (!metricsEnabled()) return; + metrics().browserErrors.inc(); +} + +/** For tests. */ +export function resetMetricsForTests(): void { + const g = globalThis as typeof globalThis & { [KEY]?: Metrics }; + g[KEY]?.registry.clear(); + delete g[KEY]; +} diff --git a/lib/metrics/upstream.ts b/lib/metrics/upstream.ts new file mode 100644 index 000000000..5aaf47fca --- /dev/null +++ b/lib/metrics/upstream.ts @@ -0,0 +1,19 @@ +import { observeUpstreamResult, type UpstreamTarget } from './index'; + +/** + * Run one request this server makes to another service and record its + * duration and outcome under a fixed target name. An answer below 500 counts + * as ok (a 401 from Stalwart is an answer); a 5xx or a thrown error as + * error. Costs nothing while metrics are off. + */ +export async function observeUpstream(target: UpstreamTarget, run: () => Promise): Promise { + const start = performance.now(); + try { + const response = await run(); + observeUpstreamResult(target, response.status < 500, (performance.now() - start) / 1000); + return response; + } catch (error) { + observeUpstreamResult(target, false, (performance.now() - start) / 1000); + throw error; + } +} diff --git a/lib/oauth/token-exchange.ts b/lib/oauth/token-exchange.ts index e002f218d..a60031595 100644 --- a/lib/oauth/token-exchange.ts +++ b/lib/oauth/token-exchange.ts @@ -1,4 +1,5 @@ import { logger } from '@/lib/logger'; +import { observeUpstream } from '@/lib/metrics/upstream'; import { discoverOAuth } from '@/lib/oauth/discovery'; import type { EndpointValidator, OAuthMetadata } from '@/lib/oauth/discovery'; import { isPublicHttpUrl } from '@/lib/security/url-guard'; @@ -209,12 +210,12 @@ export async function exchangeCodeForTokens( // The body carries the code, the PKCE verifier and the client secret. The // endpoint was validated; a redirect would re-send all of it to a host // that was not, so none is followed. - const tokenResponse = await fetch(tokenEndpoint, { + const tokenResponse = await observeUpstream('oauth', () => fetch(tokenEndpoint, { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: params.toString(), redirect: 'error', - }); + })); if (!tokenResponse.ok) { const errorText = await tokenResponse.text(); diff --git a/lib/plugin-secrets/proxy.ts b/lib/plugin-secrets/proxy.ts index 766b49dd7..6a84a3f8b 100644 --- a/lib/plugin-secrets/proxy.ts +++ b/lib/plugin-secrets/proxy.ts @@ -9,6 +9,7 @@ import { getPlugin } from '@/lib/admin/plugin-registry'; import { getPluginConfig } from '@/lib/admin/plugin-config'; import { logger } from '@/lib/logger'; import { DisallowedUrlError, fetchPublicUrl } from '@/lib/security/url-guard'; +import { observeUpstream } from '@/lib/metrics/upstream'; import { authHeaderFor, credentialOrigin, privateProxyOrigins, type CredentialInject, type CredentialSpec } from './credentials'; import { getUserCredential, setUserCredentialSecret, userKey, type VaultUser } from './vault'; import { declaredUserCredential } from './known-plugins'; @@ -239,8 +240,10 @@ async function send( signal: init.signal, ...(init.body ? { body: init.body } : {}), }; - if (trusted) return fetch(url, { ...request, redirect: 'manual' } as RequestInit); - return (await fetchPublicUrl(url, request as never)) as unknown as Response; + return observeUpstream('plugin_proxy', async () => { + if (trusted) return fetch(url, { ...request, redirect: 'manual' } as RequestInit); + return (await fetchPublicUrl(url, request as never)) as unknown as Response; + }); } function responseHeaders(upstream: Response): Headers { diff --git a/lib/route-groups.ts b/lib/route-groups.ts index e10e6c092..6c222f6a6 100644 --- a/lib/route-groups.ts +++ b/lib/route-groups.ts @@ -11,7 +11,7 @@ /** The folders under app/api. */ export const API_GROUPS: readonly string[] = [ 'account', 'account-archive', 'admin', 'auth', 'bimi', 'caldav', 'calendar-agenda', 'config', - 'connector', 'dev-jmap', 'errors', 'favicon', 'fetch-ical', 'health', 'og-image', + 'connector', 'dev-jmap', 'errors', 'favicon', 'fetch-ical', 'health', 'metrics', 'og-image', 'plugin-approval-status', 'plugin-proxy', 'plugin-secrets', 'plugin-signing-pubkey', 'plugins', 'push', 'pwa-icon', 'pwa-screenshot', 'ready', 'settings', 'setup', 'system', 'translate', 'webdav', 'wopi', diff --git a/lib/stalwart/server-fetch.ts b/lib/stalwart/server-fetch.ts index d1adc2fe3..710c2493c 100644 --- a/lib/stalwart/server-fetch.ts +++ b/lib/stalwart/server-fetch.ts @@ -19,6 +19,7 @@ import { configManager } from '@/lib/admin/config-manager'; import { logger } from '@/lib/logger'; import { parseJmapServers, resolveTrustedJmapUrl } from '@/lib/admin/jmap-servers'; import { fetchPublicUrl, type PublicFetchResponse } from '@/lib/security/url-guard'; +import { observeUpstream } from '@/lib/metrics/upstream'; import { isStoredCredentialRefused, recordStoredCredentialStatus, @@ -72,7 +73,7 @@ export async function fetchJmapServer( ): Promise { const authHeader = new Headers(givenInit.headers).get('authorization'); if (authHeader && (await isStoredCredentialRefused(authHeader))) return refusedCredentialResponse(); - const response = await fetchUnbudgeted(url, givenInit, trusted); + const response = await observeUpstream('stalwart', () => fetchUnbudgeted(url, givenInit, trusted)); // Not awaited: this process's copy is updated at once, the shared write follows. if (authHeader) { void recordStoredCredentialStatus(authHeader, response.status).catch((error) => { diff --git a/lib/wopi/discovery.ts b/lib/wopi/discovery.ts index d2150567f..971285f26 100644 --- a/lib/wopi/discovery.ts +++ b/lib/wopi/discovery.ts @@ -1,4 +1,5 @@ import { configManager } from '@/lib/admin/config-manager'; +import { observeUpstream } from '@/lib/metrics/upstream'; import { logger } from '@/lib/logger'; /** @@ -123,10 +124,10 @@ export async function getWopiActions(): Promise { async function fetchWopiActions(discoveryUrl: string): Promise { try { - const res = await fetch(discoveryUrl, { + const res = await observeUpstream('wopi', () => fetch(discoveryUrl, { headers: { Accept: 'text/xml' }, signal: AbortSignal.timeout(DISCOVERY_TIMEOUT_MS), - }); + })); if (!res.ok) { logger.warn('WOPI discovery fetch failed', { discoveryUrl, status: res.status }); return null; diff --git a/next.config.ts b/next.config.ts index d039caec1..cb20d9f8c 100644 --- a/next.config.ts +++ b/next.config.ts @@ -61,9 +61,9 @@ const nextConfig: NextConfig = { // esbuild ships native binaries + a README the bundler can't parse; load // it from node_modules at runtime instead of trying to bundle it. Used by // PLUGIN_DEV_DIR's on-the-fly bundler. ioredis (the shared store, #1066) - // is loaded from node_modules too, so the proxy and the route bundles - // don't each carry a copy. - serverExternalPackages: ["esbuild", "ioredis"], + // and prom-client (/api/metrics) are loaded from node_modules too, so the + // proxy and the route bundles don't each carry a copy. + serverExternalPackages: ["esbuild", "ioredis", "prom-client"], // The build's type check leaves out tests, which are a third of the files // and push tsc past the default heap on a 4 GB machine (#1431). // `npm run typecheck` still checks them with tsconfig.json. diff --git a/package-lock.json b/package-lock.json index 878d8cb72..055af593f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,13 @@ "@dnd-kit/utilities": "^3.2.2", "@noble/curves": "^2.4.0", "@noble/hashes": "^2.4.0", + "@opentelemetry/api": "1.9.1", + "@opentelemetry/core": "2.8.0", + "@opentelemetry/exporter-trace-otlp-http": "0.219.0", + "@opentelemetry/resources": "2.8.0", + "@opentelemetry/sdk-trace-base": "2.8.0", + "@opentelemetry/sdk-trace-node": "2.8.0", + "@opentelemetry/semantic-conventions": "1.44.0", "@tabler/icons-react": "^3.50.0", "@tanstack/react-virtual": "^3.14.23", "@tiptap/extension-color": "^3.32.1", @@ -43,6 +50,7 @@ "otpauth": "^9.5.2", "pdfjs-dist": "^6.4.299", "postal-mime": "^4.1.4", + "prom-client": "15.1.3", "qrcode": "^1.5.4", "react": "^19.3.1", "react-dom": "^19.3.1", @@ -2389,6 +2397,204 @@ "url": "https://paulmillr.com/funding/" } }, + "node_modules/@opentelemetry/api": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/api-logs": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.219.0.tgz", + "integrity": "sha512-FFx7YnaYJlIjqWW/AG/yAZ0L/NEY724PipXXXQLdtZPbLwBGbUMTGL1i/esI56TWfTUXxhLfpgrnWJCG8aUJyg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/context-async-hooks": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-2.8.0.tgz", + "integrity": "sha512-/3FIraneMcng67SUJCxvyInk/oxzwsxyadufk0wwfOBLf5wqtAGX4MoQASwSbndBPeARzBryUM9Azr5kHIdWLw==", + "license": "Apache-2.0", + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/core": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.8.0.tgz", + "integrity": "sha512-hd1Lfh8p545nNz+jq1Ejfz+Mn1hyLuxYn1YzTfFNrxr8urEWMNQLPf1Th8kjOH+HxwawCrtgBp8JpBUR4ZSgww==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-http": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-http/-/exporter-trace-otlp-http-0.219.0.tgz", + "integrity": "sha512-9t6SvBXXBEjOBcIzgozvBbd3jWrv3Gt3ngGhl1fhdZ/zRc7oZDVOFEqbi2zlBpW9BXhgDMKv422J0DL/3iQWfw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.8.0", + "@opentelemetry/otlp-exporter-base": "0.219.0", + "@opentelemetry/otlp-transformer": "0.219.0", + "@opentelemetry/resources": "2.8.0", + "@opentelemetry/sdk-trace-base": "2.8.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-exporter-base": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.219.0.tgz", + "integrity": "sha512-zvIxQX/AZUVKDU+hCuYx+7UkiP7GRdnk1ZbFQRYzHvYp47cAWR4j3IhoPhV9KaeXEv2xdGq3IA6PnpzDmLcmSA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.8.0", + "@opentelemetry/otlp-transformer": "0.219.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.219.0.tgz", + "integrity": "sha512-aaYKAyXhw9VchKZVGOopD3Gw/kPsyrX2c6IQ0AW32mTjqmZOh5Y6Gf5OYqTNqVktAeBjmFinhyFaCwW6GYK9YQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.219.0", + "@opentelemetry/core": "2.8.0", + "@opentelemetry/resources": "2.8.0", + "@opentelemetry/sdk-logs": "0.219.0", + "@opentelemetry/sdk-metrics": "2.8.0", + "@opentelemetry/sdk-trace-base": "2.8.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/resources": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.8.0.tgz", + "integrity": "sha512-qmXQ27ilDbUK/vGMqwL8D4/rhn76C+sherM4wTbjlfknR8Nvfc/hCxjRJPhkzZzUsPiNg16SA31NxMabwttRjg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.8.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-logs": { + "version": "0.219.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.219.0.tgz", + "integrity": "sha512-s6lTKRakaPClvKoWHRChxnXjDMkM/TQ30ff78jN6EBGf7MI7VzANE5PU3f4z9qDUudWjvZjOLHG0rBnBKYvoXA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.219.0", + "@opentelemetry/core": "2.8.0", + "@opentelemetry/resources": "2.8.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.4.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-metrics": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.8.0.tgz", + "integrity": "sha512-UDBGaj6W0Rgy5rTTaoxs8gVGF/aGkAKyjurJv7se6wjRxJu7FoquTLT/vt54DZfo4crbprYfhX/SOK9+BPw1qg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.8.0", + "@opentelemetry/resources": "2.8.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-base": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.8.0.tgz", + "integrity": "sha512-mhU4jp+vW0mGbFRd+GeXHvmfA4aDqWjBjLC3pE5XMpLs0IE2ryYb019Ts2AQrOq67gaTF25D91+fgvEHDZEnuQ==", + "deprecated": "Use @opentelemetry/sdk-trace package", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.8.0", + "@opentelemetry/resources": "2.8.0", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-node": { + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-node/-/sdk-trace-node-2.8.0.tgz", + "integrity": "sha512-nZt9OGufioAc3AfoLTqA9bsAeaMJAictYDdI2VcNQ+PmT+3rfKjAZDZvgPfd8VPX0O5Bw1hdQF6kDK8VSpZiWg==", + "deprecated": "Use @opentelemetry/sdk-trace package", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/context-async-hooks": "2.8.0", + "@opentelemetry/core": "2.8.0", + "@opentelemetry/sdk-trace-base": "2.8.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/semantic-conventions": { + "version": "1.44.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.44.0.tgz", + "integrity": "sha512-E3JkOlMBi3N8OGGkbl7tFyqk2MAS6/lySRZBK3Mu7AzjF1fT8oIChH9ChE48SUWzDn9Kdwta+G/p5LJ4xW4CLQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, "node_modules/@oxc-project/types": { "version": "0.155.0", "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.155.0.tgz", @@ -4943,6 +5149,12 @@ "require-from-string": "^2.0.2" } }, + "node_modules/bintrees": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", + "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==", + "license": "MIT" + }, "node_modules/brace-expansion": { "version": "5.0.12", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.12.tgz", @@ -8375,6 +8587,20 @@ "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", "license": "MIT" }, + "node_modules/prom-client": { + "version": "15.1.3", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.3.tgz", + "integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==", + "deprecated": "prom-client has been replaced by @prometheus-io/client", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.4.0", + "tdigest": "^0.1.1" + }, + "engines": { + "node": "^16 || ^18 || >=20" + } + }, "node_modules/prop-types": { "version": "15.8.1", "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", @@ -9355,6 +9581,15 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/tdigest": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.3.tgz", + "integrity": "sha512-zbRt+lT+/H4fRItHshczHErVCQnitJk8MfMT24MqFJf3YL7SJJPqGIGeuOdvxXxM/AHFzKBl7WoyaYwqO9s3Kw==", + "license": "MIT", + "dependencies": { + "bintrees": "1.0.2" + } + }, "node_modules/tinybench": { "version": "6.2.2", "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-6.2.2.tgz", diff --git a/package.json b/package.json index 6f78a0579..ae34dfde9 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,13 @@ "@dnd-kit/utilities": "^3.2.2", "@noble/curves": "^2.4.0", "@noble/hashes": "^2.4.0", + "@opentelemetry/api": "1.9.1", + "@opentelemetry/core": "2.8.0", + "@opentelemetry/exporter-trace-otlp-http": "0.219.0", + "@opentelemetry/resources": "2.8.0", + "@opentelemetry/sdk-trace-base": "2.8.0", + "@opentelemetry/sdk-trace-node": "2.8.0", + "@opentelemetry/semantic-conventions": "1.44.0", "@tabler/icons-react": "^3.50.0", "@tanstack/react-virtual": "^3.14.23", "@tiptap/extension-color": "^3.32.1", @@ -73,6 +80,7 @@ "otpauth": "^9.5.2", "pdfjs-dist": "^6.4.299", "postal-mime": "^4.1.4", + "prom-client": "15.1.3", "qrcode": "^1.5.4", "react": "^19.3.1", "react-dom": "^19.3.1", diff --git a/proxy.ts b/proxy.ts index 12f1423ed..0ba40028a 100644 --- a/proxy.ts +++ b/proxy.ts @@ -179,6 +179,8 @@ export function mailFrameAncestors(allowed: string | undefined): string { return sources.join(" "); } +const PROBE_PATHS = new Set(["/api/health", "/api/health/live", "/api/health/ready", "/api/ready", "/api/metrics"]); + function isSetupPath(pathname: string): boolean { return ( pathname === "/setup" || @@ -189,9 +191,10 @@ function isSetupPath(pathname: string): boolean { export async function proxy(request: NextRequest) { // The probes answer without config or store, so a replica that is still - // starting (or lost its shared store) can say so (#1066). + // starting (or lost its shared store) can say so (#1066). The metrics + // endpoint, too: a scrape shouldn't fail while a replica boots. const probePath = routePathOf(request.nextUrl.pathname); - if (probePath === "/api/health" || probePath === "/api/ready") { + if (PROBE_PATHS.has(probePath)) { const probe = NextResponse.next(); probe.headers.set("X-Content-Type-Options", "nosniff"); probe.headers.set("X-Frame-Options", "DENY"); diff --git a/scripts/healthcheck.mjs b/scripts/healthcheck.mjs new file mode 100644 index 000000000..26ebbc359 --- /dev/null +++ b/scripts/healthcheck.mjs @@ -0,0 +1,15 @@ +// Docker HEALTHCHECK: exit 0 when /api/health/live answers 200 within 4 s. +// Liveness only, so a Stalwart outage never marks the container unhealthy. +// The base path is baked in at build time (BULWARK_BASE_PATH in the image). +// https://bulwarkmail.org/docs/admin/monitoring#health +const port = process.env.PORT || '3000'; +const listen = process.env.HOSTNAME; +const host = !listen || listen === '0.0.0.0' || listen === '::' ? '127.0.0.1' : listen.includes(':') ? `[${listen}]` : listen; +const base = (process.env.BULWARK_BASE_PATH || '').replace(/\/+$/, ''); + +try { + const res = await fetch(`http://${host}:${port}${base}/api/health/live`, { signal: AbortSignal.timeout(4000) }); + process.exit(res.ok ? 0 : 1); +} catch { + process.exit(1); +}