From 692f965f2064df1224fc0517caa8251f6bb817d8 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:40:25 +0100 Subject: [PATCH 1/6] fix: start with ADMIN_PASSWORD when the admin config is mounted read-only #1066 --- .../admin-password-read-only.test.ts | 59 +++++++++++++++++++ lib/admin/password.ts | 32 +++++++++- 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 lib/admin/__tests__/admin-password-read-only.test.ts diff --git a/lib/admin/__tests__/admin-password-read-only.test.ts b/lib/admin/__tests__/admin-password-read-only.test.ts new file mode 100644 index 000000000..bcad786ad --- /dev/null +++ b/lib/admin/__tests__/admin-password-read-only.test.ts @@ -0,0 +1,59 @@ +// @vitest-environment node +import { afterAll, afterEach, describe, expect, it, vi } from 'vitest'; +import { existsSync, rmSync } from 'node:fs'; +import path from 'node:path'; + +// A config dir mounted read-only (Kubernetes ConfigMap, a :ro volume) without +// an admin.json, and ADMIN_PASSWORD in the environment: the admin password +// works from memory on every replica instead of failing the start, which +// kept every replica of a shared-store deployment from ever becoming ready. + +// Hoisted: isConfigReadOnly() reads the env once. +const dir = await vi.hoisted(async () => { + const { mkdtempSync, mkdirSync } = await import('node:fs'); + const { tmpdir } = await import('node:os'); + const { join } = await import('node:path'); + const root = mkdtempSync(join(tmpdir(), 'bw-admin-pw-ro-')); + mkdirSync(join(root, 'config')); + process.env.ADMIN_CONFIG_DIR = join(root, 'config'); + process.env.ADMIN_STATE_DIR = join(root, 'state'); + process.env.ADMIN_CONFIG_READONLY = 'true'; + process.env.ADMIN_PASSWORD = 'from-the-environment'; + return root; +}); + +import { initAdminPassword, isAdminEnabled, resetAdminCacheForTests, verifyAdminPassword } from '../password'; +import { setStoreForTests } from '@/lib/shared-store'; +import { resetLocalStoreForTests } from '@/lib/shared-store/local'; +import { separateFakeStore } from './shared-store-fake'; + +afterEach(() => { + resetAdminCacheForTests(); + setStoreForTests(null); + resetLocalStoreForTests(); +}); + +afterAll(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +describe('ADMIN_PASSWORD with a read-only config and no admin.json', () => { + it('works from memory on a single instance and writes nothing to the config dir', async () => { + await expect(initAdminPassword()).resolves.toBe(true); + expect(isAdminEnabled()).toBe(true); + expect(await verifyAdminPassword('from-the-environment')).toBe(true); + expect(await verifyAdminPassword('wrong')).toBe(false); + expect(existsSync(path.join(dir, 'config', 'admin.json'))).toBe(false); + }); + + it('lets every replica of a shared store start and sign in', async () => { + setStoreForTests(separateFakeStore(path.join(dir, 'store'))); + for (let replica = 0; replica < 3; replica++) { + resetAdminCacheForTests(); + await expect(initAdminPassword()).resolves.toBe(true); + expect(await verifyAdminPassword('from-the-environment')).toBe(true); + expect(await verifyAdminPassword('wrong')).toBe(false); + } + expect(existsSync(path.join(dir, 'config', 'admin.json'))).toBe(false); + }); +}); diff --git a/lib/admin/password.ts b/lib/admin/password.ts index 592991bc9..bae73d0b5 100644 --- a/lib/admin/password.ts +++ b/lib/admin/password.ts @@ -2,7 +2,7 @@ import { scrypt, randomBytes, timingSafeEqual } from 'node:crypto'; import { logger } from '@/lib/logger'; import { isSharedMode, type DocCollection, type PutOptions, type PutResult, type StoredDoc } from '@/lib/shared-store'; import { announceChange, onSharedChange } from '@/lib/shared-store/changes'; -import { assertWritable } from './paths'; +import { assertWritable, isConfigReadOnly } from './paths'; import { adminConfigDocs, adminStateDocs, isStoreUnavailable } from './admin-store'; import type { AdminConfigData, AdminStateData } from './types'; import { readFileEnv } from '../read-file-env'; @@ -131,6 +131,12 @@ interface AdminCache { initialized: boolean; /** Shared mode: one read of the state for checks that run together. */ stateRead?: Promise; + /** + * The password came from ADMIN_PASSWORD into a read-only config without an + * admin.json: it lives in this process only, and every replica makes it + * from the same variable on start. + */ + fromEnvOnly?: boolean; unlisten?: () => void; } @@ -149,6 +155,7 @@ export function resetAdminCacheForTests(): void { cache.initialized = false; cache.stateRead = undefined; cache.unlisten = undefined; + cache.fromEnvOnly = undefined; } function freshState(): AdminStateData { @@ -271,6 +278,24 @@ export async function initAdminPassword(): Promise { const hash = isHashed(envPassword) ? envPassword : await hashPassword(envPassword); const config: AdminConfigData = { passwordHash: hash, passwordHashFile: undefined }; + if (isConfigReadOnly()) { + // Nowhere to save admin.json (a ConfigMap, a :ro volume). Keep the + // password in memory instead of failing the start; each replica does the + // same from the same variable. Changing it in the admin console is + // refused, as every other write to read-only config is. + cache.config = config; + cache.fromEnvOnly = true; + cache.state = shared ? await readStateData() : (await readStateData()) ?? freshState(); + if (shared && !cache.state) { + const fresh = freshState(); + cache.state = (await writeStateData(fresh, { ifVersion: null })).ok ? fresh : (await readStateData()) ?? fresh; + } + if (!cache.initialized) { + logger.info('Admin dashboard enabled with ADMIN_PASSWORD (config is read-only, so it is not saved to admin.json)'); + } + cache.initialized = true; + return true; + } if (shared) { // Replicas booting together with ADMIN_PASSWORD: the first one writes // admin.json and moves the session generation on, the others use it. @@ -319,6 +344,11 @@ function verifyAgainst(config: AdminConfigData, password: string): Promise { + if (cache.fromEnvOnly && cache.config) { + // A mounted admin.json that appeared later still wins. + const mounted = (await readConfigDoc())?.data ?? null; + return verifyAgainst(mounted ?? cache.config, password); + } if (isSharedMode()) { cache.config = (await readConfigDoc())?.data ?? null; } else if (!cache.config) { From d2a83a4ee9d2bf68d08ebc9895741c3bdcb719be Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:41:42 +0100 Subject: [PATCH 2/6] fix: stop proxying locale rewrites back to the server when it listens on 127.0.0.1 #1487 --- lib/__tests__/proxy-loopback-origin.test.ts | 76 +++++++++++++++++++++ next.config.ts | 9 +++ proxy.ts | 21 +++--- 3 files changed, 98 insertions(+), 8 deletions(-) create mode 100644 lib/__tests__/proxy-loopback-origin.test.ts diff --git a/lib/__tests__/proxy-loopback-origin.test.ts b/lib/__tests__/proxy-loopback-origin.test.ts new file mode 100644 index 000000000..4fa64e4a1 --- /dev/null +++ b/lib/__tests__/proxy-loopback-origin.test.ts @@ -0,0 +1,76 @@ +/** + * @vitest-environment node + * + * On HOSTNAME=127.0.0.1, Next renames the host in request.nextUrl to + * "localhost" while its router keeps 127.0.0.1. A proxy rewrite or redirect + * built from nextUrl then points at another origin: rewrites were proxied + * back to the server itself (over TLS when X-Forwarded-Proto said https, an + * EPROTO 500), and with skipProxyUrlNormalize redirects would send the + * browser to localhost (#1046, #1487). + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { NextRequest } from 'next/server'; + +let setupState: 'configured' | 'bootstrap' = 'configured'; + +vi.mock('@/lib/admin/config-manager', () => ({ + configManager: { + ensureLoaded: vi.fn(async () => {}), + get: vi.fn((_key: string, fallback: unknown) => fallback), + getPolicy: vi.fn(() => ({ features: {} })), + }, +})); +vi.mock('@/lib/setup/state', () => ({ detectSetupState: vi.fn(() => setupState) })); +vi.mock('@/lib/admin/csp-frame-origins', () => ({ getEnabledPluginFrameOrigins: vi.fn(async () => []) })); +vi.mock('next-intl/middleware', async () => { + const { NextResponse } = await import('next/server'); + return { default: () => () => NextResponse.next() }; +}); + +import { proxy } from '@/proxy'; + +beforeEach(() => { + // What the build inlines for skipProxyUrlNormalize: true. NextRequest then + // keeps request.url as given instead of the renamed nextUrl. + vi.stubEnv('__NEXT_NO_MIDDLEWARE_URL_NORMALIZE', 'true'); +}); + +afterEach(() => { + setupState = 'configured'; + vi.unstubAllEnvs(); + vi.resetModules(); +}); + +describe('proxy redirects on a loopback listen address', () => { + it('sends a configured install from /setup to /admin/login on the same origin', async () => { + const response = await proxy(new NextRequest('https://127.0.0.1:3000/setup')); + + expect(response.status).toBe(307); + expect(response.headers.get('location')).toBe('https://127.0.0.1:3000/admin/login'); + }); + + it('sends every page to /setup on the same origin while the wizard runs, keeping the query', async () => { + setupState = 'bootstrap'; + const response = await proxy(new NextRequest('http://[::1]:3000/mail?x=1')); + + expect(response.status).toBe(307); + expect(response.headers.get('location')).toBe('http://[::1]:3000/setup?x=1'); + }); + + it('keeps the base path of a sub-path install', async () => { + const request = new NextRequest('https://127.0.0.1:3000/webmail/setup', { + nextConfig: { basePath: '/webmail' }, + }); + const response = await proxy(request); + + expect(response.headers.get('location')).toBe('https://127.0.0.1:3000/webmail/admin/login'); + }); +}); + +describe('next.config', () => { + it('leaves proxy rewrites on the URL the router built', async () => { + vi.resetModules(); + const { default: config } = await import('../../next.config'); + expect(config.skipProxyUrlNormalize).toBe(true); + }); +}); diff --git a/next.config.ts b/next.config.ts index d039caec1..e8cfc98f7 100644 --- a/next.config.ts +++ b/next.config.ts @@ -52,6 +52,15 @@ const nextConfig: NextConfig = { // No `X-Powered-By: Next.js`; headers() below sends // `X-Powered-By: bulwarkmail.org` instead, without a version. poweredByHeader: false, + // Keep proxy rewrites on the origin the server listens on. Next renames a + // 127.x or [::1] host to "localhost" in each proxy rewrite, but its router + // compares the rewrite with the listen address. On HOSTNAME=127.0.0.1 every + // locale rewrite then looked external and was proxied back to the server + // itself, over TLS when X-Forwarded-Proto said https: EPROTO and a 500 + // behind a reverse proxy (#1046, #1487, vercel/next.js#94745). With this set, + // request.url is the URL the router built; proxy.ts builds its redirects + // from it as well. + skipProxyUrlNormalize: true, // Directory-style pages (out/en/mail/index.html) are what static hosts and // the shipped SPA-fallback rules expect; next/image has no optimizer here. ...(isLite ? { trailingSlash: true, images: { unoptimized: true } } : {}), diff --git a/proxy.ts b/proxy.ts index 12f1423ed..6660020bd 100644 --- a/proxy.ts +++ b/proxy.ts @@ -187,6 +187,17 @@ function isSetupPath(pathname: string): boolean { ); } +/** + * A redirect to `pathname` (without the base path) on the origin of + * request.url. Not request.nextUrl: Next renames a 127.x or [::1] host there + * to "localhost", and with skipProxyUrlNormalize (next.config.ts) Next no + * longer turns that back into a relative Location. On HOSTNAME=127.0.0.1 the + * browser would be sent to https://localhost:3000/... (#1487). + */ +function redirectTo(request: NextRequest, pathname: string, search: string): NextResponse { + return NextResponse.redirect(new URL(`${request.nextUrl.basePath}${pathname}${search}`, request.url)); +} + export async function proxy(request: NextRequest) { // The probes answer without config or store, so a replica that is still // starting (or lost its shared store) can say so (#1066). @@ -231,10 +242,7 @@ export async function proxy(request: NextRequest) { { status: 503, headers: { "content-type": "application/json" } }, ); } - const url = request.nextUrl.clone(); - url.pathname = "/setup"; - url.search = request.nextUrl.search; - return NextResponse.redirect(url); + return redirectTo(request, "/setup", request.nextUrl.search); } } else if (isSetupPath(pathname)) { // Configured / env-managed: wizard is no longer reachable. @@ -244,10 +252,7 @@ export async function proxy(request: NextRequest) { if (pathname.startsWith("/api/setup")) { return new NextResponse("Not Found", { status: 404 }); } - const url = request.nextUrl.clone(); - url.pathname = "/admin/login"; - url.search = ""; - return NextResponse.redirect(url); + return redirectTo(request, "/admin/login", ""); } // Outer CSRF gate for every API route (GHSA-qvr9-m8cq-7wvg, From 8308e5e858f05b3827a8f0c444e758a4d93cbeef Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:52:37 +0100 Subject: [PATCH 3/6] feat: add Kubernetes manifests for several replicas sharing one Valkey #1066 --- .gitignore | 3 + deploy/kubernetes/base/bulwark.env | 13 ++ deploy/kubernetes/base/bulwark.yaml | 144 ++++++++++++++++++ deploy/kubernetes/base/kustomization.yaml | 32 ++++ deploy/kubernetes/base/namespace.yaml | 4 + deploy/kubernetes/base/secrets.env.example | 13 ++ deploy/kubernetes/base/valkey.yaml | 95 ++++++++++++ deploy/kubernetes/ingress.example.yaml | 28 ++++ deploy/kubernetes/kustomization.yaml | 14 ++ .../config-in-git/config-from-files.yaml | 20 +++ .../overlays/config-in-git/config.json | 4 + .../overlays/config-in-git/kustomization.yaml | 23 +++ .../overlays/config-in-git/policy.json | 5 + deploy/kubernetes/test/smoke.sh | 120 +++++++++++++++ 14 files changed, 518 insertions(+) create mode 100644 deploy/kubernetes/base/bulwark.env create mode 100644 deploy/kubernetes/base/bulwark.yaml create mode 100644 deploy/kubernetes/base/kustomization.yaml create mode 100644 deploy/kubernetes/base/namespace.yaml create mode 100644 deploy/kubernetes/base/secrets.env.example create mode 100644 deploy/kubernetes/base/valkey.yaml create mode 100644 deploy/kubernetes/ingress.example.yaml create mode 100644 deploy/kubernetes/kustomization.yaml create mode 100644 deploy/kubernetes/overlays/config-in-git/config-from-files.yaml create mode 100644 deploy/kubernetes/overlays/config-in-git/config.json create mode 100644 deploy/kubernetes/overlays/config-in-git/kustomization.yaml create mode 100644 deploy/kubernetes/overlays/config-in-git/policy.json create mode 100644 deploy/kubernetes/test/smoke.sh diff --git a/.gitignore b/.gitignore index a67d67b1c..19e0f5fa8 100644 --- a/.gitignore +++ b/.gitignore @@ -55,3 +55,6 @@ next-env.d.ts # Sibling repos /repos/ + +# Kubernetes reference manifests: the secrets users fill in +deploy/kubernetes/base/secrets.env diff --git a/deploy/kubernetes/base/bulwark.env b/deploy/kubernetes/base/bulwark.env new file mode 100644 index 000000000..aa88c979d --- /dev/null +++ b/deploy/kubernetes/base/bulwark.env @@ -0,0 +1,13 @@ +# Settings for every Bulwark replica (no secrets here; those go in secrets.env). +# Any variable from https://bulwarkmail.org/docs/getting-started/configuration/environment-reference +# can be added. + +# The JMAP server users sign in to. Both the pods and users' browsers must reach it. +JMAP_SERVER_URL=https://mail.example.com + +# Proxies in front of the pods: 1 for an ingress controller alone, 2 if a cloud +# load balancer sits in front of it. Sign-in limits key on the address this picks. +TRUSTED_PROXY_DEPTH=1 + +SETTINGS_SYNC_ENABLED=true +LOG_LEVEL=info diff --git a/deploy/kubernetes/base/bulwark.yaml b/deploy/kubernetes/base/bulwark.yaml new file mode 100644 index 000000000..5d40e9c12 --- /dev/null +++ b/deploy/kubernetes/base/bulwark.yaml @@ -0,0 +1,144 @@ +# Bulwark itself: any number of replicas, no volumes. Everything they share +# (admin config, plugins, branding, synced settings, sign-in limits, sessions) +# lives in the Valkey store; see https://bulwarkmail.org/docs/deployment/multi-instance +apiVersion: v1 +kind: Service +metadata: + name: bulwark + labels: + app.kubernetes.io/name: bulwark +spec: + selector: + app.kubernetes.io/name: bulwark + ports: + - name: http + port: 80 + targetPort: http +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: bulwark + labels: + app.kubernetes.io/name: bulwark +spec: + replicas: 3 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + selector: + matchLabels: + app.kubernetes.io/name: bulwark + template: + metadata: + labels: + app.kubernetes.io/name: bulwark + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1001 + runAsGroup: 1001 + fsGroup: 1001 + seccompProfile: + type: RuntimeDefault + # Spread the replicas over nodes where there are several. + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: ScheduleAnyway + labelSelector: + matchLabels: + app.kubernetes.io/name: bulwark + terminationGracePeriodSeconds: 30 + containers: + - name: bulwark + image: ghcr.io/bulwarkmail/webmail + ports: + - name: http + containerPort: 3000 + envFrom: + - configMapRef: + name: bulwark-env + env: + - name: SESSION_SECRET + valueFrom: + secretKeyRef: + name: bulwark-secrets + key: SESSION_SECRET + - name: ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: bulwark-secrets + key: ADMIN_PASSWORD + - name: VALKEY_PASSWORD + valueFrom: + secretKeyRef: + name: bulwark-secrets + key: VALKEY_PASSWORD + # Kubernetes fills in $(VALKEY_PASSWORD) from the variable above. + - name: REDIS_URL + value: redis://:$(VALKEY_PASSWORD)@valkey:6379/0 + # /api/health only says the process runs; /api/ready also says it + # finished starting and reaches the store. A store outage takes the + # pods out of the Service without restarting them. + startupProbe: + httpGet: + path: /api/health + port: http + periodSeconds: 2 + failureThreshold: 60 + readinessProbe: + httpGet: + path: /api/ready + port: http + periodSeconds: 5 + failureThreshold: 2 + livenessProbe: + httpGet: + path: /api/health + port: http + periodSeconds: 10 + failureThreshold: 3 + lifecycle: + # Let the Service drop the pod before it stops taking requests. + preStop: + exec: + command: ["sleep", "5"] + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 768Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + # Scratch space only; nothing here needs to survive the pod. + - name: data + mountPath: /app/data + - name: next-cache + mountPath: /app/.next/cache + - name: tmp + mountPath: /tmp + volumes: + - name: data + emptyDir: {} + - name: next-cache + emptyDir: {} + - name: tmp + emptyDir: {} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: bulwark +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: bulwark diff --git a/deploy/kubernetes/base/kustomization.yaml b/deploy/kubernetes/base/kustomization.yaml new file mode 100644 index 000000000..e7b76c1ce --- /dev/null +++ b/deploy/kubernetes/base/kustomization.yaml @@ -0,0 +1,32 @@ +# The base the top-level kustomization.yaml and the overlays use. Change the +# namespace, the image tag (pin a release, never latest in production) and +# the replica count here or, better, in an overlay. secrets.env must exist +# next to this file (copy secrets.env.example). +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: bulwark + +resources: + - namespace.yaml + - valkey.yaml + - bulwark.yaml + +configMapGenerator: + - name: bulwark-env + envs: + - bulwark.env + +secretGenerator: + - name: bulwark-secrets + envs: + - secrets.env + +images: + - name: ghcr.io/bulwarkmail/webmail + newTag: latest + +labels: + - pairs: + app.kubernetes.io/part-of: bulwark + includeSelectors: false diff --git a/deploy/kubernetes/base/namespace.yaml b/deploy/kubernetes/base/namespace.yaml new file mode 100644 index 000000000..ff5226df2 --- /dev/null +++ b/deploy/kubernetes/base/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: bulwark diff --git a/deploy/kubernetes/base/secrets.env.example b/deploy/kubernetes/base/secrets.env.example new file mode 100644 index 000000000..24cb08cf1 --- /dev/null +++ b/deploy/kubernetes/base/secrets.env.example @@ -0,0 +1,13 @@ +# Copy to secrets.env (kept out of git) and fill in. Generate each with: +# openssl rand -base64 32 +# Or delete the secretGenerator in kustomization.yaml and create the Secret +# `bulwark-secrets` with these three keys some other way (External Secrets, +# Sealed Secrets, your CI). + +# Encrypts sessions, synced settings and plugin credentials. The same on every +# replica; changing it signs everyone out. +SESSION_SECRET= +# First admin password; hashed into the store on the first start. +ADMIN_PASSWORD= +# Password of the Valkey store. +VALKEY_PASSWORD= diff --git a/deploy/kubernetes/base/valkey.yaml b/deploy/kubernetes/base/valkey.yaml new file mode 100644 index 000000000..debc8bb92 --- /dev/null +++ b/deploy/kubernetes/base/valkey.yaml @@ -0,0 +1,95 @@ +# The shared store every Bulwark replica uses. One Valkey with persistence +# (AOF) and noeviction: an evicting policy would silently drop settings and +# admin config under memory pressure. A managed Redis or Valkey works as well; +# then delete this file from kustomization.yaml and point REDIS_URL at it. +apiVersion: v1 +kind: Service +metadata: + name: valkey + labels: + app.kubernetes.io/name: valkey +spec: + clusterIP: None + selector: + app.kubernetes.io/name: valkey + ports: + - name: redis + port: 6379 + targetPort: redis +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: valkey + labels: + app.kubernetes.io/name: valkey +spec: + serviceName: valkey + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: valkey + template: + metadata: + labels: + app.kubernetes.io/name: valkey + spec: + securityContext: + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + fsGroup: 999 + seccompProfile: + type: RuntimeDefault + containers: + - name: valkey + image: valkey/valkey:8-alpine + command: ["sh", "-c"] + args: + - >- + exec valkey-server + --requirepass "$VALKEY_PASSWORD" + --appendonly yes + --maxmemory 512mb + --maxmemory-policy noeviction + --dir /data + env: + - name: VALKEY_PASSWORD + valueFrom: + secretKeyRef: + name: bulwark-secrets + key: VALKEY_PASSWORD + ports: + - name: redis + containerPort: 6379 + readinessProbe: + exec: + command: ["sh", "-c", "REDISCLI_AUTH=\"$VALKEY_PASSWORD\" valkey-cli --no-auth-warning ping | grep -q PONG"] + periodSeconds: 5 + livenessProbe: + exec: + command: ["sh", "-c", "REDISCLI_AUTH=\"$VALKEY_PASSWORD\" valkey-cli --no-auth-warning ping | grep -q PONG"] + periodSeconds: 10 + failureThreshold: 6 + resources: + requests: + cpu: 50m + memory: 64Mi + limits: + memory: 640Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: data + mountPath: /data + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi diff --git a/deploy/kubernetes/ingress.example.yaml b/deploy/kubernetes/ingress.example.yaml new file mode 100644 index 000000000..fa94692a3 --- /dev/null +++ b/deploy/kubernetes/ingress.example.yaml @@ -0,0 +1,28 @@ +# Not applied by kustomization.yaml: adapt and apply it yourself, or use a +# Gateway API HTTPRoute instead. Plain round robin is fine; Bulwark needs no +# sticky sessions. Set TRUSTED_PROXY_DEPTH in bulwark.env to the number of +# proxies in front of the pods (1 for the ingress controller alone). +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: bulwark + namespace: bulwark + annotations: + # Attachments and plugin packages can be large. + nginx.ingress.kubernetes.io/proxy-body-size: 100m +spec: + ingressClassName: nginx + tls: + - hosts: [webmail.example.com] + secretName: bulwark-tls + rules: + - host: webmail.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: bulwark + port: + name: http diff --git a/deploy/kubernetes/kustomization.yaml b/deploy/kubernetes/kustomization.yaml new file mode 100644 index 000000000..006ce308b --- /dev/null +++ b/deploy/kubernetes/kustomization.yaml @@ -0,0 +1,14 @@ +# Bulwark Webmail on Kubernetes: three replicas sharing one Valkey store. +# https://bulwarkmail.org/docs/deployment/kubernetes +# +# cp base/secrets.env.example base/secrets.env # fill in the three secrets +# $EDITOR base/bulwark.env # at least JMAP_SERVER_URL +# kubectl apply -k . # or: -k overlays/config-in-git +# +# For your own changes (namespace, image tag, replicas, resources) add an +# overlay next to overlays/config-in-git that uses ../../base. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - base diff --git a/deploy/kubernetes/overlays/config-in-git/config-from-files.yaml b/deploy/kubernetes/overlays/config-in-git/config-from-files.yaml new file mode 100644 index 000000000..fe91e62f7 --- /dev/null +++ b/deploy/kubernetes/overlays/config-in-git/config-from-files.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: bulwark +spec: + template: + spec: + containers: + - name: bulwark + env: + - name: ADMIN_CONFIG_READONLY + value: "true" + volumeMounts: + - name: admin-config + mountPath: /app/data/admin + readOnly: true + volumes: + - name: admin-config + configMap: + name: bulwark-admin-config diff --git a/deploy/kubernetes/overlays/config-in-git/config.json b/deploy/kubernetes/overlays/config-in-git/config.json new file mode 100644 index 000000000..e0754da49 --- /dev/null +++ b/deploy/kubernetes/overlays/config-in-git/config.json @@ -0,0 +1,4 @@ +{ + "appName": "Webmail", + "setupComplete": true +} diff --git a/deploy/kubernetes/overlays/config-in-git/kustomization.yaml b/deploy/kubernetes/overlays/config-in-git/kustomization.yaml new file mode 100644 index 000000000..ab85f582b --- /dev/null +++ b/deploy/kubernetes/overlays/config-in-git/kustomization.yaml @@ -0,0 +1,23 @@ +# Admin config and user policy from files in git instead of the admin console. +# Every replica reads config.json and policy.json from the ConfigMap and picks +# up a change within a few seconds; the admin console shows them but can't +# save them. Sessions, settings and the audit log still live in the store. +# +# kubectl apply -k overlays/config-in-git +# +# Export your current policy from the admin console (Organisation > Export) +# into policy.json to start from it. +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base + +configMapGenerator: + - name: bulwark-admin-config + files: + - config.json + - policy.json + +patches: + - path: config-from-files.yaml diff --git a/deploy/kubernetes/overlays/config-in-git/policy.json b/deploy/kubernetes/overlays/config-in-git/policy.json new file mode 100644 index 000000000..d5a66620e --- /dev/null +++ b/deploy/kubernetes/overlays/config-in-git/policy.json @@ -0,0 +1,5 @@ +{ + "restrictions": {}, + "features": {}, + "defaults": {} +} diff --git a/deploy/kubernetes/test/smoke.sh b/deploy/kubernetes/test/smoke.sh new file mode 100644 index 000000000..9d097aaa4 --- /dev/null +++ b/deploy/kubernetes/test/smoke.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# Smoke test for a running deployment of deploy/kubernetes: +# - Valkey and every Bulwark replica become ready, on the shared store; +# - an admin session made on one replica is valid on every other one; +# - a config change saved on one replica reaches the others within 5 s; +# - deleting a pod and a rolling restart don't fail requests through the +# Service, and the admin session survives the new pods. +# +# Usage: ADMIN_PASSWORD=... deploy/kubernetes/test/smoke.sh [namespace] +# With READ_ONLY_CONFIG=1 (overlays/config-in-git) it checks that config saves +# are refused instead of spread. +# Uses the current kubectl context. The password goes to curl on stdin, never +# on a command line. +set -euo pipefail + +NS="${1:-bulwark}" +: "${ADMIN_PASSWORD:?set ADMIN_PASSWORD to the admin password of the deployment}" +k() { kubectl -n "$NS" "$@"; } +fail() { echo "FAIL: $*" >&2; exit 1; } +step() { echo "== $*"; } + +step "rollout" +k rollout status statefulset/valkey --timeout=300s +k rollout status deployment/bulwark --timeout=600s + +step "curl pod" +if ! k get pod smoke-curl >/dev/null 2>&1; then + k run smoke-curl --image=curlimages/curl:8.11.1 --restart=Never --labels=app.kubernetes.io/name=smoke-curl \ + --command -- sleep 1800 >/dev/null +fi +k wait pod/smoke-curl --for=condition=Ready --timeout=180s >/dev/null +c() { k exec -i smoke-curl -- curl -sS --max-time 10 "$@"; } +status() { k exec smoke-curl -- curl -s -o /dev/null -w '%{http_code}' --max-time 5 "$@" || echo 000; } + +pod_ips() { k get pods -l app.kubernetes.io/name=bulwark --field-selector=status.phase=Running -o jsonpath='{.items[*].status.podIP}'; } +read -r -a IPS <<<"$(pod_ips)" +[ "${#IPS[@]}" -ge 2 ] || fail "need at least 2 replicas, found ${#IPS[@]}" +echo "replicas: ${IPS[*]}" + +step "every replica is ready on the shared store" +for ip in "${IPS[@]}"; do + out="$(c "http://$ip:3000/api/ready")" + echo "$ip $out" + grep -q '"ready":true' <<<"$out" || fail "$ip not ready" + grep -q '"store":"redis"' <<<"$out" || fail "$ip not on the shared store" +done + +step "an admin session made on one replica works on every replica" +json_string() { + local s="${1//\\/\\\\}" + s="${s//\"/\\\"}" + printf '"%s"' "$s" +} +code="$(printf '{"password":%s}' "$(json_string "$ADMIN_PASSWORD")" \ + | c -o /dev/null -w '%{http_code}' -c /tmp/jar -H 'Content-Type: application/json' --data-binary @- \ + "http://${IPS[0]}:3000/api/admin/auth")" +[ "$code" = 200 ] || fail "admin sign-in on ${IPS[0]} answered $code" +# curl's jar keeps cookies per host and every pod has its own address, so the +# cookie goes in a header. It stays in the curl pod, off every command line. +k exec smoke-curl -- sh -c "awk '{ sub(/^#HttpOnly_/, \"\") } !/^#/ && NF >= 7 { printf \"%s=%s; \", \$6, \$7 }' /tmp/jar > /tmp/cookie" +ca() { k exec -i smoke-curl -- sh -c 'curl -sS --max-time 10 -H "Cookie: $(cat /tmp/cookie)" "$@"' sh "$@"; } +for ip in "${IPS[@]}"; do + ca "http://$ip:3000/api/admin/auth" /dev/null; do + total=$((total + 1)) + [ "$(status "$SVC")" = 200 ] || failures=$((failures + 1)) + sleep 1 + done + wait "$pid" + echo "requests: $total, failed: $failures" + [ "$failures" -eq 0 ] || fail "$failures requests failed" +} + +step "deleting a pod keeps the Service answering" +victim="$(k get pods -l app.kubernetes.io/name=bulwark -o jsonpath='{.items[0].metadata.name}')" +probe_while k delete pod "$victim" --wait=true +k rollout status deployment/bulwark --timeout=300s + +step "a rolling restart keeps the Service answering" +k rollout restart deployment/bulwark +probe_while k rollout status deployment/bulwark --timeout=600s + +step "the admin session survives the new pods" +ca "http://bulwark.$NS.svc/api/admin/auth" /dev/null +echo "OK" From cffa8bb1942483dd16f3fbc6aa55dc19b62b09b5 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:52:37 +0100 Subject: [PATCH 4/6] ci: deploy the Kubernetes manifests to kind and smoke-test several replicas --- .github/workflows/kubernetes.yml | 99 ++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 .github/workflows/kubernetes.yml diff --git a/.github/workflows/kubernetes.yml b/.github/workflows/kubernetes.yml new file mode 100644 index 000000000..d521c4341 --- /dev/null +++ b/.github/workflows/kubernetes.yml @@ -0,0 +1,99 @@ +name: "CI: Kubernetes manifests" + +# Builds the image from this commit, deploys deploy/kubernetes to a kind +# cluster (three replicas and one Valkey) and runs +# deploy/kubernetes/test/smoke.sh against it: every replica ready on the +# shared store, one admin session valid everywhere, a config change reaching +# every replica, no failed requests while a pod is deleted and during a +# rolling restart. The config-in-git overlay runs next to it in its own +# namespace and must refuse config saves. +# +# It runs when the manifests change, weekly on main (an app change can break +# several replicas without touching the manifests), and on demand: +# gh workflow run kubernetes.yml + +on: + pull_request: + paths: + - deploy/kubernetes/** + - .github/workflows/kubernetes.yml + push: + branches: [main] + paths: + - deploy/kubernetes/** + - .github/workflows/kubernetes.yml + schedule: + - cron: "41 3 * * 1" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: kubernetes-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + kind: + name: kind + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + KIND_VERSION: v0.29.0 + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Build the image + run: docker build --build-arg "GIT_COMMIT=${GITHUB_SHA}" -t bulwark-local:ci . + + - name: Create the cluster + run: | + curl -fsSLo /usr/local/bin/kind "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64" + chmod +x /usr/local/bin/kind + kind create cluster --name ci --wait 180s + kind load docker-image bulwark-local:ci --name ci + + - name: Deploy the base and the config-in-git overlay + run: | + umask 077 + admin="$(openssl rand -hex 16)" + echo "::add-mask::$admin" + { + echo "SESSION_SECRET=$(openssl rand -base64 32)" + echo "ADMIN_PASSWORD=$admin" + echo "VALKEY_PASSWORD=$(openssl rand -hex 24)" + } > deploy/kubernetes/base/secrets.env + echo "ADMIN_PASSWORD=$admin" >> "$GITHUB_ENV" + for target in base:bulwark overlays/config-in-git:bulwark-ro; do + src="${target%%:*}"; ns="${target##*:}" + mkdir -p "ci/$ns" + cat > "ci/$ns/kustomization.yaml" < Date: Sun, 1 Nov 2026 02:53:49 +0100 Subject: [PATCH 5/6] docs: name the policy export button in the config-in-git overlay --- deploy/kubernetes/overlays/config-in-git/kustomization.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/kubernetes/overlays/config-in-git/kustomization.yaml b/deploy/kubernetes/overlays/config-in-git/kustomization.yaml index ab85f582b..9f604dba2 100644 --- a/deploy/kubernetes/overlays/config-in-git/kustomization.yaml +++ b/deploy/kubernetes/overlays/config-in-git/kustomization.yaml @@ -5,8 +5,8 @@ # # kubectl apply -k overlays/config-in-git # -# Export your current policy from the admin console (Organisation > Export) -# into policy.json to start from it. +# To start from your current policy, use "Export policy.json" on the admin +# console's Domains and groups page. apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization From 110f7fa96a051b9651e1ebb3fe9792bf41d4f470 Mon Sep 17 00:00:00 2001 From: Linus Rath <139418639+rathlinus@users.noreply.github.com> Date: Sun, 1 Nov 2026 02:59:41 +0100 Subject: [PATCH 6/6] ci: install kind where the runner can make it executable --- .github/workflows/kubernetes.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/kubernetes.yml b/.github/workflows/kubernetes.yml index d521c4341..e62d2d215 100644 --- a/.github/workflows/kubernetes.yml +++ b/.github/workflows/kubernetes.yml @@ -49,8 +49,12 @@ jobs: - name: Create the cluster run: | - curl -fsSLo /usr/local/bin/kind "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64" - chmod +x /usr/local/bin/kind + # The runner's own kind in /usr/local/bin belongs to root; use a pinned one. + mkdir -p "$RUNNER_TEMP/bin" + curl -fsSLo "$RUNNER_TEMP/bin/kind" "https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64" + chmod +x "$RUNNER_TEMP/bin/kind" + echo "$RUNNER_TEMP/bin" >> "$GITHUB_PATH" + export PATH="$RUNNER_TEMP/bin:$PATH" kind create cluster --name ci --wait 180s kind load docker-image bulwark-local:ci --name ci