Merge branch 'feat/multi-instance' into feat/multi-instance-ext

app/(main)/admin/_pages/plugins.tsx

@@ -85,7 +85,14 @@ export function PluginsPage() {
       });
       if (res.ok) {
         policyETag.current = res.headers.get('ETag');
-        setSavedPolicy(policy);
+        // The server sanitises; continue from what it stored.
+        const stored = ((await res.json().catch(() => null)) as { policy?: SettingsPolicy } | null)?.policy;
+        if (stored) {
+          setPolicy(stored);
+          setSavedPolicy(stored);
+        } else {
+          setSavedPolicy(policy);
+        }
         return null;
       }
       const data = await res.json().catch(() => ({}));

app/(main)/admin/_pages/policy.tsx

@@ -229,7 +229,14 @@ export function PolicyPage() {
 
       if (res.ok) {
         policyETag.current = res.headers.get('ETag');
-        setSavedPolicy(policy);
+        // The server sanitises; continue from what it stored.
+        const stored = ((await res.json().catch(() => null)) as { policy?: SettingsPolicy } | null)?.policy;
+        if (stored) {
+          setPolicy(stored);
+          setSavedPolicy(stored);
+        } else {
+          setSavedPolicy(policy);
+        }
         setMessage({ type: 'success', text: 'Policy saved. Users see the change on their next login.' });
         setDirty(false);
       } else {

app/(main)/admin/_pages/themes.tsx

@@ -160,7 +160,14 @@ export function ThemesPage() {
       });
       if (res.ok) {
         policyETag.current = res.headers.get('ETag');
-        setSavedPolicy(policy);
+        // The server sanitises; continue from what it stored.
+        const stored = ((await res.json().catch(() => null)) as { policy?: SettingsPolicy } | null)?.policy;
+        if (stored) {
+          setPolicy(withThemeDefaults(stored));
+          setSavedPolicy(withThemeDefaults(stored));
+        } else {
+          setSavedPolicy(policy);
+        }
         return null;
       }
       const data = await res.json().catch(() => ({}));

app/api/account-archive/route.ts

@@ -11,6 +11,7 @@ import { rejectCrossOriginRequest } from '@/lib/security/same-origin';
 import { MAX_ACCOUNT_SLOTS } from '@/lib/account-utils';
 import { archiveIdentity, parseArchiveEnvelope, ARCHIVE_MAX_BYTES, type ArchiveOwner } from '@/lib/account-archive';
 import { deleteArchive, loadArchive, saveArchive } from '@/lib/account-archive-storage';
+import { isStoreUnavailable } from '@/lib/stored-data-errors';
 
 export const runtime = 'nodejs';
 
@@ -18,6 +19,11 @@ const reply = (data: unknown, status = 200) => NextResponse.json(data, {
   status, headers: { 'Cache-Control': 'no-store' },
 });
 
+/** A shared store that can't be reached is a 503 the client retries; nothing was written. */
+const storageFailed = (error: unknown) => isStoreUnavailable(error)
+  ? NextResponse.json({ error: 'storage_unavailable' }, { status: 503, headers: { 'Cache-Control': 'no-store', 'Retry-After': '10' } })
+  : reply({ error: 'storage_failed' }, 500);
+
 /**
  * The owner whose archive this request may touch, or the response refusing it.
  *
@@ -67,7 +73,7 @@ export async function GET(request: NextRequest) {
   const owner = await verifiedOwner(request);
   if (owner instanceof NextResponse) return owner;
   try { return reply({ envelope: await loadArchive(owner) }); }
-  catch { return reply({ error: 'storage_failed' }, 500); }
+  catch (error) { return storageFailed(error); }
 }
 
 export async function PUT(request: NextRequest) {
@@ -83,7 +89,7 @@ export async function PUT(request: NextRequest) {
     envelope = parseArchiveEnvelope((JSON.parse(text) as { envelope?: unknown })?.envelope);
   } catch { return reply({ error: 'invalid_archive' }, 400); }
   try { await saveArchive(owner, envelope); return reply({ ok: true }); }
-  catch { return reply({ error: 'storage_failed' }, 500); }
+  catch (error) { return storageFailed(error); }
 }
 
 export async function DELETE(request: NextRequest) {
@@ -92,5 +98,5 @@ export async function DELETE(request: NextRequest) {
   const owner = await verifiedOwner(request);
   if (owner instanceof NextResponse) return owner;
   try { await deleteArchive(owner); return reply({ ok: true }); }
-  catch { return reply({ error: 'storage_failed' }, 500); }
+  catch (error) { return storageFailed(error); }
 }

app/api/admin/__tests__/config-merge-shared.test.ts

@@ -0,0 +1,99 @@
+// @vitest-environment node
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { NextRequest } from 'next/server';
+
+// Some admin saves change one entry of a list: the per-domain branding, the
+// client secrets of the JMAP servers. Built on this instance's copy of the
+// list, they dropped what another instance (or a parallel upload here) had
+// saved a moment before (#1066).
+
+vi.mock('@/lib/admin/session', () => ({
+  requireAdminAuth: async () => ({ payload: { role: 'admin' } }),
+  getClientIP: () => '127.0.0.1',
+}));
+vi.mock('@/lib/admin/audit', () => ({ auditLog: async () => {} }));
+
+import { POST as uploadBranding, DELETE as deleteBranding } from '../branding/route';
+import { PATCH as patchConfig } from '../config/route';
+import { configManager } from '@/lib/admin/config-manager';
+import { adminConfigDocs } from '@/lib/admin/admin-store';
+import { setStoreForTests } from '@/lib/shared-store';
+import { resetLocalStoreForTests } from '@/lib/shared-store/local';
+import { INIT_EPOCH, markEpochsSeen, resetChangesForTests } from '@/lib/shared-store/changes';
+import { separateFakeStore } from '@/lib/admin/__tests__/shared-store-fake';
+
+let dir: string;
+const stored = async () => JSON.parse((await adminConfigDocs().get('config.json'))!.data.toString('utf-8'));
+/** Another instance saved this; its notice hasn't arrived here yet. */
+const savedElsewhere = (config: Record<string, unknown>) => adminConfigDocs().put('config.json', JSON.stringify(config));
+
+function upload(host: string) {
+  const form = new FormData();
+  form.set('file', new File([new Uint8Array([0x89, 0x50, 0x4e, 0x47])], 'logo.png', { type: 'image/png' }));
+  form.set('slot', 'faviconUrl');
+  form.set('host', host);
+  return uploadBranding(new NextRequest('https://mail.example/api/admin/branding', { method: 'POST', body: form }));
+}
+
+beforeEach(async () => {
+  dir = await mkdtemp(path.join(tmpdir(), 'bw-config-merge-'));
+  vi.stubEnv('ADMIN_CONFIG_DIR', path.join(dir, 'config'));
+  vi.stubEnv('ADMIN_STATE_DIR', path.join(dir, 'state'));
+  resetLocalStoreForTests();
+  resetChangesForTests();
+  const store = separateFakeStore(path.join(dir, 'store'));
+  setStoreForTests(store);
+  await store.epochs.bump(INIT_EPOCH);
+  await savedElsewhere({ setupComplete: true });
+  await markEpochsSeen();
+  await configManager.load();
+});
+
+afterEach(async () => {
+  setStoreForTests(null);
+  resetChangesForTests();
+  resetLocalStoreForTests();
+  vi.unstubAllEnvs();
+  await rm(dir, { recursive: true, force: true });
+});
+
+describe('list saves with a shared store', () => {
+  it('keeps another instance\'s domain branding when uploading for another domain', async () => {
+    await savedElsewhere({ setupComplete: true, domainBranding: [{ host: 'a.example', faviconUrl: '/api/admin/branding/a.png' }] });
+    expect((await upload('b.example')).status).toBe(200);
+    const hosts = (await stored()).domainBranding.map((e: { host: string }) => e.host).sort();
+    expect(hosts).toEqual(['a.example', 'b.example']);
+
+    await savedElsewhere({ ...(await stored()), domainBranding: [...(await stored()).domainBranding, { host: 'c.example', faviconUrl: '/x.png' }] });
+    const res = await deleteBranding(new NextRequest('https://mail.example/api/admin/branding', {
+      method: 'DELETE',
+      body: JSON.stringify({ slot: 'faviconUrl', host: 'b.example' }),
+    }));
+    expect(res.status).toBe(200);
+    expect((await stored()).domainBranding.map((e: { host: string }) => e.host)).toContain('c.example');
+  });
+
+  it('keeps both of two uploads for different domains at once', async () => {
+    const results = await Promise.all([upload('a.example'), upload('b.example')]);
+    expect(results.map((r) => r.status)).toEqual([200, 200]);
+    expect((await stored()).domainBranding.map((e: { host: string }) => e.host).sort()).toEqual(['a.example', 'b.example']);
+  });
+
+  it('keeps a JMAP server\'s client secret another instance just saved', async () => {
+    const server = { id: 'corp', label: 'Corp', url: 'https://mail.corp.example', oauth: { clientId: 'webmail' } };
+    await savedElsewhere({ setupComplete: true, jmapServers: [{ ...server, oauth: { clientId: 'webmail', clientSecret: 'secret-saved-elsewhere' } }] });
+    // The dashboard never has the secret, so it sends the entry without it.
+    const res = await patchConfig(new NextRequest('https://mail.example/api/admin/config', {
+      method: 'PATCH',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({ jmapServers: [{ ...server, label: 'Corp mail' }], appName: 'Renamed' }),
+    }));
+    expect(res.status).toBe(200);
+    const config = await stored();
+    expect(config.appName).toBe('Renamed');
+    expect(config.jmapServers[0]).toMatchObject({ label: 'Corp mail', oauth: { clientId: 'webmail', clientSecret: 'secret-saved-elsewhere' } });
+  });
+});

app/api/admin/__tests__/policy-bearer-route.test.ts

@@ -28,6 +28,7 @@ vi.mock('@/lib/admin/config-manager', () => ({
       organisation: { disclaimer: { html: '<p>Acme GmbH</p>', text: 'Acme GmbH' } },
       scopes: SCOPES,
     }),
+    getPolicyWithVersion() { return { policy: this.getPolicy(), version: 'v1' }; },
   },
 }));
 const logs = vi.hoisted(() => ({ calls: [] as unknown[][] }));

app/api/admin/__tests__/policy-etag-race.test.ts

@@ -0,0 +1,84 @@
+// @vitest-environment node
+// GET /api/admin/policy waits on the store for the identity checks. A
+// change from another replica applied meanwhile must not pair the policy
+// read before with the version read after: a page saving that body with
+// that ETag would put the old policy back over the other replica's save.
+import { afterEach, beforeEach, expect, it, vi } from 'vitest';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { NextRequest } from 'next/server';
+
+const hooks: { duringIdentity?: () => Promise<void> } = {};
+vi.mock('@/lib/auth/instance-details', () => ({
+  canSeeInstanceDetails: async () => true,
+  signedInIdentity: async () => {
+    // Stands in for the store round trip verifyCurrentAdminSession makes.
+    await hooks.duringIdentity?.();
+    return { admin: true };
+  },
+}));
+vi.mock('@/lib/admin/session', () => ({
+  requireAdminAuth: async () => ({ payload: { role: 'admin' } }),
+  getClientIP: () => '127.0.0.1',
+}));
+vi.mock('@/lib/admin/audit', () => ({ auditLog: async () => {} }));
+
+import { configManager } from '@/lib/admin/config-manager';
+import { adminConfigDocs } from '@/lib/admin/admin-store';
+import { setBootState, setStoreForTests } from '@/lib/shared-store';
+import { resetLocalStoreForTests } from '@/lib/shared-store/local';
+import { INIT_EPOCH, markEpochsSeen, pollChanges, resetChangesForTests } from '@/lib/shared-store/changes';
+import { GET, PUT } from '@/app/api/admin/policy/route';
+import { separateFakeStore } from '@/lib/admin/__tests__/shared-store-fake';
+
+let dir: string;
+let store: ReturnType<typeof separateFakeStore>;
+
+beforeEach(async () => {
+  dir = await mkdtemp(path.join(tmpdir(), 'bw-review-etag-'));
+  vi.stubEnv('ADMIN_CONFIG_DIR', path.join(dir, 'config'));
+  vi.stubEnv('ADMIN_STATE_DIR', path.join(dir, 'state'));
+  resetLocalStoreForTests();
+  resetChangesForTests();
+  store = separateFakeStore(path.join(dir, 'store'));
+  setStoreForTests(store);
+  await store.epochs.bump(INIT_EPOCH);
+  await adminConfigDocs().put('policy.json', JSON.stringify({ features: { calendarEnabled: true } }));
+  await markEpochsSeen();
+  await configManager.load();
+  setBootState('ready');
+});
+
+afterEach(async () => {
+  hooks.duringIdentity = undefined;
+  setStoreForTests(null);
+  resetChangesForTests();
+  resetLocalStoreForTests();
+  vi.unstubAllEnvs();
+  await rm(dir, { recursive: true, force: true });
+});
+
+it('sends the version of the policy it sends, when a change lands during the request', async () => {
+  hooks.duringIdentity = async () => {
+    // Another replica saves calendarEnabled:false; the poll lands mid-GET.
+    await adminConfigDocs().put('policy.json', JSON.stringify({ features: { calendarEnabled: false, filesEnabled: false } }));
+    await store.epochs.bump('policy');
+    await pollChanges();
+  };
+  const res = await GET(new NextRequest('https://mail.example/api/admin/policy'));
+  hooks.duringIdentity = undefined;
+  const body = await res.json();
+  const etag = res.headers.get('ETag')!;
+  // Old body...
+  expect(body.features.calendarEnabled).toBe(true);
+  // ...saved with the ETag it came with: refused, the other save stays.
+  const put = await PUT(new NextRequest('https://mail.example/api/admin/policy', {
+    method: 'PUT',
+    headers: { 'content-type': 'application/json', 'if-match': etag },
+    body: JSON.stringify(body),
+  }));
+  expect(put.status).toBe(409);
+  const stored = JSON.parse((await adminConfigDocs().get('policy.json'))!.data.toString('utf-8'));
+  expect(stored.features).toEqual({ calendarEnabled: false, filesEnabled: false });
+});

app/api/admin/__tests__/policy-if-match.test.ts

@@ -94,6 +94,22 @@ describe('policy save with If-Match', () => {
     expect((await stored()).forceEnabledPlugins).toEqual([]);
   });
 
+  it('drops a deleted plugin without undoing a save another replica made on the same volume', async () => {
+    await load();
+    await writeFile(path.join(dir, 'policy.json'), JSON.stringify({ features: { calendarEnabled: false }, forceEnabledPlugins: ['demo', 'other'] }));
+    await forgetExtensionInPolicy('plugin', 'demo');
+    expect((await stored()).forceEnabledPlugins).toEqual(['other']);
+    expect((await stored()).features.calendarEnabled).toBe(false);
+  });
+
+  it('answers a save with what it stored', async () => {
+    const page = await load();
+    const res = await save({ ...page.policy, features: { ...(page.policy.features as object), calendarEnabled: false } }, page.etag);
+    const body = await res.json();
+    expect(body.policy.features.calendarEnabled).toBe(false);
+    expect(res.headers.get('ETag')).toBe((await load()).etag);
+  });
+
   it('replaces the policy without If-Match, as before', async () => {
     await load();
     await configManager.setPolicy({ ...configManager.getPolicy(), forceEnabledPlugins: [] });

app/api/admin/__tests__/policy-scopes-route.test.ts

@@ -29,6 +29,7 @@ vi.mock('@/lib/admin/config-manager', () => ({
       scopes: SCOPES,
     }),
     getPolicyVersion: () => 'v1',
+    getPolicyWithVersion() { return { policy: this.getPolicy(), version: 'v1' }; },
   },
 }));
 vi.mock('@/lib/logger', () => ({ logger: { error: vi.fn(), warn: vi.fn(), info: vi.fn(), debug: vi.fn() } }));

app/api/admin/__tests__/read-only-config-module-copies.test.ts

@@ -0,0 +1,53 @@
+/**
+ * @vitest-environment node
+ */
+import { afterAll, describe, expect, it, vi } from 'vitest';
+import { rmSync } from 'node:fs';
+import { NextRequest } from 'next/server';
+
+// In the server the instrumentation creates the config manager, so a write
+// it refuses for ADMIN_CONFIG_READONLY throws the instrumentation's copy of
+// ConfigReadOnlyError, not the route's. The route has to answer 409 anyway
+// (#1249), not 500.
+
+const dir = await vi.hoisted(async () => {
+  const { mkdtempSync } = await import('node:fs');
+  const { tmpdir } = await import('node:os');
+  const { join } = await import('node:path');
+  const config = mkdtempSync(join(tmpdir(), 'bw-readonly-copies-'));
+  process.env.ADMIN_CONFIG_DIR = config;
+  process.env.ADMIN_STATE_DIR = join(config, 'state');
+  process.env.ADMIN_CONFIG_READONLY = 'true';
+  return config;
+});
+
+vi.mock('@/lib/admin/session', () => ({
+  requireAdminAuth: async () => ({ payload: { role: 'admin' } }),
+  getClientIP: () => '127.0.0.1',
+}));
+vi.mock('@/lib/admin/audit', () => ({ auditLog: async () => {} }));
+
+afterAll(() => {
+  delete process.env.ADMIN_CONFIG_READONLY;
+  rmSync(dir, { recursive: true, force: true });
+});
+
+describe('read-only config across module copies', () => {
+  it('answers 409 when the config manager came from another bundle', async () => {
+    const KEY = Symbol.for('bulwark.admin.configManager');
+    const g = globalThis as Record<symbol, unknown>;
+    delete g[KEY];
+    // The instrumentation's copy makes the config manager.
+    await (await import('@/lib/admin/config-manager')).configManager.load();
+    // The route's own copy.
+    vi.resetModules();
+    const { PATCH } = await import('@/app/api/admin/config/route');
+    const res = await PATCH(new NextRequest('http://localhost/api/admin/config', {
+      method: 'PATCH',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({ appName: 'Nope' }),
+    }));
+    expect(res.status).toBe(409);
+    expect((await res.json()).error).toMatch(/read-only/);
+  });
+});

app/api/admin/__tests__/store-down-writes.test.ts

@@ -0,0 +1,113 @@
+// @vitest-environment node
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { mkdtemp, rm } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { NextRequest } from 'next/server';
+
+// While the shared store can't be reached (#1066) nothing can be saved. The
+// admin and setup routes say so with a 503 and Retry-After, so the panel
+// and the wizard can tell "try again" from a bug, instead of a bare 500.
+
+const jar = new Map<string, string>();
+vi.mock('next/headers', () => ({
+  cookies: async () => ({
+    get: (name: string) => (jar.has(name) ? { name, value: jar.get(name)! } : undefined),
+    set: (name: string, value: string) => { jar.set(name, value); },
+    delete: (name: string) => { jar.delete(name); },
+  }),
+}));
+vi.mock('@/lib/admin/session', async (importOriginal) => ({
+  ...(await importOriginal<typeof import('@/lib/admin/session')>()),
+  requireAdminAuth: async () => ({ payload: { role: 'admin' } }),
+}));
+vi.mock('@/lib/admin/audit', () => ({ auditLog: async () => {} }));
+
+import { PATCH as patchConfig, DELETE as deleteConfig } from '../config/route';
+import { PUT as putPolicy } from '../policy/route';
+import { POST as importPolicy } from '../policy/file/route';
+import { POST as changePassword } from '../change-password/route';
+import { POST as setupStep } from '@/app/api/setup/step/route';
+import { POST as setupToken } from '@/app/api/setup/token/route';
+import { GET as setupStatus } from '@/app/api/setup/status/route';
+import { configManager } from '@/lib/admin/config-manager';
+import { adminConfigDocs } from '@/lib/admin/admin-store';
+import { setStoreForTests } from '@/lib/shared-store';
+import { resetLocalStoreForTests } from '@/lib/shared-store/local';
+import { INIT_EPOCH, markEpochsSeen, resetChangesForTests } from '@/lib/shared-store/changes';
+import { ensureSetupToken } from '@/lib/setup/token';
+import { separateFakeStore } from '@/lib/admin/__tests__/shared-store-fake';
+
+let dir: string;
+let store: ReturnType<typeof separateFakeStore>;
+
+const json = (url: string, method: string, body: unknown) => new NextRequest(`https://mail.example${url}`, {
+  method,
+  headers: { 'content-type': 'application/json', 'sec-fetch-site': 'same-origin' },
+  body: JSON.stringify(body),
+});
+
+async function expect503(res: Response) {
+  expect(res.status).toBe(503);
+  expect(res.headers.get('Retry-After')).toBe('10');
+  expect((await res.json()).error).toMatch(/try again/i);
+}
+
+beforeEach(async () => {
+  jar.clear();
+  dir = await mkdtemp(path.join(tmpdir(), 'bw-store-down-'));
+  vi.stubEnv('ADMIN_CONFIG_DIR', path.join(dir, 'config'));
+  vi.stubEnv('ADMIN_STATE_DIR', path.join(dir, 'state'));
+  vi.stubEnv('JMAP_SERVER_URL', '');
+  vi.stubEnv('SESSION_SECRET', 'a'.repeat(64));
+  resetLocalStoreForTests();
+  resetChangesForTests();
+  store = separateFakeStore(path.join(dir, 'store'));
+  setStoreForTests(store);
+  await store.epochs.bump(INIT_EPOCH);
+  await markEpochsSeen();
+});
+
+afterEach(async () => {
+  setStoreForTests(null);
+  resetChangesForTests();
+  resetLocalStoreForTests();
+  vi.unstubAllEnvs();
+  await rm(dir, { recursive: true, force: true });
+});
+
+describe('admin saves while the shared store is down', () => {
+  beforeEach(async () => {
+    await adminConfigDocs().put('config.json', JSON.stringify({ setupComplete: true }));
+    await configManager.load();
+    store.down = true;
+  });
+
+  it('answers 503 for config changes', async () => {
+    await expect503(await patchConfig(json('/api/admin/config', 'PATCH', { appName: 'Nope' })));
+    await expect503(await deleteConfig(json('/api/admin/config', 'DELETE', { key: 'appName' })));
+  });
+
+  it('answers 503 for policy saves and imports', async () => {
+    await expect503(await putPolicy(json('/api/admin/policy', 'PUT', { features: {} })));
+    await expect503(await importPolicy(json('/api/admin/policy/file', 'POST', { features: {} })));
+  });
+
+  it('answers 503 for a password change', async () => {
+    await expect503(await changePassword(json('/api/admin/change-password', 'POST', { currentPassword: 'old-password-1', newPassword: 'new-password-2' })));
+  });
+});
+
+describe('the setup wizard while the shared store is down', () => {
+  beforeEach(async () => {
+    await configManager.load();
+    jar.set('bulwark_setup_token', await ensureSetupToken());
+    store.down = true;
+  });
+
+  it('answers 503 instead of failing on the token check', async () => {
+    await expect503(await setupStep(json('/api/setup/step', 'POST', { step: 'branding', data: { appName: 'Nope' } })));
+    await expect503(await setupToken(json('/api/setup/token', 'POST', { token: jar.get('bulwark_setup_token') })));
+    await expect503(await setupStatus());
+  });
+});

app/api/admin/auth/route.ts

@@ -283,7 +283,18 @@ export async function DELETE(request: NextRequest) {
       return NextResponse.json({ error: 'Cross-origin request rejected' }, { status: 403 });
     }
     const ip = getClientIP(request);
-    await clearAdminSessionCookie();
+    try {
+      await clearAdminSessionCookie();
+    } catch (error) {
+      if (!isStoreUnavailable(error)) throw error;
+      // The cookie is gone from this browser, but the sign-out couldn't be
+      // stored, so a copy of the cookie isn't refused yet.
+      await auditLog('admin.logout', { stored: false }, ip);
+      return NextResponse.json(
+        { error: 'Signed out in this browser, but the sign-out could not be saved, so a copy of the session still works until it expires. Try signing out again in a moment.' },
+        { status: 503, headers: { 'Retry-After': '10' } },
+      );
+    }
     await auditLog('admin.logout', {}, ip);
     return NextResponse.json({ ok: true });
   } catch (error) {

app/api/admin/branding/route.ts

@@ -193,9 +193,10 @@ export async function POST(request: NextRequest) {
     await configManager.ensureLoaded();
 
     if (host) {
-      const current = parseDomainBranding(configManager.get<unknown>('domainBranding', []));
-      const next = mergeDomainEntry(current, host, { [slot]: servedUrl });
-      await configManager.setAdminConfig({ domainBranding: next });
+      // Built on the stored list, so a save of another domain at the same
+      // time (here or on another instance) stays.
+      await configManager.updateAdminConfig<unknown>('domainBranding', (current) =>
+        mergeDomainEntry(parseDomainBranding(current), host, { [slot]: servedUrl }), []);
     } else {
       await configManager.setAdminConfig({ [slot]: servedUrl });
     }
@@ -269,9 +270,8 @@ export async function DELETE(request: NextRequest) {
 
     await configManager.ensureLoaded();
     if (host) {
-      const current = parseDomainBranding(configManager.get<unknown>('domainBranding', []));
-      const next = clearDomainKeys(current, host, [slot as BrandingOverrideKey]);
-      await configManager.setAdminConfig({ domainBranding: next });
+      await configManager.updateAdminConfig<unknown>('domainBranding', (current) =>
+        clearDomainKeys(parseDomainBranding(current), host, [slot as BrandingOverrideKey]), []);
     } else {
       await configManager.removeAdminOverride(slot);
     }

app/api/admin/change-password/route.ts

@@ -1,5 +1,5 @@
 import { NextRequest, NextResponse } from 'next/server';
-import { changeAdminPassword } from '@/lib/admin/password';
+import { changeAdminPassword, isAdminPasswordConflict } from '@/lib/admin/password';
 import { requireAdminAuth, getClientIP, setAdminSessionCookie } from '@/lib/admin/session';
 import { auditLog } from '@/lib/admin/audit';
 import { logger } from '@/lib/logger';
@@ -35,6 +35,7 @@ export async function POST(request: NextRequest) {
     await auditLog('admin.change-password', {}, ip);
     return NextResponse.json({ ok: true });
   } catch (error) {
+    if (isAdminPasswordConflict(error)) return NextResponse.json({ error: error.message }, { status: 409 });
     const readOnly = configReadOnlyResponse(error);
     if (readOnly) return readOnly;
     logger.error('Admin change password error', { error: error instanceof Error ? error.message : 'Unknown error' });

app/api/admin/config/route.ts

@@ -135,6 +135,7 @@ export async function PATCH(request: NextRequest) {
 
     // Normalize jmapServers: pass through the parser so invalid entries are
     // rejected (bad ids, duplicate ids, non-HTTP URLs) before they're persisted.
+    let jmapServers: unknown[] | null = null;
     if ('jmapServers' in updates) {
       const incoming = updates.jmapServers;
       if (incoming != null && !Array.isArray(incoming)) {
@@ -151,6 +152,8 @@ export async function PATCH(request: NextRequest) {
           error: 'One or more jmapServers entries are invalid (each needs a unique id, label, and HTTP(S) url).',
         }, { status: 400 });
       }
+      // Saved on its own below, with the secrets of the stored list.
+      jmapServers = Array.isArray(incoming) ? incoming : [];
       updates.jmapServers = sanitized;
     }
 
@@ -179,7 +182,18 @@ export async function PATCH(request: NextRequest) {
       oldValues[key] = configManager.get(key);
     }
 
-    await configManager.setAdminConfig(updates);
+    if (jmapServers) {
+      // The secrets the dashboard didn't send come from the stored list, not
+      // this instance's copy: a secret saved elsewhere a moment ago stays.
+      const { jmapServers: _servers, ...rest } = updates;
+      void _servers;
+      if (Object.keys(rest).length > 0) await configManager.setAdminConfig(rest);
+      const incoming = jmapServers;
+      updates.jmapServers = await configManager.updateAdminConfig<unknown>('jmapServers', (current) =>
+        parseJmapServers(withStoredClientSecrets(incoming, parseJmapServers(current))), []);
+    } else {
+      await configManager.setAdminConfig(updates);
+    }
     await auditLog('config.update', { changes: Object.keys(updates).map(k => ({ key: k, old: oldValues[k], new: updates[k] })) }, ip);
 
     return NextResponse.json({ ok: true });

app/api/admin/oauth/setup/route.ts

@@ -7,6 +7,7 @@ import { configManager } from '@/lib/admin/config-manager';
 import { isConfigReadOnly, ConfigReadOnlyError } from '@/lib/admin/paths';
 import { auditLog } from '@/lib/admin/audit';
 import { logger } from '@/lib/logger';
+import { configWriteErrorResponse } from '@/lib/admin/config-read-only-response';
 import { locales as ALL_LOCALES } from '@/i18n/routing';
 
 const CLIENT_ID = 'bulwark-webmail';
@@ -384,6 +385,8 @@ export async function POST(request: NextRequest) {
       mobileRedirectUri: MOBILE_REDIRECT_URI,
     });
   } catch (error) {
+    const refused = configWriteErrorResponse(error);
+    if (refused) return refused;
     logger.error('Admin OAuth setup error', { error: error instanceof Error ? error.message : 'Unknown error' });
     return NextResponse.json(
       { error: error instanceof Error ? error.message : 'Internal server error' },

app/api/admin/policy/route.ts

@@ -1,5 +1,5 @@
 import { NextRequest, NextResponse } from 'next/server';
-import { configManager, isPolicyConflict } from '@/lib/admin/config-manager';
+import { completePolicy, configManager, isPolicyConflict } from '@/lib/admin/config-manager';
 import { requireAdminAuth, getClientIP } from '@/lib/admin/session';
 import { auditLog } from '@/lib/admin/audit';
 import { logger } from '@/lib/logger';
@@ -37,7 +37,9 @@ function ifMatchVersion(request: NextRequest): string | undefined {
 export async function GET(request: NextRequest) {
   try {
     await configManager.ensureLoaded();
-    const policy = configManager.getPolicy();
+    // Together: the identity checks below wait on the store, and a change
+    // applied meanwhile must not pair this policy with a newer version.
+    const { policy, version } = configManager.getPolicyWithVersion();
     if (!(await canSeeInstanceDetails(request))) {
       const tokenIdentity = await bearerIdentity(request);
       if (tokenIdentity) return userPolicyResponse(policy, tokenIdentity.addresses);
@@ -70,7 +72,7 @@ export async function GET(request: NextRequest) {
     // The whole policy, as the admin pages edit it: with its version, which
     // they send back as If-Match when they save.
     return NextResponse.json({ ...policy, scopes: policy.scopes ?? [] }, {
-      headers: { 'Cache-Control': 'no-store', ETag: policyETag(configManager.getPolicyVersion()) },
+      headers: { 'Cache-Control': 'no-store', ETag: policyETag(version) },
     });
   } catch (error) {
     logger.error('Policy read error', { error: error instanceof Error ? error.message : 'Unknown error' });
@@ -146,7 +148,10 @@ export async function PUT(request: NextRequest) {
       defaultSidebarAppCount: configManager.getPolicy().defaultSidebarApps?.length ?? 0,
     }, ip);
 
-    return NextResponse.json({ ok: true }, { headers: { ETag: policyETag(version) } });
+    // What was stored, with its version: the page continues from this
+    // instead of reading it back from a replica that may not have it yet.
+    const saved = completePolicy(policy);
+    return NextResponse.json({ ok: true, policy: { ...saved, scopes: saved.scopes ?? [] } }, { headers: { ETag: policyETag(version) } });
   } catch (error) {
     const readOnly = configReadOnlyResponse(error);
     if (readOnly) return readOnly;

app/api/admin/telemetry/route.ts

@@ -4,8 +4,7 @@ import { auditLog } from '@/lib/admin/audit';
 import { logger } from '@/lib/logger';
 import {
   effectiveConsent,
-  loadState,
-  saveState,
+  updateState,
   buildPayload,
   sendOnce,
   reschedule,
@@ -13,6 +12,7 @@ import {
   getLoginCounts,
   resolveEndpointAllowed,
 } from '@/lib/telemetry';
+import { storedDataUnavailableResponse } from '@/lib/stored-data-errors';
 
 /**
  * GET /api/admin/telemetry
@@ -48,6 +48,8 @@ export async function GET(request: NextRequest) {
     logger.error('telemetry GET error', {
       error: err instanceof Error ? err.message : 'unknown',
     });
+    const unavailable = storedDataUnavailableResponse(err);
+    if (unavailable) return unavailable;
     return NextResponse.json({ error: 'Internal server error' }, { status: 500 });
   }
 }
@@ -90,13 +92,15 @@ export async function POST(request: NextRequest) {
       if (body.consent !== 'on' && body.consent !== 'off') {
         return NextResponse.json({ error: 'consent must be "on" or "off"' }, { status: 400 });
       }
-      const state = await loadState();
-      const before = state.consent;
-      state.consent = body.consent;
-      if (body.consent === 'on' && !state.consentedAt) {
-        state.consentedAt = new Date().toISOString();
-      }
-      await saveState(state);
+      const consent = body.consent;
+      const now = new Date().toISOString();
+      let before: string | undefined;
+      // Against the stored state, so a schedule written meanwhile stays.
+      await updateState((state) => {
+        before = state.consent;
+        state.consent = consent;
+        if (consent === 'on' && !state.consentedAt) state.consentedAt = now;
+      });
       await reschedule();
       await auditLog('telemetry.set-consent', { from: before, to: body.consent }, ip);
       return NextResponse.json({ ok: true });
@@ -113,11 +117,12 @@ export async function POST(request: NextRequest) {
           return NextResponse.json({ error: check.reason }, { status: 400 });
         }
       }
-      const state = await loadState();
-      const before = state.endpoint;
+      let before: string | undefined;
       // Empty means no collector: heartbeats stop until a URL is set again.
-      state.endpoint = trimmed;
-      await saveState(state);
+      const state = await updateState((current) => {
+        before = current.endpoint;
+        current.endpoint = trimmed;
+      });
       await reschedule();
       await auditLog('telemetry.set-endpoint', { from: before, to: state.endpoint }, ip);
       return NextResponse.json({ ok: true, endpoint: state.endpoint });
@@ -138,6 +143,8 @@ export async function POST(request: NextRequest) {
     logger.error('telemetry POST error', {
       error: err instanceof Error ? err.message : 'unknown',
     });
+    const unavailable = storedDataUnavailableResponse(err);
+    if (unavailable) return unavailable;
     return NextResponse.json({ error: 'Internal server error' }, { status: 500 });
   }
 }

app/api/admin/version/route.ts

@@ -10,6 +10,7 @@ import {
   DEFAULT_VERSION_ENDPOINT,
 } from '@/lib/version-check';
 import { isNoThirdPartyTrafficOn } from '@/lib/admin/third-party-mode';
+import { storedDataUnavailableResponse } from '@/lib/stored-data-errors';
 
 /**
  * GET /api/admin/version
@@ -40,6 +41,8 @@ export async function GET(request: NextRequest) {
     logger.error('version admin GET error', {
       error: err instanceof Error ? err.message : 'unknown',
     });
+    const unavailable = storedDataUnavailableResponse(err, { 'Cache-Control': 'no-store' });
+    if (unavailable) return unavailable;
     return NextResponse.json({ error: 'failed' }, { status: 500 });
   }
 }
@@ -64,6 +67,8 @@ export async function POST(req: NextRequest) {
     logger.error('version admin POST error', {
       error: err instanceof Error ? err.message : 'unknown',
     });
+    const unavailable = storedDataUnavailableResponse(err, { 'Cache-Control': 'no-store' });
+    if (unavailable) return unavailable;
     return NextResponse.json({ error: 'failed' }, { status: 500 });
   }
 }

app/api/plugin-secrets/route.ts

@@ -12,6 +12,7 @@ import {
   putUserCredential,
   type VaultUser,
 } from '@/lib/plugin-secrets/vault';
+import { storedDataUnavailableResponse } from '@/lib/stored-data-errors';
 
 const PLUGIN_ID_RE = /^[a-z0-9][a-z0-9-]*[a-z0-9]$/;
 const MAX_SECRET_LENGTH = 4096;
@@ -35,7 +36,14 @@ export async function GET(request: NextRequest) {
   if (pluginId !== undefined && !PLUGIN_ID_RE.test(pluginId)) {
     return NextResponse.json({ error: 'Invalid plugin id' }, { status: 400 });
   }
-  const credentials = await listUserCredentials(user, pluginId);
+  let credentials;
+  try {
+    credentials = await listUserCredentials(user, pluginId);
+  } catch (error) {
+    const unavailable = storedDataUnavailableResponse(error, NO_STORE);
+    if (unavailable) return unavailable;
+    throw error;
+  }
   // For one plugin, also say which of its admin credentials are set up, and
   // on which servers: the host page checks the links a plugin shows against
   // them (the plugin itself only gets yes/no, see proxyAccounts).
@@ -93,6 +101,8 @@ export async function POST(request: NextRequest) {
     });
   } catch (error) {
     if (error instanceof VaultFullError) return NextResponse.json({ error: error.message }, { status: 409 });
+    const unavailable = storedDataUnavailableResponse(error, NO_STORE);
+    if (unavailable) return unavailable;
     throw error;
   }
   return NextResponse.json({ ok: true }, { headers: NO_STORE });
@@ -108,7 +118,13 @@ export async function DELETE(request: NextRequest) {
   const credentialId = request.nextUrl.searchParams.get('credential') ?? undefined;
   if (!PLUGIN_ID_RE.test(pluginId)) return bad('Invalid plugin id');
   if (credentialId !== undefined && !CREDENTIAL_ID_RE.test(credentialId)) return bad('Invalid credential id');
-  await deleteUserCredentials(user, pluginId, credentialId);
+  try {
+    await deleteUserCredentials(user, pluginId, credentialId);
+  } catch (error) {
+    const unavailable = storedDataUnavailableResponse(error, NO_STORE);
+    if (unavailable) return unavailable;
+    throw error;
+  }
   return NextResponse.json({ ok: true }, { headers: NO_STORE });
 }
 

app/api/settings/route.ts

@@ -5,13 +5,14 @@ import { logger } from '@/lib/logger';
 import { decryptSession } from '@/lib/auth/crypto';
 import { sessionCookieName } from '@/lib/auth/session-cookie';
 import { readStalwartAuthContextFromStore } from '@/lib/stalwart/auth-context';
-import { saveUserSettings, loadUserSettings, deleteUserSettings } from '@/lib/settings-sync';
+import { updateUserSettings, loadUserSettings, deleteUserSettings } from '@/lib/settings-sync';
 import { configManager } from '@/lib/admin/config-manager';
 import { resolvePolicy } from '@/lib/admin/policy-scopes';
 import { hasSessionSecret } from '@/lib/auth/session-secret';
 import { MAX_ACCOUNT_SLOTS } from '@/lib/account-utils';
 import { mergeStoredTemplates } from '@/lib/template-merge';
 import { mergeStoredSubscriptions } from '@/lib/calendar-subscription-sync';
+import { storedDataUnavailableResponse } from '@/lib/stored-data-errors';
 
 /** Largest settings blob a client may store (templates and signatures included). */
 const SETTINGS_MAX_BYTES = 2 * 1024 * 1024;
@@ -141,6 +142,8 @@ export async function GET(request: NextRequest) {
     const message = error instanceof Error ? error.message : 'Unknown error';
     const code = (error as NodeJS.ErrnoException).code;
     logger.error('Settings load error', { error: message, code });
+    const unavailable = storedDataUnavailableResponse(error);
+    if (unavailable) return unavailable;
     const classified = classifyError(error);
     return NextResponse.json({ error: classified.message }, { status: classified.status });
   }
@@ -210,17 +213,22 @@ export async function POST(request: NextRequest) {
 
     // Templates and iCal subscriptions are merged into the stored blob rather
     // than replaced, so a stale tab or device cannot drop ones another added.
-    const stored = await loadUserSettings(accountName, storageServerUrl(serverUrl));
-    await saveUserSettings(
+    // In one step against what is stored now: two saves at once (two tabs,
+    // two replicas) each keep the other's additions.
+    await updateUserSettings(
       accountName,
       storageServerUrl(serverUrl),
-      mergeStoredSubscriptions(stored, mergeStoredTemplates(stored, filteredSettings)),
+      (stored) => mergeStoredSubscriptions(stored, mergeStoredTemplates(stored, filteredSettings)),
     );
     return NextResponse.json({ ok: true });
   } catch (error) {
     const message = error instanceof Error ? error.message : 'Unknown error';
     const code = (error as NodeJS.ErrnoException).code;
     logger.error('Settings save error', { error: message, code });
+    // Stored settings that don't decrypt, or no store: write nothing, so
+    // the user's real settings survive and the client keeps its copy.
+    const unavailable = storedDataUnavailableResponse(error);
+    if (unavailable) return unavailable;
     const classified = classifyError(error);
     return NextResponse.json({ error: classified.message }, { status: classified.status });
   }
@@ -252,6 +260,8 @@ export async function DELETE(request: NextRequest) {
     const message = error instanceof Error ? error.message : 'Unknown error';
     const code = (error as NodeJS.ErrnoException).code;
     logger.error('Settings delete error', { error: message, code });
+    const unavailable = storedDataUnavailableResponse(error);
+    if (unavailable) return unavailable;
     const classified = classifyError(error);
     return NextResponse.json({ error: classified.message }, { status: classified.status });
   }

app/api/setup/__tests__/step-session-secret.test.ts

@@ -11,7 +11,7 @@ import { NextRequest } from 'next/server';
 
 const setAdminConfig = vi.hoisted(() => vi.fn(async (_updates: Record<string, unknown>) => {}));
 vi.mock('@/lib/setup/state', () => ({ detectSetupState: () => 'bootstrap' }));
-vi.mock('@/lib/setup/session', () => ({ authenticateWizardRequest: async () => true }));
+vi.mock('@/lib/setup/session', () => ({ authenticateWizardRequest: async () => true, wizardSessionRefusal: async () => null }));
 vi.mock('@/lib/admin/config-manager', () => ({
   configManager: { ensureLoaded: async () => {}, setAdminConfig },
 }));
@@ -19,6 +19,7 @@ vi.mock('@/lib/telemetry', () => ({
   effectiveConsent: async () => ({ source: 'env' }),
   loadState: async () => ({}),
   saveState: async () => {},
+  updateState: async () => ({}),
   reschedule: async () => {},
 }));
 

app/api/setup/__tests__/step-telemetry-consent.test.ts

@@ -0,0 +1,39 @@
+/**
+ * @vitest-environment node
+ */
+import { describe, it, expect, vi } from 'vitest';
+import { NextRequest } from 'next/server';
+
+// The wizard's telemetry choice used to load the whole state, change it and
+// save it back, putting its copy over anything written meanwhile (the
+// scheduler leader storing the next heartbeat, on this or another replica).
+// It changes only the consent fields of what is stored now.
+
+vi.mock('@/lib/setup/state', () => ({ detectSetupState: () => 'bootstrap' }));
+vi.mock('@/lib/setup/session', () => ({ authenticateWizardRequest: async () => true }));
+vi.mock('@/lib/admin/config-manager', () => ({
+  configManager: { ensureLoaded: async () => {}, setAdminConfig: async () => {} },
+}));
+const stored = vi.hoisted(() => ({ consent: 'off', consentedAt: null as string | null, nextScheduledAt: 'kept' }));
+vi.mock('@/lib/telemetry', () => ({
+  effectiveConsent: async () => ({ source: 'file' }),
+  loadState: async () => { throw new Error('read-modify-write of the whole state'); },
+  saveState: async () => { throw new Error('read-modify-write of the whole state'); },
+  updateState: async (change: (s: typeof stored) => void) => { change(stored); return stored; },
+  reschedule: async () => {},
+}));
+
+import { POST } from '../step/route';
+
+describe('POST /api/setup/step (security, telemetry)', () => {
+  it('turns consent on against the stored state', async () => {
+    const res = await POST(new NextRequest('http://localhost/api/setup/step', {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({ step: 'security', values: { telemetryConsent: 'on' } }),
+    }));
+    expect(res.status).toBe(200);
+    expect(stored).toMatchObject({ consent: 'on', nextScheduledAt: 'kept' });
+    expect(stored.consentedAt).not.toBeNull();
+  });
+});

app/api/setup/branding/route.ts

@@ -3,7 +3,8 @@ import { writeFile, unlink, mkdir } from 'node:fs/promises';
 import { existsSync } from 'node:fs';
 import path from 'node:path';
 import { detectSetupState } from '@/lib/setup/state';
-import { authenticateWizardRequest } from '@/lib/setup/session';
+import { wizardSessionRefusal } from '@/lib/setup/session';
+import { configWriteErrorResponse } from '@/lib/admin/config-read-only-response';
 import { configManager } from '@/lib/admin/config-manager';
 import { getConfigDir, assertWritable } from '@/lib/admin/paths';
 import { logger } from '@/lib/logger';
@@ -66,9 +67,8 @@ export async function POST(request: NextRequest) {
   if (detectSetupState() !== 'bootstrap') {
     return NextResponse.json({ error: 'Setup is not active' }, { status: 404 });
   }
-  if (!(await authenticateWizardRequest())) {
-    return NextResponse.json({ error: 'Wizard session required' }, { status: 401 });
-  }
+  const wizardRefused = await wizardSessionRefusal();
+  if (wizardRefused) return wizardRefused;
 
   try {
     assertWritable('upload branding asset');
@@ -121,6 +121,8 @@ export async function POST(request: NextRequest) {
 
     return NextResponse.json({ url: servedUrl, filename: safeName });
   } catch (error) {
+    const refused = configWriteErrorResponse(error);
+    if (refused) return refused;
     logger.error('Wizard branding upload failed', {
       error: error instanceof Error ? error.message : 'Unknown error',
     });
@@ -138,9 +140,8 @@ export async function DELETE(request: NextRequest) {
   if (detectSetupState() !== 'bootstrap') {
     return NextResponse.json({ error: 'Setup is not active' }, { status: 404 });
   }
-  if (!(await authenticateWizardRequest())) {
-    return NextResponse.json({ error: 'Wizard session required' }, { status: 401 });
-  }
+  const wizardRefused = await wizardSessionRefusal();
+  if (wizardRefused) return wizardRefused;
 
   try {
     assertWritable('remove branding asset');
@@ -162,6 +163,8 @@ export async function DELETE(request: NextRequest) {
 
     return NextResponse.json({ ok: true });
   } catch (error) {
+    const refused = configWriteErrorResponse(error);
+    if (refused) return refused;
     logger.error('Wizard branding delete failed', {
       error: error instanceof Error ? error.message : 'Unknown error',
     });

app/api/setup/finish/route.ts

@@ -1,7 +1,8 @@
 import { NextRequest, NextResponse } from 'next/server';
 import { writeFile } from 'node:fs/promises';
 import { detectSetupState } from '@/lib/setup/state';
-import { authenticateWizardRequest, SETUP_COOKIE } from '@/lib/setup/session';
+import { wizardSessionRefusal, SETUP_COOKIE } from '@/lib/setup/session';
+import { configWriteErrorResponse } from '@/lib/admin/config-read-only-response';
 import { configManager } from '@/lib/admin/config-manager';
 import { setInitialAdminPassword } from '@/lib/admin/password';
 import { clearSetupToken } from '@/lib/setup/token';
@@ -27,9 +28,8 @@ export async function POST(request: NextRequest) {
   if (detectSetupState() !== 'bootstrap') {
     return NextResponse.json({ error: 'Setup is not active' }, { status: 404 });
   }
-  if (!(await authenticateWizardRequest())) {
-    return NextResponse.json({ error: 'Wizard session required' }, { status: 401 });
-  }
+  const wizardRefused = await wizardSessionRefusal();
+  if (wizardRefused) return wizardRefused;
 
   let body: { adminPassword?: unknown; lockConfig?: unknown };
   try {
@@ -105,6 +105,8 @@ export async function POST(request: NextRequest) {
     response.cookies.delete(SETUP_COOKIE);
     return response;
   } catch (error) {
+    const refused = configWriteErrorResponse(error);
+    if (refused) return refused;
     logger.error('Wizard finish failed', {
       error: error instanceof Error ? error.message : 'Unknown error',
     });

app/api/setup/status/route.ts

@@ -1,6 +1,7 @@
 import { NextResponse } from 'next/server';
 import { detectSetupState } from '@/lib/setup/state';
-import { authenticateWizardRequest } from '@/lib/setup/session';
+import { authenticateWizardRequest, setupStoreUnavailableResponse } from '@/lib/setup/session';
+import { isStoreUnavailableError } from '@/lib/shared-store';
 import { configManager } from '@/lib/admin/config-manager';
 import { isConfigReadOnly } from '@/lib/admin/paths';
 import { SENSITIVE_CONFIG_KEYS } from '@/lib/admin/types';
@@ -19,7 +20,13 @@ export const dynamic = 'force-dynamic';
 export async function GET() {
   await configManager.ensureLoaded();
   const state = detectSetupState();
-  const authenticated = state === 'bootstrap' ? await authenticateWizardRequest() : false;
+  let authenticated = false;
+  try {
+    authenticated = state === 'bootstrap' ? await authenticateWizardRequest() : false;
+  } catch (error) {
+    if (isStoreUnavailableError(error)) return setupStoreUnavailableResponse();
+    throw error;
+  }
 
   let partialConfig: Record<string, unknown> | null = null;
   if (state === 'bootstrap' && authenticated) {

app/api/setup/step/route.ts

@@ -1,10 +1,11 @@
 import { NextRequest, NextResponse } from 'next/server';
 import { detectSetupState } from '@/lib/setup/state';
-import { authenticateWizardRequest } from '@/lib/setup/session';
+import { wizardSessionRefusal } from '@/lib/setup/session';
+import { configWriteErrorResponse } from '@/lib/admin/config-read-only-response';
 import { configManager } from '@/lib/admin/config-manager';
 import { CONFIG_ENV_MAP } from '@/lib/admin/types';
 import { parseJmapServers } from '@/lib/admin/jmap-servers';
-import { effectiveConsent, loadState, saveState, reschedule } from '@/lib/telemetry';
+import { effectiveConsent, updateState, reschedule } from '@/lib/telemetry';
 import { logger } from '@/lib/logger';
 
 export const dynamic = 'force-dynamic';
@@ -58,9 +59,8 @@ export async function POST(request: NextRequest) {
   if (detectSetupState() !== 'bootstrap') {
     return NextResponse.json({ error: 'Setup is not active' }, { status: 404 });
   }
-  if (!(await authenticateWizardRequest())) {
-    return NextResponse.json({ error: 'Wizard session required' }, { status: 401 });
-  }
+  const wizardRefused = await wizardSessionRefusal();
+  if (wizardRefused) return wizardRefused;
 
   let body: { step?: unknown; values?: unknown };
   try {
@@ -93,12 +93,12 @@ export async function POST(request: NextRequest) {
     // A BULWARK_TELEMETRY env var hard-locks the choice; don't fight it.
     const { source } = await effectiveConsent();
     if (source !== 'env') {
-      const tstate = await loadState();
-      tstate.consent = consent;
-      if (consent === 'on' && !tstate.consentedAt) {
-        tstate.consentedAt = new Date().toISOString();
-      }
-      await saveState(tstate);
+      const now = new Date().toISOString();
+      // Against the stored state, so a schedule written meanwhile stays.
+      await updateState((tstate) => {
+        tstate.consent = consent;
+        if (consent === 'on' && !tstate.consentedAt) tstate.consentedAt = now;
+      });
       await reschedule();
     }
   }
@@ -144,6 +144,8 @@ export async function POST(request: NextRequest) {
     await configManager.setAdminConfig(updates);
     return NextResponse.json({ ok: true });
   } catch (error) {
+    const refused = configWriteErrorResponse(error);
+    if (refused) return refused;
     logger.error('Wizard step save failed', {
       step,
       error: error instanceof Error ? error.message : 'Unknown error',

app/api/setup/test-jmap/route.ts

@@ -1,6 +1,6 @@
 import { NextRequest, NextResponse } from 'next/server';
 import { detectSetupState } from '@/lib/setup/state';
-import { authenticateWizardRequest } from '@/lib/setup/session';
+import { wizardSessionRefusal } from '@/lib/setup/session';
 
 export const dynamic = 'force-dynamic';
 
@@ -52,9 +52,8 @@ export async function POST(request: NextRequest) {
   if (detectSetupState() !== 'bootstrap') {
     return NextResponse.json({ error: 'Setup is not active' }, { status: 404 });
   }
-  if (!(await authenticateWizardRequest())) {
-    return NextResponse.json({ error: 'Wizard session required' }, { status: 401 });
-  }
+  const wizardRefused = await wizardSessionRefusal();
+  if (wizardRefused) return wizardRefused;
 
   let body: { url?: unknown };
   try {

app/api/setup/token/route.ts

@@ -1,7 +1,8 @@
 import { NextRequest, NextResponse } from 'next/server';
 import { detectSetupState } from '@/lib/setup/state';
 import { verifySetupToken } from '@/lib/setup/token';
-import { buildSessionCookieAttributes } from '@/lib/setup/session';
+import { buildSessionCookieAttributes, setupStoreUnavailableResponse } from '@/lib/setup/session';
+import { isStoreUnavailableError } from '@/lib/shared-store';
 
 export const dynamic = 'force-dynamic';
 
@@ -29,7 +30,13 @@ export async function POST(request: NextRequest) {
     return NextResponse.json({ error: 'Token required' }, { status: 400 });
   }
 
-  const ok = await verifySetupToken(submitted);
+  let ok: boolean;
+  try {
+    ok = await verifySetupToken(submitted);
+  } catch (error) {
+    if (isStoreUnavailableError(error)) return setupStoreUnavailableResponse();
+    throw error;
+  }
   if (!ok) {
     // Don't differentiate between "wrong token" and "no token issued" - the
     // operator either has it from the logs or they don't.

app/api/system/update-status/route.ts

@@ -2,6 +2,7 @@ import { NextRequest, NextResponse } from 'next/server';
 import { checkOnce, freshStatus, loadState } from '@/lib/version-check';
 import { configManager } from '@/lib/admin/config-manager';
 import { canSeeInstanceDetails } from '@/lib/auth/instance-details';
+import { storedDataUnavailableResponse } from '@/lib/stored-data-errors';
 
 // Public endpoint that returns the latest cached update status. Fed by the
 // background scheduler started in instrumentation.node.ts; in production we
@@ -13,11 +14,18 @@ import { canSeeInstanceDetails } from '@/lib/auth/instance-details';
 // requiring a dev-server restart. The 5s upstream timeout in fetchStatus
 // caps the worst-case latency added to a dev reload.
 export async function GET(request: NextRequest) {
-  if (process.env.NODE_ENV === 'development') {
-    await checkOnce({ reason: 'dev-reload' });
+  let state;
+  try {
+    if (process.env.NODE_ENV === 'development') {
+      await checkOnce({ reason: 'dev-reload' });
+    }
+    state = await loadState();
+  } catch (err) {
+    // The shared store is down: the banner stays as the browser has it.
+    const unavailable = storedDataUnavailableResponse(err, { 'Cache-Control': 'no-store' });
+    if (unavailable) return unavailable;
+    throw err;
   }
-
-  const state = await loadState();
   // freshStatus drops a status persisted by an earlier build - after an
   // upgrade it would still advertise the version we just upgraded from (#913).
   let status = freshStatus(state);

components/admin/org/use-org-policy-draft.ts

@@ -58,20 +58,24 @@ export function useOrgPolicyDraft(dirtyId: string) {
   const scopeId = useOrgScopeStore((s) => s.scopeId);
   const setScopeId = useOrgScopeStore((s) => s.setScopeId);
 
+  /** Start editing from `data`, the policy stored at version `etag`. */
+  const adopt = useCallback((data: SettingsPolicy, etag: string | null) => {
+    policyETag.current = etag;
+    const normalised = { ...DEFAULT_POLICY, ...data, organisation: sanitizeOrganisationPolicy(data.organisation), scopes: data.scopes ?? [] };
+    setPolicy(normalised);
+    setSaved(clone(normalised));
+  }, []);
+
   const load = useCallback(async () => {
     try {
       const res = await apiFetch('/api/admin/policy');
       if (!res.ok) throw new Error();
-      const data = (await res.json()) as SettingsPolicy;
-      policyETag.current = res.headers.get('ETag');
-      const normalised = { ...DEFAULT_POLICY, ...data, organisation: sanitizeOrganisationPolicy(data.organisation), scopes: data.scopes ?? [] };
-      setPolicy(normalised);
-      setSaved(clone(normalised));
+      adopt((await res.json()) as SettingsPolicy, res.headers.get('ETag'));
     } catch {
       setMessage({ type: 'error', text: 'Could not load the policy' });
       setPolicy((p) => p ?? { ...DEFAULT_POLICY, organisation: clone(DEFAULT_ORGANISATION_POLICY), scopes: [] });
     }
-  }, [setMessage]);
+  }, [adopt, setMessage]);
 
   useEffect(() => { void load(); }, [load]);
 
@@ -194,8 +198,11 @@ export function useOrgPolicyDraft(dirtyId: string) {
         body: JSON.stringify(policy),
       });
       if (res.ok) {
-        // The server sanitises; show what it kept.
-        await load();
+        // The server sanitises; show what it kept. From the answer, not a new
+        // read, which another instance might serve before it has the change.
+        const stored = ((await res.json().catch(() => null)) as { policy?: SettingsPolicy } | null)?.policy;
+        if (stored) adopt(stored, res.headers.get('ETag'));
+        else await load();
         setMessage({ type: 'success', text: 'Saved. Users get the change the next time Bulwark loads.' });
       } else {
         const data = await res.json().catch(() => ({}));

instrumentation.node.ts

@@ -80,16 +80,19 @@ migrateLegacyAdminLayout()
     // or with BULWARK_TELEMETRY=on; BULWARK_TELEMETRY=off or an empty
     // collector URL keeps it from sending.
     // See https://bulwarkmail.org/docs/legal/privacy/telemetry
-    const { startScheduler, markProcessStart } = await import("./lib/telemetry");
+    const { startScheduler, stopScheduler, markProcessStart } = await import("./lib/telemetry");
     markProcessStart();
-    await startScheduler();
-  })
-  .then(async () => {
     // Hourly check against version.telemetry.bulwarkmail.org. Disable with
     // BULWARK_UPDATE_CHECK=off or override the endpoint with
     // BULWARK_UPDATE_CHECK_URL.
-    const { startScheduler } = await import("./lib/version-check");
-    await startScheduler();
+    const versionCheck = await import("./lib/version-check");
+    // Both run on one replica of the deployment, the scheduler leader
+    // (#1066); a single instance always leads.
+    const { startSchedulerLeader } = await import("./lib/scheduler-leader");
+    await startSchedulerLeader([
+      { name: "telemetry", start: startScheduler, stop: stopScheduler },
+      { name: "version-check", start: versionCheck.startScheduler, stop: versionCheck.stopScheduler },
+    ]);
   })
   .catch((err) => {
     console.warn("Admin dashboard init skipped:", err instanceof Error ? err.message : err);

lib/__tests__/account-archive-shared-store.test.ts

@@ -0,0 +1,120 @@
+// @vitest-environment node
+import { mkdtempSync, readdirSync, readFileSync, rmSync, statSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { open } from 'node:fs/promises';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { NextRequest } from 'next/server';
+import { setStoreForTests } from '@/lib/shared-store';
+import { resetLocalStoreForTests } from '@/lib/shared-store/local';
+import { separateDiskStore, type SeparateDiskStore } from './helpers/separate-disk-store';
+
+// Account archives with a shared store (#1066): one document per owner,
+// replaced whole, the same on every replica.
+
+const mocks = vi.hoisted(() => ({ contexts: new Map<number, unknown>(), verify: vi.fn() }));
+vi.mock('@/lib/admin/config-manager', () => ({ configManager: {
+  ensureLoaded: async () => {},
+  get: (_key: string, fallback: unknown) => fallback,
+  getPolicy: () => ({ features: { accountArchiveEnabled: true } }),
+} }));
+vi.mock('@/lib/auth/session-secret', () => ({ hasSessionSecret: () => true }));
+vi.mock('next/headers', () => ({ cookies: async () => ({}) }));
+vi.mock('@/lib/stalwart/auth-context', () => ({
+  readStalwartAuthContextFromStore: (_: unknown, slot: number) => mocks.contexts.get(slot) ?? null,
+}));
+vi.mock('@/lib/auth/verify-jmap-auth', async importOriginal => ({
+  ...await importOriginal<typeof import('@/lib/auth/verify-jmap-auth')>(), verifyJmapAuth: mocks.verify,
+}));
+
+import { deleteArchive, loadArchive, saveArchive } from '@/lib/account-archive-storage';
+import { GET, PUT } from '@/app/api/account-archive/route';
+import type { ArchiveEnvelope } from '@/lib/account-archive';
+
+const owner = { username: 'owner@example.com', serverUrl: 'https://mail.example.com' };
+const envelope: ArchiveEnvelope = { version: 1, iterations: 600000, salt: 'A'.repeat(22) + '==', iv: 'A'.repeat(16), ciphertext: 'A'.repeat(43) + '=' };
+const other: ArchiveEnvelope = { ...envelope, ciphertext: 'B'.repeat(43) + '=' };
+
+let disks: string;
+let store: SeparateDiskStore;
+
+beforeEach(() => {
+  disks = mkdtempSync(path.join(tmpdir(), 'bw-archive-replicas-'));
+  vi.stubEnv('SETTINGS_DATA_DIR', path.join(disks, 'a'));
+  store = separateDiskStore();
+  setStoreForTests(store);
+  mocks.contexts.clear();
+  mocks.contexts.set(0, { username: owner.username, serverUrl: owner.serverUrl, authHeader: 'Basic ' + Buffer.from(`${owner.username}:pw`).toString('base64') });
+  mocks.verify.mockReset().mockResolvedValue(owner.serverUrl);
+});
+
+afterEach(() => {
+  setStoreForTests(null);
+  resetLocalStoreForTests();
+  vi.unstubAllEnvs();
+  store.cleanup();
+  rmSync(disks, { recursive: true, force: true });
+});
+
+function request(method: string, body?: unknown) {
+  return new NextRequest('https://webmail.example.com/api/account-archive', {
+    method,
+    headers: { 'x-archive-username': owner.username, 'x-archive-server': owner.serverUrl, 'content-type': 'application/json', 'sec-fetch-site': 'same-origin' },
+    ...(body === undefined ? {} : { body: JSON.stringify(body) }),
+  });
+}
+
+describe('account archives with a shared store', () => {
+  it('returns on replica B what replica A stored, and the latest save wins', async () => {
+    await saveArchive(owner, envelope);
+    vi.stubEnv('SETTINGS_DATA_DIR', path.join(disks, 'b'));
+    expect(await loadArchive(owner)).toEqual(envelope);
+
+    await saveArchive(owner, other);
+    vi.stubEnv('SETTINGS_DATA_DIR', path.join(disks, 'a'));
+    expect(await loadArchive(owner)).toEqual(other);
+
+    await deleteArchive(owner);
+    expect(await loadArchive(owner)).toBeNull();
+  });
+
+  it('answers 503 while the store is down and keeps what is stored', async () => {
+    await saveArchive(owner, envelope);
+    store.down = true;
+    expect((await PUT(request('PUT', { envelope: other }))).status).toBe(503);
+    expect((await GET(request('GET'))).status).toBe(503);
+    store.down = false;
+    expect(await loadArchive(owner)).toEqual(envelope);
+  });
+});
+
+describe('account archives on a single instance', () => {
+  it('flush the new file to disk before it replaces the old one', async () => {
+    setStoreForTests(null);
+    const handle = await open(path.join(disks, 'probe'), 'w');
+    const proto = Object.getPrototypeOf(handle) as { sync: () => Promise<void> };
+    await handle.close();
+    const sync = vi.spyOn(proto, 'sync');
+    try {
+      await saveArchive(owner, envelope);
+      expect(sync).toHaveBeenCalledTimes(1);
+      expect(await loadArchive(owner)).toEqual(envelope);
+    } finally {
+      sync.mockRestore();
+    }
+  });
+
+  it('keep today\'s file: <sha256 of the owner>.json under account-archives, mode 0600', async () => {
+    setStoreForTests(null);
+    await saveArchive(owner, envelope);
+    const dir = path.join(disks, 'a', 'account-archives');
+    const files = readdirSync(dir);
+    expect(files).toHaveLength(1);
+    expect(JSON.parse(readFileSync(path.join(dir, files[0]), 'utf8'))).toEqual(envelope);
+    if (process.platform !== 'win32') {
+      expect(statSync(path.join(dir, files[0])).mode & 0o777).toBe(0o600);
+      expect(statSync(dir).mode & 0o777).toBe(0o700);
+    }
+    expect(files[0]).toMatch(/^[0-9a-f]{64}\.json$/);
+  });
+});

lib/__tests__/admin-session-shared.test.ts

@@ -5,6 +5,7 @@ import { mkdtempSync, rmSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import path from 'node:path';
 import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { NextRequest } from 'next/server';
 
 // With a shared store (#1066) an admin signed out on one replica has to be
 // refused on every other one at once, a new admin password has to sign out
@@ -201,4 +202,32 @@ describe('admin sessions with a shared store', () => {
     store.down = false;
     expect(await statusFor(token)).toBe(200);
   });
+
+  it('says so when a sign-out could not be stored', async () => {
+    await setInitialAdminPassword('first-password-1');
+    const token = await signIn();
+    store.down = true;
+    const { DELETE } = await import('@/app/api/admin/auth/route');
+    const res = await DELETE(new NextRequest('http://localhost/api/admin/auth', { method: 'DELETE', headers: { 'sec-fetch-site': 'same-origin' } }));
+    expect(res.status).toBe(503);
+    expect((await res.json()).error).toMatch(/could not be saved/);
+    expect(jar.has(COOKIE)).toBe(false);
+    store.down = false;
+    // As said: the copy still works.
+    expect(await statusFor(token)).toBe(200);
+  });
+
+  it('refuses the second of two password changes at once with 409, not "incorrect"', async () => {
+    await setInitialAdminPassword('first-password-1');
+    await signIn();
+    const { POST } = await import('@/app/api/admin/change-password/route');
+    const change = (newPassword: string) => POST(new NextRequest('http://localhost/api/admin/change-password', {
+      method: 'POST',
+      headers: { 'sec-fetch-site': 'same-origin', 'content-type': 'application/json' },
+      body: JSON.stringify({ currentPassword: 'first-password-1', newPassword }),
+    }));
+    const statuses = (await Promise.all([change('second-password-2'), change('third-password-3')])).map((r) => r.status);
+    expect(statuses.sort()).toEqual([200, 409]);
+    expect([await verifyAdminPassword('second-password-2'), await verifyAdminPassword('third-password-3')].filter(Boolean)).toHaveLength(1);
+  });
 });

lib/__tests__/helpers/separate-disk-store.ts

@@ -0,0 +1,69 @@
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { localCollection } from '@/lib/shared-store/local';
+import { fakeSharedStore } from '@/lib/shared-store/__tests__/fake-shared';
+import {
+  StoreUnavailableError,
+  type DocCollection,
+  type KvStore,
+  type LeaseStore,
+  type SharedStore,
+} from '@/lib/shared-store/types';
+
+export type SeparateDiskStore = SharedStore & { down: boolean; root: string; cleanup(): void };
+
+/**
+ * A fake shared store (mode 'redis') whose documents live in a folder of
+ * their own, not in the dirs a store names for local mode. Every replica has
+ * its own disk: a store that still wrote its local files would not find on
+ * "replica B" (another SETTINGS_DATA_DIR, …) what it saved on "replica A".
+ * `down` makes every document, kv and lease call fail as an outage would.
+ */
+export function separateDiskStore(): SeparateDiskStore {
+  const root = mkdtempSync(path.join(tmpdir(), 'bw-shared-store-'));
+  const base = fakeSharedStore();
+  const guard = <T>(run: () => Promise<T>): Promise<T> =>
+    store.down ? Promise.reject(new StoreUnavailableError('down (test)')) : run();
+
+  const kv: KvStore = {
+    get: (key) => guard(() => base.kv.get(key)),
+    set: (key, value, options) => guard(() => base.kv.set(key, value, options)),
+    del: (key) => guard(() => base.kv.del(key)),
+    take: (key) => guard(() => base.kv.take(key)),
+    update: (key, fn, options) => guard(() => base.kv.update(key, fn, options)),
+  };
+  const leases: LeaseStore = {
+    acquire: (name, holder, ttlMs) => guard(() => base.leases.acquire(name, holder, ttlMs)),
+    release: (name, holder) => guard(() => base.leases.release(name, holder)),
+    holder: (name) => guard(() => base.leases.holder(name)),
+  };
+
+  const store: SeparateDiskStore = {
+    ...base,
+    kv,
+    leases,
+    root,
+    get down() {
+      return base.down;
+    },
+    set down(value: boolean) {
+      base.down = value;
+    },
+    collection(name, options): DocCollection {
+      const inner = localCollection(name, { ...options, local: { ...options.local, dir: () => path.join(root, name) } });
+      return {
+        name,
+        get: (id) => guard(() => inner.get(id)),
+        put: (id, data, putOptions) => guard(() => inner.put(id, data, putOptions)),
+        update: (id, fn, updateOptions) => guard(() => inner.update(id, fn, updateOptions)),
+        delete: (id, deleteOptions) => guard(() => inner.delete(id, deleteOptions)),
+        list: (prefix) => guard(() => inner.list(prefix)),
+      };
+    },
+    cleanup() {
+      rmSync(root, { recursive: true, force: true });
+    },
+  };
+  return store;
+}

lib/__tests__/instance-details-gates.test.ts

@@ -33,6 +33,7 @@ vi.mock('@/lib/admin/config-manager', () => ({
     get: (key: string, fallback: unknown) => (key in config.values ? config.values[key] : fallback),
     getPolicy: () => POLICY,
     getPolicyVersion: () => 'v1',
+    getPolicyWithVersion() { return { policy: this.getPolicy(), version: 'v1' }; },
   },
 }));
 vi.mock('@/lib/logger', () => ({ logger: { error: vi.fn(), warn: vi.fn(), info: vi.fn(), debug: vi.fn() } }));

lib/__tests__/plugin-vault-shared-store.test.ts

@@ -0,0 +1,99 @@
+// @vitest-environment node
+import { mkdtempSync, readdirSync, rmSync, statSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { setStoreForTests } from '@/lib/shared-store';
+import { resetLocalStoreForTests } from '@/lib/shared-store/local';
+import { separateDiskStore, type SeparateDiskStore } from './helpers/separate-disk-store';
+
+// Users' plugin credentials with a shared store (#1066). The vault used to
+// serialise writes with a queue in its own module, which no other process
+// (and no other module copy) sees: two writes on two replicas lost one.
+
+vi.mock('@/lib/auth/session-secret', () => ({
+  getSessionSecret: () => 's'.repeat(64),
+  hasSessionSecret: () => true,
+}));
+vi.mock('@/lib/logger', () => ({
+  logger: { warn: () => {}, error: () => {}, info: () => {}, debug: () => {} },
+}));
+
+const user = { username: 'alice@example.org', serverUrl: 'https://mail.example.org' };
+const input = (secret: string) => ({ origin: 'https://cloud.example.org', inject: { type: 'basic' } as const, label: 'Cloud', secret });
+
+type Vault = typeof import('@/lib/plugin-secrets/vault');
+
+/** A fresh copy of the vault module, as another process (or bundle) has. */
+async function replica(): Promise<Vault> {
+  vi.resetModules();
+  return import('@/lib/plugin-secrets/vault');
+}
+
+let disks: string;
+let store: SeparateDiskStore;
+
+beforeEach(() => {
+  disks = mkdtempSync(path.join(tmpdir(), 'bw-vault-replicas-'));
+  vi.stubEnv('PLUGIN_SECRETS_DIR', path.join(disks, 'a'));
+  store = separateDiskStore();
+  setStoreForTests(store);
+});
+
+afterEach(() => {
+  setStoreForTests(null);
+  resetLocalStoreForTests();
+  vi.unstubAllEnvs();
+  store.cleanup();
+  rmSync(disks, { recursive: true, force: true });
+});
+
+describe('plugin credentials with a shared store', () => {
+  it('reads on replica B what replica A stored', async () => {
+    const a = await replica();
+    await a.putUserCredential(user, 'nextcloud', 'cloud', input('pw'));
+
+    vi.stubEnv('PLUGIN_SECRETS_DIR', path.join(disks, 'b'));
+    const b = await replica();
+    expect(await b.getUserCredential(user, 'nextcloud', 'cloud')).toMatchObject({ secret: 'pw' });
+  });
+
+  it('keeps both of two credentials saved at once through two replicas', async () => {
+    const a = await replica();
+    const b = await replica();
+    await Promise.all([
+      a.putUserCredential(user, 'nextcloud', 'cloud', input('one')),
+      b.putUserCredential(user, 'paperless', 'docs', input('two')),
+      a.putUserCredential(user, 'radicale', 'dav', input('three')),
+      b.putUserCredential(user, 'immich', 'photos', input('four')),
+    ]);
+
+    const ids = (await a.listUserCredentials(user)).map((c) => c.pluginId).sort();
+    expect(ids).toEqual(['immich', 'nextcloud', 'paperless', 'radicale']);
+  });
+
+  it('answers with StoreUnavailableError while the store is down and writes nothing', async () => {
+    const vault = await replica();
+    await vault.putUserCredential(user, 'nextcloud', 'cloud', input('pw'));
+    store.down = true;
+    await expect(vault.putUserCredential(user, 'paperless', 'docs', input('x'))).rejects.toThrow('down (test)');
+    await expect(vault.listUserCredentials(user)).rejects.toThrow('down (test)');
+    store.down = false;
+    expect((await vault.listUserCredentials(user)).map((c) => c.pluginId)).toEqual(['nextcloud']);
+  });
+});
+
+describe('plugin credentials on a single instance', () => {
+  it('keep today\'s file: <sha256>.enc in the vault dir, mode 0600, gone when the last one goes', async () => {
+    setStoreForTests(null);
+    const vault = await replica();
+    await vault.putUserCredential(user, 'nextcloud', 'cloud', input('pw'));
+    const dir = path.join(disks, 'a');
+    const files = readdirSync(dir);
+    expect(files).toEqual([`${vault.userKey(user)}.enc`]);
+    if (process.platform !== 'win32') expect(statSync(path.join(dir, files[0])).mode & 0o777).toBe(0o600);
+
+    await vault.deleteUserCredentials(user, 'nextcloud');
+    expect(readdirSync(dir)).toEqual([]);
+  });
+});

lib/__tests__/scheduler-leader.test.ts

@@ -0,0 +1,381 @@
+// @vitest-environment node
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { setTimeout as realSleep } from 'node:timers/promises';
+import { NextRequest } from 'next/server';
+import { replicaId, setStoreForTests } from '@/lib/shared-store';
+import { resetLocalStoreForTests } from '@/lib/shared-store/local';
+import { INIT_EPOCH, announceChange, pollChanges, resetChangesForTests } from '@/lib/shared-store/changes';
+import { separateDiskStore, type SeparateDiskStore } from './helpers/separate-disk-store';
+
+// The background jobs with several replicas (#1066): only the holder of the
+// scheduler-leader lease runs them, another replica takes over when the
+// holder's lease runs out and keeps the stored schedule, and a scheduled
+// heartbeat is claimed with a compare-and-set so two leaders can't both send.
+
+const log = vi.hoisted(() => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }));
+vi.mock('@/lib/logger', () => ({ logger: log }));
+vi.mock('@/lib/admin/third-party-mode', () => ({ isNoThirdPartyTrafficOn: async () => false }));
+const buildPayload = vi.hoisted(() => vi.fn(async () => ({ schema: '1' })));
+vi.mock('@/lib/telemetry/payload', () => ({ buildPayload, markProcessStart: () => {} }));
+const sent = vi.hoisted(() => vi.fn(async (_url: string, _init?: unknown) => new Response('ok')));
+vi.mock('@/lib/telemetry/endpoint-guard', () => ({
+  resolveEndpointAllowed: async () => ({ ok: true }),
+  fetchTelemetryTarget: (url: string, init?: unknown) => sent(url, init),
+}));
+vi.mock('@/lib/admin/session', () => ({
+  requireAdminAuth: async () => ({ session: { sub: 'admin' } }),
+  getClientIP: () => '127.0.0.1',
+}));
+vi.mock('@/lib/admin/audit', () => ({ auditLog: async () => {} }));
+
+import {
+  LEADER_LEASE, isSchedulerLeader, resetSchedulerLeaderForTests, schedulerLeaderSettled, startSchedulerLeader, stopSchedulerLeader,
+} from '@/lib/scheduler-leader';
+import { loadState, saveState, sendOnce, startScheduler, stopScheduler, updateState } from '@/lib/telemetry';
+import { POST as telemetryRoute } from '@/app/api/admin/telemetry/route';
+
+const MIN = 60_000;
+const COLLECTOR = 'https://collector.example.org/v1/heartbeat';
+const jobs = [{ name: 'telemetry', start: startScheduler, stop: stopScheduler }];
+
+let dir: string;
+let store: SeparateDiskStore;
+
+beforeEach(() => {
+  dir = mkdtempSync(path.join(tmpdir(), 'bw-leader-'));
+  vi.stubEnv('TELEMETRY_DATA_DIR', path.join(dir, 'telemetry'));
+  vi.stubEnv('BULWARK_TELEMETRY', '');
+  vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'setInterval', 'clearInterval', 'Date'] });
+  vi.setSystemTime(new Date('2026-10-31T12:00:00.000Z'));
+  store = separateDiskStore();
+  setStoreForTests(store);
+  sent.mockClear();
+  log.info.mockClear();
+});
+
+afterEach(async () => {
+  await stopSchedulerLeader();
+  resetSchedulerLeaderForTests();
+  await stopScheduler();
+  resetChangesForTests();
+  vi.useRealTimers();
+  setStoreForTests(null);
+  resetLocalStoreForTests();
+  vi.unstubAllEnvs();
+  store.cleanup();
+  rmSync(dir, { recursive: true, force: true });
+});
+
+async function consentOn(nextScheduledAt: string | null) {
+  await saveState({ ...(await loadState()), consent: 'on', endpoint: COLLECTOR, nextScheduledAt });
+}
+
+/** Let time pass; the leader's tries and job starts do real file I/O, so wait for them too. */
+async function pass(ms: number) {
+  for (let passed = 0; passed < ms; passed += 10_000) {
+    await vi.advanceTimersByTimeAsync(Math.min(10_000, ms - passed));
+    await schedulerLeaderSettled();
+  }
+}
+
+/** Another replica holding (and renewing) the lease while `ms` pass here. */
+async function otherReplicaLeads(ms: number) {
+  for (let passed = 0; passed < ms; passed += 10_000) {
+    expect(await store.leases.acquire(LEADER_LEASE, 'replica-a', 30_000)).toBe(true);
+    await pass(10_000);
+  }
+}
+
+/** Let a heartbeat a timer set off finish its real file I/O, so "not sent" means it. */
+async function ioSettles() {
+  await realSleep(200);
+}
+
+/** The heartbeat the timer set off does real I/O before it sends. */
+async function sends(times: number) {
+  await vi.waitFor(() => expect(sent).toHaveBeenCalledTimes(times));
+}
+
+describe('scheduler leader with a shared store', () => {
+  it('only the holder sends; when its lease runs out another replica takes over and keeps the schedule', async () => {
+    const slot = new Date(Date.now() + 20 * MIN).toISOString();
+    await consentOn(slot);
+    await store.leases.acquire(LEADER_LEASE, 'replica-a', 30_000);
+
+    await startSchedulerLeader(jobs);
+    expect(isSchedulerLeader()).toBe(false);
+    await otherReplicaLeads(5 * MIN);
+    await ioSettles();
+    expect(isSchedulerLeader()).toBe(false);
+
+    // replica-a is gone: its lease expires after 30 s, the next try takes it.
+    await pass(40_000);
+    expect(isSchedulerLeader()).toBe(true);
+    expect(log.info).toHaveBeenCalledWith('scheduler: leading', expect.objectContaining({ replica: expect.any(String) }));
+    expect((await loadState()).nextScheduledAt).toBe(slot);
+    expect(sent).not.toHaveBeenCalled();
+
+    await vi.advanceTimersByTimeAsync(new Date(slot).getTime() - Date.now() + 1000);
+    await sends(1);
+  });
+
+  it('stops its timers when it can no longer reach the store', async () => {
+    const slot = new Date(Date.now() + 5 * MIN).toISOString();
+    await consentOn(slot);
+    await startSchedulerLeader(jobs);
+    expect(isSchedulerLeader()).toBe(true);
+
+    store.down = true;
+    await pass(10_000);
+    expect(isSchedulerLeader()).toBe(false);
+    expect(log.info).toHaveBeenCalledWith('scheduler: stopped leading', expect.objectContaining({ reason: expect.stringContaining('shared store') }));
+    // Past the slot with the store still down: no heartbeat from here.
+    await pass(6 * MIN);
+    await ioSettles();
+    expect(sent).not.toHaveBeenCalled();
+    expect(isSchedulerLeader()).toBe(false);
+  });
+
+  it('stops leading when the store lost its data', async () => {
+    await store.epochs.bump(INIT_EPOCH);
+    await consentOn(new Date(Date.now() + 5 * MIN).toISOString());
+    await startSchedulerLeader(jobs);
+    expect(isSchedulerLeader()).toBe(true);
+
+    // Flushed: the init epoch is gone, and the next poll notices.
+    resetLocalStoreForTests();
+    await pollChanges();
+    await pass(10_000);
+    expect(isSchedulerLeader()).toBe(false);
+    expect(log.info).toHaveBeenCalledWith('scheduler: stopped leading', expect.objectContaining({ reason: 'the shared store lost its data' }));
+  });
+
+  it('stops its timers when another replica has taken the lease', async () => {
+    const slot = new Date(Date.now() + 5 * MIN).toISOString();
+    await consentOn(slot);
+    await startSchedulerLeader(jobs);
+    expect(isSchedulerLeader()).toBe(true);
+
+    // This replica's lease is gone (it stalled past the TTL) and replica-a took it.
+    await store.leases.release(LEADER_LEASE, replicaId());
+    await otherReplicaLeads(6 * MIN);
+    await ioSettles();
+    expect(isSchedulerLeader()).toBe(false);
+    expect(sent).not.toHaveBeenCalled();
+  });
+
+  it('does not arm a new timer after a send that was in flight when the lead moved', async () => {
+    const slot = new Date(Date.now() + MIN).toISOString();
+    await consentOn(slot);
+    let finish!: (res: Response) => void;
+    sent.mockImplementationOnce(() => new Promise<Response>((resolve) => { finish = resolve; }));
+    await startSchedulerLeader(jobs);
+
+    await vi.advanceTimersByTimeAsync(MIN + 1000);
+    await sends(1);
+    // The lead moves to replica-a while the heartbeat hangs.
+    await store.leases.release(LEADER_LEASE, replicaId());
+    expect(await store.leases.acquire(LEADER_LEASE, 'replica-a', 30_000)).toBe(true);
+    await pass(10_000);
+    expect(isSchedulerLeader()).toBe(false);
+    await stopSchedulerLeader(); // no more tries from this replica
+
+    finish(new Response('ok'));
+    await ioSettles();
+    await vi.advanceTimersByTimeAsync(27 * 60 * MIN);
+    await ioSettles();
+    expect(sent).toHaveBeenCalledTimes(1);
+    expect(vi.getTimerCount()).toBe(0);
+  });
+
+  it('stops a job whose restart finished after the replica stepped down', async () => {
+    let running = false;
+    let starts = 0;
+    let finishStart: (() => void) | null = null;
+    const job = {
+      name: 'telemetry',
+      start: async () => {
+        if (++starts === 2) await new Promise<void>((resolve) => { finishStart = resolve; });
+        running = true;
+      },
+      stop: async () => { running = false; },
+    };
+    await startSchedulerLeader([job]);
+    expect(running).toBe(true);
+
+    // Consent changed: the restart's start hangs while the lead is lost.
+    const restarted = announceChange('telemetry');
+    await vi.waitFor(() => expect(finishStart).not.toBeNull());
+    store.down = true;
+    await pass(10_000);
+    expect(isSchedulerLeader()).toBe(false);
+    finishStart!();
+    await restarted;
+    expect(running).toBe(false);
+  });
+
+  it('version check: does not arm a new timer after a check that was in flight when it stopped', async () => {
+    vi.stubEnv('VERSION_CHECK_DATA_DIR', path.join(dir, 'version-check'));
+    vi.stubEnv('BULWARK_UPDATE_CHECK', '');
+    vi.stubEnv('BULWARK_UPDATE_CHECK_URL', 'https://version.example/');
+    vi.stubEnv('NEXT_PUBLIC_APP_VERSION', '1.20.0');
+    let finish!: (res: Response) => void;
+    const fetchMock = vi.fn(() => new Promise<Response>((resolve) => { finish = resolve; }));
+    vi.stubGlobal('fetch', fetchMock);
+    const versionCheck = await import('@/lib/version-check');
+    try {
+      await versionCheck.startScheduler();
+      await vi.advanceTimersByTimeAsync(31_000);
+      await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(1));
+      await versionCheck.stopScheduler();
+      finish(new Response('{}', { status: 500 }));
+      await ioSettles();
+      await vi.advanceTimersByTimeAsync(2 * 60 * MIN);
+      await ioSettles();
+      expect(fetchMock).toHaveBeenCalledTimes(1);
+      expect(vi.getTimerCount()).toBe(0);
+    } finally {
+      await versionCheck.stopScheduler();
+      vi.unstubAllGlobals();
+    }
+  });
+
+  it('sends once when two leaders fire for the same slot, and not again after it', async () => {
+    const slot = new Date(Date.now() - 1000).toISOString();
+    await consentOn(slot);
+
+    const results = await Promise.all([
+      sendOnce({ reason: 'scheduled', slot }),
+      sendOnce({ reason: 'scheduled', slot }),
+    ]);
+    expect(results.filter((r) => r.ok)).toHaveLength(1);
+    expect(sent).toHaveBeenCalledTimes(1);
+
+    // The paused leader wakes up an hour later, still holding that slot.
+    vi.advanceTimersByTime(60 * MIN);
+    expect((await sendOnce({ reason: 'scheduled', slot })).ok).toBe(false);
+    expect(sent).toHaveBeenCalledTimes(1);
+  });
+
+  it('sends once when the old leader stalled past its slot and the new one picked a later slot', async () => {
+    const oldSlot = new Date(Date.now() - 1000).toISOString();
+    await consentOn(oldSlot);
+    // The new leader found the slot passed and scheduled an hour out.
+    const newSlot = new Date(Date.now() + 60 * MIN).toISOString();
+    // The stalled leader wakes up and sends for its slot first.
+    expect((await sendOnce({ reason: 'scheduled', slot: oldSlot })).ok).toBe(true);
+    vi.advanceTimersByTime(61 * MIN);
+    expect((await sendOnce({ reason: 'scheduled', slot: newSlot })).ok).toBe(false);
+    expect(sent).toHaveBeenCalledTimes(1);
+  });
+
+  it('sends once when a leader died after sending but before storing the next slot', async () => {
+    const slot = new Date(Date.now() - 1000).toISOString();
+    await consentOn(slot);
+    expect((await sendOnce({ reason: 'scheduled', slot })).ok).toBe(true);
+    // nextScheduledAt still names the slot just sent; the next leader
+    // finds it passed and fires an hour later.
+    vi.advanceTimersByTime(60 * MIN);
+    expect((await sendOnce({ reason: 'scheduled', slot: new Date().toISOString() })).ok).toBe(false);
+    expect(sent).toHaveBeenCalledTimes(1);
+  });
+
+  it('still sends the next scheduled heartbeat after a manual "send now"', async () => {
+    const slot = new Date(Date.now() - 1000).toISOString();
+    await consentOn(slot);
+    expect((await sendOnce({ reason: 'scheduled', slot })).ok).toBe(true);
+    vi.advanceTimersByTime(20 * 60 * MIN);
+    expect((await sendOnce({ reason: 'admin-manual' })).ok).toBe(true);
+    vi.advanceTimersByTime(2 * 60 * MIN);
+    expect((await sendOnce({ reason: 'scheduled', slot: new Date().toISOString() })).ok).toBe(true);
+    expect(sent).toHaveBeenCalledTimes(3);
+  });
+
+  it('gives the slot back when the payload cannot be built', async () => {
+    const slot = new Date(Date.now() - 1000).toISOString();
+    await consentOn(slot);
+    buildPayload.mockRejectedValueOnce(new Error('store down'));
+    expect((await sendOnce({ reason: 'scheduled', slot })).ok).toBe(false);
+    expect((await loadState()).lastSentAt).toBeNull();
+    expect((await sendOnce({ reason: 'scheduled', slot })).ok).toBe(true);
+    expect(sent).toHaveBeenCalledTimes(1);
+  });
+
+  it('gives the slot back when the send fails', async () => {
+    const slot = new Date(Date.now() - 1000).toISOString();
+    await consentOn(slot);
+    sent.mockResolvedValueOnce(new Response('no', { status: 500 }));
+    expect((await sendOnce({ reason: 'scheduled', slot })).ok).toBe(false);
+    expect((await loadState()).lastSentAt).toBeNull();
+    expect((await sendOnce({ reason: 'scheduled', slot })).ok).toBe(true);
+  });
+
+  it('sends on "send now" from a replica that does not lead, even right after a heartbeat', async () => {
+    await consentOn(null);
+    await store.leases.acquire(LEADER_LEASE, 'replica-a', 30_000);
+    await startSchedulerLeader(jobs);
+    expect(isSchedulerLeader()).toBe(false);
+    await updateState((state) => { state.lastSentAt = new Date().toISOString(); });
+
+    const res = await telemetryRoute(new NextRequest('http://localhost/api/admin/telemetry', {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({ action: 'send-now' }),
+    }));
+    expect(res.status).toBe(200);
+    expect(sent).toHaveBeenCalledTimes(1);
+  });
+
+  it('starts the heartbeat on the leader when consent is turned on through another replica', async () => {
+    await store.epochs.bump(INIT_EPOCH);
+    await startSchedulerLeader(jobs);
+    expect(isSchedulerLeader()).toBe(true);
+    expect((await loadState()).nextScheduledAt).toBeNull();
+
+    // The other replica's admin route: write, then bump the epoch.
+    await updateState((state) => { state.consent = 'on'; state.endpoint = COLLECTOR; });
+    await store.epochs.bump('telemetry');
+    await pollChanges();
+
+    expect((await loadState()).nextScheduledAt).not.toBeNull();
+    await vi.advanceTimersByTimeAsync(61 * MIN);
+    await sends(1);
+  });
+});
+
+describe('scheduler leader on a single instance', () => {
+  it('sends a scheduled heartbeat an hour after the last one, as before', async () => {
+    setStoreForTests(null);
+    const slot = new Date(Date.now() - 1000).toISOString();
+    await consentOn(slot);
+    expect((await sendOnce({ reason: 'scheduled', slot })).ok).toBe(true);
+    vi.advanceTimersByTime(60 * MIN);
+    expect((await sendOnce({ reason: 'scheduled', slot: new Date().toISOString() })).ok).toBe(true);
+    expect((await loadState()).lastScheduledSentAt).toBeUndefined();
+  });
+
+  it('leads at once and restarts the heartbeat when consent changes in this process', async () => {
+    setStoreForTests(null);
+    await startSchedulerLeader(jobs);
+    expect(isSchedulerLeader()).toBe(true);
+    expect(log.info).toHaveBeenCalledWith('scheduler: leading', expect.anything());
+
+    const res = await telemetryRoute(new NextRequest('http://localhost/api/admin/telemetry', {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({ action: 'set-consent', consent: 'on' }),
+    }));
+    expect(res.status).toBe(200);
+    expect(vi.getTimerCount()).toBe(1);
+
+    await telemetryRoute(new NextRequest('http://localhost/api/admin/telemetry', {
+      method: 'POST',
+      headers: { 'content-type': 'application/json' },
+      body: JSON.stringify({ action: 'set-consent', consent: 'off' }),
+    }));
+    expect(vi.getTimerCount()).toBe(0);
+  });
+});

lib/__tests__/scheduler-timer-module-copies.test.ts

@@ -0,0 +1,62 @@
+// @vitest-environment node
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+// Next loads lib/telemetry and lib/version-check once per bundle. The timer
+// sat in a module-level variable, so the admin route's reschedule() acted on
+// its own copy: the timer instrumentation started kept running next to the
+// new one (two heartbeats), and turning consent off didn't stop it.
+
+vi.mock('@/lib/admin/third-party-mode', () => ({ isNoThirdPartyTrafficOn: async () => false }));
+
+let dir: string;
+
+beforeEach(() => {
+  dir = mkdtempSync(path.join(tmpdir(), 'bw-scheduler-copies-'));
+  vi.stubEnv('TELEMETRY_DATA_DIR', path.join(dir, 'telemetry'));
+  vi.stubEnv('VERSION_CHECK_DATA_DIR', path.join(dir, 'version-check'));
+  vi.stubEnv('BULWARK_TELEMETRY', 'on');
+  vi.stubEnv('BULWARK_UPDATE_CHECK', '');
+  vi.stubEnv('BULWARK_UPDATE_CHECK_URL', 'https://version.example/');
+  vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] });
+});
+
+afterEach(() => {
+  vi.useRealTimers();
+  vi.unstubAllEnvs();
+  vi.resetModules();
+  rmSync(dir, { recursive: true, force: true });
+});
+
+async function copy<T>(load: () => Promise<T>): Promise<T> {
+  vi.resetModules();
+  return load();
+}
+
+describe('scheduler timers across module copies', () => {
+  it('telemetry: a stop from another copy clears the running timer, a restart leaves one', async () => {
+    const instrumentation = await copy(() => import('@/lib/telemetry/sender'));
+    const route = await copy(() => import('@/lib/telemetry/sender'));
+
+    await instrumentation.startScheduler();
+    expect(vi.getTimerCount()).toBe(1);
+    await route.startScheduler();
+    expect(vi.getTimerCount()).toBe(1);
+    await route.stopScheduler();
+    expect(vi.getTimerCount()).toBe(0);
+  });
+
+  it('version check: the same', async () => {
+    const instrumentation = await copy(() => import('@/lib/version-check/sender'));
+    const route = await copy(() => import('@/lib/version-check/sender'));
+
+    await instrumentation.startScheduler();
+    expect(vi.getTimerCount()).toBe(1);
+    await route.startScheduler();
+    expect(vi.getTimerCount()).toBe(1);
+    await route.stopScheduler();
+    expect(vi.getTimerCount()).toBe(0);
+  });
+});

lib/__tests__/settings-route-account-key.test.ts

@@ -39,6 +39,9 @@ vi.mock('@/lib/settings-sync', () => ({
   saveUserSettings: async (username: string, serverUrl: string, settings: Record<string, unknown>) => {
     store.set(`${username}|${serverUrl}`, settings);
   },
+  updateUserSettings: async (username: string, serverUrl: string, change: (stored: Record<string, unknown> | null) => Record<string, unknown>) => {
+    store.set(`${username}|${serverUrl}`, change(store.get(`${username}|${serverUrl}`) ?? null));
+  },
   deleteUserSettings: async (username: string, serverUrl: string) => { store.delete(`${username}|${serverUrl}`); },
 }));
 

lib/__tests__/settings-route-template-merge.test.ts

@@ -38,6 +38,9 @@ vi.mock('@/lib/settings-sync', () => ({
   saveUserSettings: async (username: string, serverUrl: string, settings: Record<string, unknown>) => {
     store.set(`${username}|${serverUrl}`, settings);
   },
+  updateUserSettings: async (username: string, serverUrl: string, change: (stored: Record<string, unknown> | null) => Record<string, unknown>) => {
+    store.set(`${username}|${serverUrl}`, change(store.get(`${username}|${serverUrl}`) ?? null));
+  },
   deleteUserSettings: async () => {},
 }));
 

lib/__tests__/settings-shared-store.test.ts

@@ -0,0 +1,176 @@
+// @vitest-environment node
+import { mkdtempSync, readdirSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { NextRequest } from 'next/server';
+import { setStoreForTests } from '@/lib/shared-store';
+import { resetLocalStoreForTests } from '@/lib/shared-store/local';
+import { SETTINGS_COLLECTION, settingsId } from '@/lib/settings-sync';
+import { separateDiskStore, type SeparateDiskStore } from './helpers/separate-disk-store';
+
+// Synced settings with a shared store (#1066): every replica reads and
+// writes the one store, never its own disk.
+
+vi.mock('@/lib/auth/session-secret', () => ({
+  getSessionSecret: () => 's'.repeat(64),
+  hasSessionSecret: () => true,
+}));
+vi.mock('@/lib/logger', () => ({
+  logger: { warn: () => {}, error: () => {}, info: () => {}, debug: () => {} },
+}));
+vi.mock('@/lib/admin/config-manager', () => ({
+  configManager: {
+    ensureLoaded: async () => {},
+    get: (key: string, fallback: unknown) => (key === 'settingsSyncEnabled' ? true : fallback),
+    getPolicy: () => ({ restrictions: {} }),
+  },
+}));
+
+const jar = new Map<string, string>();
+vi.mock('next/headers', () => ({
+  cookies: async () => ({
+    get: (name: string) => (jar.has(name) ? { name, value: jar.get(name)! } : undefined),
+    set: (name: string, value: string) => { jar.set(name, value); },
+    delete: (name: string) => { jar.delete(name); },
+    getAll: () => [...jar].map(([name, value]) => ({ name, value })),
+  }),
+}));
+
+const SERVER = 'https://mail.example.org';
+const USER = 'alice@example.org';
+
+let disks: string;
+let store: SeparateDiskStore;
+
+/** Point this process at one replica's own data dir. */
+function onReplica(name: 'a' | 'b') {
+  vi.stubEnv('SETTINGS_DATA_DIR', path.join(disks, name, 'settings'));
+}
+
+beforeEach(async () => {
+  disks = mkdtempSync(path.join(tmpdir(), 'bw-settings-replicas-'));
+  store = separateDiskStore();
+  setStoreForTests(store);
+  vi.stubEnv('SETTINGS_SYNC_ENABLED', 'true');
+  onReplica('a');
+  jar.clear();
+  const { setStalwartAuthContextInStore } = await import('@/lib/stalwart/auth-context');
+  setStalwartAuthContextInStore({
+    get: (n: string) => (jar.has(n) ? { name: n, value: jar.get(n)! } : undefined),
+    set: (n: string, v: string) => { jar.set(n, v); },
+    delete: (n: string) => { jar.delete(n); },
+  } as never, 0, { serverUrl: SERVER, authHeader: 'Bearer t', username: USER });
+});
+
+afterEach(() => {
+  setStoreForTests(null);
+  resetLocalStoreForTests();
+  vi.unstubAllEnvs();
+  store.cleanup();
+  rmSync(disks, { recursive: true, force: true });
+});
+
+function template(id: string) {
+  const at = '2026-10-01T00:00:00.000Z';
+  return { id, name: id, subject: '', body: '', isHTML: false, category: '', isFavorite: false, createdAt: at, updatedAt: at };
+}
+
+async function push(settings: Record<string, unknown>) {
+  const { POST } = await import('@/app/api/settings/route');
+  return POST(new NextRequest('https://webmail.example/api/settings', {
+    method: 'POST',
+    headers: { 'sec-fetch-site': 'same-origin', 'content-type': 'application/json' },
+    body: JSON.stringify({ username: USER, serverUrl: SERVER, settings }),
+  }));
+}
+
+async function fetchSettings() {
+  const { GET } = await import('@/app/api/settings/route');
+  return GET(new NextRequest('https://webmail.example/api/settings', {
+    headers: { 'x-settings-username': USER, 'x-settings-server': SERVER },
+  }));
+}
+
+function storedDoc() {
+  return store.collection('settings', SETTINGS_COLLECTION).get(settingsId(USER, SERVER));
+}
+
+describe('settings with a shared store', () => {
+  it('loads on replica B what was saved through replica A', async () => {
+    expect((await push({ theme: 'dark' })).status).toBe(200);
+
+    onReplica('b');
+    const res = await fetchSettings();
+    expect(res.status).toBe(200);
+    expect((await res.json()).settings).toEqual({ theme: 'dark' });
+  });
+
+  it('deletes them for every replica', async () => {
+    await push({ theme: 'dark' });
+    onReplica('b');
+    const { DELETE } = await import('@/app/api/settings/route');
+    const res = await DELETE(new NextRequest('https://webmail.example/api/settings', {
+      method: 'DELETE',
+      headers: { 'sec-fetch-site': 'same-origin', 'content-type': 'application/json' },
+      body: JSON.stringify({ username: USER, serverUrl: SERVER }),
+    }));
+    expect(res.status).toBe(200);
+    onReplica('a');
+    expect((await (await fetchSettings()).json()).settings).toBeNull();
+  });
+
+  it('moves a stored document that does not decrypt aside in the store before a save, keeping its bytes', async () => {
+    const garbage = Buffer.alloc(64, 7);
+    const id = settingsId(USER, SERVER);
+    await store.collection('settings', SETTINGS_COLLECTION).put(id, garbage);
+
+    expect((await (await fetchSettings()).json()).settings).toBeNull();
+    expect((await push({ theme: 'light' })).status).toBe(200);
+    expect((await (await fetchSettings()).json()).settings).toEqual({ theme: 'light' });
+
+    // In the store, not on this replica's disk.
+    const aside = readdirSync(path.join(store.root, 'settings')).filter((f) => f.startsWith(`${id}.unreadable-`));
+    expect(aside).toHaveLength(1);
+    const asideId = aside[0].slice(0, -'.enc'.length);
+    expect((await store.collection('settings', SETTINGS_COLLECTION).get(asideId))?.data.equals(garbage)).toBe(true);
+  });
+
+  it('answers 503 and writes nothing while the store is down', async () => {
+    await push({ theme: 'dark' });
+    const before = (await storedDoc())!.data;
+
+    store.down = true;
+    expect((await push({ theme: 'light' })).status).toBe(503);
+    expect((await fetchSettings()).status).toBe(503);
+    store.down = false;
+
+    expect((await storedDoc())!.data.equals(before)).toBe(true);
+  });
+});
+
+// Two saves at once, each from a tab that added a template the other
+// doesn't know yet. The route used to load, merge and save without a lock,
+// so the second save put its merge over the first: one template was lost,
+// with two tabs on one instance as well as across replicas.
+async function twoConcurrentSaves() {
+  await push({ templates: [], deletedTemplateIds: {} });
+  const [a, b] = await Promise.all([
+    push({ templates: [template('from-a')], deletedTemplateIds: {} }),
+    push({ templates: [template('from-b')], deletedTemplateIds: {} }),
+  ]);
+  expect([a.status, b.status]).toEqual([200, 200]);
+  const saved = (await (await fetchSettings()).json()).settings as { templates: { id: string }[] };
+  return saved.templates.map((t) => t.id).sort();
+}
+
+describe('two saves at once', () => {
+  it('keep both templates with a shared store', async () => {
+    expect(await twoConcurrentSaves()).toEqual(['from-a', 'from-b']);
+  });
+
+  it('keep both templates on a single instance', async () => {
+    setStoreForTests(null);
+    expect(await twoConcurrentSaves()).toEqual(['from-a', 'from-b']);
+  });
+});

lib/__tests__/setup-token-shared.test.ts

@@ -16,15 +16,33 @@ import { separateFakeStore } from '@/lib/admin/__tests__/shared-store-fake';
 
 let dir: string;
 
-/** One replica's copy of lib/setup/token.ts. */
+let replicas = 0;
+
+/**
+ * One replica's copy of lib/setup/token.ts, with a state dir of its own:
+ * replicas share the store, not a volume. Each call runs with that dir.
+ */
 async function replica() {
   vi.resetModules();
-  return import('@/lib/setup/token');
+  const token = await import('@/lib/setup/token');
+  const own = path.join(dir, `state-${++replicas}`);
+  const inOwnDir = <A extends unknown[], R>(fn: (...args: A) => Promise<R>) => (...args: A) => {
+    vi.stubEnv('ADMIN_STATE_DIR', own);
+    return fn(...args);
+  };
+  return {
+    ensureSetupToken: inOwnDir(token.ensureSetupToken),
+    verifySetupToken: inOwnDir(token.verifySetupToken),
+    clearSetupToken: inOwnDir(token.clearSetupToken),
+  };
 }
 
+/** Whether any replica wrote a token file. */
+const anyTokenFile = () => Array.from({ length: replicas }, (_, i) => path.join(dir, `state-${i + 1}`, '.setup-token')).some((f) => existsSync(f));
+
 beforeEach(() => {
   dir = mkdtempSync(path.join(tmpdir(), 'bw-setup-token-'));
-  vi.stubEnv('ADMIN_STATE_DIR', path.join(dir, 'state'));
+  replicas = 0;
   resetLocalStoreForTests();
   setStoreForTests(separateFakeStore(path.join(dir, 'store')));
 });
@@ -44,7 +62,7 @@ describe('setup token with a shared store', () => {
     expect(new Set(tokens).size).toBe(1);
     for (const r of replicas) expect(await r.verifySetupToken(tokens[0])).toBe(true);
     expect(await replicas[0].verifySetupToken('0'.repeat(64))).toBe(false);
-    expect(existsSync(path.join(dir, 'state', '.setup-token'))).toBe(false);
+    expect(anyTokenFile()).toBe(false);
   });
 
   it('keeps the token across restarts within the hour, then issues a new one', async () => {
@@ -63,6 +81,7 @@ describe('setup token with a shared store', () => {
   it('stops the token working on every replica once the wizard finished on one', async () => {
     const [a, b] = [await replica(), await replica()];
     const token = await a.ensureSetupToken();
+    expect(await b.verifySetupToken(token)).toBe(true);
     await a.clearSetupToken();
     expect(await b.verifySetupToken(token)).toBe(false);
   });

lib/__tests__/stored-data-unreadable.test.ts

@@ -0,0 +1,212 @@
+// @vitest-environment node
+import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { NextRequest } from 'next/server';
+
+// A settings file or plugin vault that exists but doesn't decrypt (written
+// with another session secret) used to read as "nothing stored", and the
+// next save wrote over it: the user's real data was gone for good. Now the
+// save first moves it aside (`<file>.unreadable-<time>`, bytes unchanged)
+// and goes on as if nothing was stored, so the user can save again and the
+// old data can still be restored with the old secret.
+
+let secret = 'a'.repeat(64);
+vi.mock('@/lib/auth/session-secret', () => ({
+  getSessionSecret: () => secret,
+  hasSessionSecret: () => true,
+}));
+vi.mock('@/lib/logger', () => ({
+  logger: { warn: () => {}, error: () => {}, info: () => {}, debug: () => {} },
+}));
+vi.mock('@/lib/admin/config-manager', () => ({
+  configManager: {
+    ensureLoaded: async () => {},
+    get: (key: string, fallback: unknown) => (key === 'settingsSyncEnabled' ? true : fallback),
+    getPolicy: () => ({ restrictions: {}, features: { pluginsEnabled: true } }),
+  },
+}));
+vi.mock('@/lib/admin/third-party-mode', () => ({ isNoThirdPartyTrafficOn: async () => false, thirdPartyRefusal: () => null }));
+// One installed plugin with a credential the admin set up.
+vi.mock('@/lib/admin/plugin-registry', () => ({
+  getPlugin: async (id: string) => (id === 'paperless'
+    ? { id, enabled: true, credentials: { shared: { scope: 'admin', label: 'Paperless', inject: { type: 'bearer' }, origin: 'https://paperless.example.com', secretKey: 'token' } } }
+    : null),
+}));
+vi.mock('@/lib/admin/plugin-config', () => ({ getPluginConfig: async () => ({ token: 'admin-token' }) }));
+
+const jar = new Map<string, string>();
+vi.mock('next/headers', () => ({
+  cookies: async () => ({
+    get: (name: string) => (jar.has(name) ? { name, value: jar.get(name)! } : undefined),
+    set: (name: string, value: string) => { jar.set(name, value); },
+    delete: (name: string) => { jar.delete(name); },
+    getAll: () => [...jar].map(([name, value]) => ({ name, value })),
+  }),
+}));
+
+const SERVER = 'https://mail.example.org';
+const USER = 'alice@example.org';
+vi.mock('@/lib/stalwart/credentials', () => ({
+  getStalwartCredentials: async () => ({ username: USER, accountName: USER, serverUrl: SERVER, authHeader: 'x' }),
+}));
+vi.mock('@/lib/plugin-secrets/known-plugins', () => ({
+  declaredUserCredential: async () => ({ scope: 'user', label: 'Nextcloud', inject: { type: 'basic' } }),
+}));
+
+let dataDir: string;
+
+beforeEach(async () => {
+  dataDir = mkdtempSync(path.join(tmpdir(), 'bw-unreadable-'));
+  vi.stubEnv('SETTINGS_DATA_DIR', path.join(dataDir, 'settings'));
+  vi.stubEnv('PLUGIN_SECRETS_DIR', path.join(dataDir, 'plugin-secrets'));
+  vi.stubEnv('SETTINGS_SYNC_ENABLED', 'true');
+  secret = 'a'.repeat(64);
+  jar.clear();
+});
+
+afterEach(() => {
+  vi.unstubAllEnvs();
+  rmSync(dataDir, { recursive: true, force: true });
+});
+
+/** Sign in after the secret change: the context cookie is sealed with the current secret. */
+async function signIn() {
+  const { setStalwartAuthContextInStore } = await import('@/lib/stalwart/auth-context');
+  setStalwartAuthContextInStore({
+    get: (n: string) => (jar.has(n) ? { name: n, value: jar.get(n)! } : undefined),
+    set: (n: string, v: string) => { jar.set(n, v); },
+    delete: (n: string) => { jar.delete(n); },
+  } as never, 0, { serverUrl: SERVER, authHeader: 'Bearer t', username: USER });
+}
+
+function files(dir: string): string[] {
+  return readdirSync(dir).sort();
+}
+
+function onlyFile(dir: string): string {
+  const [file] = files(dir).filter((name) => name.endsWith('.enc'));
+  return path.join(dir, file);
+}
+
+function asideOf(dir: string, file: string): string | undefined {
+  const name = files(dir).find((n) => n.startsWith(`${path.basename(file)}.unreadable-`));
+  return name && path.join(dir, name);
+}
+
+describe('settings that do not decrypt', () => {
+  it('read as nothing stored, and a file too short to hold any as well', async () => {
+    const { loadUserSettings, saveUserSettings } = await import('@/lib/settings-sync');
+    expect(await loadUserSettings(USER, SERVER)).toBeNull();
+
+    await saveUserSettings(USER, SERVER, { theme: 'dark' });
+    secret = 'b'.repeat(64);
+    expect(await loadUserSettings(USER, SERVER)).toBeNull();
+
+    writeFileSync(onlyFile(path.join(dataDir, 'settings')), Buffer.alloc(10));
+    expect(await loadUserSettings(USER, SERVER)).toBeNull();
+  });
+
+  it('are moved aside unchanged by the next save, which then goes through', async () => {
+    const { saveUserSettings, loadUserSettings } = await import('@/lib/settings-sync');
+    await saveUserSettings(USER, SERVER, { theme: 'dark', templates: [{ id: 'a' }] });
+    const dir = path.join(dataDir, 'settings');
+    const file = onlyFile(dir);
+    const before = readFileSync(file);
+    secret = 'b'.repeat(64);
+    await signIn();
+
+    const { GET, POST } = await import('@/app/api/settings/route');
+    const got = await GET(new NextRequest('https://webmail.example/api/settings', {
+      headers: { 'x-settings-username': USER, 'x-settings-server': SERVER },
+    }));
+    expect(got.status).toBe(200);
+    expect((await got.json()).settings).toBeNull();
+
+    const saved = await POST(new NextRequest('https://webmail.example/api/settings', {
+      method: 'POST',
+      headers: { 'sec-fetch-site': 'same-origin', 'content-type': 'application/json' },
+      body: JSON.stringify({ username: USER, serverUrl: SERVER, settings: { theme: 'light' } }),
+    }));
+    expect(saved.status).toBe(200);
+    expect(await loadUserSettings(USER, SERVER)).toEqual({ theme: 'light' });
+
+    const aside = asideOf(dir, file);
+    expect(aside).toBeDefined();
+    expect(readFileSync(aside!).equals(before)).toBe(true);
+    // With the old secret back, the moved file still holds the old settings.
+    secret = 'a'.repeat(64);
+    writeFileSync(file, readFileSync(aside!));
+    expect(await loadUserSettings(USER, SERVER)).toEqual({ theme: 'dark', templates: [{ id: 'a' }] });
+  });
+
+  it('a file too short to hold any is written over, not moved', async () => {
+    const { updateUserSettings, loadUserSettings, settingsId } = await import('@/lib/settings-sync');
+    const dir = path.join(dataDir, 'settings');
+    mkdirSync(dir, { recursive: true });
+    writeFileSync(path.join(dir, `${settingsId(USER, SERVER)}.enc`), Buffer.alloc(0));
+    await updateUserSettings(USER, SERVER, (stored) => ({ ...(stored ?? {}), theme: 'dark' }));
+    expect(files(dir)).toEqual([`${settingsId(USER, SERVER)}.enc`]);
+    expect(await loadUserSettings(USER, SERVER)).toEqual({ theme: 'dark' });
+  });
+});
+
+describe('a plugin vault that does not decrypt', () => {
+  const user = { username: USER, serverUrl: SERVER };
+  const input = { origin: 'https://cloud.example.org', inject: { type: 'basic' } as const, label: 'Nextcloud', secret: 'pw' };
+
+  it('reads as empty, and the next write moves it aside unchanged and goes through', async () => {
+    const { listUserCredentials, putUserCredential } = await import('@/lib/plugin-secrets/vault');
+    expect(await listUserCredentials(user)).toEqual([]);
+    await putUserCredential(user, 'nextcloud', 'cloud', input);
+    const dir = path.join(dataDir, 'plugin-secrets');
+    const file = onlyFile(dir);
+    const before = readFileSync(file);
+    secret = 'b'.repeat(64);
+
+    expect(await listUserCredentials(user)).toEqual([]);
+    await putUserCredential(user, 'other', 'x', input);
+    expect((await listUserCredentials(user)).map((c) => c.pluginId)).toEqual(['other']);
+    const aside = asideOf(dir, file);
+    expect(aside).toBeDefined();
+    expect(readFileSync(aside!).equals(before)).toBe(true);
+  });
+
+  it('lets the routes work again: list, connect, disconnect, and the proxy finds no credential', async () => {
+    const { putUserCredential } = await import('@/lib/plugin-secrets/vault');
+    await putUserCredential(user, 'nextcloud', 'cloud', input);
+    secret = 'b'.repeat(64);
+
+    const { GET, POST, DELETE } = await import('@/app/api/plugin-secrets/route');
+    const { POST: proxy } = await import('@/app/api/plugin-proxy/route');
+    const same = { 'sec-fetch-site': 'same-origin' };
+
+    const listed = await GET(new NextRequest('http://localhost/api/plugin-secrets?plugin=paperless'));
+    expect(listed.status).toBe(200);
+    expect(await listed.json()).toMatchObject({ credentials: [], admin: [{ credentialId: 'shared', configured: true }] });
+    const proxied = await proxy(new NextRequest(
+      'http://localhost/api/plugin-proxy?plugin=nextcloud&credential=cloud&url=https%3A%2F%2Fcloud.example.org%2F',
+      { method: 'POST', headers: same },
+    ));
+    expect(proxied.status).toBe(404);
+    const post = await POST(new NextRequest('http://localhost/api/plugin-secrets', {
+      method: 'POST',
+      headers: same,
+      body: JSON.stringify({ pluginId: 'nextcloud', credentialId: 'cloud', label: 'x', origin: 'https://cloud.example.org', inject: { type: 'basic' }, secret: 's' }),
+    }));
+    expect(post.status).toBe(200);
+    const del = await DELETE(new NextRequest('http://localhost/api/plugin-secrets?plugin=nextcloud', { method: 'DELETE', headers: same }));
+    expect(del.status).toBe(200);
+  });
+
+  it('resolves an admin credential without reading the user vault', async () => {
+    const { resolveCredential } = await import('@/lib/plugin-secrets/proxy');
+    // A vault that can't even be read (a folder where the file should be).
+    const { userKey } = await import('@/lib/plugin-secrets/vault');
+    mkdirSync(path.join(dataDir, 'plugin-secrets', `${userKey(user)}.enc`), { recursive: true });
+
+    expect(await resolveCredential(user, 'paperless', 'shared')).toMatchObject({ scope: 'admin', secret: 'admin-token' });
+    await expect(resolveCredential(user, 'nextcloud', 'cloud')).rejects.toThrow();
+  });
+});

lib/__tests__/telemetry-state-read-error.test.ts

@@ -0,0 +1,58 @@
+// @vitest-environment node
+import { mkdtempSync, readFileSync, rmSync, writeFileSync, mkdirSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+// A telemetry state file that couldn't be read (a torn write, a passing I/O
+// error) used to be replaced with the defaults, which turned the admin's
+// consent off for good. Now the read answers with the defaults and leaves
+// the file alone.
+
+vi.mock('@/lib/admin/third-party-mode', () => ({ isNoThirdPartyTrafficOn: async () => false }));
+
+let dir: string;
+
+beforeEach(() => {
+  dir = mkdtempSync(path.join(tmpdir(), 'bw-telemetry-state-'));
+  vi.stubEnv('TELEMETRY_DATA_DIR', dir);
+  vi.stubEnv('BULWARK_TELEMETRY', '');
+});
+
+afterEach(() => {
+  vi.unstubAllEnvs();
+  rmSync(dir, { recursive: true, force: true });
+});
+
+describe('telemetry state that cannot be read', () => {
+  it('keeps the stored consent', async () => {
+    const { effectiveConsent, loadState } = await import('@/lib/telemetry/state');
+    const good = JSON.stringify({ consent: 'on', endpoint: 'https://collector.example.org/', consentedAt: '2026-10-01T00:00:00.000Z', lastSentAt: null, nextScheduledAt: null });
+    mkdirSync(dir, { recursive: true });
+    const file = path.join(dir, 'state.json');
+    writeFileSync(file, good.slice(0, 20));
+
+    expect((await loadState()).consent).toBe('off');
+    expect((await effectiveConsent()).consent).toBe('off');
+    expect(readFileSync(file, 'utf8')).toBe(good.slice(0, 20));
+
+    writeFileSync(file, good);
+    expect((await effectiveConsent()).consent).toBe('on');
+  });
+
+  it('refuses to write over it', async () => {
+    const { updateState } = await import('@/lib/telemetry/state');
+    mkdirSync(dir, { recursive: true });
+    const file = path.join(dir, 'state.json');
+    writeFileSync(file, '{"consent":"on","endp');
+
+    await expect(updateState((state) => { state.nextScheduledAt = 'x'; })).rejects.toThrow('could not be read');
+    expect(readFileSync(file, 'utf8')).toBe('{"consent":"on","endp');
+  });
+
+  it('still writes the default state on a fresh install', async () => {
+    const { loadState } = await import('@/lib/telemetry/state');
+    expect((await loadState()).consent).toBe('off');
+    expect(JSON.parse(readFileSync(path.join(dir, 'state.json'), 'utf8')).consent).toBe('off');
+  });
+});

lib/__tests__/telemetry-version-check-shared-store.test.ts

@@ -0,0 +1,128 @@
+// @vitest-environment node
+import { mkdtempSync, readdirSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { setStoreForTests } from '@/lib/shared-store';
+import { resetLocalStoreForTests } from '@/lib/shared-store/local';
+import type { UpdateStatus } from '@/lib/version-check/types';
+import { separateDiskStore, type SeparateDiskStore } from './helpers/separate-disk-store';
+
+// Telemetry and the version check with a shared store (#1066): one instance
+// id, one login list and one state for the deployment.
+
+vi.mock('@/lib/admin/third-party-mode', () => ({ isNoThirdPartyTrafficOn: async () => false }));
+
+let disks: string;
+let store: SeparateDiskStore;
+
+function onReplica(name: 'a' | 'b') {
+  vi.stubEnv('TELEMETRY_DATA_DIR', path.join(disks, name, 'telemetry'));
+  vi.stubEnv('VERSION_CHECK_DATA_DIR', path.join(disks, name, 'version-check'));
+}
+
+beforeEach(() => {
+  disks = mkdtempSync(path.join(tmpdir(), 'bw-telemetry-replicas-'));
+  store = separateDiskStore();
+  setStoreForTests(store);
+  vi.stubEnv('BULWARK_TELEMETRY', '');
+  onReplica('a');
+});
+
+afterEach(() => {
+  setStoreForTests(null);
+  resetLocalStoreForTests();
+  vi.unstubAllEnvs();
+  vi.unstubAllGlobals();
+  store.cleanup();
+  rmSync(disks, { recursive: true, force: true });
+});
+
+describe('telemetry with a shared store', () => {
+  it('gives every replica one instance id, even when they ask at once', async () => {
+    const { getInstanceId } = await import('@/lib/telemetry/state');
+    const [first, second] = await Promise.all([getInstanceId(), getInstanceId()]);
+    expect(second).toBe(first);
+    onReplica('b');
+    expect(await getInstanceId()).toBe(first);
+  });
+
+  it('counts logins on every replica together and reports the same numbers everywhere', async () => {
+    // A fresh module copy per replica: nothing carried over in memory.
+    const tracker = async () => {
+      vi.resetModules();
+      return import('@/lib/telemetry/login-tracker');
+    };
+    const a = await tracker();
+    await Promise.all([
+      a.recordLogin('alice@example.org', 'https://mail.example.org'),
+      a.recordLogin('bob@example.org', 'https://mail.example.org'),
+    ]);
+    onReplica('b');
+    const b = await tracker();
+    await b.recordLogin('carol@example.org', 'https://mail.example.org');
+    expect(await b.getLoginCounts()).toEqual({ total: 3, active7d: 3 });
+    expect(await a.getLoginCounts()).toEqual({ total: 3, active7d: 3 });
+  });
+
+  it('does not report zero accounts while the store is down', async () => {
+    const { recordLogin, getLoginCounts } = await import('@/lib/telemetry/login-tracker');
+    await recordLogin('alice@example.org', 'https://mail.example.org');
+    store.down = true;
+    await expect(getLoginCounts()).rejects.toThrow('down (test)');
+    store.down = false;
+    expect(await getLoginCounts()).toEqual({ total: 1, active7d: 1 });
+  });
+
+  it('shows on replica B the consent set on replica A', async () => {
+    const { updateState, effectiveConsent } = await import('@/lib/telemetry/state');
+    await updateState((state) => { state.consent = 'on'; });
+    onReplica('b');
+    expect((await effectiveConsent()).consent).toBe('on');
+  });
+});
+
+function status(current: string, latest: string): UpdateStatus {
+  return {
+    schema: 1, current, latest, updateAvailable: current !== latest, severity: current !== latest ? 'normal' : 'none',
+    url: null, advisory: null, checkedAt: new Date().toISOString(),
+  };
+}
+
+describe('version check with a shared store', () => {
+  it('keeps each running version\'s status during a rolling update', async () => {
+    vi.stubEnv('BULWARK_UPDATE_CHECK_URL', 'https://version.example/');
+    const answer = (current: string) => vi.stubGlobal('fetch', vi.fn(async () => new Response(JSON.stringify(status(current, '1.21.0')))));
+    const { checkOnce, loadState, freshStatus } = await import('@/lib/version-check');
+
+    vi.stubEnv('NEXT_PUBLIC_APP_VERSION', '1.20.0');
+    answer('1.20.0');
+    expect((await checkOnce()).ok).toBe(true);
+
+    onReplica('b');
+    vi.stubEnv('NEXT_PUBLIC_APP_VERSION', '1.21.0');
+    answer('1.21.0');
+    expect((await checkOnce()).ok).toBe(true);
+    expect(freshStatus(await loadState())).toMatchObject({ current: '1.21.0', updateAvailable: false });
+
+    // A replica still on the old version keeps showing its banner.
+    onReplica('a');
+    vi.stubEnv('NEXT_PUBLIC_APP_VERSION', '1.20.0');
+    expect(freshStatus(await loadState())).toMatchObject({ current: '1.20.0', updateAvailable: true });
+  });
+});
+
+describe('telemetry and version check on a single instance', () => {
+  it('keep today\'s files', async () => {
+    setStoreForTests(null);
+    const { getInstanceId, loadState } = await import('@/lib/telemetry/state');
+    const { recordLogin } = await import('@/lib/telemetry/login-tracker');
+    const versionCheck = await import('@/lib/version-check');
+    await getInstanceId();
+    await loadState();
+    await recordLogin('alice@example.org', 'https://mail.example.org');
+    await versionCheck.saveState({ ...(await versionCheck.loadState()), lastCheckedAt: 'x' });
+    expect(readdirSync(path.join(disks, 'a', 'telemetry')).sort()).toEqual(['.telemetry-id', 'logins.json', 'state.json']);
+    expect(readdirSync(path.join(disks, 'a', 'version-check'))).toEqual(['state.json']);
+  });
+});

lib/__tests__/version-routes-store-down.test.ts

@@ -0,0 +1,54 @@
+// @vitest-environment node
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { NextRequest } from 'next/server';
+import { setStoreForTests } from '@/lib/shared-store';
+import { resetLocalStoreForTests } from '@/lib/shared-store/local';
+import { separateDiskStore, type SeparateDiskStore } from './helpers/separate-disk-store';
+
+// With the shared store down, the update banner's routes answered 500. They
+// answer 503 with Retry-After now, like the other routes that need the store.
+
+vi.mock('@/lib/admin/session', () => ({ requireAdminAuth: async () => ({ session: { sub: 'admin' } }) }));
+vi.mock('@/lib/admin/third-party-mode', () => ({ isNoThirdPartyTrafficOn: async () => false }));
+vi.mock('@/lib/auth/instance-details', () => ({ canSeeInstanceDetails: async () => true }));
+vi.mock('@/lib/logger', () => ({ logger: { warn: () => {}, error: () => {}, info: () => {}, debug: () => {} } }));
+
+import { GET as updateStatus } from '@/app/api/system/update-status/route';
+import { GET as adminVersion, POST as adminCheck } from '@/app/api/admin/version/route';
+
+let store: SeparateDiskStore;
+
+beforeEach(() => {
+  store = separateDiskStore();
+  setStoreForTests(store);
+  vi.stubEnv('BULWARK_UPDATE_CHECK_URL', 'https://version.example/');
+  vi.stubEnv('NEXT_PUBLIC_APP_VERSION', '1.20.0');
+});
+
+afterEach(() => {
+  setStoreForTests(null);
+  resetLocalStoreForTests();
+  vi.unstubAllEnvs();
+  store.cleanup();
+});
+
+const req = (url: string, init?: RequestInit) => new NextRequest(`http://localhost${url}`, init as never);
+
+describe('update status routes while the shared store is down', () => {
+  it('answer 503 with Retry-After, and 200 again once it is back', async () => {
+    store.down = true;
+    const responses = [
+      await updateStatus(req('/api/system/update-status')),
+      await adminVersion(req('/api/admin/version')),
+      await adminCheck(req('/api/admin/version', { method: 'POST', body: JSON.stringify({ action: 'check-now' }) })),
+    ];
+    for (const res of responses) {
+      expect(res.status).toBe(503);
+      expect(res.headers.get('retry-after')).toBe('10');
+    }
+
+    store.down = false;
+    expect((await updateStatus(req('/api/system/update-status'))).status).toBe(200);
+    expect((await adminVersion(req('/api/admin/version'))).status).toBe(200);
+  });
+});

lib/account-archive-storage.ts

@@ -1,41 +1,53 @@
-import { createHash, randomBytes } from 'node:crypto';
-import { mkdir, open, readFile, rename, unlink } from 'node:fs/promises';
+import { createHash } from 'node:crypto';
+import { mkdir } from 'node:fs/promises';
 import path from 'node:path';
+import { getStore, isSharedMode, type CollectionOptions } from '@/lib/shared-store';
 import { archiveIdentity, parseArchiveEnvelope, type ArchiveEnvelope, type ArchiveOwner } from './account-archive';
 
 /** One file per owner: `SETTINGS_DATA_DIR/account-archives/<sha256 of the owner>.json`. */
 function archiveDir(): string {
   return path.join(process.env.SETTINGS_DATA_DIR || path.join(process.cwd(), 'data', 'settings'), 'account-archives');
 }
-function archivePath(owner: ArchiveOwner): string {
-  return path.join(archiveDir(), createHash('sha256').update(archiveIdentity(owner)).digest('hex') + '.json');
+
+const ARCHIVE_FILE = /^[0-9a-f]{64}\.json$/;
+
+/**
+ * One document per owner (#1066), replaced whole on every save. Local mode:
+ * today's files, mode 0600. The archive is encrypted in the browser with
+ * the user's password; the server holds no key.
+ */
+export const ARCHIVE_COLLECTION: CollectionOptions = {
+  local: {
+    dir: archiveDir,
+    file: (id) => `${id}.json`,
+    idOf: (relative) => (ARCHIVE_FILE.test(relative) ? relative.slice(0, -'.json'.length) : null),
+    mode: 0o600,
+    // Written to disk before the rename, as it always was.
+    fsync: true,
+  },
+};
+
+function collection() {
+  return getStore().collection('account-archives', ARCHIVE_COLLECTION);
+}
+
+function archiveId(owner: ArchiveOwner): string {
+  return createHash('sha256').update(archiveIdentity(owner)).digest('hex');
 }
 
 export async function loadArchive(owner: ArchiveOwner): Promise<ArchiveEnvelope | null> {
-  try { return parseArchiveEnvelope(JSON.parse(await readFile(archivePath(owner), 'utf8'))); }
-  catch (err) {
-    if ((err as NodeJS.ErrnoException).code === 'ENOENT') return null;
-    throw err;
-  }
+  const stored = await collection().get(archiveId(owner));
+  return stored ? parseArchiveEnvelope(JSON.parse(stored.data.toString('utf8'))) : null;
 }
 
-/** Replaces the owner's archive atomically: a reader sees the old file or the new one, never half of it. */
+/** Replaces the owner's archive atomically: a reader sees the old one or the new one, never half of it. */
 export async function saveArchive(owner: ArchiveOwner, envelope: ArchiveEnvelope): Promise<void> {
-  const target = archivePath(owner);
-  const temporary = `${target}.${randomBytes(8).toString('hex')}.tmp`;
-  await mkdir(archiveDir(), { recursive: true, mode: 0o700 });
-  try {
-    const file = await open(temporary, 'wx', 0o600);
-    try { await file.writeFile(JSON.stringify(parseArchiveEnvelope(envelope))); await file.sync(); }
-    finally { await file.close(); }
-    await rename(temporary, target);
-  } finally {
-    await unlink(temporary).catch(() => {});
-  }
+  const data = JSON.stringify(parseArchiveEnvelope(envelope));
+  // The folder only its owner may list, as it has always been created.
+  if (!isSharedMode()) await mkdir(archiveDir(), { recursive: true, mode: 0o700 });
+  await collection().put(archiveId(owner), data);
 }
 
 export async function deleteArchive(owner: ArchiveOwner): Promise<void> {
-  await unlink(archivePath(owner)).catch(err => {
-    if ((err as NodeJS.ErrnoException).code !== 'ENOENT') throw err;
-  });
+  await collection().delete(archiveId(owner));
 }

lib/admin/__tests__/config-manager-shared.test.ts

@@ -39,6 +39,8 @@ import { separateFakeStore } from './shared-store-fake';
 
 let dir: string;
 let store: ReturnType<typeof separateFakeStore>;
+/** Runs inside the store's document calls, to land another replica's change mid-request. */
+const hooks: { onGet?: (id: string) => Promise<void>; afterUpdate?: (id: string) => Promise<void> } = {};
 
 const stored = async (id: string) => {
   const doc = await adminConfigDocs().get(id);
@@ -71,6 +73,24 @@ beforeEach(async () => {
   resetLocalStoreForTests();
   resetChangesForTests();
   store = separateFakeStore(path.join(dir, 'store'));
+  const collection = store.collection.bind(store);
+  store.collection = (name, options) => {
+    const docs = collection(name, options);
+    return {
+      ...docs,
+      get: async (id) => {
+        await hooks.onGet?.(id);
+        return docs.get(id);
+      },
+      update: async (id, fn, updateOptions) => {
+        const result = await docs.update(id, fn, updateOptions);
+        const hook = hooks.afterUpdate;
+        hooks.afterUpdate = undefined;
+        await hook?.(id);
+        return result;
+      },
+    };
+  };
   setStoreForTests(store);
   await store.epochs.bump(INIT_EPOCH);
   await adminConfigDocs().put('config.json', JSON.stringify({ appName: 'First', setupComplete: true }));
@@ -81,6 +101,8 @@ beforeEach(async () => {
 });
 
 afterEach(async () => {
+  hooks.onGet = undefined;
+  hooks.afterUpdate = undefined;
   setStoreForTests(null);
   resetChangesForTests();
   resetLocalStoreForTests();
@@ -160,7 +182,82 @@ describe('config manager with a shared store', () => {
   });
 });
 
+describe('change notices racing writes', () => {
+  it('fails a reload when writes of this replica kept landing while it read, so the poller tries again', async () => {
+    let reads = 0;
+    let n = 0;
+    hooks.onGet = async (id) => {
+      if (id !== 'config.json') return;
+      reads++;
+      await configManager.setAdminConfig({ loginShowVersion: ++n % 2 === 0 });
+    };
+    // The listener the change poller runs for a remote change.
+    const reload = (configManager as unknown as { reloadShared(area: string): Promise<void> }).reloadShared('config');
+    await expect(reload).rejects.toThrow(/kept changing/);
+    expect(reads).toBe(3);
+  });
+
+  it('keeps another replica\'s change that was reloaded while a save of this one was in flight', async () => {
+    hooks.afterUpdate = async () => {
+      const current = await stored('config.json');
+      await otherReplicaSaves('config.json', { ...current, appName: 'Saved elsewhere' });
+      await pollChanges();
+    };
+    await configManager.setAdminConfig({ loginShowVersion: false });
+    expect(configManager.get('appName')).toBe('Saved elsewhere');
+    expect(configManager.get('loginShowVersion')).toBe(false);
+  });
+});
+
+describe('updateAdminConfig with a shared store', () => {
+  it('builds each of two saves on two replicas on the other\'s result', async () => {
+    const other = await otherReplicaManager();
+    const add = (host: string) => (list: Array<{ host: string }>) => [...list, { host }];
+    await Promise.all([
+      configManager.updateAdminConfig('domainBranding', add('a.example'), []),
+      other.updateAdminConfig('domainBranding', add('b.example'), []),
+    ]);
+    const hosts = (await stored('config.json')).domainBranding.map((e: { host: string }) => e.host).sort();
+    expect(hosts).toEqual(['a.example', 'b.example']);
+  });
+
+  it('starts from the value get() would give and leaves the config alone on undefined', async () => {
+    vi.stubEnv('APP_NAME', 'From env');
+    expect(await configManager.updateAdminConfig<string>('appName', (current) => `${current}!`)).toBe('First!');
+    await configManager.removeAdminOverride('appName');
+    expect(await configManager.updateAdminConfig<string>('appName', (current) => `${current}!`)).toBe('From env!');
+    const before = await adminConfigDocs().get('config.json');
+    expect(await configManager.updateAdminConfig('appName', () => undefined)).toBeUndefined();
+    expect((await adminConfigDocs().get('config.json'))!.version).toBe(before!.version);
+  });
+});
+
 describe('policy saves with a shared store', () => {
+  it('lets a save replace a stored policy that doesn\'t parse', async () => {
+    await adminConfigDocs().put('policy.json', '{ not json');
+    await store.epochs.bump('policy');
+    await pollChanges();
+    const page = await load();
+    expect(page.etag).toBe(`"${(await adminConfigDocs().get('policy.json'))!.version}"`);
+    expect((await save(page.policy, page.etag)).status).toBe(200);
+    expect(await stored('policy.json')).toMatchObject({ features: page.policy.features });
+
+    // The same after a restart.
+    await adminConfigDocs().put('policy.json', '{ not json');
+    await configManager.reload();
+    const again = await load();
+    expect((await save(again.policy, again.etag)).status).toBe(200);
+  });
+
+  it('answers a save with the stored policy and its version', async () => {
+    const page = await load();
+    const res = await save({ ...page.policy, features: { ...page.policy.features, calendarEnabled: false } }, page.etag);
+    const body = await res.json();
+    expect(body.policy.features.calendarEnabled).toBe(false);
+    expect(body.policy.scopes).toEqual([]);
+    expect(res.headers.get('ETag')).toBe(`"${(await adminConfigDocs().get('policy.json'))!.version}"`);
+  });
+
   async function load() {
     const res = await GET(new NextRequest('https://mail.example/api/admin/policy'));
     return { policy: await res.json(), etag: res.headers.get('ETag')! };

lib/admin/__tests__/shared-store-fake.ts

@@ -10,13 +10,14 @@ import { fakeSharedStore } from '@/lib/shared-store/__tests__/fake-shared';
  * Redis would hold them apart from this replica's files. Without that, code
  * that still read and wrote its local files would pass for shared.
  *
- * `down` makes every document, log and counter call fail as an outage would.
+ * `down` makes every document, log, counter and kv call fail as an outage would.
  */
 export function separateFakeStore(root: string): SharedStore & { down: boolean } {
   const fake = fakeSharedStore();
   const guard = <T>(fn: () => Promise<T>): Promise<T> =>
     fake.down ? Promise.reject(new StoreUnavailableError('down (test)')) : fn();
   const counters = fake.counters;
+  const kv = fake.kv;
   return Object.assign(fake, {
     collection(name: string, options: CollectionOptions) {
       // Read-only config comes from the mounted files, as in the Redis store.
@@ -35,10 +36,19 @@ export function separateFakeStore(root: string): SharedStore & { down: boolean }
       const log = localLog(name, { ...options, local: { ...options.local, path: () => path.join(root, 'logs', `${name}.log`) } });
       return {
         append: (line: string) => guard(() => log.append(line)),
+        appendMany: (lines: string[]) => guard(() => log.appendMany(lines)),
+        clear: () => guard(() => log.clear()),
         readAll: () => guard(() => log.readAll()),
         length: () => guard(() => log.length()),
       };
     },
+    kv: {
+      get: (key: string) => guard(() => kv.get(key)),
+      set: (...args: Parameters<typeof kv.set>) => guard(() => kv.set(...args)),
+      del: (key: string) => guard(() => kv.del(key)),
+      take: (key: string) => guard(() => kv.take(key)),
+      update: (...args: Parameters<typeof kv.update>) => guard(() => kv.update(...args)),
+    },
     counters: {
       reserve: (...args: Parameters<typeof counters.reserve>) => guard(() => counters.reserve(...args)),
       refund: (...args: Parameters<typeof counters.refund>) => guard(() => counters.refund(...args)),

lib/admin/admin-store.ts

@@ -1,4 +1,4 @@
-import { getStore, type CollectionOptions, type DocCollection } from '@/lib/shared-store';
+import { getStore, isStoreUnavailableError, type CollectionOptions, type DocCollection } from '@/lib/shared-store';
 import { getConfigDir, getStateDir } from './paths';
 
 /**
@@ -32,12 +32,7 @@ export function adminStateDocs(): DocCollection {
   return getStore().collection(ADMIN_STATE_COLLECTION, adminStateOptions);
 }
 
-/**
- * Whether `error` is the shared store's StoreUnavailableError. By name, not
- * instanceof: the store on globalThis may come from another bundle's copy
- * of the shared-store module (proxy, instrumentation, routes), whose class
- * is a different object.
- */
+/** Whether `error` is the shared store's StoreUnavailableError, from any bundle's module copy. */
 export function isStoreUnavailable(error: unknown): boolean {
-  return error instanceof Error && error.name === 'StoreUnavailableError';
+  return isStoreUnavailableError(error);
 }

lib/admin/config-manager.ts

@@ -70,9 +70,12 @@ export function isPolicyConflict(error: unknown): error is PolicyConflictError {
 
 /** Policy writes in this process, one after another, so a version check and its write can't interleave. */
 const POLICY_WRITE_QUEUE = 'admin-config\0policy.json';
+/** Config read-modify-writes (updateAdminConfig) in this process, one after another. */
+const CONFIG_WRITE_QUEUE = 'admin-config\0config.json';
 
 interface ParsedDoc {
-  data: Record<string, unknown>;
+  /** null when the stored document doesn't parse (shared mode). */
+  data: Record<string, unknown> | null;
   /** As the shared store computes it: changes whenever the bytes do. */
   version: string;
 }
@@ -81,9 +84,19 @@ function parseDoc(raw: string): ParsedDoc {
   return { data: JSON.parse(raw), version: versionOf(Buffer.from(raw, 'utf-8')) };
 }
 
-/** A document from the shared store; throws when it doesn't parse. */
-function parseStored(doc: StoredDoc | null): ParsedDoc | null {
-  return doc ? { data: JSON.parse(doc.data.toString('utf-8')), version: doc.version } : null;
+/**
+ * A document from the shared store. Data that doesn't parse comes back as
+ * null with the document's version, so an If-Match save can still replace
+ * it.
+ */
+function parseStored(doc: StoredDoc | null, onError?: (error: unknown) => void): ParsedDoc | null {
+  if (!doc) return null;
+  try {
+    return { data: JSON.parse(doc.data.toString('utf-8')), version: doc.version };
+  } catch (error) {
+    onError?.(error);
+    return { data: null, version: doc.version };
+  }
 }
 
 /** Which stored document a change area reloads. */
@@ -113,6 +126,8 @@ class ConfigManager {
   private sharedLoad: Promise<void> | null = null;
   /** Shared mode: counts writes applied from their result, so a reload that raced one reads again. */
   private writes = 0;
+  /** Shared mode: counts reloads applied, so a write that raced one reads its result again. */
+  private reloads = 0;
   private unlisten: (() => void) | null = null;
 
   /**
@@ -159,12 +174,9 @@ class ConfigManager {
 
   /** As a broken file on disk: logged, and the defaults apply. */
   private parseOrWarn(filename: string, doc: StoredDoc | null): ParsedDoc | null {
-    try {
-      return parseStored(doc);
-    } catch (error) {
+    return parseStored(doc, (error) => {
       logger.warn(`Failed to read ${filename} from the shared store`, { error: error instanceof Error ? error.message : 'Unknown error' });
-      return null;
-    }
+    });
   }
 
   /** Reload config or policy when another replica changed it. Registered again on every load. */
@@ -178,24 +190,53 @@ class ConfigManager {
   }
 
   /**
-   * Read one document again. Throws when the store can't be read or the
-   * document doesn't parse, keeping the copy in memory; the change poller
-   * then tries again.
+   * Read one document again. Throws when the store can't be read, or when
+   * writes of this process kept landing while reading; the change poller
+   * then tries again. A document that doesn't parse keeps the copy in
+   * memory (the policy takes its version, so a save can replace it).
    */
   private async reloadShared(area: ConfigArea): Promise<void> {
     for (let attempt = 0; attempt < 3; attempt++) {
       const writes = this.writes;
-      const doc = parseStored(await adminConfigDocs().get(AREA_FILES[area]));
+      const doc = this.parseOrWarn(AREA_FILES[area], await adminConfigDocs().get(AREA_FILES[area]));
       // A write of this process landed while reading: its result may be newer.
       if (writes !== this.writes) continue;
-      if (area === 'config') {
-        this.adminConfig = doc?.data || {};
-        this.applyLoggerSettings();
-      } else {
-        this.applyPolicyDoc(doc);
-      }
+      this.applyReloaded(area, doc);
       return;
     }
+    throw new Error(`${AREA_FILES[area]} kept changing while it was reloaded`);
+  }
+
+  private applyReloaded(area: ConfigArea, doc: ParsedDoc | null): void {
+    this.reloads++;
+    if (doc && doc.data === null) {
+      if (area === 'policy') this.policyVersion = doc.version;
+      return;
+    }
+    if (area === 'config') {
+      this.adminConfig = doc?.data || {};
+      this.applyLoggerSettings();
+    } else {
+      this.applyPolicyDoc(doc);
+    }
+  }
+
+  /**
+   * After a write of this process: apply its result, unless a reload of
+   * another replica's change was applied while it was in flight. That one
+   * may be newer than the write, so the document is read again instead.
+   */
+  private async applyWritten(area: ConfigArea, reloadsBefore: number, apply: () => void): Promise<void> {
+    this.writes++;
+    if (this.reloads === reloadsBefore) {
+      apply();
+      return;
+    }
+    try {
+      this.applyReloaded(area, this.parseOrWarn(AREA_FILES[area], await adminConfigDocs().get(AREA_FILES[area])));
+    } catch {
+      apply();
+    }
   }
 
   /**
@@ -203,8 +244,9 @@ class ConfigManager {
    * the object it gets), so only the keys a request names change and a
    * concurrent save of other keys on another replica stays.
    */
-  private async updateSharedConfig(change: (config: Record<string, unknown>) => void): Promise<void> {
+  private async updateSharedConfig(change: (config: Record<string, unknown>) => void | false): Promise<void> {
     const inMemory = this.adminConfig;
+    const reloads = this.reloads;
     const result = await adminConfigDocs().update('config.json', (current) => {
       let base: Record<string, unknown> = {};
       if (current) {
@@ -215,12 +257,14 @@ class ConfigManager {
           base = { ...inMemory };
         }
       }
-      change(base);
+      if (change(base) === false) return undefined;
       return JSON.stringify(base, null, 2);
     });
-    this.writes++;
-    if (result.data) this.adminConfig = JSON.parse(result.data.toString('utf-8'));
-    this.applyLoggerSettings();
+    if (result.data === undefined) return;
+    await this.applyWritten('config', reloads, () => {
+      if (result.data) this.adminConfig = JSON.parse(result.data.toString('utf-8'));
+      this.applyLoggerSettings();
+    });
     await announceChange('config');
   }
 
@@ -298,12 +342,17 @@ class ConfigManager {
     // Synchronous callers can't wait: a due check runs in the background and
     // the next read sees its result.
     void this.checkForChanges();
+    return ConfigManager.resolve(this.adminConfig, key, defaultValue);
+  }
+
+  /** A value as get() reads it, from the admin overrides in `adminConfig`. */
+  private static resolve<T>(adminConfig: Record<string, unknown>, key: string, defaultValue?: T): T {
     // Admin override (highest priority)
     const mapping = CONFIG_ENV_MAP[key];
-    if (key in this.adminConfig) {
-      return this.adminConfig[key] as T;
-    } else if (mapping && mapping.fileKey && mapping.fileKey in this.adminConfig) {
-      const fileVal = readFileEnv(<string | undefined>this.adminConfig[mapping.fileKey]);
+    if (key in adminConfig) {
+      return adminConfig[key] as T;
+    } else if (mapping && mapping.fileKey && mapping.fileKey in adminConfig) {
+      const fileVal = readFileEnv(<string | undefined>adminConfig[mapping.fileKey]);
       if (fileVal !== null) {
         return parseEnvValue(fileVal, mapping.type) as T;
       }
@@ -380,7 +429,11 @@ class ConfigManager {
    */
   async setAdminConfig(updates: Record<string, unknown>): Promise<void> {
     assertWritable('update admin config');
-    if (this.usesStore()) return this.updateSharedConfig((config) => Object.assign(config, updates));
+    if (this.usesStore()) {
+      return this.updateSharedConfig((config) => {
+        Object.assign(config, updates);
+      });
+    }
     // Start from what is on disk, so a save here doesn't undo a change another
     // replica saved a moment ago. Same before every write below.
     await this.reloadIfChanged();
@@ -389,6 +442,36 @@ class ConfigManager {
     this.applyLoggerSettings();
   }
 
+  /**
+   * Set one admin override to `change(current)`, where `current` is the value
+   * as get() would return it from the stored config (override, else env,
+   * else `defaultValue`). For values built on the current one, such as a
+   * list with one entry changed: two saves at once, here or on another
+   * replica, each build on the other's result. Return undefined to leave
+   * it. Returns the value stored.
+   */
+  async updateAdminConfig<T>(key: string, change: (current: T) => T | undefined, defaultValue?: T): Promise<T | undefined> {
+    assertWritable('update admin config');
+    let stored: T | undefined;
+    if (this.usesStore()) {
+      await this.updateSharedConfig((config) => {
+        stored = change(ConfigManager.resolve(config, key, defaultValue));
+        if (stored === undefined) return false;
+        config[key] = stored;
+      });
+      return stored;
+    }
+    return serializeLocal(CONFIG_WRITE_QUEUE, async () => {
+      await this.reloadIfChanged();
+      stored = change(ConfigManager.resolve(this.adminConfig, key, defaultValue));
+      if (stored === undefined) return undefined;
+      this.adminConfig[key] = stored;
+      await this.writeJsonFile('config.json', this.adminConfig);
+      this.applyLoggerSettings();
+      return stored;
+    });
+  }
+
   /**
    * Remove an admin override, reverting to env/default.
    */
@@ -446,6 +529,15 @@ class ConfigManager {
     return this.policyVersion ?? DEFAULT_POLICY_VERSION;
   }
 
+  /**
+   * The policy and its version, read together. Use this whenever both go
+   * out (a body with its ETag): an await between two separate reads lets a
+   * change from another replica pair an old policy with the new version.
+   */
+  getPolicyWithVersion(): { policy: SettingsPolicy; version: string } {
+    return { policy: this.getPolicy(), version: this.getPolicyVersion() };
+  }
+
   /**
    * Update the settings policy. Writes to disk. With `ifVersion` (from
    * getPolicyVersion when the caller read the policy) it refuses with a
@@ -477,6 +569,8 @@ class ConfigManager {
     assertWritable('update settings policy');
     if (this.usesStore()) return this.updateSharedPolicy(change);
     await serializeLocal(POLICY_WRITE_QUEUE, async () => {
+      // Change what is on disk now, not what this process saw last.
+      await this.reloadIfChanged();
       const next = change(this.policyCache);
       if (next === undefined) return;
       this.policyCache = ConfigManager.completePolicy(next);
@@ -487,24 +581,29 @@ class ConfigManager {
   private async setSharedPolicy(policy: SettingsPolicy, ifVersion: string | undefined): Promise<string> {
     const completed = ConfigManager.completePolicy(policy);
     const expected = ifVersion === DEFAULT_POLICY_VERSION ? null : ifVersion;
+    const reloads = this.reloads;
     const result = await adminConfigDocs().put(
       'policy.json',
       JSON.stringify(completed, null, 2),
       expected === undefined ? {} : { ifVersion: expected },
     );
-    this.writes++;
     if (!result.ok) {
+      this.writes++;
       // Show what is stored now, so a retry or a reload compares with it.
-      this.applyPolicyDoc(this.parseOrWarn('policy.json', result.current));
+      this.applyReloaded('policy', this.parseOrWarn('policy.json', result.current));
       throw new PolicyConflictError();
     }
-    this.policyCache = completed;
-    this.policyVersion = result.version;
+    const version = result.version;
+    await this.applyWritten('policy', reloads, () => {
+      this.policyCache = completed;
+      this.policyVersion = version;
+    });
     await announceChange('policy');
-    return result.version;
+    return version;
   }
 
   private async updateSharedPolicy(change: (current: SettingsPolicy) => SettingsPolicy | undefined): Promise<void> {
+    const reloads = this.reloads;
     const result = await adminConfigDocs().update('policy.json', (current) => {
       const stored = current
         ? ConfigManager.completePolicy(JSON.parse(current.toString('utf-8')) as Partial<SettingsPolicy>)
@@ -513,8 +612,9 @@ class ConfigManager {
       return next === undefined ? undefined : JSON.stringify(ConfigManager.completePolicy(next), null, 2);
     });
     if (result.data === undefined) return;
-    this.writes++;
-    this.applyPolicyDoc(result.data && result.version ? parseStored({ data: result.data, version: result.version }) : null);
+    await this.applyWritten('policy', reloads, () => {
+      this.applyPolicyDoc(result.data && result.version ? parseStored({ data: result.data, version: result.version }) : null);
+    });
     await announceChange('policy');
   }
 

lib/admin/config-read-only-response.ts

@@ -1,13 +1,34 @@
 import { NextResponse } from 'next/server';
-import { ConfigReadOnlyError } from './paths';
+import { isStoreUnavailableError } from '@/lib/shared-store';
 
 /**
- * The 409 an admin route answers when a write hit ADMIN_CONFIG_READONLY,
- * carrying the ConfigReadOnlyError message so the panel can show why
- * nothing was saved (#1249). Returns null for any other error, which the
- * route keeps reporting as a 500.
+ * The answer an admin or setup route gives when a config write failed for
+ * a reason the admin should see, or null for any other error, which the
+ * route keeps reporting as a 500:
+ *
+ * - 409 when the write hit ADMIN_CONFIG_READONLY, carrying the
+ *   ConfigReadOnlyError message so the panel can show why nothing was
+ *   saved (#1249).
+ * - 503 with Retry-After when the shared store can't be reached (#1066):
+ *   nothing was saved, and trying again in a moment may work.
+ *
+ * By the error's name, not instanceof: the config manager on globalThis
+ * comes from another bundle's copy of lib/admin/paths.ts (the
+ * instrumentation creates it), whose ConfigReadOnlyError is a different
+ * class, and a read-only refusal then answered 500.
  */
-export function configReadOnlyResponse(error: unknown): NextResponse | null {
-  if (!(error instanceof ConfigReadOnlyError)) return null;
-  return NextResponse.json({ error: error.message }, { status: 409 });
+export function configWriteErrorResponse(error: unknown): NextResponse | null {
+  if (error instanceof Error && error.name === 'ConfigReadOnlyError') {
+    return NextResponse.json({ error: error.message }, { status: 409 });
+  }
+  if (isStoreUnavailableError(error)) {
+    return NextResponse.json(
+      { error: 'Nothing was saved: the shared store can\'t be reached right now. Try again in a moment.' },
+      { status: 503, headers: { 'Retry-After': '10' } },
+    );
+  }
+  return null;
 }
+
+/** The same as configWriteErrorResponse; the name the admin routes use. */
+export const configReadOnlyResponse = configWriteErrorResponse;

lib/admin/password.ts

@@ -330,7 +330,25 @@ export async function verifyAdminPassword(password: string): Promise<boolean> {
 }
 
 /**
- * Change the admin password. Returns true on success.
+ * Two password changes at once (on two replicas): the one that came second
+ * was checked against a password that no longer applies.
+ */
+export class AdminPasswordConflictError extends Error {
+  constructor() {
+    super('The admin password was changed somewhere else a moment ago. Sign in with the new password and try again.');
+    this.name = 'AdminPasswordConflictError';
+  }
+}
+
+/** By name, as the class may come from another bundle's copy of this module. */
+export function isAdminPasswordConflict(error: unknown): error is AdminPasswordConflictError {
+  return error instanceof Error && error.name === 'AdminPasswordConflictError';
+}
+
+/**
+ * Change the admin password. Returns true on success, false when the
+ * current password is wrong. Throws AdminPasswordConflictError when another
+ * change landed between the check and the write (shared store).
  */
 export async function changeAdminPassword(currentPassword: string, newPassword: string): Promise<boolean> {
   const shared = isSharedMode();
@@ -347,7 +365,7 @@ export async function changeAdminPassword(currentPassword: string, newPassword:
   const hash = await hashPassword(newPassword);
   const next: AdminConfigData = { passwordHash: hash, passwordHashFile: undefined };
   if (shared) {
-    if (!(await writeConfigData(next, { ifVersion: stored!.version })).ok) return false;
+    if (!(await writeConfigData(next, { ifVersion: stored!.version })).ok) throw new AdminPasswordConflictError();
   } else {
     await writeConfigData(next);
   }

lib/admin/session.ts

@@ -266,12 +266,13 @@ export async function clearAdminSessionCookie(): Promise<void> {
   try {
     await revokeAdminSession(payload.sid, payload.exp);
   } catch (error) {
-    // Local mode keeps it in memory regardless; only a restart forgets it.
-    // A shared store that is down keeps every admin request out until it
-    // is back, but then the token works again until it expires.
     logger.warn('Failed to save the admin session sign-out', {
       error: error instanceof Error ? error.message : 'Unknown error',
     });
+    // Local mode keeps it in memory regardless; only a restart forgets it.
+    // With a shared store nothing kept it: once the store is back a copy of
+    // the cookie would work again until it expires, so the caller is told.
+    if (isStoreUnavailable(error)) throw error;
   }
 }
 

lib/plugin-secrets/proxy.ts

@@ -12,6 +12,7 @@ import { DisallowedUrlError, fetchPublicUrl } from '@/lib/security/url-guard';
 import { authHeaderFor, credentialOrigin, privateProxyOrigins, type CredentialInject, type CredentialSpec } from './credentials';
 import { getUserCredential, setUserCredentialSecret, userKey, type VaultUser } from './vault';
 import { declaredUserCredential } from './known-plugins';
+import { isStoredDataUnavailable } from '@/lib/stored-data-errors';
 
 /** Largest request body the proxy forwards (one upload chunk). */
 export const MAX_PROXY_REQUEST_BYTES = 100 * 1024 * 1024;
@@ -68,7 +69,12 @@ export interface ResolvedCredential {
  * admin configured for an installed plugin that declares it.
  */
 export async function resolveCredential(user: VaultUser, pluginId: string, credentialId: string): Promise<ResolvedCredential | null> {
-  const stored = await getUserCredential(user, pluginId, credentialId);
+  const plugin = await getPlugin(pluginId);
+  const spec = (plugin as { credentials?: Record<string, CredentialSpec> } | null)?.credentials?.[credentialId];
+  // A credential the admin set up never depends on the user's own vault: a
+  // user can only store credentials a plugin declares for users, so there
+  // is nothing of theirs to find under this id.
+  const stored = spec?.scope === 'admin' ? null : await getUserCredential(user, pluginId, credentialId);
   if (stored) {
     // Only for a plugin this server knows declares it, on the server the
     // manifest fixes (see declaredUserCredential).
@@ -80,8 +86,6 @@ export async function resolveCredential(user: VaultUser, pluginId: string, crede
       trusted: privateProxyOrigins().includes(stored.origin), scope: 'user',
     };
   }
-  const plugin = await getPlugin(pluginId);
-  const spec = (plugin as { credentials?: Record<string, CredentialSpec> } | null)?.credentials?.[credentialId];
   if (!plugin || !plugin.enabled || !spec || spec.scope !== 'admin') return null;
   const config = await getPluginConfig(pluginId);
   const secret = config[spec.secretKey];
@@ -269,7 +273,15 @@ export async function proxyRequest(input: ProxyInput): Promise<Response> {
   const method = input.method.toUpperCase();
   if (!PROXY_METHODS.has(method)) throw new ProxyError(`Method ${method} is not allowed`, 405, 'bad_method');
 
-  const credential = await resolveCredential(input.user, input.pluginId, input.credentialId);
+  let credential: ResolvedCredential | null;
+  try {
+    credential = await resolveCredential(input.user, input.pluginId, input.credentialId);
+  } catch (error) {
+    if (isStoredDataUnavailable(error)) {
+      throw Object.assign(new ProxyError('The connected accounts could not be read right now', 503, 'store_unavailable'), { retryAfter: 10 });
+    }
+    throw error;
+  }
   if (!credential) throw new ProxyError('No such credential', 404, 'no_credential');
 
   let target: URL;
@@ -381,6 +393,12 @@ export async function proxyRequest(input: ProxyInput): Promise<Response> {
       audit(error.status, 0, error.code);
       throw error;
     }
+    // A captured secret that couldn't be stored: the vault doesn't read,
+    // or the shared store is down. Nothing was written.
+    if (isStoredDataUnavailable(error)) {
+      audit(503, 0, 'store unavailable');
+      throw Object.assign(new ProxyError('The connected accounts could not be read or saved right now', 503, 'store_unavailable'), { retryAfter: 10 });
+    }
     if (error instanceof DisallowedUrlError) {
       audit(403, 0, 'blocked address');
       throw new ProxyError('This server address is not allowed', 403, 'blocked_address');

lib/plugin-secrets/vault.ts

@@ -1,12 +1,14 @@
 // Per-user store of the credentials users connected for plugins. One
-// encrypted file per mail account; the secrets never leave the server again
-// except as an auth header on a request the proxy forwards to the
+// encrypted document per mail account; the secrets never leave the server
+// again except as an auth header on a request the proxy forwards to the
 // credential's own origin.
 
 import { createHash } from 'node:crypto';
-import { readFile, writeFile, mkdir, rename, unlink } from 'node:fs/promises';
 import path from 'node:path';
 import { decryptPayload, encryptPayload } from '@/lib/auth/crypto';
+import { getStore, type CollectionOptions } from '@/lib/shared-store';
+import { isStoreUnavailable, isStoredDataUnreadable, StoredDataUnreadableError } from '@/lib/stored-data-errors';
+import { moveUnreadableAside } from '@/lib/stored-data-quarantine';
 import type { CredentialInject } from './credentials';
 
 export interface VaultUser {
@@ -49,51 +51,89 @@ export function userKey(user: VaultUser): string {
   return createHash('sha256').update(`${user.username}:${user.serverUrl}`).digest('hex');
 }
 
-function vaultPath(user: VaultUser): string {
-  const dir = path.resolve(vaultDir());
-  const file = path.resolve(dir, `${userKey(user)}.enc`);
-  if (!file.startsWith(dir + path.sep)) throw new Error('Invalid vault path');
-  return file;
+const VAULT_FILE = /^[0-9a-f]{64}\.enc$/;
+
+/**
+ * One document per account, keyed by userKey (#1066). Local mode: today's
+ * `<vault dir>/<key>.enc`, base64 `encryptPayload` text, mode 0600.
+ */
+export const VAULT_COLLECTION: CollectionOptions = {
+  local: {
+    dir: () => vaultDir(),
+    file: (id) => `${id}.enc`,
+    idOf: (relative) => (VAULT_FILE.test(relative) ? relative.slice(0, -'.enc'.length) : null),
+    mode: 0o600,
+  },
+};
+
+function collection() {
+  return getStore().collection('plugin-secrets', VAULT_COLLECTION);
 }
 
-async function load(user: VaultUser): Promise<VaultData> {
-  let raw: string;
-  try {
-    raw = await readFile(vaultPath(user), 'utf8');
-  } catch (error) {
-    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return {};
-    throw error;
-  }
-  const payload = decryptPayload(raw, 'plugin-secrets');
+/**
+ * The user's credentials from a stored document; {} for none. One that
+ * doesn't decrypt (another session secret) throws StoredDataUnreadableError,
+ * so no write lands on top of it before it is moved aside (see modify).
+ */
+function decode(user: VaultUser, raw: Buffer | null): VaultData {
+  if (!raw) return {};
+  const payload = decryptPayload(raw.toString('utf8'), 'plugin-secrets');
+  if (!payload) throw new StoredDataUnreadableError('plugin credentials');
   // Bound to this user: a file copied over from another account's name
   // decrypts, but carries the other account's key and is ignored.
-  if (!payload || payload.u !== userKey(user)) return {};
+  if (payload.u !== userKey(user)) return {};
   const data = payload.d;
   return data && typeof data === 'object' ? (data as VaultData) : {};
 }
 
-async function save(user: VaultUser, data: VaultData): Promise<void> {
-  const file = vaultPath(user);
-  if (Object.keys(data).length === 0) {
-    try { await unlink(file); } catch { /* nothing stored */ }
-    return;
+async function load(user: VaultUser): Promise<VaultData> {
+  let stored;
+  try {
+    stored = await collection().get(userKey(user));
+  } catch (error) {
+    if (isStoreUnavailable(error)) throw error;
+    throw new StoredDataUnreadableError('plugin credentials', { cause: error });
+  }
+  try {
+    return decode(user, stored?.data ?? null);
+  } catch (error) {
+    // Reads as no credentials; the next write moves it aside.
+    if (isStoredDataUnreadable(error)) return {};
+    throw error;
   }
-  await mkdir(path.dirname(file), { recursive: true });
-  const tmp = `${file}.${process.pid}.tmp`;
-  await writeFile(tmp, encryptPayload({ u: userKey(user), d: data }, 'plugin-secrets'), { encoding: 'utf8', mode: 0o600 });
-  await rename(tmp, file);
 }
 
-// Writes for one user run one after another, so two saves can't drop each
-// other's changes.
-const queues = new Map<string, Promise<unknown>>();
-function serialize<T>(user: VaultUser, task: () => Promise<T>): Promise<T> {
-  const key = userKey(user);
-  const run = (queues.get(key) ?? Promise.resolve()).then(task, task);
-  const tail = run.catch(() => {});
-  queues.set(key, tail);
-  void tail.then(() => { if (queues.get(key) === tail) queues.delete(key); });
-  return run;
+/**
+ * Change the user's credentials in one step against what is stored now, so
+ * two writes at once (two tabs, two replicas) both land. `change` edits the
+ * data it gets and says whether it changed anything; it may run more than
+ * once. An empty vault is deleted.
+ *
+ * A vault that doesn't decrypt is moved aside first (kept next to the
+ * original as `.unreadable-<time>`, never overwritten), and the write goes
+ * on as if nothing was stored.
+ */
+async function modify<T>(user: VaultUser, change: (data: VaultData) => { result: T; changed: boolean }): Promise<T> {
+  const docs = collection();
+  const id = userKey(user);
+  for (let attempt = 0; ; attempt++) {
+    try {
+      let result!: T;
+      await docs.update(id, (current) => {
+        const data = decode(user, current);
+        const out = change(data);
+        result = out.result;
+        if (!out.changed) return undefined;
+        if (Object.keys(data).length === 0) return current ? null : undefined;
+        return encryptPayload({ u: userKey(user), d: data }, 'plugin-secrets');
+      });
+      return result;
+    } catch (error) {
+      if (!isStoredDataUnreadable(error) || attempt >= 2) throw error;
+      const current = await docs.get(id);
+      if (current) await moveUnreadableAside(docs, id, current, path.join(vaultDir(), `${id}.enc`), 'plugin credentials');
+    }
+  }
 }
 
 export async function getUserCredential(user: VaultUser, pluginId: string, credentialId: string): Promise<StoredCredential | null> {
@@ -134,8 +174,7 @@ export function putUserCredential(
   credentialId: string,
   input: { origin: string; inject: CredentialInject; label: string; username?: string; secret?: string },
 ): Promise<void> {
-  return serialize(user, async () => {
-    const data = await load(user);
+  return modify(user, (data) => {
     const existing = data[pluginId]?.[credentialId];
     const count = Object.values(data).reduce((n, c) => n + Object.keys(c).length, 0);
     if (!existing && count >= MAX_USER_CREDENTIALS) throw new VaultFullError();
@@ -153,7 +192,7 @@ export function putUserCredential(
         updatedAt: now,
       },
     };
-    await save(user, data);
+    return { result: undefined, changed: true };
   });
 }
 
@@ -165,32 +204,29 @@ export function setUserCredentialSecret(
   secret: string,
   username?: string,
 ): Promise<boolean> {
-  return serialize(user, async () => {
-    const data = await load(user);
+  return modify(user, (data) => {
     const existing = data[pluginId]?.[credentialId];
-    if (!existing) return false;
+    if (!existing) return { result: false, changed: false };
     data[pluginId][credentialId] = {
       ...existing,
       secret,
       ...(username ? { username } : {}),
       updatedAt: new Date().toISOString(),
     };
-    await save(user, data);
-    return true;
+    return { result: true, changed: true };
   });
 }
 
 /** Remove one credential, or all of a plugin's when credentialId is omitted. */
 export function deleteUserCredentials(user: VaultUser, pluginId: string, credentialId?: string): Promise<void> {
-  return serialize(user, async () => {
-    const data = await load(user);
-    if (!data[pluginId]) return;
+  return modify(user, (data) => {
+    if (!data[pluginId]) return { result: undefined, changed: false };
     if (credentialId) {
       delete data[pluginId][credentialId];
       if (Object.keys(data[pluginId]).length === 0) delete data[pluginId];
     } else {
       delete data[pluginId];
     }
-    await save(user, data);
+    return { result: undefined, changed: true };
   });
 }

lib/scheduler-leader.ts

@@ -0,0 +1,186 @@
+import { logger } from '@/lib/logger';
+import { getStore, isSharedMode, replicaId } from '@/lib/shared-store';
+import { onSharedChange, storeLostItsData } from '@/lib/shared-store/changes';
+
+/**
+ * One replica runs the background jobs (telemetry heartbeat, update check)
+ * for the whole deployment (#1066). It holds the `scheduler-leader` lease,
+ * renewing it every 10 s; every other replica tries to take it every 10 s,
+ * so when the leader dies or loses the store another one takes over within
+ * the 30 s lease plus one try. A leader that can't renew stops its jobs at
+ * once. The jobs keep their schedule in the shared state, so a new leader
+ * continues it instead of starting over.
+ *
+ * Local mode: the one process is always the leader.
+ *
+ * Everything lives on globalThis: instrumentation starts it, and the
+ * `telemetry` change listener may be run from a route's module copy.
+ */
+
+export const LEADER_LEASE = 'scheduler-leader';
+export const LEADER_TTL_MS = 30_000;
+export const LEADER_TRY_MS = 10_000;
+
+export interface ScheduledJob {
+  name: string;
+  /** Start the job's timer from its stored schedule. Must be idempotent. */
+  start(): Promise<void>;
+  stop(): Promise<void>;
+}
+
+interface LeaderGlobals {
+  jobs: ScheduledJob[];
+  /** Jobs whose start went through while leading; the others are retried on each renewal. */
+  started: Set<string>;
+  leading: boolean;
+  interval?: NodeJS.Timeout;
+  trying?: Promise<boolean>;
+  unlisten?: () => void;
+}
+
+const KEY = Symbol.for('bulwark.schedulerLeader');
+
+function globals(): LeaderGlobals {
+  const g = globalThis as typeof globalThis & { [KEY]?: LeaderGlobals };
+  return (g[KEY] ??= { jobs: [], started: new Set(), leading: false });
+}
+
+function message(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
+
+/** True while this process runs the background jobs. */
+export function isSchedulerLeader(): boolean {
+  return globals().leading;
+}
+
+async function startJobs(g: LeaderGlobals): Promise<void> {
+  for (const job of g.jobs) {
+    if (g.started.has(job.name)) continue;
+    try {
+      await job.start();
+      // Stepped down while it started: don't leave its timer running.
+      if (g.leading) g.started.add(job.name);
+      else await job.stop();
+    } catch (error) {
+      logger.warn('scheduler: could not start a job; retrying', { job: job.name, error: message(error) });
+    }
+  }
+}
+
+async function stopJobs(g: LeaderGlobals): Promise<void> {
+  g.started.clear();
+  for (const job of g.jobs) {
+    try {
+      await job.stop();
+    } catch (error) {
+      logger.warn('scheduler: could not stop a job', { job: job.name, error: message(error) });
+    }
+  }
+}
+
+async function becomeLeader(g: LeaderGlobals): Promise<void> {
+  g.leading = true;
+  logger.info('scheduler: leading', { replica: replicaId() });
+  await startJobs(g);
+}
+
+async function stepDown(g: LeaderGlobals, reason: string): Promise<void> {
+  g.leading = false;
+  logger.info('scheduler: stopped leading', { replica: replicaId(), reason });
+  await stopJobs(g);
+}
+
+/**
+ * One try: take or renew the lease, start the jobs on taking it, stop them
+ * on losing it (or on not reaching the store). Returns whether this replica
+ * leads afterwards.
+ */
+export async function tryLead(): Promise<boolean> {
+  const g = globals();
+  if (g.trying) return g.trying;
+  g.trying = (async () => {
+    let holds: boolean;
+    let reason = 'another replica holds the lease';
+    if (storeLostItsData()) {
+      // The store was emptied under us: a lease taken there now proves
+      // nothing, and the jobs' stored schedule is gone. Wait for it back.
+      holds = false;
+      reason = 'the shared store lost its data';
+    } else {
+      try {
+        holds = await getStore().leases.acquire(LEADER_LEASE, replicaId(), LEADER_TTL_MS);
+      } catch (error) {
+        holds = false;
+        reason = `could not reach the shared store: ${message(error)}`;
+      }
+    }
+    if (holds && !g.leading) await becomeLeader(g);
+    else if (holds) await startJobs(g);
+    else if (g.leading) await stepDown(g, reason);
+    return g.leading;
+  })().finally(() => {
+    g.trying = undefined;
+  });
+  return g.trying;
+}
+
+/** Restart one job on the leader (its settings changed), keeping its stored schedule. */
+async function restartJob(name: string): Promise<void> {
+  const g = globals();
+  if (!g.leading) return;
+  const job = g.jobs.find((j) => j.name === name);
+  if (!job) return;
+  g.started.delete(name);
+  await job.stop();
+  await job.start();
+  if (g.leading) g.started.add(name);
+  else await job.stop();
+}
+
+/**
+ * Start leading or trying to (instrumentation, once per process). In shared
+ * mode every replica calls it; only the lease holder runs `jobs`.
+ */
+export async function startSchedulerLeader(jobs: ScheduledJob[]): Promise<void> {
+  const g = globals();
+  g.jobs = jobs;
+  // Consent or the collector changed on some replica: the leader picks it up.
+  g.unlisten ??= onSharedChange('telemetry', () => restartJob('telemetry'));
+  if (!isSharedMode()) {
+    if (!g.leading) await becomeLeader(g);
+    return;
+  }
+  if (!g.interval) {
+    g.interval = setInterval(() => void tryLead(), LEADER_TRY_MS);
+    g.interval.unref();
+  }
+  await tryLead();
+}
+
+/** Stop the jobs and give the lease up (shutdown, tests). */
+export async function stopSchedulerLeader(): Promise<void> {
+  const g = globals();
+  if (g.interval) clearInterval(g.interval);
+  g.interval = undefined;
+  g.unlisten?.();
+  g.unlisten = undefined;
+  if (g.leading) {
+    if (isSharedMode()) await getStore().leases.release(LEADER_LEASE, replicaId()).catch(() => {});
+    await stepDown(g, 'stopped');
+  }
+}
+
+/** Resolves when the current try (and the job starts it set off) is done (tests). */
+export function schedulerLeaderSettled(): Promise<unknown> {
+  return globals().trying ?? Promise.resolve();
+}
+
+/** Forget everything (tests). */
+export function resetSchedulerLeaderForTests(): void {
+  const g = globals();
+  if (g.interval) clearInterval(g.interval);
+  g.unlisten?.();
+  const holder = globalThis as typeof globalThis & { [KEY]?: LeaderGlobals };
+  delete holder[KEY];
+}

lib/settings-sync.ts

@@ -1,14 +1,17 @@
 import { createHash, createCipheriv, createDecipheriv, randomBytes } from 'node:crypto';
-import { readFile, writeFile, unlink, mkdir, rename } from 'node:fs/promises';
-import { existsSync } from 'node:fs';
 import path from 'node:path';
 import { logger } from '@/lib/logger';
 import { getSessionSecret } from '@/lib/auth/session-secret';
+import { getStore, type CollectionOptions } from '@/lib/shared-store';
+import { isStoreUnavailable, isStoredDataUnreadable, StoredDataUnreadableError } from '@/lib/stored-data-errors';
+import { moveUnreadableAside } from '@/lib/stored-data-quarantine';
 
 const ALGORITHM = 'aes-256-gcm';
 const IV_LENGTH = 12;
 const TAG_LENGTH = 16;
 
+type Settings = Record<string, unknown>;
+
 function getKey(): Buffer {
   const secret = getSessionSecret();
   if (!secret) throw new Error('SESSION_SECRET not configured');
@@ -19,47 +22,46 @@ export function getSettingsDir(): string {
   return process.env.SETTINGS_DATA_DIR || path.join(process.cwd(), 'data', 'settings');
 }
 
-function getSettingsPath(username: string, serverUrl: string): string {
-  const hash = createHash('sha256').update(`${username}:${serverUrl}`).digest('hex');
-  const filePath = path.join(getSettingsDir(), `${hash}.enc`);
-  // Defense in depth: ensure path stays within the settings directory
-  const resolvedDir = path.resolve(getSettingsDir());
-  const resolvedPath = path.resolve(filePath);
-  if (!resolvedPath.startsWith(resolvedDir + path.sep) && resolvedPath !== resolvedDir) {
-    throw new Error('Invalid settings path');
-  }
-  return resolvedPath;
+/** The document id of an account's settings: sha256 of `user:server`, hex. */
+export function settingsId(username: string, serverUrl: string): string {
+  return createHash('sha256').update(`${username}:${serverUrl}`).digest('hex');
 }
 
-export async function saveUserSettings(username: string, serverUrl: string, settings: Record<string, unknown>): Promise<void> {
-  const dir = getSettingsDir();
-  if (!existsSync(dir)) {
-    await mkdir(dir, { recursive: true });
-  }
+const SETTINGS_FILE = /^[0-9a-f]{64}\.enc$/;
 
-  const key = getKey();
+/**
+ * One encrypted document per account (#1066). Local mode: today's
+ * `<settings dir>/<id>.enc`, `iv‖tag‖ciphertext`. The account archives live
+ * in a subfolder of the same dir and are not part of this collection.
+ */
+export const SETTINGS_COLLECTION: CollectionOptions = {
+  local: {
+    dir: () => getSettingsDir(),
+    file: (id) => `${id}.enc`,
+    idOf: (relative) => (SETTINGS_FILE.test(relative) ? relative.slice(0, -'.enc'.length) : null),
+  },
+};
+
+function collection() {
+  return getStore().collection('settings', SETTINGS_COLLECTION);
+}
+
+function encryptSettings(settings: Settings): Buffer {
   const iv = randomBytes(IV_LENGTH);
-  const cipher = createCipheriv(ALGORITHM, key, iv);
-
-  const payload = JSON.stringify(settings);
-  const encrypted = Buffer.concat([cipher.update(payload, 'utf8'), cipher.final()]);
-  const tag = cipher.getAuthTag();
-
-  const data = Buffer.concat([iv, tag, encrypted]);
-  const targetPath = getSettingsPath(username, serverUrl);
-  const tmpPath = targetPath + '.tmp';
-  await writeFile(tmpPath, data);
-  await rename(tmpPath, targetPath);
+  const cipher = createCipheriv(ALGORITHM, getKey(), iv);
+  const encrypted = Buffer.concat([cipher.update(JSON.stringify(settings), 'utf8'), cipher.final()]);
+  return Buffer.concat([iv, cipher.getAuthTag(), encrypted]);
 }
 
-export async function loadUserSettings(username: string, serverUrl: string): Promise<Record<string, unknown> | null> {
-  const filePath = getSettingsPath(username, serverUrl);
-
+/**
+ * Settings from stored bytes. Too short to hold anything reads as nothing
+ * stored, as it always has; bytes that don't decrypt throw
+ * StoredDataUnreadableError.
+ */
+function decryptSettings(data: Buffer): Settings | null {
+  if (data.length < IV_LENGTH + TAG_LENGTH) return null;
+  const key = getKey();
   try {
-    const data = await readFile(filePath);
-    if (data.length < IV_LENGTH + TAG_LENGTH) return null;
-
-    const key = getKey();
     const iv = data.subarray(0, IV_LENGTH);
     const tag = data.subarray(IV_LENGTH, IV_LENGTH + TAG_LENGTH);
     const encrypted = data.subarray(IV_LENGTH + TAG_LENGTH);
@@ -70,18 +72,74 @@ export async function loadUserSettings(username: string, serverUrl: string): Pro
     const decrypted = Buffer.concat([decipher.update(encrypted), decipher.final()]);
     return JSON.parse(decrypted.toString('utf8'));
   } catch (error) {
-    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null;
-    logger.warn('Failed to load user settings', { error: error instanceof Error ? error.message : 'Unknown error' });
-    return null;
+    logger.warn('Failed to decrypt user settings', { error: error instanceof Error ? error.message : 'Unknown error' });
+    throw new StoredDataUnreadableError('settings', { cause: error });
+  }
+}
+
+export async function saveUserSettings(username: string, serverUrl: string, settings: Settings): Promise<void> {
+  await collection().put(settingsId(username, serverUrl), encryptSettings(settings));
+}
+
+/**
+ * The user's stored settings, or null when none are stored. Settings that
+ * don't decrypt (written with another session secret) read as none too; the
+ * next save moves them aside before it writes (see updateUserSettings). A
+ * file that can't be read throws StoredDataUnreadableError, a shared store
+ * that can't be reached StoreUnavailableError.
+ */
+export async function loadUserSettings(username: string, serverUrl: string): Promise<Settings | null> {
+  let stored;
+  try {
+    stored = await collection().get(settingsId(username, serverUrl));
+  } catch (error) {
+    if (isStoreUnavailable(error)) throw error;
+    logger.warn('Failed to read user settings', { error: error instanceof Error ? error.message : 'Unknown error' });
+    throw new StoredDataUnreadableError('settings', { cause: error });
+  }
+  if (!stored) return null;
+  try {
+    return decryptSettings(stored.data);
+  } catch (error) {
+    if (isStoredDataUnreadable(error)) return null;
+    throw error;
+  }
+}
+
+/**
+ * Read, change and write the user's settings in one step, so a save that
+ * lands at the same time (another tab, another replica) isn't lost: `change`
+ * gets what is stored now (null for nothing) and returns what to store. It
+ * may run more than once.
+ *
+ * Stored settings that don't decrypt are moved aside first (kept next to
+ * the original as `.unreadable-<time>`, never overwritten), and the save
+ * goes on as if nothing was stored.
+ */
+export async function updateUserSettings(
+  username: string,
+  serverUrl: string,
+  change: (stored: Settings | null) => Settings,
+): Promise<void> {
+  const docs = collection();
+  const id = settingsId(username, serverUrl);
+  for (let attempt = 0; ; attempt++) {
+    try {
+      await docs.update(id, (current) => encryptSettings(change(current ? decryptSettings(current) : null)));
+      return;
+    } catch (error) {
+      if (!isStoredDataUnreadable(error) || attempt >= 2) throw error;
+      const current = await docs.get(id);
+      if (current) await moveUnreadableAside(docs, id, current, path.join(getSettingsDir(), `${id}.enc`), 'settings');
+    }
   }
 }
 
 export async function deleteUserSettings(username: string, serverUrl: string): Promise<void> {
   try {
-    await unlink(getSettingsPath(username, serverUrl));
+    await collection().delete(settingsId(username, serverUrl));
   } catch (error) {
-    if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
-      logger.warn('Failed to delete user settings', { error: error instanceof Error ? error.message : 'Unknown error' });
-    }
+    if (isStoreUnavailable(error)) throw error;
+    logger.warn('Failed to delete user settings', { error: error instanceof Error ? error.message : 'Unknown error' });
   }
 }

lib/setup/session.ts

@@ -1,6 +1,7 @@
 import { cookies } from 'next/headers';
-import type { NextRequest } from 'next/server';
+import { NextResponse, type NextRequest } from 'next/server';
 import { verifySetupToken } from './token';
+import { isStoreUnavailableError } from '@/lib/shared-store';
 import { secureCookieFlag } from '@/lib/security/request-protocol';
 
 export const SETUP_COOKIE = 'bulwark_setup_token';
@@ -23,6 +24,28 @@ export async function authenticateWizardRequest(): Promise<boolean> {
   return verifySetupToken(token);
 }
 
+/** 503 for a setup request while the shared store, which holds the token, can't be reached (#1066). */
+export function setupStoreUnavailableResponse(): NextResponse {
+  return NextResponse.json(
+    { error: 'Setup is temporarily unavailable: the shared store can\'t be reached. Try again in a moment.' },
+    { status: 503, headers: { 'Retry-After': '10' } },
+  );
+}
+
+/**
+ * The answer for a wizard request without a valid wizard session (401), or
+ * while the token can't be checked (503); null when it may go on.
+ */
+export async function wizardSessionRefusal(): Promise<NextResponse | null> {
+  try {
+    if (await authenticateWizardRequest()) return null;
+  } catch (error) {
+    if (isStoreUnavailableError(error)) return setupStoreUnavailableResponse();
+    throw error;
+  }
+  return NextResponse.json({ error: 'Wizard session required' }, { status: 401 });
+}
+
 export function buildSessionCookieAttributes(request?: NextRequest) {
   // Match Secure to the actual request protocol. Browsers drop Secure cookies
   // on plain HTTP, so unconditionally setting Secure in production breaks

lib/shared-store/__tests__/boot.test.ts

@@ -87,6 +87,28 @@ describe('change notices', () => {
     expect(seen).toEqual(['local', 'local', 'remote']);
   });
 
+  it('sends a change notice again when the store was unreachable right after the write', async () => {
+    const store = fakeSharedStore();
+    setStoreForTests(store);
+    await store.epochs.bump(INIT_EPOCH);
+    await markEpochsSeen();
+    const bump = store.epochs.bump.bind(store.epochs);
+    let fail = true;
+    store.epochs.bump = async (area: string) => {
+      if (fail) throw new Error('connection dropped');
+      return bump(area);
+    };
+    await announceChange('config');
+    expect((await store.epochs.all()).config).toBeUndefined();
+    await pollChanges(); // still down
+    expect((await store.epochs.all()).config).toBeUndefined();
+    fail = false;
+    await pollChanges();
+    expect((await store.epochs.all()).config).toBe(1);
+    await pollChanges();
+    expect((await store.epochs.all()).config).toBe(1); // sent once, not every poll
+  });
+
   it('retries a listener that failed on the next poll', async () => {
     const store = fakeSharedStore();
     setStoreForTests(store);

lib/shared-store/__tests__/contract.ts

@@ -307,6 +307,33 @@ export function runStoreContract(target: ContractTarget): void {
         await log.append('one\ntwo');
         expect(await log.readAll()).toEqual(['one two']);
       });
+
+      it('appends a batch in order, with no other writer\'s line inside it', async () => {
+        const [a, b] = await open(2);
+        const batch = Array.from({ length: 200 }, (_, i) => `batch-${i}`);
+        await Promise.all([
+          a.log(`audit-${run}`, logOptions()).appendMany(batch),
+          ...Array.from({ length: 20 }, (_, i) => b.log(`audit-${run}`, logOptions()).append(`single-${i}`)),
+        ]);
+        const lines = await a.log(`audit-${run}`, logOptions()).readAll();
+        expect(lines).toHaveLength(220);
+        const start = lines.indexOf('batch-0');
+        expect(lines.slice(start, start + 200)).toEqual(batch);
+        await a.log(`audit-${run}`, logOptions()).appendMany([]);
+        expect(await b.log(`audit-${run}`, logOptions()).length()).toBe(220);
+      });
+
+      it('clears the log for every replica', async () => {
+        const [a, b] = await open(2);
+        await a.log(`audit-${run}`, logOptions()).appendMany(['one', 'two\nthree']);
+        expect(await b.log(`audit-${run}`, logOptions()).readAll()).toEqual(['one', 'two three']);
+        await b.log(`audit-${run}`, logOptions()).clear();
+        expect(await a.log(`audit-${run}`, logOptions()).readAll()).toEqual([]);
+        expect(await a.log(`audit-${run}`, logOptions()).length()).toBe(0);
+        await b.log(`audit-${run}`, logOptions()).clear();
+        await a.log(`audit-${run}`, logOptions()).append('again');
+        expect(await b.log(`audit-${run}`, logOptions()).readAll()).toEqual(['again']);
+      });
     });
 
     describe('leases', () => {

lib/shared-store/__tests__/import-admin.test.ts

@@ -17,8 +17,12 @@ import { resetLocalStoreForTests } from '../local';
 import { separateFakeStore } from '@/lib/admin/__tests__/shared-store-fake';
 import { adminConfigDocs, adminStateDocs } from '@/lib/admin/admin-store';
 import { AUDIT_LOG_OPTIONS } from '@/lib/admin/audit';
+import { INIT_EPOCH } from '../changes';
 
 let dir: string;
+let store: ReturnType<typeof separateFakeStore>;
+/** Store calls of the shared audit log; `failAfter` makes the n+1st batch fail, as an outage would. */
+const calls = { append: 0, appendMany: 0, failAfter: Infinity };
 const config = (file: string) => path.join(dir, 'config', file);
 const state = (file: string) => path.join(dir, 'state', file);
 const LOCAL: Record<string, string> = {};
@@ -42,7 +46,26 @@ beforeEach(() => {
     LOCAL[file] = content;
   }
   resetLocalStoreForTests();
-  setStoreForTests(separateFakeStore(path.join(dir, 'store')));
+  store = separateFakeStore(path.join(dir, 'store'));
+  const log = store.log.bind(store);
+  calls.append = 0;
+  calls.appendMany = 0;
+  calls.failAfter = Infinity;
+  store.log = (name, options) => {
+    const inner = log(name, options);
+    return {
+      ...inner,
+      append: async (line) => {
+        calls.append++;
+        return inner.append(line);
+      },
+      appendMany: async (lines) => {
+        if (++calls.appendMany > calls.failAfter) throw new Error('store went away (test)');
+        return inner.appendMany(lines);
+      },
+    };
+  };
+  setStoreForTests(store);
 });
 
 afterEach(() => {
@@ -73,6 +96,34 @@ describe('admin import', () => {
     });
   });
 
+  it('copies a long audit log in batches, in order', async () => {
+    const lines = Array.from({ length: 1201 }, (_, i) => JSON.stringify({ ts: String(i), action: 'admin.login', detail: {}, ip: '1' }));
+    writeFileSync(state('audit.log'), lines.map((l) => `${l}\n`).join(''));
+    expect((await importLocalData(steps)).imported.audit).toBe(1201);
+    expect(calls).toMatchObject({ append: 0, appendMany: 3 });
+    expect(await getStore().log('audit', AUDIT_LOG_OPTIONS).readAll()).toEqual(lines);
+  });
+
+  it('redoes an audit import that stopped halfway, without doubling lines', async () => {
+    const lines = Array.from({ length: 1100 }, (_, i) => JSON.stringify({ ts: String(i), action: 'admin.login', detail: {}, ip: '1' }));
+    writeFileSync(state('audit.log'), lines.map((l) => `${l}\n`).join(''));
+    calls.failAfter = 1;
+    await expect(importLocalData(steps)).rejects.toThrow(/went away/);
+    expect(await getStore().log('audit', AUDIT_LOG_OPTIONS).length()).toBe(500);
+
+    calls.failAfter = Infinity;
+    expect((await importLocalData(steps)).imported.audit).toBe(1100);
+    expect(await getStore().log('audit', AUDIT_LOG_OPTIONS).readAll()).toEqual(lines);
+    expect((await importLocalData(steps)).imported.audit).toBe(0);
+  });
+
+  it('leaves the log of a running deployment alone on a forced re-run', async () => {
+    await store.epochs.bump(INIT_EPOCH);
+    await getStore().log('audit', AUDIT_LOG_OPTIONS).append('{"ts":"9","action":"admin.login","detail":{},"ip":"2"}');
+    expect((await importLocalData(steps)).imported.audit).toBe(0);
+    expect(await getStore().log('audit', AUDIT_LOG_OPTIONS).length()).toBe(1);
+  });
+
   it('keeps what the store already holds', async () => {
     await adminConfigDocs().put('config.json', '{"appName":"Already shared"}');
     expect((await importLocalData(steps)).imported['admin-config']).toBe(3);

lib/shared-store/__tests__/import-data.test.ts

@@ -0,0 +1,103 @@
+/**
+ * @vitest-environment node
+ */
+import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { setStoreForTests } from '../index';
+import { resetLocalStoreForTests } from '../local';
+import { separateDiskStore, type SeparateDiskStore } from '@/lib/__tests__/helpers/separate-disk-store';
+import { steps } from '../import/data';
+import { SETTINGS_COLLECTION } from '@/lib/settings-sync';
+import { VAULT_COLLECTION } from '@/lib/plugin-secrets/vault';
+import { ARCHIVE_COLLECTION } from '@/lib/account-archive-storage';
+import { TELEMETRY_COLLECTION } from '@/lib/telemetry/state';
+import { VERSION_CHECK_COLLECTION } from '@/lib/version-check/state';
+
+// An existing single instance's settings, plugin credentials, archives,
+// telemetry and update-check state go into an empty shared store once, as
+// they are; the local files stay untouched.
+
+let data: string;
+let store: SeparateDiskStore;
+
+const hash = (c: string) => c.repeat(64);
+
+function snapshot(dir: string): Record<string, string> {
+  const out: Record<string, string> = {};
+  const walk = (d: string) => {
+    for (const name of readdirSync(d)) {
+      const full = path.join(d, name);
+      if (statSync(full).isDirectory()) walk(full);
+      else out[path.relative(dir, full)] = readFileSync(full).toString('base64');
+    }
+  };
+  walk(dir);
+  return out;
+}
+
+beforeEach(() => {
+  data = mkdtempSync(path.join(tmpdir(), 'bw-import-data-'));
+  vi.stubEnv('SETTINGS_DATA_DIR', path.join(data, 'settings'));
+  vi.stubEnv('PLUGIN_SECRETS_DIR', path.join(data, 'plugin-secrets'));
+  vi.stubEnv('TELEMETRY_DATA_DIR', path.join(data, 'telemetry'));
+  vi.stubEnv('VERSION_CHECK_DATA_DIR', path.join(data, 'version-check'));
+  for (const dir of ['settings/account-archives', 'plugin-secrets', 'telemetry', 'version-check']) {
+    mkdirSync(path.join(data, dir), { recursive: true });
+  }
+  writeFileSync(path.join(data, 'settings', `${hash('a')}.enc`), Buffer.from([1, 2, 3, 4]));
+  writeFileSync(path.join(data, 'settings', `${hash('b')}.enc`), Buffer.from([5, 6]));
+  writeFileSync(path.join(data, 'settings', `${hash('c')}.enc.tmp`), 'half written');
+  writeFileSync(path.join(data, 'settings', 'account-archives', `${hash('d')}.json`), '{"version":1}');
+  writeFileSync(path.join(data, 'plugin-secrets', `${hash('e')}.enc`), 'c2VhbGVk');
+  writeFileSync(path.join(data, 'telemetry', 'state.json'), '{"consent":"on"}');
+  writeFileSync(path.join(data, 'telemetry', '.telemetry-id'), '0b9f1f3e-8b8e-4c4b-9a35-3c1b8a7a1e11');
+  writeFileSync(path.join(data, 'telemetry', 'logins.json'), '{"records":[]}');
+  writeFileSync(path.join(data, 'version-check', 'state.json'), '{"status":null}');
+  store = separateDiskStore();
+  setStoreForTests(store);
+});
+
+afterEach(() => {
+  setStoreForTests(null);
+  resetLocalStoreForTests();
+  vi.unstubAllEnvs();
+  store.cleanup();
+  rmSync(data, { recursive: true, force: true });
+});
+
+async function runAll(): Promise<Record<string, number>> {
+  const counts: Record<string, number> = {};
+  for (const step of steps) counts[step.name] = await step.run();
+  return counts;
+}
+
+describe('importing part data', () => {
+  it('copies every document once and leaves the local files alone', async () => {
+    const before = snapshot(data);
+
+    expect(await runAll()).toEqual({
+      settings: 2, 'plugin-secrets': 1, 'account-archives': 1, telemetry: 3, 'version-check': 1,
+    });
+    expect(await runAll()).toEqual({
+      settings: 0, 'plugin-secrets': 0, 'account-archives': 0, telemetry: 0, 'version-check': 0,
+    });
+    expect(snapshot(data)).toEqual(before);
+
+    const settings = store.collection('settings', SETTINGS_COLLECTION);
+    expect(await settings.list()).toEqual([hash('a'), hash('b')]);
+    expect((await settings.get(hash('a')))?.data).toEqual(Buffer.from([1, 2, 3, 4]));
+    expect((await store.collection('plugin-secrets', VAULT_COLLECTION).get(hash('e')))?.data.toString()).toBe('c2VhbGVk');
+    expect((await store.collection('account-archives', ARCHIVE_COLLECTION).get(hash('d')))?.data.toString()).toBe('{"version":1}');
+    expect((await store.collection('telemetry', TELEMETRY_COLLECTION).list()).sort()).toEqual(['.telemetry-id', 'logins.json', 'state.json']);
+    expect((await store.collection('version-check', VERSION_CHECK_COLLECTION).get('state.json'))?.data.toString()).toBe('{"status":null}');
+  });
+
+  it('keeps what the store already holds', async () => {
+    const settings = store.collection('settings', SETTINGS_COLLECTION);
+    await settings.put(hash('a'), Buffer.from([9]));
+    expect((await runAll()).settings).toBe(1);
+    expect((await settings.get(hash('a')))?.data).toEqual(Buffer.from([9]));
+  });
+});

lib/shared-store/__tests__/store-errors.test.ts

@@ -12,3 +12,25 @@ describe('isStoreUnavailableError', () => {
     expect(second.isStoreUnavailableError(null)).toBe(false);
   });
 });
+
+describe('redis store error mapping', () => {
+  const reply = (message: string) => Object.assign(new Error(message), { name: 'ReplyError' });
+
+  it('treats a full server (OOM under noeviction) and connection errors as unavailable', async () => {
+    const { unavailable } = await import('../redis');
+    const { isStoreUnavailableError } = await import('../types');
+    for (const error of [reply('OOM command not allowed when used memory > \'maxmemory\'.'), reply('LOADING Redis is loading'), new Error('Stream isn\'t writeable')]) {
+      let thrown: unknown;
+      try { unavailable(error); } catch (e) { thrown = e; }
+      expect(isStoreUnavailableError(thrown)).toBe(true);
+    }
+  });
+
+  it('rethrows script and command errors as they are', async () => {
+    const { unavailable } = await import('../redis');
+    const bug = reply('ERR Error running script (call to f_x): attempt to compare nil');
+    let thrown: unknown;
+    try { unavailable(bug); } catch (e) { thrown = e; }
+    expect(thrown).toBe(bug);
+  });
+});

lib/shared-store/changes.ts

@@ -33,13 +33,15 @@ interface ChangeGlobals {
   polling?: Promise<void>;
   lastPollOk: number;
   storeLost: boolean;
+  /** Areas whose epoch bump failed (store unreachable after the write). */
+  pendingBumps: Set<string>;
 }
 
 const KEY = Symbol.for('bulwark.sharedStore.changes');
 
 function globals(): ChangeGlobals {
   const g = globalThis as typeof globalThis & { [KEY]?: ChangeGlobals };
-  return (g[KEY] ??= { listeners: new Map(), seen: new Map(), lastPollOk: 0, storeLost: false });
+  return (g[KEY] ??= { listeners: new Map(), seen: new Map(), lastPollOk: 0, storeLost: false, pendingBumps: new Set() });
 }
 
 /**
@@ -74,8 +76,8 @@ async function runListeners(area: string, origin: ChangeOrigin): Promise<boolean
 /**
  * Tell every replica that `area` changed. Call it after the write went
  * through. In shared mode a failure to bump the epoch is logged, not thrown:
- * the write itself succeeded, and other replicas then see it on their next
- * reload of that area (or after their next restart) instead.
+ * the write itself succeeded. The bump is kept and retried by every poll
+ * until it goes through, so the other replicas still pick the change up.
  */
 export async function announceChange(area: string): Promise<void> {
   const g = globals();
@@ -86,7 +88,8 @@ export async function announceChange(area: string): Promise<void> {
       // otherwise the poller reloads to pick up the other replica's.
       if ((g.seen.get(area) ?? 0) === epoch - 1) g.seen.set(area, epoch);
     } catch (error) {
-      logger.warn('Could not announce a change to the other replicas', {
+      g.pendingBumps.add(area);
+      logger.warn('Could not announce a change to the other replicas; retrying with the next poll', {
         area,
         error: error instanceof Error ? error.message : 'Unknown error',
       });
@@ -95,11 +98,23 @@ export async function announceChange(area: string): Promise<void> {
   await runListeners(area, 'local');
 }
 
+/** Send the bumps that failed earlier. A failure leaves them for the next poll. */
+async function retryPendingBumps(g: ChangeGlobals): Promise<void> {
+  for (const area of [...g.pendingBumps]) {
+    await getStore().epochs.bump(area);
+    g.pendingBumps.delete(area);
+    logger.info('Announced a change to the other replicas after a retry', { area });
+  }
+}
+
 /** One poll: read the epochs, run the listeners of every area that moved. */
 export async function pollChanges(): Promise<void> {
   const g = globals();
   if (g.polling) return g.polling;
   g.polling = (async () => {
+    // Our own retried bump then shows up as a change on this replica too,
+    // which costs one reload and keeps the logic simple.
+    if (g.pendingBumps.size) await retryPendingBumps(g);
     const epochs = await getStore().epochs.all();
     if (!epochs[INIT_EPOCH]) {
       if (!g.storeLost) {

lib/shared-store/import/admin.ts

@@ -8,6 +8,33 @@ import {
   adminStateOptions,
 } from '@/lib/admin/admin-store';
 import { AUDIT_LOG_OPTIONS } from '@/lib/admin/audit';
+import { INIT_EPOCH } from '../changes';
+
+/** Set once every audit line is in the shared log. */
+export const AUDIT_IMPORT_DONE_KEY = 'import:audit-done';
+/** Lines per store call: 50 000 lines are 100 calls, not 50 000. */
+const AUDIT_BATCH = 500;
+
+/**
+ * The current audit.log, as the dashboard shows it; the rotated files stay
+ * on disk. In batches, in order. The marker is written after the last one,
+ * so an import that stopped halfway (the replica died, the store dropped)
+ * is redone from the start on the next boot instead of being left short.
+ * A store that is already initialised but has no marker (a forced re-run)
+ * only gets the lines when its log is empty: its lines may be new entries.
+ */
+async function importAuditLog(): Promise<number> {
+  const store = getStore();
+  if (await store.kv.get(AUDIT_IMPORT_DONE_KEY)) return 0;
+  const shared = store.log('audit', AUDIT_LOG_OPTIONS);
+  const initialised = !!(await store.epochs.all())[INIT_EPOCH];
+  if (initialised && (await shared.length()) > 0) return 0;
+  await shared.clear();
+  const lines = (await localLog('audit', AUDIT_LOG_OPTIONS).readAll()).slice(-AUDIT_LOG_OPTIONS.maxEntries);
+  for (let i = 0; i < lines.length; i += AUDIT_BATCH) await shared.appendMany(lines.slice(i, i + AUDIT_BATCH));
+  await store.kv.set(AUDIT_IMPORT_DONE_KEY, JSON.stringify({ at: new Date().toISOString(), lines: lines.length }));
+  return lines.length;
+}
 
 /**
  * Part `admin` of the import (#1066): the admin config, the admin password
@@ -38,17 +65,5 @@ export const steps: ImportStep[] = [
     name: 'admin-state',
     run: () => copyDocs(ADMIN_STATE_COLLECTION, adminStateOptions, ['admin-state.json']),
   },
-  {
-    // The current audit.log, as the dashboard shows it; the rotated files
-    // stay on disk. Only into an empty shared log, so a second run (or a
-    // replica that joins later) adds nothing twice.
-    name: 'audit',
-    run: async () => {
-      const shared = getStore().log('audit', AUDIT_LOG_OPTIONS);
-      if ((await shared.length()) > 0) return 0;
-      const lines = (await localLog('audit', AUDIT_LOG_OPTIONS).readAll()).slice(-AUDIT_LOG_OPTIONS.maxEntries);
-      for (const line of lines) await shared.append(line);
-      return lines.length;
-    },
-  },
+  { name: 'audit', run: importAuditLog },
 ];

lib/shared-store/import/data.ts

@@ -0,0 +1,42 @@
+import { getStore } from '@/lib/shared-store';
+import { localCollection } from '@/lib/shared-store/local';
+import type { CollectionOptions } from '@/lib/shared-store/types';
+import type { ImportStep } from '@/lib/shared-store/import';
+import { SETTINGS_COLLECTION } from '@/lib/settings-sync';
+import { VAULT_COLLECTION } from '@/lib/plugin-secrets/vault';
+import { ARCHIVE_COLLECTION } from '@/lib/account-archive-storage';
+import { TELEMETRY_COLLECTION } from '@/lib/telemetry/state';
+import { VERSION_CHECK_COLLECTION } from '@/lib/version-check/state';
+
+/**
+ * Part `data` of the shared-store import (#1066): every document of these
+ * collections, read from the local files exactly as local mode reads them,
+ * written to the shared store only where it holds nothing yet. The bytes go
+ * over as they are: settings and plugin credentials stay encrypted with the
+ * session secret, which the boot imports first.
+ */
+function copyCollection(name: string, options: CollectionOptions): ImportStep {
+  return {
+    name,
+    async run() {
+      const local = localCollection(name, options);
+      const shared = getStore().collection(name, options);
+      let written = 0;
+      for (const id of await local.list()) {
+        const doc = await local.get(id);
+        if (!doc) continue;
+        if ((await shared.put(id, doc.data, { ifVersion: null })).ok) written++;
+      }
+      return written;
+    },
+  };
+}
+
+export const steps: ImportStep[] = [
+  copyCollection('settings', SETTINGS_COLLECTION),
+  copyCollection('plugin-secrets', VAULT_COLLECTION),
+  copyCollection('account-archives', ARCHIVE_COLLECTION),
+  // state.json, logins.json and the instance id .telemetry-id
+  copyCollection('telemetry', TELEMETRY_COLLECTION),
+  copyCollection('version-check', VERSION_CHECK_COLLECTION),
+];

lib/shared-store/local.ts

@@ -1,6 +1,6 @@
 import { randomBytes } from 'node:crypto';
 import { existsSync } from 'node:fs';
-import { appendFile, mkdir, readFile, readdir, rename, stat, unlink, writeFile } from 'node:fs/promises';
+import { appendFile, mkdir, open, readFile, readdir, rename, stat, unlink, writeFile } from 'node:fs/promises';
 import path from 'node:path';
 import { assertDocId, assertKey, assertName, toBuffer, versionOf } from './keys';
 import type {
@@ -245,7 +245,20 @@ export function localCollection(name: string, options: CollectionOptions): DocCo
   async function writeCurrent(file: string, data: Buffer): Promise<void> {
     await mkdir(path.dirname(file), { recursive: true });
     const tmp = layout.tmpName ? layout.tmpName(file) : `${file}.${process.pid}.${randomBytes(4).toString('hex')}.tmp`;
-    await writeFile(tmp, data, layout.mode !== undefined ? { mode: layout.mode } : undefined);
+    if (layout.fsync) {
+      const handle = await open(tmp, 'wx', layout.mode ?? 0o666);
+      try {
+        await handle.writeFile(data);
+        await handle.sync();
+      } catch (error) {
+        await handle.close().catch(() => {});
+        await unlink(tmp).catch(() => {});
+        throw error;
+      }
+      await handle.close();
+    } else {
+      await writeFile(tmp, data, layout.mode !== undefined ? { mode: layout.mode } : undefined);
+    }
     try {
       await rename(tmp, file);
     } catch (error) {
@@ -402,6 +415,23 @@ export function localLog(name: string, options: LogOptions): AppendLog {
         await rotateIfNeeded(file);
       });
     },
+    async appendMany(lines) {
+      if (lines.length === 0) return;
+      const file = path.resolve(local.path());
+      await serializeLocal(`log\0${file}`, async () => {
+        await mkdir(path.dirname(file), { recursive: true });
+        await appendFile(file, lines.map((line) => line.replace(/[\r\n]+/g, ' ') + '\n').join(''), 'utf-8');
+        await rotateIfNeeded(file);
+      });
+    },
+    async clear() {
+      const file = path.resolve(local.path());
+      await serializeLocal(`log\0${file}`, async () => {
+        await unlink(file).catch((error) => {
+          if (!isMissing(error)) throw error;
+        });
+      });
+    },
     async readAll() {
       try {
         const content = await readFile(path.resolve(local.path()), 'utf-8');

lib/shared-store/redis.ts

@@ -128,9 +128,11 @@ type ScriptName = keyof typeof SCRIPTS;
 type Scripted = Redis & { [K in ScriptName]: (...args: Array<string | number | Buffer>) => Promise<unknown> };
 
 // Server replies that mean "not available right now", not "bug in our script".
-const UNAVAILABLE_REPLY = /^(NOAUTH|WRONGPASS|LOADING|READONLY|MASTERDOWN|BUSY|CLUSTERDOWN|TRYAGAIN|ERR max number of clients)/;
+// OOM is what a full server with the required noeviction policy answers to writes.
+const UNAVAILABLE_REPLY = /^(NOAUTH|WRONGPASS|LOADING|READONLY|MASTERDOWN|BUSY|CLUSTERDOWN|TRYAGAIN|OOM|NOREPLICAS|ERR max number of clients)/;
 
-function unavailable(error: unknown): never {
+/** Throw `error` as a StoreUnavailableError when it means the store is unavailable; rethrow it as it is otherwise. */
+export function unavailable(error: unknown): never {
   if (isStoreUnavailableError(error)) throw error;
   const name = (error as Error)?.name;
   const message = (error as Error)?.message ?? '';
@@ -233,7 +235,8 @@ export function createRedisStore(options: RedisStoreOptions): SharedStore {
         if (ok === 1) return next;
         await backoff(attempt);
       }
-      throw new Error(`Too many concurrent writes to ${key}`);
+      // Fail closed like an outage: the callers then answer 503 instead of 500.
+      throw new StoreUnavailableError(`Too many concurrent writes to ${key}`);
     },
   };
 
@@ -318,7 +321,7 @@ export function createRedisStore(options: RedisStoreOptions): SharedStore {
           }
           await backoff(attempt);
         }
-        throw new Error(`Too many concurrent writes to ${name}/${id}`);
+        throw new StoreUnavailableError(`Too many concurrent writes to ${name}/${id}`);
       },
       async delete(id, putOpts: PutOptions = {}) {
         assertDocId(id);
@@ -353,6 +356,14 @@ export function createRedisStore(options: RedisStoreOptions): SharedStore {
         const clean = line.replace(/[\r\n]+/g, ' ');
         await call(() => client.multi().rpush(key, clean).ltrim(key, -opts.maxEntries, -1).exec());
       },
+      async appendMany(lines) {
+        if (lines.length === 0) return;
+        const clean = lines.map((line) => line.replace(/[\r\n]+/g, ' '));
+        await call(() => client.multi().rpush(key, ...clean).ltrim(key, -opts.maxEntries, -1).exec());
+      },
+      async clear() {
+        await call(() => client.del(key));
+      },
       async readAll() {
         return call(() => client.lrange(key, 0, -1));
       },

lib/shared-store/types.ts

@@ -130,6 +130,11 @@ export interface LocalLayout {
    * depend on it. Default: `<target>.<pid>.<random>.tmp`.
    */
   tmpName?: (target: string) => string;
+  /**
+   * Flush the temp file to disk before the rename (open with `wx`, write,
+   * fsync, close), for documents that must survive a power cut.
+   */
+  fsync?: boolean;
 }
 
 export interface CollectionOptions {
@@ -205,6 +210,13 @@ export interface LogOptions {
 export interface AppendLog {
   /** One line, without a newline. Lines are never interleaved or torn. */
   append(line: string): Promise<void>;
+  /**
+   * Several lines in one call, in this order; no other writer's line lands
+   * between them. For bulk copies such as the import.
+   */
+  appendMany(lines: string[]): Promise<void>;
+  /** Remove every line readAll returns (local: the current file; rotated files stay). */
+  clear(): Promise<void>;
   /** Every line, oldest first (local: the current file only, as today). */
   readAll(): Promise<string[]>;
   length(): Promise<number>;

lib/stored-data-errors.ts

@@ -0,0 +1,59 @@
+import { NextResponse } from 'next/server';
+import { isStoreUnavailableError } from '@/lib/shared-store/types';
+
+/**
+ * A user's stored document exists but can't be read: it doesn't decrypt
+ * (another session secret), it is cut short, or the read failed. It is not
+ * the same as "nothing stored": a route that treated it as empty would merge
+ * into nothing and overwrite the user's real data on the next save. Routes
+ * answer 503 and write nothing.
+ */
+export class StoredDataUnreadableError extends Error {
+  constructor(what: string, options?: { cause?: unknown }) {
+    super(`Stored ${what} could not be read`, options);
+    this.name = 'StoredDataUnreadableError';
+  }
+}
+
+// By name, not instanceof: Next bundles each entry with its own copy of a
+// module, so the store's error class may not be the one this file imports.
+function named(error: unknown, name: string): boolean {
+  return error instanceof Error && error.name === name;
+}
+
+/** The shared store couldn't be reached (StoreUnavailableError from lib/shared-store). */
+export function isStoreUnavailable(error: unknown): boolean {
+  return isStoreUnavailableError(error);
+}
+
+export function isStoredDataUnreadable(error: unknown): boolean {
+  return named(error, 'StoredDataUnreadableError');
+}
+
+/** True for the errors that mean "try again later, nothing was written". */
+export function isStoredDataUnavailable(error: unknown): boolean {
+  return isStoredDataUnreadable(error) || isStoreUnavailable(error);
+}
+
+/**
+ * The 503 for an unreadable document or an unreachable shared store, or null
+ * for any other error. The client keeps its own copy and retries.
+ */
+export function storedDataUnavailableResponse(
+  error: unknown,
+  headers?: Record<string, string>,
+): NextResponse | null {
+  if (isStoredDataUnreadable(error)) {
+    return NextResponse.json(
+      { error: 'Your stored data could not be read on the server. Nothing was changed.', code: 'stored_data_unreadable' },
+      { status: 503, headers: { 'Retry-After': '10', ...headers } },
+    );
+  }
+  if (isStoreUnavailable(error)) {
+    return NextResponse.json(
+      { error: 'Storage is temporarily unavailable. Nothing was changed.', code: 'store_unavailable' },
+      { status: 503, headers: { 'Retry-After': '10', ...headers } },
+    );
+  }
+  return null;
+}

lib/stored-data-quarantine.ts

@@ -0,0 +1,50 @@
+import { readFile, rename } from 'node:fs/promises';
+import path from 'node:path';
+import { logger } from '@/lib/logger';
+import { isSharedMode } from '@/lib/shared-store';
+import { versionOf } from '@/lib/shared-store/keys';
+import { serializeLocal } from '@/lib/shared-store/local';
+import type { DocCollection, StoredDoc } from '@/lib/shared-store/types';
+
+/**
+ * Put a user's document that no longer decrypts out of the way, so the
+ * user can save again, without losing its bytes (#1066). It most likely was
+ * written with another session secret (rotated, or a replica that had its
+ * own); with that secret back it can be restored by hand.
+ *
+ * - Local mode: renamed to `<file>.unreadable-<time>` next to it.
+ * - Shared mode: copied to `<id>.unreadable-<time>`, then the original is
+ *   deleted, only if it is still the same document.
+ *
+ * Either way a concurrent write that replaced it meanwhile is left alone.
+ */
+export async function moveUnreadableAside(
+  docs: DocCollection,
+  id: string,
+  current: StoredDoc,
+  localFile: string,
+  what: string,
+): Promise<void> {
+  const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+  if (!isSharedMode()) {
+    const file = path.resolve(localFile);
+    const target = `${file}.unreadable-${stamp}`;
+    // On the collection's own queue for this file, so no write lands between
+    // the check and the rename.
+    const moved = await serializeLocal(`doc\0${file}`, async () => {
+      const now = await readFile(file).catch(() => null);
+      if (!now || versionOf(now) !== current.version) return false;
+      await rename(file, target);
+      return true;
+    });
+    if (moved) logger.warn(`Moved aside stored ${what} that could not be decrypted`, { path: target });
+    return;
+  }
+  const aside = `${id}.unreadable-${stamp}`;
+  await docs.put(aside, current.data, { ifVersion: null });
+  if (await docs.delete(id, { ifVersion: current.version })) {
+    logger.warn(`Moved aside stored ${what} that could not be decrypted`, { collection: docs.name, id: aside });
+  } else {
+    await docs.delete(aside).catch(() => {});
+  }
+}

lib/telemetry/index.ts

@@ -1,7 +1,7 @@
 export { startScheduler, stopScheduler, reschedule, sendOnce } from './sender';
 export { buildPayload, markProcessStart } from './payload';
 export {
-  loadState, saveState, getInstanceId, effectiveConsent,
+  loadState, saveState, updateState, getInstanceId, effectiveConsent,
 } from './state';
 export { recordLogin, getLoginCounts } from './login-tracker';
 export {

lib/telemetry/login-tracker.ts

@@ -1,9 +1,7 @@
-import { readFile, writeFile, mkdir, rename } from 'node:fs/promises';
-import { existsSync } from 'node:fs';
-import path from 'node:path';
 import { createHmac } from 'node:crypto';
 import { logger } from '@/lib/logger';
-import { getInstanceId } from './state';
+import { isStoreUnavailableError } from '@/lib/shared-store/types';
+import { getInstanceId, telemetryDocs } from './state';
 
 // We never store usernames or server URLs in the clear. Each login is
 // recorded as HMAC-SHA256(username + '@' + serverUrl, instance_id), so the
@@ -22,39 +20,17 @@ interface LoginsFile {
 const SEVEN_DAYS_MS = 7 * 24 * 60 * 60 * 1000;
 const RETENTION_MS = 90 * 24 * 60 * 60 * 1000;
 
-let cache: LoginsFile | null = null;
+/** `logins.json` in the telemetry collection (#1066): one list for every replica. */
+const LOGINS_DOC = 'logins.json';
 
-function getDir(): string {
-  return process.env.TELEMETRY_DATA_DIR || path.join(process.cwd(), 'data', 'telemetry');
-}
-
-function loginsPath(): string {
-  return path.join(getDir(), 'logins.json');
-}
-
-async function ensureDir(): Promise<void> {
-  const dir = getDir();
-  if (!existsSync(dir)) await mkdir(dir, { recursive: true });
-}
-
-async function loadFile(): Promise<LoginsFile> {
-  if (cache) return cache;
+function parse(data: Buffer | null): LoginsFile {
+  if (!data) return { records: [] };
   try {
-    const raw = await readFile(loginsPath(), 'utf8');
-    const parsed = JSON.parse(raw) as Partial<LoginsFile>;
-    cache = Array.isArray(parsed?.records) ? { records: parsed.records as LoginRecord[] } : { records: [] };
+    const parsed = JSON.parse(data.toString('utf8')) as Partial<LoginsFile>;
+    return Array.isArray(parsed?.records) ? { records: parsed.records as LoginRecord[] } : { records: [] };
   } catch {
-    cache = { records: [] };
+    return { records: [] };
   }
-  return cache;
-}
-
-async function saveFile(file: LoginsFile): Promise<void> {
-  await ensureDir();
-  cache = file;
-  const tmp = loginsPath() + '.tmp';
-  await writeFile(tmp, JSON.stringify(file), 'utf8');
-  await rename(tmp, loginsPath());
 }
 
 function normalizeServer(serverUrl: string): string {
@@ -71,29 +47,31 @@ async function hashIdentity(username: string, serverUrl: string): Promise<string
  * Record a successful login. Best-effort; never throws. Updates the
  * existing record's timestamp if the same identity has logged in before,
  * otherwise appends a new record. Records older than the retention window
- * are pruned on every write.
+ * are pruned on every write. Read and written in one step, so logins on
+ * several replicas at once all count.
  */
 export async function recordLogin(username: string, serverUrl: string): Promise<void> {
   if (!username || !serverUrl) return;
   try {
     const id = await hashIdentity(username, serverUrl);
-    const file = await loadFile();
     const now = new Date().toISOString();
-    const cutoff = Date.now() - RETENTION_MS;
-    const next: LoginRecord[] = [];
-    let updated = false;
-    for (const rec of file.records) {
-      const ts = new Date(rec.lastLoginAt).getTime();
-      if (Number.isNaN(ts) || ts < cutoff) continue;
-      if (rec.id === id) {
-        next.push({ id, lastLoginAt: now });
-        updated = true;
-      } else {
-        next.push(rec);
+    await telemetryDocs().update(LOGINS_DOC, (current) => {
+      const cutoff = Date.now() - RETENTION_MS;
+      const next: LoginRecord[] = [];
+      let updated = false;
+      for (const rec of parse(current).records) {
+        const ts = new Date(rec.lastLoginAt).getTime();
+        if (Number.isNaN(ts) || ts < cutoff) continue;
+        if (rec.id === id) {
+          next.push({ id, lastLoginAt: now });
+          updated = true;
+        } else {
+          next.push(rec);
+        }
       }
-    }
-    if (!updated) next.push({ id, lastLoginAt: now });
-    await saveFile({ records: next });
+      if (!updated) next.push({ id, lastLoginAt: now });
+      return JSON.stringify({ records: next });
+    });
   } catch (err) {
     logger.debug?.('telemetry: recordLogin failed', {
       error: err instanceof Error ? err.message : String(err),
@@ -103,11 +81,13 @@ export async function recordLogin(username: string, serverUrl: string): Promise<
 
 /**
  * Total distinct accounts seen in the 90-day retention window, plus those
- * with a login in the last 7 days.
+ * with a login in the last 7 days. Read from the store each time, so every
+ * replica reports the same numbers. Throws while the shared store is down
+ * rather than reporting zero accounts.
  */
 export async function getLoginCounts(): Promise<{ total: number; active7d: number }> {
   try {
-    const file = await loadFile();
+    const file = parse((await telemetryDocs().get(LOGINS_DOC))?.data ?? null);
     const cutoff = Date.now() - RETENTION_MS;
     const sevenAgo = Date.now() - SEVEN_DAYS_MS;
     let total = 0;
@@ -119,7 +99,8 @@ export async function getLoginCounts(): Promise<{ total: number; active7d: numbe
       if (ts >= sevenAgo) active7d++;
     }
     return { total, active7d };
-  } catch {
+  } catch (err) {
+    if (isStoreUnavailableError(err)) throw err;
     return { total: 0, active7d: 0 };
   }
 }

lib/telemetry/sender.ts

@@ -1,5 +1,9 @@
 import { logger } from '@/lib/logger';
-import { effectiveConsent, endpointEnabled, loadState, saveState } from './state';
+import { isSharedMode } from '@/lib/shared-store';
+import { announceChange } from '@/lib/shared-store/changes';
+import {
+  effectiveConsent, endpointEnabled, loadState, loadStateVersioned, saveStateIfUnchanged, updateState,
+} from './state';
 import { buildPayload } from './payload';
 import { fetchTelemetryTarget, resolveEndpointAllowed } from './endpoint-guard';
 import { DEFAULT_ENDPOINT } from './types';
@@ -7,15 +11,81 @@ import { DEFAULT_ENDPOINT } from './types';
 const DAY_MS = 24 * 60 * 60 * 1000;
 const JITTER_MS = 2 * 60 * 60 * 1000; // ± 2 hours
 const FIRST_DELAY_MS = 60 * 60 * 1000;  // 1 hour after consent
+/** A send this long before a scheduled slot already counts for that slot. */
+const SLOT_SLACK_MS = 60 * 1000;
+/**
+ * With a shared store, a scheduled heartbeat this recent means the day's is
+ * done. A new leader computes its own slot when the old one stalled past its
+ * slot or died before storing the next one, so the slot alone can't tell.
+ */
+const SCHEDULED_GAP_MS = 20 * 60 * 60 * 1000;
 
-let currentTimer: NodeJS.Timeout | null = null;
+// On globalThis: Next loads this module once per bundle (instrumentation,
+// each route), and a reschedule from a route must clear the timer the
+// instrumentation copy started, not a copy of its own.
+const TIMER_KEY = Symbol.for('bulwark.telemetry.timer');
+/**
+ * `generation` goes up on every stop. A start or a tick captures it and
+ * arms a timer only while it is unchanged, so a check or send still in
+ * flight when the scheduler stops (this replica lost the lead) can't set a
+ * new timer afterwards.
+ */
+interface TimerState { current: NodeJS.Timeout | null; generation: number }
+function timer(): TimerState {
+  const g = globalThis as typeof globalThis & { [TIMER_KEY]?: TimerState };
+  return (g[TIMER_KEY] ??= { current: null, generation: 0 });
+}
 
 function jitteredDelay(base: number): number {
   const j = (Math.random() * 2 - 1) * JITTER_MS;
   return Math.max(60_000, base + j);
 }
 
-export async function sendOnce(opts?: { reason?: string }): Promise<{
+type Claim =
+  | { claimed: true; previous: string | null; previousScheduled: string | null | undefined; at: string; shared: boolean }
+  | { claimed: false };
+
+/**
+ * Take the scheduled send for `slot` (its nextScheduledAt): set lastSentAt
+ * with a compare-and-set on the state, unless a send at or after the slot is
+ * already recorded. Of two leaders firing for one slot (a handover after a
+ * long pause), only one gets it.
+ */
+async function claimSlot(slot: string): Promise<Claim> {
+  const slotMs = new Date(slot).getTime();
+  const shared = isSharedMode();
+  for (let attempt = 0; attempt < 5; attempt++) {
+    const { state, version } = await loadStateVersioned();
+    if (version === null) return { claimed: false };
+    const last = state.lastSentAt ? new Date(state.lastSentAt).getTime() : NaN;
+    if (!Number.isNaN(last) && last >= slotMs - SLOT_SLACK_MS) return { claimed: false };
+    if (shared && state.lastScheduledSentAt) {
+      const scheduled = new Date(state.lastScheduledSentAt).getTime();
+      if (!Number.isNaN(scheduled) && Date.now() - scheduled < SCHEDULED_GAP_MS) return { claimed: false };
+    }
+    const at = new Date().toISOString();
+    const next = shared ? { ...state, lastSentAt: at, lastScheduledSentAt: at } : { ...state, lastSentAt: at };
+    if (await saveStateIfUnchanged(next, version)) {
+      return { claimed: true, previous: state.lastSentAt, previousScheduled: state.lastScheduledSentAt, at, shared };
+    }
+  }
+  return { claimed: false };
+}
+
+/** Give a claimed slot back after a failed send, unless another send landed meanwhile. */
+async function releaseSlot(claim: Extract<Claim, { claimed: true }>): Promise<void> {
+  await updateState((state) => {
+    if (state.lastSentAt === claim.at) state.lastSentAt = claim.previous;
+    if (claim.shared && state.lastScheduledSentAt === claim.at) state.lastScheduledSentAt = claim.previousScheduled ?? null;
+  }).catch(() => {});
+}
+
+/**
+ * Send one heartbeat. A scheduled send passes its `slot` and is skipped
+ * when another replica already sent for it; the admin's "send now" passes
+ * none and always sends.
+ */
+export async function sendOnce(opts?: { reason?: string; slot?: string }): Promise<{
   ok: boolean;
   status?: number;
   error?: string;
@@ -37,8 +107,19 @@ export async function sendOnce(opts?: { reason?: string }): Promise<{
     return { ok: false, error: `endpoint blocked: ${guard.reason}` };
   }
 
-  const payload = await buildPayload();
+  let claim: Claim | null = null;
+  if (opts?.slot) {
+    claim = await claimSlot(opts.slot);
+    if (!claim.claimed) {
+      logger.info('telemetry: heartbeat skipped, already sent for this slot', { slot: opts.slot });
+      return { ok: false, error: 'already sent' };
+    }
+  }
+
   try {
+    // Inside the try: a payload that can't be built (store down) gives the
+    // claimed slot back instead of losing the day's heartbeat.
+    const payload = await buildPayload();
     // Never follows redirects: the endpoint guard only vetted the URL we
     // were given, so a 3xx to an internal host would bypass it.
     const res = await fetchTelemetryTarget(endpoint, {
@@ -48,11 +129,11 @@ export async function sendOnce(opts?: { reason?: string }): Promise<{
       signal: AbortSignal.timeout(5000),
     });
     const ok = res.ok;
-    if (ok) {
-      const next = await loadState();
-      next.lastSentAt = new Date().toISOString();
-      await saveState(next);
+    if (ok && !claim) {
+      const at = new Date().toISOString();
+      await updateState((next) => { next.lastSentAt = at; });
     }
+    if (!ok && claim?.claimed) await releaseSlot(claim);
     logger.info('telemetry: heartbeat', {
       ok, status: res.status, reason: opts?.reason ?? 'scheduled',
     });
@@ -60,28 +141,40 @@ export async function sendOnce(opts?: { reason?: string }): Promise<{
   } catch (err) {
     const msg = err instanceof Error ? err.message : String(err);
     logger.warn('telemetry: heartbeat failed', { error: msg });
+    if (claim?.claimed) await releaseSlot(claim);
     return { ok: false, error: msg };
   }
 }
 
-async function scheduleNext(delayMs: number): Promise<void> {
-  if (currentTimer) clearTimeout(currentTimer);
+async function scheduleNext(delayMs: number, generation: number): Promise<void> {
+  const t = timer();
+  if (t.generation !== generation) return; // stopped meanwhile
+  if (t.current) clearTimeout(t.current);
   const at = new Date(Date.now() + delayMs).toISOString();
-  const state = await loadState();
-  state.nextScheduledAt = at;
-  await saveState(state);
-  currentTimer = setTimeout(() => { void tick(); }, delayMs);
+  // The timer first: a state write that fails (store down) must not end
+  // the chain of heartbeats.
+  t.current = setTimeout(() => { void tick(at, generation); }, delayMs);
   // Don't keep the process alive just for this.
-  currentTimer.unref?.();
+  t.current.unref?.();
+  try {
+    await updateState((state) => { state.nextScheduledAt = at; });
+  } catch (err) {
+    logger.warn('telemetry: could not store the schedule', { error: err instanceof Error ? err.message : String(err) });
+  }
 }
 
-async function tick(): Promise<void> {
-  await sendOnce({ reason: 'scheduled' });
-  await scheduleNext(jitteredDelay(DAY_MS));
+async function tick(slot: string, generation: number): Promise<void> {
+  try {
+    await sendOnce({ reason: 'scheduled', slot });
+  } catch (err) {
+    logger.warn('telemetry: heartbeat failed', { error: err instanceof Error ? err.message : String(err) });
+  }
+  await scheduleNext(jitteredDelay(DAY_MS), generation);
 }
 
-// Called from instrumentation. Idempotent.
+// Run by the scheduler leader (lib/scheduler-leader.ts). Idempotent.
 export async function startScheduler(): Promise<void> {
+  const generation = timer().generation;
   const { consent } = await effectiveConsent();
   if (consent !== 'on') {
     logger.info('telemetry: scheduler not started', { consent });
@@ -99,7 +192,7 @@ export async function startScheduler(): Promise<void> {
     const remaining = new Date(state.nextScheduledAt).getTime() - Date.now();
     if (remaining > 0) delay = Math.min(remaining, DAY_MS + JITTER_MS);
   }
-  await scheduleNext(delay);
+  await scheduleNext(delay, generation);
   logger.info('telemetry: scheduler started', {
     nextInMs: delay,
     endpoint: state.endpoint,
@@ -107,12 +200,15 @@ export async function startScheduler(): Promise<void> {
 }
 
 export async function stopScheduler(): Promise<void> {
-  if (currentTimer) clearTimeout(currentTimer);
-  currentTimer = null;
+  const t = timer();
+  t.generation++;
+  if (t.current) clearTimeout(t.current);
+  t.current = null;
 }
 
 // Called when consent flips on/off or the collector URL changes via the UI.
+// The replica that runs the jobs (any replica's change reaches it) restarts
+// the timer from the stored state; in local mode that is this process, at once.
 export async function reschedule(): Promise<void> {
-  await stopScheduler();
-  await startScheduler();
+  await announceChange('telemetry');
 }

lib/telemetry/state.ts

@@ -1,8 +1,8 @@
-import { readFile, writeFile, mkdir, rename } from 'node:fs/promises';
-import { existsSync } from 'node:fs';
 import path from 'node:path';
 import { randomUUID } from 'node:crypto';
 import { logger } from '@/lib/logger';
+import { getStore, type CollectionOptions } from '@/lib/shared-store';
+import { isStoreUnavailable, StoredDataUnreadableError } from '@/lib/stored-data-errors';
 import type { TelemetryStateFile, ConsentState } from './types';
 import { DEFAULT_ENDPOINT } from './types';
 import { isNoThirdPartyTrafficOn } from '@/lib/admin/third-party-mode';
@@ -12,8 +12,26 @@ export function getDir(): string {
     path.join(process.cwd(), 'data', 'telemetry');
 }
 
-function statePath(): string { return path.join(getDir(), 'state.json'); }
-function idPath(): string { return path.join(getDir(), '.telemetry-id'); }
+/**
+ * Telemetry's documents (#1066): `state.json`, `logins.json` and the
+ * instance id `.telemetry-id`. Local mode: today's files in the telemetry
+ * dir. Shared mode: one copy for the whole deployment, so every replica
+ * reports as the same instance with the same counts.
+ */
+export const TELEMETRY_COLLECTION: CollectionOptions = {
+  local: {
+    dir: () => getDir(),
+    idOf: (relative) => (['state.json', 'logins.json', '.telemetry-id'].includes(relative) ? relative : null),
+  },
+};
+
+export const STATE_DOC = 'state.json';
+const ID_DOC = '.telemetry-id';
+const UUID = /^[0-9a-f-]{36}$/i;
+
+export function telemetryDocs() {
+  return getStore().collection('telemetry', TELEMETRY_COLLECTION);
+}
 
 function envOverride(): ConsentState | null {
   const v = (process.env.BULWARK_TELEMETRY ?? '').toLowerCase();
@@ -26,21 +44,29 @@ function envOverride(): ConsentState | null {
   return null;
 }
 
-export async function ensureDir(): Promise<void> {
-  if (!existsSync(getDir())) await mkdir(getDir(), { recursive: true });
-}
-
+/**
+ * The instance id, created once: of several replicas (or module copies)
+ * asking at once, every one gets the id the first one stored.
+ */
 export async function getInstanceId(): Promise<string> {
-  await ensureDir();
-  try {
-    const id = (await readFile(idPath(), 'utf8')).trim();
-    if (/^[0-9a-f-]{36}$/i.test(id)) return id;
-  } catch { /* generate fresh */ }
-  const fresh = randomUUID();
-  const tmp = idPath() + '.tmp';
-  await writeFile(tmp, fresh, 'utf8');
-  await rename(tmp, idPath());
-  return fresh;
+  const docs = telemetryDocs();
+  for (let attempt = 0; attempt < 3; attempt++) {
+    let current;
+    try {
+      current = await docs.get(ID_DOC);
+    } catch (error) {
+      if (isStoreUnavailable(error)) throw error;
+      current = null; // unreadable: make a fresh one, as before
+    }
+    const id = current?.data.toString('utf8').trim();
+    if (id && UUID.test(id)) return id;
+    const fresh = randomUUID();
+    const result = await docs.put(ID_DOC, fresh, { ifVersion: current ? current.version : null });
+    if (result.ok) return fresh;
+    const won = result.current?.data.toString('utf8').trim();
+    if (won && UUID.test(won)) return won;
+  }
+  throw new Error('telemetry: could not store an instance id');
 }
 
 // Default consent is 'off' - telemetry is opt-in. Admins can enable it during
@@ -56,32 +82,95 @@ const DEFAULTS: TelemetryStateFile = {
   nextScheduledAt: null,
 };
 
-export async function loadState(): Promise<TelemetryStateFile> {
-  await ensureDir();
+function serialize(state: TelemetryStateFile): string {
+  return JSON.stringify(state, null, 2);
+}
+
+function parse(data: Buffer): TelemetryStateFile {
+  return { ...DEFAULTS, ...(JSON.parse(data.toString('utf8')) as Partial<TelemetryStateFile>) };
+}
+
+function readFailed(err: unknown): TelemetryStateFile {
+  // Unreadable is not missing: answer with the defaults (so nothing is
+  // sent on a guess) but leave the file alone, or a passing read error
+  // would switch the admin's consent off for good.
+  logger.warn('telemetry: state read failed', {
+    error: err instanceof Error ? err.message : String(err),
+  });
+  return { ...DEFAULTS };
+}
+
+/**
+ * The state with its stored version (null when it couldn't be read), for a
+ * compare-and-set. Throws StoreUnavailableError when the shared store is down.
+ */
+export async function loadStateVersioned(): Promise<{ state: TelemetryStateFile; version: string | null }> {
+  const docs = telemetryDocs();
+  let current;
   try {
-    const raw = await readFile(statePath(), 'utf8');
-    const parsed = JSON.parse(raw) as Partial<TelemetryStateFile>;
-    return { ...DEFAULTS, ...parsed };
+    current = await docs.get(STATE_DOC);
   } catch (err) {
-    if ((err as NodeJS.ErrnoException).code !== 'ENOENT') {
-      logger.warn('telemetry: state read failed', {
-        error: err instanceof Error ? err.message : String(err),
-      });
-    }
-    // First-ever load on a fresh install: persist the default-off state so the
-    // instance id and (lack of) consent are stable across restarts. The admin
-    // can opt in later via the UI or the BULWARK_TELEMETRY env var.
-    const fresh: TelemetryStateFile = { ...DEFAULTS };
-    await saveState(fresh);
-    return fresh;
+    if (isStoreUnavailable(err)) throw err;
+    return { state: readFailed(err), version: null };
   }
+  if (current) {
+    try {
+      return { state: parse(current.data), version: current.version };
+    } catch (err) {
+      return { state: readFailed(err), version: null };
+    }
+  }
+  // First-ever load on a fresh install: persist the default-off state so the
+  // instance id and (lack of) consent are stable across restarts. The admin
+  // can opt in later via the UI or the BULWARK_TELEMETRY env var.
+  const fresh: TelemetryStateFile = { ...DEFAULTS };
+  const written = await docs.put(STATE_DOC, serialize(fresh), { ifVersion: null });
+  if (written.ok) return { state: fresh, version: written.version };
+  // Another replica created it first.
+  return written.current
+    ? { state: parse(written.current.data), version: written.current.version }
+    : { state: fresh, version: null };
+}
+
+export async function loadState(): Promise<TelemetryStateFile> {
+  return (await loadStateVersioned()).state;
 }
 
 export async function saveState(state: TelemetryStateFile): Promise<void> {
-  await ensureDir();
-  const tmp = statePath() + '.tmp';
-  await writeFile(tmp, JSON.stringify(state, null, 2), 'utf8');
-  await rename(tmp, statePath());
+  await telemetryDocs().put(STATE_DOC, serialize(state));
+}
+
+/**
+ * Change some fields of the stored state without undoing a write that
+ * landed meanwhile (another replica scheduling, the admin changing consent).
+ * `change` may run more than once. Returns the state written. A state file
+ * that doesn't parse throws StoredDataUnreadableError and is left as it is.
+ */
+export async function updateState(change: (state: TelemetryStateFile) => void): Promise<TelemetryStateFile> {
+  let next: TelemetryStateFile = { ...DEFAULTS };
+  await telemetryDocs().update(STATE_DOC, (current) => {
+    let state: TelemetryStateFile;
+    try {
+      state = current ? parse(current) : { ...DEFAULTS };
+    } catch (err) {
+      // Unreadable is not missing, here as on reads: writing the defaults
+      // with one change would throw away the admin's consent and endpoint.
+      throw new StoredDataUnreadableError('telemetry state', { cause: err });
+    }
+    change(state);
+    next = state;
+    return serialize(state);
+  });
+  return next;
+}
+
+/**
+ * Write `state` only if the stored state is still at `version` (from
+ * loadStateVersioned). False when something else wrote first.
+ */
+export async function saveStateIfUnchanged(state: TelemetryStateFile, version: string | null): Promise<boolean> {
+  if (version === null) return false;
+  return (await telemetryDocs().put(STATE_DOC, serialize(state), { ifVersion: version })).ok;
 }
 
 // Effective consent: the no-third-party mode wins over everything (#903),

lib/telemetry/types.ts

@@ -41,6 +41,11 @@ export interface TelemetryStateFile {
   endpoint: string;
   consentedAt: string | null;
   lastSentAt: string | null;
+  /**
+   * With a shared store only: when the last scheduled heartbeat went out.
+   * Unlike lastSentAt, the admin's "send now" doesn't set it.
+   */
+  lastScheduledSentAt?: string | null;
   nextScheduledAt: string | null;
 }
 

lib/version-check/index.ts

@@ -1,6 +1,6 @@
 export { startScheduler, stopScheduler, checkOnce } from './sender';
 export {
-  loadState, saveState, effectiveEndpoint, disabledByEnv,
+  loadState, saveState, updateState, effectiveEndpoint, disabledByEnv,
   getCurrentVersion, isStatusStale, freshStatus,
 } from './state';
 export { fetchStatus, parseStatus } from './fetcher';

lib/version-check/sender.ts

@@ -1,7 +1,9 @@
 import { logger } from '@/lib/logger';
 import {
-  disabledByEnv, effectiveEndpoint, getCurrentVersion, isStatusStale, loadState, saveState,
+  disabledByEnv, effectiveEndpoint, freshStatus, getCurrentVersion, isStatusStale, loadState, saveState,
+  updateState, withVersionStatus,
 } from './state';
+import { isSharedMode } from '@/lib/shared-store';
 import { fetchStatus } from './fetcher';
 import type { UpdateStatus } from './types';
 import { isNoThirdPartyTrafficOn } from '@/lib/admin/third-party-mode';
@@ -11,7 +13,21 @@ const JITTER_MS = 5 * 60 * 1000;       // ± 5 min, keeps containers from syncin
 const FIRST_DELAY_MS = 30 * 1000;       // first check ~30s after boot
 const FAILURE_BACKOFF_MS = 15 * 60 * 1000; // after a failed fetch, retry in 15 min
 
-let currentTimer: NodeJS.Timeout | null = null;
+// On globalThis: Next loads this module once per bundle (instrumentation,
+// each route), and a reschedule from a route must clear the timer the
+// instrumentation copy started, not a copy of its own.
+const TIMER_KEY = Symbol.for('bulwark.versionCheck.timer');
+/**
+ * `generation` goes up on every stop. A start or a tick captures it and
+ * arms a timer only while it is unchanged, so a check or send still in
+ * flight when the scheduler stops (this replica lost the lead) can't set a
+ * new timer afterwards.
+ */
+interface TimerState { current: NodeJS.Timeout | null; generation: number }
+function timer(): TimerState {
+  const g = globalThis as typeof globalThis & { [TIMER_KEY]?: TimerState };
+  return (g[TIMER_KEY] ??= { current: null, generation: 0 });
+}
 
 /** What checkOnce reports while NO_THIRD_PARTY_TRAFFIC is on. */
 export const NO_THIRD_PARTY_ERROR = 'off: no third-party traffic';
@@ -39,13 +55,15 @@ export async function checkOnce(opts?: { reason?: string }): Promise<{
   const now = new Date().toISOString();
   const result = await fetchStatus(endpoint, current);
 
-  const next = await loadState();
-  next.lastCheckedAt = now;
-  if (result.ok) {
-    next.lastSuccessAt = now;
-    next.status = result.status;
-  }
-  await saveState(next);
+  const shared = isSharedMode();
+  await updateState((next) => {
+    next.lastCheckedAt = now;
+    if (result.ok) {
+      next.lastSuccessAt = now;
+      next.status = result.status;
+      if (shared) next.statuses = withVersionStatus(next.statuses, result.status);
+    }
+  });
 
   logger.info('version-check: ran', {
     ok: result.ok,
@@ -57,28 +75,38 @@ export async function checkOnce(opts?: { reason?: string }): Promise<{
   return { ok: false, error: result.error };
 }
 
-async function scheduleNext(delayMs: number): Promise<void> {
-  if (currentTimer) clearTimeout(currentTimer);
+async function scheduleNext(delayMs: number, generation: number): Promise<void> {
+  const t = timer();
+  if (t.generation !== generation) return; // stopped meanwhile
+  if (t.current) clearTimeout(t.current);
   const at = new Date(Date.now() + delayMs).toISOString();
-  const state = await loadState();
-  state.nextScheduledAt = at;
-  await saveState(state);
-  currentTimer = setTimeout(() => { void tick(); }, delayMs);
+  // The timer first: a state write that fails (store down) must not end
+  // the chain of checks.
+  t.current = setTimeout(() => { void tick(generation); }, delayMs);
   // Don't keep the process alive just for this.
-  currentTimer.unref?.();
+  t.current.unref?.();
+  try {
+    await updateState((state) => { state.nextScheduledAt = at; });
+  } catch (err) {
+    logger.warn('version-check: could not store the schedule', { error: err instanceof Error ? err.message : String(err) });
+  }
 }
 
-async function tick(): Promise<void> {
-  const result = await checkOnce({ reason: 'scheduled' });
+async function tick(generation: number): Promise<void> {
+  const result = await checkOnce({ reason: 'scheduled' }).catch((err: unknown) => ({
+    ok: false as const, error: err instanceof Error ? err.message : String(err),
+  }));
   // The scheduler keeps running while third-party traffic is off, so turning
   // the switch off at runtime resumes the checks; checkOnce skips meanwhile.
   const skipped = result.error === NO_THIRD_PARTY_ERROR;
   const delay = result.ok || skipped ? jitteredDelay(HOUR_MS) : FAILURE_BACKOFF_MS;
-  await scheduleNext(delay);
+  await scheduleNext(delay, generation);
 }
 
-// Idempotent - safe to call from instrumentation hot-reload in dev.
+// Run by the scheduler leader (lib/scheduler-leader.ts). Idempotent - safe
+// to call from instrumentation hot-reload in dev.
 export async function startScheduler(): Promise<void> {
+  const generation = timer().generation;
   if (disabledByEnv()) {
     logger.info('version-check: scheduler not started (disabled by env)');
     return;
@@ -92,8 +120,12 @@ export async function startScheduler(): Promise<void> {
   // version we no longer run, and its schedule would keep that stale answer
   // (e.g. "update available" for the version we just upgraded from) on screen
   // for up to an hour. Drop it and check promptly instead (#913).
-  const stale = isStatusStale(state.status);
-  if (stale) {
+  // With a shared store, replicas on another version may still be serving
+  // that status: keep it, and only check promptly when there is none for
+  // this version yet.
+  const shared = isSharedMode();
+  const stale = shared ? !freshStatus(state) && state.status !== null : isStatusStale(state.status);
+  if (stale && !shared) {
     logger.info('version-check: discarding status from a previous version', {
       cached: state.status?.current ?? null,
       running: getCurrentVersion(),
@@ -111,11 +143,13 @@ export async function startScheduler(): Promise<void> {
     const remaining = new Date(state.nextScheduledAt).getTime() - Date.now();
     if (remaining > 0) delay = Math.min(remaining, HOUR_MS + JITTER_MS);
   }
-  await scheduleNext(delay);
+  await scheduleNext(delay, generation);
   logger.info('version-check: scheduler started', { nextInMs: delay });
 }
 
 export async function stopScheduler(): Promise<void> {
-  if (currentTimer) clearTimeout(currentTimer);
-  currentTimer = null;
+  const t = timer();
+  t.generation++;
+  if (t.current) clearTimeout(t.current);
+  t.current = null;
 }

lib/version-check/state.ts

@@ -1,7 +1,7 @@
-import { readFile, writeFile, mkdir, rename } from 'node:fs/promises';
-import { existsSync } from 'node:fs';
 import path from 'node:path';
 import { logger } from '@/lib/logger';
+import { getStore, type CollectionOptions } from '@/lib/shared-store';
+import { isStoreUnavailable } from '@/lib/stored-data-errors';
 import type { UpdateStatus, VersionCheckStateFile } from './types';
 import { DEFAULT_VERSION_ENDPOINT } from './types';
 
@@ -10,7 +10,23 @@ function getDir(): string {
     path.join(process.cwd(), 'data', 'version-check');
 }
 
-function statePath(): string { return path.join(getDir(), 'state.json'); }
+/**
+ * `state.json` (#1066). Local mode: today's file in the version-check dir.
+ * Shared mode: one state for the deployment, written by the replica that
+ * runs the scheduled check and read by all.
+ */
+export const VERSION_CHECK_COLLECTION: CollectionOptions = {
+  local: {
+    dir: () => getDir(),
+    idOf: (relative) => (relative === 'state.json' ? relative : null),
+  },
+};
+
+export const STATE_DOC = 'state.json';
+
+function docs() {
+  return getStore().collection('version-check', VERSION_CHECK_COLLECTION);
+}
 
 const DEFAULTS: VersionCheckStateFile = {
   endpoint: DEFAULT_VERSION_ENDPOINT,
@@ -20,31 +36,44 @@ const DEFAULTS: VersionCheckStateFile = {
   status: null,
 };
 
-export async function ensureDir(): Promise<void> {
-  if (!existsSync(getDir())) await mkdir(getDir(), { recursive: true });
+function parse(data: Buffer): VersionCheckStateFile {
+  return { ...DEFAULTS, ...(JSON.parse(data.toString('utf8')) as Partial<VersionCheckStateFile>) };
 }
 
+function serialize(state: VersionCheckStateFile): string {
+  return JSON.stringify(state, null, 2);
+}
+
+/** The stored state, or the defaults. Throws StoreUnavailableError when the shared store is down. */
 export async function loadState(): Promise<VersionCheckStateFile> {
-  await ensureDir();
   try {
-    const raw = await readFile(statePath(), 'utf8');
-    const parsed = JSON.parse(raw) as Partial<VersionCheckStateFile>;
-    return { ...DEFAULTS, ...parsed };
+    const current = await docs().get(STATE_DOC);
+    return current ? parse(current.data) : { ...DEFAULTS };
   } catch (err) {
-    if ((err as NodeJS.ErrnoException).code !== 'ENOENT') {
-      logger.warn('version-check: state read failed', {
-        error: err instanceof Error ? err.message : String(err),
-      });
-    }
+    if (isStoreUnavailable(err)) throw err;
+    logger.warn('version-check: state read failed', {
+      error: err instanceof Error ? err.message : String(err),
+    });
     return { ...DEFAULTS };
   }
 }
 
 export async function saveState(state: VersionCheckStateFile): Promise<void> {
-  await ensureDir();
-  const tmp = statePath() + '.tmp';
-  await writeFile(tmp, JSON.stringify(state, null, 2), 'utf8');
-  await rename(tmp, statePath());
+  await docs().put(STATE_DOC, serialize(state));
+}
+
+/** Change some fields against what is stored now. `change` may run more than once. */
+export async function updateState(change: (state: VersionCheckStateFile) => void): Promise<void> {
+  await docs().update(STATE_DOC, (current) => {
+    let state: VersionCheckStateFile;
+    try {
+      state = current ? parse(current) : { ...DEFAULTS };
+    } catch {
+      state = { ...DEFAULTS };
+    }
+    change(state);
+    return serialize(state);
+  });
 }
 
 export function disabledByEnv(): boolean {
@@ -78,6 +107,29 @@ export function isStatusStale(
   return status.current !== current;
 }
 
+/** How many running versions keep their own status (a rolling update runs two). */
+const KEPT_VERSIONS = 4;
+
+/**
+ * The status for the version this replica runs. With a shared store,
+ * replicas on two versions during a rolling update share one state; each
+ * reads the status checked for its own version from `statuses`, so the
+ * banner doesn't come and go as the other version's checks land.
+ */
 export function freshStatus(state: VersionCheckStateFile): UpdateStatus | null {
-  return isStatusStale(state.status) ? null : state.status;
+  if (!isStatusStale(state.status)) return state.status;
+  const own = state.statuses?.[getCurrentVersion()];
+  return own && !isStatusStale(own) ? own : null;
+}
+
+/** `statuses` with `status` added for its version, keeping the newest few. */
+export function withVersionStatus(
+  statuses: Record<string, UpdateStatus> | undefined,
+  status: UpdateStatus,
+): Record<string, UpdateStatus> {
+  const next = { ...(statuses ?? {}), [status.current]: status };
+  const newest = Object.entries(next)
+    .sort(([, a], [, b]) => b.checkedAt.localeCompare(a.checkedAt))
+    .slice(0, KEPT_VERSIONS);
+  return Object.fromEntries(newest);
 }

lib/version-check/types.ts

@@ -20,6 +20,11 @@ export interface VersionCheckStateFile {
   lastSuccessAt: string | null;
   nextScheduledAt: string | null;
   status: UpdateStatus | null;
+  /**
+   * With a shared store only: the last status per running version, so
+   * replicas on two versions during a rolling update each keep their own.
+   */
+  statuses?: Record<string, UpdateStatus>;
 }
 
 export const DEFAULT_VERSION_ENDPOINT = 'https://version.telemetry.bulwarkmail.org/';