diff --git a/app/api/admin/outgoing-hosts/route.ts b/app/api/admin/outgoing-hosts/route.ts index 41a681e4c..d7eaeccce 100644 --- a/app/api/admin/outgoing-hosts/route.ts +++ b/app/api/admin/outgoing-hosts/route.ts @@ -5,7 +5,8 @@ import { isConfigReadOnly } from '@/lib/admin/paths'; import { getPluginRegistry } from '@/lib/admin/plugin-registry'; import { noThirdPartyTrafficState } from '@/lib/admin/third-party-mode'; import { listReachableHosts } from '@/lib/admin/reachable-hosts'; -import { OUTGOING_HOSTS, pluginDeclaresNetwork } from '@/lib/outgoing-hosts'; +import { pluginDeclaresNetwork } from '@/lib/outgoing-hosts'; +import { OUTGOING_HOSTS } from '@/lib/admin/outgoing-hosts-registry'; import { logger } from '@/lib/logger'; /** diff --git a/components/admin/__tests__/third-party-section.test.tsx b/components/admin/__tests__/third-party-section.test.tsx index ab811c63d..1fe1813eb 100644 --- a/components/admin/__tests__/third-party-section.test.tsx +++ b/components/admin/__tests__/third-party-section.test.tsx @@ -1,7 +1,7 @@ import { act, fireEvent, render, screen, waitFor, within } from '@testing-library/react'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { listReachableHosts } from '@/lib/admin/reachable-hosts'; -import { OUTGOING_HOSTS } from '@/lib/outgoing-hosts'; +import { OUTGOING_HOSTS } from '@/lib/admin/outgoing-hosts-registry'; import type { OrgPolicyDraft } from '@/components/admin/org/use-org-policy-draft'; const apiFetch = vi.hoisted(() => vi.fn()); diff --git a/components/admin/org/sections/third-party-section.tsx b/components/admin/org/sections/third-party-section.tsx index 824b99888..0873524f6 100644 --- a/components/admin/org/sections/third-party-section.tsx +++ b/components/admin/org/sections/third-party-section.tsx @@ -6,7 +6,7 @@ import { SettingsSection, SettingItem, ToggleSwitch } from '@/components/setting import { ConfigNote, RevertButton, SourceBadge } from '@/components/admin/config-items'; import { AdminLink } from '@/components/admin/admin-link'; import { apiFetch } from '@/lib/browser-navigation'; -import type { OutgoingHost } from '@/lib/outgoing-hosts'; +import type { OutgoingHost } from '@/lib/admin/outgoing-hosts-registry'; import type { ReachableHost } from '@/lib/admin/reachable-hosts'; import type { OrgPolicyDraft } from '../use-org-policy-draft'; diff --git a/lib/__tests__/lite-bundle-hosts.test.ts b/lib/__tests__/lite-bundle-hosts.test.ts index 46e2092bd..99fe6ddb8 100644 --- a/lib/__tests__/lite-bundle-hosts.test.ts +++ b/lib/__tests__/lite-bundle-hosts.test.ts @@ -1,70 +1,69 @@ // @vitest-environment node -import { describe, expect, it } from 'vitest'; -import { readdirSync, readFileSync, statSync } from 'node:fs'; +import { afterAll, describe, expect, it } from 'vitest'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; import path from 'node:path'; -import { NON_REQUEST_HOSTS, isPlaceholderHost, registeredLiterals } from '@/lib/outgoing-hosts'; +import { BUILT_IN_HOSTNAMES } from '@/lib/outgoing-hosts'; +import { isPlaceholderHost as sourcePlaceholder } from '@/lib/admin/outgoing-hosts-registry'; +import { LITE_BUILT_IN_HOSTS, isPlaceholderHost, scanBundleHosts } from '../../scripts/lite/scan-hosts.mjs'; -// The built Lite bundle (#903): every http(s) host literal in out/ must be a -// registered outside host, a known non-request, or one of the strings the -// bundled libraries, the demo data and the translations carry. Run after -// `npm run build:lite` with LITE_OUT_DIR pointing at its out/ folder: +// scripts/lite/scan-hosts.mjs runs in the Lite CI job after build:lite (the +// build prunes the test trees, so the scan can't be a vitest file there). +// These tests pin it to the app's host list and show what it catches. With +// LITE_OUT_DIR set it also scans a real build: // // LITE_OUT_DIR=/path/to/out npx vitest run lib/__tests__/lite-bundle-hosts.test.ts -// -// Without LITE_OUT_DIR the scan is skipped (the unit test run has no build). -const OUT = process.env.LITE_OUT_DIR ?? ''; - -/** Strings in the bundle that are never requested, by why they are there. */ -const BUNDLE_NON_REQUEST: Record = { - 'error and docs links of bundled libraries': [ - 'nextjs.org', 'react.dev', 'stuk.github.io', 'prosemirror.net', 'github.com', 'foo.bar', - ], - // Translators localise the example addresses of the login form. - 'example domains in translations': ['eksempel.dk', 'mail.eksempel.dk', 'piemers.lv', 'pasts.piemers.lv', 'mail.ornek.com', 'ornek.com'], - // lib/demo/fixtures: links inside the sample mail of the demo mode. - 'demo mail content': [ - 'zoom.us', 'notion.so', 'www.figma.com', 'vercel.com', 'i.scdn.co', 'cf.bstatic.com', 'open.spotify.com', - 'linear.app', 'sentry.io', 'app.docusign.com', - ], -}; - -function files(dir: string, out: string[] = []): string[] { - for (const name of readdirSync(dir)) { - const full = path.join(dir, name); - if (statSync(full).isDirectory()) files(full, out); - else if (/\.(js|html|css|json|txt|webmanifest)$/.test(name)) out.push(full); +const dirs: string[] = []; +function fakeOut(files: Record): string { + const dir = mkdtempSync(path.join(tmpdir(), 'lite-out-')); + dirs.push(dir); + for (const [name, text] of Object.entries(files)) { + mkdirSync(path.dirname(path.join(dir, name)), { recursive: true }); + writeFileSync(path.join(dir, name), text); } - return out; + return dir; } +afterAll(() => { for (const dir of dirs) rmSync(dir, { recursive: true, force: true }); }); -describe.skipIf(!OUT)('Lite bundle outgoing hosts (LITE_OUT_DIR)', () => { - it('holds no host that is neither registered nor a known non-request', () => { - const config = JSON.parse(readFileSync(path.join(OUT, 'config.json'), 'utf8')) as { jmapServerUrl?: string }; - const allowed = new Set([ - ...registeredLiterals(), - ...NON_REQUEST_HOSTS.map((h) => h.host), - ...Object.values(BUNDLE_NON_REQUEST).flat(), - ]); - if (config.jmapServerUrl) allowed.add(new URL(config.jmapServerUrl).host); - - const unknown = new Map(); - const all = files(OUT); - expect(all.length).toBeGreaterThan(50); - for (const file of all) { - const text = readFileSync(file, 'utf8'); - for (const m of text.matchAll(/https?:\/\/([A-Za-z0-9.*_-]+)/g)) { - const raw = m[1].toLowerCase(); - const host = raw.replace(/\.{2,}$/, ''); - if (allowed.has(raw) || allowed.has(host) || isPlaceholderHost(host)) continue; - if (!unknown.has(host)) unknown.set(host, path.relative(OUT, file)); - } +describe('Lite bundle host scan', () => { + it('knows the same built-in hosts and placeholders as the app', () => { + expect((LITE_BUILT_IN_HOSTS as string[]).filter((h) => !h.endsWith('.')).sort()).toEqual([...BUILT_IN_HOSTNAMES].sort()); + for (const host of ['mail.example.com', 'x', '10.0.0.5', '93.184.216.34', 'acme.test', 'github.com', '*.vimeo.com']) { + expect(isPlaceholderHost(host), host).toBe(sourcePlaceholder(host)); } - expect([...unknown].map(([host, file]) => `${file}: ${host}`)).toEqual([]); }); - it('ships config.json with the switch', () => { - const config = JSON.parse(readFileSync(path.join(OUT, 'config.json'), 'utf8')); - expect(config).toHaveProperty('noThirdPartyTraffic'); + it('passes a bundle with only accounted hosts', () => { + const out = fakeOut({ + 'config.json': JSON.stringify({ jmapServerUrl: 'https://mail.corp.example.net' }), + '_next/static/chunks/a1.js': 'fetch("https://mail.corp.example.net/jmap");x="https://react.dev/errors/";y="https://icons.duckduckgo.com"', + 'en/mail/index.html': '', + }); + expect(scanBundleHosts(out)).toEqual([]); + }); + + it('catches a new host, a ws host, a protocol-relative host, a public IP and a listed host in the wrong file', () => { + const out = fakeOut({ + 'config.json': '{}', + '_next/static/chunks/b2.js': [ + 'fetch("https://beacon.fakecdn.io/x")', + 'new WebSocket("WSS://Live.FakeCDN.io")', + 's.src="//cdn.fakecdn.io/lib.js"', + 'fetch("http://93.184.216.34/")', + ].join(';'), + 'en/mail/index.html': '', + }); + expect(scanBundleHosts(out).sort()).toEqual([ + '_next/static/chunks/b2.js: 93.184.216.34', + '_next/static/chunks/b2.js: beacon.fakecdn.io', + '_next/static/chunks/b2.js: cdn.fakecdn.io', + '_next/static/chunks/b2.js: live.fakecdn.io', + 'en/mail/index.html: icons.duckduckgo.com', + ]); + }); + + it.skipIf(!process.env.LITE_OUT_DIR)('finds nothing unaccounted in the build at LITE_OUT_DIR', () => { + expect(scanBundleHosts(process.env.LITE_OUT_DIR)).toEqual([]); }); }); diff --git a/lib/__tests__/no-third-party-client.test.ts b/lib/__tests__/no-third-party-client.test.ts index 9ea5f4b93..93b5a46d4 100644 --- a/lib/__tests__/no-third-party-client.test.ts +++ b/lib/__tests__/no-third-party-client.test.ts @@ -4,7 +4,8 @@ import { resyncWebPush, revokePushDevice, type PushDevice } from '@/lib/web-push import { doGetPublicKeyFromWKD } from '@/lib/plugin-sandbox/host-api'; import { applyLiteConfig } from '@/lib/lite-config'; import { setNoThirdPartyTraffic, noThirdPartyTraffic, parseNoThirdPartyEnv } from '@/lib/third-party-mode'; -import { isBuiltInHost, isPlaceholderHost, pluginAllowed, pluginDeclaresNetwork } from '@/lib/outgoing-hosts'; +import { isBuiltInHost, pluginAllowed, pluginDeclaresNetwork } from '@/lib/outgoing-hosts'; +import { isPlaceholderHost } from '@/lib/admin/outgoing-hosts-registry'; import type { IJMAPClient } from '@/lib/jmap/client-interface'; // The browser half of the "no third-party traffic" mode (#903). diff --git a/lib/__tests__/outgoing-hosts-mode.test.ts b/lib/__tests__/outgoing-hosts-mode.test.ts index be4a37094..71f52778c 100644 --- a/lib/__tests__/outgoing-hosts-mode.test.ts +++ b/lib/__tests__/outgoing-hosts-mode.test.ts @@ -4,8 +4,8 @@ import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import path from 'node:path'; import { NextRequest } from 'next/server'; +import { OUTGOING_HOSTS } from '@/lib/admin/outgoing-hosts-registry'; import { - OUTGOING_HOSTS, isBuiltInHost, mapsLinksAllowed, sampleAvatarsAllowed, @@ -106,12 +106,12 @@ describe('no third-party traffic mode', () => { expect(Object.keys(probes).sort()).toEqual(OUTGOING_HOSTS.map((h) => h.id).sort()); }); - it.each(OUTGOING_HOSTS.map((h) => [h.id, h] as const))('%s is reachable with the mode off', async (id) => { + it.each(OUTGOING_HOSTS.map((h) => h.id))('%s is reachable with the mode off', async (id) => { await setMode(false); expect(await probes[id]()).toBe(true); }); - it.each(OUTGOING_HOSTS.map((h) => [h.id, h] as const))('%s is off with the mode on', async (id) => { + it.each(OUTGOING_HOSTS.map((h) => h.id))('%s is off with the mode on', async (id) => { await setMode(true); expect(await probes[id]()).toBe(false); }); diff --git a/lib/__tests__/outgoing-hosts.test.ts b/lib/__tests__/outgoing-hosts.test.ts index 15342981e..008a4aa3f 100644 --- a/lib/__tests__/outgoing-hosts.test.ts +++ b/lib/__tests__/outgoing-hosts.test.ts @@ -3,23 +3,34 @@ import { describe, expect, it } from 'vitest'; import { readdirSync, readFileSync, statSync } from 'node:fs'; import path from 'node:path'; import { + BUILT_URL_FILES, NON_REQUEST_HOSTS, NON_REQUEST_PATHS, OUTGOING_HOSTS, isPlaceholderHost, registeredLiterals, -} from '@/lib/outgoing-hosts'; + registryHostnames, +} from '@/lib/admin/outgoing-hosts-registry'; +import { BUILT_IN_HOSTNAMES } from '@/lib/outgoing-hosts'; -// Every http(s) host literal in the shipped source must be a built-in -// outside host in lib/outgoing-hosts.ts (with a switch that turns it off) or -// a known non-request (docs link, namespace, sample text). A new outside -// host fails here until someone decides which it is (#903). +// Every host literal in the shipped source must be a built-in outside host +// in lib/admin/outgoing-hosts-registry.ts (with a switch that turns it off) +// or a known non-request (docs link, namespace, sample text), in the file it +// is listed for. URLs built from a variable must be in a file listed with a +// reason. A new outside host fails here until someone decides which it is +// (#903). const ROOT = path.resolve(__dirname, '..', '..'); -const DIRS = ['app', 'lib', 'stores', 'hooks', 'components', 'contexts']; -const FILES = ['public/sw.js', 'proxy.ts', 'instrumentation.ts', 'instrumentation.node.ts']; +const DIRS = ['app', 'lib', 'stores', 'hooks', 'components', 'contexts', 'i18n']; +const FILES = ['public/sw.js', 'proxy.ts', 'instrumentation.ts', 'instrumentation.node.ts', 'next.config.ts']; const EXTENSIONS = /\.(ts|tsx|js|jsx|mjs|cjs)$/; -const HOST_LITERAL = /https?:\/\/([A-Za-z0-9.*_-]+)/g; + +/** scheme://host, for http(s) and ws(s), any case. */ +const SCHEME_HOST = /\b(?:https?|wss?):\/\/([a-z0-9.*_-]+)/gi; +/** scheme:// followed by ${...} or the end of the string: a URL built from a variable, or the scheme alone. */ +const BUILT_URL = /\b(?:https?|wss?):\/\/(?=\$\{|["'`])/gi; +/** A protocol-relative //host at the start of a string literal. */ +const PROTOCOL_RELATIVE = /["'`]\/\/([a-z0-9-]+(?:\.[a-z0-9-]+)+)/gi; function isTestPath(rel: string): boolean { return /(^|\/)(__tests__|__mocks__|node_modules)\//.test(rel) || /\.(test|spec)\.[a-z]+$/.test(rel); @@ -35,7 +46,7 @@ function walk(rel: string, out: string[]) { } } -export function sourceFiles(): string[] { +function sourceFiles(): string[] { const out: string[] = []; for (const dir of DIRS) walk(dir, out); for (const file of FILES) { @@ -44,67 +55,109 @@ export function sourceFiles(): string[] { return out.filter((file) => !NON_REQUEST_PATHS.some((prefix) => file.startsWith(prefix))); } -/** Host literals in a source text, as written (lower case, port dropped). */ -export function hostLiterals(text: string): string[] { - return [...text.matchAll(HOST_LITERAL)].map((m) => m[1].toLowerCase()); +type Finding = { kind: 'host'; host: string } | { kind: 'built' }; + +/** Host literals (lower case) and built URLs in a source text. */ +function findings(text: string): Finding[] { + const out: Finding[] = []; + for (const m of text.matchAll(SCHEME_HOST)) out.push({ kind: 'host', host: m[1].toLowerCase() }); + for (const m of text.matchAll(PROTOCOL_RELATIVE)) out.push({ kind: 'host', host: m[1].toLowerCase() }); + for (const _m of text.matchAll(BUILT_URL)) out.push({ kind: 'built' }); + return out; } -/** Why a host literal in `file` is fine, or null when it isn't accounted for. */ -export function classify(host: string, file: string, registered = registeredLiterals()): string | null { - if (registered.has(host)) return 'registry'; - const trimmed = host.replace(/\.+$/, ''); - if (registered.has(trimmed)) return 'registry'; - const allowed = NON_REQUEST_HOSTS.find((entry) => entry.host === trimmed); - if (allowed && allowed.files.includes(file)) return 'non-request'; - if (isPlaceholderHost(host)) return 'placeholder'; +const registered = registeredLiterals(); + +/** Why a finding in `file` is fine, or null when it isn't accounted for. */ +function classify(finding: Finding, file: string): string | null { + if (finding.kind === 'built') return BUILT_URL_FILES.some((b) => b.file === file) ? 'built' : null; + const host = finding.host; + const trimmed = host.replace(/\.{2,}$/, ''); + for (const literal of [host, trimmed]) { + if (registered.get(literal)?.has(file)) return 'registry'; + if (NON_REQUEST_HOSTS.some((entry) => entry.host === literal && entry.files.includes(file))) return 'non-request'; + } + // A registered or allowed host is only fine where it is listed. + if (registered.has(trimmed) || NON_REQUEST_HOSTS.some((entry) => entry.host === trimmed)) return null; + if (isPlaceholderHost(trimmed)) return 'placeholder'; return null; } describe('outgoing host literals', () => { const files = sourceFiles(); + const texts = new Map(files.map((file) => [file, readFileSync(path.join(ROOT, file), 'utf8')])); - it('scans the shipped source, service worker included, and leaves the tests out', () => { - expect(files).toContain('public/sw.js'); - expect(files).toContain('app/api/favicon/route.ts'); + it('scans the shipped source, service worker and config included, and leaves the tests out', () => { + for (const file of ['public/sw.js', 'next.config.ts', 'app/api/favicon/route.ts', 'app/api/dev-jmap/[...path]/route.ts', 'i18n/request.ts']) { + expect(files).toContain(file); + } expect(files.some((f) => f.includes('__tests__'))).toBe(false); expect(files.some((f) => f.startsWith('lib/demo/fixtures/'))).toBe(false); expect(files.length).toBeGreaterThan(500); }); - it('finds no host that is neither registered nor a known non-request', () => { - const registered = registeredLiterals(); + it('finds no host or built URL that isn\'t accounted for', () => { const unknown: string[] = []; - for (const file of files) { - const text = readFileSync(path.join(ROOT, file), 'utf8'); - for (const host of hostLiterals(text)) { - if (!classify(host, file, registered)) unknown.push(`${file}: ${host}`); + for (const [file, text] of texts) { + for (const finding of findings(text)) { + if (!classify(finding, file)) unknown.push(`${file}: ${finding.kind === 'host' ? finding.host : 'URL built from a variable'}`); } } - expect(unknown, 'Add the host to OUTGOING_HOSTS with a switch, or to NON_REQUEST_HOSTS if Bulwark never connects to it').toEqual([]); + expect([...new Set(unknown)], 'Add the host to OUTGOING_HOSTS with a switch, or to NON_REQUEST_HOSTS / BUILT_URL_FILES with the reason').toEqual([]); }); - it('lists only hosts that are still in the source', () => { - const all = files.map((file) => ({ file, text: readFileSync(path.join(ROOT, file), 'utf8') })); - const literals = new Set(all.flatMap(({ text }) => hostLiterals(text))); + it('lists only hosts and files that are still in the source', () => { + const hostsIn = (file: string) => findings(texts.get(file) ?? '') + .flatMap((f) => (f.kind === 'host' ? [f.host, f.host.replace(/\.{2,}$/, '')] : [])); for (const entry of OUTGOING_HOSTS) { - for (const literal of entry.literals ?? [entry.host]) { - expect(literals.has(literal), `${entry.id}: ${literal}`).toBe(true); + for (const file of entry.files) { + const found = (entry.literals ?? [entry.host]).some((literal) => hostsIn(file).includes(literal)); + expect(found, `${entry.id} in ${file}`).toBe(true); } } for (const entry of NON_REQUEST_HOSTS) { - for (const file of entry.files) { - const text = all.find((f) => f.file === file)?.text ?? ''; - expect(hostLiterals(text).some((h) => h.replace(/\.+$/, '') === entry.host), `${entry.host} in ${file}`).toBe(true); - } + for (const file of entry.files) expect(hostsIn(file).includes(entry.host), `${entry.host} in ${file}`).toBe(true); + } + for (const { file } of BUILT_URL_FILES) { + expect(findings(texts.get(file) ?? '').some((f) => f.kind === 'built'), `built URL in ${file}`).toBe(true); } }); - it('classifies the way the registry says', () => { - expect(classify('icons.duckduckgo.com', 'app/api/favicon/route.ts')).toBe('registry'); - expect(classify('openpgpkey.', 'lib/plugin-sandbox/host-api.ts')).toBe('registry'); - expect(classify('github.com', 'components/settings/about-data-settings.tsx')).toBe('non-request'); - expect(classify('github.com', 'lib/some-new-file.ts')).toBeNull(); - expect(classify('mail.example.com', 'anywhere.ts')).toBe('placeholder'); - expect(classify('cdn.unpkg.com', 'lib/x.ts')).toBeNull(); + it('keeps the browser\'s host list equal to the registry', () => { + expect([...BUILT_IN_HOSTNAMES].sort()).toEqual(registryHostnames().sort()); + }); +}); + +describe('the scan catches', () => { + const unknownIn = (text: string, file = 'lib/new-module.ts') => + findings(text).filter((f) => !classify(f, file)).map((f) => (f.kind === 'host' ? f.host : 'built')); + + it('a new outside host, in any case', () => { + expect(unknownIn(`fetch('https://tracker.fakecdn.io/pixel.gif')`)).toEqual(['tracker.fakecdn.io']); + expect(unknownIn(`fetch("HTTPS://Tracker.FakeCDN.io/p")`)).toEqual(['tracker.fakecdn.io']); + expect(unknownIn(`new WebSocket('wss://live.fakecdn.io/socket')`)).toEqual(['live.fakecdn.io']); + expect(unknownIn(`new WebSocket("ws://live.fakecdn.io")`)).toEqual(['live.fakecdn.io']); + }); + + it('a protocol-relative host in a string', () => { + expect(unknownIn(`script.src = '//cdn.fakecdn.io/lib.js'`)).toEqual(['cdn.fakecdn.io']); + expect(unknownIn(`// a comment about //cdn.fakecdn.io`)).toEqual([]); + }); + + it('a URL built from a variable or the scheme alone, outside the listed files', () => { + expect(unknownIn('fetch(`https://${host}/x`)')).toEqual(['built']); + expect(unknownIn(`url.startsWith('https://')`)).toEqual(['built']); + expect(unknownIn('fetch(`https://${host}/x`)', 'lib/validation.ts')).toEqual([]); + }); + + it('a public IP literal, but not a private or reserved one', () => { + expect(unknownIn(`fetch('http://93.184.216.34/x')`)).toEqual(['93.184.216.34']); + expect(unknownIn(`fetch('http://10.0.0.5:8080'); fetch('http://192.168.1.1'); fetch('http://127.0.0.1')`)).toEqual([]); + }); + + it('a registered or allowed host in a file it isn\'t listed for', () => { + expect(unknownIn(`fetch('https://icons.duckduckgo.com/ip3/x.ico')`)).toEqual(['icons.duckduckgo.com']); + expect(unknownIn(``)).toEqual(['github.com']); + expect(unknownIn(`fetch('https://icons.duckduckgo.com/ip3/x.ico')`, 'app/api/favicon/route.ts')).toEqual([]); }); }); diff --git a/lib/admin/outgoing-hosts-registry.ts b/lib/admin/outgoing-hosts-registry.ts new file mode 100644 index 000000000..b042bac89 --- /dev/null +++ b/lib/admin/outgoing-hosts-registry.ts @@ -0,0 +1,241 @@ +/** + * Every outside host Bulwark itself knows about, what talks to it, and what + * turns it off (#903). The admin view (/api/admin/outgoing-hosts) and the + * tests read this; the browser only needs the host names and the switches in + * lib/outgoing-hosts.ts, so none of this prose ends up in client chunks. + * + * lib/__tests__/outgoing-hosts.test.ts scans the source for host literals and + * fails on any that isn't accounted for here, in the file it is listed for. + * lib/__tests__/outgoing-hosts-mode.test.ts turns the mode on and checks + * every entry's switch. + */ + +export type OutgoingSide = 'server' | 'browser'; + +export interface OutgoingHost { + /** Stable id, used by the tests and the admin list. */ + id: string; + /** The host as people should read it. */ + host: string; + /** How the host appears in the source after the scheme, when it differs from `host`. */ + literals?: string[]; + /** The files that may name it. */ + files: string[]; + /** Who opens the connection: this server, or the user's browser. */ + side: OutgoingSide; + /** What uses it, in one sentence. */ + usedBy: string; + /** How to turn it off without the mode. */ + off: string; + /** What the mode does instead. */ + inMode: string; + /** Also present in Bulwark Lite. */ + lite: boolean; +} + +export const OUTGOING_HOSTS: readonly OutgoingHost[] = [ + { + id: 'telemetry', + host: 'telemetry.bulwarkmail.org', + files: ['lib/telemetry/types.ts'], + side: 'server', + usedBy: 'Anonymous daily heartbeat, sent only after an admin turned telemetry on.', + off: 'Off by default. BULWARK_TELEMETRY=off, or Admin > Telemetry.', + inMode: 'Never sent, whatever the telemetry setting says.', + lite: false, + }, + { + id: 'version-check', + host: 'version.telemetry.bulwarkmail.org', + files: ['lib/version-check/types.ts'], + side: 'server', + usedBy: 'Hourly check for a newer Bulwark release, shown to admins.', + off: 'BULWARK_UPDATE_CHECK=off, or BULWARK_UPDATE_CHECK_URL= (empty).', + inMode: 'Not checked; Admin > Version says why. Checks resume when the switch goes off.', + lite: false, + }, + { + id: 'push-relay', + host: 'notifications.relay.bulwarkmail.org', + files: ['lib/push-relays.ts', 'app/api/dev-jmap/[...path]/route.ts'], + side: 'browser', + usedBy: 'Built-in Web Push relay that wakes the browser for new mail.', + off: 'Add your own relays under Admin > User policy > Push relays and lock the choice.', + inMode: 'Not offered. Push works only with a relay the admin configured; existing registrations move to it or are removed.', + lite: false, + }, + { + id: 'extension-directory', + host: 'extensions.bulwarkmail.org', + files: ['lib/admin/directory-keys.ts', 'lib/admin/marketplace-directory.ts', 'lib/admin/types.ts'], + side: 'server', + usedBy: 'Marketplace browsing, installs and update checks in the admin panel.', + off: 'EXTENSION_DIRECTORY_URL pointing at a self-hosted directory.', + inMode: 'Off, unless extensionDirectoryUrl points at another host.', + lite: false, + }, + { + id: 'connector', + host: 'connector.bulwarkmail.org', + files: ['lib/admin/types.ts', 'lib/connector/registry.ts'], + side: 'server', + usedBy: 'Links in the docs and the directory that open this instance; the connector asks this server what it can open.', + off: 'CONNECTOR_ENABLED=false.', + inMode: 'The capabilities endpoint the connector asks answers 404.', + lite: false, + }, + { + id: 'favicons', + host: 'icons.duckduckgo.com', + files: ['app/api/favicon/route.ts'], + side: 'server', + usedBy: 'Sender icons next to messages, fetched by /api/favicon.', + off: 'Users can turn sender icons off in Settings > Message list.', + inMode: 'Avatars show initials; /api/favicon answers 403 before any request.', + lite: false, + }, + { + id: 'translate', + host: 'api.mymemory.translated.net', + files: ['app/api/translate/route.ts'], + side: 'server', + usedBy: 'Default provider of /api/translate, which the Translate plugin calls.', + off: 'Set LIBRETRANSLATE_URL to your own LibreTranslate and pick it in the plugin.', + inMode: 'Refused with 403; LibreTranslate at LIBRETRANSLATE_URL still works.', + lite: false, + }, + { + id: 'maps', + host: 'www.google.com', + files: ['lib/event-links.ts'], + side: 'browser', + usedBy: 'Tapping the address of a calendar event opens Google Maps.', + off: 'Nothing else turns it off; it is a link the user taps.', + inMode: 'The address is plain text with a copy button.', + lite: true, + }, + { + id: 'wkd', + host: 'openpgpkey.', + literals: ['openpgpkey.'], + files: ['lib/plugin-sandbox/host-api.ts'], + side: 'browser', + usedBy: 'OpenPGP key lookup (WKD) at the recipient\'s domain, for PGP plugins.', + off: 'Don\'t install a PGP plugin that looks keys up.', + inMode: 'The lookup is refused.', + lite: false, + }, + { + id: 'dev-avatars', + host: 'randomuser.me', + literals: ['randomuser.me', 'img.freepik.com'], + files: ['components/ui/avatar.tsx', 'app/api/dev-jmap/[...path]/route.ts'], + side: 'browser', + usedBy: 'Sample sender and contact photos, only with the built-in mock server (DEV_MOCK_JMAP=true).', + off: 'Don\'t run the mock server.', + inMode: 'Not shown.', + lite: false, + }, +]; + +/** + * Hosts in the source that Bulwark never connects to on its own: docs links, + * XML namespaces, credits, examples in comments and sample text. Each is + * allowed only in the files named, so the same host turning up somewhere new + * has to be looked at again. + */ +export const NON_REQUEST_HOSTS: readonly { host: string; files: readonly string[]; why: string }[] = [ + { host: 'bulwarkmail.org', why: 'docs links and comments', files: ['app/(main)/admin/_pages/telemetry.tsx', 'instrumentation.node.ts', 'lib/telemetry/state.ts', 'lib/telemetry/types.ts'] }, + { host: 'bulwarkmail.com', why: 'JMAP capability URI', files: ['lib/jmap/client.ts'] }, + { host: 'extensions.bulwarkmail.org', why: 'signed statement type URIs', files: ['lib/admin/directory-keys.ts'] }, + { host: 'www.w3.org', why: 'XML namespaces', files: ['app/(main)/setup/page.tsx', 'components/ui/flag-icons.tsx', 'lib/favicon-badge.ts'] }, + { host: 'creativecommons.org', why: 'licence metadata inside an SVG', files: ['components/ui/flag-icons.tsx'] }, + { host: 'github.com', why: 'source code link (AGPL) and SVG metadata', files: ['components/settings/about-data-settings.tsx', 'components/ui/flag-icons.tsx'] }, + { host: 'www.rfc-editor.org', why: 'comments', files: ['lib/jmap/client.ts'] }, + { host: 'devguide.calconnect.org', why: 'comment', files: ['lib/jmap/client.ts'] }, + { host: 'stalw.art', why: 'comment', files: ['app/api/auth/impersonate/route.ts'] }, + { host: 'tabler.io', why: 'icon credit link', files: ['app/(main)/admin/_pages/policy.tsx'] }, + { host: 'www.youtube-nocookie.com', why: 'example in a comment', files: ['lib/plugin-types.ts'] }, + { host: 'bulwark-webmail.vercel.app', why: 'sample message text', files: ['stores/email-store.ts'] }, + { host: 'randomuser.me', why: 'docs links in a comment of the mock server', files: ['app/api/dev-jmap/[...path]/route.ts'] }, +]; + +/** + * Files that build an http(s) URL from a variable or name the scheme alone + * (for checks and placeholders). The host there comes from configuration, + * user input or the data around it, never from a built-in default; each is + * listed with the reason. + */ +export const BUILT_URL_FILES: readonly { file: string; why: string }[] = [ + { file: 'app/(main)/admin/_pages/marketplace-detail.tsx', why: "link to an extension's source host, from the directory's answer" }, + { file: 'app/(main)/setup/page.tsx', why: "the instance's own origin, and scheme checks of the admin-entered server URL" }, + { file: 'app/api/plugin-secrets/route.ts', why: "scheme check of a credential's server URL" }, + { file: 'components/calendar/ical-subscription-modal.tsx', why: "webcal:// rewritten to https:// for a feed the user enters" }, + { file: 'lib/admin/branding-asset.ts', why: "scheme checks of admin-configured branding URLs" }, + { file: 'lib/auth/pair-bundle.ts', why: "scheme checks of the paired server URL" }, + { file: 'lib/calendar-subscription-sync.ts', why: "webcal:// rewritten to https:// for a feed the user subscribed to" }, + { file: 'lib/email-sanitization.ts', why: "comment about links in mail" }, + { file: 'lib/idn.ts', why: "parses a host name the user typed" }, + { file: 'lib/plugin-sandbox/connect-dialog.ts', why: "adds the scheme to a server address the user enters" }, + { file: 'lib/plugin-sandbox/host-api.ts', why: "WKD lookup at the recipient's domain (registered as wkd), and an error message" }, + { file: 'lib/validation.ts', why: "scheme checks" }, + { file: 'lib/vcard.ts', why: "scheme checks of contact URLs" }, + { file: 'stores/calendar-store.ts', why: "webcal:// rewritten to https:// for a feed the user subscribed to" }, +]; + +/** Folders whose host literals are sample mail content, never requests. */ +export const NON_REQUEST_PATHS: readonly string[] = [ + 'lib/demo/fixtures/', +]; + +const RESERVED_TLDS = new Set(['example', 'test', 'invalid', 'localhost', 'local', 'internal', 'lan']); + +/** IPv4 literals in private, loopback, link-local, shared, documentation and other reserved ranges. */ +export function isReservedIPv4(ip: string): boolean { + const parts = ip.split('.').map(Number); + if (parts.length !== 4 || parts.some((p) => !Number.isInteger(p) || p < 0 || p > 255)) return false; + const [a, b, c] = parts; + return a === 0 || a === 10 || a === 127 || a >= 224 + || (a === 100 && b >= 64 && b <= 127) + || (a === 169 && b === 254) + || (a === 172 && b >= 16 && b <= 31) + || (a === 192 && b === 168) + || (a === 192 && b === 0 && (c === 0 || c === 2)) + || (a === 198 && (b === 18 || b === 19)) + || (a === 198 && b === 51 && c === 100) + || (a === 203 && b === 0 && c === 113); +} + +/** + * Hosts that can't be anyone else's: example and reserved names, single + * labels, private or reserved IP literals and CSP wildcards. Placeholders and + * tests use them. A public IP literal is not one. + */ +export function isPlaceholderHost(raw: string): boolean { + const host = raw.toLowerCase().replace(/:\d+$/, '').replace(/\.+$/, ''); + if (/^[\d.]+$/.test(host)) return isReservedIPv4(host); + if (!host.includes('.') || host.includes('*')) return true; + const labels = host.split('.'); + const tld = labels[labels.length - 1]; + if (RESERVED_TLDS.has(tld)) return true; + return /(^|\.)example\.(com|org|net)$/.test(host); +} + +/** Every host literal the registry accounts for, with the files that may name it. */ +export function registeredLiterals(): Map> { + const out = new Map>(); + for (const entry of OUTGOING_HOSTS) { + for (const literal of entry.literals ?? [entry.host]) { + const files = out.get(literal) ?? new Set(); + for (const file of entry.files) files.add(file); + out.set(literal, files); + } + } + return out; +} + +/** The exact host names of the registry, which the browser module repeats. */ +export function registryHostnames(): string[] { + return OUTGOING_HOSTS.flatMap((entry) => entry.literals ?? [entry.host]).filter((h) => !h.endsWith('.')); +} + diff --git a/lib/outgoing-hosts.ts b/lib/outgoing-hosts.ts index 5b64892e7..514e203c4 100644 --- a/lib/outgoing-hosts.ts +++ b/lib/outgoing-hosts.ts @@ -1,190 +1,28 @@ /** - * Every outside host Bulwark itself knows about, what talks to it, and what - * turns it off (#903). + * The browser-side switches of the "no third-party traffic" mode (#903). * - * "Outside" means a host the deployment didn't configure: the admin's JMAP - * servers, push relays, OAuth issuer, WOPI editor, sidebar apps and a - * self-hosted extension directory are the deployment's own choice and stay - * reachable. Everything listed here is off while `noThirdPartyTraffic` - * (env NO_THIRD_PARTY_TRAFFIC) is on. - * - * lib/__tests__/outgoing-hosts.test.ts scans the source for http(s) host - * literals and fails on any that is neither listed here nor in - * NON_REQUEST_HOSTS, so a new built-in host can't slip in unnoticed; a - * second test turns the mode on and checks every entry's switch. - * - * Plain data and pure functions: imported by the browser, the server routes, - * the admin page and the tests. + * Small on purpose: this module ships in every client chunk and in Lite. The + * registry with the admin-facing descriptions is lib/admin/outgoing-hosts-registry.ts. + * Every helper that opens an outside host asks noThirdPartyTraffic(), which is + * true while the mode is on and while the config isn't known yet. */ import { noThirdPartyTraffic } from './third-party-mode'; -export type OutgoingSide = 'server' | 'browser'; - -export interface OutgoingHost { - /** Stable id, used by the tests and the admin list. */ - id: string; - /** The host as people should read it. */ - host: string; - /** How the host appears in the source after `https://`, when it differs from `host`. */ - literals?: string[]; - /** Who opens the connection: this server, or the user's browser. */ - side: OutgoingSide; - /** What uses it, in one sentence. */ - usedBy: string; - /** How to turn it off without the mode. */ - off: string; - /** What the mode does instead. */ - inMode: string; - /** Also present in Bulwark Lite. */ - lite: boolean; -} - -export const OUTGOING_HOSTS: readonly OutgoingHost[] = [ - { - id: 'telemetry', - host: 'telemetry.bulwarkmail.org', - side: 'server', - usedBy: 'Anonymous daily heartbeat, sent only after an admin turned telemetry on.', - off: 'Off by default. BULWARK_TELEMETRY=off, or Admin > Telemetry.', - inMode: 'Never sent, whatever the telemetry setting says.', - lite: false, - }, - { - id: 'version-check', - host: 'version.telemetry.bulwarkmail.org', - side: 'server', - usedBy: 'Hourly check for a newer Bulwark release, shown to admins.', - off: 'BULWARK_UPDATE_CHECK=off, or BULWARK_UPDATE_CHECK_URL= (empty).', - inMode: 'Not checked; Admin > Version says why.', - lite: false, - }, - { - id: 'push-relay', - host: 'notifications.relay.bulwarkmail.org', - side: 'browser', - usedBy: 'Built-in Web Push relay that wakes the browser for new mail.', - off: 'Add your own relays under Admin > User policy > Push relays and lock the choice.', - inMode: 'Not offered. Push works only with a relay the admin configured.', - lite: false, - }, - { - id: 'extension-directory', - host: 'extensions.bulwarkmail.org', - side: 'server', - usedBy: 'Marketplace browsing, installs and update checks in the admin panel.', - off: 'EXTENSION_DIRECTORY_URL pointing at a self-hosted directory.', - inMode: 'Off, unless extensionDirectoryUrl points at another host.', - lite: false, - }, - { - id: 'connector', - host: 'connector.bulwarkmail.org', - side: 'server', - usedBy: 'Links in the docs and the directory that open this instance; the connector asks this server what it can open.', - off: 'CONNECTOR_ENABLED=false.', - inMode: 'The capabilities endpoint the connector asks answers 404.', - lite: false, - }, - { - id: 'favicons', - host: 'icons.duckduckgo.com', - side: 'server', - usedBy: 'Sender icons next to messages, fetched by /api/favicon.', - off: 'Users can turn sender icons off in Settings > Message list.', - inMode: 'Avatars show initials; /api/favicon answers 403 before any request.', - lite: false, - }, - { - id: 'translate', - host: 'api.mymemory.translated.net', - side: 'server', - usedBy: 'Default provider of /api/translate, which the Translate plugin calls.', - off: 'Set LIBRETRANSLATE_URL to your own LibreTranslate and pick it in the plugin.', - inMode: 'Refused with 403; LibreTranslate at LIBRETRANSLATE_URL still works.', - lite: false, - }, - { - id: 'maps', - host: 'www.google.com', - side: 'browser', - usedBy: 'Tapping the address of a calendar event opens Google Maps.', - off: 'Nothing else turns it off; it is a link the user taps.', - inMode: 'The address is plain text with a copy button.', - lite: true, - }, - { - id: 'wkd', - host: 'openpgpkey.', - literals: ['openpgpkey.'], - side: 'browser', - usedBy: 'OpenPGP key lookup (WKD) at the recipient\'s domain, for PGP plugins.', - off: 'Don\'t install a PGP plugin that looks keys up.', - inMode: 'The lookup is refused.', - lite: false, - }, - { - id: 'dev-avatars', - host: 'randomuser.me', - literals: ['randomuser.me', 'img.freepik.com'], - side: 'browser', - usedBy: 'Sample sender photos, only with the built-in mock server (DEV_MOCK_JMAP=true).', - off: 'Don\'t run the mock server.', - inMode: 'Not shown.', - lite: false, - }, +/** Exact host names of the built-in outside hosts (the registry repeats them; a test keeps both equal). */ +export const BUILT_IN_HOSTNAMES: readonly string[] = [ + 'telemetry.bulwarkmail.org', + 'version.telemetry.bulwarkmail.org', + 'notifications.relay.bulwarkmail.org', + 'extensions.bulwarkmail.org', + 'connector.bulwarkmail.org', + 'icons.duckduckgo.com', + 'api.mymemory.translated.net', + 'www.google.com', + 'randomuser.me', + 'img.freepik.com', ]; -/** - * Hosts in the source that Bulwark never connects to on its own: docs links, - * XML namespaces, credits, examples in comments and sample text. Each is - * allowed only in the files named, so the same host turning up somewhere new - * has to be looked at again. - */ -export const NON_REQUEST_HOSTS: readonly { host: string; files: readonly string[]; why: string }[] = [ - { host: 'bulwarkmail.org', why: 'docs links and comments', files: ['app/(main)/admin/_pages/telemetry.tsx', 'instrumentation.node.ts', 'lib/telemetry/state.ts', 'lib/telemetry/types.ts'] }, - { host: 'bulwarkmail.com', why: 'JMAP capability URI', files: ['lib/jmap/client.ts'] }, - { host: 'extensions.bulwarkmail.org', why: 'signed statement type URIs', files: ['lib/admin/directory-keys.ts'] }, - { host: 'www.w3.org', why: 'XML namespaces', files: ['app/(main)/setup/page.tsx', 'components/ui/flag-icons.tsx', 'lib/favicon-badge.ts'] }, - { host: 'creativecommons.org', why: 'licence metadata inside an SVG', files: ['components/ui/flag-icons.tsx'] }, - { host: 'github.com', why: 'source code link (AGPL) and SVG metadata', files: ['components/settings/about-data-settings.tsx', 'components/ui/flag-icons.tsx'] }, - { host: 'www.rfc-editor.org', why: 'comments', files: ['lib/jmap/client.ts'] }, - { host: 'devguide.calconnect.org', why: 'comment', files: ['lib/jmap/client.ts'] }, - { host: 'stalw.art', why: 'comment', files: ['app/api/auth/impersonate/route.ts'] }, - { host: 'tabler.io', why: 'icon credit link', files: ['app/(main)/admin/_pages/policy.tsx'] }, - { host: 'www.youtube-nocookie.com', why: 'example in a comment', files: ['lib/plugin-types.ts'] }, - { host: 'bulwark-webmail.vercel.app', why: 'sample message text', files: ['stores/email-store.ts'] }, -]; - -/** Folders whose host literals are sample mail content, never requests. */ -export const NON_REQUEST_PATHS: readonly string[] = [ - 'lib/demo/fixtures/', - 'app/api/dev-jmap/', -]; - -const RESERVED_TLDS = new Set(['example', 'test', 'invalid', 'localhost', 'local', 'internal', 'lan']); - -/** - * Hosts that can't be anyone else's: example and reserved names, single - * labels, IP literals and CSP wildcards. Placeholders and tests use them. - */ -export function isPlaceholderHost(raw: string): boolean { - const host = raw.toLowerCase().replace(/:\d+$/, '').replace(/\.+$/, ''); - if (!host.includes('.') || host.includes('*')) return true; - if (/^\d{1,3}(\.\d{1,3}){0,3}\.?$/.test(host)) return true; - const labels = host.split('.'); - const tld = labels[labels.length - 1]; - if (RESERVED_TLDS.has(tld)) return true; - return /(^|\.)example\.(com|org|net)$/.test(host); -} - -/** Every host literal the registry accounts for. */ -export function registeredLiterals(): Set { - const out = new Set(); - for (const entry of OUTGOING_HOSTS) for (const literal of entry.literals ?? [entry.host]) out.add(literal); - return out; -} - function hostnameOf(url: string): string | null { try { return new URL(url).hostname.toLowerCase(); @@ -193,15 +31,12 @@ function hostnameOf(url: string): string | null { } } -/** Whether a URL points at one of the built-in outside hosts above. */ +/** Whether a URL points at one of the built-in outside hosts. */ export function isBuiltInHost(url: string): boolean { const host = hostnameOf(url); - if (!host) return false; - return OUTGOING_HOSTS.some((entry) => (entry.literals ?? [entry.host]).some((literal) => literal === host)); + return !!host && BUILT_IN_HOSTNAMES.includes(host); } -// ─── Browser-side switches ──────────────────────────────────────────────── - /** A push relay may be used: always, except the built-in one in the mode. */ export function relayAllowed(url: string): boolean { return !(noThirdPartyTraffic() && isBuiltInHost(url)); @@ -227,6 +62,24 @@ export function sampleAvatarsAllowed(): boolean { return !noThirdPartyTraffic(); } +/** + * An avatar URL may be shown: a plugin's (Gravatar, Libravatar...) or a + * contact card's photo link. In the mode only when it stays on this origin or + * is inline data, so those avatars fall back to initials instead of reaching + * out (or tripping the CSP). + */ +export function avatarUrlAllowed(url: string | null | undefined, origin?: string): boolean { + if (!url) return false; + if (!noThirdPartyTraffic()) return true; + if (/^(data|blob):/i.test(url)) return true; + const base = origin ?? (typeof window !== 'undefined' ? window.location.origin : ''); + try { + return !!base && new URL(url, base).origin === new URL(base).origin; + } catch { + return false; + } +} + export interface NetworkDeclaringPlugin { permissions?: readonly string[]; httpOrigins?: readonly string[]; diff --git a/scripts/lite/scan-hosts.mjs b/scripts/lite/scan-hosts.mjs new file mode 100644 index 000000000..5a48e122a --- /dev/null +++ b/scripts/lite/scan-hosts.mjs @@ -0,0 +1,120 @@ +#!/usr/bin/env node +// Scans a finished Lite export for outside hosts (#903). Every http(s) or +// ws(s) host literal, and every protocol-relative //host in a string, must be +// the configured JMAP server, a placeholder, one of Bulwark's built-in outside +// hosts (switched off at runtime by noThirdPartyTraffic) or a known string +// that is never requested, in the kind of file it is listed for. Anything else +// fails the build in CI (.github/workflows/static-lite.yml). +// +// node scripts/lite/scan-hosts.mjs [outDir=out] +// +// LITE_BUILT_IN_HOSTS mirrors BUILT_IN_HOSTNAMES in lib/outgoing-hosts.ts (a +// TS module this script cannot import); lib/__tests__/lite-bundle-hosts.test.ts +// pins the two together. +import { readdirSync, readFileSync, statSync } from "node:fs"; +import { join, relative, sep } from "node:path"; +import { isMainModule, resolveRepoRoot } from "./lib.mjs"; + +export const LITE_BUILT_IN_HOSTS = [ + "telemetry.bulwarkmail.org", + "version.telemetry.bulwarkmail.org", + "notifications.relay.bulwarkmail.org", + "extensions.bulwarkmail.org", + "connector.bulwarkmail.org", + "icons.duckduckgo.com", + "api.mymemory.translated.net", + "www.google.com", + "randomuser.me", + "img.freepik.com", + "openpgpkey.", +]; + +const CHUNKS = /^_next\/static\/chunks\/[^/]+\.js$/; +const PAGES = /\.(html|txt)$/; + +/** Strings in the bundle that are never requested, by where they may appear and why. */ +export const LITE_BUNDLE_NON_REQUEST = [ + { files: CHUNKS, why: "Bulwark's built-in hosts, off at runtime with noThirdPartyTraffic", hosts: LITE_BUILT_IN_HOSTS }, + { files: CHUNKS, why: "error and docs links of bundled libraries (Next.js, React, JSZip, ProseMirror, core-js, date-fns)", hosts: ["nextjs.org", "react.dev", "stuk.github.io", "prosemirror.net", "github.com", "foo.bar"] }, + { files: CHUNKS, why: "example addresses the translations localise", hosts: ["eksempel.dk", "mail.eksempel.dk", "piemers.lv", "pasts.piemers.lv", "ornek.com", "mail.ornek.com"] }, + { files: CHUNKS, why: "links inside the sample mail of the demo mode (lib/demo/fixtures)", hosts: ["zoom.us", "notion.so", "www.figma.com", "vercel.com", "i.scdn.co", "cf.bstatic.com", "open.spotify.com", "linear.app", "sentry.io", "app.docusign.com", "github.com", "bulwark-webmail.vercel.app"] }, + { files: CHUNKS, why: "JMAP capability URI and XML namespaces", hosts: ["bulwarkmail.com", "www.w3.org", "creativecommons.org"] }, + { files: /./, why: "docs links in the translations and the README", hosts: ["bulwarkmail.org", "github.com"] }, + { files: PAGES, why: "XML namespaces of inline SVG", hosts: ["www.w3.org"] }, +]; + +const RESERVED_TLDS = new Set(["example", "test", "invalid", "localhost", "local", "internal", "lan"]); + +function isReservedIPv4(ip) { + const parts = ip.split(".").map(Number); + if (parts.length !== 4 || parts.some((p) => !Number.isInteger(p) || p < 0 || p > 255)) return false; + const [a, b, c] = parts; + return a === 0 || a === 10 || a === 127 || a >= 224 + || (a === 100 && b >= 64 && b <= 127) || (a === 169 && b === 254) + || (a === 172 && b >= 16 && b <= 31) || (a === 192 && b === 168) + || (a === 192 && b === 0 && (c === 0 || c === 2)) || (a === 198 && (b === 18 || b === 19)) + || (a === 198 && b === 51 && c === 100) || (a === 203 && b === 0 && c === 113); +} + +/** Same rule as isPlaceholderHost in lib/admin/outgoing-hosts-registry.ts. */ +export function isPlaceholderHost(raw) { + const host = raw.toLowerCase().replace(/:\d+$/, "").replace(/\.+$/, ""); + if (/^[\d.]+$/.test(host)) return isReservedIPv4(host); + if (!host.includes(".") || host.includes("*")) return true; + const tld = host.split(".").pop(); + if (RESERVED_TLDS.has(tld)) return true; + return /(^|\.)example\.(com|org|net)$/.test(host); +} + +const SCHEME_HOST = /\b(?:https?|wss?):\/\/([a-z0-9.*_-]+)/gi; +const PROTOCOL_RELATIVE = /["'`]\/\/([a-z0-9-]+(?:\.[a-z0-9-]+)+)/gi; + +export function bundleHosts(text) { + const out = []; + for (const m of text.matchAll(SCHEME_HOST)) out.push(m[1].toLowerCase()); + for (const m of text.matchAll(PROTOCOL_RELATIVE)) out.push(m[1].toLowerCase()); + return out; +} + +function files(dir, out = []) { + for (const name of readdirSync(dir)) { + const full = join(dir, name); + if (statSync(full).isDirectory()) files(full, out); + else if (/\.(js|mjs|html|css|json|txt|webmanifest|md|example)$/.test(name) || name === "_headers") out.push(full); + } + return out; +} + +/** "file: host" for every host in `outDir` that isn't accounted for. */ +export function scanBundleHosts(outDir) { + let jmapHost = null; + try { + const config = JSON.parse(readFileSync(join(outDir, "config.json"), "utf8")); + if (config.jmapServerUrl) jmapHost = new URL(config.jmapServerUrl).host.toLowerCase(); + } catch { /* no config: nothing configured */ } + const problems = new Set(); + for (const full of files(outDir)) { + const file = relative(outDir, full).split(sep).join("/"); + for (const raw of bundleHosts(readFileSync(full, "utf8"))) { + const host = raw.replace(/\.{2,}$/, ""); + if (host === jmapHost) continue; + const listed = LITE_BUNDLE_NON_REQUEST.some((entry) => entry.files.test(file) && (entry.hosts.includes(raw) || entry.hosts.includes(host))); + if (listed) continue; + const known = LITE_BUNDLE_NON_REQUEST.some((entry) => entry.hosts.includes(host)); + if (!known && isPlaceholderHost(host)) continue; + problems.add(`${file}: ${host}`); + } + } + return [...problems]; +} + +if (isMainModule(import.meta.url)) { + const outDir = process.argv[2] ?? join(resolveRepoRoot(import.meta.url), "out"); + const problems = scanBundleHosts(outDir); + if (problems.length > 0) { + console.error(`[lite] ${problems.length} outside host(s) in the bundle that aren't accounted for (scripts/lite/scan-hosts.mjs):`); + for (const p of problems) console.error(` - ${p}`); + process.exit(1); + } + console.log("[lite] no unaccounted outside hosts in the bundle"); +}