diff --git a/.dockerignore b/.dockerignore index 130ea406c..e61ff58d2 100644 --- a/.dockerignore +++ b/.dockerignore @@ -14,6 +14,7 @@ scripts/ !scripts/build-pst-worker.mjs !scripts/healthcheck.mjs !scripts/sourcemaps.mjs +!scripts/standalone-guard.mjs # Built in the image by scripts/build-plugin-sandbox.mjs public/plugin-sandbox-assets/ TODO.md diff --git a/Dockerfile b/Dockerfile index add3eabd1..ed2713b1f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,7 +36,8 @@ RUN rm -rf "app/api/dev-jmap" && \ -prune -exec rm -rf {} \; && \ node scripts/build-pst-worker.mjs && \ node scripts/build-plugin-sandbox.mjs && \ - npx next build --webpack + npx next build --webpack && \ + node scripts/standalone-guard.mjs # webpack has no debug IDs of its own, so sentry-cli adds them to the # chunks; then every map moves to /sourcemaps (exported with diff --git a/lib/__tests__/standalone-guard.test.ts b/lib/__tests__/standalone-guard.test.ts new file mode 100644 index 000000000..dbc69e792 --- /dev/null +++ b/lib/__tests__/standalone-guard.test.ts @@ -0,0 +1,181 @@ +// @vitest-environment node +import { afterEach, describe, expect, it } from 'vitest'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { spawnSync } from 'node:child_process'; +import os from 'node:os'; +import path from 'node:path'; +import { + findForbidden, + guardStandalone, + isTraceFileAllowed, + pruneTraces, +} from '../../scripts/standalone-guard.mjs'; + +const ROOT = path.resolve(__dirname, '../..'); +const SCRIPT = path.join(ROOT, 'scripts/standalone-guard.mjs'); +const temps: string[] = []; + +function tempDir(): string { + const dir = mkdtempSync(path.join(os.tmpdir(), 'standalone-guard-')); + temps.push(dir); + return dir; +} + +function put(root: string, rel: string, body = 'x'): void { + const file = path.join(root, rel); + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync(file, body); +} + +afterEach(() => { + while (temps.length) rmSync(temps.pop()!, { recursive: true, force: true }); +}); + +describe('standalone guard check', () => { + it('fails the build on a planted signing key', () => { + const dir = tempDir(); + put(dir, 'server.js'); + put(dir, 'data/admin/plugin-signing.key', 'FAKE'); + const run = spawnSync(process.execPath, [SCRIPT, dir], { encoding: 'utf8' }); + expect(run.status).toBe(1); + expect(run.stderr).toContain('data/'); + }); + + it('passes a clean tree', () => { + const dir = tempDir(); + put(dir, 'server.js'); + put(dir, 'package.json'); + put(dir, 'public/icon.png'); + put(dir, '.next/server/app/api/data/route.js'); + put(dir, 'node_modules/some-package/data/table.json'); + const run = spawnSync(process.execPath, [SCRIPT, dir], { encoding: 'utf8' }); + expect(run.stderr).toBe(''); + expect(run.status).toBe(0); + }); + + it.each([ + 'data/admin/admin.json', + 'local-data/notes.md', + 'repos/plugins/index.js', + '.env.local', + 'lib/plugin-signing.key', + ])('finds %s', (rel) => { + const dir = tempDir(); + put(dir, 'server.js'); + put(dir, rel); + expect(findForbidden(dir)).toHaveLength(1); + }); + + it('finds them when the project is nested under the tracing root', () => { + const dir = tempDir(); + put(dir, 'worktrees/webmail/server.js'); + put(dir, 'worktrees/webmail/data/admin/plugin-signing.key'); + expect(findForbidden(dir)).toEqual(['worktrees/webmail/data/']); + }); + + it('removes the .env files Next copies next to server.js, then passes', () => { + const dir = tempDir(); + put(dir, 'worktrees/webmail/server.js'); + put(dir, 'worktrees/webmail/.env', 'SECRET=1'); + put(dir, 'worktrees/webmail/.env.production', 'SECRET=1'); + expect(guardStandalone(dir)).toEqual([]); + expect(existsSync(path.join(dir, 'worktrees/webmail/.env'))).toBe(false); + expect(existsSync(path.join(dir, 'worktrees/webmail/.env.production'))).toBe(false); + }); + + it('does not remove a .env.local, which Next never copies, and reports it', () => { + const dir = tempDir(); + put(dir, 'server.js'); + put(dir, '.env.local', 'SECRET=1'); + expect(guardStandalone(dir)).toEqual(['.env.local']); + }); +}); + +describe('trace pruning', () => { + const project = path.resolve('/project'); + + it.each([ + ['node_modules/next/dist/server.js', true], + ['.next/server/chunks/1.js', true], + ['public/icon.png', true], + ['package.json', true], + ['data/admin/plugin-signing.key', false], + ['local-data/notes.md', false], + ['repos/plugins/index.js', false], + ['.env', false], + ['.git/config', false], + ['lib/admin/paths.ts', false], + ])('%s allowed: %s', (rel, allowed) => { + expect(isTraceFileAllowed(path.join(project, rel), project)).toBe(allowed); + }); + + it('keeps a node_modules outside the project and nothing else from there', () => { + expect(isTraceFileAllowed(path.resolve('/elsewhere/node_modules/next/x.js'), project)).toBe(true); + expect(isTraceFileAllowed(path.resolve('/elsewhere/secrets.txt'), project)).toBe(false); + }); + + it('rewrites a trace without the planted files', () => { + const projectDir = tempDir(); + const distDir = path.join(projectDir, '.next'); + const trace = path.join(distDir, 'server/middleware.js.nft.json'); + put( + projectDir, + '.next/server/middleware.js.nft.json', + JSON.stringify({ + version: 1, + files: ['../../data/admin/plugin-signing.key', '../../.env', '../../node_modules/next/index.js', 'chunks/1.js'], + }), + ); + expect(pruneTraces({ distDir, projectDir })).toBe(2); + expect(JSON.parse(readFileSync(trace, 'utf8'))).toEqual({ + version: 1, + files: ['../../node_modules/next/index.js', 'chunks/1.js'], + }); + }); + + it('keeps Turbopack hashes and symlink indexes on the right files', () => { + const projectDir = tempDir(); + const distDir = path.join(projectDir, '.next'); + const trace = path.join(distDir, 'server/middleware.js.nft.json'); + put( + projectDir, + '.next/server/middleware.js.nft.json', + JSON.stringify({ + version: 1, + files: ['../../data/x', '../../node_modules/a', '../../repos/y', '../node_modules/esbuild-1', '../../data/link'], + fileHashes: ['h0', 'h1', 'h2', 'h3', 'h4'], + symlinks: [ + [3, '../../node_modules/esbuild'], + [4, '../../outside'], + ], + }), + ); + expect(pruneTraces({ distDir, projectDir })).toBe(3); + expect(JSON.parse(readFileSync(trace, 'utf8'))).toEqual({ + version: 1, + files: ['../../node_modules/a', '../node_modules/esbuild-1'], + fileHashes: ['h1', 'h3'], + symlinks: [[1, '../../node_modules/esbuild']], + }); + }); + + it('leaves unreadable traces alone instead of failing mid-build', () => { + const projectDir = tempDir(); + put(projectDir, '.next/server/page.js.nft.json', '{ half written'); + expect(pruneTraces({ distDir: path.join(projectDir, '.next'), projectDir })).toBe(0); + }); +}); + +describe('build wiring', () => { + it('prunes after compiling and checks after every standalone build', () => { + const config = readFileSync(path.join(ROOT, 'next.config.ts'), 'utf8'); + expect(config).toMatch(/runAfterProductionCompile[\s\S]*pruneTraces/); + const pkg = JSON.parse(readFileSync(path.join(ROOT, 'package.json'), 'utf8')); + expect(pkg.scripts.build).toMatch(/next build[^&]*&& node scripts\/standalone-guard\.mjs/); + expect(readFileSync(path.join(ROOT, 'Dockerfile'), 'utf8')).toMatch( + /next build --webpack[\s\S]{0,40}node scripts\/standalone-guard\.mjs/, + ); + expect(readFileSync(path.join(ROOT, '.dockerignore'), 'utf8')).toContain('!scripts/standalone-guard.mjs'); + expect(readFileSync(path.join(ROOT, '.github/workflows/standalone-release.yml'), 'utf8')).toMatch(/run: npm run build/); + }); +}); diff --git a/next.config.ts b/next.config.ts index ba49d2cc2..e57a5f3af 100644 --- a/next.config.ts +++ b/next.config.ts @@ -4,6 +4,7 @@ import { execSync } from "child_process"; import { createHash } from "crypto"; import { readFileSync } from "fs"; import { join } from "path"; +import { pruneTraces } from "./scripts/standalone-guard.mjs"; // Prefer an explicit build arg (passed in by CI / Docker, where .git is // excluded from the build context) and fall back to `git rev-parse` for @@ -154,10 +155,9 @@ const nextConfig: NextConfig = { // Sibling repos checked out under ./repos/ are unrelated source trees that // Turbopack's NFT can otherwise rope into the trace when dynamic fs calls // confuse it. Keeps the build from ballooning memory tracing dead code. - // The trace follows fs reads, and the runtime reads its state directories, - // so a build from a checkout that has been run would copy the admin - // password hash, the plugin signing key, the audit log and .env secrets - // into .next/standalone - and from there into the image. + // These excludes are not what keeps private state out of the build: they + // miss several traces. runAfterProductionCompile below and the final check + // in scripts/standalone-guard.mjs do that. // Only "./repos" goes under "*": that key also feeds the next-server trace, // where Next matches each glob anywhere in the path, so "./data/**/*" would // drop node_modules/next/dist/lib/metadata/* and the server would not start. @@ -167,6 +167,15 @@ const nextConfig: NextConfig = { "*": ["./repos/**/*"], "!(next-server)": ["./data/**/*", "./local-data/**/*", "./.env*"], }, + ...(isLite + ? {} + : { + compiler: { + runAfterProductionCompile: async ({ distDir, projectDir }) => { + pruneTraces({ distDir, projectDir }); + }, + }, + }), turbopack: { root: process.env.NEXT_TURBOPACK_ROOT || import.meta.dirname, ...(sourceMaps ? { debugIds: true } : {}), diff --git a/package.json b/package.json index 4627ae0ef..9c1abb8ac 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,7 @@ ], "scripts": { "dev": "node scripts/build-pst-worker.mjs && node scripts/build-plugin-sandbox.mjs --watch -- next dev --turbopack", - "build": "node scripts/build-pst-worker.mjs && node scripts/build-plugin-sandbox.mjs && next build --turbopack", + "build": "node scripts/build-pst-worker.mjs && node scripts/build-plugin-sandbox.mjs && next build --turbopack && node scripts/standalone-guard.mjs", "build:lite": "node scripts/lite/build.mjs", "lite:serve": "node scripts/lite/serve.mjs out 4173", "test:lite-smoke": "playwright test -c scripts/lite/smoke/playwright.config.ts", diff --git a/scripts/standalone-guard.mjs b/scripts/standalone-guard.mjs new file mode 100644 index 000000000..80aaff1de --- /dev/null +++ b/scripts/standalone-guard.mjs @@ -0,0 +1,165 @@ +#!/usr/bin/env node +// Keeps a checkout's private state out of .next/standalone. +// +// The trace follows fs reads, and the server reads its state directories +// relative to cwd, so a build from a checkout that has been run copies the +// admin password hash, the plugin signing key, the audit log and .env secrets +// into the standalone tree, and from there into whatever ships it. +// outputFileTracingExcludes doesn't stop this: +// - Turbopack never applies it to the middleware and instrumentation traces, +// which cover the whole project, and misses "!(next-server)" per route. +// - Webpack builds the exclude globs with path.join, so on Windows they hold +// backslashes and match nothing. +// - Next copies .env and .env.production into standalone on its own. +// So next.config.ts prunes the traces right after compiling (pruneTraces), and +// every build ends with `node scripts/standalone-guard.mjs`, which removes the +// .env copies and fails if anything forbidden is left. + +import { existsSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +/** Top-level project entries the running server needs. Traces drop the rest. */ +const TRACE_ALLOWED = new Set(['node_modules', '.next', 'public', 'locales', 'package.json', 'VERSION']); + +/** Never in a standalone tree, wherever it sits outside node_modules. */ +const FORBIDDEN_DIRS = new Set(['data', 'local-data', 'repos']); +const FORBIDDEN_FILE = /^(\.env.*|plugin-signing\.key)$/; + +function toPosix(p) { + return p.split(path.sep).join('/'); +} + +/** Whether a traced file belongs in the build. `abs` is its absolute path. */ +export function isTraceFileAllowed(abs, projectDir) { + const rel = toPosix(path.relative(projectDir, abs)); + if (rel === '') return true; + const segments = rel.split('/'); + // Outside the project: a junctioned or hoisted node_modules, nothing else. + if (segments[0] === '..' || path.isAbsolute(rel)) return segments.includes('node_modules'); + return TRACE_ALLOWED.has(segments[0]); +} + +function* walk(dir) { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) yield* walk(full); + else yield full; + } +} + +/** + * Drops files from a trace in place. Turbopack's fileHashes run parallel to + * files and its symlinks point at file indexes, so those move along. + */ +function pruneTraceList(list, dir, projectDir) { + const indexMap = new Map(); + const files = []; + const fileHashes = Array.isArray(list.fileHashes) ? [] : undefined; + list.files.forEach((entry, index) => { + if (!isTraceFileAllowed(path.resolve(dir, entry), projectDir)) return; + indexMap.set(index, files.length); + files.push(entry); + fileHashes?.push(list.fileHashes[index]); + }); + const dropped = list.files.length - files.length; + if (dropped === 0) return 0; + list.files = files; + if (fileHashes) list.fileHashes = fileHashes; + if (Array.isArray(list.symlinks)) { + list.symlinks = list.symlinks + .filter(([index]) => indexMap.has(index)) + .map(([index, ...rest]) => [indexMap.get(index), ...rest]); + } + return dropped; +} + +/** Rewrites every *.nft.json under distDir without the files that must not ship. */ +export function pruneTraces({ distDir, projectDir }) { + let removed = 0; + for (const file of walk(distDir)) { + if (!file.endsWith('.nft.json')) continue; + // standalone holds copies from an earlier build, cache holds Next's own. + if (/[\\/]\.next[\\/](standalone|cache|dev)[\\/]/.test(file)) continue; + let trace; + try { + trace = JSON.parse(readFileSync(file, 'utf8')); + } catch { + // Webpack's trace worker may be rewriting it. The final check still runs. + continue; + } + if (!Array.isArray(trace.files)) continue; + const dropped = pruneTraceList(trace, path.dirname(file), projectDir); + if (dropped === 0) continue; + removed += dropped; + const tmp = `${file}.${process.pid}.tmp`; + writeFileSync(tmp, JSON.stringify(trace)); + renameSync(tmp, file); + } + return removed; +} + +/** Everything forbidden in a standalone tree, as paths relative to it. */ +export function findForbidden(standaloneDir) { + const found = []; + const visit = (dir) => { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + // Packages may hold anything, and .next/server has routes named after data. + if (entry.name === 'node_modules' || entry.name === '.next') continue; + if (FORBIDDEN_DIRS.has(entry.name)) found.push(toPosix(path.relative(standaloneDir, full)) + '/'); + else visit(full); + } else if (FORBIDDEN_FILE.test(entry.name)) { + found.push(toPosix(path.relative(standaloneDir, full))); + } + } + }; + visit(standaloneDir); + return found.sort(); +} + +/** Removes the .env files Next copies next to server.js on its own. */ +export function removeCopiedEnvFiles(standaloneDir) { + const removed = []; + const visit = (dir) => { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (entry.isDirectory() && entry.name !== 'node_modules' && entry.name !== '.next') { + visit(path.join(dir, entry.name)); + } + } + if (!existsSync(path.join(dir, 'server.js'))) return; + for (const name of ['.env', '.env.production']) { + const file = path.join(dir, name); + if (existsSync(file)) { + rmSync(file); + removed.push(toPosix(path.relative(standaloneDir, file))); + } + } + }; + visit(standaloneDir); + return removed; +} + +/** Cleans and checks a standalone tree. Returns what is still wrong with it. */ +export function guardStandalone(standaloneDir) { + removeCopiedEnvFiles(standaloneDir); + return findForbidden(standaloneDir); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const standaloneDir = path.resolve(process.argv[2] ?? '.next/standalone'); + if (!existsSync(standaloneDir)) { + console.error(`[standalone-guard] ${standaloneDir} does not exist`); + process.exit(1); + } + const forbidden = guardStandalone(standaloneDir); + if (forbidden.length > 0) { + console.error(`[standalone-guard] private files in ${standaloneDir}:`); + for (const file of forbidden.slice(0, 50)) console.error(` ${file}`); + if (forbidden.length > 50) console.error(` ... and ${forbidden.length - 50} more`); + console.error('[standalone-guard] do not ship this build. See scripts/standalone-guard.mjs.'); + process.exit(1); + } + console.log('[standalone-guard] ok'); +}