mirror of
https://github.com/bulwarkmail/webmail.git
synced 2026-11-04 00:14:12 +00:00
fix: keep private files and secrets out of standalone builds
This commit is contained in:
Linus Rath
@@ -14,6 +14,7 @@ scripts/
|
||||
!scripts/build-pst-worker.mjs
|
||||
!scripts/healthcheck.mjs
|
||||
!scripts/sourcemaps.mjs
|
||||
!scripts/standalone-guard.mjs
|
||||
# Built in the image by scripts/build-plugin-sandbox.mjs
|
||||
public/plugin-sandbox-assets/
|
||||
TODO.md
|
||||
|
||||
+2
-1
@@ -36,7 +36,8 @@ RUN rm -rf "app/api/dev-jmap" && \
|
||||
-prune -exec rm -rf {} \; && \
|
||||
node scripts/build-pst-worker.mjs && \
|
||||
node scripts/build-plugin-sandbox.mjs && \
|
||||
npx next build --webpack
|
||||
npx next build --webpack && \
|
||||
node scripts/standalone-guard.mjs
|
||||
|
||||
# webpack has no debug IDs of its own, so sentry-cli adds them to the
|
||||
# chunks; then every map moves to /sourcemaps (exported with
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
// @vitest-environment node
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import {
|
||||
findForbidden,
|
||||
guardStandalone,
|
||||
isTraceFileAllowed,
|
||||
pruneTraces,
|
||||
} from '../../scripts/standalone-guard.mjs';
|
||||
|
||||
const ROOT = path.resolve(__dirname, '../..');
|
||||
const SCRIPT = path.join(ROOT, 'scripts/standalone-guard.mjs');
|
||||
const temps: string[] = [];
|
||||
|
||||
function tempDir(): string {
|
||||
const dir = mkdtempSync(path.join(os.tmpdir(), 'standalone-guard-'));
|
||||
temps.push(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
function put(root: string, rel: string, body = 'x'): void {
|
||||
const file = path.join(root, rel);
|
||||
mkdirSync(path.dirname(file), { recursive: true });
|
||||
writeFileSync(file, body);
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
while (temps.length) rmSync(temps.pop()!, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('standalone guard check', () => {
|
||||
it('fails the build on a planted signing key', () => {
|
||||
const dir = tempDir();
|
||||
put(dir, 'server.js');
|
||||
put(dir, 'data/admin/plugin-signing.key', 'FAKE');
|
||||
const run = spawnSync(process.execPath, [SCRIPT, dir], { encoding: 'utf8' });
|
||||
expect(run.status).toBe(1);
|
||||
expect(run.stderr).toContain('data/');
|
||||
});
|
||||
|
||||
it('passes a clean tree', () => {
|
||||
const dir = tempDir();
|
||||
put(dir, 'server.js');
|
||||
put(dir, 'package.json');
|
||||
put(dir, 'public/icon.png');
|
||||
put(dir, '.next/server/app/api/data/route.js');
|
||||
put(dir, 'node_modules/some-package/data/table.json');
|
||||
const run = spawnSync(process.execPath, [SCRIPT, dir], { encoding: 'utf8' });
|
||||
expect(run.stderr).toBe('');
|
||||
expect(run.status).toBe(0);
|
||||
});
|
||||
|
||||
it.each([
|
||||
'data/admin/admin.json',
|
||||
'local-data/notes.md',
|
||||
'repos/plugins/index.js',
|
||||
'.env.local',
|
||||
'lib/plugin-signing.key',
|
||||
])('finds %s', (rel) => {
|
||||
const dir = tempDir();
|
||||
put(dir, 'server.js');
|
||||
put(dir, rel);
|
||||
expect(findForbidden(dir)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('finds them when the project is nested under the tracing root', () => {
|
||||
const dir = tempDir();
|
||||
put(dir, 'worktrees/webmail/server.js');
|
||||
put(dir, 'worktrees/webmail/data/admin/plugin-signing.key');
|
||||
expect(findForbidden(dir)).toEqual(['worktrees/webmail/data/']);
|
||||
});
|
||||
|
||||
it('removes the .env files Next copies next to server.js, then passes', () => {
|
||||
const dir = tempDir();
|
||||
put(dir, 'worktrees/webmail/server.js');
|
||||
put(dir, 'worktrees/webmail/.env', 'SECRET=1');
|
||||
put(dir, 'worktrees/webmail/.env.production', 'SECRET=1');
|
||||
expect(guardStandalone(dir)).toEqual([]);
|
||||
expect(existsSync(path.join(dir, 'worktrees/webmail/.env'))).toBe(false);
|
||||
expect(existsSync(path.join(dir, 'worktrees/webmail/.env.production'))).toBe(false);
|
||||
});
|
||||
|
||||
it('does not remove a .env.local, which Next never copies, and reports it', () => {
|
||||
const dir = tempDir();
|
||||
put(dir, 'server.js');
|
||||
put(dir, '.env.local', 'SECRET=1');
|
||||
expect(guardStandalone(dir)).toEqual(['.env.local']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('trace pruning', () => {
|
||||
const project = path.resolve('/project');
|
||||
|
||||
it.each([
|
||||
['node_modules/next/dist/server.js', true],
|
||||
['.next/server/chunks/1.js', true],
|
||||
['public/icon.png', true],
|
||||
['package.json', true],
|
||||
['data/admin/plugin-signing.key', false],
|
||||
['local-data/notes.md', false],
|
||||
['repos/plugins/index.js', false],
|
||||
['.env', false],
|
||||
['.git/config', false],
|
||||
['lib/admin/paths.ts', false],
|
||||
])('%s allowed: %s', (rel, allowed) => {
|
||||
expect(isTraceFileAllowed(path.join(project, rel), project)).toBe(allowed);
|
||||
});
|
||||
|
||||
it('keeps a node_modules outside the project and nothing else from there', () => {
|
||||
expect(isTraceFileAllowed(path.resolve('/elsewhere/node_modules/next/x.js'), project)).toBe(true);
|
||||
expect(isTraceFileAllowed(path.resolve('/elsewhere/secrets.txt'), project)).toBe(false);
|
||||
});
|
||||
|
||||
it('rewrites a trace without the planted files', () => {
|
||||
const projectDir = tempDir();
|
||||
const distDir = path.join(projectDir, '.next');
|
||||
const trace = path.join(distDir, 'server/middleware.js.nft.json');
|
||||
put(
|
||||
projectDir,
|
||||
'.next/server/middleware.js.nft.json',
|
||||
JSON.stringify({
|
||||
version: 1,
|
||||
files: ['../../data/admin/plugin-signing.key', '../../.env', '../../node_modules/next/index.js', 'chunks/1.js'],
|
||||
}),
|
||||
);
|
||||
expect(pruneTraces({ distDir, projectDir })).toBe(2);
|
||||
expect(JSON.parse(readFileSync(trace, 'utf8'))).toEqual({
|
||||
version: 1,
|
||||
files: ['../../node_modules/next/index.js', 'chunks/1.js'],
|
||||
});
|
||||
});
|
||||
|
||||
it('keeps Turbopack hashes and symlink indexes on the right files', () => {
|
||||
const projectDir = tempDir();
|
||||
const distDir = path.join(projectDir, '.next');
|
||||
const trace = path.join(distDir, 'server/middleware.js.nft.json');
|
||||
put(
|
||||
projectDir,
|
||||
'.next/server/middleware.js.nft.json',
|
||||
JSON.stringify({
|
||||
version: 1,
|
||||
files: ['../../data/x', '../../node_modules/a', '../../repos/y', '../node_modules/esbuild-1', '../../data/link'],
|
||||
fileHashes: ['h0', 'h1', 'h2', 'h3', 'h4'],
|
||||
symlinks: [
|
||||
[3, '../../node_modules/esbuild'],
|
||||
[4, '../../outside'],
|
||||
],
|
||||
}),
|
||||
);
|
||||
expect(pruneTraces({ distDir, projectDir })).toBe(3);
|
||||
expect(JSON.parse(readFileSync(trace, 'utf8'))).toEqual({
|
||||
version: 1,
|
||||
files: ['../../node_modules/a', '../node_modules/esbuild-1'],
|
||||
fileHashes: ['h1', 'h3'],
|
||||
symlinks: [[1, '../../node_modules/esbuild']],
|
||||
});
|
||||
});
|
||||
|
||||
it('leaves unreadable traces alone instead of failing mid-build', () => {
|
||||
const projectDir = tempDir();
|
||||
put(projectDir, '.next/server/page.js.nft.json', '{ half written');
|
||||
expect(pruneTraces({ distDir: path.join(projectDir, '.next'), projectDir })).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('build wiring', () => {
|
||||
it('prunes after compiling and checks after every standalone build', () => {
|
||||
const config = readFileSync(path.join(ROOT, 'next.config.ts'), 'utf8');
|
||||
expect(config).toMatch(/runAfterProductionCompile[\s\S]*pruneTraces/);
|
||||
const pkg = JSON.parse(readFileSync(path.join(ROOT, 'package.json'), 'utf8'));
|
||||
expect(pkg.scripts.build).toMatch(/next build[^&]*&& node scripts\/standalone-guard\.mjs/);
|
||||
expect(readFileSync(path.join(ROOT, 'Dockerfile'), 'utf8')).toMatch(
|
||||
/next build --webpack[\s\S]{0,40}node scripts\/standalone-guard\.mjs/,
|
||||
);
|
||||
expect(readFileSync(path.join(ROOT, '.dockerignore'), 'utf8')).toContain('!scripts/standalone-guard.mjs');
|
||||
expect(readFileSync(path.join(ROOT, '.github/workflows/standalone-release.yml'), 'utf8')).toMatch(/run: npm run build/);
|
||||
});
|
||||
});
|
||||
+13
-4
@@ -4,6 +4,7 @@ import { execSync } from "child_process";
|
||||
import { createHash } from "crypto";
|
||||
import { readFileSync } from "fs";
|
||||
import { join } from "path";
|
||||
import { pruneTraces } from "./scripts/standalone-guard.mjs";
|
||||
|
||||
// Prefer an explicit build arg (passed in by CI / Docker, where .git is
|
||||
// excluded from the build context) and fall back to `git rev-parse` for
|
||||
@@ -154,10 +155,9 @@ const nextConfig: NextConfig = {
|
||||
// Sibling repos checked out under ./repos/ are unrelated source trees that
|
||||
// Turbopack's NFT can otherwise rope into the trace when dynamic fs calls
|
||||
// confuse it. Keeps the build from ballooning memory tracing dead code.
|
||||
// The trace follows fs reads, and the runtime reads its state directories,
|
||||
// so a build from a checkout that has been run would copy the admin
|
||||
// password hash, the plugin signing key, the audit log and .env secrets
|
||||
// into .next/standalone - and from there into the image.
|
||||
// These excludes are not what keeps private state out of the build: they
|
||||
// miss several traces. runAfterProductionCompile below and the final check
|
||||
// in scripts/standalone-guard.mjs do that.
|
||||
// Only "./repos" goes under "*": that key also feeds the next-server trace,
|
||||
// where Next matches each glob anywhere in the path, so "./data/**/*" would
|
||||
// drop node_modules/next/dist/lib/metadata/* and the server would not start.
|
||||
@@ -167,6 +167,15 @@ const nextConfig: NextConfig = {
|
||||
"*": ["./repos/**/*"],
|
||||
"!(next-server)": ["./data/**/*", "./local-data/**/*", "./.env*"],
|
||||
},
|
||||
...(isLite
|
||||
? {}
|
||||
: {
|
||||
compiler: {
|
||||
runAfterProductionCompile: async ({ distDir, projectDir }) => {
|
||||
pruneTraces({ distDir, projectDir });
|
||||
},
|
||||
},
|
||||
}),
|
||||
turbopack: {
|
||||
root: process.env.NEXT_TURBOPACK_ROOT || import.meta.dirname,
|
||||
...(sourceMaps ? { debugIds: true } : {}),
|
||||
|
||||
+1
-1
@@ -23,7 +23,7 @@
|
||||
],
|
||||
"scripts": {
|
||||
"dev": "node scripts/build-pst-worker.mjs && node scripts/build-plugin-sandbox.mjs --watch -- next dev --turbopack",
|
||||
"build": "node scripts/build-pst-worker.mjs && node scripts/build-plugin-sandbox.mjs && next build --turbopack",
|
||||
"build": "node scripts/build-pst-worker.mjs && node scripts/build-plugin-sandbox.mjs && next build --turbopack && node scripts/standalone-guard.mjs",
|
||||
"build:lite": "node scripts/lite/build.mjs",
|
||||
"lite:serve": "node scripts/lite/serve.mjs out 4173",
|
||||
"test:lite-smoke": "playwright test -c scripts/lite/smoke/playwright.config.ts",
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
#!/usr/bin/env node
|
||||
// Keeps a checkout's private state out of .next/standalone.
|
||||
//
|
||||
// The trace follows fs reads, and the server reads its state directories
|
||||
// relative to cwd, so a build from a checkout that has been run copies the
|
||||
// admin password hash, the plugin signing key, the audit log and .env secrets
|
||||
// into the standalone tree, and from there into whatever ships it.
|
||||
// outputFileTracingExcludes doesn't stop this:
|
||||
// - Turbopack never applies it to the middleware and instrumentation traces,
|
||||
// which cover the whole project, and misses "!(next-server)" per route.
|
||||
// - Webpack builds the exclude globs with path.join, so on Windows they hold
|
||||
// backslashes and match nothing.
|
||||
// - Next copies .env and .env.production into standalone on its own.
|
||||
// So next.config.ts prunes the traces right after compiling (pruneTraces), and
|
||||
// every build ends with `node scripts/standalone-guard.mjs`, which removes the
|
||||
// .env copies and fails if anything forbidden is left.
|
||||
|
||||
import { existsSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
|
||||
/** Top-level project entries the running server needs. Traces drop the rest. */
|
||||
const TRACE_ALLOWED = new Set(['node_modules', '.next', 'public', 'locales', 'package.json', 'VERSION']);
|
||||
|
||||
/** Never in a standalone tree, wherever it sits outside node_modules. */
|
||||
const FORBIDDEN_DIRS = new Set(['data', 'local-data', 'repos']);
|
||||
const FORBIDDEN_FILE = /^(\.env.*|plugin-signing\.key)$/;
|
||||
|
||||
function toPosix(p) {
|
||||
return p.split(path.sep).join('/');
|
||||
}
|
||||
|
||||
/** Whether a traced file belongs in the build. `abs` is its absolute path. */
|
||||
export function isTraceFileAllowed(abs, projectDir) {
|
||||
const rel = toPosix(path.relative(projectDir, abs));
|
||||
if (rel === '') return true;
|
||||
const segments = rel.split('/');
|
||||
// Outside the project: a junctioned or hoisted node_modules, nothing else.
|
||||
if (segments[0] === '..' || path.isAbsolute(rel)) return segments.includes('node_modules');
|
||||
return TRACE_ALLOWED.has(segments[0]);
|
||||
}
|
||||
|
||||
function* walk(dir) {
|
||||
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
||||
const full = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) yield* walk(full);
|
||||
else yield full;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Drops files from a trace in place. Turbopack's fileHashes run parallel to
|
||||
* files and its symlinks point at file indexes, so those move along.
|
||||
*/
|
||||
function pruneTraceList(list, dir, projectDir) {
|
||||
const indexMap = new Map();
|
||||
const files = [];
|
||||
const fileHashes = Array.isArray(list.fileHashes) ? [] : undefined;
|
||||
list.files.forEach((entry, index) => {
|
||||
if (!isTraceFileAllowed(path.resolve(dir, entry), projectDir)) return;
|
||||
indexMap.set(index, files.length);
|
||||
files.push(entry);
|
||||
fileHashes?.push(list.fileHashes[index]);
|
||||
});
|
||||
const dropped = list.files.length - files.length;
|
||||
if (dropped === 0) return 0;
|
||||
list.files = files;
|
||||
if (fileHashes) list.fileHashes = fileHashes;
|
||||
if (Array.isArray(list.symlinks)) {
|
||||
list.symlinks = list.symlinks
|
||||
.filter(([index]) => indexMap.has(index))
|
||||
.map(([index, ...rest]) => [indexMap.get(index), ...rest]);
|
||||
}
|
||||
return dropped;
|
||||
}
|
||||
|
||||
/** Rewrites every *.nft.json under distDir without the files that must not ship. */
|
||||
export function pruneTraces({ distDir, projectDir }) {
|
||||
let removed = 0;
|
||||
for (const file of walk(distDir)) {
|
||||
if (!file.endsWith('.nft.json')) continue;
|
||||
// standalone holds copies from an earlier build, cache holds Next's own.
|
||||
if (/[\\/]\.next[\\/](standalone|cache|dev)[\\/]/.test(file)) continue;
|
||||
let trace;
|
||||
try {
|
||||
trace = JSON.parse(readFileSync(file, 'utf8'));
|
||||
} catch {
|
||||
// Webpack's trace worker may be rewriting it. The final check still runs.
|
||||
continue;
|
||||
}
|
||||
if (!Array.isArray(trace.files)) continue;
|
||||
const dropped = pruneTraceList(trace, path.dirname(file), projectDir);
|
||||
if (dropped === 0) continue;
|
||||
removed += dropped;
|
||||
const tmp = `${file}.${process.pid}.tmp`;
|
||||
writeFileSync(tmp, JSON.stringify(trace));
|
||||
renameSync(tmp, file);
|
||||
}
|
||||
return removed;
|
||||
}
|
||||
|
||||
/** Everything forbidden in a standalone tree, as paths relative to it. */
|
||||
export function findForbidden(standaloneDir) {
|
||||
const found = [];
|
||||
const visit = (dir) => {
|
||||
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
||||
const full = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
// Packages may hold anything, and .next/server has routes named after data.
|
||||
if (entry.name === 'node_modules' || entry.name === '.next') continue;
|
||||
if (FORBIDDEN_DIRS.has(entry.name)) found.push(toPosix(path.relative(standaloneDir, full)) + '/');
|
||||
else visit(full);
|
||||
} else if (FORBIDDEN_FILE.test(entry.name)) {
|
||||
found.push(toPosix(path.relative(standaloneDir, full)));
|
||||
}
|
||||
}
|
||||
};
|
||||
visit(standaloneDir);
|
||||
return found.sort();
|
||||
}
|
||||
|
||||
/** Removes the .env files Next copies next to server.js on its own. */
|
||||
export function removeCopiedEnvFiles(standaloneDir) {
|
||||
const removed = [];
|
||||
const visit = (dir) => {
|
||||
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
||||
if (entry.isDirectory() && entry.name !== 'node_modules' && entry.name !== '.next') {
|
||||
visit(path.join(dir, entry.name));
|
||||
}
|
||||
}
|
||||
if (!existsSync(path.join(dir, 'server.js'))) return;
|
||||
for (const name of ['.env', '.env.production']) {
|
||||
const file = path.join(dir, name);
|
||||
if (existsSync(file)) {
|
||||
rmSync(file);
|
||||
removed.push(toPosix(path.relative(standaloneDir, file)));
|
||||
}
|
||||
}
|
||||
};
|
||||
visit(standaloneDir);
|
||||
return removed;
|
||||
}
|
||||
|
||||
/** Cleans and checks a standalone tree. Returns what is still wrong with it. */
|
||||
export function guardStandalone(standaloneDir) {
|
||||
removeCopiedEnvFiles(standaloneDir);
|
||||
return findForbidden(standaloneDir);
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
const standaloneDir = path.resolve(process.argv[2] ?? '.next/standalone');
|
||||
if (!existsSync(standaloneDir)) {
|
||||
console.error(`[standalone-guard] ${standaloneDir} does not exist`);
|
||||
process.exit(1);
|
||||
}
|
||||
const forbidden = guardStandalone(standaloneDir);
|
||||
if (forbidden.length > 0) {
|
||||
console.error(`[standalone-guard] private files in ${standaloneDir}:`);
|
||||
for (const file of forbidden.slice(0, 50)) console.error(` ${file}`);
|
||||
if (forbidden.length > 50) console.error(` ... and ${forbidden.length - 50} more`);
|
||||
console.error('[standalone-guard] do not ship this build. See scripts/standalone-guard.mjs.');
|
||||
process.exit(1);
|
||||
}
|
||||
console.log('[standalone-guard] ok');
|
||||
}
|
||||
Reference in New Issue
Block a user