feat(cli): opt-in per-script codebase digests (per_script_digest) (#12267)

* feat(cli): opt-in per-script codebase digests from each script's imports

* fix(cli): digest only the tsconfig esbuild applies, with its extends chain

* perf(cli): skip codebase digests in the lockfile dedupe pass

* fix(cli): digest package.json build fields and package-resolved tsconfig bases

* fix(cli): resolve tsconfig extends exactly as esbuild does, BOM included

* fix(cli): fall through unmatched nested export conditions in tsconfig extends

* refactor(cli): hash every config of an extended tsconfig package

* fix(cli): hash every file of an extended tsconfig package, through symlinks

* style(cli): escape the BOM and sort package files in the digest

* docs(cli): note runtime-read files in the per_script_digest description
This commit is contained in:
Ruben Fiszel
2026-11-06 13:32:55 +01:00
committed by GitHub
parent 8a53808622
commit 706d1763f1
9 changed files with 818 additions and 49 deletions
+1 -1
View File
@@ -143,7 +143,7 @@ export async function scriptDigest(
// from the metadata file. Only bundling tells whether push adds `.tar`, so a pulled
// value that is this digest plus `.tar` is taken as is.
const codebase = language == "bun" ? findCodebase(contentFile, codebases) : undefined;
let codebaseDigest = codebase ? await codebase.getDigest() : undefined;
let codebaseDigest = codebase ? await codebase.getDigest(contentFile) : undefined;
if (codebaseDigest && meta?.codebase === codebaseDigest + ".tar") {
codebaseDigest = meta.codebase;
}
+2
View File
@@ -144,6 +144,7 @@ export const CONFIG_REFERENCE: ConfigOption[] = [
properties: {
relative_path: { type: "string", description: "Path to the codebase directory" },
extra_digest_paths: { type: "array", items: { type: "string" }, description: "Paths outside relative_path that scripts import, relative to wmill.yaml; hashed recursively" },
per_script_digest: { type: "boolean", description: "Redeploy a script only when a file it bundles changes, instead of on any change under relative_path. Each push bundles every script of the codebase once to find its inputs, so every one of them must bundle. Files a script reads at runtime rather than imports are not covered: list them in extra_digest_paths" },
includes: { type: "array", items: { type: "string" }, description: "Glob patterns for files to include in bundle" },
excludes: { type: "array", items: { type: "string" }, description: "Glob patterns for files to exclude from bundle" },
format: { type: "string", enum: ["cjs", "esm"], description: "Bundle output format" },
@@ -164,6 +165,7 @@ export const CONFIG_REFERENCE: ConfigOption[] = [
"# codebases:",
'# - relative_path: ./shared # path to the codebase',
'# # extra_digest_paths: ["../packages/utils/src"] # imported paths outside relative_path, hashed recursively (skip node_modules/dist)',
"# # per_script_digest: true # redeploy a script only when a file it bundles changes",
'# includes: ["**/*.ts"] # files to include in bundle',
'# excludes: ["node_modules/**"] # files to exclude',
'# format: esm # bundle format: "cjs" or "esm"',
+10 -7
View File
@@ -63,6 +63,7 @@ import {
} from "../../core/conf.ts";
import {
SyncCodebase,
isPerScriptDigest,
listSyncCodebases,
uncoveredBundleInputs,
} from "../../utils/codebase.ts";
@@ -451,11 +452,13 @@ export async function handleFile(
metafile: true,
}));
const endTime = performance.now();
const uncovered = uncoveredBundleInputs(
codebase,
path,
Object.keys(out.metafile?.inputs ?? {})
);
const uncovered = isPerScriptDigest(codebase)
? []
: uncoveredBundleInputs(
codebase,
path,
Object.keys(out.metafile?.inputs ?? {})
);
if (uncovered.length > 0) {
log.warnAlways(
`${path} bundles files outside codebase ${codebase.relative_path}; edits to them won't be detected as changes unless listed in extra_digest_paths: ${uncovered.join(", ")}`
@@ -569,7 +572,7 @@ export async function handleFile(
}
if (typed && codebase) {
typed.codebase = await codebase.getDigest(forceTar);
typed.codebase = await codebase.getDigest(path, forceTar);
}
// Scan for modules: folder layout (entry point inside __mod/) or flat layout
@@ -626,7 +629,7 @@ export async function handleFile(
debounce_args_to_accumulate: typed?.debounce_args_to_accumulate,
max_total_debouncing_time: typed?.max_total_debouncing_time,
max_total_debounces_amount: typed?.max_total_debounces_amount,
codebase: await codebase?.getDigest(forceTar),
codebase: await codebase?.getDigest(path, forceTar),
timeout: nonePositiveInt(typed?.timeout),
// 0 means "delete immediately after completion", so it must survive as 0
// rather than being folded into "unset" the way the positive-only settings are.
+42 -3
View File
@@ -124,6 +124,7 @@ import {
import { Workspace } from "../workspace/workspace.ts";
import { removePathPrefix } from "../../types.ts";
import {
isPerScriptDigest,
listSyncCodebases,
sameCodebaseDigest,
SyncCodebase,
@@ -571,6 +572,39 @@ export function findCodebase(
}
}
/** Computes the per-script codebase digests of every local codebase script in one
* esbuild pass, before the diff asks for them one script at a time. */
async function primeCodebaseDigests(
codebases: SyncCodebase[],
ignore: (p: string, isDirectory: boolean) => boolean,
): Promise<void> {
if (!codebases.some(isPerScriptDigest)) return;
const scripts = new Map<SyncCodebase, string[]>();
const walk = async (dir: string) => {
for (const e of await readdir(dir || ".", { withFileTypes: true })) {
const p = dir ? path.join(dir, e.name) : e.name;
if (e.isDirectory()) {
if (e.name != "node_modules" && !e.name.startsWith(".")) await walk(p);
continue;
}
if (!e.name.endsWith(".script.yaml") || ignore(p, false)) continue;
const ts = p.replace(/\.script\.yaml$/, ".ts");
const c = findCodebase(ts, codebases);
if (!c || !isPerScriptDigest(c)) continue;
if (!(await stat(ts).then((s) => s.isFile(), () => false))) continue;
scripts.set(c, [...(scripts.get(c) ?? []), ts]);
}
};
await walk("");
for (const [c, paths] of scripts) {
const start = performance.now();
await c.primeDigests(paths);
log.info(
`Codebase ${c.relative_path}: digested ${paths.length} scripts in ${((performance.now() - start) / 1000).toFixed(1)}s`,
);
}
}
/** Script metadata that differs only by the `.tar` a multi-output bundle adds to the
* deployed codebase digest. */
function sameScriptMetadataModuloTar(a: any, b: any): boolean {
@@ -618,7 +652,7 @@ async function addCodebaseDigestIfRelevant(
if (ignoreCodebaseChanges) {
parsed["codebase"] = undefined;
} else {
parsed["codebase"] = await c.getDigest();
parsed["codebase"] = await c.getDigest(replacedPath);
}
parsed["lock"] = "";
return yamlStringify(parsed, yamlOptions);
@@ -3026,7 +3060,7 @@ export async function compareDynFSElement(
continue;
}
const c = findCodebase(tsFile, codebases);
if (!sameCodebaseDigest(await c?.getDigest(), v)) {
if (!sameCodebaseDigest(await c?.getDigest(tsFile), v)) {
changes.push({
name: "edited",
path: tsFile,
@@ -4493,8 +4527,10 @@ export async function dedupeLockfilesOnDisk(args: {
failed = [],
dryRun,
} = args;
// The plan reads only locks, so codebase digests (a bundle pass per script
// under `per_script_digest`) are not computed for it.
const map = await elementsToMap(
await FSFSElement(process.cwd(), codebases, false),
await FSFSElement(process.cwd(), codebases, true),
ignore,
opts.json ?? false,
opts,
@@ -5088,6 +5124,9 @@ export async function push(
localFlowInlineMapping,
);
if (!opts.skipScripts) {
await primeCodebaseDigests(codebases, await ignoreF(opts));
}
const local = await FSFSElement(
path.join(process.cwd(), ""),
codebases,
+1
View File
@@ -120,6 +120,7 @@ export interface SyncOptions {
export interface Codebase {
relative_path: string;
extra_digest_paths?: string[];
per_script_digest?: boolean;
includes?: string[];
excludes?: string[];
assets?: {
+441 -36
View File
@@ -1,7 +1,8 @@
import { Codebase, SyncOptions } from "../core/conf.ts";
import * as log from "../core/log.ts";
import { digestDir, generateHash, generateHashFromBuffer } from "./utils.ts";
import { readFile, stat } from "node:fs/promises";
import { withEsbuild } from "./esbuild_loader.ts";
import { readdir, readFile, realpath, stat } from "node:fs/promises";
import path from "node:path";
// A bare string is accepted, like `includes`/`excludes` in findCodebase.
@@ -40,8 +41,18 @@ export function uncoveredBundleInputs(
});
}
/** Whether a codebase digests each script from its own bundle inputs. A custom
* bundler's inputs are unknown, so it keeps the directory digest. */
export function isPerScriptDigest(codebase: Codebase): boolean {
return codebase.per_script_digest === true && !codebase.customBundler;
}
export type SyncCodebase = Codebase & {
getDigest: (forceTar?: boolean) => Promise<string>;
/** The `codebase` value a push stores for the script at `scriptPath`. */
getDigest: (scriptPath: string, forceTar?: boolean) => Promise<string>;
/** Computes the digests of many scripts in one esbuild pass. Only per-script
* digest codebases do any work. */
primeDigests: (scriptPaths: string[]) => Promise<void>;
};
/** Whether two codebase digests name the same bundle. A push adds `.tar` when esbuild emits
@@ -67,43 +78,437 @@ export function listSyncCodebases(options: SyncOptions): SyncCodebase[] {
.join(", ")}`
);
}
const hasher = new FileHasher();
for (const codebase of options?.codebases ?? []) {
let _digest: string | undefined = undefined;
let alreadyPrinted = false;
let hasAssets = false;
const getDigest: (forceTar?: boolean) => Promise<string> = async (
forceTar?: boolean
) => {
if (_digest == undefined) {
// Concurrent callers read `_digest` across these awaits: assign it only once complete.
let digest = await digestDir(
codebase.relative_path,
JSON.stringify(codebase)
);
const extra = extraDigestPaths(codebase);
if (extra.length > 0) {
const hashes = await Promise.all(extra.map(digestPath));
digest = await generateHash(digest + hashes.join(""));
}
if (codebase.format == "esm") {
digest += ".esm";
}
_digest = digest;
if (!alreadyPrinted) {
alreadyPrinted = true;
log.info(`Codebase ${codebase.relative_path}, digest: ${_digest}`);
}
hasAssets =
Array.isArray(codebase.assets) && codebase.assets.length > 0;
}
if (forceTar || hasAssets) {
return _digest + ".tar";
} else {
return _digest;
}
const hasAssets =
Array.isArray(codebase.assets) && codebase.assets.length > 0;
const withSuffixes = (digest: string, forceTar?: boolean) => {
if (codebase.format == "esm") digest += ".esm";
return forceTar || hasAssets ? digest + ".tar" : digest;
};
res.push({ ...codebase, getDigest });
if (!isPerScriptDigest(codebase)) {
let dirDigest: Promise<string> | undefined;
const getDigest = async (_scriptPath: string, forceTar?: boolean) => {
if (!dirDigest) {
dirDigest = (async () => {
let digest = await digestDir(
codebase.relative_path,
JSON.stringify(codebase)
);
const extra = extraDigestPaths(codebase);
if (extra.length > 0) {
const hashes = await Promise.all(extra.map(digestPath));
digest = await generateHash(digest + hashes.join(""));
}
log.info(
`Codebase ${codebase.relative_path}, digest: ${withSuffixes(digest)}`
);
return digest;
})();
}
return withSuffixes(await dirDigest, forceTar);
};
res.push({ ...codebase, getDigest, primeDigests: async () => {} });
continue;
}
const digests = new Map<string, Promise<string>>();
const primeDigests = async (scriptPaths: string[]) => {
const missing = [
...new Set(scriptPaths.map((p) => path.resolve(p))),
].filter((p) => !digests.has(p));
if (missing.length == 0) return;
const computed = scriptDigests(codebase, missing, hasher);
for (const p of missing) {
const d = computed.then((m) => {
const v = m.get(p)!;
if (v instanceof Error) throw v;
return v;
});
// A primed script nobody asks for must not fail the run.
d.catch(() => {});
digests.set(p, d);
}
await Promise.allSettled(missing.map((p) => digests.get(p)));
};
const getDigest = async (scriptPath: string, forceTar?: boolean) => {
const abs = path.resolve(scriptPath);
if (!digests.has(abs)) await primeDigests([abs]);
return withSuffixes(await digests.get(abs)!, forceTar);
};
res.push({ ...codebase, getDigest, primeDigests });
}
return res;
}
/** Memoized content hashes, shared by every codebase of a run. */
class FileHasher {
private hashes = new Map<string, Promise<string>>();
private configs = new Map<string, Promise<string[]>>();
private packages = new Map<string, Promise<string | undefined>>();
private packageFields = new Map<string, Promise<string>>();
/** The nearest package.json at or above `dir`. */
packageFor(dir: string): Promise<string | undefined> {
let p = this.packages.get(dir);
if (!p) {
p = (async () => {
const here = path.join(dir, "package.json");
if (await isFile(here)) return here;
const parent = path.dirname(dir);
return parent == dir ? undefined : await this.packageFor(parent);
})();
this.packages.set(dir, p);
}
return p;
}
/** A hash of the build-relevant fields of a package.json, or of the whole file
* when it does not parse. */
hashPackageFields(pkgPath: string): Promise<string> {
let h = this.packageFields.get(pkgPath);
if (!h) {
h = (async () => {
const pkg = await readJson(pkgPath);
if (!pkg || typeof pkg != "object") return await this.hash(pkgPath);
const fields = BUILD_PACKAGE_FIELDS.map((f) => [f, pkg[f]]);
return await generateHash(JSON.stringify(fields));
})();
this.packageFields.set(pkgPath, h);
}
return h;
}
hash(absPath: string): Promise<string> {
let h = this.hashes.get(absPath);
if (!h) {
h = readFile(absPath).then(generateHashFromBuffer);
this.hashes.set(absPath, h);
}
return h;
}
/** The TS config esbuild applies to files in `dir`, as it picks it: the nearest
* `tsconfig.json`, else `jsconfig.json`, in `dir` or an ancestor, followed by the
* configs it `extends`. Ancestors it shadows are left out. */
configsFor(dir: string): Promise<string[]> {
let c = this.configs.get(dir);
if (!c) {
c = (async () => {
for (const name of ["tsconfig.json", "jsconfig.json"]) {
const p = path.join(dir, name);
if (await isFile(p)) return await this.extendsChain(p, new Set());
}
const parent = path.dirname(dir);
return parent == dir ? [] : await this.configsFor(parent);
})();
this.configs.set(dir, c);
}
return c;
}
private async extendsChain(config: string, seen: Set<string>): Promise<string[]> {
if (seen.has(config)) return [];
seen.add(config);
const res = [config];
for (const spec of tsconfigExtends(await readFile(config, "utf8"))) {
const dir = path.dirname(config);
const bases = isPackageSpec(spec)
? await packageConfigFiles(spec, dir)
: [await resolveRelativeExtends(spec, dir)].filter((b) => b != undefined);
for (const base of bases) res.push(...(await this.extendsChain(base, seen)));
}
return res;
}
}
const isPackageSpec = (spec: string) =>
!/^\.{1,2}(\/|\\|$)/.test(spec) && !path.isAbsolute(spec);
/** A path `extends`, as esbuild 0.28 resolves it: relative to the config, with
* `.json` appended only when it lacks it, and `.`/`..` meaning their tsconfig.json. */
async function resolveRelativeExtends(spec: string, dir: string): Promise<string | undefined> {
const p = path.resolve(dir, spec == "." || spec == ".." ? spec + "/tsconfig.json" : spec);
if (await isFile(p)) return p;
return !p.endsWith(".json") && (await isFile(p + ".json")) ? p + ".json" : undefined;
}
/**
* Every config a package `extends` could resolve to: every file of the package in
* each `node_modules` above the config, whatever its extension (an `exports` target
* can be `.jsonc` or extensionless) and through symlinks (workspace packages), plus
* the file its `tsconfig` field names. esbuild picks one through the package's
* `exports` conditions, `tsconfig` field and subpath; hashing them all covers its
* pick without reimplementing that resolution, at the cost of redeploying when any
* file of the package changes.
*/
async function packageConfigFiles(spec: string, dir: string): Promise<string[]> {
const name = spec.split("/").slice(0, spec.startsWith("@") ? 2 : 1).join("/");
const res: string[] = [];
for (let d = dir; ; d = path.dirname(d)) {
const pkgDir = path.join(d, "node_modules", name);
if (await stat(pkgDir).then((s) => s.isDirectory(), () => false)) {
res.push(...(await filesUnder(pkgDir, new Set())));
const field = (await readJson(path.join(pkgDir, "package.json")))?.tsconfig;
if (typeof field == "string" && field != "") {
const p = path.resolve(pkgDir, field);
if (await isFile(p)) res.push(p);
}
}
if (path.dirname(d) == d) return [...new Set(res)].sort();
}
}
/** The files under `dir`, following symlinks, without nested `node_modules`. */
async function filesUnder(dir: string, seenDirs: Set<string>): Promise<string[]> {
const real = await realpath(dir);
if (seenDirs.has(real)) return [];
seenDirs.add(real);
const res: string[] = [];
for (const e of (await readdir(dir)).sort()) {
const p = path.join(dir, e);
const s = await stat(p).catch(() => undefined);
if (s?.isDirectory() && e != "node_modules") res.push(...(await filesUnder(p, seenDirs)));
else if (s?.isFile()) res.push(p);
}
return res;
}
const isFile = (p: string) => stat(p).then((s) => s.isFile(), () => false);
/** The `extends` of a tsconfig, which is JSON with comments and trailing commas.
* An unparseable config extends nothing; its own content is still hashed. */
export function tsconfigExtends(text: string): string[] {
text = text.replace(/^\uFEFF/, "");
let json = "";
for (let i = 0; i < text.length; i++) {
const ch = text[i];
if (ch == '"') {
const start = i;
for (i++; i < text.length && text[i] != '"'; i++) if (text[i] == "\\") i++;
json += text.slice(start, i + 1);
} else if (ch == "/" && text[i + 1] == "/") {
while (i < text.length && text[i] != "\n") i++;
json += "\n";
} else if (ch == "/" && text[i + 1] == "*") {
i = text.indexOf("*/", i + 2);
if (i < 0) break;
i++;
} else {
json += ch;
}
}
try {
const ext = JSON.parse(json.replace(/,(\s*[}\]])/g, "$1"))?.extends;
return ([] as unknown[]).concat(ext ?? []).filter((e): e is string => typeof e == "string");
} catch {
return [];
}
}
async function readJson(p: string): Promise<any> {
try {
return JSON.parse((await readFile(p, "utf8")).replace(/^\uFEFF/, ""));
} catch {
return undefined;
}
}
/** The fields of a package.json that change how esbuild compiles or tree-shakes the
* files below it without changing which files it bundles. Hashed instead of the
* whole file, so adding a dependency does not redeploy every script; a dependency's
* new version shows up in its bundled files. */
const BUILD_PACKAGE_FIELDS = ["sideEffects", "type", "main", "module", "browser", "exports", "imports"];
/** A file input of an esbuild metafile, as opposed to "<define:x>", "(disabled):x"
* or a plugin namespace. */
function isFileInput(key: string): boolean {
return !/^[<(]|^[a-z-]{2,}:/i.test(key);
}
const toPosix = (p: string) => p.split(path.sep).join("/");
/**
* Per-script digests: each covers the files the script's bundle is built from
* (the transitive imports esbuild resolves, `node_modules` included, by path and
* content), the TS configs and package.json build fields that apply to them, the
* codebase's assets and
* `extra_digest_paths`, and the codebase config.
*
* It follows the import graph rather than the bundle output: an output's input list
* drops files whose values are inlined (TS enums), and hashing output would make
* every digest depend on esbuild's code generation. Paths are relative to the
* working directory, the sync checkout root, so a script digests the same alone and
* in a batch, on any machine.
*/
async function scriptDigests(
codebase: Codebase,
scriptPaths: string[],
hasher: FileHasher
): Promise<Map<string, string | Error>> {
const cwd = await realpath(process.cwd());
const parts = [JSON.stringify(codebase)];
for (const p of extraDigestPaths(codebase)) parts.push(await digestPath(p));
for (const a of codebase.assets ?? []) {
parts.push(`${a.to}\0${await hasher.hash(path.resolve(a.from))}`);
}
const shared = parts.join("\n");
const closures = await importClosures(codebase, scriptPaths, cwd);
const res = new Map<string, string | Error>();
await Promise.all(
scriptPaths.map(async (script) => {
const files = closures.get(script)!;
if (files instanceof Error) {
res.set(script, files);
return;
}
const configs = new Set<string>();
const packages = new Set<string>();
for (const f of files) {
const pkg = await hasher.packageFor(path.dirname(f));
if (pkg) packages.add(pkg);
if (f.split(path.sep).includes("node_modules")) continue;
for (const c of await hasher.configsFor(path.dirname(f))) {
configs.add(c);
}
}
const rel = (f: string) => toPosix(path.relative(cwd, f));
const entries = await Promise.all([
...[...files, ...configs].map(async (f) => `${rel(f)}\0${await hasher.hash(f)}`),
...[...packages].map(
async (p) => `${rel(p)}\0fields\0${await hasher.hashPackageFields(p)}`
),
]);
entries.sort();
res.set(
script,
await generateHash(shared + "\n" + entries.join("\n"))
);
})
);
return res;
}
/** Each script's bundle inputs as real absolute paths, the script itself included.
* esbuild reports real paths, so `cwd` and the entries are resolved the same way: a
* checkout under a symlink would otherwise find no inputs for its scripts. */
async function importClosures(
codebase: Codebase,
scriptPaths: string[],
cwd: string
): Promise<Map<string, Set<string> | Error>> {
const res = new Map<string, Set<string> | Error>();
const entries = new Map<string, string>();
for (const s of scriptPaths) {
try {
entries.set(s, await realpath(s));
} catch (e) {
res.set(s, bundleError(s, e));
}
}
const inject = await Promise.all(
(codebase.inject ?? []).map((p) => realpath(path.resolve(p)))
);
const graph = async (entries: string[], batched: boolean) => {
const out = await withEsbuild((esbuild) =>
esbuild.build({
...graphBuildOptions(codebase, batched),
entryPoints: entries,
absWorkingDir: cwd,
})
);
const imports = new Map<string, string[]>();
for (const [key, input] of Object.entries(out.metafile?.inputs ?? {})) {
if (!isFileInput(key)) continue;
imports.set(
path.resolve(cwd, key),
input.imports
.filter((i) => !i.external && isFileInput(i.path))
.map((i) => path.resolve(cwd, i.path))
);
}
return imports;
};
const closure = (imports: Map<string, string[]>, script: string) => {
if (!imports.has(script)) {
throw new Error(`esbuild reported no inputs for ${script}`);
}
const seen = new Set<string>();
const todo = [script, ...inject];
while (todo.length > 0) {
const f = todo.pop()!;
if (seen.has(f)) continue;
seen.add(f);
todo.push(...(imports.get(f) ?? []));
}
return seen;
};
if (entries.size > 1) {
try {
const imports = await graph([...entries.values()], true);
for (const [s, real] of entries) res.set(s, closure(imports, real));
return res;
} catch (e) {
const first = (e as { errors?: { text: string; location?: { file: string } }[] })?.errors?.[0];
const reason = first
? `${first.location?.file ? first.location.file + ": " : ""}${first.text}`
: String(e).split("\n")[0];
log.info(
`Codebase digest: the batched build failed (${reason}), digesting ${entries.size} scripts one by one`
);
}
}
// One script that does not bundle fails the whole batch; only it should fail.
const queue = [...entries];
await Promise.all(
Array.from({ length: Math.min(8, queue.length) }, async () => {
for (let q = queue.shift(); q != undefined; q = queue.shift()) {
const [s, real] = q;
try {
res.set(s, closure(await graph([real], false), real));
} catch (e) {
res.set(s, bundleError(s, e));
}
}
})
);
return res;
}
function bundleError(script: string, e: unknown): Error {
const msg = e instanceof Error ? e.message : String(e);
return new Error(
`Cannot compute the codebase digest of ${toPosix(path.relative(process.cwd(), script))}: bundling it failed. ` +
`With per_script_digest every codebase script must bundle for a push to compare it; fix the script or exclude it from the codebase.\n${msg}`
);
}
/**
* The esbuild options of the digest pass. Only the options that decide which files
* are bundled matter (`external`, `inject`, `define`, which can drop a dead
* `require`, `loader`, `platform`, `packages`); the output is discarded. A batch is
* built with code splitting so that code shared by many scripts is generated once.
*/
function graphBuildOptions(codebase: Codebase, batched: boolean) {
return {
bundle: true,
write: false,
metafile: true,
logLevel: "silent" as const,
outdir: "/",
platform: "node" as const,
packages: "bundle" as const,
external: codebase.external,
inject: codebase.inject,
define: codebase.define,
loader: codebase.loader ?? { ".node": "file" },
target: "esnext",
...(batched
? { format: "esm" as const, splitting: true }
: { format: codebase.format ?? "cjs" }),
};
}
+1
View File
@@ -146,6 +146,7 @@ export async function resolvePreviewLocalScriptState(
codebase,
codebaseDigest: codebase
? await codebase.getDigest(
filePath,
Array.isArray(codebase.assets) && codebase.assets.length > 0
)
: undefined,
+316 -2
View File
@@ -1,9 +1,10 @@
import { expect, test } from "bun:test";
import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
import { mkdtemp, mkdir, realpath, rm, symlink, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { dirname, join } from "node:path";
import {
listSyncCodebases,
tsconfigExtends,
uncoveredBundleInputs,
} from "../src/utils/codebase.ts";
@@ -33,6 +34,319 @@ test("extra_digest_paths edits change the codebase digest", async () => {
}
});
async function perScriptFixture() {
const root = await mkdtemp(join(tmpdir(), "wmill-codebase-"));
await mkdir(join(root, "f/app/lib"), { recursive: true });
await mkdir(join(root, "packages/utils"), { recursive: true });
const files: Record<string, string> = {
"f/app/a.ts": "import { greet } from './lib/greet'; export function main() { return greet() }",
"f/app/a.spec.ts": "// a test, imported by nothing",
"f/app/b.ts": "import { Kind } from './lib/kind'; export function main() { return Kind.A }",
"f/app/c.ts": "import { u } from '../../packages/utils/u'; export function main() { return u }",
"f/app/lib/greet.ts": "export const greet = () => 'hi'",
"f/app/lib/kind.ts": "export enum Kind { A = 1 }",
"packages/utils/u.ts": "export const u = 1",
};
for (const [p, c] of Object.entries(files)) await writeFile(join(root, p), c);
const scripts = ["a", "b", "c"].map((s) => join(root, `f/app/${s}.ts`));
const codebase = () =>
listSyncCodebases({
codebases: [{ relative_path: join(root, "f/app"), per_script_digest: true }],
} as any)[0];
const digests = async () => {
const c = codebase();
await c.primeDigests(scripts);
return Promise.all(scripts.map((s) => c.getDigest(s)));
};
return { root, scripts, codebase, digests };
}
test("per_script_digest changes only the digests of scripts that bundle the edited file", async () => {
const { root, digests } = await perScriptFixture();
try {
const [a, b, c] = await digests();
expect(new Set([a, b, c]).size).toBe(3);
await writeFile(join(root, "f/app/a.spec.ts"), "// edited");
expect(await digests()).toEqual([a, b, c]);
await writeFile(join(root, "f/app/lib/greet.ts"), "export const greet = () => 'hello'");
const [a2, b2, c2] = await digests();
expect(a2).not.toBe(a);
expect([b2, c2]).toEqual([b, c]);
// The enum's value is inlined into b's bundle, which then no longer lists kind.ts.
await writeFile(join(root, "f/app/lib/kind.ts"), "export enum Kind { A = 2 }");
const [, b3] = await digests();
expect(b3).not.toBe(b);
// Imports outside relative_path are covered without extra_digest_paths.
await writeFile(join(root, "packages/utils/u.ts"), "export const u = 2");
const [, , c4] = await digests();
expect(c4).not.toBe(c);
} finally {
await rm(root, { recursive: true, force: true });
}
});
test("per_script_digest is the same for a script digested alone or in a batch", async () => {
const { root, scripts, codebase, digests } = await perScriptFixture();
try {
const batched = await digests();
const alone = await Promise.all(scripts.map((s) => codebase().getDigest(s)));
expect(alone).toEqual(batched);
} finally {
await rm(root, { recursive: true, force: true });
}
});
test("per_script_digest follows the imports of a script reached through a symlink", async () => {
const { root, scripts, codebase } = await perScriptFixture();
const link = root + "-link";
try {
await symlink(root, link);
const viaLink = join(link, "f/app/a.ts");
const before = await codebase().getDigest(viaLink);
expect(before).toBe(await codebase().getDigest(scripts[0]));
await writeFile(join(root, "f/app/lib/greet.ts"), "export const greet = () => 'hello'");
expect(await codebase().getDigest(viaLink)).not.toBe(before);
} finally {
await rm(link, { force: true });
await rm(root, { recursive: true, force: true });
}
});
test("per_script_digest covers the nearest tsconfig and what it extends, not shadowed ones", async () => {
const { root, digests } = await perScriptFixture();
try {
await writeFile(join(root, "tsconfig.json"), "{}");
await writeFile(join(root, "tsconfig.base.json"), `{ "compilerOptions": {} }`);
await writeFile(
join(root, "f/app/tsconfig.json"),
`{\n // shared settings\n "extends": "../../tsconfig.base.json", /* jsonc */\n}`
);
const before = await digests();
// f/app's own tsconfig shadows the root one; only c, which also imports
// packages/utils, has a file the root tsconfig applies to.
await writeFile(join(root, "tsconfig.json"), `{ "compilerOptions": { "strict": true } }`);
const shadowed = await digests();
expect(shadowed.slice(0, 2)).toEqual(before.slice(0, 2));
expect(shadowed[2]).not.toBe(before[2]);
await writeFile(join(root, "tsconfig.base.json"), `{ "compilerOptions": { "jsx": "react" } }`);
const extended = await digests();
expect(extended[0]).not.toBe(shadowed[0]);
expect(extended[1]).not.toBe(shadowed[1]);
} finally {
await rm(root, { recursive: true, force: true });
}
});
test("per_script_digest covers build fields of package.json, not its dependencies", async () => {
const { root, digests } = await perScriptFixture();
try {
await writeFile(join(root, "package.json"), `{ "name": "x", "dependencies": {} }`);
const before = await digests();
await writeFile(join(root, "package.json"), `{ "name": "x", "dependencies": { "pg": "8" } }`);
expect(await digests()).toEqual(before);
await writeFile(join(root, "package.json"), `{ "name": "x", "sideEffects": false }`);
const after = await digests();
expect(after.every((d, i) => d != before[i])).toBe(true);
} finally {
await rm(root, { recursive: true, force: true });
}
});
// Pins the digest's tsconfig coverage to what esbuild applies: in each layout, editing
// the base config esbuild applies must change both the bundle and the digest. The decoy
// is a config esbuild does not apply (its edit leaves the bundle alone), which proves
// the layout resolves where it claims to.
const ON = `{ "compilerOptions": { "useDefineForClassFields": true } }`;
const OFF = `{ "compilerOptions": { "useDefineForClassFields": false } }`;
const tsconfigLayouts: {
name: string;
files: Record<string, string>;
links?: Record<string, string>;
applied: string;
decoy?: string;
}[] = [
{
name: ".jsonc exports target",
files: {
"node_modules/cfg/package.json": JSON.stringify({ exports: { ".": "./base.jsonc" } }),
"node_modules/cfg/base.jsonc": ON,
"node_modules/cfg/tsconfig.json": ON,
"f/tsconfig.json": `{ "extends": "cfg" }`,
},
applied: "node_modules/cfg/base.jsonc",
decoy: "node_modules/cfg/tsconfig.json",
},
{
name: "extensionless subpath",
files: {
"node_modules/cfg/package.json": "{}",
"node_modules/cfg/strict": ON,
"f/tsconfig.json": `{ "extends": "cfg/strict" }`,
},
applied: "node_modules/cfg/strict",
},
{
name: "workspace package symlinked into node_modules",
files: {
"packages/cfg/package.json": "{}",
"packages/cfg/tsconfig.json": ON,
"f/tsconfig.json": `{ "extends": "cfg" }`,
},
links: { "node_modules/cfg": "../packages/cfg" },
applied: "packages/cfg/tsconfig.json",
},
{
name: "conditional exports",
files: {
"node_modules/cfg/package.json": JSON.stringify({ exports: { ".": { import: "./esm.json", require: "./cjs.json" } } }),
"node_modules/cfg/esm.json": ON,
"node_modules/cfg/cjs.json": ON,
"f/tsconfig.json": `{ "extends": "cfg" }`,
},
applied: "node_modules/cfg/cjs.json",
decoy: "node_modules/cfg/esm.json",
},
{
name: "nested unmatched condition",
files: {
"node_modules/cfg/package.json": JSON.stringify({ exports: { ".": { node: { import: "./esm.json" }, require: "./req.json" } } }),
"node_modules/cfg/esm.json": ON,
"node_modules/cfg/req.json": ON,
"f/tsconfig.json": `{ "extends": "cfg" }`,
},
applied: "node_modules/cfg/req.json",
decoy: "node_modules/cfg/esm.json",
},
{
name: "explicit null condition",
files: {
"node_modules/cfg/package.json": JSON.stringify({ exports: { ".": { node: null, require: "./req.json" } } }),
"node_modules/cfg/req.json": ON,
"node_modules/cfg/tsconfig.json": ON,
"f/tsconfig.json": `{ "extends": "cfg" }`,
},
applied: "node_modules/cfg/tsconfig.json",
decoy: "node_modules/cfg/req.json",
},
{
name: "module condition",
files: {
"node_modules/cfg/package.json": JSON.stringify({ exports: { ".": { module: "./module.json", require: "./req.json" } } }),
"node_modules/cfg/module.json": ON,
"node_modules/cfg/req.json": ON,
"f/tsconfig.json": `{ "extends": "cfg" }`,
},
applied: "node_modules/cfg/module.json",
decoy: "node_modules/cfg/req.json",
},
{
name: "invalid target in a fallback array",
files: {
"node_modules/cfg/package.json": JSON.stringify({ exports: { ".": ["invalid", "./req.json"] } }),
"node_modules/cfg/req.json": ON,
"node_modules/cfg/tsconfig.json": ON,
"f/tsconfig.json": `{ "extends": "cfg" }`,
},
applied: "node_modules/cfg/req.json",
decoy: "node_modules/cfg/tsconfig.json",
},
{
name: "overlapping wildcard exports",
files: {
"node_modules/cfg/package.json": JSON.stringify({ exports: { "./*": "./any/*.json", "./node/*": "./node/*.json" } }),
"node_modules/cfg/any/node/20.json": ON,
"node_modules/cfg/node/20.json": ON,
"f/tsconfig.json": `{ "extends": "cfg/node/20" }`,
},
applied: "node_modules/cfg/node/20.json",
decoy: "node_modules/cfg/any/node/20.json",
},
{
name: "package tsconfig field",
files: {
"node_modules/cfg/package.json": JSON.stringify({ tsconfig: "./configs/base.json" }),
"node_modules/cfg/configs/base.json": ON,
"node_modules/cfg/tsconfig.json": ON,
"f/tsconfig.json": `{ "extends": "cfg" }`,
},
applied: "node_modules/cfg/configs/base.json",
decoy: "node_modules/cfg/tsconfig.json",
},
{
name: "BOM-prefixed config, relative extends without .json",
files: { "base.json": ON, "f/tsconfig.json": `\uFEFF{ "extends": "../base" }` },
applied: "base.json",
},
];
for (const layout of tsconfigLayouts) {
test(`per_script_digest applies the tsconfig esbuild applies: ${layout.name}`, async () => {
const esbuild = await import("esbuild");
const root = await realpath(await mkdtemp(join(tmpdir(), "wmill-tsconfig-")));
const cwd = process.cwd();
try {
for (const [p, c] of Object.entries(layout.files)) {
await mkdir(dirname(join(root, p)), { recursive: true });
await writeFile(join(root, p), c);
}
for (const [p, target] of Object.entries(layout.links ?? {})) {
await mkdir(dirname(join(root, p)), { recursive: true });
await symlink(target, join(root, p));
}
// useDefineForClassFields decides whether the field shadows the accessor.
await writeFile(
join(root, "f/a.ts"),
"class A { x = 1; constructor() { Object.defineProperty(this, 'x', { get: () => 0 }) } }\nexport const main = () => new A().x\n"
);
process.chdir(root);
const bundle = async () =>
(await esbuild.build({ entryPoints: ["f/a.ts"], bundle: true, write: false, platform: "node", logLevel: "silent", absWorkingDir: root })).outputFiles[0].text;
const digest = () =>
listSyncCodebases({ codebases: [{ relative_path: "f", per_script_digest: true }] } as any)[0].getDigest("f/a.ts");
const [b0, d0] = [await bundle(), await digest()];
if (layout.decoy) {
await writeFile(join(root, layout.decoy), OFF);
expect(await bundle()).toBe(b0);
}
const d1 = await digest();
await writeFile(join(root, layout.applied), OFF);
expect(await bundle()).not.toBe(b0);
expect(await digest()).not.toBe(d1);
} finally {
process.chdir(cwd);
await rm(root, { recursive: true, force: true });
}
});
}
test("tsconfigExtends reads extends from JSON with comments", () => {
expect(tsconfigExtends(`{ "a": "x // not a comment", // c\n "extends": ["./a", "b"], }`)).toEqual(["./a", "b"]);
expect(tsconfigExtends(`{ /* "extends": "no" */ }`)).toEqual([]);
expect(tsconfigExtends(`not json`)).toEqual([]);
expect(tsconfigExtends(`\uFEFF{ "extends": "./base.json" }`)).toEqual(["./base.json"]);
});
test("per_script_digest fails only the script that does not bundle", async () => {
const { root, scripts, codebase } = await perScriptFixture();
try {
const before = await Promise.all(scripts.map((s) => codebase().getDigest(s)));
await writeFile(join(root, "f/app/b.ts"), "import { x } from './lib/missing'; export const main = () => x");
const c = codebase();
await c.primeDigests(scripts);
expect(await c.getDigest(scripts[0])).toBe(before[0]);
await expect(c.getDigest(scripts[1])).rejects.toThrow("f/app/b.ts");
expect(await c.getDigest(scripts[2])).toBe(before[2]);
} finally {
await rm(root, { recursive: true, force: true });
}
});
test("uncoveredBundleInputs reports only files outside the digested roots", () => {
const codebase = { relative_path: "f", extra_digest_paths: ["packages/a"] };
expect(
+4
View File
@@ -154,6 +154,10 @@
},
"description": "Paths outside relative_path that scripts import, relative to wmill.yaml; hashed recursively"
},
"per_script_digest": {
"type": "boolean",
"description": "Redeploy a script only when a file it bundles changes, instead of on any change under relative_path. Each push bundles every script of the codebase once to find its inputs, so every one of them must bundle. Files a script reads at runtime rather than imports are not covered: list them in extra_digest_paths"
},
"includes": {
"type": "array",
"items": {