From 706d1763f18901751a4b6401fae1cae893e8f3c4 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 6 Nov 2026 12:32:55 +0000 Subject: [PATCH] feat(cli): opt-in per-script codebase digests (per_script_digest) (#12267) * feat(cli): opt-in per-script codebase digests from each script's imports * fix(cli): digest only the tsconfig esbuild applies, with its extends chain * perf(cli): skip codebase digests in the lockfile dedupe pass * fix(cli): digest package.json build fields and package-resolved tsconfig bases * fix(cli): resolve tsconfig extends exactly as esbuild does, BOM included * fix(cli): fall through unmatched nested export conditions in tsconfig extends * refactor(cli): hash every config of an extended tsconfig package * fix(cli): hash every file of an extended tsconfig package, through symlinks * style(cli): escape the BOM and sort package files in the digest * docs(cli): note runtime-read files in the per_script_digest description --- cli/src/commands/digest/digest.ts | 2 +- cli/src/commands/init/template.ts | 2 + cli/src/commands/script/script.ts | 17 +- cli/src/commands/sync/sync.ts | 45 ++- cli/src/core/conf.ts | 1 + cli/src/utils/codebase.ts | 477 ++++++++++++++++++++++++-- cli/src/utils/local_path_scripts.ts | 1 + cli/test/codebase_digest_unit.test.ts | 318 ++++++++++++++++- cli/wmill.schema.json | 4 + 9 files changed, 818 insertions(+), 49 deletions(-) diff --git a/cli/src/commands/digest/digest.ts b/cli/src/commands/digest/digest.ts index c293505bba..02c4c4039d 100644 --- a/cli/src/commands/digest/digest.ts +++ b/cli/src/commands/digest/digest.ts @@ -143,7 +143,7 @@ export async function scriptDigest( // from the metadata file. Only bundling tells whether push adds `.tar`, so a pulled // value that is this digest plus `.tar` is taken as is. const codebase = language == "bun" ? findCodebase(contentFile, codebases) : undefined; - let codebaseDigest = codebase ? await codebase.getDigest() : undefined; + let codebaseDigest = codebase ? await codebase.getDigest(contentFile) : undefined; if (codebaseDigest && meta?.codebase === codebaseDigest + ".tar") { codebaseDigest = meta.codebase; } diff --git a/cli/src/commands/init/template.ts b/cli/src/commands/init/template.ts index 1f78c13148..6b5f220bff 100644 --- a/cli/src/commands/init/template.ts +++ b/cli/src/commands/init/template.ts @@ -144,6 +144,7 @@ export const CONFIG_REFERENCE: ConfigOption[] = [ properties: { relative_path: { type: "string", description: "Path to the codebase directory" }, extra_digest_paths: { type: "array", items: { type: "string" }, description: "Paths outside relative_path that scripts import, relative to wmill.yaml; hashed recursively" }, + per_script_digest: { type: "boolean", description: "Redeploy a script only when a file it bundles changes, instead of on any change under relative_path. Each push bundles every script of the codebase once to find its inputs, so every one of them must bundle. Files a script reads at runtime rather than imports are not covered: list them in extra_digest_paths" }, includes: { type: "array", items: { type: "string" }, description: "Glob patterns for files to include in bundle" }, excludes: { type: "array", items: { type: "string" }, description: "Glob patterns for files to exclude from bundle" }, format: { type: "string", enum: ["cjs", "esm"], description: "Bundle output format" }, @@ -164,6 +165,7 @@ export const CONFIG_REFERENCE: ConfigOption[] = [ "# codebases:", '# - relative_path: ./shared # path to the codebase', '# # extra_digest_paths: ["../packages/utils/src"] # imported paths outside relative_path, hashed recursively (skip node_modules/dist)', + "# # per_script_digest: true # redeploy a script only when a file it bundles changes", '# includes: ["**/*.ts"] # files to include in bundle', '# excludes: ["node_modules/**"] # files to exclude', '# format: esm # bundle format: "cjs" or "esm"', diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index 7921d4ef93..38eec70eeb 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -63,6 +63,7 @@ import { } from "../../core/conf.ts"; import { SyncCodebase, + isPerScriptDigest, listSyncCodebases, uncoveredBundleInputs, } from "../../utils/codebase.ts"; @@ -451,11 +452,13 @@ export async function handleFile( metafile: true, })); const endTime = performance.now(); - const uncovered = uncoveredBundleInputs( - codebase, - path, - Object.keys(out.metafile?.inputs ?? {}) - ); + const uncovered = isPerScriptDigest(codebase) + ? [] + : uncoveredBundleInputs( + codebase, + path, + Object.keys(out.metafile?.inputs ?? {}) + ); if (uncovered.length > 0) { log.warnAlways( `${path} bundles files outside codebase ${codebase.relative_path}; edits to them won't be detected as changes unless listed in extra_digest_paths: ${uncovered.join(", ")}` @@ -569,7 +572,7 @@ export async function handleFile( } if (typed && codebase) { - typed.codebase = await codebase.getDigest(forceTar); + typed.codebase = await codebase.getDigest(path, forceTar); } // Scan for modules: folder layout (entry point inside __mod/) or flat layout @@ -626,7 +629,7 @@ export async function handleFile( debounce_args_to_accumulate: typed?.debounce_args_to_accumulate, max_total_debouncing_time: typed?.max_total_debouncing_time, max_total_debounces_amount: typed?.max_total_debounces_amount, - codebase: await codebase?.getDigest(forceTar), + codebase: await codebase?.getDigest(path, forceTar), timeout: nonePositiveInt(typed?.timeout), // 0 means "delete immediately after completion", so it must survive as 0 // rather than being folded into "unset" the way the positive-only settings are. diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 2f28b63ffd..01ecce1f29 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -124,6 +124,7 @@ import { import { Workspace } from "../workspace/workspace.ts"; import { removePathPrefix } from "../../types.ts"; import { + isPerScriptDigest, listSyncCodebases, sameCodebaseDigest, SyncCodebase, @@ -571,6 +572,39 @@ export function findCodebase( } } +/** Computes the per-script codebase digests of every local codebase script in one + * esbuild pass, before the diff asks for them one script at a time. */ +async function primeCodebaseDigests( + codebases: SyncCodebase[], + ignore: (p: string, isDirectory: boolean) => boolean, +): Promise { + if (!codebases.some(isPerScriptDigest)) return; + const scripts = new Map(); + const walk = async (dir: string) => { + for (const e of await readdir(dir || ".", { withFileTypes: true })) { + const p = dir ? path.join(dir, e.name) : e.name; + if (e.isDirectory()) { + if (e.name != "node_modules" && !e.name.startsWith(".")) await walk(p); + continue; + } + if (!e.name.endsWith(".script.yaml") || ignore(p, false)) continue; + const ts = p.replace(/\.script\.yaml$/, ".ts"); + const c = findCodebase(ts, codebases); + if (!c || !isPerScriptDigest(c)) continue; + if (!(await stat(ts).then((s) => s.isFile(), () => false))) continue; + scripts.set(c, [...(scripts.get(c) ?? []), ts]); + } + }; + await walk(""); + for (const [c, paths] of scripts) { + const start = performance.now(); + await c.primeDigests(paths); + log.info( + `Codebase ${c.relative_path}: digested ${paths.length} scripts in ${((performance.now() - start) / 1000).toFixed(1)}s`, + ); + } +} + /** Script metadata that differs only by the `.tar` a multi-output bundle adds to the * deployed codebase digest. */ function sameScriptMetadataModuloTar(a: any, b: any): boolean { @@ -618,7 +652,7 @@ async function addCodebaseDigestIfRelevant( if (ignoreCodebaseChanges) { parsed["codebase"] = undefined; } else { - parsed["codebase"] = await c.getDigest(); + parsed["codebase"] = await c.getDigest(replacedPath); } parsed["lock"] = ""; return yamlStringify(parsed, yamlOptions); @@ -3026,7 +3060,7 @@ export async function compareDynFSElement( continue; } const c = findCodebase(tsFile, codebases); - if (!sameCodebaseDigest(await c?.getDigest(), v)) { + if (!sameCodebaseDigest(await c?.getDigest(tsFile), v)) { changes.push({ name: "edited", path: tsFile, @@ -4493,8 +4527,10 @@ export async function dedupeLockfilesOnDisk(args: { failed = [], dryRun, } = args; + // The plan reads only locks, so codebase digests (a bundle pass per script + // under `per_script_digest`) are not computed for it. const map = await elementsToMap( - await FSFSElement(process.cwd(), codebases, false), + await FSFSElement(process.cwd(), codebases, true), ignore, opts.json ?? false, opts, @@ -5088,6 +5124,9 @@ export async function push( localFlowInlineMapping, ); + if (!opts.skipScripts) { + await primeCodebaseDigests(codebases, await ignoreF(opts)); + } const local = await FSFSElement( path.join(process.cwd(), ""), codebases, diff --git a/cli/src/core/conf.ts b/cli/src/core/conf.ts index b010b1f8e3..b7dd49deb7 100644 --- a/cli/src/core/conf.ts +++ b/cli/src/core/conf.ts @@ -120,6 +120,7 @@ export interface SyncOptions { export interface Codebase { relative_path: string; extra_digest_paths?: string[]; + per_script_digest?: boolean; includes?: string[]; excludes?: string[]; assets?: { diff --git a/cli/src/utils/codebase.ts b/cli/src/utils/codebase.ts index debdf012bb..9bcf42e2cd 100644 --- a/cli/src/utils/codebase.ts +++ b/cli/src/utils/codebase.ts @@ -1,7 +1,8 @@ import { Codebase, SyncOptions } from "../core/conf.ts"; import * as log from "../core/log.ts"; import { digestDir, generateHash, generateHashFromBuffer } from "./utils.ts"; -import { readFile, stat } from "node:fs/promises"; +import { withEsbuild } from "./esbuild_loader.ts"; +import { readdir, readFile, realpath, stat } from "node:fs/promises"; import path from "node:path"; // A bare string is accepted, like `includes`/`excludes` in findCodebase. @@ -40,8 +41,18 @@ export function uncoveredBundleInputs( }); } +/** Whether a codebase digests each script from its own bundle inputs. A custom + * bundler's inputs are unknown, so it keeps the directory digest. */ +export function isPerScriptDigest(codebase: Codebase): boolean { + return codebase.per_script_digest === true && !codebase.customBundler; +} + export type SyncCodebase = Codebase & { - getDigest: (forceTar?: boolean) => Promise; + /** The `codebase` value a push stores for the script at `scriptPath`. */ + getDigest: (scriptPath: string, forceTar?: boolean) => Promise; + /** Computes the digests of many scripts in one esbuild pass. Only per-script + * digest codebases do any work. */ + primeDigests: (scriptPaths: string[]) => Promise; }; /** Whether two codebase digests name the same bundle. A push adds `.tar` when esbuild emits @@ -67,43 +78,437 @@ export function listSyncCodebases(options: SyncOptions): SyncCodebase[] { .join(", ")}` ); } + const hasher = new FileHasher(); for (const codebase of options?.codebases ?? []) { - let _digest: string | undefined = undefined; - let alreadyPrinted = false; - let hasAssets = false; - const getDigest: (forceTar?: boolean) => Promise = async ( - forceTar?: boolean - ) => { - if (_digest == undefined) { - // Concurrent callers read `_digest` across these awaits: assign it only once complete. - let digest = await digestDir( - codebase.relative_path, - JSON.stringify(codebase) - ); - const extra = extraDigestPaths(codebase); - if (extra.length > 0) { - const hashes = await Promise.all(extra.map(digestPath)); - digest = await generateHash(digest + hashes.join("")); - } - if (codebase.format == "esm") { - digest += ".esm"; - } - _digest = digest; - if (!alreadyPrinted) { - alreadyPrinted = true; - log.info(`Codebase ${codebase.relative_path}, digest: ${_digest}`); - } - hasAssets = - Array.isArray(codebase.assets) && codebase.assets.length > 0; - } - if (forceTar || hasAssets) { - return _digest + ".tar"; - } else { - return _digest; - } + const hasAssets = + Array.isArray(codebase.assets) && codebase.assets.length > 0; + const withSuffixes = (digest: string, forceTar?: boolean) => { + if (codebase.format == "esm") digest += ".esm"; + return forceTar || hasAssets ? digest + ".tar" : digest; }; - res.push({ ...codebase, getDigest }); + + if (!isPerScriptDigest(codebase)) { + let dirDigest: Promise | undefined; + const getDigest = async (_scriptPath: string, forceTar?: boolean) => { + if (!dirDigest) { + dirDigest = (async () => { + let digest = await digestDir( + codebase.relative_path, + JSON.stringify(codebase) + ); + const extra = extraDigestPaths(codebase); + if (extra.length > 0) { + const hashes = await Promise.all(extra.map(digestPath)); + digest = await generateHash(digest + hashes.join("")); + } + log.info( + `Codebase ${codebase.relative_path}, digest: ${withSuffixes(digest)}` + ); + return digest; + })(); + } + return withSuffixes(await dirDigest, forceTar); + }; + res.push({ ...codebase, getDigest, primeDigests: async () => {} }); + continue; + } + + const digests = new Map>(); + const primeDigests = async (scriptPaths: string[]) => { + const missing = [ + ...new Set(scriptPaths.map((p) => path.resolve(p))), + ].filter((p) => !digests.has(p)); + if (missing.length == 0) return; + const computed = scriptDigests(codebase, missing, hasher); + for (const p of missing) { + const d = computed.then((m) => { + const v = m.get(p)!; + if (v instanceof Error) throw v; + return v; + }); + // A primed script nobody asks for must not fail the run. + d.catch(() => {}); + digests.set(p, d); + } + await Promise.allSettled(missing.map((p) => digests.get(p))); + }; + const getDigest = async (scriptPath: string, forceTar?: boolean) => { + const abs = path.resolve(scriptPath); + if (!digests.has(abs)) await primeDigests([abs]); + return withSuffixes(await digests.get(abs)!, forceTar); + }; + res.push({ ...codebase, getDigest, primeDigests }); } return res; } + +/** Memoized content hashes, shared by every codebase of a run. */ +class FileHasher { + private hashes = new Map>(); + private configs = new Map>(); + private packages = new Map>(); + private packageFields = new Map>(); + + /** The nearest package.json at or above `dir`. */ + packageFor(dir: string): Promise { + let p = this.packages.get(dir); + if (!p) { + p = (async () => { + const here = path.join(dir, "package.json"); + if (await isFile(here)) return here; + const parent = path.dirname(dir); + return parent == dir ? undefined : await this.packageFor(parent); + })(); + this.packages.set(dir, p); + } + return p; + } + + /** A hash of the build-relevant fields of a package.json, or of the whole file + * when it does not parse. */ + hashPackageFields(pkgPath: string): Promise { + let h = this.packageFields.get(pkgPath); + if (!h) { + h = (async () => { + const pkg = await readJson(pkgPath); + if (!pkg || typeof pkg != "object") return await this.hash(pkgPath); + const fields = BUILD_PACKAGE_FIELDS.map((f) => [f, pkg[f]]); + return await generateHash(JSON.stringify(fields)); + })(); + this.packageFields.set(pkgPath, h); + } + return h; + } + + hash(absPath: string): Promise { + let h = this.hashes.get(absPath); + if (!h) { + h = readFile(absPath).then(generateHashFromBuffer); + this.hashes.set(absPath, h); + } + return h; + } + + /** The TS config esbuild applies to files in `dir`, as it picks it: the nearest + * `tsconfig.json`, else `jsconfig.json`, in `dir` or an ancestor, followed by the + * configs it `extends`. Ancestors it shadows are left out. */ + configsFor(dir: string): Promise { + let c = this.configs.get(dir); + if (!c) { + c = (async () => { + for (const name of ["tsconfig.json", "jsconfig.json"]) { + const p = path.join(dir, name); + if (await isFile(p)) return await this.extendsChain(p, new Set()); + } + const parent = path.dirname(dir); + return parent == dir ? [] : await this.configsFor(parent); + })(); + this.configs.set(dir, c); + } + return c; + } + + private async extendsChain(config: string, seen: Set): Promise { + if (seen.has(config)) return []; + seen.add(config); + const res = [config]; + for (const spec of tsconfigExtends(await readFile(config, "utf8"))) { + const dir = path.dirname(config); + const bases = isPackageSpec(spec) + ? await packageConfigFiles(spec, dir) + : [await resolveRelativeExtends(spec, dir)].filter((b) => b != undefined); + for (const base of bases) res.push(...(await this.extendsChain(base, seen))); + } + return res; + } +} + +const isPackageSpec = (spec: string) => + !/^\.{1,2}(\/|\\|$)/.test(spec) && !path.isAbsolute(spec); + +/** A path `extends`, as esbuild 0.28 resolves it: relative to the config, with + * `.json` appended only when it lacks it, and `.`/`..` meaning their tsconfig.json. */ +async function resolveRelativeExtends(spec: string, dir: string): Promise { + const p = path.resolve(dir, spec == "." || spec == ".." ? spec + "/tsconfig.json" : spec); + if (await isFile(p)) return p; + return !p.endsWith(".json") && (await isFile(p + ".json")) ? p + ".json" : undefined; +} + +/** + * Every config a package `extends` could resolve to: every file of the package in + * each `node_modules` above the config, whatever its extension (an `exports` target + * can be `.jsonc` or extensionless) and through symlinks (workspace packages), plus + * the file its `tsconfig` field names. esbuild picks one through the package's + * `exports` conditions, `tsconfig` field and subpath; hashing them all covers its + * pick without reimplementing that resolution, at the cost of redeploying when any + * file of the package changes. + */ +async function packageConfigFiles(spec: string, dir: string): Promise { + const name = spec.split("/").slice(0, spec.startsWith("@") ? 2 : 1).join("/"); + const res: string[] = []; + for (let d = dir; ; d = path.dirname(d)) { + const pkgDir = path.join(d, "node_modules", name); + if (await stat(pkgDir).then((s) => s.isDirectory(), () => false)) { + res.push(...(await filesUnder(pkgDir, new Set()))); + const field = (await readJson(path.join(pkgDir, "package.json")))?.tsconfig; + if (typeof field == "string" && field != "") { + const p = path.resolve(pkgDir, field); + if (await isFile(p)) res.push(p); + } + } + if (path.dirname(d) == d) return [...new Set(res)].sort(); + } +} + +/** The files under `dir`, following symlinks, without nested `node_modules`. */ +async function filesUnder(dir: string, seenDirs: Set): Promise { + const real = await realpath(dir); + if (seenDirs.has(real)) return []; + seenDirs.add(real); + const res: string[] = []; + for (const e of (await readdir(dir)).sort()) { + const p = path.join(dir, e); + const s = await stat(p).catch(() => undefined); + if (s?.isDirectory() && e != "node_modules") res.push(...(await filesUnder(p, seenDirs))); + else if (s?.isFile()) res.push(p); + } + return res; +} + +const isFile = (p: string) => stat(p).then((s) => s.isFile(), () => false); + +/** The `extends` of a tsconfig, which is JSON with comments and trailing commas. + * An unparseable config extends nothing; its own content is still hashed. */ +export function tsconfigExtends(text: string): string[] { + text = text.replace(/^\uFEFF/, ""); + let json = ""; + for (let i = 0; i < text.length; i++) { + const ch = text[i]; + if (ch == '"') { + const start = i; + for (i++; i < text.length && text[i] != '"'; i++) if (text[i] == "\\") i++; + json += text.slice(start, i + 1); + } else if (ch == "/" && text[i + 1] == "/") { + while (i < text.length && text[i] != "\n") i++; + json += "\n"; + } else if (ch == "/" && text[i + 1] == "*") { + i = text.indexOf("*/", i + 2); + if (i < 0) break; + i++; + } else { + json += ch; + } + } + try { + const ext = JSON.parse(json.replace(/,(\s*[}\]])/g, "$1"))?.extends; + return ([] as unknown[]).concat(ext ?? []).filter((e): e is string => typeof e == "string"); + } catch { + return []; + } +} + +async function readJson(p: string): Promise { + try { + return JSON.parse((await readFile(p, "utf8")).replace(/^\uFEFF/, "")); + } catch { + return undefined; + } +} + +/** The fields of a package.json that change how esbuild compiles or tree-shakes the + * files below it without changing which files it bundles. Hashed instead of the + * whole file, so adding a dependency does not redeploy every script; a dependency's + * new version shows up in its bundled files. */ +const BUILD_PACKAGE_FIELDS = ["sideEffects", "type", "main", "module", "browser", "exports", "imports"]; + +/** A file input of an esbuild metafile, as opposed to "", "(disabled):x" + * or a plugin namespace. */ +function isFileInput(key: string): boolean { + return !/^[<(]|^[a-z-]{2,}:/i.test(key); +} + +const toPosix = (p: string) => p.split(path.sep).join("/"); + +/** + * Per-script digests: each covers the files the script's bundle is built from + * (the transitive imports esbuild resolves, `node_modules` included, by path and + * content), the TS configs and package.json build fields that apply to them, the + * codebase's assets and + * `extra_digest_paths`, and the codebase config. + * + * It follows the import graph rather than the bundle output: an output's input list + * drops files whose values are inlined (TS enums), and hashing output would make + * every digest depend on esbuild's code generation. Paths are relative to the + * working directory, the sync checkout root, so a script digests the same alone and + * in a batch, on any machine. + */ +async function scriptDigests( + codebase: Codebase, + scriptPaths: string[], + hasher: FileHasher +): Promise> { + const cwd = await realpath(process.cwd()); + const parts = [JSON.stringify(codebase)]; + for (const p of extraDigestPaths(codebase)) parts.push(await digestPath(p)); + for (const a of codebase.assets ?? []) { + parts.push(`${a.to}\0${await hasher.hash(path.resolve(a.from))}`); + } + const shared = parts.join("\n"); + + const closures = await importClosures(codebase, scriptPaths, cwd); + const res = new Map(); + await Promise.all( + scriptPaths.map(async (script) => { + const files = closures.get(script)!; + if (files instanceof Error) { + res.set(script, files); + return; + } + const configs = new Set(); + const packages = new Set(); + for (const f of files) { + const pkg = await hasher.packageFor(path.dirname(f)); + if (pkg) packages.add(pkg); + if (f.split(path.sep).includes("node_modules")) continue; + for (const c of await hasher.configsFor(path.dirname(f))) { + configs.add(c); + } + } + const rel = (f: string) => toPosix(path.relative(cwd, f)); + const entries = await Promise.all([ + ...[...files, ...configs].map(async (f) => `${rel(f)}\0${await hasher.hash(f)}`), + ...[...packages].map( + async (p) => `${rel(p)}\0fields\0${await hasher.hashPackageFields(p)}` + ), + ]); + entries.sort(); + res.set( + script, + await generateHash(shared + "\n" + entries.join("\n")) + ); + }) + ); + return res; +} + +/** Each script's bundle inputs as real absolute paths, the script itself included. + * esbuild reports real paths, so `cwd` and the entries are resolved the same way: a + * checkout under a symlink would otherwise find no inputs for its scripts. */ +async function importClosures( + codebase: Codebase, + scriptPaths: string[], + cwd: string +): Promise | Error>> { + const res = new Map | Error>(); + const entries = new Map(); + for (const s of scriptPaths) { + try { + entries.set(s, await realpath(s)); + } catch (e) { + res.set(s, bundleError(s, e)); + } + } + const inject = await Promise.all( + (codebase.inject ?? []).map((p) => realpath(path.resolve(p))) + ); + const graph = async (entries: string[], batched: boolean) => { + const out = await withEsbuild((esbuild) => + esbuild.build({ + ...graphBuildOptions(codebase, batched), + entryPoints: entries, + absWorkingDir: cwd, + }) + ); + const imports = new Map(); + for (const [key, input] of Object.entries(out.metafile?.inputs ?? {})) { + if (!isFileInput(key)) continue; + imports.set( + path.resolve(cwd, key), + input.imports + .filter((i) => !i.external && isFileInput(i.path)) + .map((i) => path.resolve(cwd, i.path)) + ); + } + return imports; + }; + const closure = (imports: Map, script: string) => { + if (!imports.has(script)) { + throw new Error(`esbuild reported no inputs for ${script}`); + } + const seen = new Set(); + const todo = [script, ...inject]; + while (todo.length > 0) { + const f = todo.pop()!; + if (seen.has(f)) continue; + seen.add(f); + todo.push(...(imports.get(f) ?? [])); + } + return seen; + }; + + if (entries.size > 1) { + try { + const imports = await graph([...entries.values()], true); + for (const [s, real] of entries) res.set(s, closure(imports, real)); + return res; + } catch (e) { + const first = (e as { errors?: { text: string; location?: { file: string } }[] })?.errors?.[0]; + const reason = first + ? `${first.location?.file ? first.location.file + ": " : ""}${first.text}` + : String(e).split("\n")[0]; + log.info( + `Codebase digest: the batched build failed (${reason}), digesting ${entries.size} scripts one by one` + ); + } + } + // One script that does not bundle fails the whole batch; only it should fail. + const queue = [...entries]; + await Promise.all( + Array.from({ length: Math.min(8, queue.length) }, async () => { + for (let q = queue.shift(); q != undefined; q = queue.shift()) { + const [s, real] = q; + try { + res.set(s, closure(await graph([real], false), real)); + } catch (e) { + res.set(s, bundleError(s, e)); + } + } + }) + ); + return res; +} + +function bundleError(script: string, e: unknown): Error { + const msg = e instanceof Error ? e.message : String(e); + return new Error( + `Cannot compute the codebase digest of ${toPosix(path.relative(process.cwd(), script))}: bundling it failed. ` + + `With per_script_digest every codebase script must bundle for a push to compare it; fix the script or exclude it from the codebase.\n${msg}` + ); +} + +/** + * The esbuild options of the digest pass. Only the options that decide which files + * are bundled matter (`external`, `inject`, `define`, which can drop a dead + * `require`, `loader`, `platform`, `packages`); the output is discarded. A batch is + * built with code splitting so that code shared by many scripts is generated once. + */ +function graphBuildOptions(codebase: Codebase, batched: boolean) { + return { + bundle: true, + write: false, + metafile: true, + logLevel: "silent" as const, + outdir: "/", + platform: "node" as const, + packages: "bundle" as const, + external: codebase.external, + inject: codebase.inject, + define: codebase.define, + loader: codebase.loader ?? { ".node": "file" }, + target: "esnext", + ...(batched + ? { format: "esm" as const, splitting: true } + : { format: codebase.format ?? "cjs" }), + }; +} diff --git a/cli/src/utils/local_path_scripts.ts b/cli/src/utils/local_path_scripts.ts index a1dcf432a8..d1e6f80e3b 100644 --- a/cli/src/utils/local_path_scripts.ts +++ b/cli/src/utils/local_path_scripts.ts @@ -146,6 +146,7 @@ export async function resolvePreviewLocalScriptState( codebase, codebaseDigest: codebase ? await codebase.getDigest( + filePath, Array.isArray(codebase.assets) && codebase.assets.length > 0 ) : undefined, diff --git a/cli/test/codebase_digest_unit.test.ts b/cli/test/codebase_digest_unit.test.ts index f12b928de4..8cffcc119a 100644 --- a/cli/test/codebase_digest_unit.test.ts +++ b/cli/test/codebase_digest_unit.test.ts @@ -1,9 +1,10 @@ import { expect, test } from "bun:test"; -import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, mkdir, realpath, rm, symlink, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { listSyncCodebases, + tsconfigExtends, uncoveredBundleInputs, } from "../src/utils/codebase.ts"; @@ -33,6 +34,319 @@ test("extra_digest_paths edits change the codebase digest", async () => { } }); +async function perScriptFixture() { + const root = await mkdtemp(join(tmpdir(), "wmill-codebase-")); + await mkdir(join(root, "f/app/lib"), { recursive: true }); + await mkdir(join(root, "packages/utils"), { recursive: true }); + const files: Record = { + "f/app/a.ts": "import { greet } from './lib/greet'; export function main() { return greet() }", + "f/app/a.spec.ts": "// a test, imported by nothing", + "f/app/b.ts": "import { Kind } from './lib/kind'; export function main() { return Kind.A }", + "f/app/c.ts": "import { u } from '../../packages/utils/u'; export function main() { return u }", + "f/app/lib/greet.ts": "export const greet = () => 'hi'", + "f/app/lib/kind.ts": "export enum Kind { A = 1 }", + "packages/utils/u.ts": "export const u = 1", + }; + for (const [p, c] of Object.entries(files)) await writeFile(join(root, p), c); + const scripts = ["a", "b", "c"].map((s) => join(root, `f/app/${s}.ts`)); + const codebase = () => + listSyncCodebases({ + codebases: [{ relative_path: join(root, "f/app"), per_script_digest: true }], + } as any)[0]; + const digests = async () => { + const c = codebase(); + await c.primeDigests(scripts); + return Promise.all(scripts.map((s) => c.getDigest(s))); + }; + return { root, scripts, codebase, digests }; +} + +test("per_script_digest changes only the digests of scripts that bundle the edited file", async () => { + const { root, digests } = await perScriptFixture(); + try { + const [a, b, c] = await digests(); + expect(new Set([a, b, c]).size).toBe(3); + + await writeFile(join(root, "f/app/a.spec.ts"), "// edited"); + expect(await digests()).toEqual([a, b, c]); + + await writeFile(join(root, "f/app/lib/greet.ts"), "export const greet = () => 'hello'"); + const [a2, b2, c2] = await digests(); + expect(a2).not.toBe(a); + expect([b2, c2]).toEqual([b, c]); + + // The enum's value is inlined into b's bundle, which then no longer lists kind.ts. + await writeFile(join(root, "f/app/lib/kind.ts"), "export enum Kind { A = 2 }"); + const [, b3] = await digests(); + expect(b3).not.toBe(b); + + // Imports outside relative_path are covered without extra_digest_paths. + await writeFile(join(root, "packages/utils/u.ts"), "export const u = 2"); + const [, , c4] = await digests(); + expect(c4).not.toBe(c); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("per_script_digest is the same for a script digested alone or in a batch", async () => { + const { root, scripts, codebase, digests } = await perScriptFixture(); + try { + const batched = await digests(); + const alone = await Promise.all(scripts.map((s) => codebase().getDigest(s))); + expect(alone).toEqual(batched); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("per_script_digest follows the imports of a script reached through a symlink", async () => { + const { root, scripts, codebase } = await perScriptFixture(); + const link = root + "-link"; + try { + await symlink(root, link); + const viaLink = join(link, "f/app/a.ts"); + const before = await codebase().getDigest(viaLink); + expect(before).toBe(await codebase().getDigest(scripts[0])); + await writeFile(join(root, "f/app/lib/greet.ts"), "export const greet = () => 'hello'"); + expect(await codebase().getDigest(viaLink)).not.toBe(before); + } finally { + await rm(link, { force: true }); + await rm(root, { recursive: true, force: true }); + } +}); + +test("per_script_digest covers the nearest tsconfig and what it extends, not shadowed ones", async () => { + const { root, digests } = await perScriptFixture(); + try { + await writeFile(join(root, "tsconfig.json"), "{}"); + await writeFile(join(root, "tsconfig.base.json"), `{ "compilerOptions": {} }`); + await writeFile( + join(root, "f/app/tsconfig.json"), + `{\n // shared settings\n "extends": "../../tsconfig.base.json", /* jsonc */\n}` + ); + const before = await digests(); + + // f/app's own tsconfig shadows the root one; only c, which also imports + // packages/utils, has a file the root tsconfig applies to. + await writeFile(join(root, "tsconfig.json"), `{ "compilerOptions": { "strict": true } }`); + const shadowed = await digests(); + expect(shadowed.slice(0, 2)).toEqual(before.slice(0, 2)); + expect(shadowed[2]).not.toBe(before[2]); + + await writeFile(join(root, "tsconfig.base.json"), `{ "compilerOptions": { "jsx": "react" } }`); + const extended = await digests(); + expect(extended[0]).not.toBe(shadowed[0]); + expect(extended[1]).not.toBe(shadowed[1]); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +test("per_script_digest covers build fields of package.json, not its dependencies", async () => { + const { root, digests } = await perScriptFixture(); + try { + await writeFile(join(root, "package.json"), `{ "name": "x", "dependencies": {} }`); + const before = await digests(); + await writeFile(join(root, "package.json"), `{ "name": "x", "dependencies": { "pg": "8" } }`); + expect(await digests()).toEqual(before); + await writeFile(join(root, "package.json"), `{ "name": "x", "sideEffects": false }`); + const after = await digests(); + expect(after.every((d, i) => d != before[i])).toBe(true); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + +// Pins the digest's tsconfig coverage to what esbuild applies: in each layout, editing +// the base config esbuild applies must change both the bundle and the digest. The decoy +// is a config esbuild does not apply (its edit leaves the bundle alone), which proves +// the layout resolves where it claims to. +const ON = `{ "compilerOptions": { "useDefineForClassFields": true } }`; +const OFF = `{ "compilerOptions": { "useDefineForClassFields": false } }`; +const tsconfigLayouts: { + name: string; + files: Record; + links?: Record; + applied: string; + decoy?: string; +}[] = [ + { + name: ".jsonc exports target", + files: { + "node_modules/cfg/package.json": JSON.stringify({ exports: { ".": "./base.jsonc" } }), + "node_modules/cfg/base.jsonc": ON, + "node_modules/cfg/tsconfig.json": ON, + "f/tsconfig.json": `{ "extends": "cfg" }`, + }, + applied: "node_modules/cfg/base.jsonc", + decoy: "node_modules/cfg/tsconfig.json", + }, + { + name: "extensionless subpath", + files: { + "node_modules/cfg/package.json": "{}", + "node_modules/cfg/strict": ON, + "f/tsconfig.json": `{ "extends": "cfg/strict" }`, + }, + applied: "node_modules/cfg/strict", + }, + { + name: "workspace package symlinked into node_modules", + files: { + "packages/cfg/package.json": "{}", + "packages/cfg/tsconfig.json": ON, + "f/tsconfig.json": `{ "extends": "cfg" }`, + }, + links: { "node_modules/cfg": "../packages/cfg" }, + applied: "packages/cfg/tsconfig.json", + }, + { + name: "conditional exports", + files: { + "node_modules/cfg/package.json": JSON.stringify({ exports: { ".": { import: "./esm.json", require: "./cjs.json" } } }), + "node_modules/cfg/esm.json": ON, + "node_modules/cfg/cjs.json": ON, + "f/tsconfig.json": `{ "extends": "cfg" }`, + }, + applied: "node_modules/cfg/cjs.json", + decoy: "node_modules/cfg/esm.json", + }, + { + name: "nested unmatched condition", + files: { + "node_modules/cfg/package.json": JSON.stringify({ exports: { ".": { node: { import: "./esm.json" }, require: "./req.json" } } }), + "node_modules/cfg/esm.json": ON, + "node_modules/cfg/req.json": ON, + "f/tsconfig.json": `{ "extends": "cfg" }`, + }, + applied: "node_modules/cfg/req.json", + decoy: "node_modules/cfg/esm.json", + }, + { + name: "explicit null condition", + files: { + "node_modules/cfg/package.json": JSON.stringify({ exports: { ".": { node: null, require: "./req.json" } } }), + "node_modules/cfg/req.json": ON, + "node_modules/cfg/tsconfig.json": ON, + "f/tsconfig.json": `{ "extends": "cfg" }`, + }, + applied: "node_modules/cfg/tsconfig.json", + decoy: "node_modules/cfg/req.json", + }, + { + name: "module condition", + files: { + "node_modules/cfg/package.json": JSON.stringify({ exports: { ".": { module: "./module.json", require: "./req.json" } } }), + "node_modules/cfg/module.json": ON, + "node_modules/cfg/req.json": ON, + "f/tsconfig.json": `{ "extends": "cfg" }`, + }, + applied: "node_modules/cfg/module.json", + decoy: "node_modules/cfg/req.json", + }, + { + name: "invalid target in a fallback array", + files: { + "node_modules/cfg/package.json": JSON.stringify({ exports: { ".": ["invalid", "./req.json"] } }), + "node_modules/cfg/req.json": ON, + "node_modules/cfg/tsconfig.json": ON, + "f/tsconfig.json": `{ "extends": "cfg" }`, + }, + applied: "node_modules/cfg/req.json", + decoy: "node_modules/cfg/tsconfig.json", + }, + { + name: "overlapping wildcard exports", + files: { + "node_modules/cfg/package.json": JSON.stringify({ exports: { "./*": "./any/*.json", "./node/*": "./node/*.json" } }), + "node_modules/cfg/any/node/20.json": ON, + "node_modules/cfg/node/20.json": ON, + "f/tsconfig.json": `{ "extends": "cfg/node/20" }`, + }, + applied: "node_modules/cfg/node/20.json", + decoy: "node_modules/cfg/any/node/20.json", + }, + { + name: "package tsconfig field", + files: { + "node_modules/cfg/package.json": JSON.stringify({ tsconfig: "./configs/base.json" }), + "node_modules/cfg/configs/base.json": ON, + "node_modules/cfg/tsconfig.json": ON, + "f/tsconfig.json": `{ "extends": "cfg" }`, + }, + applied: "node_modules/cfg/configs/base.json", + decoy: "node_modules/cfg/tsconfig.json", + }, + { + name: "BOM-prefixed config, relative extends without .json", + files: { "base.json": ON, "f/tsconfig.json": `\uFEFF{ "extends": "../base" }` }, + applied: "base.json", + }, +]; + +for (const layout of tsconfigLayouts) { + test(`per_script_digest applies the tsconfig esbuild applies: ${layout.name}`, async () => { + const esbuild = await import("esbuild"); + const root = await realpath(await mkdtemp(join(tmpdir(), "wmill-tsconfig-"))); + const cwd = process.cwd(); + try { + for (const [p, c] of Object.entries(layout.files)) { + await mkdir(dirname(join(root, p)), { recursive: true }); + await writeFile(join(root, p), c); + } + for (const [p, target] of Object.entries(layout.links ?? {})) { + await mkdir(dirname(join(root, p)), { recursive: true }); + await symlink(target, join(root, p)); + } + // useDefineForClassFields decides whether the field shadows the accessor. + await writeFile( + join(root, "f/a.ts"), + "class A { x = 1; constructor() { Object.defineProperty(this, 'x', { get: () => 0 }) } }\nexport const main = () => new A().x\n" + ); + process.chdir(root); + const bundle = async () => + (await esbuild.build({ entryPoints: ["f/a.ts"], bundle: true, write: false, platform: "node", logLevel: "silent", absWorkingDir: root })).outputFiles[0].text; + const digest = () => + listSyncCodebases({ codebases: [{ relative_path: "f", per_script_digest: true }] } as any)[0].getDigest("f/a.ts"); + const [b0, d0] = [await bundle(), await digest()]; + if (layout.decoy) { + await writeFile(join(root, layout.decoy), OFF); + expect(await bundle()).toBe(b0); + } + const d1 = await digest(); + await writeFile(join(root, layout.applied), OFF); + expect(await bundle()).not.toBe(b0); + expect(await digest()).not.toBe(d1); + } finally { + process.chdir(cwd); + await rm(root, { recursive: true, force: true }); + } + }); +} + +test("tsconfigExtends reads extends from JSON with comments", () => { + expect(tsconfigExtends(`{ "a": "x // not a comment", // c\n "extends": ["./a", "b"], }`)).toEqual(["./a", "b"]); + expect(tsconfigExtends(`{ /* "extends": "no" */ }`)).toEqual([]); + expect(tsconfigExtends(`not json`)).toEqual([]); + expect(tsconfigExtends(`\uFEFF{ "extends": "./base.json" }`)).toEqual(["./base.json"]); +}); + +test("per_script_digest fails only the script that does not bundle", async () => { + const { root, scripts, codebase } = await perScriptFixture(); + try { + const before = await Promise.all(scripts.map((s) => codebase().getDigest(s))); + await writeFile(join(root, "f/app/b.ts"), "import { x } from './lib/missing'; export const main = () => x"); + const c = codebase(); + await c.primeDigests(scripts); + expect(await c.getDigest(scripts[0])).toBe(before[0]); + await expect(c.getDigest(scripts[1])).rejects.toThrow("f/app/b.ts"); + expect(await c.getDigest(scripts[2])).toBe(before[2]); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + test("uncoveredBundleInputs reports only files outside the digested roots", () => { const codebase = { relative_path: "f", extra_digest_paths: ["packages/a"] }; expect( diff --git a/cli/wmill.schema.json b/cli/wmill.schema.json index 145a75fd26..15af745438 100644 --- a/cli/wmill.schema.json +++ b/cli/wmill.schema.json @@ -154,6 +154,10 @@ }, "description": "Paths outside relative_path that scripts import, relative to wmill.yaml; hashed recursively" }, + "per_script_digest": { + "type": "boolean", + "description": "Redeploy a script only when a file it bundles changes, instead of on any change under relative_path. Each push bundles every script of the codebase once to find its inputs, so every one of them must bundle. Files a script reads at runtime rather than imports are not covered: list them in extra_digest_paths" + }, "includes": { "type": "array", "items": {