diff --git a/cli/build-npm.ts b/cli/build-npm.ts index 665021ffdc..5c9353dd21 100644 --- a/cli/build-npm.ts +++ b/cli/build-npm.ts @@ -28,6 +28,9 @@ rmSync(outDir, { recursive: true, force: true }); // Build with bun — bundle everything except esbuild (platform-specific binary), // svelte (optional, only needed for `wmill app bundle/dev`), and parser packages // (loaded at runtime via init() with readFileSync for the .wasm binary). +// esbuild-wasm is not a dependency at all: the host/binary-mismatch fallback in +// esbuild_loader.ts downloads and caches the whole esbuild-wasm package at +// runtime, so it stays out of the bundle and the published dependencies. console.log("Bundling with bun build..."); const buildResult = Bun.spawnSync([ "bun", "build", "src/main.ts", diff --git a/cli/src/commands/app/bundle.ts b/cli/src/commands/app/bundle.ts index e2f8553e22..07ee266145 100644 --- a/cli/src/commands/app/bundle.ts +++ b/cli/src/commands/app/bundle.ts @@ -6,6 +6,7 @@ import * as log from "../../core/log.ts"; import { colors } from "@cliffy/ansi/colors"; import * as windmillUtils from "@windmill-labs/shared-utils"; import { readTextFile, readTextFileSync } from "../../utils/utils.ts"; +import { getEsbuild, stopEsbuild } from "../../utils/esbuild_loader.ts"; export interface BundleOptions { entryPoint?: string; outDir?: string; @@ -170,8 +171,9 @@ export async function ensureNodeModules(appDir?: string): Promise { export async function createBundle( options: BundleOptions = {} ): Promise { - // Dynamically import esbuild - const esbuild = await import("esbuild"); + // Native esbuild with a transparent esbuild-wasm fallback on host/binary + // version mismatch (see esbuild_loader.ts). + const esbuild = await getEsbuild(); // Detect frameworks to determine default entry point. // Use the entryPoint's directory if provided, otherwise fall back to cwd. @@ -286,6 +288,10 @@ export async function createBundle( outfile, sourcemap, minify, + // Keep outputs in memory: esbuild-wasm cannot write to the filesystem + // ("write" option unavailable), and the dist files were discarded after the + // read anyway. Native esbuild supports write:false + outputFiles too. + write: false as const, define: { "process.env.NODE_ENV": production ? '"production"' : '"development"', }, @@ -307,29 +313,24 @@ export async function createBundle( log.info(colors.green("✅ Bundle created successfully")); - // Read the generated files - const jsPath = path.join(process.cwd(), outfile); - const cssPath = path.join(process.cwd(), outDir, "bundle.css"); + const outputFiles = result.outputFiles ?? []; + const jsFile = outputFiles.find((f) => f.path.endsWith(".js")); + const cssFile = outputFiles.find((f) => f.path.endsWith(".css")); - if (!fs.existsSync(jsPath)) { - throw new Error(`Expected JS bundle at ${jsPath} but file not found`); + if (!jsFile) { + throw new Error("Expected a JS bundle in esbuild output but none found"); } - const jsContent = readTextFileSync(jsPath); - const cssContent = fs.existsSync(cssPath) - ? readTextFileSync(cssPath) - : ""; - try { fs.rmSync(distDir, { recursive: true }); } catch { //ignore } - return { js: jsContent, css: cssContent }; - + return { js: jsFile.text, css: cssFile?.text ?? "" }; + } finally { - // Stop esbuild - await esbuild.stop(); + // Stop the native esbuild service so the process can exit (no-op for wasm). + await stopEsbuild(); } } diff --git a/cli/src/commands/app/dev.ts b/cli/src/commands/app/dev.ts index c73a8685c8..1a226ec5ef 100644 --- a/cli/src/commands/app/dev.ts +++ b/cli/src/commands/app/dev.ts @@ -437,7 +437,11 @@ async function dev(opts: DevOptions, appFolder?: string) { const rawApp = (await yamlParseFile(rawAppPath)) as any; const appPath = rawApp?.custom_path ?? "u/unknown/newapp"; - // Dynamically import esbuild only when the dev command is called + // Dynamically import esbuild only when the dev command is called. + // Native-only here (no esbuild-wasm fallback via getEsbuild): dev is a local + // interactive command that relies on context()/watch, whose semantics under + // wasm are untested. The host/binary-mismatch fallback covers the bundling + // paths that run on workers/CI via `wmill sync push`. const esbuild = await import("esbuild"); const host = opts.host ?? DEFAULT_HOST; diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index 717e8acf46..ed57b1747d 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -58,6 +58,7 @@ import { SyncCodebase, listSyncCodebases } from "../../utils/codebase.ts"; import { pollJobWithQueueLogging } from "../../utils/job_polling.ts"; import fs from "node:fs"; import { createTarBlob, type TarEntry } from "../../utils/tar.ts"; +import { getEsbuild } from "../../utils/esbuild_loader.ts"; import { execSync } from "node:child_process"; import { NewScript, Script, ScriptModule } from "../../../gen/types.gen.ts"; @@ -328,7 +329,7 @@ export async function handleFile( }).toString(); log.info("Custom bundler executed for " + path); } else { - const esbuild = await import("esbuild"); + const esbuild = await getEsbuild(); log.info(`Started bundling ${path} ...`); const startTime = performance.now(); @@ -1565,7 +1566,7 @@ async function preview( maxBuffer: 1024 * 1024 * 50, }).toString(); } else { - const esbuild = await import("esbuild"); + const esbuild = await getEsbuild(); if (!opts.silent) { log.info(`Bundling ${filePath} for preview...`); diff --git a/cli/src/utils/esbuild_loader.ts b/cli/src/utils/esbuild_loader.ts new file mode 100644 index 0000000000..4801c51a46 --- /dev/null +++ b/cli/src/utils/esbuild_loader.ts @@ -0,0 +1,228 @@ +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import process from "node:process"; +import { createGunzip } from "node:zlib"; +import { Readable } from "node:stream"; +import { pathToFileURL } from "node:url"; +import * as tar from "tar-stream"; +import * as log from "../core/log.ts"; + +// esbuild splits into a JS host package and a per-platform native binary +// (@esbuild/). They must be the same version. A broken or incremental +// install can leave the on-disk binary at a different version than the pinned +// host, which crashes service start with +// Cannot start service: Host version "X" does not match binary version "Y" +// The running code can't fix what npm/bun put on disk, so when that happens we +// fall back to esbuild-wasm, whose binary is a single version-pinned .wasm. To +// keep that 14MB out of every CLI install, the esbuild-wasm package is not a +// dependency: it is downloaded once and cached on disk, on the fallback path +// only. We download the whole package (not just the .wasm) because esbuild-wasm +// reads the app's files from disk by spawning `node bin/esbuild`, which needs +// bin/esbuild + esbuild.wasm + wasm_exec*.js co-located on disk. + +type Esbuild = typeof import("esbuild"); + +// Version to fall back to if the native host's version can't be read. Keep in +// sync with the "esbuild" pin in cli/package.json. +const FALLBACK_VERSION = "0.28.0"; + +let cached: Esbuild | undefined; +let inFlight: Promise | undefined; +// Distinguishes concurrent extraction temp dirs within a process. +let extractCounter = 0; + +/** + * Returns a working esbuild module, preferring the native binary and falling + * back to esbuild-wasm only when the native host/binary versions don't match. + * Memoized for the process: concurrent first callers (e.g. a parallel + * `wmill sync push`) share one probe/download instead of each running their own. + */ +export function getEsbuild(): Promise { + if (cached) return Promise.resolve(cached); + if (inFlight) return inFlight; + inFlight = acquireEsbuild() + .then((esbuild) => { + cached = esbuild; + return esbuild; + }) + .finally(() => { + inFlight = undefined; + }); + return inFlight; +} + +async function acquireEsbuild(): Promise { + // Escape hatch: skip native entirely (e.g. a host known to have a broken + // install, or to exercise the fallback path). + if (process.env.WINDMILL_FORCE_ESBUILD_WASM) { + return loadWasmEsbuild(await nativeHostVersion()); + } + + try { + const esbuild = await import("esbuild"); + // The native service only starts on the first call; force it with the most + // trivial op so any breakage (host/binary version mismatch, a dead service) + // surfaces now rather than mid-build. The mismatch detail is printed to the + // child's stderr while the thrown error is generic ("service was stopped"), + // so we fall back on ANY smoke-test failure rather than matching a string. + await esbuild.transform(""); + return esbuild; + } catch (e) { + const msg = e instanceof Error ? e.message : String(e); + log.warn( + `native esbuild is not usable; falling back to esbuild-wasm (${msg.trim()})` + ); + } + + return loadWasmEsbuild(await nativeHostVersion()); +} + +/** + * Stops the esbuild service (native or wasm — both spawn a child process) so the + * process can exit. Safe to call repeatedly; the service restarts lazily on the + * next build. + */ +export async function stopEsbuild(): Promise { + await cached?.stop(); +} + +async function nativeHostVersion(): Promise { + try { + return (await import("esbuild")).version ?? FALLBACK_VERSION; + } catch { + return FALLBACK_VERSION; + } +} + +async function loadWasmEsbuild(version: string): Promise { + const pkgDir = await ensureWasmPackage(version); + const mainJs = path.join(pkgDir, "lib", "main.js"); + // The Node build (lib/main.js) reads app files from disk by spawning + // `node bin/esbuild`, so it works with on-disk entry points and node_modules, + // unlike the browser build. + return (await import(pathToFileURL(mainJs).href)) as unknown as Esbuild; +} + +/** + * Returns a directory containing an extracted esbuild-wasm package (with + * lib/main.js). Uses an explicit override, then an on-disk cache, then downloads + * and extracts the npm tarball. + */ +async function ensureWasmPackage(version: string): Promise { + // Explicit local override wins (air-gapped / self-hosted workers): a path to + // an already-extracted esbuild-wasm package directory. + const override = process.env.WINDMILL_ESBUILD_WASM_PATH; + if (override) return override; + + const destDir = path.join(cacheDir(), `esbuild-wasm-${version}`); + if (fs.existsSync(path.join(destDir, "lib", "main.js"))) { + return destDir; + } + + const url = process.env.WINDMILL_ESBUILD_WASM_URL ?? + `https://registry.npmjs.org/esbuild-wasm/-/esbuild-wasm-${version}.tgz`; + log.info(`Downloading esbuild-wasm@${version} from ${url} ...`); + const res = await fetch(url); + if (!res.ok || !res.body) { + throw new Error( + `Failed to download esbuild-wasm@${version} (${res.status} ${res.statusText}). ` + + `Set WINDMILL_ESBUILD_WASM_PATH to an extracted esbuild-wasm package dir, ` + + `point WINDMILL_ESBUILD_WASM_URL at a reachable tarball, or repair the native esbuild install.` + ); + } + + // Extract to a unique temp dir and rename into place so a crash or a + // concurrent writer can't leave a half-extracted package behind, and so two + // extractions never share an in-progress directory. + const tmpDir = `${destDir}.${process.pid}.${extractCounter++}.tmp`; + fs.rmSync(tmpDir, { recursive: true, force: true }); + await extractTarball(res.body, tmpDir); + if (!fs.existsSync(path.join(tmpDir, "lib", "main.js"))) { + fs.rmSync(tmpDir, { recursive: true, force: true }); + throw new Error(`esbuild-wasm@${version} tarball did not contain lib/main.js`); + } + try { + fs.renameSync(tmpDir, destDir); + } catch { + // Another process won the race, or rename across devices failed; clean up + // and let the existsSync check below decide whether the cache is usable. + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + if (!fs.existsSync(path.join(destDir, "lib", "main.js"))) { + throw new Error(`Failed to cache esbuild-wasm@${version} at ${destDir}`); + } + return destDir; +} + +/** + * Resolves a tar entry to an absolute path inside destDir, stripping the leading + * "package/" component that npm tarballs use. Returns null if the entry would + * escape destDir (tar-slip), since WINDMILL_ESBUILD_WASM_URL allows untrusted + * tarball sources. + */ +export function resolveTarEntryPath( + destDir: string, + entryName: string +): string | null { + const rel = entryName.replace(/^[^/]+\//, ""); + const root = path.resolve(destDir); + const outPath = path.resolve(root, rel); + if (outPath !== root && !outPath.startsWith(root + path.sep)) { + return null; + } + return outPath; +} + +// Extracts an npm tarball (gzipped tar) into destDir, stripping the leading +// "package/" path component that npm tarballs use. +async function extractTarball( + body: ReadableStream, + destDir: string +): Promise { + const extract = tar.extract(); + extract.on("entry", (header, stream, next) => { + if (header.type !== "file") { + stream.resume(); + stream.on("end", next); + return; + } + const outPath = resolveTarEntryPath(destDir, header.name); + if (!outPath) { + // Reject tar-slip entries that would write outside the cache dir. + stream.resume(); + stream.on("end", () => + next(new Error(`unsafe path in esbuild-wasm tarball: ${header.name}`)) + ); + return; + } + fs.mkdirSync(path.dirname(outPath), { recursive: true }); + const ws = fs.createWriteStream(outPath, { mode: header.mode ?? 0o644 }); + stream.pipe(ws); + ws.on("finish", next); + ws.on("error", next); + stream.on("error", next); + }); + + await new Promise((resolve, reject) => { + extract.on("finish", resolve); + extract.on("error", reject); + Readable.fromWeb(body as unknown as Parameters[0]) + .pipe(createGunzip()) + .on("error", reject) + .pipe(extract) + .on("error", reject); + }); +} + +function cacheDir(): string { + const explicit = process.env.WINDMILL_CACHE_DIR; + if (explicit) return explicit; + const xdg = process.env.XDG_CACHE_HOME; + if (xdg) return path.join(xdg, "windmill"); + try { + return path.join(os.homedir(), ".cache", "windmill"); + } catch { + return path.join(os.tmpdir(), "windmill"); + } +} diff --git a/cli/src/utils/local_path_scripts.ts b/cli/src/utils/local_path_scripts.ts index c9d321dd55..9e03db8a5f 100644 --- a/cli/src/utils/local_path_scripts.ts +++ b/cli/src/utils/local_path_scripts.ts @@ -1,6 +1,7 @@ import { execFileSync } from "node:child_process"; import { stat } from "node:fs/promises"; import { readTextFile } from "./utils.ts"; +import { getEsbuild } from "./esbuild_loader.ts"; import type { SyncCodebase } from "./codebase.ts"; import { parseMetadataFileIfExists } from "./metadata.ts"; import { inferContentTypeFromFilePath } from "./script_common.ts"; @@ -43,7 +44,7 @@ async function bundleSingleFileCodebaseScript( ).toString(); } - const esbuild = await import("esbuild"); + const esbuild = await getEsbuild(); const out = await esbuild.build({ entryPoints: [filePath], // Inline rawscripts are executed through the standard module wrapper, diff --git a/cli/test/esbuild_loader_unit.test.ts b/cli/test/esbuild_loader_unit.test.ts new file mode 100644 index 0000000000..9e75607dbf --- /dev/null +++ b/cli/test/esbuild_loader_unit.test.ts @@ -0,0 +1,31 @@ +/** + * Unit tests for esbuild_loader pure logic (no backend, no network). + */ + +import { expect, test, describe } from "bun:test"; +import { resolveTarEntryPath } from "../src/utils/esbuild_loader.ts"; +import { sep, resolve } from "node:path"; + +describe("resolveTarEntryPath", () => { + const dest = resolve("/tmp/cache/esbuild-wasm-0.28.0"); + + test("strips the leading package/ component and resolves inside dest", () => { + expect(resolveTarEntryPath(dest, "package/lib/main.js")).toBe( + dest + sep + "lib" + sep + "main.js" + ); + expect(resolveTarEntryPath(dest, "package/esbuild.wasm")).toBe( + dest + sep + "esbuild.wasm" + ); + }); + + test("rejects tar-slip entries that escape the dest dir", () => { + expect(resolveTarEntryPath(dest, "package/../../etc/passwd")).toBeNull(); + expect(resolveTarEntryPath(dest, "package/../../../outside")).toBeNull(); + }); + + test("rejects entries that only share a prefix with dest", () => { + // ".../esbuild-wasm-0.28.0-evil" must not be treated as inside dest + expect(resolveTarEntryPath(dest, "evil/../../esbuild-wasm-0.28.0-evil/x")) + .toBeNull(); + }); +});