import * as fs from "node:fs"; import * as os from "node:os"; import * as path from "node:path"; import process from "node:process"; import { createGunzip } from "node:zlib"; import { Readable } from "node:stream"; import { pathToFileURL } from "node:url"; import * as tar from "tar-stream"; import * as log from "../core/log.ts"; // esbuild splits into a JS host package and a per-platform native binary // (@esbuild/). They must be the same version. A broken or incremental // install can leave the on-disk binary at a different version than the pinned // host, which crashes service start with // Cannot start service: Host version "X" does not match binary version "Y" // The running code can't fix what npm/bun put on disk, so when that happens we // fall back to esbuild-wasm, whose binary is a single version-pinned .wasm. To // keep that 14MB out of every CLI install, the esbuild-wasm package is not a // dependency: it is downloaded once and cached on disk, on the fallback path // only. We download the whole package (not just the .wasm) because esbuild-wasm // reads the app's files from disk by spawning `node bin/esbuild`, which needs // bin/esbuild + esbuild.wasm + wasm_exec*.js co-located on disk. type Esbuild = typeof import("esbuild"); // Version to fall back to if the native host's version can't be read. Keep in // sync with the "esbuild" pin in cli/package.json. const FALLBACK_VERSION = "0.28.2"; let cached: Esbuild | undefined; let inFlight: Promise | undefined; // Distinguishes concurrent extraction temp dirs within a process. let extractCounter = 0; /** * Returns a working esbuild module, preferring the native binary and falling * back to esbuild-wasm only when the native host/binary versions don't match. * Memoized for the process: concurrent first callers (e.g. a parallel * `wmill sync push`) share one probe/download instead of each running their own. */ export function getEsbuild(): Promise { if (cached) return Promise.resolve(cached); if (inFlight) return inFlight; inFlight = acquireEsbuild() .then((esbuild) => { cached = esbuild; return esbuild; }) .finally(() => { inFlight = undefined; }); return inFlight; } async function acquireEsbuild(): Promise { // Escape hatch: skip native entirely (e.g. a host known to have a broken // install, or to exercise the fallback path). if (process.env.WINDMILL_FORCE_ESBUILD_WASM) { return loadWasmEsbuild(await nativeHostVersion()); } try { const esbuild = await import("esbuild"); // The native service only starts on the first call; force it with the most // trivial op so any breakage (host/binary version mismatch, a dead service) // surfaces now rather than mid-build. The mismatch detail is printed to the // child's stderr while the thrown error is generic ("service was stopped"), // so we fall back on ANY smoke-test failure rather than matching a string. await esbuild.transform(""); return esbuild; } catch (e) { const msg = e instanceof Error ? e.message : String(e); log.warn( `native esbuild is not usable; falling back to esbuild-wasm (${msg.trim()})` ); } return loadWasmEsbuild(await nativeHostVersion()); } /** * Stops the esbuild service (native or wasm — both spawn a child process) so the * process can exit. Safe to call repeatedly; the service restarts lazily on the * next build. */ export async function stopEsbuild(): Promise { await cached?.stop(); } async function nativeHostVersion(): Promise { try { return (await import("esbuild")).version ?? FALLBACK_VERSION; } catch { return FALLBACK_VERSION; } } async function loadWasmEsbuild(version: string): Promise { const pkgDir = await ensureWasmPackage(version); const mainJs = path.join(pkgDir, "lib", "main.js"); // The Node build (lib/main.js) reads app files from disk by spawning // `node bin/esbuild`, so it works with on-disk entry points and node_modules, // unlike the browser build. return (await import(pathToFileURL(mainJs).href)) as unknown as Esbuild; } /** * Returns a directory containing an extracted esbuild-wasm package (with * lib/main.js). Uses an explicit override, then an on-disk cache, then downloads * and extracts the npm tarball. */ async function ensureWasmPackage(version: string): Promise { // Explicit local override wins (air-gapped / self-hosted workers): a path to // an already-extracted esbuild-wasm package directory. const override = process.env.WINDMILL_ESBUILD_WASM_PATH; if (override) return override; const destDir = path.join(cacheDir(), `esbuild-wasm-${version}`); if (fs.existsSync(path.join(destDir, "lib", "main.js"))) { return destDir; } const url = process.env.WINDMILL_ESBUILD_WASM_URL ?? `https://registry.npmjs.org/esbuild-wasm/-/esbuild-wasm-${version}.tgz`; log.info(`Downloading esbuild-wasm@${version} from ${url} ...`); const res = await fetch(url); if (!res.ok || !res.body) { throw new Error( `Failed to download esbuild-wasm@${version} (${res.status} ${res.statusText}). ` + `Set WINDMILL_ESBUILD_WASM_PATH to an extracted esbuild-wasm package dir, ` + `point WINDMILL_ESBUILD_WASM_URL at a reachable tarball, or repair the native esbuild install.` ); } // Extract to a unique temp dir and rename into place so a crash or a // concurrent writer can't leave a half-extracted package behind, and so two // extractions never share an in-progress directory. const tmpDir = `${destDir}.${process.pid}.${extractCounter++}.tmp`; fs.rmSync(tmpDir, { recursive: true, force: true }); await extractTarball(res.body, tmpDir); if (!fs.existsSync(path.join(tmpDir, "lib", "main.js"))) { fs.rmSync(tmpDir, { recursive: true, force: true }); throw new Error(`esbuild-wasm@${version} tarball did not contain lib/main.js`); } try { fs.renameSync(tmpDir, destDir); } catch { // Another process won the race, or rename across devices failed; clean up // and let the existsSync check below decide whether the cache is usable. fs.rmSync(tmpDir, { recursive: true, force: true }); } if (!fs.existsSync(path.join(destDir, "lib", "main.js"))) { throw new Error(`Failed to cache esbuild-wasm@${version} at ${destDir}`); } return destDir; } /** * Resolves a tar entry to an absolute path inside destDir, stripping the leading * "package/" component that npm tarballs use. Returns null if the entry would * escape destDir (tar-slip), since WINDMILL_ESBUILD_WASM_URL allows untrusted * tarball sources. */ export function resolveTarEntryPath( destDir: string, entryName: string ): string | null { const rel = entryName.replace(/^[^/]+\//, ""); const root = path.resolve(destDir); const outPath = path.resolve(root, rel); if (outPath !== root && !outPath.startsWith(root + path.sep)) { return null; } return outPath; } // Extracts an npm tarball (gzipped tar) into destDir, stripping the leading // "package/" path component that npm tarballs use. async function extractTarball( body: ReadableStream, destDir: string ): Promise { const extract = tar.extract(); extract.on("entry", (header, stream, next) => { if (header.type !== "file") { stream.resume(); stream.on("end", next); return; } const outPath = resolveTarEntryPath(destDir, header.name); if (!outPath) { // Reject tar-slip entries that would write outside the cache dir. stream.resume(); stream.on("end", () => next(new Error(`unsafe path in esbuild-wasm tarball: ${header.name}`)) ); return; } fs.mkdirSync(path.dirname(outPath), { recursive: true }); const ws = fs.createWriteStream(outPath, { mode: header.mode ?? 0o644 }); stream.pipe(ws); ws.on("finish", next); ws.on("error", next); stream.on("error", next); }); await new Promise((resolve, reject) => { extract.on("finish", resolve); extract.on("error", reject); Readable.fromWeb(body as unknown as Parameters[0]) .pipe(createGunzip()) .on("error", reject) .pipe(extract) .on("error", reject); }); } function cacheDir(): string { const explicit = process.env.WINDMILL_CACHE_DIR; if (explicit) return explicit; const xdg = process.env.XDG_CACHE_HOME; if (xdg) return path.join(xdg, "windmill"); try { return path.join(os.homedir(), ".cache", "windmill"); } catch { return path.join(os.tmpdir(), "windmill"); } }