mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-08-18 16:02:10 +00:00
fix(cli): fall back to esbuild-wasm on native host/binary mismatch (#9629)
* fix(cli): fall back to esbuild-wasm on native host/binary mismatch Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(cli): guard tarball extraction, extend esbuild-wasm fallback to script bundling Address CI review: prevent tar-slip in esbuild-wasm package extraction, route codebase/script and inline-rawscript bundling through getEsbuild() too, and move the loader to utils. Add a unit test for the tar-slip guard. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(cli): make esbuild-wasm fallback concurrency-safe Address CI review (P1): memoize getEsbuild() on an in-flight promise so concurrent first callers (parallel wmill sync push) share one probe/download instead of racing, and give each extraction a unique temp dir so concurrent extractions can't clobber each other. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ruben Fiszel
GitHub
@@ -28,6 +28,9 @@ rmSync(outDir, { recursive: true, force: true });
|
||||
// Build with bun — bundle everything except esbuild (platform-specific binary),
|
||||
// svelte (optional, only needed for `wmill app bundle/dev`), and parser packages
|
||||
// (loaded at runtime via init() with readFileSync for the .wasm binary).
|
||||
// esbuild-wasm is not a dependency at all: the host/binary-mismatch fallback in
|
||||
// esbuild_loader.ts downloads and caches the whole esbuild-wasm package at
|
||||
// runtime, so it stays out of the bundle and the published dependencies.
|
||||
console.log("Bundling with bun build...");
|
||||
const buildResult = Bun.spawnSync([
|
||||
"bun", "build", "src/main.ts",
|
||||
|
||||
@@ -6,6 +6,7 @@ import * as log from "../../core/log.ts";
|
||||
import { colors } from "@cliffy/ansi/colors";
|
||||
import * as windmillUtils from "@windmill-labs/shared-utils";
|
||||
import { readTextFile, readTextFileSync } from "../../utils/utils.ts";
|
||||
import { getEsbuild, stopEsbuild } from "../../utils/esbuild_loader.ts";
|
||||
export interface BundleOptions {
|
||||
entryPoint?: string;
|
||||
outDir?: string;
|
||||
@@ -170,8 +171,9 @@ export async function ensureNodeModules(appDir?: string): Promise<void> {
|
||||
export async function createBundle(
|
||||
options: BundleOptions = {}
|
||||
): Promise<BundleResult> {
|
||||
// Dynamically import esbuild
|
||||
const esbuild = await import("esbuild");
|
||||
// Native esbuild with a transparent esbuild-wasm fallback on host/binary
|
||||
// version mismatch (see esbuild_loader.ts).
|
||||
const esbuild = await getEsbuild();
|
||||
|
||||
// Detect frameworks to determine default entry point.
|
||||
// Use the entryPoint's directory if provided, otherwise fall back to cwd.
|
||||
@@ -286,6 +288,10 @@ export async function createBundle(
|
||||
outfile,
|
||||
sourcemap,
|
||||
minify,
|
||||
// Keep outputs in memory: esbuild-wasm cannot write to the filesystem
|
||||
// ("write" option unavailable), and the dist files were discarded after the
|
||||
// read anyway. Native esbuild supports write:false + outputFiles too.
|
||||
write: false as const,
|
||||
define: {
|
||||
"process.env.NODE_ENV": production ? '"production"' : '"development"',
|
||||
},
|
||||
@@ -307,29 +313,24 @@ export async function createBundle(
|
||||
|
||||
log.info(colors.green("✅ Bundle created successfully"));
|
||||
|
||||
// Read the generated files
|
||||
const jsPath = path.join(process.cwd(), outfile);
|
||||
const cssPath = path.join(process.cwd(), outDir, "bundle.css");
|
||||
const outputFiles = result.outputFiles ?? [];
|
||||
const jsFile = outputFiles.find((f) => f.path.endsWith(".js"));
|
||||
const cssFile = outputFiles.find((f) => f.path.endsWith(".css"));
|
||||
|
||||
if (!fs.existsSync(jsPath)) {
|
||||
throw new Error(`Expected JS bundle at ${jsPath} but file not found`);
|
||||
if (!jsFile) {
|
||||
throw new Error("Expected a JS bundle in esbuild output but none found");
|
||||
}
|
||||
|
||||
const jsContent = readTextFileSync(jsPath);
|
||||
const cssContent = fs.existsSync(cssPath)
|
||||
? readTextFileSync(cssPath)
|
||||
: "";
|
||||
|
||||
try {
|
||||
fs.rmSync(distDir, { recursive: true });
|
||||
} catch {
|
||||
//ignore
|
||||
}
|
||||
return { js: jsContent, css: cssContent };
|
||||
|
||||
return { js: jsFile.text, css: cssFile?.text ?? "" };
|
||||
|
||||
} finally {
|
||||
// Stop esbuild
|
||||
await esbuild.stop();
|
||||
// Stop the native esbuild service so the process can exit (no-op for wasm).
|
||||
await stopEsbuild();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -437,7 +437,11 @@ async function dev(opts: DevOptions, appFolder?: string) {
|
||||
const rawApp = (await yamlParseFile(rawAppPath)) as any;
|
||||
const appPath = rawApp?.custom_path ?? "u/unknown/newapp";
|
||||
|
||||
// Dynamically import esbuild only when the dev command is called
|
||||
// Dynamically import esbuild only when the dev command is called.
|
||||
// Native-only here (no esbuild-wasm fallback via getEsbuild): dev is a local
|
||||
// interactive command that relies on context()/watch, whose semantics under
|
||||
// wasm are untested. The host/binary-mismatch fallback covers the bundling
|
||||
// paths that run on workers/CI via `wmill sync push`.
|
||||
const esbuild = await import("esbuild");
|
||||
|
||||
const host = opts.host ?? DEFAULT_HOST;
|
||||
|
||||
@@ -58,6 +58,7 @@ import { SyncCodebase, listSyncCodebases } from "../../utils/codebase.ts";
|
||||
import { pollJobWithQueueLogging } from "../../utils/job_polling.ts";
|
||||
import fs from "node:fs";
|
||||
import { createTarBlob, type TarEntry } from "../../utils/tar.ts";
|
||||
import { getEsbuild } from "../../utils/esbuild_loader.ts";
|
||||
|
||||
import { execSync } from "node:child_process";
|
||||
import { NewScript, Script, ScriptModule } from "../../../gen/types.gen.ts";
|
||||
@@ -328,7 +329,7 @@ export async function handleFile(
|
||||
}).toString();
|
||||
log.info("Custom bundler executed for " + path);
|
||||
} else {
|
||||
const esbuild = await import("esbuild");
|
||||
const esbuild = await getEsbuild();
|
||||
|
||||
log.info(`Started bundling ${path} ...`);
|
||||
const startTime = performance.now();
|
||||
@@ -1565,7 +1566,7 @@ async function preview(
|
||||
maxBuffer: 1024 * 1024 * 50,
|
||||
}).toString();
|
||||
} else {
|
||||
const esbuild = await import("esbuild");
|
||||
const esbuild = await getEsbuild();
|
||||
|
||||
if (!opts.silent) {
|
||||
log.info(`Bundling ${filePath} for preview...`);
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
import * as fs from "node:fs";
|
||||
import * as os from "node:os";
|
||||
import * as path from "node:path";
|
||||
import process from "node:process";
|
||||
import { createGunzip } from "node:zlib";
|
||||
import { Readable } from "node:stream";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import * as tar from "tar-stream";
|
||||
import * as log from "../core/log.ts";
|
||||
|
||||
// esbuild splits into a JS host package and a per-platform native binary
|
||||
// (@esbuild/<platform>). They must be the same version. A broken or incremental
|
||||
// install can leave the on-disk binary at a different version than the pinned
|
||||
// host, which crashes service start with
|
||||
// Cannot start service: Host version "X" does not match binary version "Y"
|
||||
// The running code can't fix what npm/bun put on disk, so when that happens we
|
||||
// fall back to esbuild-wasm, whose binary is a single version-pinned .wasm. To
|
||||
// keep that 14MB out of every CLI install, the esbuild-wasm package is not a
|
||||
// dependency: it is downloaded once and cached on disk, on the fallback path
|
||||
// only. We download the whole package (not just the .wasm) because esbuild-wasm
|
||||
// reads the app's files from disk by spawning `node bin/esbuild`, which needs
|
||||
// bin/esbuild + esbuild.wasm + wasm_exec*.js co-located on disk.
|
||||
|
||||
type Esbuild = typeof import("esbuild");
|
||||
|
||||
// Version to fall back to if the native host's version can't be read. Keep in
|
||||
// sync with the "esbuild" pin in cli/package.json.
|
||||
const FALLBACK_VERSION = "0.28.0";
|
||||
|
||||
let cached: Esbuild | undefined;
|
||||
let inFlight: Promise<Esbuild> | undefined;
|
||||
// Distinguishes concurrent extraction temp dirs within a process.
|
||||
let extractCounter = 0;
|
||||
|
||||
/**
|
||||
* Returns a working esbuild module, preferring the native binary and falling
|
||||
* back to esbuild-wasm only when the native host/binary versions don't match.
|
||||
* Memoized for the process: concurrent first callers (e.g. a parallel
|
||||
* `wmill sync push`) share one probe/download instead of each running their own.
|
||||
*/
|
||||
export function getEsbuild(): Promise<Esbuild> {
|
||||
if (cached) return Promise.resolve(cached);
|
||||
if (inFlight) return inFlight;
|
||||
inFlight = acquireEsbuild()
|
||||
.then((esbuild) => {
|
||||
cached = esbuild;
|
||||
return esbuild;
|
||||
})
|
||||
.finally(() => {
|
||||
inFlight = undefined;
|
||||
});
|
||||
return inFlight;
|
||||
}
|
||||
|
||||
async function acquireEsbuild(): Promise<Esbuild> {
|
||||
// Escape hatch: skip native entirely (e.g. a host known to have a broken
|
||||
// install, or to exercise the fallback path).
|
||||
if (process.env.WINDMILL_FORCE_ESBUILD_WASM) {
|
||||
return loadWasmEsbuild(await nativeHostVersion());
|
||||
}
|
||||
|
||||
try {
|
||||
const esbuild = await import("esbuild");
|
||||
// The native service only starts on the first call; force it with the most
|
||||
// trivial op so any breakage (host/binary version mismatch, a dead service)
|
||||
// surfaces now rather than mid-build. The mismatch detail is printed to the
|
||||
// child's stderr while the thrown error is generic ("service was stopped"),
|
||||
// so we fall back on ANY smoke-test failure rather than matching a string.
|
||||
await esbuild.transform("");
|
||||
return esbuild;
|
||||
} catch (e) {
|
||||
const msg = e instanceof Error ? e.message : String(e);
|
||||
log.warn(
|
||||
`native esbuild is not usable; falling back to esbuild-wasm (${msg.trim()})`
|
||||
);
|
||||
}
|
||||
|
||||
return loadWasmEsbuild(await nativeHostVersion());
|
||||
}
|
||||
|
||||
/**
|
||||
* Stops the esbuild service (native or wasm — both spawn a child process) so the
|
||||
* process can exit. Safe to call repeatedly; the service restarts lazily on the
|
||||
* next build.
|
||||
*/
|
||||
export async function stopEsbuild(): Promise<void> {
|
||||
await cached?.stop();
|
||||
}
|
||||
|
||||
async function nativeHostVersion(): Promise<string> {
|
||||
try {
|
||||
return (await import("esbuild")).version ?? FALLBACK_VERSION;
|
||||
} catch {
|
||||
return FALLBACK_VERSION;
|
||||
}
|
||||
}
|
||||
|
||||
async function loadWasmEsbuild(version: string): Promise<Esbuild> {
|
||||
const pkgDir = await ensureWasmPackage(version);
|
||||
const mainJs = path.join(pkgDir, "lib", "main.js");
|
||||
// The Node build (lib/main.js) reads app files from disk by spawning
|
||||
// `node bin/esbuild`, so it works with on-disk entry points and node_modules,
|
||||
// unlike the browser build.
|
||||
return (await import(pathToFileURL(mainJs).href)) as unknown as Esbuild;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a directory containing an extracted esbuild-wasm package (with
|
||||
* lib/main.js). Uses an explicit override, then an on-disk cache, then downloads
|
||||
* and extracts the npm tarball.
|
||||
*/
|
||||
async function ensureWasmPackage(version: string): Promise<string> {
|
||||
// Explicit local override wins (air-gapped / self-hosted workers): a path to
|
||||
// an already-extracted esbuild-wasm package directory.
|
||||
const override = process.env.WINDMILL_ESBUILD_WASM_PATH;
|
||||
if (override) return override;
|
||||
|
||||
const destDir = path.join(cacheDir(), `esbuild-wasm-${version}`);
|
||||
if (fs.existsSync(path.join(destDir, "lib", "main.js"))) {
|
||||
return destDir;
|
||||
}
|
||||
|
||||
const url = process.env.WINDMILL_ESBUILD_WASM_URL ??
|
||||
`https://registry.npmjs.org/esbuild-wasm/-/esbuild-wasm-${version}.tgz`;
|
||||
log.info(`Downloading esbuild-wasm@${version} from ${url} ...`);
|
||||
const res = await fetch(url);
|
||||
if (!res.ok || !res.body) {
|
||||
throw new Error(
|
||||
`Failed to download esbuild-wasm@${version} (${res.status} ${res.statusText}). ` +
|
||||
`Set WINDMILL_ESBUILD_WASM_PATH to an extracted esbuild-wasm package dir, ` +
|
||||
`point WINDMILL_ESBUILD_WASM_URL at a reachable tarball, or repair the native esbuild install.`
|
||||
);
|
||||
}
|
||||
|
||||
// Extract to a unique temp dir and rename into place so a crash or a
|
||||
// concurrent writer can't leave a half-extracted package behind, and so two
|
||||
// extractions never share an in-progress directory.
|
||||
const tmpDir = `${destDir}.${process.pid}.${extractCounter++}.tmp`;
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
await extractTarball(res.body, tmpDir);
|
||||
if (!fs.existsSync(path.join(tmpDir, "lib", "main.js"))) {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
throw new Error(`esbuild-wasm@${version} tarball did not contain lib/main.js`);
|
||||
}
|
||||
try {
|
||||
fs.renameSync(tmpDir, destDir);
|
||||
} catch {
|
||||
// Another process won the race, or rename across devices failed; clean up
|
||||
// and let the existsSync check below decide whether the cache is usable.
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
}
|
||||
if (!fs.existsSync(path.join(destDir, "lib", "main.js"))) {
|
||||
throw new Error(`Failed to cache esbuild-wasm@${version} at ${destDir}`);
|
||||
}
|
||||
return destDir;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves a tar entry to an absolute path inside destDir, stripping the leading
|
||||
* "package/" component that npm tarballs use. Returns null if the entry would
|
||||
* escape destDir (tar-slip), since WINDMILL_ESBUILD_WASM_URL allows untrusted
|
||||
* tarball sources.
|
||||
*/
|
||||
export function resolveTarEntryPath(
|
||||
destDir: string,
|
||||
entryName: string
|
||||
): string | null {
|
||||
const rel = entryName.replace(/^[^/]+\//, "");
|
||||
const root = path.resolve(destDir);
|
||||
const outPath = path.resolve(root, rel);
|
||||
if (outPath !== root && !outPath.startsWith(root + path.sep)) {
|
||||
return null;
|
||||
}
|
||||
return outPath;
|
||||
}
|
||||
|
||||
// Extracts an npm tarball (gzipped tar) into destDir, stripping the leading
|
||||
// "package/" path component that npm tarballs use.
|
||||
async function extractTarball(
|
||||
body: ReadableStream<Uint8Array>,
|
||||
destDir: string
|
||||
): Promise<void> {
|
||||
const extract = tar.extract();
|
||||
extract.on("entry", (header, stream, next) => {
|
||||
if (header.type !== "file") {
|
||||
stream.resume();
|
||||
stream.on("end", next);
|
||||
return;
|
||||
}
|
||||
const outPath = resolveTarEntryPath(destDir, header.name);
|
||||
if (!outPath) {
|
||||
// Reject tar-slip entries that would write outside the cache dir.
|
||||
stream.resume();
|
||||
stream.on("end", () =>
|
||||
next(new Error(`unsafe path in esbuild-wasm tarball: ${header.name}`))
|
||||
);
|
||||
return;
|
||||
}
|
||||
fs.mkdirSync(path.dirname(outPath), { recursive: true });
|
||||
const ws = fs.createWriteStream(outPath, { mode: header.mode ?? 0o644 });
|
||||
stream.pipe(ws);
|
||||
ws.on("finish", next);
|
||||
ws.on("error", next);
|
||||
stream.on("error", next);
|
||||
});
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
extract.on("finish", resolve);
|
||||
extract.on("error", reject);
|
||||
Readable.fromWeb(body as unknown as Parameters<typeof Readable.fromWeb>[0])
|
||||
.pipe(createGunzip())
|
||||
.on("error", reject)
|
||||
.pipe(extract)
|
||||
.on("error", reject);
|
||||
});
|
||||
}
|
||||
|
||||
function cacheDir(): string {
|
||||
const explicit = process.env.WINDMILL_CACHE_DIR;
|
||||
if (explicit) return explicit;
|
||||
const xdg = process.env.XDG_CACHE_HOME;
|
||||
if (xdg) return path.join(xdg, "windmill");
|
||||
try {
|
||||
return path.join(os.homedir(), ".cache", "windmill");
|
||||
} catch {
|
||||
return path.join(os.tmpdir(), "windmill");
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { stat } from "node:fs/promises";
|
||||
import { readTextFile } from "./utils.ts";
|
||||
import { getEsbuild } from "./esbuild_loader.ts";
|
||||
import type { SyncCodebase } from "./codebase.ts";
|
||||
import { parseMetadataFileIfExists } from "./metadata.ts";
|
||||
import { inferContentTypeFromFilePath } from "./script_common.ts";
|
||||
@@ -43,7 +44,7 @@ async function bundleSingleFileCodebaseScript(
|
||||
).toString();
|
||||
}
|
||||
|
||||
const esbuild = await import("esbuild");
|
||||
const esbuild = await getEsbuild();
|
||||
const out = await esbuild.build({
|
||||
entryPoints: [filePath],
|
||||
// Inline rawscripts are executed through the standard module wrapper,
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
/**
|
||||
* Unit tests for esbuild_loader pure logic (no backend, no network).
|
||||
*/
|
||||
|
||||
import { expect, test, describe } from "bun:test";
|
||||
import { resolveTarEntryPath } from "../src/utils/esbuild_loader.ts";
|
||||
import { sep, resolve } from "node:path";
|
||||
|
||||
describe("resolveTarEntryPath", () => {
|
||||
const dest = resolve("/tmp/cache/esbuild-wasm-0.28.0");
|
||||
|
||||
test("strips the leading package/ component and resolves inside dest", () => {
|
||||
expect(resolveTarEntryPath(dest, "package/lib/main.js")).toBe(
|
||||
dest + sep + "lib" + sep + "main.js"
|
||||
);
|
||||
expect(resolveTarEntryPath(dest, "package/esbuild.wasm")).toBe(
|
||||
dest + sep + "esbuild.wasm"
|
||||
);
|
||||
});
|
||||
|
||||
test("rejects tar-slip entries that escape the dest dir", () => {
|
||||
expect(resolveTarEntryPath(dest, "package/../../etc/passwd")).toBeNull();
|
||||
expect(resolveTarEntryPath(dest, "package/../../../outside")).toBeNull();
|
||||
});
|
||||
|
||||
test("rejects entries that only share a prefix with dest", () => {
|
||||
// ".../esbuild-wasm-0.28.0-evil" must not be treated as inside dest
|
||||
expect(resolveTarEntryPath(dest, "evil/../../esbuild-wasm-0.28.0-evil/x"))
|
||||
.toBeNull();
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user