mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-11-17 08:20:12 +00:00
fix: keep shared esbuild service alive until the last parallel build ends (#12210)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
d8773f4316
commit
8d7bd153a2
@@ -8,7 +8,7 @@ import * as log from "../../core/log.ts";
|
||||
import { colors } from "@cliffy/ansi/colors";
|
||||
import * as windmillUtils from "@windmill-labs/shared-utils";
|
||||
import { readTextFile, readTextFileSync } from "../../utils/utils.ts";
|
||||
import { getEsbuild, stopEsbuild } from "../../utils/esbuild_loader.ts";
|
||||
import { withEsbuild } from "../../utils/esbuild_loader.ts";
|
||||
export interface BundleOptions {
|
||||
entryPoint?: string;
|
||||
outDir?: string;
|
||||
@@ -371,10 +371,6 @@ export function createSharedUiPlugin(sharedUiDir: string, appDir: string): any {
|
||||
export async function createBundle(
|
||||
options: BundleOptions = {}
|
||||
): Promise<BundleResult> {
|
||||
// Native esbuild with a transparent esbuild-wasm fallback on host/binary
|
||||
// version mismatch (see esbuild_loader.ts).
|
||||
const esbuild = await getEsbuild();
|
||||
|
||||
// Detect frameworks to determine default entry point.
|
||||
// Use the entryPoint's directory if provided, otherwise fall back to cwd.
|
||||
const appDir = options.entryPoint ? path.dirname(options.entryPoint) : process.cwd();
|
||||
@@ -467,38 +463,34 @@ export async function createBundle(
|
||||
|
||||
log.info(colors.blue("📦 Building bundle..."));
|
||||
|
||||
try {
|
||||
const result = await esbuild.build(buildOptions);
|
||||
// Native esbuild with a transparent esbuild-wasm fallback on host/binary
|
||||
// version mismatch (see esbuild_loader.ts).
|
||||
const result = await withEsbuild((esbuild) => esbuild.build(buildOptions));
|
||||
|
||||
if (result.errors.length > 0) {
|
||||
log.error(colors.red("❌ Build failed:"));
|
||||
result.errors.forEach((error: any) => {
|
||||
log.error(colors.red(error.text));
|
||||
});
|
||||
throw new Error("Build failed with errors");
|
||||
}
|
||||
|
||||
log.info(colors.green("✅ Bundle created successfully"));
|
||||
|
||||
const outputFiles = result.outputFiles ?? [];
|
||||
const jsFile = outputFiles.find((f) => f.path.endsWith(".js"));
|
||||
const cssFile = outputFiles.find((f) => f.path.endsWith(".css"));
|
||||
|
||||
if (!jsFile) {
|
||||
throw new Error("Expected a JS bundle in esbuild output but none found");
|
||||
}
|
||||
|
||||
try {
|
||||
fs.rmSync(distDir, { recursive: true });
|
||||
} catch {
|
||||
//ignore
|
||||
}
|
||||
return { js: jsFile.text, css: cssFile?.text ?? "" };
|
||||
|
||||
} finally {
|
||||
// Stop the native esbuild service so the process can exit (no-op for wasm).
|
||||
await stopEsbuild();
|
||||
if (result.errors.length > 0) {
|
||||
log.error(colors.red("❌ Build failed:"));
|
||||
result.errors.forEach((error: any) => {
|
||||
log.error(colors.red(error.text));
|
||||
});
|
||||
throw new Error("Build failed with errors");
|
||||
}
|
||||
|
||||
log.info(colors.green("✅ Bundle created successfully"));
|
||||
|
||||
const outputFiles = result.outputFiles ?? [];
|
||||
const jsFile = outputFiles.find((f) => f.path.endsWith(".js"));
|
||||
const cssFile = outputFiles.find((f) => f.path.endsWith(".css"));
|
||||
|
||||
if (!jsFile) {
|
||||
throw new Error("Expected a JS bundle in esbuild output but none found");
|
||||
}
|
||||
|
||||
try {
|
||||
fs.rmSync(distDir, { recursive: true });
|
||||
} catch {
|
||||
//ignore
|
||||
}
|
||||
return { js: jsFile.text, css: cssFile?.text ?? "" };
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -69,7 +69,7 @@ import {
|
||||
import { pollJobWithQueueLogging } from "../../utils/job_polling.ts";
|
||||
import fs from "node:fs";
|
||||
import { createTarBlob, type TarEntry } from "../../utils/tar.ts";
|
||||
import { getEsbuild } from "../../utils/esbuild_loader.ts";
|
||||
import { withEsbuild } from "../../utils/esbuild_loader.ts";
|
||||
|
||||
import { execSync } from "node:child_process";
|
||||
import { NewScript, Script, ScriptModule } from "../../../gen/types.gen.ts";
|
||||
@@ -430,12 +430,10 @@ export async function handleFile(
|
||||
}).toString();
|
||||
log.info("Custom bundler executed for " + path);
|
||||
} else {
|
||||
const esbuild = await getEsbuild();
|
||||
|
||||
log.info(`Started bundling ${path} ...`);
|
||||
const startTime = performance.now();
|
||||
const format = codebase.format ?? "cjs";
|
||||
const out = await esbuild.build({
|
||||
const out = await withEsbuild((esbuild) => esbuild.build({
|
||||
entryPoints: [path],
|
||||
format: format,
|
||||
bundle: true,
|
||||
@@ -451,7 +449,7 @@ export async function handleFile(
|
||||
banner: codebase.banner,
|
||||
// ...(codebase.banner != null && { banner: codebase.banner }),
|
||||
metafile: true,
|
||||
});
|
||||
}));
|
||||
const endTime = performance.now();
|
||||
const uncovered = uncoveredBundleInputs(
|
||||
codebase,
|
||||
@@ -2001,14 +1999,12 @@ async function preview(
|
||||
maxBuffer: 1024 * 1024 * 50,
|
||||
}).toString();
|
||||
} else {
|
||||
const esbuild = await getEsbuild();
|
||||
|
||||
if (!opts.silent) {
|
||||
log.info(`Bundling ${filePath} for preview...`);
|
||||
}
|
||||
const startTime = performance.now();
|
||||
const format = codebase.format ?? "cjs";
|
||||
const out = await esbuild.build({
|
||||
const out = await withEsbuild((esbuild) => esbuild.build({
|
||||
entryPoints: [filePath],
|
||||
format: format,
|
||||
bundle: true,
|
||||
@@ -2022,7 +2018,7 @@ async function preview(
|
||||
packages: "bundle",
|
||||
target: format == "cjs" ? "node20.15.1" : "esnext",
|
||||
banner: codebase.banner,
|
||||
});
|
||||
}));
|
||||
const endTime = performance.now();
|
||||
bundledContent = out.outputFiles[0].text;
|
||||
|
||||
|
||||
@@ -38,7 +38,7 @@ let extractCounter = 0;
|
||||
* Memoized for the process: concurrent first callers (e.g. a parallel
|
||||
* `wmill sync push`) share one probe/download instead of each running their own.
|
||||
*/
|
||||
export function getEsbuild(): Promise<Esbuild> {
|
||||
function getEsbuild(): Promise<Esbuild> {
|
||||
if (cached) return Promise.resolve(cached);
|
||||
if (inFlight) return inFlight;
|
||||
inFlight = acquireEsbuild()
|
||||
@@ -78,13 +78,25 @@ async function acquireEsbuild(): Promise<Esbuild> {
|
||||
return loadWasmEsbuild(await nativeHostVersion());
|
||||
}
|
||||
|
||||
let activeUsers = 0;
|
||||
|
||||
/**
|
||||
* Stops the esbuild service (native or wasm — both spawn a child process) so the
|
||||
* process can exit. Safe to call repeatedly; the service restarts lazily on the
|
||||
* next build.
|
||||
* Runs `fn` with the shared esbuild module and stops its service (native or
|
||||
* wasm, both a child process) once no caller is using it any more. Every
|
||||
* concurrent build shares that one service, and stopping it under a pending
|
||||
* `build()` leaves the promise unsettled forever: with nothing else to do, Node
|
||||
* exits 0 mid-push. So the service must never be stopped outside this count.
|
||||
* It restarts lazily on the next call.
|
||||
*/
|
||||
export async function stopEsbuild(): Promise<void> {
|
||||
await cached?.stop();
|
||||
export async function withEsbuild<T>(
|
||||
fn: (esbuild: Esbuild) => Promise<T>
|
||||
): Promise<T> {
|
||||
activeUsers++;
|
||||
try {
|
||||
return await fn(await getEsbuild());
|
||||
} finally {
|
||||
if (--activeUsers === 0) await cached?.stop();
|
||||
}
|
||||
}
|
||||
|
||||
async function nativeHostVersion(): Promise<string> {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { stat } from "node:fs/promises";
|
||||
import { readTextFile } from "./utils.ts";
|
||||
import { getEsbuild } from "./esbuild_loader.ts";
|
||||
import { withEsbuild } from "./esbuild_loader.ts";
|
||||
import type { SyncCodebase } from "./codebase.ts";
|
||||
import { parseMetadataFileIfExists } from "./metadata.ts";
|
||||
import { inferContentTypeFromFilePath } from "./script_common.ts";
|
||||
@@ -44,8 +44,7 @@ async function bundleSingleFileCodebaseScript(
|
||||
).toString();
|
||||
}
|
||||
|
||||
const esbuild = await getEsbuild();
|
||||
const out = await esbuild.build({
|
||||
const out = await withEsbuild((esbuild) => esbuild.build({
|
||||
entryPoints: [filePath],
|
||||
// Inline rawscripts are executed through the standard module wrapper,
|
||||
// so the bundle must expose `main` as an ESM export.
|
||||
@@ -61,7 +60,7 @@ async function bundleSingleFileCodebaseScript(
|
||||
packages: "bundle",
|
||||
target: "esnext",
|
||||
banner: codebase.banner,
|
||||
});
|
||||
}));
|
||||
|
||||
if (out.outputFiles.length === 0) {
|
||||
throw new Error(`No output files found for ${filePath}`);
|
||||
|
||||
@@ -1,11 +1,45 @@
|
||||
/**
|
||||
* Unit tests for esbuild_loader pure logic (no backend, no network).
|
||||
* Unit tests for esbuild_loader (no backend, no network).
|
||||
*/
|
||||
|
||||
import { expect, test, describe } from "bun:test";
|
||||
import { resolveTarEntryPath } from "../src/utils/esbuild_loader.ts";
|
||||
import { resolveTarEntryPath, withEsbuild } from "../src/utils/esbuild_loader.ts";
|
||||
import { sep, resolve } from "node:path";
|
||||
|
||||
// A parallel `sync push` bundles several apps on the one shared service: the
|
||||
// first build to finish must not stop it under the others.
|
||||
test("a finished build does not stop the service under a concurrent one", async () => {
|
||||
let releaseSlow!: () => void;
|
||||
const slowGate = new Promise<void>((r) => (releaseSlow = r));
|
||||
const build = (name: string, gate?: Promise<void>) =>
|
||||
withEsbuild((esbuild) =>
|
||||
esbuild.build({
|
||||
entryPoints: [name],
|
||||
bundle: true,
|
||||
write: false,
|
||||
plugins: [{
|
||||
name: "virtual",
|
||||
setup(b) {
|
||||
b.onResolve({ filter: /.*/ }, (a) => ({ path: a.path, namespace: "v" }));
|
||||
b.onLoad({ filter: /.*/, namespace: "v" }, async () => {
|
||||
await gate;
|
||||
return { contents: `export const n = ${JSON.stringify(name)};`, loader: "js" };
|
||||
});
|
||||
},
|
||||
}],
|
||||
})
|
||||
);
|
||||
|
||||
const slow = build("slow", slowGate);
|
||||
await build("fast");
|
||||
releaseSlow();
|
||||
const out = await Promise.race([
|
||||
slow,
|
||||
new Promise<never>((_, rej) => setTimeout(() => rej(new Error("slow build never settled")), 5000)),
|
||||
]);
|
||||
expect(out.outputFiles[0].text).toContain('"slow"');
|
||||
});
|
||||
|
||||
describe("resolveTarEntryPath", () => {
|
||||
const dest = resolve("/tmp/cache/esbuild-wasm-0.28.0");
|
||||
|
||||
|
||||
Reference in New Issue
Block a user