starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-18 16:02:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(cli): support better esm mode for codebases

Browse Source
This commit is contained in:
Ruben Fiszel
2025-11-27 17:21:04 +00:00
parent d876c2c31c
commit e8fd36e2e7
6 changed files with 91 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json
Generated
+1 -1
View File
@@ -15,7 +15,7 @@
]
},
"nullable": [
true
null
]
},
"hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55"
backend/.sqlx/query-a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a.json → backend/.sqlx/query-7bca61bdff25cc5e4181d6a738bf29848b2c7131b1bbc9c3a6fe121c84138662.json
Generated
+8 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT content AS \"content!: String\",\n lock AS \"lock: String\", language AS \"language: Option<ScriptLang>\", envs AS \"envs: Vec<String>\", schema AS \"schema: String\", schema_validation AS \"schema_validation: bool\", codebase LIKE '%.tar' as use_tar FROM script WHERE hash = $1 LIMIT 1",
"query": "SELECT content AS \"content!: String\",\n lock AS \"lock: String\", language AS \"language: Option<ScriptLang>\", envs AS \"envs: Vec<String>\", schema AS \"schema: String\", schema_validation AS \"schema_validation: bool\", codebase LIKE '%.tar' as use_tar, codebase LIKE '%.esm%' as is_esm FROM script WHERE hash = $1 LIMIT 1",
"describe": {
"columns": [
{
@@ -68,6 +68,11 @@
"ordinal": 6,
"name": "use_tar",
"type_info": "Bool"
},
{
"ordinal": 7,
"name": "is_esm",
"type_info": "Bool"
}
],
"parameters": {
@@ -82,8 +87,9 @@
true,
true,
false,
null,
null
]
},
"hash": "a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a"
"hash": "7bca61bdff25cc5e4181d6a738bf29848b2c7131b1bbc9c3a6fe121c84138662"
}
backend/windmill-api/src/scripts.rs
+9 -3
View File
@@ -409,7 +409,7 @@ async fn create_snapshot_script(
if name == "script" {
let ns: NewScript = Some(serde_json::from_slice(&data).map_err(to_anyhow)?).unwrap();
let is_tar = ns.codebase.as_ref().is_some_and(|x| x.ends_with(".tar"));
let use_esm = ns.codebase.as_ref().is_some_and(|x| x.contains(".esm"));
let (new_hash, ntx, hdm) = create_script_internal(
ns,
w_id.clone(),
@@ -419,8 +419,14 @@ async fn create_snapshot_script(
webhook.clone(),
)
.await?;
let nh = new_hash.to_string();
script_hash = Some(if is_tar { format!("{nh}.tar") } else { nh });
let mut nh = new_hash.to_string();
if use_esm {
nh = format!("{nh}.esm");
}
if is_tar {
nh = format!("{nh}.tar");
}
script_hash = Some(nh);
tx = Some(ntx);
handle_deployment_metadata = hdm;
}
backend/windmill-common/src/cache.rs
+9 -6
View File
@@ -631,7 +631,8 @@ pub mod script {
envs AS \"envs: Vec<String>\", \
schema AS \"schema: String\", \
schema_validation AS \"schema_validation: bool\", \
codebase LIKE '%.tar' as use_tar \
codebase LIKE '%.tar' as use_tar, \
codebase LIKE '%.esm%' as is_esm \
FROM script WHERE hash = $1 LIMIT 1",
hash.0
)
@@ -647,12 +648,14 @@ pub mod script {
language: r.language,
envs: r.envs,
codebase: if let Some(use_tar) = r.use_tar {
let sh = hash.to_string();
if use_tar {
Some(format!("{sh}.tar"))
} else {
Some(sh)
let mut sh = hash.to_string();
if r.is_esm.unwrap_or(false) {
sh = format!("{sh}.esm");
}
if use_tar {
sh = format!("{sh}.tar");
}
Some(sh)
} else {
None
},
cli/src/commands/script/script.ts
+42 -39
View File
@@ -142,7 +142,7 @@ export async function findResourceFile(path: string) {
if (validCandidates.length > 1) {
throw new Error(
"Found two resource files for the same resource" +
validCandidates.join(", ")
validCandidates.join(", ")
);
}
if (validCandidates.length < 1) {
@@ -181,11 +181,11 @@ export async function handleScriptMetadata(
}
export interface OutputFile {
path: string
contents: Uint8Array
hash: string
path: string;
contents: Uint8Array;
hash: string;
/** "contents" as text (changes automatically with "contents") */
readonly text: string
readonly text: string;
}
export async function handleFile(
@@ -223,10 +223,9 @@ export async function handleFile(
let outputFiles: OutputFile[] = [];
if (codebase.customBundler) {
log.info(`Using custom bundler ${codebase.customBundler} for ${path}`);
bundleContent = execSync(
codebase.customBundler + " " + path,
{ maxBuffer: 1024 * 1024 * 50 }
).toString();
bundleContent = execSync(codebase.customBundler + " " + path, {
maxBuffer: 1024 * 1024 * 50,
}).toString();
log.info("Custom bundler executed for " + path);
} else {
const esbuild = await import("npm:esbuild");
@@ -243,7 +242,7 @@ export async function handleFile(
inject: codebase.inject,
define: codebase.define,
loader: codebase.loader ?? { ".node": "file" },
outdir: '/',
outdir: "/",
platform: "node",
packages: "bundle",
target: format == "cjs" ? "node20.15.1" : "esnext",
@@ -260,17 +259,18 @@ export async function handleFile(
if (outputFiles.length > 1) {
const archiveNpm = await import("npm:@ayonli/jsext/archive");
log.info(
`Found multiple output files for ${path}, creating a tarball... ${outputFiles.map((file) => file.path).join(", ")}`
`Found multiple output files for ${path}, creating a tarball... ${outputFiles
.map((file) => file.path)
.join(", ")}`
);
forceTar = true;
const startTime = performance.now();
const tarball = new archiveNpm.Tarball();
const mainPath = path.split(SEP).pop()?.split(".")[0] + ".js";
const content = outputFiles.find((file) => file.path == "/" + mainPath)?.text ?? '';
const content =
outputFiles.find((file) => file.path == "/" + mainPath)?.text ?? "";
log.info(`Main content: ${content.length}chars`);
tarball.append(
new File([content], "main.js", { type: "text/plain" })
);
tarball.append(new File([content], "main.js", { type: "text/plain" }));
for (const file of outputFiles) {
if (file.path == "/" + mainPath) {
continue;
@@ -318,20 +318,20 @@ export async function handleFile(
let typed = opts?.skipScriptsMetadata
? undefined
: (
await parseMetadataFile(
remotePath,
opts
? {
...opts,
path,
workspaceRemote: workspace,
schemaOnly: codebase ? true : undefined,
globalDeps,
codebases
}
: undefined,
)
)?.payload;
await parseMetadataFile(
remotePath,
opts
? {
...opts,
path,
workspaceRemote: workspace,
schemaOnly: codebase ? true : undefined,
globalDeps,
codebases,
}
: undefined
)
)?.payload;
const workspaceId = workspace.workspaceId;
@@ -401,6 +401,7 @@ export async function handleFile(
on_behalf_of_email: typed?.on_behalf_of_email,
};
// console.log(requestBodyCommon.codebase);
// log.info(JSON.stringify(requestBodyCommon, null, 2))
// log.info(JSON.stringify(opts, null, 2))
if (remote) {
@@ -418,19 +419,19 @@ export async function handleFile(
deepEqual(typed.schema, remote.schema) &&
typed.tag == remote.tag &&
(typed.ws_error_handler_muted ?? false) ==
remote.ws_error_handler_muted &&
remote.ws_error_handler_muted &&
typed.dedicated_worker == remote.dedicated_worker &&
typed.cache_ttl == remote.cache_ttl &&
typed.concurrency_time_window_s ==
remote.concurrency_time_window_s &&
remote.concurrency_time_window_s &&
typed.concurrent_limit == remote.concurrent_limit &&
Boolean(typed.restart_unless_cancelled) ==
Boolean(remote.restart_unless_cancelled) &&
Boolean(remote.restart_unless_cancelled) &&
Boolean(typed.visible_to_runner_only) ==
Boolean(remote.visible_to_runner_only) &&
Boolean(remote.visible_to_runner_only) &&
Boolean(typed.no_main_func) == Boolean(remote.no_main_func) &&
Boolean(typed.has_preprocessor) ==
Boolean(remote.has_preprocessor) &&
Boolean(remote.has_preprocessor) &&
typed.priority == Boolean(remote.priority) &&
typed.timeout == remote.timeout &&
//@ts-ignore
@@ -523,7 +524,8 @@ async function createScript(
});
} catch (e: any) {
throw Error(
`Script creation for ${body.path} with parent ${body.parent_hash
`Script creation for ${body.path} with parent ${
body.parent_hash
} was not successful: ${e.body ?? e.message} `
);
}
@@ -549,7 +551,8 @@ async function createScript(
});
if (req.status != 201) {
throw Error(
`Script snapshot creation was not successful: ${req.status} - ${req.statusText
`Script snapshot creation was not successful: ${req.status} - ${
req.statusText
} - ${await req.text()} `
);
}
@@ -561,8 +564,8 @@ export async function findContentFile(filePath: string) {
const candidates = filePath.endsWith("script.json")
? exts.map((x) => filePath.replace(".script.json", x))
: filePath.endsWith("script.lock")
? exts.map((x) => filePath.replace(".script.lock", x))
: exts.map((x) => filePath.replace(".script.yaml", x));
? exts.map((x) => filePath.replace(".script.lock", x))
: exts.map((x) => filePath.replace(".script.yaml", x));
const validCandidates = (
await Promise.all(
@@ -581,7 +584,7 @@ export async function findContentFile(filePath: string) {
if (validCandidates.length > 1) {
throw new Error(
"No content path given and more than one candidate found: " +
validCandidates.join(", ")
validCandidates.join(", ")
);
}
if (validCandidates.length < 1) {
cli/src/utils/codebase.ts
+22 -10
View File
@@ -2,34 +2,46 @@ import { Codebase, SyncOptions } from "../core/conf.ts";
import { log } from "../../deps.ts";
import { digestDir } from "./utils.ts";
export type SyncCodebase = Codebase & { getDigest: (forceTar?: boolean) => Promise<string> };
export function listSyncCodebases(
options: SyncOptions
): SyncCodebase[] {
export type SyncCodebase = Codebase & {
getDigest: (forceTar?: boolean) => Promise<string>;
};
export function listSyncCodebases(options: SyncOptions): SyncCodebase[] {
const res: SyncCodebase[] = [];
const nb_codebase = options?.codebases?.length ?? 0;
if (nb_codebase > 0) {
log.info(`Found ${nb_codebase} codebases: ${options?.codebases?.map((c) => c.relative_path).join(", ")}`);
log.info(
`Found ${nb_codebase} codebases: ${options?.codebases
?.map((c) => c.relative_path)
.join(", ")}`
);
}
for (const codebase of options?.codebases ?? []) {
let _digest: string | undefined = undefined;
let alreadyPrinted = false;
const getDigest: (forceTar?: boolean) => Promise<string> = async (forceTar?: boolean) => {
if (_digest == undefined || forceTar) {
let hasAssets = false;
const getDigest: (forceTar?: boolean) => Promise<string> = async (
forceTar?: boolean
) => {
if (_digest == undefined) {
_digest = await digestDir(
codebase.relative_path,
JSON.stringify(codebase)
);
if (forceTar || (Array.isArray(codebase.assets) && codebase.assets.length > 0)) {
_digest += ".tar";
if (codebase.format == "esm") {
_digest += ".esm";
}
if (!alreadyPrinted) {
alreadyPrinted = true;
log.info(`Codebase ${codebase.relative_path}, digest: ${_digest}`);
}
hasAssets =
Array.isArray(codebase.assets) && codebase.assets.length > 0;
}
if (forceTar || hasAssets) {
return _digest + ".tar";
} else {
return _digest;
}
return _digest;
};
res.push({ ...codebase, getDigest });
}