starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-18 16:02:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: experimental nodejs support

Browse Source
This commit is contained in:
Ruben Fiszel
2024-01-14 18:53:43 +01:00
parent e5678b4711
commit 1eff3fc9cc
4 changed files with 190 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Dockerfile
+5 -1
View File
@@ -193,6 +193,10 @@ RUN set -eux; \
ENV PATH="${PATH}:/usr/local/go/bin"
ENV GO_PATH=/usr/local/go/bin/go
RUN if [ "$nsjail" = "true" ]; then apt-get -y update \
&& apt-get install -y \
curl nodejs; fi
# go build is slower the first time it is ran, so we prewarm it in the build
RUN mkdir -p /tmp/gobuildwarm && cd /tmp/gobuildwarm && go mod init gobuildwarm && printf "package foo\nimport (\"fmt\")\nfunc main() { fmt.Println(42) }" > warm.go && go build -x && rm -rf /tmp/gobuildwarm
@@ -209,7 +213,7 @@ RUN chmod 755 /usr/bin/deno
COPY --from=nsjail /nsjail/nsjail /bin/nsjail
COPY --from=oven/bun:1.0.18 /usr/local/bin/bun /usr/bin/bun
COPY --from=oven/bun:1.0.22 /usr/local/bin/bun /usr/bin/bun
# add the docker client to call docker from a worker if enabled
COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/
backend/windmill-worker/nsjail/run.bun.config.proto
+26 -11
View File
@@ -1,14 +1,14 @@
name: "bun run script"
name: "{LANG} run script"
mode: ONCE
hostname: "bun"
hostname: "{LANG}"
log_level: ERROR
disable_rl: true
mount_proc: true
cwd: "/tmp/bun"
cwd: "/tmp/{LANG}"
clone_newnet: false
clone_newuser: {CLONE_NEWUSER}
@@ -64,14 +64,14 @@ mount {
mount {
src: "{JOB_DIR}/package.json"
dst: "/tmp/bun/package.json"
dst: "/tmp/{LANG}/package.json"
is_bind: true
mandatory: true
}
mount {
src: "{JOB_DIR}/bun.lockb"
dst: "/tmp/bun/bun.lockb"
dst: "/tmp/{LANG}/bun.lockb"
is_bind: true
mandatory: false
}
@@ -79,7 +79,14 @@ mount {
mount {
src: "{JOB_DIR}/wrapper.ts"
dst: "/tmp/bun/wrapper.ts"
dst: "/tmp/{LANG}/wrapper.ts"
is_bind: true
mandatory: false
}
mount {
src: "{JOB_DIR}/wrapper.mjs"
dst: "/tmp/{LANG}/wrapper.mjs"
is_bind: true
mandatory: false
}
@@ -87,14 +94,22 @@ mount {
mount {
src: "{JOB_DIR}/loader.bun.ts"
dst: "/tmp/bun/loader.bun.ts"
dst: "/tmp/{LANG}/loader.bun.ts"
is_bind: true
mandatory: false
}
mount {
src: "{JOB_DIR}/node_modules"
dst: "/tmp/{LANG}/node_modules"
is_bind: true
mandatory: false
}
mount {
src: "{JOB_DIR}/main.ts"
dst: "/tmp/bun/main.ts"
dst: "/tmp/{LANG}/main.ts"
is_bind: true
mandatory: false
}
@@ -102,13 +117,13 @@ mount {
mount {
src: "{JOB_DIR}/args.json"
dst: "/tmp/bun/args.json"
dst: "/tmp/{LANG}/args.json"
is_bind: true
}
mount {
src: "{JOB_DIR}/result.json"
dst: "/tmp/bun/result.json"
dst: "/tmp/{LANG}/result.json"
rw: true
is_bind: true
}
@@ -143,6 +158,6 @@ mount {
{SHARED_MOUNT}
envar: "HOME=/tmp/bun"
envar: "HOME=/tmp/{LANG}"
backend/windmill-worker/src/bun_executor.rs
+158 -44
View File
@@ -16,7 +16,7 @@ use crate::{
read_result, set_logs, start_child_process, write_file, write_file_binary,
},
AuthedClientBackgroundTask, BUNFIG_INSTALL_SCOPES, BUN_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL,
DISABLE_NUSER, HOME_ENV, NPM_CONFIG_REGISTRY, NSJAIL_PATH, PATH_ENV, TZ_ENV,
DISABLE_NUSER, HOME_ENV, NODE_PATH, NPM_CONFIG_REGISTRY, NSJAIL_PATH, PATH_ENV, TZ_ENV,
};
use tokio::{
@@ -294,6 +294,15 @@ pub async fn handle_bun_job(
let common_bun_proc_envs: HashMap<String, String> =
get_common_bun_proc_envs(&base_internal_url).await;
let nodejs_mode: bool = inner_content.starts_with("//nodejs");
#[cfg(feature = "enterprise")]
if nodejs_mode {
return Err(error::Error::ExecutionErr(
"Nodejs mode is an EE feature".to_string(),
));
}
if let Some(reqs) = requirements_o {
let splitted = reqs.split(BUN_LOCKB_SPLIT).collect::<Vec<&str>>();
if splitted.len() != 2 {
@@ -331,7 +340,7 @@ pub async fn handle_bun_job(
common_bun_proc_envs.clone(),
)
.await?;
if !has_trusted_deps {
if !has_trusted_deps && !nodejs_mode {
remove_dir_all(format!("{}/node_modules", job_dir)).await?;
}
}
@@ -362,7 +371,7 @@ pub async fn handle_bun_job(
.await?;
// }
if empty_trusted_deps && !has_custom_config_registry {
if empty_trusted_deps && !has_custom_config_registry && !nodejs_mode {
let node_modules_path = format!("{}/node_modules", job_dir);
let node_modules_exists = tokio::fs::metadata(&node_modules_path).await.is_ok();
if node_modules_exists {
@@ -371,7 +380,11 @@ pub async fn handle_bun_job(
}
}
logs.push_str("\n\n--- BUN CODE EXECUTION ---\n");
if nodejs_mode {
logs.push_str("\n\n--- NODE CODE EXECUTION ---\n");
} else {
logs.push_str("\n\n--- BUN CODE EXECUTION ---\n");
}
let logs_f = async {
set_logs(&logs, &job.id, &db).await;
@@ -446,27 +459,65 @@ run().catch(async (e) => {{
Ok(reserved_variables) as error::Result<HashMap<String, String>>
};
let write_loader_f = async {
write_file(
&job_dir,
"loader.bun.ts",
&format!(
r#"
let loader = RELATIVE_BUN_LOADER
.replace("W_ID", &job.workspace_id)
.replace("BASE_INTERNAL_URL", base_internal_url)
.replace("TOKEN", &client.get_token().await)
.replace("CURRENT_PATH", job.script_path());
let write_loader_f = async move {
if nodejs_mode {
write_file(
&job_dir,
"node_builder.ts",
&format!(
r#"
{}
import {{ readdir }} from "node:fs/promises";
let fileNames = []
try {{
fileNames = await readdir("{job_dir}/node_modules")
}} catch (e) {{
}}
const bo = await Bun.build({{
entrypoints: ["{job_dir}/wrapper.ts"],
outdir: "./",
target: "node",
plugins: [p],
external: fileNames,
}});
if (!bo.success) {{
bo.logs.forEach((l) => console.log(l));
process.exit(1);
}}
"#,
loader
),
)
.await?;
Ok(()) as error::Result<()>
} else {
write_file(
&job_dir,
"loader.bun.ts",
&format!(
r#"
import {{ plugin }} from "bun";
{}
plugin(p)
"#,
RELATIVE_BUN_LOADER
.replace("W_ID", &job.workspace_id)
.replace("BASE_INTERNAL_URL", base_internal_url)
.replace("TOKEN", &client.get_token().await)
.replace("CURRENT_PATH", job.script_path())
),
)
.await?;
Ok(()) as error::Result<()>
loader
),
)
.await?;
Ok(()) as error::Result<()>
}
};
let (reserved_variables, _, _, _) = tokio::try_join!(
@@ -476,12 +527,47 @@ plugin(p)
write_loader_f
)?;
if nodejs_mode {
let mut child = Command::new(&*BUN_PATH);
child
.current_dir(job_dir)
.env_clear()
.envs(common_bun_proc_envs.clone())
.env("PATH", PATH_ENV.as_str())
.args(vec!["run", "node_builder.ts"])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
let child_process = start_child_process(child, &*BUN_PATH).await?;
handle_child(
&job.id,
db,
logs,
mem_peak,
canceled_by,
child_process,
false,
worker_name,
&job.workspace_id,
"bun build",
job.timeout,
false,
)
.await?;
tokio::fs::rename(
format!("{job_dir}/wrapper.js"),
format!("{job_dir}/wrapper.mjs"),
)
.await
.map_err(|e| error::Error::InternalErr(format!("Could not move wrapper to mjs: {e}")))?;
}
//do not cache local dependencies
let child = if !*DISABLE_NSJAIL {
let _ = write_file(
job_dir,
"run.config.proto",
&NSJAIL_CONFIG_RUN_BUN_CONTENT
.replace("{LANG}", if nodejs_mode { "nodejs" } else { "bun" })
.replace("{JOB_DIR}", job_dir)
.replace("{CACHE_DIR}", BUN_CACHE_DIR)
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
@@ -490,14 +576,16 @@ plugin(p)
.await?;
let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
nsjail_cmd
.current_dir(job_dir)
.env_clear()
.envs(envs)
.envs(reserved_variables)
.envs(common_bun_proc_envs)
.env("PATH", PATH_ENV.as_str())
.args(vec![
let args = if nodejs_mode {
vec![
"--config",
"run.config.proto",
"--",
&NODE_PATH,
"/tmp/nodejs/wrapper.mjs",
]
} else {
vec![
"--config",
"run.config.proto",
"--",
@@ -508,31 +596,57 @@ plugin(p)
"-r",
"/tmp/bun/loader.bun.ts",
"/tmp/bun/wrapper.ts",
])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await?
} else {
let script_path = format!("{job_dir}/wrapper.ts");
let args = vec![
"run",
"-i",
"--prefer-offline",
"-r",
"./loader.bun.ts",
&script_path,
];
let mut bun_cmd = Command::new(&*BUN_PATH);
bun_cmd
]
};
nsjail_cmd
.current_dir(job_dir)
.env_clear()
.envs(envs)
.envs(reserved_variables)
.envs(common_bun_proc_envs)
.env("PATH", PATH_ENV.as_str())
.args(args)
.stdout(Stdio::piped())
.stderr(Stdio::piped());
start_child_process(bun_cmd, &*BUN_PATH).await?
start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await?
} else {
let cmd = if nodejs_mode {
let script_path = format!("{job_dir}/wrapper.mjs");
let mut bun_cmd = Command::new(&*NODE_PATH);
bun_cmd
.current_dir(job_dir)
.env_clear()
.envs(envs)
.envs(reserved_variables)
.envs(common_bun_proc_envs)
.args(vec![&script_path])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
bun_cmd
} else {
let script_path = format!("{job_dir}/wrapper.ts");
let mut bun_cmd = Command::new(&*BUN_PATH);
bun_cmd
.current_dir(job_dir)
.env_clear()
.envs(envs)
.envs(reserved_variables)
.envs(common_bun_proc_envs)
.args(vec![
"run",
"-i",
"--prefer-offline",
"-r",
"./loader.bun.ts",
&script_path,
])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
bun_cmd
};
start_child_process(cmd, &*BUN_PATH).await?
};
handle_child(
backend/windmill-worker/src/worker.rs
+1
View File
@@ -238,6 +238,7 @@ lazy_static::lazy_static! {
pub static ref HTTPS_PROXY: Option<String> = std::env::var("https_proxy").ok().or(std::env::var("HTTPS_PROXY").ok());
pub static ref DENO_PATH: String = std::env::var("DENO_PATH").unwrap_or_else(|_| "/usr/bin/deno".to_string());
pub static ref BUN_PATH: String = std::env::var("BUN_PATH").unwrap_or_else(|_| "/usr/bin/bun".to_string());
pub static ref NODE_PATH: String = std::env::var("NODE_PATH").unwrap_or_else(|_| "/usr/bin/node".to_string());
pub static ref POWERSHELL_PATH: String = std::env::var("POWERSHELL_PATH").unwrap_or_else(|_| "/usr/bin/pwsh".to_string());
pub static ref NSJAIL_PATH: String = std::env::var("NSJAIL_PATH").unwrap_or_else(|_| "nsjail".to_string());
pub static ref PATH_ENV: String = std::env::var("PATH").unwrap_or_else(|_| String::new());