diff --git a/Dockerfile b/Dockerfile index 6b4c60f6be..f056a58b01 100644 --- a/Dockerfile +++ b/Dockerfile @@ -193,6 +193,10 @@ RUN set -eux; \ ENV PATH="${PATH}:/usr/local/go/bin" ENV GO_PATH=/usr/local/go/bin/go +RUN if [ "$nsjail" = "true" ]; then apt-get -y update \ + && apt-get install -y \ + curl nodejs; fi + # go build is slower the first time it is ran, so we prewarm it in the build RUN mkdir -p /tmp/gobuildwarm && cd /tmp/gobuildwarm && go mod init gobuildwarm && printf "package foo\nimport (\"fmt\")\nfunc main() { fmt.Println(42) }" > warm.go && go build -x && rm -rf /tmp/gobuildwarm @@ -209,7 +213,7 @@ RUN chmod 755 /usr/bin/deno COPY --from=nsjail /nsjail/nsjail /bin/nsjail -COPY --from=oven/bun:1.0.18 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.0.22 /usr/local/bin/bun /usr/bin/bun # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ diff --git a/backend/windmill-worker/nsjail/run.bun.config.proto b/backend/windmill-worker/nsjail/run.bun.config.proto index a9b34e285d..d5de58cc51 100644 --- a/backend/windmill-worker/nsjail/run.bun.config.proto +++ b/backend/windmill-worker/nsjail/run.bun.config.proto @@ -1,14 +1,14 @@ -name: "bun run script" +name: "{LANG} run script" mode: ONCE -hostname: "bun" +hostname: "{LANG}" log_level: ERROR disable_rl: true mount_proc: true -cwd: "/tmp/bun" +cwd: "/tmp/{LANG}" clone_newnet: false clone_newuser: {CLONE_NEWUSER} @@ -64,14 +64,14 @@ mount { mount { src: "{JOB_DIR}/package.json" - dst: "/tmp/bun/package.json" + dst: "/tmp/{LANG}/package.json" is_bind: true mandatory: true } mount { src: "{JOB_DIR}/bun.lockb" - dst: "/tmp/bun/bun.lockb" + dst: "/tmp/{LANG}/bun.lockb" is_bind: true mandatory: false } @@ -79,7 +79,14 @@ mount { mount { src: "{JOB_DIR}/wrapper.ts" - dst: "/tmp/bun/wrapper.ts" + dst: "/tmp/{LANG}/wrapper.ts" + is_bind: true + mandatory: false +} + +mount { + src: "{JOB_DIR}/wrapper.mjs" + dst: "/tmp/{LANG}/wrapper.mjs" is_bind: true mandatory: false } @@ -87,14 +94,22 @@ mount { mount { src: "{JOB_DIR}/loader.bun.ts" - dst: "/tmp/bun/loader.bun.ts" + dst: "/tmp/{LANG}/loader.bun.ts" is_bind: true mandatory: false } +mount { + src: "{JOB_DIR}/node_modules" + dst: "/tmp/{LANG}/node_modules" + is_bind: true + mandatory: false +} + + mount { src: "{JOB_DIR}/main.ts" - dst: "/tmp/bun/main.ts" + dst: "/tmp/{LANG}/main.ts" is_bind: true mandatory: false } @@ -102,13 +117,13 @@ mount { mount { src: "{JOB_DIR}/args.json" - dst: "/tmp/bun/args.json" + dst: "/tmp/{LANG}/args.json" is_bind: true } mount { src: "{JOB_DIR}/result.json" - dst: "/tmp/bun/result.json" + dst: "/tmp/{LANG}/result.json" rw: true is_bind: true } @@ -143,6 +158,6 @@ mount { {SHARED_MOUNT} -envar: "HOME=/tmp/bun" +envar: "HOME=/tmp/{LANG}" diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index e3d021c667..3b791bbbc7 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -16,7 +16,7 @@ use crate::{ read_result, set_logs, start_child_process, write_file, write_file_binary, }, AuthedClientBackgroundTask, BUNFIG_INSTALL_SCOPES, BUN_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, - DISABLE_NUSER, HOME_ENV, NPM_CONFIG_REGISTRY, NSJAIL_PATH, PATH_ENV, TZ_ENV, + DISABLE_NUSER, HOME_ENV, NODE_PATH, NPM_CONFIG_REGISTRY, NSJAIL_PATH, PATH_ENV, TZ_ENV, }; use tokio::{ @@ -294,6 +294,15 @@ pub async fn handle_bun_job( let common_bun_proc_envs: HashMap = get_common_bun_proc_envs(&base_internal_url).await; + let nodejs_mode: bool = inner_content.starts_with("//nodejs"); + + #[cfg(feature = "enterprise")] + if nodejs_mode { + return Err(error::Error::ExecutionErr( + "Nodejs mode is an EE feature".to_string(), + )); + } + if let Some(reqs) = requirements_o { let splitted = reqs.split(BUN_LOCKB_SPLIT).collect::>(); if splitted.len() != 2 { @@ -331,7 +340,7 @@ pub async fn handle_bun_job( common_bun_proc_envs.clone(), ) .await?; - if !has_trusted_deps { + if !has_trusted_deps && !nodejs_mode { remove_dir_all(format!("{}/node_modules", job_dir)).await?; } } @@ -362,7 +371,7 @@ pub async fn handle_bun_job( .await?; // } - if empty_trusted_deps && !has_custom_config_registry { + if empty_trusted_deps && !has_custom_config_registry && !nodejs_mode { let node_modules_path = format!("{}/node_modules", job_dir); let node_modules_exists = tokio::fs::metadata(&node_modules_path).await.is_ok(); if node_modules_exists { @@ -371,7 +380,11 @@ pub async fn handle_bun_job( } } - logs.push_str("\n\n--- BUN CODE EXECUTION ---\n"); + if nodejs_mode { + logs.push_str("\n\n--- NODE CODE EXECUTION ---\n"); + } else { + logs.push_str("\n\n--- BUN CODE EXECUTION ---\n"); + } let logs_f = async { set_logs(&logs, &job.id, &db).await; @@ -446,27 +459,65 @@ run().catch(async (e) => {{ Ok(reserved_variables) as error::Result> }; - let write_loader_f = async { - write_file( - &job_dir, - "loader.bun.ts", - &format!( - r#" + let loader = RELATIVE_BUN_LOADER + .replace("W_ID", &job.workspace_id) + .replace("BASE_INTERNAL_URL", base_internal_url) + .replace("TOKEN", &client.get_token().await) + .replace("CURRENT_PATH", job.script_path()); + let write_loader_f = async move { + if nodejs_mode { + write_file( + &job_dir, + "node_builder.ts", + &format!( + r#" +{} + +import {{ readdir }} from "node:fs/promises"; + +let fileNames = [] +try {{ + fileNames = await readdir("{job_dir}/node_modules") +}} catch (e) {{ + +}} + +const bo = await Bun.build({{ + entrypoints: ["{job_dir}/wrapper.ts"], + outdir: "./", + target: "node", + plugins: [p], + external: fileNames, + }}); + +if (!bo.success) {{ + bo.logs.forEach((l) => console.log(l)); + process.exit(1); +}} +"#, + loader + ), + ) + .await?; + Ok(()) as error::Result<()> + } else { + write_file( + &job_dir, + "loader.bun.ts", + &format!( + r#" import {{ plugin }} from "bun"; {} plugin(p) "#, - RELATIVE_BUN_LOADER - .replace("W_ID", &job.workspace_id) - .replace("BASE_INTERNAL_URL", base_internal_url) - .replace("TOKEN", &client.get_token().await) - .replace("CURRENT_PATH", job.script_path()) - ), - ) - .await?; - Ok(()) as error::Result<()> + loader + ), + ) + .await?; + Ok(()) as error::Result<()> + } }; let (reserved_variables, _, _, _) = tokio::try_join!( @@ -476,12 +527,47 @@ plugin(p) write_loader_f )?; + if nodejs_mode { + let mut child = Command::new(&*BUN_PATH); + child + .current_dir(job_dir) + .env_clear() + .envs(common_bun_proc_envs.clone()) + .env("PATH", PATH_ENV.as_str()) + .args(vec!["run", "node_builder.ts"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let child_process = start_child_process(child, &*BUN_PATH).await?; + handle_child( + &job.id, + db, + logs, + mem_peak, + canceled_by, + child_process, + false, + worker_name, + &job.workspace_id, + "bun build", + job.timeout, + false, + ) + .await?; + tokio::fs::rename( + format!("{job_dir}/wrapper.js"), + format!("{job_dir}/wrapper.mjs"), + ) + .await + .map_err(|e| error::Error::InternalErr(format!("Could not move wrapper to mjs: {e}")))?; + } + //do not cache local dependencies let child = if !*DISABLE_NSJAIL { let _ = write_file( job_dir, "run.config.proto", &NSJAIL_CONFIG_RUN_BUN_CONTENT + .replace("{LANG}", if nodejs_mode { "nodejs" } else { "bun" }) .replace("{JOB_DIR}", job_dir) .replace("{CACHE_DIR}", BUN_CACHE_DIR) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) @@ -490,14 +576,16 @@ plugin(p) .await?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); - nsjail_cmd - .current_dir(job_dir) - .env_clear() - .envs(envs) - .envs(reserved_variables) - .envs(common_bun_proc_envs) - .env("PATH", PATH_ENV.as_str()) - .args(vec![ + let args = if nodejs_mode { + vec![ + "--config", + "run.config.proto", + "--", + &NODE_PATH, + "/tmp/nodejs/wrapper.mjs", + ] + } else { + vec![ "--config", "run.config.proto", "--", @@ -508,31 +596,57 @@ plugin(p) "-r", "/tmp/bun/loader.bun.ts", "/tmp/bun/wrapper.ts", - ]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await? - } else { - let script_path = format!("{job_dir}/wrapper.ts"); - let args = vec![ - "run", - "-i", - "--prefer-offline", - "-r", - "./loader.bun.ts", - &script_path, - ]; - let mut bun_cmd = Command::new(&*BUN_PATH); - bun_cmd + ] + }; + nsjail_cmd .current_dir(job_dir) .env_clear() .envs(envs) .envs(reserved_variables) .envs(common_bun_proc_envs) + .env("PATH", PATH_ENV.as_str()) .args(args) .stdout(Stdio::piped()) .stderr(Stdio::piped()); - start_child_process(bun_cmd, &*BUN_PATH).await? + start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await? + } else { + let cmd = if nodejs_mode { + let script_path = format!("{job_dir}/wrapper.mjs"); + + let mut bun_cmd = Command::new(&*NODE_PATH); + bun_cmd + .current_dir(job_dir) + .env_clear() + .envs(envs) + .envs(reserved_variables) + .envs(common_bun_proc_envs) + .args(vec![&script_path]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + bun_cmd + } else { + let script_path = format!("{job_dir}/wrapper.ts"); + + let mut bun_cmd = Command::new(&*BUN_PATH); + bun_cmd + .current_dir(job_dir) + .env_clear() + .envs(envs) + .envs(reserved_variables) + .envs(common_bun_proc_envs) + .args(vec![ + "run", + "-i", + "--prefer-offline", + "-r", + "./loader.bun.ts", + &script_path, + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + bun_cmd + }; + start_child_process(cmd, &*BUN_PATH).await? }; handle_child( diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index ef2d38153b..30593ea1ff 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -238,6 +238,7 @@ lazy_static::lazy_static! { pub static ref HTTPS_PROXY: Option = std::env::var("https_proxy").ok().or(std::env::var("HTTPS_PROXY").ok()); pub static ref DENO_PATH: String = std::env::var("DENO_PATH").unwrap_or_else(|_| "/usr/bin/deno".to_string()); pub static ref BUN_PATH: String = std::env::var("BUN_PATH").unwrap_or_else(|_| "/usr/bin/bun".to_string()); + pub static ref NODE_PATH: String = std::env::var("NODE_PATH").unwrap_or_else(|_| "/usr/bin/node".to_string()); pub static ref POWERSHELL_PATH: String = std::env::var("POWERSHELL_PATH").unwrap_or_else(|_| "/usr/bin/pwsh".to_string()); pub static ref NSJAIL_PATH: String = std::env::var("NSJAIL_PATH").unwrap_or_else(|_| "nsjail".to_string()); pub static ref PATH_ENV: String = std::env::var("PATH").unwrap_or_else(|_| String::new());