From 52df2650ea5d5c03e94c96af0b8a79275856fc37 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 29 Sep 2023 20:27:13 +0200 Subject: [PATCH] feat: add trustedDependencies escape hatch for bun (#2364) * trustedDeps * fix frontend * trustedDeps * fix frontend --- backend/windmill-worker/loader_builder.bun.ts | 2 +- backend/windmill-worker/src/bun_executor.rs | 132 ++++++++++++++---- backend/windmill-worker/src/worker.rs | 5 +- .../src/routes/(root)/(logged)/+layout.svelte | 5 +- 4 files changed, 110 insertions(+), 34 deletions(-) diff --git a/backend/windmill-worker/loader_builder.bun.ts b/backend/windmill-worker/loader_builder.bun.ts index eca8c9ef81..89fe438bcb 100644 --- a/backend/windmill-worker/loader_builder.bun.ts +++ b/backend/windmill-worker/loader_builder.bun.ts @@ -14,7 +14,7 @@ if (!bo.success) { process.exit(1); } else { let content = await fs.readFile("./out/main.js", { encoding: "utf8" }); - const imports = new Bun.Transpiler().scanImports(content); + const imports = new Bun.Transpiler().scanImports(content.replaceAll("__require", "require")); const { intersect } = require("semver-intersect"); const dependencies: Record = {}; diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index f7c033d357..23bc1415ba 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -8,6 +8,7 @@ use anyhow::Context; use base64::Engine; use itertools::Itertools; +use regex::Regex; use uuid::Uuid; #[cfg(feature = "enterprise")] @@ -42,7 +43,7 @@ use tokio::sync::mpsc::{Receiver, Sender}; use windmill_common::variables; use windmill_common::{ - error::{self, Result}, + error::{self, to_anyhow, Result}, jobs::QueuedJob, }; use windmill_parser::Typ; @@ -56,6 +57,10 @@ const NSJAIL_CONFIG_RUN_BUN_CONTENT: &str = include_str!("../nsjail/run.bun.conf pub const BUN_LOCKB_SPLIT: &str = "\n//bun.lockb\n"; pub const EMPTY_FILE: &str = ""; +lazy_static::lazy_static! { + pub static ref TRUSTED_DEP: Regex = Regex::new(r"//\s?trustedDependencies:(.*)\n").unwrap(); +} + pub async fn gen_lockfile( logs: &mut String, job_id: &Uuid, @@ -67,6 +72,7 @@ pub async fn gen_lockfile( base_internal_url: &str, worker_name: &str, export_pkg: bool, + trusted_deps: Vec, ) -> Result> { let _ = write_file( &job_dir, @@ -112,6 +118,37 @@ pub async fn gen_lockfile( ) .await?; + if trusted_deps.len() > 0 { + logs.push_str(&format!( + "\ndetected trustedDependencies: {}\n", + trusted_deps.join(", ") + )); + let mut content = "".to_string(); + { + let mut file = File::open(format!("{job_dir}/package.json")).await?; + file.read_to_string(&mut content).await?; + } + let mut value = + serde_json::from_str::>(&content) + .map_err(to_anyhow)?; + value.insert( + "trustedDependencies".to_string(), + serde_json::json!(trusted_deps), + ); + write_file( + job_dir, + "package.json", + &serde_json::to_string(&value).map_err(to_anyhow)?, + ) + .await?; + + let mut content = "".to_string(); + { + let mut file = File::open(format!("{job_dir}/package.json")).await?; + file.read_to_string(&mut content).await?; + } + } + install_lockfile( logs, job_id, @@ -181,6 +218,23 @@ pub async fn install_lockfile( Ok(()) } +pub fn get_trusted_deps(code: &str) -> Vec { + // postinstall not allowed with nsjail + if !*DISABLE_NSJAIL { + return vec![]; + } + TRUSTED_DEP + .captures(code) + .map(|x| { + x[1].to_string() + .trim() + .split(',') + .map(|y| y.trim().to_string()) + .collect_vec() + }) + .unwrap_or_default() +} + #[tracing::instrument(level = "trace", skip_all)] pub async fn handle_bun_job( requirements_o: Option, @@ -196,6 +250,7 @@ pub async fn handle_bun_job( shared_mount: &str, ) -> error::Result { let _ = write_file(job_dir, "main.ts", inner_content).await?; + let common_bun_proc_envs: HashMap = get_common_bun_proc_envs(&base_internal_url); @@ -209,16 +264,20 @@ pub async fn handle_bun_job( let _ = write_file(job_dir, "package.json", &splitted[0]).await?; let lockb = splitted[1]; if lockb != EMPTY_FILE { - let _ = write_file_binary( - job_dir, - "bun.lockb", - &base64::engine::general_purpose::STANDARD - .decode(&splitted[1]) - .map_err(|_| { - error::Error::InternalErr("Could not decode bun.lockb".to_string()) - })?, - ) - .await?; + let has_trusted_deps = &splitted[0].contains("trustedDependencies"); + + if !has_trusted_deps { + let _ = write_file_binary( + job_dir, + "bun.lockb", + &base64::engine::general_purpose::STANDARD + .decode(&splitted[1]) + .map_err(|_| { + error::Error::InternalErr("Could not decode bun.lockb".to_string()) + })?, + ) + .await?; + } install_lockfile( logs, @@ -230,25 +289,40 @@ pub async fn handle_bun_job( common_bun_proc_envs.clone(), ) .await?; - remove_dir_all(format!("{}/node_modules", job_dir)).await?; + if !has_trusted_deps { + remove_dir_all(format!("{}/node_modules", job_dir)).await?; + } + } + } else { + // TODO: remove once bun implement a reasonable set of trusted deps + let trusted_deps = get_trusted_deps(inner_content); + let empty_trusted_deps = trusted_deps.len() == 0; + if !*DISABLE_NSJAIL || !empty_trusted_deps { + logs.push_str("\n\n--- BUN INSTALL ---\n"); + set_logs(&logs, &job.id, &db).await; + let _ = gen_lockfile( + logs, + &job.id, + &job.workspace_id, + db, + &client.get_token().await, + &job.script_path(), + job_dir, + base_internal_url, + worker_name, + false, + trusted_deps, + ) + .await?; + } + + if empty_trusted_deps { + let node_modules_path = format!("{}/node_modules", job_dir); + let node_modules_exists = tokio::fs::metadata(&node_modules_path).await.is_ok(); + if node_modules_exists { + remove_dir_all(&node_modules_path).await?; + } } - } else if !*DISABLE_NSJAIL { - logs.push_str("\n\n--- BUN INSTALL ---\n"); - set_logs(&logs, &job.id, &db).await; - let _ = gen_lockfile( - logs, - &job.id, - &job.workspace_id, - db, - &client.get_token().await, - &job.script_path(), - job_dir, - base_internal_url, - worker_name, - false, - ) - .await?; - remove_dir_all(format!("{}/node_modules", job_dir)).await?; } logs.push_str("\n\n--- BUN CODE EXECUTION ---\n"); diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index cb717233ac..1cf0316a3d 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -68,7 +68,7 @@ use windmill_queue::{add_completed_job, add_completed_job_error}; use crate::{ bash_executor::{handle_bash_job, handle_powershell_job, ANSI_ESCAPE_RE}, - bun_executor::{gen_lockfile, handle_bun_job}, + bun_executor::{gen_lockfile, get_trusted_deps, handle_bun_job}, common::{hash_args, read_result, save_in_cache, transform_json_value, write_file}, deno_executor::{generate_deno_lock, handle_deno_job}, go_executor::{handle_go_job, install_go_dependencies}, @@ -2671,6 +2671,8 @@ async fn capture_dependency_job( } ScriptLang::Bun => { let _ = write_file(job_dir, "main.ts", job_raw_code).await?; + //TODO: remove once bun provides sane default fot it + let trusted_deps = get_trusted_deps(job_raw_code); let req = gen_lockfile( logs, job_id, @@ -2682,6 +2684,7 @@ async fn capture_dependency_job( base_internal_url, worker_name, true, + trusted_deps, ) .await?; Ok(req.unwrap_or_else(String::new)) diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index 8baca94fa4..93bb616e51 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -12,7 +12,7 @@ ScriptService, UserService } from '$lib/gen' - import { classNames, sendUserToast } from '$lib/utils' + import { classNames } from '$lib/utils' import WorkspaceMenu from '$lib/components/sidebar/WorkspaceMenu.svelte' import SidebarContent from '$lib/components/sidebar/SidebarContent.svelte' @@ -20,7 +20,6 @@ enterpriseLicense, starStore, superadmin, - tutorialsToDo, usageStore, userStore, workspaceStore @@ -35,7 +34,7 @@ import { SUPERADMIN_SETTINGS_HASH, USER_SETTINGS_HASH } from '$lib/components/sidebar/settings' import { isCloudHosted } from '$lib/cloud' import MultiplayerMenu from '$lib/components/sidebar/MultiplayerMenu.svelte' - import { syncTutorialsTodos, updateProgress } from '$lib/tutorialUtils' + import { syncTutorialsTodos } from '$lib/tutorialUtils' OpenAPI.WITH_CREDENTIALS = true let menuOpen = false