diff --git a/backend/src/main.rs b/backend/src/main.rs index d9ae5c6e58..432095580d 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -66,12 +66,11 @@ use windmill_common::METRICS_ADDR; use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING; use windmill_worker::{ - get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, - BUN_DEPSTAR_CACHE_DIR, CSHARP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, - DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR, - POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, PY312_CACHE_DIR, PY313_CACHE_DIR, - RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TAR_PY310_CACHE_DIR, TAR_PY311_CACHE_DIR, - TAR_PY312_CACHE_DIR, TAR_PY313_CACHE_DIR, UV_CACHE_DIR, + get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, CSHARP_CACHE_DIR, + DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, + LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, + PY312_CACHE_DIR, PY313_CACHE_DIR, RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TAR_PY310_CACHE_DIR, + TAR_PY311_CACHE_DIR, TAR_PY312_CACHE_DIR, TAR_PY313_CACHE_DIR, UV_CACHE_DIR, }; use crate::monitor::{ @@ -1042,7 +1041,6 @@ pub async fn run_workers( TAR_PY312_CACHE_DIR, TAR_PY313_CACHE_DIR, PIP_CACHE_DIR, - BUN_DEPSTAR_CACHE_DIR, BUN_BUNDLE_CACHE_DIR, GO_CACHE_DIR, GO_BIN_CACHE_DIR, diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 4fc471df91..134511472f 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -1,14 +1,12 @@ #[cfg(feature = "deno_core")] use std::time::Instant; -use std::{collections::HashMap, fs, path::Path, process::Stdio}; +use std::{collections::HashMap, fs, process::Stdio}; -use anyhow::Context; use base64::Engine; use itertools::Itertools; use serde_json::value::RawValue; -use sha2::Digest; use uuid::Uuid; use windmill_parser_ts::remove_pinned_imports; use windmill_queue::{append_logs, CanceledBy}; @@ -23,8 +21,8 @@ use crate::{ }, handle_child::handle_child, AuthedClientBackgroundTask, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, - BUN_DEPSTAR_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, - NODE_PATH, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TZ_ENV, + BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, NODE_PATH, + NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, TZ_ENV, }; #[cfg(windows)] @@ -637,51 +635,6 @@ pub async fn pull_codebase(_w_id: &str, _id: &str, _job_dir: &str) -> Result<()> )); } -#[cfg(unix)] -pub fn copy_recursively( - source: impl AsRef, - destination: impl AsRef, - skip: Option<&Vec>, -) -> Result<()> { - let mut stack = Vec::new(); - stack.push(( - source.as_ref().to_path_buf(), - destination.as_ref().to_path_buf(), - 0, - )); - while let Some((current_source, current_destination, level)) = stack.pop() { - for entry in fs::read_dir(¤t_source) - .context(format!("reading directory {current_source:?}"))? - { - let entry = entry?; - let filetype = entry.file_type()?; - let destination = current_destination.join(entry.file_name()); - if level == 0 { - if let Some(skip) = skip { - if skip.contains(&entry.file_name().to_string_lossy().to_string()) { - continue; - } - } - } - - let original = entry.path(); - - if filetype.is_dir() { - fs::create_dir_all(&destination)?; - stack.push((entry.path(), destination, level + 1)); - } else { - fs::hard_link(&original, &destination).map_err(|e| { - error::Error::internal_err(format!( - "hard linking from {original:?} to {destination:?}: {e:#}" - )) - })?; - } - } - } - - Ok(()) -} - pub async fn prebundle_bun_script( inner_content: &str, lockfile: Option<&String>, @@ -889,7 +842,6 @@ pub async fn handle_bun_job( )); } - let mut gbuntar_name: Option = None; if has_bundle_cache { let target; let symlink; @@ -924,67 +876,23 @@ pub async fn handle_bun_job( let _ = write_file(job_dir, "package.json", pkg)?; let lock = if annotation.npm { "" } else { lock.unwrap() }; if !empty { - let mut skip_install = false; - let mut create_buntar = false; - let mut buntar_path = "".to_string(); - if !annotation.npm { let _ = write_lock(lock, job_dir, is_binary).await?; - - let mut sha_path = sha2::Sha256::new(); - sha_path.update(lock.as_bytes()); - - let buntar_name = - base64::engine::general_purpose::URL_SAFE.encode(sha_path.finalize()); - buntar_path = format!("{BUN_DEPSTAR_CACHE_DIR}/{buntar_name}"); - - #[cfg(unix)] - if tokio::fs::metadata(&buntar_path).await.is_ok() { - if let Err(e) = copy_recursively(&buntar_path, job_dir, None) { - tracing::error!("Could not extract buntar: {e:#}"); - } else { - gbuntar_name = Some(buntar_name.clone()); - skip_install = true; - } - } else { - create_buntar = true; - } } - if !skip_install { - install_bun_lockfile( - mem_peak, - canceled_by, - &job.id, - &job.workspace_id, - Some(db), - job_dir, - worker_name, - common_bun_proc_envs.clone(), - annotation.npm, - &mut Some(occupancy_metrics), - ) - .await?; - - #[cfg(unix)] - if create_buntar { - fs::create_dir_all(&buntar_path)?; - if let Err(e) = copy_recursively( - job_dir, - &buntar_path, - Some(&vec![ - "main.ts".to_string(), - "package.json".to_string(), - if is_binary { "bun.lockb" } else { "bun.lock" }.to_string(), - "shared".to_string(), - "bunfig.toml".to_string(), - ]), - ) { - fs::remove_dir_all(&buntar_path).context("deleting buntar directory")?; - tracing::error!("Could not create buntar: {e}"); - } - } - } + install_bun_lockfile( + mem_peak, + canceled_by, + &job.id, + &job.workspace_id, + Some(db), + job_dir, + worker_name, + common_bun_proc_envs.clone(), + annotation.npm, + &mut Some(occupancy_metrics), + ) + .await?; } } else { // if !*DISABLE_NSJAIL || !empty_trusted_deps || has_custom_config_registry { @@ -1031,13 +939,6 @@ pub async fn handle_bun_job( "\n\n--- BUN CODE EXECUTION ---\n".to_string() }; - if let Some(gbuntar_name) = gbuntar_name { - init_logs = format!( - "\nskipping install, using cached buntar based on lockfile hash: {gbuntar_name}{}", - init_logs - ); - } - if has_bundle_cache { init_logs = format!("\n{}{}", cache_logs, init_logs); } diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index f523ce0723..5592e0d722 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -291,7 +291,6 @@ pub const RUST_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "rust"); pub const CSHARP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "csharp"); pub const BUN_CACHE_DIR: &str = concatcp!(ROOT_CACHE_NOMOUNT_DIR, "bun"); pub const BUN_BUNDLE_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "bun"); -pub const BUN_DEPSTAR_CACHE_DIR: &str = concatcp!(ROOT_CACHE_NOMOUNT_DIR, "buntar"); pub const GO_BIN_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "gobin"); pub const POWERSHELL_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "powershell");