diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 2170eaf760..30ba46c4c1 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -39,7 +39,7 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cipher", "cpufeatures", "opaque-debug", @@ -62,7 +62,7 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "const-random", "getrandom 0.2.15", "once_cell", @@ -986,7 +986,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ "addr2line", - "cfg-if 1.0.0", + "cfg-if", "libc", "miniz_oxide 0.8.2", "object", @@ -1210,7 +1210,7 @@ dependencies = [ "arrayref", "arrayvec", "cc", - "cfg-if 1.0.0", + "cfg-if", "constant_time_eq", ] @@ -1554,12 +1554,6 @@ dependencies = [ "nom", ] -[[package]] -name = "cfg-if" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" - [[package]] name = "cfg-if" version = "1.0.0" @@ -1750,7 +1744,7 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "wasm-bindgen", ] @@ -1914,7 +1908,7 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -2048,7 +2042,7 @@ version = "4.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "curve25519-dalek-derive", "digest 0.10.7", @@ -2145,7 +2139,7 @@ version = "5.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "hashbrown 0.14.5", "lock_api", "once_cell", @@ -2908,7 +2902,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "dirs-sys-next", ] @@ -3121,7 +3115,7 @@ version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", ] [[package]] @@ -3172,15 +3166,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "error-chain" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d2f06b9cac1506ece98fe3231e3cc9c4410ec3d5b1f24ae1c8946f0742cdefc" -dependencies = [ - "version_check", -] - [[package]] name = "error_reporter" version = "1.0.0" @@ -3202,7 +3187,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "home", "windows-sys 0.48.0", ] @@ -3273,7 +3258,7 @@ version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "libredox", "windows-sys 0.59.0", @@ -3759,7 +3744,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "wasi 0.9.0+wasi-snapshot-preview1", ] @@ -3770,7 +3755,7 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", @@ -3911,7 +3896,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" dependencies = [ "bytemuck", - "cfg-if 1.0.0", + "cfg-if", "crunchy", "num-traits", "rand 0.8.5", @@ -4014,7 +3999,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "447afdcdb8afb9d0a852af6dc65d9b285ce720ed7a59e42a8bf2e931c67bc1b5" dependencies = [ "async-trait", - "cfg-if 1.0.0", + "cfg-if", "data-encoding", "enum-as-inner", "futures-channel", @@ -4038,7 +4023,7 @@ version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a2e2aba9c389ce5267d31cf1e4dace82390ae276b0b364ea55630b1fa1b44b4" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "futures-util", "hickory-proto", "ipconfig", @@ -4600,7 +4585,7 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "js-sys", "wasm-bindgen", "web-sys", @@ -4861,7 +4846,7 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "windows-targets 0.52.6", ] @@ -5175,7 +5160,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "digest 0.10.7", ] @@ -5420,19 +5405,6 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" -[[package]] -name = "nix" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50e4785f2c3b7589a0d0c1dd60285e1188adac4006e8abd6dd578e1567027363" -dependencies = [ - "bitflags 1.3.2", - "cc", - "cfg-if 0.1.10", - "libc", - "void", -] - [[package]] name = "nix" version = "0.27.1" @@ -5440,7 +5412,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" dependencies = [ "bitflags 2.6.0", - "cfg-if 1.0.0", + "cfg-if", "libc", ] @@ -5660,18 +5632,6 @@ dependencies = [ "walkdir", ] -[[package]] -name = "ofiles" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6d996a29712d694df329d19f0e18632466a42c9de4aab73dca087a1a4c1bd7d" -dependencies = [ - "error-chain", - "glob", - "log", - "nix 0.17.0", -] - [[package]] name = "once_cell" version = "1.20.2" @@ -5761,7 +5721,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" dependencies = [ "bitflags 2.6.0", - "cfg-if 1.0.0", + "cfg-if", "foreign-types", "libc", "once_cell", @@ -5980,7 +5940,7 @@ version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "libc", "redox_syscall 0.5.8", "smallvec", @@ -6473,7 +6433,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "fnv", "lazy_static", "memchr", @@ -7099,7 +7059,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", - "cfg-if 1.0.0", + "cfg-if", "getrandom 0.2.15", "libc", "spin 0.9.8", @@ -7897,7 +7857,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "digest 0.10.7", ] @@ -7909,7 +7869,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ "block-buffer 0.9.0", - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "digest 0.9.0", "opaque-debug", @@ -7921,7 +7881,7 @@ version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "cpufeatures", "digest 0.10.7", ] @@ -8442,7 +8402,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "799c883d55abdb5e98af1a7b3f23b9b6de8ecada0ecac058672d7635eb48ca7b" dependencies = [ "cc", - "cfg-if 1.0.0", + "cfg-if", "libc", "psm", "windows-sys 0.59.0", @@ -8602,7 +8562,7 @@ checksum = "12d0a8eaaf1606c9207077d75828008cb2dfb51b095a766bd2b72ef893576e31" dependencies = [ "ast_node", "better_scoped_tls", - "cfg-if 1.0.0", + "cfg-if", "either", "from_variant", "new_debug_unreachable", @@ -9190,7 +9150,7 @@ version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "fastrand 2.3.0", "once_cell", "rustix", @@ -9261,7 +9221,7 @@ version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "once_cell", ] @@ -10029,7 +9989,7 @@ version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "rand 0.8.5", "static_assertions", ] @@ -10423,12 +10383,6 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "void" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" - [[package]] name = "vsimd" version = "0.8.0" @@ -10484,7 +10438,7 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "wasm-bindgen-macro", ] @@ -10509,7 +10463,7 @@ version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "js-sys", "wasm-bindgen", "web-sys", @@ -11236,9 +11190,8 @@ dependencies = [ "mappable-rc", "mysql_async", "native-tls", - "nix 0.27.1", + "nix", "object_store", - "ofiles", "once_cell", "opentelemetry", "pem 3.0.4", @@ -11490,7 +11443,7 @@ version = "0.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ - "cfg-if 1.0.0", + "cfg-if", "windows-sys 0.48.0", ] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index aa3d440e21..8e86f63caf 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -199,7 +199,6 @@ json-pointer = "^0" itertools = "^0" regex = "^1" semver = "^1" -ofiles = "0.2.0" deno_fetch = "0.203.0" deno_tls = "0.166.0" diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index b29c1691ac..890d3f3a70 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -69,7 +69,6 @@ dyn-iter.workspace = true once_cell.workspace = true tokio-postgres.workspace = true bit-vec.workspace = true -ofiles.workspace = true deno_fetch = { workspace = true, optional = true } deno_webidl = { workspace = true, optional = true } deno_web = { workspace = true, optional = true } diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 34f313e653..4e3094db84 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -15,7 +15,7 @@ use tokio::{ fs::{metadata, DirBuilder, File}, io::AsyncReadExt, process::Command, - sync::Semaphore, + sync::{RwLock, Semaphore}, task, }; use uuid::Uuid; @@ -35,6 +35,8 @@ use windmill_common::variables::get_secret_value_as_admin; use windmill_queue::{append_logs, CanceledBy}; lazy_static::lazy_static! { + static ref BUSY_VENV_P: RwLock> = RwLock::new(HashSet::new()); + static ref PYTHON_PATH: String = std::env::var("PYTHON_PATH").unwrap_or_else(|_| "/usr/local/bin/python3".to_string()); @@ -1482,15 +1484,7 @@ pub async fn handle_python_reqs( "{py_prefix}/{}", req.replace(' ', "").replace('/', "").replace(':', "") ); - if metadata(&venv_p).await.is_ok() - // Check if venv_p/.lock is in use - // If it is, it is better to wait for it's release and check if wheel has been installed after - && ofiles::opath(venv_p.clone() + "/.lock") - .unwrap_or_else(|e| { - tracing::error!("Failed to check if any PID has {}/.lock open: {e}", &venv_p); - vec![] - }) - .is_empty() + if metadata(&venv_p).await.is_ok() && !BUSY_VENV_P.read().await.contains(&venv_p) { // If dir exists or .lock is held skip installation and push path to output req_paths.push(venv_p); @@ -1710,6 +1704,7 @@ pub async fn handle_python_reqs( ); let start = std::time::Instant::now(); + BUSY_VENV_P.write().await.insert(venv_p.clone()); #[cfg(all(feature = "enterprise", feature = "parquet", unix))] if is_not_pro { if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() { @@ -1887,8 +1882,9 @@ pub async fn handle_python_reqs( ); } } else { - req_paths.push(venv_p); + req_paths.push(venv_p.clone()); } + BUSY_VENV_P.write().await.remove(&venv_p); } if has_work {