From 5ab4eca2f08d542dbc220e876b1ec8ab2bfa2f64 Mon Sep 17 00:00:00 2001 From: pyranota <92104930+pyranota@users.noreply.github.com> Date: Sat, 7 Jun 2025 03:35:28 +0200 Subject: [PATCH] fix: never consider minor version for global site packages (#5893) --- backend/src/main.rs | 7 +------ .../windmill-worker/src/python_executor.rs | 8 ++++---- .../windmill-worker/src/python_versions.rs | 20 ++++++++++++++----- backend/windmill-worker/src/worker.rs | 13 ++++-------- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/backend/src/main.rs b/backend/src/main.rs index d9283f6d9c..e469123a53 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -80,8 +80,7 @@ use windmill_worker::{ get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, CSHARP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, JAVA_CACHE_DIR, NU_CACHE_DIR, POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, - PY312_CACHE_DIR, PY313_CACHE_DIR, RUST_CACHE_DIR, TAR_JAVA_CACHE_DIR, TAR_PY310_CACHE_DIR, - TAR_PY311_CACHE_DIR, TAR_PY312_CACHE_DIR, TAR_PY313_CACHE_DIR, UV_CACHE_DIR, + PY312_CACHE_DIR, PY313_CACHE_DIR, RUST_CACHE_DIR, TAR_JAVA_CACHE_DIR, UV_CACHE_DIR, }; use crate::monitor::{ @@ -1281,10 +1280,6 @@ pub async fn run_workers( PY311_CACHE_DIR, PY312_CACHE_DIR, PY313_CACHE_DIR, - TAR_PY310_CACHE_DIR, - TAR_PY311_CACHE_DIR, - TAR_PY312_CACHE_DIR, - TAR_PY313_CACHE_DIR, BUN_BUNDLE_CACHE_DIR, GO_CACHE_DIR, GO_BIN_CACHE_DIR, diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 3a4898acc1..35815e452c 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -658,7 +658,7 @@ except BaseException as e: // Add /tmp/windmill/cache/python_x_y_z/global-site-packages to PYTHONPATH. // Usefull if certain wheels needs to be preinstalled before execution. - let global_site_packages_path = py_version.to_cache_dir() + "/global-site-packages"; + let global_site_packages_path = py_version.to_cache_dir(true) + "/global-site-packages"; let additional_python_paths_folders = { let mut paths = additional_python_paths.clone(); if std::fs::metadata(&global_site_packages_path).is_ok() { @@ -1489,7 +1489,7 @@ pub async fn handle_python_reqs( if req.starts_with('#') || req.starts_with('-') || req.trim().is_empty() { continue; } - let py_prefix = &py_version.to_cache_dir(); + let py_prefix = &py_version.to_cache_dir(false); let venv_p = format!( "{py_prefix}/{}", @@ -1740,7 +1740,7 @@ pub async fn handle_python_reqs( tokio::select! { // Cancel was called on the job _ = kill_rx.recv() => return Err(anyhow::anyhow!("S3 pull was canceled")), - pull = pull_from_tar(os, venv_p.clone(), py_version.to_cache_dir_top_level(), None, false) => { + pull = pull_from_tar(os, venv_p.clone(), py_version.to_cache_dir_top_level(false), None, false) => { if let Err(e) = pull { tracing::info!( workspace_id = %w_id, @@ -1891,7 +1891,7 @@ pub async fn handle_python_reqs( #[cfg(all(feature = "enterprise", feature = "parquet", unix))] if s3_push { if let Some(os) = windmill_common::s3_helpers::get_object_store().await { - tokio::spawn(build_tar_and_push(os, venv_p.clone(), py_version.to_cache_dir_top_level(), None, false)); + tokio::spawn(build_tar_and_push(os, venv_p.clone(), py_version.to_cache_dir_top_level(false), None, false)); } } diff --git a/backend/windmill-worker/src/python_versions.rs b/backend/windmill-worker/src/python_versions.rs index ff07b55ea1..2f12d34dc6 100644 --- a/backend/windmill-worker/src/python_versions.rs +++ b/backend/windmill-worker/src/python_versions.rs @@ -278,14 +278,24 @@ impl PyV { .into()) } } - /// e.g.: `/tmp/windmill/cache/python_3xy` - pub(crate) fn to_cache_dir(&self) -> String { + /// e.g.: `/tmp/windmill/cache/python_3_x_y` + pub(crate) fn to_cache_dir(&self, ignore_patch: bool) -> String { use windmill_common::worker::ROOT_CACHE_DIR; - format!("{ROOT_CACHE_DIR}{}", self.to_cache_dir_top_level()) + format!( + "{ROOT_CACHE_DIR}{}", + self.to_cache_dir_top_level(ignore_patch) + ) } /// e.g.: `python_3_x_y` - pub fn to_cache_dir_top_level(&self) -> String { + pub fn to_cache_dir_top_level(&self, ignore_patch: bool) -> String { + if ignore_patch { + if let [major, minor, ..] = self.release() { + return format!("python_{major}_{minor}"); + } + + tracing::warn!("failed to parse python's ({}) top level directory with no patch digit, fallback to full version.", self.to_string()); + } format!("python_{}", self.to_string().replace(".", "_")) } @@ -589,7 +599,7 @@ impl PyV { // For the default version directory created during startup (main.rs) DirBuilder::new() .recursive(true) - .create(self.to_cache_dir()) + .create(self.to_cache_dir(false)) .await .expect("could not create initial worker dir"); diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 7c01691826..56ac7c00e0 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -168,15 +168,10 @@ use windmill_common::bench::{benchmark_init, BenchmarkInfo, BenchmarkIter}; use windmill_common::add_time; -pub const PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_310"); -pub const PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_311"); -pub const PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_312"); -pub const PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_313"); - -pub const TAR_PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_310"); -pub const TAR_PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_311"); -pub const TAR_PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_312"); -pub const TAR_PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/python_313"); +pub const PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_3_10"); +pub const PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_3_11"); +pub const PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_3_12"); +pub const PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_3_13"); pub const TAR_JAVA_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/java");