From 2833d6da3f4079e36ff4a33aaffbf2442a9835e2 Mon Sep 17 00:00:00 2001 From: pyranota <92104930+pyranota@users.noreply.github.com> Date: Sat, 24 May 2025 02:13:47 +0200 Subject: [PATCH] feat(python): inline script metadata (PEP 723) (#5712) * make resolver * more updates * fix build * fix raw_dependencies job type * compat with http agent workers * refactor * rename * more refactor * cleanup * more tests * fix s3 * small fixes * more fixing * fix endpoint * nit: update comment * update ee ref * update ee ref * update ee ref * implement safer `list_available_python_versions` * add tracing to get of authed client * internal: Trigger claude when commenting with /aider (#5783) * add claude instructions files * call claude too when using aider * fix * add draft for linear claude integration * fix build * update ee ref * ignore versions <=3.9 * fix windows build * correct versions filter * fix windows build (this time for real) * inject error to debug CI * update CI * undo debug of CI * fix tests * remove outdated comment * update ee repo ref * Update ee-repo-ref.txt * Update backend/parsers/windmill-parser-py-imports/src/lib.rs Co-authored-by: graphite-app[bot] <96075541+graphite-app[bot]@users.noreply.github.com> * Update InstanceSetting.svelte --------- Co-authored-by: Ruben Fiszel Co-authored-by: centdix <40307056+centdix@users.noreply.github.com> Co-authored-by: graphite-app[bot] <96075541+graphite-app[bot]@users.noreply.github.com> --- .github/workflows/backend-test.yml | 4 +- backend/Cargo.lock | 23 + backend/Cargo.toml | 5 +- backend/ee-repo-ref.txt | 2 +- .../windmill-parser-py-imports/Cargo.toml | 3 + .../windmill-parser-py-imports/src/lib.rs | 142 ++- .../windmill-parser-py-imports/tests/tests.rs | 36 +- backend/tests/fixtures/multipython.sql | 20 + backend/tests/worker.rs | 49 +- backend/windmill-api/Cargo.toml | 1 + backend/windmill-api/openapi.yaml | 18 +- backend/windmill-api/src/configs.rs | 22 + backend/windmill-common/src/worker.rs | 1 + backend/windmill-queue/src/jobs.rs | 11 +- backend/windmill-worker/Cargo.toml | 1 + .../windmill-worker/src/ansible_executor.rs | 15 +- backend/windmill-worker/src/global_cache.rs | 5 +- backend/windmill-worker/src/lib.rs | 5 + .../windmill-worker/src/python_executor.rs | 651 +++----------- .../windmill-worker/src/python_versions.rs | 848 ++++++++++++++++++ backend/windmill-worker/src/worker.rs | 29 +- .../windmill-worker/src/worker_lockfiles.rs | 86 +- .../src/lib/components/InstanceSetting.svelte | 93 +- .../src/lib/components/instanceSettings.ts | 3 +- 24 files changed, 1420 insertions(+), 653 deletions(-) create mode 100644 backend/tests/fixtures/multipython.sql create mode 100644 backend/windmill-worker/src/python_versions.rs diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index 6d12e4f69c..c7d1c50db1 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -45,9 +45,9 @@ jobs: - uses: oven-sh/setup-bun@v2 with: bun-version: 1.1.43 - - uses: astral-sh/setup-uv@v4 + - uses: astral-sh/setup-uv@v6 with: - version: "0.4.18" + version: "0.6.2" - uses: actions-rust-lang/setup-rust-toolchain@v1 with: cache-workspaces: backend diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 5b5f1e754b..1f02308fbb 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -9045,6 +9045,18 @@ dependencies = [ "base64ct", ] +[[package]] +name = "pep440_rs" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31095ca1f396e3de32745f42b20deef7bc09077f918b085307e8eab6ddd8fb9c" +dependencies = [ + "once_cell", + "serde", + "unicode-width 0.2.0", + "unscanny", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -13858,6 +13870,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "unscanny" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9df2af067a7953e9c3831320f35c1cc0600c30d44d9f7a12b01db1cd88d6b47" + [[package]] name = "untrusted" version = "0.7.1" @@ -14440,6 +14458,7 @@ dependencies = [ "memchr", "object_store", "once_cell", + "pep440_rs", "prometheus", "quote", "rand 0.9.0", @@ -14858,12 +14877,15 @@ dependencies = [ "lazy_static", "malachite", "malachite-bigint", + "pep440_rs", "phf", "regex", "regex-lite", "rustpython-parser", + "serde", "serde_json", "sqlx", + "toml", "windmill-common", "windmill-parser", ] @@ -15040,6 +15062,7 @@ dependencies = [ "opentelemetry", "oracle", "pem 3.0.5", + "pep440_rs", "postgres-native-tls 0.5.1", "prometheus", "rand 0.9.0", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 972960baad..88aa2ffab6 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -83,7 +83,7 @@ zip = ["windmill-api/zip"] static_frontend = ["windmill-api/static_frontend"] scoped_cache = ["windmill-common/scoped_cache"] # Languages -python = ["windmill-worker/python"] +python = ["windmill-worker/python", "windmill-api/python"] rust = ["windmill-worker/rust"] mysql = ["windmill-worker/mysql"] oracledb = ["windmill-worker/oracledb"] @@ -135,10 +135,12 @@ quote.workspace = true memchr.workspace = true v8 = { workspace = true, optional = true } rustls.workspace = true +pep440_rs.workspace = true systemstat.workspace = true size.workspace = true strum.workspace = true + [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator = { optional = true, workspace = true } tikv-jemalloc-sys = { optional = true, workspace = true } @@ -219,6 +221,7 @@ git-version = "^0" malachite = "=0.4.18" malachite-bigint = "=0.2.0" rustpython-parser = "^0" +pep440_rs = "0.7.3" php-parser-rs = { git = "https://github.com/php-rust-tools/parser", rev = "ec4cb411dec09450946ef57920b7ffced7f6495d" } cron = "^0" mail-send = { version = "0.4.0", features = ["builder"], default-features=false } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index dfe9519cdd..9b39544645 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -99182f521564ff2d08dc2faf0af00f74f1834d60 \ No newline at end of file +72e6260ca886628cf1ba271bc058e6ecfdecdae5 \ No newline at end of file diff --git a/backend/parsers/windmill-parser-py-imports/Cargo.toml b/backend/parsers/windmill-parser-py-imports/Cargo.toml index 7bc558f9c0..abd363b42b 100644 --- a/backend/parsers/windmill-parser-py-imports/Cargo.toml +++ b/backend/parsers/windmill-parser-py-imports/Cargo.toml @@ -27,3 +27,6 @@ anyhow.workspace = true lazy_static.workspace = true sqlx.workspace = true async-recursion.workspace = true +toml.workspace = true +serde.workspace = true +pep440_rs.workspace = true diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs index 3b06a491d5..92f4346115 100644 --- a/backend/parsers/windmill-parser-py-imports/src/lib.rs +++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs @@ -11,7 +11,7 @@ mod mapping; use async_recursion::async_recursion; use itertools::Itertools; use lazy_static::lazy_static; -use std::collections::HashMap; +use std::{collections::HashMap, str::FromStr}; use mapping::{FULL_IMPORTS_MAP, SHORT_IMPORTS_MAP}; #[cfg(not(target_arch = "wasm32"))] @@ -25,7 +25,10 @@ use rustpython_parser::{ Parse, }; use sqlx::{Pool, Postgres}; -use windmill_common::{error, worker::PythonAnnotations}; +use windmill_common::{ + error::{self, to_anyhow}, + worker::PythonAnnotations, +}; const DEF_MAIN: &str = "def main("; @@ -242,8 +245,7 @@ pub async fn parse_python_imports( w_id: &str, path: &str, db: &Pool, - already_visited: &mut Vec, - annotated_pyv_numeric: &mut Option, + version_specifiers: &mut Vec, ) -> error::Result<(Vec, Option)> { let mut compile_error_hint: Option = None; let mut imports = parse_python_imports_inner( @@ -251,9 +253,10 @@ pub async fn parse_python_imports( w_id, path, db, - already_visited, - annotated_pyv_numeric, - &mut annotated_pyv_numeric.and_then(|_| Some(path.to_owned())), + &mut vec![], + version_specifiers, + // &mut version_specifier.and_then(|_| Some(path.to_owned())), + &mut None ) .await? .into_values() @@ -279,6 +282,7 @@ pub async fn parse_python_imports( .flatten() .collect::>>()? .into_iter() + .filter(|x| !x.trim_start().starts_with("--") && !x.trim().is_empty()) .unique() .collect_vec(); @@ -304,11 +308,34 @@ async fn parse_python_imports_inner( path: &str, db: &Pool, already_visited: &mut Vec, - annotated_pyv_numeric: &mut Option, + version_specifiers: &mut Vec, path_where_annotated_pyv: &mut Option, ) -> error::Result> { let PythonAnnotations { py310, py311, py312, py313, .. } = PythonAnnotations::parse(&code); + let mut push_version_specifiers = |perform, unparsed: String| -> error::Result<()> { + if perform { + pep440_rs::VersionSpecifiers::from_str(unparsed.as_str()) + .ok() + .map(|vs| version_specifiers.extend(vs.to_vec())); + } + Ok(()) + }; + push_version_specifiers(py310, "==3.10.*".to_owned())?; + push_version_specifiers(py311, "==3.11.*".to_owned())?; + push_version_specifiers(py312, "==3.12.*".to_owned())?; + push_version_specifiers(py313, "==3.13.*".to_owned())?; + + for x in code.lines() { + if x.starts_with("# py:") || x.starts_with("#py:") { + push_version_specifiers( + true, + x.replace('#', "").replace("py:", "").trim().to_owned(), + )?; + } else if !x.starts_with('#') { + break; + } + } // we pass only if there is none or only one annotation // Naive: @@ -323,39 +350,48 @@ async fn parse_python_imports_inner( // This way we make sure there is no multiple annotations for same script // and we get detailed span on conflicting versions - let mut check = |is_py_xyz, numeric| -> error::Result<()> { - if is_py_xyz { - if let Some(v) = annotated_pyv_numeric { - if *v != numeric { - return Err(error::Error::from(anyhow::anyhow!( - "Annotated 2 or more different python versions: \n - py{v} at {}\n - py{numeric} at {path}\nIt is possible to use only one.", - path_where_annotated_pyv.clone().unwrap_or("Unknown".to_owned()) - ))); - } - } else { - *annotated_pyv_numeric = Some(numeric); - } - *path_where_annotated_pyv = Some(path.to_owned()); - } - Ok(()) - }; + #[derive(serde::Serialize, serde::Deserialize)] + struct InlineMetadata { + requires_python: String, + dependencies: Vec, + } - check(py310, 310)?; - check(py311, 311)?; - check(py312, 312)?; - check(py313, 313)?; - - let find_requirements = code - .lines() - .find_position(|x| x.starts_with("#requirements:") || x.starts_with("# requirements:")); - if let Some((pos, _)) = find_requirements { + let find_requirements = code.lines().find_position(|x| { + x.starts_with("#requirements:") + || x.starts_with("# requirements:") + || x.starts_with("# /// script") + }); + if let Some((pos, item)) = find_requirements { let mut requirements = HashMap::new(); - code.lines() - .skip(pos + 1) - .map_while(|x| { - RE.captures(x).and_then(|x| { - x.get(1).map(|m| { - let requirement = m.as_str().to_string(); + if item.starts_with("# /// script") { + let mut incorrect = false; + let metadata = code + .lines() + .skip(pos + 1) + .map_while(|x| { + incorrect = !x.starts_with('#'); + if incorrect || x.starts_with("# ///") { + None + } else { + x.get(1..) + } + }) + .join("\n") + .parse::() + .map_err(to_anyhow)?; + + { + if let Some(v) = metadata.get("requires-python").and_then(|v| v.as_str()) { + push_version_specifiers(true, v.to_owned())?; + } + }; + + metadata + .get("dependencies") + .and_then(|dependencies| dependencies.as_array()) + .inspect(|list| { + for dependency_v in list.into_iter() { + let requirement = dependency_v.as_str().unwrap_or("ERROR").to_owned(); let key = extract_pkg_name(&requirement); requirements.insert( key.clone(), @@ -367,11 +403,31 @@ async fn parse_python_imports_inner( key, }, ); + } + }); + } else { + code.lines() + .skip(pos + 1) + .map_while(|x| { + RE.captures(x).and_then(|x| { + x.get(1).map(|m| { + let requirement = m.as_str().to_string(); + let key = extract_pkg_name(&requirement); + requirements.insert( + key.clone(), + NImportResolved::Pin { + pins: vec![ImportPin { + pkg: requirement.clone(), + path: Default::default(), + }], + key, + }, + ); + }) }) }) - }) - .collect_vec(); - + .collect_vec(); + } Ok(requirements) } else { let find_extra_requirements = code.lines().find_position(|x| { @@ -442,7 +498,7 @@ async fn parse_python_imports_inner( &rpath, db, already_visited, - annotated_pyv_numeric, + version_specifiers, path_where_annotated_pyv, ) .await? diff --git a/backend/parsers/windmill-parser-py-imports/tests/tests.rs b/backend/parsers/windmill-parser-py-imports/tests/tests.rs index 9fee9b21c9..b7fd2c0737 100644 --- a/backend/parsers/windmill-parser-py-imports/tests/tests.rs +++ b/backend/parsers/windmill-parser-py-imports/tests/tests.rs @@ -18,16 +18,8 @@ def main(): pass "; - let mut already_visited = vec![]; - let (r, ..) = parse_python_imports( - code, - "test-workspace", - "f/foo/bar", - &db, - &mut already_visited, - &mut None, - ) - .await?; + let (r, ..) = + parse_python_imports(code, "test-workspace", "f/foo/bar", &db, &mut vec![]).await?; // println!("{}", serde_json::to_string(&r)?); assert_eq!( r, @@ -59,16 +51,8 @@ def main(): pass "; - let mut already_visited = vec![]; - let (r, ..) = parse_python_imports( - code, - "test-workspace", - "f/foo/bar", - &db, - &mut already_visited, - &mut None, - ) - .await?; + let (r, ..) = + parse_python_imports(code, "test-workspace", "f/foo/bar", &db, &mut vec![]).await?; println!("{}", serde_json::to_string(&r)?); assert_eq!(r, vec!["burkina=0.4", "nigeria"]); @@ -89,17 +73,9 @@ def main(): pass "; - let mut already_visited = vec![]; - let (r, ..) = parse_python_imports( - code, - "test-workspace", - "f/foo/bar", - &db, - &mut already_visited, - &mut None, - ) - .await?; + let (r, ..) = + parse_python_imports(code, "test-workspace", "f/foo/bar", &db, &mut vec![]).await?; println!("{}", serde_json::to_string(&r)?); assert_eq!( r, diff --git a/backend/tests/fixtures/multipython.sql b/backend/tests/fixtures/multipython.sql new file mode 100644 index 0000000000..fa7d9c8c8d --- /dev/null +++ b/backend/tests/fixtures/multipython.sql @@ -0,0 +1,20 @@ +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +'# py312 +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/multipython/aliases', 2468135790, 'python3', ''); + +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'test-user', +'# py: >=3.9,!=3.12.2 +', +'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', +'', +'', +'f/multipython/script1', 2345678901, 'python3', ''); + diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 8b7509bb84..c8299796f8 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -3970,7 +3970,7 @@ async fn assert_lockfile( #[sqlx::test(fixtures("base", "lockfile_python"))] async fn test_requirements_python(db: Pool) { let content = r#" -# py311 +# py: 3.11.11 # requirements: # tiny==0.1.3 @@ -3988,7 +3988,7 @@ def main(): &db, content, ScriptLang::Python3, - vec!["# py311", "tiny==0.1.3"], + vec!["# py: 3.11.11", "tiny==0.1.3"], ) .await; } @@ -3998,7 +3998,7 @@ def main(): async fn test_extra_requirements_python(db: Pool) { { let content = r#" -# py311 +# py: ==3.11.11 # extra_requirements: # tiny @@ -4016,7 +4016,7 @@ def main(): &db, content, ScriptLang::Python3, - vec!["# py311", "bottle==0.13.2", "tiny==0.1.2"], + vec!["# py: 3.11.11", "bottle==0.13.2", "tiny==0.1.2"], ) .await; } @@ -4026,7 +4026,7 @@ def main(): #[sqlx::test(fixtures("base", "lockfile_python"))] async fn test_extra_requirements_python2(db: Pool) { let content = r#" -# py311 +# py: ==3.11.11 # extra_requirements: # tiny==0.1.3 @@ -4040,7 +4040,7 @@ def main(): &db, content, ScriptLang::Python3, - vec!["# py311", "simplejson==3.20.1", "tiny==0.1.3"], + vec!["# py: 3.11.11", "simplejson==3.20.1", "tiny==0.1.3"], ) .await; } @@ -4049,7 +4049,7 @@ def main(): #[sqlx::test(fixtures("base", "lockfile_python"))] async fn test_pins_python(db: Pool) { let content = r#" -# py311 +# py: ==3.11.11 # extra_requirements: # tiny==0.1.3 # bottle==0.13.2 @@ -4069,7 +4069,7 @@ def main(): content, ScriptLang::Python3, vec![ - "# py311", + "# py: 3.11.11", "bottle==0.13.2", "microdot==2.2.0", "simplejson==3.19.3", @@ -4078,6 +4078,39 @@ def main(): ) .await; } +#[cfg(feature = "python")] +#[sqlx::test(fixtures("base", "multipython"))] +async fn test_multipython_python(db: Pool) { + let content = r#"# py: <=3.12.2, >=3.12.0 +import f.multipython.script1 +import f.multipython.aliases +"# + .to_string(); + + assert_lockfile(&db, content, ScriptLang::Python3, vec!["# py: 3.12.1\n"]).await; +} + +#[cfg(feature = "python")] +#[sqlx::test(fixtures("base", "multipython"))] +async fn test_inline_script_metadata_python(db: Pool) { + let content = r#"# py_select_latest +# /// script +# requires-python = ">3.11,<3.12.3,!=3.12.2" +# dependencies = [ +# "tiny==0.1.3", +# ] +# /// +"# + .to_string(); + + assert_lockfile( + &db, + content, + ScriptLang::Python3, + vec!["# py: 3.12.1", "tiny==0.1.3"], + ) + .await; +} #[sqlx::test(fixtures("base", "result_format"))] async fn test_result_format(db: Pool) { let ordered_result_job_id = "1eecb96a-c8b0-4a3d-b1b6-087878c55e41"; diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 233fbeec54..276377b1ca 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -36,6 +36,7 @@ deno_core = ["dep:deno_core", "dep:deno_error"] gcp_trigger = ["dep:thiserror", "dep:google-cloud-pubsub", "dep:google-cloud-googleapis", "dep:tonic"] cloud = ["windmill-common/cloud"] mcp = ["dep:rmcp"] +python = [] [dependencies] rmcp = { git = "https://github.com/windmill-labs/rust-sdk", features = ["transport-sse-server"], optional = true } diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 166bcc7ce8..fd3e81f209 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -11105,6 +11105,23 @@ paths: items: $ref: "#/components/schemas/AutoscalingEvent" + /configs/list_available_python_versions: + get: + summary: Get currently available python versions provided by UV. + operationId: listAvailablePythonVersions + tags: + - config + # parameters: + responses: + "200": + description: List of python versions + content: + application/json: + schema: + type: array + items: + type: string + /agent_workers/create_agent_token: post: summary: create agent token @@ -16802,7 +16819,6 @@ components: type: string required: - s3 - TeamsChannel: type: object required: diff --git a/backend/windmill-api/src/configs.rs b/backend/windmill-api/src/configs.rs index 81b65216e0..e8770b2b5e 100644 --- a/backend/windmill-api/src/configs.rs +++ b/backend/windmill-api/src/configs.rs @@ -33,6 +33,10 @@ pub fn global_service() -> Router { "/list_autoscaling_events/:worker_group", get(list_autoscaling_events), ) + .route( + "/list_available_python_versions", + get(list_available_python_versions), + ) } #[derive(Serialize, Deserialize, FromRow)] @@ -205,6 +209,24 @@ async fn list_autoscaling_events( Ok(Json(events)) } +async fn list_available_python_versions() -> error::JsonResult> { + #[cfg(not(feature = "python"))] + return Err(error::Error::BadRequest( + "Python listing available only with 'python' feature enabled".to_string(), + )); + + #[cfg(feature = "python")] + use itertools::Itertools; + #[cfg(feature = "python")] + return Ok(Json( + windmill_worker::PyV::list_available_python_versions() + .await + .iter() + .map(|v| v.to_string()) + .collect_vec(), + )); +} + #[cfg(feature = "enterprise")] async fn list_configs( authed: ApiAuthed, diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index ea50393018..2d1e295432 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -516,6 +516,7 @@ fn parse_file(path: &str) -> Option { pub struct PythonAnnotations { pub no_cache: bool, pub no_postinstall: bool, + pub py_select_latest: bool, pub skip_result_postprocessing: bool, pub py310: bool, pub py311: bool, diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index f2b1e56153..de64071c42 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -2123,10 +2123,19 @@ pub struct PulledJob { pub permissioned_as_folders: Option>, } + +// NOTE: +// Precomputed by the server +// Used to offload work from agent workers to server #[derive(Serialize, Deserialize)] pub enum PrecomputedAgentInfo { Bun { local: String, remote: String }, - Python { py_version: Option, requirements: Option }, + Python { + // V1, not used anymore. Exists for compat. + // TODO: Needs to be removed eventually + py_version: Option, + py_version_v2: Option, + requirements: Option }, } #[derive(Serialize, Deserialize)] diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index c37e78d7cf..4dccc3213c 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -116,6 +116,7 @@ convert_case.workspace = true yaml-rust.workspace = true backon.workspace = true winapi = { workspace = true, optional = true } +pep440_rs.workspace = true opentelemetry = { workspace = true, optional = true } bollard = { workspace = true, optional = true } diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 7d46fc6040..ea94b2c1fb 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -30,9 +30,9 @@ use crate::{ start_child_process, transform_json, OccupancyMetrics, }, handle_child::handle_child, - python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile, PyVersion}, - AuthedClient, DISABLE_NSJAIL, DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV, - PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, + python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile}, + AuthedClient, PyVAlias, DISABLE_NSJAIL, DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, + PATH_ENV, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, }; lazy_static::lazy_static! { @@ -373,7 +373,7 @@ async fn handle_ansible_python_deps( worker_name, w_id, &mut Some(occupancy_metrics), - PyVersion::Py311, + PyVAlias::Py311.into(), false, ) .await @@ -387,10 +387,7 @@ async fn handle_ansible_python_deps( if requirements.len() > 0 { let mut venv_path = handle_python_reqs( - requirements - .split("\n") - .filter(|x| !x.starts_with("--")) - .collect(), + crate::python_executor::split_requirements(requirements), job_id, w_id, mem_peak, @@ -400,7 +397,7 @@ async fn handle_ansible_python_deps( job_dir, worker_dir, &mut Some(occupancy_metrics), - crate::python_executor::PyVersion::Py311, + PyVAlias::default().into(), ) .await?; additional_python_paths.append(&mut venv_path); diff --git a/backend/windmill-worker/src/global_cache.rs b/backend/windmill-worker/src/global_cache.rs index 155e6db8c1..6ab2acb441 100644 --- a/backend/windmill-worker/src/global_cache.rs +++ b/backend/windmill-worker/src/global_cache.rs @@ -22,6 +22,7 @@ pub async fn build_tar_and_push( platform_agnostic: bool, ) -> error::Result<()> { use object_store::path::Path; + use tokio::fs::create_dir_all; use crate::TAR_PYBASE_CACHE_DIR; @@ -36,7 +37,9 @@ pub async fn build_tar_and_push( }; let prefix = &format!("{TAR_PYBASE_CACHE_DIR}/{}", lang); - let tar_path = format!("{prefix}/{folder_name}_tar.tar",); + let tar_path = format!("{prefix}/{folder_name}_tar.tar"); + + create_dir_all(prefix).await?; let tar_file = std::fs::File::create(&tar_path)?; let mut tar = tar::Builder::new(tar_file); diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 64a9cf2c88..4ea56c090e 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -39,6 +39,8 @@ mod pg_executor; mod php_executor; #[cfg(feature = "python")] mod python_executor; +#[cfg(feature = "python")] +mod python_versions; pub mod result_processor; #[cfg(feature = "rust")] mod rust_executor; @@ -60,3 +62,6 @@ pub use bun_executor::{ prebundle_bun_script, prepare_job_dir, }; pub use deno_executor::generate_deno_lock; + +#[cfg(feature = "python")] +pub use python_versions::{PyV, PyVAlias}; diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 1ea5ebc520..0289ef1560 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -3,6 +3,7 @@ use std::{ fs, path::Path, process::Stdio, + str::FromStr, sync::Arc, }; @@ -38,12 +39,12 @@ use std::env::var; use windmill_queue::{append_logs, CanceledBy, PrecomputedAgentInfo}; lazy_static::lazy_static! { - static ref PYTHON_PATH: Option = var("PYTHON_PATH").ok().map(|v| { + pub(crate) static ref PYTHON_PATH: Option = var("PYTHON_PATH").ok().map(|v| { tracing::warn!("PYTHON_PATH is set to {} and thus python will not be managed by uv and stay static regardless of annotation and instance settings. NOT RECOMMENDED", v); v }); - static ref UV_PATH: String = + pub(crate) static ref UV_PATH: String = var("UV_PATH").unwrap_or_else(|_| "/usr/local/bin/uv".to_string()); static ref PY_CONCURRENT_DOWNLOADS: usize = @@ -77,348 +78,11 @@ use crate::{ start_child_process, OccupancyMetrics, }, handle_child::handle_child, - worker_lockfiles::LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT, worker_utils::ping_job_status, - AuthedClient, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, INSTANCE_PYTHON_VERSION, NSJAIL_PATH, - PATH_ENV, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR, + AuthedClient, PyV, PyVAlias, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, + PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR, }; -// To change latest stable version: -// 1. Change placeholder in instanceSettings.ts -// 2. Change LATEST_STABLE_PY in dockerfile -// 3. Change #[default] annotation for PyVersion in backend -#[derive(Eq, PartialEq, Clone, Copy, Default, Debug)] -pub enum PyVersion { - Py310, - #[default] - Py311, - Py312, - Py313, -} - -impl PyVersion { - pub async fn from_instance_version(job_id: &Uuid, w_id: &str, conn: &Connection) -> Self { - let mut err = None; - let pyv = match INSTANCE_PYTHON_VERSION.read().await.clone() { - Some(v) => PyVersion::from_string_with_dots(&v).unwrap_or_else(|| { - let v = PyVersion::default(); - err = Some(format!("\nCannot parse INSTANCE_PYTHON_VERSION ({:?}), fallback to latest_stable ({v:?})", *INSTANCE_PYTHON_VERSION)); - v - }), - // Use latest stable - None => PyVersion::default(), - }; - - if let Some(msg) = err { - append_logs(job_id, w_id, &msg, conn).await; - tracing::error!(msg); - } - pyv - } - /// e.g.: `/tmp/windmill/cache/python_3xy` - pub fn to_cache_dir(&self) -> String { - use windmill_common::worker::ROOT_CACHE_DIR; - format!("{ROOT_CACHE_DIR}{}", &self.to_cache_dir_top_level()) - } - /// e.g.: `python_3xy` - pub fn to_cache_dir_top_level(&self) -> String { - format!("python_{}", self.to_string_no_dot()) - } - /// e.g.: `3xy` - pub fn to_string_no_dot(&self) -> String { - self.to_string_with_dot().replace('.', "") - } - /// e.g.: `3.xy` - pub fn to_string_with_dot(&self) -> &str { - use PyVersion::*; - match self { - Py310 => "3.10", - Py311 => "3.11", - Py312 => "3.12", - Py313 => "3.13", - } - } - pub fn from_string_with_dots(value: &str) -> Option { - use PyVersion::*; - match value { - "3.10" => Some(Py310), - "3.11" => Some(Py311), - "3.12" => Some(Py312), - "3.13" => Some(Py313), - "default" => Some(PyVersion::default()), - _ => { - tracing::warn!( - "Cannot convert string (\"{value}\") to PyVersion\nExpected format x.yz" - ); - None - } - } - } - pub fn from_string_no_dots(value: &str) -> Option { - use PyVersion::*; - match value { - "310" => Some(Py310), - "311" => Some(Py311), - "312" => Some(Py312), - "313" => Some(Py313), - "default" => Some(PyVersion::default()), - _ => { - tracing::warn!( - "Cannot convert string (\"{value}\") to PyVersion\nExpected format xyz" - ); - None - } - } - } - /// e.g.: `# py3xy` -> `PyVersion::Py3XY` - pub fn parse_version(line: &str) -> Option { - Self::from_string_no_dots(line.replace(" ", "").replace("#py", "").as_str()) - } - pub fn from_py_annotations(a: PythonAnnotations) -> Option { - let PythonAnnotations { py310, py311, py312, py313, .. } = a; - use PyVersion::*; - if py313 { - Some(Py313) - } else if py312 { - Some(Py312) - } else if py311 { - Some(Py311) - } else if py310 { - Some(Py310) - } else { - None - } - } - pub fn from_numeric(n: u32) -> Option { - use PyVersion::*; - match n { - 310 => Some(Py310), - 311 => Some(Py311), - 312 => Some(Py312), - 313 => Some(Py313), - _ => None, - } - } - pub fn to_numeric(&self) -> u32 { - use PyVersion::*; - match self { - Py310 => 310, - Py311 => 311, - Py312 => 312, - Py313 => 313, - } - } - pub async fn get_python( - &self, - job_id: &Uuid, - mem_peak: &mut i32, - // canceled_by: &mut Option, - conn: &Connection, - worker_name: &str, - w_id: &str, - occupancy_metrics: &mut Option<&mut OccupancyMetrics>, - ) -> error::Result> { - // lazy_static::lazy_static! { - // static ref PYTHON_PATHS: Arc>> = Arc::new(RwLock::new(HashMap::new())); - // } - - let res = self - .get_python_inner(job_id, mem_peak, conn, worker_name, w_id, occupancy_metrics) - .await; - - if let Err(ref e) = res { - tracing::error!( - "worker_name: {worker_name}, w_id: {w_id}, job_id: {job_id}\n - Error while getting python from uv, falling back to system python: {e:?}" - ); - append_logs( - job_id, - w_id, - format!( - "\nError while getting python from uv, falling back to system python: {e:?}" - ), - conn, - ) - .await; - } - res - } - async fn get_python_inner( - self, - job_id: &Uuid, - mem_peak: &mut i32, - // canceled_by: &mut Option, - conn: &Connection, - worker_name: &str, - w_id: &str, - occupancy_metrics: &mut Option<&mut OccupancyMetrics>, - ) -> error::Result> { - let py_path = self.find_python().await; - - // Runtime is not installed - if py_path.is_err() { - // Install it - if let Err(err) = self - .install_python(job_id, mem_peak, conn, worker_name, w_id, occupancy_metrics) - .await - { - tracing::error!("Cannot install python: {err}"); - return Err(err); - } else { - // Try to find one more time - let py_path = self.find_python().await; - - if let Err(err) = py_path { - tracing::error!("Cannot find python version {err}"); - return Err(err); - } - - // TODO: Cache the result - py_path - } - } else { - py_path - } - } - async fn install_python( - self, - job_id: &Uuid, - mem_peak: &mut i32, - // canceled_by: &mut Option, - conn: &Connection, - worker_name: &str, - w_id: &str, - occupancy_metrics: &mut Option<&mut OccupancyMetrics>, - ) -> error::Result<()> { - let v = self.to_string_with_dot(); - append_logs(job_id, w_id, format!("\nINSTALLING PYTHON ({})", v), conn).await; - // Create dirs for newly installed python - // If we dont do this, NSJAIL will not be able to mount cache - // For the default version directory created during startup (main.rs) - DirBuilder::new() - .recursive(true) - .create(self.to_cache_dir()) - .await - .expect("could not create initial worker dir"); - - let logs = String::new(); - - #[cfg(windows)] - let uv_cmd = "uv"; - - #[cfg(unix)] - let uv_cmd = UV_PATH.as_str(); - - let mut child_cmd = Command::new(uv_cmd); - child_cmd - .env_clear() - .env("HOME", HOME_ENV.to_string()) - .env("PATH", PATH_ENV.to_string()) - .envs(PROXY_ENVS.clone()) - .args(["python", "install", v, "--python-preference=only-managed"]) - // TODO: Do we need these? - .envs([("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR)]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - #[cfg(windows)] - { - child_cmd - .env("SystemRoot", SYSTEM_ROOT.as_str()) - .env("USERPROFILE", crate::USERPROFILE_ENV.as_str()) - .env( - "TMP", - std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), - ) - .env( - "LOCALAPPDATA", - std::env::var("LOCALAPPDATA") - .unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())), - ); - } - - let child_process = start_child_process(child_cmd, "uv").await?; - - append_logs(&job_id, &w_id, logs, conn).await; - handle_child( - job_id, - conn, - mem_peak, - &mut None, - child_process, - false, - worker_name, - &w_id, - "uv", - None, - false, - occupancy_metrics, - None, - ) - .await - } - async fn find_python(self) -> error::Result> { - #[cfg(windows)] - let uv_cmd = "uv"; - - #[cfg(unix)] - let uv_cmd = UV_PATH.as_str(); - - let mut child_cmd = Command::new(uv_cmd); - - child_cmd.env_clear(); - - #[cfg(windows)] - { - child_cmd - .env("SystemRoot", SYSTEM_ROOT.as_str()) - .env("USERPROFILE", crate::USERPROFILE_ENV.as_str()) - .env( - "TMP", - std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), - ) - .env( - "LOCALAPPDATA", - std::env::var("LOCALAPPDATA") - .unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())), - ); - } - - let output = child_cmd - // .current_dir(job_dir) - .env("HOME", HOME_ENV.to_string()) - .env("PATH", PATH_ENV.to_string()) - .args([ - "python", - "find", - self.to_string_with_dot(), - "--system", - "--python-preference=only-managed", - ]) - .envs([ - ("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR), - ("UV_PYTHON_PREFERENCE", "only-managed"), - ]) - // .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .output() - .await?; - - // Check if the command was successful - if output.status.success() { - // Convert the output to a String - let stdout = - String::from_utf8(output.stdout).expect("Failed to convert output to String"); - return Ok(Some(stdout.replace('\n', ""))); - } else { - // If the command failed, print the error - let stderr = - String::from_utf8(output.stderr).expect("Failed to convert error output to String"); - return Err(error::Error::FindPythonError(stderr)); - } - } -} - #[cfg(windows)] use crate::SYSTEM_ROOT; @@ -465,7 +129,7 @@ pub async fn uv_pip_compile( worker_name: &str, w_id: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, - py_version: PyVersion, + py_version: PyV, // Debug-only flag no_cache: bool, ) -> error::Result { @@ -502,10 +166,11 @@ pub async fn uv_pip_compile( requirements.to_string() }; + let py_version_str = py_version.clone().to_string(); // Include python version to requirements.in // We need it because same hash based on requirements.in can get calculated even for different python versions // To prevent from overwriting same requirements.in but with different python versions, we include version to hash - let requirements = format!("# py{}\n{}", py_version.to_string_no_dot(), requirements); + let requirements = format!("# py: {}\n{}", py_version.to_string(), requirements); #[cfg(feature = "enterprise")] let requirements = replace_pip_secret(conn, w_id, &requirements, worker_name, job_id).await?; @@ -525,7 +190,7 @@ pub async fn uv_pip_compile( { logs.push_str(&format!( "\nFound cached resolution: {req_hash}, on python version: {}", - py_version.to_string_with_dot() + &py_version_str )); return Ok(cached); } @@ -539,7 +204,7 @@ pub async fn uv_pip_compile( { // Make sure we have python runtime installed py_version - .get_python(job_id, mem_peak, conn, worker_name, w_id, occupancy_metrics) + .try_get_python(job_id, mem_peak, conn, worker_name, w_id, occupancy_metrics) .await?; let mut args = vec![ @@ -561,12 +226,7 @@ pub async fn uv_pip_compile( UV_CACHE_DIR, ]; - args.extend([ - "-p", - &py_version.to_string_with_dot(), - "--python-preference", - "only-managed", - ]); + args.extend(["-p", &py_version_str, "--python-preference", "only-managed"]); if no_cache { args.extend(["--no-cache"]); @@ -666,8 +326,8 @@ pub async fn uv_pip_compile( let mut req_content = "".to_string(); file.read_to_string(&mut req_content).await?; let lockfile = format!( - "# py{}\n{}", - py_version.to_string_no_dot(), + "# py: {}\n{}", + py_version.to_string(), req_content .lines() .filter(|x| !x.trim_start().starts_with('#')) @@ -789,37 +449,6 @@ async fn postinstall( Ok(()) } -async fn get_python_path( - py_version: PyVersion, - worker_name: &str, - job_id: &Uuid, - w_id: &str, - mem_peak: &mut i32, - conn: &Connection, - occupancy_metrics: &mut Option<&mut OccupancyMetrics>, -) -> windmill_common::error::Result { - let python_path = if let Some(python_path) = PYTHON_PATH.clone() { - python_path - } else if let Some(python_path) = py_version - .get_python( - &job_id, - mem_peak, - conn, - worker_name, - w_id, - occupancy_metrics, - ) - .await? - { - python_path - } else { - return Err(Error::ExecutionErr(format!( - "uv could not manage python path. Please manage it manually by setting PYTHON_PATH environment variable to your python binary path" - ))); - }; - Ok(python_path) -} - #[tracing::instrument(level = "trace", skip_all)] pub async fn handle_python_job( requirements_o: Option<&String>, @@ -863,16 +492,16 @@ pub async fn handle_python_job( .await?; tracing::debug!("Finished handling python dependencies"); - let python_path = get_python_path( - py_version, - worker_name, - &job.id, - &job.workspace_id, - mem_peak, - conn, - &mut Some(occupancy_metrics), - ) - .await?; + let python_path = py_version + .get_python( + worker_name, + &job.id, + &job.workspace_id, + mem_peak, + conn, + &mut Some(occupancy_metrics), + ) + .await?; if !annotations.no_postinstall { if let Err(e) = postinstall(&mut additional_python_paths, job_dir, job, conn).await { @@ -887,7 +516,7 @@ pub async fn handle_python_job( &job.workspace_id, format!( "\n\n--- PYTHON ({}) CODE EXECUTION ---\n", - py_version.to_string_with_dot() + py_version.clone().to_string() ), conn, ) @@ -1026,7 +655,7 @@ except BaseException as e: let mut reserved_variables = get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; - // Add /tmp/windmill/cache/python_xyz/global-site-packages to PYTHONPATH. + // Add /tmp/windmill/cache/python_x_y_z/global-site-packages to PYTHONPATH. // Usefull if certain wheels needs to be preinstalled before execution. let global_site_packages_path = py_version.to_cache_dir() + "/global-site-packages"; let additional_python_paths_folders = { @@ -1039,9 +668,9 @@ except BaseException as e: // Since we handle mount of global_site_packages on our own, we don't want it to be mounted automatically. // We do this because existence of every wheel in cache is mandatory and if it is not there and nsjail expects it, it is a bug. // On the other side global_site_packages is purely optional. - // NOTE: This behaviour can be changed in future, so verification of wheels can be offloaded from nsjail to windmill + // NOTE: This behaviour can be changed in future, so verification of wheels can be delegated from nsjail to windmill paths.insert(0, global_site_packages_path.clone()); - // ^^^^^^^^ + // ^^^^^^ ^ // We also want this be priorotized, that's why we insert it to the beginning } paths.iter().join(":") @@ -1434,7 +1063,7 @@ async fn handle_python_deps( occupancy_metrics: &mut Option<&mut OccupancyMetrics>, precomputed_agent_info: Option, annotations: PythonAnnotations, -) -> error::Result<(PyVersion, Vec)> { +) -> error::Result<(PyV, Vec)> { create_dependencies_dir(job_dir).await; let mut additional_python_paths: Vec = WORKER_CONFIG @@ -1445,90 +1074,116 @@ async fn handle_python_deps( .unwrap_or_else(|| vec![]) .clone(); - let mut requirements; - let compilation_error_hint; - let mut annotated_pyv = None; - let mut annotated_pyv_numeric = None; - let is_deployed = requirements_o.is_some(); - let instance_pyv = PyVersion::from_instance_version(job_id, w_id, conn).await; - let requirements = match requirements_o { - Some(r) => r, + let (pyv, resolved_lines) = match requirements_o { + // Deployed + Some(r) => { + let rl = split_requirements(r); + (PyV::parse_from_requirements(&rl), rl) + } + // Preview None => { - let mut already_visited = vec![]; - - (requirements, compilation_error_hint) = match conn { + let (v, requirements_lines, error_hint) = match conn { Connection::Sql(db) => { + let mut version_specifiers = vec![]; let (r, h) = windmill_parser_py_imports::parse_python_imports( inner_content, w_id, script_path, db, - &mut already_visited, - &mut annotated_pyv_numeric, + &mut version_specifiers, ) .await?; - (r.join("\n"), h) + let v = PyV::resolve( + version_specifiers, + job_id, + w_id, + annotations.py_select_latest, + Some(conn.clone()), + None, + None, + ) + .await?; + + (v, r, h) } Connection::Http(_) => match precomputed_agent_info { - Some(PrecomputedAgentInfo::Python { py_version, requirements }) => { - annotated_pyv_numeric = py_version; - (requirements.clone().unwrap_or_else(|| "".to_string()), None) + Some(PrecomputedAgentInfo::Python { + requirements, + py_version, + py_version_v2, + }) => { + let v = { + let v_v2 = py_version_v2 + .clone() + .and_then(|s| pep440_rs::Version::from_str(&s).ok().map(PyV::from)); + let v_v1 = py_version.and_then(PyVAlias::try_from_v1).map(PyV::from); + + match v_v2.or(v_v1) { + Some(v) => v, + None => { + tracing::warn!( + workspace_id = %w_id, + " +Failed to get precomputed python version from server. Fallback to Default ({}) +Returned from server: py_version - {:?}, py_version_v2 - {:?} + ", + *PyV::default(), + py_version, + py_version_v2 + ); + Default::default() + } + } + }; + + let r = split_requirements(requirements.unwrap_or_default()); + let h = None; + + (v, r, h) } - _ => ("".to_string(), None), + _ => Default::default(), }, }; - annotated_pyv = annotated_pyv_numeric.and_then(|v| PyVersion::from_numeric(v)); - - if !requirements.is_empty() { - requirements = uv_pip_compile( - job_id, - &requirements, - mem_peak, - canceled_by, - job_dir, - conn, - worker_name, - w_id, - occupancy_metrics, - annotated_pyv.unwrap_or(instance_pyv), - annotations.no_cache, - ) - .await - .map_err(|e| { - Error::ExecutionErr(format!( - "pip compile failed: {}{}", - e.to_string(), - compilation_error_hint.unwrap_or_default() - )) - })?; - } - &requirements + ( + v.clone(), + if !requirements_lines.is_empty() { + uv_pip_compile( + job_id, + &requirements_lines.join("\n"), + mem_peak, + canceled_by, + job_dir, + conn, + worker_name, + w_id, + occupancy_metrics, + // annotated_pyv.unwrap_or(instance_pyv), + v, + annotations.no_cache, + ) + .await + .map_err(|e| { + Error::ExecutionErr(format!( + "pip compile failed: {}{}", + e.to_string(), + error_hint.unwrap_or_default() + )) + })? + .lines() + .map(|s| s.to_owned()) + .collect_vec() + } else { + vec![] + }, + ) } }; - /* - For deployed scripts we want to find out version in following order: - 1. Assigned version (written in lockfile) - 2. 3.11 - - For Previews: - 1. Annotated version - 2. Instance version - 3. Latest Stable - */ - let requirements_lines = split_requirements(requirements.as_str()); - let final_version = if is_deployed { - get_pyv_from_requirements_lines(&requirements_lines) - } else { - // This is not deployed script, meaning we test run it (Preview) - annotated_pyv.unwrap_or(instance_pyv) - }; - // If len > 0 it means there is atleast one dependency or assigned python version - if requirements.len() > 0 { + if !resolved_lines.is_empty() { let mut venv_path = handle_python_reqs( - requirements_lines, + resolved_lines, job_id, w_id, mem_peak, @@ -1538,13 +1193,13 @@ async fn handle_python_deps( job_dir, worker_dir, occupancy_metrics, - final_version, + pyv.clone(), ) .await?; additional_python_paths.append(&mut venv_path); } - Ok((final_version, additional_python_paths)) + Ok((pyv, additional_python_paths)) } lazy_static::lazy_static! { @@ -1733,7 +1388,7 @@ async fn spawn_uv_install( /// uv pip install, include cached or pull from S3 pub async fn handle_python_reqs( - requirements: Vec<&str>, + requirements: Vec, job_id: &Uuid, w_id: &str, mem_peak: &mut i32, @@ -1743,7 +1398,7 @@ pub async fn handle_python_reqs( job_dir: &str, worker_dir: &str, _occupancy_metrics: &mut Option<&mut OccupancyMetrics>, - py_version: PyVersion, + py_version: PyV, ) -> error::Result> { let worker_dir = worker_dir.to_string(); @@ -2017,7 +1672,7 @@ pub async fn handle_python_reqs( let total_time = std::time::Instant::now(); let py_path = py_version - .get_python( + .try_get_python( job_id, mem_peak, conn, @@ -2059,6 +1714,10 @@ pub async fn handle_python_reqs( let py_path = py_path.clone(); let pids = pids.clone(); let worker_dir = worker_dir.clone(); + + #[cfg(all(feature = "enterprise", feature = "parquet", unix))] + let py_version = py_version.clone(); + handles.push(task::spawn(async move { // permit will be dropped anyway if this thread exits at any point // so we dont have to drop it manually @@ -2300,36 +1959,14 @@ pub async fn handle_python_reqs( }; } -fn split_requirements(requirements: &str) -> Vec<&str> { +pub fn split_requirements>(requirements: T) -> Vec { requirements - .split("\n") + .as_ref() + .lines() .filter(|x| !x.trim_start().starts_with("--") && !x.trim().is_empty()) + .map(String::from) .collect() } -/// Check requirements/lockfile to figure out python version assigned to it. -fn get_pyv_from_requirements_lines(requirements_lines: &[&str]) -> PyVersion { - // If script is deployed we can try to parse first line to get assigned version - - let index = if requirements_lines.get(0).map_or(false, |line| { - line.starts_with(LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT) - }) { - 1 - } else { - 0 - }; - if let Some(v) = requirements_lines - .get(index) - .and_then(|line| PyVersion::parse_version(*line)) - { - // We have valid assigned version, we use it - v - } else { - // If there is no assigned version in lockfile we automatically fallback to 3.11 - // In this case we have dependencies, but no associated python version - // This is the case for old deployed scripts - PyVersion::Py311 - } -} // Returns code snippet that needs to be injected into wrapper to post-process results or leave unprocessed fn get_result_postprocessor<'a>(skip: bool) -> &'a str { @@ -2365,6 +2002,8 @@ pub async fn start_worker( jobs_rx: tokio::sync::mpsc::Receiver>, killpill_rx: tokio::sync::broadcast::Receiver<()>, ) -> error::Result<()> { + use crate::{PyV, PyVAlias}; + let mut mem_peak: i32 = 0; let mut canceled_by: Option = None; let context = variables::get_reserved_variables( @@ -2518,22 +2157,22 @@ for line in sys.stdin: proc_envs.insert("BASE_URL".to_string(), base_internal_url.to_string()); let py_version = if let Some(requirements) = requirements_o { - get_pyv_from_requirements_lines(&split_requirements(requirements.as_str())) + PyV::parse_from_requirements(&split_requirements(requirements.as_str())) } else { tracing::warn!(workspace_id = %w_id, "lockfile is empty for dedicated worker, thus python version cannot be inferred. Fallback to 3.11"); - PyVersion::Py311 + PyVAlias::Py311.into() }; - let python_path = get_python_path( - py_version, - worker_name, - &Uuid::nil(), - w_id, - &mut mem_peak, - &Connection::Sql(db.clone()), - &mut None, - ) - .await?; + let python_path = py_version + .get_python( + worker_name, + &Uuid::nil(), + w_id, + &mut mem_peak, + &Connection::Sql(db.clone()), + &mut None, + ) + .await?; handle_dedicated_process( &python_path, job_dir, diff --git a/backend/windmill-worker/src/python_versions.rs b/backend/windmill-worker/src/python_versions.rs new file mode 100644 index 0000000000..26be7aca65 --- /dev/null +++ b/backend/windmill-worker/src/python_versions.rs @@ -0,0 +1,848 @@ +use std::{ + ops::{Deref, DerefMut}, + process::Stdio, + str::FromStr, + sync::Arc, +}; + +use chrono::{DateTime, Duration, Utc}; +use itertools::Itertools; +use serde_json::Value; +use tokio::{fs::DirBuilder, process::Command, sync::RwLock}; +use uuid::Uuid; +use windmill_common::{ + error::{self, Error}, + worker::Connection, +}; + +use anyhow::{anyhow, bail}; +use windmill_queue::append_logs; + +use crate::{ + common::{start_child_process, OccupancyMetrics}, + handle_child::handle_child, + python_executor::{PYTHON_PATH, UV_PATH}, + worker_lockfiles::LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT, + HOME_ENV, INSTANCE_PYTHON_VERSION, PATH_ENV, PROXY_ENVS, PY_INSTALL_DIR, WIN_ENVS, +}; + +#[derive(Eq, PartialEq, Clone, Copy, Default, Debug)] +#[repr(u32)] +pub enum PyVAlias { + Py310 = 10, + #[default] + Py311, + Py312, + Py313, +} + +impl Into for PyVAlias { + fn into(self) -> pep440_rs::Version { + pep440_rs::Version::new([self.major() as u64, self as u64]) + } +} + +impl Into for PyVAlias { + fn into(self) -> u32 { + self.major() * 100 + self as u32 + } +} + +impl From for PyVAlias { + fn from(value: PyV) -> Self { + match value.release() { + [major, minor, ..] => { + if let Some(alias) = Self::try_from_v1(format!("{}{}", *major, *minor)) { + return alias; + } + } + _ => (), + } + + tracing::warn!( + "Failed to convert Python Full Version to Alias. Fallback to default ({})", + *PyV::default() + ); + Self::default() + } +} +impl PyVAlias { + fn all>() -> Vec { + use PyVAlias::*; + vec![Py310.into(), Py311.into(), Py312.into(), Py313.into()] + } + // Get MAJOR part of alias. (semver: MAJOR.MINOR.PATCH) + fn major(&self) -> u32 { + use PyVAlias::*; + match self { + Py310 | Py311 | Py312 | Py313 => 3, + // Py400 | Py401 => 4 + } + } + + /// Converts numeric format to alias + /// Example: + /// 310u32 (in) -> PyVAlias::Py310 (out) + pub(crate) fn try_from_v1(numeric: T) -> Option { + use PyVAlias::*; + match numeric.to_string().as_str() { + "310" => Some(Py310), + "311" => Some(Py311), + "312" => Some(Py312), + "313" => Some(Py313), + _ => None, + } + } +} + +// To change latest stable version: +// 1. Change placeholder in instanceSettings.ts +// 2. Change LATEST_STABLE_PY in dockerfile +// 3. Change #[default] annotation for PyVersion in backend +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct PyV(pub pep440_rs::Version); + +impl From for PyV { + fn from(value: pep440_rs::Version) -> Self { + Self(value) + } +} + +impl From for PyV { + fn from(value: PyVAlias) -> Self { + Self(value.into()) + } +} + +impl Default for PyV { + fn default() -> Self { + PyVAlias::default().into() + } +} + +impl Deref for PyV { + type Target = pep440_rs::Version; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} +impl DerefMut for PyV { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl PyV { + pub async fn resolve( + version_specifiers: Vec, + job_id: &Uuid, + w_id: &str, + select_latest: bool, + // Needed for logs but optional + conn: Option, + // Usually for testing + custom_versions: Option>, + // For testing + gravitational_version: Option, + ) -> Result { + // Get all versions that can be fetched + let all_versions = custom_versions.unwrap_or(PyV::list_available_python_versions().await); + + // Narrow down to those that satisfy given version specifiers + let valid = all_versions + .clone() + .into_iter() + .filter(|v| version_specifiers.iter().all(|vs| vs.contains(&*v))) + .collect_vec(); + + if !valid.is_empty() { + if select_latest { + return Ok(valid[0].clone()); + } + + // Usually INSTANCE_PYTHON_VERSION + let gv = gravitational_version + .unwrap_or(PyV::gravitational_version(job_id, w_id, conn).await); + + // Will be used to determine if picked version matches gravity version + // Once first match occure, we will stop iterating + let gravity_matcher = pep440_rs::VersionSpecifier::from_version( + pep440_rs::Operator::EqualStar, + (*gv).clone(), + ) + .map_err(|e| { + Error::ArgumentErr(format!( + "{e}\nLikely means INSTANCE_PYTHON_VERSION is set incorrectly." + )) + })?; + + // Reminder of semver: MAJOR.MINOR.PATCH + // + // - Go from up to down + // - We will iterate until find the closest version to target. + // - If closest version has the same MINOR version, use it. + // - If it differs in MINOR version, take latest PATCH version. + // + let mut result = None; + + // This represents newest version with oldest MINOR: + // + // I Iterable Newest in MINOR + // 1. 3.11.2 -> 3.11.2 + // 2. 3.11.1 -> 3.11.2 + // 3. 3.11.0 -> 3.11.2 + // 4. 3.10.2 -> 3.10.2 + // 5. 3.10.1 -> 3.10.2 + // 6. 3.10.0 -> 3.10.2 + let mut newest_in_minor = None; + for v in valid.iter() { + if result.is_none() { + result.replace(v); + } + + if v < &gv { + // We will not continue if we start looking into versions older than gravity version. + break; + } + + let [major, minor, ..] = v.release() else { + return Err(Error::InternalErr(format!("Failed to parse \"{}\". Available python versions are supposed to be in SEMVER format (MAJOR.MINOR)", **v))); + }; + + // Since we go top to down we can assume + // the first occurence of new minor version contains the latest patch version. + if matches!(newest_in_minor, Some((_, mm)) if mm != (major, minor)) + || newest_in_minor.is_none() + { + newest_in_minor = Some((v.clone(), (major, minor))); + } + + if gravity_matcher.contains(v) { + // return as soon as gravity matcher has first hit. + return Ok(v.clone()); + } + // If we are still in the loop, it means that we are getting closer to gravity version + else { + result = Some(v); + } + } + + let [gravity_major, gravity_minor, ..] = gv.release() else { + return Err(Error::internal_err(format!("Cannot get MAJOR or/and MINOR version of python gravity version ({}). Something might be wrong with INSTANCE_PYTHON_VERSION.", &*gv))); + }; + + if let Some((v, mm)) = newest_in_minor { + if (gravity_major, gravity_minor) != mm { + return Ok(v); + } + } + + result + .ok_or(Error::internal_err( + "No python candidates found. This is a bug!", + )) + .map(ToOwned::to_owned) + } else { + Err(anyhow!( + " + × No solution found when resolving python: + ╰─▶ Because you require python {}, we can conclude that your requirements are unsatisfiable. + + All versions: \n{} + \n", + version_specifiers.iter().map(|s| s.to_string()).join(", "), + all_versions + .iter() + .enumerate() + .map(|(i, v)| format!( + "{}{}", + windmill_common::worker::pad_string(&v.0.to_string(), 11), + if (i + 1) % 5 == 0 { "\n" } else { "" } + )) + .collect::() + ) + .into()) + } + } + /// e.g.: `/tmp/windmill/cache/python_3xy` + pub(crate) fn to_cache_dir(&self) -> String { + use windmill_common::worker::ROOT_CACHE_DIR; + format!("{ROOT_CACHE_DIR}{}", self.to_cache_dir_top_level()) + } + + /// e.g.: `python_3_x_y` + pub fn to_cache_dir_top_level(&self) -> String { + format!("python_{}", self.to_string().replace(".", "_")) + } + + pub async fn gravitational_version( + job_id: &Uuid, + w_id: &str, + conn: Option, + ) -> Self { + let mut err = None; + let pyv = match INSTANCE_PYTHON_VERSION.read().await.clone() { + Some(v) => pep440_rs::Version::from_str(&v).unwrap_or_else(|_| { + let v = PyVAlias::default().into(); + err = Some(format!("\nCannot parse INSTANCE_PYTHON_VERSION ({:?}), fallback to latest_stable ({v:?})", *INSTANCE_PYTHON_VERSION)); + v + }), + // Use latest stable + None => PyVAlias::default().into(), + }; + + if let Some(msg) = err { + if let Some(conn) = conn { + append_logs(job_id, w_id, &msg, &conn).await; + } + tracing::error!(msg); + } + pyv.into() + } + + pub async fn list_available_python_versions() -> Vec { + match Self::list_available_python_versions_inner().await { + Ok(pyvs) => pyvs, + Err(e) => { + tracing::error!( + "Fallback to preconfigured aliases. Cannot list python versions due to this error: {e}" + ); + PyVAlias::all() + } + } + } + async fn list_available_python_versions_inner() -> anyhow::Result> { + lazy_static::lazy_static! { + static ref CACHED_VERSIONS: Arc>>> = Arc::new(RwLock::new(None)); + static ref LAST_CHECKED: Arc>> = Arc::new(RwLock::new(Utc::now())); + } + match ( + Utc::now().signed_duration_since(*LAST_CHECKED.read().await) > Duration::minutes(30), + CACHED_VERSIONS.read().await.clone(), + ) { + (false, Some(vs)) => return Ok(vs), + _ => {} + }; + + let output = { + #[cfg(windows)] + let uv_cmd = "uv"; + + #[cfg(unix)] + let uv_cmd = UV_PATH.as_str(); + + Command::new(uv_cmd) + .env_clear() + .envs(WIN_ENVS.to_vec()) + .args([ + "python", + "list", + "--all-versions", + "--output-format", + "json", + ]) + .stderr(Stdio::piped()) + .output() + .await? + }; + + // We want to skip all versions smaller then 3.10 + // Windmill is incompatible with 3.9 and older + let filter = pep440_rs::VersionSpecifier::from_version( + pep440_rs::Operator::GreaterThanEqual, + PyVAlias::Py310.into(), + )?; + + if output.status.success() { + let res = String::from_utf8(output.stdout)?; + tracing::error!("{}", &res); + let list = serde_json::from_str::>>(&res)? + .into_iter() + .filter_map(|e| { + if e.get("implementation").and_then(Value::as_str) == Some("pypy") { + None + } else { + Some( + e.get("version") + .and_then(Value::as_str) + .and_then(|s| pep440_rs::Version::from_str(s).ok()) + .map(PyV::from) + .ok_or(Error::internal_err("version is None")), + ) + } + }) + .collect::, Error>>()? + .into_iter() + .unique() + .sorted() + .filter(|pyv| filter.contains(&*pyv)) + .rev() + .collect_vec(); + + *LAST_CHECKED.write().await = Utc::now(); + CACHED_VERSIONS.write().await.replace(list.clone()); + + Ok(list) + } else { + // If the command failed, print the error + let stderr = String::from_utf8(output.stderr)?; + bail!( + "Cannot list python versions, is uv (0.5.19 and newer) installed? Err:\n{}", + stderr + ); + } + } + + /// Parse lockfile for assigned python version. + /// If not found returns 3.11 + pub fn parse_from_requirements>(requirements_lines: &[S]) -> Self { + Self::try_parse_from_requirements(requirements_lines).unwrap_or( + // If there is no assigned version in lockfile we automatically fallback to 3.11 + // In this case we have dependencies or other metadata, but no associated python version + // This is the case for old deployed scripts + PyVAlias::Py311.into(), + ) + } + + /// Parse lockfile for assigned python version. + /// If not found returns None + pub fn try_parse_from_requirements>(requirements_lines: &[S]) -> Option { + let parse_version = |s: &str| -> Option { + // Possible inputs: + // V2: + // # py: 3.11.0 or #py:3.11.0 or #py: 3.11.0 + // + // V1: + // # py311 or #py311 + let version_unparsed = s + .to_owned() + // Remove whitespaces. That leaves us with: + // V2: #py:3.11.0 + // V1: #py311 + // + // Remove # + // V2: py:3.11.0 + // V1: py311 + // + // Remove : + // V2: py3.11.0 + // V1: py311 + .replace([' ', '#', ':'], "") + // Remove "py" + // V2: 3.11.0 + // V1: 311 + .replace("py", ""); + + // We will support reading V1 syntax, but it will be overwritten next deploy + PyVAlias::try_from_v1(&version_unparsed) + .map(PyVAlias::into) + .or(pep440_rs::Version::from_str(&version_unparsed) + .ok() + .map(pep440_rs::Version::into)) + }; + let index = if requirements_lines.get(0).map_or(false, |line| { + line.as_ref() + .starts_with(LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT) + }) { + 1 + } else { + 0 + }; + requirements_lines + .get(index) + .map(S::as_ref) + .and_then(parse_version) + } + + pub async fn get_python( + &self, + worker_name: &str, + job_id: &Uuid, + w_id: &str, + mem_peak: &mut i32, + conn: &Connection, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + ) -> windmill_common::error::Result { + let python_path = if let Some(python_path) = PYTHON_PATH.clone() { + python_path + } else if let Some(python_path) = self + .try_get_python( + &job_id, + mem_peak, + conn, + worker_name, + w_id, + occupancy_metrics, + ) + .await? + { + python_path + } else { + return Err(Error::ExecutionErr(format!( + "uv could not manage python path. Please manage it manually by setting PYTHON_PATH environment variable to your python binary path" + ))); + }; + Ok(python_path) + } + + pub async fn try_get_python( + &self, + job_id: &Uuid, + mem_peak: &mut i32, + // canceled_by: &mut Option, + conn: &Connection, + worker_name: &str, + w_id: &str, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + ) -> error::Result> { + // lazy_static::lazy_static! { + // static ref PYTHON_PATHS: Arc>> = Arc::new(RwLock::new(HashMap::new())); + // } + + let res = self + .get_python_inner(job_id, mem_peak, conn, worker_name, w_id, occupancy_metrics) + .await; + + if let Err(ref e) = res { + tracing::error!( + "worker_name: {worker_name}, w_id: {w_id}, job_id: {job_id}\n + Error while getting python from uv, falling back to system python: {e:?}" + ); + append_logs( + job_id, + w_id, + format!( + "\nError while getting python from uv, falling back to system python: {e:?}" + ), + conn, + ) + .await; + } + res + } + async fn get_python_inner( + &self, + job_id: &Uuid, + mem_peak: &mut i32, + // canceled_by: &mut Option, + conn: &Connection, + worker_name: &str, + w_id: &str, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + ) -> error::Result> { + let py_path = self.find_python().await; + + // Runtime is not installed + if py_path.is_err() { + // Install it + if let Err(err) = self + .install_python(job_id, mem_peak, conn, worker_name, w_id, occupancy_metrics) + .await + { + tracing::error!("Cannot install python: {err}"); + return Err(err); + } else { + // Try to find one more time + let py_path = self.find_python().await; + + if let Err(err) = py_path { + tracing::error!("Cannot find python version {err}"); + return Err(err); + } + + // TODO: Cache the result + py_path + } + } else { + py_path + } + } + async fn install_python( + &self, + job_id: &Uuid, + mem_peak: &mut i32, + // canceled_by: &mut Option, + conn: &Connection, + worker_name: &str, + w_id: &str, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + ) -> error::Result<()> { + let v = self.to_string(); + append_logs(job_id, w_id, format!("\nINSTALLING PYTHON ({})", v), conn).await; + // Create dirs for newly installed python + // If we dont do this, NSJAIL will not be able to mount cache + // For the default version directory created during startup (main.rs) + DirBuilder::new() + .recursive(true) + .create(self.to_cache_dir()) + .await + .expect("could not create initial worker dir"); + + let logs = String::new(); + + #[cfg(windows)] + let uv_cmd = "uv"; + + #[cfg(unix)] + let uv_cmd = UV_PATH.as_str(); + + let mut child_cmd = Command::new(uv_cmd); + child_cmd + .env_clear() + .env("HOME", HOME_ENV.to_string()) + .env("PATH", PATH_ENV.to_string()) + .envs(PROXY_ENVS.clone()) + .args(["python", "install", &v, "--python-preference=only-managed"]) + // TODO: Do we need these? + .envs([("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR)]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + #[cfg(windows)] + { + child_cmd + .env("SystemRoot", crate::SYSTEM_ROOT.as_str()) + .env("USERPROFILE", crate::USERPROFILE_ENV.as_str()) + .env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), + ) + .env( + "LOCALAPPDATA", + std::env::var("LOCALAPPDATA") + .unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())), + ); + } + + let child_process = start_child_process(child_cmd, "uv").await?; + + append_logs(&job_id, &w_id, logs, conn).await; + handle_child( + job_id, + conn, + mem_peak, + &mut None, + child_process, + false, + worker_name, + &w_id, + "uv", + None, + false, + occupancy_metrics, + None, + ) + .await + } + async fn find_python(&self) -> error::Result> { + #[cfg(windows)] + let uv_cmd = "uv"; + + #[cfg(unix)] + let uv_cmd = UV_PATH.as_str(); + + let mut child_cmd = Command::new(uv_cmd); + + child_cmd.env_clear(); + + #[cfg(windows)] + { + child_cmd + .env("SystemRoot", crate::SYSTEM_ROOT.as_str()) + .env("USERPROFILE", crate::USERPROFILE_ENV.as_str()) + .env( + "TMP", + std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), + ) + .env( + "LOCALAPPDATA", + std::env::var("LOCALAPPDATA") + .unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())), + ); + } + + let output = child_cmd + // .current_dir(job_dir) + .env("HOME", HOME_ENV.to_string()) + .env("PATH", PATH_ENV.to_string()) + .args([ + "python", + "find", + &self.to_string(), + "--system", + "--python-preference=only-managed", + ]) + .envs([ + ("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR), + ("UV_PYTHON_PREFERENCE", "only-managed"), + ]) + // .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await?; + + // Check if the command was successful + if output.status.success() { + // Convert the output to a String + let stdout = + String::from_utf8(output.stdout).expect("Failed to convert output to String"); + return Ok(Some(stdout.replace('\n', ""))); + } else { + // If the command failed, print the error + let stderr = + String::from_utf8(output.stderr).expect("Failed to convert error output to String"); + return Err(error::Error::FindPythonError(stderr)); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Unsafe helper for testing + fn pyv(value: &str) -> PyV { + pep440_rs::Version::from_str(value).unwrap().into() + } + + async fn assert_resolution( + instance_version: &str, + select_highest: bool, + specifiers: Vec<&str>, + available: Vec, + expected: PyV, + ) { + let resolved = PyV::resolve( + specifiers + .into_iter() + .map(|s| pep440_rs::VersionSpecifier::from_str(s).unwrap()) + .collect_vec(), + &Uuid::nil(), + "", + select_highest, + None, + Some(available), + Some(pyv(instance_version)), + ) + .await + .unwrap(); + assert_eq!(expected, resolved); + } + + #[tokio::test] + async fn test_python_resolution_1() { + assert_resolution( + "1.0", + false, + vec![], + vec![ + pyv("1.2.0"), + pyv("1.1.0"), + pyv("1.0.0"), + pyv("0.9.0"), // + ], + pyv("1.0.0"), // + ) + .await; + } + #[tokio::test] + async fn test_python_resolution_2() { + assert_resolution( + "1.0.0", + false, + vec!["!=1.*"], + vec![ + pyv("1.2"), + pyv("1.1"), + pyv("1.0.2"), + pyv("1.0.1"), + pyv("1.0.0"), + pyv("0.9.4"), + pyv("0.9.3"), + pyv("0.9.2"), + ], + pyv("0.9.4"), // + ) + .await; + } + #[tokio::test] + async fn test_python_resolution_3() { + assert_resolution( + "0.9", + false, + vec!["!=0.9.*"], + vec![ + pyv("1.2"), + pyv("1.1"), + pyv("1.0.2"), + pyv("1.0.1"), + pyv("1.0.0"), + pyv("0.9.4"), + pyv("0.9.3"), + pyv("0.9.2"), + pyv("0.8.2"), + pyv("0.8.1"), + pyv("0.8.0"), + ], + pyv("1.0.2"), // + ) + .await; + } + #[tokio::test] + async fn test_python_resolution_4() { + assert_resolution( + "0.9", + false, + vec!["<=0.8.1"], + vec![pyv("1.0.0"), pyv("0.9.0"), pyv("0.8.1"), pyv("0.8.0")], + pyv("0.8.1"), // + ) + .await; + } + #[tokio::test] + async fn test_python_resolution_5() { + assert_resolution( + "0.0.1", + false, + vec!["!=0.1.0"], + vec![pyv("2.1.0"), pyv("1.1.0"), pyv("0.1.0")], + pyv("1.1.0"), + ) + .await; + } + #[tokio::test] + async fn test_python_resolution_6() { + assert_resolution( + "1.1.1", + false, + vec![], + vec![ + pyv("3.0.1"), + pyv("3.0.0"), + pyv("2.2.2"), + pyv("2.2.1"), + pyv("2.2.0"), + ], + pyv("2.2.2"), + ) + .await; + } + #[tokio::test] + async fn test_python_resolution_7() { + assert_resolution( + "2.2.1", + true, + vec![], + vec![ + pyv("3.0.1"), + pyv("3.0.0"), + pyv("2.2.2"), + pyv("2.2.1"), + pyv("2.2.0"), + ], + pyv("3.0.1"), + ) + .await; + } +} diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index ede454281d..92fc0f0351 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -134,7 +134,10 @@ use crate::java_executor::{handle_java_job, JobHandlerInput as JobHandlerInputJa use crate::php_executor::handle_php_job; #[cfg(feature = "python")] -use crate::python_executor::{handle_python_job, PyVersion}; +use crate::{ + python_executor::handle_python_job, + python_versions::{PyV, PyVAlias}, +}; #[cfg(feature = "python")] use crate::ansible_executor::handle_ansible_job; @@ -363,10 +366,26 @@ lazy_static::lazy_static! { } +type Envs = Vec<(String, String)>; + #[cfg(windows)] lazy_static::lazy_static! { pub static ref SYSTEM_ROOT: String = std::env::var("SystemRoot").unwrap_or_else(|_| "C:\\Windows".to_string()); pub static ref USERPROFILE_ENV: String = std::env::var("USERPROFILE").unwrap_or_else(|_| "/tmp".to_string()); + static ref TMP: String = std::env::var("TMP").unwrap_or_else(|_| "/tmp".to_string()); + static ref LOCALAPPDATA: String = std::env::var("LOCALAPPDATA").unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())); + pub static ref WIN_ENVS: Envs = vec![ + ("SystemRoot".into(), SYSTEM_ROOT.clone()), + ("USERPROFILE".into(), USERPROFILE_ENV.clone()), + ("TMP".into(), TMP.clone()), + ("LOCALAPPDATA".into(), LOCALAPPDATA.clone()) + ]; + +} + +#[cfg(not(windows))] +lazy_static::lazy_static! { + pub static ref WIN_ENVS: Envs = vec![]; } //only matter if CLOUD_HOSTED @@ -828,9 +847,9 @@ pub async fn run_worker( worker_dir.clone(), ); tokio::spawn(async move { - if let Err(e) = PyVersion::from_instance_version(&Uuid::nil(), "", &conn) + if let Err(e) = PyV::gravitational_version(&Uuid::nil(), "", Some(conn.clone())) .await - .get_python(&Uuid::nil(), &mut 0, &conn, &worker_name, "", &mut None) + .try_get_python(&Uuid::nil(), &mut 0, &conn, &worker_name, "", &mut None) .await { tracing::error!( @@ -840,8 +859,8 @@ pub async fn run_worker( "Cannot preinstall or find Instance Python version to worker: {e}"// ); } - if let Err(e) = PyVersion::Py311 - .get_python(&Uuid::nil(), &mut 0, &conn, &worker_name, "", &mut None) + if let Err(e) = PyV::from(PyVAlias::Py311) + .try_get_python(&Uuid::nil(), &mut 0, &conn, &worker_name, "", &mut None) .await { tracing::error!( diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 7744ef341b..d32413fc1e 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -47,7 +47,7 @@ use crate::java_executor::resolve; use crate::php_executor::{composer_install, parse_php_imports}; #[cfg(feature = "python")] use crate::python_executor::{ - create_dependencies_dir, handle_python_reqs, uv_pip_compile, PyVersion, + create_dependencies_dir, handle_python_reqs, split_requirements, uv_pip_compile, }; #[cfg(feature = "rust")] use crate::rust_executor::generate_cargo_lockfile; @@ -1897,26 +1897,13 @@ async fn python_dep( w_id: &str, worker_dir: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, - annotated_pyv_numeric: Option, + py_version: crate::PyV, annotations: PythonAnnotations, ) -> std::result::Result { + use crate::python_executor::split_requirements; + create_dependencies_dir(job_dir).await; - /* - Unlike `handle_python_deps` which we use for running scripts (deployed and drafts) - This one used specifically for deploying scripts - So we can get final_version right away and include in lockfile - And the precendence is following: - - 1. Annotation version - 2. Instance version - 3. Latest Stable - */ - - let final_version = annotated_pyv_numeric - .and_then(|pyv| PyVersion::from_numeric(pyv)) - .unwrap_or(PyVersion::from_instance_version(job_id, w_id, &db.into()).await); - let req: std::result::Result = uv_pip_compile( job_id, &reqs, @@ -1927,14 +1914,15 @@ async fn python_dep( worker_name, w_id, occupancy_metrics, - final_version, + py_version, annotations.no_cache, ) .await; // install the dependencies to pre-fill the cache if let Ok(req) = req.as_ref() { let r = handle_python_reqs( - req.split("\n").filter(|x| !x.starts_with("--")).collect(), + split_requirements(req), + // req.split("\n").filter(|x| !x.starts_with("--")).collect(), job_id, w_id, mem_peak, @@ -1944,7 +1932,8 @@ async fn python_dep( job_dir, worker_dir, occupancy_metrics, - final_version, + // final_version, + crate::PyVAlias::default().into(), ) .await; @@ -1994,7 +1983,7 @@ async fn ansible_dep( w_id, worker_dir, &mut Some(occupancy_metrics), - None, + crate::PyV::gravitational_version(job_id, w_id, Some(db.clone().into())).await, PythonAnnotations::default(), ) .await?; @@ -2104,31 +2093,44 @@ async fn capture_dependency_job( )); #[cfg(feature = "python")] { - let anns = PythonAnnotations::parse(job_raw_code); - let mut annotated_pyv_numeric = None; - - let reqs = if raw_deps { + // Manually assigned version from requirements.txt + // let assigned_py_version; + let (reqs, py_version) = if raw_deps { // `wmill script generate-metadata` // should also respect annotated pyversion // can be annotated in script itself // or in requirements.txt if present - annotated_pyv_numeric = - PyVersion::from_py_annotations(anns).map(|v| v.to_numeric()); - job_raw_code.to_string() - } else { - let mut already_visited = vec![]; - windmill_parser_py_imports::parse_python_imports( - job_raw_code, - &w_id, - script_path, - &db, - &mut already_visited, - &mut annotated_pyv_numeric, + ( + job_raw_code.to_owned(), + crate::PyV::parse_from_requirements(&split_requirements(job_raw_code)), + ) + } else { + let mut version_specifiers = vec![]; + let PythonAnnotations { py_select_latest, .. } = + PythonAnnotations::parse(job_raw_code); + ( + windmill_parser_py_imports::parse_python_imports( + job_raw_code, + &w_id, + script_path, + &db, + &mut version_specifiers, + ) + .await? + .0 + .join("\n"), + crate::PyV::resolve( + version_specifiers, + job_id, + w_id, + py_select_latest, + Some(db.clone().into()), + None, + None, + ) + .await?, ) - .await? - .0 - .join("\n") }; python_dep( @@ -2142,8 +2144,8 @@ async fn capture_dependency_job( w_id, worker_dir, &mut Some(occupancy_metrics), - annotated_pyv_numeric, - anns, + py_version, + PythonAnnotations::parse(job_raw_code), ) .await .map(|res| { diff --git a/frontend/src/lib/components/InstanceSetting.svelte b/frontend/src/lib/components/InstanceSetting.svelte index 958dcb9808..6e75df9270 100644 --- a/frontend/src/lib/components/InstanceSetting.svelte +++ b/frontend/src/lib/components/InstanceSetting.svelte @@ -17,11 +17,18 @@ import ObjectStoreConfigSettings from './ObjectStoreConfigSettings.svelte' import { sendUserToast } from '$lib/toast' import ConfirmButton from './ConfirmButton.svelte' - import { IndexSearchService, SettingService, TeamsService } from '$lib/gen' + import { + ConfigService, + IndexSearchService, + SettingService, + TeamsService, + type ListAvailablePythonVersionsResponse + } from '$lib/gen' import { Button, SecondsInput, Skeleton } from './common' import Password from './Password.svelte' import { classNames } from '$lib/utils' import Popover from './Popover.svelte' + import PopoverMelt from './meltComponents/Popover.svelte' import Toggle from './Toggle.svelte' import type { Writable } from 'svelte/store' import { createEventDispatcher } from 'svelte' @@ -30,6 +37,7 @@ import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte' import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte' import SimpleEditor from './SimpleEditor.svelte' + import LoadingIcon from './apps/svelte-select/lib/LoadingIcon.svelte' import TeamSelector from './TeamSelector.svelte' import ChannelSelector from './ChannelSelector.svelte' @@ -39,7 +47,10 @@ export let loading = true const dispatch = createEventDispatcher() - if (setting.fieldType == 'select' && $values[setting.key] == undefined) { + if ( + (setting.fieldType == 'select' || setting.fieldType == 'select_python') && + $values[setting.key] == undefined + ) { $values[setting.key] = 'default' } @@ -124,6 +135,24 @@ } } + let pythonAvailableVersions: ListAvailablePythonVersionsResponse = [] + + let isPyFetching = false + async function fetch_available_python_versions() { + if (isPyFetching) return + isPyFetching = true + try { + pythonAvailableVersions = await ConfigService.listAvailablePythonVersions() + } catch (error) { + console.error('Error fetching python versions:', error) + } finally { + isPyFetching = false + } + } + if (setting.fieldType == 'select_python') { + fetch_available_python_versions() + } + async function fetchTeams() { if (isFetching) return isFetching = true @@ -193,6 +222,66 @@ {/each} + {:else if setting.fieldType == 'select_python'} +
+ + + + + {#each setting.select_items ?? [] as item} + + {/each} + + + {#if setting.select_items?.some((e) => e.label == $values[setting.key] || e.value == $values[setting.key])} + + {:else} + + {/if} + + + {#if isPyFetching} +
+ +
+ {:else} + + {#each pythonAvailableVersions as item} + + {/each} + + {/if} + + + +
{:else}