starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-21 08:02:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(python): inline script metadata (PEP 723) (#5712)

Browse Source 
* make resolver

* more updates

* fix build

* fix raw_dependencies job type

* compat with http agent workers

* refactor

* rename

* more refactor

* cleanup

* more tests

* fix s3

* small fixes

* more fixing

* fix endpoint

* nit: update comment

* update ee ref

* update ee ref

* update ee ref

* implement safer `list_available_python_versions`

* add tracing to get of authed client

* internal: Trigger claude when commenting with /aider (#5783)

* add claude instructions files

* call claude too when using aider

* fix

* add draft for linear claude integration

* fix build

* update ee ref

* ignore versions <=3.9

* fix windows build

* correct versions filter

* fix windows build (this time for real)

* inject error to debug CI

* update CI

* undo debug of CI

* fix tests

* remove outdated comment

* update ee repo ref

* Update ee-repo-ref.txt

* Update backend/parsers/windmill-parser-py-imports/src/lib.rs

Co-authored-by: graphite-app[bot] <96075541+graphite-app[bot]@users.noreply.github.com>

* Update InstanceSetting.svelte

---------

Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
Co-authored-by: centdix <40307056+centdix@users.noreply.github.com>
Co-authored-by: graphite-app[bot] <96075541+graphite-app[bot]@users.noreply.github.com>
This commit is contained in:
pyranota
2025-05-24 02:13:47 +02:00
committed by GitHub
parent 1f043b902a
commit 2833d6da3f
24 changed files with 1420 additions and 653 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/backend-test.yml
+2 -2
View File
@@ -45,9 +45,9 @@ jobs:
- uses: oven-sh/setup-bun@v2
with:
bun-version: 1.1.43
- uses: astral-sh/setup-uv@v4
- uses: astral-sh/setup-uv@v6
with:
version: "0.4.18"
version: "0.6.2"
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
cache-workspaces: backend
backend/Cargo.lock
Generated
+23
View File
@@ -9045,6 +9045,18 @@ dependencies = [
"base64ct",
]
[[package]]
name = "pep440_rs"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31095ca1f396e3de32745f42b20deef7bc09077f918b085307e8eab6ddd8fb9c"
dependencies = [
"once_cell",
"serde",
"unicode-width 0.2.0",
"unscanny",
]
[[package]]
name = "percent-encoding"
version = "2.3.1"
@@ -13858,6 +13870,12 @@ dependencies = [
"subtle",
]
[[package]]
name = "unscanny"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9df2af067a7953e9c3831320f35c1cc0600c30d44d9f7a12b01db1cd88d6b47"
[[package]]
name = "untrusted"
version = "0.7.1"
@@ -14440,6 +14458,7 @@ dependencies = [
"memchr",
"object_store",
"once_cell",
"pep440_rs",
"prometheus",
"quote",
"rand 0.9.0",
@@ -14858,12 +14877,15 @@ dependencies = [
"lazy_static",
"malachite",
"malachite-bigint",
"pep440_rs",
"phf",
"regex",
"regex-lite",
"rustpython-parser",
"serde",
"serde_json",
"sqlx",
"toml",
"windmill-common",
"windmill-parser",
]
@@ -15040,6 +15062,7 @@ dependencies = [
"opentelemetry",
"oracle",
"pem 3.0.5",
"pep440_rs",
"postgres-native-tls 0.5.1",
"prometheus",
"rand 0.9.0",
backend/Cargo.toml
+4 -1
View File
@@ -83,7 +83,7 @@ zip = ["windmill-api/zip"]
static_frontend = ["windmill-api/static_frontend"]
scoped_cache = ["windmill-common/scoped_cache"]
# Languages
python = ["windmill-worker/python"]
python = ["windmill-worker/python", "windmill-api/python"]
rust = ["windmill-worker/rust"]
mysql = ["windmill-worker/mysql"]
oracledb = ["windmill-worker/oracledb"]
@@ -135,10 +135,12 @@ quote.workspace = true
memchr.workspace = true
v8 = { workspace = true, optional = true }
rustls.workspace = true
pep440_rs.workspace = true
systemstat.workspace = true
size.workspace = true
strum.workspace = true
[target.'cfg(not(target_env = "msvc"))'.dependencies]
tikv-jemallocator = { optional = true, workspace = true }
tikv-jemalloc-sys = { optional = true, workspace = true }
@@ -219,6 +221,7 @@ git-version = "^0"
malachite = "=0.4.18"
malachite-bigint = "=0.2.0"
rustpython-parser = "^0"
pep440_rs = "0.7.3"
php-parser-rs = { git = "https://github.com/php-rust-tools/parser", rev = "ec4cb411dec09450946ef57920b7ffced7f6495d" }
cron = "^0"
mail-send = { version = "0.4.0", features = ["builder"], default-features=false }
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
99182f521564ff2d08dc2faf0af00f74f1834d60
72e6260ca886628cf1ba271bc058e6ecfdecdae5
backend/parsers/windmill-parser-py-imports/Cargo.toml
+3
View File
@@ -27,3 +27,6 @@ anyhow.workspace = true
lazy_static.workspace = true
sqlx.workspace = true
async-recursion.workspace = true
toml.workspace = true
serde.workspace = true
pep440_rs.workspace = true
backend/parsers/windmill-parser-py-imports/src/lib.rs
+99 -43
View File
@@ -11,7 +11,7 @@ mod mapping;
use async_recursion::async_recursion;
use itertools::Itertools;
use lazy_static::lazy_static;
use std::collections::HashMap;
use std::{collections::HashMap, str::FromStr};
use mapping::{FULL_IMPORTS_MAP, SHORT_IMPORTS_MAP};
#[cfg(not(target_arch = "wasm32"))]
@@ -25,7 +25,10 @@ use rustpython_parser::{
Parse,
};
use sqlx::{Pool, Postgres};
use windmill_common::{error, worker::PythonAnnotations};
use windmill_common::{
error::{self, to_anyhow},
worker::PythonAnnotations,
};
const DEF_MAIN: &str = "def main(";
@@ -242,8 +245,7 @@ pub async fn parse_python_imports(
w_id: &str,
path: &str,
db: &Pool<Postgres>,
already_visited: &mut Vec<String>,
annotated_pyv_numeric: &mut Option<u32>,
version_specifiers: &mut Vec<pep440_rs::VersionSpecifier>,
) -> error::Result<(Vec<String>, Option<String>)> {
let mut compile_error_hint: Option<String> = None;
let mut imports = parse_python_imports_inner(
@@ -251,9 +253,10 @@ pub async fn parse_python_imports(
w_id,
path,
db,
already_visited,
annotated_pyv_numeric,
&mut annotated_pyv_numeric.and_then(|_| Some(path.to_owned())),
&mut vec![],
version_specifiers,
// &mut version_specifier.and_then(|_| Some(path.to_owned())),
&mut None
)
.await?
.into_values()
@@ -279,6 +282,7 @@ pub async fn parse_python_imports(
.flatten()
.collect::<error::Result<Vec<String>>>()?
.into_iter()
.filter(|x| !x.trim_start().starts_with("--") && !x.trim().is_empty())
.unique()
.collect_vec();
@@ -304,11 +308,34 @@ async fn parse_python_imports_inner(
path: &str,
db: &Pool<Postgres>,
already_visited: &mut Vec<String>,
annotated_pyv_numeric: &mut Option<u32>,
version_specifiers: &mut Vec<pep440_rs::VersionSpecifier>,
path_where_annotated_pyv: &mut Option<String>,
) -> error::Result<HashMap<String, NImportResolved>> {
let PythonAnnotations { py310, py311, py312, py313, .. } = PythonAnnotations::parse(&code);
let mut push_version_specifiers = |perform, unparsed: String| -> error::Result<()> {
if perform {
pep440_rs::VersionSpecifiers::from_str(unparsed.as_str())
.ok()
.map(|vs| version_specifiers.extend(vs.to_vec()));
}
Ok(())
};
push_version_specifiers(py310, "==3.10.*".to_owned())?;
push_version_specifiers(py311, "==3.11.*".to_owned())?;
push_version_specifiers(py312, "==3.12.*".to_owned())?;
push_version_specifiers(py313, "==3.13.*".to_owned())?;
for x in code.lines() {
if x.starts_with("# py:") || x.starts_with("#py:") {
push_version_specifiers(
true,
x.replace('#', "").replace("py:", "").trim().to_owned(),
)?;
} else if !x.starts_with('#') {
break;
}
}
// we pass only if there is none or only one annotation
// Naive:
@@ -323,39 +350,48 @@ async fn parse_python_imports_inner(
// This way we make sure there is no multiple annotations for same script
// and we get detailed span on conflicting versions
let mut check = |is_py_xyz, numeric| -> error::Result<()> {
if is_py_xyz {
if let Some(v) = annotated_pyv_numeric {
if *v != numeric {
return Err(error::Error::from(anyhow::anyhow!(
"Annotated 2 or more different python versions: \n - py{v} at {}\n - py{numeric} at {path}\nIt is possible to use only one.",
path_where_annotated_pyv.clone().unwrap_or("Unknown".to_owned())
)));
}
} else {
*annotated_pyv_numeric = Some(numeric);
}
*path_where_annotated_pyv = Some(path.to_owned());
}
Ok(())
};
#[derive(serde::Serialize, serde::Deserialize)]
struct InlineMetadata {
requires_python: String,
dependencies: Vec<String>,
}
check(py310, 310)?;
check(py311, 311)?;
check(py312, 312)?;
check(py313, 313)?;
let find_requirements = code
.lines()
.find_position(|x| x.starts_with("#requirements:") || x.starts_with("# requirements:"));
if let Some((pos, _)) = find_requirements {
let find_requirements = code.lines().find_position(|x| {
x.starts_with("#requirements:")
|| x.starts_with("# requirements:")
|| x.starts_with("# /// script")
});
if let Some((pos, item)) = find_requirements {
let mut requirements = HashMap::new();
code.lines()
.skip(pos + 1)
.map_while(|x| {
RE.captures(x).and_then(|x| {
x.get(1).map(|m| {
let requirement = m.as_str().to_string();
if item.starts_with("# /// script") {
let mut incorrect = false;
let metadata = code
.lines()
.skip(pos + 1)
.map_while(|x| {
incorrect = !x.starts_with('#');
if incorrect || x.starts_with("# ///") {
None
} else {
x.get(1..)
}
})
.join("\n")
.parse::<toml::Table>()
.map_err(to_anyhow)?;
{
if let Some(v) = metadata.get("requires-python").and_then(|v| v.as_str()) {
push_version_specifiers(true, v.to_owned())?;
}
};
metadata
.get("dependencies")
.and_then(|dependencies| dependencies.as_array())
.inspect(|list| {
for dependency_v in list.into_iter() {
let requirement = dependency_v.as_str().unwrap_or("ERROR").to_owned();
let key = extract_pkg_name(&requirement);
requirements.insert(
key.clone(),
@@ -367,11 +403,31 @@ async fn parse_python_imports_inner(
key,
},
);
}
});
} else {
code.lines()
.skip(pos + 1)
.map_while(|x| {
RE.captures(x).and_then(|x| {
x.get(1).map(|m| {
let requirement = m.as_str().to_string();
let key = extract_pkg_name(&requirement);
requirements.insert(
key.clone(),
NImportResolved::Pin {
pins: vec![ImportPin {
pkg: requirement.clone(),
path: Default::default(),
}],
key,
},
);
})
})
})
})
.collect_vec();
.collect_vec();
}
Ok(requirements)
} else {
let find_extra_requirements = code.lines().find_position(|x| {
@@ -442,7 +498,7 @@ async fn parse_python_imports_inner(
&rpath,
db,
already_visited,
annotated_pyv_numeric,
version_specifiers,
path_where_annotated_pyv,
)
.await?
backend/parsers/windmill-parser-py-imports/tests/tests.rs
+6 -30
View File
@@ -18,16 +18,8 @@ def main():
pass
";
let mut already_visited = vec![];
let (r, ..) = parse_python_imports(
code,
"test-workspace",
"f/foo/bar",
&db,
&mut already_visited,
&mut None,
)
.await?;
let (r, ..) =
parse_python_imports(code, "test-workspace", "f/foo/bar", &db, &mut vec![]).await?;
// println!("{}", serde_json::to_string(&r)?);
assert_eq!(
r,
@@ -59,16 +51,8 @@ def main():
pass
";
let mut already_visited = vec![];
let (r, ..) = parse_python_imports(
code,
"test-workspace",
"f/foo/bar",
&db,
&mut already_visited,
&mut None,
)
.await?;
let (r, ..) =
parse_python_imports(code, "test-workspace", "f/foo/bar", &db, &mut vec![]).await?;
println!("{}", serde_json::to_string(&r)?);
assert_eq!(r, vec!["burkina=0.4", "nigeria"]);
@@ -89,17 +73,9 @@ def main():
pass
";
let mut already_visited = vec![];
let (r, ..) = parse_python_imports(
code,
"test-workspace",
"f/foo/bar",
&db,
&mut already_visited,
&mut None,
)
.await?;
let (r, ..) =
parse_python_imports(code, "test-workspace", "f/foo/bar", &db, &mut vec![]).await?;
println!("{}", serde_json::to_string(&r)?);
assert_eq!(
r,
backend/tests/fixtures/multipython.sql
Vendored
+20
View File
@@ -0,0 +1,20 @@
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'test-user',
'# py312
',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'',
'',
'f/multipython/aliases', 2468135790, 'python3', '');
INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
'test-workspace',
'test-user',
'# py: >=3.9,!=3.12.2
',
'{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}',
'',
'',
'f/multipython/script1', 2345678901, 'python3', '');
backend/tests/worker.rs
+41 -8
View File
@@ -3970,7 +3970,7 @@ async fn assert_lockfile(
#[sqlx::test(fixtures("base", "lockfile_python"))]
async fn test_requirements_python(db: Pool<Postgres>) {
let content = r#"
# py311
# py: 3.11.11
# requirements:
# tiny==0.1.3
@@ -3988,7 +3988,7 @@ def main():
&db,
content,
ScriptLang::Python3,
vec!["# py311", "tiny==0.1.3"],
vec!["# py: 3.11.11", "tiny==0.1.3"],
)
.await;
}
@@ -3998,7 +3998,7 @@ def main():
async fn test_extra_requirements_python(db: Pool<Postgres>) {
{
let content = r#"
# py311
# py: ==3.11.11
# extra_requirements:
# tiny
@@ -4016,7 +4016,7 @@ def main():
&db,
content,
ScriptLang::Python3,
vec!["# py311", "bottle==0.13.2", "tiny==0.1.2"],
vec!["# py: 3.11.11", "bottle==0.13.2", "tiny==0.1.2"],
)
.await;
}
@@ -4026,7 +4026,7 @@ def main():
#[sqlx::test(fixtures("base", "lockfile_python"))]
async fn test_extra_requirements_python2(db: Pool<Postgres>) {
let content = r#"
# py311
# py: ==3.11.11
# extra_requirements:
# tiny==0.1.3
@@ -4040,7 +4040,7 @@ def main():
&db,
content,
ScriptLang::Python3,
vec!["# py311", "simplejson==3.20.1", "tiny==0.1.3"],
vec!["# py: 3.11.11", "simplejson==3.20.1", "tiny==0.1.3"],
)
.await;
}
@@ -4049,7 +4049,7 @@ def main():
#[sqlx::test(fixtures("base", "lockfile_python"))]
async fn test_pins_python(db: Pool<Postgres>) {
let content = r#"
# py311
# py: ==3.11.11
# extra_requirements:
# tiny==0.1.3
# bottle==0.13.2
@@ -4069,7 +4069,7 @@ def main():
content,
ScriptLang::Python3,
vec![
"# py311",
"# py: 3.11.11",
"bottle==0.13.2",
"microdot==2.2.0",
"simplejson==3.19.3",
@@ -4078,6 +4078,39 @@ def main():
)
.await;
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "multipython"))]
async fn test_multipython_python(db: Pool<Postgres>) {
let content = r#"# py: <=3.12.2, >=3.12.0
import f.multipython.script1
import f.multipython.aliases
"#
.to_string();
assert_lockfile(&db, content, ScriptLang::Python3, vec!["# py: 3.12.1\n"]).await;
}
#[cfg(feature = "python")]
#[sqlx::test(fixtures("base", "multipython"))]
async fn test_inline_script_metadata_python(db: Pool<Postgres>) {
let content = r#"# py_select_latest
# /// script
# requires-python = ">3.11,<3.12.3,!=3.12.2"
# dependencies = [
# "tiny==0.1.3",
# ]
# ///
"#
.to_string();
assert_lockfile(
&db,
content,
ScriptLang::Python3,
vec!["# py: 3.12.1", "tiny==0.1.3"],
)
.await;
}
#[sqlx::test(fixtures("base", "result_format"))]
async fn test_result_format(db: Pool<Postgres>) {
let ordered_result_job_id = "1eecb96a-c8b0-4a3d-b1b6-087878c55e41";
backend/windmill-api/Cargo.toml
+1
View File
@@ -36,6 +36,7 @@ deno_core = ["dep:deno_core", "dep:deno_error"]
gcp_trigger = ["dep:thiserror", "dep:google-cloud-pubsub", "dep:google-cloud-googleapis", "dep:tonic"]
cloud = ["windmill-common/cloud"]
mcp = ["dep:rmcp"]
python = []
[dependencies]
rmcp = { git = "https://github.com/windmill-labs/rust-sdk", features = ["transport-sse-server"], optional = true }
backend/windmill-api/openapi.yaml
+17 -1
View File
@@ -11105,6 +11105,23 @@ paths:
items:
$ref: "#/components/schemas/AutoscalingEvent"
/configs/list_available_python_versions:
get:
summary: Get currently available python versions provided by UV.
operationId: listAvailablePythonVersions
tags:
- config
# parameters:
responses:
"200":
description: List of python versions
content:
application/json:
schema:
type: array
items:
type: string
/agent_workers/create_agent_token:
post:
summary: create agent token
@@ -16802,7 +16819,6 @@ components:
type: string
required:
- s3
TeamsChannel:
type: object
required:
backend/windmill-api/src/configs.rs
+22
View File
@@ -33,6 +33,10 @@ pub fn global_service() -> Router {
"/list_autoscaling_events/:worker_group",
get(list_autoscaling_events),
)
.route(
"/list_available_python_versions",
get(list_available_python_versions),
)
}
#[derive(Serialize, Deserialize, FromRow)]
@@ -205,6 +209,24 @@ async fn list_autoscaling_events(
Ok(Json(events))
}
async fn list_available_python_versions() -> error::JsonResult<Vec<String>> {
#[cfg(not(feature = "python"))]
return Err(error::Error::BadRequest(
"Python listing available only with 'python' feature enabled".to_string(),
));
#[cfg(feature = "python")]
use itertools::Itertools;
#[cfg(feature = "python")]
return Ok(Json(
windmill_worker::PyV::list_available_python_versions()
.await
.iter()
.map(|v| v.to_string())
.collect_vec(),
));
}
#[cfg(feature = "enterprise")]
async fn list_configs(
authed: ApiAuthed,
backend/windmill-common/src/worker.rs
+1
View File
@@ -516,6 +516,7 @@ fn parse_file<T: FromStr>(path: &str) -> Option<T> {
pub struct PythonAnnotations {
pub no_cache: bool,
pub no_postinstall: bool,
pub py_select_latest: bool,
pub skip_result_postprocessing: bool,
pub py310: bool,
pub py311: bool,
backend/windmill-queue/src/jobs.rs
+10 -1
View File
@@ -2123,10 +2123,19 @@ pub struct PulledJob {
pub permissioned_as_folders: Option<Vec<serde_json::Value>>,
}
// NOTE:
// Precomputed by the server
// Used to offload work from agent workers to server
#[derive(Serialize, Deserialize)]
pub enum PrecomputedAgentInfo {
Bun { local: String, remote: String },
Python { py_version: Option<u32>, requirements: Option<String> },
Python {
// V1, not used anymore. Exists for compat.
// TODO: Needs to be removed eventually
py_version: Option<u32>,
py_version_v2: Option<String>,
requirements: Option<String> },
}
#[derive(Serialize, Deserialize)]
backend/windmill-worker/Cargo.toml
+1
View File
@@ -116,6 +116,7 @@ convert_case.workspace = true
yaml-rust.workspace = true
backon.workspace = true
winapi = { workspace = true, optional = true }
pep440_rs.workspace = true
opentelemetry = { workspace = true, optional = true }
bollard = { workspace = true, optional = true }
backend/windmill-worker/src/ansible_executor.rs
+6 -9
View File
@@ -30,9 +30,9 @@ use crate::{
start_child_process, transform_json, OccupancyMetrics,
},
handle_child::handle_child,
python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile, PyVersion},
AuthedClient, DISABLE_NSJAIL, DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV,
PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV,
python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile},
AuthedClient, PyVAlias, DISABLE_NSJAIL, DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH,
PATH_ENV, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV,
};
lazy_static::lazy_static! {
@@ -373,7 +373,7 @@ async fn handle_ansible_python_deps(
worker_name,
w_id,
&mut Some(occupancy_metrics),
PyVersion::Py311,
PyVAlias::Py311.into(),
false,
)
.await
@@ -387,10 +387,7 @@ async fn handle_ansible_python_deps(
if requirements.len() > 0 {
let mut venv_path = handle_python_reqs(
requirements
.split("\n")
.filter(|x| !x.starts_with("--"))
.collect(),
crate::python_executor::split_requirements(requirements),
job_id,
w_id,
mem_peak,
@@ -400,7 +397,7 @@ async fn handle_ansible_python_deps(
job_dir,
worker_dir,
&mut Some(occupancy_metrics),
crate::python_executor::PyVersion::Py311,
PyVAlias::default().into(),
)
.await?;
additional_python_paths.append(&mut venv_path);
backend/windmill-worker/src/global_cache.rs
+4 -1
View File
@@ -22,6 +22,7 @@ pub async fn build_tar_and_push(
platform_agnostic: bool,
) -> error::Result<()> {
use object_store::path::Path;
use tokio::fs::create_dir_all;
use crate::TAR_PYBASE_CACHE_DIR;
@@ -36,7 +37,9 @@ pub async fn build_tar_and_push(
};
let prefix = &format!("{TAR_PYBASE_CACHE_DIR}/{}", lang);
let tar_path = format!("{prefix}/{folder_name}_tar.tar",);
let tar_path = format!("{prefix}/{folder_name}_tar.tar");
create_dir_all(prefix).await?;
let tar_file = std::fs::File::create(&tar_path)?;
let mut tar = tar::Builder::new(tar_file);
backend/windmill-worker/src/lib.rs
+5
View File
@@ -39,6 +39,8 @@ mod pg_executor;
mod php_executor;
#[cfg(feature = "python")]
mod python_executor;
#[cfg(feature = "python")]
mod python_versions;
pub mod result_processor;
#[cfg(feature = "rust")]
mod rust_executor;
@@ -60,3 +62,6 @@ pub use bun_executor::{
prebundle_bun_script, prepare_job_dir,
};
pub use deno_executor::generate_deno_lock;
#[cfg(feature = "python")]
pub use python_versions::{PyV, PyVAlias};
backend/windmill-worker/src/python_executor.rs
+145 -506
View File
@@ -3,6 +3,7 @@ use std::{
fs,
path::Path,
process::Stdio,
str::FromStr,
sync::Arc,
};
@@ -38,12 +39,12 @@ use std::env::var;
use windmill_queue::{append_logs, CanceledBy, PrecomputedAgentInfo};
lazy_static::lazy_static! {
static ref PYTHON_PATH: Option<String> = var("PYTHON_PATH").ok().map(|v| {
pub(crate) static ref PYTHON_PATH: Option<String> = var("PYTHON_PATH").ok().map(|v| {
tracing::warn!("PYTHON_PATH is set to {} and thus python will not be managed by uv and stay static regardless of annotation and instance settings. NOT RECOMMENDED", v);
v
});
static ref UV_PATH: String =
pub(crate) static ref UV_PATH: String =
var("UV_PATH").unwrap_or_else(|_| "/usr/local/bin/uv".to_string());
static ref PY_CONCURRENT_DOWNLOADS: usize =
@@ -77,348 +78,11 @@ use crate::{
start_child_process, OccupancyMetrics,
},
handle_child::handle_child,
worker_lockfiles::LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT,
worker_utils::ping_job_status,
AuthedClient, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, INSTANCE_PYTHON_VERSION, NSJAIL_PATH,
PATH_ENV, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR,
AuthedClient, PyV, PyVAlias, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR,
};
// To change latest stable version:
// 1. Change placeholder in instanceSettings.ts
// 2. Change LATEST_STABLE_PY in dockerfile
// 3. Change #[default] annotation for PyVersion in backend
#[derive(Eq, PartialEq, Clone, Copy, Default, Debug)]
pub enum PyVersion {
Py310,
#[default]
Py311,
Py312,
Py313,
}
impl PyVersion {
pub async fn from_instance_version(job_id: &Uuid, w_id: &str, conn: &Connection) -> Self {
let mut err = None;
let pyv = match INSTANCE_PYTHON_VERSION.read().await.clone() {
Some(v) => PyVersion::from_string_with_dots(&v).unwrap_or_else(|| {
let v = PyVersion::default();
err = Some(format!("\nCannot parse INSTANCE_PYTHON_VERSION ({:?}), fallback to latest_stable ({v:?})", *INSTANCE_PYTHON_VERSION));
v
}),
// Use latest stable
None => PyVersion::default(),
};
if let Some(msg) = err {
append_logs(job_id, w_id, &msg, conn).await;
tracing::error!(msg);
}
pyv
}
/// e.g.: `/tmp/windmill/cache/python_3xy`
pub fn to_cache_dir(&self) -> String {
use windmill_common::worker::ROOT_CACHE_DIR;
format!("{ROOT_CACHE_DIR}{}", &self.to_cache_dir_top_level())
}
/// e.g.: `python_3xy`
pub fn to_cache_dir_top_level(&self) -> String {
format!("python_{}", self.to_string_no_dot())
}
/// e.g.: `3xy`
pub fn to_string_no_dot(&self) -> String {
self.to_string_with_dot().replace('.', "")
}
/// e.g.: `3.xy`
pub fn to_string_with_dot(&self) -> &str {
use PyVersion::*;
match self {
Py310 => "3.10",
Py311 => "3.11",
Py312 => "3.12",
Py313 => "3.13",
}
}
pub fn from_string_with_dots(value: &str) -> Option<Self> {
use PyVersion::*;
match value {
"3.10" => Some(Py310),
"3.11" => Some(Py311),
"3.12" => Some(Py312),
"3.13" => Some(Py313),
"default" => Some(PyVersion::default()),
_ => {
tracing::warn!(
"Cannot convert string (\"{value}\") to PyVersion\nExpected format x.yz"
);
None
}
}
}
pub fn from_string_no_dots(value: &str) -> Option<Self> {
use PyVersion::*;
match value {
"310" => Some(Py310),
"311" => Some(Py311),
"312" => Some(Py312),
"313" => Some(Py313),
"default" => Some(PyVersion::default()),
_ => {
tracing::warn!(
"Cannot convert string (\"{value}\") to PyVersion\nExpected format xyz"
);
None
}
}
}
/// e.g.: `# py3xy` -> `PyVersion::Py3XY`
pub fn parse_version(line: &str) -> Option<Self> {
Self::from_string_no_dots(line.replace(" ", "").replace("#py", "").as_str())
}
pub fn from_py_annotations(a: PythonAnnotations) -> Option<Self> {
let PythonAnnotations { py310, py311, py312, py313, .. } = a;
use PyVersion::*;
if py313 {
Some(Py313)
} else if py312 {
Some(Py312)
} else if py311 {
Some(Py311)
} else if py310 {
Some(Py310)
} else {
None
}
}
pub fn from_numeric(n: u32) -> Option<Self> {
use PyVersion::*;
match n {
310 => Some(Py310),
311 => Some(Py311),
312 => Some(Py312),
313 => Some(Py313),
_ => None,
}
}
pub fn to_numeric(&self) -> u32 {
use PyVersion::*;
match self {
Py310 => 310,
Py311 => 311,
Py312 => 312,
Py313 => 313,
}
}
pub async fn get_python(
&self,
job_id: &Uuid,
mem_peak: &mut i32,
// canceled_by: &mut Option<CanceledBy>,
conn: &Connection,
worker_name: &str,
w_id: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> error::Result<Option<String>> {
// lazy_static::lazy_static! {
// static ref PYTHON_PATHS: Arc<RwLock<HashMap<PyVersion, String>>> = Arc::new(RwLock::new(HashMap::new()));
// }
let res = self
.get_python_inner(job_id, mem_peak, conn, worker_name, w_id, occupancy_metrics)
.await;
if let Err(ref e) = res {
tracing::error!(
"worker_name: {worker_name}, w_id: {w_id}, job_id: {job_id}\n
Error while getting python from uv, falling back to system python: {e:?}"
);
append_logs(
job_id,
w_id,
format!(
"\nError while getting python from uv, falling back to system python: {e:?}"
),
conn,
)
.await;
}
res
}
async fn get_python_inner(
self,
job_id: &Uuid,
mem_peak: &mut i32,
// canceled_by: &mut Option<CanceledBy>,
conn: &Connection,
worker_name: &str,
w_id: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> error::Result<Option<String>> {
let py_path = self.find_python().await;
// Runtime is not installed
if py_path.is_err() {
// Install it
if let Err(err) = self
.install_python(job_id, mem_peak, conn, worker_name, w_id, occupancy_metrics)
.await
{
tracing::error!("Cannot install python: {err}");
return Err(err);
} else {
// Try to find one more time
let py_path = self.find_python().await;
if let Err(err) = py_path {
tracing::error!("Cannot find python version {err}");
return Err(err);
}
// TODO: Cache the result
py_path
}
} else {
py_path
}
}
async fn install_python(
self,
job_id: &Uuid,
mem_peak: &mut i32,
// canceled_by: &mut Option<CanceledBy>,
conn: &Connection,
worker_name: &str,
w_id: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> error::Result<()> {
let v = self.to_string_with_dot();
append_logs(job_id, w_id, format!("\nINSTALLING PYTHON ({})", v), conn).await;
// Create dirs for newly installed python
// If we dont do this, NSJAIL will not be able to mount cache
// For the default version directory created during startup (main.rs)
DirBuilder::new()
.recursive(true)
.create(self.to_cache_dir())
.await
.expect("could not create initial worker dir");
let logs = String::new();
#[cfg(windows)]
let uv_cmd = "uv";
#[cfg(unix)]
let uv_cmd = UV_PATH.as_str();
let mut child_cmd = Command::new(uv_cmd);
child_cmd
.env_clear()
.env("HOME", HOME_ENV.to_string())
.env("PATH", PATH_ENV.to_string())
.envs(PROXY_ENVS.clone())
.args(["python", "install", v, "--python-preference=only-managed"])
// TODO: Do we need these?
.envs([("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR)])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
#[cfg(windows)]
{
child_cmd
.env("SystemRoot", SYSTEM_ROOT.as_str())
.env("USERPROFILE", crate::USERPROFILE_ENV.as_str())
.env(
"TMP",
std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")),
)
.env(
"LOCALAPPDATA",
std::env::var("LOCALAPPDATA")
.unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())),
);
}
let child_process = start_child_process(child_cmd, "uv").await?;
append_logs(&job_id, &w_id, logs, conn).await;
handle_child(
job_id,
conn,
mem_peak,
&mut None,
child_process,
false,
worker_name,
&w_id,
"uv",
None,
false,
occupancy_metrics,
None,
)
.await
}
async fn find_python(self) -> error::Result<Option<String>> {
#[cfg(windows)]
let uv_cmd = "uv";
#[cfg(unix)]
let uv_cmd = UV_PATH.as_str();
let mut child_cmd = Command::new(uv_cmd);
child_cmd.env_clear();
#[cfg(windows)]
{
child_cmd
.env("SystemRoot", SYSTEM_ROOT.as_str())
.env("USERPROFILE", crate::USERPROFILE_ENV.as_str())
.env(
"TMP",
std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")),
)
.env(
"LOCALAPPDATA",
std::env::var("LOCALAPPDATA")
.unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())),
);
}
let output = child_cmd
// .current_dir(job_dir)
.env("HOME", HOME_ENV.to_string())
.env("PATH", PATH_ENV.to_string())
.args([
"python",
"find",
self.to_string_with_dot(),
"--system",
"--python-preference=only-managed",
])
.envs([
("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR),
("UV_PYTHON_PREFERENCE", "only-managed"),
])
// .stdout(Stdio::piped())
.stderr(Stdio::piped())
.output()
.await?;
// Check if the command was successful
if output.status.success() {
// Convert the output to a String
let stdout =
String::from_utf8(output.stdout).expect("Failed to convert output to String");
return Ok(Some(stdout.replace('\n', "")));
} else {
// If the command failed, print the error
let stderr =
String::from_utf8(output.stderr).expect("Failed to convert error output to String");
return Err(error::Error::FindPythonError(stderr));
}
}
}
#[cfg(windows)]
use crate::SYSTEM_ROOT;
@@ -465,7 +129,7 @@ pub async fn uv_pip_compile(
worker_name: &str,
w_id: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
py_version: PyVersion,
py_version: PyV,
// Debug-only flag
no_cache: bool,
) -> error::Result<String> {
@@ -502,10 +166,11 @@ pub async fn uv_pip_compile(
requirements.to_string()
};
let py_version_str = py_version.clone().to_string();
// Include python version to requirements.in
// We need it because same hash based on requirements.in can get calculated even for different python versions
// To prevent from overwriting same requirements.in but with different python versions, we include version to hash
let requirements = format!("# py{}\n{}", py_version.to_string_no_dot(), requirements);
let requirements = format!("# py: {}\n{}", py_version.to_string(), requirements);
#[cfg(feature = "enterprise")]
let requirements = replace_pip_secret(conn, w_id, &requirements, worker_name, job_id).await?;
@@ -525,7 +190,7 @@ pub async fn uv_pip_compile(
{
logs.push_str(&format!(
"\nFound cached resolution: {req_hash}, on python version: {}",
py_version.to_string_with_dot()
&py_version_str
));
return Ok(cached);
}
@@ -539,7 +204,7 @@ pub async fn uv_pip_compile(
{
// Make sure we have python runtime installed
py_version
.get_python(job_id, mem_peak, conn, worker_name, w_id, occupancy_metrics)
.try_get_python(job_id, mem_peak, conn, worker_name, w_id, occupancy_metrics)
.await?;
let mut args = vec![
@@ -561,12 +226,7 @@ pub async fn uv_pip_compile(
UV_CACHE_DIR,
];
args.extend([
"-p",
&py_version.to_string_with_dot(),
"--python-preference",
"only-managed",
]);
args.extend(["-p", &py_version_str, "--python-preference", "only-managed"]);
if no_cache {
args.extend(["--no-cache"]);
@@ -666,8 +326,8 @@ pub async fn uv_pip_compile(
let mut req_content = "".to_string();
file.read_to_string(&mut req_content).await?;
let lockfile = format!(
"# py{}\n{}",
py_version.to_string_no_dot(),
"# py: {}\n{}",
py_version.to_string(),
req_content
.lines()
.filter(|x| !x.trim_start().starts_with('#'))
@@ -789,37 +449,6 @@ async fn postinstall(
Ok(())
}
async fn get_python_path(
py_version: PyVersion,
worker_name: &str,
job_id: &Uuid,
w_id: &str,
mem_peak: &mut i32,
conn: &Connection,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> windmill_common::error::Result<String> {
let python_path = if let Some(python_path) = PYTHON_PATH.clone() {
python_path
} else if let Some(python_path) = py_version
.get_python(
&job_id,
mem_peak,
conn,
worker_name,
w_id,
occupancy_metrics,
)
.await?
{
python_path
} else {
return Err(Error::ExecutionErr(format!(
"uv could not manage python path. Please manage it manually by setting PYTHON_PATH environment variable to your python binary path"
)));
};
Ok(python_path)
}
#[tracing::instrument(level = "trace", skip_all)]
pub async fn handle_python_job(
requirements_o: Option<&String>,
@@ -863,16 +492,16 @@ pub async fn handle_python_job(
.await?;
tracing::debug!("Finished handling python dependencies");
let python_path = get_python_path(
py_version,
worker_name,
&job.id,
&job.workspace_id,
mem_peak,
conn,
&mut Some(occupancy_metrics),
)
.await?;
let python_path = py_version
.get_python(
worker_name,
&job.id,
&job.workspace_id,
mem_peak,
conn,
&mut Some(occupancy_metrics),
)
.await?;
if !annotations.no_postinstall {
if let Err(e) = postinstall(&mut additional_python_paths, job_dir, job, conn).await {
@@ -887,7 +516,7 @@ pub async fn handle_python_job(
&job.workspace_id,
format!(
"\n\n--- PYTHON ({}) CODE EXECUTION ---\n",
py_version.to_string_with_dot()
py_version.clone().to_string()
),
conn,
)
@@ -1026,7 +655,7 @@ except BaseException as e:
let mut reserved_variables =
get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?;
// Add /tmp/windmill/cache/python_xyz/global-site-packages to PYTHONPATH.
// Add /tmp/windmill/cache/python_x_y_z/global-site-packages to PYTHONPATH.
// Usefull if certain wheels needs to be preinstalled before execution.
let global_site_packages_path = py_version.to_cache_dir() + "/global-site-packages";
let additional_python_paths_folders = {
@@ -1039,9 +668,9 @@ except BaseException as e:
// Since we handle mount of global_site_packages on our own, we don't want it to be mounted automatically.
// We do this because existence of every wheel in cache is mandatory and if it is not there and nsjail expects it, it is a bug.
// On the other side global_site_packages is purely optional.
// NOTE: This behaviour can be changed in future, so verification of wheels can be offloaded from nsjail to windmill
// NOTE: This behaviour can be changed in future, so verification of wheels can be delegated from nsjail to windmill
paths.insert(0, global_site_packages_path.clone());
// ^^^^^^^^
// ^^^^^^ ^
// We also want this be priorotized, that's why we insert it to the beginning
}
paths.iter().join(":")
@@ -1434,7 +1063,7 @@ async fn handle_python_deps(
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
precomputed_agent_info: Option<PrecomputedAgentInfo>,
annotations: PythonAnnotations,
) -> error::Result<(PyVersion, Vec<String>)> {
) -> error::Result<(PyV, Vec<String>)> {
create_dependencies_dir(job_dir).await;
let mut additional_python_paths: Vec<String> = WORKER_CONFIG
@@ -1445,90 +1074,116 @@ async fn handle_python_deps(
.unwrap_or_else(|| vec![])
.clone();
let mut requirements;
let compilation_error_hint;
let mut annotated_pyv = None;
let mut annotated_pyv_numeric = None;
let is_deployed = requirements_o.is_some();
let instance_pyv = PyVersion::from_instance_version(job_id, w_id, conn).await;
let requirements = match requirements_o {
Some(r) => r,
let (pyv, resolved_lines) = match requirements_o {
// Deployed
Some(r) => {
let rl = split_requirements(r);
(PyV::parse_from_requirements(&rl), rl)
}
// Preview
None => {
let mut already_visited = vec![];
(requirements, compilation_error_hint) = match conn {
let (v, requirements_lines, error_hint) = match conn {
Connection::Sql(db) => {
let mut version_specifiers = vec![];
let (r, h) = windmill_parser_py_imports::parse_python_imports(
inner_content,
w_id,
script_path,
db,
&mut already_visited,
&mut annotated_pyv_numeric,
&mut version_specifiers,
)
.await?;
(r.join("\n"), h)
let v = PyV::resolve(
version_specifiers,
job_id,
w_id,
annotations.py_select_latest,
Some(conn.clone()),
None,
None,
)
.await?;
(v, r, h)
}
Connection::Http(_) => match precomputed_agent_info {
Some(PrecomputedAgentInfo::Python { py_version, requirements }) => {
annotated_pyv_numeric = py_version;
(requirements.clone().unwrap_or_else(|| "".to_string()), None)
Some(PrecomputedAgentInfo::Python {
requirements,
py_version,
py_version_v2,
}) => {
let v = {
let v_v2 = py_version_v2
.clone()
.and_then(|s| pep440_rs::Version::from_str(&s).ok().map(PyV::from));
let v_v1 = py_version.and_then(PyVAlias::try_from_v1).map(PyV::from);
match v_v2.or(v_v1) {
Some(v) => v,
None => {
tracing::warn!(
workspace_id = %w_id,
"
Failed to get precomputed python version from server. Fallback to Default ({})
Returned from server: py_version - {:?}, py_version_v2 - {:?}
",
*PyV::default(),
py_version,
py_version_v2
);
Default::default()
}
}
};
let r = split_requirements(requirements.unwrap_or_default());
let h = None;
(v, r, h)
}
_ => ("".to_string(), None),
_ => Default::default(),
},
};
annotated_pyv = annotated_pyv_numeric.and_then(|v| PyVersion::from_numeric(v));
if !requirements.is_empty() {
requirements = uv_pip_compile(
job_id,
&requirements,
mem_peak,
canceled_by,
job_dir,
conn,
worker_name,
w_id,
occupancy_metrics,
annotated_pyv.unwrap_or(instance_pyv),
annotations.no_cache,
)
.await
.map_err(|e| {
Error::ExecutionErr(format!(
"pip compile failed: {}{}",
e.to_string(),
compilation_error_hint.unwrap_or_default()
))
})?;
}
&requirements
(
v.clone(),
if !requirements_lines.is_empty() {
uv_pip_compile(
job_id,
&requirements_lines.join("\n"),
mem_peak,
canceled_by,
job_dir,
conn,
worker_name,
w_id,
occupancy_metrics,
// annotated_pyv.unwrap_or(instance_pyv),
v,
annotations.no_cache,
)
.await
.map_err(|e| {
Error::ExecutionErr(format!(
"pip compile failed: {}{}",
e.to_string(),
error_hint.unwrap_or_default()
))
})?
.lines()
.map(|s| s.to_owned())
.collect_vec()
} else {
vec![]
},
)
}
};
/*
For deployed scripts we want to find out version in following order:
1. Assigned version (written in lockfile)
2. 3.11
For Previews:
1. Annotated version
2. Instance version
3. Latest Stable
*/
let requirements_lines = split_requirements(requirements.as_str());
let final_version = if is_deployed {
get_pyv_from_requirements_lines(&requirements_lines)
} else {
// This is not deployed script, meaning we test run it (Preview)
annotated_pyv.unwrap_or(instance_pyv)
};
// If len > 0 it means there is atleast one dependency or assigned python version
if requirements.len() > 0 {
if !resolved_lines.is_empty() {
let mut venv_path = handle_python_reqs(
requirements_lines,
resolved_lines,
job_id,
w_id,
mem_peak,
@@ -1538,13 +1193,13 @@ async fn handle_python_deps(
job_dir,
worker_dir,
occupancy_metrics,
final_version,
pyv.clone(),
)
.await?;
additional_python_paths.append(&mut venv_path);
}
Ok((final_version, additional_python_paths))
Ok((pyv, additional_python_paths))
}
lazy_static::lazy_static! {
@@ -1733,7 +1388,7 @@ async fn spawn_uv_install(
/// uv pip install, include cached or pull from S3
pub async fn handle_python_reqs(
requirements: Vec<&str>,
requirements: Vec<String>,
job_id: &Uuid,
w_id: &str,
mem_peak: &mut i32,
@@ -1743,7 +1398,7 @@ pub async fn handle_python_reqs(
job_dir: &str,
worker_dir: &str,
_occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
py_version: PyVersion,
py_version: PyV,
) -> error::Result<Vec<String>> {
let worker_dir = worker_dir.to_string();
@@ -2017,7 +1672,7 @@ pub async fn handle_python_reqs(
let total_time = std::time::Instant::now();
let py_path = py_version
.get_python(
.try_get_python(
job_id,
mem_peak,
conn,
@@ -2059,6 +1714,10 @@ pub async fn handle_python_reqs(
let py_path = py_path.clone();
let pids = pids.clone();
let worker_dir = worker_dir.clone();
#[cfg(all(feature = "enterprise", feature = "parquet", unix))]
let py_version = py_version.clone();
handles.push(task::spawn(async move {
// permit will be dropped anyway if this thread exits at any point
// so we dont have to drop it manually
@@ -2300,36 +1959,14 @@ pub async fn handle_python_reqs(
};
}
fn split_requirements(requirements: &str) -> Vec<&str> {
pub fn split_requirements<T: AsRef<str>>(requirements: T) -> Vec<String> {
requirements
.split("\n")
.as_ref()
.lines()
.filter(|x| !x.trim_start().starts_with("--") && !x.trim().is_empty())
.map(String::from)
.collect()
}
/// Check requirements/lockfile to figure out python version assigned to it.
fn get_pyv_from_requirements_lines(requirements_lines: &[&str]) -> PyVersion {
// If script is deployed we can try to parse first line to get assigned version
let index = if requirements_lines.get(0).map_or(false, |line| {
line.starts_with(LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT)
}) {
1
} else {
0
};
if let Some(v) = requirements_lines
.get(index)
.and_then(|line| PyVersion::parse_version(*line))
{
// We have valid assigned version, we use it
v
} else {
// If there is no assigned version in lockfile we automatically fallback to 3.11
// In this case we have dependencies, but no associated python version
// This is the case for old deployed scripts
PyVersion::Py311
}
}
// Returns code snippet that needs to be injected into wrapper to post-process results or leave unprocessed
fn get_result_postprocessor<'a>(skip: bool) -> &'a str {
@@ -2365,6 +2002,8 @@ pub async fn start_worker(
jobs_rx: tokio::sync::mpsc::Receiver<std::sync::Arc<MiniPulledJob>>,
killpill_rx: tokio::sync::broadcast::Receiver<()>,
) -> error::Result<()> {
use crate::{PyV, PyVAlias};
let mut mem_peak: i32 = 0;
let mut canceled_by: Option<CanceledBy> = None;
let context = variables::get_reserved_variables(
@@ -2518,22 +2157,22 @@ for line in sys.stdin:
proc_envs.insert("BASE_URL".to_string(), base_internal_url.to_string());
let py_version = if let Some(requirements) = requirements_o {
get_pyv_from_requirements_lines(&split_requirements(requirements.as_str()))
PyV::parse_from_requirements(&split_requirements(requirements.as_str()))
} else {
tracing::warn!(workspace_id = %w_id, "lockfile is empty for dedicated worker, thus python version cannot be inferred. Fallback to 3.11");
PyVersion::Py311
PyVAlias::Py311.into()
};
let python_path = get_python_path(
py_version,
worker_name,
&Uuid::nil(),
w_id,
&mut mem_peak,
&Connection::Sql(db.clone()),
&mut None,
)
.await?;
let python_path = py_version
.get_python(
worker_name,
&Uuid::nil(),
w_id,
&mut mem_peak,
&Connection::Sql(db.clone()),
&mut None,
)
.await?;
handle_dedicated_process(
&python_path,
job_dir,
backend/windmill-worker/src/python_versions.rs
+848
View File
@@ -0,0 +1,848 @@
use std::{
ops::{Deref, DerefMut},
process::Stdio,
str::FromStr,
sync::Arc,
};
use chrono::{DateTime, Duration, Utc};
use itertools::Itertools;
use serde_json::Value;
use tokio::{fs::DirBuilder, process::Command, sync::RwLock};
use uuid::Uuid;
use windmill_common::{
error::{self, Error},
worker::Connection,
};
use anyhow::{anyhow, bail};
use windmill_queue::append_logs;
use crate::{
common::{start_child_process, OccupancyMetrics},
handle_child::handle_child,
python_executor::{PYTHON_PATH, UV_PATH},
worker_lockfiles::LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT,
HOME_ENV, INSTANCE_PYTHON_VERSION, PATH_ENV, PROXY_ENVS, PY_INSTALL_DIR, WIN_ENVS,
};
#[derive(Eq, PartialEq, Clone, Copy, Default, Debug)]
#[repr(u32)]
pub enum PyVAlias {
Py310 = 10,
#[default]
Py311,
Py312,
Py313,
}
impl Into<pep440_rs::Version> for PyVAlias {
fn into(self) -> pep440_rs::Version {
pep440_rs::Version::new([self.major() as u64, self as u64])
}
}
impl Into<u32> for PyVAlias {
fn into(self) -> u32 {
self.major() * 100 + self as u32
}
}
impl From<PyV> for PyVAlias {
fn from(value: PyV) -> Self {
match value.release() {
[major, minor, ..] => {
if let Some(alias) = Self::try_from_v1(format!("{}{}", *major, *minor)) {
return alias;
}
}
_ => (),
}
tracing::warn!(
"Failed to convert Python Full Version to Alias. Fallback to default ({})",
*PyV::default()
);
Self::default()
}
}
impl PyVAlias {
fn all<T: From<PyVAlias>>() -> Vec<T> {
use PyVAlias::*;
vec![Py310.into(), Py311.into(), Py312.into(), Py313.into()]
}
// Get MAJOR part of alias. (semver: MAJOR.MINOR.PATCH)
fn major(&self) -> u32 {
use PyVAlias::*;
match self {
Py310 | Py311 | Py312 | Py313 => 3,
// Py400 | Py401 => 4
}
}
/// Converts numeric format to alias
/// Example:
/// 310u32 (in) -> PyVAlias::Py310 (out)
pub(crate) fn try_from_v1<T: ToString>(numeric: T) -> Option<Self> {
use PyVAlias::*;
match numeric.to_string().as_str() {
"310" => Some(Py310),
"311" => Some(Py311),
"312" => Some(Py312),
"313" => Some(Py313),
_ => None,
}
}
}
// To change latest stable version:
// 1. Change placeholder in instanceSettings.ts
// 2. Change LATEST_STABLE_PY in dockerfile
// 3. Change #[default] annotation for PyVersion in backend
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct PyV(pub pep440_rs::Version);
impl From<pep440_rs::Version> for PyV {
fn from(value: pep440_rs::Version) -> Self {
Self(value)
}
}
impl From<PyVAlias> for PyV {
fn from(value: PyVAlias) -> Self {
Self(value.into())
}
}
impl Default for PyV {
fn default() -> Self {
PyVAlias::default().into()
}
}
impl Deref for PyV {
type Target = pep440_rs::Version;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl DerefMut for PyV {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
impl PyV {
pub async fn resolve(
version_specifiers: Vec<pep440_rs::VersionSpecifier>,
job_id: &Uuid,
w_id: &str,
select_latest: bool,
// Needed for logs but optional
conn: Option<Connection>,
// Usually for testing
custom_versions: Option<Vec<PyV>>,
// For testing
gravitational_version: Option<PyV>,
) -> Result<Self, Error> {
// Get all versions that can be fetched
let all_versions = custom_versions.unwrap_or(PyV::list_available_python_versions().await);
// Narrow down to those that satisfy given version specifiers
let valid = all_versions
.clone()
.into_iter()
.filter(|v| version_specifiers.iter().all(|vs| vs.contains(&*v)))
.collect_vec();
if !valid.is_empty() {
if select_latest {
return Ok(valid[0].clone());
}
// Usually INSTANCE_PYTHON_VERSION
let gv = gravitational_version
.unwrap_or(PyV::gravitational_version(job_id, w_id, conn).await);
// Will be used to determine if picked version matches gravity version
// Once first match occure, we will stop iterating
let gravity_matcher = pep440_rs::VersionSpecifier::from_version(
pep440_rs::Operator::EqualStar,
(*gv).clone(),
)
.map_err(|e| {
Error::ArgumentErr(format!(
"{e}\nLikely means INSTANCE_PYTHON_VERSION is set incorrectly."
))
})?;
// Reminder of semver: MAJOR.MINOR.PATCH
//
// - Go from up to down
// - We will iterate until find the closest version to target.
// - If closest version has the same MINOR version, use it.
// - If it differs in MINOR version, take latest PATCH version.
//
let mut result = None;
// This represents newest version with oldest MINOR:
//
// I Iterable Newest in MINOR
// 1. 3.11.2 -> 3.11.2
// 2. 3.11.1 -> 3.11.2
// 3. 3.11.0 -> 3.11.2
// 4. 3.10.2 -> 3.10.2
// 5. 3.10.1 -> 3.10.2
// 6. 3.10.0 -> 3.10.2
let mut newest_in_minor = None;
for v in valid.iter() {
if result.is_none() {
result.replace(v);
}
if v < &gv {
// We will not continue if we start looking into versions older than gravity version.
break;
}
let [major, minor, ..] = v.release() else {
return Err(Error::InternalErr(format!("Failed to parse \"{}\". Available python versions are supposed to be in SEMVER format (MAJOR.MINOR)", **v)));
};
// Since we go top to down we can assume
// the first occurence of new minor version contains the latest patch version.
if matches!(newest_in_minor, Some((_, mm)) if mm != (major, minor))
|| newest_in_minor.is_none()
{
newest_in_minor = Some((v.clone(), (major, minor)));
}
if gravity_matcher.contains(v) {
// return as soon as gravity matcher has first hit.
return Ok(v.clone());
}
// If we are still in the loop, it means that we are getting closer to gravity version
else {
result = Some(v);
}
}
let [gravity_major, gravity_minor, ..] = gv.release() else {
return Err(Error::internal_err(format!("Cannot get MAJOR or/and MINOR version of python gravity version ({}). Something might be wrong with INSTANCE_PYTHON_VERSION.", &*gv)));
};
if let Some((v, mm)) = newest_in_minor {
if (gravity_major, gravity_minor) != mm {
return Ok(v);
}
}
result
.ok_or(Error::internal_err(
"No python candidates found. This is a bug!",
))
.map(ToOwned::to_owned)
} else {
Err(anyhow!(
"
× No solution found when resolving python:
╰─▶ Because you require python {}, we can conclude that your requirements are unsatisfiable.
All versions: \n{}
\n",
version_specifiers.iter().map(|s| s.to_string()).join(", "),
all_versions
.iter()
.enumerate()
.map(|(i, v)| format!(
"{}{}",
windmill_common::worker::pad_string(&v.0.to_string(), 11),
if (i + 1) % 5 == 0 { "\n" } else { "" }
))
.collect::<String>()
)
.into())
}
}
/// e.g.: `/tmp/windmill/cache/python_3xy`
pub(crate) fn to_cache_dir(&self) -> String {
use windmill_common::worker::ROOT_CACHE_DIR;
format!("{ROOT_CACHE_DIR}{}", self.to_cache_dir_top_level())
}
/// e.g.: `python_3_x_y`
pub fn to_cache_dir_top_level(&self) -> String {
format!("python_{}", self.to_string().replace(".", "_"))
}
pub async fn gravitational_version(
job_id: &Uuid,
w_id: &str,
conn: Option<Connection>,
) -> Self {
let mut err = None;
let pyv = match INSTANCE_PYTHON_VERSION.read().await.clone() {
Some(v) => pep440_rs::Version::from_str(&v).unwrap_or_else(|_| {
let v = PyVAlias::default().into();
err = Some(format!("\nCannot parse INSTANCE_PYTHON_VERSION ({:?}), fallback to latest_stable ({v:?})", *INSTANCE_PYTHON_VERSION));
v
}),
// Use latest stable
None => PyVAlias::default().into(),
};
if let Some(msg) = err {
if let Some(conn) = conn {
append_logs(job_id, w_id, &msg, &conn).await;
}
tracing::error!(msg);
}
pyv.into()
}
pub async fn list_available_python_versions() -> Vec<Self> {
match Self::list_available_python_versions_inner().await {
Ok(pyvs) => pyvs,
Err(e) => {
tracing::error!(
"Fallback to preconfigured aliases. Cannot list python versions due to this error: {e}"
);
PyVAlias::all()
}
}
}
async fn list_available_python_versions_inner() -> anyhow::Result<Vec<Self>> {
lazy_static::lazy_static! {
static ref CACHED_VERSIONS: Arc<RwLock<Option<Vec<PyV>>>> = Arc::new(RwLock::new(None));
static ref LAST_CHECKED: Arc<RwLock<DateTime<Utc>>> = Arc::new(RwLock::new(Utc::now()));
}
match (
Utc::now().signed_duration_since(*LAST_CHECKED.read().await) > Duration::minutes(30),
CACHED_VERSIONS.read().await.clone(),
) {
(false, Some(vs)) => return Ok(vs),
_ => {}
};
let output = {
#[cfg(windows)]
let uv_cmd = "uv";
#[cfg(unix)]
let uv_cmd = UV_PATH.as_str();
Command::new(uv_cmd)
.env_clear()
.envs(WIN_ENVS.to_vec())
.args([
"python",
"list",
"--all-versions",
"--output-format",
"json",
])
.stderr(Stdio::piped())
.output()
.await?
};
// We want to skip all versions smaller then 3.10
// Windmill is incompatible with 3.9 and older
let filter = pep440_rs::VersionSpecifier::from_version(
pep440_rs::Operator::GreaterThanEqual,
PyVAlias::Py310.into(),
)?;
if output.status.success() {
let res = String::from_utf8(output.stdout)?;
tracing::error!("{}", &res);
let list = serde_json::from_str::<Vec<serde_json::Map<String, Value>>>(&res)?
.into_iter()
.filter_map(|e| {
if e.get("implementation").and_then(Value::as_str) == Some("pypy") {
None
} else {
Some(
e.get("version")
.and_then(Value::as_str)
.and_then(|s| pep440_rs::Version::from_str(s).ok())
.map(PyV::from)
.ok_or(Error::internal_err("version is None")),
)
}
})
.collect::<Result<Vec<PyV>, Error>>()?
.into_iter()
.unique()
.sorted()
.filter(|pyv| filter.contains(&*pyv))
.rev()
.collect_vec();
*LAST_CHECKED.write().await = Utc::now();
CACHED_VERSIONS.write().await.replace(list.clone());
Ok(list)
} else {
// If the command failed, print the error
let stderr = String::from_utf8(output.stderr)?;
bail!(
"Cannot list python versions, is uv (0.5.19 and newer) installed? Err:\n{}",
stderr
);
}
}
/// Parse lockfile for assigned python version.
/// If not found returns 3.11
pub fn parse_from_requirements<S: AsRef<str>>(requirements_lines: &[S]) -> Self {
Self::try_parse_from_requirements(requirements_lines).unwrap_or(
// If there is no assigned version in lockfile we automatically fallback to 3.11
// In this case we have dependencies or other metadata, but no associated python version
// This is the case for old deployed scripts
PyVAlias::Py311.into(),
)
}
/// Parse lockfile for assigned python version.
/// If not found returns None
pub fn try_parse_from_requirements<S: AsRef<str>>(requirements_lines: &[S]) -> Option<Self> {
let parse_version = |s: &str| -> Option<PyV> {
// Possible inputs:
// V2:
// # py: 3.11.0 or #py:3.11.0 or #py: 3.11.0
//
// V1:
// # py311 or #py311
let version_unparsed = s
.to_owned()
// Remove whitespaces. That leaves us with:
// V2: #py:3.11.0
// V1: #py311
//
// Remove #
// V2: py:3.11.0
// V1: py311
//
// Remove :
// V2: py3.11.0
// V1: py311
.replace([' ', '#', ':'], "")
// Remove "py"
// V2: 3.11.0
// V1: 311
.replace("py", "");
// We will support reading V1 syntax, but it will be overwritten next deploy
PyVAlias::try_from_v1(&version_unparsed)
.map(PyVAlias::into)
.or(pep440_rs::Version::from_str(&version_unparsed)
.ok()
.map(pep440_rs::Version::into))
};
let index = if requirements_lines.get(0).map_or(false, |line| {
line.as_ref()
.starts_with(LOCKFILE_GENERATED_FROM_REQUIREMENTS_TXT)
}) {
1
} else {
0
};
requirements_lines
.get(index)
.map(S::as_ref)
.and_then(parse_version)
}
pub async fn get_python(
&self,
worker_name: &str,
job_id: &Uuid,
w_id: &str,
mem_peak: &mut i32,
conn: &Connection,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> windmill_common::error::Result<String> {
let python_path = if let Some(python_path) = PYTHON_PATH.clone() {
python_path
} else if let Some(python_path) = self
.try_get_python(
&job_id,
mem_peak,
conn,
worker_name,
w_id,
occupancy_metrics,
)
.await?
{
python_path
} else {
return Err(Error::ExecutionErr(format!(
"uv could not manage python path. Please manage it manually by setting PYTHON_PATH environment variable to your python binary path"
)));
};
Ok(python_path)
}
pub async fn try_get_python(
&self,
job_id: &Uuid,
mem_peak: &mut i32,
// canceled_by: &mut Option<CanceledBy>,
conn: &Connection,
worker_name: &str,
w_id: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> error::Result<Option<String>> {
// lazy_static::lazy_static! {
// static ref PYTHON_PATHS: Arc<RwLock<HashMap<PyVersion, String>>> = Arc::new(RwLock::new(HashMap::new()));
// }
let res = self
.get_python_inner(job_id, mem_peak, conn, worker_name, w_id, occupancy_metrics)
.await;
if let Err(ref e) = res {
tracing::error!(
"worker_name: {worker_name}, w_id: {w_id}, job_id: {job_id}\n
Error while getting python from uv, falling back to system python: {e:?}"
);
append_logs(
job_id,
w_id,
format!(
"\nError while getting python from uv, falling back to system python: {e:?}"
),
conn,
)
.await;
}
res
}
async fn get_python_inner(
&self,
job_id: &Uuid,
mem_peak: &mut i32,
// canceled_by: &mut Option<CanceledBy>,
conn: &Connection,
worker_name: &str,
w_id: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> error::Result<Option<String>> {
let py_path = self.find_python().await;
// Runtime is not installed
if py_path.is_err() {
// Install it
if let Err(err) = self
.install_python(job_id, mem_peak, conn, worker_name, w_id, occupancy_metrics)
.await
{
tracing::error!("Cannot install python: {err}");
return Err(err);
} else {
// Try to find one more time
let py_path = self.find_python().await;
if let Err(err) = py_path {
tracing::error!("Cannot find python version {err}");
return Err(err);
}
// TODO: Cache the result
py_path
}
} else {
py_path
}
}
async fn install_python(
&self,
job_id: &Uuid,
mem_peak: &mut i32,
// canceled_by: &mut Option<CanceledBy>,
conn: &Connection,
worker_name: &str,
w_id: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> error::Result<()> {
let v = self.to_string();
append_logs(job_id, w_id, format!("\nINSTALLING PYTHON ({})", v), conn).await;
// Create dirs for newly installed python
// If we dont do this, NSJAIL will not be able to mount cache
// For the default version directory created during startup (main.rs)
DirBuilder::new()
.recursive(true)
.create(self.to_cache_dir())
.await
.expect("could not create initial worker dir");
let logs = String::new();
#[cfg(windows)]
let uv_cmd = "uv";
#[cfg(unix)]
let uv_cmd = UV_PATH.as_str();
let mut child_cmd = Command::new(uv_cmd);
child_cmd
.env_clear()
.env("HOME", HOME_ENV.to_string())
.env("PATH", PATH_ENV.to_string())
.envs(PROXY_ENVS.clone())
.args(["python", "install", &v, "--python-preference=only-managed"])
// TODO: Do we need these?
.envs([("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR)])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
#[cfg(windows)]
{
child_cmd
.env("SystemRoot", crate::SYSTEM_ROOT.as_str())
.env("USERPROFILE", crate::USERPROFILE_ENV.as_str())
.env(
"TMP",
std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")),
)
.env(
"LOCALAPPDATA",
std::env::var("LOCALAPPDATA")
.unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())),
);
}
let child_process = start_child_process(child_cmd, "uv").await?;
append_logs(&job_id, &w_id, logs, conn).await;
handle_child(
job_id,
conn,
mem_peak,
&mut None,
child_process,
false,
worker_name,
&w_id,
"uv",
None,
false,
occupancy_metrics,
None,
)
.await
}
async fn find_python(&self) -> error::Result<Option<String>> {
#[cfg(windows)]
let uv_cmd = "uv";
#[cfg(unix)]
let uv_cmd = UV_PATH.as_str();
let mut child_cmd = Command::new(uv_cmd);
child_cmd.env_clear();
#[cfg(windows)]
{
child_cmd
.env("SystemRoot", crate::SYSTEM_ROOT.as_str())
.env("USERPROFILE", crate::USERPROFILE_ENV.as_str())
.env(
"TMP",
std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")),
)
.env(
"LOCALAPPDATA",
std::env::var("LOCALAPPDATA")
.unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())),
);
}
let output = child_cmd
// .current_dir(job_dir)
.env("HOME", HOME_ENV.to_string())
.env("PATH", PATH_ENV.to_string())
.args([
"python",
"find",
&self.to_string(),
"--system",
"--python-preference=only-managed",
])
.envs([
("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR),
("UV_PYTHON_PREFERENCE", "only-managed"),
])
// .stdout(Stdio::piped())
.stderr(Stdio::piped())
.output()
.await?;
// Check if the command was successful
if output.status.success() {
// Convert the output to a String
let stdout =
String::from_utf8(output.stdout).expect("Failed to convert output to String");
return Ok(Some(stdout.replace('\n', "")));
} else {
// If the command failed, print the error
let stderr =
String::from_utf8(output.stderr).expect("Failed to convert error output to String");
return Err(error::Error::FindPythonError(stderr));
}
}
}
#[cfg(test)]
mod tests {
use super::*;
/// Unsafe helper for testing
fn pyv(value: &str) -> PyV {
pep440_rs::Version::from_str(value).unwrap().into()
}
async fn assert_resolution(
instance_version: &str,
select_highest: bool,
specifiers: Vec<&str>,
available: Vec<PyV>,
expected: PyV,
) {
let resolved = PyV::resolve(
specifiers
.into_iter()
.map(|s| pep440_rs::VersionSpecifier::from_str(s).unwrap())
.collect_vec(),
&Uuid::nil(),
"",
select_highest,
None,
Some(available),
Some(pyv(instance_version)),
)
.await
.unwrap();
assert_eq!(expected, resolved);
}
#[tokio::test]
async fn test_python_resolution_1() {
assert_resolution(
"1.0",
false,
vec![],
vec![
pyv("1.2.0"),
pyv("1.1.0"),
pyv("1.0.0"),
pyv("0.9.0"), //
],
pyv("1.0.0"), //
)
.await;
}
#[tokio::test]
async fn test_python_resolution_2() {
assert_resolution(
"1.0.0",
false,
vec!["!=1.*"],
vec![
pyv("1.2"),
pyv("1.1"),
pyv("1.0.2"),
pyv("1.0.1"),
pyv("1.0.0"),
pyv("0.9.4"),
pyv("0.9.3"),
pyv("0.9.2"),
],
pyv("0.9.4"), //
)
.await;
}
#[tokio::test]
async fn test_python_resolution_3() {
assert_resolution(
"0.9",
false,
vec!["!=0.9.*"],
vec![
pyv("1.2"),
pyv("1.1"),
pyv("1.0.2"),
pyv("1.0.1"),
pyv("1.0.0"),
pyv("0.9.4"),
pyv("0.9.3"),
pyv("0.9.2"),
pyv("0.8.2"),
pyv("0.8.1"),
pyv("0.8.0"),
],
pyv("1.0.2"), //
)
.await;
}
#[tokio::test]
async fn test_python_resolution_4() {
assert_resolution(
"0.9",
false,
vec!["<=0.8.1"],
vec![pyv("1.0.0"), pyv("0.9.0"), pyv("0.8.1"), pyv("0.8.0")],
pyv("0.8.1"), //
)
.await;
}
#[tokio::test]
async fn test_python_resolution_5() {
assert_resolution(
"0.0.1",
false,
vec!["!=0.1.0"],
vec![pyv("2.1.0"), pyv("1.1.0"), pyv("0.1.0")],
pyv("1.1.0"),
)
.await;
}
#[tokio::test]
async fn test_python_resolution_6() {
assert_resolution(
"1.1.1",
false,
vec![],
vec![
pyv("3.0.1"),
pyv("3.0.0"),
pyv("2.2.2"),
pyv("2.2.1"),
pyv("2.2.0"),
],
pyv("2.2.2"),
)
.await;
}
#[tokio::test]
async fn test_python_resolution_7() {
assert_resolution(
"2.2.1",
true,
vec![],
vec![
pyv("3.0.1"),
pyv("3.0.0"),
pyv("2.2.2"),
pyv("2.2.1"),
pyv("2.2.0"),
],
pyv("3.0.1"),
)
.await;
}
}
backend/windmill-worker/src/worker.rs
+24 -5
View File
@@ -134,7 +134,10 @@ use crate::java_executor::{handle_java_job, JobHandlerInput as JobHandlerInputJa
use crate::php_executor::handle_php_job;
#[cfg(feature = "python")]
use crate::python_executor::{handle_python_job, PyVersion};
use crate::{
python_executor::handle_python_job,
python_versions::{PyV, PyVAlias},
};
#[cfg(feature = "python")]
use crate::ansible_executor::handle_ansible_job;
@@ -363,10 +366,26 @@ lazy_static::lazy_static! {
}
type Envs = Vec<(String, String)>;
#[cfg(windows)]
lazy_static::lazy_static! {
pub static ref SYSTEM_ROOT: String = std::env::var("SystemRoot").unwrap_or_else(|_| "C:\\Windows".to_string());
pub static ref USERPROFILE_ENV: String = std::env::var("USERPROFILE").unwrap_or_else(|_| "/tmp".to_string());
static ref TMP: String = std::env::var("TMP").unwrap_or_else(|_| "/tmp".to_string());
static ref LOCALAPPDATA: String = std::env::var("LOCALAPPDATA").unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str()));
pub static ref WIN_ENVS: Envs = vec![
("SystemRoot".into(), SYSTEM_ROOT.clone()),
("USERPROFILE".into(), USERPROFILE_ENV.clone()),
("TMP".into(), TMP.clone()),
("LOCALAPPDATA".into(), LOCALAPPDATA.clone())
];
}
#[cfg(not(windows))]
lazy_static::lazy_static! {
pub static ref WIN_ENVS: Envs = vec![];
}
//only matter if CLOUD_HOSTED
@@ -828,9 +847,9 @@ pub async fn run_worker(
worker_dir.clone(),
);
tokio::spawn(async move {
if let Err(e) = PyVersion::from_instance_version(&Uuid::nil(), "", &conn)
if let Err(e) = PyV::gravitational_version(&Uuid::nil(), "", Some(conn.clone()))
.await
.get_python(&Uuid::nil(), &mut 0, &conn, &worker_name, "", &mut None)
.try_get_python(&Uuid::nil(), &mut 0, &conn, &worker_name, "", &mut None)
.await
{
tracing::error!(
@@ -840,8 +859,8 @@ pub async fn run_worker(
"Cannot preinstall or find Instance Python version to worker: {e}"//
);
}
if let Err(e) = PyVersion::Py311
.get_python(&Uuid::nil(), &mut 0, &conn, &worker_name, "", &mut None)
if let Err(e) = PyV::from(PyVAlias::Py311)
.try_get_python(&Uuid::nil(), &mut 0, &conn, &worker_name, "", &mut None)
.await
{
tracing::error!(
backend/windmill-worker/src/worker_lockfiles.rs
+44 -42
View File
@@ -47,7 +47,7 @@ use crate::java_executor::resolve;
use crate::php_executor::{composer_install, parse_php_imports};
#[cfg(feature = "python")]
use crate::python_executor::{
create_dependencies_dir, handle_python_reqs, uv_pip_compile, PyVersion,
create_dependencies_dir, handle_python_reqs, split_requirements, uv_pip_compile,
};
#[cfg(feature = "rust")]
use crate::rust_executor::generate_cargo_lockfile;
@@ -1897,26 +1897,13 @@ async fn python_dep(
w_id: &str,
worker_dir: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
annotated_pyv_numeric: Option<u32>,
py_version: crate::PyV,
annotations: PythonAnnotations,
) -> std::result::Result<String, Error> {
use crate::python_executor::split_requirements;
create_dependencies_dir(job_dir).await;
/*
Unlike `handle_python_deps` which we use for running scripts (deployed and drafts)
This one used specifically for deploying scripts
So we can get final_version right away and include in lockfile
And the precendence is following:
1. Annotation version
2. Instance version
3. Latest Stable
*/
let final_version = annotated_pyv_numeric
.and_then(|pyv| PyVersion::from_numeric(pyv))
.unwrap_or(PyVersion::from_instance_version(job_id, w_id, &db.into()).await);
let req: std::result::Result<String, Error> = uv_pip_compile(
job_id,
&reqs,
@@ -1927,14 +1914,15 @@ async fn python_dep(
worker_name,
w_id,
occupancy_metrics,
final_version,
py_version,
annotations.no_cache,
)
.await;
// install the dependencies to pre-fill the cache
if let Ok(req) = req.as_ref() {
let r = handle_python_reqs(
req.split("\n").filter(|x| !x.starts_with("--")).collect(),
split_requirements(req),
// req.split("\n").filter(|x| !x.starts_with("--")).collect(),
job_id,
w_id,
mem_peak,
@@ -1944,7 +1932,8 @@ async fn python_dep(
job_dir,
worker_dir,
occupancy_metrics,
final_version,
// final_version,
crate::PyVAlias::default().into(),
)
.await;
@@ -1994,7 +1983,7 @@ async fn ansible_dep(
w_id,
worker_dir,
&mut Some(occupancy_metrics),
None,
crate::PyV::gravitational_version(job_id, w_id, Some(db.clone().into())).await,
PythonAnnotations::default(),
)
.await?;
@@ -2104,31 +2093,44 @@ async fn capture_dependency_job(
));
#[cfg(feature = "python")]
{
let anns = PythonAnnotations::parse(job_raw_code);
let mut annotated_pyv_numeric = None;
let reqs = if raw_deps {
// Manually assigned version from requirements.txt
// let assigned_py_version;
let (reqs, py_version) = if raw_deps {
// `wmill script generate-metadata`
// should also respect annotated pyversion
// can be annotated in script itself
// or in requirements.txt if present
annotated_pyv_numeric =
PyVersion::from_py_annotations(anns).map(|v| v.to_numeric());
job_raw_code.to_string()
} else {
let mut already_visited = vec![];
windmill_parser_py_imports::parse_python_imports(
job_raw_code,
&w_id,
script_path,
&db,
&mut already_visited,
&mut annotated_pyv_numeric,
(
job_raw_code.to_owned(),
crate::PyV::parse_from_requirements(&split_requirements(job_raw_code)),
)
} else {
let mut version_specifiers = vec![];
let PythonAnnotations { py_select_latest, .. } =
PythonAnnotations::parse(job_raw_code);
(
windmill_parser_py_imports::parse_python_imports(
job_raw_code,
&w_id,
script_path,
&db,
&mut version_specifiers,
)
.await?
.0
.join("\n"),
crate::PyV::resolve(
version_specifiers,
job_id,
w_id,
py_select_latest,
Some(db.clone().into()),
None,
None,
)
.await?,
)
.await?
.0
.join("\n")
};
python_dep(
@@ -2142,8 +2144,8 @@ async fn capture_dependency_job(
w_id,
worker_dir,
&mut Some(occupancy_metrics),
annotated_pyv_numeric,
anns,
py_version,
PythonAnnotations::parse(job_raw_code),
)
.await
.map(|res| {
frontend/src/lib/components/InstanceSetting.svelte
+91 -2
View File
@@ -17,11 +17,18 @@
import ObjectStoreConfigSettings from './ObjectStoreConfigSettings.svelte'
import { sendUserToast } from '$lib/toast'
import ConfirmButton from './ConfirmButton.svelte'
import { IndexSearchService, SettingService, TeamsService } from '$lib/gen'
import {
ConfigService,
IndexSearchService,
SettingService,
TeamsService,
type ListAvailablePythonVersionsResponse
} from '$lib/gen'
import { Button, SecondsInput, Skeleton } from './common'
import Password from './Password.svelte'
import { classNames } from '$lib/utils'
import Popover from './Popover.svelte'
import PopoverMelt from './meltComponents/Popover.svelte'
import Toggle from './Toggle.svelte'
import type { Writable } from 'svelte/store'
import { createEventDispatcher } from 'svelte'
@@ -30,6 +37,7 @@
import ToggleButtonGroup from './common/toggleButton-v2/ToggleButtonGroup.svelte'
import ToggleButton from './common/toggleButton-v2/ToggleButton.svelte'
import SimpleEditor from './SimpleEditor.svelte'
import LoadingIcon from './apps/svelte-select/lib/LoadingIcon.svelte'
import TeamSelector from './TeamSelector.svelte'
import ChannelSelector from './ChannelSelector.svelte'
@@ -39,7 +47,10 @@
export let loading = true
const dispatch = createEventDispatcher()
if (setting.fieldType == 'select' && $values[setting.key] == undefined) {
if (
(setting.fieldType == 'select' || setting.fieldType == 'select_python') &&
$values[setting.key] == undefined
) {
$values[setting.key] = 'default'
}
@@ -124,6 +135,24 @@
}
}
let pythonAvailableVersions: ListAvailablePythonVersionsResponse = []
let isPyFetching = false
async function fetch_available_python_versions() {
if (isPyFetching) return
isPyFetching = true
try {
pythonAvailableVersions = await ConfigService.listAvailablePythonVersions()
} catch (error) {
console.error('Error fetching python versions:', error)
} finally {
isPyFetching = false
}
}
if (setting.fieldType == 'select_python') {
fetch_available_python_versions()
}
async function fetchTeams() {
if (isFetching) return
isFetching = true
@@ -193,6 +222,66 @@
{/each}
</ToggleButtonGroup>
</div>
{:else if setting.fieldType == 'select_python'}
<div>
<!-- svelte-ignore a11y-label-has-associated-control -->
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">{setting.label}</span>
{#if setting.description}
<span class="text-secondary text-xs">
{@html setting.description}
</span>
{/if}
</label>
<ToggleButtonGroup bind:selected={$values[setting.key]} let:item={toggleButtonn}>
{#each setting.select_items ?? [] as item}
<ToggleButton
value={item.value ?? item.label}
label={item.label}
tooltip={item.tooltip}
item={toggleButtonn}
/>
{/each}
<PopoverMelt closeButton={!isPyFetching}>
<svelte:fragment slot="trigger">
{#if setting.select_items?.some((e) => e.label == $values[setting.key] || e.value == $values[setting.key])}
<Button
variant="border"
color="dark"
btnClasses="px-1.5 py-1.5 text-2xs bg-surface-secondary border-0"
nonCaptureEvent={true}>Select Custom</Button
>
{:else}
<Button
variant="border"
color="dark"
btnClasses="px-1.5 py-1.5 text-2xs border-0 shadow-md"
nonCaptureEvent={true}>Custom | {$values[setting.key]}</Button
>
{/if}
</svelte:fragment>
<svelte:fragment slot="content">
{#if isPyFetching}
<div class="p-4">
<LoadingIcon />
</div>
{:else}
<ToggleButtonGroup
bind:selected={$values[setting.key]}
let:item={toggleButtonn}
class="mr-10 h-full"
tabListClass="flex-wrap p-2"
>
{#each pythonAvailableVersions as item}
<ToggleButton value={item} label={item} tooltip={item} item={toggleButtonn} />
{/each}
</ToggleButtonGroup>
{/if}
</svelte:fragment>
</PopoverMelt>
</ToggleButtonGroup>
</div>
{:else}
<!-- svelte-ignore a11y-label-has-associated-control -->
<label class="block pb-2">
frontend/src/lib/components/instanceSettings.ts
+2 -1
View File
@@ -19,6 +19,7 @@ export interface Setting {
| 'boolean'
| 'password'
| 'select'
| 'select_python'
| 'textarea'
| 'codearea'
| 'seconds'
@@ -231,7 +232,7 @@ export const settings: Record<string, Setting[]> = {
label: 'Instance Python Version',
description: 'Default python version for newly deployed scripts',
key: 'instance_python_version',
fieldType: 'select',
fieldType: 'select_python',
// To change latest stable version:
// 1. Change placeholder in instanceSettings.ts
// 2. Change LATEST_STABLE_PY in dockerfile