starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-04 08:01:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: Handle pip install by uv (#4517)

Browse Source 
* feat: Handle `pip install` by `uv`

Dirty and untested, but already something working

* Integrate with NSJAIL and prepare fallbacks

* Refactor fallback
no_uv disable compile and install
where no_uv_install and no_uv_compile are a bit more specific

* Remove `--disable-pip-version-check`
Reason:
   warning: pip's `--disable-pip-version-check` has no effect

* Fix backend compilation error

* Pip fallback overwrite UV's cache

* Initially refactor cache (No S3)

* Support S3

* Remove unused import

* Handle flags for NSJAIL

* Return deleted flag

* Update Dockerfile

* Update docker-image.yml

* Update docker-image.yml

* Add --link-mode=copy and remove -v

* Fix NSJAIL INDEX_URL

* Fix flock and windows

* Update python_executor.rs

* Remove line from Dockerfile

We dont need it and to trigger build

* fixing for windows

* Dont pin python to specific version

* Change TMP for windows

* Revert docker-image.yml

* Disable UV for ansible

Will be enabled later.
Needs proper testing and its better to split onto 2 PRs with first modifying python and second ansible

---------

Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
Co-authored-by: Alexander Petric <petric.al@gmail.com>
This commit is contained in:
pyranota
2024-11-14 23:12:56 +00:00
committed by Alexander Petric
parent 7cfc87d5f3
commit 5403aab203
10 changed files with 285 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/main.rs
+2 -3
View File
@@ -65,8 +65,8 @@ use windmill_common::global_settings::OBJECT_STORE_CACHE_CONFIG_SETTING;
use windmill_worker::{
get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR,
BUN_DEPSTAR_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM,
GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR,
RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR,
GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, POWERSHELL_CACHE_DIR, RUST_CACHE_DIR,
TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR,
};
use crate::monitor::{
@@ -995,7 +995,6 @@ pub async fn run_workers<R: rsmq_async::RsmqConnection + Send + Sync + Clone + '
for x in [
LOCK_CACHE_DIR,
TMP_LOGS_DIR,
PIP_CACHE_DIR,
UV_CACHE_DIR,
TAR_PIP_CACHE_DIR,
DENO_CACHE_DIR,
backend/windmill-common/src/worker.rs
+2
View File
@@ -310,6 +310,8 @@ fn parse_file<T: FromStr>(path: &str) -> Option<T> {
pub struct PythonAnnotations {
pub no_cache: bool,
pub no_uv: bool,
pub no_uv_install: bool,
pub no_uv_compile: bool,
}
#[annotations("//")]
backend/windmill-worker/nsjail/download.py.pip.config.proto
+93
View File
@@ -0,0 +1,93 @@
name: "python download pip"
mode: ONCE
hostname: "python"
log_level: ERROR
time_limit: 900
rlimit_as: 2048
rlimit_cpu: 1000
rlimit_fsize: 1024
rlimit_nofile: 64
envar: "HOME=/user"
envar: "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH"
cwd: "/tmp"
clone_newnet: false
clone_newuser: {CLONE_NEWUSER}
keep_caps: true
keep_env: true
mount {
src: "/bin"
dst: "/bin"
is_bind: true
}
mount {
src: "/lib"
dst: "/lib"
is_bind: true
}
mount {
src: "/lib64"
dst: "/lib64"
is_bind: true
mandatory: false
}
mount {
src: "/usr"
dst: "/usr"
is_bind: true
}
mount {
src: "/etc"
dst: "/etc"
is_bind: true
}
mount {
src: "/dev/null"
dst: "/dev/null"
is_bind: true
rw: true
}
mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size=500000000"
}
mount {
src: "{WORKER_DIR}/download_deps.py.pip.sh"
dst: "/download_deps.sh"
is_bind: true
}
mount {
src: "{CACHE_DIR}"
dst: "{CACHE_DIR}"
is_bind: true
rw: true
}
mount {
src: "/dev/urandom"
dst: "/dev/urandom"
is_bind: true
}
exec_bin {
path: "/bin/sh"
arg: "/download_deps.sh"
}
backend/windmill-worker/nsjail/download_deps.py.pip.sh
Executable
+24
View File
@@ -0,0 +1,24 @@
#/bin/sh
INDEX_URL_ARG=$([ -z "$INDEX_URL" ] && echo ""|| echo "--index-url $INDEX_URL" )
EXTRA_INDEX_URL_ARG=$([ -z "$EXTRA_INDEX_URL" ] && echo ""|| echo "--extra-index-url $EXTRA_INDEX_URL" )
TRUSTED_HOST_ARG=$([ -z "$TRUSTED_HOST" ] && echo "" || echo "--trusted-host $TRUSTED_HOST")
if [ ! -z "$INDEX_URL" ]
then
echo "\$INDEX_URL is set to $INDEX_URL"
fi
if [ ! -z "$EXTRA_INDEX_URL" ]
then
echo "\$EXTRA_INDEX_URL is set to $EXTRA_INDEX_URL"
fi
if [ ! -z "$TRUSTED_HOST" ]
then
echo "\$TRUSTED_HOST is set to $TRUSTED_HOST"
fi
CMD="/usr/local/bin/python3 -m pip install -v \"$REQ\" -I -t \"$TARGET\" --no-cache --no-color --no-deps --isolated --no-warn-conflicts --disable-pip-version-check $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG"
echo $CMD
eval $CMD
backend/windmill-worker/nsjail/download_deps.py.sh
+13 -1
View File
@@ -19,6 +19,18 @@ then
echo "\$TRUSTED_HOST is set to $TRUSTED_HOST"
fi
CMD="/usr/local/bin/python3 -m pip install -v \"$REQ\" -I -t \"$TARGET\" --no-cache --no-color --no-deps --isolated --no-warn-conflicts --disable-pip-version-check $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG"
CMD="/usr/local/bin/uv pip install
\"$REQ\"
--target \"$TARGET\"
--no-cache
--no-config
--no-color
--no-deps
--link-mode=copy
$INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG
--index-strategy unsafe-best-match
--system
"
echo $CMD
eval $CMD
backend/windmill-worker/src/ansible_executor.rs
+2
View File
@@ -116,6 +116,8 @@ async fn handle_ansible_python_deps(
job_dir,
worker_dir,
&mut Some(occupancy_metrics),
true,
true,
)
.await?;
additional_python_paths.append(&mut venv_path);
backend/windmill-worker/src/global_cache.rs
+22 -4
View File
@@ -20,13 +20,21 @@ use std::sync::Arc;
pub async fn build_tar_and_push(
s3_client: Arc<dyn ObjectStore>,
folder: String,
no_uv: bool,
) -> error::Result<()> {
use object_store::path::Path;
use crate::PY311_CACHE_DIR;
tracing::info!("Started building and pushing piptar {folder}");
let start = Instant::now();
let folder_name = folder.split("/").last().unwrap();
let tar_path = format!("{PIP_CACHE_DIR}/{folder_name}_tar.tar",);
let prefix = if no_uv {
PIP_CACHE_DIR
} else {
PY311_CACHE_DIR
};
let tar_path = format!("{prefix}/{folder_name}_tar.tar",);
let tar_file = std::fs::File::create(&tar_path)?;
let mut tar = tar::Builder::new(tar_file);
@@ -46,7 +54,10 @@ pub async fn build_tar_and_push(
// })?;
if let Err(e) = s3_client
.put(
&Path::from(format!("/tar/pip/{folder_name}.tar")),
&Path::from(format!(
"/tar/{}/{folder_name}.tar",
if no_uv { "pip" } else { "python_311" }
)),
std::fs::read(&tar_path)?.into(),
)
.await
@@ -71,7 +82,11 @@ pub async fn build_tar_and_push(
}
#[cfg(all(feature = "enterprise", feature = "parquet"))]
pub async fn pull_from_tar(client: Arc<dyn ObjectStore>, folder: String) -> error::Result<()> {
pub async fn pull_from_tar(
client: Arc<dyn ObjectStore>,
folder: String,
no_uv: bool,
) -> error::Result<()> {
use windmill_common::s3_helpers::attempt_fetch_bytes;
let folder_name = folder.split("/").last().unwrap();
@@ -79,7 +94,10 @@ pub async fn pull_from_tar(client: Arc<dyn ObjectStore>, folder: String) -> erro
tracing::info!("Attempting to pull piptar {folder_name} from bucket");
let start = Instant::now();
let tar_path = format!("tar/pip/{folder_name}.tar");
let tar_path = format!(
"tar/{}/{folder_name}.tar",
if no_uv { "pip" } else { "python_311" }
);
let bytes = attempt_fetch_bytes(client, &tar_path).await?;
// tracing::info!("B: {target} {folder}");
backend/windmill-worker/src/python_executor.rs
+109 -35
View File
@@ -42,6 +42,10 @@ lazy_static::lazy_static! {
static ref USE_PIP_COMPILE: bool = std::env::var("USE_PIP_COMPILE")
.ok().map(|flag| flag == "true").unwrap_or(false);
/// Use pip install
static ref USE_PIP_INSTALL: bool = std::env::var("USE_PIP_INSTALL")
.ok().map(|flag| flag == "true").unwrap_or(false);
static ref RELATIVE_IMPORT_REGEX: Regex = Regex::new(r#"(import|from)\s(((u|f)\.)|\.)"#).unwrap();
@@ -50,6 +54,8 @@ lazy_static::lazy_static! {
}
const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = include_str!("../nsjail/download.py.config.proto");
const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT_FALLBACK: &str =
include_str!("../nsjail/download.py.pip.config.proto");
const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str = include_str!("../nsjail/run.python3.config.proto");
const RELATIVE_PYTHON_LOADER: &str = include_str!("../loader.py");
@@ -66,8 +72,8 @@ use crate::{
},
handle_child::handle_child,
AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, LOCK_CACHE_DIR,
NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, TZ_ENV,
UV_CACHE_DIR,
NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS,
PY311_CACHE_DIR, TZ_ENV, UV_CACHE_DIR,
};
#[cfg(windows)]
@@ -901,10 +907,10 @@ async fn handle_python_deps(
.unwrap_or_else(|| vec![])
.clone();
let annotations = windmill_common::worker::PythonAnnotations::parse(inner_content);
let requirements = match requirements_o {
Some(r) => r,
None => {
let annotation = windmill_common::worker::PythonAnnotations::parse(inner_content);
let mut already_visited = vec![];
let requirements = windmill_parser_py_imports::parse_python_imports(
@@ -929,8 +935,8 @@ async fn handle_python_deps(
worker_name,
w_id,
occupancy_metrics,
annotation.no_uv,
annotation.no_cache,
annotations.no_uv || annotations.no_uv_compile,
annotations.no_cache,
)
.await
.map_err(|e| {
@@ -955,6 +961,8 @@ async fn handle_python_deps(
job_dir,
worker_dir,
occupancy_metrics,
annotations.no_uv || annotations.no_uv_install,
false,
)
.await?;
additional_python_paths.append(&mut venv_path);
@@ -966,6 +974,7 @@ lazy_static::lazy_static! {
static ref PIP_SECRET_VARIABLE: Regex = Regex::new(r"\$\{PIP_SECRET:([^\s\}]+)\}").unwrap();
}
/// pip install, include cached or pull from S3
pub async fn handle_python_reqs(
requirements: Vec<&str>,
job_id: &Uuid,
@@ -977,12 +986,22 @@ pub async fn handle_python_reqs(
job_dir: &str,
worker_dir: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
// TODO: Remove (Deprecated)
mut no_uv_install: bool,
is_ansible: bool,
) -> error::Result<Vec<String>> {
let mut req_paths: Vec<String> = vec![];
let mut vars = vec![("PATH", PATH_ENV.as_str())];
let pip_extra_index_url;
let pip_index_url;
no_uv_install |= *USE_PIP_INSTALL;
if no_uv_install && !is_ansible {
append_logs(&job_id, w_id, "\nFallback to pip (Deprecated!)\n", db).await;
tracing::warn!("Fallback to pip");
}
if !*DISABLE_NSJAIL {
pip_extra_index_url = PIP_EXTRA_INDEX_URL
.read()
@@ -1013,10 +1032,21 @@ pub async fn handle_python_reqs(
let _ = write_file(
job_dir,
"download.config.proto",
&NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT
.replace("{WORKER_DIR}", &worker_dir)
.replace("{CACHE_DIR}", PIP_CACHE_DIR)
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()),
&(if no_uv_install {
NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT_FALLBACK
} else {
NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT
})
.replace("{WORKER_DIR}", &worker_dir)
.replace(
"{CACHE_DIR}",
if no_uv_install {
PIP_CACHE_DIR
} else {
PY311_CACHE_DIR
},
)
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()),
)?;
};
@@ -1026,8 +1056,14 @@ pub async fn handle_python_reqs(
if req.starts_with('#') {
continue;
}
let py_prefix = if no_uv_install {
PIP_CACHE_DIR
} else {
PY311_CACHE_DIR
};
let venv_p = format!(
"{PIP_CACHE_DIR}/{}",
"{py_prefix}/{}",
req.replace(' ', "").replace('/', "").replace(':', "")
);
if metadata(&venv_p).await.is_ok() {
@@ -1073,7 +1109,10 @@ pub async fn handle_python_reqs(
.map(|(req, venv_p)| {
let os = os.clone();
async move {
if pull_from_tar(os, venv_p.clone()).await.is_ok() {
if pull_from_tar(os, venv_p.clone(), no_uv_install)
.await
.is_ok()
{
PullFromTar::Pulled(venv_p.to_string())
} else {
PullFromTar::NotPulled(req.to_string(), venv_p.to_string())
@@ -1114,7 +1153,7 @@ pub async fn handle_python_reqs(
for (req, venv_p) in req_with_penv {
let mut logs1 = String::new();
logs1.push_str("\n\n--- PIP INSTALL ---\n");
logs1.push_str("\n\n--- UV PIP INSTALL ---\n");
logs1.push_str(&format!("\n{req} is being installed for the first time.\n It will be cached for all ulterior uses."));
append_logs(&job_id, w_id, logs1, db).await;
@@ -1150,21 +1189,47 @@ pub async fn handle_python_reqs(
#[cfg(windows)]
let req = format!("{}", req);
let mut command_args = vec![
PYTHON_PATH.as_str(),
"-m",
"pip",
"install",
&req,
"-I",
"--no-deps",
"--no-color",
"--isolated",
"--no-warn-conflicts",
"--disable-pip-version-check",
"-t",
venv_p.as_str(),
];
let mut command_args = if no_uv_install {
vec![
PYTHON_PATH.as_str(),
"-m",
"pip",
"install",
&req,
"-I",
"--no-deps",
"--no-color",
"--isolated",
"--no-warn-conflicts",
"--disable-pip-version-check",
"-t",
venv_p.as_str(),
]
} else {
vec![
UV_PATH.as_str(),
"pip",
"install",
&req,
"--no-deps",
"--no-color",
// "-p",
// "3.11",
// Prevent uv from discovering configuration files.
"--no-config",
"--link-mode=copy",
// TODO: Doublecheck it
"--system",
// Prefer main index over extra
// https://docs.astral.sh/uv/pip/compatibility/#packages-that-exist-on-multiple-indexes
// TODO: Use env variable that can be toggled from UI
"--index-strategy",
"unsafe-best-match",
"--target",
venv_p.as_str(),
"--no-cache",
]
};
let pip_extra_index_url = PIP_EXTRA_INDEX_URL
.read()
.await
@@ -1196,7 +1261,7 @@ pub async fn handle_python_reqs(
envs.push(("HOME", HOME_ENV.as_str()));
tracing::debug!("pip install command: {:?}", command_args);
tracing::debug!("uv pip install command: {:?}", command_args);
#[cfg(unix)]
{
@@ -1207,7 +1272,12 @@ pub async fn handle_python_reqs(
.envs(envs)
.args([
"-x",
&format!("{}/pip-{}.lock", LOCK_CACHE_DIR, fssafe_req),
&format!(
"{}/{}-{}.lock",
LOCK_CACHE_DIR,
if no_uv_install { "pip" } else { "py311" },
fssafe_req
),
"--command",
&command_args.join(" "),
])
@@ -1218,16 +1288,20 @@ pub async fn handle_python_reqs(
#[cfg(windows)]
{
let mut pip_cmd = Command::new(PYTHON_PATH.as_str());
pip_cmd
.env_clear()
let installer_path = if no_uv_install { command_args[0] } else { "uv" };
let mut cmd: Command = Command::new(&installer_path);
cmd.env_clear()
.envs(envs)
.envs(PROXY_ENVS.clone())
.env("SystemRoot", SYSTEM_ROOT.as_str())
.env(
"TMP",
std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")),
)
.args(&command_args[1..])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
start_child_process(pip_cmd, PYTHON_PATH.as_str()).await?
start_child_process(cmd, installer_path).await?
}
};
@@ -1240,7 +1314,7 @@ pub async fn handle_python_reqs(
false,
worker_name,
&w_id,
&format!("pip install {req}"),
&format!("uv pip install {req}"),
None,
false,
occupancy_metrics,
@@ -1260,7 +1334,7 @@ pub async fn handle_python_reqs(
tracing::warn!("S3 cache not available in the pro plan");
} else {
let venv_p = venv_p.clone();
tokio::spawn(build_tar_and_push(os, venv_p));
tokio::spawn(build_tar_and_push(os, venv_p, no_uv_install));
}
}
req_paths.push(venv_p);
backend/windmill-worker/src/worker.rs
+16
View File
@@ -236,7 +236,14 @@ pub const TMP_LOGS_DIR: &str = concatcp!(TMP_DIR, "/logs");
pub const ROOT_CACHE_NOMOUNT_DIR: &str = concatcp!(TMP_DIR, "/cache_nomount/");
pub const LOCK_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "lock");
// Used as fallback now
pub const PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "pip");
// pub const PY310_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_310");
pub const PY311_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_311");
// pub const PY312_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_312");
// pub const PY313_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "python_313");
pub const UV_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "uv");
pub const TAR_PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/pip");
pub const DENO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "deno");
@@ -257,6 +264,7 @@ const NUM_SECS_PING: u64 = 5;
const NUM_SECS_READINGS: u64 = 60;
const INCLUDE_DEPS_PY_SH_CONTENT: &str = include_str!("../nsjail/download_deps.py.sh");
const INCLUDE_DEPS_PY_SH_CONTENT_FALLBACK: &str = include_str!("../nsjail/download_deps.py.pip.sh");
pub const DEFAULT_CLOUD_TIMEOUT: u64 = 900;
pub const DEFAULT_SELFHOSTED_TIMEOUT: u64 = 604800; // 7 days
@@ -311,6 +319,7 @@ lazy_static::lazy_static! {
.and_then(|x| x.parse::<bool>().ok())
.unwrap_or(false);
// pub static ref DISABLE_NSJAIL: bool = false;
pub static ref DISABLE_NSJAIL: bool = std::env::var("DISABLE_NSJAIL")
.ok()
.and_then(|x| x.parse::<bool>().ok())
@@ -740,6 +749,13 @@ pub async fn run_worker<R: rsmq_async::RsmqConnection + Send + Sync + Clone + 's
"download_deps.py.sh",
INCLUDE_DEPS_PY_SH_CONTENT,
);
// TODO: Remove (Deprecated)
let _ = write_file(
&worker_dir,
"download_deps.py.pip.sh",
INCLUDE_DEPS_PY_SH_CONTENT_FALLBACK,
);
}
let mut last_ping = Instant::now() - Duration::from_secs(NUM_SECS_PING + 1);
backend/windmill-worker/src/worker_lockfiles.rs
+2
View File
@@ -1312,6 +1312,8 @@ async fn python_dep(
job_dir,
worker_dir,
occupancy_metrics,
false,
false,
)
.await;