diff --git a/README.md b/README.md index a0a3b82d5d..e655b87831 100644 --- a/README.md +++ b/README.md @@ -309,6 +309,8 @@ upcoming CLI tool. | DENO_AUTH_TOKENS | None | Custom DENO_AUTH_TOKENS to pass to worker to allow the use of private modules | Worker | | DENO_FLAGS | None | Override the flags passed to deno (default --allow-all) to tighten permissions. Minimum permissions needed are "--allow-read=args.json --allow-write=result.json" | Worker | | | +| PIP_LOCAL_DEPENDENCIES | None | Specify dependencies that are installed locally and do not need to be solved nor installed again | Worker | +| | diff --git a/backend/parsers/windmill-parser-py/src/lib.rs b/backend/parsers/windmill-parser-py/src/lib.rs index bfbbe6d28f..fe0c5395d2 100644 --- a/backend/parsers/windmill-parser-py/src/lib.rs +++ b/backend/parsers/windmill-parser-py/src/lib.rs @@ -182,6 +182,7 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma "f" => "requests", "shopify" => "ShopifyAPI", "seleniumwire" => "selenium-wire", + "openbb-terminal" => "openbb[all]", }; fn replace_import(x: String) -> String { diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 5272ba6afd..c0f32c3ee8 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -427,6 +427,9 @@ pub async fn run_worker( let deno_flags = std::env::var("DENO_FLAGS") .ok() .map(|x| x.split(' ').map(|x| x.to_string()).collect()); + let pip_local_dependencies = std::env::var("PIP_LOCAL_DEPENDENCIES") + .ok() + .map(|x| x.split(',').map(|x| x.to_string()).collect()); #[cfg(feature = "enterprise")] let tar_cache_rate = std::env::var("TAR_CACHE_RATE") @@ -447,6 +450,7 @@ pub async fn run_worker( max_log_size, deno_flags, deno_auth_tokens, + pip_local_dependencies, }; WORKER_STARTED.inc(); @@ -757,6 +761,7 @@ struct Envs { deno_auth_tokens: String, deno_flags: Option>, max_log_size: i64, + pip_local_dependencies: Option>, } fn extract_error_value(log_lines: &str) -> serde_json::Value { @@ -2101,7 +2106,14 @@ async fn pip_compile( requirements: &str, logs: &mut String, job_dir: &str, - Envs { pip_extra_index_url, pip_index_url, pip_trusted_host, max_log_size, .. }: &Envs, + Envs { + pip_extra_index_url, + pip_index_url, + pip_trusted_host, + max_log_size, + pip_local_dependencies, + .. + }: &Envs, db: &Pool, timeout: i32, ) -> error::Result { @@ -2109,6 +2121,15 @@ async fn pip_compile( set_logs(logs, job_id, db).await; logs.push_str(&format!("\ncontent of requirements:\n{}", requirements)); let file = "requirements.in"; + let requirements = if let Some(pip_local_dependencies) = pip_local_dependencies { + let deps = pip_local_dependencies.clone(); + requirements + .lines() + .filter(|s| !deps.contains(&s.to_string())) + .join("\n") + } else { + requirements.to_string() + }; write_file(job_dir, file, &requirements).await?; let mut args = vec!["-q", "--no-header", file, "--resolver=backtracking"];