From 35868ef9bf1eac650cbb735807aebc5a604dd5d6 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 2 Nov 2022 14:04:44 +0100 Subject: [PATCH] fix(backend): capture up all lockfile issues --- backend/parsers/windmill-parser-py/src/lib.rs | 3 +- backend/windmill-worker/src/worker.rs | 81 +++++++++++-------- .../src/lib/components/ScriptEditor.svelte | 2 +- .../components/common/button/Button.svelte | 40 +++++---- frontend/src/routes/__layout-root@none.svelte | 3 +- .../src/routes/user/workspaces@user.svelte | 1 + 6 files changed, 70 insertions(+), 60 deletions(-) diff --git a/backend/parsers/windmill-parser-py/src/lib.rs b/backend/parsers/windmill-parser-py/src/lib.rs index b268bd2529..37b4e31608 100644 --- a/backend/parsers/windmill-parser-py/src/lib.rs +++ b/backend/parsers/windmill-parser-py/src/lib.rs @@ -176,7 +176,8 @@ fn constant_to_value(c: &Constant) -> serde_json::Value { } static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_map! { - "psycopg2" => "psycopg2-binary" + "psycopg2" => "psycopg2-binary", + "git" => "GitPython" }; fn replace_import(x: String) -> String { diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 8ffe1a3bb5..0ce21f3c01 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -1268,7 +1268,7 @@ async fn handle_python_job( "".to_string() } else { pip_compile(job, &requirements, logs, job_dir, envs, db, timeout) - .await? + .await .map_err(|e| { Error::ExecutionErr(format!("pip compile failed: {}", e.to_string())) })? @@ -1551,37 +1551,7 @@ async fn handle_dependency_job( timeout: i32, envs: &Envs, ) -> error::Result { - let content: Result = match job.language { - Some(ScriptLang::Python3) => { - create_dependencies_dir(job_dir).await; - let requirements = &job - .raw_code - .as_ref() - .ok_or_else(|| Error::ExecutionErr("missing requirements".to_string()))? - .clone(); - pip_compile(job, requirements, logs, job_dir, envs, db, timeout).await? - } - Some(ScriptLang::Go) => { - let requirements = job - .raw_code - .as_ref() - .ok_or_else(|| Error::ExecutionErr("missing requirements".to_string()))?; - install_go_dependencies( - &job.id, - &requirements, - logs, - job_dir, - db, - timeout, - &envs.go_path, - false, - ) - .await - .map_err(|e| e.to_string()) - } - _ => Err("Language incompatible with dep job".to_string()), - }; - + let content = capture_dependency_job(job, logs, job_dir, db, timeout, envs).await; match content { Ok(content) => { sqlx::query!( @@ -1608,6 +1578,47 @@ async fn handle_dependency_job( } } +async fn capture_dependency_job( + job: &QueuedJob, + logs: &mut String, + job_dir: &str, + db: &sqlx::Pool, + timeout: i32, + envs: &Envs, +) -> error::Result { + match job.language { + Some(ScriptLang::Python3) => { + create_dependencies_dir(job_dir).await; + let requirements = &job + .raw_code + .as_ref() + .ok_or_else(|| Error::ExecutionErr("missing requirements".to_string()))? + .clone(); + pip_compile(job, requirements, logs, job_dir, envs, db, timeout).await + } + Some(ScriptLang::Go) => { + let requirements = job + .raw_code + .as_ref() + .ok_or_else(|| Error::ExecutionErr("missing requirements".to_string()))?; + install_go_dependencies( + &job.id, + &requirements, + logs, + job_dir, + db, + timeout, + &envs.go_path, + false, + ) + .await + } + _ => Err(error::Error::ExecutionErr( + "Language incompatible with dep job".to_string(), + )), + } +} + async fn pip_compile( job: &QueuedJob, requirements: &str, @@ -1616,7 +1627,7 @@ async fn pip_compile( Envs { pip_extra_index_url, pip_index_url, pip_trusted_host, .. }: &Envs, db: &Pool, timeout: i32, -) -> Result, Error> { +) -> error::Result { logs.push_str(&format!("content of requirements:\n{}\n", requirements)); let file = "requirements.in"; write_file(job_dir, file, &requirements).await?; @@ -1643,12 +1654,12 @@ async fn pip_compile( let mut file = File::open(path_lock).await?; let mut req_content = "".to_string(); file.read_to_string(&mut req_content).await?; - Ok(Ok(req_content + Ok(req_content .lines() .filter(|x| !x.trim_start().starts_with('#')) .map(|x| x.to_string()) .collect::>() - .join("\n"))) + .join("\n")) } async fn install_go_dependencies( diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index e5592806ee..3a4bdbbd35 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -162,7 +162,7 @@ -
+
{#if testIsLoading}