initial python imports relative dependency resolver

backend/Cargo.lock

@@ -7178,6 +7178,7 @@ dependencies = [
  "regex",
  "rustpython-parser",
  "serde_json",
+ "sqlx",
  "windmill-common",
  "windmill-parser",
 ]

backend/parsers/windmill-parser-py-imports/Cargo.toml

@@ -17,4 +17,5 @@ itertools.workspace = true
 regex.workspace = true
 serde_json.workspace = true
 anyhow.workspace = true
-lazy_static.workspace = true
+lazy_static.workspace = true
+sqlx.workspace = true

backend/parsers/windmill-parser-py-imports/src/lib.rs

@@ -11,6 +11,7 @@ use lazy_static::lazy_static;
 use phf::phf_map;
 use regex::Regex;
 
+use sqlx::{Pool, Postgres};
 use windmill_common::error;
 
 use rustpython_parser::ast::{Located, StmtKind};
@@ -46,7 +47,12 @@ lazy_static! {
     static ref RE: Regex = Regex::new(r"^\#\s?(\S+)$").unwrap();
 }
 
-pub fn parse_python_imports(code: &str) -> error::Result<Vec<String>> {
+pub async fn parse_python_imports(
+    code: &str,
+    w_id: &str,
+    path: &str,
+    db: &Pool<Postgres>,
+) -> error::Result<Vec<String>> {
     let find_requirements = code
         .lines()
         .find_position(|x| x.starts_with("#requirements:") || x.starts_with("# requirements:"));
@@ -119,58 +125,6 @@ pub fn parse_python_imports(code: &str) -> error::Result<Vec<String>> {
     }
 }
 
-#[cfg(test)]
-mod tests {
-
-    use super::*;
-
-    #[test]
-    fn test_parse_python_imports() -> anyhow::Result<()> {
-        //let code = "print(2 + 3, fd=sys.stderr)";
-        let code = "
-
-import os
-import wmill
-from zanzibar.estonie import talin
-import matplotlib.pyplot as plt
-from . import tests
-
-def main():
-    pass
-
-";
-        let r = parse_python_imports(code)?;
-        // println!("{}", serde_json::to_string(&r)?);
-        assert_eq!(r, vec!["matplotlib", "requests", "wmill", "zanzibar"]);
-        Ok(())
-    }
-
-    #[test]
-    fn test_parse_python_imports2() -> anyhow::Result<()> {
-        //let code = "print(2 + 3, fd=sys.stderr)";
-        let code = "
-#requirements:
-#burkina=0.4
-#nigeria
-#
-#congo
-
-import os
-import wmill
-from zanzibar.estonie import talin
-
-def main():
-    pass
-
-";
-        let r = parse_python_imports(code)?;
-        println!("{}", serde_json::to_string(&r)?);
-        assert_eq!(r, vec!["burkina=0.4", "nigeria"]);
-
-        Ok(())
-    }
-}
-
 const STDIMPORTS: [&str; 301] = [
     "__future__",
     "_abc",

backend/parsers/windmill-parser-py-imports/tests/tests.rs

@@ -0,0 +1,51 @@
+mod tests {
+
+    use sqlx::{Pool, Postgres};
+    use windmill_parser_py_imports::parse_python_imports;
+
+    #[sqlx::test(fixtures("base"))]
+    async fn test_parse_python_imports(db: Pool<Postgres>) -> anyhow::Result<()> {
+        //let code = "print(2 + 3, fd=sys.stderr)";
+        let code = "
+
+import os
+import wmill
+from zanzibar.estonie import talin
+import matplotlib.pyplot as plt
+from . import tests
+
+def main():
+    pass
+
+";
+        let r = parse_python_imports(code, "test-workspace", "f/foo/bar", &db).await?;
+        // println!("{}", serde_json::to_string(&r)?);
+        assert_eq!(r, vec!["matplotlib", "requests", "wmill", "zanzibar"]);
+        Ok(())
+    }
+
+    #[sqlx::test(fixtures("base"))]
+    async fn test_parse_python_imports2(db: Pool<Postgres>) -> anyhow::Result<()> {
+        //let code = "print(2 + 3, fd=sys.stderr)";
+        let code = "
+#requirements:
+#burkina=0.4
+#nigeria
+#
+#congo
+
+import os
+import wmill
+from zanzibar.estonie import talin
+
+def main():
+    pass
+
+";
+        let r = parse_python_imports(code, "test-workspace", "f/foo/bar", &db).await?;
+        println!("{}", serde_json::to_string(&r)?);
+        assert_eq!(r, vec!["burkina=0.4", "nigeria"]);
+
+        Ok(())
+    }
+}

backend/windmill-api/src/scripts.rs

@@ -488,7 +488,9 @@ async fn create_script(
     if needs_lock_gen {
         let dependencies = match ns.language {
             ScriptLang::Python3 => {
-                windmill_parser_py_imports::parse_python_imports(&ns.content)?.join("\n")
+                windmill_parser_py_imports::parse_python_imports(&ns.content, &w_id, &ns.path, &db)
+                    .await?
+                    .join("\n")
             }
             _ => ns.content,
         };

backend/windmill-worker/src/python_executor.rs

@@ -180,8 +180,14 @@ pub async fn handle_python_job(
     let requirements = match requirements_o {
         Some(r) => r,
         None => {
-            let requirements =
-                windmill_parser_py_imports::parse_python_imports(&inner_content)?.join("\n");
+            let requirements = windmill_parser_py_imports::parse_python_imports(
+                &inner_content,
+                &job.workspace_id,
+                &job.script_path(),
+                &db,
+            )
+            .await?
+            .join("\n");
             if requirements.is_empty() {
                 "".to_string()
             } else {

backend/windmill-worker/src/worker.rs

@@ -2140,7 +2140,7 @@ async fn handle_flow_dependency_job(
     worker_name: &str,
     worker_dir: &str,
 ) -> error::Result<()> {
-    let path = job.script_path.clone().ok_or_else(|| {
+    let job_path = job.script_path.clone().ok_or_else(|| {
         error::Error::InternalErr(
             "Cannot resolve flow dependencies for flow without path".to_string(),
         )
@@ -2159,7 +2159,7 @@ async fn handle_flow_dependency_job(
         };
         // sync with windmill-api/scripts
         let dependencies = match language {
-            ScriptLang::Python3 => windmill_parser_py_imports::parse_python_imports(&content)?.join("\n"),
+            ScriptLang::Python3 => windmill_parser_py_imports::parse_python_imports(&content, &job.workspace_id, &path.clone().unwrap_or_else(|| job_path.clone()), &db).await?.join("\n"),
             _ => content.clone(),
         };
         let new_lock = capture_dependency_job(
@@ -2232,7 +2232,7 @@ async fn handle_flow_dependency_job(
     sqlx::query!(
         "UPDATE flow SET value = $1 WHERE path = $2 AND workspace_id = $3",
         new_flow_value,
-        path,
+        job_path,
         job.workspace_id
     )
     .execute(db)