feat: resolve dependencies across relative imports for python

backend/Cargo.lock

@@ -7172,6 +7172,7 @@ name = "windmill-parser-py-imports"
 version = "1.136.0"
 dependencies = [
  "anyhow",
+ "async-recursion",
  "itertools 0.11.0",
  "lazy_static",
  "phf 0.11.2",

backend/parsers/windmill-parser-py-imports/Cargo.toml

@@ -18,4 +18,5 @@ regex.workspace = true
 serde_json.workspace = true
 anyhow.workspace = true
 lazy_static.workspace = true
-sqlx.workspace = true
+sqlx.workspace = true
+async-recursion.workspace = true

backend/parsers/windmill-parser-py-imports/src/lib.rs

@@ -6,6 +6,7 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
+use async_recursion::async_recursion;
 use itertools::Itertools;
 use lazy_static::lazy_static;
 use phf::phf_map;
@@ -24,9 +25,6 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma
     "psycopg" => "psycopg[binary, pool]",
     "yaml" => "pyyaml",
     "git" => "GitPython",
-    "u" => "requests",
-    "f" => "requests",
-    "." => "requests",
     "shopify" => "ShopifyAPI",
     "seleniumwire" => "selenium-wire",
     "openbb-terminal" => "openbb[all]",
@@ -47,11 +45,41 @@ lazy_static! {
     static ref RE: Regex = Regex::new(r"^\#\s?(\S+)$").unwrap();
 }
 
+fn process_import(module: Option<String>, path: &str, level: usize) -> Vec<String> {
+    if level > 0 {
+        let mut imports = vec!["requests".to_string()];
+        let splitted_path = path.split("/");
+        let base = splitted_path
+            .clone()
+            .take(splitted_path.count() - level)
+            .join("/");
+        if let Some(m) = module {
+            imports.push(format!("relative:{base}/{}", m.replace(".", "/")));
+        } else {
+            imports.push(format!("relative:{base}"));
+        }
+        imports
+    } else if let Some(module) = module {
+        let imprt = module.split('.').next().unwrap_or("").replace("_", "-");
+        if imprt == "u" || imprt == "f" {
+            vec![
+                "requests".to_string(),
+                format!("relative:{}", module.replace(".", "/")),
+            ]
+        } else {
+            vec![imprt]
+        }
+    } else {
+        vec![]
+    }
+}
+
+#[async_recursion]
 pub async fn parse_python_imports(
     code: &str,
-    _w_id: &str,
-    _path: &str,
-    _db: &Pool<Postgres>,
+    w_id: &str,
+    path: &str,
+    db: &Pool<Postgres>,
 ) -> error::Result<Vec<String>> {
     let find_requirements = code
         .lines()
@@ -96,21 +124,16 @@ pub async fn parse_python_imports(
                             .into_iter()
                             .map(|x| {
                                 let name = x.node.name;
-                                if name.starts_with('.') {
-                                    ".".to_string()
-                                } else {
-                                    name.split('.').next().unwrap_or("").to_string()
-                                }
+                                process_import(Some(name), path, 0)
                             })
-                            .map(replace_import)
+                            .flatten()
                             .collect::<Vec<String>>(),
                     ),
-                    StmtKind::ImportFrom { level: Some(i), .. } if i > 0 => {
-                        Some(vec!["requests".to_string()])
+                    StmtKind::ImportFrom { level: Some(i), module, .. } if i > 0 => {
+                        Some(process_import(module, path, i))
                     }
-                    StmtKind::ImportFrom { level: _, module: Some(mod_), names: _ } => {
-                        let imprt = mod_.split('.').next().unwrap_or("").replace("_", "-");
-                        Some(vec![replace_import(imprt)])
+                    StmtKind::ImportFrom { level: _, module, names: _ } => {
+                        Some(process_import(module, path, 0))
                     }
                     _ => None,
                 },
@@ -119,7 +142,28 @@ pub async fn parse_python_imports(
             .filter(|x| !STDIMPORTS.contains(&x.as_str()))
             .unique()
             .collect();
-        imports.extend(nimports);
+        for n in nimports.iter() {
+            let nested = if n.starts_with("relative:") {
+                let code = sqlx::query_scalar!(
+                    r#"
+                    SELECT content FROM script WHERE workspace_id = $1 AND path = $2
+                    "#,
+                    w_id,
+                    n.replace("relative:", "")
+                )
+                .fetch_optional(db)
+                .await?
+                .unwrap_or_else(|| "".to_string());
+                parse_python_imports(&code, w_id, path, db).await?
+            } else {
+                vec![replace_import(n.to_string())]
+            };
+            for imp in nested {
+                if !imports.contains(&imp) {
+                    imports.push(imp);
+                }
+            }
+        }
         imports.sort();
         Ok(imports)
     }

backend/parsers/windmill-parser-py-imports/tests/fixtures/base.sql

@@ -2535,3 +2535,25 @@ INSERT INTO public.workspace_key(workspace_id, kind, key) VALUES
 
 insert INTO public.token(token, email, label, super_admin) VALUES ('SECRET_TOKEN', 'test@windmill.dev', 'test token', true);
 
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'system',
+'
+import pandas as pd
+import pandas2
+',
+'{}',
+'',
+'',
+'f/foo/bar', -28028598712388162, 'python3', '');
+
+INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES (
+'test-workspace',
+'system',
+'
+import numpy as np
+',
+'{}',
+'',
+'',
+'f/foo/baz', -28028598712388161, 'python3', '');

backend/parsers/windmill-parser-py-imports/tests/tests.rs

@@ -48,4 +48,25 @@ def main():
 
         Ok(())
     }
+
+    #[sqlx::test(fixtures("base"))]
+    async fn test_parse_python_imports_local(db: Pool<Postgres>) -> anyhow::Result<()> {
+        //let code = "print(2 + 3, fd=sys.stderr)";
+        let code = "
+from f.foo.bar import main1
+from .baz import main2
+from ..foo.bar import main3
+import f.foo.bar as bar
+
+
+def main():
+    pass
+
+";
+        let r = parse_python_imports(code, "test-workspace", "f/foo/bar", &db).await?;
+        println!("{}", serde_json::to_string(&r)?);
+        assert_eq!(r, ["numpy", "pandas", "pandas2", "requests"]);
+
+        Ok(())
+    }
 }

frontend/src/lib/components/VariableEditor.svelte

@@ -179,12 +179,12 @@
 					<span class="text-sm text-tertiary mr-4">
 						({variable.value.length}/{MAX_VARIABLE_LENGTH} characters)
 					</span>
-					{#if edit && variable.is_secret}<Button
-							variant="border"
-							size="xs"
-							on:click={() => loadVariable(initialPath)}
-							>Load secret value<Tooltip>Will generate an audit log</Tooltip></Button
-						>{/if}
+					{#if edit && variable.is_secret}{#if $userStore?.operator}
+							<div class="p-2 border">Operators cannot load secret value</div>
+						{:else}
+							<Button variant="border" size="xs" on:click={() => loadVariable(initialPath)}
+								>Load secret value<Tooltip>Will generate an audit log</Tooltip></Button
+							>{/if}{/if}
 				</div>
 				<div class="flex flex-col gap-2">
 					<ToggleButtonGroup bind:selected={editorKind}>