diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 50b1922c7a..f679588f2f 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -7172,6 +7172,7 @@ name = "windmill-parser-py-imports" version = "1.136.0" dependencies = [ "anyhow", + "async-recursion", "itertools 0.11.0", "lazy_static", "phf 0.11.2", diff --git a/backend/parsers/windmill-parser-py-imports/Cargo.toml b/backend/parsers/windmill-parser-py-imports/Cargo.toml index 9a906c09cb..665b6df211 100644 --- a/backend/parsers/windmill-parser-py-imports/Cargo.toml +++ b/backend/parsers/windmill-parser-py-imports/Cargo.toml @@ -18,4 +18,5 @@ regex.workspace = true serde_json.workspace = true anyhow.workspace = true lazy_static.workspace = true -sqlx.workspace = true \ No newline at end of file +sqlx.workspace = true +async-recursion.workspace = true diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs index 100825dfb9..fd687ae1a8 100644 --- a/backend/parsers/windmill-parser-py-imports/src/lib.rs +++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs @@ -6,6 +6,7 @@ * LICENSE-AGPL for a copy of the license. */ +use async_recursion::async_recursion; use itertools::Itertools; use lazy_static::lazy_static; use phf::phf_map; @@ -24,9 +25,6 @@ static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_ma "psycopg" => "psycopg[binary, pool]", "yaml" => "pyyaml", "git" => "GitPython", - "u" => "requests", - "f" => "requests", - "." => "requests", "shopify" => "ShopifyAPI", "seleniumwire" => "selenium-wire", "openbb-terminal" => "openbb[all]", @@ -47,11 +45,41 @@ lazy_static! { static ref RE: Regex = Regex::new(r"^\#\s?(\S+)$").unwrap(); } +fn process_import(module: Option, path: &str, level: usize) -> Vec { + if level > 0 { + let mut imports = vec!["requests".to_string()]; + let splitted_path = path.split("/"); + let base = splitted_path + .clone() + .take(splitted_path.count() - level) + .join("/"); + if let Some(m) = module { + imports.push(format!("relative:{base}/{}", m.replace(".", "/"))); + } else { + imports.push(format!("relative:{base}")); + } + imports + } else if let Some(module) = module { + let imprt = module.split('.').next().unwrap_or("").replace("_", "-"); + if imprt == "u" || imprt == "f" { + vec![ + "requests".to_string(), + format!("relative:{}", module.replace(".", "/")), + ] + } else { + vec![imprt] + } + } else { + vec![] + } +} + +#[async_recursion] pub async fn parse_python_imports( code: &str, - _w_id: &str, - _path: &str, - _db: &Pool, + w_id: &str, + path: &str, + db: &Pool, ) -> error::Result> { let find_requirements = code .lines() @@ -96,21 +124,16 @@ pub async fn parse_python_imports( .into_iter() .map(|x| { let name = x.node.name; - if name.starts_with('.') { - ".".to_string() - } else { - name.split('.').next().unwrap_or("").to_string() - } + process_import(Some(name), path, 0) }) - .map(replace_import) + .flatten() .collect::>(), ), - StmtKind::ImportFrom { level: Some(i), .. } if i > 0 => { - Some(vec!["requests".to_string()]) + StmtKind::ImportFrom { level: Some(i), module, .. } if i > 0 => { + Some(process_import(module, path, i)) } - StmtKind::ImportFrom { level: _, module: Some(mod_), names: _ } => { - let imprt = mod_.split('.').next().unwrap_or("").replace("_", "-"); - Some(vec![replace_import(imprt)]) + StmtKind::ImportFrom { level: _, module, names: _ } => { + Some(process_import(module, path, 0)) } _ => None, }, @@ -119,7 +142,28 @@ pub async fn parse_python_imports( .filter(|x| !STDIMPORTS.contains(&x.as_str())) .unique() .collect(); - imports.extend(nimports); + for n in nimports.iter() { + let nested = if n.starts_with("relative:") { + let code = sqlx::query_scalar!( + r#" + SELECT content FROM script WHERE workspace_id = $1 AND path = $2 + "#, + w_id, + n.replace("relative:", "") + ) + .fetch_optional(db) + .await? + .unwrap_or_else(|| "".to_string()); + parse_python_imports(&code, w_id, path, db).await? + } else { + vec![replace_import(n.to_string())] + }; + for imp in nested { + if !imports.contains(&imp) { + imports.push(imp); + } + } + } imports.sort(); Ok(imports) } diff --git a/backend/parsers/windmill-parser-py-imports/tests/fixtures/base.sql b/backend/parsers/windmill-parser-py-imports/tests/fixtures/base.sql index 3e49f494ec..434ae3b9ee 100644 --- a/backend/parsers/windmill-parser-py-imports/tests/fixtures/base.sql +++ b/backend/parsers/windmill-parser-py-imports/tests/fixtures/base.sql @@ -2535,3 +2535,25 @@ INSERT INTO public.workspace_key(workspace_id, kind, key) VALUES insert INTO public.token(token, email, label, super_admin) VALUES ('SECRET_TOKEN', 'test@windmill.dev', 'test token', true); +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'system', +' +import pandas as pd +import pandas2 +', +'{}', +'', +'', +'f/foo/bar', -28028598712388162, 'python3', ''); + +INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( +'test-workspace', +'system', +' +import numpy as np +', +'{}', +'', +'', +'f/foo/baz', -28028598712388161, 'python3', ''); \ No newline at end of file diff --git a/backend/parsers/windmill-parser-py-imports/tests/tests.rs b/backend/parsers/windmill-parser-py-imports/tests/tests.rs index 5beeb304f6..d3e4d2bd99 100644 --- a/backend/parsers/windmill-parser-py-imports/tests/tests.rs +++ b/backend/parsers/windmill-parser-py-imports/tests/tests.rs @@ -48,4 +48,25 @@ def main(): Ok(()) } + + #[sqlx::test(fixtures("base"))] + async fn test_parse_python_imports_local(db: Pool) -> anyhow::Result<()> { + //let code = "print(2 + 3, fd=sys.stderr)"; + let code = " +from f.foo.bar import main1 +from .baz import main2 +from ..foo.bar import main3 +import f.foo.bar as bar + + +def main(): + pass + +"; + let r = parse_python_imports(code, "test-workspace", "f/foo/bar", &db).await?; + println!("{}", serde_json::to_string(&r)?); + assert_eq!(r, ["numpy", "pandas", "pandas2", "requests"]); + + Ok(()) + } } diff --git a/frontend/src/lib/components/VariableEditor.svelte b/frontend/src/lib/components/VariableEditor.svelte index 3d2e48e411..9d52e4ef33 100644 --- a/frontend/src/lib/components/VariableEditor.svelte +++ b/frontend/src/lib/components/VariableEditor.svelte @@ -179,12 +179,12 @@ ({variable.value.length}/{MAX_VARIABLE_LENGTH} characters) - {#if edit && variable.is_secret}{/if} + {#if edit && variable.is_secret}{#if $userStore?.operator} +
Operators cannot load secret value
+ {:else} + {/if}{/if}