diff --git a/Dockerfile b/Dockerfile index 5d78f6cb67..bdac008053 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,7 +54,7 @@ COPY /openflow.openapi.yaml /openflow.openapi.yaml COPY /backend/windmill-api/build_openapi.sh /backend/windmill-api/build_openapi.sh RUN cd /backend/windmill-api && . ./build_openapi.sh -COPY /backend/parsers/windmill-parser-py-wasm/pkg/ /backend/windmill-parser-py-wasm/pkg/ +COPY /backend/parsers/windmill-parser-wasm/pkg/ /backend/parsers/windmill-parser-wasm/pkg/ RUN npm run generate-backend-client ENV NODE_OPTIONS "--max-old-space-size=8192" diff --git a/backend/.vscode/settings.json b/backend/.vscode/settings.json index 5076ac3cf3..f85934fbfd 100644 --- a/backend/.vscode/settings.json +++ b/backend/.vscode/settings.json @@ -5,5 +5,6 @@ "./parsers/windmill-parser-ts-wasm/Cargo.toml", "./parsers/windmill-parser-ts-wasm/Cargo.toml", "./parsers/windmill-parser-ts-wasm/Cargo.toml" - ] + ], + "rust-analyzer.showUnlinkedFileNotification": false } diff --git a/backend/Cargo.lock b/backend/Cargo.lock index a2d4f2b925..b035ec588f 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -4766,12 +4766,6 @@ version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" -[[package]] -name = "unicode-general-category" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2281c8c1d221438e373249e065ca4989c4c36952c211ff21a0ee91c44a3869e7" - [[package]] name = "unicode-id" version = "0.3.3" @@ -5233,10 +5227,7 @@ dependencies = [ "windmill-audit", "windmill-common", "windmill-parser", - "windmill-parser-bash", - "windmill-parser-go", - "windmill-parser-py", - "windmill-parser-ts", + "windmill-parser-py-imports", "windmill-queue", ] @@ -5308,13 +5299,9 @@ name = "windmill-parser-bash" version = "1.109.1" dependencies = [ "anyhow", - "itertools", "lazy_static", - "phf 0.11.1", "regex", "serde_json", - "unicode-general-category", - "windmill-common", "windmill-parser", ] @@ -5325,15 +5312,23 @@ dependencies = [ "anyhow", "gosyn", "itertools", - "phf 0.11.1", - "unicode-general-category", - "windmill-common", "windmill-parser", ] [[package]] name = "windmill-parser-py" version = "1.109.1" +dependencies = [ + "anyhow", + "itertools", + "rustpython-parser", + "serde_json", + "windmill-parser", +] + +[[package]] +name = "windmill-parser-py-imports" +version = "1.109.1" dependencies = [ "anyhow", "itertools", @@ -5361,13 +5356,17 @@ dependencies = [ ] [[package]] -name = "windmill-parser-ts-wasm" +name = "windmill-parser-wasm" version = "1.109.1" dependencies = [ + "anyhow", "serde_json", "wasm-bindgen", "wasm-bindgen-test", "windmill-parser", + "windmill-parser-bash", + "windmill-parser-go", + "windmill-parser-py", "windmill-parser-ts", ] @@ -5432,6 +5431,7 @@ dependencies = [ "windmill-parser-bash", "windmill-parser-go", "windmill-parser-py", + "windmill-parser-py-imports", "windmill-parser-ts", "windmill-queue", ] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 06e688765c..182aea5844 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -14,9 +14,11 @@ members = [ "./windmill-api-client", "./parsers/windmill-parser", "./parsers/windmill-parser-ts", - "./parsers/windmill-parser-ts-wasm", + "./parsers/windmill-parser-wasm", "./parsers/windmill-parser-go", + "./parsers/windmill-parser-bash", "./parsers/windmill-parser-py", + "./parsers/windmill-parser-py-imports", ] [workspace.package] @@ -74,6 +76,7 @@ windmill-audit = { path = "./windmill-audit" } windmill-parser = { path = "./parsers/windmill-parser" } windmill-parser-ts = { path = "./parsers/windmill-parser-ts" } windmill-parser-py = { path = "./parsers/windmill-parser-py" } +windmill-parser-py-imports = { path = "./parsers/windmill-parser-py-imports" } windmill-parser-go = { path = "./parsers/windmill-parser-go" } windmill-parser-bash = { path = "./parsers/windmill-parser-bash" } axum = { version = "^0", features = ["headers"] } @@ -132,7 +135,6 @@ swc_common = "0.29.39" swc_ecma_parser = "0.128.2" swc_ecma_ast = "0.98.1" base64 = "0.21.0" -unicode-general-category = "^0" hmac = "0.12.1" sha2 = "0.10.6" sqlx = { version = "^0", features = [ diff --git a/backend/parsers/windmill-parser-bash/Cargo.toml b/backend/parsers/windmill-parser-bash/Cargo.toml index 22579e82c3..9d2070f6c8 100644 --- a/backend/parsers/windmill-parser-bash/Cargo.toml +++ b/backend/parsers/windmill-parser-bash/Cargo.toml @@ -10,10 +10,6 @@ path = "./src/lib.rs" [dependencies] windmill-parser.workspace = true -windmill-common.workspace = true -phf.workspace = true -unicode-general-category.workspace = true -itertools.workspace = true anyhow.workspace = true regex.workspace = true lazy_static.workspace = true diff --git a/backend/parsers/windmill-parser-bash/src/lib.rs b/backend/parsers/windmill-parser-bash/src/lib.rs index a43e079cb7..6e9b5a4eed 100644 --- a/backend/parsers/windmill-parser-bash/src/lib.rs +++ b/backend/parsers/windmill-parser-bash/src/lib.rs @@ -7,15 +7,13 @@ use serde_json::json; use std::collections::HashMap; use windmill_parser::{Arg, MainArgSignature, Typ}; -pub fn parse_bash_sig(code: &str) -> windmill_common::error::Result { +pub fn parse_bash_sig(code: &str) -> anyhow::Result { let parsed = parse_file(&code)?; if let Some(x) = parsed { let args = x; Ok(MainArgSignature { star_args: false, star_kwargs: false, args }) } else { - Err(windmill_common::error::Error::BadRequest( - "Error parsing bash script".to_string(), - )) + Err(anyhow!("Error parsing bash script".to_string())) } } diff --git a/backend/parsers/windmill-parser-go/Cargo.toml b/backend/parsers/windmill-parser-go/Cargo.toml index 6bc0be873b..83f95edd1a 100644 --- a/backend/parsers/windmill-parser-go/Cargo.toml +++ b/backend/parsers/windmill-parser-go/Cargo.toml @@ -10,9 +10,6 @@ path = "./src/lib.rs" [dependencies] windmill-parser.workspace = true -windmill-common.workspace = true -phf.workspace = true -unicode-general-category.workspace = true itertools.workspace = true anyhow.workspace = true gosyn.workspace = true \ No newline at end of file diff --git a/backend/parsers/windmill-parser-go/src/lib.rs b/backend/parsers/windmill-parser-go/src/lib.rs index 689d558234..c7e06a102a 100644 --- a/backend/parsers/windmill-parser-go/src/lib.rs +++ b/backend/parsers/windmill-parser-go/src/lib.rs @@ -8,7 +8,7 @@ use itertools::Itertools; use windmill_parser::{Arg, MainArgSignature, ObjectProperty, Typ}; -pub fn parse_go_sig(code: &str) -> windmill_common::error::Result { +pub fn parse_go_sig(code: &str) -> anyhow::Result { let filtered_code = filter_non_main(code); let file = parse_source(&filtered_code).map_err(|x| anyhow::anyhow!(x.to_string()))?; if let Some(func) = file.decl.iter().find_map(|x| match x { @@ -27,13 +27,11 @@ pub fn parse_go_sig(code: &str) -> windmill_common::error::Result windmill_common::error::Result> { +pub fn parse_go_imports(code: &str) -> anyhow::Result> { let file = parse_source(filter_non_imports(code)).map_err(|x| anyhow::anyhow!(x.to_string()))?; let mut imports: Vec = file diff --git a/backend/parsers/windmill-parser-py-imports/Cargo.toml b/backend/parsers/windmill-parser-py-imports/Cargo.toml new file mode 100644 index 0000000000..f972d950b0 --- /dev/null +++ b/backend/parsers/windmill-parser-py-imports/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "windmill-parser-py-imports" +version.workspace = true +edition.workspace = true +authors.workspace = true + +[lib] +name = "windmill_parser_py_imports" +path = "./src/lib.rs" + +[dependencies] +windmill-parser.workspace = true +windmill-common.workspace = true +rustpython-parser.workspace = true +phf.workspace = true +itertools.workspace = true +regex.workspace = true +serde_json.workspace = true +anyhow.workspace = true +lazy_static.workspace = true \ No newline at end of file diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs new file mode 100644 index 0000000000..41692f29fe --- /dev/null +++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs @@ -0,0 +1,458 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use itertools::Itertools; +use lazy_static::lazy_static; +use phf::phf_map; +use regex::Regex; + +use windmill_common::error; + +use rustpython_parser::ast::{Located, StmtKind}; +use rustpython_parser::parser::parse_program; + +const DEF_MAIN: &str = "def main("; + +static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_map! { + "psycopg2" => "psycopg2-binary", + "psycopg" => "psycopg[binary, pool]", + "yaml" => "pyyaml", + "git" => "GitPython", + "u" => "requests", + "f" => "requests", + "." => "requests", + "shopify" => "ShopifyAPI", + "seleniumwire" => "selenium-wire", + "openbb-terminal" => "openbb[all]", + "riskfolio" => "riskfolio-lib", + "smb" => "pysmb", +}; + +fn replace_import(x: String) -> String { + PYTHON_IMPORTS_REPLACEMENT + .get(&x) + .map(|x| x.to_owned()) + .unwrap_or(&x) + .to_string() +} + +lazy_static! { + static ref RE: Regex = Regex::new(r"^\#\s?(\S+)$").unwrap(); +} + +pub fn parse_python_imports(code: &str) -> error::Result> { + let find_requirements = code + .lines() + .find_position(|x| x.starts_with("#requirements:") || x.starts_with("# requirements:")); + if let Some((pos, _)) = find_requirements { + let lines = code + .lines() + .skip(pos + 1) + .map_while(|x| { + RE.captures(x) + .map(|x| x.get(1).unwrap().as_str().to_string()) + }) + .collect(); + Ok(lines) + } else { + let code = code.split(DEF_MAIN).next().unwrap_or(""); + let ast = parse_program(code, "main.py").map_err(|e| { + error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string())) + })?; + let mut imports: Vec = ast + .into_iter() + .filter_map(|x| match x { + Located { node, .. } => match node { + StmtKind::Import { names } => Some( + names + .into_iter() + .map(|x| { + let name = x.node.name; + if name.starts_with('.') { + ".".to_string() + } else { + name.split('.').next().unwrap_or("").to_string() + } + }) + .map(replace_import) + .collect::>(), + ), + StmtKind::ImportFrom { level: Some(i), .. } if i > 0 => { + Some(vec!["requests".to_string()]) + } + StmtKind::ImportFrom { level: _, module: Some(mod_), names: _ } => { + let imprt = mod_.split('.').next().unwrap_or("").replace("_", "-"); + Some(vec![replace_import(imprt)]) + } + _ => None, + }, + }) + .flatten() + .filter(|x| !STDIMPORTS.contains(&x.as_str())) + .unique() + .collect(); + imports.sort(); + Ok(imports) + } +} + +#[cfg(test)] +mod tests { + + use super::*; + + #[test] + fn test_parse_python_imports() -> anyhow::Result<()> { + //let code = "print(2 + 3, fd=sys.stderr)"; + let code = " + +import os +import wmill +from zanzibar.estonie import talin +import matplotlib.pyplot as plt +from . import tests + +def main(): + pass + +"; + let r = parse_python_imports(code)?; + // println!("{}", serde_json::to_string(&r)?); + assert_eq!(r, vec!["matplotlib", "requests", "wmill", "zanzibar"]); + Ok(()) + } + + #[test] + fn test_parse_python_imports2() -> anyhow::Result<()> { + //let code = "print(2 + 3, fd=sys.stderr)"; + let code = " +#requirements: +#burkina=0.4 +#nigeria +# +#congo + +import os +import wmill +from zanzibar.estonie import talin + +def main(): + pass + +"; + let r = parse_python_imports(code)?; + println!("{}", serde_json::to_string(&r)?); + assert_eq!(r, vec!["burkina=0.4", "nigeria"]); + + Ok(()) + } +} + +const STDIMPORTS: [&str; 301] = [ + "__future__", + "_abc", + "_aix_support", + "_ast", + "_asyncio", + "_bisect", + "_blake2", + "_bootsubprocess", + "_bz2", + "_codecs", + "_codecs_cn", + "_codecs_hk", + "_codecs_iso2022", + "_codecs_jp", + "_codecs_kr", + "_codecs_tw", + "_collections", + "_collections_abc", + "_compat_pickle", + "_compression", + "_contextvars", + "_crypt", + "_csv", + "_ctypes", + "_curses", + "_curses_panel", + "_datetime", + "_dbm", + "_decimal", + "_elementtree", + "_frozen_importlib", + "_frozen_importlib_external", + "_functools", + "_gdbm", + "_hashlib", + "_heapq", + "_imp", + "_io", + "_json", + "_locale", + "_lsprof", + "_lzma", + "_markupbase", + "_md5", + "_msi", + "_multibytecodec", + "_multiprocessing", + "_opcode", + "_operator", + "_osx_support", + "_overlapped", + "_pickle", + "_posixshmem", + "_posixsubprocess", + "_py_abc", + "_pydecimal", + "_pyio", + "_queue", + "_random", + "_sha1", + "_sha256", + "_sha3", + "_sha512", + "_signal", + "_sitebuiltins", + "_socket", + "_sqlite3", + "_sre", + "_ssl", + "_stat", + "_statistics", + "_string", + "_strptime", + "_struct", + "_symtable", + "_thread", + "_threading_local", + "_tkinter", + "_tracemalloc", + "_uuid", + "_warnings", + "_weakref", + "_weakrefset", + "_winapi", + "_zoneinfo", + "abc", + "aifc", + "antigravity", + "argparse", + "array", + "ast", + "asynchat", + "asyncio", + "asyncore", + "atexit", + "audioop", + "base64", + "bdb", + "binascii", + "binhex", + "bisect", + "builtins", + "bz2", + "cProfile", + "calendar", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "compileall", + "concurrent", + "configparser", + "contextlib", + "contextvars", + "copy", + "copyreg", + "crypt", + "csv", + "ctypes", + "curses", + "dataclasses", + "datetime", + "dbm", + "decimal", + "difflib", + "dis", + "distutils", + "doctest", + "email", + "encodings", + "ensurepip", + "enum", + "errno", + "faulthandler", + "fcntl", + "filecmp", + "fileinput", + "fnmatch", + "fractions", + "ftplib", + "functools", + "gc", + "genericpath", + "getopt", + "getpass", + "gettext", + "glob", + "graphlib", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "html", + "http", + "idlelib", + "imaplib", + "imghdr", + "imp", + "importlib", + "inspect", + "io", + "ipaddress", + "itertools", + "json", + "keyword", + "lib2to3", + "linecache", + "locale", + "logging", + "lzma", + "mailbox", + "mailcap", + "marshal", + "math", + "mimetypes", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multiprocessing", + "netrc", + "nis", + "nntplib", + "nt", + "ntpath", + "nturl2path", + "numbers", + "opcode", + "operator", + "optparse", + "os", + "ossaudiodev", + "pathlib", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "poplib", + "posix", + "posixpath", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "pydoc_data", + "pyexpat", + "queue", + "quopri", + "random", + "re", + "readline", + "reprlib", + "resource", + "rlcompleter", + "runpy", + "sched", + "secrets", + "select", + "selectors", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "socketserver", + "spwd", + "sqlite3", + "sre_compile", + "sre_constants", + "sre_parse", + "ssl", + "stat", + "statistics", + "string", + "stringprep", + "struct", + "subprocess", + "sunau", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "textwrap", + "this", + "threading", + "time", + "timeit", + "tkinter", + "token", + "tokenize", + "trace", + "traceback", + "tracemalloc", + "tty", + "turtle", + "turtledemo", + "types", + "typing", + "unicodedata", + "unittest", + "urllib", + "uu", + "uuid", + "venv", + "warnings", + "wave", + "weakref", + "webbrowser", + "winreg", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpc", + "zipapp", + "zipfile", + "zipimport", + "", +]; diff --git a/backend/parsers/windmill-parser-py/Cargo.toml b/backend/parsers/windmill-parser-py/Cargo.toml index 6476e27282..73a797ef79 100644 --- a/backend/parsers/windmill-parser-py/Cargo.toml +++ b/backend/parsers/windmill-parser-py/Cargo.toml @@ -10,11 +10,7 @@ path = "./src/lib.rs" [dependencies] windmill-parser.workspace = true -windmill-common.workspace = true rustpython-parser.workspace = true -phf.workspace = true itertools.workspace = true -regex.workspace = true serde_json.workspace = true anyhow.workspace = true -lazy_static.workspace = true \ No newline at end of file diff --git a/backend/parsers/windmill-parser-py/src/lib.rs b/backend/parsers/windmill-parser-py/src/lib.rs index 4777476a1a..d4bd08dbdd 100644 --- a/backend/parsers/windmill-parser-py/src/lib.rs +++ b/backend/parsers/windmill-parser-py/src/lib.rs @@ -9,12 +9,8 @@ use std::collections::HashMap; use itertools::Itertools; -use lazy_static::lazy_static; -use phf::phf_map; -use regex::Regex; use serde_json::json; -use windmill_common::error; use windmill_parser::{json_to_typ, Arg, MainArgSignature, Typ}; use rustpython_parser::ast::{Constant, ExprKind, Located, StmtKind}; @@ -57,16 +53,13 @@ fn filter_non_main(code: &str) -> String { return filtered_code; } -pub fn parse_python_signature(code: &str) -> error::Result { +pub fn parse_python_signature(code: &str) -> anyhow::Result { let filtered_code = filter_non_main(code); if filtered_code.is_empty() { - return Err(error::Error::BadRequest( - "No main function found".to_string(), - )); + return Err(anyhow::anyhow!("No main function found".to_string(),)); } - let ast = parse_program(&filtered_code, "main.py").map_err(|e| { - error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string())) - })?; + let ast = parse_program(&filtered_code, "main.py") + .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?; let param = ast.into_iter().find_map(|x| match x { Located { node: StmtKind::FunctionDef { name, args, .. }, .. } if &name == "main" => { Some(*args) @@ -120,7 +113,7 @@ pub fn parse_python_signature(code: &str) -> error::Result { .collect(), }) } else { - Err(error::Error::ExecutionErr( + Err(anyhow::anyhow!( "main function was not findable".to_string(), )) } @@ -171,89 +164,6 @@ fn constant_to_value(c: &Constant) -> serde_json::Value { } } -static PYTHON_IMPORTS_REPLACEMENT: phf::Map<&'static str, &'static str> = phf_map! { - "psycopg2" => "psycopg2-binary", - "psycopg" => "psycopg[binary, pool]", - "yaml" => "pyyaml", - "git" => "GitPython", - "u" => "requests", - "f" => "requests", - "." => "requests", - "shopify" => "ShopifyAPI", - "seleniumwire" => "selenium-wire", - "openbb-terminal" => "openbb[all]", - "riskfolio" => "riskfolio-lib", - "smb" => "pysmb", -}; - -fn replace_import(x: String) -> String { - PYTHON_IMPORTS_REPLACEMENT - .get(&x) - .map(|x| x.to_owned()) - .unwrap_or(&x) - .to_string() -} - -lazy_static! { - static ref RE: Regex = Regex::new(r"^\#\s?(\S+)$").unwrap(); -} - -pub fn parse_python_imports(code: &str) -> error::Result> { - let find_requirements = code - .lines() - .find_position(|x| x.starts_with("#requirements:") || x.starts_with("# requirements:")); - if let Some((pos, _)) = find_requirements { - let lines = code - .lines() - .skip(pos + 1) - .map_while(|x| { - RE.captures(x) - .map(|x| x.get(1).unwrap().as_str().to_string()) - }) - .collect(); - Ok(lines) - } else { - let code = code.split(DEF_MAIN).next().unwrap_or(""); - let ast = parse_program(code, "main.py").map_err(|e| { - error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string())) - })?; - let mut imports: Vec = ast - .into_iter() - .filter_map(|x| match x { - Located { node, .. } => match node { - StmtKind::Import { names } => Some( - names - .into_iter() - .map(|x| { - let name = x.node.name; - if name.starts_with('.') { - ".".to_string() - } else { - name.split('.').next().unwrap_or("").to_string() - } - }) - .map(replace_import) - .collect::>(), - ), - StmtKind::ImportFrom { level: Some(i), .. } if i > 0 => { - Some(vec!["requests".to_string()]) - } - StmtKind::ImportFrom { level: _, module: Some(mod_), names: _ } => { - let imprt = mod_.split('.').next().unwrap_or("").replace("_", "-"); - Some(vec![replace_import(imprt)]) - } - _ => None, - }, - }) - .flatten() - .filter(|x| !STDIMPORTS.contains(&x.as_str())) - .unique() - .collect(); - imports.sort(); - Ok(imports) - } -} - #[cfg(test)] mod tests { @@ -441,354 +351,4 @@ def main(test1: str, Ok(()) } - - #[test] - fn test_parse_python_imports() -> anyhow::Result<()> { - //let code = "print(2 + 3, fd=sys.stderr)"; - let code = " - -import os -import wmill -from zanzibar.estonie import talin -import matplotlib.pyplot as plt -from . import tests - -def main(): - pass - -"; - let r = parse_python_imports(code)?; - // println!("{}", serde_json::to_string(&r)?); - assert_eq!(r, vec!["matplotlib", "requests", "wmill", "zanzibar"]); - Ok(()) - } - - #[test] - fn test_parse_python_imports2() -> anyhow::Result<()> { - //let code = "print(2 + 3, fd=sys.stderr)"; - let code = " -#requirements: -#burkina=0.4 -#nigeria -# -#congo - -import os -import wmill -from zanzibar.estonie import talin - -def main(): - pass - -"; - let r = parse_python_imports(code)?; - println!("{}", serde_json::to_string(&r)?); - assert_eq!(r, vec!["burkina=0.4", "nigeria"]); - - Ok(()) - } } - -const STDIMPORTS: [&str; 301] = [ - "__future__", - "_abc", - "_aix_support", - "_ast", - "_asyncio", - "_bisect", - "_blake2", - "_bootsubprocess", - "_bz2", - "_codecs", - "_codecs_cn", - "_codecs_hk", - "_codecs_iso2022", - "_codecs_jp", - "_codecs_kr", - "_codecs_tw", - "_collections", - "_collections_abc", - "_compat_pickle", - "_compression", - "_contextvars", - "_crypt", - "_csv", - "_ctypes", - "_curses", - "_curses_panel", - "_datetime", - "_dbm", - "_decimal", - "_elementtree", - "_frozen_importlib", - "_frozen_importlib_external", - "_functools", - "_gdbm", - "_hashlib", - "_heapq", - "_imp", - "_io", - "_json", - "_locale", - "_lsprof", - "_lzma", - "_markupbase", - "_md5", - "_msi", - "_multibytecodec", - "_multiprocessing", - "_opcode", - "_operator", - "_osx_support", - "_overlapped", - "_pickle", - "_posixshmem", - "_posixsubprocess", - "_py_abc", - "_pydecimal", - "_pyio", - "_queue", - "_random", - "_sha1", - "_sha256", - "_sha3", - "_sha512", - "_signal", - "_sitebuiltins", - "_socket", - "_sqlite3", - "_sre", - "_ssl", - "_stat", - "_statistics", - "_string", - "_strptime", - "_struct", - "_symtable", - "_thread", - "_threading_local", - "_tkinter", - "_tracemalloc", - "_uuid", - "_warnings", - "_weakref", - "_weakrefset", - "_winapi", - "_zoneinfo", - "abc", - "aifc", - "antigravity", - "argparse", - "array", - "ast", - "asynchat", - "asyncio", - "asyncore", - "atexit", - "audioop", - "base64", - "bdb", - "binascii", - "binhex", - "bisect", - "builtins", - "bz2", - "cProfile", - "calendar", - "cgi", - "cgitb", - "chunk", - "cmath", - "cmd", - "code", - "codecs", - "codeop", - "collections", - "colorsys", - "compileall", - "concurrent", - "configparser", - "contextlib", - "contextvars", - "copy", - "copyreg", - "crypt", - "csv", - "ctypes", - "curses", - "dataclasses", - "datetime", - "dbm", - "decimal", - "difflib", - "dis", - "distutils", - "doctest", - "email", - "encodings", - "ensurepip", - "enum", - "errno", - "faulthandler", - "fcntl", - "filecmp", - "fileinput", - "fnmatch", - "fractions", - "ftplib", - "functools", - "gc", - "genericpath", - "getopt", - "getpass", - "gettext", - "glob", - "graphlib", - "grp", - "gzip", - "hashlib", - "heapq", - "hmac", - "html", - "http", - "idlelib", - "imaplib", - "imghdr", - "imp", - "importlib", - "inspect", - "io", - "ipaddress", - "itertools", - "json", - "keyword", - "lib2to3", - "linecache", - "locale", - "logging", - "lzma", - "mailbox", - "mailcap", - "marshal", - "math", - "mimetypes", - "mmap", - "modulefinder", - "msilib", - "msvcrt", - "multiprocessing", - "netrc", - "nis", - "nntplib", - "nt", - "ntpath", - "nturl2path", - "numbers", - "opcode", - "operator", - "optparse", - "os", - "ossaudiodev", - "pathlib", - "pdb", - "pickle", - "pickletools", - "pipes", - "pkgutil", - "platform", - "plistlib", - "poplib", - "posix", - "posixpath", - "pprint", - "profile", - "pstats", - "pty", - "pwd", - "py_compile", - "pyclbr", - "pydoc", - "pydoc_data", - "pyexpat", - "queue", - "quopri", - "random", - "re", - "readline", - "reprlib", - "resource", - "rlcompleter", - "runpy", - "sched", - "secrets", - "select", - "selectors", - "shelve", - "shlex", - "shutil", - "signal", - "site", - "smtpd", - "smtplib", - "sndhdr", - "socket", - "socketserver", - "spwd", - "sqlite3", - "sre_compile", - "sre_constants", - "sre_parse", - "ssl", - "stat", - "statistics", - "string", - "stringprep", - "struct", - "subprocess", - "sunau", - "symtable", - "sys", - "sysconfig", - "syslog", - "tabnanny", - "tarfile", - "telnetlib", - "tempfile", - "termios", - "textwrap", - "this", - "threading", - "time", - "timeit", - "tkinter", - "token", - "tokenize", - "trace", - "traceback", - "tracemalloc", - "tty", - "turtle", - "turtledemo", - "types", - "typing", - "unicodedata", - "unittest", - "urllib", - "uu", - "uuid", - "venv", - "warnings", - "wave", - "weakref", - "webbrowser", - "winreg", - "winsound", - "wsgiref", - "xdrlib", - "xml", - "xmlrpc", - "zipapp", - "zipfile", - "zipimport", - "", -]; diff --git a/backend/parsers/windmill-parser-ts-wasm/pkg/package-lock.json b/backend/parsers/windmill-parser-ts-wasm/pkg/package-lock.json deleted file mode 100644 index f6f37a1a9d..0000000000 --- a/backend/parsers/windmill-parser-ts-wasm/pkg/package-lock.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "windmill-parser-ts-wasm", - "version": "1.109.1", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "windmill-parser-ts-wasm", - "version": "1.109.1" - } - } -} diff --git a/backend/parsers/windmill-parser-ts-wasm/pkg/package.json b/backend/parsers/windmill-parser-ts-wasm/pkg/package.json deleted file mode 100644 index dc33248f48..0000000000 --- a/backend/parsers/windmill-parser-ts-wasm/pkg/package.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "windmill-parser-ts-wasm", - "collaborators": [ - "Ruben Fiszel " - ], - "version": "1.109.1", - "files": [ - "windmill_parser_ts_wasm_bg.wasm", - "windmill_parser_ts_wasm.js", - "windmill_parser_ts_wasm.d.ts" - ], - "module": "windmill_parser_ts_wasm.js", - "types": "windmill_parser_ts_wasm.d.ts", - "sideEffects": [ - "./snippets/*" - ] -} \ No newline at end of file diff --git a/backend/parsers/windmill-parser-ts-wasm/pkg/windmill_parser_ts_wasm_bg.wasm b/backend/parsers/windmill-parser-ts-wasm/pkg/windmill_parser_ts_wasm_bg.wasm deleted file mode 100644 index 659fcf97ae..0000000000 Binary files a/backend/parsers/windmill-parser-ts-wasm/pkg/windmill_parser_ts_wasm_bg.wasm and /dev/null differ diff --git a/backend/parsers/windmill-parser-ts-wasm/src/lib.rs b/backend/parsers/windmill-parser-ts-wasm/src/lib.rs deleted file mode 100644 index 6c48df72af..0000000000 --- a/backend/parsers/windmill-parser-ts-wasm/src/lib.rs +++ /dev/null @@ -1,13 +0,0 @@ -use serde_json; -use wasm_bindgen::prelude::*; -use windmill_parser_ts::parse_deno_signature; - -#[wasm_bindgen] -pub fn parse_deno_wasm(code: &str) -> String { - let r = parse_deno_signature(code, false); - if let Ok(r) = r { - return serde_json::to_string(&r).unwrap(); - } else { - return "{\"type\": \"Invalid\"}".to_string(); - } -} diff --git a/backend/parsers/windmill-parser-ts/src/lib.rs b/backend/parsers/windmill-parser-ts/src/lib.rs index afc0d571ea..0091ac6625 100644 --- a/backend/parsers/windmill-parser-ts/src/lib.rs +++ b/backend/parsers/windmill-parser-ts/src/lib.rs @@ -21,7 +21,7 @@ use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax, TsConfig}; #[cfg(target_arch = "wasm32")] use wasm_bindgen::prelude::*; -pub fn parse_deno_signature(code: &str, skip_dflt: bool) -> Result { +pub fn parse_deno_signature(code: &str, skip_dflt: bool) -> anyhow::Result { let cm: Lrc = Default::default(); let fm = cm.new_source_file(FileName::Custom("main.ts".into()), code.into()); let lexer = Lexer::new( @@ -42,7 +42,7 @@ pub fn parse_deno_signature(code: &str, skip_dflt: bool) -> Result Result, String>>()?, + .collect::>>()?, }; Ok(r) } else { - Err( + Err(anyhow::anyhow!( "main function was not findable (expected to find 'export function main(...)'" .to_string(), - ) + )) } } -fn parse_param(x: Param, cm: &Lrc, skip_dflt: bool) -> Result { +fn parse_param(x: Param, cm: &Lrc, skip_dflt: bool) -> anyhow::Result { let r = match x.pat { Pat::Ident(ident) => { let (name, typ, nullable) = binding_ident_to_arg(&ident); @@ -87,7 +87,7 @@ fn parse_param(x: Param, cm: &Lrc, skip_dflt: bool) -> Result { let (name, mut typ, _nullable) = left.as_ident().map(binding_ident_to_arg).ok_or_else(|| { - format!( + anyhow::anyhow!( "parameter syntax unsupported: `{}`", cm.span_to_snippet(left.span()) .unwrap_or_else(|_| cm.span_to_string(left.span())) @@ -120,7 +120,7 @@ fn parse_param(x: Param, cm: &Lrc, skip_dflt: bool) -> Result Err(format!( + _ => Err(anyhow::anyhow!( "parameter syntax unsupported: `{}`", cm.span_to_snippet(x.span()) .unwrap_or_else(|_| cm.span_to_string(x.span())) @@ -280,6 +280,6 @@ pub fn eval_sync(code: &str) -> Result { } #[cfg(not(target_arch = "wasm32"))] -pub fn eval_sync(code: &str) -> Result { +pub fn eval_sync(_code: &str) -> Result { panic!("eval_sync is only available in wasm32") } diff --git a/backend/parsers/windmill-parser-ts-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml similarity index 63% rename from backend/parsers/windmill-parser-ts-wasm/Cargo.toml rename to backend/parsers/windmill-parser-wasm/Cargo.toml index 62f46eaf6d..33ef0f5c46 100644 --- a/backend/parsers/windmill-parser-ts-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -1,19 +1,23 @@ [package] -name = "windmill-parser-ts-wasm" +name = "windmill-parser-wasm" version.workspace = true edition.workspace = true authors.workspace = true [lib] crate-type = ["cdylib"] -name = "windmill_parser_ts_wasm" +name = "windmill_parser_wasm" path = "./src/lib.rs" [dev-dependencies] -windmill-parser.workspace = true wasm-bindgen-test.workspace = true [dependencies] +anyhow.workspace = true +windmill-parser.workspace = true +windmill-parser-go.workspace = true +windmill-parser-bash.workspace = true +windmill-parser-py.workspace = true windmill-parser-ts.workspace = true wasm-bindgen.workspace = true serde_json.workspace = true \ No newline at end of file diff --git a/backend/parsers/windmill-parser-ts-wasm/package-lock.json b/backend/parsers/windmill-parser-wasm/package-lock.json similarity index 100% rename from backend/parsers/windmill-parser-ts-wasm/package-lock.json rename to backend/parsers/windmill-parser-wasm/package-lock.json diff --git a/backend/parsers/windmill-parser-ts-wasm/package.json b/backend/parsers/windmill-parser-wasm/package.json similarity index 100% rename from backend/parsers/windmill-parser-ts-wasm/package.json rename to backend/parsers/windmill-parser-wasm/package.json diff --git a/backend/parsers/windmill-parser-wasm/pkg/package.json b/backend/parsers/windmill-parser-wasm/pkg/package.json new file mode 100644 index 0000000000..3132350e51 --- /dev/null +++ b/backend/parsers/windmill-parser-wasm/pkg/package.json @@ -0,0 +1,17 @@ +{ + "name": "windmill-parser-wasm", + "collaborators": [ + "Ruben Fiszel " + ], + "version": "1.109.1", + "files": [ + "windmill_parser_wasm_bg.wasm", + "windmill_parser_wasm.js", + "windmill_parser_wasm.d.ts" + ], + "module": "windmill_parser_wasm.js", + "types": "windmill_parser_wasm.d.ts", + "sideEffects": [ + "./snippets/*" + ] +} \ No newline at end of file diff --git a/backend/parsers/windmill-parser-ts-wasm/pkg/windmill_parser_ts_wasm.d.ts b/backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm.d.ts similarity index 66% rename from backend/parsers/windmill-parser-ts-wasm/pkg/windmill_parser_ts_wasm.d.ts rename to backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm.d.ts index 0e1b0a4ad1..884128e73a 100644 --- a/backend/parsers/windmill-parser-ts-wasm/pkg/windmill_parser_ts_wasm.d.ts +++ b/backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm.d.ts @@ -4,13 +4,31 @@ * @param {string} code * @returns {string} */ -export function parse_deno_wasm(code: string): string; +export function parse_deno(code: string): string; +/** +* @param {string} code +* @returns {string} +*/ +export function parse_bash(code: string): string; +/** +* @param {string} code +* @returns {string} +*/ +export function parse_go(code: string): string; +/** +* @param {string} code +* @returns {string} +*/ +export function parse_python(code: string): string; export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module; export interface InitOutput { readonly memory: WebAssembly.Memory; - readonly parse_deno_wasm: (a: number, b: number, c: number) => void; + readonly parse_deno: (a: number, b: number, c: number) => void; + readonly parse_bash: (a: number, b: number, c: number) => void; + readonly parse_go: (a: number, b: number, c: number) => void; + readonly parse_python: (a: number, b: number, c: number) => void; readonly __wbindgen_malloc: (a: number) => number; readonly __wbindgen_realloc: (a: number, b: number, c: number) => number; readonly __wbindgen_add_to_stack_pointer: (a: number) => number; diff --git a/backend/parsers/windmill-parser-ts-wasm/pkg/windmill_parser_ts_wasm.js b/backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm.js similarity index 86% rename from backend/parsers/windmill-parser-ts-wasm/pkg/windmill_parser_ts_wasm.js rename to backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm.js index 72ed7ba7ed..bf489ad733 100644 --- a/backend/parsers/windmill-parser-ts-wasm/pkg/windmill_parser_ts_wasm.js +++ b/backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm.js @@ -201,14 +201,83 @@ function debugString(val) { * @param {string} code * @returns {string} */ -export function parse_deno_wasm(code) { +export function parse_deno(code) { let deferred2_0; let deferred2_1; try { const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); const len0 = WASM_VECTOR_LEN; - wasm.parse_deno_wasm(retptr, ptr0, len0); + wasm.parse_deno(retptr, ptr0, len0); + var r0 = getInt32Memory0()[retptr / 4 + 0]; + var r1 = getInt32Memory0()[retptr / 4 + 1]; + deferred2_0 = r0; + deferred2_1 = r1; + return getStringFromWasm0(r0, r1); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + wasm.__wbindgen_free(deferred2_0, deferred2_1); + } +} + +/** +* @param {string} code +* @returns {string} +*/ +export function parse_bash(code) { + let deferred2_0; + let deferred2_1; + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + wasm.parse_bash(retptr, ptr0, len0); + var r0 = getInt32Memory0()[retptr / 4 + 0]; + var r1 = getInt32Memory0()[retptr / 4 + 1]; + deferred2_0 = r0; + deferred2_1 = r1; + return getStringFromWasm0(r0, r1); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + wasm.__wbindgen_free(deferred2_0, deferred2_1); + } +} + +/** +* @param {string} code +* @returns {string} +*/ +export function parse_go(code) { + let deferred2_0; + let deferred2_1; + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + wasm.parse_go(retptr, ptr0, len0); + var r0 = getInt32Memory0()[retptr / 4 + 0]; + var r1 = getInt32Memory0()[retptr / 4 + 1]; + deferred2_0 = r0; + deferred2_1 = r1; + return getStringFromWasm0(r0, r1); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + wasm.__wbindgen_free(deferred2_0, deferred2_1); + } +} + +/** +* @param {string} code +* @returns {string} +*/ +export function parse_python(code) { + let deferred2_0; + let deferred2_1; + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + wasm.parse_python(retptr, ptr0, len0); var r0 = getInt32Memory0()[retptr / 4 + 0]; var r1 = getInt32Memory0()[retptr / 4 + 1]; deferred2_0 = r0; @@ -262,13 +331,13 @@ async function __wbg_load(module, imports) { function __wbg_get_imports() { const imports = {}; imports.wbg = {}; - imports.wbg.__wbindgen_object_drop_ref = function(arg0) { - takeObject(arg0); - }; imports.wbg.__wbg_eval_8130c2f52f1a6d39 = function(arg0, arg1) { const ret = eval(getStringFromWasm0(arg0, arg1)); return addHeapObject(ret); }; + imports.wbg.__wbindgen_object_drop_ref = function(arg0) { + takeObject(arg0); + }; imports.wbg.__wbindgen_string_get = function(arg0, arg1) { const obj = getObject(arg1); const ret = typeof(obj) === 'string' ? obj : undefined; @@ -468,7 +537,7 @@ async function __wbg_init(input) { if (wasm !== undefined) return wasm; if (typeof input === 'undefined') { - input = new URL('windmill_parser_ts_wasm_bg.wasm', import.meta.url); + input = new URL('windmill_parser_wasm_bg.wasm', import.meta.url); } const imports = __wbg_get_imports(); diff --git a/backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm_bg.wasm b/backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm_bg.wasm new file mode 100644 index 0000000000..e5f9817b49 Binary files /dev/null and b/backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm_bg.wasm differ diff --git a/backend/parsers/windmill-parser-ts-wasm/pkg/windmill_parser_ts_wasm_bg.wasm.d.ts b/backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm_bg.wasm.d.ts similarity index 59% rename from backend/parsers/windmill-parser-ts-wasm/pkg/windmill_parser_ts_wasm_bg.wasm.d.ts rename to backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm_bg.wasm.d.ts index 090727c83a..07a75cb832 100644 --- a/backend/parsers/windmill-parser-ts-wasm/pkg/windmill_parser_ts_wasm_bg.wasm.d.ts +++ b/backend/parsers/windmill-parser-wasm/pkg/windmill_parser_wasm_bg.wasm.d.ts @@ -1,7 +1,10 @@ /* tslint:disable */ /* eslint-disable */ export const memory: WebAssembly.Memory; -export function parse_deno_wasm(a: number, b: number, c: number): void; +export function parse_deno(a: number, b: number, c: number): void; +export function parse_bash(a: number, b: number, c: number): void; +export function parse_go(a: number, b: number, c: number): void; +export function parse_python(a: number, b: number, c: number): void; export function __wbindgen_malloc(a: number): number; export function __wbindgen_realloc(a: number, b: number, c: number): number; export function __wbindgen_add_to_stack_pointer(a: number): number; diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs new file mode 100644 index 0000000000..2cdcd6e44a --- /dev/null +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -0,0 +1,31 @@ +use serde_json; +use wasm_bindgen::prelude::*; +use windmill_parser::MainArgSignature; + +fn wrap_sig(r: anyhow::Result) -> String { + if let Ok(r) = r { + return serde_json::to_string(&r).unwrap(); + } else { + return "{\"type\": \"Invalid\"}".to_string(); + } +} + +#[wasm_bindgen] +pub fn parse_deno(code: &str) -> String { + wrap_sig(windmill_parser_ts::parse_deno_signature(code, false)) +} + +#[wasm_bindgen] +pub fn parse_bash(code: &str) -> String { + wrap_sig(windmill_parser_bash::parse_bash_sig(code)) +} + +#[wasm_bindgen] +pub fn parse_go(code: &str) -> String { + wrap_sig(windmill_parser_go::parse_go_sig(code)) +} + +#[wasm_bindgen] +pub fn parse_python(code: &str) -> String { + wrap_sig(windmill_parser_py::parse_python_signature(code)) +} diff --git a/backend/parsers/windmill-parser-ts-wasm/tests/wasm.rs b/backend/parsers/windmill-parser-wasm/tests/wasm.rs similarity index 98% rename from backend/parsers/windmill-parser-ts-wasm/tests/wasm.rs rename to backend/parsers/windmill-parser-wasm/tests/wasm.rs index e04440eddf..50c0a53c48 100644 --- a/backend/parsers/windmill-parser-ts-wasm/tests/wasm.rs +++ b/backend/parsers/windmill-parser-wasm/tests/wasm.rs @@ -4,7 +4,7 @@ use windmill_parser::{Arg, MainArgSignature, ObjectProperty, Typ}; use windmill_parser_ts::parse_deno_signature; #[wasm_bindgen_test] -fn test_parse_deno_sig() -> Result<(), String> { +fn test_parse_deno_sig() -> anyhow::Result<()> { let code = " export function main(test1?: string, test2: string = \"burkina\", test3: wmill.Resource<'postgres'>, b64: Base64, ls: Base64[], @@ -115,7 +115,7 @@ export function main(test1?: string, test2: string = \"burkina\", } #[wasm_bindgen_test] -fn test_parse_deno_sig_implicit_types() -> Result<(), String> { +fn test_parse_deno_sig_implicit_types() -> anyhow::Result<()> { let code = " export function main(test2 = \"burkina\", bool = true, diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 5d27f79560..f4ba7f4945 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -24,10 +24,7 @@ windmill-common = { workspace = true, features = [ ] } windmill-audit.workspace = true windmill-parser.workspace = true -windmill-parser-ts.workspace = true -windmill-parser-go.workspace = true -windmill-parser-py.workspace = true -windmill-parser-bash.workspace = true +windmill-parser-py-imports.workspace = true tokio.workspace = true anyhow.workspace = true argon2.workspace = true diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index c431843a9c..144a932a58 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -2278,90 +2278,6 @@ paths: items: type: string - /scripts/python/tojsonschema: - post: - summary: inspect python code to infer jsonschema of arguments - operationId: pythonToJsonschema - tags: - - script - requestBody: - description: python code with the main function - required: true - content: - application/json: - schema: - type: string - responses: - "200": - description: parsed args - content: - application/json: - schema: - $ref: "#/components/schemas/MainArgSignature" - - /scripts/deno/tojsonschema: - post: - summary: inspect deno code to infer jsonschema of arguments - operationId: denoToJsonschema - tags: - - script - requestBody: - description: deno code with the main function - required: true - content: - application/json: - schema: - type: string - responses: - "200": - description: parsed args - content: - application/json: - schema: - $ref: "#/components/schemas/MainArgSignature" - - /scripts/bash/tojsonschema: - post: - summary: inspect bash code to infer jsonschema of arguments - operationId: bashToJsonschema - tags: - - script - requestBody: - description: bash code with the main function - required: true - content: - application/json: - schema: - type: string - responses: - "200": - description: parsed args - content: - application/json: - schema: - $ref: "#/components/schemas/MainArgSignature" - - /scripts/go/tojsonschema: - post: - summary: inspect go code to infer jsonschema of arguments - operationId: goToJsonschema - tags: - - script - requestBody: - description: go code with the main function - required: true - content: - application/json: - schema: - type: string - responses: - "200": - description: parsed args - content: - application/json: - schema: - $ref: "#/components/schemas/MainArgSignature" - /w/{workspace}/scripts/archive/p/{path}: post: summary: archive script by path diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index 11507a06cd..dfdbef8190 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -76,13 +76,6 @@ pub struct ScriptWDraft { pub fn global_service() -> Router { Router::new() - .route( - "/python/tojsonschema", - post(parse_python_code_to_jsonschema), - ) - .route("/deno/tojsonschema", post(parse_deno_code_to_jsonschema)) - .route("/go/tojsonschema", post(parse_go_code_to_jsonschema)) - .route("/bash/tojsonschema", post(parse_bash_code_to_jsonschema)) .route("/hub/list", get(list_hub_scripts)) .route("/hub/get/*path", get(get_hub_script_by_path)) .route("/hub/get_full/*path", get(get_full_hub_script_by_path)) @@ -483,7 +476,7 @@ async fn create_script( if needs_lock_gen { let dependencies = match ns.language { ScriptLang::Python3 => { - windmill_parser_py::parse_python_imports(&ns.content)?.join("\n") + windmill_parser_py_imports::parse_python_imports(&ns.content)?.join("\n") } _ => ns.content, }; @@ -916,20 +909,3 @@ fn result_to_sig_parsing(result: Result) -> Json { Err(e) => Json(SigParsing::Invalid { error: e.to_string() }), } } - -async fn parse_python_code_to_jsonschema(Json(code): Json) -> Json { - result_to_sig_parsing(windmill_parser_py::parse_python_signature(&code)) -} - -async fn parse_deno_code_to_jsonschema(Json(code): Json) -> Json { - result_to_sig_parsing( - windmill_parser_ts::parse_deno_signature(&code, false).map_err(|e| Error::ExecutionErr(e)), - ) -} -async fn parse_go_code_to_jsonschema(Json(code): Json) -> Json { - result_to_sig_parsing(windmill_parser_go::parse_go_sig(&code)) -} - -async fn parse_bash_code_to_jsonschema(Json(code): Json) -> Json { - result_to_sig_parsing(windmill_parser_bash::parse_bash_sig(&code)) -} diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index 73cfe7b1df..b53bf4e461 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -27,6 +27,7 @@ windmill-parser.workspace = true windmill-parser-ts.workspace = true windmill-parser-go.workspace = true windmill-parser-py.workspace = true +windmill-parser-py-imports.workspace = true windmill-parser-bash.workspace = true sqlx.workspace = true uuid.workspace = true diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 874c616851..d5e9d1b946 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -175,7 +175,8 @@ pub async fn handle_python_job( let requirements = match requirements_o { Some(r) => r, None => { - let requirements = windmill_parser_py::parse_python_imports(&inner_content)?.join("\n"); + let requirements = + windmill_parser_py_imports::parse_python_imports(&inner_content)?.join("\n"); if requirements.is_empty() { "".to_string() } else { diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 79d85344ad..aa3e49d083 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -1755,7 +1755,7 @@ async fn handle_flow_dependency_job( }; // sync with windmill-api/scripts let dependencies = match language { - ScriptLang::Python3 => windmill_parser_py::parse_python_imports(&content)?.join("\n"), + ScriptLang::Python3 => windmill_parser_py_imports::parse_python_imports(&content)?.join("\n"), _ => content.clone(), }; let new_lock = capture_dependency_job( diff --git a/docker/DockerfileOpenbb b/docker/DockerfileOpenbb index d71f970b96..a5c15a917c 100644 --- a/docker/DockerfileOpenbb +++ b/docker/DockerfileOpenbb @@ -53,7 +53,7 @@ COPY /backend/windmill-api/openapi.yaml /backend/windmill-api/openapi.yaml COPY /openflow.openapi.yaml /openflow.openapi.yaml COPY /backend/windmill-api/build_openapi.sh /backend/windmill-api/build_openapi.sh RUN cd /backend/windmill-api && . ./build_openapi.sh -COPY /backend/parsers/windmill-parser-py-wasm/pkg/ /backend/windmill-parser-py-wasm/pkg/ +COPY /backend/parsers/windmill-parser-wasm/pkg/ /backend/parsers/windmill-parser-wasm/pkg/ RUN npm run generate-backend-client ENV NODE_OPTIONS "--max-old-space-size=8192" diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 190874259b..9141cc0b04 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -36,7 +36,7 @@ "svelte-timezone-picker": "^2.0.3", "tailwind-merge": "^1.12.0", "vscode-ws-jsonrpc": "3.0.0", - "windmill-parser-ts-wasm": "file:../backend/parsers/windmill-parser-ts-wasm/pkg", + "windmill-parser-wasm": "file:../backend/parsers/windmill-parser-wasm/pkg", "y-monaco": "^0.1.4", "y-websocket": "^1.5.0", "yjs": "^13.6.1" @@ -104,6 +104,10 @@ }, "../backend/parsers/windmill-parser-ts-wasm/pkg": { "name": "windmill-parser-ts-wasm", + "version": "1.109.1", + "extraneous": true + }, + "../backend/parsers/windmill-parser-wasm/pkg": { "version": "1.109.1" }, "node_modules/@alloc/quick-lru": { @@ -7699,8 +7703,8 @@ "string-width": "^1.0.2 || 2 || 3 || 4" } }, - "node_modules/windmill-parser-ts-wasm": { - "resolved": "../backend/parsers/windmill-parser-ts-wasm/pkg", + "node_modules/windmill-parser-wasm": { + "resolved": "../backend/parsers/windmill-parser-wasm/pkg", "link": true }, "node_modules/word-wrap": { @@ -13396,8 +13400,8 @@ "string-width": "^1.0.2 || 2 || 3 || 4" } }, - "windmill-parser-ts-wasm": { - "version": "file:../backend/parsers/windmill-parser-ts-wasm/pkg" + "windmill-parser-wasm": { + "version": "file:../backend/parsers/windmill-parser-wasm/pkg" }, "word-wrap": { "version": "1.2.3", diff --git a/frontend/package.json b/frontend/package.json index 1e91f9fcdb..7b02a98b54 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -99,7 +99,7 @@ "y-monaco": "^0.1.4", "y-websocket": "^1.5.0", "yjs": "^13.6.1", - "windmill-parser-ts-wasm": "file:../backend/parsers/windmill-parser-ts-wasm/pkg" + "windmill-parser-wasm": "file:../backend/parsers/windmill-parser-wasm/pkg" }, "peerDependencies": { "@sveltejs/kit": "^1.20.1", diff --git a/frontend/src/lib/infer.ts b/frontend/src/lib/infer.ts index bbc0802171..8d2c53ef88 100644 --- a/frontend/src/lib/infer.ts +++ b/frontend/src/lib/infer.ts @@ -1,10 +1,10 @@ -import { ScriptService, type MainArgSignature } from '$lib/gen' +import type { MainArgSignature } from '$lib/gen' import { get, writable } from 'svelte/store' import type { Schema, SchemaProperty } from './common.js' import { sortObject } from './utils.js' import { tick } from 'svelte' -import init, { parse_deno_wasm } from 'windmill-parser-ts-wasm' -import wasmUrl from 'windmill-parser-ts-wasm/windmill_parser_ts_wasm_bg.wasm?url' +import init, { parse_deno, parse_bash, parse_go, parse_python } from 'windmill-parser-wasm' +import wasmUrl from 'windmill-parser-wasm/windmill_parser_wasm_bg.wasm?url' init(wasmUrl) @@ -24,19 +24,13 @@ export async function inferArgs( code = ' ' } if (language == 'python3') { - inferedSchema = await ScriptService.pythonToJsonschema({ - requestBody: code - }) + inferedSchema = JSON.parse(parse_python(code)) } else if (language == 'deno') { - inferedSchema = JSON.parse(parse_deno_wasm(code)) + inferedSchema = JSON.parse(parse_deno(code)) } else if (language == 'go') { - inferedSchema = await ScriptService.goToJsonschema({ - requestBody: code - }) + inferedSchema = JSON.parse(parse_go(code)) } else if (language == 'bash') { - inferedSchema = await ScriptService.bashToJsonschema({ - requestBody: code - }) + inferedSchema = JSON.parse(parse_bash(code)) } else { return }