starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-19 00:02:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

Separate asset parsers (#8321)

Browse Source 
* Refactor asset_parser

* package update

* package lock
This commit is contained in:
Diego Imbert
2026-03-11 17:54:05 +01:00
committed by GitHub
parent e7047761cf
commit 577484d06a
25 changed files with 168 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/Cargo.lock
Generated
+39 -2
View File
@@ -15941,6 +15941,7 @@ dependencies = [
"windmill-parser-py",
"windmill-parser-py-imports",
"windmill-parser-sql",
"windmill-parser-sql-asset",
"windmill-parser-ts",
"windmill-queue",
"windmill-store",
@@ -16864,7 +16865,17 @@ dependencies = [
"rustpython-parser",
"serde_json",
"windmill-parser",
"windmill-parser-sql",
]
[[package]]
name = "windmill-parser-py-asset"
version = "1.653.0"
dependencies = [
"anyhow",
"rustpython-ast",
"rustpython-parser",
"windmill-parser",
"windmill-parser-sql-asset",
]
[[package]]
@@ -16932,6 +16943,17 @@ dependencies = [
"regex-lite",
"serde",
"serde_json",
"windmill-parser",
"windmill-types",
]
[[package]]
name = "windmill-parser-sql-asset"
version = "1.653.0"
dependencies = [
"anyhow",
"serde",
"serde_json",
"sqlparser 0.59.0",
"windmill-parser",
"windmill-types",
@@ -16953,7 +16975,22 @@ dependencies = [
"triomphe",
"wasm-bindgen",
"windmill-parser",
"windmill-parser-sql",
]
[[package]]
name = "windmill-parser-ts-asset"
version = "1.653.0"
dependencies = [
"anyhow",
"serde-wasm-bindgen",
"swc_common",
"swc_ecma_ast",
"swc_ecma_parser",
"swc_ecma_visit",
"triomphe",
"wasm-bindgen",
"windmill-parser",
"windmill-parser-sql-asset",
]
[[package]]
backend/Cargo.toml
+7
View File
@@ -59,6 +59,7 @@ members = [
"./windmill-oauth",
"./parsers/windmill-parser",
"./parsers/windmill-parser-ts",
"./parsers/windmill-parser-ts-asset",
"./parsers/windmill-parser-go",
"./parsers/windmill-parser-rust",
"./parsers/windmill-parser-csharp",
@@ -67,8 +68,11 @@ members = [
"./parsers/windmill-parser-ruby",
"./parsers/windmill-parser-bash",
"./parsers/windmill-parser-py",
"./parsers/windmill-parser-py-asset",
"./parsers/windmill-parser-py-imports",
"./parsers/windmill-parser-wac",
"./parsers/windmill-parser-sql",
"./parsers/windmill-parser-sql-asset",
"./parsers/windmill-sql-datatype-parser-wasm",
"./parsers/windmill-parser-yaml", "windmill-macros", "parsers/windmill-parser-nu",
"./windmill-worker-volumes",
@@ -320,7 +324,9 @@ windmill-api-workers = { path = "./windmill-api-workers" }
windmill-store = { path = "./windmill-store" }
windmill-parser = { path = "./parsers/windmill-parser" }
windmill-parser-ts = { path = "./parsers/windmill-parser-ts" }
windmill-parser-ts-asset = { path = "./parsers/windmill-parser-ts-asset" }
windmill-parser-py = { path = "./parsers/windmill-parser-py" }
windmill-parser-py-asset = { path = "./parsers/windmill-parser-py-asset" }
windmill-parser-py-imports = { path = "./parsers/windmill-parser-py-imports" }
windmill-parser-go = { path = "./parsers/windmill-parser-go" }
windmill-parser-rust = { path = "./parsers/windmill-parser-rust" }
@@ -331,6 +337,7 @@ windmill-parser-ruby = { path = "./parsers/windmill-parser-ruby" }
windmill-parser-nu = { path = "./parsers/windmill-parser-nu" }
windmill-parser-bash = { path = "./parsers/windmill-parser-bash" }
windmill-parser-sql = { path = "./parsers/windmill-parser-sql" }
windmill-parser-sql-asset = { path = "./parsers/windmill-parser-sql-asset" }
windmill-parser-graphql = { path = "./parsers/windmill-parser-graphql" }
windmill-parser-php = { path = "./parsers/windmill-parser-php" }
windmill-parser-wac = { path = "./parsers/windmill-parser-wac" }
backend/parsers/windmill-parser-py-asset/Cargo.toml
+16
View File
@@ -0,0 +1,16 @@
[package]
name = "windmill-parser-py-asset"
version.workspace = true
edition.workspace = true
authors.workspace = true
[lib]
name = "windmill_parser_py_asset"
path = "./src/lib.rs"
[dependencies]
windmill-parser.workspace = true
windmill-parser-sql-asset.workspace = true
rustpython-parser.workspace = true
rustpython-ast = { version = "0.4.0", features = ["visitor"] }
anyhow.workspace = true
backend/parsers/windmill-parser-py/src/asset_parser.rs → backend/parsers/windmill-parser-py-asset/src/lib.rs
+1 -1
View File
@@ -215,7 +215,7 @@ impl AssetsFinder {
_ => return Err(()),
};
// We use the SQL parser to detect RW, specific tables, etc.
let sql_assets = windmill_parser_sql::parse_wmill_sdk_sql_assets(
let sql_assets = windmill_parser_sql_asset::parse_wmill_sdk_sql_assets(
*kind,
path,
schema.as_deref(),
backend/parsers/windmill-parser-py/Cargo.toml
-1
View File
@@ -10,7 +10,6 @@ path = "./src/lib.rs"
[dependencies]
windmill-parser.workspace = true
windmill-parser-sql.workspace = true
rustpython-parser.workspace = true
itertools.workspace = true
serde_json.workspace = true
backend/parsers/windmill-parser-py/src/lib.rs
-3
View File
@@ -21,11 +21,8 @@ use rustpython_parser::{
Parse,
};
pub mod asset_parser;
pub mod pydantic_parser;
pub use asset_parser::parse_assets;
const FUNCTION_CALL: &str = "<function call>";
/// Cheap string-based check to see if code might contain Pydantic models or dataclasses.
backend/parsers/windmill-parser-sql-asset/Cargo.toml
+17
View File
@@ -0,0 +1,17 @@
[package]
name = "windmill-parser-sql-asset"
version.workspace = true
edition.workspace = true
authors.workspace = true
[lib]
name = "windmill_parser_sql_asset"
path = "./src/lib.rs"
[dependencies]
windmill-parser.workspace = true
windmill-types.workspace = true
anyhow.workspace = true
serde_json.workspace = true
serde.workspace = true
sqlparser = { version = "0.59.0", features = ["visitor"] }
backend/parsers/windmill-parser-sql/src/asset_parser.rs → backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs
View File
backend/parsers/windmill-parser-sql/src/asset_parser_utils.rs → backend/parsers/windmill-parser-sql-asset/src/asset_parser_utils.rs
View File
backend/parsers/windmill-parser-sql-asset/src/lib.rs
+4
View File
@@ -0,0 +1,4 @@
mod asset_parser;
mod asset_parser_utils;
pub use asset_parser::parse_assets;
pub use asset_parser_utils::parse_wmill_sdk_sql_assets;
backend/parsers/windmill-parser-sql/Cargo.toml
+1 -2
View File
@@ -20,5 +20,4 @@ windmill-types.workspace = true
anyhow.workspace = true
lazy_static.workspace = true
serde_json.workspace = true
serde.workspace = true
sqlparser = { version = "0.59.0", features = ["visitor"] }
serde.workspace = true
backend/parsers/windmill-parser-sql/src/lib.rs
-5
View File
@@ -20,11 +20,6 @@ pub use windmill_parser::{Arg, MainArgSignature, ObjectType, Typ};
pub const SANITIZED_ENUM_STR: &str = "__sanitized_enum__";
pub const SANITIZED_RAW_STRING_STR: &str = "__sanitized_raw_string__";
mod asset_parser;
mod asset_parser_utils;
pub use asset_parser::parse_assets;
pub use asset_parser_utils::parse_wmill_sdk_sql_assets;
pub fn parse_mysql_sig(code: &str) -> anyhow::Result<MainArgSignature> {
let parsed = parse_mysql_file(&code)?;
if let Some(x) = parsed {
backend/parsers/windmill-parser-ts-asset/Cargo.toml
+23
View File
@@ -0,0 +1,23 @@
[package]
name = "windmill-parser-ts-asset"
version.workspace = true
edition.workspace = true
authors.workspace = true
[lib]
name = "windmill_parser_ts_asset"
path = "./src/lib.rs"
[target.'cfg(target_arch = "wasm32")'.dependencies]
wasm-bindgen.workspace = true
serde-wasm-bindgen.workspace = true
[dependencies]
windmill-parser.workspace = true
windmill-parser-sql-asset.workspace = true
swc_common.workspace = true
triomphe.workspace = true
swc_ecma_parser.workspace = true
swc_ecma_ast.workspace = true
swc_ecma_visit.workspace = true
anyhow.workspace = true
backend/parsers/windmill-parser-ts/src/asset_parser.rs → backend/parsers/windmill-parser-ts-asset/src/lib.rs
+1 -1
View File
@@ -259,7 +259,7 @@ impl Visit for AssetsFinder {
});
// We use the SQL parser to detect RW, specific tables, etc.
let sql_assets = windmill_parser_sql::parse_wmill_sdk_sql_assets(
let sql_assets = windmill_parser_sql_asset::parse_wmill_sdk_sql_assets(
*kind,
asset_name,
schema.as_deref(),
backend/parsers/windmill-parser-ts/Cargo.toml
-1
View File
@@ -15,7 +15,6 @@ serde-wasm-bindgen.workspace = true
[dependencies]
windmill-parser.workspace = true
windmill-parser-sql.workspace = true
swc_common.workspace = true
triomphe.workspace = true
swc_ecma_parser.workspace = true
backend/parsers/windmill-parser-ts/src/lib.rs
-2
View File
@@ -211,8 +211,6 @@ pub enum TypeDecl {
Interface(TsInterfaceDecl),
Alias(TsTypeAliasDecl),
}
pub mod asset_parser;
pub use asset_parser::parse_assets;
/// skip_params is a micro optimization for when we just want to find the main
/// function without parsing all the params.
backend/parsers/windmill-parser-wasm/Cargo.toml
+4
View File
@@ -39,6 +39,7 @@ nu-parser = [ "dep:windmill-parser-nu"]
java-parser = [ "dep:windmill-parser-java"]
ruby-parser = [ "dep:windmill-parser-ruby"]
wac-parser = [ "dep:windmill-parser-wac"]
asset-parser = [ "dep:windmill-parser-ts-asset", "dep:windmill-parser-py-asset", "dep:windmill-parser-sql-asset"]
[dependencies]
anyhow.workspace = true
@@ -57,6 +58,9 @@ windmill-parser-nu = { workspace = true, optional = true }
windmill-parser-java = { workspace = true, optional = true }
windmill-parser-ruby = { workspace = true, optional = true }
windmill-parser-wac = { workspace = true, optional = true }
windmill-parser-ts-asset = { workspace = true, optional = true }
windmill-parser-py-asset = { workspace = true, optional = true }
windmill-parser-sql-asset = { workspace = true, optional = true }
wasm-bindgen.workspace = true
serde_json.workspace = true
backend/parsers/windmill-parser-wasm/build.nu
+5
View File
@@ -61,6 +61,11 @@ const targets = [
desc: "Workflow-as-Code",
features: "wac-parser",
env: "default",
}, {
ident: "asset",
desc: "Asset parsers (TS, Python, SQL) with SQL AST",
features: "asset-parser",
env: "default",
},
# ^^^ Add new entry here ^^^
];
backend/parsers/windmill-parser-wasm/publish-pkgs.sh
+3
View File
@@ -33,3 +33,6 @@ popd
pushd "pkg-java" && npm publish ${args}
popd
pushd "pkg-asset" && npm publish ${args}
popd
backend/parsers/windmill-parser-wasm/src/lib.rs
+6 -6
View File
@@ -187,28 +187,28 @@ pub fn parse_ruby(code: &str) -> String {
wrap_sig(windmill_parser_ruby::parse_ruby_signature(code))
}
#[cfg(feature = "sql-parser")]
#[cfg(feature = "asset-parser")]
#[wasm_bindgen]
pub fn parse_assets_sql(code: &str) -> String {
match windmill_parser_sql::parse_assets(code) {
match windmill_parser_sql_asset::parse_assets(code) {
Ok(r) => serde_json::to_string(&r).unwrap(),
Err(err) => format!("err: {:?}", err),
}
}
#[cfg(feature = "ts-parser")]
#[cfg(feature = "asset-parser")]
#[wasm_bindgen]
pub fn parse_assets_ts(code: &str) -> String {
match windmill_parser_ts::parse_assets(code) {
match windmill_parser_ts_asset::parse_assets(code) {
Ok(r) => serde_json::to_string(&r).unwrap(),
Err(err) => format!("err: {:?}", err),
}
}
#[cfg(feature = "py-parser")]
#[cfg(feature = "asset-parser")]
#[wasm_bindgen]
pub fn parse_assets_py(code: &str) -> String {
match windmill_parser_py::parse_assets(code) {
match windmill_parser_py_asset::parse_assets(code) {
Ok(r) => serde_json::to_string(&r).unwrap(),
Err(err) => format!("err: {:?}", err),
}
backend/windmill-api/Cargo.toml
+1
View File
@@ -63,6 +63,7 @@ windmill-object-store.workspace = true
windmill-audit.workspace = true
windmill-parser.workspace = true
windmill-parser-sql.workspace = true
windmill-parser-sql-asset.workspace = true
windmill-parser-ts.workspace = true
windmill-parser-py = { workspace = true, optional = true }
windmill-parser-py-imports = { workspace = true, optional = true }
backend/windmill-api/src/jobs.rs
+2 -2
View File
@@ -4833,7 +4833,7 @@ fn register_potential_assets_on_inline_execution(
preview: &PreviewInline,
) {
let assets = if preview.language == ScriptLang::DuckDb {
Some(windmill_parser_sql::parse_assets(&preview.content).map(|a| a.assets))
Some(windmill_parser_sql_asset::parse_assets(&preview.content).map(|a| a.assets))
} else if preview.language == ScriptLang::Postgresql {
let datatable = preview
.args
@@ -4851,7 +4851,7 @@ fn register_potential_assets_on_inline_execution(
(None, None)
};
let content = content.as_deref().unwrap_or(&preview.content);
windmill_parser_sql::parse_wmill_sdk_sql_assets(
windmill_parser_sql_asset::parse_wmill_sdk_sql_assets(
AssetKind::DataTable,
datatable,
schema.as_deref(),
frontend/package-lock.json
Generated
+18 -12
View File
@@ -75,16 +75,17 @@
"vscode-languageclient": "~9.0.1",
"vscode-uri": "~3.1.0",
"vscode-ws-jsonrpc": "~3.5.0",
"windmill-parser-wasm-asset": "1.653.0",
"windmill-parser-wasm-csharp": "1.510.1",
"windmill-parser-wasm-go": "1.510.1",
"windmill-parser-wasm-java": "1.510.1",
"windmill-parser-wasm-nu": "1.510.1",
"windmill-parser-wasm-php": "1.647.1",
"windmill-parser-wasm-py": "^1.628.3",
"windmill-parser-wasm-regex": "1.646.0",
"windmill-parser-wasm-py": "1.653.0",
"windmill-parser-wasm-regex": "1.653.0",
"windmill-parser-wasm-ruby": "1.526.1",
"windmill-parser-wasm-rust": "1.647.1",
"windmill-parser-wasm-ts": "1.647.1",
"windmill-parser-wasm-ts": "1.653.0",
"windmill-parser-wasm-yaml": "1.593.0",
"windmill-sql-datatype-parser-wasm": "1.512.0",
"windmill-utils-internal": "^1.3.4",
@@ -12698,6 +12699,11 @@
"node": ">=8"
}
},
"node_modules/windmill-parser-wasm-asset": {
"version": "1.653.0",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-asset/-/windmill-parser-wasm-asset-1.653.0.tgz",
"integrity": "sha512-Tc9smy79wZSEBxAbiad8D4NiydOVHl541amCRQ93yqqffsmFnyRr4tpBYV/dR5uQ4zdX7aTe5ROwz2jhsXODYQ=="
},
"node_modules/windmill-parser-wasm-csharp": {
"version": "1.510.1",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-csharp/-/windmill-parser-wasm-csharp-1.510.1.tgz",
@@ -12724,14 +12730,14 @@
"integrity": "sha512-u2qaMkupSdhJibxvkLh3r/y36IARvnYNTLXWvOKxcQ0G/BPUB4+yF5o/yf47vv9zUV5WZv4mrdsKDt/pZDYeDg=="
},
"node_modules/windmill-parser-wasm-py": {
"version": "1.628.3",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.628.3.tgz",
"integrity": "sha512-TlluqknZpg8cZ+A3m6JFLPseY2PpKtDsxdj26fAnCUzKPtse8TxQR+n0dwC80rfW5TwdWSulvNGRDgcNuf7CTw=="
"version": "1.653.0",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.653.0.tgz",
"integrity": "sha512-vMkSL3JpELpag7nmyGA8onhYNiAG3K1mkh2k4vwVHC3W5dUd12fSS9gsBco2FqPGUPnWv1gCaHwmjBrOBVGL1w=="
},
"node_modules/windmill-parser-wasm-regex": {
"version": "1.646.0",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.646.0.tgz",
"integrity": "sha512-sRDQBX9ML3VIcW9r1Ug8SOKkHog6uKBwCTE6OADU3K8o8C45NiJufMGCeKtBWoF/Ki08IEj+0h4dd6QEIKpeIQ=="
"version": "1.653.0",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.653.0.tgz",
"integrity": "sha512-LEvLhb8uCb/jGMzd+lwP886LcwxuTE+W04wdyqmdqy1JD9FBToKeiqW7CtZWqbWr1oI8yK9U3jmqSqyOcUCzRw=="
},
"node_modules/windmill-parser-wasm-ruby": {
"version": "1.526.1",
@@ -12744,9 +12750,9 @@
"integrity": "sha512-9yGLYZX2Hn9TdTqGY/5Fp50ftzgUsrfBkSK9vJkKJd5Amyg+yXLBGzd8pz6Org+4uxMenz/16wpsgijvo6uhhQ=="
},
"node_modules/windmill-parser-wasm-ts": {
"version": "1.647.1",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.647.1.tgz",
"integrity": "sha512-64iSAUMU5W/WtePqE1vtDvglDqtkiZVndyieYBVDX0nl7UuovS+wPgH/P3TEoKbR+FwAPacki0CX3DsEzZ/Yxw=="
"version": "1.653.0",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.653.0.tgz",
"integrity": "sha512-zwBUy7ijo58ooAKcsYflISY/+xllCw3Aq34Kj1PED6uABWVbV6A8MWHqEsjiVzC6iWfihWNtZJpck8zsRr9DCg=="
},
"node_modules/windmill-parser-wasm-yaml": {
"version": "1.593.0",
frontend/package.json
+5 -4
View File
@@ -147,16 +147,17 @@
"vscode-languageclient": "~9.0.1",
"vscode-uri": "~3.1.0",
"vscode-ws-jsonrpc": "~3.5.0",
"windmill-parser-wasm-asset": "1.653.0",
"windmill-parser-wasm-csharp": "1.510.1",
"windmill-parser-wasm-go": "1.510.1",
"windmill-parser-wasm-java": "1.510.1",
"windmill-parser-wasm-nu": "1.510.1",
"windmill-parser-wasm-php": "1.647.1",
"windmill-parser-wasm-py": "^1.628.3",
"windmill-parser-wasm-regex": "1.646.0",
"windmill-parser-wasm-py": "1.653.0",
"windmill-parser-wasm-regex": "1.653.0",
"windmill-parser-wasm-ruby": "1.526.1",
"windmill-parser-wasm-rust": "1.647.1",
"windmill-parser-wasm-ts": "1.647.1",
"windmill-parser-wasm-ts": "1.653.0",
"windmill-parser-wasm-yaml": "1.593.0",
"windmill-sql-datatype-parser-wasm": "1.512.0",
"windmill-utils-internal": "^1.3.4",
@@ -581,4 +582,4 @@
"@rollup/rollup-linux-x64-gnu": "^4.35.0",
"fsevents": "^2.3.3"
}
}
}
frontend/src/lib/infer.ts
+15 -7
View File
@@ -10,7 +10,7 @@ import type { Schema, SupportedLanguage } from './common.js'
import { emptySchema, sortObject } from './utils.js'
import { tick } from 'svelte'
import initTsParser, { parse_assets_ts, parse_deno, parse_outputs } from 'windmill-parser-wasm-ts'
import initTsParser, { parse_deno, parse_outputs } from 'windmill-parser-wasm-ts'
import initRegexParsers, {
parse_sql,
parse_mysql,
@@ -22,10 +22,14 @@ import initRegexParsers, {
parse_mssql,
parse_db_resource,
parse_bash,
parse_powershell,
parse_assets_sql
parse_powershell
} from 'windmill-parser-wasm-regex'
import initPythonParser, { parse_assets_py, parse_python } from 'windmill-parser-wasm-py'
import initPythonParser, { parse_python } from 'windmill-parser-wasm-py'
import initAssetParser, {
parse_assets_ts,
parse_assets_py,
parse_assets_sql
} from 'windmill-parser-wasm-asset'
import initGoParser, { parse_go } from 'windmill-parser-wasm-go'
import initPhpParser, { parse_php } from 'windmill-parser-wasm-php'
import initRustParser, { parse_rust } from 'windmill-parser-wasm-rust'
@@ -50,6 +54,7 @@ import wasmUrlCSharp from 'windmill-parser-wasm-csharp/windmill_parser_wasm_bg.w
import wasmUrlNu from 'windmill-parser-wasm-nu/windmill_parser_wasm_bg.wasm?url'
import wasmUrlJava from 'windmill-parser-wasm-java/windmill_parser_wasm_bg.wasm?url'
import wasmUrlRuby from 'windmill-parser-wasm-ruby/windmill_parser_wasm_bg.wasm?url'
import wasmUrlAsset from 'windmill-parser-wasm-asset/windmill_parser_wasm_bg.wasm?url'
import { workspaceStore } from './stores.js'
import { argSigToJsonSchemaType } from 'windmill-utils-internal'
import { type AssetWithAccessType } from './components/assets/lib.js'
@@ -94,6 +99,9 @@ async function initWasmJava() {
async function initWasmRuby() {
await initRubyParser(wasmUrlRuby)
}
async function initWasmAsset() {
await initAssetParser(wasmUrlAsset)
}
type InferAssetsResult =
| {
@@ -171,13 +179,13 @@ export async function inferAssets(
try {
if (language === 'duckdb') {
await initWasmRegex()
await initWasmAsset()
result = wrap(parse_assets_sql(code))
} else if (language === 'deno' || language === 'nativets' || language === 'bun') {
await initWasmTs()
await initWasmAsset()
result = wrap(parse_assets_ts(code))
} else if (language === 'python3') {
await initWasmPython()
await initWasmAsset()
result = wrap(parse_assets_py(code))
} else if (language === 'ansible') {
await initWasmYaml()