fix: parse python 3.12-3.15 syntax in the python parsers (#11675)

* fix: parse python 3.12-3.15 syntax by moving the python parsers to ruff

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test: pin async for loops in the python wac dag

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: tell main apart from a look-alike line inside a string

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: keep import split and async for as before, fix sql asset path panic

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ruben Fiszel
2026-10-11 23:26:03 +02:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 0758bdc46b
commit aae5f2efaa
15 changed files with 1350 additions and 876 deletions
+278 -176
View File
@@ -950,6 +950,35 @@ version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
[[package]]
name = "attribute-derive"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2f9ef7a68684657ce9da341c934c73b63df7a3585a61e366d84c84ed5d3d167"
dependencies = [
"attribute-derive-macro",
"manyhow 0.14.0",
"proc-macro2",
"quote",
"syn 3.0.6",
]
[[package]]
name = "attribute-derive-macro"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "300d426e5e73549125bb8e4649bdd5508f82c94ddf898fb0517246e71dfb8e98"
dependencies = [
"collection_literals",
"interpolator",
"manyhow 0.14.0",
"proc-macro-utils",
"proc-macro2",
"quote",
"quote-use",
"syn 3.0.6",
]
[[package]]
name = "autocfg"
version = "1.5.1"
@@ -2385,6 +2414,18 @@ dependencies = [
"rand_core 0.10.1",
]
[[package]]
name = "char_str"
version = "0.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c9cba78a5abb515610e10af297615434b019755b5d7be14e559ff5340dff50c4"
dependencies = [
"castaway",
"get-size2",
"itoa",
"ryu",
]
[[package]]
name = "chrono"
version = "0.4.45"
@@ -2516,6 +2557,12 @@ dependencies = [
"x509-cert",
]
[[package]]
name = "collection_literals"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2550f75b8cfac212855f6b1885455df8eaee8fe8e246b647d69146142e016084"
[[package]]
name = "colorchoice"
version = "1.0.5"
@@ -2555,6 +2602,19 @@ dependencies = [
"static_assertions",
]
[[package]]
name = "compact_str"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "79fcda08c33bb58b97008b2cdada6622500e949e060f5913361763121abd2416"
dependencies = [
"castaway",
"cfg-if",
"itoa",
"static_assertions",
"zmij",
]
[[package]]
name = "concurrent-queue"
version = "2.5.0"
@@ -4630,7 +4690,6 @@ dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
"unicode-xid",
]
[[package]]
@@ -4887,6 +4946,12 @@ dependencies = [
"text_lines",
]
[[package]]
name = "drop_bomb"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9bda8e21c04aca2ae33ffc2fd8c23134f3cac46db123ba97bd9d3f3b8a4a85e1"
[[package]]
name = "dtoa"
version = "1.0.11"
@@ -5816,6 +5881,31 @@ dependencies = [
"zeroize",
]
[[package]]
name = "get-size-derive2"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28f0daa179069133df60523e0f59d0a6bd2d3db9ccae01b630896d269c4a802e"
dependencies = [
"attribute-derive",
"quote",
"syn 3.0.6",
]
[[package]]
name = "get-size2"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "050a5d07359ebfac7f0b74916d94cfa30575e45fdc7e9bfbb0f76d5d4203325a"
dependencies = [
"compact_str 0.10.0",
"get-size-derive2",
"hashbrown 0.17.1",
"ordermap",
"smallvec",
"thin-vec",
]
[[package]]
name = "gethostname"
version = "0.4.3"
@@ -7005,6 +7095,12 @@ version = "3.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02"
[[package]]
name = "interpolator"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71dd52191aae121e8611f1e8dc3e324dd0dd1dee1e6dd91d10ee07a3cfb4d9d8"
[[package]]
name = "inventory"
version = "0.3.25"
@@ -7104,6 +7200,15 @@ dependencies = [
"either",
]
[[package]]
name = "itertools"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc"
dependencies = [
"either",
]
[[package]]
name = "itoa"
version = "1.0.18"
@@ -7493,12 +7598,6 @@ dependencies = [
"tracing",
]
[[package]]
name = "lalrpop-util"
version = "0.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553"
[[package]]
name = "lapin"
version = "2.5.5"
@@ -7917,76 +8016,30 @@ dependencies = [
"webpki-roots 0.26.11",
]
[[package]]
name = "malachite"
version = "0.4.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2fbdf9cb251732db30a7200ebb6ae5d22fe8e11397364416617d2c2cf0c51cb5"
dependencies = [
"malachite-base",
"malachite-nz",
"malachite-q",
]
[[package]]
name = "malachite-base"
version = "0.4.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ea0ed76adf7defc1a92240b5c36d5368cfe9251640dcce5bd2d0b7c1fd87aeb"
dependencies = [
"hashbrown 0.14.5",
"itertools 0.11.0",
"libm",
"ryu",
]
[[package]]
name = "malachite-bigint"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d149aaa2965d70381709d9df4c7ee1fc0de1c614a4efc2ee356f5e43d68749f8"
dependencies = [
"derive_more 1.0.0",
"malachite",
"num-integer",
"num-traits",
"paste",
]
[[package]]
name = "malachite-nz"
version = "0.4.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34a79feebb2bc9aa7762047c8e5495269a367da6b5a90a99882a0aeeac1841f7"
dependencies = [
"itertools 0.11.0",
"libm",
"malachite-base",
]
[[package]]
name = "malachite-q"
version = "0.4.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50f235d5747b1256b47620f5640c2a17a88c7569eebdf27cd9cb130e1a619191"
dependencies = [
"itertools 0.11.0",
"malachite-base",
"malachite-nz",
]
[[package]]
name = "manyhow"
version = "0.11.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b33efb3ca6d3b07393750d4030418d594ab1139cee518f0dc88db70fec873587"
dependencies = [
"manyhow-macros",
"manyhow-macros 0.11.4",
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
name = "manyhow"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c69ea6e469ca785c846b25dd69eac9e9c916b51d8ada1f212866787ee1da12d"
dependencies = [
"manyhow-macros 0.14.0",
"proc-macro2",
"quote",
"syn 3.0.6",
]
[[package]]
name = "manyhow-macros"
version = "0.11.4"
@@ -7998,6 +8051,17 @@ dependencies = [
"quote",
]
[[package]]
name = "manyhow-macros"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22fc5c1b2c4000056dae3619fea36e79ce79dabfee4907d2dcad6612b189e370"
dependencies = [
"proc-macro-utils",
"proc-macro2",
"quote",
]
[[package]]
name = "mappable-rc"
version = "0.1.1"
@@ -8265,7 +8329,7 @@ checksum = "a4db8a44120571277accfaa3f3d91e7d3989d601d817c2fc01a9391b86135666"
dependencies = [
"darling 0.23.0",
"heck",
"manyhow",
"manyhow 0.11.4",
"num-bigint",
"proc-macro-crate",
"proc-macro2",
@@ -9236,6 +9300,15 @@ dependencies = [
"num-traits",
]
[[package]]
name = "ordermap"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16a756ddcdeb6c0efc8927bdf0ed112885815a35ba350472a467be5104f01ca1"
dependencies = [
"indexmap 2.14.2",
]
[[package]]
name = "os_pipe"
version = "1.2.1"
@@ -10325,6 +10398,28 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "quote-use"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48356e493823dc3c1d7e9fc7b3ff204ecec48afbdd718fc4134091b88eb63f3d"
dependencies = [
"quote",
"quote-use-macros",
]
[[package]]
name = "quote-use-macros"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8cf1c5de6a7cc231c71e03c77aa4c7019097859ea7029156b54d2894a8430704"
dependencies = [
"proc-macro-utils",
"proc-macro2",
"quote",
"syn 3.0.6",
]
[[package]]
name = "r-efi"
version = "5.3.0"
@@ -11078,6 +11173,106 @@ dependencies = [
"zeroize",
]
[[package]]
name = "ruff_python_ast"
version = "0.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15434b06df7d930fad2818639a4e0fec4af9c770121c45a96b72da7feaae9ead"
dependencies = [
"aho-corasick",
"arrayvec",
"bitflags 2.13.2",
"char_str",
"compact_str 0.10.0",
"get-size2",
"is-macro",
"memchr",
"ruff_python_trivia",
"ruff_source_file",
"ruff_text_size",
"rustc-hash 2.1.3",
"thin-vec",
"thiserror 2.0.21",
]
[[package]]
name = "ruff_python_parser"
version = "0.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37a22811d752aa9e255d7dd72524d14352cf9bc654ef79b399bf1eeeb6640fbb"
dependencies = [
"bitflags 2.13.2",
"bstr",
"drop_bomb",
"get-size2",
"hashbrown 0.17.1",
"memchr",
"ruff_python_ast",
"ruff_python_trivia",
"ruff_text_size",
"ruff_unicode_names2",
"rustc-hash 2.1.3",
"stacker",
"static_assertions",
"thin-vec",
"unicode-ident",
"unicode-normalization",
]
[[package]]
name = "ruff_python_trivia"
version = "0.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0322a5059fe0feb2e010ca6f31c24ac289f6b7e9e38359539491b10c57f1c5bc"
dependencies = [
"itertools 0.15.0",
"memchr",
"ruff_source_file",
"ruff_text_size",
"rustc-hash 2.1.3",
"unicode-ident",
"unicode-normalization",
]
[[package]]
name = "ruff_source_file"
version = "0.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6ae809a289b5c791f82aa45df33f2a92323e7a1c1d265c10e9ac884027a2e43"
dependencies = [
"memchr",
"ruff_text_size",
]
[[package]]
name = "ruff_text_size"
version = "0.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "daf6344d6915ced9316312c1ee3f24c4300c54295bc4c3b8f4e95b633ae5443b"
dependencies = [
"get-size2",
]
[[package]]
name = "ruff_unicode_names2"
version = "0.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13418b8fd029d597e35dedc36e91f47122d26075e4c86557574d9441753fb375"
dependencies = [
"phf 0.11.3",
"ruff_unicode_names2_generator",
]
[[package]]
name = "ruff_unicode_names2_generator"
version = "0.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37fd82af49d5685d9054e68185987acaaf62542f671b6df8935155664876bc39"
dependencies = [
"phf_codegen",
"rand 0.8.5",
]
[[package]]
name = "rumqttc"
version = "0.24.0"
@@ -11414,63 +11609,6 @@ dependencies = [
"untrusted 0.9.0",
]
[[package]]
name = "rustpython-ast"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cdaf8ee5c1473b993b398c174641d3aa9da847af36e8d5eb8291930b72f31a5"
dependencies = [
"is-macro",
"malachite-bigint",
"rustpython-parser-core",
"static_assertions",
]
[[package]]
name = "rustpython-parser"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "868f724daac0caf9bd36d38caf45819905193a901e8f1c983345a68e18fb2abb"
dependencies = [
"anyhow",
"is-macro",
"itertools 0.11.0",
"lalrpop-util",
"log",
"malachite-bigint",
"num-traits",
"phf 0.11.3",
"phf_codegen",
"rustc-hash 1.1.0",
"rustpython-ast",
"rustpython-parser-core",
"tiny-keccak",
"unic-emoji-char",
"unic-ucd-ident",
"unicode_names2",
]
[[package]]
name = "rustpython-parser-core"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4b6c12fa273825edc7bccd9a734f0ad5ba4b8a2f4da5ff7efe946f066d0f4ad"
dependencies = [
"is-macro",
"memchr",
"rustpython-parser-vendored",
]
[[package]]
name = "rustpython-parser-vendored"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04fcea49a4630a3a5d940f4d514dc4f575ed63c14c3e3ed07146634aed7f67a6"
dependencies = [
"memchr",
"once_cell",
]
[[package]]
name = "rustversion"
version = "1.0.23"
@@ -12903,7 +13041,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bcf55c2d7555c93f4945e29f93b7529562be97ba16e60dd94c25724d746174ac"
dependencies = [
"ascii",
"compact_str",
"compact_str 0.7.1",
"memchr",
"num-bigint",
"once_cell",
@@ -13628,6 +13766,12 @@ dependencies = [
"unicode-width 0.2.2",
]
[[package]]
name = "thin-vec"
version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6a4b9ba8738cb4a4f399d37e266becfd475e75eb73425b87a05a2f2039ba63e"
[[package]]
name = "thiserror"
version = "1.0.69"
@@ -14677,17 +14821,6 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc"
[[package]]
name = "unic-emoji-char"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b07221e68897210270a38bde4babb655869637af0f69407f96053a34f76494d"
dependencies = [
"unic-char-property",
"unic-char-range",
"unic-ucd-version",
]
[[package]]
name = "unic-ucd-category"
version = "0.9.0"
@@ -14700,17 +14833,6 @@ dependencies = [
"unic-ucd-version",
]
[[package]]
name = "unic-ucd-ident"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e230a37c0381caa9219d67cf063aa3a375ffed5bf541a452db16e744bdab6987"
dependencies = [
"unic-char-property",
"unic-char-range",
"unic-ucd-version",
]
[[package]]
name = "unic-ucd-version"
version = "0.9.0"
@@ -14798,28 +14920,6 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
[[package]]
name = "unicode_names2"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d1673eca9782c84de5f81b82e4109dcfb3611c8ba0d52930ec4a9478f547b2dd"
dependencies = [
"phf 0.11.3",
"unicode_names2_generator",
]
[[package]]
name = "unicode_names2_generator"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b91e5b84611016120197efd7dc93ef76774f4e084cd73c9fb3ea4a86c570c56e"
dependencies = [
"getopts",
"log",
"phf_codegen",
"rand 0.8.5",
]
[[package]]
name = "universal-hash"
version = "0.5.1"
@@ -16630,8 +16730,8 @@ version = "1.830.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
"rustpython-ast",
"rustpython-parser",
"ruff_python_ast",
"ruff_python_parser",
"serde_json",
"windmill-parser",
]
@@ -16641,8 +16741,8 @@ name = "windmill-parser-py-asset"
version = "1.830.0"
dependencies = [
"anyhow",
"rustpython-ast",
"rustpython-parser",
"ruff_python_ast",
"ruff_python_parser",
"windmill-parser",
"windmill-parser-sql-asset",
]
@@ -16659,7 +16759,8 @@ dependencies = [
"phf 0.11.3",
"regex",
"regex-lite",
"rustpython-parser",
"ruff_python_ast",
"ruff_python_parser",
"serde",
"serde_json",
"sqlx",
@@ -16776,8 +16877,9 @@ name = "windmill-parser-wac"
version = "1.830.0"
dependencies = [
"anyhow",
"rustpython-ast",
"rustpython-parser",
"ruff_python_ast",
"ruff_python_parser",
"ruff_text_size",
"serde",
"serde_json",
"sha2 0.10.9",
+5 -3
View File
@@ -488,9 +488,11 @@ arc-swap = "1"
ed25519-dalek = { version = "2", features = ["rand_core"] }
magic-crypt = "^3"
git-version = "^0"
malachite = "=0.4.18"
malachite-bigint = "=0.2.0"
rustpython-parser = "^0"
# Ruff's parser crates are internal to ruff and carry no semver guarantee: pin them exactly,
# and to the same release.
ruff_python_parser = "=0.0.17"
ruff_python_ast = "=0.0.17"
ruff_text_size = "=0.0.17"
pep440_rs = "0.7.3"
php-parser-rs = { git = "https://github.com/php-rust-tools/parser", rev = "ec4cb411dec09450946ef57920b7ffced7f6495d" }
cron = "^0"
@@ -11,6 +11,6 @@ path = "./src/lib.rs"
[dependencies]
windmill-parser.workspace = true
windmill-parser-sql-asset.workspace = true
rustpython-parser.workspace = true
rustpython-ast = { version = "0.4.0", features = ["visitor"] }
ruff_python_parser.workspace = true
ruff_python_ast.workspace = true
anyhow.workspace = true
@@ -1,5 +1,8 @@
use rustpython_ast::{Constant, Expr, ExprConstant, Visitor};
use rustpython_parser::{ast::Suite, Parse};
use ruff_python_ast::{
visitor::{walk_expr, walk_stmt, Visitor},
Expr, ExprCall, ExprDict, FStringPartRef, InterpolatedStringElement, Keyword, Stmt, StmtAssign,
};
use ruff_python_parser::parse_module;
use std::collections::HashMap;
use windmill_parser::asset_parser::{
asset_was_used, merge_assets, parse_asset_syntax, parse_pipeline_annotations, AssetKind,
@@ -8,12 +11,12 @@ use windmill_parser::asset_parser::{
use AssetUsageAccessType::*;
pub fn parse_assets(input: &str) -> anyhow::Result<ParseAssetsOutput> {
let ast = Suite::parse(input, "main.py")
.map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?;
let ast = parse_module(input)
.map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?
.into_suite();
let mut assets_finder = AssetsFinder { assets: vec![], var_identifiers: HashMap::new() };
ast.into_iter()
.for_each(|stmt| assets_finder.visit_stmt(stmt));
assets_finder.visit_body(&ast);
for (kind, path, _) in assets_finder.var_identifiers.into_values() {
// if a db = wmill.datatable() was never used (e.g db.query(...)),
@@ -43,13 +46,72 @@ struct AssetsFinder {
var_identifiers: HashMap<String, (AssetKind, VarAssetName, VarAssetSchema)>,
}
impl Visitor for AssetsFinder {
impl<'a> Visitor<'a> for AssetsFinder {
fn visit_stmt(&mut self, stmt: &'a Stmt) {
if let Stmt::Assign(node) = stmt {
self.track_assignment(node);
}
// Continue with generic visit to catch any other assets in the statement
walk_stmt(self, stmt);
}
fn visit_expr(&mut self, expr: &'a Expr) {
match expr {
// A detected asset call does not recurse,
// so this is only reached when no further context was found
Expr::StringLiteral(s) => self.visit_str(s.value.to_str()),
Expr::FString(f) => {
// Literal text between two interpolations reads as one string, across the
// pieces of an implicitly concatenated f-string too.
let mut literal = String::new();
for part in &f.value {
match part {
FStringPartRef::Literal(s) => literal.push_str(&s.value),
FStringPartRef::FString(f) => {
for element in &f.elements {
match element {
InterpolatedStringElement::Literal(l) => {
literal.push_str(&l.value)
}
InterpolatedStringElement::Interpolation(_) => {
self.visit_str(&std::mem::take(&mut literal))
}
}
}
}
}
}
self.visit_str(&literal);
walk_expr(self, expr);
}
Expr::Call(node) => {
if self.visit_expr_call_inner(node).is_err() {
walk_expr(self, expr);
}
}
_ => walk_expr(self, expr),
}
}
}
impl AssetsFinder {
fn visit_str(&mut self, s: &str) {
if let Some((kind, path)) = parse_asset_syntax(s, false) {
self.assets.push(ParseAssetsResult {
kind,
path: path.to_string(),
access_type: None,
columns: None,
});
}
}
// Handle assignment statements like: x = wmill.datatable('name')
fn visit_stmt_assign(&mut self, node: rustpython_ast::StmtAssign) {
fn track_assignment(&mut self, node: &StmtAssign) {
// Check if the value is a call to a tracked function
if let Some(Expr::Name(expr_name)) = node.targets.first() {
// Remove any tracked variables with that name in case of reassignment
let Ok(var_name) = expr_name.id.parse::<String>();
let var_name = expr_name.id.to_string();
let removed = self.var_identifiers.remove(&var_name);
// if a db = wmill.datatable() or similar was removed, but never used (e.g db.query(...)),
// we still want to register the asset as unknown access type
@@ -69,59 +131,12 @@ impl Visitor for AssetsFinder {
if let Some((kind, name, schema)) = self.extract_asset_from_call(&node.value) {
// Track target variable
let Ok(var_name) = expr_name.id.parse::<String>();
self.var_identifiers
.insert(var_name, (kind.clone(), name.clone(), schema.clone()));
}
}
// Continue with generic visit to catch any other assets in the expression
self.generic_visit_stmt_assign(node);
}
// visit_call_expr will not recurse if it detects an asset,
// so this will only be called when no further context was found
fn visit_expr_constant(&mut self, node: ExprConstant) {
match node.value {
Constant::Str(s) => {
if let Some((kind, path)) = parse_asset_syntax(&s, false) {
self.assets.push(ParseAssetsResult {
kind,
path: path.to_string(),
access_type: None,
columns: None,
});
}
}
_ => self.generic_visit_expr_constant(node),
}
}
fn visit_expr_call(&mut self, node: rustpython_ast::ExprCall) {
match self.visit_expr_call_inner(&node) {
Ok(_) => {}
Err(_) => {
// Check keyword arguments for assets before falling back to generic visit
for keyword in &node.keywords {
if let Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) =
&keyword.value
{
if let Some((kind, path)) = parse_asset_syntax(s, false) {
self.assets.push(ParseAssetsResult {
kind,
path: path.to_string(),
access_type: None,
columns: None,
});
}
}
}
self.generic_visit_expr_call(node);
}
}
}
}
impl AssetsFinder {
/// Extract asset info from calls like wmill.datatable('name'), wmill.ducklake('name'), etc.
fn extract_asset_from_call(
&self,
@@ -147,18 +162,20 @@ impl AssetsFinder {
// Get the first argument (the asset name)
let name = call
.arguments
.args
.first()
.or_else(|| {
// Try keyword argument 'name' or 'path'
call.keywords
call.arguments
.keywords
.iter()
.find(|kw| matches!(kw.arg.as_deref(), Some("name") | Some("path")))
.map(|kw| &kw.value)
})
.and_then(|first_arg| {
if let Expr::Constant(ExprConstant { value: Constant::Str(name), .. }) = first_arg {
Some(name.clone())
if let Expr::StringLiteral(name) = first_arg {
Some(name.value.to_str().to_string())
} else {
None
}
@@ -179,44 +196,39 @@ impl AssetsFinder {
Some((kind, name, schema))
}
fn visit_expr_call_inner(&mut self, node: &rustpython_ast::ExprCall) -> Result<(), ()> {
fn visit_expr_call_inner(&mut self, node: &ExprCall) -> Result<(), ()> {
let ident: String = node
.func
.as_name_expr()
.and_then(|o| o.id.parse().ok())
.map(|o| o.id.to_string())
.or_else(|| {
node.func
.as_attribute_expr()
.and_then(|attr| attr.attr.parse().ok())
.map(|attr| attr.attr.to_string())
})
.ok_or(())?;
// `obj_name` is the object (receiver) - i.e., `obj` in `obj.method()`
let obj_name: String = if let Expr::Attribute(rustpython_ast::ExprAttribute {
value, ..
}) = node.func.as_ref()
{
if let Expr::Name(expr_name) = value.as_ref() {
expr_name.id.parse().map_err(|_| ())?
} else {
return Err(());
}
} else {
return Err(());
};
let obj_name: String = node
.func
.as_attribute_expr()
.and_then(|attr| attr.value.as_name_expr())
.map(|name| name.id.to_string())
.ok_or(())?;
if obj_name == "wmill" {
// Continue
} else if let Some((kind, ref path, ref schema)) = self.var_identifiers.get(&obj_name) {
if ident == "query" {
let expr_name = node.args.get(0).or_else(|| {
node.keywords
let expr_name = node.arguments.args.get(0).or_else(|| {
node.arguments
.keywords
.iter()
.find(|kw| kw.arg.as_deref() == Some("name"))
.map(|kw| &kw.value)
});
let sql = match expr_name {
Some(Expr::Constant(ExprConstant { value: Constant::Str(sql), .. })) => sql,
Some(Expr::StringLiteral(sql)) => sql.value.to_str(),
_ => return Err(()),
};
// We use the SQL parser to detect RW, specific tables, etc.
@@ -224,7 +236,7 @@ impl AssetsFinder {
*kind,
path,
schema.as_deref(),
&sql,
sql,
);
match sql_assets {
Ok(Some(sql_assets)) => self.assets.extend(sql_assets),
@@ -252,9 +264,10 @@ impl AssetsFinder {
};
let arg_val = match arg {
Arg(i, name) => node.args.get(i).or_else(|| {
Arg(i, name) => node.arguments.args.get(i).or_else(|| {
// Get arg by name
node.keywords
node.arguments
.keywords
.iter()
.find(|kw| kw.arg.as_deref() == Some(name))
.map(|kw| &kw.value)
@@ -270,8 +283,9 @@ impl AssetsFinder {
self.assets
.push(ParseAssetsResult { kind, path, access_type, columns: None });
}
Some(Expr::Constant(ExprConstant { value: Constant::Str(value), .. })) => {
let path = parse_asset_syntax(&value, false)
Some(Expr::StringLiteral(value)) => {
let value = value.value.to_str();
let path = parse_asset_syntax(value, false)
.map(|(_, p)| p)
.unwrap_or(std::borrow::Cow::Borrowed(value));
self.assets.push(ParseAssetsResult {
@@ -291,26 +305,24 @@ impl AssetsFinder {
struct Arg(usize, &'static str);
/// Extract a string-literal keyword argument, e.g. `s3="value"` in a call.
fn keyword_str_value(keywords: &[rustpython_ast::Keyword], name: &str) -> Option<String> {
fn keyword_str_value(keywords: &[Keyword], name: &str) -> Option<String> {
keywords
.iter()
.find(|kw| kw.arg.as_deref() == Some(name))
.and_then(|kw| match &kw.value {
Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => Some(s.clone()),
Expr::StringLiteral(s) => Some(s.value.to_str().to_string()),
_ => None,
})
}
/// Extract a string-literal value from a dict literal, e.g. `{"s3": "value"}`.
fn dict_str_value(dict: &rustpython_ast::ExprDict, name: &str) -> Option<String> {
dict.keys
fn dict_str_value(dict: &ExprDict, name: &str) -> Option<String> {
dict.items
.iter()
.zip(dict.values.iter())
.find_map(|(key, value)| match (key.as_ref()?, value) {
(
Expr::Constant(ExprConstant { value: Constant::Str(k), .. }),
Expr::Constant(ExprConstant { value: Constant::Str(v), .. }),
) if k.as_str() == name => Some(v.clone()),
.find_map(|item| match (item.key.as_ref()?, &item.value) {
(Expr::StringLiteral(k), Expr::StringLiteral(v)) if k.value.to_str() == name => {
Some(v.value.to_str().to_string())
}
_ => None,
})
}
@@ -328,12 +340,13 @@ fn dict_str_value(dict: &rustpython_ast::ExprDict, name: &str) -> Option<String>
/// matches the TS object form and the `# on s3:///…` trigger form exactly.
fn s3_object_arg_path(expr: &Expr) -> Option<String> {
let uri = match expr {
Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => {
Expr::StringLiteral(s) => {
let s = s.value.to_str();
match s
.strip_prefix("s3://")
.and_then(|rest| rest.split_once('/'))
{
Some((_, key)) if !key.is_empty() => s.clone(),
Some((_, key)) if !key.is_empty() => s.to_string(),
_ => return None,
}
}
@@ -347,8 +360,9 @@ fn s3_object_arg_path(expr: &Expr) -> Option<String> {
if func_name != "S3Object" {
return None;
}
let key = keyword_str_value(&call.keywords, "s3")?;
let storage = keyword_str_value(&call.keywords, "storage").unwrap_or_default();
let key = keyword_str_value(&call.arguments.keywords, "s3")?;
let storage =
keyword_str_value(&call.arguments.keywords, "storage").unwrap_or_default();
format!("s3://{storage}/{key}")
}
Expr::Dict(dict) => {
@@ -822,4 +836,62 @@ def main():
},])
);
}
#[test]
fn test_py_asset_parser_recent_syntax() {
let input = r#"
import wmill
lazy import polars
type Rows = list[dict]
def main[T](name: str):
try:
with wmill.load_s3_file_reader('s3:///in.csv') as f:
f.read()
except ValueError, TypeError:
pass
greeting = t"hello {name}"
return f"{wmill.get_resource("u/user/pg")["host"]}"
"#;
let s = parse_assets(input).map(|o| o.assets);
assert_eq!(
s.map_err(|e| e.to_string()),
Ok(vec![
ParseAssetsResult {
kind: AssetKind::S3Object,
path: "/in.csv".to_string(),
access_type: Some(R),
columns: None,
},
ParseAssetsResult {
kind: AssetKind::Resource,
path: "u/user/pg".to_string(),
access_type: None,
columns: None,
},
])
);
}
#[test]
fn test_py_asset_parser_schema_qualified_db_without_table() {
let input = r#"
import wmill
def main():
db = wmill.datatable('dt:sch')
db.query("ATTACH 'ducklake://other' AS o")
db.query("SELECT * FROM t")
"#;
let s = parse_assets(input).map(|o| o.assets);
assert_eq!(
s.map_err(|e| e.to_string())
.map(|a| a.into_iter().map(|a| (a.kind, a.path)).collect::<Vec<_>>()),
Ok(vec![
(AssetKind::DataTable, "dt".to_string()),
(AssetKind::DataTable, "dt/sch.t".to_string()),
(AssetKind::Ducklake, "other".to_string()),
])
);
}
}
@@ -23,7 +23,8 @@ tracing.workspace = true
[dependencies]
windmill-parser.workspace = true
rustpython-parser.workspace = true
ruff_python_parser.workspace = true
ruff_python_ast.workspace = true
phf.workspace = true
itertools.workspace = true
serde_json.workspace = true
@@ -13,9 +13,9 @@ use async_recursion::async_recursion;
use itertools::Itertools;
use lazy_static::lazy_static;
#[cfg(not(target_arch = "wasm32"))]
use std::str::FromStr;
#[cfg(not(target_arch = "wasm32"))]
use std::collections::HashMap;
#[cfg(not(target_arch = "wasm32"))]
use std::str::FromStr;
use mapping::{FULL_IMPORTS_MAP, SHORT_IMPORTS_MAP};
#[cfg(not(target_arch = "wasm32"))]
@@ -23,11 +23,8 @@ use regex::Regex;
#[cfg(target_arch = "wasm32")]
use regex_lite::Regex;
use rustpython_parser::{
ast::{Stmt, StmtImport, StmtImportFrom, Suite},
text_size::TextRange,
Parse,
};
use ruff_python_ast::{Stmt, StmtImport, StmtImportFrom};
use ruff_python_parser::parse_module;
#[cfg(not(target_arch = "wasm32"))]
use sqlx::{Pool, Postgres};
#[cfg(not(target_arch = "wasm32"))]
@@ -160,6 +157,7 @@ pub struct ImportPin {
}
pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result<Vec<NImport>> {
let full_code = code;
// Use regex to safely find the main function definition
let mut code = DEF_MAIN_RE
.split(code)
@@ -186,16 +184,22 @@ pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result<Vec<NImp
// This is needed because we've split off the real main function above
let code_with_fake_main = format!("{}\n\ndef main(): pass", code);
let ast = Suite::parse(&code_with_fake_main, "main.py").map_err(|e| {
anyhow::anyhow!("Error parsing code for imports: {}", e.to_string())
})?;
let (code, ast) = match parse_module(&code_with_fake_main) {
Ok(parsed) => (code, parsed.into_suite()),
// The split lands on any line reading like the definition of `main`, one inside a
// string included: what precedes it is then not a module, while the whole file is.
Err(e) => match parse_module(full_code) {
Ok(parsed) => (full_code.to_string(), parsed.into_suite()),
Err(_) => anyhow::bail!("Error parsing code for imports: {}", e.to_string()),
},
};
// Note: We're still using the original code for finding pins,
// as the TextRange values from the parsed AST would be based on code_with_fake_main
// but we want to match against the original code
let find_pin = |range: TextRange, key: String| {
let find_pin = |stmt_end: usize, key: String| {
let hs = code
.chars()
.skip(range.end().to_usize())
.skip(stmt_end)
.take_while(|e| *e != '\n')
.collect::<String>();
@@ -226,9 +230,9 @@ pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result<Vec<NImp
let mut nimports: Vec<NImport> = ast
.into_iter()
.filter_map(|x| match x {
Stmt::Import(StmtImport { names, range }) => names
Stmt::Import(StmtImport { names, range, .. }) => names
.get(0)
.and_then(|al| find_pin(range, al.name.to_string()))
.and_then(|al| find_pin(range.end().to_usize(), al.name.to_string()))
.or(Some(
names
.into_iter()
@@ -239,15 +243,11 @@ pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result<Vec<NImp
.flatten()
.collect::<Vec<NImport>>(),
)),
Stmt::ImportFrom(StmtImportFrom { level: Some(i), module, .. }) if i.to_u32() > 0 => {
Some(process_import(
module.map(|x| x.to_string()),
path,
i.to_usize(),
))
}
Stmt::ImportFrom(StmtImportFrom { level, module, .. }) if level > 0 => Some(
process_import(module.map(|x| x.to_string()), path, level as usize),
),
Stmt::ImportFrom(StmtImportFrom { level: _, module, range, .. }) => find_pin(
range,
range.end().to_usize(),
module.clone().map(|x| x.to_string()).unwrap_or_default(),
)
.or(Some(process_import(module.map(|x| x.to_string()), path, 0))),
@@ -518,8 +518,14 @@ async fn parse_python_imports_inner(
let mut nested = match n {
NImport::Relative(rpath) => {
// First try to get content from temp_script_refs cache if available
let code_from_cache = if let Some(hash) = temp_script_refs.as_ref().and_then(|dt| dt.get(&rpath)) {
tracing::debug!("Found relative import '{}' in temp_script_refs with hash '{}'", rpath, hash);
let code_from_cache = if let Some(hash) =
temp_script_refs.as_ref().and_then(|dt| dt.get(&rpath))
{
tracing::debug!(
"Found relative import '{}' in temp_script_refs with hash '{}'",
rpath,
hash
);
match windmill_common::cache::raw_script_temp::load(hash.clone(), db).await {
Ok(content) => Some(content),
Err(e) => {
@@ -534,19 +540,17 @@ async fn parse_python_imports_inner(
// Use cached content if available, otherwise fall back to deployed script
let code = match code_from_cache {
Some(content) => content,
None => {
sqlx::query_scalar!(
r#"
None => sqlx::query_scalar!(
r#"
SELECT content FROM script WHERE path = $1 AND workspace_id = $2
AND archived = false ORDER BY created_at DESC LIMIT 1
"#,
&rpath,
w_id
)
.fetch_optional(db)
.await?
.unwrap_or_else(|| "".to_string())
}
&rpath,
w_id
)
.fetch_optional(db)
.await?
.unwrap_or_else(|| "".to_string()),
};
if already_visited.contains(&rpath) {
@@ -1,7 +1,9 @@
mod tests {
use sqlx::{Pool, Postgres};
use windmill_parser_py_imports::parse_python_imports;
use windmill_parser_py_imports::{
parse_code_for_imports, parse_python_imports, ImportPin, NImport,
};
#[sqlx::test(fixtures("base"))]
async fn test_parse_python_imports(db: Pool<Postgres>) -> anyhow::Result<()> {
@@ -117,4 +119,87 @@ def main():
Ok(())
}
fn auto(pkg: &str) -> NImport {
NImport::Auto { pkg: pkg.to_string(), key: None }
}
#[test]
fn test_parse_lazy_imports() -> anyhow::Result<()> {
let code = "
lazy import requests
lazy import numpy # pin: numpy==2.1.0
lazy from pandas import DataFrame
lazy from f.shared import helper
lazy = 1
def main():
pass
";
assert_eq!(
parse_code_for_imports(code, "f/foo/bar")?,
vec![
NImport::Pin {
pins: vec![ImportPin {
pkg: "numpy==2.1.0".to_string(),
path: "f/foo/bar".to_string()
}],
key: "numpy".to_string()
},
auto("pandas"),
auto("requests"),
NImport::Relative("f/shared".to_string()),
]
);
Ok(())
}
#[test]
fn test_parse_imports_recent_syntax_before_main() -> anyhow::Result<()> {
let code = r#"
import requests
type Alias = list[int]
NAME = "x"
D = {"a": 1}
L = [[1], [2]]
TEMPLATE = t"hello {NAME}"
PEP701 = f"{D["a"]} {'\n'.join(["x"])}"
PEP798 = [*a for a in L]
try:
import ujson
except ImportError, ValueError:
pass
def main():
pass
"#;
assert_eq!(
parse_code_for_imports(code, "f/foo/bar")?,
vec![auto("requests")]
);
Ok(())
}
#[test]
fn test_parse_imports_main_lookalike_in_docstring() -> anyhow::Result<()> {
for example in ["def main[T](x: T):", "def main(x):"] {
let code = format!(
"\"\"\"Usage example:\n{example}\n return x\n\"\"\"\nimport requests # pin: requests==2.32.0\n\ndef main(value: str):\n return value\n"
);
assert_eq!(
parse_code_for_imports(&code, "f/foo/bar")?,
vec![NImport::Pin {
pins: vec![ImportPin {
pkg: "requests==2.32.0".to_string(),
path: "f/foo/bar".to_string()
}],
key: "requests".to_string()
}],
"{code}"
);
}
Ok(())
}
}
@@ -10,8 +10,8 @@ path = "./src/lib.rs"
[dependencies]
windmill-parser.workspace = true
rustpython-parser.workspace = true
ruff_python_parser.workspace = true
ruff_python_ast.workspace = true
itertools.workspace = true
serde_json.workspace = true
anyhow.workspace = true
rustpython-ast = { version = "0.4.0", features = ["visitor"] }
+206 -103
View File
@@ -13,13 +13,11 @@ use itertools::Itertools;
use serde_json::json;
use windmill_parser::{json_to_typ, Arg, MainArgSignature, ObjectType, Typ};
use rustpython_parser::{
ast::{
Constant, Expr, ExprAttribute, ExprConstant, ExprDict, ExprList, ExprName, Stmt,
StmtAssign, StmtAsyncFunctionDef, StmtClassDef, StmtFunctionDef, Suite,
},
Parse,
use ruff_python_ast::{
Expr, ExprAttribute, ExprDict, ExprList, ExprName, Number, Parameters, Stmt, StmtAssign,
StmtClassDef, StmtFunctionDef,
};
use ruff_python_parser::parse_module;
pub mod pydantic_parser;
@@ -81,22 +79,24 @@ fn should_parse_for_models(code: &str) -> bool {
|| code.contains("import dataclasses")
}
/// What follows `def <name>` or `async def <name>` on a line opening that function's
/// parameters or type parameters.
fn strip_def<'a>(line: &'a str, name: &str) -> Option<&'a str> {
let rest = line
.strip_prefix("async def ")
.or_else(|| line.strip_prefix("def "))?
.strip_prefix(name)?;
(rest.starts_with('(') || rest.starts_with('[')).then_some(rest)
}
fn filter_non_main(code: &str, main_name: &str) -> String {
let def_main = format!("def {}(", main_name);
let async_def_main = format!("async def {}(", main_name);
let mut filtered_code = String::new();
let mut code_iter = code.split("\n");
let mut remaining: String = String::new();
while let Some(line) = code_iter.next() {
if line.starts_with(&async_def_main) {
filtered_code += &async_def_main;
remaining += line.strip_prefix(&async_def_main).unwrap();
remaining += "\n";
remaining += &code_iter.join("\n");
break;
} else if line.starts_with(&def_main) {
filtered_code += &def_main;
remaining += line.strip_prefix(&def_main).unwrap();
if let Some(rest) = strip_def(line, main_name) {
filtered_code += &line[..line.len() - rest.len()];
remaining += rest;
remaining += "\n";
remaining += &code_iter.join("\n");
break;
@@ -106,6 +106,26 @@ fn filter_non_main(code: &str, main_name: &str) -> String {
return String::new();
}
let mut chars = remaining.chars();
// PEP 695 type parameters, `def main[T](x: T)`, sit between the name and the parameters.
if remaining.starts_with('[') {
let mut open_brackets = 0;
for c in chars.by_ref() {
if c == '[' {
open_brackets += 1;
} else if c == ']' {
open_brackets -= 1;
}
filtered_code.push(c);
if open_brackets == 0 {
break;
}
}
}
if chars.next() != Some('(') {
return String::new();
}
filtered_code.push('(');
let mut open_parens = 1;
while let Some(c) = chars.next() {
@@ -183,15 +203,15 @@ fn extract_code_metadata(code: &str, main_name: &str) -> CodeMetadata {
let filtered_code = filter_relevant_statements(code);
let ast = match Suite::parse(&filtered_code, "main.py") {
Ok(ast) => ast,
let ast = match parse_module(&filtered_code) {
Ok(parsed) => parsed.into_suite(),
Err(_) => return CodeMetadata { enums, descriptions },
};
for stmt in ast {
match stmt {
Stmt::ClassDef(StmtClassDef { name, body, bases, .. }) if has_enum => {
let is_enum = bases.iter().any(|base| {
Stmt::ClassDef(class_def) if has_enum => {
let is_enum = class_def.bases().iter().any(|base| {
matches!(base, Expr::Name(ExprName { id, .. })
if id == "Enum" || id == "IntEnum" || id == "StrEnum"
|| id == "Flag" || id == "IntFlag")
@@ -201,17 +221,15 @@ fn extract_code_metadata(code: &str, main_name: &str) -> CodeMetadata {
let mut values = Vec::new();
let mut members = HashMap::new();
let StmtClassDef { name, body, .. } = class_def;
for item in body {
if let Stmt::Assign(StmtAssign { targets, value, .. }) = item {
if let Some(Expr::Name(ExprName { id: target_name, .. })) =
targets.first()
{
if !target_name.starts_with('_') {
if let Expr::Constant(ExprConstant {
value: Constant::Str(val),
..
}) = value.as_ref()
{
if let Expr::StringLiteral(val) = value.as_ref() {
let val = val.value.to_str();
values.push(val.to_string());
members.insert(target_name.to_string(), val.to_string());
}
@@ -225,15 +243,13 @@ fn extract_code_metadata(code: &str, main_name: &str) -> CodeMetadata {
}
}
}
Stmt::FunctionDef(StmtFunctionDef { name: func_name, body, .. }) if has_docstring => {
if &func_name == main_name {
Stmt::FunctionDef(StmtFunctionDef {
name: func_name, body, is_async: false, ..
}) if has_docstring => {
if func_name.as_str() == main_name {
if let Some(Stmt::Expr(expr_stmt)) = body.first() {
if let Expr::Constant(ExprConstant {
value: Constant::Str(docstring),
..
}) = expr_stmt.value.as_ref()
{
descriptions = parse_docstring_args(docstring);
if let Expr::StringLiteral(docstring) = expr_stmt.value.as_ref() {
descriptions = parse_docstring_args(docstring.value.to_str());
}
}
}
@@ -294,6 +310,27 @@ fn parse_docstring_args(docstring: &str) -> HashMap<String, String> {
descriptions
}
/// A line that reads as the definition of `main_name` can also sit inside a string, a docstring
/// showing an example for instance. When several lines do, only the parsed module tells which
/// one is the function.
fn main_params_among_lookalikes(code: &str, main_name: &str) -> Option<Parameters> {
code.split('\n')
.filter(|line| strip_def(line, main_name).is_some())
.nth(1)?;
parse_module(code)
.ok()?
.into_suite()
.into_iter()
.find_map(|x| match x {
Stmt::FunctionDef(StmtFunctionDef { name, parameters, .. })
if name.as_str() == main_name =>
{
Some(*parameters)
}
_ => None,
})
}
/// skip_params is a micro optimization for when we just want to find the main
/// function without parsing all the params.
pub fn parse_python_signature(
@@ -310,41 +347,39 @@ pub fn parse_python_signature(
// - If no models: parse only the filtered main function
let (params, module) = if should_parse_for_models(code) {
// Parse full code once for both Pydantic detection and signature extraction
let ast = Suite::parse(code, "main.py")
.map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?;
let ast = parse_module(code)
.map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?
.into_suite();
// Extract main function from full AST
let params = ast.iter().find_map(|x| match x {
Stmt::FunctionDef(StmtFunctionDef { name, args, .. }) if name == &main_name => {
Some(args.as_ref().clone())
}
Stmt::AsyncFunctionDef(StmtAsyncFunctionDef { name, args, .. })
if name == &main_name =>
Stmt::FunctionDef(StmtFunctionDef { name, parameters, .. })
if name.as_str() == main_name =>
{
Some(args.as_ref().clone())
Some(parameters.as_ref().clone())
}
_ => None,
});
// Keep AST for Pydantic/dataclass detection
(params, Some(ast))
} else if let Some(params) = main_params_among_lookalikes(code, &main_name) {
(Some(params), None)
} else {
// No models detected - parse only the filtered main function
let filtered_code = filter_non_main(code, &main_name);
if filtered_code.is_empty() {
(None, None)
} else {
let ast = Suite::parse(&filtered_code, "main.py")
.map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?;
let ast = parse_module(&filtered_code)
.map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?
.into_suite();
let params = ast.into_iter().find_map(|x| match x {
Stmt::FunctionDef(StmtFunctionDef { name, args, .. }) if &name == &main_name => {
Some(*args)
}
Stmt::AsyncFunctionDef(StmtAsyncFunctionDef { name, args, .. })
if &name == &main_name =>
Stmt::FunctionDef(StmtFunctionDef { name, parameters, .. })
if name.as_str() == main_name =>
{
Some(*args)
Some(*parameters)
}
_ => None,
});
@@ -378,8 +413,6 @@ pub fn parse_python_signature(
if !skip_params && params.is_some() {
let params = params.unwrap();
let def_arg_start = params.args.len() - params.defaults().count();
// Two-pass approach for lazy metadata extraction:
// Pass 1: Parse types without enum info to determine if metadata is needed
// Pass 2: Re-parse unknown types with metadata only if necessary
@@ -392,11 +425,11 @@ pub fn parse_python_signature(
.iter()
.enumerate()
.map(|(i, x)| {
let arg_name = x.as_arg().arg.to_string();
let arg_name = x.parameter.name.to_string();
let (typ, has_default) = x
.as_arg()
.parameter
.annotation
.as_ref()
.as_deref()
.map_or((Typ::Unknown, false), |e| {
parse_expr(e, &empty_enums, module_ref)
});
@@ -423,21 +456,16 @@ pub fn parse_python_signature(
.into_iter()
.map(|(i, arg_name, mut typ, mut has_default)| {
if matches!(typ, Typ::Resource(_)) && !metadata.enums.is_empty() {
if let Some(annotation) = params.args[i].as_arg().annotation.as_ref() {
if let Some(annotation) = params.args[i].parameter.annotation.as_deref() {
(typ, has_default) =
parse_expr(annotation, &metadata.enums, module_ref);
}
}
let default = if i >= def_arg_start {
params
.defaults()
.nth(i - def_arg_start)
.map(|expr| to_value(expr, &metadata.enums))
.flatten()
} else {
None
};
let default = params.args[i]
.default
.as_deref()
.and_then(|expr| to_value(expr, &metadata.enums));
let should_get_type_from_default = match &typ {
Typ::Unknown => true,
@@ -465,10 +493,10 @@ pub fn parse_python_signature(
// debounce accumulation support. Falls back to docstring
// description if no union array pattern is found.
let union_otyp = params.args[i]
.as_arg()
.parameter
.annotation
.as_ref()
.and_then(|ann| detect_py_union_array_otyp(ann.as_ref()));
.as_deref()
.and_then(detect_py_union_array_otyp);
Arg {
otyp: union_otyp.or_else(|| {
@@ -509,12 +537,8 @@ pub fn parse_python_signature(
}
}
fn parse_expr(
e: &Box<Expr>,
enums: &HashMap<String, EnumInfo>,
module: Option<&[Stmt]>,
) -> (Typ, bool) {
match e.as_ref() {
fn parse_expr(e: &Expr, enums: &HashMap<String, EnumInfo>, module: Option<&[Stmt]>) -> (Typ, bool) {
match e {
Expr::Name(ExprName { id, .. }) => (parse_typ(id.as_ref(), enums, module), false),
Expr::Attribute(x) => {
if let Some(name) = x.value.as_name_expr() {
@@ -531,12 +555,9 @@ fn parse_expr(
}
}
Expr::BinOp(x) => {
if matches!(
x.right.as_ref(),
Expr::Constant(ExprConstant { value: Constant::None, .. })
) {
if x.right.is_none_literal_expr() {
(parse_expr(&x.left, enums, module).0, true)
} else if detect_py_union_array_otyp(e.as_ref()).is_some() {
} else if detect_py_union_array_otyp(e).is_some() {
// T | list[T] — parsed type is Unknown; otyp is set separately
(Typ::Unknown, false)
} else {
@@ -552,7 +573,7 @@ fn parse_expr(
.elts
.iter()
.map(|x| match x {
Expr::Constant(c) => c.value.as_str().map(|x| x.to_string()),
Expr::StringLiteral(s) => Some(s.value.to_str().to_string()),
_ => None,
})
.filter_map(|x| x)
@@ -625,15 +646,26 @@ fn map_resource_name(x: &str) -> String {
}
}
fn to_value<R>(et: &Expr<R>, enums: &HashMap<String, EnumInfo>) -> Option<serde_json::Value> {
fn to_value(et: &Expr, enums: &HashMap<String, EnumInfo>) -> Option<serde_json::Value> {
match et {
Expr::Constant(ExprConstant { value, .. }) => Some(constant_to_value(value)),
Expr::Dict(ExprDict { keys, values, .. }) => {
let v = keys
.into_iter()
.zip(values)
.map(|(k, v)| {
let key = k
Expr::NoneLiteral(_) => Some(json!(null)),
Expr::BooleanLiteral(b) => Some(json!(b.value)),
Expr::StringLiteral(s) => Some(json!(s.value.to_str())),
Expr::BytesLiteral(b) => Some(json!(b.value.bytes().collect::<Vec<u8>>())),
Expr::NumberLiteral(n) => Some(match &n.value {
Number::Int(i) => {
serde_json::from_str(&i.to_string()).unwrap_or(json!("invalid number"))
}
Number::Float(f) => json!(f),
Number::Complex { real, imag } => json!([real, imag]),
}),
Expr::EllipsisLiteral(_) => Some(json!("...")),
Expr::Dict(ExprDict { items, .. }) => {
let v = items
.iter()
.map(|item| {
let key = item
.key
.as_ref()
.map(|e| to_value(e, enums))
.flatten()
@@ -642,7 +674,7 @@ fn to_value<R>(et: &Expr<R>, enums: &HashMap<String, EnumInfo>) -> Option<serde_
_ => None,
})
.unwrap_or_else(|| "no_key".to_string());
(key, to_value(&v, enums))
(key, to_value(&item.value, enums))
})
.collect::<HashMap<String, _>>();
Some(json!(v))
@@ -672,20 +704,6 @@ fn to_value<R>(et: &Expr<R>, enums: &HashMap<String, EnumInfo>) -> Option<serde_
}
}
fn constant_to_value(c: &Constant) -> serde_json::Value {
match c {
Constant::None => json!(null),
Constant::Bool(b) => json!(b),
Constant::Str(s) => json!(s),
Constant::Bytes(b) => json!(b),
Constant::Int(i) => serde_json::from_str(&i.to_string()).unwrap_or(json!("invalid number")),
Constant::Tuple(t) => json!(t.iter().map(constant_to_value).collect::<Vec<_>>()),
Constant::Float(f) => json!(f),
Constant::Complex { real, imag } => json!([real, imag]),
Constant::Ellipsis => json!("..."),
}
}
#[cfg(test)]
mod tests {
@@ -1323,4 +1341,89 @@ def main(color: Color = Color.RED):
);
Ok(())
}
#[test]
fn test_parse_python_sig_type_params() -> anyhow::Result<()> {
for code in [
"def main[T](x: str, y: int = 3):\n return x\n",
"async def main[T: (int, str), *Ts](x: str, y: int = 3):\n return x\n",
// parsed whole rather than pre-filtered down to `main`
"from dataclasses import dataclass\n\ndef main[T](x: str, y: int = 3):\n return x\n",
] {
let result = parse_python_signature(code, None, false)?;
assert_eq!(result.auto_kind, None, "{code}");
assert_eq!(
result
.args
.iter()
.map(|a| (a.name.as_str(), &a.typ, &a.default))
.collect::<Vec<_>>(),
vec![
("x", &Typ::Str(None), &None),
("y", &Typ::Int, &Some(json!(3)))
],
"{code}"
);
}
Ok(())
}
// A script with a model class is parsed whole, so syntax anywhere in it reaches the parser.
#[test]
fn test_parse_python_sig_recent_syntax() -> anyhow::Result<()> {
let code = r#"
from dataclasses import dataclass
lazy import requests
lazy from pandas import DataFrame
type Alias = list[int]
@dataclass
class Point:
x: int
try:
import a
except ImportError, ValueError:
pass
D = {"a": 1}
L = [[1], [2]]
PEP701 = f"{D["a"]} {'\n'.join(["x"])}"
PEP798 = [*a for a in L]
def main(p: Point, name: str = "world"):
return t"hello {name}"
"#;
let result = parse_python_signature(code, None, false)?;
assert_eq!(result.args.len(), 2);
assert_eq!(
result.args[0].typ,
Typ::Object(ObjectType::new(
Some("Point".to_string()),
Some(vec![windmill_parser::ObjectProperty {
key: "x".to_string(),
typ: Box::new(Typ::Int)
}])
))
);
assert_eq!(result.args[1].default, Some(json!("world")));
Ok(())
}
#[test]
fn test_parse_python_sig_main_lookalike_in_docstring() -> anyhow::Result<()> {
for example in ["def main[T](x: T):", "def main(x: int, y: int):"] {
let code = format!(
"\"\"\"Usage example:\n{example}\n return x\n\"\"\"\n\ndef main(value: str):\n return value\n"
);
let result = parse_python_signature(&code, None, false)?;
assert_eq!(
result.args.iter().map(|a| &a.name).collect::<Vec<_>>(),
vec!["value"],
"{code}"
);
}
Ok(())
}
}
@@ -11,8 +11,8 @@
//! This module provides functionality to detect and parse Pydantic models and Python
//! dataclasses from Python AST, enabling automatic UI generation for complex data structures.
use rustpython_parser::ast::{
Constant, Expr, ExprAttribute, ExprCall, ExprConstant, ExprName, ExprTuple, Stmt, StmtAnnAssign,
use ruff_python_ast::{
Decorator, Expr, ExprAttribute, ExprCall, ExprName, ExprTuple, Stmt, StmtAnnAssign,
};
use std::collections::HashSet;
use windmill_parser::{ObjectProperty, ObjectType, Typ};
@@ -64,7 +64,7 @@ fn detect_model_type_impl(
// Mark as being visited
visited.insert(class_name.to_string());
let result = if is_pydantic_base(&class_def.bases) {
let result = if is_pydantic_base(class_def.bases()) {
// Pydantic BaseModel
parse_model_fields(&class_def.body, class_def.name.as_str(), module, visited)
} else if has_dataclass_decorator(&class_def.decorator_list)
@@ -115,9 +115,9 @@ fn is_pydantic_base(bases: &[Expr]) -> bool {
}
/// Checks if a class has @dataclass decorator (standard library)
fn has_dataclass_decorator(decorators: &[Expr]) -> bool {
fn has_dataclass_decorator(decorators: &[Decorator]) -> bool {
for decorator in decorators {
match decorator {
match &decorator.expression {
// Match: @dataclass
Expr::Name(ExprName { id, .. }) if id.as_str() == "dataclass" => {
return true;
@@ -160,9 +160,9 @@ fn has_dataclass_decorator(decorators: &[Expr]) -> bool {
}
/// Checks if a class has @pydantic.dataclasses.dataclass decorator (Pydantic v2)
fn has_pydantic_dataclass_decorator(decorators: &[Expr]) -> bool {
fn has_pydantic_dataclass_decorator(decorators: &[Decorator]) -> bool {
for decorator in decorators {
match decorator {
match &decorator.expression {
// Match: @pydantic.dataclasses.dataclass
Expr::Attribute(ExprAttribute { attr, value, .. }) if attr.as_str() == "dataclass" => {
if let Expr::Attribute(ExprAttribute {
@@ -350,10 +350,10 @@ fn extract_field_type(
// String annotations: "ForwardRef" (forward references)
// Not fully supported - return Unknown with warning
Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => {
Expr::StringLiteral(s) => {
eprintln!(
"Forward references like \"{forward_ref}\" are not yet supported, treating as Unknown",
forward_ref = s,
forward_ref = s.value.to_str(),
);
Typ::Unknown
}
@@ -20,13 +20,16 @@ pub fn parse_wmill_sdk_sql_assets(
Ok(mut sql_assets) => {
if let Some(schema) = schema {
for asset in &mut sql_assets.assets {
if asset.kind == kind && asset.path.starts_with(asset_name) {
asset.path = format!(
"{}/{}.{}",
asset_name,
schema,
&asset.path[asset_name.len() + 1..]
);
if asset.kind != kind {
continue;
}
// A statement can name the database alone, with no table to qualify.
let table = asset
.path
.strip_prefix(asset_name)
.and_then(|rest| rest.strip_prefix('/'));
if let Some(table) = table {
asset.path = format!("{asset_name}/{schema}.{table}");
}
}
}
@@ -9,8 +9,9 @@ name = "windmill_parser_wac"
path = "./src/lib.rs"
[dependencies]
rustpython-parser.workspace = true
rustpython-ast = { version = "0.4.0", features = ["visitor"] }
ruff_python_parser.workspace = true
ruff_python_ast.workspace = true
ruff_text_size.workspace = true
swc_common.workspace = true
swc_ecma_parser.workspace = true
swc_ecma_ast.workspace = true
+135 -190
View File
@@ -1,12 +1,12 @@
use std::collections::HashMap;
use rustpython_parser::{
ast::{
Expr, ExprAwait, ExprCall, ExprName, Stmt, StmtExpr, StmtFor, StmtIf, StmtReturn, StmtTry,
StmtTryStar, StmtWhile,
},
Parse,
use ruff_python_ast::{
CmpOp, ElifElseClause, ExceptHandler, Expr, ExprAttribute, ExprAwait, ExprCall, ExprName,
Number, Parameters, Stmt, StmtExpr, StmtFor, StmtFunctionDef, StmtIf, StmtReturn, StmtTry,
StmtWhile,
};
use ruff_python_parser::parse_module;
use ruff_text_size::{Ranged, TextSize};
use crate::dag::{DagEdge, DagNode, DagNodeType, Param, WorkflowDag};
use crate::validation::{self, CompileError};
@@ -42,9 +42,9 @@ type TaskFunctions = HashMap<String, Option<String>>;
fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions {
let mut tasks = HashMap::new();
for stmt in stmts {
if let Stmt::AsyncFunctionDef(func) = stmt {
if let Stmt::FunctionDef(func @ StmtFunctionDef { is_async: true, .. }) = stmt {
for dec in &func.decorator_list {
match dec {
match &dec.expression {
// @task (bare decorator)
Expr::Name(ExprName { id, .. }) if id.as_str() == "task" => {
tasks.insert(func.name.to_string(), None);
@@ -68,13 +68,9 @@ fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions {
if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() {
if id.as_str() == "task_script" || id.as_str() == "task_flow" {
// Extract the path from the first positional argument
let path = call.args.first().and_then(|arg| {
if let Expr::Constant(c) = arg {
if let rustpython_parser::ast::Constant::Str(s) = &c.value {
return Some(s.to_string());
}
}
None
let path = call.arguments.args.first().and_then(|arg| match arg {
Expr::StringLiteral(s) => Some(s.value.to_str().to_string()),
_ => None,
});
// Extract variable name from target
if let Some(Expr::Name(ExprName { id: var_name, .. })) =
@@ -92,13 +88,11 @@ fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions {
/// Extract the `path=` keyword argument from a `@task(path="...")` call.
fn extract_task_path_kwarg(call: &ExprCall) -> Option<String> {
for kw in &call.keywords {
for kw in &call.arguments.keywords {
if let Some(ref arg) = kw.arg {
if arg.as_str() == "path" {
if let Expr::Constant(c) = &kw.value {
if let rustpython_parser::ast::Constant::Str(s) = &c.value {
return Some(s.to_string());
}
if let Expr::StringLiteral(s) = &kw.value {
return Some(s.value.to_str().to_string());
}
}
}
@@ -150,10 +144,9 @@ impl WacWalker {
fn line_of_expr(&self, expr: &Expr) -> usize {
let offset = match expr {
Expr::Call(c) => c.range.start().to_usize(),
Expr::Await(a) => a.range.start().to_usize(),
Expr::Attribute(a) => a.range.start().to_usize(),
Expr::Name(n) => n.range.start().to_usize(),
Expr::Call(_) | Expr::Await(_) | Expr::Attribute(_) | Expr::Name(_) => {
expr.start().to_usize()
}
_ => 0,
};
self.line_index.line_of(offset)
@@ -161,17 +154,15 @@ impl WacWalker {
fn line_of_stmt(&self, stmt: &Stmt) -> usize {
let offset = match stmt {
Stmt::If(s) => s.range.start().to_usize(),
Stmt::For(s) => s.range.start().to_usize(),
Stmt::While(s) => s.range.start().to_usize(),
Stmt::Return(s) => s.range.start().to_usize(),
Stmt::Expr(s) => s.range.start().to_usize(),
Stmt::Try(s) => s.range.start().to_usize(),
Stmt::TryStar(s) => s.range.start().to_usize(),
Stmt::Assign(s) => s.range.start().to_usize(),
Stmt::AnnAssign(s) => s.range.start().to_usize(),
Stmt::FunctionDef(s) => s.range.start().to_usize(),
Stmt::AsyncFunctionDef(s) => s.range.start().to_usize(),
Stmt::If(_)
| Stmt::For(_)
| Stmt::While(_)
| Stmt::Return(_)
| Stmt::Expr(_)
| Stmt::Try(_)
| Stmt::Assign(_)
| Stmt::AnnAssign(_)
| Stmt::FunctionDef(_) => stmt.start().to_usize(),
_ => 0,
};
self.line_index.line_of(offset)
@@ -190,9 +181,7 @@ impl WacWalker {
/// Check if an expression is `asyncio.gather(...)` call
fn is_asyncio_gather_call(expr: &Expr) -> bool {
if let Expr::Call(call) = expr {
if let Expr::Attribute(rustpython_parser::ast::ExprAttribute { value, attr, .. }) =
call.func.as_ref()
{
if let Expr::Attribute(ExprAttribute { value, attr, .. }) = call.func.as_ref() {
if attr.as_str() == "gather" {
if let Expr::Name(ExprName { id, .. }) = value.as_ref() {
return id.as_str() == "asyncio";
@@ -222,20 +211,20 @@ impl WacWalker {
fn expr_to_source(expr: &Expr) -> String {
match expr {
Expr::Compare(c) => {
let left = Self::expr_to_source(&c.left);
if let Some(comparator) = c.comparators.first() {
let left = Self::expr_to_source(&c.operands[0]);
if let Some(comparator) = c.comparators().first() {
let right = Self::expr_to_source(comparator);
let op = match c.ops.first() {
Some(rustpython_parser::ast::CmpOp::Gt) => ">",
Some(rustpython_parser::ast::CmpOp::Lt) => "<",
Some(rustpython_parser::ast::CmpOp::GtE) => ">=",
Some(rustpython_parser::ast::CmpOp::LtE) => "<=",
Some(rustpython_parser::ast::CmpOp::Eq) => "==",
Some(rustpython_parser::ast::CmpOp::NotEq) => "!=",
Some(rustpython_parser::ast::CmpOp::In) => "in",
Some(rustpython_parser::ast::CmpOp::NotIn) => "not in",
Some(rustpython_parser::ast::CmpOp::Is) => "is",
Some(rustpython_parser::ast::CmpOp::IsNot) => "is not",
Some(CmpOp::Gt) => ">",
Some(CmpOp::Lt) => "<",
Some(CmpOp::GtE) => ">=",
Some(CmpOp::LtE) => "<=",
Some(CmpOp::Eq) => "==",
Some(CmpOp::NotEq) => "!=",
Some(CmpOp::In) => "in",
Some(CmpOp::NotIn) => "not in",
Some(CmpOp::Is) => "is",
Some(CmpOp::IsNot) => "is not",
None => "?",
};
format!("{left} {op} {right}")
@@ -253,14 +242,14 @@ impl WacWalker {
format!("{value}.{}", a.attr)
}
Expr::Name(n) => n.id.to_string(),
Expr::Constant(c) => match &c.value {
rustpython_parser::ast::Constant::Str(s) => format!("\"{s}\""),
rustpython_parser::ast::Constant::Int(i) => i.to_string(),
rustpython_parser::ast::Constant::Float(f) => f.to_string(),
rustpython_parser::ast::Constant::Bool(b) => b.to_string(),
rustpython_parser::ast::Constant::None => "None".to_string(),
_ => "...".to_string(),
Expr::StringLiteral(s) => format!("\"{}\"", s.value.to_str()),
Expr::NumberLiteral(n) => match &n.value {
Number::Int(i) => i.to_string(),
Number::Float(f) => f.to_string(),
Number::Complex { .. } => "...".to_string(),
},
Expr::BooleanLiteral(b) => b.value.to_string(),
Expr::NoneLiteral(_) => "None".to_string(),
_ => "...".to_string(),
}
}
@@ -279,8 +268,16 @@ impl WacWalker {
match stmt {
Stmt::Expr(StmtExpr { value, .. }) => self.expr_contains_step(value),
Stmt::Assign(a) => self.expr_contains_step(&a.value),
Stmt::If(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse),
Stmt::For(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse),
Stmt::If(s) => {
self.body_contains_step(&s.body)
|| s.elif_else_clauses
.iter()
.any(|clause| self.body_contains_step(&clause.body))
}
// `async for` is not modelled: its body is skipped like any unsupported statement
Stmt::For(s) if !s.is_async => {
self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse)
}
Stmt::While(s) => {
self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse)
}
@@ -289,19 +286,7 @@ impl WacWalker {
|| self.body_contains_step(&s.orelse)
|| self.body_contains_step(&s.finalbody)
|| s.handlers.iter().any(|h| match h {
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
self.body_contains_step(&eh.body)
}
})
}
Stmt::TryStar(s) => {
self.body_contains_step(&s.body)
|| self.body_contains_step(&s.orelse)
|| self.body_contains_step(&s.finalbody)
|| s.handlers.iter().any(|h| match h {
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
self.body_contains_step(&eh.body)
}
ExceptHandler::ExceptHandler(eh) => self.body_contains_step(&eh.body),
})
}
Stmt::Return(_) => false,
@@ -319,11 +304,12 @@ impl WacWalker {
match expr {
Expr::Await(ExprAwait { value, .. }) => self.expr_contains_step(value),
Expr::Call(call) => {
if self.is_task_fn_call(&Expr::Call(call.clone())) {
return true;
}
if Self::is_asyncio_gather_call(&Expr::Call(call.clone())) {
return call.args.iter().any(|a| self.expr_contains_step(a));
if Self::is_asyncio_gather_call(expr) {
return call
.arguments
.args
.iter()
.any(|a| self.expr_contains_step(a));
}
false
}
@@ -370,12 +356,11 @@ impl WacWalker {
Stmt::Expr(StmtExpr { value, .. }) => self.walk_expr_stmt(value),
Stmt::Assign(a) => self.walk_expr_stmt(&a.value),
Stmt::If(if_stmt) => self.walk_if(if_stmt),
Stmt::For(for_stmt) => self.walk_for(for_stmt),
Stmt::For(for_stmt) if !for_stmt.is_async => self.walk_for(for_stmt),
Stmt::While(while_stmt) => self.walk_while(while_stmt),
Stmt::Try(try_stmt) => self.walk_try(try_stmt),
Stmt::TryStar(try_stmt) => self.walk_try_star(try_stmt),
Stmt::Return(ret) => self.walk_return(ret),
Stmt::FunctionDef(_) | Stmt::AsyncFunctionDef(_) => {
Stmt::FunctionDef(_) => {
if self.stmt_contains_step(stmt) {
self.errors.push(validation::error_step_in_nested_function(
self.line_of_stmt(stmt),
@@ -392,7 +377,7 @@ impl WacWalker {
if let Expr::Await(ExprAwait { value, .. }) = expr {
// await task_fn(...)
if let Expr::Call(call) = value.as_ref() {
if self.is_task_fn_call(&Expr::Call(call.clone())) {
if self.is_task_fn_call(value) {
return self.emit_step(call, expr);
}
// Check for SDK-level calls: step(), sleep(), wait_for_approval()
@@ -430,15 +415,12 @@ impl WacWalker {
"step" => {
// step("name", fn) — extract the name from the first string argument
let name = call
.arguments
.args
.first()
.and_then(|arg| {
if let Expr::Constant(c) = arg {
if let rustpython_parser::ast::Constant::Str(s) = &c.value {
return Some(s.to_string());
}
}
None
.and_then(|arg| match arg {
Expr::StringLiteral(s) => Some(s.value.to_str().to_string()),
_ => None,
})
.unwrap_or_else(|| "step".to_string());
let id = self.next_id();
@@ -452,6 +434,7 @@ impl WacWalker {
}
"sleep" => {
let seconds = call
.arguments
.args
.first()
.map(|arg| Self::expr_to_source(arg))
@@ -517,10 +500,10 @@ impl WacWalker {
});
let mut step_ids = Vec::new();
for arg in &gather_call.args {
for arg in &gather_call.arguments.args {
// Each arg should be task_fn(...)
if let Expr::Call(call) = arg {
if self.is_task_fn_call(&Expr::Call(call.clone())) {
if self.is_task_fn_call(arg) {
let (name, script) = self
.extract_step_info_from_task_call(call)
.unwrap_or(("unknown".into(), "unknown".into()));
@@ -553,15 +536,33 @@ impl WacWalker {
}
fn walk_if(&mut self, if_stmt: &StmtIf) -> Option<(String, String)> {
let has_steps_in_body = self.body_contains_step(&if_stmt.body);
let has_steps_in_else = self.body_contains_step(&if_stmt.orelse);
self.walk_if_chain(
if_stmt.start(),
&if_stmt.test,
&if_stmt.body,
&if_stmt.elif_else_clauses,
)
}
/// An `elif` is laid out as an `if` nested alone in the `else` of the clause before it.
fn walk_if_chain(
&mut self,
start: TextSize,
test: &Expr,
body: &[Stmt],
clauses: &[ElifElseClause],
) -> Option<(String, String)> {
let has_steps_in_body = self.body_contains_step(body);
let has_steps_in_else = clauses
.iter()
.any(|clause| self.body_contains_step(&clause.body));
if !has_steps_in_body && !has_steps_in_else {
return None;
}
let line = self.line_index.line_of(if_stmt.range.start().to_usize());
let condition_source = Self::expr_to_source(&if_stmt.test);
let line = self.line_index.line_of(start.to_usize());
let condition_source = Self::expr_to_source(test);
let branch_id = self.next_id();
let branch_node_id = self.add_node(DagNode {
@@ -573,15 +574,19 @@ impl WacWalker {
let mut last_ids = Vec::new();
if let Some((true_first, true_last)) = self.walk_body(&if_stmt.body) {
if let Some((true_first, true_last)) = self.walk_body(body) {
self.add_edge(&branch_node_id, &true_first, Some("true".to_string()));
last_ids.push(true_last);
} else {
last_ids.push(branch_node_id.clone());
}
if !if_stmt.orelse.is_empty() {
if let Some((else_first, else_last)) = self.walk_body(&if_stmt.orelse) {
if let Some((clause, clauses)) = clauses.split_first() {
let orelse = match &clause.test {
Some(test) => self.walk_if_chain(clause.start(), test, &clause.body, clauses),
None => self.walk_body(&clause.body),
};
if let Some((else_first, else_last)) = orelse {
self.add_edge(&branch_node_id, &else_first, Some("false".to_string()));
last_ids.push(else_last);
} else {
@@ -611,7 +616,7 @@ impl WacWalker {
return None;
}
let line = self.line_index.line_of(for_stmt.range.start().to_usize());
let line = self.line_index.line_of(for_stmt.start().to_usize());
let iter_source = Self::expr_to_source(&for_stmt.iter);
let start_id = self.next_id();
@@ -644,7 +649,7 @@ impl WacWalker {
return None;
}
let line = self.line_index.line_of(while_stmt.range.start().to_usize());
let line = self.line_index.line_of(while_stmt.start().to_usize());
let condition = Self::expr_to_source(&while_stmt.test);
let start_id = self.next_id();
@@ -677,39 +682,14 @@ impl WacWalker {
|| self.body_contains_step(&try_stmt.orelse)
|| self.body_contains_step(&try_stmt.finalbody)
|| try_stmt.handlers.iter().any(|h| match h {
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
self.body_contains_step(&eh.body)
}
ExceptHandler::ExceptHandler(eh) => self.body_contains_step(&eh.body),
});
if !has_steps {
return None;
}
let line = self.line_index.line_of(try_stmt.range.start().to_usize());
self.emit_try_catch_branch(
&try_stmt.body,
&try_stmt.handlers,
&try_stmt.finalbody,
line,
)
}
fn walk_try_star(&mut self, try_stmt: &StmtTryStar) -> Option<(String, String)> {
let has_steps = self.body_contains_step(&try_stmt.body)
|| self.body_contains_step(&try_stmt.orelse)
|| self.body_contains_step(&try_stmt.finalbody)
|| try_stmt.handlers.iter().any(|h| match h {
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
self.body_contains_step(&eh.body)
}
});
if !has_steps {
return None;
}
let line = self.line_index.line_of(try_stmt.range.start().to_usize());
let line = self.line_index.line_of(try_stmt.start().to_usize());
self.emit_try_catch_branch(
&try_stmt.body,
&try_stmt.handlers,
@@ -721,7 +701,7 @@ impl WacWalker {
fn emit_try_catch_branch(
&mut self,
try_body: &[Stmt],
handlers: &[rustpython_parser::ast::ExceptHandler],
handlers: &[ExceptHandler],
finally_body: &[Stmt],
line: usize,
) -> Option<(String, String)> {
@@ -746,7 +726,7 @@ impl WacWalker {
// Except handlers
for handler in handlers {
match handler {
rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => {
ExceptHandler::ExceptHandler(eh) => {
if let Some((catch_first, catch_last)) = self.walk_body(&eh.body) {
self.add_edge(&branch_node_id, &catch_first, Some("except".to_string()));
last_ids.push(catch_last);
@@ -783,7 +763,7 @@ impl WacWalker {
}
fn walk_return(&mut self, ret: &StmtReturn) -> Option<(String, String)> {
let line = self.line_index.line_of(ret.range.start().to_usize());
let line = self.line_index.line_of(ret.start().to_usize());
let id = self.next_id();
let node_id = self.add_node(DagNode {
id: id.clone(),
@@ -796,12 +776,12 @@ impl WacWalker {
}
/// Extract workflow function parameters (no longer skips ctx)
fn extract_params(args: &rustpython_parser::ast::Arguments) -> Vec<Param> {
fn extract_params(args: &Parameters) -> Vec<Param> {
let mut params = Vec::new();
for arg_with_default in args.args.iter().chain(args.posonlyargs.iter()) {
let name = arg_with_default.def.arg.to_string();
let name = arg_with_default.parameter.name.to_string();
let typ = arg_with_default
.def
.parameter
.annotation
.as_ref()
.map(|ann| WacWalker::expr_to_source(ann));
@@ -811,59 +791,34 @@ fn extract_params(args: &rustpython_parser::ast::Arguments) -> Vec<Param> {
}
pub fn parse_python_workflow(code: &str) -> Result<WorkflowDag, Vec<CompileError>> {
let ast = rustpython_parser::ast::Suite::parse(code, "<workflow>")
.map_err(|e| vec![CompileError { message: format!("Parse error: {e}"), line: 0 }])?;
let ast = parse_module(code)
.map_err(|e| vec![CompileError { message: format!("Parse error: {e}"), line: 0 }])?
.into_suite();
// First pass: collect @task functions
let task_functions = collect_task_functions(&ast);
// Find the @workflow async def
let workflow_fn = ast.iter().find_map(|stmt| {
if let Stmt::AsyncFunctionDef(func) = stmt {
let has_workflow_decorator = func.decorator_list.iter().any(|dec| {
if let Expr::Name(ExprName { id, .. }) = dec {
id.as_str() == "workflow"
} else {
false
}
});
if has_workflow_decorator {
return Some(func);
let workflow_fns = || {
ast.iter().filter_map(|stmt| match stmt {
Stmt::FunctionDef(func)
if func.decorator_list.iter().any(|dec| {
matches!(&dec.expression, Expr::Name(ExprName { id, .. }) if id.as_str() == "workflow")
}) =>
{
Some(func)
}
}
// Also check non-async for error reporting
if let Stmt::FunctionDef(func) = stmt {
let has_workflow_decorator = func.decorator_list.iter().any(|dec| {
if let Expr::Name(ExprName { id, .. }) = dec {
id.as_str() == "workflow"
} else {
false
}
});
if has_workflow_decorator {
return None; // Will be reported as not-async below
}
}
None
});
_ => None,
})
};
// Check for non-async workflow function
let non_async_workflow = ast.iter().find_map(|stmt| {
if let Stmt::FunctionDef(func) = stmt {
let has_workflow_decorator = func.decorator_list.iter().any(|dec| {
if let Expr::Name(ExprName { id, .. }) = dec {
id.as_str() == "workflow"
} else {
false
}
});
if has_workflow_decorator {
let line_index = LineIndex::new(code);
return Some(line_index.line_of(func.range.start().to_usize()));
}
}
None
});
// Find the @workflow async def
let workflow_fn = workflow_fns().find(|func| func.is_async);
// Check for non-async workflow function. The statement starts at its first decorator,
// the name sits on the `def` line the error points at.
let non_async_workflow = workflow_fns()
.find(|func| !func.is_async)
.map(|func| LineIndex::new(code).line_of(func.name.start().to_usize()));
if let Some(line) = non_async_workflow {
if workflow_fn.is_none() {
@@ -875,7 +830,7 @@ pub fn parse_python_workflow(code: &str) -> Result<WorkflowDag, Vec<CompileError
vec![CompileError { message: "No @workflow async function found.".to_string(), line: 0 }]
})?;
let params = extract_params(&workflow_fn.args);
let params = extract_params(&workflow_fn.parameters);
let source_hash = compute_source_hash(code);
let mut walker = WacWalker::new(code, task_functions);
@@ -894,13 +849,3 @@ fn compute_source_hash(code: &str) -> String {
hasher.update(code.as_bytes());
format!("{:x}", hasher.finalize())
}
trait ToUsize {
fn to_usize(self) -> usize;
}
impl ToUsize for rustpython_parser::text_size::TextSize {
fn to_usize(self) -> usize {
u32::from(self) as usize
}
}
@@ -363,3 +363,77 @@ async def main(x: str):
));
assert!(matches!(dag.nodes[5].node_type, DagNodeType::Return));
}
#[test]
fn test_elif_chain_nests_branches() {
let code = r#"
from wmill import workflow, task
@task
async def a(): ...
@task
async def b(): ...
@task
async def c(): ...
@workflow
async def main(x: int):
if x > 2:
await a()
elif x == 2:
await b()
else:
await c()
"#;
let dag = parse_python_workflow(code).expect("should parse");
let branches: Vec<(&str, usize)> = dag
.nodes
.iter()
.filter_map(|n| match &n.node_type {
DagNodeType::Branch { condition_source } => Some((condition_source.as_str(), n.line)),
_ => None,
})
.collect();
assert_eq!(branches, vec![("x > 2", 13), ("x == 2", 15)]);
// The `elif` branch hangs off the `false` edge of the `if` before it
let (outer, inner) = (&dag.nodes[0].id, &dag.nodes[2].id);
assert!(dag
.edges
.iter()
.any(|e| &e.from == outer && &e.to == inner && e.label.as_deref() == Some("false")));
assert_eq!(
dag.nodes
.iter()
.filter(|n| matches!(n.node_type, DagNodeType::Merge))
.count(),
2
);
}
#[test]
fn test_workflow_with_recent_syntax() {
let code = r#"
from wmill import workflow, task
lazy import polars
type Rows = list[dict]
@task
async def load[T](rows: T): ...
@workflow
async def main(name: str):
try:
await load(t"hello {name}")
except ValueError, TypeError:
pass
"#;
let dag = parse_python_workflow(code).expect("should parse");
assert!(dag
.nodes
.iter()
.any(|n| matches!(&n.node_type, DagNodeType::Step { name, .. } if name == "load")));
}
+332 -250
View File
@@ -254,6 +254,35 @@ version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
[[package]]
name = "attribute-derive"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2f9ef7a68684657ce9da341c934c73b63df7a3585a61e366d84c84ed5d3d167"
dependencies = [
"attribute-derive-macro",
"manyhow",
"proc-macro2",
"quote",
"syn 3.0.7",
]
[[package]]
name = "attribute-derive-macro"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "300d426e5e73549125bb8e4649bdd5508f82c94ddf898fb0517246e71dfb8e98"
dependencies = [
"collection_literals",
"interpolator",
"manyhow",
"proc-macro-utils",
"proc-macro2",
"quote",
"quote-use",
"syn 3.0.7",
]
[[package]]
name = "autocfg"
version = "1.5.0"
@@ -428,7 +457,7 @@ dependencies = [
"proc-macro2",
"quote",
"regex",
"rustc-hash 2.1.1",
"rustc-hash",
"shlex",
"syn 2.0.117",
]
@@ -620,6 +649,15 @@ version = "1.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e93abca9e28e0a1b9877922aacb20576e05d4679ffa78c3d6dc22a26a216659"
[[package]]
name = "castaway"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a"
dependencies = [
"rustversion",
]
[[package]]
name = "cc"
version = "1.2.56"
@@ -670,6 +708,18 @@ dependencies = [
"rand_core 0.10.1",
]
[[package]]
name = "char_str"
version = "0.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c9cba78a5abb515610e10af297615434b019755b5d7be14e559ff5340dff50c4"
dependencies = [
"castaway",
"get-size2",
"itoa",
"ryu",
]
[[package]]
name = "chrono"
version = "0.4.44"
@@ -779,6 +829,12 @@ version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a"
[[package]]
name = "collection_literals"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2550f75b8cfac212855f6b1885455df8eaee8fe8e246b647d69146142e016084"
[[package]]
name = "colorchoice"
version = "1.0.4"
@@ -795,6 +851,19 @@ dependencies = [
"memchr",
]
[[package]]
name = "compact_str"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "79fcda08c33bb58b97008b2cdada6622500e949e060f5913361763121abd2416"
dependencies = [
"castaway",
"cfg-if",
"itoa",
"static_assertions",
"zmij",
]
[[package]]
name = "compression-codecs"
version = "0.4.37"
@@ -1019,12 +1088,6 @@ dependencies = [
"winapi",
]
[[package]]
name = "crunchy"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
[[package]]
name = "crypto-common"
version = "0.1.7"
@@ -1093,27 +1156,6 @@ dependencies = [
"powerfmt",
]
[[package]]
name = "derive_more"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05"
dependencies = [
"derive_more-impl",
]
[[package]]
name = "derive_more-impl"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.117",
"unicode-xid",
]
[[package]]
name = "des"
version = "0.7.0"
@@ -1196,6 +1238,12 @@ version = "0.15.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
[[package]]
name = "drop_bomb"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9bda8e21c04aca2ae33ffc2fd8c23134f3cac46db123ba97bd9d3f3b8a4a85e1"
[[package]]
name = "dunce"
version = "1.0.5"
@@ -1502,6 +1550,31 @@ dependencies = [
"version_check",
]
[[package]]
name = "get-size-derive2"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28f0daa179069133df60523e0f59d0a6bd2d3db9ccae01b630896d269c4a802e"
dependencies = [
"attribute-derive",
"quote",
"syn 3.0.7",
]
[[package]]
name = "get-size2"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "050a5d07359ebfac7f0b74916d94cfa30575e45fdc7e9bfbb0f76d5d4203325a"
dependencies = [
"compact_str",
"get-size-derive2",
"hashbrown 0.17.1",
"ordermap",
"smallvec",
"thin-vec",
]
[[package]]
name = "gethostname"
version = "0.4.3"
@@ -1686,6 +1759,12 @@ version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
[[package]]
name = "hashbrown"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a"
[[package]]
name = "hashlink"
version = "0.10.0"
@@ -1758,7 +1837,7 @@ dependencies = [
"hashbrown 0.14.5",
"new_debug_unreachable",
"once_cell",
"rustc-hash 2.1.1",
"rustc-hash",
"triomphe",
]
@@ -1870,7 +1949,7 @@ dependencies = [
"libc",
"percent-encoding",
"pin-project-lite",
"socket2 0.6.3",
"socket2",
"system-configuration",
"tokio",
"tower-service",
@@ -2012,16 +2091,22 @@ dependencies = [
[[package]]
name = "indexmap"
version = "2.13.0"
version = "2.14.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017"
checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855"
dependencies = [
"equivalent",
"hashbrown 0.16.1",
"hashbrown 0.17.1",
"serde",
"serde_core",
]
[[package]]
name = "interpolator"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71dd52191aae121e8611f1e8dc3e324dd0dd1dee1e6dd91d10ee07a3cfb4d9d8"
[[package]]
name = "inventory"
version = "0.3.22"
@@ -2082,15 +2167,6 @@ version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
[[package]]
name = "itertools"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
dependencies = [
"either",
]
[[package]]
name = "itertools"
version = "0.13.0"
@@ -2109,6 +2185,15 @@ dependencies = [
"either",
]
[[package]]
name = "itertools"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc"
dependencies = [
"either",
]
[[package]]
name = "itoa"
version = "1.0.17"
@@ -2212,12 +2297,6 @@ version = "0.2.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37"
[[package]]
name = "lalrpop-util"
version = "0.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553"
[[package]]
name = "lazy_static"
version = "1.5.0"
@@ -2388,61 +2467,26 @@ dependencies = [
]
[[package]]
name = "malachite"
version = "0.4.22"
name = "manyhow"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2fbdf9cb251732db30a7200ebb6ae5d22fe8e11397364416617d2c2cf0c51cb5"
checksum = "8c69ea6e469ca785c846b25dd69eac9e9c916b51d8ada1f212866787ee1da12d"
dependencies = [
"malachite-base",
"malachite-nz",
"malachite-q",
"manyhow-macros",
"proc-macro2",
"quote",
"syn 3.0.7",
]
[[package]]
name = "malachite-base"
version = "0.4.22"
name = "manyhow-macros"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ea0ed76adf7defc1a92240b5c36d5368cfe9251640dcce5bd2d0b7c1fd87aeb"
checksum = "22fc5c1b2c4000056dae3619fea36e79ce79dabfee4907d2dcad6612b189e370"
dependencies = [
"hashbrown 0.14.5",
"itertools 0.11.0",
"libm",
"ryu",
]
[[package]]
name = "malachite-bigint"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d149aaa2965d70381709d9df4c7ee1fc0de1c614a4efc2ee356f5e43d68749f8"
dependencies = [
"derive_more",
"malachite",
"num-integer",
"num-traits",
"paste",
]
[[package]]
name = "malachite-nz"
version = "0.4.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34a79feebb2bc9aa7762047c8e5495269a367da6b5a90a99882a0aeeac1841f7"
dependencies = [
"itertools 0.11.0",
"libm",
"malachite-base",
]
[[package]]
name = "malachite-q"
version = "0.4.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50f235d5747b1256b47620f5640c2a17a88c7569eebdf27cd9cb130e1a619191"
dependencies = [
"itertools 0.11.0",
"malachite-base",
"malachite-nz",
"proc-macro-utils",
"proc-macro2",
"quote",
]
[[package]]
@@ -2949,6 +2993,15 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
[[package]]
name = "ordermap"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "16a756ddcdeb6c0efc8927bdf0ed112885815a35ba350472a467be5104f01ca1"
dependencies = [
"indexmap",
]
[[package]]
name = "owo-colors"
version = "4.3.0"
@@ -2984,12 +3037,6 @@ dependencies = [
"windows-link",
]
[[package]]
name = "paste"
version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
[[package]]
name = "pem"
version = "3.0.6"
@@ -3286,6 +3333,17 @@ dependencies = [
"version_check",
]
[[package]]
name = "proc-macro-utils"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eeaf08a13de400bc215877b5bdc088f241b12eb42f0a548d3390dc1c56bb7071"
dependencies = [
"proc-macro2",
"quote",
"smallvec",
]
[[package]]
name = "proc-macro2"
version = "1.0.106"
@@ -3401,9 +3459,9 @@ dependencies = [
"pin-project-lite",
"quinn-proto",
"quinn-udp",
"rustc-hash 2.1.1",
"rustc-hash",
"rustls",
"socket2 0.6.3",
"socket2",
"thiserror 2.0.18",
"tokio",
"tracing",
@@ -3422,7 +3480,7 @@ dependencies = [
"lru-slab",
"rand 0.9.5",
"ring",
"rustc-hash 2.1.1",
"rustc-hash",
"rustls",
"rustls-pki-types",
"slab",
@@ -3441,7 +3499,7 @@ dependencies = [
"cfg_aliases",
"libc",
"once_cell",
"socket2 0.6.3",
"socket2",
"tracing",
"windows-sys 0.60.2",
]
@@ -3455,6 +3513,28 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "quote-use"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48356e493823dc3c1d7e9fc7b3ff204ecec48afbdd718fc4134091b88eb63f3d"
dependencies = [
"quote",
"quote-use-macros",
]
[[package]]
name = "quote-use-macros"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8cf1c5de6a7cc231c71e03c77aa4c7019097859ea7029156b54d2894a8430704"
dependencies = [
"proc-macro-utils",
"proc-macro2",
"quote",
"syn 3.0.7",
]
[[package]]
name = "r-efi"
version = "5.3.0"
@@ -3835,6 +3915,106 @@ dependencies = [
"zeroize",
]
[[package]]
name = "ruff_python_ast"
version = "0.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15434b06df7d930fad2818639a4e0fec4af9c770121c45a96b72da7feaae9ead"
dependencies = [
"aho-corasick",
"arrayvec",
"bitflags",
"char_str",
"compact_str",
"get-size2",
"is-macro",
"memchr",
"ruff_python_trivia",
"ruff_source_file",
"ruff_text_size",
"rustc-hash",
"thin-vec",
"thiserror 2.0.18",
]
[[package]]
name = "ruff_python_parser"
version = "0.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37a22811d752aa9e255d7dd72524d14352cf9bc654ef79b399bf1eeeb6640fbb"
dependencies = [
"bitflags",
"bstr",
"drop_bomb",
"get-size2",
"hashbrown 0.17.1",
"memchr",
"ruff_python_ast",
"ruff_python_trivia",
"ruff_text_size",
"ruff_unicode_names2",
"rustc-hash",
"stacker",
"static_assertions",
"thin-vec",
"unicode-ident",
"unicode-normalization",
]
[[package]]
name = "ruff_python_trivia"
version = "0.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0322a5059fe0feb2e010ca6f31c24ac289f6b7e9e38359539491b10c57f1c5bc"
dependencies = [
"itertools 0.15.0",
"memchr",
"ruff_source_file",
"ruff_text_size",
"rustc-hash",
"unicode-ident",
"unicode-normalization",
]
[[package]]
name = "ruff_source_file"
version = "0.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6ae809a289b5c791f82aa45df33f2a92323e7a1c1d265c10e9ac884027a2e43"
dependencies = [
"memchr",
"ruff_text_size",
]
[[package]]
name = "ruff_text_size"
version = "0.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "daf6344d6915ced9316312c1ee3f24c4300c54295bc4c3b8f4e95b633ae5443b"
dependencies = [
"get-size2",
]
[[package]]
name = "ruff_unicode_names2"
version = "0.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13418b8fd029d597e35dedc36e91f47122d26075e4c86557574d9441753fb375"
dependencies = [
"phf 0.11.3",
"ruff_unicode_names2_generator",
]
[[package]]
name = "ruff_unicode_names2_generator"
version = "0.0.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37fd82af49d5685d9054e68185987acaaf62542f671b6df8935155664876bc39"
dependencies = [
"phf_codegen",
"rand 0.8.6",
]
[[package]]
name = "rust_decimal"
version = "1.40.0"
@@ -3857,12 +4037,6 @@ version = "0.1.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d"
[[package]]
name = "rustc-hash"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
[[package]]
name = "rustc-hash"
version = "2.1.1"
@@ -3971,63 +4145,6 @@ dependencies = [
"untrusted 0.9.0",
]
[[package]]
name = "rustpython-ast"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cdaf8ee5c1473b993b398c174641d3aa9da847af36e8d5eb8291930b72f31a5"
dependencies = [
"is-macro",
"malachite-bigint",
"rustpython-parser-core",
"static_assertions",
]
[[package]]
name = "rustpython-parser"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "868f724daac0caf9bd36d38caf45819905193a901e8f1c983345a68e18fb2abb"
dependencies = [
"anyhow",
"is-macro",
"itertools 0.11.0",
"lalrpop-util",
"log",
"malachite-bigint",
"num-traits",
"phf 0.11.3",
"phf_codegen",
"rustc-hash 1.1.0",
"rustpython-ast",
"rustpython-parser-core",
"tiny-keccak",
"unic-emoji-char",
"unic-ucd-ident",
"unicode_names2",
]
[[package]]
name = "rustpython-parser-core"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4b6c12fa273825edc7bccd9a734f0ad5ba4b8a2f4da5ff7efe946f066d0f4ad"
dependencies = [
"is-macro",
"memchr",
"rustpython-parser-vendored",
]
[[package]]
name = "rustpython-parser-vendored"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04fcea49a4630a3a5d940f4d514dc4f575ed63c14c3e3ed07146634aed7f67a6"
dependencies = [
"memchr",
"once_cell",
]
[[package]]
name = "rustversion"
version = "1.0.22"
@@ -4040,6 +4157,12 @@ version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
[[package]]
name = "ryu-js"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04d056b875a9d2e6cb9a61d127afee9ac5999b9f87bcb32079d1318e505be714"
[[package]]
name = "same-file"
version = "1.0.6"
@@ -4409,16 +4532,6 @@ dependencies = [
"version_check",
]
[[package]]
name = "socket2"
version = "0.5.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678"
dependencies = [
"libc",
"windows-sys 0.52.0",
]
[[package]]
name = "socket2"
version = "0.6.3"
@@ -4680,15 +4793,15 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
[[package]]
name = "stacker"
version = "0.1.23"
version = "0.1.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08d74a23609d509411d10e2176dc2a4346e3b4aea2e7b1869f19fdedbc71c013"
checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967"
dependencies = [
"cc",
"cfg-if",
"libc",
"psm",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -4830,7 +4943,7 @@ dependencies = [
"new_debug_unreachable",
"num-bigint",
"once_cell",
"rustc-hash 2.1.1",
"rustc-hash",
"serde",
"siphasher 0.3.11",
"swc_atoms",
@@ -4852,7 +4965,7 @@ dependencies = [
"num-bigint",
"once_cell",
"phf 0.11.3",
"rustc-hash 2.1.1",
"rustc-hash",
"string_enum",
"swc_atoms",
"swc_common",
@@ -4871,7 +4984,7 @@ dependencies = [
"either",
"num-bigint",
"phf 0.11.3",
"rustc-hash 2.1.1",
"rustc-hash",
"seq-macro",
"serde",
"smallvec",
@@ -4974,6 +5087,17 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "syn"
version = "3.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d62a2e0561533f2ca2561d0cf27fd9fedb640a1bf2616ff5d5c80d99017faadc"
dependencies = [
"proc-macro2",
"quote",
"unicode-ident",
]
[[package]]
name = "sync_wrapper"
version = "1.0.2"
@@ -5102,6 +5226,12 @@ dependencies = [
"unicode-width 0.2.2",
]
[[package]]
name = "thin-vec"
version = "0.2.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6a4b9ba8738cb4a4f399d37e266becfd475e75eb73425b87a05a2f2039ba63e"
[[package]]
name = "thiserror"
version = "1.0.69"
@@ -5193,15 +5323,6 @@ dependencies = [
"time-core",
]
[[package]]
name = "tiny-keccak"
version = "2.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"
dependencies = [
"crunchy",
]
[[package]]
name = "tinystr"
version = "0.8.2"
@@ -5229,9 +5350,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
version = "1.46.1"
version = "1.47.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17"
checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038"
dependencies = [
"backtrace",
"bytes",
@@ -5242,10 +5363,10 @@ dependencies = [
"pin-project-lite",
"signal-hook-registry",
"slab",
"socket2 0.5.10",
"socket2",
"tokio-macros",
"tracing",
"windows-sys 0.52.0",
"windows-sys 0.59.0",
]
[[package]]
@@ -5289,7 +5410,7 @@ dependencies = [
"postgres-protocol",
"postgres-types",
"rand 0.10.1",
"socket2 0.6.3",
"socket2",
"tokio",
"tokio-util",
"whoami 2.1.0",
@@ -5325,6 +5446,7 @@ dependencies = [
"bytes",
"futures-core",
"futures-sink",
"futures-util",
"pin-project-lite",
"tokio",
]
@@ -5663,17 +5785,6 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc"
[[package]]
name = "unic-emoji-char"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b07221e68897210270a38bde4babb655869637af0f69407f96053a34f76494d"
dependencies = [
"unic-char-property",
"unic-char-range",
"unic-ucd-version",
]
[[package]]
name = "unic-ucd-category"
version = "0.9.0"
@@ -5686,17 +5797,6 @@ dependencies = [
"unic-ucd-version",
]
[[package]]
name = "unic-ucd-ident"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e230a37c0381caa9219d67cf063aa3a375ffed5bf541a452db16e744bdab6987"
dependencies = [
"unic-char-property",
"unic-char-range",
"unic-ucd-version",
]
[[package]]
name = "unic-ucd-version"
version = "0.9.0"
@@ -5775,28 +5875,6 @@ version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
[[package]]
name = "unicode_names2"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d1673eca9782c84de5f81b82e4109dcfb3611c8ba0d52930ec4a9478f547b2dd"
dependencies = [
"phf 0.11.3",
"unicode_names2_generator",
]
[[package]]
name = "unicode_names2_generator"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b91e5b84611016120197efd7dc93ef76774f4e084cd73c9fb3ea4a86c570c56e"
dependencies = [
"getopts",
"log",
"phf_codegen",
"rand 0.8.6",
]
[[package]]
name = "unscanny"
version = "0.1.0"
@@ -6260,12 +6338,14 @@ dependencies = [
"pin-project-lite",
"pkcs1",
"postgres-native-tls",
"postgres-protocol",
"quick_cache",
"rand 0.9.5",
"regex",
"reqwest",
"reqwest-middleware",
"reqwest-retry",
"ryu-js",
"semver",
"serde",
"serde_json",
@@ -6409,8 +6489,8 @@ version = "1.830.0"
dependencies = [
"anyhow",
"itertools 0.14.0",
"rustpython-ast",
"rustpython-parser",
"ruff_python_ast",
"ruff_python_parser",
"serde_json",
"windmill-parser",
]
@@ -6420,8 +6500,8 @@ name = "windmill-parser-py-asset"
version = "1.830.0"
dependencies = [
"anyhow",
"rustpython-ast",
"rustpython-parser",
"ruff_python_ast",
"ruff_python_parser",
"windmill-parser",
"windmill-parser-sql-asset",
]
@@ -6438,7 +6518,8 @@ dependencies = [
"phf 0.11.3",
"regex",
"regex-lite",
"rustpython-parser",
"ruff_python_ast",
"ruff_python_parser",
"serde",
"serde_json",
"sqlx",
@@ -6555,8 +6636,9 @@ name = "windmill-parser-wac"
version = "1.830.0"
dependencies = [
"anyhow",
"rustpython-ast",
"rustpython-parser",
"ruff_python_ast",
"ruff_python_parser",
"ruff_text_size",
"serde",
"serde_json",
"sha2 0.10.9",