From aae5f2efaa772d5bc6d7d1e4537d7f6748e50e61 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 11 Oct 2026 23:26:03 +0200 Subject: [PATCH] fix: parse python 3.12-3.15 syntax in the python parsers (#11675) * fix: parse python 3.12-3.15 syntax by moving the python parsers to ruff Co-Authored-By: Claude Opus 5.5 * test: pin async for loops in the python wac dag Co-Authored-By: Claude Opus 5.5 * fix: tell main apart from a look-alike line inside a string Co-Authored-By: Claude Opus 5.5 * fix: keep import split and async for as before, fix sql asset path panic Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- backend/Cargo.lock | 454 ++++++++------ backend/Cargo.toml | 8 +- .../windmill-parser-py-asset/Cargo.toml | 4 +- .../windmill-parser-py-asset/src/lib.rs | 264 +++++--- .../windmill-parser-py-imports/Cargo.toml | 3 +- .../windmill-parser-py-imports/src/lib.rs | 72 ++- .../windmill-parser-py-imports/tests/tests.rs | 87 ++- backend/parsers/windmill-parser-py/Cargo.toml | 4 +- backend/parsers/windmill-parser-py/src/lib.rs | 309 ++++++---- .../windmill-parser-py/src/pydantic_parser.rs | 18 +- .../src/asset_parser_utils.rs | 17 +- .../parsers/windmill-parser-wac/Cargo.toml | 5 +- .../parsers/windmill-parser-wac/src/python.rs | 325 ++++------ .../windmill-parser-wac/tests/python_tests.rs | 74 +++ .../parsers/windmill-parser-wasm/Cargo.lock | 582 ++++++++++-------- 15 files changed, 1350 insertions(+), 876 deletions(-) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index ce517a7487..aa3d601492 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -950,6 +950,35 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "attribute-derive" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f9ef7a68684657ce9da341c934c73b63df7a3585a61e366d84c84ed5d3d167" +dependencies = [ + "attribute-derive-macro", + "manyhow 0.14.0", + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "attribute-derive-macro" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300d426e5e73549125bb8e4649bdd5508f82c94ddf898fb0517246e71dfb8e98" +dependencies = [ + "collection_literals", + "interpolator", + "manyhow 0.14.0", + "proc-macro-utils", + "proc-macro2", + "quote", + "quote-use", + "syn 3.0.6", +] + [[package]] name = "autocfg" version = "1.5.1" @@ -2385,6 +2414,18 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "char_str" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cba78a5abb515610e10af297615434b019755b5d7be14e559ff5340dff50c4" +dependencies = [ + "castaway", + "get-size2", + "itoa", + "ryu", +] + [[package]] name = "chrono" version = "0.4.45" @@ -2516,6 +2557,12 @@ dependencies = [ "x509-cert", ] +[[package]] +name = "collection_literals" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2550f75b8cfac212855f6b1885455df8eaee8fe8e246b647d69146142e016084" + [[package]] name = "colorchoice" version = "1.0.5" @@ -2555,6 +2602,19 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "compact_str" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79fcda08c33bb58b97008b2cdada6622500e949e060f5913361763121abd2416" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "static_assertions", + "zmij", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -4630,7 +4690,6 @@ dependencies = [ "proc-macro2", "quote", "syn 2.0.119", - "unicode-xid", ] [[package]] @@ -4887,6 +4946,12 @@ dependencies = [ "text_lines", ] +[[package]] +name = "drop_bomb" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bda8e21c04aca2ae33ffc2fd8c23134f3cac46db123ba97bd9d3f3b8a4a85e1" + [[package]] name = "dtoa" version = "1.0.11" @@ -5816,6 +5881,31 @@ dependencies = [ "zeroize", ] +[[package]] +name = "get-size-derive2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28f0daa179069133df60523e0f59d0a6bd2d3db9ccae01b630896d269c4a802e" +dependencies = [ + "attribute-derive", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "get-size2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "050a5d07359ebfac7f0b74916d94cfa30575e45fdc7e9bfbb0f76d5d4203325a" +dependencies = [ + "compact_str 0.10.0", + "get-size-derive2", + "hashbrown 0.17.1", + "ordermap", + "smallvec", + "thin-vec", +] + [[package]] name = "gethostname" version = "0.4.3" @@ -7005,6 +7095,12 @@ version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" +[[package]] +name = "interpolator" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71dd52191aae121e8611f1e8dc3e324dd0dd1dee1e6dd91d10ee07a3cfb4d9d8" + [[package]] name = "inventory" version = "0.3.25" @@ -7104,6 +7200,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -7493,12 +7598,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "lalrpop-util" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553" - [[package]] name = "lapin" version = "2.5.5" @@ -7917,76 +8016,30 @@ dependencies = [ "webpki-roots 0.26.11", ] -[[package]] -name = "malachite" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fbdf9cb251732db30a7200ebb6ae5d22fe8e11397364416617d2c2cf0c51cb5" -dependencies = [ - "malachite-base", - "malachite-nz", - "malachite-q", -] - -[[package]] -name = "malachite-base" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea0ed76adf7defc1a92240b5c36d5368cfe9251640dcce5bd2d0b7c1fd87aeb" -dependencies = [ - "hashbrown 0.14.5", - "itertools 0.11.0", - "libm", - "ryu", -] - -[[package]] -name = "malachite-bigint" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d149aaa2965d70381709d9df4c7ee1fc0de1c614a4efc2ee356f5e43d68749f8" -dependencies = [ - "derive_more 1.0.0", - "malachite", - "num-integer", - "num-traits", - "paste", -] - -[[package]] -name = "malachite-nz" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34a79feebb2bc9aa7762047c8e5495269a367da6b5a90a99882a0aeeac1841f7" -dependencies = [ - "itertools 0.11.0", - "libm", - "malachite-base", -] - -[[package]] -name = "malachite-q" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f235d5747b1256b47620f5640c2a17a88c7569eebdf27cd9cb130e1a619191" -dependencies = [ - "itertools 0.11.0", - "malachite-base", - "malachite-nz", -] - [[package]] name = "manyhow" version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b33efb3ca6d3b07393750d4030418d594ab1139cee518f0dc88db70fec873587" dependencies = [ - "manyhow-macros", + "manyhow-macros 0.11.4", "proc-macro2", "quote", "syn 2.0.119", ] +[[package]] +name = "manyhow" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c69ea6e469ca785c846b25dd69eac9e9c916b51d8ada1f212866787ee1da12d" +dependencies = [ + "manyhow-macros 0.14.0", + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "manyhow-macros" version = "0.11.4" @@ -7998,6 +8051,17 @@ dependencies = [ "quote", ] +[[package]] +name = "manyhow-macros" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22fc5c1b2c4000056dae3619fea36e79ce79dabfee4907d2dcad6612b189e370" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", +] + [[package]] name = "mappable-rc" version = "0.1.1" @@ -8265,7 +8329,7 @@ checksum = "a4db8a44120571277accfaa3f3d91e7d3989d601d817c2fc01a9391b86135666" dependencies = [ "darling 0.23.0", "heck", - "manyhow", + "manyhow 0.11.4", "num-bigint", "proc-macro-crate", "proc-macro2", @@ -9236,6 +9300,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordermap" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a756ddcdeb6c0efc8927bdf0ed112885815a35ba350472a467be5104f01ca1" +dependencies = [ + "indexmap 2.14.2", +] + [[package]] name = "os_pipe" version = "1.2.1" @@ -10325,6 +10398,28 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "quote-use" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48356e493823dc3c1d7e9fc7b3ff204ecec48afbdd718fc4134091b88eb63f3d" +dependencies = [ + "quote", + "quote-use-macros", +] + +[[package]] +name = "quote-use-macros" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf1c5de6a7cc231c71e03c77aa4c7019097859ea7029156b54d2894a8430704" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "r-efi" version = "5.3.0" @@ -11078,6 +11173,106 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ruff_python_ast" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15434b06df7d930fad2818639a4e0fec4af9c770121c45a96b72da7feaae9ead" +dependencies = [ + "aho-corasick", + "arrayvec", + "bitflags 2.13.2", + "char_str", + "compact_str 0.10.0", + "get-size2", + "is-macro", + "memchr", + "ruff_python_trivia", + "ruff_source_file", + "ruff_text_size", + "rustc-hash 2.1.3", + "thin-vec", + "thiserror 2.0.21", +] + +[[package]] +name = "ruff_python_parser" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37a22811d752aa9e255d7dd72524d14352cf9bc654ef79b399bf1eeeb6640fbb" +dependencies = [ + "bitflags 2.13.2", + "bstr", + "drop_bomb", + "get-size2", + "hashbrown 0.17.1", + "memchr", + "ruff_python_ast", + "ruff_python_trivia", + "ruff_text_size", + "ruff_unicode_names2", + "rustc-hash 2.1.3", + "stacker", + "static_assertions", + "thin-vec", + "unicode-ident", + "unicode-normalization", +] + +[[package]] +name = "ruff_python_trivia" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0322a5059fe0feb2e010ca6f31c24ac289f6b7e9e38359539491b10c57f1c5bc" +dependencies = [ + "itertools 0.15.0", + "memchr", + "ruff_source_file", + "ruff_text_size", + "rustc-hash 2.1.3", + "unicode-ident", + "unicode-normalization", +] + +[[package]] +name = "ruff_source_file" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6ae809a289b5c791f82aa45df33f2a92323e7a1c1d265c10e9ac884027a2e43" +dependencies = [ + "memchr", + "ruff_text_size", +] + +[[package]] +name = "ruff_text_size" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf6344d6915ced9316312c1ee3f24c4300c54295bc4c3b8f4e95b633ae5443b" +dependencies = [ + "get-size2", +] + +[[package]] +name = "ruff_unicode_names2" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13418b8fd029d597e35dedc36e91f47122d26075e4c86557574d9441753fb375" +dependencies = [ + "phf 0.11.3", + "ruff_unicode_names2_generator", +] + +[[package]] +name = "ruff_unicode_names2_generator" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37fd82af49d5685d9054e68185987acaaf62542f671b6df8935155664876bc39" +dependencies = [ + "phf_codegen", + "rand 0.8.5", +] + [[package]] name = "rumqttc" version = "0.24.0" @@ -11414,63 +11609,6 @@ dependencies = [ "untrusted 0.9.0", ] -[[package]] -name = "rustpython-ast" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cdaf8ee5c1473b993b398c174641d3aa9da847af36e8d5eb8291930b72f31a5" -dependencies = [ - "is-macro", - "malachite-bigint", - "rustpython-parser-core", - "static_assertions", -] - -[[package]] -name = "rustpython-parser" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "868f724daac0caf9bd36d38caf45819905193a901e8f1c983345a68e18fb2abb" -dependencies = [ - "anyhow", - "is-macro", - "itertools 0.11.0", - "lalrpop-util", - "log", - "malachite-bigint", - "num-traits", - "phf 0.11.3", - "phf_codegen", - "rustc-hash 1.1.0", - "rustpython-ast", - "rustpython-parser-core", - "tiny-keccak", - "unic-emoji-char", - "unic-ucd-ident", - "unicode_names2", -] - -[[package]] -name = "rustpython-parser-core" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4b6c12fa273825edc7bccd9a734f0ad5ba4b8a2f4da5ff7efe946f066d0f4ad" -dependencies = [ - "is-macro", - "memchr", - "rustpython-parser-vendored", -] - -[[package]] -name = "rustpython-parser-vendored" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04fcea49a4630a3a5d940f4d514dc4f575ed63c14c3e3ed07146634aed7f67a6" -dependencies = [ - "memchr", - "once_cell", -] - [[package]] name = "rustversion" version = "1.0.23" @@ -12903,7 +13041,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bcf55c2d7555c93f4945e29f93b7529562be97ba16e60dd94c25724d746174ac" dependencies = [ "ascii", - "compact_str", + "compact_str 0.7.1", "memchr", "num-bigint", "once_cell", @@ -13628,6 +13766,12 @@ dependencies = [ "unicode-width 0.2.2", ] +[[package]] +name = "thin-vec" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6a4b9ba8738cb4a4f399d37e266becfd475e75eb73425b87a05a2f2039ba63e" + [[package]] name = "thiserror" version = "1.0.69" @@ -14677,17 +14821,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" -[[package]] -name = "unic-emoji-char" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b07221e68897210270a38bde4babb655869637af0f69407f96053a34f76494d" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - [[package]] name = "unic-ucd-category" version = "0.9.0" @@ -14700,17 +14833,6 @@ dependencies = [ "unic-ucd-version", ] -[[package]] -name = "unic-ucd-ident" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e230a37c0381caa9219d67cf063aa3a375ffed5bf541a452db16e744bdab6987" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - [[package]] name = "unic-ucd-version" version = "0.9.0" @@ -14798,28 +14920,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" -[[package]] -name = "unicode_names2" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1673eca9782c84de5f81b82e4109dcfb3611c8ba0d52930ec4a9478f547b2dd" -dependencies = [ - "phf 0.11.3", - "unicode_names2_generator", -] - -[[package]] -name = "unicode_names2_generator" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91e5b84611016120197efd7dc93ef76774f4e084cd73c9fb3ea4a86c570c56e" -dependencies = [ - "getopts", - "log", - "phf_codegen", - "rand 0.8.5", -] - [[package]] name = "universal-hash" version = "0.5.1" @@ -16630,8 +16730,8 @@ version = "1.830.0" dependencies = [ "anyhow", "itertools 0.14.0", - "rustpython-ast", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", "serde_json", "windmill-parser", ] @@ -16641,8 +16741,8 @@ name = "windmill-parser-py-asset" version = "1.830.0" dependencies = [ "anyhow", - "rustpython-ast", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", "windmill-parser", "windmill-parser-sql-asset", ] @@ -16659,7 +16759,8 @@ dependencies = [ "phf 0.11.3", "regex", "regex-lite", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", "serde", "serde_json", "sqlx", @@ -16776,8 +16877,9 @@ name = "windmill-parser-wac" version = "1.830.0" dependencies = [ "anyhow", - "rustpython-ast", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", + "ruff_text_size", "serde", "serde_json", "sha2 0.10.9", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 306a3ae9f6..d627328e27 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -488,9 +488,11 @@ arc-swap = "1" ed25519-dalek = { version = "2", features = ["rand_core"] } magic-crypt = "^3" git-version = "^0" -malachite = "=0.4.18" -malachite-bigint = "=0.2.0" -rustpython-parser = "^0" +# Ruff's parser crates are internal to ruff and carry no semver guarantee: pin them exactly, +# and to the same release. +ruff_python_parser = "=0.0.17" +ruff_python_ast = "=0.0.17" +ruff_text_size = "=0.0.17" pep440_rs = "0.7.3" php-parser-rs = { git = "https://github.com/php-rust-tools/parser", rev = "ec4cb411dec09450946ef57920b7ffced7f6495d" } cron = "^0" diff --git a/backend/parsers/windmill-parser-py-asset/Cargo.toml b/backend/parsers/windmill-parser-py-asset/Cargo.toml index ddd6446079..84562dcf0e 100644 --- a/backend/parsers/windmill-parser-py-asset/Cargo.toml +++ b/backend/parsers/windmill-parser-py-asset/Cargo.toml @@ -11,6 +11,6 @@ path = "./src/lib.rs" [dependencies] windmill-parser.workspace = true windmill-parser-sql-asset.workspace = true -rustpython-parser.workspace = true -rustpython-ast = { version = "0.4.0", features = ["visitor"] } +ruff_python_parser.workspace = true +ruff_python_ast.workspace = true anyhow.workspace = true diff --git a/backend/parsers/windmill-parser-py-asset/src/lib.rs b/backend/parsers/windmill-parser-py-asset/src/lib.rs index 9c70b93d64..2919b74607 100644 --- a/backend/parsers/windmill-parser-py-asset/src/lib.rs +++ b/backend/parsers/windmill-parser-py-asset/src/lib.rs @@ -1,5 +1,8 @@ -use rustpython_ast::{Constant, Expr, ExprConstant, Visitor}; -use rustpython_parser::{ast::Suite, Parse}; +use ruff_python_ast::{ + visitor::{walk_expr, walk_stmt, Visitor}, + Expr, ExprCall, ExprDict, FStringPartRef, InterpolatedStringElement, Keyword, Stmt, StmtAssign, +}; +use ruff_python_parser::parse_module; use std::collections::HashMap; use windmill_parser::asset_parser::{ asset_was_used, merge_assets, parse_asset_syntax, parse_pipeline_annotations, AssetKind, @@ -8,12 +11,12 @@ use windmill_parser::asset_parser::{ use AssetUsageAccessType::*; pub fn parse_assets(input: &str) -> anyhow::Result { - let ast = Suite::parse(input, "main.py") - .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?; + let ast = parse_module(input) + .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))? + .into_suite(); let mut assets_finder = AssetsFinder { assets: vec![], var_identifiers: HashMap::new() }; - ast.into_iter() - .for_each(|stmt| assets_finder.visit_stmt(stmt)); + assets_finder.visit_body(&ast); for (kind, path, _) in assets_finder.var_identifiers.into_values() { // if a db = wmill.datatable() was never used (e.g db.query(...)), @@ -43,13 +46,72 @@ struct AssetsFinder { var_identifiers: HashMap, } -impl Visitor for AssetsFinder { +impl<'a> Visitor<'a> for AssetsFinder { + fn visit_stmt(&mut self, stmt: &'a Stmt) { + if let Stmt::Assign(node) = stmt { + self.track_assignment(node); + } + // Continue with generic visit to catch any other assets in the statement + walk_stmt(self, stmt); + } + + fn visit_expr(&mut self, expr: &'a Expr) { + match expr { + // A detected asset call does not recurse, + // so this is only reached when no further context was found + Expr::StringLiteral(s) => self.visit_str(s.value.to_str()), + Expr::FString(f) => { + // Literal text between two interpolations reads as one string, across the + // pieces of an implicitly concatenated f-string too. + let mut literal = String::new(); + for part in &f.value { + match part { + FStringPartRef::Literal(s) => literal.push_str(&s.value), + FStringPartRef::FString(f) => { + for element in &f.elements { + match element { + InterpolatedStringElement::Literal(l) => { + literal.push_str(&l.value) + } + InterpolatedStringElement::Interpolation(_) => { + self.visit_str(&std::mem::take(&mut literal)) + } + } + } + } + } + } + self.visit_str(&literal); + walk_expr(self, expr); + } + Expr::Call(node) => { + if self.visit_expr_call_inner(node).is_err() { + walk_expr(self, expr); + } + } + _ => walk_expr(self, expr), + } + } +} + +impl AssetsFinder { + fn visit_str(&mut self, s: &str) { + if let Some((kind, path)) = parse_asset_syntax(s, false) { + self.assets.push(ParseAssetsResult { + kind, + path: path.to_string(), + access_type: None, + columns: None, + }); + } + } + // Handle assignment statements like: x = wmill.datatable('name') - fn visit_stmt_assign(&mut self, node: rustpython_ast::StmtAssign) { + fn track_assignment(&mut self, node: &StmtAssign) { // Check if the value is a call to a tracked function if let Some(Expr::Name(expr_name)) = node.targets.first() { // Remove any tracked variables with that name in case of reassignment - let Ok(var_name) = expr_name.id.parse::(); + let var_name = expr_name.id.to_string(); let removed = self.var_identifiers.remove(&var_name); // if a db = wmill.datatable() or similar was removed, but never used (e.g db.query(...)), // we still want to register the asset as unknown access type @@ -69,59 +131,12 @@ impl Visitor for AssetsFinder { if let Some((kind, name, schema)) = self.extract_asset_from_call(&node.value) { // Track target variable - let Ok(var_name) = expr_name.id.parse::(); self.var_identifiers .insert(var_name, (kind.clone(), name.clone(), schema.clone())); } } - // Continue with generic visit to catch any other assets in the expression - self.generic_visit_stmt_assign(node); } - // visit_call_expr will not recurse if it detects an asset, - // so this will only be called when no further context was found - fn visit_expr_constant(&mut self, node: ExprConstant) { - match node.value { - Constant::Str(s) => { - if let Some((kind, path)) = parse_asset_syntax(&s, false) { - self.assets.push(ParseAssetsResult { - kind, - path: path.to_string(), - access_type: None, - columns: None, - }); - } - } - _ => self.generic_visit_expr_constant(node), - } - } - - fn visit_expr_call(&mut self, node: rustpython_ast::ExprCall) { - match self.visit_expr_call_inner(&node) { - Ok(_) => {} - Err(_) => { - // Check keyword arguments for assets before falling back to generic visit - for keyword in &node.keywords { - if let Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) = - &keyword.value - { - if let Some((kind, path)) = parse_asset_syntax(s, false) { - self.assets.push(ParseAssetsResult { - kind, - path: path.to_string(), - access_type: None, - columns: None, - }); - } - } - } - self.generic_visit_expr_call(node); - } - } - } -} - -impl AssetsFinder { /// Extract asset info from calls like wmill.datatable('name'), wmill.ducklake('name'), etc. fn extract_asset_from_call( &self, @@ -147,18 +162,20 @@ impl AssetsFinder { // Get the first argument (the asset name) let name = call + .arguments .args .first() .or_else(|| { // Try keyword argument 'name' or 'path' - call.keywords + call.arguments + .keywords .iter() .find(|kw| matches!(kw.arg.as_deref(), Some("name") | Some("path"))) .map(|kw| &kw.value) }) .and_then(|first_arg| { - if let Expr::Constant(ExprConstant { value: Constant::Str(name), .. }) = first_arg { - Some(name.clone()) + if let Expr::StringLiteral(name) = first_arg { + Some(name.value.to_str().to_string()) } else { None } @@ -179,44 +196,39 @@ impl AssetsFinder { Some((kind, name, schema)) } - fn visit_expr_call_inner(&mut self, node: &rustpython_ast::ExprCall) -> Result<(), ()> { + fn visit_expr_call_inner(&mut self, node: &ExprCall) -> Result<(), ()> { let ident: String = node .func .as_name_expr() - .and_then(|o| o.id.parse().ok()) + .map(|o| o.id.to_string()) .or_else(|| { node.func .as_attribute_expr() - .and_then(|attr| attr.attr.parse().ok()) + .map(|attr| attr.attr.to_string()) }) .ok_or(())?; // `obj_name` is the object (receiver) - i.e., `obj` in `obj.method()` - let obj_name: String = if let Expr::Attribute(rustpython_ast::ExprAttribute { - value, .. - }) = node.func.as_ref() - { - if let Expr::Name(expr_name) = value.as_ref() { - expr_name.id.parse().map_err(|_| ())? - } else { - return Err(()); - } - } else { - return Err(()); - }; + let obj_name: String = node + .func + .as_attribute_expr() + .and_then(|attr| attr.value.as_name_expr()) + .map(|name| name.id.to_string()) + .ok_or(())?; if obj_name == "wmill" { // Continue } else if let Some((kind, ref path, ref schema)) = self.var_identifiers.get(&obj_name) { if ident == "query" { - let expr_name = node.args.get(0).or_else(|| { - node.keywords + let expr_name = node.arguments.args.get(0).or_else(|| { + node.arguments + .keywords .iter() .find(|kw| kw.arg.as_deref() == Some("name")) .map(|kw| &kw.value) }); let sql = match expr_name { - Some(Expr::Constant(ExprConstant { value: Constant::Str(sql), .. })) => sql, + Some(Expr::StringLiteral(sql)) => sql.value.to_str(), _ => return Err(()), }; // We use the SQL parser to detect RW, specific tables, etc. @@ -224,7 +236,7 @@ impl AssetsFinder { *kind, path, schema.as_deref(), - &sql, + sql, ); match sql_assets { Ok(Some(sql_assets)) => self.assets.extend(sql_assets), @@ -252,9 +264,10 @@ impl AssetsFinder { }; let arg_val = match arg { - Arg(i, name) => node.args.get(i).or_else(|| { + Arg(i, name) => node.arguments.args.get(i).or_else(|| { // Get arg by name - node.keywords + node.arguments + .keywords .iter() .find(|kw| kw.arg.as_deref() == Some(name)) .map(|kw| &kw.value) @@ -270,8 +283,9 @@ impl AssetsFinder { self.assets .push(ParseAssetsResult { kind, path, access_type, columns: None }); } - Some(Expr::Constant(ExprConstant { value: Constant::Str(value), .. })) => { - let path = parse_asset_syntax(&value, false) + Some(Expr::StringLiteral(value)) => { + let value = value.value.to_str(); + let path = parse_asset_syntax(value, false) .map(|(_, p)| p) .unwrap_or(std::borrow::Cow::Borrowed(value)); self.assets.push(ParseAssetsResult { @@ -291,26 +305,24 @@ impl AssetsFinder { struct Arg(usize, &'static str); /// Extract a string-literal keyword argument, e.g. `s3="value"` in a call. -fn keyword_str_value(keywords: &[rustpython_ast::Keyword], name: &str) -> Option { +fn keyword_str_value(keywords: &[Keyword], name: &str) -> Option { keywords .iter() .find(|kw| kw.arg.as_deref() == Some(name)) .and_then(|kw| match &kw.value { - Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => Some(s.clone()), + Expr::StringLiteral(s) => Some(s.value.to_str().to_string()), _ => None, }) } /// Extract a string-literal value from a dict literal, e.g. `{"s3": "value"}`. -fn dict_str_value(dict: &rustpython_ast::ExprDict, name: &str) -> Option { - dict.keys +fn dict_str_value(dict: &ExprDict, name: &str) -> Option { + dict.items .iter() - .zip(dict.values.iter()) - .find_map(|(key, value)| match (key.as_ref()?, value) { - ( - Expr::Constant(ExprConstant { value: Constant::Str(k), .. }), - Expr::Constant(ExprConstant { value: Constant::Str(v), .. }), - ) if k.as_str() == name => Some(v.clone()), + .find_map(|item| match (item.key.as_ref()?, &item.value) { + (Expr::StringLiteral(k), Expr::StringLiteral(v)) if k.value.to_str() == name => { + Some(v.value.to_str().to_string()) + } _ => None, }) } @@ -328,12 +340,13 @@ fn dict_str_value(dict: &rustpython_ast::ExprDict, name: &str) -> Option /// matches the TS object form and the `# on s3:///…` trigger form exactly. fn s3_object_arg_path(expr: &Expr) -> Option { let uri = match expr { - Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => { + Expr::StringLiteral(s) => { + let s = s.value.to_str(); match s .strip_prefix("s3://") .and_then(|rest| rest.split_once('/')) { - Some((_, key)) if !key.is_empty() => s.clone(), + Some((_, key)) if !key.is_empty() => s.to_string(), _ => return None, } } @@ -347,8 +360,9 @@ fn s3_object_arg_path(expr: &Expr) -> Option { if func_name != "S3Object" { return None; } - let key = keyword_str_value(&call.keywords, "s3")?; - let storage = keyword_str_value(&call.keywords, "storage").unwrap_or_default(); + let key = keyword_str_value(&call.arguments.keywords, "s3")?; + let storage = + keyword_str_value(&call.arguments.keywords, "storage").unwrap_or_default(); format!("s3://{storage}/{key}") } Expr::Dict(dict) => { @@ -822,4 +836,62 @@ def main(): },]) ); } + + #[test] + fn test_py_asset_parser_recent_syntax() { + let input = r#" +import wmill +lazy import polars + +type Rows = list[dict] + +def main[T](name: str): + try: + with wmill.load_s3_file_reader('s3:///in.csv') as f: + f.read() + except ValueError, TypeError: + pass + greeting = t"hello {name}" + return f"{wmill.get_resource("u/user/pg")["host"]}" +"#; + let s = parse_assets(input).map(|o| o.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ + ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/in.csv".to_string(), + access_type: Some(R), + columns: None, + }, + ParseAssetsResult { + kind: AssetKind::Resource, + path: "u/user/pg".to_string(), + access_type: None, + columns: None, + }, + ]) + ); + } + + #[test] + fn test_py_asset_parser_schema_qualified_db_without_table() { + let input = r#" +import wmill +def main(): + db = wmill.datatable('dt:sch') + db.query("ATTACH 'ducklake://other' AS o") + db.query("SELECT * FROM t") +"#; + let s = parse_assets(input).map(|o| o.assets); + assert_eq!( + s.map_err(|e| e.to_string()) + .map(|a| a.into_iter().map(|a| (a.kind, a.path)).collect::>()), + Ok(vec![ + (AssetKind::DataTable, "dt".to_string()), + (AssetKind::DataTable, "dt/sch.t".to_string()), + (AssetKind::Ducklake, "other".to_string()), + ]) + ); + } } diff --git a/backend/parsers/windmill-parser-py-imports/Cargo.toml b/backend/parsers/windmill-parser-py-imports/Cargo.toml index 565e1422bb..2ef83f5ac9 100644 --- a/backend/parsers/windmill-parser-py-imports/Cargo.toml +++ b/backend/parsers/windmill-parser-py-imports/Cargo.toml @@ -23,7 +23,8 @@ tracing.workspace = true [dependencies] windmill-parser.workspace = true -rustpython-parser.workspace = true +ruff_python_parser.workspace = true +ruff_python_ast.workspace = true phf.workspace = true itertools.workspace = true serde_json.workspace = true diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs index 72597da385..5c8de9790f 100644 --- a/backend/parsers/windmill-parser-py-imports/src/lib.rs +++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs @@ -13,9 +13,9 @@ use async_recursion::async_recursion; use itertools::Itertools; use lazy_static::lazy_static; #[cfg(not(target_arch = "wasm32"))] -use std::str::FromStr; -#[cfg(not(target_arch = "wasm32"))] use std::collections::HashMap; +#[cfg(not(target_arch = "wasm32"))] +use std::str::FromStr; use mapping::{FULL_IMPORTS_MAP, SHORT_IMPORTS_MAP}; #[cfg(not(target_arch = "wasm32"))] @@ -23,11 +23,8 @@ use regex::Regex; #[cfg(target_arch = "wasm32")] use regex_lite::Regex; -use rustpython_parser::{ - ast::{Stmt, StmtImport, StmtImportFrom, Suite}, - text_size::TextRange, - Parse, -}; +use ruff_python_ast::{Stmt, StmtImport, StmtImportFrom}; +use ruff_python_parser::parse_module; #[cfg(not(target_arch = "wasm32"))] use sqlx::{Pool, Postgres}; #[cfg(not(target_arch = "wasm32"))] @@ -160,6 +157,7 @@ pub struct ImportPin { } pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result> { + let full_code = code; // Use regex to safely find the main function definition let mut code = DEF_MAIN_RE .split(code) @@ -186,16 +184,22 @@ pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result (code, parsed.into_suite()), + // The split lands on any line reading like the definition of `main`, one inside a + // string included: what precedes it is then not a module, while the whole file is. + Err(e) => match parse_module(full_code) { + Ok(parsed) => (full_code.to_string(), parsed.into_suite()), + Err(_) => anyhow::bail!("Error parsing code for imports: {}", e.to_string()), + }, + }; // Note: We're still using the original code for finding pins, // as the TextRange values from the parsed AST would be based on code_with_fake_main // but we want to match against the original code - let find_pin = |range: TextRange, key: String| { + let find_pin = |stmt_end: usize, key: String| { let hs = code .chars() - .skip(range.end().to_usize()) + .skip(stmt_end) .take_while(|e| *e != '\n') .collect::(); @@ -226,9 +230,9 @@ pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result = ast .into_iter() .filter_map(|x| match x { - Stmt::Import(StmtImport { names, range }) => names + Stmt::Import(StmtImport { names, range, .. }) => names .get(0) - .and_then(|al| find_pin(range, al.name.to_string())) + .and_then(|al| find_pin(range.end().to_usize(), al.name.to_string())) .or(Some( names .into_iter() @@ -239,15 +243,11 @@ pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result>(), )), - Stmt::ImportFrom(StmtImportFrom { level: Some(i), module, .. }) if i.to_u32() > 0 => { - Some(process_import( - module.map(|x| x.to_string()), - path, - i.to_usize(), - )) - } + Stmt::ImportFrom(StmtImportFrom { level, module, .. }) if level > 0 => Some( + process_import(module.map(|x| x.to_string()), path, level as usize), + ), Stmt::ImportFrom(StmtImportFrom { level: _, module, range, .. }) => find_pin( - range, + range.end().to_usize(), module.clone().map(|x| x.to_string()).unwrap_or_default(), ) .or(Some(process_import(module.map(|x| x.to_string()), path, 0))), @@ -518,8 +518,14 @@ async fn parse_python_imports_inner( let mut nested = match n { NImport::Relative(rpath) => { // First try to get content from temp_script_refs cache if available - let code_from_cache = if let Some(hash) = temp_script_refs.as_ref().and_then(|dt| dt.get(&rpath)) { - tracing::debug!("Found relative import '{}' in temp_script_refs with hash '{}'", rpath, hash); + let code_from_cache = if let Some(hash) = + temp_script_refs.as_ref().and_then(|dt| dt.get(&rpath)) + { + tracing::debug!( + "Found relative import '{}' in temp_script_refs with hash '{}'", + rpath, + hash + ); match windmill_common::cache::raw_script_temp::load(hash.clone(), db).await { Ok(content) => Some(content), Err(e) => { @@ -534,19 +540,17 @@ async fn parse_python_imports_inner( // Use cached content if available, otherwise fall back to deployed script let code = match code_from_cache { Some(content) => content, - None => { - sqlx::query_scalar!( - r#" + None => sqlx::query_scalar!( + r#" SELECT content FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1 "#, - &rpath, - w_id - ) - .fetch_optional(db) - .await? - .unwrap_or_else(|| "".to_string()) - } + &rpath, + w_id + ) + .fetch_optional(db) + .await? + .unwrap_or_else(|| "".to_string()), }; if already_visited.contains(&rpath) { diff --git a/backend/parsers/windmill-parser-py-imports/tests/tests.rs b/backend/parsers/windmill-parser-py-imports/tests/tests.rs index c853c8e5b4..9086b29d47 100644 --- a/backend/parsers/windmill-parser-py-imports/tests/tests.rs +++ b/backend/parsers/windmill-parser-py-imports/tests/tests.rs @@ -1,7 +1,9 @@ mod tests { use sqlx::{Pool, Postgres}; - use windmill_parser_py_imports::parse_python_imports; + use windmill_parser_py_imports::{ + parse_code_for_imports, parse_python_imports, ImportPin, NImport, + }; #[sqlx::test(fixtures("base"))] async fn test_parse_python_imports(db: Pool) -> anyhow::Result<()> { @@ -117,4 +119,87 @@ def main(): Ok(()) } + + fn auto(pkg: &str) -> NImport { + NImport::Auto { pkg: pkg.to_string(), key: None } + } + + #[test] + fn test_parse_lazy_imports() -> anyhow::Result<()> { + let code = " +lazy import requests +lazy import numpy # pin: numpy==2.1.0 +lazy from pandas import DataFrame +lazy from f.shared import helper +lazy = 1 + +def main(): + pass +"; + assert_eq!( + parse_code_for_imports(code, "f/foo/bar")?, + vec![ + NImport::Pin { + pins: vec![ImportPin { + pkg: "numpy==2.1.0".to_string(), + path: "f/foo/bar".to_string() + }], + key: "numpy".to_string() + }, + auto("pandas"), + auto("requests"), + NImport::Relative("f/shared".to_string()), + ] + ); + Ok(()) + } + + #[test] + fn test_parse_imports_recent_syntax_before_main() -> anyhow::Result<()> { + let code = r#" +import requests + +type Alias = list[int] +NAME = "x" +D = {"a": 1} +L = [[1], [2]] +TEMPLATE = t"hello {NAME}" +PEP701 = f"{D["a"]} {'\n'.join(["x"])}" +PEP798 = [*a for a in L] + +try: + import ujson +except ImportError, ValueError: + pass + +def main(): + pass +"#; + assert_eq!( + parse_code_for_imports(code, "f/foo/bar")?, + vec![auto("requests")] + ); + Ok(()) + } + + #[test] + fn test_parse_imports_main_lookalike_in_docstring() -> anyhow::Result<()> { + for example in ["def main[T](x: T):", "def main(x):"] { + let code = format!( + "\"\"\"Usage example:\n{example}\n return x\n\"\"\"\nimport requests # pin: requests==2.32.0\n\ndef main(value: str):\n return value\n" + ); + assert_eq!( + parse_code_for_imports(&code, "f/foo/bar")?, + vec![NImport::Pin { + pins: vec![ImportPin { + pkg: "requests==2.32.0".to_string(), + path: "f/foo/bar".to_string() + }], + key: "requests".to_string() + }], + "{code}" + ); + } + Ok(()) + } } diff --git a/backend/parsers/windmill-parser-py/Cargo.toml b/backend/parsers/windmill-parser-py/Cargo.toml index 0c50e0f8d8..6971fe8fcd 100644 --- a/backend/parsers/windmill-parser-py/Cargo.toml +++ b/backend/parsers/windmill-parser-py/Cargo.toml @@ -10,8 +10,8 @@ path = "./src/lib.rs" [dependencies] windmill-parser.workspace = true -rustpython-parser.workspace = true +ruff_python_parser.workspace = true +ruff_python_ast.workspace = true itertools.workspace = true serde_json.workspace = true anyhow.workspace = true -rustpython-ast = { version = "0.4.0", features = ["visitor"] } diff --git a/backend/parsers/windmill-parser-py/src/lib.rs b/backend/parsers/windmill-parser-py/src/lib.rs index 086a354d02..5a9eabc33e 100644 --- a/backend/parsers/windmill-parser-py/src/lib.rs +++ b/backend/parsers/windmill-parser-py/src/lib.rs @@ -13,13 +13,11 @@ use itertools::Itertools; use serde_json::json; use windmill_parser::{json_to_typ, Arg, MainArgSignature, ObjectType, Typ}; -use rustpython_parser::{ - ast::{ - Constant, Expr, ExprAttribute, ExprConstant, ExprDict, ExprList, ExprName, Stmt, - StmtAssign, StmtAsyncFunctionDef, StmtClassDef, StmtFunctionDef, Suite, - }, - Parse, +use ruff_python_ast::{ + Expr, ExprAttribute, ExprDict, ExprList, ExprName, Number, Parameters, Stmt, StmtAssign, + StmtClassDef, StmtFunctionDef, }; +use ruff_python_parser::parse_module; pub mod pydantic_parser; @@ -81,22 +79,24 @@ fn should_parse_for_models(code: &str) -> bool { || code.contains("import dataclasses") } +/// What follows `def ` or `async def ` on a line opening that function's +/// parameters or type parameters. +fn strip_def<'a>(line: &'a str, name: &str) -> Option<&'a str> { + let rest = line + .strip_prefix("async def ") + .or_else(|| line.strip_prefix("def "))? + .strip_prefix(name)?; + (rest.starts_with('(') || rest.starts_with('[')).then_some(rest) +} + fn filter_non_main(code: &str, main_name: &str) -> String { - let def_main = format!("def {}(", main_name); - let async_def_main = format!("async def {}(", main_name); let mut filtered_code = String::new(); let mut code_iter = code.split("\n"); let mut remaining: String = String::new(); while let Some(line) = code_iter.next() { - if line.starts_with(&async_def_main) { - filtered_code += &async_def_main; - remaining += line.strip_prefix(&async_def_main).unwrap(); - remaining += "\n"; - remaining += &code_iter.join("\n"); - break; - } else if line.starts_with(&def_main) { - filtered_code += &def_main; - remaining += line.strip_prefix(&def_main).unwrap(); + if let Some(rest) = strip_def(line, main_name) { + filtered_code += &line[..line.len() - rest.len()]; + remaining += rest; remaining += "\n"; remaining += &code_iter.join("\n"); break; @@ -106,6 +106,26 @@ fn filter_non_main(code: &str, main_name: &str) -> String { return String::new(); } let mut chars = remaining.chars(); + + // PEP 695 type parameters, `def main[T](x: T)`, sit between the name and the parameters. + if remaining.starts_with('[') { + let mut open_brackets = 0; + for c in chars.by_ref() { + if c == '[' { + open_brackets += 1; + } else if c == ']' { + open_brackets -= 1; + } + filtered_code.push(c); + if open_brackets == 0 { + break; + } + } + } + if chars.next() != Some('(') { + return String::new(); + } + filtered_code.push('('); let mut open_parens = 1; while let Some(c) = chars.next() { @@ -183,15 +203,15 @@ fn extract_code_metadata(code: &str, main_name: &str) -> CodeMetadata { let filtered_code = filter_relevant_statements(code); - let ast = match Suite::parse(&filtered_code, "main.py") { - Ok(ast) => ast, + let ast = match parse_module(&filtered_code) { + Ok(parsed) => parsed.into_suite(), Err(_) => return CodeMetadata { enums, descriptions }, }; for stmt in ast { match stmt { - Stmt::ClassDef(StmtClassDef { name, body, bases, .. }) if has_enum => { - let is_enum = bases.iter().any(|base| { + Stmt::ClassDef(class_def) if has_enum => { + let is_enum = class_def.bases().iter().any(|base| { matches!(base, Expr::Name(ExprName { id, .. }) if id == "Enum" || id == "IntEnum" || id == "StrEnum" || id == "Flag" || id == "IntFlag") @@ -201,17 +221,15 @@ fn extract_code_metadata(code: &str, main_name: &str) -> CodeMetadata { let mut values = Vec::new(); let mut members = HashMap::new(); + let StmtClassDef { name, body, .. } = class_def; for item in body { if let Stmt::Assign(StmtAssign { targets, value, .. }) = item { if let Some(Expr::Name(ExprName { id: target_name, .. })) = targets.first() { if !target_name.starts_with('_') { - if let Expr::Constant(ExprConstant { - value: Constant::Str(val), - .. - }) = value.as_ref() - { + if let Expr::StringLiteral(val) = value.as_ref() { + let val = val.value.to_str(); values.push(val.to_string()); members.insert(target_name.to_string(), val.to_string()); } @@ -225,15 +243,13 @@ fn extract_code_metadata(code: &str, main_name: &str) -> CodeMetadata { } } } - Stmt::FunctionDef(StmtFunctionDef { name: func_name, body, .. }) if has_docstring => { - if &func_name == main_name { + Stmt::FunctionDef(StmtFunctionDef { + name: func_name, body, is_async: false, .. + }) if has_docstring => { + if func_name.as_str() == main_name { if let Some(Stmt::Expr(expr_stmt)) = body.first() { - if let Expr::Constant(ExprConstant { - value: Constant::Str(docstring), - .. - }) = expr_stmt.value.as_ref() - { - descriptions = parse_docstring_args(docstring); + if let Expr::StringLiteral(docstring) = expr_stmt.value.as_ref() { + descriptions = parse_docstring_args(docstring.value.to_str()); } } } @@ -294,6 +310,27 @@ fn parse_docstring_args(docstring: &str) -> HashMap { descriptions } +/// A line that reads as the definition of `main_name` can also sit inside a string, a docstring +/// showing an example for instance. When several lines do, only the parsed module tells which +/// one is the function. +fn main_params_among_lookalikes(code: &str, main_name: &str) -> Option { + code.split('\n') + .filter(|line| strip_def(line, main_name).is_some()) + .nth(1)?; + parse_module(code) + .ok()? + .into_suite() + .into_iter() + .find_map(|x| match x { + Stmt::FunctionDef(StmtFunctionDef { name, parameters, .. }) + if name.as_str() == main_name => + { + Some(*parameters) + } + _ => None, + }) +} + /// skip_params is a micro optimization for when we just want to find the main /// function without parsing all the params. pub fn parse_python_signature( @@ -310,41 +347,39 @@ pub fn parse_python_signature( // - If no models: parse only the filtered main function let (params, module) = if should_parse_for_models(code) { // Parse full code once for both Pydantic detection and signature extraction - let ast = Suite::parse(code, "main.py") - .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?; + let ast = parse_module(code) + .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))? + .into_suite(); // Extract main function from full AST let params = ast.iter().find_map(|x| match x { - Stmt::FunctionDef(StmtFunctionDef { name, args, .. }) if name == &main_name => { - Some(args.as_ref().clone()) - } - Stmt::AsyncFunctionDef(StmtAsyncFunctionDef { name, args, .. }) - if name == &main_name => + Stmt::FunctionDef(StmtFunctionDef { name, parameters, .. }) + if name.as_str() == main_name => { - Some(args.as_ref().clone()) + Some(parameters.as_ref().clone()) } _ => None, }); // Keep AST for Pydantic/dataclass detection (params, Some(ast)) + } else if let Some(params) = main_params_among_lookalikes(code, &main_name) { + (Some(params), None) } else { // No models detected - parse only the filtered main function let filtered_code = filter_non_main(code, &main_name); if filtered_code.is_empty() { (None, None) } else { - let ast = Suite::parse(&filtered_code, "main.py") - .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?; + let ast = parse_module(&filtered_code) + .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))? + .into_suite(); let params = ast.into_iter().find_map(|x| match x { - Stmt::FunctionDef(StmtFunctionDef { name, args, .. }) if &name == &main_name => { - Some(*args) - } - Stmt::AsyncFunctionDef(StmtAsyncFunctionDef { name, args, .. }) - if &name == &main_name => + Stmt::FunctionDef(StmtFunctionDef { name, parameters, .. }) + if name.as_str() == main_name => { - Some(*args) + Some(*parameters) } _ => None, }); @@ -378,8 +413,6 @@ pub fn parse_python_signature( if !skip_params && params.is_some() { let params = params.unwrap(); - let def_arg_start = params.args.len() - params.defaults().count(); - // Two-pass approach for lazy metadata extraction: // Pass 1: Parse types without enum info to determine if metadata is needed // Pass 2: Re-parse unknown types with metadata only if necessary @@ -392,11 +425,11 @@ pub fn parse_python_signature( .iter() .enumerate() .map(|(i, x)| { - let arg_name = x.as_arg().arg.to_string(); + let arg_name = x.parameter.name.to_string(); let (typ, has_default) = x - .as_arg() + .parameter .annotation - .as_ref() + .as_deref() .map_or((Typ::Unknown, false), |e| { parse_expr(e, &empty_enums, module_ref) }); @@ -423,21 +456,16 @@ pub fn parse_python_signature( .into_iter() .map(|(i, arg_name, mut typ, mut has_default)| { if matches!(typ, Typ::Resource(_)) && !metadata.enums.is_empty() { - if let Some(annotation) = params.args[i].as_arg().annotation.as_ref() { + if let Some(annotation) = params.args[i].parameter.annotation.as_deref() { (typ, has_default) = parse_expr(annotation, &metadata.enums, module_ref); } } - let default = if i >= def_arg_start { - params - .defaults() - .nth(i - def_arg_start) - .map(|expr| to_value(expr, &metadata.enums)) - .flatten() - } else { - None - }; + let default = params.args[i] + .default + .as_deref() + .and_then(|expr| to_value(expr, &metadata.enums)); let should_get_type_from_default = match &typ { Typ::Unknown => true, @@ -465,10 +493,10 @@ pub fn parse_python_signature( // debounce accumulation support. Falls back to docstring // description if no union array pattern is found. let union_otyp = params.args[i] - .as_arg() + .parameter .annotation - .as_ref() - .and_then(|ann| detect_py_union_array_otyp(ann.as_ref())); + .as_deref() + .and_then(detect_py_union_array_otyp); Arg { otyp: union_otyp.or_else(|| { @@ -509,12 +537,8 @@ pub fn parse_python_signature( } } -fn parse_expr( - e: &Box, - enums: &HashMap, - module: Option<&[Stmt]>, -) -> (Typ, bool) { - match e.as_ref() { +fn parse_expr(e: &Expr, enums: &HashMap, module: Option<&[Stmt]>) -> (Typ, bool) { + match e { Expr::Name(ExprName { id, .. }) => (parse_typ(id.as_ref(), enums, module), false), Expr::Attribute(x) => { if let Some(name) = x.value.as_name_expr() { @@ -531,12 +555,9 @@ fn parse_expr( } } Expr::BinOp(x) => { - if matches!( - x.right.as_ref(), - Expr::Constant(ExprConstant { value: Constant::None, .. }) - ) { + if x.right.is_none_literal_expr() { (parse_expr(&x.left, enums, module).0, true) - } else if detect_py_union_array_otyp(e.as_ref()).is_some() { + } else if detect_py_union_array_otyp(e).is_some() { // T | list[T] — parsed type is Unknown; otyp is set separately (Typ::Unknown, false) } else { @@ -552,7 +573,7 @@ fn parse_expr( .elts .iter() .map(|x| match x { - Expr::Constant(c) => c.value.as_str().map(|x| x.to_string()), + Expr::StringLiteral(s) => Some(s.value.to_str().to_string()), _ => None, }) .filter_map(|x| x) @@ -625,15 +646,26 @@ fn map_resource_name(x: &str) -> String { } } -fn to_value(et: &Expr, enums: &HashMap) -> Option { +fn to_value(et: &Expr, enums: &HashMap) -> Option { match et { - Expr::Constant(ExprConstant { value, .. }) => Some(constant_to_value(value)), - Expr::Dict(ExprDict { keys, values, .. }) => { - let v = keys - .into_iter() - .zip(values) - .map(|(k, v)| { - let key = k + Expr::NoneLiteral(_) => Some(json!(null)), + Expr::BooleanLiteral(b) => Some(json!(b.value)), + Expr::StringLiteral(s) => Some(json!(s.value.to_str())), + Expr::BytesLiteral(b) => Some(json!(b.value.bytes().collect::>())), + Expr::NumberLiteral(n) => Some(match &n.value { + Number::Int(i) => { + serde_json::from_str(&i.to_string()).unwrap_or(json!("invalid number")) + } + Number::Float(f) => json!(f), + Number::Complex { real, imag } => json!([real, imag]), + }), + Expr::EllipsisLiteral(_) => Some(json!("...")), + Expr::Dict(ExprDict { items, .. }) => { + let v = items + .iter() + .map(|item| { + let key = item + .key .as_ref() .map(|e| to_value(e, enums)) .flatten() @@ -642,7 +674,7 @@ fn to_value(et: &Expr, enums: &HashMap) -> Option None, }) .unwrap_or_else(|| "no_key".to_string()); - (key, to_value(&v, enums)) + (key, to_value(&item.value, enums)) }) .collect::>(); Some(json!(v)) @@ -672,20 +704,6 @@ fn to_value(et: &Expr, enums: &HashMap) -> Option serde_json::Value { - match c { - Constant::None => json!(null), - Constant::Bool(b) => json!(b), - Constant::Str(s) => json!(s), - Constant::Bytes(b) => json!(b), - Constant::Int(i) => serde_json::from_str(&i.to_string()).unwrap_or(json!("invalid number")), - Constant::Tuple(t) => json!(t.iter().map(constant_to_value).collect::>()), - Constant::Float(f) => json!(f), - Constant::Complex { real, imag } => json!([real, imag]), - Constant::Ellipsis => json!("..."), - } -} - #[cfg(test)] mod tests { @@ -1323,4 +1341,89 @@ def main(color: Color = Color.RED): ); Ok(()) } + + #[test] + fn test_parse_python_sig_type_params() -> anyhow::Result<()> { + for code in [ + "def main[T](x: str, y: int = 3):\n return x\n", + "async def main[T: (int, str), *Ts](x: str, y: int = 3):\n return x\n", + // parsed whole rather than pre-filtered down to `main` + "from dataclasses import dataclass\n\ndef main[T](x: str, y: int = 3):\n return x\n", + ] { + let result = parse_python_signature(code, None, false)?; + assert_eq!(result.auto_kind, None, "{code}"); + assert_eq!( + result + .args + .iter() + .map(|a| (a.name.as_str(), &a.typ, &a.default)) + .collect::>(), + vec![ + ("x", &Typ::Str(None), &None), + ("y", &Typ::Int, &Some(json!(3))) + ], + "{code}" + ); + } + Ok(()) + } + + // A script with a model class is parsed whole, so syntax anywhere in it reaches the parser. + #[test] + fn test_parse_python_sig_recent_syntax() -> anyhow::Result<()> { + let code = r#" +from dataclasses import dataclass +lazy import requests +lazy from pandas import DataFrame + +type Alias = list[int] + +@dataclass +class Point: + x: int + +try: + import a +except ImportError, ValueError: + pass + +D = {"a": 1} +L = [[1], [2]] +PEP701 = f"{D["a"]} {'\n'.join(["x"])}" +PEP798 = [*a for a in L] + +def main(p: Point, name: str = "world"): + return t"hello {name}" +"#; + let result = parse_python_signature(code, None, false)?; + assert_eq!(result.args.len(), 2); + assert_eq!( + result.args[0].typ, + Typ::Object(ObjectType::new( + Some("Point".to_string()), + Some(vec![windmill_parser::ObjectProperty { + key: "x".to_string(), + typ: Box::new(Typ::Int) + }]) + )) + ); + assert_eq!(result.args[1].default, Some(json!("world"))); + Ok(()) + } + + #[test] + fn test_parse_python_sig_main_lookalike_in_docstring() -> anyhow::Result<()> { + for example in ["def main[T](x: T):", "def main(x: int, y: int):"] { + let code = format!( + "\"\"\"Usage example:\n{example}\n return x\n\"\"\"\n\ndef main(value: str):\n return value\n" + ); + let result = parse_python_signature(&code, None, false)?; + assert_eq!( + result.args.iter().map(|a| &a.name).collect::>(), + vec!["value"], + "{code}" + ); + } + Ok(()) + } } diff --git a/backend/parsers/windmill-parser-py/src/pydantic_parser.rs b/backend/parsers/windmill-parser-py/src/pydantic_parser.rs index e229d97a8a..78a0a3c354 100644 --- a/backend/parsers/windmill-parser-py/src/pydantic_parser.rs +++ b/backend/parsers/windmill-parser-py/src/pydantic_parser.rs @@ -11,8 +11,8 @@ //! This module provides functionality to detect and parse Pydantic models and Python //! dataclasses from Python AST, enabling automatic UI generation for complex data structures. -use rustpython_parser::ast::{ - Constant, Expr, ExprAttribute, ExprCall, ExprConstant, ExprName, ExprTuple, Stmt, StmtAnnAssign, +use ruff_python_ast::{ + Decorator, Expr, ExprAttribute, ExprCall, ExprName, ExprTuple, Stmt, StmtAnnAssign, }; use std::collections::HashSet; use windmill_parser::{ObjectProperty, ObjectType, Typ}; @@ -64,7 +64,7 @@ fn detect_model_type_impl( // Mark as being visited visited.insert(class_name.to_string()); - let result = if is_pydantic_base(&class_def.bases) { + let result = if is_pydantic_base(class_def.bases()) { // Pydantic BaseModel parse_model_fields(&class_def.body, class_def.name.as_str(), module, visited) } else if has_dataclass_decorator(&class_def.decorator_list) @@ -115,9 +115,9 @@ fn is_pydantic_base(bases: &[Expr]) -> bool { } /// Checks if a class has @dataclass decorator (standard library) -fn has_dataclass_decorator(decorators: &[Expr]) -> bool { +fn has_dataclass_decorator(decorators: &[Decorator]) -> bool { for decorator in decorators { - match decorator { + match &decorator.expression { // Match: @dataclass Expr::Name(ExprName { id, .. }) if id.as_str() == "dataclass" => { return true; @@ -160,9 +160,9 @@ fn has_dataclass_decorator(decorators: &[Expr]) -> bool { } /// Checks if a class has @pydantic.dataclasses.dataclass decorator (Pydantic v2) -fn has_pydantic_dataclass_decorator(decorators: &[Expr]) -> bool { +fn has_pydantic_dataclass_decorator(decorators: &[Decorator]) -> bool { for decorator in decorators { - match decorator { + match &decorator.expression { // Match: @pydantic.dataclasses.dataclass Expr::Attribute(ExprAttribute { attr, value, .. }) if attr.as_str() == "dataclass" => { if let Expr::Attribute(ExprAttribute { @@ -350,10 +350,10 @@ fn extract_field_type( // String annotations: "ForwardRef" (forward references) // Not fully supported - return Unknown with warning - Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => { + Expr::StringLiteral(s) => { eprintln!( "Forward references like \"{forward_ref}\" are not yet supported, treating as Unknown", - forward_ref = s, + forward_ref = s.value.to_str(), ); Typ::Unknown } diff --git a/backend/parsers/windmill-parser-sql-asset/src/asset_parser_utils.rs b/backend/parsers/windmill-parser-sql-asset/src/asset_parser_utils.rs index b29c6c99ac..4d77da15bb 100644 --- a/backend/parsers/windmill-parser-sql-asset/src/asset_parser_utils.rs +++ b/backend/parsers/windmill-parser-sql-asset/src/asset_parser_utils.rs @@ -20,13 +20,16 @@ pub fn parse_wmill_sdk_sql_assets( Ok(mut sql_assets) => { if let Some(schema) = schema { for asset in &mut sql_assets.assets { - if asset.kind == kind && asset.path.starts_with(asset_name) { - asset.path = format!( - "{}/{}.{}", - asset_name, - schema, - &asset.path[asset_name.len() + 1..] - ); + if asset.kind != kind { + continue; + } + // A statement can name the database alone, with no table to qualify. + let table = asset + .path + .strip_prefix(asset_name) + .and_then(|rest| rest.strip_prefix('/')); + if let Some(table) = table { + asset.path = format!("{asset_name}/{schema}.{table}"); } } } diff --git a/backend/parsers/windmill-parser-wac/Cargo.toml b/backend/parsers/windmill-parser-wac/Cargo.toml index d354b1c653..21ee286db9 100644 --- a/backend/parsers/windmill-parser-wac/Cargo.toml +++ b/backend/parsers/windmill-parser-wac/Cargo.toml @@ -9,8 +9,9 @@ name = "windmill_parser_wac" path = "./src/lib.rs" [dependencies] -rustpython-parser.workspace = true -rustpython-ast = { version = "0.4.0", features = ["visitor"] } +ruff_python_parser.workspace = true +ruff_python_ast.workspace = true +ruff_text_size.workspace = true swc_common.workspace = true swc_ecma_parser.workspace = true swc_ecma_ast.workspace = true diff --git a/backend/parsers/windmill-parser-wac/src/python.rs b/backend/parsers/windmill-parser-wac/src/python.rs index f91067b711..e8fb72ff2f 100644 --- a/backend/parsers/windmill-parser-wac/src/python.rs +++ b/backend/parsers/windmill-parser-wac/src/python.rs @@ -1,12 +1,12 @@ use std::collections::HashMap; -use rustpython_parser::{ - ast::{ - Expr, ExprAwait, ExprCall, ExprName, Stmt, StmtExpr, StmtFor, StmtIf, StmtReturn, StmtTry, - StmtTryStar, StmtWhile, - }, - Parse, +use ruff_python_ast::{ + CmpOp, ElifElseClause, ExceptHandler, Expr, ExprAttribute, ExprAwait, ExprCall, ExprName, + Number, Parameters, Stmt, StmtExpr, StmtFor, StmtFunctionDef, StmtIf, StmtReturn, StmtTry, + StmtWhile, }; +use ruff_python_parser::parse_module; +use ruff_text_size::{Ranged, TextSize}; use crate::dag::{DagEdge, DagNode, DagNodeType, Param, WorkflowDag}; use crate::validation::{self, CompileError}; @@ -42,9 +42,9 @@ type TaskFunctions = HashMap>; fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions { let mut tasks = HashMap::new(); for stmt in stmts { - if let Stmt::AsyncFunctionDef(func) = stmt { + if let Stmt::FunctionDef(func @ StmtFunctionDef { is_async: true, .. }) = stmt { for dec in &func.decorator_list { - match dec { + match &dec.expression { // @task (bare decorator) Expr::Name(ExprName { id, .. }) if id.as_str() == "task" => { tasks.insert(func.name.to_string(), None); @@ -68,13 +68,9 @@ fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions { if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() { if id.as_str() == "task_script" || id.as_str() == "task_flow" { // Extract the path from the first positional argument - let path = call.args.first().and_then(|arg| { - if let Expr::Constant(c) = arg { - if let rustpython_parser::ast::Constant::Str(s) = &c.value { - return Some(s.to_string()); - } - } - None + let path = call.arguments.args.first().and_then(|arg| match arg { + Expr::StringLiteral(s) => Some(s.value.to_str().to_string()), + _ => None, }); // Extract variable name from target if let Some(Expr::Name(ExprName { id: var_name, .. })) = @@ -92,13 +88,11 @@ fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions { /// Extract the `path=` keyword argument from a `@task(path="...")` call. fn extract_task_path_kwarg(call: &ExprCall) -> Option { - for kw in &call.keywords { + for kw in &call.arguments.keywords { if let Some(ref arg) = kw.arg { if arg.as_str() == "path" { - if let Expr::Constant(c) = &kw.value { - if let rustpython_parser::ast::Constant::Str(s) = &c.value { - return Some(s.to_string()); - } + if let Expr::StringLiteral(s) = &kw.value { + return Some(s.value.to_str().to_string()); } } } @@ -150,10 +144,9 @@ impl WacWalker { fn line_of_expr(&self, expr: &Expr) -> usize { let offset = match expr { - Expr::Call(c) => c.range.start().to_usize(), - Expr::Await(a) => a.range.start().to_usize(), - Expr::Attribute(a) => a.range.start().to_usize(), - Expr::Name(n) => n.range.start().to_usize(), + Expr::Call(_) | Expr::Await(_) | Expr::Attribute(_) | Expr::Name(_) => { + expr.start().to_usize() + } _ => 0, }; self.line_index.line_of(offset) @@ -161,17 +154,15 @@ impl WacWalker { fn line_of_stmt(&self, stmt: &Stmt) -> usize { let offset = match stmt { - Stmt::If(s) => s.range.start().to_usize(), - Stmt::For(s) => s.range.start().to_usize(), - Stmt::While(s) => s.range.start().to_usize(), - Stmt::Return(s) => s.range.start().to_usize(), - Stmt::Expr(s) => s.range.start().to_usize(), - Stmt::Try(s) => s.range.start().to_usize(), - Stmt::TryStar(s) => s.range.start().to_usize(), - Stmt::Assign(s) => s.range.start().to_usize(), - Stmt::AnnAssign(s) => s.range.start().to_usize(), - Stmt::FunctionDef(s) => s.range.start().to_usize(), - Stmt::AsyncFunctionDef(s) => s.range.start().to_usize(), + Stmt::If(_) + | Stmt::For(_) + | Stmt::While(_) + | Stmt::Return(_) + | Stmt::Expr(_) + | Stmt::Try(_) + | Stmt::Assign(_) + | Stmt::AnnAssign(_) + | Stmt::FunctionDef(_) => stmt.start().to_usize(), _ => 0, }; self.line_index.line_of(offset) @@ -190,9 +181,7 @@ impl WacWalker { /// Check if an expression is `asyncio.gather(...)` call fn is_asyncio_gather_call(expr: &Expr) -> bool { if let Expr::Call(call) = expr { - if let Expr::Attribute(rustpython_parser::ast::ExprAttribute { value, attr, .. }) = - call.func.as_ref() - { + if let Expr::Attribute(ExprAttribute { value, attr, .. }) = call.func.as_ref() { if attr.as_str() == "gather" { if let Expr::Name(ExprName { id, .. }) = value.as_ref() { return id.as_str() == "asyncio"; @@ -222,20 +211,20 @@ impl WacWalker { fn expr_to_source(expr: &Expr) -> String { match expr { Expr::Compare(c) => { - let left = Self::expr_to_source(&c.left); - if let Some(comparator) = c.comparators.first() { + let left = Self::expr_to_source(&c.operands[0]); + if let Some(comparator) = c.comparators().first() { let right = Self::expr_to_source(comparator); let op = match c.ops.first() { - Some(rustpython_parser::ast::CmpOp::Gt) => ">", - Some(rustpython_parser::ast::CmpOp::Lt) => "<", - Some(rustpython_parser::ast::CmpOp::GtE) => ">=", - Some(rustpython_parser::ast::CmpOp::LtE) => "<=", - Some(rustpython_parser::ast::CmpOp::Eq) => "==", - Some(rustpython_parser::ast::CmpOp::NotEq) => "!=", - Some(rustpython_parser::ast::CmpOp::In) => "in", - Some(rustpython_parser::ast::CmpOp::NotIn) => "not in", - Some(rustpython_parser::ast::CmpOp::Is) => "is", - Some(rustpython_parser::ast::CmpOp::IsNot) => "is not", + Some(CmpOp::Gt) => ">", + Some(CmpOp::Lt) => "<", + Some(CmpOp::GtE) => ">=", + Some(CmpOp::LtE) => "<=", + Some(CmpOp::Eq) => "==", + Some(CmpOp::NotEq) => "!=", + Some(CmpOp::In) => "in", + Some(CmpOp::NotIn) => "not in", + Some(CmpOp::Is) => "is", + Some(CmpOp::IsNot) => "is not", None => "?", }; format!("{left} {op} {right}") @@ -253,14 +242,14 @@ impl WacWalker { format!("{value}.{}", a.attr) } Expr::Name(n) => n.id.to_string(), - Expr::Constant(c) => match &c.value { - rustpython_parser::ast::Constant::Str(s) => format!("\"{s}\""), - rustpython_parser::ast::Constant::Int(i) => i.to_string(), - rustpython_parser::ast::Constant::Float(f) => f.to_string(), - rustpython_parser::ast::Constant::Bool(b) => b.to_string(), - rustpython_parser::ast::Constant::None => "None".to_string(), - _ => "...".to_string(), + Expr::StringLiteral(s) => format!("\"{}\"", s.value.to_str()), + Expr::NumberLiteral(n) => match &n.value { + Number::Int(i) => i.to_string(), + Number::Float(f) => f.to_string(), + Number::Complex { .. } => "...".to_string(), }, + Expr::BooleanLiteral(b) => b.value.to_string(), + Expr::NoneLiteral(_) => "None".to_string(), _ => "...".to_string(), } } @@ -279,8 +268,16 @@ impl WacWalker { match stmt { Stmt::Expr(StmtExpr { value, .. }) => self.expr_contains_step(value), Stmt::Assign(a) => self.expr_contains_step(&a.value), - Stmt::If(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse), - Stmt::For(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse), + Stmt::If(s) => { + self.body_contains_step(&s.body) + || s.elif_else_clauses + .iter() + .any(|clause| self.body_contains_step(&clause.body)) + } + // `async for` is not modelled: its body is skipped like any unsupported statement + Stmt::For(s) if !s.is_async => { + self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse) + } Stmt::While(s) => { self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse) } @@ -289,19 +286,7 @@ impl WacWalker { || self.body_contains_step(&s.orelse) || self.body_contains_step(&s.finalbody) || s.handlers.iter().any(|h| match h { - rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { - self.body_contains_step(&eh.body) - } - }) - } - Stmt::TryStar(s) => { - self.body_contains_step(&s.body) - || self.body_contains_step(&s.orelse) - || self.body_contains_step(&s.finalbody) - || s.handlers.iter().any(|h| match h { - rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { - self.body_contains_step(&eh.body) - } + ExceptHandler::ExceptHandler(eh) => self.body_contains_step(&eh.body), }) } Stmt::Return(_) => false, @@ -319,11 +304,12 @@ impl WacWalker { match expr { Expr::Await(ExprAwait { value, .. }) => self.expr_contains_step(value), Expr::Call(call) => { - if self.is_task_fn_call(&Expr::Call(call.clone())) { - return true; - } - if Self::is_asyncio_gather_call(&Expr::Call(call.clone())) { - return call.args.iter().any(|a| self.expr_contains_step(a)); + if Self::is_asyncio_gather_call(expr) { + return call + .arguments + .args + .iter() + .any(|a| self.expr_contains_step(a)); } false } @@ -370,12 +356,11 @@ impl WacWalker { Stmt::Expr(StmtExpr { value, .. }) => self.walk_expr_stmt(value), Stmt::Assign(a) => self.walk_expr_stmt(&a.value), Stmt::If(if_stmt) => self.walk_if(if_stmt), - Stmt::For(for_stmt) => self.walk_for(for_stmt), + Stmt::For(for_stmt) if !for_stmt.is_async => self.walk_for(for_stmt), Stmt::While(while_stmt) => self.walk_while(while_stmt), Stmt::Try(try_stmt) => self.walk_try(try_stmt), - Stmt::TryStar(try_stmt) => self.walk_try_star(try_stmt), Stmt::Return(ret) => self.walk_return(ret), - Stmt::FunctionDef(_) | Stmt::AsyncFunctionDef(_) => { + Stmt::FunctionDef(_) => { if self.stmt_contains_step(stmt) { self.errors.push(validation::error_step_in_nested_function( self.line_of_stmt(stmt), @@ -392,7 +377,7 @@ impl WacWalker { if let Expr::Await(ExprAwait { value, .. }) = expr { // await task_fn(...) if let Expr::Call(call) = value.as_ref() { - if self.is_task_fn_call(&Expr::Call(call.clone())) { + if self.is_task_fn_call(value) { return self.emit_step(call, expr); } // Check for SDK-level calls: step(), sleep(), wait_for_approval() @@ -430,15 +415,12 @@ impl WacWalker { "step" => { // step("name", fn) — extract the name from the first string argument let name = call + .arguments .args .first() - .and_then(|arg| { - if let Expr::Constant(c) = arg { - if let rustpython_parser::ast::Constant::Str(s) = &c.value { - return Some(s.to_string()); - } - } - None + .and_then(|arg| match arg { + Expr::StringLiteral(s) => Some(s.value.to_str().to_string()), + _ => None, }) .unwrap_or_else(|| "step".to_string()); let id = self.next_id(); @@ -452,6 +434,7 @@ impl WacWalker { } "sleep" => { let seconds = call + .arguments .args .first() .map(|arg| Self::expr_to_source(arg)) @@ -517,10 +500,10 @@ impl WacWalker { }); let mut step_ids = Vec::new(); - for arg in &gather_call.args { + for arg in &gather_call.arguments.args { // Each arg should be task_fn(...) if let Expr::Call(call) = arg { - if self.is_task_fn_call(&Expr::Call(call.clone())) { + if self.is_task_fn_call(arg) { let (name, script) = self .extract_step_info_from_task_call(call) .unwrap_or(("unknown".into(), "unknown".into())); @@ -553,15 +536,33 @@ impl WacWalker { } fn walk_if(&mut self, if_stmt: &StmtIf) -> Option<(String, String)> { - let has_steps_in_body = self.body_contains_step(&if_stmt.body); - let has_steps_in_else = self.body_contains_step(&if_stmt.orelse); + self.walk_if_chain( + if_stmt.start(), + &if_stmt.test, + &if_stmt.body, + &if_stmt.elif_else_clauses, + ) + } + + /// An `elif` is laid out as an `if` nested alone in the `else` of the clause before it. + fn walk_if_chain( + &mut self, + start: TextSize, + test: &Expr, + body: &[Stmt], + clauses: &[ElifElseClause], + ) -> Option<(String, String)> { + let has_steps_in_body = self.body_contains_step(body); + let has_steps_in_else = clauses + .iter() + .any(|clause| self.body_contains_step(&clause.body)); if !has_steps_in_body && !has_steps_in_else { return None; } - let line = self.line_index.line_of(if_stmt.range.start().to_usize()); - let condition_source = Self::expr_to_source(&if_stmt.test); + let line = self.line_index.line_of(start.to_usize()); + let condition_source = Self::expr_to_source(test); let branch_id = self.next_id(); let branch_node_id = self.add_node(DagNode { @@ -573,15 +574,19 @@ impl WacWalker { let mut last_ids = Vec::new(); - if let Some((true_first, true_last)) = self.walk_body(&if_stmt.body) { + if let Some((true_first, true_last)) = self.walk_body(body) { self.add_edge(&branch_node_id, &true_first, Some("true".to_string())); last_ids.push(true_last); } else { last_ids.push(branch_node_id.clone()); } - if !if_stmt.orelse.is_empty() { - if let Some((else_first, else_last)) = self.walk_body(&if_stmt.orelse) { + if let Some((clause, clauses)) = clauses.split_first() { + let orelse = match &clause.test { + Some(test) => self.walk_if_chain(clause.start(), test, &clause.body, clauses), + None => self.walk_body(&clause.body), + }; + if let Some((else_first, else_last)) = orelse { self.add_edge(&branch_node_id, &else_first, Some("false".to_string())); last_ids.push(else_last); } else { @@ -611,7 +616,7 @@ impl WacWalker { return None; } - let line = self.line_index.line_of(for_stmt.range.start().to_usize()); + let line = self.line_index.line_of(for_stmt.start().to_usize()); let iter_source = Self::expr_to_source(&for_stmt.iter); let start_id = self.next_id(); @@ -644,7 +649,7 @@ impl WacWalker { return None; } - let line = self.line_index.line_of(while_stmt.range.start().to_usize()); + let line = self.line_index.line_of(while_stmt.start().to_usize()); let condition = Self::expr_to_source(&while_stmt.test); let start_id = self.next_id(); @@ -677,39 +682,14 @@ impl WacWalker { || self.body_contains_step(&try_stmt.orelse) || self.body_contains_step(&try_stmt.finalbody) || try_stmt.handlers.iter().any(|h| match h { - rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { - self.body_contains_step(&eh.body) - } + ExceptHandler::ExceptHandler(eh) => self.body_contains_step(&eh.body), }); if !has_steps { return None; } - let line = self.line_index.line_of(try_stmt.range.start().to_usize()); - self.emit_try_catch_branch( - &try_stmt.body, - &try_stmt.handlers, - &try_stmt.finalbody, - line, - ) - } - - fn walk_try_star(&mut self, try_stmt: &StmtTryStar) -> Option<(String, String)> { - let has_steps = self.body_contains_step(&try_stmt.body) - || self.body_contains_step(&try_stmt.orelse) - || self.body_contains_step(&try_stmt.finalbody) - || try_stmt.handlers.iter().any(|h| match h { - rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { - self.body_contains_step(&eh.body) - } - }); - - if !has_steps { - return None; - } - - let line = self.line_index.line_of(try_stmt.range.start().to_usize()); + let line = self.line_index.line_of(try_stmt.start().to_usize()); self.emit_try_catch_branch( &try_stmt.body, &try_stmt.handlers, @@ -721,7 +701,7 @@ impl WacWalker { fn emit_try_catch_branch( &mut self, try_body: &[Stmt], - handlers: &[rustpython_parser::ast::ExceptHandler], + handlers: &[ExceptHandler], finally_body: &[Stmt], line: usize, ) -> Option<(String, String)> { @@ -746,7 +726,7 @@ impl WacWalker { // Except handlers for handler in handlers { match handler { - rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { + ExceptHandler::ExceptHandler(eh) => { if let Some((catch_first, catch_last)) = self.walk_body(&eh.body) { self.add_edge(&branch_node_id, &catch_first, Some("except".to_string())); last_ids.push(catch_last); @@ -783,7 +763,7 @@ impl WacWalker { } fn walk_return(&mut self, ret: &StmtReturn) -> Option<(String, String)> { - let line = self.line_index.line_of(ret.range.start().to_usize()); + let line = self.line_index.line_of(ret.start().to_usize()); let id = self.next_id(); let node_id = self.add_node(DagNode { id: id.clone(), @@ -796,12 +776,12 @@ impl WacWalker { } /// Extract workflow function parameters (no longer skips ctx) -fn extract_params(args: &rustpython_parser::ast::Arguments) -> Vec { +fn extract_params(args: &Parameters) -> Vec { let mut params = Vec::new(); for arg_with_default in args.args.iter().chain(args.posonlyargs.iter()) { - let name = arg_with_default.def.arg.to_string(); + let name = arg_with_default.parameter.name.to_string(); let typ = arg_with_default - .def + .parameter .annotation .as_ref() .map(|ann| WacWalker::expr_to_source(ann)); @@ -811,59 +791,34 @@ fn extract_params(args: &rustpython_parser::ast::Arguments) -> Vec { } pub fn parse_python_workflow(code: &str) -> Result> { - let ast = rustpython_parser::ast::Suite::parse(code, "") - .map_err(|e| vec![CompileError { message: format!("Parse error: {e}"), line: 0 }])?; + let ast = parse_module(code) + .map_err(|e| vec![CompileError { message: format!("Parse error: {e}"), line: 0 }])? + .into_suite(); // First pass: collect @task functions let task_functions = collect_task_functions(&ast); - // Find the @workflow async def - let workflow_fn = ast.iter().find_map(|stmt| { - if let Stmt::AsyncFunctionDef(func) = stmt { - let has_workflow_decorator = func.decorator_list.iter().any(|dec| { - if let Expr::Name(ExprName { id, .. }) = dec { - id.as_str() == "workflow" - } else { - false - } - }); - if has_workflow_decorator { - return Some(func); + let workflow_fns = || { + ast.iter().filter_map(|stmt| match stmt { + Stmt::FunctionDef(func) + if func.decorator_list.iter().any(|dec| { + matches!(&dec.expression, Expr::Name(ExprName { id, .. }) if id.as_str() == "workflow") + }) => + { + Some(func) } - } - // Also check non-async for error reporting - if let Stmt::FunctionDef(func) = stmt { - let has_workflow_decorator = func.decorator_list.iter().any(|dec| { - if let Expr::Name(ExprName { id, .. }) = dec { - id.as_str() == "workflow" - } else { - false - } - }); - if has_workflow_decorator { - return None; // Will be reported as not-async below - } - } - None - }); + _ => None, + }) + }; - // Check for non-async workflow function - let non_async_workflow = ast.iter().find_map(|stmt| { - if let Stmt::FunctionDef(func) = stmt { - let has_workflow_decorator = func.decorator_list.iter().any(|dec| { - if let Expr::Name(ExprName { id, .. }) = dec { - id.as_str() == "workflow" - } else { - false - } - }); - if has_workflow_decorator { - let line_index = LineIndex::new(code); - return Some(line_index.line_of(func.range.start().to_usize())); - } - } - None - }); + // Find the @workflow async def + let workflow_fn = workflow_fns().find(|func| func.is_async); + + // Check for non-async workflow function. The statement starts at its first decorator, + // the name sits on the `def` line the error points at. + let non_async_workflow = workflow_fns() + .find(|func| !func.is_async) + .map(|func| LineIndex::new(code).line_of(func.name.start().to_usize())); if let Some(line) = non_async_workflow { if workflow_fn.is_none() { @@ -875,7 +830,7 @@ pub fn parse_python_workflow(code: &str) -> Result String { hasher.update(code.as_bytes()); format!("{:x}", hasher.finalize()) } - -trait ToUsize { - fn to_usize(self) -> usize; -} - -impl ToUsize for rustpython_parser::text_size::TextSize { - fn to_usize(self) -> usize { - u32::from(self) as usize - } -} diff --git a/backend/parsers/windmill-parser-wac/tests/python_tests.rs b/backend/parsers/windmill-parser-wac/tests/python_tests.rs index cf117da9e7..c0de489153 100644 --- a/backend/parsers/windmill-parser-wac/tests/python_tests.rs +++ b/backend/parsers/windmill-parser-wac/tests/python_tests.rs @@ -363,3 +363,77 @@ async def main(x: str): )); assert!(matches!(dag.nodes[5].node_type, DagNodeType::Return)); } + +#[test] +fn test_elif_chain_nests_branches() { + let code = r#" +from wmill import workflow, task + +@task +async def a(): ... +@task +async def b(): ... +@task +async def c(): ... + +@workflow +async def main(x: int): + if x > 2: + await a() + elif x == 2: + await b() + else: + await c() +"#; + + let dag = parse_python_workflow(code).expect("should parse"); + let branches: Vec<(&str, usize)> = dag + .nodes + .iter() + .filter_map(|n| match &n.node_type { + DagNodeType::Branch { condition_source } => Some((condition_source.as_str(), n.line)), + _ => None, + }) + .collect(); + assert_eq!(branches, vec![("x > 2", 13), ("x == 2", 15)]); + + // The `elif` branch hangs off the `false` edge of the `if` before it + let (outer, inner) = (&dag.nodes[0].id, &dag.nodes[2].id); + assert!(dag + .edges + .iter() + .any(|e| &e.from == outer && &e.to == inner && e.label.as_deref() == Some("false"))); + assert_eq!( + dag.nodes + .iter() + .filter(|n| matches!(n.node_type, DagNodeType::Merge)) + .count(), + 2 + ); +} + +#[test] +fn test_workflow_with_recent_syntax() { + let code = r#" +from wmill import workflow, task +lazy import polars + +type Rows = list[dict] + +@task +async def load[T](rows: T): ... + +@workflow +async def main(name: str): + try: + await load(t"hello {name}") + except ValueError, TypeError: + pass +"#; + + let dag = parse_python_workflow(code).expect("should parse"); + assert!(dag + .nodes + .iter() + .any(|n| matches!(&n.node_type, DagNodeType::Step { name, .. } if name == "load"))); +} diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index c0108282ee..75c10528df 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -254,6 +254,35 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "attribute-derive" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f9ef7a68684657ce9da341c934c73b63df7a3585a61e366d84c84ed5d3d167" +dependencies = [ + "attribute-derive-macro", + "manyhow", + "proc-macro2", + "quote", + "syn 3.0.7", +] + +[[package]] +name = "attribute-derive-macro" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300d426e5e73549125bb8e4649bdd5508f82c94ddf898fb0517246e71dfb8e98" +dependencies = [ + "collection_literals", + "interpolator", + "manyhow", + "proc-macro-utils", + "proc-macro2", + "quote", + "quote-use", + "syn 3.0.7", +] + [[package]] name = "autocfg" version = "1.5.0" @@ -428,7 +457,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash 2.1.1", + "rustc-hash", "shlex", "syn 2.0.117", ] @@ -620,6 +649,15 @@ version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e93abca9e28e0a1b9877922aacb20576e05d4679ffa78c3d6dc22a26a216659" +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cc" version = "1.2.56" @@ -670,6 +708,18 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "char_str" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cba78a5abb515610e10af297615434b019755b5d7be14e559ff5340dff50c4" +dependencies = [ + "castaway", + "get-size2", + "itoa", + "ryu", +] + [[package]] name = "chrono" version = "0.4.44" @@ -779,6 +829,12 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "collection_literals" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2550f75b8cfac212855f6b1885455df8eaee8fe8e246b647d69146142e016084" + [[package]] name = "colorchoice" version = "1.0.4" @@ -795,6 +851,19 @@ dependencies = [ "memchr", ] +[[package]] +name = "compact_str" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79fcda08c33bb58b97008b2cdada6622500e949e060f5913361763121abd2416" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "static_assertions", + "zmij", +] + [[package]] name = "compression-codecs" version = "0.4.37" @@ -1019,12 +1088,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "crunchy" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" - [[package]] name = "crypto-common" version = "0.1.7" @@ -1093,27 +1156,6 @@ dependencies = [ "powerfmt", ] -[[package]] -name = "derive_more" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" -dependencies = [ - "derive_more-impl", -] - -[[package]] -name = "derive_more-impl" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", - "unicode-xid", -] - [[package]] name = "des" version = "0.7.0" @@ -1196,6 +1238,12 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "drop_bomb" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bda8e21c04aca2ae33ffc2fd8c23134f3cac46db123ba97bd9d3f3b8a4a85e1" + [[package]] name = "dunce" version = "1.0.5" @@ -1502,6 +1550,31 @@ dependencies = [ "version_check", ] +[[package]] +name = "get-size-derive2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28f0daa179069133df60523e0f59d0a6bd2d3db9ccae01b630896d269c4a802e" +dependencies = [ + "attribute-derive", + "quote", + "syn 3.0.7", +] + +[[package]] +name = "get-size2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "050a5d07359ebfac7f0b74916d94cfa30575e45fdc7e9bfbb0f76d5d4203325a" +dependencies = [ + "compact_str", + "get-size-derive2", + "hashbrown 0.17.1", + "ordermap", + "smallvec", + "thin-vec", +] + [[package]] name = "gethostname" version = "0.4.3" @@ -1686,6 +1759,12 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "hashlink" version = "0.10.0" @@ -1758,7 +1837,7 @@ dependencies = [ "hashbrown 0.14.5", "new_debug_unreachable", "once_cell", - "rustc-hash 2.1.1", + "rustc-hash", "triomphe", ] @@ -1870,7 +1949,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2", "system-configuration", "tokio", "tower-service", @@ -2012,16 +2091,22 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.13.0" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "serde", "serde_core", ] +[[package]] +name = "interpolator" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71dd52191aae121e8611f1e8dc3e324dd0dd1dee1e6dd91d10ee07a3cfb4d9d8" + [[package]] name = "inventory" version = "0.3.22" @@ -2082,15 +2167,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.13.0" @@ -2109,6 +2185,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.17" @@ -2212,12 +2297,6 @@ version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" -[[package]] -name = "lalrpop-util" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553" - [[package]] name = "lazy_static" version = "1.5.0" @@ -2388,61 +2467,26 @@ dependencies = [ ] [[package]] -name = "malachite" -version = "0.4.22" +name = "manyhow" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fbdf9cb251732db30a7200ebb6ae5d22fe8e11397364416617d2c2cf0c51cb5" +checksum = "8c69ea6e469ca785c846b25dd69eac9e9c916b51d8ada1f212866787ee1da12d" dependencies = [ - "malachite-base", - "malachite-nz", - "malachite-q", + "manyhow-macros", + "proc-macro2", + "quote", + "syn 3.0.7", ] [[package]] -name = "malachite-base" -version = "0.4.22" +name = "manyhow-macros" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea0ed76adf7defc1a92240b5c36d5368cfe9251640dcce5bd2d0b7c1fd87aeb" +checksum = "22fc5c1b2c4000056dae3619fea36e79ce79dabfee4907d2dcad6612b189e370" dependencies = [ - "hashbrown 0.14.5", - "itertools 0.11.0", - "libm", - "ryu", -] - -[[package]] -name = "malachite-bigint" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d149aaa2965d70381709d9df4c7ee1fc0de1c614a4efc2ee356f5e43d68749f8" -dependencies = [ - "derive_more", - "malachite", - "num-integer", - "num-traits", - "paste", -] - -[[package]] -name = "malachite-nz" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34a79feebb2bc9aa7762047c8e5495269a367da6b5a90a99882a0aeeac1841f7" -dependencies = [ - "itertools 0.11.0", - "libm", - "malachite-base", -] - -[[package]] -name = "malachite-q" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f235d5747b1256b47620f5640c2a17a88c7569eebdf27cd9cb130e1a619191" -dependencies = [ - "itertools 0.11.0", - "malachite-base", - "malachite-nz", + "proc-macro-utils", + "proc-macro2", + "quote", ] [[package]] @@ -2949,6 +2993,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "ordermap" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a756ddcdeb6c0efc8927bdf0ed112885815a35ba350472a467be5104f01ca1" +dependencies = [ + "indexmap", +] + [[package]] name = "owo-colors" version = "4.3.0" @@ -2984,12 +3037,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - [[package]] name = "pem" version = "3.0.6" @@ -3286,6 +3333,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "proc-macro-utils" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeaf08a13de400bc215877b5bdc088f241b12eb42f0a548d3390dc1c56bb7071" +dependencies = [ + "proc-macro2", + "quote", + "smallvec", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -3401,9 +3459,9 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.1.1", + "rustc-hash", "rustls", - "socket2 0.6.3", + "socket2", "thiserror 2.0.18", "tokio", "tracing", @@ -3422,7 +3480,7 @@ dependencies = [ "lru-slab", "rand 0.9.5", "ring", - "rustc-hash 2.1.1", + "rustc-hash", "rustls", "rustls-pki-types", "slab", @@ -3441,7 +3499,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2", "tracing", "windows-sys 0.60.2", ] @@ -3455,6 +3513,28 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "quote-use" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48356e493823dc3c1d7e9fc7b3ff204ecec48afbdd718fc4134091b88eb63f3d" +dependencies = [ + "quote", + "quote-use-macros", +] + +[[package]] +name = "quote-use-macros" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf1c5de6a7cc231c71e03c77aa4c7019097859ea7029156b54d2894a8430704" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", + "syn 3.0.7", +] + [[package]] name = "r-efi" version = "5.3.0" @@ -3835,6 +3915,106 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ruff_python_ast" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15434b06df7d930fad2818639a4e0fec4af9c770121c45a96b72da7feaae9ead" +dependencies = [ + "aho-corasick", + "arrayvec", + "bitflags", + "char_str", + "compact_str", + "get-size2", + "is-macro", + "memchr", + "ruff_python_trivia", + "ruff_source_file", + "ruff_text_size", + "rustc-hash", + "thin-vec", + "thiserror 2.0.18", +] + +[[package]] +name = "ruff_python_parser" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37a22811d752aa9e255d7dd72524d14352cf9bc654ef79b399bf1eeeb6640fbb" +dependencies = [ + "bitflags", + "bstr", + "drop_bomb", + "get-size2", + "hashbrown 0.17.1", + "memchr", + "ruff_python_ast", + "ruff_python_trivia", + "ruff_text_size", + "ruff_unicode_names2", + "rustc-hash", + "stacker", + "static_assertions", + "thin-vec", + "unicode-ident", + "unicode-normalization", +] + +[[package]] +name = "ruff_python_trivia" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0322a5059fe0feb2e010ca6f31c24ac289f6b7e9e38359539491b10c57f1c5bc" +dependencies = [ + "itertools 0.15.0", + "memchr", + "ruff_source_file", + "ruff_text_size", + "rustc-hash", + "unicode-ident", + "unicode-normalization", +] + +[[package]] +name = "ruff_source_file" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6ae809a289b5c791f82aa45df33f2a92323e7a1c1d265c10e9ac884027a2e43" +dependencies = [ + "memchr", + "ruff_text_size", +] + +[[package]] +name = "ruff_text_size" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf6344d6915ced9316312c1ee3f24c4300c54295bc4c3b8f4e95b633ae5443b" +dependencies = [ + "get-size2", +] + +[[package]] +name = "ruff_unicode_names2" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13418b8fd029d597e35dedc36e91f47122d26075e4c86557574d9441753fb375" +dependencies = [ + "phf 0.11.3", + "ruff_unicode_names2_generator", +] + +[[package]] +name = "ruff_unicode_names2_generator" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37fd82af49d5685d9054e68185987acaaf62542f671b6df8935155664876bc39" +dependencies = [ + "phf_codegen", + "rand 0.8.6", +] + [[package]] name = "rust_decimal" version = "1.40.0" @@ -3857,12 +4037,6 @@ version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc-hash" version = "2.1.1" @@ -3971,63 +4145,6 @@ dependencies = [ "untrusted 0.9.0", ] -[[package]] -name = "rustpython-ast" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cdaf8ee5c1473b993b398c174641d3aa9da847af36e8d5eb8291930b72f31a5" -dependencies = [ - "is-macro", - "malachite-bigint", - "rustpython-parser-core", - "static_assertions", -] - -[[package]] -name = "rustpython-parser" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "868f724daac0caf9bd36d38caf45819905193a901e8f1c983345a68e18fb2abb" -dependencies = [ - "anyhow", - "is-macro", - "itertools 0.11.0", - "lalrpop-util", - "log", - "malachite-bigint", - "num-traits", - "phf 0.11.3", - "phf_codegen", - "rustc-hash 1.1.0", - "rustpython-ast", - "rustpython-parser-core", - "tiny-keccak", - "unic-emoji-char", - "unic-ucd-ident", - "unicode_names2", -] - -[[package]] -name = "rustpython-parser-core" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4b6c12fa273825edc7bccd9a734f0ad5ba4b8a2f4da5ff7efe946f066d0f4ad" -dependencies = [ - "is-macro", - "memchr", - "rustpython-parser-vendored", -] - -[[package]] -name = "rustpython-parser-vendored" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04fcea49a4630a3a5d940f4d514dc4f575ed63c14c3e3ed07146634aed7f67a6" -dependencies = [ - "memchr", - "once_cell", -] - [[package]] name = "rustversion" version = "1.0.22" @@ -4040,6 +4157,12 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "ryu-js" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04d056b875a9d2e6cb9a61d127afee9ac5999b9f87bcb32079d1318e505be714" + [[package]] name = "same-file" version = "1.0.6" @@ -4409,16 +4532,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "socket2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "socket2" version = "0.6.3" @@ -4680,15 +4793,15 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stacker" -version = "0.1.23" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d74a23609d509411d10e2176dc2a4346e3b4aea2e7b1869f19fdedbc71c013" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" dependencies = [ "cc", "cfg-if", "libc", "psm", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4830,7 +4943,7 @@ dependencies = [ "new_debug_unreachable", "num-bigint", "once_cell", - "rustc-hash 2.1.1", + "rustc-hash", "serde", "siphasher 0.3.11", "swc_atoms", @@ -4852,7 +4965,7 @@ dependencies = [ "num-bigint", "once_cell", "phf 0.11.3", - "rustc-hash 2.1.1", + "rustc-hash", "string_enum", "swc_atoms", "swc_common", @@ -4871,7 +4984,7 @@ dependencies = [ "either", "num-bigint", "phf 0.11.3", - "rustc-hash 2.1.1", + "rustc-hash", "seq-macro", "serde", "smallvec", @@ -4974,6 +5087,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d62a2e0561533f2ca2561d0cf27fd9fedb640a1bf2616ff5d5c80d99017faadc" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -5102,6 +5226,12 @@ dependencies = [ "unicode-width 0.2.2", ] +[[package]] +name = "thin-vec" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6a4b9ba8738cb4a4f399d37e266becfd475e75eb73425b87a05a2f2039ba63e" + [[package]] name = "thiserror" version = "1.0.69" @@ -5193,15 +5323,6 @@ dependencies = [ "time-core", ] -[[package]] -name = "tiny-keccak" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" -dependencies = [ - "crunchy", -] - [[package]] name = "tinystr" version = "0.8.2" @@ -5229,9 +5350,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.46.1" +version = "1.47.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" +checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" dependencies = [ "backtrace", "bytes", @@ -5242,10 +5363,10 @@ dependencies = [ "pin-project-lite", "signal-hook-registry", "slab", - "socket2 0.5.10", + "socket2", "tokio-macros", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5289,7 +5410,7 @@ dependencies = [ "postgres-protocol", "postgres-types", "rand 0.10.1", - "socket2 0.6.3", + "socket2", "tokio", "tokio-util", "whoami 2.1.0", @@ -5325,6 +5446,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "futures-util", "pin-project-lite", "tokio", ] @@ -5663,17 +5785,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" -[[package]] -name = "unic-emoji-char" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b07221e68897210270a38bde4babb655869637af0f69407f96053a34f76494d" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - [[package]] name = "unic-ucd-category" version = "0.9.0" @@ -5686,17 +5797,6 @@ dependencies = [ "unic-ucd-version", ] -[[package]] -name = "unic-ucd-ident" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e230a37c0381caa9219d67cf063aa3a375ffed5bf541a452db16e744bdab6987" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - [[package]] name = "unic-ucd-version" version = "0.9.0" @@ -5775,28 +5875,6 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" -[[package]] -name = "unicode_names2" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1673eca9782c84de5f81b82e4109dcfb3611c8ba0d52930ec4a9478f547b2dd" -dependencies = [ - "phf 0.11.3", - "unicode_names2_generator", -] - -[[package]] -name = "unicode_names2_generator" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91e5b84611016120197efd7dc93ef76774f4e084cd73c9fb3ea4a86c570c56e" -dependencies = [ - "getopts", - "log", - "phf_codegen", - "rand 0.8.6", -] - [[package]] name = "unscanny" version = "0.1.0" @@ -6260,12 +6338,14 @@ dependencies = [ "pin-project-lite", "pkcs1", "postgres-native-tls", + "postgres-protocol", "quick_cache", "rand 0.9.5", "regex", "reqwest", "reqwest-middleware", "reqwest-retry", + "ryu-js", "semver", "serde", "serde_json", @@ -6409,8 +6489,8 @@ version = "1.830.0" dependencies = [ "anyhow", "itertools 0.14.0", - "rustpython-ast", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", "serde_json", "windmill-parser", ] @@ -6420,8 +6500,8 @@ name = "windmill-parser-py-asset" version = "1.830.0" dependencies = [ "anyhow", - "rustpython-ast", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", "windmill-parser", "windmill-parser-sql-asset", ] @@ -6438,7 +6518,8 @@ dependencies = [ "phf 0.11.3", "regex", "regex-lite", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", "serde", "serde_json", "sqlx", @@ -6555,8 +6636,9 @@ name = "windmill-parser-wac" version = "1.830.0" dependencies = [ "anyhow", - "rustpython-ast", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", + "ruff_text_size", "serde", "serde_json", "sha2 0.10.9",