From 02fe2e75116c9389335117a4da18fdd96ba0f423 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 1 May 2026 10:57:17 +0000 Subject: [PATCH 01/21] chore(main): release 1.693.4 (#8994) * chore(main): release 1.693.4 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 7 + backend/Cargo.lock | 156 +++++++++--------- backend/Cargo.toml | 4 +- .../parsers/windmill-parser-wasm/Cargo.lock | 48 +++--- .../parsers/windmill-parser-wasm/Cargo.toml | 2 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/main.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 2 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 17 files changed, 125 insertions(+), 118 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10588505fa..7239633716 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [1.693.4](https://github.com/windmill-labs/windmill/compare/v1.693.3...v1.693.4) (2026-04-30) + + +### Bug Fixes + +* **cli:** pin wasm parser versions in published package.json ([#8993](https://github.com/windmill-labs/windmill/issues/8993)) ([7c227ec](https://github.com/windmill-labs/windmill/commit/7c227ece0d546ef0f521a963f927e4c913011e8e)) + ## [1.693.3](https://github.com/windmill-labs/windmill/compare/v1.693.2...v1.693.3) (2026-04-30) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 3d250d1b7c..d5c0b09b12 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -16020,7 +16020,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "async-nats", @@ -16101,7 +16101,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.693.3" +version = "1.693.4" dependencies = [ "async-trait", "aws-config", @@ -16125,7 +16125,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.693.3" +version = "1.693.4" dependencies = [ "axum 0.8.4", "chrono", @@ -16138,7 +16138,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "argon2", @@ -16281,7 +16281,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.693.3" +version = "1.693.4" dependencies = [ "axum 0.8.4", "chrono", @@ -16304,7 +16304,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.693.3" +version = "1.693.4" dependencies = [ "axum 0.8.4", "chrono", @@ -16317,7 +16317,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "axum 0.8.4", @@ -16343,7 +16343,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.693.3" +version = "1.693.4" dependencies = [ "reqwest 0.12.28", "serde", @@ -16353,7 +16353,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.693.3" +version = "1.693.4" dependencies = [ "axum 0.8.4", "chrono", @@ -16370,7 +16370,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.693.3" +version = "1.693.4" dependencies = [ "axum 0.8.4", "base64 0.22.1", @@ -16392,7 +16392,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "axum 0.8.4", @@ -16415,7 +16415,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.693.3" +version = "1.693.4" dependencies = [ "axum 0.8.4", "chrono", @@ -16431,7 +16431,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.693.3" +version = "1.693.4" dependencies = [ "axum 0.8.4", "chrono", @@ -16452,7 +16452,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.693.3" +version = "1.693.4" dependencies = [ "axum 0.8.4", "chrono", @@ -16473,7 +16473,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.693.3" +version = "1.693.4" dependencies = [ "axum 0.8.4", "chrono", @@ -16487,7 +16487,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "async-nats", @@ -16519,7 +16519,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "axum 0.8.4", @@ -16544,7 +16544,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.693.3" +version = "1.693.4" dependencies = [ "axum 0.8.4", "flate2", @@ -16562,7 +16562,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "axum 0.8.4", @@ -16584,7 +16584,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.693.3" +version = "1.693.4" dependencies = [ "axum 0.8.4", "chrono", @@ -16604,7 +16604,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.693.3" +version = "1.693.4" dependencies = [ "axum 0.8.4", "chrono", @@ -16634,7 +16634,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "axum 0.8.4", @@ -16662,7 +16662,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.693.3" +version = "1.693.4" dependencies = [ "lazy_static", "serde", @@ -16674,7 +16674,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.693.3" +version = "1.693.4" dependencies = [ "argon2", "axum 0.8.4", @@ -16699,7 +16699,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.693.3" +version = "1.693.4" dependencies = [ "axum 0.8.4", "chrono", @@ -16713,7 +16713,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.693.3" +version = "1.693.4" dependencies = [ "axum 0.8.4", "chrono", @@ -16746,7 +16746,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.693.3" +version = "1.693.4" dependencies = [ "chrono", "lazy_static", @@ -16760,7 +16760,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "axum 0.8.4", @@ -16779,7 +16779,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.693.3" +version = "1.693.4" dependencies = [ "aes-gcm", "aho-corasick", @@ -16880,7 +16880,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.693.3" +version = "1.693.4" dependencies = [ "chrono", "itertools 0.14.0", @@ -16899,7 +16899,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.693.3" +version = "1.693.4" dependencies = [ "regex", "serde", @@ -16914,7 +16914,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "astral-tokio-tar", @@ -16938,7 +16938,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "futures", @@ -16955,7 +16955,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.693.3" +version = "1.693.4" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -16971,7 +16971,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "async-trait", @@ -16992,7 +16992,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "async-trait", @@ -17023,7 +17023,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "arc-swap", @@ -17048,7 +17048,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "async-stream", @@ -17082,7 +17082,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "futures", @@ -17100,7 +17100,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.693.3" +version = "1.693.4" dependencies = [ "convert_case 0.6.0", "serde", @@ -17109,7 +17109,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "lazy_static", @@ -17121,7 +17121,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "serde_json", @@ -17133,7 +17133,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "gosyn", @@ -17145,7 +17145,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "lazy_static", @@ -17157,7 +17157,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "serde_json", @@ -17169,7 +17169,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "nu-parser", @@ -17180,7 +17180,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "itertools 0.14.0", @@ -17191,7 +17191,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "itertools 0.14.0", @@ -17203,7 +17203,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "rustpython-ast", @@ -17214,7 +17214,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "async-recursion", @@ -17236,7 +17236,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "serde_json", @@ -17248,7 +17248,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "lazy_static", @@ -17262,7 +17262,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -17279,7 +17279,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "lazy_static", @@ -17292,7 +17292,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "serde", @@ -17304,7 +17304,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "lazy_static", @@ -17322,7 +17322,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -17338,7 +17338,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "rustpython-ast", @@ -17354,7 +17354,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "serde", @@ -17365,7 +17365,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "async-recursion", @@ -17402,7 +17402,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "const_format", @@ -17440,7 +17440,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.693.3" +version = "1.693.4" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -17451,7 +17451,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "async-recursion", @@ -17481,7 +17481,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "async-trait", @@ -17505,7 +17505,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "async-trait", @@ -17538,7 +17538,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "async-trait", @@ -17571,7 +17571,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "async-trait", @@ -17591,7 +17591,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "async-trait", @@ -17625,7 +17625,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "async-trait", @@ -17661,7 +17661,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "async-trait", @@ -17684,7 +17684,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "async-trait", @@ -17708,7 +17708,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "async-nats", @@ -17732,7 +17732,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "async-trait", @@ -17767,7 +17767,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "async-trait", @@ -17795,7 +17795,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "async-trait", @@ -17818,7 +17818,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "bitflags 2.9.4", @@ -17837,7 +17837,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "async-once-cell", @@ -17949,7 +17949,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.693.3" +version = "1.693.4" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index f1b379e97e..a95d6d1f37 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.693.3" +version = "1.693.4" authors.workspace = true edition.workspace = true @@ -87,7 +87,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.693.3" +version = "1.693.4" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index ce541ce667..05d6baf815 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6183,7 +6183,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.693.3" +version = "1.693.4" dependencies = [ "aho-corasick", "anyhow", @@ -6263,7 +6263,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.693.3" +version = "1.693.4" dependencies = [ "proc-macro2", "quote", @@ -6275,7 +6275,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.693.3" +version = "1.693.4" dependencies = [ "convert_case", "serde", @@ -6284,7 +6284,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "lazy_static", @@ -6296,7 +6296,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "serde_json", @@ -6308,7 +6308,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "gosyn", @@ -6320,7 +6320,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "lazy_static", @@ -6332,7 +6332,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "serde_json", @@ -6344,7 +6344,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "nu-parser", @@ -6355,7 +6355,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6366,7 +6366,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6378,7 +6378,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "rustpython-ast", @@ -6389,7 +6389,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "async-recursion", @@ -6411,7 +6411,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "serde_json", @@ -6423,7 +6423,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "lazy_static", @@ -6437,7 +6437,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "convert_case", @@ -6454,7 +6454,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "lazy_static", @@ -6467,7 +6467,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "serde", @@ -6479,7 +6479,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "lazy_static", @@ -6497,7 +6497,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6513,7 +6513,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "rustpython-ast", @@ -6529,7 +6529,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6561,7 +6561,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "serde", @@ -6572,7 +6572,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.693.3" +version = "1.693.4" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 45ec8ca078..39d5655d45 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.693.3" +version = "1.693.4" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 92bb97046a..c4ac9b4c94 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.693.3 + version: 1.693.4 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 4419cd109d..1689b1bf30 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.693.3"; +export const VERSION = "v1.693.4"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/main.ts b/cli/src/main.ts index b05cbba921..cfb373d0c7 100755 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -78,7 +78,7 @@ export { token, }; -export const VERSION = "1.693.3"; +export const VERSION = "1.693.4"; // Re-exported from constants.ts to maintain backwards compatibility export { WM_FORK_PREFIX } from "./core/constants.ts"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 66541f03e9..e346c9c811 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.693.3", + "version": "1.693.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.693.3", + "version": "1.693.4", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index c0b49c4905..fae6f4cce1 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.693.3", + "version": "1.693.4", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/lsp/Pipfile b/lsp/Pipfile index 057c4fd717..5108baaaea 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.693.3" +wmill = ">=1.693.4" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index c45b270dc5..102051b6f0 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.693.3 + version: 1.693.4 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 1beaa5ad13..096962d458 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.693.3' + ModuleVersion = '1.693.4' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 835fd5ee61..3b709cde64 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.693.3" +version = "1.693.4" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 6cce29bb6f..45f1d7d40b 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.693.3", + "version": "1.693.4", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 5d3fdd8d43..57bc33cbe9 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.693.3", + "version": "1.693.4", "author": "Ruben Fiszel", "license": "Apache 2.0", "sideEffects": false, diff --git a/version.txt b/version.txt index d6e039b4cd..62fb18d7de 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.693.3 +1.693.4 From 96324ea5aed4054d33895102ec9313f5dadc77a2 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 1 May 2026 12:12:04 +0000 Subject: [PATCH 02/21] feat: ansible delegate_to_git_repo install_requirements, dynamic fields, --limit (#8997) * feat: ansible delegate_to_git_repo install_requirements, dynamic fields, --limit Co-Authored-By: Claude Opus 4.5 * fix: include .yaml variants in collections/roles requirements lookup Co-Authored-By: Claude Opus 4.5 --------- Co-authored-by: Claude Opus 4.5 --- .../parsers/windmill-parser-yaml/src/lib.rs | 76 ++++ .../windmill-worker/src/ansible_executor.rs | 367 +++++++++++++++++- 2 files changed, 426 insertions(+), 17 deletions(-) diff --git a/backend/parsers/windmill-parser-yaml/src/lib.rs b/backend/parsers/windmill-parser-yaml/src/lib.rs index 887f94fa38..39dd504240 100644 --- a/backend/parsers/windmill-parser-yaml/src/lib.rs +++ b/backend/parsers/windmill-parser-yaml/src/lib.rs @@ -215,6 +215,7 @@ pub struct AnsiblePlaybookOptions { pub timeout: Option, pub flush_cache: Option<()>, pub force_handlers: Option<()>, + pub limit: Option, } #[derive(Debug, Clone)] @@ -265,6 +266,8 @@ pub struct DelegateToGitRepoDetails { pub commit: Option, pub inventories_location: Option, pub vars_location: Option, + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub install_requirements: bool, } #[derive(Debug, Clone)] @@ -300,6 +303,7 @@ impl Default for AnsibleRequirements { timeout: None, flush_cache: None, force_handlers: None, + limit: None, }, vault_password: None, vault_id: vec![], @@ -602,6 +606,10 @@ fn extract_delegate_to_git_repo_details(value: &Yaml) -> Option Option) -> AnsiblePlaybookOptions { timeout: None, flush_cache: None, force_handlers: None, + limit: None, }; for opt in opts { if let Yaml::String(o) = opt { @@ -691,6 +701,13 @@ fn parse_ansible_options(opts: &Vec) -> AnsiblePlaybookOptions { } } } + "limit" => { + if let Yaml::String(limit) = value { + if !limit.is_empty() { + ret.limit = Some(limit.clone()); + } + } + } _ => (), } } @@ -972,4 +989,63 @@ dependencies: let a = parse_delegate_to_git_repo(p).unwrap(); println!("The resulting delegate_to_kit_repo is: {:#?}", a); } + + #[test] + fn test_parse_options_limit() { + let p = r#" +--- +options: + - vv + - limit: webservers:!db1.example.com + - forks: 5 +--- +- name: Test + hosts: all +"#; + let (_, reqs, _) = parse_ansible_reqs(p).unwrap(); + let opts = reqs.unwrap().options; + assert_eq!(opts.limit.as_deref(), Some("webservers:!db1.example.com")); + assert_eq!(opts.verbosity.as_deref(), Some("vv")); + assert_eq!(opts.forks, Some(5)); + } + + #[test] + fn test_parse_install_requirements_default_false() { + let p = r#" +--- +delegate_to_git_repo: + resource: u/admin/repo + playbook: site.yml +--- +- name: Test + hosts: all +"#; + let (_, reqs, _) = parse_ansible_reqs(p).unwrap(); + let d = reqs.unwrap().delegate_to_git_repo.unwrap(); + assert!(!d.install_requirements); + assert_eq!(d.playbook.as_deref(), Some("site.yml")); + } + + #[test] + fn test_parse_install_requirements_true() { + let p = r#" +--- +delegate_to_git_repo: + resource: u/admin/repo + playbook: "{{ playbook_name }}" + inventories_location: "inventories/{{ env }}" + install_requirements: true +--- +- name: Test + hosts: all +"#; + let (_, reqs, _) = parse_ansible_reqs(p).unwrap(); + let d = reqs.unwrap().delegate_to_git_repo.unwrap(); + assert!(d.install_requirements); + assert_eq!(d.playbook.as_deref(), Some("{{ playbook_name }}")); + assert_eq!( + d.inventories_location.as_deref(), + Some("inventories/{{ env }}") + ); + } } diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index bc16f82813..c8289b6934 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -51,6 +51,93 @@ lazy_static::lazy_static! { const NSJAIL_CONFIG_RUN_ANSIBLE_CONTENT: &str = include_str!("../nsjail/run.ansible.config.proto"); const WINDMILL_ANSIBLE_PASSWORD_FILENAME: &str = ".windmill.ansible_vault_password_file"; +const DELEGATE_GIT_REPO_TARGET: &str = "delegate_git_repository"; + +lazy_static::lazy_static! { + static ref TEMPLATE_RE: regex::Regex = regex::Regex::new(r"\{\{\s*([A-Za-z_][A-Za-z0-9_]*)\s*\}\}").unwrap(); +} + +/// Substitute `{{ arg_name }}` placeholders with values from `args`. +/// Strings are used raw; numbers/bools are stringified. Other types are rejected. +fn interpolate_template( + template: &str, + args: Option<&HashMap>>, + field_name: &str, +) -> error::Result { + let mut last_err: Option = None; + let result = TEMPLATE_RE.replace_all(template, |caps: ®ex::Captures| { + let name = &caps[1]; + let raw = args.and_then(|a| a.get(name)); + let Some(raw) = raw else { + last_err = Some(error::Error::BadRequest(format!( + "`{}` references `{{{{ {} }}}}` but no such argument was provided", + field_name, name + ))); + return String::new(); + }; + let json: serde_json::Value = match serde_json::from_str(raw.get()) { + Ok(v) => v, + Err(e) => { + last_err = Some(error::Error::BadRequest(format!( + "`{}` could not parse argument `{}` as JSON: {e}", + field_name, name + ))); + return String::new(); + } + }; + match json { + serde_json::Value::String(s) => s, + serde_json::Value::Number(n) => n.to_string(), + serde_json::Value::Bool(b) => b.to_string(), + serde_json::Value::Null => { + last_err = Some(error::Error::BadRequest(format!( + "`{}` references `{{{{ {} }}}}` but the argument is null", + field_name, name + ))); + String::new() + } + _ => { + last_err = Some(error::Error::BadRequest(format!( + "`{}` references `{{{{ {} }}}}` but the argument is not a primitive (string/number/bool)", + field_name, name + ))); + String::new() + } + } + }); + if let Some(e) = last_err { + return Err(e); + } + Ok(result.into_owned()) +} + +/// Reject absolute paths and `..` segments to prevent escaping the cloned repo directory. +fn validate_relative_path(path: &str, field_name: &str) -> error::Result<()> { + let trimmed = path.trim(); + if trimmed.is_empty() { + return Err(error::Error::BadRequest(format!( + "`{}` resolved to an empty path", + field_name + ))); + } + let p = std::path::Path::new(trimmed); + if p.is_absolute() { + return Err(error::Error::BadRequest(format!( + "`{}` must be a relative path inside the cloned repo, got: {}", + field_name, trimmed + ))); + } + for component in p.components() { + if matches!(component, std::path::Component::ParentDir) { + return Err(error::Error::BadRequest(format!( + "`{}` must not contain `..` segments, got: {}", + field_name, trimmed + ))); + } + } + Ok(()) +} + async fn clone_repo( repo: &GitRepo, job_dir: &str, @@ -439,6 +526,35 @@ pub async fn install_galaxy_collections( ) .await; + run_galaxy_install_from_requirements( + "requirements.yml", + job_dir, + job_id, + worker_name, + w_id, + mem_peak, + canceled_by, + conn, + occupancy_metrics, + git_ssh_cmd, + ) + .await +} + +/// Run `ansible-galaxy role install -r ` then `ansible-galaxy collection install -r `. +/// `requirements_path` is relative to `job_dir`. +async fn run_galaxy_install_from_requirements( + requirements_path: &str, + job_dir: &str, + job_id: &Uuid, + worker_name: &str, + w_id: &str, + mem_peak: &mut i32, + canceled_by: &mut Option, + conn: &Connection, + occupancy_metrics: &mut OccupancyMetrics, + git_ssh_cmd: &str, +) -> anyhow::Result<()> { let mut galaxy_roles_cmd = Command::new(ANSIBLE_GALAXY_PATH.as_str()); galaxy_roles_cmd .current_dir(job_dir) @@ -451,7 +567,7 @@ pub async fn install_galaxy_collections( "role", "install", "-r", - "requirements.yml", + requirements_path, "-p", "./roles", ]) @@ -489,7 +605,7 @@ pub async fn install_galaxy_collections( "collection", "install", "-r", - "requirements.yml", + requirements_path, "-p", "./", ]) @@ -519,6 +635,82 @@ pub async fn install_galaxy_collections( Ok(()) } +/// Look for `requirements.yml`, `collections/requirements.yml`, and `roles/requirements.yml` +/// inside a cloned repo (relative to `job_dir`) and run ansible-galaxy install on each one found. +async fn install_requirements_from_cloned_repo( + repo_target: &str, + job_dir: &str, + job_id: &Uuid, + worker_name: &str, + w_id: &str, + mem_peak: &mut i32, + canceled_by: &mut Option, + conn: &Connection, + occupancy_metrics: &mut OccupancyMetrics, + git_ssh_cmd: &str, +) -> anyhow::Result<()> { + let candidates = [ + "requirements.yml", + "requirements.yaml", + "collections/requirements.yml", + "collections/requirements.yaml", + "roles/requirements.yml", + "roles/requirements.yaml", + ]; + let mut found: Vec = vec![]; + for candidate in candidates { + let abs = std::path::Path::new(job_dir) + .join(repo_target) + .join(candidate); + if abs.is_file() { + found.push(format!("{}/{}", repo_target, candidate)); + } + } + + if found.is_empty() { + append_logs( + job_id, + w_id, + format!( + "\nNo requirements.yml found in `{}`, skipping repo dependency install.\n", + repo_target + ), + conn, + ) + .await; + return Ok(()); + } + + append_logs( + job_id, + w_id, + format!( + "\n\n--- INSTALLING REPO REQUIREMENTS ({}) ---\n", + found.join(", ") + ), + conn, + ) + .await; + + for path in &found { + run_galaxy_install_from_requirements( + path, + job_dir, + job_id, + worker_name, + w_id, + mem_peak, + canceled_by, + conn, + occupancy_metrics, + git_ssh_cmd, + ) + .await?; + } + + Ok(()) +} + #[derive(Serialize, Deserialize)] pub struct AnsibleDependencyLocks { pub python_lockfile: String, @@ -882,13 +1074,21 @@ pub async fn handle_ansible_job( }; write_file(job_dir, "result.json", "")?; - let cmd_options: Vec = reqs - .as_ref() - .map(|r| r.options.clone()) - .map(|r| get_cmd_options(r)) - .unwrap_or_default(); + let cmd_options: Vec = if let Some(r) = reqs.as_ref() { + let mut opts = r.options.clone(); + if let Some(limit) = opts.limit.as_ref() { + opts.limit = Some(interpolate_template( + limit, + interpolated_args.as_ref(), + "options.limit", + )?); + } + get_cmd_options(opts) + } else { + vec![] + }; - let inventories: Vec = reqs + let mut inventories: Vec = reqs .as_ref() .map(|x| -> Result, _> { let mut ret: Vec = x @@ -938,6 +1138,44 @@ pub async fn handle_ansible_job( .await?; if let Some(delegated_git_repo) = r.delegate_to_git_repo.as_ref() { + let interpolated_playbook = delegated_git_repo + .playbook + .as_ref() + .map(|p| -> error::Result { + let p = interpolate_template( + p, + interpolated_args.as_ref(), + "delegate_to_git_repo.playbook", + )?; + validate_relative_path(&p, "delegate_to_git_repo.playbook")?; + Ok(p) + }) + .transpose()?; + let interpolated_commit = delegated_git_repo + .commit + .as_ref() + .map(|c| { + interpolate_template( + c, + interpolated_args.as_ref(), + "delegate_to_git_repo.commit", + ) + }) + .transpose()?; + let interpolated_inventories_location = delegated_git_repo + .inventories_location + .as_ref() + .map(|p| -> error::Result { + let p = interpolate_template( + p, + interpolated_args.as_ref(), + "delegate_to_git_repo.inventories_location", + )?; + validate_relative_path(&p, "delegate_to_git_repo.inventories_location")?; + Ok(p) + }) + .transpose()?; + let serde_json::Value::Object(git_repo_resource) = client .get_resource_value_interpolated::( &delegated_git_repo.resource, @@ -974,11 +1212,11 @@ pub async fn handle_ansible_job( let branch = Some(git_repo_resource.get("branch").and_then(|s| s.as_str()).map(|s| s.to_string()) .ok_or(anyhow!("Failed to get branch from git repo resource, please check that the resource has the correct type (git_repository)"))?).filter(|s| !s.is_empty()); - let target_path = "delegate_git_repository".to_string(); + let target_path = DELEGATE_GIT_REPO_TARGET.to_string(); let repo = GitRepo { url: secret_url, - commit: delegated_git_repo.commit.clone(), + commit: interpolated_commit.clone(), branch, target_path, }; @@ -989,7 +1227,7 @@ pub async fn handle_ansible_job( conn, ) .await; - if let Some(commit) = delegated_git_repo.commit.as_ref() { + if let Some(commit) = interpolated_commit.as_ref() { clone_repo_without_history( &repo, commit, @@ -1033,12 +1271,29 @@ pub async fn handle_ansible_job( ) .await; - playbook_override = Some( - delegated_git_repo - .playbook - .as_ref() - .map(|p| format!("{}/{}", &repo.target_path, p)), - ); + playbook_override = + Some(interpolated_playbook.map(|p| format!("{}/{}", &repo.target_path, p))); + + if let Some(inv) = interpolated_inventories_location { + inventories.push("-i".to_string()); + inventories.push(format!("{}/{}", &repo.target_path, inv)); + } + + if delegated_git_repo.install_requirements { + install_requirements_from_cloned_repo( + &repo.target_path, + job_dir, + &job.id, + worker_name, + &job.workspace_id, + mem_peak, + canceled_by, + conn, + occupancy_metrics, + git_ssh_cmd, + ) + .await?; + } } if playbook_override.clone().flatten().is_none() && playbook.is_empty() { @@ -1322,6 +1577,11 @@ fn get_cmd_options(r: windmill_parser_yaml::AnsiblePlaybookOptions) -> Vec client.get_variable_value(&p).await?, }) } + +#[cfg(test)] +mod tests { + use super::*; + + fn args_from_json(v: serde_json::Value) -> HashMap> { + let serde_json::Value::Object(map) = v else { + panic!("expected object"); + }; + map.into_iter() + .map(|(k, v)| (k, RawValue::from_string(v.to_string()).unwrap())) + .collect() + } + + #[test] + fn test_interpolate_template_string() { + let args = args_from_json(serde_json::json!({"playbook": "site.yml"})); + let out = + interpolate_template("playbooks/{{ playbook }}", Some(&args), "playbook").unwrap(); + assert_eq!(out, "playbooks/site.yml"); + } + + #[test] + fn test_interpolate_template_number_and_bool() { + let args = args_from_json(serde_json::json!({"n": 42, "b": true})); + let out = interpolate_template("{{ n }}-{{ b }}", Some(&args), "x").unwrap(); + assert_eq!(out, "42-true"); + } + + #[test] + fn test_interpolate_template_no_placeholders() { + let out = interpolate_template("plain.yml", None, "playbook").unwrap(); + assert_eq!(out, "plain.yml"); + } + + #[test] + fn test_interpolate_template_missing_arg_errors() { + let args = args_from_json(serde_json::json!({})); + let err = interpolate_template("{{ missing }}", Some(&args), "playbook").unwrap_err(); + assert!(err.to_string().contains("missing")); + } + + #[test] + fn test_interpolate_template_object_arg_errors() { + let args = args_from_json(serde_json::json!({"o": {"k": "v"}})); + let err = interpolate_template("{{ o }}", Some(&args), "x").unwrap_err(); + assert!(err.to_string().contains("not a primitive")); + } + + #[test] + fn test_validate_relative_path_ok() { + validate_relative_path("playbooks/site.yml", "playbook").unwrap(); + validate_relative_path("./site.yml", "playbook").unwrap(); + validate_relative_path("a/b/c.yml", "playbook").unwrap(); + } + + #[test] + fn test_validate_relative_path_rejects_absolute() { + assert!(validate_relative_path("/etc/passwd", "playbook").is_err()); + } + + #[test] + fn test_validate_relative_path_rejects_parent_dir() { + assert!(validate_relative_path("../escape.yml", "playbook").is_err()); + assert!(validate_relative_path("a/../../escape.yml", "playbook").is_err()); + } + + #[test] + fn test_validate_relative_path_rejects_empty() { + assert!(validate_relative_path("", "playbook").is_err()); + assert!(validate_relative_path(" ", "playbook").is_err()); + } +} From 5d5b853f70a73453f63d14edcd5d2fac8e3d804c Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 1 May 2026 14:52:55 +0000 Subject: [PATCH 03/21] fix(cli): only preserve case for raw-app runnableIds, not app/flow summaries (#9000) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(cli): only preserve case for raw-app runnableIds, not app/flow summaries PR #8940 stopped lowercasing in sanitizeForFilesystem to fix #8939, where a raw-app runnableId like CamelCaseTSRunnable produced a CamelCase YAML metadata file but a lowercased code file, making them desync and register as duplicate runnables on push. That fix overshot. sanitizeForFilesystem is also reached by newPathAssigner, which serves normal apps and flows where the input is the script's human summary ("Get Users Data") rather than an identifier. There the on-disk filename is the only artifact — there's no companion YAML to keep in sync — so lowercasing was the right behavior. Removing it changed both the on-disk filename and the !inline reference in app.yaml / flow.yaml from get_users_data.inline_script.ts to Get_Users_Data.inline_script.ts on the next pull, surfacing as unwanted case churn for users updating to 1.693.x. Add a preserveCase option to sanitizeForFilesystem (default false → lowercase). newRawAppPathAssigner opts in; newPathAssigner stays on the default. Update unit tests accordingly and add an end-to-end raw-app round-trip in raw_app_sync.test.ts that pushes a CamelCase backend runnable, pulls it back, and asserts both YAML and code file preserve case with no lowercase orphan. Co-Authored-By: Claude Opus 4.7 (1M context) * test(cli): use readdir for exact-case orphan check on Windows The CamelCase round-trip test used fileExists("camelcasetsrunnable.ts") to assert no lowercase orphan was produced, which false-positives on Windows since the filesystem is case-insensitive and resolves the lookup to the existing CamelCaseTSRunnable.ts. Switch to readdir + toContain so the exact on-disk casing is compared identically on Linux and Windows. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- ..._scripts_failure_preprocessor_unit.test.ts | 6 +- cli/test/raw_app_sync.test.ts | 87 ++++++++++++++++++- cli/test/sync_pull_push.test.ts | 4 +- .../src/path-utils/path-assigner.ts | 12 ++- 4 files changed, 100 insertions(+), 9 deletions(-) diff --git a/cli/test/inline_scripts_failure_preprocessor_unit.test.ts b/cli/test/inline_scripts_failure_preprocessor_unit.test.ts index b9b35f0912..78af37a02e 100644 --- a/cli/test/inline_scripts_failure_preprocessor_unit.test.ts +++ b/cli/test/inline_scripts_failure_preprocessor_unit.test.ts @@ -524,7 +524,7 @@ describe("extractInlineScripts with mapping preserves file paths", () => { const contentScript = scripts.find((s) => !s.is_lock); // Should use assigner path based on summary, not mapped - expect(contentScript!.path).toContain("Get_Users_Data"); + expect(contentScript!.path).toContain("get_users_data"); }); test("mapped modules and unmapped modules coexist", () => { @@ -538,7 +538,7 @@ describe("extractInlineScripts with mapping preserves file paths", () => { const paths = scripts.filter((s) => !s.is_lock).map((s) => s.path); expect(paths[0]).toBe("my_custom_name.ts"); - expect(paths[1]).toContain("Step_B"); // assigner-generated from summary + expect(paths[1]).toContain("step_b"); // assigner-generated from summary }); test("lock path is derived from mapped content path", () => { @@ -560,7 +560,7 @@ describe("extractInlineScripts with mapping preserves file paths", () => { const scripts = extractInlineScripts([mod], {}, "/", "bun"); const lockScript = scripts.find((s) => s.is_lock); - expect(lockScript!.path).toContain("Get_Users_Data"); + expect(lockScript!.path).toContain("get_users_data"); expect(lockScript!.path).toEndWith(".lock"); }); diff --git a/cli/test/raw_app_sync.test.ts b/cli/test/raw_app_sync.test.ts index 95d812511f..ee20bc1947 100644 --- a/cli/test/raw_app_sync.test.ts +++ b/cli/test/raw_app_sync.test.ts @@ -2,7 +2,7 @@ import { expect, test } from "bun:test"; import { withTestBackend } from "./test_backend.ts"; import { addWorkspace } from "../workspace.ts"; import * as path from "node:path"; -import { writeFile, readFile, stat, rm, mkdir } from "node:fs/promises"; +import { writeFile, readFile, stat, rm, mkdir, readdir } from "node:fs/promises"; // ============================================================================= // RAW APP SYNC TESTS @@ -508,3 +508,88 @@ excludes: []`, "utf-8"); expect(hasRawApp).toBeTruthy(); }); }); + +test("Raw App: CamelCase backend runnableId round-trips without duplicates", async () => { + await withTestBackend(async (backend, tempDir) => { + const testWorkspace = { + remote: backend.baseUrl, + workspaceId: backend.workspace, + name: "raw_app_camelcase_test", + token: backend.token + }; + await addWorkspace(testWorkspace, { force: true, configDir: backend.testConfigDir }); + + await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun +includes: + - "**" +excludes: []`, "utf-8"); + + const appDir = path.join(tempDir, "f", "test", "camelcase_app.raw_app"); + const backendDir = path.join(appDir, "backend"); + await mkdir(path.join(tempDir, "f", "test"), { recursive: true }); + await createRawAppOnDisk(appDir); + + // Add a backend runnable whose id contains uppercase letters. The YAML + // metadata file is named `${runnableId}.yaml` and must stay in sync with + // the code file's casing — the bug fixed here desynced them. + await mkdir(backendDir, { recursive: true }); + await writeFile(path.join(backendDir, "CamelCaseTSRunnable.yaml"), "type: inline\n", "utf-8"); + await writeFile( + path.join(backendDir, "CamelCaseTSRunnable.ts"), + `export async function main(x: number): Promise {\n return \`Result: \${x}\`;\n}\n`, + "utf-8" + ); + + const pushResult1 = await backend.runCLICommand( + ["sync", "push", "--yes"], + tempDir, "raw_app_camelcase_test" + ); + expect(pushResult1.code).toEqual(0); + + await rm(appDir, { recursive: true }); + + const pullResult = await backend.runCLICommand( + ["sync", "pull", "--yes"], + tempDir, "raw_app_camelcase_test" + ); + expect(pullResult.code).toEqual(0); + + // YAML and code file must both come back with the original case so + // loadRunnablesFromBackend pairs them as one runnable, not two. + // Use readdir for exact-case comparison: Windows is case-insensitive at + // the filesystem level, so fileExists("camelcasetsrunnable.ts") would + // resolve to CamelCaseTSRunnable.ts and false-positive the orphan check. + const backendEntries = await readdir(backendDir); + expect(backendEntries).toContain("CamelCaseTSRunnable.yaml"); + expect(backendEntries).toContain("CamelCaseTSRunnable.ts"); + expect(backendEntries).not.toContain("camelcasetsrunnable.ts"); + expect(backendEntries).not.toContain("camelcasetsrunnable.yaml"); + + const pulledContent = await readFileContent(path.join(backendDir, "CamelCaseTSRunnable.ts")); + expect(pulledContent).toContain("Result:"); + + // A second push must not surface a duplicate lowercase runnable: the + // dry-run change list should only mention the one app, no orphan code + // file getting registered as a separate inline runnable. + const dryRun = await backend.runCLICommand( + ["sync", "push", "--dry-run", "--json-output"], + tempDir, "raw_app_camelcase_test" + ); + expect(dryRun.code).toEqual(0); + + let jsonOutput: any = null; + try { + jsonOutput = JSON.parse(dryRun.stdout.trim()); + } catch { + const jsonMatch = dryRun.stdout.match(/\{[\s\S]*\}/); + if (jsonMatch) { + try { jsonOutput = JSON.parse(jsonMatch[0]); } catch { /* ignore */ } + } + } + expect(jsonOutput !== null).toBeTruthy(); + const changes = (jsonOutput.changes ?? []) as Array<{ path: string }>; + // No change should reference a lowercased runnable filename. + const lowercased = changes.filter((c) => c.path.includes("camelcasetsrunnable")); + expect(lowercased).toEqual([]); + }); +}); diff --git a/cli/test/sync_pull_push.test.ts b/cli/test/sync_pull_push.test.ts index 2efa6ac5a4..e64af246f2 100644 --- a/cli/test/sync_pull_push.test.ts +++ b/cli/test/sync_pull_push.test.ts @@ -631,10 +631,10 @@ test("newPathAssigner with skipInlineScriptSuffix removes .inline_script. from p expect(noSuffixPyExt).toEqual("py"); }); -test("path assigners preserve mixed-case summaries", () => { +test("newPathAssigner lowercases summaries; newRawAppPathAssigner preserves case", () => { const inlineAssigner = newPathAssigner("bun", { skipInlineScriptSuffix: true }); const [inlinePath] = inlineAssigner.assignPath("CamelCaseTSRunnable", "bun"); - expect(inlinePath).toEqual("CamelCaseTSRunnable."); + expect(inlinePath).toEqual("camelcasetsrunnable."); const rawAssigner = newRawAppPathAssigner("bun"); const [rawPath] = rawAssigner.assignPath("CamelCaseTSRunnable", "bun"); diff --git a/cli/windmill-utils-internal/src/path-utils/path-assigner.ts b/cli/windmill-utils-internal/src/path-utils/path-assigner.ts index 90a70a82a8..94d9085b94 100644 --- a/cli/windmill-utils-internal/src/path-utils/path-assigner.ts +++ b/cli/windmill-utils-internal/src/path-utils/path-assigner.ts @@ -118,8 +118,11 @@ export function getLanguageFromExtension( */ const WINDOWS_RESERVED = /^(con|prn|aux|nul|com[0-9]|lpt[0-9])$/; -export function sanitizeForFilesystem(summary: string): string { - const name = summary +export function sanitizeForFilesystem( + summary: string, + options?: { preserveCase?: boolean }, +): string { + let name = summary .replaceAll(" ", "_") // Remove characters invalid on Windows/Unix/Mac: / \ : * ? " < > | // Also remove control characters (0x00-0x1F) and DEL (0x7F) @@ -129,6 +132,9 @@ export function sanitizeForFilesystem(summary: string): string { .replace(/_+/g, "_") // Trim leading/trailing dots and underscores (hidden files, Windows edge cases) .replace(/^[._]+|[._]+$/g, ""); + if (!options?.preserveCase) { + name = name.toLowerCase(); + } // Prefix Windows reserved device names (CON, PRN, AUX, NUL, COM0-9, LPT0-9) return WINDOWS_RESERVED.test(name.toLowerCase()) ? `_${name}` : name; } @@ -211,7 +217,7 @@ export function newRawAppPathAssigner(defaultTs: "bun" | "deno"): PathAssigner { ): [string, string] { let name; - name = summary ? sanitizeForFilesystem(summary) : ""; + name = summary ? sanitizeForFilesystem(summary, { preserveCase: true }) : ""; let original_name = name; From 0b959b8ec61b24d861c5a10a9242a7a0e6013707 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Fri, 1 May 2026 17:11:56 +0200 Subject: [PATCH 04/21] feat(cli): wmill-lock.yaml auto-fill + --rehash-only + path-prefix dedup (#8978) * fix(cli): canonical lockfile hashes + lock upgrade migration to v3 Co-Authored-By: Claude Opus 4.5 * fix(cli): use __app_hash subpath in rehash missing-entry check Co-Authored-By: Claude Opus 4.5 * fix(cli): run sync pull lockfile auto-fill regardless of changes Co-Authored-By: Claude Opus 4.5 * chore: regenerate system prompts for new lock and rehash-only commands Co-Authored-By: Claude Opus 4.5 * fix(cli): address review feedback on lock upgrade Co-Authored-By: Claude Opus 4.5 * fix(cli): drop v3 marker; always run fallback; fail-fast on unknown lockfile version Co-Authored-By: Claude Opus 4.5 * fix(cli): drop yaml-round-trip legacy hash variant; recover via --rehash-only Co-Authored-By: Claude Opus 4.5 * fix(cli): include legacy hash in script push staleness warning check Co-Authored-By: Claude Opus 4.5 * revert(cli): drop canonical hash formula; keep raw-bytes hashing Co-Authored-By: Claude Opus 4.5 * perf(cli): reuse change-tracker map for sync pull lockfile auto-fill Co-Authored-By: Claude Opus 4.5 * fix(cli): address review feedback on rehash-only Co-Authored-By: Claude Opus 4.5 * test(cli): pin lockfile hash + yaml format and cover regression cases Co-Authored-By: Claude Opus 4.5 * test(cli): byte-stable snapshot tests for flow.yaml format Co-Authored-By: Claude Opus 4.5 * test(cli): add app and script-metadata yaml snapshot fixtures Co-Authored-By: Claude Opus 4.5 * fix(cli): address claude review on rehash-only Co-Authored-By: Claude Opus 4.5 * refactor(cli): factorize script-path to remote-path derivation Co-Authored-By: Claude Opus 4.5 * fix(cli): address claude + cubic review (dry-run mutation, rehash short-circuit) Co-Authored-By: Claude Opus 4.5 * refactor(cli): make rehash a subcommand and factorize fs walks Co-Authored-By: Claude Opus 4.5 * fix(cli): normalize line endings in yaml snapshot tests for windows ci Co-Authored-By: Claude Opus 4.5 * fix(cli): address review feedback on rehash + auto-fill - Flat-layout scripts now clearGlobalLock before rehash write so legacy ./-prefixed duplicates get cleaned up (matches flow/app behavior). - Add MalformedLockfileError; sync pull auto-fill re-throws it alongside UnknownLockVersionError instead of silently warning + continuing. - Document the legacy step-removal false-negative in isFlowDirectlyStale / isAppDirectlyStale and the categorizeLocalFiles ignore-filter invariant. Co-Authored-By: Claude Opus 4.5 --------- Co-authored-by: Claude Opus 4.5 --- cli/src/commands/app/app_metadata.ts | 46 +- cli/src/commands/flow/flow_metadata.ts | 56 ++- .../generate-metadata/generate-metadata.ts | 344 +++++++++++--- cli/src/commands/script/script.ts | 5 +- cli/src/commands/sync/sync.ts | 67 ++- cli/src/guidance/skills.gen.ts | 9 + cli/src/utils/metadata.ts | 147 ++++-- cli/src/utils/resource_folders.ts | 17 + .../fixtures/yaml-snapshots/.gitattributes | 1 + .../fixtures/yaml-snapshots/app_basic.yaml | 39 ++ cli/test/fixtures/yaml-snapshots/app_raw.yaml | 14 + .../fixtures/yaml-snapshots/flow_basic.yaml | 25 + .../yaml-snapshots/flow_branches_loops.yaml | 61 +++ .../yaml-snapshots/flow_inline_tags.yaml | 19 + .../yaml-snapshots/flow_notes_groups.yaml | 40 ++ .../flow_retry_suspend_sleep.yaml | 58 +++ .../fixtures/yaml-snapshots/regenerate.ts | 440 ++++++++++++++++++ .../script_metadata.script.yaml | 22 + cli/test/lockfile_format_pinning_unit.test.ts | 108 +++++ cli/test/wmill_lock_unit.test.ts | 97 ++++ cli/test/yaml_format_snapshot_unit.test.ts | 60 +++ .../auto-generated/cli/cli-commands.md | 9 + system_prompts/auto-generated/prompts.ts | 9 + .../skills/cli-commands/SKILL.md | 9 + 24 files changed, 1585 insertions(+), 117 deletions(-) create mode 100644 cli/test/fixtures/yaml-snapshots/.gitattributes create mode 100644 cli/test/fixtures/yaml-snapshots/app_basic.yaml create mode 100644 cli/test/fixtures/yaml-snapshots/app_raw.yaml create mode 100644 cli/test/fixtures/yaml-snapshots/flow_basic.yaml create mode 100644 cli/test/fixtures/yaml-snapshots/flow_branches_loops.yaml create mode 100644 cli/test/fixtures/yaml-snapshots/flow_inline_tags.yaml create mode 100644 cli/test/fixtures/yaml-snapshots/flow_notes_groups.yaml create mode 100644 cli/test/fixtures/yaml-snapshots/flow_retry_suspend_sleep.yaml create mode 100644 cli/test/fixtures/yaml-snapshots/regenerate.ts create mode 100644 cli/test/fixtures/yaml-snapshots/script_metadata.script.yaml create mode 100644 cli/test/lockfile_format_pinning_unit.test.ts create mode 100644 cli/test/yaml_format_snapshot_unit.test.ts diff --git a/cli/src/commands/app/app_metadata.ts b/cli/src/commands/app/app_metadata.ts index 2589bf2fca..0a098af05a 100644 --- a/cli/src/commands/app/app_metadata.ts +++ b/cli/src/commands/app/app_metadata.ts @@ -57,6 +57,33 @@ type AppFile = RawAppFile | NormalAppFile; /** * Generates a hash for all inline scripts in an app directory */ +/** + * Check if an app folder is up-to-date against the lockfile. + * + * Mirrors the flow staleness check: when the top hash mismatches (e.g. due to + * the legacy unsorted-keys top-hash formula), accept the entry if every + * per-file hash still matches individually. + * + * Same false-negative as flows: removing a runnable on a legacy lockfile + * looks "up-to-date" until the next push rewrites the top hash. See + * isFlowDirectlyStale for the rationale. + */ +async function isAppDirectlyStale( + appFolder: string, + hashes: Record, + conf: Awaited>, +): Promise { + if (await checkifMetadataUptodate(appFolder, hashes[TOP_HASH], conf, TOP_HASH)) { + return false; + } + const fileEntries = Object.entries(hashes).filter(([k]) => k !== TOP_HASH); + if (fileEntries.length === 0) return true; + for (const [k, h] of fileEntries) { + if (!(await checkifMetadataUptodate(appFolder, h, conf, k))) return true; + } + return false; +} + async function generateAppHash( rawReqs: Record | undefined, folder: string, @@ -122,6 +149,7 @@ export async function generateAppLocksInternal( workspace: Workspace, opts: GlobalOptions & { defaultTs?: "bun" | "deno"; + rehashOnly?: boolean; }, justUpdateMetadataLock?: boolean, noStaleMessage?: boolean, @@ -131,6 +159,20 @@ export async function generateAppLocksInternal( appFolder = appFolder.substring(0, appFolder.length - 1); } + // Rehash-only fast path: write canonical hashes from disk, skip backend. + // Short-circuit before reading app.yaml / readLockfile since generateAppHash + // walks the folder itself. + // Uses empty workspace deps `{}` to match the tree-mode write path. See + // the matching comment in flow_metadata.ts for the legacy-mode caveat. + if (opts.rehashOnly) { + const hashes = await generateAppHash({}, appFolder, rawApp, opts.defaultTs); + await clearGlobalLock(appFolder); + for (const [k, v] of Object.entries(hashes)) { + await updateMetadataGlobalLock(appFolder, v, k); + } + return; + } + const remote_path = appFolder.replaceAll(SEP, "/"); if (!justUpdateMetadataLock && !noStaleMessage) { @@ -154,7 +196,7 @@ export async function generateAppLocksInternal( if (tree) { if (dryRun) { const hashes = await generateAppHash({}, appFolder, rawApp, opts.defaultTs); - const isDirectlyStale = !(await checkifMetadataUptodate(appFolder, hashes[TOP_HASH], conf, TOP_HASH)); + const isDirectlyStale = await isAppDirectlyStale(appFolder, hashes, conf); // For raw apps in new format, runnables are in separate files under backend/ let treeAppValue = structuredClone(appValue); @@ -206,7 +248,7 @@ export async function generateAppLocksInternal( filteredDeps = await filterWorkspaceDependenciesForApp(appValue, rawWorkspaceDependencies, appFolder); const hashes = await generateAppHash(filteredDeps, appFolder, rawApp, opts.defaultTs); - const isDirectlyStale = !(await checkifMetadataUptodate(appFolder, hashes[TOP_HASH], conf, TOP_HASH)); + const isDirectlyStale = await isAppDirectlyStale(appFolder, hashes, conf); if (!isDirectlyStale) { if (!noStaleMessage) { diff --git a/cli/src/commands/flow/flow_metadata.ts b/cli/src/commands/flow/flow_metadata.ts index 4110004959..25f074a50d 100644 --- a/cli/src/commands/flow/flow_metadata.ts +++ b/cli/src/commands/flow/flow_metadata.ts @@ -34,6 +34,39 @@ import { DoubleLinkedDependencyTree } from "../../utils/dependency_tree.ts"; import { pollJobWithQueueLogging } from "../../utils/job_polling.ts"; const TOP_HASH = "__flow_hash"; + +/** + * Check if a flow folder is up-to-date against the lockfile. + * + * The top hash is `sha256(JSON.stringify(sortedPerFileHashes))`. Older CLI + * versions hashed `JSON.stringify(perFileHashes)` without sorting keys, so + * lockfile entries written before that fix won't match. As a backwards-compat + * fallback, if the top hash mismatches we accept the entry as up-to-date when + * every per-file hash matches individually — that's enough to prove no file + * content changed. + * + * Known false-negative: on a legacy lockfile, *removing* a step leaves the + * remaining per-file hashes matching, so this returns "up-to-date" even + * though the flow shape changed. Self-heals on the next push (which writes + * the modern top hash). Acceptable trade-off vs. the alternative — false + * positives from the unrecoverable legacy top-hash formula. + */ +async function isFlowDirectlyStale( + folder: string, + hashes: Record, + conf: Awaited>, +): Promise { + if (await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)) { + return false; + } + const fileEntries = Object.entries(hashes).filter(([k]) => k !== TOP_HASH); + if (fileEntries.length === 0) return true; + for (const [k, h] of fileEntries) { + if (!(await checkifMetadataUptodate(folder, h, conf, k))) return true; + } + return false; +} + async function generateFlowHash( rawWorkspaceDependencies: Record, folder: string, @@ -72,6 +105,7 @@ export async function generateFlowLockInternal( workspace: Workspace, opts: GlobalOptions & { defaultTs?: "bun" | "deno"; + rehashOnly?: boolean; }, justUpdateMetadataLock?: boolean, noStaleMessage?: boolean, @@ -80,6 +114,24 @@ export async function generateFlowLockInternal( if (folder.endsWith(SEP)) { folder = folder.substring(0, folder.length - 1); } + + // Rehash-only fast path: write canonical per-file + top hashes from disk, + // no backend trip, no flow.yaml/inline_script rewrite. Short-circuit before + // yamlParseFile / extractInlineScriptsForFlows / readLockfile since none of + // those are needed — generateFlowHash walks the folder itself. + // Uses empty workspace deps `{}` to match the tree-mode dryRun and write + // paths (the modern default). A subsequent legacy non-tree push would + // see a deps-included hash mismatch — but legacy non-tree mode is opt-in + // and not the recommended workflow. + if (opts.rehashOnly) { + const hashes = await generateFlowHash({}, folder, opts.defaultTs); + await clearGlobalLock(folder); + for (const [k, v] of Object.entries(hashes)) { + await updateMetadataGlobalLock(folder, v, k); + } + return; + } + const remote_path = extractNameFromFolder(folder.replaceAll(SEP, "/"), "flow"); if (!justUpdateMetadataLock && !noStaleMessage) { log.info(`Generating lock for flow ${folder} at ${remote_path}`); @@ -122,7 +174,7 @@ export async function generateFlowLockInternal( } const hashes = await generateFlowHash({}, folder, opts.defaultTs); - const isDirectlyStale = !(await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)); + const isDirectlyStale = await isFlowDirectlyStale(folder, hashes, conf); await tree.addNode(folderNormalized, "", "bun", "", inlineScriptPaths, "flow", folderNormalized, folder, isDirectlyStale); return; @@ -134,7 +186,7 @@ export async function generateFlowLockInternal( filteredDeps = await filterWorkspaceDependenciesForFlow(flowValue.value as FlowValue, rawWorkspaceDependencies, folder); const hashes = await generateFlowHash(filteredDeps, folder, opts.defaultTs); - const isDirectlyStale = !(await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)); + const isDirectlyStale = await isFlowDirectlyStale(folder, hashes, conf); if (!isDirectlyStale) { if (!noStaleMessage) { diff --git a/cli/src/commands/generate-metadata/generate-metadata.ts b/cli/src/commands/generate-metadata/generate-metadata.ts index ee867fe9fc..e840397230 100644 --- a/cli/src/commands/generate-metadata/generate-metadata.ts +++ b/cli/src/commands/generate-metadata/generate-metadata.ts @@ -11,18 +11,17 @@ import { generateScriptMetadataInternal, getRawWorkspaceDependencies, readLockfile, - checkifMetadataUptodate, } from "../../utils/metadata.ts"; import { generateFlowLockInternal, FlowLocksResult } from "../flow/flow_metadata.ts"; -import { generateAppLocksInternal, getAppFolders, AppLocksResult } from "../app/app_metadata.ts"; +import { generateAppLocksInternal, AppLocksResult } from "../app/app_metadata.ts"; import { elementsToMap, FSFSElement, ignoreF, } from "../sync/sync.ts"; import { exts } from "../script/script.ts"; -import { isFolderResourcePathAnyFormat, isScriptModulePath, isModuleEntryPoint } from "../../utils/resource_folders.ts"; -import { listSyncCodebases } from "../../utils/codebase.ts"; +import { isFolderResourcePathAnyFormat, isScriptModulePath, isModuleEntryPoint, scriptPathToRemotePath } from "../../utils/resource_folders.ts"; +import { listSyncCodebases, SyncCodebase } from "../../utils/codebase.ts"; import { DoubleLinkedDependencyTree, uploadScripts, @@ -37,6 +36,233 @@ interface StaleItem { staleReason?: string; } +/** + * FS walk helpers — shared between the regular `generate-metadata` flow and + * the `generate-metadata rehash` subcommand. Each returns the filtered list + * of items in scope (scripts / flow folders / app items). + */ +async function walkLocalScripts( + codebases: SyncCodebase[], + ignore: (p: string, isD: boolean) => boolean, +): Promise { + const elems = await elementsToMap( + await FSFSElement(process.cwd(), codebases, false), + (p, isD) => + (!isD && !exts.some((ext) => p.endsWith(ext))) || + ignore(p, isD) || + isFolderResourcePathAnyFormat(p) || + (isScriptModulePath(p) && !isModuleEntryPoint(p)), + false, + {}, + ); + return Object.keys(elems); +} + +async function walkLocalFlowFolders( + ignore: (p: string, isD: boolean) => boolean, +): Promise { + const elems = await elementsToMap( + await FSFSElement(process.cwd(), [], true), + (p, isD) => + ignore(p, isD) || + (!isD && !p.endsWith(SEP + "flow.yaml") && !p.endsWith(SEP + "flow.json")), + false, + {}, + ); + return Object.keys(elems).map((x) => x.substring(0, x.lastIndexOf(SEP))); +} + +async function walkLocalAppItems( + ignore: (p: string, isD: boolean) => boolean, +): Promise<{ folder: string; rawApp: boolean }[]> { + const elems = await elementsToMap( + await FSFSElement(process.cwd(), [], true), + (p, isD) => + ignore(p, isD) || + (!isD && !p.endsWith(SEP + "raw_app.yaml") && !p.endsWith(SEP + "app.yaml")), + false, + {}, + ); + return Object.keys(elems).map((p) => ({ + folder: p.substring(0, p.lastIndexOf(SEP)), + rawApp: p.endsWith(SEP + "raw_app.yaml"), + })); +} + +/** + * Categorize a flat list of file paths into scripts / flow folders / app + * file paths. Used to derive item lists from a precomputed FS map (e.g. + * sync pull's change-tracker output) without re-walking the filesystem. + * + * Caller invariant: the provided paths are expected to already be filtered + * by the user-level ignore predicate (`ignoreF(opts)`). This function does + * NOT re-apply ignore patterns — it only filters by file *kind* (script vs + * flow vs app). Sync pull's localMap satisfies this since `elementsToMap` + * was called with the same `ignoreF`. + */ +function categorizeLocalFiles( + paths: Iterable, +): { scripts: string[]; flowFolders: string[]; appPaths: string[] } { + const scripts: string[] = []; + const flowFolderSet = new Set(); + const appPaths: string[] = []; + for (const p of paths) { + if (p.endsWith(SEP + "flow.yaml") || p.endsWith(SEP + "flow.json")) { + flowFolderSet.add(p.substring(0, p.lastIndexOf(SEP))); + } else if ( + p.endsWith(SEP + "raw_app.yaml") || + p.endsWith(SEP + "app.yaml") + ) { + appPaths.push(p); + } else if ( + exts.some((ext) => p.endsWith(ext)) && + !isFolderResourcePathAnyFormat(p) && + !(isScriptModulePath(p) && !isModuleEntryPoint(p)) + ) { + scripts.push(p); + } + } + return { scripts, flowFolders: [...flowFolderSet], appPaths }; +} + +/** + * Walks all local scripts/flows/apps (or those under `folder`) and writes + * canonical hashes to wmill-lock.yaml from disk content. No backend round-trip, + * no yaml/lock rewrites. Stub workspace + opts are passed through to handlers + * since the rehash-only fast path returns before any backend call. + */ +export async function rehashOnly( + opts: GlobalOptions & SyncOptions & { defaultTs?: "bun" | "deno" }, + folder?: string, + rehashFilter?: { + missingOnly?: boolean; + localFiles?: Iterable; + skipScripts?: boolean; + skipFlows?: boolean; + skipApps?: boolean; + }, +): Promise<{ scripts: number; flows: number; apps: number }> { + const codebases = await listSyncCodebases(opts); + const ignore = await ignoreF(opts); + const counts = { scripts: 0, flows: 0, apps: 0 }; + const folderFilter = folder + ?.replaceAll("\\", "/") + .replace(/^\.\//, "") + .replace(/\/$/, ""); + const inFilter = (p: string) => { + if (!folderFilter) return true; + const n = p.replaceAll("\\", "/"); + return n === folderFilter || n.startsWith(folderFilter + "/"); + }; + + const conf = rehashFilter?.missingOnly ? await readLockfile() : undefined; + const isFlatKeyed = conf?.version === "v2"; + const hasEntry = (key: string, subpath?: string): boolean => { + if (!conf?.locks) return false; + if (isFlatKeyed) { + const fullKey = subpath ? `${key}+${subpath}` : key; + return ( + conf.locks[fullKey] !== undefined || + conf.locks["./" + fullKey] !== undefined + ); + } + for (const p of [key, "./" + key]) { + const obj = conf.locks[p]; + if (obj === undefined) continue; + if (!subpath) return true; + if (typeof obj === "object" && obj?.[subpath] !== undefined) return true; + } + return false; + }; + const skipIfExisting = (remotePath: string, subpath?: string): boolean => + !!rehashFilter?.missingOnly && hasEntry(remotePath, subpath); + + // Either reuse a precomputed file list from the caller (e.g. sync pull's + // change-tracker) or do three separate FS walks here. + let scriptPaths: string[]; + let flowFolders: string[]; + let appPaths: { folder: string; rawApp: boolean }[]; + + if (rehashFilter?.localFiles) { + const cat = categorizeLocalFiles(rehashFilter.localFiles); + scriptPaths = cat.scripts; + flowFolders = cat.flowFolders; + appPaths = cat.appPaths.map((p) => ({ + folder: p.substring(0, p.lastIndexOf(SEP)), + rawApp: p.endsWith(SEP + "raw_app.yaml"), + })); + } else { + scriptPaths = await walkLocalScripts(codebases, ignore); + flowFolders = await walkLocalFlowFolders(ignore); + appPaths = await walkLocalAppItems(ignore); + } + + const stubWorkspace = {} as any; + const rehashOpts = { ...opts, rehashOnly: true } as any; + + if (!rehashFilter?.skipScripts) { + for (const e of scriptPaths) { + // Filter against the derived remote path so a folder argument like + // `f/foo` matches both flat (`f/foo.ts`) and folder-layout + // (`f/foo__mod/script.ts`) scripts uniformly. + const remotePath = scriptPathToRemotePath(e); + if (!inFilter(remotePath)) continue; + if (rehashFilter?.missingOnly) { + if (skipIfExisting(remotePath) || skipIfExisting(remotePath, "__script_hash")) continue; + } + try { + await generateScriptMetadataInternal( + e, stubWorkspace, rehashOpts, false, true, {}, codebases, false, + ); + counts.scripts++; + } catch (err) { + log.warn(`Skipping ${e}: ${err instanceof Error ? err.message : err}`); + } + } + } + + if (!rehashFilter?.skipFlows) { + for (const f of flowFolders) { + if (!inFilter(f)) continue; + if (rehashFilter?.missingOnly) { + const folderNormalized = f.replaceAll(SEP, "/"); + if (skipIfExisting(folderNormalized, "__flow_hash")) continue; + } + try { + await generateFlowLockInternal(f, false, stubWorkspace, rehashOpts, false, true); + counts.flows++; + } catch (err) { + log.warn(`Skipping ${f}: ${err instanceof Error ? err.message : err}`); + } + } + } + + if (!rehashFilter?.skipApps) { + for (const { folder: appFolder, rawApp } of appPaths) { + if (!inFilter(appFolder)) continue; + if (rehashFilter?.missingOnly) { + const folderNormalized = appFolder.replaceAll(SEP, "/"); + if (skipIfExisting(folderNormalized, "__app_hash")) continue; + } + try { + await generateAppLocksInternal(appFolder, rawApp, false, stubWorkspace, rehashOpts, false, true); + counts.apps++; + } catch (err) { + log.warn(`Skipping ${appFolder}: ${err instanceof Error ? err.message : err}`); + } + } + } + + if (counts.scripts + counts.flows + counts.apps > 0 || !rehashFilter?.missingOnly) { + log.info( + `Rehashed ${colors.bold(String(counts.scripts))} script(s), ` + + `${colors.bold(String(counts.flows))} flow(s), ` + + `${colors.bold(String(counts.apps))} app(s) from disk.`, + ); + } + return counts; +} + async function generateMetadata( opts: GlobalOptions & { yes?: boolean; @@ -85,22 +311,7 @@ async function generateMetadata( // === Collect stale scripts === if (!skipScripts) { - // TODO: run elementsToMap only once but for all runnable types. - const scriptElems = await elementsToMap( - await FSFSElement(process.cwd(), codebases, false), - (p, isD) => { - return ( - (!isD && !exts.some((ext) => p.endsWith(ext))) || - ignore(p, isD) || - isFolderResourcePathAnyFormat(p) || - (isScriptModulePath(p) && !isModuleEntryPoint(p)) - ); - }, - false, - {} - ); - - for (const e of Object.keys(scriptElems)) { + for (const e of await walkLocalScripts(codebases, ignore)) { await generateScriptMetadataInternal( e, workspace, @@ -117,23 +328,7 @@ async function generateMetadata( // === Collect stale flows === if (!skipFlows) { - const flowElems = Object.keys( - await elementsToMap( - await FSFSElement(process.cwd(), [], true), - (p, isD) => { - return ( - ignore(p, isD) || - (!isD && - !p.endsWith(SEP + "flow.yaml") && - !p.endsWith(SEP + "flow.json")) - ); - }, - false, - {} - ) - ).map((x) => x.substring(0, x.lastIndexOf(SEP))); - - for (const flowFolder of flowElems) { + for (const flowFolder of await walkLocalFlowFolders(ignore)) { await generateFlowLockInternal( flowFolder, true, // dryRun - populate tree @@ -148,40 +343,10 @@ async function generateMetadata( // === Collect stale apps === if (!skipApps) { - const elems = await elementsToMap( - await FSFSElement(process.cwd(), [], true), - (p, isD) => { - return ( - ignore(p, isD) || - (!isD && - !p.endsWith(SEP + "raw_app.yaml") && - !p.endsWith(SEP + "app.yaml")) - ); - }, - false, - {} - ); - - const rawAppFolders = getAppFolders(elems, "raw_app.yaml"); - const appFolders = getAppFolders(elems, "app.yaml"); - - for (const appFolder of rawAppFolders) { + for (const { folder: appFolder, rawApp } of await walkLocalAppItems(ignore)) { await generateAppLocksInternal( appFolder, - true, // rawApp - true, // dryRun - populate tree - workspace, - opts, - false, - true, // noStaleMessage - tree - ); - } - - for (const appFolder of appFolders) { - await generateAppLocksInternal( - appFolder, - false, // rawApp + rawApp, true, // dryRun - populate tree workspace, opts, @@ -447,6 +612,23 @@ async function generateMetadata( } } +async function rehashCommand( + opts: GlobalOptions & SyncOptions & { + skipScripts?: boolean; + skipFlows?: boolean; + skipApps?: boolean; + }, + folder?: string, +) { + if (folder === "") folder = undefined; + opts = await mergeConfigWithConfigFile(opts); + await rehashOnly(opts, folder, { + skipScripts: opts.skipScripts, + skipFlows: opts.skipFlows, + skipApps: opts.skipApps, + }); +} + const command = new Command() .description("Generate metadata (locks, schemas) for all scripts, flows, and apps") .arguments("[folder:string]") @@ -466,6 +648,28 @@ const command = new Command() "-e --excludes ", "Comma separated patterns to specify which files to exclude" ) - .action(generateMetadata as any); + .action(generateMetadata as any) + .command( + "rehash", + new Command() + .description( + "Trust on-disk content; rewrite wmill-lock.yaml hashes without backend " + + "trips or yaml/lock rewrites. Useful for bootstrapping missing lockfile " + + "entries or recovering from older-CLI hash drift." + ) + .arguments("[folder:string]") + .option("--skip-scripts", "Skip processing scripts") + .option("--skip-flows", "Skip processing flows") + .option("--skip-apps", "Skip processing apps") + .option( + "-i --includes ", + "Comma separated patterns to specify which files to include" + ) + .option( + "-e --excludes ", + "Comma separated patterns to specify which files to exclude" + ) + .action(rehashCommand as any), + ); export default command; diff --git a/cli/src/commands/script/script.ts b/cli/src/commands/script/script.ts index f89bcccfb2..db2b9b3fef 100644 --- a/cli/src/commands/script/script.ts +++ b/cli/src/commands/script/script.ts @@ -70,6 +70,7 @@ import { getModuleFolderSuffix, isModuleEntryPoint, getScriptBasePathFromModulePath, + scriptPathToRemotePath, isRawAppPath, } from "../../utils/resource_folders.ts"; @@ -299,9 +300,7 @@ export async function handleFile( log.debug(`Processing local script ${path}`); alreadySynced.push(path); - const remotePath = moduleEntryPoint - ? getScriptBasePathFromModulePath(path)!.replaceAll(SEP, "/") - : path.substring(0, path.indexOf(".")).replaceAll(SEP, "/"); + const remotePath = scriptPathToRemotePath(path); const language = inferContentTypeFromFilePath(path, opts?.defaultTs); diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index edbf27d86b..a7dd91b5fa 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -75,6 +75,8 @@ import { generateScriptMetadataInternal, getRawWorkspaceDependencies, readLockfile, + UnknownLockVersionError, + MalformedLockfileError, workspaceDependenciesPathToLanguageAndFilename, } from "../../utils/metadata.ts"; import { OpenFlow, NativeServiceName, ScriptModule } from "../../../gen/types.gen.ts"; @@ -1669,7 +1671,7 @@ async function compareDynFSElement( specificItems?: SpecificItemsConfig, branchOverride?: string, isEls1Remote?: boolean, -): Promise { +): Promise<{ changes: Change[]; localMap: Record }> { const [m1, m2] = els2 ? await Promise.all([ elementsToMap(els1, ignore, json, skips, specificItems, branchOverride, isEls1Remote), @@ -1864,7 +1866,12 @@ async function compareDynFSElement( return a.path.localeCompare(b.path); }); - return changes; + // Expose the local-side map so callers (e.g. sync pull's auto-fill) can + // reuse it instead of re-walking the filesystem. Which map is local depends + // on which side `els1` was — pull passes remote as els1 (isEls1Remote=true), + // push passes local as els1 (isEls1Remote=false). + const localMap = isEls1Remote ? m2 : m1; + return { changes, localMap }; } function getOrderFromPath(p: string) { @@ -2332,7 +2339,7 @@ export async function pull( ? await FSFSElement(process.cwd(), codebases, true) : await FSFSElement(path.join(process.cwd(), ".wmill"), [], true); - const changes = await compareDynFSElement( + const { changes, localMap } = await compareDynFSElement( remote, local, await ignoreF(opts), @@ -2600,6 +2607,7 @@ export async function pull( true, ); } + if (opts.jsonOutput) { const result = { success: true, @@ -2641,6 +2649,57 @@ export async function pull( ); } + // Auto-fill missing lockfile entries for items that exist on disk but have + // no entry yet (e.g. flows/apps that predate lockfile maintenance). Runs + // regardless of whether the pull had changes from the backend so a no-op + // pull still bootstraps the lockfile. Silent on a complete lockfile (just + // dict lookups, no hashing). Skipped under --dry-run since auto-fill writes + // to wmill-lock.yaml. + if (!opts.jsonOutput && !opts.dryRun) { + try { + // Dynamic import to avoid a circular dep between sync.ts and + // generate-metadata.ts. Don't "clean up" to a static import. + const { rehashOnly } = await import("../generate-metadata/generate-metadata.ts"); + // Reuse the local-side file list from the change-tracker so we don't + // re-walk the filesystem. Apply the just-applied changes to derive the + // post-pull state: localMap is pre-pull, but auto-fill needs to see + // additions and skip deletions. + const postPullPaths = new Set(Object.keys(localMap)); + for (const change of changes) { + if (change.name === "added") postPullPaths.add(change.path); + else if (change.name === "deleted") postPullPaths.delete(change.path); + } + const filled = await rehashOnly(opts as any, undefined, { + missingOnly: true, + localFiles: postPullPaths, + }); + const total = filled.scripts + filled.flows + filled.apps; + if (total > 0) { + log.info( + colors.gray( + `Auto-filled ${total} missing lockfile entr${total === 1 ? "y" : "ies"} ` + + `(${filled.scripts} script, ${filled.flows} flow, ${filled.apps} app) from disk.`, + ), + ); + } + } catch (e) { + // Re-throw fail-fast lockfile errors (unknown version, malformed yaml) + // so the user sees them; only swallow soft failures from the auto-fill + // walk itself. + if ( + e instanceof UnknownLockVersionError || + e instanceof MalformedLockfileError + ) { + throw e; + } + log.warn( + colors.yellow( + `Could not auto-fill missing lockfile entries: ${e instanceof Error ? e.message : e}`, + ), + ); + } + } + try { await pullSharedUi(workspace.workspaceId); } catch (e) { @@ -2908,7 +2967,7 @@ export async function push( ); const local = await FSFSElement(path.join(process.cwd(), ""), codebases, false); - const changes = await compareDynFSElement( + const { changes } = await compareDynFSElement( local, remote, await ignoreF(opts), diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index e598ac2a76..e005bc10b8 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -6496,6 +6496,15 @@ Generate metadata (locks, schemas) for all scripts, flows, and apps - \`-i --includes \` - Comma separated patterns to specify which files to include - \`-e --excludes \` - Comma separated patterns to specify which files to exclude +**Subcommands:** + +- \`generate-metadata rehash [folder:string]\` + - \`--skip-scripts\` - Skip processing scripts + - \`--skip-flows\` - Skip processing flows + - \`--skip-apps\` - Skip processing apps + - \`-i --includes \` - Comma separated patterns to specify which files to include + - \`-e --excludes \` - Comma separated patterns to specify which files to exclude + ### gitsync-settings Manage git-sync settings between local wmill.yaml and Windmill backend diff --git a/cli/src/utils/metadata.ts b/cli/src/utils/metadata.ts index f958e2d722..7aa4372ebf 100644 --- a/cli/src/utils/metadata.ts +++ b/cli/src/utils/metadata.ts @@ -19,7 +19,7 @@ import { languageNeedsLock, } from "./script_common.ts"; import { inferContentTypeFromFilePath } from "./script_common.ts"; -import { getModuleFolderSuffix, isModuleEntryPoint, getScriptBasePathFromModulePath } from "./resource_folders.ts"; +import { getModuleFolderSuffix, isModuleEntryPoint, scriptPathToRemotePath } from "./resource_folders.ts"; import { findCodebase, yamlOptions } from "../commands/sync/sync.ts"; import { generateHash, readInlinePathSync, getHeaders, readTextFile, readTextFileSync } from "./utils.ts"; @@ -56,6 +56,20 @@ export class LockfileGenerationError extends Error { } } +export class UnknownLockVersionError extends Error { + constructor(message: string) { + super(message); + this.name = "UnknownLockVersionError"; + } +} + +export class MalformedLockfileError extends Error { + constructor(message: string) { + super(message); + this.name = "MalformedLockfileError"; + } +} + export async function getRawWorkspaceDependencies(legacyBehaviour: boolean): Promise> { const rawWorkspaceDeps: Record = {}; @@ -186,6 +200,7 @@ export async function generateScriptMetadataInternal( lockOnly?: boolean | undefined; schemaOnly?: boolean | undefined; defaultTs?: "bun" | "deno"; + rehashOnly?: boolean | undefined; }, dryRun: boolean, noStaleMessage: boolean, @@ -198,9 +213,7 @@ export async function generateScriptMetadataInternal( const isFolderLayout = isModuleEntryPoint(scriptPath); // remotePath is the Windmill API path (e.g., "u/admin/my_script") - const remotePath = isFolderLayout - ? getScriptBasePathFromModulePath(scriptPath)!.replaceAll(SEP, "/") - : scriptPath.substring(0, scriptPath.indexOf(".")).replaceAll(SEP, "/"); + const remotePath = scriptPathToRemotePath(scriptPath); const language = inferContentTypeFromFilePath(scriptPath, opts.defaultTs); @@ -241,6 +254,30 @@ export async function generateScriptMetadataInternal( } const hasModuleHashes = Object.keys(moduleHashes).length > 0; + // Rehash-only fast path: trust on-disk content, write canonical hashes + // straight to the lockfile, skip staleness check and any backend trip. + // Short-circuit before computing the legacy fallback hashes since we never + // use them on this path. + if (opts.rehashOnly) { + if (hasModuleHashes) { + const sortedEntries = Object.entries(moduleHashes).sort(([a], [b]) => a.localeCompare(b)); + const metaHash = await generateHash(hash + JSON.stringify(sortedEntries)); + await clearGlobalLock(remotePath); + await updateMetadataGlobalLock(remotePath, metaHash, SCRIPT_TOP_HASH); + for (const [modulePath, moduleHash] of Object.entries(moduleHashes)) { + await updateMetadataGlobalLock(remotePath, moduleHash, modulePath); + } + } else { + // Mirror the hasModuleHashes branch: clear first so any legacy + // "./"-prefixed duplicate gets collapsed alongside the canonical write. + await clearGlobalLock(remotePath); + await updateMetadataGlobalLock(remotePath, hash); + } + return; + } + + const conf = await readLockfile(); + // If modules exist, combine main script hash + module hashes into a meta-hash let checkHash = hash; let checkSubpath: string | undefined; @@ -250,8 +287,6 @@ export async function generateScriptMetadataInternal( checkSubpath = SCRIPT_TOP_HASH; } - const conf = await readLockfile(); - // Use checkHash (includes module hashes) so module changes are detected as stale const isDirectlyStale = !(await checkifMetadataUptodate(remotePath, checkHash, conf, checkSubpath)); @@ -1129,37 +1164,63 @@ export async function parseMetadataFile( }; } -interface Lock { - version?: "v2"; +export type LockVersion = "v2"; + +export interface Lock { + version?: LockVersion; locks?: { [path: string]: string | { [subpath: string]: string } }; } const WMILL_LOCKFILE = "wmill-lock.yaml"; +const CURRENT_LOCK_VERSION: LockVersion = "v2"; +// Versions this CLI knows how to read/write. An unknown value indicates the +// lockfile was written by a newer CLI; we refuse to touch it rather than +// silently fall through to a legacy code path (the bug that 1.692.0 had with +// the proposed v3 marker). +// Real v1 lockfiles never had a version field (it was added with v2). The +// "v1" string is included here only to be lenient about manual edits that +// label a v1 lockfile explicitly — the downstream isFlatKeyed check ensures +// it still goes through the legacy nested-key path. +const KNOWN_LOCK_VERSIONS: readonly string[] = ["v1", "v2"]; const SCRIPT_TOP_HASH = "__script_hash"; /** - * Normalizes a path to use Linux separators (forward slashes). - * This ensures wmill-lock.yaml is portable across Windows and Linux. + * Normalizes a path to use Linux separators (forward slashes) and strips a + * leading "./" prefix. Forward slashes ensure wmill-lock.yaml is portable + * across Windows and Linux; stripping "./" collapses entries from older CLIs + * that joined paths against the current directory before storing. */ export function normalizeLockPath(p: string): string { - return p.replace(/\\/g, "/"); + let n = p.replace(/\\/g, "/"); + if (n.startsWith("./")) n = n.slice(2); + return n; } export async function readLockfile(): Promise { + let parsed: unknown; try { - const read = await yamlParseFile(WMILL_LOCKFILE); - if (typeof read == "object" && read != null) { - return read as Lock; - } else { - throw new Error("Invalid lockfile"); - } + parsed = await yamlParseFile(WMILL_LOCKFILE); } catch { - const lock = { locks: {}, version: "v2" as const }; + const lock: Lock = { locks: {}, version: CURRENT_LOCK_VERSION }; await writeFile(WMILL_LOCKFILE, yamlStringify(lock, yamlOptions), "utf-8"); log.info(colors.green("wmill-lock.yaml created")); - return lock; } + if (typeof parsed != "object" || parsed == null) { + throw new MalformedLockfileError( + "wmill-lock.yaml is malformed (expected an object). " + + "Refusing to operate to avoid corrupting the lockfile.", + ); + } + const conf = parsed as Lock; + if (conf.version != null && !KNOWN_LOCK_VERSIONS.includes(conf.version)) { + throw new UnknownLockVersionError( + `wmill-lock.yaml is at unknown version "${conf.version}". This was ` + + `written by a newer wmill CLI; please upgrade with \`wmill upgrade\`. ` + + `Refusing to operate to avoid corrupting the lockfile.`, + ); + } + return conf; } function v2LockPath(path: string, subpath?: string) { @@ -1182,16 +1243,21 @@ export async function checkifMetadataUptodate( if (!conf.locks) { return false; } - const isV2 = conf?.version == "v2"; + const isFlatKeyed = conf?.version === "v2"; - if (isV2) { - const current = conf.locks?.[v2LockPath(path, subpath)]; - return current == hash; - } else { - const obj = conf.locks?.[path]; - const current = subpath && typeof obj == "object" ? obj?.[subpath] : obj; - return current == hash; + // Older CLIs sometimes wrote entries with a leading "./" prefix; some + // lockfiles even contain both forms with different stale values. Accept + // either form so duplicate-key drift doesn't cause false positives. + if (isFlatKeyed) { + const key = v2LockPath(path, subpath); + return conf.locks?.[key] === hash || conf.locks?.["./" + key] === hash; } + for (const p of [path, "./" + path]) { + const obj = conf.locks?.[p]; + const v = subpath && typeof obj == "object" ? obj?.[subpath] : obj; + if (v === hash) return true; + } + return false; } export async function generateScriptHash( @@ -1249,17 +1315,26 @@ export async function clearGlobalLock(path: string): Promise { if (!conf?.locks) { conf.locks = {}; } - const isV2 = conf?.version == "v2"; + const isFlatKeyed = conf?.version === "v2"; - if (isV2) { - // Remove the specific v2 lock entry + if (isFlatKeyed) { + // Remove the specific flat-keyed lock entry. Match both the canonical + // form and the legacy "./"-prefixed form so duplicate entries left over + // from older CLIs get cleaned up alongside the canonical write. + // Match exactly `key` or `key+` (and the same for the legacy + // "./"-prefixed form). The "+" separator boundary is critical: a plain + // startsWith("f/foo") would also match "f/foobar+..." entries belonging + // to a sibling script. const key = v2LockPath(path); + const legacyKey = "./" + key; if (conf.locks) { Object.keys(conf.locks).forEach((k) => { - if (conf.locks) { - if (k.startsWith(key)) { - delete conf.locks[k]; - } + if (!conf.locks) return; + if ( + k === key || k.startsWith(key + "+") || + k === legacyKey || k.startsWith(legacyKey + "+") + ) { + delete conf.locks[k]; } }); } @@ -1280,9 +1355,9 @@ export async function updateMetadataGlobalLock( if (!conf?.locks) { conf.locks = {}; } - const isV2 = conf?.version == "v2"; + const isFlatKeyed = conf?.version === "v2"; - if (isV2) { + if (isFlatKeyed) { conf.locks[v2LockPath(path, subpath)] = hash; } else { if (subpath) { diff --git a/cli/src/utils/resource_folders.ts b/cli/src/utils/resource_folders.ts index 8a72a8b9ff..653c443935 100644 --- a/cli/src/utils/resource_folders.ts +++ b/cli/src/utils/resource_folders.ts @@ -553,6 +553,23 @@ export function getScriptBasePathFromModulePath(p: string): string | undefined { return norm.slice(0, idx); } +/** + * Convert a local script file path to its Windmill API remote path. + * Handles both folder layout (`u/admin/my_script__mod/script.ts` -> `u/admin/my_script`) + * and flat layout (`u/admin/my_script.ts` -> `u/admin/my_script`). + * + * For flat layout, splits at the FIRST `.` which means a script under a + * folder containing a `.` in its name (e.g. `u/my.folder/script.ts`) gets + * truncated; that pre-dates this helper and is preserved here for parity. + */ +export function scriptPathToRemotePath(p: string): string { + return ( + isModuleEntryPoint(p) + ? getScriptBasePathFromModulePath(p)! + : p.substring(0, p.indexOf(".")) + ).replaceAll(SEP, "/"); +} + // ============================================================================ // Sync-related Path Functions // ============================================================================ diff --git a/cli/test/fixtures/yaml-snapshots/.gitattributes b/cli/test/fixtures/yaml-snapshots/.gitattributes new file mode 100644 index 0000000000..72a8adf806 --- /dev/null +++ b/cli/test/fixtures/yaml-snapshots/.gitattributes @@ -0,0 +1 @@ +*.yaml text eol=lf diff --git a/cli/test/fixtures/yaml-snapshots/app_basic.yaml b/cli/test/fixtures/yaml-snapshots/app_basic.yaml new file mode 100644 index 0000000000..6c77fe8868 --- /dev/null +++ b/cli/test/fixtures/yaml-snapshots/app_basic.yaml @@ -0,0 +1,39 @@ +summary: Dashboard +value: + fullscreen: false + grid: + - '12': + fixed: true + h: 6 + w: 12 + x: 0 + y: 0 + '3': + fixed: true + h: 6 + w: 12 + x: 0 + y: 0 + id: table_a + data: + type: tablecomponent + componentInput: + type: runnable + fields: {} + fieldType: any + runnable: + type: runnableByPath + path: f/lib/list_users + schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: {} + required: [] + configuration: {} + hiddenInlineScripts: [] + norefreshbar: false + unusedInlineScripts: [] +policy: + execution_mode: viewer + on_behalf_of_email: admin@windmill.dev + triggerables_v2: {} diff --git a/cli/test/fixtures/yaml-snapshots/app_raw.yaml b/cli/test/fixtures/yaml-snapshots/app_raw.yaml new file mode 100644 index 0000000000..8b1bbdf223 --- /dev/null +++ b/cli/test/fixtures/yaml-snapshots/app_raw.yaml @@ -0,0 +1,14 @@ +summary: Raw HTML app +value: + files: + app.js: '!inline_fileset app.js' + index.html: '!inline_fileset index.html' + runnables: + fetch_data: + fields: {} + kind: runnableByPath + path: f/lib/fetch_data +policy: + execution_mode: viewer + on_behalf_of_email: admin@windmill.dev + triggerables_v2: {} diff --git a/cli/test/fixtures/yaml-snapshots/flow_basic.yaml b/cli/test/fixtures/yaml-snapshots/flow_basic.yaml new file mode 100644 index 0000000000..9c9cd3e2fb --- /dev/null +++ b/cli/test/fixtures/yaml-snapshots/flow_basic.yaml @@ -0,0 +1,25 @@ +summary: Basic flow +description: Two sequential script modules +value: + modules: + - id: a + value: + type: script + input_transforms: + x: + type: static + value: hello + path: f/lib/uppercase + - id: b + value: + type: script + input_transforms: + input: + type: javascript + expr: results.a + path: f/lib/print +schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: {} + required: [] diff --git a/cli/test/fixtures/yaml-snapshots/flow_branches_loops.yaml b/cli/test/fixtures/yaml-snapshots/flow_branches_loops.yaml new file mode 100644 index 0000000000..f8378aadba --- /dev/null +++ b/cli/test/fixtures/yaml-snapshots/flow_branches_loops.yaml @@ -0,0 +1,61 @@ +summary: Branches and loops +description: '' +value: + modules: + - id: loop + value: + type: forloopflow + modules: + - id: step + value: + type: rawscript + content: '!inline loop/step.bun.ts' + input_transforms: + n: + type: javascript + expr: flow_input.iter.value + language: bun + iterator: + type: javascript + expr: '[1,2,3]' + parallel: false + parallelism: 4 + skip_failures: false + - id: branchall + value: + type: branchall + branches: + - summary: Left branch + modules: + - id: left + value: + type: identity + parallel: false + skip_failure: false + - summary: Right branch + modules: + - id: right + value: + type: identity + parallel: false + skip_failure: false + parallel: true + - id: branchone + value: + type: branchone + branches: + - summary: Case A + modules: + - id: case_a + value: + type: identity + expr: results.branchall.length > 0 + default: + - id: fallback + value: + type: identity +schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: {} + required: [] diff --git a/cli/test/fixtures/yaml-snapshots/flow_inline_tags.yaml b/cli/test/fixtures/yaml-snapshots/flow_inline_tags.yaml new file mode 100644 index 0000000000..0b72cc5196 --- /dev/null +++ b/cli/test/fixtures/yaml-snapshots/flow_inline_tags.yaml @@ -0,0 +1,19 @@ +summary: Inline tag string-prefix form +description: Inline scripts and locks expressed as '!inline path' strings +value: + modules: + - id: step1 + value: + type: rawscript + content: '!inline step1/script.bun.ts' + input_transforms: + x: + type: static + value: 1 + lock: '!inline step1/script.lock' + language: bun +schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: {} + required: [] diff --git a/cli/test/fixtures/yaml-snapshots/flow_notes_groups.yaml b/cli/test/fixtures/yaml-snapshots/flow_notes_groups.yaml new file mode 100644 index 0000000000..d5ef973272 --- /dev/null +++ b/cli/test/fixtures/yaml-snapshots/flow_notes_groups.yaml @@ -0,0 +1,40 @@ +summary: Notes and groups (#8641) +description: '' +value: + modules: + - id: fetch + summary: Fetch product + value: + type: script + input_transforms: {} + is_trigger: false + path: f/api/product_get + - id: map + summary: Map item + value: + type: script + input_transforms: + bc_item: + type: javascript + expr: flow_input.bc_item + is_trigger: false + path: f/mapping/item_to_product + groups: + - summary: My group + color: green + end_id: map + start_id: fetch + notes: + - id: note-abc123 + type: group + color: blue + contained_node_ids: + - fetch + - map + locked: false + text: These steps must run together +schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: {} + required: [] diff --git a/cli/test/fixtures/yaml-snapshots/flow_retry_suspend_sleep.yaml b/cli/test/fixtures/yaml-snapshots/flow_retry_suspend_sleep.yaml new file mode 100644 index 0000000000..9afe85715e --- /dev/null +++ b/cli/test/fixtures/yaml-snapshots/flow_retry_suspend_sleep.yaml @@ -0,0 +1,58 @@ +summary: Retry / suspend / sleep +description: '' +value: + modules: + - id: with_retry + value: + type: script + input_transforms: {} + path: f/lib/may_fail + retry: + constant: + attempts: 3 + seconds: 5 + exponential: + attempts: 2 + multiplier: 2 + random_factor: 0 + seconds: 1 + sleep: + type: static + value: 1 + - id: with_suspend + value: + type: script + input_transforms: {} + path: f/lib/manual_review + suspend: + continue_on_disapproved: false + hide_cancel: false + required_events: 1 + resume_form: + schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + approved: + type: boolean + required: + - approved + timeout: 86400 + failure_module: + id: failure + value: + type: script + input_transforms: {} + path: f/lib/notify_failure + preprocessor_module: + id: preprocessor + value: + type: rawscript + content: '!inline preprocessor.bun.ts' + input_transforms: {} + language: bun +schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: {} + required: [] diff --git a/cli/test/fixtures/yaml-snapshots/regenerate.ts b/cli/test/fixtures/yaml-snapshots/regenerate.ts new file mode 100644 index 0000000000..7e8727aee0 --- /dev/null +++ b/cli/test/fixtures/yaml-snapshots/regenerate.ts @@ -0,0 +1,440 @@ +/** + * Regenerator for the yaml-snapshot fixtures. + * + * Run from the cli directory: + * + * bun run test/fixtures/yaml-snapshots/regenerate.ts + * + * Each fixture is built from the JS object below by passing it through + * `yamlStringify(obj, yamlOptions)` — the exact same path `wmill sync pull` + * uses to write `flow.yaml` / `app.yaml` / ` + + + {#snippet trigger()} +