From ab1c3ee46205630d8a3e7edd4e8cd3bd3148d25f Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 18 Jun 2026 09:10:40 +0200 Subject: [PATCH 01/41] chore(main): release 1.729.0 (#9632) * chore(main): release 1.729.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> --- CHANGELOG.md | 22 ++ backend/Cargo.lock | 331 +++++------------- backend/Cargo.toml | 4 +- .../parsers/windmill-parser-wasm/Cargo.lock | 48 +-- .../parsers/windmill-parser-wasm/Cargo.toml | 2 +- backend/windmill-api/openapi.yaml | 2 +- benchmarks/lib.ts | 2 +- cli/src/core/constants.ts | 2 +- frontend/package-lock.json | 4 +- frontend/package.json | 2 +- lsp/Pipfile | 2 +- openflow.openapi.yaml | 2 +- .../WindmillClient/WindmillClient.psd1 | 2 +- python-client/wmill/pyproject.toml | 2 +- typescript-client/jsr.json | 2 +- typescript-client/package.json | 2 +- version.txt | 2 +- 17 files changed, 155 insertions(+), 278 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11f5d8e97c..fbe24efa34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## [1.729.0](https://github.com/windmill-labs/windmill/compare/v1.728.1...v1.729.0) (2026-06-18) + + +### Features + +* add ducklake schema support to the database manager ([#9633](https://github.com/windmill-labs/windmill/issues/9633)) ([3eeccaf](https://github.com/windmill-labs/windmill/commit/3eeccaf9682b7803fdf5be8dcbc4d243e0ba2e49)) +* **ai-chat:** self-hosted docs tools via windmill.dev llms.txt + ask benchmark ([#9578](https://github.com/windmill-labs/windmill/issues/9578)) ([f4425fc](https://github.com/windmill-labs/windmill/commit/f4425fca9fb0d02b845bd72888ade54905c5a30b)) +* **frontend:** View Diff and in-place Load for other users' drafts ([#9621](https://github.com/windmill-labs/windmill/issues/9621)) ([5508f1d](https://github.com/windmill-labs/windmill/commit/5508f1da9cd04c2583eb3f7ee6bce19d067f2227)) +* per-user draft review & deploy page (gating, badges, rename, raw-app deploy fixes) ([#9625](https://github.com/windmill-labs/windmill/issues/9625)) ([e09cd58](https://github.com/windmill-labs/windmill/commit/e09cd5862cb636e143027fe8d9a5be9c7097b031)) +* queue messages typed while ai chat is streaming ([#9525](https://github.com/windmill-labs/windmill/issues/9525)) ([51bd869](https://github.com/windmill-labs/windmill/commit/51bd8692a482850f7ac8b04dd16db5876336b5b9)) +* zero-setup oauth client credentials for registry providers ([#9559](https://github.com/windmill-labs/windmill/issues/9559)) ([e26a923](https://github.com/windmill-labs/windmill/commit/e26a9239a62a25abf90ef06ade4dde7f36e791bb)) + + +### Bug Fixes + +* **ai_evals:** adapt global eval harness to DB-backed user drafts ([#9641](https://github.com/windmill-labs/windmill/issues/9641)) ([e87ff79](https://github.com/windmill-labs/windmill/commit/e87ff79ecf6a6e0958916ed1b3756fb3addf719f)) +* **drafts:** preserve original timestamp when migrating localStorage drafts ([#9638](https://github.com/windmill-labs/windmill/issues/9638)) ([8021775](https://github.com/windmill-labs/windmill/commit/8021775f5f961ef6fd01b022639b85855326a1da)) +* **frontend:** don't save drafts on leave when auto-save is off, warn instead ([#9630](https://github.com/windmill-labs/windmill/issues/9630)) ([2523465](https://github.com/windmill-labs/windmill/commit/252346500945a9571af744c839ac0c7d6870504f)) +* **frontend:** render Modal2 dialogs above the AI chat panel ([#9636](https://github.com/windmill-labs/windmill/issues/9636)) ([b67c8cf](https://github.com/windmill-labs/windmill/commit/b67c8cf42b477575fc1bc448058ec0d3b7e54fee)) +* **frontend:** show AI sessions when AI unconfigured, with disabled chat ([#9644](https://github.com/windmill-labs/windmill/issues/9644)) ([ba69d81](https://github.com/windmill-labs/windmill/commit/ba69d8147b615e160cf3d2885fc65a0777b78b71)) +* **git-sync:** bump default sync script to hub/28719 (windmill-cli 1.728.1) for WAC modules ([#9649](https://github.com/windmill-labs/windmill/issues/9649)) ([3c0e38b](https://github.com/windmill-labs/windmill/commit/3c0e38b5890d77983cb5cf5f422a62a73e7a4f22)) + ## [1.728.1](https://github.com/windmill-labs/windmill/compare/v1.728.0...v1.728.1) (2026-06-17) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index a3a2e94b11..0db0d88472 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -5066,16 +5066,14 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", "rand_core 0.10.1", - "wasip2", - "wasip3", ] [[package]] @@ -5800,7 +5798,7 @@ dependencies = [ "tokio", "tokio-rustls 0.26.4", "tower-service", - "webpki-roots 1.0.7", + "webpki-roots 1.0.8", ] [[package]] @@ -5993,12 +5991,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "id-arena" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - [[package]] name = "ident_case" version = "1.0.1" @@ -6506,12 +6498,6 @@ dependencies = [ "spin 0.9.8", ] -[[package]] -name = "leb128fmt" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" - [[package]] name = "levenshtein_automata" version = "0.2.1" @@ -7273,9 +7259,9 @@ dependencies = [ [[package]] name = "mysql_common" -version = "0.37.2" +version = "0.37.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b42ced54aa8ac97226486337973f9bc3956e24f03a23e88a6e18f640959d6e2" +checksum = "0f27695f286b461da077b8c2f72f47feaa04ce3c3f9c0976257410e90e21208a" dependencies = [ "base64 0.22.1", "bitflags 2.13.0", @@ -9112,7 +9098,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ "chacha20", - "getrandom 0.4.2", + "getrandom 0.4.3", "rand_core 0.10.1", ] @@ -9477,7 +9463,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 1.0.7", + "webpki-roots 1.0.8", ] [[package]] @@ -12025,7 +12011,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom 0.4.3", "once_cell", "rustix 1.1.4", "windows-sys 0.61.2", @@ -13358,7 +13344,7 @@ version = "1.23.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" dependencies = [ - "getrandom 0.4.2", + "getrandom 0.4.3", "js-sys", "serde_core", "wasm-bindgen", @@ -13460,16 +13446,7 @@ version = "1.0.4+wasi-0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" dependencies = [ - "wit-bindgen 0.57.1", -] - -[[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" -dependencies = [ - "wit-bindgen 0.51.0", + "wit-bindgen", ] [[package]] @@ -13575,28 +13552,6 @@ dependencies = [ "syn 2.0.118", ] -[[package]] -name = "wasm-encoder" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" -dependencies = [ - "leb128fmt", - "wasmparser", -] - -[[package]] -name = "wasm-metadata" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" -dependencies = [ - "anyhow", - "indexmap 2.14.0", - "wasm-encoder", - "wasmparser", -] - [[package]] name = "wasm-streams" version = "0.4.2" @@ -13620,18 +13575,6 @@ dependencies = [ "thiserror 2.0.18", ] -[[package]] -name = "wasmparser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" -dependencies = [ - "bitflags 2.13.0", - "hashbrown 0.15.5", - "indexmap 2.14.0", - "semver 1.0.28", -] - [[package]] name = "wasmtimer" version = "0.4.3" @@ -13680,9 +13623,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" +checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267" dependencies = [ "rustls-pki-types", ] @@ -13693,14 +13636,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.7", + "webpki-roots 1.0.8", ] [[package]] name = "webpki-roots" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +checksum = "bf85cb06032201fa7c6f829d7db5a7e5aa45bcc0655327713065f6f0576731bf" dependencies = [ "rustls-pki-types", ] @@ -13792,7 +13735,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "async-nats", @@ -13874,7 +13817,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.728.1" +version = "1.729.0" dependencies = [ "async-stream", "async-trait", @@ -13907,7 +13850,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.728.1" +version = "1.729.0" dependencies = [ "axum 0.8.9", "chrono", @@ -13920,7 +13863,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "argon2", @@ -14058,7 +14001,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.728.1" +version = "1.729.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14081,7 +14024,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.728.1" +version = "1.729.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14094,7 +14037,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14120,7 +14063,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.728.1" +version = "1.729.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -14130,7 +14073,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.728.1" +version = "1.729.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14147,7 +14090,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.728.1" +version = "1.729.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -14169,7 +14112,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14192,7 +14135,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.728.1" +version = "1.729.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14208,7 +14151,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.728.1" +version = "1.729.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14229,7 +14172,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.728.1" +version = "1.729.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14250,7 +14193,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.728.1" +version = "1.729.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14264,7 +14207,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "async-nats", @@ -14299,7 +14242,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14324,7 +14267,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.728.1" +version = "1.729.0" dependencies = [ "axum 0.8.9", "flate2", @@ -14342,7 +14285,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14364,7 +14307,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.728.1" +version = "1.729.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14384,7 +14327,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.728.1" +version = "1.729.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14415,7 +14358,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14443,7 +14386,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.728.1" +version = "1.729.0" dependencies = [ "lazy_static", "serde", @@ -14455,7 +14398,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.728.1" +version = "1.729.0" dependencies = [ "argon2", "axum 0.8.9", @@ -14480,7 +14423,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.728.1" +version = "1.729.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14494,7 +14437,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.728.1" +version = "1.729.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14527,7 +14470,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.728.1" +version = "1.729.0" dependencies = [ "chrono", "lazy_static", @@ -14541,7 +14484,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -14560,7 +14503,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.728.1" +version = "1.729.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -14662,7 +14605,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.728.1" +version = "1.729.0" dependencies = [ "chrono", "itertools 0.14.0", @@ -14681,7 +14624,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.728.1" +version = "1.729.0" dependencies = [ "regex", "serde", @@ -14696,7 +14639,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -14720,7 +14663,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "futures", @@ -14737,7 +14680,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.728.1" +version = "1.729.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -14753,7 +14696,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "async-trait", @@ -14774,7 +14717,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "async-trait", @@ -14805,7 +14748,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "arc-swap", @@ -14830,7 +14773,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "async-stream", @@ -14864,7 +14807,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "futures", @@ -14882,7 +14825,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.728.1" +version = "1.729.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -14891,7 +14834,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "lazy_static", @@ -14903,7 +14846,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "serde_json", @@ -14915,7 +14858,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "gosyn", @@ -14927,7 +14870,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "lazy_static", @@ -14939,7 +14882,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "serde_json", @@ -14951,7 +14894,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "nu-parser", @@ -14962,7 +14905,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14973,7 +14916,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -14985,7 +14928,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "rustpython-ast", @@ -14996,7 +14939,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "async-recursion", @@ -15018,7 +14961,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "serde_json", @@ -15030,7 +14973,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "lazy_static", @@ -15044,7 +14987,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15061,7 +15004,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "lazy_static", @@ -15074,7 +15017,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "serde", @@ -15086,7 +15029,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "lazy_static", @@ -15104,7 +15047,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -15120,7 +15063,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "rustpython-ast", @@ -15136,7 +15079,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "serde", @@ -15147,7 +15090,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "async-recursion", @@ -15185,7 +15128,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "const_format", @@ -15224,7 +15167,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.728.1" +version = "1.729.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -15235,7 +15178,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "async-recursion", @@ -15267,7 +15210,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "async-trait", @@ -15291,7 +15234,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "async-trait", @@ -15324,7 +15267,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "async-trait", @@ -15357,7 +15300,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "async-trait", @@ -15377,7 +15320,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "async-trait", @@ -15411,7 +15354,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "async-trait", @@ -15447,7 +15390,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "async-trait", @@ -15470,7 +15413,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "async-trait", @@ -15494,7 +15437,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "async-nats", @@ -15518,7 +15461,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "async-trait", @@ -15553,7 +15496,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "async-trait", @@ -15581,7 +15524,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "async-trait", @@ -15606,7 +15549,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "bitflags 2.13.0", @@ -15625,7 +15568,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "async-once-cell", @@ -15735,7 +15678,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.728.1" +version = "1.729.0" dependencies = [ "bytes", "futures", @@ -16348,100 +16291,12 @@ version = "0.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" -[[package]] -name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - [[package]] name = "wit-bindgen" version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" -[[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck", - "indexmap 2.14.0", - "prettyplease", - "syn 2.0.118", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn 2.0.118", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags 2.13.0", - "indexmap 2.14.0", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] - -[[package]] -name = "wit-parser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" -dependencies = [ - "anyhow", - "id-arena", - "indexmap 2.14.0", - "log", - "semver 1.0.28", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", -] - [[package]] name = "writeable" version = "0.6.3" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 5d2c556965..4d41151c57 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.728.1" +version = "1.729.0" authors.workspace = true edition.workspace = true @@ -87,7 +87,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.728.1" +version = "1.729.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index f09c528521..cd05b53b71 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6183,7 +6183,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.728.1" +version = "1.729.0" dependencies = [ "aho-corasick", "anyhow", @@ -6263,7 +6263,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.728.1" +version = "1.729.0" dependencies = [ "proc-macro2", "quote", @@ -6275,7 +6275,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.728.1" +version = "1.729.0" dependencies = [ "convert_case", "serde", @@ -6284,7 +6284,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "lazy_static", @@ -6296,7 +6296,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "serde_json", @@ -6308,7 +6308,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "gosyn", @@ -6320,7 +6320,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "lazy_static", @@ -6332,7 +6332,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "serde_json", @@ -6344,7 +6344,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "nu-parser", @@ -6355,7 +6355,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6366,7 +6366,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6378,7 +6378,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6389,7 +6389,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "async-recursion", @@ -6411,7 +6411,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "serde_json", @@ -6423,7 +6423,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "lazy_static", @@ -6437,7 +6437,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "convert_case", @@ -6454,7 +6454,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "lazy_static", @@ -6467,7 +6467,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "serde", @@ -6479,7 +6479,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "lazy_static", @@ -6497,7 +6497,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6513,7 +6513,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6529,7 +6529,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6561,7 +6561,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "serde", @@ -6572,7 +6572,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.728.1" +version = "1.729.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index 686be7d91a..6b43533f5b 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.728.1" +version = "1.729.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 0a6bdaffa9..cd5b320339 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.728.1 + version: 1.729.0 title: Windmill API contact: diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 742f715e5a..ac6780ef14 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.728.1"; +export const VERSION = "v1.729.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 0807ef50e0..be07eed833 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.728.1"; +export const VERSION = "1.729.0"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index f1c5cc4ec3..6a6d281b59 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.728.1", + "version": "1.729.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.728.1", + "version": "1.729.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 4ead8ea2cb..341d86c709 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.728.1", + "version": "1.729.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/lsp/Pipfile b/lsp/Pipfile index 09e70331c1..8834fdae89 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,7 +4,7 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.728.1" +wmill = ">=1.729.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 97fc9a3a2c..18fe9617ef 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: '3.0.3' info: - version: 1.728.1 + version: 1.729.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index 0c83d2e5f8..683e455ffa 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.728.1' + ModuleVersion = '1.729.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 62abaccf5f..5e0ae61a24 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.728.1" +version = "1.729.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 2c284bc2c1..16165a2077 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.728.1", + "version": "1.729.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./sqlUtils.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 9ac3f6dd2e..eea14bb700 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.728.1", + "version": "1.729.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/typescript-client#readme", diff --git a/version.txt b/version.txt index a6fe01083d..786a9673d1 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.728.1 +1.729.0 From 471147135b419d7c82544bec740ee8c909d99529 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Thu, 18 Jun 2026 16:44:50 +0200 Subject: [PATCH 02/41] oauth: complete Coupa managed client-credentials (instance mapping + default scopes) (#9651) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * oauth: map Coupa instance to instance_url resource arg Coupa's managed client-credentials connect collects an instance name to host-pin the token URL but had no resource_mapping, so the created resource's instance_url (the API base URL the hub scripts build on) stayed empty. Add the mapping, mirroring ServiceNow, so the entered instance fills it automatically. Co-Authored-By: Claude Fable 5 * oauth: default Coupa client-credentials scopes (cc_scopes) Prefill the connect dialog's scope field with the core.* scopes the Coupa hub scripts exercise — read+write for suppliers/purchase_orders/requisitions/invoices, read-only for contracts/expenses (the shipped scripts only read those). Scope names verified against the Coupa scope docs and corroborated in production code. The user can trim them to what their OIDC client is granted. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- backend/oauth_connect.json | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/backend/oauth_connect.json b/backend/oauth_connect.json index 0b9a19b873..421136fc77 100644 --- a/backend/oauth_connect.json +++ b/backend/oauth_connect.json @@ -133,12 +133,27 @@ }, "coupa": { "grant_types": ["client_credentials"], + "cc_scopes": [ + "core.supplier.read", + "core.supplier.write", + "core.purchase_order.read", + "core.purchase_order.write", + "core.requisition.read", + "core.requisition.write", + "core.invoice.read", + "core.invoice.write", + "core.contract.read", + "core.expense.read" + ], "connect_config_template": { "display_name": "Coupa", "label": "Coupa instance", "placeholder": "your-instance", "token_url": "https://{instance}.coupahost.com/oauth2/token", - "strip_suffix": ".coupahost.com" + "strip_suffix": ".coupahost.com", + "resource_mapping": { + "instance_url": "https://{instance}.coupahost.com" + } } }, "sage_intacct": { From fdd82f0c48f29805cd9e219649f27fba45c7fd92 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Thu, 18 Jun 2026 16:45:10 +0200 Subject: [PATCH 03/41] fix: gate agent-worker global setting reads with a blocklist (#9623) * fix: restrict agent-worker global setting reads to an allowlist Add AGENT_WORKER_READABLE_SETTINGS allowlist of the operational settings agent workers load over HTTP, with a helper used by the agent endpoint to reject any other key. Bump ee-repo-ref for the companion EE handler change. Co-Authored-By: Claude Opus 4.8 (1M context) * chore: update ee-repo-ref to 3fab9f01ecce3dad0aa9b9c544d41f1e88bc81dd This commit updates the EE repository reference after PR #615 was merged in windmill-ee-private. Previous ee-repo-ref: 8a657066fda1c5ffe225588bce6c349cffd81e98 New ee-repo-ref: 3fab9f01ecce3dad0aa9b9c544d41f1e88bc81dd Automated by sync-ee-ref workflow. * fix: make agent-worker setting gate a blocklist instead of allowlist Switch is_setting_readable_by_agent_worker to deny-by-exception: serve every global setting to agent workers except AGENT_WORKER_BLOCKED_SETTINGS (the instance secrets). Update tests and bump ee-repo-ref for the companion comment change. Co-Authored-By: Claude Opus 4.8 (1M context) * docs: remind to blocklist new secret settings for agent workers Co-Authored-By: Claude Opus 4.8 (1M context) * chore: update ee-repo-ref to 9e4dadafb44ba953a7d2af2be12b92be98d86b66 This commit updates the EE repository reference after PR #618 was merged in windmill-ee-private. Previous ee-repo-ref: 8e32afb69ffc4d5f080c0c4bc6b023d57d0f39ae New ee-repo-ref: 9e4dadafb44ba953a7d2af2be12b92be98d86b66 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: windmill-internal-app[bot] --- backend/ee-repo-ref.txt | 2 +- .../windmill-common/src/global_settings.rs | 108 ++++++++++++++++++ 2 files changed, 109 insertions(+), 1 deletion(-) diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index e8927cee6d..0ed0d2d025 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -136f4634aca61e74ccb045372358a1e3f6b23e75 +9e4dadafb44ba953a7d2af2be12b92be98d86b66 diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 732b9cdc30..bed78113b5 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -1,3 +1,8 @@ +// Adding a global setting? Decide whether agent workers may read it. Agent +// workers (remote workers connected over HTTP) fetch settings through an +// endpoint that is deny-by-exception: every key is served except those in +// AGENT_WORKER_BLOCKED_SETTINGS (defined below). If a new setting holds an +// instance secret the server should keep to itself, add its key there. pub const CUSTOM_TAGS_SETTING: &str = "custom_tags"; pub const DEFAULT_TAGS_PER_WORKSPACE_SETTING: &str = "default_tags_per_workspace"; pub const DEFAULT_TAGS_WORKSPACES_SETTING: &str = "default_tags_workspaces"; @@ -104,6 +109,53 @@ pub const WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING: &str = "workspace_fairness_max pub const WORKSPACE_FAIRNESS_DURATION_SECS_SETTING: &str = "workspace_fairness_duration_secs"; pub const WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING: &str = "workspace_fairness_min_total_jobs"; +/// Global settings an agent worker (a remote worker connected over HTTP instead +/// of to the database) must NEVER read through +/// `GET /api/agent_workers/get_global_setting/{key}`. Every other key is served. +/// +/// SECURITY: that endpoint is authenticated only by an agent-worker JWT and +/// returns the raw `global_settings` value for the requested key. Because the +/// policy is deny-by-exception (anything not listed here is readable), every +/// setting that holds an instance secret or credential an agent worker does not +/// need MUST be listed below. Missing one discloses it to every agent worker — +/// `jwt_secret` is the worst case (a token holder could forge a superadmin JWT), +/// but `oauths`, `smtp_settings`, `secret_backend`, object-store credentials, +/// etc. are instance-wide secrets too. +/// +/// NOT blocked, on purpose: the operational credentials an agent worker loads to +/// run jobs (`license_key`, `hub_api_secret`, `sandbox_registry_auth`, +/// `powershell_repo_pat`, `npmrc`, ...). Those are already within an agent +/// worker's trust boundary, and blocking them breaks worker startup or +/// dependency installation. When adding a new setting that stores a secret the +/// server keeps to itself, add it here. +pub const AGENT_WORKER_BLOCKED_SETTINGS: &[&str] = &[ + // Instance identity / auth secrets — disclosure enables privilege escalation + // or impersonation. + JWT_SECRET_SETTING, + OAUTH_SETTING, + SMTP_SETTING, + SCIM_TOKEN_SETTING, + SAML_METADATA_SETTING, + SECRET_BACKEND_SETTING, + GITHUB_ENTERPRISE_APP_SETTING, + OBJECT_STORE_CONFIG_SETTING, + AI_CONFIG_SETTING, + TEAMS_SETTING, + INDEXER_SETTING, + // Server-only configs that may embed credentials, webhook URLs or tokens and + // are never loaded by an agent worker. + CRITICAL_ERROR_CHANNELS_SETTING, + INSTANCE_EVENTS_WEBHOOK_SETTING, + OTEL_SETTING, + OTEL_TRACING_PROXY_SETTING, +]; + +/// Whether an agent worker may read the given global setting over HTTP. +/// Deny-by-exception: everything is readable except [`AGENT_WORKER_BLOCKED_SETTINGS`]. +pub fn is_setting_readable_by_agent_worker(name: &str) -> bool { + !AGENT_WORKER_BLOCKED_SETTINGS.contains(&name) +} + use std::sync::atomic::AtomicBool; lazy_static::lazy_static! { @@ -265,6 +317,62 @@ pub fn workspace_integration_auth_endpoint(client_name: &str, base_url: &str) -> } } +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn agent_workers_can_read_operational_settings() { + // Operational knobs and the credentials a worker needs to run jobs are + // intentionally NOT blocked. Deny-by-exception also means an arbitrary + // unlisted key is readable. + for key in [ + NPMRC_SETTING, + PIP_INDEX_URL_SETTING, + JOB_ISOLATION_SETTING, + LICENSE_KEY_SETTING, + HUB_API_SECRET_SETTING, + SANDBOX_REGISTRY_AUTH_SETTING, + POWERSHELL_REPO_PAT_SETTING, + "some_future_operational_setting", + ] { + assert!( + is_setting_readable_by_agent_worker(key), + "'{key}' must remain readable by agent workers" + ); + } + } + + #[test] + fn agent_workers_cannot_read_instance_secrets() { + // Disclosing any of these to a remote worker enables privilege + // escalation (jwt_secret -> forged superadmin JWT) or leaks instance + // secrets. They must never be served by the agent-worker endpoint. + for key in [ + JWT_SECRET_SETTING, + OAUTH_SETTING, + SMTP_SETTING, + SCIM_TOKEN_SETTING, + SAML_METADATA_SETTING, + SECRET_BACKEND_SETTING, + GITHUB_ENTERPRISE_APP_SETTING, + OBJECT_STORE_CONFIG_SETTING, + AI_CONFIG_SETTING, + TEAMS_SETTING, + INDEXER_SETTING, + CRITICAL_ERROR_CHANNELS_SETTING, + INSTANCE_EVENTS_WEBHOOK_SETTING, + OTEL_SETTING, + OTEL_TRACING_PROXY_SETTING, + ] { + assert!( + !is_setting_readable_by_agent_worker(key), + "'{key}' is an instance secret and must not be readable by agent workers" + ); + } + } +} + pub async fn set_value_in_global_settings( db: &Pool, setting_name: &str, From 5d553b81c06664aab61131a93b198575c088d12d Mon Sep 17 00:00:00 2001 From: centdix <40307056+centdix@users.noreply.github.com> Date: Thu, 18 Jun 2026 16:51:06 +0200 Subject: [PATCH 04/41] feat(ai-chat): summary-based conversation compaction (#9645) * feat(ai-chat): summary-based conversation compaction Replace drop-oldest compaction with summary-based partial compaction: when a send would cross the context-window trigger, summarize the older prefix into one message and keep the recent tail verbatim, replacing the prefix in both the model context and the visible transcript with a collapsible boundary. Drop-oldest remains a fallback; a circuit breaker disables the summary round-trip after repeated failures. Co-Authored-By: Claude Opus 4.8 (1M context) * nit * fix(ai-chat): address review findings on summary compaction - Stop during an in-flight summary no longer falls through to a destructive drop-oldest compaction. The aborted controller short-circuits the fallback and its save, so the cancel path rolls the unsent turn back cleanly instead of permanently dropping older history (P1). - Preserve the original chat title across compaction: once the summary boundary leads the transcript, reuse the title computed before compaction rather than re-deriving it from the first surviving tail message (P2). - Strip every block from the model's summary, not just the first, so extra scratchpad blocks can't leak into context (P2). - Reindent AIChatMessage.svelte / ContextUsageIndicator.svelte (prettier). Adds regression tests for the abort path, title preservation, and multi-analysis stripping. Co-Authored-By: Claude Opus 4.8 * nit --------- Co-authored-by: Claude Opus 4.8 (1M context) --- .../copilot/chat/AIChatDisplay.svelte | 17 +- .../copilot/chat/AIChatManager.svelte.ts | 197 ++++++++++++++++-- .../copilot/chat/AIChatManager.test.ts | 144 ++++++++++++- .../copilot/chat/AIChatMessage.svelte | 197 +++++++++--------- .../copilot/chat/CompactionBoundary.svelte | 33 +++ .../copilot/chat/ContextManager.svelte.ts | 34 +-- .../copilot/chat/ContextUsageIndicator.svelte | 22 +- .../copilot/chat/HistoryManager.svelte.ts | 16 +- .../copilot/chat/HistoryManager.test.ts | 27 +++ .../copilot/chat/compactionPrompt.test.ts | 64 ++++++ .../copilot/chat/compactionPrompt.ts | 124 +++++++++++ .../src/lib/components/copilot/chat/shared.ts | 17 +- 12 files changed, 735 insertions(+), 157 deletions(-) create mode 100644 frontend/src/lib/components/copilot/chat/CompactionBoundary.svelte create mode 100644 frontend/src/lib/components/copilot/chat/compactionPrompt.test.ts create mode 100644 frontend/src/lib/components/copilot/chat/compactionPrompt.ts diff --git a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte index cd9c50d1c2..e9ac5eed16 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatDisplay.svelte @@ -464,11 +464,13 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> {:else} {/if} @@ -538,8 +540,6 @@ the panel, or the Escape-to-stop focus check would wrongly reject them. --> {#if inputPreface} {@render inputPreface()} {/if} - - {/if} +
+ +
{#if (aiChatManager.mode === AIMode.NAVIGATOR || aiChatManager.mode === AIMode.ASK) && suggestions.length > 0 && messages.filter((m) => m.role === 'user').length === 0 && !disabled}
diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index cb775a6084..0eb9a79707 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -36,8 +36,13 @@ import { loadApiTools } from './api/apiTools' import { prepareScriptUserMessage } from './script/core' import { prepareNavigatorUserMessage } from './navigator/core' import { sendUserToast } from '$lib/toast' -import { workspaceAIClients } from '../lib' +import { workspaceAIClients, getNonStreamingCompletion } from '../lib' import { getKnownModelContextWindow } from '../modelConfig' +import { + getCompactionSummaryPrompt, + formatCompactSummary, + buildSummaryMessageContent +} from './compactionPrompt' import { dfs } from '$lib/components/flows/previousResults' import { getStringError } from './utils' import { type PasteAttachment } from './pasteTokens' @@ -81,15 +86,25 @@ import { isGlobalAiEnabled } from './global/gate' import { scopedKey, onUserChange, migrateLegacyLocalStorage } from '$lib/userScopedStorage' import { getLocalSetting, storeLocalSetting } from '$lib/utils' -// Drop-oldest compaction of the stored history: once the projected request -// size (contextTokens — the provider's report when current, a fresh chars/4 -// estimate otherwise — plus the new user message) reaches the trigger ratio -// of the model's context window, head messages are dropped until roughly the -// target ratio. The trigger headroom absorbs what the projection cannot see — -// the upcoming completion and tool results, system-prompt/tool-schema changes -// from mode switches, and the estimate's chars/4 error. +// Compaction of the stored history: once the projected request size +// (contextTokens — the provider's report when current, a fresh chars/4 +// estimate otherwise — plus the new user message) reaches the trigger ratio of +// the model's context window, the older prefix is summarized into a single +// message while the recent tail is kept verbatim, bringing the history down to +// roughly the target ratio. The trigger headroom absorbs what the projection +// cannot see — the upcoming completion and tool results, system-prompt/tool- +// schema changes from mode switches, and the estimate's chars/4 error. const COMPACTION_TRIGGER_RATIO = 0.8 const COMPACTION_TARGET_RATIO = 0.7 +// Headroom reserved within the target budget for the summary message itself, so +// the summary + kept tail + overhead land under the target ratio. +const SUMMARY_OUTPUT_RESERVE_TOKENS = 8000 +// Below this many messages in the prefix there's little to gain from a summary +// round-trip; skip straight to drop-oldest. +const MIN_PREFIX_MESSAGES_TO_SUMMARIZE = 4 +// Stop attempting summarization after this many consecutive failures and use +// drop-oldest directly; a successful summarization resets the counter. +const MAX_CONSECUTIVE_COMPACTION_FAILURES = 3 // Abort reason for a deliberate user cancel (Esc / Stop). Programmatic cancels // (panel teardown, save-and-clear) pass their own reason, so the queued-message // flush can tell "the user wants to move on" from "the turn was torn down". @@ -242,6 +257,13 @@ export class AIChatManager { * (provider never reported, turn failed, history rewound). Never holds a * guess: readers go through `contextTokens`, which estimates lazily. */ contextUsage = $state(undefined) + // Circuit breaker for summary-based compaction: after repeated failures the + // summary round-trip is skipped in favor of drop-oldest. Reset on any + // successful summarization. Not persisted — a fresh load gets a fresh chance. + private consecutiveCompactionFailures = 0 + // True while the summarization round-trip is in flight, so the UI can show a + // "Compacting conversation" label on the processing indicator. + compacting = $state(false) autonomyMode = $state(getPersistedAutonomyMode()) autoAcceptEditsAvailable = $derived(supportsAutoAcceptEdits(this.mode)) autoAcceptEditsActive = $derived( @@ -397,6 +419,122 @@ export class AIChatManager { return freed } + /** + * Summary-based partial compaction. Summarizes the older PREFIX of the stored + * history into a single user message and keeps the recent tail verbatim, + * bringing the history down to roughly the target ratio while preserving the + * intent, decisions, and recent work that drop-oldest would discard. + * + * The tail grows from the most recent message until it fills `tailBudget`, + * then snaps forward to a user-message boundary (a leading tool/assistant + * message would dangle without the turn that introduced it). The summary + * replaces the prefix in BOTH `messages` (as a user message) and + * `displayMessages` (as a `summary` boundary); surviving tail user messages + * have their restart `index` re-based onto the new history. + * + * Returns true on success. Returns false — caller falls back to drop-oldest — + * when summarization isn't worthwhile or fails (user abort, empty summary, or + * the circuit breaker being tripped). + */ + private summarizeAndCompact = async (contextWindow: number): Promise => { + if (this.consecutiveCompactionFailures >= MAX_CONSECUTIVE_COMPACTION_FAILURES) { + return false + } + const abortController = this.abortController + if (!abortController) { + return false + } + + const tailBudget = + contextWindow * COMPACTION_TARGET_RATIO - + SUMMARY_OUTPUT_RESERVE_TOKENS - + this.estimateOverheadTokens() + if (tailBudget <= 0) { + return false + } + + const last = this.messages.length - 1 + if (last < 1) { + return false + } + + // Grow the tail from the most recent message downward while it fits the + // budget; always keep at least the last message. + let keepFrom = last + let tailTokens = 0 + for (let i = last; i >= 1; i--) { + const t = this.estimateMessagesTokens([this.messages[i]]) + if (i < last && tailTokens + t > tailBudget) { + break + } + tailTokens += t + keepFrom = i + } + // The tail must start on a user message — move the boundary forward over + // any leading tool/assistant messages, folding them into the prefix. + while (keepFrom < last && this.messages[keepFrom].role !== 'user') { + keepFrom++ + } + + const prefix = this.messages.slice(0, keepFrom) + const tail = this.messages.slice(keepFrom) + if (prefix.length < MIN_PREFIX_MESSAGES_TO_SUMMARIZE || tail.length === 0) { + return false + } + + // The user message at the boundary has a display counterpart with the same + // index; resolve it before any mutation so a corrupt transcript can never + // result from an unexpected miss. + const displayKeepFrom = this.displayMessages.findIndex( + (m) => m.role === 'user' && m.index >= keepFrom + ) + if (displayKeepFrom === -1) { + this.consecutiveCompactionFailures++ + return false + } + + this.compacting = true + try { + const raw = await getNonStreamingCompletion( + [...prefix, { role: 'user', content: getCompactionSummaryPrompt() }], + abortController + ) + const formatted = formatCompactSummary(raw ?? '') + if (!formatted) { + this.consecutiveCompactionFailures++ + return false + } + + this.messages = [{ role: 'user', content: buildSummaryMessageContent(formatted) }, ...tail] + + // Replace the summarized display prefix with the boundary marker and + // re-base the surviving tail's restart indices (the summary occupies + // slot 0, so the tail now starts at slot 1). + this.displayMessages = [ + { role: 'summary', content: formatted }, + ...this.displayMessages + .slice(displayKeepFrom) + .map((m) => (m.role === 'user' ? { ...m, index: m.index - keepFrom + 1 } : m)) + ] + + // The provider report described the pre-compaction history; the new + // history is much smaller, so clear it and let readers re-estimate. + this.contextUsage = undefined + this.consecutiveCompactionFailures = 0 + return true + } catch (err) { + // A user Stop aborts the in-flight summary — that's a turn cancel, not a + // compaction failure, so it doesn't count toward the circuit breaker. + if (!abortController.signal.aborted) { + console.error('Conversation summarization failed', err) + this.consecutiveCompactionFailures++ + } + return false + } finally { + this.compacting = false + } + } + loadApiTools = async () => { try { this.apiTools = await loadApiTools() @@ -1156,7 +1294,6 @@ export class AIChatManager { // not the expanded LLM text, plus the rollback anchor after the user turn. const sentInstructions = this.instructions const sentPastes = pastes - const displayLenAfterUser = this.displayMessages.length // The LLM gets the full pasted content; the display message above keeps // the compact tokens + registry so the bubble can render/expand chips. const oldInstructions = expanded(chatDraft(this.instructions, pastes)) @@ -1226,22 +1363,38 @@ export class AIChatManager { contextWindow !== undefined && projectedContextTokens >= contextWindow * COMPACTION_TRIGGER_RATIO ) { - const freed = this.compactOldestMessages( - projectedContextTokens - contextWindow * COMPACTION_TARGET_RATIO - ) - // A report stays meaningful only debited by what was dropped; the - // estimate path needs no bookkeeping — the next read re-estimates - // the compacted history. chars/4 can underestimate the freed - // tokens, which errs toward compacting again — never toward - // overflowing. - if (this.contextUsage !== undefined) { - this.contextUsage = Math.max(0, this.contextUsage - freed) + // Preferred path: summarize the older prefix, keep the recent tail. + const summarized = await this.summarizeAndCompact(contextWindow) + // A Stop during the in-flight summary aborts this turn's controller; + // summarizeAndCompact then returns false without touching history. Skip + // the drop-oldest fallback (and its save) — it would destructively + // compact a conversation the user only meant to cancel, and the request + // can't run on an aborted controller anyway. The cancel path below rolls + // the pushed turn back cleanly on its own. + if (!this.abortController?.signal.aborted) { + if (!summarized) { + // Fallback when summarization isn't worthwhile or fails: drop the + // oldest messages. A report stays meaningful only debited by what + // was dropped; the estimate path needs no bookkeeping — the next + // read re-estimates the compacted history. chars/4 can + // underestimate the freed tokens, which errs toward compacting + // again — never toward overflowing. + const freed = this.compactOldestMessages( + projectedContextTokens - contextWindow * COMPACTION_TARGET_RATIO + ) + if (this.contextUsage !== undefined) { + this.contextUsage = Math.max(0, this.contextUsage - freed) + } + } + await this.historyManager.saveChat(this.displayMessages, this.messages, this.contextUsage) } - await this.historyManager.saveChat(this.displayMessages, this.messages, this.contextUsage) } - // Rollback anchor for restoreUnsentTurn: captured after compaction so it - // indexes into the (possibly compacted) stored history. + // Rollback anchors for restoreUnsentTurn: captured after compaction so + // they index into the (possibly compacted) stored history. The summary + // path shrinks displayMessages too, so the display anchor must also be + // read here, not before compaction. const modelLenAfterUser = this.messages.length + const displayLenAfterUser = this.displayMessages.length const params: { messages: ChatCompletionMessageParam[] diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts index 01929cefc7..16a08d7344 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.test.ts @@ -14,6 +14,7 @@ const mocks = vi.hoisted(() => ({ sendUserToast: vi.fn(), getOpenaiClient: vi.fn(), getAnthropicClient: vi.fn(), + getNonStreamingCompletion: vi.fn(), runChatLoop: vi.fn() })) @@ -60,7 +61,8 @@ vi.mock('../lib', () => ({ subscribe: () => () => undefined, getOpenaiClient: mocks.getOpenaiClient, getAnthropicClient: mocks.getAnthropicClient - } + }, + getNonStreamingCompletion: mocks.getNonStreamingCompletion })) vi.mock('./api/apiTools', () => ({ @@ -770,6 +772,146 @@ describe('AIChatManager context compaction', () => { await manager.saveAndClear() expect(manager.contextUsage).toBeUndefined() }) + + // gpt-4o resolves to a known 128k window (modelConfig unmocked): trigger at + // ~102k, target ~90k. With a summary reserve of 8k the tail budget is ~76k. + const gpt4oModel = { provider: 'openai', model: 'gpt-4o' } + + // Older prefix (4 messages, ~25k tokens each = 100k chars) plus a recent + // user+assistant pair that fits the tail budget. After the new user turn is + // pushed the budget keeps [recentQ, recentA, new] verbatim and summarizes the + // four old messages. + function seedForSummary(manager: AIChatManager) { + manager.messages = [ + { role: 'user', content: 'OLD1' + 'a'.repeat(100_000) }, + { role: 'assistant', content: 'OLD2' + 'b'.repeat(100_000) }, + { role: 'user', content: 'OLD3' + 'c'.repeat(100_000) }, + { role: 'assistant', content: 'OLD4' + 'd'.repeat(100_000) }, + { role: 'user', content: 'recentQ' + 'e'.repeat(80_000) }, + { role: 'assistant', content: 'recentA' + 'f'.repeat(80_000) } + ] + manager.displayMessages = [ + { role: 'user', content: 'old1', index: 0 }, + { role: 'assistant', content: 'old2' }, + { role: 'user', content: 'old3', index: 2 }, + { role: 'assistant', content: 'old4' }, + { role: 'user', content: 'recentQ', index: 4 }, + { role: 'assistant', content: 'recentA' } + ] + manager.instructions = 'next question' + } + + it('summarizes the older prefix and keeps the recent tail verbatim', async () => { + mocks.getCurrentModel.mockReturnValue(gpt4oModel) + mocks.tryGetCurrentModel.mockReturnValue(gpt4oModel) + mocks.getNonStreamingCompletion.mockResolvedValue( + 'scratchpadSUMMARY TEXT' + ) + const manager = new AIChatManager() + seedForSummary(manager) + + await manager.sendRequest() + + // The prefix (the four OLD messages) was sent to the summarizer, followed + // by the summary-instruction user message. + expect(mocks.getNonStreamingCompletion).toHaveBeenCalledTimes(1) + const summaryReq = mocks.getNonStreamingCompletion.mock.calls[0][0] + expect(summaryReq).toHaveLength(5) + expect(summaryReq[0].content).toContain('OLD1') + expect(summaryReq[3].content).toContain('OLD4') + expect(summaryReq[4].content).toContain('detailed summary') + + // The request that went out begins with the summary user message, then the + // recent tail verbatim, then the new question. + const sent = mocks.runChatLoop.mock.calls[0][0].messages + expect(sent).toHaveLength(4) + expect(sent[0].role).toBe('user') + expect(sent[0].content).toContain('SUMMARY TEXT') + expect(sent[0].content).toContain('continued from a previous conversation') + expect(sent[0].content).not.toContain('scratchpad') + expect(sent[1].content).toContain('recentQ') + + // The display transcript replaces the summarized bubbles with one boundary + // and re-bases the surviving tail's restart indices onto the new history. + expect(manager.displayMessages[0]).toMatchObject({ role: 'summary', content: 'SUMMARY TEXT' }) + const recentQDisplay = manager.displayMessages.find( + (m) => m.role === 'user' && m.content === 'recentQ' + ) + expect(recentQDisplay && 'index' in recentQDisplay ? recentQDisplay.index : undefined).toBe(1) + // No report describes the new history, so the readable number re-estimates + // the now-small compacted context. + expect(manager.contextUsage).toBeUndefined() + }) + + it('falls back to drop-oldest when summarization fails', async () => { + mocks.getCurrentModel.mockReturnValue(gpt4oModel) + mocks.tryGetCurrentModel.mockReturnValue(gpt4oModel) + mocks.getNonStreamingCompletion.mockRejectedValue(new Error('summary boom')) + const manager = new AIChatManager() + seedForSummary(manager) + + await manager.sendRequest() + + // Summarization was attempted, then the request still went out — via + // drop-oldest, so no summary boundary anywhere. + expect(mocks.getNonStreamingCompletion).toHaveBeenCalledTimes(1) + expect(mocks.runChatLoop).toHaveBeenCalledTimes(1) + const sent = mocks.runChatLoop.mock.calls[0][0].messages + expect(sent[0].content).not.toContain('continued from a previous conversation') + expect(manager.displayMessages.some((m) => m.role === 'summary')).toBe(false) + }) + + it('skips summarization (drop-oldest) when the prefix is too small', async () => { + mocks.getCurrentModel.mockReturnValue(anthropicModel) + mocks.tryGetCurrentModel.mockReturnValue(anthropicModel) + const manager = new AIChatManager() + manager.messages = [ + { role: 'user', content: 'a'.repeat(400_000) }, + { role: 'assistant', content: 'b'.repeat(400_000) }, + { role: 'user', content: 'c'.repeat(400) } + ] + manager.contextUsage = 850_000 + manager.instructions = 'next question' + + await manager.sendRequest() + + // A two-message prefix isn't worth a summary round-trip. + expect(mocks.getNonStreamingCompletion).not.toHaveBeenCalled() + expect(manager.displayMessages.some((m) => m.role === 'summary')).toBe(false) + }) + + it('does not drop-oldest compact when the user stops during summarization', async () => { + mocks.getCurrentModel.mockReturnValue(gpt4oModel) + mocks.tryGetCurrentModel.mockReturnValue(gpt4oModel) + // The user hits Stop while the summary request is in flight: it aborts the + // turn's controller and rejects. + mocks.getNonStreamingCompletion.mockImplementation( + async (_msgs: any, ac: AbortController) => { + ac.abort('user_cancelled') + throw new Error('aborted') + } + ) + // With the controller already aborted, the real request returns nothing; + // mirror that so the turn takes the cancel/rollback path. + mocks.runChatLoop.mockImplementation(async () => ({ + addedMessages: [], + tokenUsage: { prompt: 0, completion: 0, total: 0 }, + lastIterationUsage: null, + hitMaxIterations: false + })) + const manager = new AIChatManager() + seedForSummary(manager) + + await manager.sendRequest() + + // Summarization was attempted and aborted, but the abort must NOT trigger a + // destructive drop-oldest fallback: the full prefix survives and the unsent + // turn is rolled back to the pre-send history (the head pair is still there). + expect(mocks.getNonStreamingCompletion).toHaveBeenCalledTimes(1) + expect(manager.messages).toHaveLength(6) + expect(manager.messages[0].content).toContain('OLD1') + expect(manager.displayMessages.some((m) => m.role === 'summary')).toBe(false) + }) }) const assistantToolCall = (id: string): ChatCompletionMessageParam => ({ diff --git a/frontend/src/lib/components/copilot/chat/AIChatMessage.svelte b/frontend/src/lib/components/copilot/chat/AIChatMessage.svelte index 76be72b52f..4e2df28a50 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatMessage.svelte +++ b/frontend/src/lib/components/copilot/chat/AIChatMessage.svelte @@ -9,6 +9,7 @@ import AIChatInput from './AIChatInput.svelte' import type { ContextElement } from './context' import ToolExecutionDisplay from './ToolExecutionDisplay.svelte' + import CompactionBoundary from './CompactionBoundary.svelte' import { messageDraft, segments } from './chatDraft' import { lineCountLabel } from './pasteTokens' @@ -50,103 +51,111 @@ } -
0 && 'mt-4 mb-6', - isLast && '!mb-12', - message.role !== 'user' ? 'cursor-default' : 'cursor-pointer' - )} - role="button" - tabindex="0" - onclick={() => editMessage()} - onkeydown={() => {}} -> - {#if message.role === 'user' && message.contextElements && editingMessageIndex !== messageIndex} -
- {#each message.contextElements as element} - - {/each} -
- {/if} - {#if message.role === 'user' && editingMessageIndex === messageIndex} -
- (editingMessageIndex = null)} - onKeyDown={(e) => { - if (e.key === 'Escape') { - editingMessageIndex = null - } - }} - onEditEnd={() => (editingMessageIndex = null)} - /> -
- {:else} -
- {#if message.role === 'assistant'} -
- {:else if message.role === 'tool'} -
- {:else} -
- {#each segments(messageDraft(message)) as seg}{#if seg.type === 'text'}{seg.value}{:else if expandedPastes.has(seg.att.id)}{seg.att.content}{:else}{/if}{/each} -
- {/if} -
- {/if} - {#if message.role === 'user' && message.snapshot} -
- Saved {message.snapshot.type === 'flow' ? 'a flow' : 'an app'} snapshot - {seg.att.content}{:else}{/if}{/each} +
+ {/if} +
+ {/if} + {#if message.role === 'user' && message.snapshot} +
- Revert + Saved {message.snapshot.type === 'flow' ? 'a flow' : 'an app'} snapshot + +
+ {/if} +
+ {#if message.role === 'user' && message.error} +
+
{/if} - -{#if message.role === 'user' && message.error} -
- -
{/if} diff --git a/frontend/src/lib/components/copilot/chat/CompactionBoundary.svelte b/frontend/src/lib/components/copilot/chat/CompactionBoundary.svelte new file mode 100644 index 0000000000..af9888bc93 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/CompactionBoundary.svelte @@ -0,0 +1,33 @@ + + +
+
+
+ +
+
+ {#if expanded} +
+ {content} +
+ {/if} +
diff --git a/frontend/src/lib/components/copilot/chat/ContextManager.svelte.ts b/frontend/src/lib/components/copilot/chat/ContextManager.svelte.ts index a69d6670b4..4216b6cfa0 100644 --- a/frontend/src/lib/components/copilot/chat/ContextManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/ContextManager.svelte.ts @@ -428,21 +428,25 @@ export default class ContextManager { displayMessages: DisplayMessage[], dbSchemas: DBSchemas ): DisplayMessage[] { - return displayMessages.map((m) => ({ - ...m, - contextElements: - m.role !== 'tool' && m.contextElements - ? m.contextElements.map((c) => - c.type === 'db' - ? { - type: 'db', - title: c.title, - schema: dbSchemas[c.title] - } - : c - ) - : undefined - })) + return displayMessages.map((m) => { + // Only user/assistant messages carry contextElements; tool and summary + // messages pass through untouched. + if ((m.role === 'user' || m.role === 'assistant') && m.contextElements) { + return { + ...m, + contextElements: m.contextElements.map((c) => + c.type === 'db' + ? { + type: 'db' as const, + title: c.title, + schema: dbSchemas[c.title] + } + : c + ) + } + } + return m + }) } setSelectedModuleContext( diff --git a/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte b/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte index 4fe1c342f0..f7681e9852 100644 --- a/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte +++ b/frontend/src/lib/components/copilot/chat/ContextUsageIndicator.svelte @@ -15,13 +15,9 @@ // one describes the current history (one turn stale by nature), otherwise // a live chars/4 estimate of the stored context. let usedTokens = $derived(Math.round(aiChatManager.contextTokens)) - // With a known window, only surface once the conversation actually fills it; - // without one there is no threshold to compare against, so always show. - let visible = $derived( - usedTokens > 0 && - aiChatManager.messages.length > 0 && - (contextWindow === undefined || usedTokens >= contextWindow * 0.5) - ) + // Always surface usage once a conversation has started, at any fill level, so + // the user can watch context grow toward the compaction threshold. + let visible = $derived(usedTokens > 0 && aiChatManager.messages.length > 0) function formatTokenCount(tokens: number): string { if (tokens >= 1_000_000) { @@ -35,11 +31,9 @@ {#if visible} -
- - context window usage: ~{formatTokenCount(usedTokens)}{contextWindow - ? ` / ${formatTokenCount(contextWindow)}` - : ''} - -
+ + Context usage: ~{formatTokenCount(usedTokens)}{contextWindow + ? ` / ${formatTokenCount(contextWindow)}` + : ''} + {/if} diff --git a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts index c11c0c3907..9173e4c441 100644 --- a/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/HistoryManager.svelte.ts @@ -179,9 +179,19 @@ export default class HistoryManager { contextUsage?: number ) { if (displayMessages.length > 0) { - // Expand any collapsed-paste tokens so the title is readable text, not - // the chip label + its zero-width id chars. - const title = expanded(messageDraft(displayMessages[0])).slice(0, 50) + // Compaction replaces the original first message with a summary boundary. + // Re-deriving the title would then shift it to the first surviving tail + // message, so once that boundary leads the transcript, keep the title + // computed before compaction. Otherwise derive it from the first message, + // expanding collapsed-paste tokens so it reads as text rather than the + // chip label + its zero-width id chars. + const existingTitle = this.savedChats[this.currentChatId]?.title + const title = + displayMessages[0].role === 'summary' && existingTitle !== undefined + ? existingTitle + : expanded( + messageDraft(displayMessages.find((m) => m.role !== 'summary') ?? displayMessages[0]) + ).slice(0, 50) // we don't want to save the snapshot in the history const updatedChat = { actualMessages: $state.snapshot(messages), diff --git a/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts b/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts index 96e304fa98..38fb7ecf0d 100644 --- a/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts +++ b/frontend/src/lib/components/copilot/chat/HistoryManager.test.ts @@ -114,3 +114,30 @@ describe('HistoryManager legacy chat-history migration', () => { expect(await countChats('copilot-chat-history::admin@test')).toBe(1) }) }) + +describe('HistoryManager title across compaction', () => { + it('keeps the original title once a summary boundary leads the transcript', async () => { + const hm = new HistoryManager() + await hm.init() + const id = hm.getCurrentChatId() + + // First save derives the title from the first user message. + await hm.save( + [{ role: 'user', content: 'original first question', index: 0 }] as DisplayMessage[], + [] as ChatCompletionMessageParam[] + ) + expect(hm.getAllSavedChats().find((c) => c.id === id)?.title).toBe('original first question') + + // After compaction the transcript leads with a summary boundary; deriving + // the title now would shift it to the surviving tail message. It must stay + // the title computed before compaction. + await hm.save( + [ + { role: 'summary', content: 'summary of the earlier conversation' }, + { role: 'user', content: 'a much later follow-up', index: 1 } + ] as DisplayMessage[], + [] as ChatCompletionMessageParam[] + ) + expect(hm.getAllSavedChats().find((c) => c.id === id)?.title).toBe('original first question') + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/compactionPrompt.test.ts b/frontend/src/lib/components/copilot/chat/compactionPrompt.test.ts new file mode 100644 index 0000000000..2161182594 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/compactionPrompt.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from 'vitest' +import { + buildSummaryMessageContent, + formatCompactSummary, + getCompactionSummaryPrompt +} from './compactionPrompt' + +describe('formatCompactSummary', () => { + it('strips the analysis scratchpad and unwraps the summary block', () => { + const raw = ` +chronological thinking the model should not keep + + +1. Primary Request and Intent: build the thing +2. Pending Tasks: none +` + const formatted = formatCompactSummary(raw) + expect(formatted).not.toContain('chronological thinking') + expect(formatted).not.toContain('') + expect(formatted).not.toContain('') + expect(formatted).toContain('Primary Request and Intent: build the thing') + }) + + it('falls back to the trimmed raw text when the model omits the tags', () => { + expect(formatCompactSummary(' just a plain summary ')).toBe('just a plain summary') + }) + + it('keeps the summary even when there is no analysis block', () => { + expect(formatCompactSummary('only the summary')).toBe('only the summary') + }) + + it('collapses the blank-line runs left by stripping analysis', () => { + const raw = 'x\n\n\n\na\n\n\n\nb' + expect(formatCompactSummary(raw)).toBe('a\n\nb') + }) + + it('strips every analysis block, not just the first, when the summary is untagged', () => { + const raw = 'first\nkept one\nsecond\nkept two' + const formatted = formatCompactSummary(raw) + expect(formatted).not.toContain('first') + expect(formatted).not.toContain('second') + expect(formatted).not.toContain('') + expect(formatted).toContain('kept one') + expect(formatted).toContain('kept two') + }) +}) + +describe('buildSummaryMessageContent', () => { + it('embeds the summary and frames it as a continuation', () => { + const content = buildSummaryMessageContent('THE SUMMARY') + expect(content).toContain('THE SUMMARY') + expect(content).toContain('continued from a previous conversation') + expect(content).toContain('preserved verbatim') + }) +}) + +describe('getCompactionSummaryPrompt', () => { + it('asks for a structured, text-only summary', () => { + const prompt = getCompactionSummaryPrompt() + expect(prompt).toContain('detailed summary') + expect(prompt).toContain('') + expect(prompt).toContain('TEXT ONLY') + }) +}) diff --git a/frontend/src/lib/components/copilot/chat/compactionPrompt.ts b/frontend/src/lib/components/copilot/chat/compactionPrompt.ts new file mode 100644 index 0000000000..010a8cf8b8 --- /dev/null +++ b/frontend/src/lib/components/copilot/chat/compactionPrompt.ts @@ -0,0 +1,124 @@ +// Summary-based compaction: when a conversation approaches the model's context +// window, the older prefix is replaced by an LLM-generated structured summary +// while the recent tail is kept verbatim. The summary precedes the kept tail, +// so it is written "up to" the point of compaction — newer messages the model +// does not see here will follow it. +// +// Prompt structure and the analysis/summary split are adapted from the +// reference compaction prompt used by coding agents. + +// Reinforce text-only output. The summary call is issued without tools, but a +// strong instruction keeps weaker models from narrating a tool call instead of +// producing the summary. +const NO_TOOLS_PREAMBLE = `CRITICAL: Respond with TEXT ONLY. Do NOT call any tools. + +- You already have all the context you need in the conversation above. +- Your entire response must be plain text: an block followed by a block. + +` + +const NO_TOOLS_TRAILER = + '\n\nREMINDER: Respond with plain text only — an block followed by a block.' + +// The block is a drafting scratchpad that formatCompactSummary() +// strips before the summary reaches context. +const SUMMARY_PROMPT = `Your task is to create a detailed summary of the conversation so far. This summary will be placed at the start of a continuing session; newer messages that build on this context will follow after it (you do not see them here). Summarize thoroughly so that someone reading only your summary and then the newer messages can fully understand what happened and continue the work without losing context. + +This is a conversation with Windmill's global workspace assistant. It inspects workspace items and authors them as per-user drafts — scripts, flows, apps, resources, variables, triggers, and schedules — then deploys those drafts and test-runs scripts and flows. It works with items by their workspace path (e.g. \`u/alice/sync_orders\`, \`f/team/my_flow\`); it does NOT edit files on a filesystem. Frame the summary in those terms. + +Before providing your final summary, wrap your analysis in tags to organize your thoughts. In your analysis: + +1. Chronologically analyze each message and section of the conversation. For each section thoroughly identify: + - The user's explicit requests and intents + - Your approach to addressing the user's requests + - Key decisions, technical concepts and code patterns + - Specific details: workspace item paths and kinds (script / flow / app / resource / variable / trigger / schedule), code snippets for runnables, and the exact actions taken (drafts created or updated, items deployed, test runs and their results) + - Errors you ran into and how you fixed them + - Specific user feedback, especially if the user told you to do something differently +2. Double-check for technical accuracy and completeness. + +Your summary should include the following sections: + +1. Primary Request and Intent: Capture all of the user's explicit requests and intents in detail. +2. Key Technical Concepts: List important technical concepts, technologies, and frameworks discussed. +3. Workspace Items and Code: Enumerate the workspace items inspected, drafted, updated, deployed, or test-run — each by its path and kind (script / flow / app / resource / variable / trigger / schedule) — noting what was done and why. Include full code snippets for runnables (script code, flow inline scripts, app inline runnables) wherever code was written or changed. +4. Errors and fixes: List errors encountered and how they were fixed, including any user feedback. +5. Problem Solving: Document problems solved and any ongoing troubleshooting efforts. +6. All user messages: List ALL user messages that are not tool results. These are critical for understanding the user's feedback and changing intent. +7. Pending Tasks: Outline any pending tasks you have explicitly been asked to work on. +8. Current Work: Describe precisely what was being worked on immediately before this summary, paying special attention to the most recent messages. Include item paths and code snippets where applicable. +9. Context for Continuing Work: Summarize any context, decisions, or state needed to understand and continue the work in subsequent messages — including which items are still drafts versus deployed and the exact paths involved. If there is a clear next step directly in line with the user's most recent explicit request, state it and include a direct quote from the most recent conversation showing where you left off. + +Structure your output like this: + + +[Your thought process, ensuring all points are covered thoroughly and accurately] + + + +1. Primary Request and Intent: + [Detailed description] + +2. Key Technical Concepts: + - [Concept] + +3. Workspace Items and Code: + - [path + kind, e.g. u/alice/sync_orders (script)] + - [What was done: read / draft created / draft updated / deployed / test-run result] + - [Why it matters] + - [Code snippet, for runnables] + +4. Errors and fixes: + - [Error]: [How you fixed it] + +5. Problem Solving: + [Description] + +6. All user messages: + - [Non-tool-result user message] + +7. Pending Tasks: + - [Task] + +8. Current Work: + [Precise description of current work] + +9. Context for Continuing Work: + [Key context, decisions, or state needed to continue] + + +Please provide your summary following this structure, ensuring precision and thoroughness.` + +/** Prompt sent as the final user message of the summarization request. */ +export function getCompactionSummaryPrompt(): string { + return NO_TOOLS_PREAMBLE + SUMMARY_PROMPT + NO_TOOLS_TRAILER +} + +/** + * Strips the drafting scratchpad and unwraps the block. + * Falls back to the trimmed raw text when the model didn't use the tags, so a + * well-formed-but-untagged summary is still usable. + */ +export function formatCompactSummary(raw: string): string { + let formatted = raw.replace(/[\s\S]*?<\/analysis>/gi, '') + + const summaryMatch = formatted.match(/([\s\S]*?)<\/summary>/i) + if (summaryMatch) { + formatted = (summaryMatch[1] ?? '').trim() + } + + // Collapse the blank-line runs left behind by stripping the analysis block. + return formatted.replace(/\n{3,}/g, '\n\n').trim() +} + +/** + * Wraps a formatted summary as the content of the user message that replaces + * the summarized prefix in the conversation. + */ +export function buildSummaryMessageContent(formattedSummary: string): string { + return `This session is being continued from a previous conversation that ran out of context. The summary below covers the earlier portion of the conversation. Recent messages after the summary are preserved verbatim. + +${formattedSummary} + +Continue the conversation from where it left off. Do not re-introduce the summary or recap it to the user; pick up the work as if the break never happened.` +} diff --git a/frontend/src/lib/components/copilot/chat/shared.ts b/frontend/src/lib/components/copilot/chat/shared.ts index f32f266657..aecea1ddfc 100644 --- a/frontend/src/lib/components/copilot/chat/shared.ts +++ b/frontend/src/lib/components/copilot/chat/shared.ts @@ -525,7 +525,22 @@ export type AssistantDisplayMessage = BaseDisplayMessage & { streaming?: boolean } -export type DisplayMessage = UserDisplayMessage | ToolDisplayMessage | AssistantDisplayMessage +/** + * Compaction boundary: replaces the summarized prefix in BOTH displayMessages + * and the API messages (where it is a plain user message). It carries no index + * because it is never a restart target — only the surviving tail's user + * messages are rewound to. + */ +export type SummaryDisplayMessage = { + role: 'summary' + content: string +} + +export type DisplayMessage = + | UserDisplayMessage + | ToolDisplayMessage + | AssistantDisplayMessage + | SummaryDisplayMessage // A tool message whose askUserQuestion is still awaiting an answer: the AI loop // is paused on the user. Drives the question card's interactivity, the From 796230d90a7e6d1debc15e139ab708881e527862 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Thu, 18 Jun 2026 16:59:11 +0200 Subject: [PATCH 05/41] fix(workspaces): add instance setting to disable workspace invite/add emails (#9643) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(workspaces): add skip_email option to invite_user and add_user endpoints The workspace invite_user and add_user API endpoints unconditionally sent notification emails when SMTP was configured, with no way to suppress them per-request. This is noise for automated workflows that programmatically add users to workspaces. Add an optional `skip_email: Option` field to `NewWorkspaceInvite` and `NewWorkspaceUser`, following the existing pattern on `NewUser` used by POST /api/users/create, and guard the `send_email_if_possible` calls with `if !nu.skip_email.unwrap_or(false)`. The field is optional, so existing clients are unaffected. The auto-add code paths in workspaces_ee.rs (domain-based and instance-group auto-add) are auto-triggered and take no API parameter, so they are left as-is. Fixes WIN-2068 Co-Authored-By: Claude Opus 4.8 (1M context) * feat(workspaces): make workspace invite/add emails toggleable via instance setting Replace the per-request skip_email approach with an instance-level setting `disable_workspace_invite_emails`. When enabled, the email notifications sent by the workspace invite_user and add_user endpoints are suppressed. Useful for instances where users are added programmatically (e.g. CI pipelines that fork workspaces and add users) and the invite emails are noise. Backend: - Add `DISABLE_WORKSPACE_INVITE_EMAILS_SETTING` global setting constant. - Guard the `send_email_if_possible` calls in invite_user and add_user with a read of that setting (via the existing `load_value_from_global_settings` helper). Defaults to false, so existing behavior is unchanged. - Revert the per-request `skip_email` field on NewWorkspaceInvite / NewWorkspaceUser and the corresponding openapi additions. Frontend: - Expose the setting as a boolean toggle in the SMTP tab of the instance settings (superadmin). The auto-add paths in workspaces_ee.rs are unaffected. Fixes WIN-2068 Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): gate disable_workspace_invite_emails toggle behind EE Email delivery (send_email_if_possible) is a no-op outside the EE/private build, so the toggle has no effect on a pure-OSS instance. Add `ee_only: ''` to match the sibling SMTP settings: the toggle is grayed out (with an EE badge) on non-EE instances instead of rendering as an active no-op control. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(frontend): don't EE-gate disable_workspace_invite_emails toggle The earlier ee_only addition was based on the false premise that the workspace invite/add emails are license-gated. They are not: SMTP configuration (SmtpSettings) and email sending (send_email_if_possible) have no enterpriseLicense check — they only require the closed-source build with SMTP configured. The sibling smtp_settings carries ee_only: '' but its smtp_connect field renders no SettingCard label, so that flag is inert (no badge, no disable). On a plain boolean field ee_only is fully active, which incorrectly grayed out the toggle and showed an EE badge. Drop ee_only so the control matches the actual non-license-gated behavior. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- .../windmill-api-workspaces/src/workspaces.rs | 56 +++++++++++++------ .../windmill-common/src/global_settings.rs | 1 + .../src/lib/components/instanceSettings.ts | 8 +++ 3 files changed, 47 insertions(+), 18 deletions(-) diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 788c090aa2..6387bb27c8 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -48,7 +48,9 @@ use windmill_common::workspaces::{Ducklake, DucklakeCatalogResourceType}; use windmill_common::PgDatabase; use windmill_common::{ error::{Error, JsonResult, Result}, - global_settings::AUTOMATE_USERNAME_CREATION_SETTING, + global_settings::{ + AUTOMATE_USERNAME_CREATION_SETTING, DISABLE_WORKSPACE_INVITE_EMAILS_SETTING, + }, oauth2::WORKSPACE_SLACK_BOT_TOKEN_PATH, utils::{paginate, rd_string, require_admin, Pagination}, }; @@ -5354,6 +5356,20 @@ async fn unarchive_workspace( Ok(format!("Unarchived workspace {}", &w_id)) } +/// Whether the instance is configured to suppress the email notifications sent +/// when a user is invited or added to a workspace. Defaults to false (emails on). +async fn workspace_invite_emails_disabled(db: &DB) -> Result { + Ok( + windmill_common::global_settings::load_value_from_global_settings( + db, + DISABLE_WORKSPACE_INVITE_EMAILS_SETTING, + ) + .await? + .and_then(|v| v.as_bool()) + .unwrap_or(false), + ) +} + async fn invite_user( ApiAuthed { username, is_admin, .. }: ApiAuthed, Extension(db): Extension, @@ -5412,16 +5428,18 @@ async fn invite_user( tx.commit().await?; - send_email_if_possible( - &format!("Invited to Windmill's workspace: {w_id}"), - &format!( - "You have been granted access to Windmill's workspace {w_id} + if !workspace_invite_emails_disabled(&db).await? { + send_email_if_possible( + &format!("Invited to Windmill's workspace: {w_id}"), + &format!( + "You have been granted access to Windmill's workspace {w_id} If you do not have an account on {}, login with SSO or ask an admin to create an account for you.", - (**BASE_URL.load()).clone() - ), - &nu.email, - ); + (**BASE_URL.load()).clone() + ), + &nu.email, + ); + } webhook.send_instance_event(InstanceEvent::UserInvitedWorkspace { email: nu.email.clone(), @@ -5564,17 +5582,19 @@ async fn add_user( ) .await?; - send_email_if_possible( - &format!("Added to Windmill's workspace: {w_id}"), - &format!( - "You have been granted access to Windmill's workspace {w_id} by {} + if !workspace_invite_emails_disabled(&db).await? { + send_email_if_possible( + &format!("Added to Windmill's workspace: {w_id}"), + &format!( + "You have been granted access to Windmill's workspace {w_id} by {} If you do not have an account on {}, login with SSO or ask an admin to create an account for you.", - authed.email, - (**BASE_URL.load()).clone() - ), - &nu.email, - ); + authed.email, + (**BASE_URL.load()).clone() + ), + &nu.email, + ); + } webhook.send_instance_event(InstanceEvent::UserAddedWorkspace { workspace: w_id.clone(), diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index bed78113b5..4d1d48a53d 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -76,6 +76,7 @@ pub const OBJECT_STORE_CONFIG_SETTING: &str = "object_store_cache_config"; pub const HUB_API_SECRET_SETTING: &str = "hub_api_secret"; pub const AUTOMATE_USERNAME_CREATION_SETTING: &str = "automate_username_creation"; +pub const DISABLE_WORKSPACE_INVITE_EMAILS_SETTING: &str = "disable_workspace_invite_emails"; pub const DISABLE_PASSWORD_LOGIN_SETTING: &str = "disable_password_login"; pub const AUTO_LOGIN_PROVIDER_SETTING: &str = "auto_login_provider"; pub const HUB_BASE_URL_SETTING: &str = "hub_base_url"; diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 6af6926c4f..72ccbeca2b 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -523,6 +523,14 @@ export const settings: Record = { fieldType: 'smtp_connect', storage: 'setting', ee_only: '' + }, + { + label: 'Disable workspace invite emails', + description: + 'Do not send email notifications when a user is invited or added to a workspace. Useful for automated workflows that add users programmatically.', + key: 'disable_workspace_invite_emails', + fieldType: 'boolean', + storage: 'setting' } ], 'Auth/OAuth/SAML': [ From d7e139b191462165672d0e05215e7edf611566c0 Mon Sep 17 00:00:00 2001 From: hugocasa Date: Thu, 18 Jun 2026 17:51:35 +0200 Subject: [PATCH 06/41] oauth: add netsuite provider + icon (#9538) NetSuite is a per-instance OAuth provider (account-specific authorize/token URLs), registered via connect_config_template. Its authorize endpoint requires scope=rest_webservices, so the template mechanism gains an optional scopes field copied into the built connect_config. Co-authored-by: Claude Fable 5 --- backend/oauth_connect.json | 17 +++++++++++++++++ backend/windmill-oauth/src/lib.rs | 4 ++++ .../lib/components/InstanceSettings.svelte | 2 +- .../lib/components/icons/NetsuiteIcon.svelte | 19 +++++++++++++++++++ frontend/src/lib/components/icons/index.ts | 2 ++ 5 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 frontend/src/lib/components/icons/NetsuiteIcon.svelte diff --git a/backend/oauth_connect.json b/backend/oauth_connect.json index 421136fc77..9e74822d98 100644 --- a/backend/oauth_connect.json +++ b/backend/oauth_connect.json @@ -252,6 +252,23 @@ } } }, + "netsuite": { + "connect_config_template": { + "display_name": "NetSuite", + "label": "NetSuite Account ID", + "placeholder": " as in your NetSuite domain, e.g. 1234567 or 1234567-sb1", + "help_url": "https://docs.oracle.com/en/cloud/saas/netsuite/ns-online-help/chapter_157769826287.html", + "auth_url": "https://{instance}.app.netsuite.com/app/login/oauth2/authorize.nl", + "token_url": "https://{instance}.suitetalk.api.netsuite.com/services/rest/auth/oauth2/v1/token", + "req_body_auth": false, + "strip_suffix": ".app.netsuite.com", + "scopes": ["rest_webservices"], + "extra_params_key": "account_id", + "resource_mapping": { + "account_id": "{instance}" + } + } + }, "outreach": { "auth_url": "https://api.outreach.io/oauth/authorize", "token_url": "https://api.outreach.io/oauth/token", diff --git a/backend/windmill-oauth/src/lib.rs b/backend/windmill-oauth/src/lib.rs index bf1ea659aa..2f28a6d4c7 100644 --- a/backend/windmill-oauth/src/lib.rs +++ b/backend/windmill-oauth/src/lib.rs @@ -151,6 +151,10 @@ pub struct ConnectConfigTemplate { pub token_url: String, #[serde(skip_serializing_if = "Option::is_none")] pub req_body_auth: Option, + /// Scopes copied into the built `connect_config` (e.g. NetSuite's + /// `rest_webservices`). Templated providers default to no scopes. + #[serde(skip_serializing_if = "Option::is_none")] + pub scopes: Option>, /// Key under `connect_config.extra_params` where the instance name is /// stored (defaults to `instance`). Snowflake uses `account_identifier` for /// backward compatibility with previously-saved configs. diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index cfc9015534..1e1416295a 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -312,7 +312,7 @@ instanceInputs[name] = v if (oauths[name].connect_config?.extra_params?.[key] === v) continue oauths[name].connect_config = { - scopes: [], + scopes: tmpl.scopes ?? [], // CC-only templated providers have no auth_url; store an empty string // (not omitted) so the instance-config parser still types the entry. // The backend treats an empty auth_url as the unused placeholder for diff --git a/frontend/src/lib/components/icons/NetsuiteIcon.svelte b/frontend/src/lib/components/icons/NetsuiteIcon.svelte new file mode 100644 index 0000000000..768bbbc99e --- /dev/null +++ b/frontend/src/lib/components/icons/NetsuiteIcon.svelte @@ -0,0 +1,19 @@ + + + + + + diff --git a/frontend/src/lib/components/icons/index.ts b/frontend/src/lib/components/icons/index.ts index 3f8556a143..9c4a70c041 100644 --- a/frontend/src/lib/components/icons/index.ts +++ b/frontend/src/lib/components/icons/index.ts @@ -35,6 +35,7 @@ import TelegramIcon from './TelegramIcon.svelte' import FunkwhaleIcon from './FunkwhaleIcon.svelte' import GdocsIcon from './GdocsIcon.svelte' import NextcloudIcon from './NextcloudIcon.svelte' +import NetsuiteIcon from './NetsuiteIcon.svelte' import FaunadbIcon from './FaunadbIcon.svelte' import ClickhouseIcon from './ClickhouseIcon.svelte' import OpenaiIcon from './OpenaiIcon.svelte' @@ -250,6 +251,7 @@ export const APP_TO_ICON_COMPONENT = { hubspot: HubspotIcon, datadog: DatadogIcon, databricks: DatabricksIcon, + netsuite: NetsuiteIcon, coupa: CoupaIcon, adobe_acrobat_sign: AdobeAcrobatSignIcon, stripe: StripeIcon, From 7155a0bb96cf30bd878272a0f4c3c3b02341b261 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 18 Jun 2026 18:09:02 +0200 Subject: [PATCH 07/41] feat: Data Pipelines alpha (#9193) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add workspace asset graph view Workspace-wide canvas of assets and their producer/consumer scripts, reachable from the assets page. Left-to-right layered layout via d3-dag sugiyama, rendered with @xyflow/svelte (same stack as the flow editor). GET /w/:ws/assets/graph returns deduped nodes + edges. Follow-ups: filters (kind/folder/search), node detail drawer, inline script edit from a clicked node. Co-Authored-By: Claude Opus 4.7 (1M context) * all * all * all * update * all * all * all * feat(pipeline): output-kind picker and per-(lang, output) templates Add a third stage to PipelineInsertMenu that asks what kind of asset the new script will produce (datatable / ducklake / s3 parquet / s3 object / none). The picked kind drives a real wmill SDK skeleton — typed datatable inserts, ducklake CREATE+INSERT, s3 parquet COPY, etc. — with the upstream asset auto-wired as the input source when added from an asset node. Reorder languages to bun → duckdb → python → sql so data-shaped languages surface first. Co-Authored-By: Claude Opus 4.7 (1M context) * all * chore(main): release 1.693.4 (#8994) * chore(main): release 1.693.4 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> * feat: ansible delegate_to_git_repo install_requirements, dynamic fields, --limit (#8997) * feat: ansible delegate_to_git_repo install_requirements, dynamic fields, --limit Co-Authored-By: Claude Opus 4.5 * fix: include .yaml variants in collections/roles requirements lookup Co-Authored-By: Claude Opus 4.5 --------- Co-authored-by: Claude Opus 4.5 * fix(cli): only preserve case for raw-app runnableIds, not app/flow summaries (#9000) * fix(cli): only preserve case for raw-app runnableIds, not app/flow summaries PR #8940 stopped lowercasing in sanitizeForFilesystem to fix #8939, where a raw-app runnableId like CamelCaseTSRunnable produced a CamelCase YAML metadata file but a lowercased code file, making them desync and register as duplicate runnables on push. That fix overshot. sanitizeForFilesystem is also reached by newPathAssigner, which serves normal apps and flows where the input is the script's human summary ("Get Users Data") rather than an identifier. There the on-disk filename is the only artifact — there's no companion YAML to keep in sync — so lowercasing was the right behavior. Removing it changed both the on-disk filename and the !inline reference in app.yaml / flow.yaml from get_users_data.inline_script.ts to Get_Users_Data.inline_script.ts on the next pull, surfacing as unwanted case churn for users updating to 1.693.x. Add a preserveCase option to sanitizeForFilesystem (default false → lowercase). newRawAppPathAssigner opts in; newPathAssigner stays on the default. Update unit tests accordingly and add an end-to-end raw-app round-trip in raw_app_sync.test.ts that pushes a CamelCase backend runnable, pulls it back, and asserts both YAML and code file preserve case with no lowercase orphan. Co-Authored-By: Claude Opus 4.7 (1M context) * test(cli): use readdir for exact-case orphan check on Windows The CamelCase round-trip test used fileExists("camelcasetsrunnable.ts") to assert no lowercase orphan was produced, which false-positives on Windows since the filesystem is case-insensitive and resolves the lookup to the existing CamelCaseTSRunnable.ts. Switch to readdir + toContain so the exact on-disk casing is compared identically on Linux and Windows. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * feat(cli): wmill-lock.yaml auto-fill + --rehash-only + path-prefix dedup (#8978) * fix(cli): canonical lockfile hashes + lock upgrade migration to v3 Co-Authored-By: Claude Opus 4.5 * fix(cli): use __app_hash subpath in rehash missing-entry check Co-Authored-By: Claude Opus 4.5 * fix(cli): run sync pull lockfile auto-fill regardless of changes Co-Authored-By: Claude Opus 4.5 * chore: regenerate system prompts for new lock and rehash-only commands Co-Authored-By: Claude Opus 4.5 * fix(cli): address review feedback on lock upgrade Co-Authored-By: Claude Opus 4.5 * fix(cli): drop v3 marker; always run fallback; fail-fast on unknown lockfile version Co-Authored-By: Claude Opus 4.5 * fix(cli): drop yaml-round-trip legacy hash variant; recover via --rehash-only Co-Authored-By: Claude Opus 4.5 * fix(cli): include legacy hash in script push staleness warning check Co-Authored-By: Claude Opus 4.5 * revert(cli): drop canonical hash formula; keep raw-bytes hashing Co-Authored-By: Claude Opus 4.5 * perf(cli): reuse change-tracker map for sync pull lockfile auto-fill Co-Authored-By: Claude Opus 4.5 * fix(cli): address review feedback on rehash-only Co-Authored-By: Claude Opus 4.5 * test(cli): pin lockfile hash + yaml format and cover regression cases Co-Authored-By: Claude Opus 4.5 * test(cli): byte-stable snapshot tests for flow.yaml format Co-Authored-By: Claude Opus 4.5 * test(cli): add app and script-metadata yaml snapshot fixtures Co-Authored-By: Claude Opus 4.5 * fix(cli): address claude review on rehash-only Co-Authored-By: Claude Opus 4.5 * refactor(cli): factorize script-path to remote-path derivation Co-Authored-By: Claude Opus 4.5 * fix(cli): address claude + cubic review (dry-run mutation, rehash short-circuit) Co-Authored-By: Claude Opus 4.5 * refactor(cli): make rehash a subcommand and factorize fs walks Co-Authored-By: Claude Opus 4.5 * fix(cli): normalize line endings in yaml snapshot tests for windows ci Co-Authored-By: Claude Opus 4.5 * fix(cli): address review feedback on rehash + auto-fill - Flat-layout scripts now clearGlobalLock before rehash write so legacy ./-prefixed duplicates get cleaned up (matches flow/app behavior). - Add MalformedLockfileError; sync pull auto-fill re-throws it alongside UnknownLockVersionError instead of silently warning + continuing. - Document the legacy step-removal false-negative in isFlowDirectlyStale / isAppDirectlyStale and the categorizeLocalFiles ignore-filter invariant. Co-Authored-By: Claude Opus 4.5 --------- Co-authored-by: Claude Opus 4.5 * fix: use otel.status_message for OTLP Status.message on failed jobs (#8995) tracing-opentelemetry only recognizes otel.status_code and otel.status_message as fields that map to the OTLP Status proto. The previously-used otel.status_description fell through to the generic attribute recorder, leaving Status.message unset and preventing OTLP consumers from filtering spans on error status. Co-authored-by: Claude Opus 4.7 (1M context) * fix: route email trigger path through standard info channel (#8996) * docs(skill): document email triggers and S3 attachments Add an "Email triggers" section to the triggers skill covering the local-part config, the parsed_email/raw_email/email_extra_args payload, the URL-style extras convention, where to find trigger_path (only with a preprocessor, at event.trigger_path), and — most importantly — that binary attachments are uploaded to the workspace S3 bucket and surface as `{ s3: "windmill_emails//attachments/" }`. Scripts must use wmill.loadS3File / wmill.load_s3_file to read them. Also pulls EmailTrigger into the schema mappings so a real `email_trigger.schema.yaml` is generated, and adds Email/Azure to the trigger kinds list in the CLI agent guidance. Co-Authored-By: Claude Opus 4.7 (1M context) * chore: update ee-repo-ref for email trigger path fix Co-Authored-By: Claude Opus 4.7 (1M context) * chore: update ee-repo-ref to 26184ab7a4aadfc529dcedf038aa08d36c7ad381 This commit updates the EE repository reference after PR #553 was merged in windmill-ee-private. Previous ee-repo-ref: 318a46897a605dc9be3817901f35ba5a99a0a525 New ee-repo-ref: 26184ab7a4aadfc529dcedf038aa08d36c7ad381 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 4.7 (1M context) Co-authored-by: windmill-internal-app[bot] * update git sync version to 1.693.5 * fix: pair PG arg type with actual Rust binding to keep query_typed_raw safe (#8999) * fix: pair PG arg type with actual Rust binding to keep query_typed_raw safe Co-Authored-By: Claude Opus 4.5 * fix(pg): wrap encoder errors with arg context, add fallback test Followups on #8999 review: - Wrap rust-postgres "error serializing parameter N" failures with the arg name, JSON value kind, and asserted Postgres type plus a hint about an explicit cast — so users see actionable context instead of an opaque WrongType. - Drift-prevention meta-test: assert otyp_to_pg_type and convert_val agree on the Type for every recognised arg_t when the JSON value matches its natural Rust kind. Catches future drift if either side changes. - Integration test for the prepare + query_raw fallback path: confirms unrecognised arg_t (custom enum) is routed through prepare and the server-resolved type appears in the failure surface — flips into a test failure if a regression accidentally routes unrecognised types through query_typed_raw. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(pg): add otyp_inferred flag + regex-based placeholder renumbering Two follow-ups from the review of #8999: 1. **Issue #1 (Number/Bool + explicit text decl in WHERE)** Add `Arg::otyp_inferred: bool` to the parser. The PG SQL parser sets it `true` only at the "no info → fall back to text" site (bare `$N`, no inline cast, no `-- $N (TYPE)` decl). All other arg sources keep it `false`. In `convert_val` this flag distinguishes: - explicit text-like target (`-- $1 (text)` or `$1::text`) — coerce `Bool`/`Number` → `Box` so `WHERE text_col = $1` works (`text = text` operator). Pre-#8988 behaviour, restored. - parser-default text (bare `$N`) — bind the value's natural Rust type so the regression case (`Value::Bool` against a real `bool` column via `CAST AS bool`) keeps working. `Arg` is in `windmill-parser`; the new field has `#[serde(default)]` so persisted signatures stay backward-compatible. 2. **Issue #4 ($5/$50 substring rewrite collision)** Replace the per-index `String::replace` chain (which turned `$50` into `$10` when oidx=5 was processed first) with a single regex pass. `\d+` is greedy, so `$5` and `$50` match as distinct units; indices outside the mapping are left intact. 3. Tests: - parser: `test_parse_pgsql_otyp_inferred_flag` covers bare/inline- cast/decl/mixed shapes. - executor unit: `convert_val_bool_against_every_arg_t` and `convert_val_*_number_*` split each text-like target into explicit vs inferred expectations. - executor unit: `renumber_sparse_placeholders_no_collision`. - integration: `test_postgresql_arg_type_combinations` adds 4 cases covering decl(text)+Number/Bool in WHERE, bare $1+Bool, and sparse positional args ($5/$50). Co-Authored-By: Claude Opus 4.7 (1M context) * fix(pg+sdk): enum support, extended String arms, position-aware $N rewrite, SDK quality Backend: 1. **`AnyTextValue` ToSql/FromSql wrapper**: vanilla `tokio_postgres`'s `ToSql for String` / `FromSql for String` reject `Kind::Enum` and `Kind::Domain` even though the wire format is plain UTF-8. The wrapper accepts those kinds in both directions. End result: explicit `$1::my_enum` / `CAST($1 AS my_enum)` casts now round-trip without the ugly `CAST($1::text AS my_enum)` workaround, AND `SELECT enum_col` results come back as JSON strings instead of erroring at the FromSql layer. 2. **#10 — Value::String → numeric/real/double/oid/bool**. Without these arms, a string-encoded value (`"3.14"`, `"true"`) for a non-text / non-temporal arg_t fell through to `Box + TEXT`, which then failed at the server (no implicit cast text→numeric in expression context). Now strings are parsed into the matching native type with clear error messages on parse failure. 3. **Position-aware `$N` rewrite**: replaces the regex-based renumbering (which fixed the `$5/$50` substring collision but still walked through string literals and comments, mangling `'price: $5'` etc.) with a walk over `parse_pg_statement_arg_positions` — the same string/comment/dollar-quote-aware tokenizer used for index discovery. Adds `parse_pg_statement_arg_positions` to the parser's public API. SDK: 4. **BigInt support**: `JSON.stringify(BigInt)` throws. The SDK now stringifies bigints before serialisation; the executor accepts numeric strings into BIGINT arg slots via the existing `Value::String → INT8` parsing arm. SDK-side `inferSqlType` is split so `BigInt` always resolves to `BIGINT` (was reaching `Number.isInteger(BigInt)` which returns false → wrong default). 5. **Homogeneous array auto-tag**: `${[1,2,3]}` against an `int[]` column now emits `$1::BIGINT[]` instead of `$1::JSON`. Detection covers primitive types only (number / bigint / string / boolean); mixed or nested arrays still fall back to JSON. Mixed int/float widens to `DOUBLE PRECISION[]`. 6. **`.query()` positional bug**: previously the `.query()` method abused the template-tag builder, which appended `$N::TYPE` after the user's literal SQL string instead of binding by position (`SELECT $1, $2` became `SELECT $1, $2$1::BIGINT`). Now `.query()` builds the executor-shaped content directly: a `-- $N argN (TYPE)` declaration block followed by the user's SQL verbatim. Tests: - Parser: `test_parse_pg_statement_arg_positions_skips_strings_and_comments` asserts string literals, comments, and dollar-quoted blocks don't produce positions (so renumbering doesn't mangle them). - Executor unit: `renumber_sparse_placeholders_no_collision_no_string_mangling` uses the new position-aware path and includes string-literal + comment + `$$…$$` cases. Existing convert_val tests grow to cover new String→numeric/real/double/oid/bool arms. - Integration: `test_postgresql_arg_type_combinations` adds 13 cases (enum round-trip both directions, string→numeric/real/double/bool/oid, string-literal `$N` non-mangling). The prepare-fallback test now asserts SUCCESS (not failure) for enum encoding via AnyTextValue. - SDK: new `typescript-client/tests/sqlUtils.test.ts` (42 tests) exhaustively covering inferSqlType primitives + arrays, parseTypeAnnotation, datatable() template tag (with all the new shapes — BigInt, homogeneous arrays, RawSql, schema preamble), datatable().query() positional, and ducklake() shape. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(pg): replace DISCARD ALL with curated reset (preserves typeinfo cache) Found while exhaustively probing custom-type DX: every cached-connection reuse was running `DISCARD ALL`, whose included `DEALLOCATE ALL` deallocates *all* prepared statements server-side — including the typeinfo statements that tokio_postgres caches per-Client to resolve custom enum / domain Oids. tokio_postgres still held `Statement` objects whose names the server had forgotten, so the next custom-type query failed with intermittent "prepared statement \"sN\" does not exist" errors. The failure was easy to reproduce: any sequence that forced typeinfo lookup for two different custom-type kinds on the same cached connection (e.g. enum followed by domain) would hit it. Replace `DISCARD ALL` with a curated reset that explicitly targets the state we actually care about, *without* touching prepared statements: RESET ALL — GUC parameters (search_path, application _name, statement_timeout, …) RESET SESSION AUTHORIZATION — undoes both `SET SESSION AUTHORIZATION` and `SET ROLE` (RESET ALL does NOT — these aren't GUC parameters, so without this an elevated role from a previous job would silently leak) UNLISTEN * — drops LISTEN registrations CLOSE ALL — closes open cursors Trade-off: temp tables, advisory locks (session-scoped), and user-created PREPARE statements may persist across cached-connection reuse — rare in datatable / PG-script workloads. tokio_postgres's typeinfo cache survives intact, so custom enum / domain queries are fast on subsequent reuse. Tests: - `test_postgresql_custom_types_on_cached_connection` — runs 10× alternating enum + domain queries on a cached connection. Pre-fix this failed with `prepared statement "sN" does not exist` after the first reuse; post-fix passes. - `test_postgresql_set_role_does_not_leak_across_cached_connection` — switches `SET ROLE` and `SET SESSION AUTHORIZATION` to a non-postgres role, then runs a follow-up job and asserts current_user/session_user are restored. Specifically catches the case where someone might switch back to `RESET ALL` alone (which doesn't cover SET ROLE / SESSION AUTHORIZATION) and silently introduce a permission-leak vector. - All existing session-isolation tests (`test_postgresql_cached_connection_resets_session`, `test_postgresql_single_worker_session_isolation`, `test_postgresql_100_jobs_cached`) continue to pass. Found via end-to-end probing of datatable / PG-script DX, not previously covered: the existing isolation tests only did `SET ROLE postgres`, the connecting user, so the leak was invisible. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(pg): address PR #8999 review (cubic + claude) cubic (P1, real bug): - `convert_vec_val` for `timetz` array asserted `Type::TIMETZ_ARRAY`, but chrono `NaiveTime` only encodes for TIME (same caveat as the scalar arm). Switch to `Type::TIME_ARRAY`; rely on PG's implicit `time→timetz` assignment cast at the column site. Add an explicit unit test. claude (#1, silent failure → explicit error): - `Bool` + explicit `(char)` / `(character)` decl previously silently bound BOOL, hoping the server would cast at the use site — but PG has no implicit `bool→char` and the resulting error ("operator does not exist: bool = char") was opaque. Now error at bind time with an actionable hint to use `bool` decl or pass the value as a "t"/"f" string. claude (#2, asymmetry doc): - Object/Array still coerce to text on `matches!(typ, Typ::Str(_))` (covers both explicit AND inferred-default text), unlike Bool/Number which key on `explicit_text_target`. The asymmetry is intentional (no implicit `jsonb → text` cast in expression context vs PG having implicit `bool/int → text` casts) — added a body comment so future maintainers don't try to "align" them. claude (#3, perf): - `parse_pg_statement_arg_indices` and `parse_pg_statement_arg_positions` walked the SQL tokenizer twice. Fold into a single pass that derives the index set from the position list. claude (#4, fmt drift): - `cargo fmt` over the parser crates I touched with perl scripts in the earlier commit (windmill-parser-{sql,bash,ts,go,php,java,csharp,nu,py, rust,graphql,yaml,r}). Net cosmetic. claude (#5, parseTypeAnnotation): - One-line caveat in the SDK's `parseTypeAnnotation` that the returned string is presence-only (e.g. `${x}::DOUBLE PRECISION` returns `"DOUBLE"`, `CAST(${x} AS int)` returns `"int)"` — neither matches a real PG type, but the only consumer just checks `!== undefined`). While here — discovered + fixed independently while exhaustively probing DX: - **Replace `DISCARD ALL` with curated reset** (`RESET ALL; RESET SESSION AUTHORIZATION; UNLISTEN *; CLOSE ALL;`). DISCARD's `DEALLOCATE ALL` killed tokio_postgres' typeinfo cache, producing intermittent `prepared statement "sN" does not exist` errors on custom-type queries after cached-conn reuse. New regression tests: `test_postgresql_custom_types_on_cached_connection` and `test_postgresql_set_role_does_not_leak_across_cached_connection` (the latter catches the case where someone might switch back to `RESET ALL` alone and silently introduce a permission-leak vector — RESET ALL doesn't cover SET ROLE / SET SESSION AUTHORIZATION). - **ISO-8601 timestamp results** (`pg_cell_to_json_value`). Pre-fix `TIMESTAMP` was rendered with a space separator ("2024-01-15 10:30:00") and `TIMESTAMPTZ` with " UTC" suffix ("2024-01-15 10:30:00 UTC") — neither parseable by `date-fns parseISO`, JavaScript `new Date()` is lenient enough to handle them but several frontend `App*Input.svelte` components use parseISO and fail silently. Switched to ISO-8601 with `T` separator and `+00:00` offset; arg-parsing path still accepts the legacy " UTC" suffix for back-compat. Test coverage: - 17/17 unit (`pg_executor::tests`) - 9/9 integration (`backend/tests/worker.rs`, `test_postgresql_*`) - 27/27 parser (`windmill-parser-sql`) - 42/42 SDK (`typescript-client/tests/sqlUtils.test.ts`) Co-Authored-By: Claude Opus 4.7 (1M context) * fix(pg): bounded one-shot warning on numeric precision loss + ISO-8601 + NaN handling Found while probing PG-script DX with millions of numeric cells: 1. **Numeric precision-loss warning**: `numeric` results are still serialised as JSON Number (back-compat — switching to JSON String would silently break user code doing arithmetic on results), but we now detect `Decimal -> f64 -> Decimal` round-trip failure and emit a single job-log warning recommending a `::text` cast in the SQL. Bounded by `NUMERIC_PRECISION_CHECK_BUDGET = 256` cells per query (one atomic load + one fetch_sub on the hot path; first lossy value short-circuits to a single load thereafter). Worst-case overhead on a 1M-cell numeric-heavy query: ~25µs of checks + 5ns × N atomic loads (vs. ~100ms unbounded). 2. **ISO-8601 timestamps**: `pg_cell_to_json_value` previously returned `"2024-01-15 10:30:00"` (TIMESTAMP) and `"2024-01-15 10:30:00 UTC"` (TIMESTAMPTZ) — neither parseable by date-fns `parseISO`, which is what the apps `App*Input.svelte` components use, so timestamp values silently failed to round-trip into date pickers. Switch to ISO-8601 (`T` separator + `+00:00` offset) on the result side; arg-parser continues to accept the legacy `" UTC"`-suffixed format for back-compat. 3. **Float NaN / Infinity results**: `Number::from_f64` returns None for NaN / ±Inf, which `pg_cell_to_json_value` was raising as "invalid json-float" — failing the *entire* query if any cell held one of these special values. Now serialise them as JSON strings ("NaN", "Infinity", "-Infinity") and let the rest of the row come through. Arg-side: `s.parse::()` already accepts the same strings. Tests: - `decimal_fits_f64_losslessly_predicate` — covers fits / doesn't-fit cases for the precision-loss predicate. - `precision_check_budget_caps_per_query_overhead` — locks in the budget cap and the loss-flag short-circuit. - All 9 PG integration tests + 17 unit tests pass. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(pg): add pg_advisory_unlock_all to reset; warn on missing args; honor decl defaults While probing PG-script DX further found three more frictions: 1. **Advisory lock leak** (cubic P2): switching from `DISCARD ALL` to `RESET ALL; RESET SESSION AUTHORIZATION; UNLISTEN *; CLOSE ALL;` meant session-scoped advisory locks (`pg_advisory_lock`) leaked across cached-connection reuse. Add `SELECT pg_advisory_unlock_all()` to the chain — `DISCARD ALL` covered this implicitly via `DISCARD PLANS / DEALLOCATE / pg_advisory_unlock_all` and we lost it in the switch. 2. **Missing-arg silent NULL**: an arg declared in the SQL (e.g. `-- $1 amount (numeric)`) but not provided in the args object was bound as NULL with no error / warning. Misspelling the key in the args object silently produced a row of NULLs — a notorious DX debugging trap. Now: collect the names of declared-but-missing args during dispatch and emit a single one-shot warning to the job logs at end-of-query naming each one. Bound NULL is preserved for back-compat. 3. **Declaration defaults ignored**: `-- $1 a (int) = 5` carries `arg.default = Some(Number(5))`, but the dispatch fell straight to NULL when the arg was missing. Now: respect the default — user-supplied value > declaration default > NULL. Also fixes the warning logic above (only warn for args that *don't* have a default). Tests: existing 19 unit + 9 integration pass. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(pg): multi-word PG types with [] suffix lost the array-ness; array arms accept stringified values Two more frictions found while probing SDK end-to-end against a real datatable resource: 1. **Multi-word array types lose the [] suffix in the parser**. `transform_types_with_spaces` recognises aliases for "double precision", "character varying", "timestamp with time zone", etc. but its return type was `&'a str` — only the bare alias, never with a trailing `[]`. The `RE_CODE_PGSQL` regex's `\w+` captures stop at the first space, so the regex's own `(?:\[\])?` array-suffix branch sees only `"double"` (not `"double precision[]"`); the `[]` was silently lost. Result: `$1::double precision[]` (which the SDK now emits for homogeneous float arrays via the new auto-tag) routed through `Value::Array → Type::JSONB` and the server failed with "cannot cast type jsonb to double precision[]". Fix: switch `transform_types_with_spaces` to return `Cow<'a, str>` and re-check the trailing bytes after a multi-word match. If they start with `[]`, return `format!("{alias}[]")` — Owned. Single-word types and the no-match path keep returning Borrowed slices, so no allocation in the hot path. 2. **Array arms in `convert_vec_val` rejected stringified values for numeric / int* / bool / oid / real / double**. The scalar `convert_val` already parses strings into the matching native type for these arg_ts, but the array variant only accepted JSON-native counterparts. Sending `["1.5", "2.5", "3.5"]` against `$1::numeric[]` (e.g. via `unnest` for bulk loading, or `JSON.stringify(BigInt[])` round-trip) failed with "Mixed types in array". Now the array arms mirror the scalar ones — `as_().or_else(|| as_str().and_then(parse))` — so both shapes round-trip cleanly. Tests: 19 unit + 9 integration pass; existing parser tests cover the multi-word array forms (the regex-cap behaviour didn't break for single-word types, and Cow plumbing is transparent to all callers). Co-Authored-By: Claude Opus 4.7 (1M context) * fix(parsers): add otyp_inferred field to Arg literals in tests + 3 missed src files CI failures: the perl-driven sweep that added `otyp_inferred: false` to every `Arg { ... }` literal when I introduced the field in the parser schema covered `src/lib.rs` files but missed: - parsers/windmill-parser-bash/src/lib.rs (mass-edited but a later format pass un-applied a few sites) - parsers/windmill-parser-go/src/lib.rs (same) - parsers/windmill-parser-graphql/src/lib.rs (same) - parsers/windmill-parser-nu/tests/tests.rs (test file — not swept the first time) - parsers/windmill-parser-ts/tests/tests.rs (test file — same) Also tightened the regex to handle `oidx: None` without the trailing comma (some test files had the field as the last initialiser line). `cargo build --features --workspace --all-targets` is clean. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(sdk): Date → TIMESTAMPTZ; NaN / ±Infinity → string Two more frictions found while running the actual SDK end-to-end against a live datatable resource: 1. **JS `Date`** fell into the typeof "object" branch and was tagged `::JSON`. It worked accidentally for `${date}::timestamptz` via PG's `json → text → timestamptz` implicit cast chain, but `${date}` against a `timestamptz` column without a user-supplied cast bound the value as a JSON string and the comparison `timestamptz = json` failed. Now: `inferSqlType` recognises `Date` and tags `::TIMESTAMPTZ`; `serializeArgValue` emits `Date.toISOString()` so the executor's `Value::String → TIMESTAMPTZ` arm parses it cleanly. 2. **JS `NaN` / `±Infinity`** silently became NULL. `JSON.stringify(NaN)` returns `"null"` per the JS spec, so the value reached the executor as JSON null — the SDK's `::DOUBLE PRECISION` tag then bound a NULL double. Fix: detect non-finite numbers in `serializeArgValue` and stringify them as `"NaN" / "Infinity" / "-Infinity"`. The executor's `Value::String → FLOAT8` arm (`f64::from_str`) accepts these literals directly, and the result-side already renders the values as JSON strings (matching round-trip). SDK unit tests grow from 42 → 44 passing. Co-Authored-By: Claude Opus 4.7 (1M context) * test(pg): integration coverage for multi-word arrays + stringified array elements Locks in the two array fixes from the previous commit (`fix(pg): multi-word PG types with [] suffix lost the array-ness`) with end-to-end cases in `test_postgresql_arg_type_combinations`: - `double precision[]`, `character varying[]`, `timestamp without time zone[]` — verifies the parser keeps the `[]` suffix after multi-word alias resolution. - `numeric[]` / `int[]` / `bool[]` from stringified primitives — verifies the array arms of `convert_vec_val` apply the same string-coercion the scalar arms do. Co-Authored-By: Claude Opus 4.7 (1M context) * style: fix indentation drift on otyp_inferred lines cargo fmt cleanup of leftover indentation where the perl-driven sweep that introduced the otyp_inferred field landed at the wrong column. No behaviour change. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.5 * feat: support assigning a worker tag to app inline scripts (#9002) * feat: support assigning a worker tag to app/raw-app inline scripts Co-Authored-By: Claude Opus 4.5 * fix: omit empty tag field from inline script raw_code payload Co-Authored-By: Claude Opus 4.5 * style: shrink tag popover width --------- Co-authored-by: Claude Opus 4.5 * feat(pipeline): 2-col picker, draft path edit, save-all + leave guard Co-Authored-By: Claude Opus 4.7 (1M context) * all * all * update * fix(cli): forward HEADERS env var on every backend fetch call (#9075) Several `fetch()` callers in the CLI bypassed `OpenAPI.HEADERS` and skipped the `HEADERS` env var, causing requests to fail behind auth gateways like Cloudflare Access (same shape as #6421): - `pushScript()` `/scripts/create` and `/scripts/create_snapshot` — regressed in #8936 when the call switched from `wmill.createScript()` (SDK) to a raw `fetch` for the `skip_if_noop` query param. - Script preview `/jobs/run/preview_bundle`. - App dev `/jobs_u/getupdate_sse` SSE stream. - `wmill docs` `/api/inkeep`. All four now spread `getHeaders()` and call `detectAuthGatewayChallenge()` so a Cloudflare/SSO challenge surfaces a clear error instead of an opaque JSON parse failure. Adds `test/headers_env_var.test.ts`: spins up an auth-gateway proxy that 403s requests missing `CF-Access-Client-Id` / `CF-Access-Client-Secret` and otherwise reverse-proxies to the test backend, then runs `wmill sync push` of a fresh script through the proxy. Negative case (no `HEADERS` env) verifies the proxy actually gates; positive case asserts every request including `/scripts/create` reaches the backend with the headers attached. Co-authored-by: Claude Opus 4.7 (1M context) * feat(cli): add --parallel flag to generate-metadata (#9074) * feat(cli): add --parallel flag to generate-metadata * fix(cli): validate --parallel input and harden flush ordering * perf(flows): skip flow_env DB+transform work when no resolution is needed (#9078) * fix(cli-tests): stabilize flow lock-gen race + Windows path (#9080) * fix(cli-tests): stabilize flow lock-gen race + Windows path Three CLI test failures on the latest main, all flaky on CI: 1. `Mixed Case Paths: pull and push flow with capitalized folder` and `Integration: Mixed scripts and flows with nonDottedPaths are idempotent`: flow create/update queues an async FlowDependencies job that fills inline-script lockfiles and rewrites flow.value. The tests pulled/pushed before the worker finished, so dry-run idempotency saw phantom `*.inline_script.lock` adds and `flow.yaml` edits. Added a `waitForFlowDependencyJob` helper that polls `/flows/get` for the latest `dependency_job` and `/jobs_u/completed/get` until it lands, and called it after each API/CLI flow write in both tests. 2. `HEADERS env var is forwarded on every CLI fetch` (Windows-only, added in #9075): the new test built the CLI entrypoint via `new URL("..", import.meta.url).pathname`, which yields `/C:/...` on Windows and `Bun.spawn` rejected before reaching the proxy, leaving `rejectedRequests.length` at 0. Switched to `fileURLToPath` + `node:path.join` to match `cargo_backend.ts`. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(cli-tests): use /flows/deployment_status to actually wait for dep job CI reviewers (Claude, Codex) flagged the prior `waitForFlowDependencyJob` as a no-op: it read `flow.dependency_job` from `/api/w/{ws}/flows/get`, but `Flow` / `FlowWithStarred` (backend/windmill-types/src/flows.rs:20-60) do not include that field. The helper exited on the first iteration without polling. Switch to `/api/w/{ws}/flows/deployment_status/p/{path}`, which returns `{ lock_error_logs, job_id }`. `job_id` is the FlowDependencies UUID written into `deployment_metadata` in the same tx as the dep-job push (backend/windmill-api-flows/src/flows.rs:660-672 and :1275-1292), so by the time the create/update API call returns, the response carries the latest dep-job UUID. Then poll `/jobs_u/completed/get/{job_id}` as before. Local runtime for `mixed_case_paths.test.ts` jumps from ~9s to ~32s, confirming the helper now actually waits instead of returning immediately. The 404 short-circuit in `sync_pull_push.test.ts` still works — `get_deployment_status` returns 404 when the flow is absent. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * perf(flows): cache resolved flow_env per flow execution (#9079) * perf(flows): cache resolved flow_env per flow execution * perf(flows): tighten flow_env cache cap to 1024 and clarify memory note * perf(flows): don't cache transient flow_env resolution failures * chore(main): release 1.698.0 (#9076) * chore(main): release 1.698.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> * fix: reject root-rooted paths in ansible playbook validator on windows (#9081) * fix(native-triggers): serialize Google channel renewal across replicas (#9060) * fix(native-triggers): serialize Google channel renewal across replicas `sync_all_triggers` runs every 5 minutes on every windmill-app replica with no leader election. Multiple replicas were each rotating the webhook token, creating a new Google watch channel, and racing the trigger UPDATE — leaving the loser's new token (in `token`) and channel (in Google) orphaned. Cloud was accumulating ~5 leaked tokens/week without the silent best-effort `delete_token_by_hash` ever logging a warning. Wrap each per-trigger renewal in a transaction and acquire the row with `SELECT … FOR UPDATE SKIP LOCKED`. Contending replicas skip the row instead of duplicating the work. The lock spans `rotate_webhook_token` → Google API call → `update_native_trigger_service_config` and is only released on commit. Re-checks `should_renew_channel` after acquiring the lock so a replica that committed seconds earlier doesn't trigger a duplicate renewal. The pattern matches existing batch-cleanup paths in `monitor.rs` (job-retention sweep) and other `FOR UPDATE SKIP LOCKED` call sites. Also logs at `debug!` when `delete_token_by_hash` finds no matching row, so future investigations can distinguish "deleted" from "not found" without changing the `Ok(false)` contract. Co-Authored-By: Claude Opus 4.7 (1M context) * fixup! fix(native-triggers): serialize Google channel renewal across replicas * fixup! fix(native-triggers): serialize Google channel renewal across replicas fixup! fix(native-triggers): serialize Google channel renewal across replicas Address claude review: - #5: per-skip log info -> debug (expected outcome under SKIP LOCKED) - #2: warn moved out of delete_token_by_hash to the call site that knows the expected state (try_renew_channel_locked); other callers are race-prone and shouldn't warn - #3: NULL service_config now warns (anomalous case) - #4: post-Google-API DB-update + commit failures log distinctly so the channel-orphan case is grep-able Plus: add 14d expiry to Google webhook tokens via ServiceName::webhook_token_expiration, mint fresh ephemeral-webhook-{service}-{rd5} labels at create + rotate so the existing 'ephemeral-' filter excludes them from user-token email/critical-alert paths (no filter changes in 3 places). Orphans now self-clean via the existing expiry sweep in monitor.rs. Co-Authored-By: Claude Opus 4.7 (1M context) * fixup! fix(native-triggers): serialize Google channel renewal across replicas fixup! fix(native-triggers): serialize Google channel renewal across replicas Address second-round review: - Claude #1 (P2): username_override_from_label now strips the 'ephemeral-' prefix for ephemeral-webhook-* labels, so created_by stays webhook-{service}-{rd5} instead of changing to label-ephemeral-webhook-... (preserves audit/job-list filter compatibility) - Codex (P2): updated renew_channel doc — labels are no longer copied; rotate mints fresh ephemeral-webhook-google-{rd5} with 14d expiration - Claude #3 (optional): test_rotate_webhook_token now asserts the rotated Google token has an ephemeral-webhook-google-* label and a populated expiration Co-Authored-By: Claude Opus 4.7 (1M context) * fixup! fix(native-triggers): serialize Google channel renewal across replicas fixup! fix(native-triggers): serialize Google channel renewal across replicas Reconsider the previous fixup: stripping the 'ephemeral-' prefix made created_by no longer match token.label exactly, defeating the linking purpose. Just allowlist 'ephemeral-webhook-' alongside the other recognized webhook/email/ws prefixes — created_by becomes ephemeral-webhook-google-XXXXX, matching token.label exactly. The 'ephemeral-' substring also informs operators that this is a system-managed auto-expiring token vs a user-managed webhook trigger. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * fix(cli): bump svelte version in `wmill app new` template (#9084) * fix(cli): bump svelte version in `wmill app new` template The svelte5 template pinned `svelte` to `5.45.2`, but the Svelte compiler bundled in `wmill app dev` emits `$.delegated('click', ...)` calls. The `delegated` export was added later, so 5.45.2 doesn't have it — esbuild warns `Import "delegated" will always be undefined`, replaces the call with `void 0`, and the page crashes at first event-handler bind (white screen). Bump to `^5.55.5` so the compiler and runtime stay in sync. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(frontend): bump svelte version in raw_apps UI template Mirror the CLI fix: the UI's `Add raw app` flow scaffolds a package.json with `svelte: "5.45.2"`. That works today only because the bundled rolldown worker also pins 5.45.2 — when the worker is upgraded past 5.51.1, the compiler will emit `$.delegated()` and the runtime won't have it, producing the same white-page crash that hit the CLI. 5.55.5 still exports `event` (used by the current bundled compiler), so this is forward-compatible: it works with the 5.45.2 compiler now and won't break when the worker is upgraded. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * perf(flows): gate flow_env resolve on expr text and share cache with handle_flow (#9085) * feat: parse windmill_failure field to tag run as failure (#9073) * feat: parse windmill_failure field in job result to tag run as failure * feat: preserve top-level fields when windmill_failure tags a run as failure * fix: address review findings on windmill_manual_failure * refactor: rename windmill_manual_failure to wm_failure and add wm_* aliases * fix: prefer injected ManualFailure error over sibling name/message in OTel * fix: hide _ENTRYPOINT_OVERRIDE jobs from script/flow history panel (#9088) * fix(flows): populate error handler input args from failure picker (#9087) * fix(flows): populate error handler input args from failure picker * style(flows): fix indentation in failure-step branch * fix(python): verify wheel RECORD on cache pull/install, finalize piptar (#9090) The Python per-package dependency cache could persist an incomplete wheel extraction with `.valid.windmill` set, then propagate that broken artifact to every worker through the object store. Customer hit this on argon2-cffi==25.1.0 (missing argon2/_utils.py), and previously on botocore/httpx (truncated tars). Symptom is a runtime ImportError that looks like a missing dependency declaration rather than a Windmill bug. Three changes that together stop the propagation: 1. After `pull_from_tar`, parse the wheel's `/RECORD` and confirm every listed path exists on disk before writing `.valid.windmill`. On failure, wipe the directory and fall through to a fresh local install — the next install also self-heals the broken object-store entry by pushing a fresh tar. 2. After `uv pip install` succeeds, run the same RECORD check before queuing the piptar upload or writing `.valid.windmill`. A bad install never becomes the source of a broken tar in the object store. 3. Finalize the tar (`drop(tar.into_inner()?)`) before reading its bytes for upload, so we never push an unfinalized archive (no end-of-archive marker) to the object store. Verified with a 60-package end-to-end integration test (first-fill → clear-local-cache → re-pull-from-objectstore → corrupt-objectstore-tar → detect-and-self-heal). All 27 packages on the live test pulled cleanly, and the deliberately corrupted argon2-cffi tar was caught with the exact expected log line ("wheel RECORD lists files missing on disk: argon2/_utils.py") and replaced with a fresh tar. Co-authored-by: Claude Opus 4.7 (1M context) * chore(main): release 1.699.0 (#9082) * chore(main): release 1.699.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> * feat(cli): auto-infer args for `wmill app push` (#9091) Run `wmill app push` from inside an app folder (e.g. `f/foo/my_app.app/`) with no args. The local path defaults to CWD, and the remote path is derived from CWD relative to `wmill.yaml`, with `.app`/`.raw_app`/ `__app`/`__raw_app` suffixes stripped. Either, both, or neither positional argument can be passed. Also resolves `file_path` against the user's original CWD before `resolveWorkspace` may chdir to the wmill.yaml root, so a relative `file_path` argument is interpreted from where the user invoked the command (previously it could resolve against the wrong directory). Co-authored-by: Claude Opus 4.7 (1M context) * all * fix(pipeline): live-update graph for annotations and body assets * fix(pipeline): persist draft body edits across node switches * fix(pipeline): persist live writes per draft to keep output node fresh after switch * feat(pipeline): animate graph edges only while a runnable is executing * feat(pipeline): add run button on script nodes + recomputing hint on preview * feat(pipeline): compact preview layout, two-way Test/Run sync * fix(pipeline): test button cross-browser placement (no overflow trick) * style(log-viewer): replace took/mem-peak labels with timer/cpu icons * style(log-viewer): hyphenate Auto-scroll label and prevent wrapping * style(log-viewer): lowercase auto-scroll label, force vertical scrollbar * style(log-viewer): force horizontal scrollbar instead of vertical * fix(log-viewer): scope overflow-x to top bar so pre doesn't drive panel width * fix(pipeline): overlay live body-asset writes for persisted scripts too * fix(pipeline): persist inferred body assets at save so edges survive page reload * fix(pipeline): snapshot live draft writes at persist time so they survive reload * fix(pipeline): keep inferred body writes on the canvas across selection changes * fix(pipeline): untrack inferredWrites cache mutation to break effect loop * fix(pipeline): refetch asset graph after persisted-script save * feat(pipeline): optional AI prompt when creating a pipeline script * all * all * test: cover asset-trigger dispatch end-to-end through worker * feat(pipeline): split-button Test with optional downstream cascade * feat(pipeline): cascade option on graph Run + match button heights * style(pipeline): match caret bg/text to Test button's accent-secondary * feat(pipeline): split Run pill on graph node exposes cascade option * feat: live run activity + status badges in pipeline asset graph - folder-scoped queue poll lights up the downstream asset-trigger cascade (not just the launched script); zero requests at rest, catch-up for fast hops, auto-disarm when idle - per-runnable node badge: last-run status + session run count - animate unsaved/live-parsed edges (was unconditionally suppressed) - background-pane click no longer clears selection - run-bridge guarded so node selection/save no longer triggers a test Co-Authored-By: Claude Opus 4.7 (1M context) * feat: live activity log, optimistic badges, node-avoiding graph edges - collapsible folder activity log (PipelineEventLog): live job feed, polls only while open/active, slow idle cadence, capped + pruned - composable: observe mode + events list + run-count anchored to graph-open time (pre-existing history excluded) - optimistic node badge: launched script shows running instantly via the zero-latency activeRunnable hint, keeps the polled run count - activity pane height capped (min(18rem,40vh)) then scrolls - route asset-graph edges through sugiyama-computed waypoints so they go around nodes instead of under them; bezier fallback for adjacent-layer / draft-overlay edges Co-Authored-By: Claude Opus 4.7 (1M context) * feat: prefetch all folder script assets so graph is stable on load On pipeline load, eagerly infer body assets for every persisted folder script and seed the existing inferredWritesByPath overlay, instead of only filling it when a node is selected. Scripts whose persisted asset rows are missing (e.g. object-form writeS3File) now have their edges from first paint, so clicking a node no longer re-layouts the graph. One-shot per (workspace, base-graph) load, untracked map reads, generation-cancelled, pool-capped fetches. Co-Authored-By: Claude Opus 4.7 (1M context) * perf: guard no-op poll re-layout; dedupe write-asset extraction - skip reactive ids/states/events reassignment when unchanged, so an idle poll tick no longer re-runs the full sugiyama layout every 3-6s - bound countedJobIds (rebuilt from eventsById in lockstep with prune) - extract shared extractWrites() helper, replacing 4 copy-pasted write-asset filter/map blocks in the pipeline page - compute activeRunnable node-id once, reuse for the active-edge set and the optimistic badge (flattened ternary); trim narrating docs Co-Authored-By: Claude Opus 4.7 (1M context) * feat: live read-lineage overlay for inferred body assets Renaming e.g. duckdb read_parquet('s3://...') / loadS3File now updates the asset->reader edge live instead of only after Save re-derives the persisted asset rows. - extractReads() (+ shared refsByAccess) mirroring extractWrites - inferredReadsByPath sticky cache, filled by handleAssetsChange and the load prefetch alongside writes - replace the write-only overlay loop with one overlayLineage(map, access) helper invoked for both 'w' and 'r' (net DRY) Co-Authored-By: Claude Opus 4.7 (1M context) * fix: detect S3 assets passed as SDK object arg in ts parser Mirrors merged PR #9181 so feat/asset-graph-view is self-contained (local origin/main is stale and lacks it). Object/{ s3, storage } form of writeS3File/loadS3File is now detected, not only the bare s3:// string literal. Co-Authored-By: Claude Opus 4.7 (1M context) * chore: regenerate wasm Cargo.lock + frontend package-lock Lockfile churn from local wasm-pack (asset target) + npm operations during the asset-graph work. No source/dependency-intent change. Co-Authored-By: Claude Opus 4.7 (1M context) * fix: revert to bezier graph edges; add parsing-assets hint The sugiyama-waypoint routing looked worse than the original; revert AssetGraphEdge/assetGraphLayout to the pre-routing bezier logic (same as the flow editor's BaseEdge) and drop the now-unused route plumbing from the canvas. Add a small 'Parsing assets…' hint shown while the load-time prefetch sweep is still inferring folder scripts. Co-Authored-By: Claude Opus 4.7 (1M context) * refactor: extract pure resolveGraph merge + unit tests Move the ~230-line graphWithDraft precedence/merge (base < session- inferred < draft-seeded < open-script-live, +read/write/annotation overlays, +dedup) out of the 1648-line route into a pure, testable resolveGraph() module; the route's graphWithDraft is now a thin $derived. Behaviour extracted verbatim. 10 unit tests cover the precedence matrix. Phase 1 of the state/render split. Co-Authored-By: Claude Opus 4.7 (1M context) * style: graph controls top-right, lift minimap, hide Save when unchanged Controls -> top-right horizontal, no lock toggle; MiniMap !mb-10 so it clears the activity bar; hide the per-script Save button when the script is already at its latest save point (drafts still show Create). Co-Authored-By: Claude Opus 4.7 (1M context) * fix: scope runtime-asset prune by id to spare static lineage rows prune_runtime_assets deleted by (workspace_id, path, kind) tuple, so trimming surplus usage_kind='job' rows for an s3 path also wiped the static usage_kind='script'/'flow' producer rows for the same path — silently breaking the asset-trigger cascade (fetch_producer_writes found no writes; downstream never dispatched; required band-aid re-syncs). Delete the surplus job rows by id instead; the inner query is already scoped to usage_kind='job'. Co-Authored-By: Claude Opus 4.7 (1M context) * fix: don't re-pulse already-running jobs after they finish The catch-up pulse re-added a completed job to the active set if its start was within the (lagging) lookback window — even one we'd already animated the whole time it ran — keeping its edges lit ~a poll interval past completion (~5s after a 3.5s test). Track job ids seen in-flight and skip the pulse for them; it still fires for hops whose whole lifetime fell between two polls. Bound the set in lockstep with eventsById; cleared on dispose. Co-Authored-By: Claude Opus 4.7 (1M context) * fix: don't catch-up-pulse the runnable launched from the graph If the poll never sampled a launched run's in-flight window, the catch-up pulse re-flashed its edges one tick after it correctly stopped (the page already animated it zero-latency via activeRunnable). arm(launchedId) records the launched runnable id; catch-up skips it. Cascade hops (other ids) still pulse. launchedIds cleared on stop. Co-Authored-By: Claude Opus 4.7 (1M context) * style: nudge graph controls left to clear panel toggle Co-Authored-By: Claude Opus 4.7 (1M context) * feat: partition value resolver + asset-cascade propagation windmill-common/partition: pure resolver — time kinds (tz/format/start anchor) + dynamic $.a.b JSONPath; 9 unit tests. asset_dispatch: read the producer's resolved partition and thread it into every cascaded subscriber's args + trigger.partition, so a chain resolves once at the top. No migration (cascade needs no spec lookup). Stage 1+3 of pipeline partition runtime; run-start resolution is Stage 2. Co-Authored-By: Claude Opus 4.7 (1M context) * feat: show args form in compact pipeline preview when script has inputs AssetGraphDetailsPane keeps the compact (hideArgs) preview but, via a new previewPanel.argsAboveLogs flag, renders a compact SchemaForm between the floating Test button and the logs/result panel when the script declares inputs (e.g. a partitioned script needing a `partition` arg). The preview pane also grows ~18pts so the args form doesn't shrink logs/result. Co-Authored-By: Claude Opus 4.7 * feat: parser join-mode (`// trigger all`) + script_trigger.join_all Stage A: JoinMode{Any(default),All} + `// trigger any|all` directive in parse_pipeline_annotations; TriggerSpec::is_partition_bearing() (path contains {partition}); join_mode threaded through all 4 asset-parser crates (ts/py/sql/yaml). Stage B: reversible migration adds script_trigger.join_all; insert_script_trigger writes it; deploy path sets it from the parsed annotation. No reader yet (AND-join dispatch is the next stage) so runtime behaviour is unchanged. Co-Authored-By: Claude Opus 4.7 * feat: resolve pipeline partition at job execution time Stage C: in handle_code_execution_job, once the script content is loaded, parse the // partitioned annotation (free here) and resolve the concrete partition once — schedule fire-time (scheduled_for anchor, not wall-clock) for time kinds, triggering payload for dynamic. The value is injected into the in-memory args the body sees (via a shadowed job clone) and persisted back to v2_job.args so dispatch_asset_triggers propagates the same value down the cascade. Already-set (explicit/backfill/cascade) partitions are never re-resolved (run identity immutable); unresolvable partitioned runs fail with a clear error. Integration test exercises the full worker loop + cascade propagation. Co-Authored-By: Claude Opus 4.7 * feat: AND-join barrier for partitioned pipeline subscribers Stage D: a // trigger all subscriber no longer fires on any input. New join_pending_inputs slot table keyed (workspace, subscriber, partition); fetch_subscribers now returns join_all and the dispatch loop records each partition-bearing input arrival, pushing the subscriber once only when every partition-bearing input it declares is present for that partition. Per-partition slots, cleared on fire (re-accumulate, no double-fire), skew-immune (unlike debounce). Case-3 guard: an unpartitioned producer or a reference (non-{partition}) input never fires a partitioned join. Integration test covers wait/fire/isolation/no-double-fire. Co-Authored-By: Claude Opus 4.7 * feat: opt-in // debounce for asset-cascade subscribers (parser + schema) Stage E1+E2. Parser: script-level // debounce + per-// on debounce= override (edge wins, else script default, else none = fan-out, unchanged); TriggerSpec::Asset carries the per-edge override; split_trailing_kv_opts separates the ref from trailing key=val opts. Schema/deploy: reversible migration adds script_trigger.debounce_s; parse_duration_secs (bare int or s|m|h|d, fail-safe on garbage) resolves the effective per-edge window at deploy and writes it per row. No reader yet (dispatch wiring is E3) so runtime is unchanged. New unit tests for the parser directive and duration parsing. Co-Authored-By: Claude Opus 4.7 * feat: apply opt-in debounce to asset-cascade subscriber dispatch Stage E3. fetch_subscribers now also returns debounce_s; push_subscriber builds real DebouncingSettings (delay + a (subscriber, partition) key, so distinct partitions never collapse and latest-in-window falls out) instead of ::default() when the edge opted in. Default stays no-debounce (fan-out — the prior deliberate behaviour, now overridable rather than reversed). Wiring test asserts the dispatched job carries the configured window/key and an undebounced edge carries none. Co-Authored-By: Claude Opus 4.7 * fix: atomic AND-join gate + preserve resolved partition; drop scratch artifacts Addresses local-review findings before PR: - P1: record_and_check_join_slot was a non-atomic check-then-act on a pooled connection; concurrent completion of a subscriber's last two partition-bearing inputs on different workers could double-dispatch. Now one transaction guarded by a tx-scoped advisory lock keyed on (workspace, subscriber, partition) so the gate fires exactly once. - P2: the preprocessed-args overwrite in result_processor replaced args wholesale, dropping a partition resolved by resolve_partition_for_job; the UPDATE now preserves an existing persisted partition key. - P2: gate resolve_partition_for_job on a cheap code.contains check so non-pipeline script jobs skip the annotation scan on the hot path. - P2: remove 40 scratch screenshot PNGs, a flicker-debug script and a local scheduler lock accidentally committed; gitignore the lock. Co-Authored-By: Claude Opus 4.7 * test: AND-join fires once under concurrent upstream completion Regression for the check-then-act race fixed by the advisory-locked transactional gate: releases N producer dispatches simultaneously via a barrier and asserts the AND subscriber is pushed exactly once and the slot is cleared. The invariant holds for the correct gate regardless of interleaving; a non-atomic regression fails it. Co-Authored-By: Claude Opus 4.7 * test: fuller partitioned join + multi-hop pipeline coverage Exercises a complex pipeline combining options end to end: two partitioned producers fanning into a // trigger all join, then a multi-hop downstream chain. Asserts the resolved partition propagates unchanged at every hop, chain depth increments per hop, the AND barrier fires exactly once, and a second partition opens an independent slot with no cross-partition bleed across the whole graph. Co-Authored-By: Claude Opus 4.7 * refactor: simplify pipeline code per review (dedup, single-parse, constant) - ParseAssetsOutput::new() collapses the 6-line annotation copy-paste across the 4 asset-parser crates to one call site. - asset_dispatch: parse the cascade trigger object once and pass it to the depth/partition readers instead of deserializing it twice; add a TRIGGER_ARG constant for the previously stringly-typed key (3 sites). - scripts deploy: drop a redundant debounce_default clone. No behavior change; 29 parser + 6 dispatch integration tests green. Co-Authored-By: Claude Opus 4.7 * feat: reap abandoned AND-join slots after a TTL (default 60d, per-slot) join_pending_inputs slots are normally cleared when the join fires; partial slots whose inputs never all arrive (upstream removed/renamed, one-off dynamic partition key, permanent skew) would otherwise leak. windmill_queue::asset_dispatch::reap_stale_join_slots, called from the monitor's delete_expired_items loop, deletes a (workspace, subscriber, partition) slot only when its MOST RECENT row is older than JOIN_SLOT_TTL_SECS (60d) — per-slot, never per-row, so a legitimately slow join is not corrupted mid-accumulation. Conservative default; per-join configurable TTL via the annotation is a planned follow-up. Test covers stale-reaped / fresh-kept / mixed-slot-kept. Co-Authored-By: Claude Opus 4.7 * update * feat: path-less native trigger markers + missing-trigger placeholder * feat: pipeline // tag and // retry annotations + dispatch_event log * fix: derive test-pane min from split-axis dimension (height in bottom layout) * feat: show last run logs/result when a script node is selected * fix: backfill asset rows from script.assets for pre-feature scripts * feat: job-id link + dispatch popover above script log/result * style: drop 'dispatched' label, keep just the check icon * fix: drop tag picker from pipeline script editor (set via // tag annotation) * Nicer UI * refactor: move google ai proxy handling to windmill-ai (#9260) * refactor: add ai proxy execution mode * refactor: move google ai proxy handling * refactor: share google ai request building * fix: early return should consider failure_module result (#9241) * fix(flows): flag noLogs jobs and lazily resolve them in log panel (#9099) * fix(flows): flag noLogs jobs and lazily resolve them in log panel * fix appending to flag * fix: preserve WM_LOGS_SKIPPED sentinel on SSE/replay completion pickMoreCompleteLogs resolved both sentinel and undefined to '', so the SSE completion event (whose job field is fetched .without_logs()) would clobber the sentinel placed by flagSkippedLogs. The module log panel then saw '' instead of the sentinel, defeating the lazy-resolve path. Also wire onLogsResolved on the OutputPickerInner inline LogViewer so a lazy resolve writes back to flowStateStore.previewLogs, matching ModulePreviewResultViewer and avoiding repeated fetches on remount. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * chore(main): release 1.705.0 (#9229) * chore(main): release 1.705.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> * chore: add playwright mcp for frontend verification (#9269) * feat: CLI datatable serve / psql (#9267) * feat(cli): add datatable list and run commands * feat(cli): render datatable query results as a table * feat(cli): serve datatables as a postgres-wire endpoint * feat(cli): add 'datatable psql' to launch psql against the proxy * feat(cli): route datatable serve by client-supplied database name * override database list + password option * fix: support extended queries in datatable serve * fix: correct cloud size threshold log and parse CLI descriptions with parens/trailing comma * refactor: extract raw_output envelope encoding into pg_raw_output module --------- Co-authored-by: claude[bot] <41898282+claude[bot]@users.noreply.github.com> * oom_adj nit * feat: add UV_PYTHON_INSTALL_MIRROR env and instance setting (#9271) * feat: add UV_PYTHON_INSTALL_MIRROR env and instance setting Allows operators to point `uv python install` at a private mirror of the python-build-standalone releases. Configurable via the `UV_PYTHON_INSTALL_MIRROR` env var or the `uv_python_install_mirror` instance setting, with the env var as the boot fallback and the instance setting taking precedence at reload. Fixes WIN-1966 Co-Authored-By: Claude Opus 4.7 (1M context) * fix: hoist uv_python_install_mirror binding above sandboxing branch The non-sandboxed uv pip install branch referenced a binding that was only declared inside the sandboxed branch. Co-Authored-By: Claude Opus 4.7 (1M context) * fix: neutral placeholder for uv_python_install_mirror The previous placeholder was the default public URL the setting is meant to redirect away from. A neutral example mirror URL is clearer. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * fix(indexer): tell admins when ingress routes search to wrong pod (#9274) * [ee] fix(indexer): tell admins when ingress routes search to wrong pod When the IndexReader is absent on the pod handling a search request but another pod is actively holding the indexer lock, the EE handler now returns a tailored error pointing at the ingress/load-balancer configuration instead of the generic "indexer not running" message. The indexer status endpoint reads the DB lock so it reports "running" from any pod, but search endpoints need the in-memory IndexReader that only exists on the lock holder. In multi-replica deployments this looks like the indexer is healthy but every search 404s. Companion: windmill-labs/windmill-ee-private#TBD Fixes WIN-1968. Co-Authored-By: Claude Opus 4.7 (1M context) * chore: update ee-repo-ref to eb18d7b4c0e37fea3f6e1e2cc44e0fddd74ff817 This commit updates the EE repository reference after PR #586 was merged in windmill-ee-private. Previous ee-repo-ref: 7dd43d1850813071cc18ba49ba090583e7321f4b New ee-repo-ref: eb18d7b4c0e37fea3f6e1e2cc44e0fddd74ff817 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 4.7 (1M context) Co-authored-by: windmill-internal-app[bot] * feat(cli): add `wmill init prompts` and custom override slot (#9266) * feat(cli): add `wmill init prompts` and custom override slot Co-Authored-By: Claude Opus 4.7 (1M context) * refactor(cli): replace init prompts with refresh prompts + AGENTS.md/AGENTS.cli.md split Co-Authored-By: Claude Opus 4.7 (1M context) * feat(cli): dedupe claude skills via @-includes and add prompts freshness check Co-Authored-By: Claude Opus 4.7 (1M context) * refactor(cli): drop migration-choice flags from `refresh prompts` Co-Authored-By: Claude Opus 4.7 (1M context) * docs(cli): add 'Running and previewing local changes' section to AGENTS.cli.md Co-Authored-By: Claude Opus 4.7 (1M context) * fix(cli): write full skill content to .claude/, drop @-include wrapper Co-Authored-By: Claude Opus 4.7 (1M context) * feat(cli): reconcile CLAUDE.md the same way as AGENTS.md Co-Authored-By: Claude Opus 4.7 (1M context) * fix(cli): address PR review nits — argv parsing, lazy import, comment detection, error propagation Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * feat: add yolo mode for ai chat tools (#9258) * feat: add yolo mode for ai chat tools * nit * fix: align chat footer controls * feat: add ai chat autonomy modes * feat: add autonomy mode dropdown * fix: highlight yolo autonomy icon * fix: auto accept flow edits * fix: hide unsupported autonomy modes * fix: handle auto-accept flow editor races * fix(debugger): add non-root user support to Dockerfile (#9277) Mirrors the main Windmill Dockerfile pattern: creates a windmill user (UID/GID 1000) and makes cache/work directories world-writable so the image runs cleanly under Kubernetes securityContext.runAsNonRoot or runAsUser: 1000 without permission errors on Bun, pip, or windmill cache writes. Fixes WIN-1969 Co-authored-by: Claude Opus 4.7 (1M context) * fix(ai): enforce RLS and scope check on user-supplied X-Resource-Path (#9276) * fix(ai): enforce RLS and scope check on user-supplied X-Resource-Path The AI proxy handler accepts an X-Resource-Path header to override the configured workspace AI provider. When supplied, the handler loaded the resource value from the resource table using the root DB pool with no resources:read scope check, so any authenticated workspace user could point X-Resource-Path at a restricted AI resource (e.g. one in a folder they cannot read) and the proxy would use that resource's provider credentials for the outbound AI request. For user-supplied resource paths, now require resources:read:{path} scope and fetch the resource through user_db.begin(&authed) so RLS enforces the same folder/group boundary as the resource API. The RLS- scoped $var: resolution stays in place as defense in depth. The admin-configured workspace/instance ai_config path is unchanged. Fixes WIN-1971 Co-Authored-By: Claude Opus 4.7 (1M context) * test(ai): regression test for X-Resource-Path RLS enforcement Cover all four cases: - non-admin pointing X-Resource-Path at a restricted resource is rejected - non-admin pointing it at a resource they own still works - admin can point it at any resource - workspace-configured proxy flow (no X-Resource-Path) is unchanged Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * feat: add userdraft listing primitives (#9268) * feat: add userdraft listing primitives * fix: cancel stale userdraft discard writes * docs: remove global ai userdraft plan * feat(nsjail): optional disk-backed /tmp via instance setting (#9272) * feat(nsjail): optional disk-backed /tmp via instance setting * test(nsjail): unit-test tmp mount resolver and narrow visibility * refactor(nsjail): switch tmp backing to select + conditional UI * ui(nsjail): make tmpfs the visible default in /tmp backing select * fix(nsjail): refuse preexisting jail_tmp to block symlink escape * fix(nsjail): allow jail_tmp reuse on sequential nsjail calls Codex flagged that python/ruby/rust executors invoke nsjail twice per job_dir (install then run). The previous resolver treated any preexisting jail_tmp as hostile and silently fell back to tmpfs on the second call, so disk-backed mode never reached the main script run for those langs. Use symlink_metadata().is_dir() to distinguish a real directory left by an earlier call in the same job_dir (safe to reuse) from a symlink or other entity (still refused, as the codebase-tar escape requires). Also loosen the frontend visibility predicate: only hide nsjail settings when job_isolation is explicitly 'none' or 'unshare', so deployments that enable nsjail via DISABLE_NSJAIL=false with no DB setting can still see the controls. * chore(main): release 1.706.0 (#9270) * chore(main): release 1.706.0 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> * fix(nsjail): gate unix-symlink test behind cfg(unix) for Windows build (#9280) The disk_backed_refuses_preexisting_symlink_at_jail_tmp test calls std::os::unix::fs::symlink directly, which doesn't exist on Windows targets. Without a cfg gate, `cargo check --tests` fails on Windows with E0433. Other symlink call sites in this crate (php_executor, bun_executor, rust_executor, etc.) already follow this pattern. Fixes WIN-1972 Co-authored-by: Claude Opus 4.7 (1M context) * Reduce slim image vulnerability surface (#9279) * Reduce slim image vulnerability surface * chore(docker): drop apt-get upgrade -y from slim images apt-get upgrade hurts build reproducibility (same Dockerfile + same commit at different times produces divergent images) and trips hadolint DL3005. The freshness it buys is dominated by simply rebuilding against the periodically-refreshed debian:bookworm-slim base image. The --no-install-recommends and apt-list cleanup wins are kept. --------- Co-authored-by: Ruben Fiszel * fix(git-sync): bump to hub/28234 with stateless gpg.program wrapper (WIN-1974) (#9282) * fix(git-sync): revert LATEST_GIT_SYNC_SCRIPT_PATH to hub/28230 to restore GPG-signed deploys (WIN-1974) hub/28231 (PR #9230) is the "thin" script that hands the actual `git commit` to the CLI's hidden `sync git-deploy`. The hub script still does the GPG setup (import key into a fresh GNUPGHOME, dummy `gpg -bsau` to warm the agent passphrase cache, then `git config user.signingkey` + `commit.gpgsign` locally), but the commit no longer runs in the same `git_push` flow — it runs minutes later inside the CLI after workspace API resolution, zip pull, file extraction, and lockfile autofill. By the time the spawned `git commit` asks gpg-agent for the cached passphrase, the cache state is no longer reliable (or the spawned `gpg` ends up talking to a fresh agent), so signing fails non-interactively with `gpg failed to sign the data`. hub/28230 is hub/28217's in-script logic rebuilt with windmill-cli@1.703.3: the GPG setup and the in-script `sh_run("git commit ...")` happen back-to-back in `git_push`, so the cache is always fresh. It preserves wm_deploy / fork branch behavior, the EE deployment-callback `main()` signature is unchanged, and the only min-version check in EE (`is_script_meets_min_version(28103)`) is comfortably below 28230 — so this revert is safe. Forward fix (separate PR): publish a new thin script that, alongside the existing GPG setup, writes a `gpg.program` wrapper using `--pinentry-mode loopback --passphrase-file` so signing is independent of the agent's cache state. Re-bump past 28231 then. Fixes WIN-1974 Co-Authored-By: Claude Opus 4.7 (1M context) * chore(git-sync): check in source-of-truth for the next hub script (gpg.program wrapper) This is the script that will be published to hub.windmill.dev once verified on a customer GPG-signed deploy. It replaces hub/28231's agent-cache pre-warm (`gpg -bsau` with --passphrase) with a stateless gpg.program wrapper + chmod-600 passphrase file. Every git-invoked gpg call goes through the wrapper, which always uses --pinentry-mode loopback (and --passphrase-file when a passphrase exists). Signing no longer depends on gpg-agent having a cached passphrase by the time the CLI's `git commit` runs — which closes WIN-1974. Not wired in yet: LATEST_GIT_SYNC_SCRIPT_PATH stays on hub/28230 until this script is uploaded and the new hub id is known. This file is checked in so the diff is reviewable, future bumps have a source of truth, and a CLI regression test can `cat` it for fixture parity. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(frontend): skip format/pattern validation for $var/$res/$jsonvar references in ArgInput A resource field with a `pattern` constraint (e.g. the gpg_key.private_key field, whose pattern enforces a `-----BEGIN PGP PRIVATE KEY BLOCK-----` prefix) rejects values like `$var:u/me/gpg-private-key` with an "invalid format" error in the resource editor — even though `$var:`/`$res:`/`$jsonvar:` are placeholders the backend resolves at runtime, not the actual string that needs to match the regex. Bail out of all format/pattern checks (email, ipv4, ipv6, uuid, custom pattern) when the value is one of these references. Required/numeric bounds/array checks still apply since they're shape-level, not regex. Co-Authored-By: Claude Opus 4.7 (1M context) * feat(git-sync): bump LATEST_GIT_SYNC_SCRIPT_PATH to hub/28234 (gpg.program-wrapper fix) hub/28234 is the forward fix for WIN-1974: replaces hub/28231's agent-cache pre-warm (which became stale by the time the CLI's `git commit` ran) with a stateless `gpg.program` wrapper that uses `--pinentry-mode loopback` (and `--passphrase-file` when a passphrase exists) on every gpg invocation. Bundled CLI is windmill-cli@1.705.0. Verified via reproducer at /tmp/git-sync-diff/test-gpg-fix.sh: deliberately killing gpg-agent between GPG setup and `git commit` reproduces the customer's `gpg failed to sign the data` error verbatim under the old flow, and the wrapper signs through it. Holds for passphrase-protected keys, split-subkey [C]+[S] layouts, and unprotected keys. Drops the local source-of-truth copy (`hub-scripts/`) — hub is canonical now that 28234 is published. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(git-sync): drop verbose comment above LATEST_GIT_SYNC_SCRIPT_PATH The git history (this PR) carries the why; the constant name + value carry the what. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * refactor(cli): wmill sync git-deploy stops committing; caller owns commit+push (#9284) Single contract for the deployment-callback path: the CLI does branch checkout + pull, the caller (hub script in production, test in test) does git add + commit + push. This restores the WIN-1974 invariant — GPG setup and `git commit` run back-to-back in the same process, so the agent's pre-warmed passphrase cache is still warm at sign time — without needing a `--skip-commit` flag for the hub case and a default "also-commit" for everything else. Same behavior in every call site. Changes: - sync.ts: drop the gitSyncDeployPush call from pull()'s deploy path (both the onlyCreateBranch fast-return and the post-pull commit). `gitSyncDeployPush` stays exported for any caller that wants the same commit/push semantics — just not invoked by the CLI subcommand. - gitsync_promotion.test.ts: e2e test now does its own git add + commit + push after `wmill sync git-deploy`, mirroring what the hub script does in production. Same regression coverage (wm_deploy branch created in Case A, main untouched; main updated in Case B, no new wm_deploy). CLI typecheck unchanged (two pre-existing TarAsZip errors at lines 2578/3307, present before this PR). All 743 unit tests still pass. The accompanying hub script (option-C — CLI for branch+pull, script for commit+push) lives at /tmp/git-sync-diff/sync-script-to-git-repo-windmill.option-C.ts. Once published, a follow-up bumps LATEST_GIT_SYNC_SCRIPT_PATH to its id. Co-authored-by: Claude Opus 4.7 (1M context) * bump git sync to 28236 * fix: fork compare visibility for non-admins and stale-token superadmins (#9283) * fix: use fork-scoped authed for fork visibility in compare_workspaces * test: add EE end-to-end repro for fork rename visibility * chore: restore concurrency_locks sqlx cache lost in cleanup * test: add regression for stale-superadmin-token fork visibility bug * chore: update sqlx cache for new test queries * chore(main): release 1.706.1 (#9281) * chore(main): release 1.706.1 * Apply automatic changes --------- Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> * feat: add wmill job rerun subcommand (#9275) * feat: add wmill job rerun subcommand * feat: add wmill job restart subcommand for flow restart-at-step * chore(system_prompts): point plugin skills sync at plugins/windmill/ (#9287) * chore(system_prompts): point plugin skills sync at plugins/windmill/ The plugin checkout's plugin folder is being renamed from `plugins/windmill-code-plugin/` to `plugins/windmill/` to shorten the slash-command namespace and align with the matching Cursor plugin layout. Paired with windmill-labs/windmill-claude-plugin#8. That PR must merge first so the next sync run finds the new folder. Co-Authored-By: Claude Opus 4.7 (1M context) * docs(system_prompts): update plugin-dir example to plugins/windmill Co-authored-by: centdix --------- Co-authored-by: Claude Opus 4.7 (1M context) Co-authored-by: claude[bot] <41898282+claude[bot]@users.noreply.github.com> Co-authored-by: centdix * fix(cli): wmill sync pull updates wmill-lock.yaml for raw apps (#9289) * fix: flow recording teardown crash + rename package to @windmill-labs/components (#9288) * fix: guard against null recording during FlowRecordingReplay teardown Navigating away from a flow recording inside a workspace file-tree view threw `TypeError: Cannot read properties of null (reading 'flow')` from FlowGraphViewer once during the teardown tick. Svelte 5 compiles child component props as live getters that close over `$$props.recording.flow`. When `recording` flips to null on the parent's navigation, an outer `{#if !recording?.flow}` doesn't stop those getters from firing one more time as derived effects re-evaluate before the unmount lands — so the getter dereferences null and throws. Fix at the two layers where the deref actually happens: - FlowRecordingReplay: use `recording?.flow` at the binding sites (FlowViewer + graph-snippet FlowGraphViewer) so the compiler emits an optional-chained getter, and guard the snippet branch with `{:else if recording?.flow}` so it doesn't mount when there's nothing to show. - FlowGraphViewer: finish the optional chaining the rest of the file already used everywhere else (`flow?.value?.skip_expr`, `flow?.value?.cache_ttl`, `flow?.schema`). When the upstream binding returns undefined during teardown, the graph degrades to an empty frame instead of crashing. Co-Authored-By: Claude Opus 4.7 (1M context) * chore: rename package to @windmill-labs/components - frontend/package.json: rename `windmill-components` → `@windmill-labs/components` - frontend/publish.sh: drop the in-place sed rename dance; the checked-in name now matches what's published, so `npm run package && npm publish` is enough - frontend/package-lock.json, system_prompts/auto-generated/prompts.d.ts: regenerated by `npm run package` under the new name Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) * default script name * save logic * Keyboard nav * finish keynav * nits * CI fix * nit stop propagation * Merge branch 'main' into feat/asset-graph-view * commit * update * fix: cropped save button on small screens * progress * managed scheduled removed * all * progress * feat: add data upload pipeline trigger with auto S3 picker Co-Authored-By: Claude Opus 4.8 (1M context) * fix: avoid pane editor remount flicker when deploying a pipeline draft Co-Authored-By: Claude Opus 4.8 (1M context) * fix: show only the edited script's I/O in the asset graph, not the saved version's Co-Authored-By: Claude Opus 4.8 (1M context) * feat: derive script asset rows server-side at deploy Co-Authored-By: Claude Fable 5 * test: shared fixture corpus keeps annotation parsers in parity Co-Authored-By: Claude Fable 5 * feat: dev-run draft pipeline chains, live badges, deploy drift warning Co-Authored-By: Claude Fable 5 * refactor: ungate cascade producers, squash pipeline migrations Co-Authored-By: Claude Fable 5 * chore: drop committed cli-sync fixtures and stray screenshots Co-Authored-By: Claude Fable 5 * feat: show skip-asset-dispatch flag as badge instead of args row Co-Authored-By: Claude Fable 5 * feat: pipeline view mode default with activity feed, drafts overlay chip Co-Authored-By: Claude Fable 5 * fix: treat DROP TABLE as table-level write in sql asset parser Co-Authored-By: Claude Fable 5 * feat: wmill datatable create + actionable sql extension error Co-Authored-By: Claude Fable 5 * chore: ephemeral data-pipelines demo sync repo zip for handoff Co-Authored-By: Claude Fable 5 * feat: wmill pipeline list/show renders the asset DAG in the terminal Co-Authored-By: Claude Fable 5 * nits * nits * nits * nits * fix: defer draft persist-back past the batch so discard sticks first click Co-Authored-By: Claude Fable 5 * feat: band-reserving tidy-tree asset graph layout with join breakpoints Co-Authored-By: Claude Fable 5 * fix: route skip-layer and long graph edges around occupied columns Co-Authored-By: Claude Fable 5 * fix: seed s3 template outputs with canonical leading-slash paths Co-Authored-By: Claude Fable 5 * all * feat: bundle data-pipeline drafts into the DB-backed user draft system Pipeline drafts were browser-only (localStorage `pipeline-`), so they didn't sync across devices, weren't server-visible, and never showed in the drafts list. Store them instead as one per-user `draft` row of a new `data_pipeline` kind, keyed at the folder (`f//data_pipeline`), holding the same `{ drafts, activeDraftPath }` bundle. Stage 1 — backend kind: add `data_pipeline` to DRAFT_KIND (migration) and `UserDraftItemKind` (deployed_table=None, private). The list/update handlers and folder-path access check already cover a backing-table-less kind. Stage 2 — sync: add `GET /drafts/get_own/{kind}/{path}` so an editor with no deployed-overlay GET can load its own draft. The pipeline page now hydrates from the DB on mount (one-time localStorage import for in-flight drafts) and persists via UserDraftDbSyncer (debounce + optimistic-concurrency), keeping a localStorage crash mirror. Stage 3 — surface: the drafts review page renders the bundle as a "pipeline" row that opens `/pipeline/` (open-only; excluded from bulk deploy). Verified end-to-end in-browser: DB-seeded draft hydrates to "Edit (1)", edits persist back, and the row shows with Open pipeline / Discard. Co-Authored-By: Claude Opus 4.8 (1M context) * feat: pipeline Activity panel grouping, run↔graph highlight, deploy-conflict handling Activity panel (view mode): - Group cascade runs by the connected component of the asset-dispatch graph (new GET /jobs/asset_dispatch_edges over the dispatch_event table, incl. join_pending inputs), headed by the earliest originating run + its trigger, with a "+N" chip for joins fed by multiple triggers. - Success/failure count histogram with drag-to-filter brushing, an always-on time axis + per-bar tooltips, a Reset, and Last hour/24h/48h/7/30/90d ranges. - Node run-count/status badges now derive from the same merged historic+live events the panel shows (previously session-only). Run ↔ graph highlight: - Hovering a run row (or a group header → the whole cascade) rings the node(s), animates their incident edges, and borders the adjacent assets in the edge hue (blue write / gray read); expanding a run pins a soft-blue ring. - Switching edit→view re-surfaces the Activity feed. Deploy: - Live-content autosave for the open pipeline draft + an autosave indicator. - Re-saving a script now chains off the hash just created instead of a stale parent_hash (fixes the "lineage must be linear" error on a second save), and a genuine concurrent deploy opens a keep-mine / view-latest conflict modal. Co-Authored-By: Claude Opus 4.8 (1M context) * fix: pipeline editor badge requires asset-parse, not just main-function parse A pipeline script's asset lineage is load-bearing — a deploy that can't parse assets silently records no edges. The editor "parsable" dot only reflected inferArgs (the main function), so a body the asset parser rejects (e.g. a trailing `/////` in DuckDB) still showed green and deployed with empty lineage. ScriptEditor gains `requireValidAssets` (set by the pipeline pane); when on, the EditorBar badge is green only if BOTH the main function and inferAssets parse, with the tooltip distinguishing "Main function not parsable" / "Assets not parsable" / "Parsable". Co-Authored-By: Claude Opus 4.8 (1M context) * feat: route asset-graph edges around nodes that sit in their path Edges could draw straight through an unrelated node (a join fan-out or long cross-component edge), making it ambiguous whether that node shared the input. AssetGraphEdge only saw its own endpoints, so it could only detour the near-vertical same-column skip case. The canvas now (once per layout, O(edges × nodes) — no per-frame cost) samples each edge's straight run against every non-incident node center and, on a crossing, passes a clear gutter lane to the edge via `data.detourX`; AssetGraphEdge routes the rounded-orthogonal detour through it. Verified: 0 edge↔node box crossings on the orders pipeline. Co-Authored-By: Claude Opus 4.8 (1M context) * fix: deploy pipeline drafts with freshly-inferred assets, not a stale snapshot "Save all" spread `...draft.script` into createScript, which carries a `assets` snapshot that isn't refreshed when the body is edited. So a renamed/removed output (e.g. an old `CREATE TABLE exciting_en32z9` later changed to `exciting_880909`) was re-deployed as a phantom write edge and lingered as an orphan asset on the graph — shown with no producer, and shifting position on click as the graph re-derived. saveDraft now re-runs inferAssets on the current body and passes the result as `assets`, overriding the snapshot — mirroring the per-pane save. The backend clears+reinserts from the sent set, so a re-deploy drops the stale rows. Verified: deploying with the fresh asset set removes the orphan from the graph. Co-Authored-By: Claude Opus 4.8 (1M context) * fix: collect upstream reads from CTAS and CREATE VIEW in SQL asset parser `CREATE TABLE x AS SELECT … FROM y` (and `CREATE VIEW`) recorded only the write to x — the source read of y was silently dropped. Table-level reads are gathered in the `Statement::Query` arm via handle_table_with_joins; the generic table-factor visitor only picks up read-functions and string literals, not plain `FROM ` references. The AS-query of a CTAS isn't a `Statement::Query`, so its FROM tables were never walked. On the pipeline canvas this meant a `datatable://…` upstream consumed by a CTAS step showed no read node/edge — the step looked like it produced its output from nothing. Factor the Query arm's read collection into handle_query_reads and call it from the CreateTable (when it has an AS-query) and CreateView arms, balancing the cte_name_stack push in post_visit_statement. Updated the drop_then_create test (which had pinned the old drop-the-read behavior) and added CTAS + CREATE VIEW read coverage. Verified against the rebuilt asset wasm: the live editor now infers the read. Co-Authored-By: Claude Opus 4.8 (1M context) * update * updates * refactor: dedup asset-graph code, squash migrations, drop artifacts Co-Authored-By: Claude Opus 4.8 (1M context) * perf: gate asset dispatch on a cached per-workspace producer set Cache the producer-path→writes map per workspace and invalidate it from the asset-clear paths via the notify_event polling system, so a top-level script/preview completion that isn't an asset producer costs an in-memory lookup instead of a per-completion query. Co-Authored-By: Claude Opus 4.8 (1M context) * fix: remove dead unquote fn that failed backend check under -D warnings Co-Authored-By: Claude Opus 4.8 (1M context) * fix: green the frontend check (pin published wasm-asset, fix type errors) Pin windmill-parser-wasm-asset to the published 1.728.1 (was a file: link to a gitignored, CI-unbuilt pkg-asset). Exclude test files from svelte-check (the parity test reads a backend fixture via node:fs, which the browser app tsconfig has no @types/node for; vitest still runs them). Fix pre-existing branch type errors: drop the unsupported 2nd getScriptByPath arg, cast script.schema to Schema for inferArgs, coerce has_preprocessor to a definite boolean, and wrap the cancelJob handler so it isn't possibly-undefined. Co-Authored-By: Claude Opus 4.8 (1M context) * feat: move pipeline partition resolution to ee-private (free-CE) Partition resolution becomes a private module (partition_ee in windmill-ee-private, hidden from the public repo) with an OSS no-op fallback (partition_oss); call sites resolve via the aliased windmill_common::partition. Not enterprise-gated — free to run in CE. Bumps ee-repo-ref to the ee branch carrying partition_ee. Verified building in default, private, and private,enterprise (offline). Co-Authored-By: Claude Opus 4.8 (1M context) * feat: move asset-cascade join/debounce/retry to ee-private (free-CE) Join barrier, debounce, and retry become the private windmill_queue::cascade module (cascade_ee in windmill-ee-private); OSS gets cascade_oss no-op fallbacks (plain OR fan-out). Core cascade stays public. Bumps ee-repo-ref. Verified default/private/private,enterprise. Co-Authored-By: Claude Opus 4.8 (1M context) * feat: skeleton enterprise pipeline freshness + backfill (TODO, ee-private) Gated windmill_common::pipeline_advanced (private; pipeline_advanced_ee) with OSS fallback; entry points return a clear not-implemented error. Deploy surfaces a TODO when a script declares // freshness. Bumps ee-repo-ref. Co-Authored-By: Claude Opus 4.8 (1M context) * fix: repair asset_trigger_dispatch test after cascade carve-out + cache its queries Stage-2 moved reap_stale_join_slots to windmill_queue::cascade; update the integration test's import. Also commit the test's sqlx query cache (was never prepared with --tests, so SQLX_OFFLINE cargo test failed pre-existing). Co-Authored-By: Claude Opus 4.8 (1M context) * test: invalidate producer-cache in asset dispatch tests (mirror deploy) The tests seed asset rows directly and run no notify poller, so the per-workspace producer cache went stale across tests → 0 dispatched. Clear it at the seed point, as a deploy would via notify_event. All 8 asset_trigger_dispatch tests pass. Co-Authored-By: Claude Opus 4.8 (1M context) * chore: update ee-repo-ref to ba677ea142011462ad4dfe77e8375a6dd274cdef This commit updates the EE repository reference after PR #619 was merged in windmill-ee-private. Previous ee-repo-ref: 925c350cff55d3ea738d9e2e4098d9ce4bdda418 New ee-repo-ref: ba677ea142011462ad4dfe77e8375a6dd274cdef Automated by sync-ee-ref workflow. * test: disable producer cache in asset dispatch tests (isolated-DB safe) The .remove(WS) approach still raced: #[sqlx::test] gives each test its own DB but they share one workspace id, so the WS-keyed process-global cache clobbered across DBs under concurrent threads. Add an ASSET_PRODUCER_CACHE_DISABLED test hook and set it in the tests so every dispatch reads its own DB. 8/8 pass at --test-threads=10. Co-Authored-By: Claude Opus 4.8 (1M context) * feat: replace asset-cascade depth cap with cycle detection The hardcoded MAX_CHAIN_DEPTH=5 truncated legitimate deep pipelines (silently — the check returned before event logging). Replace it with per-edge cycle detection: carry the producer lineage in trigger.chain and skip only a subscriber already in the chain, recording a visible cycle_detected dispatch_event. Acyclic pipelines of any depth now cascade fully; a high MAX_CHAIN_LEN backstop guards against runaway. Tests + UI label updated; 8/8 pass at --test-threads=10. Co-Authored-By: Claude Opus 4.8 (1M context) * docs: update dispatch_event reason examples (depth_cap → cycle_detected) Comment-only; the migration is idempotent and already in the potentially_stale self-heal list, so the checksum change re-applies cleanly. Co-Authored-By: Claude Opus 4.8 (1M context) * fix: park cascade retry (P1 dead-end) + clear stale script_triggers on rename Two deploy-path fixes: - Retry is parked: a retried subscriber is wrapped in a SingleStepFlow, whose run is a flow step and ineligible for asset dispatch, so it would silently dead-end the cascade (P1). Stop persisting retry to script_trigger and warn at deploy; TODO(pipeline-retry) to re-enable once dispatch handles flow-wrapped producers. (Dispatch plumbing kept + still tested via direct seeding.) - Rename leaves stale script_trigger rows: clear was keyed on ns.path only, so old-path '// on' edges lingered and could trigger a script later recreated at that path. Also clear the old path on rename (assets already handled via the parent-hash clear). Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) Co-authored-by: rubenfiszel <275584+rubenfiszel@users.noreply.github.com> Co-authored-by: hugocasa Co-authored-by: windmill-internal-app[bot] Co-authored-by: Arnaud <31803803+Araden14@users.noreply.github.com> Co-authored-by: Diego Imbert Co-authored-by: centdix <40307056+centdix@users.noreply.github.com> Co-authored-by: Diego Imbert <70353967+diegoimbert@users.noreply.github.com> Co-authored-by: claude[bot] <41898282+claude[bot]@users.noreply.github.com> Co-authored-by: Aldrin Jenson Co-authored-by: centdix --- .gitignore | 3 +- ...bc7e9ee85289ff5c52af43166928337c257e7.json | 3 +- ...0dad250c59ed0fca6cab7b5be4ee2516275d0.json | 14 + ...0bbc18da7e1ed758cab017fba2598c9467b7f.json | 17 + ...83e8f8e6770b78f6ed052b99afed6cea28650.json | 17 + ...9659c1e6a24d5454a691e31e0179f58c24366.json | 41 + ...f72ece70ec92c5b04baad2622639850445900.json | 97 + ...a8c69a3151266e2bf475da10feb4916e436e9.json | 38 + ...de10b1ca1785b66a68a260d4aaf2957fbcc66.json | 24 + ...cb3adcf0175d0a71013f2497397c1c8ccc619.json | 3 +- ...be15b103cf1344b7c84069e615181508913e9.json | 14 + ...d993e37d8a8bc67cd90682a7971760442b90a.json | 24 + ...6b571ae0f3355cfd5eb95a227a2a98136e02b.json | 22 + ...31e55b702d8e68031219798945e8a8e7f191b.json | 23 + ...c23caf312ec377dd91e5307f00d3fb8ec325d.json | 3 +- ...ab84dcea21e0a38906de164889cf2dacf2e43.json | 23 + ...87562dbdac633f3248ac228e00b7c8b49b800.json | 94 + ...59b785515a79dcda3745663e4c4f2cf328389.json | 48 + ...fbd503d8dea799545e9cb96574d5c6020b90b.json | 15 + ...04af5a7eb488c76f783e690af0c2153b1d6a8.json | 3 +- ...0d8288f1d6adbeb3e545386cf66c33f8db50f.json | 23 + ...41e309d7e066ee4869880c90e90dd66dc9d4d.json | 15 + ...9c3d7b2fb156a43299e9a1209d05293c4f50f.json | 64 + ...a55a9f8fa854b29edcea83710a9170124edf0.json | 64 + ...f7616db640054e804d4ca66958cad06008fdb.json | 27 + ...a8803972c7b3e2b6a2b95c27f15068bed2ca5.json | 22 + ...013649adacf348e3a815e991b4463b1711afc.json | 28 + ...a988159130fc3a8cde91d419a1787b6bb8a51.json | 52 + ...aeebc22883bf1d10dcbce41cccebb392ffd69.json | 16 + ...8b4919c4f1dbf1972d04664a730cefc0594e2.json | 23 + ...c9d7cb5d1022da6b5bf5a17c73c61773b50f4.json | 47 + ...b76e947e491da84fbaf5f130f58fbc851594e.json | 27 + ...3460b6438d0a320b5554acec0beddbdeb008b.json | 74 + ...fbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json | 3 +- ...e33244fe8ab9a803248fc23f334034d24aad4.json | 3 +- ...541df988da527d30f663ef6f764f0c3d70d4b.json | 23 + ...725f973eeb6ba5abbd61416060fb7d675593b.json | 28 + ...bb6a8f961c119516e4c33dc928cd3b4f4edbc.json | 3 +- ...77429c51e6166c414bcc5ce2e97fac25bcd77.json | 3 +- ...9746147aa9b8d0a81642a7b7cb4738a0cad66.json | 3 +- ...9207ccf02da21984730391eeb497f7146ba66.json | 23 + ...5f1f775a8b526382d671c95a2eb8879e1d828.json | 15 + ...15edc8e637d44031b83c01ef09a96ade472ae.json | 24 + ...7bb759229805cff771cfa4275a46bfc80e1ab.json | 15 + ...e1ea2fe96a1b94f5a92953fb1c1642d15c016.json | 3 +- ...120e5b0f629f9f04e08a2982df33ff23ac7a9.json | 3 +- ...10c923df7b66d1b76d36229299d2866775ffc.json | 34 + ...e6080e3c4faae5052500b93cc24b6fe556f2b.json | 43 + ...9352d4d1e3cb3a8a08f1c9c03055a1cab1235.json | 3 +- ...d62eef3ab3b920665ca5d6f9ce0ef89e53fb3.json | 22 + ...4e8dd1370fb59c0a5cfd2403c5fdaf2135cf2.json | 51 + backend/Cargo.lock | 5 + backend/ee-repo-ref.txt | 2 +- .../20260423050000_script_trigger.down.sql | 3 + .../20260423050000_script_trigger.up.sql | 58 + ...0510174213_asset_trigger_dispatch.down.sql | 3 + ...260510174213_asset_trigger_dispatch.up.sql | 5 + ...0260516194247_join_pending_inputs.down.sql | 1 + .../20260516194247_join_pending_inputs.up.sql | 19 + .../20260523055641_dispatch_event.down.sql | 2 + .../20260523055641_dispatch_event.up.sql | 63 + ...0412_add_data_pipeline_draft_kind.down.sql | 2 + ...090412_add_data_pipeline_draft_kind.up.sql | 6 + .../windmill-parser-py-asset/src/lib.rs | 11 +- .../src/asset_parser.rs | 173 +- .../windmill-parser-ts-asset/src/lib.rs | 14 +- .../parsers/windmill-parser-wasm/Cargo.lock | 149 +- .../windmill-parser-yaml/src/asset_parser.rs | 10 +- .../windmill-parser/src/asset_parser.rs | 966 ++++++- .../tests/fixtures/pipeline_annotations.json | 238 ++ .../tests/pipeline_annotations_parity.rs | 157 + backend/src/main.rs | 7 + backend/src/monitor.rs | 4 + backend/tests/asset_trigger_dispatch.rs | 981 +++++++ backend/tests/python_jobs.rs | 2 + backend/tests/script_modules.rs | 3 + backend/tests/worker.rs | 1 + backend/windmill-api-assets/src/lib.rs | 446 ++- backend/windmill-api-scripts/Cargo.toml | 7 +- .../src/asset_inference.rs | 254 ++ backend/windmill-api-scripts/src/lib.rs | 1 + backend/windmill-api-scripts/src/scripts.rs | 138 +- backend/windmill-api/openapi.yaml | 315 ++ backend/windmill-api/src/db.rs | 6 + backend/windmill-api/src/drafts.rs | 32 + backend/windmill-api/src/jobs.rs | 194 ++ backend/windmill-common/src/assets.rs | 194 +- backend/windmill-common/src/lib.rs | 14 + backend/windmill-common/src/partition_oss.rs | 55 + .../src/pipeline_advanced_oss.rs | 16 + backend/windmill-common/src/runtime_assets.rs | 21 +- backend/windmill-common/src/user_drafts.rs | 11 +- backend/windmill-object-store/src/lib.rs | 129 +- backend/windmill-queue/src/asset_dispatch.rs | 736 +++++ backend/windmill-queue/src/cascade_oss.rs | 60 + backend/windmill-queue/src/lib.rs | 8 + backend/windmill-types/src/assets.rs | 17 + backend/windmill-types/src/jobs.rs | 4 + .../windmill-worker/src/result_processor.rs | 27 +- backend/windmill-worker/src/worker.rs | 115 + cli/src/commands/datatable/datatable.ts | 69 + cli/src/commands/pipeline/pipeline.ts | 287 ++ cli/src/guidance/skills.gen.ts | 14 + cli/src/main.ts | 3 + cli/src/utils/script_common.ts | 10 +- docs/pipelines-vs-dbt.md | 374 +++ frontend/package-lock.json | 63 +- frontend/package.json | 2 +- frontend/src/lib/attachments/arrowTabNav.ts | 59 + .../src/lib/attachments/selectAndAdvanceTo.ts | 49 + .../src/lib/components/CompareDrafts.svelte | 84 +- frontend/src/lib/components/EditorBar.svelte | 15 +- frontend/src/lib/components/JobArgs.svelte | 43 +- frontend/src/lib/components/MoveDrawer.svelte | 73 + .../src/lib/components/QueuePosition.svelte | 38 +- .../lib/components/S3FilePickerInner.svelte | 106 +- .../src/lib/components/S3FilePreview.svelte | 301 ++ .../src/lib/components/ScriptEditor.svelte | 775 +++-- .../AssetGraph/ActivityHistogram.svelte | 155 + .../assets/AssetGraph/AddNode.svelte | 139 + .../assets/AssetGraph/AssetGraphCanvas.svelte | 873 ++++++ .../AssetGraph/AssetGraphDetailsPane.svelte | 1277 +++++++++ .../assets/AssetGraph/AssetGraphEdge.svelte | 100 + .../assets/AssetGraph/AssetNode.svelte | 220 ++ .../assets/AssetGraph/AssetRunsPanel.svelte | 430 +++ .../assets/AssetGraph/DataTablePreview.svelte | 212 ++ .../assets/AssetGraph/PanToNode.svelte | 39 + .../AssetGraph/PipelineActivityPanel.svelte | 579 ++++ .../AssetGraph/PipelineAlphaAckModal.svelte | 38 + .../assets/AssetGraph/PipelineEventLog.svelte | 120 + .../AssetGraph/PipelineInsertMenu.svelte | 301 ++ .../AssetGraph/PipelineModeToggle.svelte | 46 + .../AssetGraph/PipelinePickerModal.svelte | 109 + .../AssetGraph/PipelineScriptView.svelte | 110 + .../AssetGraph/PipelineTriggerEditors.svelte | 202 ++ .../assets/AssetGraph/RunnableNode.svelte | 363 +++ .../assets/AssetGraph/TriggerNode.svelte | 387 +++ .../AssetGraph/activeRunnables.svelte.ts | 402 +++ .../AssetGraph/assetGraphLayout.test.ts | 120 + .../assets/AssetGraph/assetGraphLayout.ts | 323 +++ .../AssetGraph/cascadeOrchestrator.test.ts | 163 ++ .../assets/AssetGraph/cascadeOrchestrator.ts | 114 + .../assets/AssetGraph/deployGraphDiff.test.ts | 83 + .../assets/AssetGraph/deployGraphDiff.ts | 85 + .../assets/AssetGraph/graphTraversal.test.ts | 172 ++ .../assets/AssetGraph/graphTraversal.ts | 119 + .../lib/components/assets/AssetGraph/lib.ts | 35 + .../parsePipelineAnnotations.parity.test.ts | 131 + .../parsePipelineAnnotations.test.ts | 101 + .../AssetGraph/parsePipelineAnnotations.ts | 310 ++ .../AssetGraph/pipelineHistory.svelte.ts | 147 + .../assets/AssetGraph/pipelineLanguages.ts | 21 + .../assets/AssetGraph/pipelineTemplates.ts | 686 +++++ .../assets/AssetGraph/resolveGraph.test.ts | 467 +++ .../assets/AssetGraph/resolveGraph.ts | 483 ++++ .../lib/components/assets/AssetGraph/types.ts | 121 + frontend/src/lib/components/assets/lib.ts | 21 + .../lib/components/common/table/Row.svelte | 1 + .../components/common/table/RowIcon.svelte | 6 +- frontend/src/lib/components/custom_ui.ts | 31 + .../components/flows/CreateActionsApp.svelte | 66 +- .../components/flows/CreateActionsFlow.svelte | 33 +- .../src/lib/components/home/ItemsList.svelte | 34 + .../src/lib/components/home/TreeView.svelte | 34 +- .../lib/components/home/TreeViewRoot.svelte | 5 +- .../runs/DispatchEventsButton.svelte | 46 + .../runs/DispatchEventsPanel.svelte | 23 + .../runs/DispatchEventsTable.svelte | 98 + .../runs/useDispatchEvents.svelte.ts | 24 + .../components/scriptEditor/LogPanel.svelte | 16 +- .../PipelineLockedRunnableInfo.svelte | 26 + .../triggers/TriggerRunnablePicker.svelte | 59 + .../triggers/email/EmailTriggerEditor.svelte | 4 +- .../email/EmailTriggerEditorInner.svelte | 42 +- .../triggers/gcp/GcpTriggerEditor.svelte | 4 +- .../triggers/gcp/GcpTriggerEditorInner.svelte | 65 +- .../triggers/kafka/KafkaTriggerEditor.svelte | 4 +- .../kafka/KafkaTriggerEditorInner.svelte | 58 +- .../triggers/mqtt/MqttTriggerEditor.svelte | 4 +- .../mqtt/MqttTriggerEditorInner.svelte | 68 +- .../triggers/nats/NatsTriggerEditor.svelte | 4 +- .../nats/NatsTriggerEditorInner.svelte | 60 +- .../postgres/PostgresTriggerEditor.svelte | 4 +- .../PostgresTriggerEditorInner.svelte | 76 +- .../triggers/schedules/ScheduleEditor.svelte | 9 +- .../schedules/ScheduleEditorInner.svelte | 19 +- .../triggers/sqs/SqsTriggerEditor.svelte | 4 +- .../triggers/sqs/SqsTriggerEditorInner.svelte | 69 +- frontend/src/lib/components/triggers/utils.ts | 3 +- .../triggers/webhook/WebhookEditor.svelte | 66 + frontend/src/lib/forLater.ts | 4 +- frontend/src/lib/userDraft.svelte.ts | 3 +- frontend/src/lib/userDraftDbMigration.ts | 3 +- frontend/src/lib/utils_deployable.ts | 3 + .../(root)/(logged)/assets/+page.svelte | 16 +- .../routes/(root)/(logged)/pipeline/+page.js | 5 + .../(root)/(logged)/pipeline/+page.svelte | 72 + .../(logged)/pipeline/[folder]/+page.js | 5 + .../(logged)/pipeline/[folder]/+page.svelte | 2544 +++++++++++++++++ .../(root)/(logged)/run/[...run]/+page.svelte | 4 + frontend/tsconfig.json | 2 +- .../auto-generated/cli/cli-commands.md | 14 + system_prompts/auto-generated/prompts.ts | 14 + .../skills/cli-commands/SKILL.md | 14 + 204 files changed, 22072 insertions(+), 860 deletions(-) create mode 100644 backend/.sqlx/query-09095af7cad650fb10781d9e39b0dad250c59ed0fca6cab7b5be4ee2516275d0.json create mode 100644 backend/.sqlx/query-0e7fe0e1d7aa2072a3431d081080bbc18da7e1ed758cab017fba2598c9467b7f.json create mode 100644 backend/.sqlx/query-1fc04d31ae69dbb1df9c63cb69e83e8f8e6770b78f6ed052b99afed6cea28650.json create mode 100644 backend/.sqlx/query-2484323d94f249be30f4472ece89659c1e6a24d5454a691e31e0179f58c24366.json create mode 100644 backend/.sqlx/query-26e63135fcd8e7d48e25de190a2f72ece70ec92c5b04baad2622639850445900.json create mode 100644 backend/.sqlx/query-31bbd03932912df069cfc97fd1ca8c69a3151266e2bf475da10feb4916e436e9.json create mode 100644 backend/.sqlx/query-3c5a387c2fed905838b0c1d2e0ade10b1ca1785b66a68a260d4aaf2957fbcc66.json create mode 100644 backend/.sqlx/query-3c8a2389c47131ed89ec9069b2ebe15b103cf1344b7c84069e615181508913e9.json create mode 100644 backend/.sqlx/query-3eb137e83c0aa6389b2893d59acd993e37d8a8bc67cd90682a7971760442b90a.json create mode 100644 backend/.sqlx/query-45b6c748090a0a6bf71a995413b6b571ae0f3355cfd5eb95a227a2a98136e02b.json create mode 100644 backend/.sqlx/query-478e2ccf318be5beca35518785531e55b702d8e68031219798945e8a8e7f191b.json create mode 100644 backend/.sqlx/query-58e8e13acd9f7ff951f37d555beab84dcea21e0a38906de164889cf2dacf2e43.json create mode 100644 backend/.sqlx/query-5ae004333c20e6f7c28025f16ad87562dbdac633f3248ac228e00b7c8b49b800.json create mode 100644 backend/.sqlx/query-5c609ea0696df96ca02cba7fee359b785515a79dcda3745663e4c4f2cf328389.json create mode 100644 backend/.sqlx/query-651fc12e1b971d4fd57c98a7a7efbd503d8dea799545e9cb96574d5c6020b90b.json create mode 100644 backend/.sqlx/query-6814542fcdd01a178798ad7b0840d8288f1d6adbeb3e545386cf66c33f8db50f.json create mode 100644 backend/.sqlx/query-6a1998cb3a9a0898c0fd21c35cc41e309d7e066ee4869880c90e90dd66dc9d4d.json create mode 100644 backend/.sqlx/query-6aaddd80f8c07cfafea2021c1879c3d7b2fb156a43299e9a1209d05293c4f50f.json create mode 100644 backend/.sqlx/query-6b9348e60cc1ce158314a93fc7aa55a9f8fa854b29edcea83710a9170124edf0.json create mode 100644 backend/.sqlx/query-7300eb89029e0863241087fc10df7616db640054e804d4ca66958cad06008fdb.json create mode 100644 backend/.sqlx/query-751f836dc8f78c330387456dd68a8803972c7b3e2b6a2b95c27f15068bed2ca5.json create mode 100644 backend/.sqlx/query-754b98335e8776565d63267b395013649adacf348e3a815e991b4463b1711afc.json create mode 100644 backend/.sqlx/query-77424d40104cf271e5ee5118100a988159130fc3a8cde91d419a1787b6bb8a51.json create mode 100644 backend/.sqlx/query-80f2d2f20e93b5e05ecd1fe5afeaeebc22883bf1d10dcbce41cccebb392ffd69.json create mode 100644 backend/.sqlx/query-82dcaf94ffe43da1c8c7de2a3478b4919c4f1dbf1972d04664a730cefc0594e2.json create mode 100644 backend/.sqlx/query-8bb2f6f4526231c1ce57182a779c9d7cb5d1022da6b5bf5a17c73c61773b50f4.json create mode 100644 backend/.sqlx/query-8bce3f969b4bbbcf3dc1b8c671ab76e947e491da84fbaf5f130f58fbc851594e.json create mode 100644 backend/.sqlx/query-8cd02a5378bea03012e6cb937f23460b6438d0a320b5554acec0beddbdeb008b.json create mode 100644 backend/.sqlx/query-a9a99880d870266f474878dd6ef541df988da527d30f663ef6f764f0c3d70d4b.json create mode 100644 backend/.sqlx/query-abb36bfddf707c7897b3936e982725f973eeb6ba5abbd61416060fb7d675593b.json create mode 100644 backend/.sqlx/query-c0952a5f8a3959630f124f1bf379207ccf02da21984730391eeb497f7146ba66.json create mode 100644 backend/.sqlx/query-c2c4e95ef48bf05e971b0e8d8de5f1f775a8b526382d671c95a2eb8879e1d828.json create mode 100644 backend/.sqlx/query-c608cdc9cd2e41992fb86431faa15edc8e637d44031b83c01ef09a96ade472ae.json create mode 100644 backend/.sqlx/query-d3d87b9a4d62977dea5af95dd457bb759229805cff771cfa4275a46bfc80e1ab.json create mode 100644 backend/.sqlx/query-d6a8ab57341b8aa17d4d50ce7a710c923df7b66d1b76d36229299d2866775ffc.json create mode 100644 backend/.sqlx/query-de06f44bad94710f14e9be4c0a6e6080e3c4faae5052500b93cc24b6fe556f2b.json create mode 100644 backend/.sqlx/query-ecce519d0cf0c31df4612e0ccd8d62eef3ab3b920665ca5d6f9ce0ef89e53fb3.json create mode 100644 backend/.sqlx/query-f338943aa3595c2893a0d42e4a54e8dd1370fb59c0a5cfd2403c5fdaf2135cf2.json create mode 100644 backend/migrations/20260423050000_script_trigger.down.sql create mode 100644 backend/migrations/20260423050000_script_trigger.up.sql create mode 100644 backend/migrations/20260510174213_asset_trigger_dispatch.down.sql create mode 100644 backend/migrations/20260510174213_asset_trigger_dispatch.up.sql create mode 100644 backend/migrations/20260516194247_join_pending_inputs.down.sql create mode 100644 backend/migrations/20260516194247_join_pending_inputs.up.sql create mode 100644 backend/migrations/20260523055641_dispatch_event.down.sql create mode 100644 backend/migrations/20260523055641_dispatch_event.up.sql create mode 100644 backend/migrations/20260616090412_add_data_pipeline_draft_kind.down.sql create mode 100644 backend/migrations/20260616090412_add_data_pipeline_draft_kind.up.sql create mode 100644 backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json create mode 100644 backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs create mode 100644 backend/tests/asset_trigger_dispatch.rs create mode 100644 backend/windmill-api-scripts/src/asset_inference.rs create mode 100644 backend/windmill-common/src/partition_oss.rs create mode 100644 backend/windmill-common/src/pipeline_advanced_oss.rs create mode 100644 backend/windmill-queue/src/asset_dispatch.rs create mode 100644 backend/windmill-queue/src/cascade_oss.rs create mode 100644 cli/src/commands/pipeline/pipeline.ts create mode 100644 docs/pipelines-vs-dbt.md create mode 100644 frontend/src/lib/attachments/arrowTabNav.ts create mode 100644 frontend/src/lib/attachments/selectAndAdvanceTo.ts create mode 100644 frontend/src/lib/components/S3FilePreview.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/ActivityHistogram.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/AddNode.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/AssetGraphCanvas.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/AssetGraphDetailsPane.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/AssetGraphEdge.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/AssetNode.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/AssetRunsPanel.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/DataTablePreview.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/PanToNode.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/PipelineActivityPanel.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/PipelineAlphaAckModal.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/PipelineEventLog.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/PipelineInsertMenu.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/PipelineModeToggle.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/PipelinePickerModal.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/PipelineScriptView.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/PipelineTriggerEditors.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/RunnableNode.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/TriggerNode.svelte create mode 100644 frontend/src/lib/components/assets/AssetGraph/activeRunnables.svelte.ts create mode 100644 frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.test.ts create mode 100644 frontend/src/lib/components/assets/AssetGraph/assetGraphLayout.ts create mode 100644 frontend/src/lib/components/assets/AssetGraph/cascadeOrchestrator.test.ts create mode 100644 frontend/src/lib/components/assets/AssetGraph/cascadeOrchestrator.ts create mode 100644 frontend/src/lib/components/assets/AssetGraph/deployGraphDiff.test.ts create mode 100644 frontend/src/lib/components/assets/AssetGraph/deployGraphDiff.ts create mode 100644 frontend/src/lib/components/assets/AssetGraph/graphTraversal.test.ts create mode 100644 frontend/src/lib/components/assets/AssetGraph/graphTraversal.ts create mode 100644 frontend/src/lib/components/assets/AssetGraph/lib.ts create mode 100644 frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.parity.test.ts create mode 100644 frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.test.ts create mode 100644 frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts create mode 100644 frontend/src/lib/components/assets/AssetGraph/pipelineHistory.svelte.ts create mode 100644 frontend/src/lib/components/assets/AssetGraph/pipelineLanguages.ts create mode 100644 frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts create mode 100644 frontend/src/lib/components/assets/AssetGraph/resolveGraph.test.ts create mode 100644 frontend/src/lib/components/assets/AssetGraph/resolveGraph.ts create mode 100644 frontend/src/lib/components/assets/AssetGraph/types.ts create mode 100644 frontend/src/lib/components/runs/DispatchEventsButton.svelte create mode 100644 frontend/src/lib/components/runs/DispatchEventsPanel.svelte create mode 100644 frontend/src/lib/components/runs/DispatchEventsTable.svelte create mode 100644 frontend/src/lib/components/runs/useDispatchEvents.svelte.ts create mode 100644 frontend/src/lib/components/triggers/PipelineLockedRunnableInfo.svelte create mode 100644 frontend/src/lib/components/triggers/TriggerRunnablePicker.svelte create mode 100644 frontend/src/lib/components/triggers/webhook/WebhookEditor.svelte create mode 100644 frontend/src/routes/(root)/(logged)/pipeline/+page.js create mode 100644 frontend/src/routes/(root)/(logged)/pipeline/+page.svelte create mode 100644 frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.js create mode 100644 frontend/src/routes/(root)/(logged)/pipeline/[folder]/+page.svelte diff --git a/.gitignore b/.gitignore index 5f733611de..e80b4f32ca 100644 --- a/.gitignore +++ b/.gitignore @@ -33,4 +33,5 @@ backend/chrome_profiler.json .fast-check/ __pycache__/ .playwright-mcp/ -.codex \ No newline at end of file +.codex +.claude/scheduled_tasks.lock diff --git a/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json b/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json index 79ef0c0a81..a779aa0e95 100644 --- a/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json +++ b/backend/.sqlx/query-089d7bc7acdbb97cf477159e111bc7e9ee85289ff5c52af43166928337c257e7.json @@ -34,7 +34,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } diff --git a/backend/.sqlx/query-09095af7cad650fb10781d9e39b0dad250c59ed0fca6cab7b5be4ee2516275d0.json b/backend/.sqlx/query-09095af7cad650fb10781d9e39b0dad250c59ed0fca6cab7b5be4ee2516275d0.json new file mode 100644 index 0000000000..d114d21bf0 --- /dev/null +++ b/backend/.sqlx/query-09095af7cad650fb10781d9e39b0dad250c59ed0fca6cab7b5be4ee2516275d0.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM v2_job WHERE workspace_id = $1 AND trigger_kind = 'asset'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "09095af7cad650fb10781d9e39b0dad250c59ed0fca6cab7b5be4ee2516275d0" +} diff --git a/backend/.sqlx/query-0e7fe0e1d7aa2072a3431d081080bbc18da7e1ed758cab017fba2598c9467b7f.json b/backend/.sqlx/query-0e7fe0e1d7aa2072a3431d081080bbc18da7e1ed758cab017fba2598c9467b7f.json new file mode 100644 index 0000000000..e1386cf4d6 --- /dev/null +++ b/backend/.sqlx/query-0e7fe0e1d7aa2072a3431d081080bbc18da7e1ed758cab017fba2598c9467b7f.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO join_pending_inputs\n (workspace_id, subscriber_path, partition, trigger_ref)\n VALUES ($1, $2, $3, $4)\n ON CONFLICT DO NOTHING", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "0e7fe0e1d7aa2072a3431d081080bbc18da7e1ed758cab017fba2598c9467b7f" +} diff --git a/backend/.sqlx/query-1fc04d31ae69dbb1df9c63cb69e83e8f8e6770b78f6ed052b99afed6cea28650.json b/backend/.sqlx/query-1fc04d31ae69dbb1df9c63cb69e83e8f8e6770b78f6ed052b99afed6cea28650.json new file mode 100644 index 0000000000..9d727dfc6c --- /dev/null +++ b/backend/.sqlx/query-1fc04d31ae69dbb1df9c63cb69e83e8f8e6770b78f6ed052b99afed6cea28650.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO v2_job (id, workspace_id, kind, runnable_path, args, created_by,\n permissioned_as, permissioned_as_email, tag, script_lang)\n VALUES ($1, $2, 'script'::job_kind, $3, $4, 'test-user',\n 'u/test-user', 'test@windmill.dev', 'deno', 'bash'::script_lang)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Varchar", + "Varchar", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "1fc04d31ae69dbb1df9c63cb69e83e8f8e6770b78f6ed052b99afed6cea28650" +} diff --git a/backend/.sqlx/query-2484323d94f249be30f4472ece89659c1e6a24d5454a691e31e0179f58c24366.json b/backend/.sqlx/query-2484323d94f249be30f4472ece89659c1e6a24d5454a691e31e0179f58c24366.json new file mode 100644 index 0000000000..e350cc8233 --- /dev/null +++ b/backend/.sqlx/query-2484323d94f249be30f4472ece89659c1e6a24d5454a691e31e0179f58c24366.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT kind, path, script_path, is_flow FROM (\n SELECT 'schedule' AS kind, path, script_path, is_flow FROM schedule\n WHERE workspace_id = $1\n AND script_path IS NOT NULL\n UNION ALL\n SELECT 'email', path, script_path, is_flow FROM email_trigger\n WHERE workspace_id = $1\n UNION ALL\n SELECT 'kafka', path, script_path, is_flow FROM kafka_trigger\n WHERE workspace_id = $1\n UNION ALL\n SELECT 'mqtt', path, script_path, is_flow FROM mqtt_trigger\n WHERE workspace_id = $1\n UNION ALL\n SELECT 'nats', path, script_path, is_flow FROM nats_trigger\n WHERE workspace_id = $1\n UNION ALL\n SELECT 'postgres', path, script_path, is_flow FROM postgres_trigger\n WHERE workspace_id = $1\n UNION ALL\n SELECT 'sqs', path, script_path, is_flow FROM sqs_trigger\n WHERE workspace_id = $1\n UNION ALL\n SELECT 'gcp', path, script_path, is_flow FROM gcp_trigger\n WHERE workspace_id = $1\n ) t\n WHERE ($2::text IS NULL OR script_path LIKE $2)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "kind", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "is_flow", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null, + null, + null, + null + ] + }, + "hash": "2484323d94f249be30f4472ece89659c1e6a24d5454a691e31e0179f58c24366" +} diff --git a/backend/.sqlx/query-26e63135fcd8e7d48e25de190a2f72ece70ec92c5b04baad2622639850445900.json b/backend/.sqlx/query-26e63135fcd8e7d48e25de190a2f72ece70ec92c5b04baad2622639850445900.json new file mode 100644 index 0000000000..02a419de06 --- /dev/null +++ b/backend/.sqlx/query-26e63135fcd8e7d48e25de190a2f72ece70ec92c5b04baad2622639850445900.json @@ -0,0 +1,97 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n subscriber_path AS \"subscriber_path!\",\n asset_kind AS \"asset_kind!: windmill_common::assets::AssetKind\",\n asset_path AS \"asset_path!\",\n outcome::text AS \"outcome!\",\n child_job_id,\n partition,\n received_inputs,\n required_inputs,\n debounce_s,\n reason,\n created_at AS \"created_at!\"\n FROM dispatch_event\n WHERE producer_job_id = $1 AND workspace_id = $2\n ORDER BY id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "subscriber_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "asset_kind!: windmill_common::assets::AssetKind", + "type_info": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + } + }, + { + "ordinal": 2, + "name": "asset_path!", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "outcome!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "child_job_id", + "type_info": "Uuid" + }, + { + "ordinal": 5, + "name": "partition", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "received_inputs", + "type_info": "Int4" + }, + { + "ordinal": 7, + "name": "required_inputs", + "type_info": "Int4" + }, + { + "ordinal": 8, + "name": "debounce_s", + "type_info": "Int4" + }, + { + "ordinal": 9, + "name": "reason", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "created_at!", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + null, + true, + true, + true, + true, + true, + true, + false + ] + }, + "hash": "26e63135fcd8e7d48e25de190a2f72ece70ec92c5b04baad2622639850445900" +} diff --git a/backend/.sqlx/query-31bbd03932912df069cfc97fd1ca8c69a3151266e2bf475da10feb4916e436e9.json b/backend/.sqlx/query-31bbd03932912df069cfc97fd1ca8c69a3151266e2bf475da10feb4916e436e9.json new file mode 100644 index 0000000000..ded7b65a83 --- /dev/null +++ b/backend/.sqlx/query-31bbd03932912df069cfc97fd1ca8c69a3151266e2bf475da10feb4916e436e9.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM asset\n WHERE id IN (\n SELECT id FROM (\n SELECT a.id, ROW_NUMBER() OVER (\n PARTITION BY a.workspace_id, a.path, a.kind\n ORDER BY a.created_at DESC\n ) as rn,\n limits.max_n\n FROM asset a\n INNER JOIN (\n SELECT * FROM UNNEST(\n $1::varchar[],\n $2::varchar[],\n $3::asset_kind[],\n $4::int[]\n ) AS t(workspace_id, path, kind, max_n)\n ) limits\n ON a.workspace_id = limits.workspace_id\n AND a.path = limits.path\n AND a.kind = limits.kind\n WHERE a.usage_kind = 'job'\n ) ranked\n WHERE rn > max_n\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "VarcharArray", + "VarcharArray", + { + "Custom": { + "name": "asset_kind[]", + "kind": { + "Array": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + } + } + } + }, + "Int4Array" + ] + }, + "nullable": [] + }, + "hash": "31bbd03932912df069cfc97fd1ca8c69a3151266e2bf475da10feb4916e436e9" +} diff --git a/backend/.sqlx/query-3c5a387c2fed905838b0c1d2e0ade10b1ca1785b66a68a260d4aaf2957fbcc66.json b/backend/.sqlx/query-3c5a387c2fed905838b0c1d2e0ade10b1ca1785b66a68a260d4aaf2957fbcc66.json new file mode 100644 index 0000000000..71770359cc --- /dev/null +++ b/backend/.sqlx/query-3c5a387c2fed905838b0c1d2e0ade10b1ca1785b66a68a260d4aaf2957fbcc66.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(DISTINCT trigger_ref) AS \"n!\"\n FROM join_pending_inputs\n WHERE workspace_id = $1 AND subscriber_path = $2 AND partition = $3", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "n!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "3c5a387c2fed905838b0c1d2e0ade10b1ca1785b66a68a260d4aaf2957fbcc66" +} diff --git a/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json b/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json index 20336885a6..3568d1723e 100644 --- a/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json +++ b/backend/.sqlx/query-3c84781704b84b8a927ecce5a3fcb3adcf0175d0a71013f2497397c1c8ccc619.json @@ -127,7 +127,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } diff --git a/backend/.sqlx/query-3c8a2389c47131ed89ec9069b2ebe15b103cf1344b7c84069e615181508913e9.json b/backend/.sqlx/query-3c8a2389c47131ed89ec9069b2ebe15b103cf1344b7c84069e615181508913e9.json new file mode 100644 index 0000000000..650876cc23 --- /dev/null +++ b/backend/.sqlx/query-3c8a2389c47131ed89ec9069b2ebe15b103cf1344b7c84069e615181508913e9.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM join_pending_inputs jpi\n USING (\n SELECT workspace_id, subscriber_path, partition\n FROM join_pending_inputs\n GROUP BY workspace_id, subscriber_path, partition\n HAVING max(received_at) <= now() - ($1::bigint::text || ' s')::interval\n ) stale\n WHERE jpi.workspace_id = stale.workspace_id\n AND jpi.subscriber_path = stale.subscriber_path\n AND jpi.partition = stale.partition", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "3c8a2389c47131ed89ec9069b2ebe15b103cf1344b7c84069e615181508913e9" +} diff --git a/backend/.sqlx/query-3eb137e83c0aa6389b2893d59acd993e37d8a8bc67cd90682a7971760442b90a.json b/backend/.sqlx/query-3eb137e83c0aa6389b2893d59acd993e37d8a8bc67cd90682a7971760442b90a.json new file mode 100644 index 0000000000..9b65eb8a18 --- /dev/null +++ b/backend/.sqlx/query-3eb137e83c0aa6389b2893d59acd993e37d8a8bc67cd90682a7971760442b90a.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(DISTINCT trigger_ref) AS \"n!\"\n FROM script_trigger\n WHERE workspace_id = $1\n AND runnable_path = $2\n AND trigger_kind = 'asset'\n AND runnable_kind = 'script'\n AND trigger_ref LIKE '%' || $3 || '%'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "n!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "3eb137e83c0aa6389b2893d59acd993e37d8a8bc67cd90682a7971760442b90a" +} diff --git a/backend/.sqlx/query-45b6c748090a0a6bf71a995413b6b571ae0f3355cfd5eb95a227a2a98136e02b.json b/backend/.sqlx/query-45b6c748090a0a6bf71a995413b6b571ae0f3355cfd5eb95a227a2a98136e02b.json new file mode 100644 index 0000000000..0d9e4e859d --- /dev/null +++ b/backend/.sqlx/query-45b6c748090a0a6bf71a995413b6b571ae0f3355cfd5eb95a227a2a98136e02b.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path AS \"path!\" FROM flow WHERE workspace_id = $1 AND archived = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "45b6c748090a0a6bf71a995413b6b571ae0f3355cfd5eb95a227a2a98136e02b" +} diff --git a/backend/.sqlx/query-478e2ccf318be5beca35518785531e55b702d8e68031219798945e8a8e7f191b.json b/backend/.sqlx/query-478e2ccf318be5beca35518785531e55b702d8e68031219798945e8a8e7f191b.json new file mode 100644 index 0000000000..f6c2f22d39 --- /dev/null +++ b/backend/.sqlx/query-478e2ccf318be5beca35518785531e55b702d8e68031219798945e8a8e7f191b.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) AS \"n!\" FROM join_pending_inputs\n WHERE workspace_id = $1 AND subscriber_path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "n!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "478e2ccf318be5beca35518785531e55b702d8e68031219798945e8a8e7f191b" +} diff --git a/backend/.sqlx/query-4a43d4df6c5b2e8dda4308dcb88c23caf312ec377dd91e5307f00d3fb8ec325d.json b/backend/.sqlx/query-4a43d4df6c5b2e8dda4308dcb88c23caf312ec377dd91e5307f00d3fb8ec325d.json index 2a6930755a..7d950f6d8f 100644 --- a/backend/.sqlx/query-4a43d4df6c5b2e8dda4308dcb88c23caf312ec377dd91e5307f00d3fb8ec325d.json +++ b/backend/.sqlx/query-4a43d4df6c5b2e8dda4308dcb88c23caf312ec377dd91e5307f00d3fb8ec325d.json @@ -79,7 +79,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } diff --git a/backend/.sqlx/query-58e8e13acd9f7ff951f37d555beab84dcea21e0a38906de164889cf2dacf2e43.json b/backend/.sqlx/query-58e8e13acd9f7ff951f37d555beab84dcea21e0a38906de164889cf2dacf2e43.json new file mode 100644 index 0000000000..3a09f1ee8d --- /dev/null +++ b/backend/.sqlx/query-58e8e13acd9f7ff951f37d555beab84dcea21e0a38906de164889cf2dacf2e43.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT args AS \"args!: Json>>\"\n FROM v2_job\n WHERE workspace_id = $1 AND id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "args!: Json>>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Uuid" + ] + }, + "nullable": [ + true + ] + }, + "hash": "58e8e13acd9f7ff951f37d555beab84dcea21e0a38906de164889cf2dacf2e43" +} diff --git a/backend/.sqlx/query-5ae004333c20e6f7c28025f16ad87562dbdac633f3248ac228e00b7c8b49b800.json b/backend/.sqlx/query-5ae004333c20e6f7c28025f16ad87562dbdac633f3248ac228e00b7c8b49b800.json new file mode 100644 index 0000000000..c126b6371a --- /dev/null +++ b/backend/.sqlx/query-5ae004333c20e6f7c28025f16ad87562dbdac633f3248ac228e00b7c8b49b800.json @@ -0,0 +1,94 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n asset.kind AS \"asset_kind!: AssetKind\",\n asset.path AS \"asset_path!\",\n asset.usage_kind AS \"usage_kind!: AssetUsageKind\",\n asset.usage_path AS \"usage_path!\",\n asset.usage_access_type::text AS \"access_type\"\n FROM asset\n WHERE asset.workspace_id = $1\n AND asset.usage_kind IN ('script', 'flow')\n AND ($2::asset_kind[] IS NULL OR asset.kind = ANY($2))\n AND ($3::text IS NULL OR asset.usage_path LIKE $3)\n GROUP BY asset.kind, asset.path, asset.usage_kind, asset.usage_path, asset.usage_access_type\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "asset_kind!: AssetKind", + "type_info": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + } + }, + { + "ordinal": 1, + "name": "asset_path!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "usage_kind!: AssetUsageKind", + "type_info": { + "Custom": { + "name": "asset_usage_kind", + "kind": { + "Enum": [ + "script", + "flow", + "job" + ] + } + } + } + }, + { + "ordinal": 3, + "name": "usage_path!", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "access_type", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + { + "Custom": { + "name": "asset_kind[]", + "kind": { + "Array": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + } + } + } + }, + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + null + ] + }, + "hash": "5ae004333c20e6f7c28025f16ad87562dbdac633f3248ac228e00b7c8b49b800" +} diff --git a/backend/.sqlx/query-5c609ea0696df96ca02cba7fee359b785515a79dcda3745663e4c4f2cf328389.json b/backend/.sqlx/query-5c609ea0696df96ca02cba7fee359b785515a79dcda3745663e4c4f2cf328389.json new file mode 100644 index 0000000000..03de8472ad --- /dev/null +++ b/backend/.sqlx/query-5c609ea0696df96ca02cba7fee359b785515a79dcda3745663e4c4f2cf328389.json @@ -0,0 +1,48 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n usage_path AS \"usage_path!\",\n kind AS \"kind!: AssetKind\",\n path AS \"path!\"\n FROM asset\n WHERE workspace_id = $1\n AND usage_kind = 'script'\n AND usage_access_type IN ('w', 'rw')\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "usage_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "kind!: AssetKind", + "type_info": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + } + }, + { + "ordinal": 2, + "name": "path!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "5c609ea0696df96ca02cba7fee359b785515a79dcda3745663e4c4f2cf328389" +} diff --git a/backend/.sqlx/query-651fc12e1b971d4fd57c98a7a7efbd503d8dea799545e9cb96574d5c6020b90b.json b/backend/.sqlx/query-651fc12e1b971d4fd57c98a7a7efbd503d8dea799545e9cb96574d5c6020b90b.json new file mode 100644 index 0000000000..cb20ec2ffb --- /dev/null +++ b/backend/.sqlx/query-651fc12e1b971d4fd57c98a7a7efbd503d8dea799545e9cb96574d5c6020b90b.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job\n SET args = CASE\n WHEN args ? 'partition'\n THEN $1 || jsonb_build_object('partition', args -> 'partition')\n ELSE $1\n END,\n preprocessed = TRUE\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "651fc12e1b971d4fd57c98a7a7efbd503d8dea799545e9cb96574d5c6020b90b" +} diff --git a/backend/.sqlx/query-67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8.json b/backend/.sqlx/query-67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8.json index 50e7b53387..08ebe6bba5 100644 --- a/backend/.sqlx/query-67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8.json +++ b/backend/.sqlx/query-67e25a7c19ea0ffaf7ea5303fcd04af5a7eb488c76f783e690af0c2153b1d6a8.json @@ -160,7 +160,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } diff --git a/backend/.sqlx/query-6814542fcdd01a178798ad7b0840d8288f1d6adbeb3e545386cf66c33f8db50f.json b/backend/.sqlx/query-6814542fcdd01a178798ad7b0840d8288f1d6adbeb3e545386cf66c33f8db50f.json new file mode 100644 index 0000000000..a74aedfdb5 --- /dev/null +++ b/backend/.sqlx/query-6814542fcdd01a178798ad7b0840d8288f1d6adbeb3e545386cf66c33f8db50f.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT count(*) AS \"n!\"\n FROM join_pending_inputs\n WHERE workspace_id = $1 AND subscriber_path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "n!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "6814542fcdd01a178798ad7b0840d8288f1d6adbeb3e545386cf66c33f8db50f" +} diff --git a/backend/.sqlx/query-6a1998cb3a9a0898c0fd21c35cc41e309d7e066ee4869880c90e90dd66dc9d4d.json b/backend/.sqlx/query-6a1998cb3a9a0898c0fd21c35cc41e309d7e066ee4869880c90e90dd66dc9d4d.json new file mode 100644 index 0000000000..7f2b4a677a --- /dev/null +++ b/backend/.sqlx/query-6a1998cb3a9a0898c0fd21c35cc41e309d7e066ee4869880c90e90dd66dc9d4d.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH del AS (\n DELETE FROM asset WHERE workspace_id = $1 AND usage_kind = 'script'\n AND usage_path = (SELECT path FROM script WHERE hash = $2 AND workspace_id = $1)\n )\n INSERT INTO notify_event (channel, payload)\n VALUES ('notify_asset_producer_change', $1)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "6a1998cb3a9a0898c0fd21c35cc41e309d7e066ee4869880c90e90dd66dc9d4d" +} diff --git a/backend/.sqlx/query-6aaddd80f8c07cfafea2021c1879c3d7b2fb156a43299e9a1209d05293c4f50f.json b/backend/.sqlx/query-6aaddd80f8c07cfafea2021c1879c3d7b2fb156a43299e9a1209d05293c4f50f.json new file mode 100644 index 0000000000..144bcb13f2 --- /dev/null +++ b/backend/.sqlx/query-6aaddd80f8c07cfafea2021c1879c3d7b2fb156a43299e9a1209d05293c4f50f.json @@ -0,0 +1,64 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO dispatch_event (\n workspace_id, producer_job_id, subscriber_path,\n asset_kind, asset_path, outcome,\n child_job_id, partition,\n received_inputs, required_inputs,\n debounce_s, reason\n )\n SELECT $1, $2, sp, ak, ap, oc, cj, pt, ri, rq, db, rs\n FROM unnest(\n $3::text[], $4::ASSET_KIND[], $5::text[], $6::DISPATCH_OUTCOME[],\n $7::uuid[], $8::text[], $9::int[], $10::int[], $11::int[], $12::text[]\n ) AS t(sp, ak, ap, oc, cj, pt, ri, rq, db, rs)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Uuid", + "TextArray", + { + "Custom": { + "name": "asset_kind[]", + "kind": { + "Array": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + } + } + } + }, + "TextArray", + { + "Custom": { + "name": "dispatch_outcome[]", + "kind": { + "Array": { + "Custom": { + "name": "dispatch_outcome", + "kind": { + "Enum": [ + "dispatched", + "join_pending", + "skipped" + ] + } + } + } + } + } + }, + "UuidArray", + "TextArray", + "Int4Array", + "Int4Array", + "Int4Array", + "TextArray" + ] + }, + "nullable": [] + }, + "hash": "6aaddd80f8c07cfafea2021c1879c3d7b2fb156a43299e9a1209d05293c4f50f" +} diff --git a/backend/.sqlx/query-6b9348e60cc1ce158314a93fc7aa55a9f8fa854b29edcea83710a9170124edf0.json b/backend/.sqlx/query-6b9348e60cc1ce158314a93fc7aa55a9f8fa854b29edcea83710a9170124edf0.json new file mode 100644 index 0000000000..43d3b6d7f6 --- /dev/null +++ b/backend/.sqlx/query-6b9348e60cc1ce158314a93fc7aa55a9f8fa854b29edcea83710a9170124edf0.json @@ -0,0 +1,64 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value as \"value!: sqlx::types::Json>\", created_at\n FROM draft\n WHERE workspace_id = $1 AND path = $2 AND typ = $3 AND email = $4", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value!: sqlx::types::Json>", + "type_info": "Json" + }, + { + "ordinal": 1, + "name": "created_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github", + "data_pipeline" + ] + } + } + }, + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "6b9348e60cc1ce158314a93fc7aa55a9f8fa854b29edcea83710a9170124edf0" +} diff --git a/backend/.sqlx/query-7300eb89029e0863241087fc10df7616db640054e804d4ca66958cad06008fdb.json b/backend/.sqlx/query-7300eb89029e0863241087fc10df7616db640054e804d4ca66958cad06008fdb.json new file mode 100644 index 0000000000..cc564952cb --- /dev/null +++ b/backend/.sqlx/query-7300eb89029e0863241087fc10df7616db640054e804d4ca66958cad06008fdb.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH del AS (\n DELETE FROM asset WHERE workspace_id = $1 AND usage_path = $2 AND usage_kind = $3\n )\n INSERT INTO notify_event (channel, payload)\n SELECT 'notify_asset_producer_change', $1 WHERE $3 = 'script'", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + { + "Custom": { + "name": "asset_usage_kind", + "kind": { + "Enum": [ + "script", + "flow", + "job" + ] + } + } + } + ] + }, + "nullable": [] + }, + "hash": "7300eb89029e0863241087fc10df7616db640054e804d4ca66958cad06008fdb" +} diff --git a/backend/.sqlx/query-751f836dc8f78c330387456dd68a8803972c7b3e2b6a2b95c27f15068bed2ca5.json b/backend/.sqlx/query-751f836dc8f78c330387456dd68a8803972c7b3e2b6a2b95c27f15068bed2ca5.json new file mode 100644 index 0000000000..a784094d72 --- /dev/null +++ b/backend/.sqlx/query-751f836dc8f78c330387456dd68a8803972c7b3e2b6a2b95c27f15068bed2ca5.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "pg_advisory_xact_lock", + "type_info": "Void" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "751f836dc8f78c330387456dd68a8803972c7b3e2b6a2b95c27f15068bed2ca5" +} diff --git a/backend/.sqlx/query-754b98335e8776565d63267b395013649adacf348e3a815e991b4463b1711afc.json b/backend/.sqlx/query-754b98335e8776565d63267b395013649adacf348e3a815e991b4463b1711afc.json new file mode 100644 index 0000000000..c5266c66b5 --- /dev/null +++ b/backend/.sqlx/query-754b98335e8776565d63267b395013649adacf348e3a815e991b4463b1711afc.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT runnable_path AS \"runnable_path!\", kind::text AS \"kind!\"\n FROM v2_job\n WHERE workspace_id = $1 AND trigger_kind = 'asset'\n ORDER BY runnable_path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "runnable_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "kind!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true, + null + ] + }, + "hash": "754b98335e8776565d63267b395013649adacf348e3a815e991b4463b1711afc" +} diff --git a/backend/.sqlx/query-77424d40104cf271e5ee5118100a988159130fc3a8cde91d419a1787b6bb8a51.json b/backend/.sqlx/query-77424d40104cf271e5ee5118100a988159130fc3a8cde91d419a1787b6bb8a51.json new file mode 100644 index 0000000000..0768563002 --- /dev/null +++ b/backend/.sqlx/query-77424d40104cf271e5ee5118100a988159130fc3a8cde91d419a1787b6bb8a51.json @@ -0,0 +1,52 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n runnable_kind AS \"runnable_kind!: AssetUsageKind\",\n runnable_path AS \"runnable_path!\",\n trigger_kind::text AS \"trigger_kind!\",\n trigger_ref AS \"trigger_ref!\"\n FROM script_trigger\n WHERE workspace_id = $1\n AND trigger_kind = 'asset'\n AND ($2::text IS NULL OR runnable_path LIKE $2)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "runnable_kind!: AssetUsageKind", + "type_info": { + "Custom": { + "name": "asset_usage_kind", + "kind": { + "Enum": [ + "script", + "flow", + "job" + ] + } + } + } + }, + { + "ordinal": 1, + "name": "runnable_path!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "trigger_kind!", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "trigger_ref!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + null, + false + ] + }, + "hash": "77424d40104cf271e5ee5118100a988159130fc3a8cde91d419a1787b6bb8a51" +} diff --git a/backend/.sqlx/query-80f2d2f20e93b5e05ecd1fe5afeaeebc22883bf1d10dcbce41cccebb392ffd69.json b/backend/.sqlx/query-80f2d2f20e93b5e05ecd1fe5afeaeebc22883bf1d10dcbce41cccebb392ffd69.json new file mode 100644 index 0000000000..f0f7df5289 --- /dev/null +++ b/backend/.sqlx/query-80f2d2f20e93b5e05ecd1fe5afeaeebc22883bf1d10dcbce41cccebb392ffd69.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM join_pending_inputs\n WHERE workspace_id = $1 AND subscriber_path = $2 AND partition = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "80f2d2f20e93b5e05ecd1fe5afeaeebc22883bf1d10dcbce41cccebb392ffd69" +} diff --git a/backend/.sqlx/query-82dcaf94ffe43da1c8c7de2a3478b4919c4f1dbf1972d04664a730cefc0594e2.json b/backend/.sqlx/query-82dcaf94ffe43da1c8c7de2a3478b4919c4f1dbf1972d04664a730cefc0594e2.json new file mode 100644 index 0000000000..96587ed1d0 --- /dev/null +++ b/backend/.sqlx/query-82dcaf94ffe43da1c8c7de2a3478b4919c4f1dbf1972d04664a730cefc0594e2.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT q.runnable_settings_handle\n FROM v2_job j JOIN v2_job_queue q ON q.id = j.id\n WHERE j.workspace_id = $1 AND j.runnable_path = $2\n AND j.trigger_kind = 'asset'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "runnable_settings_handle", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "82dcaf94ffe43da1c8c7de2a3478b4919c4f1dbf1972d04664a730cefc0594e2" +} diff --git a/backend/.sqlx/query-8bb2f6f4526231c1ce57182a779c9d7cb5d1022da6b5bf5a17c73c61773b50f4.json b/backend/.sqlx/query-8bb2f6f4526231c1ce57182a779c9d7cb5d1022da6b5bf5a17c73c61773b50f4.json new file mode 100644 index 0000000000..153b11560e --- /dev/null +++ b/backend/.sqlx/query-8bb2f6f4526231c1ce57182a779c9d7cb5d1022da6b5bf5a17c73c61773b50f4.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT runnable_path AS \"runnable_path!\", join_all AS \"join_all!\", debounce_s,\n retry_count, retry_delay_s\n FROM script_trigger\n WHERE workspace_id = $1\n AND trigger_kind = 'asset'\n AND trigger_ref = $2\n AND runnable_kind = 'script'\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "runnable_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "join_all!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "debounce_s", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "retry_count", + "type_info": "Int2" + }, + { + "ordinal": 4, + "name": "retry_delay_s", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + true, + true, + true + ] + }, + "hash": "8bb2f6f4526231c1ce57182a779c9d7cb5d1022da6b5bf5a17c73c61773b50f4" +} diff --git a/backend/.sqlx/query-8bce3f969b4bbbcf3dc1b8c671ab76e947e491da84fbaf5f130f58fbc851594e.json b/backend/.sqlx/query-8bce3f969b4bbbcf3dc1b8c671ab76e947e491da84fbaf5f130f58fbc851594e.json new file mode 100644 index 0000000000..fe484078c2 --- /dev/null +++ b/backend/.sqlx/query-8bce3f969b4bbbcf3dc1b8c671ab76e947e491da84fbaf5f130f58fbc851594e.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM script_trigger\n WHERE workspace_id = $1 AND runnable_kind = $2 AND runnable_path = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + { + "Custom": { + "name": "asset_usage_kind", + "kind": { + "Enum": [ + "script", + "flow", + "job" + ] + } + } + }, + "Text" + ] + }, + "nullable": [] + }, + "hash": "8bce3f969b4bbbcf3dc1b8c671ab76e947e491da84fbaf5f130f58fbc851594e" +} diff --git a/backend/.sqlx/query-8cd02a5378bea03012e6cb937f23460b6438d0a320b5554acec0beddbdeb008b.json b/backend/.sqlx/query-8cd02a5378bea03012e6cb937f23460b6438d0a320b5554acec0beddbdeb008b.json new file mode 100644 index 0000000000..24d70efae1 --- /dev/null +++ b/backend/.sqlx/query-8cd02a5378bea03012e6cb937f23460b6438d0a320b5554acec0beddbdeb008b.json @@ -0,0 +1,74 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n de.producer_job_id AS \"producer_job_id!\",\n de.child_job_id,\n de.subscriber_path AS \"subscriber_path!\",\n de.outcome::text AS \"outcome!\",\n de.asset_kind AS \"asset_kind!: windmill_common::assets::AssetKind\",\n de.asset_path AS \"asset_path!\",\n de.created_at AS \"created_at!\"\n FROM dispatch_event de\n JOIN v2_job pj ON pj.id = de.producer_job_id\n WHERE de.workspace_id = $1\n AND de.outcome IN ('dispatched', 'join_pending')\n AND de.subscriber_path LIKE $2\n AND ($3::timestamptz IS NULL OR de.created_at >= $3)\n ORDER BY de.created_at DESC, de.id DESC\n LIMIT 4000", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "producer_job_id!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "child_job_id", + "type_info": "Uuid" + }, + { + "ordinal": 2, + "name": "subscriber_path!", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "outcome!", + "type_info": "Text" + }, + { + "ordinal": 4, + "name": "asset_kind!: windmill_common::assets::AssetKind", + "type_info": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + } + }, + { + "ordinal": 5, + "name": "asset_path!", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "created_at!", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Timestamptz" + ] + }, + "nullable": [ + false, + true, + false, + null, + false, + false, + false + ] + }, + "hash": "8cd02a5378bea03012e6cb937f23460b6438d0a320b5554acec0beddbdeb008b" +} diff --git a/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json b/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json index fd32ba2753..ad9e57801e 100644 --- a/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json +++ b/backend/.sqlx/query-9ecb404e46a4eac55f977f05a3afbafe5dc3cdecc17a3d5a7476b160c1b6e7e1.json @@ -34,7 +34,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } diff --git a/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json b/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json index f88fbc8a47..9a21f228ea 100644 --- a/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json +++ b/backend/.sqlx/query-a4b6371d33206010b2f3ffd2b09e33244fe8ab9a803248fc23f334034d24aad4.json @@ -190,7 +190,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } diff --git a/backend/.sqlx/query-a9a99880d870266f474878dd6ef541df988da527d30f663ef6f764f0c3d70d4b.json b/backend/.sqlx/query-a9a99880d870266f474878dd6ef541df988da527d30f663ef6f764f0c3d70d4b.json new file mode 100644 index 0000000000..d12d305b92 --- /dev/null +++ b/backend/.sqlx/query-a9a99880d870266f474878dd6ef541df988da527d30f663ef6f764f0c3d70d4b.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT path AS \"path!\"\n FROM script\n WHERE workspace_id = $1\n AND auto_kind = 'pipeline'\n AND archived = false\n AND deleted = false\n AND ($2::text IS NULL OR path LIKE $2)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "a9a99880d870266f474878dd6ef541df988da527d30f663ef6f764f0c3d70d4b" +} diff --git a/backend/.sqlx/query-abb36bfddf707c7897b3936e982725f973eeb6ba5abbd61416060fb7d675593b.json b/backend/.sqlx/query-abb36bfddf707c7897b3936e982725f973eeb6ba5abbd61416060fb7d675593b.json new file mode 100644 index 0000000000..7795458cef --- /dev/null +++ b/backend/.sqlx/query-abb36bfddf707c7897b3936e982725f973eeb6ba5abbd61416060fb7d675593b.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n substring(path from '^f/([^/]+)/') AS \"folder!\",\n COUNT(*) AS \"script_count!\"\n FROM script\n WHERE workspace_id = $1\n AND auto_kind = 'pipeline'\n AND archived = false\n AND deleted = false\n AND path LIKE 'f/%'\n GROUP BY substring(path from '^f/([^/]+)/')\n ORDER BY substring(path from '^f/([^/]+)/')\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "folder!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "script_count!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + null + ] + }, + "hash": "abb36bfddf707c7897b3936e982725f973eeb6ba5abbd61416060fb7d675593b" +} diff --git a/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json b/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json index ca95a3bca8..3efa843923 100644 --- a/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json +++ b/backend/.sqlx/query-b3771b690c5966272b1f42c9965bb6a8f961c119516e4c33dc928cd3b4f4edbc.json @@ -165,7 +165,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } diff --git a/backend/.sqlx/query-bcfa34cf80abea05f0c24883b9e77429c51e6166c414bcc5ce2e97fac25bcd77.json b/backend/.sqlx/query-bcfa34cf80abea05f0c24883b9e77429c51e6166c414bcc5ce2e97fac25bcd77.json index 045d470de5..f6ff25a4bf 100644 --- a/backend/.sqlx/query-bcfa34cf80abea05f0c24883b9e77429c51e6166c414bcc5ce2e97fac25bcd77.json +++ b/backend/.sqlx/query-bcfa34cf80abea05f0c24883b9e77429c51e6166c414bcc5ce2e97fac25bcd77.json @@ -79,7 +79,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } diff --git a/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json b/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json index a8bb03e1b3..8dc66064dc 100644 --- a/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json +++ b/backend/.sqlx/query-be6d2c92a62b7b284651c45af809746147aa9b8d0a81642a7b7cb4738a0cad66.json @@ -110,7 +110,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } diff --git a/backend/.sqlx/query-c0952a5f8a3959630f124f1bf379207ccf02da21984730391eeb497f7146ba66.json b/backend/.sqlx/query-c0952a5f8a3959630f124f1bf379207ccf02da21984730391eeb497f7146ba66.json new file mode 100644 index 0000000000..db89d7bcfa --- /dev/null +++ b/backend/.sqlx/query-c0952a5f8a3959630f124f1bf379207ccf02da21984730391eeb497f7146ba66.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT args AS \"args: sqlx::types::Json\"\n FROM v2_job\n WHERE workspace_id = $1 AND runnable_path = $2 AND trigger_kind IS NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "args: sqlx::types::Json", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true + ] + }, + "hash": "c0952a5f8a3959630f124f1bf379207ccf02da21984730391eeb497f7146ba66" +} diff --git a/backend/.sqlx/query-c2c4e95ef48bf05e971b0e8d8de5f1f775a8b526382d671c95a2eb8879e1d828.json b/backend/.sqlx/query-c2c4e95ef48bf05e971b0e8d8de5f1f775a8b526382d671c95a2eb8879e1d828.json new file mode 100644 index 0000000000..44e09f64d1 --- /dev/null +++ b/backend/.sqlx/query-c2c4e95ef48bf05e971b0e8d8de5f1f775a8b526382d671c95a2eb8879e1d828.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job SET args = coalesce(args, '{}'::jsonb) || jsonb_build_object('partition', $1::text) WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "c2c4e95ef48bf05e971b0e8d8de5f1f775a8b526382d671c95a2eb8879e1d828" +} diff --git a/backend/.sqlx/query-c608cdc9cd2e41992fb86431faa15edc8e637d44031b83c01ef09a96ade472ae.json b/backend/.sqlx/query-c608cdc9cd2e41992fb86431faa15edc8e637d44031b83c01ef09a96ade472ae.json new file mode 100644 index 0000000000..247d4de14a --- /dev/null +++ b/backend/.sqlx/query-c608cdc9cd2e41992fb86431faa15edc8e637d44031b83c01ef09a96ade472ae.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT created_by AS \"created_by!\"\n FROM v2_job\n WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "created_by!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text", + "TextArray" + ] + }, + "nullable": [ + false + ] + }, + "hash": "c608cdc9cd2e41992fb86431faa15edc8e637d44031b83c01ef09a96ade472ae" +} diff --git a/backend/.sqlx/query-d3d87b9a4d62977dea5af95dd457bb759229805cff771cfa4275a46bfc80e1ab.json b/backend/.sqlx/query-d3d87b9a4d62977dea5af95dd457bb759229805cff771cfa4275a46bfc80e1ab.json new file mode 100644 index 0000000000..b5d905b9b6 --- /dev/null +++ b/backend/.sqlx/query-d3d87b9a4d62977dea5af95dd457bb759229805cff771cfa4275a46bfc80e1ab.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job\n SET args = CASE\n WHEN args ? 'partition'\n THEN $1 || jsonb_build_object('partition', args -> 'partition')\n ELSE $1\n END,\n preprocessed = TRUE\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "d3d87b9a4d62977dea5af95dd457bb759229805cff771cfa4275a46bfc80e1ab" +} diff --git a/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json b/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json index 5ed2e53367..d5365ffe94 100644 --- a/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json +++ b/backend/.sqlx/query-d41ea93fd58381b89e151c965eae1ea2fe96a1b94f5a92953fb1c1642d15c016.json @@ -110,7 +110,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } diff --git a/backend/.sqlx/query-d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9.json b/backend/.sqlx/query-d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9.json index a27bea8b2a..33a5534b42 100644 --- a/backend/.sqlx/query-d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9.json +++ b/backend/.sqlx/query-d4211392e174a0e8f89c7fcebdf120e5b0f629f9f04e08a2982df33ff23ac7a9.json @@ -250,7 +250,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } diff --git a/backend/.sqlx/query-d6a8ab57341b8aa17d4d50ce7a710c923df7b66d1b76d36229299d2866775ffc.json b/backend/.sqlx/query-d6a8ab57341b8aa17d4d50ce7a710c923df7b66d1b76d36229299d2866775ffc.json new file mode 100644 index 0000000000..befd9903a9 --- /dev/null +++ b/backend/.sqlx/query-d6a8ab57341b8aa17d4d50ce7a710c923df7b66d1b76d36229299d2866775ffc.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT runnable_path AS \"runnable_path!\", trigger,\n args AS \"args: sqlx::types::Json\"\n FROM v2_job\n WHERE workspace_id = $1 AND trigger_kind = 'asset'\n ORDER BY runnable_path", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "runnable_path!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "trigger", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "args: sqlx::types::Json", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + true, + true, + true + ] + }, + "hash": "d6a8ab57341b8aa17d4d50ce7a710c923df7b66d1b76d36229299d2866775ffc" +} diff --git a/backend/.sqlx/query-de06f44bad94710f14e9be4c0a6e6080e3c4faae5052500b93cc24b6fe556f2b.json b/backend/.sqlx/query-de06f44bad94710f14e9be4c0a6e6080e3c4faae5052500b93cc24b6fe556f2b.json new file mode 100644 index 0000000000..2ba317edc6 --- /dev/null +++ b/backend/.sqlx/query-de06f44bad94710f14e9be4c0a6e6080e3c4faae5052500b93cc24b6fe556f2b.json @@ -0,0 +1,43 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n kind AS \"kind!: AssetKind\",\n path AS \"path!\"\n FROM asset\n WHERE workspace_id = $1\n AND usage_kind = 'script'\n AND usage_path = $2\n AND usage_access_type IN ('w', 'rw')\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "kind!: AssetKind", + "type_info": { + "Custom": { + "name": "asset_kind", + "kind": { + "Enum": [ + "s3object", + "resource", + "variable", + "ducklake", + "datatable", + "volume" + ] + } + } + } + }, + { + "ordinal": 1, + "name": "path!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "de06f44bad94710f14e9be4c0a6e6080e3c4faae5052500b93cc24b6fe556f2b" +} diff --git a/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json b/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json index 1a4cc407e0..470c651020 100644 --- a/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json +++ b/backend/.sqlx/query-e4d71278fb80126a7a9da73f1889352d4d1e3cb3a8a08f1c9c03055a1cab1235.json @@ -190,7 +190,8 @@ "google", "ci_test", "github", - "azure" + "azure", + "asset" ] } } diff --git a/backend/.sqlx/query-ecce519d0cf0c31df4612e0ccd8d62eef3ab3b920665ca5d6f9ce0ef89e53fb3.json b/backend/.sqlx/query-ecce519d0cf0c31df4612e0ccd8d62eef3ab3b920665ca5d6f9ce0ef89e53fb3.json new file mode 100644 index 0000000000..d40882073d --- /dev/null +++ b/backend/.sqlx/query-ecce519d0cf0c31df4612e0ccd8d62eef3ab3b920665ca5d6f9ce0ef89e53fb3.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path AS \"path!\" FROM script\n WHERE workspace_id = $1\n AND archived = false\n AND deleted = false", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path!", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "ecce519d0cf0c31df4612e0ccd8d62eef3ab3b920665ca5d6f9ce0ef89e53fb3" +} diff --git a/backend/.sqlx/query-f338943aa3595c2893a0d42e4a54e8dd1370fb59c0a5cfd2403c5fdaf2135cf2.json b/backend/.sqlx/query-f338943aa3595c2893a0d42e4a54e8dd1370fb59c0a5cfd2403c5fdaf2135cf2.json new file mode 100644 index 0000000000..dd1db109da --- /dev/null +++ b/backend/.sqlx/query-f338943aa3595c2893a0d42e4a54e8dd1370fb59c0a5cfd2403c5fdaf2135cf2.json @@ -0,0 +1,51 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO script_trigger\n (workspace_id, runnable_kind, runnable_path, trigger_kind, trigger_ref, join_all,\n debounce_s, retry_count, retry_delay_s)\n VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + { + "Custom": { + "name": "asset_usage_kind", + "kind": { + "Enum": [ + "script", + "flow", + "job" + ] + } + } + }, + "Varchar", + { + "Custom": { + "name": "script_trigger_kind", + "kind": { + "Enum": [ + "asset", + "schedule", + "webhook", + "email", + "kafka", + "mqtt", + "nats", + "postgres", + "sqs", + "gcp" + ] + } + } + }, + "Text", + "Bool", + "Int4", + "Int2", + "Int4" + ] + }, + "nullable": [] + }, + "hash": "f338943aa3595c2893a0d42e4a54e8dd1370fb59c0a5cfd2403c5fdaf2135cf2" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 0db0d88472..1c8ddf01bd 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -14351,8 +14351,13 @@ dependencies = [ "windmill-dep-map", "windmill-git-sync", "windmill-object-store", + "windmill-parser", "windmill-parser-py", + "windmill-parser-py-asset", + "windmill-parser-sql-asset", "windmill-parser-ts", + "windmill-parser-ts-asset", + "windmill-parser-yaml", "windmill-queue", ] diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 0ed0d2d025..151e85295b 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -9e4dadafb44ba953a7d2af2be12b92be98d86b66 +ba677ea142011462ad4dfe77e8375a6dd274cdef diff --git a/backend/migrations/20260423050000_script_trigger.down.sql b/backend/migrations/20260423050000_script_trigger.down.sql new file mode 100644 index 0000000000..5171e8f2d0 --- /dev/null +++ b/backend/migrations/20260423050000_script_trigger.down.sql @@ -0,0 +1,3 @@ +DROP INDEX IF EXISTS idx_script_pipeline_path; +DROP TABLE IF EXISTS script_trigger; +DROP TYPE IF EXISTS SCRIPT_TRIGGER_KIND; diff --git a/backend/migrations/20260423050000_script_trigger.up.sql b/backend/migrations/20260423050000_script_trigger.up.sql new file mode 100644 index 0000000000..cb105ba3f3 --- /dev/null +++ b/backend/migrations/20260423050000_script_trigger.up.sql @@ -0,0 +1,58 @@ +-- Execution DAG edges declared via `// on ` +-- annotations. +-- For `trigger_kind='asset'`: trigger_ref is `://` (kind from +-- parse_asset_syntax, so downstream lookups match the `asset` table). +-- The other kinds mirror the keywords the annotation parser recognises in +-- `// on ` lines (every non-integration trigger kind; their +-- trigger_ref is the trigger row path, or empty for marker-only forms). +-- +-- Per-edge columns that are in fact script-level properties (every row for +-- a given runnable carries the same value, set once at deploy) but live on +-- the edge so the dispatcher reads everything from a single query: +-- join_all `// trigger all` AND-join barrier (else OR, the default). +-- debounce_s `// on debounce=` (else script-level `// debounce`); only +-- asset-cascade edges carry one. NULL = no debounce. +-- retry_* `// retry []` cascade retry. NULL = none. +-- +-- The idempotency guards (IF NOT EXISTS / duplicate_object) are load-bearing: +-- this migration squashes several pre-release ones, so databases migrated +-- from the unsquashed history already contain the final objects and +-- re-applying must be a no-op. +DO $$ BEGIN + CREATE TYPE SCRIPT_TRIGGER_KIND AS ENUM ( + 'asset', 'schedule', 'webhook', 'email', 'kafka', 'mqtt', 'nats', + 'postgres', 'sqs', 'gcp'); +EXCEPTION WHEN duplicate_object THEN NULL; +END $$; + +CREATE TABLE IF NOT EXISTS script_trigger ( + id BIGSERIAL PRIMARY KEY, + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + runnable_kind ASSET_USAGE_KIND NOT NULL, + runnable_path VARCHAR(255) NOT NULL, + trigger_kind SCRIPT_TRIGGER_KIND NOT NULL, + trigger_ref TEXT NOT NULL, + join_all BOOLEAN NOT NULL DEFAULT FALSE, + debounce_s INTEGER, + retry_count SMALLINT, + retry_delay_s INTEGER +); + +-- Per-runnable lookup (wipe-on-deploy, list-triggers-for-script). +CREATE INDEX IF NOT EXISTS idx_script_trigger_runnable + ON script_trigger (workspace_id, runnable_kind, runnable_path); + +-- Reverse lookup: "which scripts are triggered by asset X?" (the asset → script +-- edges in the graph). trigger_ref is unbounded text so can't share the +-- asset_kind btree, but this covers the common prefix-scan use case. +CREATE INDEX IF NOT EXISTS idx_script_trigger_ref + ON script_trigger (workspace_id, trigger_kind, trigger_ref); + +-- Fast lookups for: +-- 1. "does folder F have a pipeline?" (exists check on prefix) +-- 2. "list all folders with a pipeline" (distinct folder from path) +-- The partial predicate keeps the index tiny on workspaces with few +-- pipeline scripts, and text_pattern_ops lets 'f/foo/%' LIKE scans use it. +CREATE INDEX IF NOT EXISTS idx_script_pipeline_path + ON script (workspace_id, path text_pattern_ops) + WHERE auto_kind = 'pipeline' AND archived = false AND deleted = false; diff --git a/backend/migrations/20260510174213_asset_trigger_dispatch.down.sql b/backend/migrations/20260510174213_asset_trigger_dispatch.down.sql new file mode 100644 index 0000000000..76e5b7050c --- /dev/null +++ b/backend/migrations/20260510174213_asset_trigger_dispatch.down.sql @@ -0,0 +1,3 @@ +-- Postgres has no ALTER TYPE ... DROP VALUE for enums. The 'asset' value +-- stays even on rollback; this is consistent with how other job_trigger_kind +-- values were added (see 20250323162033_add-missing-trigger-kind-...). diff --git a/backend/migrations/20260510174213_asset_trigger_dispatch.up.sql b/backend/migrations/20260510174213_asset_trigger_dispatch.up.sql new file mode 100644 index 0000000000..30cdb58ec3 --- /dev/null +++ b/backend/migrations/20260510174213_asset_trigger_dispatch.up.sql @@ -0,0 +1,5 @@ +-- Add 'asset' as a job_trigger_kind so jobs that get dispatched as a +-- consequence of an upstream pipeline script writing an asset can be +-- attributed via v2_job.trigger_kind = 'asset'. The producer's runnable +-- path goes into v2_job.trigger. +ALTER TYPE job_trigger_kind ADD VALUE IF NOT EXISTS 'asset'; diff --git a/backend/migrations/20260516194247_join_pending_inputs.down.sql b/backend/migrations/20260516194247_join_pending_inputs.down.sql new file mode 100644 index 0000000000..6e67e88e1b --- /dev/null +++ b/backend/migrations/20260516194247_join_pending_inputs.down.sql @@ -0,0 +1 @@ +DROP TABLE join_pending_inputs; diff --git a/backend/migrations/20260516194247_join_pending_inputs.up.sql b/backend/migrations/20260516194247_join_pending_inputs.up.sql new file mode 100644 index 0000000000..cbda6dd16e --- /dev/null +++ b/backend/migrations/20260516194247_join_pending_inputs.up.sql @@ -0,0 +1,19 @@ +-- AND-join barrier slot state. For a `// trigger all` subscriber, each +-- partition-bearing input arrival (an `// on` asset whose declared path +-- contains the `{partition}` token) is recorded against the +-- (subscriber, partition) slot. The subscriber is dispatched once, for a +-- given partition, only when every partition-bearing input it declares has +-- arrived for that partition — skew-immune (unlike a debounce). The slot +-- is cleared on fire so later writes re-accumulate and can re-materialize. +-- +-- trigger_ref stores the literal `{partition}`-token form (lineage is +-- partition-agnostic; the concrete value is the `partition` column), +-- matching how script_trigger / asset rows store it. +CREATE TABLE join_pending_inputs ( + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + subscriber_path VARCHAR(255) NOT NULL, + partition TEXT NOT NULL, + trigger_ref TEXT NOT NULL, + received_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (workspace_id, subscriber_path, partition, trigger_ref) +); diff --git a/backend/migrations/20260523055641_dispatch_event.down.sql b/backend/migrations/20260523055641_dispatch_event.down.sql new file mode 100644 index 0000000000..c6fdefc07c --- /dev/null +++ b/backend/migrations/20260523055641_dispatch_event.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS dispatch_event; +DROP TYPE IF EXISTS DISPATCH_OUTCOME; diff --git a/backend/migrations/20260523055641_dispatch_event.up.sql b/backend/migrations/20260523055641_dispatch_event.up.sql new file mode 100644 index 0000000000..bf68281fbd --- /dev/null +++ b/backend/migrations/20260523055641_dispatch_event.up.sql @@ -0,0 +1,63 @@ +-- Per-decision log of what the asset-trigger dispatcher did after each +-- producer job completed. One row per (producer, subscriber, asset write) +-- decision: dispatched, debounced, join_pending (partial AND-join), or +-- skipped (with reason). Surfaced on the producer's job detail page so +-- the cascade is no longer invisible when the producer "succeeds" but +-- no child appears. +-- +-- Retention: the FK to v2_job(id) ON DELETE CASCADE means the existing +-- retention sweep (monitor.rs delete_expired_jobs_batch -> DELETE FROM +-- v2_job WHERE id = ANY(...)) reaps these rows along with their producer. +-- No separate cleanup path needed. +-- +-- Idempotency guards (duplicate_object / IF NOT EXISTS) are load-bearing: +-- re-applying this migration after a squash must be a no-op. +DO $$ BEGIN + CREATE TYPE DISPATCH_OUTCOME AS ENUM ( + 'dispatched', + 'join_pending', + 'skipped' + ); +EXCEPTION WHEN duplicate_object THEN NULL; +END $$; + +CREATE TABLE IF NOT EXISTS dispatch_event ( + id BIGSERIAL PRIMARY KEY, + workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id) ON DELETE CASCADE ON UPDATE CASCADE, + producer_job_id UUID NOT NULL REFERENCES v2_job(id) ON DELETE CASCADE, + subscriber_path VARCHAR(255) NOT NULL, + asset_kind ASSET_KIND NOT NULL, + asset_path TEXT NOT NULL, + outcome DISPATCH_OUTCOME NOT NULL, + -- Set for 'dispatched'. Intentionally not FK'd: the subscriber job may + -- be retention-reaped independently, and we still want the row to + -- record "we dispatched " historically (UI renders a dead link). + child_job_id UUID, + partition TEXT, + -- AND-join progress at decision time. NULL for non-join subscribers. + received_inputs INTEGER, + required_inputs INTEGER, + -- Effective debounce window applied to this dispatch (NULL = none). + debounce_s INTEGER, + -- Free-text discriminator for 'skipped' outcomes (self_loop, + -- case3_non_partition_bearing, case3_missing_partition, cycle_detected, ...). + reason TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +-- Primary access pattern: list events for one producer (the job detail +-- panel). Ordered scans by id give chronological order for free. +CREATE INDEX IF NOT EXISTS idx_dispatch_event_producer + ON dispatch_event (producer_job_id, id); + +-- Backs the asset-graph edge listing (jobs.rs list_asset_dispatch_edges): +-- WHERE workspace_id = $1 AND subscriber_path LIKE 'prefix%' +-- AND created_at >= $3 +-- ORDER BY created_at DESC, id DESC +-- The (producer_job_id, id) index above doesn't help this access path, so +-- without this one a high-volume dispatch_event seq-scans + sorts. +-- text_pattern_ops makes the anchored LIKE prefix (built as `path_start || '%'`) +-- index-usable regardless of the column collation; created_at DESC matches +-- the ORDER BY so Postgres can satisfy ordering from the index. +CREATE INDEX IF NOT EXISTS idx_dispatch_event_subscriber + ON dispatch_event (workspace_id, subscriber_path text_pattern_ops, created_at DESC); diff --git a/backend/migrations/20260616090412_add_data_pipeline_draft_kind.down.sql b/backend/migrations/20260616090412_add_data_pipeline_draft_kind.down.sql new file mode 100644 index 0000000000..fcb5ef5bbf --- /dev/null +++ b/backend/migrations/20260616090412_add_data_pipeline_draft_kind.down.sql @@ -0,0 +1,2 @@ +-- Postgres cannot drop a single enum value; leaving 'data_pipeline' in +-- DRAFT_KIND is harmless on rollback. diff --git a/backend/migrations/20260616090412_add_data_pipeline_draft_kind.up.sql b/backend/migrations/20260616090412_add_data_pipeline_draft_kind.up.sql new file mode 100644 index 0000000000..38ae68ddcb --- /dev/null +++ b/backend/migrations/20260616090412_add_data_pipeline_draft_kind.up.sql @@ -0,0 +1,6 @@ +-- A `data_pipeline` draft bundles every unsaved pipeline script of a folder +-- into a single row keyed at the folder path (typ has no deployed backing +-- table — see UserDraftItemKind::deployed_table). Lets the asset-graph view +-- store its in-flight drafts in the per-user DB draft sync instead of +-- browser-local storage. +ALTER TYPE DRAFT_KIND ADD VALUE IF NOT EXISTS 'data_pipeline'; diff --git a/backend/parsers/windmill-parser-py-asset/src/lib.rs b/backend/parsers/windmill-parser-py-asset/src/lib.rs index 10b93a998b..e8a0a1cc40 100644 --- a/backend/parsers/windmill-parser-py-asset/src/lib.rs +++ b/backend/parsers/windmill-parser-py-asset/src/lib.rs @@ -2,8 +2,8 @@ use rustpython_ast::{Constant, Expr, ExprConstant, Visitor}; use rustpython_parser::{ast::Suite, Parse}; use std::collections::HashMap; use windmill_parser::asset_parser::{ - asset_was_used, merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType, - ParseAssetsOutput, ParseAssetsResult, + asset_was_used, merge_assets, parse_asset_syntax, parse_pipeline_annotations, AssetKind, + AssetUsageAccessType, ParseAssetsOutput, ParseAssetsResult, }; use AssetUsageAccessType::*; @@ -28,7 +28,12 @@ pub fn parse_assets(input: &str) -> anyhow::Result { } } - Ok(ParseAssetsOutput { assets: merge_assets(assets_finder.assets), ..Default::default() }) + let pipeline = parse_pipeline_annotations(input); + Ok(ParseAssetsOutput::new( + merge_assets(assets_finder.assets), + Vec::new(), + pipeline, + )) } type VarAssetName = String; diff --git a/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs b/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs index 002ed78fc0..812bfa785d 100644 --- a/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-sql-asset/src/asset_parser.rs @@ -9,8 +9,8 @@ use sqlparser::{ parser::Parser, }; use windmill_parser::asset_parser::{ - asset_was_used, merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType, - ParseAssetsOutput, ParseAssetsResult, + asset_was_used, merge_assets, parse_asset_syntax, parse_pipeline_annotations, AssetKind, + AssetUsageAccessType, ParseAssetsOutput, ParseAssetsResult, }; use AssetUsageAccessType::*; @@ -33,7 +33,12 @@ pub fn parse_assets(input: &str) -> anyhow::Result { } } - Ok(ParseAssetsOutput { assets: merge_assets(collector.assets), ..Default::default() }) + let pipeline = parse_pipeline_annotations(input); + Ok(ParseAssetsOutput::new( + merge_assets(collector.assets), + Vec::new(), + pipeline, + )) } /// Visitor that collects S3 asset literals from SQL statements @@ -260,6 +265,21 @@ impl AssetCollector { } } + // Collect the table-level reads (and column assets) of a query's top + // SELECT. Table-level reads are only gathered here and in the statement + // arms — the generic table-factor visitor picks up read-functions and + // string literals, not plain `FROM
` references. Called for both + // standalone SELECTs and the `AS SELECT` of CTAS / CREATE VIEW. + fn handle_query_reads(&mut self, query: &sqlparser::ast::Query) { + self.cte_name_stack.push(collect_cte_names(query)); + if let Some(select) = query.body.as_select() { + for t in &select.from { + self.handle_table_with_joins(t, Some(R)); + } + self.extract_column_assets(&select.projection, &select.from); + } + } + fn handle_table_with_joins( &mut self, table_with_joins: &sqlparser::ast::TableWithJoins, @@ -485,15 +505,7 @@ impl Visitor for AssetCollector { ) -> std::ops::ControlFlow { match statement { sqlparser::ast::Statement::Query(q) => { - self.cte_name_stack.push(collect_cte_names(q)); - if let Some(select) = q.body.as_select() { - // First, handle table references (adds table-level assets) - for t in &select.from { - self.handle_table_with_joins(t, Some(R)); - } - // Then, extract column-level assets - self.extract_column_assets(&select.projection, &select.from); - } + self.handle_query_reads(q); } sqlparser::ast::Statement::Insert(insert) => { @@ -647,10 +659,37 @@ impl Visitor for AssetCollector { sqlparser::ast::Statement::CreateTable(create_table) => { self.track_table_definition(&create_table.name); + // `CREATE TABLE x AS SELECT … FROM y` reads y. The AS-query + // isn't a `Statement::Query`, so its FROM tables are only + // caught here. + if let Some(query) = &create_table.query { + self.handle_query_reads(query); + } } - sqlparser::ast::Statement::CreateView { name, .. } => { + sqlparser::ast::Statement::CreateView { name, query, .. } => { self.track_table_definition(name); + self.handle_query_reads(query); + } + + // DROP TABLE/VIEW is a write to the dropped object — the + // canonical idempotent-refresh pattern (`DROP TABLE IF EXISTS x; + // CREATE TABLE x AS …`) must resolve to a table-level write. + // Without this arm, a tagged-template snippet containing only the + // DROP yields no table-level asset and the TS/Python SDK parsers + // fall back to a db-level `datatable://` reference, putting a + // stray database node on the pipeline canvas. + sqlparser::ast::Statement::Drop { object_type, names, .. } => { + if matches!( + object_type, + sqlparser::ast::ObjectType::Table + | sqlparser::ast::ObjectType::View + | sqlparser::ast::ObjectType::MaterializedView + ) { + for name in names { + self.track_table_definition(name); + } + } } sqlparser::ast::Statement::Copy { target: CopyTarget::File { filename }, .. } => { @@ -702,7 +741,16 @@ impl Visitor for AssetCollector { &mut self, statement: &sqlparser::ast::Statement, ) -> std::ops::ControlFlow { - if matches!(statement, sqlparser::ast::Statement::Query(_)) { + // Balance the push done by handle_query_reads (called from the Query, + // CreateView, and CTAS arms). + let pushed = match statement { + sqlparser::ast::Statement::Query(_) | sqlparser::ast::Statement::CreateView { .. } => { + true + } + sqlparser::ast::Statement::CreateTable(ct) => ct.query.is_some(), + _ => false, + }; + if pushed { self.cte_name_stack.pop(); } std::ops::ControlFlow::Continue(()) @@ -926,6 +974,61 @@ mod tests { ); } + #[test] + fn test_sql_asset_parser_drop_table_is_write() { + // A lone DROP (e.g. one SDK tagged-template snippet of an + // idempotent-refresh script) must resolve to a table-level write, + // not fall through to nothing. + let input = r#" + ATTACH 'datatable://main' AS dt; USE dt; + DROP TABLE IF EXISTS orders_raw; + "#; + let s = parse_assets(input).map(|s| s.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ParseAssetsResult { + kind: AssetKind::DataTable, + path: "main/orders_raw".to_string(), + access_type: Some(W), + columns: None + },]) + ); + } + + #[test] + fn test_sql_asset_parser_drop_then_create_qualified() { + // Canonical refresh pattern over an attached catalog: DROP + CREATE + // AS on the output, SELECT on the input. + let input = r#" + ATTACH 'datatable://main' AS pg; + DROP TABLE IF EXISTS pg.daily_revenue; + CREATE TABLE pg.daily_revenue AS SELECT * FROM pg.orders_clean; + "#; + let s = parse_assets(input).map(|mut s| { + s.assets.sort_by(|a, b| a.path.cmp(&b.path)); + s.assets + }); + // The DROP+CTAS combo yields a clean write of the output plus the + // read of the CTAS source (`SELECT * FROM pg.orders_clean`). + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ + ParseAssetsResult { + kind: AssetKind::DataTable, + path: "main/daily_revenue".to_string(), + access_type: Some(W), + columns: None + }, + ParseAssetsResult { + kind: AssetKind::DataTable, + path: "main/orders_clean".to_string(), + access_type: Some(R), + columns: None + }, + ]) + ); + } + // Make sure a_function is not detected as main/a_function #[test] fn test_sql_asset_parser_function_table() { @@ -1796,3 +1899,45 @@ mod tests { assert_eq!(columns.get("age"), Some(&R)); } } + +#[cfg(test)] +mod ctas_read_tests { + use super::*; + + #[test] + fn test_ctas_collects_upstream_read() { + let input = r#" + ATTACH 'datatable://main' AS pg; + CREATE TABLE IF NOT EXISTS pg.exciting_809 AS + SELECT * FROM pg.fx_rates; + "#; + let assets = parse_assets(input).unwrap().assets; + assert!( + assets.iter().any(|a| a.path == "main/fx_rates" + && a.kind == AssetKind::DataTable + && a.access_type == Some(R)), + "expected read of main/fx_rates, got {:?}", + assets + ); + assert!( + assets.iter().any(|a| a.path == "main/exciting_809" + && a.access_type == Some(W)), + "expected write of main/exciting_809, got {:?}", + assets + ); + } + + #[test] + fn test_create_view_collects_upstream_read() { + let input = r#" + ATTACH 'datatable://main' AS pg; + CREATE VIEW pg.v AS SELECT * FROM pg.fx_rates; + "#; + let assets = parse_assets(input).unwrap().assets; + assert!( + assets.iter().any(|a| a.path == "main/fx_rates" && a.access_type == Some(R)), + "expected read of main/fx_rates, got {:?}", + assets + ); + } +} diff --git a/backend/parsers/windmill-parser-ts-asset/src/lib.rs b/backend/parsers/windmill-parser-ts-asset/src/lib.rs index 39302fb6fa..b7fe0fea5c 100644 --- a/backend/parsers/windmill-parser-ts-asset/src/lib.rs +++ b/backend/parsers/windmill-parser-ts-asset/src/lib.rs @@ -5,8 +5,8 @@ use swc_ecma_ast::{CallExpr, Expr, Lit, MemberExpr, MemberProp, ObjectLit, Prop, use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax, TsSyntax}; use swc_ecma_visit::{Visit, VisitWith}; use windmill_parser::asset_parser::{ - asset_was_used, merge_assets, parse_asset_syntax, AssetKind, AssetUsageAccessType, - ParseAssetsOutput, ParseAssetsResult, SqlQueryDetails, + asset_was_used, merge_assets, parse_asset_syntax, parse_pipeline_annotations, AssetKind, + AssetUsageAccessType, ParseAssetsOutput, ParseAssetsResult, SqlQueryDetails, }; use AssetUsageAccessType::*; @@ -38,10 +38,12 @@ pub fn parse_assets(code: &str) -> anyhow::Result { let mut assets_finder = AssetsFinder { assets: vec![], sql_queries: vec![], var_identifiers: HashMap::new() }; assets_finder.visit_module_items(&ast); - Ok(ParseAssetsOutput { - assets: merge_assets(assets_finder.assets), - sql_queries: assets_finder.sql_queries, - }) + let pipeline = parse_pipeline_annotations(code); + Ok(ParseAssetsOutput::new( + merge_assets(assets_finder.assets), + assets_finder.sql_queries, + pipeline, + )) } type VarAssetName = String; diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index cd05b53b71..304623998d 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -174,11 +174,10 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "ast_node" -version = "0.9.9" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9184f2b369b3e8625712493c89b785881f27eedc6cde480a81883cef78868b2" +checksum = "0a184645bcc6f52d69d8e7639720699c6a99efb711f886e251ed1d16db8dd90e" dependencies = [ - "proc-macro2", "quote", "swc_macros_common", "syn 2.0.117", @@ -401,9 +400,9 @@ checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" [[package]] name = "better_scoped_tls" -version = "0.1.2" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "297b153aa5e573b5863108a6ddc9d5c968bd0b20e75cc614ee9821d2f45679c7" +checksum = "7cd228125315b132eed175bf47619ac79b945b26e56b848ba203ae4ea8603609" dependencies = [ "scoped-tls", ] @@ -563,9 +562,6 @@ name = "bumpalo" version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" -dependencies = [ - "allocator-api2", -] [[package]] name = "byte-unit" @@ -613,6 +609,16 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +[[package]] +name = "bytes-str" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "577d2bf5650f8554d5a372af5ac93535110a0fc75b3e702bb853369febf227c2" +dependencies = [ + "bytes", + "serde", +] + [[package]] name = "bytesize" version = "1.3.3" @@ -1372,11 +1378,10 @@ dependencies = [ [[package]] name = "from_variant" -version = "0.1.9" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32016f1242eb82af5474752d00fd8ebcd9004bd69b462b1c91de833972d08ed4" +checksum = "308530a56b099da144ebc5d8e179f343ad928fa2b3558d1eb3db9af18d6eff43" dependencies = [ - "proc-macro2", "swc_macros_common", "syn 2.0.117", ] @@ -1751,15 +1756,14 @@ dependencies = [ [[package]] name = "hstr" -version = "0.2.17" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1a26def229ea95a8709dad32868d975d0dd40235bd2ce82920e4a8fe692b5e0" +checksum = "31f11d91d7befd2ffd9d216e9e5ea1fae6174b20a2a1b67a688138003d2f4122" dependencies = [ "hashbrown 0.14.5", "new_debug_unreachable", "once_cell", - "phf 0.11.3", - "rustc-hash 1.1.0", + "rustc-hash 2.1.1", "triomphe", ] @@ -4136,10 +4140,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] -name = "serde" -version = "1.0.220" +name = "seq-macro" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ceecad4c782e936ac90ecfd6b56532322e3262b14320abf30ce89a92ffdbfe22" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ "serde_core", "serde_derive", @@ -4158,18 +4168,18 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.220" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddba47394f3b862d6ff6efdbd26ca4673e3566a307880a0ffb98f274bbe0ec32" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.220" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60e1f3b1761e96def5ec6d04a6e7421c0404fa3cf5c0155f1e2848fae3d8cc08" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", @@ -4692,11 +4702,10 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "string_enum" -version = "0.4.4" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05e383308aebc257e7d7920224fa055c632478d92744eca77f99be8fa1545b90" +checksum = "ae36a4951ca7bd1cfd991c241584a9824a70f6aff1e7d4f693fb3f2465e4030e" dependencies = [ - "proc-macro2", "quote", "swc_macros_common", "syn 2.0.117", @@ -4798,49 +4807,35 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7401a30af6cb5818bb64852270bb722533397edcfc7344954a38f420819ece2" -[[package]] -name = "swc_allocator" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76aa0eb65c0f39f9b6d82a7e5192c30f7ac9a78f084a21f270de1d8c600ca388" -dependencies = [ - "bumpalo", - "hashbrown 0.14.5", - "ptr_meta", - "rustc-hash 1.1.0", - "triomphe", -] - [[package]] name = "swc_atoms" -version = "0.6.7" +version = "7.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb6567e4e67485b3e7662b486f1565bdae54bd5b9d6b16b2ba1a9babb1e42125" +checksum = "3500dcf04c84606b38464561edc5e46f5132201cb3e23cf9613ed4033d6b1bb2" dependencies = [ "hstr", "once_cell", - "rustc-hash 1.1.0", "serde", ] [[package]] name = "swc_common" -version = "0.37.5" +version = "14.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12d0a8eaaf1606c9207077d75828008cb2dfb51b095a766bd2b72ef893576e31" +checksum = "c2bb772b3a26b8b71d4e8c112ced5b5867be2266364b58517407a270328a2696" dependencies = [ + "anyhow", "ast_node", "better_scoped_tls", - "cfg-if", + "bytes-str", "either", "from_variant", "new_debug_unreachable", "num-bigint", "once_cell", - "rustc-hash 1.1.0", + "rustc-hash 2.1.1", "serde", "siphasher 0.3.11", - "swc_allocator", "swc_atoms", "swc_eq_ignore_macros", "swc_visit", @@ -4851,32 +4846,36 @@ dependencies = [ [[package]] name = "swc_ecma_ast" -version = "0.118.2" +version = "15.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6f866d12e4d519052b92a0a86d1ac7ff17570da1272ca0c89b3d6f802cd79df" +checksum = "65c25af97d53cf8aab66a6c68f3418663313fc969ad267fc2a4d19402c329be1" dependencies = [ "bitflags", "is-macro", "num-bigint", + "once_cell", "phf 0.11.3", - "scoped-tls", + "rustc-hash 2.1.1", "string_enum", "swc_atoms", "swc_common", + "swc_visit", "unicode-id-start", ] [[package]] -name = "swc_ecma_parser" -version = "0.149.1" +name = "swc_ecma_lexer" +version = "23.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683dada14722714588b56481399c699378b35b2ba4deb5c4db2fb627a97fb54b" +checksum = "017d06ea85008234aa9fb34d805c7dc563f2ea6e03869ed5ac5a2dc27d561e4d" dependencies = [ + "arrayvec", + "bitflags", "either", - "new_debug_unreachable", "num-bigint", - "num-traits", "phf 0.11.3", + "rustc-hash 2.1.1", + "seq-macro", "serde", "smallvec", "smartstring", @@ -4885,14 +4884,29 @@ dependencies = [ "swc_common", "swc_ecma_ast", "tracing", - "typed-arena", +] + +[[package]] +name = "swc_ecma_parser" +version = "24.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9011783c975ba592ffc09cd208ced92b1dfabb2e5e0ef453559e2e25286127" +dependencies = [ + "either", + "num-bigint", + "serde", + "swc_atoms", + "swc_common", + "swc_ecma_ast", + "swc_ecma_lexer", + "tracing", ] [[package]] name = "swc_ecma_visit" -version = "0.104.8" +version = "15.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b1c6802e68e51f336e8bc9644e9ff9da75d7da9c1a6247d532f2e908aa33e81" +checksum = "75a579aa8f9e212af521588df720ccead079c09fe5c8f61007cf724324aed3a0" dependencies = [ "new_debug_unreachable", "num-bigint", @@ -4905,9 +4919,9 @@ dependencies = [ [[package]] name = "swc_eq_ignore_macros" -version = "0.1.4" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63db0adcff29d220c3d151c5b25c0eabe7e32dd936212b84cdaa1392e3130497" +checksum = "c16ce73424a6316e95e09065ba6a207eba7765496fed113702278b7711d4b632" dependencies = [ "proc-macro2", "quote", @@ -4916,9 +4930,9 @@ dependencies = [ [[package]] name = "swc_macros_common" -version = "0.3.14" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27e18fbfe83811ffae2bb23727e45829a0d19c6870bced7c0f545cc99ad248dd" +checksum = "aae1efbaa74943dc5ad2a2fb16cbd78b77d7e4d63188f3c5b4df2b4dcd2faaae" dependencies = [ "proc-macro2", "quote", @@ -4927,9 +4941,9 @@ dependencies = [ [[package]] name = "swc_visit" -version = "0.6.2" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ceb044142ba2719ef9eb3b6b454fce61ab849eb696c34d190f04651955c613d" +checksum = "62fb71484b486c185e34d2172f0eabe7f4722742aad700f426a494bb2de232a2" dependencies = [ "either", "new_debug_unreachable", @@ -5065,7 +5079,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", "rustix 1.1.4", "windows-sys 0.61.2", @@ -5595,12 +5609,6 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "typed-arena" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" - [[package]] name = "typeid" version = "1.0.3" @@ -6205,6 +6213,7 @@ dependencies = [ "croner", "dashmap", "equivalent", + "erased-serde", "futures", "futures-core", "gethostname", diff --git a/backend/parsers/windmill-parser-yaml/src/asset_parser.rs b/backend/parsers/windmill-parser-yaml/src/asset_parser.rs index 7e67d563ff..4dde12ad8e 100644 --- a/backend/parsers/windmill-parser-yaml/src/asset_parser.rs +++ b/backend/parsers/windmill-parser-yaml/src/asset_parser.rs @@ -1,5 +1,6 @@ use windmill_parser::asset_parser::{ - merge_assets, AssetKind, AssetUsageAccessType, ParseAssetsOutput, ParseAssetsResult, + merge_assets, parse_pipeline_annotations, AssetKind, AssetUsageAccessType, ParseAssetsOutput, + ParseAssetsResult, }; use crate::{parse_ansible_reqs, ResourceOrVariablePath}; @@ -39,5 +40,10 @@ pub fn parse_assets(input: &str) -> anyhow::Result { } } - Ok(ParseAssetsOutput { assets: merge_assets(assets), ..Default::default() }) + let pipeline = parse_pipeline_annotations(input); + Ok(ParseAssetsOutput::new( + merge_assets(assets), + Vec::new(), + pipeline, + )) } diff --git a/backend/parsers/windmill-parser/src/asset_parser.rs b/backend/parsers/windmill-parser/src/asset_parser.rs index 4239c1854c..ce0be81f00 100644 --- a/backend/parsers/windmill-parser/src/asset_parser.rs +++ b/backend/parsers/windmill-parser/src/asset_parser.rs @@ -1,6 +1,15 @@ use serde::Serialize; use std::collections::BTreeMap; +// Token recognized inside declared asset URIs that the runtime substitutes +// with the current partition value (e.g. `s3://lake/{partition}.parquet` +// becomes `s3://lake/2024-04-29.parquet` when materialized for that day). +// Substitution is the parser's job only at lineage/graph time — at runtime +// the pipeline worker performs the actual replacement before emitting +// asset signals. Kept as a single well-known token so the parser never +// has to guess which `{...}` placeholders are partition variables. +pub const PARTITION_TOKEN: &str = "{partition}"; + #[derive(Serialize, PartialEq, Clone, Copy, Debug)] #[serde(rename_all(serialize = "lowercase"))] pub enum AssetUsageAccessType { @@ -47,6 +56,214 @@ pub struct SqlQueryDetails { pub struct ParseAssetsOutput { pub assets: Vec, pub sql_queries: Vec, + // Bare `// pipeline` (or `#` / `--`) on its own line — opt-in marker + // that sets auto_kind='pipeline' and includes the script in its + // folder's pipeline. Pipeline membership is broader than + // materialization: scripts that only assert/notify/clean up are + // members too. Outputs (when present) come from parser-detected + // `w`/`rw` usages in `assets`, not from this marker. + #[serde(skip_serializing_if = "std::ops::Not::not", default)] + pub in_pipeline: bool, + // Trigger annotations — execution DAG edges. Each is an independent OR + // (any fires the script). Empty = script has no automatic triggers + // (still runnable manually / via existing cron triggers). Includes both + // top-level `// schedule "..."` and `// on ...` forms. + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub triggers: Vec, + // `// partitioned [opts]` — declares that this pipeline script + // produces partitioned output. The runtime resolves `{partition}` in + // declared output URIs to the current partition value before signaling + // downstream consumers. At most one per script. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub partition: Option, + // `// freshness ` — SLA stating outputs must be at most + // `duration` old. Active backstop: when no other trigger has fired the + // script within the window, a watchdog re-runs it. Distinct from + // schedule (which is producer cadence); freshness is consumer SLA and + // applies regardless of which trigger last fired. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub freshness: Option, + // `// trigger all` → AND join barrier; default (`any`) = OR (current + // behaviour). Threaded to the deploy path which persists it on the + // subscriber's trigger rows. + #[serde(skip_serializing_if = "JoinMode::is_any", default)] + pub join_mode: JoinMode, + // `// debounce ` — script-level default debounce window for this + // script's asset inputs. A per-`// on … debounce=` overrides it. Raw + // duration string, parsed to seconds at deploy (parser-light, like + // freshness). Absent = no debounce (fan-out, current behaviour). + #[serde(skip_serializing_if = "Option::is_none", default)] + pub debounce_default: Option, + // `// tag ` — overrides the script's worker tag at deploy. Source + // wins over any UI-set value, matching the wipe-and-reinsert convention + // of other pipeline annotations (`// schedule`, `// on`). Absent = keep + // whatever the caller (UI / CLI) supplied. + #[serde(skip_serializing_if = "Option::is_none", default)] + pub tag: Option, + // `// retry []` — re-run a pipeline-cascade triggered + // script up to `count` times on failure, waiting `delay` between + // attempts. Applies only to runs launched via the asset/schedule + // cascade (the rows in `script_trigger`); manual UI runs are unaffected. + // The delay is a raw duration string parsed at deploy (parser-light). + #[serde(skip_serializing_if = "Option::is_none", default)] + pub retry: Option, +} + +#[derive(Serialize, Debug, PartialEq, Clone)] +#[serde(tag = "kind", rename_all = "lowercase")] +pub enum TriggerSpec { + // Refresh when `` changes. Kind comes from parse_asset_syntax so + // it matches the `asset` table. `debounce` is the optional per-input + // `// on … debounce=` override (raw duration string); it takes + // precedence over the script-level `// debounce` default, resolved at + // deploy. + Asset { + asset_kind: AssetKind, + path: String, + #[serde(skip_serializing_if = "Option::is_none", default)] + debounce: Option, + }, + // `// on ` — marker-only declaration that this script wants to be + // triggered by a native trigger of the given kind. No path: the binding + // is the trigger row's own `script_path` field (set when the user creates + // the kafka/mqtt/schedule/… trigger in its dedicated UI). The graph + // endpoint discovers attached triggers by `WHERE script_path = ` and surfaces a "missing" placeholder when an annotation has no + // matching row. + Schedule, + Webhook, + Email, + Kafka, + Mqtt, + Nats, + Postgres, + Sqs, + Gcp, + // `// on data_upload` — UI-first entry point. Unlike the other native + // kinds there is no external event source and no trigger row anywhere: + // the script declares an `S3Object` input parameter and the user uploads + // a file via the auto-generated S3 picker, which runs the pipeline. The + // graph renders it as a clickable upload source (never a "missing" + // placeholder, mirroring webhook). + #[serde(rename = "data_upload")] + DataUpload, +} + +impl TriggerSpec { + // A `// on ` whose declared path contains the `{partition}` + // token is *partition-bearing*: in an AND join its concrete partition + // value is the join key. Non-asset triggers and assets without the + // token are reference/presence-only inputs that never define the + // partition (the case-3 guard). + pub fn is_partition_bearing(&self) -> bool { + matches!(self, TriggerSpec::Asset { path, .. } if path.contains(PARTITION_TOKEN)) + } +} + +// Partitioning declaration for a pipeline script. `daily`/`hourly`/`weekly`/ +// `monthly` are time-based with the runtime supplying the current +// partition value derived from the trigger context (schedule fire time, +// freshness window, manual run arg). `dynamic` extracts the value from the +// triggering payload via JSONPath — used for per-tenant / per-shard / +// per-event-id pipelines where the partition key isn't a wall-clock value. +#[derive(Serialize, Debug, PartialEq, Clone)] +#[serde(tag = "kind", rename_all = "lowercase")] +pub enum PartitionKind { + Daily, + Hourly, + Weekly, + Monthly, + Dynamic { key: String }, +} + +#[derive(Serialize, Debug, PartialEq, Clone)] +pub struct PartitionSpec { + #[serde(flatten)] + pub kind: PartitionKind, + #[serde(skip_serializing_if = "Option::is_none")] + pub tz: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub format: Option, + // ISO-8601 (e.g. "2024-01-01") or kind-specific start anchor. Older + // partitions before this anchor are not backfilled. + #[serde(skip_serializing_if = "Option::is_none")] + pub start: Option, +} + +// Freshness SLA. The duration is kept as a raw string ("1h", "30m", "2d") +// and validated downstream — the parser deliberately doesn't bind to a +// specific duration crate so the annotation grammar stays parser-light. +#[derive(Serialize, Debug, PartialEq, Clone)] +pub struct FreshnessSpec { + pub duration: String, +} + +// Retry policy declared via `// retry []`. The delay is kept +// as a raw duration string (mirrors freshness/debounce_default) and resolved +// to seconds at deploy via `parse_duration_secs`; absent = back-to-back +// re-runs with no inter-attempt wait. +#[derive(Serialize, Debug, PartialEq, Clone)] +pub struct RetrySpec { + pub count: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub delay: Option, +} + +// `// trigger any` (default) vs `// trigger all`. `Any` = OR: any trigger +// firing runs the script (current behaviour). `All` = AND: the script +// runs only once every partition-bearing input has materialized at the +// same partition (plus every reference input exists) — the join barrier. +#[derive(Serialize, Debug, PartialEq, Eq, Clone, Copy, Default)] +#[serde(rename_all = "lowercase")] +pub enum JoinMode { + #[default] + Any, + All, +} + +impl JoinMode { + pub fn is_any(&self) -> bool { + matches!(self, JoinMode::Any) + } +} + +// All pipeline-level annotations parsed off a script's source. Returned by +// `parse_pipeline_annotations` and forwarded into `ParseAssetsOutput`. +#[derive(Default, Debug, PartialEq, Clone)] +pub struct PipelineAnnotations { + pub in_pipeline: bool, + pub triggers: Vec, + pub partition: Option, + pub freshness: Option, + pub join_mode: JoinMode, + pub debounce_default: Option, + pub tag: Option, + pub retry: Option, +} + +impl ParseAssetsOutput { + /// Build from detected assets/queries plus the script's parsed + /// pipeline annotations, so each language asset-parser does not + /// re-list the per-annotation fields (one call site instead of six + /// lines kept in lockstep across the parser crates). + pub fn new( + assets: Vec, + sql_queries: Vec, + pipeline: PipelineAnnotations, + ) -> Self { + ParseAssetsOutput { + assets, + sql_queries, + in_pipeline: pipeline.in_pipeline, + triggers: pipeline.triggers, + partition: pipeline.partition, + freshness: pipeline.freshness, + join_mode: pipeline.join_mode, + debounce_default: pipeline.debounce_default, + tag: pipeline.tag, + retry: pipeline.retry, + } + } } #[derive(Debug, Clone, Serialize)] @@ -64,13 +281,12 @@ pub fn merge_assets(assets: Vec) -> Vec { .iter_mut() .find(|x| x.path == asset.path && x.kind == asset.kind) { - // merge access types + // merge access types — a None on either side means ambiguous + // usage (unknown access), which poisons the merge to None; + // otherwise delegate to the shared truth table. existing.access_type = match (asset.access_type, existing.access_type) { (None, _) | (_, None) => None, - (Some(R), Some(W)) | (Some(W), Some(R)) => Some(RW), - (Some(RW), _) | (_, Some(RW)) => Some(RW), - (Some(R), Some(R)) => Some(R), - (Some(W), Some(W)) => Some(W), + (Some(a), Some(b)) => Some(merge_access_types(a, b)), }; // merge columns: union the column sets and merge access types per column existing.columns = merge_column_maps(existing.columns.take(), asset.columns); @@ -153,3 +369,743 @@ pub const ASSET_KINDS: &[(&str, AssetKind)] = &[ ("datatable://", AssetKind::DataTable), ("volume://", AssetKind::Volume), ]; + +// Tokenize a `key=value [key="quoted value"] ...` option string. Bare +// values run until the next whitespace; quoted values consume until the +// matching quote. Malformed pairs (missing `=` or empty key) are skipped +// rather than aborting the whole annotation. +// Split a `// on` right-hand side into the trigger ref and any trailing +// `key=value` opts. The opts section starts at the first whitespace token +// shaped like `=…` (e.g. `debounce=60s`); everything before is the +// asset/kind ref. Asset refs aren't expected to contain a space then an +// `ident=` token — the same assumption `// partitioned` already makes. +fn split_trailing_kv_opts(s: &str) -> (&str, BTreeMap) { + let mut split_at: Option = None; + for tok in s.split_whitespace() { + // `split_whitespace` yields slices borrowed from `s`, so the exact + // byte offset is the pointer delta — substring search (`find`) would + // misfire when an earlier token also appears inside a later one. + let tok_start = tok.as_ptr() as usize - s.as_ptr() as usize; + if let Some(eq) = tok.find('=') { + let key = &tok[..eq]; + if !key.is_empty() + && key.starts_with(|c: char| c.is_ascii_alphabetic() || c == '_') + && key.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') + { + split_at = Some(tok_start); + break; + } + } + } + match split_at { + Some(i) => (s[..i].trim_end(), parse_kv_opts(&s[i..])), + None => (s.trim_end(), BTreeMap::new()), + } +} + +fn parse_kv_opts(s: &str) -> BTreeMap { + let mut out = BTreeMap::new(); + let mut chars = s.chars().peekable(); + loop { + while chars.peek().map_or(false, |c| c.is_whitespace()) { + chars.next(); + } + if chars.peek().is_none() { + break; + } + let mut key = String::new(); + while let Some(&c) = chars.peek() { + if c == '=' || c.is_whitespace() { + break; + } + key.push(c); + chars.next(); + } + if chars.peek() != Some(&'=') || key.is_empty() { + // Malformed — skip until next whitespace to recover. + while chars.peek().map_or(false, |c| !c.is_whitespace()) { + chars.next(); + } + continue; + } + chars.next(); // consume '=' + let value = match chars.peek().copied() { + Some(q @ ('"' | '\'')) => { + chars.next(); + let mut v = String::new(); + while let Some(&c) = chars.peek() { + chars.next(); + if c == q { + break; + } + v.push(c); + } + v + } + _ => { + let mut v = String::new(); + while let Some(&c) = chars.peek() { + if c.is_whitespace() { + break; + } + v.push(c); + chars.next(); + } + v + } + }; + out.insert(key, value); + } + out +} + +// Scan raw source for pipeline annotations. Language-agnostic: any line +// whose first non-whitespace tokens are a comment prefix (`//`, `#`, or +// `--`) followed by one of the recognized keywords: +// - `pipeline` → opt-in marker (must be alone on the line) +// - `on ` → asset / native trigger edge (including +// the marker-only `on schedule` form) +// - `partitioned [opts]` → partition declaration +// - `freshness ` → SLA / active backstop +// - `tag ` → worker-tag override (annotation wins +// over UI-set value at deploy) +// - `retry []` → cascade-only retry policy +// +// `// pipeline` is intentionally strict — only whitespace allowed after the +// keyword. Without that constraint, casual prose like `// pipeline broken +// on staging` would false-positive (the word "pipeline" is far more common +// in normal comments than "materialize" was). +// +// `partition`, `freshness`, `tag`, and `retry` use first-write-wins; if +// multiple lines declare them, the first one is kept (last would be +// reasonable too, but first matches the file-top convention developers +// follow). +// Try to consume `` as a complete word from `rest`. Returns the trailing +// text after the keyword if it matched (empty or whitespace-bounded), +// `None` otherwise. Prevents `partitioned` matching `partition`, `pipelines` +// matching `pipeline`, etc. Mirrors `consumeKeyword` in +// parsePipelineAnnotations.ts. +fn consume_keyword<'a>(rest: &'a str, kw: &str) -> Option<&'a str> { + let after = rest.strip_prefix(kw)?; + if after.is_empty() || after.starts_with(|c: char| c.is_whitespace()) { + Some(after) + } else { + None + } +} + +pub fn parse_pipeline_annotations(code: &str) -> PipelineAnnotations { + let mut out = PipelineAnnotations::default(); + + for raw_line in code.lines() { + let line = raw_line.trim_start(); + let rest = if let Some(r) = line.strip_prefix("//") { + r + } else if let Some(r) = line.strip_prefix("--") { + r + } else if let Some(r) = line.strip_prefix('#') { + r + } else { + continue; + }; + let rest = rest.trim_start(); + + if let Some(after_kw) = consume_keyword(rest, "pipeline") { + // Strict: keyword must be the only content on the line. Rejects + // `pipeline broken`, `pipelines`, `pipeline-related`, etc. + if after_kw.trim().is_empty() { + out.in_pipeline = true; + } + continue; + } + + if let Some(after_kw) = consume_keyword(rest, "partitioned") { + if out.partition.is_none() { + if let Some(spec) = parse_partitioned_spec(after_kw.trim()) { + out.partition = Some(spec); + } + } + continue; + } + + if let Some(after_kw) = consume_keyword(rest, "freshness") { + let dur = after_kw.trim(); + if !dur.is_empty() && out.freshness.is_none() { + out.freshness = Some(FreshnessSpec { duration: dur.to_string() }); + } + continue; + } + + if let Some(after_kw) = consume_keyword(rest, "trigger") { + match after_kw.trim() { + "all" => out.join_mode = JoinMode::All, + "any" => out.join_mode = JoinMode::Any, + // Unknown value — leave the default rather than guess. + _ => {} + } + continue; + } + + if let Some(after_kw) = consume_keyword(rest, "debounce") { + let dur = after_kw.trim(); + if !dur.is_empty() && out.debounce_default.is_none() { + out.debounce_default = Some(dur.to_string()); + } + continue; + } + + if let Some(after_kw) = consume_keyword(rest, "tag") { + let name = after_kw.trim(); + if !name.is_empty() && out.tag.is_none() { + out.tag = Some(name.to_string()); + } + continue; + } + + if let Some(after_kw) = consume_keyword(rest, "retry") { + if out.retry.is_none() { + if let Some(spec) = parse_retry_spec(after_kw.trim()) { + out.retry = Some(spec); + } + } + continue; + } + + if let Some(after_kw) = consume_keyword(rest, "on") { + let spec_text = after_kw.trim(); + if spec_text.is_empty() { + continue; + } + // Split off trailing `key=value` opts (e.g. `debounce=60s`) + // from the asset/kind ref. The per-input debounce override is + // only meaningful for asset inputs (cascade fan-out); other + // trigger kinds ignore it. + let (ref_part, opts) = split_trailing_kv_opts(spec_text); + if let Some(mut trig) = parse_trigger_spec(ref_part) { + if let TriggerSpec::Asset { debounce, .. } = &mut trig { + *debounce = opts + .get("debounce") + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty()); + } + if !out.triggers.contains(&trig) { + out.triggers.push(trig); + } + } + } + } + + out +} + +// Parse a `// retry []` right-hand side. `` is a +// non-negative decimal; `` is an optional raw duration string left +// for `parse_duration_secs` to validate at deploy. A bare zero count (or +// non-numeric token) is rejected — that is, the annotation must encode a +// real retry policy or be omitted, so a typo (`// retry retry 3`) fails +// safe rather than silently disabling cascade retries. +fn parse_retry_spec(s: &str) -> Option { + let mut split = s.splitn(2, char::is_whitespace); + let count_word = split.next()?.trim(); + let count: u32 = count_word.parse().ok()?; + if count == 0 { + return None; + } + let delay = split + .next() + .map(|d| d.trim().to_string()) + .filter(|d| !d.is_empty()); + Some(RetrySpec { count, delay }) +} + +// Parse a `// partitioned [opts]` right-hand side. Recognized kinds: +// `daily`, `hourly`, `weekly`, `monthly` (with optional tz/format/start), +// and `dynamic key=""` (plus optional format). +fn parse_partitioned_spec(s: &str) -> Option { + let mut split = s.splitn(2, char::is_whitespace); + let kind_word = split.next()?; + let opts_str = split.next().unwrap_or(""); + let opts = parse_kv_opts(opts_str); + let kind = match kind_word { + "daily" => PartitionKind::Daily, + "hourly" => PartitionKind::Hourly, + "weekly" => PartitionKind::Weekly, + "monthly" => PartitionKind::Monthly, + "dynamic" => { + let key = opts.get("key")?.clone(); + if key.is_empty() { + return None; + } + PartitionKind::Dynamic { key } + } + _ => return None, + }; + Some(PartitionSpec { + kind, + tz: opts.get("tz").cloned(), + format: opts.get("format").cloned(), + start: opts.get("start").cloned(), + }) +} + +// Parse a single `on ` right-hand side. Accepted forms: +// — where is one of +// webhook | email | kafka | mqtt | nats | postgres | sqs | gcp +// (e.g. s3://bucket/key, $res:f/foo) +// +// Native trigger keywords are *marker-only* — no trailing path. The actual +// binding lives on the native trigger row (`script_path` column). Anything +// trailing the keyword is rejected so the form stays unambiguous. +fn parse_trigger_spec(s: &str) -> Option { + // Marker-only native trigger keywords. The match table keeps the + // annotation set in lockstep with `TriggerSpec`. `schedule` is in here + // too — the cron lives on the schedule row the user creates separately; + // the annotation is just the binding declaration. + const NATIVE_KINDS: &[(&str, TriggerSpec)] = &[ + ("schedule", TriggerSpec::Schedule), + ("webhook", TriggerSpec::Webhook), + ("email", TriggerSpec::Email), + ("kafka", TriggerSpec::Kafka), + ("mqtt", TriggerSpec::Mqtt), + ("nats", TriggerSpec::Nats), + ("postgres", TriggerSpec::Postgres), + ("sqs", TriggerSpec::Sqs), + ("gcp", TriggerSpec::Gcp), + ("data_upload", TriggerSpec::DataUpload), + ]; + for (kw, spec) in NATIVE_KINDS { + if let Some(rest) = s.strip_prefix(kw) { + // Must be a complete word — `kafkalike` doesn't match `kafka`. + // Trailing whitespace alone is fine; any non-empty trailing + // content is treated as malformed (the annotation is marker-only). + if !rest.is_empty() && !rest.starts_with(|c: char| c.is_whitespace()) { + continue; + } + if !rest.trim().is_empty() { + return None; + } + return Some(spec.clone()); + } + } + + let (kind, path) = parse_asset_syntax(s, false)?; + // `debounce` is attached by the caller from the `// on` line's opts. + Some(TriggerSpec::Asset { asset_kind: kind, path: path.to_string(), debounce: None }) +} + +#[cfg(test)] +mod pipeline_annotation_tests { + use super::*; + + #[test] + fn bare_pipeline_marker() { + let out = parse_pipeline_annotations("// pipeline\nconsole.log('hi')"); + assert!(out.in_pipeline); + assert!(out.triggers.is_empty()); + } + + #[test] + fn pipeline_marker_strict_grammar_rejects_trailing_words() { + // Strict — the word "pipeline" is common in casual prose, so trailing + // content disqualifies the line. Trailing whitespace alone is fine. + let out = parse_pipeline_annotations( + "// pipeline broken on staging\n# pipeline this through grep\n-- pipeline_v2", + ); + assert!(!out.in_pipeline); + + let out = parse_pipeline_annotations("// pipeline \n"); + assert!(out.in_pipeline); + } + + #[test] + fn rejects_pipeline_keyword_variants() { + let out = parse_pipeline_annotations("// pipelines\n# pipelined\n-- pipeline-foo"); + assert!(!out.in_pipeline); + } + + #[test] + fn on_schedule_marker() { + // `// on schedule` is marker-only — the binding is the schedule row's + // own `script_path` field, just like kafka/mqtt/etc. + let out = parse_pipeline_annotations("// on schedule"); + assert_eq!(out.triggers.len(), 1); + assert_eq!(out.triggers[0], TriggerSpec::Schedule); + } + + #[test] + fn rejects_schedule_with_trailing_content() { + // Marker-only — the old `// schedule ""` form is gone, and a + // trailing path/cron on the `on schedule` form is malformed. + let out = parse_pipeline_annotations("// on schedule \"0 0 * * *\""); + assert!(out.triggers.is_empty()); + let out = parse_pipeline_annotations("// schedule \"0 0 * * *\""); + assert!(out.triggers.is_empty()); + } + + #[test] + fn on_asset_ts_py_sql() { + let code = "// on s3://a/b\n# on datatable://main\n-- on $res:f/foo"; + let out = parse_pipeline_annotations(code); + assert_eq!(out.triggers.len(), 3); + assert!(matches!( + out.triggers[0], + TriggerSpec::Asset { asset_kind: AssetKind::S3Object, .. } + )); + assert!(matches!( + out.triggers[1], + TriggerSpec::Asset { asset_kind: AssetKind::DataTable, .. } + )); + assert!(matches!( + out.triggers[2], + TriggerSpec::Asset { asset_kind: AssetKind::Resource, .. } + )); + } + + #[test] + fn on_deduplicates() { + let code = "// on s3://a/b\n# on s3://a/b"; + let out = parse_pipeline_annotations(code); + assert_eq!(out.triggers.len(), 1); + } + + #[test] + fn rejects_unknown_trigger_spec() { + let out = parse_pipeline_annotations("// on unknown://nope\n# schedule"); + assert!(out.triggers.is_empty()); + } + + #[test] + fn join_mode_defaults_to_any() { + let out = parse_pipeline_annotations("// on s3://a/b"); + assert_eq!(out.join_mode, JoinMode::Any); + assert!(out.join_mode.is_any()); + } + + #[test] + fn trigger_all_sets_and_mode() { + let out = parse_pipeline_annotations( + "// pipeline\n// on s3://lake/{partition}/x\n// trigger all", + ); + assert_eq!(out.join_mode, JoinMode::All); + assert!(!out.join_mode.is_any()); + } + + #[test] + fn trigger_any_explicit_and_other_prefixes() { + // explicit `any`, plus `#` / `--` comment prefixes are accepted. + assert_eq!( + parse_pipeline_annotations("# trigger all\n-- trigger any").join_mode, + JoinMode::Any + ); + assert_eq!( + parse_pipeline_annotations("-- trigger all").join_mode, + JoinMode::All + ); + } + + #[test] + fn trigger_unknown_or_glued_keeps_default() { + // unknown value, and `triggerall` (no whitespace) must not match. + assert_eq!( + parse_pipeline_annotations("// trigger bogus").join_mode, + JoinMode::Any + ); + assert_eq!( + parse_pipeline_annotations("// triggerall").join_mode, + JoinMode::Any + ); + } + + #[test] + fn is_partition_bearing_only_for_tokened_assets() { + let out = parse_pipeline_annotations( + "// on s3://lake/raw/{partition}/events.parquet\n\ + // on s3://lake/dim/customers.parquet\n\ + // on schedule", + ); + assert_eq!(out.triggers.len(), 3); + assert!(out.triggers[0].is_partition_bearing()); + assert!(!out.triggers[1].is_partition_bearing()); + assert!(!out.triggers[2].is_partition_bearing()); + } + + fn asset_debounce(t: &TriggerSpec) -> Option<&str> { + match t { + TriggerSpec::Asset { debounce, .. } => debounce.as_deref(), + _ => None, + } + } + + #[test] + fn debounce_none_by_default() { + let out = parse_pipeline_annotations("// on s3://a/b"); + assert_eq!(out.debounce_default, None); + assert_eq!(asset_debounce(&out.triggers[0]), None); + } + + #[test] + fn script_level_debounce_default() { + let out = parse_pipeline_annotations("// debounce 30s\n// on s3://a/b"); + assert_eq!(out.debounce_default.as_deref(), Some("30s")); + // Per-edge override absent — precedence (edge ?? default) is + // resolved at deploy, so the Asset itself stays None here. + assert_eq!(asset_debounce(&out.triggers[0]), None); + } + + #[test] + fn on_level_debounce_override() { + let out = parse_pipeline_annotations( + "// debounce 30s\n\ + // on s3://lake/{partition}/raw.parquet debounce=60s\n\ + // on $res:f/cfg", + ); + assert_eq!(out.debounce_default.as_deref(), Some("30s")); + assert_eq!(out.triggers.len(), 2); + assert_eq!(asset_debounce(&out.triggers[0]), Some("60s")); + // ref still parses correctly with the trailing opt stripped. + assert!(out.triggers[0].is_partition_bearing()); + assert_eq!(asset_debounce(&out.triggers[1]), None); + } + + #[test] + fn debounce_keyword_strictness_and_first_wins() { + // `debounced` (no space) must not match; empty ignored; first wins. + let out = parse_pipeline_annotations( + "// debounced nope\n// debounce\n// debounce 1m\n// debounce 5m", + ); + assert_eq!(out.debounce_default.as_deref(), Some("1m")); + } + + #[test] + fn native_trigger_keywords_are_marker_only() { + // Marker form: `// on kafka` parses to the unit variant. + let out = parse_pipeline_annotations("// on kafka"); + assert_eq!(out.triggers.len(), 1); + assert!(matches!(out.triggers[0], TriggerSpec::Kafka)); + + // Old path-bearing form is rejected (no path on native markers). + let out = parse_pipeline_annotations("// on webhook f/foo"); + assert!(out.triggers.is_empty()); + + // Trailing key=value opts are silently dropped by the line-level + // KV splitter before parse_trigger_spec sees them — same behaviour + // for both asset and native kinds. The opts have no meaning for a + // marker, but the marker still parses. + let out = parse_pipeline_annotations("// on mqtt debounce=30s"); + assert_eq!(out.triggers.len(), 1); + assert!(matches!(out.triggers[0], TriggerSpec::Mqtt)); + + // `kafkalike` mustn't match `kafka`. + let out = parse_pipeline_annotations("// on kafkalike"); + assert!(out.triggers.is_empty()); + } + + #[test] + fn all_native_marker_keywords_parse() { + let code = "// on webhook\n// on email\n// on kafka\n// on mqtt\n\ + // on nats\n// on postgres\n// on sqs\n// on gcp\n// on data_upload"; + let out = parse_pipeline_annotations(code); + assert_eq!(out.triggers.len(), 9); + assert!(matches!(out.triggers[0], TriggerSpec::Webhook)); + assert!(matches!(out.triggers[1], TriggerSpec::Email)); + assert!(matches!(out.triggers[2], TriggerSpec::Kafka)); + assert!(matches!(out.triggers[3], TriggerSpec::Mqtt)); + assert!(matches!(out.triggers[4], TriggerSpec::Nats)); + assert!(matches!(out.triggers[5], TriggerSpec::Postgres)); + assert!(matches!(out.triggers[6], TriggerSpec::Sqs)); + assert!(matches!(out.triggers[7], TriggerSpec::Gcp)); + assert!(matches!(out.triggers[8], TriggerSpec::DataUpload)); + } + + #[test] + fn data_upload_marker_is_marker_only() { + // `// on data_upload` parses to the unit variant — no path. + let out = parse_pipeline_annotations("// on data_upload"); + assert_eq!(out.triggers.len(), 1); + assert!(matches!(out.triggers[0], TriggerSpec::DataUpload)); + + // Trailing content makes it malformed (marker-only). + let out = parse_pipeline_annotations("// on data_upload f/foo"); + assert!(out.triggers.is_empty()); + + // `data_uploadish` mustn't match `data_upload`. + let out = parse_pipeline_annotations("// on data_uploadish"); + assert!(out.triggers.is_empty()); + } + + #[test] + fn partitioned_daily() { + let code = "// partitioned daily tz=\"UTC\" format=\"YYYY-MM-DD\" start=\"2024-01-01\""; + let out = parse_pipeline_annotations(code); + let p = out.partition.expect("partition"); + assert_eq!(p.kind, PartitionKind::Daily); + assert_eq!(p.tz.as_deref(), Some("UTC")); + assert_eq!(p.format.as_deref(), Some("YYYY-MM-DD")); + assert_eq!(p.start.as_deref(), Some("2024-01-01")); + } + + #[test] + fn partitioned_hourly_minimal() { + let out = parse_pipeline_annotations("// partitioned hourly"); + let p = out.partition.expect("partition"); + assert_eq!(p.kind, PartitionKind::Hourly); + assert!(p.tz.is_none()); + } + + #[test] + fn partitioned_dynamic_requires_key() { + let out = parse_pipeline_annotations("// partitioned dynamic"); + assert!(out.partition.is_none()); + let out = parse_pipeline_annotations("// partitioned dynamic key=\"$.tenant_id\""); + let p = out.partition.expect("partition"); + assert_eq!( + p.kind, + PartitionKind::Dynamic { key: "$.tenant_id".to_string() } + ); + } + + #[test] + fn partitioned_first_wins() { + let code = "// partitioned daily tz=\"UTC\"\n// partitioned hourly"; + let out = parse_pipeline_annotations(code); + let p = out.partition.expect("partition"); + assert_eq!(p.kind, PartitionKind::Daily); + } + + #[test] + fn partitioned_unknown_kind_is_skipped() { + let out = parse_pipeline_annotations("// partitioned bogus"); + assert!(out.partition.is_none()); + } + + #[test] + fn freshness_basic() { + let out = parse_pipeline_annotations("// freshness 1h"); + assert_eq!(out.freshness.unwrap().duration, "1h"); + } + + #[test] + fn freshness_first_wins() { + let out = parse_pipeline_annotations("// freshness 1h\n# freshness 30m"); + assert_eq!(out.freshness.unwrap().duration, "1h"); + } + + #[test] + fn tag_basic() { + let out = parse_pipeline_annotations("// tag heavy"); + assert_eq!(out.tag.as_deref(), Some("heavy")); + } + + #[test] + fn tag_first_wins() { + let out = parse_pipeline_annotations("// tag heavy\n# tag light"); + assert_eq!(out.tag.as_deref(), Some("heavy")); + } + + #[test] + fn tag_empty_is_skipped() { + let out = parse_pipeline_annotations("// tag "); + assert!(out.tag.is_none()); + } + + #[test] + fn retry_count_only() { + let out = parse_pipeline_annotations("// retry 3"); + let r = out.retry.expect("retry"); + assert_eq!(r.count, 3); + assert_eq!(r.delay, None); + } + + #[test] + fn retry_with_delay() { + let out = parse_pipeline_annotations("// retry 3 5s"); + let r = out.retry.expect("retry"); + assert_eq!(r.count, 3); + assert_eq!(r.delay.as_deref(), Some("5s")); + } + + #[test] + fn retry_first_wins() { + let out = parse_pipeline_annotations("// retry 3 5s\n# retry 1"); + let r = out.retry.expect("retry"); + assert_eq!(r.count, 3); + assert_eq!(r.delay.as_deref(), Some("5s")); + } + + #[test] + fn retry_zero_is_skipped() { + let out = parse_pipeline_annotations("// retry 0 5s"); + assert!(out.retry.is_none()); + } + + #[test] + fn retry_non_numeric_count_is_skipped() { + let out = parse_pipeline_annotations("// retry many"); + assert!(out.retry.is_none()); + } + + #[test] + fn combined() { + let code = concat!( + "// pipeline\n", + "// on schedule\n", + "// on s3://in.csv\n", + "// partitioned daily tz=\"UTC\"\n", + "// freshness 2h\n", + "// tag heavy\n", + "// retry 3 5s\n" + ); + let out = parse_pipeline_annotations(code); + assert!(out.in_pipeline); + assert_eq!(out.triggers.len(), 2); + assert!(out.partition.is_some()); + assert_eq!(out.freshness.unwrap().duration, "2h"); + assert_eq!(out.tag.as_deref(), Some("heavy")); + let r = out.retry.expect("retry"); + assert_eq!(r.count, 3); + assert_eq!(r.delay.as_deref(), Some("5s")); + } + + #[test] + fn split_trailing_kv_opts_uses_exact_token_offset() { + // Regression for the token-offset computation in + // `split_trailing_kv_opts`. The asset ref's path token contains the + // exact text of the trailing `debounce=60s` opt as a substring, and + // the same `debounce=60s` text also appears a second time before the + // real opt. Offsetting by the `&str` slice's pointer is exact; a + // substring scan (`find`) is what this guards against regressing to. + let (ref_part, opts) = split_trailing_kv_opts("s3://lake/debounce=60s/raw debounce=60s"); + assert_eq!(ref_part, "s3://lake/debounce=60s/raw"); + assert_eq!(opts.get("debounce").map(String::as_str), Some("60s")); + + // End-to-end through the annotation parser: the ref must parse to the + // full S3 path (not truncated at the embedded `debounce=`), and the + // per-edge debounce override must be picked up from the trailing opt. + let out = parse_pipeline_annotations("// on s3://lake/debounce=60s/raw debounce=90s"); + assert_eq!(out.triggers.len(), 1); + match &out.triggers[0] { + TriggerSpec::Asset { asset_kind, path, debounce } => { + assert_eq!(*asset_kind, AssetKind::S3Object); + assert_eq!(path, "lake/debounce=60s/raw"); + assert_eq!(debounce.as_deref(), Some("90s")); + } + other => panic!("expected asset trigger, got {other:?}"), + } + } + + #[test] + fn kv_opts_quoted_with_spaces() { + let m = parse_kv_opts("a=\"hello world\" b=plain c='single quoted'"); + assert_eq!(m.get("a").unwrap(), "hello world"); + assert_eq!(m.get("b").unwrap(), "plain"); + assert_eq!(m.get("c").unwrap(), "single quoted"); + } + + #[test] + fn kv_opts_malformed_recovers() { + let m = parse_kv_opts("garbage a=ok =alone b=fine"); + assert_eq!(m.get("a").unwrap(), "ok"); + assert_eq!(m.get("b").unwrap(), "fine"); + assert!(m.get("garbage").is_none()); + } +} diff --git a/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json b/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json new file mode 100644 index 0000000000..158246022c --- /dev/null +++ b/backend/parsers/windmill-parser/tests/fixtures/pipeline_annotations.json @@ -0,0 +1,238 @@ +[ + { + "name": "bare pipeline marker", + "code": "// pipeline\nexport function main() {}", + "expected": { + "in_pipeline": true, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "pipeline keyword with trailing text is not a marker", + "code": "// pipeline for billing\nexport function main() {}", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "asset triggers across kinds, duplicates deduped", + "code": "-- pipeline\n-- on datatable://main/orders\n-- on s3://bucket/raw.parquet\n-- on ducklake://lake/events\n-- on datatable://main/orders\nSELECT 1;", + "expected": { + "in_pipeline": true, + "asset_triggers": [ + "datatable:main/orders", + "s3object:bucket/raw.parquet", + "ducklake:lake/events" + ], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "native trigger markers", + "code": "# pipeline\n# on kafka\n# on schedule\n# on data_upload\nprint(1)", + "expected": { + "in_pipeline": true, + "asset_triggers": [], + "native_triggers": ["kafka", "schedule", "data_upload"], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "native keyword must be a whole word and end the line", + "code": "// on kafkalike\n// on kafka topic-extra\nexport function main() {}", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "partitioned daily with options", + "code": "// pipeline\n// partitioned daily tz=Europe/Paris format=%Y-%m-%d start=2024-01-01\nexport function main() {}", + "expected": { + "in_pipeline": true, + "asset_triggers": [], + "native_triggers": [], + "partition": { + "kind": "daily", + "tz": "Europe/Paris", + "format": "%Y-%m-%d", + "start": "2024-01-01" + }, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "partitioned dynamic requires key, quoted values parse", + "code": "// partitioned dynamic key=\"customer id\"\nexport function main() {}", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": { + "kind": "dynamic", + "key": "customer id", + "tz": null, + "format": null, + "start": null + }, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "partitioned with unknown kind is rejected", + "code": "// partitioned fortnightly\nexport function main() {}", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "partitioned keyword does not match partition prefix", + "code": "// partition daily\nexport function main() {}", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "freshness and tag, first value wins on duplicates", + "code": "# pipeline\n# freshness 1h\n# freshness 2h\n# tag heavy\n# tag light\nprint(1)", + "expected": { + "in_pipeline": true, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": "1h", + "tag": "heavy", + "retry": null + } + }, + { + "name": "retry with count and delay", + "code": "// pipeline\n// retry 3 5s\nexport function main() {}", + "expected": { + "in_pipeline": true, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": { "count": 3, "delay": "5s" } + } + }, + { + "name": "retry count only", + "code": "// retry 2\nexport function main() {}", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": { "count": 2, "delay": null } + } + }, + { + "name": "retry zero or malformed count is rejected", + "code": "// retry 0\n// retry 3foo\n// retry -1\nexport function main() {}", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "annotations only count inside comments", + "code": "on datatable://main/raw\npipeline\nconst s = 'on s3://bucket/x'\nexport function main() {}", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "partition token preserved in trigger path", + "code": "// pipeline\n// on s3://bucket/daily/{partition}/data.parquet\nexport function main() {}", + "expected": { + "in_pipeline": true, + "asset_triggers": ["s3object:bucket/daily/{partition}/data.parquet"], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "leading whitespace and mixed comment prefixes", + "code": " -- pipeline\n\t-- on datatable://main/x\nSELECT 1;", + "expected": { + "in_pipeline": true, + "asset_triggers": ["datatable:main/x"], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + }, + { + "name": "on with empty or unparseable spec is ignored", + "code": "// on\n// on \n// on notaprefix/foo\nexport function main() {}", + "expected": { + "in_pipeline": false, + "asset_triggers": [], + "native_triggers": [], + "partition": null, + "freshness": null, + "tag": null, + "retry": null + } + } +] diff --git a/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs b/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs new file mode 100644 index 0000000000..3efc9f0c9d --- /dev/null +++ b/backend/parsers/windmill-parser/tests/pipeline_annotations_parity.rs @@ -0,0 +1,157 @@ +//! Parser-parity guard: `parse_pipeline_annotations` (Rust, drives deploy) +//! and `parsePipelineAnnotations` (TS, drives the live graph preview — +//! frontend/src/lib/components/assets/AssetGraph/parsePipelineAnnotations.ts) +//! must stay behaviorally identical, or the graph the user previews is not +//! the graph that deploys. Both implementations run the SAME fixture corpus: +//! +//! tests/fixtures/pipeline_annotations.json +//! +//! The frontend counterpart is parsePipelineAnnotations.parity.test.ts. +//! When the annotation grammar changes, extend the corpus — a fixture that +//! passes on one side and fails on the other is exactly the drift this +//! exists to catch. Only the fields both parsers produce are compared +//! (join_mode / debounce_default are deploy-only, parsed solely in Rust). + +use serde::Deserialize; +use windmill_parser::asset_parser::{ + parse_pipeline_annotations, AssetKind, PartitionKind, TriggerSpec, +}; + +#[derive(Deserialize)] +struct Fixture { + name: String, + code: String, + expected: Expected, +} + +#[derive(Deserialize)] +struct Expected { + in_pipeline: bool, + /// `kind:path`, in declaration order, deduped. + asset_triggers: Vec, + native_triggers: Vec, + partition: Option, + freshness: Option, + tag: Option, + retry: Option, +} + +#[derive(Deserialize)] +struct ExpectedPartition { + kind: String, + #[serde(default)] + key: Option, + tz: Option, + format: Option, + start: Option, +} + +#[derive(Deserialize)] +struct ExpectedRetry { + count: u32, + delay: Option, +} + +fn kind_str(k: AssetKind) -> &'static str { + match k { + AssetKind::S3Object => "s3object", + AssetKind::Resource => "resource", + AssetKind::Ducklake => "ducklake", + AssetKind::DataTable => "datatable", + AssetKind::Volume => "volume", + } +} + +fn native_str(t: &TriggerSpec) -> Option<&'static str> { + Some(match t { + TriggerSpec::Asset { .. } => return None, + TriggerSpec::Schedule => "schedule", + TriggerSpec::Webhook => "webhook", + TriggerSpec::Email => "email", + TriggerSpec::Kafka => "kafka", + TriggerSpec::Mqtt => "mqtt", + TriggerSpec::Nats => "nats", + TriggerSpec::Postgres => "postgres", + TriggerSpec::Sqs => "sqs", + TriggerSpec::Gcp => "gcp", + TriggerSpec::DataUpload => "data_upload", + }) +} + +#[test] +fn pipeline_annotation_fixtures_match() { + let fixtures: Vec = + serde_json::from_str(include_str!("fixtures/pipeline_annotations.json")) + .expect("fixture corpus must deserialize"); + assert!(!fixtures.is_empty()); + + for f in fixtures { + let got = parse_pipeline_annotations(&f.code); + let ctx = format!("fixture '{}'", f.name); + + assert_eq!( + got.in_pipeline, f.expected.in_pipeline, + "{ctx}: in_pipeline" + ); + + let asset_triggers: Vec = got + .triggers + .iter() + .filter_map(|t| match t { + TriggerSpec::Asset { asset_kind, path, .. } => { + Some(format!("{}:{}", kind_str(*asset_kind), path)) + } + _ => None, + }) + .collect(); + assert_eq!( + asset_triggers, f.expected.asset_triggers, + "{ctx}: asset triggers" + ); + + let native: Vec<&str> = got.triggers.iter().filter_map(native_str).collect(); + assert_eq!(native, f.expected.native_triggers, "{ctx}: native triggers"); + + match (&got.partition, &f.expected.partition) { + (None, None) => {} + (Some(p), Some(e)) => { + let (kind, key) = match &p.kind { + PartitionKind::Daily => ("daily", None), + PartitionKind::Hourly => ("hourly", None), + PartitionKind::Weekly => ("weekly", None), + PartitionKind::Monthly => ("monthly", None), + PartitionKind::Dynamic { key } => ("dynamic", Some(key.clone())), + }; + assert_eq!(kind, e.kind, "{ctx}: partition kind"); + assert_eq!(key, e.key, "{ctx}: partition key"); + assert_eq!(p.tz, e.tz, "{ctx}: partition tz"); + assert_eq!(p.format, e.format, "{ctx}: partition format"); + assert_eq!(p.start, e.start, "{ctx}: partition start"); + } + (got, want) => panic!( + "{ctx}: partition mismatch — got {:?}, want present={}", + got, + want.is_some() + ), + } + + assert_eq!( + got.freshness.as_ref().map(|fr| fr.duration.clone()), + f.expected.freshness, + "{ctx}: freshness" + ); + assert_eq!(got.tag, f.expected.tag, "{ctx}: tag"); + match (&got.retry, &f.expected.retry) { + (None, None) => {} + (Some(r), Some(e)) => { + assert_eq!(r.count, e.count, "{ctx}: retry count"); + assert_eq!(r.delay, e.delay, "{ctx}: retry delay"); + } + (got, want) => panic!( + "{ctx}: retry mismatch — got {:?}, want present={}", + got, + want.is_some() + ), + } + } +} diff --git a/backend/src/main.rs b/backend/src/main.rs index a97320d5d9..57cc02598a 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -1646,6 +1646,13 @@ async fn process_notify_event( ); windmill_common::variables::CUSTOM_ENVS_CACHE.remove(payload); } + "notify_asset_producer_change" => { + tracing::debug!( + "Asset producer change for workspace {}, invalidating producer-writes cache", + payload + ); + windmill_queue::asset_dispatch::ASSET_PRODUCER_WRITES_CACHE.remove(payload); + } "notify_workspace_key_change" => { tracing::info!( "Workspace key change detected, invalidating workspace key cache: {}", diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index f396726087..be54edf468 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -1275,6 +1275,10 @@ pub async fn delete_expired_items(db: &DB) -> () { tracing::error!("Error deleting autoscaling event on CE: {:?}", e); } + if let Err(e) = windmill_queue::cascade::reap_stale_join_slots(db).await { + tracing::error!("Error reaping stale join_pending_inputs slots: {:?}", e); + } + match sqlx::query_scalar!( "DELETE FROM agent_token_blacklist WHERE expires_at <= now() RETURNING token", ) diff --git a/backend/tests/asset_trigger_dispatch.rs b/backend/tests/asset_trigger_dispatch.rs new file mode 100644 index 0000000000..ec61c05e8c --- /dev/null +++ b/backend/tests/asset_trigger_dispatch.rs @@ -0,0 +1,981 @@ +//! End-to-end test for asset-trigger dispatch. +//! +//! Runs a real Bash producer through a worker, lets the +//! `result_processor` hook fire `dispatch_asset_triggers`, and then makes +//! several follow-up calls into `dispatch_asset_triggers` against the same +//! seeded graph to cover the eligibility branches (self-loop, skip arg, +//! cycle guard, flow subscriber, ineligible job kinds). Direct calls share +//! the same workspace so we exercise the real query paths against real +//! `asset` / `script_trigger` rows produced by deploy-equivalent seeding. + +use serde_json::json; +use sqlx::{Pool, Postgres}; +use uuid::Uuid; +use windmill_common::jobs::{JobKind, JobPayload}; +use windmill_common::runnable_settings::prefetch_cached_from_handle; +use windmill_common::scripts::{ScriptHash, ScriptLang}; +use windmill_queue::asset_dispatch::dispatch_asset_triggers; +use windmill_queue::cascade::reap_stale_join_slots; +use windmill_queue::MiniCompletedJob; +use windmill_test_utils::{initialize_tracing, ApiServer, RunJob}; + +const WS: &str = "test-workspace"; +const PRODUCER: &str = "u/test-user/producer"; +const SUB_S3: &str = "u/test-user/sub-s3"; +const SUB_RES: &str = "u/test-user/sub-res"; +const SUB_FLOW: &str = "u/test-user/sub-flow"; + +// ── Seeding helpers ─────────────────────────────────────────────────────── + +async fn seed_script( + db: &Pool, + path: &str, + content: &str, + language: &str, +) -> anyhow::Result { + // Hash needs to be unique per (workspace, hash). Derive from path AND + // content: the worker's script cache (`cache::script::fetch`) is keyed + // by hash alone and is process-global, so tests running in the same + // process that seed the same path with different content would poison + // each other's cache if the hash came from the path only. + let mut h = 0i64; + for b in path.bytes().chain(content.bytes()) { + h = h.wrapping_mul(31).wrapping_add(b as i64); + } + sqlx::query( + r#"INSERT INTO script (workspace_id, hash, path, summary, description, content, + created_by, language, tag, lock) + VALUES ($1, $2, $3, '', '', $4, 'test-user', $5::script_lang, 'deno', '') + ON CONFLICT DO NOTHING"#, + ) + .bind(WS) + .bind(h) + .bind(path) + .bind(content) + .bind(language) + .execute(db) + .await?; + Ok(h) +} + +async fn seed_asset_write( + db: &Pool, + producer_path: &str, + kind: &str, + asset_path: &str, +) -> anyhow::Result<()> { + sqlx::query( + r#"INSERT INTO asset (workspace_id, path, kind, usage_access_type, usage_path, usage_kind) + VALUES ($1, $2, $3::asset_kind, 'w'::asset_access_type, $4, 'script'::asset_usage_kind) + ON CONFLICT DO NOTHING"#, + ) + .bind(WS) + .bind(asset_path) + .bind(kind) + .bind(producer_path) + .execute(db) + .await?; + // These tests use #[sqlx::test] isolated DBs that all share one workspace + // id, so the process-global producer cache (keyed by workspace) would + // clobber across DBs under concurrent test threads. Disable it so every + // dispatch reads the test's own DB. (Production invalidates via the + // notify_event poller instead.) + windmill_queue::asset_dispatch::ASSET_PRODUCER_CACHE_DISABLED + .store(true, std::sync::atomic::Ordering::Relaxed); + Ok(()) +} + +async fn seed_subscription( + db: &Pool, + subscriber_path: &str, + subscriber_kind: &str, + trigger_ref: &str, +) -> anyhow::Result<()> { + sqlx::query( + r#"INSERT INTO script_trigger + (workspace_id, runnable_kind, runnable_path, trigger_kind, trigger_ref) + VALUES ($1, $2::asset_usage_kind, $3, 'asset'::script_trigger_kind, $4)"#, + ) + .bind(WS) + .bind(subscriber_kind) + .bind(subscriber_path) + .bind(trigger_ref) + .execute(db) + .await?; + Ok(()) +} + +/// Insert a synthetic producer `v2_job` row used by the direct +/// `dispatch_asset_triggers` calls in the edge-case section. `args` lets the +/// test inject `_wmill_skip_asset_dispatch` or `trigger.chain` so the +/// dispatcher's arg-driven branches are exercised against real rows. +async fn seed_producer_job(db: &Pool, args: serde_json::Value) -> anyhow::Result { + let id = Uuid::new_v4(); + sqlx::query!( + r#"INSERT INTO v2_job (id, workspace_id, kind, runnable_path, args, created_by, + permissioned_as, permissioned_as_email, tag, script_lang) + VALUES ($1, $2, 'script'::job_kind, $3, $4, 'test-user', + 'u/test-user', 'test@windmill.dev', 'deno', 'bash'::script_lang)"#, + id, + WS, + PRODUCER, + args, + ) + .execute(db) + .await?; + Ok(id) +} + +/// Like `seed_producer_job` but for an arbitrary runnable path (the +/// AND-join test needs two distinct producers). +async fn seed_producer_job_path( + db: &Pool, + path: &str, + args: serde_json::Value, +) -> anyhow::Result { + let id = Uuid::new_v4(); + sqlx::query!( + r#"INSERT INTO v2_job (id, workspace_id, kind, runnable_path, args, created_by, + permissioned_as, permissioned_as_email, tag, script_lang) + VALUES ($1, $2, 'script'::job_kind, $3, $4, 'test-user', + 'u/test-user', 'test@windmill.dev', 'deno', 'bash'::script_lang)"#, + id, + WS, + path, + args, + ) + .execute(db) + .await?; + Ok(id) +} + +/// Seed an asset subscription flagged as an AND join (`// trigger all`). +async fn seed_subscription_and( + db: &Pool, + subscriber_path: &str, + trigger_ref: &str, +) -> anyhow::Result<()> { + sqlx::query( + r#"INSERT INTO script_trigger + (workspace_id, runnable_kind, runnable_path, trigger_kind, trigger_ref, join_all) + VALUES ($1, 'script'::asset_usage_kind, $2, 'asset'::script_trigger_kind, $3, TRUE)"#, + ) + .bind(WS) + .bind(subscriber_path) + .bind(trigger_ref) + .execute(db) + .await?; + Ok(()) +} + +/// Seed an asset subscription with an opt-in debounce window (seconds). +async fn seed_subscription_debounced( + db: &Pool, + subscriber_path: &str, + trigger_ref: &str, + debounce_s: i32, +) -> anyhow::Result<()> { + sqlx::query( + r#"INSERT INTO script_trigger + (workspace_id, runnable_kind, runnable_path, trigger_kind, trigger_ref, debounce_s) + VALUES ($1, 'script'::asset_usage_kind, $2, 'asset'::script_trigger_kind, $3, $4)"#, + ) + .bind(WS) + .bind(subscriber_path) + .bind(trigger_ref) + .bind(debounce_s) + .execute(db) + .await?; + Ok(()) +} + +/// Seed an asset subscription with a `// retry []` policy. +async fn seed_subscription_with_retry( + db: &Pool, + subscriber_path: &str, + trigger_ref: &str, + retry_count: i16, + retry_delay_s: i32, +) -> anyhow::Result<()> { + sqlx::query( + r#"INSERT INTO script_trigger + (workspace_id, runnable_kind, runnable_path, trigger_kind, trigger_ref, + retry_count, retry_delay_s) + VALUES ($1, 'script'::asset_usage_kind, $2, 'asset'::script_trigger_kind, $3, $4, $5)"#, + ) + .bind(WS) + .bind(subscriber_path) + .bind(trigger_ref) + .bind(retry_count) + .bind(retry_delay_s) + .execute(db) + .await?; + Ok(()) +} + +fn make_mini(id: Uuid, runnable_path: &str) -> MiniCompletedJob { + MiniCompletedJob { + id, + workspace_id: WS.to_string(), + runnable_id: Some(ScriptHash(1)), + scheduled_for: chrono::Utc::now(), + parent_job: None, + flow_innermost_root_job: None, + runnable_path: Some(runnable_path.to_string()), + kind: JobKind::Script, + started_at: Some(chrono::Utc::now()), + permissioned_as: "u/test-user".to_string(), + created_by: "test-user".to_string(), + script_lang: Some(ScriptLang::Bash), + permissioned_as_email: "test@windmill.dev".to_string(), + flow_step_id: None, + trigger_kind: None, + trigger: None, + priority: None, + concurrent_limit: None, + tag: "deno".to_string(), + cache_ttl: None, + cache_ignore_s3_path: None, + runnable_settings_handle: None, + } +} + +/// Read `v2_job` rows that were created by asset dispatch (filtered by +/// `trigger_kind = 'asset'` so dep-jobs / other infra rows don't leak in). +async fn fetch_dispatched( + db: &Pool, +) -> anyhow::Result, Option)>> { + let rows = sqlx::query!( + r#"SELECT runnable_path AS "runnable_path!", trigger, + args AS "args: sqlx::types::Json" + FROM v2_job + WHERE workspace_id = $1 AND trigger_kind = 'asset' + ORDER BY runnable_path"#, + WS, + ) + .fetch_all(db) + .await?; + Ok(rows + .into_iter() + .map(|r| (r.runnable_path, r.trigger, r.args.map(|j| j.0))) + .collect()) +} + +async fn clear_dispatched(db: &Pool) -> anyhow::Result<()> { + sqlx::query!( + "DELETE FROM v2_job WHERE workspace_id = $1 AND trigger_kind = 'asset'", + WS, + ) + .execute(db) + .await?; + Ok(()) +} + +// ── The test ───────────────────────────────────────────────────────────── + +/// One end-to-end test that: +/// 1. seeds a producer that writes two asset kinds (s3 + resource) with three +/// subscribers (two script subs + one flow sub that must be skipped), +/// 2. runs the producer through a real worker and asserts the +/// `result_processor` hook fired and pushed the right jobs with the +/// right trigger metadata, +/// 3. then drives `dispatch_asset_triggers` directly against the same +/// seeded graph to cover the arg-driven and eligibility branches that +/// can't be reached by varying the producer's runtime args alone: +/// - skip arg suppresses dispatch +/// - a subscriber already in the lineage is skipped (cycle guard) +/// - the lineage chain accumulates the producer path each hop +/// - self-loop subscriber is filtered +/// - producer with parent_job is ineligible +/// - producer with `Flow` kind is ineligible +#[sqlx::test(fixtures("base"))] +async fn end_to_end_asset_dispatch(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // ── Seed the graph ────────────────────────────────────────────────── + let producer_hash = seed_script(&db, PRODUCER, "echo producer", "bash").await?; + seed_script(&db, SUB_S3, "echo s3-subscriber", "bash").await?; + seed_script(&db, SUB_RES, "echo res-subscriber", "bash").await?; + // Self-loop subscriber: same path as producer → must be filtered. + seed_subscription(&db, PRODUCER, "script", "s3://f/blob").await?; + // Flow subscriber on the same asset → V1 hard-filters runnable_kind='flow'. + seed_subscription(&db, SUB_FLOW, "flow", "s3://f/blob").await?; + // Legit subscribers. + seed_subscription(&db, SUB_S3, "script", "s3://f/blob").await?; + seed_subscription(&db, SUB_RES, "script", "$res:f/cfg").await?; + // Two writes from one producer, distinct kinds. + seed_asset_write(&db, PRODUCER, "s3object", "f/blob").await?; + seed_asset_write(&db, PRODUCER, "resource", "f/cfg").await?; + + // ── 1. Real worker run: the hook must fire after producer success ─── + let job = JobPayload::ScriptHash { + path: PRODUCER.to_string(), + hash: ScriptHash(producer_hash), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + language: ScriptLang::Bash, + priority: None, + apply_preprocessor: false, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + labels: None, + }; + let completed = RunJob::from(job).run_until_complete(&db, false, port).await; + assert!( + completed.success, + "producer must succeed for dispatch to fire" + ); + + let mut rows = fetch_dispatched(&db).await?; + rows.sort_by(|a, b| a.0.cmp(&b.0)); + assert_eq!( + rows.len(), + 2, + "expected dispatch to the two legit script subscribers (flow sub filtered, self-loop filtered)" + ); + let by_path: std::collections::HashMap<_, _> = rows.iter().map(|r| (r.0.as_str(), r)).collect(); + + let s3_row = by_path + .get(SUB_S3) + .expect("s3 subscriber should have a job"); + let s3_trig = s3_row.2.as_ref().unwrap().get("trigger").unwrap(); + assert_eq!(s3_trig["kind"], "asset"); + assert_eq!(s3_trig["asset_kind"], "s3object"); + assert_eq!(s3_trig["asset_path"], "f/blob"); + assert_eq!(s3_trig["producer_path"], PRODUCER); + assert_eq!( + s3_trig["chain"], + json!([PRODUCER]), + "lineage starts with the producer on the first hop" + ); + assert_eq!(s3_row.1.as_deref(), Some(PRODUCER)); + + let res_row = by_path + .get(SUB_RES) + .expect("resource subscriber should have a job"); + let res_trig = res_row.2.as_ref().unwrap().get("trigger").unwrap(); + assert_eq!(res_trig["asset_kind"], "resource"); + assert_eq!(res_trig["asset_path"], "f/cfg"); + + // ── 2. Direct calls to dispatch_asset_triggers for arg / eligibility + // branches that can't be reached via the runtime path ────────── + clear_dispatched(&db).await?; + + // skip arg suppresses dispatch + let id = seed_producer_job(&db, json!({ "_wmill_skip_asset_dispatch": true })).await?; + let r = dispatch_asset_triggers(&db, &make_mini(id, PRODUCER)).await; + assert_eq!(r.dispatched.len(), 0, "skip arg suppressed dispatch"); + + // cycle guard: a subscriber already in the lineage is skipped, but its + // siblings still dispatch (only the cyclic edge is cut). + let id = seed_producer_job(&db, json!({ "trigger": { "chain": [SUB_S3] } })).await?; + let r = dispatch_asset_triggers(&db, &make_mini(id, PRODUCER)).await; + assert_eq!( + r.dispatched.len(), + 1, + "cyclic subscriber (already in lineage) skipped; sibling still dispatched" + ); + + // lineage accumulates: a fresh producer extends the chain with its own path + clear_dispatched(&db).await?; + let id = seed_producer_job(&db, json!({ "trigger": { "chain": ["f/upstream"] } })).await?; + let r = dispatch_asset_triggers(&db, &make_mini(id, PRODUCER)).await; + assert_eq!(r.dispatched.len(), 2); + let rows = fetch_dispatched(&db).await?; + for row in &rows { + assert_eq!( + row.2.as_ref().unwrap()["trigger"]["chain"], + json!(["f/upstream", PRODUCER]), + "lineage accumulates the producer path" + ); + } + + // producer with parent_job (flow step) is ineligible + clear_dispatched(&db).await?; + let id = seed_producer_job(&db, json!({})).await?; + let mut mini = make_mini(id, PRODUCER); + mini.parent_job = Some(Uuid::new_v4()); + let r = dispatch_asset_triggers(&db, &mini).await; + assert_eq!(r.dispatched.len(), 0, "flow step producer ineligible"); + + // producer with kind=Flow is ineligible + let id = seed_producer_job(&db, json!({})).await?; + let mut mini = make_mini(id, PRODUCER); + mini.kind = JobKind::Flow; + let r = dispatch_asset_triggers(&db, &mini).await; + assert_eq!(r.dispatched.len(), 0, "flow producer ineligible"); + + // Sanity: the eligible direct call (clean producer, no args) still fires — + // proves the assertions above are negative cases, not a broken setup. + let id = seed_producer_job(&db, json!({})).await?; + let r = dispatch_asset_triggers(&db, &make_mini(id, PRODUCER)).await; + assert_eq!(r.dispatched.len(), 2, "clean direct call still dispatches"); + + Ok(()) +} + +/// Stage C: a `// partitioned dynamic` producer run through a real worker +/// must (1) resolve the partition off its triggering payload at execution +/// time, (2) persist it back into its own `v2_job.args` so the cascade +/// reads it, and (3) propagate the same value into the dispatched +/// subscriber's args + `trigger.partition`. +#[sqlx::test(fixtures("base"))] +async fn partition_dynamic_resolved_persisted_and_propagated( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + // Producer declares a dynamic partition keyed off the run payload. + // (Bash uses `#` comments — the annotation parser accepts `#`/`--`/`//`.) + let producer_hash = seed_script( + &db, + PRODUCER, + "# pipeline\n# partitioned dynamic key=\"$.tenant_id\"\necho producer", + "bash", + ) + .await?; + seed_script(&db, SUB_S3, "echo s3-subscriber", "bash").await?; + seed_asset_write(&db, PRODUCER, "s3object", "f/blob").await?; + seed_subscription(&db, SUB_S3, "script", "s3://f/blob").await?; + + let job = JobPayload::ScriptHash { + path: PRODUCER.to_string(), + hash: ScriptHash(producer_hash), + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + language: ScriptLang::Bash, + priority: None, + apply_preprocessor: false, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + labels: None, + }; + let completed = RunJob::from(job) + .arg("tenant_id", json!("acme")) + .run_until_complete(&db, false, port) + .await; + assert!(completed.success, "partitioned producer must succeed"); + + // (2) resolved value persisted back into the producer's own args. + let prod = sqlx::query!( + r#"SELECT args AS "args: sqlx::types::Json" + FROM v2_job + WHERE workspace_id = $1 AND runnable_path = $2 AND trigger_kind IS NULL"#, + WS, + PRODUCER, + ) + .fetch_one(&db) + .await?; + assert_eq!( + prod.args.unwrap().0["partition"], + json!("acme"), + "Stage C must persist the resolved partition into v2_job.args" + ); + + // (3) propagated into the dispatched subscriber. + let rows = fetch_dispatched(&db).await?; + let sub = rows + .iter() + .find(|r| r.0 == SUB_S3) + .expect("subscriber must be dispatched"); + let args = sub.2.as_ref().unwrap(); + assert_eq!( + args["partition"], + json!("acme"), + "subscriber gets top-level partition arg" + ); + assert_eq!( + args["trigger"]["partition"], + json!("acme"), + "subscriber gets trigger.partition" + ); + + Ok(()) +} + +/// Stage D: an AND-join subscriber (`// trigger all`) with two +/// partition-bearing inputs must NOT dispatch until both inputs have +/// arrived for the *same* partition; then it fires exactly once. Slots +/// are per-partition and cleared on fire (re-accumulate, no double-fire). +#[sqlx::test(fixtures("base"))] +async fn and_join_waits_for_all_partition_inputs(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + const PROD_A: &str = "u/test-user/prod-a"; + const PROD_B: &str = "u/test-user/prod-b"; + const SUB_J: &str = "u/test-user/sub-join"; + + seed_script(&db, SUB_J, "echo join-subscriber", "bash").await?; + // Two partition-bearing producers, one input each (literal token form). + seed_asset_write(&db, PROD_A, "s3object", "lake/{partition}/a").await?; + seed_asset_write(&db, PROD_B, "s3object", "lake/{partition}/b").await?; + seed_subscription_and(&db, SUB_J, "s3://lake/{partition}/a").await?; + seed_subscription_and(&db, SUB_J, "s3://lake/{partition}/b").await?; + + // Input A for partition "acme" → slot 1/2, must NOT dispatch. + let a_acme = seed_producer_job_path(&db, PROD_A, json!({ "partition": "acme" })).await?; + let r = dispatch_asset_triggers(&db, &make_mini(a_acme, PROD_A)).await; + assert!( + r.dispatched.is_empty(), + "AND join must wait: only 1 of 2 inputs present" + ); + assert!( + fetch_dispatched(&db).await?.is_empty(), + "no subscriber job pushed yet" + ); + + // A different partition for input B must open its OWN slot, not + // complete acme's. + let b_globex = seed_producer_job_path(&db, PROD_B, json!({ "partition": "globex" })).await?; + let r = dispatch_asset_triggers(&db, &make_mini(b_globex, PROD_B)).await; + assert!( + r.dispatched.is_empty(), + "different partition opens a separate slot, does not complete acme" + ); + + // Input B for "acme" → acme slot now 2/2 → dispatch exactly once. + let b_acme = seed_producer_job_path(&db, PROD_B, json!({ "partition": "acme" })).await?; + let r = dispatch_asset_triggers(&db, &make_mini(b_acme, PROD_B)).await; + assert_eq!(r.dispatched.len(), 1, "AND join fires once both inputs in"); + + let rows = fetch_dispatched(&db).await?; + assert_eq!(rows.len(), 1); + let sub = &rows[0]; + assert_eq!(sub.0, SUB_J); + let args = sub.2.as_ref().unwrap(); + assert_eq!(args["partition"], json!("acme")); + assert_eq!(args["trigger"]["partition"], json!("acme")); + + // Slot cleared on fire: re-arrival of A/acme alone is 1/2 again, no + // double-fire. + clear_dispatched(&db).await?; + let a_acme2 = seed_producer_job_path(&db, PROD_A, json!({ "partition": "acme" })).await?; + let r = dispatch_asset_triggers(&db, &make_mini(a_acme2, PROD_A)).await; + assert!( + r.dispatched.is_empty(), + "slot was cleared on fire; single input must not re-fire" + ); + + Ok(()) +} + +/// Stage E3: a subscriber whose edge has a debounce window gets real +/// DebouncingSettings (delay + a (subscriber, partition) key) on the +/// dispatched job; an undebounced subscriber on the same asset gets none +/// (fan-out, unchanged). Asserts the wiring fetch→push→payload→handle; +/// the actual window-collapse is the queue subsystem's own concern. +#[sqlx::test(fixtures("base"))] +async fn debounce_setting_applied_to_dispatched_subscriber( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + seed_script(&db, SUB_S3, "echo debounced", "bash").await?; + seed_script(&db, SUB_RES, "echo plain", "bash").await?; + seed_asset_write(&db, PRODUCER, "s3object", "f/blob").await?; + seed_subscription_debounced(&db, SUB_S3, "s3://f/blob", 30).await?; + seed_subscription(&db, SUB_RES, "script", "s3://f/blob").await?; + + let id = seed_producer_job(&db, json!({})).await?; + let r = dispatch_asset_triggers(&db, &make_mini(id, PRODUCER)).await; + assert_eq!(r.dispatched.len(), 2, "both subscribers dispatched"); + + async fn debounce_of( + db: &Pool, + path: &str, + ) -> anyhow::Result<(Option, Option)> { + let handle = sqlx::query_scalar!( + r#"SELECT q.runnable_settings_handle + FROM v2_job j JOIN v2_job_queue q ON q.id = j.id + WHERE j.workspace_id = $1 AND j.runnable_path = $2 + AND j.trigger_kind = 'asset'"#, + WS, + path, + ) + .fetch_one(db) + .await?; + let (deb, _conc) = prefetch_cached_from_handle(handle, db).await?; + Ok((deb.debounce_delay_s, deb.debounce_key)) + } + + let (deb_delay, deb_key) = debounce_of(&db, SUB_S3).await?; + assert_eq!(deb_delay, Some(30), "debounced edge → 30s window"); + assert!( + deb_key + .as_deref() + .is_some_and(|k| k.starts_with("asset-cascade:")), + "debounce key is scoped to the (subscriber, partition) cascade slot, got {deb_key:?}" + ); + + let (plain_delay, _) = debounce_of(&db, SUB_RES).await?; + assert_eq!( + plain_delay, None, + "undebounced edge → no debounce (fan-out)" + ); + + Ok(()) +} + +/// `// retry []` opts the subscriber into the flow-runtime retry +/// path. Implementation-wise, the dispatcher wraps the script in a one-step +/// flow (`JobPayload::SingleStepFlow`) so the existing flow retry machinery +/// handles re-runs. Subscribers without retry continue to be pushed as +/// `JobKind::Script` (no wrapping). +#[sqlx::test(fixtures("base"))] +async fn retry_setting_wraps_dispatched_subscriber_as_flow( + db: Pool, +) -> anyhow::Result<()> { + initialize_tracing().await; + + seed_script(&db, SUB_S3, "echo retrying", "bash").await?; + seed_script(&db, SUB_RES, "echo plain", "bash").await?; + seed_asset_write(&db, PRODUCER, "s3object", "f/blob").await?; + // Retry policy on the s3 edge; res edge stays vanilla so we also assert + // the "no retry" path keeps the cheaper ScriptHash push. + seed_subscription_with_retry(&db, SUB_S3, "s3://f/blob", 3, 5).await?; + seed_subscription(&db, SUB_RES, "script", "s3://f/blob").await?; + + let id = seed_producer_job(&db, json!({})).await?; + let r = dispatch_asset_triggers(&db, &make_mini(id, PRODUCER)).await; + assert_eq!(r.dispatched.len(), 2, "both subscribers dispatched"); + + let kinds: Vec<(String, String)> = sqlx::query!( + r#"SELECT runnable_path AS "runnable_path!", kind::text AS "kind!" + FROM v2_job + WHERE workspace_id = $1 AND trigger_kind = 'asset' + ORDER BY runnable_path"#, + WS, + ) + .fetch_all(&db) + .await? + .into_iter() + .map(|r| (r.runnable_path, r.kind)) + .collect(); + + let s3_kind = kinds + .iter() + .find(|(p, _)| p == SUB_S3) + .map(|(_, k)| k.as_str()) + .unwrap_or(""); + let res_kind = kinds + .iter() + .find(|(p, _)| p == SUB_RES) + .map(|(_, k)| k.as_str()) + .unwrap_or(""); + + assert_eq!( + s3_kind, "singlestepflow", + "retry-bearing subscriber wraps as SingleStepFlow so flow-runtime retry kicks in" + ); + assert_eq!( + res_kind, "script", + "no-retry subscriber stays as the cheaper ScriptHash push" + ); + + Ok(()) +} + +/// Regression for the AND-join check-then-act race: when a subscriber's +/// last partition-bearing inputs complete concurrently (different workers +/// finishing different upstream producers at once), the barrier must +/// still fire the subscriber exactly once for the partition. Fires all N +/// producers' dispatch simultaneously (a barrier releases them together) +/// and asserts a single dispatch and a cleared slot. This invariant holds +/// for the transactional, advisory-locked gate regardless of interleaving; +/// a regression to a non-atomic check-then-act fails it. +#[sqlx::test(fixtures("base"))] +async fn and_join_fires_once_under_concurrent_completion(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + const SUB_J: &str = "u/test-user/sub-join-conc"; + const N: usize = 5; + + seed_script(&db, SUB_J, "echo join", "bash").await?; + let mut producers = Vec::new(); + for i in 0..N { + let prod = format!("u/test-user/prod-conc-{i}"); + seed_asset_write(&db, &prod, "s3object", &format!("lake/{{partition}}/i{i}")).await?; + seed_subscription_and(&db, SUB_J, &format!("s3://lake/{{partition}}/i{i}")).await?; + let id = seed_producer_job_path(&db, &prod, json!({ "partition": "acme" })).await?; + producers.push((id, prod)); + } + + let barrier = std::sync::Arc::new(tokio::sync::Barrier::new(N)); + let mut set = tokio::task::JoinSet::new(); + for (id, prod) in producers { + let db = db.clone(); + let barrier = barrier.clone(); + set.spawn(async move { + barrier.wait().await; + dispatch_asset_triggers(&db, &make_mini(id, &prod)) + .await + .dispatched + .len() + }); + } + let mut total = 0usize; + while let Some(r) = set.join_next().await { + total += r?; + } + + assert_eq!( + total, 1, + "AND join must dispatch the subscriber exactly once under concurrent completion" + ); + let fires = fetch_dispatched(&db) + .await? + .iter() + .filter(|r| r.0 == SUB_J) + .count(); + assert_eq!(fires, 1, "exactly one subscriber job pushed"); + + let leftover = sqlx::query_scalar!( + r#"SELECT count(*) AS "n!" + FROM join_pending_inputs + WHERE workspace_id = $1 AND subscriber_path = $2"#, + WS, + SUB_J, + ) + .fetch_one(&db) + .await?; + assert_eq!(leftover, 0, "join slot cleared after fire"); + + Ok(()) +} + +/// Fuller pipeline: a partitioned chain that fans in through an AND-join +/// and then fans out over several more hops. Asserts the resolved +/// partition propagates unchanged at every hop, the chain depth +/// increments per hop, the AND barrier fires once, and a different +/// partition opens an independent slot (no cross-partition bleed) across +/// the whole multi-hop graph. +/// +/// Shape: A,B (partitioned producers) ─┐ +/// ├─▶ J (// trigger all) ─▶ C ─▶ D +/// A,B ─────────────────────────┘ +#[sqlx::test(fixtures("base"))] +async fn fuller_partitioned_join_multihop_pipeline(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + const PA: &str = "u/test-user/p-a"; + const PB: &str = "u/test-user/p-b"; + const JN: &str = "u/test-user/p-join"; + const CN: &str = "u/test-user/p-c"; + const DN: &str = "u/test-user/p-d"; + + for p in [JN, CN, DN] { + seed_script(&db, p, "echo step", "bash").await?; + } + seed_asset_write(&db, PA, "s3object", "lake/{partition}/a").await?; + seed_asset_write(&db, PB, "s3object", "lake/{partition}/b").await?; + seed_asset_write(&db, JN, "s3object", "lake/{partition}/j").await?; + seed_asset_write(&db, CN, "s3object", "lake/{partition}/c").await?; + // J is an AND join over both partition-bearing inputs. + seed_subscription_and(&db, JN, "s3://lake/{partition}/a").await?; + seed_subscription_and(&db, JN, "s3://lake/{partition}/b").await?; + seed_subscription(&db, CN, "script", "s3://lake/{partition}/j").await?; + seed_subscription(&db, DN, "script", "s3://lake/{partition}/c").await?; + + // Helper: assert exactly one dispatch to `path` carrying partition + // `part`, with a cascade lineage of `chain_len` producer paths. + async fn assert_hop( + db: &Pool, + path: &str, + part: &str, + chain_len: usize, + ) -> anyhow::Result<()> { + let rows = fetch_dispatched(db).await?; + let hits: Vec<_> = rows.iter().filter(|r| r.0 == path).collect(); + assert_eq!(hits.len(), 1, "expected exactly one dispatch to {path}"); + let args = hits[0].2.as_ref().unwrap(); + assert_eq!(args["partition"], json!(part), "{path} top-level partition"); + assert_eq!( + args["trigger"]["partition"], + json!(part), + "{path} trigger.partition" + ); + assert_eq!( + args["trigger"]["chain"].as_array().map(|c| c.len()), + Some(chain_len), + "{path} lineage length" + ); + Ok(()) + } + + // day1: A arrives → J waits (1/2 partition-bearing inputs). + let pa = seed_producer_job_path( + &db, + PA, + json!({ "partition": "day1", "trigger": { "chain": ["s0"] } }), + ) + .await?; + let r = dispatch_asset_triggers(&db, &make_mini(pa, PA)).await; + assert!(r.dispatched.is_empty(), "J must wait: only A present"); + assert!(fetch_dispatched(&db).await?.is_empty()); + + // day1: B arrives → J fires once for day1 at depth 2. + let pb = seed_producer_job_path( + &db, + PB, + json!({ "partition": "day1", "trigger": { "chain": ["s0"] } }), + ) + .await?; + let r = dispatch_asset_triggers(&db, &make_mini(pb, PB)).await; + assert_eq!(r.dispatched.len(), 1, "J fires once when both inputs in"); + assert_hop(&db, JN, "day1", 2).await?; + clear_dispatched(&db).await?; + + // J completes for day1 → C runs for day1 at depth 3. + let jn = seed_producer_job_path( + &db, + JN, + json!({ "partition": "day1", "trigger": { "chain": ["s0", PB] } }), + ) + .await?; + let r = dispatch_asset_triggers(&db, &make_mini(jn, JN)).await; + assert_eq!(r.dispatched.len(), 1); + assert_hop(&db, CN, "day1", 3).await?; + clear_dispatched(&db).await?; + + // C completes for day1 → D (leaf) runs for day1 at depth 4. + let cn = seed_producer_job_path( + &db, + CN, + json!({ "partition": "day1", "trigger": { "chain": ["s0", PB, JN] } }), + ) + .await?; + let r = dispatch_asset_triggers(&db, &make_mini(cn, CN)).await; + assert_eq!(r.dispatched.len(), 1); + assert_hop(&db, DN, "day1", 4).await?; + clear_dispatched(&db).await?; + + // A different partition opens an independent J slot — no bleed from + // the completed day1 run. + let pa2 = seed_producer_job_path( + &db, + PA, + json!({ "partition": "day2", "trigger": { "chain": ["s0"] } }), + ) + .await?; + let r = dispatch_asset_triggers(&db, &make_mini(pa2, PA)).await; + assert!( + r.dispatched.is_empty(), + "day2 is a separate slot; J must not fire from day1's completion" + ); + + Ok(()) +} + +/// The TTL reaper deletes abandoned AND-join slots, but keyed on the +/// slot's MOST RECENT row: a slot still receiving input (newest row +/// fresh) is never reaped even if it also has rows older than the TTL. +/// This per-slot (not per-row) property is the correctness point — it +/// prevents corrupting a join whose inputs trickle in slowly. +#[sqlx::test(fixtures("base"))] +async fn reaper_clears_only_stale_join_slots(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + + // Seed a join_pending_inputs row with an explicit age (days old). + async fn seed_slot_row( + db: &Pool, + sub: &str, + part: &str, + tref: &str, + age_days: i64, + ) -> anyhow::Result<()> { + sqlx::query( + r#"INSERT INTO join_pending_inputs + (workspace_id, subscriber_path, partition, trigger_ref, received_at) + VALUES ($1, $2, $3, $4, now() - ($5::bigint::text || ' d')::interval)"#, + ) + .bind(WS) + .bind(sub) + .bind(part) + .bind(tref) + .bind(age_days) + .execute(db) + .await?; + Ok(()) + } + async fn slot_count(db: &Pool, sub: &str) -> anyhow::Result { + Ok(sqlx::query_scalar!( + r#"SELECT count(*) AS "n!" FROM join_pending_inputs + WHERE workspace_id = $1 AND subscriber_path = $2"#, + WS, + sub, + ) + .fetch_one(db) + .await?) + } + + // Stale: every row older than the 60d TTL → reaped. + seed_slot_row( + &db, + "u/test-user/sub-stale", + "p1", + "s3://x/{partition}/a", + 61, + ) + .await?; + seed_slot_row( + &db, + "u/test-user/sub-stale", + "p1", + "s3://x/{partition}/b", + 90, + ) + .await?; + // Fresh: recent → kept. + seed_slot_row( + &db, + "u/test-user/sub-fresh", + "p1", + "s3://y/{partition}/a", + 0, + ) + .await?; + // Mixed: one ancient row + one fresh row in the SAME slot. max(received_at) + // is fresh, so the whole slot must be kept (the correctness property). + seed_slot_row( + &db, + "u/test-user/sub-mixed", + "p1", + "s3://z/{partition}/a", + 120, + ) + .await?; + seed_slot_row( + &db, + "u/test-user/sub-mixed", + "p1", + "s3://z/{partition}/b", + 0, + ) + .await?; + + reap_stale_join_slots(&db).await?; + + assert_eq!( + slot_count(&db, "u/test-user/sub-stale").await?, + 0, + "stale slot reaped" + ); + assert_eq!( + slot_count(&db, "u/test-user/sub-fresh").await?, + 1, + "fresh slot kept" + ); + assert_eq!( + slot_count(&db, "u/test-user/sub-mixed").await?, + 2, + "slot with a recent row must be kept entirely (per-slot, not per-row)" + ); + + Ok(()) +} diff --git a/backend/tests/python_jobs.rs b/backend/tests/python_jobs.rs index 3a1550d395..0f45d09147 100644 --- a/backend/tests/python_jobs.rs +++ b/backend/tests/python_jobs.rs @@ -1113,6 +1113,7 @@ async def main(item: str, qty: int, email: str): RunJob::from(JobPayload::Code(RawCode { language: ScriptLang::Python3, content, + tag: None, ..RawCode::default() })) .arg("item", json!("widget")) @@ -1298,6 +1299,7 @@ async def main(n: int): RunJob::from(JobPayload::Code(RawCode { language: ScriptLang::Python3, content, + tag: None, ..RawCode::default() })) .arg("n", json!(1)) diff --git a/backend/tests/script_modules.rs b/backend/tests/script_modules.rs index 068555c7df..8755ebfb57 100644 --- a/backend/tests/script_modules.rs +++ b/backend/tests/script_modules.rs @@ -40,6 +40,7 @@ def main(name: str): path: Some("f/test/my_script".to_string()), language: ScriptLang::Python3, modules: Some(modules), + tag: None, ..RawCode::default() }); @@ -92,6 +93,7 @@ def main(a: int, b: int): path: Some("f/test/my_script".to_string()), language: ScriptLang::Python3, modules: Some(modules), + tag: None, ..RawCode::default() }); @@ -143,6 +145,7 @@ export function main(name: string) { path: Some("f/test/my_script".to_string()), language: ScriptLang::Bun, modules: Some(modules), + tag: None, ..RawCode::default() }); diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index b220416b55..9445e6cd6a 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -4997,6 +4997,7 @@ async fn test_workflow_as_code(db: Pool) -> anyhow::Result<()> { RunJob::from(JobPayload::Code(RawCode { language: ScriptLang::Python3, content: WORKFLOW_AS_CODE.into(), + tag: None, ..RawCode::default() })) .arg("n", json!(3)) diff --git a/backend/windmill-api-assets/src/lib.rs b/backend/windmill-api-assets/src/lib.rs index 0c25cd243d..be685b2fe6 100644 --- a/backend/windmill-api-assets/src/lib.rs +++ b/backend/windmill-api-assets/src/lib.rs @@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use sqlx::Row; use windmill_common::{ - assets::{AssetKind, AssetUsageKind}, + assets::{parse_asset_trigger_ref, AssetKind, AssetUsageKind}, db::UserDB, error::JsonResult, utils::escape_ilike_pattern, @@ -20,6 +20,8 @@ pub fn workspaced_service() -> Router { .route("/list", get(list_assets)) .route("/list_by_usages", post(list_assets_by_usages)) .route("/list_favorites", get(list_favorites)) + .route("/graph", get(asset_graph)) + .route("/pipelines", get(list_pipeline_folders)) } #[derive(Deserialize)] @@ -363,3 +365,445 @@ async fn list_favorites( Ok(Json(favorites)) } + +// ------------------------------------------------------------------ +// GET /w/:workspace/assets/graph +// ------------------------------------------------------------------ +// Workspace-wide asset ↔ runnable graph. One row per unique +// (asset_kind, asset_path, usage_kind, usage_path, access_type) — the +// frontend aggregates into nodes and edges. + +#[derive(Deserialize)] +struct GraphQuery { + pub asset_kinds: Option, + pub folder: Option, +} + +#[derive(Serialize, Debug)] +struct GraphAssetNode { + kind: AssetKind, + path: String, +} + +#[derive(Serialize, Debug)] +struct GraphRunnableNode { + path: String, + usage_kind: AssetUsageKind, + // True iff the script was deployed with `// pipeline` — drives the + // pipeline-member visual state on the frontend. + #[serde(skip_serializing_if = "std::ops::Not::not", default)] + in_pipeline: bool, +} + +// Lineage edge from parsed r/w usages. One per (runnable, asset, access_type) +// tuple. Informational — not the DAG execution edges. +#[derive(Serialize, Debug)] +struct GraphEdge { + runnable_path: String, + runnable_kind: AssetUsageKind, + asset_kind: AssetKind, + asset_path: String, + access_type: Option, +} + +// Declared `// on ` trigger edge — the actual execution DAG. +// Asset edges come from `script_trigger`; the eight native variants +// (Schedule/Email/Kafka/…/Gcp) come from the per-kind trigger tables joined +// on `script_path`. Each native variant carries just the trigger row's path; +// the config (cron, broker, topic, auth, …) lives in its own UI. +// +// `webhook` is parsed as an annotation marker but has no dedicated trigger +// table — every script gets an implicit webhook endpoint — so no variant +// here. The frontend renders the marker from the source annotations alone. +#[derive(Serialize, Debug)] +#[serde(tag = "trigger_kind", rename_all = "lowercase")] +enum TriggerEdge { + Asset { + asset_kind: AssetKind, + asset_path: String, + runnable_kind: AssetUsageKind, + runnable_path: String, + }, + Schedule { + path: String, + runnable_kind: AssetUsageKind, + runnable_path: String, + }, + Email { + path: String, + runnable_kind: AssetUsageKind, + runnable_path: String, + }, + Kafka { + path: String, + runnable_kind: AssetUsageKind, + runnable_path: String, + }, + Mqtt { + path: String, + runnable_kind: AssetUsageKind, + runnable_path: String, + }, + Nats { + path: String, + runnable_kind: AssetUsageKind, + runnable_path: String, + }, + Postgres { + path: String, + runnable_kind: AssetUsageKind, + runnable_path: String, + }, + Sqs { + path: String, + runnable_kind: AssetUsageKind, + runnable_path: String, + }, + Gcp { + path: String, + runnable_kind: AssetUsageKind, + runnable_path: String, + }, +} + +#[derive(Serialize, Debug)] +struct AssetGraphResponse { + assets: Vec, + runnables: Vec, + edges: Vec, + triggers: Vec, +} + +async fn asset_graph( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, + Query(q): Query, +) -> JsonResult { + let mut tx = user_db.begin(&authed).await?; + + let kind_filter: Option> = q.asset_kinds.as_ref().map(|s| { + s.split(',') + .filter_map(|k| { + serde_json::from_value::(Value::String(k.trim().into())).ok() + }) + .collect() + }); + let kind_filter_ref = kind_filter.as_deref(); + + let folder_filter = q.folder.as_deref().map(|f| format!("f/{}/%", f)); + + // One row per (asset_kind, asset_path, usage_kind, usage_path, access_type). + // The `usage_kind IN ('script','flow')` clause excludes `job`-kind usage rows + // (runtime-detected, ephemeral) so the graph stays stable. + let rows = sqlx::query!( + r#" + SELECT + asset.kind AS "asset_kind!: AssetKind", + asset.path AS "asset_path!", + asset.usage_kind AS "usage_kind!: AssetUsageKind", + asset.usage_path AS "usage_path!", + asset.usage_access_type::text AS "access_type" + FROM asset + WHERE asset.workspace_id = $1 + AND asset.usage_kind IN ('script', 'flow') + AND ($2::asset_kind[] IS NULL OR asset.kind = ANY($2)) + AND ($3::text IS NULL OR asset.usage_path LIKE $3) + GROUP BY asset.kind, asset.path, asset.usage_kind, asset.usage_path, asset.usage_access_type + "#, + &w_id, + kind_filter_ref as Option<&[AssetKind]>, + folder_filter.as_deref(), + ) + .fetch_all(&mut *tx) + .await?; + + // Pipeline asset trigger edges, fetched separately so we can widen the + // runnable_set for trigger-only endpoints (e.g. an asset trigger whose + // asset has no usage in the pipeline yet). Native trigger kinds + // (schedule, kafka, mqtt, …) are *not* in `script_trigger` — they're + // discovered below by querying each native trigger table directly. + let trigger_rows = sqlx::query!( + r#" + SELECT + runnable_kind AS "runnable_kind!: AssetUsageKind", + runnable_path AS "runnable_path!", + trigger_kind::text AS "trigger_kind!", + trigger_ref AS "trigger_ref!" + FROM script_trigger + WHERE workspace_id = $1 + AND trigger_kind = 'asset' + AND ($2::text IS NULL OR runnable_path LIKE $2) + "#, + &w_id, + folder_filter.as_deref(), + ) + .fetch_all(&mut *tx) + .await?; + + // Native triggers in scope. Each native trigger table stores its + // single-destination `script_path` directly, so we resolve attachment by + // joining on that field rather than via `script_trigger`. UNION ALL keeps + // it a single round trip; the `kind` column drives the TriggerEdge ctor + // below. `schedule` lives in the `schedule` table, which has its own + // shape (no workspace_id-only filter — it shares `is_flow` like the + // others), but the columns we need line up. + let native_trigger_rows = sqlx::query!( + r#" + SELECT kind, path, script_path, is_flow FROM ( + SELECT 'schedule' AS kind, path, script_path, is_flow FROM schedule + WHERE workspace_id = $1 + AND script_path IS NOT NULL + UNION ALL + SELECT 'email', path, script_path, is_flow FROM email_trigger + WHERE workspace_id = $1 + UNION ALL + SELECT 'kafka', path, script_path, is_flow FROM kafka_trigger + WHERE workspace_id = $1 + UNION ALL + SELECT 'mqtt', path, script_path, is_flow FROM mqtt_trigger + WHERE workspace_id = $1 + UNION ALL + SELECT 'nats', path, script_path, is_flow FROM nats_trigger + WHERE workspace_id = $1 + UNION ALL + SELECT 'postgres', path, script_path, is_flow FROM postgres_trigger + WHERE workspace_id = $1 + UNION ALL + SELECT 'sqs', path, script_path, is_flow FROM sqs_trigger + WHERE workspace_id = $1 + UNION ALL + SELECT 'gcp', path, script_path, is_flow FROM gcp_trigger + WHERE workspace_id = $1 + ) t + WHERE ($2::text IS NULL OR script_path LIKE $2) + "#, + &w_id, + folder_filter.as_deref(), + ) + .fetch_all(&mut *tx) + .await?; + + // Which scripts in scope are pipeline members (have `// pipeline`). + let pipeline_member_paths = sqlx::query!( + r#" + SELECT path AS "path!" + FROM script + WHERE workspace_id = $1 + AND auto_kind = 'pipeline' + AND archived = false + AND deleted = false + AND ($2::text IS NULL OR path LIKE $2) + "#, + &w_id, + folder_filter.as_deref(), + ) + .fetch_all(&mut *tx) + .await?; + + // Existing scripts / flows in the workspace. Used to filter out + // orphan trigger rows whose `script_path` no longer resolves — those + // would otherwise be added to `runnable_set` below and surface as + // phantom "deployed" runnables on the canvas (matching what the user + // can deploy a new trigger against: nothing). + let existing_script_paths = sqlx::query_scalar!( + r#"SELECT path AS "path!" FROM script + WHERE workspace_id = $1 + AND archived = false + AND deleted = false"#, + &w_id, + ) + .fetch_all(&mut *tx) + .await?; + let existing_flow_paths = sqlx::query_scalar!( + r#"SELECT path AS "path!" FROM flow WHERE workspace_id = $1 AND archived = false"#, + &w_id, + ) + .fetch_all(&mut *tx) + .await?; + + tx.commit().await?; + + let pipeline_member_script_paths: std::collections::HashSet = + pipeline_member_paths.into_iter().map(|r| r.path).collect(); + let existing_script_paths: std::collections::HashSet = + existing_script_paths.into_iter().collect(); + let existing_flow_paths: std::collections::HashSet = + existing_flow_paths.into_iter().collect(); + let runnable_exists = |kind: AssetUsageKind, path: &str| match kind { + AssetUsageKind::Script => existing_script_paths.contains(path), + AssetUsageKind::Flow => existing_flow_paths.contains(path), + // `Job` is a runtime-detected ephemeral runnable (asset usage rows + // only), never a target of a stored trigger row. Treat as existing + // so we don't accidentally drop ephemeral lineage edges. + AssetUsageKind::Job => true, + }; + + let mut edges = Vec::with_capacity(rows.len()); + let mut asset_set: std::collections::HashSet<(AssetKind, String)> = Default::default(); + let mut runnable_set: std::collections::HashSet<(AssetUsageKind, String)> = Default::default(); + + // Every pipeline member in scope goes into the graph, even when the parser + // didn't detect any asset r/w and the script has no triggers yet. Without + // this, a freshly-saved pipeline script whose template body hasn't been + // filled in would vanish from the pipeline view on graph refetch. + for path in &pipeline_member_script_paths { + runnable_set.insert((AssetUsageKind::Script, path.clone())); + } + + for r in rows { + // Drop asset usage rows whose runnable target was archived/deleted + // but whose row in `asset` is still around — those would otherwise + // surface as a phantom "deployed" runnable on the canvas with no + // way to interact with it, since the underlying script/flow no + // longer exists. + if !runnable_exists(r.usage_kind, &r.usage_path) { + continue; + } + asset_set.insert((r.asset_kind, r.asset_path.clone())); + runnable_set.insert((r.usage_kind, r.usage_path.clone())); + edges.push(GraphEdge { + runnable_path: r.usage_path, + runnable_kind: r.usage_kind, + asset_kind: r.asset_kind, + asset_path: r.asset_path, + access_type: r.access_type, + }); + } + + let mut triggers: Vec = + Vec::with_capacity(trigger_rows.len() + native_trigger_rows.len()); + for t in trigger_rows { + // Drop orphan asset-trigger rows — their target runnable no longer + // exists (script/flow archived or deleted, or was never deployed). + // Without this, an orphan row would surface as a phantom "deployed" + // runnable on the canvas (no `unsaved` flag, can't actually be + // run / re-targeted by a new trigger). + if !runnable_exists(t.runnable_kind, &t.runnable_path) { + continue; + } + runnable_set.insert((t.runnable_kind, t.runnable_path.clone())); + if t.trigger_kind.as_str() == "asset" { + // trigger_ref is `` — parse back out so both + // endpoints match what the frontend uses for node ids. + if let Some((asset_kind, asset_path)) = parse_asset_trigger_ref(&t.trigger_ref) { + // Make sure the source asset has a node even if nothing + // reads/writes it in this folder. + asset_set.insert((asset_kind, asset_path.clone())); + triggers.push(TriggerEdge::Asset { + asset_kind, + asset_path, + runnable_kind: t.runnable_kind, + runnable_path: t.runnable_path, + }); + } + } + // Native kinds (schedule, kafka, mqtt, …) come from per-kind trigger + // tables below. + } + + // Native trigger attachments — one TriggerEdge per row, the kind chosen + // from the discriminator. Add the runnable to the set so a script with + // no asset edges but a kafka/schedule attachment still renders on the + // canvas. + for t in native_trigger_rows { + let kind = t.kind.unwrap_or_default(); + let path = t.path.unwrap_or_default(); + let script_path = t.script_path.unwrap_or_default(); + let runnable_kind = if t.is_flow.unwrap_or(false) { + AssetUsageKind::Flow + } else { + AssetUsageKind::Script + }; + // Same orphan filter as the asset-trigger loop above — drop trigger + // rows whose target script/flow no longer exists so the graph + // doesn't synthesize a phantom deployed runnable. + if !runnable_exists(runnable_kind, &script_path) { + continue; + } + runnable_set.insert((runnable_kind, script_path.clone())); + let edge = match kind.as_str() { + "schedule" => TriggerEdge::Schedule { path, runnable_kind, runnable_path: script_path }, + "email" => TriggerEdge::Email { path, runnable_kind, runnable_path: script_path }, + "kafka" => TriggerEdge::Kafka { path, runnable_kind, runnable_path: script_path }, + "mqtt" => TriggerEdge::Mqtt { path, runnable_kind, runnable_path: script_path }, + "nats" => TriggerEdge::Nats { path, runnable_kind, runnable_path: script_path }, + "postgres" => TriggerEdge::Postgres { path, runnable_kind, runnable_path: script_path }, + "sqs" => TriggerEdge::Sqs { path, runnable_kind, runnable_path: script_path }, + "gcp" => TriggerEdge::Gcp { path, runnable_kind, runnable_path: script_path }, + _ => continue, + }; + triggers.push(edge); + } + + let mut assets: Vec = asset_set + .into_iter() + .map(|(kind, path)| GraphAssetNode { kind, path }) + .collect(); + assets.sort_by(|a, b| a.path.cmp(&b.path)); + + let mut runnables: Vec = runnable_set + .into_iter() + .map(|(usage_kind, path)| { + let in_pipeline = usage_kind == AssetUsageKind::Script + && pipeline_member_script_paths.contains(&path); + GraphRunnableNode { path, usage_kind, in_pipeline } + }) + .collect(); + runnables.sort_by(|a, b| a.path.cmp(&b.path)); + + Ok(Json(AssetGraphResponse { + assets, + runnables, + edges, + triggers, + })) +} + +// ------------------------------------------------------------------ +// GET /w/:workspace/assets/pipelines +// ------------------------------------------------------------------ +// Distinct folder names that contain at least one pipeline-member script +// (auto_kind='pipeline'). Used by the pipeline-editor folder picker and +// the "Pipeline" entry in folder views. Keyed by the partial index on +// `script (workspace_id, path) WHERE auto_kind='pipeline' ...` so this +// is effectively O(matches). + +#[derive(Serialize, Debug)] +struct PipelineFolder { + folder: String, + script_count: i64, +} + +async fn list_pipeline_folders( + authed: ApiAuthed, + Path(w_id): Path, + Extension(user_db): Extension, +) -> JsonResult> { + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query!( + r#" + SELECT + substring(path from '^f/([^/]+)/') AS "folder!", + COUNT(*) AS "script_count!" + FROM script + WHERE workspace_id = $1 + AND auto_kind = 'pipeline' + AND archived = false + AND deleted = false + AND path LIKE 'f/%' + GROUP BY substring(path from '^f/([^/]+)/') + ORDER BY substring(path from '^f/([^/]+)/') + "#, + &w_id, + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + + Ok(Json( + rows.into_iter() + .map(|r| PipelineFolder { folder: r.folder, script_count: r.script_count }) + .collect(), + )) +} diff --git a/backend/windmill-api-scripts/Cargo.toml b/backend/windmill-api-scripts/Cargo.toml index 5ea3088837..2dee051c93 100644 --- a/backend/windmill-api-scripts/Cargo.toml +++ b/backend/windmill-api-scripts/Cargo.toml @@ -12,7 +12,7 @@ path = "src/lib.rs" default = [] enterprise = ["windmill-common/enterprise"] private = ["windmill-common/private", "windmill-dep-map/private"] -python = ["dep:windmill-parser-py"] +python = ["dep:windmill-parser-py", "dep:windmill-parser-py-asset"] prometheus = ["dep:prometheus", "windmill-common/prometheus"] [dependencies] windmill-common = { workspace = true, default-features = false } @@ -23,6 +23,10 @@ windmill-audit.workspace = true windmill-git-sync.workspace = true windmill-dep-map.workspace = true windmill-parser-ts.workspace = true +windmill-parser.workspace = true +windmill-parser-ts-asset.workspace = true +windmill-parser-sql-asset.workspace = true +windmill-parser-yaml.workspace = true axum.workspace = true futures.workspace = true @@ -41,3 +45,4 @@ lazy_static.workspace = true tokio.workspace = true prometheus = { workspace = true, optional = true } windmill-parser-py = { workspace = true, optional = true } +windmill-parser-py-asset = { workspace = true, optional = true } diff --git a/backend/windmill-api-scripts/src/asset_inference.rs b/backend/windmill-api-scripts/src/asset_inference.rs new file mode 100644 index 0000000000..5debb0d271 --- /dev/null +++ b/backend/windmill-api-scripts/src/asset_inference.rs @@ -0,0 +1,254 @@ +//! Server-side asset inference at script deploy. +//! +//! The `asset` rows written at deploy drive the asset-trigger cascade +//! (`fetch_producer_writes` in windmill-queue). Historically they came solely +//! from the client-supplied `NewScript.assets`, so a client with broken +//! inference (e.g. a failed wasm load) deploying `assets: null` silently +//! killed the producer side of the cascade while the subscriber side (parsed +//! server-side from `// on` annotations) kept looking wired. This module makes +//! asset *presence* server-authoritative by re-parsing the deployed content +//! with the same parsers the frontend wasm builds wrap. +//! +//! Merge semantics are a union: server-parsed assets are always present; +//! client entries are kept too (they may carry `alt_access_type` — the user's +//! manual access-type override — or come from client-side detection paths the +//! server has no parser for). For duplicate `(kind, path)` keys the server's +//! parser-derived fields win and the client's `alt_access_type` is preserved. +//! Languages without a server parser (or whose parse fails) fall back to the +//! client list unchanged, matching the previous behavior. + +use std::collections::BTreeMap; + +use windmill_common::{ + assets::{ + asset_access_type_from_parser, asset_kind_from_parser, AssetKind, AssetUsageAccessType, + AssetWithAltAccessType, + }, + scripts::ScriptLang, +}; + +/// Mirror of the frontend `inferAssets` language dispatch (infer.ts): only +/// these languages have a body-asset parser. Returns `None` for unsupported +/// languages or on parse failure — callers then keep the client-supplied list. +fn parse_assets_for_lang( + lang: &ScriptLang, + content: &str, +) -> Option> { + let parsed = match lang { + ScriptLang::DuckDb => windmill_parser_sql_asset::parse_assets(content), + ScriptLang::Bun | ScriptLang::Deno | ScriptLang::Nativets => { + windmill_parser_ts_asset::parse_assets(content) + } + #[cfg(feature = "python")] + ScriptLang::Python3 => windmill_parser_py_asset::parse_assets(content), + ScriptLang::Ansible => windmill_parser_yaml::parse_assets(content), + _ => return None, + }; + match parsed { + Ok(out) => Some(out.assets), + Err(e) => { + tracing::warn!( + "server-side asset inference failed for a {} script; falling back to \ + client-supplied assets: {e:#}", + lang.as_str() + ); + None + } + } +} + +/// Mirror of the frontend `getCommentPrefix` (infer.ts) — the languages whose +/// leading comment block is scanned for `volume: ` annotations. +fn comment_prefix(lang: &ScriptLang) -> Option<&'static str> { + match lang { + ScriptLang::Python3 + | ScriptLang::Bash + | ScriptLang::Powershell + | ScriptLang::Ansible + | ScriptLang::Ruby + | ScriptLang::Rlang => Some("#"), + ScriptLang::Deno + | ScriptLang::Bun + | ScriptLang::Bunnative + | ScriptLang::Nativets + | ScriptLang::Go => Some("//"), + _ => None, + } +} + +/// Mirror of the frontend `parseVolumeAnnotations` (infer.ts): ` +/// volume: ` lines in the leading comment block, each an `rw` volume +/// asset. Scanning stops at the first non-comment line (blank lines are +/// skipped), exactly like the frontend. +fn parse_volume_annotations(content: &str, prefix: &str) -> Vec { + let mut volumes = Vec::new(); + for line in content.lines() { + let trimmed = line.trim(); + if trimmed.is_empty() { + continue; + } + let Some(after) = trimmed.strip_prefix(prefix) else { + break; + }; + let after = after.trim(); + if let Some(rest) = after.strip_prefix("volume:") { + if let Some(path) = rest.trim().split_whitespace().next() { + volumes.push(AssetWithAltAccessType { + path: path.to_string(), + kind: AssetKind::Volume, + access_type: Some(AssetUsageAccessType::RW), + alt_access_type: None, + columns: None, + }); + } + } + } + volumes +} + +/// Parse the deployed content into asset usages, mirroring the frontend's +/// `inferAssets` (body parser per language + volume annotations). `None` +/// means "no server parser produced anything for this language" — distinct +/// from `Some(vec![])`, which is an authoritative "this script uses no +/// assets". +fn infer_script_assets(lang: &ScriptLang, content: &str) -> Option> { + let body_assets = parse_assets_for_lang(lang, content); + let volume_assets = comment_prefix(lang) + .map(|p| parse_volume_annotations(content, p)) + .unwrap_or_default(); + if body_assets.is_none() && volume_assets.is_empty() { + return None; + } + let mut out: Vec = body_assets + .unwrap_or_default() + .into_iter() + .map(|a| AssetWithAltAccessType { + path: a.path, + kind: asset_kind_from_parser(a.kind), + access_type: a.access_type.map(asset_access_type_from_parser), + alt_access_type: None, + columns: a.columns.map(|cols| { + cols.into_iter() + .map(|(k, v)| (k, asset_access_type_from_parser(v))) + .collect::>() + }), + }) + .collect(); + out.extend(volume_assets); + Some(out) +} + +/// The asset list to persist at deploy: server-parsed assets unioned with the +/// client-supplied ones. See the module docs for the exact semantics. +pub fn effective_script_assets( + lang: &ScriptLang, + content: &str, + client_assets: Option>, +) -> Option> { + let Some(inferred) = infer_script_assets(lang, content) else { + return client_assets; + }; + let mut merged: Vec = inferred; + for client in client_assets.into_iter().flatten() { + if let Some(existing) = merged + .iter_mut() + .find(|a| a.kind == client.kind && a.path == client.path) + { + // Server parse wins for parser-derived fields; the client's + // alt_access_type is the user's manual override — keep it. + if existing.alt_access_type.is_none() { + existing.alt_access_type = client.alt_access_type; + } + } else { + merged.push(client); + } + } + Some(merged) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn asset( + kind: AssetKind, + path: &str, + access: Option, + ) -> AssetWithAltAccessType { + AssetWithAltAccessType { + path: path.to_string(), + kind, + access_type: access, + alt_access_type: None, + columns: None, + } + } + + // The bug class this module exists for: client deploys with no assets, + // but the content demonstrably writes one — the server parse must + // produce the producer row anyway. + #[test] + fn duckdb_write_survives_empty_client_assets() { + let content = "ATTACH 'datatable://main' AS pg;\n\ + CREATE TABLE IF NOT EXISTS pg.out_table AS\n\ + SELECT * FROM (SELECT 1 AS placeholder);"; + let got = effective_script_assets(&ScriptLang::DuckDb, content, None).unwrap(); + assert_eq!(got.len(), 1); + assert_eq!(got[0].kind, AssetKind::DataTable); + assert_eq!(got[0].path, "main/out_table"); + assert_eq!(got[0].access_type, Some(AssetUsageAccessType::W)); + } + + #[test] + fn client_alt_access_type_is_preserved_on_match() { + let content = "ATTACH 'datatable://main' AS pg;\n\ + CREATE TABLE IF NOT EXISTS pg.out_table AS SELECT 1;"; + let mut client = asset(AssetKind::DataTable, "main/out_table", None); + client.alt_access_type = Some(AssetUsageAccessType::RW); + let got = + effective_script_assets(&ScriptLang::DuckDb, content, Some(vec![client])).unwrap(); + assert_eq!(got.len(), 1); + // Parser-derived access wins; the user's alt override rides along. + assert_eq!(got[0].access_type, Some(AssetUsageAccessType::W)); + assert_eq!(got[0].alt_access_type, Some(AssetUsageAccessType::RW)); + } + + #[test] + fn client_only_entries_are_kept() { + let content = "ATTACH 'datatable://main' AS pg;\n\ + CREATE TABLE IF NOT EXISTS pg.out_table AS SELECT 1;"; + let extra = asset( + AssetKind::S3Object, + "bucket/file.parquet", + Some(AssetUsageAccessType::R), + ); + let got = effective_script_assets(&ScriptLang::DuckDb, content, Some(vec![extra])).unwrap(); + assert_eq!(got.len(), 2); + assert!(got.iter().any(|a| a.kind == AssetKind::S3Object)); + } + + #[test] + fn unsupported_language_falls_back_to_client() { + let client = vec![asset( + AssetKind::S3Object, + "b/f.json", + Some(AssetUsageAccessType::W), + )]; + let got = effective_script_assets(&ScriptLang::Go, "package main", Some(client.clone())); + assert_eq!(got.map(|v| v.len()), Some(1)); + assert_eq!( + effective_script_assets(&ScriptLang::Go, "package main", None).is_none(), + true + ); + } + + #[test] + fn volume_annotations_parsed_from_leading_comment_block() { + let content = "// volume: my_vol\n// some other comment\nconsole.log(1)\n// volume: ignored_after_code\n"; + let got = effective_script_assets(&ScriptLang::Bun, content, None).unwrap(); + let vols: Vec<_> = got.iter().filter(|a| a.kind == AssetKind::Volume).collect(); + assert_eq!(vols.len(), 1); + assert_eq!(vols[0].path, "my_vol"); + assert_eq!(vols[0].access_type, Some(AssetUsageAccessType::RW)); + } +} diff --git a/backend/windmill-api-scripts/src/lib.rs b/backend/windmill-api-scripts/src/lib.rs index 9bc8318fc4..3ac3764e96 100644 --- a/backend/windmill-api-scripts/src/lib.rs +++ b/backend/windmill-api-scripts/src/lib.rs @@ -6,4 +6,5 @@ * LICENSE-AGPL for a copy of the license. */ +pub mod asset_inference; pub mod scripts; diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index 79583835b0..135256f3d0 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -44,8 +44,9 @@ use windmill_dep_map::scoped_dependency_map::ScopedDependencyMap; use windmill_common::{ assets::{ - clear_static_asset_usage, clear_static_asset_usage_by_script_hash, - insert_static_asset_usage, AssetUsageKind, + clear_script_triggers, clear_static_asset_usage, clear_static_asset_usage_by_script_hash, + insert_script_trigger, insert_static_asset_usage, parse_duration_secs, + parse_pipeline_annotations, trigger_spec_to_row, AssetUsageKind, TriggerSpec, }, error::{self, to_anyhow}, min_version::{MIN_VERSION_SUPPORTS_DEBOUNCING, MIN_VERSION_SUPPORTS_DEBOUNCING_V2}, @@ -1235,7 +1236,63 @@ async fn create_script_internal<'c>( let ci_test_refs = windmill_common::schema::parse_ci_test_annotation(&ns.content, &lang.as_comment_lit()); - let auto_kind = if ci_test_refs.is_some() { + // `pipeline` wins over `test` and any client-supplied auto_kind. The + // bare `// pipeline` marker is the opt-in signal for pipeline + // membership; parsed writes tell us what is produced (we don't record + // them in auto_kind itself). + let pipeline_annotations = parse_pipeline_annotations(&ns.content); + // `// freshness` is parsed but enforcement is a not-yet-implemented + // enterprise feature (skeleton in windmill_common::pipeline_advanced). + // Surface a clear TODO at deploy rather than silently accepting an + // annotation that does nothing. + if pipeline_annotations.freshness.is_some() { + tracing::warn!( + "{}", + windmill_common::pipeline_advanced::freshness_enforcement_todo() + ); + } + let in_pipeline = pipeline_annotations.in_pipeline; + // `// trigger all` → AND join barrier (else OR, the default). + let pipeline_join_all = !pipeline_annotations.join_mode.is_any(); + // Script-level `// debounce ` default; a per-`// on debounce=` + // overrides it (precedence resolved per edge below). + let pipeline_debounce_default = pipeline_annotations.debounce_default; + let pipeline_triggers = pipeline_annotations.triggers; + // `// tag ` overrides the caller-supplied tag at deploy. Source + // wins, matching the wipe-and-reinsert convention of other pipeline + // annotations. Applied before the dep-job tag selection below (which + // special-cases dedicated_worker / bunnative / `$args[`) so that path + // sees the annotation-overridden value. + if let Some(t) = pipeline_annotations.tag.clone() { + ns.tag = Some(t); + } + // `// retry []` is PARSED but PARKED: a retried subscriber + // is wrapped in a SingleStepFlow, whose run is a flow step and therefore + // ineligible for asset dispatch (asset_dispatch::is_eligible_kind) — so a + // retried subscriber would silently become a cascade dead-end (P1). We do + // not persist it to script_trigger; the cascade ignores retry until this + // is fixed. TODO(pipeline-retry): re-enable once cascade dispatch handles + // flow-wrapped producers. + if pipeline_annotations.retry.is_some() { + tracing::warn!( + "`// retry` on {} is not yet supported in the asset cascade and is ignored \ + (a retried subscriber cannot trigger its downstream). TODO(pipeline-retry).", + ns.path + ); + } + // Asset presence is server-authoritative: re-parse the deployed content + // (same parsers the frontend wasm wraps) and union with the client list. + // The `asset` rows written below drive the asset-trigger cascade, so a + // client deploying `assets: null` (e.g. broken wasm inference) must not + // silently kill the producer side while `// on` subscribers stay wired. + let effective_assets = crate::asset_inference::effective_script_assets( + &ns.language, + &ns.content, + ns.assets.take(), + ); + let auto_kind = if in_pipeline { + Some("pipeline".to_string()) + } else if ci_test_refs.is_some() { Some("test".to_string()) } else { auto_kind @@ -1312,7 +1369,9 @@ async fn create_script_internal<'c>( &authed, ), validate_schema, - ns.assets.as_ref().and_then(|a| serde_json::to_value(a).ok()), + effective_assets + .as_ref() + .and_then(|a| serde_json::to_value(a).ok()), guarded_debounce_key, guarded_debounce_delay_s, ns.cache_ignore_s3_path, @@ -1551,11 +1610,64 @@ async fn create_script_internal<'c>( } clear_static_asset_usage(&mut *tx, &w_id, &script_path, AssetUsageKind::Script).await?; - for asset in ns.assets.as_ref().into_iter().flatten() { + for asset in effective_assets.as_ref().into_iter().flatten() { insert_static_asset_usage(&mut *tx, &w_id, &asset, &ns.path, AssetUsageKind::Script) .await?; } + // Pipeline trigger edges: wipe-and-reinsert per deploy so removing an + // `// on ...` annotation drops the edge. Only Asset / Schedule produce + // a row — native trigger marker annotations (`// on kafka`, etc.) are + // discovered by the graph endpoint directly from the per-kind trigger + // tables, so `trigger_spec_to_row` returns None for those. + clear_script_triggers(&mut *tx, &w_id, &ns.path, AssetUsageKind::Script).await?; + // On rename, also drop the OLD path's trigger rows. clear is keyed by + // path (no by-hash variant), and only `ns.path` is wiped above — without + // this, stale `// on` edges for the old path keep matching producers and + // would trigger a script later recreated at that path even if it has no + // annotation (P1). (Producer/asset rows for the old path are already + // cleared via clear_static_asset_usage_by_script_hash on the parent.) + if let Some(ref old) = p_path_opt { + if old != &ns.path { + clear_script_triggers(&mut *tx, &w_id, old, AssetUsageKind::Script).await?; + } + } + for spec in &pipeline_triggers { + let Some((trigger_kind, trigger_ref)) = trigger_spec_to_row(spec) else { + continue; + }; + // Effective debounce for this edge: per-`// on debounce=` wins, + // else the script-level `// debounce` default. Debounce only + // applies to asset-cascade edges; other trigger kinds get none. + let debounce_s = match spec { + TriggerSpec::Asset { debounce: Some(d), .. } => parse_duration_secs(d), + TriggerSpec::Asset { .. } => pipeline_debounce_default + .as_deref() + .and_then(parse_duration_secs), + _ => None, + }; + insert_script_trigger( + &mut *tx, + &w_id, + AssetUsageKind::Script, + &ns.path, + trigger_kind, + &trigger_ref, + pipeline_join_all, + debounce_s, + // retry parked — see TODO(pipeline-retry) above. + None, + None, + ) + .await?; + } + + // Schedule annotations (`// on schedule`) are marker-only — the binding + // lives on the schedule row's own `script_path` field, which the user + // creates separately via the schedule editor. No script-create-time + // reconciliation is needed (and there are no "managed" schedules to + // upsert/delete anymore). + let permissioned_as = username_to_permissioned_as(&authed.username); if let Some(parent_hash) = ns.parent_hash { tracing::info!( @@ -2548,6 +2660,10 @@ async fn archive_script_by_path( .map_err(|e| Error::internal_err(format!("archiving script in {w_id}: {e:#}")))?; clear_static_asset_usage(&mut *tx, &w_id, path, AssetUsageKind::Script).await?; + // Pipeline event hygiene: an archived script must not be triggered by + // anything. Wipe declared `// on ...` edges (asset-event subscribers + // look these up). + clear_script_triggers(&mut *tx, &w_id, path, AssetUsageKind::Script).await?; audit_log( &mut *tx, @@ -2625,6 +2741,9 @@ async fn archive_script_by_hash( check_scopes(&authed, || format!("scripts:write:{}", &script.path))?; clear_static_asset_usage_by_script_hash(&mut *tx, &w_id, hash).await?; + // Pipeline event hygiene: archived scripts must not be triggered by + // anything. Wipe declared `// on ...` edges. + clear_script_triggers(&mut *tx, &w_id, &script.path, AssetUsageKind::Script).await?; audit_log( &mut *tx, @@ -2685,6 +2804,10 @@ async fn delete_script_by_hash( check_scopes(&authed, || format!("scripts:write:{}", &script.path))?; clear_static_asset_usage_by_script_hash(&mut *tx, &w_id, hash).await?; + // Pipeline event hygiene: a deleted script must not be triggered by + // anything. Wipe declared `// on ...` edges. Idempotent — safe even if + // the script was never a pipeline member. + clear_script_triggers(&mut *tx, &w_id, &script.path, AssetUsageKind::Script).await?; audit_log( &mut *tx, @@ -2797,6 +2920,11 @@ async fn delete_script_by_path( .execute(&mut *tx) .await?; + // Pipeline event hygiene: a deleted script must not be triggered by + // anything. Wipe declared `// on ...` edges. Idempotent — safe even if + // the script was never a pipeline member. + clear_script_triggers(&mut *tx, &w_id, path, AssetUsageKind::Script).await?; + if !query.keep_captures.unwrap_or(false) { sqlx::query!( "DELETE FROM capture_config WHERE path = $1 AND workspace_id = $2 AND is_flow IS FALSE", diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index cd5b320339..65051a4157 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -8000,6 +8000,35 @@ paths: "404": description: no draft for that owner at that path + /w/{workspace}/drafts/get_own/{kind}/{path}: + get: + summary: fetch the current user's own draft content at a path (any kind) + operationId: getOwnDraft + tags: + - draft + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: kind + in: path + required: true + schema: + $ref: "#/components/schemas/UserDraftItemKind" + - $ref: "#/components/parameters/ScriptPath" + responses: + "200": + description: the user's draft content, or null when none exists + content: + application/json: + schema: + nullable: true + type: object + properties: + value: {} + created_at: + type: string + format: date-time + required: [value, created_at] + /w/{workspace}/drafts/update/{kind}/{path}: post: summary: upsert (or clear) the current user's draft at a path @@ -12982,6 +13011,147 @@ paths: required: - created_at + /w/{workspace}/jobs_u/dispatch_events/{id}: + get: + summary: list asset-trigger dispatch events for a producer job + description: > + Returns the chronological log of decisions the asset-trigger + dispatcher made after this producer job completed. Each row is one + (subscriber, asset write) decision: `dispatched` (with + `child_job_id`), `join_pending` (with `received_inputs` / + `required_inputs` / `partition`), or `skipped` (with `reason`). + Rows are reaped automatically when the producer's `v2_job` row is + deleted by the retention sweep. + operationId: listDispatchEvents + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/JobId" + responses: + "200": + description: dispatch events for this producer job + content: + application/json: + schema: + type: array + items: + type: object + properties: + subscriber_path: + type: string + asset_kind: + type: string + enum: + - s3object + - resource + - variable + - ducklake + - datatable + - volume + asset_path: + type: string + outcome: + type: string + enum: + - dispatched + - join_pending + - skipped + child_job_id: + type: string + format: uuid + partition: + type: string + received_inputs: + type: integer + required_inputs: + type: integer + debounce_s: + type: integer + reason: + type: string + created_at: + type: string + format: date-time + required: + - subscriber_path + - asset_kind + - asset_path + - outcome + - created_at + + /w/{workspace}/jobs/asset_dispatch_edges: + get: + summary: list asset-cascade producer→child job edges for a folder + description: > + Returns the `dispatched` asset-trigger edges (producer job → child job) + whose subscriber lives under `path_start`. Lets a pipeline view + reconstruct the cascade tree of a folder by job id and group connected + runs. Visibility follows the producer job's RLS. + operationId: listAssetDispatchEdges + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: path_start + in: query + required: true + description: Folder path prefix the children live under, e.g. `f/orders/`. + schema: + type: string + - name: created_after + in: query + required: false + description: Only edges dispatched at/after this instant. + schema: + type: string + format: date-time + responses: + "200": + description: asset-cascade edges for the folder + content: + application/json: + schema: + type: array + items: + type: object + properties: + producer_job_id: + type: string + format: uuid + child_job_id: + type: string + format: uuid + description: Set for `dispatched`; absent for `join_pending` inputs. + subscriber_path: + type: string + outcome: + type: string + enum: + - dispatched + - join_pending + asset_kind: + type: string + enum: + - s3object + - resource + - variable + - ducklake + - datatable + - volume + asset_path: + type: string + created_at: + type: string + format: date-time + required: + - producer_job_id + - subscriber_path + - outcome + - asset_kind + - asset_path + - created_at + /w/{workspace}/jobs/completed/delete/{id}: post: summary: delete completed job (erase content but keep run id) @@ -20598,6 +20768,149 @@ paths: type: string description: The asset path + /w/{workspace}/assets/graph: + get: + summary: Get the workspace-wide asset <-> runnable graph + operationId: getAssetsGraph + tags: + - asset + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: asset_kinds + in: query + description: Filter by asset kinds (comma-separated list) + schema: + type: string + - name: folder + in: query + description: Scope the graph to runnables in a single folder + schema: + type: string + responses: + "200": + description: asset graph nodes, lineage edges and trigger edges + content: + application/json: + schema: + type: object + required: [assets, runnables, edges, triggers] + properties: + assets: + type: array + items: + type: object + required: [kind, path] + properties: + kind: + $ref: "#/components/schemas/AssetKind" + path: + type: string + runnables: + type: array + items: + type: object + required: [path, usage_kind] + properties: + path: + type: string + usage_kind: + $ref: "#/components/schemas/AssetUsageKind" + in_pipeline: + type: boolean + description: True iff the script is a pipeline member (deployed with `// pipeline`). Omitted when false. + edges: + type: array + items: + type: object + required: + [runnable_path, runnable_kind, asset_kind, asset_path] + properties: + runnable_path: + type: string + runnable_kind: + $ref: "#/components/schemas/AssetUsageKind" + asset_kind: + $ref: "#/components/schemas/AssetKind" + asset_path: + type: string + access_type: + $ref: "#/components/schemas/AssetUsageAccessType" + triggers: + type: array + items: + oneOf: + - type: object + description: Asset trigger edge (`// on `) + required: + [ + trigger_kind, + asset_kind, + asset_path, + runnable_kind, + runnable_path, + ] + properties: + trigger_kind: + type: string + enum: [asset] + asset_kind: + $ref: "#/components/schemas/AssetKind" + asset_path: + type: string + runnable_kind: + $ref: "#/components/schemas/AssetUsageKind" + runnable_path: + type: string + - type: object + description: Native trigger edge (schedule, email, kafka, ...). `path` is the trigger row's path. + required: + [trigger_kind, path, runnable_kind, runnable_path] + properties: + trigger_kind: + type: string + enum: + - schedule + - email + - kafka + - mqtt + - nats + - postgres + - sqs + - gcp + path: + type: string + runnable_kind: + $ref: "#/components/schemas/AssetUsageKind" + runnable_path: + type: string + + /w/{workspace}/assets/pipelines: + get: + summary: List folders that contain at least one pipeline-member script + operationId: listPipelineFolders + tags: + - asset + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: folders containing pipeline scripts, with their script counts + content: + application/json: + schema: + type: array + items: + type: object + required: [folder, script_count] + properties: + folder: + type: string + description: The folder name (without the `f/` prefix) + script_count: + type: integer + format: int64 + description: Number of pipeline-member scripts in the folder + /w/{workspace}/volumes/list: get: summary: List all volumes in the workspace @@ -21332,6 +21645,7 @@ components: - trigger_nextcloud - trigger_google - trigger_github + - data_pipeline # Do not change next line. It is used by python-client for pre-processing # -- INLINE START -- OpenFlow: @@ -24494,6 +24808,7 @@ components: - azure - google - github + - asset TriggerMode: description: job trigger mode diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index c8ed841e19..7b8d468da8 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -282,6 +282,12 @@ pub async fn migrate( 20260207000002, 20260207000003, 20260207000004, + // Squashed pre-release pipeline migrations: the per-column ALTERs on + // script_trigger and the dispatch_event subscriber index were folded + // back into these two CREATEs, changing their checksum. Both are + // idempotent, so re-applying on an already-migrated DB is a no-op. + 20260423050000, + 20260523055641, ]; for m in migrator.migrations.iter() { if m.migration_type.is_down_migration() { diff --git a/backend/windmill-api/src/drafts.rs b/backend/windmill-api/src/drafts.rs index 5117286b96..02c169f59e 100644 --- a/backend/windmill-api/src/drafts.rs +++ b/backend/windmill-api/src/drafts.rs @@ -25,6 +25,7 @@ pub fn workspaced_service() -> Router { Router::new() .route("/list", get(list_drafts)) .route("/get/{kind}/{*path}", get(get_draft_for_user)) + .route("/get_own/{kind}/{*path}", get(get_own_draft)) .route("/update/{kind}/{*path}", post(update_draft)) .route("/migrate_legacy/{kind}/{*path}", post(migrate_legacy_draft)) } @@ -564,6 +565,37 @@ async fn get_draft_for_user( }) } +/// Fetch the AUTHED user's OWN draft at a path, for any kind — including +/// private kinds (`shares_drafts_across_users() == false`). Backs editors with +/// no deployed-item GET to overlay a draft onto: the `data_pipeline` bundle is +/// keyed at a folder path with no runnable to hang `get_draft` on, so it loads +/// its in-flight state from here. Returns `null` (200) when the user has no +/// draft there, so a fresh pipeline isn't a 404. Secret-variable values come +/// back `$encrypted:`-prefixed, same as `get_draft_for_user` — variable editors +/// use their own overlay GET, not this route. +async fn get_own_draft( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, kind, path)): Path<(String, UserDraftItemKind, windmill_common::utils::StripPath)>, +) -> Result>> { + let path = path.to_path(); + require_can_read_path(&authed, &user_db, &w_id, kind, path).await?; + let row = sqlx::query_as!( + DraftForUser, + r#"SELECT value as "value!: sqlx::types::Json>", created_at + FROM draft + WHERE workspace_id = $1 AND path = $2 AND typ = $3 AND email = $4"#, + &w_id, + path, + kind as UserDraftItemKind, + &authed.email, + ) + .fetch_optional(&db) + .await?; + Ok(Json(row)) +} + /// The deployed table RLS resolves item-level `extra_perms` against. /// Delegates to `UserDraftItemKind::deployed_table()` (the shared single /// source); `None` kinds fall through to the path-only access check. diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 483541cf2c..d10abdf43e 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -270,6 +270,7 @@ pub fn workspaced_service() -> Router { ) .route("/run/dynamic_select", post(run_dynamic_select)) .route("/list", get(list_jobs)) + .route("/asset_dispatch_edges", get(list_asset_dispatch_edges)) .route( "/list_selected_job_groups", // We use post because sending a huge array as a query param can produce @@ -403,6 +404,7 @@ pub fn workspace_unauthed_service() -> Router { get(get_completed_job_result_maybe), ) .route("/completed/get_timing/{id}", get(get_completed_job_timing)) + .route("/dispatch_events/{id}", get(get_dispatch_events)) .route("/getupdate/{id}", get(get_job_update)) .route("/getupdate_sse/{id}", get(get_job_update_sse)) .route("/get_log_file/{*file_path}", get(get_log_file)) @@ -9032,6 +9034,198 @@ struct JobTiming { duration_ms: Option, } +/// One row of the producer's "Dispatch" panel — what the asset-trigger +/// dispatcher decided for a single (subscriber, asset write) pair. See +/// `windmill_queue::asset_dispatch` for the writer and the discriminants +/// of the `outcome` / `reason` fields. +#[derive(Serialize)] +struct DispatchEvent { + subscriber_path: String, + asset_kind: windmill_common::assets::AssetKind, + asset_path: String, + outcome: String, + #[serde(skip_serializing_if = "Option::is_none")] + child_job_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + partition: Option, + #[serde(skip_serializing_if = "Option::is_none")] + received_inputs: Option, + #[serde(skip_serializing_if = "Option::is_none")] + required_inputs: Option, + #[serde(skip_serializing_if = "Option::is_none")] + debounce_s: Option, + #[serde(skip_serializing_if = "Option::is_none")] + reason: Option, + created_at: chrono::DateTime, +} + +async fn get_dispatch_events( + OptViewToken(view_token): OptViewToken, + OptAuthed(opt_authed): OptAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, id)): Path<(String, Uuid)>, +) -> error::JsonResult> { + let tags = opt_authed + .as_ref() + .map(|authed| get_scope_tags(authed)) + .flatten(); + + // Gate on the producer job's visibility, exactly like + // get_completed_job_timing on the same unauthed router: scope tags + // first, then per-job read access for authed users, anonymous-only + // jobs otherwise. The dispatch_event FK to v2_job(id) guarantees the + // producer row exists for any extant event. + let producer = sqlx::query!( + r#"SELECT created_by AS "created_by!" + FROM v2_job + WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))"#, + id, + &w_id, + tags.as_ref().map(|v| v.as_slice()) as Option<&[&str]>, + ) + .fetch_optional(&db) + .await?; + let producer = not_found_if_none(producer, "Job", id.to_string())?; + + if let Some(authed) = opt_authed.as_ref() { + require_job_read_access( + &db, + &user_db, + authed, + &w_id, + &id, + &producer.created_by, + view_token.as_deref(), + ) + .await?; + } else if producer.created_by != "anonymous" { + return Err(Error::BadRequest( + "As a non logged in user, you can only see jobs ran by anonymous users".to_string(), + )); + } + + let rows = sqlx::query!( + r#"SELECT + subscriber_path AS "subscriber_path!", + asset_kind AS "asset_kind!: windmill_common::assets::AssetKind", + asset_path AS "asset_path!", + outcome::text AS "outcome!", + child_job_id, + partition, + received_inputs, + required_inputs, + debounce_s, + reason, + created_at AS "created_at!" + FROM dispatch_event + WHERE producer_job_id = $1 AND workspace_id = $2 + ORDER BY id"#, + id, + &w_id, + ) + .fetch_all(&db) + .await?; + + Ok(Json( + rows.into_iter() + .map(|r| DispatchEvent { + subscriber_path: r.subscriber_path, + asset_kind: r.asset_kind, + asset_path: r.asset_path, + outcome: r.outcome, + child_job_id: r.child_job_id, + partition: r.partition, + received_inputs: r.received_inputs, + required_inputs: r.required_inputs, + debounce_s: r.debounce_s, + reason: r.reason, + created_at: r.created_at, + }) + .collect(), + )) +} + +/// One asset-cascade dispatch record, for reconstructing the cascade graph of a +/// pipeline folder in the Activity panel. `dispatched` rows carry the resolved +/// `child_job_id` (a real producer→child job edge); `join_pending` rows are the +/// pre-completion inputs of an AND-join (no child yet) — the client links them +/// to the eventual child of the same `subscriber_path` so a join's separate +/// trigger chains merge into one group. `skipped` is omitted. +#[derive(Serialize)] +struct AssetDispatchEdge { + producer_job_id: Uuid, + #[serde(skip_serializing_if = "Option::is_none")] + child_job_id: Option, + subscriber_path: String, + outcome: String, + asset_kind: windmill_common::assets::AssetKind, + asset_path: String, + created_at: chrono::DateTime, +} + +#[derive(Deserialize)] +struct AssetDispatchEdgesQuery { + /// Folder path prefix the children live under, e.g. `f/orders/`. Matched + /// against `subscriber_path` — every intra-pipeline cascade edge has its + /// child in the folder, so this captures the whole folder's cascades. + path_start: String, + /// Only edges dispatched at/after this instant (align with the activity + /// window the client already loaded). Omit for the default cap. + created_after: Option>, +} + +/// Asset-cascade edges for a pipeline folder. RLS on the joined `v2_job` +/// producer row limits this to cascades whose producer the caller can already +/// see (same visibility as the folder's job list). +async fn list_asset_dispatch_edges( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(query): Query, +) -> error::JsonResult> { + let like = format!("{}%", query.path_start); + let mut tx = user_db.begin(&authed).await?; + let rows = sqlx::query!( + r#"SELECT + de.producer_job_id AS "producer_job_id!", + de.child_job_id, + de.subscriber_path AS "subscriber_path!", + de.outcome::text AS "outcome!", + de.asset_kind AS "asset_kind!: windmill_common::assets::AssetKind", + de.asset_path AS "asset_path!", + de.created_at AS "created_at!" + FROM dispatch_event de + JOIN v2_job pj ON pj.id = de.producer_job_id + WHERE de.workspace_id = $1 + AND de.outcome IN ('dispatched', 'join_pending') + AND de.subscriber_path LIKE $2 + AND ($3::timestamptz IS NULL OR de.created_at >= $3) + ORDER BY de.created_at DESC, de.id DESC + LIMIT 4000"#, + &w_id, + like, + query.created_after, + ) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + + Ok(Json( + rows.into_iter() + .map(|r| AssetDispatchEdge { + producer_job_id: r.producer_job_id, + child_job_id: r.child_job_id, + subscriber_path: r.subscriber_path, + outcome: r.outcome, + asset_kind: r.asset_kind, + asset_path: r.asset_path, + created_at: r.created_at, + }) + .collect(), + )) +} + async fn get_completed_job_timing( OptViewToken(view_token): OptViewToken, OptAuthed(opt_authed): OptAuthed, diff --git a/backend/windmill-common/src/assets.rs b/backend/windmill-common/src/assets.rs index 2968493ecc..4e241b4b99 100644 --- a/backend/windmill-common/src/assets.rs +++ b/backend/windmill-common/src/assets.rs @@ -2,8 +2,24 @@ use sqlx::PgExecutor; use crate::{error, scripts::ScriptHash}; +pub use windmill_parser::asset_parser::{parse_pipeline_annotations, TriggerSpec, PARTITION_TOKEN}; pub use windmill_types::assets::*; +#[derive(sqlx::Type, Debug, Clone, Copy, PartialEq)] +#[sqlx(type_name = "SCRIPT_TRIGGER_KIND", rename_all = "lowercase")] +pub enum ScriptTriggerKind { + Asset, + Schedule, + Webhook, + Email, + Kafka, + Mqtt, + Nats, + Postgres, + Sqs, + Gcp, +} + pub async fn insert_static_asset_usage<'e>( executor: impl PgExecutor<'e>, workspace_id: &str, @@ -40,8 +56,21 @@ pub async fn clear_static_asset_usage<'e>( usage_path: &str, usage_kind: AssetUsageKind, ) -> error::Result<()> { + // Invalidate the per-workspace producer-writes cache that gates the + // asset-trigger dispatch hook (windmill-queue + // asset_dispatch::ASSET_PRODUCER_WRITES_CACHE), but only for script + // usage. The notify is unconditional (not gated on rows deleted) so a + // deploy where the script *gains* its first asset still invalidates; + // emitting it in the same statement keeps it atomic with the change and + // visible to pollers only on commit. Flow usage doesn't affect the + // script producer set, so the INSERT is skipped via the `$3 = 'script'` + // guard. sqlx::query!( - r#"DELETE FROM asset WHERE workspace_id = $1 AND usage_path = $2 AND usage_kind = $3"#, + r#"WITH del AS ( + DELETE FROM asset WHERE workspace_id = $1 AND usage_path = $2 AND usage_kind = $3 + ) + INSERT INTO notify_event (channel, payload) + SELECT 'notify_asset_producer_change', $1 WHERE $3 = 'script'"#, workspace_id, usage_path, usage_kind as AssetUsageKind @@ -56,8 +85,16 @@ pub async fn clear_static_asset_usage_by_script_hash<'e>( workspace_id: &str, script_hash: ScriptHash, ) -> error::Result<()> { + // Always script usage → unconditionally invalidate the producer-writes + // cache for this workspace (see clear_static_asset_usage), atomically + // with the delete. sqlx::query!( - "DELETE FROM asset WHERE workspace_id = $1 AND usage_kind = 'script' AND usage_path = (SELECT path FROM script WHERE hash = $2 AND workspace_id = $1)", + r#"WITH del AS ( + DELETE FROM asset WHERE workspace_id = $1 AND usage_kind = 'script' + AND usage_path = (SELECT path FROM script WHERE hash = $2 AND workspace_id = $1) + ) + INSERT INTO notify_event (channel, payload) + VALUES ('notify_asset_producer_change', $1)"#, workspace_id, script_hash.0 ) @@ -66,6 +103,159 @@ pub async fn clear_static_asset_usage_by_script_hash<'e>( Ok(()) } +// Wipe all pipeline trigger declarations held by the given runnable. Used at +// deploy time: redeploying a script wipes its prior `// on` annotations so +// removing them implicitly un-declares those edges. +pub async fn clear_script_triggers<'e>( + executor: impl PgExecutor<'e>, + workspace_id: &str, + runnable_path: &str, + runnable_kind: AssetUsageKind, +) -> error::Result<()> { + sqlx::query!( + r#"DELETE FROM script_trigger + WHERE workspace_id = $1 AND runnable_kind = $2 AND runnable_path = $3"#, + workspace_id, + runnable_kind as AssetUsageKind, + runnable_path, + ) + .execute(executor) + .await?; + Ok(()) +} + +// Insert a single trigger declaration. Caller is expected to wipe first. +// `join_all` is the script-level `// trigger all` flag (AND join barrier); +// `retry_count` / `retry_delay_s` are the `// retry []` policy. +// All three are script-level — the same value for every row of a given +// runnable — but stored per-row to keep the wipe-and-reinsert pattern and a +// single-query subscriber lookup. +pub async fn insert_script_trigger<'e>( + executor: impl PgExecutor<'e>, + workspace_id: &str, + runnable_kind: AssetUsageKind, + runnable_path: &str, + trigger_kind: ScriptTriggerKind, + trigger_ref: &str, + join_all: bool, + debounce_s: Option, + retry_count: Option, + retry_delay_s: Option, +) -> error::Result<()> { + sqlx::query!( + r#"INSERT INTO script_trigger + (workspace_id, runnable_kind, runnable_path, trigger_kind, trigger_ref, join_all, + debounce_s, retry_count, retry_delay_s) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)"#, + workspace_id, + runnable_kind as AssetUsageKind, + runnable_path, + trigger_kind as ScriptTriggerKind, + trigger_ref, + join_all, + debounce_s, + retry_count, + retry_delay_s, + ) + .execute(executor) + .await?; + Ok(()) +} + +/// Parse a debounce duration into whole seconds. Accepts a bare integer +/// (seconds) or an `` with an `s`/`m`/`h`/`d` suffix (e.g. `30s`, +/// `5m`, `2h`, `1d`). Returns `None` for empty / malformed / non-positive +/// input — the caller treats `None` as "no debounce" (fan-out), so a typo +/// fails safe rather than silently debouncing. +pub fn parse_duration_secs(s: &str) -> Option { + let s = s.trim(); + if s.is_empty() { + return None; + } + let (num, mult): (&str, i64) = match s.as_bytes().last() { + Some(b's') => (&s[..s.len() - 1], 1), + Some(b'm') => (&s[..s.len() - 1], 60), + Some(b'h') => (&s[..s.len() - 1], 3600), + Some(b'd') => (&s[..s.len() - 1], 86400), + Some(c) if c.is_ascii_digit() => (s, 1), + _ => return None, + }; + let n: i64 = num.trim().parse().ok()?; + let secs = n.checked_mul(mult)?; + if secs <= 0 || secs > i32::MAX as i64 { + return None; + } + Some(secs as i32) +} + +#[cfg(test)] +mod debounce_duration_tests { + use super::parse_duration_secs; + + #[test] + fn parses_units_and_bare_seconds() { + assert_eq!(parse_duration_secs("60"), Some(60)); + assert_eq!(parse_duration_secs("30s"), Some(30)); + assert_eq!(parse_duration_secs("5m"), Some(300)); + assert_eq!(parse_duration_secs("2h"), Some(7200)); + assert_eq!(parse_duration_secs(" 1d "), Some(86400)); + } + + #[test] + fn rejects_garbage_and_nonpositive() { + assert_eq!(parse_duration_secs(""), None); + assert_eq!(parse_duration_secs("abc"), None); + assert_eq!(parse_duration_secs("0"), None); + assert_eq!(parse_duration_secs("-5"), None); + assert_eq!(parse_duration_secs("10x"), None); + assert_eq!(parse_duration_secs("s"), None); + } +} + +// Inverse of trigger_spec_to_row for the Asset variant: parses a stored +// trigger_ref (e.g. `s3://foo`, `$res:bar`) back into the (kind, path) pair +// used as a graph node id. Returns None for refs that don't match any known +// asset prefix — callers should skip those edges. +pub fn parse_asset_trigger_ref(s: &str) -> Option<(AssetKind, String)> { + let (kind, path) = windmill_parser::asset_parser::parse_asset_syntax(s, false)?; + Some((asset_kind_from_parser(kind), path.to_string())) +} + +// Convert a parser TriggerSpec into the `(kind, ref)` pair stored in +// script_trigger. Asset refs get their canonical prefix back so the +// trigger_ref matches what downstream lookups expect. +// +// Returns `None` for native trigger kinds (Kafka, Mqtt, Postgres, …) — +// those annotations are marker-only and don't produce a `script_trigger` +// row. The actual binding lives on the trigger row's own `script_path` +// column; the graph endpoint looks it up directly per kind. +pub fn trigger_spec_to_row(spec: &TriggerSpec) -> Option<(ScriptTriggerKind, String)> { + match spec { + TriggerSpec::Asset { asset_kind, path, .. } => { + // Single source of truth for the canonical prefix lives on the + // common AssetKind; map the parser kind across first. The parser + // enum has no Variable variant, so canonical_prefix is always Some. + let prefix = asset_kind_from_parser(*asset_kind).canonical_prefix()?; + Some((ScriptTriggerKind::Asset, format!("{}{}", prefix, path))) + } + // Schedule joins the native-trigger family — no script_trigger row + // is inserted for the annotation. The binding lives on the schedule + // row's own `script_path` field, same as kafka/mqtt/etc. + TriggerSpec::Schedule + | TriggerSpec::Webhook + | TriggerSpec::Email + | TriggerSpec::Kafka + | TriggerSpec::Mqtt + | TriggerSpec::Nats + | TriggerSpec::Postgres + | TriggerSpec::Sqs + | TriggerSpec::Gcp + // data_upload is a UI-first marker — no event source, no trigger row. + // The script's S3Object input + auto-generated S3 picker drive it. + | TriggerSpec::DataUpload => None, + } +} + pub fn asset_kind_from_parser(parser_kind: windmill_parser::asset_parser::AssetKind) -> AssetKind { match parser_kind { windmill_parser::asset_parser::AssetKind::S3Object => AssetKind::S3Object, diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 1f7611a13a..c501bbed6e 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -81,6 +81,20 @@ pub mod oidc_oss; #[cfg(feature = "private")] pub mod otel_ee; pub mod otel_oss; +#[cfg(feature = "private")] +pub mod partition_ee; +pub mod partition_oss; +#[cfg(feature = "private")] +pub use partition_ee as partition; +#[cfg(not(feature = "private"))] +pub use partition_oss as partition; +#[cfg(feature = "private")] +pub mod pipeline_advanced_ee; +pub mod pipeline_advanced_oss; +#[cfg(feature = "private")] +pub use pipeline_advanced_ee as pipeline_advanced; +#[cfg(not(feature = "private"))] +pub use pipeline_advanced_oss as pipeline_advanced; pub mod query_builders; pub mod queue; pub mod result_stream; diff --git a/backend/windmill-common/src/partition_oss.rs b/backend/windmill-common/src/partition_oss.rs new file mode 100644 index 0000000000..0e9c05ac36 --- /dev/null +++ b/backend/windmill-common/src/partition_oss.rs @@ -0,0 +1,55 @@ +//! OSS fallback for pipeline partition resolution. +//! +//! Partition resolution is a `private` feature (see `partition_ee`). In the +//! public build it is absent: `resolve_partition` yields no partition, so the +//! cascade runs partition-agnostically, and the args helpers degrade to a +//! plain (non-partition-preserving) write. The real implementation lives in +//! `windmill-ee-private`. + +use std::collections::HashMap; + +use chrono::{DateTime, Utc}; +use serde_json::value::RawValue; +use sqlx::types::Json; +use sqlx::PgExecutor; +use uuid::Uuid; +use windmill_parser::asset_parser::PartitionSpec; + +use crate::error::Result; + +/// Well-known arg key the resolved partition value is injected under. +pub const PARTITION_ARG: &str = "partition"; + +/// No-op: OSS never resolves a partition, so there is nothing to persist. +pub async fn set_resolved_partition<'e>( + _executor: impl PgExecutor<'e>, + _job_id: Uuid, + _value: &str, +) -> Result<()> { + Ok(()) +} + +/// Plain args replace — OSS has no resolved partition to carry forward. +pub async fn merge_args_preserving_partition<'e>( + executor: impl PgExecutor<'e>, + job_id: Uuid, + new_args: HashMap>, +) -> Result<()> { + sqlx::query!( + "UPDATE v2_job SET args = $1, preprocessed = TRUE WHERE id = $2", + Json(new_args) as Json>>, + job_id, + ) + .execute(executor) + .await?; + Ok(()) +} + +/// No partition in the OSS build. +pub fn resolve_partition( + _spec: &PartitionSpec, + _at: DateTime, + _payload: Option<&serde_json::Value>, +) -> Result> { + Ok(None) +} diff --git a/backend/windmill-common/src/pipeline_advanced_oss.rs b/backend/windmill-common/src/pipeline_advanced_oss.rs new file mode 100644 index 0000000000..009e783bd5 --- /dev/null +++ b/backend/windmill-common/src/pipeline_advanced_oss.rs @@ -0,0 +1,16 @@ +//! OSS fallback: pipeline freshness/SLA enforcement and partition backfills +//! are enterprise features; their implementations live in windmill-ee-private +//! (see `pipeline_advanced_ee`). In the public build the entry points report +//! that the enterprise edition is required. + +use crate::error::Error; + +pub fn freshness_enforcement_todo() -> Error { + Error::internal_err( + "Pipeline freshness/SLA enforcement requires the enterprise edition".to_string(), + ) +} + +pub fn backfill_todo() -> Error { + Error::internal_err("Pipeline partition backfill requires the enterprise edition".to_string()) +} diff --git a/backend/windmill-common/src/runtime_assets.rs b/backend/windmill-common/src/runtime_assets.rs index ecd863f482..70185ca48c 100644 --- a/backend/windmill-common/src/runtime_assets.rs +++ b/backend/windmill-common/src/runtime_assets.rs @@ -127,12 +127,19 @@ async fn prune_runtime_assets( .map(|((w, p, k), v)| (w.clone(), p.clone(), k.clone(), (max_n - v.len()) as i32)) .multiunzip(); + // Delete the surplus job rows by `id`, NOT by `(workspace_id, path, + // kind)`. Deleting by tuple would also wipe the static + // `usage_kind='script'|'flow'` rows that share the same path+kind (a + // producer's persisted write/read lineage), which silently breaks the + // asset-trigger cascade (`fetch_producer_writes` finds no writes). The + // inner query already scopes to `usage_kind='job'`; keep the delete + // scoped to exactly those over-cap rows. let delete_result = sqlx::query!( r#" DELETE FROM asset - WHERE (workspace_id, path, kind) IN ( - SELECT workspace_id, path, kind FROM ( - SELECT a.workspace_id, a.path, a.kind, a.usage_kind, ROW_NUMBER() OVER ( + WHERE id IN ( + SELECT id FROM ( + SELECT a.id, ROW_NUMBER() OVER ( PARTITION BY a.workspace_id, a.path, a.kind ORDER BY a.created_at DESC ) as rn, @@ -140,14 +147,14 @@ async fn prune_runtime_assets( FROM asset a INNER JOIN ( SELECT * FROM UNNEST( - $1::varchar[], - $2::varchar[], + $1::varchar[], + $2::varchar[], $3::asset_kind[], $4::int[] ) AS t(workspace_id, path, kind, max_n) ) limits - ON a.workspace_id = limits.workspace_id - AND a.path = limits.path + ON a.workspace_id = limits.workspace_id + AND a.path = limits.path AND a.kind = limits.kind WHERE a.usage_kind = 'job' ) ranked diff --git a/backend/windmill-common/src/user_drafts.rs b/backend/windmill-common/src/user_drafts.rs index b99ede353c..620cbe2a5f 100644 --- a/backend/windmill-common/src/user_drafts.rs +++ b/backend/windmill-common/src/user_drafts.rs @@ -53,6 +53,10 @@ pub enum UserDraftItemKind { TriggerNextcloud, TriggerGoogle, TriggerGithub, + /// All unsaved scripts of one data pipeline, bundled into a single draft + /// keyed at the pipeline's folder path. Not a runnable: it has no deployed + /// backing table and is private to its owner. + DataPipeline, } impl UserDraftItemKind { @@ -84,12 +88,13 @@ impl UserDraftItemKind { UserDraftItemKind::TriggerNextcloud => "trigger_nextcloud", UserDraftItemKind::TriggerGoogle => "trigger_google", UserDraftItemKind::TriggerGithub => "trigger_github", + UserDraftItemKind::DataPipeline => "data_pipeline", } } /// Every variant, for code that must enumerate kinds (e.g. generating /// the `draft_only` existence SQL). - pub const ALL: [UserDraftItemKind; 24] = [ + pub const ALL: [UserDraftItemKind; 25] = [ UserDraftItemKind::Script, UserDraftItemKind::Flow, UserDraftItemKind::App, @@ -114,6 +119,7 @@ impl UserDraftItemKind { UserDraftItemKind::TriggerNextcloud, UserDraftItemKind::TriggerGoogle, UserDraftItemKind::TriggerGithub, + UserDraftItemKind::DataPipeline, ]; /// The deployed table backing this kind, keyed by `(workspace_id, path)`. @@ -144,6 +150,9 @@ impl UserDraftItemKind { TriggerEmail | TriggerDefaultEmail => Some("email_trigger"), TriggerWebhook | TriggerPoll | TriggerCli | TriggerNextcloud | TriggerGoogle | TriggerGithub => None, + // Keyed at a folder path, not a runnable; access falls back to the + // path-only (folder write) check. + DataPipeline => None, } } diff --git a/backend/windmill-object-store/src/lib.rs b/backend/windmill-object-store/src/lib.rs index 4b1279609f..cf30efe116 100644 --- a/backend/windmill-object-store/src/lib.rs +++ b/backend/windmill-object-store/src/lib.rs @@ -535,7 +535,134 @@ pub fn build_filesystem_client(root_path: &str) -> error::Result) -> std::fmt::Result { + self.0.fmt(f) + } +} + +#[cfg(feature = "parquet")] +#[async_trait] +impl ObjectStore for FilesystemStoreIgnoringAttributes { + async fn put_opts( + &self, + location: &object_store::path::Path, + payload: object_store::PutPayload, + mut opts: object_store::PutOptions, + ) -> object_store::Result { + opts.attributes = Default::default(); + self.0.put_opts(location, payload, opts).await + } + + async fn put_multipart_opts( + &self, + location: &object_store::path::Path, + mut opts: object_store::PutMultipartOpts, + ) -> object_store::Result> { + opts.attributes = Default::default(); + self.0.put_multipart_opts(location, opts).await + } + + async fn get_opts( + &self, + location: &object_store::path::Path, + options: object_store::GetOptions, + ) -> object_store::Result { + self.0.get_opts(location, options).await + } + + async fn get_range( + &self, + location: &object_store::path::Path, + range: std::ops::Range, + ) -> object_store::Result { + self.0.get_range(location, range).await + } + + async fn get_ranges( + &self, + location: &object_store::path::Path, + ranges: &[std::ops::Range], + ) -> object_store::Result> { + self.0.get_ranges(location, ranges).await + } + + async fn head( + &self, + location: &object_store::path::Path, + ) -> object_store::Result { + self.0.head(location).await + } + + async fn delete(&self, location: &object_store::path::Path) -> object_store::Result<()> { + self.0.delete(location).await + } + + fn list( + &self, + prefix: Option<&object_store::path::Path>, + ) -> futures::stream::BoxStream<'static, object_store::Result> { + self.0.list(prefix) + } + + fn list_with_offset( + &self, + prefix: Option<&object_store::path::Path>, + offset: &object_store::path::Path, + ) -> futures::stream::BoxStream<'static, object_store::Result> { + self.0.list_with_offset(prefix, offset) + } + + async fn list_with_delimiter( + &self, + prefix: Option<&object_store::path::Path>, + ) -> object_store::Result { + self.0.list_with_delimiter(prefix).await + } + + async fn copy( + &self, + from: &object_store::path::Path, + to: &object_store::path::Path, + ) -> object_store::Result<()> { + self.0.copy(from, to).await + } + + async fn rename( + &self, + from: &object_store::path::Path, + to: &object_store::path::Path, + ) -> object_store::Result<()> { + self.0.rename(from, to).await + } + + async fn copy_if_not_exists( + &self, + from: &object_store::path::Path, + to: &object_store::path::Path, + ) -> object_store::Result<()> { + self.0.copy_if_not_exists(from, to).await + } + + async fn rename_if_not_exists( + &self, + from: &object_store::path::Path, + to: &object_store::path::Path, + ) -> object_store::Result<()> { + self.0.rename_if_not_exists(from, to).await + } } #[cfg(feature = "parquet")] diff --git a/backend/windmill-queue/src/asset_dispatch.rs b/backend/windmill-queue/src/asset_dispatch.rs new file mode 100644 index 0000000000..7647ffc864 --- /dev/null +++ b/backend/windmill-queue/src/asset_dispatch.rs @@ -0,0 +1,736 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Runtime fan-out for asset-triggered scripts. +//! +//! When a script writes an asset and a downstream script subscribes to +//! that asset via `// on s3://...`, this module pushes a job for each +//! subscriber after the producer's job completes successfully. Any +//! asset-writing top-level script cascades — there is no `// pipeline` +//! gate on the producer side; subscriptions alone define the graph. +//! +//! Eligibility (V1, narrow on purpose): +//! - Producer kind is `Script` or `Preview`. Flows defer. +//! - Producer is top-level (no `parent_job`, no `flow_step_id`). +//! - Producer succeeded. +//! - The producer's args do not contain `_wmill_skip_asset_dispatch: true`. +//! +//! Subscribers (V1): +//! - Only `script` runnables. Flow subscribers defer. +//! - The subscriber must have at least one non-archived script row. +//! - A subscriber is skipped if its path equals the producer's path +//! (self-loop) or already appears in the cascade lineage +//! (`trigger.chain`) — cycle detection, which bounds the cascade +//! without capping legitimate depth. +//! +//! Args sent to subscribers: +//! ```json +//! { +//! "trigger": { +//! "kind": "asset", +//! "asset_kind": "s3object", +//! "asset_path": "...", +//! "producer_path": "...", +//! "producer_job_id": "...", +//! "chain": ["f/a/producer0", "f/a/producer1"] +//! } +//! } +//! ``` +//! +//! Errors are logged but never bubble up to fail the producer's job. + +use crate::{push, MiniCompletedJob, PushArgs, PushIsolationLevel}; +use serde_json::value::RawValue; +use sqlx::types::Json; +use sqlx::{Pool, Postgres}; +use std::collections::HashMap; +use std::sync::Arc; +use uuid::Uuid; +use windmill_common::assets::AssetKind; +use windmill_common::error::{self, Result}; +use windmill_common::get_latest_deployed_hash_for_path; +use windmill_common::jobs::{JobKind, JobPayload, JobTriggerKind}; +use windmill_common::partition::PARTITION_ARG; +use windmill_common::scripts::ScriptHash; +use windmill_common::triggers::TriggerMetadata; +use windmill_common::users::{get_email_from_permissioned_as, username_to_permissioned_as}; +use windmill_common::worker::to_raw_value; +use windmill_common::DB; + +/// Reserved arg key that suppresses asset-trigger dispatch for a single run. +/// Set by the test panel when the user opts out of the cascade. +pub const SKIP_ASSET_DISPATCH_ARG: &str = "_wmill_skip_asset_dispatch"; + +/// Arg key holding the cascade trigger object (carries `chain`, `partition`, +/// producer metadata) injected into every dispatched subscriber. +const TRIGGER_ARG: &str = "trigger"; + +/// Arg key (under `trigger.chain`) carrying the cascade lineage: the ordered +/// list of producer paths already run in this chain. Used to detect cycles +/// (a producer re-appearing) and stop only the cyclic edge — so deep but +/// *acyclic* pipelines are never truncated. +const CHAIN_KEY: &str = "chain"; + +/// Safety backstop on lineage length. Cycle detection already bounds an +/// acyclic cascade (a path can't repeat), so this only guards against a +/// runaway from a bug. Set far above any real pipeline depth. +const MAX_CHAIN_LEN: usize = 1000; + +/// Returned to the caller (the worker's completed-job hook) so logs can +/// reference the dispatched ids. +#[derive(Debug, Default)] +pub struct DispatchResult { + pub dispatched: Vec, +} + +/// Per-decision outcome persisted to `dispatch_event` so the producer's +/// job detail page can show what happened to each subscriber. Mirrors +/// the `DISPATCH_OUTCOME` Postgres enum exactly. +#[derive(Debug, Clone, Copy, sqlx::Type)] +#[sqlx(type_name = "DISPATCH_OUTCOME", rename_all = "snake_case")] +enum DispatchOutcome { + Dispatched, + JoinPending, + Skipped, +} + +/// Outcome-specific fields. Event constructors take the four "always-present" +/// columns positionally and bundle the rest here so each call site only +/// names what it actually carries. +#[derive(Debug, Default)] +struct EventOptions<'a> { + child_job_id: Option, + partition: Option<&'a str>, + received_inputs: Option, + required_inputs: Option, + debounce_s: Option, + reason: Option<&'a str>, +} + +/// One accumulated `dispatch_event` row. Owned (not borrowed) so the whole +/// dispatch pass can collect rows and flush them in a single batched INSERT +/// at the end, avoiding an N+1 (one INSERT per subscriber × asset write). +#[derive(Debug)] +struct EventRow { + subscriber_path: String, + asset_kind: AssetKind, + asset_path: String, + outcome: DispatchOutcome, + child_job_id: Option, + partition: Option, + received_inputs: Option, + required_inputs: Option, + debounce_s: Option, + reason: Option, +} + +impl EventRow { + fn new( + subscriber_path: &str, + asset_kind: AssetKind, + asset_path: &str, + outcome: DispatchOutcome, + opts: EventOptions<'_>, + ) -> Self { + EventRow { + subscriber_path: subscriber_path.to_string(), + asset_kind, + asset_path: asset_path.to_string(), + outcome, + child_job_id: opts.child_job_id, + partition: opts.partition.map(str::to_string), + received_inputs: opts.received_inputs, + required_inputs: opts.required_inputs, + debounce_s: opts.debounce_s, + reason: opts.reason.map(str::to_string), + } + } +} + +/// Best-effort batched insert into `dispatch_event`. Never propagates — the +/// dispatch contract is "logging failures must not retroactively fail the +/// producer's job." All rows accumulated over a dispatch pass go in one +/// INSERT (UNNEST) to avoid an N+1 across (subscriber × asset write). +async fn flush_events(db: &DB, workspace_id: &str, producer_job_id: Uuid, events: &[EventRow]) { + if events.is_empty() { + return; + } + // Column-oriented arrays for UNNEST. Each Vec is one column across all rows. + let subscriber_paths: Vec = events.iter().map(|e| e.subscriber_path.clone()).collect(); + let asset_kinds: Vec = events.iter().map(|e| e.asset_kind).collect(); + let asset_paths: Vec = events.iter().map(|e| e.asset_path.clone()).collect(); + let outcomes: Vec = events.iter().map(|e| e.outcome).collect(); + let child_job_ids: Vec> = events.iter().map(|e| e.child_job_id).collect(); + let partitions: Vec> = events.iter().map(|e| e.partition.clone()).collect(); + let received_inputs: Vec> = events.iter().map(|e| e.received_inputs).collect(); + let required_inputs: Vec> = events.iter().map(|e| e.required_inputs).collect(); + let debounce_s: Vec> = events.iter().map(|e| e.debounce_s).collect(); + let reasons: Vec> = events.iter().map(|e| e.reason.clone()).collect(); + + let res = sqlx::query!( + r#"INSERT INTO dispatch_event ( + workspace_id, producer_job_id, subscriber_path, + asset_kind, asset_path, outcome, + child_job_id, partition, + received_inputs, required_inputs, + debounce_s, reason + ) + SELECT $1, $2, sp, ak, ap, oc, cj, pt, ri, rq, db, rs + FROM unnest( + $3::text[], $4::ASSET_KIND[], $5::text[], $6::DISPATCH_OUTCOME[], + $7::uuid[], $8::text[], $9::int[], $10::int[], $11::int[], $12::text[] + ) AS t(sp, ak, ap, oc, cj, pt, ri, rq, db, rs)"#, + workspace_id, + producer_job_id, + &subscriber_paths, + asset_kinds as Vec, + &asset_paths, + outcomes as Vec, + &child_job_ids as &[Option], + &partitions as &[Option], + &received_inputs as &[Option], + &required_inputs as &[Option], + &debounce_s as &[Option], + &reasons as &[Option], + ) + .execute(db) + .await; + if let Err(e) = res { + tracing::error!( + "failed to record {} dispatch_event row(s) for producer {}: {e:#}", + events.len(), + producer_job_id + ); + } +} + +/// Top-level entry. Returns `Ok(default)` and logs on any internal failure +/// rather than propagating, because dispatch is best-effort and must not +/// retroactively fail the producer. +pub async fn dispatch_asset_triggers(db: &DB, job: &MiniCompletedJob) -> DispatchResult { + match try_dispatch(db, job).await { + Ok(r) => r, + Err(e) => { + tracing::error!("asset-trigger dispatch failed for job {}: {e:#}", job.id); + DispatchResult::default() + } + } +} + +async fn try_dispatch(db: &DB, job: &MiniCompletedJob) -> Result { + if !is_eligible_kind(job) { + return Ok(DispatchResult::default()); + } + let runnable_path = match job.runnable_path.as_deref() { + Some(p) if !p.is_empty() => p, + _ => return Ok(DispatchResult::default()), + }; + + // Producer gate (cached): this hook fires on every top-level + // script/preview completion, and the overwhelmingly common case is a + // script that writes no asset. The per-workspace producer→writes map is + // cached and invalidated by a trigger on `asset`, so a non-producer + // completion costs one in-memory lookup and zero queries. The map is + // keyed on the deploy-time `asset` table by path, so an undeployed/new + // preview (no asset rows for its path) is a non-producer and never + // cascades — same as the previous per-completion lookup. + let producers = workspace_producer_writes(db, &job.workspace_id).await?; + let Some(writes) = producers.get(runnable_path).cloned() else { + return Ok(DispatchResult::default()); + }; + + let args = fetch_args(db, &job.workspace_id, job.id).await?; + if read_skip_arg(args.as_ref()) { + return Ok(DispatchResult::default()); + } + // Parse the cascade `trigger` object once; both the lineage chain and + // the propagated partition are read from it. + let trigger_map = args + .as_ref() + .and_then(|a| a.get(TRIGGER_ARG)) + .and_then(|t| serde_json::from_str::>>(t.get()).ok()); + let chain = read_chain(trigger_map.as_ref()); + let partition = read_partition(args.as_ref(), trigger_map.as_ref()); + if chain.len() >= MAX_CHAIN_LEN { + tracing::warn!( + "asset-trigger dispatch skipped: cascade lineage length {} >= backstop {} (job {}, path {})", + chain.len(), + MAX_CHAIN_LEN, + job.id, + runnable_path + ); + return Ok(DispatchResult::default()); + } + // Lineage propagated to any subscriber pushed from this producer: the + // ancestors that already ran, plus this producer. + let mut next_chain = chain.clone(); + next_chain.push(runnable_path.to_string()); + + let mut dispatched = Vec::new(); + // Best-effort dispatch_event rows accumulated over the whole pass and + // flushed in one batched INSERT at the end (avoids an N+1 over + // subscriber × asset write). The mid-pass join-slot writes + // (record_and_check_join_slot) are a separate table and unaffected. + let mut events: Vec = Vec::new(); + for (asset_kind, asset_path) in writes { + let Some(prefix) = asset_kind.canonical_prefix() else { + continue; + }; + let trigger_ref = format!("{}{}", prefix, asset_path); + let subs = fetch_subscribers(db, &job.workspace_id, &trigger_ref).await?; + for sub in subs { + let Subscriber { path: sub_path, join_all, debounce_s, retry_count, retry_delay_s } = + sub; + if sub_path == runnable_path { + events.push(EventRow::new( + &sub_path, + asset_kind, + &asset_path, + DispatchOutcome::Skipped, + EventOptions { reason: Some("self_loop"), ..Default::default() }, + )); + continue; + } + // Cycle guard: a subscriber already in this producer's lineage + // would re-enter the chain (A→…→A), looping forever. Stop only + // this edge — sibling branches still dispatch, and acyclic chains + // of any depth are unaffected. + if chain.iter().any(|p| p == &sub_path) { + events.push(EventRow::new( + &sub_path, + asset_kind, + &asset_path, + DispatchOutcome::Skipped, + EventOptions { reason: Some("cycle_detected"), ..Default::default() }, + )); + continue; + } + if join_all { + match crate::cascade::handle_join( + db, + &job.workspace_id, + &sub_path, + &trigger_ref, + partition.as_deref(), + ) + .await + { + Ok(crate::cascade::JoinDecision::Skip(reason)) => { + events.push(EventRow::new( + &sub_path, + asset_kind, + &asset_path, + DispatchOutcome::Skipped, + EventOptions { reason: Some(reason), ..Default::default() }, + )); + continue; + } + Ok(crate::cascade::JoinDecision::Pending { received, required }) => { + events.push(EventRow::new( + &sub_path, + asset_kind, + &asset_path, + DispatchOutcome::JoinPending, + EventOptions { + partition: partition.as_deref(), + received_inputs: Some(received), + required_inputs: Some(required), + ..Default::default() + }, + )); + continue; // slot incomplete — wait for the rest + } + Ok(crate::cascade::JoinDecision::Fire) => {} // fall through to push + Err(e) => { + tracing::error!("join-slot check failed for {}: {e:#}", sub_path); + continue; + } + } + } + match push_subscriber( + db, + job, + &sub_path, + asset_kind, + &asset_path, + runnable_path, + &next_chain, + partition.as_deref(), + debounce_s, + retry_count, + retry_delay_s, + ) + .await + { + Ok(id) => { + events.push(EventRow::new( + &sub_path, + asset_kind, + &asset_path, + DispatchOutcome::Dispatched, + EventOptions { + child_job_id: Some(id), + partition: partition.as_deref(), + debounce_s, + ..Default::default() + }, + )); + dispatched.push(id); + } + Err(e) => { + tracing::error!("failed to push asset-triggered job for {}: {e:#}", sub_path) + } + } + } + } + + flush_events(db, &job.workspace_id, job.id, &events).await; + + if !dispatched.is_empty() { + tracing::info!( + "asset-trigger dispatch from job {} ({}): pushed {} downstream jobs", + job.id, + runnable_path, + dispatched.len() + ); + } + Ok(DispatchResult { dispatched }) +} + +fn is_eligible_kind(job: &MiniCompletedJob) -> bool { + if !matches!(job.kind, JobKind::Script | JobKind::Preview) { + return false; + } + if job.parent_job.is_some() || job.flow_step_id.is_some() { + return false; + } + true +} + +async fn fetch_args( + db: &Pool, + workspace_id: &str, + job_id: Uuid, +) -> Result>>> { + // Read from v2_job because args live there permanently — v2_job_completed + // is the *result* row and doesn't carry args. The producer's v2_job row + // is still present at dispatch time (deletion happens later in the + // completion pipeline, after this hook). + let row = sqlx::query!( + r#"SELECT args AS "args!: Json>>" + FROM v2_job + WHERE workspace_id = $1 AND id = $2"#, + workspace_id, + job_id, + ) + .fetch_optional(db) + .await?; + Ok(row.map(|r| r.args.0)) +} + +fn read_skip_arg(args: Option<&HashMap>>) -> bool { + args.and_then(|a| a.get(SKIP_ASSET_DISPATCH_ARG)) + .and_then(|v| serde_json::from_str::(v.get()).ok()) + .unwrap_or(false) +} + +fn read_chain(trigger_map: Option<&HashMap>>) -> Vec { + trigger_map + .and_then(|m| m.get(CHAIN_KEY)) + .and_then(|v| serde_json::from_str::>(v.get()).ok()) + .unwrap_or_default() +} + +/// The partition value the producer ran with, if any. Resolved once at the +/// top of a chain (run-start) and threaded down here so every cascaded job +/// materializes the same partition without re-resolving. Top-level +/// `partition` arg (run-start injection) takes precedence over the +/// `trigger.partition` carried from an upstream cascade hop. +fn read_partition( + args: Option<&HashMap>>, + trigger_map: Option<&HashMap>>, +) -> Option { + if let Some(v) = args.and_then(|a| a.get(PARTITION_ARG)) { + if let Ok(s) = serde_json::from_str::(v.get()) { + return Some(s); + } + } + serde_json::from_str::(trigger_map?.get(PARTITION_ARG)?.get()).ok() +} + +lazy_static::lazy_static! { + /// Per-workspace map of producer script path → the assets it writes + /// (`usage_access_type IN ('w','rw')`). Serves both the producer gate + /// (is this path a producer?) and the writes themselves, so a completion + /// that isn't a producer costs a single in-memory lookup and zero + /// queries — the dispatch hook fires on every top-level script/preview + /// completion instance-wide, the overwhelming majority of which write no + /// asset. An empty map means the workspace has no asset producers (no + /// pipelines). Invalidated per workspace by `notify_asset_producer_change` + /// (a trigger on `asset`) through the polling notify system; until the + /// next poll a freshly-deployed producer may not cascade (sub-poll lag, + /// acceptable for a data pipeline). + pub static ref ASSET_PRODUCER_WRITES_CACHE: + quick_cache::sync::Cache>>> = + quick_cache::sync::Cache::new(1000); +} + +/// Test hook: disables the producer-writes cache so every dispatch reads the +/// current DB. Integration tests use `#[sqlx::test]` isolated DBs that all +/// share one workspace id, so a process-global cache keyed by workspace would +/// clobber across DBs under concurrent test threads. Always `false` in +/// production (the cache is invalidated via the notify_event poller instead). +pub static ASSET_PRODUCER_CACHE_DISABLED: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + +/// Load (cached) the producer→writes map for a workspace. The single load +/// query replaces the per-completion producer lookup; once cached, every +/// completion in the workspace is served from memory until invalidation. +async fn workspace_producer_writes( + db: &Pool, + workspace_id: &str, +) -> Result>>> { + let use_cache = !ASSET_PRODUCER_CACHE_DISABLED.load(std::sync::atomic::Ordering::Relaxed); + if use_cache { + if let Some(map) = ASSET_PRODUCER_WRITES_CACHE.get(workspace_id) { + return Ok(map); + } + } + let rows = sqlx::query!( + r#" + SELECT + usage_path AS "usage_path!", + kind AS "kind!: AssetKind", + path AS "path!" + FROM asset + WHERE workspace_id = $1 + AND usage_kind = 'script' + AND usage_access_type IN ('w', 'rw') + "#, + workspace_id, + ) + .fetch_all(db) + .await?; + let mut map: HashMap> = HashMap::new(); + for r in rows { + map.entry(r.usage_path).or_default().push((r.kind, r.path)); + } + let map = Arc::new(map); + if use_cache { + ASSET_PRODUCER_WRITES_CACHE.insert(workspace_id.to_string(), map.clone()); + } + Ok(map) +} + +/// A subscriber row resolved from `script_trigger`. Bundles the per-edge +/// options (debounce) and the script-level policy fields (`join_all`, +/// retry) that travel together to dispatch. +struct Subscriber { + path: String, + join_all: bool, + debounce_s: Option, + retry_count: Option, + retry_delay_s: Option, +} + +async fn fetch_subscribers( + db: &Pool, + workspace_id: &str, + trigger_ref: &str, +) -> Result> { + // V1: script subscribers only. Flow subscribers (`runnable_kind = 'flow'`) + // are intentionally excluded — wiring them is straightforward but the + // payload shape and permissioning need their own pass. + // `join_all` = `// trigger all` (AND join); `debounce_s` = the opt-in + // debounce window resolved at deploy (NULL = fan-out, the default). + // `retry_count` / `retry_delay_s` = the `// retry []` policy + // (NULL = no retry). + let rows = sqlx::query!( + r#" + SELECT runnable_path AS "runnable_path!", join_all AS "join_all!", debounce_s, + retry_count, retry_delay_s + FROM script_trigger + WHERE workspace_id = $1 + AND trigger_kind = 'asset' + AND trigger_ref = $2 + AND runnable_kind = 'script' + "#, + workspace_id, + trigger_ref, + ) + .fetch_all(db) + .await?; + Ok(rows + .into_iter() + .map(|r| Subscriber { + path: r.runnable_path, + join_all: r.join_all, + debounce_s: r.debounce_s, + retry_count: r.retry_count, + retry_delay_s: r.retry_delay_s, + }) + .collect()) +} + +async fn push_subscriber( + db: &DB, + producer: &MiniCompletedJob, + subscriber_path: &str, + asset_kind: AssetKind, + asset_path: &str, + producer_path: &str, + chain: &[String], + partition: Option<&str>, + debounce_s: Option, + retry_count: Option, + retry_delay_s: Option, +) -> Result { + // Same resolution as every other trigger path (`script_path_to_payload`): + // latest deployed hash plus the script's own runnable settings + // (concurrency, debounce, timeout), resolved through the + // runnable-settings handle. The cascade must not bypass a subscriber's + // concurrency limit just because it was triggered by an asset write. + let script = get_latest_deployed_hash_for_path( + None, + db.clone(), + &producer.workspace_id, + subscriber_path, + ) + .await? + .prefetch_cached(db) + .await?; + let hash = ScriptHash(script.hash); + let tag = script.tag; + let concurrency_settings = script.runnable_settings.concurrency_settings; + + // Debounce / retry semantics are a `private` feature (see `cascade`). + // OSS degrades both: debounce falls back to the subscriber's own + // script-level settings, retry is never applied. + let debouncing_settings = crate::cascade::cascade_debouncing_settings( + subscriber_path, + partition, + debounce_s, + script.runnable_settings.debouncing_settings, + ); + + // Retry is only available via the flow runtime — wrap the script in a + // one-step flow when the cascade declares one. No retry = + // unwrapped `ScriptHash` push. + let payload = if let Some(retry) = crate::cascade::cascade_retry(retry_count, retry_delay_s) { + JobPayload::SingleStepFlow { + path: subscriber_path.to_string(), + hash: Some(hash), + flow_version: None, + args: HashMap::new(), + retry: Some(retry), + error_handler_path: None, + error_handler_args: None, + skip_handler: None, + cache_ttl: script.cache_ttl, + cache_ignore_s3_path: script.cache_ignore_s3_path, + priority: script.priority, + tag_override: tag.clone(), + trigger_path: None, + apply_preprocessor: false, + concurrency_settings, + debouncing_settings, + } + } else { + JobPayload::ScriptHash { + hash, + path: subscriber_path.to_string(), + cache_ttl: script.cache_ttl, + cache_ignore_s3_path: script.cache_ignore_s3_path, + dedicated_worker: script.dedicated_worker, + language: script.language, + priority: script.priority, + apply_preprocessor: false, + debouncing_settings, + concurrency_settings, + labels: script.labels, + } + }; + + // Run the subscriber under its deployer's identity — never the + // producer's. Subscriptions are workspace-wide, so attributing the run + // to the producer would let anyone who can deploy a `// on` script + // execute code with the permissions of whoever happens to write the + // asset (e.g. an admin's scheduled job). `on_behalf_of_email` (an + // explicit service-account opt-in at deploy) takes precedence for the + // email; otherwise the deployer's email is resolved from their + // username. + let permissioned_as = username_to_permissioned_as(&script.created_by); + let email = match script.on_behalf_of_email { + Some(obo) => obo, + None => { + get_email_from_permissioned_as(&permissioned_as, &producer.workspace_id, db).await? + } + }; + + let mut args: HashMap> = HashMap::new(); + let trigger_payload = serde_json::json!({ + "kind": "asset", + "asset_kind": serde_json::to_value(&asset_kind).expect("AssetKind serializes"), + "asset_path": asset_path, + "producer_path": producer_path, + "producer_job_id": producer.id.to_string(), + CHAIN_KEY: chain, + PARTITION_ARG: partition, + }); + args.insert(TRIGGER_ARG.to_string(), to_raw_value(&trigger_payload)); + // Carry the producer's resolved partition forward as a top-level arg so + // the subscriber's body can read it and the next cascade hop's + // `read_partition` picks it up — keeps the whole chain on one partition, + // resolved once at the top. Omitted entirely for non-partitioned chains. + if let Some(p) = partition { + args.insert(PARTITION_ARG.to_string(), to_raw_value(&p)); + } + + // Attribute the dispatched run to a synthetic user so audit logs reflect + // it came from the asset cascade, not the original human runner. + let pseudo_user = format!("asset-{producer_path}"); + + let tx = PushIsolationLevel::IsolatedRoot(db.clone()); + let (id, tx) = push( + db, + tx, + &producer.workspace_id, + payload, + PushArgs { args: &args, extra: None }, + &pseudo_user, + &email, + permissioned_as, + Some(producer_path), + None, + Some(producer_path.to_string()), + None, + None, + None, + None, + false, + false, + None, + true, + tag, + script.timeout, + None, + None, + None, + false, + None, + Some(TriggerMetadata::new( + Some(producer_path.to_string()), + JobTriggerKind::Asset, + )), + None, + ) + .await + .map_err(|e| error::Error::internal_err(format!("push asset-triggered job: {e:#}")))?; + tx.commit().await?; + Ok(id) +} diff --git a/backend/windmill-queue/src/cascade_oss.rs b/backend/windmill-queue/src/cascade_oss.rs new file mode 100644 index 0000000000..be6f93d732 --- /dev/null +++ b/backend/windmill-queue/src/cascade_oss.rs @@ -0,0 +1,60 @@ +//! OSS fallback for the asset-trigger cascade's AND-join / debounce / retry. +//! +//! The richer cascade semantics are a `private` feature (see `cascade_ee`). +//! In the public build they are absent and the cascade degrades to a plain +//! OR fan-out: every join always fires immediately, no slots are recorded or +//! reaped, debounce falls back to the subscriber's own script-level settings, +//! and retry is never applied (the subscriber pushes as a bare `ScriptHash`). +//! The real implementation lives in `windmill-ee-private`. + +use windmill_common::error::Result; +use windmill_common::flows::Retry; +use windmill_common::runnable_settings::DebouncingSettings; +use windmill_common::DB; + +/// Outcome of evaluating an AND-join barrier for one (subscriber, input). +/// Kept identical to the EE definition so the core matcher in +/// `asset_dispatch` compiles against either build. +pub enum JoinDecision { + /// Input does not advance the join (recorded as Skipped with `reason`). + Skip(&'static str), + /// Join advanced but is not yet complete (recorded as JoinPending). + Pending { received: i32, required: i32 }, + /// All required inputs are present — push the subscriber. + Fire, +} + +/// OSS has no AND-join: every input fires immediately (plain OR fan-out). +pub async fn handle_join( + _db: &DB, + _workspace_id: &str, + _sub_path: &str, + _trigger_ref: &str, + _partition: Option<&str>, +) -> Result { + Ok(JoinDecision::Fire) +} + +/// Unused in OSS (no slots are ever recorded), kept for API parity. +pub const JOIN_SLOT_TTL_SECS: i64 = 60 * 24 * 60 * 60; // 60 days + +/// No-op: OSS never records join slots, so there is nothing to reap. +pub async fn reap_stale_join_slots(_db: &DB) -> Result<()> { + Ok(()) +} + +/// No per-edge debounce in OSS — always defer to the subscriber's own +/// script-level debounce settings. +pub fn cascade_debouncing_settings( + _subscriber_path: &str, + _partition: Option<&str>, + _debounce_s: Option, + fallback: DebouncingSettings, +) -> DebouncingSettings { + fallback +} + +/// No per-edge retry in OSS — the subscriber pushes as a bare `ScriptHash`. +pub fn cascade_retry(_retry_count: Option, _retry_delay_s: Option) -> Option { + None +} diff --git a/backend/windmill-queue/src/lib.rs b/backend/windmill-queue/src/lib.rs index be0d1590b6..6f689c7026 100644 --- a/backend/windmill-queue/src/lib.rs +++ b/backend/windmill-queue/src/lib.rs @@ -6,6 +6,14 @@ * LICENSE-AGPL for a copy of the license. */ +pub mod asset_dispatch; +#[cfg(feature = "private")] +pub mod cascade_ee; +pub mod cascade_oss; +#[cfg(feature = "private")] +pub use cascade_ee as cascade; +#[cfg(not(feature = "private"))] +pub use cascade_oss as cascade; pub mod jobs; #[cfg(feature = "private")] pub mod jobs_ee; diff --git a/backend/windmill-types/src/assets.rs b/backend/windmill-types/src/assets.rs index be20cfae3e..fb624382c4 100644 --- a/backend/windmill-types/src/assets.rs +++ b/backend/windmill-types/src/assets.rs @@ -16,6 +16,23 @@ pub enum AssetKind { Volume, } +impl AssetKind { + /// The canonical URI prefix used in asset trigger refs (e.g. `s3://`, + /// `$res:`). Single source of truth for both trigger-ref construction + /// and runtime cascade dispatch. `Variable` is deprecated and has no + /// canonical ref, so it returns `None`. + pub fn canonical_prefix(&self) -> Option<&'static str> { + match self { + AssetKind::S3Object => Some("s3://"), + AssetKind::Resource => Some("$res:"), + AssetKind::Ducklake => Some("ducklake://"), + AssetKind::DataTable => Some("datatable://"), + AssetKind::Volume => Some("volume://"), + AssetKind::Variable => None, + } + } +} + #[derive( Serialize, Deserialize, Debug, PartialEq, Copy, Clone, Hash, Eq, sqlx::Type, PartialOrd, Ord, )] diff --git a/backend/windmill-types/src/jobs.rs b/backend/windmill-types/src/jobs.rs index 75cde586e9..ac15430759 100644 --- a/backend/windmill-types/src/jobs.rs +++ b/backend/windmill-types/src/jobs.rs @@ -43,6 +43,9 @@ pub enum JobTriggerKind { #[serde(rename = "ci_test")] #[sqlx(rename = "ci_test")] CiTest, + // A run dispatched because an upstream pipeline script wrote an asset + // this runnable subscribes to via `// on s3://...` annotations. + Asset, } impl std::fmt::Display for JobTriggerKind { @@ -64,6 +67,7 @@ impl std::fmt::Display for JobTriggerKind { JobTriggerKind::Google => "google", JobTriggerKind::Github => "github", JobTriggerKind::CiTest => "ci_test", + JobTriggerKind::Asset => "asset", }; write!(f, "{}", kind) } diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 1824a9e493..ab2cad4da9 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -35,9 +35,9 @@ use windmill_common::{ use windmill_common::bench::{BenchmarkInfo, BenchmarkIter}; use windmill_queue::{ - append_logs, get_mini_completed_job, is_pre_shaped_wm_failure_result, CanceledBy, FlowRunners, - JobCompleted, MiniCompletedJob, MiniPulledJob, ValidableJson, WrappedError, INIT_SCRIPT_TAG, - MANUAL_FAILURE_ERROR_NAME, + append_logs, asset_dispatch, get_mini_completed_job, is_pre_shaped_wm_failure_result, + CanceledBy, FlowRunners, JobCompleted, MiniCompletedJob, MiniPulledJob, ValidableJson, + WrappedError, INIT_SCRIPT_TAG, MANUAL_FAILURE_ERROR_NAME, }; use serde_json::{json, value::RawValue, Value}; @@ -779,13 +779,16 @@ pub async fn process_completed_job( )) })?; } else if let Some(preprocessed_args) = preprocessed_args { - // Update script args to preprocessed args - sqlx::query!( - "UPDATE v2_job SET args = $1, preprocessed = TRUE WHERE id = $2", - Json(preprocessed_args) as Json>>, - job.id + // Update script args to preprocessed args, but preserve a + // resolved pipeline `partition` (injected before the body ran + // by resolve_partition_for_job). Run identity is immutable — + // the preprocessor must not change or drop it, or the asset + // cascade would read no partition for this producer. + windmill_common::partition::merge_args_preserving_partition( + db, + job.id, + preprocessed_args, ) - .execute(db) .await?; } @@ -805,6 +808,12 @@ pub async fn process_completed_job( from_cache.unwrap_or(false), ) .await?; + + // Asset-trigger fan-out: best-effort, never propagates errors. + // Internal eligibility checks gate to top-level Script/Preview runs; + // see windmill_queue::asset_dispatch. + asset_dispatch::dispatch_asset_triggers(db, &job).await; + drop(job); add_time!(bench, "add_completed_job END"); diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 8964beb6be..984acbe756 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -4301,6 +4301,109 @@ async fn try_validate_schema( Ok(()) } +/// Pipeline partition resolution at execution time. The script content is +/// already loaded for this job, so parsing the `// partitioned` annotation +/// here is free (no extra fetch / no DB column). The concrete partition +/// value is resolved exactly once — schedule fire-time for time kinds +/// (anchored on `scheduled_for`, NOT wall-clock, so a chain crossing +/// midnight stays coherent) or the triggering payload for `dynamic`. It is +/// then (a) injected into the in-memory args the body sees and (b) +/// persisted back to `v2_job.args` so the asset-dispatch cascade reads the +/// same value at completion and propagates it downstream (run identity is +/// immutable — never re-resolve once set). +/// +/// `Ok(Some(job))` = a value was injected (caller must use the returned +/// clone). `Ok(None)` = nothing to do (no `// partitioned`, or the +/// partition is already set: explicit / backfill / cascade-propagated, or +/// before the `start` anchor). `Err` fails the job with a clear message +/// (partitioned but unresolvable — e.g. `dynamic` with no payload). +async fn resolve_partition_for_job( + job: &MiniPulledJob, + code: &str, + conn: &Connection, +) -> error::Result> { + use windmill_common::partition::{resolve_partition, PARTITION_ARG}; + use windmill_parser::asset_parser::PartitionKind; + + // Only deployed scripts participate in asset pipelines. Cheap + // substring guard so the overwhelming majority of script jobs (no + // `// partitioned` line) skip the full annotation scan on the hot + // path; a false positive only costs one extra parse, never wrong. + if !matches!(job.kind, JobKind::Script) || !code.contains("partitioned") { + return Ok(None); + } + let Some(spec) = windmill_parser::asset_parser::parse_pipeline_annotations(code).partition + else { + return Ok(None); + }; + + // Already resolved upstream — explicit run arg, backfill, or + // cascade-propagated (push_subscriber injects a top-level `partition`). + // Run identity is immutable: use it as-is, do not re-resolve. + let already_set = job.args.as_ref().is_some_and(|a| { + a.0.get(PARTITION_ARG) + .and_then(|v| serde_json::from_str::(v.get()).ok()) + .is_some_and(|s| !s.is_empty()) + }); + if already_set { + return Ok(None); + } + + // `dynamic` extracts from the triggering payload (the `trigger` object + // for a cascade/event hop, else the run args themselves). Time kinds + // ignore the payload. + let payload: Option = match &spec.kind { + PartitionKind::Dynamic { .. } => job.args.as_ref().map(|a| { + a.0.get("trigger") + .and_then(|t| serde_json::from_str::(t.get()).ok()) + .unwrap_or_else(|| { + serde_json::Value::Object( + a.0.iter() + .filter_map(|(k, v)| { + serde_json::from_str(v.get()).ok().map(|jv| (k.clone(), jv)) + }) + .collect(), + ) + }) + }), + _ => None, + }; + + let resolved = resolve_partition(&spec, job.scheduled_for, payload.as_ref()) + .map_err(|e| Error::ExecutionErr(format!("partition resolution failed: {e:#}")))?; + let Some(value) = resolved else { + // Before the `start` anchor: this run has no partition to + // materialize. v1 runs it without one (logged) rather than + // introducing a skip-the-queued-job mechanism. + tracing::warn!( + job_id = %job.id, + "partitioned script resolved to no partition (before start anchor); running without one" + ); + return Ok(None); + }; + + // Persist back so dispatch_asset_triggers (which reads the producer's + // completed v2_job.args) propagates the same value down the cascade. + if let Some(db) = conn.as_sql() { + windmill_common::partition::set_resolved_partition(db, job.id, &value).await?; + } else { + tracing::warn!( + job_id = %job.id, + "agent worker: resolved partition not persisted; downstream cascade will not propagate it" + ); + } + + // Inject into the in-memory args so the running body sees it. + let mut updated = job.clone(); + let mut map = updated.args.take().map(|j| j.0).unwrap_or_default(); + map.insert( + PARTITION_ARG.to_string(), + windmill_common::worker::to_raw_value(&value), + ); + updated.args = Some(Json(map)); + Ok(Some(updated)) +} + #[tracing::instrument(level = "trace", skip_all)] async fn handle_code_execution_job( job: &MiniPulledJob, @@ -4456,6 +4559,18 @@ async fn handle_code_execution_job( ), }; + // Pipeline partition resolution: the content is now loaded, so resolve + // `// partitioned` (if any) and shadow `job` with a clone whose args + // carry the resolved `partition` for the rest of execution. + let _job_with_partition; + let job = match resolve_partition_for_job(job, code, conn).await? { + Some(j) => { + _job_with_partition = j; + &_job_with_partition + } + None => job, + }; + // For preview jobs, extract modules from args._MODULES if not already set let modules = modules_from_data.clone().or_else(|| { job.args.as_ref().and_then(|args| { diff --git a/cli/src/commands/datatable/datatable.ts b/cli/src/commands/datatable/datatable.ts index c86c971a90..1291d4cc13 100644 --- a/cli/src/commands/datatable/datatable.ts +++ b/cli/src/commands/datatable/datatable.ts @@ -41,6 +41,61 @@ async function run( await runCatalogQuery(opts, "datatable", name, sql); } +async function create( + opts: GlobalOptions & { resource?: string; force?: boolean }, + name?: string, +) { + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + const dtName = name ?? DEFAULT_DATATABLE_NAME; + + const existing = await wmill.listDataTables({ + workspace: workspace.workspaceId, + }); + if (existing.some((d) => d.name === dtName)) { + throw new Error(`Datatable '${dtName}' already exists in this workspace`); + } + // edit_datatable_config replaces the whole settings object, and fork + // metadata on existing datatables can't be read back through the API — + // so only touch a non-empty config when explicitly asked to. + if (existing.length > 0 && !opts.force) { + throw new Error( + `Workspace already has datatable(s): ${existing + .map((d) => d.name) + .join(", ")}. Re-run with --force to add '${dtName}' ` + + "(note: fork metadata on existing datatables is not preserved)", + ); + } + + const datatables: Record< + string, + { database: { resource_type: "postgresql" | "instance"; resource_path?: string } } + > = {}; + for (const d of existing) { + datatables[d.name] = { + database: { + resource_type: d.resource_type as "postgresql" | "instance", + resource_path: d.resource_path ?? undefined, + }, + }; + } + datatables[dtName] = opts.resource + ? { database: { resource_type: "postgresql", resource_path: opts.resource } } + : { database: { resource_type: "instance", resource_path: "datatable_db" } }; + + await wmill.editDataTableConfig({ + workspace: workspace.workspaceId, + requestBody: { settings: { datatables } }, + }); + log.info( + `Datatable '${dtName}' created (${ + opts.resource + ? `postgresql resource ${opts.resource}` + : "instance-backed" + }). Scripts can now use datatable://${dtName}.`, + ); +} + async function serve( opts: GlobalOptions & { port?: number; host?: string; password?: string }, ) { @@ -69,6 +124,20 @@ const command = new Command() "Output only the final result as JSON. Useful for scripting.", ) .action(run as any) + .command( + "create", + "register a datatable database in the workspace (default: instance-backed 'main') so scripts can use datatable://", + ) + .arguments("[name:string]") + .option( + "--resource ", + "Back the datatable with an existing postgresql resource path instead of the instance database", + ) + .option( + "--force", + "Allow adding to a workspace that already has datatables (fork metadata on existing ones is not preserved)", + ) + .action(create as any) .command( "serve", "Serve all datatables as a Postgres-wire endpoint (psql, DBeaver, pgAdmin); the client picks the datatable via the database name in its connection string", diff --git a/cli/src/commands/pipeline/pipeline.ts b/cli/src/commands/pipeline/pipeline.ts new file mode 100644 index 0000000000..b4e08f6416 --- /dev/null +++ b/cli/src/commands/pipeline/pipeline.ts @@ -0,0 +1,287 @@ +import { Command } from "@cliffy/command"; +import { Table } from "@cliffy/table"; +import { colors } from "@cliffy/ansi/colors"; + +import { OpenAPI } from "../../../gen/index.ts"; +import * as wmill from "../../../gen/services.gen.ts"; +import { requireLogin } from "../../core/auth.ts"; +import { resolveWorkspace } from "../../core/context.ts"; +import * as log from "../../core/log.ts"; +import { GlobalOptions } from "../../types.ts"; + +// Mirrors the asset-graph endpoint payload (backend/windmill-api-assets). +// TODO: the checked-in generated client (cli/gen, last regenerated 2025-04) +// predates these routes, so we raw-fetch and hand-roll the types. Once +// `cli/gen` is regenerated (run `cli/gen_wm_client.sh`, which is currently +// >700 openapi.yaml commits stale and would churn the whole client), replace +// `apiGet` + these types with the generated `wmill.getAssetsGraph(...)` +// (operationId getAssetsGraph) and `wmill.listPipelineFolders(...)` +// (operationId listPipelineFolders). +type GraphRunnable = { + path: string; + usage_kind: "script" | "flow" | "job"; + in_pipeline?: boolean; +}; +type GraphEdge = { + runnable_kind: string; + runnable_path: string; + asset_kind: string; + asset_path: string; + access_type?: "r" | "w" | "rw"; +}; +type GraphTrigger = + | { + trigger_kind: "asset"; + asset_kind: string; + asset_path: string; + runnable_kind: string; + runnable_path: string; + } + | { + trigger_kind: string; + path?: string; + runnable_kind: string; + runnable_path: string; + missing?: boolean; + }; +type AssetGraph = { + runnables: GraphRunnable[]; + assets: { kind: string; path: string }[]; + edges: GraphEdge[]; + triggers: GraphTrigger[]; +}; + +async function apiGet(path: string): Promise { + const response = await fetch(`${OpenAPI.BASE}${path}`, { + headers: { Authorization: `Bearer ${OpenAPI.TOKEN}` }, + }); + if (!response.ok) { + const body = await response.text(); + throw new Error(`GET ${path} -> ${response.status}: ${body}`); + } + return (await response.json()) as T; +} + +async function list(opts: GlobalOptions & { json?: boolean }) { + if (opts.json) log.setSilent(true); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + const items = await apiGet<{ folder: string; script_count: number }[]>( + `/w/${workspace.workspaceId}/assets/pipelines`, + ); + if (opts.json) { + console.log(JSON.stringify(items)); + } else if (items.length === 0) { + log.info( + "No pipelines in this workspace. Mark scripts with a `// pipeline` comment (plus `// on ` triggers) and push them into a folder.", + ); + } else { + new Table() + .header(["Folder", "Scripts"]) + .padding(2) + .border(true) + .body(items.map((p) => [`f/${p.folder}`, String(p.script_count)])) + .render(); + } +} + +const ASSET_KINDS = "s3object,ducklake,datatable,volume"; + +function assetUri(kind: string, path: string): string { + const prefix = kind === "s3object" ? "s3" : kind; + return `${prefix}://${path}`; +} + +function shortName(scriptPath: string): string { + return scriptPath.split("/").pop() ?? scriptPath; +} + +// Append to a multimap value, creating the bucket on first use. Avoids the +// O(n^2) spread-rebuild pattern (`map.set(k, [...(map.get(k) ?? []), v])`). +function pushTo(map: Map, key: K, val: V): void { + (map.get(key) ?? map.set(key, []).get(key)!).push(val); +} + +async function show( + opts: GlobalOptions & { json?: boolean }, + folder: string, +) { + if (opts.json) log.setSilent(true); + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + + const f = folder.replace(/^f\//, "").replace(/\/$/, ""); + const graph = await apiGet( + `/w/${workspace.workspaceId}/assets/graph?folder=${encodeURIComponent(f)}&asset_kinds=${ASSET_KINDS}`, + ); + if (opts.json) { + console.log(JSON.stringify(graph)); + return; + } + if (graph.runnables.length === 0) { + log.info( + `No pipeline scripts in f/${f}. Mark scripts with a \`// pipeline\` comment and push them.`, + ); + return; + } + + // Index the graph: writes per script, subscribers per asset, native + // trigger markers per script, asset subscriptions per script. + const writesByScript = new Map(); + for (const e of graph.edges) { + if (e.access_type === "w" || e.access_type === "rw") { + const uri = assetUri(e.asset_kind, e.asset_path); + pushTo(writesByScript, e.runnable_path, uri); + } + } + const subsByAsset = new Map(); + const subsByScript = new Map(); + const nativeByScript = new Map< + string, + { kind: string; path?: string; missing?: boolean }[] + >(); + for (const t of graph.triggers) { + if (t.trigger_kind === "asset") { + const at = t as Extract; + const uri = assetUri(at.asset_kind, at.asset_path); + pushTo(subsByAsset, uri, t.runnable_path); + pushTo(subsByScript, t.runnable_path, uri); + } else { + const nt = t as Exclude; + pushTo(nativeByScript, t.runnable_path, { + kind: nt.trigger_kind, + path: nt.path, + missing: nt.missing, + }); + } + } + + function triggerBadges(script: string): string { + const out: string[] = []; + for (const t of nativeByScript.get(script) ?? []) { + if (t.kind === "data_upload") { + out.push(colors.magenta("[data upload]")); + } else if (t.missing) { + out.push(colors.red(`[${t.kind} ✗ missing]`)); + } else { + out.push(colors.yellow(`[${t.kind}${t.path ? ` ${t.path}` : ""}]`)); + } + } + return out.length > 0 ? " " + out.join(" ") : ""; + } + + const printed = new Set(); + const lines: string[] = []; + + function printScript(script: string, prefix: string, extraOn?: string[]) { + const alsoOn = + extraOn && extraOn.length > 0 + ? colors.dim(` (also on: ${extraOn.join(", ")})`) + : ""; + if (printed.has(script)) { + lines.push( + `${prefix}${colors.bold(shortName(script))}${colors.dim(" ↻ shown above")}`, + ); + return; + } + printed.add(script); + lines.push(`${prefix}${colors.bold(shortName(script))}${triggerBadges(script)}${alsoOn}`); + const childPrefix = prefix.replace(/├─ $/, "│ ").replace(/└─ $/, " "); + const writes = [...(writesByScript.get(script) ?? [])].sort(); + writes.forEach((uri, i) => { + const lastAsset = i === writes.length - 1; + const assetBranch = lastAsset ? "└─▶ " : "├─▶ "; + lines.push(`${childPrefix}${assetBranch}${colors.cyan(uri)}`); + const assetChildPrefix = childPrefix + (lastAsset ? " " : "│ "); + const subs = [...(subsByAsset.get(uri) ?? [])].sort(); + subs.forEach((sub, j) => { + const branch = j === subs.length - 1 ? "└─ " : "├─ "; + const otherOn = (subsByScript.get(sub) ?? []).filter((u) => u !== uri); + printScript(sub, assetChildPrefix + branch, otherOn); + }); + }); + } + + // Roots: pipeline scripts that aren't subscribed to any asset — sources + // (data upload, schedule, webhook) and manual entries. + const roots = graph.runnables + .map((r) => r.path) + .filter((p) => !(subsByScript.get(p)?.length)) + .sort(); + + // UI-first markers (data_upload, webhook) have no trigger row — the + // graph endpoint's trigger enum (schedule/email/kafka/mqtt/nats/postgres/ + // sqs/gcp) can't surface them, so they only exist as `// on ` + // annotations in the script body. Roots are where sources matter, so fetch + // just those bodies and lift the marker kinds the canvas would show. + // + // DRIFT RISK: this regex + MARKER_KINDS is a divergent, partial copy of the + // canonical annotation parser. The proper fix is to have the graph endpoint + // emit these UI-only markers as trigger rows (a backend change), after which + // this whole Promise.all body-fetch can be deleted and read straight from + // the response. Until then, keep this list in sync with the canonical parser. + const MARKER_KINDS = ["data_upload", "webhook", "email"]; + await Promise.all( + roots.map(async (p) => { + const r = graph.runnables.find((x) => x.path === p); + if (r?.usage_kind !== "script") return; + try { + const script = await wmill.getScriptByPath({ + workspace: workspace.workspaceId, + path: p, + }); + const existing = nativeByScript.get(p) ?? []; + for (const line of (script.content ?? "").split("\n")) { + const m = line.match(/^\s*(?:\/\/|--|#)\s*on\s+(\w+)\s*$/); + if (!m) continue; + const kind = m[1]; + if (!MARKER_KINDS.includes(kind)) continue; + if (!existing.some((t) => t.kind === kind)) { + existing.push({ kind }); + } + } + if (existing.length > 0) nativeByScript.set(p, existing); + } catch { + // body fetch is best-effort enrichment only + } + }), + ); + + const scriptCount = graph.runnables.length; + const assetCount = graph.assets.length; + log.info( + colors.bold(`Pipeline f/${f}`) + + colors.dim(` — ${scriptCount} script${scriptCount === 1 ? "" : "s"} · ${assetCount} asset${assetCount === 1 ? "" : "s"}`), + ); + lines.push(""); + for (const root of roots) { + printScript(root, ""); + lines.push(""); + } + // Anything unreachable from the roots (e.g. cycles) still gets listed. + for (const r of graph.runnables) { + if (!printed.has(r.path)) { + printScript(r.path, ""); + lines.push(""); + } + } + console.log(lines.join("\n")); +} + +const command = new Command() + .description( + "inspect asset-driven pipelines (scripts marked `// pipeline`, wired by `// on ` annotations)", + ) + .command("list", "list pipeline folders in the workspace") + .option("--json", "Output as JSON (for piping to jq)") + .action(list as any) + .command( + "show", + "render a pipeline folder's DAG (sources, lineage, subscriptions) in the terminal", + ) + .arguments("") + .option("--json", "Output the raw asset graph as JSON") + .action(show as any); + +export default command; diff --git a/cli/src/guidance/skills.gen.ts b/cli/src/guidance/skills.gen.ts index b227a07859..36f90698e7 100644 --- a/cli/src/guidance/skills.gen.ts +++ b/cli/src/guidance/skills.gen.ts @@ -6324,6 +6324,9 @@ datatable related commands - \`datatable run \` - run a SQL query on a datatable - \`-n --name \` - Datatable name (default: main) - \`-s --silent\` - Output only the final result as JSON. Useful for scripting. +- \`datatable create [name:string]\` - register a datatable database in the workspace (default: instance-backed 'main') so scripts can use datatable:// + - \`--resource \` - Back the datatable with an existing postgresql resource path instead of the instance database + - \`--force\` - Allow adding to a workspace that already has datatables (fork metadata on existing ones is not preserved) - \`datatable serve\` - Serve all datatables as a Postgres-wire endpoint (psql, DBeaver, pgAdmin); the client picks the datatable via the database name in its connection string - \`--port \` - Port to listen on (default: first free port in 5433-5500) - \`--host \` - Bind address (default: 127.0.0.1) @@ -6653,6 +6656,17 @@ Object storage (S3) related commands. Operates on the workspace's default object - \`--csv-separator \` - CSV column separator (default ,) - \`--csv-header\` - Treat the first CSV row as a header +### pipeline + +inspect asset-driven pipelines (scripts marked \`// pipeline\`, wired by \`// on \` annotations) + +**Subcommands:** + +- \`pipeline list\` - list pipeline folders in the workspace + - \`--json\` - Output as JSON (for piping to jq) +- \`pipeline show \` - render a pipeline folder's DAG (sources, lineage, subscriptions) in the terminal + - \`--json\` - Output the raw asset graph as JSON + ### protection-rules Sync workspace protection rules between protection-rules.yaml and Windmill. The file is keyed by workspace name; keys must match wmill.yaml 'workspaces'. diff --git a/cli/src/main.ts b/cli/src/main.ts index efa7df963b..ff8c576ee9 100755 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -51,6 +51,7 @@ import generateMetadata from "./commands/generate-metadata/generate-metadata.ts" import docs from "./commands/docs/docs.ts"; import config from "./commands/config/config.ts"; import datatable from "./commands/datatable/datatable.ts"; +import pipeline from "./commands/pipeline/pipeline.ts"; import ducklake from "./commands/ducklake/ducklake.ts"; import objectStorage from "./commands/object-storage/object-storage.ts"; import { fetchVersion } from "./core/context.ts"; @@ -77,6 +78,7 @@ export { docs, config, datatable, + pipeline, ducklake, objectStorage, hubPull, @@ -215,6 +217,7 @@ const command = new Command() .command("docs", docs) .command("config", config) .command("datatable", datatable) + .command("pipeline", pipeline) .command("ducklake", ducklake) .command("object-storage", objectStorage) .command("version --version", "Show version information") diff --git a/cli/src/utils/script_common.ts b/cli/src/utils/script_common.ts index f314fe0e5d..a713e41782 100644 --- a/cli/src/utils/script_common.ts +++ b/cli/src/utils/script_common.ts @@ -110,8 +110,16 @@ export function inferContentTypeFromFilePath( return "rlang"; // for related places search: ADD_NEW_LANG } else { + const ext = contentPath.substring(contentPath.lastIndexOf(".")); + let hint = ""; + if (ext === ".sql") { + hint = + "\nBare .sql is ambiguous — use a dialect extension: .pg.sql (postgresql), .my.sql (mysql), .bq.sql (bigquery), .sf.sql (snowflake), .ms.sql (mssql), .odb.sql (oracledb), .duckdb.sql (duckdb)"; + } throw new Error( - "Invalid language: " + contentPath.substring(contentPath.lastIndexOf(".")) + `Cannot infer script language from extension '${ext}' (file ${contentPath}).` + + hint + + "\nSupported extensions: .ts (bun/deno), .py, .go, .sh, .ps1, .php, .rs, .cs, .nu, .java, .rb, .r, .gql, .playbook.yml, .pg.sql, .my.sql, .bq.sql, .sf.sql, .ms.sql, .odb.sql, .duckdb.sql" ); } } diff --git a/docs/pipelines-vs-dbt.md b/docs/pipelines-vs-dbt.md new file mode 100644 index 0000000000..9ec6dd11d3 --- /dev/null +++ b/docs/pipelines-vs-dbt.md @@ -0,0 +1,374 @@ +# Pipelines vs. dbt + +Positioning analysis and architectural notes for the data-pipeline abstraction +currently landing on `feat/asset-graph-view`. Covers what we're building, how +it differs from dbt, which dbt features are real gaps vs. TODO, and a focused +deep-dive on incremental materialization — including a recommendation to +collapse it into partitioning rather than ship it as a separate concept. + +## What we're building + +Asset-centric, polyglot, annotation-driven, event-aware: + +- Assets (`datatable`, `ducklake`, `s3object`, `volume`) are graph nodes; + scripts are edges that produce/consume them. See + `backend/parsers/windmill-parser/src/asset_parser.rs:25`. +- Lineage comes from two sources: parsed annotations (`// pipeline`, + `// on datatable://...`, `// partitioned daily`, `// freshness 1h`, + `// trigger any`, `// debounce`, `// tag`, `// retry`) and body-inferred + reads/writes via the asset parser. +- Triggers are first-class: schedule, webhook, email, kafka, mqtt, nats, + postgres, sqs, gcp — all wired into the same DAG view + (`frontend/src/lib/components/assets/AssetGraph/types.ts:56`). +- Per-language scaffolds (DuckDB ATTACH, Postgres, Python, TS, Bash) generate + starter code per `PipelineOutputKind` (`datatable | ducklake | s3_parquet | + s3_object | none`). See + `frontend/src/lib/components/assets/AssetGraph/pipelineTemplates.ts:11`. + +## Differentiators vs. dbt + +1. **Event-driven + batch in one DAG.** dbt is batch-on-warehouse. Kafka → + Python normalize → DuckDB aggregate → Postgres view → Slack notify is + native here; in dbt land it's "use Airflow/Prefect for the non-SQL parts." +2. **Polyglot, not SQL+Jinja.** Python/TS/Bash/Duck/PG transformations live + in the same graph. No Jinja templating language; annotations are real + comments parsed strictly. +3. **Multi-substrate by design.** `datatable` (Postgres), `ducklake` + (lakehouse), `s3_parquet`, `s3_object` are peers. dbt's universe is + "tables in your warehouse." +4. **One platform.** Same runtime as workflows, internal apps, background + jobs, RBAC, secrets, schedules. dbt is single-purpose. +5. **Inferred lineage from code.** Body parser picks up `CREATE TABLE` / S3 + writes — annotations are not strictly required to get edges. dbt requires + explicit `{{ ref() }}` everywhere. + +## Where dbt wins today + +| Gap | Architectural blocker? | Verdict | +|---|---|---| +| Data tests | No | Pure TODO | +| Incremental materializations | No, but pick a philosophy | TODO with design decision | +| Column lineage + docs site | No | Pure TODO | +| Snapshots / SCD2 | No | New output kind | +| Selective execution grammar | No | UI/CLI surface | +| Schema contracts | No, but design metadata model | TODO with design work | +| Packages / community | Closed annotation parser starts to bind | Decide extensibility model | +| Semantic layer / metrics | No | Large additive scope | + +The three items where the current abstraction needs deliberate decisions +before more weight lands on it: **incremental philosophy, schema metadata, +and annotation extensibility**. The rest is execution. + +### 1. Data tests + +dbt: `unique`, `not_null`, `accepted_values`, custom generic tests, plus +singular tests. Run as `SELECT` statements that pass when they return 0 rows. + +Today: nothing. Annotation parser is the natural hook — +`// test unique col_name`, `// test not_null col_name`, +`// test ` for custom. Pipeline runtime already handles +failure propagation. Lowest-risk, highest-payoff item. + +### 2. Incremental materializations + +See [Incremental deep-dive](#incremental-deep-dive) below. + +### 3. Column lineage + docs + +dbt: SQL-AST parsing for column-level deps; `dbt docs serve` produces a +static lineage site with descriptions. + +Today: graph is asset-level. `SqlQueryDetails` in the parser +(`backend/parsers/windmill-parser/src/asset_parser.rs:44`) already has a +column map — the scaffolding exists. No `// column` annotation, no docs +surface. Pure TODO; no abstraction stands in the way. + +### 4. Snapshots / SCD2 + +dbt: `{% snapshot %}` blocks with `strategy='timestamp'` or `'check'`. +Today: nothing. Add as a new `PipelineOutputKind` + `// snapshot strategy= +timestamp updated_at=updated_at unique_key=id` annotation. Same shape as +other output kinds. + +### 5. Selective execution grammar + +dbt: `--select tag:nightly+ state:modified+ +my_model+`. +Today: `requestRunCascadeSignal` in the canvas, `// tag` annotation parsed. +Graph + tags + last-run state has all the inputs. UI/CLI surface, not +abstraction work. + +### 6. Schema contracts + +dbt: `contract: enforced` + `columns: [{name, data_type}]`. Compile-time +check that model output matches the declaration. + +Today: `// on datatable://users/active` is a string. Rename a column +upstream → downstream breaks at runtime, silently. + +This is the item where the current asset abstraction is thinnest. +To do contracts well: capture output schemas after a run (substrate-specific +DESCRIBE), persist them as asset metadata, validate consumer references at +save time. The asset-as-typed-node model accommodates it — but **where** +schemas live (asset row, sidecar?), **when** they're captured (post-run? +edit-time?), and **how** versioning works are non-trivial design choices. +Worth doing intentionally now while the asset surface is still young. + +### 7. Packages / community + +dbt: `dbt deps`, `dbt-utils`, `dbt-expectations`. Whole ecosystem on Jinja +macros. + +Today: closed-vocabulary annotation parser — `parsePipelineAnnotations` +hardcodes `pipeline`, `partitioned`, `freshness`, `trigger`, `debounce`, +`tag`, `retry`, `on`. No way for a package to register +`// test rows_between 100 1000000` or `// hook on_failure my_alert`. + +This is the one place the current abstraction starts to bind. Macros are +also dbt's biggest pain source — we don't have to replicate them. Possible +shapes: + +- **Hooks-as-scripts**: `// on_failure f/lib/alert`, `// pre_run f/lib/setup`. + Value is a script path. Stays inside the closed annotation set; new + hook *types* still require parser changes but third-party *behavior* + ships as scripts. +- **Test types as scripts**: a test is a script that returns 0/1, packaged + via the hub like anything else. +- **Materialization plugins**: harder; template-generator would need to be + extensible. + +Doing this *after* you've shipped 30 hardcoded annotations is much harder +than doing it now. + +### 8. Semantic layer / metrics + +dbt: `metrics:` blocks, MetricFlow, BI-tool query API. Large scope, +additive. Lowest priority of the eight. + +## Incremental deep-dive + +### How dbt incremental works + +```sql +-- models/marts/orders_daily.sql +{{ config( + materialized='incremental', + unique_key='order_id', + incremental_strategy='merge', + on_schema_change='append_new_columns' +) }} + +SELECT order_id, user_id, amount, created_at +FROM {{ ref('orders_raw') }} +{% if is_incremental() %} +WHERE created_at > (SELECT MAX(created_at) FROM {{ this }}) +{% endif %} +``` + +- **First run** (target doesn't exist): `CREATE TABLE orders_daily AS + SELECT ...` — full build, no WHERE. +- **Subsequent runs**: stage to temp table, then MERGE on `unique_key`. + +Knobs: `incremental_strategy` ∈ {`merge`, `append`, `delete+insert`, +`insert_overwrite`, `microbatch`}. `on_schema_change` ∈ {`fail`, `ignore`, +`append_new_columns`, `sync_all_columns`}. `--full-refresh` forces rebuild. + +Pain points: watermark + unique_key interaction is subtle (late-arriving +rows past the watermark are silently dropped); `on_schema_change` defaults +to `ignore` (silent column drop); Jinja `is_incremental()` runs at compile, +not runtime — debugging requires `dbt compile`; cross-warehouse MERGE +dialect is dbt's biggest internal complexity. + +### Where Windmill stands today + +- `// partitioned daily|hourly|weekly|monthly|dynamic key=...` parsed into + `PartitionSpec` at + `backend/parsers/windmill-parser/src/asset_parser.rs:172`. +- `// freshness 1h` parsed. +- Templates emit `CREATE TABLE IF NOT EXISTS ... AS SELECT *` — full + refresh, every run, no partition substitution. +- No `WM_PARTITION_*` context flowing into scripts. +- No materialized-partition state per asset. + +Annotations are present but metadata-only. Nothing actually executes +incrementally yet. + +### Path A — Literal templates ("script is the truth") + +Philosophy: WYSIWYG. Windmill never wraps. Templates scaffold boilerplate, +partition context is injected as bind / env vars, the user owns the SQL. + +```sql +-- pipeline +-- on datatable://prod/orders_raw +-- partitioned daily +-- unique_key order_id + +ATTACH 'datatable://prod' AS pg; + +CREATE TABLE IF NOT EXISTS pg.orders_daily ( + order_id BIGINT PRIMARY KEY, + user_id BIGINT, + amount NUMERIC, + created_at TIMESTAMPTZ +); + +CREATE OR REPLACE TEMP TABLE _stage AS +SELECT order_id, user_id, amount, created_at +FROM pg.orders_raw +WHERE created_at >= $WM_PARTITION_START + AND created_at < $WM_PARTITION_END; + +BEGIN; +DELETE FROM pg.orders_daily + WHERE created_at >= $WM_PARTITION_START + AND created_at < $WM_PARTITION_END; +INSERT INTO pg.orders_daily SELECT * FROM _stage; +COMMIT; +``` + +Runtime: resolve `(value, start, end)` from scheduler tick / trigger event +/ backfill range → bind as SQL params → execute script as-is → record +`(asset_path, partition_value)` on success. + +**Pros**: no compile step; backfill is trivial (idempotent DELETE+INSERT); +late-arriving data → just re-run the affected partition; no dialect +rewriting in core; Python/TS/Bash/SQL all fit the same model. + +**Cons**: boilerplate per script; materialization changes require script +edits; user owns dialect specifics. + +### Path B — dbt-style wrapping + +Philosophy: separate intent (SELECT) from execution (DDL). User declares +what; Windmill compiles to per-substrate DDL. + +User writes: + +```sql +SELECT order_id, user_id, amount, created_at +FROM pg.orders_raw +WHERE created_at >= $WM_PARTITION_START + AND created_at < $WM_PARTITION_END +``` + +Runtime parses, looks up target schema, wraps per output kind + +strategy + first-run/subsequent-run state. + +**Pros**: concise; materialization is a config flip; automatic schema-drift +handling; cross-substrate consistency. + +**Cons**: two-layer execution ("what ran?" needs a compile-output view); +SELECT-only restricts pre/post-statement work (dbt's answer: `pre_hook` / +`post_hook` — more surface); doesn't generalize to Python/TS (you end up +with two execution models); cross-substrate MERGE dialect is where dbt has +burned the most engineering — we'd inherit that tax forever; schema +introspection per substrate is its own project. + +### Path C — Hybrid (recommended) + +- **Literal-by-default**: scaffolds emit full DDL with `WM_PARTITION_*` + substitution. WYSIWYG for all languages. +- **Helper library** (e.g. `wmll.partition`, `wmll.datatable.upsert_partition`): + lifts boilerplate into library calls without hiding semantics — readable + source. +- **Opt-in wrapping** for single-SELECT SQL scripts via + `// materialized incremental wrap=true`. Limit to DuckDB first; add + others as needed. Always log the compiled SQL. +- **State + backfill UI**: persist materialized partitions per asset; UI + to backfill a range with concurrency cap. + +Ships A's 80% case first without committing to B's dialect-rewriting tax. +Wrapping becomes opt-in convenience for users who want dbt-style ergonomics. + +### Decisions either path forces + +1. **Partition window provenance.** Scheduler tick? Trigger event time + (Kafka `event_time` header)? Explicit backfill? Default = "now's bucket"? +2. **Surface.** Bind params (`$WM_PARTITION_START`), env vars + (`WM_PARTITION_START`), helper library — probably all three for + different languages, but pick canonical names. +3. **First-run bootstrap.** Template scaffolds `CREATE TABLE IF NOT + EXISTS` (A), or runtime detects "table missing → full refresh" (B). +4. **State tracking.** `materialized_partitions` keyed by `(workspace, + asset_kind, asset_path)`. Drives "run stale," backfill gap detection, + downstream waiting. +5. **Backfill execution.** N partitions → serial? Parallel with + concurrency cap per asset? +6. **Idempotency contract.** `// partitioned` should imply "re-running the + same partition is safe." Templates and helpers must enforce. + +## Partitioning vs. incremental: the reframing + +Partitioning covers ~80% of what dbt's incremental does. What it gives +for free: + +- Unit of work (one partition per run) +- Idempotency (DELETE-by-partition + INSERT is safe to rerun) +- State (track which partitions are materialized) +- Backfill (re-run a range) +- First-run vs. subsequent-run (every run is "process partition P" — no + special case) +- "Process only new data" (the partition window IS the filter) + +dbt itself has been migrating toward partition-first thinking via +`microbatch` strategy — essentially `incremental` with mandatory partition +key. + +### What partitioning alone doesn't address + +**Dedup within a partition by a separate key.** Example: partition by +`created_at` daily, but `orders_raw` is mutable — the same `order_id` can +appear multiple times in one partition (initial create, then amendments). +You want `orders_daily` to hold the latest version per `order_id`. + +DELETE-by-partition + INSERT works only if you reprocess from a +source-of-truth source. If you're consuming amendments and need dedup +*within* the slice, you need MERGE on `order_id`, not DELETE on partition. + +This is what dbt's `unique_key` does. Orthogonal to partitioning: +`partitioned` answers "which slice?"; `unique_key` answers "how do I dedup +inside the slice?" + +**Pure watermark-based incremental.** Mostly subsumed by +`// partitioned dynamic key=updated_at` — a partition becomes "everything +since the last seen value of `key`." + +### Recommended annotation shape + +Don't build "incremental" as a concept. Build: + +- `// partitioned ` — unit of work + state + backfill (already exists). +- `// unique_key ` — opt-in dedup-within-partition. Drives MERGE + template vs. DELETE+INSERT template. +- `// append` — opt-out of dedup entirely (INSERT-only, no DELETE). + +This collapses dbt's `materialized=incremental` + `incremental_strategy` + +`unique_key` into orthogonal annotations that compose. Partition-first is +the better mental model. + +Schema drift handling (`on_schema_change`) is genuinely separate — applies +to full-refresh too — and belongs with the schema-contracts work (gap #6). + +## First implementation slice + +Sequencing if we go with the hybrid + partition-first reframing: + +1. **Partition runtime context** — resolve `(value, start, end)` from + scheduler / trigger / backfill, surface as bind vars + env vars. No + materialization change yet. +2. **Helper library** — `wmll.partition.window()`, + `wmll.datatable.upsert_partition()` for Python/TS, SQL macros for + DuckDB/PG. +3. **Template updates** — when `// partitioned X` is present, scaffold + DELETE+INSERT (or MERGE when `// unique_key` also present, or INSERT + when `// append`). +4. **Materialized-partition state** — new table keyed by + `(workspace, asset_kind, asset_path, partition_value)`. Asset metadata + read API exposes it. +5. **Backfill UI** — date range picker on the pipeline folder page; fans + out runs with concurrency cap. +6. *Later, behind a flag:* opt-in wrap mode for single-SELECT DuckDB. + +Delivers dbt's pragmatic value (incremental, backfill, idempotent reruns) +without buying the compile-layer maintenance, and keeps Windmill +recognizably Windmill-shaped. diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 6a6d281b59..39cc16c056 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -78,7 +78,7 @@ "vscode-languageclient": "~9.0.1", "vscode-uri": "~3.1.0", "vscode-ws-jsonrpc": "~3.5.0", - "windmill-parser-wasm-asset": "1.673.0", + "windmill-parser-wasm-asset": "^1.728.1", "windmill-parser-wasm-csharp": "1.510.1", "windmill-parser-wasm-go": "1.510.1", "windmill-parser-wasm-java": "1.510.1", @@ -166,11 +166,6 @@ "svelte": "^5.0.0" } }, - "../backend/parsers/windmill-parser-wasm/pkg-ts": { - "name": "windmill-parser-wasm-ts", - "version": "1.673.0", - "extraneous": true - }, "node_modules/@alloc/quick-lru": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/@alloc/quick-lru/-/quick-lru-5.2.0.tgz", @@ -848,7 +843,6 @@ "version": "1.10.0", "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -860,7 +854,6 @@ "version": "1.10.0", "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -871,7 +864,6 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1361,7 +1353,6 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.4.tgz", "integrity": "sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1510,7 +1501,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1527,7 +1517,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1544,7 +1533,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1561,7 +1549,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1578,7 +1565,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1595,7 +1581,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1612,7 +1597,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1629,7 +1613,6 @@ "cpu": [ "ppc64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1646,7 +1629,6 @@ "cpu": [ "s390x" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1663,7 +1645,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1680,7 +1661,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1697,7 +1677,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1714,7 +1693,6 @@ "cpu": [ "wasm32" ], - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -1733,7 +1711,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -1750,7 +1727,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2056,7 +2032,6 @@ "version": "0.10.2", "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.2.tgz", "integrity": "sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==", - "dev": true, "license": "MIT", "optional": true, "dependencies": { @@ -6857,7 +6832,7 @@ "version": "1.21.7", "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.7.tgz", "integrity": "sha512-/imKNG4EbWNrVjoNC/1H5/9GFy+tqjGBHCaSsN+P2RnPqjsLmv6UD3Ej+Kj8nBWaRAwyk7kK5ZUc+OEatnTR3A==", - "dev": true, + "devOptional": true, "license": "MIT", "bin": { "jiti": "bin/jiti.js" @@ -7356,7 +7331,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7377,7 +7351,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7398,7 +7371,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7419,7 +7391,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7440,7 +7411,6 @@ "cpu": [ "arm" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7461,7 +7431,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7482,7 +7451,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7503,7 +7471,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7524,7 +7491,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7545,7 +7511,6 @@ "cpu": [ "arm64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -7566,7 +7531,6 @@ "cpu": [ "x64" ], - "dev": true, "license": "MPL-2.0", "optional": true, "os": [ @@ -12149,21 +12113,6 @@ } } }, - "node_modules/svelte-check/node_modules/picomatch": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", - "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", - "dev": true, - "license": "MIT", - "optional": true, - "peer": true, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/svelte-eslint-parser": { "version": "0.43.0", "resolved": "https://registry.npmjs.org/svelte-eslint-parser/-/svelte-eslint-parser-0.43.0.tgz", @@ -12894,7 +12843,7 @@ "version": "5.9.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "dev": true, + "devOptional": true, "license": "Apache-2.0", "bin": { "tsc": "bin/tsc", @@ -13667,9 +13616,9 @@ } }, "node_modules/windmill-parser-wasm-asset": { - "version": "1.673.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-asset/-/windmill-parser-wasm-asset-1.673.0.tgz", - "integrity": "sha512-sJZ9YbKhxMT67wWilmT7CDbwX0eVr+rvc6Y4OdHzOXwTwu3MixUinzVyP/lD4WDQW6xzitqOfstHBUrOxpww3A==" + "version": "1.728.1", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-asset/-/windmill-parser-wasm-asset-1.728.1.tgz", + "integrity": "sha512-73cyU6XM3gYEjFBx3qOKnv+VV1t70eAr6OiT+x0QobjFVNmqZFEdA7ayMYYVCnnix8OZxcsTNEF1hT61w1XiKw==" }, "node_modules/windmill-parser-wasm-csharp": { "version": "1.510.1", diff --git a/frontend/package.json b/frontend/package.json index 341d86c709..98838b9713 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -153,7 +153,7 @@ "vscode-languageclient": "~9.0.1", "vscode-uri": "~3.1.0", "vscode-ws-jsonrpc": "~3.5.0", - "windmill-parser-wasm-asset": "1.673.0", + "windmill-parser-wasm-asset": "1.728.1", "windmill-parser-wasm-csharp": "1.510.1", "windmill-parser-wasm-go": "1.510.1", "windmill-parser-wasm-java": "1.510.1", diff --git a/frontend/src/lib/attachments/arrowTabNav.ts b/frontend/src/lib/attachments/arrowTabNav.ts new file mode 100644 index 0000000000..e98b7db34c --- /dev/null +++ b/frontend/src/lib/attachments/arrowTabNav.ts @@ -0,0 +1,59 @@ +import { tabbable } from 'tabbable' +import type { Attachment } from 'svelte/attachments' + +export interface ArrowTabNavOptions { + /** Which arrow-key pair walks the tab order. Default `'y'` (Up/Down). */ + axis?: 'x' | 'y' + /** Use this to layer custom keys (Enter, Escape …) */ + onKeyDown?: (e: KeyboardEvent) => void +} + +/** + * Map one axis of arrow keys to next/previous in Tab order, scoped to + * the attached element. Wraps around at the ends. Bails when the + * keypress originates inside a text input / textarea / contenteditable + * so the caret can still move with the arrows. + * + *
…
+ *
…
+ *
{ … } })}>…
+ */ +export function arrowTabNav(opts: ArrowTabNavOptions = {}): Attachment { + const axis = opts.axis ?? 'y' + const nextKey = axis === 'y' ? 'ArrowDown' : 'ArrowRight' + const prevKey = axis === 'y' ? 'ArrowUp' : 'ArrowLeft' + + return (node) => { + const handler = (e: KeyboardEvent) => { + // Run the consumer's handler first so they can preventDefault + // or do their own thing before we react to arrows. + opts.onKeyDown?.(e) + + if (e.key !== nextKey && e.key !== prevKey) return + + // Let editable fields keep their native caret behavior. + const t = e.target as HTMLElement | null + if (t && (t.tagName === 'INPUT' || t.tagName === 'TEXTAREA' || t.isContentEditable)) { + return + } + + const items = tabbable(node) + if (items.length === 0) return + + const active = document.activeElement as HTMLElement | null + const i = active ? items.indexOf(active) : -1 + const dir = e.key === nextKey ? 1 : -1 + // No tabbable currently focused: jump to the first (next) or last (prev). + const next = + i === -1 + ? items[dir === 1 ? 0 : items.length - 1] + : items[(i + dir + items.length) % items.length] + + e.preventDefault() + next?.focus() + } + + node.addEventListener('keydown', handler) + return () => node.removeEventListener('keydown', handler) + } +} diff --git a/frontend/src/lib/attachments/selectAndAdvanceTo.ts b/frontend/src/lib/attachments/selectAndAdvanceTo.ts new file mode 100644 index 0000000000..0de23572c5 --- /dev/null +++ b/frontend/src/lib/attachments/selectAndAdvanceTo.ts @@ -0,0 +1,49 @@ +import { tabbable } from 'tabbable' + +/** + * Build an Enter-key handler for `arrowTabNav`'s `onKeyDown` (or any + * `keydown` listener): activate whatever is focused (so its `onClick` + * fires), then move focus to the first tabbable inside the container + * returned by `getNext`. preventDefault suppresses the browser's own + * Enter→click so we don't double-fire. + * + * By default the handler bails inside `