diff --git a/CHANGELOG.md b/CHANGELOG.md index acc2bd6c5a..d1b0a05c89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,39 @@ # Changelog +## [1.575.0](https://github.com/windmill-labs/windmill/compare/v1.574.3...v1.575.0) (2025-11-12) + + +### Features + +* SQL result_collection flag ([#7041](https://github.com/windmill-labs/windmill/issues/7041)) ([14e41ba](https://github.com/windmill-labs/windmill/commit/14e41babfe6321abb22081b1b3601e9009451257)) + + +### Bug Fixes + +* allow --yes in fork command ([#7122](https://github.com/windmill-labs/windmill/issues/7122)) ([1280617](https://github.com/windmill-labs/windmill/commit/1280617a283ef7bcf53c17d833f1413ffe69dbf8)) +* **backend:** scim email filter ([#7120](https://github.com/windmill-labs/windmill/issues/7120)) ([7a8fd86](https://github.com/windmill-labs/windmill/commit/7a8fd865e289522fd2d5eaaa9af471f01193a03d)) +* **flow chat:** fix chat in edit mode + cleaner code ([#7118](https://github.com/windmill-labs/windmill/issues/7118)) ([9791eb4](https://github.com/windmill-labs/windmill/commit/9791eb4568aa9548174941f1bbfb6a338c3bef00)) + +## [1.574.3](https://github.com/windmill-labs/windmill/compare/v1.574.2...v1.574.3) (2025-11-11) + + +### Bug Fixes + +* **backend:** add sslmode parameter to DuckLake PostgreSQL connections ([ea5b3e6](https://github.com/windmill-labs/windmill/commit/ea5b3e6b9269a0c631c7601313ab97c4bbdc2137)) +* use teams internal_id rather than id ([#7113](https://github.com/windmill-labs/windmill/issues/7113)) ([25c36c1](https://github.com/windmill-labs/windmill/commit/25c36c1e0a6cfc56e079ff5a18980ea67403d4f9)) + +## [1.574.2](https://github.com/windmill-labs/windmill/compare/v1.574.1...v1.574.2) (2025-11-10) + + +### Bug Fixes + +* **backend:** improve pwsh param block parsing ([#7096](https://github.com/windmill-labs/windmill/issues/7096)) ([51cba95](https://github.com/windmill-labs/windmill/commit/51cba95d3961ec637fd5060556f0e7a8a8bdbb88)) +* **backend:** improve sse parsing ([#7100](https://github.com/windmill-labs/windmill/issues/7100)) ([2e245ac](https://github.com/windmill-labs/windmill/commit/2e245accc5f641f64a2cd28fc181e6a806c407e2)) +* fix multiselect in list for apps ([9e4882c](https://github.com/windmill-labs/windmill/commit/9e4882c0a919303480f5a431db3c05b1d5855d37)) +* make ai chat works with unicode messages ([2d54dfb](https://github.com/windmill-labs/windmill/commit/2d54dfbf05f21faa14655b7124daa291a9d8d66d)) +* preproccessor ui and expanding preprocessor support ([#6872](https://github.com/windmill-labs/windmill/issues/6872)) ([fc5034e](https://github.com/windmill-labs/windmill/commit/fc5034e94d513bab2feb0ebb2aff822bbb42e4aa)) +* proper error handling in pulled job preprocessor ([#7098](https://github.com/windmill-labs/windmill/issues/7098)) ([84992cd](https://github.com/windmill-labs/windmill/commit/84992cd8ffa94e009aeb6b25db224a3001581e15)) + ## [1.574.1](https://github.com/windmill-labs/windmill/compare/v1.574.0...v1.574.1) (2025-11-08) diff --git a/backend/.sqlx/query-3d38720e807b379645d8f3ab61c6a968143d42c3014152608f7d1b252cd8085c.json b/backend/.sqlx/query-3d38720e807b379645d8f3ab61c6a968143d42c3014152608f7d1b252cd8085c.json deleted file mode 100644 index a97d16b89c..0000000000 --- a/backend/.sqlx/query-3d38720e807b379645d8f3ab61c6a968143d42c3014152608f7d1b252cd8085c.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO app_version\n (app_id, value, created_by, raw_app)\n SELECT app_id, value, created_by, raw_app\n FROM app_version WHERE id = $1\n RETURNING id", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - false - ] - }, - "hash": "3d38720e807b379645d8f3ab61c6a968143d42c3014152608f7d1b252cd8085c" -} diff --git a/backend/.sqlx/query-1368ccd2c15a75690041a6c87d4a2849fe6bc668654ffcbfbc22a02027280739.json b/backend/.sqlx/query-7a4e79bcae7c7d69e4bb71c8540b2adb9ca8bd15098038e3ccf8b16164944951.json similarity index 58% rename from backend/.sqlx/query-1368ccd2c15a75690041a6c87d4a2849fe6bc668654ffcbfbc22a02027280739.json rename to backend/.sqlx/query-7a4e79bcae7c7d69e4bb71c8540b2adb9ca8bd15098038e3ccf8b16164944951.json index 4dac49a78c..1eb38e8623 100644 --- a/backend/.sqlx/query-1368ccd2c15a75690041a6c87d4a2849fe6bc668654ffcbfbc22a02027280739.json +++ b/backend/.sqlx/query-7a4e79bcae7c7d69e4bb71c8540b2adb9ca8bd15098038e3ccf8b16164944951.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT COUNT(id) FROM v2_job WHERE permissioned_as_email = $1", + "query": "SELECT COUNT(j.id) FROM v2_job_queue q JOIN v2_job j USING (id) WHERE j.permissioned_as_email = $1", "describe": { "columns": [ { @@ -18,5 +18,5 @@ null ] }, - "hash": "1368ccd2c15a75690041a6c87d4a2849fe6bc668654ffcbfbc22a02027280739" + "hash": "7a4e79bcae7c7d69e4bb71c8540b2adb9ca8bd15098038e3ccf8b16164944951" } diff --git a/backend/.sqlx/query-83232f2db5eb1b6fef744998e60420ef920d472286cf4c1f78452446a4bcb604.json b/backend/.sqlx/query-83232f2db5eb1b6fef744998e60420ef920d472286cf4c1f78452446a4bcb604.json new file mode 100644 index 0000000000..27a5df6de9 --- /dev/null +++ b/backend/.sqlx/query-83232f2db5eb1b6fef744998e60420ef920d472286cf4c1f78452446a4bcb604.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO app_version\n (app_id, value, created_by, raw_app)\n SELECT app_id, value, created_by, raw_app\n FROM app_version WHERE id = $1\n RETURNING id", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "83232f2db5eb1b6fef744998e60420ef920d472286cf4c1f78452446a4bcb604" +} diff --git a/backend/.sqlx/query-a6a973dcd92d2e40fd9a1c1be42052fcd350bd47ee4f63832448b6e6f0f472f0.json b/backend/.sqlx/query-a6a973dcd92d2e40fd9a1c1be42052fcd350bd47ee4f63832448b6e6f0f472f0.json deleted file mode 100644 index 1c396d38fd..0000000000 --- a/backend/.sqlx/query-a6a973dcd92d2e40fd9a1c1be42052fcd350bd47ee4f63832448b6e6f0f472f0.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO flow_version\n (workspace_id, path, value, schema, created_by)\n\n SELECT workspace_id, path, value, schema, created_by\n FROM flow_version WHERE path = $1 AND workspace_id = $2 AND id = $3\n\n RETURNING id\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Int8" - ] - }, - "nullable": [ - false - ] - }, - "hash": "a6a973dcd92d2e40fd9a1c1be42052fcd350bd47ee4f63832448b6e6f0f472f0" -} diff --git a/backend/.sqlx/query-f0efa383f2025158de160577ad839ae72faf0c8fe097e6ad6d309aee9a8aede2.json b/backend/.sqlx/query-f0efa383f2025158de160577ad839ae72faf0c8fe097e6ad6d309aee9a8aede2.json new file mode 100644 index 0000000000..32616f298f --- /dev/null +++ b/backend/.sqlx/query-f0efa383f2025158de160577ad839ae72faf0c8fe097e6ad6d309aee9a8aede2.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO flow_version\n (workspace_id, path, value, schema, created_by)\n\n SELECT workspace_id, path, value, schema, created_by\n FROM flow_version WHERE path = $1 AND workspace_id = $2 AND id = $3\n\n RETURNING id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false + ] + }, + "hash": "f0efa383f2025158de160577ad839ae72faf0c8fe097e6ad6d309aee9a8aede2" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index cb19092a20..d224f52b22 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -556,7 +556,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -670,7 +670,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -692,7 +692,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -703,7 +703,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -1026,7 +1026,7 @@ dependencies = [ "http 1.3.1", "http-body 0.4.6", "hyper 0.14.32", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-rustls 0.24.2", "hyper-rustls 0.27.7", "hyper-util", @@ -1182,7 +1182,7 @@ dependencies = [ "http 1.3.1", "http-body 1.0.1", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-util", "itoa", "matchit", @@ -1233,7 +1233,7 @@ checksum = "57d123550fa8d071b7255cb0cc04dc302baa6c8c4a79f55701552684d8399bce" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -1373,7 +1373,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.109", + "syn 2.0.110", "which 4.4.2", ] @@ -1394,7 +1394,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -1414,7 +1414,7 @@ dependencies = [ "regex", "rustc-hash 2.1.1", "shlex", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -1569,7 +1569,7 @@ dependencies = [ "hex", "http 1.3.1", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-named-pipe", "hyper-util", "hyperlocal", @@ -1621,7 +1621,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -1644,7 +1644,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -1654,7 +1654,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17d4f95e880cfd28c4ca5a006cf7f6af52b4bcb7b5866f573b2faa126fb7affb" dependencies = [ "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -1788,7 +1788,7 @@ checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -1931,7 +1931,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b4a6cae9efc04cc6cbb8faf338d2c497c165c83e74509cf4dbedea948bbf6e5" dependencies = [ "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -2083,7 +2083,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -2138,7 +2138,7 @@ dependencies = [ "nom 7.1.3", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -2457,9 +2457,9 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", "rand_core 0.6.4", @@ -2520,7 +2520,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -2613,7 +2613,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -2627,7 +2627,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -2660,7 +2660,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -2671,7 +2671,7 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core 0.21.3", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -3163,7 +3163,7 @@ checksum = "df6f88d7ee27daf8b108ba910f9015176b36fbc72902b1ca5c2a5f1d1717e1a1" dependencies = [ "datafusion-expr", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -3601,7 +3601,7 @@ checksum = "babccedee31ce7e57c3e6dff2cb3ab8d68c49d0df8222fe0d11d628e65192790" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -3625,7 +3625,7 @@ dependencies = [ "hickory-resolver", "http 1.3.1", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-rustls 0.27.7", "hyper-util", "ipnet", @@ -3713,7 +3713,7 @@ dependencies = [ "http 1.3.1", "httparse", "hyper 0.14.32", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-util", "itertools 0.10.5", "memmem", @@ -3906,7 +3906,7 @@ dependencies = [ "hkdf", "http 1.3.1", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-util", "idna", "indexmap 2.11.1", @@ -3992,7 +3992,7 @@ dependencies = [ "stringcase", "strum 0.25.0", "strum_macros 0.25.3", - "syn 2.0.109", + "syn 2.0.110", "thiserror 2.0.17", ] @@ -4177,7 +4177,7 @@ dependencies = [ "http 1.3.1", "http-body-util", "hyper 0.14.32", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-util", "libc", "log", @@ -4231,7 +4231,7 @@ dependencies = [ "deno_error", "deno_tls", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-rustls 0.27.7", "hyper-util", "log", @@ -4249,9 +4249,9 @@ dependencies = [ [[package]] name = "deno_terminal" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f71c27009e0141dedd315f1dfa3ebb0a6ca4acce7c080fac576ea415a465f6" +checksum = "f3ba8041ae7319b3ca6a64c399df4112badcbbe0868b4517637647614bede4be" dependencies = [ "once_cell", "termcolor", @@ -4361,7 +4361,7 @@ dependencies = [ "h2 0.4.12", "http 1.3.1", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-util", "once_cell", "rustls-tokio-stream", @@ -4494,7 +4494,7 @@ checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -4515,7 +4515,7 @@ checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -4559,7 +4559,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version 0.4.1", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -4579,7 +4579,7 @@ checksum = "bda628edc44c4bb645fbe0f758797143e4e07926f7ebf4e9bdfbd3d2ce621df3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -4705,7 +4705,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -4740,7 +4740,7 @@ checksum = "788160fb30de9cdd857af31c6a2675904b16ece8fc2737b2c7127ba368c9d0f4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -4950,7 +4950,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -5016,7 +5016,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -5036,7 +5036,7 @@ checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -5056,7 +5056,7 @@ checksum = "67c78a4d8fdf9953a5c9d458f9efe940fd97a0cab0941c075a813ac594733827" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -5144,6 +5144,17 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "eventsource-stream" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74fef4569247a5f429d9156b9d0a2599914385dd189c539334c625d8099d90ab" +dependencies = [ + "futures-core", + "nom 7.1.3", + "pin-project-lite", +] + [[package]] name = "fallible-iterator" version = "0.2.0" @@ -5214,7 +5225,7 @@ dependencies = [ "base64 0.21.7", "bytes", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-util", "pin-project", "rand 0.8.5", @@ -5373,7 +5384,7 @@ checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -5411,7 +5422,7 @@ checksum = "32016f1242eb82af5474752d00fd8ebcd9004bd69b462b1c91de833972d08ed4" dependencies = [ "proc-macro2", "swc_macros_common", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -5546,7 +5557,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -5843,9 +5854,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.9" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -5931,7 +5942,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -6263,7 +6274,7 @@ checksum = "149e3ea90eb5a26ad354cfe3cb7f7401b9329032d0235f2687d03a35f30e5d4c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -6593,9 +6604,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e" +checksum = "1744436df46f0bde35af3eda22aeaba453aada65d8f1c171cd8a5f59030bd69f" dependencies = [ "atomic-waker", "bytes", @@ -6624,7 +6635,7 @@ dependencies = [ "futures-util", "headers", "http 1.3.1", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-rustls 0.27.7", "hyper-util", "pin-project-lite", @@ -6641,7 +6652,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" dependencies = [ "hex", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-util", "pin-project-lite", "tokio", @@ -6672,7 +6683,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ "http 1.3.1", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-util", "log", "rustls 0.23.29", @@ -6690,7 +6701,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.7.0", + "hyper 1.8.0", "hyper-util", "pin-project-lite", "tokio", @@ -6718,7 +6729,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-util", "native-tls", "tokio", @@ -6739,7 +6750,7 @@ dependencies = [ "futures-util", "http 1.3.1", "http-body 1.0.1", - "hyper 1.7.0", + "hyper 1.8.0", "ipnet", "libc", "percent-encoding", @@ -6760,7 +6771,7 @@ checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" dependencies = [ "hex", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-util", "pin-project-lite", "tokio", @@ -7109,7 +7120,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -7408,7 +7419,7 @@ dependencies = [ "http 1.3.1", "http-body 1.0.1", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-http-proxy", "hyper-rustls 0.27.7", "hyper-timeout", @@ -7460,7 +7471,7 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -7516,7 +7527,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -7713,9 +7724,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.22" +version = "1.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d" +checksum = "15d118bbf3771060e7311cc7bb0545b01d08a8b4a7de949198dec1fa0ca1c0f7" dependencies = [ "cc", "libc", @@ -8123,7 +8134,7 @@ checksum = "db5b29714e950dbb20d5e6f74f9dcec4edbcc1067bb7f8ed198c097b8c1a818b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -8242,7 +8253,7 @@ checksum = "c402a4092d5e204f32c9e155431046831fa712637043c58cb73bc6bc6c9663b5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -8287,7 +8298,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", "termcolor", "thiserror 2.0.17", ] @@ -8390,7 +8401,7 @@ dependencies = [ "quote", "serde", "serde_json", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -8596,7 +8607,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -8753,9 +8764,9 @@ dependencies = [ [[package]] name = "num-bigint-dig" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82c79c15c05d4bf82b6f5ef163104cc81a760d8e874d38ac50ab67c8877b647b" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" dependencies = [ "lazy_static", "libm", @@ -8864,7 +8875,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -8935,7 +8946,7 @@ dependencies = [ "http-body-util", "httparse", "humantime", - "hyper 1.7.0", + "hyper 1.8.0", "itertools 0.14.0", "md-5 0.10.6", "parking_lot 0.12.5", @@ -9092,7 +9103,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -9556,7 +9567,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -9637,7 +9648,7 @@ dependencies = [ "phf_shared 0.11.3", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -9687,7 +9698,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -9899,7 +9910,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -9963,7 +9974,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -9974,7 +9985,7 @@ checksum = "07c277e4e643ef00c1233393c673f655e3672cf7eb3ba08a00bdd0ea59139b5f" dependencies = [ "proc-macro-rules-macros", "proc-macro2", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -9986,7 +9997,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -10097,7 +10108,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.109", + "syn 2.0.110", "tempfile", ] @@ -10111,7 +10122,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -10521,7 +10532,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -10590,7 +10601,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -10701,7 +10712,7 @@ dependencies = [ "http 1.3.1", "http-body 1.0.1", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-rustls 0.27.7", "hyper-tls 0.6.0", "hyper-util", @@ -10761,7 +10772,7 @@ dependencies = [ "futures", "getrandom 0.2.16", "http 1.3.1", - "hyper 1.7.0", + "hyper 1.8.0", "parking_lot 0.11.2", "reqwest 0.12.24", "reqwest-middleware", @@ -10905,7 +10916,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -10995,7 +11006,7 @@ dependencies = [ "quote", "rust-embed-utils", "shellexpand", - "syn 2.0.109", + "syn 2.0.110", "walkdir", ] @@ -11483,7 +11494,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -11495,7 +11506,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -11684,7 +11695,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -11695,7 +11706,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -11747,7 +11758,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -11814,7 +11825,7 @@ dependencies = [ "darling 0.21.3", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -12245,7 +12256,7 @@ checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -12310,7 +12321,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -12333,7 +12344,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.109", + "syn 2.0.110", "tokio", "url", ] @@ -12497,7 +12508,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -12566,7 +12577,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -12578,7 +12589,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -12697,7 +12708,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -12746,7 +12757,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -12831,7 +12842,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -12938,7 +12949,7 @@ checksum = "63db0adcff29d220c3d151c5b25c0eabe7e32dd936212b84cdaa1392e3130497" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -12949,7 +12960,7 @@ checksum = "f486687bfb7b5c560868f69ed2d458b880cebc9babebcb67e49f31b55c5bf847" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -12972,7 +12983,7 @@ dependencies = [ "proc-macro2", "quote", "swc_macros_common", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -12988,9 +12999,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.109" +version = "2.0.110" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f17c7e013e88258aa9543dcbe81aca68a667a9ac37cd69c9fbc07858bfe0e2f" +checksum = "a99801b5bd34ede4cf3fc688c5919368fea4e4814a4664359503e6015b280aea" dependencies = [ "proc-macro2", "quote", @@ -13032,7 +13043,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -13390,7 +13401,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -13401,7 +13412,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -13647,7 +13658,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -13934,7 +13945,7 @@ dependencies = [ "http 1.3.1", "http-body 1.0.1", "http-body-util", - "hyper 1.7.0", + "hyper 1.8.0", "hyper-timeout", "hyper-util", "percent-encoding", @@ -14077,7 +14088,7 @@ checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -14330,7 +14341,7 @@ checksum = "27a7a9b72ba121f6f1f6c3632b85604cac41aedb5ddc70accbebb6cac83de846" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -14813,7 +14824,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", "wasm-bindgen-shared", ] @@ -14848,7 +14859,7 @@ checksum = "ffc003a991398a8ee604a401e194b6b3a39677b3173d6e74495eb51b82e99a32" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -14883,7 +14894,7 @@ checksum = "a369369e4360c2884c3168d22bded735c43cccae97bbc147586d4b480edd138d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -15137,7 +15148,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.574.1" +version = "1.575.0" dependencies = [ "anyhow", "aws-sdk-config", @@ -15197,7 +15208,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.574.1" +version = "1.575.0" dependencies = [ "anyhow", "argon2", @@ -15240,7 +15251,7 @@ dependencies = [ "hf-hub", "hmac", "http 1.3.1", - "hyper 1.7.0", + "hyper 1.8.0", "indexmap 2.11.1", "itertools 0.14.0", "jsonwebtoken 8.3.0", @@ -15317,7 +15328,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.574.1" +version = "1.575.0" dependencies = [ "base64 0.22.1", "chrono", @@ -15332,7 +15343,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.574.1" +version = "1.575.0" dependencies = [ "chrono", "lazy_static", @@ -15346,7 +15357,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.574.1" +version = "1.575.0" dependencies = [ "anyhow", "axum", @@ -15365,7 +15376,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.574.1" +version = "1.575.0" dependencies = [ "anyhow", "async-recursion", @@ -15393,7 +15404,7 @@ dependencies = [ "globset", "hex", "hmac", - "hyper 1.7.0", + "hyper 1.8.0", "indexmap 2.11.1", "itertools 0.14.0", "jsonwebtoken 8.3.0", @@ -15450,7 +15461,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.574.1" +version = "1.575.0" dependencies = [ "regex", "serde", @@ -15465,7 +15476,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.574.1" +version = "1.575.0" dependencies = [ "anyhow", "bytes", @@ -15489,19 +15500,19 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.574.1" +version = "1.575.0" dependencies = [ "itertools 0.14.0", "lazy_static", "proc-macro2", "quote", "regex", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] name = "windmill-parser" -version = "1.574.1" +version = "1.575.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15510,7 +15521,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.574.1" +version = "1.575.0" dependencies = [ "anyhow", "lazy_static", @@ -15522,7 +15533,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.574.1" +version = "1.575.0" dependencies = [ "anyhow", "serde_json", @@ -15534,7 +15545,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.574.1" +version = "1.575.0" dependencies = [ "anyhow", "gosyn", @@ -15546,7 +15557,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.574.1" +version = "1.575.0" dependencies = [ "anyhow", "lazy_static", @@ -15558,7 +15569,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.574.1" +version = "1.575.0" dependencies = [ "anyhow", "serde_json", @@ -15570,7 +15581,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.574.1" +version = "1.575.0" dependencies = [ "anyhow", "nu-parser", @@ -15581,7 +15592,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.574.1" +version = "1.575.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15592,7 +15603,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.574.1" +version = "1.575.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15604,7 +15615,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.574.1" +version = "1.575.0" dependencies = [ "anyhow", "async-recursion", @@ -15627,7 +15638,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.574.1" +version = "1.575.0" dependencies = [ "anyhow", "lazy_static", @@ -15641,7 +15652,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.574.1" +version = "1.575.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15651,14 +15662,14 @@ dependencies = [ "quote", "regex", "serde_json", - "syn 2.0.109", + "syn 2.0.110", "toml", "windmill-parser", ] [[package]] name = "windmill-parser-sql" -version = "1.574.1" +version = "1.575.0" dependencies = [ "anyhow", "lazy_static", @@ -15672,7 +15683,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.574.1" +version = "1.575.0" dependencies = [ "anyhow", "lazy_static", @@ -15690,7 +15701,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.574.1" +version = "1.575.0" dependencies = [ "anyhow", "serde", @@ -15701,7 +15712,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.574.1" +version = "1.575.0" dependencies = [ "anyhow", "async-recursion", @@ -15725,6 +15736,7 @@ dependencies = [ "serde_urlencoded", "sql-builder", "sqlx", + "thiserror 2.0.17", "tokio", "tracing", "ulid", @@ -15735,7 +15747,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.574.1" +version = "1.575.0" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -15745,7 +15757,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.574.1" +version = "1.575.0" dependencies = [ "anyhow", "async-once-cell", @@ -15776,6 +15788,7 @@ dependencies = [ "deno_webidl", "dotenv", "dyn-iter", + "eventsource-stream", "flume", "futures", "gcp_auth", @@ -15956,7 +15969,7 @@ checksum = "f6fc35f58ecd95a9b71c4f2329b911016e6bec66b3f2e6a4aad86bd2e99e2f9b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -15967,7 +15980,7 @@ checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -15978,7 +15991,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -15989,7 +16002,7 @@ checksum = "08990546bf4edef8f431fa6326e032865f27138718c587dc21bc0265bbcb57cc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -16000,7 +16013,7 @@ checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -16011,7 +16024,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -16508,7 +16521,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", "synstructure 0.13.2", ] @@ -16520,7 +16533,7 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", "synstructure 0.13.2", ] @@ -16541,7 +16554,7 @@ checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -16561,7 +16574,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", "synstructure 0.13.2", ] @@ -16582,7 +16595,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] @@ -16615,7 +16628,7 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.109", + "syn 2.0.110", ] [[package]] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 41e33efd9a..5702c3a8f5 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.574.1" +version = "1.575.0" authors.workspace = true edition.workspace = true @@ -33,7 +33,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.574.1" +version = "1.575.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -252,6 +252,7 @@ urlencoding = "^2" url = { version = "^2" , features = ["serde"]} async-oauth2 = "0.5.1" reqwest = { version = "^0.12", features = ["json", "stream", "gzip", "multipart"] } +eventsource-stream = "0.2.3" time = "^0" serde_urlencoded = "^0" tokio-tar = "^0" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index b9762186b4..0da534da09 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -5b7afe50da442441747e7a8f6ef461c96faa9dc2 +7cdb62d968feef22046b12a1217b711af457cced \ No newline at end of file diff --git a/backend/migrations/20251105100125_legacy_sql_result_flag.down.sql b/backend/migrations/20251105100125_legacy_sql_result_flag.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20251105100125_legacy_sql_result_flag.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20251105100125_legacy_sql_result_flag.up.sql b/backend/migrations/20251105100125_legacy_sql_result_flag.up.sql new file mode 100644 index 0000000000..dbacc6084c --- /dev/null +++ b/backend/migrations/20251105100125_legacy_sql_result_flag.up.sql @@ -0,0 +1,99 @@ +CREATE OR REPLACE FUNCTION update_string(s text) +RETURNS text +LANGUAGE plpgsql +AS $$ +DECLARE + prefix TEXT := '-- https://www.windmill.dev/docs/getting_started/scripts_quickstart/sql#result-collection +-- result_collection=legacy + +'; +BEGIN + RETURN prefix || s; +END; +$$; + +CREATE OR REPLACE FUNCTION update_all_modules(obj jsonb) +RETURNS jsonb +LANGUAGE plpgsql +AS $$ +DECLARE + result jsonb; + k text; + v jsonb; +BEGIN + IF jsonb_typeof(obj) = 'object' THEN + result := '{}'::jsonb; + + FOR k, v IN SELECT * FROM jsonb_each(obj) + LOOP + IF k = 'content' and jsonb_typeof(v) = 'string' AND obj->>'language' IN ('bigquery', 'postgresql', 'duckdb', 'mssql', 'oracledb', 'snowflake', 'mysql') THEN + result := result || jsonb_build_object('content', update_string(obj->>'content')); + ELSE + result := result || jsonb_build_object(k, update_all_modules(v)); + END IF; + END LOOP; + + RETURN result; + ELSIF jsonb_typeof(obj) = 'array' AND jsonb_array_length(obj) > 0 THEN + SELECT jsonb_agg(update_all_modules(elem)) + INTO result + FROM jsonb_array_elements(obj) elem; + + RETURN result; + ELSE + RETURN obj; + END IF; +END; +$$; + +-- Run on a flow_version_lite jsonb value. Returns an array of flow_node ids whose languages are SQL. +CREATE OR REPLACE FUNCTION find_sql_flow_nodes_ids(obj jsonb) +RETURNS BIGINT[] +LANGUAGE plpgsql +AS $$ +DECLARE + result BIGINT[] := '{}'; + k text; + v jsonb; +BEGIN + IF jsonb_typeof(obj) = 'object' THEN + IF obj->>'language' IN ('bigquery', 'postgresql', 'duckdb', 'mssql', 'oracledb', 'snowflake', 'mysql') AND jsonb_typeof(obj->'id') = 'number' THEN + result := result || (obj->>'id')::BIGINT; + END IF; + + FOR k, v IN SELECT * FROM jsonb_each(obj) + LOOP + result := result || find_sql_flow_nodes_ids(v); + END LOOP; + ELSIF jsonb_typeof(obj) = 'array' AND jsonb_array_length(obj) > 0 THEN + SELECT array_agg(result_ids) + INTO result + FROM jsonb_array_elements(obj) elem, unnest(find_sql_flow_nodes_ids(elem)) as result_ids; + END IF; + RETURN result; +END; +$$; + +DO $$ +BEGIN + UPDATE app_version SET value = update_all_modules(value::jsonb)::json; + UPDATE draft SET value = update_all_modules(value::jsonb)::json; + UPDATE flow SET value = update_all_modules(value); + UPDATE flow_version SET value = update_all_modules(value); + UPDATE flow_node SET code = update_string(code) WHERE id IN ( + SELECT v FROM flow_version_lite, unnest(find_sql_flow_nodes_ids(value)) as v + ); + UPDATE app_script SET code = update_string(code) WHERE id IN ( + SELECT v FROM app_version_lite, unnest(find_sql_flow_nodes_ids(value)) as v + ); + UPDATE script SET content = update_string(content) WHERE language IN ('bigquery', 'postgresql', 'duckdb', 'mssql', 'oracledb', 'snowflake', 'mysql'); + +EXCEPTION WHEN OTHERS THEN + -- ✅ LOG ERROR WITHOUT STOPPING THE MIGRATION + RAISE WARNING 'Migration failed: %', SQLERRM; +END; +$$; + +DROP FUNCTION IF EXISTS update_all_modules(jsonb); +DROP FUNCTION IF EXISTS update_string(text); +DROP FUNCTION IF EXISTS find_sql_flow_nodes_ids(jsonb); diff --git a/backend/parsers/windmill-parser-bash/src/lib.rs b/backend/parsers/windmill-parser-bash/src/lib.rs index 4e3d70c038..5bfe6b20c5 100644 --- a/backend/parsers/windmill-parser-bash/src/lib.rs +++ b/backend/parsers/windmill-parser-bash/src/lib.rs @@ -47,7 +47,6 @@ pub fn parse_powershell_sig(code: &str) -> anyhow::Result { lazy_static::lazy_static! { static ref RE_BASH: Regex = Regex::new(r#"(?m)^(\w+)="\$(?:(\d+)|\{(\d+)\}|\{(\d+):-(.*)\})"(?:[\t ]*)?(?:#.*)?\r?$"#).unwrap(); - pub static ref RE_POWERSHELL_PARAM: Regex = Regex::new(r#"(?m)param[\t ]*\(([^)]*)\)"#).unwrap(); static ref RE_POWERSHELL_ARGS: Regex = Regex::new(r#"(?:\[([\w\[\]]+)\])?\$(\w+)[\t ]*(?:=[\t ]*(?:(?:(?:"|')([^"\n\r\$]*)(?:"|'))|([\d.]+)))?\r?"#).unwrap(); } @@ -85,6 +84,136 @@ fn parse_bash_file(code: &str) -> anyhow::Result>> { Ok(Some(args)) } +/// Extract a PowerShell param() block, handling nested parentheses. +/// +/// # Arguments +/// * `code` - The PowerShell code to extract from +/// * `include_keyword` - If true, returns the full block including "param(...)" +/// If false, returns only the contents between the parentheses +/// +/// # Returns +/// The extracted param block or contents, or None if not found. +pub fn extract_powershell_param_block(code: &str, include_keyword: bool) -> Option<&str> { + // Find "param" keyword (case-insensitive) + let lower_code = code.to_lowercase(); + let param_start = lower_code.find("param")?; + + // Verify that only comments and whitespace appear before "param" + let before_param = &code[..param_start]; + let mut chars = before_param.chars().peekable(); + let mut in_block_comment = false; + + while let Some(ch) = chars.next() { + if in_block_comment { + // Check for end of block comment: #> + if ch == '#' && chars.peek() == Some(&'>') { + chars.next(); // consume '>' + in_block_comment = false; + } + } else { + match ch { + // Start of block comment: <# + '<' if chars.peek() == Some(&'#') => { + chars.next(); // consume '#' + in_block_comment = true; + } + // Single-line comment: consume until end of line + '#' => { + while let Some(&next_ch) = chars.peek() { + if next_ch == '\n' || next_ch == '\r' { + break; + } + chars.next(); + } + } + // Whitespace is allowed + c if c.is_whitespace() => {} + // Any other character means there's code before param + _ => return None, + } + } + } + + // If we're still in a block comment at the end, it's unclosed - invalid + if in_block_comment { + return None; + } + + // Skip whitespace and tabs after "param" + let mut chars = code[param_start + 5..].char_indices(); + let mut paren_offset = param_start + 5; + + // Skip whitespace to find opening paren + while let Some((idx, ch)) = chars.next() { + if ch == '(' { + paren_offset += idx; + break; + } else if !ch.is_whitespace() && ch != '\t' { + // Found non-whitespace, non-paren character - not a valid param block + return None; + } + } + + // Now parse from the opening parenthesis + let remaining = &code[paren_offset..]; + let mut chars = remaining.char_indices(); + + // Skip the opening '(' + if let Some((_, ch)) = chars.next() { + if ch != '(' { + return None; + } + } else { + return None; + } + + let mut depth = 1; + let mut in_single_quote = false; + let mut in_double_quote = false; + let mut escape_next = false; + let content_start = paren_offset + 1; // Start after the opening '(' + + for (idx, ch) in chars { + if escape_next { + escape_next = false; + continue; + } + + match ch { + '`' if in_double_quote => { + // PowerShell escape character + escape_next = true; + } + '\'' if !in_double_quote => { + in_single_quote = !in_single_quote; + } + '"' if !in_single_quote => { + in_double_quote = !in_double_quote; + } + '(' if !in_single_quote && !in_double_quote => { + depth += 1; + } + ')' if !in_single_quote && !in_double_quote => { + depth -= 1; + if depth == 0 { + // Found the matching closing parenthesis + // idx is the position of ')' relative to paren_offset + if include_keyword { + // Return full block including "param" keyword and closing paren + return Some(&code[param_start..paren_offset + idx + 1]); + } else { + // Return only contents between parentheses + return Some(&code[content_start..paren_offset + idx]); + } + } + } + _ => {} + } + } + + None +} + enum ParserState { Normal, InSingleQuote, @@ -137,10 +266,9 @@ fn parse_powershell_single_typ(typ: &str) -> Typ { } fn parse_powershell_file(code: &str) -> anyhow::Result>> { - let param_wrapper = RE_POWERSHELL_PARAM.captures(code); + let param_wrapper = extract_powershell_param_block(code, false); let mut args = vec![]; if let Some(param_wrapper) = param_wrapper { - let param_wrapper = param_wrapper.get(1).unwrap().as_str(); let params = split_pwsh_args(param_wrapper); for param in params { if let Some(cap) = RE_POWERSHELL_ARGS.captures(param) { @@ -266,7 +394,7 @@ non_required="${5:-}" #[test] fn test_parse_powershell_sig() -> anyhow::Result<()> { - let code = r#"param($Msg, [string]$Msg2, $Dflt = "default value, with comma", [int]$Nb = 3 , $Nb2 = 5.0, $Nb3 = 5, $Wahoo = $env:WAHOO, [PSCustomObject]$Obj, [string[]]$Arr)"#; + let code = r#"param($Msg, [string]$Msg2, $Dflt = "default value, with comma", [int]$Nb = 3 , $Nb2 = 5.0, $Nb3 = 5, $Wahoo = $env:WAHOO, [PSCustomObject]$Obj, [string[]]$Arr, [Parameter(Mandatory)][ValidateSet('Green', 'Blue', 'Red')][string]$Message)"#; assert_eq!( parse_powershell_sig(code)?, MainArgSignature { @@ -344,6 +472,14 @@ non_required="${5:-}" default: None, has_default: false, oidx: None + }, + Arg { + otyp: None, + name: "Message".to_string(), + typ: Typ::Str(None), + default: None, + has_default: false, + oidx: None } ], no_main_func: None, @@ -353,6 +489,121 @@ non_required="${5:-}" Ok(()) } + #[test] + fn test_extract_powershell_param_block() { + // Basic cases + assert_eq!( + extract_powershell_param_block("param($Name, $Age)", true), + Some("param($Name, $Age)") + ); + assert_eq!( + extract_powershell_param_block("param($Name, $Age)", false), + Some("$Name, $Age") + ); + + // Case insensitive and whitespace + assert_eq!( + extract_powershell_param_block("PARAM ($Value)", false), + Some("$Value") + ); + + // Nested parentheses + assert_eq!( + extract_powershell_param_block("param([ValidateScript({$_ -gt 0})]$Count)", false), + Some("[ValidateScript({$_ -gt 0})]$Count") + ); + + // Strings with parentheses + assert_eq!( + extract_powershell_param_block("param([string]$Path = 'C:\\file(1).txt')", false), + Some("[string]$Path = 'C:\\file(1).txt'") + ); + assert_eq!( + extract_powershell_param_block(r#"param($Msg = "Hello (world)")"#, false), + Some(r#"$Msg = "Hello (world)""#) + ); + assert_eq!( + extract_powershell_param_block( + r#"param([Parameter(Mandatory)][ValidateSet('Green', 'Blue', 'Red')][string]$Message)"#, + false + ), + Some(r#"[Parameter(Mandatory)][ValidateSet('Green', 'Blue', 'Red')][string]$Message"#) + ); + + // Escaped quotes + assert_eq!( + extract_powershell_param_block("param($Text = 'don''t')", false), + Some("$Text = 'don''t'") + ); + assert_eq!( + extract_powershell_param_block(r#"param($Text = "He said `"Hi`"")"#, false), + Some(r#"$Text = "He said `"Hi`"""#) + ); + + // Multiline + let multiline = "param(\n [string]$Name,\n [int]$Age\n)"; + assert!(extract_powershell_param_block(multiline, false).is_some()); + + // Invalid cases + assert_eq!(extract_powershell_param_block("$x = 5", false), None); + assert_eq!(extract_powershell_param_block("param", false), None); + assert_eq!(extract_powershell_param_block("param($x", false), None); + + // Valid: param at beginning with single-line comments before + assert_eq!( + extract_powershell_param_block("# This is a comment\nparam($Name)", false), + Some("$Name") + ); + assert_eq!( + extract_powershell_param_block("# Comment 1\n# Comment 2\n\nparam($Name)", false), + Some("$Name") + ); + + // Valid: param at beginning with block comment before + assert_eq!( + extract_powershell_param_block("<# Block comment #>\nparam($Name)", false), + Some("$Name") + ); + assert_eq!( + extract_powershell_param_block( + "<#\n Multi-line\n block comment\n#>\nparam($Name)", + false + ), + Some("$Name") + ); + + // Valid: mixed comments and whitespace + assert_eq!( + extract_powershell_param_block( + "# Line comment\n<# Block comment #>\n\nparam($Name)", + false + ), + Some("$Name") + ); + + // Invalid: code before param + assert_eq!( + extract_powershell_param_block("$x = 5\nparam($Name)", false), + None + ); + assert_eq!( + extract_powershell_param_block("Write-Host 'test'\nparam($Name)", false), + None + ); + + // Invalid: unclosed block comment + assert_eq!( + extract_powershell_param_block("<# Unclosed comment\nparam($Name)", false), + None + ); + + // Invalid: unclosed block comment + assert_eq!( + extract_powershell_param_block("function test-x{ param($Name)\n}", false), + None + ); + } + #[test] fn test_parse_bash_sig_with_crlf() -> anyhow::Result<()> { // Test with CRLF line endings (Windows-style) diff --git a/backend/parsers/windmill-parser-php/src/lib.rs b/backend/parsers/windmill-parser-php/src/lib.rs index 1ac2b9a63a..14f30f940c 100644 --- a/backend/parsers/windmill-parser-php/src/lib.rs +++ b/backend/parsers/windmill-parser-php/src/lib.rs @@ -44,23 +44,41 @@ fn parse_default_expr(e: Expression) -> Option { pub fn parse_php_signature( code: &str, - override_main: Option, + override_entrypoint: Option, ) -> anyhow::Result { - let main_name = override_main.unwrap_or("main".to_string()); + let entrypoint_fn_name = override_entrypoint.unwrap_or("main".to_string()); let ast = parser::parse(code) .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?; - let params = ast.into_iter().find_map(|x| match x { - Statement::Function(FunctionStatement { - name, - parameters: FunctionParameterList { parameters, .. }, - .. - }) if name.to_string() == main_name => Some(parameters), - _ => None, - }); + let mut entrypoint_params = None; + let mut has_preprocessor = None; + for node in ast.into_iter() { + match node { + Statement::Function(FunctionStatement { + name, + parameters: FunctionParameterList { parameters, .. }, + .. + }) => { + let fn_name = name.to_string(); - if let Some(params) = params { + if has_preprocessor.is_none() && fn_name == "preprocessor" { + has_preprocessor = Some(true); + } + + if entrypoint_params.is_none() && fn_name == entrypoint_fn_name { + entrypoint_params = Some(parameters); + } + + if has_preprocessor.is_some() && entrypoint_params.is_some() { + break; + } + } + _ => {} + }; + } + + if let Some(params) = entrypoint_params { let args = params .into_iter() .map(|x| { @@ -82,7 +100,7 @@ pub fn parse_php_signature( star_kwargs: false, args, no_main_func: Some(false), - has_preprocessor: None, + has_preprocessor, }) } else { Ok(MainArgSignature { @@ -90,7 +108,7 @@ pub fn parse_php_signature( star_kwargs: false, args: vec![], no_main_func: Some(true), - has_preprocessor: None, + has_preprocessor, }) } } diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs index f3b7929d38..f6de0a7b05 100644 --- a/backend/parsers/windmill-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -134,8 +134,11 @@ pub fn parse_graphql(code: &str) -> String { #[cfg(feature = "php-parser")] #[wasm_bindgen] -pub fn parse_php(code: &str) -> String { - wrap_sig(windmill_parser_php::parse_php_signature(code, None)) +pub fn parse_php(code: &str, main_override: Option) -> String { + wrap_sig(windmill_parser_php::parse_php_signature( + code, + main_override, + )) } #[cfg(feature = "rust-parser")] diff --git a/backend/substitute_ee_code.sh b/backend/substitute_ee_code.sh index 3ea73e16cc..7cd9a0b25a 100755 --- a/backend/substitute_ee_code.sh +++ b/backend/substitute_ee_code.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash set -euo pipefail script_dirpath="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" root_dirpath="$(cd "${script_dirpath}/.." && pwd)" diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 6b2ef97bc4..e6f8062c3c 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.574.1 + version: 1.575.0 title: Windmill API contact: @@ -7994,6 +7994,12 @@ paths: schema: type: boolean - $ref: "#/components/parameters/NewJobId" + - name: memory_id + description: memory ID for chat-enabled flows + in: query + schema: + type: string + format: uuid requestBody: description: preview @@ -8020,6 +8026,12 @@ paths: - job parameters: - $ref: "#/components/parameters/WorkspaceId" + - name: memory_id + description: memory ID for chat-enabled flows + in: query + schema: + type: string + format: uuid requestBody: description: preview diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 1a0c013539..77cb73b8e9 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -6533,7 +6533,15 @@ async fn run_preview_flow_job( check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; let tx = PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()); - let (uuid, tx) = push( + let chat_input_enabled = raw_flow.value.chat_input_enabled.unwrap_or(false); + let flow_path = raw_flow.path.clone().unwrap_or_default(); + let user_message = raw_flow + .args + .as_ref() + .and_then(|args| args.get("user_message")) + .cloned(); + + let (uuid, mut tx) = push( &db, tx, &w_id, @@ -6568,6 +6576,25 @@ async fn run_preview_flow_job( None, ) .await?; + + // Set memory_id if provided (for agent memory) + if let Some(memory_id) = run_query.memory_id { + set_flow_memory_id(&mut tx, uuid, memory_id).await?; + } + + // Handle conversation messages for chat-enabled flows + if chat_input_enabled { + handle_chat_conversation_messages( + &mut tx, + &authed, + &w_id, + &flow_path, + &run_query, + user_message.as_ref(), + ) + .await?; + } + tx.commit().await?; Ok((StatusCode::CREATED, uuid.to_string())) diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 802f6f0353..cd2f963680 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -62,7 +62,10 @@ use tower_http::{ }; use windmill_common::db::UserDB; use windmill_common::worker::CLOUD_HOSTED; -use windmill_common::{utils::{configure_client, GIT_VERSION}, BASE_URL, INSTANCE_NAME}; +use windmill_common::{ + utils::{configure_client, GIT_VERSION}, + BASE_URL, INSTANCE_NAME, +}; use crate::scim_oss::has_scim_token; use windmill_common::error::AppError; @@ -180,6 +183,7 @@ pub mod workspaces_ee; mod workspaces_export; mod workspaces_extra; mod workspaces_oss; +mod public_app_layer; #[cfg(feature = "mcp")] mod mcp; @@ -699,6 +703,15 @@ pub async fn run_server( ) }; + let app = if let Some(domain) = public_app_layer::PUBLIC_APP_DOMAIN.as_ref() { + tracing::info!("Public app domain filter enabled for domain: {}", domain); + app.layer(axum::middleware::from_fn( + public_app_layer::public_app_domain_filter, + )) + } else { + app + }; + let app = app.layer(CatchPanicLayer::custom(|err| { tracing::error!("panic in handler, returning 500: {:?}", err); Response::builder() diff --git a/backend/windmill-api/src/public_app_layer.rs b/backend/windmill-api/src/public_app_layer.rs new file mode 100644 index 0000000000..7d5e5a6e3b --- /dev/null +++ b/backend/windmill-api/src/public_app_layer.rs @@ -0,0 +1,229 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use axum::body::Body; +use axum::response::Response; +use http::StatusCode; + +lazy_static::lazy_static! { + pub static ref PUBLIC_APP_DOMAIN: Option = std::env::var("PUBLIC_APP_DOMAIN").ok(); +} + +/// Middleware to restrict public app domain to whitelisted routes +pub async fn public_app_domain_filter( + req: axum::extract::Request, + next: axum::middleware::Next, +) -> Response { + if let Some(public_domain) = PUBLIC_APP_DOMAIN.as_ref() { + if let Some(host) = req.headers().get(http::header::HOST) { + if let Ok(host_str) = host.to_str() { + // Extract hostname without port + let hostname = host_str.split(':').next().unwrap_or(host_str); + + if hostname == public_domain { + let path = req.uri().path(); + + // Check if route is whitelisted + let is_whitelisted = is_public_route_whitelisted(path); + + if !is_whitelisted { + tracing::warn!( + "Rejected request to {} on public app domain {}", + path, + public_domain + ); + return Response::builder() + .status(StatusCode::FORBIDDEN) + .body(Body::from( + "Access forbidden: route not allowed on public app domain", + )) + .unwrap(); + } + } + } + } + } + next.run(req).await +} + +fn is_public_route_whitelisted(path: &str) -> bool { + // Whitelisted route patterns for public app domain + let whitelist = [ + // Public app routes + "/api/ee_license", + "/api/w/*/users/whoami", + "/api/w/*/apps_u/*", + "/api/w/*/resources/list", + "/api/w/*/jobs_u/getupdate_sse/*", + "/api/auth/login", + "/api/w/*/folders/listnames", + "/api/w/*/resources/exists/*", + "/api/w/*/resources/type/get/*", + "/api/w/*/resources/type/listnames", + "/api/oauth/login/*", + "/api/oauth/connect/*", + "/oauth/callback/*", + "/user/login_callback/*", + "/api/workspaces/users", + "/api/users/whoami", + "/api/oauth/list_connects", + "/api/oauth/list_logins", + "/public/*", + "/a/*", + "/Inter-Variable.woff2", + ]; + + for pattern in &whitelist { + if matches_pattern(path, pattern) { + return true; + } + } + + false +} + +fn matches_pattern(path: &str, pattern: &str) -> bool { + // Handle exact match + if path == pattern { + return true; + } + + // Handle wildcard patterns + if !pattern.contains('*') { + return false; + } + + let parts: Vec<&str> = pattern.split('*').collect(); + + // Start from the beginning - path must start with first part + if !path.starts_with(parts[0]) { + return false; + } + + let mut remaining = &path[parts[0].len()..]; + + // Check each part in between wildcards + for i in 1..parts.len() - 1 { + let part = parts[i]; + if let Some(pos) = remaining.find(part) { + remaining = &remaining[pos + part.len()..]; + } else { + return false; + } + } + + // Check the last part + let last = parts[parts.len() - 1]; + if last.is_empty() { + // Wildcard at the end, already matched + return true; + } else { + // Must end with the last part + return remaining.ends_with(last); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_matches_pattern_exact() { + assert!(matches_pattern("/api/version", "/api/version")); + assert!(!matches_pattern("/api/versions", "/api/version")); + } + + #[test] + fn test_matches_pattern_wildcard_middle() { + assert!(matches_pattern( + "/api/w/my-workspace/apps_u/my-app", + "/api/w/*/apps_u/*" + )); + assert!(matches_pattern( + "/api/w/workspace123/jobs_u/job456", + "/api/w/*/jobs_u/*" + )); + assert!(!matches_pattern( + "/api/w/workspace/apps/my-app", + "/api/w/*/apps_u/*" + )); + } + + #[test] + fn test_matches_pattern_wildcard_end() { + assert!(matches_pattern("/api/r/some/path", "/api/r/*")); + assert!(matches_pattern("/api/auth/login", "/api/auth/*")); + assert!(matches_pattern("/public/asset.css", "/public/*")); + assert!(!matches_pattern("/api/admin/users", "/api/r/*")); + } + + #[test] + fn test_is_public_route_whitelisted() { + // Whitelisted routes based on current whitelist + assert!(is_public_route_whitelisted("/api/ee_license")); + assert!(is_public_route_whitelisted( + "/api/w/my-workspace/users/whoami" + )); + assert!(is_public_route_whitelisted( + "/api/w/my-workspace/apps_u/my-app" + )); + assert!(is_public_route_whitelisted( + "/api/w/workspace/resources/list" + )); + + // Non-whitelisted routes + assert!(!is_public_route_whitelisted("/api/version")); + assert!(!is_public_route_whitelisted("/api/uptodate")); + assert!(!is_public_route_whitelisted("/api/w/workspace/scripts")); + assert!(!is_public_route_whitelisted("/api/w/workspace/flows")); + assert!(!is_public_route_whitelisted( + "/api/w/workspace/apps/private" + )); + assert!(!is_public_route_whitelisted("/api/admin/settings")); + assert!(!is_public_route_whitelisted("/api/users")); + } + + #[test] + fn test_matches_pattern_complex_multiple_wildcards() { + // Test pattern with two wildcards like "/api/w/*/foo/*" + assert!(matches_pattern( + "/api/w/workspace1/s3_proxy/file.txt", + "/api/w/*/s3_proxy/*" + )); + assert!(matches_pattern( + "/api/w/my-workspace/s3_proxy/path/to/file.jpg", + "/api/w/*/s3_proxy/*" + )); + assert!(matches_pattern( + "/api/w/test/apps_u/myapp", + "/api/w/*/apps_u/*" + )); + + // Should not match if middle part is different + assert!(!matches_pattern( + "/api/w/workspace1/other/file.txt", + "/api/w/*/s3_proxy/*" + )); + assert!(!matches_pattern( + "/api/w/workspace/apps/myapp", + "/api/w/*/apps_u/*" + )); + + // Edge cases + assert!(matches_pattern( + "/api/w/ws/s3_proxy/a", + "/api/w/*/s3_proxy/*" + )); + + // Path must have content after the last wildcard segment + assert!(matches_pattern( + "/api/w/ws/s3_proxy/", + "/api/w/*/s3_proxy/*" + )); + } +} diff --git a/backend/windmill-common/src/cache.rs b/backend/windmill-common/src/cache.rs index 3d1524916a..c7972d3a7e 100644 --- a/backend/windmill-common/src/cache.rs +++ b/backend/windmill-common/src/cache.rs @@ -433,9 +433,9 @@ pub mod flow { make_static! { /// Flow node cache. /// FIXME: Use `Arc` for cheap cloning. - static ref NODES: { FlowNodeId => RawData } in "flow" <= 1000; + static ref NODES: { FlowNodeId => RawData } in "flow_2" <= 1000; /// Flow version value cache (version id => value). - static ref FLOWS: { i64 => Entry } in "flows" <= 1000; + static ref FLOWS: { i64 => Entry } in "flows_2" <= 1000; /// Flow version lite value cache (version id => value). static ref FLOWS_LITE: { i64 => Entry } in "flowslite" <= 1000; } @@ -567,7 +567,7 @@ pub mod script { make_static! { /// Scripts cache. /// FIXME: Use `Arc` for cheap cloning. - static ref CACHE: { ScriptHash => ScriptFull } in "script" <= 1000; + static ref CACHE: { ScriptHash => ScriptFull } in "script_2" <= 1000; } /// Clear the script cache. @@ -670,7 +670,7 @@ pub mod app { make_static! { /// App scripts cache. - static ref CACHE: { AppScriptId => Entry } in "app" <= 1000; + static ref CACHE: { AppScriptId => Entry } in "app_2" <= 1000; } /// Clear the app scripts cache. diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 29d5488be8..0b0055ed0d 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -7,7 +7,7 @@ use regex::Regex; use reqwest_middleware::ClientWithMiddleware; use semver::Version; use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use serde_json::value::RawValue; +use serde_json::{json, value::RawValue}; use sqlx::{types::Json, Pool, Postgres}; use std::{ cmp::Reverse, @@ -663,13 +663,11 @@ fn parse_file(path: &str) -> Option { .flatten() } -#[derive(Copy, Clone)] #[annotations("#")] pub struct RubyAnnotations { pub verbose: bool, } -#[derive(Copy, Clone)] #[annotations("#")] pub struct PythonAnnotations { pub no_cache: bool, @@ -682,7 +680,6 @@ pub struct PythonAnnotations { pub py313: bool, } -#[derive(Copy, Clone)] #[annotations("//")] pub struct GoAnnotations { pub go1_22_compat: bool, @@ -698,13 +695,174 @@ pub struct TypeScriptAnnotations { #[annotations("--")] pub struct SqlAnnotations { - pub return_last_result: bool, + pub return_last_result: bool, // deprecated, use result_collection instead + pub result_collection: SqlResultCollectionStrategy, } #[annotations("#")] pub struct BashAnnotations { pub docker: bool, } + +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum SqlResultCollectionStrategy { + LastStatementAllRows, + LastStatementFirstRow, + LastStatementAllRowsScalar, + LastStatementFirstRowScalar, + AllStatementsAllRows, + AllStatementsFirstRow, + AllStatementsAllRowsScalar, + AllStatementsFirstRowScalar, + Legacy, +} + +impl SqlResultCollectionStrategy { + pub fn parse(s: &str) -> Self { + use SqlResultCollectionStrategy::*; + match s { + "last_statement_all_rows" => LastStatementAllRows, + "last_statement_first_row" => LastStatementFirstRow, + "last_statement_all_rows_scalar" => LastStatementAllRowsScalar, + "last_statement_first_row_scalar" => LastStatementFirstRowScalar, + "all_statements_all_rows" => AllStatementsAllRows, + "all_statements_first_row" => AllStatementsFirstRow, + "all_statements_all_rows_scalar" => AllStatementsAllRowsScalar, + "all_statements_first_row_scalar" => AllStatementsFirstRowScalar, + "legacy" => Legacy, + _ => SqlResultCollectionStrategy::default(), + } + } + + pub fn collect_last_statement_only(&self, query_count: usize) -> bool { + use SqlResultCollectionStrategy::*; + match self { + LastStatementAllRows + | LastStatementFirstRow + | LastStatementFirstRowScalar + | LastStatementAllRowsScalar => true, + Legacy => query_count == 1, + _ => false, + } + } + pub fn collect_first_row_only(&self) -> bool { + use SqlResultCollectionStrategy::*; + match self { + LastStatementFirstRow + | LastStatementFirstRowScalar + | AllStatementsFirstRow + | AllStatementsFirstRowScalar => true, + _ => false, + } + } + pub fn collect_scalar(&self) -> bool { + use SqlResultCollectionStrategy::*; + match self { + LastStatementFirstRowScalar + | AllStatementsFirstRowScalar + | LastStatementAllRowsScalar + | AllStatementsAllRowsScalar => true, + _ => false, + } + } + + // This function transforms the shape (e.g Row[][] -> Row) + // It is the responsibility of the executor to avoid fetching unnecessary statements/rows + pub fn collect( + &self, + values: Vec>>, + ) -> error::Result> { + let null = || serde_json::value::RawValue::from_string("null".to_string()).unwrap(); + + let values = if self.collect_last_statement_only(values.len()) { + values.into_iter().rev().take(1).collect() + } else { + values + }; + + let values = if self.collect_first_row_only() { + values + .into_iter() + .map(|rows| rows.into_iter().take(1).collect()) + .collect() + } else { + values + }; + + let values = if self.collect_scalar() { + values + .into_iter() + .map(|rows| { + rows.into_iter() + .map(|row| { + // Take the first value in the object + let record = + match serde_json::from_str(row.get()) { + Ok(serde_json::Value::Object(record)) => record, + Ok(_) => return Err(error::Error::ExecutionErr( + "Could not collect sql scalar value from non-object row" + .to_string(), + )), + Err(e) => { + return Err(error::Error::ExecutionErr(format!( + "Could not collect sql scalar value (failed to parse row): {}", + e + ))) + } + }; + let Some((_, value)) = record.iter().next() else { + return Err(error::Error::ExecutionErr( + "Could not collect sql scalar value from empty row".to_string(), + )); + }; + Ok(serde_json::value::RawValue::from_string( + serde_json::to_string(value).map_err(to_anyhow)?, + ) + .map_err(to_anyhow)?) + }) + .collect::>>() + }) + .collect::>>()? + } else { + values + }; + + match ( + self.collect_last_statement_only(values.len()), + self.collect_first_row_only(), + ) { + (true, true) => { + match values + .into_iter() + .last() + .map(|rows| rows.into_iter().next()) + { + Some(Some(row)) => Ok(row.clone()), + _ => Ok(null()), + } + } + (true, false) => match values.into_iter().last() { + Some(rows) => Ok(to_raw_value(&rows)), + None => Ok(null()), + }, + (false, true) => { + let values = values + .into_iter() + .map(|rows| rows.into_iter().next().unwrap_or_else(null)) + .collect::>(); + Ok(to_raw_value(&values)) + } + (false, false) => Ok(to_raw_value(&values)), + } + } +} + +impl Default for SqlResultCollectionStrategy { + fn default() -> Self { + SqlResultCollectionStrategy::LastStatementAllRows + } +} + /// length = 5 /// value = "foo" /// output = "foo " @@ -1701,6 +1859,13 @@ pub fn load_env_vars( .collect() } +pub fn error_to_value(err: &error::Error) -> serde_json::Value { + match err { + error::Error::JsonErr(err) => err.clone(), + _ => json!({"message": err.to_string(), "name": err.name()}), + } +} + #[derive(Clone, PartialEq, Debug)] pub struct WorkspacedPath { pub workspace_id: String, diff --git a/backend/windmill-duckdb-ffi-internal/README_DEV.md b/backend/windmill-duckdb-ffi-internal/README_DEV.md index 97dd8a5eb0..0c34ebc069 100644 --- a/backend/windmill-duckdb-ffi-internal/README_DEV.md +++ b/backend/windmill-duckdb-ffi-internal/README_DEV.md @@ -12,4 +12,4 @@ INSERT INTO t VALUES (NULL); causes `Constraint Error: NOT NULL constraint failed: t.x` normally, but here we see `Unknown exception in ExecutorTask::Execute`. This opaque errors comes directly from the C++ DuckDB library : https://github.com/duckdb/duckdb/blob/f99fed1e0b16a842573f9dad529f6c170a004f6e/src/parallel/executor_task.cpp#L58 -To solve this, we compile duckdb separately from the main backend crate and call it with FFI +To solve this, we compile duckdb separately from the main backend crate and call it with FFI. It has to be loaded dynamically, if it is loaded statically it will still share lib c++ with deno_core and cause issues. diff --git a/backend/windmill-duckdb-ffi-internal/src/lib.rs b/backend/windmill-duckdb-ffi-internal/src/lib.rs index 29112baff9..501b5c5dee 100644 --- a/backend/windmill-duckdb-ffi-internal/src/lib.rs +++ b/backend/windmill-duckdb-ffi-internal/src/lib.rs @@ -1,6 +1,6 @@ use std::{ collections::HashMap, - ffi::{CStr, CString, c_char}, + ffi::{CStr, CString, c_char, c_uint}, ptr::null_mut, }; @@ -27,6 +27,14 @@ pub extern "C" fn free_cstr(string: *mut c_char) -> () { } } +#[unsafe(no_mangle)] +pub extern "C" fn get_version() -> c_uint { + // Increment when making breaking changes to the FFI interface. + // The windmill worker will check that the version matches or else refuse to call + // the FFI functions to avoid undefined behavior. + return 1; +} + #[unsafe(no_mangle)] pub extern "C" fn run_duckdb_ffi( query_block_list: *const *const c_char, @@ -36,6 +44,8 @@ pub extern "C" fn run_duckdb_ffi( base_internal_url: *const c_char, w_id: *const c_char, column_order_ptr: *mut *mut c_char, + collect_last_only: bool, + collect_first_row_only: bool, ) -> *mut c_char { let (r, column_order) = match convert_args( query_block_list, @@ -54,6 +64,8 @@ pub extern "C" fn run_duckdb_ffi( token, base_internal_url, w_id, + collect_last_only, + collect_first_row_only, ) }, ) { @@ -153,6 +165,8 @@ fn run_duckdb_internal<'a>( token: &str, base_internal_url: &str, w_id: &str, + collect_last_only: bool, + collect_first_row_only: bool, ) -> Result<(String, Option>), String> { let conn = duckdb::Connection::open_in_memory().map_err(|e| e.to_string())?; @@ -189,23 +203,23 @@ fn run_duckdb_internal<'a>( )) .map_err(|e| format!("Error setting up S3 secret: {}", e.to_string()))?; - let mut result: Option> = None; + let mut results: Vec>> = vec![]; let mut column_order = None; for (query_block_index, query_block) in query_block_list.enumerate() { - result = Some( - do_duckdb_inner( - &conn, - query_block, - &job_args, - query_block_index != query_block_list_count - 1, - &mut column_order, - ) - .map_err(|e| e.to_string())?, - ); + let result = do_duckdb_inner( + &conn, + query_block, + &job_args, + collect_last_only && query_block_index != query_block_list_count - 1, + collect_first_row_only, + &mut column_order, + ) + .map_err(|e| e.to_string())?; + results.push(result); } - let result = result.unwrap_or_else(|| RawValue::from_string("[]".to_string()).unwrap()); - Ok((result.get().to_string(), column_order)) + let results = serde_json::value::to_raw_value(&results).map_err(|e| e.to_string())?; + Ok((results.get().to_string(), column_order)) } fn do_duckdb_inner( @@ -213,8 +227,9 @@ fn do_duckdb_inner( query: &str, job_args: &HashMap, skip_collect: bool, + collect_first_row_only: bool, column_order: &mut Option>, -) -> Result, String> { +) -> Result>, String> { let mut rows_vec = vec![]; let (query, job_args) = interpolate_named_args(query, &job_args); @@ -226,7 +241,7 @@ fn do_duckdb_inner( .map_err(|e| e.to_string())?; if skip_collect { - return Ok(RawValue::from_string("[]".to_string()).unwrap()); + return Ok(vec![]); } // Statement needs to be stepped at least once or stmt.column_names() will panic let mut column_names = None; @@ -266,11 +281,14 @@ fn do_duckdb_inner( return Err(e.to_string()); } } + if collect_first_row_only { + break; + } } *column_order = column_names; - serde_json::value::to_raw_value(&rows_vec).map_err(|e| e.to_string()) + Ok(rows_vec) } // duckdb-rs does not support named parameters, diff --git a/backend/windmill-macros/src/lib.rs b/backend/windmill-macros/src/lib.rs index 25ff8cbaa2..778ac72b0f 100644 --- a/backend/windmill-macros/src/lib.rs +++ b/backend/windmill-macros/src/lib.rs @@ -1,93 +1,83 @@ use proc_macro::TokenStream; use quote::quote; -use syn::{parse_macro_input, Ident, ItemStruct, Lit}; +use syn::{parse_macro_input, ItemStruct, Lit, Type}; + +fn is_bool_type(ty: &Type) -> bool { + if let Type::Path(type_path) = ty { + if let Some(segment) = type_path.path.segments.last() { + return segment.ident == "bool"; + } + } + false +} #[proc_macro_attribute] pub fn annotations(attr: TokenStream, item: TokenStream) -> TokenStream { let input = parse_macro_input!(item as ItemStruct); let name = input.ident.clone(); - let fields = input - .fields - .iter() - .map(|f| f.ident.clone().unwrap()) - .collect::>(); - // Match on the literal to extract the string value + // Separate fields by type + let mut bool_fields = Vec::new(); + let mut custom_fields = Vec::new(); + + for field in input.fields.iter() { + let field_name = field.ident.clone().unwrap(); + let field_type = &field.ty; + + if is_bool_type(field_type) { + bool_fields.push(field_name); + } else { + custom_fields.push((field_name, field_type)); + } + } + let (custom_field_names, custom_field_types): (Vec<_>, Vec<_>) = + custom_fields.into_iter().unzip(); + + // Parse comment literal let comm_lit = match parse_macro_input!(attr as Lit) { - Lit::Str(lit_str) => lit_str.value(), // This will give "#" without quotes + Lit::Str(lit_str) => lit_str.value(), _ => panic!("Expected a string literal"), }; - // Generate regex - let mut reg = format!("^{}|", &comm_lit); - { - for field in fields.iter() { - reg.push_str(&(field.to_string())); - reg.push_str("\\b"); - } - - reg.push_str(r#"|\w+"#); - } - // Example of generated regex: - // ^# - // |ann1\b|ann2\b|ann3\b|ann4\b - // |\w+ - TokenStream::from(quote! { - #[derive(Default, Debug)] + #[derive(Default, Debug, Copy, Clone)] #input - impl std::ops::BitOrAssign for #name{ - fn bitor_assign(&mut self, rhs: Self) { - // Unfold fields - // Read more: https://docs.rs/quote/latest/quote/macro.quote.html#interpolation - #( self.#fields |= rhs.#fields; )* - } - } - impl #name { /// Autogenerated by windmill-macros - pub fn parse(inner_content: &str) -> Self{ + pub fn parse(code: &str) -> Self { let mut res = Self::default(); - lazy_static::lazy_static! { - static ref RE: regex::Regex = regex::Regex::new(#reg).unwrap(); - } - // Create lines stream - let mut lines = inner_content.lines(); - 'outer: while let Some(line) = lines.next() { - // If comment sign(s) on the right place - let mut comms = false; - // New instance - // We will apply it if in line only annotations - let mut new = Self::default(); - 'inner: for (i, mat) in RE.find_iter(line).enumerate() { + let mut lines = code.lines(); - match mat.as_str(){ - #comm_lit if i == 0 => { - comms = true; - continue 'inner; - }, - - // Will expand into something like: - // "ann1" => new.ann1 = true, - // "ann2" => new.ann2 = true, - // "ann3" => new.ann3 = true, - #( stringify!(#fields) => new.#fields = true, )* - // Non annotations - _ => continue 'outer, - }; + while let Some(line) = lines.next() { + if !line.starts_with(#comm_lit) { + break; } + let line = line[#comm_lit.len()..].trim(); + let (key, value) = line.split_once('=').unwrap_or((line, "")); - if !comms { - // We dont want to continue if line does not start with # - return res; + + match key { + #( + stringify!(#custom_field_names) => { + if value.is_empty() { + continue; + } + res.#custom_field_names = #custom_field_types::parse(value); + } + )* + #( + stringify!(#bool_fields) => { + res.#bool_fields = true; + } + )* + _ => { + // Unknown key=value annotation + continue; + } } - - // Apply changes - res |= new; } - res } } diff --git a/backend/windmill-macros/tests/annotations.rs b/backend/windmill-macros/tests/annotations.rs index fd430dfaeb..d906b72887 100644 --- a/backend/windmill-macros/tests/annotations.rs +++ b/backend/windmill-macros/tests/annotations.rs @@ -24,7 +24,7 @@ mod annotations_tests { } #[annotations("#")] - #[derive(Eq, PartialEq, Copy, Clone)] + #[derive(Eq, PartialEq)] pub struct Annotations { pub ann1: bool, pub ann2: bool, @@ -34,7 +34,7 @@ mod annotations_tests { } #[annotations("//")] - #[derive(Eq, PartialEq, Copy, Clone)] + #[derive(Eq, PartialEq)] pub struct SlashedAnnotations { pub ann1: bool, pub ann2: bool, @@ -43,7 +43,7 @@ mod annotations_tests { } #[annotations("--")] - #[derive(Eq, PartialEq, Copy, Clone)] + #[derive(Eq, PartialEq)] pub struct MinusedAnnotations { pub ann1: bool, pub ann2: bool, diff --git a/backend/windmill-queue/Cargo.toml b/backend/windmill-queue/Cargo.toml index 62b8b5962b..6349aee84e 100644 --- a/backend/windmill-queue/Cargo.toml +++ b/backend/windmill-queue/Cargo.toml @@ -46,3 +46,4 @@ serde_urlencoded.workspace = true regex.workspace = true backon.workspace = true quick_cache.workspace = true +thiserror.workspace = true diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 2b20db4129..2b2bd59829 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -2460,12 +2460,18 @@ pub struct PulledJobResult { pub job: Option, pub suspended: bool, pub missing_concurrency_key: bool, + pub error_while_preprocessing: Option } +#[derive(thiserror::Error, Debug)] pub enum PulledJobResultToJobErr { + #[error("missing concurrency key")] MissingConcurrencyKey(JobCompleted), + #[error("pulled job preprocessor error: {}", .0.result)] + ErrorWhilePreprocessing(JobCompleted), } + impl PulledJobResult { pub fn to_pulled_job(self) -> Result, PulledJobResultToJobErr> { match self { @@ -2488,9 +2494,278 @@ impl PulledJobResult { from_cache: None, }), ), + PulledJobResult { job: Some(job), error_while_preprocessing: Some(e), .. } => Err( + PulledJobResultToJobErr::ErrorWhilePreprocessing(JobCompleted { + preprocessed_args: None, + job: MiniCompletedJob::from(job.job), + success: false, + result: Arc::new(windmill_common::worker::to_raw_value(&json!({ + "name": "Pulled job preprocessing error", + "message": e + }))), + result_columns: None, + mem_peak: 0, + cached_res_path: None, + token: "".to_string(), + canceled_by: None, + duration: None, + has_stream: Some(false), + from_cache: None, + }), + ), PulledJobResult { job, .. } => Ok(job), } } + + /// Generic preprocess function + /// Can be used for any kind of preprocessing + pub async fn preprocess(&mut self, db: &DB) -> error::Result<()> { + let PulledJobResult { job: Some(ref mut pulled_job), .. } = self else { + return Ok(()); + }; + + let kind = pulled_job.kind; + // Handle dependency job debouncing cleanup when a job is pulled for execution + if kind.is_dependency() + && pulled_job + .args + .as_ref() + .map(|x| x.get("triggered_by_relative_import").is_some()) + .unwrap_or_default() + && !*WMDEBUG_NO_DJOB_DEBOUNCING + { + return Box::pin(async move { + // Only used for testing in tests/relative_imports.rs + // Give us some space to work with. + #[cfg(debug_assertions)] + if let Some(duration) = pulled_job + .args + .as_ref() + .map(|x| { + x.get("dbg_sleep_between_pull_and_debounce_key_removal") + .map(|v| serde_json::from_str::(v.get()).ok()) + .flatten() + }) + .flatten() + { + tracing::debug!("going to sleep",); + sleep(std::time::Duration::from_secs(duration as u64)).await; + } + + tracing::debug!( + "Processing debounce cleanup for dependency job {} at path {:?}", + &pulled_job.id, + &pulled_job.runnable_path + ); + + let key = format!("{}:{}:dependency", &pulled_job.workspace_id, pulled_job.runnable_path()); + let mut tx = db.begin().await?; + + // === DEBOUNCE CLEANUP === + // + // Clean up the debounce_key entry for this job (if it exists). + // + // IMPORTANT: We delete by key (not job_id) to avoid race conditions: + // If pusher has locked this row then this call will be blocked until all txs are commited. + // + // The idea is that the worker_lockfiles::trigger_dependents_to_recompute_locks will fetch the latest version of the obj. + // This object needs to be created before the djob is executed and it happens right here. + // + // This way the next pusher can fetch the latest version of object and base their djob payload on newest version. + // The concurrency limit on djobs will make sure that by the time next djob is started executing the base version it is referencing + // has already calculated all locks. This way even next djob will always use the fully finalized version of object. + // + // + // + // Note: We don't use a transaction here for performance (it's called during job pull). + // This means there's a tiny window where the job is running but key isn't deleted yet, + // which is acceptable because new requests will just accumulate data to this job. + tracing::debug!( + job_id = %pulled_job.id, + "Cleaning up debounce_key entry for completed/pulled job" + ); + + // This will either: + // 1. Block until pusher pushed. Which gives us: + // - If there was any stale data in pusher, then we will read it here (couple of lines below) + // 2. Block pusher until we are done here. This gives us: + // - We will clone objects and retrieve the latest version. So when we are done the pusher can read latest version. + sqlx::query!("DELETE FROM debounce_key WHERE key = $1", &key) + .execute(&mut *tx) + .await + .map_err(|e| { + tracing::error!( + error = %e, + job_id = %pulled_job.id, + "Failed to delete debounce_key" + ); + e + })?; + + let Some(base_hash) = pulled_job.runnable_id else { + return Err(Error::InternalErr( + "Missing runnable_id for dependency job triggered by relative import" + .to_string(), + )); + }; + + tracing::debug!( + job_id = %pulled_job.id, + base_hash = %base_hash, + job_kind = ?kind, + "Creating new version for dependency job triggered by relative import" + ); + + let new_id = match kind { + JobKind::Dependencies => { + let deployment_message = pulled_job + .args + .clone() + .map(|hashmap| { + hashmap + .get("deployment_message") + .map(|map_value| { + serde_json::from_str::(map_value.get()).ok() + }) + .flatten() + }) + .flatten(); + + // This way we tell downstream which script we should archive when the resolution is finished. + // (not used at the moment) + pulled_job.args.as_mut().map(|args| { + args.insert("base_hash".to_owned(), to_raw_value(&*base_hash)) + }); + + let cloned_script = windmill_common::scripts::clone_script( + base_hash, + &pulled_job.workspace_id, + deployment_message, + &mut tx, + ) + .await?; + + if is_generated_from_raw_requirements(&Some(cloned_script.old_script.language), &cloned_script.old_script.lock.map(|v| v.to_string())) { + return Err(Error::BadRequest(format!( + "Script at path {} is generated from raw requirements, not overriding", + pulled_job.runnable_path() + ))); + } + + cloned_script.new_hash + } + JobKind::FlowDependencies => { + sqlx::query_scalar!( + "INSERT INTO flow_version + (workspace_id, path, value, schema, created_by) + + SELECT workspace_id, path, value, schema, created_by + FROM flow_version WHERE path = $1 AND workspace_id = $2 AND id = $3 + + RETURNING id + ", + pulled_job.runnable_path(), + pulled_job.workspace_id, + *base_hash, + ) + .fetch_one(&mut *tx) + .await? + } + JobKind::AppDependencies => { + sqlx::query_scalar!( + "INSERT INTO app_version + (app_id, value, created_by, raw_app) + SELECT app_id, value, created_by, raw_app + FROM app_version WHERE id = $1 + RETURNING id", + *base_hash + ) + .fetch_one(&mut *tx) + .await? + } + _ => { + return Err(Error::InternalErr(format!( + "Matched unexpected JobKind ({:?}). This is a bug!", + kind + ))) + } + }; + + pulled_job.runnable_id.replace(new_id.into()); + + if *windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING.read().await { + // === RETRIEVE ACCUMULATED DEBOUNCE DATA === + // + // For flows and apps, retrieve all nodes/components that were accumulated + // during the debounce window. This data comes from requests that were merged + // into this job instead of creating their own jobs. + // + // Scripts don't need this because they don't have nodes/components to relock. + if let Some(to_relock_field) = match &pulled_job.kind { + JobKind::FlowDependencies => Some("nodes_to_relock"), + JobKind::AppDependencies => Some("components_to_relock"), + _ => None, // Scripts don't use accumulated stale data + } { + tracing::debug!( + job_id = %pulled_job.id, + job_kind = ?pulled_job.kind, + field = %to_relock_field, + "Retrieving accumulated stale data from debounced requests" + ); + + if let Some(stale_data) = sqlx::query_scalar!( + "DELETE FROM debounce_stale_data WHERE job_id = $1 RETURNING to_relock", + &pulled_job.id + ) + .fetch_optional(&mut *tx) + .await + .map_err(|e| { + tracing::error!( + error = %e, + job_id = %pulled_job.id, + "Failed to retrieve debounce_stale_data" + ); + e + })? + .flatten() + { + tracing::debug!( + job_id = %pulled_job.id, + node_count = stale_data.len(), + nodes = ?stale_data, + "Retrieved accumulated nodes/components from {} debounced requests", + stale_data.len() + ); + + // Replace the job's relock list with the accumulated data + // This ensures all nodes from all debounced requests are processed + if let Some(args) = pulled_job.args.as_mut() { + args.insert(to_relock_field.to_owned(), to_raw_value(&stale_data)); + tracing::debug!( + field = %to_relock_field, + "Updated job args with accumulated debounce data" + ); + } + } else { + tracing::trace!( + job_id = %pulled_job.id, + "No accumulated stale data found (no debounced requests or already cleaned up)" + ); + } + } + } else { + tracing::warn!("Debouncing is not supported on this version of Windmill. Minimum version required for debouncing support."); + + } + // This will unblock pusher. + tx.commit().await?; + + Ok(()) + }).await; + } + + Ok(()) + } } /// Pull the job from queue @@ -2516,6 +2791,7 @@ pub async fn pull( job: None, suspended: false, missing_concurrency_key: false, + error_while_preprocessing: None, }); } @@ -2571,20 +2847,21 @@ pub async fn pull( // Concurrency limit is available for either enterprise job or dependency job && (cfg!(feature = "enterprise") || (job.is_dependency() && !*WMDEBUG_NO_DJOB_DEBOUNCING)) => { - let job = crate::jobs_ee::apply_concurrency_limit( + crate::jobs_ee::apply_concurrency_limit( db, pull_loop_count, suspended, job, ) - .await?; - job.unwrap_or(PulledJobResult { + .await? + .unwrap_or(PulledJobResult { job: None, suspended, missing_concurrency_key: false, + error_while_preprocessing: None, }) } - _ => PulledJobResult { job, suspended, missing_concurrency_key: false }, + _ => PulledJobResult { job, suspended, missing_concurrency_key: false, error_while_preprocessing: None }, }; Ok::<_, Error>(pulled_job_result) @@ -2602,7 +2879,7 @@ pub async fn pull( ) .await?; let Some(job) = job else { - return Ok(PulledJobResult { job: None, suspended, missing_concurrency_key: false }); + return Ok(PulledJobResult { job: None, suspended, missing_concurrency_key: false, error_while_preprocessing: None }); }; let has_concurent_limit = job.concurrent_limit.is_some(); @@ -2630,6 +2907,7 @@ pub async fn pull( job: Some(pulled_job), suspended, missing_concurrency_key: false, + error_while_preprocessing: None, }); } @@ -2637,11 +2915,11 @@ pub async fn pull( if cfg!(feature = "enterprise") || (pulled_job.is_dependency() && !*WMDEBUG_NO_DJOB_DEBOUNCING) { - if let Some(pulled_job) = + if let Some(pulled_job_res) = crate::jobs_ee::apply_concurrency_limit(db, pull_loop_count, suspended, pulled_job) .await? { - return Ok(pulled_job); + return Ok(pulled_job_res); } } } @@ -3693,7 +3971,7 @@ pub async fn push<'c, 'd>( } let in_queue = sqlx::query_scalar!( - "SELECT COUNT(id) FROM v2_job WHERE permissioned_as_email = $1", + "SELECT COUNT(j.id) FROM v2_job_queue q JOIN v2_job j USING (id) WHERE j.permissioned_as_email = $1", email ) .fetch_one(_db) @@ -5553,245 +5831,3 @@ pub async fn get_same_worker_job( )) }) } - -pub async fn preprocess_dependency_job(job: &mut PulledJob, db: &DB) -> error::Result<()> { - let kind = job.kind; - // Handle dependency job debouncing cleanup when a job is pulled for execution - if kind.is_dependency() - && job - .args - .as_ref() - .map(|x| x.get("triggered_by_relative_import").is_some()) - .unwrap_or_default() - && !*WMDEBUG_NO_DJOB_DEBOUNCING - { - return Box::pin(async move { - // Only used for testing in tests/relative_imports.rs - // Give us some space to work with. - #[cfg(debug_assertions)] - if let Some(duration) = job - .args - .as_ref() - .map(|x| { - x.get("dbg_sleep_between_pull_and_debounce_key_removal") - .map(|v| serde_json::from_str::(v.get()).ok()) - .flatten() - }) - .flatten() - { - tracing::debug!("going to sleep",); - sleep(std::time::Duration::from_secs(duration as u64)).await; - } - - tracing::debug!( - "Processing debounce cleanup for dependency job {} at path {:?}", - &job.id, - &job.runnable_path - ); - - let key = format!("{}:{}:dependency", &job.workspace_id, job.runnable_path()); - let mut tx = db.begin().await?; - - // === DEBOUNCE CLEANUP === - // - // Clean up the debounce_key entry for this job (if it exists). - // - // IMPORTANT: We delete by key (not job_id) to avoid race conditions: - // If pusher has locked this row then this call will be blocked until all txs are commited. - // - // The idea is that the worker_lockfiles::trigger_dependents_to_recompute_locks will fetch the latest version of the obj. - // This object needs to be created before the djob is executed and it happens right here. - // - // This way the next pusher can fetch the latest version of object and base their djob payload on newest version. - // The concurrency limit on djobs will make sure that by the time next djob is started executing the base version it is referencing - // has already calculated all locks. This way even next djob will always use the fully finalized version of object. - // - // - // - // Note: We don't use a transaction here for performance (it's called during job pull). - // This means there's a tiny window where the job is running but key isn't deleted yet, - // which is acceptable because new requests will just accumulate data to this job. - tracing::debug!( - job_id = %job.id, - "Cleaning up debounce_key entry for completed/pulled job" - ); - - // This will either: - // 1. Block until pusher pushed. Which gives us: - // - If there was any stale data in pusher, then we will read it here (couple of lines below) - // 2. Block pusher until we are done here. This gives us: - // - We will clone objects and retrieve the latest version. So when we are done the pusher can read latest version. - sqlx::query!("DELETE FROM debounce_key WHERE key = $1", &key) - .execute(&mut *tx) - .await - .map_err(|e| { - tracing::error!( - error = %e, - job_id = %job.id, - "Failed to delete debounce_key" - ); - e - })?; - - let Some(base_hash) = job.runnable_id else { - return Err(Error::InternalErr( - "Missing runnable_id for dependency job triggered by relative import" - .to_string(), - )); - }; - - tracing::debug!( - job_id = %job.id, - base_hash = %base_hash, - job_kind = ?kind, - "Creating new version for dependency job triggered by relative import" - ); - - let new_id = match kind { - JobKind::Dependencies => { - let deployment_message = job - .args - .clone() - .map(|hashmap| { - hashmap - .get("deployment_message") - .map(|map_value| { - serde_json::from_str::(map_value.get()).ok() - }) - .flatten() - }) - .flatten(); - - // This way we tell downstream which script we should archive when the resolution is finished. - // (not used at the moment) - job.args.as_mut().map(|args| { - args.insert("base_hash".to_owned(), to_raw_value(&*base_hash)) - }); - - let cloned_script = windmill_common::scripts::clone_script( - base_hash, - &job.workspace_id, - deployment_message, - &mut tx, - ) - .await?; - if is_generated_from_raw_requirements(&Some(cloned_script.old_script.language), &cloned_script.old_script.lock.map(|v| v.to_string())) { - return Err(Error::BadRequest(format!( - "Script at path {} is generated from raw requirements, not overriding", - job.runnable_path() - ))); - } - cloned_script.new_hash - } - JobKind::FlowDependencies => { - sqlx::query_scalar!( - "INSERT INTO flow_version - (workspace_id, path, value, schema, created_by) - - SELECT workspace_id, path, value, schema, created_by - FROM flow_version WHERE path = $1 AND workspace_id = $2 AND id = $3 - - RETURNING id - ", - job.runnable_path(), - job.workspace_id, - *base_hash, - ) - .fetch_one(&mut *tx) - .await? - } - JobKind::AppDependencies => { - sqlx::query_scalar!( - "INSERT INTO app_version - (app_id, value, created_by, raw_app) - SELECT app_id, value, created_by, raw_app - FROM app_version WHERE id = $1 - RETURNING id", - *base_hash - ) - .fetch_one(&mut *tx) - .await? - } - _ => { - return Err(Error::InternalErr(format!( - "Matched unexpected JobKind ({:?}). This is a bug!", - kind - ))) - } - }; - - job.runnable_id.replace(new_id.into()); - - if !*windmill_common::worker::MIN_VERSION_SUPPORTS_DEBOUNCING.read().await { - tx.commit().await?; - tracing::warn!("Debouncing is not supported on this version of Windmill. Minimum version required for debouncing support."); - return Ok(()); - } - // === RETRIEVE ACCUMULATED DEBOUNCE DATA === - // - // For flows and apps, retrieve all nodes/components that were accumulated - // during the debounce window. This data comes from requests that were merged - // into this job instead of creating their own jobs. - // - // Scripts don't need this because they don't have nodes/components to relock. - if let Some(to_relock_field) = match &job.kind { - JobKind::FlowDependencies => Some("nodes_to_relock"), - JobKind::AppDependencies => Some("components_to_relock"), - _ => None, // Scripts don't use accumulated stale data - } { - tracing::debug!( - job_id = %job.id, - job_kind = ?job.kind, - field = %to_relock_field, - "Retrieving accumulated stale data from debounced requests" - ); - - if let Some(stale_data) = sqlx::query_scalar!( - "DELETE FROM debounce_stale_data WHERE job_id = $1 RETURNING to_relock", - &job.id - ) - .fetch_optional(&mut *tx) - .await - .map_err(|e| { - tracing::error!( - error = %e, - job_id = %job.id, - "Failed to retrieve debounce_stale_data" - ); - e - })? - .flatten() - { - tracing::debug!( - job_id = %job.id, - node_count = stale_data.len(), - nodes = ?stale_data, - "Retrieved accumulated nodes/components from {} debounced requests", - stale_data.len() - ); - - // Replace the job's relock list with the accumulated data - // This ensures all nodes from all debounced requests are processed - if let Some(args) = job.args.as_mut() { - args.insert(to_relock_field.to_owned(), to_raw_value(&stale_data)); - tracing::debug!( - field = %to_relock_field, - "Updated job args with accumulated debounce data" - ); - } - } else { - tracing::trace!( - job_id = %job.id, - "No accumulated stale data found (no debounced requests or already cleaned up)" - ); - } - } - - // This will unblock pusher. - tx.commit().await?; - Ok(()) - }).await; - } - - Ok(()) -} diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index f36d2c5d58..d7e0f66685 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -115,6 +115,7 @@ nix.workspace = true bytes.workspace = true reqwest.workspace = true reqwest-middleware.workspace = true +eventsource-stream.workspace = true mime_guess.workspace = true hex.workspace = true tiberius = { workspace = true, optional = true } diff --git a/backend/windmill-worker/src/ai/sse.rs b/backend/windmill-worker/src/ai/sse.rs index fcafc9cd7e..16cc418d52 100644 --- a/backend/windmill-worker/src/ai/sse.rs +++ b/backend/windmill-worker/src/ai/sse.rs @@ -1,5 +1,6 @@ use std::collections::HashMap; +use eventsource_stream::Eventsource; use reqwest::Response; use serde::Deserialize; use serde_json; @@ -41,39 +42,29 @@ pub struct OpenAISSEEvent { pub choices: Option>, } +lazy_static::lazy_static! { + static ref DEBUG_SSE_STREAM: bool = std::env::var("DEBUG_SSE_STREAM") + .unwrap_or("false".to_string()) + .parse::() + .unwrap_or(false); +} pub trait SSEParser { async fn parse_event_data(&mut self, data: &str) -> Result<(), Error>; async fn parse_events(&mut self, response: Response) -> Result<(), Error> { - let mut stream = response.bytes_stream(); - let mut buffer = String::new(); + let mut stream = response.bytes_stream().eventsource(); - while let Some(chunk_result) = stream.next().await { - let chunk = chunk_result - .map_err(|e| Error::internal_err(format!("Failed to read chunk: {}", e)))?; - - // Convert chunk to string and add to buffer - let chunk_str = String::from_utf8_lossy(&chunk); - buffer.push_str(&chunk_str); - - // Process complete lines from buffer - while let Some(newline_pos) = buffer.find("\n\n") { - let line = buffer.drain(..newline_pos + 2).collect::(); - let line = line.trim_end_matches('\n'); - - // Skip empty lines and comments - if line.is_empty() || line.starts_with(':') { - continue; - } - - // Parse SSE data field - if let Some(data) = line.strip_prefix("data: ") { - if data == "[DONE]" { - // OpenAI sends [DONE] to indicate end of stream - return Ok(()); + while let Some(event) = stream.next().await { + match event { + Ok(event) => { + if *DEBUG_SSE_STREAM { + tracing::info!("SSE event: {:?}", event); } - self.parse_event_data(data).await?; + self.parse_event_data(&event.data).await?; + } + Err(e) => { + tracing::error!("Failed to parse SSE event: {}", e); } } } @@ -102,50 +93,58 @@ impl OpenAISSEParser { impl SSEParser for OpenAISSEParser { async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> { - let event: OpenAISSEEvent = serde_json::from_str(data).map_err(|e| { - Error::internal_err(format!("Failed to parse SSE chunk {}: {}", data, e)) - })?; + if data == "[DONE]" { + return Ok(()); + } - if let Some(mut choices) = event.choices.filter(|s| !s.is_empty()) { - if let Some(delta) = choices.remove(0).delta { - if let Some(content) = delta.content.filter(|s| !s.is_empty()) { - self.accumulated_content.push_str(&content); - let event = StreamingEvent::TokenDelta { content }; - self.stream_event_processor - .send(event, &mut self.events_str) - .await?; - } + let event: Option = serde_json::from_str(data) + .inspect_err(|e| { + tracing::error!("Failed to parse SSE as an OpenAI event {}: {}", data, e); + }) + .ok(); - if let Some(tool_calls) = delta.tool_calls { - for (idx, tool_call) in tool_calls.into_iter().enumerate() { - let idx = tool_call.index.unwrap_or_else(|| idx as i64); + if let Some(event) = event { + if let Some(mut choices) = event.choices.filter(|s| !s.is_empty()) { + if let Some(delta) = choices.remove(0).delta { + if let Some(content) = delta.content.filter(|s| !s.is_empty()) { + self.accumulated_content.push_str(&content); + let event = StreamingEvent::TokenDelta { content }; + self.stream_event_processor + .send(event, &mut self.events_str) + .await?; + } - if let Some(function) = tool_call.function { - if let Some(tool_call) = self.accumulated_tool_calls.get_mut(&idx) { - if let Some(arguments) = function.arguments { - tool_call.function.arguments += &arguments; - } - } else { - let fun_name = function.name.unwrap_or_default(); - let call_id = tool_call.id.unwrap_or_else(|| rd_string(24)); - let event = StreamingEvent::ToolCall { - call_id: call_id.clone(), - function_name: fun_name.clone(), - }; - self.stream_event_processor - .send(event, &mut self.events_str) - .await?; - self.accumulated_tool_calls.insert( - idx, - OpenAIToolCall { - id: call_id, - function: OpenAIFunction { - name: fun_name, - arguments: function.arguments.unwrap_or_default(), + if let Some(tool_calls) = delta.tool_calls { + for (idx, tool_call) in tool_calls.into_iter().enumerate() { + let idx = tool_call.index.unwrap_or_else(|| idx as i64); + + if let Some(function) = tool_call.function { + if let Some(tool_call) = self.accumulated_tool_calls.get_mut(&idx) { + if let Some(arguments) = function.arguments { + tool_call.function.arguments += &arguments; + } + } else { + let fun_name = function.name.unwrap_or_default(); + let call_id = tool_call.id.unwrap_or_else(|| rd_string(24)); + let event = StreamingEvent::ToolCall { + call_id: call_id.clone(), + function_name: fun_name.clone(), + }; + self.stream_event_processor + .send(event, &mut self.events_str) + .await?; + self.accumulated_tool_calls.insert( + idx, + OpenAIToolCall { + id: call_id, + function: OpenAIFunction { + name: fun_name, + arguments: function.arguments.unwrap_or_default(), + }, + r#type: "function".to_string(), }, - r#type: "function".to_string(), - }, - ); + ); + } } } } diff --git a/backend/windmill-worker/src/ai/tools.rs b/backend/windmill-worker/src/ai/tools.rs index 2cc6c1da1a..478eca5e54 100644 --- a/backend/windmill-worker/src/ai/tools.rs +++ b/backend/windmill-worker/src/ai/tools.rs @@ -6,7 +6,7 @@ use crate::ai::utils::{ update_flow_status_module_with_actions, update_flow_status_module_with_actions_success, FlowContext, }; -use crate::common::{error_to_value, OccupancyMetrics}; +use crate::common::OccupancyMetrics; use crate::result_processor::handle_non_flow_job_error; use crate::worker_flow::{ evaluate_input_transform, raw_script_to_payload, script_to_payload, JobPayloadWithTag, @@ -592,7 +592,7 @@ async fn handle_tool_execution_error( final_events_str: &mut String, ) -> Result<(), Error> { let err_string = format!("{}: {}", err.name(), err.to_string()); - let err_json = error_to_value(&err); + let err_json = windmill_common::worker::error_to_value(&err); let _ = handle_non_flow_job_error( ctx.db, tool_job, diff --git a/backend/windmill-worker/src/bigquery_executor.rs b/backend/windmill-worker/src/bigquery_executor.rs index cf93736850..c58b73f551 100644 --- a/backend/windmill-worker/src/bigquery_executor.rs +++ b/backend/windmill-worker/src/bigquery_executor.rs @@ -7,7 +7,7 @@ use serde_json::{json, value::RawValue, Value}; use windmill_common::client::AuthedClient; use windmill_common::error::to_anyhow; use windmill_common::s3_helpers::convert_json_line_stream; -use windmill_common::worker::Connection; +use windmill_common::worker::{Connection, SqlResultCollectionStrategy}; use windmill_common::{error::Error, worker::to_raw_value}; use windmill_parser_sql::{ parse_bigquery_sig, parse_db_resource, parse_s3_mode, parse_sql_blocks, @@ -86,9 +86,11 @@ fn do_bigquery_inner<'a>( timeout_ms: u64, column_order: Option<&'a mut Option>>, skip_collect: bool, + first_row_only: bool, http_client: &'a Client, s3: Option, -) -> windmill_common::error::Result>>> { +) -> windmill_common::error::Result>>>> +{ let param_names = parse_sql_statement_named_params(query, '@'); let statement_values = all_statement_values @@ -113,7 +115,7 @@ fn do_bigquery_inner<'a>( .json(&json!({ "query": query, "useLegacySql": false, - "maxResults": 10000, + "maxResults": if first_row_only { 1 } else { 10000 }, "timeoutMs": timeout_ms, "queryParameters": statement_values, })) @@ -126,7 +128,7 @@ fn do_bigquery_inner<'a>( match response.error_for_status_ref() { Ok(_) => { if skip_collect { - return Ok(to_raw_value(&Value::Array(vec![]))); + return Ok(vec![]); } else { let result = response.json::().await.map_err(|e| { Error::ExecutionErr(format!( @@ -205,10 +207,10 @@ fn do_bigquery_inner<'a>( convert_json_line_stream(rows_stream.boxed(), s3.format).await?; s3.upload(stream.boxed()).await?; - return Ok(to_raw_value(&s3.to_return_s3_obj())); + return Ok(vec![to_raw_value(&s3.to_return_s3_obj())]); } - Ok(to_raw_value(&rows)) + Ok(rows.iter().map(to_raw_value).collect::>()) } } Err(e) => match response.json::().await { @@ -341,6 +343,11 @@ pub async fn do_bigquery( }; let annotations = windmill_common::worker::SqlAnnotations::parse(query); + let collection_strategy = if annotations.return_last_result { + SqlResultCollectionStrategy::LastStatementAllRows + } else { + annotations.result_collection + }; let service_account = CustomServiceAccount::from_json(&database) .map_err(|e| Error::ExecutionErr(e.to_string()))?; @@ -425,52 +432,34 @@ pub async fn do_bigquery( statement_values.insert(arg_n, bigquery_v); } - let result_f = if queries.len() > 1 { - let futures = queries - .iter() - .enumerate() - .map(|(i, x)| { - do_bigquery_inner( - x, - &statement_values, - &project_id, - token.as_str(), - timeout_ms, - None, - annotations.return_last_result && i < queries.len() - 1, - &http_client, - s3.clone(), - ) - }) - .collect::>>()?; + let result_f = async move { + let mut results = vec![]; + for (i, q) in queries.iter().enumerate() { + let result = do_bigquery_inner( + q, + &statement_values, + &project_id, + token.as_str(), + timeout_ms, + if i == queries.len() - 1 + && collection_strategy.collect_last_statement_only(queries.len()) + && !collection_strategy.collect_scalar() + { + Some(column_order) + } else { + None + }, + collection_strategy.collect_last_statement_only(queries.len()) + && i < queries.len() - 1, + collection_strategy.collect_first_row_only(), + &http_client, + s3.clone(), + )? + .await?; + results.push(result); + } - let f = async { - let mut res: Vec> = vec![]; - - for fut in futures { - let r = fut.await?; - res.push(r); - } - if annotations.return_last_result && res.len() > 0 { - Ok(res.pop().unwrap()) - } else { - Ok(to_raw_value(&res)) - } - }; - - f.boxed() - } else { - do_bigquery_inner( - query, - &statement_values, - &project_id, - token.as_str(), - timeout_ms, - Some(column_order), - false, - &http_client, - s3, - )? + collection_strategy.collect(results) }; let r = run_future_with_polling_update_job_poller( diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 259b12ec84..4a6edbd55d 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -539,13 +539,6 @@ pub async fn update_worker_ping_for_failed_init_script( } } -pub fn error_to_value(err: &Error) -> serde_json::Value { - match err { - Error::JsonErr(err) => err.clone(), - _ => json!({"message": err.to_string(), "name": err.name()}), - } -} - #[derive(Clone)] pub struct OccupancyMetrics { pub running_job_started_at: Option, diff --git a/backend/windmill-worker/src/duckdb_executor.rs b/backend/windmill-worker/src/duckdb_executor.rs index e3408ffca7..8acdbb8b07 100644 --- a/backend/windmill-worker/src/duckdb_executor.rs +++ b/backend/windmill-worker/src/duckdb_executor.rs @@ -1,6 +1,6 @@ use std::cell::RefCell; use std::env; -use std::ffi::{c_char, CStr, CString}; +use std::ffi::{c_char, c_uint, CStr, CString}; use std::ptr::NonNull; use std::sync::{Arc, Mutex}; @@ -12,7 +12,7 @@ use uuid::Uuid; use windmill_common::error::{to_anyhow, Error, Result}; use windmill_common::s3_helpers::{S3Object, S3_PROXY_LAST_ERRORS_CACHE}; use windmill_common::utils::sanitize_string_from_password; -use windmill_common::worker::Connection; +use windmill_common::worker::{Connection, SqlResultCollectionStrategy}; use windmill_common::workspaces::{get_ducklake_from_db_unchecked, DucklakeCatalogResourceType}; use windmill_parser_sql::{parse_duckdb_sig, parse_sql_blocks}; use windmill_queue::{CanceledBy, MiniPulledJob}; @@ -39,6 +39,20 @@ pub async fn do_duckdb( occupancy_metrics: &mut OccupancyMetrics, parent_runnable_path: Option, ) -> Result> { + let annotations = windmill_common::worker::SqlAnnotations::parse(query); + let collection_strategy = + if annotations.result_collection == SqlResultCollectionStrategy::Legacy { + // Before result_collection was introduced, duckdb ignored all statements results except the last one + SqlResultCollectionStrategy::LastStatementAllRows + } else { + annotations.result_collection + }; + if annotations.return_last_result { + return Err(Error::ExecutionErr( + "return_last_result annotation is deprecated, use result_collection=last_statement_all_rows instead".to_string(), + )); + } + let token = client.token.clone(); let hidden_passwords = Arc::new(Mutex::new(Vec::::new())); @@ -141,11 +155,13 @@ pub async fn do_duckdb( &token, &base_internal_url, &w_id, + collection_strategy, ) }) .await .map_err(|e| Error::from(to_anyhow(e))) .and_then(|r| r); + let (result, column_order) = match result { Ok(r) => r, Err(e) => { @@ -210,6 +226,8 @@ struct DuckDbFfiLib { base_internal_url: *const c_char, w_id: *const c_char, column_order_ptr: *mut *mut c_char, + collect_last_only: bool, + collect_first_row_only: bool, ) -> *mut c_char, >, free_cstr: Symbol<'static, unsafe extern "C" fn(string: *mut c_char) -> ()>, @@ -249,7 +267,28 @@ impl DuckDbFfiLib { )) })? }; + let lib = Box::leak(Box::new(lib)); + + // Version mismatch should only be possible on Windows agent workers + // We check for it because FFI interface mismatch will cause undefined behavior / crashes + unsafe { + let expected_version: c_uint = 1; + let get_version: Symbol<'static, unsafe extern "C" fn() -> c_uint> = + lib.get(b"get_version") + .map_err(|e| return Error::ExecutionErr(format!("Could not find get_version in the duckdb ffi library. If you are not using docker, consider manually upgrading windmill_duckdb_ffi_lib. {}", e.to_string())))?; + let actual_version = get_version(); + if actual_version < expected_version { + return Err(Error::InternalErr( + format!("Incompatible duckdb ffi library version. Expected: {expected_version}, actual: {actual_version}. Please update to the latest windmill_duckdb_ffi_lib."), + )); + } else if actual_version > expected_version { + return Err(Error::InternalErr( + format!("Incompatible duckdb ffi library version. Expected: {expected_version}, actual: {actual_version}. Please upgrade your worker to the latest windmill version."), + )); + } + } + Ok(DuckDbFfiLib { run_duckdb_ffi: unsafe { lib.get(b"run_duckdb_ffi").map_err(to_anyhow)? }, free_cstr: unsafe { lib.get(b"free_cstr").map_err(to_anyhow)? }, @@ -265,6 +304,7 @@ fn run_duckdb_ffi_safe<'a>( token: &str, base_internal_url: &str, w_id: &str, + collection_strategy: SqlResultCollectionStrategy, ) -> Result<(Box, Option>)> { let query_block_list = query_block_list .map(|s| { @@ -296,13 +336,18 @@ fn run_duckdb_ffi_safe<'a>( base_internal_url.as_ptr(), w_id.as_ptr(), &mut column_order, + collection_strategy.collect_last_statement_only(query_block_list_count), + collection_strategy.collect_first_row_only(), ); let str = CStr::from_ptr(ptr).to_string_lossy().to_string(); free_cstr(ptr); str }; - let column_order = if column_order.is_null() { + let column_order = if column_order.is_null() + || !collection_strategy.collect_last_statement_only(query_block_list_count) + || collection_strategy.collect_scalar() + { None } else { let str = unsafe { CStr::from_ptr(column_order).to_string_lossy().to_string() }; @@ -313,7 +358,14 @@ fn run_duckdb_ffi_safe<'a>( if result_str.starts_with("ERROR") { Err(Error::ExecutionErr(result_str[6..].to_string())) } else { - let result = serde_json::value::RawValue::from_string(result_str).map_err(to_anyhow)?; + let result = if collection_strategy == SqlResultCollectionStrategy::AllStatementsAllRows { + // Avoid parsing JSON + serde_json::value::RawValue::from_string(result_str).map_err(to_anyhow)? + } else { + let result = + serde_json::from_str::>>>(&result_str).map_err(to_anyhow)?; + collection_strategy.collect(result)? + }; Ok((result, column_order)) } } @@ -350,7 +402,7 @@ fn format_attach_db_conn_str(db_resource: Value, db_type: &str) -> Result { let res: PgDatabase = serde_json::from_value(db_resource)?; format!( - "dbname={} {} host={} {} {}", + "dbname={} {} host={} {} {} {}", res.dbname, res.user.map(|u| format!("user={}", u)).unwrap_or_default(), res.host, @@ -358,6 +410,9 @@ fn format_attach_db_conn_str(db_resource: Value, db_type: &str) -> Result 0 { - Ok(to_raw_value(&json_results.pop().unwrap())) - } else { - Ok(to_raw_value(&json_results)) - } + collection_strategy.collect(json_results) } }; diff --git a/backend/windmill-worker/src/mysql_executor.rs b/backend/windmill-worker/src/mysql_executor.rs index 55cbe35893..2163ac9890 100644 --- a/backend/windmill-worker/src/mysql_executor.rs +++ b/backend/windmill-worker/src/mysql_executor.rs @@ -16,7 +16,7 @@ use windmill_common::{ client::AuthedClient, error::{to_anyhow, Error}, s3_helpers::convert_json_line_stream, - worker::{to_raw_value, Connection}, + worker::{to_raw_value, Connection, SqlResultCollectionStrategy}, }; use windmill_parser_sql::{ parse_db_resource, parse_mysql_sig, parse_s3_mode, parse_sql_blocks, @@ -50,8 +50,10 @@ fn do_mysql_inner<'a>( conn: Arc>, column_order: Option<&'a mut Option>>, skip_collect: bool, + first_row_only: bool, s3: Option, -) -> windmill_common::error::Result>>> { +) -> windmill_common::error::Result>>>> +{ let param_names = parse_sql_statement_named_params(query, ':') .into_iter() .map(|x| x.into_bytes()) @@ -76,7 +78,7 @@ fn do_mysql_inner<'a>( .await .map_err(to_anyhow)?; - Ok(to_raw_value(&Value::Array(vec![]))) + Ok(vec![]) } else if let Some(ref s3) = s3 { let query = query.to_string(); let rows_stream = async_stream::stream! { @@ -108,14 +110,23 @@ fn do_mysql_inner<'a>( let stream = convert_json_line_stream(rows_stream.boxed(), s3.format).await?; s3.upload(stream.boxed()).await?; - Ok(to_raw_value(&s3.to_return_s3_obj())) + Ok(vec![to_raw_value(&s3.to_return_s3_obj())]) } else { - let rows: Vec = conn - .lock() - .await - .exec(query, statement_values) - .await - .map_err(to_anyhow)?; + let rows: Vec = if first_row_only { + conn.lock() + .await + .exec_first(query, statement_values) + .await + .map_err(to_anyhow)? + .into_iter() + .collect() + } else { + conn.lock() + .await + .exec(query, statement_values) + .await + .map_err(to_anyhow)? + }; if let Some(column_order) = column_order { *column_order = Some( @@ -130,12 +141,10 @@ fn do_mysql_inner<'a>( ); } - Ok(to_raw_value( - &rows - .into_iter() - .map(|x| convert_row_to_value(x)) - .collect::>(), - )) + Ok(rows + .into_iter() + .map(|x| to_raw_value(&convert_row_to_value(x))) + .collect::>()) } }; @@ -180,6 +189,11 @@ pub async fn do_mysql( }; let annotations = windmill_common::worker::SqlAnnotations::parse(query); + let collection_strategy = if annotations.return_last_result { + SqlResultCollectionStrategy::LastStatementAllRows + } else { + annotations.result_collection + }; let opts = OptsBuilder::default() .db_name(Some(database.database)) @@ -285,45 +299,32 @@ pub async fn do_mysql( let queries = parse_sql_blocks(query); - let result_f = if queries.len() > 1 { - let futures = queries - .iter() - .enumerate() - .map(|(i, x)| { - do_mysql_inner( - x, - &statement_values, - conn_a.clone(), - None, - annotations.return_last_result && i < queries.len() - 1, - s3.clone(), - ) - }) - .collect::>>()?; + let conn_a_ref = &conn_a; + let result_f = async move { + let mut results = vec![]; + for (i, query) in queries.iter().enumerate() { + let result = do_mysql_inner( + query, + &statement_values, + conn_a_ref.clone(), + if i == queries.len() - 1 + && collection_strategy.collect_last_statement_only(queries.len()) + && !collection_strategy.collect_scalar() + { + Some(column_order) + } else { + None + }, + collection_strategy.collect_last_statement_only(queries.len()) + && i < queries.len() - 1, + collection_strategy.collect_first_row_only(), + s3.clone(), + )? + .await?; + results.push(result); + } - let f = async { - let mut res: Vec> = vec![]; - for fut in futures { - let r = fut.await?; - res.push(r); - } - if annotations.return_last_result && res.len() > 0 { - Ok(res.pop().unwrap()) - } else { - Ok(to_raw_value(&res)) - } - }; - - f.boxed() - } else { - do_mysql_inner( - query, - &statement_values, - conn_a.clone(), - Some(column_order), - false, - s3, - )? + collection_strategy.collect(results) }; let result = run_future_with_polling_update_job_poller( @@ -344,11 +345,10 @@ pub async fn do_mysql( pool.disconnect().await.map_err(to_anyhow)?; - let raw_result = windmill_common::worker::to_raw_value(&json!(result)); - *mem_peak = (raw_result.get().len() / 1000) as i32; + *mem_peak = (result.get().len() / 1000) as i32; // And then check that we got back the same string we sent over. - return Ok(raw_result); + return Ok(result); } // 2023-12-01T16:18:00.000Z diff --git a/backend/windmill-worker/src/oracledb_executor.rs b/backend/windmill-worker/src/oracledb_executor.rs index a1c20d1282..d83edf0104 100644 --- a/backend/windmill-worker/src/oracledb_executor.rs +++ b/backend/windmill-worker/src/oracledb_executor.rs @@ -11,7 +11,7 @@ use serde_json::{json, value::RawValue, Value}; use windmill_common::{ error::{to_anyhow, Error}, s3_helpers::convert_json_line_stream, - worker::{to_raw_value, Connection}, + worker::{to_raw_value, Connection, SqlResultCollectionStrategy}, }; use windmill_queue::MiniPulledJob; @@ -48,8 +48,10 @@ pub fn do_oracledb_inner<'a>( conn: Arc>, column_order: Option<&'a mut Option>>, skip_collect: bool, + first_row_only: bool, s3: Option, -) -> windmill_common::error::Result>>> { +) -> windmill_common::error::Result>>>> +{ let qw = query.trim_end_matches(';').to_string(); let result_f = async move { @@ -85,7 +87,7 @@ pub fn do_oracledb_inner<'a>( .map_err(to_anyhow)? .map_err(to_anyhow)?; - Ok(to_raw_value(&Value::Array(vec![]))) + Ok(vec![]) } else { // We use an mpsc because we need an async stream for s3 mode. However since everything is sync // in rust-oracle, I assumed that calling ResultSet::next() is blocking when it has to refetch. @@ -133,6 +135,9 @@ pub fn do_oracledb_inner<'a>( break; } } + if first_row_only { + break; + } } } _ => { @@ -159,17 +164,16 @@ pub fn do_oracledb_inner<'a>( if let Some(s3) = s3 { let stream = convert_json_line_stream(rows_stream.boxed(), s3.format).await?; s3.upload(stream.boxed()).await?; - return Ok(to_raw_value(&s3.to_return_s3_obj())); + return Ok(vec![to_raw_value(&s3.to_return_s3_obj())]); } else { let rows: Vec<_> = rows_stream.collect().await; - Ok(to_raw_value( - &rows - .into_iter() - .collect::, _>>() - .map_err(to_anyhow)? - .into_iter() - .collect::>(), - )) + Ok(rows + .into_iter() + .collect::, _>>() + .map_err(to_anyhow)? + .iter() + .map(to_raw_value) + .collect::>()) } } }; @@ -377,6 +381,11 @@ pub async fn do_oracledb( }; let annotations = windmill_common::worker::SqlAnnotations::parse(query); + let collection_strategy = if annotations.return_last_result { + SqlResultCollectionStrategy::LastStatementAllRows + } else { + annotations.result_collection + }; let sig = parse_oracledb_sig(query) .map_err(|x| Error::ExecutionErr(x.to_string()))? @@ -388,7 +397,7 @@ pub async fn do_oracledb( let (query, args_to_skip) = sanitize_and_interpolate_unsafe_sql_args(query, &sig, &job_args, &reserved_variables)?; - let (statement_values, errors) = get_statement_values(sig.clone(), &job_args, &args_to_skip); + let (_, errors) = get_statement_values(sig.clone(), &job_args, &args_to_skip); if !errors.is_empty() { return Err(Error::ExecutionErr(errors.join("\n"))); @@ -412,40 +421,33 @@ pub async fn do_oracledb( let queries = parse_sql_blocks(&query); - let result_f = if queries.len() > 1 { - let f = async { - let mut res: Vec> = vec![]; - for (i, q) in queries.iter().enumerate() { - let (vals, _) = get_statement_values(sig.clone(), &job_args, &args_to_skip); - let r = do_oracledb_inner( - q, - vals, - conn_a.clone(), - None, - annotations.return_last_result && i < queries.len() - 1, - s3.clone(), - )? - .await?; - res.push(r); - } + let result_f = async move { + let mut results = vec![]; + for (i, q) in queries.iter().enumerate() { + let (vals, _) = get_statement_values(sig.clone(), &job_args, &args_to_skip); - if annotations.return_last_result && res.len() > 0 { - Ok(res.pop().unwrap()) - } else { - Ok(to_raw_value(&res)) - } - }; + let result = do_oracledb_inner( + q, + vals, + conn_a.clone(), + if i == queries.len() - 1 + && collection_strategy.collect_last_statement_only(queries.len()) + && !collection_strategy.collect_scalar() + { + Some(column_order) + } else { + None + }, + collection_strategy.collect_last_statement_only(queries.len()) + && i < queries.len() - 1, + collection_strategy.collect_first_row_only(), + s3.clone(), + )? + .await?; + results.push(result); + } - f.boxed() - } else { - do_oracledb_inner( - &query, - statement_values, - conn_a, - Some(column_order), - false, - s3, - )? + collection_strategy.collect(results) }; let result = run_future_with_polling_update_job_poller( diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index 9bb7c70cda..e240b5d87b 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -28,7 +28,9 @@ use uuid::Uuid; use windmill_common::error::to_anyhow; use windmill_common::error::{self, Error}; use windmill_common::s3_helpers::convert_json_line_stream; -use windmill_common::worker::{to_raw_value, Connection, CLOUD_HOSTED}; +use windmill_common::worker::{ + to_raw_value, Connection, SqlResultCollectionStrategy, CLOUD_HOSTED, +}; use windmill_parser::{Arg, Typ}; use windmill_parser_sql::{ parse_db_resource, parse_pg_statement_arg_indices, parse_pgsql_sig, parse_s3_mode, @@ -73,8 +75,9 @@ fn do_postgresql_inner<'a>( column_order: Option<&'a mut Option>>, siz: &'a AtomicUsize, skip_collect: bool, + first_row_only: bool, s3: Option, -) -> error::Result>>> { +) -> error::Result>>>> { let mut query_params = vec![]; let arg_indices = parse_pg_statement_arg_indices(&query); @@ -100,7 +103,7 @@ fn do_postgresql_inner<'a>( let result_f = async move { // Now we can execute a simple statement that just returns its parameter. - let mut res: Vec = vec![]; + let mut res: Vec> = vec![]; let query_params = query_params .iter() @@ -125,13 +128,19 @@ fn do_postgresql_inner<'a>( let stream = convert_json_line_stream(rows_stream.boxed(), s3.format).await?; s3.upload(stream.boxed()).await?; - return Ok(to_raw_value(&s3.to_return_s3_obj())); + return Ok(vec![to_raw_value(&s3.to_return_s3_obj())]); } else { let rows = client .query_raw(&query, query_params) .await .map_err(to_anyhow)?; + let rows = if first_row_only { + rows.take(1).boxed() + } else { + rows.boxed() + }; + let rows = rows.try_collect::>().await.map_err(to_anyhow)?; if let Some(column_order) = column_order { @@ -164,14 +173,14 @@ fn do_postgresql_inner<'a>( } } if let Ok(v) = r { - res.push(v); + res.push(to_raw_value(&v)); } else { return Err(to_anyhow(r.err().unwrap()).into()); } } } - Ok(to_raw_value(&res)) + Ok(res) }; Ok(result_f.boxed()) @@ -216,6 +225,11 @@ pub async fn do_postgresql( }; let annotations = windmill_common::worker::SqlAnnotations::parse(query); + let collection_strategy = if annotations.return_last_result { + SqlResultCollectionStrategy::LastStatementAllRows + } else { + annotations.result_collection + }; let sslmode = match database.sslmode.as_deref() { Some("allow") => "prefer".to_string(), @@ -336,47 +350,33 @@ pub async fn do_postgresql( .collect::>(); let size = AtomicUsize::new(0); - let result_f = if queries.len() > 1 { - let futures = queries - .iter() - .enumerate() - .map(|(i, x)| { - do_postgresql_inner( - x.to_string(), - ¶m_idx_to_arg_and_value, - client, - None, - &size, - annotations.return_last_result && i < queries.len() - 1, - s3.clone(), - ) - }) - .collect::>>()?; + let size_ref = &size; + let result_f = async move { + let mut results = vec![]; + for (i, query) in queries.iter().enumerate() { + let result = do_postgresql_inner( + query.to_string(), + ¶m_idx_to_arg_and_value, + client, + if i == queries.len() - 1 + && collection_strategy.collect_last_statement_only(queries.len()) + && !collection_strategy.collect_scalar() + { + Some(column_order) + } else { + None + }, + size_ref, + collection_strategy.collect_last_statement_only(queries.len()) + && i < queries.len() - 1, + collection_strategy.collect_first_row_only(), + s3.clone(), + )? + .await?; + results.push(result); + } - let f = async { - let mut res: Vec> = vec![]; - for fut in futures { - let r = fut.await?; - res.push(r); - } - if annotations.return_last_result && res.len() > 0 { - Ok(res.pop().unwrap()) - } else { - Ok(to_raw_value(&res)) - } - }; - - f.boxed() - } else { - do_postgresql_inner( - query.to_string(), - ¶m_idx_to_arg_and_value, - client, - Some(column_order), - &size, - false, - s3, - )? + collection_strategy.collect(results) }; let result = run_future_with_polling_update_job_poller( diff --git a/backend/windmill-worker/src/pwsh_executor.rs b/backend/windmill-worker/src/pwsh_executor.rs index cd3161ef12..4a4b85f8ac 100644 --- a/backend/windmill-worker/src/pwsh_executor.rs +++ b/backend/windmill-worker/src/pwsh_executor.rs @@ -426,9 +426,8 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"", }\n"; // make sure param() is first - let param_match = windmill_parser_bash::RE_POWERSHELL_PARAM.find(&content); + let param_match = windmill_parser_bash::extract_powershell_param_block(&content, true); let content: String = if let Some(param_match) = param_match { - let param_match = param_match.as_str(); format!( "{}\n{}\n{}\n{}\n{}", param_match, diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 7cfe63dc34..1e168d3a30 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -22,7 +22,7 @@ use windmill_common::{ flow_status::FlowJobDuration, jobs::JobKind, utils::WarnAfterExt, - worker::{to_raw_value, Connection, WORKER_GROUP}, + worker::{error_to_value, to_raw_value, Connection, WORKER_GROUP}, worker_group_job_stats::{accumulate_job_stats, flush_stats_to_db, JobStatsMap}, KillpillSender, DB, }; @@ -31,7 +31,8 @@ use windmill_common::{ use windmill_common::bench::{BenchmarkInfo, BenchmarkIter}; use windmill_queue::{ - CanceledBy, INIT_SCRIPT_TAG, JobCompleted, MiniCompletedJob, MiniPulledJob, ValidableJson, WrappedError, append_logs, get_mini_completed_job + append_logs, get_mini_completed_job, CanceledBy, JobCompleted, MiniCompletedJob, MiniPulledJob, + ValidableJson, WrappedError, INIT_SCRIPT_TAG, }; use serde_json::{json, value::RawValue, Value}; @@ -42,7 +43,7 @@ use windmill_queue::{add_completed_job, add_completed_job_error}; use crate::{ bash_executor::ANSI_ESCAPE_RE, - common::{error_to_value, read_result, save_in_cache}, + common::{read_result, save_in_cache}, otel_oss::add_root_flow_job_to_otlp, worker_flow::update_flow_status_after_job_completion, JobCompletedReceiver, JobCompletedSender, SameWorkerSender, SendResult, SendResultPayload, diff --git a/backend/windmill-worker/src/snowflake_executor.rs b/backend/windmill-worker/src/snowflake_executor.rs index e657ef7bb0..38eff99ae1 100644 --- a/backend/windmill-worker/src/snowflake_executor.rs +++ b/backend/windmill-worker/src/snowflake_executor.rs @@ -10,7 +10,7 @@ use sha2::{Digest, Sha256}; use std::collections::HashMap; use windmill_common::error::to_anyhow; use windmill_common::s3_helpers::convert_json_line_stream; -use windmill_common::worker::Connection; +use windmill_common::worker::{Connection, SqlResultCollectionStrategy}; use windmill_common::{error::Error, worker::to_raw_value}; use windmill_parser_sql::{ @@ -130,10 +130,12 @@ fn do_snowflake_inner<'a>( token_is_keypair: bool, column_order: Option<&'a mut Option>>, skip_collect: bool, + first_row_only: bool, http_client: &'a Client, s3: Option, reserved_variables: &HashMap, -) -> windmill_common::error::Result>>> { +) -> windmill_common::error::Result>>>> +{ let sig = parse_snowflake_sig(&query) .map_err(|x| Error::ExecutionErr(x.to_string()))? .args; @@ -179,7 +181,7 @@ fn do_snowflake_inner<'a>( if skip_collect { handle_snowflake_result(result).await?; - Ok(to_raw_value(&Value::Array(vec![]))) + Ok(vec![]) } else { let response = result .parse_snowflake_response::() @@ -254,19 +256,22 @@ fn do_snowflake_inner<'a>( row_map }); + let rows_stream = rows_stream.take(if first_row_only { 1 } else { usize::MAX }); + if let Some(s3) = s3 { let rows_stream = rows_stream.map(|r| serde_json::value::to_value(&r?).map_err(to_anyhow)); let stream = convert_json_line_stream(rows_stream.boxed(), s3.format).await?; s3.upload(stream.boxed()).await?; - Ok(to_raw_value(&s3.to_return_s3_obj())) + Ok(vec![to_raw_value(&s3.to_return_s3_obj())]) } else { let rows = rows_stream .collect::>() .await .into_iter() + .map(|x| x.map(|v| to_raw_value(&v))) .collect::, _>>()?; - Ok(to_raw_value(&rows)) + Ok(rows) } } }; @@ -312,6 +317,11 @@ pub async fn do_snowflake( }; let annotations = windmill_common::worker::SqlAnnotations::parse(query); + let collection_strategy = if annotations.return_last_result { + SqlResultCollectionStrategy::LastStatementAllRows + } else { + annotations.result_collection + }; // Check if the token is present in db_arg and use it if available let (token, token_is_keypair) = if let Some(token) = db_arg @@ -409,56 +419,38 @@ pub async fn do_snowflake( let reserved_variables = get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?; - let result_f = if queries.len() > 1 { - let futures = queries - .iter() - .enumerate() - .map(|(i, x)| { - do_snowflake_inner( - x, - &snowflake_args, - body.clone(), - &database.account_identifier, - &token, - token_is_keypair, - None, - annotations.return_last_result && i < queries.len() - 1, - &http_client, - s3.clone(), - &reserved_variables, - ) - }) - .collect::>>()?; + let result_f = async move { + let mut results = vec![]; + for (i, q) in queries.iter().enumerate() { + let result = do_snowflake_inner( + q, + &snowflake_args, + body.clone(), + &database.account_identifier, + &token, + token_is_keypair, + if i == queries.len() - 1 + && collection_strategy.collect_last_statement_only(queries.len()) + && !collection_strategy.collect_scalar() + { + Some(column_order) + } else { + None + }, + collection_strategy.collect_last_statement_only(queries.len()) + && i < queries.len() - 1, + collection_strategy.collect_first_row_only(), + &http_client, + s3.clone(), + &reserved_variables, + )? + .await?; + results.push(result); + } - let f = async { - let mut res: Vec> = vec![]; - for fut in futures { - let r = fut.await?; - res.push(r); - } - if annotations.return_last_result && res.len() > 0 { - Ok(res.pop().unwrap()) - } else { - Ok(to_raw_value(&res)) - } - }; - - f.boxed() - } else { - do_snowflake_inner( - query, - &snowflake_args, - body.clone(), - &database.account_identifier, - &token, - token_is_keypair, - Some(column_order), - false, - &http_client, - s3.clone(), - &reserved_variables, - )? + collection_strategy.collect(results) }; + let r = run_future_with_polling_update_job_poller( job.id, job.timeout, diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 0de82b9e4d..ba099c87dd 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -18,6 +18,7 @@ use windmill_common::scripts::hash_to_codebase_id; use windmill_common::scripts::is_special_codebase_hash; use windmill_common::utils::report_critical_error; use windmill_common::utils::retrieve_common_worker_prefix; +use windmill_common::worker::error_to_value; use windmill_common::{ agent_workers::DECODED_AGENT_TOKEN, apps::AppScriptId, @@ -58,7 +59,6 @@ use std::{ time::Duration, }; use windmill_parser::MainArgSignature; -use windmill_queue::preprocess_dependency_job; use windmill_queue::MiniCompletedJob; use windmill_queue::PulledJobResultToJobErr; @@ -112,7 +112,7 @@ use crate::{ bash_executor::handle_bash_job, bun_executor::handle_bun_job, common::{ - build_args_map, cached_result_path, error_to_value, get_cached_resource_value_if_valid, + build_args_map, cached_result_path, get_cached_resource_value_if_valid, get_reserved_variables, update_worker_ping_for_failed_init_script, OccupancyMetrics, }, csharp_executor::handle_csharp_job, @@ -840,10 +840,22 @@ pub fn start_interactive_worker_shell( match job { Ok(j) => match j.to_pulled_job() { - Ok(j) => Ok(j.map(NextJob::Sql)), - Err(PulledJobResultToJobErr::MissingConcurrencyKey(jc)) => { - if let Err(err) = job_completed_tx.send_job(jc, true).await { - tracing::error!("An error occurred while sending job completed (missing concurrency key): {:#?}", err) + Ok(j) => Ok(j.clone().map(NextJob::Sql)), + ref e @ (Err(PulledJobResultToJobErr::MissingConcurrencyKey( + ref jc, + )) + | Err(PulledJobResultToJobErr::ErrorWhilePreprocessing( + ref jc, + ))) => { + if let Err(err) = + job_completed_tx.send_job(jc.clone(), true).await + { + let e_fmt = match e { + Ok(_) => "unknown error".to_owned(), + Err(e) => e.to_string(), + }; + + tracing::error!("An error occurred while sending job completed ({e_fmt}): {:#?}", err) } Ok(None) } @@ -878,7 +890,7 @@ pub fn start_interactive_worker_shell( token, precomputed_agent_info: precomputed_bundle, } = extract_job_and_perms(job, &conn).await; - + let authed_client = AuthedClient::new( base_internal_url.to_owned(), job.workspace_id.clone(), @@ -887,7 +899,7 @@ pub fn start_interactive_worker_shell( ); let arc_job = Arc::new(job); - + let _ = handle_queued_job( arc_job.clone(), raw_code, @@ -1606,30 +1618,23 @@ pub async fn run_worker( } }; - // Essential debouncing job preprocessing. - if let Ok(windmill_queue::PulledJobResult { - job: Some(ref mut pulled_job), - .. - }) = &mut job - { - match timeout( + // Preprocess pulled job result + if let Ok(ref mut pulled_job_res) = job { + if let Err(e) = timeout( + // Will fail if longer than 10 seconds core::time::Duration::from_secs(10), - preprocess_dependency_job(pulled_job, &db), + pulled_job_res.preprocess(db), ) .warn_after_seconds(2) .await + // Flatten result + .map_err(error::Error::from) + .and_then(|r| r) { - Ok(Err(e)) => { - tracing::error!(worker = %worker_name, hostname = %hostname, "critical: debouncing job preprocessor failed: {e:?}"); - job = Err(e.into()); - } - Err(e) => { - tracing::error!(worker = %worker_name, hostname = %hostname, "critical: debouncing job preprocessor has timed out: {e:?}"); - job = Err(e.into()); - } - _ => {} + pulled_job_res.error_while_preprocessing = Some(e.to_string()); } } + add_time!(bench, "job pulled from DB"); let duration_pull_s = pull_time.elapsed().as_secs_f64(); let err_pull = job.is_ok(); @@ -1689,9 +1694,21 @@ pub async fn run_worker( match job { Ok(pulled_job_result) => match pulled_job_result.to_pulled_job() { Ok(j) => Ok(j.map(NextJob::Sql)), - Err(PulledJobResultToJobErr::MissingConcurrencyKey(jc)) => { - if let Err(err) = job_completed_tx.send_job(jc, true).await { - tracing::error!("An error occurred while sending job completed (missing concurrency key): {:#?}", err) + ref e @ (Err(PulledJobResultToJobErr::MissingConcurrencyKey( + ref jc, + )) + | Err(PulledJobResultToJobErr::ErrorWhilePreprocessing( + ref jc, + ))) => { + if let Err(err) = + job_completed_tx.send_job(jc.clone(), true).await + { + let e_fmt = match e { + Ok(_) => "unknown error".to_owned(), + Err(e) => e.to_string(), + }; + + tracing::error!("An error occurred while sending job completed ({e_fmt}): {:#?}", err) } Ok(None) } diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 4ddab750cf..f0bc4cfc1e 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -2232,7 +2232,7 @@ async fn push_next_flow_job( } }); - // if this is an empty module without preprocessor of if the module has already been completed, successfully, update the parent flow + // if this is an empty module without preprocessor or if the module has already been completed, successfully, update the parent flow if (flow.modules.is_empty() && !step.is_preprocessor_step()) || matches!(status_module, FlowStatusModule::Success { .. }) { @@ -3718,11 +3718,13 @@ pub struct JobPayloadWithTag { pub timeout: Option, pub on_behalf_of: Option, } +#[derive(Debug)] enum ContinuePayload { SingleJob(JobPayloadWithTag), ParallelJobs(Vec), } +#[derive(Debug)] enum NextFlowTransform { EmptyInnerFlows { branch_chosen: Option }, Continue(ContinuePayload, NextStatus), diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 14179cdde7..4038cd90b3 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.574.1"; +export const VERSION = "v1.575.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/commands/workspace/workspace.ts b/cli/src/commands/workspace/workspace.ts index e67617a562..cf74a1f4da 100644 --- a/cli/src/commands/workspace/workspace.ts +++ b/cli/src/commands/workspace/workspace.ts @@ -477,6 +477,7 @@ const command = new Command() .command("delete-fork") .description("Delete a forked workspace and git branch") .arguments("") + .option("-y --yes", "Skip confirmation prompt") .action(deleteWorkspaceFork as any); export default command; diff --git a/cli/src/main.ts b/cli/src/main.ts index 88ef77d73b..42d17a4231 100644 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -68,7 +68,7 @@ export { // } // }); -export const VERSION = "1.574.1"; +export const VERSION = "1.575.0"; export const WM_FORK_PREFIX = "wm-fork"; diff --git a/cli/wasm/php/windmill_parser_wasm.d.ts b/cli/wasm/php/windmill_parser_wasm.d.ts index 7324082938..dcb548ab52 100644 --- a/cli/wasm/php/windmill_parser_wasm.d.ts +++ b/cli/wasm/php/windmill_parser_wasm.d.ts @@ -1,3 +1,3 @@ /* tslint:disable */ /* eslint-disable */ -export function parse_php(code: string): string; +export function parse_php(code: string, main_override?: string | null): string; diff --git a/cli/wasm/php/windmill_parser_wasm.js b/cli/wasm/php/windmill_parser_wasm.js index c0cdf9229d..e95e3a5126 100644 --- a/cli/wasm/php/windmill_parser_wasm.js +++ b/cli/wasm/php/windmill_parser_wasm.js @@ -56,30 +56,41 @@ function passStringToWasm0(arg, malloc, realloc) { return ptr; } -const cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } ); +function isLikeNone(x) { + return x === undefined || x === null; +} + +let cachedTextDecoder = (typeof TextDecoder !== 'undefined' ? new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }) : { decode: () => { throw Error('TextDecoder not available') } } ); if (typeof TextDecoder !== 'undefined') { cachedTextDecoder.decode(); }; +function decodeText(ptr, len) { + return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); +} + function getStringFromWasm0(ptr, len) { ptr = ptr >>> 0; - return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); + return decodeText(ptr, len); } /** * @param {string} code + * @param {string | null} [main_override] * @returns {string} */ -export function parse_php(code) { - let deferred2_0; - let deferred2_1; +export function parse_php(code, main_override) { + let deferred3_0; + let deferred3_1; try { const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_php(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; + var ptr1 = isLikeNone(main_override) ? 0 : passStringToWasm0(main_override, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + var len1 = WASM_VECTOR_LEN; + const ret = wasm.parse_php(ptr0, len0, ptr1, len1); + deferred3_0 = ret[0]; + deferred3_1 = ret[1]; return getStringFromWasm0(ret[0], ret[1]); } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + wasm.__wbindgen_free(deferred3_0, deferred3_1, 1); } } diff --git a/cli/wasm/php/windmill_parser_wasm_bg.wasm b/cli/wasm/php/windmill_parser_wasm_bg.wasm index 2d772cc834..c92e10ebac 100644 Binary files a/cli/wasm/php/windmill_parser_wasm_bg.wasm and b/cli/wasm/php/windmill_parser_wasm_bg.wasm differ diff --git a/cli/wasm/php/windmill_parser_wasm_bg.wasm.d.ts b/cli/wasm/php/windmill_parser_wasm_bg.wasm.d.ts index a771d8a427..30c3d0eff5 100644 --- a/cli/wasm/php/windmill_parser_wasm_bg.wasm.d.ts +++ b/cli/wasm/php/windmill_parser_wasm_bg.wasm.d.ts @@ -1,7 +1,7 @@ /* tslint:disable */ /* eslint-disable */ export const memory: WebAssembly.Memory; -export const parse_php: (a: number, b: number) => [number, number]; +export const parse_php: (a: number, b: number, c: number, d: number) => [number, number]; export const __wbindgen_export_0: WebAssembly.Table; export const __wbindgen_malloc: (a: number, b: number) => number; export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number; diff --git a/cli/wasm/regex/windmill_parser_wasm.d.ts b/cli/wasm/regex/windmill_parser_wasm.d.ts index 5ad7aef180..59f34f2b18 100644 --- a/cli/wasm/regex/windmill_parser_wasm.d.ts +++ b/cli/wasm/regex/windmill_parser_wasm.d.ts @@ -1,14 +1,14 @@ /* tslint:disable */ /* eslint-disable */ -export function parse_bash(code: string): string; -export function parse_snowflake(code: string): string; export function parse_assets_sql(code: string): string; -export function parse_duckdb(code: string): string; -export function parse_graphql(code: string): string; export function parse_db_resource(code: string): string | undefined; -export function parse_mysql(code: string): string; -export function parse_sql(code: string): string; -export function parse_oracledb(code: string): string; -export function parse_bigquery(code: string): string; +export function parse_bash(code: string): string; export function parse_mssql(code: string): string; +export function parse_oracledb(code: string): string; export function parse_powershell(code: string): string; +export function parse_bigquery(code: string): string; +export function parse_graphql(code: string): string; +export function parse_snowflake(code: string): string; +export function parse_duckdb(code: string): string; +export function parse_sql(code: string): string; +export function parse_mysql(code: string): string; diff --git a/cli/wasm/regex/windmill_parser_wasm.js b/cli/wasm/regex/windmill_parser_wasm.js index ce0a45c4c4..25a60a9b8e 100644 --- a/cli/wasm/regex/windmill_parser_wasm.js +++ b/cli/wasm/regex/windmill_parser_wasm.js @@ -68,44 +68,6 @@ function getStringFromWasm0(ptr, len) { ptr = ptr >>> 0; return decodeText(ptr, len); } -/** - * @param {string} code - * @returns {string} - */ -export function parse_bash(code) { - let deferred2_0; - let deferred2_1; - try { - const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_bash(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; - return getStringFromWasm0(ret[0], ret[1]); - } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); - } -} - -/** - * @param {string} code - * @returns {string} - */ -export function parse_snowflake(code) { - let deferred2_0; - let deferred2_1; - try { - const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_snowflake(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; - return getStringFromWasm0(ret[0], ret[1]); - } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); - } -} - /** * @param {string} code * @returns {string} @@ -125,44 +87,6 @@ export function parse_assets_sql(code) { } } -/** - * @param {string} code - * @returns {string} - */ -export function parse_duckdb(code) { - let deferred2_0; - let deferred2_1; - try { - const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_duckdb(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; - return getStringFromWasm0(ret[0], ret[1]); - } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); - } -} - -/** - * @param {string} code - * @returns {string} - */ -export function parse_graphql(code) { - let deferred2_0; - let deferred2_1; - try { - const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_graphql(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; - return getStringFromWasm0(ret[0], ret[1]); - } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); - } -} - /** * @param {string} code * @returns {string | undefined} @@ -183,70 +107,13 @@ export function parse_db_resource(code) { * @param {string} code * @returns {string} */ -export function parse_mysql(code) { +export function parse_bash(code) { let deferred2_0; let deferred2_1; try { const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_mysql(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; - return getStringFromWasm0(ret[0], ret[1]); - } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); - } -} - -/** - * @param {string} code - * @returns {string} - */ -export function parse_sql(code) { - let deferred2_0; - let deferred2_1; - try { - const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_sql(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; - return getStringFromWasm0(ret[0], ret[1]); - } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); - } -} - -/** - * @param {string} code - * @returns {string} - */ -export function parse_oracledb(code) { - let deferred2_0; - let deferred2_1; - try { - const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_oracledb(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; - return getStringFromWasm0(ret[0], ret[1]); - } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); - } -} - -/** - * @param {string} code - * @returns {string} - */ -export function parse_bigquery(code) { - let deferred2_0; - let deferred2_1; - try { - const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_bigquery(ptr0, len0); + const ret = wasm.parse_bash(ptr0, len0); deferred2_0 = ret[0]; deferred2_1 = ret[1]; return getStringFromWasm0(ret[0], ret[1]); @@ -274,6 +141,25 @@ export function parse_mssql(code) { } } +/** + * @param {string} code + * @returns {string} + */ +export function parse_oracledb(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_oracledb(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + /** * @param {string} code * @returns {string} @@ -293,6 +179,120 @@ export function parse_powershell(code) { } } +/** + * @param {string} code + * @returns {string} + */ +export function parse_bigquery(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_bigquery(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + +/** + * @param {string} code + * @returns {string} + */ +export function parse_graphql(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_graphql(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + +/** + * @param {string} code + * @returns {string} + */ +export function parse_snowflake(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_snowflake(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + +/** + * @param {string} code + * @returns {string} + */ +export function parse_duckdb(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_duckdb(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + +/** + * @param {string} code + * @returns {string} + */ +export function parse_sql(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_sql(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + +/** + * @param {string} code + * @returns {string} + */ +export function parse_mysql(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_mysql(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + const imports = { __wbindgen_placeholder__: { __wbindgen_init_externref_table: function() { diff --git a/cli/wasm/regex/windmill_parser_wasm_bg.wasm b/cli/wasm/regex/windmill_parser_wasm_bg.wasm index 12a3a0f992..45a144132d 100644 Binary files a/cli/wasm/regex/windmill_parser_wasm_bg.wasm and b/cli/wasm/regex/windmill_parser_wasm_bg.wasm differ diff --git a/frontend/README.md b/frontend/README.md index 850df24593..377a4e0ed9 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -1,5 +1,5 @@ # Windmill frontend -The Windmill frontend written in Svelte + Tailwind CSS +The Windmill frontend written in Svelte 5 + Tailwind CSS The frontend is under AGPL, see the LICENSE file at the root of this repo diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 84faba5827..17007336e0 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.574.1", + "version": "1.575.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.574.1", + "version": "1.575.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { @@ -24,7 +24,7 @@ "@redocly/json-to-json-schema": "^0.0.1", "@scalar/openapi-parser": "^0.15.0", "@tanstack/svelte-table": "npm:tanstack-table-8-svelte-5@^0.1", - "@tutorlatin/svelte-tiny-virtual-list": "^3.0.2", + "@tutorlatin/svelte-tiny-virtual-list": "^3.0.16", "@windmill-labs/svelte-dnd-action": "^0.9.44", "@xterm/addon-fit": "^0.10.0", "@xyflow/svelte": "^1.0.0", @@ -77,9 +77,9 @@ "windmill-parser-wasm-go": "1.510.1", "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", - "windmill-parser-wasm-php": "1.510.1", + "windmill-parser-wasm-php": "1.574.1", "windmill-parser-wasm-py": "1.538.0", - "windmill-parser-wasm-regex": "1.565.0", + "windmill-parser-wasm-regex": "1.574.1", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.558.1", "windmill-parser-wasm-ts": "1.565.0", @@ -3021,9 +3021,9 @@ } }, "node_modules/@tutorlatin/svelte-tiny-virtual-list": { - "version": "3.0.15", - "resolved": "https://registry.npmjs.org/@tutorlatin/svelte-tiny-virtual-list/-/svelte-tiny-virtual-list-3.0.15.tgz", - "integrity": "sha512-ew61aZNXGf0b5X+UjbOAhiNwzI21vijhB/mtBs8bpNOVYQ50TG6Qx00t+fR5C72eGnmdzguewZ2WP6QPNOTQJg==", + "version": "3.0.16", + "resolved": "https://registry.npmjs.org/@tutorlatin/svelte-tiny-virtual-list/-/svelte-tiny-virtual-list-3.0.16.tgz", + "integrity": "sha512-JQSmhRDAFZbq2rTlzn+kFXJayi5VPLxeGjD01ZVyV2ti7PlQE/ov6rQFR1c8s7Y3B1OiTcv3oEWGi3ib69V8eQ==", "license": "MIT", "engines": { "node": ">=20.17.0" @@ -13689,9 +13689,9 @@ "integrity": "sha512-AJLFiUy6af+LpUe7CddDo4+JOmw3c0K/1iOWh8NdTwXcLDj90lL6089mdsVo1apyloLgrTbcuFDzZMXVGBgtCg==" }, "node_modules/windmill-parser-wasm-php": { - "version": "1.510.1", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-php/-/windmill-parser-wasm-php-1.510.1.tgz", - "integrity": "sha512-qM+yeaqPdMuAaPpqND31ZabpeHlPxxtmRWLs11cGHOCHU32FIaZ92/icNUoAxgBhdJjRhR4GXaeZG32nWit2cg==" + "version": "1.574.1", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-php/-/windmill-parser-wasm-php-1.574.1.tgz", + "integrity": "sha512-COyid6B1RYs+bpzUCInsA4HY/WZkpDLfkQ90+AqU/TVTpzYSbAC2JCbIwy0cRElBvlhI4bQ+9Wg6hSQKMpEkpA==" }, "node_modules/windmill-parser-wasm-py": { "version": "1.538.0", @@ -13699,9 +13699,9 @@ "integrity": "sha512-s+bdIgT/fA5em3zYUwF8D14uA/dZh7iu0krZYZQqZUO7txN37hwSCVfovbMkIwm4zPbsJ50mU8DRLt7UpAPZIw==" }, "node_modules/windmill-parser-wasm-regex": { - "version": "1.565.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.565.0.tgz", - "integrity": "sha512-fcAy7SkyrCML6YgnsQ5H3utpJFxLqWssGmY+hJAibdQtAYNRASsSQZl87A46nIFO8TpKKiWioTSxZDl/9Ovtrw==" + "version": "1.574.1", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.574.1.tgz", + "integrity": "sha512-KnNBnpTGcnBPSzTQHdsjTPJbRQ81iem98eUnMGO8Zt1qgBFJ/eFZPwmUg6TZMbMITzTi0QqRCJj08eQID1FcPg==" }, "node_modules/windmill-parser-wasm-ruby": { "version": "1.526.1", diff --git a/frontend/package.json b/frontend/package.json index 3f5a13c88e..df05bf5de7 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.574.1", + "version": "1.575.0", "scripts": { "dev": "vite dev", "build": "vite build", @@ -89,7 +89,7 @@ "@redocly/json-to-json-schema": "^0.0.1", "@scalar/openapi-parser": "^0.15.0", "@tanstack/svelte-table": "npm:tanstack-table-8-svelte-5@^0.1", - "@tutorlatin/svelte-tiny-virtual-list": "^3.0.2", + "@tutorlatin/svelte-tiny-virtual-list": "^3.0.16", "@windmill-labs/svelte-dnd-action": "^0.9.44", "@xterm/addon-fit": "^0.10.0", "@xyflow/svelte": "^1.0.0", @@ -142,9 +142,9 @@ "windmill-parser-wasm-go": "1.510.1", "windmill-parser-wasm-java": "1.510.1", "windmill-parser-wasm-nu": "1.510.1", - "windmill-parser-wasm-php": "1.510.1", + "windmill-parser-wasm-php": "1.574.1", "windmill-parser-wasm-py": "1.538.0", - "windmill-parser-wasm-regex": "1.565.0", + "windmill-parser-wasm-regex": "1.574.1", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.558.1", "windmill-parser-wasm-ts": "1.565.0", diff --git a/frontend/src/lib/cancelable-promise-utils.ts b/frontend/src/lib/cancelable-promise-utils.ts new file mode 100644 index 0000000000..17e8be5a93 --- /dev/null +++ b/frontend/src/lib/cancelable-promise-utils.ts @@ -0,0 +1,84 @@ +import { CancelablePromise } from './gen' + +export namespace CancelablePromiseUtils { + export function then( + promise: CancelablePromise, + f: (value: T) => CancelablePromise + ): CancelablePromise { + let promiseToBeCanceled: CancelablePromise = promise + let p = new CancelablePromise((resolve, reject) => { + promise + .then((value1) => { + let promise2 = f(value1) + promiseToBeCanceled = promise2 + promise2.then((value2) => resolve(value2)).catch((err) => reject(err)) + }) + .catch((err) => reject(err)) + }) + p.cancel = () => promiseToBeCanceled.cancel() + return p + } + + export function pure(value: T): CancelablePromise { + return new CancelablePromise((resolve) => resolve(value)) + } + + export function err(error: any): CancelablePromise { + return new CancelablePromise((_, reject) => reject(error)) + } + + export function map( + promise: CancelablePromise, + f: (value: T) => U + ): CancelablePromise { + return then(promise, (value) => pure(f(value))) + } + + export function pipe( + promise: CancelablePromise, + f: (value: T) => void + ): CancelablePromise { + promise.then((value) => { + f(value) + }) + return promise + } + + export function catchErr( + promise: CancelablePromise, + f: (error: any) => CancelablePromise + ): CancelablePromise { + let promiseToBeCanceled: CancelablePromise = promise + let p = new CancelablePromise((resolve, reject) => { + promise + .then((value) => resolve(value)) + .catch((err) => { + let promise2 = f(err) + promiseToBeCanceled = promise2 + return promise2.then((value2) => resolve(value2)).catch((err2) => reject(err2)) + }) + .catch((err) => reject(err)) + }) + p.cancel = () => promiseToBeCanceled.cancel() + return p + } + + export function finallyDo(promise: CancelablePromise, f: () => void): CancelablePromise { + promise = map(promise, (value) => (f(), value)) + promise = catchErr(promise, (e) => (f(), err(e))) + return promise + } + + // Calls onTimeout if the promise does not settle within timeoutMs milliseconds + export function onTimeout( + promise: CancelablePromise, + timeoutMs: number, + onTimeout: () => void + ): CancelablePromise { + let timeoutId: number | undefined = setTimeout(onTimeout, timeoutMs) + promise = finallyDo(promise, () => { + if (timeoutId !== undefined) clearTimeout(timeoutId) + }) + return promise + } +} diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index 3aa15b98f8..b63d3798e5 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -400,6 +400,7 @@ const UUID_PATTERN = '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$' const IPV6_PATTERN = '^(([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))$' + function validateInput(pattern: string | undefined, v: any, required: boolean): void { if (nullable && emptyString(v)) { error = '' @@ -407,6 +408,18 @@ } else if (required && (v == undefined || v == null || v === '') && inputCat != 'object') { error = 'Required' valid && (valid = false) + } else if ( + required && + inputCat == 'list' && + extra?.['nonEmpty'] == true && + Array.isArray(v) && + v.length === 0 + ) { + error = 'Required' + valid && (valid = false) + } else if (inputCat == 'list' && !Array.isArray(v)) { + error = 'Expected an array, got ' + typeof v + ' instead' + valid && (valid = false) } else { if (inputCat == 'number' && typeof v === 'number') { let min = extra['min'] @@ -523,6 +536,7 @@ }) $effect(() => { + extra?.['nonEmpty'] let args = [pattern, value, required] as const untrack(() => validateInput(...args)) }) diff --git a/frontend/src/lib/components/ArrayTypeNarrowing.svelte b/frontend/src/lib/components/ArrayTypeNarrowing.svelte index 064faa6950..7feec29713 100644 --- a/frontend/src/lib/components/ArrayTypeNarrowing.svelte +++ b/frontend/src/lib/components/ArrayTypeNarrowing.svelte @@ -24,12 +24,14 @@ properties?: { [name: string]: SchemaProperty } } | undefined + nonEmpty?: boolean | undefined } let { canEditResourceType = false, originalType = undefined, - itemsType = $bindable() + itemsType = $bindable(), + nonEmpty = $bindable() }: Props = $props() let selected: @@ -143,6 +145,12 @@ {/each} + {#if canEditResourceType || originalType == 'string[]' || originalType == 'object[]'}
+
+ +
{/if} diff --git a/frontend/src/lib/components/RunsPage.svelte b/frontend/src/lib/components/RunsPage.svelte new file mode 100644 index 0000000000..091dcc2992 --- /dev/null +++ b/frontend/src/lib/components/RunsPage.svelte @@ -0,0 +1,1365 @@ + + + + + + + { + const func = askingForConfirmation?.onConfirm + await func?.(forceCancelInPopup) + askingForConfirmation = undefined + }} + type={askingForConfirmation?.type} + loading={askingForConfirmation?.loading} + on:canceled={() => { + askingForConfirmation = undefined + }} +> + {#if askingForConfirmation?.preContent} + 
{askingForConfirmation.preContent}
+ + {#if forceCancelInPopup} +
+

+ Force cancel is enabled. This is dangerous, only do this if you have no alternatives. + Instead of being gracefully cancelled, all jobs will be immediately sent to the completed + job table regardless of them being processed or not or part of running flows. You may end + up in an inconsistent state. +

+
+ {/if} + {/if} + + + + + {#if selectedIds.length === 1} + {#if selectedIds[0] === '-'} +
There is no information available for this job
+ {:else} + + {/if} + {/if} + + + + { + reset() + loadFromQuery() + }} +/> + +{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find((_) => _.id === $workspaceStore)?.operator_settings?.runs} + +{:else} +
+ +
+
+
+

+ Runs +

+ + + All past and schedule executions of scripts and flows, including previews. You only see + your own runs or runs of groups you belong to unless you are an admin. + +
+ + + { + jobsFilter('waiting') + }} + onJobsSuspended={() => { + jobsFilter('suspended') + }} + small={innerWidth < smallScreenWidth} + /> +
+ +
+ +
+ + {#if minTs || maxTs} + + {/if} + { + minTs = new Date(detail).toISOString() + calendarChangeTimeout && clearTimeout(calendarChangeTimeout) + calendarChangeTimeout = setTimeout(() => { + jobsLoader?.loadJobs(minTs, maxTs, true) + }, 1000) + }} + on:clear={async () => { + minTs = undefined + calendarChangeTimeout && clearTimeout(calendarChangeTimeout) + calendarChangeTimeout = setTimeout(() => { + jobsLoader?.loadJobs(minTs, maxTs, true) + }, 1000) + }} + /> + + + + {#if maxTs || minTs} + + {/if} + { + maxTs = new Date(detail).toISOString() + calendarChangeTimeout && clearTimeout(calendarChangeTimeout) + calendarChangeTimeout = setTimeout(() => { + jobsLoader?.loadJobs(minTs, maxTs, true) + }, 1000) + }} + on:clear={async () => { + maxTs = undefined + calendarChangeTimeout && clearTimeout(calendarChangeTimeout) + calendarChangeTimeout = setTimeout(() => { + jobsLoader?.loadJobs(minTs, maxTs, true) + }, 1000) + }} + /> + + + {#if minTs || maxTs} + + + + {/if} +
+ + +
+ { + if (e.detail == 'running' && maxTs != undefined) { + maxTs = undefined + } + }} + {usernames} + {folders} + {paths} + mobile={innerWidth < verySmallScreenWidth} + small={innerWidth < smallScreenWidth} + calendarSmall={!minTs && !maxTs} + /> +
+
+
+ + +
+
+
+ { + graph = detail + graphIsRunsChart = graph === 'RunChart' + }} + > + {#snippet children({ item })} + + + {/snippet} + + + {#if !graphIsRunsChart} + setLookback(0), + id: '0' + }, + { + displayName: '1 day', + action: () => setLookback(1), + id: '1' + }, + { + displayName: '3 days', + action: () => setLookback(3), + id: '3' + }, + { + displayName: '7 days', + action: () => setLookback(7), + id: '7' + } + ]} + selected={lookback.toString()} + selectedDisplayName={`${lookback} days lookback`} + > + {#snippet extraLabel()} + + {#snippet text()} + How far behind the min datetime to start considering jobs for the concurrency + graph. Change this value to include jobs started before the set time window for + the computation of the graph + {/snippet} + + {/snippet} + + {/if} +
+
+ {#if graph === 'RunChart'} + { + minTs = zoom.min.toISOString() + maxTs = zoom.max.toISOString() + manualDatePicker?.resetChoice() + jobsLoader?.loadJobs(minTs, maxTs, true) + }} + onPointClicked={(ids) => { + runsTable?.scrollToRun(ids) + }} + /> + {:else if graph === 'ConcurrencyChart'} + { + minTs = zoom.min.toISOString() + maxTs = zoom.max.toISOString() + jobsLoader?.loadJobs(minTs, maxTs, true) + }} + /> + {/if} +
+ +
+ + +
+ +
+
+ {#if selectionMode && selectableJobCount} +
+
+ +
+ +
+ {/if} + + +
+ +
+
+ { + localStorage.setItem( + 'show_schedules_in_run', + showSchedules ? 'true' : 'false' + ) + }} + options={tableTopBarWidth < 800 || selectionMode + ? {} + : { right: 'Cron schedules' }} + /> + + + +
+ +
+ { + localStorage.setItem('show_future_jobs', showFutureJobs ? 'true' : 'false') + }} + id="planned-later" + options={tableTopBarWidth < 800 || selectionMode + ? {} + : { right: 'Planned later' }} + /> + + + +
+
+ { + lastFetchWentToEnd = false + jobsLoader?.loadJobs(minTs, maxTs, true) + }} + bind:minTs + bind:maxTs + bind:selectedManualDate + {loading} + bind:this={manualDatePicker} + numberOfLastJobsToFetch={perPage} + /> + { + localStorage.setItem('auto_refresh_in_runs', autoRefresh ? 'true' : 'false') + }} + options={{ right: 'Auto-refresh' }} + textClass="whitespace-nowrap" + /> +
+
+
+ + +
+ {#if jobs} + + {:else} +
+ {#each new Array(8) as _} + + {/each} +
+ {/if} +
+
+ Per page: + - {#if usernames} - {#if $userStore?.is_admin || $userStore?.is_super_admin} - - {/if} - {#each usernames as e} - {#if e == username || $userStore?.is_admin || $userStore?.is_super_admin} - - {:else} - - {/if} - {/each} - {/if} - + (resources.value?.push(r), (resource = r))} createText="Press enter to use this value" bind:value={resource} @@ -423,6 +401,7 @@ Operation - - {#each ['Create', 'Update', 'Delete', 'Execute'] as e} - - {/each} - + -
+
{#if manager.isWaitingForResponse || manager.isLoading}
- {#if !isExpanded} + {#if !manager.isSidebarExpanded}
{/if} -
+
@@ -188,7 +89,7 @@
onSelectConversation(conversation.id, conversation.isDraft)} + onclick={() => manager.selectConversation(conversation.id, conversation.isDraft)} > {getConversationTitle(conversation)}
- {#if isExpanded} + {#if manager.isSidebarExpanded}

- {conversations.length} conversation{conversations.length !== 1 ? 's' : ''} + {manager.conversations.length} conversation{manager.conversations.length !== 1 ? 's' : ''}

{/if} diff --git a/frontend/src/lib/components/flows/header/FlowPreviewButtons.svelte b/frontend/src/lib/components/flows/header/FlowPreviewButtons.svelte index 68281556ec..d349b05500 100644 --- a/frontend/src/lib/components/flows/header/FlowPreviewButtons.svelte +++ b/frontend/src/lib/components/flows/header/FlowPreviewButtons.svelte @@ -50,14 +50,14 @@ flowPreviewContent?.test() } - export async function runPreview(): Promise { + export async function runPreview(conversationId?: string): Promise { if (!previewOpen) { deferContent = true await tick() } previewMode = 'whole' flowPreviewContent?.refresh() - return await flowPreviewContent?.test() + return await flowPreviewContent?.test(conversationId) } export function cancelTest() { diff --git a/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte b/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte index 7f80abcdc7..494715ad29 100644 --- a/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte +++ b/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte @@ -494,7 +494,12 @@ {/if} {#if id !== 'preprocessor'} -
+ +
+ Load next {perPage} jobs + {/if}
{/snippet} diff --git a/frontend/src/lib/components/schema/PropertyEditor.svelte b/frontend/src/lib/components/schema/PropertyEditor.svelte index f3b94f24dd..037aea6cf9 100644 --- a/frontend/src/lib/components/schema/PropertyEditor.svelte +++ b/frontend/src/lib/components/schema/PropertyEditor.svelte @@ -32,6 +32,7 @@ title?: string | undefined placeholder?: string | undefined properties?: Record | undefined + nonEmpty?: boolean | undefined isFlowInput?: boolean isAppInput?: boolean order?: string[] | undefined @@ -57,6 +58,7 @@ pattern = $bindable(undefined), enum_ = $bindable(undefined), extra = $bindable({}), + nonEmpty = $bindable(undefined), minW = true, customErrorMessage = $bindable(undefined), title = $bindable(undefined), @@ -219,6 +221,7 @@ originalType={extra['originalType']} bind:itemsType canEditResourceType={isFlowInput || isAppInput} + bind:nonEmpty /> {:else if type == 'string' || ['number', 'integer'].includes(type ?? '')}
diff --git a/frontend/src/lib/components/select/Select.svelte b/frontend/src/lib/components/select/Select.svelte index 287c620a7b..0327444bcf 100644 --- a/frontend/src/lib/components/select/Select.svelte +++ b/frontend/src/lib/components/select/Select.svelte @@ -1,4 +1,7 @@ - - - - { - const func = askingForConfirmation?.onConfirm - await func?.(forceCancelInPopup) - askingForConfirmation = undefined - }} - type={askingForConfirmation?.type} - loading={askingForConfirmation?.loading} - on:canceled={() => { - askingForConfirmation = undefined - }} -> - {#if askingForConfirmation?.preContent} - 
{askingForConfirmation.preContent}
- - {#if forceCancelInPopup} -
-

- Force cancel is enabled. This is dangerous, only do this if you have no alternatives. - Instead of being gracefully cancelled, all jobs will be immediately sent to the completed - job table regardless of them being processed or not or part of running flows. You may end - up in an inconsistent state. -

-
- {/if} - {/if} - - - - - {#if selectedIds.length === 1} - {#if selectedIds[0] === '-'} -
There is no information available for this job
- {:else} - - {/if} - {/if} - - - - { - reset() - loadFromQuery() - }} -/> - -{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find((_) => _.id === $workspaceStore)?.operator_settings?.runs} - -{:else} -
- -
-
-
-

- Runs -

- - - All past and schedule executions of scripts and flows, including previews. You only see - your own runs or runs of groups you belong to unless you are an admin. - -
- - - { - jobsFilter('waiting') - }} - onJobsSuspended={() => { - jobsFilter('suspended') - }} - small={innerWidth < smallScreenWidth} - /> -
- -
- -
- - {#if minTs || maxTs} - - {/if} - { - minTs = new Date(detail).toISOString() - calendarChangeTimeout && clearTimeout(calendarChangeTimeout) - calendarChangeTimeout = setTimeout(() => { - jobsLoader?.loadJobs(minTs, maxTs, true) - }, 1000) - }} - on:clear={async () => { - minTs = undefined - calendarChangeTimeout && clearTimeout(calendarChangeTimeout) - calendarChangeTimeout = setTimeout(() => { - jobsLoader?.loadJobs(minTs, maxTs, true) - }, 1000) - }} - /> - - - - {#if maxTs || minTs} - - {/if} - { - maxTs = new Date(detail).toISOString() - calendarChangeTimeout && clearTimeout(calendarChangeTimeout) - calendarChangeTimeout = setTimeout(() => { - jobsLoader?.loadJobs(minTs, maxTs, true) - }, 1000) - }} - on:clear={async () => { - maxTs = undefined - calendarChangeTimeout && clearTimeout(calendarChangeTimeout) - calendarChangeTimeout = setTimeout(() => { - jobsLoader?.loadJobs(minTs, maxTs, true) - }, 1000) - }} - /> - - - {#if minTs || maxTs} - - - - {/if} -
- - -
- { - if (e.detail == 'running' && maxTs != undefined) { - maxTs = undefined - } - }} - {usernames} - {folders} - {paths} - mobile={innerWidth < verySmallScreenWidth} - small={innerWidth < smallScreenWidth} - calendarSmall={!minTs && !maxTs} - /> -
-
-
- - -
-
-
- { - graph = detail - graphIsRunsChart = graph === 'RunChart' - }} - > - {#snippet children({ item })} - - - {/snippet} - - - {#if !graphIsRunsChart} - setLookback(0), - id: '0' - }, - { - displayName: '1 day', - action: () => setLookback(1), - id: '1' - }, - { - displayName: '3 days', - action: () => setLookback(3), - id: '3' - }, - { - displayName: '7 days', - action: () => setLookback(7), - id: '7' - } - ]} - selected={lookback.toString()} - selectedDisplayName={`${lookback} days lookback`} - > - {#snippet extraLabel()} - - {#snippet text()} - How far behind the min datetime to start considering jobs for the concurrency - graph. Change this value to include jobs started before the set time window for - the computation of the graph - {/snippet} - - {/snippet} - - {/if} -
-
- {#if graph === 'RunChart'} - { - minTs = zoom.min.toISOString() - maxTs = zoom.max.toISOString() - manualDatePicker?.resetChoice() - jobsLoader?.loadJobs(minTs, maxTs, true) - }} - onPointClicked={(ids) => { - runsTable?.scrollToRun(ids) - }} - /> - {:else if graph === 'ConcurrencyChart'} - { - minTs = zoom.min.toISOString() - maxTs = zoom.max.toISOString() - jobsLoader?.loadJobs(minTs, maxTs, true) - }} - /> - {/if} -
- -
- - -
- -
-
- {#if selectionMode && selectableJobCount} -
-
- -
- -
- {/if} - - -
- -
-
- { - localStorage.setItem( - 'show_schedules_in_run', - showSchedules ? 'true' : 'false' - ) - }} - options={tableTopBarWidth < 800 || selectionMode - ? {} - : { right: 'Cron schedules' }} - /> - - - -
- -
- { - localStorage.setItem('show_future_jobs', showFutureJobs ? 'true' : 'false') - }} - id="planned-later" - options={tableTopBarWidth < 800 || selectionMode - ? {} - : { right: 'Planned later' }} - /> - - - -
-
- { - lastFetchWentToEnd = false - jobsLoader?.loadJobs(minTs, maxTs, true) - }} - bind:minTs - bind:maxTs - bind:selectedManualDate - {loading} - bind:this={manualDatePicker} - /> - { - localStorage.setItem('auto_refresh_in_runs', autoRefresh ? 'true' : 'false') - }} - options={{ right: 'Auto-refresh' }} - textClass="whitespace-nowrap" - /> -
-
-
- - -
- {#if jobs} - - {:else} -
- {#each new Array(8) as _} - - {/each} -
- {/if} -
-
- - 0}> - {#if selectionMode === 're-run'} - - {:else if selectedIds.length === 1} - {#if selectedIds[0] === '-'} -
There is no information available for this job
- {:else} - - {/if} - {:else if selectedIds.length > 1} -
There are {selectedIds.length} jobs selected. Choose 1 to see detailed information
- {/if} - - -
-
-{/if} +{#key perPage} + +{/key} diff --git a/frontend/src/routes/a/[...path]/+page.svelte b/frontend/src/routes/a/[...path]/+page.svelte index 3fec55ce0a..6aa05e6110 100644 --- a/frontend/src/routes/a/[...path]/+page.svelte +++ b/frontend/src/routes/a/[...path]/+page.svelte @@ -139,7 +139,8 @@
{#if !jwtError} { + onLoginSuccess={() => { + // window.location.reload() loadApp() app = app diff --git a/frontend/src/routes/public/[workspace]/[...secret]/+page.svelte b/frontend/src/routes/public/[workspace]/[...secret]/+page.svelte index 7b7541b386..fadae2baf9 100644 --- a/frontend/src/routes/public/[workspace]/[...secret]/+page.svelte +++ b/frontend/src/routes/public/[workspace]/[...secret]/+page.svelte @@ -125,7 +125,8 @@
{#if !jwtError} { + onLoginSuccess={() => { + console.log('login') // window.location.reload() loadUser().then(() => { loadApp() diff --git a/frontend/vite.config.js b/frontend/vite.config.js index 0ac847c255..9fc6660220 100644 --- a/frontend/vite.config.js +++ b/frontend/vite.config.js @@ -11,7 +11,7 @@ const version = JSON.parse(json) const config = { server: { https: process.env.HTTPS === 'true', - allowedHosts: ['localhost', '127.0.0.1', '0.0.0.0', 'rubendev.wimill.xyz'], + allowedHosts: ['localhost', '127.0.0.1', '0.0.0.0', 'rubendev.wimill.xyz', 'windmill.xyz'], port: 3000, proxy: { '^/api/w/[^/]+/s3_proxy/.*': { diff --git a/lsp/Pipfile b/lsp/Pipfile index 5207532db2..8e69ddb427 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.574.1" -wmill_pg = ">=1.574.1" +wmill = ">=1.575.0" +wmill_pg = ">=1.575.0" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 156eac8a54..b3dfd7c8f4 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.574.1 + version: 1.575.0 title: OpenFlow Spec contact: name: Ruben Fiszel diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index ce25df0d21..b75c5d2e72 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. - ModuleVersion = '1.574.1' + ModuleVersion = '1.575.0' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 7453c8d7de..c4d9c9329e 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.574.1" +version = "1.575.0" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index cd9d5aecc8..b148a05a68 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.574.1" +version = "1.575.0" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index 1ded4087d4..564bf9b984 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.574.1", + "version": "1.575.0", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 1bc35510dc..1efa54ef76 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.574.1", + "version": "1.575.0", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index 50a7825cc3..f227cd375d 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.574.1 +1.575.0