diff --git a/CHANGELOG.md b/CHANGELOG.md index efd125ea43..fa67e06672 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,19 @@ # Changelog +## [1.576.0](https://github.com/windmill-labs/windmill/compare/v1.575.4...v1.576.0) (2025-11-14) + + +### Features + +* add support for switch and attributes in pwsh params ([#7143](https://github.com/windmill-labs/windmill/issues/7143)) ([c16bef8](https://github.com/windmill-labs/windmill/commit/c16bef8f296645ff873f9d8d28e3dcb50a65e304)) +* **ai:** handle aws bedrock as provider ([#7131](https://github.com/windmill-labs/windmill/issues/7131)) ([30eb9aa](https://github.com/windmill-labs/windmill/commit/30eb9aae25eeb563ad119ef93f3ff1ab17c66d75)) +* webhook by flow version ([#7062](https://github.com/windmill-labs/windmill/issues/7062)) ([09cdfb4](https://github.com/windmill-labs/windmill/commit/09cdfb4556748903dc5bbf53ef3356ac97c57d90)) + + +### Bug Fixes + +* use proper TLS connector for DuckLake instance catalog setup ([#7138](https://github.com/windmill-labs/windmill/issues/7138)) ([cf36fe3](https://github.com/windmill-labs/windmill/commit/cf36fe3bb1beec80fa84dc342a8a38cc7369bc4d)) + ## [1.575.4](https://github.com/windmill-labs/windmill/compare/v1.575.3...v1.575.4) (2025-11-13) diff --git a/backend/.sqlx/query-7cab8475996f0abd7ca7888b460bd3faf3bef6e34200c258e3704981817c241e.json b/backend/.sqlx/query-7cab8475996f0abd7ca7888b460bd3faf3bef6e34200c258e3704981817c241e.json new file mode 100644 index 0000000000..4311abf5ff --- /dev/null +++ b/backend/.sqlx/query-7cab8475996f0abd7ca7888b460bd3faf3bef6e34200c258e3704981817c241e.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT \n path \n FROM \n flow_version \n WHERE \n id = $1 AND \n workspace_id = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Int8", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "7cab8475996f0abd7ca7888b460bd3faf3bef6e34200c258e3704981817c241e" +} diff --git a/backend/.sqlx/query-a7468e9054beed88636786c5495ac3b9d9a6086ae6212ad9237b39a0346d7d26.json b/backend/.sqlx/query-a7468e9054beed88636786c5495ac3b9d9a6086ae6212ad9237b39a0346d7d26.json new file mode 100644 index 0000000000..cead5d7019 --- /dev/null +++ b/backend/.sqlx/query-a7468e9054beed88636786c5495ac3b9d9a6086ae6212ad9237b39a0346d7d26.json @@ -0,0 +1,66 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n flow_version.id AS version,\n flow_version.value->>'early_return' as early_return, \n flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor, \n (flow_version.value->>'chat_input_enabled')::boolean as chat_input_enabled, \n flow.tag, \n flow.dedicated_worker, \n flow.on_behalf_of_email, \n flow.edited_by\n FROM \n flow_version\n INNER JOIN flow\n ON flow.path = flow_version.path AND\n flow.workspace_id = flow_version.workspace_id\n WHERE \n flow_version.workspace_id = $1 AND\n flow_version.path = $2 AND\n flow_version.id = $3\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "version", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "early_return", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "has_preprocessor", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "chat_input_enabled", + "type_info": "Bool" + }, + { + "ordinal": 4, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "dedicated_worker", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "on_behalf_of_email", + "type_info": "Text" + }, + { + "ordinal": 7, + "name": "edited_by", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false, + null, + null, + null, + true, + true, + true, + false + ] + }, + "hash": "a7468e9054beed88636786c5495ac3b9d9a6086ae6212ad9237b39a0346d7d26" +} diff --git a/backend/.sqlx/query-b82f97f2a858e20b1fb6d9b0b3eab5bab2bafa91689eac5073c1678c338afb27.json b/backend/.sqlx/query-b82f97f2a858e20b1fb6d9b0b3eab5bab2bafa91689eac5073c1678c338afb27.json new file mode 100644 index 0000000000..64b774cbd9 --- /dev/null +++ b/backend/.sqlx/query-b82f97f2a858e20b1fb6d9b0b3eab5bab2bafa91689eac5073c1678c338afb27.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT \n path \n FROM \n flow_version \n WHERE \n id = $1 AND \n workspace_id = $2\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Int8", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "b82f97f2a858e20b1fb6d9b0b3eab5bab2bafa91689eac5073c1678c338afb27" +} diff --git a/backend/.sqlx/query-e1409c67b93881cf68f6ac9c8bae0856cf426c7e7860c8b5b799972baa8e6945.json b/backend/.sqlx/query-e1409c67b93881cf68f6ac9c8bae0856cf426c7e7860c8b5b799972baa8e6945.json deleted file mode 100644 index da6ca296b1..0000000000 --- a/backend/.sqlx/query-e1409c67b93881cf68f6ac9c8bae0856cf426c7e7860c8b5b799972baa8e6945.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor, (flow_version.value->>'chat_input_enabled')::boolean as chat_input_enabled, on_behalf_of_email, edited_by, flow_version.id AS version\n FROM flow\n INNER JOIN flow_version\n ON flow_version.id = $3\n WHERE flow.path = $1 and flow.workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "tag", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "dedicated_worker", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "early_return", - "type_info": "Text" - }, - { - "ordinal": 3, - "name": "has_preprocessor", - "type_info": "Bool" - }, - { - "ordinal": 4, - "name": "chat_input_enabled", - "type_info": "Bool" - }, - { - "ordinal": 5, - "name": "on_behalf_of_email", - "type_info": "Text" - }, - { - "ordinal": 6, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 7, - "name": "version", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Int8" - ] - }, - "nullable": [ - true, - true, - null, - null, - null, - true, - false, - false - ] - }, - "hash": "e1409c67b93881cf68f6ac9c8bae0856cf426c7e7860c8b5b799972baa8e6945" -} diff --git a/backend/.sqlx/query-e2474b7855c8b08f927f2b987421e773e537a8eb0a113477764bbab14e1f3a3d.json b/backend/.sqlx/query-f0c9670c27d9a3b8e8f316fd28f7d3ede2ecc3225ff2879ff5dfd59c4ef236d5.json similarity index 69% rename from backend/.sqlx/query-e2474b7855c8b08f927f2b987421e773e537a8eb0a113477764bbab14e1f3a3d.json rename to backend/.sqlx/query-f0c9670c27d9a3b8e8f316fd28f7d3ede2ecc3225ff2879ff5dfd59c4ef236d5.json index c7c7214a87..19842dda7d 100644 --- a/backend/.sqlx/query-e2474b7855c8b08f927f2b987421e773e537a8eb0a113477764bbab14e1f3a3d.json +++ b/backend/.sqlx/query-f0c9670c27d9a3b8e8f316fd28f7d3ede2ecc3225ff2879ff5dfd59c4ef236d5.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT\n value->'preprocessor_module'->'value' as \"preprocessor_module: _\",\n schema as \"schema: _\"\n FROM flow_version\n WHERE\n path = $1\n AND workspace_id = $2\n ORDER BY created_at DESC\n LIMIT 1", + "query": "SELECT\n value->'preprocessor_module'->'value' as \"preprocessor_module: _\",\n schema as \"schema: _\"\n FROM flow_version\n WHERE\n id = $1\n AND workspace_id = $2", "describe": { "columns": [ { @@ -16,7 +16,7 @@ ], "parameters": { "Left": [ - "Text", + "Int8", "Text" ] }, @@ -25,5 +25,5 @@ true ] }, - "hash": "e2474b7855c8b08f927f2b987421e773e537a8eb0a113477764bbab14e1f3a3d" + "hash": "f0c9670c27d9a3b8e8f316fd28f7d3ede2ecc3225ff2879ff5dfd59c4ef236d5" } diff --git a/backend/.sqlx/query-fc9753f501974c4b570d710c7621ff1f6787a852bb27acf49593af963a6aacca.json b/backend/.sqlx/query-fc9753f501974c4b570d710c7621ff1f6787a852bb27acf49593af963a6aacca.json new file mode 100644 index 0000000000..b699cc582a --- /dev/null +++ b/backend/.sqlx/query-fc9753f501974c4b570d710c7621ff1f6787a852bb27acf49593af963a6aacca.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path FROM flow_version WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Int8", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "fc9753f501974c4b570d710c7621ff1f6787a852bb27acf49593af963a6aacca" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 5ec4e577b8..8d5f4e0ac0 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -199,22 +199,22 @@ dependencies = [ [[package]] name = "anstyle-query" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "anstyle-wincon" -version = "3.0.10" +version = "3.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1026,7 +1026,7 @@ dependencies = [ "http 1.3.1", "http-body 0.4.6", "hyper 0.14.32", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-rustls 0.24.2", "hyper-rustls 0.27.7", "hyper-util", @@ -1182,7 +1182,7 @@ dependencies = [ "http 1.3.1", "http-body 1.0.1", "http-body-util", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-util", "itoa", "matchit", @@ -1569,7 +1569,7 @@ dependencies = [ "hex", "http 1.3.1", "http-body-util", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-named-pipe", "hyper-util", "hyperlocal", @@ -1799,9 +1799,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" dependencies = [ "serde", ] @@ -1945,9 +1945,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.45" +version = "1.2.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35900b6c8d709fb1d854671ae27aeaa9eec2f8b01b364e1619a40da3e6fe2afe" +checksum = "b97463e1064cb1b1c1384ad0a0b9c8abd0988e2a91f52606c80ef14aadb63e36" dependencies = [ "find-msvc-tools", "jobserver", @@ -3625,7 +3625,7 @@ dependencies = [ "hickory-resolver", "http 1.3.1", "http-body-util", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-rustls 0.27.7", "hyper-util", "ipnet", @@ -3713,7 +3713,7 @@ dependencies = [ "http 1.3.1", "httparse", "hyper 0.14.32", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-util", "itertools 0.10.5", "memmem", @@ -3906,7 +3906,7 @@ dependencies = [ "hkdf", "http 1.3.1", "http-body-util", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-util", "idna", "indexmap 2.11.1", @@ -4177,7 +4177,7 @@ dependencies = [ "http 1.3.1", "http-body-util", "hyper 0.14.32", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-util", "libc", "log", @@ -4231,7 +4231,7 @@ dependencies = [ "deno_error", "deno_tls", "http-body-util", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-rustls 0.27.7", "hyper-util", "log", @@ -4361,7 +4361,7 @@ dependencies = [ "h2 0.4.12", "http 1.3.1", "http-body-util", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-util", "once_cell", "rustls-tokio-stream", @@ -5225,7 +5225,7 @@ dependencies = [ "base64 0.21.7", "bytes", "http-body-util", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-util", "pin-project", "rand 0.8.5", @@ -5292,9 +5292,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" +checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" [[package]] name = "fixedbitset" @@ -6604,9 +6604,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1744436df46f0bde35af3eda22aeaba453aada65d8f1c171cd8a5f59030bd69f" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" dependencies = [ "atomic-waker", "bytes", @@ -6635,7 +6635,7 @@ dependencies = [ "futures-util", "headers", "http 1.3.1", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-rustls 0.27.7", "hyper-util", "pin-project-lite", @@ -6652,7 +6652,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" dependencies = [ "hex", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-util", "pin-project-lite", "tokio", @@ -6683,7 +6683,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ "http 1.3.1", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-util", "log", "rustls 0.23.29", @@ -6701,7 +6701,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.8.0", + "hyper 1.8.1", "hyper-util", "pin-project-lite", "tokio", @@ -6729,7 +6729,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-util", "native-tls", "tokio", @@ -6750,7 +6750,7 @@ dependencies = [ "futures-util", "http 1.3.1", "http-body 1.0.1", - "hyper 1.8.0", + "hyper 1.8.1", "ipnet", "libc", "percent-encoding", @@ -6771,7 +6771,7 @@ checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" dependencies = [ "hex", "http-body-util", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-util", "pin-project-lite", "tokio", @@ -7419,7 +7419,7 @@ dependencies = [ "http 1.3.1", "http-body 1.0.1", "http-body-util", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-http-proxy", "hyper-rustls 0.27.7", "hyper-timeout", @@ -8946,7 +8946,7 @@ dependencies = [ "http-body-util", "httparse", "humantime", - "hyper 1.8.0", + "hyper 1.8.1", "itertools 0.14.0", "md-5 0.10.6", "parking_lot 0.12.5", @@ -10712,7 +10712,7 @@ dependencies = [ "http 1.3.1", "http-body 1.0.1", "http-body-util", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-rustls 0.27.7", "hyper-tls 0.6.0", "hyper-util", @@ -10772,7 +10772,7 @@ dependencies = [ "futures", "getrandom 0.2.16", "http 1.3.1", - "hyper 1.8.0", + "hyper 1.8.1", "parking_lot 0.11.2", "reqwest 0.12.24", "reqwest-middleware", @@ -13945,7 +13945,7 @@ dependencies = [ "http 1.3.1", "http-body 1.0.1", "http-body-util", - "hyper 1.8.0", + "hyper 1.8.1", "hyper-timeout", "hyper-util", "percent-encoding", @@ -15148,7 +15148,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.575.4" +version = "1.576.0" dependencies = [ "anyhow", "aws-sdk-config", @@ -15208,7 +15208,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.575.4" +version = "1.576.0" dependencies = [ "anyhow", "argon2", @@ -15251,7 +15251,7 @@ dependencies = [ "hf-hub", "hmac", "http 1.3.1", - "hyper 1.8.0", + "hyper 1.8.1", "indexmap 2.11.1", "itertools 0.14.0", "jsonwebtoken 8.3.0", @@ -15269,6 +15269,7 @@ dependencies = [ "pg_escape", "pin-project", "postgres-native-tls 0.5.0", + "postgres-native-tls 0.5.1", "prometheus", "quick_cache", "rand 0.9.0", @@ -15328,7 +15329,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.575.4" +version = "1.576.0" dependencies = [ "base64 0.22.1", "chrono", @@ -15343,7 +15344,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.575.4" +version = "1.576.0" dependencies = [ "chrono", "lazy_static", @@ -15357,7 +15358,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.575.4" +version = "1.576.0" dependencies = [ "anyhow", "axum", @@ -15376,7 +15377,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.575.4" +version = "1.576.0" dependencies = [ "anyhow", "async-recursion", @@ -15404,7 +15405,7 @@ dependencies = [ "globset", "hex", "hmac", - "hyper 1.8.0", + "hyper 1.8.1", "indexmap 2.11.1", "itertools 0.14.0", "jsonwebtoken 8.3.0", @@ -15461,7 +15462,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.575.4" +version = "1.576.0" dependencies = [ "regex", "serde", @@ -15476,7 +15477,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.575.4" +version = "1.576.0" dependencies = [ "anyhow", "bytes", @@ -15500,7 +15501,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.575.4" +version = "1.576.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15512,7 +15513,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.575.4" +version = "1.576.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15521,7 +15522,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.575.4" +version = "1.576.0" dependencies = [ "anyhow", "lazy_static", @@ -15533,7 +15534,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.575.4" +version = "1.576.0" dependencies = [ "anyhow", "serde_json", @@ -15545,7 +15546,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.575.4" +version = "1.576.0" dependencies = [ "anyhow", "gosyn", @@ -15557,7 +15558,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.575.4" +version = "1.576.0" dependencies = [ "anyhow", "lazy_static", @@ -15569,7 +15570,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.575.4" +version = "1.576.0" dependencies = [ "anyhow", "serde_json", @@ -15581,7 +15582,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.575.4" +version = "1.576.0" dependencies = [ "anyhow", "nu-parser", @@ -15592,7 +15593,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.575.4" +version = "1.576.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15603,7 +15604,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.575.4" +version = "1.576.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -15615,7 +15616,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.575.4" +version = "1.576.0" dependencies = [ "anyhow", "async-recursion", @@ -15638,7 +15639,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.575.4" +version = "1.576.0" dependencies = [ "anyhow", "lazy_static", @@ -15652,7 +15653,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.575.4" +version = "1.576.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -15669,7 +15670,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.575.4" +version = "1.576.0" dependencies = [ "anyhow", "lazy_static", @@ -15683,7 +15684,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.575.4" +version = "1.576.0" dependencies = [ "anyhow", "lazy_static", @@ -15701,7 +15702,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.575.4" +version = "1.576.0" dependencies = [ "anyhow", "serde", @@ -15712,7 +15713,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.575.4" +version = "1.576.0" dependencies = [ "anyhow", "async-recursion", @@ -15747,7 +15748,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.575.4" +version = "1.576.0" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -15757,7 +15758,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.575.4" +version = "1.576.0" dependencies = [ "anyhow", "async-once-cell", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 457c9f91dd..c78695440c 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.575.4" +version = "1.576.0" authors.workspace = true edition.workspace = true @@ -33,7 +33,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal"] [workspace.package] -version = "1.575.4" +version = "1.576.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 202518370e..2cbef722f7 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -90ef44ceb8220e7e54a2ac40756c96fe003d8695 \ No newline at end of file +0772f05a60809e9d9a11c25173a246724357520c \ No newline at end of file diff --git a/backend/parsers/windmill-parser-bash/src/lib.rs b/backend/parsers/windmill-parser-bash/src/lib.rs index 5bfe6b20c5..f1652ed5d6 100644 --- a/backend/parsers/windmill-parser-bash/src/lib.rs +++ b/backend/parsers/windmill-parser-bash/src/lib.rs @@ -46,8 +46,6 @@ pub fn parse_powershell_sig(code: &str) -> anyhow::Result { lazy_static::lazy_static! { static ref RE_BASH: Regex = Regex::new(r#"(?m)^(\w+)="\$(?:(\d+)|\{(\d+)\}|\{(\d+):-(.*)\})"(?:[\t ]*)?(?:#.*)?\r?$"#).unwrap(); - - static ref RE_POWERSHELL_ARGS: Regex = Regex::new(r#"(?:\[([\w\[\]]+)\])?\$(\w+)[\t ]*(?:=[\t ]*(?:(?:(?:"|')([^"\n\r\$]*)(?:"|'))|([\d.]+)))?\r?"#).unwrap(); } fn parse_bash_file(code: &str) -> anyhow::Result>> { @@ -98,10 +96,12 @@ pub fn extract_powershell_param_block(code: &str, include_keyword: bool) -> Opti let lower_code = code.to_lowercase(); let param_start = lower_code.find("param")?; - // Verify that only comments and whitespace appear before "param" + // Verify that only comments, whitespace, and [CmdletBinding()] appear before "param" let before_param = &code[..param_start]; let mut chars = before_param.chars().peekable(); let mut in_block_comment = false; + let mut in_attribute_bracket = false; + let mut bracket_depth = 0; while let Some(ch) = chars.next() { if in_block_comment { @@ -110,6 +110,19 @@ pub fn extract_powershell_param_block(code: &str, include_keyword: bool) -> Opti chars.next(); // consume '>' in_block_comment = false; } + } else if in_attribute_bracket { + // Track bracket depth to handle nested brackets/parens in attributes + match ch { + '[' => bracket_depth += 1, + ']' => { + bracket_depth -= 1; + if bracket_depth == 0 { + in_attribute_bracket = false; + } + } + // Allow parentheses inside attributes (e.g., [CmdletBinding()]) + _ => {} + } } else { match ch { // Start of block comment: <# @@ -126,6 +139,11 @@ pub fn extract_powershell_param_block(code: &str, include_keyword: bool) -> Opti chars.next(); } } + // Start of attribute bracket (e.g., [CmdletBinding()]) + '[' => { + in_attribute_bracket = true; + bracket_depth = 1; + } // Whitespace is allowed c if c.is_whitespace() => {} // Any other character means there's code before param @@ -134,8 +152,8 @@ pub fn extract_powershell_param_block(code: &str, include_keyword: bool) -> Opti } } - // If we're still in a block comment at the end, it's unclosed - invalid - if in_block_comment { + // If we're still in a block comment or unclosed attribute bracket, it's invalid + if in_block_comment || in_attribute_bracket { return None; } @@ -214,108 +232,269 @@ pub fn extract_powershell_param_block(code: &str, include_keyword: bool) -> Opti None } -enum ParserState { - Normal, - InSingleQuote, - InDoubleQuote, -} -fn split_pwsh_args(code: &str) -> Vec<&str> { - let mut chars = code.char_indices().peekable(); - let mut state = ParserState::Normal; - let mut splits = vec![]; - let mut last_idx = 0; - while let Some((idx, char)) = chars.next() { - match (&state, char) { - (ParserState::Normal, '\'') => { - state = ParserState::InSingleQuote; - } - (ParserState::Normal, '"') => { - state = ParserState::InDoubleQuote; - } - (ParserState::InSingleQuote, '\'') => { - state = ParserState::Normal; - } - (ParserState::InDoubleQuote, '"') => { - state = ParserState::Normal; - } - (ParserState::Normal, ',') => { - splits.push(&code[last_idx..idx]); - last_idx = idx + 1; // skip the comma - } - _ => {} - } - } - - if last_idx < code.len() { - splits.push(&code[last_idx..]); - } - - splits -} - fn parse_powershell_single_typ(typ: &str) -> Typ { match typ.to_lowercase().as_str() { "string" => Typ::Str(None), "int" | "long" => Typ::Int, "decimal" | "double" | "single" => Typ::Float, "datetime" => Typ::Datetime, - "bool" => Typ::Bool, + "bool" | "switch" => Typ::Bool, "pscustomobject" => Typ::Object(ObjectType::new(None, None)), _ => Typ::Str(None), } } -fn parse_powershell_file(code: &str) -> anyhow::Result>> { - let param_wrapper = extract_powershell_param_block(code, false); - let mut args = vec![]; - if let Some(param_wrapper) = param_wrapper { - let params = split_pwsh_args(param_wrapper); - for param in params { - if let Some(cap) = RE_POWERSHELL_ARGS.captures(param) { - let typ = cap.get(1).map(|x| x.as_str().to_string()); - let name = cap.get(2).unwrap().as_str().to_string(); +/// Single-pass PowerShell parameter parser. +/// Parses the content of a param() block and extracts all parameter information. +/// +/// This function processes PowerShell parameter declarations in a single pass, handling: +/// - Parameter attributes: [Parameter(Mandatory)], [Parameter(Mandatory=$true)], [ValidateSet(...)], etc. +/// - Type annotations: [string], [int[]], [PSCustomObject], etc. +/// - Variable names: $Name, $Value, etc. +/// - Default values: = 'text', = 25, = $env:VAR, etc. +/// - Mandatory detection: Parameters with Mandatory attribute are marked as required +fn parse_powershell_parameters(content: &str) -> anyhow::Result> { + #[derive(Debug, PartialEq)] + enum State { + Normal, + InSingleQuote, + InDoubleQuote, + InBracket, + } - let mut parsed_typ = if let Some(typ) = typ { - if typ.as_str().ends_with("[]") { - Some(Typ::List(Box::new(parse_powershell_single_typ( - typ.as_str().strip_suffix("[]").unwrap(), - )))) - } else { - Some(parse_powershell_single_typ(typ.as_str())) + let mut args = Vec::new(); + let mut chars = content.char_indices().peekable(); + let mut state = State::Normal; + let mut bracket_depth: i32 = 0; + let mut paren_depth: i32 = 0; + + // Current parameter being built + let mut type_annotation: Option = None; + let mut var_name: Option = None; + let mut default_value: Option = None; + let mut is_mandatory = false; + + // Track position for extracting text + let mut last_bracket_start = None; + let mut found_dollar = false; + + while let Some((idx, ch)) = chars.next() { + match state { + State::InSingleQuote => { + if ch == '\'' { + state = State::Normal; + } + } + State::InDoubleQuote => { + if ch == '"' { + // Check for escape character + if idx > 0 && content.chars().nth(idx - 1) != Some('`') { + state = State::Normal; } - } else { - None - }; + } + } + State::InBracket => { + match ch { + '[' => bracket_depth += 1, + ']' => { + bracket_depth -= 1; + if bracket_depth == 0 { + // Extract the bracket content + if let Some(start) = last_bracket_start { + let bracket_content = &content[start + 1..idx]; - let default = if let Some(x) = cap.get(3) { - Some(json!(x.as_str().to_string())) - } else if let Some(x) = cap.get(4) { - if parsed_typ.is_none() { - if x.as_str().parse::().is_ok() { - parsed_typ = Some(Typ::Int); - } else if x.as_str().parse::().is_ok() { - parsed_typ = Some(Typ::Float); + // Check if this is a Parameter attribute with Mandatory (case-insensitive) + let lower = bracket_content.to_lowercase(); + if lower.starts_with("parameter(") || lower.starts_with("parameter ") { + // Check for Mandatory (case-insensitive) + if lower.contains("mandatory") { + // Check if it's explicitly set to false + if !lower.contains("mandatory=$false") && !lower.contains("mandatory = $false") { + is_mandatory = true; + } + } + } + + // Check if this looks like a type (simple word, possibly with []) + let is_type = !bracket_content.contains('(') + && !bracket_content.contains('=') + && (bracket_content.chars().next().unwrap_or(' ').is_alphabetic() + || bracket_content.starts_with('[')); + + if is_type && !found_dollar { + type_annotation = Some(bracket_content.to_string()); + } + } + state = State::Normal; + last_bracket_start = None; } } - serde_json::Number::from_str(x.as_str()) - .ok() - .map(serde_json::Value::Number) - } else { - None - }; + '(' => paren_depth += 1, + ')' => paren_depth = paren_depth.saturating_sub(1), + _ => {} + } + } + State::Normal => { + match ch { + '\'' => state = State::InSingleQuote, + '"' => state = State::InDoubleQuote, + '[' => { + state = State::InBracket; + bracket_depth = 1; + last_bracket_start = Some(idx); + } + '$' => { + found_dollar = true; + // Extract variable name + let name_start = idx + 1; + let mut name_end = name_start; + while let Some(&(_, next_ch)) = chars.peek() { + if next_ch.is_alphanumeric() || next_ch == '_' { + name_end += 1; + chars.next(); + } else { + break; + } + } + var_name = Some(content[name_start..name_end].to_string()); + } + '=' if found_dollar => { + // Extract default value + // Skip whitespace after = + while let Some(&(_, next_ch)) = chars.peek() { + if next_ch.is_whitespace() { + chars.next(); + } else { + break; + } + } - args.push(Arg { - name: name, - typ: parsed_typ.unwrap_or(Typ::Str(None)), - default: default.clone(), - otyp: None, - has_default: default.is_some(), - oidx: None, - }); + let default_start = chars.peek().map(|(i, _)| *i).unwrap_or(content.len()); + let mut default_end = default_start; + let mut in_string = false; + let mut string_char = ' '; + + while let Some((i, ch)) = chars.peek().copied() { + if in_string { + if ch == string_char && content.chars().nth(i.saturating_sub(1)) != Some('`') { + in_string = false; + default_end = i + 1; + chars.next(); + } else { + default_end = i + 1; + chars.next(); + } + } else if ch == '\'' || ch == '"' { + in_string = true; + string_char = ch; + default_end = i + 1; + chars.next(); + } else if ch == ',' { + break; + } else if ch.is_whitespace() && chars.clone().skip(1).next().map(|(_, c)| c) == Some(',') { + break; + } else { + default_end = i + 1; + chars.next(); + } + } + + default_value = Some(content[default_start..default_end].trim().to_string()); + } + ',' => { + // End of parameter, finalize it + if let Some(name) = var_name.take() { + args.push(finalize_parameter(name, type_annotation.take(), default_value.take(), is_mandatory)?); + } + + // Reset for next parameter + type_annotation = None; + var_name = None; + default_value = None; + is_mandatory = false; + found_dollar = false; + } + _ => {} + } } } } - Ok(Some(args)) + + // Finalize last parameter + if let Some(name) = var_name { + args.push(finalize_parameter(name, type_annotation, default_value, is_mandatory)?); + } + + Ok(args) +} + +fn finalize_parameter( + name: String, + type_annotation: Option, + default_value: Option, + is_mandatory: bool, +) -> anyhow::Result { + // Store the original PowerShell type for use in the executor + let otyp = type_annotation.clone(); + + let mut parsed_typ = if let Some(typ) = type_annotation { + if typ.ends_with("[]") { + Some(Typ::List(Box::new(parse_powershell_single_typ( + typ.strip_suffix("[]").unwrap(), + )))) + } else { + Some(parse_powershell_single_typ(&typ)) + } + } else { + None + }; + + let default = if let Some(default_str) = default_value { + // Try to parse as string (quoted) + if (default_str.starts_with('"') && default_str.ends_with('"')) + || (default_str.starts_with('\'') && default_str.ends_with('\'')) + { + Some(json!(default_str[1..default_str.len() - 1].to_string())) + } else { + // Try to parse as number + if parsed_typ.is_none() { + if default_str.parse::().is_ok() { + parsed_typ = Some(Typ::Int); + } else if default_str.parse::().is_ok() { + parsed_typ = Some(Typ::Float); + } + } + serde_json::Number::from_str(&default_str) + .ok() + .map(serde_json::Value::Number) + } + } else { + None + }; + + // has_default semantics: + // - true: parameter is optional (has a default value OR is not mandatory) + // - false: parameter is required (marked as Mandatory AND no default value) + // Simplified: A parameter is optional unless it's mandatory without a default + let has_default = default.is_some() || !is_mandatory; + + Ok(Arg { + name, + typ: parsed_typ.unwrap_or(Typ::Str(None)), + default: default.clone(), + otyp, + has_default, + oidx: None, + }) +} + +fn parse_powershell_file(code: &str) -> anyhow::Result>> { + let param_wrapper = extract_powershell_param_block(code, false); + if let Some(param_wrapper) = param_wrapper { + Ok(Some(parse_powershell_parameters(param_wrapper)?)) + } else { + Ok(Some(vec![])) + } } #[cfg(test)] @@ -402,23 +581,23 @@ non_required="${5:-}" star_kwargs: false, args: vec![ Arg { - otyp: None, + otyp: None, // No type annotation name: "Msg".to_string(), typ: Typ::Str(None), default: None, - has_default: false, + has_default: true, // Optional (not mandatory) oidx: None }, Arg { - otyp: None, + otyp: Some("string".to_string()), // [string] name: "Msg2".to_string(), typ: Typ::Str(None), default: None, - has_default: false, + has_default: true, // Optional (not mandatory) oidx: None }, Arg { - otyp: None, + otyp: None, // No type annotation name: "Dflt".to_string(), typ: Typ::Str(None), default: Some(json!("default value, with comma")), @@ -426,7 +605,7 @@ non_required="${5:-}" oidx: None }, Arg { - otyp: None, + otyp: Some("int".to_string()), // [int] name: "Nb".to_string(), typ: Typ::Int, default: Some(json!(3)), @@ -434,7 +613,7 @@ non_required="${5:-}" oidx: None }, Arg { - otyp: None, + otyp: None, // Type inferred from default value name: "Nb2".to_string(), typ: Typ::Float, default: Some(json!(5.0)), @@ -442,7 +621,7 @@ non_required="${5:-}" oidx: None }, Arg { - otyp: None, + otyp: None, // Type inferred from default value name: "Nb3".to_string(), typ: Typ::Int, default: Some(json!(5)), @@ -450,35 +629,35 @@ non_required="${5:-}" oidx: None }, Arg { - otyp: None, + otyp: None, // No type annotation name: "Wahoo".to_string(), typ: Typ::Str(None), default: None, - has_default: false, + has_default: true, // Optional (not mandatory) oidx: None }, Arg { - otyp: None, + otyp: Some("PSCustomObject".to_string()), // [PSCustomObject] name: "Obj".to_string(), typ: Typ::Object(ObjectType::new(None, None)), default: None, - has_default: false, + has_default: true, // Optional (not mandatory) oidx: None }, Arg { - otyp: None, + otyp: Some("string[]".to_string()), // [string[]] name: "Arr".to_string(), typ: Typ::List(Box::new(Typ::Str(None))), default: None, - has_default: false, + has_default: true, // Optional (not mandatory) oidx: None }, Arg { - otyp: None, + otyp: Some("string".to_string()), // [string] (last type bracket with Mandatory) name: "Message".to_string(), typ: Typ::Str(None), default: None, - has_default: false, + has_default: false, // Required (Mandatory attribute) oidx: None } ], @@ -602,6 +781,242 @@ non_required="${5:-}" extract_powershell_param_block("function test-x{ param($Name)\n}", false), None ); + + // Valid: [CmdletBinding()] before param + assert_eq!( + extract_powershell_param_block("[CmdletBinding()]\nparam($Name)", false), + Some("$Name") + ); + assert_eq!( + extract_powershell_param_block("[CmdletBinding()]\nparam($Name, $Age)", true), + Some("param($Name, $Age)") + ); + + // Valid: [CmdletBinding()] with options before param + assert_eq!( + extract_powershell_param_block( + "[CmdletBinding(SupportsShouldProcess=$true)]\nparam($Path)", + false + ), + Some("$Path") + ); + + // Valid: Multiple attributes before param + assert_eq!( + extract_powershell_param_block( + "[CmdletBinding()]\n[OutputType([string])]\nparam($Value)", + false + ), + Some("$Value") + ); + + // Valid: CmdletBinding with comments + assert_eq!( + extract_powershell_param_block( + "# My function\n[CmdletBinding()]\nparam($Name)", + false + ), + Some("$Name") + ); + + // Valid: CmdletBinding with whitespace variations + assert_eq!( + extract_powershell_param_block( + "[CmdletBinding()] \n param($Name)", + false + ), + Some("$Name") + ); + + // Invalid: Unclosed attribute bracket + assert_eq!( + extract_powershell_param_block("[CmdletBinding(\nparam($Name)", false), + None + ); + } + + #[test] + fn test_parse_powershell_sig_with_parameter_attributes() -> anyhow::Result<()> { + // Test with [Parameter(Mandatory=$true)] attribute + let code = r#"[CmdletBinding()] +param( + [Parameter(Mandatory=$true)] + [string]$Name, + [Parameter(Mandatory=$false)] + [int]$Age = 25 +)"#; + let result = parse_powershell_sig(code)?; + assert_eq!(result.args.len(), 2); + assert_eq!(result.args[0].name, "Name"); + assert_eq!(result.args[0].typ, Typ::Str(None)); + assert_eq!(result.args[0].has_default, false); + assert_eq!(result.args[1].name, "Age"); + assert_eq!(result.args[1].typ, Typ::Int); + assert_eq!(result.args[1].has_default, true); + assert_eq!(result.args[1].default, Some(json!(25))); + + // Test with complex attributes + let code2 = r#"param( + [Parameter(Mandatory=$true, Position=0)] + [ValidateSet('Red', 'Green', 'Blue')] + [string]$Color, + [Parameter(ValueFromPipeline=$true)] + [string[]]$Items +)"#; + let result2 = parse_powershell_sig(code2)?; + assert_eq!(result2.args.len(), 2); + assert_eq!(result2.args[0].name, "Color"); + assert_eq!(result2.args[0].typ, Typ::Str(None)); + assert_eq!(result2.args[1].name, "Items"); + assert_eq!(result2.args[1].typ, Typ::List(Box::new(Typ::Str(None)))); + + Ok(()) + } + + #[test] + fn test_powershell_single_pass_parser() -> anyhow::Result<()> { + // Test the single-pass parser with a complex real-world example + let code = r#"[CmdletBinding()] +param( + [Parameter(Mandatory=$true, Position=0, HelpMessage="Enter the server name")] + [ValidateNotNullOrEmpty()] + [string]$ServerName, + + [Parameter(Mandatory=$false)] + [ValidateRange(1, 65535)] + [int]$Port = 8080, + + [Parameter(ValueFromPipeline=$true)] + [string[]]$LogFiles, + + [ValidateSet('Debug', 'Info', 'Warning', 'Error')] + [string]$LogLevel = 'Info', + + [PSCustomObject]$Config +)"#; + let result = parse_powershell_sig(code)?; + + assert_eq!(result.args.len(), 5); + + // ServerName: mandatory string with no default + assert_eq!(result.args[0].name, "ServerName"); + assert_eq!(result.args[0].typ, Typ::Str(None)); + assert_eq!(result.args[0].has_default, false); + + // Port: optional int with default + assert_eq!(result.args[1].name, "Port"); + assert_eq!(result.args[1].typ, Typ::Int); + assert_eq!(result.args[1].default, Some(json!(8080))); + assert_eq!(result.args[1].has_default, true); + + // LogFiles: string array (no mandatory, so optional) + assert_eq!(result.args[2].name, "LogFiles"); + assert_eq!(result.args[2].typ, Typ::List(Box::new(Typ::Str(None)))); + assert_eq!(result.args[2].has_default, true); // Optional (not mandatory) + + // LogLevel: string with default (ValidateSet is ignored but doesn't break parsing) + assert_eq!(result.args[3].name, "LogLevel"); + assert_eq!(result.args[3].typ, Typ::Str(None)); + assert_eq!(result.args[3].default, Some(json!("Info"))); + assert_eq!(result.args[3].has_default, true); + + // Config: PSCustomObject (no mandatory, so optional) + assert_eq!(result.args[4].name, "Config"); + assert_eq!(result.args[4].typ, Typ::Object(ObjectType::new(None, None))); + assert_eq!(result.args[4].has_default, true); // Optional (not mandatory) + + Ok(()) + } + + #[test] + fn test_powershell_mandatory_attribute() -> anyhow::Result<()> { + // Test various forms of the Mandatory attribute + let code = r#"param( + [Parameter(Mandatory)] + [string]$RequiredNoEquals, + + [Parameter(Mandatory=$true)] + [string]$RequiredWithTrue, + + [Parameter(Mandatory = $true)] + [string]$RequiredWithSpaces, + + [Parameter(Mandatory=$false)] + [string]$NotRequired, + + [Parameter(Position=0)] + [string]$NoMandatory, + + [string]$PlainRequired = "default", + + [Parameter(Mandatory=$true)] + [int]$RequiredInt +)"#; + let result = parse_powershell_sig(code)?; + + assert_eq!(result.args.len(), 7); + + // RequiredNoEquals: mandatory without =$true + assert_eq!(result.args[0].name, "RequiredNoEquals"); + assert_eq!(result.args[0].has_default, false); // Required (mandatory, no default) + + // RequiredWithTrue: mandatory with =$true + assert_eq!(result.args[1].name, "RequiredWithTrue"); + assert_eq!(result.args[1].has_default, false); // Required + + // RequiredWithSpaces: mandatory with spaces + assert_eq!(result.args[2].name, "RequiredWithSpaces"); + assert_eq!(result.args[2].has_default, false); // Required + + // NotRequired: explicitly Mandatory=$false + assert_eq!(result.args[3].name, "NotRequired"); + assert_eq!(result.args[3].has_default, true); // Optional (not mandatory) + + // NoMandatory: no Mandatory attribute + assert_eq!(result.args[4].name, "NoMandatory"); + assert_eq!(result.args[4].has_default, true); // Optional (not mandatory) + + // PlainRequired: has default value (always optional) + assert_eq!(result.args[5].name, "PlainRequired"); + assert_eq!(result.args[5].has_default, true); // Optional (has default) + assert_eq!(result.args[5].default, Some(json!("default"))); + + // RequiredInt: mandatory int + assert_eq!(result.args[6].name, "RequiredInt"); + assert_eq!(result.args[6].typ, Typ::Int); + assert_eq!(result.args[6].has_default, false); // Required + + Ok(()) + } + + #[test] + fn test_powershell_case_insensitive_parameter() -> anyhow::Result<()> { + // Test that [parameter(...)] is case-insensitive + let code = r#"param( + [parameter(Mandatory)] + [string]$LowerCase, + + [PARAMETER(MANDATORY=$TRUE)] + [string]$UpperCase, + + [Parameter(mandatory=$true)] + [string]$MixedCase +)"#; + let result = parse_powershell_sig(code)?; + + assert_eq!(result.args.len(), 3); + + // All should be detected as mandatory + assert_eq!(result.args[0].name, "LowerCase"); + assert_eq!(result.args[0].has_default, false); + + assert_eq!(result.args[1].name, "UpperCase"); + assert_eq!(result.args[1].has_default, false); + + assert_eq!(result.args[2].name, "MixedCase"); + assert_eq!(result.args[2].has_default, false); + + Ok(()) } #[test] diff --git a/backend/src/ee_oss.rs b/backend/src/ee_oss.rs index 4791a243a3..2aefaf7431 100644 --- a/backend/src/ee_oss.rs +++ b/backend/src/ee_oss.rs @@ -3,7 +3,7 @@ pub use crate::ee::*; #[cfg(not(feature = "private"))] -pub async fn set_license_key(_license_key: String) -> () { +pub async fn set_license_key(_license_key: String, _db: Option<&windmill_common::db::DB>) -> () { // Implementation is not open source } diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 978aba2faa..eaf83c2e4c 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -79,7 +79,7 @@ use windmill_common::{ OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, SERVICE_LOG_RETENTION_SECS, }; use windmill_common::{client::AuthedClient, global_settings::APP_WORKSPACED_ROUTE_SETTING}; -use windmill_queue::{SameWorkerPayload, cancel_job, get_queued_job_v2}; +use windmill_queue::{cancel_job, get_queued_job_v2, SameWorkerPayload}; use windmill_worker::{ handle_job_error, JobCompletedSender, SameWorkerSender, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, @@ -2219,7 +2219,6 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, node_n ); } - timeouts }; @@ -2253,7 +2252,6 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, node_n } } - let timeouts = non_restartable_jobs .into_iter() .map(|x| (x, ErrorMessage::RestartDisabled)) @@ -2289,59 +2287,59 @@ async fn handle_zombie_jobs(db: &Pool, base_internal_url: &str, node_n continue; } if let Some(job) = job.unwrap() { - let label = if job.permissioned_as != format!("u/{}", job.created_by) - && job.permissioned_as != job.created_by - { - format!("ephemeral-script-end-user-{}", job.created_by) - } else { - "ephemeral-script".to_string() - }; - let token = create_token_for_owner( - &db, - &job.workspace_id, - &job.permissioned_as, - &label, - *SCRIPT_TOKEN_EXPIRY, - &job.permissioned_as_email, - &job.id, - None, - Some(format!("handle_zombie_jobs")), - ) - .await - .expect("could not create job token"); + let label = if job.permissioned_as != format!("u/{}", job.created_by) + && job.permissioned_as != job.created_by + { + format!("ephemeral-script-end-user-{}", job.created_by) + } else { + "ephemeral-script".to_string() + }; + let token = create_token_for_owner( + &db, + &job.workspace_id, + &job.permissioned_as, + &label, + *SCRIPT_TOKEN_EXPIRY, + &job.permissioned_as_email, + &job.id, + None, + Some(format!("handle_zombie_jobs")), + ) + .await + .expect("could not create job token"); - let client = AuthedClient::new( - base_internal_url.to_string(), - job.workspace_id.to_string(), - token, - None, - ); + let client = AuthedClient::new( + base_internal_url.to_string(), + job.workspace_id.to_string(), + token, + None, + ); - let error_message = format!( + let error_message = format!( "Job timed out after no ping from job since {} (ZOMBIE_JOB_TIMEOUT: {}, reason: {:?}).\nThis likely means that the job died on worker {}, OOM are a common reason for worker crashes.\nCheck the workers around the time of the last ping and the exit code if any.", job.last_ping.unwrap_or_default(), *ZOMBIE_JOB_TIMEOUT, error_kind.to_string(), job.worker.clone().unwrap_or_default(), ); - let memory_peak = job.memory_peak.unwrap_or(0); - let _ = handle_job_error( - db, - &client, - &windmill_queue::MiniCompletedJob::from(job), - memory_peak, - None, - error::Error::ExecutionErr(error_message), - true, - Some(&same_worker_tx_never_used), - "", - node_name, - send_result_never_used, - #[cfg(feature = "benchmark")] - &mut windmill_common::bench::BenchmarkIter::new(), - ) - .await; - } + let memory_peak = job.memory_peak.unwrap_or(0); + let _ = handle_job_error( + db, + &client, + &windmill_queue::MiniCompletedJob::from(job), + memory_peak, + None, + error::Error::ExecutionErr(error_message), + true, + Some(&same_worker_tx_never_used), + "", + node_name, + send_result_never_used, + #[cfg(feature = "benchmark")] + &mut windmill_common::bench::BenchmarkIter::new(), + ) + .await; + } } } diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 72b6aeea52..57d7833c9d 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -2769,7 +2769,7 @@ async fn test_result_format(db: Pool) -> anyhow::Result<()> { let response = windmill_api::jobs::run_wait_result( &db, Uuid::parse_str(ordered_result_job_id).unwrap(), - "test-workspace".to_string(), + "test-workspace", None, "test-user", ) diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 34cb7adfe9..7b8888b38e 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -94,6 +94,7 @@ tempfile.workspace = true tokio-util.workspace = true tokio-tar.workspace = true tokio-postgres.workspace = true +postgres-native-tls.workspace = true hmac.workspace = true cookie.workspace = true sha2.workspace = true diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index a58ca3999e..a67a108522 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.575.4 + version: 1.576.0 title: Windmill API contact: @@ -5956,6 +5956,80 @@ paths: application/json: schema: {} + /w/{workspace}/jobs/run_wait_result/fv/{version}: + post: + summary: run flow by version and wait until completion + operationId: runWaitResultFlowByVersion + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: version + description: flow version ID + in: path + required: true + schema: + type: integer + format: int64 + - $ref: "#/components/parameters/IncludeHeader" + - $ref: "#/components/parameters/QueueLimit" + - $ref: "#/components/parameters/NewJobId" + - $ref: "#/components/parameters/SkipPreprocessor" + - name: memory_id + description: memory ID for chat-enabled flows + in: query + schema: + type: string + format: uuid + + requestBody: + description: script args + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptArgs" + + responses: + "200": + description: job result + content: + application/json: + schema: {} + + get: + summary: run flow by version with GET and wait until completion + operationId: runWaitResultFlowByVersionGet + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: version + description: flow version ID + in: path + required: true + schema: + type: integer + format: int64 + - $ref: "#/components/parameters/IncludeHeader" + - $ref: "#/components/parameters/QueueLimit" + - $ref: "#/components/parameters/Payload" + - $ref: "#/components/parameters/NewJobId" + - $ref: "#/components/parameters/SkipPreprocessor" + - name: memory_id + description: memory ID for chat-enabled flows + in: query + schema: + type: string + format: uuid + + responses: + "200": + description: job result + content: + application/json: + schema: {} + /w/{workspace}/jobs/run_and_stream/f/{path}: post: summary: run flow by path and stream updates via SSE @@ -6032,6 +6106,94 @@ paths: schema: type: string + /w/{workspace}/jobs/run_and_stream/fv/{version}: + post: + summary: run flow by version and stream updates via SSE + operationId: runAndStreamFlowByVersion + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: version + description: flow version ID + in: path + required: true + schema: + type: integer + format: int64 + - $ref: "#/components/parameters/IncludeHeader" + - $ref: "#/components/parameters/QueueLimit" + - $ref: "#/components/parameters/NewJobId" + - $ref: "#/components/parameters/SkipPreprocessor" + - name: memory_id + description: memory ID for chat-enabled flows + in: query + schema: + type: string + format: uuid + - name: poll_delay_ms + description: delay between polling for job updates in milliseconds + in: query + schema: + type: integer + format: int64 + + requestBody: + description: flow args + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptArgs" + + responses: + "200": + description: server-sent events stream of job updates + content: + text/event-stream: + schema: + type: string + + get: + summary: run flow by version with GET and stream updates via SSE + operationId: runAndStreamFlowByVersionGet + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: version + description: flow version ID + in: path + required: true + schema: + type: integer + format: int64 + - $ref: "#/components/parameters/IncludeHeader" + - $ref: "#/components/parameters/QueueLimit" + - $ref: "#/components/parameters/Payload" + - $ref: "#/components/parameters/NewJobId" + - $ref: "#/components/parameters/SkipPreprocessor" + - name: memory_id + description: memory ID for chat-enabled flows + in: query + schema: + type: string + format: uuid + - name: poll_delay_ms + description: delay between polling for job updates in milliseconds + in: query + schema: + type: integer + format: int64 + + responses: + "200": + description: server-sent events stream of job updates + content: + text/event-stream: + schema: + type: string + /w/{workspace}/jobs/run_and_stream/p/{path}: post: summary: run script by path and stream updates via SSE @@ -7680,6 +7842,64 @@ paths: type: string format: uuid + /w/{workspace}/jobs/run/fv/{version}: + post: + summary: run flow by version + operationId: runFlowByVersion + tags: + - job + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: version + description: flow version ID + in: path + required: true + schema: + type: integer + format: int64 + - name: scheduled_for + description: when to schedule this job (leave empty for immediate run) + in: query + schema: + type: string + format: date-time + - name: scheduled_in_secs + description: schedule the script to execute in the number of seconds starting now + in: query + schema: + type: integer + - $ref: "#/components/parameters/SkipPreprocessor" + - $ref: "#/components/parameters/ParentJob" + - $ref: "#/components/parameters/WorkerTag" + - $ref: "#/components/parameters/NewJobId" + - $ref: "#/components/parameters/IncludeHeader" + - name: invisible_to_owner + description: make the run invisible to the the flow owner (default false) + in: query + schema: + type: boolean + - name: memory_id + description: memory ID for chat-enabled flows + in: query + schema: + type: string + format: uuid + requestBody: + description: flow args + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ScriptArgs" + responses: + "201": + description: job created + content: + text/plain: + schema: + type: string + format: uuid + /w/{workspace}/jobs/run/batch_rerun_jobs: post: summary: re-run multiple jobs @@ -18835,6 +19055,8 @@ components: properties: lock_error_logs: type: string + version_id: + type: number ExtraPerms: type: object diff --git a/backend/windmill-api/src/ee_oss.rs b/backend/windmill-api/src/ee_oss.rs index a69c59c573..30756af8ab 100644 --- a/backend/windmill-api/src/ee_oss.rs +++ b/backend/windmill-api/src/ee_oss.rs @@ -13,7 +13,10 @@ use anyhow::anyhow; #[cfg(all(feature = "enterprise", not(feature = "private")))] use {std::sync::Arc, tokio::sync::RwLock}; #[cfg(not(feature = "private"))] -pub async fn validate_license_key(_license_key: String) -> anyhow::Result<(String, bool)> { +pub async fn validate_license_key( + _license_key: String, + _db: Option<&crate::db::DB>, +) -> anyhow::Result<(String, bool)> { // Implementation is not open source Err(anyhow!("License can't be validated in Windmill CE")) } diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index ebc09952b5..61695a9829 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -1135,6 +1135,7 @@ async fn get_flow_by_path( flow.timeout, flow.visible_to_runner_only, flow.on_behalf_of_email, + flow_version.id AS version_id, flow_version.schema, flow_version.value, flow_version.created_at AS edited_at, @@ -1174,8 +1175,9 @@ async fn get_flow_by_path( flow.timeout, flow.visible_to_runner_only, flow.on_behalf_of_email, + flow_version.id AS version_id, flow_version.schema, - flow_version.value, + flow_version.value, flow_version.created_at AS edited_at, flow_version.created_by AS edited_by, NULL AS starred diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 662b1b2349..e7009b7f2e 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -15,7 +15,6 @@ use futures::{StreamExt, TryFutureExt}; use http::{HeaderMap, HeaderName}; use itertools::Itertools; use quick_cache::sync::Cache; -use rand::SeedableRng; use serde_json::value::RawValue; use serde_json::Value; use sqlx::Pool; @@ -47,8 +46,7 @@ use windmill_common::{email_oss::send_email_html, server::load_smtp_config}; use windmill_common::variables::get_workspace_key; -use crate::triggers::trigger_helpers::ScriptId; -use crate::triggers::INACTIVE_TRIGGER_SCHEDULED_FOR_DATE; +use crate::triggers::trigger_helpers::{FlowId, ScriptId}; use crate::{ add_webhook_allowed_origin, args::{self, RawWebhookArgs}, @@ -98,8 +96,9 @@ use windmill_common::{ }; use windmill_common::{ - get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path, - get_script_info_for_hash, utils::empty_as_none, FlowVersionInfo, ScriptHashInfo, BASE_URL, + get_flow_version_info_from_version, get_latest_deployed_hash_for_path, + get_latest_flow_version_info_for_path, get_script_info_for_hash, utils::empty_as_none, + FlowVersionInfo, ScriptHashInfo, BASE_URL, }; use windmill_queue::{ cancel_job, get_result_and_success_by_id_from_flow, job_is_complete, push, PushArgs, @@ -109,56 +108,6 @@ use windmill_queue::{ use crate::flow_conversations; use windmill_common::flow_conversations::MessageType; -pub async fn generate_unique_suspend_number( - db: &Pool, - workspace_id: &str, -) -> error::Result { - use rand::Rng; - - let mut rng = rand::rngs::StdRng::from_os_rng(); - let mut attempts = 0; - let max_attempts = 10; - - loop { - attempts += 1; - if attempts > max_attempts { - return Err(error::Error::InternalErr(format!( - "Failed to generate unique suspend number after {} attempts", - max_attempts - ))); - } - - let suspend_number = rng.random_range(101..i32::MAX); - - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM v2_job_queue WHERE workspace_id = $1 AND suspend = $2)", - workspace_id, - suspend_number - ) - .fetch_one(db) - .await? - .unwrap_or(false); - - if !exists { - return Ok(suspend_number); - } - } -} - -pub async fn get_suspend_number_for_unactive_mode( - db: &Pool, - workspace_id: &str, - active_mode: Option, -) -> error::Result> { - if let Some(false) = active_mode { - Ok(Some( - generate_unique_suspend_number(db, workspace_id).await?, - )) - } else { - Ok(None) - } -} - pub fn workspaced_service() -> Router { let cors = CorsLayer::new() .allow_methods([http::Method::GET, http::Method::POST]) @@ -177,6 +126,13 @@ pub fn workspaced_service() -> Router { .layer(cors.clone()) .layer(ce_headers.clone()), ) + .route( + "/run/fv/:version", + post(run_flow_by_version) + .head(|| async { "" }) + .layer(cors.clone()) + .layer(ce_headers.clone()), + ) .route( "/run/batch_rerun_jobs", post(batch_rerun_jobs) @@ -229,6 +185,14 @@ pub fn workspaced_service() -> Router { .layer(cors.clone()) .layer(ce_headers.clone()), ) + .route( + "/run_wait_result/fv/:version", + post(run_wait_result_flow_by_version) + .get(run_wait_result_flow_by_version_get) + .head(|| async { "" }) + .layer(cors.clone()) + .layer(ce_headers.clone()), + ) .route( "/run_and_stream/f/*script_path", get(stream_flow_by_path) @@ -237,6 +201,14 @@ pub fn workspaced_service() -> Router { .layer(cors.clone()) .layer(ce_headers.clone()), ) + .route( + "/run_and_stream/fv/:version", + get(stream_flow_by_version) + .post(stream_flow_by_version) + .head(|| async { "" }) + .layer(cors.clone()) + .layer(ce_headers.clone()), + ) .route( "/run_and_stream/p/*script_path", get(stream_script_by_path) @@ -290,8 +262,6 @@ pub fn workspaced_service() -> Router { .route("/queue/position/:timestamp", get(get_queue_position)) .route("/queue/scheduled_for/:id", get(get_scheduled_for)) .route("/queue/cancel_selection", post(cancel_selection)) - .route("/queue/resume_suspended", post(resume_suspended_jobs)) - .route("/queue/cancel_suspended", post(cancel_suspended_jobs)) .route("/completed/count", get(count_completed_jobs)) .route("/completed/count_jobs", get(count_completed_jobs_detail)) .route( @@ -443,39 +413,39 @@ async fn get_flow_env_by_flow_job_id( Query(JsonPath { json_path, .. }): Query, ) -> windmill_common::error::JsonResult> { let flow_env = sqlx::query_scalar!( - r#" - SELECT - CASE - WHEN flow_version.id IS NOT NULL THEN - (flow_version.value -> 'flow_env' -> $3) #> $4 - ELSE - (root_job.raw_flow -> 'flow_env' -> $3) #> $4 - END AS "flow_env: sqlx::types::Json>" - FROM - v2_job current_job - JOIN - v2_job root_job ON root_job.id = COALESCE(current_job.root_job, current_job.flow_innermost_root_job, current_job.parent_job, current_job.id) - AND root_job.workspace_id = current_job.workspace_id - LEFT JOIN - flow_version ON flow_version.id = root_job.runnable_id - AND flow_version.path = root_job.runnable_path - AND flow_version.workspace_id = root_job.workspace_id - WHERE - current_job.id = $1 AND - current_job.workspace_id = $2"#, - flow_job_id, - w_id, - var_name, - json_path - .as_ref() - .map(|x| x.split(".").collect::>()) - .unwrap_or_default() as Vec<&str>, - ) - .fetch_optional(&db) - .await? - .map(|r| r.map(|x| x.0)) - .flatten() - .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)); + r#" + SELECT + CASE + WHEN flow_version.id IS NOT NULL THEN + (flow_version.value -> 'flow_env' -> $3) #> $4 + ELSE + (root_job.raw_flow -> 'flow_env' -> $3) #> $4 + END AS "flow_env: sqlx::types::Json>" + FROM + v2_job current_job + JOIN + v2_job root_job ON root_job.id = COALESCE(current_job.root_job, current_job.flow_innermost_root_job, current_job.parent_job, current_job.id) + AND root_job.workspace_id = current_job.workspace_id + LEFT JOIN + flow_version ON flow_version.id = root_job.runnable_id + AND flow_version.path = root_job.runnable_path + AND flow_version.workspace_id = root_job.workspace_id + WHERE + current_job.id = $1 AND + current_job.workspace_id = $2"#, + flow_job_id, + w_id, + var_name, + json_path + .as_ref() + .map(|x| x.split(".").collect::>()) + .unwrap_or_default() as Vec<&str>, + ) + .fetch_optional(&db) + .await? + .map(|r| r.map(|x| x.0)) + .flatten() + .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)); log_job_view( &db, @@ -490,12 +460,12 @@ async fn get_flow_env_by_flow_job_id( async fn compute_root_job_for_flow(db: &DB, w_id: &str, job_id: Uuid) -> error::Result { let root_job = sqlx::query_scalar!( - r#"SELECT COALESCE(root_job, flow_innermost_root_job, parent_job, id) as "root_job!" FROM v2_job WHERE id = $1 AND workspace_id = $2"#, - job_id, - w_id - ) - .fetch_one(db) - .await?; + r#"SELECT COALESCE(root_job, flow_innermost_root_job, parent_job, id) as "root_job!" FROM v2_job WHERE id = $1 AND workspace_id = $2"#, + job_id, + w_id + ) + .fetch_one(db) + .await?; Ok(root_job.to_string()) } @@ -698,10 +668,10 @@ async fn get_queue_position( // Count jobs that are scheduled before this job and are not suspended let count: i64 = sqlx::query_scalar!( "SELECT COUNT(*) as count - FROM v2_job_queue - WHERE scheduled_for < to_timestamp($1::bigint / 1000.0) - AND running = false - AND suspend_until IS NULL", + FROM v2_job_queue + WHERE scheduled_for < to_timestamp($1::bigint / 1000.0) + AND running = false + AND suspend_until IS NULL", scheduled_for, ) .fetch_one(&db) @@ -809,33 +779,33 @@ async fn list_selected_job_groups( let mut tx = user_db.begin(&authed).await?; let results = sqlx::query_scalar!( - r#"SELECT jsonb_build_object( - 'kind', jb.kind, - 'script_path', jb.runnable_path, - 'latest_schema', COALESCE( - (SELECT DISTINCT ON (s.path) s.schema FROM script s WHERE s.workspace_id = $1 AND s.path = jb.runnable_path AND jb.kind = 'script' ORDER BY s.path, s.created_at DESC), - (SELECT flow_version.schema FROM flow LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] WHERE flow.workspace_id = $1 AND flow.path = jb.runnable_path AND jb.kind = 'flow') - ), - 'schemas', ARRAY( - SELECT jsonb_build_object( - 'script_hash', LPAD(TO_HEX(COALESCE(s.hash, f.id)), 16, '0'), - 'job_ids', ARRAY_AGG(DISTINCT j.id), - 'schema', (ARRAY_AGG(COALESCE(s.schema, f.schema)))[1] - ) FROM v2_job j - LEFT JOIN script s ON s.hash = j.runnable_id AND j.kind = 'script' - LEFT JOIN flow_version f ON f.id = j.runnable_id AND j.kind = 'flow' - WHERE j.id = ANY(ARRAY_AGG(jb.id)) - GROUP BY COALESCE(s.hash, f.id) - ) - ) FROM v2_job jb - WHERE (jb.kind = 'flow' OR jb.kind = 'script') - AND jb.workspace_id = $1 AND jb.id = ANY($2) - GROUP BY jb.kind, jb.runnable_path"#, - &w_id, - &uuids - ) - .fetch_all(&mut *tx) - .await?; + r#"SELECT jsonb_build_object( + 'kind', jb.kind, + 'script_path', jb.runnable_path, + 'latest_schema', COALESCE( + (SELECT DISTINCT ON (s.path) s.schema FROM script s WHERE s.workspace_id = $1 AND s.path = jb.runnable_path AND jb.kind = 'script' ORDER BY s.path, s.created_at DESC), + (SELECT flow_version.schema FROM flow LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] WHERE flow.workspace_id = $1 AND flow.path = jb.runnable_path AND jb.kind = 'flow') + ), + 'schemas', ARRAY( + SELECT jsonb_build_object( + 'script_hash', LPAD(TO_HEX(COALESCE(s.hash, f.id)), 16, '0'), + 'job_ids', ARRAY_AGG(DISTINCT j.id), + 'schema', (ARRAY_AGG(COALESCE(s.schema, f.schema)))[1] + ) FROM v2_job j + LEFT JOIN script s ON s.hash = j.runnable_id AND j.kind = 'script' + LEFT JOIN flow_version f ON f.id = j.runnable_id AND j.kind = 'flow' + WHERE j.id = ANY(ARRAY_AGG(jb.id)) + GROUP BY COALESCE(s.hash, f.id) + ) + ) FROM v2_job jb + WHERE (jb.kind = 'flow' OR jb.kind = 'script') + AND jb.workspace_id = $1 AND jb.id = ANY($2) + GROUP BY jb.kind, jb.runnable_path"#, + &w_id, + &uuids + ) + .fetch_all(&mut *tx) + .await?; tx.commit().await?; @@ -887,71 +857,71 @@ async fn get_job( } macro_rules! get_job_query { - ("v2_job_completed", $($opts:tt)*) => { - get_job_query!( - @impl "v2_job_completed", ($($opts)*), - "v2_job_completed.duration_ms, v2_job_completed.completed_at, CASE WHEN status = 'success' OR status = 'skipped' THEN true ELSE false END as success, result_columns, deleted, status = 'skipped' as is_skipped, result->'wm_labels' as labels, \ - CASE WHEN result is null or pg_column_size(result) < 90000 THEN result ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as result", - "", - ) - }; - ("v2_job_queue", $($opts:tt)*) => { - get_job_query!( - @impl "v2_job_queue", ($($opts)*), - "scheduled_for, running, ping as last_ping, suspend, suspend_until, same_worker, pre_run_error, visible_to_owner, \ - flow_innermost_root_job AS root_job, flow_leaf_jobs AS leaf_jobs, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id, cache_ttl,\ - script_entrypoint_override", - "LEFT JOIN v2_job_runtime ON v2_job_runtime.id = v2_job_queue.id LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id", - ) - }; - (@impl $table:literal, (with_logs: $with_logs:expr, $($rest:tt)*), $additional_fields:literal, $additional_joins:literal, $($args:tt)*) => { - if $with_logs { - get_job_query!(@impl $table, ($($rest)*), $additional_fields, $additional_joins, logs = "right(job_logs.logs, 20000)", $($args)*) - } else { - get_job_query!(@impl $table, ($($rest)*), $additional_fields, $additional_joins, logs = "null", $($args)*) + ("v2_job_completed", $($opts:tt)*) => { + get_job_query!( + @impl "v2_job_completed", ($($opts)*), + "v2_job_completed.duration_ms, v2_job_completed.completed_at, CASE WHEN status = 'success' OR status = 'skipped' THEN true ELSE false END as success, result_columns, deleted, status = 'skipped' as is_skipped, result->'wm_labels' as labels, \ + CASE WHEN result is null or pg_column_size(result) < 90000 THEN result ELSE '\"WINDMILL_TOO_BIG\"'::jsonb END as result", + "", + ) + }; + ("v2_job_queue", $($opts:tt)*) => { + get_job_query!( + @impl "v2_job_queue", ($($opts)*), + "scheduled_for, running, ping as last_ping, suspend, suspend_until, same_worker, pre_run_error, visible_to_owner, \ + flow_innermost_root_job AS root_job, flow_leaf_jobs AS leaf_jobs, concurrent_limit, concurrency_time_window_s, timeout, flow_step_id, cache_ttl,\ + script_entrypoint_override", + "LEFT JOIN v2_job_runtime ON v2_job_runtime.id = v2_job_queue.id LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id", + ) + }; + (@impl $table:literal, (with_logs: $with_logs:expr, $($rest:tt)*), $additional_fields:literal, $additional_joins:literal, $($args:tt)*) => { + if $with_logs { + get_job_query!(@impl $table, ($($rest)*), $additional_fields, $additional_joins, logs = "right(job_logs.logs, 20000)", $($args)*) + } else { + get_job_query!(@impl $table, ($($rest)*), $additional_fields, $additional_joins, logs = "null", $($args)*) + } + }; + (@impl $table:literal, (with_code: $with_code:expr, $($rest:tt)*), $additional_fields:literal, $additional_joins:literal, $($args:tt)*) => { + if $with_code { + get_job_query!(@impl $table, ($($rest)*), $additional_fields, $additional_joins, lock = "raw_lock", code = "raw_code", $($args)*) + } else { + get_job_query!(@impl $table, ($($rest)*), $additional_fields, $additional_joins, lock = "null", code = "null", $($args)*) + } + }; + (@impl $table:literal, (with_flow: $with_flow:expr, $($rest:tt)*), $additional_fields:literal, $additional_joins:literal, $($args:tt)*) => { + if $with_flow { + get_job_query!(@impl $table, ($($rest)*), $additional_fields, $additional_joins, flow = "raw_flow", $($args)*) + } else { + get_job_query!(@impl $table, ($($rest)*), $additional_fields, $additional_joins, flow = "null", $($args)*) + } + }; + (@impl $table:literal, (), $additional_fields:literal, $additional_joins:literal, $($args:tt)*) => { + const_format::formatcp!( + "SELECT \ + {table}.id, {table}.workspace_id, parent_job, v2_job.created_by, v2_job.created_at, started_at, v2_job.runnable_id as script_hash, v2_job.runnable_path as script_path, \ + CASE WHEN args is null THEN NULL + WHEN pg_column_size(args) < 90000 THEN + CASE WHEN jsonb_typeof(args) = 'object' THEN args + ELSE jsonb_build_object('value', args) + END + ELSE '{{\"reason\": \"WINDMILL_TOO_BIG\"}}'::jsonb END as args, flow_status, workflow_as_code_status, \ + {logs} as logs, {code} as raw_code, canceled_by is not null as canceled, canceled_by, canceled_reason, kind as job_kind, \ + CASE WHEN trigger_kind = 'schedule'::job_trigger_kind THEN trigger END AS schedule_path, permissioned_as, \ + {flow} as raw_flow, flow_step_id IS NOT NULL AS is_flow_step, script_lang as language, \ + {lock} as raw_lock, permissioned_as_email as email, visible_to_owner, memory_peak as mem_peak, v2_job.tag, v2_job.priority, preprocessed, worker,\ + {additional_fields} \ + FROM {table} + INNER JOIN v2_job ON v2_job.id = {table}.id \ + {additional_joins} \ + LEFT JOIN job_logs ON {table}.id = job_id \ + WHERE {table}.id = $1 AND {table}.workspace_id = $2 AND ($3::text[] IS NULL OR v2_job.tag = ANY($3))", + table = $table, + additional_fields = $additional_fields, + additional_joins = $additional_joins, + $($args)* + ) } - }; - (@impl $table:literal, (with_code: $with_code:expr, $($rest:tt)*), $additional_fields:literal, $additional_joins:literal, $($args:tt)*) => { - if $with_code { - get_job_query!(@impl $table, ($($rest)*), $additional_fields, $additional_joins, lock = "raw_lock", code = "raw_code", $($args)*) - } else { - get_job_query!(@impl $table, ($($rest)*), $additional_fields, $additional_joins, lock = "null", code = "null", $($args)*) - } - }; - (@impl $table:literal, (with_flow: $with_flow:expr, $($rest:tt)*), $additional_fields:literal, $additional_joins:literal, $($args:tt)*) => { - if $with_flow { - get_job_query!(@impl $table, ($($rest)*), $additional_fields, $additional_joins, flow = "raw_flow", $($args)*) - } else { - get_job_query!(@impl $table, ($($rest)*), $additional_fields, $additional_joins, flow = "null", $($args)*) - } - }; - (@impl $table:literal, (), $additional_fields:literal, $additional_joins:literal, $($args:tt)*) => { - const_format::formatcp!( - "SELECT \ - {table}.id, {table}.workspace_id, parent_job, v2_job.created_by, v2_job.created_at, started_at, v2_job.runnable_id as script_hash, v2_job.runnable_path as script_path, \ - CASE WHEN args is null THEN NULL - WHEN pg_column_size(args) < 90000 THEN - CASE WHEN jsonb_typeof(args) = 'object' THEN args - ELSE jsonb_build_object('value', args) - END - ELSE '{{\"reason\": \"WINDMILL_TOO_BIG\"}}'::jsonb END as args, flow_status, workflow_as_code_status, \ - {logs} as logs, {code} as raw_code, canceled_by is not null as canceled, canceled_by, canceled_reason, kind as job_kind, \ - CASE WHEN trigger_kind = 'schedule'::job_trigger_kind THEN trigger END AS schedule_path, permissioned_as, \ - {flow} as raw_flow, flow_step_id IS NOT NULL AS is_flow_step, script_lang as language, \ - {lock} as raw_lock, permissioned_as_email as email, visible_to_owner, memory_peak as mem_peak, v2_job.tag, v2_job.priority, preprocessed, worker,\ - {additional_fields} \ - FROM {table} - INNER JOIN v2_job ON v2_job.id = {table}.id \ - {additional_joins} \ - LEFT JOIN job_logs ON {table}.id = job_id \ - WHERE {table}.id = $1 AND {table}.workspace_id = $2 AND ($3::text[] IS NULL OR v2_job.tag = ANY($3))", - table = $table, - additional_fields = $additional_fields, - additional_joins = $additional_joins, - $($args)* - ) } -} #[derive(Copy, Clone)] struct GetQuery<'a> { @@ -1208,13 +1178,13 @@ async fn send_workspace_trigger_failure_email_notification( let trigger_info = if trigger_kind.is_some() && trigger_path.is_some() { format!( r#" -
- Trigger path: {} -
+
+ Trigger path: {} +
-
- Trigger Type: {} -
"#, +
+ Trigger Type: {} +
"#, trigger_path.unwrap(), trigger_kind_str ) @@ -1230,51 +1200,51 @@ async fn send_workspace_trigger_failure_email_notification( let content = format!( r#" - - - - - - - -

{}

+ + + + + + + +

{}

-
- Workspace: {} -
-{} -
- Script/Flow Path: {} -
+
+ Workspace: {} +
+ {} +
+ Script/Flow Path: {} +
-
- Job ID: {} -
+
+ Job ID: {} +
-
- Error Details: - 
{}
-
+
+ Error Details: + 
{}
+
- View Job Details - - -"#, + View Job Details + + + "#, email_title, w_id, trigger_info, runnable_path, &job_id, error_details, job_url ); @@ -1288,11 +1258,11 @@ async fn send_workspace_trigger_failure_email_notification( .await { let err_msg = format!( - "Failed to send workspace email notification for trigger failure (trigger kind: {}, job ID: {}): {}", - trigger_kind_str, - &job_id, - e - ); + "Failed to send workspace email notification for trigger failure (trigger kind: {}, job ID: {}): {}", + trigger_kind_str, + &job_id, + e + ); return Err(Error::internal_err(err_msg)); } let success_msg = format!( @@ -1326,8 +1296,8 @@ async fn send_email_with_instance_smtp( if *CLOUD_HOSTED { tracing::warn!( - "Workspace trigger failure email notification is not available for cloud hosted Windmill", - ); + "Workspace trigger failure email notification is not available for cloud hosted Windmill", + ); return Err(anyhow::anyhow!("Feature not supported in cloud hosted windmill").into()); } @@ -1470,18 +1440,18 @@ async fn get_completed_job_logs_tail( .flatten(); let record = sqlx::query!( - "SELECT created_by AS \"created_by!\", coalesce(job_logs.logs, '') as logs - FROM v2_job - LEFT JOIN job_logs ON job_logs.job_id = v2_job.id - WHERE v2_job.id = $1 AND v2_job.workspace_id = $2 AND ($3::text[] IS NULL OR v2_job.tag = ANY($3)) - ORDER BY job_logs.log_offset DESC - LIMIT 100", - id, - w_id, - tags.as_ref().map(|v| v.as_slice()) - ) - .fetch_optional(&db) - .await?; + "SELECT created_by AS \"created_by!\", coalesce(job_logs.logs, '') as logs + FROM v2_job + LEFT JOIN job_logs ON job_logs.job_id = v2_job.id + WHERE v2_job.id = $1 AND v2_job.workspace_id = $2 AND ($3::text[] IS NULL OR v2_job.tag = ANY($3)) + ORDER BY job_logs.log_offset DESC + LIMIT 100", + id, + w_id, + tags.as_ref().map(|v| v.as_slice()) + ) + .fetch_optional(&db) + .await?; if let Some(record) = record { if opt_authed.is_none() && record.created_by != "anonymous" { @@ -1523,16 +1493,16 @@ async fn get_job_logs( .flatten(); let record = sqlx::query!( - "SELECT j.created_by AS \"created_by\", coalesce(job_logs.logs, '') as logs, COALESCE(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index - FROM v2_job j - LEFT JOIN job_logs ON job_logs.job_id = j.id - WHERE j.id = $1 AND j.workspace_id = $2 AND ($3::text[] IS NULL OR j.tag = ANY($3))", - id, - w_id, - tags.as_ref().map(|v| v.as_slice()) - ) - .fetch_optional(&db) - .await?; + "SELECT j.created_by AS \"created_by\", coalesce(job_logs.logs, '') as logs, COALESCE(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index + FROM v2_job j + LEFT JOIN job_logs ON job_logs.job_id = j.id + WHERE j.id = $1 AND j.workspace_id = $2 AND ($3::text[] IS NULL OR j.tag = ANY($3))", + id, + w_id, + tags.as_ref().map(|v| v.as_slice()) + ) + .fetch_optional(&db) + .await?; if let Some(record) = record { if opt_authed.is_none() && record.created_by != "anonymous" { @@ -1581,16 +1551,16 @@ async fn get_job_logs( Ok(content_plain(Body::from(logs))) } else { let text = sqlx::query!( - "SELECT j.created_by AS \"created_by!\", CONCAT(coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index - FROM v2_job j - LEFT JOIN job_logs ON job_logs.job_id = j.id - WHERE j.id = $1 AND j.workspace_id = $2 AND ($3::text[] IS NULL OR j.tag = ANY($3))", - id, - w_id, - tags.as_ref().map(|v| v.as_slice()) - ) - .fetch_optional(&db) - .await?; + "SELECT j.created_by AS \"created_by!\", CONCAT(coalesce(job_logs.logs, '')) as logs, coalesce(job_logs.log_offset, 0) as log_offset, job_logs.log_file_index + FROM v2_job j + LEFT JOIN job_logs ON job_logs.job_id = j.id + WHERE j.id = $1 AND j.workspace_id = $2 AND ($3::text[] IS NULL OR j.tag = ANY($3))", + id, + w_id, + tags.as_ref().map(|v| v.as_slice()) + ) + .fetch_optional(&db) + .await?; let text = not_found_if_none(text, "Job Logs", id.to_string())?; if opt_authed.is_none() && text.created_by != "anonymous" { @@ -1640,15 +1610,15 @@ async fn get_args( .map(|authed| get_scope_tags(authed)) .flatten(); let record = sqlx::query!( - "SELECT j.created_by AS \"created_by!\", j.args as \"args: sqlx::types::Json>\" - FROM v2_job j - WHERE j.id = $1 AND j.workspace_id = $2 AND ($3::text[] IS NULL OR j.tag = ANY($3))", - id, - &w_id, - tags.as_ref().map(|v| v.as_slice()) as Option<&[&str]>, - ) - .fetch_optional(&db) - .await?; + "SELECT j.created_by AS \"created_by!\", j.args as \"args: sqlx::types::Json>\" + FROM v2_job j + WHERE j.id = $1 AND j.workspace_id = $2 AND ($3::text[] IS NULL OR j.tag = ANY($3))", + id, + &w_id, + tags.as_ref().map(|v| v.as_slice()) as Option<&[&str]>, + ) + .fetch_optional(&db) + .await?; if let Some(record) = record { if opt_authed.is_none() && record.created_by != "anonymous" { @@ -1669,15 +1639,15 @@ async fn get_args( Ok(Json(record.args.map(|x| x.0).unwrap_or_default())) } else { let record = sqlx::query!( - "SELECT created_by AS \"created_by!\", args as \"args: sqlx::types::Json>\" - FROM v2_job - WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", - id, - &w_id, - tags.as_ref().map(|v| v.as_slice()) as Option<&[&str]>, - ) - .fetch_optional(&db) - .await?; + "SELECT created_by AS \"created_by!\", args as \"args: sqlx::types::Json>\" + FROM v2_job + WHERE id = $1 AND workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", + id, + &w_id, + tags.as_ref().map(|v| v.as_slice()) as Option<&[&str]>, + ) + .fetch_optional(&db) + .await?; let record = not_found_if_none(record, "Job Args", id.to_string())?; if opt_authed.is_none() && record.created_by != "anonymous" { return Err(Error::BadRequest( @@ -1805,6 +1775,22 @@ impl RunJobQuery { Ok(None) } } + + fn payload_as_args(&self) -> error::Result>> { + let payload_r = self.payload.clone().map(decode_payload).map(|x| { + x.map_err(|e| { + error::Error::internal_err(format!("Impossible to decode query payload: {e:#?}")) + }) + }); + + let payload_as_args = if let Some(payload) = payload_r { + payload? + } else { + HashMap::new() + }; + + Ok(payload_as_args) + } } #[derive(Deserialize, Clone)] @@ -1821,7 +1807,6 @@ pub struct ListQueueQuery { pub created_or_started_after: Option>, pub running: Option, pub schedule_path: Option, - pub trigger_path: Option, pub parent_job: Option, pub order_desc: Option, pub job_kinds: Option, @@ -1843,7 +1828,6 @@ pub struct ListQueueQuery { impl From for ListQueueQuery { fn from(lcq: ListCompletedQuery) -> Self { Self { - trigger_path: lcq.trigger_path, script_path_start: lcq.script_path_start, script_path_exact: lcq.script_path_exact, script_hash: lcq.script_hash, @@ -1998,9 +1982,6 @@ pub fn filter_list_queue_query( sqlb.and_where_eq("trigger_kind", "?".bind(&format!("{}", tk))); } - if let Some(p) = &lq.trigger_path { - sqlb.and_where_eq("trigger", "?".bind(p)); - } sqlb } @@ -2106,32 +2087,32 @@ async fn cancel_jobs( let mut uuids = vec![]; let mut tx = db.begin().await?; let trivial_jobs = sqlx::query!("INSERT INTO v2_job_completed AS cj - ( workspace_id - , id - , duration_ms - , result - , canceled_by - , canceled_reason - , flow_status - , status - , worker - ) - SELECT q.workspace_id - , q.id - , 0 - , $4 - , $1 - , 'cancel all' - , (SELECT flow_status FROM v2_job_status WHERE id = q.id) - , 'canceled'::job_status - , worker - FROM v2_job_queue q - JOIN v2_job USING (id) - WHERE q.id = any($2) AND running = false AND parent_job IS NULL AND q.workspace_id = $3 AND trigger IS NULL - FOR UPDATE SKIP LOCKED - ON CONFLICT (id) DO NOTHING RETURNING id AS \"id!\"", username, &jobs, w_id, serde_json::json!({"error": { "message": format!("Job canceled: cancel all by {username}"), "name": "Canceled", "reason": "cancel all", "canceler": username}})) - .fetch_all(&mut *tx) - .await?.into_iter().map(|x| x.id).collect::>(); + ( workspace_id + , id + , duration_ms + , result + , canceled_by + , canceled_reason + , flow_status + , status + , worker + ) + SELECT q.workspace_id + , q.id + , 0 + , $4 + , $1 + , 'cancel all' + , (SELECT flow_status FROM v2_job_status WHERE id = q.id) + , 'canceled'::job_status + , worker + FROM v2_job_queue q + JOIN v2_job USING (id) + WHERE q.id = any($2) AND running = false AND parent_job IS NULL AND q.workspace_id = $3 AND trigger IS NULL + FOR UPDATE SKIP LOCKED + ON CONFLICT (id) DO NOTHING RETURNING id AS \"id!\"", username, &jobs, w_id, serde_json::json!({"error": { "message": format!("Job canceled: cancel all by {username}"), "name": "Canceled", "reason": "cancel all", "canceler": username}})) + .fetch_all(&mut *tx) + .await?.into_iter().map(|x| x.id).collect::>(); sqlx::query!( "DELETE FROM v2_job_queue WHERE id = any($1) AND workspace_id = $2", @@ -2208,12 +2189,12 @@ async fn cancel_selection( let mut tx = user_db.begin(&authed).await?; let tags = get_scope_tags(&authed).map(|v| v.iter().map(|s| s.to_string()).collect_vec()); let jobs_to_cancel = sqlx::query_scalar!( - "SELECT j.id AS \"id!\" FROM v2_job j LEFT JOIN v2_job_queue q USING (id) WHERE j.id = ANY($1) AND j.trigger_kind IS DISTINCT FROM 'schedule'::job_trigger_kind AND ($2::text[] IS NULL OR j.tag = ANY($2))", - &jobs, - tags.as_ref().map(|v| v.as_slice()) - ) - .fetch_all(&mut *tx) - .await?; + "SELECT j.id AS \"id!\" FROM v2_job j LEFT JOIN v2_job_queue q USING (id) WHERE j.id = ANY($1) AND j.trigger_kind IS DISTINCT FROM 'schedule'::job_trigger_kind AND ($2::text[] IS NULL OR j.tag = ANY($2))", + &jobs, + tags.as_ref().map(|v| v.as_slice()) + ) + .fetch_all(&mut *tx) + .await?; tx.commit().await?; cancel_jobs( @@ -2226,132 +2207,6 @@ async fn cancel_selection( .await } -#[derive(Deserialize)] -#[serde(untagged)] -enum SuspendJobsRequest { - Trigger { trigger_path: String, trigger_kind: JobTriggerKind }, -} - -async fn resume_suspended_jobs( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, - Json(request): Json, -) -> error::JsonResult { - require_admin(authed.is_admin, &authed.username)?; - - let mut tx = user_db.begin(&authed).await?; - - match request { - SuspendJobsRequest::Trigger { trigger_path, trigger_kind } => { - let scheduled_for = INACTIVE_TRIGGER_SCHEDULED_FOR_DATE.clone(); - sqlx::query!( - r#" - UPDATE - v2_job_queue - SET - scheduled_for = NOW() - FROM - v2_job - WHERE - v2_job_queue.running is FALSE AND - v2_job_queue.id = v2_job.id - AND v2_job.workspace_id = $1 - AND v2_job_queue.scheduled_for = $2 - AND v2_job_queue.canceled_by IS NULL - AND v2_job.trigger_kind = $3 - AND v2_job.trigger = $4 - "#, - w_id, - scheduled_for, - trigger_kind as JobTriggerKind, - trigger_path - ) - .fetch_all(&mut *tx) - .await?; - } - } - - audit_log( - &mut *tx, - &authed, - "jobs.resume_suspended", - ActionKind::Update, - &w_id, - None, - None, - ) - .await?; - - tx.commit().await?; - - Ok(Json(format!("Resumed all suspended workspace jobs",))) -} - -async fn cancel_suspended_jobs( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, - Json(request): Json, -) -> error::JsonResult { - require_admin(authed.is_admin, &authed.username)?; - - let mut tx = user_db.begin(&authed).await?; - - match request { - SuspendJobsRequest::Trigger { trigger_path, trigger_kind } => { - let scheduled_for = INACTIVE_TRIGGER_SCHEDULED_FOR_DATE.clone(); - - let reason = format!( - "Canceled all suspended jobs for {} with trigger at path: {}", - &trigger_kind, &trigger_path - ); - sqlx::query!( - r#" - UPDATE - v2_job_queue - SET - canceled_by = $1, - canceled_reason = $2, - scheduled_for = NOW() - FROM - v2_job - WHERE - v2_job_queue.id = v2_job.id - AND v2_job_queue.running is FALSE - AND v2_job_queue.scheduled_for = $3 - AND v2_job.workspace_id = $4 - AND v2_job_queue.canceled_by IS NULL - AND v2_job.trigger_kind = $5 - AND v2_job.trigger = $6 - "#, - authed.username, - reason, - scheduled_for, - w_id, - trigger_kind as JobTriggerKind, - trigger_path - ) - .fetch_all(&mut *tx) - .await?; - } - } - audit_log( - &mut *tx, - &authed, - "jobs.cancel_suspended", - ActionKind::Delete, - &w_id, - None, - None, - ) - .await?; - - tx.commit().await?; - - Ok(Json(format!("Canceled all suspended workspace jobs",))) -} - async fn list_filtered_job_uuids( authed: ApiAuthed, Extension(db): Extension, @@ -2436,16 +2291,16 @@ async fn count_queue_jobs( .tags .map(|t| t.split(',').map(|s| s.to_string()).collect::>()); Ok(Json( - sqlx::query_as!( - QueueStats, - "SELECT coalesce(COUNT(*) FILTER(WHERE q.suspend = 0 AND q.running = false), 0) as \"database_length!\", coalesce(COUNT(*) FILTER(WHERE q.suspend > 0), 0) as \"suspended!\" FROM v2_job_queue q JOIN v2_job j USING (id) WHERE (j.workspace_id = $1 OR $2) AND q.scheduled_for <= now() AND ($3::text[] IS NULL OR j.tag = ANY($3))", - w_id, - w_id == "admins" && cq.all_workspaces.unwrap_or(false), - tags.as_ref().map(|v| v.as_slice()) - ) - .fetch_one(&db) - .await?, - )) + sqlx::query_as!( + QueueStats, + "SELECT coalesce(COUNT(*) FILTER(WHERE q.suspend = 0 AND q.running = false), 0) as \"database_length!\", coalesce(COUNT(*) FILTER(WHERE q.suspend > 0), 0) as \"suspended!\" FROM v2_job_queue q JOIN v2_job j USING (id) WHERE (j.workspace_id = $1 OR $2) AND q.scheduled_for <= now() AND ($3::text[] IS NULL OR j.tag = ANY($3))", + w_id, + w_id == "admins" && cq.all_workspaces.unwrap_or(false), + tags.as_ref().map(|v| v.as_slice()) + ) + .fetch_one(&db) + .await?, + )) } #[derive(Deserialize)] @@ -2506,14 +2361,14 @@ async fn count_completed_jobs( Path(w_id): Path, ) -> error::JsonResult { Ok(Json( - sqlx::query_as!( - QueueStats, - "SELECT coalesce(COUNT(*), 0) as \"database_length!\", null::bigint as suspended FROM v2_job_completed WHERE workspace_id = $1", - w_id - ) - .fetch_one(&db) - .await?, - )) + sqlx::query_as!( + QueueStats, + "SELECT coalesce(COUNT(*), 0) as \"database_length!\", null::bigint as suspended FROM v2_job_completed WHERE workspace_id = $1", + w_id + ) + .fetch_one(&db) + .await?, + )) } async fn list_jobs( @@ -2574,7 +2429,7 @@ async fn list_jobs( if sqlc.is_none() { return Err(error::Error::BadRequest( "cannot specify success, label, created_or_started_before, or starte - d_before with running" + d_before with running" .to_string(), )); } @@ -2682,8 +2537,8 @@ async fn resume_suspended_job_internal( let exists = sqlx::query_scalar!( r#" - SELECT EXISTS (SELECT 1 FROM resume_job WHERE id = $1) - "#, + SELECT EXISTS (SELECT 1 FROM resume_job WHERE id = $1) + "#, Uuid::from_u128(job_id.as_u128() ^ resume_id as u128), ) .fetch_one(&db) @@ -2830,10 +2685,10 @@ async fn insert_resume_job<'c>( ) -> error::Result<()> { sqlx::query!( r#" - INSERT INTO resume_job - (id, resume_id, job, flow, value, approver, approved) - VALUES ($1, $2, $3, $4, $5, $6, $7) - "#, + INSERT INTO resume_job + (id, resume_id, job, flow, value, approver, approved) + VALUES ($1, $2, $3, $4, $5, $6, $7) + "#, Uuid::from_u128(job_id.as_u128() ^ resume_id as u128), resume_id as i32, job_id, @@ -2859,13 +2714,13 @@ async fn get_suspended_parent_flow_info(job_id: Uuid, db: &DB) -> error::Result< let flow = sqlx::query_as!( FlowInfo, r#" - SELECT q.id, f.flow_status, q.suspend, j.runnable_path AS script_path - FROM v2_job_queue q - JOIN v2_job j USING (id) - JOIN v2_job_status f USING (id) - WHERE id = ( SELECT parent_job FROM v2_job WHERE id = $1 ) - FOR UPDATE - "#, + SELECT q.id, f.flow_status, q.suspend, j.runnable_path AS script_path + FROM v2_job_queue q + JOIN v2_job j USING (id) + JOIN v2_job_status f USING (id) + WHERE id = ( SELECT parent_job FROM v2_job WHERE id = $1 ) + FOR UPDATE + "#, job_id, ) .fetch_optional(db) @@ -2879,17 +2734,17 @@ async fn get_suspended_flow_info<'c>( tx: &mut Transaction<'c, Postgres>, ) -> error::Result<(FlowInfo, Uuid)> { let flow = sqlx::query_as!( - FlowInfo, - r#" - SELECT j.id AS "id!", COALESCE(s.flow_status, s.workflow_as_code_status) as flow_status, q.suspend AS "suspend!", j.runnable_path as script_path - FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_status s USING (id) - WHERE j.id = $1 - "#, - job_id, - ) - .fetch_optional(&mut **tx) - .await? - .ok_or_else(|| anyhow::anyhow!("parent flow job not found"))?; + FlowInfo, + r#" + SELECT j.id AS "id!", COALESCE(s.flow_status, s.workflow_as_code_status) as flow_status, q.suspend AS "suspend!", j.runnable_path as script_path + FROM v2_job_queue q JOIN v2_job j USING (id) LEFT JOIN v2_job_status s USING (id) + WHERE j.id = $1 + "#, + job_id, + ) + .fetch_optional(&mut **tx) + .await? + .ok_or_else(|| anyhow::anyhow!("parent flow job not found"))?; let job_id = flow .flow_status .as_ref() @@ -2951,10 +2806,10 @@ pub async fn get_suspended_job_flow( let flow_id = sqlx::query_scalar!( r#" - SELECT parent_job - FROM v2_job - WHERE id = $1 AND workspace_id = $2 - "#, + SELECT parent_job + FROM v2_job + WHERE id = $1 AND workspace_id = $2 + "#, job, w_id ) @@ -2991,10 +2846,10 @@ pub async fn get_suspended_job_flow( let approvers = if approvers_from_status.is_empty() { sqlx::query!( r#" - SELECT resume_id, approver - FROM resume_job - WHERE job = $1 - "#, + SELECT resume_id, approver + FROM resume_job + WHERE job = $1 + "#, job, ) .fetch_all(&db) @@ -3066,9 +2921,9 @@ fn conditionally_require_authed_user( } } let error_msg = format!( - "Only users from one of the following groups are allowed to approve this workflow: {}", - approval_conditions.user_groups_required.join(", ") - ); + "Only users from one of the following groups are allowed to approve this workflow: {}", + approval_conditions.user_groups_required.join(", ") + ); return Err(Error::PermissionDenied(error_msg)); } } @@ -3097,10 +2952,10 @@ pub async fn get_flow_user_state( let mut tx = user_db.begin(&authed).await?; let r = sqlx::query_scalar!( r#" - SELECT COALESCE(s.flow_status, s.workflow_as_code_status)->'user_states'->$1 - FROM v2_job_queue q LEFT JOIN v2_job_status s USING (id) - WHERE q.id = $2 AND q.workspace_id = $3 - "#, + SELECT COALESCE(s.flow_status, s.workflow_as_code_status)->'user_states'->$1 + FROM v2_job_queue q LEFT JOIN v2_job_status s USING (id) + WHERE q.id = $2 AND q.workspace_id = $3 + "#, key, job_id, w_id @@ -3119,19 +2974,19 @@ pub async fn set_flow_user_state( ) -> error::Result { let mut tx = user_db.begin(&authed).await?; let r = sqlx::query_scalar!( - r#" - UPDATE v2_job_status f SET flow_status = JSONB_SET(flow_status, ARRAY['user_states'], JSONB_SET(COALESCE(flow_status->'user_states', '{}'::jsonb), ARRAY[$1], $2)) - FROM v2_job j - WHERE f.id = $3 AND f.id = j.id AND j.workspace_id = $4 AND kind IN ('flow', 'flowpreview', 'flownode') RETURNING 1 - "#, - key, - value, - job_id, - w_id - ) - .fetch_optional(&mut *tx) - .await? - .flatten(); + r#" + UPDATE v2_job_status f SET flow_status = JSONB_SET(flow_status, ARRAY['user_states'], JSONB_SET(COALESCE(flow_status->'user_states', '{}'::jsonb), ARRAY[$1], $2)) + FROM v2_job j + WHERE f.id = $3 AND f.id = j.id AND j.workspace_id = $4 AND kind IN ('flow', 'flowpreview', 'flownode') RETURNING 1 + "#, + key, + value, + job_id, + w_id + ) + .fetch_optional(&mut *tx) + .await? + .flatten(); if r.is_none() { return Err(Error::NotFound("Flow job not found".to_string())); } @@ -3425,11 +3280,11 @@ impl Job { db: &Pool, ) -> Result<(), sqlx::Error> { let r = sqlx::query!( - "SELECT self_wait_time_ms, aggregate_wait_time_ms FROM outstanding_wait_time WHERE job_id = $1", - self.id() - ) - .fetch_optional(db) - .await?; + "SELECT self_wait_time_ms, aggregate_wait_time_ms FROM outstanding_wait_time WHERE job_id = $1", + self.id() + ) + .fetch_optional(db) + .await?; let (self_wait_time, aggregate_wait_time) = r .map(|x| (x.self_wait_time_ms, x.aggregate_wait_time_ms)) @@ -3922,27 +3777,27 @@ fn batch_rerun_jobs_inner( let (tx, rx) = tokio::sync::mpsc::channel(10); tokio::spawn(async move { let mut job_stream = sqlx::query_as!( - BatchReRunQueryReturnType, - r#"SELECT - j.id, - j.kind AS "kind: _", - COALESCE(s.path, f.path) AS "script_path!", - COALESCE(s.hash, f.id) AS "script_hash!: _", - COALESCE(jc.started_at, jq.scheduled_for, make_date(1970, 1, 1)) AS "scheduled_for!: _", - args AS input, - COALESCE(s.schema, f.schema) AS "schema: _" - FROM v2_job j - LEFT JOIN script s ON j.runnable_id = s.hash AND j.kind = 'script' - LEFT JOIN flow_version f ON j.runnable_id = f.id AND j.runnable_path = f.path AND j.kind = 'flow' - LEFT JOIN v2_job_completed jc ON jc.id = j.id - LEFT JOIN v2_job_queue jq ON jq.id = j.id - WHERE j.id = ANY($1) - AND j.workspace_id = $2 - AND COALESCE(s.hash, f.id) IS NOT NULL - AND COALESCE(s.path, f.path) IS NOT NULL"#, - &body.job_ids, - w_id - ).fetch(&db); + BatchReRunQueryReturnType, + r#"SELECT + j.id, + j.kind AS "kind: _", + COALESCE(s.path, f.path) AS "script_path!", + COALESCE(s.hash, f.id) AS "script_hash!: _", + COALESCE(jc.started_at, jq.scheduled_for, make_date(1970, 1, 1)) AS "scheduled_for!: _", + args AS input, + COALESCE(s.schema, f.schema) AS "schema: _" + FROM v2_job j + LEFT JOIN script s ON j.runnable_id = s.hash AND j.kind = 'script' + LEFT JOIN flow_version f ON j.runnable_id = f.id AND j.runnable_path = f.path AND j.kind = 'flow' + LEFT JOIN v2_job_completed jc ON jc.id = j.id + LEFT JOIN v2_job_queue jq ON jq.id = j.id + WHERE j.id = ANY($1) + AND j.workspace_id = $2 + AND COALESCE(s.hash, f.id) IS NOT NULL + AND COALESCE(s.path, f.path) IS NOT NULL"#, + &body.job_ids, + w_id + ).fetch(&db); while let Some(Ok(job)) = job_stream.next().await { let job_result = batch_rerun_handle_job(&job, &authed, &db, &user_db, &w_id, &body).await; @@ -3987,15 +3842,15 @@ async fn batch_rerun_handle_job( let latest_schema; let schema = if use_latest_version { latest_schema = sqlx::query_scalar!( - r#"SELECT COALESCE( - (SELECT DISTINCT ON (s.path) s.schema FROM script s WHERE s.path = jb.runnable_path AND jb.kind = 'script' ORDER BY s.path, s.created_at DESC), - (SELECT flow_version.schema FROM flow LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] WHERE flow.path = jb.runnable_path AND jb.kind = 'flow') - ) FROM v2_job jb - WHERE jb.id = $1 AND jb.workspace_id = $2 - GROUP BY jb.kind, jb.runnable_path"#, - &job.id, - &w_id - ).fetch_optional(db).await?.flatten(); + r#"SELECT COALESCE( + (SELECT DISTINCT ON (s.path) s.schema FROM script s WHERE s.path = jb.runnable_path AND jb.kind = 'script' ORDER BY s.path, s.created_at DESC), + (SELECT flow_version.schema FROM flow LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] WHERE flow.path = jb.runnable_path AND jb.kind = 'flow') + ) FROM v2_job jb + WHERE jb.id = $1 AND jb.workspace_id = $2 + GROUP BY jb.kind, jb.runnable_path"#, + &job.id, + &w_id + ).fetch_optional(db).await?.flatten(); latest_schema.as_ref() } else { job.schema.as_ref() @@ -4094,12 +3949,12 @@ async fn set_flow_memory_id( ) -> error::Result<()> { sqlx::query!( "UPDATE v2_job_status - SET flow_status = jsonb_set( - flow_status, - '{memory_id}', - to_jsonb($2::uuid) - ) - WHERE id = $1", + SET flow_status = jsonb_set( + flow_status, + '{memory_id}', + to_jsonb($2::uuid) + ) + WHERE id = $1", job_id, memory_id ) @@ -4188,24 +4043,17 @@ pub async fn run_flow_by_path( Ok((StatusCode::CREATED, uuid.to_string())) } -pub async fn push_flow_job_by_path_into_queue( - authed: ApiAuthed, - db: DB, +pub async fn run_flow( + authed: &ApiAuthed, + db: &DB, user_db: UserDB, - w_id: String, - flow_path: StripPath, + w_id: &str, + flow_path: &str, + flow_version_info: FlowVersionInfo, run_query: RunJobQuery, args: PushArgsOwned, trigger: Option, ) -> error::Result<(Uuid, Option)> { - #[cfg(feature = "enterprise")] - check_license_key_valid().await?; - - let flow_path = flow_path.to_path(); - check_scopes(&authed, || format!("jobs:run:flows:{flow_path}"))?; - - let userdb_authed = UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() }; - let FlowVersionInfo { version, tag, @@ -4216,8 +4064,7 @@ pub async fn push_flow_job_by_path_into_queue( edited_by, early_return, .. - } = get_latest_flow_version_info_for_path(Some(userdb_authed), &db, &w_id, &flow_path, true) - .await?; + } = flow_version_info; let tag = run_query.tag.clone().or(tag); @@ -4298,9 +4145,146 @@ pub async fn push_flow_job_by_path_into_queue( } tx.commit().await?; + Ok((uuid, early_return)) } +pub async fn run_flow_and_wait_result( + authed: &ApiAuthed, + db: &DB, + user_db: UserDB, + w_id: &str, + flow_path: &str, + flow_version_info: FlowVersionInfo, + run_query: RunJobQuery, + args: PushArgsOwned, + trigger: Option, +) -> error::Result { + let (uuid, early_return) = run_flow( + authed, + db, + user_db, + w_id, + flow_path, + flow_version_info, + run_query, + args, + trigger, + ) + .await?; + + run_wait_result(&db, uuid, w_id, early_return, &authed.username).await +} + +pub async fn push_flow_job_by_path_into_queue( + authed: ApiAuthed, + db: DB, + user_db: UserDB, + w_id: String, + flow_path: StripPath, + run_query: RunJobQuery, + args: PushArgsOwned, + trigger: Option, +) -> error::Result<(Uuid, Option)> { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + + let flow_path = flow_path.to_path(); + check_scopes(&authed, || format!("jobs:run:flows:{flow_path}"))?; + + let userdb_authed = UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() }; + + let flow_version_info = + get_latest_flow_version_info_for_path(Some(userdb_authed), &db, &w_id, &flow_path, true) + .await?; + + run_flow( + &authed, + &db, + user_db, + &w_id, + flow_path, + flow_version_info, + run_query, + args, + trigger, + ) + .await +} + +pub async fn run_flow_by_version( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, version)): Path<(String, i64)>, + Query(run_query): Query, + args: RawWebhookArgs, +) -> error::Result<(StatusCode, String)> { + let args = args + .to_args_from_runnable( + &authed, + &db, + &w_id, + RunnableId::from_flow_version(version), + run_query.skip_preprocessor, + ) + .await?; + + let (uuid, _) = + run_flow_by_version_inner(authed, db, user_db, w_id, version, run_query, args, None) + .await?; + + Ok((StatusCode::CREATED, uuid.to_string())) +} + +pub async fn run_flow_by_version_inner( + authed: ApiAuthed, + db: DB, + user_db: UserDB, + w_id: String, + version: i64, + run_query: RunJobQuery, + args: PushArgsOwned, + trigger: Option, +) -> error::Result<(Uuid, Option)> { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + + let flow_path = sqlx::query_scalar!( + r#" + SELECT + path + FROM + flow_version + WHERE + id = $1 AND + workspace_id = $2 + "#, + version, + &w_id + ) + .fetch_one(&db) + .await?; + + check_scopes(&authed, || format!("jobs:run:flows:{flow_path}"))?; + + let flow_version_info = + get_flow_version_info_from_version(&db, version, &w_id, &flow_path).await?; + + run_flow( + &authed, + &db, + user_db, + &w_id, + &flow_path, + flow_version_info, + run_query, + args, + trigger, + ) + .await +} + #[cfg(not(feature = "enterprise"))] pub async fn restart_flow( _authed: ApiAuthed, @@ -4336,17 +4320,17 @@ pub async fn restart_flow( let mut tx = user_db.clone().begin(&authed).await?; let completed_job = sqlx::query!( - "SELECT - j.runnable_path as script_path, j.args AS \"args: sqlx::types::Json>>\", - j.tag AS \"tag!\", j.priority - FROM v2_job j - WHERE j.id = $1 and j.workspace_id = $2", - job_id, - &w_id, - ) - .fetch_optional(&mut *tx) - .await? - .with_context(|| "Unable to find completed job with the given job UUID")?; + "SELECT + j.runnable_path as script_path, j.args AS \"args: sqlx::types::Json>>\", + j.tag AS \"tag!\", j.priority + FROM v2_job j + WHERE j.id = $1 and j.workspace_id = $2", + job_id, + &w_id, + ) + .fetch_optional(&mut *tx) + .await? + .with_context(|| "Unable to find completed job with the given job UUID")?; drop(tx); let flow_path = completed_job @@ -4683,13 +4667,13 @@ pub async fn run_workflow_as_code( if !wkflow_query.skip_update.unwrap_or(false) { sqlx::query!( "INSERT INTO v2_job_status (id, workflow_as_code_status) - VALUES ($1, JSONB_SET('{}'::JSONB, array[$2], $3)) - ON CONFLICT (id) DO UPDATE SET - workflow_as_code_status = JSONB_SET( - COALESCE(v2_job_status.workflow_as_code_status, '{}'::JSONB), - array[$2], - $3 - )", + VALUES ($1, JSONB_SET('{}'::JSONB, array[$2], $3)) + ON CONFLICT (id) DO UPDATE SET + workflow_as_code_status = JSONB_SET( + COALESCE(v2_job_status.workflow_as_code_status, '{}'::JSONB), + array[$2], + $3 + )", job_id, uuid.to_string(), serde_json::json!({ "scheduled_for": Utc::now(), "name": entrypoint }), @@ -4700,7 +4684,7 @@ pub async fn run_workflow_as_code( tracing::info!("Skipping update of flow status for job {job_id} in workspace {w_id}"); sqlx::query!( "INSERT INTO v2_job_status (id, workflow_as_code_status) VALUES ($1, '{}'::JSONB) - ON CONFLICT (id) DO NOTHING", + ON CONFLICT (id) DO NOTHING", job_id, ) .execute(&mut *tx) @@ -4788,7 +4772,7 @@ pub struct WindmillCompositeResult { pub async fn run_wait_result_internal( db: &DB, uuid: Uuid, - w_id: String, + w_id: &str, node_id_for_empty_return: Option, username: &str, ) -> error::Result<(Box, bool)> { @@ -4804,7 +4788,7 @@ pub async fn run_wait_result_internal( let mut g = Guard { done: false, id: uuid, - w_id: w_id.clone(), + w_id: w_id.to_string(), db: db.clone(), username: username.to_string(), }; @@ -4816,7 +4800,7 @@ pub async fn run_wait_result_internal( if let Some(node_id_for_empty_return) = node_id_for_empty_return.as_ref() { let result_and_success = get_result_and_success_by_id_from_flow( &db, - &w_id, + w_id, &uuid, node_id_for_empty_return, None, @@ -4832,16 +4816,16 @@ pub async fn run_wait_result_internal( if result.is_none() { let row = sqlx::query!( " - SELECT - result AS \"result: sqlx::types::Json>\", - result_columns, - status = 'success' AS \"success!\" - FROM - v2_job_completed - WHERE - id = $1 AND - workspace_id = $2 - ", + SELECT + result AS \"result: sqlx::types::Json>\", + result_columns, + status = 'success' AS \"success!\" + FROM + v2_job_completed + WHERE + id = $1 AND + workspace_id = $2 + ", uuid, &w_id ) @@ -4973,7 +4957,7 @@ pub fn result_to_response(result: Box, success: bool) -> error::Result pub async fn run_wait_result( db: &DB, uuid: Uuid, - w_id: String, + w_id: &str, node_id_for_empty_return: Option, username: &str, ) -> error::Result { @@ -5008,13 +4992,13 @@ pub async fn delete_job_metadata_after_use(db: &DB, job_uuid: Uuid) -> Result<() pub async fn check_queue_too_long(db: &DB, queue_limit: Option) -> error::Result<()> { if let Some(limit) = queue_limit { let count = sqlx::query_scalar!( - "SELECT COUNT(*) FROM v2_job_queue q WHERE q.canceled_by IS NULL AND (q.scheduled_for <= now() - OR (q.suspend_until IS NOT NULL - AND (q.suspend <= 0 OR q.suspend_until <= now())))", - ) - .fetch_one(db) - .await? - .unwrap_or(0); + "SELECT COUNT(*) FROM v2_job_queue q WHERE q.canceled_by IS NULL AND (q.scheduled_for <= now() + OR (q.suspend_until IS NOT NULL + AND (q.suspend <= 0 OR q.suspend_until <= now())))", + ) + .fetch_one(db) + .await? + .unwrap_or(0); if count > queue_limit.unwrap() { return Err(Error::internal_err(format!( @@ -5133,18 +5117,11 @@ pub async fn run_wait_result_job_by_path_get( if method == http::Method::HEAD { return Ok(Json(serde_json::json!("")).into_response()); } - let payload_r = run_query.payload.map(decode_payload).map(|x| { - x.map_err(|e| Error::internal_err(format!("Impossible to decode query payload: {e:#?}"))) - }); - let payload_args = if let Some(payload) = payload_r { - payload? - } else { - HashMap::new() - }; + let payload_as_args = run_query.payload_as_args()?; let mut args = args.process_args(&authed, &db, &w_id, None).await?; - args.body = args::Body::HashMap(payload_args); + args.body = args::Body::HashMap(payload_as_args); let args = args .to_args_from_runnable( @@ -5221,7 +5198,7 @@ pub async fn run_wait_result_job_by_path_get( .await?; tx.commit().await?; - let wait_result = run_wait_result(&db, uuid, w_id, None, &authed.username).await; + let wait_result = run_wait_result(&db, uuid, &w_id, None, &authed.username).await; if delete_after_use.unwrap_or(false) { delete_job_metadata_after_use(&db, uuid).await?; } @@ -5246,20 +5223,10 @@ pub async fn run_wait_result_flow_by_path_get( if method == http::Method::HEAD { return Ok(Json(serde_json::json!("")).into_response()); } - let payload_r = run_query.payload.clone().map(decode_payload).map(|x| { - x.map_err(|e| { - error::Error::internal_err(format!("Impossible to decode query payload: {e:#?}")) - }) - }); - - let payload_args = if let Some(payload) = payload_r { - payload? - } else { - HashMap::new() - }; + let payload_as_args = run_query.payload_as_args()?; let mut args = args.process_args(&authed, &db, &w_id, None).await?; - args.body = args::Body::HashMap(payload_args); + args.body = args::Body::HashMap(payload_as_args); let args = args .to_args_from_runnable( @@ -5376,7 +5343,7 @@ pub async fn run_wait_result_script_by_path_internal( .await?; tx.commit().await?; - let wait_result = run_wait_result(&db, uuid, w_id, None, &authed.username).await; + let wait_result = run_wait_result(&db, uuid, &w_id, None, &authed.username).await; if delete_after_use.unwrap_or(false) { delete_job_metadata_after_use(&db, uuid).await?; } @@ -5499,7 +5466,7 @@ pub async fn run_wait_result_script_by_hash( .await?; tx.commit().await?; - let wait_result = run_wait_result(&db, uuid, w_id, None, &authed.username).await; + let wait_result = run_wait_result(&db, uuid, &w_id, None, &authed.username).await; if delete_after_use.unwrap_or(false) { delete_job_metadata_after_use(&db, uuid).await?; } @@ -5556,6 +5523,28 @@ pub async fn stream_flow_by_path( .await } +pub async fn stream_flow_by_version( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((w_id, version)): Path<(String, i64)>, + Query(run_query): Query, + method: hyper::http::Method, + args: RawWebhookArgs, +) -> error::Result { + stream_job( + authed, + db, + user_db, + w_id, + RunnableId::from_flow_version(version), + args, + run_query, + method == http::Method::GET, + ) + .await +} + pub async fn stream_script_by_path( authed: ApiAuthed, Extension(db): Extension, @@ -5611,20 +5600,10 @@ pub async fn stream_job( is_get: bool, ) -> error::Result { let args = if is_get { - let payload_r = run_query.payload.clone().map(decode_payload).map(|x| { - x.map_err(|e| { - Error::internal_err(format!("Impossible to decode query payload: {e:#?}")) - }) - }); - - let payload_args = if let Some(payload) = payload_r { - payload? - } else { - HashMap::new() - }; + let payload_as_args = run_query.payload_as_args()?; let mut args = args.process_args(&authed, &db, &w_id, None).await?; - args.body = args::Body::HashMap(payload_args); + args.body = args::Body::HashMap(payload_as_args); let args = args .to_args_from_runnable(&db, &w_id, runnable_id.clone(), run_query.skip_preprocessor) @@ -5671,7 +5650,7 @@ pub async fn stream_job( .await? .0 } - RunnableId::FlowPath(flow_path) => { + RunnableId::FlowId(FlowId::FlowPath(flow_path)) => { push_flow_job_by_path_into_queue( authed.clone(), db.clone(), @@ -5685,6 +5664,20 @@ pub async fn stream_job( .await? .0 } + RunnableId::FlowId(FlowId::FlowVersion(version)) => { + run_flow_by_version_inner( + authed.clone(), + db.clone(), + user_db, + w_id.clone(), + version, + run_query, + args, + None, + ) + .await? + .0 + } }; let opt_authed = Some(authed.clone()); @@ -5740,101 +5733,137 @@ pub async fn run_wait_result_flow_by_path_internal( let flow_path = flow_path.to_path(); - let scheduled_for = run_query.get_scheduled_for(&db).await?; - let userdb_authed = UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() }; - let FlowVersionInfo { - tag, - dedicated_worker, - early_return, - has_preprocessor, - chat_input_enabled, - on_behalf_of_email, - edited_by, - version, - } = get_latest_flow_version_info_for_path(Some(userdb_authed), &db, &w_id, &flow_path, true) - .await?; + let flow_version_info = + get_latest_flow_version_info_for_path(Some(userdb_authed), &db, &w_id, &flow_path, true) + .await?; - let tag = run_query.tag.clone().or(tag); - check_tag_available_for_workspace(&db, &w_id, &tag, &authed).await?; - - let (email, permissioned_as, push_authed, tx) = - if let Some(on_behalf_of_email) = on_behalf_of_email.as_ref() { - ( - on_behalf_of_email, - username_to_permissioned_as(&edited_by), - None, - PushIsolationLevel::IsolatedRoot(db.clone()), - ) - } else { - ( - &authed.email, - username_to_permissioned_as(&authed.username), - Some(authed.clone().into()), - PushIsolationLevel::Isolated(user_db.clone(), authed.clone().into()), - ) - }; - - let (uuid, mut tx) = push( + run_flow_and_wait_result( + &authed, &db, - tx, + user_db, &w_id, - JobPayload::Flow { - path: flow_path.to_string(), - dedicated_worker, - version, - apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) - && has_preprocessor.unwrap_or(false), - }, - PushArgs { args: &args.args, extra: args.extra }, - authed.display_username(), - email, - permissioned_as, - authed.token_prefix.as_deref(), - scheduled_for, - None, - run_query.parent_job, - None, - run_query.root_job, - run_query.job_id, - false, - false, - None, - !run_query.invisible_to_owner.unwrap_or(false), - tag, - None, - None, - None, - push_authed.as_ref(), - false, - None, - None, + flow_path, + flow_version_info, + run_query, + args, None, ) + .await +} + +pub async fn run_wait_result_flow_by_version_get( + method: hyper::http::Method, + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, version)): Path<(String, i64)>, + Query(run_query): Query, + args: RawWebhookArgs, +) -> error::Result { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + + let flow_path = sqlx::query_scalar!( + "SELECT path FROM flow_version WHERE id = $1 AND workspace_id = $2", + version, + &w_id + ) + .fetch_one(&db) .await?; - // Set conversation_id if provided (for agent memory) - if let Some(memory_id) = run_query.memory_id { - set_flow_memory_id(&mut tx, uuid, memory_id).await?; + check_scopes(&authed, || format!("jobs:run:flows:{flow_path}"))?; + + if method == http::Method::HEAD { + return Ok(Json(serde_json::json!("")).into_response()); } - // Handle conversation messages for chat-enabled flows - if chat_input_enabled.unwrap_or(false) { - handle_chat_conversation_messages( - &mut tx, - &authed, + let payload_as_args = run_query.payload_as_args()?; + + let mut args = args.process_args(&authed, &db, &w_id, None).await?; + args.body = args::Body::HashMap(payload_as_args); + + let args = args + .to_args_from_runnable( + &db, &w_id, - &flow_path.to_string(), - &run_query, - args.args.get("user_message"), + RunnableId::from_flow_version(version), + run_query.skip_preprocessor, ) .await?; - } - tx.commit().await?; + let flow_version_info = + get_flow_version_info_from_version(&db, version, &w_id, &flow_path).await?; - run_wait_result(&db, uuid, w_id, early_return, &authed.username).await + run_flow_and_wait_result( + &authed, + &db, + user_db, + &w_id, + &flow_path, + flow_version_info, + run_query, + args, + None, + ) + .await +} + +pub async fn run_wait_result_flow_by_version( + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((w_id, version)): Path<(String, i64)>, + Query(run_query): Query, + args: RawWebhookArgs, +) -> error::Result { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + + let flow_path = sqlx::query_scalar!( + r#" + SELECT + path + FROM + flow_version + WHERE + id = $1 AND + workspace_id = $2 + "#, + version, + &w_id + ) + .fetch_one(&db) + .await?; + + check_scopes(&authed, || format!("jobs:run:flows:{flow_path}"))?; + + let args = args + .to_args_from_runnable( + &authed, + &db, + &w_id, + RunnableId::from_flow_version(version), + run_query.skip_preprocessor, + ) + .await?; + + let flow_version_info = + get_flow_version_info_from_version(&db, version, &w_id, &flow_path).await?; + + run_flow_and_wait_result( + &authed, + &db, + user_db, + &w_id, + &flow_path, + flow_version_info, + run_query, + args, + None, + ) + .await } async fn run_preview_script( @@ -5933,7 +5962,7 @@ async fn run_wait_result_preview_script( let uuid = uuid .parse::() .map_err(|_| Error::BadRequest("Invalid UUID".to_string()))?; - let result = run_wait_result(&db, uuid, w_id, None, &authed.username).await; + let result = run_wait_result(&db, uuid, &w_id, None, &authed.username).await; return result; } @@ -6111,8 +6140,8 @@ async fn run_dependencies_job( if req.raw_scripts.len() != 1 || req.raw_scripts[0].script_path != req.entrypoint { return Err(error::Error::internal_err( - "For now only a single raw script can be passed to this endpoint, and the entrypoint should be set to the script path".to_string(), - )); + "For now only a single raw script can be passed to this endpoint, and the entrypoint should be set to the script path".to_string(), + )); } let raw_script = req.raw_scripts[0].clone(); let script_path = raw_script.script_path; @@ -6171,7 +6200,7 @@ async fn run_dependencies_job( .await?; tx.commit().await?; - let wait_result = run_wait_result(&db, uuid, w_id, None, &authed.username).await; + let wait_result = run_wait_result(&db, uuid, &w_id, None, &authed.username).await; wait_result } @@ -6241,7 +6270,7 @@ async fn run_flow_dependencies_job( .await?; tx.commit().await?; - let wait_result = run_wait_result(&db, uuid, w_id, None, &authed.username).await; + let wait_result = run_wait_result(&db, uuid, &w_id, None, &authed.username).await; wait_result } @@ -6291,15 +6320,15 @@ async fn add_batch_jobs( let db_authed = UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() }; let ScriptHashInfo { - hash: script_hash, - concurrency_key, - concurrent_limit, - concurrency_time_window_s, - language, - dedicated_worker, - timeout, - .. // TODO: consider on_behalf_of_email and created_by for batch jobs - } = get_latest_deployed_hash_for_path(Some(db_authed), db.clone(), &w_id, &path).await?; + hash: script_hash, + concurrency_key, + concurrent_limit, + concurrency_time_window_s, + language, + dedicated_worker, + timeout, + .. // TODO: consider on_behalf_of_email and created_by for batch jobs + } = get_latest_deployed_hash_for_path(Some(db_authed), db.clone(), &w_id, &path).await?; ( Some(script_hash), Some(path), @@ -6350,17 +6379,17 @@ async fn add_batch_jobs( } else if let Some(path) = batch_info.path { let mut tx = user_db.clone().begin(&authed).await?; let value_json = sqlx::query!( - "SELECT coalesce(flow_version_lite.value, flow_version.value) as \"value!: sqlx::types::Json>\" FROM flow - LEFT JOIN flow_version - ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] - LEFT JOIN flow_version_lite - ON flow_version_lite.id = flow_version.id - WHERE flow.path = $1 AND flow.workspace_id = $2 LIMIT 1", - &path, &w_id - ) - .fetch_optional(&mut *tx) - .await? - .ok_or_else(|| Error::internal_err(format!("not found flow at path {:?}", path)))?; + "SELECT coalesce(flow_version_lite.value, flow_version.value) as \"value!: sqlx::types::Json>\" FROM flow + LEFT JOIN flow_version + ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] + LEFT JOIN flow_version_lite + ON flow_version_lite.id = flow_version.id + WHERE flow.path = $1 AND flow.workspace_id = $2 LIMIT 1", + &path, &w_id + ) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| Error::internal_err(format!("not found flow at path {:?}", path)))?; let value = serde_json::from_str::(value_json.value.get()).map_err(|err| { Error::internal_err(format!( @@ -6431,44 +6460,44 @@ async fn add_batch_jobs( let mut tx = user_db.begin(&authed).await?; let uuids = sqlx::query_scalar!( - r#"WITH uuid_table as ( - select gen_random_uuid() as uuid from generate_series(1, $16) + r#"WITH uuid_table as ( + select gen_random_uuid() as uuid from generate_series(1, $16) + ) + INSERT INTO v2_job + (id, workspace_id, raw_code, raw_lock, raw_flow, tag, runnable_id, runnable_path, kind, + script_lang, created_by, permissioned_as, permissioned_as_email, concurrent_limit, + concurrency_time_window_s, timeout, args) + (SELECT uuid, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, + ('{ "uuid": "' || uuid || '" }')::jsonb FROM uuid_table) + RETURNING id AS "id!""#, + w_id, + raw_code, + raw_lock, + raw_flow.map(sqlx::types::Json) as Option>, + tag, + hash, + path, + job_kind.clone() as JobKind, + language as ScriptLang, + authed.username, + username_to_permissioned_as(&authed.username), + authed.email, + concurrent_limit, + concurrent_time_window_s, + timeout, + n, ) - INSERT INTO v2_job - (id, workspace_id, raw_code, raw_lock, raw_flow, tag, runnable_id, runnable_path, kind, - script_lang, created_by, permissioned_as, permissioned_as_email, concurrent_limit, - concurrency_time_window_s, timeout, args) - (SELECT uuid, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, - ('{ "uuid": "' || uuid || '" }')::jsonb FROM uuid_table) - RETURNING id AS "id!""#, - w_id, - raw_code, - raw_lock, - raw_flow.map(sqlx::types::Json) as Option>, - tag, - hash, - path, - job_kind.clone() as JobKind, - language as ScriptLang, - authed.username, - username_to_permissioned_as(&authed.username), - authed.email, - concurrent_limit, - concurrent_time_window_s, - timeout, - n, - ) - .fetch_all(&mut *tx) - .await?; + .fetch_all(&mut *tx) + .await?; let uuids = sqlx::query_scalar!( r#"WITH uuid_table as ( - select unnest($4::uuid[]) as uuid - ) - INSERT INTO v2_job_queue - (id, workspace_id, scheduled_for, tag) - (SELECT uuid, $1, $2, $3 FROM uuid_table) - RETURNING id"#, + select unnest($4::uuid[]) as uuid + ) + INSERT INTO v2_job_queue + (id, workspace_id, scheduled_for, tag) + (SELECT uuid, $1, $2, $3 FROM uuid_table) + RETURNING id"#, w_id, Utc::now(), tag, @@ -6485,24 +6514,24 @@ async fn add_batch_jobs( .await?; sqlx::query!( - "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) - SELECT unnest($1::uuid[]), $2, $3, $4, $5, $6, $7, $8", - &uuids, - authed.email, - authed.username, - authed.is_admin, - authed.is_operator, - &[], - &[], - w_id, - ) - .execute(&mut *tx) - .await?; + "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) + SELECT unnest($1::uuid[]), $2, $3, $4, $5, $6, $7, $8", + &uuids, + authed.email, + authed.username, + authed.is_admin, + authed.is_operator, + &[], + &[], + w_id, + ) + .execute(&mut *tx) + .await?; if let Some(flow_status) = flow_status { sqlx::query!( "INSERT INTO v2_job_status (id, flow_status) - SELECT unnest($1::uuid[]), $2", + SELECT unnest($1::uuid[]), $2", &uuids, sqlx::types::Json(flow_status) as sqlx::types::Json ) @@ -6513,7 +6542,7 @@ async fn add_batch_jobs( if let Some(custom_concurrency_key) = custom_concurrency_key { sqlx::query!( "INSERT INTO concurrency_counter(concurrency_id, job_uuids) - VALUES ($1, '{}'::jsonb)", + VALUES ($1, '{}'::jsonb)", &custom_concurrency_key ) .execute(&mut *tx) @@ -6637,7 +6666,7 @@ async fn run_wait_result_preview_flow( let uuid = uuid .parse::() .map_err(|_| Error::BadRequest("Invalid UUID".to_string()))?; - let result = run_wait_result(&db, uuid, w_id, None, &authed.username).await; + let result = run_wait_result(&db, uuid, &w_id, None, &authed.username).await; return result; } @@ -6698,14 +6727,14 @@ async fn run_dynamic_select( None => { let dynamic_input = sqlx::query_scalar!( r#" - SELECT - schema - FROM - flow - WHERE - workspace_id = $1 AND - path = $2 - "#, + SELECT + schema + FROM + flow + WHERE + workspace_id = $1 AND + path = $2 + "#, &w_id, &path ) @@ -7039,15 +7068,15 @@ async fn get_log_file(Path((_w_id, file_p)): Path<(String, String)>) -> error::R } } else { return Err(error::Error::internal_err(format!( - "Object store client not present and file not found on server logs volume at {local_file}" - ))); + "Object store client not present and file not found on server logs volume at {local_file}" + ))); } #[cfg(not(all(feature = "enterprise", feature = "parquet")))] - return Err(error::Error::NotFound(format!( - "File not found on server logs volume /tmp/windmill/logs and no distributed logs s3 storage for {}", - file_p - ))); + return Err(error::Error::NotFound(format!( + "File not found on server logs volume /tmp/windmill/logs and no distributed logs s3 storage for {}", + file_p + ))); } async fn get_job_update( @@ -7411,12 +7440,12 @@ async fn get_flow_stream_delta( if let Some(job_id) = flow_stream_job_id { let record = sqlx::query!( " - SELECT - string_agg(stream, '' order by idx asc) as stream, - max(idx) + 1 as offset - FROM job_result_stream_v2 - WHERE job_id = $2 AND idx >= $1 - ", + SELECT + string_agg(stream, '' order by idx asc) as stream, + max(idx) + 1 as offset + FROM job_result_stream_v2 + WHERE job_id = $2 AND idx >= $1 + ", stream_offset.unwrap_or(0), job_id, ) @@ -7472,37 +7501,37 @@ async fn get_job_update_data( let (result, running, mut result_stream, mut new_stream_offset, new_flow_stream_job_id) = if let Some(tags) = tags { let r = sqlx::query!( - " - WITH result_stream AS ( + " + WITH result_stream AS ( + SELECT + string_agg(stream, '' order by idx asc) as stream, + job_id, + max(idx) + 1 as offset + FROM job_result_stream_v2 + WHERE job_id = $2 AND idx >= $3 + GROUP BY job_id + ) SELECT - string_agg(stream, '' order by idx asc) as stream, - job_id, - max(idx) + 1 as offset - FROM job_result_stream_v2 - WHERE job_id = $2 AND idx >= $3 - GROUP BY job_id + jc.result as \"result: sqlx::types::Json>\", + v2_job.tag, + v2_job_queue.running as \"running: Option\", + rs.stream AS \"result_stream: Option\", + rs.offset AS stream_offset, + CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job + FROM v2_job + LEFT JOIN v2_job_queue USING (id) + LEFT JOIN v2_job_completed jc USING (id) + LEFT JOIN v2_job_status js USING (id) + LEFT JOIN result_stream rs ON rs.job_id = $2 + WHERE v2_job.id = $2 AND v2_job.workspace_id = $1", + w_id, + job_id, + stream_offset.unwrap_or(0), + ignore_flow_stream_job_id, ) - SELECT - jc.result as \"result: sqlx::types::Json>\", - v2_job.tag, - v2_job_queue.running as \"running: Option\", - rs.stream AS \"result_stream: Option\", - rs.offset AS stream_offset, - CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job - FROM v2_job - LEFT JOIN v2_job_queue USING (id) - LEFT JOIN v2_job_completed jc USING (id) - LEFT JOIN v2_job_status js USING (id) - LEFT JOIN result_stream rs ON rs.job_id = $2 - WHERE v2_job.id = $2 AND v2_job.workspace_id = $1", - w_id, - job_id, - stream_offset.unwrap_or(0), - ignore_flow_stream_job_id, - ) - .fetch_optional(db) - .await? - .ok_or_else(|| Error::NotFound(format!("Job not found: {}", job_id)))?; + .fetch_optional(db) + .await? + .ok_or_else(|| Error::NotFound(format!("Job not found: {}", job_id)))?; if !tags.contains(&r.tag.as_str()) { return Err(Error::NotAuthorized(format!( @@ -7522,35 +7551,35 @@ async fn get_job_update_data( } else { if running.is_some_and(|x| !x) { let r = sqlx::query!( - " - WITH result_stream AS ( - SELECT - string_agg(stream, '' order by idx asc) as stream, - job_id, - max(idx) + 1 as offset - FROM job_result_stream_v2 - WHERE job_id = $1 AND idx >= $3 - GROUP BY job_id - ) - SELECT - COALESCE(jc.result, jc.result) as \"result: sqlx::types::Json>\", - jq.running as \"running: Option\", - rs.stream AS \"result_stream: Option\", - rs.offset AS stream_offset, - CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job - FROM ( - SELECT $1::uuid as job_id, $2::text as workspace_id - ) base - LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id - LEFT JOIN v2_job_queue jq ON jq.id = base.job_id AND jq.workspace_id = base.workspace_id - LEFT JOIN v2_job_status js ON js.id = base.job_id - LEFT JOIN result_stream rs ON rs.job_id = base.job_id - WHERE base.job_id = $1", - job_id, - w_id, - stream_offset.unwrap_or(0), - ignore_flow_stream_job_id, - ).fetch_optional(db).await?; + " + WITH result_stream AS ( + SELECT + string_agg(stream, '' order by idx asc) as stream, + job_id, + max(idx) + 1 as offset + FROM job_result_stream_v2 + WHERE job_id = $1 AND idx >= $3 + GROUP BY job_id + ) + SELECT + COALESCE(jc.result, jc.result) as \"result: sqlx::types::Json>\", + jq.running as \"running: Option\", + rs.stream AS \"result_stream: Option\", + rs.offset AS stream_offset, + CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job + FROM ( + SELECT $1::uuid as job_id, $2::text as workspace_id + ) base + LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id + LEFT JOIN v2_job_queue jq ON jq.id = base.job_id AND jq.workspace_id = base.workspace_id + LEFT JOIN v2_job_status js ON js.id = base.job_id + LEFT JOIN result_stream rs ON rs.job_id = base.job_id + WHERE base.job_id = $1", + job_id, + w_id, + stream_offset.unwrap_or(0), + ignore_flow_stream_job_id, + ).fetch_optional(db).await?; if let Some(r) = r { let running = r.running.as_ref().map(|x| *x); ( @@ -7565,36 +7594,36 @@ async fn get_job_update_data( } } else { let q = sqlx::query!( - " - WITH result_stream AS ( - SELECT - string_agg(stream, '' order by idx asc) as stream, - job_id, - max(idx) + 1 as offset - FROM job_result_stream_v2 - WHERE job_id = $2 AND idx >= $3 - GROUP BY job_id + " + WITH result_stream AS ( + SELECT + string_agg(stream, '' order by idx asc) as stream, + job_id, + max(idx) + 1 as offset + FROM job_result_stream_v2 + WHERE job_id = $2 AND idx >= $3 + GROUP BY job_id + ) + SELECT + COALESCE(jc.result, NULL) as \"result: sqlx::types::Json>\", + rs.stream AS \"result_stream: Option\", + rs.offset AS stream_offset, + COALESCE(js.flow_status, jc.flow_status) as \"flow_status: sqlx::types::Json>\", + CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job + FROM ( + SELECT $2::uuid as job_id, $1::text as workspace_id + ) base + LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id + LEFT JOIN v2_job_status js ON js.id = base.job_id + LEFT JOIN result_stream rs ON rs.job_id = base.job_id + WHERE base.job_id = $2", + w_id, + job_id, + stream_offset.unwrap_or(0), + ignore_flow_stream_job_id, ) - SELECT - COALESCE(jc.result, NULL) as \"result: sqlx::types::Json>\", - rs.stream AS \"result_stream: Option\", - rs.offset AS stream_offset, - COALESCE(js.flow_status, jc.flow_status) as \"flow_status: sqlx::types::Json>\", - CASE WHEN $4 THEN NULL ELSE (COALESCE(js.flow_status, jc.flow_status)->>'stream_job')::uuid END as stream_job - FROM ( - SELECT $2::uuid as job_id, $1::text as workspace_id - ) base - LEFT JOIN v2_job_completed jc ON jc.id = base.job_id AND jc.workspace_id = base.workspace_id - LEFT JOIN v2_job_status js ON js.id = base.job_id - LEFT JOIN result_stream rs ON rs.job_id = base.job_id - WHERE base.job_id = $2", - w_id, - job_id, - stream_offset.unwrap_or(0), - ignore_flow_stream_job_id, - ) - .fetch_optional(db) - .await?; + .fetch_optional(db) + .await?; if let Some(r) = q { ( r.result.map(|x| x.0), @@ -7636,56 +7665,56 @@ async fn get_job_update_data( }) } else { let mut record = sqlx::query!( - " - WITH result_stream AS ( - SELECT - string_agg(stream, '' order by idx asc) as stream, - job_id, - max(idx) + 1 as offset - FROM job_result_stream_v2 - WHERE job_id = $3 AND idx >= $8 - GROUP BY job_id + " + WITH result_stream AS ( + SELECT + string_agg(stream, '' order by idx asc) as stream, + job_id, + max(idx) + 1 as offset + FROM job_result_stream_v2 + WHERE job_id = $3 AND idx >= $8 + GROUP BY job_id + ) + SELECT + c.id IS NOT NULL AS completed, + CASE + WHEN q.id IS NOT NULL THEN (CASE WHEN NOT $5 AND q.running THEN true ELSE null END) + ELSE false + END AS running, + CASE WHEN $7::BOOLEAN THEN NULL ELSE SUBSTR(logs, GREATEST($1 - log_offset, 0)) END AS logs, + rs.stream AS new_result_stream, + COALESCE(r.memory_peak, c.memory_peak) AS mem_peak, + COALESCE(c.flow_status, f.flow_status) AS \"flow_status: sqlx::types::Json>\", + (COALESCE(c.flow_status, f.flow_status)->>'stream_job')::uuid AS stream_job, + COALESCE(c.workflow_as_code_status, f.workflow_as_code_status) AS \"workflow_as_code_status: sqlx::types::Json>\", + CASE WHEN $7::BOOLEAN THEN NULL ELSE job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 END AS log_offset, + rs.offset AS stream_offset, + created_by AS \"created_by!\", + CASE WHEN $4::BOOLEAN THEN ( + SELECT scalar_int FROM job_stats WHERE job_id = $3 AND metric_id = 'progress_perc' + ) END AS progress, + rs.stream AS \"result_stream: Option\" + FROM v2_job j + LEFT JOIN v2_job_queue q USING (id) + LEFT JOIN v2_job_runtime r USING (id) + LEFT JOIN v2_job_status f USING (id) + LEFT JOIN v2_job_completed c USING (id) + LEFT JOIN result_stream rs ON rs.job_id = $3 + LEFT JOIN job_logs ON job_logs.job_id = $3 + WHERE j.workspace_id = $2 AND j.id = $3 + AND ($6::text[] IS NULL OR j.tag = ANY($6))", + log_offset, + w_id, + job_id, + get_progress, + running, + tags.as_ref().map(|v| v.as_slice()) as Option<&[&str]>, + no_logs.unwrap_or(false), + stream_offset.unwrap_or(0), ) - SELECT - c.id IS NOT NULL AS completed, - CASE - WHEN q.id IS NOT NULL THEN (CASE WHEN NOT $5 AND q.running THEN true ELSE null END) - ELSE false - END AS running, - CASE WHEN $7::BOOLEAN THEN NULL ELSE SUBSTR(logs, GREATEST($1 - log_offset, 0)) END AS logs, - rs.stream AS new_result_stream, - COALESCE(r.memory_peak, c.memory_peak) AS mem_peak, - COALESCE(c.flow_status, f.flow_status) AS \"flow_status: sqlx::types::Json>\", - (COALESCE(c.flow_status, f.flow_status)->>'stream_job')::uuid AS stream_job, - COALESCE(c.workflow_as_code_status, f.workflow_as_code_status) AS \"workflow_as_code_status: sqlx::types::Json>\", - CASE WHEN $7::BOOLEAN THEN NULL ELSE job_logs.log_offset + CHAR_LENGTH(job_logs.logs) + 1 END AS log_offset, - rs.offset AS stream_offset, - created_by AS \"created_by!\", - CASE WHEN $4::BOOLEAN THEN ( - SELECT scalar_int FROM job_stats WHERE job_id = $3 AND metric_id = 'progress_perc' - ) END AS progress, - rs.stream AS \"result_stream: Option\" - FROM v2_job j - LEFT JOIN v2_job_queue q USING (id) - LEFT JOIN v2_job_runtime r USING (id) - LEFT JOIN v2_job_status f USING (id) - LEFT JOIN v2_job_completed c USING (id) - LEFT JOIN result_stream rs ON rs.job_id = $3 - LEFT JOIN job_logs ON job_logs.job_id = $3 - WHERE j.workspace_id = $2 AND j.id = $3 - AND ($6::text[] IS NULL OR j.tag = ANY($6))", - log_offset, - w_id, - job_id, - get_progress, - running, - tags.as_ref().map(|v| v.as_slice()) as Option<&[&str]>, - no_logs.unwrap_or(false), - stream_offset.unwrap_or(0), - ) - .fetch_optional(db) - .await? - .ok_or_else(|| Error::NotFound(format!("Job not found: {}", job_id)))?; + .fetch_optional(db) + .await? + .ok_or_else(|| Error::NotFound(format!("Job not found: {}", job_id)))?; if opt_authed.is_none() && record.created_by != "anonymous" { return Err(Error::BadRequest( @@ -7753,9 +7782,9 @@ pub fn filter_list_completed_query( if let Some(label) = &lq.label { if lq.allow_wildcards.unwrap_or(false) { let wh = format!( - "EXISTS (SELECT 1 FROM jsonb_array_elements_text(result->'wm_labels') label WHERE jsonb_typeof(result->'wm_labels') = 'array' AND label LIKE '{}')", - &label.replace("*", "%").replace("'", "''") - ); + "EXISTS (SELECT 1 FROM jsonb_array_elements_text(result->'wm_labels') label WHERE jsonb_typeof(result->'wm_labels') = 'array' AND label LIKE '{}')", + &label.replace("*", "%").replace("'", "''") + ); sqlb.and_where("result ? 'wm_labels'"); sqlb.and_where(&wh); } else { @@ -7784,10 +7813,6 @@ pub fn filter_list_completed_query( sqlb.and_where_eq("trigger_kind", "'schedule'"); } - if let Some(p) = &lq.trigger_path { - sqlb.and_where_eq("trigger", "?".bind(p)); - } - if let Some(ps) = &lq.script_path_start { sqlb.and_where_like_left("runnable_path", ps); } @@ -7962,7 +7987,6 @@ pub struct ListCompletedQuery { pub is_flow_step: Option, pub suspended: Option, pub schedule_path: Option, - pub trigger_path: Option, // filter by matching a subset of the args using base64 encoded json subset pub args: Option, // filter by matching a subset of the result using base64 encoded json subset @@ -7977,6 +8001,7 @@ pub struct ListCompletedQuery { pub worker: Option, pub allow_wildcards: Option, pub trigger_kind: Option, + pub trigger_path: Option, } async fn list_completed_jobs( @@ -8099,12 +8124,12 @@ async fn get_completed_job_result( sqlx::query_as!( RawResult, "SELECT - result #> $3 AS \"result: sqlx::types::Json>\", - result_columns, - created_by AS \"created_by!\" - FROM v2_job_completed c - JOIN v2_job USING (id) - WHERE c.id = $1 AND c.workspace_id = $2 AND ($4::text[] IS NULL OR tag = ANY($4))", + result #> $3 AS \"result: sqlx::types::Json>\", + result_columns, + created_by AS \"created_by!\" + FROM v2_job_completed c + JOIN v2_job USING (id) + WHERE c.id = $1 AND c.workspace_id = $2 AND ($4::text[] IS NULL OR tag = ANY($4))", id, &w_id, json_path.split(".").collect::>() as Vec<&str>, @@ -8116,12 +8141,12 @@ async fn get_completed_job_result( sqlx::query_as!( RawResult, "SELECT - result AS \"result: sqlx::types::Json>\", - result_columns, - created_by AS \"created_by!\" - FROM v2_job_completed c - JOIN v2_job USING (id) - WHERE c.id = $1 AND c.workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", + result AS \"result: sqlx::types::Json>\", + result_columns, + created_by AS \"created_by!\" + FROM v2_job_completed c + JOIN v2_job USING (id) + WHERE c.id = $1 AND c.workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", id, &w_id, tags.as_ref().map(|v| v.as_slice()) as Option<&[&str]>, @@ -8208,19 +8233,19 @@ async fn count_by_tag( let horizon = query.horizon_secs.unwrap_or(3600); // Default to 1 hour if not specified let counts = sqlx::query_as!( - TagCount, - r#" - SELECT j.tag as "tag!", COUNT(*) as "count!" - FROM v2_job_completed c JOIN v2_job j USING (id) - WHERE c.started_at > NOW() - make_interval(secs => $1) AND ($2::text IS NULL OR j.workspace_id = $2) - GROUP BY j.tag - ORDER BY "count!" DESC - "#, - horizon as f64, - query.workspace_id - ) - .fetch_all(&db) - .await?; + TagCount, + r#" + SELECT j.tag as "tag!", COUNT(*) as "count!" + FROM v2_job_completed c JOIN v2_job j USING (id) + WHERE c.started_at > NOW() - make_interval(secs => $1) AND ($2::text IS NULL OR j.workspace_id = $2) + GROUP BY j.tag + ORDER BY "count!" DESC + "#, + horizon as f64, + query.workspace_id + ) + .fetch_all(&db) + .await?; Ok(Json(counts)) } @@ -8251,13 +8276,13 @@ async fn get_completed_job_result_maybe( .flatten(); let result_o = sqlx::query!( "SELECT - result AS \"result: sqlx::types::Json>\", - result_columns, - status = 'success' AS \"success!\", - created_by AS \"created_by!\" - FROM v2_job_completed c - JOIN v2_job j USING (id) - WHERE c.id = $1 AND c.workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", + result AS \"result: sqlx::types::Json>\", + result_columns, + status = 'success' AS \"success!\", + created_by AS \"created_by!\" + FROM v2_job_completed c + JOIN v2_job j USING (id) + WHERE c.id = $1 AND c.workspace_id = $2 AND ($3::text[] IS NULL OR tag = ANY($3))", id, &w_id, tags.as_ref().map(|v| v.as_slice()) as Option<&[&str]>, @@ -8329,15 +8354,15 @@ async fn delete_completed_job<'a>( let tags = get_scope_tags(&authed); let job_o = sqlx::query_scalar!( "UPDATE v2_job_completed c SET - result = NULL, - deleted = TRUE - FROM v2_job j - WHERE c.id = $1 - AND j.id = c.id - AND c.workspace_id = $2 - AND ($3::TEXT[] IS NULL OR tag = ANY($3)) - RETURNING c.id - ", + result = NULL, + deleted = TRUE + FROM v2_job j + WHERE c.id = $1 + AND j.id = c.id + AND c.workspace_id = $2 + AND ($3::TEXT[] IS NULL OR tag = ANY($3)) + RETURNING c.id + ", id, &w_id, tags.as_ref().map(|v| v.as_slice()) as Option<&[&str]>, diff --git a/backend/windmill-api/src/public_app_layer.rs b/backend/windmill-api/src/public_app_layer.rs index 7d5e5a6e3b..cce8af1b35 100644 --- a/backend/windmill-api/src/public_app_layer.rs +++ b/backend/windmill-api/src/public_app_layer.rs @@ -71,10 +71,12 @@ fn is_public_route_whitelisted(path: &str) -> bool { "/user/login_callback/*", "/api/workspaces/users", "/api/users/whoami", + "/api/apps_u/*", "/api/oauth/list_connects", "/api/oauth/list_logins", "/public/*", "/a/*", + "/api/oauth/get_connect/*", "/Inter-Variable.woff2", ]; diff --git a/backend/windmill-api/src/settings.rs b/backend/windmill-api/src/settings.rs index de00d3cbe6..62cb87d391 100644 --- a/backend/windmill-api/src/settings.rs +++ b/backend/windmill-api/src/settings.rs @@ -701,23 +701,49 @@ async fn setup_ducklake_catalog_db_inner( sslmode = ssl_mode ); - let (client, connection) = tokio::time::timeout( - std::time::Duration::from_secs(20), - tokio_postgres::connect(&conn_str, tokio_postgres::NoTls), - ) - .await - .map_err(|e| error::Error::ExecutionErr(format!("timeout: {}", e.to_string())))? - .map_err(|e| error::Error::ExecutionErr(format!("error: {}", e.to_string())))?; - let join_handle = tokio::spawn(async move { connection.await }); + let (client, join_handle) = if ssl_mode == "require" { + use native_tls::TlsConnector; + use postgres_native_tls::MakeTlsConnector; + + let mut connector = TlsConnector::builder(); + connector.danger_accept_invalid_certs(true); + connector.danger_accept_invalid_hostnames(true); + + let (client, connection) = tokio::time::timeout( + std::time::Duration::from_secs(20), + tokio_postgres::connect( + &conn_str, + MakeTlsConnector::new(connector.build().map_err(to_anyhow)?), + ), + ) + .await + .map_err(|e| error::Error::ExecutionErr(format!("timeout: {}", e.to_string())))? + .map_err(|e| error::Error::ExecutionErr(format!("error: {}", e.to_string())))?; + + let join_handle = tokio::spawn(async move { connection.await }); + (client, join_handle) + } else { + let (client, connection) = tokio::time::timeout( + std::time::Duration::from_secs(20), + tokio_postgres::connect(&conn_str, tokio_postgres::NoTls), + ) + .await + .map_err(|e| error::Error::ExecutionErr(format!("timeout: {}", e.to_string())))? + .map_err(|e| error::Error::ExecutionErr(format!("error: {}", e.to_string())))?; + + let join_handle = tokio::spawn(async move { connection.await }); + (client, join_handle) + }; + logs.db_connect = "OK".to_string(); client .batch_execute(&format!( "GRANT CONNECT ON DATABASE \"{dbname}\" TO ducklake_user; - GRANT USAGE ON SCHEMA public TO ducklake_user; - GRANT CREATE ON SCHEMA public TO ducklake_user; - ALTER DEFAULT PRIVILEGES IN SCHEMA public - GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO ducklake_user;" + GRANT USAGE ON SCHEMA public TO ducklake_user; + GRANT CREATE ON SCHEMA public TO ducklake_user; + ALTER DEFAULT PRIVILEGES IN SCHEMA public + GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO ducklake_user;" )) .await .map_err(|e| { diff --git a/backend/windmill-api/src/triggers/trigger_helpers.rs b/backend/windmill-api/src/triggers/trigger_helpers.rs index e357b7b068..031b200021 100644 --- a/backend/windmill-api/src/triggers/trigger_helpers.rs +++ b/backend/windmill-api/src/triggers/trigger_helpers.rs @@ -9,9 +9,19 @@ use std::collections::HashMap; use std::future::Future; use uuid::Uuid; use windmill_common::{ - FlowVersionInfo, db::{UserDB, UserDbWithAuthed}, error::Result, flows::{FlowModuleValue, Retry}, get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path, jobs::{JobPayload, get_has_preprocessor_from_content_and_lang, script_path_to_payload}, scripts::{ScriptHash, ScriptLang, get_full_hub_script_by_path}, triggers::{ - HubOrWorkspaceId, RUNNABLE_FORMAT_VERSION_CACHE, RunnableFormat, RunnableFormatVersion, TriggerInfo, TriggerKind - }, users::username_to_permissioned_as, utils::StripPath, worker::to_raw_value + db::{UserDB, UserDbWithAuthed}, + error::Result, + flows::{FlowModuleValue, Retry}, + get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path, + jobs::{get_has_preprocessor_from_content_and_lang, script_path_to_payload, JobPayload}, + scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, + triggers::{ + HubOrWorkspaceId, RunnableFormat, RunnableFormatVersion, TriggerInfo, TriggerKind, + RUNNABLE_FORMAT_VERSION_CACHE, + }, + users::username_to_permissioned_as, + utils::StripPath, + worker::to_raw_value, }; use windmill_queue::{push, PushArgs, PushArgsOwned, PushIsolationLevel}; @@ -48,7 +58,7 @@ struct PartialSchema { #[derive(Debug, Clone, Eq, PartialEq, Hash)] pub enum RunnableId { - FlowPath(String), + FlowId(FlowId), ScriptId(ScriptId), HubScript(String), } @@ -67,7 +77,33 @@ impl RunnableId { } pub fn from_flow_path(path: &str) -> Self { - Self::FlowPath(path.to_string()) + Self::FlowId(FlowId::FlowPath(path.to_string())) + } + + pub fn from_flow_version(version: i64) -> Self { + Self::FlowId(FlowId::FlowVersion(version)) + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub enum FlowId { + FlowPath(String), + FlowVersion(i64), +} + +impl FlowId { + async fn get_flow_version_id(self, workspace_id: &str, db: &DB) -> Result { + let version_id = match self { + FlowId::FlowPath(path) => { + let info = + get_latest_flow_version_info_for_path(None, db, workspace_id, &path, true) + .await?; + info.version + } + FlowId::FlowVersion(version) => version, + }; + + Ok(version_id) } } @@ -247,9 +283,8 @@ pub async fn get_runnable_format( }, ) } - RunnableId::FlowPath(path) => { - let FlowVersionInfo { version, .. } = - get_latest_flow_version_info_for_path(None, &db, workspace_id, &path, true).await?; + RunnableId::FlowId(flow_id) => { + let version = flow_id.get_flow_version_id(workspace_id, db).await?; let key = ( HubOrWorkspaceId::WorkspaceId(workspace_id.to_string()), @@ -260,7 +295,7 @@ pub async fn get_runnable_format( let runnable_format = RUNNABLE_FORMAT_VERSION_CACHE.get(&key); if let Some(runnable_format) = runnable_format { - tracing::debug!("Using cached runnable format for flow {path}"); + tracing::debug!("Using cached runnable format for flow version {version}"); return Ok(runnable_format); } @@ -271,11 +306,9 @@ pub async fn get_runnable_format( schema as \"schema: _\" FROM flow_version WHERE - path = $1 - AND workspace_id = $2 - ORDER BY created_at DESC - LIMIT 1", - path, + id = $1 + AND workspace_id = $2", + version, workspace_id, ) .fetch_one(db) @@ -612,8 +645,7 @@ pub async fn trigger_runnable_and_wait_for_result( ) .await?; let (result, success) = - run_wait_result_internal(db, uuid, workspace_id.to_string(), early_return, &username) - .await?; + run_wait_result_internal(db, uuid, &workspace_id, early_return, &username).await?; if delete_after_use.unwrap_or(false) { delete_job_metadata_after_use(&db, uuid).await?; @@ -657,7 +689,7 @@ pub async fn trigger_runnable_and_wait_for_raw_result( .await?; let (result, success) = - run_wait_result_internal(db, uuid, workspace_id.to_string(), early_return, &username) + run_wait_result_internal(db, uuid, &workspace_id, early_return, &username) .await .with_context(|| { format!( diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index 929736eff5..5b81d35f75 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -66,6 +66,7 @@ pub struct FlowWithStarred { pub starred: Option, #[serde(skip_serializing_if = "Option::is_none")] pub lock_error_logs: Option, + pub version_id: i64, } fn is_none_or_false(b: &Option) -> bool { diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index b0a3f5e4ea..b91dcfaf95 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -769,7 +769,7 @@ where } } -pub fn get_latest_flow_version_info_for_path_from_version< +pub fn get_flow_version_info_from_version< 'a, 'e, A: sqlx::Acquire<'e, Database = Postgres> + Send + 'a, @@ -782,7 +782,6 @@ pub fn get_latest_flow_version_info_for_path_from_version< async move { // as instructed in the docstring of sqlx::Acquire let key = (w_id.to_string(), version); - match FLOW_INFO_CACHE.get(&key) { Some(info) => { tracing::debug!("Using cached flow version info for {version} ({path})"); @@ -791,21 +790,37 @@ pub fn get_latest_flow_version_info_for_path_from_version< _ => { tracing::debug!("Fetching flow version info for {version} ({path})"); let mut conn = db.acquire().await?; - let info = sqlx::query_as!( - FlowVersionInfo, - "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor, (flow_version.value->>'chat_input_enabled')::boolean as chat_input_enabled, on_behalf_of_email, edited_by, flow_version.id AS version - FROM flow - INNER JOIN flow_version - ON flow_version.id = $3 - WHERE flow.path = $1 and flow.workspace_id = $2", - path, - w_id, - version - ) - .fetch_optional(&mut *conn) - .await?; + let flow_info = + sqlx::query_as!( + FlowVersionInfo, + r#" + SELECT + flow_version.id AS version, + flow_version.value->>'early_return' as early_return, + flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor, + (flow_version.value->>'chat_input_enabled')::boolean as chat_input_enabled, + flow.tag, + flow.dedicated_worker, + flow.on_behalf_of_email, + flow.edited_by + FROM + flow_version + INNER JOIN flow + ON flow.path = flow_version.path AND + flow.workspace_id = flow_version.workspace_id + WHERE + flow_version.workspace_id = $1 AND + flow_version.path = $2 AND + flow_version.id = $3 + "#, + w_id, + path, + version, + ) + .fetch_optional(&mut *conn) + .await?; - let info = utils::not_found_if_none(info, "flow", path)?; + let info = utils::not_found_if_none(flow_info, "flow", path)?; FLOW_INFO_CACHE.insert(key, info.clone()); @@ -825,7 +840,7 @@ pub async fn get_latest_flow_version_info_for_path<'e>( // as instructed in the docstring of sqlx::Acquire let version = get_latest_flow_version_id_for_path(db_authed, &db.clone(), w_id, path, use_cache).await?; - get_latest_flow_version_info_for_path_from_version(db, version, w_id, path).await + get_flow_version_info_from_version(db, version, w_id, path).await } async fn get_latest_flow_version_for_path<'e, E: sqlx::PgExecutor<'e>>( diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index 343f005916..51641bde39 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -17,8 +17,8 @@ use std::str::FromStr; use windmill_common::db::Authed; use windmill_common::ee_oss::LICENSE_KEY_VALID; use windmill_common::flows::Retry; +use windmill_common::get_flow_version_info_from_version; use windmill_common::get_latest_flow_version_id_for_path; -use windmill_common::get_latest_flow_version_info_for_path_from_version; use windmill_common::jobs::check_tag_available_for_workspace_internal; use windmill_common::jobs::JobPayload; use windmill_common::jobs::JobTriggerKind; @@ -65,7 +65,7 @@ async fn get_schedule_metadata<'c>( .await?; let FlowVersionInfo { tag, on_behalf_of_email, edited_by, .. } = - get_latest_flow_version_info_for_path_from_version( + get_flow_version_info_from_version( &mut **tx, version, &schedule.workspace_id, @@ -293,16 +293,13 @@ pub async fn push_scheduled_job<'c>( let FlowVersionInfo { version, tag, dedicated_worker, on_behalf_of_email, edited_by, .. - } = get_latest_flow_version_info_for_path_from_version( + } = get_flow_version_info_from_version( &mut *tx, version, &schedule.workspace_id, &schedule.script_path, ) - .warn_after_seconds_with_sql( - 1, - "get_latest_flow_version_info_for_path_from_version".to_string(), - ) + .warn_after_seconds_with_sql(1, "get_flow_version_info_from_version".to_string()) .await?; ( diff --git a/backend/windmill-worker/src/pwsh_executor.rs b/backend/windmill-worker/src/pwsh_executor.rs index 4a4b85f8ac..cda448dc31 100644 --- a/backend/windmill-worker/src/pwsh_executor.rs +++ b/backend/windmill-worker/src/pwsh_executor.rs @@ -255,20 +255,42 @@ pub async fn handle_powershell_job( job.args.as_ref() }; - let args_owned = windmill_parser_bash::parse_powershell_sig(&content)? + let parsed_sig = windmill_parser_bash::parse_powershell_sig(&content)?; + + parsed_sig .args .iter() - .map(|arg| { - ( - arg.name.clone(), - job_args.and_then(|x| x.get(&arg.name).map(|x| raw_to_pwsh_param(x.get()))), - ) - }) - .collect::)>>(); + .filter_map(|arg| { + let value_opt = job_args.and_then(|x| x.get(&arg.name)); - args_owned - .into_iter() - .filter_map(|(n, v)| v.map(|v| format!("-{n} {v}"))) + // Check if this is a switch parameter (only [switch], not [bool]) + let is_switch = arg.otyp.as_ref().map(|t| { + t.to_lowercase() == "switch" + }).unwrap_or(false); + + if is_switch { + // Handle switch parameters: -SwitchName or omit + if let Some(value) = value_opt { + match serde_json::from_str::(value.get()) { + Ok(serde_json::Value::Bool(true)) => { + // Switch is enabled: just pass -SwitchName + Some(format!("-{}", arg.name)) + } + Ok(serde_json::Value::Bool(false)) | _ => { + // Switch is disabled or invalid: omit the parameter + None + } + } + } else { + // No value provided, omit the switch (defaults to false) + None + } + } else { + // Regular parameter (including [bool]): format as -ParamName Value + // For [bool] parameters, this will be -ParamName $true or -ParamName $false + value_opt.map(|v| format!("-{} {}", arg.name, raw_to_pwsh_param(v.get()))) + } + }) .collect::>() .join(" ") }; diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index d5d3bd172f..34dfefebb0 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.575.4"; +export const VERSION = "v1.576.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/src/main.ts b/cli/src/main.ts index 6dba2bce7a..94ac4d1093 100644 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -68,7 +68,7 @@ export { // } // }); -export const VERSION = "1.575.4"; +export const VERSION = "1.576.0"; export const WM_FORK_PREFIX = "wm-fork"; diff --git a/cli/wasm/regex/windmill_parser_wasm.d.ts b/cli/wasm/regex/windmill_parser_wasm.d.ts index 59f34f2b18..7886800b20 100644 --- a/cli/wasm/regex/windmill_parser_wasm.d.ts +++ b/cli/wasm/regex/windmill_parser_wasm.d.ts @@ -1,14 +1,14 @@ /* tslint:disable */ /* eslint-disable */ -export function parse_assets_sql(code: string): string; -export function parse_db_resource(code: string): string | undefined; -export function parse_bash(code: string): string; export function parse_mssql(code: string): string; -export function parse_oracledb(code: string): string; export function parse_powershell(code: string): string; -export function parse_bigquery(code: string): string; +export function parse_oracledb(code: string): string; export function parse_graphql(code: string): string; -export function parse_snowflake(code: string): string; -export function parse_duckdb(code: string): string; -export function parse_sql(code: string): string; +export function parse_assets_sql(code: string): string; export function parse_mysql(code: string): string; +export function parse_bash(code: string): string; +export function parse_duckdb(code: string): string; +export function parse_db_resource(code: string): string | undefined; +export function parse_bigquery(code: string): string; +export function parse_snowflake(code: string): string; +export function parse_sql(code: string): string; diff --git a/cli/wasm/regex/windmill_parser_wasm.js b/cli/wasm/regex/windmill_parser_wasm.js index 25a60a9b8e..b3b62ebd32 100644 --- a/cli/wasm/regex/windmill_parser_wasm.js +++ b/cli/wasm/regex/windmill_parser_wasm.js @@ -68,60 +68,6 @@ function getStringFromWasm0(ptr, len) { ptr = ptr >>> 0; return decodeText(ptr, len); } -/** - * @param {string} code - * @returns {string} - */ -export function parse_assets_sql(code) { - let deferred2_0; - let deferred2_1; - try { - const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_assets_sql(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; - return getStringFromWasm0(ret[0], ret[1]); - } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); - } -} - -/** - * @param {string} code - * @returns {string | undefined} - */ -export function parse_db_resource(code) { - const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_db_resource(ptr0, len0); - let v2; - if (ret[0] !== 0) { - v2 = getStringFromWasm0(ret[0], ret[1]).slice(); - wasm.__wbindgen_free(ret[0], ret[1] * 1, 1); - } - return v2; -} - -/** - * @param {string} code - * @returns {string} - */ -export function parse_bash(code) { - let deferred2_0; - let deferred2_1; - try { - const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_bash(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; - return getStringFromWasm0(ret[0], ret[1]); - } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); - } -} - /** * @param {string} code * @returns {string} @@ -141,25 +87,6 @@ export function parse_mssql(code) { } } -/** - * @param {string} code - * @returns {string} - */ -export function parse_oracledb(code) { - let deferred2_0; - let deferred2_1; - try { - const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); - const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_oracledb(ptr0, len0); - deferred2_0 = ret[0]; - deferred2_1 = ret[1]; - return getStringFromWasm0(ret[0], ret[1]); - } finally { - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); - } -} - /** * @param {string} code * @returns {string} @@ -183,13 +110,13 @@ export function parse_powershell(code) { * @param {string} code * @returns {string} */ -export function parse_bigquery(code) { +export function parse_oracledb(code) { let deferred2_0; let deferred2_1; try { const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_bigquery(ptr0, len0); + const ret = wasm.parse_oracledb(ptr0, len0); deferred2_0 = ret[0]; deferred2_1 = ret[1]; return getStringFromWasm0(ret[0], ret[1]); @@ -221,13 +148,51 @@ export function parse_graphql(code) { * @param {string} code * @returns {string} */ -export function parse_snowflake(code) { +export function parse_assets_sql(code) { let deferred2_0; let deferred2_1; try { const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_snowflake(ptr0, len0); + const ret = wasm.parse_assets_sql(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + +/** + * @param {string} code + * @returns {string} + */ +export function parse_mysql(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_mysql(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + +/** + * @param {string} code + * @returns {string} + */ +export function parse_bash(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_bash(ptr0, len0); deferred2_0 = ret[0]; deferred2_1 = ret[1]; return getStringFromWasm0(ret[0], ret[1]); @@ -255,17 +220,33 @@ export function parse_duckdb(code) { } } +/** + * @param {string} code + * @returns {string | undefined} + */ +export function parse_db_resource(code) { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_db_resource(ptr0, len0); + let v2; + if (ret[0] !== 0) { + v2 = getStringFromWasm0(ret[0], ret[1]).slice(); + wasm.__wbindgen_free(ret[0], ret[1] * 1, 1); + } + return v2; +} + /** * @param {string} code * @returns {string} */ -export function parse_sql(code) { +export function parse_bigquery(code) { let deferred2_0; let deferred2_1; try { const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_sql(ptr0, len0); + const ret = wasm.parse_bigquery(ptr0, len0); deferred2_0 = ret[0]; deferred2_1 = ret[1]; return getStringFromWasm0(ret[0], ret[1]); @@ -278,13 +259,32 @@ export function parse_sql(code) { * @param {string} code * @returns {string} */ -export function parse_mysql(code) { +export function parse_snowflake(code) { let deferred2_0; let deferred2_1; try { const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); const len0 = WASM_VECTOR_LEN; - const ret = wasm.parse_mysql(ptr0, len0); + const ret = wasm.parse_snowflake(ptr0, len0); + deferred2_0 = ret[0]; + deferred2_1 = ret[1]; + return getStringFromWasm0(ret[0], ret[1]); + } finally { + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + +/** + * @param {string} code + * @returns {string} + */ +export function parse_sql(code) { + let deferred2_0; + let deferred2_1; + try { + const ptr0 = passStringToWasm0(code, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.parse_sql(ptr0, len0); deferred2_0 = ret[0]; deferred2_1 = ret[1]; return getStringFromWasm0(ret[0], ret[1]); diff --git a/cli/wasm/regex/windmill_parser_wasm_bg.wasm b/cli/wasm/regex/windmill_parser_wasm_bg.wasm index 45a144132d..34d6284a98 100644 Binary files a/cli/wasm/regex/windmill_parser_wasm_bg.wasm and b/cli/wasm/regex/windmill_parser_wasm_bg.wasm differ diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 2c35d77034..60d8af50d4 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.575.4", + "version": "1.576.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.575.4", + "version": "1.576.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { @@ -79,7 +79,7 @@ "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.574.1", "windmill-parser-wasm-py": "1.538.0", - "windmill-parser-wasm-regex": "1.574.1", + "windmill-parser-wasm-regex": "1.575.4", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.558.1", "windmill-parser-wasm-ts": "1.565.0", @@ -13759,9 +13759,9 @@ "integrity": "sha512-s+bdIgT/fA5em3zYUwF8D14uA/dZh7iu0krZYZQqZUO7txN37hwSCVfovbMkIwm4zPbsJ50mU8DRLt7UpAPZIw==" }, "node_modules/windmill-parser-wasm-regex": { - "version": "1.574.1", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.574.1.tgz", - "integrity": "sha512-KnNBnpTGcnBPSzTQHdsjTPJbRQ81iem98eUnMGO8Zt1qgBFJ/eFZPwmUg6TZMbMITzTi0QqRCJj08eQID1FcPg==" + "version": "1.575.4", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.575.4.tgz", + "integrity": "sha512-G7Ni8XxdSKLNew50B1KRmwn/z9qq3obpEDaaDFS7Z74MeUdaBB/TD69SY6RxDL8SZrh0Tl7SCbS/GGLnt1AzkQ==" }, "node_modules/windmill-parser-wasm-ruby": { "version": "1.526.1", diff --git a/frontend/package.json b/frontend/package.json index 7bd2758467..f16b6cdb18 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.575.4", + "version": "1.576.0", "scripts": { "dev": "vite dev", "build": "vite build", @@ -144,7 +144,7 @@ "windmill-parser-wasm-nu": "1.510.1", "windmill-parser-wasm-php": "1.574.1", "windmill-parser-wasm-py": "1.538.0", - "windmill-parser-wasm-regex": "1.574.1", + "windmill-parser-wasm-regex": "1.575.4", "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.558.1", "windmill-parser-wasm-ts": "1.565.0", diff --git a/frontend/src/lib/components/AppConnectInner.svelte b/frontend/src/lib/components/AppConnectInner.svelte index 817a6f4f16..6e4854e550 100644 --- a/frontend/src/lib/components/AppConnectInner.svelte +++ b/frontend/src/lib/components/AppConnectInner.svelte @@ -33,6 +33,7 @@ import TextInput from './text_input/TextInput.svelte' import { usePromise } from '$lib/svelte5Utils.svelte' import { pollJobResult } from './jobs/utils' + import { sameTopDomainOrigin } from '$lib/cookies' interface Props { step?: number @@ -239,7 +240,7 @@ function popupListener(event) { console.log('Received oauth popup message', event) let data = event.data - if (event.origin == null || event.origin !== window.location.origin) { + if (!sameTopDomainOrigin(event.origin, window.location.origin)) { console.log( 'Received oauth popup message from different origin', event.origin, @@ -397,6 +398,7 @@ } window.addEventListener('message', popupListener) window.addEventListener('storage', handleStorageEvent) + console.log('opening popup', url.toString()) window.open(url.toString(), '_blank', 'popup=true') step += 1 } diff --git a/frontend/src/lib/components/AppConnectLightweightResourcePicker.svelte b/frontend/src/lib/components/AppConnectLightweightResourcePicker.svelte index 9e3fe4b55a..81dd26c04b 100644 --- a/frontend/src/lib/components/AppConnectLightweightResourcePicker.svelte +++ b/frontend/src/lib/components/AppConnectLightweightResourcePicker.svelte @@ -42,7 +42,7 @@ {/if} -