From d2787cfd78e9f28c75490cbfc53ef2ce38cc3e93 Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Fri, 9 May 2025 09:50:01 +0200 Subject: [PATCH] feat: preprocessor refactor (#5629) * feat: preprocessor refactor * nit * oneOf kind support + new capture format + fix build * fix build * fix build * update preprocessor templates * cache runnable version info * update ee ref * update hub scripts && ee ref --- ...bbc4718cbb8ad6e2f13eae6b14f129d12936.json} | 14 +- ...cbb60a5446993217baffa83f843bc12a5ac73.json | 35 + ...20b59bbc414cfb92e960c174dd63fc180e187.json | 29 - ...3c840d8aff3df4a6926f569792f69b2e9601f.json | 23 + ...35adf640d44d607e62a1fb296e38ac3fdd36.json} | 10 +- ...7f1f9f836bd5b6866d3db26b25428d39c0b23.json | 29 - ...94cb1a32c4991cbd25046282d34272587d2de.json | 130 ---- ...2c6cd2932f28ab6490ace027ed3a790b2a0f7.json | 28 - ...3c989e1575fbd20ef22766d2887a86dce15e1.json | 23 - ...486b313d45347dfd9d4793d60e2760763ca3.json} | 13 +- ...a960b89e02ec40b292f3b5198baf2a1d3dbf.json} | 8 +- ...475fd947f047a9965bc110d7338208b77783d.json | 23 + ...8807bc464a81c0883e68f5a69b89b94b0b34b.json | 70 ++ ...10e2160d1b7e123c7ab226dc77d1789f164f9.json | 47 -- ...35c7342ac9a814c73f6290e1d6ebd3a55423.json} | 4 +- ...4feaeb1b671120f5f8c98fca8c89c6587fc35.json | 118 ---- ...7b3409d9166a028701981cfc280aaab3c4436.json | 41 -- backend/ee-repo-ref.txt | 2 +- ...50414082127_update_capture_format.down.sql | 2 + ...0250414082127_update_capture_format.up.sql | 2 + ...506092215_runnable_version_notify.down.sql | 4 + ...50506092215_runnable_version_notify.up.sql | 22 + backend/parsers/windmill-parser-ts/src/lib.rs | 2 +- .../parsers/windmill-parser-wasm/src/lib.rs | 2 +- backend/src/main.rs | 24 + backend/tests/worker.rs | 3 + backend/windmill-api/openapi.yaml | 7 +- backend/windmill-api/src/args.rs | 615 +++++++++++------- backend/windmill-api/src/capture.rs | 119 ++-- backend/windmill-api/src/gcp_triggers_ee.rs | 23 +- backend/windmill-api/src/http_trigger_args.rs | 204 ++++++ backend/windmill-api/src/http_triggers.rs | 126 +--- backend/windmill-api/src/jobs.rs | 317 ++++----- backend/windmill-api/src/lib.rs | 3 + backend/windmill-api/src/mcp.rs | 2 - backend/windmill-api/src/mqtt_triggers.rs | 106 +-- .../windmill-api/src/postgres_triggers/mod.rs | 17 +- .../src/postgres_triggers/trigger.rs | 70 +- backend/windmill-api/src/trigger_helpers.rs | 332 ++++++++++ .../windmill-api/src/websocket_triggers.rs | 57 +- backend/windmill-common/src/jobs.rs | 87 +-- backend/windmill-common/src/lib.rs | 250 +++++-- backend/windmill-queue/src/jobs.rs | 53 +- backend/windmill-queue/src/schedule.rs | 16 +- backend/windmill-worker/src/worker_flow.rs | 38 +- .../windmill-worker/src/worker_lockfiles.rs | 6 +- frontend/package-lock.json | 6 +- frontend/package.json | 2 +- frontend/src/lib/components/ArgInput.svelte | 25 +- .../schema/FlowPropertyEditor.svelte | 9 +- .../components/schema/PropertyEditor.svelte | 4 +- .../components/triggers/CaptureTable.svelte | 62 +- .../triggers/gcp/GcpTriggerEditorInner.svelte | 2 +- .../http/RouteBodyTransformerOption.svelte | 4 +- .../src/lib/components/triggers/http/utils.ts | 4 +- .../kafka/KafkaTriggerEditorInner.svelte | 2 +- .../mqtt/MqttTriggerEditorInner.svelte | 2 +- .../nats/NatsTriggerEditorInner.svelte | 6 +- .../triggers/sqs/SqsTriggerEditorInner.svelte | 2 +- .../WebsocketTriggerEditorInner.svelte | 2 +- frontend/src/lib/script_helpers.ts | 326 +++++----- 61 files changed, 2068 insertions(+), 1546 deletions(-) rename backend/.sqlx/{query-75451b6d48e4c26812ae64981d0d968b8fb0bf4374a2fccc167fa879bad7078f.json => query-0937e25e89959447e7cb1816c112bbc4718cbb8ad6e2f13eae6b14f129d12936.json} (87%) create mode 100644 backend/.sqlx/query-0e296134f05593edc989c628c00cbb60a5446993217baffa83f843bc12a5ac73.json delete mode 100644 backend/.sqlx/query-3a534b4fc36171efaa7c647f48320b59bbc414cfb92e960c174dd63fc180e187.json create mode 100644 backend/.sqlx/query-4455c7e8aa7616f3d547c5eb7d93c840d8aff3df4a6926f569792f69b2e9601f.json rename backend/.sqlx/{query-7cba31d597215a343cb0bca5b204a15fbba193262f7895c2bba90feb4215d6f3.json => query-4f547c0fd54f3bc57212ce87810e35adf640d44d607e62a1fb296e38ac3fdd36.json} (66%) delete mode 100644 backend/.sqlx/query-6d36da815795d5cac2e76b4d34a7f1f9f836bd5b6866d3db26b25428d39c0b23.json delete mode 100644 backend/.sqlx/query-89de3ff8ab32e545efcbcda05f994cb1a32c4991cbd25046282d34272587d2de.json delete mode 100644 backend/.sqlx/query-8fb2581a439c26391e66ae7fac32c6cd2932f28ab6490ace027ed3a790b2a0f7.json delete mode 100644 backend/.sqlx/query-97048ce0bcabb9baecb80cde5ab3c989e1575fbd20ef22766d2887a86dce15e1.json rename backend/.sqlx/{query-a0833b9899833166891c5de926f78632fae1123e736d728bf92cb2de004b6826.json => query-9b60fa8a1003015bc5a7cdbee9a4486b313d45347dfd9d4793d60e2760763ca3.json} (69%) rename backend/.sqlx/{query-6781ba76dfce321dca4634566496ea5d698ac09d8264e35dfaa4cd8edc9e8414.json => query-9c50e3a136a8ee3ec56e083f26d3a960b89e02ec40b292f3b5198baf2a1d3dbf.json} (76%) create mode 100644 backend/.sqlx/query-9dec888d4b0666d1843fbbc4fb2475fd947f047a9965bc110d7338208b77783d.json create mode 100644 backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json delete mode 100644 backend/.sqlx/query-c794ff5e14429a1bc56f5e70cfd10e2160d1b7e123c7ab226dc77d1789f164f9.json rename backend/.sqlx/{query-f9f7f31390f8ea1f4facd8e6a888886ea136f2327169bf93f126e2ef130d0946.json => query-eac595e19e5c8e70f1514ef29dec35c7342ac9a814c73f6290e1d6ebd3a55423.json} (75%) delete mode 100644 backend/.sqlx/query-ec7836df5f9056ec70015800b7f4feaeb1b671120f5f8c98fca8c89c6587fc35.json delete mode 100644 backend/.sqlx/query-f44f1a794ab217be05a9feb21e57b3409d9166a028701981cfc280aaab3c4436.json create mode 100644 backend/migrations/20250414082127_update_capture_format.down.sql create mode 100644 backend/migrations/20250414082127_update_capture_format.up.sql create mode 100644 backend/migrations/20250506092215_runnable_version_notify.down.sql create mode 100644 backend/migrations/20250506092215_runnable_version_notify.up.sql create mode 100644 backend/windmill-api/src/http_trigger_args.rs create mode 100644 backend/windmill-api/src/trigger_helpers.rs diff --git a/backend/.sqlx/query-75451b6d48e4c26812ae64981d0d968b8fb0bf4374a2fccc167fa879bad7078f.json b/backend/.sqlx/query-0937e25e89959447e7cb1816c112bbc4718cbb8ad6e2f13eae6b14f129d12936.json similarity index 87% rename from backend/.sqlx/query-75451b6d48e4c26812ae64981d0d968b8fb0bf4374a2fccc167fa879bad7078f.json rename to backend/.sqlx/query-0937e25e89959447e7cb1816c112bbc4718cbb8ad6e2f13eae6b14f129d12936.json index cf2b476448..7d24993172 100644 --- a/backend/.sqlx/query-75451b6d48e4c26812ae64981d0d968b8fb0bf4374a2fccc167fa879bad7078f.json +++ b/backend/.sqlx/query-0937e25e89959447e7cb1816c112bbc4718cbb8ad6e2f13eae6b14f129d12936.json @@ -1,12 +1,12 @@ { "db_name": "PostgreSQL", - "query": "select path, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor, on_behalf_of_email, created_by from script where hash = $1 AND workspace_id = $2", + "query": "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor, on_behalf_of_email, created_by, path from script where hash = $1 AND workspace_id = $2", "describe": { "columns": [ { "ordinal": 0, - "name": "path", - "type_info": "Varchar" + "name": "hash", + "type_info": "Int8" }, { "ordinal": 1, @@ -101,6 +101,11 @@ "ordinal": 13, "name": "created_by", "type_info": "Varchar" + }, + { + "ordinal": 14, + "name": "path", + "type_info": "Varchar" } ], "parameters": { @@ -123,8 +128,9 @@ true, true, true, + false, false ] }, - "hash": "75451b6d48e4c26812ae64981d0d968b8fb0bf4374a2fccc167fa879bad7078f" + "hash": "0937e25e89959447e7cb1816c112bbc4718cbb8ad6e2f13eae6b14f129d12936" } diff --git a/backend/.sqlx/query-0e296134f05593edc989c628c00cbb60a5446993217baffa83f843bc12a5ac73.json b/backend/.sqlx/query-0e296134f05593edc989c628c00cbb60a5446993217baffa83f843bc12a5ac73.json new file mode 100644 index 0000000000..5b086529a6 --- /dev/null +++ b/backend/.sqlx/query-0e296134f05593edc989c628c00cbb60a5446993217baffa83f843bc12a5ac73.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n value->'preprocessor_module' IS NOT NULL as has_preprocessor,\n value->'preprocessor_module'->'value'->'input_transforms'->'wm_trigger' IS NOT NULL as is_v1_preprocessor,\n schema as \"schema: _\"\n FROM flow \n WHERE workspace_id = $1 \n AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "has_preprocessor", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "is_v1_preprocessor", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "schema: _", + "type_info": "Json" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null, + null, + true + ] + }, + "hash": "0e296134f05593edc989c628c00cbb60a5446993217baffa83f843bc12a5ac73" +} diff --git a/backend/.sqlx/query-3a534b4fc36171efaa7c647f48320b59bbc414cfb92e960c174dd63fc180e187.json b/backend/.sqlx/query-3a534b4fc36171efaa7c647f48320b59bbc414cfb92e960c174dd63fc180e187.json deleted file mode 100644 index 11d343ba4d..0000000000 --- a/backend/.sqlx/query-3a534b4fc36171efaa7c647f48320b59bbc414cfb92e960c174dd63fc180e187.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT tag, dedicated_worker from flow WHERE path = $1 and workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "tag", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "dedicated_worker", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - true, - true - ] - }, - "hash": "3a534b4fc36171efaa7c647f48320b59bbc414cfb92e960c174dd63fc180e187" -} diff --git a/backend/.sqlx/query-4455c7e8aa7616f3d547c5eb7d93c840d8aff3df4a6926f569792f69b2e9601f.json b/backend/.sqlx/query-4455c7e8aa7616f3d547c5eb7d93c840d8aff3df4a6926f569792f69b2e9601f.json new file mode 100644 index 0000000000..8c90c0c765 --- /dev/null +++ b/backend/.sqlx/query-4455c7e8aa7616f3d547c5eb7d93c840d8aff3df4a6926f569792f69b2e9601f.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "select hash from script where path = $1 AND workspace_id = $2 AND deleted = false AND lock IS not NULL AND lock_error_logs IS NULL ORDER BY created_at DESC LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "hash", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "4455c7e8aa7616f3d547c5eb7d93c840d8aff3df4a6926f569792f69b2e9601f" +} diff --git a/backend/.sqlx/query-7cba31d597215a343cb0bca5b204a15fbba193262f7895c2bba90feb4215d6f3.json b/backend/.sqlx/query-4f547c0fd54f3bc57212ce87810e35adf640d44d607e62a1fb296e38ac3fdd36.json similarity index 66% rename from backend/.sqlx/query-7cba31d597215a343cb0bca5b204a15fbba193262f7895c2bba90feb4215d6f3.json rename to backend/.sqlx/query-4f547c0fd54f3bc57212ce87810e35adf640d44d607e62a1fb296e38ac3fdd36.json index a313dd6076..15b107811b 100644 --- a/backend/.sqlx/query-7cba31d597215a343cb0bca5b204a15fbba193262f7895c2bba90feb4215d6f3.json +++ b/backend/.sqlx/query-4f547c0fd54f3bc57212ce87810e35adf640d44d607e62a1fb296e38ac3fdd36.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT \n id, \n created_at, \n trigger_kind AS \"trigger_kind: _\",\n CASE \n WHEN pg_column_size(payload) < 40000 THEN payload \n ELSE '\"WINDMILL_TOO_BIG\"'::jsonb \n END AS \"payload!: _\",\n trigger_extra AS \"trigger_extra: _\"\n FROM \n capture\n WHERE \n workspace_id = $1 \n AND path = $2 \n AND is_flow = $3 \n AND ($4::trigger_kind IS NULL OR trigger_kind = $4)\n ORDER BY \n created_at DESC\n OFFSET $5\n LIMIT $6\n ", + "query": "\n SELECT \n id, \n created_at, \n trigger_kind AS \"trigger_kind: _\",\n CASE \n WHEN pg_column_size(main_args) < 40000 THEN main_args \n ELSE '\"WINDMILL_TOO_BIG\"'::jsonb \n END AS \"main_args!: _\",\n CASE\n WHEN pg_column_size(preprocessor_args) < 40000 THEN preprocessor_args\n ELSE '\"WINDMILL_TOO_BIG\"'::jsonb\n END AS \"preprocessor_args: _\"\n FROM \n capture\n WHERE \n workspace_id = $1 \n AND path = $2 \n AND is_flow = $3 \n AND ($4::trigger_kind IS NULL OR trigger_kind = $4)\n ORDER BY \n created_at DESC\n OFFSET $5\n LIMIT $6\n ", "describe": { "columns": [ { @@ -38,12 +38,12 @@ }, { "ordinal": 3, - "name": "payload!: _", + "name": "main_args!: _", "type_info": "Jsonb" }, { "ordinal": 4, - "name": "trigger_extra: _", + "name": "preprocessor_args: _", "type_info": "Jsonb" } ], @@ -80,8 +80,8 @@ false, false, null, - true + null ] }, - "hash": "7cba31d597215a343cb0bca5b204a15fbba193262f7895c2bba90feb4215d6f3" + "hash": "4f547c0fd54f3bc57212ce87810e35adf640d44d607e62a1fb296e38ac3fdd36" } diff --git a/backend/.sqlx/query-6d36da815795d5cac2e76b4d34a7f1f9f836bd5b6866d3db26b25428d39c0b23.json b/backend/.sqlx/query-6d36da815795d5cac2e76b4d34a7f1f9f836bd5b6866d3db26b25428d39c0b23.json deleted file mode 100644 index 009c8a0e26..0000000000 --- a/backend/.sqlx/query-6d36da815795d5cac2e76b4d34a7f1f9f836bd5b6866d3db26b25428d39c0b23.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT on_behalf_of_email, edited_by FROM flow WHERE path = $1 AND workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "on_behalf_of_email", - "type_info": "Text" - }, - { - "ordinal": 1, - "name": "edited_by", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - true, - false - ] - }, - "hash": "6d36da815795d5cac2e76b4d34a7f1f9f836bd5b6866d3db26b25428d39c0b23" -} diff --git a/backend/.sqlx/query-89de3ff8ab32e545efcbcda05f994cb1a32c4991cbd25046282d34272587d2de.json b/backend/.sqlx/query-89de3ff8ab32e545efcbcda05f994cb1a32c4991cbd25046282d34272587d2de.json deleted file mode 100644 index 99c2dc90a2..0000000000 --- a/backend/.sqlx/query-89de3ff8ab32e545efcbcda05f994cb1a32c4991cbd25046282d34272587d2de.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor, on_behalf_of_email, created_by from script where path = $1 AND workspace_id = $2 AND\n created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND\n deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "hash", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "tag", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "concurrency_key", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "concurrent_limit", - "type_info": "Int4" - }, - { - "ordinal": 4, - "name": "concurrency_time_window_s", - "type_info": "Int4" - }, - { - "ordinal": 5, - "name": "cache_ttl", - "type_info": "Int4" - }, - { - "ordinal": 6, - "name": "language: ScriptLang", - "type_info": { - "Custom": { - "name": "script_lang", - "kind": { - "Enum": [ - "python3", - "deno", - "go", - "bash", - "postgresql", - "nativets", - "bun", - "mysql", - "bigquery", - "snowflake", - "graphql", - "powershell", - "mssql", - "php", - "bunnative", - "rust", - "ansible", - "csharp", - "oracledb", - "nu", - "java" - ] - } - } - } - }, - { - "ordinal": 7, - "name": "dedicated_worker", - "type_info": "Bool" - }, - { - "ordinal": 8, - "name": "priority", - "type_info": "Int2" - }, - { - "ordinal": 9, - "name": "delete_after_use", - "type_info": "Bool" - }, - { - "ordinal": 10, - "name": "timeout", - "type_info": "Int4" - }, - { - "ordinal": 11, - "name": "has_preprocessor", - "type_info": "Bool" - }, - { - "ordinal": 12, - "name": "on_behalf_of_email", - "type_info": "Text" - }, - { - "ordinal": 13, - "name": "created_by", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - true, - true, - true, - true, - true, - false, - true, - true, - true, - true, - true, - true, - false - ] - }, - "hash": "89de3ff8ab32e545efcbcda05f994cb1a32c4991cbd25046282d34272587d2de" -} diff --git a/backend/.sqlx/query-8fb2581a439c26391e66ae7fac32c6cd2932f28ab6490ace027ed3a790b2a0f7.json b/backend/.sqlx/query-8fb2581a439c26391e66ae7fac32c6cd2932f28ab6490ace027ed3a790b2a0f7.json deleted file mode 100644 index 0ae81ab6dd..0000000000 --- a/backend/.sqlx/query-8fb2581a439c26391e66ae7fac32c6cd2932f28ab6490ace027ed3a790b2a0f7.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT edited_by, on_behalf_of_email FROM flow WHERE path = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "on_behalf_of_email", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - true - ] - }, - "hash": "8fb2581a439c26391e66ae7fac32c6cd2932f28ab6490ace027ed3a790b2a0f7" -} diff --git a/backend/.sqlx/query-97048ce0bcabb9baecb80cde5ab3c989e1575fbd20ef22766d2887a86dce15e1.json b/backend/.sqlx/query-97048ce0bcabb9baecb80cde5ab3c989e1575fbd20ef22766d2887a86dce15e1.json deleted file mode 100644 index c8ab870c76..0000000000 --- a/backend/.sqlx/query-97048ce0bcabb9baecb80cde5ab3c989e1575fbd20ef22766d2887a86dce15e1.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT flow.versions[array_upper(flow.versions, 1)] AS \"version!: i64\"\n FROM flow WHERE path = $1 AND workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "version!: i64", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "97048ce0bcabb9baecb80cde5ab3c989e1575fbd20ef22766d2887a86dce15e1" -} diff --git a/backend/.sqlx/query-a0833b9899833166891c5de926f78632fae1123e736d728bf92cb2de004b6826.json b/backend/.sqlx/query-9b60fa8a1003015bc5a7cdbee9a4486b313d45347dfd9d4793d60e2760763ca3.json similarity index 69% rename from backend/.sqlx/query-a0833b9899833166891c5de926f78632fae1123e736d728bf92cb2de004b6826.json rename to backend/.sqlx/query-9b60fa8a1003015bc5a7cdbee9a4486b313d45347dfd9d4793d60e2760763ca3.json index a837158a37..f4251250be 100644 --- a/backend/.sqlx/query-a0833b9899833166891c5de926f78632fae1123e736d728bf92cb2de004b6826.json +++ b/backend/.sqlx/query-9b60fa8a1003015bc5a7cdbee9a4486b313d45347dfd9d4793d60e2760763ca3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor, on_behalf_of_email, edited_by\n FROM flow \n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n WHERE flow.path = $1 and flow.workspace_id = $2", + "query": "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor, on_behalf_of_email, edited_by, flow_version.id AS version\n FROM flow\n INNER JOIN flow_version\n ON flow_version.id = $3\n WHERE flow.path = $1 and flow.workspace_id = $2", "describe": { "columns": [ { @@ -32,12 +32,18 @@ "ordinal": 5, "name": "edited_by", "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "version", + "type_info": "Int8" } ], "parameters": { "Left": [ "Text", - "Text" + "Text", + "Int8" ] }, "nullable": [ @@ -46,8 +52,9 @@ null, null, true, + false, false ] }, - "hash": "a0833b9899833166891c5de926f78632fae1123e736d728bf92cb2de004b6826" + "hash": "9b60fa8a1003015bc5a7cdbee9a4486b313d45347dfd9d4793d60e2760763ca3" } diff --git a/backend/.sqlx/query-6781ba76dfce321dca4634566496ea5d698ac09d8264e35dfaa4cd8edc9e8414.json b/backend/.sqlx/query-9c50e3a136a8ee3ec56e083f26d3a960b89e02ec40b292f3b5198baf2a1d3dbf.json similarity index 76% rename from backend/.sqlx/query-6781ba76dfce321dca4634566496ea5d698ac09d8264e35dfaa4cd8edc9e8414.json rename to backend/.sqlx/query-9c50e3a136a8ee3ec56e083f26d3a960b89e02ec40b292f3b5198baf2a1d3dbf.json index 2e4638f1f7..c21c30c012 100644 --- a/backend/.sqlx/query-6781ba76dfce321dca4634566496ea5d698ac09d8264e35dfaa4cd8edc9e8414.json +++ b/backend/.sqlx/query-9c50e3a136a8ee3ec56e083f26d3a960b89e02ec40b292f3b5198baf2a1d3dbf.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT \n id, \n created_at, \n trigger_kind AS \"trigger_kind: _\", \n payload AS \"payload!: _\", \n trigger_extra AS \"trigger_extra: _\"\n FROM \n capture\n WHERE \n id = $1 \n AND workspace_id = $2\n ", + "query": "\n SELECT \n id, \n created_at, \n trigger_kind AS \"trigger_kind: _\", \n main_args AS \"main_args!: _\", \n preprocessor_args AS \"preprocessor_args: _\"\n FROM \n capture\n WHERE \n id = $1 \n AND workspace_id = $2\n ", "describe": { "columns": [ { @@ -38,12 +38,12 @@ }, { "ordinal": 3, - "name": "payload!: _", + "name": "main_args!: _", "type_info": "Jsonb" }, { "ordinal": 4, - "name": "trigger_extra: _", + "name": "preprocessor_args: _", "type_info": "Jsonb" } ], @@ -61,5 +61,5 @@ true ] }, - "hash": "6781ba76dfce321dca4634566496ea5d698ac09d8264e35dfaa4cd8edc9e8414" + "hash": "9c50e3a136a8ee3ec56e083f26d3a960b89e02ec40b292f3b5198baf2a1d3dbf" } diff --git a/backend/.sqlx/query-9dec888d4b0666d1843fbbc4fb2475fd947f047a9965bc110d7338208b77783d.json b/backend/.sqlx/query-9dec888d4b0666d1843fbbc4fb2475fd947f047a9965bc110d7338208b77783d.json new file mode 100644 index 0000000000..7c49fe890e --- /dev/null +++ b/backend/.sqlx/query-9dec888d4b0666d1843fbbc4fb2475fd947f047a9965bc110d7338208b77783d.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT flow_version.id from flow\n INNER JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n WHERE flow.path = $1 and flow.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "9dec888d4b0666d1843fbbc4fb2475fd947f047a9965bc110d7338208b77783d" +} diff --git a/backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json b/backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json new file mode 100644 index 0000000000..a2a19a6c35 --- /dev/null +++ b/backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json @@ -0,0 +1,70 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT has_preprocessor, language as \"language: _\", content, schema as \"schema: _\" FROM script WHERE workspace_id = $1 AND hash = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "has_preprocessor", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "language: _", + "type_info": { + "Custom": { + "name": "script_lang", + "kind": { + "Enum": [ + "python3", + "deno", + "go", + "bash", + "postgresql", + "nativets", + "bun", + "mysql", + "bigquery", + "snowflake", + "graphql", + "powershell", + "mssql", + "php", + "bunnative", + "rust", + "ansible", + "csharp", + "oracledb", + "nu", + "java" + ] + } + } + } + }, + { + "ordinal": 2, + "name": "content", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "schema: _", + "type_info": "Json" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [ + true, + false, + false, + true + ] + }, + "hash": "a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b" +} diff --git a/backend/.sqlx/query-c794ff5e14429a1bc56f5e70cfd10e2160d1b7e123c7ab226dc77d1789f164f9.json b/backend/.sqlx/query-c794ff5e14429a1bc56f5e70cfd10e2160d1b7e123c7ab226dc77d1789f164f9.json deleted file mode 100644 index c4810efe68..0000000000 --- a/backend/.sqlx/query-c794ff5e14429a1bc56f5e70cfd10e2160d1b7e123c7ab226dc77d1789f164f9.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT tag, dedicated_worker, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor, on_behalf_of_email, edited_by\n FROM flow \n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n WHERE flow.path = $1 and flow.workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "tag", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "dedicated_worker", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "has_preprocessor", - "type_info": "Bool" - }, - { - "ordinal": 3, - "name": "on_behalf_of_email", - "type_info": "Text" - }, - { - "ordinal": 4, - "name": "edited_by", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - true, - true, - null, - true, - false - ] - }, - "hash": "c794ff5e14429a1bc56f5e70cfd10e2160d1b7e123c7ab226dc77d1789f164f9" -} diff --git a/backend/.sqlx/query-f9f7f31390f8ea1f4facd8e6a888886ea136f2327169bf93f126e2ef130d0946.json b/backend/.sqlx/query-eac595e19e5c8e70f1514ef29dec35c7342ac9a814c73f6290e1d6ebd3a55423.json similarity index 75% rename from backend/.sqlx/query-f9f7f31390f8ea1f4facd8e6a888886ea136f2327169bf93f126e2ef130d0946.json rename to backend/.sqlx/query-eac595e19e5c8e70f1514ef29dec35c7342ac9a814c73f6290e1d6ebd3a55423.json index a97a131baa..8890104678 100644 --- a/backend/.sqlx/query-f9f7f31390f8ea1f4facd8e6a888886ea136f2327169bf93f126e2ef130d0946.json +++ b/backend/.sqlx/query-eac595e19e5c8e70f1514ef29dec35c7342ac9a814c73f6290e1d6ebd3a55423.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO \n capture (\n workspace_id, path, is_flow, trigger_kind, payload, trigger_extra, created_by\n )\n VALUES (\n $1, $2, $3, $4, $5, $6, $7\n )\n ", + "query": "\n INSERT INTO \n capture (\n workspace_id, path, is_flow, trigger_kind, main_args, preprocessor_args, created_by\n )\n VALUES (\n $1, $2, $3, $4, $5, $6, $7\n )\n ", "describe": { "columns": [], "parameters": { @@ -34,5 +34,5 @@ }, "nullable": [] }, - "hash": "f9f7f31390f8ea1f4facd8e6a888886ea136f2327169bf93f126e2ef130d0946" + "hash": "eac595e19e5c8e70f1514ef29dec35c7342ac9a814c73f6290e1d6ebd3a55423" } diff --git a/backend/.sqlx/query-ec7836df5f9056ec70015800b7f4feaeb1b671120f5f8c98fca8c89c6587fc35.json b/backend/.sqlx/query-ec7836df5f9056ec70015800b7f4feaeb1b671120f5f8c98fca8c89c6587fc35.json deleted file mode 100644 index 73ac1919c1..0000000000 --- a/backend/.sqlx/query-ec7836df5f9056ec70015800b7f4feaeb1b671120f5f8c98fca8c89c6587fc35.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "select tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, on_behalf_of_email, created_by from script where hash = $1 AND workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "tag", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "concurrency_key", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "concurrent_limit", - "type_info": "Int4" - }, - { - "ordinal": 3, - "name": "concurrency_time_window_s", - "type_info": "Int4" - }, - { - "ordinal": 4, - "name": "cache_ttl", - "type_info": "Int4" - }, - { - "ordinal": 5, - "name": "language: ScriptLang", - "type_info": { - "Custom": { - "name": "script_lang", - "kind": { - "Enum": [ - "python3", - "deno", - "go", - "bash", - "postgresql", - "nativets", - "bun", - "mysql", - "bigquery", - "snowflake", - "graphql", - "powershell", - "mssql", - "php", - "bunnative", - "rust", - "ansible", - "csharp", - "oracledb", - "nu", - "java" - ] - } - } - } - }, - { - "ordinal": 6, - "name": "dedicated_worker", - "type_info": "Bool" - }, - { - "ordinal": 7, - "name": "priority", - "type_info": "Int2" - }, - { - "ordinal": 8, - "name": "delete_after_use", - "type_info": "Bool" - }, - { - "ordinal": 9, - "name": "timeout", - "type_info": "Int4" - }, - { - "ordinal": 10, - "name": "on_behalf_of_email", - "type_info": "Text" - }, - { - "ordinal": 11, - "name": "created_by", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Int8", - "Text" - ] - }, - "nullable": [ - true, - true, - true, - true, - true, - false, - true, - true, - true, - true, - true, - false - ] - }, - "hash": "ec7836df5f9056ec70015800b7f4feaeb1b671120f5f8c98fca8c89c6587fc35" -} diff --git a/backend/.sqlx/query-f44f1a794ab217be05a9feb21e57b3409d9166a028701981cfc280aaab3c4436.json b/backend/.sqlx/query-f44f1a794ab217be05a9feb21e57b3409d9166a028701981cfc280aaab3c4436.json deleted file mode 100644 index 79ef441a8a..0000000000 --- a/backend/.sqlx/query-f44f1a794ab217be05a9feb21e57b3409d9166a028701981cfc280aaab3c4436.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT tag, dedicated_worker, on_behalf_of_email, edited_by from flow WHERE path = $1 and workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "tag", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "dedicated_worker", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "on_behalf_of_email", - "type_info": "Text" - }, - { - "ordinal": 3, - "name": "edited_by", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - true, - true, - true, - false - ] - }, - "hash": "f44f1a794ab217be05a9feb21e57b3409d9166a028701981cfc280aaab3c4436" -} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 4ba6ebd5b8..1e7c10394e 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -19b5681d1985ee5214f768e3b818faf6ca7d20b2 +4cc3cbd0f0536a2fa4814d98ec058f998044a70b diff --git a/backend/migrations/20250414082127_update_capture_format.down.sql b/backend/migrations/20250414082127_update_capture_format.down.sql new file mode 100644 index 0000000000..3739642e41 --- /dev/null +++ b/backend/migrations/20250414082127_update_capture_format.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE capture RENAME COLUMN preprocessor_args to trigger_extra; +ALTER TABLE capture RENAME COLUMN main_args to payload; diff --git a/backend/migrations/20250414082127_update_capture_format.up.sql b/backend/migrations/20250414082127_update_capture_format.up.sql new file mode 100644 index 0000000000..dc1653315e --- /dev/null +++ b/backend/migrations/20250414082127_update_capture_format.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE capture RENAME COLUMN trigger_extra to preprocessor_args; +ALTER TABLE capture RENAME COLUMN payload to main_args; \ No newline at end of file diff --git a/backend/migrations/20250506092215_runnable_version_notify.down.sql b/backend/migrations/20250506092215_runnable_version_notify.down.sql new file mode 100644 index 0000000000..b94db9dfdc --- /dev/null +++ b/backend/migrations/20250506092215_runnable_version_notify.down.sql @@ -0,0 +1,4 @@ +-- Add down migration script here +DROP TRIGGER script_update_trigger ON script; +DROP TRIGGER flow_update_trigger ON flow_version; +DROP FUNCTION notify_runnable_version_change(); diff --git a/backend/migrations/20250506092215_runnable_version_notify.up.sql b/backend/migrations/20250506092215_runnable_version_notify.up.sql new file mode 100644 index 0000000000..a8d11142f4 --- /dev/null +++ b/backend/migrations/20250506092215_runnable_version_notify.up.sql @@ -0,0 +1,22 @@ +-- Add up migration script here +CREATE OR REPLACE FUNCTION notify_runnable_version_change() +RETURNS TRIGGER AS $$ +DECLARE + source_type TEXT; +BEGIN + source_type := TG_ARGV[0]; + + PERFORM pg_notify('notify_runnable_version_change', NEW.workspace_id || ':' || source_type || ':' || NEW.path); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER script_update_trigger +AFTER UPDATE OF lock ON script +FOR EACH ROW +EXECUTE FUNCTION notify_runnable_version_change('script'); + +CREATE TRIGGER flow_update_trigger +AFTER INSERT ON flow_version +FOR EACH ROW +EXECUTE FUNCTION notify_runnable_version_change('flow'); diff --git a/backend/parsers/windmill-parser-ts/src/lib.rs b/backend/parsers/windmill-parser-ts/src/lib.rs index d4d34cecd6..2024f4880c 100644 --- a/backend/parsers/windmill-parser-ts/src/lib.rs +++ b/backend/parsers/windmill-parser-ts/src/lib.rs @@ -506,7 +506,7 @@ fn one_of_label(members: &Vec) -> Option { let Expr::Ident(Ident { sym, .. }) = &**key else { return None; }; - if sym != "label" { + if sym != "label" && sym != "kind" { return None; } diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs index 2f4bc83c1f..cd280e7af5 100644 --- a/backend/parsers/windmill-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -17,7 +17,7 @@ fn wrap_sig(r: anyhow::Result) -> String { #[cfg(feature = "ts-parser")] #[wasm_bindgen] -pub fn parse_deno(code: &str, main_override: Option, skip_params: Option) -> String { +pub fn parse_deno(code: &str, main_override: Option) -> String { wrap_sig(windmill_parser_ts::parse_deno_signature( code, false, diff --git a/backend/src/main.rs b/backend/src/main.rs index e6dc72be80..1e4f8010eb 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -787,6 +787,29 @@ Windmill Community Edition {GIT_VERSION} tracing::info!("Workspace premium change detected, invalidating workspace premium cache: {}", workspace_id); windmill_common::workspaces::IS_PREMIUM_CACHE.remove(workspace_id); }, + "notify_runnable_version_change" => { + let payload = n.payload(); + tracing::info!("Runnable version change detected: {}", payload); + match payload.split(':').collect::>().as_slice() { + [workspace_id, source_type, path] => { + let key = (workspace_id.to_string(), path.to_string()); + match source_type { + &"script" => { + windmill_common::DEPLOYED_SCRIPT_HASH_CACHE.remove(&key); + } + &"flow" => { + windmill_common::FLOW_VERSION_CACHE.remove(&key); + }, + _ => { + tracing::warn!("Unknown runnable version change payload: {}", payload); + } + } + }, + _ => { + tracing::warn!("Unknown runnable version change payload: {}", payload); + } + } + }, "notify_global_setting_change" => { tracing::info!("Global setting change detected: {}", n.payload()); match n.payload() { @@ -1108,6 +1131,7 @@ async fn listen_pg(url: &str) -> Option { "notify_global_setting_change", "notify_webhook_change", "notify_workspace_envs_change", + "notify_runnable_version_change", ]; #[cfg(feature = "cloud")] channels.push("notify_workspace_premium_change"); diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 4f33707eff..8b7509bb84 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -4687,6 +4687,7 @@ mod job_payload { path: "f/system/hello_with_nodes_flow".to_string(), dedicated_worker: None, apply_preprocessor: false, + version: 1443253234253454, }) .run_until_complete(&db, port) .await @@ -4731,6 +4732,7 @@ mod job_payload { path: "f/system/hello_with_preprocessor".to_string(), dedicated_worker: None, apply_preprocessor: true, + version: 1443253234253456, }) .run_until_complete_with(db, port, |id| async move { let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", id) @@ -4798,6 +4800,7 @@ mod job_payload { path: "f/system/hello_with_nodes_flow".to_string(), dedicated_worker: None, apply_preprocessor: true, + version: 1443253234253454, }) .run_until_complete(&db, port) .await diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 0e9d723cee..5041f0961a 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -16539,8 +16539,8 @@ components: properties: trigger_kind: $ref: "#/components/schemas/CaptureTriggerKind" - payload: {} - trigger_extra: {} + main_args: {} + preprocessor_args: {} id: type: integer created_at: @@ -16548,7 +16548,8 @@ components: format: date-time required: - trigger_kind - - payload + - main_args + - preprocessor_args - id - created_at CaptureConfig: diff --git a/backend/windmill-api/src/args.rs b/backend/windmill-api/src/args.rs index fd2ceefabe..567237cb05 100644 --- a/backend/windmill-api/src/args.rs +++ b/backend/windmill-api/src/args.rs @@ -1,7 +1,5 @@ use std::collections::HashMap; -#[cfg(feature = "parquet")] -use crate::job_helpers_ee::get_workspace_s3_resource; use axum::{ extract::{FromRequest, FromRequestParts, Multipart, Query, Request}, http::{HeaderMap, Uri}, @@ -9,139 +7,318 @@ use axum::{ }; use bytes::Bytes; use http::{header::CONTENT_TYPE, request::Parts, StatusCode}; -#[cfg(feature = "parquet")] -use object_store::{Attribute, Attributes}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use sqlx::types::JsonRawValue; -#[cfg(feature = "parquet")] -use windmill_common::s3_helpers::build_object_store_client; use windmill_common::{error::Error, worker::to_raw_value, DB}; -use windmill_queue::PushArgsOwned; +use windmill_queue::{PushArgsOwned, TriggerKind}; -use crate::db::ApiAuthed; -#[cfg(feature = "parquet")] -use crate::job_helpers_ee::{get_random_file_name, upload_file_internal}; +use crate::{ + db::ApiAuthed, + trigger_helpers::{get_runnable_format, RunnableFormat, RunnableFormatVersion, RunnableId}, +}; -#[derive(Debug, Default)] -pub struct WebhookArgs { - pub args: PushArgsOwned, - pub multipart: Option, - pub wrap_body: Option, +#[derive(Debug)] +pub enum RawBody { + Json(String), + CEJson(String), + Text(String), + Xml(String), + UrlEncoded(Bytes), + Multipart(Multipart), + Empty, } -impl WebhookArgs { +#[derive(Clone, Serialize)] +#[serde(untagged)] +pub enum Body { + HashMap(HashMap>), + NoHashMap(Box), +} + +#[derive(Clone, Default)] +pub struct WebhookArgsMetadata { + pub raw_string: Option, + pub headers: HashMap>, + pub method: http::Method, + pub query: HashMap>, + pub query_wrap_body: bool, + pub query_use_raw: bool, +} + +pub struct RawWebhookArgs { + pub body: RawBody, + pub metadata: WebhookArgsMetadata, +} + +#[derive(Clone)] +pub struct WebhookArgs { + pub body: Body, + pub metadata: WebhookArgsMetadata, +} + +// capture +// + +impl RawWebhookArgs { #[cfg(not(feature = "parquet"))] - pub async fn to_push_args_owned( - self, + pub async fn process_multipart( + _multipart: Multipart, _authed: &ApiAuthed, _db: &DB, _w_id: &str, - ) -> Result { - if self.multipart.is_some() { - return Err(Error::BadRequest(format!( - "multipart/form-data requires the parquet feature" - ))); - } - - Ok(self.args) + ) -> Result>, Error> { + return Err(Error::BadRequest(format!( + "multipart/form-data requires the parquet feature" + ))); } #[cfg(feature = "parquet")] - pub async fn to_push_args_owned( - mut self, + async fn process_multipart( + mut multipart: Multipart, + authed: &ApiAuthed, + db: &DB, + w_id: &str, + ) -> Result>, Error> { + use crate::job_helpers_ee::{ + get_random_file_name, get_workspace_s3_resource, upload_file_internal, + }; + use futures::TryStreamExt; + use object_store::{Attribute, Attributes}; + use windmill_common::s3_helpers::build_object_store_client; + + let (_, s3_resource) = get_workspace_s3_resource(authed, db, None, "", w_id, None).await?; + + if let Some(s3_resource) = s3_resource { + let s3_client = build_object_store_client(&s3_resource).await?; + + let mut body = HashMap::new(); + let mut files = HashMap::new(); + + while let Some(field) = multipart.next_field().await.map_err(|e| { + Error::BadRequest(format!("Error reading multipart field: {}", e.body_text())) + })? { + if let Some(name) = field.name().map(|x| x.to_string()) { + if let Some(content_type) = field.content_type() { + let ext = field + .file_name() + .map(|x| x.split('.').last()) + .flatten() + .map(|x| x.to_string()); + + let file_key = get_random_file_name(ext); + + let options = Attributes::from_iter(vec![ + (Attribute::ContentType, content_type.to_string()), + ( + Attribute::ContentDisposition, + if let Some(filename) = field.file_name() { + format!("inline; filename=\"{}\"", filename) + } else { + "inline".to_string() + }, + ), + ]) + .into(); + + let bytes_stream = field + .into_stream() + .map_err(|err| std::io::Error::new(std::io::ErrorKind::Other, err)); + + upload_file_internal(s3_client.clone(), &file_key, bytes_stream, options) + .await?; + + files.entry(name).or_insert(vec![]).push(serde_json::json!({ + "s3": &file_key + })); + } else { + body.insert(name, to_raw_value(&field.text().await.unwrap_or_default())); + } + } + } + + for (k, v) in files { + body.insert(k, to_raw_value(&v)); + } + + Ok(body) + } else { + Err(Error::BadRequest(format!( + "You need to connect your workspace to an S3 bucket to use multipart/form-data" + ))) + } + } + + pub async fn process_args( + self, + authed: &ApiAuthed, + db: &DB, + w_id: &str, + force_use_raw: Option, + ) -> Result { + let use_raw = force_use_raw.unwrap_or(self.metadata.query_use_raw); + + match self.body { + RawBody::Multipart(multipart) => { + let body = Self::process_multipart(multipart, authed, db, w_id).await?; + Ok(WebhookArgs { body: Body::HashMap(body), metadata: self.metadata }) + } + RawBody::Empty => { + let mut metadata = self.metadata; + if use_raw { + metadata.raw_string = Some("".to_string()); + } + Ok(WebhookArgs { body: Body::HashMap(HashMap::new()), metadata }) + } + RawBody::Text(s) | RawBody::Xml(s) => Ok(WebhookArgs { + body: Body::HashMap(HashMap::new()), + metadata: WebhookArgsMetadata { raw_string: Some(s), ..self.metadata }, + }), + RawBody::UrlEncoded(bytes) => { + let mut metadata = self.metadata; + if use_raw { + let raw_string = String::from_utf8(bytes.to_vec()) + .map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)))?; + metadata.raw_string = Some(raw_string); + } + let payload: HashMap> = serde_urlencoded::from_bytes(&bytes) + .map_err(|e| Error::BadRequest(format!("invalid urlencoded data: {}", e)))?; + let payload = payload + .into_iter() + .map(|(k, v)| (k, to_raw_value(&v))) + .collect::>(); + + Ok(WebhookArgs { body: Body::HashMap(payload), metadata }) + } + RawBody::Json(s) => WebhookArgs::from_json(self.metadata, use_raw, s).await, + RawBody::CEJson(s) => WebhookArgs::from_ce_json(self.metadata, use_raw, s).await, + } + } + + pub async fn to_main_args( + self, authed: &ApiAuthed, db: &DB, w_id: &str, ) -> Result { - use futures::TryStreamExt; + let args = self.process_args(authed, db, w_id, None).await?; + args.to_main_args() + } - if let Some(mut multipart) = self.multipart { - { - let (_, s3_resource) = - get_workspace_s3_resource(authed, db, None, "", w_id, None).await?; + pub async fn to_args_from_runnable( + self, + authed: &ApiAuthed, + db: &DB, + w_id: &str, + runnable_id: RunnableId, + skip_preprocessor: Option, + ) -> Result { + let args = self.process_args(authed, db, w_id, None).await?; + args.to_args_from_runnable(db, w_id, runnable_id, skip_preprocessor) + .await + } +} - if let Some(s3_resource) = s3_resource { - let s3_client = build_object_store_client(&s3_resource).await?; +#[derive(Serialize)] +struct WebhookPreprocessorEvent { + kind: String, + body: Box, + raw_string: Option, + headers: HashMap>, + query: HashMap>, +} - let mut body = HashMap::new(); - let mut files = HashMap::new(); +impl WebhookArgs { + pub fn to_main_args(self) -> Result { + self.to_args_from_format(RunnableFormat { + has_preprocessor: false, + version: RunnableFormatVersion::V2, + }) + } - while let Some(field) = multipart.next_field().await.map_err(|e| { - Error::BadRequest(format!( - "Error reading multipart field: {}", - e.body_text() - )) - })? { - if let Some(name) = field.name().map(|x| x.to_string()) { - if let Some(content_type) = field.content_type() { - let ext = field - .file_name() - .map(|x| x.split('.').last()) - .flatten() - .map(|x| x.to_string()); + pub async fn to_args_from_runnable( + self, + db: &DB, + w_id: &str, + runnable_id: RunnableId, + skip_preprocessor: Option, + ) -> Result { + if skip_preprocessor.unwrap_or(false) { + self.to_main_args() + } else { + let runnable_format = + get_runnable_format(runnable_id, w_id, db, &TriggerKind::Webhook).await?; - let file_key = get_random_file_name(ext); + self.to_args_from_format(runnable_format) + } + } - let options = Attributes::from_iter(vec![ - (Attribute::ContentType, content_type.to_string()), - ( - Attribute::ContentDisposition, - if let Some(filename) = field.file_name() { - format!("inline; filename=\"{}\"", filename) - } else { - "inline".to_string() - }, - ), - ]) - .into(); + pub fn to_args_from_format( + self, + runnable_format: RunnableFormat, + ) -> Result { + match runnable_format { + RunnableFormat { has_preprocessor: true, version: RunnableFormatVersion::V2 } => { + let mut args = HashMap::new(); - let bytes_stream = field.into_stream().map_err(|err| { - std::io::Error::new(std::io::ErrorKind::Other, err) - }); + args.insert( + "event".to_string(), + to_raw_value(&WebhookPreprocessorEvent { + kind: "webhook".to_string(), + body: to_raw_value(&self.body), + raw_string: self.metadata.raw_string, + headers: self.metadata.headers, + query: self.metadata.query, + }), + ); - upload_file_internal( - s3_client.clone(), - &file_key, - bytes_stream, - options, - ) - .await?; + Ok(PushArgsOwned { args, extra: None }) + } + RunnableFormat { has_preprocessor, .. } => { + let mut extra = HashMap::new(); - files.entry(name).or_insert(vec![]).push(serde_json::json!({ - "s3": &file_key - })); - } else { - body.insert( - name, - to_raw_value(&field.text().await.unwrap_or_default()), - ); - } + let WebhookArgsMetadata { query, query_wrap_body, headers, raw_string, .. } = + self.metadata; + + for (k, v) in headers { + extra.insert(k, v); + } + + for (k, v) in query { + extra.insert(k, v); + } + + if let Some(raw_string) = raw_string { + extra.insert("raw_string".to_string(), to_raw_value(&raw_string)); + } + + if has_preprocessor { + // if has preprocessor, it has to be v1 + extra.insert( + "wm_trigger".to_string(), + to_raw_value(&serde_json::json!({ + "kind": "webhook", + })), + ); + } + + let extra = if extra.is_empty() { None } else { Some(extra) }; + + match self.body { + Body::HashMap(mut body) => { + if query_wrap_body { + body = HashMap::from([("body".to_string(), to_raw_value(&body))]); } + Ok(PushArgsOwned { args: body, extra }) } - - for (k, v) in files { - body.insert(k, to_raw_value(&v)); + Body::NoHashMap(args) => { + let mut hm = HashMap::new(); + hm.insert("body".to_string(), args); + Ok(PushArgsOwned { args: hm, extra }) } - - if self.wrap_body.unwrap_or(false) { - self.args - .args - .insert("body".to_string(), to_raw_value(&body)); - } else { - self.args.args.extend(body); - } - - return Ok(self.args); } } - - return Err(Error::BadRequest(format!( - "You need to connect your workspace to an S3 bucket to use multipart/form-data" - ))); } - - Ok(self.args) } } @@ -166,26 +343,36 @@ async fn req_to_string( pub async fn try_from_request_body( request: Request, _state: &S, - use_raw: Option, - wrap_body: Option, -) -> Result + is_http_trigger: bool, +) -> Result where S: Send + Sync, { - let (content_type, mut extra, use_raw, wrap_body) = { + let (content_type, metadata) = { let headers_map = request.headers(); let content_type_header = headers_map.get(CONTENT_TYPE); let content_type = content_type_header.and_then(|value| value.to_str().ok()); let uri = request.uri(); - let query = Query::::try_from_uri(uri).unwrap().0; - let mut extra = build_extra(&headers_map, query.include_header); + let request_query = Query::::try_from_uri(uri).unwrap().0; + let headers = build_headers(&headers_map, request_query.include_header, is_http_trigger); let query_decode = DecodeQueries::from_uri(uri); + let mut query = HashMap::new(); if let Some(DecodeQueries(queries)) = query_decode { - extra.extend(queries); + query.extend(queries); } - let raw = query.raw.unwrap_or(use_raw.unwrap_or(false)); - let wrap_body = query.wrap_body.unwrap_or(wrap_body.unwrap_or(false)); - (content_type, extra, raw, wrap_body) + let raw = !is_http_trigger && request_query.raw.unwrap_or(false); + let wrap_body = !is_http_trigger && request_query.wrap_body.unwrap_or(false); + ( + content_type, + WebhookArgsMetadata { + headers, + query, + method: request.method().clone(), + raw_string: None, + query_wrap_body: wrap_body, + query_use_raw: raw, + }, + ) }; let no_content_type = content_type.is_none(); @@ -194,33 +381,19 @@ where .await .map_err(IntoResponse::into_response)?; if no_content_type && bytes.is_empty() { - if use_raw { - extra.insert("raw_string".to_string(), to_raw_value(&"".to_string())); - } - let mut args = HashMap::new(); - if wrap_body { - args.insert("body".to_string(), to_raw_value(&serde_json::json!({}))); - } - return Ok(WebhookArgs { - args: PushArgsOwned { extra: Some(extra), args: args }, - ..Default::default() - }); + Ok(RawWebhookArgs { body: RawBody::Empty, metadata }) + } else { + let str = String::from_utf8(bytes.to_vec()) + .map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?; + Ok(RawWebhookArgs { body: RawBody::Json(str), metadata }) } - let str = String::from_utf8(bytes.to_vec()) - .map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?; - - PushArgsOwned::from_json(extra, use_raw, wrap_body, str) - .await - .map(|args| WebhookArgs { args, ..Default::default() }) } else if content_type .unwrap() .starts_with("application/cloudevents+json") { let str = req_to_string(request, _state).await?; - PushArgsOwned::from_ce_json(extra, use_raw, str) - .await - .map(|args| WebhookArgs { args, ..Default::default() }) + Ok(RawWebhookArgs { body: RawBody::CEJson(str), metadata }) } else if content_type .unwrap() .starts_with("application/cloudevents-batch+json") @@ -231,11 +404,7 @@ where ) } else if content_type.unwrap().starts_with("text/plain") { let str = req_to_string(request, _state).await?; - extra.insert("raw_string".to_string(), to_raw_value(&str)); - Ok(WebhookArgs { - args: PushArgsOwned { extra: Some(extra), args: HashMap::new() }, - ..Default::default() - }) + Ok(RawWebhookArgs { body: RawBody::Text(str), metadata }) } else if content_type .unwrap() .starts_with("application/x-www-form-urlencoded") @@ -244,58 +413,32 @@ where .await .map_err(IntoResponse::into_response)?; - if use_raw { - let raw_string = String::from_utf8(bytes.to_vec()) - .map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?; - extra.insert("raw_string".to_string(), to_raw_value(&raw_string)); - } - - let payload: HashMap> = serde_urlencoded::from_bytes(&bytes) - .map_err(|e| { - Error::BadRequest(format!("invalid urlencoded data: {}", e)).into_response() - })?; - let payload = payload - .into_iter() - .map(|(k, v)| (k, to_raw_value(&v))) - .collect::>(); - - return Ok(WebhookArgs { - args: PushArgsOwned { extra: Some(extra), args: payload }, - ..Default::default() - }); + Ok(RawWebhookArgs { body: RawBody::UrlEncoded(bytes), metadata }) } else if content_type.unwrap().starts_with("application/xml") || content_type.unwrap().starts_with("text/xml") { let str = req_to_string(request, _state).await?; - extra.insert("raw_string".to_string(), to_raw_value(&str)); - Ok(WebhookArgs { - args: PushArgsOwned { extra: Some(extra), args: HashMap::new() }, - ..Default::default() - }) + Ok(RawWebhookArgs { body: RawBody::Xml(str), metadata }) } else if content_type.unwrap().starts_with("multipart/form-data") { let multipart = Multipart::from_request(request, _state) .await .map_err(IntoResponse::into_response)?; - Ok(WebhookArgs { - args: PushArgsOwned { extra: Some(extra), args: HashMap::new() }, - multipart: Some(multipart), - wrap_body: Some(wrap_body), - }) + Ok(RawWebhookArgs { body: RawBody::Multipart(multipart), metadata }) } else { Err(StatusCode::UNSUPPORTED_MEDIA_TYPE.into_response()) } } #[axum::async_trait] -impl FromRequest for WebhookArgs +impl FromRequest for RawWebhookArgs where S: Send + Sync, { type Rejection = Response; async fn from_request(request: Request, _state: &S) -> Result { - let args = try_from_request_body(request, _state, None, None).await?; + let args = try_from_request_body(request, _state, false).await?; Ok(args) } @@ -309,27 +452,37 @@ lazy_static::lazy_static! { .collect()).unwrap_or_default(); } -pub fn build_extra( +pub fn build_headers( headers: &HeaderMap, include_header: Option, + is_http_trigger: bool, ) -> HashMap> { - let mut args = HashMap::new(); - let whitelist = include_header - .map(|s| s.split(",").map(|s| s.to_string()).collect::>()) - .unwrap_or_default(); + let mut selected_headers = HashMap::new(); - whitelist - .iter() - .chain(INCLUDE_HEADERS.iter()) - .for_each(|h| { - if let Some(v) = headers.get(h) { - args.insert( - h.to_string().to_lowercase().replace('-', "_"), - to_raw_value(&v.to_str().unwrap().to_string()), - ); - } - }); - args + if is_http_trigger { + for (k, v) in headers.iter() { + selected_headers.insert( + k.to_string(), + to_raw_value(&v.to_str().unwrap_or("").to_string()), + ); + } + } else { + let whitelist = include_header + .map(|s| s.split(",").map(|s| s.to_string()).collect::>()) + .unwrap_or_default(); + whitelist + .iter() + .chain(INCLUDE_HEADERS.iter()) + .for_each(|h| { + if let Some(v) = headers.get(h) { + selected_headers.insert( + h.to_string().to_lowercase().replace('-', "_"), + to_raw_value(&v.to_str().unwrap_or("").to_string()), + ); + } + }); + } + selected_headers } #[derive(Deserialize)] @@ -414,69 +567,50 @@ fn restructure_cloudevents_metadata( } } -trait PushArgsOwnedExt: Sized { +impl WebhookArgs { async fn from_json( - extra: HashMap>, - use_raw: bool, - force_wrap_body: bool, - str: String, - ) -> Result; - - async fn from_ce_json( - extra: HashMap>, + mut metadata: WebhookArgsMetadata, use_raw: bool, str: String, - ) -> Result; -} - -impl PushArgsOwnedExt for PushArgsOwned { - async fn from_json( - mut extra: HashMap>, - use_raw: bool, - force_wrap_body: bool, - str: String, - ) -> Result { + ) -> Result { if use_raw { - extra.insert("raw_string".to_string(), to_raw_value(&str)); + metadata.raw_string = Some(str.clone()); } - let wrap_body = force_wrap_body || str.len() > 0 && str.chars().next().unwrap() != '{'; + let no_hashmap = str.len() > 0 && str.chars().next().unwrap() != '{'; - if wrap_body { + if no_hashmap { let args = serde_json::from_str::>>(&str) - .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)).into_response())? + .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))? .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)); - let mut hm = HashMap::new(); - hm.insert("body".to_string(), args); - Ok(PushArgsOwned { extra: Some(extra), args: hm }) + + Ok(Self { body: Body::NoHashMap(args), metadata }) } else { let hm = serde_json::from_str::>>>(&str) - .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)).into_response())? + .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))? .unwrap_or_else(HashMap::new); - Ok(PushArgsOwned { extra: Some(extra), args: hm }) + Ok(Self { body: Body::HashMap(hm), metadata }) } } async fn from_ce_json( - mut extra: HashMap>, + mut metadata: WebhookArgsMetadata, use_raw: bool, str: String, - ) -> Result { + ) -> Result { if use_raw { - extra.insert("raw_string".to_string(), to_raw_value(&str)); + metadata.raw_string = Some(str.clone()); } - let hm = serde_json::from_str::>>(&str).map_err(|e| { - Error::BadRequest(format!("invalid cloudevents+json: {}", e)).into_response() - })?; - let hm = restructure_cloudevents_metadata(hm).map_err(|e| e.into_response())?; - Ok(PushArgsOwned { extra: Some(extra), args: hm }) + let hm = serde_json::from_str::>>(&str) + .map_err(|e| Error::BadRequest(format!("invalid cloudevents+json: {}", e)))?; + let hm = restructure_cloudevents_metadata(hm)?; + Ok(Self { body: Body::HashMap(hm), metadata }) } } #[cfg(test)] mod tests { - use std::collections::HashMap; use super::*; @@ -514,24 +648,35 @@ mod tests { "data" : 1.5 } "#; - let extra = HashMap::new(); + let metadata = WebhookArgsMetadata::default(); - let a1 = PushArgsOwned::from_ce_json(extra.clone(), false, r1.to_string()) + let a1 = WebhookArgs::from_ce_json(metadata.clone(), false, r1.to_string()) .await .expect("Failed to parse the cloudevent"); - let a2 = PushArgsOwned::from_ce_json(extra.clone(), false, r2.to_string()) + let a2 = WebhookArgs::from_ce_json(metadata.clone(), false, r2.to_string()) .await .expect("Failed to parse the cloudevent"); - a1.args.get("WEBHOOK__METADATA__").expect( - "CloudEvents should generate a neighboring `webhook-metadata` field in PushArgs", - ); - assert_eq!( - a2.args - .get("body") - .expect("Cloud events with a data field with no wrapping curly brackets should be inside of a `body` field in PushArgs") - .to_string(), - "1.5" - ); + match a1.body { + Body::HashMap(body) => { + body.get("WEBHOOK__METADATA__").expect( + "CloudEvents should generate a neighboring `webhook-metadata` field in PushArgs", + ); + } + _ => panic!("Expected a HashMap"), + } + + match a2.body { + Body::HashMap(body) => { + assert_eq!( + body + .get("body") + .expect("Cloud events with a data field with no wrapping curly brackets should be inside of a `body` field in PushArgs") + .to_string(), + "1.5" + ); + } + _ => panic!("Expected a HashMap"), + } } } diff --git a/backend/windmill-api/src/capture.rs b/backend/windmill-api/src/capture.rs index 547b56954b..d296c7b2cc 100644 --- a/backend/windmill-api/src/capture.rs +++ b/backend/windmill-api/src/capture.rs @@ -8,10 +8,7 @@ #[cfg(feature = "http_trigger")] use { - crate::{ - args::try_from_request_body, - http_triggers::{build_http_trigger_extra, HttpMethod}, - }, + crate::http_trigger_args::{HttpMethod, RawHttpTriggerArgs}, axum::response::{IntoResponse, Response}, std::collections::HashMap, }; @@ -56,8 +53,9 @@ use { }; use crate::{ - args::WebhookArgs, + args::RawWebhookArgs, db::{ApiAuthed, DB}, + trigger_helpers::{RunnableFormat, RunnableFormatVersion}, users::fetch_api_authed, utils::RunnableKind, }; @@ -496,8 +494,8 @@ struct Capture { id: i64, created_at: chrono::DateTime, trigger_kind: TriggerKind, - payload: SqlxJson>, - trigger_extra: Option>>, + main_args: SqlxJson>, + preprocessor_args: Option>>, } #[derive(Deserialize)] @@ -525,10 +523,13 @@ async fn list_captures( created_at, trigger_kind AS "trigger_kind: _", CASE - WHEN pg_column_size(payload) < 40000 THEN payload + WHEN pg_column_size(main_args) < 40000 THEN main_args ELSE '"WINDMILL_TOO_BIG"'::jsonb - END AS "payload!: _", - trigger_extra AS "trigger_extra: _" + END AS "main_args!: _", + CASE + WHEN pg_column_size(preprocessor_args) < 40000 THEN preprocessor_args + ELSE '"WINDMILL_TOO_BIG"'::jsonb + END AS "preprocessor_args: _" FROM capture WHERE @@ -570,8 +571,8 @@ async fn get_capture( id, created_at, trigger_kind AS "trigger_kind: _", - payload AS "payload!: _", - trigger_extra AS "trigger_extra: _" + main_args AS "main_args!: _", + preprocessor_args AS "preprocessor_args: _" FROM capture WHERE @@ -820,15 +821,15 @@ pub async fn insert_capture_payload( path: &str, is_flow: bool, trigger_kind: &TriggerKind, - payload: PushArgsOwned, - trigger_extra: Option>, + main_args: PushArgsOwned, + preprocessor_args: PushArgsOwned, owner: &str, ) -> Result<()> { sqlx::query!( r#" INSERT INTO capture ( - workspace_id, path, is_flow, trigger_kind, payload, trigger_extra, created_by + workspace_id, path, is_flow, trigger_kind, main_args, preprocessor_args, created_by ) VALUES ( $1, $2, $3, $4, $5, $6, $7 @@ -838,11 +839,9 @@ pub async fn insert_capture_payload( path, is_flow, trigger_kind as &TriggerKind, - SqlxJson(to_raw_value(&PushArgs { - args: &payload.args, - extra: payload.extra - })) as SqlxJson>, - trigger_extra.map(SqlxJson) as Option>>, + SqlxJson(PushArgs { args: &main_args.args, extra: main_args.extra }) as SqlxJson, + SqlxJson(PushArgs { args: &preprocessor_args.args, extra: preprocessor_args.extra }) + as SqlxJson, owner, ) .execute(db) @@ -856,7 +855,7 @@ pub async fn insert_capture_payload( async fn webhook_payload( Extension(db): Extension, Path((w_id, runnable_kind, path)): Path<(String, RunnableKind, StripPath)>, - args: WebhookArgs, + args: RawWebhookArgs, ) -> Result { let (owner, email) = get_active_capture_owner_and_email( &db, @@ -868,7 +867,15 @@ async fn webhook_payload( .await?; let authed = fetch_api_authed(owner.clone(), email, &w_id, &db, None).await?; - let args = args.to_push_args_owned(&authed, &db, &w_id).await?; + + let args = args.process_args(&authed, &db, &w_id, None).await?; + + let preprocessor_args = args.clone().to_args_from_format(RunnableFormat { + has_preprocessor: true, + version: RunnableFormatVersion::V2, + })?; + + let main_args = args.to_main_args()?; insert_capture_payload( &db, @@ -876,12 +883,8 @@ async fn webhook_payload( &path.to_path(), matches!(runnable_kind, RunnableKind::Flow), &TriggerKind::Webhook, - args, - Some(to_raw_value(&serde_json::json!({ - "wm_trigger": { - "kind": "webhook", - } - }))), + main_args, + preprocessor_args, &owner, ) .await?; @@ -897,6 +900,8 @@ async fn gcp_payload( headers: HeaderMap, request: Request, ) -> Result { + use crate::{gcp_triggers_ee::GcpTrigger, trigger_helpers::TriggerJobArgs}; + let is_flow = matches!(runnable_kind, RunnableKind::Flow); let (gcp_trigger_config, owner, email): (GcpTriggerConfig, _, _) = get_capture_trigger_config_and_owner(&db, &w_id, &path, is_flow, &TriggerKind::Gcp).await?; @@ -918,9 +923,9 @@ async fn gcp_payload( ) .await?; - let (args, extra) = process_google_push_request(headers, request).await?; + let (payload, gcp) = process_google_push_request(headers, request).await?; - let payload = PushArgsOwned { args, extra: None }; + let (main_args, preprocessor_args) = GcpTrigger::build_capture_payloads(payload, gcp); let _ = insert_capture_payload( &db, @@ -928,8 +933,8 @@ async fn gcp_payload( &path, is_flow, &TriggerKind::Gcp, - payload, - Some(to_raw_value(&extra)), + main_args, + preprocessor_args, &owner, ) .await?; @@ -941,10 +946,7 @@ async fn gcp_payload( async fn http_payload( Extension(db): Extension, Path((w_id, runnable_kind, path, route_path)): Path<(String, RunnableKind, String, StripPath)>, - Query(query): Query>, - method: http::Method, - headers: HeaderMap, - request: Request, + args: RawHttpTriggerArgs, ) -> std::result::Result { let path = path.replace(".", "/"); let is_flow = matches!(runnable_kind, RunnableKind::Flow); @@ -954,20 +956,17 @@ async fn http_payload( .await .map_err(|e| e.into_response())?; - let args = try_from_request_body( - request, - &(), - http_trigger_config.raw_string, - http_trigger_config.wrap_body, - ) - .await - .map_err(|e| e.into_response())?; - let authed = fetch_api_authed(owner.clone(), email, &w_id, &db, None) .await .map_err(|e| e.into_response())?; - let mut args = args - .to_push_args_owned(&authed, &db, &w_id) + + let args = args + .process_args( + &authed, + &db, + &w_id, + http_trigger_config.raw_string.unwrap_or(false), + ) .await .map_err(|e| e.into_response())?; @@ -985,31 +984,23 @@ async fn http_payload( .map(|(k, v)| (k.to_string(), v.to_string())) .collect(); - let extra = args.extra.get_or_insert_with(HashMap::new); + let preprocessor_args = args + .clone() + .to_v2_preprocessor_args(&http_trigger_config.route_path, &route_path, ¶ms) + .map_err(|e| e.into_response())?; - extra.insert( - "wm_trigger".to_string(), - build_http_trigger_extra( - &http_trigger_config.route_path, - route_path, - &method, - ¶ms, - &query, - &headers, - ) - .await, - ); + let main_args = args + .to_main_args(http_trigger_config.wrap_body.unwrap_or(false)) + .map_err(|e| e.into_response())?; - let extra = Some(to_raw_value(&extra)); - args.extra = None; insert_capture_payload( &db, &w_id, &path, is_flow, &TriggerKind::Http, - args, - extra, + main_args, + preprocessor_args, &owner, ) .await diff --git a/backend/windmill-api/src/gcp_triggers_ee.rs b/backend/windmill-api/src/gcp_triggers_ee.rs index 4c8c72a4e4..2e38d96423 100644 --- a/backend/windmill-api/src/gcp_triggers_ee.rs +++ b/backend/windmill-api/src/gcp_triggers_ee.rs @@ -1,4 +1,5 @@ use crate::db::{ApiAuthed, DB}; +use crate::trigger_helpers::TriggerJobArgs; use axum::{extract::Request, Router}; use http::HeaderMap; use serde::{Deserialize, Serialize}; @@ -7,10 +8,12 @@ use sqlx::prelude::FromRow; use sqlx::types::Json as SqlxJson; use std::collections::HashMap; use windmill_common::db::UserDB; +use windmill_common::worker::to_raw_value; use windmill_common::{ error::{Error as WindmillError, Result as WindmillResult}, utils::empty_string_as_none, }; +use windmill_queue::TriggerKind; #[derive(sqlx::Type, Debug, Deserialize, Serialize)] #[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] @@ -90,14 +93,8 @@ pub async fn manage_google_subscription( pub async fn process_google_push_request( _headers: HeaderMap, _request: Request, -) -> Result< - ( - HashMap>, - Option>>, - ), - WindmillError, -> { - Ok((HashMap::new(), None)) +) -> Result<(String, HashMap>), WindmillError> { + Ok((String::new(), HashMap::new())) } pub async fn validate_jwt_token( @@ -137,3 +134,13 @@ pub struct GcpTrigger { pub last_server_ping: Option>, pub enabled: bool, } + +impl TriggerJobArgs for GcpTrigger { + fn v1_payload_fn(payload: String) -> HashMap> { + HashMap::from([("payload".to_string(), to_raw_value(&payload))]) + } + + fn trigger_kind() -> TriggerKind { + TriggerKind::Gcp + } +} diff --git a/backend/windmill-api/src/http_trigger_args.rs b/backend/windmill-api/src/http_trigger_args.rs new file mode 100644 index 0000000000..5675f1a1b0 --- /dev/null +++ b/backend/windmill-api/src/http_trigger_args.rs @@ -0,0 +1,204 @@ +use std::collections::HashMap; + +use axum::{ + extract::{FromRequest, Request}, + response::Response, +}; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use windmill_common::{error::Error, worker::to_raw_value, DB}; +use windmill_queue::PushArgsOwned; + +use crate::{ + args::{try_from_request_body, Body, RawWebhookArgs, WebhookArgs, WebhookArgsMetadata}, + db::ApiAuthed, + trigger_helpers::{RunnableFormat, RunnableFormatVersion}, +}; + +pub struct RawHttpTriggerArgs(pub RawWebhookArgs); + +#[derive(Serialize, Deserialize, sqlx::Type, Debug)] +#[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum HttpMethod { + Get, + Post, + Put, + Delete, + Patch, +} + +impl TryFrom<&http::Method> for HttpMethod { + type Error = Error; + fn try_from(method: &http::Method) -> Result { + match method { + &http::Method::GET => Ok(HttpMethod::Get), + &http::Method::POST => Ok(HttpMethod::Post), + &http::Method::PUT => Ok(HttpMethod::Put), + &http::Method::DELETE => Ok(HttpMethod::Delete), + &http::Method::PATCH => Ok(HttpMethod::Patch), + _ => Err(Error::BadRequest("Invalid HTTP method".to_string())), + } + } +} + +#[axum::async_trait] +impl FromRequest for RawHttpTriggerArgs +where + S: Send + Sync, +{ + type Rejection = Response; + + async fn from_request(request: Request, _state: &S) -> Result { + let args = try_from_request_body(request, _state, true).await?; + + Ok(Self(args)) + } +} + +#[derive(Clone)] +pub struct HttpTriggerArgs(pub WebhookArgs); + +impl RawHttpTriggerArgs { + pub async fn process_args( + self, + authed: &ApiAuthed, + db: &DB, + w_id: &str, + use_raw: bool, + ) -> Result { + if self.0.metadata.query_use_raw || self.0.metadata.query_wrap_body { + return Err(Error::BadRequest( + "Specifying use raw or wrap body with query args is not supported anymore on http routes, please set it in the trigger config".to_string(), + ) + .into()); + } + + let args = self.0.process_args(authed, db, w_id, Some(use_raw)).await?; + + Ok(HttpTriggerArgs(args)) + } +} + +#[derive(Serialize)] +struct HttpTriggerPreprocessorEvent<'a> { + kind: String, + route: &'a str, + path: &'a str, + body: Box, + raw_string: Option, + params: &'a HashMap, + headers: HashMap>, + query: HashMap>, + method: HttpMethod, +} + +#[derive(Serialize)] +struct HttpTriggerWmTrigger<'a> { + route: &'a str, + path: &'a str, + params: &'a HashMap, + query: &'a HashMap>, + headers: &'a HashMap>, + method: HttpMethod, +} + +impl HttpTriggerArgs { + pub fn to_main_args(self, wrap_body: bool) -> Result { + let mut extra = HashMap::new(); + + let WebhookArgsMetadata { raw_string, .. } = self.0.metadata; + + if let Some(raw_string) = raw_string { + extra.insert("raw_string".to_string(), to_raw_value(&raw_string)); + } + + let extra = if extra.is_empty() { None } else { Some(extra) }; + + match self.0.body { + Body::HashMap(mut body) => { + if wrap_body { + body = HashMap::from([("body".to_string(), to_raw_value(&body))]); + } + Ok(PushArgsOwned { args: body, extra }) + } + Body::NoHashMap(args) => { + let mut hm = HashMap::new(); + hm.insert("body".to_string(), args); + Ok(PushArgsOwned { args: hm, extra }) + } + } + } + + pub fn to_args_from_format( + self, + route_path: &str, + called_path: &str, + params: &HashMap, + format: RunnableFormat, + wrap_body: bool, + ) -> Result { + match format { + RunnableFormat { has_preprocessor: true, version: RunnableFormatVersion::V2 } => { + // we don't care about wrap_body in v2 + self.to_v2_preprocessor_args(route_path, called_path, params) + } + RunnableFormat { has_preprocessor: true, version: RunnableFormatVersion::V1 } => { + self.to_v1_preprocessor_args(route_path, called_path, params, wrap_body) + } + RunnableFormat { has_preprocessor: false, .. } => self.to_main_args(wrap_body), + } + } + + fn to_v1_preprocessor_args( + self, + route_path: &str, + called_path: &str, + params: &HashMap, + wrap_body: bool, + ) -> Result { + let mut extra = HashMap::new(); + + extra.insert( + "wm_trigger".to_string(), + to_raw_value(&HttpTriggerWmTrigger { + route: route_path, + path: called_path, + method: (&self.0.metadata.method).try_into()?, + params, + query: &self.0.metadata.query, + headers: &self.0.metadata.headers, + }), + ); + + let mut args = self.to_main_args(wrap_body)?; + + args.extra.get_or_insert_default().extend(extra); + + Ok(args) + } + + pub fn to_v2_preprocessor_args( + self, + route_path: &str, + called_path: &str, + params: &HashMap, + ) -> Result { + let mut args = HashMap::new(); + args.insert( + "event".to_string(), + to_raw_value(&HttpTriggerPreprocessorEvent { + kind: "http".to_string(), + body: to_raw_value(&self.0.body), + raw_string: self.0.metadata.raw_string, + headers: self.0.metadata.headers, + query: self.0.metadata.query, + method: (&self.0.metadata.method).try_into()?, + route: route_path, + path: called_path, + params, + }), + ); + Ok(PushArgsOwned { args, extra: None }) + } +} diff --git a/backend/windmill-api/src/http_triggers.rs b/backend/windmill-api/src/http_triggers.rs index fa26406a13..ec295404b4 100644 --- a/backend/windmill-api/src/http_triggers.rs +++ b/backend/windmill-api/src/http_triggers.rs @@ -1,10 +1,11 @@ -use crate::http_trigger_auth::{self}; +#[cfg(feature = "http_trigger")] +use crate::http_trigger_args::{HttpMethod, RawHttpTriggerArgs}; #[cfg(feature = "parquet")] use crate::job_helpers_ee::get_workspace_s3_resource; use crate::resources::try_get_resource_from_db_as; +use crate::trigger_helpers::{get_runnable_format, RunnableId}; use crate::utils::non_empty_str; use crate::{ - args::try_from_request_body, auth::{AuthCache, OptTokened}, db::{ApiAuthed, DB}, jobs::{ @@ -15,7 +16,7 @@ use crate::{ }; use axum::response::Response; use axum::{ - extract::{Path, Query, Request}, + extract::{Path, Query}, response::IntoResponse, routing::{delete, get, post}, Extension, Json, Router, @@ -38,8 +39,9 @@ use windmill_common::{ error::{self, JsonResult}, s3_helpers::S3Object, utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath}, - worker::{to_raw_value, CLOUD_HOSTED}, + worker::CLOUD_HOSTED, }; +use windmill_queue::TriggerKind; lazy_static::lazy_static! { static ref ROUTE_PATH_KEY_RE: regex::Regex = regex::Regex::new(r"/?:[-\w]+").unwrap(); @@ -81,31 +83,6 @@ pub fn workspaced_service() -> Router { .route("/route_exists", post(exists_route)) } -#[derive(Serialize, Deserialize, sqlx::Type, Debug)] -#[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")] -#[serde(rename_all = "lowercase")] -pub enum HttpMethod { - Get, - Post, - Put, - Delete, - Patch, -} - -impl TryFrom<&http::Method> for HttpMethod { - type Error = error::Error; - fn try_from(method: &http::Method) -> Result { - match method { - &http::Method::GET => Ok(HttpMethod::Get), - &http::Method::POST => Ok(HttpMethod::Post), - &http::Method::PUT => Ok(HttpMethod::Put), - &http::Method::DELETE => Ok(HttpMethod::Delete), - &http::Method::PATCH => Ok(HttpMethod::Patch), - _ => Err(error::Error::BadRequest("Invalid HTTP method".to_string())), - } - } -} - #[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone, Copy)] #[sqlx(type_name = "AUTHENTICATION_METHOD", rename_all = "snake_case")] #[serde(rename_all(serialize = "snake_case", deserialize = "snake_case"))] @@ -902,42 +879,14 @@ async fn get_http_route_trigger( Ok((trigger, route_path.0, params, authed)) } -pub async fn build_http_trigger_extra( - route_path: &str, - called_path: &str, - method: &http::Method, - params: &HashMap, - query: &HashMap, - headers: &HeaderMap, -) -> Box { - let headers = headers - .iter() - .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_string())) - .collect::>(); - - to_raw_value(&serde_json::json!({ - "kind": "http", - "http": { - "route": route_path, - "path": called_path, - "method": method.to_string().to_lowercase(), - "params": params, - "query": query, - "headers": headers - }, - })) -} - async fn route_job( Extension(db): Extension, Extension(user_db): Extension, Extension(auth_cache): Extension>, OptTokened { token }: OptTokened, Path(route_path): Path, - Query(query): Query>, - method: http::Method, headers: HeaderMap, - request: Request, + args: RawHttpTriggerArgs, ) -> Result { let route_path = route_path.to_path().trim_end_matches("/"); let (trigger, called_path, params, authed) = get_http_route_trigger( @@ -946,25 +895,13 @@ async fn route_job( token.as_ref(), &db, user_db.clone(), - &method, + &args.0.metadata.method, ) .await .map_err(|e| e.into_response())?; - let args = try_from_request_body( - request, - &(), - Some(match trigger.authentication_method { - AuthenticationMethod::CustomScript | AuthenticationMethod::Signature => true, - _ => trigger.raw_string, - }), - Some(trigger.wrap_body), - ) - .await - .map_err(|e| e.into_response())?; - - let mut args = args - .to_push_args_owned(&authed, &db, &trigger.workspace_id) + let args = args + .process_args(&authed, &db, &trigger.workspace_id, trigger.raw_string) .await .map_err(|e| e.into_response())?; @@ -984,7 +921,7 @@ async fn route_job( }; let authentication_method = - try_get_resource_from_db_as::( + try_get_resource_from_db_as::( authed.clone(), Some(user_db.clone()), &db, @@ -995,14 +932,11 @@ async fn route_job( .map_err(|e| e.into_response())?; let raw_payload = args - .extra + .0 + .metadata + .raw_string .as_ref() - .and_then(|extra| { - extra - .get("raw_string") - .and_then(|value| Some(value.to_string())) - .and_then(|raw_payload| Some(serde_json::from_str::(&raw_payload))) - }) + .map(|raw_payload| serde_json::from_str::(raw_payload)) .transpose() .map_err(|e| { windmill_common::error::Error::SerdeJson { location: e.to_string(), error: e } @@ -1130,20 +1064,28 @@ async fn route_job( } } - let extra = args.extra.get_or_insert_with(HashMap::new); + let runnable_format = get_runnable_format( + if trigger.is_flow { + RunnableId::from_flow_path(&trigger.script_path) + } else { + RunnableId::from_script_path(&trigger.script_path) + }, + &trigger.workspace_id, + &db, + &TriggerKind::Http, + ) + .await + .map_err(|e| e.into_response())?; - extra.insert( - "wm_trigger".to_string(), - build_http_trigger_extra( + let args = args + .to_args_from_format( &trigger.route_path, &called_path, - &method, ¶ms, - &query, - &headers, + runnable_format, + trigger.wrap_body, ) - .await, - ); + .map_err(|e| e.into_response())?; let run_query = RunJobQuery::default(); @@ -1157,7 +1099,6 @@ async fn route_job( StripPath(trigger.script_path.to_owned()), run_query, args, - None, ) .await .into_response() @@ -1170,7 +1111,6 @@ async fn route_job( user_db, args, trigger.workspace_id.clone(), - None, ) .await .into_response() @@ -1185,7 +1125,6 @@ async fn route_job( StripPath(trigger.script_path.to_owned()), run_query, args, - None, ) .await .into_response() @@ -1198,7 +1137,6 @@ async fn route_job( user_db, trigger.workspace_id.clone(), args, - None, ) .await .into_response() diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 7913796441..3874df575c 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -7,6 +7,7 @@ */ use axum::body::Body; +use axum::extract::Request; use axum::http::HeaderValue; #[cfg(feature = "deno_core")] use deno_core::{op2, serde_v8, v8, JsRuntime, OpState}; @@ -37,10 +38,11 @@ use crate::add_webhook_allowed_origin; use crate::concurrency_groups::join_concurrency_key; use crate::db::ApiAuthed; +use crate::trigger_helpers::RunnableId; use crate::users::get_scope_tags; use crate::utils::content_plain; use crate::{ - args::{DecodeQueries, WebhookArgs}, + args::{DecodeQueries, RawWebhookArgs}, db::DB, users::{check_scopes, require_owner_of_path, OptAuthed}, utils::require_super_admin, @@ -55,7 +57,7 @@ use axum::{ use base64::Engine; use chrono::Utc; use hmac::Mac; -use hyper::{Request, StatusCode}; +use hyper::StatusCode; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use sql_builder::prelude::*; use sqlx::types::JsonRawValue; @@ -86,7 +88,10 @@ use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS; #[cfg(feature = "prometheus")] use windmill_common::{METRICS_DEBUG_ENABLED, METRICS_ENABLED}; -use windmill_common::{get_latest_deployed_hash_for_path, BASE_URL}; +use windmill_common::{ + get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path, + get_script_info_for_hash, FlowVersionInfo, ScriptHashInfo, BASE_URL, +}; use windmill_queue::{ cancel_job, get_result_and_success_by_id_from_flow, job_is_complete, push, PushArgs, PushArgsOwned, PushIsolationLevel, @@ -545,58 +550,6 @@ async fn force_cancel( } } -pub async fn get_path_tag_limits_cache_for_hash( - mut tx: Transaction<'_, Postgres>, - w_id: &str, - hash: i64, -) -> error::Result<( - String, - Option, - Option, - Option, - Option, - Option, - ScriptLang, - Option, - Option, - Option, - Option, - Option, - Option, - String, -)> { - let script = sqlx::query!( - "select path, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor, on_behalf_of_email, created_by from script where hash = $1 AND workspace_id = $2", - hash, - w_id - ) - .fetch_optional(&mut *tx) - .await - .map_err(|e| { - Error::internal_err(format!( - "querying getting path for hash {hash} in {w_id}: {e:#}" - )) - })?.ok_or_else(|| Error::NotFound(format!( - "deployed script not found at hash {hash} in workspace {w_id}" - )))?; - Ok(( - script.path, - script.tag, - script.concurrency_key, - script.concurrent_limit, - script.concurrency_time_window_s, - script.cache_ttl, - script.language, - script.dedicated_worker, - script.priority, - script.delete_after_use, - script.timeout, - script.has_preprocessor, - script.on_behalf_of_email, - script.created_by, - )) -} - async fn get_flow_job_debug_info( OptAuthed(opt_authed): OptAuthed, Extension(db): Extension, @@ -3466,7 +3419,6 @@ async fn batch_rerun_handle_job( StripPath(job.script_path.clone()), RunJobQuery { ..Default::default() }, PushArgsOwned { extra: None, args }, - None, ) .await; if let Ok((_, uuid)) = result { @@ -3483,7 +3435,6 @@ async fn batch_rerun_handle_job( StripPath(job.script_path.clone()), RunJobQuery { ..Default::default() }, PushArgsOwned { extra: None, args }, - None, ) .await } else { @@ -3495,7 +3446,6 @@ async fn batch_rerun_handle_job( job.script_hash, RunJobQuery { ..Default::default() }, PushArgsOwned { extra: None, args }, - None, ) .await }; @@ -3516,11 +3466,19 @@ pub async fn run_flow_by_path( Extension(user_db): Extension, Path((w_id, flow_path)): Path<(String, StripPath)>, Query(run_query): Query, - args: WebhookArgs, + args: RawWebhookArgs, ) -> error::Result<(StatusCode, String)> { - let args = args.to_push_args_owned(&authed, &db, &w_id).await?; + let args = args + .to_args_from_runnable( + &authed, + &db, + &w_id, + RunnableId::from_flow_path(&flow_path.0), + run_query.skip_preprocessor, + ) + .await?; - run_flow_by_path_inner(authed, db, user_db, w_id, flow_path, run_query, args, None).await + run_flow_by_path_inner(authed, db, user_db, w_id, flow_path, run_query, args).await } pub async fn run_flow_by_path_inner( @@ -3531,7 +3489,6 @@ pub async fn run_flow_by_path_inner( flow_path: StripPath, run_query: RunJobQuery, args: PushArgsOwned, - label_prefix: Option, ) -> error::Result<(StatusCode, String)> { #[cfg(feature = "enterprise")] check_license_key_valid().await?; @@ -3539,23 +3496,17 @@ pub async fn run_flow_by_path_inner( check_scopes(&authed, || format!("run:flow/{flow_path}"))?; let mut tx = user_db.clone().begin(&authed).await?; - let (tag, dedicated_worker, has_preprocessor, on_behalf_of_email, edited_by) = sqlx::query!( - "SELECT tag, dedicated_worker, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor, on_behalf_of_email, edited_by - FROM flow - LEFT JOIN flow_version - ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] - WHERE flow.path = $1 and flow.workspace_id = $2", - flow_path, - w_id - ) - .fetch_optional(&mut *tx) - .await? - .map(|x| (x.tag, x.dedicated_worker, x.has_preprocessor, x.on_behalf_of_email, x.edited_by)) - .ok_or_else(|| { - Error::NotFound(format!( - "flow not found at path {flow_path} in workspace {w_id}" - )) - })?; + + let FlowVersionInfo { + version, + tag, + dedicated_worker, + has_preprocessor, + on_behalf_of_email, + edited_by, + .. + } = get_latest_flow_version_info_for_path(&mut *tx, &w_id, &flow_path, true).await?; + drop(tx); let tag = run_query.tag.clone().or(tag); @@ -3587,13 +3538,12 @@ pub async fn run_flow_by_path_inner( JobPayload::Flow { path: flow_path.to_string(), dedicated_worker, + version, apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) && has_preprocessor.unwrap_or(false), }, PushArgs { args: &args.args, extra: args.extra }, - &label_prefix - .map(|x| x + authed.display_username()) - .unwrap_or_else(|| authed.display_username().to_string()), + authed.display_username(), email, permissioned_as, scheduled_for, @@ -3715,20 +3665,19 @@ pub async fn run_script_by_path( Extension(user_db): Extension, Path((w_id, script_path)): Path<(String, StripPath)>, Query(run_query): Query, - args: WebhookArgs, + args: RawWebhookArgs, ) -> error::Result<(StatusCode, String)> { - let args = args.to_push_args_owned(&authed, &db, &w_id).await?; - run_script_by_path_inner( - authed, - db, - user_db, - w_id, - script_path, - run_query, - args, - None, - ) - .await + let args = args + .to_args_from_runnable( + &authed, + &db, + &w_id, + RunnableId::from_script_path(&script_path.0), + run_query.skip_preprocessor, + ) + .await?; + + run_script_by_path_inner(authed, db, user_db, w_id, script_path, run_query, args).await } pub async fn run_script_by_path_inner( @@ -3739,7 +3688,6 @@ pub async fn run_script_by_path_inner( script_path: StripPath, run_query: RunJobQuery, args: PushArgsOwned, - label_prefix: Option, ) -> error::Result<(StatusCode, String)> { #[cfg(feature = "enterprise")] check_license_key_valid().await?; @@ -3780,9 +3728,7 @@ pub async fn run_script_by_path_inner( &w_id, job_payload, PushArgs { args: &args.args, extra: args.extra }, - &label_prefix - .map(|x| x + authed.display_username()) - .unwrap_or_else(|| authed.display_username().to_string()), + authed.display_username(), email, permissioned_as, scheduled_for, @@ -4504,10 +4450,8 @@ pub async fn run_wait_result_flow_by_path_get( let args = PushArgsOwned { extra: Some(payload_args), args: HashMap::new() }; - run_wait_result_flow_by_path_internal( - db, run_query, flow_path, authed, user_db, args, w_id, None, - ) - .await + run_wait_result_flow_by_path_internal(db, run_query, flow_path, authed, user_db, args, w_id) + .await } pub async fn run_wait_result_script_by_path( @@ -4516,24 +4460,23 @@ pub async fn run_wait_result_script_by_path( Extension(db): Extension, Path((w_id, script_path)): Path<(String, StripPath)>, Query(run_query): Query, - args: WebhookArgs, + args: RawWebhookArgs, ) -> error::Result { #[cfg(feature = "enterprise")] check_license_key_valid().await?; - let args = args.to_push_args_owned(&authed, &db, &w_id).await?; + let args = args + .to_args_from_runnable( + &authed, + &db, + &w_id, + RunnableId::from_script_path(&script_path.0), + run_query.skip_preprocessor, + ) + .await?; - run_wait_result_script_by_path_internal( - db, - run_query, - script_path, - authed, - user_db, - w_id, - args, - None, - ) - .await + run_wait_result_script_by_path_internal(db, run_query, script_path, authed, user_db, w_id, args) + .await } pub async fn run_wait_result_script_by_path_internal( @@ -4544,7 +4487,6 @@ pub async fn run_wait_result_script_by_path_internal( user_db: UserDB, w_id: String, args: PushArgsOwned, - label_prefix: Option, ) -> error::Result { check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?; let script_path = script_path.to_path(); @@ -4580,9 +4522,7 @@ pub async fn run_wait_result_script_by_path_internal( &w_id, job_payload, PushArgs { args: &args.args, extra: args.extra }, - &label_prefix - .map(|x| x + authed.display_username()) - .unwrap_or_else(|| authed.display_username().to_string()), + authed.display_username(), email, permissioned_as, None, @@ -4616,19 +4556,29 @@ pub async fn run_wait_result_script_by_hash( Extension(db): Extension, Path((w_id, script_hash)): Path<(String, ScriptHash)>, Query(run_query): Query, - args: WebhookArgs, + args: RawWebhookArgs, ) -> error::Result { #[cfg(feature = "enterprise")] check_license_key_valid().await?; - let args = args.to_push_args_owned(&authed, &db, &w_id).await?; + let args = args + .to_args_from_runnable( + &authed, + &db, + &w_id, + RunnableId::from_script_hash(script_hash), + run_query.skip_preprocessor, + ) + .await?; + check_queue_too_long(&db, run_query.queue_limit).await?; let hash = script_hash.0; - let ( + let mut tx = user_db.clone().begin(&authed).await?; + let ScriptHashInfo { path, tag, - custom_concurrency_key, + concurrency_key, concurrent_limit, concurrency_time_window_s, mut cache_ttl, @@ -4640,8 +4590,8 @@ pub async fn run_wait_result_script_by_hash( has_preprocessor, on_behalf_of_email, created_by, - ) = get_path_tag_limits_cache_for_hash(user_db.clone().begin(&authed).await?, &w_id, hash) - .await?; + .. + } = get_script_info_for_hash(&mut *tx, &w_id, hash).await?; if let Some(run_query_cache_ttl) = run_query.cache_ttl { cache_ttl = Some(run_query_cache_ttl); } @@ -4674,7 +4624,7 @@ pub async fn run_wait_result_script_by_hash( JobPayload::ScriptHash { hash: ScriptHash(hash), path: path, - custom_concurrency_key, + custom_concurrency_key: concurrency_key, concurrent_limit: concurrent_limit, concurrency_time_window_s: concurrency_time_window_s, cache_ttl, @@ -4719,17 +4669,23 @@ pub async fn run_wait_result_flow_by_path( Extension(db): Extension, Path((w_id, flow_path)): Path<(String, StripPath)>, Query(run_query): Query, - args: WebhookArgs, + args: RawWebhookArgs, ) -> error::Result { #[cfg(feature = "enterprise")] check_license_key_valid().await?; - let args = args.to_push_args_owned(&authed, &db, &w_id).await?; + let args = args + .to_args_from_runnable( + &authed, + &db, + &w_id, + RunnableId::from_flow_path(&flow_path.0), + run_query.skip_preprocessor, + ) + .await?; - run_wait_result_flow_by_path_internal( - db, run_query, flow_path, authed, user_db, args, w_id, None, - ) - .await + run_wait_result_flow_by_path_internal(db, run_query, flow_path, authed, user_db, args, w_id) + .await } pub async fn run_wait_result_flow_by_path_internal( @@ -4740,7 +4696,6 @@ pub async fn run_wait_result_flow_by_path_internal( user_db: UserDB, args: PushArgsOwned, w_id: String, - label_prefix: Option, ) -> error::Result { check_queue_too_long(&db, run_query.queue_limit).await?; @@ -4750,23 +4705,16 @@ pub async fn run_wait_result_flow_by_path_internal( let scheduled_for = run_query.get_scheduled_for(&db).await?; let mut tx = user_db.clone().begin(&authed).await?; - let (tag, dedicated_worker, early_return, has_preprocessor, on_behalf_of_email, edited_by) = sqlx::query!( - "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor, on_behalf_of_email, edited_by - FROM flow - LEFT JOIN flow_version - ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] - WHERE flow.path = $1 and flow.workspace_id = $2", - flow_path, - w_id - ) - .fetch_optional(&mut *tx) - .await? - .map(|x| (x.tag, x.dedicated_worker, x.early_return, x.has_preprocessor, x.on_behalf_of_email, x.edited_by)) - .ok_or_else(|| { - Error::NotFound(format!( - "flow not found at path {flow_path} in workspace {w_id}" - )) - })?; + + let FlowVersionInfo { + tag, + dedicated_worker, + early_return, + has_preprocessor, + on_behalf_of_email, + edited_by, + version, + } = get_latest_flow_version_info_for_path(&mut *tx, &w_id, &flow_path, true).await?; let tag = run_query.tag.clone().or(tag); check_tag_available_for_workspace(&w_id, &tag, &authed).await?; @@ -4795,13 +4743,12 @@ pub async fn run_wait_result_flow_by_path_internal( JobPayload::Flow { path: flow_path.to_string(), dedicated_worker, + version, apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) && has_preprocessor.unwrap_or(false), }, PushArgs { args: &args.args, extra: args.extra }, - &label_prefix - .map(|x| x + authed.display_username()) - .unwrap_or_else(|| authed.display_username().to_string()), + authed.display_username(), email, permissioned_as, scheduled_for, @@ -5255,29 +5202,23 @@ async fn add_batch_jobs( "script" => { if let Some(path) = batch_info.path { let mut tx = user_db.clone().begin(&authed).await?; - let ( - script_hash, - _tag, - custom_concurrency_key, + let ScriptHashInfo { + hash: script_hash, + concurrency_key, concurrent_limit, concurrency_time_window_s, - _cache_ttl, language, dedicated_worker, - _priority, - _delete_after_use, timeout, - _, - _, // TODO: consider on_behalf_of_email and created_by for batch jobs - _, // ------------------------------------------ - ) = get_latest_deployed_hash_for_path(&mut *tx, &w_id, &path).await?; + .. // TODO: consider on_behalf_of_email and created_by for batch jobs + } = get_latest_deployed_hash_for_path(&mut *tx, &w_id, &path).await?; ( Some(script_hash), Some(path), JobKind::Script, Some(language), dedicated_worker, - custom_concurrency_key, + concurrency_key, concurrent_limit, concurrency_time_window_s, timeout, @@ -5417,7 +5358,7 @@ async fn add_batch_jobs( raw_lock, raw_flow.map(sqlx::types::Json) as Option>, tag, - hash.map(|h| h.0), + hash, path, job_kind.clone() as JobKind, language as ScriptLang, @@ -5562,20 +5503,19 @@ pub async fn run_job_by_hash( Extension(user_db): Extension, Path((w_id, script_hash)): Path<(String, ScriptHash)>, Query(run_query): Query, - args: WebhookArgs, + args: RawWebhookArgs, ) -> error::Result<(StatusCode, String)> { - let args = args.to_push_args_owned(&authed, &db, &w_id).await?; - run_job_by_hash_inner( - authed, - db, - user_db, - w_id, - script_hash, - run_query, - args, - None, - ) - .await + let args = args + .to_args_from_runnable( + &authed, + &db, + &w_id, + RunnableId::from_script_hash(script_hash), + run_query.skip_preprocessor, + ) + .await?; + + run_job_by_hash_inner(authed, db, user_db, w_id, script_hash, run_query, args).await } pub async fn run_job_by_hash_inner( @@ -5586,29 +5526,28 @@ pub async fn run_job_by_hash_inner( script_hash: ScriptHash, run_query: RunJobQuery, args: PushArgsOwned, - label_prefix: Option, ) -> error::Result<(StatusCode, String)> { #[cfg(feature = "enterprise")] check_license_key_valid().await?; let hash = script_hash.0; - let ( + let mut tx = user_db.clone().begin(&authed).await?; + let ScriptHashInfo { path, tag, - custom_concurrency_key, + concurrency_key, concurrent_limit, concurrency_time_window_s, mut cache_ttl, language, dedicated_worker, priority, - _delete_after_use, // not taken into account in async endpoints timeout, has_preprocessor, on_behalf_of_email, created_by, - ) = get_path_tag_limits_cache_for_hash(user_db.clone().begin(&authed).await?, &w_id, hash) - .await?; + .. // delete_after_use not taken into account in async endpoints + } = get_script_info_for_hash(&mut *tx, &w_id, hash).await?; check_scopes(&authed, || format!("run:script/{path}"))?; if let Some(run_query_cache_ttl) = run_query.cache_ttl { cache_ttl = Some(run_query_cache_ttl); @@ -5642,7 +5581,7 @@ pub async fn run_job_by_hash_inner( JobPayload::ScriptHash { hash: ScriptHash(hash), path: path, - custom_concurrency_key, + custom_concurrency_key: concurrency_key, concurrent_limit: concurrent_limit, concurrency_time_window_s: concurrency_time_window_s, cache_ttl, @@ -5653,9 +5592,7 @@ pub async fn run_job_by_hash_inner( && has_preprocessor.unwrap_or(false), }, PushArgs { args: &args.args, extra: args.extra }, - &label_prefix - .map(|x| x + authed.display_username()) - .unwrap_or_else(|| authed.display_username().to_string()), + authed.display_username(), email, permissioned_as, scheduled_for, diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 16aad3e740..d7d9498fab 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -81,6 +81,8 @@ mod folders; mod granular_acls; mod groups; #[cfg(feature = "http_trigger")] +mod http_trigger_args; +#[cfg(feature = "http_trigger")] mod http_trigger_auth; #[cfg(feature = "http_trigger")] mod http_triggers; @@ -122,6 +124,7 @@ mod slack_approvals; mod smtp_server_ee; #[cfg(all(feature = "enterprise", feature = "sqs_trigger"))] mod sqs_triggers_ee; +mod trigger_helpers; mod static_assets; #[cfg(all(feature = "stripe", feature = "enterprise"))] diff --git a/backend/windmill-api/src/mcp.rs b/backend/windmill-api/src/mcp.rs index c8c86e2e1f..431e121ba8 100644 --- a/backend/windmill-api/src/mcp.rs +++ b/backend/windmill-api/src/mcp.rs @@ -924,7 +924,6 @@ impl ServerHandler for Runner { user_db.clone(), w_id.clone(), push_args, - None, ) .await } else { @@ -936,7 +935,6 @@ impl ServerHandler for Runner { user_db.clone(), push_args, w_id.clone(), - None, ) .await }; diff --git a/backend/windmill-api/src/mqtt_triggers.rs b/backend/windmill-api/src/mqtt_triggers.rs index 67c4c12d5b..a29648be22 100644 --- a/backend/windmill-api/src/mqtt_triggers.rs +++ b/backend/windmill-api/src/mqtt_triggers.rs @@ -3,6 +3,7 @@ use crate::{ db::{ApiAuthed, DB}, jobs::{run_flow_by_path_inner, run_script_by_path_inner, RunJobQuery}, resources::try_get_resource_from_db_as, + trigger_helpers::TriggerJobArgs, users::fetch_api_authed, }; use windmill_queue::TriggerKind; @@ -16,7 +17,7 @@ use axum::{ routing::{delete, get, post}, Router, }; -use base64::prelude::*; +use base64::{engine, prelude::*}; use bytes::Bytes; use http::StatusCode; use itertools::Itertools; @@ -51,8 +52,6 @@ use rand::seq::SliceRandom; use serde_json::value::RawValue; use sqlx::types::Json as SqlxJson; -use windmill_queue::PushArgsOwned; - pub fn workspaced_service() -> Router { Router::new() .route("/create", post(create_mqtt_trigger)) @@ -82,12 +81,20 @@ enum Error { } async fn run_job( - args: Option>>, - extra: Option>>, + payload: &[u8], + trigger_info: HashMap>, db: &DB, trigger: &MqttTrigger, ) -> anyhow::Result<()> { - let args = PushArgsOwned { args: args.unwrap_or_default(), extra }; + let args = MqttTrigger::build_job_args( + &trigger.script_path, + trigger.is_flow, + &trigger.workspace_id, + db, + payload, + trigger_info, + ) + .await?; let authed = fetch_api_authed( trigger.edited_by.clone(), @@ -111,7 +118,6 @@ async fn run_job( StripPath(trigger.script_path.to_owned()), run_query, args, - None, ) .await?; } else { @@ -123,7 +129,6 @@ async fn run_job( StripPath(trigger.script_path.to_owned()), run_query, args, - None, ) .await?; } @@ -1090,31 +1095,27 @@ impl EventLoop for V3EventLoop { } async fn handle_publish_packet(db: &DB, mqtt: &MqttConfig, payload: Bytes, publish: PublishData) { - let args = HashMap::from([("payload".to_string(), to_raw_value(&payload.as_ref()))]); - let extra = Some(HashMap::from([( - "wm_trigger".to_string(), - to_raw_value(&serde_json::json!({ - "kind": "mqtt", - "mqtt": { - "topic": publish.topic, - "retain": publish.retain, - "pkid": publish.pkid, - "qos": publish.qos, - "v5": publish.v5.map(|properties| { - serde_json::json!({ - "payload_format_indicator": properties.payload_format_indicator, - "topic_alias": properties.topic_alias, - "response_topic": properties.response_topic, - "correlation_data": properties.correlation_data.as_deref(), - "user_properties": properties.user_properties, - "subscription_identifiers": properties.subscription_identifiers, - "content_type": properties.content_type, - }) + let trigger_info = HashMap::from([ + ("topic".to_string(), to_raw_value(&publish.topic)), + ("retain".to_string(), to_raw_value(&publish.retain)), + ("pkid".to_string(), to_raw_value(&publish.pkid)), + ("qos".to_string(), to_raw_value(&publish.qos)), + ( + "v5".to_string(), + to_raw_value(&publish.v5.map(|properties| { + serde_json::json!({ + "payload_format_indicator": properties.payload_format_indicator, + "topic_alias": properties.topic_alias, + "response_topic": properties.response_topic, + "correlation_data": properties.correlation_data.as_deref(), + "user_properties": properties.user_properties, + "subscription_identifiers": properties.subscription_identifiers, + "content_type": properties.content_type, }) - } - })), - )])); - mqtt.handle(&db, Some(args), extra).await; + })), + ), + ]); + mqtt.handle(&db, payload.as_ref(), trigger_info).await; } async fn handle_event(db: &DB, mqtt: &MqttConfig, handler: H, mut event_loop: E) -> () @@ -1225,12 +1226,12 @@ impl MqttConfig { async fn handle( &self, db: &DB, - args: Option>>, - extra: Option>>, + payload: &[u8], + trigger_info: HashMap>, ) -> () { match self { - MqttConfig::Trigger(trigger) => trigger.handle(&db, args, extra).await, - MqttConfig::Capture(capture) => capture.handle(&db, args, extra).await, + MqttConfig::Trigger(trigger) => trigger.handle(&db, payload, trigger_info).await, + MqttConfig::Capture(capture) => capture.handle(&db, payload, trigger_info).await, } } } @@ -1406,10 +1407,10 @@ impl MqttTrigger { async fn handle( &self, db: &DB, - args: Option>>, - extra: Option>>, + payload: &[u8], + trigger_info: HashMap>, ) -> () { - if let Err(err) = run_job(args, extra, db, self).await { + if let Err(err) = run_job(payload, trigger_info, db, self).await { report_critical_error( format!("Failed to trigger job from mqtt {}: {:?}", self.path, err), db.clone(), @@ -1421,6 +1422,21 @@ impl MqttTrigger { } } +impl TriggerJobArgs<&[u8]> for MqttTrigger { + fn v1_payload_fn(payload: &[u8]) -> HashMap> { + HashMap::from([("payload".to_string(), to_raw_value(&payload))]) + } + + fn v2_payload_fn(payload: &[u8]) -> HashMap> { + let base64_payload = engine::general_purpose::STANDARD.encode(payload); + HashMap::from([("payload".to_string(), to_raw_value(&base64_payload))]) + } + + fn trigger_kind() -> TriggerKind { + TriggerKind::Mqtt + } +} + struct PublishData { topic: String, retain: bool, @@ -1675,19 +1691,19 @@ impl CaptureConfigForMqttTrigger { async fn handle( &self, db: &DB, - args: Option>>, - extra: Option>>, + payload: &[u8], + trigger_info: HashMap>, ) -> () { - let args = PushArgsOwned { args: args.unwrap_or_default(), extra: None }; - let extra = extra.as_ref().map(to_raw_value); + let (main_args, preprocessor_args) = + MqttTrigger::build_capture_payloads(payload, trigger_info); if let Err(err) = insert_capture_payload( db, &self.workspace_id, &self.path, self.is_flow, &TriggerKind::Mqtt, - args, - extra, + main_args, + preprocessor_args, &self.owner, ) .await diff --git a/backend/windmill-api/src/postgres_triggers/mod.rs b/backend/windmill-api/src/postgres_triggers/mod.rs index d041f1f508..0be19d6c4c 100644 --- a/backend/windmill-api/src/postgres_triggers/mod.rs +++ b/backend/windmill-api/src/postgres_triggers/mod.rs @@ -2,6 +2,7 @@ use crate::{ db::{ApiAuthed, DB}, jobs::{run_flow_by_path_inner, run_script_by_path_inner, RunJobQuery}, resources::try_get_resource_from_db_as, + trigger_helpers::TriggerJobArgs, users::fetch_api_authed, }; use chrono::Utc; @@ -30,7 +31,6 @@ use handler::{ Relations, }; use windmill_common::{db::UserDB, error::Error, utils::StripPath}; -use windmill_queue::PushArgsOwned; mod bool; mod converter; mod handler; @@ -250,12 +250,19 @@ pub fn workspaced_service() -> Router { } async fn run_job( - args: Option>>, - extra: Option>>, + payload: HashMap>, db: &DB, trigger: &PostgresTrigger, ) -> anyhow::Result<()> { - let args = PushArgsOwned { args: args.unwrap_or_default(), extra }; + let args = PostgresTrigger::build_job_args( + &trigger.script_path, + trigger.is_flow, + &trigger.workspace_id, + db, + payload, + HashMap::new(), + ) + .await?; let authed = fetch_api_authed( trigger.edited_by.clone(), @@ -279,7 +286,6 @@ async fn run_job( StripPath(trigger.script_path.to_owned()), run_query, args, - None, ) .await?; } else { @@ -291,7 +297,6 @@ async fn run_job( StripPath(trigger.script_path.to_owned()), run_query, args, - None, ) .await?; } diff --git a/backend/windmill-api/src/postgres_triggers/trigger.rs b/backend/windmill-api/src/postgres_triggers/trigger.rs index 746dca99eb..f340666977 100644 --- a/backend/windmill-api/src/postgres_triggers/trigger.rs +++ b/backend/windmill-api/src/postgres_triggers/trigger.rs @@ -12,6 +12,7 @@ use crate::{ run_job, }, resources::try_get_resource_from_db_as, + trigger_helpers::TriggerJobArgs, users::fetch_api_authed, }; use windmill_queue::TriggerKind; @@ -31,7 +32,6 @@ use sqlx::types::Json as SqlxJson; use windmill_common::{ db::UserDB, error, utils::report_critical_error, worker::to_raw_value, INSTANCE_NAME, }; -use windmill_queue::PushArgsOwned; use super::{ drop_logical_replication_slot_query, drop_publication_query, get_database_connection, @@ -369,13 +369,8 @@ impl PostgresTrigger { .await } - async fn handle( - &self, - db: &DB, - args: Option>>, - extra: Option>>, - ) -> () { - if let Err(err) = run_job(args, extra, db, self).await { + async fn handle(&self, db: &DB, payload: HashMap>) -> () { + if let Err(err) = run_job(payload, db, self).await { report_critical_error( format!( "Failed to trigger job from postgres {}: {:?}", @@ -390,6 +385,20 @@ impl PostgresTrigger { } } +impl TriggerJobArgs>> for PostgresTrigger { + fn v1_payload_fn(payload: HashMap>) -> HashMap> { + payload + } + + fn v2_payload_fn(payload: HashMap>) -> HashMap> { + payload + } + + fn trigger_kind() -> TriggerKind { + TriggerKind::Postgres + } +} + struct PgInfo<'a> { postgres_resource_path: &'a str, publication_name: &'a str, @@ -467,7 +476,6 @@ impl PostgresConfig { let client = PostgresSimpleClient::new(&database).await?; - let publication = client .execute_query(&format!( "SELECT pubname FROM pg_publication WHERE pubname = {}", @@ -508,15 +516,10 @@ impl PostgresConfig { } } - async fn handle( - &self, - db: &DB, - args: Option>>, - extra: Option>>, - ) -> () { + async fn handle(&self, db: &DB, payload: HashMap>) -> () { match self { - PostgresConfig::Trigger(trigger) => trigger.handle(&db, args, extra).await, - PostgresConfig::Capture(capture) => capture.handle(&db, args, extra).await, + PostgresConfig::Trigger(trigger) => trigger.handle(&db, payload).await, + PostgresConfig::Capture(capture) => capture.handle(&db, payload).await, } } @@ -606,7 +609,7 @@ async fn listen_to_transactions( } }; - + let message = match message { Ok(message) => message, Err(err) => { @@ -678,13 +681,9 @@ async fn listen_to_transactions( ("old_row".to_string(), to_raw_value(&old_row)), ("row".to_string(), to_raw_value(&row)), ]); - let extra = Some(HashMap::from([( - "wm_trigger".to_string(), - to_raw_value(&serde_json::json!({"kind": "postgres", })), - )])); - - - let _ = pg.handle(&db, Some(database_info), extra).await; + + + let _ = pg.handle(&db, database_info).await; } Some((o_id, old_row, row, transaction_type)) => { let relation = match relations.get_relation(o_id) { @@ -694,7 +693,7 @@ async fn listen_to_transactions( continue; } }; - + if let Err(err) = old_row { tracing::error!( transaction_type = ?transaction_type, @@ -707,7 +706,7 @@ async fn listen_to_transactions( relation.name, ); } - + if let Err(err) = row { tracing::error!( transaction_type = ?transaction_type, @@ -720,7 +719,7 @@ async fn listen_to_transactions( relation.name, ); } - + } _ => {} } @@ -919,22 +918,17 @@ impl CaptureConfigForPostgresTrigger { } } - async fn handle( - &self, - db: &DB, - args: Option>>, - extra: Option>>, - ) -> () { - let args = PushArgsOwned { args: args.unwrap_or_default(), extra: None }; - let extra = extra.as_ref().map(to_raw_value); + async fn handle(&self, db: &DB, payload: HashMap>) -> () { + let main_args = PostgresTrigger::build_job_args_v2(false, payload.clone(), HashMap::new()); + let preprocessor_args = PostgresTrigger::build_job_args_v2(true, payload, HashMap::new()); if let Err(err) = insert_capture_payload( db, &self.workspace_id, &self.path, self.is_flow, &TriggerKind::Postgres, - args, - extra, + main_args, + preprocessor_args, &self.owner, ) .await diff --git a/backend/windmill-api/src/trigger_helpers.rs b/backend/windmill-api/src/trigger_helpers.rs new file mode 100644 index 0000000000..69a5a014a4 --- /dev/null +++ b/backend/windmill-api/src/trigger_helpers.rs @@ -0,0 +1,332 @@ +use quick_cache::sync::Cache; +use serde::Deserialize; +use serde_json::value::RawValue; +use std::collections::HashMap; +use windmill_common::{ + error::Result, + get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path, + scripts::{ScriptHash, ScriptLang}, + worker::to_raw_value, + FlowVersionInfo, +}; +use windmill_queue::{PushArgsOwned, TriggerKind}; + +use crate::db::DB; + +type RunnableFormatCacheKey = (String, i64, TriggerKind); + +lazy_static::lazy_static! { + pub static ref RUNNABLE_FORMAT_VERSION_CACHE: Cache = Cache::new(1000); +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash, Copy)] +pub struct RunnableFormat { + pub version: RunnableFormatVersion, + pub has_preprocessor: bool, +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash, Copy)] +pub enum RunnableFormatVersion { + V1, + V2, +} + +struct ScriptInfo { + has_preprocessor: Option, + language: ScriptLang, + content: String, + schema: Option>, +} + +struct FlowInfo { + has_preprocessor: Option, + is_v1_preprocessor: Option, + schema: Option>, +} + +#[derive(Debug, Deserialize)] +struct PropertyDefinition { + r#type: Option, +} + +#[derive(Debug, Deserialize)] +struct PartialSchema { + properties: Option>, +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub enum RunnableId { + FlowPath(String), + ScriptId(ScriptId), +} + +impl RunnableId { + pub fn from_script_hash(hash: ScriptHash) -> Self { + Self::ScriptId(ScriptId::ScriptHash(hash)) + } + + pub fn from_script_path(path: &str) -> Self { + Self::ScriptId(ScriptId::ScriptPath(path.to_string())) + } + + pub fn from_flow_path(path: &str) -> Self { + Self::FlowPath(path.to_string()) + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub enum ScriptId { + ScriptPath(String), + ScriptHash(ScriptHash), +} + +impl ScriptId { + async fn get_script_hash(self, workspace_id: &str, db: &DB) -> Result { + let hash = match self { + ScriptId::ScriptPath(path) => { + let info = get_latest_deployed_hash_for_path(db, workspace_id, &path).await?; + info.hash + } + ScriptId::ScriptHash(hash) => hash.0, + }; + + Ok(hash) + } +} + +async fn get_script_info( + db: &DB, + workspace_id: &str, + hash: i64, +) -> std::result::Result { + sqlx::query_as!(ScriptInfo, "SELECT has_preprocessor, language as \"language: _\", content, schema as \"schema: _\" FROM script WHERE workspace_id = $1 AND hash = $2", workspace_id, hash) + .fetch_one(db) + .await +} + +fn runnable_format_from_schema( + trigger_kind: &TriggerKind, + has_preprocessor: bool, + schema: Option>, +) -> RunnableFormat { + match trigger_kind { + TriggerKind::Mqtt + if schema.as_ref().is_some_and(|schema| { + schema.properties.as_ref().is_some_and(|properties| { + properties.iter().any(|(key, def)| { + key == "payload" && def.r#type.as_ref().is_some_and(|t| t == "array") + }) + }) + }) => + { + RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor } + } + TriggerKind::Kafka | TriggerKind::Nats + if schema.as_ref().is_some_and(|schema| { + schema + .properties + .as_ref() + .is_some_and(|properties| properties.keys().any(|key| key == "msg")) + }) => + { + RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor } + } + _ => RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor }, + } +} +pub async fn get_runnable_format( + runnable_id: RunnableId, + workspace_id: &str, + db: &DB, + trigger_kind: &TriggerKind, +) -> Result { + match runnable_id { + RunnableId::FlowPath(path) => { + let FlowVersionInfo { version, .. } = + get_latest_flow_version_info_for_path(db, workspace_id, &path, true).await?; + + let key = (workspace_id.to_string(), version, trigger_kind.clone()); + + let runnable_format = RUNNABLE_FORMAT_VERSION_CACHE.get(&key); + + if let Some(runnable_format) = runnable_format { + tracing::debug!("Using cached runnable format for flow {path}"); + return Ok(runnable_format); + } + + let flow_info = sqlx::query_as!( + FlowInfo, + "SELECT + value->'preprocessor_module' IS NOT NULL as has_preprocessor, + value->'preprocessor_module'->'value'->'input_transforms'->'wm_trigger' IS NOT NULL as is_v1_preprocessor, + schema as \"schema: _\" + FROM flow + WHERE workspace_id = $1 + AND path = $2", + workspace_id, + path + ) + .fetch_one(db) + .await?; + + let has_preprocessor = flow_info.has_preprocessor.unwrap_or(false); + let is_v1_preprocessor = flow_info.is_v1_preprocessor.unwrap_or(false); + + let runnable_format = if has_preprocessor && is_v1_preprocessor { + RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: true } + } else { + runnable_format_from_schema(trigger_kind, has_preprocessor, flow_info.schema) + }; + + RUNNABLE_FORMAT_VERSION_CACHE.insert(key, runnable_format); + + Ok(runnable_format) + } + RunnableId::ScriptId(script_id) => { + let hash = script_id.get_script_hash(workspace_id, db).await?; + let key = (workspace_id.to_string(), hash, trigger_kind.clone()); + let runnable_format = RUNNABLE_FORMAT_VERSION_CACHE.get(&key); + + if let Some(runnable_format) = runnable_format { + tracing::debug!("Using cached runnable format for script {hash}"); + return Ok(runnable_format); + } + + let script_info = get_script_info(db, workspace_id, hash).await?; + + let has_preprocessor = script_info.has_preprocessor.unwrap_or(false); + + let runnable_format = if has_preprocessor { + let args = match script_info.language { + ScriptLang::Bun + | ScriptLang::Bunnative + | ScriptLang::Deno + | ScriptLang::Nativets => { + let args = windmill_parser_ts::parse_deno_signature( + &script_info.content, + true, + false, + Some("preprocessor".to_string()), + )?; + Some(args.args) + } + ScriptLang::Python3 => { + let args = windmill_parser_py::parse_python_signature( + &script_info.content, + Some("preprocessor".to_string()), + false, + )?; + Some(args.args) + } + _ => None, + }; + + if args.is_some_and(|args| args.iter().any(|arg| arg.name == "wm_trigger")) { + RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: true } + } else { + runnable_format_from_schema(trigger_kind, has_preprocessor, script_info.schema) + } + } else { + runnable_format_from_schema(trigger_kind, has_preprocessor, script_info.schema) + }; + + RUNNABLE_FORMAT_VERSION_CACHE.insert(key, runnable_format); + + Ok(runnable_format) + } + } +} + +#[allow(dead_code)] +pub trait TriggerJobArgs { + fn v1_payload_fn(payload: T) -> HashMap>; + fn v2_payload_fn(payload: T) -> HashMap> { + Self::v1_payload_fn(payload) + } + fn trigger_kind() -> TriggerKind; + + fn build_job_args_v2( + has_preprocessor: bool, + payload: T, + info: HashMap>, + ) -> PushArgsOwned { + let trigger_kind = Self::trigger_kind(); + let mut args = Self::v2_payload_fn(payload); + if has_preprocessor { + args.insert("kind".to_string(), to_raw_value(&trigger_kind.to_key())); + args.extend(info); + let args = HashMap::from([("event".to_string(), to_raw_value(&args))]); + PushArgsOwned { args, extra: None } + } else { + PushArgsOwned { args, extra: None } + } + } + + fn build_job_args_v1( + has_preprocessor: bool, + payload: T, + info: HashMap>, + ) -> PushArgsOwned { + let trigger_kind = Self::trigger_kind(); + let trigger_key = trigger_kind.to_key(); + let args = Self::v1_payload_fn(payload); + let extra = if has_preprocessor { + Some(HashMap::from([( + "wm_trigger".to_string(), + to_raw_value(&serde_json::json!({ + "kind": trigger_key, + trigger_key: info + })), + )])) + } else { + None + }; + + PushArgsOwned { args, extra } + } + + async fn build_job_args( + runnable_path: &str, + is_flow: bool, + w_id: &str, + db: &DB, + payload: T, + info: HashMap>, + ) -> Result { + let runnable_id = if is_flow { + RunnableId::from_flow_path(runnable_path) + } else { + RunnableId::from_script_path(runnable_path) + }; + Self::build_job_args_from_runnable_id(runnable_id, w_id, db, payload, info).await + } + + async fn build_job_args_from_runnable_id( + runnable_id: RunnableId, + w_id: &str, + db: &DB, + payload: T, + info: HashMap>, + ) -> Result { + let runnable_format = + get_runnable_format(runnable_id, w_id, db, &Self::trigger_kind()).await?; + + match runnable_format { + RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor } => { + Ok(Self::build_job_args_v1(has_preprocessor, payload, info)) + } + RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor } => { + Ok(Self::build_job_args_v2(has_preprocessor, payload, info)) + } + } + } + + fn build_capture_payloads( + payload: T, + info: HashMap>, + ) -> (PushArgsOwned, PushArgsOwned) { + let main_args = Self::build_job_args_v2(false, payload.clone(), info.clone()); + let preprocessor_args = Self::build_job_args_v2(true, payload, info); + (main_args, preprocessor_args) + } +} diff --git a/backend/windmill-api/src/websocket_triggers.rs b/backend/windmill-api/src/websocket_triggers.rs index 9be3829d83..c23daae33f 100644 --- a/backend/windmill-api/src/websocket_triggers.rs +++ b/backend/windmill-api/src/websocket_triggers.rs @@ -38,6 +38,7 @@ use crate::{ jobs::{ run_flow_by_path_inner, run_script_by_path_inner, run_wait_result_internal, RunJobQuery, }, + trigger_helpers::TriggerJobArgs, users::fetch_api_authed, }; @@ -608,7 +609,6 @@ async fn wait_runnable_result( StripPath(path.clone()), RunJobQuery::default(), args, - None, ) .await?; @@ -635,7 +635,6 @@ async fn wait_runnable_result( StripPath(path.clone()), RunJobQuery::default(), args, - None, ) .await?; @@ -890,10 +889,11 @@ impl WebsocketTrigger { async fn handle( &self, db: &DB, - args: PushArgsOwned, + msg: &str, + trigger_info: HashMap>, return_message_channels: Option, ) -> () { - if let Err(err) = run_job(db, self, args, return_message_channels).await { + if let Err(err) = run_job(db, self, &msg, trigger_info, return_message_channels).await { report_critical_error( format!( "Failed to trigger job from WebSocket {}: {:?}", @@ -919,6 +919,16 @@ impl WebsocketTrigger { } } +impl TriggerJobArgs<&str> for WebsocketTrigger { + fn v1_payload_fn(payload: &str) -> HashMap> { + HashMap::from([("msg".to_string(), to_raw_value(&payload))]) + } + + fn trigger_kind() -> TriggerKind { + TriggerKind::Websocket + } +} + #[derive(Deserialize)] struct CaptureConfigForWebsocket { trigger_config: SqlxJson, @@ -985,15 +995,18 @@ impl CaptureConfigForWebsocket { Some(()) } - async fn handle(&self, db: &DB, args: PushArgsOwned) -> () { + async fn handle(&self, db: &DB, msg: &str, trigger_info: HashMap>) -> () { + let (main_args, preprocessor_args) = + WebsocketTrigger::build_capture_payloads(&msg, trigger_info); + if let Err(err) = insert_capture_payload( db, &self.workspace_id, &self.path, self.is_flow, &TriggerKind::Websocket, - PushArgsOwned { args: args.args, extra: None }, - args.extra.as_ref().map(to_raw_value), + main_args, + preprocessor_args, &self.owner, ) .await @@ -1259,20 +1272,15 @@ async fn listen_to_websocket( } } if should_handle { - - let args = HashMap::from([("msg".to_string(), to_raw_value(&text))]); - let extra = Some(HashMap::from([( - "wm_trigger".to_string(), - to_raw_value(&serde_json::json!({"kind": "websocket", "websocket": { "url": url }})), - )])); - - let args = PushArgsOwned { args, extra }; + let trigger_info = HashMap::from([ + ("url".to_string(), to_raw_value(&url)), + ]); match &ws { WebsocketEnum::Trigger(ws_trigger) => { - ws_trigger.handle(&db, args, return_message_channels.clone()).await; + ws_trigger.handle(&db, &text, trigger_info, return_message_channels.clone()).await; }, WebsocketEnum::Capture(capture) => { - capture.handle(&db, args).await; + capture.handle(&db, &text, trigger_info).await; }, } } @@ -1311,9 +1319,20 @@ async fn listen_to_websocket( async fn run_job( db: &DB, trigger: &WebsocketTrigger, - args: PushArgsOwned, + msg: &str, + trigger_info: HashMap>, return_message_channels: Option, ) -> anyhow::Result<()> { + let args = WebsocketTrigger::build_job_args( + &trigger.script_path, + trigger.is_flow, + &trigger.workspace_id, + db, + msg, + trigger_info, + ) + .await?; + let authed = fetch_api_authed( trigger.edited_by.clone(), trigger.email.clone(), @@ -1374,7 +1393,6 @@ async fn run_job( runnable_path, run_query, args, - None, ) .await?; } else { @@ -1386,7 +1404,6 @@ async fn run_job( runnable_path, run_query, args, - None, ) .await?; } diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 6cad6579fb..83752b125c 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -5,7 +5,7 @@ use futures_core::Stream; use indexmap::IndexMap; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; -use sqlx::{types::Json, Pool, Postgres, Transaction}; +use sqlx::{types::Json, Pool, Postgres}; use tokio::io::AsyncReadExt; use uuid::Uuid; @@ -17,10 +17,11 @@ use crate::{ error::{self, to_anyhow, Error}, flow_status::{FlowStatus, RestartedFrom}, flows::{FlowNodeId, FlowValue, Retry}, - get_latest_deployed_hash_for_path, + get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path, scripts::{ScriptHash, ScriptLang}, users::username_to_permissioned_as, worker::{to_raw_value, TMP_DIR}, + FlowVersionInfo, ScriptHashInfo, }; #[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Copy, Clone)] @@ -331,6 +332,7 @@ pub enum JobPayload { path: String, dedicated_worker: Option, apply_preprocessor: bool, + version: i64, }, RestartedFlow { completed_job_id: Uuid, @@ -385,9 +387,9 @@ pub struct OnBehalfOf { pub permissioned_as: String, } -pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgres>>( +pub async fn script_path_to_payload<'e, A: sqlx::Acquire<'e, Database = Postgres> + Send>( script_path: &str, - db: E, + db: A, w_id: &str, skip_preprocessor: Option, ) -> error::Result<( @@ -407,10 +409,10 @@ pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgre None, ) } else { - let ( - script_hash, + let ScriptHashInfo { + hash, tag, - custom_concurrency_key, + concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, @@ -418,11 +420,12 @@ pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgre dedicated_worker, priority, delete_after_use, - script_timeout, + timeout, has_preprocessor, on_behalf_of_email, created_by, - ) = get_latest_deployed_hash_for_path(db, w_id, script_path).await?; + .. + } = get_latest_deployed_hash_for_path(db, w_id, script_path).await?; let on_behalf_of = if let Some(email) = on_behalf_of_email { Some(OnBehalfOf { @@ -435,9 +438,9 @@ pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgre ( JobPayload::ScriptHash { - hash: script_hash, + hash: ScriptHash(hash), path: script_path.to_owned(), - custom_concurrency_key, + custom_concurrency_key: concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl: cache_ttl, @@ -449,7 +452,7 @@ pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgre }, tag, delete_after_use, - script_timeout, + timeout, on_behalf_of, ) }; @@ -462,52 +465,6 @@ pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgre )) } -pub async fn script_hash_to_tag_and_limits<'c>( - script_hash: &ScriptHash, - db: &mut Transaction<'c, Postgres>, - w_id: &String, -) -> error::Result<( - Option, - Option, - Option, - Option, - Option, - ScriptLang, - Option, - Option, - Option, - Option, - Option, - String, -)> { - let script = sqlx::query!( - "select tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, on_behalf_of_email, created_by from script where hash = $1 AND workspace_id = $2", - script_hash.0, - w_id - ) - .fetch_one(&mut **db) - .await - .map_err(|e| { - Error::internal_err(format!( - "querying getting tag for hash {script_hash}: {e:#}" - )) - })?; - Ok(( - script.tag, - script.concurrency_key, - script.concurrent_limit, - script.concurrency_time_window_s, - script.cache_ttl, - script.language, - script.dedicated_worker, - script.priority, - script.delete_after_use, - script.timeout, - script.on_behalf_of_email, - script.created_by, - )) -} - pub async fn get_payload_tag_from_prefixed_path( path: &str, db: &DB, @@ -517,18 +474,10 @@ pub async fn get_payload_tag_from_prefixed_path( script_path_to_payload(path.strip_prefix("script/").unwrap(), db, w_id, Some(true)).await? } else if path.starts_with("flow/") { let path = path.strip_prefix("flow/").unwrap().to_string(); - let r = sqlx::query!( - "SELECT tag, dedicated_worker from flow WHERE path = $1 and workspace_id = $2", - &path, - &w_id, - ) - .fetch_optional(db) - .await?; - let (tag, dedicated_worker) = r - .map(|x| (x.tag, x.dedicated_worker)) - .unwrap_or_else(|| (None, None)); + let FlowVersionInfo { dedicated_worker, tag, version, .. } = + get_latest_flow_version_info_for_path(db, w_id, &path, true).await?; ( - JobPayload::Flow { path, dedicated_worker, apply_preprocessor: false }, + JobPayload::Flow { path, dedicated_worker, apply_preprocessor: false, version }, tag, None, None, diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index e00e3c5c78..e1c2f4aa15 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -6,7 +6,9 @@ * LICENSE-AGPL for a copy of the license. */ +use quick_cache::sync::Cache; use std::{ + future::Future, net::SocketAddr, str::FromStr, sync::{ @@ -123,8 +125,15 @@ lazy_static::lazy_static! { pub static ref INSTANCE_NAME: String = rd_string(5); + pub static ref DEPLOYED_SCRIPT_HASH_CACHE: Cache<(String, String), ExpiringLatestVersionId> = Cache::new(1000); + pub static ref FLOW_VERSION_CACHE: Cache<(String, String), ExpiringLatestVersionId> = Cache::new(1000); + pub static ref DEPLOYED_SCRIPT_INFO_CACHE: Cache<(String, i64), ScriptHashInfo> = Cache::new(1000); + pub static ref FLOW_INFO_CACHE: Cache<(String, i64), FlowVersionInfo> = Cache::new(1000); + } +const LATEST_VERSION_ID_CACHE_TTL: std::time::Duration = std::time::Duration::from_secs(60); + pub async fn shutdown_signal( tx: KillpillSender, mut rx: tokio::sync::broadcast::Receiver<()>, @@ -336,54 +345,207 @@ type Tag = String; pub type DB = Pool; -pub async fn get_latest_deployed_hash_for_path<'e, E: sqlx::Executor<'e, Database = Postgres>>( +#[derive(Clone)] +pub struct ExpiringLatestVersionId { + id: i64, + expires_at: std::time::Instant, +} + +#[derive(Clone)] +pub struct ScriptHashInfo { + pub path: String, + pub hash: i64, + pub tag: Option, + pub concurrency_key: Option, + pub concurrent_limit: Option, + pub concurrency_time_window_s: Option, + pub cache_ttl: Option, + pub language: ScriptLang, + pub dedicated_worker: Option, + pub priority: Option, + pub delete_after_use: Option, + pub timeout: Option, + pub has_preprocessor: Option, + pub on_behalf_of_email: Option, + pub created_by: String, +} + +pub fn get_latest_deployed_hash_for_path< + 'a, + 'e, + E: sqlx::Acquire<'e, Database = Postgres> + Send + 'a, +>( + db: E, + w_id: &'a str, + script_path: &'a str, +) -> impl Future> + Send + 'a { + async move { + let mut conn = db.acquire().await?; + let cache_key = (w_id.to_string(), script_path.to_string()); + + let hash = match DEPLOYED_SCRIPT_HASH_CACHE.get(&cache_key) { + Some(cached_hash) if cached_hash.expires_at > std::time::Instant::now() => { + tracing::debug!( + "Using cached script hash {} for {script_path}", + cached_hash.id + ); + cached_hash.id + } + _ => { + tracing::debug!("Fetching script hash for {script_path}"); + let hash = sqlx::query_scalar!( + "select hash from script where path = $1 AND workspace_id = $2 AND deleted = false AND lock IS not NULL AND lock_error_logs IS NULL ORDER BY created_at DESC LIMIT 1", + script_path, + w_id + ) + .fetch_optional(&mut *conn) + .await?; + + let hash = utils::not_found_if_none(hash, "script", script_path)?; + + DEPLOYED_SCRIPT_HASH_CACHE.insert( + cache_key, + ExpiringLatestVersionId { + id: hash, + expires_at: std::time::Instant::now() + LATEST_VERSION_ID_CACHE_TTL, + }, + ); + + hash + } + }; + + get_script_info_for_hash(&mut *conn, w_id, hash).await + } +} + +pub async fn get_script_info_for_hash<'e, E: sqlx::PgExecutor<'e>>( db: E, w_id: &str, - script_path: &str, -) -> error::Result<( - scripts::ScriptHash, - Option, - Option, - Option, - Option, - Option, - ScriptLang, - Option, - Option, - Option, - Option, - Option, - Option, - String, -)> { - let r_o = sqlx::query!( - "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor, on_behalf_of_email, created_by from script where path = $1 AND workspace_id = $2 AND - created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND - deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)", - script_path, - w_id - ) - .fetch_optional(db) - .await?; + hash: i64, +) -> error::Result { + let key = (w_id.to_string(), hash); - let script = utils::not_found_if_none(r_o, "deployed script", script_path)?; + match DEPLOYED_SCRIPT_INFO_CACHE.get(&key) { + Some(info) => { + tracing::debug!("Using cached deployed script info for {hash}"); + Ok(info) + } + _ => { + tracing::debug!("Fetching deployed script info for {hash}"); + let info = sqlx::query_as!( + ScriptHashInfo, + "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor, on_behalf_of_email, created_by, path from script where hash = $1 AND workspace_id = $2", + hash, + w_id + ) + .fetch_optional(db) + .await?; - Ok(( - scripts::ScriptHash(script.hash), - script.tag, - script.concurrency_key, - script.concurrent_limit, - script.concurrency_time_window_s, - script.cache_ttl, - script.language, - script.dedicated_worker, - script.priority, - script.delete_after_use, - script.timeout, - script.has_preprocessor, - script.on_behalf_of_email, - script.created_by, - )) + let info = utils::not_found_if_none(info, "script", &hash.to_string())?; + + DEPLOYED_SCRIPT_INFO_CACHE.insert(key, info.clone()); + + Ok(info) + } + } +} + +#[derive(Clone)] +pub struct FlowVersionInfo { + pub version: i64, + pub tag: Option, + pub early_return: Option, + pub has_preprocessor: Option, + pub on_behalf_of_email: Option, + pub edited_by: String, + pub dedicated_worker: Option, +} + +pub fn get_latest_flow_version_info_for_path< + 'a, + 'e, + A: sqlx::Acquire<'e, Database = Postgres> + Send + 'a, +>( + db: A, + w_id: &'a str, + path: &'a str, + use_cache: bool, +) -> impl Future> + Send + 'a { + // as instructed in the docstring of sqlx::Acquire + async move { + let mut conn = db.acquire().await?; + + let cache_key = (w_id.to_string(), path.to_string()); + let cached_version = if use_cache { + FLOW_VERSION_CACHE.get(&cache_key) + } else { + None + }; + + let version = match cached_version { + Some(cached_version) if cached_version.expires_at > std::time::Instant::now() => { + tracing::debug!("Using cached flow version {} for {path}", cached_version.id); + cached_version.id + } + _ => { + tracing::debug!("Fetching flow version for {path}"); + let version = sqlx::query_scalar!( + "SELECT flow_version.id from flow + INNER JOIN flow_version + ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] + WHERE flow.path = $1 and flow.workspace_id = $2", + path, + w_id + ) + .fetch_optional(&mut *conn) + .await?; + + let version = utils::not_found_if_none(version, "flow", path)?; + + FLOW_VERSION_CACHE.insert( + cache_key, + ExpiringLatestVersionId { + id: version, + expires_at: std::time::Instant::now() + LATEST_VERSION_ID_CACHE_TTL, + }, + ); + + version + } + }; + + let key = (w_id.to_string(), version); + + match FLOW_INFO_CACHE.get(&key) { + Some(info) => { + tracing::debug!("Using cached flow version info for {version} ({path})"); + Ok(info) + } + _ => { + tracing::debug!("Fetching flow version info for {version} ({path})"); + let info = sqlx::query_as!( + FlowVersionInfo, + "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor, on_behalf_of_email, edited_by, flow_version.id AS version + FROM flow + INNER JOIN flow_version + ON flow_version.id = $3 + WHERE flow.path = $1 and flow.workspace_id = $2", + path, + w_id, + version + ) + .fetch_optional(&mut *conn) + .await?; + + let info = utils::not_found_if_none(info, "flow", path)?; + + FLOW_INFO_CACHE.insert(key, info.clone()); + + Ok(info) + } + } + } } pub async fn get_latest_hash_for_path<'c>( diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 93ea9d0bac..74e002f545 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -1943,7 +1943,7 @@ async fn handle_successful_schedule<'a, 'c, T: Serialize + Send + Sync>( Ok(()) } -#[derive(sqlx::Type, Serialize, Deserialize, Debug, Clone)] +#[derive(sqlx::Type, Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash)] #[sqlx(type_name = "TRIGGER_KIND", rename_all = "lowercase")] #[serde(rename_all = "lowercase")] pub enum TriggerKind { @@ -1959,6 +1959,24 @@ pub enum TriggerKind { Gcp } + +impl TriggerKind { + pub fn to_key(&self) -> String { + match self { + TriggerKind::Webhook => "webhook".to_string(), + TriggerKind::Http => "http".to_string(), + TriggerKind::Websocket => "websocket".to_string(), + TriggerKind::Kafka => "kafka".to_string(), + TriggerKind::Email => "email".to_string(), + TriggerKind::Nats => "nats".to_string(), + TriggerKind::Mqtt => "mqtt".to_string(), + TriggerKind::Sqs => "sqs".to_string(), + TriggerKind::Postgres => "postgres".to_string(), + TriggerKind::Gcp => "gcp".to_string(), + } + } +} + #[derive(sqlx::Type, Serialize, Deserialize, Debug, Clone)] #[sqlx(type_name = "JOB_TRIGGER_KIND", rename_all = "lowercase")] #[serde(rename_all = "lowercase")] @@ -3261,7 +3279,7 @@ pub async fn push<'c, 'd>( mut tx: PushIsolationLevel<'c>, workspace_id: &str, job_payload: JobPayload, - mut args: PushArgs<'d>, + args: PushArgs<'d>, user: &str, mut email: &str, mut permissioned_as: String, @@ -3477,17 +3495,10 @@ pub async fn push<'c, 'd>( priority, apply_preprocessor, } => { - let extra = args.extra.get_or_insert_with(HashMap::new); if apply_preprocessor { preprocessed = Some(false); - extra.entry("wm_trigger".to_string()).or_insert_with(|| { - to_raw_value(&serde_json::json!({ - "kind": "webhook", - })) - }); - } else { - extra.remove("wm_trigger"); - } + } + ( Some(hash.0), Some(path), @@ -3853,19 +3864,8 @@ pub async fn push<'c, 'd>( priority, ) } - JobPayload::Flow { path, dedicated_worker, apply_preprocessor } => { + JobPayload::Flow { path, dedicated_worker, apply_preprocessor, version } => { let mut ntx = tx.into_tx().await?; - // Fetch the latest version of the flow. - let version = sqlx::query_scalar!( - "SELECT flow.versions[array_upper(flow.versions, 1)] AS \"version!: i64\" - FROM flow WHERE path = $1 AND workspace_id = $2", - &path, - &workspace_id - ) - .fetch_optional(&mut *ntx) - .await? - .ok_or_else(|| Error::internal_err(format!("not found flow at path {:?}", path)))?; - // Do not use the lite version unless all workers are updated. let data = if *DISABLE_FLOW_SCRIPT || (!*MIN_VERSION_IS_AT_LEAST_1_432.read().await && !*CLOUD_HOSTED) @@ -3889,17 +3889,10 @@ pub async fn push<'c, 'd>( let concurrency_time_window_s = value.concurrency_time_window_s; let concurrent_limit = value.concurrent_limit; - let extra = args.extra.get_or_insert_with(HashMap::new); if !apply_preprocessor { value.preprocessor_module = None; - extra.remove("wm_trigger"); } else { preprocessed = Some(false); - extra.entry("wm_trigger".to_string()).or_insert_with(|| { - to_raw_value(&serde_json::json!({ - "kind": "webhook", - })) - }); } // this is a new flow being pushed, status is set to `value`. diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index af1fca6f5d..bb7d5df906 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -15,8 +15,10 @@ use std::str::FromStr; use windmill_common::db::Authed; use windmill_common::ee::LICENSE_KEY_VALID; use windmill_common::flows::Retry; +use windmill_common::get_latest_flow_version_info_for_path; use windmill_common::jobs::JobPayload; use windmill_common::schedule::schedule_to_user; +use windmill_common::FlowVersionInfo; use windmill_common::DB; use windmill_common::{ error::{self, Result}, @@ -114,21 +116,21 @@ pub async fn push_scheduled_job<'c>( } let (payload, tag, timeout, on_behalf_of_email, created_by) = if schedule.is_flow { - let r = sqlx::query!( - "SELECT tag, dedicated_worker, on_behalf_of_email, edited_by from flow WHERE path = $1 and workspace_id = $2", - &schedule.script_path, + let FlowVersionInfo { + version, tag, dedicated_worker, on_behalf_of_email, edited_by, .. + } = get_latest_flow_version_info_for_path( + &mut *tx, &schedule.workspace_id, + &schedule.script_path, + false, ) - .fetch_optional(&mut *tx) .await?; - let (tag, dedicated_worker, on_behalf_of_email, edited_by) = r - .map(|x| (x.tag, x.dedicated_worker, x.on_behalf_of_email, x.edited_by)) - .unwrap_or_else(|| (None, None, None, "".to_string())); ( JobPayload::Flow { path: schedule.script_path.clone(), dedicated_worker, apply_preprocessor: false, + version, }, tag, None, diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index fe5ffcdeca..3a88d50f9f 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -26,7 +26,6 @@ use sqlx::types::Json; use sqlx::{FromRow, Postgres, Transaction}; use tracing::instrument; use uuid::Uuid; -use windmill_common::add_time; use windmill_common::auth::JobPerms; #[cfg(feature = "benchmark")] use windmill_common::bench::BenchmarkIter; @@ -37,13 +36,16 @@ use windmill_common::flow_status::{ }; use windmill_common::flows::{add_virtual_items_if_necessary, Branch, FlowNodeId, StopAfterIf}; use windmill_common::jobs::{ - script_hash_to_tag_and_limits, script_path_to_payload, JobKind, JobPayload, OnBehalfOf, - RawCode, ENTRYPOINT_OVERRIDE, + script_path_to_payload, JobKind, JobPayload, OnBehalfOf, RawCode, ENTRYPOINT_OVERRIDE, }; use windmill_common::scripts::ScriptHash; use windmill_common::users::username_to_permissioned_as; use windmill_common::utils::WarnAfterExt; use windmill_common::worker::to_raw_value; +use windmill_common::{ + add_time, get_latest_flow_version_info_for_path, get_script_info_for_hash, FlowVersionInfo, + ScriptHashInfo, +}; use windmill_common::{ error::{self, to_anyhow, Error}, flow_status::{ @@ -3869,20 +3871,15 @@ async fn flow_to_payload( w_id: &str, db: &DB, ) -> Result { - let record = sqlx::query!( - "SELECT on_behalf_of_email, edited_by FROM flow WHERE path = $1 AND workspace_id = $2", - path, - w_id, - ) - .fetch_one(db) - .await - .map_err(|e| Error::NotFound(format!("fetching flow: {e:#}")))?; - let on_behalf_of = if let Some(email) = record.on_behalf_of_email { - Some(OnBehalfOf { email, permissioned_as: username_to_permissioned_as(&record.edited_by) }) + let FlowVersionInfo { version, on_behalf_of_email, edited_by, .. } = + get_latest_flow_version_info_for_path(db, w_id, &path, true).await?; + let on_behalf_of = if let Some(email) = on_behalf_of_email { + Some(OnBehalfOf { email, permissioned_as: username_to_permissioned_as(&edited_by) }) } else { None }; - let payload = JobPayload::Flow { path, dedicated_worker: None, apply_preprocessor: false }; + let payload = + JobPayload::Flow { path, dedicated_worker: None, apply_preprocessor: false, version }; Ok(JobPayloadWithTag { payload, tag: None, delete_after_use, timeout: None, on_behalf_of }) } @@ -3912,9 +3909,9 @@ async fn script_to_payload( } else { let hash = script_hash.unwrap(); let mut tx: sqlx::Transaction<'_, sqlx::Postgres> = db.begin().await?; - let ( + let ScriptHashInfo { tag, - custom_concurrency_key, + concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, @@ -3922,10 +3919,11 @@ async fn script_to_payload( dedicated_worker, priority, delete_after_use, - script_timeout, + timeout, on_behalf_of_email, created_by, - ) = script_hash_to_tag_and_limits(&hash, &mut tx, &flow_job.workspace_id).await?; + .. + } = get_script_info_for_hash(&mut *tx, &flow_job.workspace_id, hash.0).await?; let on_behalf_of = if let Some(email) = on_behalf_of_email { Some(OnBehalfOf { email, permissioned_as: username_to_permissioned_as(&created_by) }) } else { @@ -3935,7 +3933,7 @@ async fn script_to_payload( JobPayload::ScriptHash { hash, path: script_path, - custom_concurrency_key, + custom_concurrency_key: concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl: module.cache_ttl.map(|x| x as i32).ok_or(cache_ttl).ok(), @@ -3946,7 +3944,7 @@ async fn script_to_payload( }, tag_override.to_owned().or(tag), delete_after_use, - script_timeout, + timeout, on_behalf_of, ) }; diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 82ad8cc77b..7744ef341b 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -544,9 +544,9 @@ async fn trigger_dependents_to_recompute_dependencies( match r { Ok(r) => JobPayload::Dependencies { path: s.importer_path.clone(), - hash: r.0, - language: r.6, - dedicated_worker: r.7, + hash: ScriptHash(r.hash), + language: r.language, + dedicated_worker: r.dedicated_worker, }, Err(err) => { tracing::error!( diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 6764757796..ba2b168d3b 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -75,7 +75,7 @@ "windmill-parser-wasm-py": "^1.477.1", "windmill-parser-wasm-regex": "^1.481.0", "windmill-parser-wasm-rust": "^1.429.0", - "windmill-parser-wasm-ts": "^1.438.2", + "windmill-parser-wasm-ts": "^1.486.1", "windmill-parser-wasm-yaml": "^1.429.0", "windmill-sql-datatype-parser-wasm": "^1.318.0", "y-monaco": "^0.1.4", @@ -10913,7 +10913,9 @@ "version": "1.429.0" }, "node_modules/windmill-parser-wasm-ts": { - "version": "1.438.2" + "version": "1.486.1", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.486.1.tgz", + "integrity": "sha512-nv7nPpZA5O0Zsve6W2Sm3mxCSxMqg2rfyYHNqozIpBaGhbe8SpgnbyrSaXKsGKSc/bpwpANHE7nVCp23pQUK6Q==" }, "node_modules/windmill-parser-wasm-yaml": { "version": "1.429.0" diff --git a/frontend/package.json b/frontend/package.json index 191a6d1540..f93a3b90a6 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -144,7 +144,7 @@ "windmill-parser-wasm-py": "^1.477.1", "windmill-parser-wasm-regex": "^1.481.0", "windmill-parser-wasm-rust": "^1.429.0", - "windmill-parser-wasm-ts": "^1.438.2", + "windmill-parser-wasm-ts": "^1.486.1", "windmill-parser-wasm-yaml": "^1.429.0", "windmill-sql-datatype-parser-wasm": "^1.318.0", "y-monaco": "^0.1.4", diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index dbbf7e9e89..180e0e57e9 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -126,15 +126,18 @@ oneOf.length >= 2 && (!oneOfSelected || !oneOf.some((o) => o.title === oneOfSelected) || !value) ) { - if (value && value['label'] && oneOf.some((o) => o.title === value['label'])) { + const tagKey = oneOf.find((o) => Object.keys(o.properties ?? {}).includes('kind')) + ? 'kind' + : 'label' + if (value && value[tagKey] && oneOf.some((o) => o.title === value[tagKey])) { const existingValue = JSON.parse(JSON.stringify(value)) - oneOfSelected = value['label'] + oneOfSelected = value[tagKey] await tick() value = existingValue } else { - const label = oneOf[0]['title'] - oneOfSelected = label - value = { ...(typeof value === 'object' ? (value ?? {}) : {}), label } + const variantTitle = oneOf[0]['title'] + oneOfSelected = variantTitle + value = { ...(typeof value === 'object' ? (value ?? {}) : {}), [tagKey]: variantTitle } } } } @@ -145,8 +148,11 @@ function onOneOfChange() { const label = value?.['label'] + const kind = value?.['kind'] if (label && oneOf && oneOf.some((o) => o.title == label) && oneOfSelected != label) { oneOfSelected = label + } else if (kind && oneOf && oneOf.some((o) => o.title == kind) && oneOfSelected != kind) { + oneOfSelected = kind } } @@ -820,7 +826,10 @@ for (const key of prevValueKeys) { toKeep[key] = value[key] } - value = { ...toKeep, label: detail } + const tagKey = oneOf.find((o) => Object.keys(o.properties ?? {}).includes('kind')) + ? 'kind' + : 'label' + value = { ...toKeep, [tagKey]: detail } }} let:item > @@ -849,7 +858,7 @@ }} bind:args={value} dndType={`nested-${title}`} - hiddenArgs={['label']} + hiddenArgs={['label', 'kind']} on:reorder={(e) => { if (oneOf && oneOf[objIdx]) { const keys = e.detail @@ -868,7 +877,7 @@ {onlyMaskPassword} {disablePortal} {disabled} - hiddenArgs={['label']} + hiddenArgs={['label', 'kind']} schema={{ properties: obj.properties, order: obj.order, diff --git a/frontend/src/lib/components/schema/FlowPropertyEditor.svelte b/frontend/src/lib/components/schema/FlowPropertyEditor.svelte index e975324a42..1ef4606ab1 100644 --- a/frontend/src/lib/components/schema/FlowPropertyEditor.svelte +++ b/frontend/src/lib/components/schema/FlowPropertyEditor.svelte @@ -53,7 +53,7 @@ return oneOf.map((v) => ({ ...v, properties: Object.fromEntries( - Object.entries(v.properties ?? {}).filter(([k, v]) => k !== 'label') + Object.entries(v.properties ?? {}).filter(([k, v]) => k !== 'label' && k !== 'kind') ) })) } @@ -122,12 +122,17 @@ properties = structuredClone(changedSchema.properties) order = structuredClone(changedSchema.order) requiredProperty = structuredClone(changedSchema.required) + + const tagKey = oneOf?.find((o) => Object.keys(o.properties ?? {}).includes('kind')) + ? 'kind' + : 'label' + oneOf = changedSchema.oneOf?.map((v) => { return { ...v, properties: { ...(v.properties ?? {}), - label: { + [tagKey]: { type: 'string', enum: [v.title ?? ''] } diff --git a/frontend/src/lib/components/schema/PropertyEditor.svelte b/frontend/src/lib/components/schema/PropertyEditor.svelte index 9071a64841..6fa69f64dd 100644 --- a/frontend/src/lib/components/schema/PropertyEditor.svelte +++ b/frontend/src/lib/components/schema/PropertyEditor.svelte @@ -55,7 +55,9 @@ oneOfSchemas = oneOf.map((obj) => { return { properties: obj.properties - ? Object.fromEntries(Object.entries(obj.properties).filter(([k, v]) => k !== 'label')) + ? Object.fromEntries( + Object.entries(obj.properties).filter(([k, v]) => k !== 'label' && k !== 'kind') + ) : {}, order: obj.order } diff --git a/frontend/src/lib/components/triggers/CaptureTable.svelte b/frontend/src/lib/components/triggers/CaptureTable.svelte index 71df4c2991..cc784e33b6 100644 --- a/frontend/src/lib/components/triggers/CaptureTable.svelte +++ b/frontend/src/lib/components/triggers/CaptureTable.svelte @@ -83,31 +83,41 @@ let capturesWithPayload: CaptureWithPayload[] = captures.map((capture) => { let newCapture: CaptureWithPayload = { ...capture } - if (capture.payload === 'WINDMILL_TOO_BIG') { + + const isLarge = + capture.main_args === 'WINDMILL_TOO_BIG' || + capture.preprocessor_args === 'WINDMILL_TOO_BIG' + if (isLarge) { newCapture = { ...capture, + payloadData: 'Too big to display here, select to view', getFullCapture: () => CaptureService.getCapture({ workspace: $workspaceStore!, id: capture.id }) } + return newCapture } - const trigger_extra = isObject(capture.trigger_extra) ? capture.trigger_extra : {} - newCapture.payloadData = - kind === 'preprocessor' - ? capture.payload === 'WINDMILL_TOO_BIG' - ? { - payload: capture.payload, - ...trigger_extra - } - : typeof capture.payload === 'object' + const preprocessor_args = isObject(capture.preprocessor_args) + ? capture.preprocessor_args + : {} + + if ('wm_trigger' in preprocessor_args) { + // v1 + newCapture.payloadData = + kind === 'preprocessor' + ? typeof capture.main_args === 'object' ? { - ...capture.payload, - ...trigger_extra + ...capture.main_args, + ...preprocessor_args } - : trigger_extra - : capture.payload + : preprocessor_args + : capture.main_args + } else { + // v2 + newCapture.payloadData = kind === 'preprocessor' ? preprocessor_args : capture.main_args + } return newCapture }) @@ -138,13 +148,23 @@ let payloadData: any = {} if (capture.getFullCapture) { const fullCapture = await capture.getFullCapture() - payloadData = - testKind === 'preprocessor' - ? { - ...(typeof fullCapture.payload === 'object' ? fullCapture.payload : {}), - ...(typeof fullCapture.trigger_extra === 'object' ? fullCapture.trigger_extra : {}) - } - : fullCapture.payload + const preprocessor_args = isObject(fullCapture.preprocessor_args) + ? fullCapture.preprocessor_args + : {} + if ('wm_trigger' in preprocessor_args) { + // v1 + payloadData = + testKind === 'preprocessor' + ? { + ...(typeof fullCapture.main_args === 'object' ? fullCapture.main_args : {}), + ...preprocessor_args + } + : fullCapture.main_args + } else { + // v2 + payloadData = + testKind === 'preprocessor' ? fullCapture.preprocessor_args : fullCapture.main_args + } } else { payloadData = structuredClone(capture.payloadData) } diff --git a/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte index b59fb4c60f..61a8684234 100644 --- a/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte @@ -263,7 +263,7 @@ btnClasses="ml-4 mt-2" color="dark" size="xs" - href={itemKind === 'flow' ? '/flows/add?hub=68' : '/scripts/add?hub=hub%2F14251'} + href={itemKind === 'flow' ? '/flows/add?hub=68' : '/scripts/add?hub=hub%2F11446'} target="_blank">Create from template {/if} diff --git a/frontend/src/lib/components/triggers/http/RouteBodyTransformerOption.svelte b/frontend/src/lib/components/triggers/http/RouteBodyTransformerOption.svelte index 62d54a5a35..d9f129851f 100644 --- a/frontend/src/lib/components/triggers/http/RouteBodyTransformerOption.svelte +++ b/frontend/src/lib/components/triggers/http/RouteBodyTransformerOption.svelte @@ -30,8 +30,10 @@ Wraps the payload in an object under the 'body' key, useful for handling unknown payloads. + Wraps the payload in an object under the 'body' key, useful for handling unknown payloads. + Note that this will have no effect when using a preprocessor. + Create from template {/if} diff --git a/frontend/src/lib/components/triggers/mqtt/MqttTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/mqtt/MqttTriggerEditorInner.svelte index 6a8912d9c3..3a94f63a9d 100644 --- a/frontend/src/lib/components/triggers/mqtt/MqttTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/mqtt/MqttTriggerEditorInner.svelte @@ -239,7 +239,7 @@ btnClasses="ml-4 mt-2" color="dark" size="xs" - href={itemKind === 'flow' ? '/flows/add?hub=61' : '/scripts/add?hub=hub%2F11619'} + href={itemKind === 'flow' ? '/flows/add?hub=61' : '/scripts/add?hub=hub%2F19655'} target="_blank">Create from template {/if} diff --git a/frontend/src/lib/components/triggers/nats/NatsTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/nats/NatsTriggerEditorInner.svelte index 72b05a6a9e..6d3cab175f 100644 --- a/frontend/src/lib/components/triggers/nats/NatsTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/nats/NatsTriggerEditorInner.svelte @@ -66,8 +66,8 @@ args.nats_resource_path = nDefaultValues?.nats_resource_path ?? '' args.subjects = nDefaultValues?.subjects ?? [''] args.use_jetstream = nDefaultValues?.use_jetstream ?? false - args.stream_name = args.use_jetstream ? nDefaultValues?.stream_name ?? '' : undefined - args.consumer_name = args.use_jetstream ? nDefaultValues?.consumer_name ?? '' : undefined + args.stream_name = args.use_jetstream ? (nDefaultValues?.stream_name ?? '') : undefined + args.consumer_name = args.use_jetstream ? (nDefaultValues?.consumer_name ?? '') : undefined initialScriptPath = '' fixedScriptPath = fixedScriptPath_ ?? '' script_path = fixedScriptPath @@ -242,7 +242,7 @@ btnClasses="ml-4 mt-2" color="dark" size="xs" - href={itemKind === 'flow' ? '/flows/add?hub=66' : '/scripts/add?hub=hub%2F11634'} + href={itemKind === 'flow' ? '/flows/add?hub=66' : '/scripts/add?hub=hub%2F19663'} target="_blank">Create from template {/if} diff --git a/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorInner.svelte index 197f5382bd..eee25de840 100644 --- a/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorInner.svelte @@ -225,7 +225,7 @@ btnClasses="ml-4 mt-2" color="dark" size="xs" - href={itemKind === 'flow' ? '/flows/add?hub=59' : '/scripts/add?hub=hub%2F11637'} + href={itemKind === 'flow' ? '/flows/add?hub=59' : '/scripts/add?hub=hub%2F19657'} target="_blank">Create from template {/if} diff --git a/frontend/src/lib/components/triggers/websocket/WebsocketTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/websocket/WebsocketTriggerEditorInner.svelte index ef00609f28..fea948ee30 100644 --- a/frontend/src/lib/components/triggers/websocket/WebsocketTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/websocket/WebsocketTriggerEditorInner.svelte @@ -295,7 +295,7 @@ btnClasses="ml-4 mt-2" color="dark" size="xs" - href={itemKind === 'flow' ? '/flows/add?hub=64' : '/scripts/add?hub=hub%2F11636'} + href={itemKind === 'flow' ? '/flows/add?hub=64' : '/scripts/add?hub=hub%2F19660'} target="_blank">Create from template {/if} diff --git a/frontend/src/lib/script_helpers.ts b/frontend/src/lib/script_helpers.ts index a9a0ceb4de..3d58eac5b7 100644 --- a/frontend/src/lib/script_helpers.ts +++ b/frontend/src/lib/script_helpers.ts @@ -631,15 +631,6 @@ export const TS_PREPROCESSOR_SCRIPT_INTRO = `/** * It processes raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, Postgres, or email) * before passing it to \`main\`. This separates the trigger logic from the main logic and keeps the auto-generated runnable UI clean. * - * The preprocessor receives trigger metadata (\`wm_trigger\`) along with the main trigger arguments. - * The structure of \`wm_trigger\` and the main trigger arguments are specific to each trigger type: - * - Webhook/HTTP: \`(wm_trigger: { kind: 'http' | 'webhook', http?: { ... } }, body_key_1: any, body_key_2: any, ...)\` - * - Postgres: \`(wm_trigger: { kind: 'postgres' }, transaction_type: string, schema_name: string, table_name: string, old_row?: any, row: any)\` - * - WebSocket/Kafka/NATS/SQS: \`(wm_trigger: { kind: 'websocket' | 'kafka' | 'nats' | 'sqs', [kind]: { ... } }, msg: string)\` - * - MQTT: \`(wm_trigger: { kind: 'mqtt', [kind]: { ... } }, payload: Array)\` - * - GCP: \`(wm_trigger: { kind: 'gcp', [kind]: { ... } }, payload: string)\` - * - Email: \`(wm_trigger: { kind: 'email' }, raw_email: string, parsed_email: { ... })\` - * * The returned object defines the parameter values passed to \`main()\`. * e.g., { b: 1, a: 2 } → Calls \`main(2, 1)\`, assuming \`main\` is defined as \`main(a: number, b: number)\`. * Ensure that the parameter names in \`main\` match the keys in the returned object. @@ -652,15 +643,6 @@ export const TS_PREPROCESSOR_FLOW_INTRO = `/** * * It processes raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, Postgres, or email) * before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. - * - * The preprocessor receives trigger metadata (\`wm_trigger\`) along with the main trigger arguments. - * The structure of \`wm_trigger\` and the main trigger arguments are specific to each trigger type: - * - Webhook/HTTP: \`(wm_trigger: { kind: 'http' | 'webhook', http?: { ... } }, body_key_1: any, body_key_2: any, ...)\` - * - Postgres: \`(wm_trigger: { kind: 'postgres' }, transaction_type: string, schema_name: string, table_name: string, old_row?: any, row: any)\` - * - WebSocket/Kafka/NATS/SQS: \`(wm_trigger: { kind: 'websocket' | 'kafka' | 'nats' | 'sqs', [kind]: { ... } }, msg: string)\` - * - MQTT: \`(wm_trigger: { kind: 'mqtt', [kind]: { ... } }, payload: Array)\` - * - GCP: \`(wm_trigger: { kind: 'gcp', [kind]: { ... } }, payload: string)\`* - * - Email: \`(wm_trigger: { kind: 'email' }, raw_email: string, parsed_email: { ... })\` * * The returned object determines the parameter values passed to the flow. * e.g., \`{ b: 1, a: 2 }\` → Calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`. @@ -670,77 +652,92 @@ export const TS_PREPROCESSOR_FLOW_INTRO = `/** */\n` export const TS_PREPROCESSOR_MODULE_CODE = `export async function preprocessor( - /* - * Replace this comment with the parameters received from the trigger. - * Examples: \`bodyKey1\`, \`bodyKey2\` for Webhook/HTTP, \`msg\` for WebSocket, etc. - */ - - // The trigger metadata - wm_trigger: { - kind: 'http' | 'email' | 'webhook' | 'websocket' | 'kafka' | 'nats' | 'postgres' | 'sqs' | 'mqtt' | 'gcp', - http?: { - route: string // The route path, e.g. "/users/:id" - path: string // The actual path called, e.g. "/users/123" - method: string - params: Record // path parameters - query: Record // query parameters - headers: Record - }, - websocket?: { - url: string // The websocket url - }, - kafka?: { - brokers: string[] - topic: string - group_id: string - }, - nats?: { - servers: string[] - subject: string - headers?: Record - status?: number - description?: string - length: number - }, - sqs?: { - queue_url: string, - message_id?: string, - receipt_handle?: string, - attributes: Record, - message_attributes?: Record - }, - mqtt?: { - topic: string, - retain: boolean, - pkid: number, - qos: number, - v5?: { - payload_format_indicator?: number, - topic_alias?: number, - response_topic?: string, - correlation_data?: Array, - user_properties?: Array<[string, string]>, - subscription_identifiers?: Array, - content_type?: string + event: + | { + kind: "webhook"; + body: any, + raw_string: string | null, + query: Record; + headers: Record; + } + | { + kind: "http"; + body: any, + raw_string: string | null, + route: string; + path: string; + method: string; + params: Record; + query: Record; + headers: Record; + } + | { + kind: "email"; + parsed_email: any, + raw_email: string, + } + | { kind: "websocket"; msg: string; url: string } + | { + kind: "kafka"; + payload: string; + brokers: string[]; + topic: string; + group_id: string; + } + | { + kind: "nats"; + payload: string; + servers: string[]; + subject: string; + headers?: Record; + status?: number; + description?: string; + length: number; + } + | { + kind: "sqs"; + msg: string, + queue_url: string; + message_id?: string; + receipt_handle?: string; + attributes: Record; + message_attributes?: Record< + string, + { string_value?: string; data_type: string } + >; + } + | { + kind: "mqtt"; + payload: string, + topic: string; + retain: boolean; + pkid: number; + qos: number; + v5?: { + payload_format_indicator?: number; + topic_alias?: number; + response_topic?: string; + correlation_data?: Array; + user_properties?: Array<[string, string]>; + subscription_identifiers?: Array; + content_type?: string; + }; + } + | { + kind: "gcp"; + payload: string, + message_id: string; + subscription: string; + ordering_key?: string; + attributes?: Record; + delivery_type: "push" | "pull"; + headers?: Record; + publish_time?: string; } - }, - gcp?: { - message_id: string, - subscription: string, - ordering_key?: string, - attributes?: Record, - delivery_type: "push" | "pull", - headers?: Record, - publish_time?: string, - } - } ) { return { // return the args to be passed to the runnable - } + }; } ` @@ -778,15 +775,6 @@ export const PYTHON_PREPROCESSOR_SCRIPT_INTRO = `# Trigger preprocessor # It processes raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, Postgres, or email) # before passing it to \`main\`. This separates the trigger logic from the main logic and keeps the auto-generated UI clean. # -# The preprocessor receives trigger metadata (\`wm_trigger\`) along with the main trigger arguments. -# The structure of \`wm_trigger\` and the main trigger arguments are specific to each trigger type: -# - Webhook/HTTP: \`(wm_trigger: { kind: 'http' | 'webhook', http?: { ... } }, body_key_1: any, body_key_2: any, ...)\` -# - Postgres: \`(wm_trigger: { kind: 'postgres' }, transaction_type: string, schema_name: string, table_name: string, old_row?: any, row: any)\` -# - WebSocket/Kafka/NATS/SQS: \`(wm_trigger: { kind: 'websocket' | 'kafka' | 'nats' | 'sqs', [kind]: { ... } }, msg: string)\` -# - MQTT: \`(wm_trigger: { kind: 'mqtt', [kind]: { ... } }, payload: Array)\` -# - GCP: \`(wm_trigger: { kind: 'gcp', [kind]: { ... } }, payload: string)\` -# - Email: \`(wm_trigger: { kind: 'email' }, raw_email: string, parsed_email: { ... })\` -# # The returned object defines the parameter values passed to \`main()\`. # e.g., { b: 1, a: 2 } → Calls \`main(2, 1)\`, assuming \`main\` is defined as \`main(a: int, b: int)\`. # Ensure that the parameter names in \`main\` match the keys in the returned object. @@ -797,15 +785,6 @@ export const PYTHON_PREPROCESSOR_FLOW_INTRO = `# Trigger preprocessor # # It processes raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, Postgres, or email) # before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean. -# -# The preprocessor receives the same data the flow would if no preprocessor was used, -# plus trigger metadata in the \`wm_trigger\` object: -# - Webhook/HTTP: \`(wm_trigger: { kind: 'http' | 'webhook', http?: { ... } }, body_key_1: any, body_key_2: any, ...)\` -# - Postgres: \`(wm_trigger: { kind: 'postgres' }, transaction_type: string, schema_name: string, table_name: string, old_row?:any, row: any)\` -# - WebSocket/Kafka/NATS/SQS: \`(wm_trigger: { kind: 'websocket' | 'kafka' | 'nats' | 'sqs', [kind]: { ... } }, msg: string)\` -# - MQTT: \`(wm_trigger: { kind: 'mqtt', [kind]: { ... } }, payload: Array)\` -# - GCP: \`(wm_trigger: { kind: 'gcp', [kind]: { ... } }, payload: string)\` -# - Email: \`(wm_trigger: { kind: 'email' }, raw_email: string, parsed_email: { ... })\` # # The returned object determines the parameter values passed to the flow. # e.g., \`{ b: 1, a: 2 }\` → Calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`. @@ -813,85 +792,120 @@ export const PYTHON_PREPROCESSOR_FLOW_INTRO = `# Trigger preprocessor # # Learn more: https://www.windmill.dev/docs/core_concepts/preprocessors\n\n` -export const PYTHON_PREPROCESSOR_MODULE_CODE = `from typing import TypedDict, Literal -class Http(TypedDict): - route: str # The route path, e.g. "/users/:id" - path: str # The actual path called, e.g. "/users/123" +export const PYTHON_PREPROCESSOR_MODULE_CODE = `from typing import TypedDict, Literal, Optional, Union + + +class WebhookEvent(TypedDict): + kind: Literal["webhook"] + body: dict + raw_string: Optional[str] + query: dict[str, str] + headers: dict[str, str] + + +class HttpEvent(TypedDict): + kind: Literal["http"] + body: dict + raw_string: Optional[str] + route: str + path: str method: str params: dict[str, str] query: dict[str, str] headers: dict[str, str] -class Websocket(TypedDict): - url: str # The websocket url -class Kafka(TypedDict): - topic: str +class EmailEvent(TypedDict): + kind: Literal["email"] + parsed_email: dict + raw_email: str + + +class WebsocketEvent(TypedDict): + kind: Literal["websocket"] + msg: str + url: str + + +class KafkaEvent(TypedDict): + kind: Literal["kafka"] + payload: str brokers: list[str] + topic: str group_id: str -class Nats(TypedDict): + +class NatsEvent(TypedDict): + kind: Literal["nats"] + payload: str servers: list[str] subject: str - headers: dict[str, list[str]] | None - status: int | None - description: str | None + headers: Optional[dict[str, list[str]]] + status: Optional[int] + description: Optional[str] length: int + class MessageAttribute(TypedDict): - string_value: str | None + string_value: Optional[str] data_type: str -class Sqs(TypedDict): + +class SqsEvent(TypedDict): + kind: Literal["sqs"] + msg: str queue_url: str - message_id: str | None - receipt_handle: str | None + message_id: Optional[str] + receipt_handle: Optional[str] attributes: dict[str, str] - message_attributes: dict[str, MessageAttribute] | None + message_attributes: Optional[dict[str, MessageAttribute]] -class MqttV5Properties: - payload_format_indicator: int | None - topic_alias: int | None - response_topic: str | None - correlation_data: list[int] | None - user_properties: list[tuple[str, str]] | None - subscription_identifiers: list[int] | None - content_type: str | None -class Mqtt(TypedDict): +class MqttV5Properties(TypedDict, total=False): + payload_format_indicator: Optional[int] + topic_alias: Optional[int] + response_topic: Optional[str] + correlation_data: Optional[list[int]] + user_properties: Optional[list[tuple[str, str]]] + subscription_identifiers: Optional[list[int]] + content_type: Optional[str] + + +class MqttEvent(TypedDict): + kind: Literal["mqtt"] + payload: str topic: str retain: bool pkid: int qos: int - v5: MqttV5Properties | None + v5: Optional[MqttV5Properties] -class Gcp(TypedDict): + +class GcpEvent(TypedDict): + kind: Literal["gcp"] + payload: str message_id: str subscription: str - ordering_key: str | None - attributes: dict[str, str] | None + ordering_key: Optional[str] + attributes: Optional[dict[str, str]] delivery_type: Literal["push", "pull"] - headers: dict[str, str] | None - publish_time: str | None - - -class WmTrigger(TypedDict): - kind: Literal["http", "email", "webhook", "websocket", "kafka", "nats", "postgres", "sqs", "mqtt", "gcp"] - http: Http | None - websocket: Websocket | None - kafka: Kafka | None - nats: Nats | None - sqs: Sqs | None - mqtt: Mqtt | None - gcp: Gcp | None + headers: Optional[dict[str, str]] + publish_time: Optional[str] -def preprocessor( - # Replace this comment with the parameters received from the trigger. - # Examples: \`bodyKey1\`, \`bodyKey2\` for Webhook/HTTP, \`msg\` for WebSocket, etc. +Event = Union[ + WebhookEvent, + HttpEvent, + EmailEvent, + WebsocketEvent, + KafkaEvent, + NatsEvent, + SqsEvent, + MqttEvent, + GcpEvent, +] - # Trigger metadata - wm_trigger: WmTrigger, -): + +def preprocessor(event: Event): return { # return the args to be passed to the runnable } @@ -1037,7 +1051,7 @@ public class Main { } } ` -// for related places search: ADD_NEW_LANG +// for related places search: ADD_NEW_LANG export const INITIAL_CODE = { bun: { scriptInitCodeBlock: BUN_INIT_BLOCK, @@ -1124,8 +1138,8 @@ export const INITIAL_CODE = { }, java: { script: JAVA_INIT_CODE - }, - // for related places search: ADD_NEW_LANG + } + // for related places search: ADD_NEW_LANG } export function isInitialCode(content: string): boolean { @@ -1231,7 +1245,7 @@ export function initialCode( return INITIAL_CODE.nu.script } else if (language == 'java') { return INITIAL_CODE.java.script - // for related places search: ADD_NEW_LANG + // for related places search: ADD_NEW_LANG } else if (language == 'bun' || language == 'bunnative') { if (kind == 'trigger') { return INITIAL_CODE.bun.trigger