starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-20 08:01:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: preprocessor refactor (#5629)

Browse Source 
* feat: preprocessor refactor

* nit

* oneOf kind support + new capture format + fix build

* fix build

* fix build

* update preprocessor templates

* cache runnable version info

* update ee ref

* update hub scripts && ee ref
This commit is contained in:
HugoCasa
2025-05-09 09:50:01 +02:00
committed by GitHub
parent eceef65d65
commit d2787cfd78
61 changed files with 2068 additions and 1546 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/.sqlx/query-75451b6d48e4c26812ae64981d0d968b8fb0bf4374a2fccc167fa879bad7078f.json → backend/.sqlx/query-0937e25e89959447e7cb1816c112bbc4718cbb8ad6e2f13eae6b14f129d12936.json
Generated
+10 -4
View File
@@ -1,12 +1,12 @@
{
"db_name": "PostgreSQL",
"query": "select path, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor, on_behalf_of_email, created_by from script where hash = $1 AND workspace_id = $2",
"query": "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor, on_behalf_of_email, created_by, path from script where hash = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
"name": "hash",
"type_info": "Int8"
},
{
"ordinal": 1,
@@ -101,6 +101,11 @@
"ordinal": 13,
"name": "created_by",
"type_info": "Varchar"
},
{
"ordinal": 14,
"name": "path",
"type_info": "Varchar"
}
],
"parameters": {
@@ -123,8 +128,9 @@
true,
true,
true,
false,
false
]
},
"hash": "75451b6d48e4c26812ae64981d0d968b8fb0bf4374a2fccc167fa879bad7078f"
"hash": "0937e25e89959447e7cb1816c112bbc4718cbb8ad6e2f13eae6b14f129d12936"
}
backend/.sqlx/query-0e296134f05593edc989c628c00cbb60a5446993217baffa83f843bc12a5ac73.json
Generated
+35
View File
@@ -0,0 +1,35 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n value->'preprocessor_module' IS NOT NULL as has_preprocessor,\n value->'preprocessor_module'->'value'->'input_transforms'->'wm_trigger' IS NOT NULL as is_v1_preprocessor,\n schema as \"schema: _\"\n FROM flow \n WHERE workspace_id = $1 \n AND path = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "has_preprocessor",
"type_info": "Bool"
},
{
"ordinal": 1,
"name": "is_v1_preprocessor",
"type_info": "Bool"
},
{
"ordinal": 2,
"name": "schema: _",
"type_info": "Json"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null,
null,
true
]
},
"hash": "0e296134f05593edc989c628c00cbb60a5446993217baffa83f843bc12a5ac73"
}
backend/.sqlx/query-3a534b4fc36171efaa7c647f48320b59bbc414cfb92e960c174dd63fc180e187.json
Generated
-29
View File
@@ -1,29 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT tag, dedicated_worker from flow WHERE path = $1 and workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "tag",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "dedicated_worker",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
true,
true
]
},
"hash": "3a534b4fc36171efaa7c647f48320b59bbc414cfb92e960c174dd63fc180e187"
}
backend/.sqlx/query-4455c7e8aa7616f3d547c5eb7d93c840d8aff3df4a6926f569792f69b2e9601f.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "select hash from script where path = $1 AND workspace_id = $2 AND deleted = false AND lock IS not NULL AND lock_error_logs IS NULL ORDER BY created_at DESC LIMIT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "hash",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "4455c7e8aa7616f3d547c5eb7d93c840d8aff3df4a6926f569792f69b2e9601f"
}
backend/.sqlx/query-7cba31d597215a343cb0bca5b204a15fbba193262f7895c2bba90feb4215d6f3.json → backend/.sqlx/query-4f547c0fd54f3bc57212ce87810e35adf640d44d607e62a1fb296e38ac3fdd36.json
Generated
+5 -5
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n id, \n created_at, \n trigger_kind AS \"trigger_kind: _\",\n CASE \n WHEN pg_column_size(payload) < 40000 THEN payload \n ELSE '\"WINDMILL_TOO_BIG\"'::jsonb \n END AS \"payload!: _\",\n trigger_extra AS \"trigger_extra: _\"\n FROM \n capture\n WHERE \n workspace_id = $1 \n AND path = $2 \n AND is_flow = $3 \n AND ($4::trigger_kind IS NULL OR trigger_kind = $4)\n ORDER BY \n created_at DESC\n OFFSET $5\n LIMIT $6\n ",
"query": "\n SELECT \n id, \n created_at, \n trigger_kind AS \"trigger_kind: _\",\n CASE \n WHEN pg_column_size(main_args) < 40000 THEN main_args \n ELSE '\"WINDMILL_TOO_BIG\"'::jsonb \n END AS \"main_args!: _\",\n CASE\n WHEN pg_column_size(preprocessor_args) < 40000 THEN preprocessor_args\n ELSE '\"WINDMILL_TOO_BIG\"'::jsonb\n END AS \"preprocessor_args: _\"\n FROM \n capture\n WHERE \n workspace_id = $1 \n AND path = $2 \n AND is_flow = $3 \n AND ($4::trigger_kind IS NULL OR trigger_kind = $4)\n ORDER BY \n created_at DESC\n OFFSET $5\n LIMIT $6\n ",
"describe": {
"columns": [
{
@@ -38,12 +38,12 @@
},
{
"ordinal": 3,
"name": "payload!: _",
"name": "main_args!: _",
"type_info": "Jsonb"
},
{
"ordinal": 4,
"name": "trigger_extra: _",
"name": "preprocessor_args: _",
"type_info": "Jsonb"
}
],
@@ -80,8 +80,8 @@
false,
false,
null,
true
null
]
},
"hash": "7cba31d597215a343cb0bca5b204a15fbba193262f7895c2bba90feb4215d6f3"
"hash": "4f547c0fd54f3bc57212ce87810e35adf640d44d607e62a1fb296e38ac3fdd36"
}
backend/.sqlx/query-6d36da815795d5cac2e76b4d34a7f1f9f836bd5b6866d3db26b25428d39c0b23.json
Generated
-29
View File
@@ -1,29 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT on_behalf_of_email, edited_by FROM flow WHERE path = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "on_behalf_of_email",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "edited_by",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
true,
false
]
},
"hash": "6d36da815795d5cac2e76b4d34a7f1f9f836bd5b6866d3db26b25428d39c0b23"
}
backend/.sqlx/query-89de3ff8ab32e545efcbcda05f994cb1a32c4991cbd25046282d34272587d2de.json
Generated
-130
View File
@@ -1,130 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor, on_behalf_of_email, created_by from script where path = $1 AND workspace_id = $2 AND\n created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND\n deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "hash",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "tag",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "concurrency_key",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "concurrent_limit",
"type_info": "Int4"
},
{
"ordinal": 4,
"name": "concurrency_time_window_s",
"type_info": "Int4"
},
{
"ordinal": 5,
"name": "cache_ttl",
"type_info": "Int4"
},
{
"ordinal": 6,
"name": "language: ScriptLang",
"type_info": {
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java"
]
}
}
}
},
{
"ordinal": 7,
"name": "dedicated_worker",
"type_info": "Bool"
},
{
"ordinal": 8,
"name": "priority",
"type_info": "Int2"
},
{
"ordinal": 9,
"name": "delete_after_use",
"type_info": "Bool"
},
{
"ordinal": 10,
"name": "timeout",
"type_info": "Int4"
},
{
"ordinal": 11,
"name": "has_preprocessor",
"type_info": "Bool"
},
{
"ordinal": 12,
"name": "on_behalf_of_email",
"type_info": "Text"
},
{
"ordinal": 13,
"name": "created_by",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
true,
true,
true,
true,
true,
false,
true,
true,
true,
true,
true,
true,
false
]
},
"hash": "89de3ff8ab32e545efcbcda05f994cb1a32c4991cbd25046282d34272587d2de"
}
backend/.sqlx/query-8fb2581a439c26391e66ae7fac32c6cd2932f28ab6490ace027ed3a790b2a0f7.json
Generated
-28
View File
@@ -1,28 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT edited_by, on_behalf_of_email FROM flow WHERE path = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "on_behalf_of_email",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
true
]
},
"hash": "8fb2581a439c26391e66ae7fac32c6cd2932f28ab6490ace027ed3a790b2a0f7"
}
backend/.sqlx/query-97048ce0bcabb9baecb80cde5ab3c989e1575fbd20ef22766d2887a86dce15e1.json
Generated
-23
View File
@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT flow.versions[array_upper(flow.versions, 1)] AS \"version!: i64\"\n FROM flow WHERE path = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "version!: i64",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "97048ce0bcabb9baecb80cde5ab3c989e1575fbd20ef22766d2887a86dce15e1"
}
backend/.sqlx/query-a0833b9899833166891c5de926f78632fae1123e736d728bf92cb2de004b6826.json → backend/.sqlx/query-9b60fa8a1003015bc5a7cdbee9a4486b313d45347dfd9d4793d60e2760763ca3.json
Generated
+10 -3
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor, on_behalf_of_email, edited_by\n FROM flow \n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n WHERE flow.path = $1 and flow.workspace_id = $2",
"query": "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor, on_behalf_of_email, edited_by, flow_version.id AS version\n FROM flow\n INNER JOIN flow_version\n ON flow_version.id = $3\n WHERE flow.path = $1 and flow.workspace_id = $2",
"describe": {
"columns": [
{
@@ -32,12 +32,18 @@
"ordinal": 5,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 6,
"name": "version",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"Text"
"Text",
"Int8"
]
},
"nullable": [
@@ -46,8 +52,9 @@
null,
null,
true,
false,
false
]
},
"hash": "a0833b9899833166891c5de926f78632fae1123e736d728bf92cb2de004b6826"
"hash": "9b60fa8a1003015bc5a7cdbee9a4486b313d45347dfd9d4793d60e2760763ca3"
}
backend/.sqlx/query-6781ba76dfce321dca4634566496ea5d698ac09d8264e35dfaa4cd8edc9e8414.json → backend/.sqlx/query-9c50e3a136a8ee3ec56e083f26d3a960b89e02ec40b292f3b5198baf2a1d3dbf.json
Generated
+4 -4
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n id, \n created_at, \n trigger_kind AS \"trigger_kind: _\", \n payload AS \"payload!: _\", \n trigger_extra AS \"trigger_extra: _\"\n FROM \n capture\n WHERE \n id = $1 \n AND workspace_id = $2\n ",
"query": "\n SELECT \n id, \n created_at, \n trigger_kind AS \"trigger_kind: _\", \n main_args AS \"main_args!: _\", \n preprocessor_args AS \"preprocessor_args: _\"\n FROM \n capture\n WHERE \n id = $1 \n AND workspace_id = $2\n ",
"describe": {
"columns": [
{
@@ -38,12 +38,12 @@
},
{
"ordinal": 3,
"name": "payload!: _",
"name": "main_args!: _",
"type_info": "Jsonb"
},
{
"ordinal": 4,
"name": "trigger_extra: _",
"name": "preprocessor_args: _",
"type_info": "Jsonb"
}
],
@@ -61,5 +61,5 @@
true
]
},
"hash": "6781ba76dfce321dca4634566496ea5d698ac09d8264e35dfaa4cd8edc9e8414"
"hash": "9c50e3a136a8ee3ec56e083f26d3a960b89e02ec40b292f3b5198baf2a1d3dbf"
}
backend/.sqlx/query-9dec888d4b0666d1843fbbc4fb2475fd947f047a9965bc110d7338208b77783d.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT flow_version.id from flow\n INNER JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n WHERE flow.path = $1 and flow.workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "9dec888d4b0666d1843fbbc4fb2475fd947f047a9965bc110d7338208b77783d"
}
backend/.sqlx/query-a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b.json
Generated
+70
View File
@@ -0,0 +1,70 @@
{
"db_name": "PostgreSQL",
"query": "SELECT has_preprocessor, language as \"language: _\", content, schema as \"schema: _\" FROM script WHERE workspace_id = $1 AND hash = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "has_preprocessor",
"type_info": "Bool"
},
{
"ordinal": 1,
"name": "language: _",
"type_info": {
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java"
]
}
}
}
},
{
"ordinal": 2,
"name": "content",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "schema: _",
"type_info": "Json"
}
],
"parameters": {
"Left": [
"Text",
"Int8"
]
},
"nullable": [
true,
false,
false,
true
]
},
"hash": "a8bcae108af1eda6efe3a4b8c6f8807bc464a81c0883e68f5a69b89b94b0b34b"
}
backend/.sqlx/query-c794ff5e14429a1bc56f5e70cfd10e2160d1b7e123c7ab226dc77d1789f164f9.json
Generated
-47
View File
@@ -1,47 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT tag, dedicated_worker, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor, on_behalf_of_email, edited_by\n FROM flow \n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n WHERE flow.path = $1 and flow.workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "tag",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "dedicated_worker",
"type_info": "Bool"
},
{
"ordinal": 2,
"name": "has_preprocessor",
"type_info": "Bool"
},
{
"ordinal": 3,
"name": "on_behalf_of_email",
"type_info": "Text"
},
{
"ordinal": 4,
"name": "edited_by",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
true,
true,
null,
true,
false
]
},
"hash": "c794ff5e14429a1bc56f5e70cfd10e2160d1b7e123c7ab226dc77d1789f164f9"
}
backend/.sqlx/query-f9f7f31390f8ea1f4facd8e6a888886ea136f2327169bf93f126e2ef130d0946.json → backend/.sqlx/query-eac595e19e5c8e70f1514ef29dec35c7342ac9a814c73f6290e1d6ebd3a55423.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO \n capture (\n workspace_id, path, is_flow, trigger_kind, payload, trigger_extra, created_by\n )\n VALUES (\n $1, $2, $3, $4, $5, $6, $7\n )\n ",
"query": "\n INSERT INTO \n capture (\n workspace_id, path, is_flow, trigger_kind, main_args, preprocessor_args, created_by\n )\n VALUES (\n $1, $2, $3, $4, $5, $6, $7\n )\n ",
"describe": {
"columns": [],
"parameters": {
@@ -34,5 +34,5 @@
},
"nullable": []
},
"hash": "f9f7f31390f8ea1f4facd8e6a888886ea136f2327169bf93f126e2ef130d0946"
"hash": "eac595e19e5c8e70f1514ef29dec35c7342ac9a814c73f6290e1d6ebd3a55423"
}
backend/.sqlx/query-ec7836df5f9056ec70015800b7f4feaeb1b671120f5f8c98fca8c89c6587fc35.json
Generated
-118
View File
@@ -1,118 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "select tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, on_behalf_of_email, created_by from script where hash = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "tag",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "concurrency_key",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "concurrent_limit",
"type_info": "Int4"
},
{
"ordinal": 3,
"name": "concurrency_time_window_s",
"type_info": "Int4"
},
{
"ordinal": 4,
"name": "cache_ttl",
"type_info": "Int4"
},
{
"ordinal": 5,
"name": "language: ScriptLang",
"type_info": {
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible",
"csharp",
"oracledb",
"nu",
"java"
]
}
}
}
},
{
"ordinal": 6,
"name": "dedicated_worker",
"type_info": "Bool"
},
{
"ordinal": 7,
"name": "priority",
"type_info": "Int2"
},
{
"ordinal": 8,
"name": "delete_after_use",
"type_info": "Bool"
},
{
"ordinal": 9,
"name": "timeout",
"type_info": "Int4"
},
{
"ordinal": 10,
"name": "on_behalf_of_email",
"type_info": "Text"
},
{
"ordinal": 11,
"name": "created_by",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Int8",
"Text"
]
},
"nullable": [
true,
true,
true,
true,
true,
false,
true,
true,
true,
true,
true,
false
]
},
"hash": "ec7836df5f9056ec70015800b7f4feaeb1b671120f5f8c98fca8c89c6587fc35"
}
backend/.sqlx/query-f44f1a794ab217be05a9feb21e57b3409d9166a028701981cfc280aaab3c4436.json
Generated
-41
View File
@@ -1,41 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT tag, dedicated_worker, on_behalf_of_email, edited_by from flow WHERE path = $1 and workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "tag",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "dedicated_worker",
"type_info": "Bool"
},
{
"ordinal": 2,
"name": "on_behalf_of_email",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "edited_by",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
true,
true,
true,
false
]
},
"hash": "f44f1a794ab217be05a9feb21e57b3409d9166a028701981cfc280aaab3c4436"
}
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
19b5681d1985ee5214f768e3b818faf6ca7d20b2
4cc3cbd0f0536a2fa4814d98ec058f998044a70b
backend/migrations/20250414082127_update_capture_format.down.sql
+2
View File
@@ -0,0 +1,2 @@
ALTER TABLE capture RENAME COLUMN preprocessor_args to trigger_extra;
ALTER TABLE capture RENAME COLUMN main_args to payload;
backend/migrations/20250414082127_update_capture_format.up.sql
+2
View File
@@ -0,0 +1,2 @@
ALTER TABLE capture RENAME COLUMN trigger_extra to preprocessor_args;
ALTER TABLE capture RENAME COLUMN payload to main_args;
backend/migrations/20250506092215_runnable_version_notify.down.sql
+4
View File
@@ -0,0 +1,4 @@
-- Add down migration script here
DROP TRIGGER script_update_trigger ON script;
DROP TRIGGER flow_update_trigger ON flow_version;
DROP FUNCTION notify_runnable_version_change();
backend/migrations/20250506092215_runnable_version_notify.up.sql
+22
View File
@@ -0,0 +1,22 @@
-- Add up migration script here
CREATE OR REPLACE FUNCTION notify_runnable_version_change()
RETURNS TRIGGER AS $$
DECLARE
source_type TEXT;
BEGIN
source_type := TG_ARGV[0];
PERFORM pg_notify('notify_runnable_version_change', NEW.workspace_id || ':' || source_type || ':' || NEW.path);
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER script_update_trigger
AFTER UPDATE OF lock ON script
FOR EACH ROW
EXECUTE FUNCTION notify_runnable_version_change('script');
CREATE TRIGGER flow_update_trigger
AFTER INSERT ON flow_version
FOR EACH ROW
EXECUTE FUNCTION notify_runnable_version_change('flow');
backend/parsers/windmill-parser-ts/src/lib.rs
+1 -1
View File
@@ -506,7 +506,7 @@ fn one_of_label(members: &Vec<TsTypeElement>) -> Option<String> {
let Expr::Ident(Ident { sym, .. }) = &**key else {
return None;
};
if sym != "label" {
if sym != "label" && sym != "kind" {
return None;
}
backend/parsers/windmill-parser-wasm/src/lib.rs
+1 -1
View File
@@ -17,7 +17,7 @@ fn wrap_sig(r: anyhow::Result<MainArgSignature>) -> String {
#[cfg(feature = "ts-parser")]
#[wasm_bindgen]
pub fn parse_deno(code: &str, main_override: Option<String>, skip_params: Option<bool>) -> String {
pub fn parse_deno(code: &str, main_override: Option<String>) -> String {
wrap_sig(windmill_parser_ts::parse_deno_signature(
code,
false,
backend/src/main.rs
+24
View File
@@ -787,6 +787,29 @@ Windmill Community Edition {GIT_VERSION}
tracing::info!("Workspace premium change detected, invalidating workspace premium cache: {}", workspace_id);
windmill_common::workspaces::IS_PREMIUM_CACHE.remove(workspace_id);
},
"notify_runnable_version_change" => {
let payload = n.payload();
tracing::info!("Runnable version change detected: {}", payload);
match payload.split(':').collect::<Vec<&str>>().as_slice() {
[workspace_id, source_type, path] => {
let key = (workspace_id.to_string(), path.to_string());
match source_type {
&"script" => {
windmill_common::DEPLOYED_SCRIPT_HASH_CACHE.remove(&key);
}
&"flow" => {
windmill_common::FLOW_VERSION_CACHE.remove(&key);
},
_ => {
tracing::warn!("Unknown runnable version change payload: {}", payload);
}
}
},
_ => {
tracing::warn!("Unknown runnable version change payload: {}", payload);
}
}
},
"notify_global_setting_change" => {
tracing::info!("Global setting change detected: {}", n.payload());
match n.payload() {
@@ -1108,6 +1131,7 @@ async fn listen_pg(url: &str) -> Option<PgListener> {
"notify_global_setting_change",
"notify_webhook_change",
"notify_workspace_envs_change",
"notify_runnable_version_change",
];
#[cfg(feature = "cloud")]
channels.push("notify_workspace_premium_change");
backend/tests/worker.rs
+3
View File
@@ -4687,6 +4687,7 @@ mod job_payload {
path: "f/system/hello_with_nodes_flow".to_string(),
dedicated_worker: None,
apply_preprocessor: false,
version: 1443253234253454,
})
.run_until_complete(&db, port)
.await
@@ -4731,6 +4732,7 @@ mod job_payload {
path: "f/system/hello_with_preprocessor".to_string(),
dedicated_worker: None,
apply_preprocessor: true,
version: 1443253234253456,
})
.run_until_complete_with(db, port, |id| async move {
let job = sqlx::query!("SELECT preprocessed FROM v2_job WHERE id = $1", id)
@@ -4798,6 +4800,7 @@ mod job_payload {
path: "f/system/hello_with_nodes_flow".to_string(),
dedicated_worker: None,
apply_preprocessor: true,
version: 1443253234253454,
})
.run_until_complete(&db, port)
.await
backend/windmill-api/openapi.yaml
+4 -3
View File
@@ -16539,8 +16539,8 @@ components:
properties:
trigger_kind:
$ref: "#/components/schemas/CaptureTriggerKind"
payload: {}
trigger_extra: {}
main_args: {}
preprocessor_args: {}
id:
type: integer
created_at:
@@ -16548,7 +16548,8 @@ components:
format: date-time
required:
- trigger_kind
- payload
- main_args
- preprocessor_args
- id
- created_at
CaptureConfig:
backend/windmill-api/src/args.rs
+380 -235
View File
@@ -1,7 +1,5 @@
use std::collections::HashMap;
#[cfg(feature = "parquet")]
use crate::job_helpers_ee::get_workspace_s3_resource;
use axum::{
extract::{FromRequest, FromRequestParts, Multipart, Query, Request},
http::{HeaderMap, Uri},
@@ -9,139 +7,318 @@ use axum::{
};
use bytes::Bytes;
use http::{header::CONTENT_TYPE, request::Parts, StatusCode};
#[cfg(feature = "parquet")]
use object_store::{Attribute, Attributes};
use serde::Deserialize;
use serde::{Deserialize, Serialize};
use serde_json::value::RawValue;
use sqlx::types::JsonRawValue;
#[cfg(feature = "parquet")]
use windmill_common::s3_helpers::build_object_store_client;
use windmill_common::{error::Error, worker::to_raw_value, DB};
use windmill_queue::PushArgsOwned;
use windmill_queue::{PushArgsOwned, TriggerKind};
use crate::db::ApiAuthed;
#[cfg(feature = "parquet")]
use crate::job_helpers_ee::{get_random_file_name, upload_file_internal};
use crate::{
db::ApiAuthed,
trigger_helpers::{get_runnable_format, RunnableFormat, RunnableFormatVersion, RunnableId},
};
#[derive(Debug, Default)]
pub struct WebhookArgs {
pub args: PushArgsOwned,
pub multipart: Option<Multipart>,
pub wrap_body: Option<bool>,
#[derive(Debug)]
pub enum RawBody {
Json(String),
CEJson(String),
Text(String),
Xml(String),
UrlEncoded(Bytes),
Multipart(Multipart),
Empty,
}
impl WebhookArgs {
#[derive(Clone, Serialize)]
#[serde(untagged)]
pub enum Body {
HashMap(HashMap<String, Box<RawValue>>),
NoHashMap(Box<RawValue>),
}
#[derive(Clone, Default)]
pub struct WebhookArgsMetadata {
pub raw_string: Option<String>,
pub headers: HashMap<String, Box<RawValue>>,
pub method: http::Method,
pub query: HashMap<String, Box<RawValue>>,
pub query_wrap_body: bool,
pub query_use_raw: bool,
}
pub struct RawWebhookArgs {
pub body: RawBody,
pub metadata: WebhookArgsMetadata,
}
#[derive(Clone)]
pub struct WebhookArgs {
pub body: Body,
pub metadata: WebhookArgsMetadata,
}
// capture
//
impl RawWebhookArgs {
#[cfg(not(feature = "parquet"))]
pub async fn to_push_args_owned(
self,
pub async fn process_multipart(
_multipart: Multipart,
_authed: &ApiAuthed,
_db: &DB,
_w_id: &str,
) -> Result<PushArgsOwned, Error> {
if self.multipart.is_some() {
return Err(Error::BadRequest(format!(
"multipart/form-data requires the parquet feature"
)));
}
Ok(self.args)
) -> Result<HashMap<String, Box<RawValue>>, Error> {
return Err(Error::BadRequest(format!(
"multipart/form-data requires the parquet feature"
)));
}
#[cfg(feature = "parquet")]
pub async fn to_push_args_owned(
mut self,
async fn process_multipart(
mut multipart: Multipart,
authed: &ApiAuthed,
db: &DB,
w_id: &str,
) -> Result<HashMap<String, Box<RawValue>>, Error> {
use crate::job_helpers_ee::{
get_random_file_name, get_workspace_s3_resource, upload_file_internal,
};
use futures::TryStreamExt;
use object_store::{Attribute, Attributes};
use windmill_common::s3_helpers::build_object_store_client;
let (_, s3_resource) = get_workspace_s3_resource(authed, db, None, "", w_id, None).await?;
if let Some(s3_resource) = s3_resource {
let s3_client = build_object_store_client(&s3_resource).await?;
let mut body = HashMap::new();
let mut files = HashMap::new();
while let Some(field) = multipart.next_field().await.map_err(|e| {
Error::BadRequest(format!("Error reading multipart field: {}", e.body_text()))
})? {
if let Some(name) = field.name().map(|x| x.to_string()) {
if let Some(content_type) = field.content_type() {
let ext = field
.file_name()
.map(|x| x.split('.').last())
.flatten()
.map(|x| x.to_string());
let file_key = get_random_file_name(ext);
let options = Attributes::from_iter(vec![
(Attribute::ContentType, content_type.to_string()),
(
Attribute::ContentDisposition,
if let Some(filename) = field.file_name() {
format!("inline; filename=\"{}\"", filename)
} else {
"inline".to_string()
},
),
])
.into();
let bytes_stream = field
.into_stream()
.map_err(|err| std::io::Error::new(std::io::ErrorKind::Other, err));
upload_file_internal(s3_client.clone(), &file_key, bytes_stream, options)
.await?;
files.entry(name).or_insert(vec![]).push(serde_json::json!({
"s3": &file_key
}));
} else {
body.insert(name, to_raw_value(&field.text().await.unwrap_or_default()));
}
}
}
for (k, v) in files {
body.insert(k, to_raw_value(&v));
}
Ok(body)
} else {
Err(Error::BadRequest(format!(
"You need to connect your workspace to an S3 bucket to use multipart/form-data"
)))
}
}
pub async fn process_args(
self,
authed: &ApiAuthed,
db: &DB,
w_id: &str,
force_use_raw: Option<bool>,
) -> Result<WebhookArgs, Error> {
let use_raw = force_use_raw.unwrap_or(self.metadata.query_use_raw);
match self.body {
RawBody::Multipart(multipart) => {
let body = Self::process_multipart(multipart, authed, db, w_id).await?;
Ok(WebhookArgs { body: Body::HashMap(body), metadata: self.metadata })
}
RawBody::Empty => {
let mut metadata = self.metadata;
if use_raw {
metadata.raw_string = Some("".to_string());
}
Ok(WebhookArgs { body: Body::HashMap(HashMap::new()), metadata })
}
RawBody::Text(s) | RawBody::Xml(s) => Ok(WebhookArgs {
body: Body::HashMap(HashMap::new()),
metadata: WebhookArgsMetadata { raw_string: Some(s), ..self.metadata },
}),
RawBody::UrlEncoded(bytes) => {
let mut metadata = self.metadata;
if use_raw {
let raw_string = String::from_utf8(bytes.to_vec())
.map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)))?;
metadata.raw_string = Some(raw_string);
}
let payload: HashMap<String, Option<String>> = serde_urlencoded::from_bytes(&bytes)
.map_err(|e| Error::BadRequest(format!("invalid urlencoded data: {}", e)))?;
let payload = payload
.into_iter()
.map(|(k, v)| (k, to_raw_value(&v)))
.collect::<HashMap<_, _>>();
Ok(WebhookArgs { body: Body::HashMap(payload), metadata })
}
RawBody::Json(s) => WebhookArgs::from_json(self.metadata, use_raw, s).await,
RawBody::CEJson(s) => WebhookArgs::from_ce_json(self.metadata, use_raw, s).await,
}
}
pub async fn to_main_args(
self,
authed: &ApiAuthed,
db: &DB,
w_id: &str,
) -> Result<PushArgsOwned, Error> {
use futures::TryStreamExt;
let args = self.process_args(authed, db, w_id, None).await?;
args.to_main_args()
}
if let Some(mut multipart) = self.multipart {
{
let (_, s3_resource) =
get_workspace_s3_resource(authed, db, None, "", w_id, None).await?;
pub async fn to_args_from_runnable(
self,
authed: &ApiAuthed,
db: &DB,
w_id: &str,
runnable_id: RunnableId,
skip_preprocessor: Option<bool>,
) -> Result<PushArgsOwned, Error> {
let args = self.process_args(authed, db, w_id, None).await?;
args.to_args_from_runnable(db, w_id, runnable_id, skip_preprocessor)
.await
}
}
if let Some(s3_resource) = s3_resource {
let s3_client = build_object_store_client(&s3_resource).await?;
#[derive(Serialize)]
struct WebhookPreprocessorEvent {
kind: String,
body: Box<RawValue>,
raw_string: Option<String>,
headers: HashMap<String, Box<RawValue>>,
query: HashMap<String, Box<RawValue>>,
}
let mut body = HashMap::new();
let mut files = HashMap::new();
impl WebhookArgs {
pub fn to_main_args(self) -> Result<PushArgsOwned, Error> {
self.to_args_from_format(RunnableFormat {
has_preprocessor: false,
version: RunnableFormatVersion::V2,
})
}
while let Some(field) = multipart.next_field().await.map_err(|e| {
Error::BadRequest(format!(
"Error reading multipart field: {}",
e.body_text()
))
})? {
if let Some(name) = field.name().map(|x| x.to_string()) {
if let Some(content_type) = field.content_type() {
let ext = field
.file_name()
.map(|x| x.split('.').last())
.flatten()
.map(|x| x.to_string());
pub async fn to_args_from_runnable(
self,
db: &DB,
w_id: &str,
runnable_id: RunnableId,
skip_preprocessor: Option<bool>,
) -> Result<PushArgsOwned, Error> {
if skip_preprocessor.unwrap_or(false) {
self.to_main_args()
} else {
let runnable_format =
get_runnable_format(runnable_id, w_id, db, &TriggerKind::Webhook).await?;
let file_key = get_random_file_name(ext);
self.to_args_from_format(runnable_format)
}
}
let options = Attributes::from_iter(vec![
(Attribute::ContentType, content_type.to_string()),
(
Attribute::ContentDisposition,
if let Some(filename) = field.file_name() {
format!("inline; filename=\"{}\"", filename)
} else {
"inline".to_string()
},
),
])
.into();
pub fn to_args_from_format(
self,
runnable_format: RunnableFormat,
) -> Result<PushArgsOwned, Error> {
match runnable_format {
RunnableFormat { has_preprocessor: true, version: RunnableFormatVersion::V2 } => {
let mut args = HashMap::new();
let bytes_stream = field.into_stream().map_err(|err| {
std::io::Error::new(std::io::ErrorKind::Other, err)
});
args.insert(
"event".to_string(),
to_raw_value(&WebhookPreprocessorEvent {
kind: "webhook".to_string(),
body: to_raw_value(&self.body),
raw_string: self.metadata.raw_string,
headers: self.metadata.headers,
query: self.metadata.query,
}),
);
upload_file_internal(
s3_client.clone(),
&file_key,
bytes_stream,
options,
)
.await?;
Ok(PushArgsOwned { args, extra: None })
}
RunnableFormat { has_preprocessor, .. } => {
let mut extra = HashMap::new();
files.entry(name).or_insert(vec![]).push(serde_json::json!({
"s3": &file_key
}));
} else {
body.insert(
name,
to_raw_value(&field.text().await.unwrap_or_default()),
);
}
let WebhookArgsMetadata { query, query_wrap_body, headers, raw_string, .. } =
self.metadata;
for (k, v) in headers {
extra.insert(k, v);
}
for (k, v) in query {
extra.insert(k, v);
}
if let Some(raw_string) = raw_string {
extra.insert("raw_string".to_string(), to_raw_value(&raw_string));
}
if has_preprocessor {
// if has preprocessor, it has to be v1
extra.insert(
"wm_trigger".to_string(),
to_raw_value(&serde_json::json!({
"kind": "webhook",
})),
);
}
let extra = if extra.is_empty() { None } else { Some(extra) };
match self.body {
Body::HashMap(mut body) => {
if query_wrap_body {
body = HashMap::from([("body".to_string(), to_raw_value(&body))]);
}
Ok(PushArgsOwned { args: body, extra })
}
for (k, v) in files {
body.insert(k, to_raw_value(&v));
Body::NoHashMap(args) => {
let mut hm = HashMap::new();
hm.insert("body".to_string(), args);
Ok(PushArgsOwned { args: hm, extra })
}
if self.wrap_body.unwrap_or(false) {
self.args
.args
.insert("body".to_string(), to_raw_value(&body));
} else {
self.args.args.extend(body);
}
return Ok(self.args);
}
}
return Err(Error::BadRequest(format!(
"You need to connect your workspace to an S3 bucket to use multipart/form-data"
)));
}
Ok(self.args)
}
}
@@ -166,26 +343,36 @@ async fn req_to_string<S: Send + Sync>(
pub async fn try_from_request_body<S>(
request: Request,
_state: &S,
use_raw: Option<bool>,
wrap_body: Option<bool>,
) -> Result<WebhookArgs, Response>
is_http_trigger: bool,
) -> Result<RawWebhookArgs, Response>
where
S: Send + Sync,
{
let (content_type, mut extra, use_raw, wrap_body) = {
let (content_type, metadata) = {
let headers_map = request.headers();
let content_type_header = headers_map.get(CONTENT_TYPE);
let content_type = content_type_header.and_then(|value| value.to_str().ok());
let uri = request.uri();
let query = Query::<RequestQuery>::try_from_uri(uri).unwrap().0;
let mut extra = build_extra(&headers_map, query.include_header);
let request_query = Query::<RequestQuery>::try_from_uri(uri).unwrap().0;
let headers = build_headers(&headers_map, request_query.include_header, is_http_trigger);
let query_decode = DecodeQueries::from_uri(uri);
let mut query = HashMap::new();
if let Some(DecodeQueries(queries)) = query_decode {
extra.extend(queries);
query.extend(queries);
}
let raw = query.raw.unwrap_or(use_raw.unwrap_or(false));
let wrap_body = query.wrap_body.unwrap_or(wrap_body.unwrap_or(false));
(content_type, extra, raw, wrap_body)
let raw = !is_http_trigger && request_query.raw.unwrap_or(false);
let wrap_body = !is_http_trigger && request_query.wrap_body.unwrap_or(false);
(
content_type,
WebhookArgsMetadata {
headers,
query,
method: request.method().clone(),
raw_string: None,
query_wrap_body: wrap_body,
query_use_raw: raw,
},
)
};
let no_content_type = content_type.is_none();
@@ -194,33 +381,19 @@ where
.await
.map_err(IntoResponse::into_response)?;
if no_content_type && bytes.is_empty() {
if use_raw {
extra.insert("raw_string".to_string(), to_raw_value(&"".to_string()));
}
let mut args = HashMap::new();
if wrap_body {
args.insert("body".to_string(), to_raw_value(&serde_json::json!({})));
}
return Ok(WebhookArgs {
args: PushArgsOwned { extra: Some(extra), args: args },
..Default::default()
});
Ok(RawWebhookArgs { body: RawBody::Empty, metadata })
} else {
let str = String::from_utf8(bytes.to_vec())
.map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?;
Ok(RawWebhookArgs { body: RawBody::Json(str), metadata })
}
let str = String::from_utf8(bytes.to_vec())
.map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?;
PushArgsOwned::from_json(extra, use_raw, wrap_body, str)
.await
.map(|args| WebhookArgs { args, ..Default::default() })
} else if content_type
.unwrap()
.starts_with("application/cloudevents+json")
{
let str = req_to_string(request, _state).await?;
PushArgsOwned::from_ce_json(extra, use_raw, str)
.await
.map(|args| WebhookArgs { args, ..Default::default() })
Ok(RawWebhookArgs { body: RawBody::CEJson(str), metadata })
} else if content_type
.unwrap()
.starts_with("application/cloudevents-batch+json")
@@ -231,11 +404,7 @@ where
)
} else if content_type.unwrap().starts_with("text/plain") {
let str = req_to_string(request, _state).await?;
extra.insert("raw_string".to_string(), to_raw_value(&str));
Ok(WebhookArgs {
args: PushArgsOwned { extra: Some(extra), args: HashMap::new() },
..Default::default()
})
Ok(RawWebhookArgs { body: RawBody::Text(str), metadata })
} else if content_type
.unwrap()
.starts_with("application/x-www-form-urlencoded")
@@ -244,58 +413,32 @@ where
.await
.map_err(IntoResponse::into_response)?;
if use_raw {
let raw_string = String::from_utf8(bytes.to_vec())
.map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?;
extra.insert("raw_string".to_string(), to_raw_value(&raw_string));
}
let payload: HashMap<String, Option<String>> = serde_urlencoded::from_bytes(&bytes)
.map_err(|e| {
Error::BadRequest(format!("invalid urlencoded data: {}", e)).into_response()
})?;
let payload = payload
.into_iter()
.map(|(k, v)| (k, to_raw_value(&v)))
.collect::<HashMap<_, _>>();
return Ok(WebhookArgs {
args: PushArgsOwned { extra: Some(extra), args: payload },
..Default::default()
});
Ok(RawWebhookArgs { body: RawBody::UrlEncoded(bytes), metadata })
} else if content_type.unwrap().starts_with("application/xml")
|| content_type.unwrap().starts_with("text/xml")
{
let str = req_to_string(request, _state).await?;
extra.insert("raw_string".to_string(), to_raw_value(&str));
Ok(WebhookArgs {
args: PushArgsOwned { extra: Some(extra), args: HashMap::new() },
..Default::default()
})
Ok(RawWebhookArgs { body: RawBody::Xml(str), metadata })
} else if content_type.unwrap().starts_with("multipart/form-data") {
let multipart = Multipart::from_request(request, _state)
.await
.map_err(IntoResponse::into_response)?;
Ok(WebhookArgs {
args: PushArgsOwned { extra: Some(extra), args: HashMap::new() },
multipart: Some(multipart),
wrap_body: Some(wrap_body),
})
Ok(RawWebhookArgs { body: RawBody::Multipart(multipart), metadata })
} else {
Err(StatusCode::UNSUPPORTED_MEDIA_TYPE.into_response())
}
}
#[axum::async_trait]
impl<S> FromRequest<S, axum::body::Body> for WebhookArgs
impl<S> FromRequest<S, axum::body::Body> for RawWebhookArgs
where
S: Send + Sync,
{
type Rejection = Response;
async fn from_request(request: Request, _state: &S) -> Result<Self, Self::Rejection> {
let args = try_from_request_body(request, _state, None, None).await?;
let args = try_from_request_body(request, _state, false).await?;
Ok(args)
}
@@ -309,27 +452,37 @@ lazy_static::lazy_static! {
.collect()).unwrap_or_default();
}
pub fn build_extra(
pub fn build_headers(
headers: &HeaderMap,
include_header: Option<String>,
is_http_trigger: bool,
) -> HashMap<String, Box<RawValue>> {
let mut args = HashMap::new();
let whitelist = include_header
.map(|s| s.split(",").map(|s| s.to_string()).collect::<Vec<_>>())
.unwrap_or_default();
let mut selected_headers = HashMap::new();
whitelist
.iter()
.chain(INCLUDE_HEADERS.iter())
.for_each(|h| {
if let Some(v) = headers.get(h) {
args.insert(
h.to_string().to_lowercase().replace('-', "_"),
to_raw_value(&v.to_str().unwrap().to_string()),
);
}
});
args
if is_http_trigger {
for (k, v) in headers.iter() {
selected_headers.insert(
k.to_string(),
to_raw_value(&v.to_str().unwrap_or("").to_string()),
);
}
} else {
let whitelist = include_header
.map(|s| s.split(",").map(|s| s.to_string()).collect::<Vec<_>>())
.unwrap_or_default();
whitelist
.iter()
.chain(INCLUDE_HEADERS.iter())
.for_each(|h| {
if let Some(v) = headers.get(h) {
selected_headers.insert(
h.to_string().to_lowercase().replace('-', "_"),
to_raw_value(&v.to_str().unwrap_or("").to_string()),
);
}
});
}
selected_headers
}
#[derive(Deserialize)]
@@ -414,69 +567,50 @@ fn restructure_cloudevents_metadata(
}
}
trait PushArgsOwnedExt: Sized {
impl WebhookArgs {
async fn from_json(
extra: HashMap<String, Box<RawValue>>,
use_raw: bool,
force_wrap_body: bool,
str: String,
) -> Result<Self, Response>;
async fn from_ce_json(
extra: HashMap<String, Box<RawValue>>,
mut metadata: WebhookArgsMetadata,
use_raw: bool,
str: String,
) -> Result<Self, Response>;
}
impl PushArgsOwnedExt for PushArgsOwned {
async fn from_json(
mut extra: HashMap<String, Box<RawValue>>,
use_raw: bool,
force_wrap_body: bool,
str: String,
) -> Result<Self, Response> {
) -> Result<Self, Error> {
if use_raw {
extra.insert("raw_string".to_string(), to_raw_value(&str));
metadata.raw_string = Some(str.clone());
}
let wrap_body = force_wrap_body || str.len() > 0 && str.chars().next().unwrap() != '{';
let no_hashmap = str.len() > 0 && str.chars().next().unwrap() != '{';
if wrap_body {
if no_hashmap {
let args = serde_json::from_str::<Option<Box<RawValue>>>(&str)
.map_err(|e| Error::BadRequest(format!("invalid json: {}", e)).into_response())?
.map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))?
.unwrap_or_else(|| to_raw_value(&serde_json::Value::Null));
let mut hm = HashMap::new();
hm.insert("body".to_string(), args);
Ok(PushArgsOwned { extra: Some(extra), args: hm })
Ok(Self { body: Body::NoHashMap(args), metadata })
} else {
let hm = serde_json::from_str::<Option<HashMap<String, Box<JsonRawValue>>>>(&str)
.map_err(|e| Error::BadRequest(format!("invalid json: {}", e)).into_response())?
.map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))?
.unwrap_or_else(HashMap::new);
Ok(PushArgsOwned { extra: Some(extra), args: hm })
Ok(Self { body: Body::HashMap(hm), metadata })
}
}
async fn from_ce_json(
mut extra: HashMap<String, Box<RawValue>>,
mut metadata: WebhookArgsMetadata,
use_raw: bool,
str: String,
) -> Result<Self, Response> {
) -> Result<Self, Error> {
if use_raw {
extra.insert("raw_string".to_string(), to_raw_value(&str));
metadata.raw_string = Some(str.clone());
}
let hm = serde_json::from_str::<HashMap<String, Box<RawValue>>>(&str).map_err(|e| {
Error::BadRequest(format!("invalid cloudevents+json: {}", e)).into_response()
})?;
let hm = restructure_cloudevents_metadata(hm).map_err(|e| e.into_response())?;
Ok(PushArgsOwned { extra: Some(extra), args: hm })
let hm = serde_json::from_str::<HashMap<String, Box<RawValue>>>(&str)
.map_err(|e| Error::BadRequest(format!("invalid cloudevents+json: {}", e)))?;
let hm = restructure_cloudevents_metadata(hm)?;
Ok(Self { body: Body::HashMap(hm), metadata })
}
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use super::*;
@@ -514,24 +648,35 @@ mod tests {
"data" : 1.5
}
"#;
let extra = HashMap::new();
let metadata = WebhookArgsMetadata::default();
let a1 = PushArgsOwned::from_ce_json(extra.clone(), false, r1.to_string())
let a1 = WebhookArgs::from_ce_json(metadata.clone(), false, r1.to_string())
.await
.expect("Failed to parse the cloudevent");
let a2 = PushArgsOwned::from_ce_json(extra.clone(), false, r2.to_string())
let a2 = WebhookArgs::from_ce_json(metadata.clone(), false, r2.to_string())
.await
.expect("Failed to parse the cloudevent");
a1.args.get("WEBHOOK__METADATA__").expect(
"CloudEvents should generate a neighboring `webhook-metadata` field in PushArgs",
);
assert_eq!(
a2.args
.get("body")
.expect("Cloud events with a data field with no wrapping curly brackets should be inside of a `body` field in PushArgs")
.to_string(),
"1.5"
);
match a1.body {
Body::HashMap(body) => {
body.get("WEBHOOK__METADATA__").expect(
"CloudEvents should generate a neighboring `webhook-metadata` field in PushArgs",
);
}
_ => panic!("Expected a HashMap"),
}
match a2.body {
Body::HashMap(body) => {
assert_eq!(
body
.get("body")
.expect("Cloud events with a data field with no wrapping curly brackets should be inside of a `body` field in PushArgs")
.to_string(),
"1.5"
);
}
_ => panic!("Expected a HashMap"),
}
}
}
backend/windmill-api/src/capture.rs
+55 -64
View File
@@ -8,10 +8,7 @@
#[cfg(feature = "http_trigger")]
use {
crate::{
args::try_from_request_body,
http_triggers::{build_http_trigger_extra, HttpMethod},
},
crate::http_trigger_args::{HttpMethod, RawHttpTriggerArgs},
axum::response::{IntoResponse, Response},
std::collections::HashMap,
};
@@ -56,8 +53,9 @@ use {
};
use crate::{
args::WebhookArgs,
args::RawWebhookArgs,
db::{ApiAuthed, DB},
trigger_helpers::{RunnableFormat, RunnableFormatVersion},
users::fetch_api_authed,
utils::RunnableKind,
};
@@ -496,8 +494,8 @@ struct Capture {
id: i64,
created_at: chrono::DateTime<chrono::Utc>,
trigger_kind: TriggerKind,
payload: SqlxJson<Box<serde_json::value::RawValue>>,
trigger_extra: Option<SqlxJson<Box<serde_json::value::RawValue>>>,
main_args: SqlxJson<Box<serde_json::value::RawValue>>,
preprocessor_args: Option<SqlxJson<Box<serde_json::value::RawValue>>>,
}
#[derive(Deserialize)]
@@ -525,10 +523,13 @@ async fn list_captures(
created_at,
trigger_kind AS "trigger_kind: _",
CASE
WHEN pg_column_size(payload) < 40000 THEN payload
WHEN pg_column_size(main_args) < 40000 THEN main_args
ELSE '"WINDMILL_TOO_BIG"'::jsonb
END AS "payload!: _",
trigger_extra AS "trigger_extra: _"
END AS "main_args!: _",
CASE
WHEN pg_column_size(preprocessor_args) < 40000 THEN preprocessor_args
ELSE '"WINDMILL_TOO_BIG"'::jsonb
END AS "preprocessor_args: _"
FROM
capture
WHERE
@@ -570,8 +571,8 @@ async fn get_capture(
id,
created_at,
trigger_kind AS "trigger_kind: _",
payload AS "payload!: _",
trigger_extra AS "trigger_extra: _"
main_args AS "main_args!: _",
preprocessor_args AS "preprocessor_args: _"
FROM
capture
WHERE
@@ -820,15 +821,15 @@ pub async fn insert_capture_payload(
path: &str,
is_flow: bool,
trigger_kind: &TriggerKind,
payload: PushArgsOwned,
trigger_extra: Option<Box<RawValue>>,
main_args: PushArgsOwned,
preprocessor_args: PushArgsOwned,
owner: &str,
) -> Result<()> {
sqlx::query!(
r#"
INSERT INTO
capture (
workspace_id, path, is_flow, trigger_kind, payload, trigger_extra, created_by
workspace_id, path, is_flow, trigger_kind, main_args, preprocessor_args, created_by
)
VALUES (
$1, $2, $3, $4, $5, $6, $7
@@ -838,11 +839,9 @@ pub async fn insert_capture_payload(
path,
is_flow,
trigger_kind as &TriggerKind,
SqlxJson(to_raw_value(&PushArgs {
args: &payload.args,
extra: payload.extra
})) as SqlxJson<Box<RawValue>>,
trigger_extra.map(SqlxJson) as Option<SqlxJson<Box<RawValue>>>,
SqlxJson(PushArgs { args: &main_args.args, extra: main_args.extra }) as SqlxJson<PushArgs>,
SqlxJson(PushArgs { args: &preprocessor_args.args, extra: preprocessor_args.extra })
as SqlxJson<PushArgs>,
owner,
)
.execute(db)
@@ -856,7 +855,7 @@ pub async fn insert_capture_payload(
async fn webhook_payload(
Extension(db): Extension<DB>,
Path((w_id, runnable_kind, path)): Path<(String, RunnableKind, StripPath)>,
args: WebhookArgs,
args: RawWebhookArgs,
) -> Result<StatusCode> {
let (owner, email) = get_active_capture_owner_and_email(
&db,
@@ -868,7 +867,15 @@ async fn webhook_payload(
.await?;
let authed = fetch_api_authed(owner.clone(), email, &w_id, &db, None).await?;
let args = args.to_push_args_owned(&authed, &db, &w_id).await?;
let args = args.process_args(&authed, &db, &w_id, None).await?;
let preprocessor_args = args.clone().to_args_from_format(RunnableFormat {
has_preprocessor: true,
version: RunnableFormatVersion::V2,
})?;
let main_args = args.to_main_args()?;
insert_capture_payload(
&db,
@@ -876,12 +883,8 @@ async fn webhook_payload(
&path.to_path(),
matches!(runnable_kind, RunnableKind::Flow),
&TriggerKind::Webhook,
args,
Some(to_raw_value(&serde_json::json!({
"wm_trigger": {
"kind": "webhook",
}
}))),
main_args,
preprocessor_args,
&owner,
)
.await?;
@@ -897,6 +900,8 @@ async fn gcp_payload(
headers: HeaderMap,
request: Request,
) -> Result<StatusCode> {
use crate::{gcp_triggers_ee::GcpTrigger, trigger_helpers::TriggerJobArgs};
let is_flow = matches!(runnable_kind, RunnableKind::Flow);
let (gcp_trigger_config, owner, email): (GcpTriggerConfig, _, _) =
get_capture_trigger_config_and_owner(&db, &w_id, &path, is_flow, &TriggerKind::Gcp).await?;
@@ -918,9 +923,9 @@ async fn gcp_payload(
)
.await?;
let (args, extra) = process_google_push_request(headers, request).await?;
let (payload, gcp) = process_google_push_request(headers, request).await?;
let payload = PushArgsOwned { args, extra: None };
let (main_args, preprocessor_args) = GcpTrigger::build_capture_payloads(payload, gcp);
let _ = insert_capture_payload(
&db,
@@ -928,8 +933,8 @@ async fn gcp_payload(
&path,
is_flow,
&TriggerKind::Gcp,
payload,
Some(to_raw_value(&extra)),
main_args,
preprocessor_args,
&owner,
)
.await?;
@@ -941,10 +946,7 @@ async fn gcp_payload(
async fn http_payload(
Extension(db): Extension<DB>,
Path((w_id, runnable_kind, path, route_path)): Path<(String, RunnableKind, String, StripPath)>,
Query(query): Query<HashMap<String, String>>,
method: http::Method,
headers: HeaderMap,
request: Request,
args: RawHttpTriggerArgs,
) -> std::result::Result<StatusCode, Response> {
let path = path.replace(".", "/");
let is_flow = matches!(runnable_kind, RunnableKind::Flow);
@@ -954,20 +956,17 @@ async fn http_payload(
.await
.map_err(|e| e.into_response())?;
let args = try_from_request_body(
request,
&(),
http_trigger_config.raw_string,
http_trigger_config.wrap_body,
)
.await
.map_err(|e| e.into_response())?;
let authed = fetch_api_authed(owner.clone(), email, &w_id, &db, None)
.await
.map_err(|e| e.into_response())?;
let mut args = args
.to_push_args_owned(&authed, &db, &w_id)
let args = args
.process_args(
&authed,
&db,
&w_id,
http_trigger_config.raw_string.unwrap_or(false),
)
.await
.map_err(|e| e.into_response())?;
@@ -985,31 +984,23 @@ async fn http_payload(
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect();
let extra = args.extra.get_or_insert_with(HashMap::new);
let preprocessor_args = args
.clone()
.to_v2_preprocessor_args(&http_trigger_config.route_path, &route_path, &params)
.map_err(|e| e.into_response())?;
extra.insert(
"wm_trigger".to_string(),
build_http_trigger_extra(
&http_trigger_config.route_path,
route_path,
&method,
&params,
&query,
&headers,
)
.await,
);
let main_args = args
.to_main_args(http_trigger_config.wrap_body.unwrap_or(false))
.map_err(|e| e.into_response())?;
let extra = Some(to_raw_value(&extra));
args.extra = None;
insert_capture_payload(
&db,
&w_id,
&path,
is_flow,
&TriggerKind::Http,
args,
extra,
main_args,
preprocessor_args,
&owner,
)
.await
backend/windmill-api/src/gcp_triggers_ee.rs
+15 -8
View File
@@ -1,4 +1,5 @@
use crate::db::{ApiAuthed, DB};
use crate::trigger_helpers::TriggerJobArgs;
use axum::{extract::Request, Router};
use http::HeaderMap;
use serde::{Deserialize, Serialize};
@@ -7,10 +8,12 @@ use sqlx::prelude::FromRow;
use sqlx::types::Json as SqlxJson;
use std::collections::HashMap;
use windmill_common::db::UserDB;
use windmill_common::worker::to_raw_value;
use windmill_common::{
error::{Error as WindmillError, Result as WindmillResult},
utils::empty_string_as_none,
};
use windmill_queue::TriggerKind;
#[derive(sqlx::Type, Debug, Deserialize, Serialize)]
#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))]
@@ -90,14 +93,8 @@ pub async fn manage_google_subscription(
pub async fn process_google_push_request(
_headers: HeaderMap,
_request: Request,
) -> Result<
(
HashMap<String, Box<RawValue>>,
Option<HashMap<String, Box<RawValue>>>,
),
WindmillError,
> {
Ok((HashMap::new(), None))
) -> Result<(String, HashMap<String, Box<RawValue>>), WindmillError> {
Ok((String::new(), HashMap::new()))
}
pub async fn validate_jwt_token(
@@ -137,3 +134,13 @@ pub struct GcpTrigger {
pub last_server_ping: Option<chrono::DateTime<chrono::Utc>>,
pub enabled: bool,
}
impl TriggerJobArgs<String> for GcpTrigger {
fn v1_payload_fn(payload: String) -> HashMap<String, Box<RawValue>> {
HashMap::from([("payload".to_string(), to_raw_value(&payload))])
}
fn trigger_kind() -> TriggerKind {
TriggerKind::Gcp
}
}
backend/windmill-api/src/http_trigger_args.rs
+204
View File
@@ -0,0 +1,204 @@
use std::collections::HashMap;
use axum::{
extract::{FromRequest, Request},
response::Response,
};
use serde::{Deserialize, Serialize};
use serde_json::value::RawValue;
use windmill_common::{error::Error, worker::to_raw_value, DB};
use windmill_queue::PushArgsOwned;
use crate::{
args::{try_from_request_body, Body, RawWebhookArgs, WebhookArgs, WebhookArgsMetadata},
db::ApiAuthed,
trigger_helpers::{RunnableFormat, RunnableFormatVersion},
};
pub struct RawHttpTriggerArgs(pub RawWebhookArgs);
#[derive(Serialize, Deserialize, sqlx::Type, Debug)]
#[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")]
#[serde(rename_all = "lowercase")]
pub enum HttpMethod {
Get,
Post,
Put,
Delete,
Patch,
}
impl TryFrom<&http::Method> for HttpMethod {
type Error = Error;
fn try_from(method: &http::Method) -> Result<Self, Self::Error> {
match method {
&http::Method::GET => Ok(HttpMethod::Get),
&http::Method::POST => Ok(HttpMethod::Post),
&http::Method::PUT => Ok(HttpMethod::Put),
&http::Method::DELETE => Ok(HttpMethod::Delete),
&http::Method::PATCH => Ok(HttpMethod::Patch),
_ => Err(Error::BadRequest("Invalid HTTP method".to_string())),
}
}
}
#[axum::async_trait]
impl<S> FromRequest<S, axum::body::Body> for RawHttpTriggerArgs
where
S: Send + Sync,
{
type Rejection = Response;
async fn from_request(request: Request, _state: &S) -> Result<Self, Self::Rejection> {
let args = try_from_request_body(request, _state, true).await?;
Ok(Self(args))
}
}
#[derive(Clone)]
pub struct HttpTriggerArgs(pub WebhookArgs);
impl RawHttpTriggerArgs {
pub async fn process_args(
self,
authed: &ApiAuthed,
db: &DB,
w_id: &str,
use_raw: bool,
) -> Result<HttpTriggerArgs, Error> {
if self.0.metadata.query_use_raw || self.0.metadata.query_wrap_body {
return Err(Error::BadRequest(
"Specifying use raw or wrap body with query args is not supported anymore on http routes, please set it in the trigger config".to_string(),
)
.into());
}
let args = self.0.process_args(authed, db, w_id, Some(use_raw)).await?;
Ok(HttpTriggerArgs(args))
}
}
#[derive(Serialize)]
struct HttpTriggerPreprocessorEvent<'a> {
kind: String,
route: &'a str,
path: &'a str,
body: Box<RawValue>,
raw_string: Option<String>,
params: &'a HashMap<String, String>,
headers: HashMap<String, Box<RawValue>>,
query: HashMap<String, Box<RawValue>>,
method: HttpMethod,
}
#[derive(Serialize)]
struct HttpTriggerWmTrigger<'a> {
route: &'a str,
path: &'a str,
params: &'a HashMap<String, String>,
query: &'a HashMap<String, Box<RawValue>>,
headers: &'a HashMap<String, Box<RawValue>>,
method: HttpMethod,
}
impl HttpTriggerArgs {
pub fn to_main_args(self, wrap_body: bool) -> Result<PushArgsOwned, Error> {
let mut extra = HashMap::new();
let WebhookArgsMetadata { raw_string, .. } = self.0.metadata;
if let Some(raw_string) = raw_string {
extra.insert("raw_string".to_string(), to_raw_value(&raw_string));
}
let extra = if extra.is_empty() { None } else { Some(extra) };
match self.0.body {
Body::HashMap(mut body) => {
if wrap_body {
body = HashMap::from([("body".to_string(), to_raw_value(&body))]);
}
Ok(PushArgsOwned { args: body, extra })
}
Body::NoHashMap(args) => {
let mut hm = HashMap::new();
hm.insert("body".to_string(), args);
Ok(PushArgsOwned { args: hm, extra })
}
}
}
pub fn to_args_from_format(
self,
route_path: &str,
called_path: &str,
params: &HashMap<String, String>,
format: RunnableFormat,
wrap_body: bool,
) -> Result<PushArgsOwned, Error> {
match format {
RunnableFormat { has_preprocessor: true, version: RunnableFormatVersion::V2 } => {
// we don't care about wrap_body in v2
self.to_v2_preprocessor_args(route_path, called_path, params)
}
RunnableFormat { has_preprocessor: true, version: RunnableFormatVersion::V1 } => {
self.to_v1_preprocessor_args(route_path, called_path, params, wrap_body)
}
RunnableFormat { has_preprocessor: false, .. } => self.to_main_args(wrap_body),
}
}
fn to_v1_preprocessor_args(
self,
route_path: &str,
called_path: &str,
params: &HashMap<String, String>,
wrap_body: bool,
) -> Result<PushArgsOwned, Error> {
let mut extra = HashMap::new();
extra.insert(
"wm_trigger".to_string(),
to_raw_value(&HttpTriggerWmTrigger {
route: route_path,
path: called_path,
method: (&self.0.metadata.method).try_into()?,
params,
query: &self.0.metadata.query,
headers: &self.0.metadata.headers,
}),
);
let mut args = self.to_main_args(wrap_body)?;
args.extra.get_or_insert_default().extend(extra);
Ok(args)
}
pub fn to_v2_preprocessor_args(
self,
route_path: &str,
called_path: &str,
params: &HashMap<String, String>,
) -> Result<PushArgsOwned, Error> {
let mut args = HashMap::new();
args.insert(
"event".to_string(),
to_raw_value(&HttpTriggerPreprocessorEvent {
kind: "http".to_string(),
body: to_raw_value(&self.0.body),
raw_string: self.0.metadata.raw_string,
headers: self.0.metadata.headers,
query: self.0.metadata.query,
method: (&self.0.metadata.method).try_into()?,
route: route_path,
path: called_path,
params,
}),
);
Ok(PushArgsOwned { args, extra: None })
}
}
backend/windmill-api/src/http_triggers.rs
+32 -94
View File
@@ -1,10 +1,11 @@
use crate::http_trigger_auth::{self};
#[cfg(feature = "http_trigger")]
use crate::http_trigger_args::{HttpMethod, RawHttpTriggerArgs};
#[cfg(feature = "parquet")]
use crate::job_helpers_ee::get_workspace_s3_resource;
use crate::resources::try_get_resource_from_db_as;
use crate::trigger_helpers::{get_runnable_format, RunnableId};
use crate::utils::non_empty_str;
use crate::{
args::try_from_request_body,
auth::{AuthCache, OptTokened},
db::{ApiAuthed, DB},
jobs::{
@@ -15,7 +16,7 @@ use crate::{
};
use axum::response::Response;
use axum::{
extract::{Path, Query, Request},
extract::{Path, Query},
response::IntoResponse,
routing::{delete, get, post},
Extension, Json, Router,
@@ -38,8 +39,9 @@ use windmill_common::{
error::{self, JsonResult},
s3_helpers::S3Object,
utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath},
worker::{to_raw_value, CLOUD_HOSTED},
worker::CLOUD_HOSTED,
};
use windmill_queue::TriggerKind;
lazy_static::lazy_static! {
static ref ROUTE_PATH_KEY_RE: regex::Regex = regex::Regex::new(r"/?:[-\w]+").unwrap();
@@ -81,31 +83,6 @@ pub fn workspaced_service() -> Router {
.route("/route_exists", post(exists_route))
}
#[derive(Serialize, Deserialize, sqlx::Type, Debug)]
#[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")]
#[serde(rename_all = "lowercase")]
pub enum HttpMethod {
Get,
Post,
Put,
Delete,
Patch,
}
impl TryFrom<&http::Method> for HttpMethod {
type Error = error::Error;
fn try_from(method: &http::Method) -> Result<Self, Self::Error> {
match method {
&http::Method::GET => Ok(HttpMethod::Get),
&http::Method::POST => Ok(HttpMethod::Post),
&http::Method::PUT => Ok(HttpMethod::Put),
&http::Method::DELETE => Ok(HttpMethod::Delete),
&http::Method::PATCH => Ok(HttpMethod::Patch),
_ => Err(error::Error::BadRequest("Invalid HTTP method".to_string())),
}
}
}
#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone, Copy)]
#[sqlx(type_name = "AUTHENTICATION_METHOD", rename_all = "snake_case")]
#[serde(rename_all(serialize = "snake_case", deserialize = "snake_case"))]
@@ -902,42 +879,14 @@ async fn get_http_route_trigger(
Ok((trigger, route_path.0, params, authed))
}
pub async fn build_http_trigger_extra(
route_path: &str,
called_path: &str,
method: &http::Method,
params: &HashMap<String, String>,
query: &HashMap<String, String>,
headers: &HeaderMap,
) -> Box<serde_json::value::RawValue> {
let headers = headers
.iter()
.map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_string()))
.collect::<HashMap<String, String>>();
to_raw_value(&serde_json::json!({
"kind": "http",
"http": {
"route": route_path,
"path": called_path,
"method": method.to_string().to_lowercase(),
"params": params,
"query": query,
"headers": headers
},
}))
}
async fn route_job(
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Extension(auth_cache): Extension<Arc<AuthCache>>,
OptTokened { token }: OptTokened,
Path(route_path): Path<StripPath>,
Query(query): Query<HashMap<String, String>>,
method: http::Method,
headers: HeaderMap,
request: Request,
args: RawHttpTriggerArgs,
) -> Result<impl IntoResponse, Response> {
let route_path = route_path.to_path().trim_end_matches("/");
let (trigger, called_path, params, authed) = get_http_route_trigger(
@@ -946,25 +895,13 @@ async fn route_job(
token.as_ref(),
&db,
user_db.clone(),
&method,
&args.0.metadata.method,
)
.await
.map_err(|e| e.into_response())?;
let args = try_from_request_body(
request,
&(),
Some(match trigger.authentication_method {
AuthenticationMethod::CustomScript | AuthenticationMethod::Signature => true,
_ => trigger.raw_string,
}),
Some(trigger.wrap_body),
)
.await
.map_err(|e| e.into_response())?;
let mut args = args
.to_push_args_owned(&authed, &db, &trigger.workspace_id)
let args = args
.process_args(&authed, &db, &trigger.workspace_id, trigger.raw_string)
.await
.map_err(|e| e.into_response())?;
@@ -984,7 +921,7 @@ async fn route_job(
};
let authentication_method =
try_get_resource_from_db_as::<http_trigger_auth::AuthenticationMethod>(
try_get_resource_from_db_as::<crate::http_trigger_auth::AuthenticationMethod>(
authed.clone(),
Some(user_db.clone()),
&db,
@@ -995,14 +932,11 @@ async fn route_job(
.map_err(|e| e.into_response())?;
let raw_payload = args
.extra
.0
.metadata
.raw_string
.as_ref()
.and_then(|extra| {
extra
.get("raw_string")
.and_then(|value| Some(value.to_string()))
.and_then(|raw_payload| Some(serde_json::from_str::<String>(&raw_payload)))
})
.map(|raw_payload| serde_json::from_str::<String>(raw_payload))
.transpose()
.map_err(|e| {
windmill_common::error::Error::SerdeJson { location: e.to_string(), error: e }
@@ -1130,20 +1064,28 @@ async fn route_job(
}
}
let extra = args.extra.get_or_insert_with(HashMap::new);
let runnable_format = get_runnable_format(
if trigger.is_flow {
RunnableId::from_flow_path(&trigger.script_path)
} else {
RunnableId::from_script_path(&trigger.script_path)
},
&trigger.workspace_id,
&db,
&TriggerKind::Http,
)
.await
.map_err(|e| e.into_response())?;
extra.insert(
"wm_trigger".to_string(),
build_http_trigger_extra(
let args = args
.to_args_from_format(
&trigger.route_path,
&called_path,
&method,
&params,
&query,
&headers,
runnable_format,
trigger.wrap_body,
)
.await,
);
.map_err(|e| e.into_response())?;
let run_query = RunJobQuery::default();
@@ -1157,7 +1099,6 @@ async fn route_job(
StripPath(trigger.script_path.to_owned()),
run_query,
args,
None,
)
.await
.into_response()
@@ -1170,7 +1111,6 @@ async fn route_job(
user_db,
args,
trigger.workspace_id.clone(),
None,
)
.await
.into_response()
@@ -1185,7 +1125,6 @@ async fn route_job(
StripPath(trigger.script_path.to_owned()),
run_query,
args,
None,
)
.await
.into_response()
@@ -1198,7 +1137,6 @@ async fn route_job(
user_db,
trigger.workspace_id.clone(),
args,
None,
)
.await
.into_response()
backend/windmill-api/src/jobs.rs
+127 -190
View File
@@ -7,6 +7,7 @@
*/
use axum::body::Body;
use axum::extract::Request;
use axum::http::HeaderValue;
#[cfg(feature = "deno_core")]
use deno_core::{op2, serde_v8, v8, JsRuntime, OpState};
@@ -37,10 +38,11 @@ use crate::add_webhook_allowed_origin;
use crate::concurrency_groups::join_concurrency_key;
use crate::db::ApiAuthed;
use crate::trigger_helpers::RunnableId;
use crate::users::get_scope_tags;
use crate::utils::content_plain;
use crate::{
args::{DecodeQueries, WebhookArgs},
args::{DecodeQueries, RawWebhookArgs},
db::DB,
users::{check_scopes, require_owner_of_path, OptAuthed},
utils::require_super_admin,
@@ -55,7 +57,7 @@ use axum::{
use base64::Engine;
use chrono::Utc;
use hmac::Mac;
use hyper::{Request, StatusCode};
use hyper::StatusCode;
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use sql_builder::prelude::*;
use sqlx::types::JsonRawValue;
@@ -86,7 +88,10 @@ use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS;
#[cfg(feature = "prometheus")]
use windmill_common::{METRICS_DEBUG_ENABLED, METRICS_ENABLED};
use windmill_common::{get_latest_deployed_hash_for_path, BASE_URL};
use windmill_common::{
get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path,
get_script_info_for_hash, FlowVersionInfo, ScriptHashInfo, BASE_URL,
};
use windmill_queue::{
cancel_job, get_result_and_success_by_id_from_flow, job_is_complete, push, PushArgs,
PushArgsOwned, PushIsolationLevel,
@@ -545,58 +550,6 @@ async fn force_cancel(
}
}
pub async fn get_path_tag_limits_cache_for_hash(
mut tx: Transaction<'_, Postgres>,
w_id: &str,
hash: i64,
) -> error::Result<(
String,
Option<String>,
Option<String>,
Option<i32>,
Option<i32>,
Option<i32>,
ScriptLang,
Option<bool>,
Option<i16>,
Option<bool>,
Option<i32>,
Option<bool>,
Option<String>,
String,
)> {
let script = sqlx::query!(
"select path, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor, on_behalf_of_email, created_by from script where hash = $1 AND workspace_id = $2",
hash,
w_id
)
.fetch_optional(&mut *tx)
.await
.map_err(|e| {
Error::internal_err(format!(
"querying getting path for hash {hash} in {w_id}: {e:#}"
))
})?.ok_or_else(|| Error::NotFound(format!(
"deployed script not found at hash {hash} in workspace {w_id}"
)))?;
Ok((
script.path,
script.tag,
script.concurrency_key,
script.concurrent_limit,
script.concurrency_time_window_s,
script.cache_ttl,
script.language,
script.dedicated_worker,
script.priority,
script.delete_after_use,
script.timeout,
script.has_preprocessor,
script.on_behalf_of_email,
script.created_by,
))
}
async fn get_flow_job_debug_info(
OptAuthed(opt_authed): OptAuthed,
Extension(db): Extension<DB>,
@@ -3466,7 +3419,6 @@ async fn batch_rerun_handle_job(
StripPath(job.script_path.clone()),
RunJobQuery { ..Default::default() },
PushArgsOwned { extra: None, args },
None,
)
.await;
if let Ok((_, uuid)) = result {
@@ -3483,7 +3435,6 @@ async fn batch_rerun_handle_job(
StripPath(job.script_path.clone()),
RunJobQuery { ..Default::default() },
PushArgsOwned { extra: None, args },
None,
)
.await
} else {
@@ -3495,7 +3446,6 @@ async fn batch_rerun_handle_job(
job.script_hash,
RunJobQuery { ..Default::default() },
PushArgsOwned { extra: None, args },
None,
)
.await
};
@@ -3516,11 +3466,19 @@ pub async fn run_flow_by_path(
Extension(user_db): Extension<UserDB>,
Path((w_id, flow_path)): Path<(String, StripPath)>,
Query(run_query): Query<RunJobQuery>,
args: WebhookArgs,
args: RawWebhookArgs,
) -> error::Result<(StatusCode, String)> {
let args = args.to_push_args_owned(&authed, &db, &w_id).await?;
let args = args
.to_args_from_runnable(
&authed,
&db,
&w_id,
RunnableId::from_flow_path(&flow_path.0),
run_query.skip_preprocessor,
)
.await?;
run_flow_by_path_inner(authed, db, user_db, w_id, flow_path, run_query, args, None).await
run_flow_by_path_inner(authed, db, user_db, w_id, flow_path, run_query, args).await
}
pub async fn run_flow_by_path_inner(
@@ -3531,7 +3489,6 @@ pub async fn run_flow_by_path_inner(
flow_path: StripPath,
run_query: RunJobQuery,
args: PushArgsOwned,
label_prefix: Option<String>,
) -> error::Result<(StatusCode, String)> {
#[cfg(feature = "enterprise")]
check_license_key_valid().await?;
@@ -3539,23 +3496,17 @@ pub async fn run_flow_by_path_inner(
check_scopes(&authed, || format!("run:flow/{flow_path}"))?;
let mut tx = user_db.clone().begin(&authed).await?;
let (tag, dedicated_worker, has_preprocessor, on_behalf_of_email, edited_by) = sqlx::query!(
"SELECT tag, dedicated_worker, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor, on_behalf_of_email, edited_by
FROM flow
LEFT JOIN flow_version
ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]
WHERE flow.path = $1 and flow.workspace_id = $2",
flow_path,
w_id
)
.fetch_optional(&mut *tx)
.await?
.map(|x| (x.tag, x.dedicated_worker, x.has_preprocessor, x.on_behalf_of_email, x.edited_by))
.ok_or_else(|| {
Error::NotFound(format!(
"flow not found at path {flow_path} in workspace {w_id}"
))
})?;
let FlowVersionInfo {
version,
tag,
dedicated_worker,
has_preprocessor,
on_behalf_of_email,
edited_by,
..
} = get_latest_flow_version_info_for_path(&mut *tx, &w_id, &flow_path, true).await?;
drop(tx);
let tag = run_query.tag.clone().or(tag);
@@ -3587,13 +3538,12 @@ pub async fn run_flow_by_path_inner(
JobPayload::Flow {
path: flow_path.to_string(),
dedicated_worker,
version,
apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false)
&& has_preprocessor.unwrap_or(false),
},
PushArgs { args: &args.args, extra: args.extra },
&label_prefix
.map(|x| x + authed.display_username())
.unwrap_or_else(|| authed.display_username().to_string()),
authed.display_username(),
email,
permissioned_as,
scheduled_for,
@@ -3715,20 +3665,19 @@ pub async fn run_script_by_path(
Extension(user_db): Extension<UserDB>,
Path((w_id, script_path)): Path<(String, StripPath)>,
Query(run_query): Query<RunJobQuery>,
args: WebhookArgs,
args: RawWebhookArgs,
) -> error::Result<(StatusCode, String)> {
let args = args.to_push_args_owned(&authed, &db, &w_id).await?;
run_script_by_path_inner(
authed,
db,
user_db,
w_id,
script_path,
run_query,
args,
None,
)
.await
let args = args
.to_args_from_runnable(
&authed,
&db,
&w_id,
RunnableId::from_script_path(&script_path.0),
run_query.skip_preprocessor,
)
.await?;
run_script_by_path_inner(authed, db, user_db, w_id, script_path, run_query, args).await
}
pub async fn run_script_by_path_inner(
@@ -3739,7 +3688,6 @@ pub async fn run_script_by_path_inner(
script_path: StripPath,
run_query: RunJobQuery,
args: PushArgsOwned,
label_prefix: Option<String>,
) -> error::Result<(StatusCode, String)> {
#[cfg(feature = "enterprise")]
check_license_key_valid().await?;
@@ -3780,9 +3728,7 @@ pub async fn run_script_by_path_inner(
&w_id,
job_payload,
PushArgs { args: &args.args, extra: args.extra },
&label_prefix
.map(|x| x + authed.display_username())
.unwrap_or_else(|| authed.display_username().to_string()),
authed.display_username(),
email,
permissioned_as,
scheduled_for,
@@ -4504,10 +4450,8 @@ pub async fn run_wait_result_flow_by_path_get(
let args = PushArgsOwned { extra: Some(payload_args), args: HashMap::new() };
run_wait_result_flow_by_path_internal(
db, run_query, flow_path, authed, user_db, args, w_id, None,
)
.await
run_wait_result_flow_by_path_internal(db, run_query, flow_path, authed, user_db, args, w_id)
.await
}
pub async fn run_wait_result_script_by_path(
@@ -4516,24 +4460,23 @@ pub async fn run_wait_result_script_by_path(
Extension(db): Extension<DB>,
Path((w_id, script_path)): Path<(String, StripPath)>,
Query(run_query): Query<RunJobQuery>,
args: WebhookArgs,
args: RawWebhookArgs,
) -> error::Result<Response> {
#[cfg(feature = "enterprise")]
check_license_key_valid().await?;
let args = args.to_push_args_owned(&authed, &db, &w_id).await?;
let args = args
.to_args_from_runnable(
&authed,
&db,
&w_id,
RunnableId::from_script_path(&script_path.0),
run_query.skip_preprocessor,
)
.await?;
run_wait_result_script_by_path_internal(
db,
run_query,
script_path,
authed,
user_db,
w_id,
args,
None,
)
.await
run_wait_result_script_by_path_internal(db, run_query, script_path, authed, user_db, w_id, args)
.await
}
pub async fn run_wait_result_script_by_path_internal(
@@ -4544,7 +4487,6 @@ pub async fn run_wait_result_script_by_path_internal(
user_db: UserDB,
w_id: String,
args: PushArgsOwned,
label_prefix: Option<String>,
) -> error::Result<Response> {
check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
let script_path = script_path.to_path();
@@ -4580,9 +4522,7 @@ pub async fn run_wait_result_script_by_path_internal(
&w_id,
job_payload,
PushArgs { args: &args.args, extra: args.extra },
&label_prefix
.map(|x| x + authed.display_username())
.unwrap_or_else(|| authed.display_username().to_string()),
authed.display_username(),
email,
permissioned_as,
None,
@@ -4616,19 +4556,29 @@ pub async fn run_wait_result_script_by_hash(
Extension(db): Extension<DB>,
Path((w_id, script_hash)): Path<(String, ScriptHash)>,
Query(run_query): Query<RunJobQuery>,
args: WebhookArgs,
args: RawWebhookArgs,
) -> error::Result<Response> {
#[cfg(feature = "enterprise")]
check_license_key_valid().await?;
let args = args.to_push_args_owned(&authed, &db, &w_id).await?;
let args = args
.to_args_from_runnable(
&authed,
&db,
&w_id,
RunnableId::from_script_hash(script_hash),
run_query.skip_preprocessor,
)
.await?;
check_queue_too_long(&db, run_query.queue_limit).await?;
let hash = script_hash.0;
let (
let mut tx = user_db.clone().begin(&authed).await?;
let ScriptHashInfo {
path,
tag,
custom_concurrency_key,
concurrency_key,
concurrent_limit,
concurrency_time_window_s,
mut cache_ttl,
@@ -4640,8 +4590,8 @@ pub async fn run_wait_result_script_by_hash(
has_preprocessor,
on_behalf_of_email,
created_by,
) = get_path_tag_limits_cache_for_hash(user_db.clone().begin(&authed).await?, &w_id, hash)
.await?;
..
} = get_script_info_for_hash(&mut *tx, &w_id, hash).await?;
if let Some(run_query_cache_ttl) = run_query.cache_ttl {
cache_ttl = Some(run_query_cache_ttl);
}
@@ -4674,7 +4624,7 @@ pub async fn run_wait_result_script_by_hash(
JobPayload::ScriptHash {
hash: ScriptHash(hash),
path: path,
custom_concurrency_key,
custom_concurrency_key: concurrency_key,
concurrent_limit: concurrent_limit,
concurrency_time_window_s: concurrency_time_window_s,
cache_ttl,
@@ -4719,17 +4669,23 @@ pub async fn run_wait_result_flow_by_path(
Extension(db): Extension<DB>,
Path((w_id, flow_path)): Path<(String, StripPath)>,
Query(run_query): Query<RunJobQuery>,
args: WebhookArgs,
args: RawWebhookArgs,
) -> error::Result<Response> {
#[cfg(feature = "enterprise")]
check_license_key_valid().await?;
let args = args.to_push_args_owned(&authed, &db, &w_id).await?;
let args = args
.to_args_from_runnable(
&authed,
&db,
&w_id,
RunnableId::from_flow_path(&flow_path.0),
run_query.skip_preprocessor,
)
.await?;
run_wait_result_flow_by_path_internal(
db, run_query, flow_path, authed, user_db, args, w_id, None,
)
.await
run_wait_result_flow_by_path_internal(db, run_query, flow_path, authed, user_db, args, w_id)
.await
}
pub async fn run_wait_result_flow_by_path_internal(
@@ -4740,7 +4696,6 @@ pub async fn run_wait_result_flow_by_path_internal(
user_db: UserDB,
args: PushArgsOwned,
w_id: String,
label_prefix: Option<String>,
) -> error::Result<Response> {
check_queue_too_long(&db, run_query.queue_limit).await?;
@@ -4750,23 +4705,16 @@ pub async fn run_wait_result_flow_by_path_internal(
let scheduled_for = run_query.get_scheduled_for(&db).await?;
let mut tx = user_db.clone().begin(&authed).await?;
let (tag, dedicated_worker, early_return, has_preprocessor, on_behalf_of_email, edited_by) = sqlx::query!(
"SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor, on_behalf_of_email, edited_by
FROM flow
LEFT JOIN flow_version
ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]
WHERE flow.path = $1 and flow.workspace_id = $2",
flow_path,
w_id
)
.fetch_optional(&mut *tx)
.await?
.map(|x| (x.tag, x.dedicated_worker, x.early_return, x.has_preprocessor, x.on_behalf_of_email, x.edited_by))
.ok_or_else(|| {
Error::NotFound(format!(
"flow not found at path {flow_path} in workspace {w_id}"
))
})?;
let FlowVersionInfo {
tag,
dedicated_worker,
early_return,
has_preprocessor,
on_behalf_of_email,
edited_by,
version,
} = get_latest_flow_version_info_for_path(&mut *tx, &w_id, &flow_path, true).await?;
let tag = run_query.tag.clone().or(tag);
check_tag_available_for_workspace(&w_id, &tag, &authed).await?;
@@ -4795,13 +4743,12 @@ pub async fn run_wait_result_flow_by_path_internal(
JobPayload::Flow {
path: flow_path.to_string(),
dedicated_worker,
version,
apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false)
&& has_preprocessor.unwrap_or(false),
},
PushArgs { args: &args.args, extra: args.extra },
&label_prefix
.map(|x| x + authed.display_username())
.unwrap_or_else(|| authed.display_username().to_string()),
authed.display_username(),
email,
permissioned_as,
scheduled_for,
@@ -5255,29 +5202,23 @@ async fn add_batch_jobs(
"script" => {
if let Some(path) = batch_info.path {
let mut tx = user_db.clone().begin(&authed).await?;
let (
script_hash,
_tag,
custom_concurrency_key,
let ScriptHashInfo {
hash: script_hash,
concurrency_key,
concurrent_limit,
concurrency_time_window_s,
_cache_ttl,
language,
dedicated_worker,
_priority,
_delete_after_use,
timeout,
_,
_, // TODO: consider on_behalf_of_email and created_by for batch jobs
_, // ------------------------------------------
) = get_latest_deployed_hash_for_path(&mut *tx, &w_id, &path).await?;
.. // TODO: consider on_behalf_of_email and created_by for batch jobs
} = get_latest_deployed_hash_for_path(&mut *tx, &w_id, &path).await?;
(
Some(script_hash),
Some(path),
JobKind::Script,
Some(language),
dedicated_worker,
custom_concurrency_key,
concurrency_key,
concurrent_limit,
concurrency_time_window_s,
timeout,
@@ -5417,7 +5358,7 @@ async fn add_batch_jobs(
raw_lock,
raw_flow.map(sqlx::types::Json) as Option<sqlx::types::Json<FlowValue>>,
tag,
hash.map(|h| h.0),
hash,
path,
job_kind.clone() as JobKind,
language as ScriptLang,
@@ -5562,20 +5503,19 @@ pub async fn run_job_by_hash(
Extension(user_db): Extension<UserDB>,
Path((w_id, script_hash)): Path<(String, ScriptHash)>,
Query(run_query): Query<RunJobQuery>,
args: WebhookArgs,
args: RawWebhookArgs,
) -> error::Result<(StatusCode, String)> {
let args = args.to_push_args_owned(&authed, &db, &w_id).await?;
run_job_by_hash_inner(
authed,
db,
user_db,
w_id,
script_hash,
run_query,
args,
None,
)
.await
let args = args
.to_args_from_runnable(
&authed,
&db,
&w_id,
RunnableId::from_script_hash(script_hash),
run_query.skip_preprocessor,
)
.await?;
run_job_by_hash_inner(authed, db, user_db, w_id, script_hash, run_query, args).await
}
pub async fn run_job_by_hash_inner(
@@ -5586,29 +5526,28 @@ pub async fn run_job_by_hash_inner(
script_hash: ScriptHash,
run_query: RunJobQuery,
args: PushArgsOwned,
label_prefix: Option<String>,
) -> error::Result<(StatusCode, String)> {
#[cfg(feature = "enterprise")]
check_license_key_valid().await?;
let hash = script_hash.0;
let (
let mut tx = user_db.clone().begin(&authed).await?;
let ScriptHashInfo {
path,
tag,
custom_concurrency_key,
concurrency_key,
concurrent_limit,
concurrency_time_window_s,
mut cache_ttl,
language,
dedicated_worker,
priority,
_delete_after_use, // not taken into account in async endpoints
timeout,
has_preprocessor,
on_behalf_of_email,
created_by,
) = get_path_tag_limits_cache_for_hash(user_db.clone().begin(&authed).await?, &w_id, hash)
.await?;
.. // delete_after_use not taken into account in async endpoints
} = get_script_info_for_hash(&mut *tx, &w_id, hash).await?;
check_scopes(&authed, || format!("run:script/{path}"))?;
if let Some(run_query_cache_ttl) = run_query.cache_ttl {
cache_ttl = Some(run_query_cache_ttl);
@@ -5642,7 +5581,7 @@ pub async fn run_job_by_hash_inner(
JobPayload::ScriptHash {
hash: ScriptHash(hash),
path: path,
custom_concurrency_key,
custom_concurrency_key: concurrency_key,
concurrent_limit: concurrent_limit,
concurrency_time_window_s: concurrency_time_window_s,
cache_ttl,
@@ -5653,9 +5592,7 @@ pub async fn run_job_by_hash_inner(
&& has_preprocessor.unwrap_or(false),
},
PushArgs { args: &args.args, extra: args.extra },
&label_prefix
.map(|x| x + authed.display_username())
.unwrap_or_else(|| authed.display_username().to_string()),
authed.display_username(),
email,
permissioned_as,
scheduled_for,
backend/windmill-api/src/lib.rs
+3
View File
@@ -81,6 +81,8 @@ mod folders;
mod granular_acls;
mod groups;
#[cfg(feature = "http_trigger")]
mod http_trigger_args;
#[cfg(feature = "http_trigger")]
mod http_trigger_auth;
#[cfg(feature = "http_trigger")]
mod http_triggers;
@@ -122,6 +124,7 @@ mod slack_approvals;
mod smtp_server_ee;
#[cfg(all(feature = "enterprise", feature = "sqs_trigger"))]
mod sqs_triggers_ee;
mod trigger_helpers;
mod static_assets;
#[cfg(all(feature = "stripe", feature = "enterprise"))]
backend/windmill-api/src/mcp.rs
-2
View File
@@ -924,7 +924,6 @@ impl ServerHandler for Runner {
user_db.clone(),
w_id.clone(),
push_args,
None,
)
.await
} else {
@@ -936,7 +935,6 @@ impl ServerHandler for Runner {
user_db.clone(),
push_args,
w_id.clone(),
None,
)
.await
};
backend/windmill-api/src/mqtt_triggers.rs
+61 -45
View File
@@ -3,6 +3,7 @@ use crate::{
db::{ApiAuthed, DB},
jobs::{run_flow_by_path_inner, run_script_by_path_inner, RunJobQuery},
resources::try_get_resource_from_db_as,
trigger_helpers::TriggerJobArgs,
users::fetch_api_authed,
};
use windmill_queue::TriggerKind;
@@ -16,7 +17,7 @@ use axum::{
routing::{delete, get, post},
Router,
};
use base64::prelude::*;
use base64::{engine, prelude::*};
use bytes::Bytes;
use http::StatusCode;
use itertools::Itertools;
@@ -51,8 +52,6 @@ use rand::seq::SliceRandom;
use serde_json::value::RawValue;
use sqlx::types::Json as SqlxJson;
use windmill_queue::PushArgsOwned;
pub fn workspaced_service() -> Router {
Router::new()
.route("/create", post(create_mqtt_trigger))
@@ -82,12 +81,20 @@ enum Error {
}
async fn run_job(
args: Option<HashMap<String, Box<RawValue>>>,
extra: Option<HashMap<String, Box<RawValue>>>,
payload: &[u8],
trigger_info: HashMap<String, Box<RawValue>>,
db: &DB,
trigger: &MqttTrigger,
) -> anyhow::Result<()> {
let args = PushArgsOwned { args: args.unwrap_or_default(), extra };
let args = MqttTrigger::build_job_args(
&trigger.script_path,
trigger.is_flow,
&trigger.workspace_id,
db,
payload,
trigger_info,
)
.await?;
let authed = fetch_api_authed(
trigger.edited_by.clone(),
@@ -111,7 +118,6 @@ async fn run_job(
StripPath(trigger.script_path.to_owned()),
run_query,
args,
None,
)
.await?;
} else {
@@ -123,7 +129,6 @@ async fn run_job(
StripPath(trigger.script_path.to_owned()),
run_query,
args,
None,
)
.await?;
}
@@ -1090,31 +1095,27 @@ impl EventLoop for V3EventLoop {
}
async fn handle_publish_packet(db: &DB, mqtt: &MqttConfig, payload: Bytes, publish: PublishData) {
let args = HashMap::from([("payload".to_string(), to_raw_value(&payload.as_ref()))]);
let extra = Some(HashMap::from([(
"wm_trigger".to_string(),
to_raw_value(&serde_json::json!({
"kind": "mqtt",
"mqtt": {
"topic": publish.topic,
"retain": publish.retain,
"pkid": publish.pkid,
"qos": publish.qos,
"v5": publish.v5.map(|properties| {
serde_json::json!({
"payload_format_indicator": properties.payload_format_indicator,
"topic_alias": properties.topic_alias,
"response_topic": properties.response_topic,
"correlation_data": properties.correlation_data.as_deref(),
"user_properties": properties.user_properties,
"subscription_identifiers": properties.subscription_identifiers,
"content_type": properties.content_type,
})
let trigger_info = HashMap::from([
("topic".to_string(), to_raw_value(&publish.topic)),
("retain".to_string(), to_raw_value(&publish.retain)),
("pkid".to_string(), to_raw_value(&publish.pkid)),
("qos".to_string(), to_raw_value(&publish.qos)),
(
"v5".to_string(),
to_raw_value(&publish.v5.map(|properties| {
serde_json::json!({
"payload_format_indicator": properties.payload_format_indicator,
"topic_alias": properties.topic_alias,
"response_topic": properties.response_topic,
"correlation_data": properties.correlation_data.as_deref(),
"user_properties": properties.user_properties,
"subscription_identifiers": properties.subscription_identifiers,
"content_type": properties.content_type,
})
}
})),
)]));
mqtt.handle(&db, Some(args), extra).await;
})),
),
]);
mqtt.handle(&db, payload.as_ref(), trigger_info).await;
}
async fn handle_event<E, H>(db: &DB, mqtt: &MqttConfig, handler: H, mut event_loop: E) -> ()
@@ -1225,12 +1226,12 @@ impl MqttConfig {
async fn handle(
&self,
db: &DB,
args: Option<HashMap<String, Box<RawValue>>>,
extra: Option<HashMap<String, Box<RawValue>>>,
payload: &[u8],
trigger_info: HashMap<String, Box<RawValue>>,
) -> () {
match self {
MqttConfig::Trigger(trigger) => trigger.handle(&db, args, extra).await,
MqttConfig::Capture(capture) => capture.handle(&db, args, extra).await,
MqttConfig::Trigger(trigger) => trigger.handle(&db, payload, trigger_info).await,
MqttConfig::Capture(capture) => capture.handle(&db, payload, trigger_info).await,
}
}
}
@@ -1406,10 +1407,10 @@ impl MqttTrigger {
async fn handle(
&self,
db: &DB,
args: Option<HashMap<String, Box<RawValue>>>,
extra: Option<HashMap<String, Box<RawValue>>>,
payload: &[u8],
trigger_info: HashMap<String, Box<RawValue>>,
) -> () {
if let Err(err) = run_job(args, extra, db, self).await {
if let Err(err) = run_job(payload, trigger_info, db, self).await {
report_critical_error(
format!("Failed to trigger job from mqtt {}: {:?}", self.path, err),
db.clone(),
@@ -1421,6 +1422,21 @@ impl MqttTrigger {
}
}
impl TriggerJobArgs<&[u8]> for MqttTrigger {
fn v1_payload_fn(payload: &[u8]) -> HashMap<String, Box<RawValue>> {
HashMap::from([("payload".to_string(), to_raw_value(&payload))])
}
fn v2_payload_fn(payload: &[u8]) -> HashMap<String, Box<RawValue>> {
let base64_payload = engine::general_purpose::STANDARD.encode(payload);
HashMap::from([("payload".to_string(), to_raw_value(&base64_payload))])
}
fn trigger_kind() -> TriggerKind {
TriggerKind::Mqtt
}
}
struct PublishData {
topic: String,
retain: bool,
@@ -1675,19 +1691,19 @@ impl CaptureConfigForMqttTrigger {
async fn handle(
&self,
db: &DB,
args: Option<HashMap<String, Box<RawValue>>>,
extra: Option<HashMap<String, Box<RawValue>>>,
payload: &[u8],
trigger_info: HashMap<String, Box<RawValue>>,
) -> () {
let args = PushArgsOwned { args: args.unwrap_or_default(), extra: None };
let extra = extra.as_ref().map(to_raw_value);
let (main_args, preprocessor_args) =
MqttTrigger::build_capture_payloads(payload, trigger_info);
if let Err(err) = insert_capture_payload(
db,
&self.workspace_id,
&self.path,
self.is_flow,
&TriggerKind::Mqtt,
args,
extra,
main_args,
preprocessor_args,
&self.owner,
)
.await
backend/windmill-api/src/postgres_triggers/mod.rs
+11 -6
View File
@@ -2,6 +2,7 @@ use crate::{
db::{ApiAuthed, DB},
jobs::{run_flow_by_path_inner, run_script_by_path_inner, RunJobQuery},
resources::try_get_resource_from_db_as,
trigger_helpers::TriggerJobArgs,
users::fetch_api_authed,
};
use chrono::Utc;
@@ -30,7 +31,6 @@ use handler::{
Relations,
};
use windmill_common::{db::UserDB, error::Error, utils::StripPath};
use windmill_queue::PushArgsOwned;
mod bool;
mod converter;
mod handler;
@@ -250,12 +250,19 @@ pub fn workspaced_service() -> Router {
}
async fn run_job(
args: Option<HashMap<String, Box<RawValue>>>,
extra: Option<HashMap<String, Box<RawValue>>>,
payload: HashMap<String, Box<RawValue>>,
db: &DB,
trigger: &PostgresTrigger,
) -> anyhow::Result<()> {
let args = PushArgsOwned { args: args.unwrap_or_default(), extra };
let args = PostgresTrigger::build_job_args(
&trigger.script_path,
trigger.is_flow,
&trigger.workspace_id,
db,
payload,
HashMap::new(),
)
.await?;
let authed = fetch_api_authed(
trigger.edited_by.clone(),
@@ -279,7 +286,6 @@ async fn run_job(
StripPath(trigger.script_path.to_owned()),
run_query,
args,
None,
)
.await?;
} else {
@@ -291,7 +297,6 @@ async fn run_job(
StripPath(trigger.script_path.to_owned()),
run_query,
args,
None,
)
.await?;
}
backend/windmill-api/src/postgres_triggers/trigger.rs
+32 -38
View File
@@ -12,6 +12,7 @@ use crate::{
run_job,
},
resources::try_get_resource_from_db_as,
trigger_helpers::TriggerJobArgs,
users::fetch_api_authed,
};
use windmill_queue::TriggerKind;
@@ -31,7 +32,6 @@ use sqlx::types::Json as SqlxJson;
use windmill_common::{
db::UserDB, error, utils::report_critical_error, worker::to_raw_value, INSTANCE_NAME,
};
use windmill_queue::PushArgsOwned;
use super::{
drop_logical_replication_slot_query, drop_publication_query, get_database_connection,
@@ -369,13 +369,8 @@ impl PostgresTrigger {
.await
}
async fn handle(
&self,
db: &DB,
args: Option<HashMap<String, Box<RawValue>>>,
extra: Option<HashMap<String, Box<RawValue>>>,
) -> () {
if let Err(err) = run_job(args, extra, db, self).await {
async fn handle(&self, db: &DB, payload: HashMap<String, Box<RawValue>>) -> () {
if let Err(err) = run_job(payload, db, self).await {
report_critical_error(
format!(
"Failed to trigger job from postgres {}: {:?}",
@@ -390,6 +385,20 @@ impl PostgresTrigger {
}
}
impl TriggerJobArgs<HashMap<String, Box<RawValue>>> for PostgresTrigger {
fn v1_payload_fn(payload: HashMap<String, Box<RawValue>>) -> HashMap<String, Box<RawValue>> {
payload
}
fn v2_payload_fn(payload: HashMap<String, Box<RawValue>>) -> HashMap<String, Box<RawValue>> {
payload
}
fn trigger_kind() -> TriggerKind {
TriggerKind::Postgres
}
}
struct PgInfo<'a> {
postgres_resource_path: &'a str,
publication_name: &'a str,
@@ -467,7 +476,6 @@ impl PostgresConfig {
let client = PostgresSimpleClient::new(&database).await?;
let publication = client
.execute_query(&format!(
"SELECT pubname FROM pg_publication WHERE pubname = {}",
@@ -508,15 +516,10 @@ impl PostgresConfig {
}
}
async fn handle(
&self,
db: &DB,
args: Option<HashMap<String, Box<RawValue>>>,
extra: Option<HashMap<String, Box<RawValue>>>,
) -> () {
async fn handle(&self, db: &DB, payload: HashMap<String, Box<RawValue>>) -> () {
match self {
PostgresConfig::Trigger(trigger) => trigger.handle(&db, args, extra).await,
PostgresConfig::Capture(capture) => capture.handle(&db, args, extra).await,
PostgresConfig::Trigger(trigger) => trigger.handle(&db, payload).await,
PostgresConfig::Capture(capture) => capture.handle(&db, payload).await,
}
}
@@ -606,7 +609,7 @@ async fn listen_to_transactions(
}
};
let message = match message {
Ok(message) => message,
Err(err) => {
@@ -678,13 +681,9 @@ async fn listen_to_transactions(
("old_row".to_string(), to_raw_value(&old_row)),
("row".to_string(), to_raw_value(&row)),
]);
let extra = Some(HashMap::from([(
"wm_trigger".to_string(),
to_raw_value(&serde_json::json!({"kind": "postgres", })),
)]));
let _ = pg.handle(&db, Some(database_info), extra).await;
let _ = pg.handle(&db, database_info).await;
}
Some((o_id, old_row, row, transaction_type)) => {
let relation = match relations.get_relation(o_id) {
@@ -694,7 +693,7 @@ async fn listen_to_transactions(
continue;
}
};
if let Err(err) = old_row {
tracing::error!(
transaction_type = ?transaction_type,
@@ -707,7 +706,7 @@ async fn listen_to_transactions(
relation.name,
);
}
if let Err(err) = row {
tracing::error!(
transaction_type = ?transaction_type,
@@ -720,7 +719,7 @@ async fn listen_to_transactions(
relation.name,
);
}
}
_ => {}
}
@@ -919,22 +918,17 @@ impl CaptureConfigForPostgresTrigger {
}
}
async fn handle(
&self,
db: &DB,
args: Option<HashMap<String, Box<RawValue>>>,
extra: Option<HashMap<String, Box<RawValue>>>,
) -> () {
let args = PushArgsOwned { args: args.unwrap_or_default(), extra: None };
let extra = extra.as_ref().map(to_raw_value);
async fn handle(&self, db: &DB, payload: HashMap<String, Box<RawValue>>) -> () {
let main_args = PostgresTrigger::build_job_args_v2(false, payload.clone(), HashMap::new());
let preprocessor_args = PostgresTrigger::build_job_args_v2(true, payload, HashMap::new());
if let Err(err) = insert_capture_payload(
db,
&self.workspace_id,
&self.path,
self.is_flow,
&TriggerKind::Postgres,
args,
extra,
main_args,
preprocessor_args,
&self.owner,
)
.await
backend/windmill-api/src/trigger_helpers.rs
+332
View File
@@ -0,0 +1,332 @@
use quick_cache::sync::Cache;
use serde::Deserialize;
use serde_json::value::RawValue;
use std::collections::HashMap;
use windmill_common::{
error::Result,
get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path,
scripts::{ScriptHash, ScriptLang},
worker::to_raw_value,
FlowVersionInfo,
};
use windmill_queue::{PushArgsOwned, TriggerKind};
use crate::db::DB;
type RunnableFormatCacheKey = (String, i64, TriggerKind);
lazy_static::lazy_static! {
pub static ref RUNNABLE_FORMAT_VERSION_CACHE: Cache<RunnableFormatCacheKey, RunnableFormat> = Cache::new(1000);
}
#[derive(Debug, Clone, Eq, PartialEq, Hash, Copy)]
pub struct RunnableFormat {
pub version: RunnableFormatVersion,
pub has_preprocessor: bool,
}
#[derive(Debug, Clone, Eq, PartialEq, Hash, Copy)]
pub enum RunnableFormatVersion {
V1,
V2,
}
struct ScriptInfo {
has_preprocessor: Option<bool>,
language: ScriptLang,
content: String,
schema: Option<sqlx::types::Json<PartialSchema>>,
}
struct FlowInfo {
has_preprocessor: Option<bool>,
is_v1_preprocessor: Option<bool>,
schema: Option<sqlx::types::Json<PartialSchema>>,
}
#[derive(Debug, Deserialize)]
struct PropertyDefinition {
r#type: Option<String>,
}
#[derive(Debug, Deserialize)]
struct PartialSchema {
properties: Option<HashMap<String, PropertyDefinition>>,
}
#[derive(Debug, Clone, Eq, PartialEq, Hash)]
pub enum RunnableId {
FlowPath(String),
ScriptId(ScriptId),
}
impl RunnableId {
pub fn from_script_hash(hash: ScriptHash) -> Self {
Self::ScriptId(ScriptId::ScriptHash(hash))
}
pub fn from_script_path(path: &str) -> Self {
Self::ScriptId(ScriptId::ScriptPath(path.to_string()))
}
pub fn from_flow_path(path: &str) -> Self {
Self::FlowPath(path.to_string())
}
}
#[derive(Debug, Clone, Eq, PartialEq, Hash)]
pub enum ScriptId {
ScriptPath(String),
ScriptHash(ScriptHash),
}
impl ScriptId {
async fn get_script_hash(self, workspace_id: &str, db: &DB) -> Result<i64> {
let hash = match self {
ScriptId::ScriptPath(path) => {
let info = get_latest_deployed_hash_for_path(db, workspace_id, &path).await?;
info.hash
}
ScriptId::ScriptHash(hash) => hash.0,
};
Ok(hash)
}
}
async fn get_script_info(
db: &DB,
workspace_id: &str,
hash: i64,
) -> std::result::Result<ScriptInfo, sqlx::Error> {
sqlx::query_as!(ScriptInfo, "SELECT has_preprocessor, language as \"language: _\", content, schema as \"schema: _\" FROM script WHERE workspace_id = $1 AND hash = $2", workspace_id, hash)
.fetch_one(db)
.await
}
fn runnable_format_from_schema(
trigger_kind: &TriggerKind,
has_preprocessor: bool,
schema: Option<sqlx::types::Json<PartialSchema>>,
) -> RunnableFormat {
match trigger_kind {
TriggerKind::Mqtt
if schema.as_ref().is_some_and(|schema| {
schema.properties.as_ref().is_some_and(|properties| {
properties.iter().any(|(key, def)| {
key == "payload" && def.r#type.as_ref().is_some_and(|t| t == "array")
})
})
}) =>
{
RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor }
}
TriggerKind::Kafka | TriggerKind::Nats
if schema.as_ref().is_some_and(|schema| {
schema
.properties
.as_ref()
.is_some_and(|properties| properties.keys().any(|key| key == "msg"))
}) =>
{
RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor }
}
_ => RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor },
}
}
pub async fn get_runnable_format(
runnable_id: RunnableId,
workspace_id: &str,
db: &DB,
trigger_kind: &TriggerKind,
) -> Result<RunnableFormat> {
match runnable_id {
RunnableId::FlowPath(path) => {
let FlowVersionInfo { version, .. } =
get_latest_flow_version_info_for_path(db, workspace_id, &path, true).await?;
let key = (workspace_id.to_string(), version, trigger_kind.clone());
let runnable_format = RUNNABLE_FORMAT_VERSION_CACHE.get(&key);
if let Some(runnable_format) = runnable_format {
tracing::debug!("Using cached runnable format for flow {path}");
return Ok(runnable_format);
}
let flow_info = sqlx::query_as!(
FlowInfo,
"SELECT
value->'preprocessor_module' IS NOT NULL as has_preprocessor,
value->'preprocessor_module'->'value'->'input_transforms'->'wm_trigger' IS NOT NULL as is_v1_preprocessor,
schema as \"schema: _\"
FROM flow
WHERE workspace_id = $1
AND path = $2",
workspace_id,
path
)
.fetch_one(db)
.await?;
let has_preprocessor = flow_info.has_preprocessor.unwrap_or(false);
let is_v1_preprocessor = flow_info.is_v1_preprocessor.unwrap_or(false);
let runnable_format = if has_preprocessor && is_v1_preprocessor {
RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: true }
} else {
runnable_format_from_schema(trigger_kind, has_preprocessor, flow_info.schema)
};
RUNNABLE_FORMAT_VERSION_CACHE.insert(key, runnable_format);
Ok(runnable_format)
}
RunnableId::ScriptId(script_id) => {
let hash = script_id.get_script_hash(workspace_id, db).await?;
let key = (workspace_id.to_string(), hash, trigger_kind.clone());
let runnable_format = RUNNABLE_FORMAT_VERSION_CACHE.get(&key);
if let Some(runnable_format) = runnable_format {
tracing::debug!("Using cached runnable format for script {hash}");
return Ok(runnable_format);
}
let script_info = get_script_info(db, workspace_id, hash).await?;
let has_preprocessor = script_info.has_preprocessor.unwrap_or(false);
let runnable_format = if has_preprocessor {
let args = match script_info.language {
ScriptLang::Bun
| ScriptLang::Bunnative
| ScriptLang::Deno
| ScriptLang::Nativets => {
let args = windmill_parser_ts::parse_deno_signature(
&script_info.content,
true,
false,
Some("preprocessor".to_string()),
)?;
Some(args.args)
}
ScriptLang::Python3 => {
let args = windmill_parser_py::parse_python_signature(
&script_info.content,
Some("preprocessor".to_string()),
false,
)?;
Some(args.args)
}
_ => None,
};
if args.is_some_and(|args| args.iter().any(|arg| arg.name == "wm_trigger")) {
RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: true }
} else {
runnable_format_from_schema(trigger_kind, has_preprocessor, script_info.schema)
}
} else {
runnable_format_from_schema(trigger_kind, has_preprocessor, script_info.schema)
};
RUNNABLE_FORMAT_VERSION_CACHE.insert(key, runnable_format);
Ok(runnable_format)
}
}
}
#[allow(dead_code)]
pub trait TriggerJobArgs<T: Clone> {
fn v1_payload_fn(payload: T) -> HashMap<String, Box<RawValue>>;
fn v2_payload_fn(payload: T) -> HashMap<String, Box<RawValue>> {
Self::v1_payload_fn(payload)
}
fn trigger_kind() -> TriggerKind;
fn build_job_args_v2(
has_preprocessor: bool,
payload: T,
info: HashMap<String, Box<RawValue>>,
) -> PushArgsOwned {
let trigger_kind = Self::trigger_kind();
let mut args = Self::v2_payload_fn(payload);
if has_preprocessor {
args.insert("kind".to_string(), to_raw_value(&trigger_kind.to_key()));
args.extend(info);
let args = HashMap::from([("event".to_string(), to_raw_value(&args))]);
PushArgsOwned { args, extra: None }
} else {
PushArgsOwned { args, extra: None }
}
}
fn build_job_args_v1(
has_preprocessor: bool,
payload: T,
info: HashMap<String, Box<RawValue>>,
) -> PushArgsOwned {
let trigger_kind = Self::trigger_kind();
let trigger_key = trigger_kind.to_key();
let args = Self::v1_payload_fn(payload);
let extra = if has_preprocessor {
Some(HashMap::from([(
"wm_trigger".to_string(),
to_raw_value(&serde_json::json!({
"kind": trigger_key,
trigger_key: info
})),
)]))
} else {
None
};
PushArgsOwned { args, extra }
}
async fn build_job_args(
runnable_path: &str,
is_flow: bool,
w_id: &str,
db: &DB,
payload: T,
info: HashMap<String, Box<RawValue>>,
) -> Result<PushArgsOwned> {
let runnable_id = if is_flow {
RunnableId::from_flow_path(runnable_path)
} else {
RunnableId::from_script_path(runnable_path)
};
Self::build_job_args_from_runnable_id(runnable_id, w_id, db, payload, info).await
}
async fn build_job_args_from_runnable_id(
runnable_id: RunnableId,
w_id: &str,
db: &DB,
payload: T,
info: HashMap<String, Box<RawValue>>,
) -> Result<PushArgsOwned> {
let runnable_format =
get_runnable_format(runnable_id, w_id, db, &Self::trigger_kind()).await?;
match runnable_format {
RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor } => {
Ok(Self::build_job_args_v1(has_preprocessor, payload, info))
}
RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor } => {
Ok(Self::build_job_args_v2(has_preprocessor, payload, info))
}
}
}
fn build_capture_payloads(
payload: T,
info: HashMap<String, Box<RawValue>>,
) -> (PushArgsOwned, PushArgsOwned) {
let main_args = Self::build_job_args_v2(false, payload.clone(), info.clone());
let preprocessor_args = Self::build_job_args_v2(true, payload, info);
(main_args, preprocessor_args)
}
}
backend/windmill-api/src/websocket_triggers.rs
+37 -20
View File
@@ -38,6 +38,7 @@ use crate::{
jobs::{
run_flow_by_path_inner, run_script_by_path_inner, run_wait_result_internal, RunJobQuery,
},
trigger_helpers::TriggerJobArgs,
users::fetch_api_authed,
};
@@ -608,7 +609,6 @@ async fn wait_runnable_result(
StripPath(path.clone()),
RunJobQuery::default(),
args,
None,
)
.await?;
@@ -635,7 +635,6 @@ async fn wait_runnable_result(
StripPath(path.clone()),
RunJobQuery::default(),
args,
None,
)
.await?;
@@ -890,10 +889,11 @@ impl WebsocketTrigger {
async fn handle(
&self,
db: &DB,
args: PushArgsOwned,
msg: &str,
trigger_info: HashMap<String, Box<RawValue>>,
return_message_channels: Option<ReturnMessageChannels>,
) -> () {
if let Err(err) = run_job(db, self, args, return_message_channels).await {
if let Err(err) = run_job(db, self, &msg, trigger_info, return_message_channels).await {
report_critical_error(
format!(
"Failed to trigger job from WebSocket {}: {:?}",
@@ -919,6 +919,16 @@ impl WebsocketTrigger {
}
}
impl TriggerJobArgs<&str> for WebsocketTrigger {
fn v1_payload_fn(payload: &str) -> HashMap<String, Box<RawValue>> {
HashMap::from([("msg".to_string(), to_raw_value(&payload))])
}
fn trigger_kind() -> TriggerKind {
TriggerKind::Websocket
}
}
#[derive(Deserialize)]
struct CaptureConfigForWebsocket {
trigger_config: SqlxJson<WebsocketTriggerConfig>,
@@ -985,15 +995,18 @@ impl CaptureConfigForWebsocket {
Some(())
}
async fn handle(&self, db: &DB, args: PushArgsOwned) -> () {
async fn handle(&self, db: &DB, msg: &str, trigger_info: HashMap<String, Box<RawValue>>) -> () {
let (main_args, preprocessor_args) =
WebsocketTrigger::build_capture_payloads(&msg, trigger_info);
if let Err(err) = insert_capture_payload(
db,
&self.workspace_id,
&self.path,
self.is_flow,
&TriggerKind::Websocket,
PushArgsOwned { args: args.args, extra: None },
args.extra.as_ref().map(to_raw_value),
main_args,
preprocessor_args,
&self.owner,
)
.await
@@ -1259,20 +1272,15 @@ async fn listen_to_websocket(
}
}
if should_handle {
let args = HashMap::from([("msg".to_string(), to_raw_value(&text))]);
let extra = Some(HashMap::from([(
"wm_trigger".to_string(),
to_raw_value(&serde_json::json!({"kind": "websocket", "websocket": { "url": url }})),
)]));
let args = PushArgsOwned { args, extra };
let trigger_info = HashMap::from([
("url".to_string(), to_raw_value(&url)),
]);
match &ws {
WebsocketEnum::Trigger(ws_trigger) => {
ws_trigger.handle(&db, args, return_message_channels.clone()).await;
ws_trigger.handle(&db, &text, trigger_info, return_message_channels.clone()).await;
},
WebsocketEnum::Capture(capture) => {
capture.handle(&db, args).await;
capture.handle(&db, &text, trigger_info).await;
},
}
}
@@ -1311,9 +1319,20 @@ async fn listen_to_websocket(
async fn run_job(
db: &DB,
trigger: &WebsocketTrigger,
args: PushArgsOwned,
msg: &str,
trigger_info: HashMap<String, Box<RawValue>>,
return_message_channels: Option<ReturnMessageChannels>,
) -> anyhow::Result<()> {
let args = WebsocketTrigger::build_job_args(
&trigger.script_path,
trigger.is_flow,
&trigger.workspace_id,
db,
msg,
trigger_info,
)
.await?;
let authed = fetch_api_authed(
trigger.edited_by.clone(),
trigger.email.clone(),
@@ -1374,7 +1393,6 @@ async fn run_job(
runnable_path,
run_query,
args,
None,
)
.await?;
} else {
@@ -1386,7 +1404,6 @@ async fn run_job(
runnable_path,
run_query,
args,
None,
)
.await?;
}
backend/windmill-common/src/jobs.rs
+18 -69
View File
@@ -5,7 +5,7 @@ use futures_core::Stream;
use indexmap::IndexMap;
use serde::{Deserialize, Serialize};
use serde_json::value::RawValue;
use sqlx::{types::Json, Pool, Postgres, Transaction};
use sqlx::{types::Json, Pool, Postgres};
use tokio::io::AsyncReadExt;
use uuid::Uuid;
@@ -17,10 +17,11 @@ use crate::{
error::{self, to_anyhow, Error},
flow_status::{FlowStatus, RestartedFrom},
flows::{FlowNodeId, FlowValue, Retry},
get_latest_deployed_hash_for_path,
get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path,
scripts::{ScriptHash, ScriptLang},
users::username_to_permissioned_as,
worker::{to_raw_value, TMP_DIR},
FlowVersionInfo, ScriptHashInfo,
};
#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Copy, Clone)]
@@ -331,6 +332,7 @@ pub enum JobPayload {
path: String,
dedicated_worker: Option<bool>,
apply_preprocessor: bool,
version: i64,
},
RestartedFlow {
completed_job_id: Uuid,
@@ -385,9 +387,9 @@ pub struct OnBehalfOf {
pub permissioned_as: String,
}
pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgres>>(
pub async fn script_path_to_payload<'e, A: sqlx::Acquire<'e, Database = Postgres> + Send>(
script_path: &str,
db: E,
db: A,
w_id: &str,
skip_preprocessor: Option<bool>,
) -> error::Result<(
@@ -407,10 +409,10 @@ pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgre
None,
)
} else {
let (
script_hash,
let ScriptHashInfo {
hash,
tag,
custom_concurrency_key,
concurrency_key,
concurrent_limit,
concurrency_time_window_s,
cache_ttl,
@@ -418,11 +420,12 @@ pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgre
dedicated_worker,
priority,
delete_after_use,
script_timeout,
timeout,
has_preprocessor,
on_behalf_of_email,
created_by,
) = get_latest_deployed_hash_for_path(db, w_id, script_path).await?;
..
} = get_latest_deployed_hash_for_path(db, w_id, script_path).await?;
let on_behalf_of = if let Some(email) = on_behalf_of_email {
Some(OnBehalfOf {
@@ -435,9 +438,9 @@ pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgre
(
JobPayload::ScriptHash {
hash: script_hash,
hash: ScriptHash(hash),
path: script_path.to_owned(),
custom_concurrency_key,
custom_concurrency_key: concurrency_key,
concurrent_limit,
concurrency_time_window_s,
cache_ttl: cache_ttl,
@@ -449,7 +452,7 @@ pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgre
},
tag,
delete_after_use,
script_timeout,
timeout,
on_behalf_of,
)
};
@@ -462,52 +465,6 @@ pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgre
))
}
pub async fn script_hash_to_tag_and_limits<'c>(
script_hash: &ScriptHash,
db: &mut Transaction<'c, Postgres>,
w_id: &String,
) -> error::Result<(
Option<Tag>,
Option<String>,
Option<i32>,
Option<i32>,
Option<i32>,
ScriptLang,
Option<bool>,
Option<i16>,
Option<bool>,
Option<i32>,
Option<String>,
String,
)> {
let script = sqlx::query!(
"select tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, on_behalf_of_email, created_by from script where hash = $1 AND workspace_id = $2",
script_hash.0,
w_id
)
.fetch_one(&mut **db)
.await
.map_err(|e| {
Error::internal_err(format!(
"querying getting tag for hash {script_hash}: {e:#}"
))
})?;
Ok((
script.tag,
script.concurrency_key,
script.concurrent_limit,
script.concurrency_time_window_s,
script.cache_ttl,
script.language,
script.dedicated_worker,
script.priority,
script.delete_after_use,
script.timeout,
script.on_behalf_of_email,
script.created_by,
))
}
pub async fn get_payload_tag_from_prefixed_path(
path: &str,
db: &DB,
@@ -517,18 +474,10 @@ pub async fn get_payload_tag_from_prefixed_path(
script_path_to_payload(path.strip_prefix("script/").unwrap(), db, w_id, Some(true)).await?
} else if path.starts_with("flow/") {
let path = path.strip_prefix("flow/").unwrap().to_string();
let r = sqlx::query!(
"SELECT tag, dedicated_worker from flow WHERE path = $1 and workspace_id = $2",
&path,
&w_id,
)
.fetch_optional(db)
.await?;
let (tag, dedicated_worker) = r
.map(|x| (x.tag, x.dedicated_worker))
.unwrap_or_else(|| (None, None));
let FlowVersionInfo { dedicated_worker, tag, version, .. } =
get_latest_flow_version_info_for_path(db, w_id, &path, true).await?;
(
JobPayload::Flow { path, dedicated_worker, apply_preprocessor: false },
JobPayload::Flow { path, dedicated_worker, apply_preprocessor: false, version },
tag,
None,
None,
backend/windmill-common/src/lib.rs
+206 -44
View File
@@ -6,7 +6,9 @@
* LICENSE-AGPL for a copy of the license.
*/
use quick_cache::sync::Cache;
use std::{
future::Future,
net::SocketAddr,
str::FromStr,
sync::{
@@ -123,8 +125,15 @@ lazy_static::lazy_static! {
pub static ref INSTANCE_NAME: String = rd_string(5);
pub static ref DEPLOYED_SCRIPT_HASH_CACHE: Cache<(String, String), ExpiringLatestVersionId> = Cache::new(1000);
pub static ref FLOW_VERSION_CACHE: Cache<(String, String), ExpiringLatestVersionId> = Cache::new(1000);
pub static ref DEPLOYED_SCRIPT_INFO_CACHE: Cache<(String, i64), ScriptHashInfo> = Cache::new(1000);
pub static ref FLOW_INFO_CACHE: Cache<(String, i64), FlowVersionInfo> = Cache::new(1000);
}
const LATEST_VERSION_ID_CACHE_TTL: std::time::Duration = std::time::Duration::from_secs(60);
pub async fn shutdown_signal(
tx: KillpillSender,
mut rx: tokio::sync::broadcast::Receiver<()>,
@@ -336,54 +345,207 @@ type Tag = String;
pub type DB = Pool<Postgres>;
pub async fn get_latest_deployed_hash_for_path<'e, E: sqlx::Executor<'e, Database = Postgres>>(
#[derive(Clone)]
pub struct ExpiringLatestVersionId {
id: i64,
expires_at: std::time::Instant,
}
#[derive(Clone)]
pub struct ScriptHashInfo {
pub path: String,
pub hash: i64,
pub tag: Option<String>,
pub concurrency_key: Option<String>,
pub concurrent_limit: Option<i32>,
pub concurrency_time_window_s: Option<i32>,
pub cache_ttl: Option<i32>,
pub language: ScriptLang,
pub dedicated_worker: Option<bool>,
pub priority: Option<i16>,
pub delete_after_use: Option<bool>,
pub timeout: Option<i32>,
pub has_preprocessor: Option<bool>,
pub on_behalf_of_email: Option<String>,
pub created_by: String,
}
pub fn get_latest_deployed_hash_for_path<
'a,
'e,
E: sqlx::Acquire<'e, Database = Postgres> + Send + 'a,
>(
db: E,
w_id: &'a str,
script_path: &'a str,
) -> impl Future<Output = error::Result<ScriptHashInfo>> + Send + 'a {
async move {
let mut conn = db.acquire().await?;
let cache_key = (w_id.to_string(), script_path.to_string());
let hash = match DEPLOYED_SCRIPT_HASH_CACHE.get(&cache_key) {
Some(cached_hash) if cached_hash.expires_at > std::time::Instant::now() => {
tracing::debug!(
"Using cached script hash {} for {script_path}",
cached_hash.id
);
cached_hash.id
}
_ => {
tracing::debug!("Fetching script hash for {script_path}");
let hash = sqlx::query_scalar!(
"select hash from script where path = $1 AND workspace_id = $2 AND deleted = false AND lock IS not NULL AND lock_error_logs IS NULL ORDER BY created_at DESC LIMIT 1",
script_path,
w_id
)
.fetch_optional(&mut *conn)
.await?;
let hash = utils::not_found_if_none(hash, "script", script_path)?;
DEPLOYED_SCRIPT_HASH_CACHE.insert(
cache_key,
ExpiringLatestVersionId {
id: hash,
expires_at: std::time::Instant::now() + LATEST_VERSION_ID_CACHE_TTL,
},
);
hash
}
};
get_script_info_for_hash(&mut *conn, w_id, hash).await
}
}
pub async fn get_script_info_for_hash<'e, E: sqlx::PgExecutor<'e>>(
db: E,
w_id: &str,
script_path: &str,
) -> error::Result<(
scripts::ScriptHash,
Option<Tag>,
Option<String>,
Option<i32>,
Option<i32>,
Option<i32>,
ScriptLang,
Option<bool>,
Option<i16>,
Option<bool>,
Option<i32>,
Option<bool>,
Option<String>,
String,
)> {
let r_o = sqlx::query!(
"select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor, on_behalf_of_email, created_by from script where path = $1 AND workspace_id = $2 AND
created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND
deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)",
script_path,
w_id
)
.fetch_optional(db)
.await?;
hash: i64,
) -> error::Result<ScriptHashInfo> {
let key = (w_id.to_string(), hash);
let script = utils::not_found_if_none(r_o, "deployed script", script_path)?;
match DEPLOYED_SCRIPT_INFO_CACHE.get(&key) {
Some(info) => {
tracing::debug!("Using cached deployed script info for {hash}");
Ok(info)
}
_ => {
tracing::debug!("Fetching deployed script info for {hash}");
let info = sqlx::query_as!(
ScriptHashInfo,
"select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor, on_behalf_of_email, created_by, path from script where hash = $1 AND workspace_id = $2",
hash,
w_id
)
.fetch_optional(db)
.await?;
Ok((
scripts::ScriptHash(script.hash),
script.tag,
script.concurrency_key,
script.concurrent_limit,
script.concurrency_time_window_s,
script.cache_ttl,
script.language,
script.dedicated_worker,
script.priority,
script.delete_after_use,
script.timeout,
script.has_preprocessor,
script.on_behalf_of_email,
script.created_by,
))
let info = utils::not_found_if_none(info, "script", &hash.to_string())?;
DEPLOYED_SCRIPT_INFO_CACHE.insert(key, info.clone());
Ok(info)
}
}
}
#[derive(Clone)]
pub struct FlowVersionInfo {
pub version: i64,
pub tag: Option<String>,
pub early_return: Option<String>,
pub has_preprocessor: Option<bool>,
pub on_behalf_of_email: Option<String>,
pub edited_by: String,
pub dedicated_worker: Option<bool>,
}
pub fn get_latest_flow_version_info_for_path<
'a,
'e,
A: sqlx::Acquire<'e, Database = Postgres> + Send + 'a,
>(
db: A,
w_id: &'a str,
path: &'a str,
use_cache: bool,
) -> impl Future<Output = error::Result<FlowVersionInfo>> + Send + 'a {
// as instructed in the docstring of sqlx::Acquire
async move {
let mut conn = db.acquire().await?;
let cache_key = (w_id.to_string(), path.to_string());
let cached_version = if use_cache {
FLOW_VERSION_CACHE.get(&cache_key)
} else {
None
};
let version = match cached_version {
Some(cached_version) if cached_version.expires_at > std::time::Instant::now() => {
tracing::debug!("Using cached flow version {} for {path}", cached_version.id);
cached_version.id
}
_ => {
tracing::debug!("Fetching flow version for {path}");
let version = sqlx::query_scalar!(
"SELECT flow_version.id from flow
INNER JOIN flow_version
ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]
WHERE flow.path = $1 and flow.workspace_id = $2",
path,
w_id
)
.fetch_optional(&mut *conn)
.await?;
let version = utils::not_found_if_none(version, "flow", path)?;
FLOW_VERSION_CACHE.insert(
cache_key,
ExpiringLatestVersionId {
id: version,
expires_at: std::time::Instant::now() + LATEST_VERSION_ID_CACHE_TTL,
},
);
version
}
};
let key = (w_id.to_string(), version);
match FLOW_INFO_CACHE.get(&key) {
Some(info) => {
tracing::debug!("Using cached flow version info for {version} ({path})");
Ok(info)
}
_ => {
tracing::debug!("Fetching flow version info for {version} ({path})");
let info = sqlx::query_as!(
FlowVersionInfo,
"SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor, on_behalf_of_email, edited_by, flow_version.id AS version
FROM flow
INNER JOIN flow_version
ON flow_version.id = $3
WHERE flow.path = $1 and flow.workspace_id = $2",
path,
w_id,
version
)
.fetch_optional(&mut *conn)
.await?;
let info = utils::not_found_if_none(info, "flow", path)?;
FLOW_INFO_CACHE.insert(key, info.clone());
Ok(info)
}
}
}
}
pub async fn get_latest_hash_for_path<'c>(
backend/windmill-queue/src/jobs.rs
+23 -30
View File
@@ -1943,7 +1943,7 @@ async fn handle_successful_schedule<'a, 'c, T: Serialize + Send + Sync>(
Ok(())
}
#[derive(sqlx::Type, Serialize, Deserialize, Debug, Clone)]
#[derive(sqlx::Type, Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash)]
#[sqlx(type_name = "TRIGGER_KIND", rename_all = "lowercase")]
#[serde(rename_all = "lowercase")]
pub enum TriggerKind {
@@ -1959,6 +1959,24 @@ pub enum TriggerKind {
Gcp
}
impl TriggerKind {
pub fn to_key(&self) -> String {
match self {
TriggerKind::Webhook => "webhook".to_string(),
TriggerKind::Http => "http".to_string(),
TriggerKind::Websocket => "websocket".to_string(),
TriggerKind::Kafka => "kafka".to_string(),
TriggerKind::Email => "email".to_string(),
TriggerKind::Nats => "nats".to_string(),
TriggerKind::Mqtt => "mqtt".to_string(),
TriggerKind::Sqs => "sqs".to_string(),
TriggerKind::Postgres => "postgres".to_string(),
TriggerKind::Gcp => "gcp".to_string(),
}
}
}
#[derive(sqlx::Type, Serialize, Deserialize, Debug, Clone)]
#[sqlx(type_name = "JOB_TRIGGER_KIND", rename_all = "lowercase")]
#[serde(rename_all = "lowercase")]
@@ -3261,7 +3279,7 @@ pub async fn push<'c, 'd>(
mut tx: PushIsolationLevel<'c>,
workspace_id: &str,
job_payload: JobPayload,
mut args: PushArgs<'d>,
args: PushArgs<'d>,
user: &str,
mut email: &str,
mut permissioned_as: String,
@@ -3477,17 +3495,10 @@ pub async fn push<'c, 'd>(
priority,
apply_preprocessor,
} => {
let extra = args.extra.get_or_insert_with(HashMap::new);
if apply_preprocessor {
preprocessed = Some(false);
extra.entry("wm_trigger".to_string()).or_insert_with(|| {
to_raw_value(&serde_json::json!({
"kind": "webhook",
}))
});
} else {
extra.remove("wm_trigger");
}
}
(
Some(hash.0),
Some(path),
@@ -3853,19 +3864,8 @@ pub async fn push<'c, 'd>(
priority,
)
}
JobPayload::Flow { path, dedicated_worker, apply_preprocessor } => {
JobPayload::Flow { path, dedicated_worker, apply_preprocessor, version } => {
let mut ntx = tx.into_tx().await?;
// Fetch the latest version of the flow.
let version = sqlx::query_scalar!(
"SELECT flow.versions[array_upper(flow.versions, 1)] AS \"version!: i64\"
FROM flow WHERE path = $1 AND workspace_id = $2",
&path,
&workspace_id
)
.fetch_optional(&mut *ntx)
.await?
.ok_or_else(|| Error::internal_err(format!("not found flow at path {:?}", path)))?;
// Do not use the lite version unless all workers are updated.
let data = if *DISABLE_FLOW_SCRIPT
|| (!*MIN_VERSION_IS_AT_LEAST_1_432.read().await && !*CLOUD_HOSTED)
@@ -3889,17 +3889,10 @@ pub async fn push<'c, 'd>(
let concurrency_time_window_s = value.concurrency_time_window_s;
let concurrent_limit = value.concurrent_limit;
let extra = args.extra.get_or_insert_with(HashMap::new);
if !apply_preprocessor {
value.preprocessor_module = None;
extra.remove("wm_trigger");
} else {
preprocessed = Some(false);
extra.entry("wm_trigger".to_string()).or_insert_with(|| {
to_raw_value(&serde_json::json!({
"kind": "webhook",
}))
});
}
// this is a new flow being pushed, status is set to `value`.
backend/windmill-queue/src/schedule.rs
+9 -7
View File
@@ -15,8 +15,10 @@ use std::str::FromStr;
use windmill_common::db::Authed;
use windmill_common::ee::LICENSE_KEY_VALID;
use windmill_common::flows::Retry;
use windmill_common::get_latest_flow_version_info_for_path;
use windmill_common::jobs::JobPayload;
use windmill_common::schedule::schedule_to_user;
use windmill_common::FlowVersionInfo;
use windmill_common::DB;
use windmill_common::{
error::{self, Result},
@@ -114,21 +116,21 @@ pub async fn push_scheduled_job<'c>(
}
let (payload, tag, timeout, on_behalf_of_email, created_by) = if schedule.is_flow {
let r = sqlx::query!(
"SELECT tag, dedicated_worker, on_behalf_of_email, edited_by from flow WHERE path = $1 and workspace_id = $2",
&schedule.script_path,
let FlowVersionInfo {
version, tag, dedicated_worker, on_behalf_of_email, edited_by, ..
} = get_latest_flow_version_info_for_path(
&mut *tx,
&schedule.workspace_id,
&schedule.script_path,
false,
)
.fetch_optional(&mut *tx)
.await?;
let (tag, dedicated_worker, on_behalf_of_email, edited_by) = r
.map(|x| (x.tag, x.dedicated_worker, x.on_behalf_of_email, x.edited_by))
.unwrap_or_else(|| (None, None, None, "".to_string()));
(
JobPayload::Flow {
path: schedule.script_path.clone(),
dedicated_worker,
apply_preprocessor: false,
version,
},
tag,
None,
backend/windmill-worker/src/worker_flow.rs
+18 -20
View File
@@ -26,7 +26,6 @@ use sqlx::types::Json;
use sqlx::{FromRow, Postgres, Transaction};
use tracing::instrument;
use uuid::Uuid;
use windmill_common::add_time;
use windmill_common::auth::JobPerms;
#[cfg(feature = "benchmark")]
use windmill_common::bench::BenchmarkIter;
@@ -37,13 +36,16 @@ use windmill_common::flow_status::{
};
use windmill_common::flows::{add_virtual_items_if_necessary, Branch, FlowNodeId, StopAfterIf};
use windmill_common::jobs::{
script_hash_to_tag_and_limits, script_path_to_payload, JobKind, JobPayload, OnBehalfOf,
RawCode, ENTRYPOINT_OVERRIDE,
script_path_to_payload, JobKind, JobPayload, OnBehalfOf, RawCode, ENTRYPOINT_OVERRIDE,
};
use windmill_common::scripts::ScriptHash;
use windmill_common::users::username_to_permissioned_as;
use windmill_common::utils::WarnAfterExt;
use windmill_common::worker::to_raw_value;
use windmill_common::{
add_time, get_latest_flow_version_info_for_path, get_script_info_for_hash, FlowVersionInfo,
ScriptHashInfo,
};
use windmill_common::{
error::{self, to_anyhow, Error},
flow_status::{
@@ -3869,20 +3871,15 @@ async fn flow_to_payload(
w_id: &str,
db: &DB,
) -> Result<JobPayloadWithTag, Error> {
let record = sqlx::query!(
"SELECT on_behalf_of_email, edited_by FROM flow WHERE path = $1 AND workspace_id = $2",
path,
w_id,
)
.fetch_one(db)
.await
.map_err(|e| Error::NotFound(format!("fetching flow: {e:#}")))?;
let on_behalf_of = if let Some(email) = record.on_behalf_of_email {
Some(OnBehalfOf { email, permissioned_as: username_to_permissioned_as(&record.edited_by) })
let FlowVersionInfo { version, on_behalf_of_email, edited_by, .. } =
get_latest_flow_version_info_for_path(db, w_id, &path, true).await?;
let on_behalf_of = if let Some(email) = on_behalf_of_email {
Some(OnBehalfOf { email, permissioned_as: username_to_permissioned_as(&edited_by) })
} else {
None
};
let payload = JobPayload::Flow { path, dedicated_worker: None, apply_preprocessor: false };
let payload =
JobPayload::Flow { path, dedicated_worker: None, apply_preprocessor: false, version };
Ok(JobPayloadWithTag { payload, tag: None, delete_after_use, timeout: None, on_behalf_of })
}
@@ -3912,9 +3909,9 @@ async fn script_to_payload(
} else {
let hash = script_hash.unwrap();
let mut tx: sqlx::Transaction<'_, sqlx::Postgres> = db.begin().await?;
let (
let ScriptHashInfo {
tag,
custom_concurrency_key,
concurrency_key,
concurrent_limit,
concurrency_time_window_s,
cache_ttl,
@@ -3922,10 +3919,11 @@ async fn script_to_payload(
dedicated_worker,
priority,
delete_after_use,
script_timeout,
timeout,
on_behalf_of_email,
created_by,
) = script_hash_to_tag_and_limits(&hash, &mut tx, &flow_job.workspace_id).await?;
..
} = get_script_info_for_hash(&mut *tx, &flow_job.workspace_id, hash.0).await?;
let on_behalf_of = if let Some(email) = on_behalf_of_email {
Some(OnBehalfOf { email, permissioned_as: username_to_permissioned_as(&created_by) })
} else {
@@ -3935,7 +3933,7 @@ async fn script_to_payload(
JobPayload::ScriptHash {
hash,
path: script_path,
custom_concurrency_key,
custom_concurrency_key: concurrency_key,
concurrent_limit,
concurrency_time_window_s,
cache_ttl: module.cache_ttl.map(|x| x as i32).ok_or(cache_ttl).ok(),
@@ -3946,7 +3944,7 @@ async fn script_to_payload(
},
tag_override.to_owned().or(tag),
delete_after_use,
script_timeout,
timeout,
on_behalf_of,
)
};
backend/windmill-worker/src/worker_lockfiles.rs
+3 -3
View File
@@ -544,9 +544,9 @@ async fn trigger_dependents_to_recompute_dependencies(
match r {
Ok(r) => JobPayload::Dependencies {
path: s.importer_path.clone(),
hash: r.0,
language: r.6,
dedicated_worker: r.7,
hash: ScriptHash(r.hash),
language: r.language,
dedicated_worker: r.dedicated_worker,
},
Err(err) => {
tracing::error!(
frontend/package-lock.json
Generated
+4 -2
View File
@@ -75,7 +75,7 @@
"windmill-parser-wasm-py": "^1.477.1",
"windmill-parser-wasm-regex": "^1.481.0",
"windmill-parser-wasm-rust": "^1.429.0",
"windmill-parser-wasm-ts": "^1.438.2",
"windmill-parser-wasm-ts": "^1.486.1",
"windmill-parser-wasm-yaml": "^1.429.0",
"windmill-sql-datatype-parser-wasm": "^1.318.0",
"y-monaco": "^0.1.4",
@@ -10913,7 +10913,9 @@
"version": "1.429.0"
},
"node_modules/windmill-parser-wasm-ts": {
"version": "1.438.2"
"version": "1.486.1",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.486.1.tgz",
"integrity": "sha512-nv7nPpZA5O0Zsve6W2Sm3mxCSxMqg2rfyYHNqozIpBaGhbe8SpgnbyrSaXKsGKSc/bpwpANHE7nVCp23pQUK6Q=="
},
"node_modules/windmill-parser-wasm-yaml": {
"version": "1.429.0"
frontend/package.json
+1 -1
View File
@@ -144,7 +144,7 @@
"windmill-parser-wasm-py": "^1.477.1",
"windmill-parser-wasm-regex": "^1.481.0",
"windmill-parser-wasm-rust": "^1.429.0",
"windmill-parser-wasm-ts": "^1.438.2",
"windmill-parser-wasm-ts": "^1.486.1",
"windmill-parser-wasm-yaml": "^1.429.0",
"windmill-sql-datatype-parser-wasm": "^1.318.0",
"y-monaco": "^0.1.4",
frontend/src/lib/components/ArgInput.svelte
+17 -8
View File
@@ -126,15 +126,18 @@
oneOf.length >= 2 &&
(!oneOfSelected || !oneOf.some((o) => o.title === oneOfSelected) || !value)
) {
if (value && value['label'] && oneOf.some((o) => o.title === value['label'])) {
const tagKey = oneOf.find((o) => Object.keys(o.properties ?? {}).includes('kind'))
? 'kind'
: 'label'
if (value && value[tagKey] && oneOf.some((o) => o.title === value[tagKey])) {
const existingValue = JSON.parse(JSON.stringify(value))
oneOfSelected = value['label']
oneOfSelected = value[tagKey]
await tick()
value = existingValue
} else {
const label = oneOf[0]['title']
oneOfSelected = label
value = { ...(typeof value === 'object' ? (value ?? {}) : {}), label }
const variantTitle = oneOf[0]['title']
oneOfSelected = variantTitle
value = { ...(typeof value === 'object' ? (value ?? {}) : {}), [tagKey]: variantTitle }
}
}
}
@@ -145,8 +148,11 @@
function onOneOfChange() {
const label = value?.['label']
const kind = value?.['kind']
if (label && oneOf && oneOf.some((o) => o.title == label) && oneOfSelected != label) {
oneOfSelected = label
} else if (kind && oneOf && oneOf.some((o) => o.title == kind) && oneOfSelected != kind) {
oneOfSelected = kind
}
}
@@ -820,7 +826,10 @@
for (const key of prevValueKeys) {
toKeep[key] = value[key]
}
value = { ...toKeep, label: detail }
const tagKey = oneOf.find((o) => Object.keys(o.properties ?? {}).includes('kind'))
? 'kind'
: 'label'
value = { ...toKeep, [tagKey]: detail }
}}
let:item
>
@@ -849,7 +858,7 @@
}}
bind:args={value}
dndType={`nested-${title}`}
hiddenArgs={['label']}
hiddenArgs={['label', 'kind']}
on:reorder={(e) => {
if (oneOf && oneOf[objIdx]) {
const keys = e.detail
@@ -868,7 +877,7 @@
{onlyMaskPassword}
{disablePortal}
{disabled}
hiddenArgs={['label']}
hiddenArgs={['label', 'kind']}
schema={{
properties: obj.properties,
order: obj.order,
frontend/src/lib/components/schema/FlowPropertyEditor.svelte
+7 -2
View File
@@ -53,7 +53,7 @@
return oneOf.map((v) => ({
...v,
properties: Object.fromEntries(
Object.entries(v.properties ?? {}).filter(([k, v]) => k !== 'label')
Object.entries(v.properties ?? {}).filter(([k, v]) => k !== 'label' && k !== 'kind')
)
}))
}
@@ -122,12 +122,17 @@
properties = structuredClone(changedSchema.properties)
order = structuredClone(changedSchema.order)
requiredProperty = structuredClone(changedSchema.required)
const tagKey = oneOf?.find((o) => Object.keys(o.properties ?? {}).includes('kind'))
? 'kind'
: 'label'
oneOf = changedSchema.oneOf?.map((v) => {
return {
...v,
properties: {
...(v.properties ?? {}),
label: {
[tagKey]: {
type: 'string',
enum: [v.title ?? '']
}
frontend/src/lib/components/schema/PropertyEditor.svelte
+3 -1
View File
@@ -55,7 +55,9 @@
oneOfSchemas = oneOf.map((obj) => {
return {
properties: obj.properties
? Object.fromEntries(Object.entries(obj.properties).filter(([k, v]) => k !== 'label'))
? Object.fromEntries(
Object.entries(obj.properties).filter(([k, v]) => k !== 'label' && k !== 'kind')
)
: {},
order: obj.order
}
frontend/src/lib/components/triggers/CaptureTable.svelte
+41 -21
View File
@@ -83,31 +83,41 @@
let capturesWithPayload: CaptureWithPayload[] = captures.map((capture) => {
let newCapture: CaptureWithPayload = { ...capture }
if (capture.payload === 'WINDMILL_TOO_BIG') {
const isLarge =
capture.main_args === 'WINDMILL_TOO_BIG' ||
capture.preprocessor_args === 'WINDMILL_TOO_BIG'
if (isLarge) {
newCapture = {
...capture,
payloadData: 'Too big to display here, select to view',
getFullCapture: () =>
CaptureService.getCapture({
workspace: $workspaceStore!,
id: capture.id
})
}
return newCapture
}
const trigger_extra = isObject(capture.trigger_extra) ? capture.trigger_extra : {}
newCapture.payloadData =
kind === 'preprocessor'
? capture.payload === 'WINDMILL_TOO_BIG'
? {
payload: capture.payload,
...trigger_extra
}
: typeof capture.payload === 'object'
const preprocessor_args = isObject(capture.preprocessor_args)
? capture.preprocessor_args
: {}
if ('wm_trigger' in preprocessor_args) {
// v1
newCapture.payloadData =
kind === 'preprocessor'
? typeof capture.main_args === 'object'
? {
...capture.payload,
...trigger_extra
...capture.main_args,
...preprocessor_args
}
: trigger_extra
: capture.payload
: preprocessor_args
: capture.main_args
} else {
// v2
newCapture.payloadData = kind === 'preprocessor' ? preprocessor_args : capture.main_args
}
return newCapture
})
@@ -138,13 +148,23 @@
let payloadData: any = {}
if (capture.getFullCapture) {
const fullCapture = await capture.getFullCapture()
payloadData =
testKind === 'preprocessor'
? {
...(typeof fullCapture.payload === 'object' ? fullCapture.payload : {}),
...(typeof fullCapture.trigger_extra === 'object' ? fullCapture.trigger_extra : {})
}
: fullCapture.payload
const preprocessor_args = isObject(fullCapture.preprocessor_args)
? fullCapture.preprocessor_args
: {}
if ('wm_trigger' in preprocessor_args) {
// v1
payloadData =
testKind === 'preprocessor'
? {
...(typeof fullCapture.main_args === 'object' ? fullCapture.main_args : {}),
...preprocessor_args
}
: fullCapture.main_args
} else {
// v2
payloadData =
testKind === 'preprocessor' ? fullCapture.preprocessor_args : fullCapture.main_args
}
} else {
payloadData = structuredClone(capture.payloadData)
}
frontend/src/lib/components/triggers/gcp/GcpTriggerEditorInner.svelte
+1 -1
View File
@@ -263,7 +263,7 @@
btnClasses="ml-4 mt-2"
color="dark"
size="xs"
href={itemKind === 'flow' ? '/flows/add?hub=68' : '/scripts/add?hub=hub%2F14251'}
href={itemKind === 'flow' ? '/flows/add?hub=68' : '/scripts/add?hub=hub%2F11446'}
target="_blank">Create from template</Button
>
{/if}
frontend/src/lib/components/triggers/http/RouteBodyTransformerOption.svelte
+3 -1
View File
@@ -30,8 +30,10 @@
<svelte:fragment slot="header">
<Tooltip
documentationLink="https://www.windmill.dev/docs/core_concepts/http_routing#body-processing-options"
>Wraps the payload in an object under the 'body' key, useful for handling unknown payloads.</Tooltip
>
Wraps the payload in an object under the 'body' key, useful for handling unknown payloads.
Note that this will have no effect when using a preprocessor.
</Tooltip>
</svelte:fragment>
<svelte:fragment slot="action">
<Toggle
frontend/src/lib/components/triggers/http/utils.ts
+2 -2
View File
@@ -2,7 +2,7 @@ import { base } from '$lib/base'
import { isCloudHosted } from '$lib/cloud'
export const SECRET_KEY_PATH = 'secret_key_path'
export const HUB_SCRIPT_ID = 11663
export const HUB_SCRIPT_ID = 19661
export const SIGNATURE_TEMPLATE_SCRIPT_HUB_PATH: string = `hub/${HUB_SCRIPT_ID}`
export const SIGNATURE_TEMPLATE_FLOW_HUB_ID = '67'
@@ -10,7 +10,7 @@ export function getHttpRoute(
route_prefix: string,
route_path: string | undefined,
workspaced_route: boolean,
workspace_id: string,
workspace_id: string
) {
return `${location.origin}${base}/api/${route_prefix}/${
isCloudHosted() || workspaced_route ? workspace_id + '/' : ''
frontend/src/lib/components/triggers/kafka/KafkaTriggerEditorInner.svelte
+1 -1
View File
@@ -235,7 +235,7 @@
btnClasses="ml-4 mt-2"
color="dark"
size="xs"
href={itemKind === 'flow' ? '/flows/add?hub=65' : '/scripts/add?hub=hub%2F11635'}
href={itemKind === 'flow' ? '/flows/add?hub=65' : '/scripts/add?hub=hub%2F19659'}
target="_blank">Create from template</Button
>
{/if}
frontend/src/lib/components/triggers/mqtt/MqttTriggerEditorInner.svelte
+1 -1
View File
@@ -239,7 +239,7 @@
btnClasses="ml-4 mt-2"
color="dark"
size="xs"
href={itemKind === 'flow' ? '/flows/add?hub=61' : '/scripts/add?hub=hub%2F11619'}
href={itemKind === 'flow' ? '/flows/add?hub=61' : '/scripts/add?hub=hub%2F19655'}
target="_blank">Create from template</Button
>
{/if}
frontend/src/lib/components/triggers/nats/NatsTriggerEditorInner.svelte
+3 -3
View File
@@ -66,8 +66,8 @@
args.nats_resource_path = nDefaultValues?.nats_resource_path ?? ''
args.subjects = nDefaultValues?.subjects ?? ['']
args.use_jetstream = nDefaultValues?.use_jetstream ?? false
args.stream_name = args.use_jetstream ? nDefaultValues?.stream_name ?? '' : undefined
args.consumer_name = args.use_jetstream ? nDefaultValues?.consumer_name ?? '' : undefined
args.stream_name = args.use_jetstream ? (nDefaultValues?.stream_name ?? '') : undefined
args.consumer_name = args.use_jetstream ? (nDefaultValues?.consumer_name ?? '') : undefined
initialScriptPath = ''
fixedScriptPath = fixedScriptPath_ ?? ''
script_path = fixedScriptPath
@@ -242,7 +242,7 @@
btnClasses="ml-4 mt-2"
color="dark"
size="xs"
href={itemKind === 'flow' ? '/flows/add?hub=66' : '/scripts/add?hub=hub%2F11634'}
href={itemKind === 'flow' ? '/flows/add?hub=66' : '/scripts/add?hub=hub%2F19663'}
target="_blank">Create from template</Button
>
{/if}
frontend/src/lib/components/triggers/sqs/SqsTriggerEditorInner.svelte
+1 -1
View File
@@ -225,7 +225,7 @@
btnClasses="ml-4 mt-2"
color="dark"
size="xs"
href={itemKind === 'flow' ? '/flows/add?hub=59' : '/scripts/add?hub=hub%2F11637'}
href={itemKind === 'flow' ? '/flows/add?hub=59' : '/scripts/add?hub=hub%2F19657'}
target="_blank">Create from template</Button
>
{/if}
frontend/src/lib/components/triggers/websocket/WebsocketTriggerEditorInner.svelte
+1 -1
View File
@@ -295,7 +295,7 @@
btnClasses="ml-4 mt-2"
color="dark"
size="xs"
href={itemKind === 'flow' ? '/flows/add?hub=64' : '/scripts/add?hub=hub%2F11636'}
href={itemKind === 'flow' ? '/flows/add?hub=64' : '/scripts/add?hub=hub%2F19660'}
target="_blank">Create from template</Button
>
{/if}
frontend/src/lib/script_helpers.ts
+170 -156
View File
@@ -631,15 +631,6 @@ export const TS_PREPROCESSOR_SCRIPT_INTRO = `/**
* It processes raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, Postgres, or email)
* before passing it to \`main\`. This separates the trigger logic from the main logic and keeps the auto-generated runnable UI clean.
*
* The preprocessor receives trigger metadata (\`wm_trigger\`) along with the main trigger arguments.
* The structure of \`wm_trigger\` and the main trigger arguments are specific to each trigger type:
* - Webhook/HTTP: \`(wm_trigger: { kind: 'http' | 'webhook', http?: { ... } }, body_key_1: any, body_key_2: any, ...)\`
* - Postgres: \`(wm_trigger: { kind: 'postgres' }, transaction_type: string, schema_name: string, table_name: string, old_row?: any, row: any)\`
* - WebSocket/Kafka/NATS/SQS: \`(wm_trigger: { kind: 'websocket' | 'kafka' | 'nats' | 'sqs', [kind]: { ... } }, msg: string)\`
* - MQTT: \`(wm_trigger: { kind: 'mqtt', [kind]: { ... } }, payload: Array<number>)\`
* - GCP: \`(wm_trigger: { kind: 'gcp', [kind]: { ... } }, payload: string)\`
* - Email: \`(wm_trigger: { kind: 'email' }, raw_email: string, parsed_email: { ... })\`
*
* The returned object defines the parameter values passed to \`main()\`.
* e.g., { b: 1, a: 2 } → Calls \`main(2, 1)\`, assuming \`main\` is defined as \`main(a: number, b: number)\`.
* Ensure that the parameter names in \`main\` match the keys in the returned object.
@@ -652,15 +643,6 @@ export const TS_PREPROCESSOR_FLOW_INTRO = `/**
*
* It processes raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, Postgres, or email)
* before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean.
*
* The preprocessor receives trigger metadata (\`wm_trigger\`) along with the main trigger arguments.
* The structure of \`wm_trigger\` and the main trigger arguments are specific to each trigger type:
* - Webhook/HTTP: \`(wm_trigger: { kind: 'http' | 'webhook', http?: { ... } }, body_key_1: any, body_key_2: any, ...)\`
* - Postgres: \`(wm_trigger: { kind: 'postgres' }, transaction_type: string, schema_name: string, table_name: string, old_row?: any, row: any)\`
* - WebSocket/Kafka/NATS/SQS: \`(wm_trigger: { kind: 'websocket' | 'kafka' | 'nats' | 'sqs', [kind]: { ... } }, msg: string)\`
* - MQTT: \`(wm_trigger: { kind: 'mqtt', [kind]: { ... } }, payload: Array<number>)\`
* - GCP: \`(wm_trigger: { kind: 'gcp', [kind]: { ... } }, payload: string)\`*
* - Email: \`(wm_trigger: { kind: 'email' }, raw_email: string, parsed_email: { ... })\`
*
* The returned object determines the parameter values passed to the flow.
* e.g., \`{ b: 1, a: 2 }\` → Calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`.
@@ -670,77 +652,92 @@ export const TS_PREPROCESSOR_FLOW_INTRO = `/**
*/\n`
export const TS_PREPROCESSOR_MODULE_CODE = `export async function preprocessor(
/*
* Replace this comment with the parameters received from the trigger.
* Examples: \`bodyKey1\`, \`bodyKey2\` for Webhook/HTTP, \`msg\` for WebSocket, etc.
*/
// The trigger metadata
wm_trigger: {
kind: 'http' | 'email' | 'webhook' | 'websocket' | 'kafka' | 'nats' | 'postgres' | 'sqs' | 'mqtt' | 'gcp',
http?: {
route: string // The route path, e.g. "/users/:id"
path: string // The actual path called, e.g. "/users/123"
method: string
params: Record<string, string> // path parameters
query: Record<string, string> // query parameters
headers: Record<string, string>
},
websocket?: {
url: string // The websocket url
},
kafka?: {
brokers: string[]
topic: string
group_id: string
},
nats?: {
servers: string[]
subject: string
headers?: Record<string, string[]>
status?: number
description?: string
length: number
},
sqs?: {
queue_url: string,
message_id?: string,
receipt_handle?: string,
attributes: Record<string, string>,
message_attributes?: Record<string, {
string_value?: string,
data_type: string
}>
},
mqtt?: {
topic: string,
retain: boolean,
pkid: number,
qos: number,
v5?: {
payload_format_indicator?: number,
topic_alias?: number,
response_topic?: string,
correlation_data?: Array<number>,
user_properties?: Array<[string, string]>,
subscription_identifiers?: Array<number>,
content_type?: string
event:
| {
kind: "webhook";
body: any,
raw_string: string | null,
query: Record<string, string>;
headers: Record<string, string>;
}
| {
kind: "http";
body: any,
raw_string: string | null,
route: string;
path: string;
method: string;
params: Record<string, string>;
query: Record<string, string>;
headers: Record<string, string>;
}
| {
kind: "email";
parsed_email: any,
raw_email: string,
}
| { kind: "websocket"; msg: string; url: string }
| {
kind: "kafka";
payload: string;
brokers: string[];
topic: string;
group_id: string;
}
| {
kind: "nats";
payload: string;
servers: string[];
subject: string;
headers?: Record<string, string[]>;
status?: number;
description?: string;
length: number;
}
| {
kind: "sqs";
msg: string,
queue_url: string;
message_id?: string;
receipt_handle?: string;
attributes: Record<string, string>;
message_attributes?: Record<
string,
{ string_value?: string; data_type: string }
>;
}
| {
kind: "mqtt";
payload: string,
topic: string;
retain: boolean;
pkid: number;
qos: number;
v5?: {
payload_format_indicator?: number;
topic_alias?: number;
response_topic?: string;
correlation_data?: Array<number>;
user_properties?: Array<[string, string]>;
subscription_identifiers?: Array<number>;
content_type?: string;
};
}
| {
kind: "gcp";
payload: string,
message_id: string;
subscription: string;
ordering_key?: string;
attributes?: Record<string, string>;
delivery_type: "push" | "pull";
headers?: Record<string, string>;
publish_time?: string;
}
},
gcp?: {
message_id: string,
subscription: string,
ordering_key?: string,
attributes?: Record<string, string>,
delivery_type: "push" | "pull",
headers?: Record<string, string>,
publish_time?: string,
}
}
) {
return {
// return the args to be passed to the runnable
}
};
}
`
@@ -778,15 +775,6 @@ export const PYTHON_PREPROCESSOR_SCRIPT_INTRO = `# Trigger preprocessor
# It processes raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, Postgres, or email)
# before passing it to \`main\`. This separates the trigger logic from the main logic and keeps the auto-generated UI clean.
#
# The preprocessor receives trigger metadata (\`wm_trigger\`) along with the main trigger arguments.
# The structure of \`wm_trigger\` and the main trigger arguments are specific to each trigger type:
# - Webhook/HTTP: \`(wm_trigger: { kind: 'http' | 'webhook', http?: { ... } }, body_key_1: any, body_key_2: any, ...)\`
# - Postgres: \`(wm_trigger: { kind: 'postgres' }, transaction_type: string, schema_name: string, table_name: string, old_row?: any, row: any)\`
# - WebSocket/Kafka/NATS/SQS: \`(wm_trigger: { kind: 'websocket' | 'kafka' | 'nats' | 'sqs', [kind]: { ... } }, msg: string)\`
# - MQTT: \`(wm_trigger: { kind: 'mqtt', [kind]: { ... } }, payload: Array<number>)\`
# - GCP: \`(wm_trigger: { kind: 'gcp', [kind]: { ... } }, payload: string)\`
# - Email: \`(wm_trigger: { kind: 'email' }, raw_email: string, parsed_email: { ... })\`
#
# The returned object defines the parameter values passed to \`main()\`.
# e.g., { b: 1, a: 2 } → Calls \`main(2, 1)\`, assuming \`main\` is defined as \`main(a: int, b: int)\`.
# Ensure that the parameter names in \`main\` match the keys in the returned object.
@@ -797,15 +785,6 @@ export const PYTHON_PREPROCESSOR_FLOW_INTRO = `# Trigger preprocessor
#
# It processes raw trigger data from various sources (webhook, custom HTTP route, SQS, WebSocket, Kafka, NATS, MQTT, Postgres, or email)
# before passing it to the flow. This separates the trigger logic from the flow logic and keeps the auto-generated UI clean.
#
# The preprocessor receives the same data the flow would if no preprocessor was used,
# plus trigger metadata in the \`wm_trigger\` object:
# - Webhook/HTTP: \`(wm_trigger: { kind: 'http' | 'webhook', http?: { ... } }, body_key_1: any, body_key_2: any, ...)\`
# - Postgres: \`(wm_trigger: { kind: 'postgres' }, transaction_type: string, schema_name: string, table_name: string, old_row?:any, row: any)\`
# - WebSocket/Kafka/NATS/SQS: \`(wm_trigger: { kind: 'websocket' | 'kafka' | 'nats' | 'sqs', [kind]: { ... } }, msg: string)\`
# - MQTT: \`(wm_trigger: { kind: 'mqtt', [kind]: { ... } }, payload: Array<number>)\`
# - GCP: \`(wm_trigger: { kind: 'gcp', [kind]: { ... } }, payload: string)\`
# - Email: \`(wm_trigger: { kind: 'email' }, raw_email: string, parsed_email: { ... })\`
#
# The returned object determines the parameter values passed to the flow.
# e.g., \`{ b: 1, a: 2 }\` → Calls the flow with \`a = 2\` and \`b = 1\`, assuming the flow has two inputs called \`a\` and \`b\`.
@@ -813,85 +792,120 @@ export const PYTHON_PREPROCESSOR_FLOW_INTRO = `# Trigger preprocessor
#
# Learn more: https://www.windmill.dev/docs/core_concepts/preprocessors\n\n`
export const PYTHON_PREPROCESSOR_MODULE_CODE = `from typing import TypedDict, Literal
class Http(TypedDict):
route: str # The route path, e.g. "/users/:id"
path: str # The actual path called, e.g. "/users/123"
export const PYTHON_PREPROCESSOR_MODULE_CODE = `from typing import TypedDict, Literal, Optional, Union
class WebhookEvent(TypedDict):
kind: Literal["webhook"]
body: dict
raw_string: Optional[str]
query: dict[str, str]
headers: dict[str, str]
class HttpEvent(TypedDict):
kind: Literal["http"]
body: dict
raw_string: Optional[str]
route: str
path: str
method: str
params: dict[str, str]
query: dict[str, str]
headers: dict[str, str]
class Websocket(TypedDict):
url: str # The websocket url
class Kafka(TypedDict):
topic: str
class EmailEvent(TypedDict):
kind: Literal["email"]
parsed_email: dict
raw_email: str
class WebsocketEvent(TypedDict):
kind: Literal["websocket"]
msg: str
url: str
class KafkaEvent(TypedDict):
kind: Literal["kafka"]
payload: str
brokers: list[str]
topic: str
group_id: str
class Nats(TypedDict):
class NatsEvent(TypedDict):
kind: Literal["nats"]
payload: str
servers: list[str]
subject: str
headers: dict[str, list[str]] | None
status: int | None
description: str | None
headers: Optional[dict[str, list[str]]]
status: Optional[int]
description: Optional[str]
length: int
class MessageAttribute(TypedDict):
string_value: str | None
string_value: Optional[str]
data_type: str
class Sqs(TypedDict):
class SqsEvent(TypedDict):
kind: Literal["sqs"]
msg: str
queue_url: str
message_id: str | None
receipt_handle: str | None
message_id: Optional[str]
receipt_handle: Optional[str]
attributes: dict[str, str]
message_attributes: dict[str, MessageAttribute] | None
message_attributes: Optional[dict[str, MessageAttribute]]
class MqttV5Properties:
payload_format_indicator: int | None
topic_alias: int | None
response_topic: str | None
correlation_data: list[int] | None
user_properties: list[tuple[str, str]] | None
subscription_identifiers: list[int] | None
content_type: str | None
class Mqtt(TypedDict):
class MqttV5Properties(TypedDict, total=False):
payload_format_indicator: Optional[int]
topic_alias: Optional[int]
response_topic: Optional[str]
correlation_data: Optional[list[int]]
user_properties: Optional[list[tuple[str, str]]]
subscription_identifiers: Optional[list[int]]
content_type: Optional[str]
class MqttEvent(TypedDict):
kind: Literal["mqtt"]
payload: str
topic: str
retain: bool
pkid: int
qos: int
v5: MqttV5Properties | None
v5: Optional[MqttV5Properties]
class Gcp(TypedDict):
class GcpEvent(TypedDict):
kind: Literal["gcp"]
payload: str
message_id: str
subscription: str
ordering_key: str | None
attributes: dict[str, str] | None
ordering_key: Optional[str]
attributes: Optional[dict[str, str]]
delivery_type: Literal["push", "pull"]
headers: dict[str, str] | None
publish_time: str | None
class WmTrigger(TypedDict):
kind: Literal["http", "email", "webhook", "websocket", "kafka", "nats", "postgres", "sqs", "mqtt", "gcp"]
http: Http | None
websocket: Websocket | None
kafka: Kafka | None
nats: Nats | None
sqs: Sqs | None
mqtt: Mqtt | None
gcp: Gcp | None
headers: Optional[dict[str, str]]
publish_time: Optional[str]
def preprocessor(
# Replace this comment with the parameters received from the trigger.
# Examples: \`bodyKey1\`, \`bodyKey2\` for Webhook/HTTP, \`msg\` for WebSocket, etc.
Event = Union[
WebhookEvent,
HttpEvent,
EmailEvent,
WebsocketEvent,
KafkaEvent,
NatsEvent,
SqsEvent,
MqttEvent,
GcpEvent,
]
# Trigger metadata
wm_trigger: WmTrigger,
):
def preprocessor(event: Event):
return {
# return the args to be passed to the runnable
}
@@ -1037,7 +1051,7 @@ public class Main {
}
}
`
// for related places search: ADD_NEW_LANG
// for related places search: ADD_NEW_LANG
export const INITIAL_CODE = {
bun: {
scriptInitCodeBlock: BUN_INIT_BLOCK,
@@ -1124,8 +1138,8 @@ export const INITIAL_CODE = {
},
java: {
script: JAVA_INIT_CODE
},
// for related places search: ADD_NEW_LANG
}
// for related places search: ADD_NEW_LANG
}
export function isInitialCode(content: string): boolean {
@@ -1231,7 +1245,7 @@ export function initialCode(
return INITIAL_CODE.nu.script
} else if (language == 'java') {
return INITIAL_CODE.java.script
// for related places search: ADD_NEW_LANG
// for related places search: ADD_NEW_LANG
} else if (language == 'bun' || language == 'bunnative') {
if (kind == 'trigger') {
return INITIAL_CODE.bun.trigger