diff --git a/backend/.sqlx/query-02c945b5f18a56a826721f6884846d79167747742de236ce57f395561685adc0.json b/backend/.sqlx/query-02c945b5f18a56a826721f6884846d79167747742de236ce57f395561685adc0.json new file mode 100644 index 0000000000..8fd07f3bf9 --- /dev/null +++ b/backend/.sqlx/query-02c945b5f18a56a826721f6884846d79167747742de236ce57f395561685adc0.json @@ -0,0 +1,77 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n v2_job.permissioned_as_email,\n v2_job.created_by,\n v2_job.parent_job,\n v2_job.permissioned_as,\n v2_job.runnable_path,\n CASE WHEN v2_job.trigger_kind = 'schedule'::job_trigger_kind THEN v2_job.trigger END AS schedule_path,\n v2_job.flow_step_id,\n v2_job.flow_innermost_root_job,\n v2_job.root_job,\n v2_job_queue.scheduled_for AS \"scheduled_for: chrono::DateTime\"\n FROM v2_job INNER JOIN v2_job_queue ON v2_job.id = v2_job_queue.id\n WHERE v2_job.id = $1 AND v2_job.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "permissioned_as_email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "created_by", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "parent_job", + "type_info": "Uuid" + }, + { + "ordinal": 3, + "name": "permissioned_as", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "schedule_path", + "type_info": "Varchar" + }, + { + "ordinal": 6, + "name": "flow_step_id", + "type_info": "Varchar" + }, + { + "ordinal": 7, + "name": "flow_innermost_root_job", + "type_info": "Uuid" + }, + { + "ordinal": 8, + "name": "root_job", + "type_info": "Uuid" + }, + { + "ordinal": 9, + "name": "scheduled_for: chrono::DateTime", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + false, + false, + true, + false, + true, + null, + true, + true, + true, + false + ] + }, + "hash": "02c945b5f18a56a826721f6884846d79167747742de236ce57f395561685adc0" +} diff --git a/backend/.sqlx/query-0382065b3dfd78b384e26f81317af91de289f52462e74343770a8b0d47d0577d.json b/backend/.sqlx/query-0382065b3dfd78b384e26f81317af91de289f52462e74343770a8b0d47d0577d.json new file mode 100644 index 0000000000..7e2371e9d8 --- /dev/null +++ b/backend/.sqlx/query-0382065b3dfd78b384e26f81317af91de289f52462e74343770a8b0d47d0577d.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO kafka_trigger (\n workspace_id,\n path,\n kafka_resource_path,\n group_id,\n topics,\n script_path,\n is_flow,\n enabled,\n edited_by,\n email,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, now(), $11, $12, $13\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "VarcharArray", + "Varchar", + "Bool", + "Bool", + "Varchar", + "Varchar", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "0382065b3dfd78b384e26f81317af91de289f52462e74343770a8b0d47d0577d" +} diff --git a/backend/.sqlx/query-04a8482180a3f8e07761742f13ac30afc081eeed1151a0b5e043d785baebfc9d.json b/backend/.sqlx/query-04a8482180a3f8e07761742f13ac30afc081eeed1151a0b5e043d785baebfc9d.json deleted file mode 100644 index 2fe60e6570..0000000000 --- a/backend/.sqlx/query-04a8482180a3f8e07761742f13ac30afc081eeed1151a0b5e043d785baebfc9d.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n mqtt_trigger \n SET\n mqtt_resource_path = $1,\n subscribe_topics = $2,\n client_version = $3,\n client_id = $4,\n v3_config = $5,\n v5_config = $6,\n is_flow = $7, \n edited_by = $8, \n email = $9,\n script_path = $10,\n path = $11,\n edited_at = now(), \n error = NULL,\n server_id = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16\n WHERE \n workspace_id = $12 AND \n path = $13\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "JsonbArray", - { - "Custom": { - "name": "mqtt_client_version", - "kind": { - "Enum": [ - "v3", - "v5" - ] - } - } - }, - "Varchar", - "Jsonb", - "Jsonb", - "Bool", - "Varchar", - "Varchar", - "Varchar", - "Varchar", - "Text", - "Text", - "Varchar", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "04a8482180a3f8e07761742f13ac30afc081eeed1151a0b5e043d785baebfc9d" -} diff --git a/backend/.sqlx/query-06af027f6ed10200de2006a2fc48771a8f42c28c87b78220eec1bddeae1f648f.json b/backend/.sqlx/query-06af027f6ed10200de2006a2fc48771a8f42c28c87b78220eec1bddeae1f648f.json new file mode 100644 index 0000000000..8566af0709 --- /dev/null +++ b/backend/.sqlx/query-06af027f6ed10200de2006a2fc48771a8f42c28c87b78220eec1bddeae1f648f.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO postgres_trigger (\n workspace_id,\n path,\n postgres_resource_path,\n replication_slot_name,\n publication_name,\n script_path,\n is_flow,\n enabled,\n edited_by,\n email,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, now(), $11, $12, $13\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Bool", + "Varchar", + "Varchar", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "06af027f6ed10200de2006a2fc48771a8f42c28c87b78220eec1bddeae1f648f" +} diff --git a/backend/.sqlx/query-070b8ad0b59f485fa5bf68082b060f5c3561c37e9c6f2834d234a862a475a6eb.json b/backend/.sqlx/query-070b8ad0b59f485fa5bf68082b060f5c3561c37e9c6f2834d234a862a475a6eb.json deleted file mode 100644 index 280db517dd..0000000000 --- a/backend/.sqlx/query-070b8ad0b59f485fa5bf68082b060f5c3561c37e9c6f2834d234a862a475a6eb.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n gcp_trigger \n SET \n enabled = $1, \n email = $2, \n edited_by = $3, \n edited_at = now(), \n server_id = NULL, \n error = NULL\n WHERE \n path = $4 AND \n workspace_id = $5 \n RETURNING 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Bool", - "Varchar", - "Varchar", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "070b8ad0b59f485fa5bf68082b060f5c3561c37e9c6f2834d234a862a475a6eb" -} diff --git a/backend/.sqlx/query-0d8415a79d8312db4d46da46fdf348ab17dee7b8358b6719fa399511962bc3e9.json b/backend/.sqlx/query-0d8415a79d8312db4d46da46fdf348ab17dee7b8358b6719fa399511962bc3e9.json deleted file mode 100644 index 2040b0ed88..0000000000 --- a/backend/.sqlx/query-0d8415a79d8312db4d46da46fdf348ab17dee7b8358b6719fa399511962bc3e9.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n nats_trigger\n SET\n nats_resource_path = $1,\n subjects = $2,\n stream_name = $3,\n consumer_name = $4,\n use_jetstream = $5,\n script_path = $6,\n path = $7,\n is_flow = $8,\n edited_by = $9,\n email = $10,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $13,\n error_handler_args = $14,\n retry = $15\n WHERE\n workspace_id = $11 AND path = $12\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "VarcharArray", - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Varchar", - "Text", - "Text", - "Varchar", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "0d8415a79d8312db4d46da46fdf348ab17dee7b8358b6719fa399511962bc3e9" -} diff --git a/backend/.sqlx/query-0e14ab95a08572f0672db266187335f578c622eb335cfc7cd0969633d85c9f73.json b/backend/.sqlx/query-0e14ab95a08572f0672db266187335f578c622eb335cfc7cd0969633d85c9f73.json new file mode 100644 index 0000000000..34208d501c --- /dev/null +++ b/backend/.sqlx/query-0e14ab95a08572f0672db266187335f578c622eb335cfc7cd0969633d85c9f73.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(root_job, flow_innermost_root_job, parent_job, id) as \"root_job!\" FROM v2_job WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "root_job!", + "type_info": "Uuid" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "0e14ab95a08572f0672db266187335f578c622eb335cfc7cd0969633d85c9f73" +} diff --git a/backend/.sqlx/query-0ef1e5bbbefc117a4cdaf414b3652354641c2f735d071540f858bc064f2432cd.json b/backend/.sqlx/query-0ef1e5bbbefc117a4cdaf414b3652354641c2f735d071540f858bc064f2432cd.json new file mode 100644 index 0000000000..a867ace2cb --- /dev/null +++ b/backend/.sqlx/query-0ef1e5bbbefc117a4cdaf414b3652354641c2f735d071540f858bc064f2432cd.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE nats_trigger \n SET \n nats_resource_path = $1,\n subjects = $2,\n stream_name = $3,\n consumer_name = $4,\n use_jetstream = $5,\n script_path = $6,\n path = $7,\n is_flow = $8,\n edited_by = $9,\n email = $10,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $13,\n error_handler_args = $14,\n retry = $15\n WHERE \n workspace_id = $11 AND path = $12\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "VarcharArray", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Text", + "Text", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "0ef1e5bbbefc117a4cdaf414b3652354641c2f735d071540f858bc064f2432cd" +} diff --git a/backend/.sqlx/query-0f697b1ab3105e2ea036f8ecace2d54f97bc2d0ef52f5812244a97c289523592.json b/backend/.sqlx/query-0f697b1ab3105e2ea036f8ecace2d54f97bc2d0ef52f5812244a97c289523592.json new file mode 100644 index 0000000000..ec215e0f29 --- /dev/null +++ b/backend/.sqlx/query-0f697b1ab3105e2ea036f8ecace2d54f97bc2d0ef52f5812244a97c289523592.json @@ -0,0 +1,39 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE \n mqtt_trigger \n SET\n mqtt_resource_path = $1,\n subscribe_topics = $2,\n client_version = $3,\n client_id = $4,\n v3_config = $5,\n v5_config = $6,\n is_flow = $7, \n edited_by = $8, \n email = $9,\n script_path = $10,\n path = $11,\n edited_at = now(), \n error = NULL,\n server_id = NULL,\n error_handler_path = $14,\n error_handler_args = $15,\n retry = $16\n WHERE \n workspace_id = $12 AND \n path = $13\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "JsonbArray", + { + "Custom": { + "name": "mqtt_client_version", + "kind": { + "Enum": [ + "v3", + "v5" + ] + } + } + }, + "Varchar", + "Jsonb", + "Jsonb", + "Bool", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Text", + "Text", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "0f697b1ab3105e2ea036f8ecace2d54f97bc2d0ef52f5812244a97c289523592" +} diff --git a/backend/.sqlx/query-13444bbd5547e101c41206c5f97ac4dded0536faf52c370d704ed9a451041caf.json b/backend/.sqlx/query-13444bbd5547e101c41206c5f97ac4dded0536faf52c370d704ed9a451041caf.json deleted file mode 100644 index 6b8e275167..0000000000 --- a/backend/.sqlx/query-13444bbd5547e101c41206c5f97ac4dded0536faf52c370d704ed9a451041caf.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT COUNT(*) FROM sqs_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text", - "Bool", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "13444bbd5547e101c41206c5f97ac4dded0536faf52c370d704ed9a451041caf" -} diff --git a/backend/.sqlx/query-1625a84fbcf8c5f77eb0519f60d9418f85d4d3d599f4177403fad5ad99380715.json b/backend/.sqlx/query-1625a84fbcf8c5f77eb0519f60d9418f85d4d3d599f4177403fad5ad99380715.json deleted file mode 100644 index f0ee89dbc9..0000000000 --- a/backend/.sqlx/query-1625a84fbcf8c5f77eb0519f60d9418f85d4d3d599f4177403fad5ad99380715.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n sqs_trigger \n SET \n enabled = $1, \n email = $2, \n edited_by = $3, \n edited_at = now(), \n server_id = NULL, \n error = NULL\n WHERE \n path = $4 AND \n workspace_id = $5 \n RETURNING 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Bool", - "Varchar", - "Varchar", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "1625a84fbcf8c5f77eb0519f60d9418f85d4d3d599f4177403fad5ad99380715" -} diff --git a/backend/.sqlx/query-1e333ccb7657bc721d18ea2230cb20f15e4a7839c87c853f280623bafb30b80c.json b/backend/.sqlx/query-1e333ccb7657bc721d18ea2230cb20f15e4a7839c87c853f280623bafb30b80c.json deleted file mode 100644 index 19ae194f04..0000000000 --- a/backend/.sqlx/query-1e333ccb7657bc721d18ea2230cb20f15e4a7839c87c853f280623bafb30b80c.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO nats_trigger (\n workspace_id,\n path,\n nats_resource_path,\n subjects,\n stream_name,\n consumer_name,\n use_jetstream,\n script_path,\n is_flow,\n enabled,\n edited_by,\n email,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now(), $13, $14, $15\n )\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Varchar", - "VarcharArray", - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Bool", - "Bool", - "Varchar", - "Varchar", - "Varchar", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "1e333ccb7657bc721d18ea2230cb20f15e4a7839c87c853f280623bafb30b80c" -} diff --git a/backend/.sqlx/query-208f30f8a570db41ee52f01a4402d76f7e174e39c45ede29dcd6aa9c65850fef.json b/backend/.sqlx/query-208f30f8a570db41ee52f01a4402d76f7e174e39c45ede29dcd6aa9c65850fef.json deleted file mode 100644 index 24744c96fb..0000000000 --- a/backend/.sqlx/query-208f30f8a570db41ee52f01a4402d76f7e174e39c45ede29dcd6aa9c65850fef.json +++ /dev/null @@ -1,153 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n mqtt_resource_path,\n subscribe_topics as \"subscribe_topics!: Vec>\",\n v3_config as \"v3_config!: Option>\",\n v5_config as \"v5_config!: Option>\",\n client_version AS \"client_version: _\",\n client_id,\n workspace_id,\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\"\n FROM \n mqtt_trigger\n WHERE \n workspace_id = $1 AND \n path = $2\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "mqtt_resource_path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "subscribe_topics!: Vec>", - "type_info": "JsonbArray" - }, - { - "ordinal": 2, - "name": "v3_config!: Option>", - "type_info": "Jsonb" - }, - { - "ordinal": 3, - "name": "v5_config!: Option>", - "type_info": "Jsonb" - }, - { - "ordinal": 4, - "name": "client_version: _", - "type_info": { - "Custom": { - "name": "mqtt_client_version", - "kind": { - "Enum": [ - "v3", - "v5" - ] - } - } - } - }, - { - "ordinal": 5, - "name": "client_id", - "type_info": "Varchar" - }, - { - "ordinal": 6, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 7, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 8, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 9, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 10, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 11, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 12, - "name": "edited_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 13, - "name": "server_id", - "type_info": "Varchar" - }, - { - "ordinal": 14, - "name": "last_server_ping", - "type_info": "Timestamptz" - }, - { - "ordinal": 15, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 16, - "name": "error", - "type_info": "Text" - }, - { - "ordinal": 17, - "name": "enabled", - "type_info": "Bool" - }, - { - "ordinal": 18, - "name": "error_handler_path", - "type_info": "Varchar" - }, - { - "ordinal": 19, - "name": "error_handler_args: _", - "type_info": "Jsonb" - }, - { - "ordinal": 20, - "name": "retry: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - false, - true, - true, - false, - true, - false, - false, - false, - false, - false, - false, - false, - true, - true, - false, - true, - false, - true, - true, - true - ] - }, - "hash": "208f30f8a570db41ee52f01a4402d76f7e174e39c45ede29dcd6aa9c65850fef" -} diff --git a/backend/.sqlx/query-2139f1fb1877294bbf55d786000c5c32f582a3911efcf88e437aa90d7d5a49b5.json b/backend/.sqlx/query-2139f1fb1877294bbf55d786000c5c32f582a3911efcf88e437aa90d7d5a49b5.json deleted file mode 100644 index fef0a60ca7..0000000000 --- a/backend/.sqlx/query-2139f1fb1877294bbf55d786000c5c32f582a3911efcf88e437aa90d7d5a49b5.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT COUNT(*) FROM kafka_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text", - "Bool", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "2139f1fb1877294bbf55d786000c5c32f582a3911efcf88e437aa90d7d5a49b5" -} diff --git a/backend/.sqlx/query-2238aaed46031f71b17c1cabcef935a0d3b2ca6d057f446ed9b02d4386e2ddd9.json b/backend/.sqlx/query-2238aaed46031f71b17c1cabcef935a0d3b2ca6d057f446ed9b02d4386e2ddd9.json deleted file mode 100644 index aa32886ba8..0000000000 --- a/backend/.sqlx/query-2238aaed46031f71b17c1cabcef935a0d3b2ca6d057f446ed9b02d4386e2ddd9.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT EXISTS(SELECT 1 FROM websocket_trigger WHERE path = $1 AND workspace_id = $2)", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "exists", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "2238aaed46031f71b17c1cabcef935a0d3b2ca6d057f446ed9b02d4386e2ddd9" -} diff --git a/backend/.sqlx/query-23eb4d45bf2df21e22fc6c9590b96b0a7dbdd27f85c7d886eded79b3af83731a.json b/backend/.sqlx/query-23eb4d45bf2df21e22fc6c9590b96b0a7dbdd27f85c7d886eded79b3af83731a.json new file mode 100644 index 0000000000..ecaefdeb62 --- /dev/null +++ b/backend/.sqlx/query-23eb4d45bf2df21e22fc6c9590b96b0a7dbdd27f85c7d886eded79b3af83731a.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO websocket_trigger (\n workspace_id,\n path,\n url,\n script_path,\n is_flow,\n enabled,\n filters,\n initial_messages,\n url_runnable_args,\n edited_by,\n can_return_message,\n email,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now(), $13, $14, $15\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Bool", + "JsonbArray", + "JsonbArray", + "Jsonb", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "23eb4d45bf2df21e22fc6c9590b96b0a7dbdd27f85c7d886eded79b3af83731a" +} diff --git a/backend/.sqlx/query-25975935d59f88df117bae0fa5016cf95ad56f790f0eddb6e2fbe3d83d3accaa.json b/backend/.sqlx/query-25975935d59f88df117bae0fa5016cf95ad56f790f0eddb6e2fbe3d83d3accaa.json deleted file mode 100644 index 4ae1f07aa8..0000000000 --- a/backend/.sqlx/query-25975935d59f88df117bae0fa5016cf95ad56f790f0eddb6e2fbe3d83d3accaa.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE websocket_trigger SET enabled = $1, email = $2, edited_by = $3, edited_at = now(), server_id = NULL, error = NULL\n WHERE path = $4 AND workspace_id = $5 RETURNING 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Bool", - "Varchar", - "Varchar", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "25975935d59f88df117bae0fa5016cf95ad56f790f0eddb6e2fbe3d83d3accaa" -} diff --git a/backend/.sqlx/query-264bc51ec372dfc6c1c4eb3608b32d8a250bd0d65ce33d7eb19eaea62466a803.json b/backend/.sqlx/query-264bc51ec372dfc6c1c4eb3608b32d8a250bd0d65ce33d7eb19eaea62466a803.json deleted file mode 100644 index 3e9ba6053e..0000000000 --- a/backend/.sqlx/query-264bc51ec372dfc6c1c4eb3608b32d8a250bd0d65ce33d7eb19eaea62466a803.json +++ /dev/null @@ -1,140 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n aws_auth_resource_type AS \"aws_auth_resource_type: _\",\n aws_resource_path,\n message_attributes,\n queue_url,\n workspace_id,\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\"\n FROM \n sqs_trigger\n WHERE \n workspace_id = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "aws_auth_resource_type: _", - "type_info": { - "Custom": { - "name": "aws_auth_resource_type", - "kind": { - "Enum": [ - "oidc", - "credentials" - ] - } - } - } - }, - { - "ordinal": 1, - "name": "aws_resource_path", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "message_attributes", - "type_info": "TextArray" - }, - { - "ordinal": 3, - "name": "queue_url", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 6, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 7, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 8, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 9, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 10, - "name": "edited_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 11, - "name": "server_id", - "type_info": "Varchar" - }, - { - "ordinal": 12, - "name": "last_server_ping", - "type_info": "Timestamptz" - }, - { - "ordinal": 13, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 14, - "name": "error", - "type_info": "Text" - }, - { - "ordinal": 15, - "name": "enabled", - "type_info": "Bool" - }, - { - "ordinal": 16, - "name": "error_handler_path", - "type_info": "Varchar" - }, - { - "ordinal": 17, - "name": "error_handler_args: _", - "type_info": "Jsonb" - }, - { - "ordinal": 18, - "name": "retry: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - false, - true, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - true, - false, - true, - true, - true - ] - }, - "hash": "264bc51ec372dfc6c1c4eb3608b32d8a250bd0d65ce33d7eb19eaea62466a803" -} diff --git a/backend/.sqlx/query-2e6165543e34216dfaedf6e10729f733cff812e62dea1c1bce8cefd3a3979b14.json b/backend/.sqlx/query-2e6165543e34216dfaedf6e10729f733cff812e62dea1c1bce8cefd3a3979b14.json deleted file mode 100644 index 4a15e2d4fa..0000000000 --- a/backend/.sqlx/query-2e6165543e34216dfaedf6e10729f733cff812e62dea1c1bce8cefd3a3979b14.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT COUNT(*) FROM websocket_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text", - "Bool", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "2e6165543e34216dfaedf6e10729f733cff812e62dea1c1bce8cefd3a3979b14" -} diff --git a/backend/.sqlx/query-31bc3dcea29be9cc0242771d25a232f173446d29c08fc29ddb8d55294f2c070e.json b/backend/.sqlx/query-31bc3dcea29be9cc0242771d25a232f173446d29c08fc29ddb8d55294f2c070e.json deleted file mode 100644 index 49fc84c349..0000000000 --- a/backend/.sqlx/query-31bc3dcea29be9cc0242771d25a232f173446d29c08fc29ddb8d55294f2c070e.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT COUNT(*) FROM http_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text", - "Bool", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "31bc3dcea29be9cc0242771d25a232f173446d29c08fc29ddb8d55294f2c070e" -} diff --git a/backend/.sqlx/query-33ee913ce263600a3f94f90e4a42cf0e4086030f3b7994e4892392765cbe1517.json b/backend/.sqlx/query-33ee913ce263600a3f94f90e4a42cf0e4086030f3b7994e4892392765cbe1517.json deleted file mode 100644 index 9b7581526e..0000000000 --- a/backend/.sqlx/query-33ee913ce263600a3f94f90e4a42cf0e4086030f3b7994e4892392765cbe1517.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT EXISTS(\n SELECT \n 1 \n FROM \n gcp_trigger \n WHERE \n path = $1 AND \n workspace_id = $2\n )", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "exists", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "33ee913ce263600a3f94f90e4a42cf0e4086030f3b7994e4892392765cbe1517" -} diff --git a/backend/.sqlx/query-3997dcf2c11817e59bf50fd896381d870b847a8ba07e6197212fdf85ec901b09.json b/backend/.sqlx/query-3997dcf2c11817e59bf50fd896381d870b847a8ba07e6197212fdf85ec901b09.json deleted file mode 100644 index 4c90c24dd6..0000000000 --- a/backend/.sqlx/query-3997dcf2c11817e59bf50fd896381d870b847a8ba07e6197212fdf85ec901b09.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM kafka_trigger WHERE workspace_id = $1 AND path = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "3997dcf2c11817e59bf50fd896381d870b847a8ba07e6197212fdf85ec901b09" -} diff --git a/backend/.sqlx/query-3a6c93a91d9a21aaf9e9709932ac810221c5fc8fc29ca4f1984c819e64e29437.json b/backend/.sqlx/query-3a6c93a91d9a21aaf9e9709932ac810221c5fc8fc29ca4f1984c819e64e29437.json deleted file mode 100644 index c820c649f4..0000000000 --- a/backend/.sqlx/query-3a6c93a91d9a21aaf9e9709932ac810221c5fc8fc29ca4f1984c819e64e29437.json +++ /dev/null @@ -1,125 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n workspace_id,\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled,\n replication_slot_name,\n publication_name,\n postgres_resource_path,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\"\n FROM \n postgres_trigger\n WHERE \n workspace_id = $1 AND \n path = $2\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 4, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 6, - "name": "edited_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 7, - "name": "server_id", - "type_info": "Varchar" - }, - { - "ordinal": 8, - "name": "last_server_ping", - "type_info": "Timestamptz" - }, - { - "ordinal": 9, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 10, - "name": "error", - "type_info": "Text" - }, - { - "ordinal": 11, - "name": "enabled", - "type_info": "Bool" - }, - { - "ordinal": 12, - "name": "replication_slot_name", - "type_info": "Varchar" - }, - { - "ordinal": 13, - "name": "publication_name", - "type_info": "Varchar" - }, - { - "ordinal": 14, - "name": "postgres_resource_path", - "type_info": "Varchar" - }, - { - "ordinal": 15, - "name": "error_handler_path", - "type_info": "Varchar" - }, - { - "ordinal": 16, - "name": "error_handler_args: _", - "type_info": "Jsonb" - }, - { - "ordinal": 17, - "name": "retry: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - true, - false, - false, - false, - false, - true, - true, - true - ] - }, - "hash": "3a6c93a91d9a21aaf9e9709932ac810221c5fc8fc29ca4f1984c819e64e29437" -} diff --git a/backend/.sqlx/query-3b6bd7b41f130ce6df62fdecb351a3e01be0726d02d3f863e0ea5c476a8e785e.json b/backend/.sqlx/query-3b6bd7b41f130ce6df62fdecb351a3e01be0726d02d3f863e0ea5c476a8e785e.json new file mode 100644 index 0000000000..fa18a36f44 --- /dev/null +++ b/backend/.sqlx/query-3b6bd7b41f130ce6df62fdecb351a3e01be0726d02d3f863e0ea5c476a8e785e.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE kafka_trigger \n SET \n kafka_resource_path = $1,\n group_id = $2,\n topics = $3,\n script_path = $4,\n path = $5,\n is_flow = $6,\n edited_by = $7,\n email = $8,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $11,\n error_handler_args = $12,\n retry = $13\n WHERE \n workspace_id = $9 AND path = $10\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "VarcharArray", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Text", + "Text", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "3b6bd7b41f130ce6df62fdecb351a3e01be0726d02d3f863e0ea5c476a8e785e" +} diff --git a/backend/.sqlx/query-3d8fdf7adf42b27808a5b223df3a660bfd5a8c0a451a0deac24632f0ccf66165.json b/backend/.sqlx/query-3d8fdf7adf42b27808a5b223df3a660bfd5a8c0a451a0deac24632f0ccf66165.json deleted file mode 100644 index c48927fdf2..0000000000 --- a/backend/.sqlx/query-3d8fdf7adf42b27808a5b223df3a660bfd5a8c0a451a0deac24632f0ccf66165.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE nats_trigger SET enabled = $1, email = $2, edited_by = $3, edited_at = now(), server_id = NULL, error = NULL\n WHERE path = $4 AND workspace_id = $5 RETURNING 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Bool", - "Varchar", - "Varchar", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "3d8fdf7adf42b27808a5b223df3a660bfd5a8c0a451a0deac24632f0ccf66165" -} diff --git a/backend/.sqlx/query-3ebf788b86d6aed59b8c61c08f1d9323e6c66107a1b7b37dadbbb7b91339e9d5.json b/backend/.sqlx/query-3ebf788b86d6aed59b8c61c08f1d9323e6c66107a1b7b37dadbbb7b91339e9d5.json deleted file mode 100644 index 493a2ba2ec..0000000000 --- a/backend/.sqlx/query-3ebf788b86d6aed59b8c61c08f1d9323e6c66107a1b7b37dadbbb7b91339e9d5.json +++ /dev/null @@ -1,124 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT \n workspace_id,\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled,\n replication_slot_name,\n publication_name,\n postgres_resource_path,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\"\n FROM \n postgres_trigger\n WHERE \n workspace_id = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 4, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 6, - "name": "edited_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 7, - "name": "server_id", - "type_info": "Varchar" - }, - { - "ordinal": 8, - "name": "last_server_ping", - "type_info": "Timestamptz" - }, - { - "ordinal": 9, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 10, - "name": "error", - "type_info": "Text" - }, - { - "ordinal": 11, - "name": "enabled", - "type_info": "Bool" - }, - { - "ordinal": 12, - "name": "replication_slot_name", - "type_info": "Varchar" - }, - { - "ordinal": 13, - "name": "publication_name", - "type_info": "Varchar" - }, - { - "ordinal": 14, - "name": "postgres_resource_path", - "type_info": "Varchar" - }, - { - "ordinal": 15, - "name": "error_handler_path", - "type_info": "Varchar" - }, - { - "ordinal": 16, - "name": "error_handler_args: _", - "type_info": "Jsonb" - }, - { - "ordinal": 17, - "name": "retry: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - true, - false, - false, - false, - false, - true, - true, - true - ] - }, - "hash": "3ebf788b86d6aed59b8c61c08f1d9323e6c66107a1b7b37dadbbb7b91339e9d5" -} diff --git a/backend/.sqlx/query-494d49e92dce7a1cd7074156704ccbbff020a2042409fc318e045852323cef5e.json b/backend/.sqlx/query-494d49e92dce7a1cd7074156704ccbbff020a2042409fc318e045852323cef5e.json deleted file mode 100644 index 25d6e0e6d3..0000000000 --- a/backend/.sqlx/query-494d49e92dce7a1cd7074156704ccbbff020a2042409fc318e045852323cef5e.json +++ /dev/null @@ -1,125 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT \n workspace_id,\n path,\n kafka_resource_path,\n group_id,\n topics,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\"\n FROM \n kafka_trigger \n WHERE \n workspace_id = $1 AND \n path = $2\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "kafka_resource_path", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "group_id", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "topics", - "type_info": "VarcharArray" - }, - { - "ordinal": 5, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 6, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 7, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 8, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 9, - "name": "edited_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 10, - "name": "server_id", - "type_info": "Varchar" - }, - { - "ordinal": 11, - "name": "last_server_ping", - "type_info": "Timestamptz" - }, - { - "ordinal": 12, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 13, - "name": "error", - "type_info": "Text" - }, - { - "ordinal": 14, - "name": "enabled", - "type_info": "Bool" - }, - { - "ordinal": 15, - "name": "error_handler_path", - "type_info": "Varchar" - }, - { - "ordinal": 16, - "name": "error_handler_args: _", - "type_info": "Jsonb" - }, - { - "ordinal": 17, - "name": "retry: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - false, - true, - false, - true, - true, - true - ] - }, - "hash": "494d49e92dce7a1cd7074156704ccbbff020a2042409fc318e045852323cef5e" -} diff --git a/backend/.sqlx/query-4bfa42153df81184e02ffc7ab6c8c591605211c070e52b2919cf9d7dede0ea16.json b/backend/.sqlx/query-4bfa42153df81184e02ffc7ab6c8c591605211c070e52b2919cf9d7dede0ea16.json deleted file mode 100644 index 94ea76acc2..0000000000 --- a/backend/.sqlx/query-4bfa42153df81184e02ffc7ab6c8c591605211c070e52b2919cf9d7dede0ea16.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE kafka_trigger\n SET\n kafka_resource_path = $1,\n group_id = $2,\n topics = $3,\n script_path = $4,\n path = $5,\n is_flow = $6,\n edited_by = $7,\n email = $8,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $11,\n error_handler_args = $12,\n retry = $13\n WHERE\n workspace_id = $9 AND path = $10\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "VarcharArray", - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Varchar", - "Text", - "Text", - "Varchar", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "4bfa42153df81184e02ffc7ab6c8c591605211c070e52b2919cf9d7dede0ea16" -} diff --git a/backend/.sqlx/query-4c6fbed80ba44defe3d125c258bc3449b02925c8caf85d9203529471479fdde8.json b/backend/.sqlx/query-4c6fbed80ba44defe3d125c258bc3449b02925c8caf85d9203529471479fdde8.json deleted file mode 100644 index bb5c6096f8..0000000000 --- a/backend/.sqlx/query-4c6fbed80ba44defe3d125c258bc3449b02925c8caf85d9203529471479fdde8.json +++ /dev/null @@ -1,124 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT \n workspace_id,\n path,\n kafka_resource_path,\n group_id,\n topics,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\"\n FROM kafka_trigger\n WHERE workspace_id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "kafka_resource_path", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "group_id", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "topics", - "type_info": "VarcharArray" - }, - { - "ordinal": 5, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 6, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 7, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 8, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 9, - "name": "edited_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 10, - "name": "server_id", - "type_info": "Varchar" - }, - { - "ordinal": 11, - "name": "last_server_ping", - "type_info": "Timestamptz" - }, - { - "ordinal": 12, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 13, - "name": "error", - "type_info": "Text" - }, - { - "ordinal": 14, - "name": "enabled", - "type_info": "Bool" - }, - { - "ordinal": 15, - "name": "error_handler_path", - "type_info": "Varchar" - }, - { - "ordinal": 16, - "name": "error_handler_args: _", - "type_info": "Jsonb" - }, - { - "ordinal": 17, - "name": "retry: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - false, - true, - false, - true, - true, - true - ] - }, - "hash": "4c6fbed80ba44defe3d125c258bc3449b02925c8caf85d9203529471479fdde8" -} diff --git a/backend/.sqlx/query-4eab48b7e86788835c3ba739cca764231013b32e7dc26fb3b775e872d2222005.json b/backend/.sqlx/query-4eab48b7e86788835c3ba739cca764231013b32e7dc26fb3b775e872d2222005.json deleted file mode 100644 index af68ad8234..0000000000 --- a/backend/.sqlx/query-4eab48b7e86788835c3ba739cca764231013b32e7dc26fb3b775e872d2222005.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE postgres_trigger \n SET \n script_path = $1, \n path = $2, \n is_flow = $3, \n edited_by = $4, \n email = $5, \n postgres_resource_path = $6, \n replication_slot_name = $7,\n publication_name = $8,\n edited_at = now(), \n error = NULL,\n server_id = NULL,\n error_handler_path = $11,\n error_handler_args = $12,\n retry = $13\n WHERE \n workspace_id = $9 AND \n path = $10\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Varchar", - "Varchar", - "Varchar", - "Varchar", - "Text", - "Text", - "Varchar", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "4eab48b7e86788835c3ba739cca764231013b32e7dc26fb3b775e872d2222005" -} diff --git a/backend/.sqlx/query-4ed74bbda2ad0ca5e4648787fe2d9c89e9b83571e30059861b6998935b35f78b.json b/backend/.sqlx/query-4ed74bbda2ad0ca5e4648787fe2d9c89e9b83571e30059861b6998935b35f78b.json deleted file mode 100644 index e5b87450d3..0000000000 --- a/backend/.sqlx/query-4ed74bbda2ad0ca5e4648787fe2d9c89e9b83571e30059861b6998935b35f78b.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT EXISTS(\n SELECT \n 1 \n FROM \n mqtt_trigger \n WHERE \n path = $1 AND \n workspace_id = $2\n )", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "exists", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "4ed74bbda2ad0ca5e4648787fe2d9c89e9b83571e30059861b6998935b35f78b" -} diff --git a/backend/.sqlx/query-508d267b0d77fd12446654a502bf4968ecebec1614580e55de3d5895f0595e52.json b/backend/.sqlx/query-508d267b0d77fd12446654a502bf4968ecebec1614580e55de3d5895f0595e52.json deleted file mode 100644 index 4a95ebaf74..0000000000 --- a/backend/.sqlx/query-508d267b0d77fd12446654a502bf4968ecebec1614580e55de3d5895f0595e52.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n DELETE FROM postgres_trigger \n WHERE \n workspace_id = $1 AND \n path = $2\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "508d267b0d77fd12446654a502bf4968ecebec1614580e55de3d5895f0595e52" -} diff --git a/backend/.sqlx/query-5140cfccd4a8fb7d64087de34bc2fbe1f9e0172b38492a99be19147d04a961fe.json b/backend/.sqlx/query-5140cfccd4a8fb7d64087de34bc2fbe1f9e0172b38492a99be19147d04a961fe.json deleted file mode 100644 index c80c36b18d..0000000000 --- a/backend/.sqlx/query-5140cfccd4a8fb7d64087de34bc2fbe1f9e0172b38492a99be19147d04a961fe.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO kafka_trigger (\n workspace_id,\n path,\n kafka_resource_path,\n group_id,\n topics,\n script_path,\n is_flow,\n enabled,\n edited_by,\n email,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, now(), $11, $12, $13\n )\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Varchar", - "Varchar", - "VarcharArray", - "Varchar", - "Bool", - "Bool", - "Varchar", - "Varchar", - "Varchar", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "5140cfccd4a8fb7d64087de34bc2fbe1f9e0172b38492a99be19147d04a961fe" -} diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 36ddb8ab9f..713ccb9dd3 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - true + null ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-aaa8dde748e5bcd462122d3235e54c784399b2cac7b27f0de33e78bd77661565.json b/backend/.sqlx/query-5cc0d9e3dcd9c20e6e6ec1acf38c8f97b5ece60bfe24fc3783a83de47e3aa583.json similarity index 51% rename from backend/.sqlx/query-aaa8dde748e5bcd462122d3235e54c784399b2cac7b27f0de33e78bd77661565.json rename to backend/.sqlx/query-5cc0d9e3dcd9c20e6e6ec1acf38c8f97b5ece60bfe24fc3783a83de47e3aa583.json index db7e60bbf5..fc2540e85d 100644 --- a/backend/.sqlx/query-aaa8dde748e5bcd462122d3235e54c784399b2cac7b27f0de33e78bd77661565.json +++ b/backend/.sqlx/query-5cc0d9e3dcd9c20e6e6ec1acf38c8f97b5ece60bfe24fc3783a83de47e3aa583.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO http_trigger (\n workspace_id, \n path, \n route_path, \n route_path_key,\n workspaced_route,\n authentication_resource_path,\n wrap_body,\n raw_string,\n script_path, \n summary,\n description,\n is_flow, \n is_async, \n authentication_method, \n http_method, \n static_asset_config, \n edited_by, \n email, \n edited_at, \n is_static_website,\n error_handler_path,\n error_handler_args,\n retry\n ) \n VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, now(), $19, $20, $21, $22\n )\n ", + "query": "\n INSERT INTO http_trigger (\n workspace_id, \n path, \n route_path, \n route_path_key,\n workspaced_route,\n authentication_resource_path,\n wrap_body,\n raw_string,\n script_path, \n summary,\n description,\n is_flow, \n is_async, \n authentication_method, \n http_method, \n static_asset_config, \n edited_by, \n email, \n edited_at, \n is_static_website,\n error_handler_path,\n error_handler_args,\n retry\n ) \n VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, now(), $19, $20, $21, $22\n )\n ", "describe": { "columns": [], "parameters": { @@ -58,5 +58,5 @@ }, "nullable": [] }, - "hash": "aaa8dde748e5bcd462122d3235e54c784399b2cac7b27f0de33e78bd77661565" + "hash": "5cc0d9e3dcd9c20e6e6ec1acf38c8f97b5ece60bfe24fc3783a83de47e3aa583" } diff --git a/backend/.sqlx/query-6a19c440a7a8064f3969cf6f48adea0bfdb683de9555e374ce5731e0b3c379f9.json b/backend/.sqlx/query-6a19c440a7a8064f3969cf6f48adea0bfdb683de9555e374ce5731e0b3c379f9.json deleted file mode 100644 index 4ea7dabd04..0000000000 --- a/backend/.sqlx/query-6a19c440a7a8064f3969cf6f48adea0bfdb683de9555e374ce5731e0b3c379f9.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT COUNT(*) FROM gcp_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text", - "Bool", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "6a19c440a7a8064f3969cf6f48adea0bfdb683de9555e374ce5731e0b3c379f9" -} diff --git a/backend/.sqlx/query-d580bac9f4be4329da29781efa00cd0b5379955a42b7e070c7e11f8479441482.json b/backend/.sqlx/query-6ca3ae85ddeb2334c9ae4bd7682daef744a1b10b4f47bc7355a97011bc366aae.json similarity index 77% rename from backend/.sqlx/query-d580bac9f4be4329da29781efa00cd0b5379955a42b7e070c7e11f8479441482.json rename to backend/.sqlx/query-6ca3ae85ddeb2334c9ae4bd7682daef744a1b10b4f47bc7355a97011bc366aae.json index 2e063cf7d3..658b99f032 100644 --- a/backend/.sqlx/query-d580bac9f4be4329da29781efa00cd0b5379955a42b7e070c7e11f8479441482.json +++ b/backend/.sqlx/query-6ca3ae85ddeb2334c9ae4bd7682daef744a1b10b4f47bc7355a97011bc366aae.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE \n http_trigger \n SET \n workspaced_route = $1,\n wrap_body = $2,\n raw_string = $3,\n authentication_resource_path = $4,\n script_path = $5, \n path = $6, \n is_flow = $7, \n http_method = $8, \n static_asset_config = $9, \n edited_by = $10, \n email = $11, \n is_async = $12, \n authentication_method = $13, \n edited_at = now(), \n is_static_website = $14,\n error_handler_path = $15,\n error_handler_args = $16,\n retry = $17\n WHERE \n workspace_id = $18 AND \n path = $19\n ", + "query": "\n UPDATE \n http_trigger \n SET \n workspaced_route = $1,\n wrap_body = $2,\n raw_string = $3,\n authentication_resource_path = $4,\n script_path = $5, \n path = $6, \n is_flow = $7, \n http_method = $8, \n static_asset_config = $9, \n edited_by = $10, \n email = $11, \n is_async = $12, \n authentication_method = $13, \n summary = $14,\n description = $15,\n edited_at = now(), \n is_static_website = $16,\n error_handler_path = $17,\n error_handler_args = $18,\n retry = $19\n WHERE \n workspace_id = $20 AND \n path = $21\n ", "describe": { "columns": [], "parameters": { @@ -45,6 +45,8 @@ } } }, + "Varchar", + "Text", "Bool", "Varchar", "Jsonb", @@ -55,5 +57,5 @@ }, "nullable": [] }, - "hash": "d580bac9f4be4329da29781efa00cd0b5379955a42b7e070c7e11f8479441482" + "hash": "6ca3ae85ddeb2334c9ae4bd7682daef744a1b10b4f47bc7355a97011bc366aae" } diff --git a/backend/.sqlx/query-6d418b5cd7a4df54cfe3ec06e2a957c87f2307a218115ca749ea0e71eca2002e.json b/backend/.sqlx/query-6d418b5cd7a4df54cfe3ec06e2a957c87f2307a218115ca749ea0e71eca2002e.json deleted file mode 100644 index c20295fe97..0000000000 --- a/backend/.sqlx/query-6d418b5cd7a4df54cfe3ec06e2a957c87f2307a218115ca749ea0e71eca2002e.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT EXISTS(SELECT 1 FROM kafka_trigger WHERE path = $1 AND workspace_id = $2)", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "exists", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "6d418b5cd7a4df54cfe3ec06e2a957c87f2307a218115ca749ea0e71eca2002e" -} diff --git a/backend/.sqlx/query-711e665344873454428d84e35f300edb7e0ebad5de40ba5645c1ce3d4a6b84be.json b/backend/.sqlx/query-711e665344873454428d84e35f300edb7e0ebad5de40ba5645c1ce3d4a6b84be.json deleted file mode 100644 index 3abff2fa36..0000000000 --- a/backend/.sqlx/query-711e665344873454428d84e35f300edb7e0ebad5de40ba5645c1ce3d4a6b84be.json +++ /dev/null @@ -1,136 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT \n workspace_id,\n path,\n nats_resource_path,\n subjects,\n stream_name,\n consumer_name,\n use_jetstream,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\"\n FROM \n nats_trigger\n WHERE \n workspace_id = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "nats_resource_path", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "subjects", - "type_info": "VarcharArray" - }, - { - "ordinal": 4, - "name": "stream_name", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "consumer_name", - "type_info": "Varchar" - }, - { - "ordinal": 6, - "name": "use_jetstream", - "type_info": "Bool" - }, - { - "ordinal": 7, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 8, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 9, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 10, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 11, - "name": "edited_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 12, - "name": "server_id", - "type_info": "Varchar" - }, - { - "ordinal": 13, - "name": "last_server_ping", - "type_info": "Timestamptz" - }, - { - "ordinal": 14, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 15, - "name": "error", - "type_info": "Text" - }, - { - "ordinal": 16, - "name": "enabled", - "type_info": "Bool" - }, - { - "ordinal": 17, - "name": "error_handler_path", - "type_info": "Varchar" - }, - { - "ordinal": 18, - "name": "error_handler_args: _", - "type_info": "Jsonb" - }, - { - "ordinal": 19, - "name": "retry: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - true, - true, - false, - false, - false, - false, - false, - false, - true, - true, - false, - true, - false, - true, - true, - true - ] - }, - "hash": "711e665344873454428d84e35f300edb7e0ebad5de40ba5645c1ce3d4a6b84be" -} diff --git a/backend/.sqlx/query-746ee16a04267cd251d4cefbc44a37fcc985bf61978cc07167ed35d7dbf92d11.json b/backend/.sqlx/query-746ee16a04267cd251d4cefbc44a37fcc985bf61978cc07167ed35d7dbf92d11.json deleted file mode 100644 index 4a7e81455e..0000000000 --- a/backend/.sqlx/query-746ee16a04267cd251d4cefbc44a37fcc985bf61978cc07167ed35d7dbf92d11.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT EXISTS(SELECT 1 FROM nats_trigger WHERE path = $1 AND workspace_id = $2)", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "exists", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "746ee16a04267cd251d4cefbc44a37fcc985bf61978cc07167ed35d7dbf92d11" -} diff --git a/backend/.sqlx/query-7f039cc86874128ee8c9f577897340a155562fd6cde328298a66496c90d8cd59.json b/backend/.sqlx/query-7f039cc86874128ee8c9f577897340a155562fd6cde328298a66496c90d8cd59.json deleted file mode 100644 index 51928cf94c..0000000000 --- a/backend/.sqlx/query-7f039cc86874128ee8c9f577897340a155562fd6cde328298a66496c90d8cd59.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO postgres_trigger (\n publication_name,\n replication_slot_name,\n workspace_id, \n path, \n script_path, \n is_flow, \n email, \n enabled, \n postgres_resource_path, \n edited_by,\n error_handler_path,\n error_handler_args,\n retry\n ) \n VALUES (\n $1, \n $2, \n $3, \n $4, \n $5, \n $6, \n $7, \n $8, \n $9, \n $10,\n $11,\n $12,\n $13\n )", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Varchar", - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Bool", - "Varchar", - "Varchar", - "Varchar", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "7f039cc86874128ee8c9f577897340a155562fd6cde328298a66496c90d8cd59" -} diff --git a/backend/.sqlx/query-f2a78401a363442f165d018d61a294b4575e2cdc31e8da3d453addd0edc3edf6.json b/backend/.sqlx/query-85d945cf5ade707291a161078ff96ddb29140dd72ce5115657418eec503b205d.json similarity index 56% rename from backend/.sqlx/query-f2a78401a363442f165d018d61a294b4575e2cdc31e8da3d453addd0edc3edf6.json rename to backend/.sqlx/query-85d945cf5ade707291a161078ff96ddb29140dd72ce5115657418eec503b205d.json index c9c1ac057e..fae73f363c 100644 --- a/backend/.sqlx/query-f2a78401a363442f165d018d61a294b4575e2cdc31e8da3d453addd0edc3edf6.json +++ b/backend/.sqlx/query-85d945cf5ade707291a161078ff96ddb29140dd72ce5115657418eec503b205d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE \n http_trigger \n SET \n route_path = $1, \n route_path_key = $2, \n workspaced_route = $3,\n wrap_body = $4,\n raw_string = $5,\n authentication_resource_path = $6,\n script_path = $7, \n path = $8, \n is_flow = $9, \n http_method = $10, \n static_asset_config = $11, \n edited_by = $12, \n email = $13, \n is_async = $14, \n authentication_method = $15, \n summary = $16,\n description = $17,\n edited_at = now(), \n is_static_website = $18,\n error_handler_path = $19,\n error_handler_args = $20,\n retry = $21\n WHERE \n workspace_id = $22 AND \n path = $23\n ", + "query": "\n UPDATE \n http_trigger \n SET \n route_path = $1, \n route_path_key = $2,\n workspaced_route = $3,\n wrap_body = $4,\n raw_string = $5,\n authentication_resource_path = $6,\n script_path = $7, \n path = $8, \n is_flow = $9, \n http_method = $10, \n static_asset_config = $11, \n edited_by = $12, \n email = $13, \n is_async = $14, \n authentication_method = $15, \n summary = $16,\n description = $17,\n edited_at = now(), \n is_static_website = $18,\n error_handler_path = $19,\n error_handler_args = $20,\n retry = $21\n WHERE \n workspace_id = $22 AND \n path = $23\n ", "describe": { "columns": [], "parameters": { @@ -59,5 +59,5 @@ }, "nullable": [] }, - "hash": "f2a78401a363442f165d018d61a294b4575e2cdc31e8da3d453addd0edc3edf6" + "hash": "85d945cf5ade707291a161078ff96ddb29140dd72ce5115657418eec503b205d" } diff --git a/backend/.sqlx/query-870b12a46f26c9e29889dd28a5c7832c5a674a5553674fd9762c981e0d03bb57.json b/backend/.sqlx/query-870b12a46f26c9e29889dd28a5c7832c5a674a5553674fd9762c981e0d03bb57.json deleted file mode 100644 index f5d544c5aa..0000000000 --- a/backend/.sqlx/query-870b12a46f26c9e29889dd28a5c7832c5a674a5553674fd9762c981e0d03bb57.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM nats_trigger WHERE workspace_id = $1 AND path = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "870b12a46f26c9e29889dd28a5c7832c5a674a5553674fd9762c981e0d03bb57" -} diff --git a/backend/.sqlx/query-89be9b595a892c82f2a9f3cfa4da213e360ebe8827d5b4680d3c227fb8ddec82.json b/backend/.sqlx/query-89be9b595a892c82f2a9f3cfa4da213e360ebe8827d5b4680d3c227fb8ddec82.json deleted file mode 100644 index 9b33a91ae2..0000000000 --- a/backend/.sqlx/query-89be9b595a892c82f2a9f3cfa4da213e360ebe8827d5b4680d3c227fb8ddec82.json +++ /dev/null @@ -1,187 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT \n workspace_id, \n workspaced_route,\n path, \n route_path, \n route_path_key, \n authentication_resource_path,\n script_path, \n is_flow, \n summary,\n description,\n edited_by, \n edited_at, \n email, \n extra_perms, \n is_async, \n authentication_method AS \"authentication_method: _\", \n http_method AS \"http_method: _\", \n static_asset_config AS \"static_asset_config: _\", \n is_static_website,\n wrap_body,\n raw_string,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\"\n FROM \n http_trigger\n WHERE \n workspace_id = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "workspaced_route", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "route_path", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "route_path_key", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "authentication_resource_path", - "type_info": "Varchar" - }, - { - "ordinal": 6, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 7, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 8, - "name": "summary", - "type_info": "Varchar" - }, - { - "ordinal": 9, - "name": "description", - "type_info": "Text" - }, - { - "ordinal": 10, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 11, - "name": "edited_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 12, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 13, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 14, - "name": "is_async", - "type_info": "Bool" - }, - { - "ordinal": 15, - "name": "authentication_method: _", - "type_info": { - "Custom": { - "name": "authentication_method", - "kind": { - "Enum": [ - "none", - "windmill", - "api_key", - "basic_http", - "custom_script", - "signature" - ] - } - } - } - }, - { - "ordinal": 16, - "name": "http_method: _", - "type_info": { - "Custom": { - "name": "http_method", - "kind": { - "Enum": [ - "get", - "post", - "put", - "delete", - "patch" - ] - } - } - } - }, - { - "ordinal": 17, - "name": "static_asset_config: _", - "type_info": "Jsonb" - }, - { - "ordinal": 18, - "name": "is_static_website", - "type_info": "Bool" - }, - { - "ordinal": 19, - "name": "wrap_body", - "type_info": "Bool" - }, - { - "ordinal": 20, - "name": "raw_string", - "type_info": "Bool" - }, - { - "ordinal": 21, - "name": "error_handler_path", - "type_info": "Varchar" - }, - { - "ordinal": 22, - "name": "error_handler_args: _", - "type_info": "Jsonb" - }, - { - "ordinal": 23, - "name": "retry: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - false, - true, - false, - false, - true, - true, - false, - false, - false, - false, - false, - false, - false, - true, - false, - false, - false, - true, - true, - true - ] - }, - "hash": "89be9b595a892c82f2a9f3cfa4da213e360ebe8827d5b4680d3c227fb8ddec82" -} diff --git a/backend/.sqlx/query-8d31b4a531c59a2385210d1213c205100d6673a94e90000c8db4eb5809f17365.json b/backend/.sqlx/query-8d31b4a531c59a2385210d1213c205100d6673a94e90000c8db4eb5809f17365.json deleted file mode 100644 index 0d30c8ca0a..0000000000 --- a/backend/.sqlx/query-8d31b4a531c59a2385210d1213c205100d6673a94e90000c8db4eb5809f17365.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n mqtt_trigger \n SET \n enabled = $1, \n email = $2, \n edited_by = $3, \n edited_at = now(), \n server_id = NULL, \n error = NULL\n WHERE \n path = $4 AND \n workspace_id = $5 \n RETURNING 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Bool", - "Varchar", - "Varchar", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "8d31b4a531c59a2385210d1213c205100d6673a94e90000c8db4eb5809f17365" -} diff --git a/backend/.sqlx/query-8e1afb488096330890b1675d2b3052d2064fcc8f373fecfebd40914768b2b1cf.json b/backend/.sqlx/query-8e1afb488096330890b1675d2b3052d2064fcc8f373fecfebd40914768b2b1cf.json deleted file mode 100644 index a6780ad262..0000000000 --- a/backend/.sqlx/query-8e1afb488096330890b1675d2b3052d2064fcc8f373fecfebd40914768b2b1cf.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT COUNT(*) FROM postgres_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text", - "Bool", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "8e1afb488096330890b1675d2b3052d2064fcc8f373fecfebd40914768b2b1cf" -} diff --git a/backend/.sqlx/query-8ffd68b82cfce337fc32a6e80c8d56c53acd02eb350b46f6b815a5ddbb69d163.json b/backend/.sqlx/query-8ffd68b82cfce337fc32a6e80c8d56c53acd02eb350b46f6b815a5ddbb69d163.json deleted file mode 100644 index 40ef813403..0000000000 --- a/backend/.sqlx/query-8ffd68b82cfce337fc32a6e80c8d56c53acd02eb350b46f6b815a5ddbb69d163.json +++ /dev/null @@ -1,137 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n workspace_id,\n path,\n nats_resource_path,\n subjects,\n stream_name,\n consumer_name,\n use_jetstream,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\"\n FROM nats_trigger\n WHERE workspace_id = $1 AND path = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "nats_resource_path", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "subjects", - "type_info": "VarcharArray" - }, - { - "ordinal": 4, - "name": "stream_name", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "consumer_name", - "type_info": "Varchar" - }, - { - "ordinal": 6, - "name": "use_jetstream", - "type_info": "Bool" - }, - { - "ordinal": 7, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 8, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 9, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 10, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 11, - "name": "edited_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 12, - "name": "server_id", - "type_info": "Varchar" - }, - { - "ordinal": 13, - "name": "last_server_ping", - "type_info": "Timestamptz" - }, - { - "ordinal": 14, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 15, - "name": "error", - "type_info": "Text" - }, - { - "ordinal": 16, - "name": "enabled", - "type_info": "Bool" - }, - { - "ordinal": 17, - "name": "error_handler_path", - "type_info": "Varchar" - }, - { - "ordinal": 18, - "name": "error_handler_args: _", - "type_info": "Jsonb" - }, - { - "ordinal": 19, - "name": "retry: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - true, - true, - false, - false, - false, - false, - false, - false, - true, - true, - false, - true, - false, - true, - true, - true - ] - }, - "hash": "8ffd68b82cfce337fc32a6e80c8d56c53acd02eb350b46f6b815a5ddbb69d163" -} diff --git a/backend/.sqlx/query-9abbb75c238372cc3dbaebf1dbc50ff78df6c5dcf92b08d611afe23c5b865521.json b/backend/.sqlx/query-9abbb75c238372cc3dbaebf1dbc50ff78df6c5dcf92b08d611afe23c5b865521.json new file mode 100644 index 0000000000..65dc60db73 --- /dev/null +++ b/backend/.sqlx/query-9abbb75c238372cc3dbaebf1dbc50ff78df6c5dcf92b08d611afe23c5b865521.json @@ -0,0 +1,37 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO sqs_trigger (\n workspace_id,\n path,\n queue_url,\n aws_resource_path,\n message_attributes,\n aws_auth_resource_type,\n script_path,\n is_flow,\n enabled,\n edited_by,\n email,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now(), $12, $13, $14\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "TextArray", + { + "Custom": { + "name": "aws_auth_resource_type", + "kind": { + "Enum": [ + "oidc", + "credentials" + ] + } + } + }, + "Varchar", + "Bool", + "Bool", + "Varchar", + "Varchar", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "9abbb75c238372cc3dbaebf1dbc50ff78df6c5dcf92b08d611afe23c5b865521" +} diff --git a/backend/.sqlx/query-9e88a43de7315052668619002321aadbc27ff5bd0c554bf5339060cc35797d3f.json b/backend/.sqlx/query-9e88a43de7315052668619002321aadbc27ff5bd0c554bf5339060cc35797d3f.json deleted file mode 100644 index bb2a3c682b..0000000000 --- a/backend/.sqlx/query-9e88a43de7315052668619002321aadbc27ff5bd0c554bf5339060cc35797d3f.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n DELETE \n FROM \n mqtt_trigger \n WHERE \n workspace_id = $1 AND \n path = $2\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "9e88a43de7315052668619002321aadbc27ff5bd0c554bf5339060cc35797d3f" -} diff --git a/backend/.sqlx/query-a269c388056eabe4b045948f451ea74ffbb4c0ed7e694f8f03d92f2a7c118af9.json b/backend/.sqlx/query-a269c388056eabe4b045948f451ea74ffbb4c0ed7e694f8f03d92f2a7c118af9.json deleted file mode 100644 index e81acc3584..0000000000 --- a/backend/.sqlx/query-a269c388056eabe4b045948f451ea74ffbb4c0ed7e694f8f03d92f2a7c118af9.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM http_trigger \n WHERE workspace_id = $1 \n AND path = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "a269c388056eabe4b045948f451ea74ffbb4c0ed7e694f8f03d92f2a7c118af9" -} diff --git a/backend/.sqlx/query-a8b470b463ca4b7c00c7ef6e9f36c23f8bbcefc288a56d61122bfd6fe5ca7e8d.json b/backend/.sqlx/query-a8b470b463ca4b7c00c7ef6e9f36c23f8bbcefc288a56d61122bfd6fe5ca7e8d.json deleted file mode 100644 index 6ac4ee6755..0000000000 --- a/backend/.sqlx/query-a8b470b463ca4b7c00c7ef6e9f36c23f8bbcefc288a56d61122bfd6fe5ca7e8d.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT COUNT(*) FROM mqtt_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text", - "Bool", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "a8b470b463ca4b7c00c7ef6e9f36c23f8bbcefc288a56d61122bfd6fe5ca7e8d" -} diff --git a/backend/.sqlx/query-aae8699bbaa4d6111eabee715a6f4a3600c1ccfe6847bd526a751bc7baf825c5.json b/backend/.sqlx/query-aae8699bbaa4d6111eabee715a6f4a3600c1ccfe6847bd526a751bc7baf825c5.json deleted file mode 100644 index 9b94f6bab7..0000000000 --- a/backend/.sqlx/query-aae8699bbaa4d6111eabee715a6f4a3600c1ccfe6847bd526a751bc7baf825c5.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT EXISTS(\n SELECT 1 \n FROM postgres_trigger \n WHERE \n path = $1 AND \n workspace_id = $2\n )", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "exists", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "aae8699bbaa4d6111eabee715a6f4a3600c1ccfe6847bd526a751bc7baf825c5" -} diff --git a/backend/.sqlx/query-ad07b57ad928a4e5260833f4fe2a4bf51211f2f73d78c546c894c8e147677ac7.json b/backend/.sqlx/query-ad07b57ad928a4e5260833f4fe2a4bf51211f2f73d78c546c894c8e147677ac7.json deleted file mode 100644 index 5fc32269c2..0000000000 --- a/backend/.sqlx/query-ad07b57ad928a4e5260833f4fe2a4bf51211f2f73d78c546c894c8e147677ac7.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT COUNT(*) FROM nats_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text", - "Bool", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "ad07b57ad928a4e5260833f4fe2a4bf51211f2f73d78c546c894c8e147677ac7" -} diff --git a/backend/.sqlx/query-adb0090afd3ce918d8b80ff51d9f6104a430a11d7c5cb9447025d11506585708.json b/backend/.sqlx/query-adb0090afd3ce918d8b80ff51d9f6104a430a11d7c5cb9447025d11506585708.json deleted file mode 100644 index bbe9b8bb86..0000000000 --- a/backend/.sqlx/query-adb0090afd3ce918d8b80ff51d9f6104a430a11d7c5cb9447025d11506585708.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT EXISTS(\n SELECT 1 FROM http_trigger \n WHERE path = $1 AND workspace_id = $2\n )", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "exists", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "adb0090afd3ce918d8b80ff51d9f6104a430a11d7c5cb9447025d11506585708" -} diff --git a/backend/.sqlx/query-b2e4dfaaee713604d0700ea4675ed2d4534e08471a60005bff85b952874c54c2.json b/backend/.sqlx/query-b2e4dfaaee713604d0700ea4675ed2d4534e08471a60005bff85b952874c54c2.json deleted file mode 100644 index fabcec9b2a..0000000000 --- a/backend/.sqlx/query-b2e4dfaaee713604d0700ea4675ed2d4534e08471a60005bff85b952874c54c2.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n email AS \"email!\",\n created_by AS \"created_by!\",\n parent_job, permissioned_as AS \"permissioned_as!\",\n script_path, schedule_path, flow_step_id, root_job,\n scheduled_for AS \"scheduled_for!: chrono::DateTime\"\n FROM v2_as_queue WHERE id = $1 AND workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "email!", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "created_by!", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "parent_job", - "type_info": "Uuid" - }, - { - "ordinal": 3, - "name": "permissioned_as!", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "schedule_path", - "type_info": "Varchar" - }, - { - "ordinal": 6, - "name": "flow_step_id", - "type_info": "Varchar" - }, - { - "ordinal": 7, - "name": "root_job", - "type_info": "Uuid" - }, - { - "ordinal": 8, - "name": "scheduled_for!: chrono::DateTime", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Text" - ] - }, - "nullable": [ - true, - true, - true, - true, - true, - true, - true, - true, - true - ] - }, - "hash": "b2e4dfaaee713604d0700ea4675ed2d4534e08471a60005bff85b952874c54c2" -} diff --git a/backend/.sqlx/query-b386d320f9fe1d569a16e6626b723b4376a93342702856da2ac70f6bbfc7933e.json b/backend/.sqlx/query-b386d320f9fe1d569a16e6626b723b4376a93342702856da2ac70f6bbfc7933e.json deleted file mode 100644 index 13ac3c15ea..0000000000 --- a/backend/.sqlx/query-b386d320f9fe1d569a16e6626b723b4376a93342702856da2ac70f6bbfc7933e.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT EXISTS(\n SELECT \n 1 \n FROM \n sqs_trigger \n WHERE \n path = $1 AND \n workspace_id = $2\n )", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "exists", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "b386d320f9fe1d569a16e6626b723b4376a93342702856da2ac70f6bbfc7933e" -} diff --git a/backend/.sqlx/query-c19a60a9dc3f95af218baf40c62f14572ac204cfe377166aa1d91cf58f731f50.json b/backend/.sqlx/query-c19a60a9dc3f95af218baf40c62f14572ac204cfe377166aa1d91cf58f731f50.json deleted file mode 100644 index 65cc991931..0000000000 --- a/backend/.sqlx/query-c19a60a9dc3f95af218baf40c62f14572ac204cfe377166aa1d91cf58f731f50.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n DELETE \n FROM \n gcp_trigger \n WHERE \n workspace_id = $1 AND \n path = $2\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "c19a60a9dc3f95af218baf40c62f14572ac204cfe377166aa1d91cf58f731f50" -} diff --git a/backend/.sqlx/query-c22438f9bff27ccd3228a186a8093c551c30ed85eabe8ec11cafa6358f354c8b.json b/backend/.sqlx/query-c22438f9bff27ccd3228a186a8093c551c30ed85eabe8ec11cafa6358f354c8b.json deleted file mode 100644 index 836709359e..0000000000 --- a/backend/.sqlx/query-c22438f9bff27ccd3228a186a8093c551c30ed85eabe8ec11cafa6358f354c8b.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO sqs_trigger (\n aws_auth_resource_type,\n aws_resource_path,\n queue_url,\n message_attributes,\n workspace_id, \n path, \n script_path, \n is_flow, \n email, \n enabled, \n edited_by,\n error_handler_path,\n error_handler_args,\n retry\n ) \n VALUES (\n $1, \n $2, \n $3, \n $4, \n $5, \n $6, \n $7,\n $8,\n $9,\n $10,\n $11,\n $12,\n $13,\n $14\n )", - "describe": { - "columns": [], - "parameters": { - "Left": [ - { - "Custom": { - "name": "aws_auth_resource_type", - "kind": { - "Enum": [ - "oidc", - "credentials" - ] - } - } - }, - "Varchar", - "Varchar", - "TextArray", - "Varchar", - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Bool", - "Varchar", - "Varchar", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "c22438f9bff27ccd3228a186a8093c551c30ed85eabe8ec11cafa6358f354c8b" -} diff --git a/backend/.sqlx/query-c287f025c77c34e4d521204e7a787e794b461a7a965912bcb598619451eced31.json b/backend/.sqlx/query-c287f025c77c34e4d521204e7a787e794b461a7a965912bcb598619451eced31.json deleted file mode 100644 index e6facf2719..0000000000 --- a/backend/.sqlx/query-c287f025c77c34e4d521204e7a787e794b461a7a965912bcb598619451eced31.json +++ /dev/null @@ -1,136 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT \n workspace_id,\n path,\n url,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled,\n filters AS \"filters: _\",\n initial_messages AS \"initial_messages: _\",\n url_runnable_args AS \"url_runnable_args: _\",\n can_return_message,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\"\n FROM \n websocket_trigger\n WHERE \n workspace_id = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "url", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 5, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 6, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 7, - "name": "edited_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 8, - "name": "server_id", - "type_info": "Varchar" - }, - { - "ordinal": 9, - "name": "last_server_ping", - "type_info": "Timestamptz" - }, - { - "ordinal": 10, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 11, - "name": "error", - "type_info": "Text" - }, - { - "ordinal": 12, - "name": "enabled", - "type_info": "Bool" - }, - { - "ordinal": 13, - "name": "filters: _", - "type_info": "JsonbArray" - }, - { - "ordinal": 14, - "name": "initial_messages: _", - "type_info": "JsonbArray" - }, - { - "ordinal": 15, - "name": "url_runnable_args: _", - "type_info": "Jsonb" - }, - { - "ordinal": 16, - "name": "can_return_message", - "type_info": "Bool" - }, - { - "ordinal": 17, - "name": "error_handler_path", - "type_info": "Varchar" - }, - { - "ordinal": 18, - "name": "error_handler_args: _", - "type_info": "Jsonb" - }, - { - "ordinal": 19, - "name": "retry: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - false, - true, - false, - false, - true, - true, - false, - true, - true, - true - ] - }, - "hash": "c287f025c77c34e4d521204e7a787e794b461a7a965912bcb598619451eced31" -} diff --git a/backend/.sqlx/query-c3730ae025a4dcc25b2713e3e670ce006b1ceed26e29d812d524c711d178167e.json b/backend/.sqlx/query-c3730ae025a4dcc25b2713e3e670ce006b1ceed26e29d812d524c711d178167e.json deleted file mode 100644 index daa7b50c85..0000000000 --- a/backend/.sqlx/query-c3730ae025a4dcc25b2713e3e670ce006b1ceed26e29d812d524c711d178167e.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n sqs_trigger \n SET \n aws_auth_resource_type = $1,\n aws_resource_path = $2,\n queue_url = $3,\n message_attributes = $4, \n is_flow = $5, \n edited_by = $6, \n email = $7,\n script_path = $8,\n path = $9,\n edited_at = now(), \n error = NULL,\n server_id = NULL,\n error_handler_path = $12,\n error_handler_args = $13,\n retry = $14\n WHERE \n workspace_id = $10 AND \n path = $11\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - { - "Custom": { - "name": "aws_auth_resource_type", - "kind": { - "Enum": [ - "oidc", - "credentials" - ] - } - } - }, - "Varchar", - "Varchar", - "TextArray", - "Bool", - "Varchar", - "Varchar", - "Varchar", - "Varchar", - "Text", - "Text", - "Varchar", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "c3730ae025a4dcc25b2713e3e670ce006b1ceed26e29d812d524c711d178167e" -} diff --git a/backend/.sqlx/query-c4cd9668733b2b439ac7a911d15041ac728553e20adf722eb795d01d6650ff0f.json b/backend/.sqlx/query-c4cd9668733b2b439ac7a911d15041ac728553e20adf722eb795d01d6650ff0f.json deleted file mode 100644 index dc8629a042..0000000000 --- a/backend/.sqlx/query-c4cd9668733b2b439ac7a911d15041ac728553e20adf722eb795d01d6650ff0f.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO websocket_trigger (\n workspace_id,\n path,\n url,\n script_path,\n is_flow,\n enabled,\n filters,\n initial_messages,\n url_runnable_args,\n edited_by,\n can_return_message,\n email,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now(), $13, $14, $15\n )\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Varchar", - "Varchar", - "Varchar", - "Bool", - "Bool", - "JsonbArray", - "JsonbArray", - "Jsonb", - "Varchar", - "Bool", - "Varchar", - "Varchar", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "c4cd9668733b2b439ac7a911d15041ac728553e20adf722eb795d01d6650ff0f" -} diff --git a/backend/.sqlx/query-c61df65f2e4f8bc146b0f70e605f5243ec9aa66ea1a9da9662fc877f4becb04e.json b/backend/.sqlx/query-c61df65f2e4f8bc146b0f70e605f5243ec9aa66ea1a9da9662fc877f4becb04e.json deleted file mode 100644 index 528ed0ce7d..0000000000 --- a/backend/.sqlx/query-c61df65f2e4f8bc146b0f70e605f5243ec9aa66ea1a9da9662fc877f4becb04e.json +++ /dev/null @@ -1,168 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n gcp_resource_path,\n subscription_id,\n topic_id,\n workspace_id,\n delivery_type AS \"delivery_type: _\",\n delivery_config AS \"delivery_config: _\",\n subscription_mode AS \"subscription_mode: _\",\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\",\n auto_acknowledge_msg\n FROM \n gcp_trigger\n WHERE \n workspace_id = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "gcp_resource_path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "subscription_id", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "topic_id", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "delivery_type: _", - "type_info": { - "Custom": { - "name": "delivery_mode", - "kind": { - "Enum": [ - "push", - "pull" - ] - } - } - } - }, - { - "ordinal": 5, - "name": "delivery_config: _", - "type_info": "Jsonb" - }, - { - "ordinal": 6, - "name": "subscription_mode: _", - "type_info": { - "Custom": { - "name": "gcp_subscription_mode", - "kind": { - "Enum": [ - "create_update", - "existing" - ] - } - } - } - }, - { - "ordinal": 7, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 8, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 9, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 10, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 11, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 12, - "name": "edited_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 13, - "name": "server_id", - "type_info": "Varchar" - }, - { - "ordinal": 14, - "name": "last_server_ping", - "type_info": "Timestamptz" - }, - { - "ordinal": 15, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 16, - "name": "error", - "type_info": "Text" - }, - { - "ordinal": 17, - "name": "enabled", - "type_info": "Bool" - }, - { - "ordinal": 18, - "name": "error_handler_path", - "type_info": "Varchar" - }, - { - "ordinal": 19, - "name": "error_handler_args: _", - "type_info": "Jsonb" - }, - { - "ordinal": 20, - "name": "retry: _", - "type_info": "Jsonb" - }, - { - "ordinal": 21, - "name": "auto_acknowledge_msg", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - false, - true, - false, - false, - false, - false, - false, - false, - false, - true, - true, - false, - true, - false, - true, - true, - true, - true - ] - }, - "hash": "c61df65f2e4f8bc146b0f70e605f5243ec9aa66ea1a9da9662fc877f4becb04e" -} diff --git a/backend/.sqlx/query-d3bc837633eb92cd9a8d37ad09a72e8da54885cdd135e0877dec9cf407064158.json b/backend/.sqlx/query-d3bc837633eb92cd9a8d37ad09a72e8da54885cdd135e0877dec9cf407064158.json deleted file mode 100644 index 5c129f5088..0000000000 --- a/backend/.sqlx/query-d3bc837633eb92cd9a8d37ad09a72e8da54885cdd135e0877dec9cf407064158.json +++ /dev/null @@ -1,141 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n aws_auth_resource_type AS \"aws_auth_resource_type: _\",\n aws_resource_path,\n message_attributes,\n queue_url,\n workspace_id,\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\"\n FROM \n sqs_trigger\n WHERE \n workspace_id = $1 AND \n path = $2\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "aws_auth_resource_type: _", - "type_info": { - "Custom": { - "name": "aws_auth_resource_type", - "kind": { - "Enum": [ - "oidc", - "credentials" - ] - } - } - } - }, - { - "ordinal": 1, - "name": "aws_resource_path", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "message_attributes", - "type_info": "TextArray" - }, - { - "ordinal": 3, - "name": "queue_url", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 6, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 7, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 8, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 9, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 10, - "name": "edited_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 11, - "name": "server_id", - "type_info": "Varchar" - }, - { - "ordinal": 12, - "name": "last_server_ping", - "type_info": "Timestamptz" - }, - { - "ordinal": 13, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 14, - "name": "error", - "type_info": "Text" - }, - { - "ordinal": 15, - "name": "enabled", - "type_info": "Bool" - }, - { - "ordinal": 16, - "name": "error_handler_path", - "type_info": "Varchar" - }, - { - "ordinal": 17, - "name": "error_handler_args: _", - "type_info": "Jsonb" - }, - { - "ordinal": 18, - "name": "retry: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - false, - true, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - true, - false, - true, - true, - true - ] - }, - "hash": "d3bc837633eb92cd9a8d37ad09a72e8da54885cdd135e0877dec9cf407064158" -} diff --git a/backend/.sqlx/query-d54840373df5da9662ed11ed0a605cddac517bdc06dca0a8be071330338e948a.json b/backend/.sqlx/query-d54840373df5da9662ed11ed0a605cddac517bdc06dca0a8be071330338e948a.json deleted file mode 100644 index 5b298f8db3..0000000000 --- a/backend/.sqlx/query-d54840373df5da9662ed11ed0a605cddac517bdc06dca0a8be071330338e948a.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM websocket_trigger WHERE workspace_id = $1 AND path = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "d54840373df5da9662ed11ed0a605cddac517bdc06dca0a8be071330338e948a" -} diff --git a/backend/.sqlx/query-dea056c89313f4facd62cbbc9fa33ba30fa85efcc83fafe4dd7b4e535b96a8d8.json b/backend/.sqlx/query-dea056c89313f4facd62cbbc9fa33ba30fa85efcc83fafe4dd7b4e535b96a8d8.json deleted file mode 100644 index 12e4b6d81b..0000000000 --- a/backend/.sqlx/query-dea056c89313f4facd62cbbc9fa33ba30fa85efcc83fafe4dd7b4e535b96a8d8.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n DELETE \n FROM \n sqs_trigger \n WHERE \n workspace_id = $1 AND \n path = $2\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "dea056c89313f4facd62cbbc9fa33ba30fa85efcc83fafe4dd7b4e535b96a8d8" -} diff --git a/backend/.sqlx/query-e56c71de41ba0750ae9ae8540c903cb0bf241d959a6c8f7bb896160001a6672b.json b/backend/.sqlx/query-e56c71de41ba0750ae9ae8540c903cb0bf241d959a6c8f7bb896160001a6672b.json new file mode 100644 index 0000000000..f19d888a67 --- /dev/null +++ b/backend/.sqlx/query-e56c71de41ba0750ae9ae8540c903cb0bf241d959a6c8f7bb896160001a6672b.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE postgres_trigger \n SET \n postgres_resource_path = $1,\n replication_slot_name = $2,\n publication_name = $3,\n script_path = $4,\n path = $5,\n is_flow = $6,\n edited_by = $7,\n email = $8,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $11,\n error_handler_args = $12,\n retry = $13\n WHERE \n workspace_id = $9 AND path = $10\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Text", + "Text", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "e56c71de41ba0750ae9ae8540c903cb0bf241d959a6c8f7bb896160001a6672b" +} diff --git a/backend/.sqlx/query-e9941e580a21848fa71b38c58030078a25f117a3bfec78d5270a4c4970f2465e.json b/backend/.sqlx/query-e9941e580a21848fa71b38c58030078a25f117a3bfec78d5270a4c4970f2465e.json new file mode 100644 index 0000000000..1e67891f48 --- /dev/null +++ b/backend/.sqlx/query-e9941e580a21848fa71b38c58030078a25f117a3bfec78d5270a4c4970f2465e.json @@ -0,0 +1,37 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE sqs_trigger \n SET \n queue_url = $1,\n aws_resource_path = $2,\n message_attributes = $3,\n aws_auth_resource_type = $4,\n script_path = $5,\n path = $6,\n is_flow = $7,\n edited_by = $8,\n email = $9,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $12,\n error_handler_args = $13,\n retry = $14\n WHERE \n workspace_id = $10 AND path = $11\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "TextArray", + { + "Custom": { + "name": "aws_auth_resource_type", + "kind": { + "Enum": [ + "oidc", + "credentials" + ] + } + } + }, + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Text", + "Text", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "e9941e580a21848fa71b38c58030078a25f117a3bfec78d5270a4c4970f2465e" +} diff --git a/backend/.sqlx/query-e9b4938b8c5fbd57845cf560fc2fd2d5749015c66d7de277081598ba3bb6dd8f.json b/backend/.sqlx/query-e9b4938b8c5fbd57845cf560fc2fd2d5749015c66d7de277081598ba3bb6dd8f.json deleted file mode 100644 index 5b8934eec6..0000000000 --- a/backend/.sqlx/query-e9b4938b8c5fbd57845cf560fc2fd2d5749015c66d7de277081598ba3bb6dd8f.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO mqtt_trigger (\n mqtt_resource_path,\n subscribe_topics,\n client_version,\n client_id,\n v3_config,\n v5_config,\n workspace_id,\n path, \n script_path, \n is_flow, \n email, \n enabled, \n edited_by,\n error_handler_path,\n error_handler_args,\n retry\n ) \n VALUES (\n $1, \n $2, \n $3, \n $4, \n $5, \n $6, \n $7,\n $8,\n $9,\n $10,\n $11,\n $12,\n $13,\n $14,\n $15,\n $16\n )", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "JsonbArray", - { - "Custom": { - "name": "mqtt_client_version", - "kind": { - "Enum": [ - "v3", - "v5" - ] - } - } - }, - "Varchar", - "Jsonb", - "Jsonb", - "Varchar", - "Varchar", - "Varchar", - "Bool", - "Varchar", - "Bool", - "Varchar", - "Varchar", - "Jsonb", - "Jsonb" - ] - }, - "nullable": [] - }, - "hash": "e9b4938b8c5fbd57845cf560fc2fd2d5749015c66d7de277081598ba3bb6dd8f" -} diff --git a/backend/.sqlx/query-ea59131c1fba8fb14a95b4a486db29ba44c4ff4d2c5b38369f0ede61d4708a21.json b/backend/.sqlx/query-ea59131c1fba8fb14a95b4a486db29ba44c4ff4d2c5b38369f0ede61d4708a21.json deleted file mode 100644 index f9c079f0f6..0000000000 --- a/backend/.sqlx/query-ea59131c1fba8fb14a95b4a486db29ba44c4ff4d2c5b38369f0ede61d4708a21.json +++ /dev/null @@ -1,188 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT \n workspace_id, \n path, \n route_path, \n route_path_key,\n workspaced_route,\n script_path, \n summary,\n description,\n is_flow, \n http_method as \"http_method: _\", \n edited_by, \n email, \n edited_at, \n extra_perms, \n is_async, \n authentication_method as \"authentication_method: _\", \n static_asset_config as \"static_asset_config: _\", \n is_static_website,\n authentication_resource_path,\n wrap_body,\n raw_string,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\"\n FROM \n http_trigger\n WHERE \n workspace_id = $1 AND \n path = $2\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "route_path", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "route_path_key", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "workspaced_route", - "type_info": "Bool" - }, - { - "ordinal": 5, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 6, - "name": "summary", - "type_info": "Varchar" - }, - { - "ordinal": 7, - "name": "description", - "type_info": "Text" - }, - { - "ordinal": 8, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 9, - "name": "http_method: _", - "type_info": { - "Custom": { - "name": "http_method", - "kind": { - "Enum": [ - "get", - "post", - "put", - "delete", - "patch" - ] - } - } - } - }, - { - "ordinal": 10, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 11, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 12, - "name": "edited_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 13, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 14, - "name": "is_async", - "type_info": "Bool" - }, - { - "ordinal": 15, - "name": "authentication_method: _", - "type_info": { - "Custom": { - "name": "authentication_method", - "kind": { - "Enum": [ - "none", - "windmill", - "api_key", - "basic_http", - "custom_script", - "signature" - ] - } - } - } - }, - { - "ordinal": 16, - "name": "static_asset_config: _", - "type_info": "Jsonb" - }, - { - "ordinal": 17, - "name": "is_static_website", - "type_info": "Bool" - }, - { - "ordinal": 18, - "name": "authentication_resource_path", - "type_info": "Varchar" - }, - { - "ordinal": 19, - "name": "wrap_body", - "type_info": "Bool" - }, - { - "ordinal": 20, - "name": "raw_string", - "type_info": "Bool" - }, - { - "ordinal": 21, - "name": "error_handler_path", - "type_info": "Varchar" - }, - { - "ordinal": 22, - "name": "error_handler_args: _", - "type_info": "Jsonb" - }, - { - "ordinal": 23, - "name": "retry: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - true, - true, - false, - false, - false, - false, - false, - false, - false, - false, - true, - false, - true, - false, - false, - true, - true, - true - ] - }, - "hash": "ea59131c1fba8fb14a95b4a486db29ba44c4ff4d2c5b38369f0ede61d4708a21" -} diff --git a/backend/.sqlx/query-eda1e5d1109a13feb14be254eaca630be29176bd5d1246c974642d40201782fc.json b/backend/.sqlx/query-eda1e5d1109a13feb14be254eaca630be29176bd5d1246c974642d40201782fc.json deleted file mode 100644 index fdc34ee1c7..0000000000 --- a/backend/.sqlx/query-eda1e5d1109a13feb14be254eaca630be29176bd5d1246c974642d40201782fc.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE kafka_trigger SET enabled = $1, email = $2, edited_by = $3, edited_at = now(), server_id = NULL, error = NULL\n WHERE path = $4 AND workspace_id = $5 RETURNING 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Bool", - "Varchar", - "Varchar", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "eda1e5d1109a13feb14be254eaca630be29176bd5d1246c974642d40201782fc" -} diff --git a/backend/.sqlx/query-eebc9b609d9b595524223ada91e0f7fa1ca50b45cc82e2fd9d4e61768b258735.json b/backend/.sqlx/query-eebc9b609d9b595524223ada91e0f7fa1ca50b45cc82e2fd9d4e61768b258735.json deleted file mode 100644 index 1a9deeb950..0000000000 --- a/backend/.sqlx/query-eebc9b609d9b595524223ada91e0f7fa1ca50b45cc82e2fd9d4e61768b258735.json +++ /dev/null @@ -1,150 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n mqtt_resource_path,\n subscribe_topics as \"subscribe_topics: _\",\n v3_config as \"v3_config: _\",\n v5_config as \"v5_config: _\",\n client_version AS \"client_version: _\",\n client_id,\n workspace_id,\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\"\n FROM \n mqtt_trigger\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "mqtt_resource_path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "subscribe_topics: _", - "type_info": "JsonbArray" - }, - { - "ordinal": 2, - "name": "v3_config: _", - "type_info": "Jsonb" - }, - { - "ordinal": 3, - "name": "v5_config: _", - "type_info": "Jsonb" - }, - { - "ordinal": 4, - "name": "client_version: _", - "type_info": { - "Custom": { - "name": "mqtt_client_version", - "kind": { - "Enum": [ - "v3", - "v5" - ] - } - } - } - }, - { - "ordinal": 5, - "name": "client_id", - "type_info": "Varchar" - }, - { - "ordinal": 6, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 7, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 8, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 9, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 10, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 11, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 12, - "name": "edited_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 13, - "name": "server_id", - "type_info": "Varchar" - }, - { - "ordinal": 14, - "name": "last_server_ping", - "type_info": "Timestamptz" - }, - { - "ordinal": 15, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 16, - "name": "error", - "type_info": "Text" - }, - { - "ordinal": 17, - "name": "enabled", - "type_info": "Bool" - }, - { - "ordinal": 18, - "name": "error_handler_path", - "type_info": "Varchar" - }, - { - "ordinal": 19, - "name": "error_handler_args: _", - "type_info": "Jsonb" - }, - { - "ordinal": 20, - "name": "retry: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - true, - true, - false, - true, - false, - false, - false, - false, - false, - false, - false, - true, - true, - false, - true, - false, - true, - true, - true - ] - }, - "hash": "eebc9b609d9b595524223ada91e0f7fa1ca50b45cc82e2fd9d4e61768b258735" -} diff --git a/backend/.sqlx/query-f28b71a304721643dd90fa468f61f351d6df8282415e54f0ea4f905417657ba8.json b/backend/.sqlx/query-f28b71a304721643dd90fa468f61f351d6df8282415e54f0ea4f905417657ba8.json new file mode 100644 index 0000000000..b0095248ef --- /dev/null +++ b/backend/.sqlx/query-f28b71a304721643dd90fa468f61f351d6df8282415e54f0ea4f905417657ba8.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO nats_trigger (\n workspace_id,\n path,\n nats_resource_path,\n subjects,\n stream_name,\n consumer_name,\n use_jetstream,\n script_path,\n is_flow,\n enabled,\n edited_by,\n email,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now(), $13, $14, $15\n )\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "VarcharArray", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Bool", + "Bool", + "Varchar", + "Varchar", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "f28b71a304721643dd90fa468f61f351d6df8282415e54f0ea4f905417657ba8" +} diff --git a/backend/.sqlx/query-4aaab98ebdaa90f1edf49ac96fba6c391c4d0054a618b861464ee37239f1f1e0.json b/backend/.sqlx/query-f3f96e066716e61042519a645d487b578bc63792cdb0f7ddaeb82e9771287c22.json similarity index 90% rename from backend/.sqlx/query-4aaab98ebdaa90f1edf49ac96fba6c391c4d0054a618b861464ee37239f1f1e0.json rename to backend/.sqlx/query-f3f96e066716e61042519a645d487b578bc63792cdb0f7ddaeb82e9771287c22.json index fa33c3b8e0..28c0756b0f 100644 --- a/backend/.sqlx/query-4aaab98ebdaa90f1edf49ac96fba6c391c4d0054a618b861464ee37239f1f1e0.json +++ b/backend/.sqlx/query-f3f96e066716e61042519a645d487b578bc63792cdb0f7ddaeb82e9771287c22.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT \n v2_job_queue.workspace_id,\n v2_job_queue.id,\n v2_job.args as \"args: sqlx::types::Json>>\",\n v2_job.parent_job,\n v2_job.created_by,\n v2_job_queue.started_at,\n scheduled_for,\n runnable_path,\n kind as \"kind: JobKind\",\n runnable_id as \"runnable_id: ScriptHash\",\n canceled_reason,\n canceled_by,\n permissioned_as,\n permissioned_as_email,\n flow_status as \"flow_status: sqlx::types::Json>\",\n v2_job.tag,\n script_lang as \"script_lang: ScriptLang\",\n same_worker,\n pre_run_error,\n concurrent_limit,\n concurrency_time_window_s,\n flow_innermost_root_job,\n timeout,\n flow_step_id,\n cache_ttl,\n v2_job_queue.priority,\n preprocessed,\n script_entrypoint_override,\n trigger,\n trigger_kind as \"trigger_kind: JobTriggerKind\",\n visible_to_owner\n FROM v2_job_queue INNER JOIN v2_job ON v2_job.id = v2_job_queue.id LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id WHERE v2_job_queue.id = $1", + "query": "SELECT \n v2_job_queue.workspace_id,\n v2_job_queue.id,\n v2_job.args as \"args: sqlx::types::Json>>\",\n v2_job.parent_job,\n v2_job.created_by,\n v2_job_queue.started_at,\n scheduled_for,\n runnable_path,\n kind as \"kind: JobKind\",\n runnable_id as \"runnable_id: ScriptHash\",\n canceled_reason,\n canceled_by,\n permissioned_as,\n permissioned_as_email,\n flow_status as \"flow_status: sqlx::types::Json>\",\n v2_job.tag,\n script_lang as \"script_lang: ScriptLang\",\n same_worker,\n pre_run_error,\n concurrent_limit,\n concurrency_time_window_s,\n flow_innermost_root_job,\n root_job,\n timeout,\n flow_step_id,\n cache_ttl,\n v2_job_queue.priority,\n preprocessed,\n script_entrypoint_override,\n trigger,\n trigger_kind as \"trigger_kind: JobTriggerKind\",\n visible_to_owner\n FROM v2_job_queue INNER JOIN v2_job ON v2_job.id = v2_job_queue.id LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id WHERE v2_job_queue.id = $1", "describe": { "columns": [ { @@ -173,41 +173,46 @@ }, { "ordinal": 22, + "name": "root_job", + "type_info": "Uuid" + }, + { + "ordinal": 23, "name": "timeout", "type_info": "Int4" }, { - "ordinal": 23, + "ordinal": 24, "name": "flow_step_id", "type_info": "Varchar" }, { - "ordinal": 24, + "ordinal": 25, "name": "cache_ttl", "type_info": "Int4" }, { - "ordinal": 25, + "ordinal": 26, "name": "priority", "type_info": "Int2" }, { - "ordinal": 26, + "ordinal": 27, "name": "preprocessed", "type_info": "Bool" }, { - "ordinal": 27, + "ordinal": 28, "name": "script_entrypoint_override", "type_info": "Varchar" }, { - "ordinal": 28, + "ordinal": 29, "name": "trigger", "type_info": "Varchar" }, { - "ordinal": 29, + "ordinal": 30, "name": "trigger_kind: JobTriggerKind", "type_info": { "Custom": { @@ -233,7 +238,7 @@ } }, { - "ordinal": 30, + "ordinal": 31, "name": "visible_to_owner", "type_info": "Bool" } @@ -274,8 +279,9 @@ true, true, true, + true, false ] }, - "hash": "4aaab98ebdaa90f1edf49ac96fba6c391c4d0054a618b861464ee37239f1f1e0" + "hash": "f3f96e066716e61042519a645d487b578bc63792cdb0f7ddaeb82e9771287c22" } diff --git a/backend/.sqlx/query-f434cac3a8eded700b09a8c99dcc5f7c93fbdb1e7d090b11612aa284342e86a1.json b/backend/.sqlx/query-f434cac3a8eded700b09a8c99dcc5f7c93fbdb1e7d090b11612aa284342e86a1.json deleted file mode 100644 index 21d2ddb41e..0000000000 --- a/backend/.sqlx/query-f434cac3a8eded700b09a8c99dcc5f7c93fbdb1e7d090b11612aa284342e86a1.json +++ /dev/null @@ -1,169 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n gcp_resource_path,\n subscription_id,\n topic_id,\n workspace_id,\n delivery_type AS \"delivery_type: _\",\n delivery_config AS \"delivery_config: _\",\n subscription_mode AS \"subscription_mode: _\",\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\",\n auto_acknowledge_msg\n FROM \n gcp_trigger\n WHERE \n workspace_id = $1 AND \n path = $2\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "gcp_resource_path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "subscription_id", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "topic_id", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "delivery_type: _", - "type_info": { - "Custom": { - "name": "delivery_mode", - "kind": { - "Enum": [ - "push", - "pull" - ] - } - } - } - }, - { - "ordinal": 5, - "name": "delivery_config: _", - "type_info": "Jsonb" - }, - { - "ordinal": 6, - "name": "subscription_mode: _", - "type_info": { - "Custom": { - "name": "gcp_subscription_mode", - "kind": { - "Enum": [ - "create_update", - "existing" - ] - } - } - } - }, - { - "ordinal": 7, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 8, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 9, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 10, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 11, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 12, - "name": "edited_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 13, - "name": "server_id", - "type_info": "Varchar" - }, - { - "ordinal": 14, - "name": "last_server_ping", - "type_info": "Timestamptz" - }, - { - "ordinal": 15, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 16, - "name": "error", - "type_info": "Text" - }, - { - "ordinal": 17, - "name": "enabled", - "type_info": "Bool" - }, - { - "ordinal": 18, - "name": "error_handler_path", - "type_info": "Varchar" - }, - { - "ordinal": 19, - "name": "error_handler_args: _", - "type_info": "Jsonb" - }, - { - "ordinal": 20, - "name": "retry: _", - "type_info": "Jsonb" - }, - { - "ordinal": 21, - "name": "auto_acknowledge_msg", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - false, - true, - false, - false, - false, - false, - false, - false, - false, - true, - true, - false, - true, - false, - true, - true, - true, - true - ] - }, - "hash": "f434cac3a8eded700b09a8c99dcc5f7c93fbdb1e7d090b11612aa284342e86a1" -} diff --git a/backend/.sqlx/query-fc243af1bc70f04e28c006364d6dfdfc49981568e8c496fb6a163c499c3e4ad1.json b/backend/.sqlx/query-fc243af1bc70f04e28c006364d6dfdfc49981568e8c496fb6a163c499c3e4ad1.json deleted file mode 100644 index 57ae674eb4..0000000000 --- a/backend/.sqlx/query-fc243af1bc70f04e28c006364d6dfdfc49981568e8c496fb6a163c499c3e4ad1.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT flow_version.value->>'early_return' as early_return\n FROM flow \n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n WHERE flow.path = $1 and flow.workspace_id = $2", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "early_return", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "fc243af1bc70f04e28c006364d6dfdfc49981568e8c496fb6a163c499c3e4ad1" -} diff --git a/backend/.sqlx/query-fd1db7530acf3c84b2ab696504905a50d1ed4f69629c43de7d874769c340d909.json b/backend/.sqlx/query-fd1db7530acf3c84b2ab696504905a50d1ed4f69629c43de7d874769c340d909.json deleted file mode 100644 index 06cf05a25d..0000000000 --- a/backend/.sqlx/query-fd1db7530acf3c84b2ab696504905a50d1ed4f69629c43de7d874769c340d909.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE postgres_trigger \n SET \n enabled = $1, \n email = $2, \n edited_by = $3, \n edited_at = now(), \n server_id = NULL, \n error = NULL\n WHERE \n path = $4 AND \n workspace_id = $5 \n RETURNING 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Bool", - "Varchar", - "Varchar", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "fd1db7530acf3c84b2ab696504905a50d1ed4f69629c43de7d874769c340d909" -} diff --git a/backend/.sqlx/query-fe78796c5ea2384bdb6191b66943d746de874ca6600178f5db9797eda44f1733.json b/backend/.sqlx/query-fe78796c5ea2384bdb6191b66943d746de874ca6600178f5db9797eda44f1733.json new file mode 100644 index 0000000000..6881d0aef0 --- /dev/null +++ b/backend/.sqlx/query-fe78796c5ea2384bdb6191b66943d746de874ca6600178f5db9797eda44f1733.json @@ -0,0 +1,39 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO mqtt_trigger (\n mqtt_resource_path,\n subscribe_topics,\n client_version,\n client_id,\n v3_config,\n v5_config,\n workspace_id,\n path, \n script_path, \n is_flow, \n email, \n enabled, \n edited_by,\n error_handler_path,\n error_handler_args,\n retry\n ) \n VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16\n )", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "JsonbArray", + { + "Custom": { + "name": "mqtt_client_version", + "kind": { + "Enum": [ + "v3", + "v5" + ] + } + } + }, + "Varchar", + "Jsonb", + "Jsonb", + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Varchar", + "Bool", + "Varchar", + "Varchar", + "Jsonb", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "fe78796c5ea2384bdb6191b66943d746de874ca6600178f5db9797eda44f1733" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index f0fe796f51..3a8ff6af01 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -15166,6 +15166,7 @@ dependencies = [ "async-oauth2", "async-recursion", "async-stream", + "async-trait", "async_zip", "aws-config", "aws-sdk-sqs", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index f1335287db..a64e0d3ec9 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -248,6 +248,7 @@ itertools = "^0" regex = "^1" semver = "^1" duckdb = { version = "1.3.2", features = ["bundled"] } +async-trait = "0.1.88" v8 = "=130.0.7" # Exact version NOTE: Do not forget to update version and hash in flake.nix deno_fetch = "0.214.0" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index dfa2a01850..48a07783b5 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -d16e52d570f10dfdabb04a9061fe7ebdbf5a2aa1 \ No newline at end of file +66381121ea7ba806a54cd44416870c354e461098 \ No newline at end of file diff --git a/backend/migrations/20250902070305_variable_resource_cache_notifications.down.sql b/backend/migrations/20250902070305_variable_resource_cache_notifications.down.sql new file mode 100644 index 0000000000..7a40964be5 --- /dev/null +++ b/backend/migrations/20250902070305_variable_resource_cache_notifications.down.sql @@ -0,0 +1,4 @@ +-- Add down migration script here +DROP TRIGGER IF EXISTS variable_cache_invalidate_trigger ON variable; +DROP TRIGGER IF EXISTS resource_cache_invalidate_trigger ON resource; +DROP FUNCTION IF EXISTS notify_var_resource_cache_change(); \ No newline at end of file diff --git a/backend/migrations/20250902070305_variable_resource_cache_notifications.up.sql b/backend/migrations/20250902070305_variable_resource_cache_notifications.up.sql new file mode 100644 index 0000000000..b96bcc2e50 --- /dev/null +++ b/backend/migrations/20250902070305_variable_resource_cache_notifications.up.sql @@ -0,0 +1,36 @@ +-- Add up migration script here + +-- Create notification function for variable and resource cache invalidation +CREATE OR REPLACE FUNCTION notify_var_resource_cache_change() +RETURNS TRIGGER AS $$ +BEGIN + IF TG_TABLE_NAME = 'variable' THEN + PERFORM pg_notify('var_cache_invalidation', + json_build_object( + 'workspace_id', COALESCE(NEW.workspace_id, OLD.workspace_id), + 'path', COALESCE(NEW.path, OLD.path), + 'operation', TG_OP + )::text + ); + ELSIF TG_TABLE_NAME = 'resource' THEN + PERFORM pg_notify('resource_cache_invalidation', + json_build_object( + 'workspace_id', COALESCE(NEW.workspace_id, OLD.workspace_id), + 'path', COALESCE(NEW.path, OLD.path), + 'operation', TG_OP + )::text + ); + END IF; + RETURN COALESCE(NEW, OLD); +END; +$$ LANGUAGE plpgsql; + +-- Create triggers for variable table +CREATE TRIGGER variable_cache_invalidate_trigger + AFTER INSERT OR UPDATE OR DELETE ON variable + FOR EACH ROW EXECUTE FUNCTION notify_var_resource_cache_change(); + +-- Create triggers for resource table +CREATE TRIGGER resource_cache_invalidate_trigger + AFTER INSERT OR UPDATE OR DELETE ON resource + FOR EACH ROW EXECUTE FUNCTION notify_var_resource_cache_change(); \ No newline at end of file diff --git a/backend/migrations/20250902085503_notify_resource_cache_on_variable_change.down.sql b/backend/migrations/20250902085503_notify_resource_cache_on_variable_change.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20250902085503_notify_resource_cache_on_variable_change.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20250902085503_notify_resource_cache_on_variable_change.up.sql b/backend/migrations/20250902085503_notify_resource_cache_on_variable_change.up.sql new file mode 100644 index 0000000000..1f3c0716a3 --- /dev/null +++ b/backend/migrations/20250902085503_notify_resource_cache_on_variable_change.up.sql @@ -0,0 +1,4 @@ +-- Add up migration script here +DROP TRIGGER variable_cache_invalidate_trigger ON variable; +DROP TRIGGER resource_cache_invalidate_trigger ON resource; +DROP FUNCTION notify_var_resource_cache_change(); \ No newline at end of file diff --git a/backend/src/main.rs b/backend/src/main.rs index 1dce88b3dc..6ec6e9a1bc 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -473,6 +473,9 @@ async fn windmill_main() -> anyhow::Result<()> { } else { // This time we use a pool of connections let db = windmill_common::connect_db(server_mode, indexer_mode, worker_mode).await?; + + // NOTE: Variable/resource cache initialization moved to API server in windmill-api + Connection::Sql(db) }; @@ -916,7 +919,7 @@ Windmill Community Edition {GIT_VERSION} #[cfg(feature = "http_trigger")] "notify_http_trigger_change" => { tracing::info!("HTTP trigger change detected: {}", n.payload()); - match windmill_api::http_triggers::refresh_routers(&db).await { + match windmill_api::triggers::http::refresh_routers(&db).await { Ok((true, _)) => { tracing::info!("Refreshed HTTP routers (trigger change)"); }, @@ -933,6 +936,26 @@ Windmill Community Edition {GIT_VERSION} tracing::info!("Token invalidation detected for token: {}...", &token[..token.len().min(8)]); windmill_api::auth::invalidate_token_from_cache(token); }, + "var_cache_invalidation" => { + if let Ok(payload) = serde_json::from_str::(n.payload()) { + if let (Some(workspace_id), Some(path)) = + (payload.get("workspace_id").and_then(|v| v.as_str()), + payload.get("path").and_then(|v| v.as_str())) { + tracing::info!("Variable cache invalidation detected: {}:{}", workspace_id, path); + windmill_api::var_resource_cache::invalidate_variable_cache(&workspace_id, &path); + } + } + }, + "resource_cache_invalidation" => { + if let Ok(payload) = serde_json::from_str::(n.payload()) { + if let (Some(workspace_id), Some(path)) = + (payload.get("workspace_id").and_then(|v| v.as_str()), + payload.get("path").and_then(|v| v.as_str())) { + tracing::info!("Resource cache invalidation detected: {}:{}", workspace_id, path); + windmill_api::var_resource_cache::invalidate_resource_cache(&workspace_id, &path); + } + } + }, "notify_global_setting_change" => { tracing::info!("Global setting change detected: {}", n.payload()); match n.payload() { diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 85d7555e1b..dcf20ae402 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -949,6 +949,7 @@ impl RunJob { /* schedule_path */ None, /* parent_job */ None, /* root job */ None, + /* flow_innermost_root_job */ None, /* job_id */ None, /* is_flow_step */ false, /* same_worker */ false, diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 3f4a4528f9..39b3a4ed5f 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -146,6 +146,7 @@ aws-sdk-sts = { workspace = true, optional = true } rustls = { workspace = true } aws-config = { workspace = true, optional = true } +async-trait.workspace = true google-cloud-pubsub = { workspace = true, optional = true } google-cloud-googleapis = { workspace = true , optional = true } tonic = { workspace = true, optional = true } diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 2fa7ff8c9a..836ac97cb5 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -3254,6 +3254,12 @@ paths: parameters: - $ref: "#/components/parameters/WorkspaceId" - $ref: "#/components/parameters/Path" + - name: allow_cache + description: | + allow getting a cached value for improved performance + in: query + schema: + type: boolean responses: "200": description: variable @@ -4017,6 +4023,11 @@ paths: schema: type: string format: uuid + - name: allow_cache + description: allow getting a cached value for improved performance + in: query + schema: + type: boolean responses: "200": description: resource value @@ -14137,6 +14148,10 @@ components: $ref: "#/components/schemas/AIProviderModel" code_completion_model: $ref: "#/components/schemas/AIProviderModel" + custom_prompts: + type: object + additionalProperties: + type: string Alert: type: object @@ -15229,7 +15244,7 @@ components: nu, java, ruby, - duckdb + duckdb, # for related places search: ADD_NEW_LANG ] diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index a41840404d..fc60c65c3c 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -353,6 +353,8 @@ pub struct AIConfig { pub default_model: Option, #[serde(skip_serializing_if = "Option::is_none")] pub code_completion_model: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub custom_prompts: Option>, } pub fn global_service() -> Router { diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 37743eae34..2aef45acb5 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -1089,6 +1089,7 @@ async fn create_app_internal<'a>( None, None, None, + None, false, false, None, @@ -1467,6 +1468,7 @@ async fn update_app_internal<'a>( None, None, None, + None, false, false, None, @@ -1780,6 +1782,7 @@ async fn execute_component( None, None, None, + None, job_id, false, false, @@ -2514,6 +2517,7 @@ async fn build_args( &path, None, "", + false ) .await?; if res.is_none() { diff --git a/backend/windmill-api/src/args.rs b/backend/windmill-api/src/args.rs index 0b8f4d5210..a541f8c738 100644 --- a/backend/windmill-api/src/args.rs +++ b/backend/windmill-api/src/args.rs @@ -20,7 +20,7 @@ use windmill_queue::PushArgsOwned; use crate::{ db::ApiAuthed, - trigger_helpers::{get_runnable_format, RunnableId}, + triggers::trigger_helpers::{get_runnable_format, RunnableId}, }; #[derive(Debug)] diff --git a/backend/windmill-api/src/auth.rs b/backend/windmill-api/src/auth.rs index a1ccc69769..01147c0ed8 100644 --- a/backend/windmill-api/src/auth.rs +++ b/backend/windmill-api/src/auth.rs @@ -3,12 +3,13 @@ use crate::ee_oss::ExternalJwks; use axum::{ async_trait, extract::{FromRequestParts, OriginalUri, Query}, - Extension, + Extension, Json, }; use chrono::TimeZone; use http::{request::Parts, StatusCode}; use quick_cache::sync::Cache; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; +use sqlx::FromRow; use tower_cookies::Cookies; use tracing::Span; @@ -22,7 +23,7 @@ use tokio::sync::RwLock; use windmill_common::{ auth::{get_folders_for_user, get_groups_for_user, JWTAuthClaims, TOKEN_PREFIX_LEN}, - error::Error, + error::{Error, JsonResult}, jwt, users::{COOKIE_NAME, SUPERADMIN_SECRET_EMAIL}, }; @@ -641,3 +642,71 @@ fn username_override_from_label(label: Option) -> Option { _ => None, } } + +#[derive(FromRow, Serialize)] +pub struct TruncatedTokenWithEmail { + pub label: Option, + pub token_prefix: Option, + pub expiration: Option>, + pub created_at: chrono::DateTime, + pub last_used_at: chrono::DateTime, + pub scopes: Option>, + pub email: Option, +} + +pub async fn list_tokens_internal( + db: &DB, + w_id: &str, + path: &str, + is_flow: bool, +) -> JsonResult> { + let tokens = if is_flow { + sqlx::query_as!( + TruncatedTokenWithEmail, + r#" + SELECT label, + concat(substring(token for 10)) AS token_prefix, + expiration, + created_at, + last_used_at, + scopes, + email + FROM token + WHERE workspace_id = $1 + AND ( + scopes @> ARRAY['jobs:run:flows:' || $2]::text[] + OR scopes @> ARRAY['run:flow/' || $2]::text[] + ) + "#, + w_id, + path + ) + .fetch_all(db) + .await? + } else { + sqlx::query_as!( + TruncatedTokenWithEmail, + r#" + SELECT label, + concat(substring(token for 10)) AS token_prefix, + expiration, + created_at, + last_used_at, + scopes, + email + FROM token + WHERE workspace_id = $1 + AND ( + scopes @> ARRAY['jobs:run:scripts:' || $2]::text[] + OR scopes @> ARRAY['run:script/' || $2]::text[] + ) + "#, + w_id, + path + ) + .fetch_all(db) + .await? + }; + + Ok(Json(tokens)) +} diff --git a/backend/windmill-api/src/capture.rs b/backend/windmill-api/src/capture.rs index 4bd2161248..0b56bd143d 100644 --- a/backend/windmill-api/src/capture.rs +++ b/backend/windmill-api/src/capture.rs @@ -8,54 +8,54 @@ #[cfg(feature = "http_trigger")] use { - crate::http_trigger_args::{HttpMethod, RawHttpTriggerArgs}, + crate::triggers::http::{http_trigger_args::RawHttpTriggerArgs, HttpMethod}, axum::response::{IntoResponse, Response}, std::collections::HashMap, }; -#[cfg(all(feature = "enterprise", feature = "gcp_trigger"))] +#[cfg(all(feature = "enterprise", feature = "gcp_trigger", feature = "private"))] use { - crate::gcp_triggers_oss::{ + crate::triggers::gcp::{ manage_google_subscription, process_google_push_request, validate_jwt_token, - CreateUpdateConfig, SubscriptionMode, + CreateUpdateConfig, GcpSubscriptionMode, }, axum::extract::Request, http::HeaderMap, }; #[cfg(any( - all(feature = "enterprise", feature = "gcp_trigger"), + all(feature = "enterprise", feature = "gcp_trigger", feature = "private"), feature = "postgres_trigger" ))] use windmill_common::utils::empty_as_none; -#[cfg(all(feature = "enterprise", feature = "sqs_trigger"))] +#[cfg(all(feature = "enterprise", feature = "sqs_trigger", feature = "private"))] use windmill_common::auth::aws::AwsAuthResourceType; #[cfg(any( feature = "http_trigger", - all(feature = "enterprise", feature = "gcp_trigger") + all(feature = "enterprise", feature = "gcp_trigger", feature = "private") ))] use serde::de::DeserializeOwned; #[cfg(any( feature = "http_trigger", feature = "postgres_trigger", - all(feature = "enterprise", feature = "gcp_trigger") + all(feature = "enterprise", feature = "gcp_trigger", feature = "private") ))] use windmill_common::error::Error; -#[cfg(all(feature = "enterprise", feature = "kafka"))] -use crate::kafka_triggers_oss::KafkaTriggerConfigConnection; +#[cfg(all(feature = "enterprise", feature = "kafka", feature = "private"))] +use crate::kafka_triggers_ee::KafkaTriggerConfigConnection; #[cfg(feature = "mqtt_trigger")] use crate::mqtt_triggers::{MqttClientVersion, MqttV3Config, MqttV5Config, SubscribeTopic}; -#[cfg(all(feature = "enterprise", feature = "nats"))] -use crate::nats_triggers_oss::NatsTriggerConfigConnection; +#[cfg(all(feature = "enterprise", feature = "nats", feature = "private"))] +use crate::triggers::nats::NatsTriggerConfigConnection; #[cfg(feature = "postgres_trigger")] -use crate::postgres_triggers::{ +use crate::triggers::postgres::{ create_logical_replication_slot, create_pg_publication, generate_random_string, get_default_pg_connection, PublicationData, }; @@ -122,7 +122,7 @@ pub fn workspaced_unauthed_service() -> Router { head(|| async {}).fallback(http_payload) }); - #[cfg(all(feature = "enterprise", feature = "gcp_trigger"))] + #[cfg(all(feature = "enterprise", feature = "gcp_trigger", feature = "private"))] let router = router.route("/gcp/:runnable_kind/*path", post(gcp_payload)); router @@ -146,7 +146,7 @@ struct HttpTriggerConfig { wrap_body: Option, } -#[cfg(all(feature = "enterprise", feature = "kafka"))] +#[cfg(all(feature = "enterprise", feature = "kafka", feature = "private"))] #[derive(Serialize, Deserialize)] pub struct KafkaTriggerConfig { #[serde(flatten)] @@ -155,7 +155,7 @@ pub struct KafkaTriggerConfig { pub group_id: String, } -#[cfg(all(feature = "enterprise", feature = "sqs_trigger"))] +#[cfg(all(feature = "enterprise", feature = "sqs_trigger", feature = "private"))] #[derive(Debug, Serialize, Deserialize)] pub struct SqsTriggerConfig { pub queue_url: String, @@ -164,11 +164,11 @@ pub struct SqsTriggerConfig { pub aws_auth_resource_type: AwsAuthResourceType, } -#[cfg(all(feature = "enterprise", feature = "gcp_trigger"))] +#[cfg(all(feature = "enterprise", feature = "gcp_trigger", feature = "private"))] #[derive(Debug, Serialize, Deserialize)] pub struct GcpTriggerConfig { pub gcp_resource_path: String, - pub subscription_mode: SubscriptionMode, + pub subscription_mode: GcpSubscriptionMode, #[serde(default, deserialize_with = "empty_as_none")] pub subscription_id: Option, #[serde(default, deserialize_with = "empty_as_none")] @@ -179,7 +179,7 @@ pub struct GcpTriggerConfig { pub auto_acknowledge_msg: Option, } -#[cfg(all(feature = "enterprise", feature = "nats"))] +#[cfg(all(feature = "enterprise", feature = "nats", feature = "private"))] #[derive(Serialize, Deserialize)] pub struct NatsTriggerConfig { #[serde(flatten)] @@ -231,15 +231,15 @@ enum TriggerConfig { Postgres(PostgresTriggerConfig), #[cfg(feature = "websocket")] Websocket(WebsocketTriggerConfig), - #[cfg(all(feature = "enterprise", feature = "sqs_trigger"))] + #[cfg(all(feature = "enterprise", feature = "sqs_trigger", feature = "private"))] Sqs(SqsTriggerConfig), - #[cfg(all(feature = "enterprise", feature = "kafka"))] + #[cfg(all(feature = "enterprise", feature = "kafka", feature = "private"))] Kafka(KafkaTriggerConfig), - #[cfg(all(feature = "enterprise", feature = "nats"))] + #[cfg(all(feature = "enterprise", feature = "nats", feature = "private"))] Nats(NatsTriggerConfig), #[cfg(feature = "mqtt_trigger")] Mqtt(MqttTriggerConfig), - #[cfg(all(feature = "enterprise", feature = "gcp_trigger"))] + #[cfg(all(feature = "enterprise", feature = "gcp_trigger", feature = "private"))] Gcp(GcpTriggerConfig), } @@ -365,7 +365,7 @@ async fn set_postgres_trigger_config( Ok(capture_config) } -#[cfg(all(feature = "enterprise", feature = "gcp_trigger"))] +#[cfg(all(feature = "enterprise", feature = "gcp_trigger", feature = "private"))] async fn set_gcp_trigger_config( w_id: &str, authed: ApiAuthed, @@ -392,14 +392,14 @@ async fn set_gcp_trigger_config( ) .await?; gcp_config.create_update = Some(config); - gcp_config.subscription_mode = SubscriptionMode::CreateUpdate; + gcp_config.subscription_mode = GcpSubscriptionMode::CreateUpdate; capture_config.trigger_config = Some(TriggerConfig::Gcp(gcp_config)); Ok(capture_config) } #[inline] -#[cfg(not(all(feature = "enterprise", feature = "gcp_trigger")))] +#[cfg(not(all(feature = "enterprise", feature = "gcp_trigger", feature = "private")))] async fn set_gcp_trigger_config( _w_id: &str, _authed: ApiAuthed, @@ -721,7 +721,7 @@ pub async fn get_active_capture_owner_and_email( #[cfg(any( feature = "http_trigger", - all(feature = "enterprise", feature = "gcp_trigger") + all(feature = "enterprise", feature = "gcp_trigger", feature = "private") ))] async fn get_capture_trigger_config_and_owner( db: &DB, @@ -900,7 +900,7 @@ async fn webhook_payload( Ok(StatusCode::NO_CONTENT) } -#[cfg(all(feature = "enterprise", feature = "gcp_trigger"))] +#[cfg(all(feature = "enterprise", feature = "gcp_trigger", feature = "private"))] async fn gcp_payload( Extension(db): Extension, Extension(user_db): Extension, @@ -908,7 +908,7 @@ async fn gcp_payload( headers: HeaderMap, request: Request, ) -> Result { - use crate::{gcp_triggers_oss::GcpTrigger, trigger_helpers::TriggerJobArgs}; + use crate::triggers::{gcp::GcpTrigger, trigger_helpers::TriggerJobArgs}; let is_flow = matches!(runnable_kind, RunnableKind::Flow); let (gcp_trigger_config, owner, email): (GcpTriggerConfig, _, _) = @@ -931,9 +931,9 @@ async fn gcp_payload( ) .await?; - let (payload, gcp) = process_google_push_request(headers, request).await?; + let (payload, trigger_info) = process_google_push_request(headers, request).await?; - let (main_args, preprocessor_args) = GcpTrigger::build_capture_payloads(payload, gcp); + let (main_args, preprocessor_args) = GcpTrigger::build_capture_payloads(&payload, trigger_info); let _ = insert_capture_payload( &db, diff --git a/backend/windmill-api/src/db.rs b/backend/windmill-api/src/db.rs index a3a8444d3a..9d8d494599 100644 --- a/backend/windmill-api/src/db.rs +++ b/backend/windmill-api/src/db.rs @@ -10,19 +10,18 @@ use futures::FutureExt; use sqlx::{ migrate::{Migrate, MigrateError}, pool::PoolConnection, - Executor, PgConnection, Pool, Postgres, + Executor, PgConnection, Postgres, }; use tokio::task::JoinHandle; use windmill_audit::audit_oss::{AuditAuthor, AuditAuthorable}; -use windmill_common::utils::generate_lock_id; +pub use windmill_common::db::DB; use windmill_common::{ db::{Authable, Authed}, error::Error, + utils::generate_lock_id, }; -pub type DB = Pool; - async fn current_database(conn: &mut PgConnection) -> Result { // language=SQL Ok(sqlx::query_scalar("SELECT current_database()") diff --git a/backend/windmill-api/src/ee_oss.rs b/backend/windmill-api/src/ee_oss.rs index 3fb1271a73..cec4e4db24 100644 --- a/backend/windmill-api/src/ee_oss.rs +++ b/backend/windmill-api/src/ee_oss.rs @@ -1,6 +1,12 @@ #[cfg(feature = "private")] #[allow(unused)] pub use crate::ee::*; +#[cfg(all( + feature = "enterprise", + any(feature = "nats", feature = "kafka", feature = "sqs_trigger"), + not(feature = "private") +))] +use {crate::db::ApiAuthed, windmill_common::DB}; #[cfg(not(feature = "private"))] use anyhow::anyhow; @@ -36,3 +42,20 @@ impl ExternalJwks { None } } + +#[cfg(all( + feature = "enterprise", + any(feature = "nats", feature = "kafka", feature = "sqs_trigger"), + not(feature = "private") +))] +pub async fn interpolate( + _authed: &ApiAuthed, + _db: &DB, + _w_id: &str, + _s: String, +) -> Result { + // Implementation is not open source + Err(anyhow!( + "Interpolation is not available in open source version" + )) +} diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index 9ea840a2d1..69bb707ad7 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -10,11 +10,10 @@ use std::collections::HashMap; use crate::utils::WithStarredInfoQuery; use crate::{ + auth::{list_tokens_internal, TruncatedTokenWithEmail}, db::{ApiAuthed, DB}, schedule::clear_schedule, - triggers::{ - get_triggers_count_internal, list_tokens_internal, TriggersCount, TruncatedTokenWithEmail, - }, + triggers::{get_triggers_count_internal, TriggersCount}, users::{maybe_refresh_folders, require_owner_of_path}, utils::check_scopes, webhook_util::{WebhookMessage, WebhookShared}, @@ -510,6 +509,7 @@ async fn create_flow( None, None, None, + None, false, false, None, @@ -977,6 +977,7 @@ async fn update_flow( None, None, None, + None, false, false, None, diff --git a/backend/windmill-api/src/gcp_triggers_oss.rs b/backend/windmill-api/src/gcp_triggers_oss.rs index b18e61db95..ce2f81f3c9 100644 --- a/backend/windmill-api/src/gcp_triggers_oss.rs +++ b/backend/windmill-api/src/gcp_triggers_oss.rs @@ -2,27 +2,19 @@ #[allow(unused)] pub use crate::gcp_triggers_ee::*; -use serde_json::value::RawValue; -use sqlx::prelude::FromRow; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use sqlx::types::Json as SqlxJson; -use windmill_common::worker::to_raw_value; -use windmill_common::triggers::TriggerKind; -use crate::trigger_helpers::TriggerJobArgs; - #[cfg(not(feature = "private"))] use { - crate::db::{ApiAuthed, DB}, - axum::{extract::Request, Router}, - http::HeaderMap, - windmill_common::db::UserDB, - windmill_common::{ - error::{Error as WindmillError, Result as WindmillResult}, - utils::empty_as_none, - }, + crate::{db::DB, trigger_helpers::TriggerJobArgs}, + serde::{Deserialize, Serialize}, + serde_json::value::RawValue, + sqlx::FromRow, + std::collections::HashMap, + windmill_common::{triggers::TriggerKind, utils::empty_as_none, worker::to_raw_value}, }; +#[cfg(not(feature = "private"))] +type SqlxJson = sqlx::types::Json; + #[derive(sqlx::Type, Debug, Deserialize, Serialize)] #[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] #[sqlx(type_name = "DELIVERY_MODE", rename_all = "lowercase")] @@ -77,11 +69,6 @@ pub enum SubscriptionMode { CreateUpdate, } -#[cfg(not(feature = "private"))] -pub fn workspaced_service() -> Router { - Router::new() -} - #[cfg(not(feature = "private"))] pub fn start_consuming_gcp_pubsub_event( _db: DB, @@ -91,49 +78,6 @@ pub fn start_consuming_gcp_pubsub_event( } #[cfg(not(feature = "private"))] -pub async fn manage_google_subscription( - _authed: ApiAuthed, - _db: &DB, - _workspace_id: &str, - _gcp_resource_path: &str, - _path: &str, - _topic_id: &str, - _subscription_id: &mut Option, - _base_endpoint: &mut Option, - _subscription_mode: SubscriptionMode, - _create_update_config: Option, - _trigger_mode: bool, - _is_flow: bool, -) -> WindmillResult { - Ok(CreateUpdateConfig::default()) -} - -#[cfg(not(feature = "private"))] -pub async fn process_google_push_request( - _headers: HeaderMap, - _request: Request, -) -> Result<(String, HashMap>), WindmillError> { - Ok((String::new(), HashMap::new())) -} - -#[cfg(not(feature = "private"))] -pub async fn validate_jwt_token( - _db: &DB, - _user_db: UserDB, - _authed: ApiAuthed, - _headers: &HeaderMap, - _gcp_resource_path: &str, - _workspace_id: &str, - _delivery_config: &PushConfig, -) -> Result<(), windmill_common::error::Error> { - Ok(()) -} - -#[cfg(not(feature = "private"))] -pub fn gcp_push_route_handler() -> Router { - Router::new() -} - #[derive(FromRow, Deserialize, Serialize, Debug)] pub struct GcpTrigger { pub gcp_resource_path: String, @@ -164,6 +108,7 @@ pub struct GcpTrigger { pub auto_acknowledge_msg: Option, } +#[cfg(not(feature = "private"))] impl TriggerJobArgs for GcpTrigger { fn v1_payload_fn(payload: String) -> HashMap> { HashMap::from([("payload".to_string(), to_raw_value(&payload))]) @@ -172,4 +117,4 @@ impl TriggerJobArgs for GcpTrigger { fn trigger_kind() -> TriggerKind { TriggerKind::Gcp } -} \ No newline at end of file +} diff --git a/backend/windmill-api/src/http_triggers.rs b/backend/windmill-api/src/http_triggers.rs deleted file mode 100644 index b8a52a58ba..0000000000 --- a/backend/windmill-api/src/http_triggers.rs +++ /dev/null @@ -1,1469 +0,0 @@ -#[cfg(feature = "http_trigger")] -use crate::http_trigger_args::{HttpMethod, RawHttpTriggerArgs}; -#[cfg(feature = "parquet")] -use crate::job_helpers_oss::get_workspace_s3_resource; -use crate::{ - auth::{AuthCache, OptTokened}, - db::{ApiAuthed, DB}, - resources::try_get_resource_from_db_as, - trigger_helpers::{ - get_runnable_format, trigger_runnable, trigger_runnable_and_wait_for_result, RunnableId, - }, - users::fetch_api_authed, - utils::{check_scopes, non_empty_str, ExpiringCacheEntry}, -}; -use anyhow::anyhow; -use axum::response::Response; -use axum::{ - extract::{Path, Query}, - response::IntoResponse, - routing::{delete, get, post}, - Extension, Json, Router, -}; -#[cfg(feature = "parquet")] -use http::header::IF_NONE_MATCH; -use http::{HeaderMap, StatusCode}; -use quick_cache::sync::Cache; -use serde::{Deserialize, Serialize}; -use serde_json::value::RawValue; -use sql_builder::{bind::Bind, SqlBuilder}; -use sqlx::prelude::FromRow; -use sqlx::PgConnection; -use std::borrow::Cow; -use std::collections::HashSet; -use std::{collections::HashMap, sync::Arc}; -use tokio::sync::{RwLock, RwLockReadGuard}; -use tower_http::cors::CorsLayer; -use windmill_audit::{audit_oss::audit_log, ActionKind}; -use windmill_common::error::{Error, Result as WindmillResult}; -use windmill_common::flows::Retry; -#[cfg(feature = "parquet")] -use windmill_common::s3_helpers::build_object_store_client; -use windmill_common::{ - db::UserDB, - error::{self, JsonResult}, - s3_helpers::S3Object, - triggers::TriggerKind, - utils::{empty_as_none, not_found_if_none, paginate, require_admin, Pagination, StripPath}, - worker::CLOUD_HOSTED, -}; -use windmill_git_sync::handle_deployment_metadata; - -lazy_static::lazy_static! { - static ref ROUTE_PATH_KEY_RE: regex::Regex = regex::Regex::new(r"/?:[-\w]+").unwrap(); - static ref VALID_ROUTE_PATH_RE: regex::Regex = regex::Regex::new(r"^:?[-\w]+(/:?[-\w]+)*$").unwrap(); -} - -pub fn routes_global_service() -> Router { - let cors = CorsLayer::new() - .allow_methods([ - http::Method::GET, - http::Method::POST, - http::Method::DELETE, - http::Method::PUT, - http::Method::PATCH, - ]) - .allow_headers([http::header::CONTENT_TYPE, http::header::AUTHORIZATION]) - .allow_origin(tower_http::cors::Any); - Router::new() - .route( - "/*path", - get(route_job) - .post(route_job) - .delete(route_job) - .put(route_job) - .patch(route_job) - .head(|| async { "" }), - ) - .layer(cors) -} - -pub fn workspaced_service() -> Router { - Router::new() - .route("/create", post(create_trigger)) - .route("/create_many", post(create_many_http_trigger)) - .route("/list", get(list_triggers)) - .route("/get/*path", get(get_trigger)) - .route("/update/*path", post(update_trigger)) - .route("/delete/*path", delete(delete_trigger)) - .route("/exists/*path", get(exists_trigger)) - .route("/route_exists", post(exists_route)) -} - -#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone, Copy)] -#[sqlx(type_name = "AUTHENTICATION_METHOD", rename_all = "snake_case")] -#[serde(rename_all(serialize = "snake_case", deserialize = "snake_case"))] -pub enum AuthenticationMethod { - None, - Windmill, - ApiKey, - BasicHttp, - CustomScript, - Signature, -} - -#[derive(Debug, Deserialize)] -struct NewTrigger { - path: String, - route_path: String, - script_path: String, - is_flow: bool, - is_async: bool, - authentication_resource_path: Option, - authentication_method: AuthenticationMethod, - static_asset_config: Option>, - http_method: HttpMethod, - workspaced_route: Option, - summary: Option, - description: Option, - is_static_website: bool, - wrap_body: Option, - raw_string: Option, - error_handler_path: Option, - error_handler_args: Option>>>, - retry: Option>, -} - -#[derive(FromRow, Serialize)] -pub struct HttpTrigger { - pub workspace_id: String, - pub path: String, - pub route_path: String, - pub route_path_key: String, - pub script_path: String, - pub is_flow: bool, - pub edited_by: String, - pub email: String, - pub edited_at: chrono::DateTime, - pub extra_perms: serde_json::Value, - pub is_async: bool, - pub authentication_method: AuthenticationMethod, - pub http_method: HttpMethod, - pub summary: Option, - pub description: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub static_asset_config: Option>, - pub is_static_website: bool, - pub authentication_resource_path: Option, - pub workspaced_route: bool, - pub wrap_body: bool, - pub raw_string: bool, - pub error_handler_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_args: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub retry: Option>>, -} - -#[derive(Deserialize)] -struct EditTrigger { - path: String, - route_path: Option, - script_path: String, - is_flow: bool, - is_async: bool, - authentication_method: AuthenticationMethod, - #[serde(deserialize_with = "non_empty_str")] - authentication_resource_path: Option, - summary: Option, - description: Option, - http_method: HttpMethod, - static_asset_config: Option>, - workspaced_route: Option, - is_static_website: bool, - wrap_body: Option, - raw_string: Option, - error_handler_path: Option, - error_handler_args: Option>>>, - retry: Option>, -} - -#[derive(Deserialize)] -pub struct ListTriggerQuery { - pub page: Option, - pub per_page: Option, - pub path: Option, - pub is_flow: Option, - #[serde(default, deserialize_with = "empty_as_none")] - pub path_start: Option, -} - -async fn list_triggers( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, - Query(lst): Query, -) -> error::JsonResult> { - let mut tx = user_db.begin(&authed).await?; - let (per_page, offset) = paginate(Pagination { per_page: lst.per_page, page: lst.page }); - let mut sqlb = SqlBuilder::select_from("http_trigger"); - sqlb.fields(&[ - "workspace_id", - "path", - "route_path", - "route_path_key", - "workspaced_route", - "wrap_body", - "raw_string", - "script_path", - "summary", - "description", - "is_flow", - "http_method", - "edited_by", - "email", - "edited_at", - "extra_perms", - "is_async", - "authentication_method", - "static_asset_config", - "is_static_website", - "authentication_resource_path", - "error_handler_path", - "error_handler_args", - "retry", - ]) - .order_by("edited_at", true) - .and_where("workspace_id = ?".bind(&w_id)) - .offset(offset) - .limit(per_page); - if let Some(path) = lst.path { - sqlb.and_where_eq("script_path", "?".bind(&path)); - } - if let Some(is_flow) = lst.is_flow { - sqlb.and_where_eq("is_flow", "?".bind(&is_flow)); - } - if let Some(path_start) = &lst.path_start { - sqlb.and_where_like_left("path", path_start); - } - let sql = sqlb - .sql() - .map_err(|e| error::Error::internal_err(e.to_string()))?; - let rows = sqlx::query_as::<_, HttpTrigger>(&sql) - .fetch_all(&mut *tx) - .await?; - tx.commit().await?; - - Ok(Json(rows)) -} - -async fn get_trigger( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> error::JsonResult { - let path = path.to_path(); - check_scopes(&authed, || format!("http_triggers:read:{}", path))?; - let mut tx = user_db.begin(&authed).await?; - let trigger = sqlx::query_as!( - HttpTrigger, - r#" - SELECT - workspace_id, - path, - route_path, - route_path_key, - workspaced_route, - script_path, - summary, - description, - is_flow, - http_method as "http_method: _", - edited_by, - email, - edited_at, - extra_perms, - is_async, - authentication_method as "authentication_method: _", - static_asset_config as "static_asset_config: _", - is_static_website, - authentication_resource_path, - wrap_body, - raw_string, - error_handler_path, - error_handler_args as "error_handler_args: _", - retry as "retry: _" - FROM - http_trigger - WHERE - workspace_id = $1 AND - path = $2 - "#, - w_id, - path, - ) - .fetch_optional(&mut *tx) - .await?; - tx.commit().await?; - - let trigger = not_found_if_none(trigger, "Trigger", path)?; - - Ok(Json(trigger)) -} - -fn validate_authentication_method( - authentication_method: AuthenticationMethod, - raw_string: Option, -) -> WindmillResult<()> { - match (authentication_method, raw_string) { - (AuthenticationMethod::CustomScript, raw) if !raw.unwrap_or(false) == true => { - return Err(Error::BadRequest( - "To use custom script authentication, please enable the raw body option." - .to_string(), - )); - } - _ => {} - } - - Ok(()) -} - -async fn increase_trigger_version(tx: &mut PgConnection) -> WindmillResult<()> { - sqlx::query!("SELECT nextval('http_trigger_version_seq')",) - .fetch_one(tx) - .await?; - - Ok(()) -} - -async fn create_trigger_inner( - tx: &mut PgConnection, - w_id: &str, - authed: &ApiAuthed, - new_http_trigger: &NewTrigger, - route_path_key: &str, -) -> WindmillResult<()> { - check_scopes(&authed, || { - format!("http_triggers:write:{}", &new_http_trigger.path) - })?; - - sqlx::query!( - r#" - INSERT INTO http_trigger ( - workspace_id, - path, - route_path, - route_path_key, - workspaced_route, - authentication_resource_path, - wrap_body, - raw_string, - script_path, - summary, - description, - is_flow, - is_async, - authentication_method, - http_method, - static_asset_config, - edited_by, - email, - edited_at, - is_static_website, - error_handler_path, - error_handler_args, - retry - ) - VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, now(), $19, $20, $21, $22 - ) - "#, - w_id, - new_http_trigger.path, - new_http_trigger.route_path, - &route_path_key, - new_http_trigger.workspaced_route, - new_http_trigger.authentication_resource_path, - new_http_trigger.wrap_body.unwrap_or(false), - new_http_trigger.raw_string.unwrap_or(false), - new_http_trigger.script_path, - new_http_trigger.summary, - new_http_trigger.description, - new_http_trigger.is_flow, - new_http_trigger.is_async, - new_http_trigger.authentication_method as _, - new_http_trigger.http_method as _, - new_http_trigger.static_asset_config as _, - &authed.username, - &authed.email, - new_http_trigger.is_static_website, - new_http_trigger.error_handler_path, - new_http_trigger.error_handler_args as _, - new_http_trigger.retry as _ - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - authed, - "http_triggers.create", - ActionKind::Create, - &w_id, - Some(new_http_trigger.path.as_str()), - None, - ) - .await?; - - increase_trigger_version(tx).await?; - - Ok(()) -} - -fn check_no_duplicates<'trigger>( - new_http_triggers: &[NewTrigger], - route_path_key: &[Cow<'trigger, str>], -) -> Result<(), Error> { - let mut seen = HashSet::with_capacity(new_http_triggers.len()); - - for (i, trigger) in new_http_triggers.iter().enumerate() { - if !seen.insert(( - &route_path_key[i], - trigger.http_method, - trigger.workspaced_route, - )) { - return Err(Error::BadRequest(format!( - "Duplicate HTTP route detected: '{}'. Each HTTP route must have a unique 'route_path'.", - &trigger.route_path - ))); - } - } - - Ok(()) -} - -async fn create_many_http_trigger( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path(w_id): Path, - Json(new_http_triggers): Json>, -) -> WindmillResult<(StatusCode, String)> { - require_admin(authed.is_admin, &authed.username)?; - - let error_wrapper = |path: &str, error: Error| -> Error { - anyhow!( - "Error occurred for HTTP route at route path: {}, error: {}", - path, - error - ) - .into() - }; - - let mut route_path_keys = Vec::with_capacity(new_http_triggers.len()); - - for new_http_trigger in new_http_triggers.iter() { - let route_path_key = validate_http_trigger(&db, &w_id, new_http_trigger) - .await - .map_err(|err| error_wrapper(&new_http_trigger.route_path, err))?; - - route_path_keys.push(route_path_key); - } - - check_no_duplicates(&new_http_triggers, &route_path_keys)?; - - let mut tx = user_db.begin(&authed).await?; - - for (i, new_http_trigger) in new_http_triggers.iter().enumerate() { - create_trigger_inner( - &mut tx, - &w_id, - &authed, - new_http_trigger, - &route_path_keys[i], - ) - .await - .map_err(|err| error_wrapper(&new_http_trigger.route_path, err))?; - } - - tx.commit().await?; - - for http_trigger in new_http_triggers.into_iter() { - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::HttpTrigger { path: http_trigger.path.clone() }, - Some(format!("HTTP route '{}' created", http_trigger.path)), - true, - ) - .await?; - } - - Ok((StatusCode::CREATED, format!("Created all HTTP routes"))) -} - -async fn validate_http_trigger<'trigger>( - db: &DB, - w_id: &str, - new_http_trigger: &'trigger NewTrigger, -) -> WindmillResult> { - if !VALID_ROUTE_PATH_RE.is_match(&new_http_trigger.route_path) { - return Err(error::Error::BadRequest("Invalid route path".to_string())); - } - - validate_authentication_method( - new_http_trigger.authentication_method, - new_http_trigger.raw_string, - )?; - - // route path key is extracted from the route path to check for uniqueness - // it replaces /?:{key} with :key - // it will also remove the leading / if present, not an issue as we only allow : after slashes - let route_path_key = ROUTE_PATH_KEY_RE.replace_all(&new_http_trigger.route_path, ":key"); - - let exists = route_path_key_exists( - &route_path_key, - &new_http_trigger.http_method, - &w_id, - None, - new_http_trigger.workspaced_route, - db, - ) - .await?; - - if exists { - return Err(error::Error::BadRequest( - "A route already exists with this path".to_string(), - )); - } - - if *CLOUD_HOSTED - && (new_http_trigger.is_static_website || new_http_trigger.static_asset_config.is_some()) - { - return Err(error::Error::BadRequest( - "Static website and static asset are not supported on cloud".to_string(), - )); - } - - Ok(route_path_key) -} - -async fn create_trigger( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path(w_id): Path, - Json(new_http_trigger): Json, -) -> WindmillResult<(StatusCode, String)> { - require_admin(authed.is_admin, &authed.username)?; - - let route_path_key = validate_http_trigger(&db, &w_id, &new_http_trigger).await?; - - let mut tx = user_db.begin(&authed).await?; - - let http_trigger_path = new_http_trigger.path.clone(); - - create_trigger_inner(&mut tx, &w_id, &authed, &new_http_trigger, &route_path_key).await?; - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::HttpTrigger { path: new_http_trigger.path.clone() }, - Some(format!("HTTP route '{}' created", new_http_trigger.path)), - true, - ) - .await?; - - Ok((StatusCode::CREATED, format!("{}", http_trigger_path))) -} - -async fn update_trigger( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Json(ct): Json, -) -> WindmillResult { - let path = path.to_path(); - check_scopes(&authed, || format!("http_triggers:write:{}", path))?; - - if *CLOUD_HOSTED && (ct.is_static_website || ct.static_asset_config.is_some()) { - return Err(error::Error::BadRequest( - "Static website and static asset are not supported on cloud".to_string(), - )); - } - - validate_authentication_method(ct.authentication_method, ct.raw_string)?; - - let mut tx; - if authed.is_admin { - let Some(route_path) = ct.route_path else { - return Err(error::Error::BadRequest( - "route_path is required".to_string(), - )); - }; - - if !VALID_ROUTE_PATH_RE.is_match(&route_path) { - return Err(error::Error::BadRequest("Invalid route path".to_string())); - } - - let route_path_key = ROUTE_PATH_KEY_RE.replace_all(&route_path, ":key"); - - let exists = route_path_key_exists( - &route_path_key, - &ct.http_method, - &w_id, - Some(&path), - ct.workspaced_route, - &db, - ) - .await?; - if exists { - return Err(error::Error::BadRequest( - "A route already exists with this path".to_string(), - )); - } - - tx = user_db.begin(&authed).await?; - sqlx::query!( - r#" - UPDATE - http_trigger - SET - route_path = $1, - route_path_key = $2, - workspaced_route = $3, - wrap_body = $4, - raw_string = $5, - authentication_resource_path = $6, - script_path = $7, - path = $8, - is_flow = $9, - http_method = $10, - static_asset_config = $11, - edited_by = $12, - email = $13, - is_async = $14, - authentication_method = $15, - summary = $16, - description = $17, - edited_at = now(), - is_static_website = $18, - error_handler_path = $19, - error_handler_args = $20, - retry = $21 - WHERE - workspace_id = $22 AND - path = $23 - "#, - route_path, - &route_path_key, - ct.workspaced_route, - ct.wrap_body, - ct.raw_string, - ct.authentication_resource_path, - ct.script_path, - ct.path, - ct.is_flow, - ct.http_method as _, - ct.static_asset_config as _, - &authed.username, - &authed.email, - ct.is_async, - ct.authentication_method as _, - ct.summary, - ct.description, - ct.is_static_website, - ct.error_handler_path, - ct.error_handler_args as _, - ct.retry as _, - w_id, - path, - ) - .execute(&mut *tx) - .await?; - } else { - tx = user_db.begin(&authed).await?; - sqlx::query!( - r#" - UPDATE - http_trigger - SET - workspaced_route = $1, - wrap_body = $2, - raw_string = $3, - authentication_resource_path = $4, - script_path = $5, - path = $6, - is_flow = $7, - http_method = $8, - static_asset_config = $9, - edited_by = $10, - email = $11, - is_async = $12, - authentication_method = $13, - edited_at = now(), - is_static_website = $14, - error_handler_path = $15, - error_handler_args = $16, - retry = $17 - WHERE - workspace_id = $18 AND - path = $19 - "#, - ct.workspaced_route, - ct.wrap_body, - ct.raw_string, - ct.authentication_resource_path, - ct.script_path, - ct.path, - ct.is_flow, - ct.http_method as _, - ct.static_asset_config as _, - &authed.username, - &authed.email, - ct.is_async, - ct.authentication_method as _, - ct.is_static_website, - ct.error_handler_path, - ct.error_handler_args as _, - ct.retry as _, - w_id, - path, - ) - .execute(&mut *tx) - .await?; - } - - audit_log( - &mut *tx, - &authed, - "http_triggers.update", - ActionKind::Update, - &w_id, - Some(&ct.path), - None, - ) - .await?; - - increase_trigger_version(&mut tx).await?; - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::HttpTrigger { path: ct.path.clone() }, - Some(format!("HTTP route '{}' updated", ct.path)), - true, - ) - .await?; - - Ok(ct.path.to_string()) -} - -async fn delete_trigger( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> WindmillResult { - let path = path.to_path(); - check_scopes(&authed, || format!("http_triggers:write:{}", path))?; - require_admin(authed.is_admin, &authed.username)?; - let mut tx = user_db.begin(&authed).await?; - sqlx::query!( - "DELETE FROM http_trigger - WHERE workspace_id = $1 - AND path = $2", - w_id, - path - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "http_triggers.delete", - ActionKind::Delete, - &w_id, - Some(path), - None, - ) - .await?; - - increase_trigger_version(&mut tx).await?; - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::HttpTrigger { path: path.to_string() }, - Some(format!("HTTP route '{}' deleted", path)), - true, - ) - .await?; - - Ok(format!("HTTP route {path} deleted")) -} - -async fn exists_trigger( - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - let exists = sqlx::query_scalar!( - "SELECT EXISTS( - SELECT 1 FROM http_trigger - WHERE path = $1 AND workspace_id = $2 - )", - path, - w_id - ) - .fetch_one(&db) - .await? - .unwrap_or(false); - - Ok(Json(exists)) -} - -#[derive(Deserialize)] -struct RouteExists { - route_path: String, - http_method: HttpMethod, - trigger_path: Option, - workspaced_route: Option, -} - -async fn route_path_key_exists( - route_path_key: &str, - http_method: &HttpMethod, - w_id: &str, - trigger_path: Option<&str>, - workspaced_route: Option, - db: &DB, -) -> WindmillResult { - let exists = if *CLOUD_HOSTED { - sqlx::query_scalar!( - r#" - SELECT EXISTS( - SELECT 1 - FROM http_trigger - WHERE - route_path_key = $1 - AND workspace_id = $2 - AND http_method = $3 - AND ($4::TEXT IS NULL OR path != $4) - ) - "#, - &route_path_key, - w_id, - http_method as &HttpMethod, - trigger_path - ) - .fetch_one(db) - .await? - .unwrap_or(false) - } else { - let route_path_key = match workspaced_route { - Some(true) => Cow::Owned(format!("{}/{}", w_id, route_path_key.trim_matches('/'))), - _ => Cow::Borrowed(route_path_key), - }; - sqlx::query_scalar!( - r#" - SELECT EXISTS( - SELECT 1 - FROM http_trigger - WHERE - ((workspaced_route IS TRUE AND workspace_id || '/' || route_path_key = $1) - OR (workspaced_route IS FALSE AND route_path_key = $1)) - AND http_method = $2 - AND ($3::TEXT IS NULL OR path != $3) - ) - "#, - &route_path_key, - http_method as &HttpMethod, - trigger_path - ) - .fetch_one(db) - .await? - .unwrap_or(false) - }; - - Ok(exists) -} - -async fn exists_route( - Extension(db): Extension, - Path(w_id): Path, - Json(RouteExists { route_path, http_method, trigger_path, workspaced_route }): Json< - RouteExists, - >, -) -> JsonResult { - let route_path_key = ROUTE_PATH_KEY_RE.replace_all(route_path.as_str(), ":key"); - - let exists = route_path_key_exists( - &route_path_key, - &http_method, - &w_id, - trigger_path.as_deref(), - workspaced_route, - &db, - ) - .await?; - - Ok(Json(exists)) -} - -#[derive(Debug, Deserialize, Clone)] -pub struct TriggerRoute { - path: String, - script_path: String, - is_flow: bool, - route_path: String, - workspace_id: String, - is_async: bool, - authentication_method: AuthenticationMethod, - edited_by: String, - email: String, - static_asset_config: Option>, - is_static_website: bool, - authentication_resource_path: Option, - workspaced_route: bool, - wrap_body: bool, - raw_string: bool, - error_handler_path: Option, - error_handler_args: Option>>>, - retry: Option>, -} - -pub struct RoutersCache { - routers: HashMap>, - version: i64, -} - -lazy_static::lazy_static! { - static ref HTTP_ACCESS_CACHE: Cache<(String, String, ApiAuthed), ExpiringCacheEntry<()>> = Cache::new(100); - static ref HTTP_AUTH_CACHE: Cache<(String, String, ApiAuthed), ExpiringCacheEntry> = Cache::new(100); - - static ref HTTP_ROUTERS_CACHE: RwLock = RwLock::new(RoutersCache { - routers: HashMap::new(), - version: 0, - }); -} - -pub async fn refresh_routers_loop( - db: &DB, - mut killpill_rx: tokio::sync::broadcast::Receiver<()>, -) -> () { - match refresh_routers(db).await { - Ok(_) => { - tracing::info!("Loaded HTTP routers"); - } - Err(err) => { - tracing::error!("Error loading HTTP routers: {err:#}"); - } - }; - let db = db.clone(); - tokio::spawn(async move { - loop { - tokio::select! { - _ = killpill_rx.recv() => { - break; - } - _ = tokio::time::sleep(std::time::Duration::from_secs(60)) => { - match refresh_routers(&db).await { - Ok((true, _)) => { - tracing::info!("Refreshed HTTP routers"); - } - Err(err) => { - tracing::error!("Error refreshing HTTP routers: {err:#}"); - } - _ => {} - } - } - } - } - }); -} - -pub async fn refresh_routers(db: &DB) -> Result<(bool, RwLockReadGuard<'_, RoutersCache>), Error> { - let version = sqlx::query_scalar!("SELECT last_value FROM http_trigger_version_seq",) - .fetch_one(db) - .await?; - let routers_cache = HTTP_ROUTERS_CACHE.read().await; - if routers_cache.version == 0 || version > routers_cache.version { - drop(routers_cache); - let mut routers = HashMap::new(); - - for http_method in [ - HttpMethod::Get, - HttpMethod::Post, - HttpMethod::Put, - HttpMethod::Patch, - HttpMethod::Delete, - ] { - let triggers = sqlx::query_as!( - TriggerRoute, - r#" - SELECT - path, - script_path, - is_flow, - route_path, - authentication_resource_path, - workspace_id, - is_async, - authentication_method AS "authentication_method: _", - edited_by, - email, - static_asset_config AS "static_asset_config: _", - wrap_body, - raw_string, - workspaced_route, - is_static_website, - error_handler_path, - error_handler_args as "error_handler_args: _", - retry as "retry: _" - FROM - http_trigger - WHERE - http_method = $1 - "#, - &http_method as &HttpMethod - ) - .fetch_all(db) - .await?; - - let mut router = matchit::Router::new(); - - for trigger in triggers { - let full_path = if trigger.workspaced_route || *CLOUD_HOSTED { - format!("/{}/{}", trigger.workspace_id, trigger.route_path) - } else { - format!("/{}", trigger.route_path) - }; - - if trigger.is_static_website { - router - .insert(format!("{}/*wm_subpath", full_path), trigger.clone()) - .unwrap_or_else(|e| { - tracing::warn!( - "Failed to consider HTTP route {}/*wm_subpath: {:?}", - full_path, - e, - ); - }); - } - router - .insert(full_path.clone(), trigger.clone()) - .unwrap_or_else(|e| { - tracing::warn!("Failed to consider HTTP route {}: {:?}", full_path, e,); - }); - } - - routers.insert(http_method, router); - } - - let mut routers_cache = HTTP_ROUTERS_CACHE.write().await; - *routers_cache = RoutersCache { routers, version }; - - Ok((true, routers_cache.downgrade())) - } else { - tracing::debug!("No HTTP routers refresh needed"); - Ok((false, routers_cache)) - } -} - -async fn get_http_route_trigger( - route_path: &str, - auth_cache: &Arc, - token: Option<&String>, - db: &DB, - user_db: UserDB, - method: &http::Method, -) -> WindmillResult<(TriggerRoute, String, HashMap, ApiAuthed)> { - let http_method: HttpMethod = method.try_into()?; - - let requested_path = format!("/{}", route_path); - - let routers_cache = HTTP_ROUTERS_CACHE.read().await; - - let routers_cache = if routers_cache.routers.is_empty() { - tracing::warn!("HTTP routers are not loaded, loading from db"); - let (_, routers_cache) = refresh_routers(db).await?; - routers_cache - } else { - routers_cache - }; - - let router = routers_cache - .routers - .get(&http_method) - .ok_or(error::Error::internal_err( - "HTTP routers could not be loaded".to_string(), - ))?; - - let trigger_match = router.at(requested_path.as_str()).ok(); - - let matchit::Match { value: trigger, params } = - not_found_if_none(trigger_match, "Trigger", requested_path.as_str())?; - - let params: HashMap = params - .iter() - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect(); - - let username_override = if let AuthenticationMethod::Windmill = trigger.authentication_method { - let opt_authed = if let Some(token) = token { - auth_cache - .get_authed(Some(trigger.workspace_id.clone()), token) - .await - } else { - None - }; - if let Some(authed) = opt_authed { - // check that the user has access to the trigger - let cache_key = ( - trigger.workspace_id.clone(), - trigger.path.clone(), - authed.clone(), - ); - let exists = match HTTP_ACCESS_CACHE.get(&cache_key) { - Some(cache_entry) if cache_entry.expiry > std::time::Instant::now() => { - tracing::debug!("HTTP access cache hit for route {}", trigger.path); - true - } - _ => { - tracing::debug!("HTTP access cache miss for route {}", trigger.path); - let mut tx = user_db.begin(&authed).await?; - let exists = sqlx::query_scalar!( - r#" - SELECT EXISTS( - SELECT 1 - FROM - http_trigger - WHERE - workspace_id = $1 AND - path = $2 - ) - "#, - trigger.workspace_id, - trigger.path - ) - .fetch_one(&mut *tx) - .await? - .unwrap_or(false); - if exists { - HTTP_ACCESS_CACHE.insert( - cache_key, - ExpiringCacheEntry { - value: (), - expiry: std::time::Instant::now() - + std::time::Duration::from_secs(10), - }, - ); - } - exists - } - }; - if exists { - Some(authed.display_username().to_owned()) - } else { - return Err(error::Error::NotAuthorized("Unauthorized".to_string())); - } - } else { - return Err(error::Error::NotAuthorized( - "Requires authentication".to_string(), - )); - } - } else { - None - }; - - let authed = fetch_api_authed( - trigger.edited_by.clone(), - trigger.email.clone(), - &trigger.workspace_id, - &db, - Some(username_override.unwrap_or(format!("HTTP-{}", trigger.path))), - ) - .await?; - - Ok((trigger.clone(), route_path.to_string(), params, authed)) -} - -async fn route_job( - Extension(db): Extension, - Extension(user_db): Extension, - Extension(auth_cache): Extension>, - OptTokened { token }: OptTokened, - Path(route_path): Path, - headers: HeaderMap, - args: RawHttpTriggerArgs, -) -> Result { - let route_path = route_path.to_path().trim_end_matches("/"); - let (trigger, called_path, params, authed) = get_http_route_trigger( - route_path, - &auth_cache, - token.as_ref(), - &db, - user_db.clone(), - &args.0.metadata.method, - ) - .await - .map_err(|e| e.into_response())?; - - if trigger.script_path.is_empty() && trigger.static_asset_config.is_none() { - return Err(Error::NotFound(format!( - "Runnable path of HTTP route at path: {}", - trigger.path - )) - .into_response()); - } - - let args = args - .process_args( - &authed, - &db, - &trigger.workspace_id, - match trigger.authentication_method { - AuthenticationMethod::CustomScript | AuthenticationMethod::Signature => true, - _ => trigger.raw_string, - }, - ) - .await - .map_err(|e| e.into_response())?; - - match trigger.authentication_method { - AuthenticationMethod::None - | AuthenticationMethod::Windmill - | AuthenticationMethod::CustomScript => {} - _ => { - let resource_path = match trigger.authentication_resource_path { - Some(resource_path) => resource_path, - None => { - return Err(Error::BadRequest( - "Missing authentication resource path".to_string(), - ) - .into_response()) - } - }; - - let cache_key = ( - trigger.workspace_id.clone(), - resource_path.clone(), - authed.clone(), - ); - - let authentication_method = match HTTP_AUTH_CACHE.get(&cache_key) { - Some(cache_entry) if cache_entry.expiry > std::time::Instant::now() => { - tracing::debug!("HTTP auth method cache hit for route {}", trigger.path); - cache_entry.value - } - _ => { - tracing::debug!("HTTP auth method cache miss for route {}", trigger.path); - let auth_method = try_get_resource_from_db_as::< - crate::http_trigger_auth::AuthenticationMethod, - >( - &authed, - Some(user_db.clone()), - &db, - &resource_path, - &trigger.workspace_id, - ) - .await - .map_err(|e| e.into_response())?; - HTTP_AUTH_CACHE.insert( - cache_key, - ExpiringCacheEntry { - value: auth_method.clone(), - expiry: std::time::Instant::now() + std::time::Duration::from_secs(60), - }, - ); - auth_method - } - }; - - let raw_payload = args.0.metadata.raw_string.as_ref(); - - let response = authentication_method - .authenticate_http_request(&headers, raw_payload) - .map_err(|e| e.into_response())?; - - if let Some(response) = response { - return Ok(response); - } - } - } - - #[cfg(not(feature = "parquet"))] - if trigger.static_asset_config.is_some() { - return Err(error::Error::internal_err( - "Static asset configuration is not supported in this build".to_string(), - ) - .into_response()); - } - - #[cfg(feature = "parquet")] - if let Some(sqlx::types::Json(config)) = trigger.static_asset_config { - let build_static_response_f = async { - let (_, s3_resource_opt) = get_workspace_s3_resource( - &authed, - &db, - None, - &"NO_TOKEN".to_string(), // no token is provided in this case - &trigger.workspace_id, - config.storage, - ) - .await?; - let s3_resource = s3_resource_opt.ok_or(error::Error::internal_err( - "No files storage resource defined at the workspace level".to_string(), - ))?; - let s3_client = build_object_store_client(&s3_resource).await?; - - let path = if trigger.is_static_website { - let subpath = params - .get("wm_subpath") - .cloned() - .unwrap_or("index.html".to_string()); - tracing::info!("subpath: {}", subpath); - format!("{}/{}", config.s3.trim_end_matches('/'), subpath) - } else { - config.s3.clone() - }; - let path = object_store::path::Path::from(path); - let s3_object = s3_client.get(&path).await; - - let s3_object = match s3_object { - Err(object_store::Error::NotFound { .. }) if trigger.is_static_website => { - // fallback to index.html if the file is not found - let path = object_store::path::Path::from(format!( - "{}/index.html", - config.s3.trim_end_matches('/') - )); - s3_client.get(&path).await - } - r => r, - }; - - let s3_object = s3_object.map_err(|err| { - tracing::warn!("Error retrieving file from S3: {:?}", err); - error::Error::internal_err(format!("Error retrieving file: {}", err.to_string())) - })?; - - let mut response_headers = http::HeaderMap::new(); - if let Some(ref e_tag) = s3_object.meta.e_tag { - if let Some(if_none_match) = headers.get(IF_NONE_MATCH) { - if if_none_match == e_tag { - return Ok::<_, error::Error>(( - StatusCode::NOT_MODIFIED, - response_headers, - axum::body::Body::empty(), - )); - } - } - if let Ok(e_tag) = e_tag.parse() { - response_headers.insert("etag", e_tag); - } - } - response_headers.insert( - "content-type", - s3_object - .attributes - .get(&object_store::Attribute::ContentType) - .map(|s| s.parse().ok()) - .flatten() - .unwrap_or("application/octet-stream".parse().unwrap()), - ); - if !trigger.is_static_website { - response_headers.insert( - "content-disposition", - config.filename.as_ref().map_or_else( - || { - s3_object - .attributes - .get(&object_store::Attribute::ContentDisposition) - .map(|s| s.parse().ok()) - .flatten() - .unwrap_or("inline".parse().unwrap()) - }, - |filename| { - format!("inline; filename=\"{}\"", filename) - .parse() - .unwrap_or("inline".parse().unwrap()) - }, - ), - ); - } - - let body_stream = axum::body::Body::from_stream(s3_object.into_stream()); - Ok::<_, error::Error>((StatusCode::OK, response_headers, body_stream)) - }; - match build_static_response_f.await { - Ok((status, headers, body_stream)) => { - return Ok((status, headers, body_stream).into_response()) - } - Err(e) => return Err(e.into_response()), - } - } - - let runnable_format = get_runnable_format( - if trigger.is_flow { - RunnableId::from_flow_path(&trigger.script_path) - } else { - RunnableId::from_script_path(&trigger.script_path) - }, - &trigger.workspace_id, - &db, - &TriggerKind::Http, - ) - .await - .map_err(|e| e.into_response())?; - - let args = args - .to_args_from_format( - &trigger.route_path, - &called_path, - ¶ms, - runnable_format, - trigger.wrap_body, - ) - .map_err(|e| e.into_response())?; - - if trigger.is_async { - trigger_runnable( - &db, - Some(user_db), - authed, - &trigger.workspace_id, - &trigger.script_path, - trigger.is_flow, - args, - trigger.retry.as_ref(), - trigger.error_handler_path.as_deref(), - trigger.error_handler_args.as_ref(), - format!("http_trigger/{}", trigger.path), - ) - .await - .map_err(|e| e.into_response()) - } else { - trigger_runnable_and_wait_for_result( - &db, - Some(user_db), - authed, - &trigger.workspace_id, - &trigger.script_path, - trigger.is_flow, - args, - trigger.retry.as_ref(), - trigger.error_handler_path.as_deref(), - trigger.error_handler_args.as_ref(), - format!("http_trigger/{}", trigger.path), - ) - .await - .map_err(|e| e.into_response()) - } -} diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 5332bcb6c4..cbdc5eb5cc 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -48,7 +48,7 @@ use crate::{ auth::{OptTokened, Tokened}, concurrency_groups::join_concurrency_key, db::{ApiAuthed, DB}, - trigger_helpers::RunnableId, + triggers::trigger_helpers::RunnableId, users::{get_scope_tags, require_owner_of_path, OptAuthed}, utils::{check_scopes, content_plain, require_super_admin}, }; @@ -332,21 +332,16 @@ async fn get_root_job( Ok(Json(res)) } -async fn compute_root_job_for_flow(db: &DB, w_id: &str, mut job_id: Uuid) -> error::Result { - // TODO: use `root_job` ? - loop { - job_id = match sqlx::query_scalar!( - "SELECT parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2", - job_id, - w_id - ) - .fetch_one(db) - .await - { - Ok(Some(job_id)) => job_id, - _ => return Ok(job_id.to_string()), - } - } +async fn compute_root_job_for_flow(db: &DB, w_id: &str, job_id: Uuid) -> error::Result { + let root_job = sqlx::query_scalar!( + r#"SELECT COALESCE(root_job, flow_innermost_root_job, parent_job, id) as "root_job!" FROM v2_job WHERE id = $1 AND workspace_id = $2"#, + job_id, + w_id + ) + .fetch_one(db) + .await?; + + Ok(root_job.to_string()) } async fn get_db_clock(Extension(db): Extension) -> windmill_common::error::JsonResult { @@ -3875,6 +3870,7 @@ pub async fn run_flow_by_path_inner( scheduled_for, None, run_query.parent_job, + None, run_query.root_job, run_query.job_id, false, @@ -3970,6 +3966,7 @@ pub async fn restart_flow( scheduled_for, None, run_query.parent_job, + None, run_query.root_job, run_query.job_id, false, @@ -4066,6 +4063,7 @@ pub async fn run_script_by_path_inner( scheduled_for, None, run_query.parent_job, + None, run_query.root_job, run_query.job_id, false, @@ -4215,6 +4213,7 @@ pub async fn run_workflow_as_code( scheduled_for, None, Some(job_id), + None, job.root_job.or(Some(job_id)), run_query.job_id, false, @@ -4746,6 +4745,7 @@ pub async fn run_wait_result_job_by_path_get( None, None, run_query.parent_job, + None, run_query.root_job, run_query.job_id, false, @@ -4897,6 +4897,7 @@ pub async fn run_wait_result_script_by_path_internal( None, None, run_query.parent_job, + None, run_query.root_job, run_query.job_id, false, @@ -5012,6 +5013,7 @@ pub async fn run_wait_result_script_by_hash( None, None, run_query.parent_job, + None, run_query.root_job, run_query.job_id, false, @@ -5130,6 +5132,7 @@ pub async fn run_wait_result_flow_by_path_internal( scheduled_for, None, run_query.parent_job, + None, run_query.root_job, run_query.job_id, false, @@ -5201,6 +5204,7 @@ async fn run_preview_script( None, None, None, + None, run_query.job_id, false, false, @@ -5312,6 +5316,7 @@ async fn run_bundle_preview_script( None, None, None, + None, run_query.job_id, false, false, @@ -5479,6 +5484,7 @@ async fn run_dependencies_job( None, None, None, + None, false, false, None, @@ -5545,6 +5551,7 @@ async fn run_flow_dependencies_job( None, None, None, + None, false, false, None, @@ -5885,6 +5892,7 @@ async fn run_preview_flow_job( None, None, None, + None, run_query.job_id, false, false, @@ -6024,6 +6032,7 @@ pub async fn run_job_by_hash_inner( scheduled_for, None, run_query.parent_job, + None, run_query.root_job, run_query.job_id, false, diff --git a/backend/windmill-api/src/kafka_triggers_oss.rs b/backend/windmill-api/src/kafka_triggers_oss.rs index 22cbf3785d..4227023648 100644 --- a/backend/windmill-api/src/kafka_triggers_oss.rs +++ b/backend/windmill-api/src/kafka_triggers_oss.rs @@ -5,19 +5,12 @@ pub use crate::kafka_triggers_ee::*; #[cfg(not(feature = "private"))] use crate::db::DB; #[cfg(not(feature = "private"))] -use axum::Router; -#[cfg(not(feature = "private"))] use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] #[cfg(not(feature = "private"))] pub struct KafkaResourceSecurity {} -#[cfg(not(feature = "private"))] -pub fn workspaced_service() -> Router { - Router::new() -} - #[cfg(not(feature = "private"))] pub fn start_kafka_consumers( _db: DB, @@ -54,7 +47,9 @@ pub struct KafkaTrigger { #[serde(skip_serializing_if = "Option::is_none")] pub error_handler_path: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_args: Option>>>, + pub error_handler_args: Option< + sqlx::types::Json>>, + >, #[serde(skip_serializing_if = "Option::is_none")] pub retry: Option>, } diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 842d8af40c..6d9fd79b60 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -89,12 +89,6 @@ mod flows; mod folders; mod granular_acls; mod groups; -#[cfg(feature = "http_trigger")] -mod http_trigger_args; -#[cfg(feature = "http_trigger")] -mod http_trigger_auth; -#[cfg(feature = "http_trigger")] -pub mod http_triggers; #[cfg(feature = "private")] pub mod indexer_ee; mod indexer_oss; @@ -106,6 +100,7 @@ mod integration; mod live_migrations; #[cfg(feature = "postgres_trigger")] mod postgres_triggers; +mod trigger_helpers; pub mod openapi; @@ -170,7 +165,6 @@ mod sqs_triggers_oss; #[cfg(feature = "private")] pub mod teams_approvals_ee; mod teams_approvals_oss; -mod trigger_helpers; mod static_assets; #[cfg(all(feature = "stripe", feature = "enterprise", feature = "private"))] @@ -182,12 +176,13 @@ pub mod teams_ee; mod teams_oss; mod token; mod tracing_init; -mod triggers; +pub mod triggers; mod users; #[cfg(feature = "private")] pub mod users_ee; mod users_oss; mod utils; +pub mod var_resource_cache; mod variables; pub mod webhook_util; #[cfg(feature = "websocket")] @@ -305,6 +300,7 @@ pub async fn run_server( )); let argon2 = Arc::new(Argon2::default()); + let disable_response_logs = std::env::var("DISABLE_RESPONSE_LOGS") .ok() .map(|x| x == "true") @@ -379,101 +375,14 @@ pub async fn run_server( } }; - let kafka_triggers_service = { - #[cfg(all(feature = "enterprise", feature = "kafka"))] - { - kafka_triggers_oss::workspaced_service() - } - - #[cfg(not(all(feature = "enterprise", feature = "kafka")))] - { - Router::new() - } - }; - - let nats_triggers_service = { - #[cfg(all(feature = "enterprise", feature = "nats"))] - { - nats_triggers_oss::workspaced_service() - } - - #[cfg(not(all(feature = "enterprise", feature = "nats")))] - { - Router::new() - } - }; - - let mqtt_triggers_service = { - #[cfg(all(feature = "mqtt_trigger"))] - { - mqtt_triggers::workspaced_service() - } - - #[cfg(not(feature = "mqtt_trigger"))] - { - Router::new() - } - }; - - let gcp_triggers_service = { - #[cfg(all(feature = "enterprise", feature = "gcp_trigger"))] - { - gcp_triggers_oss::workspaced_service() - } - - #[cfg(not(all(feature = "enterprise", feature = "gcp_trigger")))] - { - Router::new() - } - }; - - let sqs_triggers_service = { - #[cfg(all(feature = "enterprise", feature = "sqs_trigger"))] - { - sqs_triggers_oss::workspaced_service() - } - - #[cfg(not(all(feature = "enterprise", feature = "sqs_trigger")))] - { - Router::new() - } - }; - - let websocket_triggers_service = { - #[cfg(feature = "websocket")] - { - websocket_triggers::workspaced_service() - } - - #[cfg(not(feature = "websocket"))] - Router::new() - }; - - let http_triggers_service = { - #[cfg(feature = "http_trigger")] - { - http_triggers::workspaced_service() - } - - #[cfg(not(feature = "http_trigger"))] - Router::new() - }; - + // Initialize HTTP trigger refresh loop #[cfg(feature = "http_trigger")] { let http_killpill_rx = killpill_rx.resubscribe(); - http_triggers::refresh_routers_loop(&db, http_killpill_rx).await; + triggers::http::refresh_routers_loop(&db, http_killpill_rx).await; } - let postgres_triggers_service = { - #[cfg(feature = "postgres_trigger")] - { - postgres_triggers::workspaced_service() - } - - #[cfg(not(feature = "postgres_trigger"))] - Router::new() - }; + let triggers_service = triggers::generate_trigger_routers(); if !*CLOUD_HOSTED && server_mode && !mcp_mode { #[cfg(feature = "websocket")] @@ -617,14 +526,7 @@ pub async fn run_server( .nest("/workspaces", workspaces::workspaced_service()) .nest("/oidc", oidc_oss::workspaced_service()) .nest("/openapi", openapi::openapi_service()) - .nest("/http_triggers", http_triggers_service) - .nest("/websocket_triggers", websocket_triggers_service) - .nest("/kafka_triggers", kafka_triggers_service) - .nest("/nats_triggers", nats_triggers_service) - .nest("/mqtt_triggers", mqtt_triggers_service) - .nest("/sqs_triggers", sqs_triggers_service) - .nest("/gcp_triggers", gcp_triggers_service) - .nest("/postgres_triggers", postgres_triggers_service), + .merge(triggers_service), ) .nest("/workspaces", workspaces::global_service()) .nest( @@ -773,7 +675,7 @@ pub async fn run_server( { #[cfg(feature = "http_trigger")] { - http_triggers::routes_global_service() + triggers::http::handler::http_route_trigger_handler() } #[cfg(not(feature = "http_trigger"))] @@ -786,11 +688,19 @@ pub async fn run_server( .nest( "/gcp/w/:workspace_id", { - #[cfg(all(feature = "enterprise", feature = "gcp_trigger"))] + #[cfg(all( + feature = "enterprise", + feature = "gcp_trigger", + feature = "private" + ))] { - gcp_triggers_oss::gcp_push_route_handler() + triggers::gcp::handler_oss::gcp_push_route_handler() } - #[cfg(not(all(feature = "enterprise", feature = "gcp_trigger")))] + #[cfg(not(all( + feature = "enterprise", + feature = "gcp_trigger", + feature = "private" + )))] { Router::new() } diff --git a/backend/windmill-api/src/mcp/server.rs b/backend/windmill-api/src/mcp/server.rs index 8bfe35a009..e51d008e3c 100644 --- a/backend/windmill-api/src/mcp/server.rs +++ b/backend/windmill-api/src/mcp/server.rs @@ -190,9 +190,10 @@ impl ServerHandler for Runner { } // Continue with script/flow logic - let (tool_type, path, is_hub) = - reverse_transform(&request.name).unwrap_or_default(); - + let (tool_type, path, is_hub) = reverse_transform(&request.name).map_err(|e| { + Error::internal_error(format!("Failed to reverse transform path: {}", e), None) + })?; + let item_schema = if is_hub { get_hub_script_schema(&format!("hub/{}", path), db).await? } else { diff --git a/backend/windmill-api/src/mcp/utils/transform.rs b/backend/windmill-api/src/mcp/utils/transform.rs index 6574173bc4..c40fe37263 100644 --- a/backend/windmill-api/src/mcp/utils/transform.rs +++ b/backend/windmill-api/src/mcp/utils/transform.rs @@ -12,16 +12,9 @@ use super::models::SchemaType; /// have names with slashes. Because we replace slashes with underscores, /// we also need to escape underscores. pub fn transform_path(path: &str, type_str: &str) -> String { - // Only apply special underscore escaping for paths starting with "f/" - let transformed = if path.starts_with("f/") { - let escaped_path = path.replace('_', "__"); - escaped_path.replace('/', "_") - } else { - path.replace('/', "_") - }; - + let escaped_path = path.replace('_', "__").replace('/', "_"); // first letter of type_str is used as prefix, only one letter to avoid reaching 60 char name limit - format!("{}-{}", &type_str[..1], transformed) + format!("{}-{}", &type_str[..1], escaped_path) } /// Reverse the transformation of a path @@ -53,19 +46,15 @@ pub fn reverse_transform(transformed_path: &str) -> Result<(&str, String, bool), let mangled_path = &transformed_path[2..]; - // Check if this path was previously transformed with special underscore handling - let is_special_path = mangled_path.starts_with("f_"); - let original_path = if is_hub { let parts = mangled_path.split("-").collect::>(); + if parts.is_empty() { + return Err(format!("Invalid transformed path: {}", transformed_path)); + } parts[0].to_string() - } else if is_special_path { - const TEMP_PLACEHOLDER: &str = "@@UNDERSCORE@@"; - let path_with_placeholder = mangled_path.replace("__", TEMP_PLACEHOLDER); - let path_with_slashes = path_with_placeholder.replace('_', "/"); - path_with_slashes.replace(TEMP_PLACEHOLDER, "_") } else { - mangled_path.replacen('_', "/", 2) + const TEMP_PLACEHOLDER: &str = "@@UNDERSCORE@@"; + mangled_path.replace("__", TEMP_PLACEHOLDER).replace('_', "/").replace(TEMP_PLACEHOLDER, "_") }; Ok((type_str, original_path, is_hub)) diff --git a/backend/windmill-api/src/mqtt_triggers.rs b/backend/windmill-api/src/mqtt_triggers.rs index 5659c9b944..2b8c16fc6d 100644 --- a/backend/windmill-api/src/mqtt_triggers.rs +++ b/backend/windmill-api/src/mqtt_triggers.rs @@ -4,22 +4,11 @@ use crate::{ resources::try_get_resource_from_db_as, trigger_helpers::{trigger_runnable, TriggerJobArgs}, users::fetch_api_authed, - utils::check_scopes, }; -use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; -use axum::{ - async_trait, - extract::{Path, Query}, - Extension, Json, -}; -use axum::{ - routing::{delete, get, post}, - Router, -}; +use axum::async_trait; use base64::{engine, prelude::*}; use bytes::Bytes; -use http::StatusCode; use itertools::Itertools; use rumqttc::{ v5::{ @@ -35,17 +24,15 @@ use rumqttc::{ TlsConfiguration, Transport, }; use serde::{Deserialize, Serialize}; -use sql_builder::{bind::Bind, SqlBuilder}; use sqlx::{FromRow, Type}; use std::collections::HashMap; use std::time::Duration; -use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::{ db::UserDB, - error::{self, JsonResult}, + error::{to_anyhow, Error, Result}, triggers::TriggerKind, - utils::{not_found_if_none, paginate, report_critical_error, Pagination, StripPath}, - worker::{to_raw_value, CLOUD_HOSTED}, + utils::report_critical_error, + worker::to_raw_value, INSTANCE_NAME, }; @@ -53,34 +40,6 @@ use rand::seq::SliceRandom; use serde_json::value::RawValue; use sqlx::types::Json as SqlxJson; -pub fn workspaced_service() -> Router { - Router::new() - .route("/create", post(create_mqtt_trigger)) - .route("/list", get(list_mqtt_triggers)) - .route("/get/*path", get(get_mqtt_trigger)) - .route("/update/*path", post(update_mqtt_trigger)) - .route("/delete/*path", delete(delete_mqtt_trigger)) - .route("/exists/*path", get(exists_mqtt_trigger)) - .route("/setenabled/*path", post(set_enabled)) - .route("/test", post(test_mqtt_connection)) -} - -#[derive(Debug, thiserror::Error)] -enum Error { - #[error("{0}")] - Common(#[from] windmill_common::error::Error), - #[error("{0}")] - V5RumqttClient(#[from] rumqttc::v5::ClientError), - #[error("{0}")] - V5ConnectionError(#[from] rumqttc::v5::ConnectionError), - #[error("{0}")] - V3RumqttClient(#[from] rumqttc::ClientError), - #[error("{0}")] - V3ConnectionError(#[from] rumqttc::ConnectionError), - #[error("{0}")] - Base64Decode(#[from] base64::DecodeError), -} - async fn run_job( payload: &[u8], trigger_info: HashMap>, @@ -124,6 +83,18 @@ async fn run_job( Ok(()) } +#[derive(Debug, Deserialize, Serialize)] +pub struct MqttV3Config { + clean_session: Option, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct MqttV5Config { + clean_start: Option, + session_expiry_interval: Option, + topic_alias_maximum: Option, +} + #[derive(Clone, Debug, Deserialize, Serialize, Type)] #[serde(rename_all = "lowercase")] pub enum QualityOfService { @@ -152,18 +123,6 @@ impl From for V5QoS { } } -#[derive(Debug, Deserialize, Serialize)] -pub struct MqttV3Config { - clean_session: Option, -} - -#[derive(Debug, Deserialize, Serialize)] -pub struct MqttV5Config { - clean_start: Option, - session_expiry_interval: Option, - topic_alias_maximum: Option, -} - #[derive(Debug, Deserialize, Serialize, Type)] #[sqlx(type_name = "MQTT_CLIENT_VERSION")] #[sqlx(rename_all = "lowercase")] @@ -201,87 +160,6 @@ pub struct SubscribeTopic { topic: String, } -#[derive(Debug, Serialize, Deserialize)] -pub struct NewMqttTrigger { - mqtt_resource_path: String, - subscribe_topics: Vec, - v3_config: Option, - v5_config: Option, - #[serde(skip_serializing_if = "Option::is_none")] - client_version: Option, - client_id: Option, - path: String, - script_path: String, - is_flow: bool, - enabled: bool, - error_handler_path: Option, - error_handler_args: Option>>>, - retry: Option>, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct EditMqttTrigger { - mqtt_resource_path: String, - subscribe_topics: Vec, - v3_config: Option, - v5_config: Option, - client_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - client_version: Option, - path: String, - script_path: String, - is_flow: bool, - error_handler_path: Option, - error_handler_args: Option>>>, - retry: Option>, -} - -#[derive(Debug, Serialize, Deserialize, FromRow)] -pub struct MqttTrigger { - pub mqtt_resource_path: String, - pub subscribe_topics: Vec>, - pub v3_config: Option>, - pub v5_config: Option>, - pub client_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub client_version: Option, - pub path: String, - pub script_path: String, - pub is_flow: bool, - pub workspace_id: String, - pub edited_by: String, - pub email: String, - pub edited_at: chrono::DateTime, - pub extra_perms: Option, - pub error: Option, - pub server_id: Option, - pub last_server_ping: Option>, - pub enabled: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_args: Option>>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub retry: Option>, -} - -#[derive(Deserialize, Serialize)] -pub struct ListMqttTriggerQuery { - page: Option, - per_page: Option, - path: Option, - is_flow: Option, - path_start: Option, -} - -#[derive(Deserialize)] -pub struct SetEnabled { - enabled: bool, -} - -const KEEP_ALIVE: u64 = 60; -const CLIENT_CONNECTION_TIMEOUT: u64 = 60; -const TOPIC_ALIAS_MAXIMUM: u16 = 65535; struct MqttClientBuilder<'client> { mqtt_resource: MqttResource, client_id: &'client str, @@ -310,14 +188,14 @@ impl<'client> MqttClientBuilder<'client> { } } - async fn build_client(&self) -> Result { + async fn build_client(&self) -> Result { match self.mqtt_client_version { Some(MqttClientVersion::V5) | None => self.build_v5_client().await, Some(MqttClientVersion::V3) => self.build_v3_client().await, } } - fn get_tls_configuration(&self) -> Result, Error> { + fn get_tls_configuration(&self) -> Result> { let transport = match self.mqtt_resource.tls { Some(ref tls) if tls.enabled => { let transport = match tls.ca_certificate.trim().is_empty() { @@ -329,8 +207,9 @@ impl<'client> MqttClientBuilder<'client> { Some(client_certificate) if !client_certificate.trim().is_empty() => { - let client_certificate = - BASE64_STANDARD.decode(client_certificate)?; + let client_certificate = BASE64_STANDARD + .decode(client_certificate) + .map_err(to_anyhow)?; let password = tls .pkcs12_certificate_password .clone() @@ -351,7 +230,7 @@ impl<'client> MqttClientBuilder<'client> { Ok(transport) } - async fn build_v5_client(&self) -> Result { + async fn build_v5_client(&self) -> Result { let mut mqtt_options = V5MqttOptions::new( self.client_id, &self.mqtt_resource.broker, @@ -398,12 +277,15 @@ impl<'client> MqttClientBuilder<'client> { .map(|topic| Filter::new(topic.topic.clone(), topic.qos.clone().into())) .collect_vec(); - async_client.subscribe_many(subscribe_filters).await?; + async_client + .subscribe_many(subscribe_filters) + .await + .map_err(to_anyhow)?; } Ok(MqttClientResult::V5((V5MqttHandler, event_loop))) } - async fn build_v3_client(&self) -> Result { + async fn build_v3_client(&self) -> Result { let mut mqtt_options = V3MqttOptions::new( self.client_id, &self.mqtt_resource.broker, @@ -435,574 +317,62 @@ impl<'client> MqttClientBuilder<'client> { .map(|topic| SubscribeFilter::new(topic.topic.clone(), topic.qos.clone().into())) .collect_vec(); - async_client.subscribe_many(subscribe_filters).await?; + async_client + .subscribe_many(subscribe_filters) + .await + .map_err(to_anyhow)?; } Ok(MqttClientResult::V3((V3MqttHandler, event_loop))) } } -fn convert_disconnect_packet_into_string( - disconnect: rumqttc::v5::mqttbytes::v5::Disconnect, -) -> String { +#[derive(Debug, Serialize, Deserialize, FromRow)] +pub struct MqttTrigger { + pub mqtt_resource_path: String, + pub subscribe_topics: Vec>, + pub v3_config: Option>, + pub v5_config: Option>, + pub client_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub client_version: Option, + pub path: String, + pub script_path: String, + pub is_flow: bool, + pub workspace_id: String, + pub edited_by: String, + pub email: String, + pub edited_at: chrono::DateTime, + pub extra_perms: Option, + pub error: Option, + pub server_id: Option, + pub last_server_ping: Option>, + pub enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option>>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub retry: Option>, +} + +const KEEP_ALIVE: u64 = 60; +const CLIENT_CONNECTION_TIMEOUT: u64 = 60; +const TOPIC_ALIAS_MAXIMUM: u16 = 65535; + +fn convert_disconnect_packet_into_err(disconnect: rumqttc::v5::mqttbytes::v5::Disconnect) -> Error { let err_message = disconnect .properties .map(|properties| properties.reason_string) .flatten(); let reason_code = disconnect.reason_code as u8; - format!( + anyhow::anyhow!( "Disconnected by the broker, reason code: {}, {}", reason_code, err_message .map(|err| format!("message: {}", err)) .unwrap_or("".to_string()) ) -} - -#[derive(Debug, Deserialize)] -pub struct TestMqttConnection { - mqtt_resource_path: String, - client_version: Option, - v3_config: Option, - v5_config: Option, -} - -pub async fn test_mqtt_connection( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path(workspace_id): Path, - Json(test_postgres): Json, -) -> error::Result<()> { - let TestMqttConnection { mqtt_resource_path, client_version, v3_config, v5_config } = - test_postgres; - - let mqtt_resource = try_get_resource_from_db_as::( - &authed, - Some(user_db), - &db, - &mqtt_resource_path, - &workspace_id, - ) - .await?; - - let connect_f = async { - let client_builder = MqttClientBuilder::new( - mqtt_resource, - Some(""), - vec![], - v3_config.as_ref(), - v5_config.as_ref(), - client_version.as_ref(), - ); - - client_builder.build_client().await.map_err(|err| { - error::Error::BadConfig(format!( - "Error connecting to mqtt broker: {}", - err.to_string() - )) - }) - }; - tokio::time::timeout(tokio::time::Duration::from_secs(30), connect_f) - .await - .map_err(|_| { - error::Error::BadConfig(format!( - "Timeout occurred while trying to connect to mqtt broker after 30 seconds" - )) - })??; - - Ok(()) -} - -pub async fn create_mqtt_trigger( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path(w_id): Path, - Json(new_mqtt_trigger): Json, -) -> error::Result<(StatusCode, String)> { - check_scopes(&authed, || { - format!("mqtt_triggers:write:{}", &new_mqtt_trigger.path) - })?; - if *CLOUD_HOSTED { - return Err(error::Error::BadRequest( - "MQTT triggers are not supported on multi-tenant cloud, use dedicated cloud or self-host".to_string(), - )); - } - - let NewMqttTrigger { - mqtt_resource_path, - subscribe_topics, - path, - script_path, - enabled, - is_flow, - v3_config, - v5_config, - client_version, - client_id, - error_handler_path, - error_handler_args, - retry, - } = new_mqtt_trigger; - - let mut tx = user_db.begin(&authed).await?; - - let subscribe_topics = subscribe_topics.into_iter().map(SqlxJson).collect_vec(); - let v3_config = v3_config.map(SqlxJson); - let v5_config = v5_config.map(SqlxJson); - - sqlx::query!( - r#" - INSERT INTO mqtt_trigger ( - mqtt_resource_path, - subscribe_topics, - client_version, - client_id, - v3_config, - v5_config, - workspace_id, - path, - script_path, - is_flow, - email, - enabled, - edited_by, - error_handler_path, - error_handler_args, - retry - ) - VALUES ( - $1, - $2, - $3, - $4, - $5, - $6, - $7, - $8, - $9, - $10, - $11, - $12, - $13, - $14, - $15, - $16 - )"#, - mqtt_resource_path, - subscribe_topics.as_slice() as &[SqlxJson], - client_version as Option, - client_id, - v3_config as Option>, - v5_config as Option>, - &w_id, - &path, - script_path, - is_flow, - &authed.email, - enabled, - &authed.username, - error_handler_path, - error_handler_args as _, - retry as _ - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "mqtt_triggers.create", - ActionKind::Create, - &w_id, - Some(path.as_str()), - None, - ) - .await?; - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::MqttTrigger { path: path.to_string() }, - Some(format!("MQTT trigger '{}' created", path)), - true, - ) - .await?; - - Ok((StatusCode::CREATED, format!("{}", path.to_string()))) -} - -pub async fn list_mqtt_triggers( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, - Query(lst): Query, -) -> error::JsonResult> { - let mut tx = user_db.begin(&authed).await?; - let (per_page, offset) = paginate(Pagination { per_page: lst.per_page, page: lst.page }); - let mut sqlb = SqlBuilder::select_from("mqtt_trigger"); - sqlb.fields(&[ - "mqtt_resource_path", - "subscribe_topics", - "v3_config", - "v5_config", - "client_version", - "client_id", - "workspace_id", - "path", - "script_path", - "is_flow", - "edited_by", - "email", - "edited_at", - "server_id", - "last_server_ping", - "extra_perms", - "error", - "enabled", - "error_handler_path", - "error_handler_args", - "retry", - ]) - .order_by("edited_at", true) - .and_where("workspace_id = ?".bind(&w_id)) - .offset(offset) - .limit(per_page); - if let Some(path) = lst.path { - sqlb.and_where_eq("script_path", "?".bind(&path)); - } - if let Some(is_flow) = lst.is_flow { - sqlb.and_where_eq("is_flow", "?".bind(&is_flow)); - } - if let Some(path_start) = &lst.path_start { - sqlb.and_where_like_left("path", path_start); - } - let sql = sqlb - .sql() - .map_err(|e| error::Error::InternalErr(e.to_string()))?; - let rows = sqlx::query_as::<_, MqttTrigger>(&sql) - .fetch_all(&mut *tx) - .await - .map_err(|e| { - tracing::debug!("Error fetching mqtt_trigger: {:#?}", e); - windmill_common::error::Error::InternalErr("server error".to_string()) - })?; - tx.commit().await.map_err(|e| { - tracing::debug!("Error committing mqtt_trigger: {:#?}", e); - windmill_common::error::Error::InternalErr("server error".to_string()) - })?; - - Ok(Json(rows)) -} - -pub async fn get_mqtt_trigger( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - check_scopes(&authed, || format!("mqtt_triggers:read:{}", path))?; - - let mut tx = user_db.begin(&authed).await?; - let trigger = sqlx::query_as!( - MqttTrigger, - r#" - SELECT - mqtt_resource_path, - subscribe_topics as "subscribe_topics!: Vec>", - v3_config as "v3_config!: Option>", - v5_config as "v5_config!: Option>", - client_version AS "client_version: _", - client_id, - workspace_id, - path, - script_path, - is_flow, - edited_by, - email, - edited_at, - server_id, - last_server_ping, - extra_perms, - error, - enabled, - error_handler_path, - error_handler_args as "error_handler_args: _", - retry as "retry: _" - FROM - mqtt_trigger - WHERE - workspace_id = $1 AND - path = $2 - "#, - w_id, - &path - ) - .fetch_optional(&mut *tx) - .await?; - tx.commit().await?; - - let trigger = not_found_if_none(trigger, "Mqtt Trigger", path)?; - - Ok(Json(trigger)) -} - -pub async fn update_mqtt_trigger( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Json(mqtt_trigger): Json, -) -> error::Result { - let workspace_path = path.to_path(); - check_scopes(&authed, || { - format!("mqtt_triggers:write:{}", workspace_path) - })?; - - let EditMqttTrigger { - mqtt_resource_path, - subscribe_topics, - script_path, - path, - is_flow, - v3_config, - v5_config, - client_version, - client_id, - error_handler_path, - error_handler_args, - retry, - } = mqtt_trigger; - - let mut tx = user_db.begin(&authed).await?; - - let subscribe_topics = subscribe_topics.into_iter().map(SqlxJson).collect_vec(); - - let v3_config = v3_config.map(SqlxJson); - let v5_config = v5_config.map(SqlxJson); - - sqlx::query!( - r#" - UPDATE - mqtt_trigger - SET - mqtt_resource_path = $1, - subscribe_topics = $2, - client_version = $3, - client_id = $4, - v3_config = $5, - v5_config = $6, - is_flow = $7, - edited_by = $8, - email = $9, - script_path = $10, - path = $11, - edited_at = now(), - error = NULL, - server_id = NULL, - error_handler_path = $14, - error_handler_args = $15, - retry = $16 - WHERE - workspace_id = $12 AND - path = $13 - "#, - mqtt_resource_path, - subscribe_topics.as_slice() as &[SqlxJson], - client_version as Option, - client_id, - v3_config as Option>, - v5_config as Option>, - is_flow, - &authed.username, - &authed.email, - script_path, - path, - w_id, - workspace_path, - error_handler_path, - error_handler_args as _, - retry as _ - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "mqtt_triggers.update", - ActionKind::Update, - &w_id, - Some(&path), - None, - ) - .await?; - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::MqttTrigger { path: path.clone() }, - Some(format!("MQTT trigger '{}' updated", path)), - true, - ) - .await?; - - Ok(path.to_string()) -} - -pub async fn delete_mqtt_trigger( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> error::Result { - let path = path.to_path(); - check_scopes(&authed, || format!("mqtt_triggers:write:{}", path))?; - - let mut tx = user_db.begin(&authed).await?; - sqlx::query!( - r#" - DELETE - FROM - mqtt_trigger - WHERE - workspace_id = $1 AND - path = $2 - "#, - w_id, - path, - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "mqtt_triggers.delete", - ActionKind::Delete, - &w_id, - Some(path), - None, - ) - .await?; - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::MqttTrigger { path: path.to_string() }, - Some(format!("MQTT trigger '{}' deleted", path)), - true, - ) - .await?; - - Ok(format!("MQTT trigger {path} deleted")) -} - -pub async fn exists_mqtt_trigger( - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - let exists = sqlx::query_scalar!( - r#" - SELECT EXISTS( - SELECT - 1 - FROM - mqtt_trigger - WHERE - path = $1 AND - workspace_id = $2 - )"#, - path, - w_id, - ) - .fetch_one(&db) - .await? - .unwrap_or(false); - Ok(Json(exists)) -} - -pub async fn set_enabled( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Json(payload): Json, -) -> error::Result { - let path = path.to_path(); - check_scopes(&authed, || format!("mqtt_triggers:write:{}", path))?; - let mut tx = user_db.begin(&authed).await?; - - // important to set server_id, last_server_ping and error to NULL to stop current mqtt listener - let one_o = sqlx::query_scalar!( - r#" - UPDATE - mqtt_trigger - SET - enabled = $1, - email = $2, - edited_by = $3, - edited_at = now(), - server_id = NULL, - error = NULL - WHERE - path = $4 AND - workspace_id = $5 - RETURNING 1 - "#, - payload.enabled, - &authed.email, - &authed.username, - path, - w_id, - ) - .fetch_optional(&mut *tx) - .await? - .flatten(); - - not_found_if_none(one_o, "Mqtt trigger", path)?; - - audit_log( - &mut *tx, - &authed, - "mqtt_triggers.setenabled", - ActionKind::Update, - &w_id, - Some(path), - Some([("enabled", payload.enabled.to_string().as_ref())].into()), - ) - .await?; - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::MqttTrigger { path: path.to_string() }, - Some(format!("MQTT trigger '{}' updated", path)), - true, - ) - .await?; - - Ok(format!( - "successfully updated mqtt trigger at path {} to status {}", - path, payload.enabled - )) + .into() } async fn loop_ping(db: &DB, mqtt: &MqttConfig, error: Option<&str>) { @@ -1026,7 +396,7 @@ trait MqttEvent { type Event; fn handle_publish_packet(publish_packet: Self::PublishPacket) -> PublishData; - fn handle_event(&self, event: Self::Event) -> Result, String>; + fn handle_event(&self, event: Self::Event) -> Result>; } struct V5MqttHandler; @@ -1046,7 +416,7 @@ impl MqttEvent for V5MqttHandler { ) } - fn handle_event(&self, event: Self::Event) -> Result, String> { + fn handle_event(&self, event: Self::Event) -> Result> { tracing::debug!("Inside V5 event"); match event { Self::Event::Incoming(packet) => match packet { @@ -1057,7 +427,7 @@ impl MqttEvent for V5MqttHandler { ))) } Self::IncomingPacket::Disconnect(disconnect) => { - return Err(convert_disconnect_packet_into_string(disconnect)); + return Err(convert_disconnect_packet_into_err(disconnect)); } packet => { tracing::debug!("Received = {:#?}", packet); @@ -1089,7 +459,7 @@ impl MqttEvent for V3MqttHandler { ) } - fn handle_event(&self, event: Self::Event) -> Result, String> { + fn handle_event(&self, event: Self::Event) -> Result> { tracing::debug!("Inside V3 event"); match event { Self::Event::Incoming(packet) => match packet { @@ -1116,67 +486,65 @@ const TIMEOUT_DURATION: u64 = 10; const CONNECTION_TIMEOUT: Duration = Duration::from_secs(TIMEOUT_DURATION); #[async_trait] -trait EventLoop { +pub trait EventLoop { type Event; type Error; - async fn poll(&mut self) -> Result; - async fn verify_connection(&mut self) -> Result<(), Error>; + async fn poll(&mut self) -> Result; + async fn verify_connection(&mut self) -> Result<()>; } #[async_trait] impl EventLoop for V5EventLoop { type Event = V5Event; type Error = rumqttc::v5::ConnectionError; - async fn poll(&mut self) -> Result { - self.poll().await + async fn poll(&mut self) -> Result { + self.poll().await.map_err(|err| to_anyhow(err).into()) } - async fn verify_connection(&mut self) -> Result<(), Error> { + async fn verify_connection(&mut self) -> Result<()> { let start = std::time::Instant::now(); while start.elapsed() < CONNECTION_TIMEOUT { - match self.poll().await? { + match self.poll().await.map_err(to_anyhow)? { Self::Event::Incoming(V5Incoming::ConnAck(_)) => return Ok(()), Self::Event::Incoming(V5Incoming::Disconnect(disconnect)) => { - return Err(Error::Common(error::Error::BadConfig( - convert_disconnect_packet_into_string(disconnect), - ))); + return Err(convert_disconnect_packet_into_err(disconnect)); } _ => continue, } } - Err(Error::Common(error::Error::BadConfig(format!( + Err(Error::BadConfig(format!( "Timeout occurred while trying to connect to mqtt broker after {} seconds", TIMEOUT_DURATION - )))) + ))) } } #[async_trait] impl EventLoop for V3EventLoop { type Event = V3Event; - type Error = rumqttc::ConnectionError; + type Error = Error; - async fn poll(&mut self) -> Result { - self.poll().await + async fn poll(&mut self) -> Result { + self.poll().await.map_err(|err| to_anyhow(err).into()) } - async fn verify_connection(&mut self) -> Result<(), Error> { + async fn verify_connection(&mut self) -> Result<()> { let start = std::time::Instant::now(); while start.elapsed() < CONNECTION_TIMEOUT { - match self.poll().await? { + match self.poll().await.map_err(to_anyhow)? { Self::Event::Incoming(rumqttc::Packet::ConnAck(_)) => return Ok(()), _ => continue, } } - Err(Error::Common(error::Error::BadConfig(format!( + Err(Error::BadConfig(format!( "Timeout occurred while trying to connect to mqtt broker after {} seconds", TIMEOUT_DURATION - )))) + ))) } } @@ -1479,7 +847,7 @@ impl MqttTrigger { } } - async fn fetch_authed(&self, db: &DB) -> error::Result { + async fn fetch_authed(&self, db: &DB) -> Result { fetch_api_authed( self.edited_by.clone(), self.email.clone(), @@ -1722,7 +1090,7 @@ impl CaptureConfigForMqttTrigger { Some(()) } - async fn fetch_authed(&self, db: &DB) -> error::Result { + async fn fetch_authed(&self, db: &DB) -> Result { fetch_api_authed( self.owner.clone(), self.email.clone(), diff --git a/backend/windmill-api/src/nats_triggers_oss.rs b/backend/windmill-api/src/nats_triggers_oss.rs index 3e85ff45cf..53e8fd88ec 100644 --- a/backend/windmill-api/src/nats_triggers_oss.rs +++ b/backend/windmill-api/src/nats_triggers_oss.rs @@ -5,19 +5,12 @@ pub use crate::nats_triggers_ee::*; #[cfg(not(feature = "private"))] use crate::db::DB; #[cfg(not(feature = "private"))] -use axum::Router; -#[cfg(not(feature = "private"))] use serde::{Deserialize, Serialize}; #[cfg(not(feature = "private"))] #[derive(Serialize, Deserialize)] pub struct NatsResourceAuth {} -#[cfg(not(feature = "private"))] -pub fn workspaced_service() -> Router { - Router::new() -} - #[cfg(not(feature = "private"))] pub fn start_nats_consumers(_db: DB, mut _killpill_rx: tokio::sync::broadcast::Receiver<()>) -> () { // implementation is not open source diff --git a/backend/windmill-api/src/openapi.rs b/backend/windmill-api/src/openapi.rs index 95f73f95ba..6cc950523a 100644 --- a/backend/windmill-api/src/openapi.rs +++ b/backend/windmill-api/src/openapi.rs @@ -25,8 +25,11 @@ use crate::db::ApiAuthed; #[cfg(feature = "http_trigger")] use { crate::{ - http_trigger_args::HttpMethod, http_trigger_auth::ApiKeyAuthentication, - http_triggers::AuthenticationMethod, resources::try_get_resource_from_db_as, + resources::try_get_resource_from_db_as, + triggers::http::{ + http_trigger_args::HttpMethod, http_trigger_auth::ApiKeyAuthentication, + AuthenticationMethod, + }, }, itertools::Itertools, }; diff --git a/backend/windmill-api/src/postgres_triggers/mod.rs b/backend/windmill-api/src/postgres_triggers/mod.rs index 1e224df3dc..17d6d44678 100644 --- a/backend/windmill-api/src/postgres_triggers/mod.rs +++ b/backend/windmill-api/src/postgres_triggers/mod.rs @@ -1,47 +1,28 @@ use crate::{ db::{ApiAuthed, DB}, resources::try_get_resource_from_db_as, - trigger_helpers::{trigger_runnable, TriggerJobArgs}, - users::fetch_api_authed, }; -use chrono::Utc; -use itertools::Itertools; use native_tls::{Certificate, TlsConnector}; use pg_escape::quote_identifier; -use rand::Rng; use rust_postgres::{config::SslMode, Client, Config, NoTls}; use rust_postgres_native_tls::MakeTlsConnector; +use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; +use sqlx::FromRow; use std::collections::HashMap; -use axum::{ - routing::{delete, get, post}, - Router, -}; -pub use handler::PostgresTrigger; -use handler::{ - alter_publication, create_postgres_trigger, create_publication, create_slot, - create_template_script, delete_postgres_trigger, delete_publication, drop_slot_name, - exists_postgres_trigger, get_postgres_trigger, get_postgres_version, - get_postgres_version_internal, get_publication_info, get_template_script, - is_database_in_logical_level, list_database_publication, list_postgres_triggers, - list_slot_name, set_enabled, test_postgres_connection, update_postgres_trigger, Postgres, - Relations, -}; use windmill_common::{ db::UserDB, error::{to_anyhow, Error, Result}, + utils::empty_as_none, }; mod bool; mod converter; -mod handler; mod hex; -mod mapper; mod relation; mod replication_message; mod trigger; -pub use handler::PublicationData; pub use trigger::start_database; const ERROR_REPLICATION_SLOT_NOT_EXISTS: &str = r#"The replication slot associated with this trigger no longer exists. Recreate a new replication slot or select an existing one in the advanced tab, or delete and recreate a new trigger"#; @@ -88,6 +69,19 @@ fn build_tls_connector( ))) } +#[derive(FromRow, Serialize, Deserialize, Debug)] +pub struct Postgres { + pub user: String, + pub password: String, + pub host: String, + pub port: Option, + pub dbname: String, + #[serde(default)] + pub sslmode: String, + #[serde(default, deserialize_with = "empty_as_none")] + pub root_certificate_pem: Option, +} + pub async fn get_raw_postgres_connection( database: &Postgres, logical_mode: bool, @@ -174,124 +168,65 @@ pub async fn get_default_pg_connection( get_pg_connection(authed, user_db, db, postgres_resource_path, w_id, false).await } -pub async fn create_logical_replication_slot(tx: &Client, slot_name: &str) -> Result<()> { - tx.execute( - &format!("SELECT * FROM pg_create_logical_replication_slot($1, 'pgoutput')"), - &[&slot_name], - ) - .await - .map_err(to_anyhow)?; - Ok(()) +#[derive(FromRow, Deserialize, Serialize, Debug)] +pub struct PostgresTrigger { + pub path: String, + pub script_path: String, + pub is_flow: bool, + pub workspace_id: String, + pub edited_by: String, + pub email: String, + pub edited_at: chrono::DateTime, + #[serde(skip_serializing_if = "Option::is_none")] + pub extra_perms: Option, + pub postgres_resource_path: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub server_id: Option, + pub replication_slot_name: String, + pub publication_name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub last_server_ping: Option>, + pub enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option>>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub retry: Option>, } -async fn check_if_valid_publication_for_postgres_version( - pg_connection: &Client, - table_to_track: Option<&[Relations]>, -) -> Result { - let postgres_version = get_postgres_version_internal(pg_connection).await?; +pub async fn drop_logical_replication_slot(pg_connection: &Client, slot_name: &str) -> Result<()> { + let row = pg_connection + .query_opt( + r#" + SELECT + active_pid + FROM + pg_replication_slots + WHERE + slot_name = $1 + "#, + &[&slot_name], + ) + .await + .map_err(to_anyhow)?; - let pg_14 = postgres_version.starts_with("14"); - if pg_14 { - let unsupported_publication = table_to_track - .and_then(|relations| { - relations.iter().find(|relation| { - let invalid_relation = relation.table_to_track.iter().find(|table_to_track| { - table_to_track.where_clause.is_some() - || table_to_track.columns_name.is_some() - }); + let active_pid = row.map(|r| r.get::<_, Option>(0)).flatten(); - relation.table_to_track.is_empty() || invalid_relation.is_some() - }) - }) - .is_some(); - - if unsupported_publication { - return Err(Error::BadRequest( - "Your PostgreSQL database is running version 14, which does not support the following publication features: \ - - WHERE clause filtering, \ - - selective column tracking, and \ - - tracking all tables within a schema.\n\ - These features are only available in PostgreSQL 15 and above.".to_string(), - )); - } - } - Ok(pg_14) -} - -pub async fn create_pg_publication( - pg_connection: &Client, - publication_name: &str, - table_to_track: Option<&[Relations]>, - transaction_to_track: &[String], -) -> Result<()> { - let pg_14 = - check_if_valid_publication_for_postgres_version(pg_connection, table_to_track).await?; - let mut query = String::from("CREATE PUBLICATION "); - - query.push_str("e_identifier(publication_name)); - - match table_to_track { - Some(database_component) if !database_component.is_empty() => { - query.push_str(" FOR"); - let mut first = true; - for (i, schema) in database_component.iter().enumerate() { - if schema.table_to_track.is_empty() { - query.push_str(" TABLES IN SCHEMA "); - query.push_str("e_identifier(&schema.schema_name)); - } else { - if pg_14 && first { - query.push_str(" TABLE ONLY "); - first = false - } else if !pg_14 { - query.push_str(" TABLE ONLY "); - } - for (j, table) in schema.table_to_track.iter().enumerate() { - let table_name = quote_identifier(&table.table_name); - let schema_name = quote_identifier(&schema.schema_name); - let full_name = format!("{}.{}", &schema_name, &table_name); - query.push_str(&full_name); - if let Some(columns) = table.columns_name.as_ref() { - query.push_str(" ("); - let columns = columns - .iter() - .map(|column| quote_identifier(column)) - .join(", "); - query.push_str(&columns); - query.push_str(")"); - } - - if let Some(where_clause) = &table.where_clause { - query.push_str(" WHERE ("); - query.push_str(where_clause); - query.push(')'); - } - - if j + 1 != schema.table_to_track.len() { - query.push_str(", "); - } - } - } - if i < database_component.len() - 1 { - query.push_str(", "); - } - } - } - _ => { - query.push_str(" FOR ALL TABLES "); - } - }; - - if !transaction_to_track.is_empty() { - let transactions = || transaction_to_track.iter().join(", "); - query.push_str(" WITH (publish = '"); - query.push_str(&transactions()); - query.push_str("');"); + if let Some(pid) = active_pid { + pg_connection + .execute("SELECT pg_terminate_backend($1)", &[&pid]) + .await + .map_err(to_anyhow)?; } pg_connection - .execute(&query, &[]) + .execute("SELECT pg_drop_replication_slot($1)", &[&slot_name]) .await .map_err(to_anyhow)?; + Ok(()) } @@ -307,106 +242,3 @@ pub async fn drop_publication(pg_connection: &Client, publication_name: &str) -> Ok(()) } - -pub fn generate_random_string() -> String { - let timestamp = Utc::now().timestamp_millis().to_string(); - let mut rng = rand::rng(); - let charset = "abcdefghijklmnopqrstuvwxyz0123456789"; - - let random_part = (0..10) - .map(|_| { - charset - .chars() - .nth(rng.random_range(0..charset.len())) - .unwrap() - }) - .collect::(); - - format!("{}_{}", timestamp, random_part) -} - -fn publication_service() -> Router { - Router::new() - .route("/get/:publication_name/*path", get(get_publication_info)) - .route("/create/:publication_name/*path", post(create_publication)) - .route("/update/:publication_name/*path", post(alter_publication)) - .route( - "/delete/:publication_name/*path", - delete(delete_publication), - ) - .route("/list/*path", get(list_database_publication)) -} - -fn slot_service() -> Router { - Router::new() - .route("/list/*path", get(list_slot_name)) - .route("/create/*path", post(create_slot)) - .route("/delete/*path", delete(drop_slot_name)) -} - -fn postgres_service() -> Router { - Router::new().route("/version/*path", get(get_postgres_version)) -} - -pub fn workspaced_service() -> Router { - Router::new() - .route("/test", post(test_postgres_connection)) - .route("/create", post(create_postgres_trigger)) - .route("/list", get(list_postgres_triggers)) - .route("/get/*path", get(get_postgres_trigger)) - .route("/update/*path", post(update_postgres_trigger)) - .route("/delete/*path", delete(delete_postgres_trigger)) - .route("/exists/*path", get(exists_postgres_trigger)) - .route("/setenabled/*path", post(set_enabled)) - .route("/get_template_script/:id", get(get_template_script)) - .route("/create_template_script", post(create_template_script)) - .route( - "/is_valid_postgres_configuration/*path", - get(is_database_in_logical_level), - ) - .nest("/publication", publication_service()) - .nest("/slot", slot_service()) - .nest("/postgres", postgres_service()) -} - -async fn run_job( - payload: HashMap>, - db: &DB, - trigger: &PostgresTrigger, -) -> anyhow::Result<()> { - let args = PostgresTrigger::build_job_args( - &trigger.script_path, - trigger.is_flow, - &trigger.workspace_id, - db, - payload, - HashMap::new(), - ) - .await?; - - let authed = fetch_api_authed( - trigger.edited_by.clone(), - trigger.email.clone(), - &trigger.workspace_id, - db, - Some(format!("postgres-{}", trigger.path)), - ) - .await?; - - trigger_runnable( - db, - None, - authed, - &trigger.workspace_id, - &trigger.script_path, - trigger.is_flow, - args, - trigger.retry.as_ref(), - trigger.error_handler_path.as_deref(), - trigger.error_handler_args.as_ref(), - format!("postgres_trigger/{}", trigger.path), - ) - .await?; - - Ok(()) -} diff --git a/backend/windmill-api/src/postgres_triggers/trigger.rs b/backend/windmill-api/src/postgres_triggers/trigger.rs index 832fa997f3..6bd8877fec 100644 --- a/backend/windmill-api/src/postgres_triggers/trigger.rs +++ b/backend/windmill-api/src/postgres_triggers/trigger.rs @@ -9,10 +9,9 @@ use crate::{ LogicalReplicationMessage::{Begin, Commit, Delete, Insert, Relation, Type, Update}, ReplicationMessage, }, - run_job, }, resources::try_get_resource_from_db_as, - trigger_helpers::TriggerJobArgs, + trigger_helpers::{trigger_runnable, TriggerJobArgs}, users::fetch_api_authed, }; @@ -36,10 +35,9 @@ use windmill_common::{ }; use super::{ - drop_publication, get_default_pg_connection, get_raw_postgres_connection, - handler::{drop_logical_replication_slot, Postgres, PostgresTrigger}, - replication_message::PrimaryKeepAliveBody, - Error, ERROR_PUBLICATION_NAME_NOT_EXISTS, ERROR_REPLICATION_SLOT_NOT_EXISTS, + drop_logical_replication_slot, drop_publication, get_default_pg_connection, + get_raw_postgres_connection, replication_message::PrimaryKeepAliveBody, Error, Postgres, + PostgresTrigger, ERROR_PUBLICATION_NAME_NOT_EXISTS, ERROR_REPLICATION_SLOT_NOT_EXISTS, }; pub struct LogicalReplicationSettings { @@ -350,6 +348,48 @@ impl TriggerJobArgs>> for PostgresTrigger { } } +async fn run_job( + payload: HashMap>, + db: &DB, + trigger: &PostgresTrigger, +) -> anyhow::Result<()> { + let args = PostgresTrigger::build_job_args( + &trigger.script_path, + trigger.is_flow, + &trigger.workspace_id, + db, + payload, + HashMap::new(), + ) + .await?; + + let authed = fetch_api_authed( + trigger.edited_by.clone(), + trigger.email.clone(), + &trigger.workspace_id, + db, + Some(format!("postgres-{}", trigger.path)), + ) + .await?; + + trigger_runnable( + db, + None, + authed, + &trigger.workspace_id, + &trigger.script_path, + trigger.is_flow, + args, + trigger.retry.as_ref(), + trigger.error_handler_path.as_deref(), + trigger.error_handler_args.as_ref(), + format!("postgres_trigger/{}", trigger.path), + ) + .await?; + + Ok(()) +} + struct PgInfo<'a> { postgres_resource_path: &'a str, publication_name: &'a str, diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index b474e710db..6381843842 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -9,10 +9,7 @@ use std::collections::HashMap; use crate::{ - db::{ApiAuthed, DB}, - users::{maybe_refresh_folders, require_owner_of_path, Tokened}, - utils::check_scopes, - webhook_util::{WebhookMessage, WebhookShared}, + db::{ApiAuthed, DB}, users::{maybe_refresh_folders, require_owner_of_path, Tokened}, utils::check_scopes, var_resource_cache::{cache_resource, get_cached_resource}, webhook_util::{WebhookMessage, WebhookShared} }; use axum::{ body::Body, @@ -326,6 +323,7 @@ async fn exists_resource( Ok(Json(exists)) } + async fn get_resource_value( authed: ApiAuthed, Extension(user_db): Extension, @@ -334,6 +332,8 @@ async fn get_resource_value( ) -> JsonResult> { let path = path.to_path(); check_scopes(&authed, || format!("resources:read:{}", path))?; + + let mut tx = user_db.begin(&authed).await?; let value_o = sqlx::query_scalar!( @@ -350,6 +350,9 @@ async fn get_resource_value( } let value = not_found_if_none(value_o, "Resource", path)?; + + + Ok(Json(value)) } @@ -416,7 +419,9 @@ async fn custom_component( #[derive(Deserialize)] struct JobInfo { job_id: Option, + allow_cache: Option, } + async fn get_resource_value_interpolated( authed: ApiAuthed, Extension(user_db): Extension, @@ -428,6 +433,7 @@ async fn get_resource_value_interpolated( let path = path.to_path(); check_scopes(&authed, || format!("resources:read:{}", path))?; + return get_resource_value_interpolated_internal( &authed, Some(user_db), @@ -436,6 +442,7 @@ async fn get_resource_value_interpolated( path, job_info.job_id, token.as_str(), + job_info.allow_cache.unwrap_or(false), ) .await .map(|success| Json(success)); @@ -452,7 +459,13 @@ pub async fn get_resource_value_interpolated_internal( path: &str, job_id: Option, token: &str, + allow_cache: bool, ) -> Result> { + if allow_cache { + if let Some(cached_value) = get_cached_resource(&workspace, &path) { + return Ok(Some(cached_value)); + } + } let mut tx = authed_transaction_or_default(authed, user_db.clone(), db).await?; let value_o = sqlx::query_scalar!( @@ -469,18 +482,20 @@ pub async fn get_resource_value_interpolated_internal( let value = not_found_if_none(value_o, "Resource", path)?; if let Some(value) = value { - Ok(Some( - transform_json_value( - authed, - user_db.clone(), - db, - workspace, - value, - &job_id, - token, - ) - .await?, - )) + let r = transform_json_value( + authed, + user_db.clone(), + db, + workspace, + value, + &job_id, + token, + ) + .await?; + if allow_cache { + cache_resource(&workspace, &path, r.clone()); + } + Ok(Some(r)) } else { Ok(None) } @@ -516,6 +531,7 @@ pub async fn transform_json_value<'c>( username_override: None, token_prefix: None, }), + false ) .await?; Ok(Value::String(v)) @@ -549,12 +565,18 @@ pub async fn transform_json_value<'c>( let job_id = job_id.unwrap(); let job = sqlx::query!( "SELECT - email AS \"email!\", - created_by AS \"created_by!\", - parent_job, permissioned_as AS \"permissioned_as!\", - script_path, schedule_path, flow_step_id, root_job, - scheduled_for AS \"scheduled_for!: chrono::DateTime\" - FROM v2_as_queue WHERE id = $1 AND workspace_id = $2", + v2_job.permissioned_as_email, + v2_job.created_by, + v2_job.parent_job, + v2_job.permissioned_as, + v2_job.runnable_path, + CASE WHEN v2_job.trigger_kind = 'schedule'::job_trigger_kind THEN v2_job.trigger END AS schedule_path, + v2_job.flow_step_id, + v2_job.flow_innermost_root_job, + v2_job.root_job, + v2_job_queue.scheduled_for AS \"scheduled_for: chrono::DateTime\" + FROM v2_job INNER JOIN v2_job_queue ON v2_job.id = v2_job_queue.id + WHERE v2_job.id = $1 AND v2_job.workspace_id = $2", job_id, workspace ) @@ -581,15 +603,16 @@ pub async fn transform_json_value<'c>( &db.into(), workspace, token, - &job.email, + &job.permissioned_as_email, &job.created_by, &job_id.to_string(), &job.permissioned_as, - job.script_path.clone(), + job.runnable_path.clone(), job.parent_job.map(|x| x.to_string()), flow_path, job.schedule_path.clone(), job.flow_step_id.clone(), + job.flow_innermost_root_job.map(|x| x.to_string()), job.root_job.map(|x| x.to_string()), Some(job.scheduled_for.clone()), None, @@ -1239,7 +1262,12 @@ async fn update_resource_type( feature = "mqtt_trigger", all( feature = "enterprise", - any(feature = "sqs_trigger", feature = "gcp_trigger") + any( + feature = "sqs_trigger", + feature = "gcp_trigger", + feature = "kafka", + feature = "nats" + ) ) ))] pub async fn try_get_resource_from_db_as( diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index f0b2c4c483..bf1447b51b 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -10,9 +10,8 @@ use crate::{ auth::AuthCache, db::{ApiAuthed, DB}, schedule::clear_schedule, - triggers::{ - get_triggers_count_internal, list_tokens_internal, TriggersCount, TruncatedTokenWithEmail, - }, + triggers::{get_triggers_count_internal, TriggersCount}, + auth::{list_tokens_internal, TruncatedTokenWithEmail}, users::{maybe_refresh_folders, require_owner_of_path}, utils::{check_scopes, WithStarredInfoQuery}, webhook_util::{WebhookMessage, WebhookShared}, @@ -1007,6 +1006,7 @@ async fn create_script_internal<'c>( None, None, None, + None, false, false, None, diff --git a/backend/windmill-api/src/sqs_triggers_oss.rs b/backend/windmill-api/src/sqs_triggers_oss.rs index 7d993bf1cc..49fffb43e9 100644 --- a/backend/windmill-api/src/sqs_triggers_oss.rs +++ b/backend/windmill-api/src/sqs_triggers_oss.rs @@ -5,17 +5,10 @@ pub use crate::sqs_triggers_ee::*; #[cfg(not(feature = "private"))] use crate::db::DB; #[cfg(not(feature = "private"))] -use axum::Router; -#[cfg(not(feature = "private"))] use serde::{Deserialize, Serialize}; #[cfg(not(feature = "private"))] use windmill_common::auth::aws::AwsAuthResourceType; -#[cfg(not(feature = "private"))] -pub fn workspaced_service() -> Router { - Router::new() -} - #[cfg(not(feature = "private"))] pub fn start_sqs(_db: DB, mut _killpill_rx: tokio::sync::broadcast::Receiver<()>) -> () { // implementation is not open source @@ -43,7 +36,9 @@ pub struct SqsTrigger { #[serde(skip_serializing_if = "Option::is_none")] pub error_handler_path: Option, #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_args: Option>>>, + pub error_handler_args: Option< + sqlx::types::Json>>, + >, #[serde(skip_serializing_if = "Option::is_none")] pub retry: Option>, } diff --git a/backend/windmill-api/src/trigger_helpers.rs b/backend/windmill-api/src/trigger_helpers.rs index 1c2bdc029c..3670622128 100644 --- a/backend/windmill-api/src/trigger_helpers.rs +++ b/backend/windmill-api/src/trigger_helpers.rs @@ -60,6 +60,7 @@ pub enum RunnableId { } impl RunnableId { + #[allow(dead_code)] pub fn from_script_hash(hash: ScriptHash) -> Self { Self::ScriptId(ScriptId::ScriptHash(hash)) } @@ -78,6 +79,7 @@ impl RunnableId { } #[derive(Debug, Clone, Eq, PartialEq, Hash)] +#[allow(unused)] pub enum ScriptId { ScriptPath(String), ScriptHash(ScriptHash), @@ -774,6 +776,7 @@ async fn trigger_script_with_retry_and_error_handler( None, None, None, + None, false, false, None, diff --git a/backend/windmill-api/src/triggers.rs b/backend/windmill-api/src/triggers.rs deleted file mode 100644 index 82d189df5c..0000000000 --- a/backend/windmill-api/src/triggers.rs +++ /dev/null @@ -1,249 +0,0 @@ -use axum::Json; -use serde::{Deserialize, Serialize}; -use sqlx::FromRow; -use windmill_common::error::JsonResult; - -use crate::db::DB; - -#[derive(Serialize, Deserialize, Debug)] -pub struct TriggerPrimarySchedule { - schedule: String, -} - -#[derive(Serialize, Deserialize, Debug)] -pub struct TriggersCount { - primary_schedule: Option, - schedule_count: i64, - http_routes_count: i64, - webhook_count: i64, - email_count: i64, - websocket_count: i64, - kafka_count: i64, - nats_count: i64, - postgres_count: i64, - mqtt_count: i64, - sqs_count: i64, - gcp_count: i64, -} -pub(crate) async fn get_triggers_count_internal( - db: &DB, - w_id: &str, - path: &str, - is_flow: bool, -) -> JsonResult { - let primary_schedule = sqlx::query_scalar!( - "SELECT schedule FROM schedule WHERE path = $1 AND script_path = $1 AND is_flow = $2 AND workspace_id = $3", - path, - is_flow, - w_id - ) - .fetch_optional(db) - .await?; - - let schedule_count = sqlx::query_scalar!( - "SELECT COUNT(*) FROM schedule WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", - path, - is_flow, - w_id - ) - .fetch_one(db) - .await? - .unwrap_or(0); - - let http_routes_count = sqlx::query_scalar!( - "SELECT COUNT(*) FROM http_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", - path, - is_flow, - w_id - ) - .fetch_one(db) - .await? - .unwrap_or(0); - - let websocket_count = sqlx::query_scalar!( - "SELECT COUNT(*) FROM websocket_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", - path, - is_flow, - w_id - ) - .fetch_one(db) - .await? - .unwrap_or(0); - - let kafka_count = sqlx::query_scalar!( - "SELECT COUNT(*) FROM kafka_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", - path, - is_flow, - w_id - ) - .fetch_one(db) - .await? - .unwrap_or(0); - - let nats_count = sqlx::query_scalar!( - "SELECT COUNT(*) FROM nats_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", - path, - is_flow, - w_id - ) - .fetch_one(db) - .await? - .unwrap_or(0); - - let postgres_count = sqlx::query_scalar!( - "SELECT COUNT(*) FROM postgres_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", - path, - is_flow, - w_id - ) - .fetch_one(db) - .await? - .unwrap_or(0); - - let mqtt_count = sqlx::query_scalar!( - "SELECT COUNT(*) FROM mqtt_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", - path, - is_flow, - w_id - ) - .fetch_one(db) - .await? - .unwrap_or(0); - - let sqs_count = sqlx::query_scalar!( - "SELECT COUNT(*) FROM sqs_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", - path, - is_flow, - w_id - ) - .fetch_one(db) - .await? - .unwrap_or(0); - - let gcp_count = sqlx::query_scalar!( - "SELECT COUNT(*) FROM gcp_trigger WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", - path, - is_flow, - w_id - ) - .fetch_one(db) - .await? - .unwrap_or(0); - - let webhook_count = (if is_flow { - sqlx::query_scalar!( - "SELECT COUNT(*) FROM token WHERE label LIKE 'webhook-%' AND workspace_id = $1 AND scopes @> ARRAY['run:flow/' || $2]::text[]", - w_id, - path, - ) - - } else { - sqlx::query_scalar!( - "SELECT COUNT(*) FROM token WHERE label LIKE 'webhook-%' AND workspace_id = $1 AND scopes @> ARRAY['run:' || $2]::text[]", - w_id, - path, - ) - }).fetch_one(db) - .await? - .unwrap_or(0); - - let email_count = (if is_flow { - sqlx::query_scalar!( - "SELECT COUNT(*) FROM token WHERE label LIKE 'email-%' AND workspace_id = $1 AND scopes @> ARRAY['run:flow/' || $2]::text[]", - w_id, - path, - ) - - } else { - sqlx::query_scalar!( - "SELECT COUNT(*) FROM token WHERE label LIKE 'email-%' AND workspace_id = $1 AND scopes @> ARRAY['run:script/' || $2]::text[]", - w_id, - path, - ) - }).fetch_one(db) - .await? - .unwrap_or(0); - - Ok(Json(TriggersCount { - primary_schedule: primary_schedule.map(|s| TriggerPrimarySchedule { schedule: s }), - schedule_count, - http_routes_count, - webhook_count, - email_count, - websocket_count, - kafka_count, - nats_count, - postgres_count, - mqtt_count, - gcp_count, - sqs_count, - })) -} - -#[derive(FromRow, Serialize)] -pub struct TruncatedTokenWithEmail { - pub label: Option, - pub token_prefix: Option, - pub expiration: Option>, - pub created_at: chrono::DateTime, - pub last_used_at: chrono::DateTime, - pub scopes: Option>, - pub email: Option, -} - -pub async fn list_tokens_internal( - db: &DB, - w_id: &str, - path: &str, - is_flow: bool, -) -> JsonResult> { - let tokens = if is_flow { - sqlx::query_as!( - TruncatedTokenWithEmail, - r#" - SELECT label, - concat(substring(token for 10)) AS token_prefix, - expiration, - created_at, - last_used_at, - scopes, - email - FROM token - WHERE workspace_id = $1 - AND ( - scopes @> ARRAY['jobs:run:flows:' || $2]::text[] - OR scopes @> ARRAY['run:flow/' || $2]::text[] - ) - "#, - w_id, - path - ) - .fetch_all(db) - .await? - } else { - sqlx::query_as!( - TruncatedTokenWithEmail, - r#" - SELECT label, - concat(substring(token for 10)) AS token_prefix, - expiration, - created_at, - last_used_at, - scopes, - email - FROM token - WHERE workspace_id = $1 - AND ( - scopes @> ARRAY['jobs:run:scripts:' || $2]::text[] - OR scopes @> ARRAY['run:script/' || $2]::text[] - ) - "#, - w_id, - path - ) - .fetch_all(db) - .await? - }; - - Ok(Json(tokens)) -} diff --git a/backend/windmill-api/src/triggers/gcp/handler_oss.rs b/backend/windmill-api/src/triggers/gcp/handler_oss.rs new file mode 100644 index 0000000000..825c7f71e0 --- /dev/null +++ b/backend/windmill-api/src/triggers/gcp/handler_oss.rs @@ -0,0 +1,64 @@ +#[cfg(feature = "private")] +pub use super::handler_ee::*; + +#[cfg(not(feature = "private"))] +use { + super::GcpTrigger, + crate::{ + db::{ApiAuthed, DB}, + triggers::{TriggerCrud, TriggerData}, + }, + axum::async_trait, + sqlx::PgConnection, + windmill_common::error::{Error, Result}, + windmill_git_sync::DeployedObject, +}; + +#[cfg(not(feature = "private"))] +#[async_trait] +impl TriggerCrud for GcpTrigger { + type Trigger = (); + type TriggerConfig = (); + type TriggerConfigRequest = (); + type TestConnectionConfig = (); + + const TABLE_NAME: &'static str = ""; + const TRIGGER_TYPE: &'static str = ""; + const SUPPORTS_ENABLED: bool = false; + const SUPPORTS_SERVER_STATE: bool = false; + const SUPPORTS_TEST_CONNECTION: bool = false; + const ROUTE_PREFIX: &'static str = "/gcp_triggers"; + const DEPLOYMENT_NAME: &'static str = ""; + const IS_CLOUD_HOSTED: bool = false; + + fn get_deployed_object(path: String) -> DeployedObject { + DeployedObject::GcpTrigger { path } + } + + async fn create_trigger( + &self, + _db: &DB, + _executor: &mut PgConnection, + _authed: &ApiAuthed, + _w_id: &str, + _trigger: TriggerData, + ) -> Result<()> { + Err(Error::BadRequest( + "GCP triggers are not available in open source version".to_string(), + )) + } + + async fn update_trigger( + &self, + _db: &DB, + _executor: &mut PgConnection, + _authed: &ApiAuthed, + _workspace_id: &str, + _path: &str, + _trigger: TriggerData, + ) -> Result<()> { + Err(Error::BadRequest( + "GCP triggers are not available in open source version".to_string(), + )) + } +} diff --git a/backend/windmill-api/src/triggers/gcp/mod.rs b/backend/windmill-api/src/triggers/gcp/mod.rs new file mode 100644 index 0000000000..f11ba94883 --- /dev/null +++ b/backend/windmill-api/src/triggers/gcp/mod.rs @@ -0,0 +1,11 @@ +#[cfg(feature = "private")] +mod handler_ee; +pub mod handler_oss; + +#[cfg(feature = "private")] +mod mod_ee; +#[cfg(feature = "private")] +pub use mod_ee::*; + +#[derive(Clone, Copy)] +pub struct GcpTrigger; diff --git a/backend/windmill-api/src/triggers/handler.rs b/backend/windmill-api/src/triggers/handler.rs new file mode 100644 index 0000000000..7138b2661e --- /dev/null +++ b/backend/windmill-api/src/triggers/handler.rs @@ -0,0 +1,923 @@ +use crate::{ + db::ApiAuthed, + triggers::{StandardTriggerQuery, TriggerData}, +}; +use async_trait::async_trait; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use sql_builder::{bind::Bind, SqlBuilder}; +use sqlx::{FromRow, PgConnection}; +use std::fmt::Debug; +use windmill_common::{ + db::UserDB, + error::{Error, JsonResult, Result}, + utils::{paginate, Pagination, StripPath}, + worker::CLOUD_HOSTED, + DB, +}; +use windmill_git_sync::DeployedObject; + +use axum::{ + extract::{Path, Query}, + http::StatusCode, + routing::{delete, get, post}, + Extension, Json, Router, +}; +use std::sync::Arc; +use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_git_sync::handle_deployment_metadata; + +use crate::utils::check_scopes; + +#[async_trait] +pub trait TriggerCrud: Send + Sync + 'static { + type Trigger: Serialize + + DeserializeOwned + + for<'r> FromRow<'r, sqlx::postgres::PgRow> + + Send + + Sync + + Unpin; + + type TriggerConfig: Debug + + DeserializeOwned + + for<'r> FromRow<'r, sqlx::postgres::PgRow> + + Serialize + + Send + + Sync + + Unpin; + + type TriggerConfigRequest: Debug + DeserializeOwned + Serialize + Send + Sync; + type TestConnectionConfig: Debug + DeserializeOwned + Serialize + Send + Sync; + + const TABLE_NAME: &'static str; + const TRIGGER_TYPE: &'static str; + const SUPPORTS_ENABLED: bool; + const SUPPORTS_SERVER_STATE: bool; + const SUPPORTS_TEST_CONNECTION: bool; + const ROUTE_PREFIX: &'static str; + const DEPLOYMENT_NAME: &'static str; + const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[]; + const IS_ALLOWED_ON_CLOUD: bool; + + fn get_deployed_object(path: String) -> DeployedObject; + + async fn validate_new( + &self, + db: &DB, + workspace_id: &str, + new: &Self::TriggerConfigRequest, + ) -> Result<()> { + self.validate_config(db, new, workspace_id).await + } + + async fn validate_edit( + &self, + db: &DB, + workspace_id: &str, + edit: &Self::TriggerConfigRequest, + _path: &str, + ) -> Result<()> { + self.validate_config(db, edit, workspace_id).await + } + + async fn validate_config( + &self, + _db: &DB, + _config: &Self::TriggerConfigRequest, + _workspace_id: &str, + ) -> Result<()> { + Ok(()) + } + + fn scope_domain_name() -> &'static str { + &Self::ROUTE_PREFIX[1..] + } + + async fn create_trigger( + &self, + db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + w_id: &str, + trigger: TriggerData, + ) -> Result<()>; + + async fn update_trigger( + &self, + db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + workspace_id: &str, + path: &str, + trigger: TriggerData, + ) -> Result<()>; + + async fn test_connection( + &self, + _db: &DB, + _authed: &ApiAuthed, + _user_db: &UserDB, + _workspace_id: &str, + _config: Self::TestConnectionConfig, + ) -> Result<()> { + Err( + anyhow::anyhow!("Test connection not supported for this trigger type".to_string(),) + .into(), + ) + } + + fn additional_routes(&self) -> axum::Router { + axum::Router::new() + } + + async fn get_trigger_by_path( + &self, + tx: &mut PgConnection, + workspace_id: &str, + path: &str, + ) -> Result { + let mut fields = vec![ + "workspace_id", + "path", + "script_path", + "is_flow", + "edited_by", + "email", + "edited_at", + "extra_perms", + ]; + + if Self::SUPPORTS_SERVER_STATE { + fields.extend_from_slice(&["enabled", "server_id", "last_server_ping", "error"]); + } + + fields.extend_from_slice(&["error_handler_path", "error_handler_args", "retry"]); + fields.extend_from_slice(Self::ADDITIONAL_SELECT_FIELDS); + + let sql = format!( + r#"SELECT + {} + FROM + {} + WHERE + workspace_id = $1 AND + path = $2 + "#, + fields.join(", "), + Self::TABLE_NAME + ); + + sqlx::query_as(&sql) + .bind(workspace_id) + .bind(path) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| Error::NotFound(format!("Trigger not found at path: {}", path))) + } + + async fn exists(&self, db: &DB, workspace_id: &str, path: &str) -> Result { + let exists = sqlx::query_scalar(&format!( + "SELECT EXISTS(SELECT 1 FROM {} WHERE workspace_id = $1 AND path = $2)", + Self::TABLE_NAME + )) + .bind(workspace_id) + .bind(path) + .fetch_one(db) + .await?; + + Ok(exists) + } + + async fn delete_by_path( + &self, + tx: &mut PgConnection, + workspace_id: &str, + path: &str, + ) -> Result { + let deleted = sqlx::query(&format!( + "DELETE FROM {} WHERE workspace_id = $1 AND path = $2", + Self::TABLE_NAME + )) + .bind(workspace_id) + .bind(path) + .execute(&mut *tx) + .await? + .rows_affected(); + + Ok(deleted > 0) + } + + async fn set_enabled( + &self, + authed: &ApiAuthed, + tx: &mut PgConnection, + workspace_id: &str, + path: &str, + enabled: bool, + ) -> Result { + if !Self::SUPPORTS_SERVER_STATE { + return Err(anyhow::anyhow!( + "Enable/disable not supported for this trigger type".to_string(), + ) + .into()); + } + + let updated = sqlx::query(&format!( + r#" + UPDATE + {} + SET + enabled = $1, + email = $2, + edited_by = $3, + edited_at = now(), + server_id = NULL, + error = NULL, + last_server_ping = NULL + WHERE + workspace_id = $4 AND + path = $5 + "#, + Self::TABLE_NAME + )) + .bind(enabled) + .bind(&authed.email) + .bind(&authed.username) + .bind(workspace_id) + .bind(path) + .execute(&mut *tx) + .await? + .rows_affected(); + + Ok(updated > 0) + } + + #[allow(unused)] + async fn trigger_count( + &self, + tx: &mut PgConnection, + workspace_id: &str, + is_flow: bool, + script_path: &str, + ) -> i64 { + let count = sqlx::query_scalar(&format!( + r#" + SELECT + COUNT(*) + FROM + {} + WHERE + workspace_id = $1 AND + is_flow = $2 AND + script_path = $3 + "#, + Self::TABLE_NAME + )) + .bind(workspace_id) + .bind(is_flow) + .bind(script_path) + .fetch_one(&mut *tx) + .await + .unwrap_or(0); + + count + } + + async fn list_triggers( + &self, + tx: &mut PgConnection, + workspace_id: &str, + query: Option<&StandardTriggerQuery>, + ) -> Result> { + let mut fields = vec![ + "workspace_id", + "path", + "script_path", + "is_flow", + "edited_by", + "email", + "edited_at", + "extra_perms", + ]; + + if Self::SUPPORTS_SERVER_STATE { + fields.extend_from_slice(&["enabled", "server_id", "last_server_ping", "error"]); + } + + fields.extend_from_slice(&["error_handler_path", "error_handler_args", "retry"]); + fields.extend_from_slice(Self::ADDITIONAL_SELECT_FIELDS); + + let mut sqlb = SqlBuilder::select_from(Self::TABLE_NAME); + + sqlb.fields(&fields) + .order_by("edited_at", true) + .and_where("workspace_id = ?".bind(&workspace_id)); + + if let Some(query) = query { + let (per_page, offset) = + paginate(Pagination { per_page: query.per_page, page: query.page }); + if let Some(path) = &query.path { + sqlb.and_where_eq("script_path", "?".bind(path)); + } + + if let Some(is_flow) = query.is_flow { + sqlb.and_where_eq("is_flow", "?".bind(&is_flow)); + } + + if let Some(path_start) = &query.path_start { + sqlb.and_where_like_left("path", path_start); + } + + sqlb.offset(offset).limit(per_page); + } + + let sql = sqlb + .sql() + .map_err(|e| Error::InternalErr(format!("SQL error: {}", e)))?; + + let triggers = sqlx::query_as(&sql).fetch_all(&mut *tx).await?; + + Ok(triggers) + } +} + +pub fn trigger_routes() -> Router { + let mut router = Router::new() + .route("/create", post(create_trigger::)) + .route("/list", get(list_triggers::)) + .route("/get/*path", get(get_trigger::)) + .route("/update/*path", post(update_trigger::)) + .route("/delete/*path", delete(delete_trigger::)) + .route("/exists/*path", get(exists_trigger::)); + + if T::SUPPORTS_ENABLED { + router = router.route("/setenabled/*path", post(set_enabled_trigger::)); + } + + if T::SUPPORTS_TEST_CONNECTION { + router = router.route("/test", post(test_connection::)); + } + + router +} + +async fn create_trigger( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(workspace_id): Path, + Json(new_trigger): Json>, +) -> Result<(StatusCode, String)> { + check_scopes(&authed, || { + format!( + "{}:write:{}", + T::scope_domain_name(), + &new_trigger.base.path + ) + })?; + + if *CLOUD_HOSTED && !T::IS_ALLOWED_ON_CLOUD { + return Err(Error::BadRequest(format!( + "{} triggers are not supported on multi-tenant cloud, use dedicated cloud or self-host", + T::TRIGGER_TYPE + ))); + } + + handler + .validate_new(&db, &workspace_id, &new_trigger.config) + .await?; + + let mut tx = user_db.begin(&authed).await?; + + let new_path = new_trigger.base.path.clone(); + + handler + .create_trigger(&db, &mut *tx, &authed, &workspace_id, new_trigger) + .await?; + + audit_log( + &mut *tx, + &authed, + &format!("{}_triggers.create", T::TRIGGER_TYPE), + ActionKind::Create, + &workspace_id, + Some(&new_path), + None, + ) + .await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &workspace_id, + T::get_deployed_object(new_path.clone()), + Some(format!("{} '{}' created", T::DEPLOYMENT_NAME, new_path)), + true, + ) + .await?; + + tx.commit().await?; + + Ok((StatusCode::CREATED, new_path)) +} + +async fn list_triggers( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(user_db): Extension, + Path(workspace_id): Path, + Query(query): Query, +) -> JsonResult> { + let mut tx = user_db.begin(&authed).await?; + let triggers = handler + .list_triggers(&mut *tx, &workspace_id, Some(&query)) + .await?; + tx.commit().await?; + + Ok(Json(triggers)) +} + +async fn get_trigger( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(user_db): Extension, + Path((workspace_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + check_scopes(&authed, || { + format!("{}:read:{}", T::scope_domain_name(), &path) + })?; + + let mut tx = user_db.begin(&authed).await?; + let trigger = handler + .get_trigger_by_path(&mut *tx, &workspace_id, path) + .await?; + + tx.commit().await?; + + Ok(Json(trigger)) +} + +async fn update_trigger( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path((workspace_id, path)): Path<(String, StripPath)>, + Json(edit_trigger): Json>, +) -> Result { + let path = path.to_path(); + check_scopes(&authed, || { + format!( + "{}:write:{}", + T::scope_domain_name(), + &edit_trigger.base.path + ) + })?; + + handler + .validate_edit(&db, &workspace_id, &edit_trigger.config, path) + .await?; + + let mut tx = user_db.begin(&authed).await?; + + let new_path = edit_trigger.base.path.to_string(); + handler + .update_trigger(&db, &mut *tx, &authed, &workspace_id, path, edit_trigger) + .await?; + + audit_log( + &mut *tx, + &authed, + &format!("{}_triggers.update", T::TRIGGER_TYPE), + ActionKind::Update, + &workspace_id, + Some(&new_path), + None, + ) + .await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &workspace_id, + T::get_deployed_object(new_path.clone()), + Some(format!("{} '{}' updated", T::DEPLOYMENT_NAME, new_path)), + true, + ) + .await?; + + tx.commit().await?; + + Ok(format!("Trigger '{}' updated", path)) +} + +async fn delete_trigger( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(user_db): Extension, + Path((workspace_id, path)): Path<(String, StripPath)>, +) -> Result { + let path = path.to_path(); + check_scopes(&authed, || { + format!("{}:write:{}", T::scope_domain_name(), &path) + })?; + + let mut tx = user_db.begin(&authed).await?; + let deleted = handler + .delete_by_path(&mut *tx, &workspace_id, path) + .await?; + + if !deleted { + return Err(Error::NotFound(format!( + "Trigger not found at path: {}", + path + ))); + } + + audit_log( + &mut *tx, + &authed, + &format!("{}_triggers.delete", T::TRIGGER_TYPE), + ActionKind::Delete, + &workspace_id, + Some(&path), + None, + ) + .await?; + + tx.commit().await?; + + Ok(format!("Trigger '{}' deleted", path)) +} + +async fn exists_trigger( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(db): Extension, + Path((workspace_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + check_scopes(&authed, || { + format!("{}:read:{}", T::scope_domain_name(), path) + })?; + let exists = handler.exists(&db, &workspace_id, path).await?; + + Ok(Json(exists)) +} + +#[derive(serde::Deserialize)] +struct SetEnabledPayload { + enabled: bool, +} + +async fn set_enabled_trigger( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(user_db): Extension, + Extension(db): Extension, + Path((workspace_id, path)): Path<(String, StripPath)>, + Json(payload): Json, +) -> Result { + let path = path.to_path(); + check_scopes(&authed, || format!("{}:write", T::scope_domain_name()))?; + + let mut tx = user_db.begin(&authed).await?; + let updated = handler + .set_enabled(&authed, &mut *tx, &workspace_id, path, payload.enabled) + .await?; + + if !updated { + return Err(Error::NotFound(format!( + "Trigger not found at path: {}", + path + ))); + } + + tx.commit().await?; + + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &workspace_id, + T::get_deployed_object(path.to_owned()), + Some(format!("{} trigger '{}' updated", T::DEPLOYMENT_NAME, path)), + true, + ) + .await?; + + Ok(format!( + "Trigger '{}' {}", + path, + if payload.enabled { + "enabled" + } else { + "disabled" + } + )) +} + +async fn test_connection( + Extension(handler): Extension>, + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(workspace_id): Path, + Json(config): Json, +) -> Result<()> { + let connect_f = async move { + handler + .test_connection(&db, &authed, &user_db, &workspace_id, config) + .await + }; + + tokio::time::timeout(tokio::time::Duration::from_secs(30), connect_f) + .await + .map_err(|_| { + Error::BadConfig(format!("Timeout connecting to service after 30 seconds")) + })??; + Ok(()) +} + +#[allow(unused)] +pub fn complete_trigger_routes(handler: T) -> Router { + let standard_routes = trigger_routes::(); + + let additional_routes = handler.additional_routes(); + + standard_routes + .merge(additional_routes) + .layer(Extension(Arc::new(handler))) +} + +pub fn generate_trigger_routers() -> Router { + #[allow(unused_mut)] + let mut router = Router::new(); + + #[cfg(feature = "http_trigger")] + { + use crate::triggers::http::handler::HttpTrigger; + + router = router.nest( + HttpTrigger::ROUTE_PREFIX, + complete_trigger_routes(HttpTrigger), + ); + } + + #[cfg(feature = "websocket")] + { + use crate::triggers::websocket::WebsocketTrigger; + + router = router.nest( + WebsocketTrigger::ROUTE_PREFIX, + complete_trigger_routes(WebsocketTrigger), + ); + } + + #[cfg(all(feature = "enterprise", feature = "kafka", feature = "private"))] + { + use crate::triggers::kafka::KafkaTrigger; + + router = router.nest( + KafkaTrigger::ROUTE_PREFIX, + complete_trigger_routes(KafkaTrigger), + ); + } + + #[cfg(all(feature = "enterprise", feature = "nats", feature = "private"))] + { + use crate::triggers::nats::NatsTrigger; + + router = router.nest( + NatsTrigger::ROUTE_PREFIX, + complete_trigger_routes(NatsTrigger), + ); + } + + #[cfg(feature = "mqtt_trigger")] + { + use crate::triggers::mqtt::MqttTrigger; + + router = router.nest( + MqttTrigger::ROUTE_PREFIX, + complete_trigger_routes(MqttTrigger), + ); + } + + #[cfg(all(feature = "enterprise", feature = "sqs_trigger", feature = "private"))] + { + use crate::triggers::sqs::SqsTrigger; + + router = router.nest( + SqsTrigger::ROUTE_PREFIX, + complete_trigger_routes(SqsTrigger), + ); + } + + #[cfg(all(feature = "enterprise", feature = "gcp_trigger", feature = "private"))] + { + use crate::triggers::gcp::GcpTrigger; + + router = router.nest( + GcpTrigger::ROUTE_PREFIX, + complete_trigger_routes(GcpTrigger), + ); + } + + #[cfg(feature = "postgres_trigger")] + { + use crate::triggers::postgres::PostgresTrigger; + + router = router.nest( + PostgresTrigger::ROUTE_PREFIX, + complete_trigger_routes(PostgresTrigger), + ); + } + + router +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct TriggerPrimarySchedule { + schedule: String, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct TriggersCount { + primary_schedule: Option, + schedule_count: i64, + http_routes_count: i64, + webhook_count: i64, + email_count: i64, + websocket_count: i64, + kafka_count: i64, + nats_count: i64, + postgres_count: i64, + mqtt_count: i64, + sqs_count: i64, + gcp_count: i64, +} + +pub async fn get_triggers_count_internal( + db: &DB, + w_id: &str, + path: &str, + is_flow: bool, +) -> JsonResult { + let primary_schedule = sqlx::query_scalar!( + "SELECT schedule FROM schedule WHERE path = $1 AND script_path = $1 AND is_flow = $2 AND workspace_id = $3", + path, + is_flow, + w_id + ) + .fetch_optional(db) + .await?; + + let schedule_count = sqlx::query_scalar!( + "SELECT COUNT(*) FROM schedule WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", + path, + is_flow, + w_id + ) + .fetch_one(db) + .await? + .unwrap_or(0); + + #[allow(unused)] + let mut tx = db.begin().await?; + + #[cfg(feature = "http_trigger")] + let http_routes_count = { + use crate::triggers::http::handler::HttpTrigger; + let count = HttpTrigger + .trigger_count(&mut tx, w_id, is_flow, path) + .await; + count + }; + #[cfg(not(feature = "http_trigger"))] + let http_routes_count = 0; + + #[cfg(feature = "websocket")] + let websocket_count = { + use crate::triggers::websocket::WebsocketTrigger; + let count = WebsocketTrigger + .trigger_count(&mut tx, w_id, is_flow, path) + .await; + count + }; + #[cfg(not(feature = "websocket"))] + let websocket_count = 0; + + #[cfg(all(feature = "kafka", feature = "enterprise", feature = "private"))] + let kafka_count = { + use crate::triggers::kafka::KafkaTrigger; + let count = KafkaTrigger + .trigger_count(&mut tx, w_id, is_flow, path) + .await; + count + }; + #[cfg(not(all(feature = "kafka", feature = "enterprise", feature = "private")))] + let kafka_count = 0; + + #[cfg(all(feature = "nats", feature = "enterprise", feature = "private"))] + let nats_count = { + use crate::triggers::nats::NatsTrigger; + let count = NatsTrigger + .trigger_count(&mut tx, w_id, is_flow, path) + .await; + count + }; + #[cfg(not(all(feature = "nats", feature = "enterprise", feature = "private")))] + let nats_count = 0; + + #[cfg(feature = "postgres_trigger")] + let postgres_count = { + use crate::triggers::postgres::PostgresTrigger; + let count = PostgresTrigger + .trigger_count(&mut tx, w_id, is_flow, path) + .await; + count + }; + #[cfg(not(feature = "postgres_trigger"))] + let postgres_count = 0; + + #[cfg(feature = "mqtt_trigger")] + let mqtt_count = { + use crate::triggers::mqtt::MqttTrigger; + let count = MqttTrigger + .trigger_count(&mut tx, w_id, is_flow, path) + .await; + count + }; + #[cfg(not(feature = "mqtt_trigger"))] + let mqtt_count = 0; + + #[cfg(all(feature = "sqs_trigger", feature = "enterprise", feature = "private"))] + let sqs_count = { + use crate::triggers::sqs::SqsTrigger; + let count = SqsTrigger.trigger_count(&mut tx, w_id, is_flow, path).await; + count + }; + #[cfg(not(all(feature = "sqs_trigger", feature = "enterprise", feature = "private")))] + let sqs_count = 0; + + #[cfg(all(feature = "gcp_trigger", feature = "enterprise", feature = "private"))] + let gcp_count = { + use crate::triggers::gcp::GcpTrigger; + let count = GcpTrigger.trigger_count(&mut tx, w_id, is_flow, path).await; + count + }; + #[cfg(not(all(feature = "gcp_trigger", feature = "enterprise", feature = "private")))] + let gcp_count = 0; + tx.commit().await?; + + let webhook_count = (if is_flow { + sqlx::query_scalar!( + "SELECT COUNT(*) FROM token WHERE label LIKE 'webhook-%' AND workspace_id = $1 AND scopes @> ARRAY['run:flow/' || $2]::text[]", + w_id, + path, + ) + } else { + sqlx::query_scalar!( + "SELECT COUNT(*) FROM token WHERE label LIKE 'webhook-%' AND workspace_id = $1 AND scopes @> ARRAY['run:' || $2]::text[]", + w_id, + path, + ) + }).fetch_one(db) + .await? + .unwrap_or(0); + + let email_count = (if is_flow { + sqlx::query_scalar!( + "SELECT COUNT(*) FROM token WHERE label LIKE 'email-%' AND workspace_id = $1 AND scopes @> ARRAY['run:flow/' || $2]::text[]", + w_id, + path, + ) + } else { + sqlx::query_scalar!( + "SELECT COUNT(*) FROM token WHERE label LIKE 'email-%' AND workspace_id = $1 AND scopes @> ARRAY['run:script/' || $2]::text[]", + w_id, + path, + ) + }).fetch_one(db) + .await? + .unwrap_or(0); + + Ok(Json(TriggersCount { + primary_schedule: primary_schedule.map(|s| TriggerPrimarySchedule { schedule: s }), + schedule_count, + http_routes_count, + webhook_count, + email_count, + websocket_count, + kafka_count, + nats_count, + postgres_count, + mqtt_count, + gcp_count, + sqs_count, + })) +} diff --git a/backend/windmill-api/src/triggers/http/handler.rs b/backend/windmill-api/src/triggers/http/handler.rs new file mode 100644 index 0000000000..78ea4029c5 --- /dev/null +++ b/backend/windmill-api/src/triggers/http/handler.rs @@ -0,0 +1,1015 @@ +use super::{ + http_trigger_args::RawHttpTriggerArgs, AuthenticationMethod, HttpMethod, TriggerRoute, + HTTP_ACCESS_CACHE, HTTP_AUTH_CACHE, HTTP_ROUTERS_CACHE, +}; +use crate::{ + auth::{AuthCache, OptTokened}, + db::{ApiAuthed, DB}, + resources::try_get_resource_from_db_as, + triggers::{ + http::{ + refresh_routers, validate_authentication_method, HttpConfig, HttpConfigRequest, + RouteExists, ROUTE_PATH_KEY_RE, VALID_ROUTE_PATH_RE, + }, + trigger_helpers::{ + get_runnable_format, trigger_runnable, trigger_runnable_and_wait_for_result, RunnableId, + }, + Trigger, TriggerCrud, TriggerData, + }, + users::fetch_api_authed, + utils::ExpiringCacheEntry, +}; +use axum::{ + async_trait, + extract::Path, + response::{IntoResponse, Response}, + routing::{get, post}, + Extension, Json, Router, +}; +use http::{HeaderMap, StatusCode}; +use sqlx::PgConnection; +use std::{ + borrow::Cow, + collections::{HashMap, HashSet}, + sync::Arc, +}; +use tower_http::cors::CorsLayer; +use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_common::{ + db::UserDB, + error::{Error, Result}, + triggers::TriggerKind, + utils::{not_found_if_none, require_admin, StripPath}, + worker::CLOUD_HOSTED, +}; +use windmill_git_sync::handle_deployment_metadata; + +#[cfg(feature = "parquet")] +use { + crate::job_helpers_oss::get_workspace_s3_resource, + windmill_common::s3_helpers::build_object_store_client, +}; + +use windmill_git_sync::DeployedObject; + +pub async fn increase_trigger_version(tx: &mut PgConnection) -> Result<()> { + sqlx::query!("SELECT nextval('http_trigger_version_seq')") + .fetch_one(tx) + .await?; + Ok(()) +} + +pub fn generate_route_path_key(route_path: &str) -> String { + ROUTE_PATH_KEY_RE.replace_all(route_path, "/*").to_string() +} + +pub async fn route_path_key_exists( + route_path_key: &str, + http_method: &HttpMethod, + w_id: &str, + trigger_path: Option<&str>, + workspaced_route: Option, + db: &DB, +) -> Result { + let exists = if *CLOUD_HOSTED { + sqlx::query_scalar!( + r#" + SELECT EXISTS( + SELECT 1 + FROM http_trigger + WHERE + route_path_key = $1 + AND workspace_id = $2 + AND http_method = $3 + AND ($4::TEXT IS NULL OR path != $4) + ) + "#, + &route_path_key, + w_id, + http_method as &HttpMethod, + trigger_path + ) + .fetch_one(db) + .await? + .unwrap_or(false) + } else { + let route_path_key = match workspaced_route { + Some(true) => Cow::Owned(format!("{}/{}", w_id, route_path_key.trim_matches('/'))), + _ => Cow::Borrowed(route_path_key), + }; + + sqlx::query_scalar!( + r#" + SELECT EXISTS( + SELECT 1 + FROM http_trigger + WHERE + ((workspaced_route IS TRUE AND workspace_id || '/' || route_path_key = $1) + OR (workspaced_route IS FALSE AND route_path_key = $1)) + AND http_method = $2 + AND ($3::TEXT IS NULL OR path != $3) + ) + "#, + &route_path_key, + http_method as &HttpMethod, + trigger_path + ) + .fetch_one(db) + .await? + .unwrap_or(false) + }; + + Ok(exists) +} + +pub async fn exists_route( + Extension(db): Extension, + Path(w_id): Path, + Json(RouteExists { route_path, http_method, trigger_path, workspaced_route }): Json< + RouteExists, + >, +) -> Result> { + let route_path_key = generate_route_path_key(&route_path); + + let exists = route_path_key_exists( + &route_path_key, + &http_method, + &w_id, + trigger_path.as_deref(), + workspaced_route, + &db, + ) + .await?; + + Ok(Json(exists)) +} + +fn check_no_duplicates<'trigger>( + new_http_triggers: &[TriggerData], + route_path_key: &[String], +) -> Result<()> { + let mut seen = HashSet::with_capacity(new_http_triggers.len()); + + for (i, trigger) in new_http_triggers.iter().enumerate() { + if !seen.insert(( + &route_path_key[i], + trigger.config.http_method, + trigger.config.workspaced_route, + )) { + return Err(Error::BadRequest(format!( + "Duplicate HTTP route detected: '{}'. Each HTTP route must have a unique 'route_path'.", + &trigger.config.route_path + ))); + } + } + + Ok(()) +} + +pub async fn insert_new_trigger_into_db( + authed: &ApiAuthed, + tx: &mut PgConnection, + w_id: &str, + trigger: &TriggerData, + route_path_key: &str, +) -> Result<()> { + require_admin(authed.is_admin, &authed.username)?; + + sqlx::query!( + r#" + INSERT INTO http_trigger ( + workspace_id, + path, + route_path, + route_path_key, + workspaced_route, + authentication_resource_path, + wrap_body, + raw_string, + script_path, + summary, + description, + is_flow, + is_async, + authentication_method, + http_method, + static_asset_config, + edited_by, + email, + edited_at, + is_static_website, + error_handler_path, + error_handler_args, + retry + ) + VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, now(), $19, $20, $21, $22 + ) + "#, + w_id, + trigger.base.path, + trigger.config.route_path, + route_path_key, + trigger.config.workspaced_route.unwrap_or(false), + trigger.config.authentication_resource_path, + trigger.config.wrap_body.unwrap_or(false), + trigger.config.raw_string.unwrap_or(false), + trigger.base.script_path, + trigger.config.summary, + trigger.config.description, + trigger.base.is_flow, + trigger.config.is_async, + trigger.config.authentication_method as _, + trigger.config.http_method as _, + trigger.config.static_asset_config as _, + &authed.username, + &authed.email, + trigger.config.is_static_website, + trigger.error_handling.error_handler_path, + trigger.error_handling.error_handler_args as _, + trigger.error_handling.retry as _ + ) + .execute(&mut *tx) + .await?; + Ok(()) +} + +pub async fn create_many_http_triggers( + authed: ApiAuthed, + Extension(db): Extension, + Extension(user_db): Extension, + Path(w_id): Path, + Json(new_http_triggers): Json>>, +) -> Result<(StatusCode, String)> { + require_admin(authed.is_admin, &authed.username)?; + + let handler = HttpTrigger; + + let error_wrapper = |route_path: &str, error: Error| -> Error { + anyhow::anyhow!( + "Error occurred for HTTP route at route path: {}, error: {}", + route_path, + error + ) + .into() + }; + + let mut route_path_keys = Vec::with_capacity(new_http_triggers.len()); + + for new_http_trigger in new_http_triggers.iter() { + handler + .validate_new(&db, &w_id, &new_http_trigger.config) + .await + .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err))?; + + let route_path_key = + check_if_route_exist(&db, &new_http_trigger.config, &w_id, None).await?; + + route_path_keys.push(route_path_key.clone()); + } + + check_no_duplicates(&new_http_triggers, &route_path_keys)?; + + let mut tx = user_db.begin(&authed).await?; + + for (new_http_trigger, route_path_key) in new_http_triggers.iter().zip(route_path_keys.iter()) { + insert_new_trigger_into_db(&authed, &mut tx, &w_id, new_http_trigger, route_path_key) + .await + .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err))?; + + audit_log( + &mut *tx, + &authed, + "http_trigger.create", + ActionKind::Create, + &w_id, + Some(&new_http_trigger.base.path), + None, + ) + .await + .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err.into()))?; + + increase_trigger_version(&mut tx) + .await + .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err.into()))?; + } + + tx.commit().await?; + + for http_trigger in new_http_triggers.into_iter() { + handle_deployment_metadata( + &authed.email, + &authed.username, + &db, + &w_id, + windmill_git_sync::DeployedObject::HttpTrigger { path: http_trigger.base.path.clone() }, + Some(format!("HTTP trigger '{}' created", http_trigger.base.path)), + true, + ) + .await + .map_err(|err| error_wrapper(&http_trigger.config.route_path, err.into()))?; + } + Ok((StatusCode::CREATED, "Created all HTTP routes".to_string())) +} + +async fn check_if_route_exist( + db: &DB, + config: &HttpConfigRequest, + workspace_id: &str, + trigger_path: Option<&str>, +) -> Result { + let route_path_key = ROUTE_PATH_KEY_RE.replace_all(&config.route_path, ":key"); + + let exists = route_path_key_exists( + &route_path_key, + &config.http_method, + workspace_id, + trigger_path, + config.workspaced_route, + db, + ) + .await?; + + if exists { + return Err(Error::BadRequest( + "A route already exists with this path".to_string(), + )); + } + + Ok(route_path_key.into_owned()) +} + +pub struct HttpTrigger; + +#[async_trait] +impl TriggerCrud for HttpTrigger { + type TriggerConfig = HttpConfig; + type Trigger = Trigger; + type TriggerConfigRequest = HttpConfigRequest; + type TestConnectionConfig = (); + + const TABLE_NAME: &'static str = "http_trigger"; + const TRIGGER_TYPE: &'static str = "http"; + const SUPPORTS_ENABLED: bool = false; + const SUPPORTS_SERVER_STATE: bool = false; + const SUPPORTS_TEST_CONNECTION: bool = false; + const ROUTE_PREFIX: &'static str = "/http_triggers"; + const DEPLOYMENT_NAME: &'static str = "HTTP trigger"; + const IS_ALLOWED_ON_CLOUD: bool = true; + const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[ + "route_path", + "route_path_key", + "is_async", + "authentication_method", + "http_method", + "summary", + "description", + "static_asset_config", + "is_static_website", + "authentication_resource_path", + "workspaced_route", + "wrap_body", + "raw_string", + ]; + + fn get_deployed_object(path: String) -> DeployedObject { + DeployedObject::HttpTrigger { path } + } + + fn additional_routes(&self) -> Router { + Router::new() + .route("/create_many", post(create_many_http_triggers)) + .route("/route_exists", post(exists_route)) + } + + async fn validate_new( + &self, + _db: &DB, + _workspace_id: &str, + new: &Self::TriggerConfigRequest, + ) -> Result<()> { + if *CLOUD_HOSTED && (new.is_static_website || new.static_asset_config.is_some()) { + return Err(Error::BadRequest( + "Static website and static asset are not supported on cloud".to_string(), + )); + } + + if !VALID_ROUTE_PATH_RE.is_match(&new.route_path) { + return Err(Error::BadRequest("Invalid route path".to_string())); + } + + validate_authentication_method(new.authentication_method, new.raw_string)?; + + Ok(()) + } + + async fn validate_edit( + &self, + _db: &DB, + _workspace_id: &str, + edit: &Self::TriggerConfigRequest, + _path: &str, + ) -> Result<()> { + if *CLOUD_HOSTED && (edit.is_static_website || edit.static_asset_config.is_some()) { + return Err(Error::BadRequest( + "Static website and static asset are not supported on cloud".to_string(), + )); + } + + validate_authentication_method(edit.authentication_method, edit.raw_string)?; + + Ok(()) + } + + async fn create_trigger( + &self, + db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + w_id: &str, + trigger: TriggerData, + ) -> Result<()> { + let route_path_key = check_if_route_exist(db, &trigger.config, &w_id, None).await?; + + insert_new_trigger_into_db(authed, tx, w_id, &trigger, &route_path_key).await?; + + increase_trigger_version(tx).await?; + + Ok(()) + } + + async fn update_trigger( + &self, + db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + workspace_id: &str, + path: &str, + trigger: TriggerData, + ) -> Result<()> { + if authed.is_admin { + if trigger.config.route_path.is_empty() { + return Err(Error::BadRequest("route_path is required".to_string())); + }; + + let route_path = &trigger.config.route_path; + if !VALID_ROUTE_PATH_RE.is_match(route_path) { + return Err(Error::BadRequest("Invalid route path".to_string())); + } + + let route_path_key = + check_if_route_exist(db, &trigger.config, workspace_id, Some(path)) + .await?; + + sqlx::query!( + r#" + UPDATE + http_trigger + SET + route_path = $1, + route_path_key = $2, + workspaced_route = $3, + wrap_body = $4, + raw_string = $5, + authentication_resource_path = $6, + script_path = $7, + path = $8, + is_flow = $9, + http_method = $10, + static_asset_config = $11, + edited_by = $12, + email = $13, + is_async = $14, + authentication_method = $15, + summary = $16, + description = $17, + edited_at = now(), + is_static_website = $18, + error_handler_path = $19, + error_handler_args = $20, + retry = $21 + WHERE + workspace_id = $22 AND + path = $23 + "#, + route_path, + &route_path_key, + trigger.config.workspaced_route, + trigger.config.wrap_body, + trigger.config.raw_string, + trigger.config.authentication_resource_path, + trigger.base.script_path, + trigger.base.path, + trigger.base.is_flow, + trigger.config.http_method as _, + trigger.config.static_asset_config as _, + &authed.username, + &authed.email, + trigger.config.is_async, + trigger.config.authentication_method as _, + trigger.config.summary, + trigger.config.description, + trigger.config.is_static_website, + trigger.error_handling.error_handler_path, + trigger.error_handling.error_handler_args as _, + trigger.error_handling.retry as _, + workspace_id, + path, + ) + .execute(&mut *tx) + .await?; + } else { + sqlx::query!( + r#" + UPDATE + http_trigger + SET + workspaced_route = $1, + wrap_body = $2, + raw_string = $3, + authentication_resource_path = $4, + script_path = $5, + path = $6, + is_flow = $7, + http_method = $8, + static_asset_config = $9, + edited_by = $10, + email = $11, + is_async = $12, + authentication_method = $13, + summary = $14, + description = $15, + edited_at = now(), + is_static_website = $16, + error_handler_path = $17, + error_handler_args = $18, + retry = $19 + WHERE + workspace_id = $20 AND + path = $21 + "#, + trigger.config.workspaced_route, + trigger.config.wrap_body, + trigger.config.raw_string, + trigger.config.authentication_resource_path, + trigger.base.script_path, + trigger.base.path, + trigger.base.is_flow, + trigger.config.http_method as _, + trigger.config.static_asset_config as _, + &authed.username, + &authed.email, + trigger.config.is_async, + trigger.config.authentication_method as _, + trigger.config.summary, + trigger.config.description, + trigger.config.is_static_website, + trigger.error_handling.error_handler_path, + trigger.error_handling.error_handler_args as _, + trigger.error_handling.retry as _, + workspace_id, + path, + ) + .execute(&mut *tx) + .await?; + } + + increase_trigger_version(tx).await?; + + Ok(()) + } + + async fn delete_by_path( + &self, + tx: &mut PgConnection, + workspace_id: &str, + path: &str, + ) -> Result { + let deleted = sqlx::query(&format!( + "DELETE FROM {} WHERE workspace_id = $1 AND path = $2", + Self::TABLE_NAME + )) + .bind(workspace_id) + .bind(path) + .execute(&mut *tx) + .await? + .rows_affected(); + + increase_trigger_version(tx).await?; + + Ok(deleted > 0) + } +} + +pub fn http_route_trigger_handler() -> Router { + let cors = CorsLayer::new() + .allow_methods([ + http::Method::GET, + http::Method::POST, + http::Method::DELETE, + http::Method::PUT, + http::Method::PATCH, + ]) + .allow_headers([http::header::CONTENT_TYPE, http::header::AUTHORIZATION]) + .allow_origin(tower_http::cors::Any); + Router::new() + .route( + "/*path", + get(route_job) + .post(route_job) + .delete(route_job) + .put(route_job) + .patch(route_job) + .head(|| async { "" }), + ) + .layer(cors) +} + +async fn get_http_route_trigger( + route_path: &str, + auth_cache: &Arc, + token: Option<&String>, + db: &DB, + user_db: UserDB, + method: &http::Method, +) -> Result<(TriggerRoute, String, HashMap, ApiAuthed)> { + let http_method: HttpMethod = method.try_into()?; + + let requested_path = format!("/{}", route_path); + + let routers_cache = HTTP_ROUTERS_CACHE.read().await; + + let routers_cache = if routers_cache.routers.is_empty() { + tracing::warn!("HTTP routers are not loaded, loading from db"); + let (_, routers_cache) = refresh_routers(db).await?; + routers_cache + } else { + routers_cache + }; + + let router = routers_cache + .routers + .get(&http_method) + .ok_or(Error::internal_err( + "HTTP routers could not be loaded".to_string(), + ))?; + + let trigger_match = router.at(requested_path.as_str()).ok(); + + let matchit::Match { value: trigger, params } = + not_found_if_none(trigger_match, "Trigger", requested_path.as_str())?; + + let params: HashMap = params + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + + let username_override = if let AuthenticationMethod::Windmill = trigger.authentication_method { + let opt_authed = if let Some(token) = token { + auth_cache + .get_authed(Some(trigger.workspace_id.clone()), token) + .await + } else { + None + }; + if let Some(authed) = opt_authed { + // check that the user has access to the trigger + let cache_key = ( + trigger.workspace_id.clone(), + trigger.path.clone(), + authed.clone(), + ); + let exists = match HTTP_ACCESS_CACHE.get(&cache_key) { + Some(cache_entry) if cache_entry.expiry > std::time::Instant::now() => { + tracing::debug!("HTTP access cache hit for route {}", trigger.path); + true + } + _ => { + tracing::debug!("HTTP access cache miss for route {}", trigger.path); + let mut tx = user_db.begin(&authed).await?; + let exists = sqlx::query_scalar!( + r#" + SELECT EXISTS( + SELECT 1 + FROM + http_trigger + WHERE + workspace_id = $1 AND + path = $2 + ) + "#, + trigger.workspace_id, + trigger.path + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false); + if exists { + HTTP_ACCESS_CACHE.insert( + cache_key, + ExpiringCacheEntry { + value: (), + expiry: std::time::Instant::now() + + std::time::Duration::from_secs(10), + }, + ); + } + exists + } + }; + if exists { + Some(authed.display_username().to_owned()) + } else { + return Err(Error::NotAuthorized("Unauthorized".to_string())); + } + } else { + return Err(Error::NotAuthorized("Requires authentication".to_string())); + } + } else { + None + }; + + let authed = fetch_api_authed( + trigger.edited_by.clone(), + trigger.email.clone(), + &trigger.workspace_id, + &db, + Some(username_override.unwrap_or(format!("HTTP-{}", trigger.path))), + ) + .await?; + + Ok((trigger.clone(), route_path.to_string(), params, authed)) +} + +async fn route_job( + Extension(db): Extension, + Extension(user_db): Extension, + Extension(auth_cache): Extension>, + OptTokened { token }: OptTokened, + Path(route_path): Path, + headers: HeaderMap, + args: RawHttpTriggerArgs, +) -> std::result::Result { + let route_path = route_path.to_path().trim_end_matches("/"); + let (trigger, called_path, params, authed) = get_http_route_trigger( + route_path, + &auth_cache, + token.as_ref(), + &db, + user_db.clone(), + &args.0.metadata.method, + ) + .await + .map_err(|e| e.into_response())?; + + if trigger.script_path.is_empty() && trigger.static_asset_config.is_none() { + return Err(Error::NotFound(format!( + "Runnable path of HTTP route at path: {}", + trigger.path + )) + .into_response()); + } + + let args = args + .process_args( + &authed, + &db, + &trigger.workspace_id, + match trigger.authentication_method { + AuthenticationMethod::CustomScript | AuthenticationMethod::Signature => true, + _ => trigger.raw_string, + }, + ) + .await + .map_err(|e| e.into_response())?; + + match trigger.authentication_method { + AuthenticationMethod::None + | AuthenticationMethod::Windmill + | AuthenticationMethod::CustomScript => {} + _ => { + let resource_path = match trigger.authentication_resource_path { + Some(resource_path) => resource_path, + None => { + return Err(Error::BadRequest( + "Missing authentication resource path".to_string(), + ) + .into_response()) + } + }; + + let cache_key = ( + trigger.workspace_id.clone(), + resource_path.clone(), + authed.clone(), + ); + + let authentication_method = match HTTP_AUTH_CACHE.get(&cache_key) { + Some(cache_entry) if cache_entry.expiry > std::time::Instant::now() => { + tracing::debug!("HTTP auth method cache hit for route {}", trigger.path); + cache_entry.value + } + _ => { + tracing::debug!("HTTP auth method cache miss for route {}", trigger.path); + let auth_method = try_get_resource_from_db_as::< + super::http_trigger_auth::AuthenticationMethod, + >( + &authed, + Some(user_db.clone()), + &db, + &resource_path, + &trigger.workspace_id, + ) + .await + .map_err(|e| e.into_response())?; + HTTP_AUTH_CACHE.insert( + cache_key, + ExpiringCacheEntry { + value: auth_method.clone(), + expiry: std::time::Instant::now() + std::time::Duration::from_secs(60), + }, + ); + auth_method + } + }; + + let raw_payload = args.0.metadata.raw_string.as_ref(); + + let response = authentication_method + .authenticate_http_request(&headers, raw_payload) + .map_err(|e| e.into_response())?; + + if let Some(response) = response { + return Ok(response); + } + } + } + + #[cfg(not(feature = "parquet"))] + if trigger.static_asset_config.is_some() { + return Err(Error::internal_err( + "Static asset configuration is not supported in this build".to_string(), + ) + .into_response()); + } + + #[cfg(feature = "parquet")] + if let Some(sqlx::types::Json(config)) = trigger.static_asset_config { + let build_static_response_f = async { + let (_, s3_resource_opt) = get_workspace_s3_resource( + &authed, + &db, + None, + &"NO_TOKEN".to_string(), // no token is provided in this case + &trigger.workspace_id, + config.storage, + ) + .await?; + let s3_resource = s3_resource_opt.ok_or(Error::internal_err( + "No files storage resource defined at the workspace level".to_string(), + ))?; + let s3_client = build_object_store_client(&s3_resource).await?; + + let path = if trigger.is_static_website { + let subpath = params + .get("wm_subpath") + .cloned() + .unwrap_or("index.html".to_string()); + tracing::info!("subpath: {}", subpath); + format!("{}/{}", config.s3.trim_end_matches('/'), subpath) + } else { + config.s3.clone() + }; + let path = object_store::path::Path::from(path); + let s3_object = s3_client.get(&path).await; + + let s3_object = match s3_object { + Err(object_store::Error::NotFound { .. }) if trigger.is_static_website => { + // fallback to index.html if the file is not found + let path = object_store::path::Path::from(format!( + "{}/index.html", + config.s3.trim_end_matches('/') + )); + s3_client.get(&path).await + } + r => r, + }; + + let s3_object = s3_object.map_err(|err| { + tracing::warn!("Error retrieving file from S3: {:?}", err); + Error::internal_err(format!("Error retrieving file: {}", err.to_string())) + })?; + + let mut response_headers = http::HeaderMap::new(); + if let Some(ref e_tag) = s3_object.meta.e_tag { + if let Some(if_none_match) = headers.get(http::header::IF_NONE_MATCH) { + if if_none_match == e_tag { + return Ok::<_, Error>(( + StatusCode::NOT_MODIFIED, + response_headers, + axum::body::Body::empty(), + )); + } + } + if let Ok(e_tag) = e_tag.parse() { + response_headers.insert("etag", e_tag); + } + } + response_headers.insert( + "content-type", + s3_object + .attributes + .get(&object_store::Attribute::ContentType) + .map(|s| s.parse().ok()) + .flatten() + .unwrap_or("application/octet-stream".parse().unwrap()), + ); + if !trigger.is_static_website { + response_headers.insert( + "content-disposition", + config.filename.as_ref().map_or_else( + || { + s3_object + .attributes + .get(&object_store::Attribute::ContentDisposition) + .map(|s| s.parse().ok()) + .flatten() + .unwrap_or("inline".parse().unwrap()) + }, + |filename| { + format!("inline; filename=\"{}\"", filename) + .parse() + .unwrap_or("inline".parse().unwrap()) + }, + ), + ); + } + + let body_stream = axum::body::Body::from_stream(s3_object.into_stream()); + Ok::<_, Error>((StatusCode::OK, response_headers, body_stream)) + }; + match build_static_response_f.await { + Ok((status, headers, body_stream)) => { + return Ok((status, headers, body_stream).into_response()) + } + Err(e) => return Err(e.into_response()), + } + } + + let runnable_format = get_runnable_format( + if trigger.is_flow { + RunnableId::from_flow_path(&trigger.script_path) + } else { + RunnableId::from_script_path(&trigger.script_path) + }, + &trigger.workspace_id, + &db, + &TriggerKind::Http, + ) + .await + .map_err(|e| e.into_response())?; + + let args = args + .to_args_from_format( + &trigger.route_path, + &called_path, + ¶ms, + runnable_format, + trigger.wrap_body, + ) + .map_err(|e| e.into_response())?; + + if trigger.is_async { + trigger_runnable( + &db, + Some(user_db), + authed, + &trigger.workspace_id, + &trigger.script_path, + trigger.is_flow, + args, + trigger.retry.as_ref(), + trigger.error_handler_path.as_deref(), + trigger.error_handler_args.as_ref(), + format!("http_trigger/{}", trigger.path), + ) + .await + .map_err(|e| e.into_response()) + } else { + trigger_runnable_and_wait_for_result( + &db, + Some(user_db), + authed, + &trigger.workspace_id, + &trigger.script_path, + trigger.is_flow, + args, + trigger.retry.as_ref(), + trigger.error_handler_path.as_deref(), + trigger.error_handler_args.as_ref(), + format!("http_trigger/{}", trigger.path), + ) + .await + .map_err(|e| e.into_response()) + } +} diff --git a/backend/windmill-api/src/http_trigger_args.rs b/backend/windmill-api/src/triggers/http/http_trigger_args.rs similarity index 100% rename from backend/windmill-api/src/http_trigger_args.rs rename to backend/windmill-api/src/triggers/http/http_trigger_args.rs diff --git a/backend/windmill-api/src/http_trigger_auth.rs b/backend/windmill-api/src/triggers/http/http_trigger_auth.rs similarity index 100% rename from backend/windmill-api/src/http_trigger_auth.rs rename to backend/windmill-api/src/triggers/http/http_trigger_auth.rs diff --git a/backend/windmill-api/src/triggers/http/mod.rs b/backend/windmill-api/src/triggers/http/mod.rs new file mode 100644 index 0000000000..2b29f0e644 --- /dev/null +++ b/backend/windmill-api/src/triggers/http/mod.rs @@ -0,0 +1,279 @@ +use std::collections::HashMap; + +use quick_cache::sync::Cache; +use serde::{Deserialize, Serialize}; +use sqlx::{types::Json as SqlxJson, FromRow}; +use tokio::sync::{RwLock, RwLockReadGuard}; +use windmill_common::{ + error::{Error, Result}, + flows::Retry, + s3_helpers::S3Object, + worker::CLOUD_HOSTED, + DB, +}; + +use crate::{db::ApiAuthed, utils::ExpiringCacheEntry}; + +pub mod handler; +pub mod http_trigger_args; +pub mod http_trigger_auth; + +lazy_static::lazy_static! { + static ref HTTP_ACCESS_CACHE: Cache<(String, String, ApiAuthed), ExpiringCacheEntry<()>> = Cache::new(100); + static ref HTTP_AUTH_CACHE: Cache<(String, String, ApiAuthed), ExpiringCacheEntry> = Cache::new(100); + + static ref HTTP_ROUTERS_CACHE: RwLock = RwLock::new(RoutersCache { + routers: HashMap::new(), + version: 0, + }); +} + +#[derive(Debug, Deserialize, Clone)] +pub struct TriggerRoute { + path: String, + script_path: String, + is_flow: bool, + route_path: String, + workspace_id: String, + is_async: bool, + authentication_method: AuthenticationMethod, + edited_by: String, + email: String, + static_asset_config: Option>, + is_static_website: bool, + authentication_resource_path: Option, + workspaced_route: bool, + wrap_body: bool, + raw_string: bool, + error_handler_path: Option, + error_handler_args: Option>>, + retry: Option>, +} + +pub struct RoutersCache { + routers: HashMap>, + version: i64, +} + +#[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Copy, Hash, Eq, PartialEq)] +#[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum HttpMethod { + Get, + Post, + Put, + Delete, + Patch, +} + +impl TryFrom<&http::Method> for HttpMethod { + type Error = Error; + fn try_from(method: &http::Method) -> Result { + match method { + &http::Method::GET => Ok(HttpMethod::Get), + &http::Method::POST => Ok(HttpMethod::Post), + &http::Method::PUT => Ok(HttpMethod::Put), + &http::Method::DELETE => Ok(HttpMethod::Delete), + &http::Method::PATCH => Ok(HttpMethod::Patch), + _ => Err(Error::BadRequest("Invalid HTTP method".to_string())), + } + } +} + +#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone, Copy)] +#[sqlx(type_name = "AUTHENTICATION_METHOD", rename_all = "snake_case")] +#[serde(rename_all(serialize = "snake_case", deserialize = "snake_case"))] +pub enum AuthenticationMethod { + None, + Windmill, + ApiKey, + BasicHttp, + CustomScript, + Signature, +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct HttpConfig { + pub route_path: String, + pub route_path_key: String, + pub is_async: bool, + pub authentication_method: AuthenticationMethod, + pub http_method: HttpMethod, + pub summary: Option, + pub description: Option, + pub static_asset_config: Option>, + pub is_static_website: bool, + pub authentication_resource_path: Option, + pub workspaced_route: bool, + pub wrap_body: bool, + pub raw_string: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HttpConfigRequest { + #[serde(default)] + pub route_path: String, + pub is_async: bool, + pub authentication_method: AuthenticationMethod, + pub http_method: HttpMethod, + pub summary: Option, + pub description: Option, + pub static_asset_config: Option>, + pub is_static_website: bool, + pub authentication_resource_path: Option, + pub workspaced_route: Option, + pub wrap_body: Option, + pub raw_string: Option, +} + +// Regex patterns for route validation +lazy_static::lazy_static! { + static ref ROUTE_PATH_KEY_RE: regex::Regex = regex::Regex::new(r"/?:[-\w]+").unwrap(); + static ref VALID_ROUTE_PATH_RE: regex::Regex = regex::Regex::new(r"^:?[-\w]+(/:?[-\w]+)*$").unwrap(); +} + +#[derive(Deserialize)] +pub struct RouteExists { + pub route_path: String, + pub http_method: HttpMethod, + pub trigger_path: Option, + pub workspaced_route: Option, +} + +pub fn validate_authentication_method( + authentication_method: AuthenticationMethod, + raw_string: Option, +) -> Result<()> { + match (authentication_method, raw_string) { + (AuthenticationMethod::CustomScript, raw) if !raw.unwrap_or(false) => { + Err(Error::BadRequest( + "To use custom script authentication, please enable the raw body option." + .to_string(), + )) + } + _ => Ok(()), + } +} + +pub async fn refresh_routers(db: &DB) -> Result<(bool, RwLockReadGuard<'_, RoutersCache>)> { + let version = sqlx::query_scalar!("SELECT last_value FROM http_trigger_version_seq",) + .fetch_one(db) + .await?; + let routers_cache = HTTP_ROUTERS_CACHE.read().await; + if routers_cache.version == 0 || version > routers_cache.version { + drop(routers_cache); + let mut routers = HashMap::new(); + + for http_method in [ + HttpMethod::Get, + HttpMethod::Post, + HttpMethod::Put, + HttpMethod::Patch, + HttpMethod::Delete, + ] { + let triggers = sqlx::query_as!( + TriggerRoute, + r#" + SELECT + path, + script_path, + is_flow, + route_path, + authentication_resource_path, + workspace_id, + is_async, + authentication_method AS "authentication_method: _", + edited_by, + email, + static_asset_config AS "static_asset_config: _", + wrap_body, + raw_string, + workspaced_route, + is_static_website, + error_handler_path, + error_handler_args as "error_handler_args: _", + retry as "retry: _" + FROM + http_trigger + WHERE + http_method = $1 + "#, + &http_method as &HttpMethod + ) + .fetch_all(db) + .await?; + + let mut router = matchit::Router::new(); + + for trigger in triggers { + let full_path = if trigger.workspaced_route || *CLOUD_HOSTED { + format!("/{}/{}", trigger.workspace_id, trigger.route_path) + } else { + format!("/{}", trigger.route_path) + }; + + if trigger.is_static_website { + router + .insert(format!("{}/*wm_subpath", full_path), trigger.clone()) + .unwrap_or_else(|e| { + tracing::warn!( + "Failed to consider HTTP route {}/*wm_subpath: {:?}", + full_path, + e, + ); + }); + } + router + .insert(full_path.clone(), trigger.clone()) + .unwrap_or_else(|e| { + tracing::warn!("Failed to consider HTTP route {}: {:?}", full_path, e,); + }); + } + + routers.insert(http_method, router); + } + + let mut routers_cache = HTTP_ROUTERS_CACHE.write().await; + *routers_cache = RoutersCache { routers, version }; + + Ok((true, routers_cache.downgrade())) + } else { + tracing::debug!("No HTTP routers refresh needed"); + Ok((false, routers_cache)) + } +} + +pub async fn refresh_routers_loop( + db: &DB, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, +) -> () { + match refresh_routers(db).await { + Ok(_) => { + tracing::info!("Loaded HTTP routers"); + } + Err(err) => { + tracing::error!("Error loading HTTP routers: {err:#}"); + } + }; + let db = db.clone(); + tokio::spawn(async move { + loop { + tokio::select! { + _ = killpill_rx.recv() => { + break; + } + _ = tokio::time::sleep(std::time::Duration::from_secs(60)) => { + match refresh_routers(&db).await { + Ok((true, _)) => { + tracing::info!("Refreshed HTTP routers"); + } + Err(err) => { + tracing::error!("Error refreshing HTTP routers: {err:#}"); + } + _ => {} + } + } + } + } + }); +} diff --git a/backend/windmill-api/src/triggers/kafka/handler_oss.rs b/backend/windmill-api/src/triggers/kafka/handler_oss.rs new file mode 100644 index 0000000000..3922b684ce --- /dev/null +++ b/backend/windmill-api/src/triggers/kafka/handler_oss.rs @@ -0,0 +1,68 @@ +#[cfg(not(feature = "private"))] +use crate::triggers::TriggerData; + +#[allow(unused)] +#[cfg(feature = "private")] +pub use super::handler_ee::*; + +#[cfg(not(feature = "private"))] +use { + super::KafkaTrigger, + crate::{ + db::{ApiAuthed, DB}, + triggers::TriggerCrud, + }, + axum::async_trait, + sqlx::PgConnection, + windmill_common::error::{Error, Result}, + windmill_git_sync::DeployedObject, +}; + +#[cfg(not(feature = "private"))] +#[async_trait] +impl TriggerCrud for KafkaTrigger { + type Trigger = (); + type TriggerConfig = (); + type TriggerConfigRequest = (); + type TestConnectionConfig = (); + + const TABLE_NAME: &'static str = ""; + const TRIGGER_TYPE: &'static str = ""; + const SUPPORTS_ENABLED: bool = false; + const SUPPORTS_SERVER_STATE: bool = false; + const SUPPORTS_TEST_CONNECTION: bool = false; + const ROUTE_PREFIX: &'static str = "/kafka_triggers"; + const DEPLOYMENT_NAME: &'static str = ""; + const IS_CLOUD_HOSTED: bool = false; + + fn get_deployed_object(path: String) -> DeployedObject { + DeployedObject::KafkaTrigger { path } + } + + async fn create_trigger( + &self, + _db: &DB, + _tx: &mut PgConnection, + _authed: &ApiAuthed, + _w_id: &str, + _trigger: TriggerData, + ) -> Result<()> { + Err(Error::BadRequest( + "Kafka triggers are not available in open source version".to_string(), + )) + } + + async fn update_trigger( + &self, + _db: &DB, + _executor: &mut PgConnection, + _authed: &ApiAuthed, + _workspace_id: &str, + _path: &str, + _trigger: TriggerData, + ) -> Result<()> { + Err(Error::BadRequest( + "Kafka triggers are not available in open source version".to_string(), + )) + } +} diff --git a/backend/windmill-api/src/triggers/kafka/mod.rs b/backend/windmill-api/src/triggers/kafka/mod.rs new file mode 100644 index 0000000000..700b76b056 --- /dev/null +++ b/backend/windmill-api/src/triggers/kafka/mod.rs @@ -0,0 +1,11 @@ +#[cfg(feature = "private")] +mod handler_ee; +pub mod handler_oss; + +#[cfg(feature = "private")] +mod mod_ee; +#[cfg(feature = "private")] +pub use mod_ee::*; + +#[derive(Copy, Clone)] +pub struct KafkaTrigger; diff --git a/backend/windmill-api/src/triggers/mod.rs b/backend/windmill-api/src/triggers/mod.rs new file mode 100644 index 0000000000..2bfe6b0225 --- /dev/null +++ b/backend/windmill-api/src/triggers/mod.rs @@ -0,0 +1,140 @@ +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use sqlx::{types::Json as SqlxJson, FromRow}; +use std::{collections::HashMap, fmt::Debug}; + +#[cfg(all(feature = "gcp_trigger", feature = "enterprise", feature = "private"))] +pub mod gcp; +#[cfg(feature = "http_trigger")] +pub mod http; +#[cfg(all(feature = "kafka", feature = "enterprise", feature = "private"))] +pub mod kafka; +#[cfg(feature = "mqtt_trigger")] +pub mod mqtt; +#[cfg(all(feature = "nats", feature = "enterprise", feature = "private"))] +pub mod nats; +#[cfg(feature = "postgres_trigger")] +pub mod postgres; +#[cfg(all(feature = "sqs_trigger", feature = "enterprise", feature = "private"))] +pub mod sqs; +#[cfg(feature = "websocket")] +pub mod websocket; + +mod handler; +pub mod trigger_helpers; + +#[allow(unused)] +pub(crate) use handler::TriggerCrud; +pub use handler::{generate_trigger_routers, get_triggers_count_internal, TriggersCount}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StandardTriggerQuery { + pub page: Option, + pub per_page: Option, + pub path: Option, + pub is_flow: Option, + pub path_start: Option, +} + +#[derive(Debug, FromRow, Clone, Serialize, Deserialize)] +pub struct BaseTrigger { + pub workspace_id: String, + pub path: String, + pub script_path: String, + pub is_flow: bool, + pub edited_by: String, + pub email: String, + pub edited_at: DateTime, + pub extra_perms: Option, +} + +#[derive(Debug, FromRow, Clone, Serialize, Deserialize)] +pub struct ServerState { + pub enabled: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub server_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub last_server_ping: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +#[derive(Debug, FromRow, Clone, Serialize, Deserialize)] +pub struct TriggerErrorHandling { + #[serde(skip_serializing_if = "Option::is_none")] + pub error_handler_path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub error_handler_args: Option>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub retry: Option>, +} + +#[derive(Serialize, Deserialize, Clone)] +pub struct Trigger +where + T: for<'r> FromRow<'r, sqlx::postgres::PgRow>, +{ + #[serde(flatten)] + pub base: BaseTrigger, + + #[serde(flatten)] + pub config: T, + + #[serde(flatten)] + pub server_state: Option, + + #[serde(flatten)] + pub error_handling: TriggerErrorHandling, +} + +impl FromRow<'_, sqlx::postgres::PgRow> for Trigger +where + T: for<'r> FromRow<'r, sqlx::postgres::PgRow>, +{ + fn from_row(row: &sqlx::postgres::PgRow) -> std::result::Result { + Ok(Trigger { + base: BaseTrigger::from_row(row)?, + config: T::from_row(row)?, + server_state: ServerState::from_row(row).ok(), + error_handling: TriggerErrorHandling::from_row(row)?, + }) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BaseTriggerData { + pub path: String, + pub script_path: String, + pub is_flow: bool, + pub enabled: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TriggerData { + #[serde(flatten)] + pub base: BaseTriggerData, + + #[serde(flatten)] + pub config: T, + + #[serde(flatten)] + pub error_handling: TriggerErrorHandling, +} + +impl StandardTriggerQuery { + pub fn offset(&self) -> i64 { + let page = self.page.unwrap_or(0); + let per_page = self.per_page.unwrap_or(100); + (page * per_page) as i64 + } + + pub fn limit(&self) -> i64 { + self.per_page.unwrap_or(100) as i64 + } +} + +impl Default for StandardTriggerQuery { + fn default() -> Self { + Self { page: Some(0), per_page: Some(100), path: None, path_start: None, is_flow: None } + } +} diff --git a/backend/windmill-api/src/triggers/mqtt/handler.rs b/backend/windmill-api/src/triggers/mqtt/handler.rs new file mode 100644 index 0000000000..d8e314a836 --- /dev/null +++ b/backend/windmill-api/src/triggers/mqtt/handler.rs @@ -0,0 +1,238 @@ +use crate::{ + db::{ApiAuthed, DB}, + resources::try_get_resource_from_db_as, + triggers::{Trigger, TriggerCrud, TriggerData}, +}; +use axum::async_trait; +use itertools::Itertools; +use sqlx::{types::Json as SqlxJson, PgConnection}; +use windmill_common::{ + db::UserDB, + error::{Error, Result}, +}; +use windmill_git_sync::DeployedObject; + +use super::{ + MqttClientBuilder, MqttClientVersion, MqttConfig, MqttConfigRequest, MqttResource, MqttTrigger, + MqttV3Config, MqttV5Config, SubscribeTopic, TestMqttConfig, +}; + +#[async_trait] +impl TriggerCrud for MqttTrigger { + type TriggerConfig = MqttConfig; + type Trigger = Trigger; + type TriggerConfigRequest = MqttConfigRequest; + type TestConnectionConfig = TestMqttConfig; + + const TABLE_NAME: &'static str = "mqtt_trigger"; + const TRIGGER_TYPE: &'static str = "mqtt"; + const SUPPORTS_ENABLED: bool = true; + const SUPPORTS_SERVER_STATE: bool = true; + const SUPPORTS_TEST_CONNECTION: bool = true; + const ROUTE_PREFIX: &'static str = "/mqtt_triggers"; + const DEPLOYMENT_NAME: &'static str = "MQTT trigger"; + const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[ + "mqtt_resource_path", + "subscribe_topics", + "v3_config", + "v5_config", + "client_id", + "client_version", + ]; + const IS_ALLOWED_ON_CLOUD: bool = false; + + fn get_deployed_object(path: String) -> DeployedObject { + DeployedObject::MqttTrigger { path } + } + + async fn validate_config( + &self, + _db: &DB, + config: &Self::TriggerConfigRequest, + _workspace_id: &str, + ) -> Result<()> { + if config.mqtt_resource_path.trim().is_empty() { + return Err(Error::BadRequest( + "MQTT resource path cannot be empty".to_string(), + )); + } + + if config.subscribe_topics.is_empty() { + return Err(Error::BadRequest( + "At least one subscribe topic must be specified".to_string(), + )); + } + + Ok(()) + } + + async fn create_trigger( + &self, + _db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + w_id: &str, + trigger: TriggerData, + ) -> Result<()> { + let subscribe_topics = trigger + .config + .subscribe_topics + .into_iter() + .map(SqlxJson) + .collect_vec(); + let v3_config = trigger.config.v3_config.map(SqlxJson); + let v5_config = trigger.config.v5_config.map(SqlxJson); + + sqlx::query!( + r#" + INSERT INTO mqtt_trigger ( + mqtt_resource_path, + subscribe_topics, + client_version, + client_id, + v3_config, + v5_config, + workspace_id, + path, + script_path, + is_flow, + email, + enabled, + edited_by, + error_handler_path, + error_handler_args, + retry + ) + VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16 + )"#, + trigger.config.mqtt_resource_path, + subscribe_topics.as_slice() as &[SqlxJson], + trigger.config.client_version as Option, + trigger.config.client_id, + v3_config as Option>, + v5_config as Option>, + w_id, + trigger.base.path, + trigger.base.script_path, + trigger.base.is_flow, + authed.email, + trigger.base.enabled.unwrap_or(true), + authed.username, + trigger.error_handling.error_handler_path, + trigger.error_handling.error_handler_args as _, + trigger.error_handling.retry as _ + ) + .execute(tx) + .await?; + + Ok(()) + } + + async fn update_trigger( + &self, + _db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + workspace_id: &str, + path: &str, + trigger: TriggerData, + ) -> Result<()> { + let subscribe_topics = trigger + .config + .subscribe_topics + .into_iter() + .map(SqlxJson) + .collect_vec(); + let v3_config = trigger.config.v3_config.map(SqlxJson); + let v5_config = trigger.config.v5_config.map(SqlxJson); + + // Important to set server_id to NULL to stop current mqtt listener + sqlx::query!( + r#" + UPDATE + mqtt_trigger + SET + mqtt_resource_path = $1, + subscribe_topics = $2, + client_version = $3, + client_id = $4, + v3_config = $5, + v5_config = $6, + is_flow = $7, + edited_by = $8, + email = $9, + script_path = $10, + path = $11, + edited_at = now(), + error = NULL, + server_id = NULL, + error_handler_path = $14, + error_handler_args = $15, + retry = $16 + WHERE + workspace_id = $12 AND + path = $13 + "#, + trigger.config.mqtt_resource_path, + subscribe_topics.as_slice() as &[SqlxJson], + trigger.config.client_version as Option, + trigger.config.client_id, + v3_config as Option>, + v5_config as Option>, + trigger.base.is_flow, + authed.username, + authed.email, + trigger.base.script_path, + trigger.base.path, + workspace_id, + path, + trigger.error_handling.error_handler_path, + trigger.error_handling.error_handler_args as _, + trigger.error_handling.retry as _ + ) + .execute(tx) + .await?; + + Ok(()) + } + + async fn test_connection( + &self, + db: &DB, + authed: &ApiAuthed, + user_db: &UserDB, + workspace_id: &str, + config: Self::TestConnectionConfig, + ) -> Result<()> { + let mqtt_resource = try_get_resource_from_db_as::( + authed, + Some(user_db.clone()), + db, + &config.mqtt_resource_path, + workspace_id, + ) + .await?; + + let connect_f = async { + let client_builder = MqttClientBuilder::new( + mqtt_resource, + Some(""), + vec![], + config.v3_config.as_ref(), + config.v5_config.as_ref(), + config.client_version.as_ref(), + ); + + client_builder.build_client().await.map_err(|err| { + Error::BadConfig(format!( + "Error connecting to mqtt broker: {}", + err.to_string() + )) + }) + }; + + connect_f.await?; + Ok(()) + } +} diff --git a/backend/windmill-api/src/triggers/mqtt/mod.rs b/backend/windmill-api/src/triggers/mqtt/mod.rs new file mode 100644 index 0000000000..1ac287b82a --- /dev/null +++ b/backend/windmill-api/src/triggers/mqtt/mod.rs @@ -0,0 +1,346 @@ +use base64::{engine, prelude::*}; +use itertools::Itertools; +use rumqttc::{ + v5::{ + mqttbytes::{ + v5::{ConnectProperties, Filter}, + QoS as V5QoS, + }, + AsyncClient as V5AsyncClient, EventLoop as V5EventLoop, MqttOptions as V5MqttOptions, + }, + AsyncClient as V3AsyncClient, EventLoop as V3EventLoop, MqttOptions as V3MqttOptions, + QoS as V3QoS, SubscribeFilter, TlsConfiguration, Transport, +}; +use crate::mqtt_triggers::EventLoop; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use sqlx::{types::Json as SqlxJson, FromRow, Type}; +use std::{collections::HashMap, time::Duration}; +use windmill_common::{ + error::{to_anyhow, Error}, + triggers::TriggerKind, + worker::to_raw_value, +}; + +use crate::triggers::{ trigger_helpers::TriggerJobArgs}; + +pub mod handler; + +#[derive(Clone, Copy)] +pub struct MqttTrigger; + +impl TriggerJobArgs for MqttTrigger { + type Payload = Vec; + const TRIGGER_KIND: TriggerKind = TriggerKind::Mqtt; + + fn v1_payload_fn(payload: &Self::Payload) -> HashMap> { + HashMap::from([("payload".to_string(), to_raw_value(&payload))]) + } + + fn v2_payload_fn(payload: &Self::Payload) -> HashMap> { + let base64_payload = engine::general_purpose::STANDARD.encode(payload); + HashMap::from([("payload".to_string(), to_raw_value(&base64_payload))]) + } +} + +#[derive(Clone, Debug, Deserialize, Serialize, Type)] +#[serde(rename_all = "lowercase")] +pub enum QualityOfService { + Qos0, + Qos1, + Qos2, +} + +impl From for V3QoS { + fn from(value: QualityOfService) -> Self { + match value { + QualityOfService::Qos0 => V3QoS::AtMostOnce, + QualityOfService::Qos1 => V3QoS::AtLeastOnce, + QualityOfService::Qos2 => V3QoS::ExactlyOnce, + } + } +} + +impl From for V5QoS { + fn from(value: QualityOfService) -> Self { + match value { + QualityOfService::Qos0 => V5QoS::AtMostOnce, + QualityOfService::Qos1 => V5QoS::AtLeastOnce, + QualityOfService::Qos2 => V5QoS::ExactlyOnce, + } + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct MqttV3Config { + clean_session: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct MqttV5Config { + clean_start: Option, + session_expiry_interval: Option, + topic_alias_maximum: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize, Type)] +#[sqlx(type_name = "MQTT_CLIENT_VERSION")] +#[sqlx(rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum MqttClientVersion { + V3, + V5, +} + +#[derive(Debug, Deserialize)] +pub struct Tls { + enabled: bool, + ca_certificate: String, + pkcs12_client_certificate: Option, + pkcs12_certificate_password: Option, +} + +#[derive(Debug, Deserialize)] +pub struct Credentials { + username: Option, + password: Option, +} + +#[derive(Debug, Deserialize)] +pub struct MqttResource { + broker: String, + port: u16, + credentials: Option, + tls: Option, +} + +#[derive(Clone, Debug, FromRow, Serialize, Deserialize)] +pub struct SubscribeTopic { + qos: QualityOfService, + topic: String, +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct MqttConfig { + pub mqtt_resource_path: String, + pub subscribe_topics: Vec>, + pub v3_config: Option>, + pub v5_config: Option>, + pub client_id: Option, + pub client_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MqttConfigRequest { + pub mqtt_resource_path: String, + pub subscribe_topics: Vec, + pub v3_config: Option, + pub v5_config: Option, + pub client_id: Option, + pub client_version: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TestMqttConfig { + pub mqtt_resource_path: String, + pub client_version: Option, + pub v3_config: Option, + pub v5_config: Option, +} + +// Constants +pub const KEEP_ALIVE: u64 = 60; +pub const CLIENT_CONNECTION_TIMEOUT: u64 = 60; +pub const TOPIC_ALIAS_MAXIMUM: u16 = 65535; +pub const TIMEOUT_DURATION: u64 = 10; +pub const CONNECTION_TIMEOUT: Duration = Duration::from_secs(TIMEOUT_DURATION); + +pub struct V3MqttHandler; +pub struct V5MqttHandler; + +pub enum MqttClientResult { + V3((V3MqttHandler, V3EventLoop)), + V5((V5MqttHandler, V5EventLoop)), +} + +#[derive(Debug, thiserror::Error)] +pub enum MqttError { + #[error("{0}")] + Common(#[from] Error), + #[error("{0}")] + V5RumqttClient(#[from] rumqttc::v5::ClientError), + #[error("{0}")] + V5ConnectionError(#[from] rumqttc::v5::ConnectionError), + #[error("{0}")] + V3RumqttClient(#[from] rumqttc::ClientError), + #[error("{0}")] + V3ConnectionError(#[from] rumqttc::ConnectionError), + #[error("{0}")] + Base64Decode(#[from] base64::DecodeError), +} + +pub struct MqttClientBuilder<'client> { + mqtt_resource: MqttResource, + client_id: &'client str, + subscribe_topics: Vec, + v3_config: Option<&'client MqttV3Config>, + v5_config: Option<&'client MqttV5Config>, + mqtt_client_version: Option<&'client MqttClientVersion>, +} + +impl<'client> MqttClientBuilder<'client> { + pub fn new( + mqtt_resource: MqttResource, + client_id: Option<&'client str>, + subscribe_topics: Vec, + v3_config: Option<&'client MqttV3Config>, + v5_config: Option<&'client MqttV5Config>, + mqtt_client_version: Option<&'client MqttClientVersion>, + ) -> Self { + Self { + mqtt_resource, + client_id: client_id.unwrap_or(""), + subscribe_topics, + v3_config, + v5_config, + mqtt_client_version, + } + } + + pub async fn build_client(&self) -> Result { + match self.mqtt_client_version { + Some(MqttClientVersion::V5) | None => self.build_v5_client().await, + Some(MqttClientVersion::V3) => self.build_v3_client().await, + } + } + + fn get_tls_configuration(&self) -> Result, Error> { + let transport = match self.mqtt_resource.tls { + Some(ref tls) if tls.enabled => { + let transport = match tls.ca_certificate.trim().is_empty() { + true => rumqttc::Transport::Tls(TlsConfiguration::Native), + false => rumqttc::Transport::Tls(TlsConfiguration::SimpleNative { + ca: tls.ca_certificate.as_bytes().to_vec(), + client_auth: { + match tls.pkcs12_client_certificate.as_ref() { + Some(client_certificate) + if !client_certificate.trim().is_empty() => + { + let client_certificate = BASE64_STANDARD + .decode(client_certificate) + .map_err(to_anyhow)?; + let password = tls + .pkcs12_certificate_password + .clone() + .unwrap_or("".to_string()); + Some((client_certificate, password)) + } + _ => None, + } + }, + }), + }; + + Some(transport) + } + _ => None, + }; + + Ok(transport) + } + + async fn build_v5_client(&self) -> Result { + let mut mqtt_options = V5MqttOptions::new( + self.client_id, + &self.mqtt_resource.broker, + self.mqtt_resource.port, + ); + + if let Some(credentials) = &self.mqtt_resource.credentials { + let username = credentials.username.as_deref().unwrap_or(""); + let password = credentials.password.as_deref().unwrap_or(""); + mqtt_options.set_credentials(username, password); + } + + if let Some(transport) = self.get_tls_configuration()? { + mqtt_options.set_transport(transport); + } + + mqtt_options.set_connection_timeout(CLIENT_CONNECTION_TIMEOUT); + + mqtt_options.set_keep_alive(Duration::from_secs(KEEP_ALIVE)); + + if let Some(v5_config) = self.v5_config { + mqtt_options.set_clean_start(v5_config.clean_start.unwrap_or(true)); + mqtt_options.set_connect_properties(ConnectProperties { + session_expiry_interval: v5_config.session_expiry_interval, + receive_maximum: None, + max_packet_size: None, + topic_alias_max: v5_config.topic_alias_maximum.or(Some(TOPIC_ALIAS_MAXIMUM)), + request_response_info: None, + request_problem_info: None, + user_properties: vec![], + authentication_method: None, + authentication_data: None, + }); + } + + let (async_client, mut event_loop) = + V5AsyncClient::new(mqtt_options, self.subscribe_topics.len()); + event_loop.verify_connection().await?; + + if !self.subscribe_topics.is_empty() { + let subscribe_filters = self + .subscribe_topics + .iter() + .map(|topic| Filter::new(topic.topic.clone(), topic.qos.clone().into())) + .collect_vec(); + + async_client + .subscribe_many(subscribe_filters) + .await + .map_err(to_anyhow)?; + } + Ok(MqttClientResult::V5((V5MqttHandler, event_loop))) + } + + async fn build_v3_client(&self) -> Result { + let mut mqtt_options = V3MqttOptions::new( + self.client_id, + &self.mqtt_resource.broker, + self.mqtt_resource.port, + ); + + if let Some(credentials) = &self.mqtt_resource.credentials { + let username = credentials.username.as_deref().unwrap_or(""); + let password = credentials.password.as_deref().unwrap_or(""); + mqtt_options.set_credentials(username, password); + } + + if let Some(transport) = self.get_tls_configuration()? { + mqtt_options.set_transport(transport); + } + mqtt_options.set_keep_alive(Duration::from_secs(KEEP_ALIVE)); + if let Some(v3_config) = self.v3_config { + mqtt_options.set_clean_session(v3_config.clean_session.unwrap_or(true)); + } + + let (async_client, mut event_loop) = + V3AsyncClient::new(mqtt_options, self.subscribe_topics.len()); + event_loop.verify_connection().await?; + + if !self.subscribe_topics.is_empty() { + let subscribe_filters = self + .subscribe_topics + .iter() + .map(|topic| SubscribeFilter::new(topic.topic.clone(), topic.qos.clone().into())) + .collect_vec(); + + async_client + .subscribe_many(subscribe_filters) + .await + .map_err(to_anyhow)?; + } + Ok(MqttClientResult::V3((V3MqttHandler, event_loop))) + } +} diff --git a/backend/windmill-api/src/triggers/nats/handler_oss.rs b/backend/windmill-api/src/triggers/nats/handler_oss.rs new file mode 100644 index 0000000000..6b03dafef0 --- /dev/null +++ b/backend/windmill-api/src/triggers/nats/handler_oss.rs @@ -0,0 +1,65 @@ +#[allow(unused)] +#[cfg(feature = "private")] +pub use super::handler_ee::*; + +#[cfg(not(feature = "private"))] +use { + super::NatsTrigger, + crate::{ + db::{ApiAuthed, DB}, + triggers::{TriggerCrud, TriggerData}, + }, + axum::async_trait, + sqlx::PgConnection, + windmill_common::error::{Error, Result}, + windmill_git_sync::DeployedObject, +}; + +#[cfg(not(feature = "private"))] +#[async_trait] +impl TriggerCrud for NatsTrigger { + type Trigger = (); + type TriggerConfig = (); + type TriggerConfigRequest = (); + type TestConnectionConfig = (); + + const TABLE_NAME: &'static str = ""; + const TRIGGER_TYPE: &'static str = ""; + const SUPPORTS_ENABLED: bool = false; + const SUPPORTS_SERVER_STATE: bool = false; + const SUPPORTS_TEST_CONNECTION: bool = false; + const ROUTE_PREFIX: &'static str = "/nats_triggers"; + const DEPLOYMENT_NAME: &'static str = ""; + const IS_CLOUD_HOSTED: bool = false; + + fn get_deployed_object(path: String) -> DeployedObject { + DeployedObject::NatsTrigger { path } + } + + async fn create_trigger( + &self, + _db: &DB, + _executor: &mut PgConnection, + _authed: &ApiAuthed, + _w_id: &str, + _trigger: TriggerData, + ) -> Result<()> { + Err(Error::BadRequest( + "NATS triggers are not available in open source version".to_string(), + )) + } + + async fn update_trigger( + &self, + _db: &DB, + _executor: &mut PgConnection, + _authed: &ApiAuthed, + _workspace_id: &str, + _path: &str, + _trigger: TriggerData, + ) -> Result<()> { + Err(Error::BadRequest( + "NATS triggers are not available in open source version".to_string(), + )) + } +} diff --git a/backend/windmill-api/src/triggers/nats/mod.rs b/backend/windmill-api/src/triggers/nats/mod.rs new file mode 100644 index 0000000000..b69f2eb3c8 --- /dev/null +++ b/backend/windmill-api/src/triggers/nats/mod.rs @@ -0,0 +1,11 @@ +#[cfg(feature = "private")] +mod handler_ee; +pub mod handler_oss; + +#[cfg(feature = "private")] +mod mod_ee; +#[cfg(feature = "private")] +pub use mod_ee::*; + +#[derive(Copy, Clone)] +pub struct NatsTrigger; diff --git a/backend/windmill-api/src/postgres_triggers/handler.rs b/backend/windmill-api/src/triggers/postgres/handler.rs similarity index 50% rename from backend/windmill-api/src/postgres_triggers/handler.rs rename to backend/windmill-api/src/triggers/postgres/handler.rs index 71d53dcf8b..5dce0345c6 100644 --- a/backend/windmill-api/src/postgres_triggers/handler.rs +++ b/backend/windmill-api/src/triggers/postgres/handler.rs @@ -1,284 +1,319 @@ -use std::collections::{ - hash_map::Entry::{Occupied, Vacant}, - HashMap, -}; +use std::collections::HashMap; -use crate::{ - db::{ApiAuthed, DB}, - postgres_triggers::mapper::{Mapper, MappingInfo}, - utils::check_scopes, -}; use axum::{ - extract::{Path, Query}, - Extension, Json, + async_trait, + extract::Path, + routing::{delete, get, post}, + Extension, Json, Router, }; -use http::StatusCode; use itertools::Itertools; +use lazy_static::lazy_static; use pg_escape::{quote_identifier, quote_literal}; use quick_cache::sync::Cache; use rust_postgres::{types::Type, Client}; -use serde::{Deserialize, Deserializer, Serialize}; -use serde_json::value::RawValue; -use sql_builder::{bind::Bind, SqlBuilder}; -use sqlx::FromRow; -use windmill_audit::{audit_oss::audit_log, ActionKind}; +use sqlx::PgConnection; +use uuid; use windmill_common::{ db::UserDB, - error::{self, to_anyhow, Error, JsonResult, Result}, - utils::{empty_as_none, not_found_if_none, paginate, Pagination, StripPath}, - worker::CLOUD_HOSTED, + error::{self, to_anyhow, Error, Result}, +}; +use windmill_git_sync::DeployedObject; + +use crate::{ + db::{ApiAuthed, DB}, + triggers::{postgres::PostgresTrigger, Trigger, TriggerCrud, TriggerData}, }; -use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; use super::{ check_if_valid_publication_for_postgres_version, create_logical_replication_slot, create_pg_publication, drop_publication, generate_random_string, get_default_pg_connection, + mapper::{Mapper, MappingInfo}, + PostgresConfig, PostgresConfigRequest, PostgresPublicationReplication, PublicationData, + Relations, Slot, SlotList, TableToTrack, TemplateScript, TestPostgresConfig, ERROR_PUBLICATION_NAME_NOT_EXISTS, }; -use anyhow::anyhow; -use lazy_static::lazy_static; - -#[derive(FromRow, Serialize, Deserialize, Debug)] -pub struct Postgres { - pub user: String, - pub password: String, - pub host: String, - pub port: Option, - pub dbname: String, - #[serde(default)] - pub sslmode: String, - #[serde(default, deserialize_with = "empty_as_none")] - pub root_certificate_pem: Option, -} - -#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] -pub struct TableToTrack { - pub table_name: String, - #[serde(default, deserialize_with = "empty_as_none")] - pub where_clause: Option, - #[serde(default, deserialize_with = "empty_as_none")] - pub columns_name: Option>, -} - -impl TableToTrack { - fn new( - table_name: String, - where_clause: Option, - columns_name: Option>, - ) -> TableToTrack { - TableToTrack { table_name, where_clause, columns_name } - } -} +// Lazy static template cache lazy_static! { pub static ref TEMPLATE: Cache = Cache::new(50); } -#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] -pub struct Relations { - pub schema_name: String, - pub table_to_track: Vec, -} +#[async_trait] +impl TriggerCrud for PostgresTrigger { + type TriggerConfig = PostgresConfig; + type Trigger = Trigger; + type TriggerConfigRequest = PostgresConfigRequest; + type TestConnectionConfig = TestPostgresConfig; -impl Relations { - fn new(schema_name: String, table_to_track: Vec) -> Relations { - Relations { schema_name, table_to_track } + const TABLE_NAME: &'static str = "postgres_trigger"; + const TRIGGER_TYPE: &'static str = "postgres"; + const SUPPORTS_ENABLED: bool = true; + const SUPPORTS_SERVER_STATE: bool = true; + const SUPPORTS_TEST_CONNECTION: bool = true; + const ROUTE_PREFIX: &'static str = "/postgres_triggers"; + const DEPLOYMENT_NAME: &'static str = "PostgreSQL trigger"; + const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[ + "postgres_resource_path", + "replication_slot_name", + "publication_name", + "NULL::text AS basic_mode", + ]; + const IS_ALLOWED_ON_CLOUD: bool = false; + + fn get_deployed_object(path: String) -> DeployedObject { + DeployedObject::PostgresTrigger { path } } - fn add_new_table(&mut self, table_to_track: TableToTrack) { - self.table_to_track.push(table_to_track); - } -} -#[derive(Debug, Deserialize)] -pub struct EditPostgresTrigger { - replication_slot_name: String, - publication_name: String, - path: String, - script_path: String, - is_flow: bool, - postgres_resource_path: String, - publication: Option, - error_handler_path: Option, - error_handler_args: Option>>>, - retry: Option>, -} + async fn create_trigger( + &self, + db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + w_id: &str, + trigger: TriggerData, + ) -> Result<()> { + let Self::TriggerConfigRequest { + postgres_resource_path, + publication_name, + replication_slot_name, + publication, + } = trigger.config; -#[derive(Deserialize, Serialize, Debug)] -pub struct NewPostgresTrigger { - path: String, - script_path: String, - is_flow: bool, - enabled: bool, - postgres_resource_path: String, - replication_slot_name: Option, - publication_name: Option, - publication: Option, - error_handler_path: Option, - error_handler_args: Option>>>, - retry: Option>, -} + let (pub_name, slot_name) = + if publication_name.is_empty() && replication_slot_name.is_empty() { + if publication.is_none() { + return Err(Error::BadRequest("publication must be set".to_string())); + } -#[derive(Serialize, Deserialize)] -pub struct TestPostgres { - pub postgres_resource_path: String, -} + let PostgresPublicationReplication { publication_name, replication_slot_name } = + create_custom_slot_and_publication_inner( + authed.clone(), + UserDB::new(db.clone()), + &db, + &postgres_resource_path, + &w_id, + &publication.unwrap(), + ) + .await?; -pub async fn test_postgres_connection( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path(workspace_id): Path, - Json(test_postgres): Json, -) -> Result<()> { - let connect_f = async { - get_default_pg_connection( - authed, - Some(user_db), - &db, - &test_postgres.postgres_resource_path, - &workspace_id, - ) - .await - .map_err(|err| { - error::Error::BadConfig(format!("Error connecting to postgres: {}", err.to_string())) - }) - }; - tokio::time::timeout(tokio::time::Duration::from_secs(30), connect_f) - .await - .map_err(|_| { - error::Error::BadConfig(format!("Timeout connecting to postgres after 30 seconds")) - })??; - - Ok(()) -} - -#[derive(Deserialize, Debug)] -pub enum Language { - #[serde(rename = "typescript", alias = "Typescript")] - Typescript, -} - -#[derive(Debug, Deserialize)] -pub struct TemplateScript { - postgres_resource_path: String, - #[serde(deserialize_with = "check_if_valid_relation")] - relations: Option>, - language: Language, -} - -fn check_if_valid_relation<'de, D>( - relations: D, -) -> std::result::Result>, D::Error> -where - D: Deserializer<'de>, -{ - let relations: Option> = Option::deserialize(relations)?; - let mut track_all_table_in_schema = false; - let mut track_specific_columns_in_table = false; - match relations { - Some(relations) => { - for relation in relations.iter() { - if relation.schema_name.is_empty() { - return Err(serde::de::Error::custom( - "Schema Name must not be empty".to_string(), + (publication_name, replication_slot_name) + } else { + if publication_name.is_empty() { + return Err(Error::BadRequest( + "Publication name must not be empty".to_string(), + )); + } else if replication_slot_name.is_empty() { + return Err(Error::BadRequest( + "Replication slot name must not be empty".to_string(), )); } + (publication_name, replication_slot_name) + }; - if !track_all_table_in_schema && relation.table_to_track.is_empty() { - track_all_table_in_schema = true; - continue; - } + sqlx::query!( + r#" + INSERT INTO postgres_trigger ( + workspace_id, + path, + postgres_resource_path, + replication_slot_name, + publication_name, + script_path, + is_flow, + enabled, + edited_by, + email, + edited_at, + error_handler_path, + error_handler_args, + retry + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, now(), $11, $12, $13 + ) + "#, + w_id, + trigger.base.path, + postgres_resource_path, + slot_name, + pub_name, + trigger.base.script_path, + trigger.base.is_flow, + trigger.base.enabled.unwrap_or(true), + authed.username, + authed.email, + trigger.error_handling.error_handler_path, + trigger.error_handling.error_handler_args as _, + trigger.error_handling.retry as _ + ) + .execute(tx) + .await?; + Ok(()) + } - for table_to_track in relation.table_to_track.iter() { - if table_to_track.table_name.trim().is_empty() { - return Err(serde::de::Error::custom( - "Table name must not be empty".to_string(), - )); - } + async fn update_trigger( + &self, + db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + w_id: &str, + path: &str, + trigger: TriggerData, + ) -> Result<()> { + let Self::TriggerConfigRequest { + replication_slot_name, + publication_name, + postgres_resource_path, + publication, + } = trigger.config; - if !track_specific_columns_in_table && table_to_track.columns_name.is_some() { - track_specific_columns_in_table = true; - } - } + let mut pg_connection = get_default_pg_connection( + authed.clone(), + Some(UserDB::new(db.clone())), + db, + &postgres_resource_path, + &w_id, + ) + .await + .map_err(to_anyhow)?; - if track_all_table_in_schema && track_specific_columns_in_table { - return Err(serde::de::Error::custom("Incompatible tracking options. Schema-level tracking and specific table tracking with column selection cannot be used together. Refer to the documentation for valid configurations.")); - } - } + let exists = + check_if_logical_replication_slot_exist(&mut pg_connection, &replication_slot_name) + .await?; - if !relations - .iter() - .map(|relation| relation.schema_name.as_str()) - .all_unique() - { - return Err(serde::de::Error::custom( - "You cannot choose a schema more than one time".to_string(), - )); - } + let remote_db_tx = pg_connection.transaction().await.map_err(to_anyhow)?; - Ok(Some(relations)) + if !exists { + tracing::debug!( + "Logical replication slot named: {} does not exists creating it...", + &replication_slot_name + ); + create_logical_replication_slot(remote_db_tx.client(), &replication_slot_name) + .await + .map_err(to_anyhow)?; } - None => Ok(None), + + if let Some(publication) = publication { + let publication_data = + get_publication_scope_and_transaction(remote_db_tx.client(), &publication_name) + .await + .map_err(to_anyhow)?; + + update_pg_publication( + remote_db_tx.client(), + &publication_name, + publication, + publication_data.map(|publication| publication.0), + ) + .await + .map_err(to_anyhow)?; + } + + remote_db_tx.commit().await.map_err(to_anyhow)?; + + sqlx::query!( + r#" + UPDATE postgres_trigger + SET + postgres_resource_path = $1, + replication_slot_name = $2, + publication_name = $3, + script_path = $4, + path = $5, + is_flow = $6, + edited_by = $7, + email = $8, + edited_at = now(), + server_id = NULL, + error = NULL, + error_handler_path = $11, + error_handler_args = $12, + retry = $13 + WHERE + workspace_id = $9 AND path = $10 + "#, + postgres_resource_path, + replication_slot_name, + publication_name, + trigger.base.script_path, + trigger.base.path, + trigger.base.is_flow, + authed.username, + authed.email, + w_id, + path, + trigger.error_handling.error_handler_path, + trigger.error_handling.error_handler_args as _, + trigger.error_handling.retry as _ + ) + .execute(tx) + .await?; + Ok(()) + } + + async fn test_connection( + &self, + db: &DB, + authed: &ApiAuthed, + user_db: &UserDB, + workspace_id: &str, + config: Self::TestConnectionConfig, + ) -> Result<()> { + let connect_f = async { + get_default_pg_connection( + authed.clone(), + Some(user_db.clone()), + db, + &config.postgres_resource_path, + workspace_id, + ) + .await + .map_err(|err| { + Error::BadConfig(format!("Error connecting to postgres: {}", err.to_string())) + })?; + + Ok::<(), Error>(()) + }; + + connect_f.await?; + Ok(()) + } + + fn additional_routes(&self) -> Router { + Router::new() + .route("/get_template_script/:id", get(get_template_script)) + .route("/create_template_script", post(create_template_script)) + .route( + "/is_valid_postgres_configuration/*path", + get(is_database_in_logical_level), + ) + .nest("/publication", publication_service()) + .nest("/slot", slot_service()) + .nest("/postgres", postgres_service()) } } -#[derive(FromRow, Deserialize, Serialize, Debug)] -pub struct PostgresTrigger { - pub path: String, - pub script_path: String, - pub is_flow: bool, - pub workspace_id: String, - pub edited_by: String, - pub email: String, - pub edited_at: chrono::DateTime, - #[serde(skip_serializing_if = "Option::is_none")] - pub extra_perms: Option, - pub postgres_resource_path: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub server_id: Option, - pub replication_slot_name: String, - pub publication_name: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub last_server_ping: Option>, - pub enabled: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_args: Option>>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub retry: Option>, +fn publication_service() -> Router { + Router::new() + .route("/get/:publication_name/*path", get(get_publication_info)) + .route("/create/:publication_name/*path", post(create_publication)) + .route("/update/:publication_name/*path", post(alter_publication)) + .route( + "/delete/:publication_name/*path", + delete(delete_publication), + ) + .route("/list/*path", get(list_database_publication)) } -#[derive(Deserialize, Serialize)] -pub struct ListPostgresTriggerQuery { - pub page: Option, - pub per_page: Option, - pub path: Option, - pub is_flow: Option, - pub path_start: Option, +fn slot_service() -> Router { + Router::new() + .route("/list/*path", get(list_slot_name)) + .route("/create/*path", post(create_slot)) + .route("/delete/*path", delete(drop_slot_name)) } -#[derive(Deserialize)] -pub struct SetEnabled { - pub enabled: bool, -} - -#[derive(Serialize, Deserialize)] -pub struct PostgresPublicationReplication { - publication_name: String, - replication_slot_name: String, -} - -impl PostgresPublicationReplication { - pub fn new( - publication_name: String, - replication_slot_name: String, - ) -> PostgresPublicationReplication { - PostgresPublicationReplication { publication_name, replication_slot_name } - } +fn postgres_service() -> Router { + Router::new().route("/version/*path", get(get_postgres_version)) } async fn check_if_logical_replication_slot_exist( @@ -367,264 +402,6 @@ pub async fn get_postgres_version( Ok(postgres_version) } -pub async fn create_postgres_trigger( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Path(w_id): Path, - Json(new_postgres_trigger): Json, -) -> Result<(StatusCode, String)> { - check_scopes(&authed, || { - format!("postgres_triggers:write:{}", new_postgres_trigger.path) - })?; - - if *CLOUD_HOSTED { - return Err(error::Error::BadRequest( - "Postgres triggers are not supported on multi-tenant cloud, use dedicated cloud or self-host".to_string(), - )); - } - - let NewPostgresTrigger { - postgres_resource_path, - path, - script_path, - enabled, - is_flow, - publication_name, - replication_slot_name, - publication, - error_handler_path, - error_handler_args, - retry, - } = new_postgres_trigger; - - if publication_name.is_none() && publication.is_none() { - return Err(error::Error::BadRequest( - "Publication data is missing".to_string(), - )); - } - let (pub_name, slot_name) = if publication_name.is_none() && replication_slot_name.is_none() { - if publication.is_none() { - return Err(Error::BadRequest("publication must be set".to_string())); - } - - let PostgresPublicationReplication { publication_name, replication_slot_name } = - create_custom_slot_and_publication_inner( - authed.clone(), - user_db.clone(), - &db, - &postgres_resource_path, - &w_id, - &publication.unwrap(), - ) - .await?; - - (publication_name, replication_slot_name) - } else { - if publication_name.is_none() { - return Err(Error::BadRequest("Missing publication name".to_string())); - } else if replication_slot_name.is_none() { - return Err(Error::BadRequest( - "Missing replication slot name".to_string(), - )); - } - (publication_name.unwrap(), replication_slot_name.unwrap()) - }; - - let mut tx = user_db.begin(&authed).await?; - - sqlx::query!( - r#" - INSERT INTO postgres_trigger ( - publication_name, - replication_slot_name, - workspace_id, - path, - script_path, - is_flow, - email, - enabled, - postgres_resource_path, - edited_by, - error_handler_path, - error_handler_args, - retry - ) - VALUES ( - $1, - $2, - $3, - $4, - $5, - $6, - $7, - $8, - $9, - $10, - $11, - $12, - $13 - )"#, - pub_name, - slot_name, - &w_id, - &path, - script_path, - is_flow, - &authed.email, - enabled, - postgres_resource_path, - &authed.username, - error_handler_path, - error_handler_args as _, - retry as _ - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "postgres_triggers.create", - ActionKind::Create, - &w_id, - Some(path.as_str()), - None, - ) - .await?; - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::PostgresTrigger { path: path.to_string() }, - Some(format!("Postgres trigger '{}' created", path)), - true, - ) - .await?; - - Ok((StatusCode::CREATED, path.to_string())) -} - -pub async fn list_postgres_triggers( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, - Query(lst): Query, -) -> error::JsonResult> { - let mut tx = user_db.begin(&authed).await?; - let (per_page, offset) = paginate(Pagination { per_page: lst.per_page, page: lst.page }); - let mut sqlb = SqlBuilder::select_from("postgres_trigger"); - sqlb.fields(&[ - "workspace_id", - "path", - "script_path", - "is_flow", - "edited_by", - "email", - "edited_at", - "server_id", - "last_server_ping", - "extra_perms", - "error", - "enabled", - "postgres_resource_path", - "replication_slot_name", - "publication_name", - "error_handler_path", - "error_handler_args", - "retry", - ]) - .order_by("edited_at", true) - .and_where("workspace_id = ?".bind(&w_id)) - .offset(offset) - .limit(per_page); - if let Some(path) = lst.path { - sqlb.and_where_eq("script_path", "?".bind(&path)); - } - if let Some(is_flow) = lst.is_flow { - sqlb.and_where_eq("is_flow", "?".bind(&is_flow)); - } - if let Some(path_start) = &lst.path_start { - sqlb.and_where_like_left("path", path_start); - } - let sql = sqlb - .sql() - .map_err(|e| error::Error::InternalErr(e.to_string()))?; - let rows = sqlx::query_as::<_, PostgresTrigger>(&sql) - .fetch_all(&mut *tx) - .await - .map_err(|e| { - tracing::debug!("Error fetching postgres_trigger: {:#?}", e); - windmill_common::error::Error::InternalErr("server error".to_string()) - })?; - tx.commit().await.map_err(|e| { - tracing::debug!("Error commiting postgres_trigger: {:#?}", e); - windmill_common::error::Error::InternalErr("server error".to_string()) - })?; - - Ok(Json(rows)) -} - -#[derive(Deserialize, Serialize, Debug)] -pub struct PublicationData { - #[serde(default, deserialize_with = "check_if_valid_relation")] - pub table_to_track: Option>, - #[serde(deserialize_with = "check_if_valid_transaction_type")] - pub transaction_to_track: Vec, -} - -fn check_if_valid_transaction_type<'de, D>( - transaction_type: D, -) -> std::result::Result, D::Error> -where - D: Deserializer<'de>, -{ - let mut transaction_type: Vec = Vec::deserialize(transaction_type)?; - if transaction_type.len() > 3 { - return Err(serde::de::Error::custom( - "More than 3 transaction type which is not authorized, you are only allowed to those 3 transaction types: Insert, Update and Delete" - .to_string(), - )); - } - transaction_type.sort_unstable(); - transaction_type.dedup(); - - for transaction in transaction_type.iter() { - match transaction.to_lowercase().as_ref() { - "insert" => {}, - "update" => {}, - "delete" => {}, - _ => { - return Err(serde::de::Error::custom( - "Only the following transaction types are allowed: Insert, Update and Delete (case insensitive)" - .to_string(), - )) - } - } - } - - Ok(transaction_type) -} - -impl PublicationData { - fn new( - table_to_track: Option>, - transaction_to_track: Vec, - ) -> PublicationData { - PublicationData { table_to_track, transaction_to_track } - } -} - -#[derive(FromRow, Debug, Serialize)] -pub struct SlotList { - slot_name: Option, - active: Option, -} - pub async fn list_slot_name( authed: ApiAuthed, Extension(user_db): Extension, @@ -666,11 +443,6 @@ pub async fn list_slot_name( Ok(Json(slots)) } -#[derive(Debug, Serialize, Deserialize)] -pub struct Slot { - name: String, -} - pub async fn create_slot( authed: ApiAuthed, Extension(user_db): Extension, @@ -1117,31 +889,32 @@ pub async fn get_tracked_relations( let columns: Option> = row.get("columns"); let where_clause: Option = row.get("where_clause"); - let schema_name = schema_name.ok_or_else::( || { - anyhow!( + let schema_name = schema_name.ok_or_else::(|| { + anyhow::anyhow!( "Unexpected NULL `schema_name` in publication entry (pubname: `{}`). This should never happen unless PostgreSQL internals are corrupted.", publication_name, - ).into() - } - )?; + ) + .into() + })?; let table_name = table_name.ok_or_else::(|| { - anyhow!( + anyhow::anyhow!( "Unexpected NULL `table_name` for schema `{}` in publication `{}`. This should never happen unless PostgreSQL internals are corrupted.", schema_name, publication_name, - ).into() + ) + .into() })?; let entry = table_to_track.entry(schema_name.clone()); - let table_to_track = TableToTrack::new(table_name, where_clause, columns); + let table_to_track_item = TableToTrack::new(table_name, where_clause, columns); match entry { - std::collections::hash_map::Entry::Occupied(mut occuped) => { - occuped.get_mut().add_new_table(table_to_track); + std::collections::hash_map::Entry::Occupied(mut occupied) => { + occupied.get_mut().add_new_table(table_to_track_item); } std::collections::hash_map::Entry::Vacant(vacant) => { - vacant.insert(Relations::new(schema_name, vec![table_to_track])); + vacant.insert(Relations::new(schema_name, vec![table_to_track_item])); } } } @@ -1149,331 +922,6 @@ pub async fn get_tracked_relations( Ok(table_to_track.into_values().collect_vec()) } -pub async fn get_postgres_trigger( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - check_scopes(&authed, || format!("postgres_triggers:read:{}", path))?; - let mut tx = user_db.begin(&authed).await?; - let trigger = sqlx::query_as!( - PostgresTrigger, - r#" - SELECT - workspace_id, - path, - script_path, - is_flow, - edited_by, - email, - edited_at, - server_id, - last_server_ping, - extra_perms, - error, - enabled, - replication_slot_name, - publication_name, - postgres_resource_path, - error_handler_path, - error_handler_args as "error_handler_args: _", - retry as "retry: _" - FROM - postgres_trigger - WHERE - workspace_id = $1 AND - path = $2 - "#, - &w_id, - &path - ) - .fetch_optional(&mut *tx) - .await?; - tx.commit().await?; - - let trigger = not_found_if_none(trigger, "Trigger", path)?; - - Ok(Json(trigger)) -} - -pub async fn update_postgres_trigger( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Json(postgres_trigger): Json, -) -> Result { - let workspace_path = path.to_path(); - check_scopes(&authed, || { - format!("postgres_triggers:write:{}", workspace_path) - })?; - - let EditPostgresTrigger { - replication_slot_name, - publication_name, - script_path, - path, - is_flow, - postgres_resource_path, - publication, - error_handler_path, - error_handler_args, - retry, - } = postgres_trigger; - - let mut pg_connection = get_default_pg_connection( - authed.clone(), - Some(user_db.clone()), - &db, - &postgres_resource_path, - &w_id, - ) - .await - .map_err(to_anyhow)?; - - let exists = - check_if_logical_replication_slot_exist(&mut pg_connection, &replication_slot_name).await?; - - let tx = pg_connection.transaction().await.map_err(to_anyhow)?; - - if !exists { - tracing::debug!( - "Logical replication slot named: {} does not exists creating it...", - &replication_slot_name - ); - create_logical_replication_slot(tx.client(), &replication_slot_name) - .await - .map_err(to_anyhow)?; - } - - if let Some(publication) = publication { - let publication_data = - get_publication_scope_and_transaction(tx.client(), &publication_name) - .await - .map_err(to_anyhow)?; - - update_pg_publication( - tx.client(), - &publication_name, - publication, - publication_data.map(|publication| publication.0), - ) - .await - .map_err(to_anyhow)?; - } - - tx.commit().await.map_err(to_anyhow)?; - - let mut tx = user_db.begin(&authed).await?; - - sqlx::query!( - r#" - UPDATE postgres_trigger - SET - script_path = $1, - path = $2, - is_flow = $3, - edited_by = $4, - email = $5, - postgres_resource_path = $6, - replication_slot_name = $7, - publication_name = $8, - edited_at = now(), - error = NULL, - server_id = NULL, - error_handler_path = $11, - error_handler_args = $12, - retry = $13 - WHERE - workspace_id = $9 AND - path = $10 - "#, - script_path, - path, - is_flow, - &authed.username, - &authed.email, - postgres_resource_path, - replication_slot_name, - publication_name, - w_id, - workspace_path, - error_handler_path, - error_handler_args as _, - retry as _ - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "postgres_triggers.update", - ActionKind::Update, - &w_id, - Some(&path), - None, - ) - .await?; - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::PostgresTrigger { path: path.to_string() }, - Some(format!("Postgres trigger '{}' updated", path)), - true, - ) - .await?; - - Ok(workspace_path.to_string()) -} - -pub async fn delete_postgres_trigger( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> Result { - let path = path.to_path(); - check_scopes(&authed, || format!("postgres_triggers:write:{}", path))?; - let mut tx = user_db.begin(&authed).await?; - sqlx::query!( - r#" - DELETE FROM postgres_trigger - WHERE - workspace_id = $1 AND - path = $2 - "#, - w_id, - path, - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "postgres_triggers.delete", - ActionKind::Delete, - &w_id, - Some(path), - None, - ) - .await?; - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::PostgresTrigger { path: path.to_string() }, - Some(format!("Postgres trigger '{}' deleted", path)), - true, - ) - .await?; - - Ok(format!("Postgres trigger {path} deleted")) -} - -pub async fn exists_postgres_trigger( - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - let exists = sqlx::query_scalar!( - r#" - SELECT EXISTS( - SELECT 1 - FROM postgres_trigger - WHERE - path = $1 AND - workspace_id = $2 - )"#, - path, - w_id, - ) - .fetch_one(&db) - .await? - .unwrap_or(false); - Ok(Json(exists)) -} - -pub async fn set_enabled( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Json(payload): Json, -) -> Result { - let path = path.to_path(); - check_scopes(&authed, || format!("postgres_triggers:write:{}", path))?; - let mut tx = user_db.begin(&authed).await?; - - // important to set server_id, last_server_ping and error to NULL to stop current postgres listener - let one_o = sqlx::query_scalar!( - r#" - UPDATE postgres_trigger - SET - enabled = $1, - email = $2, - edited_by = $3, - edited_at = now(), - server_id = NULL, - error = NULL - WHERE - path = $4 AND - workspace_id = $5 - RETURNING 1 - "#, - payload.enabled, - &authed.email, - &authed.username, - path, - w_id, - ) - .fetch_optional(&mut *tx) - .await? - .flatten(); - - not_found_if_none(one_o, "Postgres trigger", path)?; - - audit_log( - &mut *tx, - &authed, - "postgres_triggers.setenabled", - ActionKind::Update, - &w_id, - Some(path), - Some([("enabled", payload.enabled.to_string().as_ref())].into()), - ) - .await?; - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - DeployedObject::PostgresTrigger { path: path.to_string() }, - Some(format!("Postgres trigger '{}' updated", path)), - true, - ) - .await?; - - Ok(format!( - "succesfully updated postgres trigger at path {} to status {}", - path, payload.enabled - )) -} - pub async fn get_template_script(Path((_, id)): Path<(String, String)>) -> Result { let template = if let Some((_, template)) = TEMPLATE.remove(&id) { template @@ -1494,7 +942,11 @@ pub async fn create_template_script( let relations = match relations { Some(r) => r, - None => return Err(anyhow!("You must at least choose schema to fetch table from").into()), + None => { + return Err( + anyhow::anyhow!("You must at least choose schema to fetch table from").into(), + ) + } }; let pg_connection: Client = get_default_pg_connection( @@ -1596,15 +1048,17 @@ pub async fn create_template_script( MappingInfo::new(info.column_name, Type::from_oid(info.oid), info.is_nullable); match schema_map.entry(info.table_schema) { - Occupied(mut schema_entry) => match schema_entry.get_mut().entry(info.table_name) { - Occupied(mut table_entry) => { - table_entry.get_mut().push(mapped_info); + std::collections::hash_map::Entry::Occupied(mut schema_entry) => { + match schema_entry.get_mut().entry(info.table_name) { + std::collections::hash_map::Entry::Occupied(mut table_entry) => { + table_entry.get_mut().push(mapped_info); + } + std::collections::hash_map::Entry::Vacant(v) => { + v.insert(vec![mapped_info]); + } } - Vacant(v) => { - v.insert(vec![mapped_info]); - } - }, - Vacant(schema_vacant) => { + } + std::collections::hash_map::Entry::Vacant(schema_vacant) => { let mut table_map = HashMap::new(); table_map.insert(info.table_name, vec![mapped_info]); schema_vacant.insert(table_map); diff --git a/backend/windmill-api/src/postgres_triggers/mapper.rs b/backend/windmill-api/src/triggers/postgres/mapper.rs similarity index 99% rename from backend/windmill-api/src/postgres_triggers/mapper.rs rename to backend/windmill-api/src/triggers/postgres/mapper.rs index bbc7b9c011..a24fb65378 100644 --- a/backend/windmill-api/src/postgres_triggers/mapper.rs +++ b/backend/windmill-api/src/triggers/postgres/mapper.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use rust_postgres::types::Type; -use super::handler::Language; +use super::Language; fn postgres_to_typescript_type(postgres_type: Option) -> String { let data_type = match postgres_type { diff --git a/backend/windmill-api/src/triggers/postgres/mod.rs b/backend/windmill-api/src/triggers/postgres/mod.rs new file mode 100644 index 0000000000..d94b08f67d --- /dev/null +++ b/backend/windmill-api/src/triggers/postgres/mod.rs @@ -0,0 +1,547 @@ +use std::collections::HashMap; + +use crate::{ + db::{ApiAuthed, DB}, + resources::try_get_resource_from_db_as, + triggers::trigger_helpers::TriggerJobArgs, +}; +use chrono::Utc; +use itertools::Itertools; +use native_tls::{Certificate, TlsConnector}; +use pg_escape::quote_identifier; +use rand::Rng; +use rust_postgres::{config::SslMode, Client, Config, NoTls}; +use rust_postgres_native_tls::MakeTlsConnector; +use serde::{Deserialize, Deserializer, Serialize}; +use serde_json::value::RawValue; +use sqlx::FromRow; +use windmill_common::{ + db::UserDB, + error::{to_anyhow, Error, Result}, + triggers::TriggerKind, + utils::empty_as_none, +}; + +pub mod handler; +mod mapper; + +#[derive(Clone, Copy)] +pub struct PostgresTrigger; + +impl TriggerJobArgs for PostgresTrigger { + type Payload = HashMap>; + const TRIGGER_KIND: TriggerKind = TriggerKind::Postgres; + fn v1_payload_fn(payload: &HashMap>) -> HashMap> { + payload.to_owned() + } +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct PostgresConfig { + pub postgres_resource_path: String, + pub replication_slot_name: String, + pub publication_name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub basic_mode: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PostgresConfigRequest { + postgres_resource_path: String, + #[serde(default)] + replication_slot_name: String, + #[serde(default)] + publication_name: String, + publication: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TestPostgresConfig { + pub postgres_resource_path: String, +} + +fn check_if_valid_relation<'de, D>( + relations: D, +) -> std::result::Result>, D::Error> +where + D: Deserializer<'de>, +{ + let relations: Option> = Option::deserialize(relations)?; + let mut track_all_table_in_schema = false; + let mut track_specific_columns_in_table = false; + match relations { + Some(relations) => { + for relation in relations.iter() { + if relation.schema_name.is_empty() { + return Err(serde::de::Error::custom( + "Schema Name must not be empty".to_string(), + )); + } + + if !track_all_table_in_schema && relation.table_to_track.is_empty() { + track_all_table_in_schema = true; + continue; + } + + for table_to_track in relation.table_to_track.iter() { + if table_to_track.table_name.trim().is_empty() { + return Err(serde::de::Error::custom( + "Table name must not be empty".to_string(), + )); + } + + if !track_specific_columns_in_table && table_to_track.columns_name.is_some() { + track_specific_columns_in_table = true; + } + } + + if track_all_table_in_schema && track_specific_columns_in_table { + return Err(serde::de::Error::custom("Incompatible tracking options. Schema-level tracking and specific table tracking with column selection cannot be used together. Refer to the documentation for valid configurations.")); + } + } + + if !relations + .iter() + .map(|relation| relation.schema_name.as_str()) + .all_unique() + { + return Err(serde::de::Error::custom( + "You cannot choose a schema more than one time".to_string(), + )); + } + + Ok(Some(relations)) + } + None => Ok(None), + } +} + +fn check_if_valid_transaction_type<'de, D>( + transaction_type: D, +) -> std::result::Result, D::Error> +where + D: Deserializer<'de>, +{ + let mut transaction_type: Vec = Vec::deserialize(transaction_type)?; + if transaction_type.len() > 3 { + return Err(serde::de::Error::custom( + "More than 3 transaction type which is not authorized, you are only allowed to those 3 transaction types: Insert, Update and Delete" + .to_string(), + )); + } + transaction_type.sort_unstable(); + transaction_type.dedup(); + + for transaction in transaction_type.iter() { + match transaction.to_lowercase().as_ref() { + "insert" => {}, + "update" => {}, + "delete" => {}, + _ => { + return Err(serde::de::Error::custom( + "Only the following transaction types are allowed: Insert, Update and Delete (case insensitive)" + .to_string(), + )) + } + } + } + + Ok(transaction_type) +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct PublicationData { + #[serde(default, deserialize_with = "check_if_valid_relation")] + pub table_to_track: Option>, + #[serde(deserialize_with = "check_if_valid_transaction_type")] + pub transaction_to_track: Vec, +} + +impl PublicationData { + pub fn new( + table_to_track: Option>, + transaction_to_track: Vec, + ) -> PublicationData { + PublicationData { table_to_track, transaction_to_track } + } +} + +// Slot list struct +#[derive(FromRow, Debug, Serialize)] +pub struct SlotList { + pub slot_name: Option, + pub active: Option, +} + +// Slot struct +#[derive(Debug, Serialize, Deserialize)] +pub struct Slot { + pub name: String, +} + +// Template script struct +#[derive(Debug, Deserialize)] +pub struct TemplateScript { + pub postgres_resource_path: String, + #[serde(deserialize_with = "check_if_valid_relation")] + pub relations: Option>, + pub language: Language, +} + +// Language enum +#[derive(Deserialize, Debug)] +pub enum Language { + #[serde(rename = "typescript", alias = "Typescript")] + Typescript, +} + +// Test postgres struct +#[derive(Serialize, Deserialize)] +pub struct TestPostgres { + pub postgres_resource_path: String, +} + +// PostgreSQL publication replication struct +#[derive(Serialize, Deserialize)] +pub struct PostgresPublicationReplication { + pub publication_name: String, + pub replication_slot_name: String, +} + +impl PostgresPublicationReplication { + pub fn new( + publication_name: String, + replication_slot_name: String, + ) -> PostgresPublicationReplication { + PostgresPublicationReplication { publication_name, replication_slot_name } + } +} + +pub const ERROR_PUBLICATION_NAME_NOT_EXISTS: &str = r#"The publication associated with this trigger no longer exists. Recreate a new publication or select an existing one in the advanced tab, or delete and recreate a new trigger"#; + +#[derive(FromRow, Serialize, Deserialize, Debug)] +pub struct Postgres { + pub user: String, + pub password: String, + pub host: String, + pub port: Option, + pub dbname: String, + #[serde(default)] + pub sslmode: String, + #[serde(default, deserialize_with = "empty_as_none")] + pub root_certificate_pem: Option, +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct TableToTrack { + pub table_name: String, + #[serde(default, deserialize_with = "empty_as_none")] + pub where_clause: Option, + #[serde(default, deserialize_with = "empty_as_none")] + pub columns_name: Option>, +} + +impl TableToTrack { + pub fn new( + table_name: String, + where_clause: Option, + columns_name: Option>, + ) -> TableToTrack { + TableToTrack { table_name, where_clause, columns_name } + } +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct Relations { + pub schema_name: String, + pub table_to_track: Vec, +} + +impl Relations { + pub fn new(schema_name: String, table_to_track: Vec) -> Relations { + Relations { schema_name, table_to_track } + } + + pub fn add_new_table(&mut self, table_to_track: TableToTrack) { + self.table_to_track.push(table_to_track); + } +} + +fn build_tls_connector( + ssl_mode: SslMode, + root_certificate_pem: Option<&String>, +) -> Result> { + let get_tls_builder_for_verify = |root_certificate: Option<&String>| { + let mut builder = TlsConnector::builder(); + if let Some(root_certificate) = root_certificate { + let root_certificate_pem = + Certificate::from_pem(root_certificate.as_bytes()).map_err(to_anyhow)?; + builder.add_root_certificate(root_certificate_pem); + } + Ok::<_, Error>(builder) + }; + let connector = match ssl_mode { + SslMode::Disable => return Ok(None), + SslMode::Require | SslMode::Prefer => { + let mut builder = TlsConnector::builder(); + builder.danger_accept_invalid_certs(true); + builder.danger_accept_invalid_hostnames(true); + builder + } + + SslMode::VerifyCa => { + let mut builder = get_tls_builder_for_verify(root_certificate_pem)?; + builder.danger_accept_invalid_hostnames(true); + builder + } + + SslMode::VerifyFull => { + let builder = get_tls_builder_for_verify(root_certificate_pem)?; + builder + } + _ => unreachable!(), + }; + + Ok(Some(MakeTlsConnector::new( + connector.build().map_err(to_anyhow)?, + ))) +} + +pub async fn get_raw_postgres_connection( + database: &Postgres, + logical_mode: bool, +) -> Result { + let ssl_mode = match database.sslmode.as_ref() { + "disable" => SslMode::Disable, + "" | "prefer" | "allow" => SslMode::Prefer, + "require" => SslMode::Require, + "verify-ca" => SslMode::VerifyCa, + "verify-full" => SslMode::VerifyFull, + ssl_mode => { + return Err(Error::BadRequest( + format!("Invalid ssl mode for postgres: {}, please put a valid ssl_mode among the following available ssl mode: ['disable', 'allow', 'prefer', 'verify-ca', 'verify-full']", ssl_mode), + )) + } + }; + + let mut config = Config::new(); + config + .dbname(&database.dbname) + .host(&database.host) + .user(&database.user) + .ssl_mode(ssl_mode); + + if logical_mode { + config.replication_mode(rust_postgres::config::ReplicationMode::Logical); + } + + if let Some(port) = database.port { + config.port(port); + }; + + if !database.password.is_empty() { + config.password(&database.password); + } + + let connector = build_tls_connector(ssl_mode, database.root_certificate_pem.as_ref())?; + + let client = if let Some(connector) = connector { + let (client, connection) = config.connect(connector).await.map_err(to_anyhow)?; + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::debug!("{:#?}", e); + }; + tracing::info!("Successfully Connected into database"); + }); + client + } else { + let (client, connection) = config.connect(NoTls).await.map_err(to_anyhow)?; + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::debug!("{:#?}", e); + }; + tracing::info!("Successfully Connected into database"); + }); + client + }; + + Ok(client) +} + +pub async fn get_pg_connection( + authed: ApiAuthed, + user_db: Option, + db: &DB, + postgres_resource_path: &str, + w_id: &str, + logical_mode: bool, +) -> Result { + let database = + try_get_resource_from_db_as::(&authed, user_db, db, postgres_resource_path, w_id) + .await?; + + Ok(get_raw_postgres_connection(&database, logical_mode).await?) +} + +pub async fn get_default_pg_connection( + authed: ApiAuthed, + user_db: Option, + db: &DB, + postgres_resource_path: &str, + w_id: &str, +) -> Result { + get_pg_connection(authed, user_db, db, postgres_resource_path, w_id, false).await +} + +pub async fn create_logical_replication_slot(tx: &Client, slot_name: &str) -> Result<()> { + tx.execute( + &format!("SELECT * FROM pg_create_logical_replication_slot($1, 'pgoutput')"), + &[&slot_name], + ) + .await + .map_err(to_anyhow)?; + Ok(()) +} + +pub async fn check_if_valid_publication_for_postgres_version( + pg_connection: &Client, + table_to_track: Option<&[Relations]>, +) -> Result { + use crate::triggers::postgres::handler::get_postgres_version_internal; + + let postgres_version = get_postgres_version_internal(pg_connection).await?; + + let pg_14 = postgres_version.starts_with("14"); + if pg_14 { + let unsupported_publication = table_to_track + .and_then(|relations| { + relations.iter().find(|relation| { + let invalid_relation = relation.table_to_track.iter().find(|table_to_track| { + table_to_track.where_clause.is_some() + || table_to_track.columns_name.is_some() + }); + + relation.table_to_track.is_empty() || invalid_relation.is_some() + }) + }) + .is_some(); + + if unsupported_publication { + return Err(Error::BadRequest( + "Your PostgreSQL database is running version 14, which does not support the following publication features: \ + - WHERE clause filtering, \ + - selective column tracking, and \ + - tracking all tables within a schema.\n\ + These features are only available in PostgreSQL 15 and above.".to_string(), + )); + } + } + Ok(pg_14) +} + +pub async fn create_pg_publication( + pg_connection: &Client, + publication_name: &str, + table_to_track: Option<&[Relations]>, + transaction_to_track: &[String], +) -> Result<()> { + let pg_14 = + check_if_valid_publication_for_postgres_version(pg_connection, table_to_track).await?; + let mut query = String::from("CREATE PUBLICATION "); + + query.push_str("e_identifier(publication_name)); + + match table_to_track { + Some(database_component) if !database_component.is_empty() => { + query.push_str(" FOR"); + let mut first = true; + for (i, schema) in database_component.iter().enumerate() { + if schema.table_to_track.is_empty() { + query.push_str(" TABLES IN SCHEMA "); + query.push_str("e_identifier(&schema.schema_name)); + } else { + if pg_14 && first { + query.push_str(" TABLE ONLY "); + first = false + } else if !pg_14 { + query.push_str(" TABLE ONLY "); + } + for (j, table) in schema.table_to_track.iter().enumerate() { + let table_name = quote_identifier(&table.table_name); + let schema_name = quote_identifier(&schema.schema_name); + let full_name = format!("{}.{}", &schema_name, &table_name); + query.push_str(&full_name); + if let Some(columns) = table.columns_name.as_ref() { + query.push_str(" ("); + let columns = columns + .iter() + .map(|column| quote_identifier(column)) + .join(", "); + query.push_str(&columns); + query.push_str(")"); + } + + if let Some(where_clause) = &table.where_clause { + query.push_str(" WHERE ("); + query.push_str(where_clause); + query.push(')'); + } + + if j + 1 != schema.table_to_track.len() { + query.push_str(", "); + } + } + } + if i < database_component.len() - 1 { + query.push_str(", "); + } + } + } + _ => { + query.push_str(" FOR ALL TABLES "); + } + }; + + if !transaction_to_track.is_empty() { + let transactions = || transaction_to_track.iter().join(", "); + query.push_str(" WITH (publish = '"); + query.push_str(&transactions()); + query.push_str("');"); + } + + pg_connection + .execute(&query, &[]) + .await + .map_err(to_anyhow)?; + Ok(()) +} + +pub async fn drop_publication(pg_connection: &Client, publication_name: &str) -> Result<()> { + let mut query = String::from("DROP PUBLICATION IF EXISTS "); + let quoted_publication_name = quote_identifier(publication_name); + query.push_str("ed_publication_name); + + pg_connection + .execute(&query, &[]) + .await + .map_err(to_anyhow)?; + + Ok(()) +} + +pub fn generate_random_string() -> String { + let timestamp = Utc::now().timestamp_millis().to_string(); + let mut rng = rand::rng(); + let charset = "abcdefghijklmnopqrstuvwxyz0123456789"; + + let random_part = (0..10) + .map(|_| { + charset + .chars() + .nth(rng.random_range(0..charset.len())) + .unwrap() + }) + .collect::(); + + format!("{}_{}", timestamp, random_part) +} diff --git a/backend/windmill-api/src/triggers/sqs/handler_oss.rs b/backend/windmill-api/src/triggers/sqs/handler_oss.rs new file mode 100644 index 0000000000..53365def58 --- /dev/null +++ b/backend/windmill-api/src/triggers/sqs/handler_oss.rs @@ -0,0 +1,66 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub use super::handler_ee::*; + +#[cfg(not(feature = "private"))] +use { + super::SqsTrigger, + crate::{ + db::{ApiAuthed, DB}, + triggers::{Trigger, TriggerCrud, TriggerData}, + }, + axum::async_trait, + sqlx::PgConnection, + windmill_common::error::{Error, Result}, + windmill_git_sync::DeployedObject, +}; + +#[cfg(not(feature = "private"))] +#[async_trait] +#[cfg(not(feature = "private"))] +impl TriggerCrud for SqsTrigger { + type Trigger = Trigger; + type TriggerConfig = (); + type TriggerConfigRequest = (); + type TestConnectionConfig = (); + + const TABLE_NAME: &'static str = ""; + const TRIGGER_TYPE: &'static str = ""; + const SUPPORTS_ENABLED: bool = false; + const SUPPORTS_SERVER_STATE: bool = false; + const SUPPORTS_TEST_CONNECTION: bool = false; + const ROUTE_PREFIX: &'static str = "/sqs_triggers"; + const DEPLOYMENT_NAME: &'static str = ""; + const IS_CLOUD_HOSTED: bool = false; + + fn get_deployed_object(path: String) -> DeployedObject { + DeployedObject::SqsTrigger { path } + } + + async fn create_trigger( + &self, + _db: &DB, + _tx: &mut PgConnection, + _authed: &ApiAuthed, + _w_id: &str, + _trigger: TriggerData, + ) -> Result<()> { + Err(Error::BadRequest( + "SQS triggers are not available in open source version".to_string(), + )) + } + + async fn update_trigger( + &self, + _db: &DB, + _tx: &mut PgConnection, + _authed: &ApiAuthed, + _workspace_id: &str, + _path: &str, + _trigger: TriggerData, + ) -> Result<()> { + Err(Error::BadRequest( + "SQS triggers are not available in open source version".to_string(), + )) + } +} diff --git a/backend/windmill-api/src/triggers/sqs/mod.rs b/backend/windmill-api/src/triggers/sqs/mod.rs new file mode 100644 index 0000000000..74a93c12a5 --- /dev/null +++ b/backend/windmill-api/src/triggers/sqs/mod.rs @@ -0,0 +1,11 @@ +#[cfg(feature = "private")] +mod handler_ee; +pub mod handler_oss; + +#[cfg(feature = "private")] +mod mod_ee; +#[cfg(feature = "private")] +pub use mod_ee::*; + +#[derive(Copy, Clone)] +pub struct SqsTrigger; diff --git a/backend/windmill-api/src/triggers/trigger_helpers.rs b/backend/windmill-api/src/triggers/trigger_helpers.rs new file mode 100644 index 0000000000..21a698b62d --- /dev/null +++ b/backend/windmill-api/src/triggers/trigger_helpers.rs @@ -0,0 +1,811 @@ +use anyhow::Context; +use axum::response::IntoResponse; +use http::StatusCode; +use serde::Deserialize; +use serde_json::value::RawValue; +use sqlx::types::Json; +use std::collections::HashMap; +use std::future::Future; +use uuid::Uuid; +use windmill_common::{ + db::UserDB, + error::Result, + flows::{FlowModuleValue, Retry}, + get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path, + jobs::{get_has_preprocessor_from_content_and_lang, script_path_to_payload, JobPayload}, + scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, + triggers::{ + HubOrWorkspaceId, RunnableFormat, RunnableFormatVersion, TriggerKind, + RUNNABLE_FORMAT_VERSION_CACHE, + }, + users::username_to_permissioned_as, + utils::StripPath, + worker::to_raw_value, + FlowVersionInfo, +}; +use windmill_queue::{push, PushArgs, PushArgsOwned, PushIsolationLevel}; + +#[cfg(feature = "enterprise")] +use crate::jobs::check_license_key_valid; +use crate::{ + db::{ApiAuthed, DB}, + jobs::{ + check_tag_available_for_workspace, delete_job_metadata_after_use, result_to_response, + run_flow_by_path_inner, run_script_by_path_inner, run_wait_result_internal, RunJobQuery, + }, + utils::check_scopes, + HTTP_CLIENT, +}; + +struct ScriptInfo { + has_preprocessor: Option, + language: ScriptLang, + content: String, + schema: Option>, +} + +#[derive(Debug, Deserialize)] +struct PropertyDefinition { + r#type: Option, +} + +#[derive(Debug, Deserialize)] +struct PartialSchema { + properties: Option>, +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub enum RunnableId { + FlowPath(String), + ScriptId(ScriptId), + HubScript(String), +} + +impl RunnableId { + pub fn from_script_hash(hash: ScriptHash) -> Self { + Self::ScriptId(ScriptId::ScriptHash(hash)) + } + + pub fn from_script_path(path: &str) -> Self { + if path.starts_with("hub/") { + Self::HubScript(path.to_string()) + } else { + Self::ScriptId(ScriptId::ScriptPath(path.to_string())) + } + } + + pub fn from_flow_path(path: &str) -> Self { + Self::FlowPath(path.to_string()) + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub enum ScriptId { + ScriptPath(String), + ScriptHash(ScriptHash), +} + +impl ScriptId { + async fn get_script_hash(self, workspace_id: &str, db: &DB) -> Result { + let hash = match self { + ScriptId::ScriptPath(path) => { + let info = get_latest_deployed_hash_for_path(db, workspace_id, &path).await?; + info.hash + } + ScriptId::ScriptHash(hash) => hash.0, + }; + + Ok(hash) + } +} + +async fn get_script_info( + db: &DB, + workspace_id: &str, + hash: i64, +) -> std::result::Result { + sqlx::query_as!(ScriptInfo, "SELECT has_preprocessor, language as \"language: _\", content, schema as \"schema: _\" FROM script WHERE workspace_id = $1 AND hash = $2", workspace_id, hash) + .fetch_one(db) + .await +} + +fn runnable_format_from_schema_without_preprocessor( + trigger_kind: &TriggerKind, + schema: Option>, +) -> RunnableFormat { + match trigger_kind { + TriggerKind::Mqtt + if schema.as_ref().is_some_and(|schema| { + schema.properties.as_ref().is_some_and(|properties| { + properties.iter().any(|(key, def)| { + key == "payload" && def.r#type.as_ref().is_some_and(|t| t == "array") + }) + }) + }) => + { + RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: false } + } + TriggerKind::Kafka | TriggerKind::Nats + if schema.as_ref().is_some_and(|schema| { + schema + .properties + .as_ref() + .is_some_and(|properties| properties.keys().any(|key| key == "msg")) + }) => + { + RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: false } + } + _ => RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor: false }, + } +} + +fn runnable_format_from_preprocessor_args( + args: Option>, +) -> RunnableFormat { + if let Some(args) = args { + if args.iter().any(|arg| arg.name == "wm_trigger") + || (args.len() > 0 && args.iter().all(|arg| arg.name != "event")) + { + RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: true } + } else { + RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor: true } + } + } else { + RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor: true } + } +} + +enum PreprocessorInfo { + Preprocessor { content: String, language: ScriptLang }, + NoPreprocessor { schema: Option> }, +} + +#[derive(Debug, Deserialize)] +struct FlowInfo { + preprocessor_module: Option>, + schema: Option>, +} + +fn get_preprocessor_args_from_content_and_language( + content: &str, + language: &ScriptLang, +) -> Result>> { + let args = match language { + ScriptLang::Bun | ScriptLang::Bunnative | ScriptLang::Deno | ScriptLang::Nativets => { + let args = windmill_parser_ts::parse_deno_signature( + &content, + true, + false, + Some("preprocessor".to_string()), + )?; + Some(args.args) + } + ScriptLang::Python3 => { + let args = windmill_parser_py::parse_python_signature( + &content, + Some("preprocessor".to_string()), + false, + )?; + Some(args.args) + } + _ => None, + }; + Ok(args) +} + +pub async fn get_runnable_format( + runnable_id: RunnableId, + workspace_id: &str, + db: &DB, + trigger_kind: &TriggerKind, +) -> Result { + let (key, preprocessor_info) = match runnable_id { + RunnableId::HubScript(path) => { + let Some(version) = path.split("/").nth(1) else { + return Err(windmill_common::error::Error::internal_err( + "Invalid hub script path".to_string(), + )); + }; + + let version = match version.parse::() { + Ok(version) => version, + Err(_) => { + return Err(windmill_common::error::Error::internal_err( + "Invalid hub script version".to_string(), + )); + } + }; + + let key = (HubOrWorkspaceId::Hub, version, trigger_kind.clone()); + + let runnable_format = RUNNABLE_FORMAT_VERSION_CACHE.get(&key); + + if let Some(runnable_format) = runnable_format { + tracing::debug!("Using cached runnable format for hub script {path}"); + return Ok(runnable_format); + } + + let hub_script = + get_full_hub_script_by_path(StripPath(path.to_string()), &HTTP_CLIENT, Some(db)) + .await?; + + let has_preprocessor = get_has_preprocessor_from_content_and_lang( + &hub_script.content, + &hub_script.language, + )?; + + let partial_schema = serde_json::from_str(hub_script.schema.get())?; + + ( + key, + if has_preprocessor { + PreprocessorInfo::Preprocessor { + content: hub_script.content, + language: hub_script.language, + } + } else { + PreprocessorInfo::NoPreprocessor { + schema: Some(sqlx::types::Json(partial_schema)), + } + }, + ) + } + RunnableId::FlowPath(path) => { + let FlowVersionInfo { version, .. } = + get_latest_flow_version_info_for_path(db, workspace_id, &path, true).await?; + + let key = ( + HubOrWorkspaceId::WorkspaceId(workspace_id.to_string()), + version, + trigger_kind.clone(), + ); + + let runnable_format = RUNNABLE_FORMAT_VERSION_CACHE.get(&key); + + if let Some(runnable_format) = runnable_format { + tracing::debug!("Using cached runnable format for flow {path}"); + return Ok(runnable_format); + } + + let flow_info = sqlx::query_as!( + FlowInfo, + "SELECT + value->'preprocessor_module'->'value' as \"preprocessor_module: _\", + schema as \"schema: _\" + FROM flow_version + WHERE + path = $1 + AND workspace_id = $2 + ORDER BY created_at DESC + LIMIT 1", + path, + workspace_id, + ) + .fetch_one(db) + .await?; + + if let Some(preprocessor_module) = flow_info.preprocessor_module { + match preprocessor_module.0 { + FlowModuleValue::RawScript { content, language, .. } => { + (key, PreprocessorInfo::Preprocessor { content, language }) + } + FlowModuleValue::Script { path, hash, .. } => { + let hash = if let Some(hash) = hash { + hash.0 + } else { + let script_hash = + get_latest_deployed_hash_for_path(db, workspace_id, &path).await?; + script_hash.hash + }; + let script_info = get_script_info(db, workspace_id, hash).await?; + ( + key, + PreprocessorInfo::Preprocessor { + content: script_info.content, + language: script_info.language, + }, + ) + } + _ => { + return Err(windmill_common::error::Error::internal_err( + "Unsupported preprocessor module".to_string(), + )); + } + } + } else { + ( + key, + PreprocessorInfo::NoPreprocessor { schema: flow_info.schema }, + ) + } + } + RunnableId::ScriptId(script_id) => { + let hash = script_id.get_script_hash(workspace_id, db).await?; + let key = ( + HubOrWorkspaceId::WorkspaceId(workspace_id.to_string()), + hash, + trigger_kind.clone(), + ); + let runnable_format = RUNNABLE_FORMAT_VERSION_CACHE.get(&key); + + if let Some(runnable_format) = runnable_format { + tracing::debug!("Using cached runnable format for script {hash}"); + return Ok(runnable_format); + } + + let script_info = get_script_info(db, workspace_id, hash).await?; + + if script_info.has_preprocessor.unwrap_or(false) { + ( + key, + PreprocessorInfo::Preprocessor { + content: script_info.content, + language: script_info.language, + }, + ) + } else { + ( + key, + PreprocessorInfo::NoPreprocessor { schema: script_info.schema }, + ) + } + } + }; + + let runnable_format = match preprocessor_info { + PreprocessorInfo::Preprocessor { content, language } => { + let args = get_preprocessor_args_from_content_and_language(&content, &language)?; + runnable_format_from_preprocessor_args(args) + } + PreprocessorInfo::NoPreprocessor { schema } => { + runnable_format_from_schema_without_preprocessor(trigger_kind, schema) + } + }; + + RUNNABLE_FORMAT_VERSION_CACHE.insert(key, runnable_format); + + Ok(runnable_format) +} + +#[allow(dead_code)] + +pub trait TriggerJobArgs { + type Payload: Send + Sync; + const TRIGGER_KIND: TriggerKind; + + fn v1_payload_fn(payload: &Self::Payload) -> HashMap>; + fn v2_payload_fn(payload: &Self::Payload) -> HashMap> { + Self::v1_payload_fn(payload) + } + + fn build_job_args_v2( + has_preprocessor: bool, + payload: &Self::Payload, + info: HashMap>, + ) -> PushArgsOwned { + let mut args = Self::v2_payload_fn(payload); + if has_preprocessor { + args.insert( + "kind".to_string(), + to_raw_value(&Self::TRIGGER_KIND.to_key()), + ); + args.extend(info); + let args = HashMap::from([("event".to_string(), to_raw_value(&args))]); + PushArgsOwned { args, extra: None } + } else { + PushArgsOwned { args, extra: None } + } + } + + fn build_job_args_v1( + has_preprocessor: bool, + payload: &Self::Payload, + info: HashMap>, + ) -> PushArgsOwned { + let trigger_key = Self::TRIGGER_KIND.to_key(); + let args = Self::v1_payload_fn(payload); + let extra = if has_preprocessor { + Some(HashMap::from([( + "wm_trigger".to_string(), + to_raw_value(&serde_json::json!({ + "kind": trigger_key, + trigger_key: info + })), + )])) + } else { + None + }; + PushArgsOwned { args, extra } + } + + fn build_job_args( + runnable_path: &str, + is_flow: bool, + w_id: &str, + db: &DB, + payload: Self::Payload, + info: HashMap>, + ) -> impl Future> + Send { + async move { + let runnable_id = if is_flow { + RunnableId::from_flow_path(runnable_path) + } else { + RunnableId::from_script_path(runnable_path) + }; + Self::build_job_args_from_runnable_id(runnable_id, w_id, db, payload, info).await + } + } + + fn build_job_args_from_runnable_id( + runnable_id: RunnableId, + w_id: &str, + db: &DB, + payload: Self::Payload, + trigger_info: HashMap>, + ) -> impl Future> + Send { + async move { + let runnable_format = + get_runnable_format(runnable_id, w_id, db, &Self::TRIGGER_KIND).await?; + let job_args = match runnable_format { + RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor } => { + Self::build_job_args_v1(has_preprocessor, &payload, trigger_info) + } + RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor } => { + Self::build_job_args_v2(has_preprocessor, &payload, trigger_info) + } + }; + Ok(job_args) + } + } + + fn build_capture_payloads( + payload: &Self::Payload, + info: HashMap>, + ) -> (PushArgsOwned, PushArgsOwned) { + let main_args = Self::build_job_args_v2(false, payload, info.clone()); + let preprocessor_args = Self::build_job_args_v2(true, payload, info); + (main_args, preprocessor_args) + } +} + +#[allow(dead_code)] +async fn trigger_runnable_inner( + db: &DB, + user_db: Option, + authed: ApiAuthed, + workspace_id: &str, + runnable_path: &str, + is_flow: bool, + args: PushArgsOwned, + retry: Option<&sqlx::types::Json>, + error_handler_path: Option<&str>, + error_handler_args: Option<&sqlx::types::Json>>, + trigger_path: String, +) -> Result<(Uuid, Option, Option)> { + let error_handler_args = error_handler_args.map(|args| { + let args = args + .0 + .iter() + .map(|(key, value)| (key.to_owned(), to_raw_value(&value))) + .collect::>>(); + Json(args) + }); + + let user_db = user_db.unwrap_or_else(|| UserDB::new(db.clone())); + let (uuid, delete_after_use, early_return) = if is_flow { + let run_query = RunJobQuery::default(); + let path = StripPath(runnable_path.to_string()); + let (uuid, early_return) = run_flow_by_path_inner( + authed, + db.clone(), + user_db, + workspace_id.to_string(), + path, + run_query, + args, + ) + .await?; + (uuid, None, early_return) + } else { + let (uuid, delete_after_use) = trigger_script_internal( + db, + user_db, + authed, + workspace_id, + runnable_path, + args, + retry, + error_handler_path, + error_handler_args.as_ref(), + trigger_path, + ) + .await?; + (uuid, delete_after_use, None) + }; + + Ok((uuid, delete_after_use, early_return)) +} + +#[allow(dead_code)] +pub async fn trigger_runnable( + db: &DB, + user_db: Option, + authed: ApiAuthed, + workspace_id: &str, + runnable_path: &str, + is_flow: bool, + args: PushArgsOwned, + retry: Option<&sqlx::types::Json>, + error_handler_path: Option<&str>, + error_handler_args: Option<&sqlx::types::Json>>, + trigger_path: String, +) -> Result { + let (uuid, _, _) = trigger_runnable_inner( + db, + user_db, + authed, + workspace_id, + runnable_path, + is_flow, + args, + retry, + error_handler_path, + error_handler_args, + trigger_path, + ) + .await?; + Ok((StatusCode::CREATED, uuid.to_string()).into_response()) +} + +#[allow(dead_code)] +pub async fn trigger_runnable_and_wait_for_result( + db: &DB, + user_db: Option, + authed: ApiAuthed, + workspace_id: &str, + runnable_path: &str, + is_flow: bool, + args: PushArgsOwned, + retry: Option<&sqlx::types::Json>, + error_handler_path: Option<&str>, + error_handler_args: Option<&sqlx::types::Json>>, + trigger_path: String, +) -> Result { + let username = authed.username.clone(); + let (uuid, delete_after_use, early_return) = trigger_runnable_inner( + db, + user_db, + authed, + workspace_id, + runnable_path, + is_flow, + args, + retry, + error_handler_path, + error_handler_args, + trigger_path, + ) + .await?; + let (result, success) = + run_wait_result_internal(db, uuid, workspace_id.to_string(), early_return, &username) + .await?; + + if delete_after_use.unwrap_or(false) { + delete_job_metadata_after_use(&db, uuid).await?; + } + + result_to_response(result, success) +} + +#[allow(dead_code)] +pub async fn trigger_runnable_and_wait_for_raw_result( + db: &DB, + user_db: Option, + authed: ApiAuthed, + workspace_id: &str, + runnable_path: &str, + is_flow: bool, + args: PushArgsOwned, + retry: Option<&sqlx::types::Json>, + error_handler_path: Option<&str>, + error_handler_args: Option<&sqlx::types::Json>>, + trigger_path: String, +) -> Result> { + let username = authed.username.clone(); + let (uuid, delete_after_use, early_return) = trigger_runnable_inner( + db, + user_db, + authed, + workspace_id, + runnable_path, + is_flow, + args, + retry, + error_handler_path, + error_handler_args, + trigger_path, + ) + .await?; + + let (result, success) = + run_wait_result_internal(db, uuid, workspace_id.to_string(), early_return, &username) + .await + .with_context(|| { + format!( + "Error fetching job result for {} {}", + if is_flow { "flow" } else { "script" }, + runnable_path + ) + })?; + + if delete_after_use.unwrap_or(false) { + delete_job_metadata_after_use(&db, uuid).await?; + } + + if !success { + Err(windmill_common::error::Error::internal_err(format!( + "{} {runnable_path} failed: {:?}", + if is_flow { "Flow" } else { "Script" }, + result + ))) + } else { + Ok(result) + } +} + +async fn trigger_script_internal( + db: &DB, + user_db: UserDB, + authed: ApiAuthed, + workspace_id: &str, + script_path: &str, + args: PushArgsOwned, + retry: Option<&sqlx::types::Json>, + error_handler_path: Option<&str>, + error_handler_args: Option<&sqlx::types::Json>>>, + trigger_path: String, +) -> Result<(Uuid, Option)> { + if retry.is_none() && error_handler_path.is_none() { + let run_query = RunJobQuery::default(); + let path = StripPath(script_path.to_string()); + run_script_by_path_inner( + authed, + db.clone(), + user_db, + workspace_id.to_string(), + path, + run_query, + args, + ) + .await + } else { + trigger_script_with_retry_and_error_handler( + db, + user_db, + authed, + workspace_id, + script_path, + args, + retry, + error_handler_path, + error_handler_args, + trigger_path, + ) + .await + } +} + +async fn trigger_script_with_retry_and_error_handler( + db: &DB, + user_db: UserDB, + authed: ApiAuthed, + workspace_id: &str, + script_path: &str, + args: PushArgsOwned, + retry: Option<&sqlx::types::Json>, + error_handler_path: Option<&str>, + error_handler_args: Option<&sqlx::types::Json>>>, + trigger_path: String, +) -> Result<(Uuid, Option)> { + #[cfg(feature = "enterprise")] + check_license_key_valid().await?; + + check_scopes(&authed, || format!("jobs:run:scripts:{script_path}"))?; + + let retry = retry.map(|r| r.0.clone()); + let error_handler_path = error_handler_path.map(|p| p.to_string()); + let error_handler_args = error_handler_args.map(|args| args.0.clone()); + + let (job_payload, tag, delete_after_use, timeout, on_behalf_of) = { + let mut tx = user_db.clone().begin(&authed).await?; + script_path_to_payload(script_path, &mut *tx, &workspace_id, Some(false)).await? + }; + + check_tag_available_for_workspace(&db, &workspace_id, &tag, &authed).await?; + + let (email, permissioned_as, push_authed, tx) = + if let Some(on_behalf_of) = on_behalf_of.as_ref() { + ( + on_behalf_of.email.as_str(), + on_behalf_of.permissioned_as.clone(), + None, + PushIsolationLevel::IsolatedRoot(db.clone()), + ) + } else { + ( + authed.email.as_str(), + username_to_permissioned_as(&authed.username), + Some(authed.clone().into()), + PushIsolationLevel::Isolated(user_db, authed.clone().into()), + ) + }; + + let push_args = PushArgs { args: &args.args, extra: args.extra }; + + let retryable_job_payload = match job_payload { + JobPayload::ScriptHash { + hash, + path, + custom_concurrency_key, + concurrent_limit, + concurrency_time_window_s, + cache_ttl, + priority, + apply_preprocessor, + .. + } => JobPayload::SingleScriptFlow { + path, + hash, + args: HashMap::from(&push_args), + retry, + error_handler_path, + error_handler_args, + custom_concurrency_key, + concurrent_limit, + concurrency_time_window_s, + cache_ttl, + priority, + tag_override: tag.clone(), + apply_preprocessor, + trigger_path: Some(trigger_path), + }, + _ => { + return Err(windmill_common::error::Error::internal_err(format!( + "Unsupported job payload: {:?}", + job_payload + ))) + } + }; + + let (uuid, tx) = push( + &db, + tx, + &workspace_id, + retryable_job_payload, + push_args, + authed.display_username(), + email, + permissioned_as, + authed.token_prefix.as_deref(), + None, + None, + None, + None, + None, + None, + false, + false, + None, + true, + tag, + timeout, + None, + None, + push_authed.as_ref(), + false, + ) + .await?; + tx.commit().await?; + + Ok((uuid, delete_after_use)) +} diff --git a/backend/windmill-api/src/triggers/websocket/handler.rs b/backend/windmill-api/src/triggers/websocket/handler.rs new file mode 100644 index 0000000000..0a321a450c --- /dev/null +++ b/backend/windmill-api/src/triggers/websocket/handler.rs @@ -0,0 +1,258 @@ +use std::borrow::Cow; + +use crate::{ + db::{ApiAuthed, DB}, + triggers::{Trigger, TriggerCrud, TriggerData}, +}; +use axum::async_trait; +use itertools::Itertools; +use serde_json::value::RawValue; +use sqlx::{types::Json as SqlxJson, PgConnection}; +use tokio_tungstenite::connect_async; +use windmill_common::{ + db::UserDB, + error::{Error, Result}, +}; +use windmill_git_sync::DeployedObject; + +use super::{ + get_url_from_runnable_value, TestWebsocketConfig, WebsocketConfig, WebsocketConfigRequest, + WebsocketTrigger, +}; + +#[async_trait] +impl TriggerCrud for WebsocketTrigger { + type TriggerConfig = WebsocketConfig; + type Trigger = Trigger; + type TriggerConfigRequest = WebsocketConfigRequest; + type TestConnectionConfig = TestWebsocketConfig; + + const TABLE_NAME: &'static str = "websocket_trigger"; + const TRIGGER_TYPE: &'static str = "websocket"; + const SUPPORTS_ENABLED: bool = true; + const SUPPORTS_SERVER_STATE: bool = true; + const SUPPORTS_TEST_CONNECTION: bool = true; + const ROUTE_PREFIX: &'static str = "/websocket_triggers"; + const DEPLOYMENT_NAME: &'static str = "WebSocket trigger"; + const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[ + "url", + "filters", + "initial_messages", + "url_runnable_args", + "can_return_message", + ]; + const IS_ALLOWED_ON_CLOUD: bool = false; + + fn get_deployed_object(path: String) -> DeployedObject { + DeployedObject::WebsocketTrigger { path } + } + + async fn validate_config( + &self, + _db: &DB, + config: &Self::TriggerConfigRequest, + _workspace_id: &str, + ) -> Result<()> { + if config.url.trim().is_empty() { + return Err(Error::BadRequest( + "WebSocket URL cannot be empty".to_string(), + )); + } + + if let Some(args) = &config.url_runnable_args { + if !args.is_object() { + return Err(Error::BadRequest( + "url_runnable_args must be an object".to_string(), + )); + } + } + + Ok(()) + } + + async fn create_trigger( + &self, + _db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + w_id: &str, + trigger: TriggerData, + ) -> Result<()> { + let filters = trigger + .config + .filters + .into_iter() + .map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap())) + .collect_vec(); + let initial_messages = trigger + .config + .initial_messages + .unwrap_or_default() + .into_iter() + .map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap())) + .collect_vec(); + sqlx::query!( + r#" + INSERT INTO websocket_trigger ( + workspace_id, + path, + url, + script_path, + is_flow, + enabled, + filters, + initial_messages, + url_runnable_args, + edited_by, + can_return_message, + email, + edited_at, + error_handler_path, + error_handler_args, + retry + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now(), $13, $14, $15 + ) + "#, + w_id, + trigger.base.path, + trigger.config.url, + trigger.base.script_path, + trigger.base.is_flow, + trigger.base.enabled.unwrap_or(true), + &filters as _, + &initial_messages as _, + trigger + .config + .url_runnable_args + .map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap())) as _, + authed.username, + trigger.config.can_return_message, + authed.email, + trigger.error_handling.error_handler_path, + trigger.error_handling.error_handler_args as _, + trigger.error_handling.retry as _ + ) + .execute(&mut *tx) + .await?; + Ok(()) + } + + async fn update_trigger( + &self, + _db: &DB, + tx: &mut PgConnection, + authed: &ApiAuthed, + w_id: &str, + path: &str, + trigger: TriggerData, + ) -> Result<()> { + let filters = trigger + .config + .filters + .into_iter() + .map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap())) + .collect_vec(); + let initial_messages = trigger + .config + .initial_messages + .unwrap_or_default() + .into_iter() + .map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap())) + .collect_vec(); + + // important to update server_id to NULL to stop current websocket listener + sqlx::query!( + " + UPDATE + websocket_trigger + SET + url = $1, + script_path = $2, + path = $3, + is_flow = $4, + filters = $5, + initial_messages = $6, + url_runnable_args = $7, + edited_by = $8, + email = $9, + can_return_message = $10, + edited_at = now(), + server_id = NULL, + error = NULL, + error_handler_path = $13, + error_handler_args = $14, + retry = $15 + WHERE + workspace_id = $11 AND path = $12 + ", + trigger.config.url, + trigger.base.script_path, + trigger.base.path, + trigger.base.is_flow, + filters.as_slice() as &[SqlxJson>], + initial_messages.as_slice() as &[SqlxJson>], + trigger + .config + .url_runnable_args + .map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap())) + as Option>>, + &authed.username, + &authed.email, + trigger.config.can_return_message, + w_id, + path, + trigger.error_handling.error_handler_path, + trigger.error_handling.error_handler_args as _, + trigger.error_handling.retry as _ + ) + .execute(&mut *tx) + .await?; + + Ok(()) + } + + async fn test_connection( + &self, + db: &DB, + authed: &ApiAuthed, + _user_db: &UserDB, + workspace_id: &str, + config: Self::TestConnectionConfig, + ) -> Result<()> { + let url = &config.url; + + let connect_url: Cow = if url.starts_with("$") { + if url.starts_with("$flow:") || url.starts_with("$script:") { + let path = url.splitn(2, ':').nth(1).unwrap(); + Cow::Owned( + get_url_from_runnable_value( + path, + url.starts_with("$flow:"), + &db, + authed.clone(), + config.url_runnable_args.as_ref(), + &workspace_id, + ) + .await?, + ) + } else { + return Err(Error::BadConfig(format!( + "Invalid WebSocket runnable path: {}", + url + ))); + } + } else { + Cow::Borrowed(&url) + }; + + connect_async(connect_url.as_ref()).await.map_err(|err| { + Error::BadConfig(format!( + "Error connecting to WebSocket: {}", + err.to_string() + )) + })?; + + Ok(()) + } +} diff --git a/backend/windmill-api/src/triggers/websocket/mod.rs b/backend/windmill-api/src/triggers/websocket/mod.rs new file mode 100644 index 0000000000..c367811406 --- /dev/null +++ b/backend/windmill-api/src/triggers/websocket/mod.rs @@ -0,0 +1,121 @@ +use std::collections::HashMap; + +use crate::{ + db::ApiAuthed, + triggers::trigger_helpers::{trigger_runnable_and_wait_for_raw_result, TriggerJobArgs}, +}; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use sqlx::{types::Json as SqlxJson, FromRow}; +use windmill_common::{ + error::{Error, Result}, + triggers::TriggerKind, + worker::to_raw_value, + DB, +}; +use windmill_queue::PushArgsOwned; + +mod handler; + +#[derive(Copy, Clone)] +pub struct WebsocketTrigger; + +impl TriggerJobArgs for WebsocketTrigger { + type Payload = String; + const TRIGGER_KIND: TriggerKind = TriggerKind::Websocket; + fn v1_payload_fn(payload: &Self::Payload) -> HashMap> { + HashMap::from([("msg".to_string(), to_raw_value(&payload))]) + } +} + +#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] +pub struct WebsocketConfig { + pub url: String, + #[serde(default)] + pub filters: Vec>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub initial_messages: Option>>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub url_runnable_args: Option>>, + #[serde(default)] + pub can_return_message: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WebsocketConfigRequest { + url: String, + filters: Vec, + initial_messages: Option>, + url_runnable_args: Option, + can_return_message: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TestWebsocketConfig { + url: String, + url_runnable_args: Option, +} + +pub fn value_to_args_hashmap( + args: Option<&serde_json::Value>, +) -> Result>> { + let args = if let Some(args) = args { + let args_map: Option> = + serde_json::from_value(args.clone()) + .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))?; + + args_map + .unwrap_or_else(HashMap::new) + .into_iter() + .map(|(k, v)| { + let raw_value = serde_json::value::to_raw_value(&v).map_err(|e| { + Error::BadRequest(format!("failed to convert to raw value: {}", e)) + })?; + Ok((k, raw_value)) + }) + .collect::>>>() + } else { + Ok(HashMap::new()) + }?; + Ok(args) +} + +pub async fn get_url_from_runnable_value( + path: &str, + is_flow: bool, + db: &DB, + authed: ApiAuthed, + args: Option<&serde_json::Value>, + workspace_id: &str, +) -> Result { + tracing::info!( + "Running {} {} to get WebSocket URL", + if is_flow { "flow" } else { "script" }, + path + ); + + let args = value_to_args_hashmap(args)?; + + let result = trigger_runnable_and_wait_for_raw_result( + db, + None, + authed, + workspace_id, + path, + is_flow, + PushArgsOwned { args, extra: None }, + None, + None, + None, + "".to_string(), // doesn't matter as no retry/error handler + ) + .await?; + + serde_json::from_str::(result.get()).map_err(|_| { + Error::BadConfig(format!( + "{} {} did not return a string", + if is_flow { "Flow" } else { "Script" }, + path, + )) + }) +} diff --git a/backend/windmill-api/src/utils.rs b/backend/windmill-api/src/utils.rs index ebbcad481c..2026099141 100644 --- a/backend/windmill-api/src/utils.rs +++ b/backend/windmill-api/src/utils.rs @@ -447,11 +447,12 @@ pub struct ExpiringCacheEntry { pub expiry: std::time::Instant, } +lazy_static::lazy_static! { + static ref DUCKLAKE_INSTANCE_PG_PASSWORD: std::sync::RwLock> = std::sync::RwLock::new(None); +} + #[cfg(all(feature = "kafka", feature = "enterprise", feature = "private"))] pub async fn update_rw_lock(lock: std::sync::Arc>, value: T) -> () { let mut w = lock.write().await; *w = value; } -lazy_static::lazy_static! { - static ref DUCKLAKE_INSTANCE_PG_PASSWORD: std::sync::RwLock> = std::sync::RwLock::new(None); -} diff --git a/backend/windmill-api/src/var_resource_cache.rs b/backend/windmill-api/src/var_resource_cache.rs new file mode 100644 index 0000000000..d97416b871 --- /dev/null +++ b/backend/windmill-api/src/var_resource_cache.rs @@ -0,0 +1,122 @@ +/* + * Author: Claude + * Copyright: Windmill Labs, Inc 2025 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use quick_cache::sync::Cache; +use serde_json::Value; +use std::time::{SystemTime, UNIX_EPOCH}; + +/// Cache TTL for variables and resources (30seconds) +const CACHE_TTL_SECS: u64 = 30; + +/// Cache entry with timestamp and value (following raw script cache pattern) +#[derive(Clone, Debug)] +pub struct CacheEntry { + pub timestamp: u64, + pub value: T, +} + +impl CacheEntry { + pub fn new(value: T) -> Self { + Self { + timestamp: SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(), + value, + } + } + + pub fn is_expired(&self) -> bool { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + now > self.timestamp + CACHE_TTL_SECS + } +} + + +lazy_static::lazy_static! { + /// Cache for individual variable values: key = "workspace_id:path" + pub static ref VARIABLE_CACHE: Cache> = Cache::new(1000); + + /// Cache for resource values: key = "workspace_id:path" + pub static ref RESOURCE_CACHE: Cache> = Cache::new(1000); +} + + +/// Generate cache key for variables and resources +pub fn cache_key(workspace_id: &str, path: &str) -> String { + format!("{}:{}", workspace_id, path) +} + +/// Get cached variable if available and not expired +pub fn get_cached_variable(workspace_id: &str, path: &str) -> Option { + let key = cache_key(workspace_id, path); + VARIABLE_CACHE.get(&key).and_then(|entry| { + if entry.is_expired() { + VARIABLE_CACHE.remove(&key); + None + } else { + tracing::debug!("Cache hit for variable {}", key); + Some(entry.value.clone()) + } + }) +} + +/// Cache variable data +pub fn cache_variable(workspace_id: &str, path: &str, email: &str, variable: String) { + let key = format!("{}:{}", email, cache_key(workspace_id, path)); + let entry = CacheEntry::new(variable); + VARIABLE_CACHE.insert(key.clone(), entry); + tracing::debug!("Cached variable {}", key); +} + +/// Get cached resource if available and not expired +pub fn get_cached_resource(workspace_id: &str, path: &str) -> Option { + let key = cache_key(workspace_id, path); + RESOURCE_CACHE.get(&key).and_then(|entry| { + if entry.is_expired() { + RESOURCE_CACHE.remove(&key); + None + } else { + tracing::debug!("Cache hit for resource {}", key); + Some(entry.value.clone()) + } + }) +} + +/// Cache resource data +pub fn cache_resource(workspace_id: &str, path: &str, resource: Value) { + let key = cache_key(workspace_id, path); + let entry = CacheEntry::new(resource); + RESOURCE_CACHE.insert(key.clone(), entry); + tracing::debug!("Cached resource {}", key); +} + +/// Invalidate specific variable from cache +pub fn invalidate_variable_cache(workspace_id: &str, path: &str) { + let key = cache_key(workspace_id, path); + VARIABLE_CACHE.remove(&key); + tracing::info!("Variable cache invalidated for {}", key); +} + +/// Invalidate specific resource from cache +pub fn invalidate_resource_cache(workspace_id: &str, path: &str) { + let key = cache_key(workspace_id, path); + RESOURCE_CACHE.remove(&key); + tracing::info!("Resource cache invalidated for {}", key); +} + +/// Clear all caches (for testing/debugging) +#[allow(dead_code)] +pub fn clear_all_caches() { + VARIABLE_CACHE.clear(); + RESOURCE_CACHE.clear(); + tracing::debug!("All variable/resource caches cleared"); +} \ No newline at end of file diff --git a/backend/windmill-api/src/variables.rs b/backend/windmill-api/src/variables.rs index f7476d9f26..ff3a0110fc 100644 --- a/backend/windmill-api/src/variables.rs +++ b/backend/windmill-api/src/variables.rs @@ -39,6 +39,7 @@ use serde::Deserialize; use sqlx::{Postgres, Transaction}; use windmill_common::variables::{decrypt, encrypt}; use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; +use crate::var_resource_cache::{get_cached_variable, cache_variable}; lazy_static! { pub static ref SECRET_SALT: Option = std::env::var("SECRET_SALT").ok(); @@ -77,6 +78,7 @@ async fn list_contextual_variables( Some("u/user/triggering_flow_path".to_string()), Some("c".to_string()), Some("017e0ad5-f499-73b6-5488-92a61c5196dd".to_string()), + Some("017e0ad5-f499-73b6-5488-92a61c5196dd".to_string()), Some(chrono::offset::Utc::now()), Some(ScriptHash(1234567890)), ) @@ -144,6 +146,7 @@ async fn get_variable( ) -> JsonResult { let path = path.to_path(); check_scopes(&authed, || format!("variables:read:{}", path))?; + let mut tx = user_db.begin(&authed).await?; let variable_o = sqlx::query_as::<_, ListableVariable>( @@ -217,20 +220,25 @@ async fn get_variable( variable }; + Ok(Json(r)) } +#[derive(Deserialize)] +struct GetValueQuery { + allow_cache: Option, +} async fn get_value( authed: ApiAuthed, Extension(user_db): Extension, Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, + Query(q): Query, ) -> JsonResult { let path = path.to_path(); check_scopes(&authed, || format!("variables:read:{}", path))?; let tx = user_db.begin(&authed).await?; - return get_value_internal(tx, &db, &w_id, &path, &authed) + return get_value_internal(tx, &db, &w_id, &path, &authed, q.allow_cache.unwrap_or(false)) .await .map(Json); } @@ -692,7 +700,16 @@ pub async fn get_value_internal<'c>( w_id: &str, path: &str, audit_author: &impl AuditAuthorable, + allow_cache: bool, ) -> Result { + + + if allow_cache { + if let Some(cached_variable) = get_cached_variable(&w_id, &path) { + return Ok(cached_variable); + } + } + let variable_o = sqlx::query!( "SELECT value, account, (now() > account.expires_at) as is_expired, is_secret, path from variable LEFT JOIN account ON variable.account = account.id WHERE variable.path = $1 AND variable.workspace_id = $2", path, w_id @@ -708,6 +725,8 @@ pub async fn get_value_internal<'c>( unreachable!() }; + + let r = if variable.is_secret { audit_log( &mut *tx, @@ -745,6 +764,12 @@ pub async fn get_value_internal<'c>( variable.value }; + // Cache the result when explicitly allowed and caching appropriate + if allow_cache { + cache_variable(&w_id, &path, audit_author.email(), r.clone()); + } + + Ok(r) } diff --git a/backend/windmill-api/src/websocket_triggers.rs b/backend/windmill-api/src/websocket_triggers.rs index a75652e555..43e234a8c7 100644 --- a/backend/windmill-api/src/websocket_triggers.rs +++ b/backend/windmill-api/src/websocket_triggers.rs @@ -1,11 +1,6 @@ use anyhow::Context; -use axum::{ - extract::{Path, Query}, - routing::{delete, get, post}, - Extension, Json, Router, -}; + use futures::{stream::SplitSink, SinkExt, StreamExt}; -use http::StatusCode; use itertools::Itertools; use rand::seq::SliceRandom; use serde::{ @@ -13,24 +8,18 @@ use serde::{ Deserialize, Deserializer, Serialize, }; use serde_json::{value::RawValue, Value}; -use sql_builder::{bind::Bind, SqlBuilder}; use sqlx::prelude::FromRow; use sqlx::types::Json as SqlxJson; use std::{collections::HashMap, fmt}; use tokio::net::TcpStream; use tokio_tungstenite::{connect_async, tungstenite::Message, MaybeTlsStream, WebSocketStream}; -use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::{ - db::UserDB, - error::{self, to_anyhow, JsonResult}, + error::{self, to_anyhow}, triggers::TriggerKind, - utils::{ - empty_as_none, not_found_if_none, paginate, report_critical_error, Pagination, StripPath, - }, - worker::{to_raw_value, CLOUD_HOSTED}, + utils::report_critical_error, + worker::to_raw_value, INSTANCE_NAME, }; -use windmill_git_sync::handle_deployment_metadata; use windmill_queue::PushArgsOwned; use crate::{ @@ -38,40 +27,10 @@ use crate::{ db::{ApiAuthed, DB}, trigger_helpers::{trigger_runnable, trigger_runnable_and_wait_for_raw_result, TriggerJobArgs}, users::fetch_api_authed, - utils::check_scopes, }; use std::borrow::Cow; -pub fn workspaced_service() -> Router { - Router::new() - .route("/create", post(create_websocket_trigger)) - .route("/list", get(list_websocket_triggers)) - .route("/get/*path", get(get_websocket_trigger)) - .route("/update/*path", post(update_websocket_trigger)) - .route("/delete/*path", delete(delete_websocket_trigger)) - .route("/exists/*path", get(exists_websocket_trigger)) - .route("/setenabled/*path", post(set_enabled)) - .route("/test", post(test_websocket_connection)) -} - -#[derive(Deserialize)] -struct NewWebsocketTrigger { - path: String, - url: String, - script_path: String, - is_flow: bool, - enabled: Option, - filters: Vec>, - initial_messages: Option>>, - url_runnable_args: Option>, - can_return_message: bool, - #[serde(default, deserialize_with = "empty_as_none")] - error_handler_path: Option, - error_handler_args: Option>>>, - retry: Option>, -} - #[derive(Deserialize)] pub struct JsonFilter { key: String, @@ -123,496 +82,6 @@ pub struct WebsocketTrigger { pub retry: Option>, } -#[derive(Deserialize)] -struct EditWebsocketTrigger { - path: String, - url: String, - script_path: String, - is_flow: bool, - filters: Vec>, - initial_messages: Option>>, - url_runnable_args: Option>, - can_return_message: bool, - error_handler_path: Option, - error_handler_args: Option>>>, - retry: Option>, -} - -#[derive(Deserialize)] -pub struct ListWebsocketTriggerQuery { - pub page: Option, - pub per_page: Option, - pub path: Option, - pub is_flow: Option, - pub path_start: Option, -} - -async fn list_websocket_triggers( - authed: ApiAuthed, - Extension(user_db): Extension, - Path(w_id): Path, - Query(lst): Query, -) -> error::JsonResult> { - let mut tx = user_db.begin(&authed).await?; - let (per_page, offset) = paginate(Pagination { per_page: lst.per_page, page: lst.page }); - let mut sqlb = SqlBuilder::select_from("websocket_trigger"); - - sqlb.fields(&[ - "workspace_id", - "path", - "url", - "script_path", - "is_flow", - "edited_by", - "email", - "edited_at", - "server_id", - "last_server_ping", - "extra_perms", - "error", - "enabled", - "filters", - "initial_messages", - "url_runnable_args", - "can_return_message", - "error_handler_path", - "error_handler_args", - "retry", - ]) - .order_by("edited_at", true) - .and_where("workspace_id = ?".bind(&w_id)) - .offset(offset) - .limit(per_page); - if let Some(path) = lst.path { - sqlb.and_where_eq("script_path", "?".bind(&path)); - } - if let Some(is_flow) = lst.is_flow { - sqlb.and_where_eq("is_flow", "?".bind(&is_flow)); - } - if let Some(path_start) = &lst.path_start { - sqlb.and_where_like_left("path", path_start); - } - let sql = sqlb - .sql() - .map_err(|e| error::Error::internal_err(e.to_string()))?; - let rows = sqlx::query_as::<_, WebsocketTrigger>(&sql) - .fetch_all(&mut *tx) - .await?; - tx.commit().await?; - - Ok(Json(rows)) -} - -async fn get_websocket_trigger( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> error::JsonResult { - let path = path.to_path(); - check_scopes(&authed, || format!("websocket_triggers:read:{}", path))?; - let mut tx = user_db.begin(&authed).await?; - let trigger = sqlx::query_as::<_, WebsocketTrigger>( - r#" - SELECT - workspace_id, - path, - url, - script_path, - is_flow, - edited_by, - email, - edited_at, - server_id, - last_server_ping, - extra_perms, - error, - enabled, - filters, - initial_messages, - url_runnable_args, - can_return_message, - error_handler_path, - error_handler_args, - retry - FROM - websocket_trigger - WHERE - workspace_id = $1 AND path = $2 - "#, - ) - .bind(w_id) - .bind(path) - .fetch_optional(&mut *tx) - .await?; - - tx.commit().await?; - - let trigger = not_found_if_none(trigger, "Trigger", path)?; - - Ok(Json(trigger)) -} - -async fn create_websocket_trigger( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path(w_id): Path, - Json(ct): Json, -) -> error::Result<(StatusCode, String)> { - check_scopes(&authed, || format!("websocket_triggers:write:{}", ct.path))?; - - if *CLOUD_HOSTED { - return Err(error::Error::BadRequest( - "WebSocket triggers are not supported on multi-tenant cloud, use dedicated cloud or self-host".to_string(), - )); - } - - let mut tx = user_db.begin(&authed).await?; - - let filters = ct.filters.into_iter().map(SqlxJson).collect_vec(); - let initial_messages = ct - .initial_messages - .unwrap_or_default() - .into_iter() - .map(SqlxJson) - .collect_vec(); - sqlx::query!( - r#" - INSERT INTO websocket_trigger ( - workspace_id, - path, - url, - script_path, - is_flow, - enabled, - filters, - initial_messages, - url_runnable_args, - edited_by, - can_return_message, - email, - edited_at, - error_handler_path, - error_handler_args, - retry - ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now(), $13, $14, $15 - ) - "#, - w_id, - ct.path, - ct.url, - ct.script_path, - ct.is_flow, - ct.enabled.unwrap_or(true), - &filters as _, - &initial_messages as _, - ct.url_runnable_args.map(SqlxJson) as _, - authed.username, - ct.can_return_message, - authed.email, - ct.error_handler_path, - ct.error_handler_args as _, - ct.retry as _ - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "websocket_triggers.create", - ActionKind::Create, - &w_id, - Some(ct.path.as_str()), - None, - ) - .await?; - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::WebsocketTrigger { path: ct.path.clone() }, - Some(format!("WebSocket trigger '{}' created", ct.path)), - true, - ) - .await?; - - Ok((StatusCode::CREATED, format!("{}", ct.path))) -} - -async fn update_websocket_trigger( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Json(ct): Json, -) -> error::Result { - let path = path.to_path(); - check_scopes(&authed, || format!("websocket_triggers:write:{}", path))?; - let mut tx = user_db.begin(&authed).await?; - - let filters = ct.filters.into_iter().map(SqlxJson).collect_vec(); - let initial_messages = ct - .initial_messages - .unwrap_or_default() - .into_iter() - .map(SqlxJson) - .collect_vec(); - - // important to update server_id to NULL to stop current websocket listener - sqlx::query!( - " - UPDATE - websocket_trigger - SET - url = $1, - script_path = $2, - path = $3, - is_flow = $4, - filters = $5, - initial_messages = $6, - url_runnable_args = $7, - edited_by = $8, - email = $9, - can_return_message = $10, - edited_at = now(), - server_id = NULL, - error = NULL, - error_handler_path = $13, - error_handler_args = $14, - retry = $15 - WHERE - workspace_id = $11 AND path = $12 - ", - ct.url, - ct.script_path, - ct.path, - ct.is_flow, - filters.as_slice() as &[SqlxJson>], - initial_messages.as_slice() as &[SqlxJson>], - ct.url_runnable_args.map(SqlxJson) as Option>>, - &authed.username, - &authed.email, - ct.can_return_message, - w_id, - path, - ct.error_handler_path, - ct.error_handler_args as _, - ct.retry as _ - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "websocket_triggers.update", - ActionKind::Update, - &w_id, - Some(&ct.path), - None, - ) - .await?; - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::WebsocketTrigger { path: ct.path.clone() }, - Some(format!("WebSocket trigger '{}' updated", ct.path)), - true, - ) - .await?; - - Ok(ct.path.to_string()) -} - -#[derive(Deserialize)] -pub struct SetEnabled { - pub enabled: bool, -} - -pub async fn set_enabled( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, - Json(payload): Json, -) -> error::Result { - let path = path.to_path(); - check_scopes(&authed, || format!("websocket_triggers:write:{}", path))?; - let mut tx = user_db.begin(&authed).await?; - - // important to set server_id, last_server_ping and error to NULL to stop current websocket listener - let one_o = sqlx::query_scalar!( - "UPDATE websocket_trigger SET enabled = $1, email = $2, edited_by = $3, edited_at = now(), server_id = NULL, error = NULL - WHERE path = $4 AND workspace_id = $5 RETURNING 1", - payload.enabled, - &authed.email, - &authed.username, - path, - w_id, - ).fetch_optional(&mut *tx).await?; - - not_found_if_none(one_o.flatten(), "WebSocket trigger", path)?; - - audit_log( - &mut *tx, - &authed, - "websocket_triggers.setenabled", - ActionKind::Update, - &w_id, - Some(path), - Some([("enabled", payload.enabled.to_string().as_ref())].into()), - ) - .await?; - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::WebsocketTrigger { path: path.to_string() }, - Some(format!("WebSocket trigger '{}' updated", path)), - true, - ) - .await?; - - Ok(format!( - "succesfully updated WebSocket trigger at path {} to status {}", - path, payload.enabled - )) -} - -async fn delete_websocket_trigger( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> error::Result { - let path = path.to_path(); - check_scopes(&authed, || format!("websocket_triggers:write:{}", path))?; - - let mut tx = user_db.begin(&authed).await?; - sqlx::query!( - "DELETE FROM websocket_trigger WHERE workspace_id = $1 AND path = $2", - w_id, - path, - ) - .execute(&mut *tx) - .await?; - - audit_log( - &mut *tx, - &authed, - "websocket_triggers.delete", - ActionKind::Delete, - &w_id, - Some(path), - None, - ) - .await?; - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::WebsocketTrigger { path: path.to_string() }, - Some(format!("WebSocket trigger '{}' deleted", path)), - true, - ) - .await?; - - Ok(format!("WebSocket trigger {path} deleted")) -} - -async fn exists_websocket_trigger( - Extension(db): Extension, - Path((w_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM websocket_trigger WHERE path = $1 AND workspace_id = $2)", - path, - w_id, - ) - .fetch_one(&db) - .await? - .unwrap_or(false); - Ok(Json(exists)) -} - -#[derive(Debug, Deserialize)] -struct TestWebsocket { - url: String, - url_runnable_args: Option>, -} - -async fn test_websocket_connection( - authed: ApiAuthed, - Extension(db): Extension, - Path(workspace_id): Path, - Json(test_websocket): Json, -) -> error::Result<()> { - let url = test_websocket.url; - - let connect_f = async { - let connect_url: Cow = if url.starts_with("$") { - if url.starts_with("$flow:") || url.starts_with("$script:") { - let path = url.splitn(2, ':').nth(1).unwrap(); - Cow::Owned( - get_url_from_runnable( - path, - url.starts_with("$flow:"), - &db, - authed, - test_websocket.url_runnable_args.as_ref(), - &workspace_id, - ) - .await?, - ) - } else { - return Err(error::Error::BadConfig(format!( - "Invalid WebSocket runnable path: {}", - url - ))); - } - } else { - Cow::Borrowed(&url) - }; - - connect_async(connect_url.as_ref()).await.map_err(|err| { - error::Error::BadConfig(format!( - "Error connecting to WebSocket: {}", - err.to_string() - )) - })?; - - Ok(()) - }; - - tokio::time::timeout(tokio::time::Duration::from_secs(30), connect_f) - .await - .map_err(|_| { - error::Error::BadConfig(format!("Timeout connecting to WebSocket after 30 seconds")) - })??; - - Ok(()) -} - async fn listen_to_unlistened_websockets( db: &DB, killpill_rx: &tokio::sync::broadcast::Receiver<()>, diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index fb0ea74f26..c2b9e448c2 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -720,6 +720,7 @@ async fn edit_deploy_to() -> Result { } pub const BANNED_DOMAINS: &str = include_str!("../banned_domains.txt"); +pub const MAX_CUSTOM_PROMPT_LENGTH: usize = 5000; async fn is_allowed_auto_domain(ApiAuthed { email, .. }: ApiAuthed) -> JsonResult { let domain = email.split('@').last().unwrap(); @@ -819,6 +820,20 @@ async fn edit_copilot_config( ) -> Result { require_admin(is_admin, &username)?; + // Validate custom prompts length + if let Some(ref custom_prompts) = ai_config.custom_prompts { + for (mode, prompt) in custom_prompts.iter() { + if prompt.len() > MAX_CUSTOM_PROMPT_LENGTH { + return Err(Error::BadRequest(format!( + "Custom prompt for mode '{}' exceeds maximum length of {} characters (current: {})", + mode, + MAX_CUSTOM_PROMPT_LENGTH, + prompt.len() + ))); + } + } + } + let mut tx = db.begin().await?; sqlx::query!( @@ -887,6 +902,7 @@ async fn get_copilot_info( providers: None, default_model: None, code_completion_model: None, + custom_prompts: None, })) } } diff --git a/backend/windmill-api/src/workspaces_export.rs b/backend/windmill-api/src/workspaces_export.rs index 1b1f107be6..7758e024b7 100644 --- a/backend/windmill-api/src/workspaces_export.rs +++ b/backend/windmill-api/src/workspaces_export.rs @@ -17,6 +17,24 @@ use crate::{ resources::{Resource, ResourceType}, }; +#[cfg(any( + feature = "http_trigger", + feature = "websocket", + feature = "postgres_trigger", + feature = "mqtt_trigger", + all( + feature = "enterprise", + any( + feature = "kafka", + feature = "sqs_trigger", + feature = "gcp_trigger", + feature = "nats", + ), + feature = "private" + ) +))] +use crate::triggers::TriggerCrud; + use axum::{ extract::{Extension, Path, Query}, response::IntoResponse, @@ -537,357 +555,137 @@ pub(crate) async fn tarball_workspace( if include_triggers.unwrap_or(false) { #[cfg(feature = "http_trigger")] { - let http_triggers = sqlx::query_as!( - crate::http_triggers::HttpTrigger, - r#" - SELECT - workspace_id, - workspaced_route, - path, - route_path, - route_path_key, - authentication_resource_path, - script_path, - is_flow, - summary, - description, - edited_by, - edited_at, - email, - extra_perms, - is_async, - authentication_method AS "authentication_method: _", - http_method AS "http_method: _", - static_asset_config AS "static_asset_config: _", - is_static_website, - wrap_body, - raw_string, - error_handler_path, - error_handler_args as "error_handler_args: _", - retry as "retry: _" - FROM - http_trigger - WHERE - workspace_id = $1 - "#, - &w_id - ) - .fetch_all(&mut *tx) - .await?; + use crate::triggers::http::handler::HttpTrigger; + let handler = HttpTrigger; + let http_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?; for trigger in http_triggers { let trigger_str = &to_string_without_metadata(&trigger, false, None).unwrap(); archive - .write_to_archive(&trigger_str, &format!("{}.http_trigger.json", trigger.path)) + .write_to_archive( + &trigger_str, + &format!("{}.http_trigger.json", trigger.base.path), + ) .await?; } } #[cfg(feature = "websocket")] { - let websocket_triggers = sqlx::query_as!( - crate::websocket_triggers::WebsocketTrigger, - r#" - SELECT - workspace_id, - path, - url, - script_path, - is_flow, - edited_by, - email, - edited_at, - server_id, - last_server_ping, - extra_perms, - error, - enabled, - filters AS "filters: _", - initial_messages AS "initial_messages: _", - url_runnable_args AS "url_runnable_args: _", - can_return_message, - error_handler_path, - error_handler_args as "error_handler_args: _", - retry as "retry: _" - FROM - websocket_trigger - WHERE - workspace_id = $1 - "#, - &w_id - ) - .fetch_all(&mut *tx) - .await?; + use crate::triggers::websocket::WebsocketTrigger; + let handler = WebsocketTrigger; + let websocket_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?; for trigger in websocket_triggers { let trigger_str = &to_string_without_metadata(&trigger, false, None).unwrap(); archive .write_to_archive( &trigger_str, - &format!("{}.websocket_trigger.json", trigger.path), + &format!("{}.websocket_trigger.json", trigger.base.path), ) .await?; } } - #[cfg(all(feature = "enterprise", feature = "kafka"))] + #[cfg(all(feature = "enterprise", feature = "kafka", feature = "private"))] { - let kafka_triggers = sqlx::query_as!( - crate::kafka_triggers_oss::KafkaTrigger, - r#"SELECT - workspace_id, - path, - kafka_resource_path, - group_id, - topics, - script_path, - is_flow, - edited_by, - email, - edited_at, - server_id, - last_server_ping, - extra_perms, - error, - enabled, - error_handler_path, - error_handler_args as "error_handler_args: _", - retry as "retry: _" - FROM kafka_trigger - WHERE workspace_id = $1"#, - &w_id - ) - .fetch_all(&mut *tx) - .await?; + use crate::triggers::kafka::KafkaTrigger; + let handler = KafkaTrigger; + let kafka_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?; for trigger in kafka_triggers { let trigger_str = &to_string_without_metadata(&trigger, false, None).unwrap(); archive .write_to_archive( &trigger_str, - &format!("{}.kafka_trigger.json", trigger.path), + &format!("{}.kafka_trigger.json", trigger.base.path), ) .await?; } } - #[cfg(all(feature = "enterprise", feature = "sqs_trigger"))] + #[cfg(all(feature = "enterprise", feature = "sqs_trigger", feature = "private"))] { - let sqs_triggers = sqlx::query_as!( - crate::sqs_triggers_oss::SqsTrigger, - r#" - SELECT - aws_auth_resource_type AS "aws_auth_resource_type: _", - aws_resource_path, - message_attributes, - queue_url, - workspace_id, - path, - script_path, - is_flow, - edited_by, - email, - edited_at, - server_id, - last_server_ping, - extra_perms, - error, - enabled, - error_handler_path, - error_handler_args as "error_handler_args: _", - retry as "retry: _" - FROM - sqs_trigger - WHERE - workspace_id = $1 - "#, - &w_id - ) - .fetch_all(&mut *tx) - .await?; + use crate::triggers::sqs::SqsTrigger; + let handler = SqsTrigger; + let sqs_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?; for trigger in sqs_triggers { let trigger_str = &to_string_without_metadata(&trigger, false, None).unwrap(); archive - .write_to_archive(&trigger_str, &format!("{}.sqs_trigger.json", trigger.path)) + .write_to_archive( + &trigger_str, + &format!("{}.sqs_trigger.json", trigger.base.path), + ) .await?; } } - #[cfg(all(feature = "enterprise", feature = "gcp_trigger"))] + #[cfg(all(feature = "enterprise", feature = "gcp_trigger", feature = "private"))] { - let gcp_triggers = sqlx::query_as!( - crate::gcp_triggers_oss::GcpTrigger, - r#" - SELECT - gcp_resource_path, - subscription_id, - topic_id, - workspace_id, - delivery_type AS "delivery_type: _", - delivery_config AS "delivery_config: _", - subscription_mode AS "subscription_mode: _", - path, - script_path, - is_flow, - edited_by, - email, - edited_at, - server_id, - last_server_ping, - extra_perms, - error, - enabled, - error_handler_path, - error_handler_args as "error_handler_args: _", - retry as "retry: _", - auto_acknowledge_msg - FROM - gcp_trigger - WHERE - workspace_id = $1 - "#, - &w_id - ) - .fetch_all(&mut *tx) - .await?; + use crate::triggers::gcp::GcpTrigger; + let handler = GcpTrigger; + let gcp_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?; for trigger in gcp_triggers { let trigger_str = &to_string_without_metadata(&trigger, false, None).unwrap(); archive - .write_to_archive(&trigger_str, &format!("{}.gcp_trigger.json", trigger.path)) + .write_to_archive( + &trigger_str, + &format!("{}.gcp_trigger.json", trigger.base.path), + ) .await?; } } - #[cfg(all(feature = "enterprise", feature = "nats"))] + #[cfg(all(feature = "enterprise", feature = "nats", feature = "private"))] { - let nats_triggers = sqlx::query_as!( - crate::nats_triggers_oss::NatsTrigger, - r#" - SELECT - workspace_id, - path, - nats_resource_path, - subjects, - stream_name, - consumer_name, - use_jetstream, - script_path, - is_flow, - edited_by, - email, - edited_at, - server_id, - last_server_ping, - extra_perms, - error, - enabled, - error_handler_path, - error_handler_args as "error_handler_args: _", - retry as "retry: _" - FROM - nats_trigger - WHERE - workspace_id = $1 - "#, - &w_id - ) - .fetch_all(&mut *tx) - .await?; + use crate::triggers::nats::NatsTrigger; + let handler = NatsTrigger; + let nats_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?; for trigger in nats_triggers { let trigger_str: &String = &to_string_without_metadata(&trigger, false, None).unwrap(); archive - .write_to_archive(&trigger_str, &format!("{}.nats_trigger.json", trigger.path)) + .write_to_archive( + &trigger_str, + &format!("{}.nats_trigger.json", trigger.base.path), + ) .await?; } } #[cfg(feature = "postgres_trigger")] { - let postgres_triggers = sqlx::query_as!( - crate::postgres_triggers::PostgresTrigger, - r#" - SELECT - workspace_id, - path, - script_path, - is_flow, - edited_by, - email, - edited_at, - server_id, - last_server_ping, - extra_perms, - error, - enabled, - replication_slot_name, - publication_name, - postgres_resource_path, - error_handler_path, - error_handler_args as "error_handler_args: _", - retry as "retry: _" - FROM - postgres_trigger - WHERE - workspace_id = $1 - "#, - &w_id - ) - .fetch_all(&mut *tx) - .await?; + use crate::triggers::postgres::PostgresTrigger; + let handler = PostgresTrigger; + let postgres_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?; for trigger in postgres_triggers { let trigger_str = &to_string_without_metadata(&trigger, false, None).unwrap(); archive .write_to_archive( &trigger_str, - &format!("{}.postgres_trigger.json", trigger.path), + &format!("{}.postgres_trigger.json", trigger.base.path), ) .await?; } } - #[cfg(all(feature = "enterprise", feature = "mqtt_trigger"))] + #[cfg(feature = "mqtt_trigger")] { - let mqtt_triggers = sqlx::query_as!( - crate::mqtt_triggers::MqttTrigger, - r#" - SELECT - mqtt_resource_path, - subscribe_topics as "subscribe_topics: _", - v3_config as "v3_config: _", - v5_config as "v5_config: _", - client_version AS "client_version: _", - client_id, - workspace_id, - path, - script_path, - is_flow, - edited_by, - email, - edited_at, - server_id, - last_server_ping, - extra_perms, - error, - enabled, - error_handler_path, - error_handler_args as "error_handler_args: _", - retry as "retry: _" - FROM - mqtt_trigger - "#, - ) - .fetch_all(&mut *tx) - .await?; + use crate::triggers::mqtt::MqttTrigger; + let handler = MqttTrigger; + let mqtt_triggers = handler.list_triggers(&mut *tx, &w_id, None).await?; for trigger in mqtt_triggers { let trigger_str = &to_string_without_metadata(&trigger, false, None).unwrap(); archive - .write_to_archive(&trigger_str, &format!("{}.mqtt_trigger.json", trigger.path)) + .write_to_archive( + &trigger_str, + &format!("{}.mqtt_trigger.json", trigger.base.path), + ) .await?; } } diff --git a/backend/windmill-common/src/cache.rs b/backend/windmill-common/src/cache.rs index f43d14845b..4890b9715b 100644 --- a/backend/windmill-common/src/cache.rs +++ b/backend/windmill-common/src/cache.rs @@ -539,7 +539,7 @@ pub mod flow { } pub mod script { - use crate::{worker::Connection, DB}; + use crate::{db::DB, worker::Connection}; use super::*; diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index 60583ef50f..5a905c65ca 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -20,11 +20,11 @@ use sqlx::types::JsonRawValue; use crate::{ assets::AssetWithAltAccessType, cache, + db::DB, error::Error, more_serde::{default_empty_string, default_id, default_null, default_true, is_default}, scripts::{Schema, ScriptHash, ScriptLang}, worker::{to_raw_value, Connection}, - DB, }; #[derive(Debug, Serialize, Deserialize, sqlx::FromRow)] diff --git a/backend/windmill-common/src/indexer.rs b/backend/windmill-common/src/indexer.rs index e581196af8..45a924748d 100644 --- a/backend/windmill-common/src/indexer.rs +++ b/backend/windmill-common/src/indexer.rs @@ -1,6 +1,6 @@ use serde::{Deserialize, Serialize}; -use crate::{error, DB}; +use crate::{db::DB, error}; pub const BYTES_PER_MB: u64 = 1_048_576; pub const BYTES_PER_KB: u64 = 1024; diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index ba5c6c006e..0c53426eab 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -5,7 +5,7 @@ use futures_core::Stream; use indexmap::IndexMap; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; -use sqlx::{types::Json, Pool, Postgres}; +use sqlx::{types::Json, Postgres}; use tokio::io::AsyncReadExt; use uuid::Uuid; @@ -17,6 +17,7 @@ pub const EMAIL_ERROR_HANDLER_USER_EMAIL: &str = "email_error_handler@windmill.d use crate::{ apps::AppScriptId, auth::is_super_admin_email, + db::DB, error::{self, to_anyhow, Error}, flow_status::{FlowStatus, RestartedFrom}, flows::{FlowNodeId, FlowValue, Retry}, @@ -28,6 +29,42 @@ use crate::{ FlowVersionInfo, ScriptHashInfo, }; +#[derive(sqlx::Type, Serialize, Deserialize, Debug, Clone)] +#[sqlx(type_name = "JOB_TRIGGER_KIND", rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +pub enum JobTriggerKind { + Webhook, + Http, + Websocket, + Kafka, + Email, + Nats, + Mqtt, + Sqs, + Postgres, + Schedule, + Gcp, +} + +impl std::fmt::Display for JobTriggerKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let kind = match self { + JobTriggerKind::Webhook => "webhook", + JobTriggerKind::Http => "http", + JobTriggerKind::Websocket => "websocket", + JobTriggerKind::Kafka => "kafka", + JobTriggerKind::Email => "email", + JobTriggerKind::Nats => "nats", + JobTriggerKind::Mqtt => "mqtt", + JobTriggerKind::Sqs => "sqs", + JobTriggerKind::Postgres => "postgres", + JobTriggerKind::Schedule => "schedule", + JobTriggerKind::Gcp => "gcp", + }; + write!(f, "{}", kind) + } +} + #[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Copy, Clone)] #[sqlx(type_name = "JOB_KIND", rename_all = "lowercase")] #[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] @@ -397,8 +434,6 @@ pub struct RawCode { type Tag = String; -pub type DB = Pool; - #[derive(Clone, Debug)] pub struct OnBehalfOf { pub email: String, diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index adf58871ba..ed35ab84bf 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -22,7 +22,7 @@ use tokio::sync::broadcast; use ee_oss::CriticalErrorChannel; use error::Error; use scripts::ScriptLang; -use sqlx::{Pool, Postgres}; +use sqlx::Postgres; pub mod agent_workers; pub mod apps; @@ -55,7 +55,7 @@ pub mod job_s3_helpers_oss; pub mod oidc_ee; #[cfg(all(feature = "enterprise", feature = "openidconnect"))] pub mod oidc_oss; - +pub mod triggers; pub mod jobs; pub mod jwt; pub mod more_serde; @@ -81,7 +81,6 @@ pub mod utils; pub mod variables; pub mod worker; pub mod workspaces; -pub mod triggers; pub mod result_stream; pub mod stream; @@ -422,7 +421,7 @@ pub async fn connect( type Tag = String; -pub type DB = Pool; +pub use db::DB; #[derive(Clone)] pub struct ExpiringLatestVersionId { diff --git a/backend/windmill-common/src/triggers.rs b/backend/windmill-common/src/triggers.rs index df68d0543c..08356f24ad 100644 --- a/backend/windmill-common/src/triggers.rs +++ b/backend/windmill-common/src/triggers.rs @@ -1,31 +1,8 @@ +use lazy_static::lazy_static; use quick_cache::sync::Cache; use serde::{Deserialize, Serialize}; use std::fmt; -use strum::EnumIter; - -#[derive(Eq, PartialEq, Hash)] -pub enum HubOrWorkspaceId { - Hub, - WorkspaceId(String), -} - -type RunnableFormatCacheKey = (HubOrWorkspaceId, i64, TriggerKind); - -#[derive(Debug, Clone, Eq, PartialEq, Hash, Copy)] -pub struct RunnableFormat { - pub version: RunnableFormatVersion, - pub has_preprocessor: bool, -} - -#[derive(Debug, Clone, Eq, PartialEq, Hash, Copy)] -pub enum RunnableFormatVersion { - V1, - V2, -} - -lazy_static::lazy_static! { - pub static ref RUNNABLE_FORMAT_VERSION_CACHE: Cache = Cache::new(1000); -} +use strum_macros::EnumIter; #[derive(sqlx::Type, Serialize, Deserialize, Debug, Clone, Eq, PartialEq, Hash, EnumIter)] #[sqlx(type_name = "TRIGGER_KIND", rename_all = "lowercase")] @@ -77,3 +54,28 @@ impl fmt::Display for TriggerKind { write!(f, "{}", s) } } + +#[derive(Eq, PartialEq, Hash)] +pub enum HubOrWorkspaceId { + Hub, + WorkspaceId(String), +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash, Copy)] +pub struct RunnableFormat { + pub version: RunnableFormatVersion, + pub has_preprocessor: bool, +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash, Copy)] +pub enum RunnableFormatVersion { + V1, + V2, +} + +pub type RunnableFormatCacheKey = (HubOrWorkspaceId, i64, TriggerKind); + +lazy_static! { + pub static ref RUNNABLE_FORMAT_VERSION_CACHE: Cache = + Cache::new(1000); +} diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index f43292755e..ae81512a0a 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -29,7 +29,7 @@ pub struct ContextualVariable { pub is_custom: bool, } -#[derive(Serialize, Deserialize, sqlx::FromRow)] +#[derive(Serialize, Deserialize, sqlx::FromRow, Clone)] pub struct ListableVariable { pub workspace_id: String, @@ -210,7 +210,8 @@ pub async fn get_reserved_variables( flow_path: Option, schedule_path: Option, step_id: Option, - root_flow_id: Option, + flow_innermost_root_job: Option, + root_job_id: Option, scheduled_for: Option>, runnable_id: Option, ) -> Vec { @@ -322,8 +323,14 @@ pub async fn get_reserved_variables( }, ContextualVariable { name: "WM_ROOT_FLOW_JOB_ID".to_string(), - value: root_flow_id.unwrap_or_else(|| "".to_string()), - description: "Job id of the root flow if the job is a flow step".to_string(), + value: flow_innermost_root_job.unwrap_or_else(|| "".to_string()), + description: "Job id of the innermost root flow if the job is a flow step".to_string(), + is_custom: false, + }, + ContextualVariable { + name: "WM_ROOT_JOB_ID".to_string(), + value: root_job_id.unwrap_or_else(|| "".to_string()), + description: "Job id of the root job".to_string(), is_custom: false, }, ContextualVariable { diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 7ec4cd09f5..745102bf66 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -380,7 +380,7 @@ fn format_pull_query(peek: String) -> String { id, workspace_id, parent_job, created_by, created_at, runnable_id, runnable_path, args, kind, trigger, trigger_kind, permissioned_as, permissioned_as_email, script_lang, - flow_innermost_root_job, flow_step_id, + flow_innermost_root_job, root_job, flow_step_id, same_worker, pre_run_error, visible_to_owner, tag, concurrent_limit, concurrency_time_window_s, timeout, cache_ttl, priority, raw_code, raw_lock, raw_flow, script_entrypoint_override, preprocessed @@ -391,7 +391,7 @@ fn format_pull_query(peek: String) -> String { canceled_reason, j.kind, j.trigger, j.trigger_kind, j.permissioned_as, flow_status, j.script_lang, j.same_worker, j.pre_run_error, j.visible_to_owner, - j.tag, j.concurrent_limit, j.concurrency_time_window_s, j.flow_innermost_root_job, + j.tag, j.concurrent_limit, j.concurrency_time_window_s, j.flow_innermost_root_job, j.root_job, j.timeout, j.flow_step_id, j.cache_ttl, j.priority, j.raw_code, j.raw_lock, j.raw_flow, j.script_entrypoint_override, j.preprocessed, pj.runnable_path as parent_runnable_path, COALESCE(p.email, j.permissioned_as_email) as permissioned_as_email, p.username as permissioned_as_username, p.is_admin as permissioned_as_is_admin, diff --git a/backend/windmill-git-sync/src/lib.rs b/backend/windmill-git-sync/src/lib.rs index 55ab86af3f..df63736acf 100644 --- a/backend/windmill-git-sync/src/lib.rs +++ b/backend/windmill-git-sync/src/lib.rs @@ -6,16 +6,13 @@ * LICENSE-AGPL for a copy of the license. */ -use sqlx::{Pool, Postgres}; - -use windmill_common::scripts::ScriptHash; +use windmill_common::{scripts::ScriptHash, DB}; #[cfg(feature = "private")] pub mod git_sync_ee; pub mod git_sync_oss; pub use git_sync_oss::handle_deployment_metadata; -pub type DB = Pool; #[derive(Clone, Debug)] pub enum DeployedObject { @@ -69,7 +66,11 @@ impl DeployedObject { pub fn get_ignore_regex_filter(&self) -> bool { match self { - Self::User { .. } | Self::Group { .. } | Self::ResourceType { .. } | Self::Settings { .. } | Self::Key { .. } => true, + Self::User { .. } + | Self::Group { .. } + | Self::ResourceType { .. } + | Self::Settings { .. } + | Self::Key { .. } => true, _ => false, } } diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 3667239119..fcefc458f7 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -33,7 +33,7 @@ use windmill_common::add_time; use windmill_common::auth::JobPerms; #[cfg(feature = "benchmark")] use windmill_common::bench::BenchmarkIter; -use windmill_common::jobs::EMAIL_ERROR_HANDLER_USER_EMAIL; +use windmill_common::jobs::{JobTriggerKind, EMAIL_ERROR_HANDLER_USER_EMAIL}; use windmill_common::utils::now_from_db; use windmill_common::worker::{Connection, SCRIPT_TOKEN_EXPIRY}; #[cfg(feature = "enterprise")] @@ -439,6 +439,7 @@ pub async fn push_init_job<'c>( None, None, None, + None, false, true, None, @@ -492,6 +493,7 @@ pub async fn push_periodic_bash_job<'c>( None, None, None, + None, false, true, None, @@ -1267,6 +1269,7 @@ async fn restart_job_if_perpetual_inner( None, None, None, + None, false, false, None, @@ -2029,6 +2032,7 @@ pub async fn push_error_handler<'a, 'c, T: Serialize + Send + Sync>( None, None, Some(job_id), + None, Some(job_id), None, false, @@ -2138,6 +2142,7 @@ async fn handle_recovered_schedule<'a, 'c, T: Serialize + Send + Sync>( None, None, Some(job_id), + None, Some(job_id), None, false, @@ -2228,6 +2233,7 @@ async fn handle_successful_schedule<'a, 'c, T: Serialize + Send + Sync>( None, None, Some(job_id), + None, Some(job_id), None, false, @@ -2251,42 +2257,6 @@ async fn handle_successful_schedule<'a, 'c, T: Serialize + Send + Sync>( Ok(()) } -#[derive(sqlx::Type, Serialize, Deserialize, Debug, Clone)] -#[sqlx(type_name = "JOB_TRIGGER_KIND", rename_all = "lowercase")] -#[serde(rename_all = "lowercase")] -pub enum JobTriggerKind { - Webhook, - Http, - Websocket, - Kafka, - Email, - Nats, - Mqtt, - Sqs, - Postgres, - Schedule, - Gcp, -} - -impl std::fmt::Display for JobTriggerKind { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let kind = match self { - JobTriggerKind::Webhook => "webhook", - JobTriggerKind::Http => "http", - JobTriggerKind::Websocket => "websocket", - JobTriggerKind::Kafka => "kafka", - JobTriggerKind::Email => "email", - JobTriggerKind::Nats => "nats", - JobTriggerKind::Mqtt => "mqtt", - JobTriggerKind::Sqs => "sqs", - JobTriggerKind::Postgres => "postgres", - JobTriggerKind::Schedule => "schedule", - JobTriggerKind::Gcp => "gcp", - }; - write!(f, "{}", kind) - } -} - #[derive(sqlx::FromRow, Debug, Clone, Serialize, Deserialize)] pub struct MiniPulledJob { pub workspace_id: String, @@ -2311,6 +2281,7 @@ pub struct MiniPulledJob { pub concurrent_limit: Option, pub concurrency_time_window_s: Option, pub flow_innermost_root_job: Option, + pub root_job: Option, pub timeout: Option, pub flow_step_id: Option, pub cache_ttl: Option, @@ -2369,7 +2340,8 @@ impl MiniPulledJob { pre_run_error: job.pre_run_error.clone(), concurrent_limit: job.concurrent_limit.clone(), concurrency_time_window_s: job.concurrency_time_window_s.clone(), - flow_innermost_root_job: job.root_job.clone(), + flow_innermost_root_job: job.root_job.clone(), // QueuedJob is taken from v2_as_queue, where root_job corresponds to flow_innermost_root_job in v2_job + root_job: None, timeout: job.timeout.clone(), flow_step_id: job.flow_step_id.clone(), cache_ttl: job.cache_ttl.clone(), @@ -2574,6 +2546,7 @@ pub async fn get_mini_pulled_job<'c>( concurrent_limit, concurrency_time_window_s, flow_innermost_root_job, + root_job, timeout, flow_step_id, cache_ttl, @@ -3623,6 +3596,7 @@ pub async fn push<'c, 'd>( schedule_path: Option, parent_job: Option, root_job: Option, + flow_innermost_root_job: Option, job_id: Option, _is_flow_step: bool, mut same_worker: bool, // whether the job will be executed on the same worker: if true, the job will be set to running but started_at will not be set. @@ -4683,6 +4657,16 @@ pub async fn push<'c, 'd>( None }; + let root_job = if root_job.is_some() + && (root_job == flow_innermost_root_job.or(parent_job).or(Some(job_id))) + { + // We only save the root job if it's not the innermost root job, parent job, or the job itself as an optimization + // Reference: see [`windmill_worker::common::get_root_job_id`] for logic on determining the root job. + None + } else { + root_job + }; + sqlx::query!( "WITH inserted_job AS ( INSERT INTO v2_job (id, workspace_id, raw_code, raw_lock, raw_flow, tag, parent_job, @@ -4724,7 +4708,7 @@ pub async fn push<'c, 'd>( pre_run_error.map(|e| e.to_string()), email, visible_to_owner, - root_job, + flow_innermost_root_job, concurrent_limit, if concurrent_limit.is_some() { concurrency_time_window_s @@ -4746,7 +4730,7 @@ pub async fn push<'c, 'd>( job_authed.is_operator, folders.as_slice(), job_authed.groups.as_slice(), - root_job.or(parent_job), + root_job, trigger_kind as Option, running, ) @@ -5137,6 +5121,7 @@ pub async fn get_same_worker_job( v2_job.concurrent_limit, v2_job.concurrency_time_window_s, v2_job.flow_innermost_root_job, + v2_job.root_job, v2_job.timeout, v2_job.flow_step_id, v2_job.cache_ttl, diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index ae3653881a..cd682629c2 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -294,6 +294,7 @@ pub async fn push_scheduled_job<'c>( None, None, None, + None, false, false, None, diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index c8ee996bc7..398d8f876d 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -9,11 +9,12 @@ use windmill_common::{ auth::get_job_perms, cache, client::AuthedClient, + db::DB, error::{self, to_anyhow, Error}, flow_status::AgentAction, flows::{FlowModule, FlowModuleValue}, get_latest_hash_for_path, - jobs::{JobKind, DB}, + jobs::JobKind, scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, utils::{StripPath, HTTP_CLIENT}, worker::{to_raw_value, Connection}, @@ -484,6 +485,7 @@ async fn call_tool( agent_job.schedule_path(), Some(agent_job.id), None, + None, Some(job_id), false, false, diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index d0ad991bf2..d64926b580 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -1590,6 +1590,7 @@ pub async fn start_worker( None, None, None, + None, ) .await; let context_envs = build_envs_map(context.to_vec()).await; diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 1df44c34df..ce1619da8b 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -211,25 +211,6 @@ pub fn parse_npm_config(s: &str) -> (String, Option) { return (url, token_opt); } -#[async_recursion] -pub async fn get_root_job_id(job: &Uuid, db: &Pool) -> anyhow::Result { - let njob = sqlx::query_scalar!( - "SELECT flow_innermost_root_job FROM v2_job WHERE id = $1", - job - ) - .fetch_optional(db) - .await? - .flatten(); - if let Some(root_job) = njob { - if root_job == *job { - return Ok(job.to_owned()); - } - get_root_job_id(&root_job, db).await - } else { - Ok(job.to_owned()) - } -} - #[async_recursion] pub async fn transform_json_value( name: &str, @@ -272,9 +253,7 @@ pub async fn transform_json_value( Connection::Sql(db) => { let encrypted = y.strip_prefix("$encrypted:").unwrap(); - let root_job_id = - get_root_job_id(&job.flow_innermost_root_job.unwrap_or_else(|| job.id), db) - .await?; + let root_job_id = get_root_job_id(&job); let mc = build_crypt_with_key_suffix( &db, &job.workspace_id, @@ -463,6 +442,7 @@ pub async fn get_reserved_variables( job.schedule_path(), job.flow_step_id.clone(), job.flow_innermost_root_job.clone().map(|x| x.to_string()), + Some(get_root_job_id(job).to_string()), Some(job.scheduled_for.clone()), job.runnable_id, ) @@ -1073,6 +1053,14 @@ pub fn build_http_client(timeout_duration: std::time::Duration) -> error::Result .map_err(|e| Error::internal_err(format!("Error building http client: {e:#}"))) } +pub fn get_root_job_id(job: &MiniPulledJob) -> uuid::Uuid { + // fallback to flow_innermost_root_job and parent_job as root_job is not set if equal to innermost root job or parent job + job.root_job + .or(job.flow_innermost_root_job) + .or(job.parent_job) + .unwrap_or(job.id) +} + #[derive(Clone)] pub struct S3ModeWorkerData { pub client: AuthedClient, diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 607e6375d2..a61b581262 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -545,6 +545,7 @@ pub async fn start_worker( None, None, None, + None, ) .await; let context_envs = build_envs_map(context.to_vec()).await; diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 70f980052b..e8291f771a 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -2146,6 +2146,7 @@ pub async fn start_worker( None, None, None, + None, ) .await .to_vec(); @@ -2266,6 +2267,7 @@ for line in sys.stdin: None, None, None, + None, ) .await; diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index f4874f6456..405ed49307 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -11,7 +11,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::time::Duration; -use crate::common::{cached_result_path, save_in_cache}; +use crate::common::{cached_result_path, get_root_job_id, save_in_cache}; use crate::js_eval::{eval_timeout, IdContext}; use crate::worker_utils::get_tag_and_concurrency; use crate::{ @@ -48,7 +48,7 @@ use windmill_common::utils::WarnAfterExt; use windmill_common::worker::to_raw_value; use windmill_common::{ add_time, get_latest_flow_version_info_for_path, get_script_info_for_hash, FlowVersionInfo, - ScriptHashInfo, + ScriptHashInfo, DB, }; use windmill_common::{ error::{self, to_anyhow, Error}, @@ -66,8 +66,6 @@ use windmill_queue::{ MiniPulledJob, PushArgs, PushIsolationLevel, SameWorkerPayload, WrappedError, }; -type DB = sqlx::Pool; - use windmill_audit::audit_oss::{audit_log, AuditAuthor}; use windmill_audit::ActionKind; use windmill_queue::{canceled_job_to_result, push}; @@ -2816,7 +2814,7 @@ async fn push_next_flow_job( let value_with_parallel = module.get_value_with_parallel()?; - let root_job = if { + let flow_innermost_root_job = if { value_with_parallel.type_ == "flow" || (value_with_parallel.type_ == "forloopflow" && value_with_parallel.parallel.is_some_and(|x| x)) @@ -2828,19 +2826,13 @@ async fn push_next_flow_job( .or_else(|| Some(flow_job.id)) }; + let flow_root_job = get_root_job_id(&flow_job); + // forward root job permissions to the new job - let job_perms: Option = { - if let Some(root_job) = &flow_job - .flow_innermost_root_job - .or_else(|| Some(flow_job.id)) - { - get_job_perms(&mut *tx, root_job, &flow_job.workspace_id) - .await? - .map(|x| x.into()) - } else { - None - } - }; + let job_perms: Option = + get_job_perms(&mut *tx, &flow_root_job, &flow_job.workspace_id) + .await? + .map(|x| x.into()); tracing::debug!(id = %flow_job.id, root_id = %job_root, "computed perms for job {i} of {len}"); let tag = if !matches!(step, Step::PreprocessorStep) @@ -2860,6 +2852,7 @@ async fn push_next_flow_job( flow_job.permissioned_as.to_owned(), ) }; + let tx2 = PushIsolationLevel::Transaction(tx); let (uuid, mut inner_tx) = push( &db, @@ -2877,7 +2870,8 @@ async fn push_next_flow_job( scheduled_for_o, flow_job.schedule_path(), Some(flow_job.id), - root_job, + Some(flow_root_job), + flow_innermost_root_job, None, true, continue_on_same_worker, @@ -2938,7 +2932,7 @@ async fn push_next_flow_job( SET flow_status = JSONB_SET(flow_status, ARRAY['cleanup_module', 'flow_jobs_to_clean'], COALESCE(flow_status->'cleanup_module'->'flow_jobs_to_clean', '[]'::jsonb) || $1) WHERE id = $2", uuid_singleton_json, - root_job.unwrap_or(flow_job.id) + flow_innermost_root_job.unwrap_or(flow_job.id) ) .execute(&mut *inner_tx) .warn_after_seconds(3) diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 358e93e0a1..38a451d8fa 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -739,6 +739,7 @@ pub async fn trigger_dependents_to_recompute_dependencies( None, None, None, + None, false, false, None, diff --git a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts index dcd91e1d1f..ed7ea56157 100644 --- a/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/AIChatManager.svelte.ts @@ -37,7 +37,8 @@ import { getStringError } from './utils' import type { FlowModuleState, FlowState } from '$lib/components/flows/flowState' import type { CurrentEditor, ExtendedOpenFlow } from '$lib/components/flows/types' import { untrack } from 'svelte' -import { getCurrentModel, type DBSchemas } from '$lib/stores' +import { get } from 'svelte/store' +import { getCurrentModel, type DBSchemas, copilotInfo } from '$lib/stores' import { askTools, prepareAskSystemMessage, prepareAskUserMessage } from './ask/core' import { chatState, DEFAULT_SIZE, triggerablesByAi } from './sharedChatState.svelte' import type { ContextElement } from './context' @@ -208,7 +209,8 @@ class AIChatManager { this.mode = mode this.pendingPrompt = pendingPrompt ?? '' if (mode === AIMode.SCRIPT) { - this.systemMessage = prepareScriptSystemMessage() + const customPrompt = get(copilotInfo).customPrompts?.[mode] + this.systemMessage = prepareScriptSystemMessage(customPrompt) this.systemMessage.content = this.NAVIGATION_SYSTEM_PROMPT + this.systemMessage.content const context = this.contextManager.getSelectedContext() const lang = this.scriptEditorOptions?.lang ?? 'bun' @@ -243,20 +245,24 @@ class AIChatManager { } } } else if (mode === AIMode.FLOW) { - this.systemMessage = prepareFlowSystemMessage() + const customPrompt = get(copilotInfo).customPrompts?.[mode] + this.systemMessage = prepareFlowSystemMessage(customPrompt) this.systemMessage.content = this.NAVIGATION_SYSTEM_PROMPT + this.systemMessage.content this.tools = [this.changeModeTool, ...flowTools] this.helpers = this.flowAiChatHelpers } else if (mode === AIMode.NAVIGATOR) { - this.systemMessage = prepareNavigatorSystemMessage() + const customPrompt = get(copilotInfo).customPrompts?.[mode] + this.systemMessage = prepareNavigatorSystemMessage(customPrompt) this.tools = [this.changeModeTool, ...navigatorTools] this.helpers = {} } else if (mode === AIMode.ASK) { - this.systemMessage = prepareAskSystemMessage() + const customPrompt = get(copilotInfo).customPrompts?.[mode] + this.systemMessage = prepareAskSystemMessage(customPrompt) this.tools = [...askTools] this.helpers = {} } else if (mode === AIMode.API) { - this.systemMessage = prepareApiSystemMessage() + const customPrompt = get(copilotInfo).customPrompts?.[mode] + this.systemMessage = prepareApiSystemMessage(customPrompt) this.tools = [...this.apiTools] this.helpers = {} } diff --git a/frontend/src/lib/components/copilot/chat/api/core.ts b/frontend/src/lib/components/copilot/chat/api/core.ts index cd7038595a..4e47baea72 100644 --- a/frontend/src/lib/components/copilot/chat/api/core.ts +++ b/frontend/src/lib/components/copilot/chat/api/core.ts @@ -57,10 +57,17 @@ export async function getApiTools(): Promise[]> { export const apiTools: Tool<{}>[] = [getDocumentationTool] -export function prepareApiSystemMessage(): ChatCompletionSystemMessageParam { +export function prepareApiSystemMessage(customPrompt?: string): ChatCompletionSystemMessageParam { + let content = CHAT_SYSTEM_PROMPT(get(userStore)?.username ?? '') + + // If there's a custom prompt, append it to the system prompt + if (customPrompt?.trim()) { + content = `${content}\n\nUSER GIVEN INSTRUCTIONS:\n${customPrompt.trim()}` + } + return { role: 'system', - content: CHAT_SYSTEM_PROMPT(get(userStore)?.username ?? '') + content } } diff --git a/frontend/src/lib/components/copilot/chat/ask/core.ts b/frontend/src/lib/components/copilot/chat/ask/core.ts index 997594cc88..f9ba219599 100644 --- a/frontend/src/lib/components/copilot/chat/ask/core.ts +++ b/frontend/src/lib/components/copilot/chat/ask/core.ts @@ -25,10 +25,17 @@ GENERAL PRINCIPLES: export const askTools: Tool<{}>[] = [getDocumentationTool] -export function prepareAskSystemMessage(): ChatCompletionSystemMessageParam { +export function prepareAskSystemMessage(customPrompt?: string): ChatCompletionSystemMessageParam { + let content = CHAT_SYSTEM_PROMPT + + // If there's a custom prompt, append it to the system prompt + if (customPrompt?.trim()) { + content = `${content}\n\nUSER GIVEN INSTRUCTIONS:\n${customPrompt.trim()}` + } + return { role: 'system', - content: CHAT_SYSTEM_PROMPT + content } } diff --git a/frontend/src/lib/components/copilot/chat/flow/core.ts b/frontend/src/lib/components/copilot/chat/flow/core.ts index dae637f156..1fdd8895ff 100644 --- a/frontend/src/lib/components/copilot/chat/flow/core.ts +++ b/frontend/src/lib/components/copilot/chat/flow/core.ts @@ -885,8 +885,8 @@ export const flowTools: Tool[] = [ } ] -export function prepareFlowSystemMessage(): ChatCompletionSystemMessageParam { - const content = `You are a helpful assistant that creates and edits workflows on the Windmill platform. You're provided with a bunch of tools to help you edit the flow. +export function prepareFlowSystemMessage(customPrompt?: string): ChatCompletionSystemMessageParam { + let content = `You are a helpful assistant that creates and edits workflows on the Windmill platform. You're provided with a bunch of tools to help you edit the flow. Follow the user instructions carefully. Go step by step, and explain what you're doing as you're doing it. DO NOT wait for user confirmation before performing an action. Only do it if the user explicitly asks you to wait in their initial instructions. @@ -1012,6 +1012,11 @@ If the user needs a resource as flow input, you should set the property type in If the user wants a specific resource as step input, you should set the step value to a static string in the following format: "$res:path/to/resource". ` + // If there's a custom prompt, append it to the system prompt + if (customPrompt?.trim()) { + content = `${content}\n\nUSER GIVEN INSTRUCTIONS:\n${customPrompt.trim()}` + } + return { role: 'system', content diff --git a/frontend/src/lib/components/copilot/chat/navigator/core.ts b/frontend/src/lib/components/copilot/chat/navigator/core.ts index f7488b3fa8..4c9cf38b70 100644 --- a/frontend/src/lib/components/copilot/chat/navigator/core.ts +++ b/frontend/src/lib/components/copilot/chat/navigator/core.ts @@ -350,10 +350,17 @@ export const navigatorTools: Tool<{}>[] = [ getAvailableResourcesTool ] -export function prepareNavigatorSystemMessage(): ChatCompletionSystemMessageParam { +export function prepareNavigatorSystemMessage(customPrompt?: string): ChatCompletionSystemMessageParam { + let content = CHAT_SYSTEM_PROMPT + + // If there's a custom prompt, append it to the system prompt + if (customPrompt?.trim()) { + content = `${content}\n\nUSER GIVEN INSTRUCTIONS:\n${customPrompt.trim()}` + } + return { role: 'system', - content: CHAT_SYSTEM_PROMPT + content } } diff --git a/frontend/src/lib/components/copilot/chat/script/core.ts b/frontend/src/lib/components/copilot/chat/script/core.ts index 9deb4f0412..ca9bde59e4 100644 --- a/frontend/src/lib/components/copilot/chat/script/core.ts +++ b/frontend/src/lib/components/copilot/chat/script/core.ts @@ -452,10 +452,19 @@ WINDMILL LANGUAGE CONTEXT: ` -export function prepareScriptSystemMessage(): ChatCompletionSystemMessageParam { +export function prepareScriptSystemMessage( + customPrompt?: string +): ChatCompletionSystemMessageParam { + let content = CHAT_SYSTEM_PROMPT + + // If there's a custom prompt, prepend it to the system prompt + if (customPrompt?.trim()) { + content = `${content}\n\nUSER GIVEN INSTRUCTIONS:\n${customPrompt.trim()}` + } + return { role: 'system', - content: CHAT_SYSTEM_PROMPT + content } } diff --git a/frontend/src/lib/components/flows/content/FlowModuleSuspend.svelte b/frontend/src/lib/components/flows/content/FlowModuleSuspend.svelte index c457e036ff..ab17bdb28b 100644 --- a/frontend/src/lib/components/flows/content/FlowModuleSuspend.svelte +++ b/frontend/src/lib/components/flows/content/FlowModuleSuspend.svelte @@ -70,7 +70,6 @@ let jsonView: boolean = $state(false) -A{JSON.stringify(flowModule.suspend?.resume_form)}B
{#snippet action()} diff --git a/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte b/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte index 819765403d..59be11236b 100644 --- a/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte +++ b/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte @@ -4,7 +4,6 @@ import Popover from '$lib/components/Popover.svelte' import { classNames, type StateStore } from '$lib/utils' import { - AlertTriangle, Bed, Database, Gauge, @@ -16,7 +15,8 @@ Pin, X, Play, - Loader2 + Loader2, + TriangleAlert } from 'lucide-svelte' import { createEventDispatcher, getContext } from 'svelte' import { fade } from 'svelte/transition' @@ -460,60 +460,67 @@ {#if deletable && !action} - - - {#if id !== 'preprocessor'} + {#if id !== 'preprocessor'} + + {/if} - {/if} + {#if (id && Object.values($flowInputsStore?.[id]?.flowStepWarnings || {}).length > 0) || Boolean(warningMessage)} -
- - {#snippet text()} -
    - {#if id} - {#each Object.values($flowInputsStore?.[id]?.flowStepWarnings || {}) as m} -
  • - {m.message} -
    • - {/each} - {/if} -
- {/snippet} -
x.type === 'error' - ) - ? 'border-red-600 text-red-600 bg-red-100 hover:bg-red-300' - : 'border-yellow-600 text-yellow-600 bg-yellow-100 hover:bg-yellow-300' - )} - > - -
- -
+ x.type === 'error' + ) + ? 'border-red-600 text-red-600 bg-red-100 hover:bg-red-300' + : ' text-yellow-600 bg-yellow-100 hover:bg-yellow-300' + )} + > + {#snippet text()} +
    + {#if id} + {#each Object.values($flowInputsStore?.[id]?.flowStepWarnings || {}) as m} +
  • + {m.message} +
    • + {/each} + {/if} +
+ {/snippet} + + + {/if} {/if} @@ -521,7 +528,7 @@ outline-[1px] outline dark:outline-gray-500 outline-gray-300 bg-surface duration {#if editMode && enableTestRun && flowJob?.type !== 'QueuedJob'}
(hover = true)} onmouseleave={() => (hover = false)} > @@ -529,11 +536,11 @@ outline-[1px] outline dark:outline-gray-500 outline-gray-300 bg-surface duration
{#if !testIsLoading} {/if} {:else if id?.startsWith('subflow:')} diff --git a/frontend/src/lib/components/graph/renderers/nodes/BranchAllStart.svelte b/frontend/src/lib/components/graph/renderers/nodes/BranchAllStart.svelte index 923a82b13c..09ba0d81d8 100644 --- a/frontend/src/lib/components/graph/renderers/nodes/BranchAllStart.svelte +++ b/frontend/src/lib/components/graph/renderers/nodes/BranchAllStart.svelte @@ -40,9 +40,8 @@ {#if data.insertable}
{/if} + {#if Object.keys(aiProviders).length > 0} +
+

Custom system prompts

+
+ + + +
+
+ {/if} +