diff --git a/backend/.sqlx/query-011c7638eeeda710deb86a216a9e10df9c3e9458e85bcdde466b01011a1f2ac2.json b/backend/.sqlx/query-011c7638eeeda710deb86a216a9e10df9c3e9458e85bcdde466b01011a1f2ac2.json deleted file mode 100644 index 749b684fd9..0000000000 --- a/backend/.sqlx/query-011c7638eeeda710deb86a216a9e10df9c3e9458e85bcdde466b01011a1f2ac2.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n path,\n is_flow,\n workspace_id,\n owner,\n email,\n trigger_config as \"trigger_config!: _\"\n FROM\n capture_config\n WHERE\n trigger_kind = 'postgres' AND\n last_client_ping > NOW() - INTERVAL '10 seconds' AND\n trigger_config IS NOT NULL AND\n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "owner", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "trigger_config!: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - false, - false, - true - ] - }, - "hash": "011c7638eeeda710deb86a216a9e10df9c3e9458e85bcdde466b01011a1f2ac2" -} diff --git a/backend/.sqlx/query-031d0d70b0aff52feaad487bddb74e5ef0aaa2505facbea8c764003dfc8fffb1.json b/backend/.sqlx/query-031d0d70b0aff52feaad487bddb74e5ef0aaa2505facbea8c764003dfc8fffb1.json deleted file mode 100644 index 8fce6ae729..0000000000 --- a/backend/.sqlx/query-031d0d70b0aff52feaad487bddb74e5ef0aaa2505facbea8c764003dfc8fffb1.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE capture_config SET last_server_ping = now(), error = $1 WHERE workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'websocket' AND server_id = $5 AND last_client_ping > NOW() - INTERVAL '10 seconds' RETURNING 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Bool", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "031d0d70b0aff52feaad487bddb74e5ef0aaa2505facbea8c764003dfc8fffb1" -} diff --git a/backend/.sqlx/query-115544a96173f9cb1d27757e7b931fb27912cfd05ba768a42cf9b3dfd7205e9a.json b/backend/.sqlx/query-115544a96173f9cb1d27757e7b931fb27912cfd05ba768a42cf9b3dfd7205e9a.json deleted file mode 100644 index 66a2152ae9..0000000000 --- a/backend/.sqlx/query-115544a96173f9cb1d27757e7b931fb27912cfd05ba768a42cf9b3dfd7205e9a.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n mqtt_trigger \n SET \n server_id = $1, \n last_server_ping = now(),\n error = 'Connecting...'\n WHERE \n enabled IS TRUE \n AND workspace_id = $2 \n AND path = $3 \n AND (last_server_ping IS NULL \n OR last_server_ping < now() - INTERVAL '15 seconds'\n ) \n RETURNING true\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "115544a96173f9cb1d27757e7b931fb27912cfd05ba768a42cf9b3dfd7205e9a" -} diff --git a/backend/.sqlx/query-12e868b63a7c622c76713db5a5577a927efca4ae49a15c2b999e2410f2a312ff.json b/backend/.sqlx/query-12e868b63a7c622c76713db5a5577a927efca4ae49a15c2b999e2410f2a312ff.json deleted file mode 100644 index 49ad10ac07..0000000000 --- a/backend/.sqlx/query-12e868b63a7c622c76713db5a5577a927efca4ae49a15c2b999e2410f2a312ff.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE \n capture_config \n SET \n last_server_ping = NULL \n WHERE \n workspace_id = $1 AND \n path = $2 AND \n is_flow = $3 AND \n trigger_kind = 'postgres' AND \n server_id IS NULL\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "12e868b63a7c622c76713db5a5577a927efca4ae49a15c2b999e2410f2a312ff" -} diff --git a/backend/.sqlx/query-1488e1b5007752e1ebae4235ad04c398fe6398745e16fd119008b8ea67662416.json b/backend/.sqlx/query-1488e1b5007752e1ebae4235ad04c398fe6398745e16fd119008b8ea67662416.json deleted file mode 100644 index 197298f788..0000000000 --- a/backend/.sqlx/query-1488e1b5007752e1ebae4235ad04c398fe6398745e16fd119008b8ea67662416.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE postgres_trigger \n SET \n server_id = $1, \n last_server_ping = now(),\n error = 'Connecting...'\n WHERE \n enabled IS TRUE \n AND workspace_id = $2 \n AND path = $3 \n AND (last_server_ping IS NULL \n OR last_server_ping < now() - INTERVAL '15 seconds'\n ) \n RETURNING true\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "1488e1b5007752e1ebae4235ad04c398fe6398745e16fd119008b8ea67662416" -} diff --git a/backend/.sqlx/query-186aef850c2eeb89c186ac6b2934dd3a703e2b9428096801e1d2d61fdbb99c9e.json b/backend/.sqlx/query-186aef850c2eeb89c186ac6b2934dd3a703e2b9428096801e1d2d61fdbb99c9e.json deleted file mode 100644 index 648c5dda19..0000000000 --- a/backend/.sqlx/query-186aef850c2eeb89c186ac6b2934dd3a703e2b9428096801e1d2d61fdbb99c9e.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n mqtt_trigger \n SET\n last_server_ping = NULL \n WHERE \n workspace_id = $1 \n AND path = $2 \n AND server_id IS NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "186aef850c2eeb89c186ac6b2934dd3a703e2b9428096801e1d2d61fdbb99c9e" -} diff --git a/backend/.sqlx/query-1974bd65bbf40024773aad4dee1c50b12e110e76bb58e6de25bec094e758a71c.json b/backend/.sqlx/query-1974bd65bbf40024773aad4dee1c50b12e110e76bb58e6de25bec094e758a71c.json deleted file mode 100644 index f19670704d..0000000000 --- a/backend/.sqlx/query-1974bd65bbf40024773aad4dee1c50b12e110e76bb58e6de25bec094e758a71c.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n last_server_ping = now(), \n error = $1 \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'postgres' AND \n server_id = $5 AND \n last_client_ping > NOW() - INTERVAL '10 seconds' \n RETURNING 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Bool", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "1974bd65bbf40024773aad4dee1c50b12e110e76bb58e6de25bec094e758a71c" -} diff --git a/backend/.sqlx/query-1b33393fbbc7e681b4d355f6096d982b52b48c8c3da392b14963a1ec86811546.json b/backend/.sqlx/query-1b33393fbbc7e681b4d355f6096d982b52b48c8c3da392b14963a1ec86811546.json deleted file mode 100644 index 271c60b480..0000000000 --- a/backend/.sqlx/query-1b33393fbbc7e681b4d355f6096d982b52b48c8c3da392b14963a1ec86811546.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n path,\n is_flow,\n workspace_id,\n owner,\n email,\n trigger_config as \"trigger_config!: _\"\n FROM\n capture_config\n WHERE\n trigger_kind = 'sqs' AND\n last_client_ping > NOW() - INTERVAL '10 seconds' AND\n trigger_config IS NOT NULL AND\n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "owner", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "trigger_config!: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - false, - false, - true - ] - }, - "hash": "1b33393fbbc7e681b4d355f6096d982b52b48c8c3da392b14963a1ec86811546" -} diff --git a/backend/.sqlx/query-1c37f91192aa4f535c7fff80fa809260b906dc988f44a0db60952a2bf8b1cdaf.json b/backend/.sqlx/query-1c37f91192aa4f535c7fff80fa809260b906dc988f44a0db60952a2bf8b1cdaf.json deleted file mode 100644 index 946783d745..0000000000 --- a/backend/.sqlx/query-1c37f91192aa4f535c7fff80fa809260b906dc988f44a0db60952a2bf8b1cdaf.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE kafka_trigger SET last_server_ping = NULL WHERE workspace_id = $1 AND path = $2 AND server_id IS NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "1c37f91192aa4f535c7fff80fa809260b906dc988f44a0db60952a2bf8b1cdaf" -} diff --git a/backend/.sqlx/query-1cad25c24d0f80d58a50d4da923f8b0672797299545ee0754b6ad74ece92c77e.json b/backend/.sqlx/query-1cad25c24d0f80d58a50d4da923f8b0672797299545ee0754b6ad74ece92c77e.json deleted file mode 100644 index 0729adc8ba..0000000000 --- a/backend/.sqlx/query-1cad25c24d0f80d58a50d4da923f8b0672797299545ee0754b6ad74ece92c77e.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n last_server_ping = now(), \n error = $1 \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'sqs' AND \n server_id = $5 AND \n last_client_ping > NOW() - INTERVAL '10 seconds' \n RETURNING 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Bool", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "1cad25c24d0f80d58a50d4da923f8b0672797299545ee0754b6ad74ece92c77e" -} diff --git a/backend/.sqlx/query-203fa78d423ec5a8c5ff6166aed591b28cbf9ea8f61d379b84ee6e14c033035d.json b/backend/.sqlx/query-203fa78d423ec5a8c5ff6166aed591b28cbf9ea8f61d379b84ee6e14c033035d.json deleted file mode 100644 index 65b1b24efb..0000000000 --- a/backend/.sqlx/query-203fa78d423ec5a8c5ff6166aed591b28cbf9ea8f61d379b84ee6e14c033035d.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE capture_config SET error = $1, server_id = NULL, last_server_ping = NULL WHERE workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'kafka'", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "203fa78d423ec5a8c5ff6166aed591b28cbf9ea8f61d379b84ee6e14c033035d" -} diff --git a/backend/.sqlx/query-20e01ecb5d4aa4c532a8f906365c776994ae2c990bb7bb307c52296b9543fbb8.json b/backend/.sqlx/query-20e01ecb5d4aa4c532a8f906365c776994ae2c990bb7bb307c52296b9543fbb8.json deleted file mode 100644 index c9f7733011..0000000000 --- a/backend/.sqlx/query-20e01ecb5d4aa4c532a8f906365c776994ae2c990bb7bb307c52296b9543fbb8.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE \n capture_config \n SET \n last_server_ping = NULL \n WHERE \n workspace_id = $1 AND \n path = $2 AND \n is_flow = $3 AND \n trigger_kind = 'sqs' AND \n server_id IS NULL\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "20e01ecb5d4aa4c532a8f906365c776994ae2c990bb7bb307c52296b9543fbb8" -} diff --git a/backend/.sqlx/query-22dcd953d900fb0ddbe2099ccee27bcf833f56475bbc1aa3c8f6d79f146f41bf.json b/backend/.sqlx/query-22dcd953d900fb0ddbe2099ccee27bcf833f56475bbc1aa3c8f6d79f146f41bf.json deleted file mode 100644 index f94ca24f0f..0000000000 --- a/backend/.sqlx/query-22dcd953d900fb0ddbe2099ccee27bcf833f56475bbc1aa3c8f6d79f146f41bf.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n sqs_trigger\n SET \n last_server_ping = now(),\n error = $1\n WHERE\n workspace_id = $2\n AND path = $3\n AND server_id = $4 \n AND enabled IS TRUE\n RETURNING 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "22dcd953d900fb0ddbe2099ccee27bcf833f56475bbc1aa3c8f6d79f146f41bf" -} diff --git a/backend/.sqlx/query-2659fe2e121ac15da08030c9e72bdb79a580711cba2139f0cf901b30bb491fd5.json b/backend/.sqlx/query-2659fe2e121ac15da08030c9e72bdb79a580711cba2139f0cf901b30bb491fd5.json deleted file mode 100644 index f1d2942f51..0000000000 --- a/backend/.sqlx/query-2659fe2e121ac15da08030c9e72bdb79a580711cba2139f0cf901b30bb491fd5.json +++ /dev/null @@ -1,150 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n mqtt_resource_path,\n subscribe_topics as \"subscribe_topics!: Vec>\",\n v3_config as \"v3_config!: Option>\",\n v5_config as \"v5_config!: Option>\",\n client_version as \"client_version: _\",\n client_id,\n workspace_id,\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\"\n FROM\n mqtt_trigger\n WHERE\n enabled IS TRUE\n AND (last_server_ping IS NULL OR\n last_server_ping < now() - interval '15 seconds'\n )\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "mqtt_resource_path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "subscribe_topics!: Vec>", - "type_info": "JsonbArray" - }, - { - "ordinal": 2, - "name": "v3_config!: Option>", - "type_info": "Jsonb" - }, - { - "ordinal": 3, - "name": "v5_config!: Option>", - "type_info": "Jsonb" - }, - { - "ordinal": 4, - "name": "client_version: _", - "type_info": { - "Custom": { - "name": "mqtt_client_version", - "kind": { - "Enum": [ - "v3", - "v5" - ] - } - } - } - }, - { - "ordinal": 5, - "name": "client_id", - "type_info": "Varchar" - }, - { - "ordinal": 6, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 7, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 8, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 9, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 10, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 11, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 12, - "name": "edited_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 13, - "name": "server_id", - "type_info": "Varchar" - }, - { - "ordinal": 14, - "name": "last_server_ping", - "type_info": "Timestamptz" - }, - { - "ordinal": 15, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 16, - "name": "error", - "type_info": "Text" - }, - { - "ordinal": 17, - "name": "enabled", - "type_info": "Bool" - }, - { - "ordinal": 18, - "name": "error_handler_path", - "type_info": "Varchar" - }, - { - "ordinal": 19, - "name": "error_handler_args: _", - "type_info": "Jsonb" - }, - { - "ordinal": 20, - "name": "retry: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - true, - true, - false, - true, - false, - false, - false, - false, - false, - false, - false, - true, - true, - false, - true, - false, - true, - true, - true - ] - }, - "hash": "2659fe2e121ac15da08030c9e72bdb79a580711cba2139f0cf901b30bb491fd5" -} diff --git a/backend/.sqlx/query-299e16725162888c01712f371785199960264b54c1ddf928c0c654ab15176f63.json b/backend/.sqlx/query-299e16725162888c01712f371785199960264b54c1ddf928c0c654ab15176f63.json deleted file mode 100644 index d64d8f404b..0000000000 --- a/backend/.sqlx/query-299e16725162888c01712f371785199960264b54c1ddf928c0c654ab15176f63.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n server_id = $1,\n last_server_ping = now(), \n error = 'Connecting...' \n WHERE \n last_client_ping > NOW() - INTERVAL '10 seconds' AND \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'gcp' AND \n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') \n RETURNING true\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text", - "Bool" - ] - }, - "nullable": [ - null - ] - }, - "hash": "299e16725162888c01712f371785199960264b54c1ddf928c0c654ab15176f63" -} diff --git a/backend/.sqlx/query-29f096ec62c4abb1435a5667e2b30e9c1724e419cdc23ef1b300e84c02a20427.json b/backend/.sqlx/query-29f096ec62c4abb1435a5667e2b30e9c1724e419cdc23ef1b300e84c02a20427.json deleted file mode 100644 index b7075f9df1..0000000000 --- a/backend/.sqlx/query-29f096ec62c4abb1435a5667e2b30e9c1724e419cdc23ef1b300e84c02a20427.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'postgres'\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "29f096ec62c4abb1435a5667e2b30e9c1724e419cdc23ef1b300e84c02a20427" -} diff --git a/backend/.sqlx/query-33aaf2dd14397d0b50b986ed55bc458337b85b6a9a9ab3d93d8f33d0f57e4b0f.json b/backend/.sqlx/query-33aaf2dd14397d0b50b986ed55bc458337b85b6a9a9ab3d93d8f33d0f57e4b0f.json deleted file mode 100644 index 188dcc8d5b..0000000000 --- a/backend/.sqlx/query-33aaf2dd14397d0b50b986ed55bc458337b85b6a9a9ab3d93d8f33d0f57e4b0f.json +++ /dev/null @@ -1,166 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n gcp_resource_path,\n subscription_id,\n topic_id,\n workspace_id,\n delivery_type AS \"delivery_type: _\",\n delivery_config AS \"delivery_config: _\",\n subscription_mode AS \"subscription_mode: _\",\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\",\n auto_acknowledge_msg\n FROM\n gcp_trigger\n WHERE\n delivery_type != 'push'::DELIVERY_MODE AND\n enabled IS TRUE\n AND (last_server_ping IS NULL OR\n last_server_ping < now() - interval '15 seconds'\n )\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "gcp_resource_path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "subscription_id", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "topic_id", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "delivery_type: _", - "type_info": { - "Custom": { - "name": "delivery_mode", - "kind": { - "Enum": [ - "push", - "pull" - ] - } - } - } - }, - { - "ordinal": 5, - "name": "delivery_config: _", - "type_info": "Jsonb" - }, - { - "ordinal": 6, - "name": "subscription_mode: _", - "type_info": { - "Custom": { - "name": "gcp_subscription_mode", - "kind": { - "Enum": [ - "create_update", - "existing" - ] - } - } - } - }, - { - "ordinal": 7, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 8, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 9, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 10, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 11, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 12, - "name": "edited_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 13, - "name": "server_id", - "type_info": "Varchar" - }, - { - "ordinal": 14, - "name": "last_server_ping", - "type_info": "Timestamptz" - }, - { - "ordinal": 15, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 16, - "name": "error", - "type_info": "Text" - }, - { - "ordinal": 17, - "name": "enabled", - "type_info": "Bool" - }, - { - "ordinal": 18, - "name": "error_handler_path", - "type_info": "Varchar" - }, - { - "ordinal": 19, - "name": "error_handler_args: _", - "type_info": "Jsonb" - }, - { - "ordinal": 20, - "name": "retry: _", - "type_info": "Jsonb" - }, - { - "ordinal": 21, - "name": "auto_acknowledge_msg", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - false, - false, - true, - false, - false, - false, - false, - false, - false, - false, - true, - true, - false, - true, - false, - true, - true, - true, - true - ] - }, - "hash": "33aaf2dd14397d0b50b986ed55bc458337b85b6a9a9ab3d93d8f33d0f57e4b0f" -} diff --git a/backend/.sqlx/query-3e33469f448fa86c7e0d0deb65ee2c13b5dcc4cbc4fcb0fe2bde1fdcb6d20e74.json b/backend/.sqlx/query-3e33469f448fa86c7e0d0deb65ee2c13b5dcc4cbc4fcb0fe2bde1fdcb6d20e74.json deleted file mode 100644 index 113f888545..0000000000 --- a/backend/.sqlx/query-3e33469f448fa86c7e0d0deb65ee2c13b5dcc4cbc4fcb0fe2bde1fdcb6d20e74.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE \n capture_config \n SET \n last_server_ping = NULL \n WHERE \n workspace_id = $1 AND \n path = $2 AND \n is_flow = $3 AND \n trigger_kind = 'mqtt' AND \n server_id IS NULL\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "3e33469f448fa86c7e0d0deb65ee2c13b5dcc4cbc4fcb0fe2bde1fdcb6d20e74" -} diff --git a/backend/.sqlx/query-3e64c894c89ef82c4527180b82c4e82c1b7060ba0ca288dc6a7c7afcd76212e8.json b/backend/.sqlx/query-3e64c894c89ef82c4527180b82c4e82c1b7060ba0ca288dc6a7c7afcd76212e8.json deleted file mode 100644 index 8831155670..0000000000 --- a/backend/.sqlx/query-3e64c894c89ef82c4527180b82c4e82c1b7060ba0ca288dc6a7c7afcd76212e8.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE kafka_trigger SET enabled = FALSE, error = $1, server_id = NULL, last_server_ping = NULL WHERE workspace_id = $2 AND path = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "3e64c894c89ef82c4527180b82c4e82c1b7060ba0ca288dc6a7c7afcd76212e8" -} diff --git a/backend/.sqlx/query-3f67e7cf8d9f021a075f1c88703287a9f42e252a1246a584249a224afdbbbf8a.json b/backend/.sqlx/query-3f67e7cf8d9f021a075f1c88703287a9f42e252a1246a584249a224afdbbbf8a.json deleted file mode 100644 index 8dec90897c..0000000000 --- a/backend/.sqlx/query-3f67e7cf8d9f021a075f1c88703287a9f42e252a1246a584249a224afdbbbf8a.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n sqs_trigger \n SET \n enabled = FALSE, \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "3f67e7cf8d9f021a075f1c88703287a9f42e252a1246a584249a224afdbbbf8a" -} diff --git a/backend/.sqlx/query-42030372693d8d6b8d03947bb6702024cfa226735eeeb720bbab46bd0e3cedcc.json b/backend/.sqlx/query-42030372693d8d6b8d03947bb6702024cfa226735eeeb720bbab46bd0e3cedcc.json deleted file mode 100644 index cef1da340c..0000000000 --- a/backend/.sqlx/query-42030372693d8d6b8d03947bb6702024cfa226735eeeb720bbab46bd0e3cedcc.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT path, is_flow, workspace_id, trigger_config as \"trigger_config!: _\", owner, email FROM capture_config WHERE trigger_kind = 'nats' AND last_client_ping > NOW() - INTERVAL '10 seconds' AND trigger_config IS NOT NULL AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "trigger_config!: _", - "type_info": "Jsonb" - }, - { - "ordinal": 4, - "name": "owner", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "email", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - true, - false, - false - ] - }, - "hash": "42030372693d8d6b8d03947bb6702024cfa226735eeeb720bbab46bd0e3cedcc" -} diff --git a/backend/.sqlx/query-47560aa3d3af93167663e763ad873088eb4f57785df3c383bbf6e5ceab268980.json b/backend/.sqlx/query-47560aa3d3af93167663e763ad873088eb4f57785df3c383bbf6e5ceab268980.json deleted file mode 100644 index 541634afdf..0000000000 --- a/backend/.sqlx/query-47560aa3d3af93167663e763ad873088eb4f57785df3c383bbf6e5ceab268980.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT path, is_flow, workspace_id, trigger_config as \"trigger_config!: _\", owner, email FROM capture_config WHERE trigger_kind = 'websocket' AND last_client_ping > NOW() - INTERVAL '10 seconds' AND trigger_config IS NOT NULL AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "trigger_config!: _", - "type_info": "Jsonb" - }, - { - "ordinal": 4, - "name": "owner", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "email", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - true, - false, - false - ] - }, - "hash": "47560aa3d3af93167663e763ad873088eb4f57785df3c383bbf6e5ceab268980" -} diff --git a/backend/.sqlx/query-4844a797ddec207f1d2cbf41836e65d6435840f6d0740f300d7c2cf88f8fe46a.json b/backend/.sqlx/query-4844a797ddec207f1d2cbf41836e65d6435840f6d0740f300d7c2cf88f8fe46a.json deleted file mode 100644 index 3f66c4c9b4..0000000000 --- a/backend/.sqlx/query-4844a797ddec207f1d2cbf41836e65d6435840f6d0740f300d7c2cf88f8fe46a.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n mqtt_trigger\n SET \n last_server_ping = now(),\n error = $1\n WHERE\n workspace_id = $2\n AND path = $3\n AND server_id = $4 \n AND enabled IS TRUE\n RETURNING 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "4844a797ddec207f1d2cbf41836e65d6435840f6d0740f300d7c2cf88f8fe46a" -} diff --git a/backend/.sqlx/query-51f37d683d5a48b96f6224111639c364ba9c41572eb6799f7c822c66d00d2500.json b/backend/.sqlx/query-51f37d683d5a48b96f6224111639c364ba9c41572eb6799f7c822c66d00d2500.json deleted file mode 100644 index 36198820c2..0000000000 --- a/backend/.sqlx/query-51f37d683d5a48b96f6224111639c364ba9c41572eb6799f7c822c66d00d2500.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE websocket_trigger SET last_server_ping = NULL WHERE workspace_id = $1 AND path = $2 AND server_id IS NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "51f37d683d5a48b96f6224111639c364ba9c41572eb6799f7c822c66d00d2500" -} diff --git a/backend/.sqlx/query-548a1424a6b9ac9998b8fc5312bcff671ee7fe59154e9ef32800c117e64bce19.json b/backend/.sqlx/query-548a1424a6b9ac9998b8fc5312bcff671ee7fe59154e9ef32800c117e64bce19.json deleted file mode 100644 index 554b0fff0b..0000000000 --- a/backend/.sqlx/query-548a1424a6b9ac9998b8fc5312bcff671ee7fe59154e9ef32800c117e64bce19.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n path,\n is_flow,\n workspace_id,\n owner,\n email,\n trigger_config as \"trigger_config!: _\"\n FROM\n capture_config\n WHERE\n trigger_kind = 'mqtt' AND\n last_client_ping > NOW() - INTERVAL '10 seconds' AND\n trigger_config IS NOT NULL AND\n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "owner", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "trigger_config!: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - false, - false, - true - ] - }, - "hash": "548a1424a6b9ac9998b8fc5312bcff671ee7fe59154e9ef32800c117e64bce19" -} diff --git a/backend/.sqlx/query-58ddf5c76455d30dc42f46a0553e761461c327e7502b707c2d2742946d7d1f4c.json b/backend/.sqlx/query-58ddf5c76455d30dc42f46a0553e761461c327e7502b707c2d2742946d7d1f4c.json deleted file mode 100644 index 2a91229fbd..0000000000 --- a/backend/.sqlx/query-58ddf5c76455d30dc42f46a0553e761461c327e7502b707c2d2742946d7d1f4c.json +++ /dev/null @@ -1,138 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n aws_auth_resource_type AS \"aws_auth_resource_type: _\",\n queue_url,\n aws_resource_path,\n message_attributes,\n workspace_id,\n path,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\"\n FROM\n sqs_trigger\n WHERE\n enabled IS TRUE\n AND (last_server_ping IS NULL OR\n last_server_ping < now() - interval '15 seconds'\n )\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "aws_auth_resource_type: _", - "type_info": { - "Custom": { - "name": "aws_auth_resource_type", - "kind": { - "Enum": [ - "oidc", - "credentials" - ] - } - } - } - }, - { - "ordinal": 1, - "name": "queue_url", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "aws_resource_path", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "message_attributes", - "type_info": "TextArray" - }, - { - "ordinal": 4, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 6, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 7, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 8, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 9, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 10, - "name": "edited_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 11, - "name": "server_id", - "type_info": "Varchar" - }, - { - "ordinal": 12, - "name": "last_server_ping", - "type_info": "Timestamptz" - }, - { - "ordinal": 13, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 14, - "name": "error", - "type_info": "Text" - }, - { - "ordinal": 15, - "name": "enabled", - "type_info": "Bool" - }, - { - "ordinal": 16, - "name": "error_handler_path", - "type_info": "Varchar" - }, - { - "ordinal": 17, - "name": "error_handler_args: _", - "type_info": "Jsonb" - }, - { - "ordinal": 18, - "name": "retry: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - true, - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - true, - false, - true, - true, - true - ] - }, - "hash": "58ddf5c76455d30dc42f46a0553e761461c327e7502b707c2d2742946d7d1f4c" -} diff --git a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json index 713ccb9dd3..36ddb8ab9f 100644 --- a/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json +++ b/backend/.sqlx/query-5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55.json @@ -15,7 +15,7 @@ ] }, "nullable": [ - null + true ] }, "hash": "5a219a2532517869578c4504ff3153c43903f929ae5d62fbba12610f89c36d55" diff --git a/backend/.sqlx/query-5bd8ae8d694ac9f6afef762276141d95ba195f5543120e76e68f6e6715b65a71.json b/backend/.sqlx/query-5bd8ae8d694ac9f6afef762276141d95ba195f5543120e76e68f6e6715b65a71.json deleted file mode 100644 index 31700319e2..0000000000 --- a/backend/.sqlx/query-5bd8ae8d694ac9f6afef762276141d95ba195f5543120e76e68f6e6715b65a71.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n sqs_trigger \n SET \n server_id = $1, \n last_server_ping = now(),\n error = 'Connecting...'\n WHERE \n enabled IS TRUE \n AND workspace_id = $2 \n AND path = $3 \n AND (last_server_ping IS NULL \n OR last_server_ping < now() - INTERVAL '15 seconds'\n ) \n RETURNING true\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "5bd8ae8d694ac9f6afef762276141d95ba195f5543120e76e68f6e6715b65a71" -} diff --git a/backend/.sqlx/query-615d832a452a6c64de50cd0efada0be238fb16daacb3464bbcf47ca2e21bdaae.json b/backend/.sqlx/query-615d832a452a6c64de50cd0efada0be238fb16daacb3464bbcf47ca2e21bdaae.json new file mode 100644 index 0000000000..fca8d2b883 --- /dev/null +++ b/backend/.sqlx/query-615d832a452a6c64de50cd0efada0be238fb16daacb3464bbcf47ca2e21bdaae.json @@ -0,0 +1,45 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE \n capture_config\n SET \n last_server_ping = now(), error = $1\n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = $5 AND \n server_id = $6 AND \n last_client_ping > NOW() - INTERVAL '10 seconds'\n RETURNING 1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Bool", + { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "postgres", + "sqs", + "mqtt", + "gcp" + ] + } + } + }, + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "615d832a452a6c64de50cd0efada0be238fb16daacb3464bbcf47ca2e21bdaae" +} diff --git a/backend/.sqlx/query-61bed1bc6d3e6a3c1d640eeacc290a85d8b63ee36c39dfbf4348d120f6e561ae.json b/backend/.sqlx/query-61bed1bc6d3e6a3c1d640eeacc290a85d8b63ee36c39dfbf4348d120f6e561ae.json deleted file mode 100644 index 7f9886ccb6..0000000000 --- a/backend/.sqlx/query-61bed1bc6d3e6a3c1d640eeacc290a85d8b63ee36c39dfbf4348d120f6e561ae.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n postgres_trigger \n SET\n last_server_ping = NULL \n WHERE \n workspace_id = $1 \n AND path = $2 \n AND server_id IS NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "61bed1bc6d3e6a3c1d640eeacc290a85d8b63ee36c39dfbf4348d120f6e561ae" -} diff --git a/backend/.sqlx/query-673564e6c4dcf30dae3d7a75c397998ea800860ede856e5fc6cd1f57d5408333.json b/backend/.sqlx/query-673564e6c4dcf30dae3d7a75c397998ea800860ede856e5fc6cd1f57d5408333.json deleted file mode 100644 index 8b9a46c4d0..0000000000 --- a/backend/.sqlx/query-673564e6c4dcf30dae3d7a75c397998ea800860ede856e5fc6cd1f57d5408333.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE \n capture_config \n SET \n last_server_ping = NULL \n WHERE \n workspace_id = $1 AND \n path = $2 AND \n is_flow = $3 AND \n trigger_kind = 'gcp' AND \n server_id IS NULL\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "673564e6c4dcf30dae3d7a75c397998ea800860ede856e5fc6cd1f57d5408333" -} diff --git a/backend/.sqlx/query-6b776693091191f36eaf9e35fb3cabb01481a19bef5a956dc36cac41acad0e53.json b/backend/.sqlx/query-6b776693091191f36eaf9e35fb3cabb01481a19bef5a956dc36cac41acad0e53.json deleted file mode 100644 index 97079df64f..0000000000 --- a/backend/.sqlx/query-6b776693091191f36eaf9e35fb3cabb01481a19bef5a956dc36cac41acad0e53.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n server_id = $1,\n last_server_ping = now(), \n error = 'Connecting...' \n WHERE \n last_client_ping > NOW() - INTERVAL '10 seconds' AND \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'sqs' AND \n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') \n RETURNING true\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text", - "Bool" - ] - }, - "nullable": [ - null - ] - }, - "hash": "6b776693091191f36eaf9e35fb3cabb01481a19bef5a956dc36cac41acad0e53" -} diff --git a/backend/.sqlx/query-6e70ebf078ac04a2933d2f83791973e8fc108d9f32be8a8501391052d76e191e.json b/backend/.sqlx/query-6e70ebf078ac04a2933d2f83791973e8fc108d9f32be8a8501391052d76e191e.json deleted file mode 100644 index d95c8c9b5c..0000000000 --- a/backend/.sqlx/query-6e70ebf078ac04a2933d2f83791973e8fc108d9f32be8a8501391052d76e191e.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE kafka_trigger SET last_server_ping = now(), error = $1 WHERE workspace_id = $2 AND path = $3 AND server_id = $4 AND enabled IS TRUE RETURNING 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "6e70ebf078ac04a2933d2f83791973e8fc108d9f32be8a8501391052d76e191e" -} diff --git a/backend/.sqlx/query-74d928f4c3f0de191f414471b9a4fbe9c20f9685b06ad5bbded424948b2dc88c.json b/backend/.sqlx/query-74d928f4c3f0de191f414471b9a4fbe9c20f9685b06ad5bbded424948b2dc88c.json deleted file mode 100644 index 0f72e41b9f..0000000000 --- a/backend/.sqlx/query-74d928f4c3f0de191f414471b9a4fbe9c20f9685b06ad5bbded424948b2dc88c.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n server_id = $1,\n last_server_ping = now(), \n error = 'Connecting...' \n WHERE \n last_client_ping > NOW() - INTERVAL '10 seconds' AND \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'postgres' AND \n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') \n RETURNING true\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text", - "Bool" - ] - }, - "nullable": [ - null - ] - }, - "hash": "74d928f4c3f0de191f414471b9a4fbe9c20f9685b06ad5bbded424948b2dc88c" -} diff --git a/backend/.sqlx/query-7e31c47e02492b74cbb5702dfc5ef9f4c8faa986843024a2cd58e212f706aae8.json b/backend/.sqlx/query-7e31c47e02492b74cbb5702dfc5ef9f4c8faa986843024a2cd58e212f706aae8.json deleted file mode 100644 index eea1cfa434..0000000000 --- a/backend/.sqlx/query-7e31c47e02492b74cbb5702dfc5ef9f4c8faa986843024a2cd58e212f706aae8.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'sqs'\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "7e31c47e02492b74cbb5702dfc5ef9f4c8faa986843024a2cd58e212f706aae8" -} diff --git a/backend/.sqlx/query-9116102c6ccad5b0d752d5d690c233dfe48062aef23072b4f4ae4ab5ca269082.json b/backend/.sqlx/query-9116102c6ccad5b0d752d5d690c233dfe48062aef23072b4f4ae4ab5ca269082.json deleted file mode 100644 index 6e58a79dd3..0000000000 --- a/backend/.sqlx/query-9116102c6ccad5b0d752d5d690c233dfe48062aef23072b4f4ae4ab5ca269082.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n postgres_trigger\n SET \n last_server_ping = now(),\n error = $1\n WHERE\n workspace_id = $2\n AND path = $3\n AND server_id = $4 \n AND enabled IS TRUE\n RETURNING 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "9116102c6ccad5b0d752d5d690c233dfe48062aef23072b4f4ae4ab5ca269082" -} diff --git a/backend/.sqlx/query-92c2b66eb6287449f5f8cc9f8d1329f748006ab131489d2fcac21d32641bb633.json b/backend/.sqlx/query-92c2b66eb6287449f5f8cc9f8d1329f748006ab131489d2fcac21d32641bb633.json deleted file mode 100644 index 14d37809d5..0000000000 --- a/backend/.sqlx/query-92c2b66eb6287449f5f8cc9f8d1329f748006ab131489d2fcac21d32641bb633.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n gcp_trigger \n SET \n server_id = $1, \n last_server_ping = now(),\n error = 'Connecting...'\n WHERE \n enabled IS TRUE \n AND workspace_id = $2 \n AND path = $3 \n AND (last_server_ping IS NULL \n OR last_server_ping < now() - INTERVAL '15 seconds'\n ) \n RETURNING true\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "92c2b66eb6287449f5f8cc9f8d1329f748006ab131489d2fcac21d32641bb633" -} diff --git a/backend/.sqlx/query-92e60af0d3ae8c73d74ae68d70e20ae18f79cc84626097b766307cd42722baa3.json b/backend/.sqlx/query-92e60af0d3ae8c73d74ae68d70e20ae18f79cc84626097b766307cd42722baa3.json deleted file mode 100644 index acc6769652..0000000000 --- a/backend/.sqlx/query-92e60af0d3ae8c73d74ae68d70e20ae18f79cc84626097b766307cd42722baa3.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE websocket_trigger SET server_id = $1, last_server_ping = now(), error = 'Connecting...' WHERE enabled IS TRUE AND workspace_id = $2 AND path = $3 AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') RETURNING true", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "92e60af0d3ae8c73d74ae68d70e20ae18f79cc84626097b766307cd42722baa3" -} diff --git a/backend/.sqlx/query-93d64930c74ccb1abdc9bda8287540a354f8eec49913aaf07fc1e737e7c93330.json b/backend/.sqlx/query-93d64930c74ccb1abdc9bda8287540a354f8eec49913aaf07fc1e737e7c93330.json deleted file mode 100644 index 5b30dd2cbd..0000000000 --- a/backend/.sqlx/query-93d64930c74ccb1abdc9bda8287540a354f8eec49913aaf07fc1e737e7c93330.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE capture_config SET last_server_ping = NULL WHERE workspace_id = $1 AND path = $2 AND is_flow = $3 AND trigger_kind = 'nats' AND server_id IS NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "93d64930c74ccb1abdc9bda8287540a354f8eec49913aaf07fc1e737e7c93330" -} diff --git a/backend/.sqlx/query-999edc6f54a9efb6dc6237992dad59f418ed6fc2a98ddb9bbc33dce5f029d904.json b/backend/.sqlx/query-999edc6f54a9efb6dc6237992dad59f418ed6fc2a98ddb9bbc33dce5f029d904.json deleted file mode 100644 index 8748572af7..0000000000 --- a/backend/.sqlx/query-999edc6f54a9efb6dc6237992dad59f418ed6fc2a98ddb9bbc33dce5f029d904.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n gcp_trigger \n SET\n last_server_ping = NULL \n WHERE \n workspace_id = $1 \n AND path = $2 \n AND server_id IS NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "999edc6f54a9efb6dc6237992dad59f418ed6fc2a98ddb9bbc33dce5f029d904" -} diff --git a/backend/.sqlx/query-99a2c935acf5d6bbeb70ea1255679115b6e9042800d40899a36b3867049c5c46.json b/backend/.sqlx/query-99a2c935acf5d6bbeb70ea1255679115b6e9042800d40899a36b3867049c5c46.json deleted file mode 100644 index f38b293acb..0000000000 --- a/backend/.sqlx/query-99a2c935acf5d6bbeb70ea1255679115b6e9042800d40899a36b3867049c5c46.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE capture_config SET server_id = $1, last_server_ping = now(), error = 'Connecting...' WHERE last_client_ping > NOW() - INTERVAL '10 seconds' AND workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'websocket' AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') RETURNING true", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text", - "Bool" - ] - }, - "nullable": [ - null - ] - }, - "hash": "99a2c935acf5d6bbeb70ea1255679115b6e9042800d40899a36b3867049c5c46" -} diff --git a/backend/.sqlx/query-a5fbef9db2308920ea26f6154f0f3490f88ced636c9ebdecba2205f964b1d0f1.json b/backend/.sqlx/query-a5fbef9db2308920ea26f6154f0f3490f88ced636c9ebdecba2205f964b1d0f1.json deleted file mode 100644 index 219558ac68..0000000000 --- a/backend/.sqlx/query-a5fbef9db2308920ea26f6154f0f3490f88ced636c9ebdecba2205f964b1d0f1.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n sqs_trigger \n SET\n last_server_ping = NULL \n WHERE \n workspace_id = $1 \n AND path = $2 \n AND server_id IS NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "a5fbef9db2308920ea26f6154f0f3490f88ced636c9ebdecba2205f964b1d0f1" -} diff --git a/backend/.sqlx/query-a7ffc5b983d365159ef379ec6a2ab0dc7217d1b3f86f362c213982984d2cf652.json b/backend/.sqlx/query-a7ffc5b983d365159ef379ec6a2ab0dc7217d1b3f86f362c213982984d2cf652.json deleted file mode 100644 index 03b05af6fb..0000000000 --- a/backend/.sqlx/query-a7ffc5b983d365159ef379ec6a2ab0dc7217d1b3f86f362c213982984d2cf652.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n last_server_ping = now(), \n error = $1 \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'gcp' AND \n server_id = $5 AND \n last_client_ping > NOW() - INTERVAL '10 seconds' \n RETURNING 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Bool", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "a7ffc5b983d365159ef379ec6a2ab0dc7217d1b3f86f362c213982984d2cf652" -} diff --git a/backend/.sqlx/query-ac9037b8adce156b95390a0ffac04e38ab8474849e0cacb3be1443d7f3265d30.json b/backend/.sqlx/query-ac9037b8adce156b95390a0ffac04e38ab8474849e0cacb3be1443d7f3265d30.json new file mode 100644 index 0000000000..4896bdeae1 --- /dev/null +++ b/backend/.sqlx/query-ac9037b8adce156b95390a0ffac04e38ab8474849e0cacb3be1443d7f3265d30.json @@ -0,0 +1,44 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE \n capture_config \n SET \n server_id = $1,\n last_server_ping = now(), \n error = 'Connecting...' \n WHERE \n last_client_ping > NOW() - INTERVAL '10 seconds' AND \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = $5 AND \n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') \n RETURNING true\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "?column?", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Text", + "Text", + "Bool", + { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "postgres", + "sqs", + "mqtt", + "gcp" + ] + } + } + } + ] + }, + "nullable": [ + null + ] + }, + "hash": "ac9037b8adce156b95390a0ffac04e38ab8474849e0cacb3be1443d7f3265d30" +} diff --git a/backend/.sqlx/query-add48c8e7c6fa2c549ad6293cbee22889d35e919d3267c1d2a265d868fa8a7d1.json b/backend/.sqlx/query-add48c8e7c6fa2c549ad6293cbee22889d35e919d3267c1d2a265d868fa8a7d1.json deleted file mode 100644 index 0468efc99f..0000000000 --- a/backend/.sqlx/query-add48c8e7c6fa2c549ad6293cbee22889d35e919d3267c1d2a265d868fa8a7d1.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE nats_trigger SET enabled = FALSE, error = $1, server_id = NULL, last_server_ping = NULL WHERE workspace_id = $2 AND path = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "add48c8e7c6fa2c549ad6293cbee22889d35e919d3267c1d2a265d868fa8a7d1" -} diff --git a/backend/.sqlx/query-b223c56f55a138abef8c8cb2df40472a87b7e267a143709728b09ac063c33b96.json b/backend/.sqlx/query-b223c56f55a138abef8c8cb2df40472a87b7e267a143709728b09ac063c33b96.json deleted file mode 100644 index f542ec075a..0000000000 --- a/backend/.sqlx/query-b223c56f55a138abef8c8cb2df40472a87b7e267a143709728b09ac063c33b96.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE capture_config SET last_server_ping = NULL WHERE workspace_id = $1 AND path = $2 AND is_flow = $3 AND trigger_kind = 'kafka' AND server_id IS NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "b223c56f55a138abef8c8cb2df40472a87b7e267a143709728b09ac063c33b96" -} diff --git a/backend/.sqlx/query-bcc4f786fa2771a4a265aec8d1c403e660002503aad860caa12ebee2dbab3f0f.json b/backend/.sqlx/query-bcc4f786fa2771a4a265aec8d1c403e660002503aad860caa12ebee2dbab3f0f.json deleted file mode 100644 index 5839ea94b4..0000000000 --- a/backend/.sqlx/query-bcc4f786fa2771a4a265aec8d1c403e660002503aad860caa12ebee2dbab3f0f.json +++ /dev/null @@ -1,134 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT\n workspace_id,\n path,\n nats_resource_path,\n subjects,\n stream_name,\n consumer_name,\n use_jetstream,\n script_path,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\"\n FROM nats_trigger\n WHERE enabled IS TRUE AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "nats_resource_path", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "subjects", - "type_info": "VarcharArray" - }, - { - "ordinal": 4, - "name": "stream_name", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "consumer_name", - "type_info": "Varchar" - }, - { - "ordinal": 6, - "name": "use_jetstream", - "type_info": "Bool" - }, - { - "ordinal": 7, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 8, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 9, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 10, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 11, - "name": "edited_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 12, - "name": "server_id", - "type_info": "Varchar" - }, - { - "ordinal": 13, - "name": "last_server_ping", - "type_info": "Timestamptz" - }, - { - "ordinal": 14, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 15, - "name": "error", - "type_info": "Text" - }, - { - "ordinal": 16, - "name": "enabled", - "type_info": "Bool" - }, - { - "ordinal": 17, - "name": "error_handler_path", - "type_info": "Varchar" - }, - { - "ordinal": 18, - "name": "error_handler_args: _", - "type_info": "Jsonb" - }, - { - "ordinal": 19, - "name": "retry: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - false, - true, - true, - false, - false, - false, - false, - false, - false, - true, - true, - false, - true, - false, - true, - true, - true - ] - }, - "hash": "bcc4f786fa2771a4a265aec8d1c403e660002503aad860caa12ebee2dbab3f0f" -} diff --git a/backend/.sqlx/query-bfc534d87d701d7ac78cc97d0054d829165ba3f22fba75c3161e4cddb72264ee.json b/backend/.sqlx/query-bfc534d87d701d7ac78cc97d0054d829165ba3f22fba75c3161e4cddb72264ee.json deleted file mode 100644 index 76f0f1486f..0000000000 --- a/backend/.sqlx/query-bfc534d87d701d7ac78cc97d0054d829165ba3f22fba75c3161e4cddb72264ee.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n postgres_trigger \n SET \n enabled = FALSE, \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "bfc534d87d701d7ac78cc97d0054d829165ba3f22fba75c3161e4cddb72264ee" -} diff --git a/backend/.sqlx/query-c0e6dbce7a401b06e1bf45155c3f81572818177a6024e743b12f558b46edf74c.json b/backend/.sqlx/query-c0e6dbce7a401b06e1bf45155c3f81572818177a6024e743b12f558b46edf74c.json deleted file mode 100644 index e65e3cb60e..0000000000 --- a/backend/.sqlx/query-c0e6dbce7a401b06e1bf45155c3f81572818177a6024e743b12f558b46edf74c.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n path,\n is_flow,\n workspace_id,\n owner,\n email,\n trigger_config as \"trigger_config!: _\"\n FROM\n capture_config\n WHERE\n trigger_kind = 'gcp' AND\n last_client_ping > NOW() - INTERVAL '10 seconds' AND\n trigger_config IS NOT NULL AND\n trigger_config->>'delivery_type' IS DISTINCT FROM 'push' AND\n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "owner", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "trigger_config!: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - false, - false, - true - ] - }, - "hash": "c0e6dbce7a401b06e1bf45155c3f81572818177a6024e743b12f558b46edf74c" -} diff --git a/backend/.sqlx/query-c5063e79aafa70b974276f5bea43ad71135d44b2e84c9efac43a6678f0cf9a18.json b/backend/.sqlx/query-c5063e79aafa70b974276f5bea43ad71135d44b2e84c9efac43a6678f0cf9a18.json deleted file mode 100644 index 9ab57a7e3a..0000000000 --- a/backend/.sqlx/query-c5063e79aafa70b974276f5bea43ad71135d44b2e84c9efac43a6678f0cf9a18.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n mqtt_trigger \n SET \n enabled = FALSE, \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "c5063e79aafa70b974276f5bea43ad71135d44b2e84c9efac43a6678f0cf9a18" -} diff --git a/backend/.sqlx/query-cba3bfb174829ee3b08ea195831fdba4335ca221a5fd25c72519ac376c593e44.json b/backend/.sqlx/query-cba3bfb174829ee3b08ea195831fdba4335ca221a5fd25c72519ac376c593e44.json deleted file mode 100644 index ebda0390f9..0000000000 --- a/backend/.sqlx/query-cba3bfb174829ee3b08ea195831fdba4335ca221a5fd25c72519ac376c593e44.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE capture_config SET last_server_ping = NULL WHERE workspace_id = $1 AND path = $2 AND is_flow = $3 AND trigger_kind = 'websocket' AND server_id IS NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "cba3bfb174829ee3b08ea195831fdba4335ca221a5fd25c72519ac376c593e44" -} diff --git a/backend/.sqlx/query-cd33a9d63f4706a7e3b1e23cd0a4b2e3ecb30aae6510d1fcd08493b07c8b0952.json b/backend/.sqlx/query-cd33a9d63f4706a7e3b1e23cd0a4b2e3ecb30aae6510d1fcd08493b07c8b0952.json deleted file mode 100644 index 36b1323980..0000000000 --- a/backend/.sqlx/query-cd33a9d63f4706a7e3b1e23cd0a4b2e3ecb30aae6510d1fcd08493b07c8b0952.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE websocket_trigger SET enabled = FALSE, error = $1, server_id = NULL, last_server_ping = NULL WHERE workspace_id = $2 AND path = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "cd33a9d63f4706a7e3b1e23cd0a4b2e3ecb30aae6510d1fcd08493b07c8b0952" -} diff --git a/backend/.sqlx/query-cd5d62d456b74237b941bc72ea8de007185263167bf1dfa2e469d319cc4da674.json b/backend/.sqlx/query-cd5d62d456b74237b941bc72ea8de007185263167bf1dfa2e469d319cc4da674.json deleted file mode 100644 index 912b9a53de..0000000000 --- a/backend/.sqlx/query-cd5d62d456b74237b941bc72ea8de007185263167bf1dfa2e469d319cc4da674.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE nats_trigger SET server_id = $1, last_server_ping = now(), error = 'Connecting...' WHERE enabled IS TRUE AND workspace_id = $2 AND path = $3 AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') RETURNING true", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "cd5d62d456b74237b941bc72ea8de007185263167bf1dfa2e469d319cc4da674" -} diff --git a/backend/.sqlx/query-cf76aedfdffc25057a64fc94fd6be0791d5970458810a2b465c27f02ad1d4b2f.json b/backend/.sqlx/query-cf76aedfdffc25057a64fc94fd6be0791d5970458810a2b465c27f02ad1d4b2f.json deleted file mode 100644 index 6557fadd07..0000000000 --- a/backend/.sqlx/query-cf76aedfdffc25057a64fc94fd6be0791d5970458810a2b465c27f02ad1d4b2f.json +++ /dev/null @@ -1,122 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n workspace_id,\n path,\n script_path,\n replication_slot_name,\n publication_name,\n is_flow,\n edited_by,\n email,\n edited_at,\n server_id,\n last_server_ping,\n extra_perms,\n error,\n enabled,\n postgres_resource_path,\n error_handler_path,\n error_handler_args as \"error_handler_args: _\",\n retry as \"retry: _\"\n FROM\n postgres_trigger\n WHERE\n enabled IS TRUE\n AND (last_server_ping IS NULL OR\n last_server_ping < now() - interval '15 seconds'\n )\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "script_path", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "replication_slot_name", - "type_info": "Varchar" - }, - { - "ordinal": 4, - "name": "publication_name", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 6, - "name": "edited_by", - "type_info": "Varchar" - }, - { - "ordinal": 7, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 8, - "name": "edited_at", - "type_info": "Timestamptz" - }, - { - "ordinal": 9, - "name": "server_id", - "type_info": "Varchar" - }, - { - "ordinal": 10, - "name": "last_server_ping", - "type_info": "Timestamptz" - }, - { - "ordinal": 11, - "name": "extra_perms", - "type_info": "Jsonb" - }, - { - "ordinal": 12, - "name": "error", - "type_info": "Text" - }, - { - "ordinal": 13, - "name": "enabled", - "type_info": "Bool" - }, - { - "ordinal": 14, - "name": "postgres_resource_path", - "type_info": "Varchar" - }, - { - "ordinal": 15, - "name": "error_handler_path", - "type_info": "Varchar" - }, - { - "ordinal": 16, - "name": "error_handler_args: _", - "type_info": "Jsonb" - }, - { - "ordinal": 17, - "name": "retry: _", - "type_info": "Jsonb" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - false, - false, - false, - false, - false, - false, - true, - true, - true, - true, - false, - false, - true, - true, - true - ] - }, - "hash": "cf76aedfdffc25057a64fc94fd6be0791d5970458810a2b465c27f02ad1d4b2f" -} diff --git a/backend/.sqlx/query-cff764601318bfecb8e0f15b64ff9f430d7b1efe8ba863cbcc4a49bab4951794.json b/backend/.sqlx/query-cff764601318bfecb8e0f15b64ff9f430d7b1efe8ba863cbcc4a49bab4951794.json deleted file mode 100644 index a9e508d01d..0000000000 --- a/backend/.sqlx/query-cff764601318bfecb8e0f15b64ff9f430d7b1efe8ba863cbcc4a49bab4951794.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n last_server_ping = now(), \n error = $1 \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'mqtt' AND \n server_id = $5 AND \n last_client_ping > NOW() - INTERVAL '10 seconds' \n RETURNING 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Bool", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "cff764601318bfecb8e0f15b64ff9f430d7b1efe8ba863cbcc4a49bab4951794" -} diff --git a/backend/.sqlx/query-d06efdc24706e0d7479bffc0b19a0c5976ee60125289e9f7b0b04090bce4a3a3.json b/backend/.sqlx/query-d06efdc24706e0d7479bffc0b19a0c5976ee60125289e9f7b0b04090bce4a3a3.json deleted file mode 100644 index 79fe0873dd..0000000000 --- a/backend/.sqlx/query-d06efdc24706e0d7479bffc0b19a0c5976ee60125289e9f7b0b04090bce4a3a3.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE capture_config SET server_id = $1, last_server_ping = now(), error = 'Connecting...' WHERE last_client_ping > NOW() - INTERVAL '10 seconds' AND workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'kafka' AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') RETURNING true", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text", - "Bool" - ] - }, - "nullable": [ - null - ] - }, - "hash": "d06efdc24706e0d7479bffc0b19a0c5976ee60125289e9f7b0b04090bce4a3a3" -} diff --git a/backend/.sqlx/query-d08f34000c3d96ccd0f44ca8520f966d751a4dda554d8215eedb8f65be98e100.json b/backend/.sqlx/query-d08f34000c3d96ccd0f44ca8520f966d751a4dda554d8215eedb8f65be98e100.json deleted file mode 100644 index 431ffc04a2..0000000000 --- a/backend/.sqlx/query-d08f34000c3d96ccd0f44ca8520f966d751a4dda554d8215eedb8f65be98e100.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE capture_config SET error = $1, server_id = NULL, last_server_ping = NULL WHERE workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'websocket'", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "d08f34000c3d96ccd0f44ca8520f966d751a4dda554d8215eedb8f65be98e100" -} diff --git a/backend/.sqlx/query-d78ecf85c1e1e95650c380c9488aa90d8d8c5c76f3971484e4c515ff60293d3b.json b/backend/.sqlx/query-d78ecf85c1e1e95650c380c9488aa90d8d8c5c76f3971484e4c515ff60293d3b.json deleted file mode 100644 index cdb8b928b3..0000000000 --- a/backend/.sqlx/query-d78ecf85c1e1e95650c380c9488aa90d8d8c5c76f3971484e4c515ff60293d3b.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE kafka_trigger SET server_id = $1, last_server_ping = now(), error = 'Connecting...' WHERE enabled IS TRUE AND workspace_id = $2 AND path = $3 AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') RETURNING true", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "d78ecf85c1e1e95650c380c9488aa90d8d8c5c76f3971484e4c515ff60293d3b" -} diff --git a/backend/.sqlx/query-d8ea17fba0e417333e9c6cf82ad36a57830a791c8358e30e9be012b815f5e8e3.json b/backend/.sqlx/query-d8ea17fba0e417333e9c6cf82ad36a57830a791c8358e30e9be012b815f5e8e3.json deleted file mode 100644 index dcaee3cfe8..0000000000 --- a/backend/.sqlx/query-d8ea17fba0e417333e9c6cf82ad36a57830a791c8358e30e9be012b815f5e8e3.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT path, is_flow, workspace_id, trigger_config as \"trigger_config!: _\", owner, email FROM capture_config WHERE trigger_kind = 'kafka' AND last_client_ping > NOW() - INTERVAL '10 seconds' AND trigger_config IS NOT NULL AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "path", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "is_flow", - "type_info": "Bool" - }, - { - "ordinal": 2, - "name": "workspace_id", - "type_info": "Varchar" - }, - { - "ordinal": 3, - "name": "trigger_config!: _", - "type_info": "Jsonb" - }, - { - "ordinal": 4, - "name": "owner", - "type_info": "Varchar" - }, - { - "ordinal": 5, - "name": "email", - "type_info": "Varchar" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false, - false, - false, - true, - false, - false - ] - }, - "hash": "d8ea17fba0e417333e9c6cf82ad36a57830a791c8358e30e9be012b815f5e8e3" -} diff --git a/backend/.sqlx/query-d9a6f75e4c4a1f61e55b313cc09bceffac637548841897341672da427a9140fc.json b/backend/.sqlx/query-d9a6f75e4c4a1f61e55b313cc09bceffac637548841897341672da427a9140fc.json deleted file mode 100644 index 3ccfcc9582..0000000000 --- a/backend/.sqlx/query-d9a6f75e4c4a1f61e55b313cc09bceffac637548841897341672da427a9140fc.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE capture_config SET last_server_ping = now(), error = $1 WHERE workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'kafka' AND server_id = $5 AND last_client_ping > NOW() - INTERVAL '10 seconds' RETURNING 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Bool", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "d9a6f75e4c4a1f61e55b313cc09bceffac637548841897341672da427a9140fc" -} diff --git a/backend/.sqlx/query-43c8cd9f8560412bb06d9966ccfa2524943ba277c5f9a37c06bd5ee14fd46bda.json b/backend/.sqlx/query-db7b39335049f7b5fbb1ba2b99618eeeccdd4b7e14a0c0077af9d978f99ae899.json similarity index 51% rename from backend/.sqlx/query-43c8cd9f8560412bb06d9966ccfa2524943ba277c5f9a37c06bd5ee14fd46bda.json rename to backend/.sqlx/query-db7b39335049f7b5fbb1ba2b99618eeeccdd4b7e14a0c0077af9d978f99ae899.json index 7d6ecdbf32..5cb129c672 100644 --- a/backend/.sqlx/query-43c8cd9f8560412bb06d9966ccfa2524943ba277c5f9a37c06bd5ee14fd46bda.json +++ b/backend/.sqlx/query-db7b39335049f7b5fbb1ba2b99618eeeccdd4b7e14a0c0077af9d978f99ae899.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'gcp'\n ", + "query": "\n UPDATE \n capture_config \n SET \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = $5\n ", "describe": { "columns": [], "parameters": { @@ -8,10 +8,29 @@ "Text", "Text", "Text", - "Bool" + "Bool", + { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "postgres", + "sqs", + "mqtt", + "gcp" + ] + } + } + } ] }, "nullable": [] }, - "hash": "43c8cd9f8560412bb06d9966ccfa2524943ba277c5f9a37c06bd5ee14fd46bda" + "hash": "db7b39335049f7b5fbb1ba2b99618eeeccdd4b7e14a0c0077af9d978f99ae899" } diff --git a/backend/.sqlx/query-dcaf17a826e8f4cba4145abcf72bf749ad1d4381fa3b9df8b5bf534f9c13692e.json b/backend/.sqlx/query-dcaf17a826e8f4cba4145abcf72bf749ad1d4381fa3b9df8b5bf534f9c13692e.json new file mode 100644 index 0000000000..a86bfa2b3e --- /dev/null +++ b/backend/.sqlx/query-dcaf17a826e8f4cba4145abcf72bf749ad1d4381fa3b9df8b5bf534f9c13692e.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE \n capture_config\n SET \n last_server_ping = NULL\n WHERE \n workspace_id = $1 AND \n path = $2 AND \n is_flow = $3 AND \n trigger_kind = $4 AND \n server_id IS NULL\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text", + "Bool", + { + "Custom": { + "name": "trigger_kind", + "kind": { + "Enum": [ + "webhook", + "http", + "websocket", + "kafka", + "email", + "nats", + "postgres", + "sqs", + "mqtt", + "gcp" + ] + } + } + } + ] + }, + "nullable": [] + }, + "hash": "dcaf17a826e8f4cba4145abcf72bf749ad1d4381fa3b9df8b5bf534f9c13692e" +} diff --git a/backend/.sqlx/query-dcf03ff4b922b93be37d2be5da6884ae3e8c6cf7eaa2d9c62056366eb42f2276.json b/backend/.sqlx/query-dcf03ff4b922b93be37d2be5da6884ae3e8c6cf7eaa2d9c62056366eb42f2276.json deleted file mode 100644 index 989e4612ec..0000000000 --- a/backend/.sqlx/query-dcf03ff4b922b93be37d2be5da6884ae3e8c6cf7eaa2d9c62056366eb42f2276.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE nats_trigger SET last_server_ping = now(), error = $1 WHERE workspace_id = $2 AND path = $3 AND server_id = $4 AND enabled IS TRUE RETURNING 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "dcf03ff4b922b93be37d2be5da6884ae3e8c6cf7eaa2d9c62056366eb42f2276" -} diff --git a/backend/.sqlx/query-df2a426658f0a36683cc2163fe70912d0221d191b8ab091601cb53cdd932ad58.json b/backend/.sqlx/query-df2a426658f0a36683cc2163fe70912d0221d191b8ab091601cb53cdd932ad58.json deleted file mode 100644 index f98dd01404..0000000000 --- a/backend/.sqlx/query-df2a426658f0a36683cc2163fe70912d0221d191b8ab091601cb53cdd932ad58.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n gcp_trigger\n SET \n last_server_ping = now(),\n error = $1\n WHERE\n workspace_id = $2\n AND path = $3\n AND server_id = $4 \n AND enabled IS TRUE\n RETURNING 1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "df2a426658f0a36683cc2163fe70912d0221d191b8ab091601cb53cdd932ad58" -} diff --git a/backend/.sqlx/query-e37ba13aa3174931f0bfcff26dbc141fe8a346ab6a1a3bc924794bfe3c9af306.json b/backend/.sqlx/query-e37ba13aa3174931f0bfcff26dbc141fe8a346ab6a1a3bc924794bfe3c9af306.json deleted file mode 100644 index cec42e51c6..0000000000 --- a/backend/.sqlx/query-e37ba13aa3174931f0bfcff26dbc141fe8a346ab6a1a3bc924794bfe3c9af306.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE capture_config SET last_server_ping = now(), error = $1 WHERE workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'nats' AND server_id = $5 AND last_client_ping > NOW() - INTERVAL '10 seconds' RETURNING 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Bool", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "e37ba13aa3174931f0bfcff26dbc141fe8a346ab6a1a3bc924794bfe3c9af306" -} diff --git a/backend/.sqlx/query-e38a66d15382703b99054a9b609af9f3854bb65393d33b265476c0e68aec4f61.json b/backend/.sqlx/query-e38a66d15382703b99054a9b609af9f3854bb65393d33b265476c0e68aec4f61.json deleted file mode 100644 index 78e5750992..0000000000 --- a/backend/.sqlx/query-e38a66d15382703b99054a9b609af9f3854bb65393d33b265476c0e68aec4f61.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE nats_trigger SET last_server_ping = NULL WHERE workspace_id = $1 AND path = $2 AND server_id IS NULL", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "e38a66d15382703b99054a9b609af9f3854bb65393d33b265476c0e68aec4f61" -} diff --git a/backend/.sqlx/query-f050be8da0d99aa1af64f5d32a8df01c5a59b036b2669878cc557537efb492c8.json b/backend/.sqlx/query-f050be8da0d99aa1af64f5d32a8df01c5a59b036b2669878cc557537efb492c8.json deleted file mode 100644 index 7334c70fb4..0000000000 --- a/backend/.sqlx/query-f050be8da0d99aa1af64f5d32a8df01c5a59b036b2669878cc557537efb492c8.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'mqtt'\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "f050be8da0d99aa1af64f5d32a8df01c5a59b036b2669878cc557537efb492c8" -} diff --git a/backend/.sqlx/query-f16b00bad2880f896f4452e5894fe101cd5c2fe3e4c143c0b979668143d85dd2.json b/backend/.sqlx/query-f16b00bad2880f896f4452e5894fe101cd5c2fe3e4c143c0b979668143d85dd2.json deleted file mode 100644 index a1ce88bd88..0000000000 --- a/backend/.sqlx/query-f16b00bad2880f896f4452e5894fe101cd5c2fe3e4c143c0b979668143d85dd2.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE capture_config SET server_id = $1, last_server_ping = now(), error = 'Connecting...' WHERE last_client_ping > NOW() - INTERVAL '10 seconds' AND workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'nats' AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') RETURNING true", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text", - "Bool" - ] - }, - "nullable": [ - null - ] - }, - "hash": "f16b00bad2880f896f4452e5894fe101cd5c2fe3e4c143c0b979668143d85dd2" -} diff --git a/backend/.sqlx/query-fc932ecf697b921fe9143c36872e9cc4e9aca7f9129b0466aacae51334a3db96.json b/backend/.sqlx/query-fc932ecf697b921fe9143c36872e9cc4e9aca7f9129b0466aacae51334a3db96.json deleted file mode 100644 index 091d205b40..0000000000 --- a/backend/.sqlx/query-fc932ecf697b921fe9143c36872e9cc4e9aca7f9129b0466aacae51334a3db96.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n capture_config \n SET \n server_id = $1,\n last_server_ping = now(), \n error = 'Connecting...' \n WHERE \n last_client_ping > NOW() - INTERVAL '10 seconds' AND \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'mqtt' AND \n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') \n RETURNING true\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text", - "Bool" - ] - }, - "nullable": [ - null - ] - }, - "hash": "fc932ecf697b921fe9143c36872e9cc4e9aca7f9129b0466aacae51334a3db96" -} diff --git a/backend/.sqlx/query-fd0a0ca4a107dc813240ad71f372ab7e0bf26431158fbfb5f0023ed847ca7dc5.json b/backend/.sqlx/query-fd0a0ca4a107dc813240ad71f372ab7e0bf26431158fbfb5f0023ed847ca7dc5.json deleted file mode 100644 index d47bb3fd68..0000000000 --- a/backend/.sqlx/query-fd0a0ca4a107dc813240ad71f372ab7e0bf26431158fbfb5f0023ed847ca7dc5.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE \n gcp_trigger \n SET \n enabled = FALSE, \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text" - ] - }, - "nullable": [] - }, - "hash": "fd0a0ca4a107dc813240ad71f372ab7e0bf26431158fbfb5f0023ed847ca7dc5" -} diff --git a/backend/.sqlx/query-febd70e6d5304ad912c232d3d8fc3f8ce0133d8a1a9e5ca1124eff9a988dc2d7.json b/backend/.sqlx/query-febd70e6d5304ad912c232d3d8fc3f8ce0133d8a1a9e5ca1124eff9a988dc2d7.json deleted file mode 100644 index 7958f9d2f3..0000000000 --- a/backend/.sqlx/query-febd70e6d5304ad912c232d3d8fc3f8ce0133d8a1a9e5ca1124eff9a988dc2d7.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE websocket_trigger SET last_server_ping = now(), error = $1 WHERE workspace_id = $2 AND path = $3 AND server_id = $4 AND enabled IS TRUE RETURNING 1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "?column?", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "febd70e6d5304ad912c232d3d8fc3f8ce0133d8a1a9e5ca1124eff9a988dc2d7" -} diff --git a/backend/.sqlx/query-ffb6b2a40b605f9c55826a0e519af80b100d2ffa58ec567223e27099d364fcc8.json b/backend/.sqlx/query-ffb6b2a40b605f9c55826a0e519af80b100d2ffa58ec567223e27099d364fcc8.json deleted file mode 100644 index c75cb3385a..0000000000 --- a/backend/.sqlx/query-ffb6b2a40b605f9c55826a0e519af80b100d2ffa58ec567223e27099d364fcc8.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE capture_config SET error = $1, server_id = NULL, last_server_ping = NULL WHERE workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'nats'", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Text", - "Text", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "ffb6b2a40b605f9c55826a0e519af80b100d2ffa58ec567223e27099d364fcc8" -} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index f53e3b3b76..fed66058fd 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -ef50587d21cc765c2f32ad95622f84d68d1e91a3 +f860277e8a5719c59d8961856fd84e3be5c3a83b \ No newline at end of file diff --git a/backend/windmill-api/src/capture.rs b/backend/windmill-api/src/capture.rs index 1231bcf042..603c040475 100644 --- a/backend/windmill-api/src/capture.rs +++ b/backend/windmill-api/src/capture.rs @@ -46,10 +46,10 @@ use serde::de::DeserializeOwned; use windmill_common::error::Error; #[cfg(all(feature = "enterprise", feature = "kafka", feature = "private"))] -use crate::kafka_triggers_ee::KafkaTriggerConfigConnection; +use crate::triggers::kafka::KafkaTriggerConfigConnection; #[cfg(feature = "mqtt_trigger")] -use crate::mqtt_triggers::{MqttClientVersion, MqttV3Config, MqttV5Config, SubscribeTopic}; +use crate::triggers::mqtt::{MqttClientVersion, MqttV3Config, MqttV5Config, SubscribeTopic}; #[cfg(all(feature = "enterprise", feature = "nats", feature = "private"))] use crate::triggers::nats::NatsTriggerConfigConnection; diff --git a/backend/windmill-api/src/ee_oss.rs b/backend/windmill-api/src/ee_oss.rs index cec4e4db24..a69c59c573 100644 --- a/backend/windmill-api/src/ee_oss.rs +++ b/backend/windmill-api/src/ee_oss.rs @@ -11,10 +11,7 @@ use {crate::db::ApiAuthed, windmill_common::DB}; #[cfg(not(feature = "private"))] use anyhow::anyhow; #[cfg(all(feature = "enterprise", not(feature = "private")))] -use std::sync::Arc; -#[cfg(all(feature = "enterprise", not(feature = "private")))] -use tokio::sync::RwLock; - +use {std::sync::Arc, tokio::sync::RwLock}; #[cfg(not(feature = "private"))] pub async fn validate_license_key(_license_key: String) -> anyhow::Result<(String, bool)> { // Implementation is not open source diff --git a/backend/windmill-api/src/gcp_triggers_oss.rs b/backend/windmill-api/src/gcp_triggers_oss.rs deleted file mode 100644 index ce2f81f3c9..0000000000 --- a/backend/windmill-api/src/gcp_triggers_oss.rs +++ /dev/null @@ -1,120 +0,0 @@ -#[cfg(feature = "private")] -#[allow(unused)] -pub use crate::gcp_triggers_ee::*; - -#[cfg(not(feature = "private"))] -use { - crate::{db::DB, trigger_helpers::TriggerJobArgs}, - serde::{Deserialize, Serialize}, - serde_json::value::RawValue, - sqlx::FromRow, - std::collections::HashMap, - windmill_common::{triggers::TriggerKind, utils::empty_as_none, worker::to_raw_value}, -}; - -#[cfg(not(feature = "private"))] -type SqlxJson = sqlx::types::Json; - -#[derive(sqlx::Type, Debug, Deserialize, Serialize)] -#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] -#[sqlx(type_name = "DELIVERY_MODE", rename_all = "lowercase")] -#[allow(unused)] -#[cfg(not(feature = "private"))] -pub enum DeliveryType { - Pull, - Push, -} - -#[cfg(not(feature = "private"))] -impl Default for DeliveryType { - fn default() -> Self { - Self::Pull - } -} - -#[derive(FromRow, Deserialize, Serialize, Debug)] -#[allow(unused)] -#[cfg(not(feature = "private"))] -pub struct PushConfig { - #[serde(deserialize_with = "empty_as_none")] - route_path: Option, - #[serde(deserialize_with = "empty_as_none")] - audience: Option, - authenticate: bool, - base_endpoint: String, -} -#[derive(Default, Debug, Serialize, Deserialize)] -#[allow(unused)] -#[cfg(not(feature = "private"))] -pub struct CreateUpdateConfig { - pub delivery_type: DeliveryType, - #[serde(default, deserialize_with = "empty_as_none")] - pub subscription_id: Option, - pub delivery_config: Option>, -} - -#[derive(Debug, Deserialize, Serialize)] -#[cfg(not(feature = "private"))] -pub struct ExistingGcpSubscription { - pub subscription_id: String, - pub base_endpoint: String, -} - -#[derive(Debug, Deserialize, Serialize, sqlx::Type)] -#[serde(rename_all = "snake_case")] -#[sqlx(type_name = "GCP_SUBSCRIPTION_MODE", rename_all = "snake_case")] -#[cfg(not(feature = "private"))] -pub enum SubscriptionMode { - Existing, - CreateUpdate, -} - -#[cfg(not(feature = "private"))] -pub fn start_consuming_gcp_pubsub_event( - _db: DB, - mut _killpill_rx: tokio::sync::broadcast::Receiver<()>, -) -> () { - // implementation is not open source -} - -#[cfg(not(feature = "private"))] -#[derive(FromRow, Deserialize, Serialize, Debug)] -pub struct GcpTrigger { - pub gcp_resource_path: String, - pub subscription_id: String, - pub delivery_type: DeliveryType, - pub delivery_config: Option>, - pub subscription_mode: SubscriptionMode, - pub topic_id: String, - pub path: String, - pub script_path: String, - pub is_flow: bool, - pub workspace_id: String, - pub edited_by: String, - pub email: String, - pub edited_at: chrono::DateTime, - pub extra_perms: Option, - pub error: Option, - pub server_id: Option, - pub last_server_ping: Option>, - pub enabled: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_args: Option>>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub retry: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub auto_acknowledge_msg: Option, -} - -#[cfg(not(feature = "private"))] -impl TriggerJobArgs for GcpTrigger { - fn v1_payload_fn(payload: String) -> HashMap> { - HashMap::from([("payload".to_string(), to_raw_value(&payload))]) - } - - fn trigger_kind() -> TriggerKind { - TriggerKind::Gcp - } -} diff --git a/backend/windmill-api/src/kafka_triggers_oss.rs b/backend/windmill-api/src/kafka_triggers_oss.rs deleted file mode 100644 index 4227023648..0000000000 --- a/backend/windmill-api/src/kafka_triggers_oss.rs +++ /dev/null @@ -1,55 +0,0 @@ -#[cfg(feature = "private")] -#[allow(unused)] -pub use crate::kafka_triggers_ee::*; - -#[cfg(not(feature = "private"))] -use crate::db::DB; -#[cfg(not(feature = "private"))] -use serde::{Deserialize, Serialize}; - -#[derive(Serialize, Deserialize)] -#[cfg(not(feature = "private"))] -pub struct KafkaResourceSecurity {} - -#[cfg(not(feature = "private"))] -pub fn start_kafka_consumers( - _db: DB, - mut _killpill_rx: tokio::sync::broadcast::Receiver<()>, -) -> () { - // implementation is not open source -} - -#[derive(Serialize, Deserialize)] -#[cfg(not(feature = "private"))] -pub enum KafkaTriggerConfigConnection {} - -#[derive(Serialize, Clone)] -#[cfg(not(feature = "private"))] -pub struct KafkaTrigger { - pub workspace_id: String, - pub path: String, - pub kafka_resource_path: String, - pub group_id: String, - pub topics: Vec, - pub script_path: String, - pub is_flow: bool, - pub edited_by: String, - pub email: String, - pub edited_at: chrono::DateTime, - #[serde(skip_serializing_if = "Option::is_none")] - pub server_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub last_server_ping: Option>, - pub extra_perms: serde_json::Value, - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, - pub enabled: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_args: Option< - sqlx::types::Json>>, - >, - #[serde(skip_serializing_if = "Option::is_none")] - pub retry: Option>, -} diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index c57bec62a1..26e2a803c7 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -20,7 +20,7 @@ use crate::smtp_server_oss::SmtpServer; #[cfg(feature = "mcp")] use crate::mcp::{extract_and_store_workspace_id, setup_mcp_server, shutdown_mcp_server}; -use http::StatusCode; +use crate::triggers::start_all_listeners; #[cfg(feature = "mcp")] use rmcp::transport::streamable_http_server::session::local::LocalSessionManager; use tower_http::catch_panic::CatchPanicLayer; @@ -100,14 +100,9 @@ mod inkeep_oss; mod inputs; mod integration; mod live_migrations; -#[cfg(feature = "postgres_trigger")] -mod postgres_triggers; #[cfg(all(feature = "private", feature = "parquet"))] pub mod s3_proxy_ee; mod s3_proxy_oss; - -mod trigger_helpers; - pub mod openapi; mod approvals; @@ -115,10 +110,6 @@ mod approvals; pub mod apps_ee; #[cfg(feature = "enterprise")] mod apps_oss; -#[cfg(all(feature = "enterprise", feature = "gcp_trigger", feature = "private"))] -pub mod gcp_triggers_ee; -#[cfg(all(feature = "enterprise", feature = "gcp_trigger"))] -mod gcp_triggers_oss; #[cfg(all(feature = "enterprise", feature = "private"))] pub mod git_sync_ee; #[cfg(feature = "enterprise")] @@ -129,16 +120,6 @@ pub mod job_helpers_ee; mod job_helpers_oss; pub mod job_metrics; pub mod jobs; -#[cfg(all(feature = "enterprise", feature = "kafka", feature = "private"))] -pub mod kafka_triggers_ee; -#[cfg(all(feature = "enterprise", feature = "kafka"))] -mod kafka_triggers_oss; -#[cfg(feature = "mqtt_trigger")] -mod mqtt_triggers; -#[cfg(all(feature = "enterprise", feature = "nats", feature = "private"))] -pub mod nats_triggers_ee; -#[cfg(all(feature = "enterprise", feature = "nats"))] -mod nats_triggers_oss; #[cfg(all(feature = "oauth2", feature = "private"))] pub mod oauth2_ee; #[cfg(feature = "oauth2")] @@ -164,10 +145,6 @@ mod slack_approvals; pub mod smtp_server_ee; #[cfg(feature = "smtp")] mod smtp_server_oss; -#[cfg(all(feature = "enterprise", feature = "sqs_trigger", feature = "private"))] -pub mod sqs_triggers_ee; -#[cfg(all(feature = "enterprise", feature = "sqs_trigger"))] -mod sqs_triggers_oss; #[cfg(feature = "private")] pub mod teams_approvals_ee; mod teams_approvals_oss; @@ -191,8 +168,6 @@ mod utils; pub mod var_resource_cache; mod variables; pub mod webhook_util; -#[cfg(feature = "websocket")] -mod websocket_triggers; mod workers; mod workspaces; #[cfg(feature = "private")] @@ -391,47 +366,7 @@ pub async fn run_server( let triggers_service = triggers::generate_trigger_routers(); if !*CLOUD_HOSTED && server_mode && !mcp_mode { - #[cfg(feature = "websocket")] - { - let ws_killpill_rx = killpill_rx.resubscribe(); - websocket_triggers::start_websockets(db.clone(), ws_killpill_rx); - } - - #[cfg(all(feature = "enterprise", feature = "kafka"))] - { - let kafka_killpill_rx = killpill_rx.resubscribe(); - kafka_triggers_oss::start_kafka_consumers(db.clone(), kafka_killpill_rx); - } - - #[cfg(all(feature = "enterprise", feature = "nats"))] - { - let nats_killpill_rx = killpill_rx.resubscribe(); - nats_triggers_oss::start_nats_consumers(db.clone(), nats_killpill_rx); - } - - #[cfg(feature = "postgres_trigger")] - { - let db_killpill_rx = killpill_rx.resubscribe(); - postgres_triggers::start_database(db.clone(), db_killpill_rx); - } - - #[cfg(feature = "mqtt_trigger")] - { - let mqtt_killpill_rx = killpill_rx.resubscribe(); - mqtt_triggers::start_mqtt_consumer(db.clone(), mqtt_killpill_rx); - } - - #[cfg(all(feature = "enterprise", feature = "sqs_trigger"))] - { - let sqs_killpill_rx = killpill_rx.resubscribe(); - sqs_triggers_oss::start_sqs(db.clone(), sqs_killpill_rx); - } - - #[cfg(all(feature = "enterprise", feature = "gcp_trigger"))] - { - let gcp_killpill_rx = killpill_rx.resubscribe(); - gcp_triggers_oss::start_consuming_gcp_pubsub_event(db.clone(), gcp_killpill_rx); - } + start_all_listeners(db.clone(), &killpill_rx); } let listener = tokio::net::TcpListener::bind(addr) @@ -740,7 +675,7 @@ pub async fn run_server( let app = app.layer(CatchPanicLayer::custom(|err| { tracing::error!("panic in handler, returning 500: {:?}", err); Response::builder() - .status(StatusCode::INTERNAL_SERVER_ERROR) + .status(http::StatusCode::INTERNAL_SERVER_ERROR) .body(Body::from("Internal Server Error")) .unwrap() })); diff --git a/backend/windmill-api/src/mqtt_triggers.rs b/backend/windmill-api/src/mqtt_triggers.rs deleted file mode 100644 index 2b8c16fc6d..0000000000 --- a/backend/windmill-api/src/mqtt_triggers.rs +++ /dev/null @@ -1,1277 +0,0 @@ -use crate::{ - capture::{insert_capture_payload, MqttTriggerConfig}, - db::{ApiAuthed, DB}, - resources::try_get_resource_from_db_as, - trigger_helpers::{trigger_runnable, TriggerJobArgs}, - users::fetch_api_authed, -}; - -use axum::async_trait; -use base64::{engine, prelude::*}; -use bytes::Bytes; -use itertools::Itertools; -use rumqttc::{ - v5::{ - mqttbytes::{ - v5::{ConnectProperties, Filter, PublishProperties}, - QoS as V5QoS, - }, - AsyncClient as V5AsyncClient, Event as V5Event, EventLoop as V5EventLoop, - Incoming as V5Incoming, MqttOptions as V5MqttOptions, - }, - AsyncClient as V3AsyncClient, Event as V3Event, EventLoop as V3EventLoop, - Incoming as V3Incoming, MqttOptions as V3MqttOptions, QoS as V3QoS, SubscribeFilter, - TlsConfiguration, Transport, -}; -use serde::{Deserialize, Serialize}; -use sqlx::{FromRow, Type}; -use std::collections::HashMap; -use std::time::Duration; -use windmill_common::{ - db::UserDB, - error::{to_anyhow, Error, Result}, - triggers::TriggerKind, - utils::report_critical_error, - worker::to_raw_value, - INSTANCE_NAME, -}; - -use rand::seq::SliceRandom; -use serde_json::value::RawValue; -use sqlx::types::Json as SqlxJson; - -async fn run_job( - payload: &[u8], - trigger_info: HashMap>, - db: &DB, - trigger: &MqttTrigger, -) -> anyhow::Result<()> { - let args = MqttTrigger::build_job_args( - &trigger.script_path, - trigger.is_flow, - &trigger.workspace_id, - db, - payload, - trigger_info, - ) - .await?; - - let authed = fetch_api_authed( - trigger.edited_by.clone(), - trigger.email.clone(), - &trigger.workspace_id, - db, - Some(format!("mqtt-{}", trigger.path)), - ) - .await?; - - trigger_runnable( - db, - None, - authed, - &trigger.workspace_id, - &trigger.script_path, - trigger.is_flow, - args, - trigger.retry.as_ref(), - trigger.error_handler_path.as_deref(), - trigger.error_handler_args.as_ref(), - format!("mqtt_trigger/{}", trigger.path), - ) - .await?; - - Ok(()) -} - -#[derive(Debug, Deserialize, Serialize)] -pub struct MqttV3Config { - clean_session: Option, -} - -#[derive(Debug, Deserialize, Serialize)] -pub struct MqttV5Config { - clean_start: Option, - session_expiry_interval: Option, - topic_alias_maximum: Option, -} - -#[derive(Clone, Debug, Deserialize, Serialize, Type)] -#[serde(rename_all = "lowercase")] -pub enum QualityOfService { - Qos0, - Qos1, - Qos2, -} - -impl From for V3QoS { - fn from(value: QualityOfService) -> Self { - match value { - QualityOfService::Qos0 => V3QoS::AtMostOnce, - QualityOfService::Qos1 => V3QoS::AtLeastOnce, - QualityOfService::Qos2 => V3QoS::ExactlyOnce, - } - } -} - -impl From for V5QoS { - fn from(value: QualityOfService) -> Self { - match value { - QualityOfService::Qos0 => V5QoS::AtMostOnce, - QualityOfService::Qos1 => V5QoS::AtLeastOnce, - QualityOfService::Qos2 => V5QoS::ExactlyOnce, - } - } -} - -#[derive(Debug, Deserialize, Serialize, Type)] -#[sqlx(type_name = "MQTT_CLIENT_VERSION")] -#[sqlx(rename_all = "lowercase")] -#[serde(rename_all = "lowercase")] -pub enum MqttClientVersion { - V3, - V5, -} - -#[derive(Debug, Deserialize)] -pub struct Tls { - enabled: bool, - ca_certificate: String, - //encoded in base64 - pkcs12_client_certificate: Option, - pkcs12_certificate_password: Option, -} - -#[derive(Debug, Deserialize)] -pub struct Credentials { - username: Option, - password: Option, -} - -#[derive(Debug, Deserialize)] -pub struct MqttResource { - broker: String, - port: u16, - credentials: Option, - tls: Option, -} -#[derive(Clone, Debug, FromRow, Serialize, Deserialize)] -pub struct SubscribeTopic { - qos: QualityOfService, - topic: String, -} - -struct MqttClientBuilder<'client> { - mqtt_resource: MqttResource, - client_id: &'client str, - subscribe_topics: Vec, - v3_config: Option<&'client MqttV3Config>, - v5_config: Option<&'client MqttV5Config>, - mqtt_client_version: Option<&'client MqttClientVersion>, -} - -impl<'client> MqttClientBuilder<'client> { - fn new( - mqtt_resource: MqttResource, - client_id: Option<&'client str>, - subscribe_topics: Vec, - v3_config: Option<&'client MqttV3Config>, - v5_config: Option<&'client MqttV5Config>, - mqtt_client_version: Option<&'client MqttClientVersion>, - ) -> Self { - Self { - mqtt_resource, - client_id: client_id.unwrap_or(""), - subscribe_topics, - v3_config, - v5_config, - mqtt_client_version, - } - } - - async fn build_client(&self) -> Result { - match self.mqtt_client_version { - Some(MqttClientVersion::V5) | None => self.build_v5_client().await, - Some(MqttClientVersion::V3) => self.build_v3_client().await, - } - } - - fn get_tls_configuration(&self) -> Result> { - let transport = match self.mqtt_resource.tls { - Some(ref tls) if tls.enabled => { - let transport = match tls.ca_certificate.trim().is_empty() { - true => rumqttc::Transport::Tls(TlsConfiguration::Native), - false => rumqttc::Transport::Tls(TlsConfiguration::SimpleNative { - ca: tls.ca_certificate.as_bytes().to_vec(), - client_auth: { - match tls.pkcs12_client_certificate.as_ref() { - Some(client_certificate) - if !client_certificate.trim().is_empty() => - { - let client_certificate = BASE64_STANDARD - .decode(client_certificate) - .map_err(to_anyhow)?; - let password = tls - .pkcs12_certificate_password - .clone() - .unwrap_or("".to_string()); - Some((client_certificate, password)) - } - _ => None, - } - }, - }), - }; - - Some(transport) - } - _ => None, - }; - - Ok(transport) - } - - async fn build_v5_client(&self) -> Result { - let mut mqtt_options = V5MqttOptions::new( - self.client_id, - &self.mqtt_resource.broker, - self.mqtt_resource.port, - ); - - if let Some(credentials) = &self.mqtt_resource.credentials { - let username = credentials.username.as_deref().unwrap_or(""); - let password = credentials.password.as_deref().unwrap_or(""); - mqtt_options.set_credentials(username, password); - } - - if let Some(transport) = self.get_tls_configuration()? { - mqtt_options.set_transport(transport); - } - - mqtt_options.set_connection_timeout(CLIENT_CONNECTION_TIMEOUT); - - mqtt_options.set_keep_alive(Duration::from_secs(KEEP_ALIVE)); - - if let Some(v5_config) = self.v5_config { - mqtt_options.set_clean_start(v5_config.clean_start.unwrap_or(true)); - mqtt_options.set_connect_properties(ConnectProperties { - session_expiry_interval: v5_config.session_expiry_interval, - receive_maximum: None, - max_packet_size: None, - topic_alias_max: v5_config.topic_alias_maximum.or(Some(TOPIC_ALIAS_MAXIMUM)), - request_response_info: None, - request_problem_info: None, - user_properties: vec![], - authentication_method: None, - authentication_data: None, - }); - } - - let (async_client, mut event_loop) = - V5AsyncClient::new(mqtt_options, self.subscribe_topics.len()); - event_loop.verify_connection().await?; - - if !self.subscribe_topics.is_empty() { - let subscribe_filters = self - .subscribe_topics - .iter() - .map(|topic| Filter::new(topic.topic.clone(), topic.qos.clone().into())) - .collect_vec(); - - async_client - .subscribe_many(subscribe_filters) - .await - .map_err(to_anyhow)?; - } - Ok(MqttClientResult::V5((V5MqttHandler, event_loop))) - } - - async fn build_v3_client(&self) -> Result { - let mut mqtt_options = V3MqttOptions::new( - self.client_id, - &self.mqtt_resource.broker, - self.mqtt_resource.port, - ); - - if let Some(credentials) = &self.mqtt_resource.credentials { - let username = credentials.username.as_deref().unwrap_or(""); - let password = credentials.password.as_deref().unwrap_or(""); - mqtt_options.set_credentials(username, password); - } - - if let Some(transport) = self.get_tls_configuration()? { - mqtt_options.set_transport(transport); - } - mqtt_options.set_keep_alive(Duration::from_secs(KEEP_ALIVE)); - if let Some(v3_config) = self.v3_config { - mqtt_options.set_clean_session(v3_config.clean_session.unwrap_or(true)); - } - - let (async_client, mut event_loop) = - V3AsyncClient::new(mqtt_options, self.subscribe_topics.len()); - event_loop.verify_connection().await?; - - if !self.subscribe_topics.is_empty() { - let subscribe_filters = self - .subscribe_topics - .iter() - .map(|topic| SubscribeFilter::new(topic.topic.clone(), topic.qos.clone().into())) - .collect_vec(); - - async_client - .subscribe_many(subscribe_filters) - .await - .map_err(to_anyhow)?; - } - Ok(MqttClientResult::V3((V3MqttHandler, event_loop))) - } -} - -#[derive(Debug, Serialize, Deserialize, FromRow)] -pub struct MqttTrigger { - pub mqtt_resource_path: String, - pub subscribe_topics: Vec>, - pub v3_config: Option>, - pub v5_config: Option>, - pub client_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub client_version: Option, - pub path: String, - pub script_path: String, - pub is_flow: bool, - pub workspace_id: String, - pub edited_by: String, - pub email: String, - pub edited_at: chrono::DateTime, - pub extra_perms: Option, - pub error: Option, - pub server_id: Option, - pub last_server_ping: Option>, - pub enabled: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_args: Option>>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub retry: Option>, -} - -const KEEP_ALIVE: u64 = 60; -const CLIENT_CONNECTION_TIMEOUT: u64 = 60; -const TOPIC_ALIAS_MAXIMUM: u16 = 65535; - -fn convert_disconnect_packet_into_err(disconnect: rumqttc::v5::mqttbytes::v5::Disconnect) -> Error { - let err_message = disconnect - .properties - .map(|properties| properties.reason_string) - .flatten(); - let reason_code = disconnect.reason_code as u8; - anyhow::anyhow!( - "Disconnected by the broker, reason code: {}, {}", - reason_code, - err_message - .map(|err| format!("message: {}", err)) - .unwrap_or("".to_string()) - ) - .into() -} - -async fn loop_ping(db: &DB, mqtt: &MqttConfig, error: Option<&str>) { - loop { - if mqtt.update_ping(db, error).await.is_none() { - return; - } - - tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; - } -} - -enum MqttClientResult { - V3((V3MqttHandler, V3EventLoop)), - V5((V5MqttHandler, V5EventLoop)), -} - -trait MqttEvent { - type IncomingPacket; - type PublishPacket; - type Event; - - fn handle_publish_packet(publish_packet: Self::PublishPacket) -> PublishData; - fn handle_event(&self, event: Self::Event) -> Result>; -} - -struct V5MqttHandler; - -impl MqttEvent for V5MqttHandler { - type IncomingPacket = V5Incoming; - type PublishPacket = rumqttc::v5::mqttbytes::v5::Publish; - type Event = V5Event; - - fn handle_publish_packet(publish_packet: Self::PublishPacket) -> PublishData { - PublishData::new( - String::from_utf8(publish_packet.topic.as_ref().to_vec()).unwrap_or("".to_string()), - publish_packet.retain, - publish_packet.pkid, - publish_packet.properties, - publish_packet.qos as u8, - ) - } - - fn handle_event(&self, event: Self::Event) -> Result> { - tracing::debug!("Inside V5 event"); - match event { - Self::Event::Incoming(packet) => match packet { - Self::IncomingPacket::Publish(publish_packet) => { - return Ok(Some(( - publish_packet.payload.clone(), - Self::handle_publish_packet(publish_packet), - ))) - } - Self::IncomingPacket::Disconnect(disconnect) => { - return Err(convert_disconnect_packet_into_err(disconnect)); - } - packet => { - tracing::debug!("Received = {:#?}", packet); - } - }, - Self::Event::Outgoing(packet) => { - tracing::debug!("Outgoing Received = {:#?}", packet); - } - } - - Ok(None) - } -} - -struct V3MqttHandler; - -impl MqttEvent for V3MqttHandler { - type IncomingPacket = V3Incoming; - type PublishPacket = rumqttc::mqttbytes::v4::Publish; - type Event = V3Event; - - fn handle_publish_packet(publish_packet: Self::PublishPacket) -> PublishData { - PublishData::new( - publish_packet.topic, - publish_packet.retain, - publish_packet.pkid, - None, - publish_packet.qos as u8, - ) - } - - fn handle_event(&self, event: Self::Event) -> Result> { - tracing::debug!("Inside V3 event"); - match event { - Self::Event::Incoming(packet) => match packet { - Self::IncomingPacket::Publish(publish_packet) => { - return Ok(Some(( - publish_packet.payload.clone(), - Self::handle_publish_packet(publish_packet), - ))) - } - packet => { - tracing::debug!("Received = {:?}", packet); - } - }, - Self::Event::Outgoing(packet) => { - tracing::debug!("Outgoing Received = {:?}", packet); - } - } - - Ok(None) - } -} - -const TIMEOUT_DURATION: u64 = 10; -const CONNECTION_TIMEOUT: Duration = Duration::from_secs(TIMEOUT_DURATION); - -#[async_trait] -pub trait EventLoop { - type Event; - type Error; - - async fn poll(&mut self) -> Result; - async fn verify_connection(&mut self) -> Result<()>; -} - -#[async_trait] -impl EventLoop for V5EventLoop { - type Event = V5Event; - type Error = rumqttc::v5::ConnectionError; - async fn poll(&mut self) -> Result { - self.poll().await.map_err(|err| to_anyhow(err).into()) - } - - async fn verify_connection(&mut self) -> Result<()> { - let start = std::time::Instant::now(); - - while start.elapsed() < CONNECTION_TIMEOUT { - match self.poll().await.map_err(to_anyhow)? { - Self::Event::Incoming(V5Incoming::ConnAck(_)) => return Ok(()), - Self::Event::Incoming(V5Incoming::Disconnect(disconnect)) => { - return Err(convert_disconnect_packet_into_err(disconnect)); - } - _ => continue, - } - } - - Err(Error::BadConfig(format!( - "Timeout occurred while trying to connect to mqtt broker after {} seconds", - TIMEOUT_DURATION - ))) - } -} - -#[async_trait] -impl EventLoop for V3EventLoop { - type Event = V3Event; - type Error = Error; - - async fn poll(&mut self) -> Result { - self.poll().await.map_err(|err| to_anyhow(err).into()) - } - - async fn verify_connection(&mut self) -> Result<()> { - let start = std::time::Instant::now(); - - while start.elapsed() < CONNECTION_TIMEOUT { - match self.poll().await.map_err(to_anyhow)? { - Self::Event::Incoming(rumqttc::Packet::ConnAck(_)) => return Ok(()), - _ => continue, - } - } - - Err(Error::BadConfig(format!( - "Timeout occurred while trying to connect to mqtt broker after {} seconds", - TIMEOUT_DURATION - ))) - } -} - -async fn handle_publish_packet(db: &DB, mqtt: &MqttConfig, payload: Bytes, publish: PublishData) { - let trigger_info = HashMap::from([ - ("topic".to_string(), to_raw_value(&publish.topic)), - ("retain".to_string(), to_raw_value(&publish.retain)), - ("pkid".to_string(), to_raw_value(&publish.pkid)), - ("qos".to_string(), to_raw_value(&publish.qos)), - ( - "v5".to_string(), - to_raw_value(&publish.v5.map(|properties| { - serde_json::json!({ - "payload_format_indicator": properties.payload_format_indicator, - "topic_alias": properties.topic_alias, - "response_topic": properties.response_topic, - "correlation_data": properties.correlation_data.as_deref(), - "user_properties": properties.user_properties, - "subscription_identifiers": properties.subscription_identifiers, - "content_type": properties.content_type, - }) - })), - ), - ]); - mqtt.handle(&db, payload.as_ref(), trigger_info).await; -} - -async fn handle_event(db: &DB, mqtt: &MqttConfig, handler: H, mut event_loop: E) -> () -where - H: MqttEvent, - E: EventLoop, - E::Error: ToString, -{ - loop { - let event = event_loop.poll().await; - - match event { - Ok(event) => { - let publish_data = handler.handle_event(event); - if let Ok(Some((payload, publish_data))) = publish_data { - handle_publish_packet(db, mqtt, payload, publish_data).await; - } - } - Err(err) => { - let err = err.to_string(); - tracing::debug!("Error: {}", &err); - mqtt.disable_with_error(&db, err).await; - return; - } - } - } -} - -#[derive(Debug)] -enum MqttConfig { - Trigger(MqttTrigger), - Capture(CaptureConfigForMqttTrigger), -} - -impl MqttConfig { - async fn update_ping(&self, db: &DB, error: Option<&str>) -> Option<()> { - match self { - MqttConfig::Trigger(trigger) => trigger.update_ping(db, error).await, - MqttConfig::Capture(capture) => capture.update_ping(db, error).await, - } - } - - async fn disable_with_error(&self, db: &DB, error: String) -> () { - match self { - MqttConfig::Trigger(trigger) => trigger.disable_with_error(&db, error).await, - MqttConfig::Capture(capture) => capture.disable_with_error(db, error).await, - } - } - - async fn start_consuming_messages( - &self, - db: &DB, - ) -> std::result::Result { - let mqtt_resource_path; - let subscribe_topics; - let workspace_id; - let authed; - let client_version; - let client_id; - let v3_config; - let v5_config; - match self { - MqttConfig::Capture(capture) => { - mqtt_resource_path = &capture.trigger_config.0.mqtt_resource_path; - subscribe_topics = capture.trigger_config.0.subscribe_topics.clone(); - workspace_id = &capture.workspace_id; - authed = capture.fetch_authed(&db).await?; - client_version = capture.trigger_config.0.client_version.as_ref(); - client_id = capture.trigger_config.0.client_id.as_deref(); - v3_config = capture.trigger_config.0.v3_config.as_ref(); - v5_config = capture.trigger_config.0.v5_config.as_ref(); - } - MqttConfig::Trigger(trigger) => { - mqtt_resource_path = &trigger.mqtt_resource_path; - subscribe_topics = trigger - .subscribe_topics - .iter() - .map(|topic| topic.0.clone()) - .collect_vec(); - workspace_id = &trigger.workspace_id; - client_version = trigger.client_version.as_ref(); - authed = trigger.fetch_authed(&db).await?; - client_id = trigger.client_id.as_deref(); - v3_config = trigger.v3_config.as_ref().map(|v3_config| &v3_config.0); - v5_config = trigger.v5_config.as_ref().map(|v5_config| &v5_config.0); - } - } - let mqtt_resource = try_get_resource_from_db_as::( - &authed, - Some(UserDB::new(db.clone())), - db, - mqtt_resource_path, - workspace_id, - ) - .await?; - let client_builder = MqttClientBuilder::new( - mqtt_resource, - client_id, - subscribe_topics, - v3_config, - v5_config, - client_version, - ); - - client_builder.build_client().await - } - - async fn handle( - &self, - db: &DB, - payload: &[u8], - trigger_info: HashMap>, - ) -> () { - match self { - MqttConfig::Trigger(trigger) => trigger.handle(&db, payload, trigger_info).await, - MqttConfig::Capture(capture) => capture.handle(&db, payload, trigger_info).await, - } - } -} - -impl MqttTrigger { - async fn try_to_listen_to_mqtt_messages( - self, - db: DB, - killpill_rx: tokio::sync::broadcast::Receiver<()>, - ) -> () { - let mqtt_trigger = sqlx::query_scalar!( - r#" - UPDATE - mqtt_trigger - SET - server_id = $1, - last_server_ping = now(), - error = 'Connecting...' - WHERE - enabled IS TRUE - AND workspace_id = $2 - AND path = $3 - AND (last_server_ping IS NULL - OR last_server_ping < now() - INTERVAL '15 seconds' - ) - RETURNING true - "#, - *INSTANCE_NAME, - self.workspace_id, - self.path, - ) - .fetch_optional(&db) - .await; - match mqtt_trigger { - Ok(has_lock) => { - if has_lock.flatten().unwrap_or(false) { - tracing::info!("Spawning new task to listen to mqtt notifications"); - tokio::spawn(async move { - listen_to_messages(MqttConfig::Trigger(self), db.clone(), killpill_rx) - .await; - }); - } else { - tracing::info!("Mqtt trigger {} already being listened to", self.path); - } - } - Err(err) => { - tracing::error!( - "Error acquiring lock for mqtt trigger {}: {:?}", - self.path, - err - ); - } - }; - } - - async fn update_ping(&self, db: &DB, error: Option<&str>) -> Option<()> { - let updated = sqlx::query_scalar!( - r#" - UPDATE - mqtt_trigger - SET - last_server_ping = now(), - error = $1 - WHERE - workspace_id = $2 - AND path = $3 - AND server_id = $4 - AND enabled IS TRUE - RETURNING 1 - "#, - error, - &self.workspace_id, - &self.path, - *INSTANCE_NAME - ) - .fetch_optional(db) - .await; - - match updated { - Ok(updated) => { - if updated.flatten().is_none() { - // allow faster restart of mqtt trigger - sqlx::query!( - r#" - UPDATE - mqtt_trigger - SET - last_server_ping = NULL - WHERE - workspace_id = $1 - AND path = $2 - AND server_id IS NULL"#, - &self.workspace_id, - &self.path, - ) - .execute(db) - .await - .ok(); - tracing::info!( - "Mqtt trigger {} changed, disabled, or deleted, stopping...", - self.path - ); - return None; - } - } - Err(err) => { - tracing::warn!( - "Error updating ping of mqtt trigger {}: {:?}", - self.path, - err - ); - } - }; - - Some(()) - } - - async fn disable_with_error(&self, db: &DB, error: String) -> () { - match sqlx::query!( - r#" - UPDATE - mqtt_trigger - SET - enabled = FALSE, - error = $1, - server_id = NULL, - last_server_ping = NULL - WHERE - workspace_id = $2 AND - path = $3 - "#, - error, - self.workspace_id, - self.path, - ) - .execute(db) - .await - { - Ok(_) => { - report_critical_error( - format!( - "Disabling mqtt trigger {} because of error: {}", - self.path, error - ), - db.clone(), - Some(&self.workspace_id), - None, - ) - .await; - } - Err(disable_err) => { - report_critical_error( - format!("Could not disable mqtt trigger {} with err {}, disabling because of error {}", self.path, disable_err, error), - db.clone(), - Some(&self.workspace_id), - None, - ).await; - } - } - } - - async fn fetch_authed(&self, db: &DB) -> Result { - fetch_api_authed( - self.edited_by.clone(), - self.email.clone(), - &self.workspace_id, - db, - Some(format!("mqtt-{}", self.path)), - ) - .await - } - - async fn handle( - &self, - db: &DB, - payload: &[u8], - trigger_info: HashMap>, - ) -> () { - if let Err(err) = run_job(payload, trigger_info, db, self).await { - report_critical_error( - format!("Failed to trigger job from mqtt {}: {:?}", self.path, err), - db.clone(), - Some(&self.workspace_id), - None, - ) - .await; - }; - } -} - -impl TriggerJobArgs<&[u8]> for MqttTrigger { - fn v1_payload_fn(payload: &[u8]) -> HashMap> { - HashMap::from([("payload".to_string(), to_raw_value(&payload))]) - } - - fn v2_payload_fn(payload: &[u8]) -> HashMap> { - let base64_payload = engine::general_purpose::STANDARD.encode(payload); - HashMap::from([("payload".to_string(), to_raw_value(&base64_payload))]) - } - - fn trigger_kind() -> TriggerKind { - TriggerKind::Mqtt - } -} - -struct PublishData { - topic: String, - retain: bool, - pkid: u16, - v5: Option, - qos: u8, -} - -impl PublishData { - fn new( - topic: String, - retain: bool, - pkid: u16, - v5: Option, - qos: u8, - ) -> PublishData { - PublishData { topic, retain, pkid, v5, qos } - } -} - -async fn listen_to_messages( - mqtt: MqttConfig, - db: DB, - mut killpill_rx: tokio::sync::broadcast::Receiver<()>, -) { - tokio::select! { - biased; - - _ = killpill_rx.recv() => { - return; - } - - _ = loop_ping(&db, &mqtt, Some("Connecting...")) => { - return; - } - - result = mqtt.start_consuming_messages(&db) => { - tokio::select! { - biased; - - _ = killpill_rx.recv() => { - return; - } - - _ = loop_ping(&db, &mqtt, None) => { - return; - } - - _ = async { - match result { - Ok(connection) => { - match connection { - MqttClientResult::V3((v3_handler, event_loop)) => handle_event(&db, &mqtt, v3_handler, event_loop).await, - MqttClientResult::V5((v5_handler, event_loop)) => handle_event(&db, &mqtt, v5_handler, event_loop).await, - } - } - Err(err) => { - tracing::error!( - "Mqtt trigger error while trying to start listening to notifications: {}", - &err - ); - mqtt.disable_with_error(&db, err.to_string()).await - } - } - } => {} - } - } - } -} - -#[derive(Debug, Deserialize)] -struct CaptureConfigForMqttTrigger { - trigger_config: SqlxJson, - path: String, - is_flow: bool, - workspace_id: String, - owner: String, - email: String, -} - -impl CaptureConfigForMqttTrigger { - async fn try_to_listen_to_mqtt_messages( - self, - db: DB, - killpill_rx: tokio::sync::broadcast::Receiver<()>, - ) -> () { - match sqlx::query_scalar!( - r#" - UPDATE - capture_config - SET - server_id = $1, - last_server_ping = now(), - error = 'Connecting...' - WHERE - last_client_ping > NOW() - INTERVAL '10 seconds' AND - workspace_id = $2 AND - path = $3 AND - is_flow = $4 AND - trigger_kind = 'mqtt' AND - (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') - RETURNING true - "#, - *INSTANCE_NAME, - self.workspace_id, - self.path, - self.is_flow, - ) - .fetch_optional(&db) - .await - { - Ok(has_lock) => { - if has_lock.flatten().unwrap_or(false) { - tokio::spawn(listen_to_messages( - MqttConfig::Capture(self), - db, - killpill_rx, - )); - } else { - tracing::info!("Mqtt {} already being listened to", self.path); - } - } - Err(err) => { - tracing::error!( - "Error acquiring lock for capture mqtt {}: {:?}", - self.path, - err - ); - } - }; - } - - async fn update_ping(&self, db: &DB, error: Option<&str>) -> Option<()> { - match sqlx::query_scalar!( - r#" - UPDATE - capture_config - SET - last_server_ping = now(), - error = $1 - WHERE - workspace_id = $2 AND - path = $3 AND - is_flow = $4 AND - trigger_kind = 'mqtt' AND - server_id = $5 AND - last_client_ping > NOW() - INTERVAL '10 seconds' - RETURNING 1 - "#, - error, - self.workspace_id, - self.path, - self.is_flow, - *INSTANCE_NAME - ) - .fetch_optional(db) - .await - { - Ok(updated) => { - if updated.flatten().is_none() { - // allow faster restart of mqtt capture - sqlx::query!( - r#"UPDATE - capture_config - SET - last_server_ping = NULL - WHERE - workspace_id = $1 AND - path = $2 AND - is_flow = $3 AND - trigger_kind = 'mqtt' AND - server_id IS NULL - "#, - self.workspace_id, - self.path, - self.is_flow, - ) - .execute(db) - .await - .ok(); - tracing::info!( - "Mqtt capture {} changed, disabled, or deleted, stopping...", - self.path - ); - return None; - } - } - Err(err) => { - tracing::warn!( - "Error updating ping of capture mqtt {}: {:?}", - self.path, - err - ); - } - }; - - Some(()) - } - - async fn fetch_authed(&self, db: &DB) -> Result { - fetch_api_authed( - self.owner.clone(), - self.email.clone(), - &self.workspace_id, - db, - Some(format!("mqtt-{}", self.get_trigger_path())), - ) - .await - } - - fn get_trigger_path(&self) -> String { - format!( - "{}-{}", - if self.is_flow { "flow" } else { "script" }, - self.path - ) - } - - async fn disable_with_error(&self, db: &DB, error: String) -> () { - if let Err(err) = sqlx::query!( - r#" - UPDATE - capture_config - SET - error = $1, - server_id = NULL, - last_server_ping = NULL - WHERE - workspace_id = $2 AND - path = $3 AND - is_flow = $4 AND - trigger_kind = 'mqtt' - "#, - error, - self.workspace_id, - self.path, - self.is_flow, - ) - .execute(db) - .await - { - tracing::error!( - "Could not disable mqtt capture {} ({}) with err {}, disabling because of error {}", - self.path, - self.workspace_id, - err, - error - ); - } - } - - async fn handle( - &self, - db: &DB, - payload: &[u8], - trigger_info: HashMap>, - ) -> () { - let (main_args, preprocessor_args) = - MqttTrigger::build_capture_payloads(payload, trigger_info); - if let Err(err) = insert_capture_payload( - db, - &self.workspace_id, - &self.path, - self.is_flow, - &TriggerKind::Mqtt, - main_args, - preprocessor_args, - &self.owner, - ) - .await - { - tracing::error!("Error inserting capture payload: {:?}", err); - } - } -} - -async fn listen_to_unlistened_mqtt_events( - db: &DB, - killpill_rx: &tokio::sync::broadcast::Receiver<()>, -) { - let mqtt_triggers = sqlx::query_as!( - MqttTrigger, - r#" - SELECT - mqtt_resource_path, - subscribe_topics as "subscribe_topics!: Vec>", - v3_config as "v3_config!: Option>", - v5_config as "v5_config!: Option>", - client_version as "client_version: _", - client_id, - workspace_id, - path, - script_path, - is_flow, - edited_by, - email, - edited_at, - server_id, - last_server_ping, - extra_perms, - error, - enabled, - error_handler_path, - error_handler_args as "error_handler_args: _", - retry as "retry: _" - FROM - mqtt_trigger - WHERE - enabled IS TRUE - AND (last_server_ping IS NULL OR - last_server_ping < now() - interval '15 seconds' - ) - "# - ) - .fetch_all(db) - .await; - - match mqtt_triggers { - Ok(mut triggers) => { - triggers.shuffle(&mut rand::rng()); - for trigger in triggers { - trigger - .try_to_listen_to_mqtt_messages(db.clone(), killpill_rx.resubscribe()) - .await; - } - } - Err(err) => { - tracing::error!("Error fetching mqtt triggers: {:?}", err); - } - }; - - let mqtt_triggers_capture = sqlx::query_as!( - CaptureConfigForMqttTrigger, - r#" - SELECT - path, - is_flow, - workspace_id, - owner, - email, - trigger_config as "trigger_config!: _" - FROM - capture_config - WHERE - trigger_kind = 'mqtt' AND - last_client_ping > NOW() - INTERVAL '10 seconds' AND - trigger_config IS NOT NULL AND - (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') - "# - ) - .fetch_all(db) - .await; - - match mqtt_triggers_capture { - Ok(mut captures) => { - captures.shuffle(&mut rand::rng()); - for capture in captures { - capture - .try_to_listen_to_mqtt_messages(db.clone(), killpill_rx.resubscribe()) - .await; - } - } - Err(err) => { - tracing::error!("Error fetching captures mqtt triggers: {:?}", err); - } - }; -} - -pub fn start_mqtt_consumer(db: DB, mut killpill_rx: tokio::sync::broadcast::Receiver<()>) { - tokio::spawn(async move { - listen_to_unlistened_mqtt_events(&db, &killpill_rx).await; - loop { - tokio::select! { - biased; - _ = killpill_rx.recv() => { - return; - } - _ = tokio::time::sleep(tokio::time::Duration::from_secs(15)) => { - listen_to_unlistened_mqtt_events(&db, &killpill_rx).await - } - } - } - }); -} diff --git a/backend/windmill-api/src/nats_triggers_oss.rs b/backend/windmill-api/src/nats_triggers_oss.rs deleted file mode 100644 index 53e8fd88ec..0000000000 --- a/backend/windmill-api/src/nats_triggers_oss.rs +++ /dev/null @@ -1,54 +0,0 @@ -#[cfg(feature = "private")] -#[allow(unused)] -pub use crate::nats_triggers_ee::*; - -#[cfg(not(feature = "private"))] -use crate::db::DB; -#[cfg(not(feature = "private"))] -use serde::{Deserialize, Serialize}; - -#[cfg(not(feature = "private"))] -#[derive(Serialize, Deserialize)] -pub struct NatsResourceAuth {} - -#[cfg(not(feature = "private"))] -pub fn start_nats_consumers(_db: DB, mut _killpill_rx: tokio::sync::broadcast::Receiver<()>) -> () { - // implementation is not open source -} - -#[derive(Serialize, Deserialize)] -#[cfg(not(feature = "private"))] -pub enum NatsTriggerConfigConnection {} - -#[derive(Serialize, Clone)] -#[cfg(not(feature = "private"))] -pub struct NatsTrigger { - pub workspace_id: String, - pub path: String, - pub nats_resource_path: String, - pub subjects: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - pub stream_name: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub consumer_name: Option, - pub use_jetstream: bool, - pub script_path: String, - pub is_flow: bool, - pub edited_by: String, - pub email: String, - pub edited_at: chrono::DateTime, - #[serde(skip_serializing_if = "Option::is_none")] - pub server_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub last_server_ping: Option>, - pub extra_perms: serde_json::Value, - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, - pub enabled: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_args: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub retry: Option, -} diff --git a/backend/windmill-api/src/postgres_triggers/mod.rs b/backend/windmill-api/src/postgres_triggers/mod.rs deleted file mode 100644 index 17d6d44678..0000000000 --- a/backend/windmill-api/src/postgres_triggers/mod.rs +++ /dev/null @@ -1,244 +0,0 @@ -use crate::{ - db::{ApiAuthed, DB}, - resources::try_get_resource_from_db_as, -}; -use native_tls::{Certificate, TlsConnector}; -use pg_escape::quote_identifier; -use rust_postgres::{config::SslMode, Client, Config, NoTls}; -use rust_postgres_native_tls::MakeTlsConnector; -use serde::{Deserialize, Serialize}; -use serde_json::value::RawValue; -use sqlx::FromRow; -use std::collections::HashMap; - -use windmill_common::{ - db::UserDB, - error::{to_anyhow, Error, Result}, - utils::empty_as_none, -}; -mod bool; -mod converter; -mod hex; -mod relation; -mod replication_message; -mod trigger; - -pub use trigger::start_database; - -const ERROR_REPLICATION_SLOT_NOT_EXISTS: &str = r#"The replication slot associated with this trigger no longer exists. Recreate a new replication slot or select an existing one in the advanced tab, or delete and recreate a new trigger"#; - -const ERROR_PUBLICATION_NAME_NOT_EXISTS: &str = r#"The publication associated with this trigger no longer exists. Recreate a new publication or select an existing one in the advanced tab, or delete and recreate a new trigger"#; - -fn build_tls_connector( - ssl_mode: SslMode, - root_certificate_pem: Option<&String>, -) -> Result> { - let get_tls_builder_for_verify = |root_certificate: Option<&String>| { - let mut builder = TlsConnector::builder(); - if let Some(root_certificate) = root_certificate { - let root_certificate_pem = - Certificate::from_pem(root_certificate.as_bytes()).map_err(to_anyhow)?; - builder.add_root_certificate(root_certificate_pem); - } - Ok::<_, Error>(builder) - }; - let connector = match ssl_mode { - SslMode::Disable => return Ok(None), - SslMode::Require | SslMode::Prefer => { - let mut builder = TlsConnector::builder(); - builder.danger_accept_invalid_certs(true); - builder.danger_accept_invalid_hostnames(true); - builder - } - - SslMode::VerifyCa => { - let mut builder = get_tls_builder_for_verify(root_certificate_pem)?; - builder.danger_accept_invalid_hostnames(true); - builder - } - - SslMode::VerifyFull => { - let builder = get_tls_builder_for_verify(root_certificate_pem)?; - builder - } - _ => unreachable!(), - }; - - Ok(Some(MakeTlsConnector::new( - connector.build().map_err(to_anyhow)?, - ))) -} - -#[derive(FromRow, Serialize, Deserialize, Debug)] -pub struct Postgres { - pub user: String, - pub password: String, - pub host: String, - pub port: Option, - pub dbname: String, - #[serde(default)] - pub sslmode: String, - #[serde(default, deserialize_with = "empty_as_none")] - pub root_certificate_pem: Option, -} - -pub async fn get_raw_postgres_connection( - database: &Postgres, - logical_mode: bool, -) -> Result { - let ssl_mode = match database.sslmode.as_ref() { - "disable" => SslMode::Disable, - "" | "prefer" | "allow" => SslMode::Prefer, - "require" => SslMode::Require, - "verify-ca" => SslMode::VerifyCa, - "verify-full" => SslMode::VerifyFull, - ssl_mode => { - return Err(Error::BadRequest( - format!("Invalid ssl mode for postgres: {}, please put a valid ssl_mode among the following available ssl mode: ['disable', 'allow', 'prefer', 'verify-ca', 'verify-full']", ssl_mode), - )) - } - }; - - let mut config = Config::new(); - config - .dbname(&database.dbname) - .host(&database.host) - .user(&database.user) - .ssl_mode(ssl_mode); - - if logical_mode { - config.replication_mode(rust_postgres::config::ReplicationMode::Logical); - } - - if let Some(port) = database.port { - config.port(port); - }; - - if !database.password.is_empty() { - config.password(&database.password); - } - - let connector = build_tls_connector(ssl_mode, database.root_certificate_pem.as_ref())?; - - let client = if let Some(connector) = connector { - let (client, connection) = config.connect(connector).await.map_err(to_anyhow)?; - tokio::spawn(async move { - if let Err(e) = connection.await { - tracing::debug!("{:#?}", e); - }; - tracing::info!("Successfully Connected into database"); - }); - client - } else { - let (client, connection) = config.connect(NoTls).await.map_err(to_anyhow)?; - tokio::spawn(async move { - if let Err(e) = connection.await { - tracing::debug!("{:#?}", e); - }; - tracing::info!("Successfully Connected into database"); - }); - client - }; - - Ok(client) -} - -pub async fn get_pg_connection( - authed: ApiAuthed, - user_db: Option, - db: &DB, - postgres_resource_path: &str, - w_id: &str, - logical_mode: bool, -) -> Result { - let database = - try_get_resource_from_db_as::(&authed, user_db, db, postgres_resource_path, w_id) - .await?; - - Ok(get_raw_postgres_connection(&database, logical_mode).await?) -} - -pub async fn get_default_pg_connection( - authed: ApiAuthed, - user_db: Option, - db: &DB, - postgres_resource_path: &str, - w_id: &str, -) -> Result { - get_pg_connection(authed, user_db, db, postgres_resource_path, w_id, false).await -} - -#[derive(FromRow, Deserialize, Serialize, Debug)] -pub struct PostgresTrigger { - pub path: String, - pub script_path: String, - pub is_flow: bool, - pub workspace_id: String, - pub edited_by: String, - pub email: String, - pub edited_at: chrono::DateTime, - #[serde(skip_serializing_if = "Option::is_none")] - pub extra_perms: Option, - pub postgres_resource_path: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub server_id: Option, - pub replication_slot_name: String, - pub publication_name: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub last_server_ping: Option>, - pub enabled: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_args: Option>>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub retry: Option>, -} - -pub async fn drop_logical_replication_slot(pg_connection: &Client, slot_name: &str) -> Result<()> { - let row = pg_connection - .query_opt( - r#" - SELECT - active_pid - FROM - pg_replication_slots - WHERE - slot_name = $1 - "#, - &[&slot_name], - ) - .await - .map_err(to_anyhow)?; - - let active_pid = row.map(|r| r.get::<_, Option>(0)).flatten(); - - if let Some(pid) = active_pid { - pg_connection - .execute("SELECT pg_terminate_backend($1)", &[&pid]) - .await - .map_err(to_anyhow)?; - } - - pg_connection - .execute("SELECT pg_drop_replication_slot($1)", &[&slot_name]) - .await - .map_err(to_anyhow)?; - - Ok(()) -} - -pub async fn drop_publication(pg_connection: &Client, publication_name: &str) -> Result<()> { - let mut query = String::from("DROP PUBLICATION IF EXISTS "); - let quoted_publication_name = quote_identifier(publication_name); - query.push_str("ed_publication_name); - - pg_connection - .execute(&query, &[]) - .await - .map_err(to_anyhow)?; - - Ok(()) -} diff --git a/backend/windmill-api/src/postgres_triggers/trigger.rs b/backend/windmill-api/src/postgres_triggers/trigger.rs deleted file mode 100644 index 6bd8877fec..0000000000 --- a/backend/windmill-api/src/postgres_triggers/trigger.rs +++ /dev/null @@ -1,1038 +0,0 @@ -use std::{collections::HashMap, pin::Pin}; - -use crate::{ - capture::{insert_capture_payload, PostgresTriggerConfig}, - db::{ApiAuthed, DB}, - postgres_triggers::{ - relation::RelationConverter, - replication_message::{ - LogicalReplicationMessage::{Begin, Commit, Delete, Insert, Relation, Type, Update}, - ReplicationMessage, - }, - }, - resources::try_get_resource_from_db_as, - trigger_helpers::{trigger_runnable, TriggerJobArgs}, - users::fetch_api_authed, -}; - -use bytes::{BufMut, Bytes, BytesMut}; -use chrono::TimeZone; -use futures::{pin_mut, SinkExt, StreamExt}; -use pg_escape::{quote_identifier, quote_literal}; -use rand::seq::SliceRandom; -use rust_postgres::{Client, CopyBothDuplex, SimpleQueryMessage}; -use serde::Deserialize; -use serde_json::value::RawValue; -use sqlx::types::Json as SqlxJson; - -use windmill_common::{ - db::UserDB, - error::{self, to_anyhow}, - triggers::TriggerKind, - utils::report_critical_error, - worker::to_raw_value, - INSTANCE_NAME, -}; - -use super::{ - drop_logical_replication_slot, drop_publication, get_default_pg_connection, - get_raw_postgres_connection, replication_message::PrimaryKeepAliveBody, Error, Postgres, - PostgresTrigger, ERROR_PUBLICATION_NAME_NOT_EXISTS, ERROR_REPLICATION_SLOT_NOT_EXISTS, -}; - -pub struct LogicalReplicationSettings { - pub streaming: bool, -} - -impl LogicalReplicationSettings { - pub fn new(streaming: bool) -> Self { - Self { streaming } - } -} - -#[allow(unused)] -trait RowExist { - fn row_exist(&self) -> bool; -} - -impl RowExist for Vec { - fn row_exist(&self) -> bool { - self.iter() - .find_map(|element| { - if let SimpleQueryMessage::CommandComplete(value) = element { - Some(*value) - } else { - None - } - }) - .is_some_and(|value| value > 0) - } -} - -pub struct PostgresSimpleClient(Client); - -impl PostgresSimpleClient { - async fn new(database: &Postgres) -> Result { - let client = get_raw_postgres_connection(database, true).await?; - - Ok(PostgresSimpleClient(client)) - } - - async fn execute_query( - &self, - query: &str, - ) -> Result, rust_postgres::Error> { - self.0.simple_query(query).await - } - - async fn get_logical_replication_stream( - &self, - publication_name: &str, - logical_replication_slot_name: &str, - ) -> Result<(CopyBothDuplex, LogicalReplicationSettings), Error> { - let options = format!( - r#"("proto_version" '2', "publication_names" {})"#, - quote_literal(publication_name), - ); - - let query = format!( - r#"START_REPLICATION SLOT {} LOGICAL 0/0 {}"#, - quote_identifier(logical_replication_slot_name), - options - ); - - Ok(( - self.0 - .copy_both_simple::(query.as_str()) - .await - .map_err(to_anyhow)?, - LogicalReplicationSettings::new(false), - )) - } - - async fn send_status_update( - primary_keep_alive: PrimaryKeepAliveBody, - copy_both_stream: &mut Pin<&mut CopyBothDuplex>, - ) { - let mut buf = BytesMut::new(); - let ts = chrono::Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); - let ts = chrono::Utc::now() - .signed_duration_since(ts) - .num_microseconds() - .unwrap_or(0); - - buf.put_u8(b'r'); - buf.put_u64(primary_keep_alive.wal_end); - buf.put_u64(primary_keep_alive.wal_end); - buf.put_u64(primary_keep_alive.wal_end); - buf.put_i64(ts); - buf.put_u8(0); - copy_both_stream.send(buf.freeze()).await.unwrap(); - tracing::debug!("Send update status message"); - } -} - -async fn loop_ping(db: &DB, pg: &PostgresConfig, error: Option<&str>) { - loop { - if pg.update_ping(db, error).await.is_none() { - return; - } - - tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; - } -} - -enum PostgresConfig { - Trigger(PostgresTrigger), - Capture(CaptureConfigForPostgresTrigger), -} - -impl PostgresTrigger { - async fn try_to_listen_to_database_transactions( - self, - db: DB, - killpill_rx: tokio::sync::broadcast::Receiver<()>, - ) -> () { - let postgres_trigger = sqlx::query_scalar!( - r#" - UPDATE postgres_trigger - SET - server_id = $1, - last_server_ping = now(), - error = 'Connecting...' - WHERE - enabled IS TRUE - AND workspace_id = $2 - AND path = $3 - AND (last_server_ping IS NULL - OR last_server_ping < now() - INTERVAL '15 seconds' - ) - RETURNING true - "#, - *INSTANCE_NAME, - self.workspace_id, - self.path, - ) - .fetch_optional(&db) - .await; - match postgres_trigger { - Ok(has_lock) => { - if has_lock.flatten().unwrap_or(false) { - tracing::info!("Spawning new task to listen_to_database_transaction"); - tokio::spawn(async move { - listen_to_transactions( - PostgresConfig::Trigger(self), - db.clone(), - killpill_rx, - ) - .await; - }); - } else { - tracing::info!("Postgres trigger {} already being listened to", self.path); - } - } - Err(err) => { - tracing::error!( - "Error acquiring lock for postgres trigger {}: {:?}", - self.path, - err - ); - } - }; - } - - async fn update_ping(&self, db: &DB, error: Option<&str>) -> Option<()> { - let updated = sqlx::query_scalar!( - r#" - UPDATE - postgres_trigger - SET - last_server_ping = now(), - error = $1 - WHERE - workspace_id = $2 - AND path = $3 - AND server_id = $4 - AND enabled IS TRUE - RETURNING 1 - "#, - error, - &self.workspace_id, - &self.path, - *INSTANCE_NAME - ) - .fetch_optional(db) - .await; - - match updated { - Ok(updated) => { - if updated.flatten().is_none() { - // allow faster restart of database trigger - sqlx::query!( - r#" - UPDATE - postgres_trigger - SET - last_server_ping = NULL - WHERE - workspace_id = $1 - AND path = $2 - AND server_id IS NULL"#, - &self.workspace_id, - &self.path, - ) - .execute(db) - .await - .ok(); - tracing::info!( - "Postgres trigger {} changed, disabled, or deleted, stopping...", - self.path - ); - return None; - } - } - Err(err) => { - tracing::warn!( - "Error updating ping of postgres trigger {}: {:?}", - self.path, - err - ); - } - }; - - Some(()) - } - - async fn disable_with_error(&self, db: &DB, error: String) -> () { - match sqlx::query!( - r#" - UPDATE - postgres_trigger - SET - enabled = FALSE, - error = $1, - server_id = NULL, - last_server_ping = NULL - WHERE - workspace_id = $2 AND - path = $3 - "#, - error, - self.workspace_id, - self.path, - ) - .execute(db) - .await - { - Ok(_) => { - report_critical_error( - format!( - "Disabling postgres trigger {} because of error: {}", - self.path, error - ), - db.clone(), - Some(&self.workspace_id), - None, - ) - .await; - } - Err(disable_err) => { - report_critical_error( - format!("Could not disable postgres trigger {} with err {}, disabling because of error {}", self.path, disable_err, error), - db.clone(), - Some(&self.workspace_id), - None, - ).await; - } - } - } - - async fn fetch_authed(&self, db: &DB) -> error::Result { - fetch_api_authed( - self.edited_by.clone(), - self.email.clone(), - &self.workspace_id, - db, - Some(format!("pg-{}", self.path)), - ) - .await - } - - async fn handle(&self, db: &DB, payload: HashMap>) -> () { - if let Err(err) = run_job(payload, db, self).await { - report_critical_error( - format!( - "Failed to trigger job from postgres {}: {:?}", - self.path, err - ), - db.clone(), - Some(&self.workspace_id), - None, - ) - .await; - }; - } -} - -impl TriggerJobArgs>> for PostgresTrigger { - fn v1_payload_fn(payload: HashMap>) -> HashMap> { - payload - } - - fn v2_payload_fn(payload: HashMap>) -> HashMap> { - payload - } - - fn trigger_kind() -> TriggerKind { - TriggerKind::Postgres - } -} - -async fn run_job( - payload: HashMap>, - db: &DB, - trigger: &PostgresTrigger, -) -> anyhow::Result<()> { - let args = PostgresTrigger::build_job_args( - &trigger.script_path, - trigger.is_flow, - &trigger.workspace_id, - db, - payload, - HashMap::new(), - ) - .await?; - - let authed = fetch_api_authed( - trigger.edited_by.clone(), - trigger.email.clone(), - &trigger.workspace_id, - db, - Some(format!("postgres-{}", trigger.path)), - ) - .await?; - - trigger_runnable( - db, - None, - authed, - &trigger.workspace_id, - &trigger.script_path, - trigger.is_flow, - args, - trigger.retry.as_ref(), - trigger.error_handler_path.as_deref(), - trigger.error_handler_args.as_ref(), - format!("postgres_trigger/{}", trigger.path), - ) - .await?; - - Ok(()) -} - -struct PgInfo<'a> { - postgres_resource_path: &'a str, - publication_name: &'a str, - replication_slot_name: &'a str, - workspace_id: &'a str, -} - -impl PostgresConfig { - async fn update_ping(&self, db: &DB, error: Option<&str>) -> Option<()> { - match self { - PostgresConfig::Trigger(trigger) => trigger.update_ping(db, error).await, - PostgresConfig::Capture(capture) => capture.update_ping(db, error).await, - } - } - - async fn disable_with_error(&self, db: &DB, error: String) -> () { - match self { - PostgresConfig::Trigger(trigger) => trigger.disable_with_error(&db, error).await, - PostgresConfig::Capture(capture) => capture.disable_with_error(db, error).await, - } - } - - fn retrieve_info(&self) -> PgInfo<'_> { - let postgres_resource_path; - let publication_name; - let replication_slot_name; - let workspace_id; - - match self { - PostgresConfig::Trigger(trigger) => { - postgres_resource_path = &trigger.postgres_resource_path; - publication_name = &trigger.publication_name; - replication_slot_name = &trigger.replication_slot_name; - workspace_id = &trigger.workspace_id; - } - PostgresConfig::Capture(capture) => { - postgres_resource_path = &capture.trigger_config.postgres_resource_path; - workspace_id = &capture.workspace_id; - publication_name = capture.trigger_config.publication_name.as_ref().unwrap(); - replication_slot_name = capture - .trigger_config - .replication_slot_name - .as_ref() - .unwrap(); - } - }; - - PgInfo { postgres_resource_path, replication_slot_name, workspace_id, publication_name } - } - - async fn start_logical_replication_streaming( - &self, - db: &DB, - ) -> std::result::Result<(CopyBothDuplex, LogicalReplicationSettings), Error> { - let PgInfo { - publication_name, - replication_slot_name, - workspace_id, - postgres_resource_path, - } = self.retrieve_info(); - - let authed = match self { - PostgresConfig::Trigger(trigger) => trigger.fetch_authed(db).await?, - PostgresConfig::Capture(capture) => capture.fetch_authed(db).await?, - }; - - let database = try_get_resource_from_db_as::( - &authed, - Some(UserDB::new(db.clone())), - &db, - postgres_resource_path, - workspace_id, - ) - .await?; - - let client = PostgresSimpleClient::new(&database).await?; - - let publication = client - .execute_query(&format!( - "SELECT pubname FROM pg_publication WHERE pubname = {}", - quote_literal(&publication_name) - )) - .await - .map_err(to_anyhow)?; - - if !publication.row_exist() { - return Err(Error::BadConfig( - ERROR_PUBLICATION_NAME_NOT_EXISTS.to_string(), - )); - } - - let replication_slot = client - .execute_query(&format!( - "SELECT slot_name FROM pg_replication_slots WHERE slot_name = {}", - quote_literal(&replication_slot_name) - )) - .await - .map_err(to_anyhow)?; - - if !replication_slot.row_exist() { - return Err(Error::BadConfig( - ERROR_REPLICATION_SLOT_NOT_EXISTS.to_string(), - )); - } - - let (logical_replication_stream, logical_replication_settings) = client - .get_logical_replication_stream(&publication_name, &replication_slot_name) - .await - .map_err(to_anyhow)?; - - Ok((logical_replication_stream, logical_replication_settings)) - } - - fn get_path(&self) -> &str { - match self { - PostgresConfig::Trigger(trigger) => &trigger.path, - PostgresConfig::Capture(capture) => &capture.path, - } - } - - async fn handle(&self, db: &DB, payload: HashMap>) -> () { - match self { - PostgresConfig::Trigger(trigger) => trigger.handle(&db, payload).await, - PostgresConfig::Capture(capture) => capture.handle(&db, payload).await, - } - } - - async fn cleanup(&self, db: &DB) -> Result<(), Error> { - match self { - PostgresConfig::Trigger(_) => Ok(()), - PostgresConfig::Capture(capture) => { - let publication_name = capture.trigger_config.publication_name.as_ref().unwrap(); - let replication_slot_name = capture - .trigger_config - .replication_slot_name - .as_ref() - .unwrap(); - let postgres_resource_path = &capture.trigger_config.postgres_resource_path; - let workspace_id = &capture.workspace_id; - let authed = capture.fetch_authed(&db).await?; - - let user_db = UserDB::new(db.clone()); - - let mut pg_connection = get_default_pg_connection( - authed.clone(), - Some(user_db.clone()), - &db, - postgres_resource_path, - workspace_id, - ) - .await?; - - if capture.trigger_config.basic_mode.unwrap_or(false) { - drop_logical_replication_slot(&mut pg_connection, replication_slot_name) - .await?; - - drop_publication(&mut pg_connection, publication_name).await?; - } - - Ok(()) - } - } - } -} - -async fn listen_to_transactions( - pg: PostgresConfig, - db: DB, - mut killpill_rx: tokio::sync::broadcast::Receiver<()>, -) { - tokio::select! { - biased; - _ = killpill_rx.recv() => { - let _ = pg.cleanup(&db).await; - return; - } - _ = loop_ping(&db, &pg, Some("Connecting...")) => { - let _ = pg.cleanup(&db).await; - return; - } - result = pg.start_logical_replication_streaming(&db) => { - tokio::select! { - biased; - _ = killpill_rx.recv() => { - let _ = pg.cleanup(&db).await; - return; - } - _ = loop_ping(&db, &pg, None) => { - let _ = pg.cleanup(&db).await; - return; - } - _ = { - async { - match result { - Ok((logical_replication_stream, logical_replication_settings)) => { - pin_mut!(logical_replication_stream); - let mut relations = RelationConverter::new(); - tracing::info!("Starting to listen for postgres trigger {}", pg.get_path()); - loop { - let message = logical_replication_stream.next().await; - - let message = match message { - Some(message) => message, - None => { - tracing::error!("Stream for postgres trigger {} closed", pg.get_path()); - if let None = pg.update_ping(&db, Some("Stream closed")).await { - return; - } - return; - } - }; - - - let message = match message { - Ok(message) => message, - Err(err) => { - let err = format!("Postgres trigger named {} had an error while receiving a message : {}", pg.get_path(), err.to_string()); - pg.disable_with_error(&db, err).await; - return; - } - }; - - let logical_message = match ReplicationMessage::parse(message) { - Ok(logical_message) => logical_message, - Err(err) => { - let err = format!("Postgres trigger named: {} had an error while parsing message: {}", pg.get_path(), err.to_string()); - pg.disable_with_error(&db, err).await; - return; - } - }; - - - match logical_message { - ReplicationMessage::PrimaryKeepAlive(primary_keep_alive) => { - if primary_keep_alive.reply { - PostgresSimpleClient::send_status_update(primary_keep_alive, &mut logical_replication_stream).await; - } - } - ReplicationMessage::XLogData(x_log_data) => { - let logical_replication_message = match x_log_data.parse(&logical_replication_settings) { - Ok(logical_replication_message) => logical_replication_message, - Err(err) => { - tracing::error!("Postgres trigger named: {} had an error while trying to parse incomming stream message: {}", pg.get_path(), err.to_string()); - continue; - } - }; - - let json = match logical_replication_message { - Relation(relation_body) => { - relations.add_relation(relation_body); - None - } - Begin | Type | Commit => { - None - } - Insert(insert) => { - Some((insert.o_id, Ok(None), relations.row_to_json((insert.o_id, insert.tuple)), "insert")) - } - Update(update) => { - let old_row = update.old_tuple.map(|old_tuple| relations.row_to_json((update.o_id, old_tuple))).transpose(); - let row = relations.row_to_json((update.o_id, update.new_tuple)); - Some((update.o_id, old_row, row, "update")) - } - Delete(delete) => { - let row = delete.old_tuple.unwrap_or_else(|| delete.key_tuple.unwrap()); - Some((delete.o_id, Ok(None), relations.row_to_json((delete.o_id, row)), "delete")) - } - }; - match json { - Some((o_id, Ok(old_row), Ok(row), transaction_type)) => { - let relation = match relations.get_relation(o_id) { - Ok(relation) => relation, - Err(err) => { - tracing::error!("Postgres trigger named: {}, error: {}", pg.get_path(), err.to_string()); - continue; - } - }; - let database_info = HashMap::from([ - ("schema_name".to_string(), to_raw_value(&relation.namespace)), - ("table_name".to_string(), to_raw_value(&relation.name)), - ("transaction_type".to_string(), to_raw_value(&transaction_type)), - ("old_row".to_string(), to_raw_value(&old_row)), - ("row".to_string(), to_raw_value(&row)), - ]); - - - let _ = pg.handle(&db, database_info).await; - } - Some((o_id, old_row, row, transaction_type)) => { - let relation = match relations.get_relation(o_id) { - Ok(relation) => relation, - Err(err) => { - tracing::error!("Postgres trigger named: {}, error: {}", pg.get_path(), err.to_string()); - continue; - } - }; - - if let Err(err) = old_row { - tracing::error!( - transaction_type = ?transaction_type, - schema = %relation.namespace, - table = %relation.name, - error = %err, - "Failed to decode OLD row for {} transaction on {}.{}", - transaction_type, - relation.namespace, - relation.name, - ); - } - - if let Err(err) = row { - tracing::error!( - transaction_type = ?transaction_type, - schema = %relation.namespace, - table = %relation.name, - error = %err, - "Failed to decode NEW row for {} transaction on {}.{}", - transaction_type, - relation.namespace, - relation.name, - ); - } - - } - _ => {} - } - - } - } - } - } - Err(err) => { - tracing::error!("Postgres trigger error while trying to start logical replication streaming: {}", &err); - pg.disable_with_error(&db, err.to_string()).await - } - } - } - } => { - let _ = pg.cleanup(&db).await; - return; - } - } - } - } -} - -#[derive(Deserialize)] -struct CaptureConfigForPostgresTrigger { - trigger_config: SqlxJson, - path: String, - is_flow: bool, - workspace_id: String, - owner: String, - email: String, -} - -impl CaptureConfigForPostgresTrigger { - async fn try_to_listen_to_database_transactions( - self, - db: DB, - killpill_rx: tokio::sync::broadcast::Receiver<()>, - ) -> () { - match sqlx::query_scalar!( - r#" - UPDATE - capture_config - SET - server_id = $1, - last_server_ping = now(), - error = 'Connecting...' - WHERE - last_client_ping > NOW() - INTERVAL '10 seconds' AND - workspace_id = $2 AND - path = $3 AND - is_flow = $4 AND - trigger_kind = 'postgres' AND - (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') - RETURNING true - "#, - *INSTANCE_NAME, - self.workspace_id, - self.path, - self.is_flow, - ) - .fetch_optional(&db) - .await - { - Ok(has_lock) => { - if has_lock.flatten().unwrap_or(false) { - tokio::spawn(listen_to_transactions( - PostgresConfig::Capture(self), - db, - killpill_rx, - )); - } else { - tracing::info!("Postgres {} already being listened to", self.path); - } - } - Err(err) => { - tracing::error!( - "Error acquiring lock for capture postgres {}: {:?}", - self.path, - err - ); - } - }; - } - - async fn update_ping(&self, db: &DB, error: Option<&str>) -> Option<()> { - match sqlx::query_scalar!( - r#" - UPDATE - capture_config - SET - last_server_ping = now(), - error = $1 - WHERE - workspace_id = $2 AND - path = $3 AND - is_flow = $4 AND - trigger_kind = 'postgres' AND - server_id = $5 AND - last_client_ping > NOW() - INTERVAL '10 seconds' - RETURNING 1 - "#, - error, - self.workspace_id, - self.path, - self.is_flow, - *INSTANCE_NAME - ) - .fetch_optional(db) - .await - { - Ok(updated) => { - if updated.flatten().is_none() { - // allow faster restart of postgres capture - sqlx::query!( - r#"UPDATE - capture_config - SET - last_server_ping = NULL - WHERE - workspace_id = $1 AND - path = $2 AND - is_flow = $3 AND - trigger_kind = 'postgres' AND - server_id IS NULL - "#, - self.workspace_id, - self.path, - self.is_flow, - ) - .execute(db) - .await - .ok(); - tracing::info!( - "Postgres capture {} changed, disabled, or deleted, stopping...", - self.path - ); - return None; - } - } - Err(err) => { - tracing::warn!( - "Error updating ping of capture postgres {}: {:?}", - self.path, - err - ); - } - }; - - Some(()) - } - - async fn fetch_authed(&self, db: &DB) -> error::Result { - fetch_api_authed( - self.owner.clone(), - self.email.clone(), - &self.workspace_id, - db, - Some(format!("postgres-{}", self.get_trigger_path())), - ) - .await - } - - fn get_trigger_path(&self) -> String { - format!( - "{}-{}", - if self.is_flow { "flow" } else { "script" }, - self.path - ) - } - - async fn disable_with_error(&self, db: &DB, error: String) -> () { - if let Err(err) = sqlx::query!( - r#" - UPDATE - capture_config - SET - error = $1, - server_id = NULL, - last_server_ping = NULL - WHERE - workspace_id = $2 AND - path = $3 AND - is_flow = $4 AND - trigger_kind = 'postgres' - "#, - error, - self.workspace_id, - self.path, - self.is_flow, - ) - .execute(db) - .await - { - tracing::error!("Could not disable postgres capture {} ({}) with err {}, disabling because of error {}", self.path, self.workspace_id, err, error); - } - } - - async fn handle(&self, db: &DB, payload: HashMap>) -> () { - let main_args = PostgresTrigger::build_job_args_v2(false, payload.clone(), HashMap::new()); - let preprocessor_args = PostgresTrigger::build_job_args_v2(true, payload, HashMap::new()); - if let Err(err) = insert_capture_payload( - db, - &self.workspace_id, - &self.path, - self.is_flow, - &TriggerKind::Postgres, - main_args, - preprocessor_args, - &self.owner, - ) - .await - { - tracing::error!("Error inserting capture payload: {:?}", err); - } - } -} - -async fn listen_to_unlistened_database_events( - db: &DB, - killpill_rx: &tokio::sync::broadcast::Receiver<()>, -) { - let postgres_triggers = sqlx::query_as!( - PostgresTrigger, - r#" - SELECT - workspace_id, - path, - script_path, - replication_slot_name, - publication_name, - is_flow, - edited_by, - email, - edited_at, - server_id, - last_server_ping, - extra_perms, - error, - enabled, - postgres_resource_path, - error_handler_path, - error_handler_args as "error_handler_args: _", - retry as "retry: _" - FROM - postgres_trigger - WHERE - enabled IS TRUE - AND (last_server_ping IS NULL OR - last_server_ping < now() - interval '15 seconds' - ) - "# - ) - .fetch_all(db) - .await; - - match postgres_triggers { - Ok(mut triggers) => { - triggers.shuffle(&mut rand::rng()); - for trigger in triggers { - trigger - .try_to_listen_to_database_transactions(db.clone(), killpill_rx.resubscribe()) - .await; - } - } - Err(err) => { - tracing::error!("Error fetching postgres triggers: {:?}", err); - } - }; - - let postgres_triggers_capture = sqlx::query_as!( - CaptureConfigForPostgresTrigger, - r#" - SELECT - path, - is_flow, - workspace_id, - owner, - email, - trigger_config as "trigger_config!: _" - FROM - capture_config - WHERE - trigger_kind = 'postgres' AND - last_client_ping > NOW() - INTERVAL '10 seconds' AND - trigger_config IS NOT NULL AND - (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') - "# - ) - .fetch_all(db) - .await; - - match postgres_triggers_capture { - Ok(mut captures) => { - captures.shuffle(&mut rand::rng()); - for capture in captures { - capture - .try_to_listen_to_database_transactions(db.clone(), killpill_rx.resubscribe()) - .await; - } - } - Err(err) => { - tracing::error!("Error fetching captures postgres triggers: {:?}", err); - } - }; -} - -pub fn start_database(db: DB, mut killpill_rx: tokio::sync::broadcast::Receiver<()>) { - tokio::spawn(async move { - listen_to_unlistened_database_events(&db, &killpill_rx).await; - loop { - tokio::select! { - biased; - _ = killpill_rx.recv() => { - return; - } - _ = tokio::time::sleep(tokio::time::Duration::from_secs(15)) => { - listen_to_unlistened_database_events(&db, &killpill_rx).await - } - } - } - }); -} diff --git a/backend/windmill-api/src/sqs_triggers_oss.rs b/backend/windmill-api/src/sqs_triggers_oss.rs deleted file mode 100644 index 49fffb43e9..0000000000 --- a/backend/windmill-api/src/sqs_triggers_oss.rs +++ /dev/null @@ -1,44 +0,0 @@ -#[cfg(feature = "private")] -#[allow(unused)] -pub use crate::sqs_triggers_ee::*; - -#[cfg(not(feature = "private"))] -use crate::db::DB; -#[cfg(not(feature = "private"))] -use serde::{Deserialize, Serialize}; -#[cfg(not(feature = "private"))] -use windmill_common::auth::aws::AwsAuthResourceType; - -#[cfg(not(feature = "private"))] -pub fn start_sqs(_db: DB, mut _killpill_rx: tokio::sync::broadcast::Receiver<()>) -> () { - // implementation is not open source -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -#[cfg(not(feature = "private"))] -pub struct SqsTrigger { - pub queue_url: String, - pub aws_auth_resource_type: AwsAuthResourceType, - pub aws_resource_path: String, - pub message_attributes: Option>, - pub path: String, - pub script_path: String, - pub is_flow: bool, - pub workspace_id: String, - pub edited_by: String, - pub email: String, - pub edited_at: chrono::DateTime, - pub extra_perms: Option, - pub error: Option, - pub server_id: Option, - pub last_server_ping: Option>, - pub enabled: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_args: Option< - sqlx::types::Json>>, - >, - #[serde(skip_serializing_if = "Option::is_none")] - pub retry: Option>, -} diff --git a/backend/windmill-api/src/trigger_helpers.rs b/backend/windmill-api/src/trigger_helpers.rs deleted file mode 100644 index 14f9e42005..0000000000 --- a/backend/windmill-api/src/trigger_helpers.rs +++ /dev/null @@ -1,808 +0,0 @@ -use anyhow::Context; -use axum::response::IntoResponse; -use http::StatusCode; -use serde::Deserialize; -use serde_json::value::RawValue; -use std::collections::HashMap; -use uuid::Uuid; -use windmill_common::{ - db::{UserDB, UserDbWithAuthed}, - error::Result, - flows::{FlowModuleValue, Retry}, - get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path, - jobs::{get_has_preprocessor_from_content_and_lang, script_path_to_payload, JobPayload}, - scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, - triggers::{ - HubOrWorkspaceId, RunnableFormat, RunnableFormatVersion, TriggerKind, - RUNNABLE_FORMAT_VERSION_CACHE, - }, - users::username_to_permissioned_as, - utils::StripPath, - worker::to_raw_value, - FlowVersionInfo, -}; -use windmill_queue::{push, PushArgs, PushArgsOwned, PushIsolationLevel}; - -#[cfg(feature = "enterprise")] -use crate::jobs::check_license_key_valid; -use crate::{ - db::{ApiAuthed, DB}, - jobs::{ - check_tag_available_for_workspace, delete_job_metadata_after_use, result_to_response, - run_flow_by_path_inner, run_script_by_path_inner, run_wait_result_internal, RunJobQuery, - }, - utils::check_scopes, - HTTP_CLIENT, -}; - -struct ScriptInfo { - has_preprocessor: Option, - language: ScriptLang, - content: String, - schema: Option>, -} - -#[derive(Debug, Deserialize)] -struct PropertyDefinition { - r#type: Option, -} - -#[derive(Debug, Deserialize)] -struct PartialSchema { - properties: Option>, -} - -#[derive(Debug, Clone, Eq, PartialEq, Hash)] -pub enum RunnableId { - FlowPath(String), - ScriptId(ScriptId), - HubScript(String), -} - -impl RunnableId { - #[allow(dead_code)] - pub fn from_script_hash(hash: ScriptHash) -> Self { - Self::ScriptId(ScriptId::ScriptHash(hash)) - } - - pub fn from_script_path(path: &str) -> Self { - if path.starts_with("hub/") { - Self::HubScript(path.to_string()) - } else { - Self::ScriptId(ScriptId::ScriptPath(path.to_string())) - } - } - - pub fn from_flow_path(path: &str) -> Self { - Self::FlowPath(path.to_string()) - } -} - -#[derive(Debug, Clone, Eq, PartialEq, Hash)] -#[allow(unused)] -pub enum ScriptId { - ScriptPath(String), - ScriptHash(ScriptHash), -} - -impl ScriptId { - async fn get_script_hash(self, workspace_id: &str, db: &DB) -> Result { - let hash = match self { - ScriptId::ScriptPath(path) => { - let info = get_latest_deployed_hash_for_path(None, db.clone(), workspace_id, &path) - .await?; - info.hash - } - ScriptId::ScriptHash(hash) => hash.0, - }; - - Ok(hash) - } -} - -async fn get_script_info( - db: &DB, - workspace_id: &str, - hash: i64, -) -> std::result::Result { - sqlx::query_as!(ScriptInfo, "SELECT has_preprocessor, language as \"language: _\", content, schema as \"schema: _\" FROM script WHERE workspace_id = $1 AND hash = $2", workspace_id, hash) - .fetch_one(db) - .await -} - -fn runnable_format_from_schema_without_preprocessor( - trigger_kind: &TriggerKind, - schema: Option>, -) -> RunnableFormat { - match trigger_kind { - TriggerKind::Mqtt - if schema.as_ref().is_some_and(|schema| { - schema.properties.as_ref().is_some_and(|properties| { - properties.iter().any(|(key, def)| { - key == "payload" && def.r#type.as_ref().is_some_and(|t| t == "array") - }) - }) - }) => - { - RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: false } - } - TriggerKind::Kafka | TriggerKind::Nats - if schema.as_ref().is_some_and(|schema| { - schema - .properties - .as_ref() - .is_some_and(|properties| properties.keys().any(|key| key == "msg")) - }) => - { - RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: false } - } - _ => RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor: false }, - } -} - -fn runnable_format_from_preprocessor_args( - args: Option>, -) -> RunnableFormat { - if let Some(args) = args { - if args.iter().any(|arg| arg.name == "wm_trigger") - || (args.len() > 0 && args.iter().all(|arg| arg.name != "event")) - { - RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: true } - } else { - RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor: true } - } - } else { - RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor: true } - } -} - -enum PreprocessorInfo { - Preprocessor { content: String, language: ScriptLang }, - NoPreprocessor { schema: Option> }, -} - -#[derive(Debug, Deserialize)] -struct FlowInfo { - preprocessor_module: Option>, - schema: Option>, -} - -fn get_preprocessor_args_from_content_and_language( - content: &str, - language: &ScriptLang, -) -> Result>> { - let args = match language { - ScriptLang::Bun | ScriptLang::Bunnative | ScriptLang::Deno | ScriptLang::Nativets => { - let args = windmill_parser_ts::parse_deno_signature( - &content, - true, - false, - Some("preprocessor".to_string()), - )?; - Some(args.args) - } - ScriptLang::Python3 => { - let args = windmill_parser_py::parse_python_signature( - &content, - Some("preprocessor".to_string()), - false, - )?; - Some(args.args) - } - _ => None, - }; - Ok(args) -} - -pub async fn get_runnable_format( - runnable_id: RunnableId, - workspace_id: &str, - db: &DB, - trigger_kind: &TriggerKind, -) -> Result { - let (key, preprocessor_info) = match runnable_id { - RunnableId::HubScript(path) => { - let Some(version) = path.split("/").nth(1) else { - return Err(windmill_common::error::Error::internal_err( - "Invalid hub script path".to_string(), - )); - }; - - let version = match version.parse::() { - Ok(version) => version, - Err(_) => { - return Err(windmill_common::error::Error::internal_err( - "Invalid hub script version".to_string(), - )); - } - }; - - let key = (HubOrWorkspaceId::Hub, version, trigger_kind.clone()); - - let runnable_format = RUNNABLE_FORMAT_VERSION_CACHE.get(&key); - - if let Some(runnable_format) = runnable_format { - tracing::debug!("Using cached runnable format for hub script {path}"); - return Ok(runnable_format); - } - - let hub_script = - get_full_hub_script_by_path(StripPath(path.to_string()), &HTTP_CLIENT, Some(db)) - .await?; - - let has_preprocessor = get_has_preprocessor_from_content_and_lang( - &hub_script.content, - &hub_script.language, - )?; - - let partial_schema = serde_json::from_str(hub_script.schema.get())?; - - ( - key, - if has_preprocessor { - PreprocessorInfo::Preprocessor { - content: hub_script.content, - language: hub_script.language, - } - } else { - PreprocessorInfo::NoPreprocessor { - schema: Some(sqlx::types::Json(partial_schema)), - } - }, - ) - } - RunnableId::FlowPath(path) => { - let FlowVersionInfo { version, .. } = - get_latest_flow_version_info_for_path(None, &db, workspace_id, &path, true).await?; - - let key = ( - HubOrWorkspaceId::WorkspaceId(workspace_id.to_string()), - version, - trigger_kind.clone(), - ); - - let runnable_format = RUNNABLE_FORMAT_VERSION_CACHE.get(&key); - - if let Some(runnable_format) = runnable_format { - tracing::debug!("Using cached runnable format for flow {path}"); - return Ok(runnable_format); - } - - let flow_info = sqlx::query_as!( - FlowInfo, - "SELECT - value->'preprocessor_module'->'value' as \"preprocessor_module: _\", - schema as \"schema: _\" - FROM flow_version - WHERE - path = $1 - AND workspace_id = $2 - ORDER BY created_at DESC - LIMIT 1", - path, - workspace_id, - ) - .fetch_one(db) - .await?; - - if let Some(preprocessor_module) = flow_info.preprocessor_module { - match preprocessor_module.0 { - FlowModuleValue::RawScript { content, language, .. } => { - (key, PreprocessorInfo::Preprocessor { content, language }) - } - FlowModuleValue::Script { path, hash, .. } => { - let hash = if let Some(hash) = hash { - hash.0 - } else { - let script_hash = get_latest_deployed_hash_for_path( - None, - db.clone(), - workspace_id, - &path, - ) - .await?; - script_hash.hash - }; - let script_info = get_script_info(db, workspace_id, hash).await?; - ( - key, - PreprocessorInfo::Preprocessor { - content: script_info.content, - language: script_info.language, - }, - ) - } - _ => { - return Err(windmill_common::error::Error::internal_err( - "Unsupported preprocessor module".to_string(), - )); - } - } - } else { - ( - key, - PreprocessorInfo::NoPreprocessor { schema: flow_info.schema }, - ) - } - } - RunnableId::ScriptId(script_id) => { - let hash = script_id.get_script_hash(workspace_id, db).await?; - let key = ( - HubOrWorkspaceId::WorkspaceId(workspace_id.to_string()), - hash, - trigger_kind.clone(), - ); - let runnable_format = RUNNABLE_FORMAT_VERSION_CACHE.get(&key); - - if let Some(runnable_format) = runnable_format { - tracing::debug!("Using cached runnable format for script {hash}"); - return Ok(runnable_format); - } - - let script_info = get_script_info(db, workspace_id, hash).await?; - - if script_info.has_preprocessor.unwrap_or(false) { - ( - key, - PreprocessorInfo::Preprocessor { - content: script_info.content, - language: script_info.language, - }, - ) - } else { - ( - key, - PreprocessorInfo::NoPreprocessor { schema: script_info.schema }, - ) - } - } - }; - - let runnable_format = match preprocessor_info { - PreprocessorInfo::Preprocessor { content, language } => { - let args = get_preprocessor_args_from_content_and_language(&content, &language)?; - runnable_format_from_preprocessor_args(args) - } - PreprocessorInfo::NoPreprocessor { schema } => { - runnable_format_from_schema_without_preprocessor(trigger_kind, schema) - } - }; - - RUNNABLE_FORMAT_VERSION_CACHE.insert(key, runnable_format); - - Ok(runnable_format) -} - -#[allow(dead_code)] -pub trait TriggerJobArgs { - fn v1_payload_fn(payload: T) -> HashMap>; - fn v2_payload_fn(payload: T) -> HashMap> { - Self::v1_payload_fn(payload) - } - fn trigger_kind() -> TriggerKind; - - fn build_job_args_v2( - has_preprocessor: bool, - payload: T, - info: HashMap>, - ) -> PushArgsOwned { - let trigger_kind = Self::trigger_kind(); - let mut args = Self::v2_payload_fn(payload); - if has_preprocessor { - args.insert("kind".to_string(), to_raw_value(&trigger_kind.to_key())); - args.extend(info); - let args = HashMap::from([("event".to_string(), to_raw_value(&args))]); - PushArgsOwned { args, extra: None } - } else { - PushArgsOwned { args, extra: None } - } - } - - fn build_job_args_v1( - has_preprocessor: bool, - payload: T, - info: HashMap>, - ) -> PushArgsOwned { - let trigger_kind = Self::trigger_kind(); - let trigger_key = trigger_kind.to_key(); - let args = Self::v1_payload_fn(payload); - let extra = if has_preprocessor { - Some(HashMap::from([( - "wm_trigger".to_string(), - to_raw_value(&serde_json::json!({ - "kind": trigger_key, - trigger_key: info - })), - )])) - } else { - None - }; - - PushArgsOwned { args, extra } - } - - async fn build_job_args( - runnable_path: &str, - is_flow: bool, - w_id: &str, - db: &DB, - payload: T, - info: HashMap>, - ) -> Result { - let runnable_id = if is_flow { - RunnableId::from_flow_path(runnable_path) - } else { - RunnableId::from_script_path(runnable_path) - }; - Self::build_job_args_from_runnable_id(runnable_id, w_id, db, payload, info).await - } - - async fn build_job_args_from_runnable_id( - runnable_id: RunnableId, - w_id: &str, - db: &DB, - payload: T, - info: HashMap>, - ) -> Result { - let runnable_format = - get_runnable_format(runnable_id, w_id, db, &Self::trigger_kind()).await?; - - match runnable_format { - RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor } => { - Ok(Self::build_job_args_v1(has_preprocessor, payload, info)) - } - RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor } => { - Ok(Self::build_job_args_v2(has_preprocessor, payload, info)) - } - } - } - - fn build_capture_payloads( - payload: T, - info: HashMap>, - ) -> (PushArgsOwned, PushArgsOwned) { - let main_args = Self::build_job_args_v2(false, payload.clone(), info.clone()); - let preprocessor_args = Self::build_job_args_v2(true, payload, info); - (main_args, preprocessor_args) - } -} - -#[allow(dead_code)] -async fn trigger_runnable_inner( - db: &DB, - user_db: Option, - authed: ApiAuthed, - workspace_id: &str, - runnable_path: &str, - is_flow: bool, - args: PushArgsOwned, - retry: Option<&sqlx::types::Json>, - error_handler_path: Option<&str>, - error_handler_args: Option<&sqlx::types::Json>>>, - trigger_path: String, -) -> Result<(Uuid, Option, Option)> { - let user_db = user_db.unwrap_or_else(|| UserDB::new(db.clone())); - let (uuid, delete_after_use, early_return) = if is_flow { - let run_query = RunJobQuery::default(); - let path = StripPath(runnable_path.to_string()); - let (uuid, early_return) = run_flow_by_path_inner( - authed, - db.clone(), - user_db, - workspace_id.to_string(), - path, - run_query, - args, - ) - .await?; - (uuid, None, early_return) - } else { - let (uuid, delete_after_use) = trigger_script_internal( - db, - user_db, - authed, - workspace_id, - runnable_path, - args, - retry, - error_handler_path, - error_handler_args, - trigger_path, - ) - .await?; - (uuid, delete_after_use, None) - }; - - Ok((uuid, delete_after_use, early_return)) -} - -#[allow(dead_code)] -pub async fn trigger_runnable( - db: &DB, - user_db: Option, - authed: ApiAuthed, - workspace_id: &str, - runnable_path: &str, - is_flow: bool, - args: PushArgsOwned, - retry: Option<&sqlx::types::Json>, - error_handler_path: Option<&str>, - error_handler_args: Option<&sqlx::types::Json>>>, - trigger_path: String, -) -> Result { - let (uuid, _, _) = trigger_runnable_inner( - db, - user_db, - authed, - workspace_id, - runnable_path, - is_flow, - args, - retry, - error_handler_path, - error_handler_args, - trigger_path, - ) - .await?; - Ok((StatusCode::CREATED, uuid.to_string()).into_response()) -} - -#[allow(dead_code)] -pub async fn trigger_runnable_and_wait_for_result( - db: &DB, - user_db: Option, - authed: ApiAuthed, - workspace_id: &str, - runnable_path: &str, - is_flow: bool, - args: PushArgsOwned, - retry: Option<&sqlx::types::Json>, - error_handler_path: Option<&str>, - error_handler_args: Option<&sqlx::types::Json>>>, - trigger_path: String, -) -> Result { - let username = authed.username.clone(); - let (uuid, delete_after_use, early_return) = trigger_runnable_inner( - db, - user_db, - authed, - workspace_id, - runnable_path, - is_flow, - args, - retry, - error_handler_path, - error_handler_args, - trigger_path, - ) - .await?; - let (result, success) = - run_wait_result_internal(db, uuid, workspace_id.to_string(), early_return, &username) - .await?; - - if delete_after_use.unwrap_or(false) { - delete_job_metadata_after_use(&db, uuid).await?; - } - - result_to_response(result, success) -} - -#[allow(dead_code)] -pub async fn trigger_runnable_and_wait_for_raw_result( - db: &DB, - user_db: Option, - authed: ApiAuthed, - workspace_id: &str, - runnable_path: &str, - is_flow: bool, - args: PushArgsOwned, - retry: Option<&sqlx::types::Json>, - error_handler_path: Option<&str>, - error_handler_args: Option<&sqlx::types::Json>>>, - trigger_path: String, -) -> Result> { - let username = authed.username.clone(); - let (uuid, delete_after_use, early_return) = trigger_runnable_inner( - db, - user_db, - authed, - workspace_id, - runnable_path, - is_flow, - args, - retry, - error_handler_path, - error_handler_args, - trigger_path, - ) - .await?; - - let (result, success) = - run_wait_result_internal(db, uuid, workspace_id.to_string(), early_return, &username) - .await - .with_context(|| { - format!( - "Error fetching job result for {} {}", - if is_flow { "flow" } else { "script" }, - runnable_path - ) - })?; - - if delete_after_use.unwrap_or(false) { - delete_job_metadata_after_use(&db, uuid).await?; - } - - if !success { - Err(windmill_common::error::Error::internal_err(format!( - "{} {runnable_path} failed: {:?}", - if is_flow { "Flow" } else { "Script" }, - result - ))) - } else { - Ok(result) - } -} - -async fn trigger_script_internal( - db: &DB, - user_db: UserDB, - authed: ApiAuthed, - workspace_id: &str, - script_path: &str, - args: PushArgsOwned, - retry: Option<&sqlx::types::Json>, - error_handler_path: Option<&str>, - error_handler_args: Option<&sqlx::types::Json>>>, - trigger_path: String, -) -> Result<(Uuid, Option)> { - if retry.is_none() && error_handler_path.is_none() { - let run_query = RunJobQuery::default(); - let path = StripPath(script_path.to_string()); - run_script_by_path_inner( - authed, - db.clone(), - user_db, - workspace_id.to_string(), - path, - run_query, - args, - ) - .await - } else { - trigger_script_with_retry_and_error_handler( - db, - user_db, - authed, - workspace_id, - script_path, - args, - retry, - error_handler_path, - error_handler_args, - trigger_path, - ) - .await - } -} - -async fn trigger_script_with_retry_and_error_handler( - db: &DB, - user_db: UserDB, - authed: ApiAuthed, - workspace_id: &str, - script_path: &str, - args: PushArgsOwned, - retry: Option<&sqlx::types::Json>, - error_handler_path: Option<&str>, - error_handler_args: Option<&sqlx::types::Json>>>, - trigger_path: String, -) -> Result<(Uuid, Option)> { - #[cfg(feature = "enterprise")] - check_license_key_valid().await?; - - check_scopes(&authed, || format!("jobs:run:scripts:{script_path}"))?; - - let retry = retry.map(|r| r.0.clone()); - let error_handler_path = error_handler_path.map(|p| p.to_string()); - let error_handler_args = error_handler_args.map(|args| args.0.clone()); - - let (job_payload, tag, delete_after_use, timeout, on_behalf_of) = { - let db_authed = UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() }; - script_path_to_payload( - script_path, - Some(db_authed), - db.clone(), - &workspace_id, - Some(false), - ) - .await? - }; - - check_tag_available_for_workspace(&db, &workspace_id, &tag, &authed).await?; - - let (email, permissioned_as, push_authed, tx) = - if let Some(on_behalf_of) = on_behalf_of.as_ref() { - ( - on_behalf_of.email.as_str(), - on_behalf_of.permissioned_as.clone(), - None, - PushIsolationLevel::IsolatedRoot(db.clone()), - ) - } else { - ( - authed.email.as_str(), - username_to_permissioned_as(&authed.username), - Some(authed.clone().into()), - PushIsolationLevel::Isolated(user_db, authed.clone().into()), - ) - }; - - let push_args = PushArgs { args: &args.args, extra: args.extra }; - - let retryable_job_payload = match job_payload { - JobPayload::ScriptHash { - hash, - path, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, - cache_ttl, - priority, - apply_preprocessor, - .. - } => JobPayload::SingleScriptFlow { - path, - hash, - args: HashMap::from(&push_args), - retry, - error_handler_path, - error_handler_args, - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, - cache_ttl, - priority, - tag_override: tag.clone(), - apply_preprocessor, - trigger_path: Some(trigger_path), - }, - _ => { - return Err(windmill_common::error::Error::internal_err(format!( - "Unsupported job payload: {:?}", - job_payload - ))) - } - }; - - let (uuid, tx) = push( - &db, - tx, - &workspace_id, - retryable_job_payload, - push_args, - authed.display_username(), - email, - permissioned_as, - authed.token_prefix.as_deref(), - None, - None, - None, - None, - None, - None, - false, - false, - None, - true, - tag, - timeout, - None, - None, - push_authed.as_ref(), - false, - ) - .await?; - tx.commit().await?; - - Ok((uuid, delete_after_use)) -} diff --git a/backend/windmill-api/src/triggers/gcp/handler_oss.rs b/backend/windmill-api/src/triggers/gcp/handler_oss.rs index 825c7f71e0..4b52ff9c68 100644 --- a/backend/windmill-api/src/triggers/gcp/handler_oss.rs +++ b/backend/windmill-api/src/triggers/gcp/handler_oss.rs @@ -1,3 +1,4 @@ +#[allow(unused)] #[cfg(feature = "private")] pub use super::handler_ee::*; @@ -29,7 +30,7 @@ impl TriggerCrud for GcpTrigger { const SUPPORTS_TEST_CONNECTION: bool = false; const ROUTE_PREFIX: &'static str = "/gcp_triggers"; const DEPLOYMENT_NAME: &'static str = ""; - const IS_CLOUD_HOSTED: bool = false; + const IS_ALLOWED_ON_CLOUD: bool = false; fn get_deployed_object(path: String) -> DeployedObject { DeployedObject::GcpTrigger { path } diff --git a/backend/windmill-api/src/triggers/gcp/listener_oss.rs b/backend/windmill-api/src/triggers/gcp/listener_oss.rs new file mode 100644 index 0000000000..a14168968a --- /dev/null +++ b/backend/windmill-api/src/triggers/gcp/listener_oss.rs @@ -0,0 +1,41 @@ +#[allow(unused)] + +#[cfg(feature = "private")] +pub use super::listener_ee::*; + +#[cfg(not(feature = "private"))] +use { + super::GcpTrigger, + crate::triggers::{listener::ListeningTrigger, Listener}, + std::sync::Arc, + tokio::sync::RwLock, + windmill_common::{error::Result, jobs::JobTriggerKind, DB}, +}; + +#[cfg(not(feature = "private"))] +#[async_trait::async_trait] +impl Listener for GcpTrigger { + type Consumer = (); + type Extra = (); + const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Gcp; + + async fn get_consumer( + &self, + _db: &DB, + _listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> Result> { + Ok(None) + } + async fn consume( + &self, + _db: &DB, + _consumer: Self::Consumer, + _listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) { + () + } +} diff --git a/backend/windmill-api/src/triggers/gcp/mod.rs b/backend/windmill-api/src/triggers/gcp/mod.rs index f11ba94883..dd2f2a65df 100644 --- a/backend/windmill-api/src/triggers/gcp/mod.rs +++ b/backend/windmill-api/src/triggers/gcp/mod.rs @@ -2,6 +2,11 @@ mod handler_ee; pub mod handler_oss; + +#[cfg(feature = "private")] +mod listener_ee; +pub mod listener_oss; + #[cfg(feature = "private")] mod mod_ee; #[cfg(feature = "private")] diff --git a/backend/windmill-api/src/triggers/kafka/handler_oss.rs b/backend/windmill-api/src/triggers/kafka/handler_oss.rs index 3922b684ce..bf5537b941 100644 --- a/backend/windmill-api/src/triggers/kafka/handler_oss.rs +++ b/backend/windmill-api/src/triggers/kafka/handler_oss.rs @@ -33,7 +33,7 @@ impl TriggerCrud for KafkaTrigger { const SUPPORTS_TEST_CONNECTION: bool = false; const ROUTE_PREFIX: &'static str = "/kafka_triggers"; const DEPLOYMENT_NAME: &'static str = ""; - const IS_CLOUD_HOSTED: bool = false; + const IS_ALLOWED_ON_CLOUD: bool = false; fn get_deployed_object(path: String) -> DeployedObject { DeployedObject::KafkaTrigger { path } diff --git a/backend/windmill-api/src/triggers/kafka/listener_oss.rs b/backend/windmill-api/src/triggers/kafka/listener_oss.rs new file mode 100644 index 0000000000..c5dca4ab37 --- /dev/null +++ b/backend/windmill-api/src/triggers/kafka/listener_oss.rs @@ -0,0 +1,41 @@ +#[allow(unused)] + +#[cfg(feature = "private")] +pub use super::listener_ee::*; + +#[cfg(not(feature = "private"))] +use { + super::KafkaTrigger, + crate::triggers::{listener::ListeningTrigger, Listener}, + std::sync::Arc, + tokio::sync::RwLock, + windmill_common::{error::Result, jobs::JobTriggerKind, DB}, +}; + +#[cfg(not(feature = "private"))] +#[async_trait::async_trait] +impl Listener for KafkaTrigger { + type Consumer = (); + type Extra = (); + const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Kafka; + + async fn get_consumer( + &self, + _db: &DB, + _listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> Result> { + Ok(None) + } + async fn consume( + &self, + _db: &DB, + _consumer: Self::Consumer, + _listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) { + () + } +} diff --git a/backend/windmill-api/src/triggers/kafka/mod.rs b/backend/windmill-api/src/triggers/kafka/mod.rs index 700b76b056..f09b2283ff 100644 --- a/backend/windmill-api/src/triggers/kafka/mod.rs +++ b/backend/windmill-api/src/triggers/kafka/mod.rs @@ -2,6 +2,10 @@ mod handler_ee; pub mod handler_oss; +#[cfg(feature = "private")] +mod listener_ee; +pub mod listener_oss; + #[cfg(feature = "private")] mod mod_ee; #[cfg(feature = "private")] diff --git a/backend/windmill-api/src/triggers/listener.rs b/backend/windmill-api/src/triggers/listener.rs new file mode 100644 index 0000000000..e6a0110b2a --- /dev/null +++ b/backend/windmill-api/src/triggers/listener.rs @@ -0,0 +1,901 @@ +use std::{collections::HashMap, fmt::Debug, sync::Arc}; + +use crate::{ + capture::insert_capture_payload, + db::ApiAuthed, + triggers::{ + handler::TriggerCrud, + trigger_helpers::{trigger_runnable, TriggerJobArgs}, + Trigger, TriggerErrorHandling, + }, + users::fetch_api_authed, +}; +use async_trait::async_trait; +use itertools::Itertools; +use rand::seq::SliceRandom; +use serde::de::DeserializeOwned; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use sql_builder::SqlBuilder; +use sqlx::{FromRow, Row}; +use tokio::sync::RwLock; +use windmill_common::{ + error::{Error, Result}, + jobs::JobTriggerKind, + triggers::TriggerKind, + utils::report_critical_error, + DB, INSTANCE_NAME, +}; + +#[allow(unused)] +#[async_trait] +pub trait Listener: TriggerCrud + TriggerJobArgs { + type Consumer: Send; + type Extra: Send + Sync; + + //to use in next PR to add job trigger kind to eow + #[allow(unused)] + const JOB_TRIGGER_KIND: JobTriggerKind; + const EXTRA_TRIGGER_AND_WHERE_CLAUSE: &[&'static str] = &[]; + const EXTRA_CAPTURE_AND_WHERE_CLAUSE: &[&'static str] = &[]; + + async fn get_consumer( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + err_message: Arc>>, + killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> Result>; + async fn consume( + &self, + db: &DB, + consumer: Self::Consumer, + listening_trigger: &ListeningTrigger, + err_message: Arc>>, + killpill_rx: tokio::sync::broadcast::Receiver<()>, + ); + async fn fetch_enabled_unlistened_triggers( + &self, + db: &DB, + ) -> Result>> { + let mut fields = vec![ + "workspace_id", + "path", + "script_path", + "is_flow", + "edited_by", + "email", + "edited_at", + "extra_perms", + ]; + + if Self::SUPPORTS_SERVER_STATE { + fields.extend_from_slice(&["enabled", "server_id", "last_server_ping", "error"]); + } + fields.extend_from_slice(&["error_handler_path", "error_handler_args", "retry"]); + fields.extend_from_slice(Self::ADDITIONAL_SELECT_FIELDS); + + let mut sqlb = SqlBuilder::select_from(Self::TABLE_NAME); + + sqlb.fields(&fields).and_where("enabled IS TRUE").and_where( + "(last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')", + ); + + for where_clause in Self::EXTRA_TRIGGER_AND_WHERE_CLAUSE { + sqlb.and_where(where_clause); + } + + let sql = sqlb + .sql() + .map_err(|e| Error::InternalErr(format!("SQL error: {}", e)))?; + + let triggers: Vec> = + sqlx::query_as(&sql).fetch_all(db).await?; + + let triggers = triggers + .into_iter() + .map(|trigger| ListeningTrigger { + path: trigger.base.path, + workspace_id: trigger.base.workspace_id, + is_flow: trigger.base.is_flow, + username: trigger.base.edited_by, + email: trigger.base.email, + script_path: trigger.base.script_path, + trigger_config: trigger.config, + error_handling: Some(trigger.error_handling), + trigger_mode: true, + }) + .collect_vec(); + + Ok(triggers) + } + + async fn fetch_unlistened_captures( + &self, + db: &DB, + ) -> Result>> { + let fields = vec![ + "path", + "is_flow", + "workspace_id", + "owner AS username", + "email", + "trigger_config", + ]; + + let mut sqlb = SqlBuilder::select_from("capture_config"); + sqlb.fields(&fields) + .and_where(format!("trigger_kind = '{}'", Self::TRIGGER_KIND.to_key())) + .and_where("last_client_ping > NOW() - INTERVAL '10 seconds'") + .and_where("trigger_config IS NOT NULL") + .and_where( + "(last_server_ping IS NULL OR last_server_ping < NOW() - INTERVAL '15 seconds')", + ); + + for where_clause in Self::EXTRA_CAPTURE_AND_WHERE_CLAUSE { + sqlb.and_where(where_clause); + } + + let sql = sqlb.sql().expect("failed to build SQL"); + + let captures: Vec> = + sqlx::query_as(&sql).fetch_all(db).await?; + + let captures = captures + .into_iter() + .map(|capture| ListeningTrigger { + username: capture.username, + path: capture.path, + workspace_id: capture.workspace_id, + script_path: "".to_string(), + email: capture.email, + trigger_config: capture.trigger_config, + trigger_mode: false, + is_flow: capture.is_flow, + error_handling: None, + }) + .collect_vec(); + + Ok(captures) + } + + async fn cleanup( + &self, + _db: &DB, + _listening_trigger: &ListeningTrigger, + ) -> Result<()> { + Ok(()) + } + + async fn loop_ping( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + status: Arc>>, + error_message: Option, + ) { + update_rw_lock(status.clone(), error_message).await; + loop { + if let None = self + .update_ping(db, listening_trigger, status.read().await.as_deref()) + .await + { + return; + } + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + } + } + + async fn update_ping( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + error: Option<&str>, + ) -> Option<()> { + if listening_trigger.trigger_mode { + self.update_trigger_ping(db, listening_trigger, error).await + } else { + self.update_capture_ping(db, listening_trigger, error).await + } + } + + async fn update_ping_and_loop_ping_status( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + loop_ping_status: Arc>>, + error: Option, + ) -> Option<()> { + // update immediately the ping status and update the loop ping status so that the next loop pings will display the new status + update_rw_lock(loop_ping_status.clone(), error.clone()).await; + if let None = self + .update_ping(db, listening_trigger, error.as_deref()) + .await + { + return None; + } + Some(()) + } + + async fn update_trigger_ping( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + error: Option<&str>, + ) -> Option<()> { + let updated = sqlx::query_scalar::<_, i32>(&format!( + r#" + UPDATE + {} + SET + last_server_ping = now(), error = $1 + WHERE + workspace_id = $2 AND + path = $3 AND + server_id = $4 AND + enabled IS TRUE + RETURNING 1 + "#, + Self::TABLE_NAME + )) + .bind(error) + .bind(&listening_trigger.workspace_id) + .bind(&listening_trigger.path) + .bind(&*INSTANCE_NAME) + .fetch_optional(db) + .await; + + self.handle_ping_result(updated, db, listening_trigger, "trigger") + .await + } + + async fn update_capture_ping( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + error: Option<&str>, + ) -> Option<()> { + let updated = sqlx::query_scalar!( + r#" + UPDATE + capture_config + SET + last_server_ping = now(), error = $1 + WHERE + workspace_id = $2 AND + path = $3 AND + is_flow = $4 AND + trigger_kind = $5 AND + server_id = $6 AND + last_client_ping > NOW() - INTERVAL '10 seconds' + RETURNING 1 + "#, + error, + &listening_trigger.workspace_id, + &listening_trigger.path, + &listening_trigger.is_flow, + Self::TRIGGER_KIND as TriggerKind, + &*INSTANCE_NAME + ) + .fetch_optional(db) + .await + .map(|result| result.flatten()); + + self.handle_ping_result(updated, db, listening_trigger, "capture") + .await + } + + async fn handle_ping_result( + &self, + result: sqlx::Result>, + db: &DB, + listening_trigger: &ListeningTrigger, + entity_type: &str, + ) -> Option<()> { + match result { + Ok(updated) => { + if updated.is_none() { + self.reset_ping_for_restart(db, listening_trigger).await; + tracing::info!( + "{} {} {} changed, disabled, or deleted, stopping...", + Self::TRIGGER_KIND, + entity_type, + listening_trigger.path + ); + return None; + } + } + Err(error) => { + tracing::warn!( + "Error updating ping of {} {} {}: {:?}", + Self::TRIGGER_KIND, + entity_type, + &listening_trigger.path, + error + ); + } + } + + Some(()) + } + + async fn reset_ping_for_restart( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + ) { + if listening_trigger.trigger_mode { + let _ = sqlx::query(&format!( + r#" + UPDATE + {} + SET + last_server_ping = NULL + WHERE + workspace_id = $1 AND + path = $2 AND + server_id IS NULL + "#, + Self::TABLE_NAME + )) + .bind(&listening_trigger.workspace_id) + .bind(&listening_trigger.path) + .execute(db) + .await; + } else { + let _ = sqlx::query!( + r#" + UPDATE + capture_config + SET + last_server_ping = NULL + WHERE + workspace_id = $1 AND + path = $2 AND + is_flow = $3 AND + trigger_kind = $4 AND + server_id IS NULL + "#, + &listening_trigger.workspace_id, + &listening_trigger.path, + &listening_trigger.is_flow, + Self::TRIGGER_KIND as TriggerKind + ) + .execute(db) + .await; + } + } + + async fn disable_with_error( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + error: String, + ) { + if listening_trigger.trigger_mode { + let report_status = sqlx::query(&format!( + r#" + UPDATE + {} + SET + enabled = FALSE, + error = $1, + server_id = NULL, + last_server_ping = NULL + WHERE + workspace_id = $2 AND + path = $3 + "#, + Self::TABLE_NAME + )) + .bind(&error) + .bind(&listening_trigger.workspace_id) + .bind(&listening_trigger.path) + .execute(db) + .await; + + match report_status { + Ok(_) => { + report_critical_error( + format!( + "Disabling {} trigger {} because of error: {}", + Self::TRIGGER_KIND, + listening_trigger.path, + error + ), + db.clone(), + Some(&listening_trigger.workspace_id), + None, + ) + .await; + } + Err(disable_err) => { + report_critical_error( + format!("Could not disable {} trigger {} with err {}, disabling because of error {}", Self::TRIGGER_KIND, listening_trigger.path, disable_err, error), + db.clone(), + Some(&listening_trigger.workspace_id), + None, + ).await; + } + } + return; + } + + let report_status = sqlx::query!( + r#" + UPDATE + capture_config + SET + error = $1, + server_id = NULL, + last_server_ping = NULL + WHERE + workspace_id = $2 AND + path = $3 AND + is_flow = $4 AND + trigger_kind = $5 + "#, + error, + listening_trigger.workspace_id, + listening_trigger.path, + listening_trigger.is_flow, + Self::TRIGGER_KIND as TriggerKind + ) + .execute(db) + .await; + + if let Err(disable_err) = report_status { + tracing::error!( + "Could not disable {} capture {} ({}) with err {}, disabling because of error {}", + Self::TRIGGER_KIND, + listening_trigger.path, + listening_trigger.workspace_id, + disable_err, + error + ) + } + } + + async fn handle_trigger( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + payload: Self::Payload, + trigger_info: HashMap>, + _extra: Option, + ) -> Result<()> { + let args = Self::build_job_args( + &listening_trigger.script_path, + listening_trigger.is_flow, + &listening_trigger.workspace_id, + db, + payload, + trigger_info, + ) + .await?; + + let authed = listening_trigger + .authed(db, &Self::TRIGGER_KIND.to_string()) + .await?; + + let (retry, error_handler_path, error_handler_args) = + match listening_trigger.error_handling.as_ref() { + Some(error_handling) => ( + error_handling.retry.as_ref(), + error_handling.error_handler_path.as_deref(), + error_handling.error_handler_args.as_ref(), + ), + None => (None, None, None), + }; + + trigger_runnable( + db, + None, + authed, + &listening_trigger.workspace_id, + &listening_trigger.script_path, + listening_trigger.is_flow, + args, + retry, + error_handler_path.as_deref(), + error_handler_args, + format!("{}_trigger/{}", Self::TRIGGER_KIND, listening_trigger.path), + ) + .await?; + + Ok(()) + } + + async fn handle_event( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + payload: Self::Payload, + trigger_info: HashMap>, + extra: Option, + ) -> Result<()> { + if listening_trigger.trigger_mode { + if let Err(err) = self + .handle_trigger(db, listening_trigger, payload, trigger_info, extra) + .await + { + report_critical_error( + format!( + "Failed to trigger job from {} event {}: {:?}", + Self::TRIGGER_KIND, + listening_trigger.path, + err + ), + db.clone(), + Some(&listening_trigger.workspace_id), + None, + ) + .await; + return Err(err); + }; + return Ok(()); + } + + let (main_args, preprocessor_args) = Self::build_capture_payloads(&payload, trigger_info); + if let Err(err) = insert_capture_payload( + db, + &listening_trigger.workspace_id, + &listening_trigger.path, + listening_trigger.is_flow, + &Self::TRIGGER_KIND, + main_args, + preprocessor_args, + &listening_trigger.username, + ) + .await + { + tracing::error!("Error inserting capture payload: {:?}", err); + return Err(err); + } + Ok(()) + } +} + +#[allow(unused)] +async fn listening( + db: DB, + listener: T, + listening_trigger: ListeningTrigger, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, +) { + let killpill_rx_consumer = killpill_rx.resubscribe(); + let killpill_rx_get_consumer = killpill_rx.resubscribe(); + + let loop_ping_status = Arc::new(RwLock::new(None)); + + tokio::select! { + biased; + _ = killpill_rx.recv() => { + let _ = listener.cleanup(&db, &listening_trigger).await; + } + _ = listener.loop_ping(&db, &listening_trigger, loop_ping_status.clone(), Some("Connecting...".to_string())) => { + let _ = listener.cleanup(&db, &listening_trigger).await; + } + consumer = { + listener.get_consumer(&db, &listening_trigger, loop_ping_status.clone(), killpill_rx_get_consumer) + } => { + tokio::select! { + biased; + _ = killpill_rx.recv() => { + let _ = listener.cleanup(&db, &listening_trigger).await; + return; + } + _ = listener.loop_ping(&db, &listening_trigger, loop_ping_status.clone(), None) => { + let _ = listener.cleanup(&db, &listening_trigger).await; + return; + } + _ = async { + match consumer { + Ok(Some(consumer)) => { + listener.update_ping_and_loop_ping_status(&db, &listening_trigger, loop_ping_status.clone(), None).await; + let _ = listener.consume(&db, consumer, &listening_trigger, loop_ping_status.clone(), killpill_rx_consumer).await; + tracing::debug!("Stopping consumer for trigger"); + } + Err(error) => { + tracing::warn!("Disabling trigger due to consumer error: {}", error); + listener.disable_with_error(&db, &listening_trigger, error.to_string()).await; + } + _ => {} + } + } => { + let _ = listener.cleanup(&db, &listening_trigger).await; + return; + } + } + } + } +} + +#[allow(unused)] +async fn listen_to_unlistened_events( + listener: T, + db: DB, + killpill_rx: &tokio::sync::broadcast::Receiver<()>, +) { + let unlistend_enabled_triggers = listener.fetch_enabled_unlistened_triggers(&db).await; + + match unlistend_enabled_triggers { + Ok(mut unlistend_enabled_triggers) => { + unlistend_enabled_triggers.shuffle(&mut rand::rng()); + for trigger in unlistend_enabled_triggers { + let has_lock = sqlx::query_scalar(&format!( + r#" + UPDATE + {} + SET + server_id = $1, + last_server_ping = now(), + error = 'Connecting...' + WHERE + enabled IS TRUE + AND workspace_id = $2 + AND path = $3 + AND (last_server_ping IS NULL + OR last_server_ping < now() - INTERVAL '15 seconds' + ) + RETURNING true + "#, + T::TABLE_NAME, + )) + .bind(&*INSTANCE_NAME) + .bind(&trigger.workspace_id) + .bind(&trigger.path) + .fetch_optional(&db) + .await; + match has_lock { + Ok(has_lock) => { + if has_lock.flatten().unwrap_or(false) { + tracing::info!( + "Spawning new task to listen for {} event", + T::TABLE_NAME + ); + tokio::spawn({ + let db = db.clone(); + let killpill_rx = killpill_rx.resubscribe(); + async move { listening(db, listener, trigger, killpill_rx).await } + }); + } else { + tracing::info!( + "{} trigger {} already being listened to", + T::TRIGGER_KIND, + trigger.path + ); + } + } + Err(err) => { + tracing::error!( + "Error acquiring lock for {} trigger {}: {:?}", + T::TRIGGER_KIND, + trigger.path, + err + ); + } + }; + } + } + Err(err) => { + tracing::error!("Error fetching {} triggers: {:?}", T::TRIGGER_KIND, err,); + } + } + + let unlisted_captures = listener.fetch_unlistened_captures(&db).await; + + match unlisted_captures { + Ok(unlistened_captures) => { + for capture in unlistened_captures { + let has_lock = sqlx::query_scalar!( + r#" + UPDATE + capture_config + SET + server_id = $1, + last_server_ping = now(), + error = 'Connecting...' + WHERE + last_client_ping > NOW() - INTERVAL '10 seconds' AND + workspace_id = $2 AND + path = $3 AND + is_flow = $4 AND + trigger_kind = $5 AND + (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') + RETURNING true + "#, + *INSTANCE_NAME, + &capture.workspace_id, + &capture.path, + &capture.is_flow, + T::TRIGGER_KIND as TriggerKind + ) + .fetch_optional(&db) + .await; + match has_lock { + Ok(has_lock) => { + if has_lock.flatten().unwrap_or(false) { + tokio::spawn({ + let db = db.clone(); + let killpill_rx = killpill_rx.resubscribe(); + async move { listening(db, listener, capture, killpill_rx).await } + }); + } else { + tracing::info!( + "{} capture {} already being listened to", + T::TRIGGER_KIND.to_string(), + capture.path + ); + } + } + Err(err) => { + tracing::error!( + "Error acquiring lock for capture {} {}: {:?}", + T::TRIGGER_KIND, + capture.path, + err + ); + } + }; + } + } + Err(err) => { + tracing::error!( + "Error fetching captures {} triggers: {:?}", + T::TRIGGER_KIND, + err + ); + } + } +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +struct Capture +where + T: for<'r> FromRow<'r, sqlx::postgres::PgRow>, +{ + path: String, + is_flow: bool, + workspace_id: String, + username: String, + email: String, + #[serde(flatten)] + trigger_config: T, +} + +impl FromRow<'_, sqlx::postgres::PgRow> for Capture +where + T: for<'r> FromRow<'r, sqlx::postgres::PgRow> + DeserializeOwned, +{ + fn from_row(row: &sqlx::postgres::PgRow) -> std::result::Result { + let trigger_config_value = row.try_get("trigger_config")?; + let trigger_config: T = serde_json::from_value(trigger_config_value) + .map_err(|e| sqlx::Error::Decode(Box::new(e)))?; + Ok(Capture { + path: row.try_get("path")?, + is_flow: row.try_get("is_flow")?, + workspace_id: row.try_get("workspace_id")?, + username: row.try_get("username")?, + email: row.try_get("email")?, + trigger_config, + }) + } +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ListeningTrigger { + pub path: String, + pub is_flow: bool, + pub workspace_id: String, + pub username: String, + pub email: String, + pub trigger_config: T, + pub script_path: String, + pub trigger_mode: bool, + pub error_handling: Option, +} + +impl ListeningTrigger { + pub async fn authed(&self, db: &DB, username: &str) -> Result { + fetch_api_authed( + self.username.clone(), + self.email.clone(), + &self.workspace_id, + db, + Some(format!("{}-{}", username, self.path)), + ) + .await + } +} + +#[allow(unused)] +pub async fn update_rw_lock(lock: std::sync::Arc>, value: T) -> () { + let mut w = lock.write().await; + *w = value; +} + +#[allow(unused)] +fn listen_to( + trigger: T, + db: DB, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, +) { + tokio::spawn(async move { + listen_to_unlistened_events(trigger, db.clone(), &killpill_rx).await; + loop { + tokio::select! { + biased; + _ = killpill_rx.recv() => { + return; + } + _ = tokio::time::sleep(tokio::time::Duration::from_secs(15)) => { + listen_to_unlistened_events(trigger, db.clone(), &killpill_rx).await + } + } + } + }); +} + +#[allow(unused)] +pub fn start_all_listeners(db: DB, killpill_rx: &tokio::sync::broadcast::Receiver<()>) { + tracing::info!("Starting trigger listeners based on available features..."); + + #[cfg(feature = "postgres_trigger")] + { + let postgres_killpill_rx = killpill_rx.resubscribe(); + use crate::triggers::postgres::PostgresTrigger; + + listen_to(PostgresTrigger, db.clone(), postgres_killpill_rx) + } + + #[cfg(feature = "mqtt_trigger")] + { + let mqtt_killpill_rx = killpill_rx.resubscribe(); + use crate::triggers::mqtt::MqttTrigger; + + listen_to(MqttTrigger, db.clone(), mqtt_killpill_rx) + } + + #[cfg(feature = "websocket")] + { + let mqtt_killpill_rx = killpill_rx.resubscribe(); + use crate::triggers::websocket::WebsocketTrigger; + + listen_to(WebsocketTrigger, db.clone(), mqtt_killpill_rx) + } + + #[cfg(all(feature = "gcp_trigger", feature = "enterprise", feature = "private"))] + { + let gcp_killpill_rx = killpill_rx.resubscribe(); + use crate::triggers::gcp::GcpTrigger; + + listen_to(GcpTrigger, db.clone(), gcp_killpill_rx); + } + + #[cfg(all(feature = "sqs_trigger", feature = "enterprise", feature = "private"))] + { + let gcp_killpill_rx = killpill_rx.resubscribe(); + use crate::triggers::sqs::SqsTrigger; + + listen_to(SqsTrigger, db.clone(), gcp_killpill_rx); + } + + #[cfg(all(feature = "nats", feature = "enterprise", feature = "private"))] + { + let gcp_killpill_rx = killpill_rx.resubscribe(); + use crate::triggers::nats::NatsTrigger; + + listen_to(NatsTrigger, db.clone(), gcp_killpill_rx); + } + + #[cfg(all(feature = "kafka", feature = "enterprise", feature = "private"))] + { + let gcp_killpill_rx = killpill_rx.resubscribe(); + use crate::triggers::kafka::KafkaTrigger; + + listen_to(KafkaTrigger, db.clone(), gcp_killpill_rx); + } + + tracing::info!("All available trigger listeners have been started"); +} diff --git a/backend/windmill-api/src/triggers/mod.rs b/backend/windmill-api/src/triggers/mod.rs index a5e6dc61f2..9a7f22e3e5 100644 --- a/backend/windmill-api/src/triggers/mod.rs +++ b/backend/windmill-api/src/triggers/mod.rs @@ -23,11 +23,15 @@ pub mod sqs; pub mod websocket; mod handler; +mod listener; pub mod trigger_helpers; #[allow(unused)] pub(crate) use handler::TriggerCrud; pub use handler::{generate_trigger_routers, get_triggers_count_internal, TriggersCount}; +pub use listener::start_all_listeners; +#[allow(unused)] +pub(crate) use listener::Listener; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct StandardTriggerQuery { diff --git a/backend/windmill-api/src/triggers/mqtt/listener.rs b/backend/windmill-api/src/triggers/mqtt/listener.rs new file mode 100644 index 0000000000..34af0b5eb1 --- /dev/null +++ b/backend/windmill-api/src/triggers/mqtt/listener.rs @@ -0,0 +1,374 @@ +use std::{collections::HashMap, sync::Arc}; + +use async_trait::async_trait; +use bytes::Bytes; +use rumqttc::{ + v5::{ + mqttbytes::v5::PublishProperties, Event as V5Event, EventLoop as V5EventLoop, + Incoming as V5Incoming, + }, + Event as V3Event, EventLoop as V3EventLoop, Incoming as V3Incoming, +}; +use std::time::Duration; +use tokio::sync::RwLock; +use windmill_common::{ + db::UserDB, + error::{to_anyhow, Error, Result}, + jobs::JobTriggerKind, + worker::to_raw_value, + DB, +}; + +use crate::{ + resources::try_get_resource_from_db_as, + triggers::{ + listener::ListeningTrigger, + mqtt::{ + MqttClientBuilder, MqttClientResult, MqttConfig, MqttResource, MqttTrigger, + V3MqttHandler, V5MqttHandler, + }, + trigger_helpers::TriggerJobArgs, + Listener, + }, +}; + +#[async_trait] +impl Listener for MqttTrigger { + type Consumer = MqttClientResult; + type Extra = (); + const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Mqtt; + + async fn get_consumer( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> Result> { + let ListeningTrigger:: { workspace_id, trigger_config, .. } = + listening_trigger; + + let MqttConfig { + mqtt_resource_path, + subscribe_topics, + v3_config, + v5_config, + client_id, + client_version, + .. + } = trigger_config; + + let authed = listening_trigger + .authed(db, &Self::TRIGGER_KIND.to_string()) + .await?; + + let mqtt_resource = try_get_resource_from_db_as::( + &authed, + Some(UserDB::new(db.clone())), + &db, + mqtt_resource_path, + workspace_id, + ) + .await?; + + let subscribe_topics = subscribe_topics + .iter() + .map(|topic| topic.0.clone()) + .collect(); + + let client_builder = MqttClientBuilder::new( + mqtt_resource, + client_id.as_deref(), + subscribe_topics, + v3_config.as_ref().map(|c| &c.0), + v5_config.as_ref().map(|c| &c.0), + client_version.as_ref(), + ); + + let client_result = client_builder + .build_client() + .await + .map_err(|e| Error::BadConfig(format!("Failed to build MQTT client: {}", e)))?; + + Ok(Some(client_result)) + } + + async fn consume( + &self, + db: &DB, + consumer: Self::Consumer, + listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) { + tracing::info!( + "Starting to listen for MQTT trigger {}", + &listening_trigger.path + ); + + match consumer { + MqttClientResult::V3((v3_handler, event_loop)) => { + handle_event(&db, self, listening_trigger, v3_handler, event_loop).await + } + MqttClientResult::V5((v5_handler, event_loop)) => { + handle_event(&db, self, listening_trigger, v5_handler, event_loop).await + } + } + } +} + +const TIMEOUT_DURATION: u64 = 10; +const CONNECTION_TIMEOUT: Duration = Duration::from_secs(TIMEOUT_DURATION); + +fn convert_disconnect_packet_into_string( + disconnect: rumqttc::v5::mqttbytes::v5::Disconnect, +) -> String { + let err_message = disconnect + .properties + .map(|properties| properties.reason_string) + .flatten(); + let reason_code = disconnect.reason_code as u8; + format!( + "Disconnected by the broker, reason code: {}, {}", + reason_code, + err_message + .map(|err| format!("message: {}", err)) + .unwrap_or("".to_string()) + ) +} + +#[async_trait] +pub trait EventLoop { + type Event; + type Error; + + async fn poll(&mut self) -> Result; + async fn verify_connection(&mut self) -> Result<()>; +} + +#[async_trait] +impl EventLoop for V5EventLoop { + type Event = V5Event; + type Error = rumqttc::v5::ConnectionError; + + async fn poll(&mut self) -> Result { + self.poll().await.map_err(|err| to_anyhow(err).into()) + } + + async fn verify_connection(&mut self) -> Result<()> { + let start = std::time::Instant::now(); + + while start.elapsed() < CONNECTION_TIMEOUT { + match self.poll().await.map_err(to_anyhow)? { + Self::Event::Incoming(V5Incoming::ConnAck(_)) => return Ok(()), + Self::Event::Incoming(V5Incoming::Disconnect(disconnect)) => { + return Err(Error::BadConfig(convert_disconnect_packet_into_string( + disconnect, + ))); + } + _ => continue, + } + } + + Err(Error::BadConfig(format!( + "Timeout occurred while trying to connect to mqtt broker after {} seconds", + TIMEOUT_DURATION + ))) + } +} + +#[async_trait] +impl EventLoop for V3EventLoop { + type Event = V3Event; + type Error = rumqttc::ConnectionError; + + async fn poll(&mut self) -> Result { + self.poll().await.map_err(|err| to_anyhow(err).into()) + } + + async fn verify_connection(&mut self) -> Result<()> { + let start = std::time::Instant::now(); + + while start.elapsed() < CONNECTION_TIMEOUT { + match self.poll().await.map_err(to_anyhow)? { + Self::Event::Incoming(rumqttc::Packet::ConnAck(_)) => return Ok(()), + _ => continue, + } + } + + Err(Error::BadConfig(format!( + "Timeout occurred while trying to connect to mqtt broker after {} seconds", + TIMEOUT_DURATION + ))) + } +} + +async fn handle_event( + db: &DB, + listener: &T, + listening_trigger: &ListeningTrigger, + handler: H, + mut event_loop: E, +) -> () +where + T: Listener, + H: MqttEvent, + E: EventLoop, + E::Error: ToString, + ::Payload: From, +{ + loop { + let event = event_loop.poll().await; + + match event { + Ok(event) => { + let publish_data = handler.handle_event(event); + if let Ok(Some((payload, publish_data))) = publish_data { + let trigger_info = HashMap::from([ + ("topic".to_string(), to_raw_value(&publish_data.topic)), + ("retain".to_string(), to_raw_value(&publish_data.retain)), + ("pkid".to_string(), to_raw_value(&publish_data.pkid)), + ("qos".to_string(), to_raw_value(&publish_data.qos)), + ( + "v5".to_string(), + to_raw_value(&publish_data.v5.map(|properties| { + serde_json::json!({ + "payload_format_indicator": properties.payload_format_indicator, + "topic_alias": properties.topic_alias, + "response_topic": properties.response_topic, + "correlation_data": properties.correlation_data.as_deref(), + "user_properties": properties.user_properties, + "subscription_identifiers": properties.subscription_identifiers, + "content_type": properties.content_type, + }) + })), + ), + ]); + let _ = listener + .handle_event(db, listening_trigger, payload.into(), trigger_info, None) + .await; + } + } + Err(err) => { + let error = err.to_string(); + tracing::debug!("Error: {}", &err); + listener + .disable_with_error(db, listening_trigger, error) + .await; + return; + } + } + } +} + +#[derive(Clone)] +#[allow(unused)] +pub struct PublishData { + topic: String, + retain: bool, + pkid: u16, + v5: Option, + qos: u8, +} + +impl PublishData { + pub fn new( + topic: String, + retain: bool, + pkid: u16, + v5: Option, + qos: u8, + ) -> PublishData { + PublishData { topic, retain, pkid, v5, qos } + } +} + +trait MqttEvent { + type IncomingPacket; + type PublishPacket; + type Event; + + fn handle_publish_packet(publish_packet: Self::PublishPacket) -> PublishData; + fn handle_event(&self, event: Self::Event) -> Result>; +} + +impl MqttEvent for V5MqttHandler { + type IncomingPacket = V5Incoming; + type PublishPacket = rumqttc::v5::mqttbytes::v5::Publish; + type Event = V5Event; + + fn handle_publish_packet(publish_packet: Self::PublishPacket) -> PublishData { + PublishData::new( + String::from_utf8(publish_packet.topic.as_ref().to_vec()).unwrap_or("".to_string()), + publish_packet.retain, + publish_packet.pkid, + publish_packet.properties, + publish_packet.qos as u8, + ) + } + + fn handle_event(&self, event: Self::Event) -> Result> { + tracing::debug!("Inside V5 event"); + match event { + Self::Event::Incoming(packet) => match packet { + Self::IncomingPacket::Publish(publish_packet) => { + return Ok(Some(( + publish_packet.payload.clone(), + Self::handle_publish_packet(publish_packet), + ))) + } + Self::IncomingPacket::Disconnect(disconnect) => { + return Err( + anyhow::anyhow!(convert_disconnect_packet_into_string(disconnect)).into(), + ); + } + packet => { + tracing::debug!("Received = {:#?}", packet); + } + }, + Self::Event::Outgoing(packet) => { + tracing::debug!("Outgoing Received = {:#?}", packet); + } + } + + Ok(None) + } +} + +impl MqttEvent for V3MqttHandler { + type IncomingPacket = V3Incoming; + type PublishPacket = rumqttc::mqttbytes::v4::Publish; + type Event = V3Event; + + fn handle_publish_packet(publish_packet: Self::PublishPacket) -> PublishData { + PublishData::new( + publish_packet.topic, + publish_packet.retain, + publish_packet.pkid, + None, + publish_packet.qos as u8, + ) + } + + fn handle_event(&self, event: Self::Event) -> Result> { + tracing::debug!("Inside V3 event"); + match event { + Self::Event::Incoming(packet) => match packet { + Self::IncomingPacket::Publish(publish_packet) => { + return Ok(Some(( + publish_packet.payload.clone(), + Self::handle_publish_packet(publish_packet), + ))) + } + packet => { + tracing::debug!("Received = {:?}", packet); + } + }, + Self::Event::Outgoing(packet) => { + tracing::debug!("Outgoing Received = {:?}", packet); + } + } + + Ok(None) + } +} diff --git a/backend/windmill-api/src/triggers/mqtt/mod.rs b/backend/windmill-api/src/triggers/mqtt/mod.rs index 1ac287b82a..cfc8e36900 100644 --- a/backend/windmill-api/src/triggers/mqtt/mod.rs +++ b/backend/windmill-api/src/triggers/mqtt/mod.rs @@ -11,7 +11,6 @@ use rumqttc::{ AsyncClient as V3AsyncClient, EventLoop as V3EventLoop, MqttOptions as V3MqttOptions, QoS as V3QoS, SubscribeFilter, TlsConfiguration, Transport, }; -use crate::mqtt_triggers::EventLoop; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use sqlx::{types::Json as SqlxJson, FromRow, Type}; @@ -22,9 +21,10 @@ use windmill_common::{ worker::to_raw_value, }; -use crate::triggers::{ trigger_helpers::TriggerJobArgs}; +use crate::triggers::{mqtt::listener::EventLoop, trigger_helpers::TriggerJobArgs}; pub mod handler; +pub mod listener; #[derive(Clone, Copy)] pub struct MqttTrigger; diff --git a/backend/windmill-api/src/triggers/nats/handler_oss.rs b/backend/windmill-api/src/triggers/nats/handler_oss.rs index 6b03dafef0..41f060003f 100644 --- a/backend/windmill-api/src/triggers/nats/handler_oss.rs +++ b/backend/windmill-api/src/triggers/nats/handler_oss.rs @@ -30,7 +30,7 @@ impl TriggerCrud for NatsTrigger { const SUPPORTS_TEST_CONNECTION: bool = false; const ROUTE_PREFIX: &'static str = "/nats_triggers"; const DEPLOYMENT_NAME: &'static str = ""; - const IS_CLOUD_HOSTED: bool = false; + const IS_ALLOWED_ON_CLOUD: bool = false; fn get_deployed_object(path: String) -> DeployedObject { DeployedObject::NatsTrigger { path } diff --git a/backend/windmill-api/src/triggers/nats/listener_oss.rs b/backend/windmill-api/src/triggers/nats/listener_oss.rs new file mode 100644 index 0000000000..80f1befded --- /dev/null +++ b/backend/windmill-api/src/triggers/nats/listener_oss.rs @@ -0,0 +1,41 @@ +#[allow(unused)] + +#[cfg(feature = "private")] +pub use super::listener_ee::*; + +#[cfg(not(feature = "private"))] +use { + super::NatsTrigger, + crate::triggers::{listener::ListeningTrigger, Listener}, + std::sync::Arc, + tokio::sync::RwLock, + windmill_common::{error::Result, jobs::JobTriggerKind, DB}, +}; + +#[cfg(not(feature = "private"))] +#[async_trait::async_trait] +impl Listener for NatsTrigger { + type Consumer = (); + type Extra = (); + const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Nats; + + async fn get_consumer( + &self, + _db: &DB, + _listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> Result> { + Ok(None) + } + async fn consume( + &self, + _db: &DB, + _consumer: Self::Consumer, + _listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) { + () + } +} diff --git a/backend/windmill-api/src/triggers/nats/mod.rs b/backend/windmill-api/src/triggers/nats/mod.rs index b69f2eb3c8..668df073c9 100644 --- a/backend/windmill-api/src/triggers/nats/mod.rs +++ b/backend/windmill-api/src/triggers/nats/mod.rs @@ -2,6 +2,10 @@ mod handler_ee; pub mod handler_oss; +#[cfg(feature = "private")] +mod listener_ee; +pub mod listener_oss; + #[cfg(feature = "private")] mod mod_ee; #[cfg(feature = "private")] diff --git a/backend/windmill-api/src/postgres_triggers/bool.rs b/backend/windmill-api/src/triggers/postgres/bool.rs similarity index 94% rename from backend/windmill-api/src/postgres_triggers/bool.rs rename to backend/windmill-api/src/triggers/postgres/bool.rs index 9c415780a5..13b0a8fa68 100644 --- a/backend/windmill-api/src/postgres_triggers/bool.rs +++ b/backend/windmill-api/src/triggers/postgres/bool.rs @@ -3,10 +3,10 @@ use thiserror::Error; /** * This implementation is inspired by Postgres replication functionality * from https://github.com/supabase/pg_replicate -* -* Original implementation: +* +* Original implementation: * - https://github.dev/supabase/pg_replicate/blob/main/pg_replicate/src/conversions/bool.rs -* +* */ #[derive(Debug, Error)] diff --git a/backend/windmill-api/src/postgres_triggers/converter.rs b/backend/windmill-api/src/triggers/postgres/converter.rs similarity index 100% rename from backend/windmill-api/src/postgres_triggers/converter.rs rename to backend/windmill-api/src/triggers/postgres/converter.rs diff --git a/backend/windmill-api/src/postgres_triggers/hex.rs b/backend/windmill-api/src/triggers/postgres/hex.rs similarity index 100% rename from backend/windmill-api/src/postgres_triggers/hex.rs rename to backend/windmill-api/src/triggers/postgres/hex.rs diff --git a/backend/windmill-api/src/triggers/postgres/listener.rs b/backend/windmill-api/src/triggers/postgres/listener.rs new file mode 100644 index 0000000000..7bdfab0ed2 --- /dev/null +++ b/backend/windmill-api/src/triggers/postgres/listener.rs @@ -0,0 +1,438 @@ +use std::{collections::HashMap, pin::Pin, sync::Arc}; + +use bytes::{BufMut, Bytes, BytesMut}; +use chrono::TimeZone; +use futures::{pin_mut, SinkExt}; +use pg_escape::{quote_identifier, quote_literal}; +use rust_postgres::{Client, CopyBothDuplex, SimpleQueryMessage}; +use tokio::sync::RwLock; +use tokio_stream::StreamExt; +use windmill_common::{ + db::UserDB, + error::{to_anyhow, Error, Result}, + jobs::JobTriggerKind, + worker::to_raw_value, + DB, +}; + +use crate::{ + resources::try_get_resource_from_db_as, + triggers::{ + listener::ListeningTrigger, + postgres::{ + drop_publication, get_default_pg_connection, get_raw_postgres_connection, + handler::drop_logical_replication_slot, + relation::RelationConverter, + replication_message::{ + LogicalReplicationMessage::{ + Begin, Commit, Delete, Insert, Relation, Type, Update, + }, + PrimaryKeepAliveBody, ReplicationMessage, + }, + Postgres, PostgresConfig, PostgresTrigger, ERROR_PUBLICATION_NAME_NOT_EXISTS, + }, + trigger_helpers::TriggerJobArgs, + Listener, + }, +}; + +const ERROR_REPLICATION_SLOT_NOT_EXISTS: &str = r#"The replication slot associated with this trigger no longer exists. Recreate a new replication slot or select an existing one in the advanced tab, or delete and recreate a new trigger"#; + +pub struct LogicalReplicationSettings { + pub streaming: bool, +} + +impl LogicalReplicationSettings { + pub fn new(streaming: bool) -> Self { + Self { streaming } + } +} + +pub struct PostgresSimpleClient(Client); + +trait RowExist { + fn row_exist(&self) -> bool; +} + +impl RowExist for Vec { + fn row_exist(&self) -> bool { + self.iter() + .find_map(|element| { + if let SimpleQueryMessage::CommandComplete(value) = element { + Some(*value) + } else { + None + } + }) + .is_some_and(|value| value > 0) + } +} + +impl PostgresSimpleClient { + async fn new(database: &Postgres) -> Result { + let client = get_raw_postgres_connection(database, true).await?; + + Ok(PostgresSimpleClient(client)) + } + + async fn execute_query( + &self, + query: &str, + ) -> std::result::Result, rust_postgres::Error> { + self.0.simple_query(query).await + } + + async fn get_logical_replication_stream( + &self, + publication_name: &str, + logical_replication_slot_name: &str, + ) -> Result<(CopyBothDuplex, LogicalReplicationSettings)> { + let options = format!( + r#"("proto_version" '2', "publication_names" {})"#, + quote_literal(publication_name), + ); + + let query = format!( + r#"START_REPLICATION SLOT {} LOGICAL 0/0 {}"#, + quote_identifier(logical_replication_slot_name), + options + ); + + Ok(( + self.0 + .copy_both_simple::(query.as_str()) + .await + .map_err(to_anyhow)?, + LogicalReplicationSettings::new(false), + )) + } + + async fn send_status_update( + primary_keep_alive: PrimaryKeepAliveBody, + copy_both_stream: &mut Pin<&mut CopyBothDuplex>, + ) { + let mut buf = BytesMut::new(); + let ts = chrono::Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); + let ts = chrono::Utc::now() + .signed_duration_since(ts) + .num_microseconds() + .unwrap_or(0); + + buf.put_u8(b'r'); + buf.put_u64(primary_keep_alive.wal_end); + buf.put_u64(primary_keep_alive.wal_end); + buf.put_u64(primary_keep_alive.wal_end); + buf.put_i64(ts); + buf.put_u8(0); + copy_both_stream.send(buf.freeze()).await.unwrap(); + tracing::debug!("Send update status message"); + } +} + +#[async_trait::async_trait] +impl Listener for PostgresTrigger { + type Consumer = (CopyBothDuplex, LogicalReplicationSettings); + type Extra = (); + const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Postgres; + + async fn get_consumer( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> Result> { + let ListeningTrigger:: { workspace_id, trigger_config, .. } = + listening_trigger; + + let PostgresConfig { + postgres_resource_path, publication_name, replication_slot_name, .. + } = trigger_config; + + let authed = listening_trigger + .authed(db, &Self::TRIGGER_KIND.to_string()) + .await?; + + let database = try_get_resource_from_db_as::( + &authed, + Some(UserDB::new(db.clone())), + &db, + postgres_resource_path, + workspace_id, + ) + .await?; + + let client = PostgresSimpleClient::new(&database).await?; + + let publication = client + .execute_query(&format!( + "SELECT pubname FROM pg_publication WHERE pubname = {}", + quote_literal(&publication_name) + )) + .await + .map_err(to_anyhow)?; + + if !publication.row_exist() { + return Err(Error::BadConfig( + ERROR_PUBLICATION_NAME_NOT_EXISTS.to_string(), + )); + } + + let replication_slot = client + .execute_query(&format!( + "SELECT slot_name FROM pg_replication_slots WHERE slot_name = {}", + quote_literal(&replication_slot_name) + )) + .await + .map_err(to_anyhow)?; + + if !replication_slot.row_exist() { + return Err(Error::BadConfig( + ERROR_REPLICATION_SLOT_NOT_EXISTS.to_string(), + )); + } + + let (logical_replication_stream, logical_replication_settings) = client + .get_logical_replication_stream(&publication_name, &replication_slot_name) + .await + .map_err(to_anyhow)?; + + Ok(Some(( + logical_replication_stream, + logical_replication_settings, + ))) + } + async fn consume( + &self, + db: &DB, + consumer: Self::Consumer, + listening_trigger: &ListeningTrigger, + err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) { + let (logical_replication_stream, logical_replication_settings) = consumer; + pin_mut!(logical_replication_stream); + let mut relations = RelationConverter::new(); + tracing::info!( + "Starting to listen for postgres trigger {}", + &listening_trigger.path + ); + loop { + let message = logical_replication_stream.next().await; + let message = match message { + Some(message) => message, + None => { + tracing::error!( + "Stream for postgres trigger {} closed", + &listening_trigger.path + ); + if let None = self + .update_ping_and_loop_ping_status( + db, + listening_trigger, + err_message.clone(), + Some("Stream closed".to_string()), + ) + .await + { + return; + } + return; + } + }; + + let message = match message { + Ok(message) => message, + Err(err) => { + let err = format!( + "Postgres trigger named {} had an error while receiving a message : {}", + &listening_trigger.path, + err.to_string() + ); + self.disable_with_error(db, listening_trigger, err).await; + return; + } + }; + + let logical_message = match ReplicationMessage::parse(message) { + Ok(logical_message) => logical_message, + Err(err) => { + let err = format!( + "Postgres trigger named: {} had an error while parsing message: {}", + &listening_trigger.path, + err.to_string() + ); + self.disable_with_error(db, listening_trigger, err).await; + return; + } + }; + + match logical_message { + ReplicationMessage::PrimaryKeepAlive(primary_keep_alive) => { + if primary_keep_alive.reply { + PostgresSimpleClient::send_status_update( + primary_keep_alive, + &mut logical_replication_stream, + ) + .await; + } + } + ReplicationMessage::XLogData(x_log_data) => { + let logical_replication_message = match x_log_data + .parse(&logical_replication_settings) + { + Ok(logical_replication_message) => logical_replication_message, + Err(err) => { + tracing::error!("Postgres trigger named: {} had an error while trying to parse incomming stream message: {}", &listening_trigger.path, err.to_string()); + continue; + } + }; + + let json = match logical_replication_message { + Relation(relation_body) => { + relations.add_relation(relation_body); + None + } + Begin | Type | Commit => None, + Insert(insert) => Some(( + insert.o_id, + Ok(None), + relations.row_to_json((insert.o_id, insert.tuple)), + "insert", + )), + Update(update) => { + let old_row = update + .old_tuple + .map(|old_tuple| relations.row_to_json((update.o_id, old_tuple))) + .transpose(); + let row = relations.row_to_json((update.o_id, update.new_tuple)); + Some((update.o_id, old_row, row, "update")) + } + Delete(delete) => { + let row = delete + .old_tuple + .unwrap_or_else(|| delete.key_tuple.unwrap()); + Some(( + delete.o_id, + Ok(None), + relations.row_to_json((delete.o_id, row)), + "delete", + )) + } + }; + match json { + Some((o_id, Ok(old_row), Ok(row), transaction_type)) => { + let relation = match relations.get_relation(o_id) { + Ok(relation) => relation, + Err(err) => { + tracing::error!( + "Postgres trigger named: {}, error: {}", + &listening_trigger.path, + err.to_string() + ); + continue; + } + }; + let database_info = HashMap::from([ + ("schema_name".to_string(), to_raw_value(&relation.namespace)), + ("table_name".to_string(), to_raw_value(&relation.name)), + ( + "transaction_type".to_string(), + to_raw_value(&transaction_type), + ), + ("old_row".to_string(), to_raw_value(&old_row)), + ("row".to_string(), to_raw_value(&row)), + ]); + let _ = self + .handle_event( + db, + listening_trigger, + database_info, + HashMap::new(), + None, + ) + .await; + } + Some((o_id, old_row, row, transaction_type)) => { + let relation = match relations.get_relation(o_id) { + Ok(relation) => relation, + Err(err) => { + tracing::error!( + "Postgres trigger named: {}, error: {}", + &listening_trigger.path, + err.to_string() + ); + continue; + } + }; + + if let Err(err) = old_row { + tracing::error!( + transaction_type = ?transaction_type, + schema = %relation.namespace, + table = %relation.name, + error = %err, + "Failed to decode OLD row for {} transaction on {}.{}", + transaction_type, + relation.namespace, + relation.name, + ); + } + + if let Err(err) = row { + tracing::error!( + transaction_type = ?transaction_type, + schema = %relation.namespace, + table = %relation.name, + error = %err, + "Failed to decode NEW row for {} transaction on {}.{}", + transaction_type, + relation.namespace, + relation.name, + ); + } + } + _ => {} + } + } + } + } + } + + async fn cleanup( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + ) -> Result<()> { + let authed = listening_trigger + .authed(db, &Self::TRIGGER_KIND.to_string()) + .await?; + + let user_db = UserDB::new(db.clone()); + + let mut pg_connection = get_default_pg_connection( + authed, + Some(user_db), + &db, + &listening_trigger.trigger_config.postgres_resource_path, + &listening_trigger.workspace_id, + ) + .await?; + + if listening_trigger.trigger_config.basic_mode.unwrap_or(false) { + drop_logical_replication_slot( + &mut pg_connection, + &listening_trigger.trigger_config.replication_slot_name, + ) + .await?; + + drop_publication( + &mut pg_connection, + &listening_trigger.trigger_config.publication_name, + ) + .await?; + } + + Ok(()) + } +} diff --git a/backend/windmill-api/src/triggers/postgres/mod.rs b/backend/windmill-api/src/triggers/postgres/mod.rs index d94b08f67d..191c57b166 100644 --- a/backend/windmill-api/src/triggers/postgres/mod.rs +++ b/backend/windmill-api/src/triggers/postgres/mod.rs @@ -22,8 +22,14 @@ use windmill_common::{ utils::empty_as_none, }; +mod bool; +mod converter; pub mod handler; +mod hex; +pub mod listener; mod mapper; +mod relation; +mod replication_message; #[derive(Clone, Copy)] pub struct PostgresTrigger; diff --git a/backend/windmill-api/src/postgres_triggers/relation.rs b/backend/windmill-api/src/triggers/postgres/relation.rs similarity index 100% rename from backend/windmill-api/src/postgres_triggers/relation.rs rename to backend/windmill-api/src/triggers/postgres/relation.rs diff --git a/backend/windmill-api/src/postgres_triggers/replication_message.rs b/backend/windmill-api/src/triggers/postgres/replication_message.rs similarity index 99% rename from backend/windmill-api/src/postgres_triggers/replication_message.rs rename to backend/windmill-api/src/triggers/postgres/replication_message.rs index e33f4bbf86..5ae7f8cf2f 100644 --- a/backend/windmill-api/src/postgres_triggers/replication_message.rs +++ b/backend/windmill-api/src/triggers/postgres/replication_message.rs @@ -12,7 +12,7 @@ use bytes::Bytes; use rust_postgres::types::{Oid, Type}; use thiserror::Error; -use super::trigger::LogicalReplicationSettings; +use super::listener::LogicalReplicationSettings; const PRIMARY_KEEPALIVE_BYTE: u8 = b'k'; const X_LOG_DATA_BYTE: u8 = b'w'; diff --git a/backend/windmill-api/src/triggers/sqs/handler_oss.rs b/backend/windmill-api/src/triggers/sqs/handler_oss.rs index 53365def58..6b6d18b72e 100644 --- a/backend/windmill-api/src/triggers/sqs/handler_oss.rs +++ b/backend/windmill-api/src/triggers/sqs/handler_oss.rs @@ -31,7 +31,7 @@ impl TriggerCrud for SqsTrigger { const SUPPORTS_TEST_CONNECTION: bool = false; const ROUTE_PREFIX: &'static str = "/sqs_triggers"; const DEPLOYMENT_NAME: &'static str = ""; - const IS_CLOUD_HOSTED: bool = false; + const IS_ALLOWED_ON_CLOUD: bool = false; fn get_deployed_object(path: String) -> DeployedObject { DeployedObject::SqsTrigger { path } diff --git a/backend/windmill-api/src/triggers/sqs/listener_oss.rs b/backend/windmill-api/src/triggers/sqs/listener_oss.rs new file mode 100644 index 0000000000..0aaf5360c3 --- /dev/null +++ b/backend/windmill-api/src/triggers/sqs/listener_oss.rs @@ -0,0 +1,41 @@ +#[allow(unused)] + +#[cfg(feature = "private")] +pub use super::listener_ee::*; + +#[cfg(not(feature = "private"))] +use { + super::SqsTrigger, + crate::triggers::{listener::ListeningTrigger, Listener}, + std::sync::Arc, + tokio::sync::RwLock, + windmill_common::{error::Result, jobs::JobTriggerKind, DB}, +}; + +#[cfg(not(feature = "private"))] +#[async_trait::async_trait] +impl Listener for SqsTrigger { + type Consumer = (); + type Extra = (); + const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Sqs; + + async fn get_consumer( + &self, + _db: &DB, + _listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> Result> { + Ok(None) + } + async fn consume( + &self, + _db: &DB, + _consumer: Self::Consumer, + _listening_trigger: &ListeningTrigger, + _err_message: Arc>>, + _killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) { + () + } +} diff --git a/backend/windmill-api/src/triggers/sqs/mod.rs b/backend/windmill-api/src/triggers/sqs/mod.rs index 74a93c12a5..8c7217725c 100644 --- a/backend/windmill-api/src/triggers/sqs/mod.rs +++ b/backend/windmill-api/src/triggers/sqs/mod.rs @@ -2,6 +2,10 @@ mod handler_ee; pub mod handler_oss; +#[cfg(feature = "private")] +mod listener_ee; +pub mod listener_oss; + #[cfg(feature = "private")] mod mod_ee; #[cfg(feature = "private")] diff --git a/backend/windmill-api/src/triggers/websocket/listener.rs b/backend/windmill-api/src/triggers/websocket/listener.rs new file mode 100644 index 0000000000..ae8ba01d2e --- /dev/null +++ b/backend/windmill-api/src/triggers/websocket/listener.rs @@ -0,0 +1,571 @@ +use super::WebsocketTrigger; +use crate::triggers::{ + listener::ListeningTrigger, + trigger_helpers::{trigger_runnable, trigger_runnable_and_wait_for_raw_result, TriggerJobArgs}, + websocket::WebsocketConfig, + Listener, +}; +use anyhow::Context; +use async_trait::async_trait; +use futures::{stream::SplitSink, SinkExt, StreamExt}; +use http::Response; +use itertools::Itertools; +use serde::{ + de::{self, MapAccess, Visitor}, + Deserialize, Deserializer, +}; +use serde_json::{value::RawValue, Value}; +use std::{borrow::Cow, collections::HashMap, fmt, sync::Arc}; +use tokio::{net::TcpStream, sync::RwLock}; +use tokio_tungstenite::{connect_async, tungstenite::Message, MaybeTlsStream, WebSocketStream}; +use windmill_common::{ + error::{to_anyhow, Error, Result}, + jobs::JobTriggerKind, + utils::report_critical_error, + worker::to_raw_value, + DB, +}; +use windmill_queue::PushArgsOwned; + +impl ListeningTrigger { + async fn get_url_from_runnable(&self, db: &DB) -> Result { + let runnable_kind = if self.is_flow { "Flow" } else { "Script" }; + tracing::info!( + "Running {} {} to get WebSocket URL", + runnable_kind.to_lowercase(), + self.path + ); + + let authed = self.authed(db, "ws").await?; + + let args = raw_value_to_args_hashmap( + self.trigger_config.url_runnable_args.as_ref().map(|r| &r.0), + )?; + + let result = trigger_runnable_and_wait_for_raw_result( + db, + None, + authed, + &self.workspace_id, + &self.script_path, + self.is_flow, + PushArgsOwned { args, extra: None }, + None, + None, + None, + "".to_string(), // doesn't matter as no retry/error handler + ) + .await?; + + serde_json::from_str::(result.get()).map_err(|_| { + Error::BadConfig(format!( + "{} {} did not return a string", + runnable_kind, self.path, + )) + }) + } + + async fn send_initial_messages( + &self, + writer: &mut SplitSink>, Message>, + db: &DB, + ) -> Result<()> { + let initial_messages: Vec = self + .trigger_config + .initial_messages + .as_deref() + .unwrap_or_default() + .iter() + .filter_map(|m| serde_json::from_str(m.get()).ok()) + .collect_vec(); + + let WebsocketConfig { ref url, .. } = self.trigger_config; + let runnable_kind = if self.is_flow { "flow" } else { "script" }; + let mut authed_o = None; + for start_message in initial_messages { + match start_message { + InitialMessage::RawMessage(msg) => { + let msg = if msg.starts_with("\"") && msg.ends_with("\"") { + msg[1..msg.len() - 1].to_string() + } else { + msg + }; + tracing::info!( + "Sending raw message initial message to WebSocket {}: {}", + url, + msg + ); + writer + .send(tokio_tungstenite::tungstenite::Message::Text(msg)) + .await + .map_err(to_anyhow) + .with_context(|| "failed to send raw message")?; + } + InitialMessage::RunnableResult { path, is_flow, args } => { + tracing::info!( + "Running {} {} for initial message to WebSocket {}", + runnable_kind, + path, + url, + ); + + let args = raw_value_to_args_hashmap(Some(&args))?; + + if authed_o.is_none() { + authed_o = Some(self.authed(db, "ws").await?); + } + let authed = authed_o.clone().unwrap(); + + let result = trigger_runnable_and_wait_for_raw_result( + db, + None, + authed.clone(), + &self.workspace_id, + &path, + is_flow, + PushArgsOwned { args, extra: None }, + None, + None, + None, + "".to_string(), // doesn't matter as no retry/error handler + ) + .await + .map(|r| r.get().to_owned())?; + + tracing::info!( + "Sending {} {} result to WebSocket {}", + runnable_kind, + path, + url + ); + + // if the `result` was just a single string, the below removes the surrounding quotes by parsing it as a string. + // it falls back to the original serialized JSON if it doesn't work. + let result = serde_json::from_str::(result.as_str()).unwrap_or(result); + + writer + .send(tokio_tungstenite::tungstenite::Message::Text(result)) + .await + .map_err(to_anyhow) + .with_context(|| { + format!("Failed to send {} {} result", runnable_kind, path) + })?; + } + } + } + Ok(()) + } +} + +#[async_trait] +impl Listener for WebsocketTrigger { + type Consumer = ( + WebSocketStream>, + Response>>, + ); + type Extra = ReturnMessageChannels; + const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Websocket; + async fn get_consumer( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + err_message: Arc>>, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) -> Result> { + let url = &listening_trigger.trigger_config.url; + let connect_url: Cow = if url.starts_with("$") { + if url.starts_with("$flow:") || url.starts_with("$script:") { + tokio::select! { + biased; + _ = killpill_rx.recv() => { + return Ok(None); + }, + _ = self.loop_ping(&db, listening_trigger, err_message.clone(), Some( + "Waiting on runnable to return WebSocket URL...".to_string() + )) => { + return Ok(None); + }, + + url_result = listening_trigger.get_url_from_runnable(&db) => match url_result { + Ok(url) => Cow::Owned(url), + Err(err) => { + return Err(anyhow::anyhow!("Error getting WebSocket URL from runnable after 5 tries: {:?}", err).into()); + } + }, + } + } else { + return Err(anyhow::anyhow!("Invalid WebSocket runnable path: {}", url).into()); + } + } else { + Cow::Borrowed(&url) + }; + + let connection = connect_async(connect_url.as_ref()) + .await + .map(|conn| Some(conn)) + .map_err(|err| to_anyhow(err).into()); + + connection + } + async fn consume( + &self, + db: &DB, + consumer: Self::Consumer, + listening_trigger: &ListeningTrigger, + err_message: Arc>>, + mut killpill_rx: tokio::sync::broadcast::Receiver<()>, + ) { + let WebsocketConfig { ref url, .. } = listening_trigger.trigger_config; + + tracing::info!("Connected to WebSocket {}", url); + + let (ws_stream, _) = consumer; + + let (mut writer, mut reader) = ws_stream.split(); + + // send initial messages + if listening_trigger.trigger_mode { + tokio::select! { + biased; + _ = killpill_rx.recv() => { + return; + }, + _ = self.loop_ping(db, listening_trigger, err_message.clone(), Some("Sending initial messages...".to_string())) => { + return; + }, + result = listening_trigger.send_initial_messages(&mut writer, &db) => { + if let Err(err) = result { + self.disable_with_error(&db, listening_trigger, format!("Error sending initial messages: {:?}", err)).await; + return + } else { + tracing::debug!("Initial messages sent successfully to WebSocket {}", url); + } + } + } + } + + let (return_message_channels, message_sender_handle) = if listening_trigger.trigger_mode + && listening_trigger.trigger_config.can_return_message + { + let (send_message_tx, mut rx) = tokio::sync::mpsc::channel::(100); + let w_id = listening_trigger.workspace_id.clone(); + let url = url.clone(); + let db = db.clone(); + let handle = tokio::spawn(async move { + while let Some(message) = rx.recv().await { + if let Err(err) = writer + .send(tokio_tungstenite::tungstenite::Message::Text(message)) + .await + { + report_critical_error(format!("Could not send runnable result to WebSocket {} because of error: {}", url, err), db.clone(), Some(&w_id), None).await; + } + } + }); + + let killpill_rx = killpill_rx.resubscribe(); + + let return_message_channels = ReturnMessageChannels { send_message_tx, killpill_rx }; + + (Some(return_message_channels), Some(handle)) + } else { + (None, None) + }; + + tokio::select! { + biased; + _ = killpill_rx.recv() => { + }, + _ = self.loop_ping(db, listening_trigger, err_message.clone(), None) => { + }, + _ = async { + let filters: Vec = if listening_trigger.trigger_mode { + listening_trigger + .trigger_config + .filters + .iter() + .filter_map(|m| serde_json::from_str(m.get()).ok()) + .collect_vec() + } else { + vec![] + }; + loop { + if let Some(msg) = reader.next().await { + match msg { + Ok(msg) => { + match msg { + tokio_tungstenite::tungstenite::Message::Text(text) => { + tracing::debug!("Received text message from WebSocket {}: {}", url, text); + let mut should_handle = true; + for filter in &filters { + match filter { + Filter::JsonFilter(JsonFilter { key, value }) => { + let mut deserializer = serde_json::Deserializer::from_str(text.as_str()); + should_handle = match is_value_superset(&mut deserializer, key, &value) { + Ok(filter_match) => { + filter_match + }, + Err(err) => { + tracing::warn!("Error deserializing filter for WebSocket {}: {:?}", url, err); + false + } + }; + } + } + if !should_handle { + break; + } + } + if should_handle { + let trigger_info = HashMap::from([ + ("url".to_string(), to_raw_value(&listening_trigger.trigger_config.url)), + ]); + let _ = self.handle_event(db, listening_trigger, text, trigger_info, return_message_channels.clone()).await; + } + }, + a @ _ => { + tracing::debug!("Received non text-message from WebSocket {}: {:?}", url, a); + } + } + }, + Err(err) => { + tracing::error!("Error reading from WebSocket {}: {:?}", url, err); + } + } + } else { + tracing::error!("WebSocket {} closed", url); + self.update_ping_and_loop_ping_status(db, listening_trigger, err_message.clone(), Some("WebSocket closed".to_string())).await; + break; + } + } + } => {} + } + // make sure to stop return message handler + if let Some(message_sender_handle) = message_sender_handle { + message_sender_handle.abort(); + } + } + + async fn handle_trigger( + &self, + db: &DB, + listening_trigger: &ListeningTrigger, + payload: Self::Payload, + trigger_info: HashMap>, + extra: Option, + ) -> Result<()> { + let ListeningTrigger { + path, + is_flow, + workspace_id, + trigger_config, + script_path, + error_handling, + .. + } = listening_trigger; + + let WebsocketConfig { url, .. } = trigger_config; + + let args = WebsocketTrigger::build_job_args( + &script_path, + *is_flow, + workspace_id, + db, + payload, + trigger_info, + ) + .await; + + let args = match args { + Ok(args) => args, + Err(err) => { + return Err(err); + } + }; + + let authed = listening_trigger.authed(db, "ws").await?; + + let (retry, error_handler_path, error_handler_args) = match error_handling.as_ref() { + Some(error_handling) => ( + error_handling.retry.as_ref(), + error_handling.error_handler_path.as_deref(), + error_handling.error_handler_args.as_ref(), + ), + None => (None, None, None), + }; + if let Some(ReturnMessageChannels { send_message_tx, mut killpill_rx }) = extra { + let db_ = db.clone(); + let url = url.to_owned(); + let script_path = script_path.to_owned(); + let is_flow = *is_flow; + let w_id = workspace_id.to_owned(); + let retry = retry.cloned(); + let error_handler_path = error_handler_path.map(|s| s.to_string()); + let error_handler_args = error_handler_args.cloned(); + let trigger_path = path.clone(); + let handle_response_f = async move { + tokio::select! { + _ = killpill_rx.recv() => { + return; + }, + result = trigger_runnable_and_wait_for_raw_result( + &db_, + None, + authed, + &w_id, + &script_path, + is_flow, + args, + retry.as_ref(), + error_handler_path.as_deref(), + error_handler_args.as_ref(), + format!("websocket_trigger/{}", trigger_path), + ) => { + if let Ok(result) = result.map(|r| r.get().to_owned()) { + // only send the result if it's not null + if result != "null" { + tracing::info!("Sending job result to WebSocket {}", url); + // if the `result` was just a single string, the below removes the surrounding quotes by parsing it as a string. + // it falls back to the original serialized JSON if it doesn't work. + let result = serde_json::from_str::(result.as_str()).unwrap_or(result); + if let Err(err) = send_message_tx.send(result).await { + report_critical_error(format!("Could not send runnable result to WebSocket {} because of error: {}", url, err), db_.clone(), Some(&w_id), None).await; + } + } + } + } + }; + }; + + tokio::spawn(handle_response_f); + } else { + trigger_runnable( + db, + None, + authed, + &workspace_id, + &script_path, + *is_flow, + args, + retry, + error_handler_path, + error_handler_args, + format!("websocket_trigger/{}", listening_trigger.path), + ) + .await?; + } + + Ok(()) + } +} + +#[derive(Deserialize)] +pub struct JsonFilter { + key: String, + value: Value, +} + +#[derive(Deserialize)] +#[serde(untagged)] +pub enum Filter { + JsonFilter(JsonFilter), +} + +pub struct ReturnMessageChannels { + send_message_tx: tokio::sync::mpsc::Sender, + killpill_rx: tokio::sync::broadcast::Receiver<()>, +} + +impl Clone for ReturnMessageChannels { + fn clone(&self) -> Self { + Self { + send_message_tx: self.send_message_tx.clone(), + killpill_rx: self.killpill_rx.resubscribe(), + } + } +} + +#[derive(Debug, Deserialize)] +enum InitialMessage { + #[serde(rename = "raw_message")] + RawMessage(String), + #[serde(rename = "runnable_result")] + RunnableResult { path: String, args: Box, is_flow: bool }, +} + +struct SupersetVisitor<'a> { + key: &'a str, + value_to_check: &'a Value, +} + +impl<'de, 'a> Visitor<'de> for SupersetVisitor<'a> { + type Value = bool; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("a JSON object with a specific key at the top level") + } + + fn visit_map(self, mut map: V) -> std::result::Result + where + V: MapAccess<'de>, + { + while let Some(key) = map.next_key::()? { + if key == self.key { + // Deserialize the value for the key and check if it's a superset + let json_value: Value = map.next_value()?; + return Ok(is_superset(&json_value, self.value_to_check)); + } else { + // Skip the value if it's not the one we're interested in + let _ = map.next_value::()?; + } + } + // If the key was not found, return false + Ok(false) + } +} + +fn is_superset(json_value: &Value, value_to_check: &Value) -> bool { + match (json_value, value_to_check) { + (Value::Object(json_map), Value::Object(check_map)) => { + // Check that all keys and values in check_map exist and match in json_map + check_map.iter().all(|(k, v)| { + json_map + .get(k) + .map_or(false, |json_val| is_superset(json_val, v)) + }) + } + (Value::Array(json_array), Value::Array(check_array)) => { + // Check that all elements in check_array exist in json_array + check_array.iter().all(|check_item| { + json_array + .iter() + .any(|json_item| is_superset(json_item, check_item)) + }) + } + _ => json_value == value_to_check, + } +} + +// A function to deserialize and check if the value at the given key is a superset of a passed value +fn is_value_superset<'a, 'de, D>( + deserializer: D, + key: &'a str, + value_to_check: &'a Value, +) -> std::result::Result +where + D: Deserializer<'de>, +{ + deserializer.deserialize_map(SupersetVisitor { key, value_to_check }) +} + +fn raw_value_to_args_hashmap( + args: Option<&Box>, +) -> Result>> { + let args = if let Some(args) = args { + serde_json::from_str::>>>(args.get()) + .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))? + .unwrap_or_else(HashMap::new) + } else { + HashMap::new() + }; + Ok(args) +} diff --git a/backend/windmill-api/src/triggers/websocket/mod.rs b/backend/windmill-api/src/triggers/websocket/mod.rs index c367811406..7ed67e1565 100644 --- a/backend/windmill-api/src/triggers/websocket/mod.rs +++ b/backend/windmill-api/src/triggers/websocket/mod.rs @@ -16,6 +16,7 @@ use windmill_common::{ use windmill_queue::PushArgsOwned; mod handler; +mod listener; #[derive(Copy, Clone)] pub struct WebsocketTrigger; diff --git a/backend/windmill-api/src/utils.rs b/backend/windmill-api/src/utils.rs index d3b820ccc0..e65473aea6 100644 --- a/backend/windmill-api/src/utils.rs +++ b/backend/windmill-api/src/utils.rs @@ -456,9 +456,3 @@ pub struct ExpiringCacheEntry { lazy_static::lazy_static! { static ref DUCKLAKE_INSTANCE_PG_PASSWORD: std::sync::RwLock> = std::sync::RwLock::new(None); } - -#[cfg(all(feature = "kafka", feature = "enterprise", feature = "private"))] -pub async fn update_rw_lock(lock: std::sync::Arc>, value: T) -> () { - let mut w = lock.write().await; - *w = value; -} diff --git a/backend/windmill-api/src/websocket_triggers.rs b/backend/windmill-api/src/websocket_triggers.rs deleted file mode 100644 index 43e234a8c7..0000000000 --- a/backend/windmill-api/src/websocket_triggers.rs +++ /dev/null @@ -1,1005 +0,0 @@ -use anyhow::Context; - -use futures::{stream::SplitSink, SinkExt, StreamExt}; -use itertools::Itertools; -use rand::seq::SliceRandom; -use serde::{ - de::{self, MapAccess, Visitor}, - Deserialize, Deserializer, Serialize, -}; -use serde_json::{value::RawValue, Value}; -use sqlx::prelude::FromRow; -use sqlx::types::Json as SqlxJson; -use std::{collections::HashMap, fmt}; -use tokio::net::TcpStream; -use tokio_tungstenite::{connect_async, tungstenite::Message, MaybeTlsStream, WebSocketStream}; -use windmill_common::{ - error::{self, to_anyhow}, - triggers::TriggerKind, - utils::report_critical_error, - worker::to_raw_value, - INSTANCE_NAME, -}; -use windmill_queue::PushArgsOwned; - -use crate::{ - capture::{insert_capture_payload, WebsocketTriggerConfig}, - db::{ApiAuthed, DB}, - trigger_helpers::{trigger_runnable, trigger_runnable_and_wait_for_raw_result, TriggerJobArgs}, - users::fetch_api_authed, -}; - -use std::borrow::Cow; - -#[derive(Deserialize)] -pub struct JsonFilter { - key: String, - value: serde_json::Value, -} - -#[derive(Deserialize)] -#[serde(untagged)] -pub enum Filter { - JsonFilter(JsonFilter), -} - -#[derive(Deserialize)] -enum InitialMessage { - #[serde(rename = "raw_message")] - RawMessage(String), - #[serde(rename = "runnable_result")] - RunnableResult { path: String, args: Box, is_flow: bool }, -} - -#[derive(FromRow, Serialize, Clone)] -pub struct WebsocketTrigger { - pub workspace_id: String, - pub path: String, - pub url: String, - pub script_path: String, - pub is_flow: bool, - pub edited_by: String, - pub email: String, - pub edited_at: chrono::DateTime, - #[serde(skip_serializing_if = "Option::is_none")] - pub server_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub last_server_ping: Option>, - pub extra_perms: serde_json::Value, - pub error: Option, - pub enabled: bool, - pub filters: Vec>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub initial_messages: Option>>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub url_runnable_args: Option>>, - pub can_return_message: bool, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_args: Option>>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub retry: Option>, -} - -async fn listen_to_unlistened_websockets( - db: &DB, - killpill_rx: &tokio::sync::broadcast::Receiver<()>, -) { - let websocket_triggers = sqlx::query_as::<_, WebsocketTrigger>( - r#" - SELECT - workspace_id, - path, - url, - script_path, - is_flow, - edited_by, - email, - edited_at, - server_id, - last_server_ping, - extra_perms, - error, - enabled, - filters, - initial_messages, - url_runnable_args, - can_return_message, - error_handler_path, - error_handler_args, - retry - FROM websocket_trigger - WHERE - enabled IS TRUE - AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') - "#, - ) - .fetch_all(db) - .await; - - match websocket_triggers { - Ok(mut triggers) => { - triggers.shuffle(&mut rand::rng()); - for trigger in triggers { - trigger - .maybe_listen_to_websocket(db.clone(), killpill_rx.resubscribe()) - .await; - } - } - Err(err) => { - tracing::error!("Error fetching WebSocket triggers: {:?}", err); - } - }; - - match sqlx::query_as!( - CaptureConfigForWebsocket, - r#"SELECT path, is_flow, workspace_id, trigger_config as "trigger_config!: _", owner, email FROM capture_config WHERE trigger_kind = 'websocket' AND last_client_ping > NOW() - INTERVAL '10 seconds' AND trigger_config IS NOT NULL AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')"# - ) - .fetch_all(db) - .await - { - Ok(mut captures) => { - captures.shuffle(&mut rand::rng()); - for capture in captures { - capture.maybe_listen_to_websocket(db.clone(), killpill_rx.resubscribe()).await; - } - } - Err(err) => { - tracing::error!("Error fetching capture WebSocket triggers: {:?}", err); - } - } -} - -pub fn start_websockets(db: DB, mut killpill_rx: tokio::sync::broadcast::Receiver<()>) -> () { - tokio::spawn(async move { - listen_to_unlistened_websockets(&db, &killpill_rx).await; - loop { - tokio::select! { - biased; - _ = killpill_rx.recv() => { - return; - } - _ = tokio::time::sleep(tokio::time::Duration::from_secs(15)) => { - listen_to_unlistened_websockets(&db, &killpill_rx).await; - } - } - } - }); -} - -struct SupersetVisitor<'a> { - key: &'a str, - value_to_check: &'a Value, -} - -impl<'de, 'a> Visitor<'de> for SupersetVisitor<'a> { - type Value = bool; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("a JSON object with a specific key at the top level") - } - - fn visit_map(self, mut map: V) -> Result - where - V: MapAccess<'de>, - { - while let Some(key) = map.next_key::()? { - if key == self.key { - // Deserialize the value for the key and check if it's a superset - let json_value: Value = map.next_value()?; - return Ok(is_superset(&json_value, self.value_to_check)); - } else { - // Skip the value if it's not the one we're interested in - let _ = map.next_value::()?; - } - } - // If the key was not found, return false - Ok(false) - } -} - -// Function to check if json_value is a superset of value_to_check -fn is_superset(json_value: &Value, value_to_check: &Value) -> bool { - match (json_value, value_to_check) { - (Value::Object(json_map), Value::Object(check_map)) => { - // Check that all keys and values in check_map exist and match in json_map - check_map.iter().all(|(k, v)| { - json_map - .get(k) - .map_or(false, |json_val| is_superset(json_val, v)) - }) - } - (Value::Array(json_array), Value::Array(check_array)) => { - // Check that all elements in check_array exist in json_array - check_array.iter().all(|check_item| { - json_array - .iter() - .any(|json_item| is_superset(json_item, check_item)) - }) - } - _ => json_value == value_to_check, - } -} - -// A function to deserialize and check if the value at the given key is a superset of a passed value -fn is_value_superset<'a, 'de, D>( - deserializer: D, - key: &'a str, - value_to_check: &'a Value, -) -> Result -where - D: Deserializer<'de>, -{ - deserializer.deserialize_map(SupersetVisitor { key, value_to_check }) -} - -fn raw_value_to_args_hashmap( - args: Option<&Box>, -) -> error::Result>> { - let args = if let Some(args) = args { - serde_json::from_str::>>>(args.get()) - .map_err(|e| error::Error::BadRequest(format!("invalid json: {}", e)))? - .unwrap_or_else(HashMap::new) - } else { - HashMap::new() - }; - Ok(args) -} - -async fn loop_ping(db: &DB, ws: &WebsocketEnum, error: Option<&str>) -> () { - loop { - if let None = ws.update_ping(db, error).await { - return; - } - tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; - } -} - -async fn get_url_from_runnable( - path: &str, - is_flow: bool, - db: &DB, - authed: ApiAuthed, - args: Option<&Box>, - workspace_id: &str, -) -> error::Result { - tracing::info!( - "Running {} {} to get WebSocket URL", - if is_flow { "flow" } else { "script" }, - path - ); - - let args = raw_value_to_args_hashmap(args)?; - - let result = trigger_runnable_and_wait_for_raw_result( - db, - None, - authed, - workspace_id, - path, - is_flow, - PushArgsOwned { args, extra: None }, - None, - None, - None, - "".to_string(), // doesn't matter as no retry/error handler - ) - .await?; - - serde_json::from_str::(result.get()).map_err(|_| { - error::Error::BadConfig(format!( - "{} {} did not return a string", - if is_flow { "Flow" } else { "Script" }, - path, - )) - }) -} - -impl WebsocketTrigger { - async fn maybe_listen_to_websocket( - self, - db: DB, - killpill_rx: tokio::sync::broadcast::Receiver<()>, - ) -> () { - match sqlx::query_scalar!( - "UPDATE websocket_trigger SET server_id = $1, last_server_ping = now(), error = 'Connecting...' WHERE enabled IS TRUE AND workspace_id = $2 AND path = $3 AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') RETURNING true", - *INSTANCE_NAME, - self.workspace_id, - self.path, - ).fetch_optional(&db).await { - Ok(has_lock) => { - if has_lock.flatten().unwrap_or(false) { - tokio::spawn(listen_to_websocket(WebsocketEnum::Trigger(self), db, killpill_rx)); - } else { - tracing::info!("WebSocket {} already being listened to", self.url); - } - }, - Err(err) => { - tracing::error!("Error acquiring lock for WebSocket {}: {:?}", self.path, err); - } - }; - } - - async fn update_ping(&self, db: &DB, error: Option<&str>) -> Option<()> { - match sqlx::query_scalar!( - "UPDATE websocket_trigger SET last_server_ping = now(), error = $1 WHERE workspace_id = $2 AND path = $3 AND server_id = $4 AND enabled IS TRUE RETURNING 1", - error, - self.workspace_id, - self.path, - *INSTANCE_NAME - ).fetch_optional(db).await { - Ok(updated) => { - if updated.flatten().is_none() { - // allow faster restart of websocket trigger - sqlx::query!( - "UPDATE websocket_trigger SET last_server_ping = NULL WHERE workspace_id = $1 AND path = $2 AND server_id IS NULL", - self.workspace_id, - self.path, - ).execute(db).await.ok(); - tracing::info!("WebSocket {} changed, disabled, or deleted, stopping...", self.url); - return None; - } - }, - Err(err) => { - tracing::warn!("Error updating ping of WebSocket {}: {:?}", self.url, err); - } - }; - - Some(()) - } - - async fn disable_with_error(&self, db: &DB, error: String) -> () { - match sqlx::query!( - "UPDATE websocket_trigger SET enabled = FALSE, error = $1, server_id = NULL, last_server_ping = NULL WHERE workspace_id = $2 AND path = $3", - error, - self.workspace_id, - self.path, - ) - .execute(db).await { - Ok(_) => { - report_critical_error(format!("Disabling WebSocket {} because of error: {}", self.url, error), db.clone(), Some(&self.workspace_id), None).await; - }, - Err(disable_err) => { - report_critical_error( - format!("Could not disable WebSocket {} with err {}, disabling because of error {}", self.path, disable_err, error), - db.clone(), - Some(&self.workspace_id), - None, - ).await; - } - } - } - - async fn get_url_from_runnable( - &self, - path: &str, - is_flow: bool, - db: &DB, - ) -> error::Result { - get_url_from_runnable( - &path, - is_flow, - db, - self.fetch_authed(db).await?, - self.url_runnable_args.as_ref().map(|r| &r.0), - &self.workspace_id, - ) - .await - } - - async fn send_initial_messages( - &self, - writer: &mut SplitSink>, Message>, - db: &DB, - ) -> error::Result<()> { - let initial_messages: Vec = self - .initial_messages - .as_deref() - .unwrap_or_default() - .iter() - .filter_map(|m| serde_json::from_str(m.get()).ok()) - .collect_vec(); - - let mut authed_o = None; - for start_message in initial_messages { - match start_message { - InitialMessage::RawMessage(msg) => { - let msg = if msg.starts_with("\"") && msg.ends_with("\"") { - msg[1..msg.len() - 1].to_string() - } else { - msg - }; - tracing::info!( - "Sending raw message initial message to WebSocket {}: {}", - self.url, - msg - ); - writer - .send(tokio_tungstenite::tungstenite::Message::Text(msg)) - .await - .map_err(to_anyhow) - .with_context(|| "failed to send raw message")?; - } - InitialMessage::RunnableResult { path, is_flow, args } => { - tracing::info!( - "Running {} {} for initial message to WebSocket {}", - if is_flow { "flow" } else { "script" }, - path, - self.url, - ); - - let args = raw_value_to_args_hashmap(Some(&args))?; - - if authed_o.is_none() { - authed_o = Some(self.fetch_authed(db).await?); - } - let authed = authed_o.clone().unwrap(); - - let result = trigger_runnable_and_wait_for_raw_result( - db, - None, - authed.clone(), - &self.workspace_id, - &path, - is_flow, - PushArgsOwned { args, extra: None }, - None, - None, - None, - "".to_string(), // doesn't matter as no retry/error handler - ) - .await - .map(|r| r.get().to_owned())?; - - tracing::info!( - "Sending {} {} result to WebSocket {}", - if is_flow { "flow" } else { "script" }, - path, - self.url - ); - - // if the `result` was just a single string, the below removes the surrounding quotes by parsing it as a string. - // it falls back to the original serialized JSON if it doesn't work. - let result = serde_json::from_str::(result.as_str()).unwrap_or(result); - - writer - .send(tokio_tungstenite::tungstenite::Message::Text(result)) - .await - .map_err(to_anyhow) - .with_context(|| { - format!( - "Failed to send {} {} result", - if is_flow { "flow" } else { "script" }, - path - ) - })?; - } - } - } - - Ok(()) - } - - async fn handle( - &self, - db: &DB, - msg: &str, - trigger_info: HashMap>, - return_message_channels: Option, - ) -> () { - if let Err(err) = run_job(db, self, &msg, trigger_info, return_message_channels).await { - report_critical_error( - format!( - "Failed to trigger job from WebSocket {}: {:?}", - self.url, err - ), - db.clone(), - Some(&self.workspace_id), - None, - ) - .await; - }; - } - - async fn fetch_authed(&self, db: &DB) -> error::Result { - fetch_api_authed( - self.edited_by.clone(), - self.email.clone(), - &self.workspace_id, - db, - Some(format!("ws-{}", self.path)), - ) - .await - } -} - -impl TriggerJobArgs<&str> for WebsocketTrigger { - fn v1_payload_fn(payload: &str) -> HashMap> { - HashMap::from([("msg".to_string(), to_raw_value(&payload))]) - } - - fn trigger_kind() -> TriggerKind { - TriggerKind::Websocket - } -} - -#[derive(Deserialize)] -struct CaptureConfigForWebsocket { - trigger_config: SqlxJson, - path: String, - is_flow: bool, - workspace_id: String, - owner: String, - email: String, -} - -impl CaptureConfigForWebsocket { - async fn maybe_listen_to_websocket( - self, - db: DB, - killpill_rx: tokio::sync::broadcast::Receiver<()>, - ) -> () { - match sqlx::query_scalar!( - "UPDATE capture_config SET server_id = $1, last_server_ping = now(), error = 'Connecting...' WHERE last_client_ping > NOW() - INTERVAL '10 seconds' AND workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'websocket' AND (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') RETURNING true", - *INSTANCE_NAME, - self.workspace_id, - self.path, - self.is_flow, - ).fetch_optional(&db).await { - Ok(has_lock) => { - if has_lock.flatten().unwrap_or(false) { - tokio::spawn(listen_to_websocket(WebsocketEnum::Capture(self), db, killpill_rx)); - } else { - tracing::info!("WebSocket {} already being listened to", self.trigger_config.url); - } - }, - Err(err) => { - tracing::error!("Error acquiring lock for capture WebSocket {}: {:?}", self.path, err); - } - }; - } - - async fn update_ping(&self, db: &DB, error: Option<&str>) -> Option<()> { - match sqlx::query_scalar!( - "UPDATE capture_config SET last_server_ping = now(), error = $1 WHERE workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'websocket' AND server_id = $5 AND last_client_ping > NOW() - INTERVAL '10 seconds' RETURNING 1", - error, - self.workspace_id, - self.path, - self.is_flow, - *INSTANCE_NAME - ).fetch_optional(db).await { - Ok(updated) => { - if updated.flatten().is_none() { - // allow faster restart of websocket capture - sqlx::query!( - "UPDATE capture_config SET last_server_ping = NULL WHERE workspace_id = $1 AND path = $2 AND is_flow = $3 AND trigger_kind = 'websocket' AND server_id IS NULL", - self.workspace_id, - self.path, - self.is_flow, - ).execute(db).await.ok(); - tracing::info!("WebSocket capture {} changed, disabled, or deleted, stopping...", self.trigger_config.url); - return None; - } - }, - Err(err) => { - tracing::warn!("Error updating ping of capture WebSocket {}: {:?}", self.trigger_config.url, err); - } - }; - - Some(()) - } - - async fn handle(&self, db: &DB, msg: &str, trigger_info: HashMap>) -> () { - let (main_args, preprocessor_args) = - WebsocketTrigger::build_capture_payloads(&msg, trigger_info); - - if let Err(err) = insert_capture_payload( - db, - &self.workspace_id, - &self.path, - self.is_flow, - &TriggerKind::Websocket, - main_args, - preprocessor_args, - &self.owner, - ) - .await - { - tracing::error!("Error inserting capture payload: {:?}", err); - } - } - - async fn get_url_from_runnable( - &self, - path: &str, - is_flow: bool, - db: &DB, - ) -> error::Result { - let url_runnable_args = self - .trigger_config - .url_runnable_args - .as_ref() - .map(to_raw_value); - get_url_from_runnable( - &path, - is_flow, - db, - self.fetch_authed(db).await?, - url_runnable_args.as_ref(), - &self.workspace_id, - ) - .await - } - - async fn fetch_authed(&self, db: &DB) -> error::Result { - fetch_api_authed( - self.owner.clone(), - self.email.clone(), - &self.workspace_id, - db, - Some(format!("ws-{}", self.get_trigger_path())), - ) - .await - } - - async fn disable_with_error(&self, db: &DB, error: String) -> () { - if let Err(err) = sqlx::query!( - "UPDATE capture_config SET error = $1, server_id = NULL, last_server_ping = NULL WHERE workspace_id = $2 AND path = $3 AND is_flow = $4 AND trigger_kind = 'websocket'", - error, - self.workspace_id, - self.path, - self.is_flow, - ) - .execute(db).await { - tracing::error!("Could not disable WebSocket capture {} ({}) with err {}, disabling because of error {}", self.path, self.workspace_id, err, error); - } - } - - fn get_trigger_path(&self) -> String { - format!( - "{}-{}", - if self.is_flow { "flow" } else { "script" }, - self.path - ) - } -} - -enum WebsocketEnum { - Trigger(WebsocketTrigger), - Capture(CaptureConfigForWebsocket), -} - -impl WebsocketEnum { - async fn update_ping(&self, db: &DB, error: Option<&str>) -> Option<()> { - match self { - WebsocketEnum::Trigger(ws) => ws.update_ping(db, error).await, - WebsocketEnum::Capture(capture) => capture.update_ping(db, error).await, - } - } - - async fn get_url_from_runnable( - &self, - path: &str, - is_flow: bool, - db: &DB, - ) -> error::Result { - match self { - WebsocketEnum::Trigger(ws) => ws.get_url_from_runnable(path, is_flow, db).await, - WebsocketEnum::Capture(capture) => { - capture.get_url_from_runnable(path, is_flow, db).await - } - } - } - - async fn disable_with_error(&self, db: &DB, error: String) -> () { - match self { - WebsocketEnum::Trigger(ws) => ws.disable_with_error(db, error).await, - WebsocketEnum::Capture(capture) => capture.disable_with_error(db, error).await, - } - } -} - -struct ReturnMessageChannels { - send_message_tx: tokio::sync::mpsc::Sender, - killpill_rx: tokio::sync::broadcast::Receiver<()>, -} - -impl Clone for ReturnMessageChannels { - fn clone(&self) -> Self { - Self { - send_message_tx: self.send_message_tx.clone(), - killpill_rx: self.killpill_rx.resubscribe(), - } - } -} - -async fn listen_to_websocket( - ws: WebsocketEnum, - db: DB, - mut killpill_rx: tokio::sync::broadcast::Receiver<()>, -) -> () { - let url = match &ws { - WebsocketEnum::Trigger(ws_trigger) => ws_trigger.url.clone(), - WebsocketEnum::Capture(capture) => capture.trigger_config.url.clone(), - }; - - let filters: Vec = match &ws { - WebsocketEnum::Trigger(ws_trigger) => ws_trigger - .filters - .iter() - .filter_map(|m| serde_json::from_str(m.get()).ok()) - .collect_vec(), - WebsocketEnum::Capture(_) => vec![], - }; - - let connect_url: Cow = if url.starts_with("$") { - if url.starts_with("$flow:") || url.starts_with("$script:") { - let path = url.splitn(2, ':').nth(1).unwrap(); - tokio::select! { - biased; - _ = killpill_rx.recv() => { - return; - }, - _ = loop_ping(&db, &ws, Some( - "Waiting on runnable to return WebSocket URL..." - )) => { - return; - }, - - url_result = ws.get_url_from_runnable(path, url.starts_with("$flow:"), &db) => match url_result { - Ok(url) => Cow::Owned(url), - Err(err) => { - ws.disable_with_error(&db, format!( - "Error getting WebSocket URL from runnable after 5 tries: {:?}", - err - ), - ) - .await; - return; - } - }, - } - } else { - ws.disable_with_error(&db, format!("Invalid WebSocket runnable path: {}", url)) - .await; - return; - } - } else { - Cow::Borrowed(&url) - }; - - tokio::select! { - biased; - _ = killpill_rx.recv() => { - return; - }, - _ = loop_ping(&db, &ws, Some("Connecting...")) => { - return; - }, - connection = connect_async(connect_url.as_ref()) => { - match connection { - Ok((ws_stream, _)) => { - tracing::info!("Connected to WebSocket {}", url); - let (mut writer, mut reader) = ws_stream.split(); - - // send initial messages - match &ws { - WebsocketEnum::Trigger(ws_trigger) => { - tokio::select! { - biased; - _ = killpill_rx.recv() => { - return; - }, - _ = loop_ping(&db, &ws, Some("Sending initial messages...")) => { - return; - }, - result = ws_trigger.send_initial_messages(&mut writer, &db) => { - if let Err(err) = result { - ws_trigger.disable_with_error(&db, format!("Error sending initial messages: {:?}", err)).await; - return - } else { - tracing::debug!("Initial messages sent successfully to WebSocket {}", url); - } - } - } - }, - _ => { - } - } - - let (return_message_channels, message_sender_handle) = match &ws { - WebsocketEnum::Trigger(ws_trigger) if ws_trigger.can_return_message => { - let (send_message_tx, mut rx) = tokio::sync::mpsc::channel::(100); - let w_id = ws_trigger.workspace_id.clone(); - let url = ws_trigger.url.clone(); - let db = db.clone(); - let handle = tokio::spawn(async move { - while let Some(message) = rx.recv().await { - if let Err(err) = writer.send(tokio_tungstenite::tungstenite::Message::Text(message)).await { - report_critical_error(format!("Could not send runnable result to WebSocket {} because of error: {}", url, err), db.clone(), Some(&w_id), None).await; - } - } - }); - - let killpill_rx = killpill_rx.resubscribe(); - - let return_message_channels = ReturnMessageChannels { - send_message_tx, - killpill_rx - }; - - (Some(return_message_channels), Some(handle)) - }, - _ => (None, None) - }; - - tokio::select! { - biased; - _ = killpill_rx.recv() => {}, - _ = loop_ping(&db, &ws, None) => {}, - _ = async { - loop { - if let Some(msg) = reader.next().await { - match msg { - Ok(msg) => { - match msg { - tokio_tungstenite::tungstenite::Message::Text(text) => { - tracing::debug!("Received text message from WebSocket {}: {}", url, text); - let mut should_handle = true; - for filter in &filters { - match filter { - Filter::JsonFilter(JsonFilter { key, value }) => { - let mut deserializer = serde_json::Deserializer::from_str(text.as_str()); - should_handle = match is_value_superset(&mut deserializer, key, &value) { - Ok(filter_match) => { - filter_match - }, - Err(err) => { - tracing::warn!("Error deserializing filter for WebSocket {}: {:?}", url, err); - false - } - }; - } - } - if !should_handle { - break; - } - } - if should_handle { - let trigger_info = HashMap::from([ - ("url".to_string(), to_raw_value(&url)), - ]); - match &ws { - WebsocketEnum::Trigger(ws_trigger) => { - ws_trigger.handle(&db, &text, trigger_info, return_message_channels.clone()).await; - }, - WebsocketEnum::Capture(capture) => { - capture.handle(&db, &text, trigger_info).await; - }, - } - } - }, - a @ _ => { - tracing::debug!("Received non text-message from WebSocket {}: {:?}", url, a); - } - } - }, - Err(err) => { - tracing::error!("Error reading from WebSocket {}: {:?}", url, err); - } - } - } else { - tracing::error!("WebSocket {} closed", url); - ws.update_ping(&db, Some("WebSocket closed")).await; - break; - } - } - } => {} - } - // make sure to stop return message handler - if let Some(message_sender_handle) = message_sender_handle { - message_sender_handle.abort(); - } - } - Err(err) => { - tracing::error!("Error connecting to WebSocket {}: {:?}", url, err); - ws.update_ping(&db, Some(err.to_string().as_str())).await; - } - } - } - } -} - -async fn run_job( - db: &DB, - trigger: &WebsocketTrigger, - msg: &str, - trigger_info: HashMap>, - return_message_channels: Option, -) -> anyhow::Result<()> { - let args = WebsocketTrigger::build_job_args( - &trigger.script_path, - trigger.is_flow, - &trigger.workspace_id, - db, - msg, - trigger_info, - ) - .await?; - - let authed = fetch_api_authed( - trigger.edited_by.clone(), - trigger.email.clone(), - &trigger.workspace_id, - db, - Some(format!("ws-{}", trigger.path)), - ) - .await?; - - if let Some(ReturnMessageChannels { send_message_tx, mut killpill_rx }) = - return_message_channels - { - let db_ = db.clone(); - let url = trigger.url.clone(); - let script_path = trigger.script_path.clone(); - let is_flow = trigger.is_flow; - let w_id = trigger.workspace_id.clone(); - let retry = trigger.retry.clone(); - let error_handler_path = trigger.error_handler_path.clone(); - let error_handler_args = trigger.error_handler_args.clone(); - let trigger_path = trigger.path.clone(); - let handle_response_f = async move { - tokio::select! { - _ = killpill_rx.recv() => { - return; - }, - result = trigger_runnable_and_wait_for_raw_result( - &db_, - None, - authed, - &w_id, - &script_path, - is_flow, - args, - retry.as_ref(), - error_handler_path.as_deref(), - error_handler_args.as_ref(), - format!("websocket_trigger/{}", trigger_path), - ) => { - if let Ok(result) = result.map(|r| r.get().to_owned()) { - // only send the result if it's not null - if result != "null" { - tracing::info!("Sending job result to WebSocket {}", url); - // if the `result` was just a single string, the below removes the surrounding quotes by parsing it as a string. - // it falls back to the original serialized JSON if it doesn't work. - let result = serde_json::from_str::(result.as_str()).unwrap_or(result); - if let Err(err) = send_message_tx.send(result).await { - report_critical_error(format!("Could not send runnable result to WebSocket {} because of error: {}", url, err), db_.clone(), Some(&w_id), None).await; - } - } - } - } - }; - }; - - tokio::spawn(handle_response_f); - } else { - trigger_runnable( - db, - None, - authed, - &trigger.workspace_id, - &trigger.script_path, - trigger.is_flow, - args, - trigger.retry.as_ref(), - trigger.error_handler_path.as_deref(), - trigger.error_handler_args.as_ref(), - format!("websocket_trigger/{}", trigger.path), - ) - .await?; - } - - Ok(()) -}