starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-18 16:02:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add postgres trigger captures (#5165)

Browse Source 
* feat: 🚧 add postgres trigger config in back

* feat: add capture section for postgres

* feat: update postgres trigger panel, update capture migration

* fix: ci and casing style

* feat: capture done

* fix: update sqlx

* fix: remove unused

* feat: handle persisting data

* fix: add feature flag

* feat: capture done

* fix: feature flag

* feat: capture done

* nits: add plus into postgres capture label

* feat: trigger done

* Update PostgresTriggerEditorInner.svelte

* Update script_helpers.ts

* nit

* chore: update to meet pr comment requirement

* refactor:

* fix:

* fix: miss closing curly braces

* fix:

* fix: ci

* Update mod.rs

* polishing postgres trigger

* update minor changes on UI

* nits: remove log and update function parameters

* fix: remove wrong name

* nits: change icon

* fix: nit migration

* nits: refacto and remove alert

* nits: change variable name

* update workspaces_export

* fix: ci

---------

Co-authored-by: HugoCasa <hugo@casademont.ch>
Co-authored-by: Guilhem <guilhemlemouel@gmail.com>
This commit is contained in:
dieriba
2025-02-11 18:16:12 +01:00
committed by GitHub
parent aae3683fe9
commit 57cfa4045b
59 changed files with 3349 additions and 981 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/.sqlx/query-011c7638eeeda710deb86a216a9e10df9c3e9458e85bcdde466b01011a1f2ac2.json
Generated
+50
View File
@@ -0,0 +1,50 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n path,\n is_flow,\n workspace_id,\n owner,\n email,\n trigger_config as \"trigger_config!: _\"\n FROM\n capture_config\n WHERE\n trigger_kind = 'postgres' AND\n last_client_ping > NOW() - INTERVAL '10 seconds' AND\n trigger_config IS NOT NULL AND\n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 2,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "owner",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "trigger_config!: _",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
false,
false,
false,
false,
true
]
},
"hash": "011c7638eeeda710deb86a216a9e10df9c3e9458e85bcdde466b01011a1f2ac2"
}
backend/.sqlx/query-07da723ce5c9ee2d7c236e8eabe254c783fc34b617c8a9a95a0eb0cda535dab5.json
Generated
+2 -1
View File
@@ -18,7 +18,8 @@
"websocket",
"kafka",
"email",
"nats"
"nats",
"postgres"
]
}
}
backend/.sqlx/query-12e868b63a7c622c76713db5a5577a927efca4ae49a15c2b999e2410f2a312ff.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE \n capture_config \n SET \n last_server_ping = NULL \n WHERE \n workspace_id = $1 AND \n path = $2 AND \n is_flow = $3 AND \n trigger_kind = 'postgres' AND \n server_id IS NULL\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Bool"
]
},
"nullable": []
},
"hash": "12e868b63a7c622c76713db5a5577a927efca4ae49a15c2b999e2410f2a312ff"
}
backend/.sqlx/query-1488e1b5007752e1ebae4235ad04c398fe6398745e16fd119008b8ea67662416.json
Generated
+24
View File
@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE postgres_trigger \n SET \n server_id = $1, \n last_server_ping = now(),\n error = 'Connecting...'\n WHERE \n enabled IS TRUE \n AND workspace_id = $2 \n AND path = $3 \n AND (last_server_ping IS NULL \n OR last_server_ping < now() - INTERVAL '15 seconds'\n ) \n RETURNING true\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Varchar",
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "1488e1b5007752e1ebae4235ad04c398fe6398745e16fd119008b8ea67662416"
}
backend/.sqlx/query-1974bd65bbf40024773aad4dee1c50b12e110e76bb58e6de25bec094e758a71c.json
Generated
+26
View File
@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n capture_config \n SET \n last_server_ping = now(), \n error = $1 \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'postgres' AND \n server_id = $5 AND \n last_client_ping > NOW() - INTERVAL '10 seconds' \n RETURNING 1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Text",
"Bool",
"Text"
]
},
"nullable": [
null
]
},
"hash": "1974bd65bbf40024773aad4dee1c50b12e110e76bb58e6de25bec094e758a71c"
}
backend/.sqlx/query-1aa8ead10f8d994f6685d266fcbd409b0fff43111d9600e64b2348401ed8929d.json
Generated
+118
View File
@@ -0,0 +1,118 @@
{
"db_name": "PostgreSQL",
"query": "SELECT workspace_id, path, url, script_path, is_flow, edited_by, email, edited_at, server_id, last_server_ping, extra_perms, error, enabled, filters as \"filters: _\", initial_messages as \"initial_messages: _\", url_runnable_args as \"url_runnable_args: _\", can_return_message FROM websocket_trigger\n WHERE workspace_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "url",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 5,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 6,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 8,
"name": "server_id",
"type_info": "Varchar"
},
{
"ordinal": 9,
"name": "last_server_ping",
"type_info": "Timestamptz"
},
{
"ordinal": 10,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 11,
"name": "error",
"type_info": "Text"
},
{
"ordinal": 12,
"name": "enabled",
"type_info": "Bool"
},
{
"ordinal": 13,
"name": "filters: _",
"type_info": "JsonbArray"
},
{
"ordinal": 14,
"name": "initial_messages: _",
"type_info": "JsonbArray"
},
{
"ordinal": 15,
"name": "url_runnable_args: _",
"type_info": "Jsonb"
},
{
"ordinal": 16,
"name": "can_return_message",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
false,
true,
true,
false,
true,
false,
false,
true,
true,
false
]
},
"hash": "1aa8ead10f8d994f6685d266fcbd409b0fff43111d9600e64b2348401ed8929d"
}
backend/.sqlx/query-29f096ec62c4abb1435a5667e2b30e9c1724e419cdc23ef1b300e84c02a20427.json
Generated
+17
View File
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n capture_config \n SET \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'postgres'\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text",
"Bool"
]
},
"nullable": []
},
"hash": "29f096ec62c4abb1435a5667e2b30e9c1724e419cdc23ef1b300e84c02a20427"
}
backend/.sqlx/query-4931d4752357078ae3ae01f37742639dba0dde680ab934ba78abcb5fdda8117a.json
Generated
+106
View File
@@ -0,0 +1,106 @@
{
"db_name": "PostgreSQL",
"query": "SELECT * FROM postgres_trigger\n WHERE workspace_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 3,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 6,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 7,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 8,
"name": "postgres_resource_path",
"type_info": "Varchar"
},
{
"ordinal": 9,
"name": "error",
"type_info": "Text"
},
{
"ordinal": 10,
"name": "server_id",
"type_info": "Varchar"
},
{
"ordinal": 11,
"name": "last_server_ping",
"type_info": "Timestamptz"
},
{
"ordinal": 12,
"name": "replication_slot_name",
"type_info": "Varchar"
},
{
"ordinal": 13,
"name": "publication_name",
"type_info": "Varchar"
},
{
"ordinal": 14,
"name": "enabled",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
true,
false,
true,
true,
true,
false,
false,
false
]
},
"hash": "4931d4752357078ae3ae01f37742639dba0dde680ab934ba78abcb5fdda8117a"
}
backend/.sqlx/query-5c1de8473e0e96c1063a9a735a064c5a91e3ed8d9260c72b783fc12542b88fbd.json
Generated
+4 -2
View File
@@ -26,7 +26,8 @@
"websocket",
"kafka",
"email",
"nats"
"nats",
"postgres"
]
}
}
@@ -58,7 +59,8 @@
"websocket",
"kafka",
"email",
"nats"
"nats",
"postgres"
]
}
}
backend/.sqlx/query-61bed1bc6d3e6a3c1d640eeacc290a85d8b63ee36c39dfbf4348d120f6e561ae.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n postgres_trigger \n SET\n last_server_ping = NULL \n WHERE \n workspace_id = $1 \n AND path = $2 \n AND server_id IS NULL",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "61bed1bc6d3e6a3c1d640eeacc290a85d8b63ee36c39dfbf4348d120f6e561ae"
}
backend/.sqlx/query-61e6070b8a1e3a138818c327d6dbe7efbe27f9e2c8e02258cf7aa06e1779fddb.json
Generated
+118
View File
@@ -0,0 +1,118 @@
{
"db_name": "PostgreSQL",
"query": "SELECT * FROM nats_trigger\n WHERE workspace_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "nats_resource_path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "subjects",
"type_info": "VarcharArray"
},
{
"ordinal": 3,
"name": "stream_name",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "consumer_name",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "use_jetstream",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 8,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 9,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 10,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 11,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 12,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 13,
"name": "server_id",
"type_info": "Varchar"
},
{
"ordinal": 14,
"name": "last_server_ping",
"type_info": "Timestamptz"
},
{
"ordinal": 15,
"name": "error",
"type_info": "Text"
},
{
"ordinal": 16,
"name": "enabled",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false,
false,
true,
true,
false,
false,
false,
false,
false,
false,
false,
false,
true,
true,
true,
false
]
},
"hash": "61e6070b8a1e3a138818c327d6dbe7efbe27f9e2c8e02258cf7aa06e1779fddb"
}
backend/.sqlx/query-62475252dcf54f32433b97ae011daf5d4205d160d2aedf463c7dfe944e93257a.json
Generated
+2 -1
View File
@@ -18,7 +18,8 @@
"websocket",
"kafka",
"email",
"nats"
"nats",
"postgres"
]
}
}
backend/.sqlx/query-6f56acb985aa7141ea1891d7ad58a32c35d1b02fe7070c92a2e62c1a5339c396.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT slot_name FROM pg_replication_slots where slot_name = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "slot_name",
"type_info": "Name"
}
],
"parameters": {
"Left": [
"Name"
]
},
"nullable": [
true
]
},
"hash": "6f56acb985aa7141ea1891d7ad58a32c35d1b02fe7070c92a2e62c1a5339c396"
}
backend/.sqlx/query-71d51bbc35da7b9930e3ea3a634451217ccb9f1bc35b1ad6e10d16bc19c41447.json
Generated
+2 -1
View File
@@ -29,7 +29,8 @@
"websocket",
"kafka",
"email",
"nats"
"nats",
"postgres"
]
}
}
backend/.sqlx/query-74d928f4c3f0de191f414471b9a4fbe9c20f9685b06ad5bbded424948b2dc88c.json
Generated
+25
View File
@@ -0,0 +1,25 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n capture_config \n SET \n server_id = $1,\n last_server_ping = now(), \n error = 'Connecting...' \n WHERE \n last_client_ping > NOW() - INTERVAL '10 seconds' AND \n workspace_id = $2 AND \n path = $3 AND \n is_flow = $4 AND \n trigger_kind = 'postgres' AND \n (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') \n RETURNING true\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Varchar",
"Text",
"Text",
"Bool"
]
},
"nullable": [
null
]
},
"hash": "74d928f4c3f0de191f414471b9a4fbe9c20f9685b06ad5bbded424948b2dc88c"
}
backend/.sqlx/query-8b784784fe63d91cc5ebe27022f803caf85d5916960308cff512047d4f0dcba4.json
Generated
+106
View File
@@ -0,0 +1,106 @@
{
"db_name": "PostgreSQL",
"query": "SELECT * FROM kafka_trigger\n WHERE workspace_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "kafka_resource_path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "topics",
"type_info": "VarcharArray"
},
{
"ordinal": 3,
"name": "group_id",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 9,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 10,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 11,
"name": "server_id",
"type_info": "Varchar"
},
{
"ordinal": 12,
"name": "last_server_ping",
"type_info": "Timestamptz"
},
{
"ordinal": 13,
"name": "error",
"type_info": "Text"
},
{
"ordinal": 14,
"name": "enabled",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
false,
false,
false,
false,
true,
true,
true,
false
]
},
"hash": "8b784784fe63d91cc5ebe27022f803caf85d5916960308cff512047d4f0dcba4"
}
backend/.sqlx/query-9116102c6ccad5b0d752d5d690c233dfe48062aef23072b4f4ae4ab5ca269082.json
Generated
+25
View File
@@ -0,0 +1,25 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n postgres_trigger\n SET \n last_server_ping = now(),\n error = $1\n WHERE\n workspace_id = $2\n AND path = $3\n AND server_id = $4 \n AND enabled IS TRUE\n RETURNING 1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "9116102c6ccad5b0d752d5d690c233dfe48062aef23072b4f4ae4ab5ca269082"
}
backend/.sqlx/query-baa1dddc616419bf4b923715f0a863bc0ff69c98db0f0c8f55e4ac89fdde7a60.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT pubname FROM pg_publication WHERE pubname = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "pubname",
"type_info": "Name"
}
],
"parameters": {
"Left": [
"Name"
]
},
"nullable": [
false
]
},
"hash": "baa1dddc616419bf4b923715f0a863bc0ff69c98db0f0c8f55e4ac89fdde7a60"
}
backend/.sqlx/query-bfc534d87d701d7ac78cc97d0054d829165ba3f22fba75c3161e4cddb72264ee.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE \n postgres_trigger \n SET \n enabled = FALSE, \n error = $1, \n server_id = NULL, \n last_server_ping = NULL \n WHERE \n workspace_id = $2 AND \n path = $3\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "bfc534d87d701d7ac78cc97d0054d829165ba3f22fba75c3161e4cddb72264ee"
}
backend/.sqlx/query-c223f8b7fa4ef1aa06e1ba2a56d677774aa237508d5610714efd2e9b8b93c7b8.json
Generated
+2 -1
View File
@@ -21,7 +21,8 @@
"websocket",
"kafka",
"email",
"nats"
"nats",
"postgres"
]
}
}
backend/.sqlx/query-c5270ee815689e42b65df507b850da43239c9a5aaea41c9aed7ed33a6219a534.json
Generated
+2 -1
View File
@@ -18,7 +18,8 @@
"websocket",
"kafka",
"email",
"nats"
"nats",
"postgres"
]
}
}
backend/.sqlx/query-cc6e21bf16d7b92764aa2b261cee94f6daf6a1eedd8a68742a2f510f0452cc32.json
Generated
+46
View File
@@ -0,0 +1,46 @@
{
"db_name": "PostgreSQL",
"query": "SELECT g_.workspace_id, name, summary, extra_perms, array_agg(u2g.usr) filter (where u2g.usr is not null) as members \n FROM usr u\n JOIN usr_to_group u2g ON u2g.usr = u.username AND u2g.workspace_id = u.workspace_id\n RIGHT JOIN group_ g_ ON g_.workspace_id = u.workspace_id AND g_.name = u2g.group_\n WHERE g_.workspace_id = $1 AND g_.name != 'all'\n GROUP BY g_.workspace_id, name, summary, extra_perms",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "summary",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 4,
"name": "members",
"type_info": "VarcharArray"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false,
true,
false,
null
]
},
"hash": "cc6e21bf16d7b92764aa2b261cee94f6daf6a1eedd8a68742a2f510f0452cc32"
}
backend/.sqlx/query-d8aa1a5813fedbb22128112fcdf9d16769811cd5d5ad3b66362cc5bb1750ae6b.json
Generated
+119
View File
@@ -0,0 +1,119 @@
{
"db_name": "PostgreSQL",
"query": "SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, edited_by, edited_at, email, extra_perms, is_async, requires_auth, http_method as \"http_method: _\", static_asset_config as \"static_asset_config: _\", is_static_website FROM http_trigger\n WHERE workspace_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "route_path",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "route_path_key",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 7,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 8,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 9,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 10,
"name": "is_async",
"type_info": "Bool"
},
{
"ordinal": 11,
"name": "requires_auth",
"type_info": "Bool"
},
{
"ordinal": 12,
"name": "http_method: _",
"type_info": {
"Custom": {
"name": "http_method",
"kind": {
"Enum": [
"get",
"post",
"put",
"delete",
"patch"
]
}
}
}
},
{
"ordinal": 13,
"name": "static_asset_config: _",
"type_info": "Jsonb"
},
{
"ordinal": 14,
"name": "is_static_website",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
false,
false,
false,
false,
false,
false,
true,
false
]
},
"hash": "d8aa1a5813fedbb22128112fcdf9d16769811cd5d5ad3b66362cc5bb1750ae6b"
}
backend/.sqlx/query-dc165e2d3e6cfc52d92b48500b5ca7dd94b46263c58163071c0ded1c54535727.json
Generated
+130
View File
@@ -0,0 +1,130 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n -- slack_team_id, \n -- slack_name, \n -- slack_command_script, \n -- CASE WHEN slack_email = 'missing@email.xyz' THEN NULL ELSE slack_email END AS slack_email,\n auto_invite_domain IS NOT NULL AS \"auto_invite_enabled!\",\n CASE WHEN auto_invite_operator IS TRUE THEN 'operator' ELSE 'developer' END AS \"auto_invite_as!\", \n CASE WHEN auto_add IS TRUE THEN 'add' ELSE 'invite' END AS \"auto_invite_mode!\", \n webhook, \n deploy_to, \n error_handler, \n ai_resource, \n ai_models,\n code_completion_model,\n error_handler_extra_args, \n error_handler_muted_on_cancel, \n large_file_storage, \n git_sync,\n default_app,\n default_scripts,\n workspace.name,\n mute_critical_alerts,\n color,\n operator_settings\n FROM workspace_settings\n LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id\n WHERE workspace_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "auto_invite_enabled!",
"type_info": "Bool"
},
{
"ordinal": 1,
"name": "auto_invite_as!",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "auto_invite_mode!",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "webhook",
"type_info": "Text"
},
{
"ordinal": 4,
"name": "deploy_to",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "error_handler",
"type_info": "Varchar"
},
{
"ordinal": 6,
"name": "ai_resource",
"type_info": "Jsonb"
},
{
"ordinal": 7,
"name": "ai_models",
"type_info": "VarcharArray"
},
{
"ordinal": 8,
"name": "code_completion_model",
"type_info": "Varchar"
},
{
"ordinal": 9,
"name": "error_handler_extra_args",
"type_info": "Json"
},
{
"ordinal": 10,
"name": "error_handler_muted_on_cancel",
"type_info": "Bool"
},
{
"ordinal": 11,
"name": "large_file_storage",
"type_info": "Jsonb"
},
{
"ordinal": 12,
"name": "git_sync",
"type_info": "Jsonb"
},
{
"ordinal": 13,
"name": "default_app",
"type_info": "Varchar"
},
{
"ordinal": 14,
"name": "default_scripts",
"type_info": "Jsonb"
},
{
"ordinal": 15,
"name": "name",
"type_info": "Varchar"
},
{
"ordinal": 16,
"name": "mute_critical_alerts",
"type_info": "Bool"
},
{
"ordinal": 17,
"name": "color",
"type_info": "Varchar"
},
{
"ordinal": 18,
"name": "operator_settings",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null,
null,
null,
true,
true,
true,
true,
false,
true,
true,
false,
true,
true,
true,
true,
false,
true,
true,
true
]
},
"hash": "dc165e2d3e6cfc52d92b48500b5ca7dd94b46263c58163071c0ded1c54535727"
}
backend/.sqlx/query-e17ec84003e2ec414622d100f5dfdda86bee33f31835317df512a20c805b35d7.json
Generated
+2 -1
View File
@@ -26,7 +26,8 @@
"websocket",
"kafka",
"email",
"nats"
"nats",
"postgres"
]
}
}
backend/.sqlx/query-e23e110e1f0438d21534fc4323e0e7bc1f0dbeca2e4f44ced05bae0ca5ca1039.json
Generated
+2 -1
View File
@@ -34,7 +34,8 @@
"websocket",
"kafka",
"email",
"nats"
"nats",
"postgres"
]
}
}
backend/.sqlx/query-e5fb3531f8bc7ef1f7484524f8c3bc9c48f71a44827ba0d01ac5588dc31082a2.json
Generated
+64
View File
@@ -0,0 +1,64 @@
{
"db_name": "PostgreSQL",
"query": "SELECT * FROM usr\n WHERE workspace_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "username",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "is_admin",
"type_info": "Bool"
},
{
"ordinal": 4,
"name": "created_at",
"type_info": "Timestamptz"
},
{
"ordinal": 5,
"name": "operator",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "disabled",
"type_info": "Bool"
},
{
"ordinal": 7,
"name": "role",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
true
]
},
"hash": "e5fb3531f8bc7ef1f7484524f8c3bc9c48f71a44827ba0d01ac5588dc31082a2"
}
backend/.sqlx/query-fd5754fe3c6346ae28818a9d60d144a40f8884f47e5bbdd2824e939dafd8f154.json
Generated
+40
View File
@@ -0,0 +1,40 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n schemaname AS schema_name,\n tablename AS table_name,\n CASE\n WHEN array_length(attnames, 1) = (SELECT COUNT(*) FROM information_schema.columns WHERE table_schema = pg_publication_tables.schemaname AND table_name = pg_publication_tables.tablename)\n THEN NULL\n ELSE attnames\n END AS columns,\n rowfilter AS where_clause\n FROM\n pg_publication_tables\n WHERE\n pubname = $1;\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "schema_name",
"type_info": "Name"
},
{
"ordinal": 1,
"name": "table_name",
"type_info": "Name"
},
{
"ordinal": 2,
"name": "columns",
"type_info": "NameArray"
},
{
"ordinal": 3,
"name": "where_clause",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Name"
]
},
"nullable": [
true,
true,
null,
true
]
},
"hash": "fd5754fe3c6346ae28818a9d60d144a40f8884f47e5bbdd2824e939dafd8f154"
}
backend/migrations/20250204192651_add_postgres_type_value_to_trigger_kind_type.down.sql
View File
backend/migrations/20250204192651_add_postgres_type_value_to_trigger_kind_type.up.sql
+2
View File
@@ -0,0 +1,2 @@
-- Add up migration script here
ALTER TYPE TRIGGER_KIND ADD VALUE IF NOT EXISTS 'postgres';
backend/windmill-api/openapi.yaml
+30 -2
View File
@@ -8760,6 +8760,34 @@ paths:
schema:
type: string
/w/{workspace}/postgres_triggers/test:
post:
summary: test postgres connection
operationId: testPostgresConnection
tags:
- postgres_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
description: test postgres connection
required: true
content:
application/json:
schema:
type: object
properties:
database:
type: string
required:
- database
responses:
"200":
description: successfuly connected to postgres
content:
text/plain:
schema:
type: string
/groups/list:
get:
summary: list instance groups
@@ -13323,7 +13351,7 @@ components:
type: string
required:
- transaction_to_track
TableToTrack:
type: array
items:
@@ -14644,7 +14672,7 @@ components:
CaptureTriggerKind:
type: string
enum: [webhook, http, websocket, kafka, email, nats]
enum: [webhook, http, websocket, kafka, email, nats, postgres]
Capture:
type: object
backend/windmill-api/src/capture.rs
+97 -2
View File
@@ -38,6 +38,16 @@ use crate::http_triggers::{build_http_trigger_extra, HttpMethod};
use crate::kafka_triggers_ee::KafkaTriggerConfigConnection;
#[cfg(all(feature = "enterprise", feature = "nats"))]
use crate::nats_triggers_ee::NatsTriggerConfigConnection;
#[cfg(feature = "postgres_trigger")]
use crate::postgres_triggers::{
create_logical_replication_slot_query, create_publication_query, drop_publication_query,
generate_random_string, get_database_connection, PublicationData,
};
#[cfg(feature = "postgres_trigger")]
use itertools::Itertools;
#[cfg(feature = "postgres_trigger")]
use pg_escape::quote_literal;
use crate::{
args::WebhookArgs,
db::{ApiAuthed, DB},
@@ -78,7 +88,7 @@ pub fn workspaced_unauthed_service() -> Router {
}
}
#[derive(sqlx::Type, Serialize, Deserialize)]
#[derive(sqlx::Type, Serialize, Deserialize, Debug)]
#[sqlx(type_name = "TRIGGER_KIND", rename_all = "lowercase")]
#[serde(rename_all = "lowercase")]
pub enum TriggerKind {
@@ -88,6 +98,7 @@ pub enum TriggerKind {
Kafka,
Email,
Nats,
Postgres,
}
impl fmt::Display for TriggerKind {
@@ -99,6 +110,7 @@ impl fmt::Display for TriggerKind {
TriggerKind::Kafka => "kafka",
TriggerKind::Email => "email",
TriggerKind::Nats => "nats",
TriggerKind::Postgres => "postgres",
};
write!(f, "{}", s)
}
@@ -133,6 +145,16 @@ pub struct NatsTriggerConfig {
pub use_jetstream: bool,
}
#[cfg(feature = "postgres_trigger")]
#[derive(Serialize, Deserialize, Debug)]
pub struct PostgresTriggerConfig {
pub postgres_resource_path: String,
pub publication_name: Option<String>,
pub replication_slot_name: Option<String>,
pub publication: PublicationData,
}
#[cfg(feature = "websocket")]
#[derive(Serialize, Deserialize, Debug)]
pub struct WebsocketTriggerConfig {
pub url: String,
@@ -145,6 +167,9 @@ pub struct WebsocketTriggerConfig {
enum TriggerConfig {
#[cfg(feature = "http_trigger")]
Http(HttpTriggerConfig),
#[cfg(feature = "postgres_trigger")]
Postgres(PostgresTriggerConfig),
#[cfg(feature = "websocket")]
Websocket(WebsocketTriggerConfig),
#[cfg(all(feature = "enterprise", feature = "kafka"))]
Kafka(KafkaTriggerConfig),
@@ -186,18 +211,88 @@ async fn get_configs(
)
.fetch_all(&mut *tx)
.await?;
tx.commit().await?;
Ok(Json(configs))
}
#[cfg(feature = "postgres_trigger")]
async fn set_postgres_trigger_config(
w_id: &str,
authed: ApiAuthed,
db: &DB,
user_db: UserDB,
mut capture_config: NewCaptureConfig,
) -> Result<NewCaptureConfig> {
let Some(TriggerConfig::Postgres(mut postgres_config)) = capture_config.trigger_config else {
return Err(windmill_common::error::Error::BadRequest(
"Invalid postgres config".to_string(),
));
};
let mut connection = get_database_connection(
authed,
Some(user_db),
&db,
&postgres_config.postgres_resource_path,
&w_id,
)
.await?;
let publication_name = postgres_config
.publication_name
.get_or_insert(format!("windmill_capture_{}", generate_random_string()));
let replication_slot_name = postgres_config
.replication_slot_name
.get_or_insert(publication_name.clone());
let query = drop_publication_query(&publication_name);
sqlx::query(&query).execute(&mut connection).await?;
let query = create_publication_query(
&publication_name,
postgres_config.publication.table_to_track.as_deref(),
&postgres_config
.publication
.transaction_to_track
.iter()
.map(AsRef::as_ref)
.collect_vec(),
);
sqlx::query(&query).execute(&mut connection).await?;
let query = format!(
"SELECT 1 from pg_replication_slots WHERE slot_name = {}",
quote_literal(replication_slot_name)
);
let row = sqlx::query(&query).fetch_optional(&mut connection).await?;
if row.is_none() {
let query = create_logical_replication_slot_query(&replication_slot_name);
sqlx::query(&query).execute(&mut connection).await?;
}
capture_config.trigger_config = Some(TriggerConfig::Postgres(postgres_config));
Ok(capture_config)
}
async fn set_config(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
#[cfg(feature = "postgres_trigger")] Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Json(nc): Json<NewCaptureConfig>,
) -> Result<()> {
#[cfg(feature = "postgres_trigger")]
let nc = if let TriggerKind::Postgres = nc.trigger_kind {
set_postgres_trigger_config(&w_id, authed.clone(), &db, user_db.clone(), nc).await?
}
else {
nc
};
let mut tx = user_db.begin(&authed).await?;
sqlx::query!(
backend/windmill-api/src/http_triggers.rs
+1 -1
View File
@@ -75,7 +75,7 @@ pub fn workspaced_service() -> Router {
.route("/route_exists", post(exists_route))
}
#[derive(Serialize, Deserialize, sqlx::Type)]
#[derive(Serialize, Deserialize, sqlx::Type, Debug)]
#[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")]
#[serde(rename_all = "lowercase")]
pub enum HttpMethod {
backend/windmill-api/src/postgres_triggers/handler.rs
+290 -293
View File
@@ -1,9 +1,6 @@
use std::{
collections::{
hash_map::Entry::{Occupied, Vacant},
HashMap,
},
str::FromStr,
use std::collections::{
hash_map::Entry::{Occupied, Vacant},
HashMap,
};
use crate::{
@@ -14,29 +11,28 @@ use axum::{
extract::{Path, Query},
Extension, Json,
};
use chrono::Utc;
use http::StatusCode;
use itertools::Itertools;
use pg_escape::{quote_identifier, quote_literal};
use quick_cache::sync::Cache;
use rand::Rng;
use rust_postgres::types::Type;
use serde::{Deserialize, Deserializer, Serialize};
use sql_builder::{bind::Bind, SqlBuilder};
use sqlx::{
postgres::{types::Oid, PgConnectOptions, PgSslMode},
Connection, FromRow, PgConnection, QueryBuilder,
};
use sqlx::{postgres::types::Oid, FromRow, PgConnection};
use windmill_audit::{audit_ee::audit_log, ActionKind};
use windmill_common::error::Error;
use windmill_common::{
db::UserDB,
error::{self, JsonResult},
error::{self, JsonResult, Result},
utils::{not_found_if_none, paginate, Pagination, StripPath},
worker::CLOUD_HOSTED,
};
use super::get_database_resource;
use super::{
create_logical_replication_slot_query, create_publication_query,
drop_logical_replication_slot_query, drop_publication_query, generate_random_string,
get_database_connection, ERROR_PUBLICATION_NAME_NOT_EXISTS, ERROR_REPLICATION_SLOT_NOT_EXISTS,
};
use lazy_static::lazy_static;
#[derive(FromRow, Serialize, Deserialize, Debug)]
@@ -88,7 +84,7 @@ impl Relations {
}
}
#[derive(Deserialize)]
#[derive(Debug, Deserialize)]
pub struct EditPostgresTrigger {
replication_slot_name: String,
publication_name: String,
@@ -112,48 +108,39 @@ pub struct NewPostgresTrigger {
publication: Option<PublicationData>,
}
pub async fn get_database_connection(
authed: ApiAuthed,
user_db: Option<UserDB>,
db: &DB,
postgres_resource_path: &str,
w_id: &str,
) -> Result<PgConnection, windmill_common::error::Error> {
let database = get_database_resource(authed, user_db, db, postgres_resource_path, w_id).await?;
Ok(get_raw_postgres_connection(&database).await?)
#[derive(Serialize, Deserialize)]
pub struct TestPostgres {
pub postgres_resource_path: String,
}
pub async fn get_raw_postgres_connection(db: &Database) -> Result<PgConnection, Error> {
let options = {
let sslmode = if !db.sslmode.is_empty() {
PgSslMode::from_str(&db.sslmode)?
} else {
PgSslMode::Prefer
};
let options = PgConnectOptions::new()
.host(&db.host)
.database(&db.dbname)
.port(db.port)
.ssl_mode(sslmode)
.username(&db.user);
let options = if !db.root_certificate_pem.is_empty() {
options.ssl_root_cert_from_pem(db.root_certificate_pem.as_bytes().to_vec())
} else {
options
};
if !db.password.is_empty() {
options.password(&db.password)
} else {
options
}
};
PgConnection::connect_with(&options)
pub async fn test_postgres_connection(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path(workspace_id): Path<String>,
Json(test_postgres): Json<TestPostgres>,
) -> Result<()> {
let connect_f = async {
get_database_connection(
authed,
Some(user_db),
&db,
&test_postgres.postgres_resource_path,
&workspace_id,
)
.await
.map_err(|e| e.into())
.map_err(|err| {
error::Error::BadConfig(format!("Error connecting to postgres: {}", err.to_string()))
})
};
tokio::time::timeout(tokio::time::Duration::from_secs(30), connect_f)
.await
.map_err(|_| {
error::Error::BadConfig(format!("Timeout connecting to postgres after 30 seconds"))
})??;
Ok(())
}
#[derive(Deserialize, Debug)]
@@ -165,19 +152,20 @@ pub enum Language {
#[derive(Debug, Deserialize)]
pub struct TemplateScript {
postgres_resource_path: String,
#[serde(deserialize_with = "check_if_not_duplication_relation")]
#[serde(deserialize_with = "check_if_valid_relation")]
relations: Option<Vec<Relations>>,
language: Language,
}
fn check_if_not_duplication_relation<'de, D>(
fn check_if_valid_relation<'de, D>(
relations: D,
) -> std::result::Result<Option<Vec<Relations>>, D::Error>
where
D: Deserializer<'de>,
{
let relations: Option<Vec<Relations>> = Option::deserialize(relations)?;
let mut track_all_table_in_schema = false;
let mut track_specific_columns_in_table = false;
match relations {
Some(relations) => {
for relation in relations.iter() {
@@ -187,12 +175,25 @@ where
));
}
if !track_all_table_in_schema && relation.table_to_track.is_empty() {
track_all_table_in_schema = true;
continue;
}
for table_to_track in relation.table_to_track.iter() {
if table_to_track.table_name.trim().is_empty() {
return Err(serde::de::Error::custom(
"Table name must not be empty".to_string(),
));
}
if !track_specific_columns_in_table && !table_to_track.columns_name.is_empty() {
track_specific_columns_in_table = true;
}
}
if track_all_table_in_schema && track_specific_columns_in_table {
return Err(serde::de::Error::custom("Incompatible tracking options. Schema-level tracking and specific table tracking with column selection cannot be used together. Refer to the documentation for valid configurations."));
}
}
@@ -249,13 +250,105 @@ pub struct SetEnabled {
pub enabled: bool,
}
#[derive(Serialize, Deserialize)]
pub struct PostgresPublicationReplication {
publication_name: String,
replication_slot_name: String,
}
impl PostgresPublicationReplication {
pub fn new(
publication_name: String,
replication_slot_name: String,
) -> PostgresPublicationReplication {
PostgresPublicationReplication { publication_name, replication_slot_name }
}
}
async fn check_if_publication_exist(
connection: &mut PgConnection,
publication_name: &str,
) -> Result<()> {
sqlx::query!(
"SELECT pubname FROM pg_publication WHERE pubname = $1",
publication_name
)
.fetch_one(connection)
.await
.map_err(|err| match err {
sqlx::Error::RowNotFound => {
Error::BadRequest(ERROR_PUBLICATION_NAME_NOT_EXISTS.to_string())
}
err => Error::SqlErr { error: err, location: "pg_trigger".to_string() },
})?;
Ok(())
}
async fn check_if_logical_replication_slot_exist(
connection: &mut PgConnection,
replication_slot_name: &str,
) -> Result<()> {
sqlx::query!(
"SELECT slot_name FROM pg_replication_slots where slot_name = $1",
&replication_slot_name
)
.fetch_one(connection)
.await
.map_err(|err| match err {
_ => Error::BadRequest(ERROR_REPLICATION_SLOT_NOT_EXISTS.to_string()),
})?;
Ok(())
}
async fn create_custom_slot_and_publication_inner(
authed: ApiAuthed,
user_db: UserDB,
db: &DB,
postgres_resource_path: &str,
w_id: &str,
publication: &PublicationData,
) -> Result<PostgresPublicationReplication> {
let publication_name = format!("windmill_trigger_{}", generate_random_string());
let replication_slot_name = publication_name.clone();
let query = create_publication_query(
&publication_name,
publication.table_to_track.as_deref(),
&publication
.transaction_to_track
.iter()
.map(AsRef::as_ref)
.collect_vec(),
);
let mut connection = get_database_connection(
authed.clone(),
Some(user_db.clone()),
&db,
&postgres_resource_path,
&w_id,
)
.await?;
sqlx::query(&query).execute(&mut connection).await?;
let query = create_logical_replication_slot_query(&replication_slot_name);
sqlx::query(&query).execute(&mut connection).await?;
Ok(PostgresPublicationReplication::new(
publication_name,
replication_slot_name,
))
}
pub async fn create_postgres_trigger(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Json(new_postgres_trigger): Json<NewPostgresTrigger>,
) -> error::Result<(StatusCode, String)> {
) -> Result<(StatusCode, String)> {
if *CLOUD_HOSTED {
return Err(error::Error::BadRequest(
"Postgres triggers are not supported on multi-tenant cloud, use dedicated cloud or self-host".to_string(),
@@ -278,59 +371,32 @@ pub async fn create_postgres_trigger(
"Publication data is missing".to_string(),
));
}
let (pub_name, slot_name) = if publication_name.is_none() && replication_slot_name.is_none() {
if publication.is_none() {
return Err(Error::BadRequest("publication must be set".to_string()));
}
let PostgresPublicationReplication { publication_name, replication_slot_name } =
create_custom_slot_and_publication_inner(
authed.clone(),
user_db.clone(),
&db,
&postgres_resource_path,
&w_id,
&publication.unwrap(),
)
.await?;
let create_slot = replication_slot_name.is_none();
let create_publication = publication_name.is_none();
let name;
let mut pub_name = publication_name.as_deref().unwrap_or_default();
let mut slot_name = replication_slot_name.as_deref().unwrap_or_default();
if create_publication || create_slot {
let generate_random_string = move || {
let timestamp = Utc::now().timestamp_millis().to_string();
let mut rng = rand::rng();
let charset = "abcdefghijklmnopqrstuvwxyz0123456789";
let random_part = (0..10)
.map(|_| {
charset
.chars()
.nth(rng.random_range(0..charset.len()))
.unwrap()
})
.collect::<String>();
format!("{}_{}", timestamp, random_part)
};
name = format!("windmill_{}", generate_random_string());
pub_name = &name;
slot_name = &name;
let publication = publication.unwrap();
let mut connection = get_database_connection(
authed.clone(),
Some(user_db.clone()),
&db,
&postgres_resource_path,
&w_id,
)
.await?;
new_publication(
&mut connection,
pub_name,
publication.table_to_track.as_deref(),
&publication
.transaction_to_track
.iter()
.map(AsRef::as_ref)
.collect_vec(),
)
.await?;
new_slot(&mut connection, slot_name).await?;
}
(publication_name, replication_slot_name)
} else {
if publication_name.is_none() {
return Err(Error::BadRequest("Missing publication name".to_string()));
} else if replication_slot_name.is_none() {
return Err(Error::BadRequest(
"Missing replication slot name".to_string(),
));
}
(replication_slot_name.unwrap(), publication_name.unwrap())
};
let mut tx = user_db.begin(&authed).await?;
@@ -450,10 +516,10 @@ pub async fn list_postgres_triggers(
#[derive(Deserialize, Serialize, Debug)]
pub struct PublicationData {
#[serde(default, deserialize_with = "check_if_not_duplication_relation")]
table_to_track: Option<Vec<Relations>>,
#[serde(default, deserialize_with = "check_if_valid_relation")]
pub table_to_track: Option<Vec<Relations>>,
#[serde(deserialize_with = "check_if_valid_transaction_type")]
transaction_to_track: Vec<String>,
pub transaction_to_track: Vec<String>,
}
fn check_if_valid_transaction_type<'de, D>(
@@ -509,7 +575,7 @@ pub async fn list_slot_name(
Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Path((w_id, postgres_resource_path)): Path<(String, String)>,
) -> error::Result<Json<Vec<SlotList>>> {
) -> Result<Json<Vec<SlotList>>> {
let mut connection = get_database_connection(
authed.clone(),
Some(user_db.clone()),
@@ -543,28 +609,13 @@ pub struct Slot {
name: String,
}
async fn new_slot(connection: &mut PgConnection, name: &str) -> error::Result<()> {
let query = format!(
r#"
SELECT
*
FROM
pg_create_logical_replication_slot({}, 'pgoutput');"#,
quote_literal(&name)
);
sqlx::query(&query).execute(connection).await?;
Ok(())
}
pub async fn create_slot(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Path((w_id, postgres_resource_path)): Path<(String, String)>,
Json(Slot { name }): Json<Slot>,
) -> error::Result<String> {
) -> Result<String> {
let mut connection = get_database_connection(
authed.clone(),
Some(user_db.clone()),
@@ -574,7 +625,9 @@ pub async fn create_slot(
)
.await?;
new_slot(&mut connection, &name).await?;
let query = create_logical_replication_slot_query(&name);
sqlx::query(&query).execute(&mut connection).await?;
Ok(format!("Slot {} created!", name))
}
@@ -585,7 +638,7 @@ pub async fn drop_slot_name(
Extension(db): Extension<DB>,
Path((w_id, postgres_resource_path)): Path<(String, String)>,
Json(Slot { name }): Json<Slot>,
) -> error::Result<String> {
) -> Result<String> {
let mut connection = get_database_connection(
authed.clone(),
Some(user_db.clone()),
@@ -595,7 +648,7 @@ pub async fn drop_slot_name(
)
.await?;
let query = format!("SELECT pg_drop_replication_slot({});", quote_literal(&name));
let query = drop_logical_replication_slot_query(&name);
sqlx::query(&query).execute(&mut connection).await?;
Ok(format!("Slot name {} deleted!", name))
@@ -610,7 +663,7 @@ pub async fn list_database_publication(
Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Path((w_id, postgres_resource_path)): Path<(String, String)>,
) -> error::Result<Json<Vec<String>>> {
) -> Result<Json<Vec<String>>> {
let mut connection = get_database_connection(
authed.clone(),
Some(user_db.clone()),
@@ -640,7 +693,7 @@ pub async fn get_publication_info(
Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>,
) -> error::Result<Json<PublicationData>> {
) -> Result<Json<PublicationData>> {
let mut connection = get_database_connection(
authed.clone(),
Some(user_db.clone()),
@@ -651,13 +704,13 @@ pub async fn get_publication_info(
.await?;
let publication_data =
get_publication_scope_and_transaction(&publication_name, &mut connection).await;
get_publication_scope_and_transaction(&mut connection, &publication_name).await;
let (all_table, transaction_to_track) = match publication_data {
let (all_table, transaction_to_track) = match publication_data {
Ok(pub_data) => pub_data,
Err(Error::SqlErr { error: sqlx::Error::RowNotFound, .. }) => {
return Err(Error::NotFound(
"Publication was not found, please create a new publication".to_string(),
ERROR_PUBLICATION_NAME_NOT_EXISTS.to_string(),
))
}
Err(e) => return Err(e),
@@ -674,82 +727,13 @@ pub async fn get_publication_info(
)))
}
async fn new_publication(
connection: &mut PgConnection,
publication_name: &str,
table_to_track: Option<&[Relations]>,
transaction_to_track: &[&str],
) -> Result<(), Error> {
let mut query = QueryBuilder::new("CREATE PUBLICATION ");
query.push(quote_identifier(publication_name));
match table_to_track {
Some(database_component) if !database_component.is_empty() => {
query.push(" FOR");
for (i, schema) in database_component.iter().enumerate() {
if schema.table_to_track.is_empty() {
query.push(" TABLES IN SCHEMA ");
query.push(quote_identifier(&schema.schema_name));
} else {
query.push(" TABLE ONLY ");
for (j, table) in schema.table_to_track.iter().enumerate() {
let table_name = quote_identifier(&table.table_name);
let schema_name = quote_identifier(&schema.schema_name);
let full_name = format!("{}.{}", &schema_name, &table_name);
query.push(full_name);
if !table.columns_name.is_empty() {
query.push(" (");
let columns = table
.columns_name
.iter()
.map(|column| quote_identifier(column))
.join(", ");
query.push(&columns);
query.push(")");
}
if let Some(where_clause) = &table.where_clause {
query.push(" WHERE (");
query.push(where_clause);
query.push(')');
}
if j + 1 != schema.table_to_track.len() {
query.push(", ");
}
}
}
if i < database_component.len() - 1 {
query.push(", ");
}
}
}
_ => {
query.push(" FOR ALL TABLES ");
}
};
if !transaction_to_track.is_empty() {
let transactions = || transaction_to_track.iter().join(", ");
query.push(" WITH (publish = '");
query.push(transactions());
query.push("');");
}
let query = query.build();
query.execute(&mut *connection).await?;
Ok(())
}
pub async fn create_publication(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>,
Json(publication_data): Json<PublicationData>,
) -> error::Result<String> {
) -> Result<String> {
let PublicationData { table_to_track, transaction_to_track } = publication_data;
let mut connection = get_database_connection(
@@ -761,13 +745,13 @@ pub async fn create_publication(
)
.await?;
new_publication(
&mut connection,
let query = create_publication_query(
&publication_name,
table_to_track.as_deref(),
&transaction_to_track.iter().map(AsRef::as_ref).collect_vec(),
)
.await?;
);
sqlx::query(&query).execute(&mut connection).await?;
Ok(format!(
"Publication {} successfully created!",
@@ -775,24 +759,12 @@ pub async fn create_publication(
))
}
async fn drop_publication(
publication_name: &str,
connection: &mut PgConnection,
) -> Result<(), Error> {
let mut query = QueryBuilder::new("DROP PUBLICATION IF EXISTS ");
let quoted_publication_name = quote_identifier(publication_name);
query.push(quoted_publication_name);
query.push(";");
query.build().execute(&mut *connection).await?;
Ok(())
}
pub async fn delete_publication(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>,
) -> error::Result<String> {
) -> Result<String> {
let mut connection = get_database_connection(
authed.clone(),
Some(user_db.clone()),
@@ -802,7 +774,9 @@ pub async fn delete_publication(
)
.await?;
drop_publication(&publication_name, &mut connection).await?;
let query = drop_publication_query(&publication_name);
sqlx::query(&query).execute(&mut connection).await?;
Ok(format!(
"Publication {} successfully deleted!",
@@ -810,65 +784,61 @@ pub async fn delete_publication(
))
}
async fn update_publication(
connection: &mut PgConnection,
pub fn get_update_publication_query(
publication_name: &str,
PublicationData { table_to_track, transaction_to_track }: PublicationData,
) -> error::Result<String> {
let (all_table, _) =
get_publication_scope_and_transaction(&publication_name, connection).await?;
let mut query = QueryBuilder::new("");
all_table: bool,
) -> Vec<String> {
let quoted_publication_name = quote_identifier(&publication_name);
let transaction_to_track_as_str = transaction_to_track.iter().join(",");
let mut queries = Vec::with_capacity(2);
match table_to_track {
Some(ref relations) if !relations.is_empty() => {
if all_table {
drop_publication(&publication_name, connection).await?;
new_publication(
connection,
queries.push(drop_publication_query(&publication_name));
queries.push(create_publication_query(
&publication_name,
table_to_track.as_deref(),
&transaction_to_track.iter().map(AsRef::as_ref).collect_vec(),
)
.await?;
));
} else {
query.push("ALTER PUBLICATION ");
query.push(&quoted_publication_name);
query.push(" SET");
let mut query = String::from("");
query.push_str("ALTER PUBLICATION ");
query.push_str(&quoted_publication_name);
query.push_str(" SET");
for (i, schema) in relations.iter().enumerate() {
if schema.table_to_track.is_empty() {
query.push(" TABLES IN SCHEMA ");
query.push_str(" TABLES IN SCHEMA ");
let quoted_schema = quote_identifier(&schema.schema_name);
query.push(&quoted_schema);
query.push_str(&quoted_schema);
} else {
query.push(" TABLE ONLY ");
query.push_str(" TABLE ONLY ");
for (j, table) in schema.table_to_track.iter().enumerate() {
let table_name = quote_identifier(&table.table_name);
let schema_name = quote_identifier(&schema.schema_name);
let full_name = format!("{}.{}", &schema_name, &table_name);
query.push(&full_name);
query.push_str(&full_name);
if !table.columns_name.is_empty() {
query.push(" (");
query.push_str(" (");
let columns = table
.columns_name
.iter()
.map(|column| quote_identifier(column))
.join(", ");
query.push(&columns);
query.push(") ");
query.push_str(&columns);
query.push_str(") ");
}
if let Some(where_clause) = &table.where_clause {
query.push(" WHERE (");
query.push(where_clause);
query.push_str(" WHERE (");
query.push_str(where_clause);
query.push(')');
}
if j + 1 != schema.table_to_track.len() {
query.push(", ");
query.push_str(", ");
}
}
}
@@ -876,36 +846,35 @@ async fn update_publication(
query.push(',');
}
}
query.push(";");
query.build().execute(&mut *connection).await?;
query.reset();
query.push("ALTER PUBLICATION ");
query.push(&quoted_publication_name);
query.push(format!(
query.push(';');
queries.push(query);
let mut query = String::new();
query.push_str("ALTER PUBLICATION ");
query.push_str(&quoted_publication_name);
query.push_str(&format!(
" SET (publish = '{}');",
transaction_to_track_as_str
));
queries.push(query);
}
}
_ => {
drop_publication(&publication_name, connection).await?;
queries.push(drop_publication_query(&publication_name));
let to_execute = format!(
r#"
CREATE
PUBLICATION {} FOR ALL TABLES WITH (publish = '{}')
PUBLICATION {} FOR ALL TABLES WITH (publish = '{}');
"#,
quoted_publication_name, transaction_to_track_as_str
);
query.push(&to_execute);
queries.push(to_execute);
}
};
query.build().execute(&mut *connection).await?;
Ok(format!(
"Publication {} successfully updated!",
publication_name
))
queries
}
pub async fn alter_publication(
@@ -914,7 +883,7 @@ pub async fn alter_publication(
Extension(db): Extension<DB>,
Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>,
Json(publication_data): Json<PublicationData>,
) -> error::Result<String> {
) -> Result<String> {
let mut connection = get_database_connection(
authed.clone(),
Some(user_db.clone()),
@@ -923,15 +892,28 @@ pub async fn alter_publication(
&w_id,
)
.await?;
let message = update_publication(&mut connection, &publication_name, publication_data).await?;
Ok(message)
check_if_publication_exist(&mut connection, &publication_name).await?;
let (all_table, _) =
get_publication_scope_and_transaction(&mut connection, &publication_name).await?;
let queries = get_update_publication_query(&publication_name, publication_data, all_table);
for query in queries {
sqlx::query(&query).execute(&mut connection).await?;
}
Ok(format!(
"Publication {} updated with success",
publication_name
))
}
async fn get_publication_scope_and_transaction(
publication_name: &str,
connection: &mut PgConnection,
) -> Result<(bool, Vec<String>), Error> {
publication_name: &str,
) -> std::result::Result<(bool, Vec<String>), Error> {
#[derive(Debug, Deserialize, FromRow)]
struct PublicationTransaction {
all_table: bool,
@@ -976,7 +958,7 @@ async fn get_publication_scope_and_transaction(
async fn get_tracked_relations(
connection: &mut PgConnection,
publication_name: &str,
) -> error::Result<Vec<Relations>> {
) -> Result<Vec<Relations>> {
#[derive(Debug, Deserialize, FromRow)]
struct PublicationData {
schema_name: Option<String>,
@@ -989,14 +971,18 @@ async fn get_tracked_relations(
PublicationData,
r#"
SELECT
schemaname AS schema_name,
tablename AS table_name,
attnames AS columns,
rowfilter AS where_clause
schemaname AS schema_name,
tablename AS table_name,
CASE
WHEN array_length(attnames, 1) = (SELECT COUNT(*) FROM information_schema.columns WHERE table_schema = pg_publication_tables.schemaname AND table_name = pg_publication_tables.tablename)
THEN NULL
ELSE attnames
END AS columns,
rowfilter AS where_clause
FROM
pg_publication_tables
WHERE
pubname = $1
pubname = $1;
"#,
publication_name
)
@@ -1011,7 +997,7 @@ async fn get_tracked_relations(
let table_to_track = TableToTrack::new(
publication.table_name.unwrap(),
publication.where_clause,
publication.columns.unwrap(),
publication.columns.unwrap_or_default(),
);
match entry {
Occupied(mut occuped) => {
@@ -1075,8 +1061,9 @@ pub async fn update_postgres_trigger(
Extension(db): Extension<DB>,
Path((w_id, path)): Path<(String, StripPath)>,
Json(postgres_trigger): Json<EditPostgresTrigger>,
) -> error::Result<String> {
) -> Result<String> {
let workspace_path = path.to_path();
let EditPostgresTrigger {
replication_slot_name,
publication_name,
@@ -1087,16 +1074,26 @@ pub async fn update_postgres_trigger(
publication,
} = postgres_trigger;
let mut connection = get_database_connection(
authed.clone(),
Some(user_db.clone()),
&db,
&postgres_resource_path,
&w_id,
)
.await?;
check_if_logical_replication_slot_exist(&mut connection, &replication_slot_name).await?;
if let Some(publication) = publication {
let mut connection = get_database_connection(
authed.clone(),
Some(user_db.clone()),
&db,
&postgres_resource_path,
&w_id,
)
.await?;
update_publication(&mut connection, &publication_name, publication).await?;
check_if_publication_exist(&mut connection, &publication_name).await?;
let (all_table, _) =
get_publication_scope_and_transaction(&mut connection, &publication_name).await?;
let queries = get_update_publication_query(&publication_name, publication, all_table);
for query in queries {
sqlx::query(&query).execute(&mut connection).await?;
}
}
let mut tx = user_db.begin(&authed).await?;
@@ -1153,7 +1150,7 @@ pub async fn delete_postgres_trigger(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> error::Result<String> {
) -> Result<String> {
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
sqlx::query!(
@@ -1213,7 +1210,7 @@ pub async fn set_enabled(
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
Json(payload): Json<SetEnabled>,
) -> error::Result<String> {
) -> Result<String> {
let mut tx = user_db.begin(&authed).await?;
let path = path.to_path();
@@ -1264,7 +1261,7 @@ pub async fn set_enabled(
))
}
pub async fn get_template_script(Path((_, id)): Path<(String, String)>) -> error::Result<String> {
pub async fn get_template_script(Path((_, id)): Path<(String, String)>) -> Result<String> {
let template = if let Some((_, template)) = TEMPLATE.remove(&id) {
template
} else {
@@ -1279,7 +1276,7 @@ pub async fn create_template_script(
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Json(template_script): Json<TemplateScript>,
) -> error::Result<String> {
) -> Result<String> {
let TemplateScript { postgres_resource_path, relations, language } = template_script;
if relations.is_none() {
return Err(Error::BadRequest(
@@ -1460,4 +1457,4 @@ pub async fn is_database_in_logical_level(
};
Ok(Json(is_logical))
}
}
backend/windmill-api/src/postgres_triggers/mod.rs
+172 -2
View File
@@ -4,8 +4,17 @@ use crate::{
resources::get_resource_value_interpolated_internal,
users::fetch_api_authed,
};
use chrono::Utc;
use itertools::Itertools;
use pg_escape::{quote_identifier, quote_literal};
use rand::Rng;
use serde_json::value::RawValue;
use sqlx::{
postgres::{PgConnectOptions, PgSslMode},
Connection, PgConnection,
};
use std::collections::HashMap;
use std::str::FromStr;
use axum::{
routing::{delete, get, post},
@@ -16,8 +25,10 @@ use handler::{
create_template_script, delete_postgres_trigger, delete_publication, drop_slot_name,
exists_postgres_trigger, get_postgres_trigger, get_publication_info, get_template_script,
is_database_in_logical_level, list_database_publication, list_postgres_triggers,
list_slot_name, set_enabled, update_postgres_trigger, Database,
list_slot_name, set_enabled, test_postgres_connection, update_postgres_trigger, Database,
Relations,
};
pub use handler::PostgresTrigger;
use windmill_common::{db::UserDB, error::Error, utils::StripPath};
use windmill_queue::PushArgsOwned;
@@ -30,9 +41,167 @@ mod relation;
mod replication_message;
mod trigger;
pub use handler::PostgresTrigger;
pub use handler::PublicationData;
pub use trigger::start_database;
const ERROR_REPLICATION_SLOT_NOT_EXISTS: &str = r#"The replication slot associated with this trigger no longer exists. Recreate a new replication slot or select an existing one in the advanced tab, or delete and recreate a new trigger"#;
const ERROR_PUBLICATION_NAME_NOT_EXISTS: &str = r#"The publication associated with this trigger no longer exists. Recreate a new publication or select an existing one in the advanced tab, or delete and recreate a new trigger"#;
pub async fn get_database_connection(
authed: ApiAuthed,
user_db: Option<UserDB>,
db: &DB,
postgres_resource_path: &str,
w_id: &str,
) -> std::result::Result<PgConnection, windmill_common::error::Error> {
let database = get_database_resource(authed, user_db, db, postgres_resource_path, w_id).await?;
Ok(get_raw_postgres_connection(&database).await?)
}
pub async fn get_raw_postgres_connection(
db: &Database,
) -> std::result::Result<PgConnection, Error> {
let options = {
let sslmode = if !db.sslmode.is_empty() {
PgSslMode::from_str(&db.sslmode)?
} else {
PgSslMode::Prefer
};
let options = PgConnectOptions::new()
.host(&db.host)
.database(&db.dbname)
.port(db.port)
.ssl_mode(sslmode)
.username(&db.user);
let options = if !db.root_certificate_pem.is_empty() {
options.ssl_root_cert_from_pem(db.root_certificate_pem.as_bytes().to_vec())
} else {
options
};
if !db.password.is_empty() {
options.password(&db.password)
} else {
options
}
};
Ok(PgConnection::connect_with(&options).await?)
}
pub fn create_logical_replication_slot_query(name: &str) -> String {
let query = format!(
r#"
SELECT
*
FROM
pg_create_logical_replication_slot({}, 'pgoutput');"#,
quote_literal(&name)
);
query
}
pub fn create_publication_query(
publication_name: &str,
table_to_track: Option<&[Relations]>,
transaction_to_track: &[&str],
) -> String {
let mut query = String::from("CREATE PUBLICATION ");
query.push_str(&quote_identifier(publication_name));
match table_to_track {
Some(database_component) if !database_component.is_empty() => {
query.push_str(" FOR");
for (i, schema) in database_component.iter().enumerate() {
if schema.table_to_track.is_empty() {
query.push_str(" TABLES IN SCHEMA ");
query.push_str(&quote_identifier(&schema.schema_name));
} else {
query.push_str(" TABLE ONLY ");
for (j, table) in schema.table_to_track.iter().enumerate() {
let table_name = quote_identifier(&table.table_name);
let schema_name = quote_identifier(&schema.schema_name);
let full_name = format!("{}.{}", &schema_name, &table_name);
query.push_str(&full_name);
if !table.columns_name.is_empty() {
query.push_str(" (");
let columns = table
.columns_name
.iter()
.map(|column| quote_identifier(column))
.join(", ");
query.push_str(&columns);
query.push_str(")");
}
if let Some(where_clause) = &table.where_clause {
query.push_str(" WHERE (");
query.push_str(where_clause);
query.push(')');
}
if j + 1 != schema.table_to_track.len() {
query.push_str(", ");
}
}
}
if i < database_component.len() - 1 {
query.push_str(", ");
}
}
}
_ => {
query.push_str(" FOR ALL TABLES ");
}
};
if !transaction_to_track.is_empty() {
let transactions = || transaction_to_track.iter().join(", ");
query.push_str(" WITH (publish = '");
query.push_str(&transactions());
query.push_str("');");
}
query
}
pub fn drop_publication_query(publication_name: &str) -> String {
let mut query = String::from("DROP PUBLICATION IF EXISTS ");
let quoted_publication_name = quote_identifier(publication_name);
query.push_str(&quoted_publication_name);
query.push_str(";");
query
}
pub fn drop_logical_replication_slot_query(replication_slot_name: &str) -> String {
format!(
"SELECT pg_drop_replication_slot({});",
quote_literal(&replication_slot_name)
)
}
pub fn generate_random_string() -> String {
let timestamp = Utc::now().timestamp_millis().to_string();
let mut rng = rand::rng();
let charset = "abcdefghijklmnopqrstuvwxyz0123456789";
let random_part = (0..10)
.map(|_| {
charset
.chars()
.nth(rng.random_range(0..charset.len()))
.unwrap()
})
.collect::<String>();
format!("{}_{}", timestamp, random_part)
}
pub async fn get_database_resource(
authed: ApiAuthed,
user_db: Option<UserDB>,
@@ -87,6 +256,7 @@ fn slot_service() -> Router {
pub fn workspaced_service() -> Router {
Router::new()
.route("/test", post(test_postgres_connection))
.route("/create", post(create_postgres_trigger))
.route("/list", get(list_postgres_triggers))
.route("/get/*path", get(get_postgres_trigger))
backend/windmill-api/src/postgres_triggers/trigger.rs
+628 -176
View File
@@ -1,7 +1,8 @@
use std::{collections::HashMap, pin::Pin};
use crate::{
db::DB,
capture::{insert_capture_payload, PostgresTriggerConfig, TriggerKind},
db::{ApiAuthed, DB},
postgres_triggers::{
get_database_resource,
relation::RelationConverter,
@@ -21,13 +22,20 @@ use pg_escape::{quote_identifier, quote_literal};
use rand::seq::SliceRandom;
use rust_postgres::{config::SslMode, Client, Config, CopyBothDuplex, SimpleQueryMessage};
use rust_postgres_native_tls::MakeTlsConnector;
use serde::Deserialize;
use serde_json::value::RawValue;
use sqlx::types::Json as SqlxJson;
use windmill_common::{
db::UserDB, utils::report_critical_error, worker::to_raw_value, INSTANCE_NAME,
db::UserDB, error, utils::report_critical_error, worker::to_raw_value, INSTANCE_NAME,
};
use windmill_queue::PushArgsOwned;
use super::{
drop_logical_replication_slot_query, drop_publication_query, get_database_connection,
handler::{Database, PostgresTrigger},
replication_message::PrimaryKeepAliveBody,
ERROR_PUBLICATION_NAME_NOT_EXISTS, ERROR_REPLICATION_SLOT_NOT_EXISTS,
};
pub struct LogicalReplicationSettings {
@@ -106,6 +114,7 @@ impl PostgresSimpleClient {
let connector = MakeTlsConnector::new(TlsConnector::new()?);
let (client, connection) = config.connect(connector).await?;
tokio::spawn(async move {
if let Err(e) = connection.await {
tracing::debug!("{:#?}", e);
@@ -116,6 +125,13 @@ impl PostgresSimpleClient {
Ok(PostgresSimpleClient(client))
}
async fn execute_query(
&self,
query: &str,
) -> Result<Vec<SimpleQueryMessage>, rust_postgres::Error> {
self.0.simple_query(query).await
}
async fn get_logical_replication_stream(
&self,
publication_name: &str,
@@ -162,75 +178,9 @@ impl PostgresSimpleClient {
}
}
async fn update_ping(
db: &DB,
postgres_trigger: &PostgresTrigger,
error: Option<&str>,
) -> Option<()> {
let updated = sqlx::query_scalar!(
r#"
UPDATE
postgres_trigger
SET
last_server_ping = now(),
error = $1
WHERE
workspace_id = $2
AND path = $3
AND server_id = $4
AND enabled IS TRUE
RETURNING 1
"#,
error,
&postgres_trigger.workspace_id,
&postgres_trigger.path,
*INSTANCE_NAME
)
.fetch_optional(db)
.await;
match updated {
Ok(updated) => {
if updated.flatten().is_none() {
// allow faster restart of database trigger
sqlx::query!(
r#"
UPDATE
postgres_trigger
SET
last_server_ping = NULL
WHERE
workspace_id = $1
AND path = $2
AND server_id IS NULL"#,
&postgres_trigger.workspace_id,
&postgres_trigger.path,
)
.execute(db)
.await
.ok();
tracing::info!(
"Postgres trigger {} changed, disabled, or deleted, stopping...",
postgres_trigger.path
);
return None;
}
}
Err(err) => {
tracing::warn!(
"Error updating ping of postgres trigger {}: {:?}",
postgres_trigger.path,
err
);
}
};
Some(())
}
async fn loop_ping(db: &DB, postgres_trigger: &PostgresTrigger, error: Option<&str>) {
async fn loop_ping(db: &DB, pg: &PostgresConfig, error: Option<&str>) {
loop {
if update_ping(db, postgres_trigger, error).await.is_none() {
if pg.update_ping(db, error).await.is_none() {
return;
}
@@ -238,78 +188,395 @@ async fn loop_ping(db: &DB, postgres_trigger: &PostgresTrigger, error: Option<&s
}
}
async fn disable_with_error(postgres_trigger: &PostgresTrigger, db: &DB, error: String) -> () {
match sqlx::query!(
"UPDATE postgres_trigger SET enabled = FALSE, error = $1, server_id = NULL, last_server_ping = NULL WHERE workspace_id = $2 AND path = $3",
error,
postgres_trigger.workspace_id,
postgres_trigger.path,
)
.execute(db).await {
Ok(_) => {
report_critical_error(format!("Disabling postgres trigger {} because of error: {}", postgres_trigger.path, error), db.clone(), Some(&postgres_trigger.workspace_id), None).await;
},
Err(disable_err) => {
report_critical_error(
format!("Could not disable postgres trigger {} with err {}, disabling because of error {}", postgres_trigger.path, disable_err, error),
db.clone(),
Some(&postgres_trigger.workspace_id),
None,
).await;
enum PostgresConfig {
Trigger(PostgresTrigger),
Capture(CaptureConfigForPostgresTrigger),
}
impl PostgresTrigger {
async fn try_to_listen_to_database_transactions(
self,
db: DB,
killpill_rx: tokio::sync::broadcast::Receiver<()>,
) -> () {
let postgres_trigger = sqlx::query_scalar!(
r#"
UPDATE postgres_trigger
SET
server_id = $1,
last_server_ping = now(),
error = 'Connecting...'
WHERE
enabled IS TRUE
AND workspace_id = $2
AND path = $3
AND (last_server_ping IS NULL
OR last_server_ping < now() - INTERVAL '15 seconds'
)
RETURNING true
"#,
*INSTANCE_NAME,
self.workspace_id,
self.path,
)
.fetch_optional(&db)
.await;
match postgres_trigger {
Ok(has_lock) => {
if has_lock.flatten().unwrap_or(false) {
tracing::info!("Spawning new task to listen_to_database_transaction");
tokio::spawn(async move {
listen_to_transactions(
PostgresConfig::Trigger(self),
db.clone(),
killpill_rx,
)
.await;
});
} else {
tracing::info!("Postgres trigger {} already being listened to", self.path);
}
}
Err(err) => {
tracing::error!(
"Error acquiring lock for postgres trigger {}: {:?}",
self.path,
err
);
}
};
}
async fn update_ping(&self, db: &DB, error: Option<&str>) -> Option<()> {
let updated = sqlx::query_scalar!(
r#"
UPDATE
postgres_trigger
SET
last_server_ping = now(),
error = $1
WHERE
workspace_id = $2
AND path = $3
AND server_id = $4
AND enabled IS TRUE
RETURNING 1
"#,
error,
&self.workspace_id,
&self.path,
*INSTANCE_NAME
)
.fetch_optional(db)
.await;
match updated {
Ok(updated) => {
if updated.flatten().is_none() {
// allow faster restart of database trigger
sqlx::query!(
r#"
UPDATE
postgres_trigger
SET
last_server_ping = NULL
WHERE
workspace_id = $1
AND path = $2
AND server_id IS NULL"#,
&self.workspace_id,
&self.path,
)
.execute(db)
.await
.ok();
tracing::info!(
"Postgres trigger {} changed, disabled, or deleted, stopping...",
self.path
);
return None;
}
}
Err(err) => {
tracing::warn!(
"Error updating ping of postgres trigger {}: {:?}",
self.path,
err
);
}
};
Some(())
}
async fn disable_with_error(&self, db: &DB, error: String) -> () {
match sqlx::query!(
r#"
UPDATE
postgres_trigger
SET
enabled = FALSE,
error = $1,
server_id = NULL,
last_server_ping = NULL
WHERE
workspace_id = $2 AND
path = $3
"#,
error,
self.workspace_id,
self.path,
)
.execute(db)
.await
{
Ok(_) => {
report_critical_error(
format!(
"Disabling postgres trigger {} because of error: {}",
self.path, error
),
db.clone(),
Some(&self.workspace_id),
None,
)
.await;
}
Err(disable_err) => {
report_critical_error(
format!("Could not disable postgres trigger {} with err {}, disabling because of error {}", self.path, disable_err, error),
db.clone(),
Some(&self.workspace_id),
None,
).await;
}
}
}
async fn fetch_authed(&self, db: &DB) -> error::Result<ApiAuthed> {
fetch_api_authed(
self.edited_by.clone(),
self.email.clone(),
&self.workspace_id,
db,
Some(format!("pg-{}", self.path)),
)
.await
}
async fn handle(
&self,
db: &DB,
args: Option<HashMap<String, Box<RawValue>>>,
extra: Option<HashMap<String, Box<RawValue>>>,
) -> () {
if let Err(err) = run_job(args, extra, db, self).await {
report_critical_error(
format!(
"Failed to trigger job from postgres {}: {:?}",
self.path, err
),
db.clone(),
Some(&self.workspace_id),
None,
)
.await;
};
}
}
async fn listen_to_transactions(
postgres_trigger: &PostgresTrigger,
db: DB,
mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
) {
let start_logical_replication_streaming = async {
let authed = fetch_api_authed(
postgres_trigger.edited_by.clone(),
postgres_trigger.email.clone(),
&postgres_trigger.workspace_id,
&db,
None,
)
.await?;
struct PgInfo<'a> {
postgres_resource_path: &'a str,
publication_name: &'a str,
replication_slot_name: &'a str,
workspace_id: &'a str,
}
impl PostgresConfig {
async fn update_ping(&self, db: &DB, error: Option<&str>) -> Option<()> {
match self {
PostgresConfig::Trigger(trigger) => trigger.update_ping(db, error).await,
PostgresConfig::Capture(capture) => capture.update_ping(db, error).await,
}
}
async fn disable_with_error(&self, db: &DB, error: String) -> () {
match self {
PostgresConfig::Trigger(trigger) => trigger.disable_with_error(&db, error).await,
PostgresConfig::Capture(capture) => capture.disable_with_error(db, error).await,
}
}
fn retrieve_info(&self) -> PgInfo {
let postgres_resource_path;
let publication_name;
let replication_slot_name;
let workspace_id;
match self {
PostgresConfig::Trigger(trigger) => {
postgres_resource_path = &trigger.postgres_resource_path;
publication_name = &trigger.publication_name;
replication_slot_name = &trigger.replication_slot_name;
workspace_id = &trigger.workspace_id;
}
PostgresConfig::Capture(capture) => {
postgres_resource_path = &capture.trigger_config.postgres_resource_path;
workspace_id = &capture.workspace_id;
publication_name = capture.trigger_config.publication_name.as_ref().unwrap();
replication_slot_name = capture
.trigger_config
.replication_slot_name
.as_ref()
.unwrap();
}
};
PgInfo { postgres_resource_path, replication_slot_name, workspace_id, publication_name }
}
async fn start_logical_replication_streaming(
&self,
db: &DB,
) -> std::result::Result<(CopyBothDuplex<Bytes>, LogicalReplicationSettings), Error> {
let PgInfo {
publication_name,
replication_slot_name,
workspace_id,
postgres_resource_path,
} = self.retrieve_info();
let authed = match self {
PostgresConfig::Trigger(trigger) => trigger.fetch_authed(db).await?,
PostgresConfig::Capture(capture) => capture.fetch_authed(db).await?,
};
let database = get_database_resource(
authed,
Some(UserDB::new(db.clone())),
&db,
&postgres_trigger.postgres_resource_path,
&postgres_trigger.workspace_id,
postgres_resource_path,
workspace_id,
)
.await?;
let client = PostgresSimpleClient::new(&database).await?;
let (logical_replication_stream, logical_replication_settings) = client
.get_logical_replication_stream(
&postgres_trigger.publication_name,
&postgres_trigger.replication_slot_name,
)
let publication = client
.execute_query(&format!(
"SELECT pubname FROM pg_publication WHERE pubname = {}",
quote_literal(&publication_name)
))
.await?;
Ok::<_, Error>((logical_replication_stream, logical_replication_settings))
};
if !publication.row_exist() {
return Err(Error::Common(error::Error::BadConfig(
ERROR_PUBLICATION_NAME_NOT_EXISTS.to_string(),
)));
}
let replication_slot = client
.execute_query(&format!(
"SELECT slot_name FROM pg_replication_slots WHERE slot_name = {}",
quote_literal(&replication_slot_name)
))
.await?;
if !replication_slot.row_exist() {
return Err(Error::Common(error::Error::BadConfig(
ERROR_REPLICATION_SLOT_NOT_EXISTS.to_string(),
)));
}
let (logical_replication_stream, logical_replication_settings) = client
.get_logical_replication_stream(&publication_name, &replication_slot_name)
.await?;
Ok((logical_replication_stream, logical_replication_settings))
}
fn get_path(&self) -> &str {
match self {
PostgresConfig::Trigger(trigger) => &trigger.path,
PostgresConfig::Capture(capture) => &capture.path,
}
}
async fn handle(
&self,
db: &DB,
args: Option<HashMap<String, Box<RawValue>>>,
extra: Option<HashMap<String, Box<RawValue>>>,
) -> () {
match self {
PostgresConfig::Trigger(trigger) => trigger.handle(&db, args, extra).await,
PostgresConfig::Capture(capture) => capture.handle(&db, args, extra).await,
}
}
async fn cleanup(&self, db: &DB) -> Result<(), Error> {
match self {
PostgresConfig::Trigger(_) => Ok(()),
PostgresConfig::Capture(capture) => {
let publication_name = capture.trigger_config.publication_name.as_ref().unwrap();
let replication_slot_name = capture
.trigger_config
.replication_slot_name
.as_ref()
.unwrap();
let postgres_resource_path = &capture.trigger_config.postgres_resource_path;
let workspace_id = &capture.workspace_id;
let authed = capture.fetch_authed(&db).await?;
let user_db = UserDB::new(db.clone());
let mut connection = get_database_connection(
authed.clone(),
Some(user_db.clone()),
&db,
postgres_resource_path,
workspace_id,
)
.await?;
let query = drop_logical_replication_slot_query(replication_slot_name);
let _ = sqlx::query(&query).execute(&mut connection).await;
let query = drop_publication_query(publication_name);
let _ = sqlx::query(&query).execute(&mut connection).await;
Ok(())
}
}
}
}
async fn listen_to_transactions(
pg: PostgresConfig,
db: DB,
mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
) {
tokio::select! {
biased;
_ = killpill_rx.recv() => {
let _ = pg.cleanup(&db).await;
return;
}
_ = loop_ping(&db, postgres_trigger, Some("Connecting...")) => {
_ = loop_ping(&db, &pg, Some("Connecting...")) => {
let _ = pg.cleanup(&db).await;
return;
}
result = start_logical_replication_streaming => {
result = pg.start_logical_replication_streaming(&db) => {
tokio::select! {
biased;
_ = killpill_rx.recv() => {
let _ = pg.cleanup(&db).await;
return;
}
_ = loop_ping(&db, postgres_trigger, None) => {
_ = loop_ping(&db, &pg, None) => {
let _ = pg.cleanup(&db).await;
return;
}
_ = {
@@ -318,15 +585,15 @@ async fn listen_to_transactions(
Ok((logical_replication_stream, logical_replication_settings)) => {
pin_mut!(logical_replication_stream);
let mut relations = RelationConverter::new();
tracing::info!("Starting to listen for postgres trigger {}", postgres_trigger.path);
tracing::info!("Starting to listen for postgres trigger {}", pg.get_path());
loop {
let message = logical_replication_stream.next().await;
let message = match message {
Some(message) => message,
None => {
tracing::error!("Stream for postgres trigger {} closed", postgres_trigger.path);
if let None = update_ping(&db, postgres_trigger, Some("Stream closed")).await {
tracing::error!("Stream for postgres trigger {} closed", pg.get_path());
if let None = pg.update_ping(&db, Some("Stream closed")).await {
return;
}
return;
@@ -336,8 +603,8 @@ async fn listen_to_transactions(
let message = match message {
Ok(message) => message,
Err(err) => {
let err = format!("Postgres trigger named {} had an error while receiving a message : {}", &postgres_trigger.path, err.to_string());
disable_with_error(&postgres_trigger, &db, err).await;
let err = format!("Postgres trigger named {} had an error while receiving a message : {}", pg.get_path(), err.to_string());
pg.disable_with_error(&db, err).await;
return;
}
};
@@ -345,8 +612,8 @@ async fn listen_to_transactions(
let logical_message = match ReplicationMessage::parse(message) {
Ok(logical_message) => logical_message,
Err(err) => {
let err = format!("Postgres trigger named: {} had an error while parsing message: {}", postgres_trigger.path, err.to_string());
disable_with_error(&postgres_trigger, &db, err).await;
let err = format!("Postgres trigger named: {} had an error while parsing message: {}", pg.get_path(), err.to_string());
pg.disable_with_error(&db, err).await;
return;
}
};
@@ -362,7 +629,7 @@ async fn listen_to_transactions(
let logical_replication_message = match x_log_data.parse(&logical_replication_settings) {
Ok(logical_replication_message) => logical_replication_message,
Err(err) => {
tracing::error!("Postgres trigger named: {} had an error while trying to parse incomming stream message: {}", &postgres_trigger.path, err.to_string());
tracing::error!("Postgres trigger named: {} had an error while trying to parse incomming stream message: {}", pg.get_path(), err.to_string());
continue;
}
};
@@ -390,7 +657,7 @@ async fn listen_to_transactions(
let relation = match relations.get_relation(o_id) {
Ok(relation) => relation,
Err(err) => {
tracing::error!("Postgres trigger named: {}, error: {}", &postgres_trigger.path, err.to_string());
tracing::error!("Postgres trigger named: {}, error: {}", pg.get_path(), err.to_string());
continue;
}
};
@@ -404,7 +671,9 @@ async fn listen_to_transactions(
"wm_trigger".to_string(),
to_raw_value(&serde_json::json!({"kind": "postgres", })),
)]));
let _ = run_job(Some(database_info), extra, &db, postgres_trigger).await;
let _ = pg.handle(&db, Some(database_info), extra).await;
}
}
@@ -413,11 +682,12 @@ async fn listen_to_transactions(
}
Err(err) => {
tracing::error!("Postgres trigger error while trying to start logical replication streaming: {}", &err);
disable_with_error(&postgres_trigger, &db, err.to_string()).await
pg.disable_with_error(&db, err.to_string()).await
}
}
}
} => {
let _ = pg.cleanup(&db).await;
return;
}
}
@@ -425,55 +695,204 @@ async fn listen_to_transactions(
}
}
async fn try_to_listen_to_database_transactions(
pg_trigger: PostgresTrigger,
db: DB,
killpill_rx: tokio::sync::broadcast::Receiver<()>,
) {
let postgres_trigger = sqlx::query_scalar!(
r#"
UPDATE postgres_trigger
SET
server_id = $1,
last_server_ping = now(),
error = 'Connecting...'
WHERE
enabled IS TRUE
AND workspace_id = $2
AND path = $3
AND (last_server_ping IS NULL
OR last_server_ping < now() - INTERVAL '15 seconds'
)
RETURNING true
"#,
*INSTANCE_NAME,
pg_trigger.workspace_id,
pg_trigger.path,
)
.fetch_optional(&db)
.await;
match postgres_trigger {
Ok(has_lock) => {
if has_lock.flatten().unwrap_or(false) {
tracing::info!("Spawning new task to listen_to_database_transaction");
tokio::spawn(async move {
listen_to_transactions(&pg_trigger, db.clone(), killpill_rx).await;
});
} else {
tracing::info!(
"Postgres trigger {} already being listened to",
pg_trigger.path
#[derive(Deserialize)]
struct CaptureConfigForPostgresTrigger {
trigger_config: SqlxJson<PostgresTriggerConfig>,
path: String,
is_flow: bool,
workspace_id: String,
owner: String,
email: String,
}
impl CaptureConfigForPostgresTrigger {
async fn try_to_listen_to_database_transactions(
self,
db: DB,
killpill_rx: tokio::sync::broadcast::Receiver<()>,
) -> () {
match sqlx::query_scalar!(
r#"
UPDATE
capture_config
SET
server_id = $1,
last_server_ping = now(),
error = 'Connecting...'
WHERE
last_client_ping > NOW() - INTERVAL '10 seconds' AND
workspace_id = $2 AND
path = $3 AND
is_flow = $4 AND
trigger_kind = 'postgres' AND
(last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')
RETURNING true
"#,
*INSTANCE_NAME,
self.workspace_id,
self.path,
self.is_flow,
)
.fetch_optional(&db)
.await
{
Ok(has_lock) => {
if has_lock.flatten().unwrap_or(false) {
tokio::spawn(listen_to_transactions(
PostgresConfig::Capture(self),
db,
killpill_rx,
));
} else {
tracing::info!("Postgres {} already being listened to", self.path);
}
}
Err(err) => {
tracing::error!(
"Error acquiring lock for capture postgres {}: {:?}",
self.path,
err
);
}
};
}
async fn update_ping(&self, db: &DB, error: Option<&str>) -> Option<()> {
match sqlx::query_scalar!(
r#"
UPDATE
capture_config
SET
last_server_ping = now(),
error = $1
WHERE
workspace_id = $2 AND
path = $3 AND
is_flow = $4 AND
trigger_kind = 'postgres' AND
server_id = $5 AND
last_client_ping > NOW() - INTERVAL '10 seconds'
RETURNING 1
"#,
error,
self.workspace_id,
self.path,
self.is_flow,
*INSTANCE_NAME
)
.fetch_optional(db)
.await
{
Ok(updated) => {
if updated.flatten().is_none() {
// allow faster restart of postgres capture
sqlx::query!(
r#"UPDATE
capture_config
SET
last_server_ping = NULL
WHERE
workspace_id = $1 AND
path = $2 AND
is_flow = $3 AND
trigger_kind = 'postgres' AND
server_id IS NULL
"#,
self.workspace_id,
self.path,
self.is_flow,
)
.execute(db)
.await
.ok();
tracing::info!(
"Postgres capture {} changed, disabled, or deleted, stopping...",
self.path
);
return None;
}
}
Err(err) => {
tracing::warn!(
"Error updating ping of capture postgres {}: {:?}",
self.path,
err
);
}
};
Some(())
}
async fn fetch_authed(&self, db: &DB) -> error::Result<ApiAuthed> {
fetch_api_authed(
self.owner.clone(),
self.email.clone(),
&self.workspace_id,
db,
Some(format!("postgres-{}", self.get_trigger_path())),
)
.await
}
fn get_trigger_path(&self) -> String {
format!(
"{}-{}",
if self.is_flow { "flow" } else { "script" },
self.path
)
}
async fn disable_with_error(&self, db: &DB, error: String) -> () {
if let Err(err) = sqlx::query!(
r#"
UPDATE
capture_config
SET
error = $1,
server_id = NULL,
last_server_ping = NULL
WHERE
workspace_id = $2 AND
path = $3 AND
is_flow = $4 AND
trigger_kind = 'postgres'
"#,
error,
self.workspace_id,
self.path,
self.is_flow,
)
.execute(db)
.await
{
tracing::error!("Could not disable postgres capture {} ({}) with err {}, disabling because of error {}", self.path, self.workspace_id, err, error);
}
Err(err) => {
tracing::error!(
"Error acquiring lock for postgres trigger {}: {:?}",
pg_trigger.path,
err
);
}
async fn handle(
&self,
db: &DB,
args: Option<HashMap<String, Box<RawValue>>>,
extra: Option<HashMap<String, Box<RawValue>>>,
) -> () {
let args = PushArgsOwned { args: args.unwrap_or_default(), extra };
let extra = args.extra.as_ref().map(to_raw_value);
if let Err(err) = insert_capture_payload(
db,
&self.workspace_id,
&self.path,
self.is_flow,
&TriggerKind::Postgres,
args,
extra,
&self.owner,
)
.await
{
tracing::error!("Error inserting capture payload: {:?}", err);
}
};
}
}
async fn listen_to_unlistened_database_events(
@@ -515,18 +934,51 @@ async fn listen_to_unlistened_database_events(
Ok(mut triggers) => {
triggers.shuffle(&mut rand::rng());
for trigger in triggers {
try_to_listen_to_database_transactions(
trigger,
db.clone(),
killpill_rx.resubscribe(),
)
.await;
trigger
.try_to_listen_to_database_transactions(db.clone(), killpill_rx.resubscribe())
.await;
}
}
Err(err) => {
tracing::error!("Error fetching postgres triggers: {:?}", err);
}
};
let postgres_triggers_capture = sqlx::query_as!(
CaptureConfigForPostgresTrigger,
r#"
SELECT
path,
is_flow,
workspace_id,
owner,
email,
trigger_config as "trigger_config!: _"
FROM
capture_config
WHERE
trigger_kind = 'postgres' AND
last_client_ping > NOW() - INTERVAL '10 seconds' AND
trigger_config IS NOT NULL AND
(last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')
"#
)
.fetch_all(db)
.await;
match postgres_triggers_capture {
Ok(mut captures) => {
captures.shuffle(&mut rand::rng());
for capture in captures {
capture
.try_to_listen_to_database_transactions(db.clone(), killpill_rx.resubscribe())
.await;
}
}
Err(err) => {
tracing::error!("Error fetching captures postgres triggers: {:?}", err);
}
};
}
pub fn start_database(db: DB, mut killpill_rx: tokio::sync::broadcast::Receiver<()>) {
backend/windmill-api/src/workspaces_export.rs
+91 -91
View File
@@ -336,8 +336,8 @@ pub(crate) async fn tarball_workspace(
{
let scripts = sqlx::query_as::<_, Script>(
"SELECT * FROM script as o WHERE workspace_id = $1 AND archived = false
AND created_at = (select max(created_at) from script where path = o.path AND \
workspace_id = $1)",
AND created_at = (select max(created_at) from script where path = o.path AND \
workspace_id = $1)",
)
.bind(&w_id)
.fetch_all(&mut *tx)
@@ -413,12 +413,12 @@ pub(crate) async fn tarball_workspace(
if !skip_resources.unwrap_or(false) {
let resources = sqlx::query_as!(
Resource,
"SELECT * FROM resource WHERE workspace_id = $1 AND resource_type != 'state' AND resource_type != 'cache'",
&w_id
)
.fetch_all(&mut *tx)
.await?;
Resource,
"SELECT * FROM resource WHERE workspace_id = $1 AND resource_type != 'state' AND resource_type != 'cache'",
&w_id
)
.fetch_all(&mut *tx)
.await?;
for resource in resources {
let resource_str = &to_string_without_metadata(&resource, false, None).unwrap();
@@ -450,14 +450,14 @@ pub(crate) async fn tarball_workspace(
{
let flows = sqlx::query_as::<_, Flow>(
"SELECT flow.workspace_id, flow.path, flow.summary, flow.description, flow.archived, flow.extra_perms, flow.draft_only, flow.dedicated_worker, flow.tag, flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, flow.on_behalf_of_email, flow_version.schema, flow_version.value, flow_version.created_at as edited_at, flow_version.created_by as edited_by
FROM flow
LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]
WHERE flow.workspace_id = $1 AND flow.archived = false",
)
.bind(&w_id)
.fetch_all(&mut *tx)
.await?;
"SELECT flow.workspace_id, flow.path, flow.summary, flow.description, flow.archived, flow.extra_perms, flow.draft_only, flow.dedicated_worker, flow.tag, flow.ws_error_handler_muted, flow.timeout, flow.visible_to_runner_only, flow.on_behalf_of_email, flow_version.schema, flow_version.value, flow_version.created_at as edited_at, flow_version.created_by as edited_by
FROM flow
LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]
WHERE flow.workspace_id = $1 AND flow.archived = false",
)
.bind(&w_id)
.fetch_all(&mut *tx)
.await?;
for flow in flows {
let flow_str = &to_string_without_metadata(&flow, false, None).unwrap();
@@ -469,14 +469,14 @@ pub(crate) async fn tarball_workspace(
if !skip_variables.unwrap_or(false) {
let variables =
sqlx::query_as::<_, ExportableListableVariable>(if !skip_secrets.unwrap_or(false) {
"SELECT * FROM variable WHERE workspace_id = $1 AND expires_at IS NULL"
} else {
"SELECT * FROM variable WHERE workspace_id = $1 AND is_secret = false AND expires_at IS NULL"
})
.bind(&w_id)
.fetch_all(&mut *tx)
.await?;
sqlx::query_as::<_, ExportableListableVariable>(if !skip_secrets.unwrap_or(false) {
"SELECT * FROM variable WHERE workspace_id = $1 AND expires_at IS NULL"
} else {
"SELECT * FROM variable WHERE workspace_id = $1 AND is_secret = false AND expires_at IS NULL"
})
.bind(&w_id)
.fetch_all(&mut *tx)
.await?;
let mc = build_crypt(&db, &w_id).await?;
@@ -496,14 +496,14 @@ pub(crate) async fn tarball_workspace(
{
let apps = sqlx::query_as::<_, AppWithLastVersion>(
"SELECT app.id, app.path, app.summary, app.versions, app.policy, app.custom_path,
app.extra_perms, app_version.value,
app_version.created_at, app_version.created_by from app, app_version
WHERE app.workspace_id = $1 AND app_version.id = app.versions[array_upper(app.versions, 1)]",
)
.bind(&w_id)
.fetch_all(&mut *tx)
.await?;
"SELECT app.id, app.path, app.summary, app.versions, app.policy, app.custom_path,
app.extra_perms, app_version.value,
app_version.created_at, app_version.created_by from app, app_version
WHERE app.workspace_id = $1 AND app_version.id = app.versions[array_upper(app.versions, 1)]",
)
.bind(&w_id)
.fetch_all(&mut *tx)
.await?;
for app in apps {
let app_str = &to_string_without_metadata(&app, false, None).unwrap();
@@ -516,7 +516,7 @@ pub(crate) async fn tarball_workspace(
if include_schedules.unwrap_or(false) {
let schedules = sqlx::query_as::<_, Schedule>(
"SELECT * FROM schedule
WHERE workspace_id = $1",
WHERE workspace_id = $1",
)
.bind(&w_id)
.fetch_all(&mut *tx)
@@ -534,13 +534,13 @@ pub(crate) async fn tarball_workspace(
#[cfg(feature = "http_trigger")]
{
let http_triggers = sqlx::query_as!(
crate::http_triggers::HttpTrigger,
"SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, edited_by, edited_at, email, extra_perms, is_async, requires_auth, http_method as \"http_method: _\", static_asset_config as \"static_asset_config: _\", is_static_website FROM http_trigger
WHERE workspace_id = $1",
&w_id
)
.fetch_all(&mut *tx)
.await?;
crate::http_triggers::HttpTrigger,
"SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, edited_by, edited_at, email, extra_perms, is_async, requires_auth, http_method as \"http_method: _\", static_asset_config as \"static_asset_config: _\", is_static_website FROM http_trigger
WHERE workspace_id = $1",
&w_id
)
.fetch_all(&mut *tx)
.await?;
for trigger in http_triggers {
let trigger_str = &to_string_without_metadata(&trigger, false, None).unwrap();
@@ -553,13 +553,13 @@ pub(crate) async fn tarball_workspace(
#[cfg(feature = "websocket")]
{
let websocket_triggers = sqlx::query_as!(
crate::websocket_triggers::WebsocketTrigger,
"SELECT workspace_id, path, url, script_path, is_flow, edited_by, email, edited_at, server_id, last_server_ping, extra_perms, error, enabled, filters as \"filters: _\", initial_messages as \"initial_messages: _\", url_runnable_args as \"url_runnable_args: _\", can_return_message FROM websocket_trigger
WHERE workspace_id = $1",
&w_id
)
.fetch_all(&mut *tx)
.await?;
crate::websocket_triggers::WebsocketTrigger,
"SELECT workspace_id, path, url, script_path, is_flow, edited_by, email, edited_at, server_id, last_server_ping, extra_perms, error, enabled, filters as \"filters: _\", initial_messages as \"initial_messages: _\", url_runnable_args as \"url_runnable_args: _\", can_return_message FROM websocket_trigger
WHERE workspace_id = $1",
&w_id
)
.fetch_all(&mut *tx)
.await?;
for trigger in websocket_triggers {
let trigger_str = &to_string_without_metadata(&trigger, false, None).unwrap();
@@ -577,7 +577,7 @@ pub(crate) async fn tarball_workspace(
let kafka_triggers = sqlx::query_as!(
crate::kafka_triggers_ee::KafkaTrigger,
"SELECT * FROM kafka_trigger
WHERE workspace_id = $1",
WHERE workspace_id = $1",
&w_id
)
.fetch_all(&mut *tx)
@@ -599,7 +599,7 @@ pub(crate) async fn tarball_workspace(
let nats_triggers = sqlx::query_as!(
crate::nats_triggers_ee::NatsTrigger,
"SELECT * FROM nats_trigger
WHERE workspace_id = $1",
WHERE workspace_id = $1",
&w_id
)
.fetch_all(&mut *tx)
@@ -619,7 +619,7 @@ pub(crate) async fn tarball_workspace(
let postgres_triggers = sqlx::query_as!(
crate::postgres_triggers::PostgresTrigger,
"SELECT * FROM postgres_trigger
WHERE workspace_id = $1",
WHERE workspace_id = $1",
&w_id
)
.fetch_all(&mut *tx)
@@ -640,7 +640,7 @@ pub(crate) async fn tarball_workspace(
if include_users.unwrap_or(false) {
let users = sqlx::query!(
"SELECT * FROM usr
WHERE workspace_id = $1",
WHERE workspace_id = $1",
&w_id
)
.fetch_all(&mut *tx)
@@ -668,16 +668,16 @@ pub(crate) async fn tarball_workspace(
if include_groups.unwrap_or(false) {
let groups = sqlx::query!(
r#"SELECT g_.workspace_id, name, summary, extra_perms, array_agg(u2g.usr) filter (where u2g.usr is not null) as members
FROM usr u
JOIN usr_to_group u2g ON u2g.usr = u.username AND u2g.workspace_id = u.workspace_id
RIGHT JOIN group_ g_ ON g_.workspace_id = u.workspace_id AND g_.name = u2g.group_
WHERE g_.workspace_id = $1 AND g_.name != 'all'
GROUP BY g_.workspace_id, name, summary, extra_perms"#,
&w_id
)
.fetch_all(&mut *tx)
.await?;
r#"SELECT g_.workspace_id, name, summary, extra_perms, array_agg(u2g.usr) filter (where u2g.usr is not null) as members
FROM usr u
JOIN usr_to_group u2g ON u2g.usr = u.username AND u2g.workspace_id = u.workspace_id
RIGHT JOIN group_ g_ ON g_.workspace_id = u.workspace_id AND g_.name = u2g.group_
WHERE g_.workspace_id = $1 AND g_.name != 'all'
GROUP BY g_.workspace_id, name, summary, extra_perms"#,
&w_id
)
.fetch_all(&mut *tx)
.await?;
for group in groups {
let extra_perms: HashMap<String, bool> = serde_json::from_value(group.extra_perms)
@@ -728,36 +728,36 @@ pub(crate) async fn tarball_workspace(
if include_settings.unwrap_or(false) {
let settings = sqlx::query_as!(
SimplifiedSettings,
r#"SELECT
-- slack_team_id,
-- slack_name,
-- slack_command_script,
-- CASE WHEN slack_email = 'missing@email.xyz' THEN NULL ELSE slack_email END AS slack_email,
auto_invite_domain IS NOT NULL AS "auto_invite_enabled!",
CASE WHEN auto_invite_operator IS TRUE THEN 'operator' ELSE 'developer' END AS "auto_invite_as!",
CASE WHEN auto_add IS TRUE THEN 'add' ELSE 'invite' END AS "auto_invite_mode!",
webhook,
deploy_to,
error_handler,
ai_resource,
ai_models,
code_completion_model,
error_handler_extra_args,
error_handler_muted_on_cancel,
large_file_storage,
git_sync,
default_app,
default_scripts,
workspace.name,
mute_critical_alerts,
color,
operator_settings
FROM workspace_settings
LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id
WHERE workspace_id = $1"#,
&w_id
).fetch_one(&mut *tx).await?;
SimplifiedSettings,
r#"SELECT
-- slack_team_id,
-- slack_name,
-- slack_command_script,
-- CASE WHEN slack_email = 'missing@email.xyz' THEN NULL ELSE slack_email END AS slack_email,
auto_invite_domain IS NOT NULL AS "auto_invite_enabled!",
CASE WHEN auto_invite_operator IS TRUE THEN 'operator' ELSE 'developer' END AS "auto_invite_as!",
CASE WHEN auto_add IS TRUE THEN 'add' ELSE 'invite' END AS "auto_invite_mode!",
webhook,
deploy_to,
error_handler,
ai_resource,
ai_models,
code_completion_model,
error_handler_extra_args,
error_handler_muted_on_cancel,
large_file_storage,
git_sync,
default_app,
default_scripts,
workspace.name,
mute_critical_alerts,
color,
operator_settings
FROM workspace_settings
LEFT JOIN workspace ON workspace.id = workspace_settings.workspace_id
WHERE workspace_id = $1"#,
&w_id
).fetch_one(&mut *tx).await?;
let settings_str = serde_json::to_value(settings)
.map(|v| serde_json::to_string_pretty(&v).ok())
frontend/src/lib/components/Label.svelte
+6 -4
View File
@@ -13,10 +13,12 @@
<div class="flex flex-row justify-between items-center w-full">
{#if !headless}
<div class="flex flex-row items-center gap-2">
<span class="{primary ? 'text-primary' : 'text-secondary'} text-sm leading-6">{label}</span>
{#if required}
<Required required={true} />
{/if}
<span class="{primary ? 'text-primary' : 'text-secondary'} text-sm leading-6"
>{label}
{#if required}
<Required required={true} />
{/if}
</span>
<slot name="header" />
</div>
{/if}
frontend/src/lib/components/Section.svelte
+2
View File
@@ -46,6 +46,8 @@
<slot name="header" />
{#if tooltip}
<Tooltip {documentationLink}>{tooltip}</Tooltip>
{:else if $$slots.tooltip}
<slot name="tooltip" />
{/if}
{#if eeOnly}
{#if !$enterpriseLicense}
frontend/src/lib/components/ShareModal.svelte
+1
View File
@@ -28,6 +28,7 @@
| 'websocket_trigger'
| 'kafka_trigger'
| 'nats_trigger'
| 'postgres_trigger'
let kind: Kind
let path: string = ''
frontend/src/lib/components/details/DetailPageTriggerPanel.svelte
+5 -2
View File
@@ -7,7 +7,10 @@
Terminal,
Webhook,
Unplug,
PlugZap
PlugZap,
Database
} from 'lucide-svelte'
import HighlightTheme from '../HighlightTheme.svelte'
@@ -69,7 +72,7 @@
</Tab>
<Tab value="postgres">
<span class="flex flex-row gap-2 items-center text-xs">
<Unplug size={12} />
<Database size={12} />
Postgres
</span>
</Tab>
frontend/src/lib/components/schema/AddPropertyFormV2.svelte
+3 -2
View File
@@ -5,6 +5,7 @@
import Popover from '$lib/components/meltComponents/Popover.svelte'
let name: string = ''
export let customName: string | undefined = undefined
const dispatch = createEventDispatcher()
@@ -24,7 +25,7 @@
<div class="flex flex-row gap-2 p-2 rounded-md">
<input
bind:value={name}
placeholder="Field name"
placeholder={`${customName ?? 'Field'} name`}
on:keydown={(event) => {
if (event.key === 'Enter') {
addField()
@@ -44,7 +45,7 @@
disabled={!name}
shortCut={{ Icon: CornerDownLeft, withoutModifier: true }}
>
Add field
Add {customName ? customName.toLowerCase() : 'field'}
</Button>
</div>
</svelte:fragment>
frontend/src/lib/components/triggers.ts
+2
View File
@@ -65,6 +65,8 @@ export function captureTriggerKindToTriggerKind(kind: CaptureTriggerKind): Trigg
return 'kafka'
case 'nats':
return 'nats'
case 'postgres':
return 'postgres'
default:
throw new Error(`Unknown CaptureTriggerKind: ${kind}`)
}
frontend/src/lib/components/triggers/CaptureButton.svelte
+10 -1
View File
@@ -1,7 +1,7 @@
<script lang="ts">
import Popover from '$lib/components/meltComponents/Popover.svelte'
import { Button } from '$lib/components/common'
import { Webhook, Route, Unplug, Mail, Plus } from 'lucide-svelte'
import { Webhook, Route, Unplug, Mail, Plus, Database } from 'lucide-svelte'
import KafkaIcon from '$lib/components/icons/KafkaIcon.svelte'
import { enterpriseLicense } from '$lib/stores'
import { type CaptureTriggerKind } from '$lib/gen'
@@ -82,6 +82,15 @@
<p class="text-xs text-secondary">Websocket</p>
</div>
</button>
<button
class="hover:bg-surface-hover p-2 transition-colors duration-150"
on:click={() => handleClick('postgres')}
>
<div class="flex flex-row items-center gap-2">
<Database size={16} />
<p class="text-xs text-secondary">Postgres</p>
</div>
</button>
<button
class="hover:bg-surface-hover p-2 transition-colors duration-150"
on:click={() => handleClick('email')}
frontend/src/lib/components/triggers/CaptureWrapper.svelte
+60 -21
View File
@@ -14,6 +14,8 @@
import type { CaptureInfo } from './CaptureSection.svelte'
import CaptureTable from './CaptureTable.svelte'
import NatsTriggersConfigSection from './nats/NatsTriggersConfigSection.svelte'
import PostgresEditorConfigSection from './postgres/PostgresEditorConfigSection.svelte'
import { invalidRelations } from './postgres/utils'
export let isFlow: boolean
export let path: string
@@ -26,16 +28,37 @@
export let args: Record<string, any> = {}
export let captureTable: CaptureTable | undefined = undefined
export async function setConfig() {
await CaptureService.setCaptureConfig({
requestBody: {
trigger_kind: captureType,
path,
is_flow: isFlow,
trigger_config: args && Object.keys(args).length > 0 ? args : undefined
},
workspace: $workspaceStore!
})
export async function setConfig(): Promise<boolean> {
if (captureType === 'postgres') {
if (!args?.publication?.table_to_track) {
sendUserToast('Table to track must be set', true)
return false
}
if (
invalidRelations(args.publication.table_to_track, {
showError: true,
trackSchemaTableError: true
}) === true
) {
return false
}
}
try {
await CaptureService.setCaptureConfig({
requestBody: {
trigger_kind: captureType,
path,
is_flow: isFlow,
trigger_config: args && Object.keys(args).length > 0 ? args : undefined
},
workspace: $workspaceStore!
})
return true
} catch (error) {
sendUserToast(error.body, true)
return false
}
}
let captureActive = false
@@ -55,7 +78,10 @@
return acc
}, {})
if ((captureType === 'websocket' || captureType === 'kafka') && captureActive) {
if (
(captureType === 'postgres' || captureType === 'websocket' || captureType === 'kafka') &&
captureActive
) {
const config = captureConfigs[captureType]
if (config && config.error) {
const serverEnabled = getServerEnabled(config)
@@ -114,24 +140,22 @@
if (captureActive || e.detail.disableOnly) {
captureActive = false
} else {
await setConfig()
capture()
const configSet = await setConfig()
if (configSet) {
capture()
}
}
}
let config: CaptureConfig | undefined
$: config = captureConfigs[captureType]
let cloudDisabled =
(captureType === 'websocket' || captureType === 'kafka' || captureType === 'nats') &&
isCloudHosted()
const streamingTrigger = ['postgres', 'websocket', 'kafka', 'nats']
let cloudDisabled = streamingTrigger.includes(captureType) && isCloudHosted()
function updateConnectionInfo(config: CaptureConfig | undefined, captureActive: boolean) {
if (
(captureType === 'websocket' || captureType === 'kafka' || captureType === 'nats') &&
config &&
captureActive
) {
if (streamingTrigger.includes(captureType) && config && captureActive) {
const serverEnabled = getServerEnabled(config)
const connected = serverEnabled && !config.error
const message = connected
@@ -181,6 +205,21 @@
on:captureToggle={handleCapture}
on:testWithArgs
/>
{:else if captureType === 'postgres'}
<PostgresEditorConfigSection
bind:postgres_resource_path={args.postgres_resource_path}
bind:publication={args.publication}
{showCapture}
{captureInfo}
can_write={true}
headless={true}
bind:captureTable
on:applyArgs
on:updateSchema
on:addPreprocessor
on:captureToggle={handleCapture}
on:testWithArgs
/>
{:else if captureType === 'webhook'}
<WebhooksConfigSection
{isFlow}
frontend/src/lib/components/triggers/TestTriggerConnection.svelte
+27 -17
View File
@@ -3,24 +3,27 @@
CancelablePromise,
KafkaTriggerService,
NatsTriggerService,
PostgresTriggerService,
WebsocketTriggerService
} from '$lib/gen'
import { workspaceStore } from '$lib/stores'
import { sendUserToast } from '$lib/toast'
import Button from '../common/button/Button.svelte'
export let kind: 'websocket' | 'nats' | 'kafka'
export let kind: 'websocket' | 'nats' | 'kafka' | 'postgres'
export let args: Record<string, any>
export let noButton = false
export let testLoading: boolean = false
const kindToName: { [key: string]: string } = {
websocket: 'WebSocket',
nats: 'NATS server(s)',
kafka: 'Kafka broker(s)'
kafka: 'Kafka broker(s)',
postgres: 'Postgres'
}
let testLoading: boolean = false
let promise: CancelablePromise<any> | null = null
async function testTriggerConnection() {
export async function testTriggerConnection() {
if (testLoading) {
promise?.cancel()
return
@@ -43,6 +46,11 @@
workspace: $workspaceStore!,
requestBody: args as any
})
} else if (kind === 'postgres') {
promise = PostgresTriggerService.testPostgresConnection({
workspace: $workspaceStore!,
requestBody: args as any
})
}
await promise
sendUserToast(`Successfully connected to ${kindToName[kind]}`)
@@ -56,16 +64,18 @@
}
</script>
<div class="flex flex-row justify-end mt-1">
<Button
spacingSize="sm"
size="xs"
color="light"
variant="border"
on:click={testTriggerConnection}
loading={testLoading}
clickableWhileLoading
>
Test connection
</Button>
</div>
{#if !noButton}
<div class="flex flex-row justify-end mt-1">
<Button
spacingSize="sm"
size="xs"
color="light"
variant="border"
on:click={testTriggerConnection}
loading={testLoading}
clickableWhileLoading
>
Test connection
</Button>
</div>
{/if}
frontend/src/lib/components/triggers/TriggersEditor.svelte
+12 -1
View File
@@ -140,7 +140,18 @@
</div>
{:else if $selectedTrigger === 'postgres'}
<div class="p-4">
<PostgresTriggersPanel {newItem} path={currentPath} {isFlow} />
<PostgresTriggersPanel
on:applyArgs
on:addPreprocessor
on:updateSchema
on:testWithArgs
{newItem}
path={currentPath}
{isFlow}
{canHavePreprocessor}
{hasPreprocessor}
isEditor={true}
/>
</div>
{:else if $selectedTrigger === 'kafka' || $selectedTrigger === 'nats'}
<div class="p-4 flex flex-col gap-2">
frontend/src/lib/components/triggers/TriggersEditorSection.svelte
+2 -1
View File
@@ -31,7 +31,8 @@
webhook: 'Webhook',
kafka: '+ New Kafka trigger',
email: 'Email trigger',
nats: '+ New NATS trigger'
nats: '+ New NATS trigger',
postgres: '+ New Postgres trigger'
}
const { captureOn } = getContext<TriggerContext>('TriggerContext')
frontend/src/lib/components/triggers/TriggersWrapper.svelte
+9
View File
@@ -8,6 +8,7 @@
import EmailTriggerConfigSection from '../details/EmailTriggerConfigSection.svelte'
import KafkaTriggersConfigSection from './kafka/KafkaTriggersConfigSection.svelte'
import NatsTriggersConfigSection from './nats/NatsTriggersConfigSection.svelte'
import PostgresEditorConfigSection from './postgres/PostgresEditorConfigSection.svelte'
export let triggerType: CaptureTriggerKind = 'webhook'
export let cloudDisabled: boolean = false
@@ -30,6 +31,14 @@
bind:url_runnable_args={args.url_runnable_args}
showCapture={false}
/>
{:else if triggerType === 'postgres'}
<PostgresEditorConfigSection
can_write={true}
headless={true}
showCapture={false}
bind:publication={args.publication}
bind:postgres_resource_path={args.postgres_resource_path}
/>
{:else if triggerType === 'webhook'}
<WebhooksConfigSection
{isFlow}
frontend/src/lib/components/triggers/postgres/CheckPostgresRequirement.svelte
+79
View File
@@ -0,0 +1,79 @@
<script lang="ts">
import { Button } from '$lib/components/common'
import Tooltip from '$lib/components/Tooltip.svelte'
import { PostgresTriggerService } from '$lib/gen'
import { workspaceStore } from '$lib/stores'
import { sendUserToast } from '$lib/toast'
import { emptyString } from '$lib/utils'
let loadingConfiguration = false
const checkDatabaseConfiguration = async () => {
if (emptyString(postgres_resource_path)) {
sendUserToast('You must first pick a database resource', true)
return
}
try {
const invalidConfig = !(await PostgresTriggerService.isValidPostgresConfiguration({
workspace: $workspaceStore!,
path: postgres_resource_path
}))
let msg = 'Database is in logical mode. Triggers can be used.'
if (invalidConfig) {
msg =
'Database is NOT in logical mode. Triggers cannot be used. Refer to the PostgreSQL documentation for configuration requirements.'
}
sendUserToast(msg, invalidConfig)
} catch (error) {
sendUserToast(error.body, true)
}
loadingConfiguration = false
}
const checkConnectionAndDatabaseConfiguration = async () => {
try {
loadingConfiguration = true
if (checkConnection) {
await checkConnection()
}
await checkDatabaseConfiguration()
} catch (error) {
sendUserToast(error.body, true)
}
loadingConfiguration = false
}
export let can_write: boolean
export let postgres_resource_path: string
export let checkConnection: any | undefined = undefined
console.log('dbg check connection', checkConnection)
</script>
{#if postgres_resource_path}
<div class="flex flex-col justify-end mt-1 gap-2">
<Button
disabled={!can_write}
loading={loadingConfiguration}
on:click={checkConnectionAndDatabaseConfiguration}
size="xs"
color="light"
spacingSize="sm"
variant="border"
>
{`Check database configuration ${checkConnection ? 'and connection' : ''}`}
<Tooltip
documentationLink="https://www.windmill.dev/docs/core_concepts/postgres_triggers#requirements"
>
<p class="text-sm">
Verifies whether the database is configured with the required <strong>settings</strong>.
{checkConnection && 'Also checks whether the connection to the database is working.'}
</p>
</Tooltip>
</Button>
</div>
{/if}
frontend/src/lib/components/triggers/postgres/PostgresEditorConfigSection.svelte
+158
View File
@@ -0,0 +1,158 @@
<script lang="ts">
import ResourcePicker from '$lib/components/ResourcePicker.svelte'
import Section from '$lib/components/Section.svelte'
import Label from '$lib/components/Label.svelte'
import CaptureSection, { type CaptureInfo } from '../CaptureSection.svelte'
import CaptureTable from '../CaptureTable.svelte'
import TestTriggerConnection from '../TestTriggerConnection.svelte'
import Required from '$lib/components/Required.svelte'
import MultiSelect from 'svelte-multiselect'
import RelationPicker from './RelationPicker.svelte'
import type { PublicationData } from '$lib/gen'
import { emptyString } from '$lib/utils'
import CheckPostgresRequirement from './CheckPostgresRequirement.svelte'
import Tooltip from '$lib/components/Tooltip.svelte'
import { X } from 'lucide-svelte'
const DEFAULT_PUBLICATION: PublicationData = {
transaction_to_track: ['Insert', 'Update', 'Delete'],
table_to_track: [
{
schema_name: 'public',
table_to_track: []
}
]
}
let transactionType: string[] = ['Insert', 'Update', 'Delete']
export let headless: boolean = false
export let can_write: boolean = false
export let showCapture: boolean = false
export let captureTable: CaptureTable | undefined = undefined
export let captureInfo: CaptureInfo | undefined = undefined
export let isValid: boolean = false
export let postgres_resource_path: string = ''
export let publication: PublicationData = {
transaction_to_track: ['Insert', 'Update', 'Delete'],
table_to_track: [
{
schema_name: 'public',
table_to_track: []
}
]
}
let selectedTable: 'all' | 'specific' = 'all'
function updateConfig(publication: PublicationData) {
if (publication === undefined) {
publication = { ...DEFAULT_PUBLICATION }
selectedTable = 'specific'
} else {
selectedTable =
publication.table_to_track && publication.table_to_track.length > 0 ? 'specific' : 'all'
}
const notEmpty = publication.table_to_track && publication.table_to_track.length > 0
selectedTable = notEmpty ? 'specific' : 'all'
isValid =
!emptyString(postgres_resource_path) &&
publication.transaction_to_track.length > 0 &&
(selectedTable === 'all' || (notEmpty ?? false))
}
$: updateConfig(publication)
let testTriggerConnection: TestTriggerConnection | undefined = undefined
</script>
<div class="h-full">
{#if showCapture && captureInfo}
<CaptureSection
disabled={!isValid}
on:captureToggle
captureType="postgres"
{captureInfo}
on:applyArgs
on:updateSchema
on:addPreprocessor
on:testWithArgs
bind:captureTable
/>
{/if}
<Section label="Postgres config" {headless}>
<div class="flex flex-col gap-4">
<div class="mb-2">
<p class="text-xs mb-1 text-tertiary">
Pick a database to connect to <Required required={true} />
</p>
<ResourcePicker
disabled={!can_write}
bind:value={postgres_resource_path}
resourceType={'postgresql'}
on:change={() => {
if (emptyString(postgres_resource_path)) {
selectedTable = 'specific'
publication = { ...DEFAULT_PUBLICATION }
}
}}
/>
{#if postgres_resource_path}
<TestTriggerConnection
kind="postgres"
args={{ postgres_resource_path }}
noButton
bind:this={testTriggerConnection}
/>
<CheckPostgresRequirement
bind:postgres_resource_path
bind:can_write
checkConnection={testTriggerConnection?.testTriggerConnection}
/>
{/if}
</div>
{#if postgres_resource_path}
<Label label="Transactions">
<svelte:fragment slot="header">
<Tooltip small>
Choose the types of database transactions that should trigger a script or flow. You
can select from <strong>Insert</strong>, <strong>Update</strong>,
<strong>Delete</strong>, or any combination of these operations to define when the
trigger should activate.
</Tooltip>
</svelte:fragment>
<MultiSelect
noMatchingOptionsMsg=""
createOptionMsg={null}
duplicates={false}
options={transactionType}
allowUserOptions="append"
bind:selected={publication.transaction_to_track}
ulOptionsClass={'!bg-surface !text-sm'}
ulSelectedClass="!text-sm"
outerDivClass="!bg-surface !min-h-[38px] !border-[#d1d5db]"
placeholder="Select transactions"
--sms-options-margin="4px"
>
<svelte:fragment slot="remove-icon">
<div class="hover:text-primary p-0.5">
<X size={12} />
</div>
</svelte:fragment>
</MultiSelect>
</Label>
<Label label="Table tracking">
<svelte:fragment slot="header">
<Tooltip small>
Select the tables to track. You can choose to track
<strong>all tables in your database</strong>,
<strong>all tables within a specific schema</strong>,
<strong>specific tables in a schema</strong>, or even
<strong>specific columns of a table</strong>. Additionally, you can apply a
<strong>filter</strong> to retrieve only rows that do not match the specified criteria.
</Tooltip>
</svelte:fragment>
<RelationPicker bind:selectedTable bind:relations={publication.table_to_track} />
</Label>
{/if}
</div>
</Section>
</div>
frontend/src/lib/components/triggers/postgres/PostgresTriggerEditor.svelte
+2 -2
View File
@@ -9,10 +9,10 @@
drawer?.openEdit(ePath, isFlow)
}
export async function openNew(is_flow: boolean, initial_script_path?: string) {
export async function openNew(is_flow: boolean, initial_script_path?: string, defaultValues?: Record<string, any>) {
open = true
await tick()
drawer?.openNew(is_flow, initial_script_path)
drawer?.openNew(is_flow, initial_script_path, defaultValues)
}
let drawer: PostgresTriggerEditorInner
frontend/src/lib/components/triggers/postgres/PostgresTriggerEditorInner.svelte
+122 -145
View File
@@ -10,7 +10,7 @@
import { canWrite, emptyString, emptyStringTrimmed, sendUserToast } from '$lib/utils'
import { createEventDispatcher } from 'svelte'
import Section from '$lib/components/Section.svelte'
import { Loader2, Save } from 'lucide-svelte'
import { Loader2, Save, X } from 'lucide-svelte'
import Label from '$lib/components/Label.svelte'
import Toggle from '$lib/components/Toggle.svelte'
import ResourcePicker from '$lib/components/ResourcePicker.svelte'
@@ -25,6 +25,8 @@
import Tabs from '$lib/components/common/tabs/Tabs.svelte'
import Tab from '$lib/components/common/tabs/Tab.svelte'
import RelationPicker from './RelationPicker.svelte'
import { invalidRelations } from './utils'
import CheckPostgresRequirement from './CheckPostgresRequirement.svelte'
let drawer: Drawer
let is_flow: boolean = false
@@ -53,13 +55,7 @@
let publicationItems: string[] = []
let transactionType: string[] = ['Insert', 'Update', 'Delete']
let selectedTable: 'all' | 'specific' = 'specific'
let tab: 'advanced' | 'basic'
let config: { isLogical: boolean; show: boolean } = { isLogical: false, show: false }
let loadingConfiguration = false
$: table_to_track = selectedTable === 'all' ? [] : relations
$: if (postgres_resource_path === undefined) {
config.show = false
}
let tab: 'advanced' | 'basic' = 'basic'
async function createPublication() {
try {
const message = await PostgresTriggerService.createPostgresPublication({
@@ -68,7 +64,7 @@
workspace: $workspaceStore!,
requestBody: {
transaction_to_track: transaction_to_track,
table_to_track
table_to_track: relations
}
})
@@ -107,7 +103,6 @@
dirtyPath = false
selectedPublicationAction = 'get'
selectedSlotAction = 'get'
config.show = false
selectedPublicationAction = selectedPublicationAction
selectedSlotAction = selectedSlotAction
relations = []
@@ -121,7 +116,11 @@
}
}
export async function openNew(nis_flow: boolean, fixedScriptPath_?: string) {
export async function openNew(
nis_flow: boolean,
fixedScriptPath_?: string,
defaultValues?: Record<string, any>
) {
drawerLoading = true
try {
selectedPublicationAction = 'create'
@@ -137,16 +136,17 @@
script_path = fixedScriptPath
path = ''
initialPath = ''
replication_slot_name = ''
publication_name = ''
postgres_resource_path = ''
postgres_resource_path = defaultValues?.postgres_resource_path ?? ''
edit = false
dirtyPath = false
config.show = false
publication_name = `windmill_publication_${random_adj()}`
replication_slot_name = `windmill_replication_${random_adj()}`
transaction_to_track = ['Insert', 'Update', 'Delete']
relations = [
transaction_to_track = defaultValues?.publication.transaction_to_track || [
'Insert',
'Update',
'Delete'
]
relations = defaultValues?.publication.table_to_track || [
{
schema_name: 'public',
table_to_track: []
@@ -184,6 +184,15 @@
}
async function updateTrigger(): Promise<void> {
if (
selectedTable === 'specific' &&
invalidRelations(relations, {
showError: true,
trackSchemaTableError: true
}) === true
) {
return
}
if (edit) {
await PostgresTriggerService.updatePostgresTrigger({
workspace: $workspaceStore!,
@@ -200,7 +209,7 @@
tab === 'basic'
? {
transaction_to_track,
table_to_track
table_to_track: relations
}
: undefined
}
@@ -219,7 +228,7 @@
publication_name: tab === 'basic' ? undefined : publication_name,
publication: {
transaction_to_track,
table_to_track
table_to_track: relations
}
}
})
@@ -256,24 +265,6 @@
sendUserToast(error.body, true)
}
}
const checkDatabaseConfiguration = async () => {
if (emptyString(postgres_resource_path)) {
sendUserToast('You must first pick a database resource', true)
return
}
try {
loadingConfiguration = true
config.isLogical = await PostgresTriggerService.isValidPostgresConfiguration({
workspace: $workspaceStore!,
path: postgres_resource_path
})
config.show = true
} catch (error) {
sendUserToast(error.body, true)
}
loadingConfiguration = false
}
</script>
<Drawer size="800px" bind:this={drawer}>
@@ -311,9 +302,10 @@
disabled={pathError != '' ||
emptyString(postgres_resource_path) ||
emptyString(script_path) ||
((emptyString(replication_slot_name) || emptyString(publication_name)) &&
tab === 'advanced') ||
(relations.length === 0 && tab === 'basic') ||
(tab === 'advanced' && emptyString(replication_slot_name)) ||
emptyString(publication_name) ||
(selectedTable !== 'all' && tab === 'basic' && relations.length === 0) ||
transaction_to_track.length === 0 ||
!can_write}
on:click={updateTrigger}
>
@@ -327,74 +319,28 @@
<p>Loading...</p>
</div>
{:else}
<div class="flex flex-col gap-5">
<Alert title="Info" type="info">
{#if edit}
Changes can take up to 30 seconds to take effect.
{:else}
New postgres triggers can take up to 30 seconds to start listening.
{/if}
</Alert>
</div>
<Alert title="Info" type="info">
{#if edit}
Changes can take up to 30 seconds to take effect.
{:else}
New postgres triggers can take up to 30 seconds to start listening.
{/if}
</Alert>
<div class="flex flex-col gap-12 mt-6">
<div class="flex flex-col gap-4">
<Label label="Path">
<Path
bind:dirty={dirtyPath}
bind:error={pathError}
bind:path
{initialPath}
checkInitialPathExistence={!edit}
namePlaceholder="postgres_trigger"
kind="postgres_trigger"
disabled={!can_write}
/>
</Label>
</div>
<Section label="Database">
<p class="text-xs mb-1 text-tertiary">
Pick a database to connect to <Required required={true} />
</p>
<div class="flex flex-col mb-2 gap-3">
<ResourcePicker bind:value={postgres_resource_path} resourceType={'postgresql'} />
{#if postgres_resource_path}
<Button
loading={loadingConfiguration}
on:click={checkDatabaseConfiguration}
color="gray"
size="sm"
>Check Database Configuration
<Tooltip>
<p class="text-sm">
Verifies whether the database is configured with the required <strong
>settings</strong
>.<br /> The <strong>logical wal_level</strong> setting is essential for the streaming
feature to works. If it is not set, the trigger feature will not work, and the database
configuration must be updated.
</p>
</Tooltip>
</Button>
{#if config.show}
<Alert
title="Postgres configuration"
type={config.isLogical === true ? 'success' : 'error'}
>
{#if config.isLogical}
Your database is correctly configured with logical replication enabled. You can
proceed with using the streaming feature
{:else}
Logical replication is not enabled on your database. To use this feature, your
Postgres database must have <code>wal_level</code> configured as 'logical' in your
database configuration.
{/if}
</Alert>
{/if}
{/if}
</div>
</Section>
<Label label="Path">
<Path
bind:dirty={dirtyPath}
bind:error={pathError}
bind:path
{initialPath}
checkInitialPathExistence={!edit}
namePlaceholder="postgres_trigger"
kind="postgres_trigger"
disabled={!can_write}
/>
</Label>
<Section label="Runnable">
<p class="text-xs mb-1 text-tertiary">
<p class="text-xs text-tertiary">
Pick a script or flow to be triggered <Required required={true} />
</p>
<div class="flex flex-row mb-2">
@@ -411,6 +357,7 @@
{#if script_path === undefined && is_flow === false}
<div class="flex">
<Button
disabled={!can_write}
btnClasses="ml-4 mt-2"
color="dark"
size="xs"
@@ -429,42 +376,66 @@
{/if}
</div>
</Section>
{#if postgres_resource_path}
<Section label="Configuration">
<div class="flex flex-col gap-5">
<p class="text-xs mb-3 text-tertiary">
Choose which table of your database to track as well as what kind of transaction
should fire the script.<br />
You must pick a database resource first to make the configuration of your trigger
<Required required={true} />
</p>
<Section label="Transactions">
<p class="text-xs mb-3 text-tertiary">
Choose the types of database transactions that should trigger a script or flow.
You can select from <strong>Insert</strong>, <strong>Update</strong>,
<strong>Delete</strong>, or any combination of these operations to define when the
trigger should activate.
</p>
<Section label="Database">
<p class="text-xs text-tertiary mb-2">
Pick a database to connect to <Required required={true} />
</p>
<div class="flex flex-col gap-8">
<div class="flex flex-col gap-2">
<ResourcePicker
disabled={!can_write}
bind:value={postgres_resource_path}
resourceType={'postgresql'}
/>
<CheckPostgresRequirement bind:postgres_resource_path bind:can_write />
</div>
{#if postgres_resource_path}
<Label label="Transactions">
<svelte:fragment slot="header">
<Tooltip>
<p>
Choose the types of database transactions that should trigger a script or
flow. You can select from <strong>Insert</strong>, <strong>Update</strong>,
<strong>Delete</strong>, or any combination of these operations to define when
the trigger should activate.
</p>
</Tooltip>
</svelte:fragment>
<MultiSelect
ulOptionsClass={'!bg-surface-secondary'}
noMatchingOptionsMsg=""
createOptionMsg={null}
duplicates={false}
bind:value={transaction_to_track}
options={transactionType}
allowUserOptions="append"
bind:selected={transaction_to_track}
/>
</Section>
<Section label="Table Tracking">
<p class="text-xs mb-3 text-tertiary">
Select the tables to track. You can choose to track
<strong>all tables in your database</strong>,
<strong>all tables within a specific schema</strong>,
<strong>specific tables in a schema</strong>, or even
<strong>specific columns of a table</strong>. Additionally, you can apply a
<strong>filter</strong> to retrieve only rows that do not match the specified criteria.
</p>
ulOptionsClass={'!bg-surface !text-sm'}
ulSelectedClass="!text-sm"
outerDivClass="!bg-surface !min-h-[38px] !border-[#d1d5db]"
placeholder="Select transactions"
--sms-options-margin="4px"
--sms-open-z-index="100"
>
<svelte:fragment slot="remove-icon">
<div class="hover:text-primary p-0.5">
<X size={12} />
</div>
</svelte:fragment>
</MultiSelect>
</Label>
<Label label="Table Tracking">
<svelte:fragment slot="header">
<Tooltip>
<p>
Select the tables to track. You can choose to track
<strong>all tables in your database</strong>,
<strong>all tables within a specific schema</strong>,
<strong>specific tables in a schema</strong>, or even
<strong>specific columns of a table</strong>. Additionally, you can apply a
<strong>filter</strong> to retrieve only rows that do not match the specified criteria.
</p>
</Tooltip>
</svelte:fragment>
<Tabs bind:selected={tab}>
<Tab value="basic"
><div class="flex flex-row gap-1"
@@ -500,12 +471,13 @@
<svelte:fragment slot="content">
<div class="mt-5 overflow-hidden bg-surface">
<TabContent value="basic">
<RelationPicker bind:selectedTable bind:relations />
<RelationPicker {can_write} bind:selectedTable bind:relations />
</TabContent>
<TabContent value="advanced">
<div class="flex flex-col gap-6"
><Section
label="Replication slot management"
small
label="Replication slot"
tooltip="Choose and manage the slots for your trigger. You can create or delete slots. Both non-active slots and the currently used slot by the trigger (if any) will be retrieved from your database for management."
documentationLink="https://www.windmill.dev/docs/core_concepts/postgres_triggers#managing-postgres-replication-slots"
>
@@ -515,6 +487,7 @@
on:selected={() => {
replication_slot_name = ''
}}
disabled={!can_write}
>
<ToggleButton value="create" label="Create Slot" />
<ToggleButton value="get" label="Get Slot" />
@@ -545,13 +518,15 @@
</Section>
<Section
label="Publication management"
small
label="Publication"
tooltip="Select and manage the publications for tracking data. You can create, update, or delete publications. Only existing publications in your database will be available for selection, giving you full control over what data is tracked."
documentationLink="https://www.windmill.dev/docs/core_concepts/postgres_triggers#managing-postgres-publications"
>
<div class="flex flex-col gap-3">
<ToggleButtonGroup
bind:selected={selectedPublicationAction}
disabled={!can_write}
on:selected={() => {
if (selectedPublicationAction === 'create') {
selectedTable = 'specific'
@@ -571,6 +546,7 @@
{#if selectedPublicationAction === 'create'}
<div class="flex gap-3">
<input
disabled={!can_write}
type="text"
bind:value={publication_name}
placeholder={'Publication Name'}
@@ -580,22 +556,23 @@
size="xs"
variant="border"
disabled={emptyStringTrimmed(publication_name) ||
(selectedTable != 'all' && relations.length === 0)}
(selectedTable != 'all' && relations.length === 0) ||
!can_write}
on:click={createPublication}>Create</Button
>
</div>
{:else}
<PublicationPicker
{can_write}
{postgres_resource_path}
bind:transaction_to_track
bind:table_to_track
bind:relations
bind:items={publicationItems}
bind:publication_name
bind:selectedTable
/>
{/if}
<RelationPicker bind:selectedTable bind:relations />
<RelationPicker {can_write} bind:selectedTable bind:relations />
</div>
</Section></div
>
@@ -603,10 +580,10 @@
</div>
</svelte:fragment>
</Tabs>
</Section>
</div>
</Section>
{/if}
</Label>
{/if}
</div>
</Section>
</div>
{/if}
</DrawerContent>
frontend/src/lib/components/triggers/postgres/PostgresTriggersPanel.svelte
+82 -60
View File
@@ -1,29 +1,48 @@
<script lang="ts">
import { userStore, workspaceStore } from '$lib/stores'
import { PostgresTriggerService, type PostgresTrigger } from '$lib/gen'
import { UnplugIcon } from 'lucide-svelte'
import { canWrite, sendUserToast } from '$lib/utils'
import { getContext } from 'svelte'
import { getContext, onMount } from 'svelte'
import { isCloudHosted } from '$lib/cloud'
import { Alert, Button, Skeleton } from '$lib/components/common'
import { Alert } from '$lib/components/common'
import type { TriggerContext } from '$lib/components/triggers'
import PostgresTriggerEditor from './PostgresTriggerEditor.svelte'
import Description from '$lib/components/Description.svelte'
import TriggersEditorSection from '../TriggersEditorSection.svelte'
import Section from '$lib/components/Section.svelte'
export let isFlow: boolean
export let path: string
export let newItem: boolean = false
export let canHavePreprocessor: boolean = false
export let hasPreprocessor: boolean = false
export let isEditor: boolean = false
let postgresTriggerEditor: PostgresTriggerEditor
let openForm = true
let dontCloseOnLoad = false
$: path && loadTriggers()
const { triggersCount } = getContext<TriggerContext>('TriggerContext')
const { triggersCount, selectedTrigger, defaultValues } =
getContext<TriggerContext>('TriggerContext')
let databaseTriggers: (PostgresTrigger & { canWrite: boolean })[] | undefined = undefined
onMount(() => {
if (
defaultValues &&
$selectedTrigger === 'postgres' &&
Object.keys($defaultValues ?? {}).length > 0
) {
postgresTriggerEditor.openNew(isFlow, path, $defaultValues)
defaultValues.set(undefined)
}
})
let postgresTriggers: (PostgresTrigger & { canWrite: boolean })[] | undefined = undefined
export async function loadTriggers() {
try {
databaseTriggers = (
postgresTriggers = (
await PostgresTriggerService.listPostgresTriggers({
workspace: $workspaceStore ?? '',
path,
@@ -32,7 +51,8 @@
).map((x) => {
return { canWrite: canWrite(x.path, x.extra_perms!, $userStore), ...x }
})
$triggersCount = { ...($triggersCount ?? {}), postgres_count: databaseTriggers?.length }
$triggersCount = { ...($triggersCount ?? {}), postgres_count: postgresTriggers?.length }
openForm = postgresTriggers?.length === 0 || dontCloseOnLoad
} catch (err) {
sendUserToast(`Could not load postgres triggers ${err.body}`, true)
}
@@ -45,59 +65,61 @@
}}
bind:this={postgresTriggerEditor}
/>
{#if isCloudHosted()}
<Alert title="Not compatible with multi-tenant cloud" type="warning">
Postgres triggers are disabled in the multi-tenant cloud.
</Alert>
{:else}
<div class="flex flex-col gap-4">
<Description link="https://www.windmill.dev/docs/core_concepts/postgres_triggers">
Windmill can connect to a Postgres database and trigger runnables (scripts, flows) in response
to database transactions (INSERT, UPDATE, DELETE) on specified tables, schemas, or the entire
database. Listening is done using Postgres's logical replication streaming protocol, ensuring
efficient and low-latency triggering.
</Description>
<div class="flex flex-col gap-4">
{#if !newItem}
{#if isCloudHosted()}
<Alert title="Not compatible with multi-tenant cloud" type="warning">
Postgres triggers are disabled in the multi-tenant cloud.
</Alert>
{:else if $userStore?.is_admin || $userStore?.is_super_admin}
<Button
on:click={() => postgresTriggerEditor?.openNew(isFlow, path)}
variant="border"
color="light"
size="xs"
startIcon={{ icon: UnplugIcon }}
>
New Postgres trigger
</Button>
{:else}
<Alert title="Only workspace admins can create routes" type="warning" size="xs" />
{/if}
{/if}
{#if databaseTriggers}
{#if databaseTriggers.length == 0}
<div class="text-xs text-secondary"> No Postgres triggers </div>
{:else}
<div class="flex flex-col divide-y pt-2">
{#each databaseTriggers as databaseTriggers (databaseTriggers.path)}
<div class="grid grid-cols-5 text-2xs items-center py-2">
<div class="col-span-2 truncate">{databaseTriggers.path}</div>
<div class="flex justify-end">
<button
on:click={() => postgresTriggerEditor?.openEdit(databaseTriggers.path, isFlow)}
class="px-2"
>
{#if databaseTriggers.canWrite}
Edit
{:else}
View
{/if}
</button>
</div>
{#if !newItem && postgresTriggers && postgresTriggers.length > 0}
<Section label="Postgres">
<div class="flex flex-col gap-4">
<div class="flex flex-col divide-y pt-2">
{#each postgresTriggers as postgresTriggers (postgresTriggers.path)}
<div class="grid grid-cols-5 text-2xs items-center py-2">
<div class="col-span-2 truncate">{postgresTriggers.path}</div>
<div class="flex justify-end">
<button
on:click={() => postgresTriggerEditor?.openEdit(postgresTriggers.path, isFlow)}
class="px-2"
>
{#if postgresTriggers.canWrite}
Edit
{:else}
View
{/if}
</button>
</div>
</div>
{/each}
</div>
{/each}
</div>
</div>
</Section>
{/if}
{:else}
<Skeleton layout={[[8]]} />
{/if}
{#if newItem}
<Alert title="Triggers disabled" type="warning" size="xs">
Deploy the {isFlow ? 'flow' : 'script'} to add Postgres triggers.
</Alert>
{/if}
</div>
<TriggersEditorSection
on:applyArgs
on:saveTrigger={(e) => {
postgresTriggerEditor?.openNew(isFlow, path, e.detail.config)
}}
on:addPreprocessor
on:updateSchema
on:testWithArgs
cloudDisabled={false}
triggerType="postgres"
{isFlow}
{path}
{isEditor}
{canHavePreprocessor}
{hasPreprocessor}
{newItem}
{openForm}
/>
</div>
{/if}
frontend/src/lib/components/triggers/postgres/PublicationPicker.svelte
+6 -4
View File
@@ -11,9 +11,9 @@
import { RefreshCw } from 'lucide-svelte'
export let items: string[] = []
export let can_write: boolean = true
export let publication_name: string = ''
export let postgres_resource_path: string = ''
export let table_to_track: Relations[] = []
export let relations: Relations[] = []
export let transaction_to_track: string[] = []
export let selectedTable: 'all' | 'specific' = 'specific'
@@ -38,7 +38,7 @@
workspace: $workspaceStore!,
publication: publication_name,
requestBody: {
table_to_track,
table_to_track: relations,
transaction_to_track: transaction_to_track
}
})
@@ -96,6 +96,7 @@
<div class="flex gap-1">
<Select
disabled={!can_write}
class="grow shrink max-w-full"
bind:justValue={publication_name}
value={publication_name}
@@ -109,6 +110,7 @@
on:select={getAllRelations}
/>
<Button
disabled={!can_write}
variant="border"
color="light"
wrapperClasses="self-stretch"
@@ -120,14 +122,14 @@
color="light"
size="xs"
variant="border"
disabled={emptyString(publication_name)}
disabled={emptyString(publication_name) || !can_write}
on:click={updatePublication}>Update</Button
>
<Button
color="light"
size="xs"
variant="border"
disabled={emptyString(publication_name)}
disabled={emptyString(publication_name) || !can_write}
on:click={deletePublication}>Delete</Button
>
</div>
frontend/src/lib/components/triggers/postgres/RelationPicker.svelte
+223 -143
View File
@@ -4,174 +4,254 @@
import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte'
import Tooltip from '$lib/components/Tooltip.svelte'
import type { Relations } from '$lib/gen'
import { Plus, X } from 'lucide-svelte'
import { Plus, Trash, X } from 'lucide-svelte'
import MultiSelect from 'svelte-multiselect'
import { fade } from 'svelte/transition'
import { invalidRelations } from './utils'
import AddPropertyFormV2 from '$lib/components/schema/AddPropertyFormV2.svelte'
import Label from '$lib/components/Label.svelte'
import { emptyStringTrimmed, sendUserToast } from '$lib/utils'
export let relations: Relations[] = []
export let selectedTable: 'all' | 'specific'
export let can_write: boolean = true
let cached: Relations[] = relations
function addTable(name: string, index: number) {
if (relations == undefined || !Array.isArray(relations)) {
relations = []
relations = relations.concat({
schema_name: 'public',
table_to_track: []
})
}
relations[index].table_to_track = relations[index].table_to_track.concat({
table_name: name,
columns_name: []
})
}
</script>
<div class="flex flex-col gap-4">
<ToggleButtonGroup bind:selected={selectedTable}>
<ToggleButton value="all" label="All Tables" />
<ToggleButton value="specific" label="Specific Tables" />
</ToggleButtonGroup>
<div class="flex flex-col gap-4 h-full">
<div class="grow-0 flex flex-row gap-2 w-full items-center">
<div class="grow-0">
<ToggleButtonGroup
on:selected={() => {
if (selectedTable === 'all') {
cached = relations
relations = []
} else {
relations = cached
}
}}
bind:selected={selectedTable}
>
<ToggleButton value="all" label="All Tables" />
<ToggleButton value="specific" label="Specific Tables" />
</ToggleButtonGroup>
</div>
</div>
{#if selectedTable !== 'all'}
{#if relations && relations.length > 0}
{#each relations as v, i}
<div class="flex w-full gap-2 items-center">
<div class="w-full flex flex-col gap-2 border p-2 rounded-md">
<label class="flex flex-col w-full">
<div class="text-secondary text-sm mb-2 flex flex-row gap-1"
><p>Schema Name</p><Tooltip
><p>
Enter the name of the <strong>schema</strong> that contains the table(s) you
want to track.
<br />If no tables are added, all tables within the <strong>schema</strong> will
be tracked for the selected transactions (insert, update, delete).
</p></Tooltip
></div
>
<input type="text" bind:value={v.schema_name} />
</label>
{#each v.table_to_track as table_to_track, j}
<div class="flex w-full gap-2 items-center p-5">
<div class="rounded shadow-inner p-2 flex w-full flex-col gap-4 mt-1">
<label class="flex flex-col w-full">
<div class="text-secondary text-sm mb-2 flex flex-row gap-1"
><p>Table Name</p><Tooltip
>Enter the name of the table you want to track.</Tooltip
></div
>
<input type="text" bind:value={table_to_track.table_name} />
</label>
<!-- svelte-ignore a11y-label-has-associated-control -->
<label class="flex flex-col w-full">
<div class="text-secondary text-sm mb-2 flex flex-row gap-1"
><p>Columns</p><Tooltip
documentationLink="https://www.windmill.dev/docs/core_concepts/postgres_triggers#selecting-specific-columns"
><p>
Enter the names of the <strong>columns</strong> you want to track.
<br />If no columns are specified, all columns in the table will be
tracked for the selected transactions (insert, update, delete).
</p>
<p class="text-xs text-gray-500 mt-1">
<strong class="font-semibold">Note:</strong>
<br />- If your trigger contains <strong>UPDATE</strong> or
<strong>DELETE</strong>
transactions, the row filter WHERE clause must contain only columns covered
by the <strong>replica identity</strong> (see REPLICA IDENTITY).
<br />- If your trigger contains only <strong>INSERT</strong> transactions,
the row filter WHERE clause can use any column.
</p></Tooltip
></div
>
<p class="text-xs mb-3 text-tertiary">
Enter the names of the <strong>columns</strong> you want to track.
<br />If no columns are specified, all columns in the table will be tracked.
<div class="flex flex-col gap-4 grow min-h-0 overflow-y-auto">
{#each relations as v, i}
<div class="flex w-full gap-3 items-center">
<div class="w-full flex flex-col gap-2 border py-2 px-4 rounded-md">
<Label label="Schema Name" required class="w-full">
<svelte:fragment slot="header">
<Tooltip small>
<p>
Enter the name of the <strong>schema</strong> that contains the table(s) you
want to track.
<br />If no tables are added, all tables within the <strong>schema</strong> will
be tracked for the selected transactions (insert, update, delete).
</p>
<MultiSelect
options={table_to_track.columns_name ?? []}
allowUserOptions="append"
bind:selected={table_to_track.columns_name}
ulOptionsClass={'!bg-surface-secondary'}
noMatchingOptionsMsg=""
createOptionMsg={null}
duplicates={false}
/>
</label>
<label class="flex flex-col w-full">
<div class="text-secondary text-sm mb-2 flex flex-row gap-1"
><p>Where Clause</p><Tooltip
documentationLink="https://www.windmill.dev/docs/core_concepts/postgres_triggers#filtering-rows-with-where-condition"
>
<p class="text-sm">
Use this field to define a row filter for the selected table. The <strong
>WHERE</strong
> clause allows only simple expressions and cannot include user-defined functions,
operators, types, collations, system column references, or non-immutable built-in
functions.
</p>
<p class="text-xs text-gray-500 mt-1">
<strong class="font-semibold">Note:</strong>
<br />- If your trigger contains <strong>UPDATE</strong> or
<strong>DELETE</strong>
transactions, the row filter WHERE clause must contain only columns covered
by the <strong>replica identity</strong> (see REPLICA IDENTITY).
<br />- If your trigger contains only <strong>INSERT</strong> transactions,
the row filter WHERE clause can use any column.
</p>
</Tooltip></div
>
<input type="text" bind:value={table_to_track.where_clause} />
</label>
<button
transition:fade|local={{ duration: 100 }}
class="rounded items-center p-1 bg-surface-secondary duration-200 hover:bg-surface-hover"
aria-label="Clear"
on:click={() => {
v.table_to_track = v.table_to_track.filter((_, index) => index !== j)
}}
</Tooltip>
</svelte:fragment>
<input type="text" bind:value={v.schema_name} />
</Label>
<div class="flex flex-col w-full gap-4 items-center p-5">
{#each v.table_to_track as table_to_track, j}
<div
class="relative rounded bg-surface-disabled p-2 flex w-full flex-col gap-4 group"
>
Remove
</button>
</div>
<Label label="Table Name" required>
<svelte:fragment slot="header">
<Tooltip small>Enter the name of the table you want to track.</Tooltip>
</svelte:fragment>
<input
type="text"
bind:value={table_to_track.table_name}
class="!bg-surface"
/>
</Label>
<!-- svelte-ignore a11y-label-has-associated-control -->
<Label label="Columns">
<svelte:fragment slot="header">
<Tooltip
documentationLink="https://www.windmill.dev/docs/core_concepts/postgres_triggers#selecting-specific-columns"
small
>
<p>
Enter the names of the <strong>columns</strong> you want to track.
<br />If no columns are specified, all columns in the table will be
tracked for the selected transactions (insert, update, delete).
</p>
<p class="text-xs text-gray-500 mt-1">
<strong class="font-semibold">Note:</strong>
<br />- If your trigger contains <strong>UPDATE</strong> or
<strong>DELETE</strong>
transactions, the row filter WHERE clause must contain only columns covered
by the <strong>replica identity</strong> (see REPLICA IDENTITY).
<br />- If your trigger contains only <strong>INSERT</strong> transactions,
the row filter WHERE clause can use any column.
</p>
</Tooltip>
</svelte:fragment>
<MultiSelect
options={table_to_track.columns_name ?? []}
allowUserOptions="append"
bind:selected={table_to_track.columns_name}
ulOptionsClass={'!bg-surface !text-sm'}
ulSelectedClass="!text-sm"
outerDivClass="!bg-surface !min-h-[38px] !border-[#d1d5db]"
noMatchingOptionsMsg=""
createOptionMsg={null}
duplicates={false}
placeholder="Select columns"
--sms-options-margin="4px"
>
<svelte:fragment slot="remove-icon">
<div class="hover:text-primary p-0.5">
<X size={12} />
</div>
</svelte:fragment>
</MultiSelect>
</Label>
<Label label="Where Clause">
<svelte:fragment slot="header">
<Tooltip
documentationLink="https://www.windmill.dev/docs/core_concepts/postgres_triggers#filtering-rows-with-where-condition"
small
>
<p class="text-sm">
Use this field to define a row filter for the selected table. The <strong
>WHERE</strong
> clause allows only simple expressions and cannot include user-defined functions,
operators, types, collations, system column references, or non-immutable
built-in functions.
</p>
<p class="text-xs text-gray-500 mt-1">
<strong class="font-semibold">Note:</strong>
<br />- If your trigger contains <strong>UPDATE</strong> or
<strong>DELETE</strong>
transactions, the row filter WHERE clause must contain only columns covered
by the <strong>replica identity</strong> (see REPLICA IDENTITY).
<br />- If your trigger contains only <strong>INSERT</strong> transactions,
the row filter WHERE clause can use any column.
</p>
</Tooltip>
</svelte:fragment>
<input
type="text"
bind:value={table_to_track.where_clause}
class="!bg-surface"
/>
</Label>
<Button
variant="border"
wrapperClasses="absolute -top-3 -right-3"
btnClasses="hidden group-hover:block hover:bg-red-500 hover:text-white p-2 rounded-full hover:border-red-500 transition-all duration-300"
color="light"
size="xs"
on:click={() => {
v.table_to_track = v.table_to_track.filter((_, index) => index !== j)
}}
iconOnly
startIcon={{ icon: Trash }}
/>
</div>
{/each}
<AddPropertyFormV2
customName="Table"
on:add={({ detail }) => {
addTable(detail.name, i)
}}
>
<svelte:fragment slot="trigger">
<Button
wrapperClasses="w-full border border-dashed rounded-md"
color="light"
size="xs"
startIcon={{ icon: Plus }}
nonCaptureEvent
>
Add table
</Button>
</svelte:fragment>
</AddPropertyFormV2>
</div>
{/each}
</div>
<Button
variant="border"
color="light"
size="xs"
btnClasses="mt-1"
btnClasses="bg-surface-secondary hover:bg-red-500 hover:text-white p-2 rounded-full"
aria-label="Clear"
on:click={() => {
if (
relations[i].table_to_track == undefined ||
!Array.isArray(relations[i].table_to_track)
) {
relations[i].table_to_track = []
}
relations[i].table_to_track = relations[i].table_to_track.concat({
table_name: '',
columns_name: []
})
relations = relations.filter((_, index) => index !== i)
}}
startIcon={{ icon: Plus }}
>
Add Table
<Trash size={14} />
</Button>
</div>
<button
transition:fade|local={{ duration: 100 }}
class="rounded-full p-1 bg-surface-secondary duration-200 hover:bg-surface-hover"
aria-label="Clear"
on:click={() => {
relations = relations.filter((_, index) => index !== i)
}}
>
<X size={14} />
</button>
</div>
{/each}
{/each}
</div>
{/if}
<div class="flex items-baseline">
<Button
variant="border"
color="light"
size="xs"
btnClasses="mt-1"
on:click={() => {
<div class="grow min-w-0 pr-10">
<AddPropertyFormV2
customName="Schema"
on:add={({ detail }) => {
if (relations == undefined || !Array.isArray(relations)) {
relations = []
relations = relations.concat({
schema_name: 'public',
table_to_track: []
})
} else if (emptyStringTrimmed(detail.name)) {
sendUserToast('Schema name must not be empty', true)
} else {
const appendedRelations = relations.concat({
schema_name: detail.name,
table_to_track: []
})
if (
invalidRelations(appendedRelations, {
showError: true,
trackSchemaTableError: false
}) === false
) {
relations = appendedRelations
}
}
relations = relations.concat({
schema_name: '',
table_to_track: []
})
}}
startIcon={{ icon: Plus }}
>
Add Schema
</Button>
<svelte:fragment slot="trigger">
<Button
variant="border"
color="light"
size="xs"
btnClasses="w-full"
disabled={!can_write}
startIcon={{ icon: Plus }}
nonCaptureEvent
>
Add schema
</Button>
</svelte:fragment>
</AddPropertyFormV2>
</div>
{/if}
</div>
frontend/src/lib/components/triggers/postgres/utils.ts
+102
View File
@@ -0,0 +1,102 @@
import type { Relations } from '$lib/gen'
import { sendUserToast } from '$lib/toast'
import { emptyString } from '$lib/utils'
type RelationError = {
schemaIndex: number
tableIndex: number
schemaError: boolean
tableError: boolean
schemaName?: string
trackAllTablesInSchema: boolean
trackSpecificColumnsInTable: boolean
duplicateSchemaName: boolean | undefined
}
export function invalidRelations(
relations: Relations[],
options?: {
trackSchemaTableError?: boolean
showError?: boolean
}
): boolean {
let error: RelationError = {
schemaIndex: -1,
tableIndex: -1,
schemaError: false,
tableError: false,
trackAllTablesInSchema: false,
trackSpecificColumnsInTable: false,
duplicateSchemaName: undefined
}
const duplicateName: Set<string> = new Set()
for (const [schemaIndex, relation] of relations.entries()) {
error.schemaIndex = schemaIndex + 1
error.schemaName = relation.schema_name
if (emptyString(relation.schema_name)) {
error.schemaError = true
break
} else {
if (duplicateName.has(relation.schema_name)) {
error.duplicateSchemaName = true
break
}
duplicateName.add(relation.schema_name)
const tableToTrack = relation.table_to_track
if (tableToTrack.length > 0) {
for (const [tableIndex, table] of tableToTrack.entries()) {
if (emptyString(table.table_name)) {
error.tableError = true
error.tableIndex = tableIndex + 1
break
}
if (
!error.trackSpecificColumnsInTable &&
table.columns_name &&
table.columns_name.length > 0
) {
error.trackSpecificColumnsInTable = true
}
}
if (error.tableError) {
break
}
} else if (!error.trackAllTablesInSchema) {
error.trackAllTablesInSchema = true
}
if (
options?.trackSchemaTableError &&
error.trackAllTablesInSchema &&
error.trackSpecificColumnsInTable
) {
break
}
}
}
const errorFound =
error.tableError ||
error.schemaError ||
error.duplicateSchemaName ||
((options?.trackSchemaTableError ?? false) &&
error.trackAllTablesInSchema &&
error.trackSpecificColumnsInTable)
if ((options?.showError ?? false) && errorFound) {
let errorMessage: string = ''
if (error.schemaError) {
errorMessage = `Schema Error: Please enter a name for schema number ${error.schemaIndex}`
} else if (error.tableError) {
errorMessage = `Table Error: Please enter a name for table number ${error.tableIndex} inside schema number ${error.schemaIndex}`
errorMessage += emptyString(error.schemaName) ? '' : ` named: ${error.schemaName}`
} else if (error.duplicateSchemaName) {
errorMessage = `Schema Error: schema name '${error.schemaName}' is already taken`
} else {
errorMessage =
'Configuration Error: Schema-level tracking and specific table tracking with column selection cannot be used together. Refer to the documentation for valid configurations.'
}
sendUserToast(errorMessage, true)
}
return errorFound
}
frontend/src/lib/script_helpers.ts
+1 -1
View File
@@ -705,7 +705,7 @@ class Nats(TypedDict):
length: int
class WmTrigger(TypedDict):
kind: Literal["http", "email", "webhook", "websocket", "kafka", "nats"]
kind: Literal["http", "email", "webhook", "websocket", "kafka", "nats", "postgres"]
http: Http | None
websocket: Websocket | None
kafka: Kafka | None
frontend/src/routes/(root)/(logged)/postgres_triggers/+page.svelte
+1 -1
View File
@@ -389,7 +389,7 @@
displayName: canWrite ? 'Share' : 'See Permissions',
icon: Share,
action: () => {
shareModal.openDrawer(path, 'websocket_trigger')
shareModal.openDrawer(path, 'postgres_trigger')
}
}
]}