mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-11-17 00:20:51 +00:00
feat: optionally run trigger messages the filters cannot read (#12226)
* fix: route trigger messages the filters cannot parse to the error handler A Kafka or websocket trigger with filters silently skipped a message that was not UTF-8 or not JSON, and Kafka committed its offset, so the message was lost without a trace. Such a message now goes to the trigger's error handler (script targets) or the workspace error handler, with the raw payload and its topic/partition/offset. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: take the unfilterable-message handler as a RunnableRef after merging main Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: keep none_of semantics for non-object JSON, honour workspace handler mutes, throttle the no-handler alert Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * docs: state the caller contract of push_error_handler_with_parent Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * chore: point ee-repo-ref at the kafka companion branch Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: hold unparseable-message handlers of a suspended trigger, throttle them on websocket A suspended trigger now holds the handler run like its other runs, tagged with the trigger so resuming releases it. Websocket text keepalives would start a handler run per frame, so websocket caps it at one per trigger per minute; Kafka keeps one per message. A parentless handler run carries no token prefix. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: release a held error handler run in place when resuming an edited trigger Resuming re-pushes runs held before the trigger's last edit against its target. A held error handler run would then run the target with the handler's args, so it is released in place. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * feat: add run_on_filter_error to kafka and websocket triggers Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix: treat a truncated array as unreadable, not as a filter mismatch Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * chore: update ee-repo-ref to 695eddf214f6a0e6cf1249ee2c20e716788f0766 This commit updates the EE repository reference after PR #954 was merged in windmill-ee-private. Previous ee-repo-ref: c8a26463f72ed76423b14e6a1aaf6376e58f2c4c New ee-repo-ref: 695eddf214f6a0e6cf1249ee2c20e716788f0766 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com> Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
This commit is contained in:
co-authored by
Claude Opus 5.5
windmill-internal-app[bot]
Ruben Fiszel
parent
964f1af303
commit
a4ae82fd69
+4
-3
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n UPDATE\n websocket_trigger\n SET\n url = $1,\n script_path = $2,\n path = $3,\n is_flow = $4,\n filters = $5,\n filter_logic = $6,\n initial_messages = $7,\n url_runnable_args = $8,\n edited_by = $9,\n permissioned_as = $10,\n can_return_message = $11,\n can_return_error_result = $12,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $15,\n error_handler_args = $16,\n retry = $17,\n heartbeat = $18\n WHERE\n workspace_id = $13 AND path = $14\n ",
|
||||
"query": "\n UPDATE\n websocket_trigger\n SET\n url = $1,\n script_path = $2,\n path = $3,\n is_flow = $4,\n filters = $5,\n filter_logic = $6,\n initial_messages = $7,\n url_runnable_args = $8,\n edited_by = $9,\n permissioned_as = $10,\n can_return_message = $11,\n can_return_error_result = $12,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $15,\n error_handler_args = $16,\n retry = $17,\n heartbeat = $18,\n run_on_filter_error = $19\n WHERE\n workspace_id = $13 AND path = $14\n ",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
@@ -22,10 +22,11 @@
|
||||
"Varchar",
|
||||
"Jsonb",
|
||||
"Jsonb",
|
||||
"Jsonb"
|
||||
"Jsonb",
|
||||
"Bool"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "492edd53e497a45c314d41044e85a5e3492227b5049dc6870d60817ad929e7af"
|
||||
"hash": "418d257ac50ec518d960aa286a003caf748a5423b208d897ff4e23aae0330007"
|
||||
}
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO websocket_trigger (\n path, url, script_path, is_flow, workspace_id, edited_by, edited_at,\n extra_perms, server_id, last_server_ping, error, filters, initial_messages,\n url_runnable_args, can_return_message, error_handler_path, error_handler_args,\n retry, can_return_error_result, mode, permissioned_as, filter_logic, labels,\n heartbeat\n )\n SELECT\n path, url, script_path, is_flow, $1, edited_by, edited_at,\n extra_perms, NULL, NULL, NULL, filters, initial_messages,\n url_runnable_args, can_return_message, error_handler_path, error_handler_args,\n retry, can_return_error_result, 'disabled'::TRIGGER_MODE, permissioned_as, filter_logic, labels,\n heartbeat\n FROM websocket_trigger WHERE workspace_id = $2",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "5fd769bde29e88eb43f5dc0ecd7f4a1fe20d216d818d69eea4ddbaace5233137"
|
||||
}
|
||||
+4
-3
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n UPDATE kafka_trigger\n SET\n kafka_resource_path = $1,\n group_id = $2,\n topics = $3,\n filters = $4,\n filter_logic = $5,\n auto_offset_reset = $6,\n auto_commit = $7,\n script_path = $8,\n path = $9,\n is_flow = $10,\n edited_by = $11,\n permissioned_as = $12,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $15,\n error_handler_args = $16,\n retry = $17\n WHERE\n workspace_id = $13 AND path = $14\n ",
|
||||
"query": "\n UPDATE kafka_trigger\n SET\n kafka_resource_path = $1,\n group_id = $2,\n topics = $3,\n filters = $4,\n filter_logic = $5,\n auto_offset_reset = $6,\n auto_commit = $7,\n script_path = $8,\n path = $9,\n is_flow = $10,\n edited_by = $11,\n permissioned_as = $12,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $15,\n error_handler_args = $16,\n retry = $17,\n run_on_filter_error = $18\n WHERE\n workspace_id = $13 AND path = $14\n ",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
@@ -21,10 +21,11 @@
|
||||
"Text",
|
||||
"Varchar",
|
||||
"Jsonb",
|
||||
"Jsonb"
|
||||
"Jsonb",
|
||||
"Bool"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "68c19cb0e18b94870bbe81f9aab92ba37da67cd2a56834c9d1378eab7551284d"
|
||||
"hash": "6f16ddb07e8543e12a12bd5b312da224e00c5c0ac1ba4abc16fe104d28b59da1"
|
||||
}
|
||||
+4
-3
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n INSERT INTO kafka_trigger (\n workspace_id,\n path,\n kafka_resource_path,\n group_id,\n topics,\n filters,\n filter_logic,\n auto_offset_reset,\n auto_commit,\n script_path,\n is_flow,\n mode,\n edited_by,\n permissioned_as,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, now(), $15, $16, $17\n )\n ",
|
||||
"query": "\n INSERT INTO kafka_trigger (\n workspace_id,\n path,\n kafka_resource_path,\n group_id,\n topics,\n filters,\n filter_logic,\n auto_offset_reset,\n auto_commit,\n script_path,\n is_flow,\n mode,\n edited_by,\n permissioned_as,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry,\n run_on_filter_error\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, now(), $15, $16, $17, $18\n )\n ",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
@@ -32,10 +32,11 @@
|
||||
"Varchar",
|
||||
"Varchar",
|
||||
"Jsonb",
|
||||
"Jsonb"
|
||||
"Jsonb",
|
||||
"Bool"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "6a8f4ed9946bb2a3c5e90695c90b70aa2e83fcb5aa0c953febdd9bac2d95bbec"
|
||||
"hash": "8d9c5c4bcdc1ca66388fdf35a1573ba28a1891f46c80bcd3b3e4f058e037c26a"
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO kafka_trigger (\n path, kafka_resource_path, topics, group_id, script_path, is_flow,\n workspace_id, edited_by, edited_at, extra_perms, server_id,\n last_server_ping, error, error_handler_path, error_handler_args, retry,\n mode, filters, auto_offset_reset, reset_offset, auto_commit,\n permissioned_as, filter_logic, labels, run_on_filter_error\n )\n SELECT\n path, kafka_resource_path, topics, group_id, script_path, is_flow,\n $1, edited_by, edited_at, extra_perms, NULL,\n NULL, NULL, error_handler_path, error_handler_args, retry,\n 'disabled'::TRIGGER_MODE, filters, auto_offset_reset, reset_offset, auto_commit,\n permissioned_as, filter_logic, labels, run_on_filter_error\n FROM kafka_trigger WHERE workspace_id = $2",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "a8f792d147acc47791a8cbc473a13a9440d22500eef72b11065b9badbeb5101a"
|
||||
}
|
||||
-15
@@ -1,15 +0,0 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO kafka_trigger (\n path, kafka_resource_path, topics, group_id, script_path, is_flow,\n workspace_id, edited_by, edited_at, extra_perms, server_id,\n last_server_ping, error, error_handler_path, error_handler_args, retry,\n mode, filters, auto_offset_reset, reset_offset, auto_commit,\n permissioned_as, filter_logic, labels\n )\n SELECT\n path, kafka_resource_path, topics, group_id, script_path, is_flow,\n $1, edited_by, edited_at, extra_perms, NULL,\n NULL, NULL, error_handler_path, error_handler_args, retry,\n 'disabled'::TRIGGER_MODE, filters, auto_offset_reset, reset_offset, auto_commit,\n permissioned_as, filter_logic, labels\n FROM kafka_trigger WHERE workspace_id = $2",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "c9c0b92c1fea9b4bdafba32da60e5579a4c2940bed63097ffec7f757d9882667"
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO websocket_trigger (\n path, url, script_path, is_flow, workspace_id, edited_by, edited_at,\n extra_perms, server_id, last_server_ping, error, filters, initial_messages,\n url_runnable_args, can_return_message, error_handler_path, error_handler_args,\n retry, can_return_error_result, mode, permissioned_as, filter_logic, labels,\n heartbeat, run_on_filter_error\n )\n SELECT\n path, url, script_path, is_flow, $1, edited_by, edited_at,\n extra_perms, NULL, NULL, NULL, filters, initial_messages,\n url_runnable_args, can_return_message, error_handler_path, error_handler_args,\n retry, can_return_error_result, 'disabled'::TRIGGER_MODE, permissioned_as, filter_logic, labels,\n heartbeat, run_on_filter_error\n FROM websocket_trigger WHERE workspace_id = $2",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Varchar",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "cd54e1e83205fc161e4046f8b5e3bfda4e38156cd89e7a0b7f8164c98cf1c608"
|
||||
}
|
||||
+4
-3
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n INSERT INTO websocket_trigger (\n workspace_id,\n path,\n url,\n script_path,\n is_flow,\n mode,\n filters,\n filter_logic,\n initial_messages,\n url_runnable_args,\n edited_by,\n can_return_message,\n can_return_error_result,\n permissioned_as,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry,\n heartbeat\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, now(), $15, $16, $17, $18\n )\n ",
|
||||
"query": "\n INSERT INTO websocket_trigger (\n workspace_id,\n path,\n url,\n script_path,\n is_flow,\n mode,\n filters,\n filter_logic,\n initial_messages,\n url_runnable_args,\n edited_by,\n can_return_message,\n can_return_error_result,\n permissioned_as,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry,\n heartbeat,\n run_on_filter_error\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, now(), $15, $16, $17, $18, $19\n )\n ",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
@@ -33,10 +33,11 @@
|
||||
"Varchar",
|
||||
"Jsonb",
|
||||
"Jsonb",
|
||||
"Jsonb"
|
||||
"Jsonb",
|
||||
"Bool"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "11fd92de8688ef6b4d524aade507850a9cb3e097f2d957219d04ad134a5e0399"
|
||||
"hash": "cdc2371af8d627a410e9ba0287eb1a35b08cbab2fe0d2a0e0d65a98658d42e22"
|
||||
}
|
||||
@@ -1 +1 @@
|
||||
0b763cc905bb67b17728794bb52bfdf336fe25fb
|
||||
695eddf214f6a0e6cf1249ee2c20e716788f0766
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
ALTER TABLE kafka_trigger DROP COLUMN run_on_filter_error;
|
||||
ALTER TABLE websocket_trigger DROP COLUMN run_on_filter_error;
|
||||
@@ -0,0 +1,2 @@
|
||||
ALTER TABLE kafka_trigger ADD COLUMN run_on_filter_error BOOLEAN NOT NULL DEFAULT false;
|
||||
ALTER TABLE websocket_trigger ADD COLUMN run_on_filter_error BOOLEAN NOT NULL DEFAULT false;
|
||||
@@ -7744,14 +7744,14 @@ async fn clone_triggers_and_schedules(
|
||||
extra_perms, server_id, last_server_ping, error, filters, initial_messages,
|
||||
url_runnable_args, can_return_message, error_handler_path, error_handler_args,
|
||||
retry, can_return_error_result, mode, permissioned_as, filter_logic, labels,
|
||||
heartbeat
|
||||
heartbeat, run_on_filter_error
|
||||
)
|
||||
SELECT
|
||||
path, url, script_path, is_flow, $1, edited_by, edited_at,
|
||||
extra_perms, NULL, NULL, NULL, filters, initial_messages,
|
||||
url_runnable_args, can_return_message, error_handler_path, error_handler_args,
|
||||
retry, can_return_error_result, 'disabled'::TRIGGER_MODE, permissioned_as, filter_logic, labels,
|
||||
heartbeat
|
||||
heartbeat, run_on_filter_error
|
||||
FROM websocket_trigger WHERE workspace_id = $2"#,
|
||||
target_workspace_id,
|
||||
source_workspace_id,
|
||||
@@ -7765,14 +7765,14 @@ async fn clone_triggers_and_schedules(
|
||||
workspace_id, edited_by, edited_at, extra_perms, server_id,
|
||||
last_server_ping, error, error_handler_path, error_handler_args, retry,
|
||||
mode, filters, auto_offset_reset, reset_offset, auto_commit,
|
||||
permissioned_as, filter_logic, labels
|
||||
permissioned_as, filter_logic, labels, run_on_filter_error
|
||||
)
|
||||
SELECT
|
||||
path, kafka_resource_path, topics, group_id, script_path, is_flow,
|
||||
$1, edited_by, edited_at, extra_perms, NULL,
|
||||
NULL, NULL, error_handler_path, error_handler_args, retry,
|
||||
'disabled'::TRIGGER_MODE, filters, auto_offset_reset, reset_offset, auto_commit,
|
||||
permissioned_as, filter_logic, labels
|
||||
permissioned_as, filter_logic, labels, run_on_filter_error
|
||||
FROM kafka_trigger WHERE workspace_id = $2"#,
|
||||
target_workspace_id,
|
||||
source_workspace_id,
|
||||
|
||||
@@ -35155,6 +35155,10 @@ components:
|
||||
- or
|
||||
default: and
|
||||
description: "Logic to apply when evaluating the top-level filters. 'and' requires all of them to match, 'or' requires any of them to match. Nested `any_of`/`all_of`/`none_of` groups carry their own logic."
|
||||
run_on_filter_error:
|
||||
type: boolean
|
||||
default: false
|
||||
description: "Whether a message the filters cannot read (not UTF-8, or not JSON) still runs the target, as if the trigger had no filters. A preprocessor then finds the reason in `event.filter_error`. When false, such a message is skipped."
|
||||
initial_messages:
|
||||
type: array
|
||||
nullable: true
|
||||
@@ -35222,6 +35226,10 @@ components:
|
||||
- or
|
||||
default: and
|
||||
description: "Logic to apply when evaluating the top-level filters. 'and' requires all of them to match, 'or' requires any of them to match. Nested `any_of`/`all_of`/`none_of` groups carry their own logic."
|
||||
run_on_filter_error:
|
||||
type: boolean
|
||||
default: false
|
||||
description: "Whether a message the filters cannot read (not UTF-8, or not JSON) still runs the target, as if the trigger had no filters. A preprocessor then finds the reason in `event.filter_error`. When false, such a message is skipped."
|
||||
initial_messages:
|
||||
type: array
|
||||
nullable: true
|
||||
@@ -35300,6 +35308,10 @@ components:
|
||||
- or
|
||||
default: and
|
||||
description: "Logic to apply when evaluating the top-level filters. 'and' requires all of them to match, 'or' requires any of them to match. Nested `any_of`/`all_of`/`none_of` groups carry their own logic."
|
||||
run_on_filter_error:
|
||||
type: boolean
|
||||
default: false
|
||||
description: "Whether a message the filters cannot read (not UTF-8, or not JSON) still runs the target, as if the trigger had no filters. A preprocessor then finds the reason in `event.filter_error`. When false, such a message is skipped."
|
||||
initial_messages:
|
||||
type: array
|
||||
nullable: true
|
||||
@@ -36981,6 +36993,10 @@ components:
|
||||
- or
|
||||
default: and
|
||||
description: "Logic to apply when evaluating the top-level filters. 'and' requires all of them to match, 'or' requires any of them to match. Nested `any_of`/`all_of`/`none_of` groups carry their own logic."
|
||||
run_on_filter_error:
|
||||
type: boolean
|
||||
default: false
|
||||
description: "Whether a message the filters cannot read (not UTF-8, or not JSON) still runs the target, as if the trigger had no filters. A preprocessor then finds the reason in `event.filter_error`. When false, such a message is skipped."
|
||||
auto_offset_reset:
|
||||
type: string
|
||||
enum:
|
||||
@@ -37055,6 +37071,10 @@ components:
|
||||
- or
|
||||
default: and
|
||||
description: "Logic to apply when evaluating the top-level filters. 'and' requires all of them to match, 'or' requires any of them to match. Nested `any_of`/`all_of`/`none_of` groups carry their own logic."
|
||||
run_on_filter_error:
|
||||
type: boolean
|
||||
default: false
|
||||
description: "Whether a message the filters cannot read (not UTF-8, or not JSON) still runs the target, as if the trigger had no filters. A preprocessor then finds the reason in `event.filter_error`. When false, such a message is skipped."
|
||||
auto_offset_reset:
|
||||
type: string
|
||||
enum:
|
||||
@@ -37125,6 +37145,10 @@ components:
|
||||
- or
|
||||
default: and
|
||||
description: "Logic to apply when evaluating the top-level filters. 'and' requires all of them to match, 'or' requires any of them to match. Nested `any_of`/`all_of`/`none_of` groups carry their own logic."
|
||||
run_on_filter_error:
|
||||
type: boolean
|
||||
default: false
|
||||
description: "Whether a message the filters cannot read (not UTF-8, or not JSON) still runs the target, as if the trigger had no filters. A preprocessor then finds the reason in `event.filter_error`. When false, such a message is skipped."
|
||||
auto_offset_reset:
|
||||
type: string
|
||||
enum:
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -74,6 +74,7 @@ impl TriggerCrud for WebsocketTrigger {
|
||||
"url",
|
||||
"filters",
|
||||
"filter_logic",
|
||||
"run_on_filter_error",
|
||||
"initial_messages",
|
||||
"url_runnable_args",
|
||||
"can_return_message",
|
||||
@@ -176,9 +177,10 @@ impl TriggerCrud for WebsocketTrigger {
|
||||
error_handler_path,
|
||||
error_handler_args,
|
||||
retry,
|
||||
heartbeat
|
||||
heartbeat,
|
||||
run_on_filter_error
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, now(), $15, $16, $17, $18
|
||||
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, now(), $15, $16, $17, $18, $19
|
||||
)
|
||||
"#,
|
||||
w_id,
|
||||
@@ -201,7 +203,8 @@ impl TriggerCrud for WebsocketTrigger {
|
||||
trigger.error_handling.error_handler_path,
|
||||
trigger.error_handling.error_handler_args as _,
|
||||
trigger.error_handling.retry as _,
|
||||
trigger.config.heartbeat.map(SqlxJson) as _
|
||||
trigger.config.heartbeat.map(SqlxJson) as _,
|
||||
trigger.config.run_on_filter_error
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
@@ -258,7 +261,8 @@ impl TriggerCrud for WebsocketTrigger {
|
||||
error_handler_path = $15,
|
||||
error_handler_args = $16,
|
||||
retry = $17,
|
||||
heartbeat = $18
|
||||
heartbeat = $18,
|
||||
run_on_filter_error = $19
|
||||
WHERE
|
||||
workspace_id = $13 AND path = $14
|
||||
",
|
||||
@@ -283,7 +287,8 @@ impl TriggerCrud for WebsocketTrigger {
|
||||
trigger.error_handling.error_handler_path,
|
||||
trigger.error_handling.error_handler_args as _,
|
||||
trigger.error_handling.retry as _,
|
||||
trigger.config.heartbeat.map(SqlxJson) as _
|
||||
trigger.config.heartbeat.map(SqlxJson) as _,
|
||||
trigger.config.run_on_filter_error
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
@@ -51,6 +51,8 @@ pub struct WebsocketConfig {
|
||||
pub filters: Vec<SqlxJson<Box<RawValue>>>,
|
||||
#[serde(default = "default_filter_logic")]
|
||||
pub filter_logic: String,
|
||||
#[serde(default)]
|
||||
pub run_on_filter_error: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub initial_messages: Option<Vec<SqlxJson<Box<RawValue>>>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
@@ -69,6 +71,8 @@ pub struct WebsocketConfigRequest {
|
||||
filters: Vec<serde_json::Value>,
|
||||
#[serde(default = "default_filter_logic")]
|
||||
filter_logic: String,
|
||||
#[serde(default)]
|
||||
run_on_filter_error: bool,
|
||||
initial_messages: Option<Vec<serde_json::Value>>,
|
||||
url_runnable_args: Option<serde_json::Value>,
|
||||
can_return_message: bool,
|
||||
|
||||
@@ -527,10 +527,21 @@ impl Listener for WebsocketTrigger {
|
||||
}
|
||||
}
|
||||
|
||||
if filters.matches(&text) {
|
||||
let trigger_info = HashMap::from([
|
||||
("url".to_string(), to_raw_value(&listening_trigger.trigger_config.url)),
|
||||
]);
|
||||
let mut trigger_info = HashMap::from([
|
||||
("url".to_string(), to_raw_value(&listening_trigger.trigger_config.url)),
|
||||
]);
|
||||
let should_handle = match filters.matches(&text) {
|
||||
Ok(matched) => matched,
|
||||
Err(error) if listening_trigger.trigger_config.run_on_filter_error => {
|
||||
trigger_info.insert("filter_error".to_string(), to_raw_value(&error));
|
||||
true
|
||||
}
|
||||
Err(error) => {
|
||||
tracing::warn!("WebSocket trigger {} skipped a message: its filters could not be applied, {}", listening_trigger.path, error);
|
||||
false
|
||||
}
|
||||
};
|
||||
if should_handle {
|
||||
let _ = self.handle_event(db, listening_trigger, text, trigger_info, return_message_channels.clone()).await;
|
||||
}
|
||||
},
|
||||
|
||||
@@ -80,6 +80,7 @@ fn trigger(url: String, trigger_mode: bool) -> ListeningTrigger<WebsocketConfig>
|
||||
url,
|
||||
filters: vec![],
|
||||
filter_logic: "and".to_string(),
|
||||
run_on_filter_error: false,
|
||||
initial_messages: None,
|
||||
url_runnable_args: None,
|
||||
can_return_message: false,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use serde::{
|
||||
de::{self, MapAccess, Visitor},
|
||||
de::{self, IgnoredAny, MapAccess, Visitor},
|
||||
Deserialize, Deserializer,
|
||||
};
|
||||
use serde_json::{value::RawValue, Value};
|
||||
@@ -113,18 +113,35 @@ impl CompiledFilters {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Whether `text`, parsed as a JSON object, satisfies the filters.
|
||||
pub fn matches(&self, text: &str) -> bool {
|
||||
/// Whether `text`, parsed as a JSON object, satisfies the filters. Anything else is judged
|
||||
/// as an object with no keys, which a `none_of` filter accepts. When that rejects it, JSON
|
||||
/// that is not an object is a mismatch, and text that is not JSON an error the trigger's
|
||||
/// `run_on_filter_error` setting decides on.
|
||||
pub fn matches(&self, text: &str) -> std::result::Result<bool, String> {
|
||||
if self.filters.is_empty() {
|
||||
return true;
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
let mut deserializer = serde_json::Deserializer::from_str(text);
|
||||
let values =
|
||||
Deserializer::deserialize_map(&mut deserializer, KeysVisitor { keys: &self.keys })
|
||||
.unwrap_or_default();
|
||||
let values = match Deserializer::deserialize_map(
|
||||
&mut deserializer,
|
||||
KeysVisitor { keys: &self.keys },
|
||||
) {
|
||||
Ok(values) => values,
|
||||
Err(_) if eval_all(&self.filters, self.use_or_logic, &HashMap::new()) => {
|
||||
return Ok(true)
|
||||
}
|
||||
// A data error is raised on the first token (`[`, `"`, ...), before the rest is read:
|
||||
// parse it all so a truncated array is not taken for valid JSON that is not an object.
|
||||
Err(err) if err.is_data() => {
|
||||
return serde_json::from_str::<IgnoredAny>(text)
|
||||
.map(|_| false)
|
||||
.map_err(|err| format!("payload is not valid JSON: {err}"))
|
||||
}
|
||||
Err(err) => return Err(format!("payload is not valid JSON: {err}")),
|
||||
};
|
||||
|
||||
eval_all(&self.filters, self.use_or_logic, &values)
|
||||
Ok(eval_all(&self.filters, self.use_or_logic, &values))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -325,7 +342,9 @@ mod tests {
|
||||
|
||||
fn matches(payload: &str, filters: serde_json::Value, use_or_logic: bool) -> bool {
|
||||
let filters: Vec<Filter> = serde_json::from_value(filters).unwrap();
|
||||
CompiledFilters::new(filters, use_or_logic).matches(payload)
|
||||
CompiledFilters::new(filters, use_or_logic)
|
||||
.matches(payload)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -388,7 +407,22 @@ mod tests {
|
||||
fn test_non_object_payload_never_matches() {
|
||||
let filters = json!([{"key": "a", "value": 1}]);
|
||||
assert!(!matches("[1, 2]", filters.clone(), false));
|
||||
assert!(!matches("nope", filters, true));
|
||||
assert!(!matches("\"nope\"", filters, true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_non_json_payload_is_an_error_unless_accepted_without_keys() {
|
||||
let compile = |filters: serde_json::Value| {
|
||||
CompiledFilters::new(serde_json::from_value(filters).unwrap(), false)
|
||||
};
|
||||
let keyed = compile(json!([{"key": "a", "value": 1}]));
|
||||
assert!(keyed.matches("not json").is_err());
|
||||
assert!(keyed.matches(r#"{"a": 1, "b""#).is_err());
|
||||
assert!(keyed.matches("[1,").is_err());
|
||||
assert!(!keyed.matches("[1, 2]").unwrap());
|
||||
let negated = compile(json!([{"none_of": [{"key": "a", "value": 1}]}]));
|
||||
assert!(negated.matches("not json").unwrap());
|
||||
assert!(negated.matches("[1, 2]").unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -592,9 +626,9 @@ mod tests {
|
||||
false,
|
||||
"u/admin/trigger",
|
||||
);
|
||||
assert!(filters.matches(r#"{"event": "created", "b": 2}"#));
|
||||
assert!(!filters.matches(r#"{"event": "created", "b": 3}"#));
|
||||
assert!(!filters.matches(r#"{"event": "other", "a": 1}"#));
|
||||
assert!(filters.matches(r#"{"event": "created", "b": 2}"#).unwrap());
|
||||
assert!(!filters.matches(r#"{"event": "created", "b": 3}"#).unwrap());
|
||||
assert!(!filters.matches(r#"{"event": "other", "a": 1}"#).unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Generated
+10
@@ -10270,6 +10270,11 @@ properties:
|
||||
description: Logic to apply when evaluating the top-level filters. 'and' requires
|
||||
all of them to match, 'or' requires any of them to match. Nested \`any_of\`/\`all_of\`/\`none_of\`
|
||||
groups carry their own logic.
|
||||
run_on_filter_error:
|
||||
type: boolean
|
||||
description: Whether a message the filters cannot read (not UTF-8, or not JSON)
|
||||
still runs the target, as if the trigger had no filters. A preprocessor then
|
||||
finds the reason in \`event.filter_error\`. When false, such a message is skipped.
|
||||
auto_offset_reset:
|
||||
type: string
|
||||
enum:
|
||||
@@ -11004,6 +11009,11 @@ properties:
|
||||
description: Logic to apply when evaluating the top-level filters. 'and' requires
|
||||
all of them to match, 'or' requires any of them to match. Nested \`any_of\`/\`all_of\`/\`none_of\`
|
||||
groups carry their own logic.
|
||||
run_on_filter_error:
|
||||
type: boolean
|
||||
description: Whether a message the filters cannot read (not UTF-8, or not JSON)
|
||||
still runs the target, as if the trigger had no filters. A preprocessor then
|
||||
finds the reason in \`event.filter_error\`. When false, such a message is skipped.
|
||||
initial_messages:
|
||||
type: array
|
||||
items:
|
||||
|
||||
@@ -112,6 +112,7 @@ export const websocketTriggerRequestSchema = z.object({
|
||||
"none_of": z.array(z.record(z.string(), z.any()))
|
||||
})]).describe("Either a leaf filter, matching a field of the message (parsed as JSON) against a value by equality (or superset, when the value is an object or array) \u2014 addressed by `key` for a top-level field or `path` for a dotted path into nested objects \u2014 or a group nesting sub-filters under a boolean operator (`none_of` matches when none of its sub-filters do).\n")).describe("Filters to match incoming messages (only matching messages trigger the script). Each entry is either a leaf `{key, value}` (top-level field) or `{path, value}` (dotted path into nested objects), or a group `{any_of: [...]}` / `{all_of: [...]}` / `{none_of: [...]}` nesting more entries. Entries at the top level are combined with `filter_logic`."),
|
||||
"filter_logic": z.enum(["and", "or"]).describe("Logic to apply when evaluating the top-level filters. 'and' requires all of them to match, 'or' requires any of them to match. Nested `any_of`/`all_of`/`none_of` groups carry their own logic.").default("and").optional(),
|
||||
"run_on_filter_error": z.boolean().describe("Whether a message the filters cannot read (not UTF-8, or not JSON) still runs the target, as if the trigger had no filters. A preprocessor then finds the reason in `event.filter_error`. When false, such a message is skipped.").default(false).optional(),
|
||||
"initial_messages": z.array(z.union([z.object({
|
||||
"raw_message": z.string()
|
||||
}), z.object({
|
||||
@@ -172,6 +173,7 @@ export const kafkaTriggerRequestSchema = z.object({
|
||||
"none_of": z.array(z.record(z.string(), z.any()))
|
||||
})]).describe("Either a leaf filter, matching a field of the message (parsed as JSON) against a value by equality (or superset, when the value is an object or array) \u2014 addressed by `key` for a top-level field or `path` for a dotted path into nested objects \u2014 or a group nesting sub-filters under a boolean operator (`none_of` matches when none of its sub-filters do).\n")).describe("Filters to match incoming messages (only matching messages trigger the script). Each entry is either a leaf `{key, value}` (top-level field) or `{path, value}` (dotted path into nested objects), or a group `{any_of: [...]}` / `{all_of: [...]}` / `{none_of: [...]}` nesting more entries. Entries at the top level are combined with `filter_logic`."),
|
||||
"filter_logic": z.enum(["and", "or"]).describe("Logic to apply when evaluating the top-level filters. 'and' requires all of them to match, 'or' requires any of them to match. Nested `any_of`/`all_of`/`none_of` groups carry their own logic.").default("and").optional(),
|
||||
"run_on_filter_error": z.boolean().describe("Whether a message the filters cannot read (not UTF-8, or not JSON) still runs the target, as if the trigger had no filters. A preprocessor then finds the reason in `event.filter_error`. When false, such a message is skipped.").default(false).optional(),
|
||||
"auto_offset_reset": z.enum(["latest", "earliest"]).describe("Initial offset behavior when consumer group has no committed offset.").default("latest").optional(),
|
||||
"auto_commit": z.boolean().describe("When true (default), offsets are committed automatically after receiving each message. When false, you must manually commit offsets using the commit_offsets endpoint.").default(true).optional(),
|
||||
"mode": z.enum(["enabled", "disabled", "suspended"]).describe("job trigger mode").optional(),
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
<script lang="ts">
|
||||
import Section from '$lib/components/Section.svelte'
|
||||
import Toggle from '$lib/components/Toggle.svelte'
|
||||
import TriggerFilterList from './TriggerFilterList.svelte'
|
||||
import type { FilterLogic, FilterNode, GroupOp } from './filters'
|
||||
|
||||
interface Props {
|
||||
filters: FilterNode[]
|
||||
filterLogic: FilterLogic
|
||||
runOnFilterError: boolean
|
||||
disabled?: boolean
|
||||
// Set when the runnable receives the payload base64-encoded (e.g. Kafka).
|
||||
// Filters always run on the message parsed as JSON, so we clarify the distinction.
|
||||
@@ -15,6 +17,7 @@
|
||||
let {
|
||||
filters = $bindable(),
|
||||
filterLogic = $bindable(),
|
||||
runOnFilterError = $bindable(),
|
||||
disabled = false,
|
||||
payloadBase64Encoded = false
|
||||
}: Props = $props()
|
||||
@@ -48,4 +51,18 @@
|
||||
<div class="mt-1">
|
||||
<TriggerFilterList bind:filters bind:logic={() => filterLogic, setRootLogic} {disabled} />
|
||||
</div>
|
||||
{#if filters.length > 0}
|
||||
<div class="mt-2">
|
||||
<Toggle
|
||||
bind:checked={runOnFilterError}
|
||||
options={{
|
||||
right: "Run on messages the filters can't read",
|
||||
rightTooltip:
|
||||
'Filters cannot read a message that is not valid JSON. When on, such a message runs the runnable as if the trigger had no filters, and a preprocessor finds the reason in event.filter_error. When off, it is skipped.'
|
||||
}}
|
||||
{disabled}
|
||||
size="xs"
|
||||
/>
|
||||
</div>
|
||||
{/if}
|
||||
</Section>
|
||||
|
||||
@@ -134,6 +134,7 @@
|
||||
let retry: Retry | undefined = $state()
|
||||
let filters: FilterNode[] = $state([])
|
||||
let filterLogic = $state<'and' | 'or'>('and')
|
||||
let runOnFilterError = $state(false)
|
||||
|
||||
let suspendedJobsModal = $state<TriggerSuspendedJobsModal | null>(null)
|
||||
let originalConfig = $state<Record<string, any> | undefined>(undefined)
|
||||
@@ -264,6 +265,7 @@
|
||||
retry = nDefaultValues?.retry ?? undefined
|
||||
filters = nDefaultValues?.filters ?? []
|
||||
filterLogic = 'and'
|
||||
runOnFilterError = false
|
||||
mode = nDefaultValues?.mode ?? 'enabled'
|
||||
permissionedAs = undefined
|
||||
selectedPermissionedAs = undefined
|
||||
@@ -304,6 +306,7 @@
|
||||
retry = cfg?.retry
|
||||
filters = cfg?.filters ?? []
|
||||
filterLogic = cfg?.filter_logic ?? 'and'
|
||||
runOnFilterError = cfg?.run_on_filter_error ?? false
|
||||
permissionedAs = cfg?.permissioned_as
|
||||
selectedPermissionedAs = cfg?.permissioned_as
|
||||
preservePermissionedAs = !!cfg?.permissioned_as
|
||||
@@ -342,6 +345,7 @@
|
||||
topics: kafkaCfg.topics,
|
||||
filters,
|
||||
filter_logic: filterLogic,
|
||||
run_on_filter_error: runOnFilterError,
|
||||
auto_offset_reset: autoOffsetReset,
|
||||
auto_commit: autoCommit,
|
||||
mode,
|
||||
@@ -711,6 +715,7 @@
|
||||
<TriggerFilters
|
||||
bind:filters
|
||||
bind:filterLogic
|
||||
bind:runOnFilterError
|
||||
disabled={!can_write}
|
||||
payloadBase64Encoded
|
||||
/>
|
||||
|
||||
@@ -25,6 +25,7 @@ export async function saveKafkaTriggerFromCfg(
|
||||
topics: cfg.topics,
|
||||
filters: cfg.filters ?? [],
|
||||
filter_logic: cfg.filter_logic ?? 'and',
|
||||
run_on_filter_error: cfg.run_on_filter_error ?? false,
|
||||
auto_offset_reset: cfg.auto_offset_reset ?? 'latest',
|
||||
auto_commit: cfg.auto_commit ?? true,
|
||||
...errorHandlerAndRetries,
|
||||
|
||||
@@ -122,6 +122,7 @@
|
||||
let dirtyUrl = $state(false)
|
||||
let filters: FilterNode[] = $state([])
|
||||
let filterLogic = $state<'and' | 'or'>('and')
|
||||
let runOnFilterError = $state(false)
|
||||
let initial_messages: WebsocketTriggerInitialMessage[] = $state([])
|
||||
let url_runnable_args: Record<string, any> | undefined = $state({})
|
||||
let can_return_message = $state(false)
|
||||
@@ -263,6 +264,7 @@
|
||||
initialPath = ''
|
||||
filters = []
|
||||
filterLogic = 'and'
|
||||
runOnFilterError = false
|
||||
initial_messages = []
|
||||
url_runnable_args = defaultValues?.url_runnable_args ?? {}
|
||||
dirtyPath = false
|
||||
@@ -297,6 +299,7 @@
|
||||
url = cfg?.url
|
||||
filters = cfg?.filters ?? []
|
||||
filterLogic = cfg?.filter_logic ?? 'and'
|
||||
runOnFilterError = cfg?.run_on_filter_error ?? false
|
||||
initial_messages = cfg?.initial_messages ?? []
|
||||
url_runnable_args = cfg?.url_runnable_args
|
||||
can_return_message = cfg?.can_return_message
|
||||
@@ -329,6 +332,7 @@
|
||||
url,
|
||||
filters,
|
||||
filter_logic: filterLogic,
|
||||
run_on_filter_error: runOnFilterError,
|
||||
initial_messages,
|
||||
url_runnable_args,
|
||||
can_return_message,
|
||||
@@ -911,7 +915,7 @@
|
||||
/>
|
||||
{/snippet}
|
||||
<div class="flex flex-col gap-6">
|
||||
<TriggerFilters bind:filters bind:filterLogic disabled={!can_write} />
|
||||
<TriggerFilters bind:filters bind:filterLogic bind:runOnFilterError disabled={!can_write} />
|
||||
<div class="min-h-96">
|
||||
<Tabs bind:selected={optionTabSelected}>
|
||||
<Tab value="error_handler" label="Error Handler" />
|
||||
|
||||
@@ -26,6 +26,7 @@ export async function saveWebsocketTriggerFromCfg(
|
||||
url: triggerCfg.url,
|
||||
filters: triggerCfg.filters,
|
||||
filter_logic: triggerCfg.filter_logic ?? 'and',
|
||||
run_on_filter_error: triggerCfg.run_on_filter_error ?? false,
|
||||
initial_messages: triggerCfg.initial_messages,
|
||||
url_runnable_args: triggerCfg.url_runnable_args,
|
||||
can_return_message: triggerCfg.can_return_message,
|
||||
|
||||
@@ -116,6 +116,7 @@ export const TRIGGER_KINDS: Record<
|
||||
'url',
|
||||
'filters',
|
||||
'filter_logic',
|
||||
'run_on_filter_error',
|
||||
'initial_messages',
|
||||
'url_runnable_args',
|
||||
'can_return_message',
|
||||
@@ -180,6 +181,7 @@ export const TRIGGER_KINDS: Record<
|
||||
'topics',
|
||||
'filters',
|
||||
'filter_logic',
|
||||
'run_on_filter_error',
|
||||
'auto_offset_reset',
|
||||
'auto_commit'
|
||||
],
|
||||
|
||||
@@ -100,6 +100,11 @@ properties:
|
||||
description: Logic to apply when evaluating the top-level filters. 'and' requires
|
||||
all of them to match, 'or' requires any of them to match. Nested `any_of`/`all_of`/`none_of`
|
||||
groups carry their own logic.
|
||||
run_on_filter_error:
|
||||
type: boolean
|
||||
description: Whether a message the filters cannot read (not UTF-8, or not JSON)
|
||||
still runs the target, as if the trigger had no filters. A preprocessor then
|
||||
finds the reason in `event.filter_error`. When false, such a message is skipped.
|
||||
auto_offset_reset:
|
||||
type: string
|
||||
enum:
|
||||
|
||||
@@ -93,6 +93,11 @@ properties:
|
||||
description: Logic to apply when evaluating the top-level filters. 'and' requires
|
||||
all of them to match, 'or' requires any of them to match. Nested `any_of`/`all_of`/`none_of`
|
||||
groups carry their own logic.
|
||||
run_on_filter_error:
|
||||
type: boolean
|
||||
description: Whether a message the filters cannot read (not UTF-8, or not JSON)
|
||||
still runs the target, as if the trigger had no filters. A preprocessor then
|
||||
finds the reason in `event.filter_error`. When false, such a message is skipped.
|
||||
initial_messages:
|
||||
type: array
|
||||
items:
|
||||
|
||||
Reference in New Issue
Block a user