feat: optionally run trigger messages the filters cannot read (#12226)

* fix: route trigger messages the filters cannot parse to the error handler

A Kafka or websocket trigger with filters silently skipped a message that
was not UTF-8 or not JSON, and Kafka committed its offset, so the message
was lost without a trace. Such a message now goes to the trigger's error
handler (script targets) or the workspace error handler, with the raw
payload and its topic/partition/offset.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: take the unfilterable-message handler as a RunnableRef after merging main

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: keep none_of semantics for non-object JSON, honour workspace handler mutes, throttle the no-handler alert

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* docs: state the caller contract of push_error_handler_with_parent

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* chore: point ee-repo-ref at the kafka companion branch

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: hold unparseable-message handlers of a suspended trigger, throttle them on websocket

A suspended trigger now holds the handler run like its other runs, tagged
with the trigger so resuming releases it. Websocket text keepalives would
start a handler run per frame, so websocket caps it at one per trigger per
minute; Kafka keeps one per message. A parentless handler run carries no
token prefix.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: release a held error handler run in place when resuming an edited trigger

Resuming re-pushes runs held before the trigger's last edit against its
target. A held error handler run would then run the target with the
handler's args, so it is released in place.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* feat: add run_on_filter_error to kafka and websocket triggers

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: treat a truncated array as unreadable, not as a filter mismatch

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* chore: update ee-repo-ref to 695eddf214f6a0e6cf1249ee2c20e716788f0766

This commit updates the EE repository reference after PR #954 was merged in windmill-ee-private.

Previous ee-repo-ref: c8a26463f72ed76423b14e6a1aaf6376e58f2c4c

New ee-repo-ref: 695eddf214f6a0e6cf1249ee2c20e716788f0766

Automated by sync-ee-ref workflow.

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com>
Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
This commit is contained in:
hugocasa
2026-11-05 23:33:47 +01:00
committed by GitHub
co-authored by Claude Opus 5.5 windmill-internal-app[bot] Ruben Fiszel
parent 964f1af303
commit a4ae82fd69
29 changed files with 211 additions and 72 deletions
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE\n websocket_trigger\n SET\n url = $1,\n script_path = $2,\n path = $3,\n is_flow = $4,\n filters = $5,\n filter_logic = $6,\n initial_messages = $7,\n url_runnable_args = $8,\n edited_by = $9,\n permissioned_as = $10,\n can_return_message = $11,\n can_return_error_result = $12,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $15,\n error_handler_args = $16,\n retry = $17,\n heartbeat = $18\n WHERE\n workspace_id = $13 AND path = $14\n ",
"query": "\n UPDATE\n websocket_trigger\n SET\n url = $1,\n script_path = $2,\n path = $3,\n is_flow = $4,\n filters = $5,\n filter_logic = $6,\n initial_messages = $7,\n url_runnable_args = $8,\n edited_by = $9,\n permissioned_as = $10,\n can_return_message = $11,\n can_return_error_result = $12,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $15,\n error_handler_args = $16,\n retry = $17,\n heartbeat = $18,\n run_on_filter_error = $19\n WHERE\n workspace_id = $13 AND path = $14\n ",
"describe": {
"columns": [],
"parameters": {
@@ -22,10 +22,11 @@
"Varchar",
"Jsonb",
"Jsonb",
"Jsonb"
"Jsonb",
"Bool"
]
},
"nullable": []
},
"hash": "492edd53e497a45c314d41044e85a5e3492227b5049dc6870d60817ad929e7af"
"hash": "418d257ac50ec518d960aa286a003caf748a5423b208d897ff4e23aae0330007"
}
@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO websocket_trigger (\n path, url, script_path, is_flow, workspace_id, edited_by, edited_at,\n extra_perms, server_id, last_server_ping, error, filters, initial_messages,\n url_runnable_args, can_return_message, error_handler_path, error_handler_args,\n retry, can_return_error_result, mode, permissioned_as, filter_logic, labels,\n heartbeat\n )\n SELECT\n path, url, script_path, is_flow, $1, edited_by, edited_at,\n extra_perms, NULL, NULL, NULL, filters, initial_messages,\n url_runnable_args, can_return_message, error_handler_path, error_handler_args,\n retry, can_return_error_result, 'disabled'::TRIGGER_MODE, permissioned_as, filter_logic, labels,\n heartbeat\n FROM websocket_trigger WHERE workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text"
]
},
"nullable": []
},
"hash": "5fd769bde29e88eb43f5dc0ecd7f4a1fe20d216d818d69eea4ddbaace5233137"
}
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE kafka_trigger\n SET\n kafka_resource_path = $1,\n group_id = $2,\n topics = $3,\n filters = $4,\n filter_logic = $5,\n auto_offset_reset = $6,\n auto_commit = $7,\n script_path = $8,\n path = $9,\n is_flow = $10,\n edited_by = $11,\n permissioned_as = $12,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $15,\n error_handler_args = $16,\n retry = $17\n WHERE\n workspace_id = $13 AND path = $14\n ",
"query": "\n UPDATE kafka_trigger\n SET\n kafka_resource_path = $1,\n group_id = $2,\n topics = $3,\n filters = $4,\n filter_logic = $5,\n auto_offset_reset = $6,\n auto_commit = $7,\n script_path = $8,\n path = $9,\n is_flow = $10,\n edited_by = $11,\n permissioned_as = $12,\n edited_at = now(),\n server_id = NULL,\n error = NULL,\n error_handler_path = $15,\n error_handler_args = $16,\n retry = $17,\n run_on_filter_error = $18\n WHERE\n workspace_id = $13 AND path = $14\n ",
"describe": {
"columns": [],
"parameters": {
@@ -21,10 +21,11 @@
"Text",
"Varchar",
"Jsonb",
"Jsonb"
"Jsonb",
"Bool"
]
},
"nullable": []
},
"hash": "68c19cb0e18b94870bbe81f9aab92ba37da67cd2a56834c9d1378eab7551284d"
"hash": "6f16ddb07e8543e12a12bd5b312da224e00c5c0ac1ba4abc16fe104d28b59da1"
}
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO kafka_trigger (\n workspace_id,\n path,\n kafka_resource_path,\n group_id,\n topics,\n filters,\n filter_logic,\n auto_offset_reset,\n auto_commit,\n script_path,\n is_flow,\n mode,\n edited_by,\n permissioned_as,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, now(), $15, $16, $17\n )\n ",
"query": "\n INSERT INTO kafka_trigger (\n workspace_id,\n path,\n kafka_resource_path,\n group_id,\n topics,\n filters,\n filter_logic,\n auto_offset_reset,\n auto_commit,\n script_path,\n is_flow,\n mode,\n edited_by,\n permissioned_as,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry,\n run_on_filter_error\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, now(), $15, $16, $17, $18\n )\n ",
"describe": {
"columns": [],
"parameters": {
@@ -32,10 +32,11 @@
"Varchar",
"Varchar",
"Jsonb",
"Jsonb"
"Jsonb",
"Bool"
]
},
"nullable": []
},
"hash": "6a8f4ed9946bb2a3c5e90695c90b70aa2e83fcb5aa0c953febdd9bac2d95bbec"
"hash": "8d9c5c4bcdc1ca66388fdf35a1573ba28a1891f46c80bcd3b3e4f058e037c26a"
}
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO kafka_trigger (\n path, kafka_resource_path, topics, group_id, script_path, is_flow,\n workspace_id, edited_by, edited_at, extra_perms, server_id,\n last_server_ping, error, error_handler_path, error_handler_args, retry,\n mode, filters, auto_offset_reset, reset_offset, auto_commit,\n permissioned_as, filter_logic, labels, run_on_filter_error\n )\n SELECT\n path, kafka_resource_path, topics, group_id, script_path, is_flow,\n $1, edited_by, edited_at, extra_perms, NULL,\n NULL, NULL, error_handler_path, error_handler_args, retry,\n 'disabled'::TRIGGER_MODE, filters, auto_offset_reset, reset_offset, auto_commit,\n permissioned_as, filter_logic, labels, run_on_filter_error\n FROM kafka_trigger WHERE workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text"
]
},
"nullable": []
},
"hash": "a8f792d147acc47791a8cbc473a13a9440d22500eef72b11065b9badbeb5101a"
}
@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO kafka_trigger (\n path, kafka_resource_path, topics, group_id, script_path, is_flow,\n workspace_id, edited_by, edited_at, extra_perms, server_id,\n last_server_ping, error, error_handler_path, error_handler_args, retry,\n mode, filters, auto_offset_reset, reset_offset, auto_commit,\n permissioned_as, filter_logic, labels\n )\n SELECT\n path, kafka_resource_path, topics, group_id, script_path, is_flow,\n $1, edited_by, edited_at, extra_perms, NULL,\n NULL, NULL, error_handler_path, error_handler_args, retry,\n 'disabled'::TRIGGER_MODE, filters, auto_offset_reset, reset_offset, auto_commit,\n permissioned_as, filter_logic, labels\n FROM kafka_trigger WHERE workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text"
]
},
"nullable": []
},
"hash": "c9c0b92c1fea9b4bdafba32da60e5579a4c2940bed63097ffec7f757d9882667"
}
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO websocket_trigger (\n path, url, script_path, is_flow, workspace_id, edited_by, edited_at,\n extra_perms, server_id, last_server_ping, error, filters, initial_messages,\n url_runnable_args, can_return_message, error_handler_path, error_handler_args,\n retry, can_return_error_result, mode, permissioned_as, filter_logic, labels,\n heartbeat, run_on_filter_error\n )\n SELECT\n path, url, script_path, is_flow, $1, edited_by, edited_at,\n extra_perms, NULL, NULL, NULL, filters, initial_messages,\n url_runnable_args, can_return_message, error_handler_path, error_handler_args,\n retry, can_return_error_result, 'disabled'::TRIGGER_MODE, permissioned_as, filter_logic, labels,\n heartbeat, run_on_filter_error\n FROM websocket_trigger WHERE workspace_id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text"
]
},
"nullable": []
},
"hash": "cd54e1e83205fc161e4046f8b5e3bfda4e38156cd89e7a0b7f8164c98cf1c608"
}
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO websocket_trigger (\n workspace_id,\n path,\n url,\n script_path,\n is_flow,\n mode,\n filters,\n filter_logic,\n initial_messages,\n url_runnable_args,\n edited_by,\n can_return_message,\n can_return_error_result,\n permissioned_as,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry,\n heartbeat\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, now(), $15, $16, $17, $18\n )\n ",
"query": "\n INSERT INTO websocket_trigger (\n workspace_id,\n path,\n url,\n script_path,\n is_flow,\n mode,\n filters,\n filter_logic,\n initial_messages,\n url_runnable_args,\n edited_by,\n can_return_message,\n can_return_error_result,\n permissioned_as,\n edited_at,\n error_handler_path,\n error_handler_args,\n retry,\n heartbeat,\n run_on_filter_error\n ) VALUES (\n $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, now(), $15, $16, $17, $18, $19\n )\n ",
"describe": {
"columns": [],
"parameters": {
@@ -33,10 +33,11 @@
"Varchar",
"Jsonb",
"Jsonb",
"Jsonb"
"Jsonb",
"Bool"
]
},
"nullable": []
},
"hash": "11fd92de8688ef6b4d524aade507850a9cb3e097f2d957219d04ad134a5e0399"
"hash": "cdc2371af8d627a410e9ba0287eb1a35b08cbab2fe0d2a0e0d65a98658d42e22"
}
+1 -1
View File
@@ -1 +1 @@
0b763cc905bb67b17728794bb52bfdf336fe25fb
695eddf214f6a0e6cf1249ee2c20e716788f0766
@@ -0,0 +1,2 @@
ALTER TABLE kafka_trigger DROP COLUMN run_on_filter_error;
ALTER TABLE websocket_trigger DROP COLUMN run_on_filter_error;
@@ -0,0 +1,2 @@
ALTER TABLE kafka_trigger ADD COLUMN run_on_filter_error BOOLEAN NOT NULL DEFAULT false;
ALTER TABLE websocket_trigger ADD COLUMN run_on_filter_error BOOLEAN NOT NULL DEFAULT false;
@@ -7744,14 +7744,14 @@ async fn clone_triggers_and_schedules(
extra_perms, server_id, last_server_ping, error, filters, initial_messages,
url_runnable_args, can_return_message, error_handler_path, error_handler_args,
retry, can_return_error_result, mode, permissioned_as, filter_logic, labels,
heartbeat
heartbeat, run_on_filter_error
)
SELECT
path, url, script_path, is_flow, $1, edited_by, edited_at,
extra_perms, NULL, NULL, NULL, filters, initial_messages,
url_runnable_args, can_return_message, error_handler_path, error_handler_args,
retry, can_return_error_result, 'disabled'::TRIGGER_MODE, permissioned_as, filter_logic, labels,
heartbeat
heartbeat, run_on_filter_error
FROM websocket_trigger WHERE workspace_id = $2"#,
target_workspace_id,
source_workspace_id,
@@ -7765,14 +7765,14 @@ async fn clone_triggers_and_schedules(
workspace_id, edited_by, edited_at, extra_perms, server_id,
last_server_ping, error, error_handler_path, error_handler_args, retry,
mode, filters, auto_offset_reset, reset_offset, auto_commit,
permissioned_as, filter_logic, labels
permissioned_as, filter_logic, labels, run_on_filter_error
)
SELECT
path, kafka_resource_path, topics, group_id, script_path, is_flow,
$1, edited_by, edited_at, extra_perms, NULL,
NULL, NULL, error_handler_path, error_handler_args, retry,
'disabled'::TRIGGER_MODE, filters, auto_offset_reset, reset_offset, auto_commit,
permissioned_as, filter_logic, labels
permissioned_as, filter_logic, labels, run_on_filter_error
FROM kafka_trigger WHERE workspace_id = $2"#,
target_workspace_id,
source_workspace_id,
+24
View File
@@ -35155,6 +35155,10 @@ components:
- or
default: and
description: "Logic to apply when evaluating the top-level filters. 'and' requires all of them to match, 'or' requires any of them to match. Nested `any_of`/`all_of`/`none_of` groups carry their own logic."
run_on_filter_error:
type: boolean
default: false
description: "Whether a message the filters cannot read (not UTF-8, or not JSON) still runs the target, as if the trigger had no filters. A preprocessor then finds the reason in `event.filter_error`. When false, such a message is skipped."
initial_messages:
type: array
nullable: true
@@ -35222,6 +35226,10 @@ components:
- or
default: and
description: "Logic to apply when evaluating the top-level filters. 'and' requires all of them to match, 'or' requires any of them to match. Nested `any_of`/`all_of`/`none_of` groups carry their own logic."
run_on_filter_error:
type: boolean
default: false
description: "Whether a message the filters cannot read (not UTF-8, or not JSON) still runs the target, as if the trigger had no filters. A preprocessor then finds the reason in `event.filter_error`. When false, such a message is skipped."
initial_messages:
type: array
nullable: true
@@ -35300,6 +35308,10 @@ components:
- or
default: and
description: "Logic to apply when evaluating the top-level filters. 'and' requires all of them to match, 'or' requires any of them to match. Nested `any_of`/`all_of`/`none_of` groups carry their own logic."
run_on_filter_error:
type: boolean
default: false
description: "Whether a message the filters cannot read (not UTF-8, or not JSON) still runs the target, as if the trigger had no filters. A preprocessor then finds the reason in `event.filter_error`. When false, such a message is skipped."
initial_messages:
type: array
nullable: true
@@ -36981,6 +36993,10 @@ components:
- or
default: and
description: "Logic to apply when evaluating the top-level filters. 'and' requires all of them to match, 'or' requires any of them to match. Nested `any_of`/`all_of`/`none_of` groups carry their own logic."
run_on_filter_error:
type: boolean
default: false
description: "Whether a message the filters cannot read (not UTF-8, or not JSON) still runs the target, as if the trigger had no filters. A preprocessor then finds the reason in `event.filter_error`. When false, such a message is skipped."
auto_offset_reset:
type: string
enum:
@@ -37055,6 +37071,10 @@ components:
- or
default: and
description: "Logic to apply when evaluating the top-level filters. 'and' requires all of them to match, 'or' requires any of them to match. Nested `any_of`/`all_of`/`none_of` groups carry their own logic."
run_on_filter_error:
type: boolean
default: false
description: "Whether a message the filters cannot read (not UTF-8, or not JSON) still runs the target, as if the trigger had no filters. A preprocessor then finds the reason in `event.filter_error`. When false, such a message is skipped."
auto_offset_reset:
type: string
enum:
@@ -37125,6 +37145,10 @@ components:
- or
default: and
description: "Logic to apply when evaluating the top-level filters. 'and' requires all of them to match, 'or' requires any of them to match. Nested `any_of`/`all_of`/`none_of` groups carry their own logic."
run_on_filter_error:
type: boolean
default: false
description: "Whether a message the filters cannot read (not UTF-8, or not JSON) still runs the target, as if the trigger had no filters. A preprocessor then finds the reason in `event.filter_error`. When false, such a message is skipped."
auto_offset_reset:
type: string
enum:
File diff suppressed because one or more lines are too long
@@ -74,6 +74,7 @@ impl TriggerCrud for WebsocketTrigger {
"url",
"filters",
"filter_logic",
"run_on_filter_error",
"initial_messages",
"url_runnable_args",
"can_return_message",
@@ -176,9 +177,10 @@ impl TriggerCrud for WebsocketTrigger {
error_handler_path,
error_handler_args,
retry,
heartbeat
heartbeat,
run_on_filter_error
) VALUES (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, now(), $15, $16, $17, $18
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, now(), $15, $16, $17, $18, $19
)
"#,
w_id,
@@ -201,7 +203,8 @@ impl TriggerCrud for WebsocketTrigger {
trigger.error_handling.error_handler_path,
trigger.error_handling.error_handler_args as _,
trigger.error_handling.retry as _,
trigger.config.heartbeat.map(SqlxJson) as _
trigger.config.heartbeat.map(SqlxJson) as _,
trigger.config.run_on_filter_error
)
.execute(&mut *tx)
.await?;
@@ -258,7 +261,8 @@ impl TriggerCrud for WebsocketTrigger {
error_handler_path = $15,
error_handler_args = $16,
retry = $17,
heartbeat = $18
heartbeat = $18,
run_on_filter_error = $19
WHERE
workspace_id = $13 AND path = $14
",
@@ -283,7 +287,8 @@ impl TriggerCrud for WebsocketTrigger {
trigger.error_handling.error_handler_path,
trigger.error_handling.error_handler_args as _,
trigger.error_handling.retry as _,
trigger.config.heartbeat.map(SqlxJson) as _
trigger.config.heartbeat.map(SqlxJson) as _,
trigger.config.run_on_filter_error
)
.execute(&mut *tx)
.await?;
@@ -51,6 +51,8 @@ pub struct WebsocketConfig {
pub filters: Vec<SqlxJson<Box<RawValue>>>,
#[serde(default = "default_filter_logic")]
pub filter_logic: String,
#[serde(default)]
pub run_on_filter_error: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub initial_messages: Option<Vec<SqlxJson<Box<RawValue>>>>,
#[serde(skip_serializing_if = "Option::is_none")]
@@ -69,6 +71,8 @@ pub struct WebsocketConfigRequest {
filters: Vec<serde_json::Value>,
#[serde(default = "default_filter_logic")]
filter_logic: String,
#[serde(default)]
run_on_filter_error: bool,
initial_messages: Option<Vec<serde_json::Value>>,
url_runnable_args: Option<serde_json::Value>,
can_return_message: bool,
@@ -527,10 +527,21 @@ impl Listener for WebsocketTrigger {
}
}
if filters.matches(&text) {
let trigger_info = HashMap::from([
("url".to_string(), to_raw_value(&listening_trigger.trigger_config.url)),
]);
let mut trigger_info = HashMap::from([
("url".to_string(), to_raw_value(&listening_trigger.trigger_config.url)),
]);
let should_handle = match filters.matches(&text) {
Ok(matched) => matched,
Err(error) if listening_trigger.trigger_config.run_on_filter_error => {
trigger_info.insert("filter_error".to_string(), to_raw_value(&error));
true
}
Err(error) => {
tracing::warn!("WebSocket trigger {} skipped a message: its filters could not be applied, {}", listening_trigger.path, error);
false
}
};
if should_handle {
let _ = self.handle_event(db, listening_trigger, text, trigger_info, return_message_channels.clone()).await;
}
},
@@ -80,6 +80,7 @@ fn trigger(url: String, trigger_mode: bool) -> ListeningTrigger<WebsocketConfig>
url,
filters: vec![],
filter_logic: "and".to_string(),
run_on_filter_error: false,
initial_messages: None,
url_runnable_args: None,
can_return_message: false,
+47 -13
View File
@@ -1,5 +1,5 @@
use serde::{
de::{self, MapAccess, Visitor},
de::{self, IgnoredAny, MapAccess, Visitor},
Deserialize, Deserializer,
};
use serde_json::{value::RawValue, Value};
@@ -113,18 +113,35 @@ impl CompiledFilters {
Ok(())
}
/// Whether `text`, parsed as a JSON object, satisfies the filters.
pub fn matches(&self, text: &str) -> bool {
/// Whether `text`, parsed as a JSON object, satisfies the filters. Anything else is judged
/// as an object with no keys, which a `none_of` filter accepts. When that rejects it, JSON
/// that is not an object is a mismatch, and text that is not JSON an error the trigger's
/// `run_on_filter_error` setting decides on.
pub fn matches(&self, text: &str) -> std::result::Result<bool, String> {
if self.filters.is_empty() {
return true;
return Ok(true);
}
let mut deserializer = serde_json::Deserializer::from_str(text);
let values =
Deserializer::deserialize_map(&mut deserializer, KeysVisitor { keys: &self.keys })
.unwrap_or_default();
let values = match Deserializer::deserialize_map(
&mut deserializer,
KeysVisitor { keys: &self.keys },
) {
Ok(values) => values,
Err(_) if eval_all(&self.filters, self.use_or_logic, &HashMap::new()) => {
return Ok(true)
}
// A data error is raised on the first token (`[`, `"`, ...), before the rest is read:
// parse it all so a truncated array is not taken for valid JSON that is not an object.
Err(err) if err.is_data() => {
return serde_json::from_str::<IgnoredAny>(text)
.map(|_| false)
.map_err(|err| format!("payload is not valid JSON: {err}"))
}
Err(err) => return Err(format!("payload is not valid JSON: {err}")),
};
eval_all(&self.filters, self.use_or_logic, &values)
Ok(eval_all(&self.filters, self.use_or_logic, &values))
}
}
@@ -325,7 +342,9 @@ mod tests {
fn matches(payload: &str, filters: serde_json::Value, use_or_logic: bool) -> bool {
let filters: Vec<Filter> = serde_json::from_value(filters).unwrap();
CompiledFilters::new(filters, use_or_logic).matches(payload)
CompiledFilters::new(filters, use_or_logic)
.matches(payload)
.unwrap()
}
#[test]
@@ -388,7 +407,22 @@ mod tests {
fn test_non_object_payload_never_matches() {
let filters = json!([{"key": "a", "value": 1}]);
assert!(!matches("[1, 2]", filters.clone(), false));
assert!(!matches("nope", filters, true));
assert!(!matches("\"nope\"", filters, true));
}
#[test]
fn test_non_json_payload_is_an_error_unless_accepted_without_keys() {
let compile = |filters: serde_json::Value| {
CompiledFilters::new(serde_json::from_value(filters).unwrap(), false)
};
let keyed = compile(json!([{"key": "a", "value": 1}]));
assert!(keyed.matches("not json").is_err());
assert!(keyed.matches(r#"{"a": 1, "b""#).is_err());
assert!(keyed.matches("[1,").is_err());
assert!(!keyed.matches("[1, 2]").unwrap());
let negated = compile(json!([{"none_of": [{"key": "a", "value": 1}]}]));
assert!(negated.matches("not json").unwrap());
assert!(negated.matches("[1, 2]").unwrap());
}
#[test]
@@ -592,9 +626,9 @@ mod tests {
false,
"u/admin/trigger",
);
assert!(filters.matches(r#"{"event": "created", "b": 2}"#));
assert!(!filters.matches(r#"{"event": "created", "b": 3}"#));
assert!(!filters.matches(r#"{"event": "other", "a": 1}"#));
assert!(filters.matches(r#"{"event": "created", "b": 2}"#).unwrap());
assert!(!filters.matches(r#"{"event": "created", "b": 3}"#).unwrap());
assert!(!filters.matches(r#"{"event": "other", "a": 1}"#).unwrap());
}
#[test]
+10
View File
@@ -10270,6 +10270,11 @@ properties:
description: Logic to apply when evaluating the top-level filters. 'and' requires
all of them to match, 'or' requires any of them to match. Nested \`any_of\`/\`all_of\`/\`none_of\`
groups carry their own logic.
run_on_filter_error:
type: boolean
description: Whether a message the filters cannot read (not UTF-8, or not JSON)
still runs the target, as if the trigger had no filters. A preprocessor then
finds the reason in \`event.filter_error\`. When false, such a message is skipped.
auto_offset_reset:
type: string
enum:
@@ -11004,6 +11009,11 @@ properties:
description: Logic to apply when evaluating the top-level filters. 'and' requires
all of them to match, 'or' requires any of them to match. Nested \`any_of\`/\`all_of\`/\`none_of\`
groups carry their own logic.
run_on_filter_error:
type: boolean
description: Whether a message the filters cannot read (not UTF-8, or not JSON)
still runs the target, as if the trigger had no filters. A preprocessor then
finds the reason in \`event.filter_error\`. When false, such a message is skipped.
initial_messages:
type: array
items:
@@ -112,6 +112,7 @@ export const websocketTriggerRequestSchema = z.object({
"none_of": z.array(z.record(z.string(), z.any()))
})]).describe("Either a leaf filter, matching a field of the message (parsed as JSON) against a value by equality (or superset, when the value is an object or array) \u2014 addressed by `key` for a top-level field or `path` for a dotted path into nested objects \u2014 or a group nesting sub-filters under a boolean operator (`none_of` matches when none of its sub-filters do).\n")).describe("Filters to match incoming messages (only matching messages trigger the script). Each entry is either a leaf `{key, value}` (top-level field) or `{path, value}` (dotted path into nested objects), or a group `{any_of: [...]}` / `{all_of: [...]}` / `{none_of: [...]}` nesting more entries. Entries at the top level are combined with `filter_logic`."),
"filter_logic": z.enum(["and", "or"]).describe("Logic to apply when evaluating the top-level filters. 'and' requires all of them to match, 'or' requires any of them to match. Nested `any_of`/`all_of`/`none_of` groups carry their own logic.").default("and").optional(),
"run_on_filter_error": z.boolean().describe("Whether a message the filters cannot read (not UTF-8, or not JSON) still runs the target, as if the trigger had no filters. A preprocessor then finds the reason in `event.filter_error`. When false, such a message is skipped.").default(false).optional(),
"initial_messages": z.array(z.union([z.object({
"raw_message": z.string()
}), z.object({
@@ -172,6 +173,7 @@ export const kafkaTriggerRequestSchema = z.object({
"none_of": z.array(z.record(z.string(), z.any()))
})]).describe("Either a leaf filter, matching a field of the message (parsed as JSON) against a value by equality (or superset, when the value is an object or array) \u2014 addressed by `key` for a top-level field or `path` for a dotted path into nested objects \u2014 or a group nesting sub-filters under a boolean operator (`none_of` matches when none of its sub-filters do).\n")).describe("Filters to match incoming messages (only matching messages trigger the script). Each entry is either a leaf `{key, value}` (top-level field) or `{path, value}` (dotted path into nested objects), or a group `{any_of: [...]}` / `{all_of: [...]}` / `{none_of: [...]}` nesting more entries. Entries at the top level are combined with `filter_logic`."),
"filter_logic": z.enum(["and", "or"]).describe("Logic to apply when evaluating the top-level filters. 'and' requires all of them to match, 'or' requires any of them to match. Nested `any_of`/`all_of`/`none_of` groups carry their own logic.").default("and").optional(),
"run_on_filter_error": z.boolean().describe("Whether a message the filters cannot read (not UTF-8, or not JSON) still runs the target, as if the trigger had no filters. A preprocessor then finds the reason in `event.filter_error`. When false, such a message is skipped.").default(false).optional(),
"auto_offset_reset": z.enum(["latest", "earliest"]).describe("Initial offset behavior when consumer group has no committed offset.").default("latest").optional(),
"auto_commit": z.boolean().describe("When true (default), offsets are committed automatically after receiving each message. When false, you must manually commit offsets using the commit_offsets endpoint.").default(true).optional(),
"mode": z.enum(["enabled", "disabled", "suspended"]).describe("job trigger mode").optional(),
@@ -1,11 +1,13 @@
<script lang="ts">
import Section from '$lib/components/Section.svelte'
import Toggle from '$lib/components/Toggle.svelte'
import TriggerFilterList from './TriggerFilterList.svelte'
import type { FilterLogic, FilterNode, GroupOp } from './filters'
interface Props {
filters: FilterNode[]
filterLogic: FilterLogic
runOnFilterError: boolean
disabled?: boolean
// Set when the runnable receives the payload base64-encoded (e.g. Kafka).
// Filters always run on the message parsed as JSON, so we clarify the distinction.
@@ -15,6 +17,7 @@
let {
filters = $bindable(),
filterLogic = $bindable(),
runOnFilterError = $bindable(),
disabled = false,
payloadBase64Encoded = false
}: Props = $props()
@@ -48,4 +51,18 @@
<div class="mt-1">
<TriggerFilterList bind:filters bind:logic={() => filterLogic, setRootLogic} {disabled} />
</div>
{#if filters.length > 0}
<div class="mt-2">
<Toggle
bind:checked={runOnFilterError}
options={{
right: "Run on messages the filters can't read",
rightTooltip:
'Filters cannot read a message that is not valid JSON. When on, such a message runs the runnable as if the trigger had no filters, and a preprocessor finds the reason in event.filter_error. When off, it is skipped.'
}}
{disabled}
size="xs"
/>
</div>
{/if}
</Section>
@@ -134,6 +134,7 @@
let retry: Retry | undefined = $state()
let filters: FilterNode[] = $state([])
let filterLogic = $state<'and' | 'or'>('and')
let runOnFilterError = $state(false)
let suspendedJobsModal = $state<TriggerSuspendedJobsModal | null>(null)
let originalConfig = $state<Record<string, any> | undefined>(undefined)
@@ -264,6 +265,7 @@
retry = nDefaultValues?.retry ?? undefined
filters = nDefaultValues?.filters ?? []
filterLogic = 'and'
runOnFilterError = false
mode = nDefaultValues?.mode ?? 'enabled'
permissionedAs = undefined
selectedPermissionedAs = undefined
@@ -304,6 +306,7 @@
retry = cfg?.retry
filters = cfg?.filters ?? []
filterLogic = cfg?.filter_logic ?? 'and'
runOnFilterError = cfg?.run_on_filter_error ?? false
permissionedAs = cfg?.permissioned_as
selectedPermissionedAs = cfg?.permissioned_as
preservePermissionedAs = !!cfg?.permissioned_as
@@ -342,6 +345,7 @@
topics: kafkaCfg.topics,
filters,
filter_logic: filterLogic,
run_on_filter_error: runOnFilterError,
auto_offset_reset: autoOffsetReset,
auto_commit: autoCommit,
mode,
@@ -711,6 +715,7 @@
<TriggerFilters
bind:filters
bind:filterLogic
bind:runOnFilterError
disabled={!can_write}
payloadBase64Encoded
/>
@@ -25,6 +25,7 @@ export async function saveKafkaTriggerFromCfg(
topics: cfg.topics,
filters: cfg.filters ?? [],
filter_logic: cfg.filter_logic ?? 'and',
run_on_filter_error: cfg.run_on_filter_error ?? false,
auto_offset_reset: cfg.auto_offset_reset ?? 'latest',
auto_commit: cfg.auto_commit ?? true,
...errorHandlerAndRetries,
@@ -122,6 +122,7 @@
let dirtyUrl = $state(false)
let filters: FilterNode[] = $state([])
let filterLogic = $state<'and' | 'or'>('and')
let runOnFilterError = $state(false)
let initial_messages: WebsocketTriggerInitialMessage[] = $state([])
let url_runnable_args: Record<string, any> | undefined = $state({})
let can_return_message = $state(false)
@@ -263,6 +264,7 @@
initialPath = ''
filters = []
filterLogic = 'and'
runOnFilterError = false
initial_messages = []
url_runnable_args = defaultValues?.url_runnable_args ?? {}
dirtyPath = false
@@ -297,6 +299,7 @@
url = cfg?.url
filters = cfg?.filters ?? []
filterLogic = cfg?.filter_logic ?? 'and'
runOnFilterError = cfg?.run_on_filter_error ?? false
initial_messages = cfg?.initial_messages ?? []
url_runnable_args = cfg?.url_runnable_args
can_return_message = cfg?.can_return_message
@@ -329,6 +332,7 @@
url,
filters,
filter_logic: filterLogic,
run_on_filter_error: runOnFilterError,
initial_messages,
url_runnable_args,
can_return_message,
@@ -911,7 +915,7 @@
/>
{/snippet}
<div class="flex flex-col gap-6">
<TriggerFilters bind:filters bind:filterLogic disabled={!can_write} />
<TriggerFilters bind:filters bind:filterLogic bind:runOnFilterError disabled={!can_write} />
<div class="min-h-96">
<Tabs bind:selected={optionTabSelected}>
<Tab value="error_handler" label="Error Handler" />
@@ -26,6 +26,7 @@ export async function saveWebsocketTriggerFromCfg(
url: triggerCfg.url,
filters: triggerCfg.filters,
filter_logic: triggerCfg.filter_logic ?? 'and',
run_on_filter_error: triggerCfg.run_on_filter_error ?? false,
initial_messages: triggerCfg.initial_messages,
url_runnable_args: triggerCfg.url_runnable_args,
can_return_message: triggerCfg.can_return_message,
@@ -116,6 +116,7 @@ export const TRIGGER_KINDS: Record<
'url',
'filters',
'filter_logic',
'run_on_filter_error',
'initial_messages',
'url_runnable_args',
'can_return_message',
@@ -180,6 +181,7 @@ export const TRIGGER_KINDS: Record<
'topics',
'filters',
'filter_logic',
'run_on_filter_error',
'auto_offset_reset',
'auto_commit'
],
@@ -100,6 +100,11 @@ properties:
description: Logic to apply when evaluating the top-level filters. 'and' requires
all of them to match, 'or' requires any of them to match. Nested `any_of`/`all_of`/`none_of`
groups carry their own logic.
run_on_filter_error:
type: boolean
description: Whether a message the filters cannot read (not UTF-8, or not JSON)
still runs the target, as if the trigger had no filters. A preprocessor then
finds the reason in `event.filter_error`. When false, such a message is skipped.
auto_offset_reset:
type: string
enum:
@@ -93,6 +93,11 @@ properties:
description: Logic to apply when evaluating the top-level filters. 'and' requires
all of them to match, 'or' requires any of them to match. Nested `any_of`/`all_of`/`none_of`
groups carry their own logic.
run_on_filter_error:
type: boolean
description: Whether a message the filters cannot read (not UTF-8, or not JSON)
still runs the target, as if the trigger had no filters. A preprocessor then
finds the reason in `event.filter_error`. When false, such a message is skipped.
initial_messages:
type: array
items: