Files
windmill/backend/windmill-trigger-websocket/src/handler.rs
T
a4ae82fd69 feat: optionally run trigger messages the filters cannot read (#12226)
* fix: route trigger messages the filters cannot parse to the error handler

A Kafka or websocket trigger with filters silently skipped a message that
was not UTF-8 or not JSON, and Kafka committed its offset, so the message
was lost without a trace. Such a message now goes to the trigger's error
handler (script targets) or the workspace error handler, with the raw
payload and its topic/partition/offset.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: take the unfilterable-message handler as a RunnableRef after merging main

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: keep none_of semantics for non-object JSON, honour workspace handler mutes, throttle the no-handler alert

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* docs: state the caller contract of push_error_handler_with_parent

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* chore: point ee-repo-ref at the kafka companion branch

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: hold unparseable-message handlers of a suspended trigger, throttle them on websocket

A suspended trigger now holds the handler run like its other runs, tagged
with the trigger so resuming releases it. Websocket text keepalives would
start a handler run per frame, so websocket caps it at one per trigger per
minute; Kafka keeps one per message. A parentless handler run carries no
token prefix.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: release a held error handler run in place when resuming an edited trigger

Resuming re-pushes runs held before the trigger's last edit against its
target. A held error handler run would then run the target with the
handler's args, so it is released in place.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* feat: add run_on_filter_error to kafka and websocket triggers

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* fix: treat a truncated array as unreadable, not as a filter mismatch

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* chore: update ee-repo-ref to 695eddf214f6a0e6cf1249ee2c20e716788f0766

This commit updates the EE repository reference after PR #954 was merged in windmill-ee-private.

Previous ee-repo-ref: c8a26463f72ed76423b14e6a1aaf6376e58f2c4c

New ee-repo-ref: 695eddf214f6a0e6cf1249ee2c20e716788f0766

Automated by sync-ee-ref workflow.

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com>
Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
2026-11-05 23:33:47 +01:00

344 lines
11 KiB
Rust

use std::borrow::Cow;
use async_trait::async_trait;
use itertools::Itertools;
use serde_json::value::RawValue;
use sqlx::{types::Json as SqlxJson, PgConnection};
use windmill_api_auth::{check_scopes, ApiAuthed};
use windmill_common::DB;
use windmill_common::{
db::UserDB,
error::{Error, Result},
runnable_ref::RunnableRef,
worker::to_raw_value,
};
use windmill_git_sync::DeployedObject;
use windmill_trigger::{filter::CompiledFilters, Trigger, TriggerCrud, TriggerData};
use super::{
get_url_from_runnable_value, listener::InitialMessage, proxy::connect_async_with_proxy,
validate_websocket_url_for_ssrf, TestWebsocketConfig, WebsocketConfig, WebsocketConfigRequest,
WebsocketTrigger,
};
/// A websocket_triggers:write token can configure secondary runnables that the
/// listener later executes under the trigger owner's identity: a `$flow:`/
/// `$script:` URL resolver and `initial_messages` of kind `runnable_result`.
/// That execution happens in a background task where the reconstructed authed is
/// scopeless (so its check_scopes is a no-op), so enforce run scope here, at
/// create/update time, against the API caller's token. Reading the paths here also
/// refuses, for any caller, one that does not parse.
fn check_secondary_runnable_scopes(
authed: &ApiAuthed,
config: &WebsocketConfigRequest,
) -> Result<()> {
let check = |runnable: RunnableRef| {
let kind = if runnable.is_flow() {
"flows"
} else {
"scripts"
};
check_scopes(authed, || format!("jobs:run:{kind}:{}", runnable.path()))
};
if let Some(runnable) = RunnableRef::from_ws_url(&config.url) {
check(runnable?)?;
}
if let Some(messages) = config.initial_messages.as_ref() {
for msg in messages {
if let Ok(InitialMessage::RunnableResult { path, is_flow, .. }) =
serde_json::from_value::<InitialMessage>(msg.clone())
{
check(RunnableRef::from_path(&path, is_flow)?)?;
}
}
}
Ok(())
}
#[async_trait]
impl TriggerCrud for WebsocketTrigger {
type TriggerConfig = WebsocketConfig;
type Trigger = Trigger<Self::TriggerConfig>;
type TriggerConfigRequest = WebsocketConfigRequest;
type TestConnectionConfig = TestWebsocketConfig;
const TABLE_NAME: &'static str = "websocket_trigger";
const TRIGGER_TYPE: &'static str = "websocket";
const DRAFT_KIND: windmill_common::user_drafts::UserDraftItemKind =
windmill_common::user_drafts::UserDraftItemKind::TriggerWebsocket;
const SUPPORTS_SERVER_STATE: bool = true;
const SUPPORTS_TEST_CONNECTION: bool = true;
const ROUTE_PREFIX: &'static str = "/websocket_triggers";
const DEPLOYMENT_NAME: &'static str = "WebSocket trigger";
const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[
"url",
"filters",
"filter_logic",
"run_on_filter_error",
"initial_messages",
"url_runnable_args",
"can_return_message",
"can_return_error_result",
"heartbeat",
];
const IS_ALLOWED_ON_CLOUD: bool = false;
fn get_deployed_object(path: String, parent_path: Option<String>) -> DeployedObject {
DeployedObject::WebsocketTrigger { path, parent_path }
}
async fn validate_config(
&self,
_db: &DB,
config: &Self::TriggerConfigRequest,
_workspace_id: &str,
) -> Result<()> {
if config.url.trim().is_empty() {
return Err(Error::BadRequest(
"WebSocket URL cannot be empty".to_string(),
));
}
// Reject SSRF targets at save time for static URLs. A `$flow:`/`$script:`
// URL is only known at runtime, so it is validated at connect time
// instead (in the listener and test handler).
if !config.url.starts_with('$') {
validate_websocket_url_for_ssrf(&config.url).await?;
}
if let Some(args) = &config.url_runnable_args {
if !args.is_object() {
return Err(Error::BadRequest(
"url_runnable_args must be an object".to_string(),
));
}
}
CompiledFilters::validate(&config.filters)?;
if let Some(ref hb) = config.heartbeat {
if hb.interval_secs < 1 {
return Err(Error::BadRequest(
"heartbeat interval_secs must be at least 1".to_string(),
));
}
if hb.message.is_empty() {
return Err(Error::BadRequest(
"heartbeat message cannot be empty".to_string(),
));
}
}
Ok(())
}
async fn create_trigger(
&self,
_db: &DB,
tx: &mut PgConnection,
authed: &ApiAuthed,
w_id: &str,
trigger: TriggerData<Self::TriggerConfigRequest>,
) -> Result<()> {
check_secondary_runnable_scopes(authed, &trigger.config)?;
let resolved_edited_by = trigger.base.resolve_edited_by(authed);
let resolved_permissioned_as = trigger.base.resolve_permissioned_as(authed);
let filters = trigger
.config
.filters
.into_iter()
.map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap()))
.collect_vec();
let initial_messages = trigger
.config
.initial_messages
.unwrap_or_default()
.into_iter()
.map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap()))
.collect_vec();
sqlx::query!(
r#"
INSERT INTO websocket_trigger (
workspace_id,
path,
url,
script_path,
is_flow,
mode,
filters,
filter_logic,
initial_messages,
url_runnable_args,
edited_by,
can_return_message,
can_return_error_result,
permissioned_as,
edited_at,
error_handler_path,
error_handler_args,
retry,
heartbeat,
run_on_filter_error
) VALUES (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, now(), $15, $16, $17, $18, $19
)
"#,
w_id,
trigger.base.path,
trigger.config.url,
trigger.base.script_path,
trigger.base.is_flow,
trigger.base.mode() as _,
&filters as _,
trigger.config.filter_logic,
&initial_messages as _,
trigger
.config
.url_runnable_args
.map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap())) as _,
&resolved_edited_by,
trigger.config.can_return_message,
trigger.config.can_return_error_result,
resolved_permissioned_as,
trigger.error_handling.error_handler_path,
trigger.error_handling.error_handler_args as _,
trigger.error_handling.retry as _,
trigger.config.heartbeat.map(SqlxJson) as _,
trigger.config.run_on_filter_error
)
.execute(&mut *tx)
.await?;
Ok(())
}
async fn update_trigger(
&self,
_db: &DB,
tx: &mut PgConnection,
authed: &ApiAuthed,
w_id: &str,
path: &str,
trigger: TriggerData<Self::TriggerConfigRequest>,
) -> Result<()> {
check_secondary_runnable_scopes(authed, &trigger.config)?;
let resolved_edited_by = trigger.base.resolve_edited_by(authed);
let resolved_permissioned_as = trigger.base.resolve_permissioned_as(authed);
let filters = trigger
.config
.filters
.into_iter()
.map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap()))
.collect_vec();
let initial_messages = trigger
.config
.initial_messages
.unwrap_or_default()
.into_iter()
.map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap()))
.collect_vec();
// important to update server_id to NULL to stop current websocket listener
sqlx::query!(
"
UPDATE
websocket_trigger
SET
url = $1,
script_path = $2,
path = $3,
is_flow = $4,
filters = $5,
filter_logic = $6,
initial_messages = $7,
url_runnable_args = $8,
edited_by = $9,
permissioned_as = $10,
can_return_message = $11,
can_return_error_result = $12,
edited_at = now(),
server_id = NULL,
error = NULL,
error_handler_path = $15,
error_handler_args = $16,
retry = $17,
heartbeat = $18,
run_on_filter_error = $19
WHERE
workspace_id = $13 AND path = $14
",
trigger.config.url,
trigger.base.script_path,
trigger.base.path,
trigger.base.is_flow,
filters.as_slice() as &[SqlxJson<Box<RawValue>>],
trigger.config.filter_logic,
initial_messages.as_slice() as &[SqlxJson<Box<RawValue>>],
trigger
.config
.url_runnable_args
.map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap()))
as Option<SqlxJson<Box<RawValue>>>,
&resolved_edited_by,
resolved_permissioned_as,
trigger.config.can_return_message,
trigger.config.can_return_error_result,
w_id,
path,
trigger.error_handling.error_handler_path,
trigger.error_handling.error_handler_args as _,
trigger.error_handling.retry as _,
trigger.config.heartbeat.map(SqlxJson) as _,
trigger.config.run_on_filter_error
)
.execute(&mut *tx)
.await?;
Ok(())
}
async fn test_connection(
&self,
db: &DB,
authed: &ApiAuthed,
_user_db: &UserDB,
workspace_id: &str,
config: Self::TestConnectionConfig,
) -> Result<()> {
let url = &config.url;
let connect_url: Cow<str> = if url.starts_with("$") {
let Some(runnable) = RunnableRef::from_ws_url(url) else {
return Err(Error::BadConfig(format!(
"Invalid WebSocket runnable path: {}",
url
)));
};
Cow::Owned(
get_url_from_runnable_value(
&runnable?,
&db,
authed.clone(),
config.url_runnable_args.as_ref().map(to_raw_value).as_ref(),
&workspace_id,
)
.await?,
)
} else {
Cow::Borrowed(&url)
};
let validated = validate_websocket_url_for_ssrf(&connect_url).await?;
connect_async_with_proxy(&*connect_url, validated.pinned_addrs())
.await
.map_err(|err| {
Error::BadConfig(format!(
"Error connecting to WebSocket: {}",
err.to_string()
))
})?;
Ok(())
}
}