diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 826508bf7d..c1525fa250 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -e44abce4b4c37f7cafc2840184274b14574103f1 +0c8882d93f3d49cb144e539378d0bc167f2b89c5 diff --git a/backend/tests/untrusted_interpolation.rs b/backend/tests/untrusted_interpolation.rs index 1994013c11..12df3ab790 100644 --- a/backend/tests/untrusted_interpolation.rs +++ b/backend/tests/untrusted_interpolation.rs @@ -149,7 +149,7 @@ async fn error_handler_resolves_nothing_the_failed_job_wrote( Some("u/test-user/failing".to_string()), false, "test-workspace", - "script/f/handlers/h", + &windmill_common::runnable_ref::RunnableRef::from_prefixed("script/f/handlers/h")?, Json(&failed_result()), None, None, @@ -185,7 +185,7 @@ async fn success_handler_resolves_nothing_the_job_returned( Some("u/test-user/succeeding".to_string()), false, "test-workspace", - "script/f/handlers/h", + &windmill_common::runnable_ref::RunnableRef::from_prefixed("script/f/handlers/h")?, Json(&result), None, Some(extra_args()), diff --git a/backend/tests/variable_expiration.rs b/backend/tests/variable_expiration.rs index 94a4ed3f78..fede973f7d 100644 --- a/backend/tests/variable_expiration.rs +++ b/backend/tests/variable_expiration.rs @@ -457,8 +457,8 @@ async fn test_removing_a_failing_handler_resolves_its_alert( Ok(()) } -/// The handler endpoint: admin-only, existence-checked, and clearing by omission, which -/// `wmill sync push` relies on to propagate an absent handler. +/// The handler endpoint: admin-only, existence-checked, stored in its clean form, and clearing +/// by omission, which `wmill sync push` relies on to propagate an absent handler. #[sqlx::test(fixtures("base"))] async fn test_edit_handler_endpoint(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; @@ -487,7 +487,7 @@ async fn test_edit_handler_endpoint(db: Pool) -> anyhow::Result<()> { }; seed_handler_script(&db, "u/test-user/expiry_handler").await?; - let handler = serde_json::json!({ "path": "script/u/test-user/expiry_handler" }); + let handler = serde_json::json!({ "path": "script//u/test-user/expiry_handler" }); assert_eq!(edit("SECRET_TOKEN_2", handler.clone()).await?, 403); assert!(stored().await?.is_none()); diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index 4cb0c18895..7d764c0251 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -12,6 +12,7 @@ use windmill_api_auth::{ }; use windmill_api_users::users::WorkspaceInvite; use windmill_common::email_oss::send_email_if_possible; +use windmill_common::runnable_ref::RunnableRef; use windmill_common::ssrf::{validate_webhook_url, webhook_ssrf_error_message}; use windmill_common::usernames::{get_instance_username_or_create_pending, validate_username}; use windmill_common::webhook::WebhookShared; @@ -1722,8 +1723,10 @@ async fn run_slack_message_test_job( Json(req): Json, ) -> JsonResult { // Runs as the error handler identity, handed the Slack bot token. - if !(windmill_queue::is_preset_handler_path(&req.hub_script_path) - && req.hub_script_path.ends_with("-slack")) + let handler = RunnableRef::from_path(&req.hub_script_path, false)?; + if !handler + .preset_handler() + .is_some_and(|p| p.ends_with("-slack")) { return Err(Error::BadRequest( "Only a preset Slack handler from the hub can be tested".to_string(), @@ -1747,7 +1750,7 @@ async fn run_slack_message_test_job( Some("slack_message_test".to_string()), false, w_id.as_str(), - &format!("script/{}", req.hub_script_path.as_str()), + &handler, sqlx::types::Json(&fake_result), None, Some(Utc::now()), @@ -6388,30 +6391,28 @@ async fn edit_variable_expiration_handler( )); } - // Any hub script is left unchecked, so saving never needs the hub. - let exists = if let Some(p) = path.strip_prefix("script/") { - p.starts_with("hub/") + // Any hub script is left unchecked, so saving never needs the hub. A hub flow is not + // exempt: it is looked up in the workspace like any flow, and so refused. + let handler = RunnableRef::from_prefixed(path)?; + let exists = if !handler.is_flow() { + handler.path().starts_with("hub/") || sqlx::query_scalar!( "SELECT EXISTS(SELECT 1 FROM script WHERE workspace_id = $1 AND path = $2 AND archived = false AND deleted = false)", w_id, - p + handler.path() ) .fetch_one(&mut *tx) .await? .unwrap_or(false) - } else if let Some(p) = path.strip_prefix("flow/") { + } else { sqlx::query_scalar!( "SELECT EXISTS(SELECT 1 FROM flow WHERE workspace_id = $1 AND path = $2 AND archived = false)", w_id, - p + handler.path() ) .fetch_one(&mut *tx) .await? .unwrap_or(false) - } else { - return Err(Error::BadRequest(format!( - "Field `path` must start with script/ or flow/ (got {path})" - ))); }; if !exists { return Err(Error::BadRequest(format!( @@ -6421,7 +6422,7 @@ async fn edit_variable_expiration_handler( // Persisted even when false, like `error_handler`'s. let mut variable_expiration_handler = serde_json::json!({ - "path": path, + "path": handler.prefixed(), "muted_on_user_path": es.muted_on_user_path, }); if let Some(extra_args) = &es.extra_args { diff --git a/backend/windmill-common/src/runnable_ref.rs b/backend/windmill-common/src/runnable_ref.rs index 20dd9d1464..8fc9cb4042 100644 --- a/backend/windmill-common/src/runnable_ref.rs +++ b/backend/windmill-common/src/runnable_ref.rs @@ -1,7 +1,7 @@ -//! The one parser for a script or flow named by path. A check and the resolution after it read -//! this value, so they cannot disagree on what a string means. Save-time guards that only refuse -//! a spelling may read the raw value: they can refuse more, never run something the parse did not -//! name. +//! The one parser for a script or flow named by path. A check, the resolution after it and preset +//! handler identity read this value, so they cannot disagree on what a string means. Save-time +//! guards that only refuse a spelling may read the raw value: they can refuse more, never run +//! something the parse did not name. use crate::error::{Error, Result}; use crate::utils::{parse_hub_id, RunnableKind}; @@ -13,6 +13,28 @@ pub struct RunnableRef { } impl RunnableRef { + /// `script/` or `flow/`: schedule and workspace handlers. + pub fn from_prefixed(s: &str) -> Result { + if let Some(rest) = s.strip_prefix("script/") { + Self::from_path(rest, false) + } else if let Some(rest) = s.strip_prefix("flow/") { + Self::from_path(rest, true) + } else { + Err(Error::BadRequest(format!( + "path must start with script/ or flow/ (got {s})" + ))) + } + } + + /// A setting that takes the prefixed form or a bare script path: the global error handler. + pub fn from_setting(s: &str) -> Result { + if s.starts_with("script/") || s.starts_with("flow/") { + Self::from_prefixed(s) + } else { + Self::from_path(s, false) + } + } + /// A bare path with a separate flag, as the run routes take it. /// One leading `/` is tolerated, as the run routes have always stripped it. A second one is /// refused: a stored path normalized once and stripped again when read must not change meaning. @@ -47,8 +69,53 @@ impl RunnableRef { pub fn path(&self) -> &str { &self.path } + + /// The canonical `script/` or `flow/` form. + pub fn prefixed(&self) -> String { + format!("{}/{}", self.kind, self.path) + } + + /// The preset handler Windmill ships on the hub that this names, `None` for user code. + pub fn preset_handler(&self) -> Option<&'static str> { + if self.kind != RunnableKind::Script { + return None; + } + PRESET_HANDLER_PATHS.into_iter().find(|p| *p == self.path) + } } +/// Every handler version the frontend has offered as a preset (`frontend/src/lib/hubPaths.json` +/// and its history, which stored schedules still point at). Matched whole: a hub script is +/// resolved by its numeric id alone, so the name after it proves nothing about its code. +const PRESET_HANDLER_PATHS: [&str; 26] = [ + "hub/6512/workspace-or-schedule-error-handler-slack", + "hub/5792/workspace-or-schedule-error-handler-slack", + "hub/9079/workspace-or-schedule-error-handler-slack", + "hub/9206/workspace-or-schedule-error-handler-slack", + "hub/19741/workspace-or-schedule-error-handler-slack", + "hub/28241/workspace-or-schedule-error-handler-slack", + "hub/28794/workspace-or-schedule-error-handler-slack", + "hub/30057/workspace-or-schedule-error-handler-slack", + "hub/2431/slack/schedule-error-handler-slack", + "hub/2430/slack/schedule-recovery-handler-slack", + "hub/9067/slack/schedule-recovery-handler-slack", + "hub/9080/slack/schedule-recovery-handler-slack", + "hub/28239/slack/schedule-recovery-handler-slack", + "hub/28791/slack/schedule-recovery-handler-slack", + "hub/30054/slack/schedule-recovery-handler-slack", + "hub/9069/slack/schedule-success-handler-slack", + "hub/9072/slack/schedule-success-handler-slack", + "hub/28220/slack/schedule-success-handler-slack", + "hub/28240/slack/schedule-success-handler-slack", + "hub/28793/slack/schedule-success-handler-slack", + "hub/30056/slack/schedule-success-handler-slack", + "hub/11598/workspace-or-schedule-error-handler-teams", + "hub/19742/workspace-or-schedule-error-handler-teams", + "hub/11593/schedule-recovery-handler-teams", + "hub/11596/schedule-success-handler-teams", + "hub/19795/workspace-or-error-handler-email", +]; + #[cfg(test)] mod tests { use super::*; @@ -93,5 +160,50 @@ mod tests { ] { assert!(got.is_err(), "{name}: {got:?}"); } + + let p = |s: &str| RunnableRef::from_prefixed(s).map(|r| (r.is_flow(), r.prefixed())); + assert_eq!( + p("script//u/me/x").unwrap(), + (false, "script/u/me/x".into()) + ); + assert_eq!(p("flow/f/a/b").unwrap(), (true, "flow/f/a/b".into())); + assert_eq!( + p("script/script/hub/1/x").unwrap(), + (false, "script/script/hub/1/x".into()), + "exactly one prefix is stripped" + ); + assert_eq!( + RunnableRef::from_setting("u/me/x").unwrap().prefixed(), + "script/u/me/x" + ); + for bad in ["script///u/me/x", "resource/u/me/x", "u/me/x"] { + assert!(p(bad).is_err(), "{bad}"); + } + } + + #[test] + fn only_pinned_hub_scripts_are_presets() { + let preset = |s: &str| RunnableRef::from_prefixed(s).unwrap().preset_handler(); + assert!(preset("script/hub/28791/slack/schedule-recovery-handler-slack").is_some()); + assert!(preset("script//hub/6512/workspace-or-schedule-error-handler-slack").is_some()); + // The hub resolves by id: any other id is arbitrary hub code whatever its name. + assert!(preset("script/hub/13968/workspace-or-schedule-error-handler-slack").is_none()); + assert!(preset("script/u/me/workspace-or-schedule-error-handler-slack").is_none()); + assert!(preset("flow/hub/6512/workspace-or-schedule-error-handler-slack").is_none()); + } + + /// A preset version bumped in the frontend without being pinned here would silently run as + /// the schedule's owner and lose access to the Slack bot token. + #[test] + fn frontend_presets_are_pinned() { + let hub_paths: std::collections::HashMap = + serde_json::from_str(include_str!("../../../frontend/src/lib/hubPaths.json")).unwrap(); + for (name, path) in hub_paths { + // Variable expiration handlers run as their own group whatever their path. + if path.contains("-handler-") && !path.contains("variable-expiration") { + let r = RunnableRef::from_path(&path, false).unwrap(); + assert!(r.preset_handler().is_some(), "{name}: {path} is not pinned"); + } + } } } diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 19bb2fae41..fca04090c7 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -77,6 +77,7 @@ use windmill_common::{ }, jobs::{get_payload_tag_from_prefixed_path, JobKind, JobPayload, RawCode}, min_version::{MIN_VERSION_IS_AT_LEAST_1_432, MIN_VERSION_IS_AT_LEAST_1_440}, + runnable_ref::RunnableRef, schedule::Schedule, scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL}, @@ -2779,13 +2780,7 @@ pub async fn send_error_to_global_handler<'a, T: Serialize + Send + Sync>( result: Json<&T>, ) -> Result<(), Error> { if let Some(ref global_error_handler) = *GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE { - let prefixed_global_error_handler_path = if global_error_handler.starts_with("script/") - || global_error_handler.starts_with("flow/") - { - global_error_handler.clone() - } else { - format!("script/{}", global_error_handler) - }; + let global_error_handler = RunnableRef::from_setting(global_error_handler)?; push_error_handler( db, queued_job.id, @@ -2793,7 +2788,7 @@ pub async fn send_error_to_global_handler<'a, T: Serialize + Send + Sync>( queued_job.runnable_path.clone(), queued_job.is_flow(), &queued_job.workspace_id, - &prefixed_global_error_handler_path, + &global_error_handler, result, None, queued_job.started_at, @@ -2836,29 +2831,33 @@ pub async fn report_error_to_workspace_handler_or_critical_side_channel( let (error_handler, error_handler_extra_args) = row_result; if let Some(error_handler) = error_handler { - if let Err(err) = push_error_handler( - db, - queued_job.id, - queued_job.schedule_path(), - queued_job.runnable_path.clone(), - queued_job.is_flow(), - w_id, - &error_handler, - Json(&json!({ - "error": { - "message": error_message - } - })), - None, - queued_job.started_at, - error_handler_extra_args, - &queued_job.permissioned_as_email, - None, - false, - None, - ) - .await - { + let pushed = async { + let error_handler = RunnableRef::from_prefixed(&error_handler)?; + push_error_handler( + db, + queued_job.id, + queued_job.schedule_path(), + queued_job.runnable_path.clone(), + queued_job.is_flow(), + w_id, + &error_handler, + Json(&json!({ + "error": { + "message": error_message + } + })), + None, + queued_job.started_at, + error_handler_extra_args, + &queued_job.permissioned_as_email, + None, + false, + None, + ) + .await + } + .await; + if let Err(err) = pushed { tracing::error!( "Could not push workspace error handler trying to handle critical error ({error_message}) of failed job ({}): {}", queued_job.id, @@ -3022,7 +3021,7 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync> queued_job.runnable_path.clone(), queued_job.is_flow(), &queued_job.workspace_id, - &error_handler, + &RunnableRef::from_prefixed(&error_handler)?, result, None, queued_job.started_at, @@ -3130,7 +3129,7 @@ pub async fn send_success_to_workspace_handler<'a, 'c, T: Serialize + Send + Syn queued_job.runnable_path.clone(), queued_job.is_flow(), &queued_job.workspace_id, - &success_handler, + &RunnableRef::from_prefixed(&success_handler)?, result, queued_job.started_at, success_handler_extra_args, @@ -3392,50 +3391,6 @@ pub async fn try_schedule_next_job<'c>( const ERROR_HANDLER_PATH_EMAIL: &str = "/workspace-or-error-handler-email"; -/// Every handler version the frontend has offered as a preset (`frontend/src/lib/hubPaths.json` -/// and its history, which stored schedules still point at). Matched whole: a hub script is -/// resolved by its numeric id alone, so the name after it proves nothing about its code. -const PRESET_HANDLER_PATHS: [&str; 26] = [ - "hub/6512/workspace-or-schedule-error-handler-slack", - "hub/5792/workspace-or-schedule-error-handler-slack", - "hub/9079/workspace-or-schedule-error-handler-slack", - "hub/9206/workspace-or-schedule-error-handler-slack", - "hub/19741/workspace-or-schedule-error-handler-slack", - "hub/28241/workspace-or-schedule-error-handler-slack", - "hub/28794/workspace-or-schedule-error-handler-slack", - "hub/30057/workspace-or-schedule-error-handler-slack", - "hub/2431/slack/schedule-error-handler-slack", - "hub/2430/slack/schedule-recovery-handler-slack", - "hub/9067/slack/schedule-recovery-handler-slack", - "hub/9080/slack/schedule-recovery-handler-slack", - "hub/28239/slack/schedule-recovery-handler-slack", - "hub/28791/slack/schedule-recovery-handler-slack", - "hub/30054/slack/schedule-recovery-handler-slack", - "hub/9069/slack/schedule-success-handler-slack", - "hub/9072/slack/schedule-success-handler-slack", - "hub/28220/slack/schedule-success-handler-slack", - "hub/28240/slack/schedule-success-handler-slack", - "hub/28793/slack/schedule-success-handler-slack", - "hub/30056/slack/schedule-success-handler-slack", - "hub/11598/workspace-or-schedule-error-handler-teams", - "hub/19742/workspace-or-schedule-error-handler-teams", - "hub/11593/schedule-recovery-handler-teams", - "hub/11596/schedule-success-handler-teams", - "hub/19795/workspace-or-error-handler-email", -]; - -/// The preset handler Windmill ships on the hub that `handler_path` names, `None` for user code. -fn preset_handler(handler_path: &str) -> Option<&'static str> { - let path = handler_path.strip_prefix("script/").unwrap_or(handler_path); - PRESET_HANDLER_PATHS.into_iter().find(|p| *p == path) -} - -/// Whether `handler_path` is a preset hub handler, which runs as the shared handler identity -/// (Slack bot token, instance SMTP) instead of as the schedule or user that set it. -pub fn is_preset_handler_path(handler_path: &str) -> bool { - preset_handler(handler_path).is_some() -} - /// `(email, permissioned_as)` a schedule handler runs as. The handler identity reads the /// workspace Slack bot token and may send through the instance SMTP, and a schedule's handlers /// are set by anyone who can write the schedule: only the preset hub handlers get it, custom @@ -3443,20 +3398,21 @@ pub fn is_preset_handler_path(handler_path: &str) -> bool { pub async fn schedule_handler_identity( db: &Pool, w_id: &str, - handler_path: &str, + handler: &RunnableRef, schedule_permissioned_as: &str, preset_email: &str, ) -> error::Result<(String, String)> { - if is_preset_handler_path(handler_path) { + if handler.preset_handler().is_some() { return Ok(( preset_email.to_string(), ERROR_HANDLER_USER_GROUP.to_string(), )); } - if handler_path.contains("hub/") && handler_path.contains("-handler-") { + if handler.path().starts_with("hub/") && handler.path().contains("-handler-") { tracing::warn!( - "schedule handler {handler_path} in {w_id} is named like a preset but is not a pinned \ - preset version: it runs as the schedule ({schedule_permissioned_as})" + "schedule handler {} in {w_id} is named like a preset but is not a pinned preset \ + version: it runs as the schedule ({schedule_permissioned_as})", + handler.prefixed() ); } let email = @@ -3468,7 +3424,7 @@ pub async fn schedule_handler_identity( async fn error_handler_identity( db: &Pool, w_id: &str, - error_handler_path: &str, + error_handler: &RunnableRef, is_global_error_handler: bool, schedule_permissioned_as: Option<&str>, ) -> error::Result<(String, String)> { @@ -3482,13 +3438,14 @@ async fn error_handler_identity( return schedule_handler_identity( db, w_id, - error_handler_path, + error_handler, schedule_permissioned_as, SCHEDULE_ERROR_HANDLER_USER_EMAIL, ) .await; } - let email = if preset_handler(error_handler_path) + let email = if error_handler + .preset_handler() .is_some_and(|p| p.ends_with(ERROR_HANDLER_PATH_EMAIL)) { EMAIL_ERROR_HANDLER_USER_EMAIL @@ -3505,7 +3462,7 @@ pub async fn push_error_handler<'a, 'c, T: Serialize + Send + Sync>( script_path: Option, is_flow: bool, w_id: &str, - on_failure_path: &str, + on_failure: &RunnableRef, result: Json<&'a T>, failed_times: Option, started_at: Option>, @@ -3522,7 +3479,7 @@ pub async fn push_error_handler<'a, 'c, T: Serialize + Send + Sync>( w_id }; let (payload, tag, on_behalf_of) = - get_payload_tag_from_prefixed_path(on_failure_path, db, handler_w_id).await?; + get_payload_tag_from_prefixed_path(&on_failure.prefixed(), db, handler_w_id).await?; let mut extra = HashMap::new(); if let Some(schedule_path) = schedule_path { @@ -3557,7 +3514,7 @@ pub async fn push_error_handler<'a, 'c, T: Serialize + Send + Sync>( error_handler_identity( db, w_id, - on_failure_path, + on_failure, is_global_error_handler, schedule_permissioned_as, ) @@ -3709,7 +3666,7 @@ pub async fn push_success_handler<'a, 'c, T: Serialize + Send + Sync>( script_path: Option, is_flow: bool, w_id: &str, - on_success_path: &str, + on_success: &RunnableRef, result: Json<&'a T>, started_at: Option>, extra_args: Option>>, @@ -3717,7 +3674,7 @@ pub async fn push_success_handler<'a, 'c, T: Serialize + Send + Sync>( priority: Option, ) -> windmill_common::error::Result { let (payload, tag, on_behalf_of) = - get_payload_tag_from_prefixed_path(on_success_path, db, w_id).await?; + get_payload_tag_from_prefixed_path(&on_success.prefixed(), db, w_id).await?; let mut extra = HashMap::new(); if let Some(schedule_path) = schedule_path { @@ -9140,42 +9097,6 @@ mod worker_own_script_tests { } } -#[cfg(test)] -mod preset_handler_path_tests { - use super::is_preset_handler_path; - - #[test] - fn only_pinned_hub_scripts_are_presets() { - assert!(is_preset_handler_path( - "script/hub/28791/slack/schedule-recovery-handler-slack" - )); - // The hub resolves by id: any other id is arbitrary hub code whatever its name. - assert!(!is_preset_handler_path( - "hub/13968/workspace-or-schedule-error-handler-slack" - )); - assert!(!is_preset_handler_path( - "script/u/me/workspace-or-schedule-error-handler-slack" - )); - } - - /// A preset version bumped in the frontend without being pinned here would silently run as - /// the schedule's owner and lose access to the Slack bot token. - #[test] - fn frontend_presets_are_pinned() { - let hub_paths: std::collections::HashMap = - serde_json::from_str(include_str!("../../../frontend/src/lib/hubPaths.json")).unwrap(); - for (name, path) in hub_paths { - // Variable expiration handlers run as their own group whatever their path. - if path.contains("-handler-") && !path.contains("variable-expiration") { - assert!( - is_preset_handler_path(&path), - "{name}: {path} is not pinned" - ); - } - } - } -} - #[cfg(test)] mod result_metadata_tests { use super::{ResultMetadata, ValidableJson}; diff --git a/backend/windmill-queue/src/variable_expiration.rs b/backend/windmill-queue/src/variable_expiration.rs index 74c0eaf393..cfc141afd0 100644 --- a/backend/windmill-queue/src/variable_expiration.rs +++ b/backend/windmill-queue/src/variable_expiration.rs @@ -13,6 +13,7 @@ use std::collections::{HashMap, HashSet}; use serde_json::value::RawValue; use sqlx::types::Json; use windmill_common::jobs::get_payload_tag_from_prefixed_path; +use windmill_common::runnable_ref::RunnableRef; use windmill_common::utils::{report_critical_error, report_recovered_critical_error}; use windmill_common::DB; @@ -116,19 +117,23 @@ async fn dispatch_expiring_variables_inner(db: &DB) { } // Resolved before the claim's transaction opens, so a pass never holds one connection // while waiting on another. - let handler = match failed_hub_handlers.get(&row.handler_path) { - Some(error) => Err(error.clone()), - None => get_payload_tag_from_prefixed_path(&row.handler_path, db, &row.workspace_id) - .await - .map_err(|e| { - let error = format!("{e:#}"); - if row.handler_path.starts_with("script/hub/") - || row.handler_path.starts_with("flow/hub/") - { - failed_hub_handlers.insert(row.handler_path.clone(), error.clone()); - } - error - }), + let handler = match ( + failed_hub_handlers.get(&row.handler_path), + RunnableRef::from_prefixed(&row.handler_path), + ) { + (Some(error), _) => Err(error.clone()), + (None, Err(e)) => Err(format!("{e:#}")), + (None, Ok(handler)) => { + get_payload_tag_from_prefixed_path(&handler.prefixed(), db, &row.workspace_id) + .await + .map_err(|e| { + let error = format!("{e:#}"); + if handler.path().starts_with("hub/") { + failed_hub_handlers.insert(row.handler_path.clone(), error.clone()); + } + error + }) + } }; let handler = match handler { Ok(handler) => handler,