diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 9fe5297045..7ffc40ef5c 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -171,7 +171,7 @@ pub mod teams_approvals_ee; mod teams_approvals_oss; #[cfg(feature = "native_trigger")] -pub mod native_triggers; +pub use windmill_triggers::native_triggers; mod public_app_layer; mod public_app_rate_limit; mod static_assets; @@ -187,7 +187,7 @@ pub mod teams_ee; mod teams_oss; mod token; mod tracing_init; -pub mod triggers; +pub use windmill_triggers::triggers; mod users; #[cfg(feature = "private")] pub mod users_ee; diff --git a/backend/windmill-api/src/native_triggers/handler.rs b/backend/windmill-api/src/native_triggers/handler.rs deleted file mode 100644 index 3c42b5261b..0000000000 --- a/backend/windmill-api/src/native_triggers/handler.rs +++ /dev/null @@ -1,566 +0,0 @@ -use crate::{ - db::ApiAuthed, - native_triggers::{ - delete_native_trigger, delete_token_by_prefix, get_native_trigger, get_token_by_prefix, - get_workspace_integration, list_native_triggers, store_native_trigger, - update_native_trigger_error, External, NativeTrigger, NativeTriggerConfig, - NativeTriggerData, ServiceName, - }, - users::{create_token_internal, NewToken}, - utils::check_scopes, -}; -use axum::{ - extract::{Path, Query}, - routing::{delete, get, post}, - Extension, Json, Router, -}; -use serde::{Deserialize, Serialize}; -use sqlx::PgConnection; -use std::sync::Arc; -use windmill_audit::{audit_oss::audit_log, ActionKind}; -use windmill_common::{ - db::UserDB, - error::{Error, JsonResult, Result}, - utils::rd_string, - DB, -}; - -async fn require_is_writer_on_runnable( - authed: &ApiAuthed, - path: &str, - is_flow: bool, - w_id: &str, - db: DB, -) -> Result<()> { - if is_flow { - crate::flows::require_is_writer(authed, path, w_id, db).await - } else { - crate::scripts::require_is_writer(authed, path, w_id, db).await - } -} - -#[derive(Debug, Deserialize)] -pub struct ListQuery { - pub page: Option, - pub per_page: Option, - pub path: Option, - pub is_flow: Option, -} - -#[derive(Debug, Serialize)] -pub struct FullTriggerResponse { - #[serde(flatten)] - pub windmill_data: NativeTrigger, - pub external_data: T, -} - -#[derive(Debug, Serialize)] -pub struct CreateTriggerResponse { - pub external_id: String, -} - -async fn new_webhook_token( - tx: &mut PgConnection, - db: &DB, - authed: &ApiAuthed, - script_path: &str, - is_flow: bool, - workspace_id: &str, - service_name: ServiceName, -) -> Result { - let kind = if is_flow { "flows" } else { "scripts" }; - - let scopes = vec![format!("jobs:run:{kind}:{script_path}")]; - let label = format!("webhook-{}-{}", service_name.as_str(), rd_string(5)); - let token_config = NewToken::new( - Some(label), - None, - None, - Some(scopes), - Some(workspace_id.to_owned()), - ); - let token = create_token_internal(&mut *tx, &db, &authed, token_config).await?; - - Ok(token) -} - -async fn create_native_trigger( - Extension(handler): Extension>, - Extension(service_name): Extension, - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path(workspace_id): Path, - Json(data): Json>, -) -> JsonResult { - check_scopes(&authed, || { - format!("native_triggers:write:{}", &data.script_path) - })?; - require_is_writer_on_runnable( - &authed, - &data.script_path, - data.is_flow, - &workspace_id, - db.clone(), - ) - .await?; - - let mut tx = user_db.begin(&authed).await?; - - let webhook_token = new_webhook_token( - &mut *tx, - &db, - &authed, - &data.script_path, - data.is_flow, - &workspace_id, - service_name, - ) - .await?; - - let integration = get_workspace_integration(&mut *tx, &workspace_id, service_name).await?; - - let oauth_data: T::OAuthData = serde_json::from_value(integration.oauth_data).map_err(|e| { - Error::InternalErr(format!( - "Failed to parse {} OAuth data: {}", - T::DISPLAY_NAME, - e - )) - })?; - - let resp = handler - .create( - &workspace_id, - &oauth_data, - &webhook_token, - &data, - &db, - &mut tx, - ) - .await?; - - let (external_id, _) = handler.external_id_and_metadata_from_response(&resp); - - // update the created external trigger with a new uri containing the external_id - handler - .update( - &workspace_id, - &oauth_data, - &external_id, - &webhook_token, - &data, - &db, - &mut tx, - ) - .await?; - - // Fetch the updated trigger data from the external service and extract service_config - let trigger_data = handler - .get(&workspace_id, &oauth_data, &external_id, &db, &mut tx) - .await?; - let service_config = handler.extract_service_config_from_trigger_data(&trigger_data)?; - - let config = NativeTriggerConfig { - script_path: data.script_path.clone(), - is_flow: data.is_flow, - webhook_token, - }; - - store_native_trigger( - &mut *tx, - &workspace_id, - service_name, - &external_id, - &config, - service_config, - ) - .await?; - - audit_log( - &mut *tx, - &authed, - &format!("native_triggers.{}.create", service_name), - ActionKind::Create, - &workspace_id, - Some(&external_id), - None, - ) - .await?; - - tx.commit().await?; - - Ok(Json(CreateTriggerResponse { external_id })) -} - -async fn update_native_trigger_handler( - Extension(handler): Extension>, - Extension(service_name): Extension, - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((workspace_id, external_id)): Path<(String, String)>, - Json(data): Json>, -) -> Result { - check_scopes(&authed, || { - format!("native_triggers:write:{}", &data.script_path) - })?; - require_is_writer_on_runnable( - &authed, - &data.script_path, - data.is_flow, - &workspace_id, - db.clone(), - ) - .await?; - - let mut tx = user_db.begin(&authed).await?; - - let existing = get_native_trigger(&mut *tx, &workspace_id, service_name, &external_id) - .await? - .ok_or_else(|| Error::NotFound(format!("Native trigger not found: {}", external_id)))?; - - // Look up the full token using the stored prefix (use db, not tx, for token table) - let webhook_token = match get_token_by_prefix(&db, &existing.webhook_token_prefix).await? { - Some(token) => token, - None => { - tracing::warn!( - "Webhook token not found for trigger {} (prefix: {}), recreating token", - external_id, - existing.webhook_token_prefix - ); - new_webhook_token( - &mut *tx, - &db, - &authed, - &data.script_path, - data.is_flow, - &workspace_id, - service_name, - ) - .await? - } - }; - - let integration = get_workspace_integration(&mut *tx, &workspace_id, service_name).await?; - - let oauth_data: T::OAuthData = serde_json::from_value(integration.oauth_data).map_err(|e| { - Error::InternalErr(format!( - "Failed to parse {} OAuth data: {}", - T::DISPLAY_NAME, - e - )) - })?; - - handler - .update( - &workspace_id, - &oauth_data, - &external_id, - &webhook_token, - &data, - &db, - &mut tx, - ) - .await?; - - // Fetch the updated trigger data from the external service and extract service_config - let trigger_data = handler - .get(&workspace_id, &oauth_data, &external_id, &db, &mut tx) - .await?; - let service_config = handler.extract_service_config_from_trigger_data(&trigger_data)?; - - let config = NativeTriggerConfig { - script_path: data.script_path.clone(), - is_flow: data.is_flow, - webhook_token, - }; - - store_native_trigger( - &mut *tx, - &workspace_id, - service_name, - &external_id, - &config, - service_config, - ) - .await?; - - audit_log( - &mut *tx, - &authed, - &format!("native_triggers.{}.update", service_name), - ActionKind::Update, - &workspace_id, - Some(&external_id), - None, - ) - .await?; - - tx.commit().await?; - - Ok(format!("Native trigger updated")) -} - -async fn get_native_trigger_handler( - Extension(handler): Extension>, - Extension(service_name): Extension, - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((workspace_id, external_id)): Path<(String, String)>, -) -> JsonResult> { - let mut tx = user_db.begin(&authed).await?; - - let windmill_trigger = get_native_trigger(&mut *tx, &workspace_id, service_name, &external_id) - .await? - .ok_or_else(|| Error::NotFound(format!("Native trigger not found: {}", external_id)))?; - - check_scopes(&authed, || { - format!("native_triggers:read:{}", &windmill_trigger.script_path) - })?; - require_is_writer_on_runnable( - &authed, - &windmill_trigger.script_path, - windmill_trigger.is_flow, - &workspace_id, - db.clone(), - ) - .await?; - - let integration = get_workspace_integration(&mut *tx, &workspace_id, service_name).await?; - - let oauth_data: T::OAuthData = serde_json::from_value(integration.oauth_data).map_err(|e| { - Error::InternalErr(format!( - "Failed to parse {} OAuth data: {}", - T::DISPLAY_NAME, - e - )) - })?; - - let native_trigger = handler - .get(&workspace_id, &oauth_data, &external_id, &db, &mut tx) - .await; - - let native_trigger_config = match native_trigger { - Ok(native_cfg) => { - // Clear error if it was set - if windmill_trigger.error.is_some() { - update_native_trigger_error( - &mut *tx, - &workspace_id, - service_name, - &external_id, - None, - ) - .await?; - } - native_cfg - } - Err(Error::NotFound(_)) => { - let error_msg = "Trigger no longer exists on external service".to_string(); - tracing::warn!( - "Native trigger no longer exists on external service {}, setting error", - service_name - ); - - update_native_trigger_error( - &mut *tx, - &workspace_id, - service_name, - &external_id, - Some(&error_msg), - ) - .await?; - - tx.commit().await?; - - return Err(Error::NotFound(format!( - "Trigger '{}' no longer exists on external service {}", - external_id, service_name - ))); - } - Err(e) => return Err(e), - }; - - let full_resp = Json(FullTriggerResponse { - windmill_data: windmill_trigger, - external_data: native_trigger_config, - }); - - Ok(full_resp) -} - -async fn delete_native_trigger_handler( - Extension(handler): Extension>, - Extension(service_name): Extension, - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((workspace_id, external_id)): Path<(String, String)>, -) -> Result { - let mut tx = user_db.begin(&authed).await?; - - let existing = get_native_trigger(&mut *tx, &workspace_id, service_name, &external_id) - .await? - .ok_or_else(|| Error::NotFound(format!("Native trigger not found: {}", external_id)))?; - - check_scopes(&authed, || { - format!("native_triggers:write:{}", &existing.script_path) - })?; - require_is_writer_on_runnable( - &authed, - &existing.script_path, - existing.is_flow, - &workspace_id, - db.clone(), - ) - .await?; - - let integration = get_workspace_integration(&mut *tx, &workspace_id, service_name).await?; - - let oauth_data: T::OAuthData = serde_json::from_value(integration.oauth_data).map_err(|e| { - Error::InternalErr(format!( - "Failed to parse {} OAuth data: {}", - T::DISPLAY_NAME, - e - )) - })?; - - handler - .delete(&workspace_id, &oauth_data, &external_id, &db, &mut tx) - .await?; - - let deleted = - delete_native_trigger(&mut *tx, &workspace_id, service_name, &external_id).await?; - - if !deleted { - return Err(Error::NotFound(format!("Native trigger not found"))); - } - - // Delete the webhook token using its prefix - if !delete_token_by_prefix(&db, &existing.webhook_token_prefix).await? { - tracing::warn!( - "Webhook token not found when deleting trigger {} (prefix: {})", - external_id, - existing.webhook_token_prefix - ); - } - - audit_log( - &mut *tx, - &authed, - &format!("native_triggers.{}.delete", service_name), - ActionKind::Delete, - &workspace_id, - Some(&external_id), - None, - ) - .await?; - - tx.commit().await?; - - Ok(format!("Native trigger deleted")) -} - -async fn exists_native_trigger_handler( - Extension(service_name): Extension, - _authed: ApiAuthed, - Extension(db): Extension, - Path((workspace_id, external_id)): Path<(String, String)>, -) -> JsonResult { - let exists = sqlx::query_scalar!( - r#" - SELECT EXISTS( - SELECT 1 - FROM native_trigger - WHERE - workspace_id = $1 AND - service_name = $2 AND - external_id = $3 - ) - "#, - workspace_id, - service_name as ServiceName, - external_id - ) - .fetch_one(&db) - .await? - .unwrap_or(false); - - Ok(Json(exists)) -} - -async fn list_native_triggers_handler( - Extension(service_name): Extension, - authed: ApiAuthed, - Extension(user_db): Extension, - Path(workspace_id): Path, - Query(query): Query, -) -> JsonResult> { - let mut tx = user_db.begin(&authed).await?; - let triggers = list_native_triggers( - &mut *tx, - &workspace_id, - service_name, - query.page, - query.per_page, - query.path.as_deref(), - query.is_flow, - ) - .await?; - tx.commit().await?; - Ok(Json(triggers)) -} - -pub fn service_routes(handler: T) -> Router { - let additional_routes = handler.additional_routes(); - let service_name = T::SERVICE_NAME; - - let handler_arc = Arc::new(handler); - - let standard_routes = Router::new() - .route("/create", post(create_native_trigger::)) - .route("/list", get(list_native_triggers_handler::)) - .route("/get/:external_id", get(get_native_trigger_handler::)) - .route( - "/update/:external_id", - post(update_native_trigger_handler::), - ) - .route( - "/delete/:external_id", - delete(delete_native_trigger_handler::), - ) - .route( - "/exists/:external_id", - get(exists_native_trigger_handler::), - ); - - standard_routes - .merge(additional_routes) - .layer(Extension(handler_arc)) - .layer(Extension(service_name)) -} - -/// Generates routes for all registered native trigger services. -/// When adding a new service, add a new `.nest()` call here. -pub fn generate_native_trigger_routers() -> Router { - let router = Router::new(); - - #[cfg(feature = "native_trigger")] - { - use crate::native_triggers::nextcloud::NextCloud; - - // Register all service routes here - // When adding a new service: - // 1. Import the handler: use crate::native_triggers::newservice::NewServiceHandler; - // 2. Add the route: .nest("/newservice", service_routes(NewServiceHandler)) - return router.nest("/nextcloud", service_routes(NextCloud)); - // Add new services here: - // .nest("/newservice", service_routes(NewServiceHandler)) - } - - #[cfg(not(feature = "native_trigger"))] - { - router - } -} diff --git a/backend/windmill-api/src/native_triggers/mod.rs b/backend/windmill-api/src/native_triggers/mod.rs deleted file mode 100644 index 8c0b829478..0000000000 --- a/backend/windmill-api/src/native_triggers/mod.rs +++ /dev/null @@ -1,1053 +0,0 @@ -//! Native Triggers Module -//! -//! This module provides integration with external services (like Nextcloud) that can -//! trigger Windmill scripts/flows via webhooks. -//! -//! ## Adding a New Native Trigger Service -//! -//! When adding a new service (e.g., "NewService"), you need to update the following locations: -//! -//! ### 1. This file (mod.rs): -//! - Add `pub mod newservice;` under the `#[cfg(feature = "native_trigger")]` block -//! - Add `NewService` variant to `ServiceName` enum -//! - Update `ServiceName::as_str()` - add match arm returning `"newservice"` -//! - Update `TryFrom for ServiceName` - add match arm for `"newservice"` -//! - Update `ServiceName::as_trigger_kind()` - add match arm (requires TriggerKind::NewService in windmill_common) -//! - Update `ServiceName::as_job_trigger_kind()` - add match arm (requires JobTriggerKind::NewService in windmill_common) -//! - Update `ServiceName::fmt()` (Display impl) - add match arm -//! -//! ### 2. sync.rs: -//! - Add `sync_service!()` macro call in `sync_all_triggers()` -//! -//! ### 3. handler.rs: -//! - Add `.nest("/newservice", service_routes(NewServiceHandler))` in `generate_native_trigger_routers()` -//! -//! ### 4. Database migration: -//! - Add `'newservice'` to the `native_trigger_service` enum type -//! -//! ### 5. windmill_common (if needed): -//! - Add `NewService` variant to `TriggerKind` enum -//! - Add `'newservice'` to `job_trigger_kind` enum type in migration -//! -//! The generic code (trait definitions, route handlers, database operations) does NOT -//! need modification when adding new services. - -use async_trait::async_trait; -use chrono::{DateTime, Utc}; -use http::StatusCode; -use itertools::Itertools; -use reqwest::{Client, Method}; -use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use serde_json::json; -use serde_json::value::RawValue; -use sqlx::{FromRow, PgConnection, Postgres}; -use std::{collections::HashMap, fmt::Debug}; -use strum::{EnumIter, IntoEnumIterator}; -use tokio::task; -use windmill_common::{ - error::{to_anyhow, Error, Result}, - triggers::TriggerKind, - variables::{build_crypt, decrypt, encrypt}, - DB, -}; -use windmill_queue::PushArgsOwned; - -#[cfg(feature = "native_trigger")] -use windmill_oauth::{OClient, RefreshToken, Url, OAUTH_HTTP_CLIENT}; - -use crate::db::ApiAuthed; -pub mod handler; -pub mod sync; -pub mod workspace_integrations; - -// Service modules - add new services here: -#[cfg(feature = "native_trigger")] -pub mod nextcloud; -// #[cfg(feature = "native_trigger")] -// pub mod newservice; - -/// Enum of all supported native trigger services. -/// When adding a new service, add a variant here (e.g., `NewService`). -#[derive(EnumIter, sqlx::Type, Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[sqlx(type_name = "native_trigger_service", rename_all = "lowercase")] -#[serde(rename_all = "lowercase")] -pub enum ServiceName { - Nextcloud, - // Add new services here: - // NewService, -} - -impl TryFrom for ServiceName { - type Error = Error; - fn try_from(value: String) -> std::result::Result { - // Add new service match arms here: - let service = match value.as_str() { - "nextcloud" => ServiceName::Nextcloud, - // "newservice" => ServiceName::NewService, - _ => { - return Err(anyhow::anyhow!( - "Unknown service, currently supported services are: [{}]", - ServiceName::iter().join(",") - ) - .into()) - } - }; - - Ok(service) - } -} - -impl ServiceName { - /// Returns the lowercase string identifier for this service. - /// Add new service match arms here. - pub fn as_str(&self) -> &'static str { - match self { - ServiceName::Nextcloud => "nextcloud", - // ServiceName::NewService => "newservice", - } - } - - /// Returns the corresponding TriggerKind for this service. - /// Requires adding the variant to TriggerKind in windmill_common. - pub fn as_trigger_kind(&self) -> TriggerKind { - match self { - ServiceName::Nextcloud => TriggerKind::Nextcloud, - // ServiceName::NewService => TriggerKind::NewService, - } - } - - /// Returns the corresponding JobTriggerKind for this service. - /// Requires adding the variant to JobTriggerKind in windmill_common. - pub fn as_job_trigger_kind(&self) -> windmill_common::jobs::JobTriggerKind { - match self { - ServiceName::Nextcloud => windmill_common::jobs::JobTriggerKind::Nextcloud, - // ServiceName::NewService => windmill_common::jobs::JobTriggerKind::NewService, - } - } - - /// Returns the OAuth token endpoint path for this service. - /// Used for building OAuth clients dynamically. - pub fn token_endpoint(&self) -> &'static str { - match self { - ServiceName::Nextcloud => "/apps/oauth2/api/v1/token", - // ServiceName::NewService => "/oauth/token", - } - } - - /// Returns the OAuth authorization endpoint path for this service. - /// Used for building OAuth authorization URLs. - pub fn auth_endpoint(&self) -> &'static str { - match self { - ServiceName::Nextcloud => "/apps/oauth2/authorize", - // ServiceName::NewService => "/oauth/authorize", - } - } -} - -impl std::fmt::Display for ServiceName { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.as_str()) - } -} - -#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] -pub struct NativeTrigger { - pub external_id: String, - pub workspace_id: String, - pub service_name: ServiceName, - pub script_path: String, - pub is_flow: bool, - pub webhook_token_prefix: String, - pub service_config: Option, - pub error: Option, - pub created_at: DateTime, - pub updated_at: DateTime, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct NativeTriggerConfig { - pub script_path: String, - pub is_flow: bool, - pub webhook_token: String, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct NativeTriggerData { - pub script_path: String, - pub is_flow: bool, - pub service_config: C, -} - -#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] -pub struct WorkspaceIntegration { - pub workspace_id: String, - pub service_name: ServiceName, - pub oauth_data: serde_json::Value, - pub created_at: DateTime, - pub updated_at: DateTime, - pub created_by: String, -} - -#[async_trait] -pub trait External: Send + Sync + 'static { - type ServiceConfig: Debug + DeserializeOwned + Serialize + Send + Sync; - type TriggerData: Debug + Serialize + Send + Sync; - type OAuthData: DeserializeOwned + Serialize + Clone + Send + Sync; - type CreateResponse: DeserializeOwned + Send + Sync; - - const SUPPORT_WEBHOOK: bool; - const SERVICE_NAME: ServiceName; - const DISPLAY_NAME: &'static str; - const TOKEN_ENDPOINT: &'static str; - const REFRESH_ENDPOINT: &'static str; - - async fn create( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - webhook_token: &str, - data: &NativeTriggerData, - db: &DB, - tx: &mut PgConnection, - ) -> Result; - - async fn update( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - external_id: &str, - webhook_token: &str, - data: &NativeTriggerData, - db: &DB, - tx: &mut PgConnection, - ) -> Result<()>; - - async fn get( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - external_id: &str, - db: &DB, - tx: &mut PgConnection, - ) -> Result; - - async fn delete( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - external_id: &str, - db: &DB, - tx: &mut PgConnection, - ) -> Result<()>; - - #[allow(unused)] - async fn exists( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - external_id: &str, - db: &DB, - tx: &mut PgConnection, - ) -> Result; - - async fn list_all( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - db: &DB, - tx: &mut PgConnection, - ) -> Result>; - - async fn prepare_webhook( - &self, - _db: &DB, - _w_id: &str, - _header: HashMap, - _body: String, - _script_path: &str, - _is_flow: bool, - ) -> Result { - Ok(PushArgsOwned { extra: None, args: HashMap::new() }) - } - - fn external_id_and_metadata_from_response( - &self, - resp: &Self::CreateResponse, - ) -> (String, Option); - - fn get_external_id_from_trigger_data(&self, data: &Self::TriggerData) -> String; - - /// Extracts the service-specific config from trigger data (from external service). - /// Used for comparison during sync to detect config drift. - /// Default implementation converts the trigger data to a JSON value - /// If you need to exclude some fields, skip serializing attributes on the TriggerData struct or override this method. - fn extract_service_config_from_trigger_data( - &self, - data: &Self::TriggerData, - ) -> Result { - serde_json::to_value(data).map_err(|e| { - Error::internal_err(format!("Failed to convert trigger data to JSON: {}", e)) - }) - } - - fn additional_routes(&self) -> axum::Router { - axum::Router::new() - } - - async fn http_client_request( - &self, - url: &str, - method: Method, - workspace_id: &str, - tx: &mut PgConnection, - db: &DB, - headers: Option>, - body: Option<&B>, - ) -> Result { - let oauth_config: OAuthConfig = - decrypt_oauth_data(tx, db, workspace_id, Self::SERVICE_NAME).await?; - - let result = make_http_request( - url, - method.clone(), - headers.clone(), - body.as_ref(), - &oauth_config.access_token, - ) - .await; - - match result { - Ok(response) => Ok(response), - Err(err) - if err.status() == Some(StatusCode::UNAUTHORIZED) - || err.status() == Some(StatusCode::FORBIDDEN) => - { - tracing::info!( - "HTTP auth error ({}), attempting token refresh", - err.status().unwrap() - ); - - let refreshed_oauth_config = - refresh_oauth_tokens(&oauth_config, Self::REFRESH_ENDPOINT).await?; - - task::spawn({ - let db_clone = db.clone(); - let workspace_id_clone = workspace_id.to_string(); - let refreshed_json = oauth_config_to_json(&refreshed_oauth_config); - async move { - update_workspace_integration_tokens_helper( - db_clone, - workspace_id_clone, - Self::SERVICE_NAME, - refreshed_json, - ) - .await; - } - }); - - let response = make_http_request( - url, - method, - headers, - body.as_ref(), - &refreshed_oauth_config.access_token, - ) - .await - .map_err(to_anyhow)?; - Ok(response) - } - Err(e) => Err(to_anyhow(e).into()), - } - } -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct OAuthConfig { - pub base_url: String, - pub access_token: String, - pub refresh_token: Option, - pub client_id: String, - pub client_secret: String, -} - -pub async fn make_http_request( - url: &str, - method: Method, - headers: Option>, - body: Option<&B>, - access_token: &str, -) -> std::result::Result { - let client = Client::new(); - let mut request = client.request(method, url); - - request = request - .header("Accept", "application/json") - .header("Authorization", format!("Bearer {}", access_token)); - - if body.is_some() { - request = request.header("Content-Type", "application/json"); - } - - if let Some(custom_headers) = headers { - for (key, value) in custom_headers { - request = request.header(key, value); - } - } - - if let Some(body_content) = body { - request = request.json(body_content); - } - - let response = request.send().await?.error_for_status()?; - - let response_json = response.json().await?; - - Ok(response_json) -} - -pub async fn decrypt_oauth_data< - 'c, - E: sqlx::Executor<'c, Database = Postgres>, - T: DeserializeOwned, ->( - tx: E, - db: &DB, - workspace_id: &str, - service_name: ServiceName, -) -> Result { - let integration = get_workspace_integration(tx, workspace_id, service_name).await?; - - let mc = build_crypt(db, workspace_id).await?; - let mut oauth_data: serde_json::Value = integration.oauth_data; - - if let Some(encrypted_access_token) = oauth_data.get("access_token").and_then(|v| v.as_str()) { - let decrypted_access_token = decrypt(&mc, encrypted_access_token.to_string()) - .map_err(|e| Error::InternalErr(format!("Failed to decrypt access token: {}", e)))?; - oauth_data["access_token"] = serde_json::Value::String(decrypted_access_token); - } - - if let Some(encrypted_refresh_token) = oauth_data.get("refresh_token").and_then(|v| v.as_str()) - { - let decrypted_refresh_token = decrypt(&mc, encrypted_refresh_token.to_string()) - .map_err(|e| Error::InternalErr(format!("Failed to decrypt refresh token: {}", e)))?; - oauth_data["refresh_token"] = serde_json::Value::String(decrypted_refresh_token); - } - - serde_json::from_value(oauth_data) - .map_err(|e| Error::InternalErr(format!("Failed to deserialize OAuth data: {}", e))) -} - -#[allow(unused)] -pub fn oauth_data_to_config(oauth_data: &serde_json::Value) -> Result { - let base_url = oauth_data - .get("base_url") - .and_then(|v| v.as_str()) - .ok_or_else(|| Error::InternalErr("No base_url in OAuth data".to_string()))? - .to_string(); - - let access_token = oauth_data - .get("access_token") - .and_then(|v| v.as_str()) - .ok_or_else(|| Error::InternalErr("No access_token in OAuth data".to_string()))? - .to_string(); - - let refresh_token = oauth_data - .get("refresh_token") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()); - - let client_id = oauth_data - .get("client_id") - .and_then(|v| v.as_str()) - .ok_or_else(|| Error::InternalErr("No client_id in OAuth data".to_string()))? - .to_string(); - - let client_secret = oauth_data - .get("client_secret") - .and_then(|v| v.as_str()) - .ok_or_else(|| Error::InternalErr("No client_secret in OAuth data".to_string()))? - .to_string(); - - Ok(OAuthConfig { base_url, access_token, refresh_token, client_id, client_secret }) -} - -#[inline] -pub fn oauth_config_to_json(config: &OAuthConfig) -> serde_json::Value { - let mut json = json!({ - "base_url": config.base_url, - "access_token": config.access_token, - "client_id": config.client_id, - "client_secret": config.client_secret, - }); - - if let Some(refresh_token) = &config.refresh_token { - json["refresh_token"] = serde_json::Value::String(refresh_token.clone()); - } - - json -} - -/// Token refresh response -#[cfg(feature = "native_trigger")] -#[derive(Debug, Deserialize)] -struct RefreshTokenResponse { - access_token: String, - refresh_token: Option, -} - -/// Refresh OAuth tokens using windmill-oauth. -#[cfg(feature = "native_trigger")] -pub async fn refresh_oauth_tokens( - oauth_config: &OAuthConfig, - refresh_endpoint: &str, -) -> Result { - let refresh_token_str = oauth_config - .refresh_token - .as_ref() - .ok_or_else(|| Error::InternalErr("No refresh token available".to_string()))?; - - // Build OAuth client for token refresh - // Auth URL is not used for refresh, but required by the client constructor - let auth_url = Url::parse(&format!("{}/oauth/authorize", oauth_config.base_url)) - .map_err(|e| Error::InternalErr(format!("Invalid auth URL: {}", e)))?; - let token_url = Url::parse(&format!("{}{}", oauth_config.base_url, refresh_endpoint)) - .map_err(|e| Error::InternalErr(format!("Invalid token URL: {}", e)))?; - - let mut client = OClient::new(oauth_config.client_id.clone(), auth_url, token_url); - client.set_client_secret(oauth_config.client_secret.clone()); - - let token_response: RefreshTokenResponse = client - .exchange_refresh_token(&RefreshToken::from(refresh_token_str.as_str())) - .with_client(&*OAUTH_HTTP_CLIENT) - .execute() - .await - .map_err(|e| Error::InternalErr(format!("Failed to refresh token: {:?}", e)))?; - - Ok(OAuthConfig { - base_url: oauth_config.base_url.clone(), - access_token: token_response.access_token, - refresh_token: token_response - .refresh_token - .or_else(|| oauth_config.refresh_token.clone()), - client_id: oauth_config.client_id.clone(), - client_secret: oauth_config.client_secret.clone(), - }) -} - -/// Fallback refresh without native_triggers feature -#[cfg(not(feature = "native_trigger"))] -pub async fn refresh_oauth_tokens( - _oauth_config: &OAuthConfig, - _refresh_endpoint: &str, -) -> Result { - Err(Error::InternalErr( - "Native triggers feature is not enabled".to_string(), - )) -} - -async fn update_workspace_integration_tokens_helper( - db: DB, - workspace_id: String, - service_name: ServiceName, - oauth_data: serde_json::Value, -) { - let result = async { - let mut tx = db.begin().await?; - let mc = build_crypt(&db, &workspace_id).await?; - let mut encrypted_oauth_data = oauth_data; - - if let Some(access_token) = encrypted_oauth_data - .get("access_token") - .and_then(|v| v.as_str()) - { - let encrypted_access_token = encrypt(&mc, access_token); - encrypted_oauth_data["access_token"] = - serde_json::Value::String(encrypted_access_token); - } - - if let Some(refresh_token) = encrypted_oauth_data - .get("refresh_token") - .and_then(|v| v.as_str()) - { - let encrypted_refresh_token = encrypt(&mc, refresh_token); - encrypted_oauth_data["refresh_token"] = - serde_json::Value::String(encrypted_refresh_token); - } - - sqlx::query!( - r#" - UPDATE workspace_integrations - SET oauth_data = $1, updated_at = now() - WHERE workspace_id = $2 AND service_name = $3 - "#, - encrypted_oauth_data, - workspace_id, - service_name as ServiceName, - ) - .execute(&mut *tx) - .await?; - - tx.commit().await?; - Ok::<(), Error>(()) - } - .await; - - if let Err(e) = result { - tracing::error!("Critical error: Failed to update workspace integration tokens for {} in workspace {}: {}", - service_name, workspace_id, e); - } -} - -/// Look up the full token from the token table using its prefix -pub async fn get_token_by_prefix<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - token_prefix: &str, -) -> Result> { - let token = sqlx::query_scalar!( - r#" - SELECT token - FROM token - WHERE token LIKE concat($1::text, '%') - LIMIT 1 - "#, - token_prefix - ) - .fetch_optional(db) - .await?; - - Ok(token) -} - -/// Delete a token from the token table using its prefix -pub async fn delete_token_by_prefix<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - token_prefix: &str, -) -> Result { - let deleted = sqlx::query!( - r#" - DELETE FROM token - WHERE token LIKE concat($1::text, '%') - "#, - token_prefix - ) - .execute(db) - .await? - .rows_affected(); - - Ok(deleted > 0) -} - -pub async fn store_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>, C: Serialize>( - db: E, - workspace_id: &str, - service_name: ServiceName, - external_id: &str, - config: &NativeTriggerConfig, - service_config: C, -) -> Result<()> { - // Store only the first 10 characters of the webhook token as a prefix - let webhook_token_prefix: String = config.webhook_token.chars().take(10).collect(); - - sqlx::query!( - r#" - INSERT INTO native_trigger ( - external_id, - workspace_id, - service_name, - script_path, - is_flow, - webhook_token_prefix, - service_config - ) VALUES ( - $1, $2, $3, $4, $5, $6, $7 - ) - ON CONFLICT (external_id, workspace_id, service_name) - DO UPDATE SET script_path = $4, is_flow = $5, webhook_token_prefix = $6, service_config = $7, error = NULL, updated_at = NOW() - "#, - external_id, - workspace_id, - service_name as ServiceName, - config.script_path, - config.is_flow, - webhook_token_prefix, - sqlx::types::Json(service_config) as _, - ) - .execute(db) - .await?; - - Ok(()) -} - -pub async fn update_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - workspace_id: &str, - service_name: ServiceName, - external_id: &str, - config: &NativeTriggerConfig, - service_config: Option<&RawValue>, -) -> Result<()> { - // Store only the first 10 characters of the webhook token as a prefix - let webhook_token_prefix: String = config.webhook_token.chars().take(10).collect(); - - sqlx::query!( - r#" - UPDATE native_trigger - SET script_path = $1, is_flow = $2, webhook_token_prefix = $3, service_config = $4, error = NULL, updated_at = NOW() - WHERE - workspace_id = $5 - AND service_name = $6 - AND external_id = $7 - "#, - config.script_path, - config.is_flow, - webhook_token_prefix, - service_config.map(sqlx::types::Json) as _, - workspace_id, - service_name as ServiceName, - external_id, - ) - .execute(db) - .await?; - - Ok(()) -} - -pub async fn delete_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - workspace_id: &str, - service_name: ServiceName, - external_id: &str, -) -> Result { - let deleted = sqlx::query!( - r#" - DELETE FROM native_trigger - WHERE - workspace_id = $1 - AND service_name = $2 - AND external_id = $3 - "#, - workspace_id, - service_name as ServiceName, - external_id, - ) - .execute(db) - .await? - .rows_affected(); - - Ok(deleted > 0) -} -pub async fn get_native_trigger<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - workspace_id: &str, - service_name: ServiceName, - external_id: &str, -) -> Result> { - let trigger = sqlx::query_as!( - NativeTrigger, - r#" - SELECT - external_id, - workspace_id, - service_name AS "service_name!: ServiceName", - script_path, - is_flow, - webhook_token_prefix, - service_config, - error, - created_at, - updated_at - FROM - native_trigger - WHERE - workspace_id = $1 - AND service_name = $2 - AND external_id = $3 - "#, - workspace_id, - service_name as ServiceName, - external_id - ) - .fetch_optional(db) - .await?; - - Ok(trigger) -} - -pub async fn get_native_trigger_by_script<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - workspace_id: &str, - service_name: ServiceName, - script_path: &str, - is_flow: bool, -) -> Result> { - let trigger = sqlx::query_as!( - NativeTrigger, - r#" - SELECT - external_id, - workspace_id, - service_name AS "service_name!: ServiceName", - script_path, - is_flow, - webhook_token_prefix, - service_config, - error, - created_at, - updated_at - FROM - native_trigger - WHERE - workspace_id = $1 - AND service_name = $2 - AND script_path = $3 - AND is_flow = $4 - LIMIT 1 - "#, - workspace_id, - service_name as ServiceName, - script_path, - is_flow - ) - .fetch_optional(db) - .await?; - - Ok(trigger) -} - -pub async fn list_native_triggers<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - workspace_id: &str, - service_name: ServiceName, - page: Option, - per_page: Option, - path: Option<&str>, - is_flow: Option, -) -> Result> { - let offset = (page.unwrap_or(0) * per_page.unwrap_or(100)) as i64; - let limit = per_page.unwrap_or(100) as i64; - - let triggers = sqlx::query_as!( - NativeTrigger, - r#" - SELECT - nt.external_id, - nt.workspace_id, - nt.service_name AS "service_name!: ServiceName", - nt.script_path, - nt.is_flow, - nt.webhook_token_prefix, - nt.service_config, - nt.error, - nt.created_at, - nt.updated_at - FROM - native_trigger nt - WHERE - nt.workspace_id = $1 AND - nt.service_name = $2 AND - ($5::text IS NULL OR nt.script_path = $5) AND - ($6::bool IS NULL OR nt.is_flow = $6) AND - ( - (nt.is_flow = false AND EXISTS ( - SELECT 1 FROM script s - WHERE s.workspace_id = nt.workspace_id - AND s.path = nt.script_path - )) - OR - (nt.is_flow = true AND EXISTS ( - SELECT 1 FROM flow f - WHERE f.workspace_id = nt.workspace_id - AND f.path = nt.script_path - )) - ) - LIMIT $3 - OFFSET $4 - "#, - workspace_id, - service_name as ServiceName, - limit, - offset, - path, - is_flow - ) - .fetch_all(db) - .await?; - - Ok(triggers) -} - -pub async fn update_native_trigger_error<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - workspace_id: &str, - service_name: ServiceName, - external_id: &str, - error: Option<&str>, -) -> Result<()> { - sqlx::query!( - r#" - UPDATE native_trigger - SET error = $1 - WHERE - workspace_id = $2 - AND service_name = $3 - AND external_id = $4 - "#, - error, - workspace_id, - service_name as ServiceName, - external_id, - ) - .execute(db) - .await?; - - Ok(()) -} - -pub async fn update_native_trigger_service_config< - 'c, - E: sqlx::Executor<'c, Database = Postgres>, ->( - db: E, - workspace_id: &str, - service_name: ServiceName, - external_id: &str, - service_config: &serde_json::Value, -) -> Result<()> { - sqlx::query!( - r#" - UPDATE native_trigger - SET service_config = $1, updated_at = NOW() - WHERE - workspace_id = $2 - AND service_name = $3 - AND external_id = $4 - "#, - service_config, - workspace_id, - service_name as ServiceName, - external_id, - ) - .execute(db) - .await?; - - Ok(()) -} - -pub async fn store_workspace_integration( - tx: &mut PgConnection, - authed: &ApiAuthed, - workspace_id: &str, - service_name: ServiceName, - oauth_data: serde_json::Value, -) -> Result<()> { - sqlx::query!( - r#" - INSERT INTO workspace_integrations ( - workspace_id, - service_name, - oauth_data, - created_by, - created_at, - updated_at - ) VALUES ( - $1, $2, $3, $4, now(), now() - ) - ON CONFLICT (workspace_id, service_name) - DO UPDATE SET - oauth_data = $3, - updated_at = now() - "#, - workspace_id, - service_name as ServiceName, - oauth_data, - authed.username, - ) - .execute(&mut *tx) - .await?; - - Ok(()) -} - -pub async fn get_workspace_integration<'c, E: sqlx::Executor<'c, Database = Postgres>>( - db: E, - workspace_id: &str, - service_name: ServiceName, -) -> Result { - let integration = sqlx::query_as!( - WorkspaceIntegration, - r#" - SELECT - workspace_id, - service_name AS "service_name!: ServiceName", - oauth_data, - created_at, - updated_at, - created_by - FROM - workspace_integrations - WHERE - workspace_id = $1 - AND service_name = $2 - "#, - workspace_id, - service_name as ServiceName, - ) - .fetch_one(db) - .await?; - - Ok(integration) -} - -pub async fn delete_workspace_integration( - tx: &mut PgConnection, - workspace_id: &str, - service_name: ServiceName, -) -> Result { - let deleted = sqlx::query!( - r#" - DELETE FROM workspace_integrations - WHERE - workspace_id = $1 - AND service_name = $2 - "#, - workspace_id, - service_name as ServiceName, - ) - .execute(&mut *tx) - .await? - .rows_affected(); - - Ok(deleted > 0) -} - -/// Generates the webhook URL that external services will call. -/// -/// `external_id` is optional because during CREATE we don't have it yet -/// (it's returned by the external service). During UPDATE, we have it. -pub fn generate_webhook_service_url( - base_url: &str, - w_id: &str, - script_path: &str, - is_flow: bool, - external_id: Option<&str>, - service_name: ServiceName, - webhook_token: &str, -) -> String { - let runnable_prefix = if is_flow { "f" } else { "p" }; - - let mut url = format!( - "{}/api/w/{}/jobs/run/{}/{}?token={}&service_name={}", - base_url, - w_id, - runnable_prefix, - script_path, - &webhook_token, - service_name.as_str(), - ); - - if let Some(id) = external_id { - url.push_str(&format!("&trigger_external_id={}", id)); - } - - url -} diff --git a/backend/windmill-api/src/native_triggers/nextcloud/external.rs b/backend/windmill-api/src/native_triggers/nextcloud/external.rs deleted file mode 100644 index 028ba131c3..0000000000 --- a/backend/windmill-api/src/native_triggers/nextcloud/external.rs +++ /dev/null @@ -1,304 +0,0 @@ -use reqwest::Method; -use serde::{Deserialize, Serialize}; -use serde_json::value::to_raw_value; -use sqlx::PgConnection; -use std::collections::HashMap; -use windmill_common::{ - error::{Error, Result}, - BASE_URL, DB, -}; - -use crate::native_triggers::{ - generate_webhook_service_url, - nextcloud::{ - routes, NextCloud, NextCloudOAuthData, NextCloudTriggerData, NextcloudServiceConfig, - OcsResponse, - }, - External, NativeTriggerData, ServiceName, -}; - -lazy_static::lazy_static! { - pub static ref TOKEN_NEEDED: Box = to_raw_value(&serde_json::json!({ - "user_roles": ["owner", "trigger"] - })).unwrap(); -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct FullNextcloudPayload { - pub http_method: String, - pub uri: String, - pub token_needed: Box, - #[serde(flatten)] - service_config: NextcloudServiceConfig, -} - -impl FullNextcloudPayload { - async fn new( - w_id: &str, - external_id: Option<&str>, - webhook_token: &str, - data: &NativeTriggerData, - ) -> FullNextcloudPayload { - let base_url = &*BASE_URL.read().await; - let uri = generate_webhook_service_url( - base_url, - w_id, - &data.script_path, - data.is_flow, - external_id, - ServiceName::Nextcloud, - webhook_token, - ); - - FullNextcloudPayload { - http_method: http::Method::POST.to_string().to_uppercase(), - uri, - token_needed: TOKEN_NEEDED.clone(), - service_config: data.service_config.clone(), - } - } -} - -#[derive(Debug, Deserialize)] -pub struct RegisterWebhookResponse { - pub id: i64, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct User { - pub uid: String, - #[serde(rename = "displayName")] - pub display_name: Option, -} -#[derive(Debug, Serialize, Deserialize)] -pub struct WebhookPayload { - pub event: EventPayload, - pub user: User, - pub time: i64, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct EventPayload { - pub node: Node, - #[serde(rename = "class")] - pub class_name: String, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct Node { - pub id: i64, - pub path: String, -} - -#[async_trait::async_trait] -impl External for NextCloud { - type ServiceConfig = NextcloudServiceConfig; - type TriggerData = NextCloudTriggerData; - type OAuthData = NextCloudOAuthData; - type CreateResponse = RegisterWebhookResponse; - const SERVICE_NAME: ServiceName = ServiceName::Nextcloud; - const DISPLAY_NAME: &'static str = "Nextcloud"; - const SUPPORT_WEBHOOK: bool = true; - const TOKEN_ENDPOINT: &'static str = "/apps/oauth2/api/v1/token"; - const REFRESH_ENDPOINT: &'static str = "/apps/oauth2/api/v1/token"; - - async fn create( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - webhook_token: &str, - data: &NativeTriggerData, - db: &DB, - tx: &mut PgConnection, - ) -> Result { - // During create, we don't have external_id yet (it comes from NextCloud's response) - let full_nextcloud_payload = - FullNextcloudPayload::new(w_id, None, webhook_token, data).await; - - let url = format!( - "{}/ocs/v2.php/apps/webhook_listeners/api/v1/webhooks", - oauth_data.base_url - ); - - let mut headers = HashMap::new(); - headers.insert("OCS-APIRequest".to_string(), "true".to_string()); - - let ocs_response = self - .http_client_request::, _>( - &url, - Method::POST, - w_id, - tx, - db, - Some(headers), - Some(&full_nextcloud_payload), - ) - .await?; - - Ok(ocs_response.ocs.data) - } - - async fn update( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - external_id: &str, - webhook_token: &str, - data: &NativeTriggerData, - db: &DB, - tx: &mut PgConnection, - ) -> Result<()> { - // During update, we have the external_id so include it in the webhook URL - let full_nextcloud_payload = - FullNextcloudPayload::new(w_id, Some(external_id), webhook_token, data).await; - - let url = format!( - "{}/ocs/v2.php/apps/webhook_listeners/api/v1/webhooks/{}", - oauth_data.base_url, external_id - ); - - let mut headers = HashMap::new(); - headers.insert("OCS-APIRequest".to_string(), "true".to_string()); - - let _ = self - .http_client_request::( - &url, - Method::POST, - w_id, - tx, - db, - Some(headers), - Some(&full_nextcloud_payload), - ) - .await?; - - Ok(()) - } - - async fn get( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - external_id: &str, - db: &DB, - tx: &mut PgConnection, - ) -> Result { - let url = format!( - "{}/ocs/v2.php/apps/webhook_listeners/api/v1/webhooks/{}", - oauth_data.base_url, external_id - ); - - let mut headers = HashMap::new(); - headers.insert("OCS-APIRequest".to_string(), "true".to_string()); - - let ocs_response: OcsResponse = self - .http_client_request::<_, ()>(&url, Method::GET, w_id, tx, db, Some(headers), None) - .await?; - - Ok(ocs_response.ocs.data) - } - - async fn delete( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - external_id: &str, - db: &DB, - tx: &mut PgConnection, - ) -> Result<()> { - let url = format!( - "{}/ocs/v2.php/apps/webhook_listeners/api/v1/webhooks/{}", - oauth_data.base_url, external_id - ); - - let mut headers = HashMap::new(); - headers.insert("OCS-APIRequest".to_string(), "true".to_string()); - - let _: serde_json::Value = self - .http_client_request::<_, ()>(&url, Method::DELETE, w_id, tx, db, Some(headers), None) - .await - .or_else(|e| match &e { - Error::InternalErr(msg) if msg.contains("404") => Ok(serde_json::Value::Null), - _ => Err(e), - })?; - - Ok(()) - } - - async fn exists( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - external_id: &str, - db: &DB, - tx: &mut PgConnection, - ) -> Result { - let url = format!( - "{}/ocs/v2.php/apps/webhook_listeners/api/v1/webhooks/{}", - oauth_data.base_url, external_id - ); - - let mut headers = HashMap::new(); - headers.insert("OCS-APIRequest".to_string(), "true".to_string()); - - let _ = self - .http_client_request::( - &url, - Method::GET, - w_id, - tx, - db, - Some(headers), - None, - ) - .await?; - - Ok(true) - } - - async fn list_all( - &self, - w_id: &str, - oauth_data: &Self::OAuthData, - db: &DB, - tx: &mut PgConnection, - ) -> Result> { - let url = format!( - "{}/ocs/v2.php/apps/webhook_listeners/api/v1/webhooks", - oauth_data.base_url - ); - - let mut headers = HashMap::new(); - headers.insert("OCS-APIRequest".to_string(), "true".to_string()); - - let ocs_response = self - .http_client_request::>, ()>( - &url, - Method::GET, - w_id, - tx, - db, - Some(headers), - None, - ) - .await?; - - Ok(ocs_response.ocs.data) - } - - fn external_id_and_metadata_from_response( - &self, - resp: &Self::CreateResponse, - ) -> (String, Option) { - (resp.id.to_string(), None) - } - - fn get_external_id_from_trigger_data(&self, data: &Self::TriggerData) -> String { - data.id.to_string() - } - - fn additional_routes(&self) -> axum::Router { - routes::nextcloud_routes(self.clone()) - } -} diff --git a/backend/windmill-api/src/native_triggers/nextcloud/mod.rs b/backend/windmill-api/src/native_triggers/nextcloud/mod.rs deleted file mode 100644 index fda8ad9f6e..0000000000 --- a/backend/windmill-api/src/native_triggers/nextcloud/mod.rs +++ /dev/null @@ -1,64 +0,0 @@ -use serde::{Deserialize, Serialize}; - -pub mod external; -mod routes; - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct NextCloudOAuthData { - pub base_url: String, - pub access_token: String, - pub refresh_token: Option, - pub token_expires_at: Option>, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct OcsResponse { - pub ocs: OcsData, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct Meta { - status: String, - #[serde(rename = "statuscode")] - status_code: u16, - message: String, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct OcsData { - pub meta: Meta, - pub data: T, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct NextCloudEventType { - pub name: String, - pub description: Option, - pub path: Option, - pub parameters: Option, -} - -#[derive(Copy, Clone)] -pub struct NextCloud; - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct NextcloudServiceConfig { - pub event: String, - pub event_filter: Option>, - pub user_id_filter: Option, - pub headers: Option>, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct NextCloudTriggerData { - #[serde(skip_serializing)] - pub id: i64, - #[serde(skip_serializing)] - pub uri: String, - pub event: String, - pub event_filter: Option>, - pub user_id_filter: Option, - pub headers: Option>, -} diff --git a/backend/windmill-api/src/native_triggers/nextcloud/routes.rs b/backend/windmill-api/src/native_triggers/nextcloud/routes.rs deleted file mode 100644 index 40633dc248..0000000000 --- a/backend/windmill-api/src/native_triggers/nextcloud/routes.rs +++ /dev/null @@ -1,66 +0,0 @@ -use std::{collections::HashMap, sync::Arc}; - -use axum::{extract::Path, routing::get, Extension, Json, Router}; -use http::Method; -use windmill_common::{ - db::UserDB, - error::{Error, JsonResult}, - DB, -}; - -use crate::{ - db::ApiAuthed, - native_triggers::{ - get_workspace_integration, - nextcloud::{NextCloudEventType, OcsResponse}, - External, OAuthConfig, ServiceName, - }, -}; - -async fn list_available_events( - authed: ApiAuthed, - Extension(handler): Extension>, - Extension(db): Extension, - Extension(user_db): Extension, - Path(workspace_id): Path, -) -> JsonResult> { - let mut tx = user_db.clone().begin(&authed).await?; - let integration = - get_workspace_integration(&mut *tx, &workspace_id, ServiceName::Nextcloud).await?; - - let auth = serde_json::from_value::(integration.oauth_data) - .map_err(|e| Error::InternalErr(format!("Failed to parse NextCloud OAuth data: {}", e)))?; - - let url = format!( - "{}/ocs/v2.php/apps/integration_windmill/api/v1/list/events", - &auth.base_url, - ); - - let mut headers = HashMap::new(); - headers.insert("OCS-APIRequest".to_string(), "true".to_string()); - - let ocs_response = handler - .http_client_request::( - &url, - Method::GET, - &workspace_id, - &mut *tx, - &db, - Some(headers), - None, - ) - .await?; - tx.commit().await?; - - let events = serde_json::from_str(&ocs_response.ocs.data) - .map_err(|e| Error::InternalErr(format!("Failed to parse NextCloud events data: {}", e)))?; - - Ok(Json(events)) -} - -pub fn nextcloud_routes(service: T) -> Router { - let service = Arc::new(service); - Router::new() - .route("/events", get(list_available_events::)) - .layer(Extension(service)) -} diff --git a/backend/windmill-api/src/native_triggers/sync.rs b/backend/windmill-api/src/native_triggers/sync.rs deleted file mode 100644 index ca955630aa..0000000000 --- a/backend/windmill-api/src/native_triggers/sync.rs +++ /dev/null @@ -1,416 +0,0 @@ -use std::collections::HashMap; -use windmill_common::error::Result; -use windmill_common::DB; - -use serde::Serialize; - -use crate::native_triggers::{ - decrypt_oauth_data, list_native_triggers, update_native_trigger_error, - update_native_trigger_service_config, External, ServiceName, -}; - -#[derive(Debug, Serialize)] -pub struct TriggerSyncInfo { - pub external_id: String, - pub script_path: String, - pub action: SyncAction, -} - -#[derive(Debug, Serialize)] -pub enum SyncAction { - ErrorSet(String), - ErrorCleared, - ConfigUpdated, -} - -#[derive(Debug, Serialize)] -pub struct SyncError { - pub resource_path: String, - pub error_message: String, - pub error_type: String, -} - -#[derive(Debug)] -pub struct BackgroundSyncResult { - pub workspaces_processed: usize, - pub total_synced: usize, - pub total_errors: usize, - pub service_results: HashMap, -} - -#[derive(Debug)] -pub struct ServiceSyncResult { - pub synced_triggers: Vec, - pub errors: Vec, -} - -pub async fn sync_all_triggers(db: &DB) -> Result { - tracing::info!("Starting native triggers sync"); - - let mut service_results: HashMap = HashMap::new(); - let mut total_synced = 0; - let mut total_errors = 0; - let mut workspaces_processed = 0; - - // Sync all registered services - // Each service only syncs workspaces that have the corresponding integration configured - #[cfg(feature = "native_trigger")] - { - use crate::native_triggers::nextcloud::NextCloud; - - let (service_name, result) = sync_service_triggers(db, NextCloud).await; - total_synced += result.synced_triggers.len(); - total_errors += result.errors.len(); - service_results.insert(service_name, result); - - // Add new services here: - // use crate::native_triggers::newservice::NewService; - // let (service_name, result) = sync_service_triggers(db, NewService).await; - // total_synced += result.synced_triggers.len(); - // total_errors += result.errors.len(); - // service_results.insert(service_name, result); - } - - // Count unique workspaces processed across all services - for result in service_results.values() { - workspaces_processed += result - .synced_triggers - .iter() - .map(|t| &t.external_id) - .collect::>() - .len(); - } - - let result = - BackgroundSyncResult { workspaces_processed, total_synced, total_errors, service_results }; - - tracing::info!( - "Completed native triggers sync: {} updated, {} errors", - result.total_synced, - result.total_errors - ); - - Ok(result) -} - -async fn sync_service_triggers( - db: &DB, - handler: T, -) -> (ServiceName, ServiceSyncResult) { - let mut all_synced_triggers = Vec::new(); - let mut all_errors = Vec::new(); - - // Only sync workspaces that have the corresponding integration configured - let workspaces_with_integration = match sqlx::query_scalar!( - r#" - SELECT wi.workspace_id - FROM workspace_integrations wi - JOIN workspace w ON w.id = wi.workspace_id - WHERE wi.service_name = $1 - AND wi.oauth_data IS NOT NULL - AND w.deleted = false - "#, - T::SERVICE_NAME as ServiceName - ) - .fetch_all(db) - .await - { - Ok(workspaces) => workspaces, - Err(e) => { - tracing::error!( - "Error querying workspaces with {} integration: {:#}", - T::SERVICE_NAME.as_str(), - e - ); - all_errors.push(SyncError { - resource_path: "database".to_string(), - error_message: format!("Failed to query workspaces: {}", e), - error_type: "database_error".to_string(), - }); - return ( - T::SERVICE_NAME, - ServiceSyncResult { synced_triggers: Vec::new(), errors: all_errors }, - ); - } - }; - - if workspaces_with_integration.is_empty() { - tracing::debug!( - "No workspaces with {} integration configured, skipping sync", - T::SERVICE_NAME.as_str() - ); - return ( - T::SERVICE_NAME, - ServiceSyncResult { synced_triggers: Vec::new(), errors: Vec::new() }, - ); - } - - tracing::info!( - "Found {} workspaces with {} integration configured", - workspaces_with_integration.len(), - T::SERVICE_NAME.as_str() - ); - - for workspace_id in workspaces_with_integration { - let sync_result = sync_workspace_triggers::(db, &workspace_id, &handler).await; - - match sync_result { - Ok((synced_triggers, errors)) => { - all_synced_triggers.extend(synced_triggers); - all_errors.extend(errors); - } - Err(e) => { - tracing::error!( - "Error syncing {} triggers for workspace {}: {:#}", - T::SERVICE_NAME.as_str(), - workspace_id, - e - ); - all_errors.push(SyncError { - resource_path: format!("workspace:{}", workspace_id), - error_message: format!("Failed to sync workspace: {}", e), - error_type: "workspace_sync_error".to_string(), - }); - } - } - } - - ( - T::SERVICE_NAME, - ServiceSyncResult { synced_triggers: all_synced_triggers, errors: all_errors }, - ) -} - -#[cfg(feature = "native_trigger")] -pub async fn sync_workspace_triggers( - db: &DB, - workspace_id: &str, - handler: &T, -) -> Result<(Vec, Vec)> { - tracing::info!( - "Syncing {} triggers for workspace '{}'", - T::SERVICE_NAME.as_str(), - workspace_id - ); - - let windmill_triggers = - list_native_triggers(db, workspace_id, T::SERVICE_NAME, None, None, None, None).await?; - - if windmill_triggers.is_empty() { - tracing::info!( - "No {} triggers found for workspace '{}'", - T::SERVICE_NAME.as_str(), - workspace_id - ); - return Ok((Vec::new(), Vec::new())); - } - - let mut all_synced_triggers = Vec::new(); - let mut all_sync_errors = Vec::new(); - - let oauth_data = { - match decrypt_oauth_data(db, db, workspace_id, T::SERVICE_NAME).await { - Ok(oauth_data) => oauth_data, - Err(e) => { - tracing::error!( - "Failed to get workspace integration OAuth data for {}: {}", - workspace_id, - e - ); - all_sync_errors.push(SyncError { - resource_path: format!("workspace:{}", workspace_id), - error_message: format!("Failed to get workspace integration OAuth data: {}", e), - error_type: "oauth_error".to_string(), - }); - return Ok((Vec::new(), all_sync_errors)); - } - } - }; - - let mut tx = db.begin().await?; - let external_triggers = match handler - .list_all(workspace_id, &oauth_data, db, &mut tx) - .await - { - Ok(triggers) => triggers, - Err(e) => { - tracing::error!( - "Failed to fetch external triggers for {}: {}", - workspace_id, - e - ); - all_sync_errors.push(SyncError { - resource_path: format!("workspace:{}", workspace_id), - error_message: format!("Failed to fetch external triggers: {}", e), - error_type: "external_service_error".to_string(), - }); - return Ok((Vec::new(), all_sync_errors)); - } - }; - tx.commit().await?; - - // Build a map of external trigger IDs to their data - let mut external_trigger_map: HashMap = HashMap::new(); - for external_trigger in &external_triggers { - let external_id = handler.get_external_id_from_trigger_data(external_trigger); - external_trigger_map.insert(external_id, external_trigger); - } - - for trigger in &windmill_triggers { - if !external_trigger_map.contains_key(&trigger.external_id) { - // Trigger no longer exists on external service - set error - let error_msg = "Trigger no longer exists on external service".to_string(); - - if trigger.error.as_deref() != Some(&error_msg) { - tracing::info!( - "Trigger (external_id: '{}', script_path: '{}') no longer exists in external service, setting error", - trigger.external_id, - trigger.script_path - ); - - match update_native_trigger_error( - db, - workspace_id, - T::SERVICE_NAME, - &trigger.external_id, - Some(&error_msg), - ) - .await - { - Ok(()) => { - all_synced_triggers.push(TriggerSyncInfo { - external_id: trigger.external_id.clone(), - script_path: trigger.script_path.clone(), - action: SyncAction::ErrorSet(error_msg), - }); - } - Err(e) => { - tracing::error!( - "Failed to update error for trigger (external_id: '{}'): {}", - trigger.external_id, - e - ); - all_sync_errors.push(SyncError { - resource_path: format!("workspace:{}", workspace_id), - error_message: format!( - "Failed to update error for trigger (external_id: '{}'): {}", - trigger.external_id, e - ), - error_type: "database_update_error".to_string(), - }); - } - } - } - } else { - // Trigger exists on external service - let external_trigger_data = external_trigger_map.get(&trigger.external_id).unwrap(); - - // Clear error if it was set - if trigger.error.is_some() { - tracing::info!( - "Trigger (external_id: '{}', script_path: '{}') exists on external service, clearing error", - trigger.external_id, - trigger.script_path - ); - - match update_native_trigger_error( - db, - workspace_id, - T::SERVICE_NAME, - &trigger.external_id, - None, - ) - .await - { - Ok(()) => { - all_synced_triggers.push(TriggerSyncInfo { - external_id: trigger.external_id.clone(), - script_path: trigger.script_path.clone(), - action: SyncAction::ErrorCleared, - }); - } - Err(e) => { - tracing::error!( - "Failed to clear error for trigger (external_id: '{}'): {}", - trigger.external_id, - e - ); - all_sync_errors.push(SyncError { - resource_path: format!("workspace:{}", workspace_id), - error_message: format!( - "Failed to clear error for trigger (external_id: '{}'): {}", - trigger.external_id, e - ), - error_type: "database_update_error".to_string(), - }); - } - } - } - - // Compare service_config and update if different - let external_service_config = - handler.extract_service_config_from_trigger_data(external_trigger_data)?; - let stored_service_config = trigger - .service_config - .clone() - .unwrap_or(serde_json::Value::Null); - - if external_service_config != stored_service_config { - tracing::info!( - "Trigger (external_id: '{}', script_path: '{}') config differs from external service, updating local config", - trigger.external_id, - trigger.script_path - ); - - match update_native_trigger_service_config( - db, - workspace_id, - T::SERVICE_NAME, - &trigger.external_id, - &external_service_config, - ) - .await - { - Ok(()) => { - all_synced_triggers.push(TriggerSyncInfo { - external_id: trigger.external_id.clone(), - script_path: trigger.script_path.clone(), - action: SyncAction::ConfigUpdated, - }); - } - Err(e) => { - tracing::error!( - "Failed to update config for trigger (external_id: '{}'): {}", - trigger.external_id, - e - ); - all_sync_errors.push(SyncError { - resource_path: format!("workspace:{}", workspace_id), - error_message: format!( - "Failed to update config for trigger (external_id: '{}'): {}", - trigger.external_id, e - ), - error_type: "database_update_error".to_string(), - }); - } - } - } else { - tracing::info!( - "Trigger (external_id: '{}', script_path: '{}') config is the same as external service, no update needed", - trigger.external_id, - trigger.script_path - ); - } - } - } - - tracing::info!( - "Sync completed for {} in workspace '{}'. Updated: {}, Errors: {}", - T::SERVICE_NAME.as_str(), - workspace_id, - all_synced_triggers.len(), - all_sync_errors.len() - ); - - Ok((all_synced_triggers, all_sync_errors)) -} diff --git a/backend/windmill-api/src/native_triggers/workspace_integrations.rs b/backend/windmill-api/src/native_triggers/workspace_integrations.rs deleted file mode 100644 index dd823b1344..0000000000 --- a/backend/windmill-api/src/native_triggers/workspace_integrations.rs +++ /dev/null @@ -1,522 +0,0 @@ -use axum::{ - extract::Path, - routing::{delete, get, post}, - Extension, Json, Router, -}; - -#[cfg(feature = "native_trigger")] -use serde_json::to_value; -use sqlx::prelude::FromRow; -use strum::IntoEnumIterator; - -use serde::{de::DeserializeOwned, Deserialize, Serialize}; -#[cfg(feature = "native_trigger")] -use windmill_audit::{audit_oss::audit_log, ActionKind}; -use windmill_common::{ - db::UserDB, - error::{Error, JsonResult, Result}, - utils::require_admin, - variables::{build_crypt, encrypt}, - DB, -}; - -#[cfg(feature = "native_trigger")] -use crate::{ - db::ApiAuthed, - native_triggers::{delete_workspace_integration, store_workspace_integration, ServiceName}, -}; - -#[cfg(feature = "native_trigger")] -use windmill_oauth::{OClient, Url, OAUTH_HTTP_CLIENT}; - -use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; -use hmac::{Hmac, Mac}; -use sha2::Sha256; - -type HmacSha256 = Hmac; - -const STATE_EXPIRATION_SECONDS: i64 = 600; // 10 minutes - -/// Generate a signed OAuth state that is cluster-safe. -/// The state contains: workspace_id, service_name, timestamp, and nonce. -/// It's signed with HMAC-SHA256 using the workspace key. -#[cfg(feature = "native_trigger")] -async fn generate_signed_state( - db: &DB, - workspace_id: &str, - service_name: ServiceName, -) -> Result { - use windmill_common::variables::get_workspace_key; - - let nonce = uuid::Uuid::new_v4().to_string(); - let timestamp = chrono::Utc::now().timestamp(); - let payload = format!( - "{}:{}:{}:{}", - workspace_id, - service_name.as_str(), - timestamp, - nonce - ); - - // Get workspace key for signing - let key = get_workspace_key(workspace_id, db).await?; - let mut mac = - HmacSha256::new_from_slice(key.as_bytes()).map_err(|e| Error::InternalErr(e.to_string()))?; - mac.update(payload.as_bytes()); - let signature = mac.finalize().into_bytes(); - - // Encode as: base64(payload):base64(signature) - let encoded_payload = URL_SAFE_NO_PAD.encode(payload.as_bytes()); - let encoded_signature = URL_SAFE_NO_PAD.encode(signature); - - Ok(format!("{}:{}", encoded_payload, encoded_signature)) -} - -/// Validate a signed OAuth state. -/// Returns true if the state is valid (correct signature and not expired). -#[cfg(feature = "native_trigger")] -async fn validate_signed_state(db: &DB, state: &str, workspace_id: &str) -> Result { - use windmill_common::variables::get_workspace_key; - - let parts: Vec<&str> = state.split(':').collect(); - if parts.len() != 2 { - return Ok(false); - } - - let encoded_payload = parts[0]; - let encoded_signature = parts[1]; - - // Decode payload - let payload_bytes = match URL_SAFE_NO_PAD.decode(encoded_payload) { - Ok(bytes) => bytes, - Err(_) => return Ok(false), - }; - let payload = match String::from_utf8(payload_bytes) { - Ok(s) => s, - Err(_) => return Ok(false), - }; - - // Parse payload: workspace_id:service_name:timestamp:nonce - let payload_parts: Vec<&str> = payload.split(':').collect(); - if payload_parts.len() != 4 { - return Ok(false); - } - - let state_workspace_id = payload_parts[0]; - let timestamp: i64 = match payload_parts[2].parse() { - Ok(ts) => ts, - Err(_) => return Ok(false), - }; - - // Verify workspace_id matches - if state_workspace_id != workspace_id { - return Ok(false); - } - - // Check expiration - let now = chrono::Utc::now().timestamp(); - if now - timestamp > STATE_EXPIRATION_SECONDS { - return Ok(false); - } - - // Verify signature - let key = get_workspace_key(workspace_id, db).await?; - let mut mac = - HmacSha256::new_from_slice(key.as_bytes()).map_err(|e| Error::InternalErr(e.to_string()))?; - mac.update(payload.as_bytes()); - - let received_signature = match URL_SAFE_NO_PAD.decode(encoded_signature) { - Ok(bytes) => bytes, - Err(_) => return Ok(false), - }; - - Ok(mac.verify_slice(&received_signature).is_ok()) -} - -#[derive(Debug, Serialize)] -pub struct IntegrationStatusResponse { - pub connected: bool, - pub service_name: ServiceName, - pub created_at: Option>, - pub created_by: Option, -} - -#[derive(Debug, Serialize)] -pub struct ListIntegrationsResponse { - pub integrations: Vec, -} - -#[derive(Debug, Serialize)] -pub struct ConnectIntegrationResponse { - pub auth_url: String, -} - -#[derive(FromRow, Debug, Clone, Serialize, Deserialize)] -pub struct WorkspaceOAuthConfig { - pub client_id: String, - pub client_secret: String, - pub base_url: String, - pub access_token: Option -} - -#[derive(Debug, Serialize)] -pub struct OAuthConfigResponse { - pub configured: bool, - pub base_url: Option, - pub redirect_uri: Option, -} - -#[cfg(feature = "native_trigger")] -async fn generate_connect_url( - authed: ApiAuthed, - Extension(db): Extension, - Path((workspace_id, service_name)): Path<(String, ServiceName)>, - Json(RedirectUri { redirect_uri }): Json, -) -> JsonResult { - require_admin(authed.is_admin, &workspace_id)?; - - let oauth_config = - get_workspace_oauth_config_as_oauth_config(&db, &workspace_id, service_name).await?; - - // Generate a signed state that is cluster-safe - let state = generate_signed_state(&db, &workspace_id, service_name).await?; - let auth_url = build_authorization_url(&oauth_config, &state, &redirect_uri); - Ok(Json(auth_url)) -} - -#[cfg(feature = "native_trigger")] -async fn delete_integration( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((workspace_id, service_name)): Path<(String, ServiceName)>, -) -> JsonResult { - require_admin(authed.is_admin, &workspace_id)?; - - let mut tx = user_db.begin(&authed).await?; - - let deleted = delete_workspace_integration(&mut *tx, &workspace_id, service_name).await?; - - if !deleted { - return Err(Error::NotFound(format!( - "{} integration not found for workspace", - service_name - ))); - } - - audit_log( - &mut *tx, - &authed, - &format!("workspace_integrations.{}.disconnect", service_name), - ActionKind::Delete, - &workspace_id, - Some(&format!("Disconnected {} integration", service_name)), - None, - ) - .await?; - - tx.commit().await?; - - Ok(Json(format!( - "{} integration disconnected successfully", - service_name - ))) -} - -#[derive(FromRow, Debug, Deserialize, Serialize)] -struct WorkspaceIntegrations { - service_name: ServiceName, - oauth_data: Option>, -} - -#[cfg(feature = "native_trigger")] -async fn list_integrations( - authed: ApiAuthed, - Extension(db): Extension, - Extension(_user_db): Extension, - Path(workspace_id): Path, -) -> JsonResult> { - require_admin(authed.is_admin, &workspace_id)?; - let mut tx = db.begin().await?; - let integrations = sqlx::query_as!( - WorkspaceIntegrations, - r#" - SELECT - oauth_data as "oauth_data!: sqlx::types::Json", - service_name as "service_name!: ServiceName" - FROM - workspace_integrations - WHERE - workspace_id = $1 - "#, - workspace_id - ) - .fetch_all(&mut *tx) - .await?; - - let key_value = integrations - .into_iter() - .map(|integration| (integration.service_name, integration.oauth_data)) - .collect::>(); - - let integrations = ServiceName::iter() - .map(|service_name| WorkspaceIntegrations { - service_name: service_name, - oauth_data: key_value.get(&service_name).cloned().flatten(), - }) - .collect::>(); - - tx.commit().await?; - - Ok(Json(integrations)) -} - -async fn integration_exist( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((workspace_id, service_name)): Path<(String, ServiceName)>, -) -> JsonResult { - let mut tx = user_db.begin(&authed).await?; - let exists = sqlx::query_scalar!( - r#" - SELECT EXISTS ( - SELECT 1 - FROM workspace_integrations - WHERE workspace_id = $1 - AND service_name = $2 - AND oauth_data IS NOT NULL - ) - "#, - workspace_id, - service_name as ServiceName - ) - .fetch_one(&mut *tx) - .await? - .unwrap_or(false); - - Ok(Json(exists)) -} - -#[derive(Debug, Deserialize)] -struct RedirectUri { - redirect_uri: String, -} - -#[cfg(feature = "native_trigger")] -async fn oauth_callback( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((workspace_id, service_name, code, state)): Path<(String, ServiceName, String, String)>, - Json(RedirectUri { redirect_uri }): Json, -) -> JsonResult { - require_admin(authed.is_admin, &workspace_id)?; - - // Validate the signed state (cluster-safe, no DB storage needed) - let state_was_valid = validate_signed_state(&db, &state, &workspace_id).await?; - - if !state_was_valid { - return Err(Error::BadRequest( - "Invalid or expired state parameter".to_string(), - )); - } - - let oauth_config = - get_workspace_oauth_config::(&db, &workspace_id, service_name) - .await?; - - let token_response = - exchange_code_for_token(&oauth_config, service_name, &code, &redirect_uri).await?; - - let mut tx = user_db.begin(&authed).await?; - - let mc = build_crypt(&db, &workspace_id).await?; - let mut oauth_data = serde_json::to_value(oauth_config).unwrap(); - - let encrypted_access_token = encrypt(&mc, &token_response.access_token); - oauth_data["access_token"] = serde_json::Value::String(encrypted_access_token); - - if let Some(refresh_token) = token_response.refresh_token { - let encrypted_refresh_token = encrypt(&mc, &refresh_token); - oauth_data["refresh_token"] = serde_json::Value::String(encrypted_refresh_token); - } - if let Some(expires_in) = token_response.expires_in { - let expires_at = chrono::Utc::now() + chrono::Duration::seconds(expires_in as i64); - oauth_data["token_expires_at"] = serde_json::Value::String(expires_at.to_rfc3339()); - } - - store_workspace_integration(&mut *tx, &authed, &workspace_id, service_name, oauth_data).await?; - - audit_log( - &mut *tx, - &authed, - &format!("workspace_integrations.{}.connect", service_name), - ActionKind::Create, - &workspace_id, - Some(&format!("Connected {} integration via OAuth", service_name)), - None, - ) - .await?; - - tx.commit().await?; - - Ok(Json(format!( - "{} integration connected successfully via OAuth", - service_name - ))) -} - -/// Token response from OAuth token exchange -#[derive(Debug, Deserialize)] -struct TokenResponse { - access_token: String, - refresh_token: Option, - #[serde(default)] - expires_in: Option, -} - -/// Build an OAuth client for native trigger services using windmill-oauth. -#[cfg(feature = "native_trigger")] -fn build_native_oauth_client( - config: &WorkspaceOAuthConfig, - service_name: ServiceName, - redirect_uri: &str, -) -> Result { - let auth_url = Url::parse(&format!("{}{}", config.base_url, service_name.auth_endpoint())) - .map_err(|e| Error::InternalErr(format!("Invalid auth URL: {}", e)))?; - let token_url = Url::parse(&format!("{}{}", config.base_url, service_name.token_endpoint())) - .map_err(|e| Error::InternalErr(format!("Invalid token URL: {}", e)))?; - let redirect = Url::parse(redirect_uri).map_err(|e| { - Error::BadRequest(format!( - "Invalid redirect URI '{}': {}. The redirect URI must be an absolute URL (e.g., https://example.com/callback)", - redirect_uri, e - )) - })?; - - let mut client = OClient::new(config.client_id.clone(), auth_url, token_url); - client.set_client_secret(config.client_secret.clone()); - client.set_redirect_url(redirect); - - Ok(client) -} - -/// Exchange authorization code for tokens using windmill-oauth. -#[cfg(feature = "native_trigger")] -async fn exchange_code_for_token( - config: &WorkspaceOAuthConfig, - service_name: ServiceName, - code: &str, - redirect_uri: &str, -) -> Result { - let client = build_native_oauth_client(config, service_name, redirect_uri)?; - - let token_response: TokenResponse = client - .exchange_code(code.to_string()) - .with_client(&*OAUTH_HTTP_CLIENT) - .execute() - .await - .map_err(|e| Error::InternalErr(format!("Failed to exchange code for token: {:?}", e)))?; - - Ok(token_response) -} - -async fn get_workspace_oauth_config( - db: &DB, - workspace_id: &str, - service_name: ServiceName, -) -> Result { - let oauth_configs = sqlx::query_scalar!( - r#" - SELECT - oauth_data - FROM - workspace_integrations - WHERE - workspace_id = $1 AND - service_name = $2 - "#, - workspace_id, - service_name as ServiceName - ) - .fetch_optional(db) - .await? - .ok_or(Error::NotFound(format!( - "Integration for service {} not found", - service_name.as_str() - )))?; - - let config = serde_json::from_value::(oauth_configs) - .map_err(|e| Error::InternalErr(format!("Failed to parse OAuth config: {}", e)))?; - - Ok(config) -} - -#[cfg(feature = "native_trigger")] -pub async fn create_workspace_integration( - authed: ApiAuthed, - Extension(user_db): Extension, - Path((workspace_id, service_name)): Path<(String, ServiceName)>, - Json(oauth_data): Json, -) -> Result<()> { - require_admin(authed.is_admin, &workspace_id)?; - - let mut tx = user_db.begin(&authed).await?; - - store_workspace_integration( - &mut tx, - &authed, - &workspace_id, - service_name, - to_value(oauth_data).unwrap(), - ) - .await?; - - tx.commit().await?; - - Ok(()) -} - -#[inline] -async fn get_workspace_oauth_config_as_oauth_config( - db: &DB, - workspace_id: &str, - service_name: ServiceName, -) -> Result { - get_workspace_oauth_config::(db, workspace_id, service_name).await -} - -fn build_authorization_url( - config: &WorkspaceOAuthConfig, - state: &str, - redirect_uri: &str, -) -> String { - let params = [ - ("response_type", "code"), - ("client_id", &config.client_id), - ("redirect_uri", redirect_uri), - ("state", state), - ("scope", "read write"), - ]; - - let query_string = params - .iter() - .map(|(k, v)| format!("{}={}", urlencoding::encode(k), urlencoding::encode(v))) - .collect::>() - .join("&"); - - format!("{}/apps/oauth2/authorize?{}", config.base_url, query_string) -} - -pub fn workspaced_service() -> Router { - let router = Router::new() - .route("/list", get(list_integrations)) - .route("/:service_name/exists", get(integration_exist)) - .route("/:service_name/create", post(create_workspace_integration)) - .route( - "/:service_name/generate_connect_url", - post(generate_connect_url), - ) - .route("/:service_name/delete", delete(delete_integration)) - .route("/:service_name/callback/:code/:state", post(oauth_callback)); - - Router::new().nest("/integrations", router) -} diff --git a/backend/windmill-api/src/triggers/email/handler_oss.rs b/backend/windmill-api/src/triggers/email/handler_oss.rs deleted file mode 100644 index 9a688c2cf1..0000000000 --- a/backend/windmill-api/src/triggers/email/handler_oss.rs +++ /dev/null @@ -1,67 +0,0 @@ -#[cfg(not(feature = "private"))] -use crate::triggers::TriggerData; - -#[allow(unused)] -#[cfg(feature = "private")] -pub use super::handler_ee::*; - -#[cfg(not(feature = "private"))] -use { - super::EmailTrigger, - crate::{ - db::{ApiAuthed, DB}, - triggers::TriggerCrud, - }, - axum::async_trait, - sqlx::PgConnection, - windmill_common::error::{Error, Result}, - windmill_git_sync::DeployedObject, -}; - -#[cfg(not(feature = "private"))] -#[async_trait] -impl TriggerCrud for EmailTrigger { - type Trigger = (); - type TriggerConfig = (); - type TriggerConfigRequest = (); - type TestConnectionConfig = (); - - const TABLE_NAME: &'static str = ""; - const TRIGGER_TYPE: &'static str = ""; - const SUPPORTS_SERVER_STATE: bool = false; - const SUPPORTS_TEST_CONNECTION: bool = false; - const ROUTE_PREFIX: &'static str = "/email_triggers"; - const DEPLOYMENT_NAME: &'static str = ""; - const IS_ALLOWED_ON_CLOUD: bool = false; - - fn get_deployed_object(path: String) -> DeployedObject { - DeployedObject::EmailTrigger { path } - } - - async fn create_trigger( - &self, - _db: &DB, - _tx: &mut PgConnection, - _authed: &ApiAuthed, - _w_id: &str, - _trigger: TriggerData, - ) -> Result<()> { - Err(Error::BadRequest( - "Email triggers are not available in open source version".to_string(), - )) - } - - async fn update_trigger( - &self, - _db: &DB, - _executor: &mut PgConnection, - _authed: &ApiAuthed, - _workspace_id: &str, - _path: &str, - _trigger: TriggerData, - ) -> Result<()> { - Err(Error::BadRequest( - "Email triggers are not available in open source version".to_string(), - )) - } -} diff --git a/backend/windmill-api/src/triggers/email/mod.rs b/backend/windmill-api/src/triggers/email/mod.rs deleted file mode 100644 index 836af77fa0..0000000000 --- a/backend/windmill-api/src/triggers/email/mod.rs +++ /dev/null @@ -1,11 +0,0 @@ -#[cfg(feature = "private")] -mod handler_ee; -pub mod handler_oss; - -#[cfg(feature = "private")] -mod mod_ee; -#[cfg(feature = "private")] -pub use mod_ee::*; - -#[derive(Copy, Clone)] -pub struct EmailTrigger; diff --git a/backend/windmill-api/src/triggers/filter.rs b/backend/windmill-api/src/triggers/filter.rs deleted file mode 100644 index 553e0a70f5..0000000000 --- a/backend/windmill-api/src/triggers/filter.rs +++ /dev/null @@ -1,144 +0,0 @@ -use serde::{ - de::{self, MapAccess, Visitor}, - Deserialize, Deserializer, -}; -use serde_json::Value; -use std::fmt; - -#[derive(Deserialize)] -pub struct JsonFilter { - pub key: String, - pub value: Value, -} - -#[derive(Deserialize)] -#[serde(untagged)] -pub enum Filter { - JsonFilter(JsonFilter), -} - -struct SupersetVisitor<'a> { - key: &'a str, - value_to_check: &'a Value, -} - -impl<'de, 'a> Visitor<'de> for SupersetVisitor<'a> { - type Value = bool; - - fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { - formatter.write_str("a JSON object with a specific key at the top level") - } - - fn visit_map(self, mut map: V) -> std::result::Result - where - V: MapAccess<'de>, - { - let mut result = false; - let mut found = false; - - // Must consume entire map to satisfy deserializer contract - while let Some(key) = map.next_key::()? { - if !found && key == self.key { - let json_value: Value = map.next_value()?; - result = is_superset(&json_value, self.value_to_check); - found = true; - } else { - // Skip values we don't need (cheaper than full deserialization) - let _ = map.next_value::()?; - } - } - Ok(result) - } -} - -pub fn is_superset(json_value: &Value, value_to_check: &Value) -> bool { - match (json_value, value_to_check) { - (Value::Object(json_map), Value::Object(check_map)) => { - check_map.iter().all(|(k, v)| { - json_map - .get(k) - .map_or(false, |json_val| is_superset(json_val, v)) - }) - } - (Value::Array(json_array), Value::Array(check_array)) => { - check_array.iter().all(|check_item| { - json_array - .iter() - .any(|json_item| is_superset(json_item, check_item)) - }) - } - _ => json_value == value_to_check, - } -} - -pub fn is_value_superset<'a, 'de, D>( - deserializer: D, - key: &'a str, - value_to_check: &'a Value, -) -> std::result::Result -where - D: Deserializer<'de>, -{ - deserializer.deserialize_map(SupersetVisitor { key, value_to_check }) -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn test_filter_with_other_top_level_keys() { - let payload = r#"{"event_type": "test", "other": "data"}"#; - let key = "event_type"; - let value = json!("test"); - - let mut deserializer = serde_json::Deserializer::from_str(payload); - let result = is_value_superset(&mut deserializer, key, &value).unwrap(); - assert!(result, "Should match when key exists with correct value"); - } - - #[test] - fn test_filter_with_key_not_first() { - let payload = r#"{"other": "data", "event_type": "test"}"#; - let key = "event_type"; - let value = json!("test"); - - let mut deserializer = serde_json::Deserializer::from_str(payload); - let result = is_value_superset(&mut deserializer, key, &value).unwrap(); - assert!(result, "Should match even when key is not first"); - } - - #[test] - fn test_filter_with_nested_object() { - let payload = r#"{"data": {"status": "active", "count": 5}, "other": "value"}"#; - let key = "data"; - let value = json!({"status": "active"}); - - let mut deserializer = serde_json::Deserializer::from_str(payload); - let result = is_value_superset(&mut deserializer, key, &value).unwrap(); - assert!(result, "Should match when nested object is superset"); - } - - #[test] - fn test_filter_no_match() { - let payload = r#"{"event_type": "other", "data": "value"}"#; - let key = "event_type"; - let value = json!("test"); - - let mut deserializer = serde_json::Deserializer::from_str(payload); - let result = is_value_superset(&mut deserializer, key, &value).unwrap(); - assert!(!result, "Should not match when value differs"); - } - - #[test] - fn test_filter_key_not_found() { - let payload = r#"{"other": "data"}"#; - let key = "event_type"; - let value = json!("test"); - - let mut deserializer = serde_json::Deserializer::from_str(payload); - let result = is_value_superset(&mut deserializer, key, &value).unwrap(); - assert!(!result, "Should not match when key doesn't exist"); - } -} diff --git a/backend/windmill-api/src/triggers/gcp/handler_oss.rs b/backend/windmill-api/src/triggers/gcp/handler_oss.rs deleted file mode 100644 index b75a9721eb..0000000000 --- a/backend/windmill-api/src/triggers/gcp/handler_oss.rs +++ /dev/null @@ -1,64 +0,0 @@ -#[allow(unused)] -#[cfg(feature = "private")] -pub use super::handler_ee::*; - -#[cfg(not(feature = "private"))] -use { - super::GcpTrigger, - crate::{ - db::{ApiAuthed, DB}, - triggers::{TriggerCrud, TriggerData}, - }, - axum::async_trait, - sqlx::PgConnection, - windmill_common::error::{Error, Result}, - windmill_git_sync::DeployedObject, -}; - -#[cfg(not(feature = "private"))] -#[async_trait] -impl TriggerCrud for GcpTrigger { - type Trigger = (); - type TriggerConfig = (); - type TriggerConfigRequest = (); - type TestConnectionConfig = (); - - const TABLE_NAME: &'static str = ""; - const TRIGGER_TYPE: &'static str = ""; - const SUPPORTS_SERVER_STATE: bool = false; - const SUPPORTS_TEST_CONNECTION: bool = false; - const ROUTE_PREFIX: &'static str = "/gcp_triggers"; - const DEPLOYMENT_NAME: &'static str = ""; - const IS_ALLOWED_ON_CLOUD: bool = false; - - fn get_deployed_object(path: String) -> DeployedObject { - DeployedObject::GcpTrigger { path } - } - - async fn create_trigger( - &self, - _db: &DB, - _executor: &mut PgConnection, - _authed: &ApiAuthed, - _w_id: &str, - _trigger: TriggerData, - ) -> Result<()> { - Err(Error::BadRequest( - "GCP triggers are not available in open source version".to_string(), - )) - } - - async fn update_trigger( - &self, - _db: &DB, - _executor: &mut PgConnection, - _authed: &ApiAuthed, - _workspace_id: &str, - _path: &str, - _trigger: TriggerData, - ) -> Result<()> { - Err(Error::BadRequest( - "GCP triggers are not available in open source version".to_string(), - )) - } -} diff --git a/backend/windmill-api/src/triggers/gcp/listener_oss.rs b/backend/windmill-api/src/triggers/gcp/listener_oss.rs deleted file mode 100644 index a14168968a..0000000000 --- a/backend/windmill-api/src/triggers/gcp/listener_oss.rs +++ /dev/null @@ -1,41 +0,0 @@ -#[allow(unused)] - -#[cfg(feature = "private")] -pub use super::listener_ee::*; - -#[cfg(not(feature = "private"))] -use { - super::GcpTrigger, - crate::triggers::{listener::ListeningTrigger, Listener}, - std::sync::Arc, - tokio::sync::RwLock, - windmill_common::{error::Result, jobs::JobTriggerKind, DB}, -}; - -#[cfg(not(feature = "private"))] -#[async_trait::async_trait] -impl Listener for GcpTrigger { - type Consumer = (); - type Extra = (); - const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Gcp; - - async fn get_consumer( - &self, - _db: &DB, - _listening_trigger: &ListeningTrigger, - _err_message: Arc>>, - _killpill_rx: tokio::sync::broadcast::Receiver<()>, - ) -> Result> { - Ok(None) - } - async fn consume( - &self, - _db: &DB, - _consumer: Self::Consumer, - _listening_trigger: &ListeningTrigger, - _err_message: Arc>>, - _killpill_rx: tokio::sync::broadcast::Receiver<()>, - ) { - () - } -} diff --git a/backend/windmill-api/src/triggers/gcp/mod.rs b/backend/windmill-api/src/triggers/gcp/mod.rs deleted file mode 100644 index dd2f2a65df..0000000000 --- a/backend/windmill-api/src/triggers/gcp/mod.rs +++ /dev/null @@ -1,16 +0,0 @@ -#[cfg(feature = "private")] -mod handler_ee; -pub mod handler_oss; - - -#[cfg(feature = "private")] -mod listener_ee; -pub mod listener_oss; - -#[cfg(feature = "private")] -mod mod_ee; -#[cfg(feature = "private")] -pub use mod_ee::*; - -#[derive(Clone, Copy)] -pub struct GcpTrigger; diff --git a/backend/windmill-api/src/triggers/global_handler.rs b/backend/windmill-api/src/triggers/global_handler.rs deleted file mode 100644 index 3143d6806f..0000000000 --- a/backend/windmill-api/src/triggers/global_handler.rs +++ /dev/null @@ -1,324 +0,0 @@ -use crate::{ - db::{ApiAuthed, DB}, - jobs::cancel_jobs, - triggers::trigger_helpers::trigger_runnable_inner, -}; -use axum::{ - extract::{Extension, Path}, - response::Json, -}; -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; -use serde_json::value::RawValue; -use sqlx::PgConnection; -use std::collections::HashMap; -use uuid::Uuid; -use windmill_common::{ - db::UserDB, - error::{self, Error, Result}, - jobs::JobTriggerKind, - triggers::TriggerMetadata, -}; - -#[derive(sqlx::FromRow)] -pub struct SuspendedTrigger { - pub script_path: String, - pub is_flow: bool, - pub edited_by: String, - pub email: String, - pub edited_at: DateTime, - pub error_handler_path: Option, - pub error_handler_args: Option>>, - pub retry: Option>, -} - -async fn get_suspended_trigger( - tx: &mut PgConnection, - workspace_id: &str, - trigger_kind: &JobTriggerKind, - path: &str, -) -> Result { - match trigger_kind { - JobTriggerKind::Webhook | JobTriggerKind::Schedule => { - return Err(Error::BadRequest(format!( - "{} triggers do not support job reassignment", - trigger_kind - ))); - } - _ => {} - } - - let table_name = format!("{}_trigger", trigger_kind.to_string()); - - let fields = vec![ - "script_path", - "is_flow", - "edited_by", - "email", - "edited_at", - "error_handler_path", - "error_handler_args", - "retry", - ]; - - let sql = format!( - r#"SELECT - {} - FROM - {} - WHERE - workspace_id = $1 AND - path = $2 - "#, - fields.join(", "), - table_name - ); - - sqlx::query_as(&sql) - .bind(workspace_id) - .bind(path) - .fetch_optional(&mut *tx) - .await? - .ok_or_else(|| Error::NotFound(format!("Trigger not found at path: {}", path))) -} - -struct JobWithArgs { - id: Uuid, - args: Option>>>, - created_at: chrono::DateTime, -} - -#[derive(Deserialize, Serialize, Default)] -pub struct ReassignJobsBody { - #[serde(default)] - pub job_ids: Option>, -} - -pub async fn resume_suspended_trigger_jobs( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, trigger_kind, trigger_path)): Path<(String, JobTriggerKind, String)>, - Json(body): Json, -) -> error::Result> { - let mut tx = user_db.clone().begin(&authed).await?; - - let trigger = get_suspended_trigger(&mut *tx, &w_id, &trigger_kind, &trigger_path).await?; - - let jobs = if let Some(job_ids) = body.job_ids.as_ref() { - if job_ids.is_empty() { - vec![] - } else { - sqlx::query_as!( - JobWithArgs, - r#" - SELECT - id, - args as "args: _", - created_at - FROM v2_job - WHERE workspace_id = $1 - AND ( - kind = 'unassigned_script'::JOB_KIND OR - kind = 'unassigned_flow'::JOB_KIND OR - kind = 'unassigned_singlestepflow'::JOB_KIND - ) - AND trigger_kind = $2 - AND trigger = $3 - AND id = ANY($4) - "#, - w_id, - trigger_kind as _, - trigger_path, - job_ids as _, - ) - .fetch_all(&mut *tx) - .await? - } - } else { - sqlx::query_as!( - JobWithArgs, - r#" - SELECT - id, - args as "args: _", - created_at - FROM v2_job - WHERE workspace_id = $1 - AND ( - kind = 'unassigned_script'::JOB_KIND OR - kind = 'unassigned_flow'::JOB_KIND OR - kind = 'unassigned_singlestepflow'::JOB_KIND - ) - AND trigger_kind = $2 - AND trigger = $3 - "#, - w_id, - trigger_kind as _, - trigger_path, - ) - .fetch_all(&mut *tx) - .await? - }; - - let trigger_metadata = TriggerMetadata::new(Some(trigger_path.clone()), trigger_kind); - - let l = jobs.len(); - - for job in jobs { - // If job was created before trigger was edited, simply update it to unsuspend - // instead of deleting and repushing - if job.created_at > trigger.edited_at { - let job_kind = if trigger.is_flow { - windmill_common::jobs::JobKind::Flow - } else { - windmill_common::jobs::JobKind::Script - }; - - sqlx::query!( - "UPDATE v2_job SET kind = $1 WHERE id = $2", - job_kind as _, - job.id - ) - .execute(&mut *tx) - .await?; - - // Update the job to unsuspend it and set the correct kind - sqlx::query!( - "UPDATE v2_job_queue SET scheduled_for = now() WHERE id = $1", - job.id - ) - .execute(&mut *tx) - .await?; - } else { - // Job was created after trigger edit - delete and repush with new configuration - // Pass the transaction to trigger_runnable_inner so everything is in the same transaction - let (_uuid, _delete_after_use, _early_return, tx_o) = trigger_runnable_inner( - &db, - Some(tx), - Some(user_db.clone()), - authed.clone(), - &w_id, - &trigger.script_path, - trigger.is_flow, - windmill_queue::PushArgsOwned { - extra: None, - args: job.args.map(|a| a.0).unwrap_or_default(), - }, - trigger.retry.as_ref(), - trigger.error_handler_path.as_deref(), - trigger.error_handler_args.as_ref(), - trigger_path.clone(), - None, - trigger_metadata.clone(), - None, - ) - .await?; - - tx = match tx_o { - Some(tx) => tx, - None => { - return Err(error::Error::internal_err( - "Transaction should be returned when passed in".to_string(), - )); - } - }; - - // Delete the unassigned job from all related tables - sqlx::query!("DELETE FROM v2_job_queue WHERE id = $1", job.id) - .execute(&mut *tx) - .await?; - - sqlx::query!("DELETE FROM v2_job_runtime WHERE id = $1", job.id) - .execute(&mut *tx) - .await?; - - sqlx::query!("DELETE FROM job_perms WHERE job_id = $1", job.id) - .execute(&mut *tx) - .await?; - - sqlx::query!("DELETE FROM concurrency_key WHERE job_id = $1", job.id) - .execute(&mut *tx) - .await?; - - sqlx::query!("DELETE FROM debounce_key WHERE job_id = $1", job.id) - .execute(&mut *tx) - .await?; - - sqlx::query!("DELETE FROM debounce_stale_data WHERE job_id = $1", job.id) - .execute(&mut *tx) - .await?; - - sqlx::query!("DELETE FROM v2_job WHERE id = $1", job.id) - .execute(&mut *tx) - .await?; - } - } - - tx.commit().await?; - - Ok(Json(format!("Reassigned {} jobs", l))) -} - -pub async fn cancel_suspended_trigger_jobs( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, trigger_kind, trigger_path)): Path<(String, JobTriggerKind, String)>, - Json(body): Json, -) -> error::Result> { - let mut tx = user_db.clone().begin(&authed).await?; - - // Get the list of job IDs to cancel - let jobs_to_cancel = if let Some(job_ids) = body.job_ids.as_ref() { - if job_ids.is_empty() { - vec![] - } else { - sqlx::query_scalar!( - "SELECT id FROM v2_job - WHERE workspace_id = $1 - AND (kind = 'unassigned_script'::JOB_KIND OR kind = 'unassigned_flow'::JOB_KIND OR kind = 'unassigned_singlestepflow'::JOB_KIND) - AND trigger_kind = $2 - AND trigger = $3 - AND id = ANY($4)", - w_id, - trigger_kind as _, - trigger_path, - job_ids as _, - ) - .fetch_all(&mut *tx) - .await? - } - } else { - sqlx::query_scalar!( - "SELECT id FROM v2_job - WHERE workspace_id = $1 - AND (kind = 'unassigned_script'::JOB_KIND OR kind = 'unassigned_flow'::JOB_KIND OR kind = 'unassigned_singlestepflow'::JOB_KIND) - AND trigger_kind = $2 - AND trigger = $3", - w_id, - trigger_kind as _, - trigger_path, - ) - .fetch_all(&mut *tx) - .await? - }; - - tx.commit().await?; - - let count = jobs_to_cancel.len(); - - if count > 0 { - let cancelled_jobs = cancel_jobs( - jobs_to_cancel, - &db, - authed.username.as_str(), - w_id.as_str(), - true, - ) - .await?; - Ok(Json(format!("Canceled {} jobs", cancelled_jobs.0.len()))) - } else { - Ok(Json(format!("No jobs to cancel"))) - } -} diff --git a/backend/windmill-api/src/triggers/handler.rs b/backend/windmill-api/src/triggers/handler.rs deleted file mode 100644 index 4ca7e2e15e..0000000000 --- a/backend/windmill-api/src/triggers/handler.rs +++ /dev/null @@ -1,1003 +0,0 @@ -use crate::{ - db::ApiAuthed, - triggers::{StandardTriggerQuery, TriggerData, TriggerMode}, -}; -use async_trait::async_trait; -use serde::{de::DeserializeOwned, Deserialize, Serialize}; -use sql_builder::{bind::Bind, SqlBuilder}; -use sqlx::{FromRow, PgConnection}; -use std::fmt::Debug; -use windmill_common::{ - db::UserDB, - error::{Error, JsonResult, Result}, - utils::{paginate, Pagination, StripPath}, - worker::CLOUD_HOSTED, - DB, -}; -use windmill_git_sync::DeployedObject; - -use axum::{ - extract::{Path, Query}, - http::StatusCode, - routing::{delete, get, post}, - Extension, Json, Router, -}; -use std::sync::Arc; -use windmill_audit::{audit_oss::audit_log, ActionKind}; -use windmill_git_sync::handle_deployment_metadata; - -use crate::utils::check_scopes; - -#[async_trait] -pub trait TriggerCrud: Send + Sync + 'static { - type Trigger: Serialize - + DeserializeOwned - + for<'r> FromRow<'r, sqlx::postgres::PgRow> - + Send - + Sync - + Unpin; - - type TriggerConfig: Debug - + DeserializeOwned - + for<'r> FromRow<'r, sqlx::postgres::PgRow> - + Serialize - + Send - + Sync - + Unpin; - - type TriggerConfigRequest: Debug + DeserializeOwned + Serialize + Send + Sync; - type TestConnectionConfig: Debug + DeserializeOwned + Serialize + Send + Sync; - - const TABLE_NAME: &'static str; - const TRIGGER_TYPE: &'static str; - const SUPPORTS_SERVER_STATE: bool; - const SUPPORTS_TEST_CONNECTION: bool; - const ROUTE_PREFIX: &'static str; - const DEPLOYMENT_NAME: &'static str; - const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[]; - const IS_ALLOWED_ON_CLOUD: bool; - - fn get_deployed_object(path: String) -> DeployedObject; - - async fn validate_new( - &self, - db: &DB, - workspace_id: &str, - new: &Self::TriggerConfigRequest, - ) -> Result<()> { - self.validate_config(db, new, workspace_id).await - } - - async fn validate_edit( - &self, - db: &DB, - workspace_id: &str, - edit: &Self::TriggerConfigRequest, - _path: &str, - ) -> Result<()> { - self.validate_config(db, edit, workspace_id).await - } - - async fn validate_config( - &self, - _db: &DB, - _config: &Self::TriggerConfigRequest, - _workspace_id: &str, - ) -> Result<()> { - Ok(()) - } - - fn scope_domain_name() -> &'static str { - &Self::ROUTE_PREFIX[1..] - } - - async fn create_trigger( - &self, - db: &DB, - tx: &mut PgConnection, - authed: &ApiAuthed, - w_id: &str, - trigger: TriggerData, - ) -> Result<()>; - - async fn update_trigger( - &self, - db: &DB, - tx: &mut PgConnection, - authed: &ApiAuthed, - workspace_id: &str, - path: &str, - trigger: TriggerData, - ) -> Result<()>; - - async fn test_connection( - &self, - _db: &DB, - _authed: &ApiAuthed, - _user_db: &UserDB, - _workspace_id: &str, - _config: Self::TestConnectionConfig, - ) -> Result<()> { - Err( - anyhow::anyhow!("Test connection not supported for this trigger type".to_string(),) - .into(), - ) - } - - fn additional_routes(&self) -> axum::Router { - axum::Router::new() - } - - async fn get_trigger_by_path( - &self, - tx: &mut PgConnection, - workspace_id: &str, - path: &str, - ) -> Result { - let mut fields = vec![ - "workspace_id", - "path", - "script_path", - "is_flow", - "edited_by", - "email", - "edited_at", - "extra_perms", - "mode", - ]; - - if Self::SUPPORTS_SERVER_STATE { - fields.extend_from_slice(&["server_id", "last_server_ping", "error"]); - } - - fields.extend_from_slice(&["error_handler_path", "error_handler_args", "retry"]); - fields.extend_from_slice(Self::ADDITIONAL_SELECT_FIELDS); - - let sql = format!( - r#"SELECT - {} - FROM - {} - WHERE - workspace_id = $1 AND - path = $2 - "#, - fields.join(", "), - Self::TABLE_NAME - ); - - sqlx::query_as(&sql) - .bind(workspace_id) - .bind(path) - .fetch_optional(&mut *tx) - .await? - .ok_or_else(|| Error::NotFound(format!("Trigger not found at path: {}", path))) - } - - async fn exists(&self, db: &DB, workspace_id: &str, path: &str) -> Result { - let exists = sqlx::query_scalar(&format!( - "SELECT EXISTS(SELECT 1 FROM {} WHERE workspace_id = $1 AND path = $2)", - Self::TABLE_NAME - )) - .bind(workspace_id) - .bind(path) - .fetch_one(db) - .await?; - - Ok(exists) - } - - async fn delete_by_path( - &self, - tx: &mut PgConnection, - workspace_id: &str, - path: &str, - ) -> Result { - let deleted = sqlx::query(&format!( - "DELETE FROM {} WHERE workspace_id = $1 AND path = $2", - Self::TABLE_NAME - )) - .bind(workspace_id) - .bind(path) - .execute(&mut *tx) - .await? - .rows_affected(); - - Ok(deleted > 0) - } - - async fn set_trigger_mode_extra_action(&self, _: &mut PgConnection) -> Result<()> { - Ok(()) - } - - async fn set_trigger_mode( - &self, - authed: &ApiAuthed, - tx: &mut PgConnection, - workspace_id: &str, - path: &str, - mode: &TriggerMode, - ) -> Result { - let updated = if Self::SUPPORTS_SERVER_STATE { - sqlx::query(&format!( - r#" - UPDATE - {} - SET - mode = $1, - email = $2, - edited_by = $3, - edited_at = now(), - server_id = NULL, - error = NULL - WHERE - workspace_id = $4 AND - path = $5 - "#, - Self::TABLE_NAME - )) - .bind(mode) - .bind(&authed.email) - .bind(&authed.username) - .bind(workspace_id) - .bind(path) - .execute(&mut *tx) - .await? - .rows_affected() - } else { - sqlx::query(&format!( - r#" - UPDATE - {} - SET - mode = $1, - email = $2, - edited_by = $3, - edited_at = now() - WHERE - workspace_id = $4 AND - path = $5 - "#, - Self::TABLE_NAME - )) - .bind(mode) - .bind(&authed.email) - .bind(&authed.username) - .bind(workspace_id) - .bind(path) - .execute(&mut *tx) - .await? - .rows_affected() - }; - - self.set_trigger_mode_extra_action(&mut *tx).await?; - - Ok(updated > 0) - } - - #[allow(unused)] - async fn trigger_count( - &self, - tx: &mut PgConnection, - workspace_id: &str, - is_flow: bool, - script_path: &str, - ) -> i64 { - let count = sqlx::query_scalar(&format!( - r#" - SELECT - COUNT(*) - FROM - {} - WHERE - workspace_id = $1 AND - is_flow = $2 AND - script_path = $3 - "#, - Self::TABLE_NAME - )) - .bind(workspace_id) - .bind(is_flow) - .bind(script_path) - .fetch_one(&mut *tx) - .await - .unwrap_or(0); - - count - } - - async fn list_triggers( - &self, - tx: &mut PgConnection, - workspace_id: &str, - query: Option<&StandardTriggerQuery>, - ) -> Result> { - let mut fields = vec![ - "workspace_id", - "path", - "script_path", - "is_flow", - "edited_by", - "email", - "edited_at", - "extra_perms", - "mode", - ]; - - if Self::SUPPORTS_SERVER_STATE { - fields.extend_from_slice(&["server_id", "last_server_ping", "error"]); - } - - fields.extend_from_slice(&["error_handler_path", "error_handler_args", "retry"]); - fields.extend_from_slice(Self::ADDITIONAL_SELECT_FIELDS); - - let mut sqlb = SqlBuilder::select_from(Self::TABLE_NAME); - - sqlb.fields(&fields) - .order_by("edited_at", true) - .and_where("workspace_id = ?".bind(&workspace_id)); - - if let Some(query) = query { - let (per_page, offset) = - paginate(Pagination { per_page: query.per_page, page: query.page }); - if let Some(path) = &query.path { - sqlb.and_where_eq("script_path", "?".bind(path)); - } - - if let Some(is_flow) = query.is_flow { - sqlb.and_where_eq("is_flow", "?".bind(&is_flow)); - } - - if let Some(path_start) = &query.path_start { - sqlb.and_where_like_left("path", path_start); - } - - sqlb.offset(offset).limit(per_page); - } - - let sql = sqlb - .sql() - .map_err(|e| Error::InternalErr(format!("SQL error: {}", e)))?; - - let triggers = sqlx::query_as(&sql).fetch_all(&mut *tx).await?; - - Ok(triggers) - } -} - -pub fn trigger_routes() -> Router { - let mut router = Router::new() - .route("/create", post(create_trigger::)) - .route("/list", get(list_triggers::)) - .route("/get/*path", get(get_trigger::)) - .route("/update/*path", post(update_trigger::)) - .route("/delete/*path", delete(delete_trigger::)) - .route("/exists/*path", get(exists_trigger::)) - .route("/setmode/*path", post(set_trigger_mode::)); - - if T::SUPPORTS_TEST_CONNECTION { - router = router.route("/test", post(test_connection::)); - } - - router -} - -async fn create_trigger( - Extension(handler): Extension>, - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path(workspace_id): Path, - Json(new_trigger): Json>, -) -> Result<(StatusCode, String)> { - check_scopes(&authed, || { - format!( - "{}:write:{}", - T::scope_domain_name(), - &new_trigger.base.path - ) - })?; - - if *CLOUD_HOSTED && !T::IS_ALLOWED_ON_CLOUD { - return Err(Error::BadRequest(format!( - "{} triggers are not supported on multi-tenant cloud, use dedicated cloud or self-host", - T::TRIGGER_TYPE - ))); - } - - handler - .validate_new(&db, &workspace_id, &new_trigger.config) - .await?; - - let mut tx = user_db.begin(&authed).await?; - - let new_path = new_trigger.base.path.clone(); - - handler - .create_trigger(&db, &mut *tx, &authed, &workspace_id, new_trigger) - .await?; - - audit_log( - &mut *tx, - &authed, - &format!("{}_triggers.create", T::TRIGGER_TYPE), - ActionKind::Create, - &workspace_id, - Some(&new_path), - None, - ) - .await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &workspace_id, - T::get_deployed_object(new_path.clone()), - Some(format!("{} '{}' created", T::DEPLOYMENT_NAME, new_path)), - true, - None, - ) - .await?; - - tx.commit().await?; - - Ok((StatusCode::CREATED, new_path)) -} - -async fn list_triggers( - Extension(handler): Extension>, - authed: ApiAuthed, - Extension(user_db): Extension, - Path(workspace_id): Path, - Query(query): Query, -) -> JsonResult> { - let mut tx = user_db.begin(&authed).await?; - let triggers = handler - .list_triggers(&mut *tx, &workspace_id, Some(&query)) - .await?; - tx.commit().await?; - - Ok(Json(triggers)) -} - -async fn get_trigger( - Extension(handler): Extension>, - authed: ApiAuthed, - Extension(user_db): Extension, - Path((workspace_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - check_scopes(&authed, || { - format!("{}:read:{}", T::scope_domain_name(), &path) - })?; - - let mut tx = user_db.begin(&authed).await?; - let trigger = handler - .get_trigger_by_path(&mut *tx, &workspace_id, path) - .await?; - - tx.commit().await?; - - Ok(Json(trigger)) -} - -async fn update_trigger( - Extension(handler): Extension>, - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((workspace_id, path)): Path<(String, StripPath)>, - Json(edit_trigger): Json>, -) -> Result { - let path = path.to_path(); - check_scopes(&authed, || { - format!( - "{}:write:{}", - T::scope_domain_name(), - &edit_trigger.base.path - ) - })?; - - handler - .validate_edit(&db, &workspace_id, &edit_trigger.config, path) - .await?; - - let mut tx = user_db.begin(&authed).await?; - - let new_path = edit_trigger.base.path.to_string(); - - handler - .update_trigger(&db, &mut *tx, &authed, &workspace_id, path, edit_trigger) - .await?; - - audit_log( - &mut *tx, - &authed, - &format!("{}_triggers.update", T::TRIGGER_TYPE), - ActionKind::Update, - &workspace_id, - Some(&new_path), - None, - ) - .await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &workspace_id, - T::get_deployed_object(new_path.clone()), - Some(format!("{} '{}' updated", T::DEPLOYMENT_NAME, new_path)), - true, - None, - ) - .await?; - - tx.commit().await?; - - Ok(format!("Trigger '{}' updated", path)) -} - -async fn delete_trigger( - Extension(handler): Extension>, - authed: ApiAuthed, - Extension(user_db): Extension, - Path((workspace_id, path)): Path<(String, StripPath)>, -) -> Result { - let path = path.to_path(); - check_scopes(&authed, || { - format!("{}:write:{}", T::scope_domain_name(), &path) - })?; - - let mut tx = user_db.begin(&authed).await?; - let deleted = handler - .delete_by_path(&mut *tx, &workspace_id, path) - .await?; - - if !deleted { - return Err(Error::NotFound(format!( - "Trigger not found at path: {}", - path - ))); - } - - audit_log( - &mut *tx, - &authed, - &format!("{}_triggers.delete", T::TRIGGER_TYPE), - ActionKind::Delete, - &workspace_id, - Some(&path), - None, - ) - .await?; - - tx.commit().await?; - - Ok(format!("Trigger '{}' deleted", path)) -} - -async fn exists_trigger( - Extension(handler): Extension>, - authed: ApiAuthed, - Extension(db): Extension, - Path((workspace_id, path)): Path<(String, StripPath)>, -) -> JsonResult { - let path = path.to_path(); - check_scopes(&authed, || { - format!("{}:read:{}", T::scope_domain_name(), path) - })?; - let exists = handler.exists(&db, &workspace_id, path).await?; - - Ok(Json(exists)) -} - -#[derive(serde::Deserialize)] -struct SetTriggerModePayload { - mode: TriggerMode, -} - -async fn set_trigger_mode( - Extension(handler): Extension>, - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Path((workspace_id, path)): Path<(String, StripPath)>, - Json(payload): Json, -) -> Result { - let path = path.to_path(); - check_scopes(&authed, || format!("{}:write", T::scope_domain_name()))?; - - let mut tx = user_db.begin(&authed).await?; - let updated = handler - .set_trigger_mode(&authed, &mut *tx, &workspace_id, path, &payload.mode) - .await?; - - if !updated { - return Err(Error::NotFound(format!( - "Trigger not found at path: {}", - path - ))); - } - - tx.commit().await?; - - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &workspace_id, - T::get_deployed_object(path.to_owned()), - Some(format!("{} trigger '{}' updated", T::DEPLOYMENT_NAME, path)), - true, - None, - ) - .await?; - - Ok(format!( - "Trigger '{}' {}", - path, - if payload.mode == TriggerMode::Enabled { - "enabled" - } else if payload.mode == TriggerMode::Disabled { - "disabled" - } else { - "suspended" - } - )) -} - -async fn test_connection( - Extension(handler): Extension>, - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path(workspace_id): Path, - Json(config): Json, -) -> Result<()> { - let connect_f = async move { - handler - .test_connection(&db, &authed, &user_db, &workspace_id, config) - .await - }; - - tokio::time::timeout(tokio::time::Duration::from_secs(30), connect_f) - .await - .map_err(|_| { - Error::BadConfig(format!("Timeout connecting to service after 30 seconds")) - })??; - Ok(()) -} - -#[allow(unused)] -pub fn complete_trigger_routes(handler: T) -> Router { - let standard_routes = trigger_routes::(); - - let additional_routes = handler.additional_routes(); - - standard_routes - .merge(additional_routes) - .layer(Extension(Arc::new(handler))) -} - -pub fn generate_trigger_routers() -> Router { - #[allow(unused_mut)] - let mut router = Router::new(); - - #[cfg(feature = "http_trigger")] - { - use crate::triggers::http::handler::HttpTrigger; - - router = router.nest( - HttpTrigger::ROUTE_PREFIX, - complete_trigger_routes(HttpTrigger), - ); - } - - #[cfg(feature = "websocket")] - { - use crate::triggers::websocket::WebsocketTrigger; - - router = router.nest( - WebsocketTrigger::ROUTE_PREFIX, - complete_trigger_routes(WebsocketTrigger), - ); - } - - #[cfg(all(feature = "enterprise", feature = "kafka", feature = "private"))] - { - use crate::triggers::kafka::KafkaTrigger; - - router = router.nest( - KafkaTrigger::ROUTE_PREFIX, - complete_trigger_routes(KafkaTrigger), - ); - } - - #[cfg(all(feature = "enterprise", feature = "nats", feature = "private"))] - { - use crate::triggers::nats::NatsTrigger; - - router = router.nest( - NatsTrigger::ROUTE_PREFIX, - complete_trigger_routes(NatsTrigger), - ); - } - - #[cfg(feature = "mqtt_trigger")] - { - use crate::triggers::mqtt::MqttTrigger; - - router = router.nest( - MqttTrigger::ROUTE_PREFIX, - complete_trigger_routes(MqttTrigger), - ); - } - - #[cfg(all(feature = "enterprise", feature = "sqs_trigger", feature = "private"))] - { - use crate::triggers::sqs::SqsTrigger; - - router = router.nest( - SqsTrigger::ROUTE_PREFIX, - complete_trigger_routes(SqsTrigger), - ); - } - - #[cfg(all(feature = "enterprise", feature = "gcp_trigger", feature = "private"))] - { - use crate::triggers::gcp::GcpTrigger; - - router = router.nest( - GcpTrigger::ROUTE_PREFIX, - complete_trigger_routes(GcpTrigger), - ); - } - - #[cfg(feature = "postgres_trigger")] - { - use crate::triggers::postgres::PostgresTrigger; - - router = router.nest( - PostgresTrigger::ROUTE_PREFIX, - complete_trigger_routes(PostgresTrigger), - ); - } - - #[cfg(all(feature = "smtp", feature = "private"))] - { - use crate::triggers::email::EmailTrigger; - - router = router.nest( - EmailTrigger::ROUTE_PREFIX, - complete_trigger_routes(EmailTrigger), - ); - } - - { - use crate::triggers::global_handler::{ - cancel_suspended_trigger_jobs, resume_suspended_trigger_jobs, - }; - - router = router - .route( - "/trigger/:trigger_kind/resume_suspended_trigger_jobs/*trigger_path", - post(resume_suspended_trigger_jobs), - ) - .route( - "/trigger/:trigger_kind/cancel_suspended_trigger_jobs/*trigger_path", - post(cancel_suspended_trigger_jobs), - ); - } - - router -} - -#[derive(Serialize, Deserialize, Debug)] -pub struct TriggerPrimarySchedule { - schedule: String, -} - -#[derive(Serialize, Deserialize, Debug)] -pub struct TriggersCount { - primary_schedule: Option, - schedule_count: i64, - http_routes_count: i64, - webhook_count: i64, - email_count: i64, - default_email_count: i64, - websocket_count: i64, - kafka_count: i64, - nats_count: i64, - postgres_count: i64, - mqtt_count: i64, - sqs_count: i64, - gcp_count: i64, - nextcloud_count: i64, -} - -pub async fn get_triggers_count_internal( - db: &DB, - w_id: &str, - path: &str, - is_flow: bool, -) -> JsonResult { - let primary_schedule = sqlx::query_scalar!( - "SELECT schedule FROM schedule WHERE path = $1 AND script_path = $1 AND is_flow = $2 AND workspace_id = $3", - path, - is_flow, - w_id - ) - .fetch_optional(db) - .await?; - - let schedule_count = sqlx::query_scalar!( - "SELECT COUNT(*) FROM schedule WHERE script_path = $1 AND is_flow = $2 AND workspace_id = $3", - path, - is_flow, - w_id - ) - .fetch_one(db) - .await? - .unwrap_or(0); - - #[allow(unused)] - let mut tx = db.begin().await?; - - #[cfg(feature = "http_trigger")] - let http_routes_count = { - use crate::triggers::http::handler::HttpTrigger; - let count = HttpTrigger - .trigger_count(&mut tx, w_id, is_flow, path) - .await; - count - }; - #[cfg(not(feature = "http_trigger"))] - let http_routes_count = 0; - - #[cfg(feature = "websocket")] - let websocket_count = { - use crate::triggers::websocket::WebsocketTrigger; - let count = WebsocketTrigger - .trigger_count(&mut tx, w_id, is_flow, path) - .await; - count - }; - #[cfg(not(feature = "websocket"))] - let websocket_count = 0; - - #[cfg(all(feature = "kafka", feature = "enterprise", feature = "private"))] - let kafka_count = { - use crate::triggers::kafka::KafkaTrigger; - let count = KafkaTrigger - .trigger_count(&mut tx, w_id, is_flow, path) - .await; - count - }; - #[cfg(not(all(feature = "kafka", feature = "enterprise", feature = "private")))] - let kafka_count = 0; - - #[cfg(all(feature = "nats", feature = "enterprise", feature = "private"))] - let nats_count = { - use crate::triggers::nats::NatsTrigger; - let count = NatsTrigger - .trigger_count(&mut tx, w_id, is_flow, path) - .await; - count - }; - #[cfg(not(all(feature = "nats", feature = "enterprise", feature = "private")))] - let nats_count = 0; - - #[cfg(feature = "postgres_trigger")] - let postgres_count = { - use crate::triggers::postgres::PostgresTrigger; - let count = PostgresTrigger - .trigger_count(&mut tx, w_id, is_flow, path) - .await; - count - }; - #[cfg(not(feature = "postgres_trigger"))] - let postgres_count = 0; - - #[cfg(feature = "mqtt_trigger")] - let mqtt_count = { - use crate::triggers::mqtt::MqttTrigger; - let count = MqttTrigger - .trigger_count(&mut tx, w_id, is_flow, path) - .await; - count - }; - #[cfg(not(feature = "mqtt_trigger"))] - let mqtt_count = 0; - - #[cfg(all(feature = "sqs_trigger", feature = "enterprise", feature = "private"))] - let sqs_count = { - use crate::triggers::sqs::SqsTrigger; - let count = SqsTrigger.trigger_count(&mut tx, w_id, is_flow, path).await; - count - }; - #[cfg(not(all(feature = "sqs_trigger", feature = "enterprise", feature = "private")))] - let sqs_count = 0; - - #[cfg(all(feature = "gcp_trigger", feature = "enterprise", feature = "private"))] - let gcp_count = { - use crate::triggers::gcp::GcpTrigger; - let count = GcpTrigger.trigger_count(&mut tx, w_id, is_flow, path).await; - count - }; - #[cfg(not(all(feature = "gcp_trigger", feature = "enterprise", feature = "private")))] - let gcp_count = 0; - - #[cfg(all(feature = "smtp", feature = "enterprise", feature = "private"))] - let email_count = { - use crate::triggers::email::EmailTrigger; - let count = EmailTrigger - .trigger_count(&mut tx, w_id, is_flow, path) - .await; - count - }; - #[cfg(not(all(feature = "smtp", feature = "enterprise", feature = "private")))] - let email_count = 0; - - tx.commit().await?; - - let webhook_count = (if is_flow { - sqlx::query_scalar!( - "SELECT COUNT(*) FROM token WHERE label LIKE 'webhook-%' AND workspace_id = $1 AND scopes @> ARRAY['run:flow/' || $2]::text[]", - w_id, - path, - ) - } else { - sqlx::query_scalar!( - "SELECT COUNT(*) FROM token WHERE label LIKE 'webhook-%' AND workspace_id = $1 AND scopes @> ARRAY['run:' || $2]::text[]", - w_id, - path, - ) - }).fetch_one(db) - .await? - .unwrap_or(0); - - let default_email_count = (if is_flow { - sqlx::query_scalar!( - "SELECT COUNT(*) FROM token WHERE label LIKE 'email-%' AND workspace_id = $1 AND scopes @> ARRAY['run:flow/' || $2]::text[]", - w_id, - path, - ) - } else { - sqlx::query_scalar!( - "SELECT COUNT(*) FROM token WHERE label LIKE 'email-%' AND workspace_id = $1 AND scopes @> ARRAY['run:script/' || $2]::text[]", - w_id, - path, - ) - }).fetch_one(db) - .await? - .unwrap_or(0); - - let nextcloud_count = sqlx::query_scalar!( - "SELECT COUNT(*) FROM native_trigger WHERE workspace_id = $1 AND script_path = $2 AND is_flow = $3 AND service_name = 'nextcloud'", - w_id, - path, - is_flow, - ) - .fetch_one(db) - .await? - .unwrap_or(0); - - Ok(Json(TriggersCount { - primary_schedule: primary_schedule.map(|s| TriggerPrimarySchedule { schedule: s }), - schedule_count, - http_routes_count, - webhook_count, - default_email_count, - email_count, - websocket_count, - kafka_count, - nats_count, - postgres_count, - mqtt_count, - gcp_count, - sqs_count, - nextcloud_count, - })) -} diff --git a/backend/windmill-api/src/triggers/http/handler.rs b/backend/windmill-api/src/triggers/http/handler.rs deleted file mode 100644 index a38532c46d..0000000000 --- a/backend/windmill-api/src/triggers/http/handler.rs +++ /dev/null @@ -1,1185 +0,0 @@ -use super::{ - http_trigger_args::RawHttpTriggerArgs, AuthenticationMethod, HttpMethod, RequestType, - TriggerRoute, HTTP_ACCESS_CACHE, HTTP_AUTH_CACHE, HTTP_ROUTERS_CACHE, -}; -use crate::{ - auth::{AuthCache, OptTokened}, - db::{ApiAuthed, DB}, - jobs::start_job_update_sse_stream, - resources::try_get_resource_from_db_as, - triggers::{ - http::{ - refresh_routers, validate_authentication_method, HttpConfig, HttpConfigRequest, - RouteExists, ROUTE_PATH_KEY_RE, VALID_ROUTE_PATH_RE, - }, - trigger_helpers::{ - get_runnable_format, trigger_runnable, trigger_runnable_and_wait_for_result, - trigger_runnable_inner, RunnableId, - }, - Trigger, TriggerCrud, TriggerData, TriggerMode, - }, - users::fetch_api_authed, - utils::{check_scopes, ExpiringCacheEntry}, -}; -use axum::{ - async_trait, - extract::Path, - response::{IntoResponse, Response}, - routing::{get, post}, - Extension, Json, Router, -}; -use futures::StreamExt; -use http::{HeaderMap, StatusCode}; -use sqlx::PgConnection; -use std::{ - borrow::Cow, - collections::{HashMap, HashSet}, - sync::Arc, -}; -use windmill_audit::{audit_oss::audit_log, ActionKind}; -use windmill_common::{ - db::UserDB, - error::{Error, Result}, - jobs::JobTriggerKind, - triggers::{TriggerKind, TriggerMetadata}, - utils::{not_found_if_none, require_admin, StripPath}, - worker::CLOUD_HOSTED, -}; -use windmill_git_sync::handle_deployment_metadata; - -#[cfg(feature = "parquet")] -use { - crate::job_helpers_oss::get_workspace_s3_resource, - windmill_common::s3_helpers::build_object_store_client, -}; - -use windmill_git_sync::DeployedObject; - -pub async fn increase_trigger_version(tx: &mut PgConnection) -> Result<()> { - sqlx::query!("SELECT nextval('http_trigger_version_seq')") - .fetch_one(tx) - .await?; - Ok(()) -} - -pub fn generate_route_path_key(route_path: &str) -> String { - ROUTE_PATH_KEY_RE - .replace_all(route_path, "${1}${2}key") - .to_string() -} - -pub async fn route_path_key_exists( - route_path_key: &str, - http_method: &HttpMethod, - w_id: &str, - trigger_path: Option<&str>, - workspaced_route: Option, - db: &DB, -) -> Result { - let exists = if *CLOUD_HOSTED { - sqlx::query_scalar!( - r#" - SELECT EXISTS( - SELECT 1 - FROM http_trigger - WHERE - route_path_key = $1 - AND workspace_id = $2 - AND http_method = $3 - AND ($4::TEXT IS NULL OR path != $4) - ) - "#, - &route_path_key, - w_id, - http_method as &HttpMethod, - trigger_path - ) - .fetch_one(db) - .await? - .unwrap_or(false) - } else { - let route_path_key = match workspaced_route { - Some(true) => Cow::Owned(format!("{}/{}", w_id, route_path_key.trim_matches('/'))), - _ => Cow::Borrowed(route_path_key), - }; - - sqlx::query_scalar!( - r#" - SELECT EXISTS( - SELECT 1 - FROM http_trigger - WHERE - ((workspaced_route IS TRUE AND workspace_id || '/' || route_path_key = $1) - OR (workspaced_route IS FALSE AND route_path_key = $1)) - AND http_method = $2 - AND ($3::TEXT IS NULL OR path != $3) - ) - "#, - &route_path_key, - http_method as &HttpMethod, - trigger_path - ) - .fetch_one(db) - .await? - .unwrap_or(false) - }; - - Ok(exists) -} - -pub async fn exists_route( - Extension(db): Extension, - Path(w_id): Path, - Json(RouteExists { route_path, http_method, trigger_path, workspaced_route }): Json< - RouteExists, - >, -) -> Result> { - let route_path_key = generate_route_path_key(&route_path); - - let exists = route_path_key_exists( - &route_path_key, - &http_method, - &w_id, - trigger_path.as_deref(), - workspaced_route, - &db, - ) - .await?; - - Ok(Json(exists)) -} - -fn check_no_duplicates<'trigger>( - new_http_triggers: &[TriggerData], - route_path_key: &[String], -) -> Result<()> { - let mut seen = HashSet::with_capacity(new_http_triggers.len()); - - for (i, trigger) in new_http_triggers.iter().enumerate() { - if !seen.insert(( - &route_path_key[i], - trigger.config.http_method, - trigger.config.workspaced_route, - )) { - return Err(Error::BadRequest(format!( - "Duplicate HTTP route detected: '{}'. Each HTTP route must have a unique 'route_path'.", - &trigger.config.route_path - ))); - } - } - - Ok(()) -} - -pub async fn insert_new_trigger_into_db( - authed: &ApiAuthed, - tx: &mut PgConnection, - w_id: &str, - trigger: &TriggerData, - route_path_key: &str, -) -> Result<()> { - require_admin(authed.is_admin, &authed.username)?; - - let request_type = trigger.config.request_type; - - sqlx::query!( - r#" - INSERT INTO http_trigger ( - workspace_id, - path, - route_path, - route_path_key, - workspaced_route, - authentication_resource_path, - wrap_body, - raw_string, - script_path, - summary, - description, - is_flow, - mode, - request_type, - authentication_method, - http_method, - static_asset_config, - edited_by, - email, - edited_at, - is_static_website, - error_handler_path, - error_handler_args, - retry - ) - VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, now(), $20, $21, $22, $23 - ) - "#, - w_id, - trigger.base.path, - trigger.config.route_path, - route_path_key, - trigger.config.workspaced_route.unwrap_or(false), - trigger.config.authentication_resource_path, - trigger.config.wrap_body.unwrap_or(false), - trigger.config.raw_string.unwrap_or(false), - trigger.base.script_path, - trigger.config.summary, - trigger.config.description, - trigger.base.is_flow, - trigger.base.mode() as _, - request_type as _, - trigger.config.authentication_method as _, - trigger.config.http_method as _, - trigger.config.static_asset_config as _, - &authed.username, - &authed.email, - trigger.config.is_static_website, - trigger.error_handling.error_handler_path, - trigger.error_handling.error_handler_args as _, - trigger.error_handling.retry as _ - ) - .execute(&mut *tx) - .await?; - Ok(()) -} - -pub async fn create_many_http_triggers( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path(w_id): Path, - Json(new_http_triggers): Json>>, -) -> Result<(StatusCode, String)> { - require_admin(authed.is_admin, &authed.username)?; - - let handler = HttpTrigger; - - let error_wrapper = |route_path: &str, error: Error| -> Error { - anyhow::anyhow!( - "Error occurred for HTTP route at route path: {}, error: {}", - route_path, - error - ) - .into() - }; - - let mut route_path_keys = Vec::with_capacity(new_http_triggers.len()); - - for new_http_trigger in new_http_triggers.iter() { - handler - .validate_new(&db, &w_id, &new_http_trigger.config) - .await - .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err))?; - - let route_path_key = - check_if_route_exist(&db, &new_http_trigger.config, &w_id, None).await?; - - route_path_keys.push(route_path_key.clone()); - } - - check_no_duplicates(&new_http_triggers, &route_path_keys)?; - - let mut tx = user_db.begin(&authed).await?; - - for (new_http_trigger, route_path_key) in new_http_triggers.iter().zip(route_path_keys.iter()) { - insert_new_trigger_into_db(&authed, &mut tx, &w_id, new_http_trigger, route_path_key) - .await - .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err))?; - - audit_log( - &mut *tx, - &authed, - "http_trigger.create", - ActionKind::Create, - &w_id, - Some(&new_http_trigger.base.path), - None, - ) - .await - .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err.into()))?; - - increase_trigger_version(&mut tx) - .await - .map_err(|err| error_wrapper(&new_http_trigger.config.route_path, err.into()))?; - } - - tx.commit().await?; - - for http_trigger in new_http_triggers.into_iter() { - handle_deployment_metadata( - &authed.email, - &authed.username, - &db, - &w_id, - windmill_git_sync::DeployedObject::HttpTrigger { path: http_trigger.base.path.clone() }, - Some(format!("HTTP trigger '{}' created", http_trigger.base.path)), - true, - None, - ) - .await - .map_err(|err| error_wrapper(&http_trigger.config.route_path, err.into()))?; - } - Ok((StatusCode::CREATED, "Created all HTTP routes".to_string())) -} - -async fn check_if_route_exist( - db: &DB, - config: &HttpConfigRequest, - workspace_id: &str, - trigger_path: Option<&str>, -) -> Result { - let route_path_key = generate_route_path_key(&config.route_path); - - let exists = route_path_key_exists( - &route_path_key, - &config.http_method, - workspace_id, - trigger_path, - config.workspaced_route, - db, - ) - .await?; - - if exists { - return Err(Error::BadRequest( - "A route already exists with this path".to_string(), - )); - } - - Ok(route_path_key) -} - -pub struct HttpTrigger; - -#[async_trait] -impl TriggerCrud for HttpTrigger { - type TriggerConfig = HttpConfig; - type Trigger = Trigger; - type TriggerConfigRequest = HttpConfigRequest; - type TestConnectionConfig = (); - - const TABLE_NAME: &'static str = "http_trigger"; - const TRIGGER_TYPE: &'static str = "http"; - const SUPPORTS_SERVER_STATE: bool = false; - const SUPPORTS_TEST_CONNECTION: bool = false; - const ROUTE_PREFIX: &'static str = "/http_triggers"; - const DEPLOYMENT_NAME: &'static str = "HTTP trigger"; - const IS_ALLOWED_ON_CLOUD: bool = true; - const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[ - "route_path", - "route_path_key", - "request_type", - "authentication_method", - "http_method", - "summary", - "description", - "static_asset_config", - "is_static_website", - "authentication_resource_path", - "workspaced_route", - "wrap_body", - "raw_string", - ]; - - fn get_deployed_object(path: String) -> DeployedObject { - DeployedObject::HttpTrigger { path } - } - - fn additional_routes(&self) -> Router { - Router::new() - .route("/create_many", post(create_many_http_triggers)) - .route("/route_exists", post(exists_route)) - } - - async fn validate_new( - &self, - _db: &DB, - _workspace_id: &str, - new: &Self::TriggerConfigRequest, - ) -> Result<()> { - if *CLOUD_HOSTED && (new.is_static_website || new.static_asset_config.is_some()) { - return Err(Error::BadRequest( - "Static website and static asset are not supported on cloud".to_string(), - )); - } - - if !VALID_ROUTE_PATH_RE.is_match(&new.route_path) { - return Err(Error::BadRequest("Invalid route path".to_string())); - } - - validate_authentication_method(new.authentication_method, new.raw_string)?; - - Ok(()) - } - - async fn validate_edit( - &self, - _db: &DB, - _workspace_id: &str, - edit: &Self::TriggerConfigRequest, - _path: &str, - ) -> Result<()> { - if *CLOUD_HOSTED && (edit.is_static_website || edit.static_asset_config.is_some()) { - return Err(Error::BadRequest( - "Static website and static asset are not supported on cloud".to_string(), - )); - } - - validate_authentication_method(edit.authentication_method, edit.raw_string)?; - - Ok(()) - } - - async fn create_trigger( - &self, - db: &DB, - tx: &mut PgConnection, - authed: &ApiAuthed, - w_id: &str, - trigger: TriggerData, - ) -> Result<()> { - let route_path_key = check_if_route_exist(db, &trigger.config, &w_id, None).await?; - - insert_new_trigger_into_db(authed, tx, w_id, &trigger, &route_path_key).await?; - - increase_trigger_version(tx).await?; - - Ok(()) - } - - async fn update_trigger( - &self, - db: &DB, - tx: &mut PgConnection, - authed: &ApiAuthed, - workspace_id: &str, - path: &str, - trigger: TriggerData, - ) -> Result<()> { - if authed.is_admin { - if trigger.config.route_path.is_empty() { - return Err(Error::BadRequest("route_path is required".to_string())); - }; - - let route_path = &trigger.config.route_path; - if !VALID_ROUTE_PATH_RE.is_match(route_path) { - return Err(Error::BadRequest("Invalid route path".to_string())); - } - - let route_path_key = - check_if_route_exist(db, &trigger.config, workspace_id, Some(path)).await?; - - let request_type = trigger.config.request_type; - - sqlx::query!( - r#" - UPDATE - http_trigger - SET - route_path = $1, - route_path_key = $2, - workspaced_route = $3, - wrap_body = $4, - raw_string = $5, - authentication_resource_path = $6, - script_path = $7, - path = $8, - is_flow = $9, - mode = $10, - http_method = $11, - static_asset_config = $12, - edited_by = $13, - email = $14, - request_type = $15, - authentication_method = $16, - summary = $17, - description = $18, - edited_at = now(), - is_static_website = $19, - error_handler_path = $20, - error_handler_args = $21, - retry = $22 - WHERE - workspace_id = $23 AND - path = $24 - "#, - route_path, - &route_path_key, - trigger.config.workspaced_route, - trigger.config.wrap_body, - trigger.config.raw_string, - trigger.config.authentication_resource_path, - trigger.base.script_path, - trigger.base.path, - trigger.base.is_flow, - trigger.base.mode() as _, - trigger.config.http_method as _, - trigger.config.static_asset_config as _, - &authed.username, - &authed.email, - request_type as _, - trigger.config.authentication_method as _, - trigger.config.summary, - trigger.config.description, - trigger.config.is_static_website, - trigger.error_handling.error_handler_path, - trigger.error_handling.error_handler_args as _, - trigger.error_handling.retry as _, - workspace_id, - path, - ) - .execute(&mut *tx) - .await?; - } else { - let request_type = trigger.config.request_type; - - sqlx::query!( - r#" - UPDATE - http_trigger - SET - wrap_body = $1, - raw_string = $2, - authentication_resource_path = $3, - script_path = $4, - path = $5, - is_flow = $6, - mode = $7, - http_method = $8, - static_asset_config = $9, - edited_by = $10, - email = $11, - request_type = $12, - authentication_method = $13, - summary = $14, - description = $15, - edited_at = now(), - is_static_website = $16, - error_handler_path = $17, - error_handler_args = $18, - retry = $19 - WHERE - workspace_id = $20 AND - path = $21 - "#, - trigger.config.wrap_body, - trigger.config.raw_string, - trigger.config.authentication_resource_path, - trigger.base.script_path, - trigger.base.path, - trigger.base.is_flow, - trigger.base.mode() as _, - trigger.config.http_method as _, - trigger.config.static_asset_config as _, - &authed.username, - &authed.email, - request_type as _, - trigger.config.authentication_method as _, - trigger.config.summary, - trigger.config.description, - trigger.config.is_static_website, - trigger.error_handling.error_handler_path, - trigger.error_handling.error_handler_args as _, - trigger.error_handling.retry as _, - workspace_id, - path, - ) - .execute(&mut *tx) - .await?; - } - - increase_trigger_version(tx).await?; - - Ok(()) - } - - async fn set_trigger_mode_extra_action(&self, tx: &mut PgConnection) -> Result<()> { - increase_trigger_version(tx).await - } - - async fn delete_by_path( - &self, - tx: &mut PgConnection, - workspace_id: &str, - path: &str, - ) -> Result { - let deleted = sqlx::query(&format!( - "DELETE FROM {} WHERE workspace_id = $1 AND path = $2", - Self::TABLE_NAME - )) - .bind(workspace_id) - .bind(path) - .execute(&mut *tx) - .await? - .rows_affected(); - - increase_trigger_version(tx).await?; - - Ok(deleted > 0) - } -} - -async fn conditional_cors_middleware( - req: axum::extract::Request, - next: axum::middleware::Next, -) -> Response { - let mut response = next.run(req).await; - - let headers = response.headers_mut(); - - // Check existing headers first to determine what not to insert - let mut not_insert_origin = false; - let mut not_insert_methods = false; - let mut not_insert_headers = false; - - for key in headers.keys() { - if !not_insert_origin && key == http::header::ACCESS_CONTROL_ALLOW_ORIGIN { - not_insert_origin = true; - } - if !not_insert_methods && key == http::header::ACCESS_CONTROL_ALLOW_METHODS { - not_insert_methods = true; - } - if !not_insert_headers && key == http::header::ACCESS_CONTROL_ALLOW_HEADERS { - not_insert_headers = true; - } - - // Early exit if all headers are already present - if not_insert_origin && not_insert_methods && not_insert_headers { - break; - } - } - - // Insert only the missing headers - if !not_insert_origin { - headers.insert( - http::header::ACCESS_CONTROL_ALLOW_ORIGIN, - http::HeaderValue::from_static("*"), - ); - } - - if !not_insert_methods { - headers.insert( - http::header::ACCESS_CONTROL_ALLOW_METHODS, - http::HeaderValue::from_static("GET, POST, PUT, DELETE, PATCH, HEAD, OPTIONS"), - ); - } - - if !not_insert_headers { - headers.insert( - http::header::ACCESS_CONTROL_ALLOW_HEADERS, - http::HeaderValue::from_static("content-type, authorization"), - ); - } - - response -} - -pub fn http_route_trigger_handler() -> Router { - Router::new() - .route( - "/*path", - get(route_job) - .post(route_job) - .delete(route_job) - .put(route_job) - .patch(route_job) - .head(|| async { "" }) - .options(|| async { "" }), - ) - .layer(axum::middleware::from_fn(conditional_cors_middleware)) -} - -async fn get_http_route_trigger( - route_path: &str, - auth_cache: &Arc, - token: Option<&String>, - db: &DB, - user_db: UserDB, - method: &http::Method, -) -> Result<(TriggerRoute, String, HashMap, ApiAuthed)> { - let http_method: HttpMethod = method.try_into()?; - - let requested_path = format!("/{}", route_path); - - let routers_cache = HTTP_ROUTERS_CACHE.read().await; - - let routers_cache = if routers_cache.routers.is_empty() { - tracing::warn!("HTTP routers are not loaded, loading from db"); - let (_, routers_cache) = refresh_routers(db).await?; - routers_cache - } else { - routers_cache - }; - - let router = routers_cache - .routers - .get(&http_method) - .ok_or(Error::internal_err( - "HTTP routers could not be loaded".to_string(), - ))?; - - let trigger_match = router.at(requested_path.as_str()).ok(); - - let matchit::Match { value: trigger, params } = - not_found_if_none(trigger_match, "Trigger", requested_path.as_str())?; - - let params: HashMap = params - .iter() - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect(); - - let username_override = if let AuthenticationMethod::Windmill = trigger.authentication_method { - let opt_authed = if let Some(token) = token { - auth_cache - .get_authed(Some(trigger.workspace_id.clone()), token) - .await - } else { - None - }; - if let Some(authed) = opt_authed { - check_scopes(&authed, || format!("http_triggers:read:{}", &trigger.path))?; - - // check that the user has access to the trigger - let cache_key = ( - trigger.workspace_id.clone(), - trigger.path.clone(), - authed.clone(), - ); - let exists = match HTTP_ACCESS_CACHE.get(&cache_key) { - Some(cache_entry) if cache_entry.expiry > std::time::Instant::now() => { - tracing::debug!("HTTP access cache hit for route {}", trigger.path); - true - } - _ => { - tracing::debug!("HTTP access cache miss for route {}", trigger.path); - let mut tx = user_db.begin(&authed).await?; - let exists = sqlx::query_scalar!( - r#" - SELECT EXISTS( - SELECT 1 - FROM - http_trigger - WHERE - workspace_id = $1 AND - path = $2 - ) - "#, - trigger.workspace_id, - trigger.path - ) - .fetch_one(&mut *tx) - .await? - .unwrap_or(false); - if exists { - HTTP_ACCESS_CACHE.insert( - cache_key, - ExpiringCacheEntry { - value: (), - expiry: std::time::Instant::now() - + std::time::Duration::from_secs(10), - }, - ); - } - exists - } - }; - if exists { - Some(authed.display_username().to_owned()) - } else { - return Err(Error::NotAuthorized("Unauthorized".to_string())); - } - } else { - return Err(Error::NotAuthorized("Requires authentication".to_string())); - } - } else { - None - }; - - let authed = fetch_api_authed( - trigger.edited_by.clone(), - trigger.email.clone(), - &trigger.workspace_id, - &db, - Some(username_override.unwrap_or(format!("HTTP-{}", trigger.path))), - ) - .await?; - - Ok((trigger.clone(), route_path.to_string(), params, authed)) -} - -async fn route_job( - Extension(db): Extension, - Extension(user_db): Extension, - Extension(auth_cache): Extension>, - OptTokened { token }: OptTokened, - Path(route_path): Path, - headers: HeaderMap, - args: RawHttpTriggerArgs, -) -> std::result::Result { - let route_path = route_path.to_path().trim_end_matches("/"); - - let (trigger, called_path, params, authed) = get_http_route_trigger( - route_path, - &auth_cache, - token.as_ref(), - &db, - user_db.clone(), - &args.0.metadata.method, - ) - .await - .map_err(|e| e.into_response())?; - - if trigger.script_path.is_empty() && trigger.static_asset_config.is_none() { - return Err(Error::NotFound(format!( - "Runnable path of HTTP route at path: {}", - trigger.path - )) - .into_response()); - } - - let args = args - .process_args( - &authed, - &db, - &trigger.workspace_id, - match trigger.authentication_method { - AuthenticationMethod::CustomScript | AuthenticationMethod::Signature => true, - _ => trigger.raw_string, - }, - ) - .await - .map_err(|e| e.into_response())?; - - match trigger.authentication_method { - AuthenticationMethod::None - | AuthenticationMethod::Windmill - | AuthenticationMethod::CustomScript => {} - _ => { - let resource_path = match trigger.authentication_resource_path { - Some(resource_path) => resource_path, - None => { - return Err(Error::BadRequest( - "Missing authentication resource path".to_string(), - ) - .into_response()) - } - }; - - let cache_key = ( - trigger.workspace_id.clone(), - resource_path.clone(), - authed.clone(), - ); - - let authentication_method = match HTTP_AUTH_CACHE.get(&cache_key) { - Some(cache_entry) if cache_entry.expiry > std::time::Instant::now() => { - tracing::debug!("HTTP auth method cache hit for route {}", trigger.path); - cache_entry.value - } - _ => { - tracing::debug!("HTTP auth method cache miss for route {}", trigger.path); - let auth_method = try_get_resource_from_db_as::< - super::http_trigger_auth::AuthenticationMethod, - >( - &authed, - Some(user_db.clone()), - &db, - &resource_path, - &trigger.workspace_id, - ) - .await - .map_err(|e| e.into_response())?; - HTTP_AUTH_CACHE.insert( - cache_key, - ExpiringCacheEntry { - value: auth_method.clone(), - expiry: std::time::Instant::now() + std::time::Duration::from_secs(60), - }, - ); - auth_method - } - }; - - let raw_payload = args.0.metadata.raw_string.as_ref(); - - let response = authentication_method - .authenticate_http_request(&headers, raw_payload) - .map_err(|e| e.into_response())?; - - if let Some(response) = response { - return Ok(response); - } - } - } - - #[cfg(not(feature = "parquet"))] - if trigger.static_asset_config.is_some() { - return Err(Error::internal_err( - "Static asset configuration is not supported in this build".to_string(), - ) - .into_response()); - } - - #[cfg(feature = "parquet")] - if let Some(sqlx::types::Json(config)) = trigger.static_asset_config { - let build_static_response_f = async { - let (_, s3_resource_opt) = get_workspace_s3_resource( - &authed, - &db, - None, - &trigger.workspace_id, - config.storage, - ) - .await?; - let s3_resource = s3_resource_opt.ok_or(Error::internal_err( - "No files storage resource defined at the workspace level".to_string(), - ))?; - let s3_client = build_object_store_client(&s3_resource).await?; - - let path = if trigger.is_static_website { - let subpath = params - .get("wm_subpath") - .cloned() - .unwrap_or("index.html".to_string()); - tracing::info!("subpath: {}", subpath); - format!("{}/{}", config.s3.trim_end_matches('/'), subpath) - } else { - config.s3.clone() - }; - let path = object_store::path::Path::from(path); - let s3_object = s3_client.get(&path).await; - - let s3_object = match s3_object { - Err(object_store::Error::NotFound { .. }) if trigger.is_static_website => { - // fallback to index.html if the file is not found - let path = object_store::path::Path::from(format!( - "{}/index.html", - config.s3.trim_end_matches('/') - )); - s3_client.get(&path).await - } - r => r, - }; - - let s3_object = s3_object.map_err(|err| { - tracing::warn!("Error retrieving file from S3: {:?}", err); - let mut msg = format!("Error retrieving file: {err}"); - let mut source = std::error::Error::source(&err); - while let Some(e) = source { - msg.push_str(&format!("\n caused by: {e}")); - source = e.source(); - } - Error::internal_err(msg) - })?; - - let mut response_headers = http::HeaderMap::new(); - if let Some(ref e_tag) = s3_object.meta.e_tag { - if let Some(if_none_match) = headers.get(http::header::IF_NONE_MATCH) { - if if_none_match == e_tag { - return Ok::<_, Error>(( - StatusCode::NOT_MODIFIED, - response_headers, - axum::body::Body::empty(), - )); - } - } - if let Ok(e_tag) = e_tag.parse() { - response_headers.insert("etag", e_tag); - } - } - response_headers.insert( - "content-type", - s3_object - .attributes - .get(&object_store::Attribute::ContentType) - .map(|s| s.parse().ok()) - .flatten() - .unwrap_or("application/octet-stream".parse().unwrap()), - ); - if !trigger.is_static_website { - response_headers.insert( - "content-disposition", - config.filename.as_ref().map_or_else( - || { - s3_object - .attributes - .get(&object_store::Attribute::ContentDisposition) - .map(|s| s.parse().ok()) - .flatten() - .unwrap_or("inline".parse().unwrap()) - }, - |filename| { - format!("inline; filename=\"{}\"", filename) - .parse() - .unwrap_or("inline".parse().unwrap()) - }, - ), - ); - } - - let body_stream = axum::body::Body::from_stream(s3_object.into_stream()); - Ok::<_, Error>((StatusCode::OK, response_headers, body_stream)) - }; - match build_static_response_f.await { - Ok((status, headers, body_stream)) => { - return Ok((status, headers, body_stream).into_response()) - } - Err(e) => return Err(e.into_response()), - } - } - - let runnable_format = get_runnable_format( - if trigger.is_flow { - RunnableId::from_flow_path(&trigger.script_path) - } else { - RunnableId::from_script_path(&trigger.script_path) - }, - &trigger.workspace_id, - &db, - &TriggerKind::Http, - ) - .await - .map_err(|e| e.into_response())?; - - let args = args - .to_args_from_format( - &trigger.route_path, - &called_path, - ¶ms, - runnable_format, - trigger.wrap_body, - ) - .map_err(|e| e.into_response())?; - - let trigger_info = TriggerMetadata::new(Some(trigger.path.clone()), JobTriggerKind::Http); - if trigger.mode == TriggerMode::Suspended { - let _ = trigger_runnable( - &db, - Some(user_db), - authed, - &trigger.workspace_id, - &trigger.script_path, - trigger.is_flow, - args, - trigger.retry.as_ref(), - trigger.error_handler_path.as_deref(), - trigger.error_handler_args.as_ref(), - format!("http_trigger/{}", trigger.path), - None, - true, - trigger_info, - ) - .await - .map_err(|e| e.into_response())?; - - return Ok(( - StatusCode::OK, - format!( - "Trigger {} is in suspended mode, jobs are added to the queue but suspended", - &trigger.path - ), - ) - .into_response()); - } - - // Handle execution based on the execution mode - match trigger.request_type { - RequestType::SyncSse => { - // Trigger the job (always async when streaming) - let (uuid, _, _, _) = trigger_runnable_inner( - &db, - None, - Some(user_db.clone()), - authed.clone(), - &trigger.workspace_id, - &trigger.script_path, - trigger.is_flow, - args, - trigger.retry.as_ref(), - trigger.error_handler_path.as_deref(), - trigger.error_handler_args.as_ref(), - format!("http_trigger/{}", trigger.path), - None, - trigger_info, - None, - ) - .await - .map_err(|e| e.into_response())?; - - // Set up SSE stream - let opt_authed = Some(authed.clone()); - let opt_tokened = OptTokened { token: None }; - let (tx, rx) = tokio::sync::mpsc::channel(32); - - let stream = tokio_stream::wrappers::ReceiverStream::new(rx).map(|x| { - format!( - "data: {}\n\n", - serde_json::to_string(&x).unwrap_or_default() - ) - }); - - start_job_update_sse_stream( - opt_authed, - opt_tokened, - db.clone(), - trigger.workspace_id.clone(), - uuid, - None, - None, - None, - None, - Some(true), - Some(true), - None, - None, - tx, - None, - ); - - let body = axum::body::Body::from_stream( - stream.map(std::result::Result::<_, std::convert::Infallible>::Ok), - ); - - Ok(Response::builder() - .status(200) - .header("Content-Type", "text/event-stream") - .header("Cache-Control", "no-cache") - .body(body) - .map_err(|e| Error::internal_err(e.to_string()).into_response())?) - } - RequestType::Async => trigger_runnable( - &db, - Some(user_db), - authed, - &trigger.workspace_id, - &trigger.script_path, - trigger.is_flow, - args, - trigger.retry.as_ref(), - trigger.error_handler_path.as_deref(), - trigger.error_handler_args.as_ref(), - format!("http_trigger/{}", trigger.path), - None, - false, - trigger_info, - ) - .await - .map_err(|e| e.into_response()), - RequestType::Sync => trigger_runnable_and_wait_for_result( - &db, - Some(user_db), - authed, - &trigger.workspace_id, - &trigger.script_path, - trigger.is_flow, - args, - trigger.retry.as_ref(), - trigger.error_handler_path.as_deref(), - trigger.error_handler_args.as_ref(), - format!("http_trigger/{}", trigger.path), - trigger_info, - ) - .await - .map_err(|e| e.into_response()), - } -} diff --git a/backend/windmill-api/src/triggers/http/http_trigger_args.rs b/backend/windmill-api/src/triggers/http/http_trigger_args.rs deleted file mode 100644 index af829cec53..0000000000 --- a/backend/windmill-api/src/triggers/http/http_trigger_args.rs +++ /dev/null @@ -1,225 +0,0 @@ -use std::collections::HashMap; - -use axum::{ - extract::{FromRequest, Request}, - response::Response, -}; -use serde::{Deserialize, Serialize}; -use serde_json::value::RawValue; -use windmill_common::{ - error::Error, - triggers::{RunnableFormat, RunnableFormatVersion}, - worker::to_raw_value, - DB, -}; -use windmill_queue::PushArgsOwned; - -use crate::{ - args::{ - build_headers, build_query, try_from_request_body, Body, RawWebhookArgs, WebhookArgs, - WebhookArgsMetadata, - }, - db::ApiAuthed, -}; - -pub struct RawHttpTriggerArgs(pub RawWebhookArgs); - -#[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Copy, Hash, Eq, PartialEq)] -#[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")] -#[serde(rename_all = "lowercase")] -pub enum HttpMethod { - Get, - Post, - Put, - Delete, - Patch, -} - -impl TryFrom<&http::Method> for HttpMethod { - type Error = Error; - fn try_from(method: &http::Method) -> Result { - match method { - &http::Method::GET => Ok(HttpMethod::Get), - &http::Method::POST => Ok(HttpMethod::Post), - &http::Method::PUT => Ok(HttpMethod::Put), - &http::Method::DELETE => Ok(HttpMethod::Delete), - &http::Method::PATCH => Ok(HttpMethod::Patch), - _ => Err(Error::BadRequest("Invalid HTTP method".to_string())), - } - } -} - -#[axum::async_trait] -impl FromRequest for RawHttpTriggerArgs -where - S: Send + Sync, -{ - type Rejection = Response; - - async fn from_request(request: Request, _state: &S) -> Result { - let args = try_from_request_body(request, _state, true).await?; - - Ok(Self(args)) - } -} - -#[derive(Debug, Clone)] -pub struct HttpTriggerArgs(pub WebhookArgs); - -impl RawHttpTriggerArgs { - pub async fn process_args( - self, - authed: &ApiAuthed, - db: &DB, - w_id: &str, - use_raw: bool, - ) -> Result { - if self.0.metadata.query_use_raw || self.0.metadata.query_wrap_body { - return Err(Error::BadRequest( - "Specifying use raw or wrap body with query args is not supported anymore on http routes, please set it in the trigger config".to_string(), - ) - .into()); - } - - let args = self.0.process_args(authed, db, w_id, Some(use_raw)).await?; - - Ok(HttpTriggerArgs(args)) - } -} - -#[derive(Serialize)] -struct HttpTriggerPreprocessorEvent<'a> { - kind: String, - route: &'a str, - path: &'a str, - body: Box, - raw_string: Option, - params: &'a HashMap, - headers: HashMap>, - query: HashMap>, - method: HttpMethod, -} - -#[derive(Serialize)] -struct HttpTriggerWmTrigger<'a> { - route: &'a str, - path: &'a str, - params: &'a HashMap, - query: HashMap>, - headers: HashMap>, - method: HttpMethod, -} - -impl HttpTriggerArgs { - pub fn to_main_args(self, wrap_body: bool) -> Result { - let mut extra = HashMap::new(); - - let WebhookArgsMetadata { raw_string, .. } = self.0.metadata; - - if let Some(raw_string) = raw_string { - extra.insert("raw_string".to_string(), to_raw_value(&raw_string)); - } - - let extra = if extra.is_empty() { None } else { Some(extra) }; - - match self.0.body { - Body::HashMap(mut body) => { - if wrap_body { - body = HashMap::from([("body".to_string(), to_raw_value(&body))]); - } - Ok(PushArgsOwned { args: body, extra }) - } - Body::NoHashMap(args) => { - let mut hm = HashMap::new(); - hm.insert("body".to_string(), args); - Ok(PushArgsOwned { args: hm, extra }) - } - } - } - - pub fn to_args_from_format( - self, - route_path: &str, - called_path: &str, - params: &HashMap, - format: RunnableFormat, - wrap_body: bool, - ) -> Result { - let headers = build_headers(&self.0.metadata.headers, None, true); - let query = build_query(self.0.metadata.query.as_deref(), None, true); - match format { - RunnableFormat { has_preprocessor: true, version: RunnableFormatVersion::V2 } => { - // we don't care about wrap_body in v2 - self.to_v2_preprocessor_args(route_path, called_path, params, headers, query) - } - RunnableFormat { has_preprocessor: true, version: RunnableFormatVersion::V1 } => self - .to_v1_preprocessor_args( - route_path, - called_path, - params, - wrap_body, - headers, - query, - ), - RunnableFormat { has_preprocessor: false, .. } => self.to_main_args(wrap_body), - } - } - - fn to_v1_preprocessor_args( - self, - route_path: &str, - called_path: &str, - params: &HashMap, - wrap_body: bool, - headers: HashMap>, - query: HashMap>, - ) -> Result { - let mut extra = HashMap::new(); - let mut wm_trigger = HashMap::new(); - wm_trigger.insert("kind".to_string(), to_raw_value(&"http".to_string())); - wm_trigger.insert( - "http".to_string(), - to_raw_value(&HttpTriggerWmTrigger { - route: route_path, - path: called_path, - method: (&self.0.metadata.method).try_into()?, - params, - query, - headers, - }), - ); - extra.insert("wm_trigger".to_string(), to_raw_value(&wm_trigger)); - - let mut args = self.to_main_args(wrap_body)?; - - args.extra.get_or_insert_default().extend(extra); - - Ok(args) - } - - pub fn to_v2_preprocessor_args( - self, - route_path: &str, - called_path: &str, - params: &HashMap, - headers: HashMap>, - query: HashMap>, - ) -> Result { - let mut args = HashMap::new(); - args.insert( - "event".to_string(), - to_raw_value(&HttpTriggerPreprocessorEvent { - kind: "http".to_string(), - body: to_raw_value(&self.0.body), - raw_string: self.0.metadata.raw_string, - headers, - query, - method: (&self.0.metadata.method).try_into()?, - route: route_path, - path: called_path, - params, - }), - ); - Ok(PushArgsOwned { args, extra: None }) - } -} diff --git a/backend/windmill-api/src/triggers/http/http_trigger_auth.rs b/backend/windmill-api/src/triggers/http/http_trigger_auth.rs deleted file mode 100644 index 85288d87b8..0000000000 --- a/backend/windmill-api/src/triggers/http/http_trigger_auth.rs +++ /dev/null @@ -1,751 +0,0 @@ -use axum::response::{IntoResponse, Response}; -use base64::{ - prelude::{BASE64_STANDARD, BASE64_URL_SAFE}, - Engine, -}; -use hmac::{Hmac, Mac}; -use http::{header, HeaderMap, HeaderValue, StatusCode}; -use itertools::Itertools; -use serde::{Deserialize, Serialize}; -use serde_json::json; -use sha1::Sha1; -use sha2::{Sha256, Sha512}; -use std::{borrow::Cow, collections::HashMap}; - -pub type HmacSha256 = Hmac; -pub type HmacSha512 = Hmac; -pub type HmacSha1 = Hmac; - -mod github { - use super::*; - pub struct Github; - - impl WebhookHandler for Github { - fn handle_challenge_request<'header>( - &self, - _: &'header HeaderMap, - _: &SignatureConfigData, - _: &str, - ) -> Result, AuthenticationError> { - Ok(None) - } - - fn get_hmac_authentication_data<'payload, 'header, 'prefix>( - &self, - headers: &'header HeaderMap, - raw_payload: &'payload str, - ) -> Result, AuthenticationError> - { - let github_secret_header = headers.try_get_webhook_header("X-Hub-Signature-256")?; - - let authentication_data = SignatureAuthenticationData::new( - Cow::Borrowed(raw_payload), - github_secret_header, - Some("sha256="), - SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), - ); - - Ok(authentication_data) - } - } -} - -mod slack { - use super::*; - pub struct Slack; - - impl WebhookHandler for Slack { - fn handle_challenge_request<'header>( - &self, - _: &'header HeaderMap, - _: &SignatureConfigData, - _: &str, - ) -> Result, AuthenticationError> { - Ok(None) - } - - fn get_hmac_authentication_data<'payload, 'header, 'prefix>( - &self, - headers: &'header HeaderMap, - raw_payload: &'payload str, - ) -> Result, AuthenticationError> - { - let slack_secret_signature = headers.try_get_webhook_header("X-Slack-Signature")?; - let slack_timestamp_header = - headers.try_get_webhook_header("X-Slack-Request-Timestamp")?; - let signed_payload = format!("v0:{}:{}", slack_timestamp_header, raw_payload); - - Ok(SignatureAuthenticationData::new( - Cow::Owned(signed_payload), - slack_secret_signature, - Some("v0="), - SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), - )) - } - } -} - -mod stripe { - use super::*; - - pub struct Stripe; - - impl WebhookHandler for Stripe { - fn handle_challenge_request<'header>( - &self, - _: &'header HeaderMap, - _: &SignatureConfigData, - _: &str, - ) -> Result, AuthenticationError> { - Ok(None) - } - - fn get_hmac_authentication_data<'payload, 'header, 'prefix>( - &self, - headers: &'header HeaderMap, - raw_payload: &'payload str, - ) -> Result, AuthenticationError> - { - let stripe_signature_header = headers.try_get_webhook_header("STRIPE-SIGNATURE")?; - - let stripe_signature = parse_signature(stripe_signature_header, (",", "=")); - - let timestamp = *stripe_signature - .get("t") - .ok_or(AuthenticationError::InvalidTimestamp)?; - let v1 = *stripe_signature - .get("v1") - .ok_or(AuthenticationError::InvalidSignature)?; - - let signed_payload = format!("{}.{}", timestamp, raw_payload); - - Ok(SignatureAuthenticationData::new( - Cow::Owned(signed_payload), - v1, - None, - SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), - )) - } - } -} - -mod tiktok { - use super::*; - - pub struct TikTok; - - impl WebhookHandler for TikTok { - fn handle_challenge_request<'header>( - &self, - _: &'header HeaderMap, - _: &SignatureConfigData, - _: &str, - ) -> Result, AuthenticationError> { - Ok(None) - } - - fn get_hmac_authentication_data<'payload, 'header, 'prefix>( - &self, - headers: &'header HeaderMap, - raw_payload: &'payload str, - ) -> Result, AuthenticationError> - { - let tiktok_secret_signature = headers.try_get_webhook_header("TikTok-Signature")?; - - let stripe_signature = parse_signature(tiktok_secret_signature, (",", "=")); - - let timestamp = *stripe_signature - .get("t") - .ok_or(AuthenticationError::InvalidTimestamp)?; - let s = *stripe_signature - .get("s") - .ok_or(AuthenticationError::InvalidSignature)?; - - let signed_payload = format!("{}.{}", timestamp, raw_payload); - - Ok(SignatureAuthenticationData::new( - Cow::Owned(signed_payload), - s, - None, - SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), - )) - } - } -} - -mod twitch { - use super::*; - use http::header; - use serde_json::value::RawValue; - #[derive(Debug, Deserialize)] - struct TwitchCrcBody { - challenge: String, - #[allow(unused)] - subscription: Box, - } - - pub struct Twitch; - - impl WebhookHandler for Twitch { - fn get_hmac_authentication_data<'payload, 'header, 'prefix>( - &self, - headers: &'header HeaderMap, - raw_payload: &'payload str, - ) -> Result, AuthenticationError> - { - let twitch_secret_signature = - headers.try_get_webhook_header("Twitch-Eventsub-Message-Signature")?; - let twitch_message_id_header = - headers.try_get_webhook_header("Twitch-Eventsub-Message-Id")?; - let twitch_timestamp_header = - headers.try_get_webhook_header("Twitch-Eventsub-Message-Timestamp")?; - - let message = format!( - "{}{}{}", - twitch_message_id_header, twitch_timestamp_header, raw_payload - ); - - Ok(SignatureAuthenticationData::new( - Cow::Owned(message), - twitch_secret_signature, - Some("sha256="), - SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), - )) - } - - fn handle_challenge_request<'header>( - &self, - headers: &'header HeaderMap, - signature_config_data: &SignatureConfigData, - raw_payload: &str, - ) -> Result, AuthenticationError> { - let authentication_data = self.get_hmac_authentication_data(headers, raw_payload)?; - verify_hmac_signature(authentication_data, &signature_config_data.secret_key)?; - - let twitch_eventsub_message_type = - headers.try_get_webhook_header("Twitch-Eventsub-Message-Type")?; - - if twitch_eventsub_message_type != "webhook_callback_verification" { - return Ok(None); - } - let twitch_crc_body = - serde_json::from_str::(raw_payload).map_err(|e| { - AuthenticationError::InvalidChallengeResponse(format!( - "Twitch :{}", - e.to_string() - )) - })?; - - let response = ( - StatusCode::OK, - [(header::CONTENT_TYPE, "text/plain")], - twitch_crc_body.challenge.to_string(), - ); - - Ok(Some(response.into_response())) - } - } -} - -mod zoom { - use axum::Json; - - use super::*; - - #[derive(Debug, Deserialize)] - struct ZoomPayload { - #[serde(rename = "plainToken")] - plain_token: String, - } - - #[derive(Debug, Deserialize)] - #[allow(unused)] - struct ZoomChallengeResponse { - payload: ZoomPayload, - event_ts: u64, - event: String, - } - - pub struct Zoom; - - impl WebhookHandler for Zoom { - fn handle_challenge_request<'header>( - &self, - _: &'header HeaderMap, - signature_config_data: &SignatureConfigData, - raw_payload: &str, - ) -> Result, AuthenticationError> { - let Ok(zoom_request_body) = serde_json::from_str::(raw_payload) - else { - return Ok(None); - }; - - if zoom_request_body.event != "endpoint.url_validation" { - return Ok(None); - } - - let hmac_signature = calculate_hmac_signature( - HmacAlgorithm::Sha256, - &signature_config_data.secret_key, - &zoom_request_body.payload.plain_token, - ); - - let encoded_hmac_signature = encode_hmac_signature(Encoding::Hex, &hmac_signature); - - let response = ( - StatusCode::OK, - Json(json!({ - "plainToken": zoom_request_body.payload.plain_token, - "encryptedToken": encoded_hmac_signature - })), - ); - - Ok(Some(response.into_response())) - } - - fn get_hmac_authentication_data<'payload, 'header, 'prefix>( - &self, - headers: &'header HeaderMap, - raw_payload: &'payload str, - ) -> Result, AuthenticationError> - { - let zoom_signature_header = headers.try_get_webhook_header("x-zm-signature")?; - let zoom_timestamp_header = headers.try_get_webhook_header("x-zm-request-timestamp")?; - - let message = format!("v0:{}:{}", zoom_timestamp_header, raw_payload); - - Ok(SignatureAuthenticationData::new( - Cow::Owned(message), - zoom_signature_header, - Some("v0="), - SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex), - )) - } - } -} - -use constant_time_eq::constant_time_eq; -use github::Github; -use slack::Slack; -use stripe::Stripe; -use tiktok::TikTok; -use twitch::Twitch; -use zoom::Zoom; - -#[derive(Debug)] -pub struct SignatureAuthenticationDetails { - pub algorithm_to_use: HmacAlgorithm, - pub header_key_encoding: Encoding, -} - -impl SignatureAuthenticationDetails { - #[inline] - fn new(algorithm_to_use: HmacAlgorithm, header_key_encoding: Encoding) -> Self { - Self { algorithm_to_use, header_key_encoding } - } -} - -fn parse_signature<'header>( - signature: &'header str, - splitters: (&str, &str), -) -> HashMap<&'header str, &'header str> { - let headers: HashMap<&str, &str> = signature - .split(splitters.0) - .map(|header| { - let mut key_and_value = header.split(splitters.1); - let key = key_and_value.next(); - let value = key_and_value.next(); - (key, value) - }) - .filter_map(|(key, value)| match (key, value) { - (Some(key), Some(value)) => Some((key, value)), - _ => None, - }) - .collect(); - headers -} - -#[derive(Debug)] -pub struct SignatureAuthenticationData<'payload, 'header, 'prefix> { - pub signed_payload: Cow<'payload, str>, - pub header_key_value: &'header str, - pub signature_prefix: Option<&'prefix str>, - pub config: SignatureAuthenticationDetails, -} - -impl<'payload, 'header, 'prefix> SignatureAuthenticationData<'payload, 'header, 'prefix> { - pub fn new( - signed_payload: Cow<'payload, str>, - header_key_value: &'header str, - signature_prefix: Option<&'prefix str>, - config: SignatureAuthenticationDetails, - ) -> Self { - Self { signed_payload, header_key_value, signature_prefix, config } - } -} - -pub trait WebhookHandler { - fn handle_challenge_request<'header>( - &self, - headers: &'header HeaderMap, - signature_config_data: &SignatureConfigData, - raw_payload: &str, - ) -> Result, AuthenticationError>; - - fn get_hmac_authentication_data<'payload, 'header, 'prefix>( - &self, - headers: &'header HeaderMap, - raw_payload: &'payload str, - ) -> Result, AuthenticationError>; -} - -#[derive(Clone, Copy, Debug, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum HmacAlgorithm { - Sha1, - Sha256, - Sha512, -} - -#[derive(Clone, Copy, Debug, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum Encoding { - Base64, - Base64Uri, - Hex, -} -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct SignatureAuthenticationMethod { - algorithm: HmacAlgorithm, - encoding: Encoding, - signature_header_name: String, - signature_prefix: Option, -} - -pub struct SignatureConfigData<'config> { - secret_key: &'config str, -} - -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct SignatureAuthentication { - signature_provider: WebhookType, - secret_key: String, - authentication_config: Option, -} - -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct BasicAuthAuthentication { - pub username: String, - pub password: String, -} - -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct ApiKeyAuthentication { - pub api_key_header: String, - pub api_key_secret: String, -} - -#[derive(Debug, Eq, PartialEq, Hash, Clone, Copy, Serialize, Deserialize)] -#[non_exhaustive] -pub enum WebhookType { - Github, - Slack, - Stripe, - TikTok, - Twitch, - Zoom, - Custom, -} - -impl WebhookType { - pub fn get_webhook_handler(&self) -> Option<&'static dyn WebhookHandler> { - let handler: &'static dyn WebhookHandler = match *self { - WebhookType::Github => &Github, - WebhookType::Slack => &Slack, - WebhookType::Stripe => &Stripe, - WebhookType::TikTok => &TikTok, - WebhookType::Twitch => &Twitch, - WebhookType::Zoom => &Zoom, - WebhookType::Custom => return None, - }; - Some(handler) - } -} - -trait TryGetWebhookHeader { - fn try_get_webhook_header<'header>( - &'header self, - header_name: &str, - ) -> Result<&'header str, AuthenticationError>; -} - -impl TryGetWebhookHeader for HeaderMap { - fn try_get_webhook_header<'header>( - &'header self, - header_name: &str, - ) -> Result<&'header str, AuthenticationError> { - let Some(signature_header) = self.get(header_name) else { - return Err(AuthenticationError::MissingHeader(header_name.to_string())); - }; - let Some(signature_header) = signature_header.to_str().ok() else { - return Err(AuthenticationError::InvalidHeader(header_name.to_string())); - }; - - Ok(signature_header) - } -} - -pub fn calculate_hmac_signature(algorithm: HmacAlgorithm, secret: &str, payload: &str) -> Vec { - match algorithm { - HmacAlgorithm::Sha1 => { - let mut mac = - HmacSha1::new_from_slice(secret.as_bytes()).expect("HMAC can take key of any size"); - mac.update(payload.as_bytes()); - mac.finalize().into_bytes().to_vec() - } - HmacAlgorithm::Sha256 => { - let mut mac = HmacSha256::new_from_slice(secret.as_bytes()) - .expect("HMAC can take key of any size"); - mac.update(payload.as_bytes()); - mac.finalize().into_bytes().to_vec() - } - HmacAlgorithm::Sha512 => { - let mut mac = HmacSha512::new_from_slice(secret.as_bytes()) - .expect("HMAC can take key of any size"); - mac.update(payload.as_bytes()); - mac.finalize().into_bytes().to_vec() - } - } -} - -pub fn encode_hmac_signature(encoding: Encoding, hmac_signature: &[u8]) -> String { - match encoding { - Encoding::Hex => hex::encode(hmac_signature), - Encoding::Base64 => BASE64_STANDARD.encode(hmac_signature), - Encoding::Base64Uri => BASE64_URL_SAFE.encode(hmac_signature), - } -} - -pub fn verify_hmac_signature( - authentication_data: SignatureAuthenticationData, - webhook_signing_secret: &str, -) -> Result<(), AuthenticationError> { - let hmac_signature = calculate_hmac_signature( - authentication_data.config.algorithm_to_use, - &webhook_signing_secret, - &authentication_data.signed_payload, - ); - - let encoded_signature = encode_hmac_signature( - authentication_data.config.header_key_encoding, - &hmac_signature, - ); - - let final_expected_signature = - if let Some(signature_prefix) = authentication_data.signature_prefix { - format!("{}{}", signature_prefix, encoded_signature) - } else { - encoded_signature - }; - - if !constant_time_eq( - final_expected_signature.as_bytes(), - authentication_data.header_key_value.as_bytes(), - ) { - return Err(AuthenticationError::InvalidSignature); - } - - Ok(()) -} - -#[derive(Debug, Serialize, Deserialize, Clone)] -#[serde(untagged)] -pub enum AuthenticationMethod { - Signature(SignatureAuthentication), - BasicAuth(BasicAuthAuthentication), - ApiKey(ApiKeyAuthentication), -} - -impl AuthenticationMethod { - pub fn authenticate_http_request( - &self, - headers: &HeaderMap, - raw_payload: Option<&String>, - ) -> Result, AuthenticationError> { - match self { - AuthenticationMethod::Signature(SignatureAuthentication { - secret_key, - authentication_config, - signature_provider, - }) => { - let raw_payload = raw_payload.ok_or(AuthenticationError::InvalidPayload)?; - let config_data = SignatureConfigData { secret_key: &secret_key }; - let handler = signature_provider.get_webhook_handler(); - let challenge_response = handler - .map(|handler| { - handler.handle_challenge_request(headers, &config_data, raw_payload) - }) - .transpose()? - .flatten(); - - if let Some(challenge_response) = challenge_response { - return Ok(Some(challenge_response)); - } - - let authentication_data = match handler { - Some(handler) => handler.get_hmac_authentication_data(headers, raw_payload)?, - None => { - let authentication_config = authentication_config - .as_ref() - .ok_or(AuthenticationError::InvalidCustomConfig)?; - let signature_header_value = headers - .try_get_webhook_header(&authentication_config.signature_header_name)?; - SignatureAuthenticationData::new( - Cow::Borrowed(raw_payload), - signature_header_value, - authentication_config.signature_prefix.as_deref(), - SignatureAuthenticationDetails::new( - authentication_config.algorithm, - authentication_config.encoding, - ), - ) - } - }; - - verify_hmac_signature(authentication_data, &secret_key)?; - } - AuthenticationMethod::ApiKey(ApiKeyAuthentication { - api_key_header, - api_key_secret, - }) => { - let api_key_to_cmp = headers - .try_get_webhook_header(&api_key_header) - .map_err(|_| AuthenticationError::InvalidApiKey)?; - if api_key_to_cmp != api_key_secret { - return Err(AuthenticationError::InvalidApiKey); - } - } - AuthenticationMethod::BasicAuth(BasicAuthAuthentication { username, password }) => { - let mut credentials_store = headers - .try_get_webhook_header("Authorization") - .map_err(|_| AuthenticationError::UnauthorizedBasicHttpAuth)? - .split(' '); - - let _ = credentials_store - .next() - .filter(|r#type| *r#type == "Basic") - .ok_or(AuthenticationError::UnauthorizedBasicHttpAuth)?; - - let credentials_as_base64 = credentials_store - .next() - .ok_or(AuthenticationError::UnauthorizedBasicHttpAuth)?; - - let credentials_from_base64_as_bytes = BASE64_STANDARD - .decode(credentials_as_base64.as_bytes()) - .map_err(|_| AuthenticationError::UnauthorizedBasicHttpAuth)?; - - let credentials_separated_with_colon = - String::from_utf8(credentials_from_base64_as_bytes) - .map_err(|_| AuthenticationError::UnauthorizedBasicHttpAuth)?; - - let credentials = credentials_separated_with_colon.split(':').collect_vec(); - - if credentials.len() != 2 { - return Err(AuthenticationError::UnauthorizedBasicHttpAuth); - } - - if credentials.get(0).unwrap() != username - || credentials.get(1).unwrap() != password - { - return Err(AuthenticationError::UnauthorizedBasicHttpAuth); - } - } - } - - Ok(None) - } -} - -#[derive(thiserror::Error, Debug)] -#[allow(unused)] -pub enum AuthenticationError { - #[error("failed to parse timestamp")] - InvalidTimestamp, - - #[error("invalid secret")] - InvalidSecret(#[from] base64::DecodeError), - - #[error("invalid header `{0}`")] - InvalidHeader(String), - - #[error("signature timestamp too old")] - TimestampTooOldError, - - #[error("signature timestamp too far in future")] - FutureTimestampError, - - #[error("missing header {0}")] - MissingHeader(String), - - #[error("signature invalid")] - InvalidSignature, - - #[error("payload invalid")] - InvalidPayload, - - #[error("invalid custom config")] - InvalidCustomConfig, - - #[error("invalid auth header: {0}")] - InvalidAuthHeader(String), - - #[error("invalid api key")] - InvalidApiKey, - - #[error("invalid challenge response: {0}")] - InvalidChallengeResponse(String), - - #[error("")] - UnauthorizedBasicHttpAuth, -} - -impl IntoResponse for AuthenticationError { - fn into_response(self) -> Response { - let (status, error_message) = match &self { - AuthenticationError::InvalidTimestamp - | AuthenticationError::InvalidPayload - | AuthenticationError::InvalidHeader(_) - | AuthenticationError::MissingHeader(_) - | AuthenticationError::TimestampTooOldError - | AuthenticationError::FutureTimestampError - | AuthenticationError::InvalidCustomConfig - | AuthenticationError::InvalidChallengeResponse(_) => { - (StatusCode::BAD_REQUEST, self.to_string()) - } - - AuthenticationError::InvalidSecret(_) - | AuthenticationError::InvalidSignature - | AuthenticationError::InvalidAuthHeader(_) => { - (StatusCode::UNAUTHORIZED, self.to_string()) - } - AuthenticationError::UnauthorizedBasicHttpAuth => { - return ( - StatusCode::UNAUTHORIZED, - [(header::WWW_AUTHENTICATE, r#"Basic realm="Restricted Area""#)], - "Unauthorized", - ) - .into_response() - } - AuthenticationError::InvalidApiKey => { - return (StatusCode::UNAUTHORIZED, "Unauthorized").into_response() - } - }; - - let body = json!({ "error": error_message }); - - let mut headers = HeaderMap::new(); - headers.insert("Content-Type", HeaderValue::from_static("application/json")); - - (status, headers, body.to_string()).into_response() - } -} diff --git a/backend/windmill-api/src/triggers/http/mod.rs b/backend/windmill-api/src/triggers/http/mod.rs deleted file mode 100644 index 291d9eadfe..0000000000 --- a/backend/windmill-api/src/triggers/http/mod.rs +++ /dev/null @@ -1,410 +0,0 @@ -use std::collections::HashMap; - -use quick_cache::sync::Cache; -use serde::{Deserialize, Serialize}; -use sqlx::{types::Json as SqlxJson, FromRow}; -use tokio::sync::{RwLock, RwLockReadGuard}; -use windmill_common::{ - error::{Error, Result}, - flows::Retry, - s3_helpers::S3Object, - worker::CLOUD_HOSTED, - DB, -}; - -use crate::{db::ApiAuthed, triggers::TriggerMode, utils::ExpiringCacheEntry}; - -pub mod handler; -pub mod http_trigger_args; -pub mod http_trigger_auth; - -lazy_static::lazy_static! { - static ref HTTP_ACCESS_CACHE: Cache<(String, String, ApiAuthed), ExpiringCacheEntry<()>> = Cache::new(100); - static ref HTTP_AUTH_CACHE: Cache<(String, String, ApiAuthed), ExpiringCacheEntry> = Cache::new(100); - - static ref HTTP_ROUTERS_CACHE: RwLock = RwLock::new(RoutersCache { - routers: HashMap::new(), - version: 0, - }); -} - -#[derive(Debug, Deserialize, Clone)] -pub struct TriggerRoute { - path: String, - script_path: String, - is_flow: bool, - route_path: String, - workspace_id: String, - request_type: RequestType, - authentication_method: AuthenticationMethod, - edited_by: String, - email: String, - static_asset_config: Option>, - is_static_website: bool, - authentication_resource_path: Option, - workspaced_route: bool, - wrap_body: bool, - raw_string: bool, - error_handler_path: Option, - error_handler_args: Option>>, - retry: Option>, - mode: TriggerMode, -} - -pub struct RoutersCache { - routers: HashMap>, - version: i64, -} - -#[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Copy, Hash, Eq, PartialEq)] -#[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")] -#[serde(rename_all = "lowercase")] -pub enum HttpMethod { - Get, - Post, - Put, - Delete, - Patch, -} - -#[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Copy, PartialEq)] -#[sqlx(type_name = "REQUEST_TYPE", rename_all = "snake_case")] -#[serde(rename_all = "snake_case")] -pub enum RequestType { - Sync, - Async, - SyncSse, -} - -impl TryFrom<&http::Method> for HttpMethod { - type Error = Error; - fn try_from(method: &http::Method) -> Result { - match method { - &http::Method::GET => Ok(HttpMethod::Get), - &http::Method::POST => Ok(HttpMethod::Post), - &http::Method::PUT => Ok(HttpMethod::Put), - &http::Method::DELETE => Ok(HttpMethod::Delete), - &http::Method::PATCH => Ok(HttpMethod::Patch), - _ => Err(Error::BadRequest("Invalid HTTP method".to_string())), - } - } -} - -#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone, Copy)] -#[sqlx(type_name = "AUTHENTICATION_METHOD", rename_all = "snake_case")] -#[serde(rename_all(serialize = "snake_case", deserialize = "snake_case"))] -pub enum AuthenticationMethod { - None, - Windmill, - ApiKey, - BasicHttp, - CustomScript, - Signature, -} - -#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] -pub struct HttpConfig { - pub route_path: String, - pub route_path_key: String, - pub request_type: RequestType, - pub authentication_method: AuthenticationMethod, - pub http_method: HttpMethod, - pub summary: Option, - pub description: Option, - pub static_asset_config: Option>, - pub is_static_website: bool, - pub authentication_resource_path: Option, - pub workspaced_route: bool, - pub wrap_body: bool, - pub raw_string: bool, -} - -#[derive(Debug, Clone, Serialize)] -pub struct HttpConfigRequest { - #[serde(default)] - pub route_path: String, - pub request_type: RequestType, - pub authentication_method: AuthenticationMethod, - pub http_method: HttpMethod, - pub summary: Option, - pub description: Option, - pub static_asset_config: Option>, - pub is_static_website: bool, - pub authentication_resource_path: Option, - pub workspaced_route: Option, - pub wrap_body: Option, - pub raw_string: Option, -} - -#[derive(Deserialize)] -struct HttpConfigRequestHelper { - #[serde(default)] - route_path: String, - request_type: Option, - is_async: Option, - authentication_method: AuthenticationMethod, - http_method: HttpMethod, - summary: Option, - description: Option, - static_asset_config: Option>, - is_static_website: bool, - authentication_resource_path: Option, - workspaced_route: Option, - wrap_body: Option, - raw_string: Option, -} - -impl<'de> Deserialize<'de> for HttpConfigRequest { - fn deserialize(deserializer: D) -> std::result::Result - where - D: serde::Deserializer<'de>, - { - let helper = HttpConfigRequestHelper::deserialize(deserializer)?; - - // Determine request_type with backward compatibility - let request_type = if let Some(mode) = helper.request_type { - mode - } else if let Some(is_async) = helper.is_async { - if is_async { - RequestType::Async - } else { - RequestType::Sync - } - } else { - RequestType::Sync - }; - - Ok(HttpConfigRequest { - route_path: helper.route_path, - request_type, - authentication_method: helper.authentication_method, - http_method: helper.http_method, - summary: helper.summary, - description: helper.description, - static_asset_config: helper.static_asset_config, - is_static_website: helper.is_static_website, - authentication_resource_path: helper.authentication_resource_path, - workspaced_route: helper.workspaced_route, - wrap_body: helper.wrap_body, - raw_string: helper.raw_string, - }) - } -} - -// Regex patterns for route validation -lazy_static::lazy_static! { - // Matches named params like :id or wildcards like :* or * - static ref ROUTE_PATH_KEY_RE: regex::Regex = regex::Regex::new(r"(/)?(:|\*)[-\w]+").unwrap(); - static ref VALID_ROUTE_PATH_RE: regex::Regex = regex::Regex::new(r"^(\*[-\w]+$|:?[-\w]+)(/(\*[-\w]+$|:?[-\w]+))*$").unwrap(); -} - -#[derive(Deserialize)] -pub struct RouteExists { - pub route_path: String, - pub http_method: HttpMethod, - pub trigger_path: Option, - pub workspaced_route: Option, -} - -pub fn validate_authentication_method( - authentication_method: AuthenticationMethod, - raw_string: Option, -) -> Result<()> { - match (authentication_method, raw_string) { - (AuthenticationMethod::CustomScript, raw) if !raw.unwrap_or(false) => { - Err(Error::BadRequest( - "To use custom script authentication, please enable the raw body option." - .to_string(), - )) - } - _ => Ok(()), - } -} - -pub async fn refresh_routers(db: &DB) -> Result<(bool, RwLockReadGuard<'_, RoutersCache>)> { - let version = sqlx::query_scalar!("SELECT last_value FROM http_trigger_version_seq",) - .fetch_one(db) - .await?; - let routers_cache = HTTP_ROUTERS_CACHE.read().await; - if routers_cache.version == 0 || version > routers_cache.version { - drop(routers_cache); - let mut routers = HashMap::new(); - - for http_method in [ - HttpMethod::Get, - HttpMethod::Post, - HttpMethod::Put, - HttpMethod::Patch, - HttpMethod::Delete, - ] { - let triggers = sqlx::query_as!( - TriggerRoute, - r#" - SELECT - path, - script_path, - is_flow, - route_path, - authentication_resource_path, - workspace_id, - request_type AS "request_type: _", - authentication_method AS "authentication_method: _", - edited_by, - email, - static_asset_config AS "static_asset_config: _", - wrap_body, - raw_string, - workspaced_route, - is_static_website, - error_handler_path, - error_handler_args as "error_handler_args: _", - retry as "retry: _", - mode as "mode: _" - FROM - http_trigger - WHERE - http_method = $1 AND - (mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE) - "#, - &http_method as &HttpMethod - ) - .fetch_all(db) - .await?; - - let mut router = matchit::Router::new(); - - for trigger in triggers { - let full_path = if trigger.workspaced_route || *CLOUD_HOSTED { - format!("/{}/{}", trigger.workspace_id, trigger.route_path) - } else { - format!("/{}", trigger.route_path) - }; - - if trigger.is_static_website { - router - .insert(format!("{}/*wm_subpath", full_path), trigger.clone()) - .unwrap_or_else(|e| { - tracing::warn!( - "Failed to consider HTTP route {}/*wm_subpath: {:?}", - full_path, - e, - ); - }); - } - router - .insert(full_path.clone(), trigger.clone()) - .unwrap_or_else(|e| { - tracing::warn!("Failed to consider HTTP route {}: {:?}", full_path, e,); - }); - } - - routers.insert(http_method, router); - } - - let mut routers_cache = HTTP_ROUTERS_CACHE.write().await; - *routers_cache = RoutersCache { routers, version }; - - Ok((true, routers_cache.downgrade())) - } else { - tracing::debug!("No HTTP routers refresh needed"); - Ok((false, routers_cache)) - } -} - -pub async fn refresh_routers_loop( - db: &DB, - mut killpill_rx: tokio::sync::broadcast::Receiver<()>, -) -> () { - match refresh_routers(db).await { - Ok(_) => { - tracing::info!("Loaded HTTP routers"); - } - Err(err) => { - tracing::error!("Error loading HTTP routers: {err:#}"); - } - }; - let db = db.clone(); - tokio::spawn(async move { - loop { - tokio::select! { - _ = killpill_rx.recv() => { - break; - } - _ = tokio::time::sleep(std::time::Duration::from_secs(60)) => { - match refresh_routers(&db).await { - Ok((true, _)) => { - tracing::info!("Refreshed HTTP routers"); - } - Err(err) => { - tracing::error!("Error refreshing HTTP routers: {err:#}"); - } - _ => {} - } - } - } - } - }); -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_request_type_backward_compatibility() { - // Test with new request_type field - let json_new = r#"{ - "route_path": "/test", - "request_type": "sync_sse", - "authentication_method": "none", - "http_method": "get", - "is_static_website": false - }"#; - let config: HttpConfigRequest = serde_json::from_str(json_new).unwrap(); - assert_eq!(config.request_type, RequestType::SyncSse); - - // Test with legacy is_async = true - let json_legacy_async = r#"{ - "route_path": "/test", - "is_async": true, - "authentication_method": "none", - "http_method": "get", - "is_static_website": false - }"#; - let config: HttpConfigRequest = serde_json::from_str(json_legacy_async).unwrap(); - assert_eq!(config.request_type, RequestType::Async); - - // Test with legacy is_async = false - let json_legacy_sync = r#"{ - "route_path": "/test", - "is_async": false, - "authentication_method": "none", - "http_method": "get", - "is_static_website": false - }"#; - let config: HttpConfigRequest = serde_json::from_str(json_legacy_sync).unwrap(); - assert_eq!(config.request_type, RequestType::Sync); - - // Test with neither field (default to sync) - let json_default = r#"{ - "route_path": "/test", - "authentication_method": "none", - "http_method": "get", - "is_static_website": false - }"#; - let config: HttpConfigRequest = serde_json::from_str(json_default).unwrap(); - assert_eq!(config.request_type, RequestType::Sync); - - // Test that request_type takes precedence over is_async - let json_both = r#"{ - "route_path": "/test", - "request_type": "sync_sse", - "is_async": true, - "authentication_method": "none", - "http_method": "get", - "is_static_website": false - }"#; - let config: HttpConfigRequest = serde_json::from_str(json_both).unwrap(); - assert_eq!(config.request_type, RequestType::SyncSse); - } -} diff --git a/backend/windmill-api/src/triggers/kafka/handler_oss.rs b/backend/windmill-api/src/triggers/kafka/handler_oss.rs deleted file mode 100644 index 9fda83916f..0000000000 --- a/backend/windmill-api/src/triggers/kafka/handler_oss.rs +++ /dev/null @@ -1,67 +0,0 @@ -#[cfg(not(feature = "private"))] -use crate::triggers::TriggerData; - -#[allow(unused)] -#[cfg(feature = "private")] -pub use super::handler_ee::*; - -#[cfg(not(feature = "private"))] -use { - super::KafkaTrigger, - crate::{ - db::{ApiAuthed, DB}, - triggers::TriggerCrud, - }, - axum::async_trait, - sqlx::PgConnection, - windmill_common::error::{Error, Result}, - windmill_git_sync::DeployedObject, -}; - -#[cfg(not(feature = "private"))] -#[async_trait] -impl TriggerCrud for KafkaTrigger { - type Trigger = (); - type TriggerConfig = (); - type TriggerConfigRequest = (); - type TestConnectionConfig = (); - - const TABLE_NAME: &'static str = ""; - const TRIGGER_TYPE: &'static str = ""; - const SUPPORTS_SERVER_STATE: bool = false; - const SUPPORTS_TEST_CONNECTION: bool = false; - const ROUTE_PREFIX: &'static str = "/kafka_triggers"; - const DEPLOYMENT_NAME: &'static str = ""; - const IS_ALLOWED_ON_CLOUD: bool = false; - - fn get_deployed_object(path: String) -> DeployedObject { - DeployedObject::KafkaTrigger { path } - } - - async fn create_trigger( - &self, - _db: &DB, - _tx: &mut PgConnection, - _authed: &ApiAuthed, - _w_id: &str, - _trigger: TriggerData, - ) -> Result<()> { - Err(Error::BadRequest( - "Kafka triggers are not available in open source version".to_string(), - )) - } - - async fn update_trigger( - &self, - _db: &DB, - _executor: &mut PgConnection, - _authed: &ApiAuthed, - _workspace_id: &str, - _path: &str, - _trigger: TriggerData, - ) -> Result<()> { - Err(Error::BadRequest( - "Kafka triggers are not available in open source version".to_string(), - )) - } -} diff --git a/backend/windmill-api/src/triggers/kafka/listener_oss.rs b/backend/windmill-api/src/triggers/kafka/listener_oss.rs deleted file mode 100644 index c5dca4ab37..0000000000 --- a/backend/windmill-api/src/triggers/kafka/listener_oss.rs +++ /dev/null @@ -1,41 +0,0 @@ -#[allow(unused)] - -#[cfg(feature = "private")] -pub use super::listener_ee::*; - -#[cfg(not(feature = "private"))] -use { - super::KafkaTrigger, - crate::triggers::{listener::ListeningTrigger, Listener}, - std::sync::Arc, - tokio::sync::RwLock, - windmill_common::{error::Result, jobs::JobTriggerKind, DB}, -}; - -#[cfg(not(feature = "private"))] -#[async_trait::async_trait] -impl Listener for KafkaTrigger { - type Consumer = (); - type Extra = (); - const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Kafka; - - async fn get_consumer( - &self, - _db: &DB, - _listening_trigger: &ListeningTrigger, - _err_message: Arc>>, - _killpill_rx: tokio::sync::broadcast::Receiver<()>, - ) -> Result> { - Ok(None) - } - async fn consume( - &self, - _db: &DB, - _consumer: Self::Consumer, - _listening_trigger: &ListeningTrigger, - _err_message: Arc>>, - _killpill_rx: tokio::sync::broadcast::Receiver<()>, - ) { - () - } -} diff --git a/backend/windmill-api/src/triggers/kafka/mod.rs b/backend/windmill-api/src/triggers/kafka/mod.rs deleted file mode 100644 index f09b2283ff..0000000000 --- a/backend/windmill-api/src/triggers/kafka/mod.rs +++ /dev/null @@ -1,15 +0,0 @@ -#[cfg(feature = "private")] -mod handler_ee; -pub mod handler_oss; - -#[cfg(feature = "private")] -mod listener_ee; -pub mod listener_oss; - -#[cfg(feature = "private")] -mod mod_ee; -#[cfg(feature = "private")] -pub use mod_ee::*; - -#[derive(Copy, Clone)] -pub struct KafkaTrigger; diff --git a/backend/windmill-api/src/triggers/listener.rs b/backend/windmill-api/src/triggers/listener.rs deleted file mode 100644 index ce4f651ba9..0000000000 --- a/backend/windmill-api/src/triggers/listener.rs +++ /dev/null @@ -1,928 +0,0 @@ -use std::{collections::HashMap, fmt::Debug, sync::Arc}; - -use crate::{ - capture::insert_capture_payload, - db::ApiAuthed, - triggers::{ - handler::TriggerCrud, - trigger_helpers::{trigger_runnable, TriggerJobArgs}, - Trigger, TriggerErrorHandling, TriggerMode, - }, - users::fetch_api_authed, -}; -use async_trait::async_trait; -use itertools::Itertools; -use rand::seq::SliceRandom; -use serde::de::DeserializeOwned; -use serde::{Deserialize, Serialize}; -use serde_json::value::RawValue; -use sql_builder::SqlBuilder; -use sqlx::{FromRow, Row}; -use tokio::sync::RwLock; -use windmill_common::{ - error::{Error, Result}, - jobs::JobTriggerKind, - triggers::{TriggerKind, TriggerMetadata}, - utils::report_critical_error, - DB, INSTANCE_NAME, -}; - -#[allow(unused)] -#[async_trait] -pub trait Listener: TriggerCrud + TriggerJobArgs { - type Consumer: Send; - type Extra: Send + Sync; - type ExtraState: Send + Sync; - - const JOB_TRIGGER_KIND: JobTriggerKind; - const EXTRA_TRIGGER_AND_WHERE_CLAUSE: &[&'static str] = &[]; - const EXTRA_CAPTURE_AND_WHERE_CLAUSE: &[&'static str] = &[]; - - async fn get_consumer( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - err_message: Arc>>, - killpill_rx: tokio::sync::broadcast::Receiver<()>, - ) -> Result>; - async fn consume( - &self, - db: &DB, - consumer: Self::Consumer, - listening_trigger: &ListeningTrigger, - err_message: Arc>>, - killpill_rx: tokio::sync::broadcast::Receiver<()>, - extra: Option<&Self::ExtraState>, - ); - async fn fetch_enabled_unlistened_triggers( - &self, - db: &DB, - ) -> Result>> { - let mut fields = vec![ - "workspace_id", - "path", - "script_path", - "is_flow", - "edited_by", - "email", - "edited_at", - "extra_perms", - "mode", - "error_handler_path", - "error_handler_args", - "retry", - ]; - - fields.extend_from_slice(Self::ADDITIONAL_SELECT_FIELDS); - - let mut sqlb = SqlBuilder::select_from(Self::TABLE_NAME); - - sqlb.fields(&fields) - .and_where("(mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE)") - .and_where( - "(last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')", - ); - - for where_clause in Self::EXTRA_TRIGGER_AND_WHERE_CLAUSE { - sqlb.and_where(where_clause); - } - - let sql = sqlb - .sql() - .map_err(|e| Error::InternalErr(format!("SQL error: {}", e)))?; - - let triggers: Vec> = - sqlx::query_as(&sql).fetch_all(db).await?; - - let triggers = triggers - .into_iter() - .map(|trigger| ListeningTrigger { - path: trigger.base.path, - workspace_id: trigger.base.workspace_id, - is_flow: trigger.base.is_flow, - username: trigger.base.edited_by, - email: trigger.base.email, - script_path: trigger.base.script_path, - trigger_config: trigger.config, - error_handling: Some(trigger.error_handling), - trigger_mode: true, - suspended_mode: trigger.base.mode == TriggerMode::Suspended, - }) - .collect_vec(); - - Ok(triggers) - } - - async fn fetch_unlistened_captures( - &self, - db: &DB, - ) -> Result>> { - let fields = vec![ - "path", - "is_flow", - "workspace_id", - "owner AS username", - "email", - "trigger_config", - ]; - - let mut sqlb = SqlBuilder::select_from("capture_config"); - sqlb.fields(&fields) - .and_where(format!("trigger_kind = '{}'", Self::TRIGGER_KIND.to_key())) - .and_where("last_client_ping > NOW() - INTERVAL '10 seconds'") - .and_where("trigger_config IS NOT NULL") - .and_where( - "(last_server_ping IS NULL OR last_server_ping < NOW() - INTERVAL '15 seconds')", - ); - - for where_clause in Self::EXTRA_CAPTURE_AND_WHERE_CLAUSE { - sqlb.and_where(where_clause); - } - - let sql = sqlb.sql().expect("failed to build SQL"); - - let captures: Vec> = - sqlx::query_as(&sql).fetch_all(db).await?; - - let captures = captures - .into_iter() - .map(|capture| ListeningTrigger { - username: capture.username, - path: capture.path, - workspace_id: capture.workspace_id, - script_path: "".to_string(), - email: capture.email, - trigger_config: capture.trigger_config, - trigger_mode: false, - is_flow: capture.is_flow, - error_handling: None, - suspended_mode: false, - }) - .collect_vec(); - - Ok(captures) - } - - async fn get_extra_state(&self) -> Option { - None - } - - async fn cleanup( - &self, - _db: &DB, - _listening_trigger: &ListeningTrigger, - _extra: Option<&Self::ExtraState>, - ) -> Result<()> { - Ok(()) - } - - async fn loop_ping( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - status: Arc>>, - error_message: Option, - ) { - update_rw_lock(status.clone(), error_message).await; - loop { - if let None = self - .update_ping(db, listening_trigger, status.read().await.as_deref()) - .await - { - return; - } - tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; - } - } - - async fn update_ping( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - error: Option<&str>, - ) -> Option<()> { - if listening_trigger.trigger_mode { - self.update_trigger_ping(db, listening_trigger, error).await - } else { - self.update_capture_ping(db, listening_trigger, error).await - } - } - - async fn update_ping_and_loop_ping_status( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - loop_ping_status: Arc>>, - error: Option, - ) -> Option<()> { - // update immediately the ping status and update the loop ping status so that the next loop pings will display the new status - update_rw_lock(loop_ping_status.clone(), error.clone()).await; - if let None = self - .update_ping(db, listening_trigger, error.as_deref()) - .await - { - return None; - } - Some(()) - } - - async fn update_trigger_ping( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - error: Option<&str>, - ) -> Option<()> { - let updated = sqlx::query_scalar::<_, i32>(&format!( - r#" - UPDATE - {} - SET - last_server_ping = now(), error = $1 - WHERE - workspace_id = $2 AND - path = $3 AND - server_id = $4 AND - (mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE) - RETURNING 1 - "#, - Self::TABLE_NAME - )) - .bind(error) - .bind(&listening_trigger.workspace_id) - .bind(&listening_trigger.path) - .bind(&*INSTANCE_NAME) - .fetch_optional(db) - .await; - - self.handle_ping_result(updated, db, listening_trigger, "trigger") - .await - } - - async fn update_capture_ping( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - error: Option<&str>, - ) -> Option<()> { - let updated = sqlx::query_scalar!( - r#" - UPDATE - capture_config - SET - last_server_ping = now(), error = $1 - WHERE - workspace_id = $2 AND - path = $3 AND - is_flow = $4 AND - trigger_kind = $5 AND - server_id = $6 AND - last_client_ping > NOW() - INTERVAL '10 seconds' - RETURNING 1 - "#, - error, - &listening_trigger.workspace_id, - &listening_trigger.path, - &listening_trigger.is_flow, - Self::TRIGGER_KIND as TriggerKind, - &*INSTANCE_NAME - ) - .fetch_optional(db) - .await - .map(|result| result.flatten()); - - self.handle_ping_result(updated, db, listening_trigger, "capture") - .await - } - - async fn handle_ping_result( - &self, - result: sqlx::Result>, - db: &DB, - listening_trigger: &ListeningTrigger, - entity_type: &str, - ) -> Option<()> { - match result { - Ok(updated) => { - if updated.is_none() { - self.reset_ping_for_restart(db, listening_trigger).await; - tracing::info!( - "{} {} {} changed, disabled, or deleted, stopping...", - Self::TRIGGER_KIND, - entity_type, - listening_trigger.path - ); - return None; - } - } - Err(error) => { - tracing::warn!( - "Error updating ping of {} {} {}: {:?}", - Self::TRIGGER_KIND, - entity_type, - &listening_trigger.path, - error - ); - } - } - - Some(()) - } - - async fn reset_ping_for_restart( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - ) { - if listening_trigger.trigger_mode { - let _ = sqlx::query(&format!( - r#" - UPDATE - {} - SET - last_server_ping = NULL - WHERE - workspace_id = $1 AND - path = $2 AND - server_id IS NULL - "#, - Self::TABLE_NAME - )) - .bind(&listening_trigger.workspace_id) - .bind(&listening_trigger.path) - .execute(db) - .await; - } else { - let _ = sqlx::query!( - r#" - UPDATE - capture_config - SET - last_server_ping = NULL - WHERE - workspace_id = $1 AND - path = $2 AND - is_flow = $3 AND - trigger_kind = $4 AND - server_id IS NULL - "#, - &listening_trigger.workspace_id, - &listening_trigger.path, - &listening_trigger.is_flow, - Self::TRIGGER_KIND as TriggerKind - ) - .execute(db) - .await; - } - } - - async fn disable_with_error( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - error: String, - ) { - if listening_trigger.trigger_mode { - let report_status = sqlx::query(&format!( - r#" - UPDATE - {} - SET - mode = 'disabled'::TRIGGER_MODE, - error = $1, - server_id = NULL, - last_server_ping = NULL - WHERE - workspace_id = $2 AND - path = $3 - "#, - Self::TABLE_NAME - )) - .bind(&error) - .bind(&listening_trigger.workspace_id) - .bind(&listening_trigger.path) - .execute(db) - .await; - - match report_status { - Ok(_) => { - report_critical_error( - format!( - "Disabling {} trigger {} because of error: {}", - Self::TRIGGER_KIND, - listening_trigger.path, - error - ), - db.clone(), - Some(&listening_trigger.workspace_id), - None, - ) - .await; - } - Err(disable_err) => { - report_critical_error( - format!("Could not disable {} trigger {} with err {}, disabling because of error {}", Self::TRIGGER_KIND, listening_trigger.path, disable_err, error), - db.clone(), - Some(&listening_trigger.workspace_id), - None, - ).await; - } - } - return; - } - - let report_status = sqlx::query!( - r#" - UPDATE - capture_config - SET - error = $1, - server_id = NULL, - last_server_ping = NULL - WHERE - workspace_id = $2 AND - path = $3 AND - is_flow = $4 AND - trigger_kind = $5 - "#, - error, - listening_trigger.workspace_id, - listening_trigger.path, - listening_trigger.is_flow, - Self::TRIGGER_KIND as TriggerKind - ) - .execute(db) - .await; - - if let Err(disable_err) = report_status { - tracing::error!( - "Could not disable {} capture {} ({}) with err {}, disabling because of error {}", - Self::TRIGGER_KIND, - listening_trigger.path, - listening_trigger.workspace_id, - disable_err, - error - ) - } - } - - async fn handle_trigger( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - payload: Self::Payload, - trigger_info: HashMap>, - _extra: Option, - ) -> Result<()> { - let args = Self::build_job_args( - &listening_trigger.script_path, - listening_trigger.is_flow, - &listening_trigger.workspace_id, - db, - payload, - trigger_info, - ) - .await?; - - let authed = listening_trigger - .authed(db, &Self::TRIGGER_KIND.to_string()) - .await?; - - let (retry, error_handler_path, error_handler_args) = - match listening_trigger.error_handling.as_ref() { - Some(error_handling) => ( - error_handling.retry.as_ref(), - error_handling.error_handler_path.as_deref(), - error_handling.error_handler_args.as_ref(), - ), - None => (None, None, None), - }; - - tracing::debug!( - "Triggering job from {} event {} with args {:?}", - Self::TRIGGER_KIND, - listening_trigger.path, - args - ); - - trigger_runnable( - db, - None, - authed, - &listening_trigger.workspace_id, - &listening_trigger.script_path, - listening_trigger.is_flow, - args, - retry, - error_handler_path.as_deref(), - error_handler_args, - format!("{}_trigger/{}", Self::TRIGGER_KIND, listening_trigger.path), - None, - listening_trigger.suspended_mode, - TriggerMetadata::new(Some(listening_trigger.path.clone()), Self::JOB_TRIGGER_KIND), - ) - .await?; - - Ok(()) - } - - async fn handle_event( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - payload: Self::Payload, - trigger_info: HashMap>, - extra: Option, - ) -> Result<()> { - if listening_trigger.trigger_mode { - if let Err(err) = self - .handle_trigger(db, listening_trigger, payload, trigger_info, extra) - .await - { - report_critical_error( - format!( - "Failed to trigger job from {} event {}: {:?}", - Self::TRIGGER_KIND, - listening_trigger.path, - err - ), - db.clone(), - Some(&listening_trigger.workspace_id), - None, - ) - .await; - return Err(err); - }; - return Ok(()); - } - - let (main_args, preprocessor_args) = Self::build_capture_payloads(&payload, trigger_info); - if let Err(err) = insert_capture_payload( - db, - &listening_trigger.workspace_id, - &listening_trigger.path, - listening_trigger.is_flow, - &Self::TRIGGER_KIND, - main_args, - preprocessor_args, - &listening_trigger.username, - ) - .await - { - tracing::error!("Error inserting capture payload: {:?}", err); - return Err(err); - } - Ok(()) - } -} - -#[allow(unused)] -async fn listening( - db: DB, - listener: T, - listening_trigger: ListeningTrigger, - mut killpill_rx: tokio::sync::broadcast::Receiver<()>, -) { - let killpill_rx_consumer = killpill_rx.resubscribe(); - let killpill_rx_get_consumer = killpill_rx.resubscribe(); - - let loop_ping_status = Arc::new(RwLock::new(None)); - let extra_state = listener.get_extra_state().await; - let path = listening_trigger.path.clone(); - tokio::select! { - biased; - _ = killpill_rx.recv() => { - let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; - } - _ = listener.loop_ping(&db, &listening_trigger, loop_ping_status.clone(), Some("Connecting...".to_string())) => { - let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; - } - consumer = { - tracing::info!("[{}] Getting consumer for trigger {}", T::TRIGGER_KIND, path); - listener.get_consumer(&db, &listening_trigger, loop_ping_status.clone(), killpill_rx_get_consumer) - } => { - tokio::select! { - biased; - _ = killpill_rx.recv() => { - tracing::info!("[{}] Killing pill received, stopping consumer for trigger {}", T::TRIGGER_KIND, path); - let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; - return; - } - _ = listener.loop_ping(&db, &listening_trigger, loop_ping_status.clone(), None) => { - tracing::info!("[{}] Loop ping exited, stopping consumer for trigger {}", T::TRIGGER_KIND, path); - let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; - return; - } - _ = async { - match consumer { - Ok(Some(consumer)) => { - listener.update_ping_and_loop_ping_status(&db, &listening_trigger, loop_ping_status.clone(), None).await; - tracing::info!("[{}] Starting consumer for trigger {}", T::TRIGGER_KIND, path); - listener.consume(&db, consumer, &listening_trigger, loop_ping_status.clone(), killpill_rx_consumer, extra_state.as_ref()).await; - tracing::info!("[{}] Consumer stopped for trigger {}", T::TRIGGER_KIND, path); - } - Err(error) => { - tracing::error!("[{}] Disabling trigger {} due to consumer error: {}", T::TRIGGER_KIND, path, error); - listener.disable_with_error(&db, &listening_trigger, error.to_string()).await; - } - Ok(None) => { - tracing::error!("[{}] Consumer is None for trigger {}", T::TRIGGER_KIND, path); - } - } - } => { - let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await; - return; - } - } - } - } -} - -#[allow(unused)] -async fn listen_to_unlistened_events( - listener: T, - db: DB, - killpill_rx: &tokio::sync::broadcast::Receiver<()>, -) { - let unlistend_enabled_triggers = listener.fetch_enabled_unlistened_triggers(&db).await; - - match unlistend_enabled_triggers { - Ok(mut unlistend_enabled_triggers) => { - unlistend_enabled_triggers.shuffle(&mut rand::rng()); - for trigger in unlistend_enabled_triggers { - let has_lock = sqlx::query_scalar(&format!( - r#" - UPDATE - {} - SET - server_id = $1, - last_server_ping = now(), - error = 'Connecting...' - WHERE - (mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE) - AND workspace_id = $2 - AND path = $3 - AND (last_server_ping IS NULL - OR last_server_ping < now() - INTERVAL '15 seconds' - ) - RETURNING true - "#, - T::TABLE_NAME, - )) - .bind(&*INSTANCE_NAME) - .bind(&trigger.workspace_id) - .bind(&trigger.path) - .fetch_optional(&db) - .await; - match has_lock { - Ok(has_lock) => { - if has_lock.flatten().unwrap_or(false) { - tracing::info!( - "Spawning new task to listen for {} event", - T::TABLE_NAME - ); - tokio::spawn({ - let db = db.clone(); - let killpill_rx = killpill_rx.resubscribe(); - async move { listening(db, listener, trigger, killpill_rx).await } - }); - } else { - tracing::info!( - "{} trigger {} already being listened to", - T::TRIGGER_KIND, - trigger.path - ); - } - } - Err(err) => { - tracing::error!( - "Error acquiring lock for {} trigger {}: {:?}", - T::TRIGGER_KIND, - trigger.path, - err - ); - } - }; - } - } - Err(err) => { - tracing::error!("Error fetching {} triggers: {:?}", T::TRIGGER_KIND, err,); - } - } - - let unlisted_captures = listener.fetch_unlistened_captures(&db).await; - - match unlisted_captures { - Ok(unlistened_captures) => { - for capture in unlistened_captures { - let has_lock = sqlx::query_scalar!( - r#" - UPDATE - capture_config - SET - server_id = $1, - last_server_ping = now(), - error = 'Connecting...' - WHERE - last_client_ping > NOW() - INTERVAL '10 seconds' AND - workspace_id = $2 AND - path = $3 AND - is_flow = $4 AND - trigger_kind = $5 AND - (last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds') - RETURNING true - "#, - *INSTANCE_NAME, - &capture.workspace_id, - &capture.path, - &capture.is_flow, - T::TRIGGER_KIND as TriggerKind - ) - .fetch_optional(&db) - .await; - match has_lock { - Ok(has_lock) => { - if has_lock.flatten().unwrap_or(false) { - tokio::spawn({ - let db = db.clone(); - let killpill_rx = killpill_rx.resubscribe(); - async move { listening(db, listener, capture, killpill_rx).await } - }); - } else { - tracing::info!( - "{} capture {} already being listened to", - T::TRIGGER_KIND.to_string(), - capture.path - ); - } - } - Err(err) => { - tracing::error!( - "Error acquiring lock for capture {} {}: {:?}", - T::TRIGGER_KIND, - capture.path, - err - ); - } - }; - } - } - Err(err) => { - tracing::error!( - "Error fetching captures {} triggers: {:?}", - T::TRIGGER_KIND, - err - ); - } - } -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -struct Capture -where - T: for<'r> FromRow<'r, sqlx::postgres::PgRow>, -{ - path: String, - is_flow: bool, - workspace_id: String, - username: String, - email: String, - #[serde(flatten)] - trigger_config: T, -} - -impl FromRow<'_, sqlx::postgres::PgRow> for Capture -where - T: for<'r> FromRow<'r, sqlx::postgres::PgRow> + DeserializeOwned, -{ - fn from_row(row: &sqlx::postgres::PgRow) -> std::result::Result { - let trigger_config_value = row.try_get("trigger_config")?; - let trigger_config: T = serde_json::from_value(trigger_config_value) - .map_err(|e| sqlx::Error::Decode(Box::new(e)))?; - Ok(Capture { - path: row.try_get("path")?, - is_flow: row.try_get("is_flow")?, - workspace_id: row.try_get("workspace_id")?, - username: row.try_get("username")?, - email: row.try_get("email")?, - trigger_config, - }) - } -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ListeningTrigger { - pub path: String, - pub is_flow: bool, - pub workspace_id: String, - pub username: String, - pub email: String, - pub trigger_config: T, - pub script_path: String, - pub trigger_mode: bool, - pub error_handling: Option, - pub suspended_mode: bool, -} - -impl ListeningTrigger { - pub async fn authed(&self, db: &DB, username: &str) -> Result { - fetch_api_authed( - self.username.clone(), - self.email.clone(), - &self.workspace_id, - db, - Some(format!("{}-{}", username, self.path)), - ) - .await - } -} - -#[allow(unused)] -pub async fn update_rw_lock(lock: std::sync::Arc>, value: T) -> () { - let mut w = lock.write().await; - *w = value; -} - -#[allow(unused)] -fn listen_to( - trigger: T, - db: DB, - mut killpill_rx: tokio::sync::broadcast::Receiver<()>, -) { - tokio::spawn(async move { - listen_to_unlistened_events(trigger, db.clone(), &killpill_rx).await; - loop { - tokio::select! { - biased; - _ = killpill_rx.recv() => { - return; - } - _ = tokio::time::sleep(tokio::time::Duration::from_secs(15)) => { - listen_to_unlistened_events(trigger, db.clone(), &killpill_rx).await - } - } - } - }); -} - -#[allow(unused)] -pub fn start_all_listeners(db: DB, killpill_rx: &tokio::sync::broadcast::Receiver<()>) { - tracing::info!("Starting trigger listeners based on available features..."); - - #[cfg(feature = "postgres_trigger")] - { - let postgres_killpill_rx = killpill_rx.resubscribe(); - use crate::triggers::postgres::PostgresTrigger; - - listen_to(PostgresTrigger, db.clone(), postgres_killpill_rx) - } - - #[cfg(feature = "mqtt_trigger")] - { - let mqtt_killpill_rx = killpill_rx.resubscribe(); - use crate::triggers::mqtt::MqttTrigger; - - listen_to(MqttTrigger, db.clone(), mqtt_killpill_rx) - } - - #[cfg(feature = "websocket")] - { - let mqtt_killpill_rx = killpill_rx.resubscribe(); - use crate::triggers::websocket::WebsocketTrigger; - - listen_to(WebsocketTrigger, db.clone(), mqtt_killpill_rx) - } - - #[cfg(all(feature = "gcp_trigger", feature = "enterprise", feature = "private"))] - { - let gcp_killpill_rx = killpill_rx.resubscribe(); - use crate::triggers::gcp::GcpTrigger; - - listen_to(GcpTrigger, db.clone(), gcp_killpill_rx); - } - - #[cfg(all(feature = "sqs_trigger", feature = "enterprise", feature = "private"))] - { - let gcp_killpill_rx = killpill_rx.resubscribe(); - use crate::triggers::sqs::SqsTrigger; - - listen_to(SqsTrigger, db.clone(), gcp_killpill_rx); - } - - #[cfg(all(feature = "nats", feature = "enterprise", feature = "private"))] - { - let gcp_killpill_rx = killpill_rx.resubscribe(); - use crate::triggers::nats::NatsTrigger; - - listen_to(NatsTrigger, db.clone(), gcp_killpill_rx); - } - - #[cfg(all(feature = "kafka", feature = "enterprise", feature = "private"))] - { - let gcp_killpill_rx = killpill_rx.resubscribe(); - use crate::triggers::kafka::KafkaTrigger; - - listen_to(KafkaTrigger, db.clone(), gcp_killpill_rx); - } - - tracing::info!("All available trigger listeners have been started"); -} diff --git a/backend/windmill-api/src/triggers/mod.rs b/backend/windmill-api/src/triggers/mod.rs deleted file mode 100644 index ce8538de2d..0000000000 --- a/backend/windmill-api/src/triggers/mod.rs +++ /dev/null @@ -1,182 +0,0 @@ -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; -use sqlx::{types::Json as SqlxJson, FromRow}; -use std::{collections::HashMap, fmt::Debug}; -use windmill_common::jobs::JobTriggerKind; - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum HandlerAction { - Trigger { path: String, trigger_kind: JobTriggerKind }, - // Future variants can be added here (e.g., Script, Flow, etc.) -} - -#[cfg(all(feature = "smtp", feature = "private"))] -pub mod email; -#[cfg(all(feature = "gcp_trigger", feature = "enterprise", feature = "private"))] -pub mod gcp; -#[cfg(feature = "http_trigger")] -pub mod http; -#[cfg(all(feature = "kafka", feature = "enterprise", feature = "private"))] -pub mod kafka; -#[cfg(feature = "mqtt_trigger")] -pub mod mqtt; -#[cfg(all(feature = "nats", feature = "enterprise", feature = "private"))] -pub mod nats; -#[cfg(feature = "postgres_trigger")] -pub mod postgres; -#[cfg(all(feature = "sqs_trigger", feature = "enterprise", feature = "private"))] -pub mod sqs; -#[cfg(feature = "websocket")] -pub mod websocket; - -pub mod filter; -pub mod global_handler; -mod handler; -mod listener; -pub mod trigger_helpers; - -#[allow(unused)] -pub(crate) use handler::TriggerCrud; -pub use handler::{generate_trigger_routers, get_triggers_count_internal, TriggersCount}; -pub use listener::start_all_listeners; -#[allow(unused)] -pub(crate) use listener::Listener; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct StandardTriggerQuery { - pub page: Option, - pub per_page: Option, - pub path: Option, - pub is_flow: Option, - pub path_start: Option, -} - -#[derive(Debug, FromRow, Clone, Serialize, Deserialize)] -pub struct BaseTrigger { - pub workspace_id: String, - pub path: String, - pub script_path: String, - pub mode: TriggerMode, - pub is_flow: bool, - pub edited_by: String, - pub email: String, - pub edited_at: DateTime, - pub extra_perms: Option, -} - -#[derive(Debug, FromRow, Clone, Serialize, Deserialize)] -pub struct ServerState { - #[serde(skip_serializing_if = "Option::is_none")] - pub server_id: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub last_server_ping: Option>, - #[serde(skip_serializing_if = "Option::is_none")] - pub error: Option, -} - -#[derive(Debug, FromRow, Clone, Serialize, Deserialize)] -pub struct TriggerErrorHandling { - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_path: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub error_handler_args: Option>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub retry: Option>, -} - -#[derive(Serialize, Deserialize, Clone)] -pub struct Trigger -where - T: for<'r> FromRow<'r, sqlx::postgres::PgRow>, -{ - #[serde(flatten)] - pub base: BaseTrigger, - - #[serde(flatten)] - pub config: T, - - #[serde(flatten)] - pub server_state: Option, - - #[serde(flatten)] - pub error_handling: TriggerErrorHandling, -} - -impl FromRow<'_, sqlx::postgres::PgRow> for Trigger -where - T: for<'r> FromRow<'r, sqlx::postgres::PgRow>, -{ - fn from_row(row: &sqlx::postgres::PgRow) -> std::result::Result { - let base = BaseTrigger::from_row(row)?; - - Ok(Trigger { - base, - config: T::from_row(row)?, - server_state: ServerState::from_row(row).ok(), - error_handling: TriggerErrorHandling::from_row(row)?, - }) - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct BaseTriggerData { - pub path: String, - pub script_path: String, - pub is_flow: bool, - #[deprecated(note = "Use mode instead")] - enabled: Option, // Kept for backwards compatibility, use mode instead - mode: Option, -} - -impl BaseTriggerData { - pub fn mode(&self) -> &TriggerMode { - self.mode.as_ref().unwrap_or( - #[allow(deprecated)] - if self.enabled.unwrap_or(true) { - &TriggerMode::Enabled - } else { - &TriggerMode::Disabled - }, - ) - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TriggerData { - #[serde(flatten)] - pub base: BaseTriggerData, - - #[serde(flatten)] - pub config: T, - - #[serde(flatten)] - pub error_handling: TriggerErrorHandling, -} - -impl StandardTriggerQuery { - pub fn offset(&self) -> i64 { - let page = self.page.unwrap_or(0); - let per_page = self.per_page.unwrap_or(100); - (page * per_page) as i64 - } - - pub fn limit(&self) -> i64 { - self.per_page.unwrap_or(100) as i64 - } -} - -impl Default for StandardTriggerQuery { - fn default() -> Self { - Self { page: Some(0), per_page: Some(100), path: None, path_start: None, is_flow: None } - } -} - -#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)] -#[sqlx(type_name = "TRIGGER_MODE", rename_all = "lowercase")] -#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))] -pub enum TriggerMode { - Enabled, - Disabled, - Suspended, -} diff --git a/backend/windmill-api/src/triggers/mqtt/handler.rs b/backend/windmill-api/src/triggers/mqtt/handler.rs deleted file mode 100644 index ac4c027fff..0000000000 --- a/backend/windmill-api/src/triggers/mqtt/handler.rs +++ /dev/null @@ -1,237 +0,0 @@ -use crate::{ - db::{ApiAuthed, DB}, - resources::try_get_resource_from_db_as, - triggers::{Trigger, TriggerCrud, TriggerData}, -}; -use axum::async_trait; -use itertools::Itertools; -use sqlx::{types::Json as SqlxJson, PgConnection}; -use windmill_common::{ - db::UserDB, - error::{Error, Result}, -}; -use windmill_git_sync::DeployedObject; - -use super::{ - MqttClientBuilder, MqttClientVersion, MqttConfig, MqttConfigRequest, MqttResource, MqttTrigger, - MqttV3Config, MqttV5Config, SubscribeTopic, TestMqttConfig, -}; - -#[async_trait] -impl TriggerCrud for MqttTrigger { - type TriggerConfig = MqttConfig; - type Trigger = Trigger; - type TriggerConfigRequest = MqttConfigRequest; - type TestConnectionConfig = TestMqttConfig; - - const TABLE_NAME: &'static str = "mqtt_trigger"; - const TRIGGER_TYPE: &'static str = "mqtt"; - const SUPPORTS_SERVER_STATE: bool = true; - const SUPPORTS_TEST_CONNECTION: bool = true; - const ROUTE_PREFIX: &'static str = "/mqtt_triggers"; - const DEPLOYMENT_NAME: &'static str = "MQTT trigger"; - const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[ - "mqtt_resource_path", - "subscribe_topics", - "v3_config", - "v5_config", - "client_id", - "client_version", - ]; - const IS_ALLOWED_ON_CLOUD: bool = false; - - fn get_deployed_object(path: String) -> DeployedObject { - DeployedObject::MqttTrigger { path } - } - - async fn validate_config( - &self, - _db: &DB, - config: &Self::TriggerConfigRequest, - _workspace_id: &str, - ) -> Result<()> { - if config.mqtt_resource_path.trim().is_empty() { - return Err(Error::BadRequest( - "MQTT resource path cannot be empty".to_string(), - )); - } - - if config.subscribe_topics.is_empty() { - return Err(Error::BadRequest( - "At least one subscribe topic must be specified".to_string(), - )); - } - - Ok(()) - } - - async fn create_trigger( - &self, - _db: &DB, - tx: &mut PgConnection, - authed: &ApiAuthed, - w_id: &str, - trigger: TriggerData, - ) -> Result<()> { - let subscribe_topics = trigger - .config - .subscribe_topics - .into_iter() - .map(SqlxJson) - .collect_vec(); - let v3_config = trigger.config.v3_config.map(SqlxJson); - let v5_config = trigger.config.v5_config.map(SqlxJson); - - sqlx::query!( - r#" - INSERT INTO mqtt_trigger ( - mqtt_resource_path, - subscribe_topics, - client_version, - client_id, - v3_config, - v5_config, - workspace_id, - path, - script_path, - is_flow, - email, - mode, - edited_by, - error_handler_path, - error_handler_args, - retry - ) - VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16 - )"#, - trigger.config.mqtt_resource_path, - subscribe_topics.as_slice() as &[SqlxJson], - trigger.config.client_version as Option, - trigger.config.client_id, - v3_config as Option>, - v5_config as Option>, - w_id, - trigger.base.path, - trigger.base.script_path, - trigger.base.is_flow, - authed.email, - trigger.base.mode() as _, - authed.username, - trigger.error_handling.error_handler_path, - trigger.error_handling.error_handler_args as _, - trigger.error_handling.retry as _ - ) - .execute(tx) - .await?; - - Ok(()) - } - - async fn update_trigger( - &self, - _db: &DB, - tx: &mut PgConnection, - authed: &ApiAuthed, - workspace_id: &str, - path: &str, - trigger: TriggerData, - ) -> Result<()> { - let subscribe_topics = trigger - .config - .subscribe_topics - .into_iter() - .map(SqlxJson) - .collect_vec(); - let v3_config = trigger.config.v3_config.map(SqlxJson); - let v5_config = trigger.config.v5_config.map(SqlxJson); - - // Important to set server_id to NULL to stop current mqtt listener - sqlx::query!( - r#" - UPDATE - mqtt_trigger - SET - mqtt_resource_path = $1, - subscribe_topics = $2, - client_version = $3, - client_id = $4, - v3_config = $5, - v5_config = $6, - is_flow = $7, - edited_by = $8, - email = $9, - script_path = $10, - path = $11, - edited_at = now(), - error = NULL, - server_id = NULL, - error_handler_path = $14, - error_handler_args = $15, - retry = $16 - WHERE - workspace_id = $12 AND - path = $13 - "#, - trigger.config.mqtt_resource_path, - subscribe_topics.as_slice() as &[SqlxJson], - trigger.config.client_version as Option, - trigger.config.client_id, - v3_config as Option>, - v5_config as Option>, - trigger.base.is_flow, - authed.username, - authed.email, - trigger.base.script_path, - trigger.base.path, - workspace_id, - path, - trigger.error_handling.error_handler_path, - trigger.error_handling.error_handler_args as _, - trigger.error_handling.retry as _ - ) - .execute(tx) - .await?; - - Ok(()) - } - - async fn test_connection( - &self, - db: &DB, - authed: &ApiAuthed, - user_db: &UserDB, - workspace_id: &str, - config: Self::TestConnectionConfig, - ) -> Result<()> { - let mqtt_resource = try_get_resource_from_db_as::( - authed, - Some(user_db.clone()), - db, - &config.mqtt_resource_path, - workspace_id, - ) - .await?; - - let connect_f = async { - let client_builder = MqttClientBuilder::new( - mqtt_resource, - Some(""), - vec![], - config.v3_config.as_ref(), - config.v5_config.as_ref(), - config.client_version.as_ref(), - ); - - client_builder.build_client().await.map_err(|err| { - Error::BadConfig(format!( - "Error connecting to mqtt broker: {}", - err.to_string() - )) - }) - }; - - connect_f.await?; - Ok(()) - } -} diff --git a/backend/windmill-api/src/triggers/mqtt/listener.rs b/backend/windmill-api/src/triggers/mqtt/listener.rs deleted file mode 100644 index af2d94b113..0000000000 --- a/backend/windmill-api/src/triggers/mqtt/listener.rs +++ /dev/null @@ -1,376 +0,0 @@ -use std::{collections::HashMap, sync::Arc}; - -use async_trait::async_trait; -use bytes::Bytes; -use rumqttc::{ - v5::{ - mqttbytes::v5::PublishProperties, Event as V5Event, EventLoop as V5EventLoop, - Incoming as V5Incoming, - }, - Event as V3Event, EventLoop as V3EventLoop, Incoming as V3Incoming, -}; -use std::time::Duration; -use tokio::sync::RwLock; -use windmill_common::{ - db::UserDB, - error::{to_anyhow, Error, Result}, - jobs::JobTriggerKind, - worker::to_raw_value, - DB, -}; - -use crate::{ - resources::try_get_resource_from_db_as, - triggers::{ - listener::ListeningTrigger, - mqtt::{ - MqttClientBuilder, MqttClientResult, MqttConfig, MqttResource, MqttTrigger, - V3MqttHandler, V5MqttHandler, - }, - trigger_helpers::TriggerJobArgs, - Listener, - }, -}; - -#[async_trait] -impl Listener for MqttTrigger { - type Consumer = MqttClientResult; - type Extra = (); - type ExtraState = (); - const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Mqtt; - - async fn get_consumer( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - _err_message: Arc>>, - _killpill_rx: tokio::sync::broadcast::Receiver<()>, - ) -> Result> { - let ListeningTrigger:: { workspace_id, trigger_config, .. } = - listening_trigger; - - let MqttConfig { - mqtt_resource_path, - subscribe_topics, - v3_config, - v5_config, - client_id, - client_version, - .. - } = trigger_config; - - let authed = listening_trigger - .authed(db, &Self::TRIGGER_KIND.to_string()) - .await?; - - let mqtt_resource = try_get_resource_from_db_as::( - &authed, - Some(UserDB::new(db.clone())), - &db, - mqtt_resource_path, - workspace_id, - ) - .await?; - - let subscribe_topics = subscribe_topics - .iter() - .map(|topic| topic.0.clone()) - .collect(); - - let client_builder = MqttClientBuilder::new( - mqtt_resource, - client_id.as_deref(), - subscribe_topics, - v3_config.as_ref().map(|c| &c.0), - v5_config.as_ref().map(|c| &c.0), - client_version.as_ref(), - ); - - let client_result = client_builder - .build_client() - .await - .map_err(|e| Error::BadConfig(format!("Failed to build MQTT client: {}", e)))?; - - Ok(Some(client_result)) - } - - async fn consume( - &self, - db: &DB, - consumer: Self::Consumer, - listening_trigger: &ListeningTrigger, - _err_message: Arc>>, - _killpill_rx: tokio::sync::broadcast::Receiver<()>, - _extra_state: Option<&Self::ExtraState>, - ) { - tracing::info!( - "Starting to listen for MQTT trigger {}", - &listening_trigger.path - ); - - match consumer { - MqttClientResult::V3((v3_handler, event_loop)) => { - handle_event(&db, self, listening_trigger, v3_handler, event_loop).await - } - MqttClientResult::V5((v5_handler, event_loop)) => { - handle_event(&db, self, listening_trigger, v5_handler, event_loop).await - } - } - } -} - -const TIMEOUT_DURATION: u64 = 10; -const CONNECTION_TIMEOUT: Duration = Duration::from_secs(TIMEOUT_DURATION); - -fn convert_disconnect_packet_into_string( - disconnect: rumqttc::v5::mqttbytes::v5::Disconnect, -) -> String { - let err_message = disconnect - .properties - .map(|properties| properties.reason_string) - .flatten(); - let reason_code = disconnect.reason_code as u8; - format!( - "Disconnected by the broker, reason code: {}, {}", - reason_code, - err_message - .map(|err| format!("message: {}", err)) - .unwrap_or("".to_string()) - ) -} - -#[async_trait] -pub trait EventLoop { - type Event; - type Error; - - async fn poll(&mut self) -> Result; - async fn verify_connection(&mut self) -> Result<()>; -} - -#[async_trait] -impl EventLoop for V5EventLoop { - type Event = V5Event; - type Error = rumqttc::v5::ConnectionError; - - async fn poll(&mut self) -> Result { - self.poll().await.map_err(|err| to_anyhow(err).into()) - } - - async fn verify_connection(&mut self) -> Result<()> { - let start = std::time::Instant::now(); - - while start.elapsed() < CONNECTION_TIMEOUT { - match self.poll().await.map_err(to_anyhow)? { - Self::Event::Incoming(V5Incoming::ConnAck(_)) => return Ok(()), - Self::Event::Incoming(V5Incoming::Disconnect(disconnect)) => { - return Err(Error::BadConfig(convert_disconnect_packet_into_string( - disconnect, - ))); - } - _ => continue, - } - } - - Err(Error::BadConfig(format!( - "Timeout occurred while trying to connect to mqtt broker after {} seconds", - TIMEOUT_DURATION - ))) - } -} - -#[async_trait] -impl EventLoop for V3EventLoop { - type Event = V3Event; - type Error = rumqttc::ConnectionError; - - async fn poll(&mut self) -> Result { - self.poll().await.map_err(|err| to_anyhow(err).into()) - } - - async fn verify_connection(&mut self) -> Result<()> { - let start = std::time::Instant::now(); - - while start.elapsed() < CONNECTION_TIMEOUT { - match self.poll().await.map_err(to_anyhow)? { - Self::Event::Incoming(rumqttc::Packet::ConnAck(_)) => return Ok(()), - _ => continue, - } - } - - Err(Error::BadConfig(format!( - "Timeout occurred while trying to connect to mqtt broker after {} seconds", - TIMEOUT_DURATION - ))) - } -} - -async fn handle_event( - db: &DB, - listener: &T, - listening_trigger: &ListeningTrigger, - handler: H, - mut event_loop: E, -) -> () -where - T: Listener, - H: MqttEvent, - E: EventLoop, - E::Error: ToString, - ::Payload: From, -{ - loop { - let event = event_loop.poll().await; - - match event { - Ok(event) => { - let publish_data = handler.handle_event(event); - if let Ok(Some((payload, publish_data))) = publish_data { - let trigger_info = HashMap::from([ - ("topic".to_string(), to_raw_value(&publish_data.topic)), - ("retain".to_string(), to_raw_value(&publish_data.retain)), - ("pkid".to_string(), to_raw_value(&publish_data.pkid)), - ("qos".to_string(), to_raw_value(&publish_data.qos)), - ( - "v5".to_string(), - to_raw_value(&publish_data.v5.map(|properties| { - serde_json::json!({ - "payload_format_indicator": properties.payload_format_indicator, - "topic_alias": properties.topic_alias, - "response_topic": properties.response_topic, - "correlation_data": properties.correlation_data.as_deref(), - "user_properties": properties.user_properties, - "subscription_identifiers": properties.subscription_identifiers, - "content_type": properties.content_type, - }) - })), - ), - ]); - let _ = listener - .handle_event(db, listening_trigger, payload.into(), trigger_info, None) - .await; - } - } - Err(err) => { - let error = err.to_string(); - tracing::debug!("Error: {}", &err); - listener - .disable_with_error(db, listening_trigger, error) - .await; - return; - } - } - } -} - -#[derive(Clone)] -#[allow(unused)] -pub struct PublishData { - topic: String, - retain: bool, - pkid: u16, - v5: Option, - qos: u8, -} - -impl PublishData { - pub fn new( - topic: String, - retain: bool, - pkid: u16, - v5: Option, - qos: u8, - ) -> PublishData { - PublishData { topic, retain, pkid, v5, qos } - } -} - -trait MqttEvent { - type IncomingPacket; - type PublishPacket; - type Event; - - fn handle_publish_packet(publish_packet: Self::PublishPacket) -> PublishData; - fn handle_event(&self, event: Self::Event) -> Result>; -} - -impl MqttEvent for V5MqttHandler { - type IncomingPacket = V5Incoming; - type PublishPacket = rumqttc::v5::mqttbytes::v5::Publish; - type Event = V5Event; - - fn handle_publish_packet(publish_packet: Self::PublishPacket) -> PublishData { - PublishData::new( - String::from_utf8(publish_packet.topic.as_ref().to_vec()).unwrap_or("".to_string()), - publish_packet.retain, - publish_packet.pkid, - publish_packet.properties, - publish_packet.qos as u8, - ) - } - - fn handle_event(&self, event: Self::Event) -> Result> { - tracing::debug!("Inside V5 event"); - match event { - Self::Event::Incoming(packet) => match packet { - Self::IncomingPacket::Publish(publish_packet) => { - return Ok(Some(( - publish_packet.payload.clone(), - Self::handle_publish_packet(publish_packet), - ))) - } - Self::IncomingPacket::Disconnect(disconnect) => { - return Err( - anyhow::anyhow!(convert_disconnect_packet_into_string(disconnect)).into(), - ); - } - packet => { - tracing::debug!("Received = {:#?}", packet); - } - }, - Self::Event::Outgoing(packet) => { - tracing::debug!("Outgoing Received = {:#?}", packet); - } - } - - Ok(None) - } -} - -impl MqttEvent for V3MqttHandler { - type IncomingPacket = V3Incoming; - type PublishPacket = rumqttc::mqttbytes::v4::Publish; - type Event = V3Event; - - fn handle_publish_packet(publish_packet: Self::PublishPacket) -> PublishData { - PublishData::new( - publish_packet.topic, - publish_packet.retain, - publish_packet.pkid, - None, - publish_packet.qos as u8, - ) - } - - fn handle_event(&self, event: Self::Event) -> Result> { - tracing::debug!("Inside V3 event"); - match event { - Self::Event::Incoming(packet) => match packet { - Self::IncomingPacket::Publish(publish_packet) => { - return Ok(Some(( - publish_packet.payload.clone(), - Self::handle_publish_packet(publish_packet), - ))) - } - packet => { - tracing::debug!("Received = {:?}", packet); - } - }, - Self::Event::Outgoing(packet) => { - tracing::debug!("Outgoing Received = {:?}", packet); - } - } - - Ok(None) - } -} diff --git a/backend/windmill-api/src/triggers/mqtt/mod.rs b/backend/windmill-api/src/triggers/mqtt/mod.rs deleted file mode 100644 index cfc8e36900..0000000000 --- a/backend/windmill-api/src/triggers/mqtt/mod.rs +++ /dev/null @@ -1,346 +0,0 @@ -use base64::{engine, prelude::*}; -use itertools::Itertools; -use rumqttc::{ - v5::{ - mqttbytes::{ - v5::{ConnectProperties, Filter}, - QoS as V5QoS, - }, - AsyncClient as V5AsyncClient, EventLoop as V5EventLoop, MqttOptions as V5MqttOptions, - }, - AsyncClient as V3AsyncClient, EventLoop as V3EventLoop, MqttOptions as V3MqttOptions, - QoS as V3QoS, SubscribeFilter, TlsConfiguration, Transport, -}; -use serde::{Deserialize, Serialize}; -use serde_json::value::RawValue; -use sqlx::{types::Json as SqlxJson, FromRow, Type}; -use std::{collections::HashMap, time::Duration}; -use windmill_common::{ - error::{to_anyhow, Error}, - triggers::TriggerKind, - worker::to_raw_value, -}; - -use crate::triggers::{mqtt::listener::EventLoop, trigger_helpers::TriggerJobArgs}; - -pub mod handler; -pub mod listener; - -#[derive(Clone, Copy)] -pub struct MqttTrigger; - -impl TriggerJobArgs for MqttTrigger { - type Payload = Vec; - const TRIGGER_KIND: TriggerKind = TriggerKind::Mqtt; - - fn v1_payload_fn(payload: &Self::Payload) -> HashMap> { - HashMap::from([("payload".to_string(), to_raw_value(&payload))]) - } - - fn v2_payload_fn(payload: &Self::Payload) -> HashMap> { - let base64_payload = engine::general_purpose::STANDARD.encode(payload); - HashMap::from([("payload".to_string(), to_raw_value(&base64_payload))]) - } -} - -#[derive(Clone, Debug, Deserialize, Serialize, Type)] -#[serde(rename_all = "lowercase")] -pub enum QualityOfService { - Qos0, - Qos1, - Qos2, -} - -impl From for V3QoS { - fn from(value: QualityOfService) -> Self { - match value { - QualityOfService::Qos0 => V3QoS::AtMostOnce, - QualityOfService::Qos1 => V3QoS::AtLeastOnce, - QualityOfService::Qos2 => V3QoS::ExactlyOnce, - } - } -} - -impl From for V5QoS { - fn from(value: QualityOfService) -> Self { - match value { - QualityOfService::Qos0 => V5QoS::AtMostOnce, - QualityOfService::Qos1 => V5QoS::AtLeastOnce, - QualityOfService::Qos2 => V5QoS::ExactlyOnce, - } - } -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct MqttV3Config { - clean_session: Option, -} - -#[derive(Clone, Debug, Deserialize, Serialize)] -pub struct MqttV5Config { - clean_start: Option, - session_expiry_interval: Option, - topic_alias_maximum: Option, -} - -#[derive(Clone, Debug, Deserialize, Serialize, Type)] -#[sqlx(type_name = "MQTT_CLIENT_VERSION")] -#[sqlx(rename_all = "lowercase")] -#[serde(rename_all = "lowercase")] -pub enum MqttClientVersion { - V3, - V5, -} - -#[derive(Debug, Deserialize)] -pub struct Tls { - enabled: bool, - ca_certificate: String, - pkcs12_client_certificate: Option, - pkcs12_certificate_password: Option, -} - -#[derive(Debug, Deserialize)] -pub struct Credentials { - username: Option, - password: Option, -} - -#[derive(Debug, Deserialize)] -pub struct MqttResource { - broker: String, - port: u16, - credentials: Option, - tls: Option, -} - -#[derive(Clone, Debug, FromRow, Serialize, Deserialize)] -pub struct SubscribeTopic { - qos: QualityOfService, - topic: String, -} - -#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] -pub struct MqttConfig { - pub mqtt_resource_path: String, - pub subscribe_topics: Vec>, - pub v3_config: Option>, - pub v5_config: Option>, - pub client_id: Option, - pub client_version: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MqttConfigRequest { - pub mqtt_resource_path: String, - pub subscribe_topics: Vec, - pub v3_config: Option, - pub v5_config: Option, - pub client_id: Option, - pub client_version: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TestMqttConfig { - pub mqtt_resource_path: String, - pub client_version: Option, - pub v3_config: Option, - pub v5_config: Option, -} - -// Constants -pub const KEEP_ALIVE: u64 = 60; -pub const CLIENT_CONNECTION_TIMEOUT: u64 = 60; -pub const TOPIC_ALIAS_MAXIMUM: u16 = 65535; -pub const TIMEOUT_DURATION: u64 = 10; -pub const CONNECTION_TIMEOUT: Duration = Duration::from_secs(TIMEOUT_DURATION); - -pub struct V3MqttHandler; -pub struct V5MqttHandler; - -pub enum MqttClientResult { - V3((V3MqttHandler, V3EventLoop)), - V5((V5MqttHandler, V5EventLoop)), -} - -#[derive(Debug, thiserror::Error)] -pub enum MqttError { - #[error("{0}")] - Common(#[from] Error), - #[error("{0}")] - V5RumqttClient(#[from] rumqttc::v5::ClientError), - #[error("{0}")] - V5ConnectionError(#[from] rumqttc::v5::ConnectionError), - #[error("{0}")] - V3RumqttClient(#[from] rumqttc::ClientError), - #[error("{0}")] - V3ConnectionError(#[from] rumqttc::ConnectionError), - #[error("{0}")] - Base64Decode(#[from] base64::DecodeError), -} - -pub struct MqttClientBuilder<'client> { - mqtt_resource: MqttResource, - client_id: &'client str, - subscribe_topics: Vec, - v3_config: Option<&'client MqttV3Config>, - v5_config: Option<&'client MqttV5Config>, - mqtt_client_version: Option<&'client MqttClientVersion>, -} - -impl<'client> MqttClientBuilder<'client> { - pub fn new( - mqtt_resource: MqttResource, - client_id: Option<&'client str>, - subscribe_topics: Vec, - v3_config: Option<&'client MqttV3Config>, - v5_config: Option<&'client MqttV5Config>, - mqtt_client_version: Option<&'client MqttClientVersion>, - ) -> Self { - Self { - mqtt_resource, - client_id: client_id.unwrap_or(""), - subscribe_topics, - v3_config, - v5_config, - mqtt_client_version, - } - } - - pub async fn build_client(&self) -> Result { - match self.mqtt_client_version { - Some(MqttClientVersion::V5) | None => self.build_v5_client().await, - Some(MqttClientVersion::V3) => self.build_v3_client().await, - } - } - - fn get_tls_configuration(&self) -> Result, Error> { - let transport = match self.mqtt_resource.tls { - Some(ref tls) if tls.enabled => { - let transport = match tls.ca_certificate.trim().is_empty() { - true => rumqttc::Transport::Tls(TlsConfiguration::Native), - false => rumqttc::Transport::Tls(TlsConfiguration::SimpleNative { - ca: tls.ca_certificate.as_bytes().to_vec(), - client_auth: { - match tls.pkcs12_client_certificate.as_ref() { - Some(client_certificate) - if !client_certificate.trim().is_empty() => - { - let client_certificate = BASE64_STANDARD - .decode(client_certificate) - .map_err(to_anyhow)?; - let password = tls - .pkcs12_certificate_password - .clone() - .unwrap_or("".to_string()); - Some((client_certificate, password)) - } - _ => None, - } - }, - }), - }; - - Some(transport) - } - _ => None, - }; - - Ok(transport) - } - - async fn build_v5_client(&self) -> Result { - let mut mqtt_options = V5MqttOptions::new( - self.client_id, - &self.mqtt_resource.broker, - self.mqtt_resource.port, - ); - - if let Some(credentials) = &self.mqtt_resource.credentials { - let username = credentials.username.as_deref().unwrap_or(""); - let password = credentials.password.as_deref().unwrap_or(""); - mqtt_options.set_credentials(username, password); - } - - if let Some(transport) = self.get_tls_configuration()? { - mqtt_options.set_transport(transport); - } - - mqtt_options.set_connection_timeout(CLIENT_CONNECTION_TIMEOUT); - - mqtt_options.set_keep_alive(Duration::from_secs(KEEP_ALIVE)); - - if let Some(v5_config) = self.v5_config { - mqtt_options.set_clean_start(v5_config.clean_start.unwrap_or(true)); - mqtt_options.set_connect_properties(ConnectProperties { - session_expiry_interval: v5_config.session_expiry_interval, - receive_maximum: None, - max_packet_size: None, - topic_alias_max: v5_config.topic_alias_maximum.or(Some(TOPIC_ALIAS_MAXIMUM)), - request_response_info: None, - request_problem_info: None, - user_properties: vec![], - authentication_method: None, - authentication_data: None, - }); - } - - let (async_client, mut event_loop) = - V5AsyncClient::new(mqtt_options, self.subscribe_topics.len()); - event_loop.verify_connection().await?; - - if !self.subscribe_topics.is_empty() { - let subscribe_filters = self - .subscribe_topics - .iter() - .map(|topic| Filter::new(topic.topic.clone(), topic.qos.clone().into())) - .collect_vec(); - - async_client - .subscribe_many(subscribe_filters) - .await - .map_err(to_anyhow)?; - } - Ok(MqttClientResult::V5((V5MqttHandler, event_loop))) - } - - async fn build_v3_client(&self) -> Result { - let mut mqtt_options = V3MqttOptions::new( - self.client_id, - &self.mqtt_resource.broker, - self.mqtt_resource.port, - ); - - if let Some(credentials) = &self.mqtt_resource.credentials { - let username = credentials.username.as_deref().unwrap_or(""); - let password = credentials.password.as_deref().unwrap_or(""); - mqtt_options.set_credentials(username, password); - } - - if let Some(transport) = self.get_tls_configuration()? { - mqtt_options.set_transport(transport); - } - mqtt_options.set_keep_alive(Duration::from_secs(KEEP_ALIVE)); - if let Some(v3_config) = self.v3_config { - mqtt_options.set_clean_session(v3_config.clean_session.unwrap_or(true)); - } - - let (async_client, mut event_loop) = - V3AsyncClient::new(mqtt_options, self.subscribe_topics.len()); - event_loop.verify_connection().await?; - - if !self.subscribe_topics.is_empty() { - let subscribe_filters = self - .subscribe_topics - .iter() - .map(|topic| SubscribeFilter::new(topic.topic.clone(), topic.qos.clone().into())) - .collect_vec(); - - async_client - .subscribe_many(subscribe_filters) - .await - .map_err(to_anyhow)?; - } - Ok(MqttClientResult::V3((V3MqttHandler, event_loop))) - } -} diff --git a/backend/windmill-api/src/triggers/nats/handler_oss.rs b/backend/windmill-api/src/triggers/nats/handler_oss.rs deleted file mode 100644 index 3ab3430e5f..0000000000 --- a/backend/windmill-api/src/triggers/nats/handler_oss.rs +++ /dev/null @@ -1,64 +0,0 @@ -#[allow(unused)] -#[cfg(feature = "private")] -pub use super::handler_ee::*; - -#[cfg(not(feature = "private"))] -use { - super::NatsTrigger, - crate::{ - db::{ApiAuthed, DB}, - triggers::{TriggerCrud, TriggerData}, - }, - axum::async_trait, - sqlx::PgConnection, - windmill_common::error::{Error, Result}, - windmill_git_sync::DeployedObject, -}; - -#[cfg(not(feature = "private"))] -#[async_trait] -impl TriggerCrud for NatsTrigger { - type Trigger = (); - type TriggerConfig = (); - type TriggerConfigRequest = (); - type TestConnectionConfig = (); - - const TABLE_NAME: &'static str = ""; - const TRIGGER_TYPE: &'static str = ""; - const SUPPORTS_SERVER_STATE: bool = false; - const SUPPORTS_TEST_CONNECTION: bool = false; - const ROUTE_PREFIX: &'static str = "/nats_triggers"; - const DEPLOYMENT_NAME: &'static str = ""; - const IS_ALLOWED_ON_CLOUD: bool = false; - - fn get_deployed_object(path: String) -> DeployedObject { - DeployedObject::NatsTrigger { path } - } - - async fn create_trigger( - &self, - _db: &DB, - _executor: &mut PgConnection, - _authed: &ApiAuthed, - _w_id: &str, - _trigger: TriggerData, - ) -> Result<()> { - Err(Error::BadRequest( - "NATS triggers are not available in open source version".to_string(), - )) - } - - async fn update_trigger( - &self, - _db: &DB, - _executor: &mut PgConnection, - _authed: &ApiAuthed, - _workspace_id: &str, - _path: &str, - _trigger: TriggerData, - ) -> Result<()> { - Err(Error::BadRequest( - "NATS triggers are not available in open source version".to_string(), - )) - } -} diff --git a/backend/windmill-api/src/triggers/nats/listener_oss.rs b/backend/windmill-api/src/triggers/nats/listener_oss.rs deleted file mode 100644 index 80f1befded..0000000000 --- a/backend/windmill-api/src/triggers/nats/listener_oss.rs +++ /dev/null @@ -1,41 +0,0 @@ -#[allow(unused)] - -#[cfg(feature = "private")] -pub use super::listener_ee::*; - -#[cfg(not(feature = "private"))] -use { - super::NatsTrigger, - crate::triggers::{listener::ListeningTrigger, Listener}, - std::sync::Arc, - tokio::sync::RwLock, - windmill_common::{error::Result, jobs::JobTriggerKind, DB}, -}; - -#[cfg(not(feature = "private"))] -#[async_trait::async_trait] -impl Listener for NatsTrigger { - type Consumer = (); - type Extra = (); - const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Nats; - - async fn get_consumer( - &self, - _db: &DB, - _listening_trigger: &ListeningTrigger, - _err_message: Arc>>, - _killpill_rx: tokio::sync::broadcast::Receiver<()>, - ) -> Result> { - Ok(None) - } - async fn consume( - &self, - _db: &DB, - _consumer: Self::Consumer, - _listening_trigger: &ListeningTrigger, - _err_message: Arc>>, - _killpill_rx: tokio::sync::broadcast::Receiver<()>, - ) { - () - } -} diff --git a/backend/windmill-api/src/triggers/nats/mod.rs b/backend/windmill-api/src/triggers/nats/mod.rs deleted file mode 100644 index 668df073c9..0000000000 --- a/backend/windmill-api/src/triggers/nats/mod.rs +++ /dev/null @@ -1,15 +0,0 @@ -#[cfg(feature = "private")] -mod handler_ee; -pub mod handler_oss; - -#[cfg(feature = "private")] -mod listener_ee; -pub mod listener_oss; - -#[cfg(feature = "private")] -mod mod_ee; -#[cfg(feature = "private")] -pub use mod_ee::*; - -#[derive(Copy, Clone)] -pub struct NatsTrigger; diff --git a/backend/windmill-api/src/triggers/postgres/bool.rs b/backend/windmill-api/src/triggers/postgres/bool.rs deleted file mode 100644 index 13b0a8fa68..0000000000 --- a/backend/windmill-api/src/triggers/postgres/bool.rs +++ /dev/null @@ -1,24 +0,0 @@ -use thiserror::Error; - -/** -* This implementation is inspired by Postgres replication functionality -* from https://github.com/supabase/pg_replicate -* -* Original implementation: -* - https://github.dev/supabase/pg_replicate/blob/main/pg_replicate/src/conversions/bool.rs -* -*/ - -#[derive(Debug, Error)] -pub enum ParseBoolError { - #[error("invalid input value: {0}")] - InvalidInput(String), -} - -pub fn parse_bool(s: &str) -> Result { - match s { - "t" => Ok(true), - "f" => Ok(false), - _ => Err(ParseBoolError::InvalidInput(s.to_string())), - } -} diff --git a/backend/windmill-api/src/triggers/postgres/converter.rs b/backend/windmill-api/src/triggers/postgres/converter.rs deleted file mode 100644 index f57268c4b8..0000000000 --- a/backend/windmill-api/src/triggers/postgres/converter.rs +++ /dev/null @@ -1,255 +0,0 @@ -use core::str; -use std::{ - num::{ParseFloatError, ParseIntError}, - str::FromStr, -}; - -use super::{ - bool::{parse_bool, ParseBoolError}, - hex::{from_bytea_hex, ByteaHexParseError}, -}; -use chrono::{DateTime, FixedOffset, NaiveDate, NaiveDateTime, NaiveTime, Utc}; -use rust_decimal::Decimal; -use rust_postgres::types::Type; -use serde_json::{to_value, Number, Value}; -use thiserror::Error; -use uuid::Uuid; - -/** -* This implementation is inspired by Postgres replication functionality -* from https://github.com/supabase/pg_replicate -* -* Original implementation: -* - https://github.com/supabase/pg_replicate/blob/main/pg_replicate/src/conversions/text.rs -* -*/ - -#[derive(Debug, Error)] -pub enum ConverterError { - #[error("invalid bool value")] - InvalidBool(#[from] ParseBoolError), - - #[error("invalid int value")] - InvalidInt(#[from] ParseIntError), - - #[error("invalid float value")] - InvalidFloat(#[from] ParseFloatError), - - #[error("invalid numeric: {0}")] - InvalidNumeric(#[from] rust_decimal::Error), - - #[error("invalid bytea: {0}")] - InvalidBytea(#[from] ByteaHexParseError), - - #[error("invalid uuid: {0}")] - InvalidUuid(#[from] uuid::Error), - - #[error("invalid json: {0}")] - InvalidJson(#[from] serde_json::Error), - - #[error("invalid timestamp: {0} ")] - InvalidTimestamp(#[from] chrono::ParseError), - - #[error("invalid array: {0}")] - InvalidArray(#[from] ArrayParseError), - - #[error("{0}")] - Custom(String), -} - -fn convert_into(number: T) -> Number -where - T: Sized, - serde_json::Number: From, -{ - serde_json::Number::from(number) -} - -pub struct Converter; - -#[derive(Debug, Error)] -pub enum ArrayParseError { - #[error("input too short")] - InputTooShort, - - #[error("missing braces")] - MissingBraces, -} - -fn f64_to_json_number(raw_val: f64) -> Result { - let temp = serde_json::Number::from_f64(raw_val.into()) - .ok_or(ConverterError::Custom("invalid json-float".to_string()))?; - Ok(Value::Number(temp)) -} - -impl Converter { - pub fn try_from_str(typ: Option, str: &str) -> Result { - let value = match typ.unwrap_or(Type::TEXT) { - Type::BOOL => Value::Bool(parse_bool(str)?), - Type::BOOL_ARRAY => { - Converter::parse_array(str, |str| Ok(Value::Bool(parse_bool(str)?)))? - } - Type::CHAR | Type::BPCHAR | Type::VARCHAR | Type::NAME | Type::TEXT => { - Value::String(str.to_string()) - } - Type::CHAR_ARRAY - | Type::BPCHAR_ARRAY - | Type::VARCHAR_ARRAY - | Type::NAME_ARRAY - | Type::TEXT_ARRAY => { - Converter::parse_array(str, |str| Ok(Value::String(str.to_string())))? - } - Type::INT2 => Value::Number(convert_into(str.parse::()?)), - Type::INT2_ARRAY => Converter::parse_array(str, |str| { - Ok(Value::Number(convert_into(str.parse::()?))) - })?, - Type::INT4 => Value::Number(convert_into(str.parse::()?)), - Type::INT4_ARRAY => Converter::parse_array(str, |str| { - Ok(Value::Number(convert_into(str.parse::()?))) - })?, - Type::INT8 => Value::Number(convert_into(str.parse::()?)), - Type::INT8_ARRAY => Converter::parse_array(str, |str| { - Ok(Value::Number(convert_into(str.parse::()?))) - })?, - Type::FLOAT4 => f64_to_json_number(str.parse::()?)?, - Type::FLOAT4_ARRAY => { - Converter::parse_array(str, |str| f64_to_json_number(str.parse::()?))? - } - Type::FLOAT8 => f64_to_json_number(str.parse::()?)?, - Type::FLOAT8_ARRAY => { - Converter::parse_array(str, |str| f64_to_json_number(str.parse::()?))? - } - Type::NUMERIC => serde_json::json!(Decimal::from_str(str)?), - Type::NUMERIC_ARRAY => { - Converter::parse_array(str, |str| Ok(serde_json::json!(Decimal::from_str(str)?)))? - } - Type::BYTEA => to_value(from_bytea_hex(str)?).unwrap(), - Type::BYTEA_ARRAY => { - Converter::parse_array(str, |str| Ok(to_value(from_bytea_hex(str)?).unwrap()))? - } - Type::DATE => { - let date = NaiveDate::parse_from_str(str, "%Y-%m-%d")?; - Value::String(date.to_string()) - } - Type::DATE_ARRAY => Converter::parse_array(str, |str| { - let date = NaiveDate::parse_from_str(str, "%Y-%m-%d")?; - Ok(Value::String(date.to_string())) - })?, - Type::TIME => { - let time = NaiveTime::parse_from_str(str, "%H:%M:%S%.f")?; - Value::String(time.to_string()) - } - Type::TIME_ARRAY => Converter::parse_array(str, |str| { - let time = NaiveTime::parse_from_str(str, "%H:%M:%S%.f")?; - Ok(Value::String(time.to_string())) - })?, - Type::TIMESTAMP => { - let timestamp = NaiveDateTime::parse_from_str(str, "%Y-%m-%d %H:%M:%S%.f")?; - Value::String(timestamp.to_string()) - } - Type::TIMESTAMP_ARRAY => Converter::parse_array(str, |str| { - let timestamp = NaiveDateTime::parse_from_str(str, "%Y-%m-%d %H:%M:%S%.f")?; - Ok(Value::String(timestamp.to_string())) - })?, - Type::TIMESTAMPTZ => { - let val = - match DateTime::::parse_from_str(str, "%Y-%m-%d %H:%M:%S%.f%#z") { - Ok(val) => val, - Err(_) => { - DateTime::::parse_from_str(str, "%Y-%m-%d %H:%M:%S%.f%:z")? - } - }; - let utc: DateTime = val.into(); - Value::String(utc.to_string()) - } - Type::TIMESTAMPTZ_ARRAY => { - match Converter::parse_array(str, |str| { - let utc: DateTime = - DateTime::::parse_from_str(str, "%Y-%m-%d %H:%M:%S%.f%#z")? - .into(); - Ok(Value::String(utc.to_string())) - }) { - Ok(val) => val, - Err(_) => Converter::parse_array(str, |str| { - let utc: DateTime = DateTime::::parse_from_str( - str, - "%Y-%m-%d %H:%M:%S%.f%#z", - )? - .into(); - Ok(Value::String(utc.to_string())) - })?, - } - } - Type::UUID => Value::String(Uuid::parse_str(str)?.to_string()), - Type::UUID_ARRAY => Converter::parse_array(str, |str| { - Ok(Value::String(Uuid::parse_str(str)?.to_string())) - })?, - Type::JSON | Type::JSONB => serde_json::from_str::(str)?, - Type::JSON_ARRAY | Type::JSONB_ARRAY => Converter::parse_array(str, |str| { - Ok(serde_json::from_str::(str)?) - })?, - Type::OID => Value::Number(convert_into(str.parse::()?)), - Type::OID_ARRAY => Converter::parse_array(str, |str| { - Ok(Value::Number(convert_into(str.parse::()?))) - })?, - _ => Value::String(str.to_string()), - }; - - Ok(value) - } - - fn parse_array

(str: &str, mut parse: P) -> Result - where - P: FnMut(&str) -> Result, - { - if str.len() < 2 { - return Err(ArrayParseError::InputTooShort.into()); - } - - if !str.starts_with('{') || !str.ends_with('}') { - return Err(ArrayParseError::MissingBraces.into()); - } - - let mut res = vec![]; - let str = &str[1..(str.len() - 1)]; - let mut val_str = String::with_capacity(10); - let mut in_quotes = false; - let mut in_escape = false; - let mut chars = str.chars(); - let mut done = str.is_empty(); - - while !done { - loop { - match chars.next() { - Some(c) => match c { - c if in_escape => { - val_str.push(c); - in_escape = false; - } - '"' => in_quotes = !in_quotes, - '\\' => in_escape = true, - ',' if !in_quotes => { - break; - } - c => { - val_str.push(c); - } - }, - None => { - done = true; - break; - } - } - } - let val = if val_str.to_lowercase() == "null" { - Value::Null - } else { - parse(&val_str)? - }; - res.push(val); - val_str.clear(); - } - let arr = Value::Array(res); - Ok(arr) - } -} diff --git a/backend/windmill-api/src/triggers/postgres/handler.rs b/backend/windmill-api/src/triggers/postgres/handler.rs deleted file mode 100644 index c46441cd0c..0000000000 --- a/backend/windmill-api/src/triggers/postgres/handler.rs +++ /dev/null @@ -1,1103 +0,0 @@ -use std::collections::HashMap; - -use axum::{ - async_trait, - extract::Path, - routing::{delete, get, post}, - Extension, Json, Router, -}; -use itertools::Itertools; -use lazy_static::lazy_static; -use pg_escape::{quote_identifier, quote_literal}; -use quick_cache::sync::Cache; -use rust_postgres::{types::Type, Client}; -use sqlx::PgConnection; -use uuid; -use windmill_common::{ - db::UserDB, - error::{self, to_anyhow, Error, Result}, -}; -use windmill_git_sync::DeployedObject; - -use crate::{ - db::{ApiAuthed, DB}, - triggers::{postgres::PostgresTrigger, Trigger, TriggerCrud, TriggerData}, -}; - -use super::{ - check_if_valid_publication_for_postgres_version, create_logical_replication_slot, - create_pg_publication, drop_publication, generate_random_string, get_default_pg_connection, - mapper::{Mapper, MappingInfo}, - PostgresConfig, PostgresConfigRequest, PostgresPublicationReplication, PublicationData, - Relations, Slot, SlotList, TableToTrack, TemplateScript, TestPostgresConfig, - ERROR_PUBLICATION_NAME_NOT_EXISTS, -}; - -// Lazy static template cache -lazy_static! { - pub static ref TEMPLATE: Cache = Cache::new(50); -} - -#[async_trait] -impl TriggerCrud for PostgresTrigger { - type TriggerConfig = PostgresConfig; - type Trigger = Trigger; - type TriggerConfigRequest = PostgresConfigRequest; - type TestConnectionConfig = TestPostgresConfig; - - const TABLE_NAME: &'static str = "postgres_trigger"; - const TRIGGER_TYPE: &'static str = "postgres"; - const SUPPORTS_SERVER_STATE: bool = true; - const SUPPORTS_TEST_CONNECTION: bool = true; - const ROUTE_PREFIX: &'static str = "/postgres_triggers"; - const DEPLOYMENT_NAME: &'static str = "PostgreSQL trigger"; - const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[ - "postgres_resource_path", - "replication_slot_name", - "publication_name", - "NULL::text AS basic_mode", - ]; - const IS_ALLOWED_ON_CLOUD: bool = false; - - fn get_deployed_object(path: String) -> DeployedObject { - DeployedObject::PostgresTrigger { path } - } - - async fn create_trigger( - &self, - db: &DB, - tx: &mut PgConnection, - authed: &ApiAuthed, - w_id: &str, - trigger: TriggerData, - ) -> Result<()> { - let Self::TriggerConfigRequest { - postgres_resource_path, - publication_name, - replication_slot_name, - publication, - } = trigger.config; - - let (pub_name, slot_name) = - if publication_name.is_empty() && replication_slot_name.is_empty() { - if publication.is_none() { - return Err(Error::BadRequest("publication must be set".to_string())); - } - - let PostgresPublicationReplication { publication_name, replication_slot_name } = - create_custom_slot_and_publication_inner( - authed.clone(), - UserDB::new(db.clone()), - &db, - &postgres_resource_path, - &w_id, - &publication.unwrap(), - ) - .await?; - - (publication_name, replication_slot_name) - } else { - if publication_name.is_empty() { - return Err(Error::BadRequest( - "Publication name must not be empty".to_string(), - )); - } else if replication_slot_name.is_empty() { - return Err(Error::BadRequest( - "Replication slot name must not be empty".to_string(), - )); - } - (publication_name, replication_slot_name) - }; - - sqlx::query!( - r#" - INSERT INTO postgres_trigger ( - workspace_id, - path, - postgres_resource_path, - replication_slot_name, - publication_name, - script_path, - is_flow, - mode, - edited_by, - email, - edited_at, - error_handler_path, - error_handler_args, - retry - ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, now(), $11, $12, $13 - ) - "#, - w_id, - trigger.base.path, - postgres_resource_path, - slot_name, - pub_name, - trigger.base.script_path, - trigger.base.is_flow, - trigger.base.mode() as _, - authed.username, - authed.email, - trigger.error_handling.error_handler_path, - trigger.error_handling.error_handler_args as _, - trigger.error_handling.retry as _ - ) - .execute(tx) - .await?; - Ok(()) - } - - async fn update_trigger( - &self, - db: &DB, - tx: &mut PgConnection, - authed: &ApiAuthed, - w_id: &str, - path: &str, - trigger: TriggerData, - ) -> Result<()> { - let Self::TriggerConfigRequest { - replication_slot_name, - publication_name, - postgres_resource_path, - publication, - } = trigger.config; - - let mut pg_connection = get_default_pg_connection( - authed.clone(), - Some(UserDB::new(db.clone())), - db, - &postgres_resource_path, - &w_id, - ) - .await - .map_err(to_anyhow)?; - - let exists = - check_if_logical_replication_slot_exist(&mut pg_connection, &replication_slot_name) - .await?; - - let remote_db_tx = pg_connection.transaction().await.map_err(to_anyhow)?; - - if !exists { - tracing::debug!( - "Logical replication slot named: {} does not exists creating it...", - &replication_slot_name - ); - create_logical_replication_slot(remote_db_tx.client(), &replication_slot_name) - .await - .map_err(to_anyhow)?; - } - - if let Some(publication) = publication { - let publication_data = - get_publication_scope_and_transaction(remote_db_tx.client(), &publication_name) - .await - .map_err(to_anyhow)?; - - update_pg_publication( - remote_db_tx.client(), - &publication_name, - publication, - publication_data.map(|publication| publication.0), - ) - .await - .map_err(to_anyhow)?; - } - - remote_db_tx.commit().await.map_err(to_anyhow)?; - - sqlx::query!( - r#" - UPDATE postgres_trigger - SET - postgres_resource_path = $1, - replication_slot_name = $2, - publication_name = $3, - script_path = $4, - path = $5, - is_flow = $6, - edited_by = $7, - email = $8, - edited_at = now(), - server_id = NULL, - error = NULL, - error_handler_path = $11, - error_handler_args = $12, - retry = $13 - WHERE - workspace_id = $9 AND path = $10 - "#, - postgres_resource_path, - replication_slot_name, - publication_name, - trigger.base.script_path, - trigger.base.path, - trigger.base.is_flow, - authed.username, - authed.email, - w_id, - path, - trigger.error_handling.error_handler_path, - trigger.error_handling.error_handler_args as _, - trigger.error_handling.retry as _ - ) - .execute(tx) - .await?; - Ok(()) - } - - async fn test_connection( - &self, - db: &DB, - authed: &ApiAuthed, - user_db: &UserDB, - workspace_id: &str, - config: Self::TestConnectionConfig, - ) -> Result<()> { - let connect_f = async { - get_default_pg_connection( - authed.clone(), - Some(user_db.clone()), - db, - &config.postgres_resource_path, - workspace_id, - ) - .await - .map_err(|err| { - Error::BadConfig(format!("Error connecting to postgres: {}", err.to_string())) - })?; - - Ok::<(), Error>(()) - }; - - connect_f.await?; - Ok(()) - } - - fn additional_routes(&self) -> Router { - Router::new() - .route("/get_template_script/:id", get(get_template_script)) - .route("/create_template_script", post(create_template_script)) - .route( - "/is_valid_postgres_configuration/*path", - get(is_database_in_logical_level), - ) - .nest("/publication", publication_service()) - .nest("/slot", slot_service()) - .nest("/postgres", postgres_service()) - } -} - -fn publication_service() -> Router { - Router::new() - .route("/get/:publication_name/*path", get(get_publication_info)) - .route("/create/:publication_name/*path", post(create_publication)) - .route("/update/:publication_name/*path", post(alter_publication)) - .route( - "/delete/:publication_name/*path", - delete(delete_publication), - ) - .route("/list/*path", get(list_database_publication)) -} - -fn slot_service() -> Router { - Router::new() - .route("/list/*path", get(list_slot_name)) - .route("/create/*path", post(create_slot)) - .route("/delete/*path", delete(drop_slot_name)) -} - -fn postgres_service() -> Router { - Router::new().route("/version/*path", get(get_postgres_version)) -} - -async fn check_if_logical_replication_slot_exist( - pg_connection: &mut Client, - replication_slot_name: &str, -) -> Result { - let row = pg_connection - .query_opt( - "SELECT slot_name FROM pg_replication_slots WHERE slot_name = $1", - &[&replication_slot_name], - ) - .await - .map_err(to_anyhow)?; - Ok(row.is_some()) -} - -async fn create_custom_slot_and_publication_inner( - authed: ApiAuthed, - user_db: UserDB, - db: &DB, - postgres_resource_path: &str, - w_id: &str, - publication: &PublicationData, -) -> Result { - let mut pg_connection = get_default_pg_connection( - authed.clone(), - Some(user_db), - &db, - &postgres_resource_path, - &w_id, - ) - .await - .map_err(to_anyhow)?; - - let tx = pg_connection.transaction().await.map_err(to_anyhow)?; - - let publication_name = format!("windmill_trigger_{}", generate_random_string()); - let replication_slot_name = publication_name.clone(); - - create_logical_replication_slot(tx.client(), &replication_slot_name).await?; - create_pg_publication( - &tx.client(), - &publication_name, - publication.table_to_track.as_deref(), - &publication.transaction_to_track, - ) - .await?; - - tx.commit().await.map_err(to_anyhow)?; - - Ok(PostgresPublicationReplication::new( - publication_name, - replication_slot_name, - )) -} - -pub async fn get_postgres_version_internal(pg_connection: &Client) -> Result { - let row = pg_connection - .query_one("SHOW server_version;", &[]) - .await - .map_err(to_anyhow)?; - - let postgres_version: String = row.get(0); - - Ok(postgres_version) -} - -pub async fn get_postgres_version( - authed: ApiAuthed, - Extension(db): Extension, - Extension(user_db): Extension, - Path((w_id, postgres_resource_path)): Path<(String, String)>, -) -> Result { - let pg_connection = get_default_pg_connection( - authed.clone(), - Some(user_db), - &db, - &postgres_resource_path, - &w_id, - ) - .await - .map_err(to_anyhow)?; - - let postgres_version = get_postgres_version_internal(&pg_connection).await?; - - Ok(postgres_version) -} - -pub async fn list_slot_name( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Path((w_id, postgres_resource_path)): Path<(String, String)>, -) -> Result>> { - let pg_connection: Client = get_default_pg_connection( - authed.clone(), - Some(user_db.clone()), - &db, - &postgres_resource_path, - &w_id, - ) - .await - .map_err(to_anyhow)?; - - let rows = pg_connection - .query( - r#" - SELECT - slot_name, - active - FROM - pg_replication_slots - WHERE - plugin = 'pgoutput' AND - slot_type = 'logical'; - "#, - &[], - ) - .await - .map_err(to_anyhow)?; - - let slots = rows - .into_iter() - .map(|row| SlotList { slot_name: row.get("slot_name"), active: row.get("active") }) - .collect(); - - Ok(Json(slots)) -} - -pub async fn create_slot( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Path((w_id, postgres_resource_path)): Path<(String, String)>, - Json(Slot { name }): Json, -) -> Result { - let pg_connection = get_default_pg_connection( - authed.clone(), - Some(user_db.clone()), - &db, - &postgres_resource_path, - &w_id, - ) - .await - .map_err(to_anyhow)?; - - create_logical_replication_slot(&pg_connection, &name).await?; - - Ok(format!("Replication slot {} created!", name)) -} - -pub async fn drop_logical_replication_slot(pg_connection: &Client, slot_name: &str) -> Result<()> { - let row = pg_connection - .query_opt( - r#" - SELECT - active_pid - FROM - pg_replication_slots - WHERE - slot_name = $1 - "#, - &[&slot_name], - ) - .await - .map_err(to_anyhow)?; - - let active_pid = row.map(|r| r.get::<_, Option>(0)).flatten(); - - if let Some(pid) = active_pid { - pg_connection - .execute("SELECT pg_terminate_backend($1)", &[&pid]) - .await - .map_err(to_anyhow)?; - } - - pg_connection - .execute("SELECT pg_drop_replication_slot($1)", &[&slot_name]) - .await - .map_err(to_anyhow)?; - - Ok(()) -} - -pub async fn drop_slot_name( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Path((w_id, postgres_resource_path)): Path<(String, String)>, - Json(Slot { name }): Json, -) -> Result { - let pg_connection = - get_default_pg_connection(authed, Some(user_db), &db, &postgres_resource_path, &w_id) - .await - .map_err(to_anyhow)?; - - drop_logical_replication_slot(&pg_connection, &name) - .await - .map_err(to_anyhow)?; - - Ok(format!("Replication slot {} deleted!", name)) -} - -pub async fn list_database_publication( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Path((w_id, postgres_resource_path)): Path<(String, String)>, -) -> Result>> { - let pg_connection = get_default_pg_connection( - authed.clone(), - Some(user_db.clone()), - &db, - &postgres_resource_path, - &w_id, - ) - .await - .map_err(to_anyhow)?; - - let rows = pg_connection - .query( - "SELECT pubname AS publication_name FROM pg_publication;", - &[], - ) - .await - .map_err(to_anyhow)?; - - let publications = rows - .into_iter() - .map(|row| row.get::<_, String>("publication_name")) - .collect_vec(); - - Ok(Json(publications)) -} - -pub async fn get_publication_info( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>, -) -> Result> { - let mut pg_connection = get_default_pg_connection( - authed.clone(), - Some(user_db.clone()), - &db, - &postgres_resource_path, - &w_id, - ) - .await - .map_err(to_anyhow)?; - - let publication_data = - get_publication_scope_and_transaction(&mut pg_connection, &publication_name).await; - - let (all_table, transaction_to_track) = match publication_data { - Ok(Some(pub_data)) => pub_data, - Ok(None) => { - return Err(Error::NotFound( - ERROR_PUBLICATION_NAME_NOT_EXISTS.to_string(), - )) - } - Err(e) => return Err(e), - }; - - let table_to_track = if !all_table { - Some(get_tracked_relations(&mut pg_connection, &publication_name).await?) - } else { - None - }; - Ok(Json(PublicationData::new( - table_to_track, - transaction_to_track, - ))) -} - -pub async fn create_publication( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>, - Json(publication_data): Json, -) -> Result { - let mut pg_connection = get_default_pg_connection( - authed.clone(), - Some(user_db.clone()), - &db, - &postgres_resource_path, - &w_id, - ) - .await - .map_err(to_anyhow)?; - - let PublicationData { table_to_track, transaction_to_track } = publication_data; - - let tx = pg_connection.transaction().await.map_err(to_anyhow)?; - - create_pg_publication( - tx.client(), - &publication_name, - table_to_track.as_deref(), - &transaction_to_track, - ) - .await?; - - tx.commit().await.map_err(to_anyhow)?; - - Ok(format!( - "Publication {} successfully created!", - publication_name - )) -} - -pub async fn delete_publication( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>, -) -> Result { - let mut pg_connection = get_default_pg_connection( - authed.clone(), - Some(user_db.clone()), - &db, - &postgres_resource_path, - &w_id, - ) - .await - .map_err(to_anyhow)?; - - drop_publication(&mut pg_connection, &publication_name).await?; - - Ok(format!( - "Publication {} successfully deleted!", - publication_name - )) -} - -pub async fn update_pg_publication( - pg_connection: &Client, - publication_name: &str, - PublicationData { table_to_track, transaction_to_track }: PublicationData, - all_table: Option, -) -> Result<()> { - let quoted_publication_name = quote_identifier(publication_name); - let transaction_to_track_as_str = transaction_to_track.iter().join(","); - match table_to_track { - Some(ref relations) if !relations.is_empty() => { - // If all_table is None, the publication does not exist yet - if all_table.unwrap_or(true) { - if all_table.is_some_and(|all_table| all_table) { - drop_publication(pg_connection, publication_name) - .await - .map_err(to_anyhow)?; - } - create_pg_publication( - pg_connection, - publication_name, - table_to_track.as_deref(), - &transaction_to_track, - ) - .await - .map_err(to_anyhow)?; - } else { - let pg_14 = check_if_valid_publication_for_postgres_version( - pg_connection, - table_to_track.as_deref(), - ) - .await - .map_err(to_anyhow)?; - - let mut query = format!("ALTER PUBLICATION {} SET ", quoted_publication_name); - let mut first = true; - - for (i, schema) in relations.iter().enumerate() { - if schema.table_to_track.is_empty() { - query.push_str("TABLES IN SCHEMA "); - query.push_str("e_identifier(&schema.schema_name)); - } else { - if pg_14 && first { - query.push_str("TABLE ONLY "); - first = false; - } else if !pg_14 { - query.push_str("TABLE ONLY "); - } - - for (j, table) in schema.table_to_track.iter().enumerate() { - let table_name = quote_identifier(&table.table_name); - let schema_name = quote_identifier(&schema.schema_name); - let full_name = format!("{}.{}", schema_name, table_name); - query.push_str(&full_name); - - if let Some(columns) = table.columns_name.as_ref() { - let cols = - columns.iter().map(|col| quote_identifier(col)).join(", "); - query.push_str(&format!(" ({})", cols)); - } - - if let Some(where_clause) = &table.where_clause { - query.push_str(&format!(" WHERE ({})", where_clause)); - } - - if j + 1 != schema.table_to_track.len() { - query.push_str(", "); - } - } - } - - if i + 1 != relations.len() { - query.push_str(", "); - } - } - - pg_connection - .execute(&query, &[]) - .await - .map_err(to_anyhow)?; - - let publish_query = format!( - "ALTER PUBLICATION {} SET (publish = '{}');", - quoted_publication_name, transaction_to_track_as_str - ); - pg_connection - .execute(&publish_query, &[]) - .await - .map_err(to_anyhow)?; - } - } - _ => { - drop_publication(pg_connection, publication_name) - .await - .map_err(to_anyhow)?; - let create_all_query = format!( - "CREATE PUBLICATION {} FOR ALL TABLES WITH (publish = '{}');", - quoted_publication_name, transaction_to_track_as_str - ); - pg_connection - .execute(&create_all_query, &[]) - .await - .map_err(to_anyhow)?; - } - } - - Ok(()) -} - -pub async fn alter_publication( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Path((w_id, publication_name, postgres_resource_path)): Path<(String, String, String)>, - Json(publication_data): Json, -) -> Result { - let mut pg_connection = get_default_pg_connection( - authed.clone(), - Some(user_db.clone()), - &db, - &postgres_resource_path, - &w_id, - ) - .await - .map_err(to_anyhow)?; - - let tx = pg_connection.transaction().await.map_err(to_anyhow)?; - - let publication = get_publication_scope_and_transaction(tx.client(), &publication_name) - .await - .map_err(to_anyhow)?; - - update_pg_publication( - tx.client(), - &publication_name, - publication_data, - publication.map(|publication| publication.0), - ) - .await - .map_err(to_anyhow)?; - - tx.commit().await.map_err(to_anyhow)?; - - Ok(format!( - "Publication {} updated with success", - publication_name - )) -} - -pub async fn get_publication_scope_and_transaction( - pg_connection: &Client, - publication_name: &str, -) -> Result)>> { - let row_opt = pg_connection - .query_opt( - r#" - SELECT - puballtables AS all_table, - pubinsert AS insert, - pubupdate AS update, - pubdelete AS delete - FROM - pg_publication - WHERE - pubname = $1 - "#, - &[&publication_name], - ) - .await - .map_err(to_anyhow)?; - - let row = match row_opt { - Some(r) => r, - None => return Ok(None), - }; - - let all_table: bool = row.get("all_table"); - let pub_insert: bool = row.get("insert"); - let pub_update: bool = row.get("update"); - let pub_delete: bool = row.get("delete"); - - let mut transaction_to_track = Vec::with_capacity(3); - if pub_insert { - transaction_to_track.push("insert".to_string()); - } - if pub_update { - transaction_to_track.push("update".to_string()); - } - if pub_delete { - transaction_to_track.push("delete".to_string()); - } - - Ok(Some((all_table, transaction_to_track))) -} - -pub async fn get_tracked_relations( - pg_connection: &Client, - publication_name: &str, -) -> Result> { - let pg_version = get_postgres_version_internal(pg_connection).await?; - - let query = if pg_version.starts_with("14") { - r#" - SELECT - schemaname AS schema_name, - tablename AS table_name, - NULL::text[] AS columns, - NULL::text AS where_clause - FROM - pg_publication_tables - WHERE - pubname = $1; - "# - } else { - r#" - SELECT - schemaname AS schema_name, - tablename AS table_name, - attnames AS columns, - rowfilter AS where_clause - FROM - pg_publication_tables - WHERE - pubname = $1; - "# - }; - - let rows = pg_connection - .query(query, &[&publication_name]) - .await - .map_err(to_anyhow)?; - - let mut table_to_track: HashMap = HashMap::new(); - - for row in rows { - let schema_name: Option = row.get("schema_name"); - let table_name: Option = row.get("table_name"); - let columns: Option> = row.get("columns"); - let where_clause: Option = row.get("where_clause"); - - let schema_name = schema_name.ok_or_else::(|| { - anyhow::anyhow!( - "Unexpected NULL `schema_name` in publication entry (pubname: `{}`). This should never happen unless PostgreSQL internals are corrupted.", - publication_name, - ) - .into() - })?; - - let table_name = table_name.ok_or_else::(|| { - anyhow::anyhow!( - "Unexpected NULL `table_name` for schema `{}` in publication `{}`. This should never happen unless PostgreSQL internals are corrupted.", - schema_name, - publication_name, - ) - .into() - })?; - - let entry = table_to_track.entry(schema_name.clone()); - let table_to_track_item = TableToTrack::new(table_name, where_clause, columns); - - match entry { - std::collections::hash_map::Entry::Occupied(mut occupied) => { - occupied.get_mut().add_new_table(table_to_track_item); - } - std::collections::hash_map::Entry::Vacant(vacant) => { - vacant.insert(Relations::new(schema_name, vec![table_to_track_item])); - } - } - } - - Ok(table_to_track.into_values().collect_vec()) -} - -pub async fn get_template_script(Path((_, id)): Path<(String, String)>) -> Result { - let template = if let Some((_, template)) = TEMPLATE.remove(&id) { - template - } else { - "".to_string() - }; - Ok(template) -} - -pub async fn create_template_script( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Path(w_id): Path, - Json(template_script): Json, -) -> Result { - let TemplateScript { postgres_resource_path, relations, language } = template_script; - - let relations = match relations { - Some(r) => r, - None => { - return Err( - anyhow::anyhow!("You must at least choose schema to fetch table from").into(), - ) - } - }; - - let pg_connection: Client = get_default_pg_connection( - authed.clone(), - Some(user_db.clone()), - &db, - &postgres_resource_path, - &w_id, - ) - .await - .map_err(to_anyhow)?; - - let mut schema_or_fully_qualified_name = Vec::with_capacity(relations.len()); - let mut columns_list = Vec::with_capacity(relations.len()); - - for relation in relations { - if !relation.table_to_track.is_empty() { - for table in relation.table_to_track { - let fully_qualified_name = format!("{}.{}", relation.schema_name, table.table_name); - schema_or_fully_qualified_name.push(quote_literal(&fully_qualified_name)); - let columns = table - .columns_name - .map(|c| quote_literal(&c.join(","))) - .unwrap_or_else(|| "''".to_string()); - columns_list.push(columns); - } - } else { - schema_or_fully_qualified_name.push(quote_literal(&relation.schema_name)); - columns_list.push("''".to_string()); - } - } - - let tables_name = schema_or_fully_qualified_name.join(","); - let columns_list = columns_list.join(","); - - let query = format!( - r#" - WITH table_column_mapping AS ( - SELECT - unnest(ARRAY[{}]) AS table_name, - unnest(ARRAY[{}]) AS column_list - ), - parsed_columns AS ( - SELECT - tcm.table_name, - CASE - WHEN tcm.column_list = '' THEN NULL - ELSE string_to_array(tcm.column_list, ',') - END AS columns - FROM table_column_mapping tcm - ) - SELECT - ns.nspname AS table_schema, - cls.relname AS table_name, - attr.attname AS column_name, - attr.atttypid AS oid, - attr.attnotnull AS is_nullable - FROM pg_attribute attr - JOIN pg_class cls ON attr.attrelid = cls.oid - JOIN pg_namespace ns ON cls.relnamespace = ns.oid - JOIN parsed_columns pc - ON ns.nspname || '.' || cls.relname = pc.table_name - OR ns.nspname = pc.table_name - WHERE - attr.attnum > 0 - AND NOT attr.attisdropped - AND cls.relkind = 'r' - AND ( - pc.columns IS NULL - OR attr.attname = ANY(pc.columns) - ); - "#, - tables_name, columns_list - ); - - let rows = pg_connection.query(&query, &[]).await.map_err(to_anyhow)?; - - let mut schema_map: HashMap>> = HashMap::new(); - - #[derive(Debug)] - struct ColumnInfo { - table_schema: String, - table_name: String, - column_name: String, - oid: u32, - is_nullable: bool, - } - - for row in rows { - let info = ColumnInfo { - table_schema: row.get("table_schema"), - table_name: row.get("table_name"), - column_name: row.get("column_name"), - oid: row.get::<_, u32>("oid"), - is_nullable: row.get::<_, bool>("is_nullable"), - }; - - let mapped_info = - MappingInfo::new(info.column_name, Type::from_oid(info.oid), info.is_nullable); - - match schema_map.entry(info.table_schema) { - std::collections::hash_map::Entry::Occupied(mut schema_entry) => { - match schema_entry.get_mut().entry(info.table_name) { - std::collections::hash_map::Entry::Occupied(mut table_entry) => { - table_entry.get_mut().push(mapped_info); - } - std::collections::hash_map::Entry::Vacant(v) => { - v.insert(vec![mapped_info]); - } - } - } - std::collections::hash_map::Entry::Vacant(schema_vacant) => { - let mut table_map = HashMap::new(); - table_map.insert(info.table_name, vec![mapped_info]); - schema_vacant.insert(table_map); - } - } - } - - let mapper = Mapper::new(schema_map, language); - let template = mapper.get_template(); - - let id = format!("{}-{}", w_id, uuid::Uuid::new_v4()); - - TEMPLATE.insert(id.clone(), template); - - Ok(id) -} - -pub async fn is_database_in_logical_level( - authed: ApiAuthed, - Extension(user_db): Extension, - Extension(db): Extension, - Path((w_id, postgres_resource_path)): Path<(String, String)>, -) -> error::JsonResult { - let pg_connection = get_default_pg_connection( - authed.clone(), - Some(user_db.clone()), - &db, - &postgres_resource_path, - &w_id, - ) - .await - .map_err(to_anyhow)?; - - let row_opt = pg_connection - .query_opt("SHOW wal_level;", &[]) - .await - .map_err(to_anyhow)?; - - let wal_level: Option = row_opt.map(|row| row.get(0)); - - let is_logical = matches!(wal_level.as_deref(), Some("logical")); - - Ok(Json(is_logical)) -} diff --git a/backend/windmill-api/src/triggers/postgres/hex.rs b/backend/windmill-api/src/triggers/postgres/hex.rs deleted file mode 100644 index 2ada70dd35..0000000000 --- a/backend/windmill-api/src/triggers/postgres/hex.rs +++ /dev/null @@ -1,45 +0,0 @@ -use std::num::ParseIntError; - - -/** -* This implementation is inspired by Postgres replication functionality -* from https://github.com/supabase/pg_replicate -* -* Original implementation: -* - https://github.dev/supabase/pg_replicate/blob/main/pg_replicate/src/conversions/hex.rs -* -*/ - -use thiserror::Error; - -#[derive(Debug, Error)] -pub enum ByteaHexParseError { - #[error("missing prefix '\\x'")] - InvalidPrefix, - - #[error("invalid byte")] - OddNumerOfDigits, - - #[error("parse int result: {0}")] - ParseInt(#[from] ParseIntError), -} - -pub fn from_bytea_hex(s: &str) -> Result, ByteaHexParseError> { - if s.len() < 2 || &s[..2] != "\\x" { - return Err(ByteaHexParseError::InvalidPrefix); - } - - let mut result = Vec::with_capacity((s.len() - 2) / 2); - let s = &s[2..]; - - if s.len() % 2 != 0 { - return Err(ByteaHexParseError::OddNumerOfDigits); - } - - for i in (0..s.len()).step_by(2) { - let val = u8::from_str_radix(&s[i..i + 2], 16)?; - result.push(val); - } - - Ok(result) -} diff --git a/backend/windmill-api/src/triggers/postgres/listener.rs b/backend/windmill-api/src/triggers/postgres/listener.rs deleted file mode 100644 index bfb527e3e7..0000000000 --- a/backend/windmill-api/src/triggers/postgres/listener.rs +++ /dev/null @@ -1,441 +0,0 @@ -use std::{collections::HashMap, pin::Pin, sync::Arc}; - -use bytes::{BufMut, Bytes, BytesMut}; -use chrono::TimeZone; -use futures::{pin_mut, SinkExt}; -use pg_escape::{quote_identifier, quote_literal}; -use rust_postgres::{Client, CopyBothDuplex, SimpleQueryMessage}; -use tokio::sync::RwLock; -use tokio_stream::StreamExt; -use windmill_common::{ - db::UserDB, - error::{to_anyhow, Error, Result}, - jobs::JobTriggerKind, - worker::to_raw_value, - DB, -}; - -use crate::{ - resources::try_get_resource_from_db_as, - triggers::{ - listener::ListeningTrigger, - postgres::{ - drop_publication, get_default_pg_connection, get_raw_postgres_connection, - handler::drop_logical_replication_slot, - relation::RelationConverter, - replication_message::{ - LogicalReplicationMessage::{ - Begin, Commit, Delete, Insert, Relation, Type, Update, - }, - PrimaryKeepAliveBody, ReplicationMessage, - }, - Postgres, PostgresConfig, PostgresTrigger, ERROR_PUBLICATION_NAME_NOT_EXISTS, - }, - trigger_helpers::TriggerJobArgs, - Listener, - }, -}; - -const ERROR_REPLICATION_SLOT_NOT_EXISTS: &str = r#"The replication slot associated with this trigger no longer exists. Recreate a new replication slot or select an existing one in the advanced tab, or delete and recreate a new trigger"#; - -pub struct LogicalReplicationSettings { - pub streaming: bool, -} - -impl LogicalReplicationSettings { - pub fn new(streaming: bool) -> Self { - Self { streaming } - } -} - -pub struct PostgresSimpleClient(Client); - -trait RowExist { - fn row_exist(&self) -> bool; -} - -impl RowExist for Vec { - fn row_exist(&self) -> bool { - self.iter() - .find_map(|element| { - if let SimpleQueryMessage::CommandComplete(value) = element { - Some(*value) - } else { - None - } - }) - .is_some_and(|value| value > 0) - } -} - -impl PostgresSimpleClient { - async fn new(database: &Postgres) -> Result { - let client = get_raw_postgres_connection(database, true).await?; - - Ok(PostgresSimpleClient(client)) - } - - async fn execute_query( - &self, - query: &str, - ) -> std::result::Result, rust_postgres::Error> { - self.0.simple_query(query).await - } - - async fn get_logical_replication_stream( - &self, - publication_name: &str, - logical_replication_slot_name: &str, - ) -> Result<(CopyBothDuplex, LogicalReplicationSettings)> { - let options = format!( - r#"("proto_version" '2', "publication_names" {})"#, - quote_literal(publication_name), - ); - - let query = format!( - r#"START_REPLICATION SLOT {} LOGICAL 0/0 {}"#, - quote_identifier(logical_replication_slot_name), - options - ); - - Ok(( - self.0 - .copy_both_simple::(query.as_str()) - .await - .map_err(to_anyhow)?, - LogicalReplicationSettings::new(false), - )) - } - - async fn send_status_update( - primary_keep_alive: PrimaryKeepAliveBody, - copy_both_stream: &mut Pin<&mut CopyBothDuplex>, - ) { - let mut buf = BytesMut::new(); - let ts = chrono::Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap(); - let ts = chrono::Utc::now() - .signed_duration_since(ts) - .num_microseconds() - .unwrap_or(0); - - buf.put_u8(b'r'); - buf.put_u64(primary_keep_alive.wal_end); - buf.put_u64(primary_keep_alive.wal_end); - buf.put_u64(primary_keep_alive.wal_end); - buf.put_i64(ts); - buf.put_u8(0); - copy_both_stream.send(buf.freeze()).await.unwrap(); - tracing::debug!("Send update status message"); - } -} - -#[async_trait::async_trait] -impl Listener for PostgresTrigger { - type Consumer = (CopyBothDuplex, LogicalReplicationSettings); - type Extra = (); - type ExtraState = (); - const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Postgres; - - async fn get_consumer( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - _err_message: Arc>>, - _killpill_rx: tokio::sync::broadcast::Receiver<()>, - ) -> Result> { - let ListeningTrigger:: { workspace_id, trigger_config, .. } = - listening_trigger; - - let PostgresConfig { - postgres_resource_path, publication_name, replication_slot_name, .. - } = trigger_config; - - let authed = listening_trigger - .authed(db, &Self::TRIGGER_KIND.to_string()) - .await?; - - let database = try_get_resource_from_db_as::( - &authed, - Some(UserDB::new(db.clone())), - &db, - postgres_resource_path, - workspace_id, - ) - .await?; - - let client = PostgresSimpleClient::new(&database).await?; - - let publication = client - .execute_query(&format!( - "SELECT pubname FROM pg_publication WHERE pubname = {}", - quote_literal(&publication_name) - )) - .await - .map_err(to_anyhow)?; - - if !publication.row_exist() { - return Err(Error::BadConfig( - ERROR_PUBLICATION_NAME_NOT_EXISTS.to_string(), - )); - } - - let replication_slot = client - .execute_query(&format!( - "SELECT slot_name FROM pg_replication_slots WHERE slot_name = {}", - quote_literal(&replication_slot_name) - )) - .await - .map_err(to_anyhow)?; - - if !replication_slot.row_exist() { - return Err(Error::BadConfig( - ERROR_REPLICATION_SLOT_NOT_EXISTS.to_string(), - )); - } - - let (logical_replication_stream, logical_replication_settings) = client - .get_logical_replication_stream(&publication_name, &replication_slot_name) - .await - .map_err(to_anyhow)?; - - Ok(Some(( - logical_replication_stream, - logical_replication_settings, - ))) - } - async fn consume( - &self, - db: &DB, - consumer: Self::Consumer, - listening_trigger: &ListeningTrigger, - err_message: Arc>>, - _killpill_rx: tokio::sync::broadcast::Receiver<()>, - _extra_state: Option<&Self::ExtraState>, - ) { - let (logical_replication_stream, logical_replication_settings) = consumer; - pin_mut!(logical_replication_stream); - let mut relations = RelationConverter::new(); - tracing::info!( - "Starting to listen for postgres trigger {}", - &listening_trigger.path - ); - loop { - let message = logical_replication_stream.next().await; - let message = match message { - Some(message) => message, - None => { - tracing::error!( - "Stream for postgres trigger {} closed", - &listening_trigger.path - ); - if let None = self - .update_ping_and_loop_ping_status( - db, - listening_trigger, - err_message.clone(), - Some("Stream closed".to_string()), - ) - .await - { - return; - } - return; - } - }; - - let message = match message { - Ok(message) => message, - Err(err) => { - let err = format!( - "Postgres trigger named {} had an error while receiving a message : {}", - &listening_trigger.path, - err.to_string() - ); - self.disable_with_error(db, listening_trigger, err).await; - return; - } - }; - - let logical_message = match ReplicationMessage::parse(message) { - Ok(logical_message) => logical_message, - Err(err) => { - let err = format!( - "Postgres trigger named: {} had an error while parsing message: {}", - &listening_trigger.path, - err.to_string() - ); - self.disable_with_error(db, listening_trigger, err).await; - return; - } - }; - - match logical_message { - ReplicationMessage::PrimaryKeepAlive(primary_keep_alive) => { - if primary_keep_alive.reply { - PostgresSimpleClient::send_status_update( - primary_keep_alive, - &mut logical_replication_stream, - ) - .await; - } - } - ReplicationMessage::XLogData(x_log_data) => { - let logical_replication_message = match x_log_data - .parse(&logical_replication_settings) - { - Ok(logical_replication_message) => logical_replication_message, - Err(err) => { - tracing::error!("Postgres trigger named: {} had an error while trying to parse incomming stream message: {}", &listening_trigger.path, err.to_string()); - continue; - } - }; - - let json = match logical_replication_message { - Relation(relation_body) => { - relations.add_relation(relation_body); - None - } - Begin | Type | Commit => None, - Insert(insert) => Some(( - insert.o_id, - Ok(None), - relations.row_to_json((insert.o_id, insert.tuple)), - "insert", - )), - Update(update) => { - let old_row = update - .old_tuple - .map(|old_tuple| relations.row_to_json((update.o_id, old_tuple))) - .transpose(); - let row = relations.row_to_json((update.o_id, update.new_tuple)); - Some((update.o_id, old_row, row, "update")) - } - Delete(delete) => { - let row = delete - .old_tuple - .unwrap_or_else(|| delete.key_tuple.unwrap()); - Some(( - delete.o_id, - Ok(None), - relations.row_to_json((delete.o_id, row)), - "delete", - )) - } - }; - match json { - Some((o_id, Ok(old_row), Ok(row), transaction_type)) => { - let relation = match relations.get_relation(o_id) { - Ok(relation) => relation, - Err(err) => { - tracing::error!( - "Postgres trigger named: {}, error: {}", - &listening_trigger.path, - err.to_string() - ); - continue; - } - }; - let database_info = HashMap::from([ - ("schema_name".to_string(), to_raw_value(&relation.namespace)), - ("table_name".to_string(), to_raw_value(&relation.name)), - ( - "transaction_type".to_string(), - to_raw_value(&transaction_type), - ), - ("old_row".to_string(), to_raw_value(&old_row)), - ("row".to_string(), to_raw_value(&row)), - ]); - let _ = self - .handle_event( - db, - listening_trigger, - database_info, - HashMap::new(), - None, - ) - .await; - } - Some((o_id, old_row, row, transaction_type)) => { - let relation = match relations.get_relation(o_id) { - Ok(relation) => relation, - Err(err) => { - tracing::error!( - "Postgres trigger named: {}, error: {}", - &listening_trigger.path, - err.to_string() - ); - continue; - } - }; - - if let Err(err) = old_row { - tracing::error!( - transaction_type = ?transaction_type, - schema = %relation.namespace, - table = %relation.name, - error = %err, - "Failed to decode OLD row for {} transaction on {}.{}", - transaction_type, - relation.namespace, - relation.name, - ); - } - - if let Err(err) = row { - tracing::error!( - transaction_type = ?transaction_type, - schema = %relation.namespace, - table = %relation.name, - error = %err, - "Failed to decode NEW row for {} transaction on {}.{}", - transaction_type, - relation.namespace, - relation.name, - ); - } - } - _ => {} - } - } - } - } - } - - async fn cleanup( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - _extra_state: Option<&Self::ExtraState>, - ) -> Result<()> { - let authed = listening_trigger - .authed(db, &Self::TRIGGER_KIND.to_string()) - .await?; - - let user_db = UserDB::new(db.clone()); - - let mut pg_connection = get_default_pg_connection( - authed, - Some(user_db), - &db, - &listening_trigger.trigger_config.postgres_resource_path, - &listening_trigger.workspace_id, - ) - .await?; - - if listening_trigger.trigger_config.basic_mode.unwrap_or(false) { - drop_logical_replication_slot( - &mut pg_connection, - &listening_trigger.trigger_config.replication_slot_name, - ) - .await?; - - drop_publication( - &mut pg_connection, - &listening_trigger.trigger_config.publication_name, - ) - .await?; - } - - Ok(()) - } -} diff --git a/backend/windmill-api/src/triggers/postgres/mapper.rs b/backend/windmill-api/src/triggers/postgres/mapper.rs deleted file mode 100644 index a24fb65378..0000000000 --- a/backend/windmill-api/src/triggers/postgres/mapper.rs +++ /dev/null @@ -1,136 +0,0 @@ -use std::collections::HashMap; - -use rust_postgres::types::Type; - -use super::Language; - -fn postgres_to_typescript_type(postgres_type: Option) -> String { - let data_type = match postgres_type { - Some(postgres_type) => match postgres_type { - Type::BOOL => "boolean", - Type::BOOL_ARRAY => "Array", - Type::CHAR | Type::BPCHAR | Type::VARCHAR | Type::NAME | Type::TEXT => "string", - Type::CHAR_ARRAY - | Type::BPCHAR_ARRAY - | Type::VARCHAR_ARRAY - | Type::NAME_ARRAY - | Type::TEXT_ARRAY => "Array", - Type::INT2 | Type::INT4 | Type::INT8 | Type::NUMERIC => "number", - Type::INT2_ARRAY | Type::INT4_ARRAY | Type::INT8_ARRAY => "Array", - Type::FLOAT4 | Type::FLOAT8 => "number", - Type::FLOAT8_ARRAY | Type::FLOAT4_ARRAY => "Array", - Type::NUMERIC_ARRAY => "Array", - Type::BYTEA => "Array", - Type::BYTEA_ARRAY => "Array>", - Type::DATE => "string", - Type::DATE_ARRAY => "Array", - Type::TIME => "string", - Type::TIME_ARRAY => "Array", - Type::TIMESTAMPTZ | Type::TIMESTAMP => "string", - Type::TIMESTAMPTZ_ARRAY | Type::TIMESTAMP_ARRAY => "Array", - Type::UUID => "string", - Type::UUID_ARRAY => "Array", - Type::JSON | Type::JSONB | Type::JSON_ARRAY | Type::JSONB_ARRAY => "unknown", - Type::OID => "number", - Type::OID_ARRAY => "Array", - _ => "string", - }, - None => "string", - }; - - data_type.to_string() -} - -fn into_body_struct(language: Language, mapped_info: Vec) -> String { - let mut block = String::new(); - match language { - Language::Typescript => { - block.push_str("{\r\n"); - for field in mapped_info { - let typescript_type = postgres_to_typescript_type(field.data_type); - let mut key = field.column_name; - if field.is_nullable { - key.push('?'); - } - let full_field = format!("\t\t{}: {},\r\n", key, typescript_type); - block.push_str(&full_field); - } - block.push_str("\t}"); - } - } - block -} - -#[derive(Debug)] -pub struct MappingInfo { - data_type: Option, - is_nullable: bool, - column_name: String, -} - -impl MappingInfo { - pub fn new(column_name: String, data_type: Option, is_nullable: bool) -> Self { - Self { column_name, data_type, is_nullable } - } -} - -pub struct Mapper { - to_template: HashMap>>, - language: Language, -} - -impl Mapper { - pub fn new( - to_template: HashMap>>, - language: Language, - ) -> Self { - Self { to_template, language } - } - - fn into_typescript_template(self) -> Vec { - let mut struct_definitions = Vec::new(); - for (_, mapping_info) in self.to_template { - let last_elem = mapping_info.len() - 1; - for (i, (_, mapped_info)) in mapping_info.into_iter().enumerate() { - let mut struct_body = into_body_struct(Language::Typescript, mapped_info); - let struct_body = if i != last_elem { - struct_body.push_str("\r\n"); - struct_body - } else { - struct_body - }; - struct_definitions.push(struct_body); - } - } - struct_definitions - } - - pub fn get_template(self) -> String { - let struct_definition = match self.language { - Language::Typescript => self.into_typescript_template(), - }; - - let struct_definition = if struct_definition.is_empty() { - "any".to_string() - } else { - struct_definition.join("\t| ") - }; - - format!( - r#" - - -export async function main( - transaction_type: "insert" | "update" | "delete", - schema_name: string, - table_name: string, - row: {}, - old_row?: {} -) {{ -}} - "#, - &struct_definition, - &struct_definition - ) - } -} diff --git a/backend/windmill-api/src/triggers/postgres/mod.rs b/backend/windmill-api/src/triggers/postgres/mod.rs deleted file mode 100644 index ea267721a2..0000000000 --- a/backend/windmill-api/src/triggers/postgres/mod.rs +++ /dev/null @@ -1,553 +0,0 @@ -use std::collections::HashMap; - -use crate::{ - db::{ApiAuthed, DB}, - resources::try_get_resource_from_db_as, - triggers::trigger_helpers::TriggerJobArgs, -}; -use chrono::Utc; -use itertools::Itertools; -use native_tls::{Certificate, TlsConnector}; -use pg_escape::quote_identifier; -use rand::Rng; -use rust_postgres::{config::SslMode, Client, Config, NoTls}; -use rust_postgres_native_tls::MakeTlsConnector; -use serde::{Deserialize, Deserializer, Serialize}; -use serde_json::value::RawValue; -use sqlx::FromRow; -use windmill_common::{ - db::UserDB, - error::{to_anyhow, Error, Result}, - triggers::TriggerKind, - utils::empty_as_none, -}; - -mod bool; -mod converter; -pub mod handler; -mod hex; -pub mod listener; -mod mapper; -mod relation; -mod replication_message; - -#[derive(Clone, Copy)] -pub struct PostgresTrigger; - -impl TriggerJobArgs for PostgresTrigger { - type Payload = HashMap>; - const TRIGGER_KIND: TriggerKind = TriggerKind::Postgres; - fn v1_payload_fn(payload: &HashMap>) -> HashMap> { - payload.to_owned() - } -} - -#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] -pub struct PostgresConfig { - pub postgres_resource_path: String, - pub replication_slot_name: String, - pub publication_name: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub basic_mode: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PostgresConfigRequest { - postgres_resource_path: String, - #[serde(default)] - replication_slot_name: String, - #[serde(default)] - publication_name: String, - publication: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TestPostgresConfig { - pub postgres_resource_path: String, -} - -fn check_if_valid_relation<'de, D>( - relations: D, -) -> std::result::Result>, D::Error> -where - D: Deserializer<'de>, -{ - let relations: Option> = Option::deserialize(relations)?; - let mut track_all_table_in_schema = false; - let mut track_specific_columns_in_table = false; - match relations { - Some(relations) => { - for relation in relations.iter() { - if relation.schema_name.is_empty() { - return Err(serde::de::Error::custom( - "Schema Name must not be empty".to_string(), - )); - } - - if !track_all_table_in_schema && relation.table_to_track.is_empty() { - track_all_table_in_schema = true; - continue; - } - - for table_to_track in relation.table_to_track.iter() { - if table_to_track.table_name.trim().is_empty() { - return Err(serde::de::Error::custom( - "Table name must not be empty".to_string(), - )); - } - - if !track_specific_columns_in_table && table_to_track.columns_name.is_some() { - track_specific_columns_in_table = true; - } - } - - if track_all_table_in_schema && track_specific_columns_in_table { - return Err(serde::de::Error::custom("Incompatible tracking options. Schema-level tracking and specific table tracking with column selection cannot be used together. Refer to the documentation for valid configurations.")); - } - } - - if !relations - .iter() - .map(|relation| relation.schema_name.as_str()) - .all_unique() - { - return Err(serde::de::Error::custom( - "You cannot choose a schema more than one time".to_string(), - )); - } - - Ok(Some(relations)) - } - None => Ok(None), - } -} - -fn check_if_valid_transaction_type<'de, D>( - transaction_type: D, -) -> std::result::Result, D::Error> -where - D: Deserializer<'de>, -{ - let mut transaction_type: Vec = Vec::deserialize(transaction_type)?; - if transaction_type.len() > 3 { - return Err(serde::de::Error::custom( - "More than 3 transaction type which is not authorized, you are only allowed to those 3 transaction types: Insert, Update and Delete" - .to_string(), - )); - } - transaction_type.sort_unstable(); - transaction_type.dedup(); - - for transaction in transaction_type.iter() { - match transaction.to_lowercase().as_ref() { - "insert" => {}, - "update" => {}, - "delete" => {}, - _ => { - return Err(serde::de::Error::custom( - "Only the following transaction types are allowed: Insert, Update and Delete (case insensitive)" - .to_string(), - )) - } - } - } - - Ok(transaction_type) -} - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct PublicationData { - #[serde(default, deserialize_with = "check_if_valid_relation")] - pub table_to_track: Option>, - #[serde(deserialize_with = "check_if_valid_transaction_type")] - pub transaction_to_track: Vec, -} - -impl PublicationData { - pub fn new( - table_to_track: Option>, - transaction_to_track: Vec, - ) -> PublicationData { - PublicationData { table_to_track, transaction_to_track } - } -} - -// Slot list struct -#[derive(FromRow, Debug, Serialize)] -pub struct SlotList { - pub slot_name: Option, - pub active: Option, -} - -// Slot struct -#[derive(Debug, Serialize, Deserialize)] -pub struct Slot { - pub name: String, -} - -// Template script struct -#[derive(Debug, Deserialize)] -pub struct TemplateScript { - pub postgres_resource_path: String, - #[serde(deserialize_with = "check_if_valid_relation")] - pub relations: Option>, - pub language: Language, -} - -// Language enum -#[derive(Deserialize, Debug)] -pub enum Language { - #[serde(rename = "typescript", alias = "Typescript")] - Typescript, -} - -// Test postgres struct -#[derive(Serialize, Deserialize)] -pub struct TestPostgres { - pub postgres_resource_path: String, -} - -// PostgreSQL publication replication struct -#[derive(Serialize, Deserialize)] -pub struct PostgresPublicationReplication { - pub publication_name: String, - pub replication_slot_name: String, -} - -impl PostgresPublicationReplication { - pub fn new( - publication_name: String, - replication_slot_name: String, - ) -> PostgresPublicationReplication { - PostgresPublicationReplication { publication_name, replication_slot_name } - } -} - -pub const ERROR_PUBLICATION_NAME_NOT_EXISTS: &str = r#"The publication associated with this trigger no longer exists. Recreate a new publication or select an existing one in the advanced tab, or delete and recreate a new trigger"#; - -#[derive(FromRow, Serialize, Deserialize, Debug)] -pub struct Postgres { - pub user: String, - pub password: String, - pub host: String, - pub port: Option, - pub dbname: String, - #[serde(default)] - pub sslmode: String, - #[serde(default, deserialize_with = "empty_as_none")] - pub root_certificate_pem: Option, -} - -#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] -pub struct TableToTrack { - pub table_name: String, - #[serde(default, deserialize_with = "empty_as_none")] - pub where_clause: Option, - #[serde(default, deserialize_with = "empty_as_none")] - pub columns_name: Option>, -} - -impl TableToTrack { - pub fn new( - table_name: String, - where_clause: Option, - columns_name: Option>, - ) -> TableToTrack { - TableToTrack { table_name, where_clause, columns_name } - } -} - -#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] -pub struct Relations { - pub schema_name: String, - pub table_to_track: Vec, -} - -impl Relations { - pub fn new(schema_name: String, table_to_track: Vec) -> Relations { - Relations { schema_name, table_to_track } - } - - pub fn add_new_table(&mut self, table_to_track: TableToTrack) { - self.table_to_track.push(table_to_track); - } -} - -fn build_tls_connector( - ssl_mode: SslMode, - root_certificate_pem: Option<&String>, -) -> Result> { - let get_tls_builder_for_verify = |root_certificate: Option<&String>| { - let mut builder = TlsConnector::builder(); - if let Some(root_certificate) = root_certificate { - let root_certificate_pem = - Certificate::from_pem(root_certificate.as_bytes()).map_err(to_anyhow)?; - builder.add_root_certificate(root_certificate_pem); - } - Ok::<_, Error>(builder) - }; - let connector = match ssl_mode { - SslMode::Disable => return Ok(None), - SslMode::Require | SslMode::Prefer => { - let mut builder = TlsConnector::builder(); - builder.danger_accept_invalid_certs(true); - builder.danger_accept_invalid_hostnames(true); - builder - } - - SslMode::VerifyCa => { - let mut builder = get_tls_builder_for_verify(root_certificate_pem)?; - builder.danger_accept_invalid_hostnames(true); - builder - } - - SslMode::VerifyFull => { - let builder = get_tls_builder_for_verify(root_certificate_pem)?; - builder - } - _ => unreachable!(), - }; - - Ok(Some(MakeTlsConnector::new( - connector.build().map_err(to_anyhow)?, - ))) -} - -pub async fn get_raw_postgres_connection( - database: &Postgres, - logical_mode: bool, -) -> Result { - let ssl_mode = match database.sslmode.as_ref() { - "disable" => SslMode::Disable, - "" | "prefer" | "allow" => SslMode::Prefer, - "require" => SslMode::Require, - "verify-ca" => SslMode::VerifyCa, - "verify-full" => SslMode::VerifyFull, - ssl_mode => { - return Err(Error::BadRequest( - format!("Invalid ssl mode for postgres: {}, please put a valid ssl_mode among the following available ssl mode: ['disable', 'allow', 'prefer', 'verify-ca', 'verify-full']", ssl_mode), - )) - } - }; - - let mut config = Config::new(); - config - .dbname(&database.dbname) - .host(&database.host) - .user(&database.user) - .ssl_mode(ssl_mode); - - if logical_mode { - config.replication_mode(rust_postgres::config::ReplicationMode::Logical); - } - - if let Some(port) = database.port { - config.port(port); - }; - - if !database.password.is_empty() { - config.password(&database.password); - } - - let connector = build_tls_connector(ssl_mode, database.root_certificate_pem.as_ref())?; - let client = if let Some(connector) = connector { - let (client, connection) = config.connect(connector).await.map_err(to_anyhow)?; - tokio::spawn(async move { - tracing::info!("Successfully connected to PostgreSQL database for trigger execution"); - if let Err(e) = connection.await { - tracing::debug!("Error during PostgreSQL trigger connection: {:#?}", e); - }; - tracing::info!("PostgreSQL trigger connection closed"); - }); - client - } else { - let (client, connection) = config.connect(NoTls).await.map_err(to_anyhow)?; - tokio::spawn(async move { - if let Err(e) = connection.await { - tracing::debug!("{:#?}", e); - }; - tracing::info!("Successfully Connected into database"); - }); - client - }; - - Ok(client) -} - -pub async fn get_pg_connection( - authed: ApiAuthed, - user_db: Option, - db: &DB, - postgres_resource_path: &str, - w_id: &str, - logical_mode: bool, -) -> Result { - let database = - try_get_resource_from_db_as::(&authed, user_db, db, postgres_resource_path, w_id) - .await?; - - Ok(get_raw_postgres_connection(&database, logical_mode).await?) -} - -pub async fn get_default_pg_connection( - authed: ApiAuthed, - user_db: Option, - db: &DB, - postgres_resource_path: &str, - w_id: &str, -) -> Result { - get_pg_connection(authed, user_db, db, postgres_resource_path, w_id, false).await -} - -pub async fn create_logical_replication_slot(tx: &Client, slot_name: &str) -> Result<()> { - tx.execute( - &format!("SELECT * FROM pg_create_logical_replication_slot($1, 'pgoutput')"), - &[&slot_name], - ) - .await - .map_err(to_anyhow)?; - Ok(()) -} - -pub async fn check_if_valid_publication_for_postgres_version( - pg_connection: &Client, - table_to_track: Option<&[Relations]>, -) -> Result { - use crate::triggers::postgres::handler::get_postgres_version_internal; - - let postgres_version = get_postgres_version_internal(pg_connection).await?; - - let pg_14 = postgres_version.starts_with("14"); - if pg_14 { - let unsupported_publication = table_to_track - .and_then(|relations| { - relations.iter().find(|relation| { - let invalid_relation = relation.table_to_track.iter().find(|table_to_track| { - table_to_track.where_clause.is_some() - || table_to_track.columns_name.is_some() - }); - - relation.table_to_track.is_empty() || invalid_relation.is_some() - }) - }) - .is_some(); - - if unsupported_publication { - return Err(Error::BadRequest( - "Your PostgreSQL database is running version 14, which does not support the following publication features: \ - - WHERE clause filtering, \ - - selective column tracking, and \ - - tracking all tables within a schema.\n\ - These features are only available in PostgreSQL 15 and above.".to_string(), - )); - } - } - Ok(pg_14) -} - -pub async fn create_pg_publication( - pg_connection: &Client, - publication_name: &str, - table_to_track: Option<&[Relations]>, - transaction_to_track: &[String], -) -> Result<()> { - let pg_14 = - check_if_valid_publication_for_postgres_version(pg_connection, table_to_track).await?; - let mut query = String::from("CREATE PUBLICATION "); - - query.push_str("e_identifier(publication_name)); - - match table_to_track { - Some(database_component) if !database_component.is_empty() => { - query.push_str(" FOR"); - let mut first = true; - for (i, schema) in database_component.iter().enumerate() { - if schema.table_to_track.is_empty() { - query.push_str(" TABLES IN SCHEMA "); - query.push_str("e_identifier(&schema.schema_name)); - } else { - if pg_14 && first { - query.push_str(" TABLE ONLY "); - first = false - } else if !pg_14 { - query.push_str(" TABLE ONLY "); - } - for (j, table) in schema.table_to_track.iter().enumerate() { - let table_name = quote_identifier(&table.table_name); - let schema_name = quote_identifier(&schema.schema_name); - let full_name = format!("{}.{}", &schema_name, &table_name); - query.push_str(&full_name); - if let Some(columns) = table.columns_name.as_ref() { - query.push_str(" ("); - let columns = columns - .iter() - .map(|column| quote_identifier(column)) - .join(", "); - query.push_str(&columns); - query.push_str(")"); - } - - if let Some(where_clause) = &table.where_clause { - query.push_str(" WHERE ("); - query.push_str(where_clause); - query.push(')'); - } - - if j + 1 != schema.table_to_track.len() { - query.push_str(", "); - } - } - } - if i < database_component.len() - 1 { - query.push_str(", "); - } - } - } - _ => { - query.push_str(" FOR ALL TABLES "); - } - }; - - if !transaction_to_track.is_empty() { - let transactions = || transaction_to_track.iter().join(", "); - query.push_str(" WITH (publish = '"); - query.push_str(&transactions()); - query.push_str("');"); - } - - pg_connection - .execute(&query, &[]) - .await - .map_err(to_anyhow)?; - Ok(()) -} - -pub async fn drop_publication(pg_connection: &Client, publication_name: &str) -> Result<()> { - let mut query = String::from("DROP PUBLICATION IF EXISTS "); - let quoted_publication_name = quote_identifier(publication_name); - query.push_str("ed_publication_name); - - pg_connection - .execute(&query, &[]) - .await - .map_err(to_anyhow)?; - - Ok(()) -} - -pub fn generate_random_string() -> String { - let timestamp = Utc::now().timestamp_millis().to_string(); - let mut rng = rand::rng(); - let charset = "abcdefghijklmnopqrstuvwxyz0123456789"; - - let random_part = (0..10) - .map(|_| { - charset - .chars() - .nth(rng.random_range(0..charset.len())) - .unwrap() - }) - .collect::(); - - format!("{}_{}", timestamp, random_part) -} diff --git a/backend/windmill-api/src/triggers/postgres/relation.rs b/backend/windmill-api/src/triggers/postgres/relation.rs deleted file mode 100644 index f313efb893..0000000000 --- a/backend/windmill-api/src/triggers/postgres/relation.rs +++ /dev/null @@ -1,74 +0,0 @@ -use core::str; - -use serde_json::{Map, Value}; -use std::{collections::HashMap, str::Utf8Error}; - -use super::{ - converter::{Converter, ConverterError}, - replication_message::{Columns, RelationBody, TupleData}, -}; -use rust_postgres::types::Oid; -#[derive(Debug, thiserror::Error)] -pub enum RelationConversionError { - #[error("Could not find matching table")] - FailToFindMatchingTable, - - #[error("Binary data not supported")] - BinaryFormatNotSupported, - - #[error("decode error: {0}")] - FromBytes(#[from] ConverterError), - - #[error("invalid string value")] - InvalidStr(#[from] Utf8Error), -} - -pub struct RelationConverter(HashMap); - -impl RelationConverter { - pub fn new() -> Self { - Self(HashMap::new()) - } - - pub fn add_relation(&mut self, relation: RelationBody) { - self.0.insert(relation.o_id, relation); - } - - pub fn get_columns(&self, o_id: Oid) -> Result<&Columns, RelationConversionError> { - self.0 - .get(&o_id) - .map(|relation_body| &relation_body.columns) - .ok_or(RelationConversionError::FailToFindMatchingTable) - } - - pub fn get_relation(&self, o_id: Oid) -> Result<&RelationBody, RelationConversionError> { - self.0 - .get(&o_id) - .ok_or(RelationConversionError::FailToFindMatchingTable) - } - - pub fn row_to_json( - &self, - to_decode: (Oid, Vec), - ) -> Result, RelationConversionError> { - let (o_id, tuple_data) = to_decode; - let mut object: Map = Map::new(); - let columns = self.get_columns(o_id)?; - - for (i, column) in columns.iter().enumerate() { - let value = match &tuple_data[i] { - TupleData::Null | TupleData::UnchangedToast => Value::Null, - TupleData::Binary(_) => { - return Err(RelationConversionError::BinaryFormatNotSupported) - } - TupleData::Text(bytes) => { - let str = str::from_utf8(&bytes[..])?; - Converter::try_from_str(column.type_o_id.clone(), str)? - } - }; - - object.insert(column.name.clone(), value); - } - Ok(object) - } -} diff --git a/backend/windmill-api/src/triggers/postgres/replication_message.rs b/backend/windmill-api/src/triggers/postgres/replication_message.rs deleted file mode 100644 index 5ae7f8cf2f..0000000000 --- a/backend/windmill-api/src/triggers/postgres/replication_message.rs +++ /dev/null @@ -1,510 +0,0 @@ -#![allow(unused)] - -use core::str; -use std::{ - cmp, - io::{self, Cursor, Read}, - str::Utf8Error, -}; - -use byteorder::{BigEndian, ReadBytesExt}; -use bytes::Bytes; -use rust_postgres::types::{Oid, Type}; -use thiserror::Error; - -use super::listener::LogicalReplicationSettings; -const PRIMARY_KEEPALIVE_BYTE: u8 = b'k'; -const X_LOG_DATA_BYTE: u8 = b'w'; - -/** -* This implementation is inspired by Postgres replication functionality -* from https://github.com/supabase/pg_replicate -* -* Original implementation: -* - https://github.com/supabase/pg_replicate/blob/main/pg_replicate/src/conversions/cdc_event.rs -* -*/ - -#[derive(Debug)] -pub struct PrimaryKeepAliveBody { - pub wal_end: u64, - pub timestamp: i64, - pub reply: bool, -} - -impl PrimaryKeepAliveBody { - pub fn new(wal_end: u64, timestamp: i64, reply: bool) -> PrimaryKeepAliveBody { - PrimaryKeepAliveBody { wal_end, timestamp, reply } - } -} - -const BEGIN_BYTE: u8 = b'B'; -const COMMIT_BYTE: u8 = b'C'; -const ORIGIN_BYTE: u8 = b'O'; -const RELATION_BYTE: u8 = b'R'; -const TYPE_BYTE: u8 = b'Y'; -const INSERT_BYTE: u8 = b'I'; -const UPDATE_BYTE: u8 = b'U'; -const DELETE_BYTE: u8 = b'D'; -const TUPLE_NEW_BYTE: u8 = b'N'; -const TUPLE_KEY_BYTE: u8 = b'K'; -const TUPLE_OLD_BYTE: u8 = b'O'; -const TUPLE_DATA_NULL_BYTE: u8 = b'n'; -const TUPLE_DATA_TOAST_BYTE: u8 = b'u'; -const TUPLE_DATA_TEXT_BYTE: u8 = b't'; -const TUPLE_DATA_BINARY_BYTE: u8 = b'b'; - -const REPLICA_IDENTITY_DEFAULT_BYTE: i8 = 0x64; -const REPLICA_IDENTITY_NOTHING_BYTE: i8 = 0x6E; -const REPLICA_IDENTITY_FULL_BYTE: i8 = 0x66; -const REPLICA_IDENTITY_INDEX_BYTE: i8 = 0x69; - -#[derive(Debug)] -pub enum ReplicaIdentity { - Default, - Nothing, - Full, - Index, -} - -#[derive(Debug)] -pub struct Column { - pub flags: i8, - pub name: String, - pub type_o_id: Option, - pub type_modifier: i32, -} - -impl Column { - pub fn new(flags: i8, name: String, type_o_id: Option, type_modifier: i32) -> Self { - Self { flags, name, type_o_id, type_modifier } - } -} - -pub type Columns = Vec; - -#[derive(Debug)] -pub struct RelationBody { - pub transaction_id: Option, - pub o_id: Oid, - pub namespace: String, - pub name: String, - pub replica_identity: ReplicaIdentity, - pub columns: Columns, -} - -impl RelationBody { - pub fn new( - transaction_id: Option, - o_id: Oid, - namespace: String, - name: String, - replica_identity: ReplicaIdentity, - columns: Columns, - ) -> Self { - Self { transaction_id, o_id, namespace, name, replica_identity, columns } - } -} - -#[derive(Debug)] -pub struct InsertBody { - pub transaction_id: Option, - pub o_id: Oid, - pub tuple: Vec, -} - -impl InsertBody { - pub fn new(transaction_id: Option, o_id: Oid, tuple: Vec) -> Self { - Self { transaction_id, o_id, tuple } - } -} - -#[derive(Debug)] -pub struct UpdateBody { - transaction_id: Option, - pub o_id: Oid, - pub old_tuple: Option>, - pub key_tuple: Option>, - pub new_tuple: Vec, -} - -impl UpdateBody { - pub fn new( - transaction_id: Option, - o_id: Oid, - old_tuple: Option>, - key_tuple: Option>, - new_tuple: Vec, - ) -> Self { - Self { transaction_id, o_id, old_tuple, key_tuple, new_tuple } - } -} - -#[derive(Debug)] -pub struct DeleteBody { - transaction_id: Option, - pub o_id: Oid, - pub old_tuple: Option>, - pub key_tuple: Option>, -} - -impl DeleteBody { - pub fn new( - transaction_id: Option, - o_id: Oid, - old_tuple: Option>, - key_tuple: Option>, - ) -> Self { - Self { transaction_id, o_id, old_tuple, key_tuple } - } -} - -#[derive(Debug)] -pub enum TupleData { - Null, - UnchangedToast, - Text(Bytes), - Binary(Bytes), -} - -impl TupleData { - fn parse(buf: &mut Buffer) -> Result, ConversionError> { - let number_of_columns = buf.read_i16::()?; - let mut tuples = Vec::with_capacity(number_of_columns as usize); - for _ in 0..number_of_columns { - let byte = buf.read_u8()?; - let tuple_data = match byte { - TUPLE_DATA_NULL_BYTE => TupleData::Null, - TUPLE_DATA_TOAST_BYTE => TupleData::UnchangedToast, - TUPLE_DATA_TEXT_BYTE => { - let len = buf.read_i32::()?; - let mut data = vec![0; len as usize]; - buf.read_exact(&mut data)?; - TupleData::Text(data.into()) - } - TUPLE_DATA_BINARY_BYTE => { - let len = buf.read_i32::()?; - let mut data = vec![0; len as usize]; - buf.read_exact(&mut data)?; - TupleData::Binary(data.into()) - } - byte => { - return Err(ConversionError::Io(io::Error::new( - io::ErrorKind::InvalidInput, - format!("unknown replication message byte `{}`", byte), - ))); - } - }; - - tuples.push(tuple_data); - } - - Ok(tuples) - } -} - -#[derive(Debug)] -pub enum TransactionBody { - Insert(InsertBody), - Update(UpdateBody), - Delete(DeleteBody), -} - -#[non_exhaustive] -#[derive(Debug)] -pub enum LogicalReplicationMessage { - Begin, - Commit, - Relation(RelationBody), - Type, - Insert(InsertBody), - Update(UpdateBody), - Delete(DeleteBody), -} - -#[derive(Debug)] -pub struct XLogDataBody { - pub wal_start: u64, - pub wal_end: u64, - pub timestamp: i64, - pub data: Bytes, -} - -#[derive(Error, Debug)] -pub enum ConversionError { - #[error("Error: {0}")] - Io(#[from] io::Error), - #[error("Utf8Error conversion: {0}")] - Utf8(#[from] Utf8Error), -} - -struct Buffer { - bytes: Bytes, - idx: usize, -} - -impl Buffer { - pub fn new(bytes: Bytes, idx: usize) -> Buffer { - Buffer { bytes, idx } - } - - fn slice(&self) -> &[u8] { - &self.bytes[self.idx..] - } - - fn read_cstr(&mut self) -> Result { - match self.slice().iter().position(|&x| x == 0) { - Some(pos) => { - let start = self.idx; - let end = start + pos; - let cstr = str::from_utf8(&self.bytes[start..end])?.to_owned(); - self.idx = end + 1; - Ok(cstr) - } - None => Err(ConversionError::Io(io::Error::new( - io::ErrorKind::UnexpectedEof, - "unexpected EOF", - ))), - } - } -} - -impl Read for Buffer { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - let len = { - let slice = self.slice(); - let len = cmp::min(slice.len(), buf.len()); - buf[..len].copy_from_slice(&slice[..len]); - len - }; - self.idx += len; - Ok(len) - } -} - -impl XLogDataBody { - pub fn new(wal_start: u64, wal_end: u64, timestamp: i64, data: Bytes) -> XLogDataBody { - XLogDataBody { wal_start, wal_end, timestamp, data } - } - - pub fn parse( - self, - logical_replication_settings: &LogicalReplicationSettings, - ) -> Result { - let mut buf = Buffer::new(self.data.clone(), 0); - let byte = buf.read_u8()?; - - let logical_replication_message = match byte { - BEGIN_BYTE => { - buf.read_i64::()?; - buf.read_i64::()?; - buf.read_i32::()?; - - LogicalReplicationMessage::Begin - } - COMMIT_BYTE => { - buf.read_i8()?; - buf.read_u64::()?; - buf.read_u64::()?; - buf.read_i64::()?; - LogicalReplicationMessage::Commit - } - RELATION_BYTE => { - let transaction_id = match logical_replication_settings.streaming { - true => Some(buf.read_i32::()?), - false => None, - }; - - let o_id = buf.read_u32::()?; - let namespace = buf.read_cstr()?; - let name = buf.read_cstr()?; - let replica_identity = match buf.read_i8()? { - REPLICA_IDENTITY_DEFAULT_BYTE => ReplicaIdentity::Default, - REPLICA_IDENTITY_NOTHING_BYTE => ReplicaIdentity::Nothing, - REPLICA_IDENTITY_FULL_BYTE => ReplicaIdentity::Full, - REPLICA_IDENTITY_INDEX_BYTE => ReplicaIdentity::Index, - byte => { - return Err(ConversionError::Io(io::Error::new( - io::ErrorKind::InvalidInput, - format!("unknown replica identity byte `{}`", byte), - ))); - } - }; - - let num_of_column = buf.read_i16::()?; - - let mut columns = Vec::with_capacity(num_of_column as usize); - for _ in 0..num_of_column { - let flags = buf.read_i8()?; - let name = buf.read_cstr()?; - let o_id = buf.read_u32::()?; - let type_modifier = buf.read_i32::()?; - let type_o_id = Type::from_oid(o_id); - let column = Column::new(flags, name, type_o_id, type_modifier); - - columns.push(column); - } - - LogicalReplicationMessage::Relation(RelationBody::new( - transaction_id, - o_id, - namespace, - name, - replica_identity, - columns, - )) - } - TYPE_BYTE => { - buf.read_u32::()?; - buf.read_cstr()?; - buf.read_cstr()?; - - LogicalReplicationMessage::Type - } - INSERT_BYTE => { - let transaction_id = match logical_replication_settings.streaming { - true => Some(buf.read_i32::()?), - false => None, - }; - let o_id = buf.read_u32::()?; - let byte = buf.read_u8()?; - - let tuple = match byte { - TUPLE_NEW_BYTE => TupleData::parse(&mut buf)?, - byte => { - return Err(ConversionError::Io(io::Error::new( - io::ErrorKind::InvalidInput, - format!("unexpected tuple byte `{}`", byte), - ))); - } - }; - - LogicalReplicationMessage::Insert(InsertBody::new(transaction_id, o_id, tuple)) - } - UPDATE_BYTE => { - let transaction_id = match logical_replication_settings.streaming { - true => Some(buf.read_i32::()?), - false => None, - }; - let o_id = buf.read_u32::()?; - let byte = buf.read_u8()?; - let mut key_tuple = None; - let mut old_tuple = None; - - let new_tuple = match byte { - TUPLE_NEW_BYTE => TupleData::parse(&mut buf)?, - TUPLE_OLD_BYTE | TUPLE_KEY_BYTE => { - if byte == TUPLE_OLD_BYTE { - old_tuple = Some(TupleData::parse(&mut buf)?); - } else { - key_tuple = Some(TupleData::parse(&mut buf)?); - } - match buf.read_u8()? { - TUPLE_NEW_BYTE => TupleData::parse(&mut buf)?, - byte => { - return Err(ConversionError::Io(io::Error::new( - io::ErrorKind::InvalidInput, - format!("unexpected tuple byte `{}`", byte), - ))); - } - } - } - byte => { - return Err(ConversionError::Io(io::Error::new( - io::ErrorKind::InvalidInput, - format!("unknown tuple byte `{}`", byte), - ))); - } - }; - - LogicalReplicationMessage::Update(UpdateBody::new( - transaction_id, - o_id, - old_tuple, - key_tuple, - new_tuple, - )) - } - DELETE_BYTE => { - let transaction_id = match logical_replication_settings.streaming { - true => Some(buf.read_i32::()?), - false => None, - }; - let o_id = buf.read_u32::()?; - let tag = buf.read_u8()?; - - let mut key_tuple = None; - let mut old_tuple = None; - - match tag { - TUPLE_OLD_BYTE => old_tuple = Some(TupleData::parse(&mut buf)?), - TUPLE_KEY_BYTE => key_tuple = Some(TupleData::parse(&mut buf)?), - tag => { - return Err(ConversionError::Io(io::Error::new( - io::ErrorKind::InvalidInput, - format!("unknown tuple tag `{}`", tag), - ))); - } - } - - LogicalReplicationMessage::Delete(DeleteBody::new( - transaction_id, - o_id, - old_tuple, - key_tuple, - )) - } - byte => { - return Err(ConversionError::Io(io::Error::new( - io::ErrorKind::InvalidInput, - format!("unknown replication message tag `{}`", byte), - ))); - } - }; - - Ok(logical_replication_message) - } -} - -#[non_exhaustive] -#[derive(Debug)] -pub enum ReplicationMessage { - XLogData(XLogDataBody), - PrimaryKeepAlive(PrimaryKeepAliveBody), -} - -impl ReplicationMessage { - pub fn parse(buf: Bytes) -> io::Result { - let (byte, mut message) = buf.split_first().unwrap(); - - let replication_message = match *byte { - X_LOG_DATA_BYTE => { - let len = buf.len(); - let wal_start = message.read_u64::()?; - let wal_end = message.read_u64::()?; - let timestamp = message.read_i64::()?; - let len = len - message.len(); - let data = buf.slice(len..); - ReplicationMessage::XLogData(XLogDataBody::new(wal_start, wal_end, timestamp, data)) - } - PRIMARY_KEEPALIVE_BYTE => { - let wal_end = message.read_u64::()?; - let timestamp = message.read_i64::()?; - let reply = message.read_u8()?; - ReplicationMessage::PrimaryKeepAlive(PrimaryKeepAliveBody::new( - wal_end, - timestamp, - reply == 1, - )) - } - byte => { - return Err(io::Error::new( - io::ErrorKind::InvalidInput, - format!("unknown replication message byte `{}`", byte), - )); - } - }; - - Ok(replication_message) - } -} diff --git a/backend/windmill-api/src/triggers/sqs/handler_oss.rs b/backend/windmill-api/src/triggers/sqs/handler_oss.rs deleted file mode 100644 index 21200e8e38..0000000000 --- a/backend/windmill-api/src/triggers/sqs/handler_oss.rs +++ /dev/null @@ -1,65 +0,0 @@ -#[cfg(feature = "private")] -#[allow(unused)] -pub use super::handler_ee::*; - -#[cfg(not(feature = "private"))] -use { - super::SqsTrigger, - crate::{ - db::{ApiAuthed, DB}, - triggers::{Trigger, TriggerCrud, TriggerData}, - }, - axum::async_trait, - sqlx::PgConnection, - windmill_common::error::{Error, Result}, - windmill_git_sync::DeployedObject, -}; - -#[cfg(not(feature = "private"))] -#[async_trait] -#[cfg(not(feature = "private"))] -impl TriggerCrud for SqsTrigger { - type Trigger = Trigger; - type TriggerConfig = (); - type TriggerConfigRequest = (); - type TestConnectionConfig = (); - - const TABLE_NAME: &'static str = ""; - const TRIGGER_TYPE: &'static str = ""; - const SUPPORTS_SERVER_STATE: bool = false; - const SUPPORTS_TEST_CONNECTION: bool = false; - const ROUTE_PREFIX: &'static str = "/sqs_triggers"; - const DEPLOYMENT_NAME: &'static str = ""; - const IS_ALLOWED_ON_CLOUD: bool = false; - - fn get_deployed_object(path: String) -> DeployedObject { - DeployedObject::SqsTrigger { path } - } - - async fn create_trigger( - &self, - _db: &DB, - _tx: &mut PgConnection, - _authed: &ApiAuthed, - _w_id: &str, - _trigger: TriggerData, - ) -> Result<()> { - Err(Error::BadRequest( - "SQS triggers are not available in open source version".to_string(), - )) - } - - async fn update_trigger( - &self, - _db: &DB, - _tx: &mut PgConnection, - _authed: &ApiAuthed, - _workspace_id: &str, - _path: &str, - _trigger: TriggerData, - ) -> Result<()> { - Err(Error::BadRequest( - "SQS triggers are not available in open source version".to_string(), - )) - } -} diff --git a/backend/windmill-api/src/triggers/sqs/listener_oss.rs b/backend/windmill-api/src/triggers/sqs/listener_oss.rs deleted file mode 100644 index 0aaf5360c3..0000000000 --- a/backend/windmill-api/src/triggers/sqs/listener_oss.rs +++ /dev/null @@ -1,41 +0,0 @@ -#[allow(unused)] - -#[cfg(feature = "private")] -pub use super::listener_ee::*; - -#[cfg(not(feature = "private"))] -use { - super::SqsTrigger, - crate::triggers::{listener::ListeningTrigger, Listener}, - std::sync::Arc, - tokio::sync::RwLock, - windmill_common::{error::Result, jobs::JobTriggerKind, DB}, -}; - -#[cfg(not(feature = "private"))] -#[async_trait::async_trait] -impl Listener for SqsTrigger { - type Consumer = (); - type Extra = (); - const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Sqs; - - async fn get_consumer( - &self, - _db: &DB, - _listening_trigger: &ListeningTrigger, - _err_message: Arc>>, - _killpill_rx: tokio::sync::broadcast::Receiver<()>, - ) -> Result> { - Ok(None) - } - async fn consume( - &self, - _db: &DB, - _consumer: Self::Consumer, - _listening_trigger: &ListeningTrigger, - _err_message: Arc>>, - _killpill_rx: tokio::sync::broadcast::Receiver<()>, - ) { - () - } -} diff --git a/backend/windmill-api/src/triggers/sqs/mod.rs b/backend/windmill-api/src/triggers/sqs/mod.rs deleted file mode 100644 index 8c7217725c..0000000000 --- a/backend/windmill-api/src/triggers/sqs/mod.rs +++ /dev/null @@ -1,15 +0,0 @@ -#[cfg(feature = "private")] -mod handler_ee; -pub mod handler_oss; - -#[cfg(feature = "private")] -mod listener_ee; -pub mod listener_oss; - -#[cfg(feature = "private")] -mod mod_ee; -#[cfg(feature = "private")] -pub use mod_ee::*; - -#[derive(Copy, Clone)] -pub struct SqsTrigger; diff --git a/backend/windmill-api/src/triggers/trigger_helpers.rs b/backend/windmill-api/src/triggers/trigger_helpers.rs deleted file mode 100644 index c533dcaaeb..0000000000 --- a/backend/windmill-api/src/triggers/trigger_helpers.rs +++ /dev/null @@ -1,959 +0,0 @@ -use anyhow::Context; -use axum::response::IntoResponse; -use http::StatusCode; -use serde::Deserialize; -use serde_json::value::RawValue; -use sqlx::types::Json; -use std::collections::HashMap; -use std::future::Future; -use uuid::Uuid; -use windmill_common::{ - db::{UserDB, UserDbWithAuthed}, - error::Result, - flows::{FlowModuleValue, Retry}, - get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path, - jobs::{get_has_preprocessor_from_content_and_lang, script_path_to_payload, JobPayload}, - scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, - triggers::{ - HubOrWorkspaceId, RunnableFormat, RunnableFormatVersion, TriggerKind, TriggerMetadata, - RUNNABLE_FORMAT_VERSION_CACHE, - }, - users::username_to_permissioned_as, - utils::StripPath, - worker::to_raw_value, -}; -use windmill_queue::{push, PushArgs, PushArgsOwned, PushIsolationLevel}; - -#[cfg(feature = "enterprise")] -use crate::jobs::check_license_key_valid; -use crate::{ - db::{ApiAuthed, DB}, - jobs::{ - check_tag_available_for_workspace, delete_job_metadata_after_use, - push_flow_job_by_path_into_queue, push_script_job_by_path_into_queue, result_to_response, - run_wait_result_internal, RunJobQuery, - }, - utils::check_scopes, - HTTP_CLIENT, -}; - -struct ScriptInfo { - has_preprocessor: Option, - language: ScriptLang, - content: String, - schema: Option>, -} - -#[derive(Debug, Deserialize)] -struct PropertyDefinition { - r#type: Option>, -} - -#[derive(Debug, Deserialize)] -struct PartialSchema { - properties: Option>, -} - -#[derive(Debug, Clone, Eq, PartialEq, Hash)] -pub enum RunnableId { - FlowId(FlowId), - ScriptId(ScriptId), - HubScript(String), -} - -impl RunnableId { - pub fn from_script_hash(hash: ScriptHash) -> Self { - Self::ScriptId(ScriptId::ScriptHash(hash)) - } - - pub fn from_script_path(path: &str) -> Self { - if path.starts_with("hub/") { - Self::HubScript(path.to_string()) - } else { - Self::ScriptId(ScriptId::ScriptPath(path.to_string())) - } - } - - pub fn from_flow_path(path: &str) -> Self { - Self::FlowId(FlowId::FlowPath(path.to_string())) - } - - pub fn from_flow_version(version: i64) -> Self { - Self::FlowId(FlowId::FlowVersion(version)) - } -} - -#[derive(Debug, Clone, Eq, PartialEq, Hash)] -pub enum FlowId { - FlowPath(String), - FlowVersion(i64), -} - -impl FlowId { - async fn get_flow_version_id(self, workspace_id: &str, db: &DB) -> Result { - let version_id = match self { - FlowId::FlowPath(path) => { - let info = - get_latest_flow_version_info_for_path(None, db, workspace_id, &path, true) - .await?; - info.version - } - FlowId::FlowVersion(version) => version, - }; - - Ok(version_id) - } -} - -#[derive(Debug, Clone, Eq, PartialEq, Hash)] -pub enum ScriptId { - ScriptPath(String), - ScriptHash(ScriptHash), -} - -impl ScriptId { - async fn get_script_hash(self, workspace_id: &str, db: &DB) -> Result { - let hash = match self { - ScriptId::ScriptPath(path) => { - let info = get_latest_deployed_hash_for_path(None, db.clone(), workspace_id, &path) - .await?; - info.hash - } - ScriptId::ScriptHash(hash) => hash.0, - }; - - Ok(hash) - } -} - -async fn get_script_info( - db: &DB, - workspace_id: &str, - hash: i64, -) -> std::result::Result { - sqlx::query_as!(ScriptInfo, "SELECT has_preprocessor, language as \"language: _\", content, schema as \"schema: _\" FROM script WHERE workspace_id = $1 AND hash = $2", workspace_id, hash) - .fetch_one(db) - .await -} - -fn runnable_format_from_schema_without_preprocessor( - trigger_kind: &TriggerKind, - schema: Option>, -) -> RunnableFormat { - match trigger_kind { - TriggerKind::Mqtt - if schema.as_ref().is_some_and(|schema| { - schema.properties.as_ref().is_some_and(|properties| { - properties.iter().any(|(key, def)| { - key == "payload" - && def.r#type.as_ref().is_some_and(|t| { - let typ = t.get().trim(); - typ == "array" || (typ.starts_with('[') && typ.ends_with(']')) - }) - }) - }) - }) => - { - RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: false } - } - TriggerKind::Kafka | TriggerKind::Nats - if schema.as_ref().is_some_and(|schema| { - schema - .properties - .as_ref() - .is_some_and(|properties| properties.keys().any(|key| key == "msg")) - }) => - { - RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: false } - } - _ => RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor: false }, - } -} - -fn runnable_format_from_preprocessor_args( - args: Option>, -) -> RunnableFormat { - if let Some(args) = args { - if args.iter().any(|arg| arg.name == "wm_trigger") - || (args.len() > 0 && args.iter().all(|arg| arg.name != "event")) - { - RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: true } - } else { - RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor: true } - } - } else { - RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor: true } - } -} - -enum PreprocessorInfo { - Preprocessor { content: String, language: ScriptLang }, - NoPreprocessor { schema: Option> }, -} - -#[derive(Debug, Deserialize)] -struct FlowInfo { - preprocessor_module: Option>, - schema: Option>, -} - -fn get_preprocessor_args_from_content_and_language( - content: &str, - language: &ScriptLang, -) -> Result>> { - let args = match language { - ScriptLang::Bun | ScriptLang::Bunnative | ScriptLang::Deno | ScriptLang::Nativets => { - let args = windmill_parser_ts::parse_deno_signature( - &content, - true, - false, - Some("preprocessor".to_string()), - )?; - Some(args.args) - } - ScriptLang::Python3 => { - let args = windmill_parser_py::parse_python_signature( - &content, - Some("preprocessor".to_string()), - false, - )?; - Some(args.args) - } - _ => None, - }; - Ok(args) -} - -pub async fn get_runnable_format( - runnable_id: RunnableId, - workspace_id: &str, - db: &DB, - trigger_kind: &TriggerKind, -) -> Result { - let (key, preprocessor_info) = match runnable_id { - RunnableId::HubScript(path) => { - let Some(version) = path.split("/").nth(1) else { - return Err(windmill_common::error::Error::internal_err( - "Invalid hub script path".to_string(), - )); - }; - - let version = match version.parse::() { - Ok(version) => version, - Err(_) => { - return Err(windmill_common::error::Error::internal_err( - "Invalid hub script version".to_string(), - )); - } - }; - - let key = (HubOrWorkspaceId::Hub, version, trigger_kind.clone()); - - let runnable_format = RUNNABLE_FORMAT_VERSION_CACHE.get(&key); - - if let Some(runnable_format) = runnable_format { - tracing::debug!("Using cached runnable format for hub script {path}"); - return Ok(runnable_format); - } - - let hub_script = - get_full_hub_script_by_path(StripPath(path.to_string()), &HTTP_CLIENT, Some(db)) - .await?; - - let has_preprocessor = get_has_preprocessor_from_content_and_lang( - &hub_script.content, - &hub_script.language, - )?; - - let partial_schema = serde_json::from_str(hub_script.schema.get())?; - - ( - key, - if has_preprocessor { - PreprocessorInfo::Preprocessor { - content: hub_script.content, - language: hub_script.language, - } - } else { - PreprocessorInfo::NoPreprocessor { - schema: Some(sqlx::types::Json(partial_schema)), - } - }, - ) - } - RunnableId::FlowId(flow_id) => { - let version = flow_id.get_flow_version_id(workspace_id, db).await?; - - let key = ( - HubOrWorkspaceId::WorkspaceId(workspace_id.to_string()), - version, - trigger_kind.clone(), - ); - - let runnable_format = RUNNABLE_FORMAT_VERSION_CACHE.get(&key); - - if let Some(runnable_format) = runnable_format { - tracing::debug!("Using cached runnable format for flow version {version}"); - return Ok(runnable_format); - } - - let flow_info = sqlx::query_as!( - FlowInfo, - "SELECT - value->'preprocessor_module'->'value' as \"preprocessor_module: _\", - schema as \"schema: _\" - FROM flow_version - WHERE - id = $1 - AND workspace_id = $2", - version, - workspace_id, - ) - .fetch_one(db) - .await?; - - if let Some(preprocessor_module) = flow_info.preprocessor_module { - match preprocessor_module.0 { - FlowModuleValue::RawScript { content, language, .. } => { - (key, PreprocessorInfo::Preprocessor { content, language }) - } - FlowModuleValue::Script { path, hash, .. } => { - let hash = if let Some(hash) = hash { - hash.0 - } else { - let script_hash = get_latest_deployed_hash_for_path( - None, - db.clone(), - workspace_id, - &path, - ) - .await?; - script_hash.hash - }; - let script_info = get_script_info(db, workspace_id, hash).await?; - ( - key, - PreprocessorInfo::Preprocessor { - content: script_info.content, - language: script_info.language, - }, - ) - } - _ => { - return Err(windmill_common::error::Error::internal_err( - "Unsupported preprocessor module".to_string(), - )); - } - } - } else { - ( - key, - PreprocessorInfo::NoPreprocessor { schema: flow_info.schema }, - ) - } - } - RunnableId::ScriptId(script_id) => { - let hash = script_id.get_script_hash(workspace_id, db).await?; - let key = ( - HubOrWorkspaceId::WorkspaceId(workspace_id.to_string()), - hash, - trigger_kind.clone(), - ); - let runnable_format = RUNNABLE_FORMAT_VERSION_CACHE.get(&key); - - if let Some(runnable_format) = runnable_format { - tracing::debug!("Using cached runnable format for script {hash}"); - return Ok(runnable_format); - } - - let script_info = get_script_info(db, workspace_id, hash).await?; - - if script_info.has_preprocessor.unwrap_or(false) { - ( - key, - PreprocessorInfo::Preprocessor { - content: script_info.content, - language: script_info.language, - }, - ) - } else { - ( - key, - PreprocessorInfo::NoPreprocessor { schema: script_info.schema }, - ) - } - } - }; - - let runnable_format = match preprocessor_info { - PreprocessorInfo::Preprocessor { content, language } => { - let args = get_preprocessor_args_from_content_and_language(&content, &language)?; - runnable_format_from_preprocessor_args(args) - } - PreprocessorInfo::NoPreprocessor { schema } => { - runnable_format_from_schema_without_preprocessor(trigger_kind, schema) - } - }; - - RUNNABLE_FORMAT_VERSION_CACHE.insert(key, runnable_format); - - Ok(runnable_format) -} - -#[allow(dead_code)] - -pub trait TriggerJobArgs { - type Payload: Send + Sync; - const TRIGGER_KIND: TriggerKind; - - fn v1_payload_fn(payload: &Self::Payload) -> HashMap>; - fn v2_payload_fn(payload: &Self::Payload) -> HashMap> { - Self::v1_payload_fn(payload) - } - - fn build_job_args_v2( - has_preprocessor: bool, - payload: &Self::Payload, - info: HashMap>, - ) -> PushArgsOwned { - let mut args = Self::v2_payload_fn(payload); - if has_preprocessor { - args.insert( - "kind".to_string(), - to_raw_value(&Self::TRIGGER_KIND.to_key()), - ); - args.extend(info); - let args = HashMap::from([("event".to_string(), to_raw_value(&args))]); - PushArgsOwned { args, extra: None } - } else { - PushArgsOwned { args, extra: None } - } - } - - fn build_job_args_v1( - has_preprocessor: bool, - payload: &Self::Payload, - info: HashMap>, - ) -> PushArgsOwned { - let trigger_key = Self::TRIGGER_KIND.to_key(); - let args = Self::v1_payload_fn(payload); - let extra = if has_preprocessor { - Some(HashMap::from([( - "wm_trigger".to_string(), - to_raw_value(&serde_json::json!({ - "kind": trigger_key, - trigger_key: info - })), - )])) - } else { - None - }; - PushArgsOwned { args, extra } - } - - fn build_job_args( - runnable_path: &str, - is_flow: bool, - w_id: &str, - db: &DB, - payload: Self::Payload, - info: HashMap>, - ) -> impl Future> + Send { - async move { - let runnable_id = if is_flow { - RunnableId::from_flow_path(runnable_path) - } else { - RunnableId::from_script_path(runnable_path) - }; - Self::build_job_args_from_runnable_id(runnable_id, w_id, db, payload, info).await - } - } - - fn build_job_args_from_runnable_id( - runnable_id: RunnableId, - w_id: &str, - db: &DB, - payload: Self::Payload, - trigger_info: HashMap>, - ) -> impl Future> + Send { - async move { - tracing::debug!("Building job args for {runnable_id:?}"); - let runnable_format = - get_runnable_format(runnable_id, w_id, db, &Self::TRIGGER_KIND).await?; - let job_args = match runnable_format { - RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor } => { - Self::build_job_args_v1(has_preprocessor, &payload, trigger_info) - } - RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor } => { - Self::build_job_args_v2(has_preprocessor, &payload, trigger_info) - } - }; - - Ok(job_args) - } - } - - fn build_capture_payloads( - payload: &Self::Payload, - info: HashMap>, - ) -> (PushArgsOwned, PushArgsOwned) { - let main_args = Self::build_job_args_v2(false, payload, info.clone()); - let preprocessor_args = Self::build_job_args_v2(true, payload, info); - (main_args, preprocessor_args) - } -} - -#[allow(dead_code)] -pub async fn trigger_runnable_inner<'c>( - db: &DB, - tx_o: Option>, - user_db: Option, - authed: ApiAuthed, - workspace_id: &str, - runnable_path: &str, - is_flow: bool, - args: PushArgsOwned, - retry: Option<&sqlx::types::Json>, - error_handler_path: Option<&str>, - error_handler_args: Option<&sqlx::types::Json>>, - trigger_path: String, - job_id: Option, - trigger: TriggerMetadata, - suspended_mode: Option, -) -> Result<( - Uuid, - Option, - Option, - Option>, -)> { - let error_handler_args = error_handler_args.map(|args| { - let args = args - .0 - .iter() - .map(|(key, value)| (key.to_owned(), to_raw_value(&value))) - .collect::>>(); - Json(args) - }); - - let user_db = user_db.unwrap_or_else(|| UserDB::new(db.clone())); - let (uuid, delete_after_use, early_return, tx_out) = if is_flow { - let run_query = RunJobQuery { job_id, suspended_mode, ..Default::default() }; - let path = StripPath(runnable_path.to_string()); - let (uuid, early_return, tx_out) = push_flow_job_by_path_into_queue( - authed, - db.clone(), - tx_o, - user_db, - workspace_id.to_string(), - path, - run_query, - args, - Some(trigger), - ) - .await?; - (uuid, None, early_return, tx_out) - } else { - let (uuid, delete_after_use, tx_out) = trigger_script_internal( - db, - tx_o, - user_db, - authed, - workspace_id, - runnable_path, - args, - retry, - error_handler_path, - error_handler_args.as_ref(), - trigger_path, - job_id, - trigger, - suspended_mode, - ) - .await?; - (uuid, delete_after_use, None, tx_out) - }; - - Ok((uuid, delete_after_use, early_return, tx_out)) -} - -#[allow(dead_code)] -pub async fn trigger_runnable( - db: &DB, - user_db: Option, - authed: ApiAuthed, - workspace_id: &str, - runnable_path: &str, - is_flow: bool, - args: PushArgsOwned, - retry: Option<&sqlx::types::Json>, - error_handler_path: Option<&str>, - error_handler_args: Option<&sqlx::types::Json>>, - trigger_path: String, - job_id: Option, - suspended_mode: bool, - trigger: TriggerMetadata, -) -> Result { - let uuid = trigger_runnable_inner( - db, - None, - user_db, - authed, - workspace_id, - runnable_path, - is_flow, - args, - retry, - error_handler_path, - error_handler_args, - trigger_path, - job_id, - trigger, - Some(suspended_mode), - ) - .await? - .0; - Ok((StatusCode::CREATED, uuid.to_string()).into_response()) -} - -#[allow(dead_code)] -pub async fn trigger_runnable_and_wait_for_result( - db: &DB, - user_db: Option, - authed: ApiAuthed, - workspace_id: &str, - runnable_path: &str, - is_flow: bool, - args: PushArgsOwned, - retry: Option<&sqlx::types::Json>, - error_handler_path: Option<&str>, - error_handler_args: Option<&sqlx::types::Json>>, - trigger_path: String, - trigger: TriggerMetadata, -) -> Result { - let username = authed.username.clone(); - let (uuid, delete_after_use, early_return, _) = trigger_runnable_inner( - db, - None, - user_db, - authed, - workspace_id, - runnable_path, - is_flow, - args, - retry, - error_handler_path, - error_handler_args, - trigger_path, - None, - trigger, - None, - ) - .await?; - let (result, success) = - run_wait_result_internal(db, uuid, &workspace_id, early_return, &username).await?; - - if delete_after_use.unwrap_or(false) { - delete_job_metadata_after_use(&db, uuid).await?; - } - - result_to_response(result, success) -} - -#[allow(dead_code)] -pub async fn trigger_runnable_and_wait_for_raw_result( - db: &DB, - user_db: Option, - authed: ApiAuthed, - workspace_id: &str, - runnable_path: &str, - is_flow: bool, - args: PushArgsOwned, - retry: Option<&sqlx::types::Json>, - error_handler_path: Option<&str>, - error_handler_args: Option<&sqlx::types::Json>>, - trigger_path: String, - trigger: TriggerMetadata, -) -> Result<(Box, bool)> { - let username = authed.username.clone(); - let (uuid, delete_after_use, early_return, _) = trigger_runnable_inner( - db, - None, - user_db, - authed, - workspace_id, - runnable_path, - is_flow, - args, - retry, - error_handler_path, - error_handler_args, - trigger_path, - None, - trigger, - None, - ) - .await?; - - let (result, success) = - run_wait_result_internal(db, uuid, &workspace_id, early_return, &username) - .await - .with_context(|| { - format!( - "Error fetching job result for {} {}", - if is_flow { "flow" } else { "script" }, - runnable_path - ) - })?; - - if delete_after_use.unwrap_or(false) { - delete_job_metadata_after_use(&db, uuid).await?; - } - - Ok((result, success)) -} - -pub async fn trigger_runnable_and_wait_for_raw_result_with_error_ctx( - db: &DB, - user_db: Option, - authed: ApiAuthed, - workspace_id: &str, - runnable_path: &str, - is_flow: bool, - args: PushArgsOwned, - retry: Option<&sqlx::types::Json>, - error_handler_path: Option<&str>, - error_handler_args: Option<&sqlx::types::Json>>, - trigger_path: String, - trigger: TriggerMetadata, -) -> Result> { - let (result, success) = trigger_runnable_and_wait_for_raw_result( - db, - user_db, - authed, - workspace_id, - runnable_path, - is_flow, - args, - retry, - error_handler_path, - error_handler_args, - trigger_path, - trigger, - ) - .await?; - - if !success { - Err(windmill_common::error::Error::internal_err(format!( - "{} {runnable_path} failed: {:?}", - if is_flow { "Flow" } else { "Script" }, - result - ))) - } else { - Ok(result) - } -} - -async fn trigger_script_internal<'c>( - db: &DB, - tx_o: Option>, - user_db: UserDB, - authed: ApiAuthed, - workspace_id: &str, - script_path: &str, - args: PushArgsOwned, - retry: Option<&sqlx::types::Json>, - error_handler_path: Option<&str>, - error_handler_args: Option<&sqlx::types::Json>>>, - trigger_path: String, - job_id: Option, - trigger: TriggerMetadata, - suspended_mode: Option, -) -> Result<( - Uuid, - Option, - Option>, -)> { - if retry.is_none() && error_handler_path.is_none() { - let run_query = RunJobQuery { job_id, suspended_mode, ..Default::default() }; - let path = StripPath(script_path.to_string()); - let (uuid, delete_after_use, tx_out) = push_script_job_by_path_into_queue( - authed, - db.clone(), - tx_o, - user_db, - workspace_id.to_string(), - path, - run_query, - args, - Some(trigger), - ) - .await?; - Ok((uuid, delete_after_use, tx_out)) - } else { - let (uuid, delete_after_use, tx_out) = trigger_script_with_retry_and_error_handler( - db, - tx_o, - user_db, - authed, - workspace_id, - script_path, - args, - retry, - error_handler_path, - error_handler_args, - trigger_path, - job_id, - trigger, - suspended_mode, - ) - .await?; - Ok((uuid, delete_after_use, tx_out)) - } -} - -async fn trigger_script_with_retry_and_error_handler<'c>( - db: &DB, - tx_o: Option>, - user_db: UserDB, - authed: ApiAuthed, - workspace_id: &str, - script_path: &str, - args: PushArgsOwned, - retry: Option<&sqlx::types::Json>, - error_handler_path: Option<&str>, - error_handler_args: Option<&sqlx::types::Json>>>, - trigger_path: String, - job_id: Option, - trigger: TriggerMetadata, - suspended_mode: Option, -) -> Result<( - Uuid, - Option, - Option>, -)> { - #[cfg(feature = "enterprise")] - check_license_key_valid().await?; - - check_scopes(&authed, || format!("jobs:run:scripts:{script_path}"))?; - - let retry = retry.map(|r| r.0.clone()); - let error_handler_path = error_handler_path.map(|p| p.to_string()); - let error_handler_args = error_handler_args.map(|args| args.0.clone()); - - let (job_payload, tag, delete_after_use, timeout, on_behalf_of) = { - let db_authed = UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() }; - script_path_to_payload( - script_path, - Some(db_authed), - db.clone(), - &workspace_id, - Some(false), - ) - .await? - }; - - check_tag_available_for_workspace(&db, &workspace_id, &tag, &authed).await?; - - let return_tx = tx_o.is_some(); - - let (email, permissioned_as, push_authed, tx) = if let Some(tx) = tx_o { - ( - authed.email.as_str(), - username_to_permissioned_as(&authed.username), - Some(authed.clone().into()), - PushIsolationLevel::Transaction(tx), - ) - } else if let Some(on_behalf_of) = on_behalf_of.as_ref() { - ( - on_behalf_of.email.as_str(), - on_behalf_of.permissioned_as.clone(), - None, - PushIsolationLevel::IsolatedRoot(db.clone()), - ) - } else { - ( - authed.email.as_str(), - username_to_permissioned_as(&authed.username), - Some(authed.clone().into()), - PushIsolationLevel::Isolated(user_db, authed.clone().into()), - ) - }; - - let push_args = PushArgs { args: &args.args, extra: args.extra }; - - let retryable_job_payload = match job_payload { - JobPayload::ScriptHash { - hash, - path, - concurrency_settings, - debouncing_settings, - cache_ttl, - cache_ignore_s3_path, - priority, - apply_preprocessor, - .. - } => JobPayload::SingleStepFlow { - path, - hash: Some(hash), - flow_version: None, - args: HashMap::from(&push_args), - retry, - error_handler_path, - error_handler_args, - skip_handler: None, - cache_ttl, - cache_ignore_s3_path, - priority, - tag_override: tag.clone(), - apply_preprocessor, - trigger_path: Some(trigger_path.clone()), - concurrency_settings, - debouncing_settings, - }, - _ => { - return Err(windmill_common::error::Error::internal_err(format!( - "Unsupported job payload: {:?}", - job_payload - ))) - } - }; - let (uuid, tx) = push( - &db, - tx, - &workspace_id, - retryable_job_payload, - push_args, - authed.display_username(), - email, - permissioned_as, - authed.token_prefix.as_deref(), - None, - None, - None, - None, - None, - job_id, - false, - false, - None, - true, - tag, - timeout, - None, - None, - push_authed.as_ref(), - false, - None, - Some(trigger), - suspended_mode, - ) - .await?; - - // If we were given a transaction, return it; otherwise commit it - if return_tx { - Ok((uuid, delete_after_use, Some(tx))) - } else { - tx.commit().await?; - Ok((uuid, delete_after_use, None)) - } -} diff --git a/backend/windmill-api/src/triggers/websocket/handler.rs b/backend/windmill-api/src/triggers/websocket/handler.rs deleted file mode 100644 index bea4f195b2..0000000000 --- a/backend/windmill-api/src/triggers/websocket/handler.rs +++ /dev/null @@ -1,263 +0,0 @@ -use std::borrow::Cow; - -use crate::{ - db::{ApiAuthed, DB}, - triggers::{Trigger, TriggerCrud, TriggerData}, -}; -use axum::async_trait; -use itertools::Itertools; -use serde_json::value::RawValue; -use sqlx::{types::Json as SqlxJson, PgConnection}; -use tokio_tungstenite::connect_async; -use windmill_common::{ - db::UserDB, - error::{Error, Result}, - worker::to_raw_value, -}; -use windmill_git_sync::DeployedObject; - -use super::{ - get_url_from_runnable_value, TestWebsocketConfig, WebsocketConfig, WebsocketConfigRequest, - WebsocketTrigger, -}; - -#[async_trait] -impl TriggerCrud for WebsocketTrigger { - type TriggerConfig = WebsocketConfig; - type Trigger = Trigger; - type TriggerConfigRequest = WebsocketConfigRequest; - type TestConnectionConfig = TestWebsocketConfig; - - const TABLE_NAME: &'static str = "websocket_trigger"; - const TRIGGER_TYPE: &'static str = "websocket"; - const SUPPORTS_SERVER_STATE: bool = true; - const SUPPORTS_TEST_CONNECTION: bool = true; - const ROUTE_PREFIX: &'static str = "/websocket_triggers"; - const DEPLOYMENT_NAME: &'static str = "WebSocket trigger"; - const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[ - "url", - "filters", - "initial_messages", - "url_runnable_args", - "can_return_message", - "can_return_error_result", - ]; - const IS_ALLOWED_ON_CLOUD: bool = false; - - fn get_deployed_object(path: String) -> DeployedObject { - DeployedObject::WebsocketTrigger { path } - } - - async fn validate_config( - &self, - _db: &DB, - config: &Self::TriggerConfigRequest, - _workspace_id: &str, - ) -> Result<()> { - if config.url.trim().is_empty() { - return Err(Error::BadRequest( - "WebSocket URL cannot be empty".to_string(), - )); - } - - if let Some(args) = &config.url_runnable_args { - if !args.is_object() { - return Err(Error::BadRequest( - "url_runnable_args must be an object".to_string(), - )); - } - } - - Ok(()) - } - - async fn create_trigger( - &self, - _db: &DB, - tx: &mut PgConnection, - authed: &ApiAuthed, - w_id: &str, - trigger: TriggerData, - ) -> Result<()> { - let filters = trigger - .config - .filters - .into_iter() - .map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap())) - .collect_vec(); - let initial_messages = trigger - .config - .initial_messages - .unwrap_or_default() - .into_iter() - .map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap())) - .collect_vec(); - sqlx::query!( - r#" - INSERT INTO websocket_trigger ( - workspace_id, - path, - url, - script_path, - is_flow, - mode, - filters, - initial_messages, - url_runnable_args, - edited_by, - can_return_message, - can_return_error_result, - email, - edited_at, - error_handler_path, - error_handler_args, - retry - ) VALUES ( - $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, now(), $14, $15, $16 - ) - "#, - w_id, - trigger.base.path, - trigger.config.url, - trigger.base.script_path, - trigger.base.is_flow, - trigger.base.mode() as _, - &filters as _, - &initial_messages as _, - trigger - .config - .url_runnable_args - .map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap())) as _, - authed.username, - trigger.config.can_return_message, - trigger.config.can_return_error_result, - authed.email, - trigger.error_handling.error_handler_path, - trigger.error_handling.error_handler_args as _, - trigger.error_handling.retry as _ - ) - .execute(&mut *tx) - .await?; - Ok(()) - } - - async fn update_trigger( - &self, - _db: &DB, - tx: &mut PgConnection, - authed: &ApiAuthed, - w_id: &str, - path: &str, - trigger: TriggerData, - ) -> Result<()> { - let filters = trigger - .config - .filters - .into_iter() - .map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap())) - .collect_vec(); - let initial_messages = trigger - .config - .initial_messages - .unwrap_or_default() - .into_iter() - .map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap())) - .collect_vec(); - - // important to update server_id to NULL to stop current websocket listener - sqlx::query!( - " - UPDATE - websocket_trigger - SET - url = $1, - script_path = $2, - path = $3, - is_flow = $4, - filters = $5, - initial_messages = $6, - url_runnable_args = $7, - edited_by = $8, - email = $9, - can_return_message = $10, - can_return_error_result = $11, - edited_at = now(), - server_id = NULL, - error = NULL, - error_handler_path = $14, - error_handler_args = $15, - retry = $16 - WHERE - workspace_id = $12 AND path = $13 - ", - trigger.config.url, - trigger.base.script_path, - trigger.base.path, - trigger.base.is_flow, - filters.as_slice() as &[SqlxJson>], - initial_messages.as_slice() as &[SqlxJson>], - trigger - .config - .url_runnable_args - .map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap())) - as Option>>, - &authed.username, - &authed.email, - trigger.config.can_return_message, - trigger.config.can_return_error_result, - w_id, - path, - trigger.error_handling.error_handler_path, - trigger.error_handling.error_handler_args as _, - trigger.error_handling.retry as _ - ) - .execute(&mut *tx) - .await?; - - Ok(()) - } - - async fn test_connection( - &self, - db: &DB, - authed: &ApiAuthed, - _user_db: &UserDB, - workspace_id: &str, - config: Self::TestConnectionConfig, - ) -> Result<()> { - let url = &config.url; - - let connect_url: Cow = if url.starts_with("$") { - if url.starts_with("$flow:") || url.starts_with("$script:") { - let path = url.splitn(2, ':').nth(1).unwrap(); - Cow::Owned( - get_url_from_runnable_value( - path, - url.starts_with("$flow:"), - &db, - authed.clone(), - config.url_runnable_args.as_ref().map(to_raw_value).as_ref(), - &workspace_id, - ) - .await?, - ) - } else { - return Err(Error::BadConfig(format!( - "Invalid WebSocket runnable path: {}", - url - ))); - } - } else { - Cow::Borrowed(&url) - }; - - connect_async(&*connect_url).await.map_err(|err| { - Error::BadConfig(format!( - "Error connecting to WebSocket: {}", - err.to_string() - )) - })?; - - Ok(()) - } -} diff --git a/backend/windmill-api/src/triggers/websocket/listener.rs b/backend/windmill-api/src/triggers/websocket/listener.rs deleted file mode 100644 index 0f22dffdc3..0000000000 --- a/backend/windmill-api/src/triggers/websocket/listener.rs +++ /dev/null @@ -1,477 +0,0 @@ -use super::WebsocketTrigger; -use crate::triggers::{ - filter::{is_value_superset, Filter, JsonFilter}, - listener::ListeningTrigger, - trigger_helpers::{ - trigger_runnable, trigger_runnable_and_wait_for_raw_result, - trigger_runnable_and_wait_for_raw_result_with_error_ctx, TriggerJobArgs, - }, - websocket::{get_url_from_runnable_value, WebsocketConfig}, - Listener, -}; -use anyhow::Context; -use async_trait::async_trait; -use futures::{stream::SplitSink, SinkExt, StreamExt}; -use http::Response; -use itertools::Itertools; -use serde::Deserialize; -use serde_json::value::RawValue; -use std::{borrow::Cow, collections::HashMap, sync::Arc}; -use tokio::{net::TcpStream, sync::RwLock}; -use tokio_tungstenite::{connect_async, tungstenite::Message, MaybeTlsStream, WebSocketStream}; -use windmill_common::{ - error::{to_anyhow, Error, Result}, - jobs::JobTriggerKind, - triggers::TriggerMetadata, - utils::report_critical_error, - worker::to_raw_value, - DB, -}; -use windmill_queue::PushArgsOwned; - -impl ListeningTrigger { - async fn send_initial_messages( - &self, - writer: &mut SplitSink>, Message>, - db: &DB, - ) -> Result<()> { - let initial_messages: Vec = self - .trigger_config - .initial_messages - .as_deref() - .unwrap_or_default() - .iter() - .filter_map(|m| serde_json::from_str(m.get()).ok()) - .collect_vec(); - - let WebsocketConfig { ref url, .. } = self.trigger_config; - let runnable_kind = if self.is_flow { "flow" } else { "script" }; - let mut authed_o = None; - for start_message in initial_messages { - match start_message { - InitialMessage::RawMessage(msg) => { - let msg = if msg.starts_with("\"") && msg.ends_with("\"") { - msg[1..msg.len() - 1].to_string() - } else { - msg - }; - tracing::info!( - "Sending raw message initial message to WebSocket {}: {}", - url, - msg - ); - writer - .send(tokio_tungstenite::tungstenite::Message::Text(msg)) - .await - .map_err(to_anyhow) - .with_context(|| "failed to send raw message")?; - } - InitialMessage::RunnableResult { path, is_flow, args } => { - tracing::info!( - "Running {} {} for initial message to WebSocket {}", - runnable_kind, - path, - url, - ); - - let args = raw_value_to_args_hashmap(Some(&args))?; - - if authed_o.is_none() { - authed_o = Some(self.authed(db, "ws").await?); - } - let authed = authed_o.clone().unwrap(); - - let result = trigger_runnable_and_wait_for_raw_result_with_error_ctx( - db, - None, - authed.clone(), - &self.workspace_id, - &path, - is_flow, - PushArgsOwned { args, extra: None }, - None, - None, - None, - "".to_string(), // doesn't matter as no retry/error handler - TriggerMetadata::new(Some(self.path.to_owned()), JobTriggerKind::Websocket), - ) - .await - .map(|r| r.get().to_owned())?; - - tracing::info!( - "Sending {} {} result to WebSocket {}", - runnable_kind, - path, - url - ); - - // if the `result` was just a single string, the below removes the surrounding quotes by parsing it as a string. - // it falls back to the original serialized JSON if it doesn't work. - let result = serde_json::from_str::(result.as_str()).unwrap_or(result); - - writer - .send(tokio_tungstenite::tungstenite::Message::Text(result)) - .await - .map_err(to_anyhow) - .with_context(|| { - format!("Failed to send {} {} result", runnable_kind, path) - })?; - } - } - } - Ok(()) - } -} - -#[async_trait] -impl Listener for WebsocketTrigger { - type Consumer = ( - WebSocketStream>, - Response>>, - ); - type Extra = ReturnMessageChannels; - type ExtraState = (); - const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Websocket; - async fn get_consumer( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - err_message: Arc>>, - mut killpill_rx: tokio::sync::broadcast::Receiver<()>, - ) -> Result> { - let url = &listening_trigger.trigger_config.url; - let connect_url: Cow = if url.starts_with("$") { - if url.starts_with("$flow:") || url.starts_with("$script:") { - tokio::select! { - biased; - _ = killpill_rx.recv() => { - return Ok(None); - }, - _ = self.loop_ping(&db, listening_trigger, err_message.clone(), Some( - "Waiting on runnable to return WebSocket URL...".to_string() - )) => { - return Ok(None); - }, - url_result = { - let authed = listening_trigger.authed(db, "ws").await?; - let args = listening_trigger.trigger_config.url_runnable_args.as_ref().map(|r| &r.0); - let path = url.splitn(2, ':').nth(1).unwrap(); - get_url_from_runnable_value(path, url.starts_with("$flow:"), db, authed, args, &listening_trigger.workspace_id) - } => match url_result { - Ok(url) => Cow::Owned(url), - Err(err) => { - return Err(anyhow::anyhow!("Error getting WebSocket URL from runnable after 5 tries: {:?}", err).into()); - } - }, - } - } else { - return Err(anyhow::anyhow!("Invalid WebSocket runnable path: {}", url).into()); - } - } else { - Cow::Borrowed(&url) - }; - - let connection = connect_async(&*connect_url) - .await - .map(|conn| Some(conn)) - .map_err(|err| to_anyhow(err).into()); - - connection - } - async fn consume( - &self, - db: &DB, - consumer: Self::Consumer, - listening_trigger: &ListeningTrigger, - err_message: Arc>>, - mut killpill_rx: tokio::sync::broadcast::Receiver<()>, - _extra_state: Option<&Self::ExtraState>, - ) { - let WebsocketConfig { ref url, .. } = listening_trigger.trigger_config; - - tracing::info!("Connected to WebSocket {}", url); - - let (ws_stream, _) = consumer; - - let (mut writer, mut reader) = ws_stream.split(); - - // send initial messages - if listening_trigger.trigger_mode { - tokio::select! { - biased; - _ = killpill_rx.recv() => { - return; - }, - _ = self.loop_ping(db, listening_trigger, err_message.clone(), Some("Sending initial messages...".to_string())) => { - return; - }, - result = listening_trigger.send_initial_messages(&mut writer, &db) => { - if let Err(err) = result { - self.disable_with_error(&db, listening_trigger, format!("Error sending initial messages: {:?}", err)).await; - return - } else { - tracing::debug!("Initial messages sent successfully to WebSocket {}", url); - } - } - } - } - - let (return_message_channels, message_sender_handle) = if listening_trigger.trigger_mode - && listening_trigger.trigger_config.can_return_message - { - let (send_message_tx, mut rx) = tokio::sync::mpsc::channel::(100); - let w_id = listening_trigger.workspace_id.clone(); - let url = url.clone(); - let db = db.clone(); - let handle = tokio::spawn(async move { - while let Some(message) = rx.recv().await { - if let Err(err) = writer - .send(tokio_tungstenite::tungstenite::Message::Text(message)) - .await - { - report_critical_error(format!("Could not send runnable result to WebSocket {} because of error: {}", url, err), db.clone(), Some(&w_id), None).await; - } - } - }); - - let killpill_rx = killpill_rx.resubscribe(); - - let return_message_channels = ReturnMessageChannels { send_message_tx, killpill_rx }; - - (Some(return_message_channels), Some(handle)) - } else { - (None, None) - }; - - tokio::select! { - biased; - _ = killpill_rx.recv() => { - }, - _ = self.loop_ping(db, listening_trigger, err_message.clone(), None) => { - }, - _ = async { - let filters: Vec = if listening_trigger.trigger_mode { - listening_trigger - .trigger_config - .filters - .iter() - .filter_map(|m| serde_json::from_str(m.get()).ok()) - .collect_vec() - } else { - vec![] - }; - loop { - if let Some(msg) = reader.next().await { - match msg { - Ok(msg) => { - match msg { - tokio_tungstenite::tungstenite::Message::Text(text) => { - tracing::debug!("Received text message from WebSocket {}: {}", url, text); - let mut should_handle = true; - for filter in &filters { - match filter { - Filter::JsonFilter(JsonFilter { key, value }) => { - let mut deserializer = serde_json::Deserializer::from_str(text.as_str()); - should_handle = match is_value_superset(&mut deserializer, key, &value) { - Ok(filter_match) => { - filter_match - }, - Err(err) => { - tracing::warn!("Error deserializing filter for WebSocket {}: {:?}", url, err); - false - } - }; - } - } - if !should_handle { - break; - } - } - if should_handle { - let trigger_info = HashMap::from([ - ("url".to_string(), to_raw_value(&listening_trigger.trigger_config.url)), - ]); - let _ = self.handle_event(db, listening_trigger, text, trigger_info, return_message_channels.clone()).await; - } - }, - a @ _ => { - tracing::debug!("Received non text-message from WebSocket {}: {:?}", url, a); - } - } - }, - Err(err) => { - tracing::error!("Error reading from WebSocket {}: {:?}", url, err); - } - } - } else { - tracing::error!("WebSocket {} closed", url); - self.update_ping_and_loop_ping_status(db, listening_trigger, err_message.clone(), Some("WebSocket closed".to_string())).await; - break; - } - } - } => {} - } - // make sure to stop return message handler - if let Some(message_sender_handle) = message_sender_handle { - message_sender_handle.abort(); - } - } - - async fn handle_trigger( - &self, - db: &DB, - listening_trigger: &ListeningTrigger, - payload: Self::Payload, - trigger_info: HashMap>, - extra: Option, - ) -> Result<()> { - let ListeningTrigger { - path, - is_flow, - workspace_id, - trigger_config, - script_path, - error_handling, - suspended_mode, - .. - } = listening_trigger; - - let WebsocketConfig { url, .. } = trigger_config; - - let args = WebsocketTrigger::build_job_args( - &script_path, - *is_flow, - workspace_id, - db, - payload, - trigger_info, - ) - .await; - - let args = match args { - Ok(args) => args, - Err(err) => { - return Err(err); - } - }; - - let authed = listening_trigger.authed(db, "ws").await?; - - let (retry, error_handler_path, error_handler_args) = match error_handling.as_ref() { - Some(error_handling) => ( - error_handling.retry.as_ref(), - error_handling.error_handler_path.as_deref(), - error_handling.error_handler_args.as_ref(), - ), - None => (None, None, None), - }; - let trigger = TriggerMetadata::new(Some(path.to_owned()), Self::JOB_TRIGGER_KIND); - if *suspended_mode || extra.is_none() { - trigger_runnable( - db, - None, - authed, - &workspace_id, - &script_path, - *is_flow, - args, - retry, - error_handler_path, - error_handler_args, - format!("websocket_trigger/{}", listening_trigger.path), - None, - *suspended_mode, - trigger, - ) - .await?; - } else if let Some(ReturnMessageChannels { send_message_tx, mut killpill_rx }) = extra { - let db_ = db.clone(); - let url = url.to_owned(); - let script_path = script_path.to_owned(); - let is_flow = *is_flow; - let w_id = workspace_id.to_owned(); - let retry = retry.cloned(); - let error_handler_path = error_handler_path.map(|s| s.to_string()); - let error_handler_args = error_handler_args.cloned(); - let trigger_path = path.clone(); - let can_return_error_result = trigger_config.can_return_error_result; - let handle_response_f = async move { - tokio::select! { - _ = killpill_rx.recv() => { - return; - }, - result = trigger_runnable_and_wait_for_raw_result( - &db_, - None, - authed, - &w_id, - &script_path, - is_flow, - args, - retry.as_ref(), - error_handler_path.as_deref(), - error_handler_args.as_ref(), - format!("websocket_trigger/{}", trigger_path), - trigger, - ) => { - if let Ok((result, success)) = result { - if !success && !can_return_error_result { - return; - } - let result = result.get().to_owned(); - // only send the result if it's not null - if result != "null" { - tracing::info!("Sending job result to WebSocket {}", url); - // if the `result` was just a single string, the below removes the surrounding quotes by parsing it as a string. - // it falls back to the original serialized JSON if it doesn't work. - let result = serde_json::from_str::(result.as_str()).unwrap_or(result); - if let Err(err) = send_message_tx.send(result).await { - report_critical_error(format!("Could not send runnable result to WebSocket {} because of error: {}", url, err), db_.clone(), Some(&w_id), None).await; - } - } - } - } - }; - }; - - tokio::spawn(handle_response_f); - } - - Ok(()) - } -} - -pub struct ReturnMessageChannels { - send_message_tx: tokio::sync::mpsc::Sender, - killpill_rx: tokio::sync::broadcast::Receiver<()>, -} - -impl Clone for ReturnMessageChannels { - fn clone(&self) -> Self { - Self { - send_message_tx: self.send_message_tx.clone(), - killpill_rx: self.killpill_rx.resubscribe(), - } - } -} - -#[derive(Debug, Deserialize)] -enum InitialMessage { - #[serde(rename = "raw_message")] - RawMessage(String), - #[serde(rename = "runnable_result")] - RunnableResult { path: String, args: Box, is_flow: bool }, -} - -fn raw_value_to_args_hashmap( - args: Option<&Box>, -) -> Result>> { - let args = if let Some(args) = args { - serde_json::from_str::>>>(args.get()) - .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))? - .unwrap_or_else(HashMap::new) - } else { - HashMap::new() - }; - Ok(args) -} diff --git a/backend/windmill-api/src/triggers/websocket/mod.rs b/backend/windmill-api/src/triggers/websocket/mod.rs deleted file mode 100644 index f20b6334f7..0000000000 --- a/backend/windmill-api/src/triggers/websocket/mod.rs +++ /dev/null @@ -1,128 +0,0 @@ -use std::collections::HashMap; - -use crate::{ - db::ApiAuthed, - triggers::trigger_helpers::{ - trigger_runnable_and_wait_for_raw_result_with_error_ctx, TriggerJobArgs, - }, -}; -use serde::{Deserialize, Serialize}; -use serde_json::value::RawValue; -use sqlx::{types::Json as SqlxJson, FromRow}; -use windmill_common::{ - error::{Error, Result}, - jobs::JobTriggerKind, - triggers::{TriggerMetadata, TriggerKind}, - worker::to_raw_value, - DB, -}; -use windmill_queue::PushArgsOwned; - -mod handler; -mod listener; - -#[derive(Copy, Clone)] -pub struct WebsocketTrigger; - -impl TriggerJobArgs for WebsocketTrigger { - type Payload = String; - const TRIGGER_KIND: TriggerKind = TriggerKind::Websocket; - fn v1_payload_fn(payload: &Self::Payload) -> HashMap> { - HashMap::from([("msg".to_string(), to_raw_value(&payload))]) - } -} - -#[derive(Debug, Clone, FromRow, Serialize, Deserialize)] -pub struct WebsocketConfig { - pub url: String, - #[serde(default)] - pub filters: Vec>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub initial_messages: Option>>>, - #[serde(skip_serializing_if = "Option::is_none")] - pub url_runnable_args: Option>>, - #[serde(default)] - pub can_return_message: bool, - #[serde(default)] - pub can_return_error_result: bool, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct WebsocketConfigRequest { - url: String, - filters: Vec, - initial_messages: Option>, - url_runnable_args: Option, - can_return_message: bool, - can_return_error_result: bool, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TestWebsocketConfig { - url: String, - url_runnable_args: Option, -} - -pub fn value_to_args_hashmap( - args: Option<&Box>, -) -> Result>> { - let args = if let Some(args) = args { - let args_map: Option> = serde_json::from_str(args.get()) - .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))?; - - args_map - .unwrap_or_else(HashMap::new) - .into_iter() - .map(|(k, v)| { - let raw_value = serde_json::value::to_raw_value(&v).map_err(|e| { - Error::BadRequest(format!("failed to convert to raw value: {}", e)) - })?; - Ok((k, raw_value)) - }) - .collect::>>>() - } else { - Ok(HashMap::new()) - }?; - Ok(args) -} - -pub async fn get_url_from_runnable_value( - path: &str, - is_flow: bool, - db: &DB, - authed: ApiAuthed, - args: Option<&Box>, - workspace_id: &str, -) -> Result { - tracing::info!( - "Running {} {} to get WebSocket URL", - if is_flow { "flow" } else { "script" }, - path - ); - - let args = value_to_args_hashmap(args)?; - - let result = trigger_runnable_and_wait_for_raw_result_with_error_ctx( - db, - None, - authed, - workspace_id, - path, - is_flow, - PushArgsOwned { args, extra: None }, - None, - None, - None, - "".to_string(), // doesn't matter as no retry/error handler - TriggerMetadata::new(Some(path.to_owned()), JobTriggerKind::Websocket), - ) - .await?; - - serde_json::from_str::(result.get()).map_err(|_| { - Error::BadConfig(format!( - "{} {} did not return a string", - if is_flow { "Flow" } else { "Script" }, - path, - )) - }) -}