starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-06 00:02:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: complete trigger extraction by removing originals from windmill-api

Browse Source 
Remove the original triggers/ and native_triggers/ directories from
windmill-api/src/ and replace `pub mod` declarations with `pub use`
re-exports from the windmill-triggers crate. This completes the extraction
by ensuring trigger code is only compiled as part of windmill-triggers.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Ruben Fiszel
2026-02-06 12:42:24 +00:00
co-authored by Claude Opus 4.6
parent 9b165361d9
commit 5aaecf0da3
47 changed files with 2 additions and 14635 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/windmill-api/src/lib.rs
+2 -2
View File
@@ -171,7 +171,7 @@ pub mod teams_approvals_ee;
mod teams_approvals_oss;
#[cfg(feature = "native_trigger")]
pub mod native_triggers;
pub use windmill_triggers::native_triggers;
mod public_app_layer;
mod public_app_rate_limit;
mod static_assets;
@@ -187,7 +187,7 @@ pub mod teams_ee;
mod teams_oss;
mod token;
mod tracing_init;
pub mod triggers;
pub use windmill_triggers::triggers;
mod users;
#[cfg(feature = "private")]
pub mod users_ee;
backend/windmill-api/src/native_triggers/handler.rs
-566
View File
@@ -1,566 +0,0 @@
use crate::{
db::ApiAuthed,
native_triggers::{
delete_native_trigger, delete_token_by_prefix, get_native_trigger, get_token_by_prefix,
get_workspace_integration, list_native_triggers, store_native_trigger,
update_native_trigger_error, External, NativeTrigger, NativeTriggerConfig,
NativeTriggerData, ServiceName,
},
users::{create_token_internal, NewToken},
utils::check_scopes,
};
use axum::{
extract::{Path, Query},
routing::{delete, get, post},
Extension, Json, Router,
};
use serde::{Deserialize, Serialize};
use sqlx::PgConnection;
use std::sync::Arc;
use windmill_audit::{audit_oss::audit_log, ActionKind};
use windmill_common::{
db::UserDB,
error::{Error, JsonResult, Result},
utils::rd_string,
DB,
};
async fn require_is_writer_on_runnable(
authed: &ApiAuthed,
path: &str,
is_flow: bool,
w_id: &str,
db: DB,
) -> Result<()> {
if is_flow {
crate::flows::require_is_writer(authed, path, w_id, db).await
} else {
crate::scripts::require_is_writer(authed, path, w_id, db).await
}
}
#[derive(Debug, Deserialize)]
pub struct ListQuery {
pub page: Option<usize>,
pub per_page: Option<usize>,
pub path: Option<String>,
pub is_flow: Option<bool>,
}
#[derive(Debug, Serialize)]
pub struct FullTriggerResponse<T: Serialize> {
#[serde(flatten)]
pub windmill_data: NativeTrigger,
pub external_data: T,
}
#[derive(Debug, Serialize)]
pub struct CreateTriggerResponse {
pub external_id: String,
}
async fn new_webhook_token(
tx: &mut PgConnection,
db: &DB,
authed: &ApiAuthed,
script_path: &str,
is_flow: bool,
workspace_id: &str,
service_name: ServiceName,
) -> Result<String> {
let kind = if is_flow { "flows" } else { "scripts" };
let scopes = vec![format!("jobs:run:{kind}:{script_path}")];
let label = format!("webhook-{}-{}", service_name.as_str(), rd_string(5));
let token_config = NewToken::new(
Some(label),
None,
None,
Some(scopes),
Some(workspace_id.to_owned()),
);
let token = create_token_internal(&mut *tx, &db, &authed, token_config).await?;
Ok(token)
}
async fn create_native_trigger<T: External>(
Extension(handler): Extension<Arc<T>>,
Extension(service_name): Extension<ServiceName>,
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path(workspace_id): Path<String>,
Json(data): Json<NativeTriggerData<T::ServiceConfig>>,
) -> JsonResult<CreateTriggerResponse> {
check_scopes(&authed, || {
format!("native_triggers:write:{}", &data.script_path)
})?;
require_is_writer_on_runnable(
&authed,
&data.script_path,
data.is_flow,
&workspace_id,
db.clone(),
)
.await?;
let mut tx = user_db.begin(&authed).await?;
let webhook_token = new_webhook_token(
&mut *tx,
&db,
&authed,
&data.script_path,
data.is_flow,
&workspace_id,
service_name,
)
.await?;
let integration = get_workspace_integration(&mut *tx, &workspace_id, service_name).await?;
let oauth_data: T::OAuthData = serde_json::from_value(integration.oauth_data).map_err(|e| {
Error::InternalErr(format!(
"Failed to parse {} OAuth data: {}",
T::DISPLAY_NAME,
e
))
})?;
let resp = handler
.create(
&workspace_id,
&oauth_data,
&webhook_token,
&data,
&db,
&mut tx,
)
.await?;
let (external_id, _) = handler.external_id_and_metadata_from_response(&resp);
// update the created external trigger with a new uri containing the external_id
handler
.update(
&workspace_id,
&oauth_data,
&external_id,
&webhook_token,
&data,
&db,
&mut tx,
)
.await?;
// Fetch the updated trigger data from the external service and extract service_config
let trigger_data = handler
.get(&workspace_id, &oauth_data, &external_id, &db, &mut tx)
.await?;
let service_config = handler.extract_service_config_from_trigger_data(&trigger_data)?;
let config = NativeTriggerConfig {
script_path: data.script_path.clone(),
is_flow: data.is_flow,
webhook_token,
};
store_native_trigger(
&mut *tx,
&workspace_id,
service_name,
&external_id,
&config,
service_config,
)
.await?;
audit_log(
&mut *tx,
&authed,
&format!("native_triggers.{}.create", service_name),
ActionKind::Create,
&workspace_id,
Some(&external_id),
None,
)
.await?;
tx.commit().await?;
Ok(Json(CreateTriggerResponse { external_id }))
}
async fn update_native_trigger_handler<T: External>(
Extension(handler): Extension<Arc<T>>,
Extension(service_name): Extension<ServiceName>,
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path((workspace_id, external_id)): Path<(String, String)>,
Json(data): Json<NativeTriggerData<T::ServiceConfig>>,
) -> Result<String> {
check_scopes(&authed, || {
format!("native_triggers:write:{}", &data.script_path)
})?;
require_is_writer_on_runnable(
&authed,
&data.script_path,
data.is_flow,
&workspace_id,
db.clone(),
)
.await?;
let mut tx = user_db.begin(&authed).await?;
let existing = get_native_trigger(&mut *tx, &workspace_id, service_name, &external_id)
.await?
.ok_or_else(|| Error::NotFound(format!("Native trigger not found: {}", external_id)))?;
// Look up the full token using the stored prefix (use db, not tx, for token table)
let webhook_token = match get_token_by_prefix(&db, &existing.webhook_token_prefix).await? {
Some(token) => token,
None => {
tracing::warn!(
"Webhook token not found for trigger {} (prefix: {}), recreating token",
external_id,
existing.webhook_token_prefix
);
new_webhook_token(
&mut *tx,
&db,
&authed,
&data.script_path,
data.is_flow,
&workspace_id,
service_name,
)
.await?
}
};
let integration = get_workspace_integration(&mut *tx, &workspace_id, service_name).await?;
let oauth_data: T::OAuthData = serde_json::from_value(integration.oauth_data).map_err(|e| {
Error::InternalErr(format!(
"Failed to parse {} OAuth data: {}",
T::DISPLAY_NAME,
e
))
})?;
handler
.update(
&workspace_id,
&oauth_data,
&external_id,
&webhook_token,
&data,
&db,
&mut tx,
)
.await?;
// Fetch the updated trigger data from the external service and extract service_config
let trigger_data = handler
.get(&workspace_id, &oauth_data, &external_id, &db, &mut tx)
.await?;
let service_config = handler.extract_service_config_from_trigger_data(&trigger_data)?;
let config = NativeTriggerConfig {
script_path: data.script_path.clone(),
is_flow: data.is_flow,
webhook_token,
};
store_native_trigger(
&mut *tx,
&workspace_id,
service_name,
&external_id,
&config,
service_config,
)
.await?;
audit_log(
&mut *tx,
&authed,
&format!("native_triggers.{}.update", service_name),
ActionKind::Update,
&workspace_id,
Some(&external_id),
None,
)
.await?;
tx.commit().await?;
Ok(format!("Native trigger updated"))
}
async fn get_native_trigger_handler<T: External>(
Extension(handler): Extension<Arc<T>>,
Extension(service_name): Extension<ServiceName>,
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path((workspace_id, external_id)): Path<(String, String)>,
) -> JsonResult<FullTriggerResponse<T::TriggerData>> {
let mut tx = user_db.begin(&authed).await?;
let windmill_trigger = get_native_trigger(&mut *tx, &workspace_id, service_name, &external_id)
.await?
.ok_or_else(|| Error::NotFound(format!("Native trigger not found: {}", external_id)))?;
check_scopes(&authed, || {
format!("native_triggers:read:{}", &windmill_trigger.script_path)
})?;
require_is_writer_on_runnable(
&authed,
&windmill_trigger.script_path,
windmill_trigger.is_flow,
&workspace_id,
db.clone(),
)
.await?;
let integration = get_workspace_integration(&mut *tx, &workspace_id, service_name).await?;
let oauth_data: T::OAuthData = serde_json::from_value(integration.oauth_data).map_err(|e| {
Error::InternalErr(format!(
"Failed to parse {} OAuth data: {}",
T::DISPLAY_NAME,
e
))
})?;
let native_trigger = handler
.get(&workspace_id, &oauth_data, &external_id, &db, &mut tx)
.await;
let native_trigger_config = match native_trigger {
Ok(native_cfg) => {
// Clear error if it was set
if windmill_trigger.error.is_some() {
update_native_trigger_error(
&mut *tx,
&workspace_id,
service_name,
&external_id,
None,
)
.await?;
}
native_cfg
}
Err(Error::NotFound(_)) => {
let error_msg = "Trigger no longer exists on external service".to_string();
tracing::warn!(
"Native trigger no longer exists on external service {}, setting error",
service_name
);
update_native_trigger_error(
&mut *tx,
&workspace_id,
service_name,
&external_id,
Some(&error_msg),
)
.await?;
tx.commit().await?;
return Err(Error::NotFound(format!(
"Trigger '{}' no longer exists on external service {}",
external_id, service_name
)));
}
Err(e) => return Err(e),
};
let full_resp = Json(FullTriggerResponse {
windmill_data: windmill_trigger,
external_data: native_trigger_config,
});
Ok(full_resp)
}
async fn delete_native_trigger_handler<T: External>(
Extension(handler): Extension<Arc<T>>,
Extension(service_name): Extension<ServiceName>,
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path((workspace_id, external_id)): Path<(String, String)>,
) -> Result<String> {
let mut tx = user_db.begin(&authed).await?;
let existing = get_native_trigger(&mut *tx, &workspace_id, service_name, &external_id)
.await?
.ok_or_else(|| Error::NotFound(format!("Native trigger not found: {}", external_id)))?;
check_scopes(&authed, || {
format!("native_triggers:write:{}", &existing.script_path)
})?;
require_is_writer_on_runnable(
&authed,
&existing.script_path,
existing.is_flow,
&workspace_id,
db.clone(),
)
.await?;
let integration = get_workspace_integration(&mut *tx, &workspace_id, service_name).await?;
let oauth_data: T::OAuthData = serde_json::from_value(integration.oauth_data).map_err(|e| {
Error::InternalErr(format!(
"Failed to parse {} OAuth data: {}",
T::DISPLAY_NAME,
e
))
})?;
handler
.delete(&workspace_id, &oauth_data, &external_id, &db, &mut tx)
.await?;
let deleted =
delete_native_trigger(&mut *tx, &workspace_id, service_name, &external_id).await?;
if !deleted {
return Err(Error::NotFound(format!("Native trigger not found")));
}
// Delete the webhook token using its prefix
if !delete_token_by_prefix(&db, &existing.webhook_token_prefix).await? {
tracing::warn!(
"Webhook token not found when deleting trigger {} (prefix: {})",
external_id,
existing.webhook_token_prefix
);
}
audit_log(
&mut *tx,
&authed,
&format!("native_triggers.{}.delete", service_name),
ActionKind::Delete,
&workspace_id,
Some(&external_id),
None,
)
.await?;
tx.commit().await?;
Ok(format!("Native trigger deleted"))
}
async fn exists_native_trigger_handler<T: External>(
Extension(service_name): Extension<ServiceName>,
_authed: ApiAuthed,
Extension(db): Extension<DB>,
Path((workspace_id, external_id)): Path<(String, String)>,
) -> JsonResult<bool> {
let exists = sqlx::query_scalar!(
r#"
SELECT EXISTS(
SELECT 1
FROM native_trigger
WHERE
workspace_id = $1 AND
service_name = $2 AND
external_id = $3
)
"#,
workspace_id,
service_name as ServiceName,
external_id
)
.fetch_one(&db)
.await?
.unwrap_or(false);
Ok(Json(exists))
}
async fn list_native_triggers_handler<T: External>(
Extension(service_name): Extension<ServiceName>,
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path(workspace_id): Path<String>,
Query(query): Query<ListQuery>,
) -> JsonResult<Vec<NativeTrigger>> {
let mut tx = user_db.begin(&authed).await?;
let triggers = list_native_triggers(
&mut *tx,
&workspace_id,
service_name,
query.page,
query.per_page,
query.path.as_deref(),
query.is_flow,
)
.await?;
tx.commit().await?;
Ok(Json(triggers))
}
pub fn service_routes<T: External + 'static>(handler: T) -> Router {
let additional_routes = handler.additional_routes();
let service_name = T::SERVICE_NAME;
let handler_arc = Arc::new(handler);
let standard_routes = Router::new()
.route("/create", post(create_native_trigger::<T>))
.route("/list", get(list_native_triggers_handler::<T>))
.route("/get/:external_id", get(get_native_trigger_handler::<T>))
.route(
"/update/:external_id",
post(update_native_trigger_handler::<T>),
)
.route(
"/delete/:external_id",
delete(delete_native_trigger_handler::<T>),
)
.route(
"/exists/:external_id",
get(exists_native_trigger_handler::<T>),
);
standard_routes
.merge(additional_routes)
.layer(Extension(handler_arc))
.layer(Extension(service_name))
}
/// Generates routes for all registered native trigger services.
/// When adding a new service, add a new `.nest()` call here.
pub fn generate_native_trigger_routers() -> Router {
let router = Router::new();
#[cfg(feature = "native_trigger")]
{
use crate::native_triggers::nextcloud::NextCloud;
// Register all service routes here
// When adding a new service:
// 1. Import the handler: use crate::native_triggers::newservice::NewServiceHandler;
// 2. Add the route: .nest("/newservice", service_routes(NewServiceHandler))
return router.nest("/nextcloud", service_routes(NextCloud));
// Add new services here:
// .nest("/newservice", service_routes(NewServiceHandler))
}
#[cfg(not(feature = "native_trigger"))]
{
router
}
}
backend/windmill-api/src/native_triggers/mod.rs
-1053
View File
File diff suppressed because it is too large Load Diff
backend/windmill-api/src/native_triggers/nextcloud/external.rs
-304
View File
@@ -1,304 +0,0 @@
use reqwest::Method;
use serde::{Deserialize, Serialize};
use serde_json::value::to_raw_value;
use sqlx::PgConnection;
use std::collections::HashMap;
use windmill_common::{
error::{Error, Result},
BASE_URL, DB,
};
use crate::native_triggers::{
generate_webhook_service_url,
nextcloud::{
routes, NextCloud, NextCloudOAuthData, NextCloudTriggerData, NextcloudServiceConfig,
OcsResponse,
},
External, NativeTriggerData, ServiceName,
};
lazy_static::lazy_static! {
pub static ref TOKEN_NEEDED: Box<serde_json::value::RawValue> = to_raw_value(&serde_json::json!({
"user_roles": ["owner", "trigger"]
})).unwrap();
}
#[derive(Debug, Serialize)]
#[serde(rename_all = "camelCase")]
struct FullNextcloudPayload {
pub http_method: String,
pub uri: String,
pub token_needed: Box<serde_json::value::RawValue>,
#[serde(flatten)]
service_config: NextcloudServiceConfig,
}
impl FullNextcloudPayload {
async fn new(
w_id: &str,
external_id: Option<&str>,
webhook_token: &str,
data: &NativeTriggerData<NextcloudServiceConfig>,
) -> FullNextcloudPayload {
let base_url = &*BASE_URL.read().await;
let uri = generate_webhook_service_url(
base_url,
w_id,
&data.script_path,
data.is_flow,
external_id,
ServiceName::Nextcloud,
webhook_token,
);
FullNextcloudPayload {
http_method: http::Method::POST.to_string().to_uppercase(),
uri,
token_needed: TOKEN_NEEDED.clone(),
service_config: data.service_config.clone(),
}
}
}
#[derive(Debug, Deserialize)]
pub struct RegisterWebhookResponse {
pub id: i64,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct User {
pub uid: String,
#[serde(rename = "displayName")]
pub display_name: Option<String>,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct WebhookPayload {
pub event: EventPayload,
pub user: User,
pub time: i64,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct EventPayload {
pub node: Node,
#[serde(rename = "class")]
pub class_name: String,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct Node {
pub id: i64,
pub path: String,
}
#[async_trait::async_trait]
impl External for NextCloud {
type ServiceConfig = NextcloudServiceConfig;
type TriggerData = NextCloudTriggerData;
type OAuthData = NextCloudOAuthData;
type CreateResponse = RegisterWebhookResponse;
const SERVICE_NAME: ServiceName = ServiceName::Nextcloud;
const DISPLAY_NAME: &'static str = "Nextcloud";
const SUPPORT_WEBHOOK: bool = true;
const TOKEN_ENDPOINT: &'static str = "/apps/oauth2/api/v1/token";
const REFRESH_ENDPOINT: &'static str = "/apps/oauth2/api/v1/token";
async fn create(
&self,
w_id: &str,
oauth_data: &Self::OAuthData,
webhook_token: &str,
data: &NativeTriggerData<Self::ServiceConfig>,
db: &DB,
tx: &mut PgConnection,
) -> Result<Self::CreateResponse> {
// During create, we don't have external_id yet (it comes from NextCloud's response)
let full_nextcloud_payload =
FullNextcloudPayload::new(w_id, None, webhook_token, data).await;
let url = format!(
"{}/ocs/v2.php/apps/webhook_listeners/api/v1/webhooks",
oauth_data.base_url
);
let mut headers = HashMap::new();
headers.insert("OCS-APIRequest".to_string(), "true".to_string());
let ocs_response = self
.http_client_request::<OcsResponse<RegisterWebhookResponse>, _>(
&url,
Method::POST,
w_id,
tx,
db,
Some(headers),
Some(&full_nextcloud_payload),
)
.await?;
Ok(ocs_response.ocs.data)
}
async fn update(
&self,
w_id: &str,
oauth_data: &Self::OAuthData,
external_id: &str,
webhook_token: &str,
data: &NativeTriggerData<Self::ServiceConfig>,
db: &DB,
tx: &mut PgConnection,
) -> Result<()> {
// During update, we have the external_id so include it in the webhook URL
let full_nextcloud_payload =
FullNextcloudPayload::new(w_id, Some(external_id), webhook_token, data).await;
let url = format!(
"{}/ocs/v2.php/apps/webhook_listeners/api/v1/webhooks/{}",
oauth_data.base_url, external_id
);
let mut headers = HashMap::new();
headers.insert("OCS-APIRequest".to_string(), "true".to_string());
let _ = self
.http_client_request::<serde_json::Value, _>(
&url,
Method::POST,
w_id,
tx,
db,
Some(headers),
Some(&full_nextcloud_payload),
)
.await?;
Ok(())
}
async fn get(
&self,
w_id: &str,
oauth_data: &Self::OAuthData,
external_id: &str,
db: &DB,
tx: &mut PgConnection,
) -> Result<Self::TriggerData> {
let url = format!(
"{}/ocs/v2.php/apps/webhook_listeners/api/v1/webhooks/{}",
oauth_data.base_url, external_id
);
let mut headers = HashMap::new();
headers.insert("OCS-APIRequest".to_string(), "true".to_string());
let ocs_response: OcsResponse<NextCloudTriggerData> = self
.http_client_request::<_, ()>(&url, Method::GET, w_id, tx, db, Some(headers), None)
.await?;
Ok(ocs_response.ocs.data)
}
async fn delete(
&self,
w_id: &str,
oauth_data: &Self::OAuthData,
external_id: &str,
db: &DB,
tx: &mut PgConnection,
) -> Result<()> {
let url = format!(
"{}/ocs/v2.php/apps/webhook_listeners/api/v1/webhooks/{}",
oauth_data.base_url, external_id
);
let mut headers = HashMap::new();
headers.insert("OCS-APIRequest".to_string(), "true".to_string());
let _: serde_json::Value = self
.http_client_request::<_, ()>(&url, Method::DELETE, w_id, tx, db, Some(headers), None)
.await
.or_else(|e| match &e {
Error::InternalErr(msg) if msg.contains("404") => Ok(serde_json::Value::Null),
_ => Err(e),
})?;
Ok(())
}
async fn exists(
&self,
w_id: &str,
oauth_data: &Self::OAuthData,
external_id: &str,
db: &DB,
tx: &mut PgConnection,
) -> Result<bool> {
let url = format!(
"{}/ocs/v2.php/apps/webhook_listeners/api/v1/webhooks/{}",
oauth_data.base_url, external_id
);
let mut headers = HashMap::new();
headers.insert("OCS-APIRequest".to_string(), "true".to_string());
let _ = self
.http_client_request::<serde_json::Value, ()>(
&url,
Method::GET,
w_id,
tx,
db,
Some(headers),
None,
)
.await?;
Ok(true)
}
async fn list_all(
&self,
w_id: &str,
oauth_data: &Self::OAuthData,
db: &DB,
tx: &mut PgConnection,
) -> Result<Vec<Self::TriggerData>> {
let url = format!(
"{}/ocs/v2.php/apps/webhook_listeners/api/v1/webhooks",
oauth_data.base_url
);
let mut headers = HashMap::new();
headers.insert("OCS-APIRequest".to_string(), "true".to_string());
let ocs_response = self
.http_client_request::<OcsResponse<Vec<NextCloudTriggerData>>, ()>(
&url,
Method::GET,
w_id,
tx,
db,
Some(headers),
None,
)
.await?;
Ok(ocs_response.ocs.data)
}
fn external_id_and_metadata_from_response(
&self,
resp: &Self::CreateResponse,
) -> (String, Option<serde_json::Value>) {
(resp.id.to_string(), None)
}
fn get_external_id_from_trigger_data(&self, data: &Self::TriggerData) -> String {
data.id.to_string()
}
fn additional_routes(&self) -> axum::Router {
routes::nextcloud_routes(self.clone())
}
}
backend/windmill-api/src/native_triggers/nextcloud/mod.rs
-64
View File
@@ -1,64 +0,0 @@
use serde::{Deserialize, Serialize};
pub mod external;
mod routes;
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct NextCloudOAuthData {
pub base_url: String,
pub access_token: String,
pub refresh_token: Option<String>,
pub token_expires_at: Option<chrono::DateTime<chrono::Utc>>,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct OcsResponse<T = String> {
pub ocs: OcsData<T>,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct Meta {
status: String,
#[serde(rename = "statuscode")]
status_code: u16,
message: String,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct OcsData<T> {
pub meta: Meta,
pub data: T,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct NextCloudEventType {
pub name: String,
pub description: Option<String>,
pub path: Option<String>,
pub parameters: Option<serde_json::Value>,
}
#[derive(Copy, Clone)]
pub struct NextCloud;
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct NextcloudServiceConfig {
pub event: String,
pub event_filter: Option<Box<serde_json::value::RawValue>>,
pub user_id_filter: Option<String>,
pub headers: Option<Box<serde_json::value::RawValue>>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct NextCloudTriggerData {
#[serde(skip_serializing)]
pub id: i64,
#[serde(skip_serializing)]
pub uri: String,
pub event: String,
pub event_filter: Option<Box<serde_json::value::RawValue>>,
pub user_id_filter: Option<String>,
pub headers: Option<Box<serde_json::value::RawValue>>,
}
backend/windmill-api/src/native_triggers/nextcloud/routes.rs
-66
View File
@@ -1,66 +0,0 @@
use std::{collections::HashMap, sync::Arc};
use axum::{extract::Path, routing::get, Extension, Json, Router};
use http::Method;
use windmill_common::{
db::UserDB,
error::{Error, JsonResult},
DB,
};
use crate::{
db::ApiAuthed,
native_triggers::{
get_workspace_integration,
nextcloud::{NextCloudEventType, OcsResponse},
External, OAuthConfig, ServiceName,
},
};
async fn list_available_events<T: External>(
authed: ApiAuthed,
Extension(handler): Extension<Arc<T>>,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path(workspace_id): Path<String>,
) -> JsonResult<Vec<NextCloudEventType>> {
let mut tx = user_db.clone().begin(&authed).await?;
let integration =
get_workspace_integration(&mut *tx, &workspace_id, ServiceName::Nextcloud).await?;
let auth = serde_json::from_value::<OAuthConfig>(integration.oauth_data)
.map_err(|e| Error::InternalErr(format!("Failed to parse NextCloud OAuth data: {}", e)))?;
let url = format!(
"{}/ocs/v2.php/apps/integration_windmill/api/v1/list/events",
&auth.base_url,
);
let mut headers = HashMap::new();
headers.insert("OCS-APIRequest".to_string(), "true".to_string());
let ocs_response = handler
.http_client_request::<OcsResponse, ()>(
&url,
Method::GET,
&workspace_id,
&mut *tx,
&db,
Some(headers),
None,
)
.await?;
tx.commit().await?;
let events = serde_json::from_str(&ocs_response.ocs.data)
.map_err(|e| Error::InternalErr(format!("Failed to parse NextCloud events data: {}", e)))?;
Ok(Json(events))
}
pub fn nextcloud_routes<T: External>(service: T) -> Router {
let service = Arc::new(service);
Router::new()
.route("/events", get(list_available_events::<T>))
.layer(Extension(service))
}
backend/windmill-api/src/native_triggers/sync.rs
-416
View File
@@ -1,416 +0,0 @@
use std::collections::HashMap;
use windmill_common::error::Result;
use windmill_common::DB;
use serde::Serialize;
use crate::native_triggers::{
decrypt_oauth_data, list_native_triggers, update_native_trigger_error,
update_native_trigger_service_config, External, ServiceName,
};
#[derive(Debug, Serialize)]
pub struct TriggerSyncInfo {
pub external_id: String,
pub script_path: String,
pub action: SyncAction,
}
#[derive(Debug, Serialize)]
pub enum SyncAction {
ErrorSet(String),
ErrorCleared,
ConfigUpdated,
}
#[derive(Debug, Serialize)]
pub struct SyncError {
pub resource_path: String,
pub error_message: String,
pub error_type: String,
}
#[derive(Debug)]
pub struct BackgroundSyncResult {
pub workspaces_processed: usize,
pub total_synced: usize,
pub total_errors: usize,
pub service_results: HashMap<ServiceName, ServiceSyncResult>,
}
#[derive(Debug)]
pub struct ServiceSyncResult {
pub synced_triggers: Vec<TriggerSyncInfo>,
pub errors: Vec<SyncError>,
}
pub async fn sync_all_triggers(db: &DB) -> Result<BackgroundSyncResult> {
tracing::info!("Starting native triggers sync");
let mut service_results: HashMap<ServiceName, ServiceSyncResult> = HashMap::new();
let mut total_synced = 0;
let mut total_errors = 0;
let mut workspaces_processed = 0;
// Sync all registered services
// Each service only syncs workspaces that have the corresponding integration configured
#[cfg(feature = "native_trigger")]
{
use crate::native_triggers::nextcloud::NextCloud;
let (service_name, result) = sync_service_triggers(db, NextCloud).await;
total_synced += result.synced_triggers.len();
total_errors += result.errors.len();
service_results.insert(service_name, result);
// Add new services here:
// use crate::native_triggers::newservice::NewService;
// let (service_name, result) = sync_service_triggers(db, NewService).await;
// total_synced += result.synced_triggers.len();
// total_errors += result.errors.len();
// service_results.insert(service_name, result);
}
// Count unique workspaces processed across all services
for result in service_results.values() {
workspaces_processed += result
.synced_triggers
.iter()
.map(|t| &t.external_id)
.collect::<std::collections::HashSet<_>>()
.len();
}
let result =
BackgroundSyncResult { workspaces_processed, total_synced, total_errors, service_results };
tracing::info!(
"Completed native triggers sync: {} updated, {} errors",
result.total_synced,
result.total_errors
);
Ok(result)
}
async fn sync_service_triggers<T: External>(
db: &DB,
handler: T,
) -> (ServiceName, ServiceSyncResult) {
let mut all_synced_triggers = Vec::new();
let mut all_errors = Vec::new();
// Only sync workspaces that have the corresponding integration configured
let workspaces_with_integration = match sqlx::query_scalar!(
r#"
SELECT wi.workspace_id
FROM workspace_integrations wi
JOIN workspace w ON w.id = wi.workspace_id
WHERE wi.service_name = $1
AND wi.oauth_data IS NOT NULL
AND w.deleted = false
"#,
T::SERVICE_NAME as ServiceName
)
.fetch_all(db)
.await
{
Ok(workspaces) => workspaces,
Err(e) => {
tracing::error!(
"Error querying workspaces with {} integration: {:#}",
T::SERVICE_NAME.as_str(),
e
);
all_errors.push(SyncError {
resource_path: "database".to_string(),
error_message: format!("Failed to query workspaces: {}", e),
error_type: "database_error".to_string(),
});
return (
T::SERVICE_NAME,
ServiceSyncResult { synced_triggers: Vec::new(), errors: all_errors },
);
}
};
if workspaces_with_integration.is_empty() {
tracing::debug!(
"No workspaces with {} integration configured, skipping sync",
T::SERVICE_NAME.as_str()
);
return (
T::SERVICE_NAME,
ServiceSyncResult { synced_triggers: Vec::new(), errors: Vec::new() },
);
}
tracing::info!(
"Found {} workspaces with {} integration configured",
workspaces_with_integration.len(),
T::SERVICE_NAME.as_str()
);
for workspace_id in workspaces_with_integration {
let sync_result = sync_workspace_triggers::<T>(db, &workspace_id, &handler).await;
match sync_result {
Ok((synced_triggers, errors)) => {
all_synced_triggers.extend(synced_triggers);
all_errors.extend(errors);
}
Err(e) => {
tracing::error!(
"Error syncing {} triggers for workspace {}: {:#}",
T::SERVICE_NAME.as_str(),
workspace_id,
e
);
all_errors.push(SyncError {
resource_path: format!("workspace:{}", workspace_id),
error_message: format!("Failed to sync workspace: {}", e),
error_type: "workspace_sync_error".to_string(),
});
}
}
}
(
T::SERVICE_NAME,
ServiceSyncResult { synced_triggers: all_synced_triggers, errors: all_errors },
)
}
#[cfg(feature = "native_trigger")]
pub async fn sync_workspace_triggers<T: External>(
db: &DB,
workspace_id: &str,
handler: &T,
) -> Result<(Vec<TriggerSyncInfo>, Vec<SyncError>)> {
tracing::info!(
"Syncing {} triggers for workspace '{}'",
T::SERVICE_NAME.as_str(),
workspace_id
);
let windmill_triggers =
list_native_triggers(db, workspace_id, T::SERVICE_NAME, None, None, None, None).await?;
if windmill_triggers.is_empty() {
tracing::info!(
"No {} triggers found for workspace '{}'",
T::SERVICE_NAME.as_str(),
workspace_id
);
return Ok((Vec::new(), Vec::new()));
}
let mut all_synced_triggers = Vec::new();
let mut all_sync_errors = Vec::new();
let oauth_data = {
match decrypt_oauth_data(db, db, workspace_id, T::SERVICE_NAME).await {
Ok(oauth_data) => oauth_data,
Err(e) => {
tracing::error!(
"Failed to get workspace integration OAuth data for {}: {}",
workspace_id,
e
);
all_sync_errors.push(SyncError {
resource_path: format!("workspace:{}", workspace_id),
error_message: format!("Failed to get workspace integration OAuth data: {}", e),
error_type: "oauth_error".to_string(),
});
return Ok((Vec::new(), all_sync_errors));
}
}
};
let mut tx = db.begin().await?;
let external_triggers = match handler
.list_all(workspace_id, &oauth_data, db, &mut tx)
.await
{
Ok(triggers) => triggers,
Err(e) => {
tracing::error!(
"Failed to fetch external triggers for {}: {}",
workspace_id,
e
);
all_sync_errors.push(SyncError {
resource_path: format!("workspace:{}", workspace_id),
error_message: format!("Failed to fetch external triggers: {}", e),
error_type: "external_service_error".to_string(),
});
return Ok((Vec::new(), all_sync_errors));
}
};
tx.commit().await?;
// Build a map of external trigger IDs to their data
let mut external_trigger_map: HashMap<String, &T::TriggerData> = HashMap::new();
for external_trigger in &external_triggers {
let external_id = handler.get_external_id_from_trigger_data(external_trigger);
external_trigger_map.insert(external_id, external_trigger);
}
for trigger in &windmill_triggers {
if !external_trigger_map.contains_key(&trigger.external_id) {
// Trigger no longer exists on external service - set error
let error_msg = "Trigger no longer exists on external service".to_string();
if trigger.error.as_deref() != Some(&error_msg) {
tracing::info!(
"Trigger (external_id: '{}', script_path: '{}') no longer exists in external service, setting error",
trigger.external_id,
trigger.script_path
);
match update_native_trigger_error(
db,
workspace_id,
T::SERVICE_NAME,
&trigger.external_id,
Some(&error_msg),
)
.await
{
Ok(()) => {
all_synced_triggers.push(TriggerSyncInfo {
external_id: trigger.external_id.clone(),
script_path: trigger.script_path.clone(),
action: SyncAction::ErrorSet(error_msg),
});
}
Err(e) => {
tracing::error!(
"Failed to update error for trigger (external_id: '{}'): {}",
trigger.external_id,
e
);
all_sync_errors.push(SyncError {
resource_path: format!("workspace:{}", workspace_id),
error_message: format!(
"Failed to update error for trigger (external_id: '{}'): {}",
trigger.external_id, e
),
error_type: "database_update_error".to_string(),
});
}
}
}
} else {
// Trigger exists on external service
let external_trigger_data = external_trigger_map.get(&trigger.external_id).unwrap();
// Clear error if it was set
if trigger.error.is_some() {
tracing::info!(
"Trigger (external_id: '{}', script_path: '{}') exists on external service, clearing error",
trigger.external_id,
trigger.script_path
);
match update_native_trigger_error(
db,
workspace_id,
T::SERVICE_NAME,
&trigger.external_id,
None,
)
.await
{
Ok(()) => {
all_synced_triggers.push(TriggerSyncInfo {
external_id: trigger.external_id.clone(),
script_path: trigger.script_path.clone(),
action: SyncAction::ErrorCleared,
});
}
Err(e) => {
tracing::error!(
"Failed to clear error for trigger (external_id: '{}'): {}",
trigger.external_id,
e
);
all_sync_errors.push(SyncError {
resource_path: format!("workspace:{}", workspace_id),
error_message: format!(
"Failed to clear error for trigger (external_id: '{}'): {}",
trigger.external_id, e
),
error_type: "database_update_error".to_string(),
});
}
}
}
// Compare service_config and update if different
let external_service_config =
handler.extract_service_config_from_trigger_data(external_trigger_data)?;
let stored_service_config = trigger
.service_config
.clone()
.unwrap_or(serde_json::Value::Null);
if external_service_config != stored_service_config {
tracing::info!(
"Trigger (external_id: '{}', script_path: '{}') config differs from external service, updating local config",
trigger.external_id,
trigger.script_path
);
match update_native_trigger_service_config(
db,
workspace_id,
T::SERVICE_NAME,
&trigger.external_id,
&external_service_config,
)
.await
{
Ok(()) => {
all_synced_triggers.push(TriggerSyncInfo {
external_id: trigger.external_id.clone(),
script_path: trigger.script_path.clone(),
action: SyncAction::ConfigUpdated,
});
}
Err(e) => {
tracing::error!(
"Failed to update config for trigger (external_id: '{}'): {}",
trigger.external_id,
e
);
all_sync_errors.push(SyncError {
resource_path: format!("workspace:{}", workspace_id),
error_message: format!(
"Failed to update config for trigger (external_id: '{}'): {}",
trigger.external_id, e
),
error_type: "database_update_error".to_string(),
});
}
}
} else {
tracing::info!(
"Trigger (external_id: '{}', script_path: '{}') config is the same as external service, no update needed",
trigger.external_id,
trigger.script_path
);
}
}
}
tracing::info!(
"Sync completed for {} in workspace '{}'. Updated: {}, Errors: {}",
T::SERVICE_NAME.as_str(),
workspace_id,
all_synced_triggers.len(),
all_sync_errors.len()
);
Ok((all_synced_triggers, all_sync_errors))
}
backend/windmill-api/src/native_triggers/workspace_integrations.rs
-522
View File
@@ -1,522 +0,0 @@
use axum::{
extract::Path,
routing::{delete, get, post},
Extension, Json, Router,
};
#[cfg(feature = "native_trigger")]
use serde_json::to_value;
use sqlx::prelude::FromRow;
use strum::IntoEnumIterator;
use serde::{de::DeserializeOwned, Deserialize, Serialize};
#[cfg(feature = "native_trigger")]
use windmill_audit::{audit_oss::audit_log, ActionKind};
use windmill_common::{
db::UserDB,
error::{Error, JsonResult, Result},
utils::require_admin,
variables::{build_crypt, encrypt},
DB,
};
#[cfg(feature = "native_trigger")]
use crate::{
db::ApiAuthed,
native_triggers::{delete_workspace_integration, store_workspace_integration, ServiceName},
};
#[cfg(feature = "native_trigger")]
use windmill_oauth::{OClient, Url, OAUTH_HTTP_CLIENT};
use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine};
use hmac::{Hmac, Mac};
use sha2::Sha256;
type HmacSha256 = Hmac<Sha256>;
const STATE_EXPIRATION_SECONDS: i64 = 600; // 10 minutes
/// Generate a signed OAuth state that is cluster-safe.
/// The state contains: workspace_id, service_name, timestamp, and nonce.
/// It's signed with HMAC-SHA256 using the workspace key.
#[cfg(feature = "native_trigger")]
async fn generate_signed_state(
db: &DB,
workspace_id: &str,
service_name: ServiceName,
) -> Result<String> {
use windmill_common::variables::get_workspace_key;
let nonce = uuid::Uuid::new_v4().to_string();
let timestamp = chrono::Utc::now().timestamp();
let payload = format!(
"{}:{}:{}:{}",
workspace_id,
service_name.as_str(),
timestamp,
nonce
);
// Get workspace key for signing
let key = get_workspace_key(workspace_id, db).await?;
let mut mac =
HmacSha256::new_from_slice(key.as_bytes()).map_err(|e| Error::InternalErr(e.to_string()))?;
mac.update(payload.as_bytes());
let signature = mac.finalize().into_bytes();
// Encode as: base64(payload):base64(signature)
let encoded_payload = URL_SAFE_NO_PAD.encode(payload.as_bytes());
let encoded_signature = URL_SAFE_NO_PAD.encode(signature);
Ok(format!("{}:{}", encoded_payload, encoded_signature))
}
/// Validate a signed OAuth state.
/// Returns true if the state is valid (correct signature and not expired).
#[cfg(feature = "native_trigger")]
async fn validate_signed_state(db: &DB, state: &str, workspace_id: &str) -> Result<bool> {
use windmill_common::variables::get_workspace_key;
let parts: Vec<&str> = state.split(':').collect();
if parts.len() != 2 {
return Ok(false);
}
let encoded_payload = parts[0];
let encoded_signature = parts[1];
// Decode payload
let payload_bytes = match URL_SAFE_NO_PAD.decode(encoded_payload) {
Ok(bytes) => bytes,
Err(_) => return Ok(false),
};
let payload = match String::from_utf8(payload_bytes) {
Ok(s) => s,
Err(_) => return Ok(false),
};
// Parse payload: workspace_id:service_name:timestamp:nonce
let payload_parts: Vec<&str> = payload.split(':').collect();
if payload_parts.len() != 4 {
return Ok(false);
}
let state_workspace_id = payload_parts[0];
let timestamp: i64 = match payload_parts[2].parse() {
Ok(ts) => ts,
Err(_) => return Ok(false),
};
// Verify workspace_id matches
if state_workspace_id != workspace_id {
return Ok(false);
}
// Check expiration
let now = chrono::Utc::now().timestamp();
if now - timestamp > STATE_EXPIRATION_SECONDS {
return Ok(false);
}
// Verify signature
let key = get_workspace_key(workspace_id, db).await?;
let mut mac =
HmacSha256::new_from_slice(key.as_bytes()).map_err(|e| Error::InternalErr(e.to_string()))?;
mac.update(payload.as_bytes());
let received_signature = match URL_SAFE_NO_PAD.decode(encoded_signature) {
Ok(bytes) => bytes,
Err(_) => return Ok(false),
};
Ok(mac.verify_slice(&received_signature).is_ok())
}
#[derive(Debug, Serialize)]
pub struct IntegrationStatusResponse {
pub connected: bool,
pub service_name: ServiceName,
pub created_at: Option<chrono::DateTime<chrono::Utc>>,
pub created_by: Option<String>,
}
#[derive(Debug, Serialize)]
pub struct ListIntegrationsResponse {
pub integrations: Vec<IntegrationStatusResponse>,
}
#[derive(Debug, Serialize)]
pub struct ConnectIntegrationResponse {
pub auth_url: String,
}
#[derive(FromRow, Debug, Clone, Serialize, Deserialize)]
pub struct WorkspaceOAuthConfig {
pub client_id: String,
pub client_secret: String,
pub base_url: String,
pub access_token: Option<String>
}
#[derive(Debug, Serialize)]
pub struct OAuthConfigResponse {
pub configured: bool,
pub base_url: Option<String>,
pub redirect_uri: Option<String>,
}
#[cfg(feature = "native_trigger")]
async fn generate_connect_url(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Path((workspace_id, service_name)): Path<(String, ServiceName)>,
Json(RedirectUri { redirect_uri }): Json<RedirectUri>,
) -> JsonResult<String> {
require_admin(authed.is_admin, &workspace_id)?;
let oauth_config =
get_workspace_oauth_config_as_oauth_config(&db, &workspace_id, service_name).await?;
// Generate a signed state that is cluster-safe
let state = generate_signed_state(&db, &workspace_id, service_name).await?;
let auth_url = build_authorization_url(&oauth_config, &state, &redirect_uri);
Ok(Json(auth_url))
}
#[cfg(feature = "native_trigger")]
async fn delete_integration(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path((workspace_id, service_name)): Path<(String, ServiceName)>,
) -> JsonResult<String> {
require_admin(authed.is_admin, &workspace_id)?;
let mut tx = user_db.begin(&authed).await?;
let deleted = delete_workspace_integration(&mut *tx, &workspace_id, service_name).await?;
if !deleted {
return Err(Error::NotFound(format!(
"{} integration not found for workspace",
service_name
)));
}
audit_log(
&mut *tx,
&authed,
&format!("workspace_integrations.{}.disconnect", service_name),
ActionKind::Delete,
&workspace_id,
Some(&format!("Disconnected {} integration", service_name)),
None,
)
.await?;
tx.commit().await?;
Ok(Json(format!(
"{} integration disconnected successfully",
service_name
)))
}
#[derive(FromRow, Debug, Deserialize, Serialize)]
struct WorkspaceIntegrations {
service_name: ServiceName,
oauth_data: Option<sqlx::types::Json<WorkspaceOAuthConfig>>,
}
#[cfg(feature = "native_trigger")]
async fn list_integrations(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(_user_db): Extension<UserDB>,
Path(workspace_id): Path<String>,
) -> JsonResult<Vec<WorkspaceIntegrations>> {
require_admin(authed.is_admin, &workspace_id)?;
let mut tx = db.begin().await?;
let integrations = sqlx::query_as!(
WorkspaceIntegrations,
r#"
SELECT
oauth_data as "oauth_data!: sqlx::types::Json<WorkspaceOAuthConfig>",
service_name as "service_name!: ServiceName"
FROM
workspace_integrations
WHERE
workspace_id = $1
"#,
workspace_id
)
.fetch_all(&mut *tx)
.await?;
let key_value = integrations
.into_iter()
.map(|integration| (integration.service_name, integration.oauth_data))
.collect::<std::collections::HashMap<_, _>>();
let integrations = ServiceName::iter()
.map(|service_name| WorkspaceIntegrations {
service_name: service_name,
oauth_data: key_value.get(&service_name).cloned().flatten(),
})
.collect::<Vec<_>>();
tx.commit().await?;
Ok(Json(integrations))
}
async fn integration_exist(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path((workspace_id, service_name)): Path<(String, ServiceName)>,
) -> JsonResult<bool> {
let mut tx = user_db.begin(&authed).await?;
let exists = sqlx::query_scalar!(
r#"
SELECT EXISTS (
SELECT 1
FROM workspace_integrations
WHERE workspace_id = $1
AND service_name = $2
AND oauth_data IS NOT NULL
)
"#,
workspace_id,
service_name as ServiceName
)
.fetch_one(&mut *tx)
.await?
.unwrap_or(false);
Ok(Json(exists))
}
#[derive(Debug, Deserialize)]
struct RedirectUri {
redirect_uri: String,
}
#[cfg(feature = "native_trigger")]
async fn oauth_callback(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path((workspace_id, service_name, code, state)): Path<(String, ServiceName, String, String)>,
Json(RedirectUri { redirect_uri }): Json<RedirectUri>,
) -> JsonResult<String> {
require_admin(authed.is_admin, &workspace_id)?;
// Validate the signed state (cluster-safe, no DB storage needed)
let state_was_valid = validate_signed_state(&db, &state, &workspace_id).await?;
if !state_was_valid {
return Err(Error::BadRequest(
"Invalid or expired state parameter".to_string(),
));
}
let oauth_config =
get_workspace_oauth_config::<WorkspaceOAuthConfig>(&db, &workspace_id, service_name)
.await?;
let token_response =
exchange_code_for_token(&oauth_config, service_name, &code, &redirect_uri).await?;
let mut tx = user_db.begin(&authed).await?;
let mc = build_crypt(&db, &workspace_id).await?;
let mut oauth_data = serde_json::to_value(oauth_config).unwrap();
let encrypted_access_token = encrypt(&mc, &token_response.access_token);
oauth_data["access_token"] = serde_json::Value::String(encrypted_access_token);
if let Some(refresh_token) = token_response.refresh_token {
let encrypted_refresh_token = encrypt(&mc, &refresh_token);
oauth_data["refresh_token"] = serde_json::Value::String(encrypted_refresh_token);
}
if let Some(expires_in) = token_response.expires_in {
let expires_at = chrono::Utc::now() + chrono::Duration::seconds(expires_in as i64);
oauth_data["token_expires_at"] = serde_json::Value::String(expires_at.to_rfc3339());
}
store_workspace_integration(&mut *tx, &authed, &workspace_id, service_name, oauth_data).await?;
audit_log(
&mut *tx,
&authed,
&format!("workspace_integrations.{}.connect", service_name),
ActionKind::Create,
&workspace_id,
Some(&format!("Connected {} integration via OAuth", service_name)),
None,
)
.await?;
tx.commit().await?;
Ok(Json(format!(
"{} integration connected successfully via OAuth",
service_name
)))
}
/// Token response from OAuth token exchange
#[derive(Debug, Deserialize)]
struct TokenResponse {
access_token: String,
refresh_token: Option<String>,
#[serde(default)]
expires_in: Option<u64>,
}
/// Build an OAuth client for native trigger services using windmill-oauth.
#[cfg(feature = "native_trigger")]
fn build_native_oauth_client(
config: &WorkspaceOAuthConfig,
service_name: ServiceName,
redirect_uri: &str,
) -> Result<OClient> {
let auth_url = Url::parse(&format!("{}{}", config.base_url, service_name.auth_endpoint()))
.map_err(|e| Error::InternalErr(format!("Invalid auth URL: {}", e)))?;
let token_url = Url::parse(&format!("{}{}", config.base_url, service_name.token_endpoint()))
.map_err(|e| Error::InternalErr(format!("Invalid token URL: {}", e)))?;
let redirect = Url::parse(redirect_uri).map_err(|e| {
Error::BadRequest(format!(
"Invalid redirect URI '{}': {}. The redirect URI must be an absolute URL (e.g., https://example.com/callback)",
redirect_uri, e
))
})?;
let mut client = OClient::new(config.client_id.clone(), auth_url, token_url);
client.set_client_secret(config.client_secret.clone());
client.set_redirect_url(redirect);
Ok(client)
}
/// Exchange authorization code for tokens using windmill-oauth.
#[cfg(feature = "native_trigger")]
async fn exchange_code_for_token(
config: &WorkspaceOAuthConfig,
service_name: ServiceName,
code: &str,
redirect_uri: &str,
) -> Result<TokenResponse> {
let client = build_native_oauth_client(config, service_name, redirect_uri)?;
let token_response: TokenResponse = client
.exchange_code(code.to_string())
.with_client(&*OAUTH_HTTP_CLIENT)
.execute()
.await
.map_err(|e| Error::InternalErr(format!("Failed to exchange code for token: {:?}", e)))?;
Ok(token_response)
}
async fn get_workspace_oauth_config<T: DeserializeOwned>(
db: &DB,
workspace_id: &str,
service_name: ServiceName,
) -> Result<T> {
let oauth_configs = sqlx::query_scalar!(
r#"
SELECT
oauth_data
FROM
workspace_integrations
WHERE
workspace_id = $1 AND
service_name = $2
"#,
workspace_id,
service_name as ServiceName
)
.fetch_optional(db)
.await?
.ok_or(Error::NotFound(format!(
"Integration for service {} not found",
service_name.as_str()
)))?;
let config = serde_json::from_value::<T>(oauth_configs)
.map_err(|e| Error::InternalErr(format!("Failed to parse OAuth config: {}", e)))?;
Ok(config)
}
#[cfg(feature = "native_trigger")]
pub async fn create_workspace_integration(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path((workspace_id, service_name)): Path<(String, ServiceName)>,
Json(oauth_data): Json<WorkspaceOAuthConfig>,
) -> Result<()> {
require_admin(authed.is_admin, &workspace_id)?;
let mut tx = user_db.begin(&authed).await?;
store_workspace_integration(
&mut tx,
&authed,
&workspace_id,
service_name,
to_value(oauth_data).unwrap(),
)
.await?;
tx.commit().await?;
Ok(())
}
#[inline]
async fn get_workspace_oauth_config_as_oauth_config(
db: &DB,
workspace_id: &str,
service_name: ServiceName,
) -> Result<WorkspaceOAuthConfig> {
get_workspace_oauth_config::<WorkspaceOAuthConfig>(db, workspace_id, service_name).await
}
fn build_authorization_url(
config: &WorkspaceOAuthConfig,
state: &str,
redirect_uri: &str,
) -> String {
let params = [
("response_type", "code"),
("client_id", &config.client_id),
("redirect_uri", redirect_uri),
("state", state),
("scope", "read write"),
];
let query_string = params
.iter()
.map(|(k, v)| format!("{}={}", urlencoding::encode(k), urlencoding::encode(v)))
.collect::<Vec<_>>()
.join("&");
format!("{}/apps/oauth2/authorize?{}", config.base_url, query_string)
}
pub fn workspaced_service() -> Router {
let router = Router::new()
.route("/list", get(list_integrations))
.route("/:service_name/exists", get(integration_exist))
.route("/:service_name/create", post(create_workspace_integration))
.route(
"/:service_name/generate_connect_url",
post(generate_connect_url),
)
.route("/:service_name/delete", delete(delete_integration))
.route("/:service_name/callback/:code/:state", post(oauth_callback));
Router::new().nest("/integrations", router)
}
backend/windmill-api/src/triggers/email/handler_oss.rs
-67
View File
@@ -1,67 +0,0 @@
#[cfg(not(feature = "private"))]
use crate::triggers::TriggerData;
#[allow(unused)]
#[cfg(feature = "private")]
pub use super::handler_ee::*;
#[cfg(not(feature = "private"))]
use {
super::EmailTrigger,
crate::{
db::{ApiAuthed, DB},
triggers::TriggerCrud,
},
axum::async_trait,
sqlx::PgConnection,
windmill_common::error::{Error, Result},
windmill_git_sync::DeployedObject,
};
#[cfg(not(feature = "private"))]
#[async_trait]
impl TriggerCrud for EmailTrigger {
type Trigger = ();
type TriggerConfig = ();
type TriggerConfigRequest = ();
type TestConnectionConfig = ();
const TABLE_NAME: &'static str = "";
const TRIGGER_TYPE: &'static str = "";
const SUPPORTS_SERVER_STATE: bool = false;
const SUPPORTS_TEST_CONNECTION: bool = false;
const ROUTE_PREFIX: &'static str = "/email_triggers";
const DEPLOYMENT_NAME: &'static str = "";
const IS_ALLOWED_ON_CLOUD: bool = false;
fn get_deployed_object(path: String) -> DeployedObject {
DeployedObject::EmailTrigger { path }
}
async fn create_trigger(
&self,
_db: &DB,
_tx: &mut PgConnection,
_authed: &ApiAuthed,
_w_id: &str,
_trigger: TriggerData<Self::TriggerConfigRequest>,
) -> Result<()> {
Err(Error::BadRequest(
"Email triggers are not available in open source version".to_string(),
))
}
async fn update_trigger(
&self,
_db: &DB,
_executor: &mut PgConnection,
_authed: &ApiAuthed,
_workspace_id: &str,
_path: &str,
_trigger: TriggerData<Self::TriggerConfigRequest>,
) -> Result<()> {
Err(Error::BadRequest(
"Email triggers are not available in open source version".to_string(),
))
}
}
backend/windmill-api/src/triggers/email/mod.rs
-11
View File
@@ -1,11 +0,0 @@
#[cfg(feature = "private")]
mod handler_ee;
pub mod handler_oss;
#[cfg(feature = "private")]
mod mod_ee;
#[cfg(feature = "private")]
pub use mod_ee::*;
#[derive(Copy, Clone)]
pub struct EmailTrigger;
backend/windmill-api/src/triggers/filter.rs
-144
View File
@@ -1,144 +0,0 @@
use serde::{
de::{self, MapAccess, Visitor},
Deserialize, Deserializer,
};
use serde_json::Value;
use std::fmt;
#[derive(Deserialize)]
pub struct JsonFilter {
pub key: String,
pub value: Value,
}
#[derive(Deserialize)]
#[serde(untagged)]
pub enum Filter {
JsonFilter(JsonFilter),
}
struct SupersetVisitor<'a> {
key: &'a str,
value_to_check: &'a Value,
}
impl<'de, 'a> Visitor<'de> for SupersetVisitor<'a> {
type Value = bool;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
formatter.write_str("a JSON object with a specific key at the top level")
}
fn visit_map<V>(self, mut map: V) -> std::result::Result<Self::Value, V::Error>
where
V: MapAccess<'de>,
{
let mut result = false;
let mut found = false;
// Must consume entire map to satisfy deserializer contract
while let Some(key) = map.next_key::<String>()? {
if !found && key == self.key {
let json_value: Value = map.next_value()?;
result = is_superset(&json_value, self.value_to_check);
found = true;
} else {
// Skip values we don't need (cheaper than full deserialization)
let _ = map.next_value::<de::IgnoredAny>()?;
}
}
Ok(result)
}
}
pub fn is_superset(json_value: &Value, value_to_check: &Value) -> bool {
match (json_value, value_to_check) {
(Value::Object(json_map), Value::Object(check_map)) => {
check_map.iter().all(|(k, v)| {
json_map
.get(k)
.map_or(false, |json_val| is_superset(json_val, v))
})
}
(Value::Array(json_array), Value::Array(check_array)) => {
check_array.iter().all(|check_item| {
json_array
.iter()
.any(|json_item| is_superset(json_item, check_item))
})
}
_ => json_value == value_to_check,
}
}
pub fn is_value_superset<'a, 'de, D>(
deserializer: D,
key: &'a str,
value_to_check: &'a Value,
) -> std::result::Result<bool, D::Error>
where
D: Deserializer<'de>,
{
deserializer.deserialize_map(SupersetVisitor { key, value_to_check })
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn test_filter_with_other_top_level_keys() {
let payload = r#"{"event_type": "test", "other": "data"}"#;
let key = "event_type";
let value = json!("test");
let mut deserializer = serde_json::Deserializer::from_str(payload);
let result = is_value_superset(&mut deserializer, key, &value).unwrap();
assert!(result, "Should match when key exists with correct value");
}
#[test]
fn test_filter_with_key_not_first() {
let payload = r#"{"other": "data", "event_type": "test"}"#;
let key = "event_type";
let value = json!("test");
let mut deserializer = serde_json::Deserializer::from_str(payload);
let result = is_value_superset(&mut deserializer, key, &value).unwrap();
assert!(result, "Should match even when key is not first");
}
#[test]
fn test_filter_with_nested_object() {
let payload = r#"{"data": {"status": "active", "count": 5}, "other": "value"}"#;
let key = "data";
let value = json!({"status": "active"});
let mut deserializer = serde_json::Deserializer::from_str(payload);
let result = is_value_superset(&mut deserializer, key, &value).unwrap();
assert!(result, "Should match when nested object is superset");
}
#[test]
fn test_filter_no_match() {
let payload = r#"{"event_type": "other", "data": "value"}"#;
let key = "event_type";
let value = json!("test");
let mut deserializer = serde_json::Deserializer::from_str(payload);
let result = is_value_superset(&mut deserializer, key, &value).unwrap();
assert!(!result, "Should not match when value differs");
}
#[test]
fn test_filter_key_not_found() {
let payload = r#"{"other": "data"}"#;
let key = "event_type";
let value = json!("test");
let mut deserializer = serde_json::Deserializer::from_str(payload);
let result = is_value_superset(&mut deserializer, key, &value).unwrap();
assert!(!result, "Should not match when key doesn't exist");
}
}
backend/windmill-api/src/triggers/gcp/handler_oss.rs
-64
View File
@@ -1,64 +0,0 @@
#[allow(unused)]
#[cfg(feature = "private")]
pub use super::handler_ee::*;
#[cfg(not(feature = "private"))]
use {
super::GcpTrigger,
crate::{
db::{ApiAuthed, DB},
triggers::{TriggerCrud, TriggerData},
},
axum::async_trait,
sqlx::PgConnection,
windmill_common::error::{Error, Result},
windmill_git_sync::DeployedObject,
};
#[cfg(not(feature = "private"))]
#[async_trait]
impl TriggerCrud for GcpTrigger {
type Trigger = ();
type TriggerConfig = ();
type TriggerConfigRequest = ();
type TestConnectionConfig = ();
const TABLE_NAME: &'static str = "";
const TRIGGER_TYPE: &'static str = "";
const SUPPORTS_SERVER_STATE: bool = false;
const SUPPORTS_TEST_CONNECTION: bool = false;
const ROUTE_PREFIX: &'static str = "/gcp_triggers";
const DEPLOYMENT_NAME: &'static str = "";
const IS_ALLOWED_ON_CLOUD: bool = false;
fn get_deployed_object(path: String) -> DeployedObject {
DeployedObject::GcpTrigger { path }
}
async fn create_trigger(
&self,
_db: &DB,
_executor: &mut PgConnection,
_authed: &ApiAuthed,
_w_id: &str,
_trigger: TriggerData<Self::TriggerConfigRequest>,
) -> Result<()> {
Err(Error::BadRequest(
"GCP triggers are not available in open source version".to_string(),
))
}
async fn update_trigger(
&self,
_db: &DB,
_executor: &mut PgConnection,
_authed: &ApiAuthed,
_workspace_id: &str,
_path: &str,
_trigger: TriggerData<Self::TriggerConfigRequest>,
) -> Result<()> {
Err(Error::BadRequest(
"GCP triggers are not available in open source version".to_string(),
))
}
}
backend/windmill-api/src/triggers/gcp/listener_oss.rs
-41
View File
@@ -1,41 +0,0 @@
#[allow(unused)]
#[cfg(feature = "private")]
pub use super::listener_ee::*;
#[cfg(not(feature = "private"))]
use {
super::GcpTrigger,
crate::triggers::{listener::ListeningTrigger, Listener},
std::sync::Arc,
tokio::sync::RwLock,
windmill_common::{error::Result, jobs::JobTriggerKind, DB},
};
#[cfg(not(feature = "private"))]
#[async_trait::async_trait]
impl Listener for GcpTrigger {
type Consumer = ();
type Extra = ();
const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Gcp;
async fn get_consumer(
&self,
_db: &DB,
_listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
_err_message: Arc<RwLock<Option<String>>>,
_killpill_rx: tokio::sync::broadcast::Receiver<()>,
) -> Result<Option<Self::Consumer>> {
Ok(None)
}
async fn consume(
&self,
_db: &DB,
_consumer: Self::Consumer,
_listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
_err_message: Arc<RwLock<Option<String>>>,
_killpill_rx: tokio::sync::broadcast::Receiver<()>,
) {
()
}
}
backend/windmill-api/src/triggers/gcp/mod.rs
-16
View File
@@ -1,16 +0,0 @@
#[cfg(feature = "private")]
mod handler_ee;
pub mod handler_oss;
#[cfg(feature = "private")]
mod listener_ee;
pub mod listener_oss;
#[cfg(feature = "private")]
mod mod_ee;
#[cfg(feature = "private")]
pub use mod_ee::*;
#[derive(Clone, Copy)]
pub struct GcpTrigger;
backend/windmill-api/src/triggers/global_handler.rs
-324
View File
@@ -1,324 +0,0 @@
use crate::{
db::{ApiAuthed, DB},
jobs::cancel_jobs,
triggers::trigger_helpers::trigger_runnable_inner,
};
use axum::{
extract::{Extension, Path},
response::Json,
};
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use serde_json::value::RawValue;
use sqlx::PgConnection;
use std::collections::HashMap;
use uuid::Uuid;
use windmill_common::{
db::UserDB,
error::{self, Error, Result},
jobs::JobTriggerKind,
triggers::TriggerMetadata,
};
#[derive(sqlx::FromRow)]
pub struct SuspendedTrigger {
pub script_path: String,
pub is_flow: bool,
pub edited_by: String,
pub email: String,
pub edited_at: DateTime<Utc>,
pub error_handler_path: Option<String>,
pub error_handler_args: Option<sqlx::types::Json<HashMap<String, serde_json::Value>>>,
pub retry: Option<sqlx::types::Json<windmill_common::flows::Retry>>,
}
async fn get_suspended_trigger(
tx: &mut PgConnection,
workspace_id: &str,
trigger_kind: &JobTriggerKind,
path: &str,
) -> Result<SuspendedTrigger> {
match trigger_kind {
JobTriggerKind::Webhook | JobTriggerKind::Schedule => {
return Err(Error::BadRequest(format!(
"{} triggers do not support job reassignment",
trigger_kind
)));
}
_ => {}
}
let table_name = format!("{}_trigger", trigger_kind.to_string());
let fields = vec![
"script_path",
"is_flow",
"edited_by",
"email",
"edited_at",
"error_handler_path",
"error_handler_args",
"retry",
];
let sql = format!(
r#"SELECT
{}
FROM
{}
WHERE
workspace_id = $1 AND
path = $2
"#,
fields.join(", "),
table_name
);
sqlx::query_as(&sql)
.bind(workspace_id)
.bind(path)
.fetch_optional(&mut *tx)
.await?
.ok_or_else(|| Error::NotFound(format!("Trigger not found at path: {}", path)))
}
struct JobWithArgs {
id: Uuid,
args: Option<sqlx::types::Json<HashMap<String, Box<RawValue>>>>,
created_at: chrono::DateTime<chrono::Utc>,
}
#[derive(Deserialize, Serialize, Default)]
pub struct ReassignJobsBody {
#[serde(default)]
pub job_ids: Option<Vec<Uuid>>,
}
pub async fn resume_suspended_trigger_jobs(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path((w_id, trigger_kind, trigger_path)): Path<(String, JobTriggerKind, String)>,
Json(body): Json<ReassignJobsBody>,
) -> error::Result<Json<String>> {
let mut tx = user_db.clone().begin(&authed).await?;
let trigger = get_suspended_trigger(&mut *tx, &w_id, &trigger_kind, &trigger_path).await?;
let jobs = if let Some(job_ids) = body.job_ids.as_ref() {
if job_ids.is_empty() {
vec![]
} else {
sqlx::query_as!(
JobWithArgs,
r#"
SELECT
id,
args as "args: _",
created_at
FROM v2_job
WHERE workspace_id = $1
AND (
kind = 'unassigned_script'::JOB_KIND OR
kind = 'unassigned_flow'::JOB_KIND OR
kind = 'unassigned_singlestepflow'::JOB_KIND
)
AND trigger_kind = $2
AND trigger = $3
AND id = ANY($4)
"#,
w_id,
trigger_kind as _,
trigger_path,
job_ids as _,
)
.fetch_all(&mut *tx)
.await?
}
} else {
sqlx::query_as!(
JobWithArgs,
r#"
SELECT
id,
args as "args: _",
created_at
FROM v2_job
WHERE workspace_id = $1
AND (
kind = 'unassigned_script'::JOB_KIND OR
kind = 'unassigned_flow'::JOB_KIND OR
kind = 'unassigned_singlestepflow'::JOB_KIND
)
AND trigger_kind = $2
AND trigger = $3
"#,
w_id,
trigger_kind as _,
trigger_path,
)
.fetch_all(&mut *tx)
.await?
};
let trigger_metadata = TriggerMetadata::new(Some(trigger_path.clone()), trigger_kind);
let l = jobs.len();
for job in jobs {
// If job was created before trigger was edited, simply update it to unsuspend
// instead of deleting and repushing
if job.created_at > trigger.edited_at {
let job_kind = if trigger.is_flow {
windmill_common::jobs::JobKind::Flow
} else {
windmill_common::jobs::JobKind::Script
};
sqlx::query!(
"UPDATE v2_job SET kind = $1 WHERE id = $2",
job_kind as _,
job.id
)
.execute(&mut *tx)
.await?;
// Update the job to unsuspend it and set the correct kind
sqlx::query!(
"UPDATE v2_job_queue SET scheduled_for = now() WHERE id = $1",
job.id
)
.execute(&mut *tx)
.await?;
} else {
// Job was created after trigger edit - delete and repush with new configuration
// Pass the transaction to trigger_runnable_inner so everything is in the same transaction
let (_uuid, _delete_after_use, _early_return, tx_o) = trigger_runnable_inner(
&db,
Some(tx),
Some(user_db.clone()),
authed.clone(),
&w_id,
&trigger.script_path,
trigger.is_flow,
windmill_queue::PushArgsOwned {
extra: None,
args: job.args.map(|a| a.0).unwrap_or_default(),
},
trigger.retry.as_ref(),
trigger.error_handler_path.as_deref(),
trigger.error_handler_args.as_ref(),
trigger_path.clone(),
None,
trigger_metadata.clone(),
None,
)
.await?;
tx = match tx_o {
Some(tx) => tx,
None => {
return Err(error::Error::internal_err(
"Transaction should be returned when passed in".to_string(),
));
}
};
// Delete the unassigned job from all related tables
sqlx::query!("DELETE FROM v2_job_queue WHERE id = $1", job.id)
.execute(&mut *tx)
.await?;
sqlx::query!("DELETE FROM v2_job_runtime WHERE id = $1", job.id)
.execute(&mut *tx)
.await?;
sqlx::query!("DELETE FROM job_perms WHERE job_id = $1", job.id)
.execute(&mut *tx)
.await?;
sqlx::query!("DELETE FROM concurrency_key WHERE job_id = $1", job.id)
.execute(&mut *tx)
.await?;
sqlx::query!("DELETE FROM debounce_key WHERE job_id = $1", job.id)
.execute(&mut *tx)
.await?;
sqlx::query!("DELETE FROM debounce_stale_data WHERE job_id = $1", job.id)
.execute(&mut *tx)
.await?;
sqlx::query!("DELETE FROM v2_job WHERE id = $1", job.id)
.execute(&mut *tx)
.await?;
}
}
tx.commit().await?;
Ok(Json(format!("Reassigned {} jobs", l)))
}
pub async fn cancel_suspended_trigger_jobs(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path((w_id, trigger_kind, trigger_path)): Path<(String, JobTriggerKind, String)>,
Json(body): Json<ReassignJobsBody>,
) -> error::Result<Json<String>> {
let mut tx = user_db.clone().begin(&authed).await?;
// Get the list of job IDs to cancel
let jobs_to_cancel = if let Some(job_ids) = body.job_ids.as_ref() {
if job_ids.is_empty() {
vec![]
} else {
sqlx::query_scalar!(
"SELECT id FROM v2_job
WHERE workspace_id = $1
AND (kind = 'unassigned_script'::JOB_KIND OR kind = 'unassigned_flow'::JOB_KIND OR kind = 'unassigned_singlestepflow'::JOB_KIND)
AND trigger_kind = $2
AND trigger = $3
AND id = ANY($4)",
w_id,
trigger_kind as _,
trigger_path,
job_ids as _,
)
.fetch_all(&mut *tx)
.await?
}
} else {
sqlx::query_scalar!(
"SELECT id FROM v2_job
WHERE workspace_id = $1
AND (kind = 'unassigned_script'::JOB_KIND OR kind = 'unassigned_flow'::JOB_KIND OR kind = 'unassigned_singlestepflow'::JOB_KIND)
AND trigger_kind = $2
AND trigger = $3",
w_id,
trigger_kind as _,
trigger_path,
)
.fetch_all(&mut *tx)
.await?
};
tx.commit().await?;
let count = jobs_to_cancel.len();
if count > 0 {
let cancelled_jobs = cancel_jobs(
jobs_to_cancel,
&db,
authed.username.as_str(),
w_id.as_str(),
true,
)
.await?;
Ok(Json(format!("Canceled {} jobs", cancelled_jobs.0.len())))
} else {
Ok(Json(format!("No jobs to cancel")))
}
}
backend/windmill-api/src/triggers/handler.rs
-1003
View File
File diff suppressed because it is too large Load Diff
backend/windmill-api/src/triggers/http/handler.rs
-1185
View File
File diff suppressed because it is too large Load Diff
backend/windmill-api/src/triggers/http/http_trigger_args.rs
-225
View File
@@ -1,225 +0,0 @@
use std::collections::HashMap;
use axum::{
extract::{FromRequest, Request},
response::Response,
};
use serde::{Deserialize, Serialize};
use serde_json::value::RawValue;
use windmill_common::{
error::Error,
triggers::{RunnableFormat, RunnableFormatVersion},
worker::to_raw_value,
DB,
};
use windmill_queue::PushArgsOwned;
use crate::{
args::{
build_headers, build_query, try_from_request_body, Body, RawWebhookArgs, WebhookArgs,
WebhookArgsMetadata,
},
db::ApiAuthed,
};
pub struct RawHttpTriggerArgs(pub RawWebhookArgs);
#[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Copy, Hash, Eq, PartialEq)]
#[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")]
#[serde(rename_all = "lowercase")]
pub enum HttpMethod {
Get,
Post,
Put,
Delete,
Patch,
}
impl TryFrom<&http::Method> for HttpMethod {
type Error = Error;
fn try_from(method: &http::Method) -> Result<Self, Self::Error> {
match method {
&http::Method::GET => Ok(HttpMethod::Get),
&http::Method::POST => Ok(HttpMethod::Post),
&http::Method::PUT => Ok(HttpMethod::Put),
&http::Method::DELETE => Ok(HttpMethod::Delete),
&http::Method::PATCH => Ok(HttpMethod::Patch),
_ => Err(Error::BadRequest("Invalid HTTP method".to_string())),
}
}
}
#[axum::async_trait]
impl<S> FromRequest<S, axum::body::Body> for RawHttpTriggerArgs
where
S: Send + Sync,
{
type Rejection = Response;
async fn from_request(request: Request, _state: &S) -> Result<Self, Self::Rejection> {
let args = try_from_request_body(request, _state, true).await?;
Ok(Self(args))
}
}
#[derive(Debug, Clone)]
pub struct HttpTriggerArgs(pub WebhookArgs);
impl RawHttpTriggerArgs {
pub async fn process_args(
self,
authed: &ApiAuthed,
db: &DB,
w_id: &str,
use_raw: bool,
) -> Result<HttpTriggerArgs, Error> {
if self.0.metadata.query_use_raw || self.0.metadata.query_wrap_body {
return Err(Error::BadRequest(
"Specifying use raw or wrap body with query args is not supported anymore on http routes, please set it in the trigger config".to_string(),
)
.into());
}
let args = self.0.process_args(authed, db, w_id, Some(use_raw)).await?;
Ok(HttpTriggerArgs(args))
}
}
#[derive(Serialize)]
struct HttpTriggerPreprocessorEvent<'a> {
kind: String,
route: &'a str,
path: &'a str,
body: Box<RawValue>,
raw_string: Option<String>,
params: &'a HashMap<String, String>,
headers: HashMap<String, Box<RawValue>>,
query: HashMap<String, Box<RawValue>>,
method: HttpMethod,
}
#[derive(Serialize)]
struct HttpTriggerWmTrigger<'a> {
route: &'a str,
path: &'a str,
params: &'a HashMap<String, String>,
query: HashMap<String, Box<RawValue>>,
headers: HashMap<String, Box<RawValue>>,
method: HttpMethod,
}
impl HttpTriggerArgs {
pub fn to_main_args(self, wrap_body: bool) -> Result<PushArgsOwned, Error> {
let mut extra = HashMap::new();
let WebhookArgsMetadata { raw_string, .. } = self.0.metadata;
if let Some(raw_string) = raw_string {
extra.insert("raw_string".to_string(), to_raw_value(&raw_string));
}
let extra = if extra.is_empty() { None } else { Some(extra) };
match self.0.body {
Body::HashMap(mut body) => {
if wrap_body {
body = HashMap::from([("body".to_string(), to_raw_value(&body))]);
}
Ok(PushArgsOwned { args: body, extra })
}
Body::NoHashMap(args) => {
let mut hm = HashMap::new();
hm.insert("body".to_string(), args);
Ok(PushArgsOwned { args: hm, extra })
}
}
}
pub fn to_args_from_format(
self,
route_path: &str,
called_path: &str,
params: &HashMap<String, String>,
format: RunnableFormat,
wrap_body: bool,
) -> Result<PushArgsOwned, Error> {
let headers = build_headers(&self.0.metadata.headers, None, true);
let query = build_query(self.0.metadata.query.as_deref(), None, true);
match format {
RunnableFormat { has_preprocessor: true, version: RunnableFormatVersion::V2 } => {
// we don't care about wrap_body in v2
self.to_v2_preprocessor_args(route_path, called_path, params, headers, query)
}
RunnableFormat { has_preprocessor: true, version: RunnableFormatVersion::V1 } => self
.to_v1_preprocessor_args(
route_path,
called_path,
params,
wrap_body,
headers,
query,
),
RunnableFormat { has_preprocessor: false, .. } => self.to_main_args(wrap_body),
}
}
fn to_v1_preprocessor_args(
self,
route_path: &str,
called_path: &str,
params: &HashMap<String, String>,
wrap_body: bool,
headers: HashMap<String, Box<RawValue>>,
query: HashMap<String, Box<RawValue>>,
) -> Result<PushArgsOwned, Error> {
let mut extra = HashMap::new();
let mut wm_trigger = HashMap::new();
wm_trigger.insert("kind".to_string(), to_raw_value(&"http".to_string()));
wm_trigger.insert(
"http".to_string(),
to_raw_value(&HttpTriggerWmTrigger {
route: route_path,
path: called_path,
method: (&self.0.metadata.method).try_into()?,
params,
query,
headers,
}),
);
extra.insert("wm_trigger".to_string(), to_raw_value(&wm_trigger));
let mut args = self.to_main_args(wrap_body)?;
args.extra.get_or_insert_default().extend(extra);
Ok(args)
}
pub fn to_v2_preprocessor_args(
self,
route_path: &str,
called_path: &str,
params: &HashMap<String, String>,
headers: HashMap<String, Box<RawValue>>,
query: HashMap<String, Box<RawValue>>,
) -> Result<PushArgsOwned, Error> {
let mut args = HashMap::new();
args.insert(
"event".to_string(),
to_raw_value(&HttpTriggerPreprocessorEvent {
kind: "http".to_string(),
body: to_raw_value(&self.0.body),
raw_string: self.0.metadata.raw_string,
headers,
query,
method: (&self.0.metadata.method).try_into()?,
route: route_path,
path: called_path,
params,
}),
);
Ok(PushArgsOwned { args, extra: None })
}
}
backend/windmill-api/src/triggers/http/http_trigger_auth.rs
-751
View File
@@ -1,751 +0,0 @@
use axum::response::{IntoResponse, Response};
use base64::{
prelude::{BASE64_STANDARD, BASE64_URL_SAFE},
Engine,
};
use hmac::{Hmac, Mac};
use http::{header, HeaderMap, HeaderValue, StatusCode};
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use serde_json::json;
use sha1::Sha1;
use sha2::{Sha256, Sha512};
use std::{borrow::Cow, collections::HashMap};
pub type HmacSha256 = Hmac<Sha256>;
pub type HmacSha512 = Hmac<Sha512>;
pub type HmacSha1 = Hmac<Sha1>;
mod github {
use super::*;
pub struct Github;
impl WebhookHandler for Github {
fn handle_challenge_request<'header>(
&self,
_: &'header HeaderMap,
_: &SignatureConfigData,
_: &str,
) -> Result<Option<Response>, AuthenticationError> {
Ok(None)
}
fn get_hmac_authentication_data<'payload, 'header, 'prefix>(
&self,
headers: &'header HeaderMap,
raw_payload: &'payload str,
) -> Result<SignatureAuthenticationData<'payload, 'header, 'prefix>, AuthenticationError>
{
let github_secret_header = headers.try_get_webhook_header("X-Hub-Signature-256")?;
let authentication_data = SignatureAuthenticationData::new(
Cow::Borrowed(raw_payload),
github_secret_header,
Some("sha256="),
SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex),
);
Ok(authentication_data)
}
}
}
mod slack {
use super::*;
pub struct Slack;
impl WebhookHandler for Slack {
fn handle_challenge_request<'header>(
&self,
_: &'header HeaderMap,
_: &SignatureConfigData,
_: &str,
) -> Result<Option<Response>, AuthenticationError> {
Ok(None)
}
fn get_hmac_authentication_data<'payload, 'header, 'prefix>(
&self,
headers: &'header HeaderMap,
raw_payload: &'payload str,
) -> Result<SignatureAuthenticationData<'payload, 'header, 'prefix>, AuthenticationError>
{
let slack_secret_signature = headers.try_get_webhook_header("X-Slack-Signature")?;
let slack_timestamp_header =
headers.try_get_webhook_header("X-Slack-Request-Timestamp")?;
let signed_payload = format!("v0:{}:{}", slack_timestamp_header, raw_payload);
Ok(SignatureAuthenticationData::new(
Cow::Owned(signed_payload),
slack_secret_signature,
Some("v0="),
SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex),
))
}
}
}
mod stripe {
use super::*;
pub struct Stripe;
impl WebhookHandler for Stripe {
fn handle_challenge_request<'header>(
&self,
_: &'header HeaderMap,
_: &SignatureConfigData,
_: &str,
) -> Result<Option<Response>, AuthenticationError> {
Ok(None)
}
fn get_hmac_authentication_data<'payload, 'header, 'prefix>(
&self,
headers: &'header HeaderMap,
raw_payload: &'payload str,
) -> Result<SignatureAuthenticationData<'payload, 'header, 'prefix>, AuthenticationError>
{
let stripe_signature_header = headers.try_get_webhook_header("STRIPE-SIGNATURE")?;
let stripe_signature = parse_signature(stripe_signature_header, (",", "="));
let timestamp = *stripe_signature
.get("t")
.ok_or(AuthenticationError::InvalidTimestamp)?;
let v1 = *stripe_signature
.get("v1")
.ok_or(AuthenticationError::InvalidSignature)?;
let signed_payload = format!("{}.{}", timestamp, raw_payload);
Ok(SignatureAuthenticationData::new(
Cow::Owned(signed_payload),
v1,
None,
SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex),
))
}
}
}
mod tiktok {
use super::*;
pub struct TikTok;
impl WebhookHandler for TikTok {
fn handle_challenge_request<'header>(
&self,
_: &'header HeaderMap,
_: &SignatureConfigData,
_: &str,
) -> Result<Option<Response>, AuthenticationError> {
Ok(None)
}
fn get_hmac_authentication_data<'payload, 'header, 'prefix>(
&self,
headers: &'header HeaderMap,
raw_payload: &'payload str,
) -> Result<SignatureAuthenticationData<'payload, 'header, 'prefix>, AuthenticationError>
{
let tiktok_secret_signature = headers.try_get_webhook_header("TikTok-Signature")?;
let stripe_signature = parse_signature(tiktok_secret_signature, (",", "="));
let timestamp = *stripe_signature
.get("t")
.ok_or(AuthenticationError::InvalidTimestamp)?;
let s = *stripe_signature
.get("s")
.ok_or(AuthenticationError::InvalidSignature)?;
let signed_payload = format!("{}.{}", timestamp, raw_payload);
Ok(SignatureAuthenticationData::new(
Cow::Owned(signed_payload),
s,
None,
SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex),
))
}
}
}
mod twitch {
use super::*;
use http::header;
use serde_json::value::RawValue;
#[derive(Debug, Deserialize)]
struct TwitchCrcBody {
challenge: String,
#[allow(unused)]
subscription: Box<RawValue>,
}
pub struct Twitch;
impl WebhookHandler for Twitch {
fn get_hmac_authentication_data<'payload, 'header, 'prefix>(
&self,
headers: &'header HeaderMap,
raw_payload: &'payload str,
) -> Result<SignatureAuthenticationData<'payload, 'header, 'prefix>, AuthenticationError>
{
let twitch_secret_signature =
headers.try_get_webhook_header("Twitch-Eventsub-Message-Signature")?;
let twitch_message_id_header =
headers.try_get_webhook_header("Twitch-Eventsub-Message-Id")?;
let twitch_timestamp_header =
headers.try_get_webhook_header("Twitch-Eventsub-Message-Timestamp")?;
let message = format!(
"{}{}{}",
twitch_message_id_header, twitch_timestamp_header, raw_payload
);
Ok(SignatureAuthenticationData::new(
Cow::Owned(message),
twitch_secret_signature,
Some("sha256="),
SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex),
))
}
fn handle_challenge_request<'header>(
&self,
headers: &'header HeaderMap,
signature_config_data: &SignatureConfigData,
raw_payload: &str,
) -> Result<Option<Response>, AuthenticationError> {
let authentication_data = self.get_hmac_authentication_data(headers, raw_payload)?;
verify_hmac_signature(authentication_data, &signature_config_data.secret_key)?;
let twitch_eventsub_message_type =
headers.try_get_webhook_header("Twitch-Eventsub-Message-Type")?;
if twitch_eventsub_message_type != "webhook_callback_verification" {
return Ok(None);
}
let twitch_crc_body =
serde_json::from_str::<TwitchCrcBody>(raw_payload).map_err(|e| {
AuthenticationError::InvalidChallengeResponse(format!(
"Twitch :{}",
e.to_string()
))
})?;
let response = (
StatusCode::OK,
[(header::CONTENT_TYPE, "text/plain")],
twitch_crc_body.challenge.to_string(),
);
Ok(Some(response.into_response()))
}
}
}
mod zoom {
use axum::Json;
use super::*;
#[derive(Debug, Deserialize)]
struct ZoomPayload {
#[serde(rename = "plainToken")]
plain_token: String,
}
#[derive(Debug, Deserialize)]
#[allow(unused)]
struct ZoomChallengeResponse {
payload: ZoomPayload,
event_ts: u64,
event: String,
}
pub struct Zoom;
impl WebhookHandler for Zoom {
fn handle_challenge_request<'header>(
&self,
_: &'header HeaderMap,
signature_config_data: &SignatureConfigData,
raw_payload: &str,
) -> Result<Option<Response>, AuthenticationError> {
let Ok(zoom_request_body) = serde_json::from_str::<ZoomChallengeResponse>(raw_payload)
else {
return Ok(None);
};
if zoom_request_body.event != "endpoint.url_validation" {
return Ok(None);
}
let hmac_signature = calculate_hmac_signature(
HmacAlgorithm::Sha256,
&signature_config_data.secret_key,
&zoom_request_body.payload.plain_token,
);
let encoded_hmac_signature = encode_hmac_signature(Encoding::Hex, &hmac_signature);
let response = (
StatusCode::OK,
Json(json!({
"plainToken": zoom_request_body.payload.plain_token,
"encryptedToken": encoded_hmac_signature
})),
);
Ok(Some(response.into_response()))
}
fn get_hmac_authentication_data<'payload, 'header, 'prefix>(
&self,
headers: &'header HeaderMap,
raw_payload: &'payload str,
) -> Result<SignatureAuthenticationData<'payload, 'header, 'prefix>, AuthenticationError>
{
let zoom_signature_header = headers.try_get_webhook_header("x-zm-signature")?;
let zoom_timestamp_header = headers.try_get_webhook_header("x-zm-request-timestamp")?;
let message = format!("v0:{}:{}", zoom_timestamp_header, raw_payload);
Ok(SignatureAuthenticationData::new(
Cow::Owned(message),
zoom_signature_header,
Some("v0="),
SignatureAuthenticationDetails::new(HmacAlgorithm::Sha256, Encoding::Hex),
))
}
}
}
use constant_time_eq::constant_time_eq;
use github::Github;
use slack::Slack;
use stripe::Stripe;
use tiktok::TikTok;
use twitch::Twitch;
use zoom::Zoom;
#[derive(Debug)]
pub struct SignatureAuthenticationDetails {
pub algorithm_to_use: HmacAlgorithm,
pub header_key_encoding: Encoding,
}
impl SignatureAuthenticationDetails {
#[inline]
fn new(algorithm_to_use: HmacAlgorithm, header_key_encoding: Encoding) -> Self {
Self { algorithm_to_use, header_key_encoding }
}
}
fn parse_signature<'header>(
signature: &'header str,
splitters: (&str, &str),
) -> HashMap<&'header str, &'header str> {
let headers: HashMap<&str, &str> = signature
.split(splitters.0)
.map(|header| {
let mut key_and_value = header.split(splitters.1);
let key = key_and_value.next();
let value = key_and_value.next();
(key, value)
})
.filter_map(|(key, value)| match (key, value) {
(Some(key), Some(value)) => Some((key, value)),
_ => None,
})
.collect();
headers
}
#[derive(Debug)]
pub struct SignatureAuthenticationData<'payload, 'header, 'prefix> {
pub signed_payload: Cow<'payload, str>,
pub header_key_value: &'header str,
pub signature_prefix: Option<&'prefix str>,
pub config: SignatureAuthenticationDetails,
}
impl<'payload, 'header, 'prefix> SignatureAuthenticationData<'payload, 'header, 'prefix> {
pub fn new(
signed_payload: Cow<'payload, str>,
header_key_value: &'header str,
signature_prefix: Option<&'prefix str>,
config: SignatureAuthenticationDetails,
) -> Self {
Self { signed_payload, header_key_value, signature_prefix, config }
}
}
pub trait WebhookHandler {
fn handle_challenge_request<'header>(
&self,
headers: &'header HeaderMap,
signature_config_data: &SignatureConfigData,
raw_payload: &str,
) -> Result<Option<Response>, AuthenticationError>;
fn get_hmac_authentication_data<'payload, 'header, 'prefix>(
&self,
headers: &'header HeaderMap,
raw_payload: &'payload str,
) -> Result<SignatureAuthenticationData<'payload, 'header, 'prefix>, AuthenticationError>;
}
#[derive(Clone, Copy, Debug, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum HmacAlgorithm {
Sha1,
Sha256,
Sha512,
}
#[derive(Clone, Copy, Debug, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum Encoding {
Base64,
Base64Uri,
Hex,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct SignatureAuthenticationMethod {
algorithm: HmacAlgorithm,
encoding: Encoding,
signature_header_name: String,
signature_prefix: Option<String>,
}
pub struct SignatureConfigData<'config> {
secret_key: &'config str,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct SignatureAuthentication {
signature_provider: WebhookType,
secret_key: String,
authentication_config: Option<SignatureAuthenticationMethod>,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct BasicAuthAuthentication {
pub username: String,
pub password: String,
}
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct ApiKeyAuthentication {
pub api_key_header: String,
pub api_key_secret: String,
}
#[derive(Debug, Eq, PartialEq, Hash, Clone, Copy, Serialize, Deserialize)]
#[non_exhaustive]
pub enum WebhookType {
Github,
Slack,
Stripe,
TikTok,
Twitch,
Zoom,
Custom,
}
impl WebhookType {
pub fn get_webhook_handler(&self) -> Option<&'static dyn WebhookHandler> {
let handler: &'static dyn WebhookHandler = match *self {
WebhookType::Github => &Github,
WebhookType::Slack => &Slack,
WebhookType::Stripe => &Stripe,
WebhookType::TikTok => &TikTok,
WebhookType::Twitch => &Twitch,
WebhookType::Zoom => &Zoom,
WebhookType::Custom => return None,
};
Some(handler)
}
}
trait TryGetWebhookHeader {
fn try_get_webhook_header<'header>(
&'header self,
header_name: &str,
) -> Result<&'header str, AuthenticationError>;
}
impl TryGetWebhookHeader for HeaderMap<HeaderValue> {
fn try_get_webhook_header<'header>(
&'header self,
header_name: &str,
) -> Result<&'header str, AuthenticationError> {
let Some(signature_header) = self.get(header_name) else {
return Err(AuthenticationError::MissingHeader(header_name.to_string()));
};
let Some(signature_header) = signature_header.to_str().ok() else {
return Err(AuthenticationError::InvalidHeader(header_name.to_string()));
};
Ok(signature_header)
}
}
pub fn calculate_hmac_signature(algorithm: HmacAlgorithm, secret: &str, payload: &str) -> Vec<u8> {
match algorithm {
HmacAlgorithm::Sha1 => {
let mut mac =
HmacSha1::new_from_slice(secret.as_bytes()).expect("HMAC can take key of any size");
mac.update(payload.as_bytes());
mac.finalize().into_bytes().to_vec()
}
HmacAlgorithm::Sha256 => {
let mut mac = HmacSha256::new_from_slice(secret.as_bytes())
.expect("HMAC can take key of any size");
mac.update(payload.as_bytes());
mac.finalize().into_bytes().to_vec()
}
HmacAlgorithm::Sha512 => {
let mut mac = HmacSha512::new_from_slice(secret.as_bytes())
.expect("HMAC can take key of any size");
mac.update(payload.as_bytes());
mac.finalize().into_bytes().to_vec()
}
}
}
pub fn encode_hmac_signature(encoding: Encoding, hmac_signature: &[u8]) -> String {
match encoding {
Encoding::Hex => hex::encode(hmac_signature),
Encoding::Base64 => BASE64_STANDARD.encode(hmac_signature),
Encoding::Base64Uri => BASE64_URL_SAFE.encode(hmac_signature),
}
}
pub fn verify_hmac_signature(
authentication_data: SignatureAuthenticationData,
webhook_signing_secret: &str,
) -> Result<(), AuthenticationError> {
let hmac_signature = calculate_hmac_signature(
authentication_data.config.algorithm_to_use,
&webhook_signing_secret,
&authentication_data.signed_payload,
);
let encoded_signature = encode_hmac_signature(
authentication_data.config.header_key_encoding,
&hmac_signature,
);
let final_expected_signature =
if let Some(signature_prefix) = authentication_data.signature_prefix {
format!("{}{}", signature_prefix, encoded_signature)
} else {
encoded_signature
};
if !constant_time_eq(
final_expected_signature.as_bytes(),
authentication_data.header_key_value.as_bytes(),
) {
return Err(AuthenticationError::InvalidSignature);
}
Ok(())
}
#[derive(Debug, Serialize, Deserialize, Clone)]
#[serde(untagged)]
pub enum AuthenticationMethod {
Signature(SignatureAuthentication),
BasicAuth(BasicAuthAuthentication),
ApiKey(ApiKeyAuthentication),
}
impl AuthenticationMethod {
pub fn authenticate_http_request(
&self,
headers: &HeaderMap,
raw_payload: Option<&String>,
) -> Result<Option<Response>, AuthenticationError> {
match self {
AuthenticationMethod::Signature(SignatureAuthentication {
secret_key,
authentication_config,
signature_provider,
}) => {
let raw_payload = raw_payload.ok_or(AuthenticationError::InvalidPayload)?;
let config_data = SignatureConfigData { secret_key: &secret_key };
let handler = signature_provider.get_webhook_handler();
let challenge_response = handler
.map(|handler| {
handler.handle_challenge_request(headers, &config_data, raw_payload)
})
.transpose()?
.flatten();
if let Some(challenge_response) = challenge_response {
return Ok(Some(challenge_response));
}
let authentication_data = match handler {
Some(handler) => handler.get_hmac_authentication_data(headers, raw_payload)?,
None => {
let authentication_config = authentication_config
.as_ref()
.ok_or(AuthenticationError::InvalidCustomConfig)?;
let signature_header_value = headers
.try_get_webhook_header(&authentication_config.signature_header_name)?;
SignatureAuthenticationData::new(
Cow::Borrowed(raw_payload),
signature_header_value,
authentication_config.signature_prefix.as_deref(),
SignatureAuthenticationDetails::new(
authentication_config.algorithm,
authentication_config.encoding,
),
)
}
};
verify_hmac_signature(authentication_data, &secret_key)?;
}
AuthenticationMethod::ApiKey(ApiKeyAuthentication {
api_key_header,
api_key_secret,
}) => {
let api_key_to_cmp = headers
.try_get_webhook_header(&api_key_header)
.map_err(|_| AuthenticationError::InvalidApiKey)?;
if api_key_to_cmp != api_key_secret {
return Err(AuthenticationError::InvalidApiKey);
}
}
AuthenticationMethod::BasicAuth(BasicAuthAuthentication { username, password }) => {
let mut credentials_store = headers
.try_get_webhook_header("Authorization")
.map_err(|_| AuthenticationError::UnauthorizedBasicHttpAuth)?
.split(' ');
let _ = credentials_store
.next()
.filter(|r#type| *r#type == "Basic")
.ok_or(AuthenticationError::UnauthorizedBasicHttpAuth)?;
let credentials_as_base64 = credentials_store
.next()
.ok_or(AuthenticationError::UnauthorizedBasicHttpAuth)?;
let credentials_from_base64_as_bytes = BASE64_STANDARD
.decode(credentials_as_base64.as_bytes())
.map_err(|_| AuthenticationError::UnauthorizedBasicHttpAuth)?;
let credentials_separated_with_colon =
String::from_utf8(credentials_from_base64_as_bytes)
.map_err(|_| AuthenticationError::UnauthorizedBasicHttpAuth)?;
let credentials = credentials_separated_with_colon.split(':').collect_vec();
if credentials.len() != 2 {
return Err(AuthenticationError::UnauthorizedBasicHttpAuth);
}
if credentials.get(0).unwrap() != username
|| credentials.get(1).unwrap() != password
{
return Err(AuthenticationError::UnauthorizedBasicHttpAuth);
}
}
}
Ok(None)
}
}
#[derive(thiserror::Error, Debug)]
#[allow(unused)]
pub enum AuthenticationError {
#[error("failed to parse timestamp")]
InvalidTimestamp,
#[error("invalid secret")]
InvalidSecret(#[from] base64::DecodeError),
#[error("invalid header `{0}`")]
InvalidHeader(String),
#[error("signature timestamp too old")]
TimestampTooOldError,
#[error("signature timestamp too far in future")]
FutureTimestampError,
#[error("missing header {0}")]
MissingHeader(String),
#[error("signature invalid")]
InvalidSignature,
#[error("payload invalid")]
InvalidPayload,
#[error("invalid custom config")]
InvalidCustomConfig,
#[error("invalid auth header: {0}")]
InvalidAuthHeader(String),
#[error("invalid api key")]
InvalidApiKey,
#[error("invalid challenge response: {0}")]
InvalidChallengeResponse(String),
#[error("")]
UnauthorizedBasicHttpAuth,
}
impl IntoResponse for AuthenticationError {
fn into_response(self) -> Response {
let (status, error_message) = match &self {
AuthenticationError::InvalidTimestamp
| AuthenticationError::InvalidPayload
| AuthenticationError::InvalidHeader(_)
| AuthenticationError::MissingHeader(_)
| AuthenticationError::TimestampTooOldError
| AuthenticationError::FutureTimestampError
| AuthenticationError::InvalidCustomConfig
| AuthenticationError::InvalidChallengeResponse(_) => {
(StatusCode::BAD_REQUEST, self.to_string())
}
AuthenticationError::InvalidSecret(_)
| AuthenticationError::InvalidSignature
| AuthenticationError::InvalidAuthHeader(_) => {
(StatusCode::UNAUTHORIZED, self.to_string())
}
AuthenticationError::UnauthorizedBasicHttpAuth => {
return (
StatusCode::UNAUTHORIZED,
[(header::WWW_AUTHENTICATE, r#"Basic realm="Restricted Area""#)],
"Unauthorized",
)
.into_response()
}
AuthenticationError::InvalidApiKey => {
return (StatusCode::UNAUTHORIZED, "Unauthorized").into_response()
}
};
let body = json!({ "error": error_message });
let mut headers = HeaderMap::new();
headers.insert("Content-Type", HeaderValue::from_static("application/json"));
(status, headers, body.to_string()).into_response()
}
}
backend/windmill-api/src/triggers/http/mod.rs
-410
View File
@@ -1,410 +0,0 @@
use std::collections::HashMap;
use quick_cache::sync::Cache;
use serde::{Deserialize, Serialize};
use sqlx::{types::Json as SqlxJson, FromRow};
use tokio::sync::{RwLock, RwLockReadGuard};
use windmill_common::{
error::{Error, Result},
flows::Retry,
s3_helpers::S3Object,
worker::CLOUD_HOSTED,
DB,
};
use crate::{db::ApiAuthed, triggers::TriggerMode, utils::ExpiringCacheEntry};
pub mod handler;
pub mod http_trigger_args;
pub mod http_trigger_auth;
lazy_static::lazy_static! {
static ref HTTP_ACCESS_CACHE: Cache<(String, String, ApiAuthed), ExpiringCacheEntry<()>> = Cache::new(100);
static ref HTTP_AUTH_CACHE: Cache<(String, String, ApiAuthed), ExpiringCacheEntry<http_trigger_auth::AuthenticationMethod>> = Cache::new(100);
static ref HTTP_ROUTERS_CACHE: RwLock<RoutersCache> = RwLock::new(RoutersCache {
routers: HashMap::new(),
version: 0,
});
}
#[derive(Debug, Deserialize, Clone)]
pub struct TriggerRoute {
path: String,
script_path: String,
is_flow: bool,
route_path: String,
workspace_id: String,
request_type: RequestType,
authentication_method: AuthenticationMethod,
edited_by: String,
email: String,
static_asset_config: Option<sqlx::types::Json<S3Object>>,
is_static_website: bool,
authentication_resource_path: Option<String>,
workspaced_route: bool,
wrap_body: bool,
raw_string: bool,
error_handler_path: Option<String>,
error_handler_args: Option<sqlx::types::Json<HashMap<String, serde_json::Value>>>,
retry: Option<sqlx::types::Json<Retry>>,
mode: TriggerMode,
}
pub struct RoutersCache {
routers: HashMap<HttpMethod, matchit::Router<TriggerRoute>>,
version: i64,
}
#[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Copy, Hash, Eq, PartialEq)]
#[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")]
#[serde(rename_all = "lowercase")]
pub enum HttpMethod {
Get,
Post,
Put,
Delete,
Patch,
}
#[derive(Serialize, Deserialize, sqlx::Type, Debug, Clone, Copy, PartialEq)]
#[sqlx(type_name = "REQUEST_TYPE", rename_all = "snake_case")]
#[serde(rename_all = "snake_case")]
pub enum RequestType {
Sync,
Async,
SyncSse,
}
impl TryFrom<&http::Method> for HttpMethod {
type Error = Error;
fn try_from(method: &http::Method) -> Result<Self> {
match method {
&http::Method::GET => Ok(HttpMethod::Get),
&http::Method::POST => Ok(HttpMethod::Post),
&http::Method::PUT => Ok(HttpMethod::Put),
&http::Method::DELETE => Ok(HttpMethod::Delete),
&http::Method::PATCH => Ok(HttpMethod::Patch),
_ => Err(Error::BadRequest("Invalid HTTP method".to_string())),
}
}
}
#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone, Copy)]
#[sqlx(type_name = "AUTHENTICATION_METHOD", rename_all = "snake_case")]
#[serde(rename_all(serialize = "snake_case", deserialize = "snake_case"))]
pub enum AuthenticationMethod {
None,
Windmill,
ApiKey,
BasicHttp,
CustomScript,
Signature,
}
#[derive(Debug, Clone, FromRow, Serialize, Deserialize)]
pub struct HttpConfig {
pub route_path: String,
pub route_path_key: String,
pub request_type: RequestType,
pub authentication_method: AuthenticationMethod,
pub http_method: HttpMethod,
pub summary: Option<String>,
pub description: Option<String>,
pub static_asset_config: Option<SqlxJson<S3Object>>,
pub is_static_website: bool,
pub authentication_resource_path: Option<String>,
pub workspaced_route: bool,
pub wrap_body: bool,
pub raw_string: bool,
}
#[derive(Debug, Clone, Serialize)]
pub struct HttpConfigRequest {
#[serde(default)]
pub route_path: String,
pub request_type: RequestType,
pub authentication_method: AuthenticationMethod,
pub http_method: HttpMethod,
pub summary: Option<String>,
pub description: Option<String>,
pub static_asset_config: Option<SqlxJson<S3Object>>,
pub is_static_website: bool,
pub authentication_resource_path: Option<String>,
pub workspaced_route: Option<bool>,
pub wrap_body: Option<bool>,
pub raw_string: Option<bool>,
}
#[derive(Deserialize)]
struct HttpConfigRequestHelper {
#[serde(default)]
route_path: String,
request_type: Option<RequestType>,
is_async: Option<bool>,
authentication_method: AuthenticationMethod,
http_method: HttpMethod,
summary: Option<String>,
description: Option<String>,
static_asset_config: Option<SqlxJson<S3Object>>,
is_static_website: bool,
authentication_resource_path: Option<String>,
workspaced_route: Option<bool>,
wrap_body: Option<bool>,
raw_string: Option<bool>,
}
impl<'de> Deserialize<'de> for HttpConfigRequest {
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let helper = HttpConfigRequestHelper::deserialize(deserializer)?;
// Determine request_type with backward compatibility
let request_type = if let Some(mode) = helper.request_type {
mode
} else if let Some(is_async) = helper.is_async {
if is_async {
RequestType::Async
} else {
RequestType::Sync
}
} else {
RequestType::Sync
};
Ok(HttpConfigRequest {
route_path: helper.route_path,
request_type,
authentication_method: helper.authentication_method,
http_method: helper.http_method,
summary: helper.summary,
description: helper.description,
static_asset_config: helper.static_asset_config,
is_static_website: helper.is_static_website,
authentication_resource_path: helper.authentication_resource_path,
workspaced_route: helper.workspaced_route,
wrap_body: helper.wrap_body,
raw_string: helper.raw_string,
})
}
}
// Regex patterns for route validation
lazy_static::lazy_static! {
// Matches named params like :id or wildcards like :* or *
static ref ROUTE_PATH_KEY_RE: regex::Regex = regex::Regex::new(r"(/)?(:|\*)[-\w]+").unwrap();
static ref VALID_ROUTE_PATH_RE: regex::Regex = regex::Regex::new(r"^(\*[-\w]+$|:?[-\w]+)(/(\*[-\w]+$|:?[-\w]+))*$").unwrap();
}
#[derive(Deserialize)]
pub struct RouteExists {
pub route_path: String,
pub http_method: HttpMethod,
pub trigger_path: Option<String>,
pub workspaced_route: Option<bool>,
}
pub fn validate_authentication_method(
authentication_method: AuthenticationMethod,
raw_string: Option<bool>,
) -> Result<()> {
match (authentication_method, raw_string) {
(AuthenticationMethod::CustomScript, raw) if !raw.unwrap_or(false) => {
Err(Error::BadRequest(
"To use custom script authentication, please enable the raw body option."
.to_string(),
))
}
_ => Ok(()),
}
}
pub async fn refresh_routers(db: &DB) -> Result<(bool, RwLockReadGuard<'_, RoutersCache>)> {
let version = sqlx::query_scalar!("SELECT last_value FROM http_trigger_version_seq",)
.fetch_one(db)
.await?;
let routers_cache = HTTP_ROUTERS_CACHE.read().await;
if routers_cache.version == 0 || version > routers_cache.version {
drop(routers_cache);
let mut routers = HashMap::new();
for http_method in [
HttpMethod::Get,
HttpMethod::Post,
HttpMethod::Put,
HttpMethod::Patch,
HttpMethod::Delete,
] {
let triggers = sqlx::query_as!(
TriggerRoute,
r#"
SELECT
path,
script_path,
is_flow,
route_path,
authentication_resource_path,
workspace_id,
request_type AS "request_type: _",
authentication_method AS "authentication_method: _",
edited_by,
email,
static_asset_config AS "static_asset_config: _",
wrap_body,
raw_string,
workspaced_route,
is_static_website,
error_handler_path,
error_handler_args as "error_handler_args: _",
retry as "retry: _",
mode as "mode: _"
FROM
http_trigger
WHERE
http_method = $1 AND
(mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE)
"#,
&http_method as &HttpMethod
)
.fetch_all(db)
.await?;
let mut router = matchit::Router::new();
for trigger in triggers {
let full_path = if trigger.workspaced_route || *CLOUD_HOSTED {
format!("/{}/{}", trigger.workspace_id, trigger.route_path)
} else {
format!("/{}", trigger.route_path)
};
if trigger.is_static_website {
router
.insert(format!("{}/*wm_subpath", full_path), trigger.clone())
.unwrap_or_else(|e| {
tracing::warn!(
"Failed to consider HTTP route {}/*wm_subpath: {:?}",
full_path,
e,
);
});
}
router
.insert(full_path.clone(), trigger.clone())
.unwrap_or_else(|e| {
tracing::warn!("Failed to consider HTTP route {}: {:?}", full_path, e,);
});
}
routers.insert(http_method, router);
}
let mut routers_cache = HTTP_ROUTERS_CACHE.write().await;
*routers_cache = RoutersCache { routers, version };
Ok((true, routers_cache.downgrade()))
} else {
tracing::debug!("No HTTP routers refresh needed");
Ok((false, routers_cache))
}
}
pub async fn refresh_routers_loop(
db: &DB,
mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
) -> () {
match refresh_routers(db).await {
Ok(_) => {
tracing::info!("Loaded HTTP routers");
}
Err(err) => {
tracing::error!("Error loading HTTP routers: {err:#}");
}
};
let db = db.clone();
tokio::spawn(async move {
loop {
tokio::select! {
_ = killpill_rx.recv() => {
break;
}
_ = tokio::time::sleep(std::time::Duration::from_secs(60)) => {
match refresh_routers(&db).await {
Ok((true, _)) => {
tracing::info!("Refreshed HTTP routers");
}
Err(err) => {
tracing::error!("Error refreshing HTTP routers: {err:#}");
}
_ => {}
}
}
}
}
});
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_request_type_backward_compatibility() {
// Test with new request_type field
let json_new = r#"{
"route_path": "/test",
"request_type": "sync_sse",
"authentication_method": "none",
"http_method": "get",
"is_static_website": false
}"#;
let config: HttpConfigRequest = serde_json::from_str(json_new).unwrap();
assert_eq!(config.request_type, RequestType::SyncSse);
// Test with legacy is_async = true
let json_legacy_async = r#"{
"route_path": "/test",
"is_async": true,
"authentication_method": "none",
"http_method": "get",
"is_static_website": false
}"#;
let config: HttpConfigRequest = serde_json::from_str(json_legacy_async).unwrap();
assert_eq!(config.request_type, RequestType::Async);
// Test with legacy is_async = false
let json_legacy_sync = r#"{
"route_path": "/test",
"is_async": false,
"authentication_method": "none",
"http_method": "get",
"is_static_website": false
}"#;
let config: HttpConfigRequest = serde_json::from_str(json_legacy_sync).unwrap();
assert_eq!(config.request_type, RequestType::Sync);
// Test with neither field (default to sync)
let json_default = r#"{
"route_path": "/test",
"authentication_method": "none",
"http_method": "get",
"is_static_website": false
}"#;
let config: HttpConfigRequest = serde_json::from_str(json_default).unwrap();
assert_eq!(config.request_type, RequestType::Sync);
// Test that request_type takes precedence over is_async
let json_both = r#"{
"route_path": "/test",
"request_type": "sync_sse",
"is_async": true,
"authentication_method": "none",
"http_method": "get",
"is_static_website": false
}"#;
let config: HttpConfigRequest = serde_json::from_str(json_both).unwrap();
assert_eq!(config.request_type, RequestType::SyncSse);
}
}
backend/windmill-api/src/triggers/kafka/handler_oss.rs
-67
View File
@@ -1,67 +0,0 @@
#[cfg(not(feature = "private"))]
use crate::triggers::TriggerData;
#[allow(unused)]
#[cfg(feature = "private")]
pub use super::handler_ee::*;
#[cfg(not(feature = "private"))]
use {
super::KafkaTrigger,
crate::{
db::{ApiAuthed, DB},
triggers::TriggerCrud,
},
axum::async_trait,
sqlx::PgConnection,
windmill_common::error::{Error, Result},
windmill_git_sync::DeployedObject,
};
#[cfg(not(feature = "private"))]
#[async_trait]
impl TriggerCrud for KafkaTrigger {
type Trigger = ();
type TriggerConfig = ();
type TriggerConfigRequest = ();
type TestConnectionConfig = ();
const TABLE_NAME: &'static str = "";
const TRIGGER_TYPE: &'static str = "";
const SUPPORTS_SERVER_STATE: bool = false;
const SUPPORTS_TEST_CONNECTION: bool = false;
const ROUTE_PREFIX: &'static str = "/kafka_triggers";
const DEPLOYMENT_NAME: &'static str = "";
const IS_ALLOWED_ON_CLOUD: bool = false;
fn get_deployed_object(path: String) -> DeployedObject {
DeployedObject::KafkaTrigger { path }
}
async fn create_trigger(
&self,
_db: &DB,
_tx: &mut PgConnection,
_authed: &ApiAuthed,
_w_id: &str,
_trigger: TriggerData<Self::TriggerConfigRequest>,
) -> Result<()> {
Err(Error::BadRequest(
"Kafka triggers are not available in open source version".to_string(),
))
}
async fn update_trigger(
&self,
_db: &DB,
_executor: &mut PgConnection,
_authed: &ApiAuthed,
_workspace_id: &str,
_path: &str,
_trigger: TriggerData<Self::TriggerConfigRequest>,
) -> Result<()> {
Err(Error::BadRequest(
"Kafka triggers are not available in open source version".to_string(),
))
}
}
backend/windmill-api/src/triggers/kafka/listener_oss.rs
-41
View File
@@ -1,41 +0,0 @@
#[allow(unused)]
#[cfg(feature = "private")]
pub use super::listener_ee::*;
#[cfg(not(feature = "private"))]
use {
super::KafkaTrigger,
crate::triggers::{listener::ListeningTrigger, Listener},
std::sync::Arc,
tokio::sync::RwLock,
windmill_common::{error::Result, jobs::JobTriggerKind, DB},
};
#[cfg(not(feature = "private"))]
#[async_trait::async_trait]
impl Listener for KafkaTrigger {
type Consumer = ();
type Extra = ();
const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Kafka;
async fn get_consumer(
&self,
_db: &DB,
_listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
_err_message: Arc<RwLock<Option<String>>>,
_killpill_rx: tokio::sync::broadcast::Receiver<()>,
) -> Result<Option<Self::Consumer>> {
Ok(None)
}
async fn consume(
&self,
_db: &DB,
_consumer: Self::Consumer,
_listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
_err_message: Arc<RwLock<Option<String>>>,
_killpill_rx: tokio::sync::broadcast::Receiver<()>,
) {
()
}
}
backend/windmill-api/src/triggers/kafka/mod.rs
-15
View File
@@ -1,15 +0,0 @@
#[cfg(feature = "private")]
mod handler_ee;
pub mod handler_oss;
#[cfg(feature = "private")]
mod listener_ee;
pub mod listener_oss;
#[cfg(feature = "private")]
mod mod_ee;
#[cfg(feature = "private")]
pub use mod_ee::*;
#[derive(Copy, Clone)]
pub struct KafkaTrigger;
backend/windmill-api/src/triggers/listener.rs
-928
View File
@@ -1,928 +0,0 @@
use std::{collections::HashMap, fmt::Debug, sync::Arc};
use crate::{
capture::insert_capture_payload,
db::ApiAuthed,
triggers::{
handler::TriggerCrud,
trigger_helpers::{trigger_runnable, TriggerJobArgs},
Trigger, TriggerErrorHandling, TriggerMode,
},
users::fetch_api_authed,
};
use async_trait::async_trait;
use itertools::Itertools;
use rand::seq::SliceRandom;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use serde_json::value::RawValue;
use sql_builder::SqlBuilder;
use sqlx::{FromRow, Row};
use tokio::sync::RwLock;
use windmill_common::{
error::{Error, Result},
jobs::JobTriggerKind,
triggers::{TriggerKind, TriggerMetadata},
utils::report_critical_error,
DB, INSTANCE_NAME,
};
#[allow(unused)]
#[async_trait]
pub trait Listener: TriggerCrud + TriggerJobArgs {
type Consumer: Send;
type Extra: Send + Sync;
type ExtraState: Send + Sync;
const JOB_TRIGGER_KIND: JobTriggerKind;
const EXTRA_TRIGGER_AND_WHERE_CLAUSE: &[&'static str] = &[];
const EXTRA_CAPTURE_AND_WHERE_CLAUSE: &[&'static str] = &[];
async fn get_consumer(
&self,
db: &DB,
listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
err_message: Arc<RwLock<Option<String>>>,
killpill_rx: tokio::sync::broadcast::Receiver<()>,
) -> Result<Option<Self::Consumer>>;
async fn consume(
&self,
db: &DB,
consumer: Self::Consumer,
listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
err_message: Arc<RwLock<Option<String>>>,
killpill_rx: tokio::sync::broadcast::Receiver<()>,
extra: Option<&Self::ExtraState>,
);
async fn fetch_enabled_unlistened_triggers(
&self,
db: &DB,
) -> Result<Vec<ListeningTrigger<Self::TriggerConfig>>> {
let mut fields = vec![
"workspace_id",
"path",
"script_path",
"is_flow",
"edited_by",
"email",
"edited_at",
"extra_perms",
"mode",
"error_handler_path",
"error_handler_args",
"retry",
];
fields.extend_from_slice(Self::ADDITIONAL_SELECT_FIELDS);
let mut sqlb = SqlBuilder::select_from(Self::TABLE_NAME);
sqlb.fields(&fields)
.and_where("(mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE)")
.and_where(
"(last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')",
);
for where_clause in Self::EXTRA_TRIGGER_AND_WHERE_CLAUSE {
sqlb.and_where(where_clause);
}
let sql = sqlb
.sql()
.map_err(|e| Error::InternalErr(format!("SQL error: {}", e)))?;
let triggers: Vec<Trigger<Self::TriggerConfig>> =
sqlx::query_as(&sql).fetch_all(db).await?;
let triggers = triggers
.into_iter()
.map(|trigger| ListeningTrigger {
path: trigger.base.path,
workspace_id: trigger.base.workspace_id,
is_flow: trigger.base.is_flow,
username: trigger.base.edited_by,
email: trigger.base.email,
script_path: trigger.base.script_path,
trigger_config: trigger.config,
error_handling: Some(trigger.error_handling),
trigger_mode: true,
suspended_mode: trigger.base.mode == TriggerMode::Suspended,
})
.collect_vec();
Ok(triggers)
}
async fn fetch_unlistened_captures(
&self,
db: &DB,
) -> Result<Vec<ListeningTrigger<Self::TriggerConfig>>> {
let fields = vec![
"path",
"is_flow",
"workspace_id",
"owner AS username",
"email",
"trigger_config",
];
let mut sqlb = SqlBuilder::select_from("capture_config");
sqlb.fields(&fields)
.and_where(format!("trigger_kind = '{}'", Self::TRIGGER_KIND.to_key()))
.and_where("last_client_ping > NOW() - INTERVAL '10 seconds'")
.and_where("trigger_config IS NOT NULL")
.and_where(
"(last_server_ping IS NULL OR last_server_ping < NOW() - INTERVAL '15 seconds')",
);
for where_clause in Self::EXTRA_CAPTURE_AND_WHERE_CLAUSE {
sqlb.and_where(where_clause);
}
let sql = sqlb.sql().expect("failed to build SQL");
let captures: Vec<Capture<Self::TriggerConfig>> =
sqlx::query_as(&sql).fetch_all(db).await?;
let captures = captures
.into_iter()
.map(|capture| ListeningTrigger {
username: capture.username,
path: capture.path,
workspace_id: capture.workspace_id,
script_path: "".to_string(),
email: capture.email,
trigger_config: capture.trigger_config,
trigger_mode: false,
is_flow: capture.is_flow,
error_handling: None,
suspended_mode: false,
})
.collect_vec();
Ok(captures)
}
async fn get_extra_state(&self) -> Option<Self::ExtraState> {
None
}
async fn cleanup(
&self,
_db: &DB,
_listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
_extra: Option<&Self::ExtraState>,
) -> Result<()> {
Ok(())
}
async fn loop_ping(
&self,
db: &DB,
listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
status: Arc<RwLock<Option<String>>>,
error_message: Option<String>,
) {
update_rw_lock(status.clone(), error_message).await;
loop {
if let None = self
.update_ping(db, listening_trigger, status.read().await.as_deref())
.await
{
return;
}
tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
}
}
async fn update_ping(
&self,
db: &DB,
listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
error: Option<&str>,
) -> Option<()> {
if listening_trigger.trigger_mode {
self.update_trigger_ping(db, listening_trigger, error).await
} else {
self.update_capture_ping(db, listening_trigger, error).await
}
}
async fn update_ping_and_loop_ping_status(
&self,
db: &DB,
listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
loop_ping_status: Arc<RwLock<Option<String>>>,
error: Option<String>,
) -> Option<()> {
// update immediately the ping status and update the loop ping status so that the next loop pings will display the new status
update_rw_lock(loop_ping_status.clone(), error.clone()).await;
if let None = self
.update_ping(db, listening_trigger, error.as_deref())
.await
{
return None;
}
Some(())
}
async fn update_trigger_ping(
&self,
db: &DB,
listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
error: Option<&str>,
) -> Option<()> {
let updated = sqlx::query_scalar::<_, i32>(&format!(
r#"
UPDATE
{}
SET
last_server_ping = now(), error = $1
WHERE
workspace_id = $2 AND
path = $3 AND
server_id = $4 AND
(mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE)
RETURNING 1
"#,
Self::TABLE_NAME
))
.bind(error)
.bind(&listening_trigger.workspace_id)
.bind(&listening_trigger.path)
.bind(&*INSTANCE_NAME)
.fetch_optional(db)
.await;
self.handle_ping_result(updated, db, listening_trigger, "trigger")
.await
}
async fn update_capture_ping(
&self,
db: &DB,
listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
error: Option<&str>,
) -> Option<()> {
let updated = sqlx::query_scalar!(
r#"
UPDATE
capture_config
SET
last_server_ping = now(), error = $1
WHERE
workspace_id = $2 AND
path = $3 AND
is_flow = $4 AND
trigger_kind = $5 AND
server_id = $6 AND
last_client_ping > NOW() - INTERVAL '10 seconds'
RETURNING 1
"#,
error,
&listening_trigger.workspace_id,
&listening_trigger.path,
&listening_trigger.is_flow,
Self::TRIGGER_KIND as TriggerKind,
&*INSTANCE_NAME
)
.fetch_optional(db)
.await
.map(|result| result.flatten());
self.handle_ping_result(updated, db, listening_trigger, "capture")
.await
}
async fn handle_ping_result(
&self,
result: sqlx::Result<Option<i32>>,
db: &DB,
listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
entity_type: &str,
) -> Option<()> {
match result {
Ok(updated) => {
if updated.is_none() {
self.reset_ping_for_restart(db, listening_trigger).await;
tracing::info!(
"{} {} {} changed, disabled, or deleted, stopping...",
Self::TRIGGER_KIND,
entity_type,
listening_trigger.path
);
return None;
}
}
Err(error) => {
tracing::warn!(
"Error updating ping of {} {} {}: {:?}",
Self::TRIGGER_KIND,
entity_type,
&listening_trigger.path,
error
);
}
}
Some(())
}
async fn reset_ping_for_restart(
&self,
db: &DB,
listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
) {
if listening_trigger.trigger_mode {
let _ = sqlx::query(&format!(
r#"
UPDATE
{}
SET
last_server_ping = NULL
WHERE
workspace_id = $1 AND
path = $2 AND
server_id IS NULL
"#,
Self::TABLE_NAME
))
.bind(&listening_trigger.workspace_id)
.bind(&listening_trigger.path)
.execute(db)
.await;
} else {
let _ = sqlx::query!(
r#"
UPDATE
capture_config
SET
last_server_ping = NULL
WHERE
workspace_id = $1 AND
path = $2 AND
is_flow = $3 AND
trigger_kind = $4 AND
server_id IS NULL
"#,
&listening_trigger.workspace_id,
&listening_trigger.path,
&listening_trigger.is_flow,
Self::TRIGGER_KIND as TriggerKind
)
.execute(db)
.await;
}
}
async fn disable_with_error(
&self,
db: &DB,
listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
error: String,
) {
if listening_trigger.trigger_mode {
let report_status = sqlx::query(&format!(
r#"
UPDATE
{}
SET
mode = 'disabled'::TRIGGER_MODE,
error = $1,
server_id = NULL,
last_server_ping = NULL
WHERE
workspace_id = $2 AND
path = $3
"#,
Self::TABLE_NAME
))
.bind(&error)
.bind(&listening_trigger.workspace_id)
.bind(&listening_trigger.path)
.execute(db)
.await;
match report_status {
Ok(_) => {
report_critical_error(
format!(
"Disabling {} trigger {} because of error: {}",
Self::TRIGGER_KIND,
listening_trigger.path,
error
),
db.clone(),
Some(&listening_trigger.workspace_id),
None,
)
.await;
}
Err(disable_err) => {
report_critical_error(
format!("Could not disable {} trigger {} with err {}, disabling because of error {}", Self::TRIGGER_KIND, listening_trigger.path, disable_err, error),
db.clone(),
Some(&listening_trigger.workspace_id),
None,
).await;
}
}
return;
}
let report_status = sqlx::query!(
r#"
UPDATE
capture_config
SET
error = $1,
server_id = NULL,
last_server_ping = NULL
WHERE
workspace_id = $2 AND
path = $3 AND
is_flow = $4 AND
trigger_kind = $5
"#,
error,
listening_trigger.workspace_id,
listening_trigger.path,
listening_trigger.is_flow,
Self::TRIGGER_KIND as TriggerKind
)
.execute(db)
.await;
if let Err(disable_err) = report_status {
tracing::error!(
"Could not disable {} capture {} ({}) with err {}, disabling because of error {}",
Self::TRIGGER_KIND,
listening_trigger.path,
listening_trigger.workspace_id,
disable_err,
error
)
}
}
async fn handle_trigger(
&self,
db: &DB,
listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
payload: Self::Payload,
trigger_info: HashMap<String, Box<RawValue>>,
_extra: Option<Self::Extra>,
) -> Result<()> {
let args = Self::build_job_args(
&listening_trigger.script_path,
listening_trigger.is_flow,
&listening_trigger.workspace_id,
db,
payload,
trigger_info,
)
.await?;
let authed = listening_trigger
.authed(db, &Self::TRIGGER_KIND.to_string())
.await?;
let (retry, error_handler_path, error_handler_args) =
match listening_trigger.error_handling.as_ref() {
Some(error_handling) => (
error_handling.retry.as_ref(),
error_handling.error_handler_path.as_deref(),
error_handling.error_handler_args.as_ref(),
),
None => (None, None, None),
};
tracing::debug!(
"Triggering job from {} event {} with args {:?}",
Self::TRIGGER_KIND,
listening_trigger.path,
args
);
trigger_runnable(
db,
None,
authed,
&listening_trigger.workspace_id,
&listening_trigger.script_path,
listening_trigger.is_flow,
args,
retry,
error_handler_path.as_deref(),
error_handler_args,
format!("{}_trigger/{}", Self::TRIGGER_KIND, listening_trigger.path),
None,
listening_trigger.suspended_mode,
TriggerMetadata::new(Some(listening_trigger.path.clone()), Self::JOB_TRIGGER_KIND),
)
.await?;
Ok(())
}
async fn handle_event(
&self,
db: &DB,
listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
payload: Self::Payload,
trigger_info: HashMap<String, Box<RawValue>>,
extra: Option<Self::Extra>,
) -> Result<()> {
if listening_trigger.trigger_mode {
if let Err(err) = self
.handle_trigger(db, listening_trigger, payload, trigger_info, extra)
.await
{
report_critical_error(
format!(
"Failed to trigger job from {} event {}: {:?}",
Self::TRIGGER_KIND,
listening_trigger.path,
err
),
db.clone(),
Some(&listening_trigger.workspace_id),
None,
)
.await;
return Err(err);
};
return Ok(());
}
let (main_args, preprocessor_args) = Self::build_capture_payloads(&payload, trigger_info);
if let Err(err) = insert_capture_payload(
db,
&listening_trigger.workspace_id,
&listening_trigger.path,
listening_trigger.is_flow,
&Self::TRIGGER_KIND,
main_args,
preprocessor_args,
&listening_trigger.username,
)
.await
{
tracing::error!("Error inserting capture payload: {:?}", err);
return Err(err);
}
Ok(())
}
}
#[allow(unused)]
async fn listening<T: Listener>(
db: DB,
listener: T,
listening_trigger: ListeningTrigger<T::TriggerConfig>,
mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
) {
let killpill_rx_consumer = killpill_rx.resubscribe();
let killpill_rx_get_consumer = killpill_rx.resubscribe();
let loop_ping_status = Arc::new(RwLock::new(None));
let extra_state = listener.get_extra_state().await;
let path = listening_trigger.path.clone();
tokio::select! {
biased;
_ = killpill_rx.recv() => {
let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await;
}
_ = listener.loop_ping(&db, &listening_trigger, loop_ping_status.clone(), Some("Connecting...".to_string())) => {
let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await;
}
consumer = {
tracing::info!("[{}] Getting consumer for trigger {}", T::TRIGGER_KIND, path);
listener.get_consumer(&db, &listening_trigger, loop_ping_status.clone(), killpill_rx_get_consumer)
} => {
tokio::select! {
biased;
_ = killpill_rx.recv() => {
tracing::info!("[{}] Killing pill received, stopping consumer for trigger {}", T::TRIGGER_KIND, path);
let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await;
return;
}
_ = listener.loop_ping(&db, &listening_trigger, loop_ping_status.clone(), None) => {
tracing::info!("[{}] Loop ping exited, stopping consumer for trigger {}", T::TRIGGER_KIND, path);
let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await;
return;
}
_ = async {
match consumer {
Ok(Some(consumer)) => {
listener.update_ping_and_loop_ping_status(&db, &listening_trigger, loop_ping_status.clone(), None).await;
tracing::info!("[{}] Starting consumer for trigger {}", T::TRIGGER_KIND, path);
listener.consume(&db, consumer, &listening_trigger, loop_ping_status.clone(), killpill_rx_consumer, extra_state.as_ref()).await;
tracing::info!("[{}] Consumer stopped for trigger {}", T::TRIGGER_KIND, path);
}
Err(error) => {
tracing::error!("[{}] Disabling trigger {} due to consumer error: {}", T::TRIGGER_KIND, path, error);
listener.disable_with_error(&db, &listening_trigger, error.to_string()).await;
}
Ok(None) => {
tracing::error!("[{}] Consumer is None for trigger {}", T::TRIGGER_KIND, path);
}
}
} => {
let _ = listener.cleanup(&db, &listening_trigger, extra_state.as_ref()).await;
return;
}
}
}
}
}
#[allow(unused)]
async fn listen_to_unlistened_events<T: Copy + Listener>(
listener: T,
db: DB,
killpill_rx: &tokio::sync::broadcast::Receiver<()>,
) {
let unlistend_enabled_triggers = listener.fetch_enabled_unlistened_triggers(&db).await;
match unlistend_enabled_triggers {
Ok(mut unlistend_enabled_triggers) => {
unlistend_enabled_triggers.shuffle(&mut rand::rng());
for trigger in unlistend_enabled_triggers {
let has_lock = sqlx::query_scalar(&format!(
r#"
UPDATE
{}
SET
server_id = $1,
last_server_ping = now(),
error = 'Connecting...'
WHERE
(mode = 'enabled'::TRIGGER_MODE OR mode = 'suspended'::TRIGGER_MODE)
AND workspace_id = $2
AND path = $3
AND (last_server_ping IS NULL
OR last_server_ping < now() - INTERVAL '15 seconds'
)
RETURNING true
"#,
T::TABLE_NAME,
))
.bind(&*INSTANCE_NAME)
.bind(&trigger.workspace_id)
.bind(&trigger.path)
.fetch_optional(&db)
.await;
match has_lock {
Ok(has_lock) => {
if has_lock.flatten().unwrap_or(false) {
tracing::info!(
"Spawning new task to listen for {} event",
T::TABLE_NAME
);
tokio::spawn({
let db = db.clone();
let killpill_rx = killpill_rx.resubscribe();
async move { listening(db, listener, trigger, killpill_rx).await }
});
} else {
tracing::info!(
"{} trigger {} already being listened to",
T::TRIGGER_KIND,
trigger.path
);
}
}
Err(err) => {
tracing::error!(
"Error acquiring lock for {} trigger {}: {:?}",
T::TRIGGER_KIND,
trigger.path,
err
);
}
};
}
}
Err(err) => {
tracing::error!("Error fetching {} triggers: {:?}", T::TRIGGER_KIND, err,);
}
}
let unlisted_captures = listener.fetch_unlistened_captures(&db).await;
match unlisted_captures {
Ok(unlistened_captures) => {
for capture in unlistened_captures {
let has_lock = sqlx::query_scalar!(
r#"
UPDATE
capture_config
SET
server_id = $1,
last_server_ping = now(),
error = 'Connecting...'
WHERE
last_client_ping > NOW() - INTERVAL '10 seconds' AND
workspace_id = $2 AND
path = $3 AND
is_flow = $4 AND
trigger_kind = $5 AND
(last_server_ping IS NULL OR last_server_ping < now() - interval '15 seconds')
RETURNING true
"#,
*INSTANCE_NAME,
&capture.workspace_id,
&capture.path,
&capture.is_flow,
T::TRIGGER_KIND as TriggerKind
)
.fetch_optional(&db)
.await;
match has_lock {
Ok(has_lock) => {
if has_lock.flatten().unwrap_or(false) {
tokio::spawn({
let db = db.clone();
let killpill_rx = killpill_rx.resubscribe();
async move { listening(db, listener, capture, killpill_rx).await }
});
} else {
tracing::info!(
"{} capture {} already being listened to",
T::TRIGGER_KIND.to_string(),
capture.path
);
}
}
Err(err) => {
tracing::error!(
"Error acquiring lock for capture {} {}: {:?}",
T::TRIGGER_KIND,
capture.path,
err
);
}
};
}
}
Err(err) => {
tracing::error!(
"Error fetching captures {} triggers: {:?}",
T::TRIGGER_KIND,
err
);
}
}
}
#[derive(Debug, Clone, Deserialize, Serialize)]
struct Capture<T>
where
T: for<'r> FromRow<'r, sqlx::postgres::PgRow>,
{
path: String,
is_flow: bool,
workspace_id: String,
username: String,
email: String,
#[serde(flatten)]
trigger_config: T,
}
impl<T> FromRow<'_, sqlx::postgres::PgRow> for Capture<T>
where
T: for<'r> FromRow<'r, sqlx::postgres::PgRow> + DeserializeOwned,
{
fn from_row(row: &sqlx::postgres::PgRow) -> std::result::Result<Self, sqlx::Error> {
let trigger_config_value = row.try_get("trigger_config")?;
let trigger_config: T = serde_json::from_value(trigger_config_value)
.map_err(|e| sqlx::Error::Decode(Box::new(e)))?;
Ok(Capture {
path: row.try_get("path")?,
is_flow: row.try_get("is_flow")?,
workspace_id: row.try_get("workspace_id")?,
username: row.try_get("username")?,
email: row.try_get("email")?,
trigger_config,
})
}
}
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct ListeningTrigger<T> {
pub path: String,
pub is_flow: bool,
pub workspace_id: String,
pub username: String,
pub email: String,
pub trigger_config: T,
pub script_path: String,
pub trigger_mode: bool,
pub error_handling: Option<TriggerErrorHandling>,
pub suspended_mode: bool,
}
impl<T> ListeningTrigger<T> {
pub async fn authed(&self, db: &DB, username: &str) -> Result<ApiAuthed> {
fetch_api_authed(
self.username.clone(),
self.email.clone(),
&self.workspace_id,
db,
Some(format!("{}-{}", username, self.path)),
)
.await
}
}
#[allow(unused)]
pub async fn update_rw_lock<T>(lock: std::sync::Arc<tokio::sync::RwLock<T>>, value: T) -> () {
let mut w = lock.write().await;
*w = value;
}
#[allow(unused)]
fn listen_to<T: Copy + Listener>(
trigger: T,
db: DB,
mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
) {
tokio::spawn(async move {
listen_to_unlistened_events(trigger, db.clone(), &killpill_rx).await;
loop {
tokio::select! {
biased;
_ = killpill_rx.recv() => {
return;
}
_ = tokio::time::sleep(tokio::time::Duration::from_secs(15)) => {
listen_to_unlistened_events(trigger, db.clone(), &killpill_rx).await
}
}
}
});
}
#[allow(unused)]
pub fn start_all_listeners(db: DB, killpill_rx: &tokio::sync::broadcast::Receiver<()>) {
tracing::info!("Starting trigger listeners based on available features...");
#[cfg(feature = "postgres_trigger")]
{
let postgres_killpill_rx = killpill_rx.resubscribe();
use crate::triggers::postgres::PostgresTrigger;
listen_to(PostgresTrigger, db.clone(), postgres_killpill_rx)
}
#[cfg(feature = "mqtt_trigger")]
{
let mqtt_killpill_rx = killpill_rx.resubscribe();
use crate::triggers::mqtt::MqttTrigger;
listen_to(MqttTrigger, db.clone(), mqtt_killpill_rx)
}
#[cfg(feature = "websocket")]
{
let mqtt_killpill_rx = killpill_rx.resubscribe();
use crate::triggers::websocket::WebsocketTrigger;
listen_to(WebsocketTrigger, db.clone(), mqtt_killpill_rx)
}
#[cfg(all(feature = "gcp_trigger", feature = "enterprise", feature = "private"))]
{
let gcp_killpill_rx = killpill_rx.resubscribe();
use crate::triggers::gcp::GcpTrigger;
listen_to(GcpTrigger, db.clone(), gcp_killpill_rx);
}
#[cfg(all(feature = "sqs_trigger", feature = "enterprise", feature = "private"))]
{
let gcp_killpill_rx = killpill_rx.resubscribe();
use crate::triggers::sqs::SqsTrigger;
listen_to(SqsTrigger, db.clone(), gcp_killpill_rx);
}
#[cfg(all(feature = "nats", feature = "enterprise", feature = "private"))]
{
let gcp_killpill_rx = killpill_rx.resubscribe();
use crate::triggers::nats::NatsTrigger;
listen_to(NatsTrigger, db.clone(), gcp_killpill_rx);
}
#[cfg(all(feature = "kafka", feature = "enterprise", feature = "private"))]
{
let gcp_killpill_rx = killpill_rx.resubscribe();
use crate::triggers::kafka::KafkaTrigger;
listen_to(KafkaTrigger, db.clone(), gcp_killpill_rx);
}
tracing::info!("All available trigger listeners have been started");
}
backend/windmill-api/src/triggers/mod.rs
-182
View File
@@ -1,182 +0,0 @@
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use sqlx::{types::Json as SqlxJson, FromRow};
use std::{collections::HashMap, fmt::Debug};
use windmill_common::jobs::JobTriggerKind;
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum HandlerAction {
Trigger { path: String, trigger_kind: JobTriggerKind },
// Future variants can be added here (e.g., Script, Flow, etc.)
}
#[cfg(all(feature = "smtp", feature = "private"))]
pub mod email;
#[cfg(all(feature = "gcp_trigger", feature = "enterprise", feature = "private"))]
pub mod gcp;
#[cfg(feature = "http_trigger")]
pub mod http;
#[cfg(all(feature = "kafka", feature = "enterprise", feature = "private"))]
pub mod kafka;
#[cfg(feature = "mqtt_trigger")]
pub mod mqtt;
#[cfg(all(feature = "nats", feature = "enterprise", feature = "private"))]
pub mod nats;
#[cfg(feature = "postgres_trigger")]
pub mod postgres;
#[cfg(all(feature = "sqs_trigger", feature = "enterprise", feature = "private"))]
pub mod sqs;
#[cfg(feature = "websocket")]
pub mod websocket;
pub mod filter;
pub mod global_handler;
mod handler;
mod listener;
pub mod trigger_helpers;
#[allow(unused)]
pub(crate) use handler::TriggerCrud;
pub use handler::{generate_trigger_routers, get_triggers_count_internal, TriggersCount};
pub use listener::start_all_listeners;
#[allow(unused)]
pub(crate) use listener::Listener;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StandardTriggerQuery {
pub page: Option<usize>,
pub per_page: Option<usize>,
pub path: Option<String>,
pub is_flow: Option<bool>,
pub path_start: Option<String>,
}
#[derive(Debug, FromRow, Clone, Serialize, Deserialize)]
pub struct BaseTrigger {
pub workspace_id: String,
pub path: String,
pub script_path: String,
pub mode: TriggerMode,
pub is_flow: bool,
pub edited_by: String,
pub email: String,
pub edited_at: DateTime<Utc>,
pub extra_perms: Option<serde_json::Value>,
}
#[derive(Debug, FromRow, Clone, Serialize, Deserialize)]
pub struct ServerState {
#[serde(skip_serializing_if = "Option::is_none")]
pub server_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub last_server_ping: Option<DateTime<Utc>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub error: Option<String>,
}
#[derive(Debug, FromRow, Clone, Serialize, Deserialize)]
pub struct TriggerErrorHandling {
#[serde(skip_serializing_if = "Option::is_none")]
pub error_handler_path: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub error_handler_args: Option<SqlxJson<HashMap<String, serde_json::Value>>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub retry: Option<sqlx::types::Json<windmill_common::flows::Retry>>,
}
#[derive(Serialize, Deserialize, Clone)]
pub struct Trigger<T>
where
T: for<'r> FromRow<'r, sqlx::postgres::PgRow>,
{
#[serde(flatten)]
pub base: BaseTrigger,
#[serde(flatten)]
pub config: T,
#[serde(flatten)]
pub server_state: Option<ServerState>,
#[serde(flatten)]
pub error_handling: TriggerErrorHandling,
}
impl<T> FromRow<'_, sqlx::postgres::PgRow> for Trigger<T>
where
T: for<'r> FromRow<'r, sqlx::postgres::PgRow>,
{
fn from_row(row: &sqlx::postgres::PgRow) -> std::result::Result<Self, sqlx::Error> {
let base = BaseTrigger::from_row(row)?;
Ok(Trigger {
base,
config: T::from_row(row)?,
server_state: ServerState::from_row(row).ok(),
error_handling: TriggerErrorHandling::from_row(row)?,
})
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BaseTriggerData {
pub path: String,
pub script_path: String,
pub is_flow: bool,
#[deprecated(note = "Use mode instead")]
enabled: Option<bool>, // Kept for backwards compatibility, use mode instead
mode: Option<TriggerMode>,
}
impl BaseTriggerData {
pub fn mode(&self) -> &TriggerMode {
self.mode.as_ref().unwrap_or(
#[allow(deprecated)]
if self.enabled.unwrap_or(true) {
&TriggerMode::Enabled
} else {
&TriggerMode::Disabled
},
)
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TriggerData<T: Debug> {
#[serde(flatten)]
pub base: BaseTriggerData,
#[serde(flatten)]
pub config: T,
#[serde(flatten)]
pub error_handling: TriggerErrorHandling,
}
impl StandardTriggerQuery {
pub fn offset(&self) -> i64 {
let page = self.page.unwrap_or(0);
let per_page = self.per_page.unwrap_or(100);
(page * per_page) as i64
}
pub fn limit(&self) -> i64 {
self.per_page.unwrap_or(100) as i64
}
}
impl Default for StandardTriggerQuery {
fn default() -> Self {
Self { page: Some(0), per_page: Some(100), path: None, path_start: None, is_flow: None }
}
}
#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Clone)]
#[sqlx(type_name = "TRIGGER_MODE", rename_all = "lowercase")]
#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))]
pub enum TriggerMode {
Enabled,
Disabled,
Suspended,
}
backend/windmill-api/src/triggers/mqtt/handler.rs
-237
View File
@@ -1,237 +0,0 @@
use crate::{
db::{ApiAuthed, DB},
resources::try_get_resource_from_db_as,
triggers::{Trigger, TriggerCrud, TriggerData},
};
use axum::async_trait;
use itertools::Itertools;
use sqlx::{types::Json as SqlxJson, PgConnection};
use windmill_common::{
db::UserDB,
error::{Error, Result},
};
use windmill_git_sync::DeployedObject;
use super::{
MqttClientBuilder, MqttClientVersion, MqttConfig, MqttConfigRequest, MqttResource, MqttTrigger,
MqttV3Config, MqttV5Config, SubscribeTopic, TestMqttConfig,
};
#[async_trait]
impl TriggerCrud for MqttTrigger {
type TriggerConfig = MqttConfig;
type Trigger = Trigger<Self::TriggerConfig>;
type TriggerConfigRequest = MqttConfigRequest;
type TestConnectionConfig = TestMqttConfig;
const TABLE_NAME: &'static str = "mqtt_trigger";
const TRIGGER_TYPE: &'static str = "mqtt";
const SUPPORTS_SERVER_STATE: bool = true;
const SUPPORTS_TEST_CONNECTION: bool = true;
const ROUTE_PREFIX: &'static str = "/mqtt_triggers";
const DEPLOYMENT_NAME: &'static str = "MQTT trigger";
const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[
"mqtt_resource_path",
"subscribe_topics",
"v3_config",
"v5_config",
"client_id",
"client_version",
];
const IS_ALLOWED_ON_CLOUD: bool = false;
fn get_deployed_object(path: String) -> DeployedObject {
DeployedObject::MqttTrigger { path }
}
async fn validate_config(
&self,
_db: &DB,
config: &Self::TriggerConfigRequest,
_workspace_id: &str,
) -> Result<()> {
if config.mqtt_resource_path.trim().is_empty() {
return Err(Error::BadRequest(
"MQTT resource path cannot be empty".to_string(),
));
}
if config.subscribe_topics.is_empty() {
return Err(Error::BadRequest(
"At least one subscribe topic must be specified".to_string(),
));
}
Ok(())
}
async fn create_trigger(
&self,
_db: &DB,
tx: &mut PgConnection,
authed: &ApiAuthed,
w_id: &str,
trigger: TriggerData<Self::TriggerConfigRequest>,
) -> Result<()> {
let subscribe_topics = trigger
.config
.subscribe_topics
.into_iter()
.map(SqlxJson)
.collect_vec();
let v3_config = trigger.config.v3_config.map(SqlxJson);
let v5_config = trigger.config.v5_config.map(SqlxJson);
sqlx::query!(
r#"
INSERT INTO mqtt_trigger (
mqtt_resource_path,
subscribe_topics,
client_version,
client_id,
v3_config,
v5_config,
workspace_id,
path,
script_path,
is_flow,
email,
mode,
edited_by,
error_handler_path,
error_handler_args,
retry
)
VALUES (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16
)"#,
trigger.config.mqtt_resource_path,
subscribe_topics.as_slice() as &[SqlxJson<SubscribeTopic>],
trigger.config.client_version as Option<MqttClientVersion>,
trigger.config.client_id,
v3_config as Option<SqlxJson<MqttV3Config>>,
v5_config as Option<SqlxJson<MqttV5Config>>,
w_id,
trigger.base.path,
trigger.base.script_path,
trigger.base.is_flow,
authed.email,
trigger.base.mode() as _,
authed.username,
trigger.error_handling.error_handler_path,
trigger.error_handling.error_handler_args as _,
trigger.error_handling.retry as _
)
.execute(tx)
.await?;
Ok(())
}
async fn update_trigger(
&self,
_db: &DB,
tx: &mut PgConnection,
authed: &ApiAuthed,
workspace_id: &str,
path: &str,
trigger: TriggerData<Self::TriggerConfigRequest>,
) -> Result<()> {
let subscribe_topics = trigger
.config
.subscribe_topics
.into_iter()
.map(SqlxJson)
.collect_vec();
let v3_config = trigger.config.v3_config.map(SqlxJson);
let v5_config = trigger.config.v5_config.map(SqlxJson);
// Important to set server_id to NULL to stop current mqtt listener
sqlx::query!(
r#"
UPDATE
mqtt_trigger
SET
mqtt_resource_path = $1,
subscribe_topics = $2,
client_version = $3,
client_id = $4,
v3_config = $5,
v5_config = $6,
is_flow = $7,
edited_by = $8,
email = $9,
script_path = $10,
path = $11,
edited_at = now(),
error = NULL,
server_id = NULL,
error_handler_path = $14,
error_handler_args = $15,
retry = $16
WHERE
workspace_id = $12 AND
path = $13
"#,
trigger.config.mqtt_resource_path,
subscribe_topics.as_slice() as &[SqlxJson<SubscribeTopic>],
trigger.config.client_version as Option<MqttClientVersion>,
trigger.config.client_id,
v3_config as Option<SqlxJson<MqttV3Config>>,
v5_config as Option<SqlxJson<MqttV5Config>>,
trigger.base.is_flow,
authed.username,
authed.email,
trigger.base.script_path,
trigger.base.path,
workspace_id,
path,
trigger.error_handling.error_handler_path,
trigger.error_handling.error_handler_args as _,
trigger.error_handling.retry as _
)
.execute(tx)
.await?;
Ok(())
}
async fn test_connection(
&self,
db: &DB,
authed: &ApiAuthed,
user_db: &UserDB,
workspace_id: &str,
config: Self::TestConnectionConfig,
) -> Result<()> {
let mqtt_resource = try_get_resource_from_db_as::<MqttResource>(
authed,
Some(user_db.clone()),
db,
&config.mqtt_resource_path,
workspace_id,
)
.await?;
let connect_f = async {
let client_builder = MqttClientBuilder::new(
mqtt_resource,
Some(""),
vec![],
config.v3_config.as_ref(),
config.v5_config.as_ref(),
config.client_version.as_ref(),
);
client_builder.build_client().await.map_err(|err| {
Error::BadConfig(format!(
"Error connecting to mqtt broker: {}",
err.to_string()
))
})
};
connect_f.await?;
Ok(())
}
}
backend/windmill-api/src/triggers/mqtt/listener.rs
-376
View File
@@ -1,376 +0,0 @@
use std::{collections::HashMap, sync::Arc};
use async_trait::async_trait;
use bytes::Bytes;
use rumqttc::{
v5::{
mqttbytes::v5::PublishProperties, Event as V5Event, EventLoop as V5EventLoop,
Incoming as V5Incoming,
},
Event as V3Event, EventLoop as V3EventLoop, Incoming as V3Incoming,
};
use std::time::Duration;
use tokio::sync::RwLock;
use windmill_common::{
db::UserDB,
error::{to_anyhow, Error, Result},
jobs::JobTriggerKind,
worker::to_raw_value,
DB,
};
use crate::{
resources::try_get_resource_from_db_as,
triggers::{
listener::ListeningTrigger,
mqtt::{
MqttClientBuilder, MqttClientResult, MqttConfig, MqttResource, MqttTrigger,
V3MqttHandler, V5MqttHandler,
},
trigger_helpers::TriggerJobArgs,
Listener,
},
};
#[async_trait]
impl Listener for MqttTrigger {
type Consumer = MqttClientResult;
type Extra = ();
type ExtraState = ();
const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Mqtt;
async fn get_consumer(
&self,
db: &DB,
listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
_err_message: Arc<RwLock<Option<String>>>,
_killpill_rx: tokio::sync::broadcast::Receiver<()>,
) -> Result<Option<Self::Consumer>> {
let ListeningTrigger::<Self::TriggerConfig> { workspace_id, trigger_config, .. } =
listening_trigger;
let MqttConfig {
mqtt_resource_path,
subscribe_topics,
v3_config,
v5_config,
client_id,
client_version,
..
} = trigger_config;
let authed = listening_trigger
.authed(db, &Self::TRIGGER_KIND.to_string())
.await?;
let mqtt_resource = try_get_resource_from_db_as::<MqttResource>(
&authed,
Some(UserDB::new(db.clone())),
&db,
mqtt_resource_path,
workspace_id,
)
.await?;
let subscribe_topics = subscribe_topics
.iter()
.map(|topic| topic.0.clone())
.collect();
let client_builder = MqttClientBuilder::new(
mqtt_resource,
client_id.as_deref(),
subscribe_topics,
v3_config.as_ref().map(|c| &c.0),
v5_config.as_ref().map(|c| &c.0),
client_version.as_ref(),
);
let client_result = client_builder
.build_client()
.await
.map_err(|e| Error::BadConfig(format!("Failed to build MQTT client: {}", e)))?;
Ok(Some(client_result))
}
async fn consume(
&self,
db: &DB,
consumer: Self::Consumer,
listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
_err_message: Arc<RwLock<Option<String>>>,
_killpill_rx: tokio::sync::broadcast::Receiver<()>,
_extra_state: Option<&Self::ExtraState>,
) {
tracing::info!(
"Starting to listen for MQTT trigger {}",
&listening_trigger.path
);
match consumer {
MqttClientResult::V3((v3_handler, event_loop)) => {
handle_event(&db, self, listening_trigger, v3_handler, event_loop).await
}
MqttClientResult::V5((v5_handler, event_loop)) => {
handle_event(&db, self, listening_trigger, v5_handler, event_loop).await
}
}
}
}
const TIMEOUT_DURATION: u64 = 10;
const CONNECTION_TIMEOUT: Duration = Duration::from_secs(TIMEOUT_DURATION);
fn convert_disconnect_packet_into_string(
disconnect: rumqttc::v5::mqttbytes::v5::Disconnect,
) -> String {
let err_message = disconnect
.properties
.map(|properties| properties.reason_string)
.flatten();
let reason_code = disconnect.reason_code as u8;
format!(
"Disconnected by the broker, reason code: {}, {}",
reason_code,
err_message
.map(|err| format!("message: {}", err))
.unwrap_or("".to_string())
)
}
#[async_trait]
pub trait EventLoop {
type Event;
type Error;
async fn poll(&mut self) -> Result<Self::Event>;
async fn verify_connection(&mut self) -> Result<()>;
}
#[async_trait]
impl EventLoop for V5EventLoop {
type Event = V5Event;
type Error = rumqttc::v5::ConnectionError;
async fn poll(&mut self) -> Result<Self::Event> {
self.poll().await.map_err(|err| to_anyhow(err).into())
}
async fn verify_connection(&mut self) -> Result<()> {
let start = std::time::Instant::now();
while start.elapsed() < CONNECTION_TIMEOUT {
match self.poll().await.map_err(to_anyhow)? {
Self::Event::Incoming(V5Incoming::ConnAck(_)) => return Ok(()),
Self::Event::Incoming(V5Incoming::Disconnect(disconnect)) => {
return Err(Error::BadConfig(convert_disconnect_packet_into_string(
disconnect,
)));
}
_ => continue,
}
}
Err(Error::BadConfig(format!(
"Timeout occurred while trying to connect to mqtt broker after {} seconds",
TIMEOUT_DURATION
)))
}
}
#[async_trait]
impl EventLoop for V3EventLoop {
type Event = V3Event;
type Error = rumqttc::ConnectionError;
async fn poll(&mut self) -> Result<Self::Event> {
self.poll().await.map_err(|err| to_anyhow(err).into())
}
async fn verify_connection(&mut self) -> Result<()> {
let start = std::time::Instant::now();
while start.elapsed() < CONNECTION_TIMEOUT {
match self.poll().await.map_err(to_anyhow)? {
Self::Event::Incoming(rumqttc::Packet::ConnAck(_)) => return Ok(()),
_ => continue,
}
}
Err(Error::BadConfig(format!(
"Timeout occurred while trying to connect to mqtt broker after {} seconds",
TIMEOUT_DURATION
)))
}
}
async fn handle_event<T, E, H>(
db: &DB,
listener: &T,
listening_trigger: &ListeningTrigger<T::TriggerConfig>,
handler: H,
mut event_loop: E,
) -> ()
where
T: Listener,
H: MqttEvent,
E: EventLoop<Event = H::Event>,
E::Error: ToString,
<T as TriggerJobArgs>::Payload: From<Bytes>,
{
loop {
let event = event_loop.poll().await;
match event {
Ok(event) => {
let publish_data = handler.handle_event(event);
if let Ok(Some((payload, publish_data))) = publish_data {
let trigger_info = HashMap::from([
("topic".to_string(), to_raw_value(&publish_data.topic)),
("retain".to_string(), to_raw_value(&publish_data.retain)),
("pkid".to_string(), to_raw_value(&publish_data.pkid)),
("qos".to_string(), to_raw_value(&publish_data.qos)),
(
"v5".to_string(),
to_raw_value(&publish_data.v5.map(|properties| {
serde_json::json!({
"payload_format_indicator": properties.payload_format_indicator,
"topic_alias": properties.topic_alias,
"response_topic": properties.response_topic,
"correlation_data": properties.correlation_data.as_deref(),
"user_properties": properties.user_properties,
"subscription_identifiers": properties.subscription_identifiers,
"content_type": properties.content_type,
})
})),
),
]);
let _ = listener
.handle_event(db, listening_trigger, payload.into(), trigger_info, None)
.await;
}
}
Err(err) => {
let error = err.to_string();
tracing::debug!("Error: {}", &err);
listener
.disable_with_error(db, listening_trigger, error)
.await;
return;
}
}
}
}
#[derive(Clone)]
#[allow(unused)]
pub struct PublishData {
topic: String,
retain: bool,
pkid: u16,
v5: Option<PublishProperties>,
qos: u8,
}
impl PublishData {
pub fn new(
topic: String,
retain: bool,
pkid: u16,
v5: Option<PublishProperties>,
qos: u8,
) -> PublishData {
PublishData { topic, retain, pkid, v5, qos }
}
}
trait MqttEvent {
type IncomingPacket;
type PublishPacket;
type Event;
fn handle_publish_packet(publish_packet: Self::PublishPacket) -> PublishData;
fn handle_event(&self, event: Self::Event) -> Result<Option<(Bytes, PublishData)>>;
}
impl MqttEvent for V5MqttHandler {
type IncomingPacket = V5Incoming;
type PublishPacket = rumqttc::v5::mqttbytes::v5::Publish;
type Event = V5Event;
fn handle_publish_packet(publish_packet: Self::PublishPacket) -> PublishData {
PublishData::new(
String::from_utf8(publish_packet.topic.as_ref().to_vec()).unwrap_or("".to_string()),
publish_packet.retain,
publish_packet.pkid,
publish_packet.properties,
publish_packet.qos as u8,
)
}
fn handle_event(&self, event: Self::Event) -> Result<Option<(Bytes, PublishData)>> {
tracing::debug!("Inside V5 event");
match event {
Self::Event::Incoming(packet) => match packet {
Self::IncomingPacket::Publish(publish_packet) => {
return Ok(Some((
publish_packet.payload.clone(),
Self::handle_publish_packet(publish_packet),
)))
}
Self::IncomingPacket::Disconnect(disconnect) => {
return Err(
anyhow::anyhow!(convert_disconnect_packet_into_string(disconnect)).into(),
);
}
packet => {
tracing::debug!("Received = {:#?}", packet);
}
},
Self::Event::Outgoing(packet) => {
tracing::debug!("Outgoing Received = {:#?}", packet);
}
}
Ok(None)
}
}
impl MqttEvent for V3MqttHandler {
type IncomingPacket = V3Incoming;
type PublishPacket = rumqttc::mqttbytes::v4::Publish;
type Event = V3Event;
fn handle_publish_packet(publish_packet: Self::PublishPacket) -> PublishData {
PublishData::new(
publish_packet.topic,
publish_packet.retain,
publish_packet.pkid,
None,
publish_packet.qos as u8,
)
}
fn handle_event(&self, event: Self::Event) -> Result<Option<(Bytes, PublishData)>> {
tracing::debug!("Inside V3 event");
match event {
Self::Event::Incoming(packet) => match packet {
Self::IncomingPacket::Publish(publish_packet) => {
return Ok(Some((
publish_packet.payload.clone(),
Self::handle_publish_packet(publish_packet),
)))
}
packet => {
tracing::debug!("Received = {:?}", packet);
}
},
Self::Event::Outgoing(packet) => {
tracing::debug!("Outgoing Received = {:?}", packet);
}
}
Ok(None)
}
}
backend/windmill-api/src/triggers/mqtt/mod.rs
-346
View File
@@ -1,346 +0,0 @@
use base64::{engine, prelude::*};
use itertools::Itertools;
use rumqttc::{
v5::{
mqttbytes::{
v5::{ConnectProperties, Filter},
QoS as V5QoS,
},
AsyncClient as V5AsyncClient, EventLoop as V5EventLoop, MqttOptions as V5MqttOptions,
},
AsyncClient as V3AsyncClient, EventLoop as V3EventLoop, MqttOptions as V3MqttOptions,
QoS as V3QoS, SubscribeFilter, TlsConfiguration, Transport,
};
use serde::{Deserialize, Serialize};
use serde_json::value::RawValue;
use sqlx::{types::Json as SqlxJson, FromRow, Type};
use std::{collections::HashMap, time::Duration};
use windmill_common::{
error::{to_anyhow, Error},
triggers::TriggerKind,
worker::to_raw_value,
};
use crate::triggers::{mqtt::listener::EventLoop, trigger_helpers::TriggerJobArgs};
pub mod handler;
pub mod listener;
#[derive(Clone, Copy)]
pub struct MqttTrigger;
impl TriggerJobArgs for MqttTrigger {
type Payload = Vec<u8>;
const TRIGGER_KIND: TriggerKind = TriggerKind::Mqtt;
fn v1_payload_fn(payload: &Self::Payload) -> HashMap<String, Box<RawValue>> {
HashMap::from([("payload".to_string(), to_raw_value(&payload))])
}
fn v2_payload_fn(payload: &Self::Payload) -> HashMap<String, Box<RawValue>> {
let base64_payload = engine::general_purpose::STANDARD.encode(payload);
HashMap::from([("payload".to_string(), to_raw_value(&base64_payload))])
}
}
#[derive(Clone, Debug, Deserialize, Serialize, Type)]
#[serde(rename_all = "lowercase")]
pub enum QualityOfService {
Qos0,
Qos1,
Qos2,
}
impl From<QualityOfService> for V3QoS {
fn from(value: QualityOfService) -> Self {
match value {
QualityOfService::Qos0 => V3QoS::AtMostOnce,
QualityOfService::Qos1 => V3QoS::AtLeastOnce,
QualityOfService::Qos2 => V3QoS::ExactlyOnce,
}
}
}
impl From<QualityOfService> for V5QoS {
fn from(value: QualityOfService) -> Self {
match value {
QualityOfService::Qos0 => V5QoS::AtMostOnce,
QualityOfService::Qos1 => V5QoS::AtLeastOnce,
QualityOfService::Qos2 => V5QoS::ExactlyOnce,
}
}
}
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct MqttV3Config {
clean_session: Option<bool>,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct MqttV5Config {
clean_start: Option<bool>,
session_expiry_interval: Option<u32>,
topic_alias_maximum: Option<u16>,
}
#[derive(Clone, Debug, Deserialize, Serialize, Type)]
#[sqlx(type_name = "MQTT_CLIENT_VERSION")]
#[sqlx(rename_all = "lowercase")]
#[serde(rename_all = "lowercase")]
pub enum MqttClientVersion {
V3,
V5,
}
#[derive(Debug, Deserialize)]
pub struct Tls {
enabled: bool,
ca_certificate: String,
pkcs12_client_certificate: Option<String>,
pkcs12_certificate_password: Option<String>,
}
#[derive(Debug, Deserialize)]
pub struct Credentials {
username: Option<String>,
password: Option<String>,
}
#[derive(Debug, Deserialize)]
pub struct MqttResource {
broker: String,
port: u16,
credentials: Option<Credentials>,
tls: Option<Tls>,
}
#[derive(Clone, Debug, FromRow, Serialize, Deserialize)]
pub struct SubscribeTopic {
qos: QualityOfService,
topic: String,
}
#[derive(Debug, Clone, FromRow, Serialize, Deserialize)]
pub struct MqttConfig {
pub mqtt_resource_path: String,
pub subscribe_topics: Vec<SqlxJson<SubscribeTopic>>,
pub v3_config: Option<SqlxJson<MqttV3Config>>,
pub v5_config: Option<SqlxJson<MqttV5Config>>,
pub client_id: Option<String>,
pub client_version: Option<MqttClientVersion>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MqttConfigRequest {
pub mqtt_resource_path: String,
pub subscribe_topics: Vec<SubscribeTopic>,
pub v3_config: Option<MqttV3Config>,
pub v5_config: Option<MqttV5Config>,
pub client_id: Option<String>,
pub client_version: Option<MqttClientVersion>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TestMqttConfig {
pub mqtt_resource_path: String,
pub client_version: Option<MqttClientVersion>,
pub v3_config: Option<MqttV3Config>,
pub v5_config: Option<MqttV5Config>,
}
// Constants
pub const KEEP_ALIVE: u64 = 60;
pub const CLIENT_CONNECTION_TIMEOUT: u64 = 60;
pub const TOPIC_ALIAS_MAXIMUM: u16 = 65535;
pub const TIMEOUT_DURATION: u64 = 10;
pub const CONNECTION_TIMEOUT: Duration = Duration::from_secs(TIMEOUT_DURATION);
pub struct V3MqttHandler;
pub struct V5MqttHandler;
pub enum MqttClientResult {
V3((V3MqttHandler, V3EventLoop)),
V5((V5MqttHandler, V5EventLoop)),
}
#[derive(Debug, thiserror::Error)]
pub enum MqttError {
#[error("{0}")]
Common(#[from] Error),
#[error("{0}")]
V5RumqttClient(#[from] rumqttc::v5::ClientError),
#[error("{0}")]
V5ConnectionError(#[from] rumqttc::v5::ConnectionError),
#[error("{0}")]
V3RumqttClient(#[from] rumqttc::ClientError),
#[error("{0}")]
V3ConnectionError(#[from] rumqttc::ConnectionError),
#[error("{0}")]
Base64Decode(#[from] base64::DecodeError),
}
pub struct MqttClientBuilder<'client> {
mqtt_resource: MqttResource,
client_id: &'client str,
subscribe_topics: Vec<SubscribeTopic>,
v3_config: Option<&'client MqttV3Config>,
v5_config: Option<&'client MqttV5Config>,
mqtt_client_version: Option<&'client MqttClientVersion>,
}
impl<'client> MqttClientBuilder<'client> {
pub fn new(
mqtt_resource: MqttResource,
client_id: Option<&'client str>,
subscribe_topics: Vec<SubscribeTopic>,
v3_config: Option<&'client MqttV3Config>,
v5_config: Option<&'client MqttV5Config>,
mqtt_client_version: Option<&'client MqttClientVersion>,
) -> Self {
Self {
mqtt_resource,
client_id: client_id.unwrap_or(""),
subscribe_topics,
v3_config,
v5_config,
mqtt_client_version,
}
}
pub async fn build_client(&self) -> Result<MqttClientResult, Error> {
match self.mqtt_client_version {
Some(MqttClientVersion::V5) | None => self.build_v5_client().await,
Some(MqttClientVersion::V3) => self.build_v3_client().await,
}
}
fn get_tls_configuration(&self) -> Result<Option<Transport>, Error> {
let transport = match self.mqtt_resource.tls {
Some(ref tls) if tls.enabled => {
let transport = match tls.ca_certificate.trim().is_empty() {
true => rumqttc::Transport::Tls(TlsConfiguration::Native),
false => rumqttc::Transport::Tls(TlsConfiguration::SimpleNative {
ca: tls.ca_certificate.as_bytes().to_vec(),
client_auth: {
match tls.pkcs12_client_certificate.as_ref() {
Some(client_certificate)
if !client_certificate.trim().is_empty() =>
{
let client_certificate = BASE64_STANDARD
.decode(client_certificate)
.map_err(to_anyhow)?;
let password = tls
.pkcs12_certificate_password
.clone()
.unwrap_or("".to_string());
Some((client_certificate, password))
}
_ => None,
}
},
}),
};
Some(transport)
}
_ => None,
};
Ok(transport)
}
async fn build_v5_client(&self) -> Result<MqttClientResult, Error> {
let mut mqtt_options = V5MqttOptions::new(
self.client_id,
&self.mqtt_resource.broker,
self.mqtt_resource.port,
);
if let Some(credentials) = &self.mqtt_resource.credentials {
let username = credentials.username.as_deref().unwrap_or("");
let password = credentials.password.as_deref().unwrap_or("");
mqtt_options.set_credentials(username, password);
}
if let Some(transport) = self.get_tls_configuration()? {
mqtt_options.set_transport(transport);
}
mqtt_options.set_connection_timeout(CLIENT_CONNECTION_TIMEOUT);
mqtt_options.set_keep_alive(Duration::from_secs(KEEP_ALIVE));
if let Some(v5_config) = self.v5_config {
mqtt_options.set_clean_start(v5_config.clean_start.unwrap_or(true));
mqtt_options.set_connect_properties(ConnectProperties {
session_expiry_interval: v5_config.session_expiry_interval,
receive_maximum: None,
max_packet_size: None,
topic_alias_max: v5_config.topic_alias_maximum.or(Some(TOPIC_ALIAS_MAXIMUM)),
request_response_info: None,
request_problem_info: None,
user_properties: vec![],
authentication_method: None,
authentication_data: None,
});
}
let (async_client, mut event_loop) =
V5AsyncClient::new(mqtt_options, self.subscribe_topics.len());
event_loop.verify_connection().await?;
if !self.subscribe_topics.is_empty() {
let subscribe_filters = self
.subscribe_topics
.iter()
.map(|topic| Filter::new(topic.topic.clone(), topic.qos.clone().into()))
.collect_vec();
async_client
.subscribe_many(subscribe_filters)
.await
.map_err(to_anyhow)?;
}
Ok(MqttClientResult::V5((V5MqttHandler, event_loop)))
}
async fn build_v3_client(&self) -> Result<MqttClientResult, Error> {
let mut mqtt_options = V3MqttOptions::new(
self.client_id,
&self.mqtt_resource.broker,
self.mqtt_resource.port,
);
if let Some(credentials) = &self.mqtt_resource.credentials {
let username = credentials.username.as_deref().unwrap_or("");
let password = credentials.password.as_deref().unwrap_or("");
mqtt_options.set_credentials(username, password);
}
if let Some(transport) = self.get_tls_configuration()? {
mqtt_options.set_transport(transport);
}
mqtt_options.set_keep_alive(Duration::from_secs(KEEP_ALIVE));
if let Some(v3_config) = self.v3_config {
mqtt_options.set_clean_session(v3_config.clean_session.unwrap_or(true));
}
let (async_client, mut event_loop) =
V3AsyncClient::new(mqtt_options, self.subscribe_topics.len());
event_loop.verify_connection().await?;
if !self.subscribe_topics.is_empty() {
let subscribe_filters = self
.subscribe_topics
.iter()
.map(|topic| SubscribeFilter::new(topic.topic.clone(), topic.qos.clone().into()))
.collect_vec();
async_client
.subscribe_many(subscribe_filters)
.await
.map_err(to_anyhow)?;
}
Ok(MqttClientResult::V3((V3MqttHandler, event_loop)))
}
}
backend/windmill-api/src/triggers/nats/handler_oss.rs
-64
View File
@@ -1,64 +0,0 @@
#[allow(unused)]
#[cfg(feature = "private")]
pub use super::handler_ee::*;
#[cfg(not(feature = "private"))]
use {
super::NatsTrigger,
crate::{
db::{ApiAuthed, DB},
triggers::{TriggerCrud, TriggerData},
},
axum::async_trait,
sqlx::PgConnection,
windmill_common::error::{Error, Result},
windmill_git_sync::DeployedObject,
};
#[cfg(not(feature = "private"))]
#[async_trait]
impl TriggerCrud for NatsTrigger {
type Trigger = ();
type TriggerConfig = ();
type TriggerConfigRequest = ();
type TestConnectionConfig = ();
const TABLE_NAME: &'static str = "";
const TRIGGER_TYPE: &'static str = "";
const SUPPORTS_SERVER_STATE: bool = false;
const SUPPORTS_TEST_CONNECTION: bool = false;
const ROUTE_PREFIX: &'static str = "/nats_triggers";
const DEPLOYMENT_NAME: &'static str = "";
const IS_ALLOWED_ON_CLOUD: bool = false;
fn get_deployed_object(path: String) -> DeployedObject {
DeployedObject::NatsTrigger { path }
}
async fn create_trigger(
&self,
_db: &DB,
_executor: &mut PgConnection,
_authed: &ApiAuthed,
_w_id: &str,
_trigger: TriggerData<Self::TriggerConfigRequest>,
) -> Result<()> {
Err(Error::BadRequest(
"NATS triggers are not available in open source version".to_string(),
))
}
async fn update_trigger(
&self,
_db: &DB,
_executor: &mut PgConnection,
_authed: &ApiAuthed,
_workspace_id: &str,
_path: &str,
_trigger: TriggerData<Self::TriggerConfigRequest>,
) -> Result<()> {
Err(Error::BadRequest(
"NATS triggers are not available in open source version".to_string(),
))
}
}
backend/windmill-api/src/triggers/nats/listener_oss.rs
-41
View File
@@ -1,41 +0,0 @@
#[allow(unused)]
#[cfg(feature = "private")]
pub use super::listener_ee::*;
#[cfg(not(feature = "private"))]
use {
super::NatsTrigger,
crate::triggers::{listener::ListeningTrigger, Listener},
std::sync::Arc,
tokio::sync::RwLock,
windmill_common::{error::Result, jobs::JobTriggerKind, DB},
};
#[cfg(not(feature = "private"))]
#[async_trait::async_trait]
impl Listener for NatsTrigger {
type Consumer = ();
type Extra = ();
const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Nats;
async fn get_consumer(
&self,
_db: &DB,
_listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
_err_message: Arc<RwLock<Option<String>>>,
_killpill_rx: tokio::sync::broadcast::Receiver<()>,
) -> Result<Option<Self::Consumer>> {
Ok(None)
}
async fn consume(
&self,
_db: &DB,
_consumer: Self::Consumer,
_listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
_err_message: Arc<RwLock<Option<String>>>,
_killpill_rx: tokio::sync::broadcast::Receiver<()>,
) {
()
}
}
backend/windmill-api/src/triggers/nats/mod.rs
-15
View File
@@ -1,15 +0,0 @@
#[cfg(feature = "private")]
mod handler_ee;
pub mod handler_oss;
#[cfg(feature = "private")]
mod listener_ee;
pub mod listener_oss;
#[cfg(feature = "private")]
mod mod_ee;
#[cfg(feature = "private")]
pub use mod_ee::*;
#[derive(Copy, Clone)]
pub struct NatsTrigger;
backend/windmill-api/src/triggers/postgres/bool.rs
-24
View File
@@ -1,24 +0,0 @@
use thiserror::Error;
/**
* This implementation is inspired by Postgres replication functionality
* from https://github.com/supabase/pg_replicate
*
* Original implementation:
* - https://github.dev/supabase/pg_replicate/blob/main/pg_replicate/src/conversions/bool.rs
*
*/
#[derive(Debug, Error)]
pub enum ParseBoolError {
#[error("invalid input value: {0}")]
InvalidInput(String),
}
pub fn parse_bool(s: &str) -> Result<bool, ParseBoolError> {
match s {
"t" => Ok(true),
"f" => Ok(false),
_ => Err(ParseBoolError::InvalidInput(s.to_string())),
}
}
backend/windmill-api/src/triggers/postgres/converter.rs
-255
View File
@@ -1,255 +0,0 @@
use core::str;
use std::{
num::{ParseFloatError, ParseIntError},
str::FromStr,
};
use super::{
bool::{parse_bool, ParseBoolError},
hex::{from_bytea_hex, ByteaHexParseError},
};
use chrono::{DateTime, FixedOffset, NaiveDate, NaiveDateTime, NaiveTime, Utc};
use rust_decimal::Decimal;
use rust_postgres::types::Type;
use serde_json::{to_value, Number, Value};
use thiserror::Error;
use uuid::Uuid;
/**
* This implementation is inspired by Postgres replication functionality
* from https://github.com/supabase/pg_replicate
*
* Original implementation:
* - https://github.com/supabase/pg_replicate/blob/main/pg_replicate/src/conversions/text.rs
*
*/
#[derive(Debug, Error)]
pub enum ConverterError {
#[error("invalid bool value")]
InvalidBool(#[from] ParseBoolError),
#[error("invalid int value")]
InvalidInt(#[from] ParseIntError),
#[error("invalid float value")]
InvalidFloat(#[from] ParseFloatError),
#[error("invalid numeric: {0}")]
InvalidNumeric(#[from] rust_decimal::Error),
#[error("invalid bytea: {0}")]
InvalidBytea(#[from] ByteaHexParseError),
#[error("invalid uuid: {0}")]
InvalidUuid(#[from] uuid::Error),
#[error("invalid json: {0}")]
InvalidJson(#[from] serde_json::Error),
#[error("invalid timestamp: {0} ")]
InvalidTimestamp(#[from] chrono::ParseError),
#[error("invalid array: {0}")]
InvalidArray(#[from] ArrayParseError),
#[error("{0}")]
Custom(String),
}
fn convert_into<T>(number: T) -> Number
where
T: Sized,
serde_json::Number: From<T>,
{
serde_json::Number::from(number)
}
pub struct Converter;
#[derive(Debug, Error)]
pub enum ArrayParseError {
#[error("input too short")]
InputTooShort,
#[error("missing braces")]
MissingBraces,
}
fn f64_to_json_number(raw_val: f64) -> Result<Value, ConverterError> {
let temp = serde_json::Number::from_f64(raw_val.into())
.ok_or(ConverterError::Custom("invalid json-float".to_string()))?;
Ok(Value::Number(temp))
}
impl Converter {
pub fn try_from_str(typ: Option<Type>, str: &str) -> Result<Value, ConverterError> {
let value = match typ.unwrap_or(Type::TEXT) {
Type::BOOL => Value::Bool(parse_bool(str)?),
Type::BOOL_ARRAY => {
Converter::parse_array(str, |str| Ok(Value::Bool(parse_bool(str)?)))?
}
Type::CHAR | Type::BPCHAR | Type::VARCHAR | Type::NAME | Type::TEXT => {
Value::String(str.to_string())
}
Type::CHAR_ARRAY
| Type::BPCHAR_ARRAY
| Type::VARCHAR_ARRAY
| Type::NAME_ARRAY
| Type::TEXT_ARRAY => {
Converter::parse_array(str, |str| Ok(Value::String(str.to_string())))?
}
Type::INT2 => Value::Number(convert_into(str.parse::<i16>()?)),
Type::INT2_ARRAY => Converter::parse_array(str, |str| {
Ok(Value::Number(convert_into(str.parse::<i16>()?)))
})?,
Type::INT4 => Value::Number(convert_into(str.parse::<i32>()?)),
Type::INT4_ARRAY => Converter::parse_array(str, |str| {
Ok(Value::Number(convert_into(str.parse::<i32>()?)))
})?,
Type::INT8 => Value::Number(convert_into(str.parse::<i64>()?)),
Type::INT8_ARRAY => Converter::parse_array(str, |str| {
Ok(Value::Number(convert_into(str.parse::<i64>()?)))
})?,
Type::FLOAT4 => f64_to_json_number(str.parse::<f64>()?)?,
Type::FLOAT4_ARRAY => {
Converter::parse_array(str, |str| f64_to_json_number(str.parse::<f64>()?))?
}
Type::FLOAT8 => f64_to_json_number(str.parse::<f64>()?)?,
Type::FLOAT8_ARRAY => {
Converter::parse_array(str, |str| f64_to_json_number(str.parse::<f64>()?))?
}
Type::NUMERIC => serde_json::json!(Decimal::from_str(str)?),
Type::NUMERIC_ARRAY => {
Converter::parse_array(str, |str| Ok(serde_json::json!(Decimal::from_str(str)?)))?
}
Type::BYTEA => to_value(from_bytea_hex(str)?).unwrap(),
Type::BYTEA_ARRAY => {
Converter::parse_array(str, |str| Ok(to_value(from_bytea_hex(str)?).unwrap()))?
}
Type::DATE => {
let date = NaiveDate::parse_from_str(str, "%Y-%m-%d")?;
Value::String(date.to_string())
}
Type::DATE_ARRAY => Converter::parse_array(str, |str| {
let date = NaiveDate::parse_from_str(str, "%Y-%m-%d")?;
Ok(Value::String(date.to_string()))
})?,
Type::TIME => {
let time = NaiveTime::parse_from_str(str, "%H:%M:%S%.f")?;
Value::String(time.to_string())
}
Type::TIME_ARRAY => Converter::parse_array(str, |str| {
let time = NaiveTime::parse_from_str(str, "%H:%M:%S%.f")?;
Ok(Value::String(time.to_string()))
})?,
Type::TIMESTAMP => {
let timestamp = NaiveDateTime::parse_from_str(str, "%Y-%m-%d %H:%M:%S%.f")?;
Value::String(timestamp.to_string())
}
Type::TIMESTAMP_ARRAY => Converter::parse_array(str, |str| {
let timestamp = NaiveDateTime::parse_from_str(str, "%Y-%m-%d %H:%M:%S%.f")?;
Ok(Value::String(timestamp.to_string()))
})?,
Type::TIMESTAMPTZ => {
let val =
match DateTime::<FixedOffset>::parse_from_str(str, "%Y-%m-%d %H:%M:%S%.f%#z") {
Ok(val) => val,
Err(_) => {
DateTime::<FixedOffset>::parse_from_str(str, "%Y-%m-%d %H:%M:%S%.f%:z")?
}
};
let utc: DateTime<Utc> = val.into();
Value::String(utc.to_string())
}
Type::TIMESTAMPTZ_ARRAY => {
match Converter::parse_array(str, |str| {
let utc: DateTime<Utc> =
DateTime::<FixedOffset>::parse_from_str(str, "%Y-%m-%d %H:%M:%S%.f%#z")?
.into();
Ok(Value::String(utc.to_string()))
}) {
Ok(val) => val,
Err(_) => Converter::parse_array(str, |str| {
let utc: DateTime<Utc> = DateTime::<FixedOffset>::parse_from_str(
str,
"%Y-%m-%d %H:%M:%S%.f%#z",
)?
.into();
Ok(Value::String(utc.to_string()))
})?,
}
}
Type::UUID => Value::String(Uuid::parse_str(str)?.to_string()),
Type::UUID_ARRAY => Converter::parse_array(str, |str| {
Ok(Value::String(Uuid::parse_str(str)?.to_string()))
})?,
Type::JSON | Type::JSONB => serde_json::from_str::<serde_json::Value>(str)?,
Type::JSON_ARRAY | Type::JSONB_ARRAY => Converter::parse_array(str, |str| {
Ok(serde_json::from_str::<serde_json::Value>(str)?)
})?,
Type::OID => Value::Number(convert_into(str.parse::<u32>()?)),
Type::OID_ARRAY => Converter::parse_array(str, |str| {
Ok(Value::Number(convert_into(str.parse::<u32>()?)))
})?,
_ => Value::String(str.to_string()),
};
Ok(value)
}
fn parse_array<P>(str: &str, mut parse: P) -> Result<Value, ConverterError>
where
P: FnMut(&str) -> Result<Value, ConverterError>,
{
if str.len() < 2 {
return Err(ArrayParseError::InputTooShort.into());
}
if !str.starts_with('{') || !str.ends_with('}') {
return Err(ArrayParseError::MissingBraces.into());
}
let mut res = vec![];
let str = &str[1..(str.len() - 1)];
let mut val_str = String::with_capacity(10);
let mut in_quotes = false;
let mut in_escape = false;
let mut chars = str.chars();
let mut done = str.is_empty();
while !done {
loop {
match chars.next() {
Some(c) => match c {
c if in_escape => {
val_str.push(c);
in_escape = false;
}
'"' => in_quotes = !in_quotes,
'\\' => in_escape = true,
',' if !in_quotes => {
break;
}
c => {
val_str.push(c);
}
},
None => {
done = true;
break;
}
}
}
let val = if val_str.to_lowercase() == "null" {
Value::Null
} else {
parse(&val_str)?
};
res.push(val);
val_str.clear();
}
let arr = Value::Array(res);
Ok(arr)
}
}
backend/windmill-api/src/triggers/postgres/handler.rs
-1103
View File
File diff suppressed because it is too large Load Diff
backend/windmill-api/src/triggers/postgres/hex.rs
-45
View File
@@ -1,45 +0,0 @@
use std::num::ParseIntError;
/**
* This implementation is inspired by Postgres replication functionality
* from https://github.com/supabase/pg_replicate
*
* Original implementation:
* - https://github.dev/supabase/pg_replicate/blob/main/pg_replicate/src/conversions/hex.rs
*
*/
use thiserror::Error;
#[derive(Debug, Error)]
pub enum ByteaHexParseError {
#[error("missing prefix '\\x'")]
InvalidPrefix,
#[error("invalid byte")]
OddNumerOfDigits,
#[error("parse int result: {0}")]
ParseInt(#[from] ParseIntError),
}
pub fn from_bytea_hex(s: &str) -> Result<Vec<u8>, ByteaHexParseError> {
if s.len() < 2 || &s[..2] != "\\x" {
return Err(ByteaHexParseError::InvalidPrefix);
}
let mut result = Vec::with_capacity((s.len() - 2) / 2);
let s = &s[2..];
if s.len() % 2 != 0 {
return Err(ByteaHexParseError::OddNumerOfDigits);
}
for i in (0..s.len()).step_by(2) {
let val = u8::from_str_radix(&s[i..i + 2], 16)?;
result.push(val);
}
Ok(result)
}
backend/windmill-api/src/triggers/postgres/listener.rs
-441
View File
@@ -1,441 +0,0 @@
use std::{collections::HashMap, pin::Pin, sync::Arc};
use bytes::{BufMut, Bytes, BytesMut};
use chrono::TimeZone;
use futures::{pin_mut, SinkExt};
use pg_escape::{quote_identifier, quote_literal};
use rust_postgres::{Client, CopyBothDuplex, SimpleQueryMessage};
use tokio::sync::RwLock;
use tokio_stream::StreamExt;
use windmill_common::{
db::UserDB,
error::{to_anyhow, Error, Result},
jobs::JobTriggerKind,
worker::to_raw_value,
DB,
};
use crate::{
resources::try_get_resource_from_db_as,
triggers::{
listener::ListeningTrigger,
postgres::{
drop_publication, get_default_pg_connection, get_raw_postgres_connection,
handler::drop_logical_replication_slot,
relation::RelationConverter,
replication_message::{
LogicalReplicationMessage::{
Begin, Commit, Delete, Insert, Relation, Type, Update,
},
PrimaryKeepAliveBody, ReplicationMessage,
},
Postgres, PostgresConfig, PostgresTrigger, ERROR_PUBLICATION_NAME_NOT_EXISTS,
},
trigger_helpers::TriggerJobArgs,
Listener,
},
};
const ERROR_REPLICATION_SLOT_NOT_EXISTS: &str = r#"The replication slot associated with this trigger no longer exists. Recreate a new replication slot or select an existing one in the advanced tab, or delete and recreate a new trigger"#;
pub struct LogicalReplicationSettings {
pub streaming: bool,
}
impl LogicalReplicationSettings {
pub fn new(streaming: bool) -> Self {
Self { streaming }
}
}
pub struct PostgresSimpleClient(Client);
trait RowExist {
fn row_exist(&self) -> bool;
}
impl RowExist for Vec<SimpleQueryMessage> {
fn row_exist(&self) -> bool {
self.iter()
.find_map(|element| {
if let SimpleQueryMessage::CommandComplete(value) = element {
Some(*value)
} else {
None
}
})
.is_some_and(|value| value > 0)
}
}
impl PostgresSimpleClient {
async fn new(database: &Postgres) -> Result<Self> {
let client = get_raw_postgres_connection(database, true).await?;
Ok(PostgresSimpleClient(client))
}
async fn execute_query(
&self,
query: &str,
) -> std::result::Result<Vec<SimpleQueryMessage>, rust_postgres::Error> {
self.0.simple_query(query).await
}
async fn get_logical_replication_stream(
&self,
publication_name: &str,
logical_replication_slot_name: &str,
) -> Result<(CopyBothDuplex<Bytes>, LogicalReplicationSettings)> {
let options = format!(
r#"("proto_version" '2', "publication_names" {})"#,
quote_literal(publication_name),
);
let query = format!(
r#"START_REPLICATION SLOT {} LOGICAL 0/0 {}"#,
quote_identifier(logical_replication_slot_name),
options
);
Ok((
self.0
.copy_both_simple::<bytes::Bytes>(query.as_str())
.await
.map_err(to_anyhow)?,
LogicalReplicationSettings::new(false),
))
}
async fn send_status_update(
primary_keep_alive: PrimaryKeepAliveBody,
copy_both_stream: &mut Pin<&mut CopyBothDuplex<Bytes>>,
) {
let mut buf = BytesMut::new();
let ts = chrono::Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap();
let ts = chrono::Utc::now()
.signed_duration_since(ts)
.num_microseconds()
.unwrap_or(0);
buf.put_u8(b'r');
buf.put_u64(primary_keep_alive.wal_end);
buf.put_u64(primary_keep_alive.wal_end);
buf.put_u64(primary_keep_alive.wal_end);
buf.put_i64(ts);
buf.put_u8(0);
copy_both_stream.send(buf.freeze()).await.unwrap();
tracing::debug!("Send update status message");
}
}
#[async_trait::async_trait]
impl Listener for PostgresTrigger {
type Consumer = (CopyBothDuplex<Bytes>, LogicalReplicationSettings);
type Extra = ();
type ExtraState = ();
const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Postgres;
async fn get_consumer(
&self,
db: &DB,
listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
_err_message: Arc<RwLock<Option<String>>>,
_killpill_rx: tokio::sync::broadcast::Receiver<()>,
) -> Result<Option<Self::Consumer>> {
let ListeningTrigger::<Self::TriggerConfig> { workspace_id, trigger_config, .. } =
listening_trigger;
let PostgresConfig {
postgres_resource_path, publication_name, replication_slot_name, ..
} = trigger_config;
let authed = listening_trigger
.authed(db, &Self::TRIGGER_KIND.to_string())
.await?;
let database = try_get_resource_from_db_as::<Postgres>(
&authed,
Some(UserDB::new(db.clone())),
&db,
postgres_resource_path,
workspace_id,
)
.await?;
let client = PostgresSimpleClient::new(&database).await?;
let publication = client
.execute_query(&format!(
"SELECT pubname FROM pg_publication WHERE pubname = {}",
quote_literal(&publication_name)
))
.await
.map_err(to_anyhow)?;
if !publication.row_exist() {
return Err(Error::BadConfig(
ERROR_PUBLICATION_NAME_NOT_EXISTS.to_string(),
));
}
let replication_slot = client
.execute_query(&format!(
"SELECT slot_name FROM pg_replication_slots WHERE slot_name = {}",
quote_literal(&replication_slot_name)
))
.await
.map_err(to_anyhow)?;
if !replication_slot.row_exist() {
return Err(Error::BadConfig(
ERROR_REPLICATION_SLOT_NOT_EXISTS.to_string(),
));
}
let (logical_replication_stream, logical_replication_settings) = client
.get_logical_replication_stream(&publication_name, &replication_slot_name)
.await
.map_err(to_anyhow)?;
Ok(Some((
logical_replication_stream,
logical_replication_settings,
)))
}
async fn consume(
&self,
db: &DB,
consumer: Self::Consumer,
listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
err_message: Arc<RwLock<Option<String>>>,
_killpill_rx: tokio::sync::broadcast::Receiver<()>,
_extra_state: Option<&Self::ExtraState>,
) {
let (logical_replication_stream, logical_replication_settings) = consumer;
pin_mut!(logical_replication_stream);
let mut relations = RelationConverter::new();
tracing::info!(
"Starting to listen for postgres trigger {}",
&listening_trigger.path
);
loop {
let message = logical_replication_stream.next().await;
let message = match message {
Some(message) => message,
None => {
tracing::error!(
"Stream for postgres trigger {} closed",
&listening_trigger.path
);
if let None = self
.update_ping_and_loop_ping_status(
db,
listening_trigger,
err_message.clone(),
Some("Stream closed".to_string()),
)
.await
{
return;
}
return;
}
};
let message = match message {
Ok(message) => message,
Err(err) => {
let err = format!(
"Postgres trigger named {} had an error while receiving a message : {}",
&listening_trigger.path,
err.to_string()
);
self.disable_with_error(db, listening_trigger, err).await;
return;
}
};
let logical_message = match ReplicationMessage::parse(message) {
Ok(logical_message) => logical_message,
Err(err) => {
let err = format!(
"Postgres trigger named: {} had an error while parsing message: {}",
&listening_trigger.path,
err.to_string()
);
self.disable_with_error(db, listening_trigger, err).await;
return;
}
};
match logical_message {
ReplicationMessage::PrimaryKeepAlive(primary_keep_alive) => {
if primary_keep_alive.reply {
PostgresSimpleClient::send_status_update(
primary_keep_alive,
&mut logical_replication_stream,
)
.await;
}
}
ReplicationMessage::XLogData(x_log_data) => {
let logical_replication_message = match x_log_data
.parse(&logical_replication_settings)
{
Ok(logical_replication_message) => logical_replication_message,
Err(err) => {
tracing::error!("Postgres trigger named: {} had an error while trying to parse incomming stream message: {}", &listening_trigger.path, err.to_string());
continue;
}
};
let json = match logical_replication_message {
Relation(relation_body) => {
relations.add_relation(relation_body);
None
}
Begin | Type | Commit => None,
Insert(insert) => Some((
insert.o_id,
Ok(None),
relations.row_to_json((insert.o_id, insert.tuple)),
"insert",
)),
Update(update) => {
let old_row = update
.old_tuple
.map(|old_tuple| relations.row_to_json((update.o_id, old_tuple)))
.transpose();
let row = relations.row_to_json((update.o_id, update.new_tuple));
Some((update.o_id, old_row, row, "update"))
}
Delete(delete) => {
let row = delete
.old_tuple
.unwrap_or_else(|| delete.key_tuple.unwrap());
Some((
delete.o_id,
Ok(None),
relations.row_to_json((delete.o_id, row)),
"delete",
))
}
};
match json {
Some((o_id, Ok(old_row), Ok(row), transaction_type)) => {
let relation = match relations.get_relation(o_id) {
Ok(relation) => relation,
Err(err) => {
tracing::error!(
"Postgres trigger named: {}, error: {}",
&listening_trigger.path,
err.to_string()
);
continue;
}
};
let database_info = HashMap::from([
("schema_name".to_string(), to_raw_value(&relation.namespace)),
("table_name".to_string(), to_raw_value(&relation.name)),
(
"transaction_type".to_string(),
to_raw_value(&transaction_type),
),
("old_row".to_string(), to_raw_value(&old_row)),
("row".to_string(), to_raw_value(&row)),
]);
let _ = self
.handle_event(
db,
listening_trigger,
database_info,
HashMap::new(),
None,
)
.await;
}
Some((o_id, old_row, row, transaction_type)) => {
let relation = match relations.get_relation(o_id) {
Ok(relation) => relation,
Err(err) => {
tracing::error!(
"Postgres trigger named: {}, error: {}",
&listening_trigger.path,
err.to_string()
);
continue;
}
};
if let Err(err) = old_row {
tracing::error!(
transaction_type = ?transaction_type,
schema = %relation.namespace,
table = %relation.name,
error = %err,
"Failed to decode OLD row for {} transaction on {}.{}",
transaction_type,
relation.namespace,
relation.name,
);
}
if let Err(err) = row {
tracing::error!(
transaction_type = ?transaction_type,
schema = %relation.namespace,
table = %relation.name,
error = %err,
"Failed to decode NEW row for {} transaction on {}.{}",
transaction_type,
relation.namespace,
relation.name,
);
}
}
_ => {}
}
}
}
}
}
async fn cleanup(
&self,
db: &DB,
listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
_extra_state: Option<&Self::ExtraState>,
) -> Result<()> {
let authed = listening_trigger
.authed(db, &Self::TRIGGER_KIND.to_string())
.await?;
let user_db = UserDB::new(db.clone());
let mut pg_connection = get_default_pg_connection(
authed,
Some(user_db),
&db,
&listening_trigger.trigger_config.postgres_resource_path,
&listening_trigger.workspace_id,
)
.await?;
if listening_trigger.trigger_config.basic_mode.unwrap_or(false) {
drop_logical_replication_slot(
&mut pg_connection,
&listening_trigger.trigger_config.replication_slot_name,
)
.await?;
drop_publication(
&mut pg_connection,
&listening_trigger.trigger_config.publication_name,
)
.await?;
}
Ok(())
}
}
backend/windmill-api/src/triggers/postgres/mapper.rs
-136
View File
@@ -1,136 +0,0 @@
use std::collections::HashMap;
use rust_postgres::types::Type;
use super::Language;
fn postgres_to_typescript_type(postgres_type: Option<Type>) -> String {
let data_type = match postgres_type {
Some(postgres_type) => match postgres_type {
Type::BOOL => "boolean",
Type::BOOL_ARRAY => "Array<boolean>",
Type::CHAR | Type::BPCHAR | Type::VARCHAR | Type::NAME | Type::TEXT => "string",
Type::CHAR_ARRAY
| Type::BPCHAR_ARRAY
| Type::VARCHAR_ARRAY
| Type::NAME_ARRAY
| Type::TEXT_ARRAY => "Array<string>",
Type::INT2 | Type::INT4 | Type::INT8 | Type::NUMERIC => "number",
Type::INT2_ARRAY | Type::INT4_ARRAY | Type::INT8_ARRAY => "Array<number>",
Type::FLOAT4 | Type::FLOAT8 => "number",
Type::FLOAT8_ARRAY | Type::FLOAT4_ARRAY => "Array<number>",
Type::NUMERIC_ARRAY => "Array<number>",
Type::BYTEA => "Array<number>",
Type::BYTEA_ARRAY => "Array<Array<number>>",
Type::DATE => "string",
Type::DATE_ARRAY => "Array<string>",
Type::TIME => "string",
Type::TIME_ARRAY => "Array<string>",
Type::TIMESTAMPTZ | Type::TIMESTAMP => "string",
Type::TIMESTAMPTZ_ARRAY | Type::TIMESTAMP_ARRAY => "Array<string>",
Type::UUID => "string",
Type::UUID_ARRAY => "Array<string>",
Type::JSON | Type::JSONB | Type::JSON_ARRAY | Type::JSONB_ARRAY => "unknown",
Type::OID => "number",
Type::OID_ARRAY => "Array<number>",
_ => "string",
},
None => "string",
};
data_type.to_string()
}
fn into_body_struct(language: Language, mapped_info: Vec<MappingInfo>) -> String {
let mut block = String::new();
match language {
Language::Typescript => {
block.push_str("{\r\n");
for field in mapped_info {
let typescript_type = postgres_to_typescript_type(field.data_type);
let mut key = field.column_name;
if field.is_nullable {
key.push('?');
}
let full_field = format!("\t\t{}: {},\r\n", key, typescript_type);
block.push_str(&full_field);
}
block.push_str("\t}");
}
}
block
}
#[derive(Debug)]
pub struct MappingInfo {
data_type: Option<Type>,
is_nullable: bool,
column_name: String,
}
impl MappingInfo {
pub fn new(column_name: String, data_type: Option<Type>, is_nullable: bool) -> Self {
Self { column_name, data_type, is_nullable }
}
}
pub struct Mapper {
to_template: HashMap<String, HashMap<String, Vec<MappingInfo>>>,
language: Language,
}
impl Mapper {
pub fn new(
to_template: HashMap<String, HashMap<String, Vec<MappingInfo>>>,
language: Language,
) -> Self {
Self { to_template, language }
}
fn into_typescript_template(self) -> Vec<String> {
let mut struct_definitions = Vec::new();
for (_, mapping_info) in self.to_template {
let last_elem = mapping_info.len() - 1;
for (i, (_, mapped_info)) in mapping_info.into_iter().enumerate() {
let mut struct_body = into_body_struct(Language::Typescript, mapped_info);
let struct_body = if i != last_elem {
struct_body.push_str("\r\n");
struct_body
} else {
struct_body
};
struct_definitions.push(struct_body);
}
}
struct_definitions
}
pub fn get_template(self) -> String {
let struct_definition = match self.language {
Language::Typescript => self.into_typescript_template(),
};
let struct_definition = if struct_definition.is_empty() {
"any".to_string()
} else {
struct_definition.join("\t| ")
};
format!(
r#"
export async function main(
transaction_type: "insert" | "update" | "delete",
schema_name: string,
table_name: string,
row: {},
old_row?: {}
) {{
}}
"#,
&struct_definition,
&struct_definition
)
}
}
backend/windmill-api/src/triggers/postgres/mod.rs
-553
View File
@@ -1,553 +0,0 @@
use std::collections::HashMap;
use crate::{
db::{ApiAuthed, DB},
resources::try_get_resource_from_db_as,
triggers::trigger_helpers::TriggerJobArgs,
};
use chrono::Utc;
use itertools::Itertools;
use native_tls::{Certificate, TlsConnector};
use pg_escape::quote_identifier;
use rand::Rng;
use rust_postgres::{config::SslMode, Client, Config, NoTls};
use rust_postgres_native_tls::MakeTlsConnector;
use serde::{Deserialize, Deserializer, Serialize};
use serde_json::value::RawValue;
use sqlx::FromRow;
use windmill_common::{
db::UserDB,
error::{to_anyhow, Error, Result},
triggers::TriggerKind,
utils::empty_as_none,
};
mod bool;
mod converter;
pub mod handler;
mod hex;
pub mod listener;
mod mapper;
mod relation;
mod replication_message;
#[derive(Clone, Copy)]
pub struct PostgresTrigger;
impl TriggerJobArgs for PostgresTrigger {
type Payload = HashMap<String, Box<RawValue>>;
const TRIGGER_KIND: TriggerKind = TriggerKind::Postgres;
fn v1_payload_fn(payload: &HashMap<String, Box<RawValue>>) -> HashMap<String, Box<RawValue>> {
payload.to_owned()
}
}
#[derive(Debug, Clone, FromRow, Serialize, Deserialize)]
pub struct PostgresConfig {
pub postgres_resource_path: String,
pub replication_slot_name: String,
pub publication_name: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub basic_mode: Option<bool>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PostgresConfigRequest {
postgres_resource_path: String,
#[serde(default)]
replication_slot_name: String,
#[serde(default)]
publication_name: String,
publication: Option<PublicationData>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TestPostgresConfig {
pub postgres_resource_path: String,
}
fn check_if_valid_relation<'de, D>(
relations: D,
) -> std::result::Result<Option<Vec<Relations>>, D::Error>
where
D: Deserializer<'de>,
{
let relations: Option<Vec<Relations>> = Option::deserialize(relations)?;
let mut track_all_table_in_schema = false;
let mut track_specific_columns_in_table = false;
match relations {
Some(relations) => {
for relation in relations.iter() {
if relation.schema_name.is_empty() {
return Err(serde::de::Error::custom(
"Schema Name must not be empty".to_string(),
));
}
if !track_all_table_in_schema && relation.table_to_track.is_empty() {
track_all_table_in_schema = true;
continue;
}
for table_to_track in relation.table_to_track.iter() {
if table_to_track.table_name.trim().is_empty() {
return Err(serde::de::Error::custom(
"Table name must not be empty".to_string(),
));
}
if !track_specific_columns_in_table && table_to_track.columns_name.is_some() {
track_specific_columns_in_table = true;
}
}
if track_all_table_in_schema && track_specific_columns_in_table {
return Err(serde::de::Error::custom("Incompatible tracking options. Schema-level tracking and specific table tracking with column selection cannot be used together. Refer to the documentation for valid configurations."));
}
}
if !relations
.iter()
.map(|relation| relation.schema_name.as_str())
.all_unique()
{
return Err(serde::de::Error::custom(
"You cannot choose a schema more than one time".to_string(),
));
}
Ok(Some(relations))
}
None => Ok(None),
}
}
fn check_if_valid_transaction_type<'de, D>(
transaction_type: D,
) -> std::result::Result<Vec<String>, D::Error>
where
D: Deserializer<'de>,
{
let mut transaction_type: Vec<String> = Vec::deserialize(transaction_type)?;
if transaction_type.len() > 3 {
return Err(serde::de::Error::custom(
"More than 3 transaction type which is not authorized, you are only allowed to those 3 transaction types: Insert, Update and Delete"
.to_string(),
));
}
transaction_type.sort_unstable();
transaction_type.dedup();
for transaction in transaction_type.iter() {
match transaction.to_lowercase().as_ref() {
"insert" => {},
"update" => {},
"delete" => {},
_ => {
return Err(serde::de::Error::custom(
"Only the following transaction types are allowed: Insert, Update and Delete (case insensitive)"
.to_string(),
))
}
}
}
Ok(transaction_type)
}
#[derive(Debug, Clone, Deserialize, Serialize)]
pub struct PublicationData {
#[serde(default, deserialize_with = "check_if_valid_relation")]
pub table_to_track: Option<Vec<Relations>>,
#[serde(deserialize_with = "check_if_valid_transaction_type")]
pub transaction_to_track: Vec<String>,
}
impl PublicationData {
pub fn new(
table_to_track: Option<Vec<Relations>>,
transaction_to_track: Vec<String>,
) -> PublicationData {
PublicationData { table_to_track, transaction_to_track }
}
}
// Slot list struct
#[derive(FromRow, Debug, Serialize)]
pub struct SlotList {
pub slot_name: Option<String>,
pub active: Option<bool>,
}
// Slot struct
#[derive(Debug, Serialize, Deserialize)]
pub struct Slot {
pub name: String,
}
// Template script struct
#[derive(Debug, Deserialize)]
pub struct TemplateScript {
pub postgres_resource_path: String,
#[serde(deserialize_with = "check_if_valid_relation")]
pub relations: Option<Vec<Relations>>,
pub language: Language,
}
// Language enum
#[derive(Deserialize, Debug)]
pub enum Language {
#[serde(rename = "typescript", alias = "Typescript")]
Typescript,
}
// Test postgres struct
#[derive(Serialize, Deserialize)]
pub struct TestPostgres {
pub postgres_resource_path: String,
}
// PostgreSQL publication replication struct
#[derive(Serialize, Deserialize)]
pub struct PostgresPublicationReplication {
pub publication_name: String,
pub replication_slot_name: String,
}
impl PostgresPublicationReplication {
pub fn new(
publication_name: String,
replication_slot_name: String,
) -> PostgresPublicationReplication {
PostgresPublicationReplication { publication_name, replication_slot_name }
}
}
pub const ERROR_PUBLICATION_NAME_NOT_EXISTS: &str = r#"The publication associated with this trigger no longer exists. Recreate a new publication or select an existing one in the advanced tab, or delete and recreate a new trigger"#;
#[derive(FromRow, Serialize, Deserialize, Debug)]
pub struct Postgres {
pub user: String,
pub password: String,
pub host: String,
pub port: Option<u16>,
pub dbname: String,
#[serde(default)]
pub sslmode: String,
#[serde(default, deserialize_with = "empty_as_none")]
pub root_certificate_pem: Option<String>,
}
#[derive(Debug, Clone, FromRow, Serialize, Deserialize)]
pub struct TableToTrack {
pub table_name: String,
#[serde(default, deserialize_with = "empty_as_none")]
pub where_clause: Option<String>,
#[serde(default, deserialize_with = "empty_as_none")]
pub columns_name: Option<Vec<String>>,
}
impl TableToTrack {
pub fn new(
table_name: String,
where_clause: Option<String>,
columns_name: Option<Vec<String>>,
) -> TableToTrack {
TableToTrack { table_name, where_clause, columns_name }
}
}
#[derive(Debug, Clone, FromRow, Serialize, Deserialize)]
pub struct Relations {
pub schema_name: String,
pub table_to_track: Vec<TableToTrack>,
}
impl Relations {
pub fn new(schema_name: String, table_to_track: Vec<TableToTrack>) -> Relations {
Relations { schema_name, table_to_track }
}
pub fn add_new_table(&mut self, table_to_track: TableToTrack) {
self.table_to_track.push(table_to_track);
}
}
fn build_tls_connector(
ssl_mode: SslMode,
root_certificate_pem: Option<&String>,
) -> Result<Option<MakeTlsConnector>> {
let get_tls_builder_for_verify = |root_certificate: Option<&String>| {
let mut builder = TlsConnector::builder();
if let Some(root_certificate) = root_certificate {
let root_certificate_pem =
Certificate::from_pem(root_certificate.as_bytes()).map_err(to_anyhow)?;
builder.add_root_certificate(root_certificate_pem);
}
Ok::<_, Error>(builder)
};
let connector = match ssl_mode {
SslMode::Disable => return Ok(None),
SslMode::Require | SslMode::Prefer => {
let mut builder = TlsConnector::builder();
builder.danger_accept_invalid_certs(true);
builder.danger_accept_invalid_hostnames(true);
builder
}
SslMode::VerifyCa => {
let mut builder = get_tls_builder_for_verify(root_certificate_pem)?;
builder.danger_accept_invalid_hostnames(true);
builder
}
SslMode::VerifyFull => {
let builder = get_tls_builder_for_verify(root_certificate_pem)?;
builder
}
_ => unreachable!(),
};
Ok(Some(MakeTlsConnector::new(
connector.build().map_err(to_anyhow)?,
)))
}
pub async fn get_raw_postgres_connection(
database: &Postgres,
logical_mode: bool,
) -> Result<Client> {
let ssl_mode = match database.sslmode.as_ref() {
"disable" => SslMode::Disable,
"" | "prefer" | "allow" => SslMode::Prefer,
"require" => SslMode::Require,
"verify-ca" => SslMode::VerifyCa,
"verify-full" => SslMode::VerifyFull,
ssl_mode => {
return Err(Error::BadRequest(
format!("Invalid ssl mode for postgres: {}, please put a valid ssl_mode among the following available ssl mode: ['disable', 'allow', 'prefer', 'verify-ca', 'verify-full']", ssl_mode),
))
}
};
let mut config = Config::new();
config
.dbname(&database.dbname)
.host(&database.host)
.user(&database.user)
.ssl_mode(ssl_mode);
if logical_mode {
config.replication_mode(rust_postgres::config::ReplicationMode::Logical);
}
if let Some(port) = database.port {
config.port(port);
};
if !database.password.is_empty() {
config.password(&database.password);
}
let connector = build_tls_connector(ssl_mode, database.root_certificate_pem.as_ref())?;
let client = if let Some(connector) = connector {
let (client, connection) = config.connect(connector).await.map_err(to_anyhow)?;
tokio::spawn(async move {
tracing::info!("Successfully connected to PostgreSQL database for trigger execution");
if let Err(e) = connection.await {
tracing::debug!("Error during PostgreSQL trigger connection: {:#?}", e);
};
tracing::info!("PostgreSQL trigger connection closed");
});
client
} else {
let (client, connection) = config.connect(NoTls).await.map_err(to_anyhow)?;
tokio::spawn(async move {
if let Err(e) = connection.await {
tracing::debug!("{:#?}", e);
};
tracing::info!("Successfully Connected into database");
});
client
};
Ok(client)
}
pub async fn get_pg_connection(
authed: ApiAuthed,
user_db: Option<UserDB>,
db: &DB,
postgres_resource_path: &str,
w_id: &str,
logical_mode: bool,
) -> Result<Client> {
let database =
try_get_resource_from_db_as::<Postgres>(&authed, user_db, db, postgres_resource_path, w_id)
.await?;
Ok(get_raw_postgres_connection(&database, logical_mode).await?)
}
pub async fn get_default_pg_connection(
authed: ApiAuthed,
user_db: Option<UserDB>,
db: &DB,
postgres_resource_path: &str,
w_id: &str,
) -> Result<Client> {
get_pg_connection(authed, user_db, db, postgres_resource_path, w_id, false).await
}
pub async fn create_logical_replication_slot(tx: &Client, slot_name: &str) -> Result<()> {
tx.execute(
&format!("SELECT * FROM pg_create_logical_replication_slot($1, 'pgoutput')"),
&[&slot_name],
)
.await
.map_err(to_anyhow)?;
Ok(())
}
pub async fn check_if_valid_publication_for_postgres_version(
pg_connection: &Client,
table_to_track: Option<&[Relations]>,
) -> Result<bool> {
use crate::triggers::postgres::handler::get_postgres_version_internal;
let postgres_version = get_postgres_version_internal(pg_connection).await?;
let pg_14 = postgres_version.starts_with("14");
if pg_14 {
let unsupported_publication = table_to_track
.and_then(|relations| {
relations.iter().find(|relation| {
let invalid_relation = relation.table_to_track.iter().find(|table_to_track| {
table_to_track.where_clause.is_some()
|| table_to_track.columns_name.is_some()
});
relation.table_to_track.is_empty() || invalid_relation.is_some()
})
})
.is_some();
if unsupported_publication {
return Err(Error::BadRequest(
"Your PostgreSQL database is running version 14, which does not support the following publication features: \
- WHERE clause filtering, \
- selective column tracking, and \
- tracking all tables within a schema.\n\
These features are only available in PostgreSQL 15 and above.".to_string(),
));
}
}
Ok(pg_14)
}
pub async fn create_pg_publication(
pg_connection: &Client,
publication_name: &str,
table_to_track: Option<&[Relations]>,
transaction_to_track: &[String],
) -> Result<()> {
let pg_14 =
check_if_valid_publication_for_postgres_version(pg_connection, table_to_track).await?;
let mut query = String::from("CREATE PUBLICATION ");
query.push_str(&quote_identifier(publication_name));
match table_to_track {
Some(database_component) if !database_component.is_empty() => {
query.push_str(" FOR");
let mut first = true;
for (i, schema) in database_component.iter().enumerate() {
if schema.table_to_track.is_empty() {
query.push_str(" TABLES IN SCHEMA ");
query.push_str(&quote_identifier(&schema.schema_name));
} else {
if pg_14 && first {
query.push_str(" TABLE ONLY ");
first = false
} else if !pg_14 {
query.push_str(" TABLE ONLY ");
}
for (j, table) in schema.table_to_track.iter().enumerate() {
let table_name = quote_identifier(&table.table_name);
let schema_name = quote_identifier(&schema.schema_name);
let full_name = format!("{}.{}", &schema_name, &table_name);
query.push_str(&full_name);
if let Some(columns) = table.columns_name.as_ref() {
query.push_str(" (");
let columns = columns
.iter()
.map(|column| quote_identifier(column))
.join(", ");
query.push_str(&columns);
query.push_str(")");
}
if let Some(where_clause) = &table.where_clause {
query.push_str(" WHERE (");
query.push_str(where_clause);
query.push(')');
}
if j + 1 != schema.table_to_track.len() {
query.push_str(", ");
}
}
}
if i < database_component.len() - 1 {
query.push_str(", ");
}
}
}
_ => {
query.push_str(" FOR ALL TABLES ");
}
};
if !transaction_to_track.is_empty() {
let transactions = || transaction_to_track.iter().join(", ");
query.push_str(" WITH (publish = '");
query.push_str(&transactions());
query.push_str("');");
}
pg_connection
.execute(&query, &[])
.await
.map_err(to_anyhow)?;
Ok(())
}
pub async fn drop_publication(pg_connection: &Client, publication_name: &str) -> Result<()> {
let mut query = String::from("DROP PUBLICATION IF EXISTS ");
let quoted_publication_name = quote_identifier(publication_name);
query.push_str(&quoted_publication_name);
pg_connection
.execute(&query, &[])
.await
.map_err(to_anyhow)?;
Ok(())
}
pub fn generate_random_string() -> String {
let timestamp = Utc::now().timestamp_millis().to_string();
let mut rng = rand::rng();
let charset = "abcdefghijklmnopqrstuvwxyz0123456789";
let random_part = (0..10)
.map(|_| {
charset
.chars()
.nth(rng.random_range(0..charset.len()))
.unwrap()
})
.collect::<String>();
format!("{}_{}", timestamp, random_part)
}
backend/windmill-api/src/triggers/postgres/relation.rs
-74
View File
@@ -1,74 +0,0 @@
use core::str;
use serde_json::{Map, Value};
use std::{collections::HashMap, str::Utf8Error};
use super::{
converter::{Converter, ConverterError},
replication_message::{Columns, RelationBody, TupleData},
};
use rust_postgres::types::Oid;
#[derive(Debug, thiserror::Error)]
pub enum RelationConversionError {
#[error("Could not find matching table")]
FailToFindMatchingTable,
#[error("Binary data not supported")]
BinaryFormatNotSupported,
#[error("decode error: {0}")]
FromBytes(#[from] ConverterError),
#[error("invalid string value")]
InvalidStr(#[from] Utf8Error),
}
pub struct RelationConverter(HashMap<Oid, RelationBody>);
impl RelationConverter {
pub fn new() -> Self {
Self(HashMap::new())
}
pub fn add_relation(&mut self, relation: RelationBody) {
self.0.insert(relation.o_id, relation);
}
pub fn get_columns(&self, o_id: Oid) -> Result<&Columns, RelationConversionError> {
self.0
.get(&o_id)
.map(|relation_body| &relation_body.columns)
.ok_or(RelationConversionError::FailToFindMatchingTable)
}
pub fn get_relation(&self, o_id: Oid) -> Result<&RelationBody, RelationConversionError> {
self.0
.get(&o_id)
.ok_or(RelationConversionError::FailToFindMatchingTable)
}
pub fn row_to_json(
&self,
to_decode: (Oid, Vec<TupleData>),
) -> Result<Map<String, Value>, RelationConversionError> {
let (o_id, tuple_data) = to_decode;
let mut object: Map<String, Value> = Map::new();
let columns = self.get_columns(o_id)?;
for (i, column) in columns.iter().enumerate() {
let value = match &tuple_data[i] {
TupleData::Null | TupleData::UnchangedToast => Value::Null,
TupleData::Binary(_) => {
return Err(RelationConversionError::BinaryFormatNotSupported)
}
TupleData::Text(bytes) => {
let str = str::from_utf8(&bytes[..])?;
Converter::try_from_str(column.type_o_id.clone(), str)?
}
};
object.insert(column.name.clone(), value);
}
Ok(object)
}
}
backend/windmill-api/src/triggers/postgres/replication_message.rs
-510
View File
@@ -1,510 +0,0 @@
#![allow(unused)]
use core::str;
use std::{
cmp,
io::{self, Cursor, Read},
str::Utf8Error,
};
use byteorder::{BigEndian, ReadBytesExt};
use bytes::Bytes;
use rust_postgres::types::{Oid, Type};
use thiserror::Error;
use super::listener::LogicalReplicationSettings;
const PRIMARY_KEEPALIVE_BYTE: u8 = b'k';
const X_LOG_DATA_BYTE: u8 = b'w';
/**
* This implementation is inspired by Postgres replication functionality
* from https://github.com/supabase/pg_replicate
*
* Original implementation:
* - https://github.com/supabase/pg_replicate/blob/main/pg_replicate/src/conversions/cdc_event.rs
*
*/
#[derive(Debug)]
pub struct PrimaryKeepAliveBody {
pub wal_end: u64,
pub timestamp: i64,
pub reply: bool,
}
impl PrimaryKeepAliveBody {
pub fn new(wal_end: u64, timestamp: i64, reply: bool) -> PrimaryKeepAliveBody {
PrimaryKeepAliveBody { wal_end, timestamp, reply }
}
}
const BEGIN_BYTE: u8 = b'B';
const COMMIT_BYTE: u8 = b'C';
const ORIGIN_BYTE: u8 = b'O';
const RELATION_BYTE: u8 = b'R';
const TYPE_BYTE: u8 = b'Y';
const INSERT_BYTE: u8 = b'I';
const UPDATE_BYTE: u8 = b'U';
const DELETE_BYTE: u8 = b'D';
const TUPLE_NEW_BYTE: u8 = b'N';
const TUPLE_KEY_BYTE: u8 = b'K';
const TUPLE_OLD_BYTE: u8 = b'O';
const TUPLE_DATA_NULL_BYTE: u8 = b'n';
const TUPLE_DATA_TOAST_BYTE: u8 = b'u';
const TUPLE_DATA_TEXT_BYTE: u8 = b't';
const TUPLE_DATA_BINARY_BYTE: u8 = b'b';
const REPLICA_IDENTITY_DEFAULT_BYTE: i8 = 0x64;
const REPLICA_IDENTITY_NOTHING_BYTE: i8 = 0x6E;
const REPLICA_IDENTITY_FULL_BYTE: i8 = 0x66;
const REPLICA_IDENTITY_INDEX_BYTE: i8 = 0x69;
#[derive(Debug)]
pub enum ReplicaIdentity {
Default,
Nothing,
Full,
Index,
}
#[derive(Debug)]
pub struct Column {
pub flags: i8,
pub name: String,
pub type_o_id: Option<Type>,
pub type_modifier: i32,
}
impl Column {
pub fn new(flags: i8, name: String, type_o_id: Option<Type>, type_modifier: i32) -> Self {
Self { flags, name, type_o_id, type_modifier }
}
}
pub type Columns = Vec<Column>;
#[derive(Debug)]
pub struct RelationBody {
pub transaction_id: Option<i32>,
pub o_id: Oid,
pub namespace: String,
pub name: String,
pub replica_identity: ReplicaIdentity,
pub columns: Columns,
}
impl RelationBody {
pub fn new(
transaction_id: Option<i32>,
o_id: Oid,
namespace: String,
name: String,
replica_identity: ReplicaIdentity,
columns: Columns,
) -> Self {
Self { transaction_id, o_id, namespace, name, replica_identity, columns }
}
}
#[derive(Debug)]
pub struct InsertBody {
pub transaction_id: Option<i32>,
pub o_id: Oid,
pub tuple: Vec<TupleData>,
}
impl InsertBody {
pub fn new(transaction_id: Option<i32>, o_id: Oid, tuple: Vec<TupleData>) -> Self {
Self { transaction_id, o_id, tuple }
}
}
#[derive(Debug)]
pub struct UpdateBody {
transaction_id: Option<i32>,
pub o_id: Oid,
pub old_tuple: Option<Vec<TupleData>>,
pub key_tuple: Option<Vec<TupleData>>,
pub new_tuple: Vec<TupleData>,
}
impl UpdateBody {
pub fn new(
transaction_id: Option<i32>,
o_id: Oid,
old_tuple: Option<Vec<TupleData>>,
key_tuple: Option<Vec<TupleData>>,
new_tuple: Vec<TupleData>,
) -> Self {
Self { transaction_id, o_id, old_tuple, key_tuple, new_tuple }
}
}
#[derive(Debug)]
pub struct DeleteBody {
transaction_id: Option<i32>,
pub o_id: Oid,
pub old_tuple: Option<Vec<TupleData>>,
pub key_tuple: Option<Vec<TupleData>>,
}
impl DeleteBody {
pub fn new(
transaction_id: Option<i32>,
o_id: Oid,
old_tuple: Option<Vec<TupleData>>,
key_tuple: Option<Vec<TupleData>>,
) -> Self {
Self { transaction_id, o_id, old_tuple, key_tuple }
}
}
#[derive(Debug)]
pub enum TupleData {
Null,
UnchangedToast,
Text(Bytes),
Binary(Bytes),
}
impl TupleData {
fn parse(buf: &mut Buffer) -> Result<Vec<TupleData>, ConversionError> {
let number_of_columns = buf.read_i16::<BigEndian>()?;
let mut tuples = Vec::with_capacity(number_of_columns as usize);
for _ in 0..number_of_columns {
let byte = buf.read_u8()?;
let tuple_data = match byte {
TUPLE_DATA_NULL_BYTE => TupleData::Null,
TUPLE_DATA_TOAST_BYTE => TupleData::UnchangedToast,
TUPLE_DATA_TEXT_BYTE => {
let len = buf.read_i32::<BigEndian>()?;
let mut data = vec![0; len as usize];
buf.read_exact(&mut data)?;
TupleData::Text(data.into())
}
TUPLE_DATA_BINARY_BYTE => {
let len = buf.read_i32::<BigEndian>()?;
let mut data = vec![0; len as usize];
buf.read_exact(&mut data)?;
TupleData::Binary(data.into())
}
byte => {
return Err(ConversionError::Io(io::Error::new(
io::ErrorKind::InvalidInput,
format!("unknown replication message byte `{}`", byte),
)));
}
};
tuples.push(tuple_data);
}
Ok(tuples)
}
}
#[derive(Debug)]
pub enum TransactionBody {
Insert(InsertBody),
Update(UpdateBody),
Delete(DeleteBody),
}
#[non_exhaustive]
#[derive(Debug)]
pub enum LogicalReplicationMessage {
Begin,
Commit,
Relation(RelationBody),
Type,
Insert(InsertBody),
Update(UpdateBody),
Delete(DeleteBody),
}
#[derive(Debug)]
pub struct XLogDataBody {
pub wal_start: u64,
pub wal_end: u64,
pub timestamp: i64,
pub data: Bytes,
}
#[derive(Error, Debug)]
pub enum ConversionError {
#[error("Error: {0}")]
Io(#[from] io::Error),
#[error("Utf8Error conversion: {0}")]
Utf8(#[from] Utf8Error),
}
struct Buffer {
bytes: Bytes,
idx: usize,
}
impl Buffer {
pub fn new(bytes: Bytes, idx: usize) -> Buffer {
Buffer { bytes, idx }
}
fn slice(&self) -> &[u8] {
&self.bytes[self.idx..]
}
fn read_cstr(&mut self) -> Result<String, ConversionError> {
match self.slice().iter().position(|&x| x == 0) {
Some(pos) => {
let start = self.idx;
let end = start + pos;
let cstr = str::from_utf8(&self.bytes[start..end])?.to_owned();
self.idx = end + 1;
Ok(cstr)
}
None => Err(ConversionError::Io(io::Error::new(
io::ErrorKind::UnexpectedEof,
"unexpected EOF",
))),
}
}
}
impl Read for Buffer {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
let len = {
let slice = self.slice();
let len = cmp::min(slice.len(), buf.len());
buf[..len].copy_from_slice(&slice[..len]);
len
};
self.idx += len;
Ok(len)
}
}
impl XLogDataBody {
pub fn new(wal_start: u64, wal_end: u64, timestamp: i64, data: Bytes) -> XLogDataBody {
XLogDataBody { wal_start, wal_end, timestamp, data }
}
pub fn parse(
self,
logical_replication_settings: &LogicalReplicationSettings,
) -> Result<LogicalReplicationMessage, ConversionError> {
let mut buf = Buffer::new(self.data.clone(), 0);
let byte = buf.read_u8()?;
let logical_replication_message = match byte {
BEGIN_BYTE => {
buf.read_i64::<BigEndian>()?;
buf.read_i64::<BigEndian>()?;
buf.read_i32::<BigEndian>()?;
LogicalReplicationMessage::Begin
}
COMMIT_BYTE => {
buf.read_i8()?;
buf.read_u64::<BigEndian>()?;
buf.read_u64::<BigEndian>()?;
buf.read_i64::<BigEndian>()?;
LogicalReplicationMessage::Commit
}
RELATION_BYTE => {
let transaction_id = match logical_replication_settings.streaming {
true => Some(buf.read_i32::<BigEndian>()?),
false => None,
};
let o_id = buf.read_u32::<BigEndian>()?;
let namespace = buf.read_cstr()?;
let name = buf.read_cstr()?;
let replica_identity = match buf.read_i8()? {
REPLICA_IDENTITY_DEFAULT_BYTE => ReplicaIdentity::Default,
REPLICA_IDENTITY_NOTHING_BYTE => ReplicaIdentity::Nothing,
REPLICA_IDENTITY_FULL_BYTE => ReplicaIdentity::Full,
REPLICA_IDENTITY_INDEX_BYTE => ReplicaIdentity::Index,
byte => {
return Err(ConversionError::Io(io::Error::new(
io::ErrorKind::InvalidInput,
format!("unknown replica identity byte `{}`", byte),
)));
}
};
let num_of_column = buf.read_i16::<BigEndian>()?;
let mut columns = Vec::with_capacity(num_of_column as usize);
for _ in 0..num_of_column {
let flags = buf.read_i8()?;
let name = buf.read_cstr()?;
let o_id = buf.read_u32::<BigEndian>()?;
let type_modifier = buf.read_i32::<BigEndian>()?;
let type_o_id = Type::from_oid(o_id);
let column = Column::new(flags, name, type_o_id, type_modifier);
columns.push(column);
}
LogicalReplicationMessage::Relation(RelationBody::new(
transaction_id,
o_id,
namespace,
name,
replica_identity,
columns,
))
}
TYPE_BYTE => {
buf.read_u32::<BigEndian>()?;
buf.read_cstr()?;
buf.read_cstr()?;
LogicalReplicationMessage::Type
}
INSERT_BYTE => {
let transaction_id = match logical_replication_settings.streaming {
true => Some(buf.read_i32::<BigEndian>()?),
false => None,
};
let o_id = buf.read_u32::<BigEndian>()?;
let byte = buf.read_u8()?;
let tuple = match byte {
TUPLE_NEW_BYTE => TupleData::parse(&mut buf)?,
byte => {
return Err(ConversionError::Io(io::Error::new(
io::ErrorKind::InvalidInput,
format!("unexpected tuple byte `{}`", byte),
)));
}
};
LogicalReplicationMessage::Insert(InsertBody::new(transaction_id, o_id, tuple))
}
UPDATE_BYTE => {
let transaction_id = match logical_replication_settings.streaming {
true => Some(buf.read_i32::<BigEndian>()?),
false => None,
};
let o_id = buf.read_u32::<BigEndian>()?;
let byte = buf.read_u8()?;
let mut key_tuple = None;
let mut old_tuple = None;
let new_tuple = match byte {
TUPLE_NEW_BYTE => TupleData::parse(&mut buf)?,
TUPLE_OLD_BYTE | TUPLE_KEY_BYTE => {
if byte == TUPLE_OLD_BYTE {
old_tuple = Some(TupleData::parse(&mut buf)?);
} else {
key_tuple = Some(TupleData::parse(&mut buf)?);
}
match buf.read_u8()? {
TUPLE_NEW_BYTE => TupleData::parse(&mut buf)?,
byte => {
return Err(ConversionError::Io(io::Error::new(
io::ErrorKind::InvalidInput,
format!("unexpected tuple byte `{}`", byte),
)));
}
}
}
byte => {
return Err(ConversionError::Io(io::Error::new(
io::ErrorKind::InvalidInput,
format!("unknown tuple byte `{}`", byte),
)));
}
};
LogicalReplicationMessage::Update(UpdateBody::new(
transaction_id,
o_id,
old_tuple,
key_tuple,
new_tuple,
))
}
DELETE_BYTE => {
let transaction_id = match logical_replication_settings.streaming {
true => Some(buf.read_i32::<BigEndian>()?),
false => None,
};
let o_id = buf.read_u32::<BigEndian>()?;
let tag = buf.read_u8()?;
let mut key_tuple = None;
let mut old_tuple = None;
match tag {
TUPLE_OLD_BYTE => old_tuple = Some(TupleData::parse(&mut buf)?),
TUPLE_KEY_BYTE => key_tuple = Some(TupleData::parse(&mut buf)?),
tag => {
return Err(ConversionError::Io(io::Error::new(
io::ErrorKind::InvalidInput,
format!("unknown tuple tag `{}`", tag),
)));
}
}
LogicalReplicationMessage::Delete(DeleteBody::new(
transaction_id,
o_id,
old_tuple,
key_tuple,
))
}
byte => {
return Err(ConversionError::Io(io::Error::new(
io::ErrorKind::InvalidInput,
format!("unknown replication message tag `{}`", byte),
)));
}
};
Ok(logical_replication_message)
}
}
#[non_exhaustive]
#[derive(Debug)]
pub enum ReplicationMessage {
XLogData(XLogDataBody),
PrimaryKeepAlive(PrimaryKeepAliveBody),
}
impl ReplicationMessage {
pub fn parse(buf: Bytes) -> io::Result<Self> {
let (byte, mut message) = buf.split_first().unwrap();
let replication_message = match *byte {
X_LOG_DATA_BYTE => {
let len = buf.len();
let wal_start = message.read_u64::<BigEndian>()?;
let wal_end = message.read_u64::<BigEndian>()?;
let timestamp = message.read_i64::<BigEndian>()?;
let len = len - message.len();
let data = buf.slice(len..);
ReplicationMessage::XLogData(XLogDataBody::new(wal_start, wal_end, timestamp, data))
}
PRIMARY_KEEPALIVE_BYTE => {
let wal_end = message.read_u64::<BigEndian>()?;
let timestamp = message.read_i64::<BigEndian>()?;
let reply = message.read_u8()?;
ReplicationMessage::PrimaryKeepAlive(PrimaryKeepAliveBody::new(
wal_end,
timestamp,
reply == 1,
))
}
byte => {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
format!("unknown replication message byte `{}`", byte),
));
}
};
Ok(replication_message)
}
}
backend/windmill-api/src/triggers/sqs/handler_oss.rs
-65
View File
@@ -1,65 +0,0 @@
#[cfg(feature = "private")]
#[allow(unused)]
pub use super::handler_ee::*;
#[cfg(not(feature = "private"))]
use {
super::SqsTrigger,
crate::{
db::{ApiAuthed, DB},
triggers::{Trigger, TriggerCrud, TriggerData},
},
axum::async_trait,
sqlx::PgConnection,
windmill_common::error::{Error, Result},
windmill_git_sync::DeployedObject,
};
#[cfg(not(feature = "private"))]
#[async_trait]
#[cfg(not(feature = "private"))]
impl TriggerCrud for SqsTrigger {
type Trigger = Trigger<Self::TriggerConfig>;
type TriggerConfig = ();
type TriggerConfigRequest = ();
type TestConnectionConfig = ();
const TABLE_NAME: &'static str = "";
const TRIGGER_TYPE: &'static str = "";
const SUPPORTS_SERVER_STATE: bool = false;
const SUPPORTS_TEST_CONNECTION: bool = false;
const ROUTE_PREFIX: &'static str = "/sqs_triggers";
const DEPLOYMENT_NAME: &'static str = "";
const IS_ALLOWED_ON_CLOUD: bool = false;
fn get_deployed_object(path: String) -> DeployedObject {
DeployedObject::SqsTrigger { path }
}
async fn create_trigger(
&self,
_db: &DB,
_tx: &mut PgConnection,
_authed: &ApiAuthed,
_w_id: &str,
_trigger: TriggerData<Self::TriggerConfigRequest>,
) -> Result<()> {
Err(Error::BadRequest(
"SQS triggers are not available in open source version".to_string(),
))
}
async fn update_trigger(
&self,
_db: &DB,
_tx: &mut PgConnection,
_authed: &ApiAuthed,
_workspace_id: &str,
_path: &str,
_trigger: TriggerData<Self::TriggerConfigRequest>,
) -> Result<()> {
Err(Error::BadRequest(
"SQS triggers are not available in open source version".to_string(),
))
}
}
backend/windmill-api/src/triggers/sqs/listener_oss.rs
-41
View File
@@ -1,41 +0,0 @@
#[allow(unused)]
#[cfg(feature = "private")]
pub use super::listener_ee::*;
#[cfg(not(feature = "private"))]
use {
super::SqsTrigger,
crate::triggers::{listener::ListeningTrigger, Listener},
std::sync::Arc,
tokio::sync::RwLock,
windmill_common::{error::Result, jobs::JobTriggerKind, DB},
};
#[cfg(not(feature = "private"))]
#[async_trait::async_trait]
impl Listener for SqsTrigger {
type Consumer = ();
type Extra = ();
const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Sqs;
async fn get_consumer(
&self,
_db: &DB,
_listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
_err_message: Arc<RwLock<Option<String>>>,
_killpill_rx: tokio::sync::broadcast::Receiver<()>,
) -> Result<Option<Self::Consumer>> {
Ok(None)
}
async fn consume(
&self,
_db: &DB,
_consumer: Self::Consumer,
_listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
_err_message: Arc<RwLock<Option<String>>>,
_killpill_rx: tokio::sync::broadcast::Receiver<()>,
) {
()
}
}
backend/windmill-api/src/triggers/sqs/mod.rs
-15
View File
@@ -1,15 +0,0 @@
#[cfg(feature = "private")]
mod handler_ee;
pub mod handler_oss;
#[cfg(feature = "private")]
mod listener_ee;
pub mod listener_oss;
#[cfg(feature = "private")]
mod mod_ee;
#[cfg(feature = "private")]
pub use mod_ee::*;
#[derive(Copy, Clone)]
pub struct SqsTrigger;
backend/windmill-api/src/triggers/trigger_helpers.rs
-959
View File
@@ -1,959 +0,0 @@
use anyhow::Context;
use axum::response::IntoResponse;
use http::StatusCode;
use serde::Deserialize;
use serde_json::value::RawValue;
use sqlx::types::Json;
use std::collections::HashMap;
use std::future::Future;
use uuid::Uuid;
use windmill_common::{
db::{UserDB, UserDbWithAuthed},
error::Result,
flows::{FlowModuleValue, Retry},
get_latest_deployed_hash_for_path, get_latest_flow_version_info_for_path,
jobs::{get_has_preprocessor_from_content_and_lang, script_path_to_payload, JobPayload},
scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang},
triggers::{
HubOrWorkspaceId, RunnableFormat, RunnableFormatVersion, TriggerKind, TriggerMetadata,
RUNNABLE_FORMAT_VERSION_CACHE,
},
users::username_to_permissioned_as,
utils::StripPath,
worker::to_raw_value,
};
use windmill_queue::{push, PushArgs, PushArgsOwned, PushIsolationLevel};
#[cfg(feature = "enterprise")]
use crate::jobs::check_license_key_valid;
use crate::{
db::{ApiAuthed, DB},
jobs::{
check_tag_available_for_workspace, delete_job_metadata_after_use,
push_flow_job_by_path_into_queue, push_script_job_by_path_into_queue, result_to_response,
run_wait_result_internal, RunJobQuery,
},
utils::check_scopes,
HTTP_CLIENT,
};
struct ScriptInfo {
has_preprocessor: Option<bool>,
language: ScriptLang,
content: String,
schema: Option<sqlx::types::Json<PartialSchema>>,
}
#[derive(Debug, Deserialize)]
struct PropertyDefinition {
r#type: Option<Box<RawValue>>,
}
#[derive(Debug, Deserialize)]
struct PartialSchema {
properties: Option<HashMap<String, PropertyDefinition>>,
}
#[derive(Debug, Clone, Eq, PartialEq, Hash)]
pub enum RunnableId {
FlowId(FlowId),
ScriptId(ScriptId),
HubScript(String),
}
impl RunnableId {
pub fn from_script_hash(hash: ScriptHash) -> Self {
Self::ScriptId(ScriptId::ScriptHash(hash))
}
pub fn from_script_path(path: &str) -> Self {
if path.starts_with("hub/") {
Self::HubScript(path.to_string())
} else {
Self::ScriptId(ScriptId::ScriptPath(path.to_string()))
}
}
pub fn from_flow_path(path: &str) -> Self {
Self::FlowId(FlowId::FlowPath(path.to_string()))
}
pub fn from_flow_version(version: i64) -> Self {
Self::FlowId(FlowId::FlowVersion(version))
}
}
#[derive(Debug, Clone, Eq, PartialEq, Hash)]
pub enum FlowId {
FlowPath(String),
FlowVersion(i64),
}
impl FlowId {
async fn get_flow_version_id(self, workspace_id: &str, db: &DB) -> Result<i64> {
let version_id = match self {
FlowId::FlowPath(path) => {
let info =
get_latest_flow_version_info_for_path(None, db, workspace_id, &path, true)
.await?;
info.version
}
FlowId::FlowVersion(version) => version,
};
Ok(version_id)
}
}
#[derive(Debug, Clone, Eq, PartialEq, Hash)]
pub enum ScriptId {
ScriptPath(String),
ScriptHash(ScriptHash),
}
impl ScriptId {
async fn get_script_hash(self, workspace_id: &str, db: &DB) -> Result<i64> {
let hash = match self {
ScriptId::ScriptPath(path) => {
let info = get_latest_deployed_hash_for_path(None, db.clone(), workspace_id, &path)
.await?;
info.hash
}
ScriptId::ScriptHash(hash) => hash.0,
};
Ok(hash)
}
}
async fn get_script_info(
db: &DB,
workspace_id: &str,
hash: i64,
) -> std::result::Result<ScriptInfo, sqlx::Error> {
sqlx::query_as!(ScriptInfo, "SELECT has_preprocessor, language as \"language: _\", content, schema as \"schema: _\" FROM script WHERE workspace_id = $1 AND hash = $2", workspace_id, hash)
.fetch_one(db)
.await
}
fn runnable_format_from_schema_without_preprocessor(
trigger_kind: &TriggerKind,
schema: Option<sqlx::types::Json<PartialSchema>>,
) -> RunnableFormat {
match trigger_kind {
TriggerKind::Mqtt
if schema.as_ref().is_some_and(|schema| {
schema.properties.as_ref().is_some_and(|properties| {
properties.iter().any(|(key, def)| {
key == "payload"
&& def.r#type.as_ref().is_some_and(|t| {
let typ = t.get().trim();
typ == "array" || (typ.starts_with('[') && typ.ends_with(']'))
})
})
})
}) =>
{
RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: false }
}
TriggerKind::Kafka | TriggerKind::Nats
if schema.as_ref().is_some_and(|schema| {
schema
.properties
.as_ref()
.is_some_and(|properties| properties.keys().any(|key| key == "msg"))
}) =>
{
RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: false }
}
_ => RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor: false },
}
}
fn runnable_format_from_preprocessor_args(
args: Option<Vec<windmill_parser::Arg>>,
) -> RunnableFormat {
if let Some(args) = args {
if args.iter().any(|arg| arg.name == "wm_trigger")
|| (args.len() > 0 && args.iter().all(|arg| arg.name != "event"))
{
RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor: true }
} else {
RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor: true }
}
} else {
RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor: true }
}
}
enum PreprocessorInfo {
Preprocessor { content: String, language: ScriptLang },
NoPreprocessor { schema: Option<sqlx::types::Json<PartialSchema>> },
}
#[derive(Debug, Deserialize)]
struct FlowInfo {
preprocessor_module: Option<sqlx::types::Json<FlowModuleValue>>,
schema: Option<sqlx::types::Json<PartialSchema>>,
}
fn get_preprocessor_args_from_content_and_language(
content: &str,
language: &ScriptLang,
) -> Result<Option<Vec<windmill_parser::Arg>>> {
let args = match language {
ScriptLang::Bun | ScriptLang::Bunnative | ScriptLang::Deno | ScriptLang::Nativets => {
let args = windmill_parser_ts::parse_deno_signature(
&content,
true,
false,
Some("preprocessor".to_string()),
)?;
Some(args.args)
}
ScriptLang::Python3 => {
let args = windmill_parser_py::parse_python_signature(
&content,
Some("preprocessor".to_string()),
false,
)?;
Some(args.args)
}
_ => None,
};
Ok(args)
}
pub async fn get_runnable_format(
runnable_id: RunnableId,
workspace_id: &str,
db: &DB,
trigger_kind: &TriggerKind,
) -> Result<RunnableFormat> {
let (key, preprocessor_info) = match runnable_id {
RunnableId::HubScript(path) => {
let Some(version) = path.split("/").nth(1) else {
return Err(windmill_common::error::Error::internal_err(
"Invalid hub script path".to_string(),
));
};
let version = match version.parse::<i64>() {
Ok(version) => version,
Err(_) => {
return Err(windmill_common::error::Error::internal_err(
"Invalid hub script version".to_string(),
));
}
};
let key = (HubOrWorkspaceId::Hub, version, trigger_kind.clone());
let runnable_format = RUNNABLE_FORMAT_VERSION_CACHE.get(&key);
if let Some(runnable_format) = runnable_format {
tracing::debug!("Using cached runnable format for hub script {path}");
return Ok(runnable_format);
}
let hub_script =
get_full_hub_script_by_path(StripPath(path.to_string()), &HTTP_CLIENT, Some(db))
.await?;
let has_preprocessor = get_has_preprocessor_from_content_and_lang(
&hub_script.content,
&hub_script.language,
)?;
let partial_schema = serde_json::from_str(hub_script.schema.get())?;
(
key,
if has_preprocessor {
PreprocessorInfo::Preprocessor {
content: hub_script.content,
language: hub_script.language,
}
} else {
PreprocessorInfo::NoPreprocessor {
schema: Some(sqlx::types::Json(partial_schema)),
}
},
)
}
RunnableId::FlowId(flow_id) => {
let version = flow_id.get_flow_version_id(workspace_id, db).await?;
let key = (
HubOrWorkspaceId::WorkspaceId(workspace_id.to_string()),
version,
trigger_kind.clone(),
);
let runnable_format = RUNNABLE_FORMAT_VERSION_CACHE.get(&key);
if let Some(runnable_format) = runnable_format {
tracing::debug!("Using cached runnable format for flow version {version}");
return Ok(runnable_format);
}
let flow_info = sqlx::query_as!(
FlowInfo,
"SELECT
value->'preprocessor_module'->'value' as \"preprocessor_module: _\",
schema as \"schema: _\"
FROM flow_version
WHERE
id = $1
AND workspace_id = $2",
version,
workspace_id,
)
.fetch_one(db)
.await?;
if let Some(preprocessor_module) = flow_info.preprocessor_module {
match preprocessor_module.0 {
FlowModuleValue::RawScript { content, language, .. } => {
(key, PreprocessorInfo::Preprocessor { content, language })
}
FlowModuleValue::Script { path, hash, .. } => {
let hash = if let Some(hash) = hash {
hash.0
} else {
let script_hash = get_latest_deployed_hash_for_path(
None,
db.clone(),
workspace_id,
&path,
)
.await?;
script_hash.hash
};
let script_info = get_script_info(db, workspace_id, hash).await?;
(
key,
PreprocessorInfo::Preprocessor {
content: script_info.content,
language: script_info.language,
},
)
}
_ => {
return Err(windmill_common::error::Error::internal_err(
"Unsupported preprocessor module".to_string(),
));
}
}
} else {
(
key,
PreprocessorInfo::NoPreprocessor { schema: flow_info.schema },
)
}
}
RunnableId::ScriptId(script_id) => {
let hash = script_id.get_script_hash(workspace_id, db).await?;
let key = (
HubOrWorkspaceId::WorkspaceId(workspace_id.to_string()),
hash,
trigger_kind.clone(),
);
let runnable_format = RUNNABLE_FORMAT_VERSION_CACHE.get(&key);
if let Some(runnable_format) = runnable_format {
tracing::debug!("Using cached runnable format for script {hash}");
return Ok(runnable_format);
}
let script_info = get_script_info(db, workspace_id, hash).await?;
if script_info.has_preprocessor.unwrap_or(false) {
(
key,
PreprocessorInfo::Preprocessor {
content: script_info.content,
language: script_info.language,
},
)
} else {
(
key,
PreprocessorInfo::NoPreprocessor { schema: script_info.schema },
)
}
}
};
let runnable_format = match preprocessor_info {
PreprocessorInfo::Preprocessor { content, language } => {
let args = get_preprocessor_args_from_content_and_language(&content, &language)?;
runnable_format_from_preprocessor_args(args)
}
PreprocessorInfo::NoPreprocessor { schema } => {
runnable_format_from_schema_without_preprocessor(trigger_kind, schema)
}
};
RUNNABLE_FORMAT_VERSION_CACHE.insert(key, runnable_format);
Ok(runnable_format)
}
#[allow(dead_code)]
pub trait TriggerJobArgs {
type Payload: Send + Sync;
const TRIGGER_KIND: TriggerKind;
fn v1_payload_fn(payload: &Self::Payload) -> HashMap<String, Box<RawValue>>;
fn v2_payload_fn(payload: &Self::Payload) -> HashMap<String, Box<RawValue>> {
Self::v1_payload_fn(payload)
}
fn build_job_args_v2(
has_preprocessor: bool,
payload: &Self::Payload,
info: HashMap<String, Box<RawValue>>,
) -> PushArgsOwned {
let mut args = Self::v2_payload_fn(payload);
if has_preprocessor {
args.insert(
"kind".to_string(),
to_raw_value(&Self::TRIGGER_KIND.to_key()),
);
args.extend(info);
let args = HashMap::from([("event".to_string(), to_raw_value(&args))]);
PushArgsOwned { args, extra: None }
} else {
PushArgsOwned { args, extra: None }
}
}
fn build_job_args_v1(
has_preprocessor: bool,
payload: &Self::Payload,
info: HashMap<String, Box<RawValue>>,
) -> PushArgsOwned {
let trigger_key = Self::TRIGGER_KIND.to_key();
let args = Self::v1_payload_fn(payload);
let extra = if has_preprocessor {
Some(HashMap::from([(
"wm_trigger".to_string(),
to_raw_value(&serde_json::json!({
"kind": trigger_key,
trigger_key: info
})),
)]))
} else {
None
};
PushArgsOwned { args, extra }
}
fn build_job_args(
runnable_path: &str,
is_flow: bool,
w_id: &str,
db: &DB,
payload: Self::Payload,
info: HashMap<String, Box<RawValue>>,
) -> impl Future<Output = Result<PushArgsOwned>> + Send {
async move {
let runnable_id = if is_flow {
RunnableId::from_flow_path(runnable_path)
} else {
RunnableId::from_script_path(runnable_path)
};
Self::build_job_args_from_runnable_id(runnable_id, w_id, db, payload, info).await
}
}
fn build_job_args_from_runnable_id(
runnable_id: RunnableId,
w_id: &str,
db: &DB,
payload: Self::Payload,
trigger_info: HashMap<String, Box<RawValue>>,
) -> impl Future<Output = Result<PushArgsOwned>> + Send {
async move {
tracing::debug!("Building job args for {runnable_id:?}");
let runnable_format =
get_runnable_format(runnable_id, w_id, db, &Self::TRIGGER_KIND).await?;
let job_args = match runnable_format {
RunnableFormat { version: RunnableFormatVersion::V1, has_preprocessor } => {
Self::build_job_args_v1(has_preprocessor, &payload, trigger_info)
}
RunnableFormat { version: RunnableFormatVersion::V2, has_preprocessor } => {
Self::build_job_args_v2(has_preprocessor, &payload, trigger_info)
}
};
Ok(job_args)
}
}
fn build_capture_payloads(
payload: &Self::Payload,
info: HashMap<String, Box<RawValue>>,
) -> (PushArgsOwned, PushArgsOwned) {
let main_args = Self::build_job_args_v2(false, payload, info.clone());
let preprocessor_args = Self::build_job_args_v2(true, payload, info);
(main_args, preprocessor_args)
}
}
#[allow(dead_code)]
pub async fn trigger_runnable_inner<'c>(
db: &DB,
tx_o: Option<sqlx::Transaction<'c, sqlx::Postgres>>,
user_db: Option<UserDB>,
authed: ApiAuthed,
workspace_id: &str,
runnable_path: &str,
is_flow: bool,
args: PushArgsOwned,
retry: Option<&sqlx::types::Json<Retry>>,
error_handler_path: Option<&str>,
error_handler_args: Option<&sqlx::types::Json<HashMap<String, serde_json::Value>>>,
trigger_path: String,
job_id: Option<Uuid>,
trigger: TriggerMetadata,
suspended_mode: Option<bool>,
) -> Result<(
Uuid,
Option<bool>,
Option<String>,
Option<sqlx::Transaction<'c, sqlx::Postgres>>,
)> {
let error_handler_args = error_handler_args.map(|args| {
let args = args
.0
.iter()
.map(|(key, value)| (key.to_owned(), to_raw_value(&value)))
.collect::<HashMap<String, Box<RawValue>>>();
Json(args)
});
let user_db = user_db.unwrap_or_else(|| UserDB::new(db.clone()));
let (uuid, delete_after_use, early_return, tx_out) = if is_flow {
let run_query = RunJobQuery { job_id, suspended_mode, ..Default::default() };
let path = StripPath(runnable_path.to_string());
let (uuid, early_return, tx_out) = push_flow_job_by_path_into_queue(
authed,
db.clone(),
tx_o,
user_db,
workspace_id.to_string(),
path,
run_query,
args,
Some(trigger),
)
.await?;
(uuid, None, early_return, tx_out)
} else {
let (uuid, delete_after_use, tx_out) = trigger_script_internal(
db,
tx_o,
user_db,
authed,
workspace_id,
runnable_path,
args,
retry,
error_handler_path,
error_handler_args.as_ref(),
trigger_path,
job_id,
trigger,
suspended_mode,
)
.await?;
(uuid, delete_after_use, None, tx_out)
};
Ok((uuid, delete_after_use, early_return, tx_out))
}
#[allow(dead_code)]
pub async fn trigger_runnable(
db: &DB,
user_db: Option<UserDB>,
authed: ApiAuthed,
workspace_id: &str,
runnable_path: &str,
is_flow: bool,
args: PushArgsOwned,
retry: Option<&sqlx::types::Json<Retry>>,
error_handler_path: Option<&str>,
error_handler_args: Option<&sqlx::types::Json<HashMap<String, serde_json::Value>>>,
trigger_path: String,
job_id: Option<Uuid>,
suspended_mode: bool,
trigger: TriggerMetadata,
) -> Result<axum::response::Response> {
let uuid = trigger_runnable_inner(
db,
None,
user_db,
authed,
workspace_id,
runnable_path,
is_flow,
args,
retry,
error_handler_path,
error_handler_args,
trigger_path,
job_id,
trigger,
Some(suspended_mode),
)
.await?
.0;
Ok((StatusCode::CREATED, uuid.to_string()).into_response())
}
#[allow(dead_code)]
pub async fn trigger_runnable_and_wait_for_result(
db: &DB,
user_db: Option<UserDB>,
authed: ApiAuthed,
workspace_id: &str,
runnable_path: &str,
is_flow: bool,
args: PushArgsOwned,
retry: Option<&sqlx::types::Json<Retry>>,
error_handler_path: Option<&str>,
error_handler_args: Option<&sqlx::types::Json<HashMap<String, serde_json::Value>>>,
trigger_path: String,
trigger: TriggerMetadata,
) -> Result<axum::response::Response> {
let username = authed.username.clone();
let (uuid, delete_after_use, early_return, _) = trigger_runnable_inner(
db,
None,
user_db,
authed,
workspace_id,
runnable_path,
is_flow,
args,
retry,
error_handler_path,
error_handler_args,
trigger_path,
None,
trigger,
None,
)
.await?;
let (result, success) =
run_wait_result_internal(db, uuid, &workspace_id, early_return, &username).await?;
if delete_after_use.unwrap_or(false) {
delete_job_metadata_after_use(&db, uuid).await?;
}
result_to_response(result, success)
}
#[allow(dead_code)]
pub async fn trigger_runnable_and_wait_for_raw_result(
db: &DB,
user_db: Option<UserDB>,
authed: ApiAuthed,
workspace_id: &str,
runnable_path: &str,
is_flow: bool,
args: PushArgsOwned,
retry: Option<&sqlx::types::Json<Retry>>,
error_handler_path: Option<&str>,
error_handler_args: Option<&sqlx::types::Json<HashMap<String, serde_json::Value>>>,
trigger_path: String,
trigger: TriggerMetadata,
) -> Result<(Box<RawValue>, bool)> {
let username = authed.username.clone();
let (uuid, delete_after_use, early_return, _) = trigger_runnable_inner(
db,
None,
user_db,
authed,
workspace_id,
runnable_path,
is_flow,
args,
retry,
error_handler_path,
error_handler_args,
trigger_path,
None,
trigger,
None,
)
.await?;
let (result, success) =
run_wait_result_internal(db, uuid, &workspace_id, early_return, &username)
.await
.with_context(|| {
format!(
"Error fetching job result for {} {}",
if is_flow { "flow" } else { "script" },
runnable_path
)
})?;
if delete_after_use.unwrap_or(false) {
delete_job_metadata_after_use(&db, uuid).await?;
}
Ok((result, success))
}
pub async fn trigger_runnable_and_wait_for_raw_result_with_error_ctx(
db: &DB,
user_db: Option<UserDB>,
authed: ApiAuthed,
workspace_id: &str,
runnable_path: &str,
is_flow: bool,
args: PushArgsOwned,
retry: Option<&sqlx::types::Json<Retry>>,
error_handler_path: Option<&str>,
error_handler_args: Option<&sqlx::types::Json<HashMap<String, serde_json::Value>>>,
trigger_path: String,
trigger: TriggerMetadata,
) -> Result<Box<RawValue>> {
let (result, success) = trigger_runnable_and_wait_for_raw_result(
db,
user_db,
authed,
workspace_id,
runnable_path,
is_flow,
args,
retry,
error_handler_path,
error_handler_args,
trigger_path,
trigger,
)
.await?;
if !success {
Err(windmill_common::error::Error::internal_err(format!(
"{} {runnable_path} failed: {:?}",
if is_flow { "Flow" } else { "Script" },
result
)))
} else {
Ok(result)
}
}
async fn trigger_script_internal<'c>(
db: &DB,
tx_o: Option<sqlx::Transaction<'c, sqlx::Postgres>>,
user_db: UserDB,
authed: ApiAuthed,
workspace_id: &str,
script_path: &str,
args: PushArgsOwned,
retry: Option<&sqlx::types::Json<Retry>>,
error_handler_path: Option<&str>,
error_handler_args: Option<&sqlx::types::Json<HashMap<String, Box<RawValue>>>>,
trigger_path: String,
job_id: Option<Uuid>,
trigger: TriggerMetadata,
suspended_mode: Option<bool>,
) -> Result<(
Uuid,
Option<bool>,
Option<sqlx::Transaction<'c, sqlx::Postgres>>,
)> {
if retry.is_none() && error_handler_path.is_none() {
let run_query = RunJobQuery { job_id, suspended_mode, ..Default::default() };
let path = StripPath(script_path.to_string());
let (uuid, delete_after_use, tx_out) = push_script_job_by_path_into_queue(
authed,
db.clone(),
tx_o,
user_db,
workspace_id.to_string(),
path,
run_query,
args,
Some(trigger),
)
.await?;
Ok((uuid, delete_after_use, tx_out))
} else {
let (uuid, delete_after_use, tx_out) = trigger_script_with_retry_and_error_handler(
db,
tx_o,
user_db,
authed,
workspace_id,
script_path,
args,
retry,
error_handler_path,
error_handler_args,
trigger_path,
job_id,
trigger,
suspended_mode,
)
.await?;
Ok((uuid, delete_after_use, tx_out))
}
}
async fn trigger_script_with_retry_and_error_handler<'c>(
db: &DB,
tx_o: Option<sqlx::Transaction<'c, sqlx::Postgres>>,
user_db: UserDB,
authed: ApiAuthed,
workspace_id: &str,
script_path: &str,
args: PushArgsOwned,
retry: Option<&sqlx::types::Json<Retry>>,
error_handler_path: Option<&str>,
error_handler_args: Option<&sqlx::types::Json<HashMap<String, Box<RawValue>>>>,
trigger_path: String,
job_id: Option<Uuid>,
trigger: TriggerMetadata,
suspended_mode: Option<bool>,
) -> Result<(
Uuid,
Option<bool>,
Option<sqlx::Transaction<'c, sqlx::Postgres>>,
)> {
#[cfg(feature = "enterprise")]
check_license_key_valid().await?;
check_scopes(&authed, || format!("jobs:run:scripts:{script_path}"))?;
let retry = retry.map(|r| r.0.clone());
let error_handler_path = error_handler_path.map(|p| p.to_string());
let error_handler_args = error_handler_args.map(|args| args.0.clone());
let (job_payload, tag, delete_after_use, timeout, on_behalf_of) = {
let db_authed = UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() };
script_path_to_payload(
script_path,
Some(db_authed),
db.clone(),
&workspace_id,
Some(false),
)
.await?
};
check_tag_available_for_workspace(&db, &workspace_id, &tag, &authed).await?;
let return_tx = tx_o.is_some();
let (email, permissioned_as, push_authed, tx) = if let Some(tx) = tx_o {
(
authed.email.as_str(),
username_to_permissioned_as(&authed.username),
Some(authed.clone().into()),
PushIsolationLevel::Transaction(tx),
)
} else if let Some(on_behalf_of) = on_behalf_of.as_ref() {
(
on_behalf_of.email.as_str(),
on_behalf_of.permissioned_as.clone(),
None,
PushIsolationLevel::IsolatedRoot(db.clone()),
)
} else {
(
authed.email.as_str(),
username_to_permissioned_as(&authed.username),
Some(authed.clone().into()),
PushIsolationLevel::Isolated(user_db, authed.clone().into()),
)
};
let push_args = PushArgs { args: &args.args, extra: args.extra };
let retryable_job_payload = match job_payload {
JobPayload::ScriptHash {
hash,
path,
concurrency_settings,
debouncing_settings,
cache_ttl,
cache_ignore_s3_path,
priority,
apply_preprocessor,
..
} => JobPayload::SingleStepFlow {
path,
hash: Some(hash),
flow_version: None,
args: HashMap::from(&push_args),
retry,
error_handler_path,
error_handler_args,
skip_handler: None,
cache_ttl,
cache_ignore_s3_path,
priority,
tag_override: tag.clone(),
apply_preprocessor,
trigger_path: Some(trigger_path.clone()),
concurrency_settings,
debouncing_settings,
},
_ => {
return Err(windmill_common::error::Error::internal_err(format!(
"Unsupported job payload: {:?}",
job_payload
)))
}
};
let (uuid, tx) = push(
&db,
tx,
&workspace_id,
retryable_job_payload,
push_args,
authed.display_username(),
email,
permissioned_as,
authed.token_prefix.as_deref(),
None,
None,
None,
None,
None,
job_id,
false,
false,
None,
true,
tag,
timeout,
None,
None,
push_authed.as_ref(),
false,
None,
Some(trigger),
suspended_mode,
)
.await?;
// If we were given a transaction, return it; otherwise commit it
if return_tx {
Ok((uuid, delete_after_use, Some(tx)))
} else {
tx.commit().await?;
Ok((uuid, delete_after_use, None))
}
}
backend/windmill-api/src/triggers/websocket/handler.rs
-263
View File
@@ -1,263 +0,0 @@
use std::borrow::Cow;
use crate::{
db::{ApiAuthed, DB},
triggers::{Trigger, TriggerCrud, TriggerData},
};
use axum::async_trait;
use itertools::Itertools;
use serde_json::value::RawValue;
use sqlx::{types::Json as SqlxJson, PgConnection};
use tokio_tungstenite::connect_async;
use windmill_common::{
db::UserDB,
error::{Error, Result},
worker::to_raw_value,
};
use windmill_git_sync::DeployedObject;
use super::{
get_url_from_runnable_value, TestWebsocketConfig, WebsocketConfig, WebsocketConfigRequest,
WebsocketTrigger,
};
#[async_trait]
impl TriggerCrud for WebsocketTrigger {
type TriggerConfig = WebsocketConfig;
type Trigger = Trigger<Self::TriggerConfig>;
type TriggerConfigRequest = WebsocketConfigRequest;
type TestConnectionConfig = TestWebsocketConfig;
const TABLE_NAME: &'static str = "websocket_trigger";
const TRIGGER_TYPE: &'static str = "websocket";
const SUPPORTS_SERVER_STATE: bool = true;
const SUPPORTS_TEST_CONNECTION: bool = true;
const ROUTE_PREFIX: &'static str = "/websocket_triggers";
const DEPLOYMENT_NAME: &'static str = "WebSocket trigger";
const ADDITIONAL_SELECT_FIELDS: &[&'static str] = &[
"url",
"filters",
"initial_messages",
"url_runnable_args",
"can_return_message",
"can_return_error_result",
];
const IS_ALLOWED_ON_CLOUD: bool = false;
fn get_deployed_object(path: String) -> DeployedObject {
DeployedObject::WebsocketTrigger { path }
}
async fn validate_config(
&self,
_db: &DB,
config: &Self::TriggerConfigRequest,
_workspace_id: &str,
) -> Result<()> {
if config.url.trim().is_empty() {
return Err(Error::BadRequest(
"WebSocket URL cannot be empty".to_string(),
));
}
if let Some(args) = &config.url_runnable_args {
if !args.is_object() {
return Err(Error::BadRequest(
"url_runnable_args must be an object".to_string(),
));
}
}
Ok(())
}
async fn create_trigger(
&self,
_db: &DB,
tx: &mut PgConnection,
authed: &ApiAuthed,
w_id: &str,
trigger: TriggerData<Self::TriggerConfigRequest>,
) -> Result<()> {
let filters = trigger
.config
.filters
.into_iter()
.map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap()))
.collect_vec();
let initial_messages = trigger
.config
.initial_messages
.unwrap_or_default()
.into_iter()
.map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap()))
.collect_vec();
sqlx::query!(
r#"
INSERT INTO websocket_trigger (
workspace_id,
path,
url,
script_path,
is_flow,
mode,
filters,
initial_messages,
url_runnable_args,
edited_by,
can_return_message,
can_return_error_result,
email,
edited_at,
error_handler_path,
error_handler_args,
retry
) VALUES (
$1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, now(), $14, $15, $16
)
"#,
w_id,
trigger.base.path,
trigger.config.url,
trigger.base.script_path,
trigger.base.is_flow,
trigger.base.mode() as _,
&filters as _,
&initial_messages as _,
trigger
.config
.url_runnable_args
.map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap())) as _,
authed.username,
trigger.config.can_return_message,
trigger.config.can_return_error_result,
authed.email,
trigger.error_handling.error_handler_path,
trigger.error_handling.error_handler_args as _,
trigger.error_handling.retry as _
)
.execute(&mut *tx)
.await?;
Ok(())
}
async fn update_trigger(
&self,
_db: &DB,
tx: &mut PgConnection,
authed: &ApiAuthed,
w_id: &str,
path: &str,
trigger: TriggerData<Self::TriggerConfigRequest>,
) -> Result<()> {
let filters = trigger
.config
.filters
.into_iter()
.map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap()))
.collect_vec();
let initial_messages = trigger
.config
.initial_messages
.unwrap_or_default()
.into_iter()
.map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap()))
.collect_vec();
// important to update server_id to NULL to stop current websocket listener
sqlx::query!(
"
UPDATE
websocket_trigger
SET
url = $1,
script_path = $2,
path = $3,
is_flow = $4,
filters = $5,
initial_messages = $6,
url_runnable_args = $7,
edited_by = $8,
email = $9,
can_return_message = $10,
can_return_error_result = $11,
edited_at = now(),
server_id = NULL,
error = NULL,
error_handler_path = $14,
error_handler_args = $15,
retry = $16
WHERE
workspace_id = $12 AND path = $13
",
trigger.config.url,
trigger.base.script_path,
trigger.base.path,
trigger.base.is_flow,
filters.as_slice() as &[SqlxJson<Box<RawValue>>],
initial_messages.as_slice() as &[SqlxJson<Box<RawValue>>],
trigger
.config
.url_runnable_args
.map(|v| SqlxJson(serde_json::value::to_raw_value(&v).unwrap()))
as Option<SqlxJson<Box<RawValue>>>,
&authed.username,
&authed.email,
trigger.config.can_return_message,
trigger.config.can_return_error_result,
w_id,
path,
trigger.error_handling.error_handler_path,
trigger.error_handling.error_handler_args as _,
trigger.error_handling.retry as _
)
.execute(&mut *tx)
.await?;
Ok(())
}
async fn test_connection(
&self,
db: &DB,
authed: &ApiAuthed,
_user_db: &UserDB,
workspace_id: &str,
config: Self::TestConnectionConfig,
) -> Result<()> {
let url = &config.url;
let connect_url: Cow<str> = if url.starts_with("$") {
if url.starts_with("$flow:") || url.starts_with("$script:") {
let path = url.splitn(2, ':').nth(1).unwrap();
Cow::Owned(
get_url_from_runnable_value(
path,
url.starts_with("$flow:"),
&db,
authed.clone(),
config.url_runnable_args.as_ref().map(to_raw_value).as_ref(),
&workspace_id,
)
.await?,
)
} else {
return Err(Error::BadConfig(format!(
"Invalid WebSocket runnable path: {}",
url
)));
}
} else {
Cow::Borrowed(&url)
};
connect_async(&*connect_url).await.map_err(|err| {
Error::BadConfig(format!(
"Error connecting to WebSocket: {}",
err.to_string()
))
})?;
Ok(())
}
}
backend/windmill-api/src/triggers/websocket/listener.rs
-477
View File
@@ -1,477 +0,0 @@
use super::WebsocketTrigger;
use crate::triggers::{
filter::{is_value_superset, Filter, JsonFilter},
listener::ListeningTrigger,
trigger_helpers::{
trigger_runnable, trigger_runnable_and_wait_for_raw_result,
trigger_runnable_and_wait_for_raw_result_with_error_ctx, TriggerJobArgs,
},
websocket::{get_url_from_runnable_value, WebsocketConfig},
Listener,
};
use anyhow::Context;
use async_trait::async_trait;
use futures::{stream::SplitSink, SinkExt, StreamExt};
use http::Response;
use itertools::Itertools;
use serde::Deserialize;
use serde_json::value::RawValue;
use std::{borrow::Cow, collections::HashMap, sync::Arc};
use tokio::{net::TcpStream, sync::RwLock};
use tokio_tungstenite::{connect_async, tungstenite::Message, MaybeTlsStream, WebSocketStream};
use windmill_common::{
error::{to_anyhow, Error, Result},
jobs::JobTriggerKind,
triggers::TriggerMetadata,
utils::report_critical_error,
worker::to_raw_value,
DB,
};
use windmill_queue::PushArgsOwned;
impl ListeningTrigger<WebsocketConfig> {
async fn send_initial_messages(
&self,
writer: &mut SplitSink<WebSocketStream<MaybeTlsStream<TcpStream>>, Message>,
db: &DB,
) -> Result<()> {
let initial_messages: Vec<InitialMessage> = self
.trigger_config
.initial_messages
.as_deref()
.unwrap_or_default()
.iter()
.filter_map(|m| serde_json::from_str(m.get()).ok())
.collect_vec();
let WebsocketConfig { ref url, .. } = self.trigger_config;
let runnable_kind = if self.is_flow { "flow" } else { "script" };
let mut authed_o = None;
for start_message in initial_messages {
match start_message {
InitialMessage::RawMessage(msg) => {
let msg = if msg.starts_with("\"") && msg.ends_with("\"") {
msg[1..msg.len() - 1].to_string()
} else {
msg
};
tracing::info!(
"Sending raw message initial message to WebSocket {}: {}",
url,
msg
);
writer
.send(tokio_tungstenite::tungstenite::Message::Text(msg))
.await
.map_err(to_anyhow)
.with_context(|| "failed to send raw message")?;
}
InitialMessage::RunnableResult { path, is_flow, args } => {
tracing::info!(
"Running {} {} for initial message to WebSocket {}",
runnable_kind,
path,
url,
);
let args = raw_value_to_args_hashmap(Some(&args))?;
if authed_o.is_none() {
authed_o = Some(self.authed(db, "ws").await?);
}
let authed = authed_o.clone().unwrap();
let result = trigger_runnable_and_wait_for_raw_result_with_error_ctx(
db,
None,
authed.clone(),
&self.workspace_id,
&path,
is_flow,
PushArgsOwned { args, extra: None },
None,
None,
None,
"".to_string(), // doesn't matter as no retry/error handler
TriggerMetadata::new(Some(self.path.to_owned()), JobTriggerKind::Websocket),
)
.await
.map(|r| r.get().to_owned())?;
tracing::info!(
"Sending {} {} result to WebSocket {}",
runnable_kind,
path,
url
);
// if the `result` was just a single string, the below removes the surrounding quotes by parsing it as a string.
// it falls back to the original serialized JSON if it doesn't work.
let result = serde_json::from_str::<String>(result.as_str()).unwrap_or(result);
writer
.send(tokio_tungstenite::tungstenite::Message::Text(result))
.await
.map_err(to_anyhow)
.with_context(|| {
format!("Failed to send {} {} result", runnable_kind, path)
})?;
}
}
}
Ok(())
}
}
#[async_trait]
impl Listener for WebsocketTrigger {
type Consumer = (
WebSocketStream<MaybeTlsStream<TcpStream>>,
Response<Option<Vec<u8>>>,
);
type Extra = ReturnMessageChannels;
type ExtraState = ();
const JOB_TRIGGER_KIND: JobTriggerKind = JobTriggerKind::Websocket;
async fn get_consumer(
&self,
db: &DB,
listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
err_message: Arc<RwLock<Option<String>>>,
mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
) -> Result<Option<Self::Consumer>> {
let url = &listening_trigger.trigger_config.url;
let connect_url: Cow<str> = if url.starts_with("$") {
if url.starts_with("$flow:") || url.starts_with("$script:") {
tokio::select! {
biased;
_ = killpill_rx.recv() => {
return Ok(None);
},
_ = self.loop_ping(&db, listening_trigger, err_message.clone(), Some(
"Waiting on runnable to return WebSocket URL...".to_string()
)) => {
return Ok(None);
},
url_result = {
let authed = listening_trigger.authed(db, "ws").await?;
let args = listening_trigger.trigger_config.url_runnable_args.as_ref().map(|r| &r.0);
let path = url.splitn(2, ':').nth(1).unwrap();
get_url_from_runnable_value(path, url.starts_with("$flow:"), db, authed, args, &listening_trigger.workspace_id)
} => match url_result {
Ok(url) => Cow::Owned(url),
Err(err) => {
return Err(anyhow::anyhow!("Error getting WebSocket URL from runnable after 5 tries: {:?}", err).into());
}
},
}
} else {
return Err(anyhow::anyhow!("Invalid WebSocket runnable path: {}", url).into());
}
} else {
Cow::Borrowed(&url)
};
let connection = connect_async(&*connect_url)
.await
.map(|conn| Some(conn))
.map_err(|err| to_anyhow(err).into());
connection
}
async fn consume(
&self,
db: &DB,
consumer: Self::Consumer,
listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
err_message: Arc<RwLock<Option<String>>>,
mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
_extra_state: Option<&Self::ExtraState>,
) {
let WebsocketConfig { ref url, .. } = listening_trigger.trigger_config;
tracing::info!("Connected to WebSocket {}", url);
let (ws_stream, _) = consumer;
let (mut writer, mut reader) = ws_stream.split();
// send initial messages
if listening_trigger.trigger_mode {
tokio::select! {
biased;
_ = killpill_rx.recv() => {
return;
},
_ = self.loop_ping(db, listening_trigger, err_message.clone(), Some("Sending initial messages...".to_string())) => {
return;
},
result = listening_trigger.send_initial_messages(&mut writer, &db) => {
if let Err(err) = result {
self.disable_with_error(&db, listening_trigger, format!("Error sending initial messages: {:?}", err)).await;
return
} else {
tracing::debug!("Initial messages sent successfully to WebSocket {}", url);
}
}
}
}
let (return_message_channels, message_sender_handle) = if listening_trigger.trigger_mode
&& listening_trigger.trigger_config.can_return_message
{
let (send_message_tx, mut rx) = tokio::sync::mpsc::channel::<String>(100);
let w_id = listening_trigger.workspace_id.clone();
let url = url.clone();
let db = db.clone();
let handle = tokio::spawn(async move {
while let Some(message) = rx.recv().await {
if let Err(err) = writer
.send(tokio_tungstenite::tungstenite::Message::Text(message))
.await
{
report_critical_error(format!("Could not send runnable result to WebSocket {} because of error: {}", url, err), db.clone(), Some(&w_id), None).await;
}
}
});
let killpill_rx = killpill_rx.resubscribe();
let return_message_channels = ReturnMessageChannels { send_message_tx, killpill_rx };
(Some(return_message_channels), Some(handle))
} else {
(None, None)
};
tokio::select! {
biased;
_ = killpill_rx.recv() => {
},
_ = self.loop_ping(db, listening_trigger, err_message.clone(), None) => {
},
_ = async {
let filters: Vec<Filter> = if listening_trigger.trigger_mode {
listening_trigger
.trigger_config
.filters
.iter()
.filter_map(|m| serde_json::from_str(m.get()).ok())
.collect_vec()
} else {
vec![]
};
loop {
if let Some(msg) = reader.next().await {
match msg {
Ok(msg) => {
match msg {
tokio_tungstenite::tungstenite::Message::Text(text) => {
tracing::debug!("Received text message from WebSocket {}: {}", url, text);
let mut should_handle = true;
for filter in &filters {
match filter {
Filter::JsonFilter(JsonFilter { key, value }) => {
let mut deserializer = serde_json::Deserializer::from_str(text.as_str());
should_handle = match is_value_superset(&mut deserializer, key, &value) {
Ok(filter_match) => {
filter_match
},
Err(err) => {
tracing::warn!("Error deserializing filter for WebSocket {}: {:?}", url, err);
false
}
};
}
}
if !should_handle {
break;
}
}
if should_handle {
let trigger_info = HashMap::from([
("url".to_string(), to_raw_value(&listening_trigger.trigger_config.url)),
]);
let _ = self.handle_event(db, listening_trigger, text, trigger_info, return_message_channels.clone()).await;
}
},
a @ _ => {
tracing::debug!("Received non text-message from WebSocket {}: {:?}", url, a);
}
}
},
Err(err) => {
tracing::error!("Error reading from WebSocket {}: {:?}", url, err);
}
}
} else {
tracing::error!("WebSocket {} closed", url);
self.update_ping_and_loop_ping_status(db, listening_trigger, err_message.clone(), Some("WebSocket closed".to_string())).await;
break;
}
}
} => {}
}
// make sure to stop return message handler
if let Some(message_sender_handle) = message_sender_handle {
message_sender_handle.abort();
}
}
async fn handle_trigger(
&self,
db: &DB,
listening_trigger: &ListeningTrigger<Self::TriggerConfig>,
payload: Self::Payload,
trigger_info: HashMap<String, Box<RawValue>>,
extra: Option<Self::Extra>,
) -> Result<()> {
let ListeningTrigger {
path,
is_flow,
workspace_id,
trigger_config,
script_path,
error_handling,
suspended_mode,
..
} = listening_trigger;
let WebsocketConfig { url, .. } = trigger_config;
let args = WebsocketTrigger::build_job_args(
&script_path,
*is_flow,
workspace_id,
db,
payload,
trigger_info,
)
.await;
let args = match args {
Ok(args) => args,
Err(err) => {
return Err(err);
}
};
let authed = listening_trigger.authed(db, "ws").await?;
let (retry, error_handler_path, error_handler_args) = match error_handling.as_ref() {
Some(error_handling) => (
error_handling.retry.as_ref(),
error_handling.error_handler_path.as_deref(),
error_handling.error_handler_args.as_ref(),
),
None => (None, None, None),
};
let trigger = TriggerMetadata::new(Some(path.to_owned()), Self::JOB_TRIGGER_KIND);
if *suspended_mode || extra.is_none() {
trigger_runnable(
db,
None,
authed,
&workspace_id,
&script_path,
*is_flow,
args,
retry,
error_handler_path,
error_handler_args,
format!("websocket_trigger/{}", listening_trigger.path),
None,
*suspended_mode,
trigger,
)
.await?;
} else if let Some(ReturnMessageChannels { send_message_tx, mut killpill_rx }) = extra {
let db_ = db.clone();
let url = url.to_owned();
let script_path = script_path.to_owned();
let is_flow = *is_flow;
let w_id = workspace_id.to_owned();
let retry = retry.cloned();
let error_handler_path = error_handler_path.map(|s| s.to_string());
let error_handler_args = error_handler_args.cloned();
let trigger_path = path.clone();
let can_return_error_result = trigger_config.can_return_error_result;
let handle_response_f = async move {
tokio::select! {
_ = killpill_rx.recv() => {
return;
},
result = trigger_runnable_and_wait_for_raw_result(
&db_,
None,
authed,
&w_id,
&script_path,
is_flow,
args,
retry.as_ref(),
error_handler_path.as_deref(),
error_handler_args.as_ref(),
format!("websocket_trigger/{}", trigger_path),
trigger,
) => {
if let Ok((result, success)) = result {
if !success && !can_return_error_result {
return;
}
let result = result.get().to_owned();
// only send the result if it's not null
if result != "null" {
tracing::info!("Sending job result to WebSocket {}", url);
// if the `result` was just a single string, the below removes the surrounding quotes by parsing it as a string.
// it falls back to the original serialized JSON if it doesn't work.
let result = serde_json::from_str::<String>(result.as_str()).unwrap_or(result);
if let Err(err) = send_message_tx.send(result).await {
report_critical_error(format!("Could not send runnable result to WebSocket {} because of error: {}", url, err), db_.clone(), Some(&w_id), None).await;
}
}
}
}
};
};
tokio::spawn(handle_response_f);
}
Ok(())
}
}
pub struct ReturnMessageChannels {
send_message_tx: tokio::sync::mpsc::Sender<String>,
killpill_rx: tokio::sync::broadcast::Receiver<()>,
}
impl Clone for ReturnMessageChannels {
fn clone(&self) -> Self {
Self {
send_message_tx: self.send_message_tx.clone(),
killpill_rx: self.killpill_rx.resubscribe(),
}
}
}
#[derive(Debug, Deserialize)]
enum InitialMessage {
#[serde(rename = "raw_message")]
RawMessage(String),
#[serde(rename = "runnable_result")]
RunnableResult { path: String, args: Box<RawValue>, is_flow: bool },
}
fn raw_value_to_args_hashmap(
args: Option<&Box<RawValue>>,
) -> Result<HashMap<String, Box<RawValue>>> {
let args = if let Some(args) = args {
serde_json::from_str::<Option<HashMap<String, Box<RawValue>>>>(args.get())
.map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))?
.unwrap_or_else(HashMap::new)
} else {
HashMap::new()
};
Ok(args)
}
backend/windmill-api/src/triggers/websocket/mod.rs
-128
View File
@@ -1,128 +0,0 @@
use std::collections::HashMap;
use crate::{
db::ApiAuthed,
triggers::trigger_helpers::{
trigger_runnable_and_wait_for_raw_result_with_error_ctx, TriggerJobArgs,
},
};
use serde::{Deserialize, Serialize};
use serde_json::value::RawValue;
use sqlx::{types::Json as SqlxJson, FromRow};
use windmill_common::{
error::{Error, Result},
jobs::JobTriggerKind,
triggers::{TriggerMetadata, TriggerKind},
worker::to_raw_value,
DB,
};
use windmill_queue::PushArgsOwned;
mod handler;
mod listener;
#[derive(Copy, Clone)]
pub struct WebsocketTrigger;
impl TriggerJobArgs for WebsocketTrigger {
type Payload = String;
const TRIGGER_KIND: TriggerKind = TriggerKind::Websocket;
fn v1_payload_fn(payload: &Self::Payload) -> HashMap<String, Box<RawValue>> {
HashMap::from([("msg".to_string(), to_raw_value(&payload))])
}
}
#[derive(Debug, Clone, FromRow, Serialize, Deserialize)]
pub struct WebsocketConfig {
pub url: String,
#[serde(default)]
pub filters: Vec<SqlxJson<Box<RawValue>>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub initial_messages: Option<Vec<SqlxJson<Box<RawValue>>>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub url_runnable_args: Option<SqlxJson<Box<RawValue>>>,
#[serde(default)]
pub can_return_message: bool,
#[serde(default)]
pub can_return_error_result: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WebsocketConfigRequest {
url: String,
filters: Vec<serde_json::Value>,
initial_messages: Option<Vec<serde_json::Value>>,
url_runnable_args: Option<serde_json::Value>,
can_return_message: bool,
can_return_error_result: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TestWebsocketConfig {
url: String,
url_runnable_args: Option<serde_json::Value>,
}
pub fn value_to_args_hashmap(
args: Option<&Box<RawValue>>,
) -> Result<HashMap<String, Box<RawValue>>> {
let args = if let Some(args) = args {
let args_map: Option<HashMap<String, serde_json::Value>> = serde_json::from_str(args.get())
.map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))?;
args_map
.unwrap_or_else(HashMap::new)
.into_iter()
.map(|(k, v)| {
let raw_value = serde_json::value::to_raw_value(&v).map_err(|e| {
Error::BadRequest(format!("failed to convert to raw value: {}", e))
})?;
Ok((k, raw_value))
})
.collect::<Result<HashMap<String, Box<RawValue>>>>()
} else {
Ok(HashMap::new())
}?;
Ok(args)
}
pub async fn get_url_from_runnable_value(
path: &str,
is_flow: bool,
db: &DB,
authed: ApiAuthed,
args: Option<&Box<RawValue>>,
workspace_id: &str,
) -> Result<String> {
tracing::info!(
"Running {} {} to get WebSocket URL",
if is_flow { "flow" } else { "script" },
path
);
let args = value_to_args_hashmap(args)?;
let result = trigger_runnable_and_wait_for_raw_result_with_error_ctx(
db,
None,
authed,
workspace_id,
path,
is_flow,
PushArgsOwned { args, extra: None },
None,
None,
None,
"".to_string(), // doesn't matter as no retry/error handler
TriggerMetadata::new(Some(path.to_owned()), JobTriggerKind::Websocket),
)
.await?;
serde_json::from_str::<String>(result.get()).map_err(|_| {
Error::BadConfig(format!(
"{} {} did not return a string",
if is_flow { "Flow" } else { "Script" },
path,
))
})
}